@opengeni/core 0.11.2 → 0.12.0

package/src/sandbox/routing.ts

@@ -2,11 +2,10 @@
 // real DB pointer + the live NATS control plane for the API-DIRECT Channel-A path
 // (M7). Symmetric with apps/worker/src/sandbox-routing.ts (the turn path).
 //
-// A Channel-A op resumes the group box by id and runs ONE op against it. With
-// hot-swap, the op must land on the session's CURRENTLY-active sandbox, not
-// always the group box: if the session swapped to a selfhosted machine, an
-// fs.read / git.status / exec from the API must reach THAT machine. So the
-// established group session is wrapped in a `RoutingSandboxSession` that re-reads
+// A Channel-A op runs against one established home: either a lease-owned provider
+// box or a Connected Machine home that intentionally has no cloud lease. With
+// hot-swap, the op must land on the session's CURRENTLY-active sandbox. The
+// established session is wrapped in a `RoutingSandboxSession` that re-reads
 // (active_sandbox_id, active_epoch) and dispatches to the active backend.
 //
 // The DB-coupled glue (readActiveSandbox / getSandbox / the selfhosted ControlRpc
@@ -16,10 +15,12 @@ import type { Settings } from "@opengeni/config";
 import {
   advanceWorkspaceGenerationForDirectRequest,
   advanceWorkspaceGenerationForRetainedProcess,
+  getRetainedProcess,
   getSandbox,
   markWarmLeaseInstanceLost,
   readActiveSandbox,
   retainWorkspaceMutationProcess,
+  retainedProcessSettlementIdentity,
   settleRetainedProcess,
   verifyDirectWorkspaceMutationSettlement,
   verifyRetainedProcessMutationSettlement,
@@ -55,7 +56,12 @@ export type ChannelARoutingServices = {
 export function relayConfigFromSettings(settings: Settings): SelfhostedRelayConfig {
   const raw = settings.selfhostedRelayUrl?.trim();
   if (!raw) {
-    return { host: "relay.opengeni.local", port: 443, tls: true, path: "/stream" };
+    return {
+      host: "relay.opengeni.local",
+      port: 443,
+      tls: true,
+      path: "/stream",
+    };
   }
   try {
     const url = new URL(raw.includes("://") ? raw : `wss://${raw}`);
@@ -105,11 +111,10 @@ export function routingEnabled(settings: Settings): boolean {
 }
 
 /**
- * Wrap an established group-box session in a `RoutingSandboxSession` so a
- * Channel-A op routes to the session's currently-active sandbox. Returns the
- * established handle with its `session` replaced by the stable proxy. With the
- * default pointer (active_sandbox_id == null) this routes to the group box
- * unchanged; a selfhosted active pointer routes the op to the machine.
+ * Wrap an established home session in a `RoutingSandboxSession` so a Channel-A
+ * op routes to the session's currently-active sandbox. Provider homes supply a
+ * lease; machine homes supply a pinned selfhosted identity and no lease. Returns
+ * the established handle with its `session` replaced by the stable proxy.
  */
 export function wrapChannelABoxWithRouting(
   services: ChannelARoutingServices,
@@ -117,12 +122,19 @@ export function wrapChannelABoxWithRouting(
     accountId: string;
     workspaceId: string;
     sessionId: string;
-    homeLease: {
+    homeLease?: {
       sandboxGroupId: string;
       leaseEpoch: number;
       instanceId: string;
       backend: string;
     };
+    /** A machine-home Channel-A request has no cloud/home lease. Its already
+     * established SelfhostedSession is pinned to the durable active pointer so
+     * the first command uses the exact machine instance and epoch. */
+    pinnedSelfhosted?: {
+      sandboxId: string;
+      epoch: number;
+    };
     directRequest: {
       requestId: string;
       holderId: string;
@@ -131,180 +143,212 @@ export function wrapChannelABoxWithRouting(
   established: EstablishedSandboxSession,
 ): EstablishedSandboxSession {
   const { db, settings, bus } = services;
-  const beforeMutation = async ({
-    op,
-    backend,
-  }: {
-    op: string;
-    backend: ResolvedActiveBackend;
-  }): Promise<SandboxWorkspaceMutationAdmission | null> => {
-    // Connected Machines and other non-persistable targets intentionally do
-    // not dirty or advance the cloud-home archive generation.
-    if (
-      backend.sandboxId !== null ||
-      backend.leaseEpoch === undefined ||
-      backend.providerInstanceId === undefined
-    ) {
-      return null;
-    }
-    if (backend.activeEpoch === undefined) {
-      throw new Error("API-direct workspace mutation resolved without an active route epoch");
-    }
-    return await advanceWorkspaceGenerationForDirectRequest(db, {
-      accountId: ids.accountId,
-      workspaceId: ids.workspaceId,
-      sessionId: ids.sessionId,
-      requestId: ids.directRequest.requestId,
-      holderId: ids.directRequest.holderId,
-      sandboxGroupId: ids.homeLease.sandboxGroupId,
-      expectedEpoch: backend.leaseEpoch,
-      expectedInstanceId: backend.providerInstanceId,
-      routeTargetId: backend.sandboxId,
-      routeEpoch: backend.activeEpoch,
-      operation: op,
-    });
-  };
-  const afterMutation = async ({
-    op,
-    backend,
-    admission,
-    outcome,
-    retainedProcess,
-  }: {
-    op: string;
-    backend: ResolvedActiveBackend;
-    admission: unknown;
-    outcome: "resolved" | "rejected";
-    result?: unknown;
-    retainedProcess?: RoutingRetainedProcess;
-  }): Promise<void> => {
-    if (admission === null) return;
-    if (
-      !admission ||
-      typeof admission !== "object" ||
-      typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).id !== "string" ||
-      typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).workspaceGeneration !==
-        "number" ||
-      backend.leaseEpoch === undefined ||
-      backend.providerInstanceId === undefined ||
-      backend.activeEpoch === undefined
-    ) {
-      throw new Error("API-direct workspace mutation settlement lacked its exact admission");
-    }
-    const exactAdmission = admission as SandboxWorkspaceMutationAdmission;
-    if (outcome === "resolved" && retainedProcess) {
-      await retainWorkspaceMutationProcess(db, {
-        accountId: ids.accountId,
-        workspaceId: ids.workspaceId,
-        sessionId: ids.sessionId,
-        processId: retainedProcess.id,
-        providerSessionId: retainedProcess.providerSessionId,
-        admissionId: exactAdmission.id,
-        admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
-        operation: op,
-        owner: {
-          kind: "direct",
+  const homeLease = ids.homeLease;
+  const beforeMutation = homeLease
+    ? async ({
+        op,
+        backend,
+      }: {
+        op: string;
+        backend: ResolvedActiveBackend;
+      }): Promise<SandboxWorkspaceMutationAdmission | null> => {
+        // Connected Machines and other non-persistable targets intentionally do
+        // not dirty or advance the cloud-home archive generation.
+        if (
+          backend.sandboxId !== null ||
+          backend.leaseEpoch === undefined ||
+          backend.providerInstanceId === undefined
+        ) {
+          return null;
+        }
+        if (backend.activeEpoch === undefined) {
+          throw new Error("API-direct workspace mutation resolved without an active route epoch");
+        }
+        return await advanceWorkspaceGenerationForDirectRequest(db, {
+          accountId: ids.accountId,
+          workspaceId: ids.workspaceId,
+          sessionId: ids.sessionId,
+          requestId: ids.directRequest.requestId,
+          holderId: ids.directRequest.holderId,
+          sandboxGroupId: homeLease.sandboxGroupId,
+          expectedEpoch: backend.leaseEpoch,
+          expectedInstanceId: backend.providerInstanceId,
+          routeTargetId: backend.sandboxId,
+          routeEpoch: backend.activeEpoch,
+          operation: op,
+        });
+      }
+    : undefined;
+  const afterMutation = homeLease
+    ? async ({
+        op,
+        backend,
+        admission,
+        outcome,
+        retainedProcess,
+      }: {
+        op: string;
+        backend: ResolvedActiveBackend;
+        admission: unknown;
+        outcome: "resolved" | "rejected";
+        result?: unknown;
+        retainedProcess?: RoutingRetainedProcess;
+      }): Promise<void> => {
+        if (admission === null) return;
+        if (
+          !admission ||
+          typeof admission !== "object" ||
+          typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).id !== "string" ||
+          typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).workspaceGeneration !==
+            "number" ||
+          backend.leaseEpoch === undefined ||
+          backend.providerInstanceId === undefined ||
+          backend.activeEpoch === undefined
+        ) {
+          throw new Error("API-direct workspace mutation settlement lacked its exact admission");
+        }
+        const exactAdmission = admission as SandboxWorkspaceMutationAdmission;
+        if (outcome === "resolved" && retainedProcess) {
+          await retainWorkspaceMutationProcess(db, {
+            accountId: ids.accountId,
+            workspaceId: ids.workspaceId,
+            sessionId: ids.sessionId,
+            processId: retainedProcess.id,
+            providerSessionId: retainedProcess.providerSessionId,
+            admissionId: exactAdmission.id,
+            admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
+            operation: op,
+            owner: {
+              kind: "direct",
+              requestId: ids.directRequest.requestId,
+              holderId: ids.directRequest.holderId,
+              sandboxGroupId: homeLease.sandboxGroupId,
+              expectedEpoch: backend.leaseEpoch,
+              expectedInstanceId: backend.providerInstanceId,
+              routeTargetId: exactAdmission.routeTargetId,
+              routeEpoch: exactAdmission.routeEpoch,
+            },
+          });
+          return;
+        }
+        await verifyDirectWorkspaceMutationSettlement(db, {
+          accountId: ids.accountId,
+          workspaceId: ids.workspaceId,
+          sessionId: ids.sessionId,
           requestId: ids.directRequest.requestId,
           holderId: ids.directRequest.holderId,
-          sandboxGroupId: ids.homeLease.sandboxGroupId,
+          sandboxGroupId: homeLease.sandboxGroupId,
           expectedEpoch: backend.leaseEpoch,
           expectedInstanceId: backend.providerInstanceId,
           routeTargetId: exactAdmission.routeTargetId,
           routeEpoch: exactAdmission.routeEpoch,
-        },
-      });
-      return;
-    }
-    await verifyDirectWorkspaceMutationSettlement(db, {
-      accountId: ids.accountId,
-      workspaceId: ids.workspaceId,
-      sessionId: ids.sessionId,
-      requestId: ids.directRequest.requestId,
-      holderId: ids.directRequest.holderId,
-      sandboxGroupId: ids.homeLease.sandboxGroupId,
-      expectedEpoch: backend.leaseEpoch,
-      expectedInstanceId: backend.providerInstanceId,
-      routeTargetId: exactAdmission.routeTargetId,
-      routeEpoch: exactAdmission.routeEpoch,
-      admission: exactAdmission,
-      operation: op,
-      outcome,
-    });
-  };
-  const beforeProcessMutation = async ({
-    op,
-    process,
-  }: {
-    op: string;
-    backend: ResolvedActiveBackend;
-    process: RoutingRetainedProcess;
-  }): Promise<SandboxWorkspaceMutationAdmission> =>
-    await advanceWorkspaceGenerationForRetainedProcess(db, {
-      accountId: ids.accountId,
-      workspaceId: ids.workspaceId,
-      sessionId: ids.sessionId,
-      processId: process.id,
-      operation: op,
-    });
-  const afterProcessMutation = async ({
-    op,
-    process,
-    admission,
-    outcome,
-  }: {
-    op: string;
-    backend: ResolvedActiveBackend;
-    process: RoutingRetainedProcess;
-    admission: unknown;
-    outcome: "resolved" | "rejected";
-    result?: unknown;
-  }): Promise<void> => {
-    if (
-      !admission ||
-      typeof admission !== "object" ||
-      typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).id !== "string" ||
-      typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).workspaceGeneration !==
-        "number"
-    ) {
-      throw new Error("API retained-process mutation settlement lacked its exact admission");
-    }
-    await verifyRetainedProcessMutationSettlement(db, {
-      accountId: ids.accountId,
-      workspaceId: ids.workspaceId,
-      sessionId: ids.sessionId,
-      processId: process.id,
-      admission: admission as SandboxWorkspaceMutationAdmission,
-      operation: op,
-      outcome,
-    });
-  };
-  const settleProcess = async ({
-    backend,
-    process,
-    proof,
-  }: {
-    backend: ResolvedActiveBackend;
-    process: RoutingRetainedProcess;
-    proof: RoutingRetainedProcessTerminalProof;
-  }): Promise<void> => {
-    if (
-      backend.sandboxId !== null ||
-      backend.leaseEpoch === undefined ||
-      backend.providerInstanceId === undefined
-    ) {
-      return;
-    }
-    await settleRetainedProcess(db, {
-      accountId: ids.accountId,
-      workspaceId: ids.workspaceId,
-      sessionId: ids.sessionId,
-      processId: process.id,
-      outcome: proof.outcome,
-      exitCode: proof.exitCode,
-      reason: proof.reason,
-      idleGraceMs: settings.sandboxIdleGraceMs,
-    });
-  };
+          admission: exactAdmission,
+          operation: op,
+          outcome,
+        });
+      }
+    : undefined;
+  const beforeProcessMutation = homeLease
+    ? async ({
+        op,
+        process,
+      }: {
+        op: string;
+        backend: ResolvedActiveBackend;
+        process: RoutingRetainedProcess;
+      }): Promise<SandboxWorkspaceMutationAdmission> =>
+        await advanceWorkspaceGenerationForRetainedProcess(db, {
+          accountId: ids.accountId,
+          workspaceId: ids.workspaceId,
+          sessionId: ids.sessionId,
+          processId: process.id,
+          operation: op,
+        })
+    : undefined;
+  const afterProcessMutation = homeLease
+    ? async ({
+        op,
+        process,
+        admission,
+        outcome,
+      }: {
+        op: string;
+        backend: ResolvedActiveBackend;
+        process: RoutingRetainedProcess;
+        admission: unknown;
+        outcome: "resolved" | "rejected";
+        result?: unknown;
+      }): Promise<void> => {
+        if (
+          !admission ||
+          typeof admission !== "object" ||
+          typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).id !== "string" ||
+          typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).workspaceGeneration !==
+            "number"
+        ) {
+          throw new Error("API retained-process mutation settlement lacked its exact admission");
+        }
+        await verifyRetainedProcessMutationSettlement(db, {
+          accountId: ids.accountId,
+          workspaceId: ids.workspaceId,
+          sessionId: ids.sessionId,
+          processId: process.id,
+          admission: admission as SandboxWorkspaceMutationAdmission,
+          operation: op,
+          outcome,
+        });
+      }
+    : undefined;
+  const settleProcess = homeLease
+    ? async ({
+        backend,
+        process,
+        proof,
+      }: {
+        backend: ResolvedActiveBackend;
+        process: RoutingRetainedProcess;
+        proof: RoutingRetainedProcessTerminalProof;
+      }): Promise<void> => {
+        if (
+          backend.sandboxId !== null ||
+          backend.leaseEpoch === undefined ||
+          backend.providerInstanceId === undefined ||
+          backend.activeEpoch === undefined
+        ) {
+          return;
+        }
+        const durable = await getRetainedProcess(db, {
+          workspaceId: ids.workspaceId,
+          sessionId: ids.sessionId,
+          processId: process.id,
+        });
+        if (
+          !durable ||
+          durable.providerSessionId !== process.providerSessionId ||
+          durable.providerBackend !== backend.kind ||
+          durable.providerInstanceId !== backend.providerInstanceId ||
+          durable.leaseEpoch !== backend.leaseEpoch ||
+          durable.routeKind !== (backend.sandboxId === null ? "home" : "active") ||
+          durable.routeTargetId !== backend.sandboxId ||
+          durable.routeEpoch !== backend.activeEpoch
+        ) {
+          throw new Error(
+            "API retained-process settlement lost its exact durable backend identity",
+          );
+        }
+        await settleRetainedProcess(db, {
+          accountId: ids.accountId,
+          workspaceId: ids.workspaceId,
+          sessionId: ids.sessionId,
+          processId: process.id,
+          expected: retainedProcessSettlementIdentity(durable),
+          outcome: proof.outcome,
+          exitCode: proof.exitCode,
+          reason: proof.reason,
+          idleGraceMs: settings.sandboxIdleGraceMs,
+        });
+      }
+    : undefined;
   const resolver = makeActiveBackendResolver({
     workspaceId: ids.workspaceId,
     defaultBackend: established.session as RoutableBackendSession,
@@ -322,13 +366,28 @@ export function wrapChannelABoxWithRouting(
     },
     controlRpcFactory: controlRpcFactory(bus),
     relay: relayConfigFromSettings(settings),
-    resolveDefaultBackend: async () => ({
-      session: established.session as RoutableBackendSession,
-      sandboxId: null,
-      kind: established.backendId,
-      leaseEpoch: ids.homeLease.leaseEpoch,
-      providerInstanceId: ids.homeLease.instanceId,
-    }),
+    selfhostedTimeoutMs: settings.sandboxSelfhostedControlTimeoutMs,
+    selfhostedExecTimeoutMs: settings.sandboxSelfhostedExecTimeoutMs,
+    ...(ids.pinnedSelfhosted
+      ? {
+          pinnedSelfhosted: {
+            sandboxId: ids.pinnedSelfhosted.sandboxId,
+            epoch: ids.pinnedSelfhosted.epoch,
+            session: established.session as RoutableBackendSession,
+          },
+        }
+      : {}),
+    ...(homeLease
+      ? {
+          resolveDefaultBackend: async () => ({
+            session: established.session as RoutableBackendSession,
+            sandboxId: null,
+            kind: established.backendId,
+            leaseEpoch: homeLease.leaseEpoch,
+            providerInstanceId: homeLease.instanceId,
+          }),
+        }
+      : {}),
   });
 
   const proxy = new RoutingSandboxSession({
@@ -336,8 +395,12 @@ export function wrapChannelABoxWithRouting(
       session: established.session as RoutableBackendSession,
       sandboxId: null,
       kind: established.backendId,
-      leaseEpoch: ids.homeLease.leaseEpoch,
-      providerInstanceId: ids.homeLease.instanceId,
+      ...(homeLease
+        ? {
+            leaseEpoch: homeLease.leaseEpoch,
+            providerInstanceId: homeLease.instanceId,
+          }
+        : {}),
     },
     readPointer: async () => {
       if (!routingEnabled(settings)) {
@@ -347,43 +410,47 @@ export function wrapChannelABoxWithRouting(
       return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
     },
     resolveActiveBackend: resolver,
-    beforeMutation,
-    afterMutation,
-    beforeProcessMutation,
-    afterProcessMutation,
-    settleProcess,
-    onDefaultBackendError: async ({ error }) => {
-      if (!isProviderSandboxGoneDuringRoutedOperation(ids.homeLease.backend, error)) return null;
-      const marked = await markWarmLeaseInstanceLost(db, {
-        accountId: ids.accountId,
-        workspaceId: ids.workspaceId,
-        sandboxGroupId: ids.homeLease.sandboxGroupId,
-        expectedEpoch: ids.homeLease.leaseEpoch,
-        expectedInstanceId: ids.homeLease.instanceId,
-        diagnostic: "provider_not_found_during_routed_operation",
-      });
-      if (marked.status === "marked" && bus) {
-        await appendAndPublishEvents(db, bus, ids.workspaceId, ids.sessionId, [
-          {
-            type: "sandbox.box.lost",
-            payload: { sandboxId: ids.homeLease.instanceId },
+    ...(beforeMutation ? { beforeMutation } : {}),
+    ...(afterMutation ? { afterMutation } : {}),
+    ...(beforeProcessMutation ? { beforeProcessMutation } : {}),
+    ...(afterProcessMutation ? { afterProcessMutation } : {}),
+    ...(settleProcess ? { settleProcess } : {}),
+    ...(homeLease
+      ? {
+          onDefaultBackendError: async ({ error }: { error: unknown }) => {
+            if (!isProviderSandboxGoneDuringRoutedOperation(homeLease.backend, error)) return null;
+            const marked = await markWarmLeaseInstanceLost(db, {
+              accountId: ids.accountId,
+              workspaceId: ids.workspaceId,
+              sandboxGroupId: homeLease.sandboxGroupId,
+              expectedEpoch: homeLease.leaseEpoch,
+              expectedInstanceId: homeLease.instanceId,
+              diagnostic: "provider_not_found_during_routed_operation",
+            });
+            if (marked.status === "marked" && bus) {
+              await appendAndPublishEvents(db, bus, ids.workspaceId, ids.sessionId, [
+                {
+                  type: "sandbox.box.lost",
+                  payload: { sandboxId: homeLease.instanceId },
+                },
+              ]).catch(() => undefined);
+            }
+            const lease = marked.lease;
+            const restore = lease?.recovery.restore.status;
+            return {
+              leaseEpoch: lease?.leaseEpoch ?? homeLease.leaseEpoch,
+              recovery:
+                marked.status === "stale"
+                  ? ("superseded" as const)
+                  : restore === "pending"
+                    ? ("pending" as const)
+                    : restore === "degraded"
+                      ? ("degraded" as const)
+                      : ("unrecoverable" as const),
+            };
           },
-        ]).catch(() => undefined);
-      }
-      const lease = marked.lease;
-      const restore = lease?.recovery.restore.status;
-      return {
-        leaseEpoch: lease?.leaseEpoch ?? ids.homeLease.leaseEpoch,
-        recovery:
-          marked.status === "stale"
-            ? ("superseded" as const)
-            : restore === "pending"
-              ? ("pending" as const)
-              : restore === "degraded"
-                ? ("degraded" as const)
-                : ("unrecoverable" as const),
-      };
-    },
+        }
+      : {}),
   });
 
   return { ...established, session: proxy };