@opengeni/core 3.0.1 → 3.1.2

package/src/sandbox/routing.ts

@@ -15,23 +15,27 @@ import { sandboxLifecycleTransitionWaitMs, type Settings } from "@opengeni/confi
 import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
 import {
   createProviderCommandRetainer,
-  retainedProviderCommandPersistence,
+  supervisedCommandProtocolReady,
 } from "@opengeni/db/retained-provider-commands";
 import {
   advanceWorkspaceGenerationForDirectRequest,
+  retainedProviderCommandPersistence,
   advanceWorkspaceGenerationForRetainedProcess,
   getLiveEnrollmentConnection,
   getRetainedProcess,
   getSandbox,
   markWarmLeaseInstanceLost,
   readActiveSandbox,
+  resolvePersonalMachineConnectionForAttempt,
   retainWorkspaceMutationProcess,
   SandboxRetainedProcessPromotionFencedError,
+  SandboxWorkspaceMutationOutputRejectedError,
   retainedProcessSettlementIdentity,
   settleRetainedProcess,
   verifyDirectWorkspaceMutationSettlement,
   verifyRetainedProcessMutationSettlement,
   type Database,
+  type EnrollmentRecord,
   type SandboxRetainedProcess,
   type SandboxWorkspaceMutationAdmission,
 } from "@opengeni/db";
@@ -42,6 +46,7 @@ import {
   NatsControlRpc,
   NatsOpStreamTransport,
   RoutingSandboxSession,
+  RoutingMutationOutputRejectedError,
   resolveModalCheckpointProviderBindingForSession,
   type ControlRpc,
   type EstablishedSandboxSession,
@@ -52,6 +57,7 @@ import {
   type RoutingRetainedProcess,
   type RoutingRetainedProcessTerminalProof,
   type RoutingSandboxOperationObserver,
+  type RoutingSandboxCaptureWaitObserver,
   type SelfhostedRelayConfig,
   type SelfhostedConnectionBinding,
   type SelfhostedOpStreamDeps,
@@ -79,6 +85,18 @@ type DirectRetainedProcessRoute = {
   routeEpoch: number;
 };
 
+/** Inline filesystem subprocesses stay durably retained for cancellation and
+ * recovery, but are not model-visible session background commands. */
+export function directRetainedProcessBackgroundCommand(
+  process: Pick<RoutingRetainedProcess, "id">,
+  operation: string,
+  purpose?: "synchronous_filesystem",
+): { commandId: string; command: string } | undefined {
+  return purpose === "synchronous_filesystem"
+    ? undefined
+    : { commandId: process.id, command: operation };
+}
+
 /** API-direct requests always use active-route authority. The default active
  * pointer is represented by a null target id; it is not a home-route write. */
 export function directRetainedProcessMatchesBackend(
@@ -116,9 +134,77 @@ export type ChannelARoutingServices = {
   settings: Settings;
   bus?: EventBus;
   onSandboxOperation?: RoutingSandboxOperationObserver;
+  onSandboxCaptureWait?: RoutingSandboxCaptureWaitObserver;
   waitSignal?: AbortSignal;
 };
 
+/**
+ * Whose access a Channel-A request uses to reach a Connected Machine. A person
+ * uses their own. An agent acts for the person who started its turn, never as
+ * itself, so it reaches the same machines that person can; a personal (Only me)
+ * machine additionally needs that exact running attempt's per-chat permission,
+ * rechecked before every routed operation like the worker's turn path.
+ */
+export type ChannelAMachineAuthority = {
+  subjectId: string;
+  personalMachineAttempt: {
+    turnId: string;
+    attemptId: string;
+    executionGeneration: number;
+  } | null;
+};
+
+/**
+ * Resolve a Connected Machine's live runner for one Channel-A request. The
+ * machine's audience can change in place, so its current scope is read from the
+ * enrollment itself. An agent attempt reaches a personal machine only through
+ * the same attempt admission as swap and run_on; everything else keeps the
+ * caller-scoped lookup.
+ */
+export async function resolveChannelAMachineConnection(
+  db: Database,
+  input: {
+    accountId: string;
+    workspaceId: string;
+    sessionId: string;
+    machineWorkspaceId: string;
+    enrollmentId: string;
+  },
+  authority: ChannelAMachineAuthority,
+): Promise<EnrollmentRecord | null> {
+  const attempt = authority.personalMachineAttempt;
+  if (attempt) {
+    const current = await getLiveEnrollmentConnection(
+      db,
+      input.machineWorkspaceId,
+      input.enrollmentId,
+    );
+    if (!current) return null;
+    if (current.scope === "user") {
+      return await resolvePersonalMachineConnectionForAttempt(db, {
+        accountId: input.accountId,
+        workspaceId: input.workspaceId,
+        subjectId: authority.subjectId,
+        sessionId: input.sessionId,
+        turnId: attempt.turnId,
+        attemptId: attempt.attemptId,
+        executionGeneration: attempt.executionGeneration,
+        enrollmentId: input.enrollmentId,
+        requireActiveSandbox: true,
+      });
+    }
+  }
+  return await getLiveEnrollmentConnection(
+    db,
+    {
+      accountId: input.accountId,
+      workspaceId: input.workspaceId,
+      subjectId: authority.subjectId,
+    },
+    input.enrollmentId,
+  );
+}
+
 /** Map the deployment relay URL to the leaf's `SelfhostedRelayConfig` shape. The
  *  relay URL (`OPENGENI_SELFHOSTED_RELAY_URL`) may carry a path (the relay's wss
  *  route); a path-less URL defaults to the relay's `/stream` route (M8b). */
@@ -175,11 +261,22 @@ function controlRpcFactory(bus: EventBus | undefined): () => ControlRpc {
 
 async function resolveSelfhostedConnection(
   services: ChannelARoutingServices,
-  access: string | { accountId: string; workspaceId: string; subjectId: string },
+  ids: { accountId: string; workspaceId: string; sessionId: string },
+  authority: ChannelAMachineAuthority,
   sandbox: RoutableSandbox,
 ): Promise<SelfhostedConnectionBinding | null> {
   if (!sandbox.enrollmentId) return null;
-  const enrollment = await getLiveEnrollmentConnection(services.db, access, sandbox.enrollmentId);
+  const enrollment = await resolveChannelAMachineConnection(
+    services.db,
+    {
+      accountId: ids.accountId,
+      workspaceId: ids.workspaceId,
+      sessionId: ids.sessionId,
+      machineWorkspaceId: sandbox.workspaceId,
+      enrollmentId: sandbox.enrollmentId,
+    },
+    authority,
+  );
   if (!enrollment?.connectionInstanceId) return null;
   const opStream: SelfhostedOpStreamDeps | undefined =
     services.settings.agentOpStreamEnabled === true &&
@@ -220,6 +317,10 @@ export function wrapChannelABoxWithRouting(
     workspaceId: string;
     sessionId: string;
     resourceSubjectId: string;
+    /** Whose access reaches Connected Machines (see ChannelAMachineAuthority).
+     * Resolved lazily, once, only when a request routes to a machine. Omitted,
+     * the request uses `resourceSubjectId`'s own access. */
+    machineAuthority?: () => Promise<ChannelAMachineAuthority>;
     homeLease?: {
       sandboxGroupId: string;
       leaseEpoch: number;
@@ -242,6 +343,16 @@ export function wrapChannelABoxWithRouting(
 ): EstablishedSandboxSession {
   const { db, settings, bus } = services;
   const homeLease = ids.homeLease;
+  let machineAuthority: Promise<ChannelAMachineAuthority> | undefined;
+  const resolveMachineAuthority = (): Promise<ChannelAMachineAuthority> => {
+    machineAuthority ??= ids.machineAuthority
+      ? ids.machineAuthority()
+      : Promise.resolve({
+          subjectId: ids.resourceSubjectId,
+          personalMachineAttempt: null,
+        });
+    return machineAuthority;
+  };
   const beforeMutation = homeLease
     ? async ({
         op,
@@ -292,13 +403,15 @@ export function wrapChannelABoxWithRouting(
         admission,
         outcome,
         retainedProcess,
+        retainedProcessPurpose,
       }: {
         op: string;
         backend: ResolvedActiveBackend;
         admission: unknown;
-        outcome: "resolved" | "rejected";
+        outcome: "resolved" | "rejected" | "outcome_unknown";
         result?: unknown;
         retainedProcess?: RoutingRetainedProcess;
+        retainedProcessPurpose?: "synchronous_filesystem";
       }): Promise<void> => {
         if (admission === null) return;
         if (
@@ -320,7 +433,12 @@ export function wrapChannelABoxWithRouting(
         ) {
           throw new Error("API-direct workspace mutation settlement lacked its bound admission");
         }
-        if (outcome === "resolved" && retainedProcess) {
+        if ((outcome === "resolved" || outcome === "outcome_unknown") && retainedProcess) {
+          const backgroundCommand = directRetainedProcessBackgroundCommand(
+            retainedProcess,
+            op,
+            retainedProcessPurpose,
+          );
           await retainWorkspaceProviderCommand(db, {
             accountId: ids.accountId,
             workspaceId: ids.workspaceId,
@@ -334,10 +452,7 @@ export function wrapChannelABoxWithRouting(
             admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
             operation: op,
             providerBinding: boundAdmission.providerBinding ?? null,
-            backgroundCommand: {
-              commandId: retainedProcess.id,
-              command: op,
-            },
+            ...(backgroundCommand ? { backgroundCommand } : {}),
             owner: {
               kind: "direct",
               requestId: ids.directRequest.requestId,
@@ -352,22 +467,42 @@ export function wrapChannelABoxWithRouting(
           });
           return;
         }
-        await verifyDirectWorkspaceMutationSettlement(db, {
-          accountId: ids.accountId,
-          workspaceId: ids.workspaceId,
-          sessionId: ids.sessionId,
-          requestId: ids.directRequest.requestId,
-          holderId: ids.directRequest.holderId,
-          initiatorSubjectId: ids.resourceSubjectId,
-          sandboxGroupId: homeLease.sandboxGroupId,
-          expectedEpoch: backend.leaseEpoch,
-          expectedInstanceId: backend.providerInstanceId,
-          routeTargetId: exactAdmission.routeTargetId,
-          routeEpoch: exactAdmission.routeEpoch,
-          admission: exactAdmission,
-          operation: op,
-          outcome,
-        });
+        if (outcome === "outcome_unknown")
+          throw new Error(
+            "Outcome-unknown command settlement requires its exact retained invocation",
+          );
+        try {
+          await verifyDirectWorkspaceMutationSettlement(db, {
+            accountId: ids.accountId,
+            workspaceId: ids.workspaceId,
+            sessionId: ids.sessionId,
+            requestId: ids.directRequest.requestId,
+            holderId: ids.directRequest.holderId,
+            initiatorSubjectId: ids.resourceSubjectId,
+            sandboxGroupId: homeLease.sandboxGroupId,
+            expectedEpoch: backend.leaseEpoch,
+            expectedInstanceId: backend.providerInstanceId,
+            routeTargetId: exactAdmission.routeTargetId,
+            routeEpoch: exactAdmission.routeEpoch,
+            admission: exactAdmission,
+            operation: op,
+            outcome,
+          });
+        } catch (error) {
+          if (
+            error instanceof SandboxWorkspaceMutationOutputRejectedError &&
+            error.matchesPhysicalSettlement({
+              accountId: ids.accountId,
+              workspaceId: ids.workspaceId,
+              admission: exactAdmission,
+              operation: op,
+              outcome,
+            })
+          ) {
+            throw new RoutingMutationOutputRejectedError(op, error.code, { cause: error });
+          }
+          throw error;
+        }
       }
     : undefined;
   const beforeProcessMutation = homeLease
@@ -412,15 +547,31 @@ export function wrapChannelABoxWithRouting(
         ) {
           throw new Error("API retained-process mutation settlement lacked its exact admission");
         }
-        await verifyRetainedProcessMutationSettlement(db, {
-          accountId: ids.accountId,
-          workspaceId: ids.workspaceId,
-          sessionId: ids.sessionId,
-          processId: process.id,
-          admission: admission as SandboxWorkspaceMutationAdmission,
-          operation: op,
-          outcome,
-        });
+        try {
+          await verifyRetainedProcessMutationSettlement(db, {
+            accountId: ids.accountId,
+            workspaceId: ids.workspaceId,
+            sessionId: ids.sessionId,
+            processId: process.id,
+            admission: admission as SandboxWorkspaceMutationAdmission,
+            operation: op,
+            outcome,
+          });
+        } catch (error) {
+          if (
+            error instanceof SandboxWorkspaceMutationOutputRejectedError &&
+            error.matchesPhysicalSettlement({
+              accountId: ids.accountId,
+              workspaceId: ids.workspaceId,
+              admission: admission as SandboxWorkspaceMutationAdmission,
+              operation: op,
+              outcome,
+            })
+          ) {
+            throw new RoutingMutationOutputRejectedError(op, error.code, { cause: error });
+          }
+          throw error;
+        }
       }
     : undefined;
   const settleProcess = homeLease
@@ -474,12 +625,13 @@ export function wrapChannelABoxWithRouting(
     defaultBackend: established.session as RoutableBackendSession,
     defaultKind: established.backendId,
     getSandbox: async (sandboxId): Promise<RoutableSandbox | null> => {
+      const authority = await resolveMachineAuthority();
       const sandbox = await getSandbox(
         db,
         {
           accountId: ids.accountId,
           workspaceId: ids.workspaceId,
-          subjectId: ids.resourceSubjectId,
+          subjectId: authority.subjectId,
         },
         sandboxId,
       );
@@ -494,14 +646,15 @@ export function wrapChannelABoxWithRouting(
         : null;
     },
     controlRpcFactory: controlRpcFactory(bus),
-    resolveSelfhostedConnection: (sandbox) =>
+    resolveSelfhostedConnection: async (sandbox) =>
       resolveSelfhostedConnection(
         services,
         {
           accountId: ids.accountId,
           workspaceId: ids.workspaceId,
-          subjectId: ids.resourceSubjectId,
+          sessionId: ids.sessionId,
         },
+        await resolveMachineAuthority(),
         sandbox,
       ),
     relay: relayConfigFromSettings(settings),
@@ -530,17 +683,23 @@ export function wrapChannelABoxWithRouting(
   });
 
   const proxy = new RoutingSandboxSession({
+    providerSupervisionReady: async () =>
+      settings.modalCommandSupervisionEnabled && (await supervisedCommandProtocolReady(db)),
     providerCommandHandle: (value) =>
       value && typeof value === "object"
         ? (value as PersistableMutationAdmission).admission?.workspaceGeneration
         : undefined,
     providerCommandPersistence: (process) =>
-      retainedProviderCommandPersistence(db, {
-        accountId: ids.accountId,
-        workspaceId: ids.workspaceId,
-        sessionId: ids.sessionId,
-        processId: process.id,
-      }),
+      retainedProviderCommandPersistence(
+        db,
+        {
+          accountId: ids.accountId,
+          workspaceId: ids.workspaceId,
+          sessionId: ids.sessionId,
+          processId: process.id,
+        },
+        bus ? (events) => bus.publish(ids.workspaceId, ids.sessionId, events) : undefined,
+      ),
     captureProcessOutput: async ({ process, chunkId, chunk, stream, streamFidelity }) => {
       const events = await appendSessionCommandOutput(db, {
         accountId: ids.accountId,
@@ -576,6 +735,7 @@ export function wrapChannelABoxWithRouting(
     },
     resolveActiveBackend: resolver,
     ...(services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {}),
+    ...(services.onSandboxCaptureWait ? { onCaptureWait: services.onSandboxCaptureWait } : {}),
     ...(beforeMutation ? { beforeMutation } : {}),
     ...(afterMutation ? { afterMutation } : {}),
     ...(beforeProcessMutation ? { beforeProcessMutation } : {}),
@@ -591,6 +751,7 @@ export function wrapChannelABoxWithRouting(
               sandboxGroupId: homeLease.sandboxGroupId,
               expectedEpoch: homeLease.leaseEpoch,
               expectedInstanceId: homeLease.instanceId,
+              expectedBackend: homeLease.backend,
               diagnostic: "provider_not_found_during_routed_operation",
             });
             if (marked.status === "marked" && bus) {