@opengeni/api-router 3.0.1 → 3.1.2

package/src/sandbox/channel-a.ts

@@ -22,19 +22,17 @@ import {
   hasGitCredentialRepositorySelection,
   hasGitHubRepositorySelection,
   sandboxLifecycleTransitionWaitMs,
+  sandboxWarmRateMicrosPerSecond,
   stableSandboxEnvironmentForRun,
   type Settings,
 } from "@opengeni/config";
 import { githubAppBotIdentity } from "@opengeni/github";
-import type { Session } from "@opengeni/contracts";
+import type { AccessGrant, Session } from "@opengeni/contracts";
 import {
   acquireLease,
   getSandboxSessionEnvelope,
-  getLiveEnrollmentConnection,
   getSandbox,
-  getScheduledScopedRigVersionMetadata,
   touchLeaseHolder,
-  loadWorkspaceEnvironmentForRun,
   markWarmLeaseInstanceLost,
   readActiveSandbox,
   readLease,
@@ -42,15 +40,18 @@ import {
   SandboxImageConflictError,
   SandboxProviderReadLockUnavailableError,
   SandboxRigConflictError,
+  SandboxPaidComputeAdmissionError,
   withSandboxProviderReadLock,
   type Database,
   type LeaseSnapshot,
 } from "@opengeni/db";
 import { appendAndPublishEvents, type EventBus } from "@opengeni/events";
 import {
+  errorCauseAttributes,
   recordTenancyCompatibilityLaneUse,
   sandboxLeaseTelemetryKey,
   sandboxOperationMetricObserver,
+  sandboxCaptureWaitMetricObserver,
   type Observability,
 } from "@opengeni/observability";
 import { HTTPException } from "hono/http-exception";
@@ -67,6 +68,7 @@ import {
   SandboxResumeIdentityMismatchError,
   SandboxResumeIdentityUnavailableError,
   RoutingActiveRouteChangedError,
+  RoutingMutationOutputRejectedError,
   RoutingWorkspaceRootChangedError,
   SelfhostedWorkspaceRootChangedError,
   ChannelAConflictError,
@@ -74,6 +76,7 @@ import {
   ChannelANotFoundError,
   ChannelAUnsupportedError,
   ChannelAUnavailableError,
+  SynchronousCommandOutcomeUnknownError,
   ChannelAValidationError,
   BrowserControlRequestError,
   BrowserControlTransportError,
@@ -89,12 +92,16 @@ import {
   type RoutingSandboxSession,
 } from "@opengeni/runtime/sandbox";
 import {
+  grantHasAgentAttemptAuthority,
   managedSessionGroupBackend,
   managedSessionGroupOs,
   providerSettingsForSessionSandboxRuntime,
   relayConfigFromSettings,
+  requireLiveAgentAttemptAuthorization,
+  resolveChannelAMachineConnection,
   resolveSessionSandboxRuntime,
   wrapChannelABoxWithRouting,
+  type ChannelAMachineAuthority,
 } from "@opengeni/core";
 import { establishApiSandboxSpawner } from "./rematerialize";
 
@@ -147,6 +154,9 @@ export type ChannelAContext = {
   session: Session;
   // The principal that drives the op (for emit attribution + pty opened_by).
   subjectId: string;
+  /** The authenticated route grant, when one drives the op. It resolves the
+   *  session's Sandbox Environment for an agent attempt as its initiating human. */
+  grant?: AccessGrant | undefined;
   /** Cancel lifecycle waiting when the originating HTTP request disconnects. */
   waitSignal?: AbortSignal | undefined;
   /** Bounded route identity for metrics and safe operator diagnostics. */
@@ -156,6 +166,10 @@ export type ChannelAContext = {
    * provider handle and replay the request. Tab/lifecycle mutations that lack
    * a controller operation id must never opt into this recovery. */
   retryControllerTransport?: boolean | undefined;
+  /** Ephemeral browser callbacks must never replay after an ambiguous result. */
+  allowOperationReplay?: boolean | undefined;
+  /** Exact existing interaction provider; may not spawn/rotate on image drift. */
+  retainedInstanceId?: string | undefined;
 };
 
 export type ChannelAOperationFailureReason =
@@ -189,6 +203,10 @@ export type ChannelAHandle = {
    * active-sandbox pointer and is safe for placement-bound controllers. */
   homeSession: ChannelASession;
   routingSession: RoutingSandboxSession;
+  /** Whose access this request uses to reach a Connected Machine: the person
+   * who started an agent's turn, or the calling person. Routes that look up
+   * the routed machine again must use it rather than the raw caller grant. */
+  machineAuthority: () => Promise<ChannelAMachineAuthority>;
   requestId: string;
 };
 
@@ -403,6 +421,8 @@ type ChannelAReadRecoveryOptions = {
   /** Modal may expose one more stale command-router route after the first
    * successful handle rebuild. Keep this closed and statically bounded. */
   maxFreshHandleRetries?: 1 | 2;
+  /** Explicit opt-out overrides every transport/provider retry classification. */
+  allowOperationReplay?: boolean | undefined;
   /** Never start another provider attempt after the originating request ends. */
   waitSignal?: AbortSignal | undefined;
   /** Additional callback-specific failure that is safe to replay. */
@@ -429,7 +449,11 @@ export async function runChannelAReadWithFreshHandleRetry<T>(
     } catch (error) {
       const retryable =
         error instanceof ChannelAUnavailableError || options.retryableError?.(error) === true;
-      if (!retryable || retries >= maxFreshHandleRetries) {
+      if (
+        options.allowOperationReplay === false ||
+        !retryable ||
+        retries >= maxFreshHandleRetries
+      ) {
         throw error;
       }
       options.waitSignal?.throwIfAborted();
@@ -521,6 +545,36 @@ export async function withChannelARead<T>(
   return await withChannelAOperation(services, ctx, true, fn);
 }
 
+/**
+ * Whose access this request uses to reach a Connected Machine. A person uses
+ * their own. An agent acts for the person who started its turn (as swap and
+ * run_on do), so the machines that person can use stay usable from browser,
+ * computer, file and terminal requests; a personal machine also needs the
+ * attempt's own per-chat permission, checked by the connection resolver.
+ */
+async function channelAMachineAuthority(
+  db: Database,
+  ctx: ChannelAContext,
+): Promise<ChannelAMachineAuthority> {
+  const grant = ctx.grant;
+  const callerSessionId = grant?.metadata?.["sessionId"];
+  if (!grant || !grantHasAgentAttemptAuthority(grant) || typeof callerSessionId !== "string") {
+    return { subjectId: ctx.subjectId, personalMachineAttempt: null };
+  }
+  const actor = await requireLiveAgentAttemptAuthorization(db, grant, callerSessionId);
+  if (!actor.initiatingHumanSubjectId) {
+    return { subjectId: ctx.subjectId, personalMachineAttempt: null };
+  }
+  return {
+    subjectId: actor.initiatingHumanSubjectId,
+    personalMachineAttempt: {
+      turnId: actor.turnId,
+      attemptId: actor.attemptId,
+      executionGeneration: actor.executionGeneration,
+    },
+  };
+}
+
 async function withChannelAOperation<T>(
   services: ChannelAServices,
   ctx: ChannelAContext,
@@ -531,62 +585,39 @@ async function withChannelAOperation<T>(
   const onSandboxOperation = services.observability
     ? sandboxOperationMetricObserver(services.observability)
     : undefined;
+  const onSandboxCaptureWait = services.observability
+    ? sandboxCaptureWaitMetricObserver(services.observability)
+    : undefined;
   const { accountId, workspaceId, session } = ctx;
 
-  if (session.sandboxBackend === "none") {
-    throw new HTTPException(409, { message: "sandbox not available" });
-  }
-
   const sandboxGroupId = session.sandboxGroupId;
   const requestId = crypto.randomUUID();
   const holderId = `direct:${requestId}`;
   const operationStartedAt = performance.now();
   const operation = ctx.operation ?? (readOnly ? "read" : "mutation");
   const leaseTtlMs = settings.sandboxLeaseTtlMs;
+  let machineAuthorityResult: Promise<ChannelAMachineAuthority> | undefined;
+  const machineAuthority = (): Promise<ChannelAMachineAuthority> =>
+    (machineAuthorityResult ??= channelAMachineAuthority(db, ctx));
 
   // The STABLE run-environment used by both a cloud home and a machine home.
-  // It also carries the per-session Codemode pointer selected below.
-  const workspaceEnvironmentValues: Record<string, string> = {};
-  const rigVersion =
-    session.rigId && session.rigVersionId
-      ? await getScheduledScopedRigVersionMetadata(
-          db,
-          {
-            accountId,
-            workspaceId,
-            subjectId: ctx.subjectId ?? "session-attach",
-          },
-          session.rigId,
-          session.rigVersionId,
-        )
-      : null;
-  for (const variableSetId of rigVersion?.version.defaultVariableSetIds ?? []) {
-    const workspaceEnvironment = await loadWorkspaceEnvironmentForRun(db, settings, {
-      accountId,
-      workspaceId,
-      variableSetId,
-      authority: { kind: "session_attach", sessionId: session.id, subjectId: ctx.subjectId },
-    });
-    Object.assign(workspaceEnvironmentValues, workspaceEnvironment?.values ?? {});
-  }
-  for (const variableSetId of session.variableSetIds) {
-    const workspaceEnvironment = await loadWorkspaceEnvironmentForRun(db, settings, {
-      accountId,
-      workspaceId,
-      variableSetId,
-      authority: { kind: "session_attach", sessionId: session.id, subjectId: ctx.subjectId },
-    });
-    Object.assign(workspaceEnvironmentValues, workspaceEnvironment?.values ?? {});
+  // It also carries the per-session Codemode pointer selected below. Constants
+  // only: user secrets are resolved per command (W1), never seeded here.
+  // `none` describes only the durable home. A session without a home sandbox
+  // that attached a Connected Machine runs its turns there (the worker gives
+  // such a turn the effective selfhosted backend), so its direct operations,
+  // such as opening a computer or browser on that machine, follow the same
+  // active pointer.
+  const machineOnlyHome = session.sandboxBackend === "none";
+  if (machineOnlyHome && !(await readActiveSandbox(db, workspaceId, session.id))?.activeSandboxId) {
+    throw new HTTPException(409, { message: "sandbox not available" });
   }
+  const effectiveBackend = machineOnlyHome ? "selfhosted" : session.sandboxBackend;
   const settingsForSession =
-    session.sandboxBackend !== settings.sandboxBackend
-      ? { ...settings, sandboxBackend: session.sandboxBackend }
+    effectiveBackend !== settings.sandboxBackend
+      ? { ...settings, sandboxBackend: effectiveBackend }
       : settings;
-  const environment = stableSandboxEnvironmentForRun(
-    settingsForSession,
-    workspaceEnvironmentValues,
-    { workspaceId },
-  );
+  const environment = stableSandboxEnvironmentForRun(settingsForSession, { workspaceId });
   if (hasGitCredentialRepositorySelection(session.resources)) {
     applyGitAuthPointerEnvironment(
       environment,
@@ -635,6 +666,7 @@ async function withChannelAOperation<T>(
       lease,
       homeSession,
       routingSession,
+      machineAuthority,
       requestId,
     });
     // The direct request has accepted the result in memory. Finalize every
@@ -647,12 +679,15 @@ async function withChannelAOperation<T>(
   // A machine-targeted top-level session has an honest selfhosted HOME label.
   // It has no cloud provider box and therefore must not acquire or establish a
   // phantom home lease before following its active machine pointer.
-  if (session.sandboxBackend === "selfhosted") {
+  if (effectiveBackend === "selfhosted") {
     let established: EstablishedSandboxSession | undefined;
     try {
       ctx.waitSignal?.throwIfAborted();
       const pointer = await readActiveSandbox(db, workspaceId, session.id);
       if (!pointer?.activeSandboxId) {
+        if (machineOnlyHome) {
+          throw new HTTPException(409, { message: "sandbox not available" });
+        }
         // A machine-home session with no selected machine uses the deployment's
         // managed group box, exactly like the worker turn path. Keep the durable
         // home label honest; only this request's effective backend changes.
@@ -679,9 +714,10 @@ async function withChannelAOperation<T>(
           message: "machine-home session has no active Connected Machine or managed sandbox",
         });
       }
+      const requestMachineAuthority = await machineAuthority();
       const sandbox = await getSandbox(
         db,
-        { accountId: ctx.accountId, workspaceId, subjectId: ctx.subjectId },
+        { accountId: ctx.accountId, workspaceId, subjectId: requestMachineAuthority.subjectId },
         pointer.activeSandboxId,
       );
       if (sandbox?.kind !== "selfhosted" || !sandbox.enrollmentId) {
@@ -690,10 +726,16 @@ async function withChannelAOperation<T>(
         });
       }
       const originWorkspaceId = sandbox.workspaceId;
-      const enrollment = await getLiveEnrollmentConnection(
+      const enrollment = await resolveChannelAMachineConnection(
         db,
-        { accountId, workspaceId, subjectId: ctx.subjectId },
-        sandbox.enrollmentId,
+        {
+          accountId,
+          workspaceId,
+          sessionId: session.id,
+          machineWorkspaceId: originWorkspaceId,
+          enrollmentId: sandbox.enrollmentId,
+        },
+        requestMachineAuthority,
       );
       if (!enrollment?.connectionInstanceId) {
         // Preserve causal machine liveness through Channel-A. Generic callers
@@ -753,6 +795,7 @@ async function withChannelAOperation<T>(
           settings,
           bus,
           ...(onSandboxOperation ? { onSandboxOperation } : {}),
+          ...(onSandboxCaptureWait ? { onSandboxCaptureWait } : {}),
           ...(ctx.waitSignal ? { waitSignal: ctx.waitSignal } : {}),
         },
         {
@@ -760,6 +803,7 @@ async function withChannelAOperation<T>(
           workspaceId,
           sessionId: session.id,
           resourceSubjectId: ctx.subjectId,
+          machineAuthority,
           pinnedSelfhosted: {
             sandboxId: sandbox.id,
             epoch: pointer.activeEpoch,
@@ -787,9 +831,14 @@ async function withChannelAOperation<T>(
 
   // One session has one logical runtime across turns and every API-direct
   // surface. Without this, Terminal/Files/Browser/Computer/viewers could rearm
-  // a stale deployment image after the worker had resolved a newer Pack/Rig or
+
   // deployment image for the same durable sandbox group.
-  const sandboxRuntime = await resolveSessionSandboxRuntime(db, settings, session);
+  const sandboxRuntime = await resolveSessionSandboxRuntime(
+    db,
+    settings,
+    session,
+    ctx.grant ? { grant: ctx.grant } : { subjectId: ctx.subjectId },
+  );
 
   const release = async (): Promise<void> => {
     await releaseLeaseHolder(db, {
@@ -818,8 +867,14 @@ async function withChannelAOperation<T>(
       holderId,
       subjectId: session.id,
       backend: session.sandboxBackend,
+      warmBilling: {
+        mode: settings.sandboxWarmBillingMode,
+        rateMicrosPerSecond: sandboxWarmRateMicrosPerSecond(settings, session.sandboxBackend),
+      },
       os: session.sandboxOs,
       image: sandboxRuntime.image,
+      imagePolicy: "new_creates_only",
+      ...(ctx.retainedInstanceId ? { retainedInstanceId: ctx.retainedInstanceId } : {}),
       rigVersionId: session.rigVersionId,
       leaseTtlMs,
       warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
@@ -1060,6 +1115,7 @@ async function withChannelAOperation<T>(
           settings,
           bus,
           ...(onSandboxOperation ? { onSandboxOperation } : {}),
+          ...(onSandboxCaptureWait ? { onSandboxCaptureWait } : {}),
           ...(ctx.waitSignal ? { waitSignal: ctx.waitSignal } : {}),
         },
         {
@@ -1067,6 +1123,7 @@ async function withChannelAOperation<T>(
           workspaceId,
           sessionId: session.id,
           resourceSubjectId: ctx.subjectId,
+          machineAuthority,
           homeLease: {
             sandboxGroupId,
             leaseEpoch: leaseSnapshot.leaseEpoch,
@@ -1168,6 +1225,7 @@ async function withChannelAOperation<T>(
           },
           {
             maxFreshHandleRetries: session.sandboxBackend === "modal" ? 2 : 1,
+            allowOperationReplay: ctx.allowOperationReplay,
             ...(ctx.waitSignal ? { waitSignal: ctx.waitSignal } : {}),
             ...(ctx.retryControllerTransport
               ? { retryableError: isRetryableControllerTransport }
@@ -1204,6 +1262,8 @@ async function withChannelAOperation<T>(
  *  already-HTTPException unchanged. */
 export function mapChannelAError(error: unknown, waitSignal?: AbortSignal): unknown {
   if (error instanceof HTTPException) return error;
+  if (error instanceof SandboxPaidComputeAdmissionError)
+    return new HTTPException(402, { message: error.message, cause: error });
   if (isChannelARequestCancellation(error, waitSignal))
     return new HTTPException(499 as never, {
       message: "request cancelled",
@@ -1214,6 +1274,20 @@ export function mapChannelAError(error: unknown, waitSignal?: AbortSignal): unkn
     error instanceof SandboxResumeIdentityUnavailableError
   )
     return new HTTPException(409, { message: error.message });
+  if (error instanceof RoutingMutationOutputRejectedError) {
+    const revoked = error.reasonCode === "authority_revoked";
+    return new ApiHttpError(revoked ? 403 : 409, {
+      code: revoked ? "forbidden" : "conflict",
+      message: error.message,
+      retryable: false,
+      outcomeUnknown: false,
+      details: {
+        code: error.code,
+        reasonCode: error.reasonCode,
+        physicalOutcome: "resolved",
+      },
+    });
+  }
   if (
     error instanceof RoutingActiveRouteChangedError ||
     error instanceof RoutingWorkspaceRootChangedError ||
@@ -1235,8 +1309,27 @@ export function mapChannelAError(error: unknown, waitSignal?: AbortSignal): unkn
     });
   if (error instanceof SelfhostedControlError && error.agentOffline)
     return new HTTPException(409, { message: error.message, cause: error });
+  if (error instanceof SelfhostedControlError && error.payloadTooLarge) {
+    const outbound = error.detail.direction === "request";
+    // Project only the bounded direction. Native free-form diagnostics can
+    // contain file contents or private paths. A large reply is not evidence
+    // that the operation failed before execution, and must not invite replay.
+    return new ApiHttpError(outbound ? 413 : 502, {
+      code: "limit_exceeded",
+      message: outbound
+        ? "The request exceeds the connected machine's per-message size limit. Large file content requires a machine that supports bounded file transfers."
+        : "The connected machine's reply exceeds its per-message size limit. The operation may have completed; inspect its result before repeating it. Read large files in bounded ranges.",
+      retryable: false,
+      details: {
+        code: "machine_transport_payload_too_large",
+        direction: outbound ? "request" : "response",
+      },
+    });
+  }
   if (error instanceof ChannelAUnavailableError)
     return new HTTPException(503, { message: error.message });
+  if (error instanceof SynchronousCommandOutcomeUnknownError)
+    return new HTTPException(409, { message: error.message });
   if (error instanceof ChannelAValidationError)
     return new HTTPException(400, { message: error.message });
   if (error instanceof ChannelANotFoundError)
@@ -1292,6 +1385,13 @@ export function channelAOperationFailureDiagnostic(
       errorCode: "sandbox_channel_a_cancelled",
     };
   }
+  if (error instanceof RoutingMutationOutputRejectedError) {
+    return {
+      reason: "request_rejected",
+      status: error.reasonCode === "authority_revoked" ? 403 : 409,
+      errorCode: "sandbox_channel_a_operation_failed",
+    };
+  }
   if (error instanceof SandboxProviderReadLockUnavailableError) {
     return {
       reason: "provider_read_busy",
@@ -1299,6 +1399,13 @@ export function channelAOperationFailureDiagnostic(
       errorCode: "sandbox_channel_a_provider_busy",
     };
   }
+  if (error instanceof SelfhostedControlError && error.payloadTooLarge) {
+    return {
+      reason: error.detail.direction === "request" ? "request_rejected" : "provider_unavailable",
+      status: error.detail.direction === "request" ? 413 : 502,
+      errorCode: "sandbox_channel_a_operation_failed",
+    };
+  }
   if (error instanceof SandboxImageConflictError || error instanceof SandboxRigConflictError) {
     return {
       reason: "lifecycle_conflict",
@@ -1398,6 +1505,9 @@ export function observeChannelAOperationFailure(
     errorClass: "SandboxChannelAOperationError",
     errorCode: diagnostic.errorCode,
     origin: "api",
+    // Which component refused it (controller, machine, database), as reviewed
+    // closed values only; "unexpected" alone cannot be diagnosed after the fact.
+    ...errorCauseAttributes(input.error),
   } as const;
   try {
     services.observability.incrementCounter({