@opengeni/api-router 0.30.3 → 2.2.0-canary.0

package/src/sandbox/channel-a.ts

@@ -30,25 +30,30 @@ import type { Session } from "@opengeni/contracts";
 import {
   acquireLease,
   getSandboxSessionEnvelope,
-  getEnrollment,
+  getLiveEnrollmentConnection,
   getSandbox,
+  touchLeaseHolder,
   loadWorkspaceEnvironmentForRun,
   markWarmLeaseInstanceLost,
   readActiveSandbox,
   readLease,
   releaseLeaseHolder,
+  SandboxImageConflictError,
   SandboxProviderReadLockUnavailableError,
+  SandboxRigConflictError,
   withSandboxProviderReadLock,
   type Database,
   type LeaseSnapshot,
 } from "@opengeni/db";
 import { appendAndPublishEvents, type EventBus } from "@opengeni/events";
 import {
+  recordTenancyCompatibilityLaneUse,
   sandboxLeaseTelemetryKey,
   sandboxOperationMetricObserver,
   type Observability,
 } from "@opengeni/observability";
 import { HTTPException } from "hono/http-exception";
+import { ApiHttpError } from "../http/api-error";
 
 import {
   buildSelfhostedBackendSession,
@@ -59,6 +64,8 @@ import {
   NatsOpStreamTransport,
   SandboxResumeIdentityMismatchError,
   SandboxResumeIdentityUnavailableError,
+  RoutingWorkspaceRootChangedError,
+  SelfhostedWorkspaceRootChangedError,
   ChannelAConflictError,
   ChannelANotFoundError,
   ChannelAUnsupportedError,
@@ -66,14 +73,23 @@ import {
   ChannelAValidationError,
   BrowserControlRequestError,
   BrowserControlTransportError,
+  SelfhostedControlError,
+  agentErrorToControlError,
   codemodeTokenFileFromEnvironment,
+  offlineAgentError,
+  resolveConnectedMachineWorkspaceRoot,
   withCodemodeTokenSession,
   withRunCredentialsSession,
   type ChannelASession,
   type EstablishedSandboxSession,
   type RoutingSandboxSession,
 } from "@opengeni/runtime/sandbox";
-import { relayConfigFromSettings, wrapChannelABoxWithRouting } from "@opengeni/core";
+import {
+  providerSettingsForSessionSandboxRuntime,
+  relayConfigFromSettings,
+  resolveSessionSandboxRuntime,
+  wrapChannelABoxWithRouting,
+} from "@opengeni/core";
 import { establishApiSandboxSpawner } from "./rematerialize";
 
 export type ChannelAServices = {
@@ -87,6 +103,7 @@ export type ChannelAOperation =
   | "fs.list"
   | "fs.list-batch"
   | "fs.read"
+  | "artifact.publish"
   | "fs.write"
   | "fs.delete"
   | "fs.move"
@@ -522,12 +539,12 @@ async function withChannelAOperation<T>(
 
   // The STABLE run-environment used by both a cloud home and a machine home.
   // It also carries the per-session Codemode pointer selected below.
-  const workspaceEnvironment = await loadWorkspaceEnvironmentForRun(
-    db,
-    settings,
+  const workspaceEnvironment = await loadWorkspaceEnvironmentForRun(db, settings, {
+    accountId,
     workspaceId,
-    session.environmentId,
-  );
+    variableSetId: session.environmentId,
+    authority: { kind: "session_attach", sessionId: session.id, subjectId: ctx.subjectId },
+  });
   const settingsForSession =
     session.sandboxBackend !== settings.sandboxBackend
       ? { ...settings, sandboxBackend: session.sandboxBackend }
@@ -566,12 +583,20 @@ async function withChannelAOperation<T>(
           codemodeTokenFileFromEnvironment(environment, session.id),
         )
       : credentialSession;
+    const fileSystemRoot = await routingSession.fileSystemRoot();
     const service = new SandboxChannelAService({
       session: scopedSession as ChannelASession,
+      workspaceRoot: fileSystemRoot,
       leaseEpoch: lease?.leaseEpoch ?? session.activeEpoch,
       emit,
     });
-    const result = await fn({ service, lease, homeSession, routingSession, requestId });
+    const result = await fn({
+      service,
+      lease,
+      homeSession,
+      routingSession,
+      requestId,
+    });
     // The direct request has accepted the result in memory. Finalize every
     // Connected Machine backend the routing proxy reached so a mid-request
     // route transition cannot leave completed output retained until TTL.
@@ -588,27 +613,76 @@ async function withChannelAOperation<T>(
       ctx.waitSignal?.throwIfAborted();
       const pointer = await readActiveSandbox(db, workspaceId, session.id);
       if (!pointer?.activeSandboxId) {
+        // A machine-home session with no selected machine uses the deployment's
+        // managed group box, exactly like the worker turn path. Keep the durable
+        // home label honest; only this request's effective backend changes.
+        if (settings.sandboxBackend !== "none" && settings.sandboxBackend !== "selfhosted") {
+          return await withChannelAOperation(
+            services,
+            {
+              ...ctx,
+              session: {
+                ...session,
+                sandboxBackend: settings.sandboxBackend,
+                sandboxOs: "linux",
+              },
+            },
+            readOnly,
+            fn,
+          );
+        }
         throw new HTTPException(409, {
           message: "machine-home session has no active Connected Machine",
         });
       }
-      const sandbox = await getSandbox(db, workspaceId, pointer.activeSandboxId);
+      const sandbox = await getSandbox(
+        db,
+        { accountId: ctx.accountId, workspaceId, subjectId: ctx.subjectId },
+        pointer.activeSandboxId,
+      );
       if (sandbox?.kind !== "selfhosted" || !sandbox.enrollmentId) {
         throw new HTTPException(409, {
           message: "machine-home session points to an unavailable Connected Machine",
         });
       }
-      const enrollment = await getEnrollment(db, workspaceId, sandbox.enrollmentId);
+      const originWorkspaceId = sandbox.workspaceId;
+      const enrollment = await getLiveEnrollmentConnection(
+        db,
+        { accountId, workspaceId, subjectId: ctx.subjectId },
+        sandbox.enrollmentId,
+      );
+      if (!enrollment?.connectionInstanceId) {
+        // Preserve causal machine liveness through Channel-A. Generic callers
+        // retain the established 409 mapping below; Browser/Computer callers can
+        // project the same bounded `agent_offline` contract as control RPC.
+        throw agentErrorToControlError(
+          offlineAgentError("Connected Machine has no live runner connection", true),
+        );
+      }
+      if (!enrollment.workspaceRoot) {
+        throw new HTTPException(409, {
+          message:
+            "Connected Machine has not reported an absolute workspace root; reconnect it with a current agent",
+        });
+      }
       const built = await buildSelfhostedBackendSession({
-        workspaceId,
+        workspaceId: originWorkspaceId,
         agentId: sandbox.enrollmentId,
+        connectionInstanceId: enrollment.connectionInstanceId,
+        workspaceRoot: resolveConnectedMachineWorkspaceRoot(
+          enrollment.workspaceRoot,
+          pointer.workingDir,
+        ),
         relay: relayConfigFromSettings(settings),
         controlRpcFactory: () => new NatsControlRpc(async () => bus.getRequestConnection()),
         epoch: pointer.activeEpoch,
         environment,
-        workingDir: pointer.workingDir,
         timeoutMs: settings.sandboxSelfhostedControlTimeoutMs,
         execTimeoutMs: settings.sandboxSelfhostedExecTimeoutMs,
+        operationResourcePolicy: enrollment.operationPolicy,
+        operationResourcePolicySupported:
+          enrollment.agentCapabilities.operationResourcePolicy === true,
+        operationCpuQuotaSupported: enrollment.agentCapabilities.operationCpuQuota === true,
         ...(settings.agentOpStreamEnabled === true &&
         enrollment?.opStream === true &&
         bus.getOpStreamConnection
@@ -640,6 +714,7 @@ async function withChannelAOperation<T>(
           accountId,
           workspaceId,
           sessionId: session.id,
+          resourceSubjectId: ctx.subjectId,
           pinnedSelfhosted: {
             sandboxId: sandbox.id,
             epoch: pointer.activeEpoch,
@@ -665,6 +740,12 @@ async function withChannelAOperation<T>(
     }
   }
 
+  // One session has one logical runtime across turns and every API-direct
+  // surface. Without this, Terminal/Files/Browser/Computer/viewers could rearm
+  // a stale deployment image after the worker had resolved a newer Pack/Rig or
+  // deployment image for the same durable sandbox group.
+  const sandboxRuntime = await resolveSessionSandboxRuntime(db, settings, session);
+
   const release = async (): Promise<void> => {
     await releaseLeaseHolder(db, {
       accountId,
@@ -693,6 +774,8 @@ async function withChannelAOperation<T>(
       subjectId: session.id,
       backend: session.sandboxBackend,
       os: session.sandboxOs,
+      image: sandboxRuntime.image,
+      rigVersionId: session.rigVersionId,
       leaseTtlMs,
       warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
       captureWaitMs: sandboxLifecycleTransitionWaitMs(settings),
@@ -716,6 +799,56 @@ async function withChannelAOperation<T>(
       });
     }
   } catch (error) {
+    let mappedError: unknown = error;
+    if (error instanceof SandboxImageConflictError) {
+      // A durable Browser/Computer holder can outlive its provider container.
+      // Runtime drift is checked before a new direct holder is admitted, so a
+      // dead old container would otherwise strand the lease behind a permanent
+      // conflict. Probe the exact old identity without replacement; only an
+      // authoritative provider-missing result may retire it and unblock the
+      // next request's cold successor election.
+      const live = await readLease(db, workspaceId, sandboxGroupId).catch(() => null);
+      if (live?.liveness === "warm" && live.instanceId !== null && live.resumeState !== null) {
+        let probe: EstablishedSandboxSession | undefined;
+        try {
+          probe = await establishSandboxSessionFromEnvelope(
+            sandboxRuntime.settings,
+            live.resumeState,
+            {
+              sessionId: session.id,
+              recovery: "resume-only",
+              backendOverride: session.sandboxBackend,
+              environment,
+            },
+          );
+        } catch (probeError) {
+          if (isProviderSandboxNotFoundError(session.sandboxBackend, probeError)) {
+            const marked = await markWarmLeaseInstanceLost(db, {
+              accountId,
+              workspaceId,
+              sandboxGroupId,
+              expectedEpoch: live.leaseEpoch,
+              expectedInstanceId: live.instanceId,
+            }).catch(() => null);
+            if (marked?.status === "marked") {
+              await appendAndPublishEvents(db, bus, workspaceId, session.id, [
+                {
+                  type: "sandbox.box.lost",
+                  payload: { sandboxId: live.instanceId },
+                },
+              ]).catch(() => undefined);
+              mappedError = new ApiHttpError(409, {
+                code: "conflict",
+                message: "sandbox instance was lost; retry to restore it",
+                retryable: true,
+              });
+            }
+          }
+        } finally {
+          await dropEstablishedHandle(probe);
+        }
+      }
+    }
     // Release is idempotent. If acquisition committed before the request was
     // cancelled, this removes that exact direct holder; a transient DB failure
     // must not overwrite the original structural error (holder TTL is the final
@@ -727,12 +860,28 @@ async function withChannelAOperation<T>(
       backend: session.sandboxBackend,
       operation,
       durationMs: performance.now() - operationStartedAt,
-      error,
+      error: mappedError,
       waitSignal: ctx.waitSignal,
     });
-    throw mapChannelAError(error, ctx.waitSignal);
+    throw mapChannelAError(mappedError, ctx.waitSignal);
   }
 
+  // Keep the exact direct-request owner visible for the full operation. A
+  // yielded command is also tracked by a non-TTL process holder, but the
+  // retained-process reconciler must not poll that provider session while this
+  // request is still its active owner. Stale direct holders remain crash-
+  // recoverable through the ordinary holder TTL reaper.
+  const directHolderHeartbeat = setInterval(() => {
+    void touchLeaseHolder(db, {
+      accountId,
+      workspaceId,
+      sandboxGroupId,
+      kind: "direct",
+      holderId,
+    }).catch(() => undefined);
+  }, 10_000);
+  directHolderHeartbeat.unref?.();
+
   let established: EstablishedSandboxSession | undefined;
   let leaseSnapshot: LeaseSnapshot = acquired.lease;
   let establishedCacheKey: string | null = null;
@@ -747,7 +896,7 @@ async function withChannelAOperation<T>(
   ): Promise<{ established: EstablishedSandboxSession; cacheKey: string }> => {
     const cacheKey = establishedHandleCacheKey(workspaceId, session.id, live);
     const establish = () =>
-      establishSandboxSessionFromEnvelope(settings, live.resumeState, {
+      establishSandboxSessionFromEnvelope(sandboxRuntime.settings, live.resumeState, {
         sessionId: session.id,
         recovery: "resume-only",
         backendOverride: session.sandboxBackend,
@@ -781,8 +930,10 @@ async function withChannelAOperation<T>(
           },
         ]);
       }
-      throw new HTTPException(409, {
+      throw new ApiHttpError(409, {
+        code: "conflict",
         message: `sandbox instance was lost; retry to restore it`,
+        retryable: true,
       });
     }
   };
@@ -802,9 +953,13 @@ async function withChannelAOperation<T>(
       // resume_state is the lease's authoritative box descriptor; the session
       // `_sandbox` envelope is only the per-session fallback.
       try {
+        const providerSettings = await providerSettingsForSessionSandboxRuntime(
+          sandboxRuntime,
+          session.sandboxBackend,
+        );
         const result = await establishApiSandboxSpawner({
           db,
-          settings,
+          settings: providerSettings,
           accountId,
           workspaceId,
           sandboxGroupId,
@@ -863,6 +1018,7 @@ async function withChannelAOperation<T>(
           accountId,
           workspaceId,
           sessionId: session.id,
+          resourceSubjectId: ctx.subjectId,
           homeLease: {
             sandboxGroupId,
             leaseEpoch: leaseSnapshot.leaseEpoch,
@@ -990,6 +1146,7 @@ async function withChannelAOperation<T>(
     });
     throw mapChannelAError(error, ctx.waitSignal);
   } finally {
+    clearInterval(directHolderHeartbeat);
     await release();
     await dropEstablishedHandle(established);
   }
@@ -1009,8 +1166,25 @@ export function mapChannelAError(error: unknown, waitSignal?: AbortSignal): unkn
     error instanceof SandboxResumeIdentityUnavailableError
   )
     return new HTTPException(409, { message: error.message });
+  if (
+    error instanceof RoutingWorkspaceRootChangedError ||
+    error instanceof SelfhostedWorkspaceRootChangedError
+  )
+    return new ApiHttpError(409, {
+      code: "conflict",
+      message: error.message,
+      retryable: true,
+    });
   if (error instanceof SandboxProviderReadLockUnavailableError)
     return new HTTPException(503, { message: error.message });
+  if (error instanceof SandboxImageConflictError || error instanceof SandboxRigConflictError)
+    return new ApiHttpError(409, {
+      code: "conflict",
+      message: "sandbox runtime changed while this session still has active operations; retry",
+      retryable: true,
+    });
+  if (error instanceof SelfhostedControlError && error.agentOffline)
+    return new HTTPException(409, { message: error.message, cause: error });
   if (error instanceof ChannelAUnavailableError)
     return new HTTPException(503, { message: error.message });
   if (error instanceof ChannelAValidationError)
@@ -1021,9 +1195,31 @@ export function mapChannelAError(error: unknown, waitSignal?: AbortSignal): unkn
     return new HTTPException(409, { message: error.message });
   if (error instanceof ChannelAUnsupportedError)
     return new HTTPException(409, { message: error.message });
+  const authorityFence = workspaceMutationAuthorityFenceCode(error);
+  // A revoked grant is an authorization outcome, not a server fault; an
+  // unattributed pre-0277 writer is a conflict the caller resolves by starting
+  // fresh work. Both must be visible instead of surfacing as a 500.
+  if (authorityFence === "authority_revoked")
+    return new HTTPException(403, { message: (error as Error).message });
+  if (authorityFence === "authority_unattributed")
+    return new HTTPException(409, { message: (error as Error).message });
   return error;
 }
 
+/** Structural check: the fence type lives in `@opengeni/db` and is raised deep
+ * inside admission, so match it the same way the worker does. */
+function workspaceMutationAuthorityFenceCode(error: unknown): string | null {
+  if (
+    !(error instanceof Error) ||
+    (error.name !== "SandboxWorkspaceMutationFencedError" &&
+      error.name !== "SandboxRetainedProcessPromotionFencedError")
+  ) {
+    return null;
+  }
+  const code = (error as Error & { code?: unknown }).code;
+  return typeof code === "string" ? code : null;
+}
+
 export function isChannelARequestCancellation(error: unknown, waitSignal?: AbortSignal): boolean {
   if (waitSignal?.aborted !== true) return false;
   const isSignalReason = waitSignal.reason !== undefined && error === waitSignal.reason;
@@ -1053,7 +1249,14 @@ export function channelAOperationFailureDiagnostic(
       errorCode: "sandbox_channel_a_provider_busy",
     };
   }
-  if (error instanceof ChannelAUnavailableError) {
+  if (error instanceof SandboxImageConflictError || error instanceof SandboxRigConflictError) {
+    return {
+      reason: "lifecycle_conflict",
+      status: 409,
+      errorCode: "sandbox_channel_a_lifecycle_conflict",
+    };
+  }
+  if (error instanceof ChannelAUnavailableError || error instanceof BrowserControlTransportError) {
     return {
       reason: "provider_unavailable",
       status: 503,
@@ -1063,6 +1266,8 @@ export function channelAOperationFailureDiagnostic(
   if (
     error instanceof SandboxResumeIdentityMismatchError ||
     error instanceof SandboxResumeIdentityUnavailableError ||
+    error instanceof RoutingWorkspaceRootChangedError ||
+    error instanceof SelfhostedWorkspaceRootChangedError ||
     (error instanceof HTTPException && error.status === 409)
   ) {
     return {
@@ -1091,6 +1296,16 @@ export function channelAOperationFailureDiagnostic(
       errorCode: "sandbox_channel_a_operation_failed",
     };
   }
+  // An authority fence is a deliberate rejection, not a fault: keep it out of
+  // the unexpected-failure signal that operators page on.
+  const authorityFence = workspaceMutationAuthorityFenceCode(error);
+  if (authorityFence === "authority_revoked" || authorityFence === "authority_unattributed") {
+    return {
+      reason: "request_rejected",
+      status: authorityFence === "authority_revoked" ? 403 : 409,
+      errorCode: "sandbox_channel_a_operation_failed",
+    };
+  }
   return {
     reason: "unexpected",
     status: 500,
@@ -1098,7 +1313,8 @@ export function channelAOperationFailureDiagnostic(
   };
 }
 
-function observeChannelAOperationFailure(
+/** Exported for the telemetry contract tests; not a route surface. */
+export function observeChannelAOperationFailure(
   services: ChannelAServices,
   input: {
     workspaceId: string;
@@ -1111,6 +1327,13 @@ function observeChannelAOperationFailure(
   },
 ): void {
   if (!services.observability) return;
+  // A writer with no recorded authority is a compatibility lane, not a fault:
+  // count the lane itself so an operator can see whether it is still live,
+  // separately from the structural failure signal operators page on below.
+  // Lane name only - never the workspace, sandbox group, backend, or operation.
+  if (workspaceMutationAuthorityFenceCode(input.error) === "authority_unattributed") {
+    recordTenancyCompatibilityLaneUse(services.observability, "workspace_writer_unattributed");
+  }
   const diagnostic = channelAOperationFailureDiagnostic(input.error, input.waitSignal);
   const attributes = {
     sandboxLeaseKey: sandboxLeaseTelemetryKey(input.workspaceId, input.sandboxGroupId),

package/src/sandbox/connection-authority.ts

@@ -0,0 +1,3 @@
+/** Five-second heartbeats get four missed intervals before another daemon may
+ * claim the enrollment. Server time is authoritative. */
+export const AGENT_CONNECTION_LEASE_MS = 20_000;

package/src/sandbox/enrollment.ts

@@ -172,7 +172,7 @@ export async function startDeviceEnrollment(
     throw lastError instanceof Error ? lastError : new Error("failed to start device enrollment");
   }
 
-  const base = input.verificationOrigin.replace(/\/$/, "");
+  const base = input.verificationOrigin.replace(/\/+$/, "");
   const verificationUri = `${base}/device`;
   const verificationUriComplete = `${verificationUri}?user_code=${encodeURIComponent(request.userCode)}`;
   return {
@@ -193,12 +193,18 @@ export async function approveDeviceEnrollment(
   input: {
     accountId: string;
     workspaceId: string;
+    scope?: "organization" | "workspace" | "user";
+    allowOrganization?: boolean;
     userCode: string;
     allowScreenControl: boolean;
     approvedBySubjectId: string;
     approvedBySubjectLabel?: string | null;
   },
-): Promise<{ enrollmentId: string; sandboxId: string; allowScreenControl: boolean } | null> {
+): Promise<{
+  enrollmentId: string;
+  sandboxId: string;
+  allowScreenControl: boolean;
+} | null> {
   const { db } = services;
   const pending = await getPendingDeviceEnrollmentRequestByUserCode(
     db,
@@ -213,6 +219,8 @@ export async function approveDeviceEnrollment(
   const result = await approveDeviceEnrollmentRequest(db, {
     accountId: input.accountId,
     workspaceId: input.workspaceId,
+    ...(input.scope ? { scope: input.scope } : {}),
+    allowOrganization: input.allowOrganization === true,
     requestId: pending.id,
     allowScreenControl: input.allowScreenControl,
     approvedBySubjectId: input.approvedBySubjectId,
@@ -291,7 +299,11 @@ export async function denyDeviceEnrollment(
  *  mirror poll's "disabled" handling. The token value is NEVER logged. */
 export async function mintEnrollToken(
   services: EnrollmentServices,
-  input: { accountId: string; workspaceId: string; allowScreenControl: boolean },
+  input: {
+    accountId: string;
+    workspaceId: string;
+    allowScreenControl: boolean;
+  },
 ): Promise<MintEnrollTokenResponse | null> {
   const { settings } = services;
   const secret = resolveEnrollmentSigningSecret(settings);