@opengeni/api-router 0.5.3 → 0.5.5

package/src/sandbox/channel-a.ts

@@ -17,7 +17,13 @@
 // IMPORT DISCIPLINE: sandbox symbols come ONLY from @opengeni/runtime/sandbox
 // (the agent-loop-free leaf) — enforced by sandbox-access-import-guard.test.ts.
 
-import { applyGitAuthPointerEnvironment, hasGitHubRepositorySelection, stableSandboxEnvironmentForRun, type Settings } from "@opengeni/config";
+import {
+  applyGitAuthPointerEnvironment,
+  hasGitCredentialRepositorySelection,
+  hasGitHubRepositorySelection,
+  stableSandboxEnvironmentForRun,
+  type Settings,
+} from "@opengeni/config";
 import { githubAppBotIdentity } from "@opengeni/github";
 import type { Session } from "@opengeni/contracts";
 import {
@@ -26,9 +32,9 @@ import {
   failWarmingToCold,
   getSandboxSessionEnvelope,
   loadWorkspaceEnvironmentForRun,
+  markWarmLeaseInstanceLost,
   readLease,
   releaseLeaseHolder,
-  SandboxLeaseSupersededError,
   type Database,
   type LeaseSnapshot,
 } from "@opengeni/db";
@@ -37,6 +43,8 @@ import { HTTPException } from "hono/http-exception";
 
 import {
   establishSandboxSessionFromEnvelope,
+  isProviderSandboxNotFoundError,
+  SandboxResumeStateUnavailableError,
   serializeEstablishedSandboxEnvelope,
   SandboxChannelAService,
   ChannelAConflictError,
@@ -84,7 +92,7 @@ export async function withChannelA<T>(
   fn: (handle: ChannelAHandle) => Promise<T>,
 ): Promise<T> {
   const { db, settings, bus } = services;
-  const { accountId, workspaceId, session, subjectId } = ctx;
+  const { accountId, workspaceId, session } = ctx;
 
   if (session.sandboxBackend === "none") {
     throw new HTTPException(409, { message: "sandbox not available" });
@@ -116,11 +124,14 @@ export async function withChannelA<T>(
     backend: session.sandboxBackend,
     os: session.sandboxOs,
     leaseTtlMs,
+    warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
   });
 
   if (acquired.role === "fenced") {
     await release();
-    throw new HTTPException(409, { message: `sandbox lease superseded (epoch ${acquired.lease.leaseEpoch}); retry` });
+    throw new HTTPException(409, {
+      message: `sandbox lease superseded (epoch ${acquired.lease.leaseEpoch}); retry`,
+    });
   }
 
   let established: EstablishedSandboxSession | undefined;
@@ -136,13 +147,26 @@ export async function withChannelA<T>(
     // the SESSION's backend (the establish below passes backendOverride:
     // session.sandboxBackend, and HOME/token-file/askpass are backend-derived),
     // NOT the deployment default — mirrors sessionAttachEnvironment.
-    const workspaceEnvironment = await loadWorkspaceEnvironmentForRun(db, settings, workspaceId, session.environmentId);
-    const settingsForSession = session.sandboxBackend !== settings.sandboxBackend
-      ? { ...settings, sandboxBackend: session.sandboxBackend }
-      : settings;
-    const environment = stableSandboxEnvironmentForRun(settingsForSession, workspaceEnvironment?.values ?? {}, { workspaceId });
-    if (hasGitHubRepositorySelection(session.resources)) {
-      applyGitAuthPointerEnvironment(environment, githubAppBotIdentity(settings));
+    const workspaceEnvironment = await loadWorkspaceEnvironmentForRun(
+      db,
+      settings,
+      workspaceId,
+      session.environmentId,
+    );
+    const settingsForSession =
+      session.sandboxBackend !== settings.sandboxBackend
+        ? { ...settings, sandboxBackend: session.sandboxBackend }
+        : settings;
+    const environment = stableSandboxEnvironmentForRun(
+      settingsForSession,
+      workspaceEnvironment?.values ?? {},
+      { workspaceId },
+    );
+    if (hasGitCredentialRepositorySelection(session.resources)) {
+      applyGitAuthPointerEnvironment(
+        environment,
+        hasGitHubRepositorySelection(session.resources) ? githubAppBotIdentity(settings) : null,
+      );
     }
 
     if (acquired.role === "spawner") {
@@ -161,18 +185,22 @@ export async function withChannelA<T>(
       try {
         established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
           sessionId: session.id,
+          recovery: "create-or-restore",
           backendOverride: session.sandboxBackend,
           environment,
         });
       } catch (error) {
         await failWarmingToCold(db, { accountId, workspaceId, sandboxGroupId, expectedEpoch });
-        throw new HTTPException(409, { message: `sandbox not available (${error instanceof Error ? error.message : "spawn failed"})` });
+        throw new HTTPException(409, {
+          message: `sandbox not available (${error instanceof Error ? error.message : "spawn failed"})`,
+        });
       }
       // Persist the LIVE box as the lease's resume_state so the NEXT op resumes
       // this box by id rather than cold-creating a rival (the box-churn the
       // prove-it surfaced). Fall back to the session envelope when serialize is
       // unavailable.
-      const resumeEnvelope = (await serializeEstablishedSandboxEnvelope(established)) ?? envelope ?? null;
+      const resumeEnvelope =
+        (await serializeEstablishedSandboxEnvelope(established)) ?? envelope ?? null;
       const committed = await commitWarmingToWarm(db, {
         accountId,
         workspaceId,
@@ -185,20 +213,59 @@ export async function withChannelA<T>(
         leaseTtlMs,
       });
       if (!committed.committed || !committed.lease) {
-        throw new HTTPException(409, { message: `sandbox lease superseded (epoch ${expectedEpoch}); retry` });
+        throw new HTTPException(409, {
+          message: `sandbox lease superseded (epoch ${expectedEpoch}); retry`,
+        });
       }
       leaseSnapshot = committed.lease;
     } else {
       // ATTACHED / REARMED: the box is live. Read the lease to get the
       // authoritative resume_state, then resume by id for this op.
       const live = await readLease(db, workspaceId, sandboxGroupId);
-      if (live) leaseSnapshot = live;
-      const resumeEnvelope = leaseSnapshot.resumeState ?? envelope;
-      established = await establishSandboxSessionFromEnvelope(settings, resumeEnvelope, {
-        sessionId: session.id,
-        backendOverride: session.sandboxBackend,
-        environment,
-      });
+      if (
+        !live ||
+        live.liveness !== "warm" ||
+        live.leaseEpoch !== acquired.lease.leaseEpoch ||
+        live.instanceId === null
+      ) {
+        throw new HTTPException(409, {
+          message: `sandbox lease is not attachable; retry`,
+        });
+      }
+      leaseSnapshot = live;
+      try {
+        established = await establishSandboxSessionFromEnvelope(settings, live.resumeState, {
+          sessionId: session.id,
+          recovery: "resume-only",
+          backendOverride: session.sandboxBackend,
+          environment,
+        });
+      } catch (error) {
+        if (
+          !(error instanceof SandboxResumeStateUnavailableError) &&
+          !isProviderSandboxNotFoundError(session.sandboxBackend, error)
+        ) {
+          throw error;
+        }
+        const marked = await markWarmLeaseInstanceLost(db, {
+          accountId,
+          workspaceId,
+          sandboxGroupId,
+          expectedEpoch: live.leaseEpoch,
+          expectedInstanceId: live.instanceId,
+        });
+        if (marked.status === "marked") {
+          await appendAndPublishEvents(db, bus, workspaceId, session.id, [
+            {
+              type: "sandbox.box.lost",
+              payload: { sandboxId: live.instanceId },
+            },
+          ]);
+        }
+        throw new HTTPException(409, {
+          message: `sandbox instance was lost; retry to restore it`,
+        });
+      }
     }
 
     const emit = async (events: { type: string; payload: unknown }[]): Promise<void> => {
@@ -220,7 +287,11 @@ export async function withChannelA<T>(
     // default, or a swapped-to selfhosted machine). With the flag off the
     // established group session is used unchanged.
     const routedSession = routingEnabled(settings)
-      ? wrapChannelABoxWithRouting({ db, settings, bus }, { workspaceId, sessionId: session.id }, established).session
+      ? wrapChannelABoxWithRouting(
+          { db, settings, bus },
+          { workspaceId, sessionId: session.id },
+          established,
+        ).session
       : established.session;
 
     const service = new SandboxChannelAService({
@@ -242,10 +313,14 @@ export async function withChannelA<T>(
  *  already-HTTPException unchanged. */
 export function mapChannelAError(error: unknown): unknown {
   if (error instanceof HTTPException) return error;
-  if (error instanceof ChannelAValidationError) return new HTTPException(400, { message: error.message });
-  if (error instanceof ChannelANotFoundError) return new HTTPException(404, { message: error.message });
-  if (error instanceof ChannelAConflictError) return new HTTPException(409, { message: error.message });
-  if (error instanceof ChannelAUnsupportedError) return new HTTPException(409, { message: error.message });
+  if (error instanceof ChannelAValidationError)
+    return new HTTPException(400, { message: error.message });
+  if (error instanceof ChannelANotFoundError)
+    return new HTTPException(404, { message: error.message });
+  if (error instanceof ChannelAConflictError)
+    return new HTTPException(409, { message: error.message });
+  if (error instanceof ChannelAUnsupportedError)
+    return new HTTPException(409, { message: error.message });
   return error;
 }
 
@@ -258,7 +333,9 @@ export function mapChannelAError(error: unknown): unknown {
 // destroyed the box mid-flight, so a subsequent fs.read/git/exec hit a different
 // (cold-restored) box and 404'd. We DO NOT close the session; only the reaper
 // (provider stop at refcount 0) terminates a box.
-async function dropEstablishedHandle(established: EstablishedSandboxSession | undefined): Promise<void> {
+async function dropEstablishedHandle(
+  established: EstablishedSandboxSession | undefined,
+): Promise<void> {
   // No-op beyond dropping the reference: the lease owns lifecycle, the reaper
   // owns teardown. Never session.close()/terminate() a non-owned handle here.
   void established;

package/src/sandbox/enrollment.ts

@@ -199,7 +199,11 @@ export async function approveDeviceEnrollment(
   },
 ): Promise<{ enrollmentId: string; sandboxId: string; allowScreenControl: boolean } | null> {
   const { db } = services;
-  const pending = await getPendingDeviceEnrollmentRequestByUserCode(db, input.workspaceId, input.userCode);
+  const pending = await getPendingDeviceEnrollmentRequestByUserCode(
+    db,
+    input.workspaceId,
+    input.userCode,
+  );
   if (!pending) {
     return null;
   }
@@ -240,7 +244,9 @@ export async function lookupDeviceEnrollment(
 
 /** Project a resolved pending record to the presentational lookup response (no
  *  secrets, no device_code) the approve screen (EnrollmentConsent) renders. */
-export function toLookupResponse(record: DeviceEnrollmentRequestRecord): DeviceEnrollmentLookupResponse {
+export function toLookupResponse(
+  record: DeviceEnrollmentRequestRecord,
+): DeviceEnrollmentLookupResponse {
   return {
     workspaceId: record.workspaceId,
     userCode: record.userCode,
@@ -263,7 +269,11 @@ export async function denyDeviceEnrollment(
   input: { accountId: string; workspaceId: string; userCode: string },
 ): Promise<{ denied: boolean }> {
   const { db } = services;
-  const pending = await getPendingDeviceEnrollmentRequestByUserCode(db, input.workspaceId, input.userCode);
+  const pending = await getPendingDeviceEnrollmentRequestByUserCode(
+    db,
+    input.workspaceId,
+    input.userCode,
+  );
   if (!pending) {
     return { denied: false };
   }

package/src/sandbox/machines.ts

@@ -31,11 +31,7 @@ import {
   type MachineMetricsRow,
 } from "@opengeni/db";
 import type { EventBus } from "@opengeni/events";
-import {
-  MachineView,
-  MetricSample,
-  type MachinesResponse,
-} from "@opengeni/contracts";
+import { MachineView, MetricSample, type MachinesResponse } from "@opengeni/contracts";
 import {
   NatsControlRpc,
   selfhostedLiveness,
@@ -95,7 +91,11 @@ async function probeEnrollment(
   services: MachinesServices,
   workspaceId: string,
   enrollment: EnrollmentRecord,
-): Promise<{ state: "online" | "reconnecting" | "offline"; consented: boolean; hasDisplay: boolean }> {
+): Promise<{
+  state: "online" | "reconnecting" | "offline";
+  consented: boolean;
+  hasDisplay: boolean;
+}> {
   const { settings, bus } = services;
   let probeResponded = false;
   if (enrollment.status === "active") {
@@ -119,6 +119,8 @@ async function probeEnrollment(
       allowScreenControl: enrollment.allowScreenControl,
       hasDisplay: enrollment.hasDisplay,
       lastSeenAt: enrollment.lastSeenAt,
+      wentOfflineAt: enrollment.wentOfflineAt,
+      wentOfflineReason: enrollment.wentOfflineReason,
     },
     probeResponded,
   });
@@ -186,25 +188,27 @@ export async function listMachines(
   // null active pointer routes to. Only present in an in-session view.
   if (session) {
     const groupActive = activeSandboxId === null;
-    machines.push(MachineView.parse({
-      sandboxId: session.sandboxGroupId,
-      enrollmentId: null,
-      name: "session sandbox",
-      kind: session.sandboxBackend === "selfhosted" ? "selfhosted" : "modal",
-      state: "online",
-      active: groupActive,
-      isSessionGroup: true,
-      // The Modal group box is a cloud Linux box; its precise OS/arch is not
-      // surfaced as a metric, so the dashboard shows the canonical linux/x86_64.
-      os: "linux",
-      arch: "x86_64",
-      hasDisplay: false,
-      desktopUnavailableReason: null,
-      allowScreenControl: false,
-      sharedSessionCount: 1,
-      lastSeenAt: null,
-      metrics: null,
-    }));
+    machines.push(
+      MachineView.parse({
+        sandboxId: session.sandboxGroupId,
+        enrollmentId: null,
+        name: "session sandbox",
+        kind: session.sandboxBackend === "selfhosted" ? "selfhosted" : "modal",
+        state: "online",
+        active: groupActive,
+        isSessionGroup: true,
+        // The Modal group box is a cloud Linux box; its precise OS/arch is not
+        // surfaced as a metric, so the dashboard shows the canonical linux/x86_64.
+        os: "linux",
+        arch: "x86_64",
+        hasDisplay: false,
+        desktopUnavailableReason: null,
+        allowScreenControl: false,
+        sharedSessionCount: 1,
+        lastSeenAt: null,
+        metrics: null,
+      }),
+    );
   }
 
   // The workspace's enrolled selfhosted machines. One bulk metrics read joined
@@ -216,42 +220,47 @@ export async function listMachines(
   ]);
   const enrollmentById = new Map(enrollments.map((e) => [e.id, e]));
 
-  for (const sandbox of sandboxes) {
-    if (sandbox.kind !== "selfhosted" || !sandbox.enrollmentId) {
-      continue;
-    }
-    const enrollment = enrollmentById.get(sandbox.enrollmentId) ?? null;
-    if (!enrollment) {
-      continue;
-    }
-    const probe = await probeEnrollment(services, workspaceId, enrollment);
-    const state = machineStateFor(probe.state, probe.hasDisplay);
+  const machineViews = await Promise.all(
+    sandboxes.map(async (sandbox): Promise<MachineView | null> => {
+      if (sandbox.kind !== "selfhosted" || !sandbox.enrollmentId) {
+        return null;
+      }
+      const enrollment = enrollmentById.get(sandbox.enrollmentId) ?? null;
+      if (!enrollment) {
+        return null;
+      }
+      const [probe, lease] = await Promise.all([
+        probeEnrollment(services, workspaceId, enrollment),
+        readLease(db, workspaceId, sandbox.id),
+      ]);
+      const state = machineStateFor(probe.state, probe.hasDisplay);
 
-    // sharedSessionCount = the lease refcount for this machine's group. The
-    // selfhosted sandbox id IS the lease group key (maxSandboxes:1, N sessions
-    // share via refcount). No lease yet → 0 sessions sharing.
-    const lease = await readLease(db, workspaceId, sandbox.id);
-    const sharedSessionCount = lease?.refcount ?? 0;
+      // sharedSessionCount = the lease refcount for this machine's group. The
+      // selfhosted sandbox id IS the lease group key (maxSandboxes:1, N sessions
+      // share via refcount). No lease yet → 0 sessions sharing.
+      const sharedSessionCount = lease?.refcount ?? 0;
 
-    const metricsRow = metricsByEnrollment.get(enrollment.id) ?? null;
-    machines.push(MachineView.parse({
-      sandboxId: sandbox.id,
-      enrollmentId: enrollment.id,
-      name: sandbox.name,
-      kind: "selfhosted",
-      state,
-      active: activeSandboxId === sandbox.id,
-      isSessionGroup: false,
-      os: enrollment.os,
-      arch: enrollment.arch,
-      hasDisplay: enrollment.hasDisplay,
-      desktopUnavailableReason: enrollment.desktopUnavailableReason,
-      allowScreenControl: enrollment.allowScreenControl,
-      sharedSessionCount,
-      lastSeenAt: enrollment.lastSeenAt,
-      metrics: metricsRow ? metricRowToSample(metricsRow) : null,
-    }));
-  }
+      const metricsRow = metricsByEnrollment.get(enrollment.id) ?? null;
+      return MachineView.parse({
+        sandboxId: sandbox.id,
+        enrollmentId: enrollment.id,
+        name: sandbox.name,
+        kind: "selfhosted",
+        state,
+        active: activeSandboxId === sandbox.id,
+        isSessionGroup: false,
+        os: enrollment.os,
+        arch: enrollment.arch,
+        hasDisplay: enrollment.hasDisplay,
+        desktopUnavailableReason: enrollment.desktopUnavailableReason,
+        allowScreenControl: enrollment.allowScreenControl,
+        sharedSessionCount,
+        lastSeenAt: enrollment.lastSeenAt,
+        metrics: metricsRow ? metricRowToSample(metricsRow) : null,
+      });
+    }),
+  );
+  machines.push(...machineViews.filter((machine): machine is MachineView => machine !== null));
 
   return { activeSandboxId, activeEpoch, machines };
 }