@opengeni/runtime 0.6.0 → 0.7.0

package/src/sandbox/routing/routing-session.ts

@@ -28,6 +28,8 @@
 // `@opengeni/db`.
 
 import type { ExposedPortEndpoint } from "../stream-port";
+import { SelfhostedControlError } from "../selfhosted/control-rpc";
+import { renderSelfhostedFault } from "../selfhosted/fault-rendering";
 
 /** The per-session active-sandbox pointer the proxy re-reads on every op. Mirror
  *  of `@opengeni/db`'s `ActiveSandboxPointer` (structural, so the leaf does not
@@ -74,7 +76,13 @@ export interface RoutableBackendSession {
   // `event` is kept `unknown` (mirroring the interface's structural style + avoiding
   // a proto import into the leaf); the SelfhostedSession takes `DesktopInputRequest["event"]`.
   desktopInput?(event: unknown): Promise<void>;
-  screenshot?(): Promise<{ png: Uint8Array; width: number; height: number; nativeWidth: number; nativeHeight: number }>;
+  screenshot?(): Promise<{
+    png: Uint8Array;
+    width: number;
+    height: number;
+    nativeWidth: number;
+    nativeHeight: number;
+  }>;
 }
 
 /** The resolved active backend for an epoch: the live session + the sandbox id it
@@ -148,10 +156,12 @@ function isFenceError(error: unknown): boolean {
   if ((error as { fenced?: unknown }).fenced === true) {
     return true;
   }
-  const name = typeof (error as { name?: unknown }).name === "string" ? (error as { name: string }).name : "";
-  const message = error instanceof Error ? error.message : String((error as { message?: unknown }).message ?? "");
+  const name =
+    typeof (error as { name?: unknown }).name === "string" ? (error as { name: string }).name : "";
+  const message =
+    error instanceof Error ? error.message : String((error as { message?: unknown }).message ?? "");
   const haystack = `${name} ${message}`.toLowerCase();
-  return haystack.includes("fenced") || haystack.includes("epoch") && haystack.includes("super");
+  return haystack.includes("fenced") || (haystack.includes("epoch") && haystack.includes("super"));
 }
 
 /**
@@ -172,9 +182,17 @@ function isFenceError(error: unknown): boolean {
 export class RoutingSandboxSession implements RoutableBackendSession {
   private readonly deps: RoutingSandboxSessionDeps;
   private readonly maxFenceRetries: number;
-  // The per-epoch resolved-backend cache. Keyed by activeEpoch: a swap bumps the
-  // epoch, invalidating the cache so the NEXT op re-resolves the new backend.
+  // The resolved-backend cache. Keyed by the FULL pointer tuple
+  // `(activeEpoch, activeSandboxId)` — NOT the epoch alone. A swap bumps the epoch,
+  // but a pointer can also change its target id WITHOUT an epoch bump: the
+  // `sessions.active_sandbox_id` FK is `ON DELETE SET NULL`, so a cascade that
+  // deletes the pointed-at sandbox row nulls the id at the SAME epoch. Keying on the
+  // epoch alone would then keep serving the deleted/stale backend for that epoch
+  // (issue #341 §5.2 — a swap-free route to the Shape-3 symptom). Keying on the tuple
+  // makes any target change — epoch-bumped OR not — invalidate the cache so the next
+  // op re-resolves (and, for a null id, re-resolves the session HOME).
   private cachedEpoch: number | undefined;
+  private cachedSandboxId: string | null | undefined;
   private cached: ResolvedActiveBackend | undefined;
   // The last-resolved backend, exposed via the `state` getter (a method-free read
   // of the active backend's `state`). Updated on every resolve.
@@ -189,7 +207,13 @@ export class RoutingSandboxSession implements RoutableBackendSession {
   // that cannot serve them. So the constructor assigns them ONLY when the
   // construction-time default backend actually implements the native surface (below).
   desktopInput?: (event: unknown) => Promise<void>;
-  screenshot?: () => Promise<{ png: Uint8Array; width: number; height: number; nativeWidth: number; nativeHeight: number }>;
+  screenshot?: () => Promise<{
+    png: Uint8Array;
+    width: number;
+    height: number;
+    nativeWidth: number;
+    nativeHeight: number;
+  }>;
 
   constructor(deps: RoutingSandboxSessionDeps) {
     this.deps = deps;
@@ -252,16 +276,34 @@ export class RoutingSandboxSession implements RoutableBackendSession {
    */
   private async resolve(): Promise<ResolvedActiveBackend> {
     const pointer = await this.deps.readPointer();
-    if (this.cachedEpoch === pointer.activeEpoch && this.cached) {
+    if (
+      this.cachedEpoch === pointer.activeEpoch &&
+      this.cachedSandboxId === pointer.activeSandboxId &&
+      this.cached
+    ) {
       return this.cached;
     }
     const fromEpoch = this.cachedEpoch ?? pointer.activeEpoch;
     const resolved = await this.deps.resolveActiveBackend(pointer);
+    // Re-entrancy guard: a resolver that returns THIS proxy as the active backend
+    // makes every op (exec/readFile/…) dispatch back into resolve() -> the same
+    // backend -> forever (a silent async infinite recursion that HANGS the turn,
+    // not a stack overflow). Fail loud instead — a wiring bug must surface as a
+    // legible error, never a hung turn.
+    if ((resolved.session as unknown) === this) {
+      throw new Error(
+        "RoutingSandboxSession.resolveActiveBackend returned the proxy itself as the active backend (re-entrancy) — the resolver must return the underlying box session, not the routing proxy.",
+      );
+    }
     this.cachedEpoch = pointer.activeEpoch;
+    this.cachedSandboxId = pointer.activeSandboxId;
     this.cached = resolved;
     this.lastResolved = resolved;
     this.deps.onTransition?.({
-      type: this.cachedEpoch !== undefined && fromEpoch !== pointer.activeEpoch ? "epoch-changed" : "resolved",
+      type:
+        this.cachedEpoch !== undefined && fromEpoch !== pointer.activeEpoch
+          ? "epoch-changed"
+          : "resolved",
       fromEpoch,
       toEpoch: pointer.activeEpoch,
       sandboxId: resolved.sandboxId,
@@ -281,7 +323,10 @@ export class RoutingSandboxSession implements RoutableBackendSession {
    * A non-fence error propagates immediately (it is a real op failure, not a swap
    * race).
    */
-  private async dispatch<T>(op: string, fn: (session: RoutableBackendSession) => Promise<T>): Promise<T> {
+  private async dispatch<T>(
+    op: string,
+    fn: (session: RoutableBackendSession) => Promise<T>,
+  ): Promise<T> {
     let attempt = 0;
     let lastError: unknown;
     while (attempt <= this.maxFenceRetries) {
@@ -297,6 +342,7 @@ export class RoutingSandboxSession implements RoutableBackendSession {
         // active sandbox (the fenced-retry role). Bounded by maxFenceRetries.
         lastError = error;
         this.cachedEpoch = undefined;
+        this.cachedSandboxId = undefined;
         this.cached = undefined;
         this.deps.onTransition?.({
           type: "fenced-retry",
@@ -313,6 +359,31 @@ export class RoutingSandboxSession implements RoutableBackendSession {
     throw lastError ?? new Error(`routing op "${op}" exhausted fence retries`);
   }
 
+  /**
+   * The failure-visibility boundary for the `exec_command` SDK capability tool — the
+   * dominant fault surface, and the one whose thrown `SelfhostedControlError` reaches
+   * the model wrapped by the SDK's generic tool-error function as
+   * "…Please try again. Error: …" — actively wrong for a machine that is offline, a
+   * consent that is not granted, or an oversized reply. Since the SDK closure-captures
+   * its `errorFunction` (there is no seam to attach one to its internally-built tools),
+   * we render the fault into the doctrine's four fields HERE and return it as the
+   * tool's string result — legible, in-band, and free of the misleading wrapper.
+   * (`apply_patch` already surfaces `error.message` via the SDK's own catch; the
+   * skills / `view_image` tools consume their session methods internally, so they are
+   * rendered elsewhere or left to their own SDK renderers — not this string boundary.)
+   *
+   * A FENCE error is re-thrown, NEVER rendered: `dispatch` already retries it against
+   * a re-resolved backend, and a fence that escapes retries is a routing condition the
+   * turn handles, not a model-facing fault. Any non-selfhosted error (a Modal fault, a
+   * `RoutingUnsupportedError`) is re-thrown unchanged.
+   */
+  private renderSelfhostedFaultOrThrow(error: unknown): string {
+    if (error instanceof SelfhostedControlError && !error.fenced) {
+      return renderSelfhostedFault(error);
+    }
+    throw error;
+  }
+
   // ── The forwarded structural surface ──────────────────────────────────────
   // Every method is PRESENT on the proxy (the SDK binds presence once) and
   // dispatches to the active backend at call-time. A missing backend method
@@ -332,16 +403,22 @@ export class RoutingSandboxSession implements RoutableBackendSession {
   }
 
   async execCommand(args: unknown): Promise<string> {
-    return this.dispatch("execCommand", async (s) => {
-      if (s.execCommand) {
-        return s.execCommand(args);
-      }
-      if (s.exec) {
-        const r = (await s.exec(args)) as { stdout?: string; output?: string };
-        return r.stdout ?? r.output ?? "";
-      }
-      throw new RoutingUnsupportedError("execCommand", this.cached?.kind ?? "unknown");
-    });
+    try {
+      return await this.dispatch("execCommand", async (s) => {
+        if (s.execCommand) {
+          return s.execCommand(args);
+        }
+        if (s.exec) {
+          const r = (await s.exec(args)) as { stdout?: string; output?: string };
+          return r.stdout ?? r.output ?? "";
+        }
+        throw new RoutingUnsupportedError("execCommand", this.cached?.kind ?? "unknown");
+      });
+    } catch (error) {
+      // Render a terminal selfhosted fault as the tool's result (four fields, correct
+      // verdict) instead of letting the SDK mislabel it "Please try again".
+      return this.renderSelfhostedFaultOrThrow(error);
+    }
   }
 
   async writeStdin(args: unknown): Promise<string> {
@@ -417,12 +494,39 @@ export class RoutingSandboxSession implements RoutableBackendSession {
     return Boolean(s?.supportsPty?.());
   }
 
-  /** createEditor is a synchronous factory in the SDK surface; it binds to the
-   *  last-resolved backend's editor (or the default backend before the first op).
-   *  Returns undefined when the active backend has no editor (channel-a falls back
-   *  to its exec-based write path). */
+  /** createEditor is a synchronous factory in the SDK surface. The SDK's filesystem
+   *  capability calls it ONCE at tool-BIND time — `FilesystemCapability.tools()`,
+   *  every turn, before any tool runs — and throws "Filesystem sandbox sessions must
+   *  provide createEditor()" if it returns falsy. When a backend is already resolved
+   *  (eager/selfhosted routing) we bind to its editor directly, byte-for-byte as
+   *  before. But under LAZY provisioning the backend is not established yet
+   *  (defaultResolved is the synthetic unprovisioned session with no editor), so a
+   *  direct delegate returns undefined and every lazy turn would die at bind. Return a
+   *  LAZY EDITOR PROXY instead: a non-null editor whose async ops resolve the active
+   *  backend (establishing the box on first use, via `dispatch`) and delegate to its
+   *  real editor — mirroring how this proxy defers exec/readFile. */
   createEditor(runAs?: string): unknown {
-    return (this.lastResolved ?? this.deps.defaultResolved)?.session.createEditor?.(runAs);
+    const eager = (this.lastResolved ?? this.deps.defaultResolved)?.session.createEditor?.(runAs);
+    if (eager) {
+      return eager;
+    }
+    const op =
+      (name: "createFile" | "updateFile" | "deleteFile") =>
+      (operation: unknown, context?: unknown): Promise<unknown> =>
+        this.dispatch(`editor.${name}`, async (s) => {
+          const editor = s.createEditor?.(runAs) as
+            | Record<string, (operation: unknown, context?: unknown) => Promise<unknown>>
+            | undefined;
+          if (!editor?.[name]) {
+            throw new RoutingUnsupportedError(`editor.${name}`, this.cached?.kind ?? "unknown");
+          }
+          return editor[name](operation, context);
+        });
+    return {
+      createFile: op("createFile"),
+      updateFile: op("updateFile"),
+      deleteFile: op("deleteFile"),
+    };
   }
 
   async resolveExposedPort(port: number): Promise<ExposedPortEndpoint> {

package/src/sandbox/select.ts

@@ -165,7 +165,14 @@ export function negotiateCapabilities(ctx: NegotiationContext): SessionCapabilit
 
   const fileSystem = (() => {
     if (osReason) {
-      return { available: false, readOnly: true, root: descriptor.workspaceRoot, pathSep: "/" as const, treeMode: "lazy" as const, reason: osReason };
+      return {
+        available: false,
+        readOnly: true,
+        root: descriptor.workspaceRoot,
+        pathSep: "/" as const,
+        treeMode: "lazy" as const,
+        reason: osReason,
+      };
     }
     const cap = descriptor.capabilities.FileSystem;
     return {
@@ -181,10 +188,26 @@ export function negotiateCapabilities(ctx: NegotiationContext): SessionCapabilit
   const terminal = (() => {
     const cap = descriptor.capabilities.Terminal;
     if (osReason) {
-      return { transport: null, ptyCapable: false, shell: "/bin/bash", url: null, token: null, expiresAt: null, reason: osReason };
+      return {
+        transport: null,
+        ptyCapable: false,
+        shell: "/bin/bash",
+        url: null,
+        token: null,
+        expiresAt: null,
+        reason: osReason,
+      };
     }
     if (!cap.available) {
-      return { transport: null, ptyCapable: false, shell: "/bin/bash", url: null, token: null, expiresAt: null, reason: "backend_unsupported" as const };
+      return {
+        transport: null,
+        ptyCapable: false,
+        shell: "/bin/bash",
+        url: null,
+        token: null,
+        expiresAt: null,
+        reason: "backend_unsupported" as const,
+      };
     }
     // The REAL PTY (ttyd pty-ws) rides the SAME tunnel as the desktop, so it is
     // gated identically: a real-PTY backend (cap.pty), the terminal policy toggle
@@ -231,7 +254,11 @@ export function negotiateCapabilities(ctx: NegotiationContext): SessionCapabilit
     if (osReason) {
       return { available: false, repos: [], reason: osReason };
     }
-    return { available: cap.available, repos: [], reason: cap.available ? null : ("backend_unsupported" as const) };
+    return {
+      available: cap.available,
+      repos: [],
+      reason: cap.available ? null : ("backend_unsupported" as const),
+    };
   })();
 
   const desktop = (() => {
@@ -317,14 +344,32 @@ export function negotiateCapabilities(ctx: NegotiationContext): SessionCapabilit
   const recording = (() => {
     const cap = descriptor.capabilities.Recording;
     if (osReason) {
-      return { available: false, modes: [] as ("manual" | "on-turn" | "on-verify")[], codecs: [] as ("h264-mp4" | "vp9-webm")[], reason: osReason };
+      return {
+        available: false,
+        modes: [] as ("manual" | "on-turn" | "on-verify")[],
+        codecs: [] as ("h264-mp4" | "vp9-webm")[],
+        reason: osReason,
+      };
     }
     if (!cap.available) {
-      return { available: false, modes: [] as ("manual" | "on-turn" | "on-verify")[], codecs: [] as ("h264-mp4" | "vp9-webm")[], reason: descriptor.tier === "headless" ? ("tier_headless" as const) : ("backend_unsupported" as const) };
+      return {
+        available: false,
+        modes: [] as ("manual" | "on-turn" | "on-verify")[],
+        codecs: [] as ("h264-mp4" | "vp9-webm")[],
+        reason:
+          descriptor.tier === "headless"
+            ? ("tier_headless" as const)
+            : ("backend_unsupported" as const),
+      };
     }
     // Recording feasibility tracks desktop; policy-gate it the same way.
     if (!ctx.desktopEnabled) {
-      return { available: false, modes: [] as ("manual" | "on-turn" | "on-verify")[], codecs: [] as ("h264-mp4" | "vp9-webm")[], reason: "disabled_by_policy" as const };
+      return {
+        available: false,
+        modes: [] as ("manual" | "on-turn" | "on-verify")[],
+        codecs: [] as ("h264-mp4" | "vp9-webm")[],
+        reason: "disabled_by_policy" as const,
+      };
     }
     return {
       available: true,
@@ -345,7 +390,14 @@ export function negotiateCapabilities(ctx: NegotiationContext): SessionCapabilit
       return { available: false, readOnly, reason: osReason };
     }
     if (!desktopCapable) {
-      return { available: false, readOnly, reason: descriptor.tier === "headless" ? ("tier_headless" as const) : ("backend_unsupported" as const) };
+      return {
+        available: false,
+        readOnly,
+        reason:
+          descriptor.tier === "headless"
+            ? ("tier_headless" as const)
+            : ("backend_unsupported" as const),
+      };
     }
     if (!ctx.desktopEnabled || ctx.computerUseEnabled === false) {
       return { available: false, readOnly, reason: "disabled_by_policy" as const };

package/src/sandbox/selfhosted/capabilities.ts

@@ -44,6 +44,13 @@ export interface SelfhostedEnrollment {
   allowScreenControl: boolean;
   hasDisplay: boolean;
   lastSeenAt: string | null;
+  /** An un-cleared clean going-offline marker (the machine announced a typed
+   *  GoingOffline). When set, the derivation reads the machine OFFLINE immediately,
+   *  regardless of a still-fresh `lastSeenAt`. NULL ⇒ no pending goodbye. */
+  wentOfflineAt: string | null;
+  /** The typed reason string of the pending goodbye (rides alongside
+   *  `wentOfflineAt`; NULL when there is no un-cleared marker). */
+  wentOfflineReason: string | null;
 }
 
 /** The derived liveness state of a selfhosted machine (the online/offline/
@@ -72,6 +79,8 @@ export const SELFHOSTED_RECONNECT_WINDOW_MS = 30_000;
  * (stale / never seen).
  *
  *   - no enrollment / revoked         → offline (the machine isn't enrolled).
+ *   - un-cleared clean goodbye        → offline (an announced GoingOffline beats
+ *                                       last_seen aging AND a lingering probe).
  *   - probe responded                 → online.
  *   - probe missed, lastSeenAt recent → reconnecting (a transient blip).
  *   - probe missed, lastSeenAt stale  → offline.
@@ -89,6 +98,15 @@ export function selfhostedLiveness(input: {
   }
   const consented = enrollment.exposure === "whole-machine" && enrollment.allowScreenControl;
   const hasDisplay = enrollment.hasDisplay;
+  // A CLEAN going-offline the machine announced takes precedence over BOTH
+  // last_seen aging and a lingering probe: a machine that said "I'm stopping" must
+  // read offline immediately (so no new work is routed at it) until a NEWER
+  // liveness signal — a reconnect Hello or a fresher heartbeat — clears the marker
+  // in the row. `status: "revoked"` above still trumps this (a revoked machine is
+  // offline regardless). This is the #348 lease-waits-on-dead-detect fix.
+  if (enrollment.wentOfflineAt) {
+    return { state: "offline", consented, hasDisplay };
+  }
   if (input.probeResponded) {
     return { state: "online", consented, hasDisplay };
   }
@@ -130,8 +148,11 @@ export interface SelfhostedNegotiationInput {
  * with the online/offline/reconnecting/consent_required/display_unavailable cells
  * correctly decided. Async because it issues the liveness probe.
  */
-export async function negotiateSelfhostedCapabilities(input: SelfhostedNegotiationInput): Promise<SessionCapabilities> {
-  const probeResponded = input.probeResponded ?? (input.session ? await input.session.ping() : false);
+export async function negotiateSelfhostedCapabilities(
+  input: SelfhostedNegotiationInput,
+): Promise<SessionCapabilities> {
+  const probeResponded =
+    input.probeResponded ?? (input.session ? await input.session.ping() : false);
   const liveness = selfhostedLiveness({
     enrollment: input.enrollment,
     probeResponded,
@@ -142,7 +163,8 @@ export async function negotiateSelfhostedCapabilities(input: SelfhostedNegotiati
   // descriptor-shaped cells (FS/Terminal/Git/Desktop) negotiate as on a warm box
   // when online, and a cold box when not reachable (no live tunnel). The
   // selfhosted overlay below then stamps the selfhosted-specific reasons.
-  const baseLiveness: NegotiationContext["liveness"] = liveness.state === "online" ? "warm" : "cold";
+  const baseLiveness: NegotiationContext["liveness"] =
+    liveness.state === "online" ? "warm" : "cold";
   const base: NegotiationContext = {
     sessionId: input.sessionId,
     backend: "selfhosted",
@@ -152,7 +174,9 @@ export async function negotiateSelfhostedCapabilities(input: SelfhostedNegotiati
     desktopEnabled: input.desktopEnabled ?? true,
     terminalEnabled: input.terminalEnabled ?? true,
     computerUseEnabled: input.computerUseEnabled ?? true,
-    ...(input.desktopAcknowledged !== undefined ? { desktopAcknowledged: input.desktopAcknowledged } : {}),
+    ...(input.desktopAcknowledged !== undefined
+      ? { desktopAcknowledged: input.desktopAcknowledged }
+      : {}),
     ...(input.shared !== undefined ? { shared: input.shared } : {}),
     ...(input.sharedSessionIds !== undefined ? { sharedSessionIds: input.sharedSessionIds } : {}),
     ...(input.now ? { now: input.now } : {}),
@@ -166,11 +190,19 @@ export async function negotiateSelfhostedCapabilities(input: SelfhostedNegotiati
   // is the dominant degrade (like os_unsupported): an offline/reconnecting agent
   // knocks out every reachable capability with the single coherent reason.
   if (liveness.state !== "online") {
-    const reason: CapabilityUnavailableReason = liveness.state === "offline" ? "agent_offline" : "agent_reconnecting";
+    const reason: CapabilityUnavailableReason =
+      liveness.state === "offline" ? "agent_offline" : "agent_reconnecting";
     return {
       ...caps,
       FileSystem: { ...caps.FileSystem, available: false, readOnly: true, reason },
-      Terminal: { ...caps.Terminal, transport: null, url: null, token: null, expiresAt: null, reason },
+      Terminal: {
+        ...caps.Terminal,
+        transport: null,
+        url: null,
+        token: null,
+        expiresAt: null,
+        reason,
+      },
       Git: { ...caps.Git, available: false, reason },
       DesktopStream: {
         ...caps.DesktopStream,
@@ -209,22 +241,23 @@ export async function negotiateSelfhostedCapabilities(input: SelfhostedNegotiati
     const reason: CapabilityUnavailableReason = "display_unavailable";
     return {
       ...caps,
-      DesktopStream: caps.DesktopStream.transport !== null
-        ? {
-          ...caps.DesktopStream,
-          transport: null,
-          client: null,
-          mode: "read-only",
-          url: null,
-          token: null,
-          expiresAt: null,
-          requiresAcknowledgment: false,
-          acknowledged: false,
-          shared: false,
-          sharedSessionIds: [],
-          reason,
-        }
-        : caps.DesktopStream,
+      DesktopStream:
+        caps.DesktopStream.transport !== null
+          ? {
+              ...caps.DesktopStream,
+              transport: null,
+              client: null,
+              mode: "read-only",
+              url: null,
+              token: null,
+              expiresAt: null,
+              requiresAcknowledgment: false,
+              acknowledged: false,
+              shared: false,
+              sharedSessionIds: [],
+              reason,
+            }
+          : caps.DesktopStream,
       Recording: caps.Recording.available
         ? { ...caps.Recording, available: false, modes: [], codecs: [], reason }
         : caps.Recording,
@@ -240,9 +273,10 @@ export async function negotiateSelfhostedCapabilities(input: SelfhostedNegotiati
     // so no input is forwarded even if the base offered an interactive mode.
     return {
       ...caps,
-      DesktopStream: caps.DesktopStream.transport !== null
-        ? { ...caps.DesktopStream, mode: "read-only" }
-        : caps.DesktopStream,
+      DesktopStream:
+        caps.DesktopStream.transport !== null
+          ? { ...caps.DesktopStream, mode: "read-only" }
+          : caps.DesktopStream,
       ComputerUse: caps.ComputerUse.available
         ? { ...caps.ComputerUse, available: false, reason: "consent_required" }
         : caps.ComputerUse,