@opengeni/runtime 0.13.0 → 0.13.4

package/src/sandbox/provider-errors.ts

@@ -0,0 +1,175 @@
+/**
+ * Provider sandbox failure classification.
+ *
+ * This is deliberately structural and fail-closed. DNS failures often contain
+ * the word "not found" (`ENOTFOUND`) while gRPC status values are also used by
+ * unrelated SDKs and command routers. A bare numeric status (including gRPC's
+ * numeric 5) is therefore not enough to retire a provider instance. Only
+ * authoritative, non-transient evidence may retire a provider instance and
+ * license lease recovery.
+ */
+export type ProviderSandboxFailureKind = "not_found" | "transient_transport" | "other";
+
+export type ProviderSandboxFailure = {
+  kind: ProviderSandboxFailureKind;
+  /** Bounded, non-secret diagnostic suitable for logs and lifecycle events. */
+  diagnostic: string;
+};
+
+const GRPC_TRANSIENT = new Set([1, 2, 4, 13, 14]);
+const TRANSIENT_CODES = new Set([
+  "CANCELLED",
+  "UNKNOWN",
+  "DEADLINE_EXCEEDED",
+  "INTERNAL",
+  "UNAVAILABLE",
+  "ECONNRESET",
+  "ECONNREFUSED",
+  "ETIMEDOUT",
+  "EAI_AGAIN",
+  "ENOTFOUND",
+]);
+const NOT_FOUND_CODES = new Set(["404", "NOT_FOUND", "RESOURCE_NOT_FOUND", "SANDBOX_NOT_FOUND"]);
+const SAFE_CODE = /^[A-Z0-9._:-]{1,64}$/;
+
+type ErrorSignals = {
+  numbers: number[];
+  codes: string[];
+  messages: string[];
+};
+
+function safeRead(record: object, key: string): unknown {
+  try {
+    return Reflect.get(record, key);
+  } catch {
+    return undefined;
+  }
+}
+
+function collectSignals(value: unknown): ErrorSignals {
+  const out: ErrorSignals = { numbers: [], codes: [], messages: [] };
+  const seen = new Set<object>();
+  const visit = (current: unknown, depth: number): void => {
+    if (depth > 3 || current === null || current === undefined) return;
+    if (typeof current === "string") {
+      if (current.length <= 512) out.messages.push(current);
+      return;
+    }
+    if (typeof current !== "object" || seen.has(current)) return;
+    seen.add(current);
+    for (const key of [
+      "status",
+      "statusCode",
+      "httpStatus",
+      "httpStatusCode",
+      "responseStatus",
+      "errorCode",
+      "code",
+    ]) {
+      const signal = safeRead(current, key);
+      if (typeof signal === "number" && Number.isFinite(signal)) out.numbers.push(signal);
+      if (typeof signal === "string") {
+        const code = signal.trim().toUpperCase();
+        if (SAFE_CODE.test(code)) out.codes.push(code);
+      }
+    }
+    for (const key of ["name", "message", "details", "cause"]) {
+      const signal = safeRead(current, key);
+      if (typeof signal === "string" && signal.length <= 512) out.messages.push(signal);
+      else visit(signal, depth + 1);
+    }
+    visit(safeRead(current, "response"), depth + 1);
+    visit(safeRead(current, "error"), depth + 1);
+  };
+  visit(value, 0);
+  return out;
+}
+
+function safeDiagnostic(signals: ErrorSignals): string {
+  const code = signals.codes.find(Boolean);
+  const numeric = signals.numbers.find((value) => Number.isFinite(value));
+  return [code ? `code=${code}` : "", numeric !== undefined ? `status=${numeric}` : ""]
+    .filter(Boolean)
+    .join(" ");
+}
+
+function hasTransientSignal(signals: ErrorSignals): boolean {
+  return (
+    signals.codes.some((code) => TRANSIENT_CODES.has(code)) ||
+    signals.numbers.some((status) => GRPC_TRANSIENT.has(status))
+  );
+}
+
+function hasModalTerminalSandboxMessage(signals: ErrorSignals): boolean {
+  return signals.messages.some(
+    (message) =>
+      /^Modal sandbox [A-Za-z0-9_-]+ is no longer running\.$/.test(message) ||
+      /^Modal sandbox [A-Za-z0-9_-]+ not found \(has been terminated\)$/.test(message),
+  );
+}
+
+/** Typed transport evidence always dominates nested/string NotFound text. */
+export function classifyProviderSandboxFailure(
+  backendId: string,
+  error: unknown,
+): ProviderSandboxFailure {
+  if (backendId === "selfhosted" || !error) {
+    return { kind: "other", diagnostic: "selfhosted-or-empty" };
+  }
+  const signals = collectSignals(error);
+  const diagnostic = safeDiagnostic(signals) || "unclassified_provider_failure";
+
+  if (hasTransientSignal(signals)) {
+    return { kind: "transient_transport", diagnostic };
+  }
+  if (
+    signals.numbers.some((status) => status === 404) ||
+    signals.codes.some((code) => NOT_FOUND_CODES.has(code))
+  ) {
+    return { kind: "not_found", diagnostic };
+  }
+
+  // Modal's SDK emits these exact terminal grammars after resolving one exact
+  // sandbox id. Generic "not found" prose is intentionally not authoritative.
+  if (backendId === "modal" && hasModalTerminalSandboxMessage(signals)) {
+    return {
+      kind: "not_found",
+      diagnostic:
+        diagnostic === "unclassified_provider_failure"
+          ? "modal_terminal_message"
+          : `${diagnostic} modal_terminal_message`,
+    };
+  }
+
+  return { kind: "other", diagnostic };
+}
+
+export function isProviderSandboxNotFoundError(backendId: string, error: unknown): boolean {
+  return classifyProviderSandboxFailure(backendId, error).kind === "not_found";
+}
+
+/**
+ * Whether a failure from an operation routed through an already-established
+ * sandbox proves that exact provider sandbox disappeared.
+ *
+ * Routed operations can legitimately return HTTP 404 / `NOT_FOUND` for paths,
+ * files, ports, processes, and other subresources. Those generic signals are
+ * sufficient for provider create/resume APIs, but they must never retire the
+ * live sandbox identity after an arbitrary operation. Only an explicit
+ * sandbox-scoped provider code or Modal's exact terminal sandbox grammar is
+ * authoritative on this path. Transient transport evidence still dominates.
+ */
+export function isProviderSandboxGoneDuringRoutedOperation(
+  backendId: string,
+  error: unknown,
+): boolean {
+  if (backendId === "selfhosted" || !error) return false;
+  const signals = collectSignals(error);
+  if (hasTransientSignal(signals)) return false;
+  if (signals.codes.includes("SANDBOX_NOT_FOUND")) return true;
+  return backendId === "modal" && hasModalTerminalSandboxMessage(signals);
+}
+
+export function isProviderSandboxTransientError(backendId: string, error: unknown): boolean {
+  return classifyProviderSandboxFailure(backendId, error).kind === "transient_transport";
+}

package/src/sandbox/routing/backend-resolver.ts

@@ -47,6 +47,18 @@ export interface ActiveBackendResolverDeps {
   defaultBackend: RoutableBackendSession;
   /** A label for the default backend (its backend id: "modal"/"selfhosted"/…). */
   defaultKind: string;
+  /**
+   * Re-resolve the session's HOME backend after the route cache is fenced.
+   *
+   * The normal path returns `defaultBackend` (the turn's already-established
+   * handle), but an API-side same-target repair can rematerialize that lease
+   * while a worker turn is still alive. Such a repair advances the route epoch
+   * without changing the null pointer; callers that can resume the current
+   * lease-owned identity may provide this callback so the next epoch does not
+   * keep dispatching to the dead pre-repair object. This callback must never
+   * create a provider or replay an operation.
+   */
+  resolveDefaultBackend?: (pointer: ActivePointer) => Promise<ResolvedActiveBackend>;
   /** Look up a first-class sandbox by id (the swap target). Returns null when the
    *  id is unknown or not in this workspace (the caller 409s the swap). */
   getSandbox(sandboxId: string): Promise<RoutableSandbox | null>;
@@ -236,6 +248,9 @@ export function makeActiveBackendResolver(
           "the active sandbox was detached to the session default (home), but this turn started pinned to a machine and established no home box; the detach is accepted and takes effect on the next turn — this turn has no active home box",
         );
       }
+      if (deps.resolveDefaultBackend) {
+        return await deps.resolveDefaultBackend(pointer);
+      }
       return { session: deps.defaultBackend, sandboxId: null, kind: deps.defaultKind };
     }