@opengeni/worker-bundle 2.3.1 → 2.3.2-canary.37090756480001

package/src/sandbox-routing.ts

@@ -1,3 +1,4 @@
+import { isDeepStrictEqual } from "node:util";
 // apps/worker/src/sandbox-routing.ts — wire the agent-loop-free routing proxy
 // (`@opengeni/runtime` RoutingSandboxSession + makeActiveBackendResolver) to the
 // real DB pointer + the live NATS control plane for the WORKER TURN path (M7).
@@ -47,6 +48,7 @@ import { observeSessionBackgroundCommandCompletion } from "@opengeni/db/session-
 import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
 import type { OpStreamOutputFrame } from "@opengeni/runtime/sandbox";
 import type { EventBus } from "@opengeni/events";
+import { publishDurableSessionEvents } from "./session-event-fanout";
 import {
   buildSelfhostedBackendSession,
   ActiveBackendUnresolvableError,
@@ -401,6 +403,11 @@ async function resolveCurrentHomeBackend(
       diagnostic: "provider_not_found_during_home_route_rebind",
     });
     if (marked.status === "marked") {
+      await publishDurableSessionEvents(
+        services.bus,
+        ids.workspaceId,
+        marked.backgroundCommandEvents,
+      );
       await services.onHomeSandboxLost?.({
         sandboxGroupId: ids.sandboxGroupId,
         instanceId: lease.instanceId,
@@ -785,6 +792,62 @@ function settleRetainedProcessForTurn(
   };
 }
 
+/** Cleanup can observe the independently settled row even when this worker's
+ * pending output receipt or original provider transport can no longer advance. */
+function isRetainedProcessSettledForTurn(services: RoutingWiringServices, ids: RoutingWiringIds) {
+  const fence = ids.workspaceMutationFence;
+  if (!fence) return undefined;
+  return async ({
+    backend,
+    process,
+  }: {
+    backend: ResolvedActiveBackend;
+    process: RoutingRetainedProcess;
+  }): Promise<boolean> => {
+    const durable = await getRetainedProcess(services.db, {
+      workspaceId: ids.workspaceId,
+      sessionId: ids.sessionId,
+      processId: process.id,
+    });
+    if (
+      !durable ||
+      durable.state === "active" ||
+      durable.providerSessionId !== process.providerSessionId ||
+      durable.providerBackend !== (sandboxBackendForSdkBackendId(backend.kind) ?? backend.kind) ||
+      durable.providerInstanceId !== backend.providerInstanceId ||
+      durable.leaseEpoch !== backend.leaseEpoch ||
+      durable.routeKind !== (backend.sandboxId === null ? "home" : "active") ||
+      durable.routeTargetId !== backend.sandboxId ||
+      durable.routeEpoch !== backend.activeEpoch
+    )
+      return false;
+    if (process.providerCommand) {
+      // The general process projection deliberately omits the provider locator.
+      // Load its immutable identity through the exact protected persistence seam.
+      const command = await retainedProviderCommandPersistence(services.db, {
+        accountId: fence.accountId,
+        workspaceId: ids.workspaceId,
+        sessionId: ids.sessionId,
+        processId: process.id,
+      }).load();
+      if (
+        !command ||
+        command.kind !== process.providerCommand.kind ||
+        command.execId !== process.providerCommand.execId ||
+        command.taskId !== process.providerCommand.taskId ||
+        command.sandboxId !== process.providerCommand.sandboxId ||
+        Boolean(command.pty) !== Boolean(process.providerCommand.pty) ||
+        (command.kind === "modal-router-v1" &&
+          process.providerCommand.kind === "modal-router-v1" &&
+          !isDeepStrictEqual(command.supervision, process.providerCommand.supervision))
+      )
+        return false;
+    }
+    // Terminal rows are written only through proof-gated physical settlement.
+    return true;
+  };
+}
+
 function adoptRetainedProcessAsBackgroundCommandForTurn(
   services: RoutingWiringServices,
   ids: RoutingWiringIds,
@@ -867,6 +930,7 @@ export function wrapTurnBoxWithRouting(
   const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
   const afterProcessMutation = afterRetainedProcessMutation(services, ids);
   const settleProcess = settleRetainedProcessForTurn(services, ids);
+  const isProcessSettled = isRetainedProcessSettledForTurn(services, ids);
   const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
     services,
     ids,
@@ -1063,6 +1127,7 @@ export function wrapTurnBoxWithRouting(
     ...(beforeProcessMutation ? { beforeProcessMutation } : {}),
     ...(afterProcessMutation ? { afterProcessMutation } : {}),
     ...(settleProcess ? { settleProcess } : {}),
+    ...(isProcessSettled ? { isProcessSettled } : {}),
     ...(ids.workspaceMutationFence
       ? { observeProcessTerminal: observeRetainedProcessTerminalForTurn(services, ids)! }
       : {}),
@@ -1093,6 +1158,11 @@ export function wrapTurnBoxWithRouting(
               diagnostic: "provider_not_found_during_routed_operation",
             });
             if (marked.status === "marked") {
+              await publishDurableSessionEvents(
+                services.bus,
+                ids.workspaceId,
+                marked.backgroundCommandEvents,
+              );
               await services.onHomeSandboxLost?.({
                 sandboxGroupId: home.sandboxGroupId,
                 instanceId: expectedInstanceId,
@@ -1149,6 +1219,7 @@ export function wrapLazyTurnBoxWithRouting(
   const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
   const afterProcessMutation = afterRetainedProcessMutation(services, ids);
   const settleProcess = settleRetainedProcessForTurn(services, ids);
+  const isProcessSettled = isRetainedProcessSettledForTurn(services, ids);
   const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
     services,
     ids,
@@ -1324,6 +1395,7 @@ export function wrapLazyTurnBoxWithRouting(
     ...(beforeProcessMutation ? { beforeProcessMutation } : {}),
     ...(afterProcessMutation ? { afterProcessMutation } : {}),
     ...(settleProcess ? { settleProcess } : {}),
+    ...(isProcessSettled ? { isProcessSettled } : {}),
     ...(ids.workspaceMutationFence
       ? { observeProcessTerminal: observeRetainedProcessTerminalForTurn(services, ids)! }
       : {}),
@@ -1355,6 +1427,11 @@ export function wrapLazyTurnBoxWithRouting(
               diagnostic: "provider_not_found_during_routed_operation",
             });
             if (marked.status === "marked") {
+              await publishDurableSessionEvents(
+                services.bus,
+                ids.workspaceId,
+                marked.backgroundCommandEvents,
+              );
               await services.onHomeSandboxLost?.({
                 sandboxGroupId: home.sandboxGroupId,
                 instanceId: backend.providerInstanceId,

package/src/session-event-fanout.ts

@@ -0,0 +1,23 @@
+import type { SessionEvent } from "@opengeni/contracts";
+import type { EventBus } from "@opengeni/events";
+
+/** Best-effort live fanout of events that are already durable, grouped by
+ * session. Durable catch-up remains the source of truth, so failures are only
+ * reported. */
+export async function publishDurableSessionEvents(
+  bus: EventBus | null | undefined,
+  workspaceId: string,
+  events: readonly SessionEvent[] | undefined,
+  onError: (error: unknown) => void = () => undefined,
+): Promise<void> {
+  if (!bus || !events?.length) return;
+  for (const sessionId of new Set(events.map((event) => event.sessionId))) {
+    await bus
+      .publish(
+        workspaceId,
+        sessionId,
+        events.filter((event) => event.sessionId === sessionId),
+      )
+      .catch(onError);
+  }
+}

package/src/worker-service-lifecycle.ts

@@ -172,3 +172,56 @@ function safeLifecycleLog(log: () => void): void {
     // Telemetry cannot change worker lifecycle state or shutdown delivery.
   }
 }
+
+/** A cleanup stall first drains peers through WORKER_SHUTDOWN. The standalone
+ * process owns the final exit, because Temporal cannot kill JavaScript writers.
+ * Embedded hosts may supply their own termination policy at this same edge. */
+export function createWorkerCleanupContainment(input: {
+  drain: () => boolean;
+  terminate?: () => void;
+  observability: Observability;
+  timeoutMs?: number;
+}) {
+  let requested = false;
+  let finished = false;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  return {
+    request() {
+      if (requested || finished) return;
+      requested = true;
+      if (input.terminate) {
+        timer = setTimeout(() => {
+          safeLifecycleLog(() =>
+            input.observability.error("worker cleanup drain exhausted; terminating host", {
+              errorClass: "WorkerLifecycleOperationError",
+              errorCode: "worker_cleanup_drain_exhausted",
+              origin: "worker-lifecycle",
+            }),
+          );
+          input.terminate?.();
+        }, input.timeoutMs ?? 100_000);
+        timer.unref?.();
+      }
+      let accepted = false;
+      try {
+        accepted = input.drain();
+      } catch {
+        /* The host backstop remains armed. */
+      }
+      if (!accepted)
+        safeLifecycleLog(() =>
+          input.observability.error("worker cleanup drain request failed", {
+            errorClass: "WorkerLifecycleOperationError",
+            errorCode: "worker_shutdown_request_failed",
+            origin: "worker-lifecycle",
+          }),
+        );
+    },
+    finished() {
+      // Call only after Worker.run resolves. ForceShutdownError does not prove
+      // its activity promises stopped, so rejection must retain the backstop.
+      finished = true;
+      clearTimeout(timer);
+    },
+  };
+}

package/src/workflows/file-upload-reaper.ts

@@ -8,6 +8,7 @@ const reaperActivity = proxyActivities<
     | "maintainRetainedScreenshots"
     | "maintainBrowserStateArtifacts"
     | "recoverVideoGenerationWorkflows"
+    | "reconcileRecentModelCallFacts"
   >
 >({
   startToCloseTimeout: "5 minutes",
@@ -20,4 +21,5 @@ export async function fileUploadReaperWorkflow(): Promise<void> {
   await reaperActivity.maintainRetainedScreenshots();
   await reaperActivity.maintainBrowserStateArtifacts();
   await reaperActivity.recoverVideoGenerationWorkflows();
+  await reaperActivity.reconcileRecentModelCallFacts();
 }

package/src/workflows/session.ts

@@ -367,7 +367,12 @@ export function postClaimDatabaseRecoveryDetail(
   if (
     (detail.sandboxSetupOutcomeUnknown !== undefined &&
       detail.sandboxSetupOutcomeUnknown !== true) ||
-    (detail.sandboxSetupOutcomeUnknown === true && hasProviderRecoveryCount) ||
+    (detail.sandboxSetupRecoveryExhausted !== undefined &&
+      detail.sandboxSetupRecoveryExhausted !== true) ||
+    (detail.sandboxSetupOutcomeUnknown === true && detail.sandboxSetupRecoveryExhausted === true) ||
+    ((detail.sandboxSetupOutcomeUnknown === true ||
+      detail.sandboxSetupRecoveryExhausted === true) &&
+      hasProviderRecoveryCount) ||
     hasProviderRecoveryCount !== hasProviderFailureCode ||
     (hasProviderRecoveryCount &&
       (!Number.isSafeInteger(detail.providerRecoveryCount) ||