@opengeni/worker-bundle 1.0.0-canary.0 → 1.0.0

package/dist/editable-artifact-materializer-service.d.ts

@@ -1,4 +1,4 @@
-import { type Settings } from "@opengeni/config";
+import { type ArtifactMaterializerSettings } from "@opengeni/config";
 import { type Database } from "@opengeni/db";
 import { type Observability } from "@opengeni/observability";
 import { type ObjectStorage } from "@opengeni/storage";
@@ -36,7 +36,7 @@ export type CreateEditableArtifactMaterializerServiceInput = Readonly<{
     }>;
 }>;
 export type ProductionEditableArtifactMaterializerServiceOptions = Readonly<{
-    settings: Settings;
+    settings: ArtifactMaterializerSettings;
     /** Dedicated DSN; never defaults to settings.databaseUrl. */
     databaseUrl: string;
     declaredDatabaseRole: string;

package/dist/editable-artifact-outbox-service.d.ts

@@ -1,4 +1,4 @@
-import { type Settings } from "@opengeni/config";
+import { type ArtifactOutboxSettings } from "@opengeni/config";
 import type { EditableArtifactOutboxDispatchSummary } from "@opengeni/core";
 import { type Observability } from "@opengeni/observability";
 import type { EditableArtifactHintNatsAuth } from "./editable-artifact-hint-broker.js";
@@ -29,7 +29,7 @@ export type CreateEditableArtifactOutboxServiceInput = Readonly<{
     readinessTimeoutMs?: number;
 }>;
 export type ProductionEditableArtifactOutboxServiceOptions = Readonly<{
-    settings: Settings;
+    settings: ArtifactOutboxSettings;
     databaseUrl: string;
     declaredDatabaseRole: string;
     natsUrl: string;

package/dist/host-export-pump.d.ts

@@ -1,4 +1,4 @@
-import type { HostEventSink, HostUsageSink } from "@opengeni/contracts";
+import type { HostEventSink, HostLifecycleFactSink, HostUsageSink } from "@opengeni/contracts";
 import { type Database, type HostExportConsumerStatus, type HostExportKind } from "@opengeni/db";
 import type { Observability } from "@opengeni/observability";
 export type HostExportDrainResult = {
@@ -24,6 +24,8 @@ export type HostExportPumpOptions = {
     db: Database;
     eventSink?: HostEventSink;
     usageSink?: HostUsageSink;
+    /** Content-free per-person product lifecycle facts (`lifecycle_fact`). */
+    lifecycleSink?: HostLifecycleFactSink;
     observability?: Observability;
     instanceId?: string;
     pollIntervalMs?: number;

package/dist/host-export.d.ts

@@ -6,4 +6,4 @@
  * events or usage must not need that native runtime (or its libc contract).
  */
 export { createHostExportPump, type HostExportDrainResult, type HostExportPump, type HostExportPumpOptions, } from "./host-export-pump.js";
-export type { HostEventExport, HostEventExportBatch, HostEventSink, HostUsageExport, HostUsageExportBatch, HostUsageSink, } from "@opengeni/contracts";
+export type { HostEventExport, HostEventExportBatch, HostEventSink, HostLifecycleFactExport, HostLifecycleFactExportBatch, HostLifecycleFactSink, HostUsageExport, HostUsageExportBatch, HostUsageSink, } from "@opengeni/contracts";

package/dist/host-export.js

@@ -1,6 +1,6 @@
 import {
   createHostExportPump
-} from "./chunk-RAMNYSAW.js";
+} from "./chunk-4ER3YXOT.js";
 import "./chunk-PZ5AY32C.js";
 export {
   createHostExportPump

package/dist/http.d.ts

@@ -27,6 +27,6 @@ export type WorkerHttpServerInput = {
 export declare function createWorkerHttpHandler(input: WorkerHttpServerInput): (request: Request) => Promise<Response>;
 export declare function startWorkerHttpServer(input: WorkerHttpServerInput): ReturnType<typeof Bun.serve>;
 export declare function runReadinessChecks(checks: ReadinessChecks, timeoutMs?: number): Promise<ReadinessResult>;
-export declare function dbReadyCheck(db: Database, posture?: RuntimeDatabasePostureOptions): () => Promise<void>;
+export declare function dbReadyCheck(db: Database, posture?: RuntimeDatabasePostureOptions, afterDatabaseReady?: () => Promise<void>): () => Promise<void>;
 export declare function natsReadyCheck(bus: EventBus): () => void;
 export declare function temporalReadyCheck(connection: Pick<import("@temporalio/worker").NativeConnection, "workflowService" | "withDeadline">): () => Promise<void>;

package/dist/index.d.ts

@@ -10,7 +10,7 @@ import type { InspectSessionAttemptActivity, SignalCodexCapacityWorkflow, Signal
 import { type WorkerLifecycleState } from "./http.js";
 import { type TurnTaskQueueStats } from "./observability-metrics.js";
 import { type WorkerRunTarget } from "./worker-service-lifecycle.js";
-export { createHostExportPump, type HostExportDrainResult, type HostExportPump, type HostExportPumpOptions, type HostEventExport, type HostEventExportBatch, type HostEventSink, type HostUsageExport, type HostUsageExportBatch, type HostUsageSink, } from "./host-export.js";
+export { createHostExportPump, type HostExportDrainResult, type HostExportPump, type HostExportPumpOptions, type HostEventExport, type HostEventExportBatch, type HostEventSink, type HostLifecycleFactExport, type HostLifecycleFactExportBatch, type HostLifecycleFactSink, type HostUsageExport, type HostUsageExportBatch, type HostUsageSink, } from "./host-export.js";
 export declare const KNOWLEDGE_INDEXING_SCHEDULE_ID = "opengeni-knowledge-indexing";
 export declare const KNOWLEDGE_INDEXING_PERIOD_MS = 15000;
 export declare const FILE_UPLOAD_REAPER_SCHEDULE_ID = "opengeni-file-upload-reaper";
@@ -56,7 +56,18 @@ export declare function createDefaultWorkerActivities(role: OpenGeniWorkerRole,
         unavailable: number;
     }>;
     getCodexCapacityWait: (input: import("./activities.js").GetCodexCapacityWaitInput) => Promise<{
-        provider?: "xai";
+        provider: "xai";
+        waiterId: string;
+        generation: number;
+        nextCheckAt: string;
+        wakeRevision: number;
+    } | {
+        provider: "claude";
+        waiterId: string;
+        generation: number;
+        nextCheckAt: string;
+        wakeRevision: number;
+    } | {
         waiterId: string;
         generation: number;
         nextCheckAt: string;
@@ -64,6 +75,21 @@ export declare function createDefaultWorkerActivities(role: OpenGeniWorkerRole,
     } | null>;
     reconcileCodexCapacityWait: (input: import("./activities.js").ReconcileCodexCapacityWaitInput) => Promise<import("./activities.js").ReconcileCodexCapacityWaitResult>;
     maintainBrowserStateArtifacts: () => Promise<import("./activities/browser-state-artifact-reaper.js").MaintainBrowserStateArtifactsResult>;
+    listDueBrowserCheckpoints(): Promise<{
+        accountId: string;
+        workspaceId: string;
+        sandboxGroupId: string;
+        leaseId: string;
+        leaseEpoch: number;
+        instanceId: string;
+        browserSessionId: string;
+        controllerGeneration: string;
+    }[]>;
+    checkpointBrowserBeforeDeadline(target: import("@opengeni/db").BrowserDeadlineCheckpointTarget): Promise<{
+        status: "skipped";
+    } | {
+        status: "suspended";
+    }>;
     settleAutomationRunFailure: (input: import("./activities.js").DispatchAutomationRunInput) => Promise<void>;
     dispatchAutomationRun: (input: import("./activities.js").DispatchAutomationRunInput) => Promise<import("./activities.js").DispatchAutomationRunResult>;
     indexDocument: (input: import("./activities.js").IndexDocumentInput) => Promise<{
@@ -114,6 +140,7 @@ export declare function createDefaultWorkerActivities(role: OpenGeniWorkerRole,
     }>;
     enqueueGoalRetryWake: (input: import("./activities.js").MaybeContinueGoalInput) => Promise<void>;
     maybeContinueGoal: (input: import("./activities.js").MaybeContinueGoalInput) => Promise<import("./activities.js").MaybeContinueGoalResult>;
+    reconcileRecentModelCallFacts: () => Promise<import("./activities/model-call-fact-reconciler.js").ReconcileRecentModelCallFactsResult>;
     runKnowledgeSourceSyncBatch: (input: import("./activities.js").RunKnowledgeSourceSyncBatchInput) => Promise<import("./activities.js").RunKnowledgeSourceSyncBatchResult>;
     maintainRetainedScreenshots: () => Promise<import("./activities/retained-screenshot-reaper.js").MaintainRetainedScreenshotsResult>;
     verifyRigChange: (input: {
@@ -199,12 +226,74 @@ export declare function createDefaultWorkerActivities(role: OpenGeniWorkerRole,
     }>;
     persistSessionAttemptQuiescence: (input: import("./activities.js").PersistSessionAttemptQuiescenceInput) => Promise<void>;
     reconcileSessionAttemptQuiescence: (input: import("./activities.js").ReconcileSessionAttemptQuiescenceInput) => Promise<import("./activities.js").ReconcileSessionAttemptQuiescenceResult>;
+    reconcileSettledSessionAttempt: (input: import("./activities/types.js").ReconcileSettledSessionAttemptInput) => Promise<import("./activities/types.js").ReconcileSettledSessionAttemptResult>;
     recoverDispatch: (input: import("./activities.js").RecoverDispatchInput) => Promise<import("./activities.js").RecoverDispatchResult>;
     recoverEscapedMcpTimeout: (input: import("./activities.js").RecoverEscapedMcpTimeoutInput) => Promise<import("./activities.js").RecoverEscapedMcpTimeoutResult>;
-    peekSessionWork: (input: import("./activities/types.js").PeekSessionWorkInput) => Promise<import("@opengeni/db").SessionWorkPeek>;
+    peekSessionWork: (input: import("./activities/types.js").PeekSessionWorkInput) => Promise<{
+        kind: "unavailable";
+    } | {
+        kind: "attempt-owned";
+        turnId: string;
+        attemptId: string;
+        executionGeneration: number;
+        activityRef: import("@opengeni/db").SessionAttemptActivityRef;
+    } | {
+        kind: "runnable";
+        admissionFence?: import("@opengeni/db").SessionAdmissionFence;
+    } | {
+        kind: "admission-blocked";
+        reason?: "sandbox_setup_outcome_unknown" | "sandbox_setup_recovery_exhausted";
+        ref?: import("@opengeni/db").SandboxSetupOutcomeUnknown | import("@opengeni/db").SandboxSetupRecoveryExhausted;
+    } | {
+        kind: "sandbox-lifecycle-wait";
+        ref: import("@opengeni/db").SandboxLifecycleWait;
+    } | {
+        kind: "approval-pending";
+        triggerEventId: string;
+        admissionFence?: import("@opengeni/db").SessionAdmissionFence;
+    } | {
+        kind: "approval-wait";
+        humanInputRequestId?: string;
+        interactionInterventionId?: string;
+        scheduledRunTimeout?: {
+            runId: string;
+            turnId: string;
+        };
+        expiresAt?: string;
+    } | {
+        kind: "capacity-wait";
+        ref: {
+            provider?: "codex" | "xai" | "claude";
+            waiterId: string;
+            generation: number;
+            nextCheckAt: string;
+            wakeRevision: number;
+        };
+    } | {
+        kind: "interruption-pending";
+        attemptId: string;
+    } | {
+        kind: "cancellation-wait";
+        attemptId: string;
+    } | {
+        kind: "input-wait";
+        disposition: "held" | "timeout" | "superseded";
+        waitTurnId: string;
+        deadlineAt: string;
+    } | {
+        kind: "idle";
+    } | {
+        kind: "attempt-owned";
+        turnId: string;
+        attemptId: string;
+        executionGeneration: number;
+        activityRef: import("@opengeni/db").SessionAttemptActivityRef;
+        ownerActivityState: "pending" | "settled" | "unknown";
+    }>;
     settleSessionInputWait: (input: import("./activities/types.js").SettleSessionInputWaitInput) => Promise<import("./activities/types.js").SettleSessionInputWaitResult>;
     expireSessionHumanInput: (input: import("./activities.js").ExpireSessionHumanInputInput) => Promise<import("./activities.js").ExpireSessionHumanInputResult>;
     expireSessionInteractionIntervention: (input: import("./activities/types.js").ExpireSessionInteractionInterventionInput) => Promise<import("./activities/types.js").ExpireSessionInteractionInterventionResult>;
+    expireScheduledRunHumanWait: (input: import("./activities/types.js").ExpireScheduledRunHumanWaitInput) => Promise<import("./activities/types.js").ExpireScheduledRunHumanWaitResult>;
     markSessionIdle: (input: import("./activities/types.js").MarkSessionIdleInput) => Promise<void>;
     dispatchSessionWorkflowWakes(): Promise<import("./activities/workflow-wake.js").DispatchSessionWorkflowWakesResult>;
 } | {
@@ -304,6 +393,9 @@ export type OpenGeniWorkerServiceOptions = Omit<WorkerOptions, "activityDependen
         db: Database;
         bus: EventBus;
     };
+    /** Host-owned final containment after a stalled cleanup has drained peers.
+     * The standalone process exits; embedded hosts own their termination policy. */
+    terminateWorker?: () => void;
     /**
      * Exact catalog posture required by the standalone runtime. Embedded hosts
      * may omit this when they own an equivalent database isolation contract.

package/dist/index.js

@@ -1,6 +1,6 @@
 import {
   createHostExportPump
-} from "./chunk-RAMNYSAW.js";
+} from "./chunk-4ER3YXOT.js";
 import {
   CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
   CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
@@ -15,11 +15,11 @@ import {
   initializeContextCompactionMetrics,
   initializeWorkerOutcomeMetrics,
   normalizeTurnTaskQueueStats,
-  observabilityEventLogger,
+  observabilityEventBusOptions,
   startContextCompactionPendingMonitor,
   startSessionRecoveryMonitor,
   startTurnCapacityMonitor
-} from "./chunk-A4XQWVZK.js";
+} from "./chunk-3ZTRWDMN.js";
 import "./chunk-PZ5AY32C.js";
 
 // src/index.ts
@@ -33,6 +33,7 @@ import {
 } from "@opengeni/config";
 import {
   assertRuntimeDatabasePosture as assertRuntimeDatabasePosture2,
+  isRetryableRuntimeDatabaseStartupError,
   countSessionRecoveryBacklog,
   createDb,
   getContextCompactionPendingSummary,
@@ -59,7 +60,12 @@ import { existsSync } from "fs";
 import { fileURLToPath } from "url";
 
 // src/workflows/activities.ts
-import { ActivityCancellationType, proxyActivities } from "@temporalio/workflow";
+import {
+  ActivityCancellationType,
+  ActivityFailure,
+  ApplicationFailure,
+  proxyActivities
+} from "@temporalio/workflow";
 
 // src/knowledge-source-sync-activity-policy.ts
 var KNOWLEDGE_SOURCE_SYNC_ACTIVITY_HEARTBEAT_TIMEOUT_MS = 12e4;
@@ -209,16 +215,17 @@ async function runReadinessChecks(checks, timeoutMs = 2e3) {
     checks: result
   };
 }
-function dbReadyCheck(db, posture) {
+function dbReadyCheck(db, posture, afterDatabaseReady) {
   let inFlight;
   return () => {
     if (inFlight) return inFlight;
     inFlight = (async () => {
       if (posture) {
         await assertRuntimeDatabasePosture(db, posture);
-        return;
+      } else {
+        await db.execute(dbSql`select 1`);
       }
-      await db.execute(dbSql`select 1`);
+      await afterDatabaseReady?.();
     })().finally(() => {
       inFlight = void 0;
     });
@@ -408,6 +415,47 @@ function safeLifecycleLog(log) {
   } catch {
   }
 }
+function createWorkerCleanupContainment(input) {
+  let requested = false;
+  let finished = false;
+  let timer;
+  return {
+    request() {
+      if (requested || finished) return;
+      requested = true;
+      if (input.terminate) {
+        timer = setTimeout(() => {
+          safeLifecycleLog(
+            () => input.observability.error("worker cleanup drain exhausted; terminating host", {
+              errorClass: "WorkerLifecycleOperationError",
+              errorCode: "worker_cleanup_drain_exhausted",
+              origin: "worker-lifecycle"
+            })
+          );
+          input.terminate?.();
+        }, input.timeoutMs ?? 1e5);
+        timer.unref?.();
+      }
+      let accepted = false;
+      try {
+        accepted = input.drain();
+      } catch {
+      }
+      if (!accepted)
+        safeLifecycleLog(
+          () => input.observability.error("worker cleanup drain request failed", {
+            errorClass: "WorkerLifecycleOperationError",
+            errorCode: "worker_shutdown_request_failed",
+            origin: "worker-lifecycle"
+          })
+        );
+    },
+    finished() {
+      finished = true;
+      clearTimeout(timer);
+    }
+  };
+}
 
 // src/memory-pressure-guard.ts
 import { readFileSync } from "fs";
@@ -762,6 +810,13 @@ function installWorkerUnhandledRejectionBoundary(runtimeProcess = process, repor
   return () => runtimeProcess.off("unhandledRejection", onUnhandledRejection);
 }
 
+// src/browser-deadline-checkpoint-contract.ts
+var BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID = "opengeni-browser-deadline-checkpoints-v1";
+var QUEUE_SUFFIX = "-browser-checkpoint-v1";
+function browserDeadlineCheckpointTaskQueue(base) {
+  return base.endsWith(QUEUE_SUFFIX) ? base : `${base}${QUEUE_SUFFIX}`;
+}
+
 // src/index.ts
 var SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
 var KNOWLEDGE_INDEXING_SCHEDULE_ID = "opengeni-knowledge-indexing";
@@ -784,7 +839,7 @@ function temporalWorkflowExecutionNotFound(error) {
   return isGrpcServiceError(error) && error.code === 5;
 }
 async function createDefaultWorkerActivities(role, dependencies = {}) {
-  return role === "control" ? (await import("./activities-control-HMFGSKC5.js")).createControlActivities(dependencies) : (await import("./activities-turn-AP67JADS.js")).createTurnActivities(dependencies);
+  return role === "control" ? (await import("./activities-control-BNPTRTOE.js")).createControlActivities(dependencies) : (await import("./activities-turn-5LGZOSVU.js")).createTurnActivities(dependencies);
 }
 function resolveOpenGeniWorkflowDefinition(moduleUrl = import.meta.url) {
   const modulePath = fileURLToPath(moduleUrl).replaceAll("\\", "/");
@@ -839,10 +894,18 @@ async function createOpenGeniWorker(options) {
       }
     ),
     async (connection) => {
+      let turnWorker;
+      let cleanupDrainRequested = false;
       const activityDependencies = {
         ...options.activityDependencies,
         settings,
-        observability
+        observability,
+        requestWorkerDrain: options.activityDependencies?.requestWorkerDrain ?? (() => {
+          if (cleanupDrainRequested) return;
+          if (!turnWorker) throw new Error("turn worker lifecycle is not initialized");
+          turnWorker.shutdown();
+          cleanupDrainRequested = true;
+        })
       };
       const activities = options.activities ?? await createDefaultWorkerActivities(options.role, activityDependencies);
       const turnConcurrency = options.role === "turn" ? createTurnWorkerConcurrencyPlan(settings, { observability }) : null;
@@ -876,28 +939,38 @@ async function createOpenGeniWorker(options) {
         } : turnConcurrency.options
       });
       if (options.role !== "control") {
+        turnWorker = worker;
         turnConcurrency?.admission?.finalizeStartupBaseline();
         return { worker, connection };
       }
+      const ownedWorkers = [worker];
       try {
-        const sandboxLifecycleWorker = await Worker.create({
-          ...sharedWorkerOptions,
-          ...workflowDefinition,
-          taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
-          reuseV8Context: true,
-          workflowThreadPoolSize: 1,
-          maxCachedWorkflows: CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
-          maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
-          maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES
-        });
+        for (const taskQueue of [
+          sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
+          browserDeadlineCheckpointTaskQueue(settings.temporalTaskQueue)
+        ]) {
+          const lifecycleWorker = await Worker.create({
+            ...sharedWorkerOptions,
+            ...workflowDefinition,
+            taskQueue,
+            reuseV8Context: true,
+            workflowThreadPoolSize: 1,
+            maxCachedWorkflows: CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
+            maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
+            maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES
+          });
+          ownedWorkers.push(lifecycleWorker);
+        }
         return {
-          worker: combineWorkerRunTargets([worker, sandboxLifecycleWorker]),
+          worker: combineWorkerRunTargets(ownedWorkers),
           connection
         };
       } catch (error) {
-        try {
-          worker.shutdown();
-        } catch {
+        for (const ownedWorker of ownedWorkers) {
+          try {
+            ownedWorker.shutdown();
+          } catch {
+          }
         }
         throw error;
       }
@@ -1006,20 +1079,26 @@ async function createWorkerWorkflowSignaler(settings, db) {
       return normalizeTurnTaskQueueStats(response.stats);
     },
     startSandboxReaperWorkflow: async () => {
-      try {
-        await temporal.workflow.start("sandboxReaperWorkflowV2", {
+      const starts = await Promise.allSettled([
+        temporal.workflow.start("sandboxReaperWorkflowV2", {
           taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
           workflowId: SANDBOX_REAPER_V2_WORKFLOW_ID,
           workflowIdReusePolicy: "ALLOW_DUPLICATE",
           args: []
-        });
-        return "started";
-      } catch (error) {
-        if (error instanceof WorkflowExecutionAlreadyStartedError) {
-          return "already_running";
+        }),
+        temporal.workflow.start("browserDeadlineCheckpointSweepWorkflow", {
+          taskQueue: browserDeadlineCheckpointTaskQueue(settings.temporalTaskQueue),
+          workflowId: BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID,
+          workflowIdReusePolicy: "ALLOW_DUPLICATE",
+          args: []
+        })
+      ]);
+      for (const start of starts) {
+        if (start.status === "rejected" && !(start.reason instanceof WorkflowExecutionAlreadyStartedError)) {
+          throw start.reason;
         }
-        throw error;
       }
+      return starts[0].status === "fulfilled" ? "started" : "already_running";
     },
     startVideoGenerationWorkflow: async ({ accountId, workspaceId, operationId }) => {
       try {
@@ -1271,6 +1350,7 @@ async function createOpenGeniWorkerService(options) {
   const retryOptions = startupRetryOptions(settings);
   const onRetry = (event) => logStartupDependencyRetry(observability, event);
   let lifecycle;
+  let cleanupContainment;
   let signaler;
   let workerBundle;
   let turnCapacityMonitor;
@@ -1321,7 +1401,11 @@ async function createOpenGeniWorkerService(options) {
         inspectSessionAttemptActivity,
         signalCodexCapacityWorkflow,
         startSandboxReaperWorkflow,
-        startVideoGenerationWorkflow
+        startVideoGenerationWorkflow,
+        requestWorkerDrain: () => {
+          if (!cleanupContainment) throw new Error("worker cleanup containment is not initialized");
+          cleanupContainment.request();
+        }
       }
     });
     if (options.role === "turn") {
@@ -1380,18 +1464,15 @@ async function createOpenGeniWorkerService(options) {
       );
     }
     if (options.http !== false) {
-      const databaseReady = dbReadyCheck(
-        options.http?.readinessDb ?? options.activityDependencies.db,
-        options.databasePosture
-      );
+      const readinessDb = options.http?.readinessDb ?? options.activityDependencies.db;
+      const databaseReady = dbReadyCheck(readinessDb, options.databasePosture, async () => {
+        await resolveCatalogSettings(readinessDb, settings);
+      });
       httpServer = startWorkerHttpServer({
         settings,
         observability,
         checks: {
-          db: async () => {
-            await databaseReady();
-            await resolveCatalogSettings(options.activityDependencies.db, settings);
-          },
+          db: databaseReady,
           nats: natsReadyCheck(options.activityDependencies.bus),
           temporal: temporalReadyCheck(workerBundle.connection)
         },
@@ -1416,9 +1497,21 @@ async function createOpenGeniWorkerService(options) {
   if (!activeWorkerBundle) {
     throw new Error("OpenGeni worker service initialization did not complete");
   }
+  cleanupContainment = createWorkerCleanupContainment({
+    drain: () => lifecycle?.drain("stalled turn finalization") ?? false,
+    ...options.terminateWorker ? { terminate: options.terminateWorker } : {},
+    observability
+  });
+  const activeCleanupContainment = cleanupContainment;
   lifecycle = createWorkerServiceLifecycle({
     role: options.role,
-    worker: activeWorkerBundle.worker,
+    worker: {
+      shutdown: () => activeWorkerBundle.worker.shutdown(),
+      run: async () => {
+        await activeWorkerBundle.worker.run();
+        activeCleanupContainment.finished();
+      }
+    },
     observability,
     closeOwnedResources: async () => {
       memoryPressureGuard?.close();
@@ -1526,19 +1619,20 @@ async function startWorker() {
     await retryStartupDependency(
       "PostgreSQL runtime posture",
       () => assertRuntimeDatabasePosture2(dbClient.db, databasePosture),
-      { ...retryOptions, onRetry }
+      { ...retryOptions, onRetry, shouldRetry: isRetryableRuntimeDatabaseStartupError }
     );
     bus = await retryStartupDependency(
       "NATS",
       () => createNatsEventBus(
         settings.natsUrl,
         controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
-        { logger: observabilityEventLogger(observability) }
+        observabilityEventBusOptions(observability)
       ),
       { ...retryOptions, onRetry }
     );
     await runOpenGeniWorker({
       role,
+      terminateWorker: () => process.exit(1),
       settings,
       databasePosture,
       http: { readinessDb: readinessDbClient.db },