@opengeni/worker-bundle 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,11 @@
1
1
  import {
2
2
  applyGitAuthPointerEnvironment,
3
+ firstPartyMcpWorkspaceUrl,
3
4
  stableSandboxEnvironmentForRun,
4
5
  type Settings,
5
6
  } from "@opengeni/config";
6
7
  import {
8
+ signDelegatedAccessToken,
7
9
  type ConnectionCredentialsPort,
8
10
  type GitCredentials,
9
11
  type ResourceRef,
@@ -104,8 +106,11 @@ export async function sandboxEnvironmentForRun(
104
106
  skipGitHubToken?: boolean;
105
107
  scope?: ConnectionScope;
106
108
  gitCredentials?: ConnectionCredentialsPort["gitCredentials"];
109
+ sessionId?: string;
110
+ runId?: string;
111
+ skipToolspaceToken?: boolean;
107
112
  } = {},
108
- ): Promise<{ environment: Record<string, string>; gitToken?: string }> {
113
+ ): Promise<{ environment: Record<string, string>; gitToken?: string; toolspaceToken?: string }> {
109
114
  // Precedence: deployment allowlist < git identity < workspace environment
110
115
  // < backend-aware HOME (the STABLE base, shared with the API-direct attach
111
116
  // paths via stableSandboxEnvironmentForRun) < platform run-scoped GitHub auth
@@ -122,7 +127,28 @@ export async function sandboxEnvironmentForRun(
122
127
  // OPENGENI_GIT_TOKEN_FILE), so the token VALUE never rides the manifest and the
123
128
  // SDK's per-turn provided-session env delta stays empty even though the token
124
129
  // rotates. The agent can refresh the token mid-turn via the `github_token` MCP tool.
125
- const environment = stableSandboxEnvironmentForRun(settings, workspaceEnvironment);
130
+ const stableOptions = options.scope ? { workspaceId: options.scope.workspaceId } : {};
131
+ const environment = stableSandboxEnvironmentForRun(settings, workspaceEnvironment, stableOptions);
132
+ let toolspaceToken: string | undefined;
133
+ if (
134
+ settings.toolspaceEnabled
135
+ && !options.skipToolspaceToken
136
+ && settings.delegationSecret
137
+ && options.scope
138
+ && options.sessionId
139
+ && options.runId
140
+ ) {
141
+ toolspaceToken = await signDelegatedAccessToken(settings.delegationSecret, {
142
+ accountId: options.scope.accountId,
143
+ workspaceId: options.scope.workspaceId,
144
+ subjectId: `sandbox:${options.runId}`,
145
+ subjectLabel: "sandbox toolspace",
146
+ permissions: ["toolspace:call"],
147
+ sessionId: options.sessionId,
148
+ exp: Math.floor(Date.now() / 1000) + 60 * 60,
149
+ });
150
+ environment.OPENGENI_TOOLSPACE_URL ??= firstPartyMcpWorkspaceUrl(settings, options.scope.workspaceId);
151
+ }
126
152
  const selection = githubRepositorySelection(resources);
127
153
  // NO-TOKEN SKIP (Stage D, change B): when the turn's EFFECTIVE compute backend is
128
154
  // a connected machine (selfhosted), the platform GitHub App installation token is
@@ -134,7 +160,7 @@ export async function sandboxEnvironmentForRun(
134
160
  // (validateNoEnvironmentDelta). The API-direct viewer attach path already drops the
135
161
  // token under this exact contract — proof a box runs fine without it.
136
162
  if (!selection || options.skipGitHubToken) {
137
- return { environment };
163
+ return { environment, ...(toolspaceToken ? { toolspaceToken } : {}) };
138
164
  }
139
165
  // Run-scoped sandbox preparation for GitHub App repository resources. When a
140
166
  // host binds `gitCredentials`, the HOST mints the installation token (BYO
@@ -173,7 +199,7 @@ export async function sandboxEnvironmentForRun(
173
199
  // cold-creates the box for a repo-attached session — an attach-warmed box
174
200
  // missing these keys kills the next repo turn on the SDK's manifest-env guard.
175
201
  applyGitAuthPointerEnvironment(environment, identity);
176
- return { environment, gitToken: token };
202
+ return { environment, gitToken: token, ...(toolspaceToken ? { toolspaceToken } : {}) };
177
203
  }
178
204
 
179
205
  function githubRepositorySelection(resources: ResourceRef[]): { installationId: number; repositoryIds: number[] } | null {
@@ -33,6 +33,8 @@
33
33
  import {
34
34
  accrueWarmSeconds,
35
35
  confirmDrainCold,
36
+ countQueuedTurns,
37
+ countSandboxLeasesByLiveness,
36
38
  forceDrainOverLimitViewerOnlyBoxes,
37
39
  getBillingBalance,
38
40
  listLiveModalSandboxLeaseAttributions,
@@ -60,6 +62,12 @@ import {
60
62
  terminateModalSandboxById,
61
63
  } from "@opengeni/runtime";
62
64
  import type { ActivityServices } from "./types";
65
+ import {
66
+ recordCreditMicros,
67
+ recordSandboxLeaseGauges,
68
+ recordSandboxOrphansTerminated,
69
+ recordTurnsQueuedGauge,
70
+ } from "../observability-metrics";
63
71
 
64
72
  export type ReapSandboxLeasesResult = {
65
73
  /** Stale viewer holders + warming-death rows the sweep touched is folded into
@@ -166,6 +174,7 @@ export function createSandboxLeaseActivities(
166
174
  if (!settings.sandboxOwnershipEnabled) {
167
175
  return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
168
176
  }
177
+ await refreshQueueAndLeaseGauges(db, observability);
169
178
 
170
179
  // (0) Warm-meter tick (P2.1) — accrue warm-seconds for every WARM viewer-only
171
180
  // box (turn-held boxes meter on the turn heartbeat, so the list fn excludes
@@ -213,12 +222,15 @@ export function createSandboxLeaseActivities(
213
222
 
214
223
  try {
215
224
  modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
225
+ recordSandboxOrphansTerminated(observability, modalOrphansTerminated);
216
226
  } catch (error) {
217
227
  observability.warn("sandbox reaper: Modal orphan sweep failed", {
218
228
  error: error instanceof Error ? error.message : String(error),
219
229
  });
220
230
  }
221
231
 
232
+ await refreshQueueAndLeaseGauges(db, observability);
233
+
222
234
  if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
223
235
  observability.info("sandbox reaper swept", {
224
236
  drainable: drainable.length,
@@ -275,6 +287,7 @@ async function accrueWarmTick(
275
287
  if (result.accrued) {
276
288
  accrued += 1;
277
289
  }
290
+ recordCreditMicros(observability, "usage", result.costMicros);
278
291
  } catch (error) {
279
292
  observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
280
293
  workspaceId: lease.workspaceId,
@@ -286,6 +299,26 @@ async function accrueWarmTick(
286
299
  return { accrued, workspaceIds };
287
300
  }
288
301
 
302
+ async function refreshQueueAndLeaseGauges(
303
+ db: ActivityServices["db"],
304
+ observability: ActivityServices["observability"],
305
+ ): Promise<void> {
306
+ try {
307
+ recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
308
+ } catch (error) {
309
+ observability.warn("sandbox reaper: queued-turn gauge refresh failed", {
310
+ error: error instanceof Error ? error.message : String(error),
311
+ });
312
+ }
313
+ try {
314
+ recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
315
+ } catch (error) {
316
+ observability.warn("sandbox reaper: sandbox-lease gauge refresh failed", {
317
+ error: error instanceof Error ? error.message : String(error),
318
+ });
319
+ }
320
+ }
321
+
289
322
  /**
290
323
  * The per-workspace warm-cap + force-drain pass (P2.1). For each workspace with a
291
324
  * live warm box, under the usage lock: if it is at 0 balance (when a billing /
@@ -1,5 +1,6 @@
1
1
  import {
2
2
  claimNextQueuedTurn as claimNextQueuedTurnDb,
3
+ countQueuedTurns,
3
4
  countTurnSessionHistoryItems,
4
5
  finishTurn,
5
6
  getSessionEvent,
@@ -13,6 +14,7 @@ import { appendAndPublishEvents } from "@opengeni/events";
13
14
  import { WORKER_DEATH_RESUME_TEXT } from "./agent-turn";
14
15
  import { isSteerInterrupt, pauseActiveGoalOnInterrupt } from "./goals";
15
16
  import { notifyParentOfChildTerminal } from "./parent-wake";
17
+ import { recordTurnsQueuedGauge } from "../observability-metrics";
16
18
  import type {
17
19
  ActivityServices,
18
20
  ClaimNextQueuedTurnInput,
@@ -67,7 +69,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
67
69
  }
68
70
 
69
71
  async function interruptActiveTurn(input: RunAgentTurnInput): Promise<void> {
70
- const { db, bus } = await services();
72
+ const { db, bus, observability } = await services();
71
73
  const session = await requireSession(db, input.workspaceId, input.sessionId);
72
74
  const trigger = await getSessionEvent(db, input.workspaceId, input.triggerEventId);
73
75
  // Pause an active goal before the early return below: an interrupt can
@@ -96,6 +98,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
96
98
  ]);
97
99
  await finishTurn(db, input.workspaceId, session.activeTurnId, "cancelled");
98
100
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null);
101
+ await refreshQueuedTurnsGauge(db, observability);
99
102
  }
100
103
 
101
104
  /**
@@ -116,7 +119,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
116
119
  * WORKER_DEATH_MAX_REDISPATCHES, persisted on the turn row.
117
120
  */
118
121
  async function requeueTurnAfterWorkerDeath(input: RequeueTurnAfterWorkerDeathInput): Promise<RequeueTurnAfterWorkerDeathResult> {
119
- const { settings, db, bus } = await services();
122
+ const { settings, db, bus, observability } = await services();
120
123
  const turn = await getSessionTurn(db, input.workspaceId, input.turnId);
121
124
  if (!turn || (turn.status !== "running" && turn.status !== "requires_action")) {
122
125
  // The timed-out attempt was a zombie that actually settled the turn
@@ -169,12 +172,15 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
169
172
  throw requeueError;
170
173
  }
171
174
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null);
175
+ await refreshQueuedTurnsGauge(db, observability);
172
176
  return { action: "requeued", redispatches };
173
177
  }
174
178
 
175
179
  async function claimNextQueuedTurn(input: ClaimNextQueuedTurnInput) {
176
- const { db } = await services();
177
- return await claimNextQueuedTurnDb(db, input.workspaceId, input.sessionId, input.workflowId);
180
+ const { db, observability } = await services();
181
+ const turn = await claimNextQueuedTurnDb(db, input.workspaceId, input.sessionId, input.workflowId);
182
+ await refreshQueuedTurnsGauge(db, observability);
183
+ return turn;
178
184
  }
179
185
 
180
186
  async function markSessionIdle(input: MarkSessionIdleInput): Promise<void> {
@@ -183,6 +189,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
183
189
  if (session.status === "queued" || session.status === "running") {
184
190
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
185
191
  }
192
+ await refreshQueuedTurnsGauge(db, observability);
186
193
  // The workflow reaches markSessionIdle exactly when it has decided to stop
187
194
  // for now (no queued turn, no goal continuation): the terminal-for-now
188
195
  // point for a spawned worker, whatever the cause (goal completed, agent or
@@ -199,3 +206,14 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
199
206
  markSessionIdle,
200
207
  };
201
208
  }
209
+
210
+ async function refreshQueuedTurnsGauge(
211
+ db: ActivityServices["db"],
212
+ observability: ActivityServices["observability"],
213
+ ): Promise<void> {
214
+ try {
215
+ recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
216
+ } catch {
217
+ // Best-effort telemetry; session state transitions remain authoritative.
218
+ }
219
+ }
@@ -15,6 +15,7 @@ export function createRuntimeBatcher(flushEvents: (events: AppendEventInput[]) =
15
15
  "agent.toolCall.created",
16
16
  "agent.toolCall.output",
17
17
  "agent.message.completed",
18
+ "tool.auth_needed",
18
19
  "session.requiresAction",
19
20
  "turn.completed",
20
21
  "turn.failed",
package/src/activities.ts CHANGED
@@ -12,6 +12,7 @@ import { createSandboxLeaseActivities } from "./activities/sandbox-lease";
12
12
  import { createScheduledTaskActivities } from "./activities/scheduled-tasks";
13
13
  import { createSessionStateActivities } from "./activities/session-state";
14
14
  import type { ActivityDependencies, ActivityServices } from "./activities/types";
15
+ import { observabilityEventLogger, runtimeMetricsHooksForObservability } from "./observability-metrics";
15
16
 
16
17
  export type {
17
18
  ActivityDependencies,
@@ -33,6 +34,7 @@ export function createActivities(dependencies: ActivityDependencies = {}) {
33
34
  async function services(): Promise<ActivityServices> {
34
35
  servicesPromise ??= (async () => {
35
36
  const settings = dependencies.settings ?? getSettings();
37
+ const observability = dependencies.observability ?? createObservability(settings, { component: "worker" });
36
38
  // Step I: when not injected, build the standalone handle — searchPath
37
39
  // undefined for standalone (public), scoped to the dedicated schema +
38
40
  // host RLS strategy when embedded config is set. An embedded host injects
@@ -67,11 +69,14 @@ export function createActivities(dependencies: ActivityDependencies = {}) {
67
69
  (await createNatsEventBus(
68
70
  settings.natsUrl,
69
71
  controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : undefined,
72
+ { logger: observabilityEventLogger(observability) },
70
73
  )),
71
- runtime: dependencies.runtime ?? createProductionAgentRuntime(),
74
+ runtime: dependencies.runtime ?? createProductionAgentRuntime({
75
+ metrics: runtimeMetricsHooksForObservability(observability),
76
+ }),
72
77
  objectStorage: dependencies.objectStorage ?? createObjectStorage(settings),
73
78
  documentServices: dependencies.documentServices ?? createDocumentServices(settings),
74
- observability: dependencies.observability ?? createObservability(settings, { component: "worker" }),
79
+ observability,
75
80
  wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
76
81
  // §7.5 P3 — host-entitlements port. No constructed default: standalone
77
82
  // has no host meter, so unset → null → `ensureRunAllowed` reads the
package/src/http.ts ADDED
@@ -0,0 +1,100 @@
1
+ import type { Settings } from "@opengeni/config";
2
+ import { dbSql, type Database } from "@opengeni/db";
3
+ import type { EventBus } from "@opengeni/events";
4
+ import type { Observability } from "@opengeni/observability";
5
+
6
+ export type ReadinessCheckName = "db" | "nats" | "temporal";
7
+ export type ReadinessChecks = Record<ReadinessCheckName, () => Promise<void> | void>;
8
+
9
+ export type ReadinessResult = {
10
+ ok: boolean;
11
+ checks: Record<ReadinessCheckName, { ok: boolean; error?: string }>;
12
+ };
13
+
14
+ export function startWorkerHttpServer(input: {
15
+ settings: Settings;
16
+ observability: Observability;
17
+ checks: ReadinessChecks;
18
+ timeoutMs?: number;
19
+ }): ReturnType<typeof Bun.serve> {
20
+ const { settings, observability, checks } = input;
21
+ const timeoutMs = input.timeoutMs ?? 2_000;
22
+ return Bun.serve({
23
+ hostname: "0.0.0.0",
24
+ port: settings.workerHttpPort,
25
+ async fetch(request) {
26
+ const url = new URL(request.url);
27
+ if (request.method !== "GET") {
28
+ return Response.json({ error: "method_not_allowed" }, { status: 405 });
29
+ }
30
+ if (url.pathname === "/healthz") {
31
+ return Response.json({
32
+ service: settings.serviceName,
33
+ environment: settings.environment,
34
+ deploymentRevision: settings.deploymentRevision,
35
+ ok: true,
36
+ });
37
+ }
38
+ if (url.pathname === "/metrics") {
39
+ return new Response(await observability.prometheusMetrics(), {
40
+ status: 200,
41
+ headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" },
42
+ });
43
+ }
44
+ if (url.pathname === "/readyz") {
45
+ const result = await runReadinessChecks(checks, timeoutMs);
46
+ return Response.json(result, { status: result.ok ? 200 : 503 });
47
+ }
48
+ return Response.json({ error: "not_found" }, { status: 404 });
49
+ },
50
+ });
51
+ }
52
+
53
+ export async function runReadinessChecks(checks: ReadinessChecks, timeoutMs = 2_000): Promise<ReadinessResult> {
54
+ const entries = await Promise.all(
55
+ (Object.entries(checks) as Array<[ReadinessCheckName, () => Promise<void> | void]>)
56
+ .map(async ([name, check]) => {
57
+ try {
58
+ await withTimeout(Promise.resolve().then(check), timeoutMs);
59
+ return [name, { ok: true }] as const;
60
+ } catch (error) {
61
+ return [name, { ok: false, error: error instanceof Error ? error.message : String(error) }] as const;
62
+ }
63
+ }),
64
+ );
65
+ const result = Object.fromEntries(entries) as ReadinessResult["checks"];
66
+ return {
67
+ ok: Object.values(result).every((check) => check.ok),
68
+ checks: result,
69
+ };
70
+ }
71
+
72
+ export function dbReadyCheck(db: Database): () => Promise<void> {
73
+ return async () => {
74
+ await db.execute(dbSql`select 1`);
75
+ };
76
+ }
77
+
78
+ export function natsReadyCheck(bus: EventBus): () => void {
79
+ return () => {
80
+ if (bus.isConnected && !bus.isConnected()) {
81
+ throw new Error("NATS is not connected");
82
+ }
83
+ };
84
+ }
85
+
86
+ async function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
87
+ let timer: ReturnType<typeof setTimeout> | undefined;
88
+ try {
89
+ return await Promise.race([
90
+ promise,
91
+ new Promise<never>((_, reject) => {
92
+ timer = setTimeout(() => reject(new Error(`readiness check timed out after ${timeoutMs}ms`)), timeoutMs);
93
+ }),
94
+ ]);
95
+ } finally {
96
+ if (timer) {
97
+ clearTimeout(timer);
98
+ }
99
+ }
100
+ }
package/src/index.ts CHANGED
@@ -1,9 +1,13 @@
1
- import { getSettings, retryStartupDependency, startupRetryOptions, type Settings } from "@opengeni/config";
1
+ import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth, retryStartupDependency, startupRetryOptions, type Settings } from "@opengeni/config";
2
+ import { createDb } from "@opengeni/db";
3
+ import { createNatsEventBus } from "@opengeni/events";
2
4
  import { createObservability, logStartupDependencyRetry, type Observability } from "@opengeni/observability";
3
5
  import { Connection, ScheduleAlreadyRunning, ScheduleOverlapPolicy, Client as TemporalClient } from "@temporalio/client";
4
6
  import { NativeConnection, Worker } from "@temporalio/worker";
5
7
  import { createActivities, type ActivityDependencies } from "./activities";
6
8
  import type { WakeSessionWorkflowSignal } from "./activities/types";
9
+ import { dbReadyCheck, natsReadyCheck, startWorkerHttpServer } from "./http";
10
+ import { observabilityEventLogger } from "./observability-metrics";
7
11
 
8
12
  // The deterministic id of the ONE global reaper Schedule. A single id means
9
13
  // create() is idempotent across every worker in the pool: the first worker to
@@ -57,7 +61,11 @@ export async function createOpenGeniWorker(options: WorkerOptions = {}): Promise
57
61
  // idled and let its run finish, so a plain signal would not start one).
58
62
  // Separate from the worker's NativeConnection: the @temporalio/client
59
63
  // Connection is what exposes workflow.signalWithStart.
60
- export async function createWorkerWorkflowSignaler(settings: Settings): Promise<{ wakeSessionWorkflow: WakeSessionWorkflowSignal; close: () => Promise<void> }> {
64
+ export async function createWorkerWorkflowSignaler(settings: Settings): Promise<{
65
+ wakeSessionWorkflow: WakeSessionWorkflowSignal;
66
+ check: () => Promise<void>;
67
+ close: () => Promise<void>;
68
+ }> {
61
69
  const connection = await Connection.connect({ address: settings.temporalHost });
62
70
  const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
63
71
  return {
@@ -70,6 +78,9 @@ export async function createWorkerWorkflowSignaler(settings: Settings): Promise<
70
78
  signal: "queueChanged",
71
79
  });
72
80
  },
81
+ check: async () => {
82
+ await connection.workflowService.getSystemInfo({});
83
+ },
73
84
  close: async () => {
74
85
  await connection.close();
75
86
  },
@@ -148,39 +159,76 @@ export async function registerSandboxReaperSchedule(
148
159
  export async function startWorker() {
149
160
  const settings = getSettings();
150
161
  const observability = createObservability(settings, { component: "worker" });
151
- const signaler = await retryStartupDependency(
152
- "Temporal client",
153
- () => createWorkerWorkflowSignaler(settings),
154
- {
155
- ...startupRetryOptions(settings),
156
- onRetry: (event) => logStartupDependencyRetry(observability, event),
157
- },
158
- );
159
- const { worker, connection } = await createOpenGeniWorker({
160
- settings,
161
- activityDependencies: { observability, wakeSessionWorkflow: signaler.wakeSessionWorkflow },
162
- });
163
- // Register the ONE global reaper Schedule (no-op when sandboxOwnershipEnabled
164
- // is false). Idempotent across the pool: only the first worker creates it. The
165
- // static global-queue Worker.create above is UNCHANGED — there is no
166
- // per-session worker factory.
167
- const reaperSchedule = await retryStartupDependency(
168
- "Temporal schedule (sandbox reaper)",
169
- () => registerSandboxReaperSchedule(settings, observability),
170
- {
171
- ...startupRetryOptions(settings),
172
- onRetry: (event) => logStartupDependencyRetry(observability, event),
173
- },
174
- );
175
- observability.info("OpenGeni worker listening", {
176
- temporalTaskQueue: settings.temporalTaskQueue,
162
+ const retryOptions = startupRetryOptions(settings);
163
+ const onRetry = (event: Parameters<typeof logStartupDependencyRetry>[1]) => logStartupDependencyRetry(observability, event);
164
+ const searchPath = dbSearchPath(settings);
165
+ const dbClient = createDb(settings.databaseUrl, {
166
+ ...(searchPath ? { searchPath } : {}),
167
+ rlsStrategy: settings.rlsStrategy,
177
168
  });
169
+ const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
170
+ let bus: Awaited<ReturnType<typeof createNatsEventBus>> | undefined;
171
+ let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
172
+ let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
173
+ let reaperSchedule: Awaited<ReturnType<typeof registerSandboxReaperSchedule>> | undefined;
174
+ let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
178
175
  try {
179
- await worker.run();
176
+ bus = await retryStartupDependency(
177
+ "NATS",
178
+ () =>
179
+ createNatsEventBus(
180
+ settings.natsUrl,
181
+ controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : undefined,
182
+ { logger: observabilityEventLogger(observability) },
183
+ ),
184
+ { ...retryOptions, onRetry },
185
+ );
186
+ signaler = await retryStartupDependency(
187
+ "Temporal client",
188
+ () => createWorkerWorkflowSignaler(settings),
189
+ { ...retryOptions, onRetry },
190
+ );
191
+ workerBundle = await createOpenGeniWorker({
192
+ settings,
193
+ activityDependencies: {
194
+ observability,
195
+ wakeSessionWorkflow: signaler.wakeSessionWorkflow,
196
+ db: dbClient.db,
197
+ bus,
198
+ },
199
+ });
200
+ httpServer = startWorkerHttpServer({
201
+ settings,
202
+ observability,
203
+ checks: {
204
+ db: dbReadyCheck(dbClient.db),
205
+ nats: natsReadyCheck(bus),
206
+ temporal: signaler.check,
207
+ },
208
+ });
209
+ // Register the ONE global reaper Schedule (no-op when sandboxOwnershipEnabled
210
+ // is false). Idempotent across the pool: only the first worker creates it. The
211
+ // static global-queue Worker.create above is UNCHANGED — there is no
212
+ // per-session worker factory.
213
+ reaperSchedule = await retryStartupDependency(
214
+ "Temporal schedule (sandbox reaper)",
215
+ () => registerSandboxReaperSchedule(settings, observability),
216
+ { ...retryOptions, onRetry },
217
+ );
218
+ observability.info("OpenGeni worker listening", {
219
+ temporalTaskQueue: settings.temporalTaskQueue,
220
+ httpPort: settings.workerHttpPort,
221
+ });
222
+ await workerBundle.worker.run();
180
223
  } finally {
181
- await connection.close();
182
- await signaler.close();
183
- await reaperSchedule.close();
224
+ httpServer?.stop(true);
225
+ await Promise.allSettled([
226
+ workerBundle?.connection.close(),
227
+ signaler?.close(),
228
+ reaperSchedule?.close(),
229
+ bus?.close(),
230
+ dbClient.close(),
231
+ ]);
184
232
  }
185
233
  }
186
234