@opengeni/worker-bundle 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@opengeni/worker-bundle",
3
- "version": "0.4.0",
3
+ "version": "0.4.1",
4
4
  "description": "OpenGeni worker entry: runOpenGeniWorker/createOpenGeniWorker(deps) + the un-bundled Temporal workflow source tree. An engine-distribution surface — its runtime closure includes engine-internal packages and the workflow bundle ships as on-disk TS source.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -38,14 +38,14 @@
38
38
  },
39
39
  "dependencies": {
40
40
  "@opengeni/codex": "^0.2.1",
41
- "@opengeni/config": "^0.2.4",
42
- "@opengeni/contracts": "^0.6.0",
43
- "@opengeni/db": "^0.4.0",
44
- "@opengeni/documents": "^0.2.4",
45
- "@opengeni/events": "^0.2.4",
41
+ "@opengeni/config": "^0.2.5",
42
+ "@opengeni/contracts": "^0.7.0",
43
+ "@opengeni/db": "^0.4.1",
44
+ "@opengeni/documents": "^0.2.5",
45
+ "@opengeni/events": "^0.2.5",
46
46
  "@opengeni/observability": "^0.2.1",
47
- "@opengeni/runtime": "^0.3.0",
48
- "@opengeni/storage": "^0.2.4",
47
+ "@opengeni/runtime": "^0.3.1",
48
+ "@opengeni/storage": "^0.2.5",
49
49
  "@temporalio/activity": "^1.17.0",
50
50
  "@temporalio/client": "^1.17.0",
51
51
  "@temporalio/worker": "^1.17.0",
@@ -94,6 +94,7 @@ import type {
94
94
  } from "./types";
95
95
  import { resumeBoxForTurn, acquireSelfhostedLeaseForTurn, type ResumedTurnSandbox } from "../sandbox-resume";
96
96
  import { wrapTurnBoxWithRouting, establishSelfhostedTurnSession, routingEnabled } from "../sandbox-routing";
97
+ import { recordCreditMicros, runtimeMetricsHooksForObservability, turnLifecycleMetricsFor, type TurnOutcome } from "../observability-metrics";
97
98
  import { beginRecording, discardRecording, finalizeRecording, type ActiveRecording } from "./recording";
98
99
  import { createObjectStorage, type ObjectStorage } from "@opengeni/storage";
99
100
  import { desktopCapableBackend, sandboxRunAs } from "@opengeni/runtime";
@@ -510,7 +511,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
510
511
  "opengeni.workflow_id": input.workflowId,
511
512
  "opengeni.trigger_event_id": input.triggerEventId,
512
513
  });
513
- let activityStatus = "unknown";
514
+ let activityStatus: RunAgentTurnResult["status"] | "unknown" = "unknown";
515
+ let turnMetricOutcome: TurnOutcome | null = null;
514
516
  let activityError: unknown;
515
517
  let turnId: string | undefined;
516
518
  let heartbeatTimer: ReturnType<typeof startActivityHeartbeat> | undefined;
@@ -680,6 +682,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
680
682
  throw new Error(`Session turn not found for trigger: ${input.triggerEventId}`);
681
683
  }
682
684
  turnId = turn.id;
685
+ turnLifecycleMetricsFor(observability).start(turnId);
683
686
  // Canonical codex-billed predicate (codex/<slug> + feature enabled + active
684
687
  // workspace credential). Computed once and threaded through every billing
685
688
  // gate + the usage recorder so a turn paid by the user's ChatGPT/Codex plan
@@ -836,6 +839,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
836
839
  { type: "session.status.changed", payload: { status: "idle" } },
837
840
  ], true);
838
841
  await finishTurn(db, input.workspaceId, turnId, "failed");
842
+ turnMetricOutcome = "failed";
839
843
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
840
844
  activityStatus = "idle";
841
845
  // idleUntilReset marks this a MANDATORY hold: session.ts must wait the full
@@ -1100,7 +1104,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1100
1104
  };
1101
1105
  } else {
1102
1106
  resolvedSandbox = await resumeBoxForTurn(
1103
- { db, settings },
1107
+ { db, settings, sandboxMetrics: runtimeMetricsHooksForObservability(observability) },
1104
1108
  {
1105
1109
  accountId: input.accountId,
1106
1110
  workspaceId: input.workspaceId,
@@ -1192,7 +1196,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1192
1196
  expectedEpoch: heartbeatEpoch,
1193
1197
  warmRateMicrosPerSecond: warmRate,
1194
1198
  subjectId: input.sessionId,
1195
- }).catch(() => undefined);
1199
+ })
1200
+ .then((result) => recordCreditMicros(observability, "usage", result.costMicros))
1201
+ .catch(() => undefined);
1196
1202
  }, 10_000);
1197
1203
  if ("unref" in leaseHeartbeatTimer && typeof leaseHeartbeatTimer.unref === "function") {
1198
1204
  leaseHeartbeatTimer.unref();
@@ -1534,6 +1540,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1534
1540
  dispatchId,
1535
1541
  positionalKey: `response-${responseUsageCount}`,
1536
1542
  }),
1543
+ observability,
1537
1544
  });
1538
1545
  await reconcileConversationTruth();
1539
1546
  try {
@@ -1588,6 +1595,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1588
1595
  isCodexTurn,
1589
1596
  usage: stream.state.usage,
1590
1597
  sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
1598
+ observability,
1591
1599
  });
1592
1600
  }
1593
1601
  if (lastInputTokensObserved !== null) {
@@ -1640,6 +1648,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1640
1648
  { type: "session.status.changed", payload: { status: "idle" } },
1641
1649
  ], true);
1642
1650
  await finishTurn(db, input.workspaceId, activeTurnId, "idle");
1651
+ turnMetricOutcome = "completed";
1643
1652
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
1644
1653
  await recordUsageEvent(db, {
1645
1654
  accountId: input.accountId,
@@ -1735,6 +1744,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1735
1744
  await requeuePreemptedTurn(db, input.workspaceId, activeTurnId, preemptedEvent ? preemptedEvent.id : input.triggerEventId);
1736
1745
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
1737
1746
  activityStatus = "preempted";
1747
+ turnMetricOutcome = "preempted";
1738
1748
  observability.info("context compaction recovery succeeded by compacting and auto-continuing", {
1739
1749
  sessionId: input.sessionId,
1740
1750
  turnId: activeTurnId,
@@ -1757,6 +1767,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1757
1767
  { type: "session.status.changed", payload: { status: "idle" } },
1758
1768
  ], true);
1759
1769
  await finishTurn(db, input.workspaceId, activeTurnId, "failed");
1770
+ turnMetricOutcome = "failed";
1760
1771
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
1761
1772
  activityStatus = "idle";
1762
1773
  activityError = attemptError;
@@ -1808,6 +1819,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1808
1819
  await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, input.triggerEventId);
1809
1820
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
1810
1821
  activityStatus = "preempted";
1822
+ turnMetricOutcome = "preempted";
1811
1823
  return { status: "preempted" };
1812
1824
  } catch (requeueError) {
1813
1825
  console.error("sandbox lease supersession requeue failed; falling back", requeueError);
@@ -1868,6 +1880,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1868
1880
  await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
1869
1881
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
1870
1882
  activityStatus = "preempted";
1883
+ turnMetricOutcome = "preempted";
1871
1884
  return { status: "preempted" };
1872
1885
  } catch (preemptError) {
1873
1886
  // A failing checkpoint/requeue must not surface as an arbitrary
@@ -1885,6 +1898,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1885
1898
  await flushRuntimeBatcher();
1886
1899
  if (preemptTurnId) {
1887
1900
  await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => undefined);
1901
+ turnMetricOutcome = "cancelled";
1888
1902
  }
1889
1903
  throw error;
1890
1904
  }
@@ -1920,6 +1934,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1920
1934
  { type: "session.status.changed", payload: { status: "idle" } },
1921
1935
  ], true);
1922
1936
  await finishTurn(db, input.workspaceId, turnId, "idle");
1937
+ turnMetricOutcome = "completed";
1923
1938
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
1924
1939
  await recordUsageEvent(db, {
1925
1940
  accountId: input.accountId,
@@ -2044,6 +2059,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2044
2059
  { type: "session.status.changed", payload: { status: "idle" } },
2045
2060
  ], true);
2046
2061
  await finishTurn(db, input.workspaceId, turnId, "failed");
2062
+ turnMetricOutcome = "failed";
2047
2063
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2048
2064
  activityStatus = "idle";
2049
2065
  activityError = error;
@@ -2096,6 +2112,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2096
2112
  { type: "session.status.changed", payload: { status: "idle" } },
2097
2113
  ], true);
2098
2114
  await finishTurn(db, input.workspaceId, turnId, "idle");
2115
+ turnMetricOutcome = "completed";
2099
2116
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2100
2117
  await recordUsageEvent(db, {
2101
2118
  accountId: input.accountId,
@@ -2146,6 +2163,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2146
2163
  { type: "session.status.changed", payload: { status: "idle" } },
2147
2164
  ], true);
2148
2165
  await finishTurn(db, input.workspaceId, turnId, "failed");
2166
+ turnMetricOutcome = "failed";
2149
2167
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2150
2168
  activityStatus = "idle";
2151
2169
  activityError = error;
@@ -2161,6 +2179,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2161
2179
  { type: "session.status.changed", payload: { status: "failed" } },
2162
2180
  ], true);
2163
2181
  await finishTurn(db, input.workspaceId, turnId, "failed");
2182
+ turnMetricOutcome = "failed";
2164
2183
  await setSessionStatus(db, input.workspaceId, input.sessionId, "failed", null);
2165
2184
  // The common failure path ends here: runAgentTurn marks the session
2166
2185
  // failed and returns "failed", and the session workflow then exits
@@ -2178,6 +2197,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2178
2197
  status: activityStatus,
2179
2198
  durationSeconds,
2180
2199
  });
2200
+ if (turnId && activityStatus !== "unknown") {
2201
+ turnLifecycleMetricsFor(observability).finish(turnId, turnMetricOutcome, durationSeconds);
2202
+ }
2181
2203
  activitySpan.end({
2182
2204
  attributes: {
2183
2205
  "opengeni.turn_id": turnId ?? "",
@@ -2469,6 +2491,7 @@ export async function recordModelUsageAndDebitCredits(settings: Settings, db: Ac
2469
2491
  isCodexTurn: boolean;
2470
2492
  usage?: ModelUsageInput | null;
2471
2493
  sourceKey: string;
2494
+ observability?: ActivityServices["observability"];
2472
2495
  }): Promise<void> {
2473
2496
  if (!input.usage) {
2474
2497
  return;
@@ -2529,7 +2552,7 @@ export async function recordModelUsageAndDebitCredits(settings: Settings, db: Ac
2529
2552
  idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`,
2530
2553
  });
2531
2554
  if (costMicros > 0) {
2532
- await applyCreditDebitUpToBalance(db, {
2555
+ const result = await applyCreditDebitUpToBalance(db, {
2533
2556
  accountId: input.accountId,
2534
2557
  workspaceId: input.workspaceId,
2535
2558
  type: "model_usage_debit",
@@ -2547,6 +2570,7 @@ export async function recordModelUsageAndDebitCredits(settings: Settings, db: Ac
2547
2570
  totalTokens,
2548
2571
  },
2549
2572
  });
2573
+ recordCreditMicros(input.observability, "usage", result.debitedMicros);
2550
2574
  }
2551
2575
  }
2552
2576
 
@@ -33,6 +33,8 @@
33
33
  import {
34
34
  accrueWarmSeconds,
35
35
  confirmDrainCold,
36
+ countQueuedTurns,
37
+ countSandboxLeasesByLiveness,
36
38
  forceDrainOverLimitViewerOnlyBoxes,
37
39
  getBillingBalance,
38
40
  listLiveModalSandboxLeaseAttributions,
@@ -60,6 +62,12 @@ import {
60
62
  terminateModalSandboxById,
61
63
  } from "@opengeni/runtime";
62
64
  import type { ActivityServices } from "./types";
65
+ import {
66
+ recordCreditMicros,
67
+ recordSandboxLeaseGauges,
68
+ recordSandboxOrphansTerminated,
69
+ recordTurnsQueuedGauge,
70
+ } from "../observability-metrics";
63
71
 
64
72
  export type ReapSandboxLeasesResult = {
65
73
  /** Stale viewer holders + warming-death rows the sweep touched is folded into
@@ -166,6 +174,7 @@ export function createSandboxLeaseActivities(
166
174
  if (!settings.sandboxOwnershipEnabled) {
167
175
  return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
168
176
  }
177
+ await refreshQueueAndLeaseGauges(db, observability);
169
178
 
170
179
  // (0) Warm-meter tick (P2.1) — accrue warm-seconds for every WARM viewer-only
171
180
  // box (turn-held boxes meter on the turn heartbeat, so the list fn excludes
@@ -213,12 +222,15 @@ export function createSandboxLeaseActivities(
213
222
 
214
223
  try {
215
224
  modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
225
+ recordSandboxOrphansTerminated(observability, modalOrphansTerminated);
216
226
  } catch (error) {
217
227
  observability.warn("sandbox reaper: Modal orphan sweep failed", {
218
228
  error: error instanceof Error ? error.message : String(error),
219
229
  });
220
230
  }
221
231
 
232
+ await refreshQueueAndLeaseGauges(db, observability);
233
+
222
234
  if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
223
235
  observability.info("sandbox reaper swept", {
224
236
  drainable: drainable.length,
@@ -275,6 +287,7 @@ async function accrueWarmTick(
275
287
  if (result.accrued) {
276
288
  accrued += 1;
277
289
  }
290
+ recordCreditMicros(observability, "usage", result.costMicros);
278
291
  } catch (error) {
279
292
  observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
280
293
  workspaceId: lease.workspaceId,
@@ -286,6 +299,26 @@ async function accrueWarmTick(
286
299
  return { accrued, workspaceIds };
287
300
  }
288
301
 
302
+ async function refreshQueueAndLeaseGauges(
303
+ db: ActivityServices["db"],
304
+ observability: ActivityServices["observability"],
305
+ ): Promise<void> {
306
+ try {
307
+ recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
308
+ } catch (error) {
309
+ observability.warn("sandbox reaper: queued-turn gauge refresh failed", {
310
+ error: error instanceof Error ? error.message : String(error),
311
+ });
312
+ }
313
+ try {
314
+ recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
315
+ } catch (error) {
316
+ observability.warn("sandbox reaper: sandbox-lease gauge refresh failed", {
317
+ error: error instanceof Error ? error.message : String(error),
318
+ });
319
+ }
320
+ }
321
+
289
322
  /**
290
323
  * The per-workspace warm-cap + force-drain pass (P2.1). For each workspace with a
291
324
  * live warm box, under the usage lock: if it is at 0 balance (when a billing /
@@ -1,5 +1,6 @@
1
1
  import {
2
2
  claimNextQueuedTurn as claimNextQueuedTurnDb,
3
+ countQueuedTurns,
3
4
  countTurnSessionHistoryItems,
4
5
  finishTurn,
5
6
  getSessionEvent,
@@ -13,6 +14,7 @@ import { appendAndPublishEvents } from "@opengeni/events";
13
14
  import { WORKER_DEATH_RESUME_TEXT } from "./agent-turn";
14
15
  import { isSteerInterrupt, pauseActiveGoalOnInterrupt } from "./goals";
15
16
  import { notifyParentOfChildTerminal } from "./parent-wake";
17
+ import { recordTurnsQueuedGauge } from "../observability-metrics";
16
18
  import type {
17
19
  ActivityServices,
18
20
  ClaimNextQueuedTurnInput,
@@ -67,7 +69,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
67
69
  }
68
70
 
69
71
  async function interruptActiveTurn(input: RunAgentTurnInput): Promise<void> {
70
- const { db, bus } = await services();
72
+ const { db, bus, observability } = await services();
71
73
  const session = await requireSession(db, input.workspaceId, input.sessionId);
72
74
  const trigger = await getSessionEvent(db, input.workspaceId, input.triggerEventId);
73
75
  // Pause an active goal before the early return below: an interrupt can
@@ -96,6 +98,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
96
98
  ]);
97
99
  await finishTurn(db, input.workspaceId, session.activeTurnId, "cancelled");
98
100
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null);
101
+ await refreshQueuedTurnsGauge(db, observability);
99
102
  }
100
103
 
101
104
  /**
@@ -116,7 +119,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
116
119
  * WORKER_DEATH_MAX_REDISPATCHES, persisted on the turn row.
117
120
  */
118
121
  async function requeueTurnAfterWorkerDeath(input: RequeueTurnAfterWorkerDeathInput): Promise<RequeueTurnAfterWorkerDeathResult> {
119
- const { settings, db, bus } = await services();
122
+ const { settings, db, bus, observability } = await services();
120
123
  const turn = await getSessionTurn(db, input.workspaceId, input.turnId);
121
124
  if (!turn || (turn.status !== "running" && turn.status !== "requires_action")) {
122
125
  // The timed-out attempt was a zombie that actually settled the turn
@@ -169,12 +172,15 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
169
172
  throw requeueError;
170
173
  }
171
174
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null);
175
+ await refreshQueuedTurnsGauge(db, observability);
172
176
  return { action: "requeued", redispatches };
173
177
  }
174
178
 
175
179
  async function claimNextQueuedTurn(input: ClaimNextQueuedTurnInput) {
176
- const { db } = await services();
177
- return await claimNextQueuedTurnDb(db, input.workspaceId, input.sessionId, input.workflowId);
180
+ const { db, observability } = await services();
181
+ const turn = await claimNextQueuedTurnDb(db, input.workspaceId, input.sessionId, input.workflowId);
182
+ await refreshQueuedTurnsGauge(db, observability);
183
+ return turn;
178
184
  }
179
185
 
180
186
  async function markSessionIdle(input: MarkSessionIdleInput): Promise<void> {
@@ -183,6 +189,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
183
189
  if (session.status === "queued" || session.status === "running") {
184
190
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
185
191
  }
192
+ await refreshQueuedTurnsGauge(db, observability);
186
193
  // The workflow reaches markSessionIdle exactly when it has decided to stop
187
194
  // for now (no queued turn, no goal continuation): the terminal-for-now
188
195
  // point for a spawned worker, whatever the cause (goal completed, agent or
@@ -199,3 +206,14 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
199
206
  markSessionIdle,
200
207
  };
201
208
  }
209
+
210
+ async function refreshQueuedTurnsGauge(
211
+ db: ActivityServices["db"],
212
+ observability: ActivityServices["observability"],
213
+ ): Promise<void> {
214
+ try {
215
+ recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
216
+ } catch {
217
+ // Best-effort telemetry; session state transitions remain authoritative.
218
+ }
219
+ }
package/src/activities.ts CHANGED
@@ -12,6 +12,7 @@ import { createSandboxLeaseActivities } from "./activities/sandbox-lease";
12
12
  import { createScheduledTaskActivities } from "./activities/scheduled-tasks";
13
13
  import { createSessionStateActivities } from "./activities/session-state";
14
14
  import type { ActivityDependencies, ActivityServices } from "./activities/types";
15
+ import { observabilityEventLogger, runtimeMetricsHooksForObservability } from "./observability-metrics";
15
16
 
16
17
  export type {
17
18
  ActivityDependencies,
@@ -33,6 +34,7 @@ export function createActivities(dependencies: ActivityDependencies = {}) {
33
34
  async function services(): Promise<ActivityServices> {
34
35
  servicesPromise ??= (async () => {
35
36
  const settings = dependencies.settings ?? getSettings();
37
+ const observability = dependencies.observability ?? createObservability(settings, { component: "worker" });
36
38
  // Step I: when not injected, build the standalone handle — searchPath
37
39
  // undefined for standalone (public), scoped to the dedicated schema +
38
40
  // host RLS strategy when embedded config is set. An embedded host injects
@@ -67,11 +69,14 @@ export function createActivities(dependencies: ActivityDependencies = {}) {
67
69
  (await createNatsEventBus(
68
70
  settings.natsUrl,
69
71
  controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : undefined,
72
+ { logger: observabilityEventLogger(observability) },
70
73
  )),
71
- runtime: dependencies.runtime ?? createProductionAgentRuntime(),
74
+ runtime: dependencies.runtime ?? createProductionAgentRuntime({
75
+ metrics: runtimeMetricsHooksForObservability(observability),
76
+ }),
72
77
  objectStorage: dependencies.objectStorage ?? createObjectStorage(settings),
73
78
  documentServices: dependencies.documentServices ?? createDocumentServices(settings),
74
- observability: dependencies.observability ?? createObservability(settings, { component: "worker" }),
79
+ observability,
75
80
  wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
76
81
  // §7.5 P3 — host-entitlements port. No constructed default: standalone
77
82
  // has no host meter, so unset → null → `ensureRunAllowed` reads the
package/src/http.ts ADDED
@@ -0,0 +1,100 @@
1
+ import type { Settings } from "@opengeni/config";
2
+ import { dbSql, type Database } from "@opengeni/db";
3
+ import type { EventBus } from "@opengeni/events";
4
+ import type { Observability } from "@opengeni/observability";
5
+
6
+ export type ReadinessCheckName = "db" | "nats" | "temporal";
7
+ export type ReadinessChecks = Record<ReadinessCheckName, () => Promise<void> | void>;
8
+
9
+ export type ReadinessResult = {
10
+ ok: boolean;
11
+ checks: Record<ReadinessCheckName, { ok: boolean; error?: string }>;
12
+ };
13
+
14
+ export function startWorkerHttpServer(input: {
15
+ settings: Settings;
16
+ observability: Observability;
17
+ checks: ReadinessChecks;
18
+ timeoutMs?: number;
19
+ }): ReturnType<typeof Bun.serve> {
20
+ const { settings, observability, checks } = input;
21
+ const timeoutMs = input.timeoutMs ?? 2_000;
22
+ return Bun.serve({
23
+ hostname: "0.0.0.0",
24
+ port: settings.workerHttpPort,
25
+ async fetch(request) {
26
+ const url = new URL(request.url);
27
+ if (request.method !== "GET") {
28
+ return Response.json({ error: "method_not_allowed" }, { status: 405 });
29
+ }
30
+ if (url.pathname === "/healthz") {
31
+ return Response.json({
32
+ service: settings.serviceName,
33
+ environment: settings.environment,
34
+ deploymentRevision: settings.deploymentRevision,
35
+ ok: true,
36
+ });
37
+ }
38
+ if (url.pathname === "/metrics") {
39
+ return new Response(await observability.prometheusMetrics(), {
40
+ status: 200,
41
+ headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" },
42
+ });
43
+ }
44
+ if (url.pathname === "/readyz") {
45
+ const result = await runReadinessChecks(checks, timeoutMs);
46
+ return Response.json(result, { status: result.ok ? 200 : 503 });
47
+ }
48
+ return Response.json({ error: "not_found" }, { status: 404 });
49
+ },
50
+ });
51
+ }
52
+
53
+ export async function runReadinessChecks(checks: ReadinessChecks, timeoutMs = 2_000): Promise<ReadinessResult> {
54
+ const entries = await Promise.all(
55
+ (Object.entries(checks) as Array<[ReadinessCheckName, () => Promise<void> | void]>)
56
+ .map(async ([name, check]) => {
57
+ try {
58
+ await withTimeout(Promise.resolve().then(check), timeoutMs);
59
+ return [name, { ok: true }] as const;
60
+ } catch (error) {
61
+ return [name, { ok: false, error: error instanceof Error ? error.message : String(error) }] as const;
62
+ }
63
+ }),
64
+ );
65
+ const result = Object.fromEntries(entries) as ReadinessResult["checks"];
66
+ return {
67
+ ok: Object.values(result).every((check) => check.ok),
68
+ checks: result,
69
+ };
70
+ }
71
+
72
+ export function dbReadyCheck(db: Database): () => Promise<void> {
73
+ return async () => {
74
+ await db.execute(dbSql`select 1`);
75
+ };
76
+ }
77
+
78
+ export function natsReadyCheck(bus: EventBus): () => void {
79
+ return () => {
80
+ if (bus.isConnected && !bus.isConnected()) {
81
+ throw new Error("NATS is not connected");
82
+ }
83
+ };
84
+ }
85
+
86
+ async function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
87
+ let timer: ReturnType<typeof setTimeout> | undefined;
88
+ try {
89
+ return await Promise.race([
90
+ promise,
91
+ new Promise<never>((_, reject) => {
92
+ timer = setTimeout(() => reject(new Error(`readiness check timed out after ${timeoutMs}ms`)), timeoutMs);
93
+ }),
94
+ ]);
95
+ } finally {
96
+ if (timer) {
97
+ clearTimeout(timer);
98
+ }
99
+ }
100
+ }
package/src/index.ts CHANGED
@@ -1,9 +1,13 @@
1
- import { getSettings, retryStartupDependency, startupRetryOptions, type Settings } from "@opengeni/config";
1
+ import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth, retryStartupDependency, startupRetryOptions, type Settings } from "@opengeni/config";
2
+ import { createDb } from "@opengeni/db";
3
+ import { createNatsEventBus } from "@opengeni/events";
2
4
  import { createObservability, logStartupDependencyRetry, type Observability } from "@opengeni/observability";
3
5
  import { Connection, ScheduleAlreadyRunning, ScheduleOverlapPolicy, Client as TemporalClient } from "@temporalio/client";
4
6
  import { NativeConnection, Worker } from "@temporalio/worker";
5
7
  import { createActivities, type ActivityDependencies } from "./activities";
6
8
  import type { WakeSessionWorkflowSignal } from "./activities/types";
9
+ import { dbReadyCheck, natsReadyCheck, startWorkerHttpServer } from "./http";
10
+ import { observabilityEventLogger } from "./observability-metrics";
7
11
 
8
12
  // The deterministic id of the ONE global reaper Schedule. A single id means
9
13
  // create() is idempotent across every worker in the pool: the first worker to
@@ -57,7 +61,11 @@ export async function createOpenGeniWorker(options: WorkerOptions = {}): Promise
57
61
  // idled and let its run finish, so a plain signal would not start one).
58
62
  // Separate from the worker's NativeConnection: the @temporalio/client
59
63
  // Connection is what exposes workflow.signalWithStart.
60
- export async function createWorkerWorkflowSignaler(settings: Settings): Promise<{ wakeSessionWorkflow: WakeSessionWorkflowSignal; close: () => Promise<void> }> {
64
+ export async function createWorkerWorkflowSignaler(settings: Settings): Promise<{
65
+ wakeSessionWorkflow: WakeSessionWorkflowSignal;
66
+ check: () => Promise<void>;
67
+ close: () => Promise<void>;
68
+ }> {
61
69
  const connection = await Connection.connect({ address: settings.temporalHost });
62
70
  const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
63
71
  return {
@@ -70,6 +78,9 @@ export async function createWorkerWorkflowSignaler(settings: Settings): Promise<
70
78
  signal: "queueChanged",
71
79
  });
72
80
  },
81
+ check: async () => {
82
+ await connection.workflowService.getSystemInfo({});
83
+ },
73
84
  close: async () => {
74
85
  await connection.close();
75
86
  },
@@ -148,39 +159,76 @@ export async function registerSandboxReaperSchedule(
148
159
  export async function startWorker() {
149
160
  const settings = getSettings();
150
161
  const observability = createObservability(settings, { component: "worker" });
151
- const signaler = await retryStartupDependency(
152
- "Temporal client",
153
- () => createWorkerWorkflowSignaler(settings),
154
- {
155
- ...startupRetryOptions(settings),
156
- onRetry: (event) => logStartupDependencyRetry(observability, event),
157
- },
158
- );
159
- const { worker, connection } = await createOpenGeniWorker({
160
- settings,
161
- activityDependencies: { observability, wakeSessionWorkflow: signaler.wakeSessionWorkflow },
162
- });
163
- // Register the ONE global reaper Schedule (no-op when sandboxOwnershipEnabled
164
- // is false). Idempotent across the pool: only the first worker creates it. The
165
- // static global-queue Worker.create above is UNCHANGED — there is no
166
- // per-session worker factory.
167
- const reaperSchedule = await retryStartupDependency(
168
- "Temporal schedule (sandbox reaper)",
169
- () => registerSandboxReaperSchedule(settings, observability),
170
- {
171
- ...startupRetryOptions(settings),
172
- onRetry: (event) => logStartupDependencyRetry(observability, event),
173
- },
174
- );
175
- observability.info("OpenGeni worker listening", {
176
- temporalTaskQueue: settings.temporalTaskQueue,
162
+ const retryOptions = startupRetryOptions(settings);
163
+ const onRetry = (event: Parameters<typeof logStartupDependencyRetry>[1]) => logStartupDependencyRetry(observability, event);
164
+ const searchPath = dbSearchPath(settings);
165
+ const dbClient = createDb(settings.databaseUrl, {
166
+ ...(searchPath ? { searchPath } : {}),
167
+ rlsStrategy: settings.rlsStrategy,
177
168
  });
169
+ const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
170
+ let bus: Awaited<ReturnType<typeof createNatsEventBus>> | undefined;
171
+ let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
172
+ let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
173
+ let reaperSchedule: Awaited<ReturnType<typeof registerSandboxReaperSchedule>> | undefined;
174
+ let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
178
175
  try {
179
- await worker.run();
176
+ bus = await retryStartupDependency(
177
+ "NATS",
178
+ () =>
179
+ createNatsEventBus(
180
+ settings.natsUrl,
181
+ controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : undefined,
182
+ { logger: observabilityEventLogger(observability) },
183
+ ),
184
+ { ...retryOptions, onRetry },
185
+ );
186
+ signaler = await retryStartupDependency(
187
+ "Temporal client",
188
+ () => createWorkerWorkflowSignaler(settings),
189
+ { ...retryOptions, onRetry },
190
+ );
191
+ workerBundle = await createOpenGeniWorker({
192
+ settings,
193
+ activityDependencies: {
194
+ observability,
195
+ wakeSessionWorkflow: signaler.wakeSessionWorkflow,
196
+ db: dbClient.db,
197
+ bus,
198
+ },
199
+ });
200
+ httpServer = startWorkerHttpServer({
201
+ settings,
202
+ observability,
203
+ checks: {
204
+ db: dbReadyCheck(dbClient.db),
205
+ nats: natsReadyCheck(bus),
206
+ temporal: signaler.check,
207
+ },
208
+ });
209
+ // Register the ONE global reaper Schedule (no-op when sandboxOwnershipEnabled
210
+ // is false). Idempotent across the pool: only the first worker creates it. The
211
+ // static global-queue Worker.create above is UNCHANGED — there is no
212
+ // per-session worker factory.
213
+ reaperSchedule = await retryStartupDependency(
214
+ "Temporal schedule (sandbox reaper)",
215
+ () => registerSandboxReaperSchedule(settings, observability),
216
+ { ...retryOptions, onRetry },
217
+ );
218
+ observability.info("OpenGeni worker listening", {
219
+ temporalTaskQueue: settings.temporalTaskQueue,
220
+ httpPort: settings.workerHttpPort,
221
+ });
222
+ await workerBundle.worker.run();
180
223
  } finally {
181
- await connection.close();
182
- await signaler.close();
183
- await reaperSchedule.close();
224
+ httpServer?.stop(true);
225
+ await Promise.allSettled([
226
+ workerBundle?.connection.close(),
227
+ signaler?.close(),
228
+ reaperSchedule?.close(),
229
+ bus?.close(),
230
+ dbClient.close(),
231
+ ]);
184
232
  }
185
233
  }
186
234