@opengeni/worker-bundle 0.3.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@opengeni/worker-bundle",
3
- "version": "0.3.0",
3
+ "version": "0.4.1",
4
4
  "description": "OpenGeni worker entry: runOpenGeniWorker/createOpenGeniWorker(deps) + the un-bundled Temporal workflow source tree. An engine-distribution surface — its runtime closure includes engine-internal packages and the workflow bundle ships as on-disk TS source.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -38,14 +38,14 @@
38
38
  },
39
39
  "dependencies": {
40
40
  "@opengeni/codex": "^0.2.1",
41
- "@opengeni/config": "^0.2.3",
42
- "@opengeni/contracts": "^0.5.0",
43
- "@opengeni/db": "^0.3.0",
44
- "@opengeni/documents": "^0.2.3",
45
- "@opengeni/events": "^0.2.3",
41
+ "@opengeni/config": "^0.2.5",
42
+ "@opengeni/contracts": "^0.7.0",
43
+ "@opengeni/db": "^0.4.1",
44
+ "@opengeni/documents": "^0.2.5",
45
+ "@opengeni/events": "^0.2.5",
46
46
  "@opengeni/observability": "^0.2.1",
47
- "@opengeni/runtime": "^0.2.3",
48
- "@opengeni/storage": "^0.2.3",
47
+ "@opengeni/runtime": "^0.3.1",
48
+ "@opengeni/storage": "^0.2.5",
49
49
  "@temporalio/activity": "^1.17.0",
50
50
  "@temporalio/client": "^1.17.0",
51
51
  "@temporalio/worker": "^1.17.0",
@@ -94,6 +94,7 @@ import type {
94
94
  } from "./types";
95
95
  import { resumeBoxForTurn, acquireSelfhostedLeaseForTurn, type ResumedTurnSandbox } from "../sandbox-resume";
96
96
  import { wrapTurnBoxWithRouting, establishSelfhostedTurnSession, routingEnabled } from "../sandbox-routing";
97
+ import { recordCreditMicros, runtimeMetricsHooksForObservability, turnLifecycleMetricsFor, type TurnOutcome } from "../observability-metrics";
97
98
  import { beginRecording, discardRecording, finalizeRecording, type ActiveRecording } from "./recording";
98
99
  import { createObjectStorage, type ObjectStorage } from "@opengeni/storage";
99
100
  import { desktopCapableBackend, sandboxRunAs } from "@opengeni/runtime";
@@ -510,7 +511,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
510
511
  "opengeni.workflow_id": input.workflowId,
511
512
  "opengeni.trigger_event_id": input.triggerEventId,
512
513
  });
513
- let activityStatus = "unknown";
514
+ let activityStatus: RunAgentTurnResult["status"] | "unknown" = "unknown";
515
+ let turnMetricOutcome: TurnOutcome | null = null;
514
516
  let activityError: unknown;
515
517
  let turnId: string | undefined;
516
518
  let heartbeatTimer: ReturnType<typeof startActivityHeartbeat> | undefined;
@@ -680,6 +682,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
680
682
  throw new Error(`Session turn not found for trigger: ${input.triggerEventId}`);
681
683
  }
682
684
  turnId = turn.id;
685
+ turnLifecycleMetricsFor(observability).start(turnId);
683
686
  // Canonical codex-billed predicate (codex/<slug> + feature enabled + active
684
687
  // workspace credential). Computed once and threaded through every billing
685
688
  // gate + the usage recorder so a turn paid by the user's ChatGPT/Codex plan
@@ -836,6 +839,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
836
839
  { type: "session.status.changed", payload: { status: "idle" } },
837
840
  ], true);
838
841
  await finishTurn(db, input.workspaceId, turnId, "failed");
842
+ turnMetricOutcome = "failed";
839
843
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
840
844
  activityStatus = "idle";
841
845
  // idleUntilReset marks this a MANDATORY hold: session.ts must wait the full
@@ -1100,7 +1104,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1100
1104
  };
1101
1105
  } else {
1102
1106
  resolvedSandbox = await resumeBoxForTurn(
1103
- { db, settings },
1107
+ { db, settings, sandboxMetrics: runtimeMetricsHooksForObservability(observability) },
1104
1108
  {
1105
1109
  accountId: input.accountId,
1106
1110
  workspaceId: input.workspaceId,
@@ -1192,7 +1196,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1192
1196
  expectedEpoch: heartbeatEpoch,
1193
1197
  warmRateMicrosPerSecond: warmRate,
1194
1198
  subjectId: input.sessionId,
1195
- }).catch(() => undefined);
1199
+ })
1200
+ .then((result) => recordCreditMicros(observability, "usage", result.costMicros))
1201
+ .catch(() => undefined);
1196
1202
  }, 10_000);
1197
1203
  if ("unref" in leaseHeartbeatTimer && typeof leaseHeartbeatTimer.unref === "function") {
1198
1204
  leaseHeartbeatTimer.unref();
@@ -1335,6 +1341,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1335
1341
  : { computerToolMode: computerToolModeForTurn(null) }),
1336
1342
  ...(packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {}),
1337
1343
  ...(workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {}),
1344
+ // Per-session persona tier (session > workspace > deployment default).
1345
+ // Composed system-level AFTER the workspace persona so it refines it for
1346
+ // this one session; absent ⇒ byte-identical to today's composition.
1347
+ ...(session.instructions ? { sessionInstructions: session.instructions } : {}),
1338
1348
  ...(workspaceEnvironment
1339
1349
  ? {
1340
1350
  workspaceEnvironment: {
@@ -1530,6 +1540,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1530
1540
  dispatchId,
1531
1541
  positionalKey: `response-${responseUsageCount}`,
1532
1542
  }),
1543
+ observability,
1533
1544
  });
1534
1545
  await reconcileConversationTruth();
1535
1546
  try {
@@ -1584,6 +1595,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1584
1595
  isCodexTurn,
1585
1596
  usage: stream.state.usage,
1586
1597
  sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
1598
+ observability,
1587
1599
  });
1588
1600
  }
1589
1601
  if (lastInputTokensObserved !== null) {
@@ -1636,6 +1648,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1636
1648
  { type: "session.status.changed", payload: { status: "idle" } },
1637
1649
  ], true);
1638
1650
  await finishTurn(db, input.workspaceId, activeTurnId, "idle");
1651
+ turnMetricOutcome = "completed";
1639
1652
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
1640
1653
  await recordUsageEvent(db, {
1641
1654
  accountId: input.accountId,
@@ -1731,6 +1744,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1731
1744
  await requeuePreemptedTurn(db, input.workspaceId, activeTurnId, preemptedEvent ? preemptedEvent.id : input.triggerEventId);
1732
1745
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
1733
1746
  activityStatus = "preempted";
1747
+ turnMetricOutcome = "preempted";
1734
1748
  observability.info("context compaction recovery succeeded by compacting and auto-continuing", {
1735
1749
  sessionId: input.sessionId,
1736
1750
  turnId: activeTurnId,
@@ -1753,6 +1767,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1753
1767
  { type: "session.status.changed", payload: { status: "idle" } },
1754
1768
  ], true);
1755
1769
  await finishTurn(db, input.workspaceId, activeTurnId, "failed");
1770
+ turnMetricOutcome = "failed";
1756
1771
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
1757
1772
  activityStatus = "idle";
1758
1773
  activityError = attemptError;
@@ -1804,6 +1819,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1804
1819
  await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, input.triggerEventId);
1805
1820
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
1806
1821
  activityStatus = "preempted";
1822
+ turnMetricOutcome = "preempted";
1807
1823
  return { status: "preempted" };
1808
1824
  } catch (requeueError) {
1809
1825
  console.error("sandbox lease supersession requeue failed; falling back", requeueError);
@@ -1864,6 +1880,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1864
1880
  await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
1865
1881
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
1866
1882
  activityStatus = "preempted";
1883
+ turnMetricOutcome = "preempted";
1867
1884
  return { status: "preempted" };
1868
1885
  } catch (preemptError) {
1869
1886
  // A failing checkpoint/requeue must not surface as an arbitrary
@@ -1881,6 +1898,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1881
1898
  await flushRuntimeBatcher();
1882
1899
  if (preemptTurnId) {
1883
1900
  await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => undefined);
1901
+ turnMetricOutcome = "cancelled";
1884
1902
  }
1885
1903
  throw error;
1886
1904
  }
@@ -1916,6 +1934,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1916
1934
  { type: "session.status.changed", payload: { status: "idle" } },
1917
1935
  ], true);
1918
1936
  await finishTurn(db, input.workspaceId, turnId, "idle");
1937
+ turnMetricOutcome = "completed";
1919
1938
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
1920
1939
  await recordUsageEvent(db, {
1921
1940
  accountId: input.accountId,
@@ -2040,6 +2059,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2040
2059
  { type: "session.status.changed", payload: { status: "idle" } },
2041
2060
  ], true);
2042
2061
  await finishTurn(db, input.workspaceId, turnId, "failed");
2062
+ turnMetricOutcome = "failed";
2043
2063
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2044
2064
  activityStatus = "idle";
2045
2065
  activityError = error;
@@ -2092,6 +2112,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2092
2112
  { type: "session.status.changed", payload: { status: "idle" } },
2093
2113
  ], true);
2094
2114
  await finishTurn(db, input.workspaceId, turnId, "idle");
2115
+ turnMetricOutcome = "completed";
2095
2116
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2096
2117
  await recordUsageEvent(db, {
2097
2118
  accountId: input.accountId,
@@ -2142,6 +2163,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2142
2163
  { type: "session.status.changed", payload: { status: "idle" } },
2143
2164
  ], true);
2144
2165
  await finishTurn(db, input.workspaceId, turnId, "failed");
2166
+ turnMetricOutcome = "failed";
2145
2167
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2146
2168
  activityStatus = "idle";
2147
2169
  activityError = error;
@@ -2157,6 +2179,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2157
2179
  { type: "session.status.changed", payload: { status: "failed" } },
2158
2180
  ], true);
2159
2181
  await finishTurn(db, input.workspaceId, turnId, "failed");
2182
+ turnMetricOutcome = "failed";
2160
2183
  await setSessionStatus(db, input.workspaceId, input.sessionId, "failed", null);
2161
2184
  // The common failure path ends here: runAgentTurn marks the session
2162
2185
  // failed and returns "failed", and the session workflow then exits
@@ -2174,6 +2197,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2174
2197
  status: activityStatus,
2175
2198
  durationSeconds,
2176
2199
  });
2200
+ if (turnId && activityStatus !== "unknown") {
2201
+ turnLifecycleMetricsFor(observability).finish(turnId, turnMetricOutcome, durationSeconds);
2202
+ }
2177
2203
  activitySpan.end({
2178
2204
  attributes: {
2179
2205
  "opengeni.turn_id": turnId ?? "",
@@ -2465,6 +2491,7 @@ export async function recordModelUsageAndDebitCredits(settings: Settings, db: Ac
2465
2491
  isCodexTurn: boolean;
2466
2492
  usage?: ModelUsageInput | null;
2467
2493
  sourceKey: string;
2494
+ observability?: ActivityServices["observability"];
2468
2495
  }): Promise<void> {
2469
2496
  if (!input.usage) {
2470
2497
  return;
@@ -2525,7 +2552,7 @@ export async function recordModelUsageAndDebitCredits(settings: Settings, db: Ac
2525
2552
  idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`,
2526
2553
  });
2527
2554
  if (costMicros > 0) {
2528
- await applyCreditDebitUpToBalance(db, {
2555
+ const result = await applyCreditDebitUpToBalance(db, {
2529
2556
  accountId: input.accountId,
2530
2557
  workspaceId: input.workspaceId,
2531
2558
  type: "model_usage_debit",
@@ -2543,6 +2570,7 @@ export async function recordModelUsageAndDebitCredits(settings: Settings, db: Ac
2543
2570
  totalTokens,
2544
2571
  },
2545
2572
  });
2573
+ recordCreditMicros(input.observability, "usage", result.debitedMicros);
2546
2574
  }
2547
2575
  }
2548
2576
 
@@ -33,8 +33,11 @@
33
33
  import {
34
34
  accrueWarmSeconds,
35
35
  confirmDrainCold,
36
+ countQueuedTurns,
37
+ countSandboxLeasesByLiveness,
36
38
  forceDrainOverLimitViewerOnlyBoxes,
37
39
  getBillingBalance,
40
+ listLiveModalSandboxLeaseAttributions,
38
41
  listMeterableWarmLeases,
39
42
  persistDrainSnapshot,
40
43
  readLease,
@@ -55,8 +58,16 @@ import {
55
58
  deletePriorPersistedSnapshot,
56
59
  deserializeSandboxSessionStateEnvelope,
57
60
  isProviderSandboxNotFoundError,
61
+ sweepModalOrphanSandboxes,
62
+ terminateModalSandboxById,
58
63
  } from "@opengeni/runtime";
59
64
  import type { ActivityServices } from "./types";
65
+ import {
66
+ recordCreditMicros,
67
+ recordSandboxLeaseGauges,
68
+ recordSandboxOrphansTerminated,
69
+ recordTurnsQueuedGauge,
70
+ } from "../observability-metrics";
60
71
 
61
72
  export type ReapSandboxLeasesResult = {
62
73
  /** Stale viewer holders + warming-death rows the sweep touched is folded into
@@ -73,6 +84,8 @@ export type ReapSandboxLeasesResult = {
73
84
  /** Viewer-only boxes force-drained because their workspace is over a limit
74
85
  * (0 balance / over the warm cap) — turn-held boxes are never drained (P2.1). */
75
86
  forceDrained: number;
87
+ /** Provider-side Modal orphan sandboxes terminated by the defensive sweep. */
88
+ modalOrphansTerminated: number;
76
89
  };
77
90
 
78
91
  // The structural slice of a provider SandboxClient we need to resume-by-id and
@@ -129,10 +142,19 @@ export type TerminateBoxFn = (
129
142
  persistArchive: PersistArchiveFn,
130
143
  ) => Promise<boolean>;
131
144
 
145
+ export type SweepModalOrphansFn = (
146
+ settings: ActivityServices["settings"],
147
+ db: ActivityServices["db"],
148
+ observability: ActivityServices["observability"],
149
+ ) => Promise<number>;
150
+
132
151
  export type SandboxLeaseActivityOptions = {
133
152
  /** Override the provider terminate (tests spy this; defaults to the real
134
153
  * resume-by-id + provider stop()). */
135
154
  terminateBox?: TerminateBoxFn;
155
+ /** Override the provider-side Modal orphan sweep (tests spy this; defaults to
156
+ * Modal list+tag comparison when Modal is configured). */
157
+ sweepModalOrphans?: SweepModalOrphansFn;
136
158
  };
137
159
 
138
160
  export function createSandboxLeaseActivities(
@@ -140,6 +162,7 @@ export function createSandboxLeaseActivities(
140
162
  options: SandboxLeaseActivityOptions = {},
141
163
  ) {
142
164
  const terminateBox: TerminateBoxFn = options.terminateBox ?? terminateProviderBox;
165
+ const sweepModalOrphans: SweepModalOrphansFn = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
143
166
  /**
144
167
  * The one global reaper sweep. Idempotent; concurrency-safe with itself.
145
168
  * Gated by the caller (the Schedule is only registered when
@@ -149,8 +172,9 @@ export function createSandboxLeaseActivities(
149
172
  async function reapSandboxLeases(): Promise<ReapSandboxLeasesResult> {
150
173
  const { db, settings, observability } = await services();
151
174
  if (!settings.sandboxOwnershipEnabled) {
152
- return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0 };
175
+ return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
153
176
  }
177
+ await refreshQueueAndLeaseGauges(db, observability);
154
178
 
155
179
  // (0) Warm-meter tick (P2.1) — accrue warm-seconds for every WARM viewer-only
156
180
  // box (turn-held boxes meter on the turn heartbeat, so the list fn excludes
@@ -172,6 +196,7 @@ export function createSandboxLeaseActivities(
172
196
 
173
197
  let terminated = 0;
174
198
  let skipped = 0;
199
+ let modalOrphansTerminated = 0;
175
200
 
176
201
  // (2) Terminate each drainable box, then CAS draining->cold. Per-row failures
177
202
  // are isolated: one box's provider error must not abort the whole sweep (the
@@ -195,17 +220,29 @@ export function createSandboxLeaseActivities(
195
220
  }
196
221
  }
197
222
 
198
- if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0) {
223
+ try {
224
+ modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
225
+ recordSandboxOrphansTerminated(observability, modalOrphansTerminated);
226
+ } catch (error) {
227
+ observability.warn("sandbox reaper: Modal orphan sweep failed", {
228
+ error: error instanceof Error ? error.message : String(error),
229
+ });
230
+ }
231
+
232
+ await refreshQueueAndLeaseGauges(db, observability);
233
+
234
+ if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
199
235
  observability.info("sandbox reaper swept", {
200
236
  drainable: drainable.length,
201
237
  terminated,
202
238
  skipped,
203
239
  metered: metered.accrued,
204
240
  forceDrained,
241
+ modalOrphansTerminated,
205
242
  });
206
243
  }
207
244
 
208
- return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained };
245
+ return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained, modalOrphansTerminated };
209
246
  }
210
247
 
211
248
  return { reapSandboxLeases };
@@ -250,6 +287,7 @@ async function accrueWarmTick(
250
287
  if (result.accrued) {
251
288
  accrued += 1;
252
289
  }
290
+ recordCreditMicros(observability, "usage", result.costMicros);
253
291
  } catch (error) {
254
292
  observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
255
293
  workspaceId: lease.workspaceId,
@@ -261,6 +299,26 @@ async function accrueWarmTick(
261
299
  return { accrued, workspaceIds };
262
300
  }
263
301
 
302
+ async function refreshQueueAndLeaseGauges(
303
+ db: ActivityServices["db"],
304
+ observability: ActivityServices["observability"],
305
+ ): Promise<void> {
306
+ try {
307
+ recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
308
+ } catch (error) {
309
+ observability.warn("sandbox reaper: queued-turn gauge refresh failed", {
310
+ error: error instanceof Error ? error.message : String(error),
311
+ });
312
+ }
313
+ try {
314
+ recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
315
+ } catch (error) {
316
+ observability.warn("sandbox reaper: sandbox-lease gauge refresh failed", {
317
+ error: error instanceof Error ? error.message : String(error),
318
+ });
319
+ }
320
+ }
321
+
264
322
  /**
265
323
  * The per-workspace warm-cap + force-drain pass (P2.1). For each workspace with a
266
324
  * live warm box, under the usage lock: if it is at 0 balance (when a billing /
@@ -310,6 +368,46 @@ async function forceDrainOverLimitWorkspaces(
310
368
  return forceDrained;
311
369
  }
312
370
 
371
+ async function sweepModalOrphansForConfiguredBackend(
372
+ settings: ActivityServices["settings"],
373
+ db: ActivityServices["db"],
374
+ observability: ActivityServices["observability"],
375
+ ): Promise<number> {
376
+ // Keep the provider-side sweep tightly scoped to deployments that have a Modal
377
+ // app path configured. Local/docker-only workers should not attempt Modal API
378
+ // calls just because modalAppName has a default.
379
+ if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret) {
380
+ return 0;
381
+ }
382
+ let liveLeases: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>;
383
+ try {
384
+ liveLeases = await listLiveModalSandboxLeaseAttributions(db);
385
+ } catch (error) {
386
+ observability.warn("sandbox reaper: live Modal lease attribution read failed", {
387
+ error: error instanceof Error ? error.message : String(error),
388
+ });
389
+ return 0;
390
+ }
391
+
392
+ const result = await sweepModalOrphanSandboxes(settings, liveLeases);
393
+ for (const terminated of result.terminated) {
394
+ observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
395
+ sandboxId: terminated.sandboxId,
396
+ reason: terminated.reason,
397
+ tags: JSON.stringify(terminated.tags),
398
+ });
399
+ }
400
+ if (result.examined > 0) {
401
+ observability.info("sandbox reaper: Modal orphan sweep completed", {
402
+ examined: result.examined,
403
+ terminated: result.terminated.length,
404
+ skipped: result.skipped,
405
+ appName: settings.modalAppName,
406
+ });
407
+ }
408
+ return result.terminated.length;
409
+ }
410
+
313
411
  /**
314
412
  * Terminate one drainable box by id, then CAS its lease draining->cold under the
315
413
  * epoch fence. Returns true when the lease went cold (the box is ours to stop
@@ -458,6 +556,35 @@ export async function terminateProviderBox(
458
556
  return true;
459
557
  }
460
558
 
559
+ // A warming-death row can have a provider instance id recorded immediately
560
+ // after create() but no resumable envelope yet. For Modal, instance_id is
561
+ // enough to terminate directly; CAS-check first so a re-arm during the sweep
562
+ // leaves the box running.
563
+ if (backend === "modal" && !lease.resumeState && lease.instanceId) {
564
+ const { wrote } = await persistArchive(null);
565
+ if (!wrote) {
566
+ observability.info("sandbox reaper: Modal lease re-armed before direct terminate — leaving sandbox RUNNING", {
567
+ sandboxGroupId: lease.sandboxGroupId,
568
+ backend,
569
+ instanceId: lease.instanceId,
570
+ });
571
+ return false;
572
+ }
573
+ try {
574
+ await terminateModalSandboxById(settings, lease.instanceId);
575
+ } catch (error) {
576
+ if (!isProviderSandboxNotFoundError("modal", error)) {
577
+ throw error;
578
+ }
579
+ observability.info("sandbox reaper: Modal sandbox already gone during direct terminate", {
580
+ sandboxGroupId: lease.sandboxGroupId,
581
+ backend,
582
+ instanceId: lease.instanceId,
583
+ });
584
+ }
585
+ return true;
586
+ }
587
+
461
588
  // resume_state is the folded group box-envelope (the provider sessionState the
462
589
  // box was last persisted as). No envelope -> no live box to stop.
463
590
  if (!lease.resumeState) {
@@ -1,5 +1,6 @@
1
1
  import {
2
2
  claimNextQueuedTurn as claimNextQueuedTurnDb,
3
+ countQueuedTurns,
3
4
  countTurnSessionHistoryItems,
4
5
  finishTurn,
5
6
  getSessionEvent,
@@ -13,6 +14,7 @@ import { appendAndPublishEvents } from "@opengeni/events";
13
14
  import { WORKER_DEATH_RESUME_TEXT } from "./agent-turn";
14
15
  import { isSteerInterrupt, pauseActiveGoalOnInterrupt } from "./goals";
15
16
  import { notifyParentOfChildTerminal } from "./parent-wake";
17
+ import { recordTurnsQueuedGauge } from "../observability-metrics";
16
18
  import type {
17
19
  ActivityServices,
18
20
  ClaimNextQueuedTurnInput,
@@ -67,7 +69,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
67
69
  }
68
70
 
69
71
  async function interruptActiveTurn(input: RunAgentTurnInput): Promise<void> {
70
- const { db, bus } = await services();
72
+ const { db, bus, observability } = await services();
71
73
  const session = await requireSession(db, input.workspaceId, input.sessionId);
72
74
  const trigger = await getSessionEvent(db, input.workspaceId, input.triggerEventId);
73
75
  // Pause an active goal before the early return below: an interrupt can
@@ -96,6 +98,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
96
98
  ]);
97
99
  await finishTurn(db, input.workspaceId, session.activeTurnId, "cancelled");
98
100
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null);
101
+ await refreshQueuedTurnsGauge(db, observability);
99
102
  }
100
103
 
101
104
  /**
@@ -116,7 +119,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
116
119
  * WORKER_DEATH_MAX_REDISPATCHES, persisted on the turn row.
117
120
  */
118
121
  async function requeueTurnAfterWorkerDeath(input: RequeueTurnAfterWorkerDeathInput): Promise<RequeueTurnAfterWorkerDeathResult> {
119
- const { settings, db, bus } = await services();
122
+ const { settings, db, bus, observability } = await services();
120
123
  const turn = await getSessionTurn(db, input.workspaceId, input.turnId);
121
124
  if (!turn || (turn.status !== "running" && turn.status !== "requires_action")) {
122
125
  // The timed-out attempt was a zombie that actually settled the turn
@@ -169,12 +172,15 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
169
172
  throw requeueError;
170
173
  }
171
174
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null);
175
+ await refreshQueuedTurnsGauge(db, observability);
172
176
  return { action: "requeued", redispatches };
173
177
  }
174
178
 
175
179
  async function claimNextQueuedTurn(input: ClaimNextQueuedTurnInput) {
176
- const { db } = await services();
177
- return await claimNextQueuedTurnDb(db, input.workspaceId, input.sessionId, input.workflowId);
180
+ const { db, observability } = await services();
181
+ const turn = await claimNextQueuedTurnDb(db, input.workspaceId, input.sessionId, input.workflowId);
182
+ await refreshQueuedTurnsGauge(db, observability);
183
+ return turn;
178
184
  }
179
185
 
180
186
  async function markSessionIdle(input: MarkSessionIdleInput): Promise<void> {
@@ -183,6 +189,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
183
189
  if (session.status === "queued" || session.status === "running") {
184
190
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
185
191
  }
192
+ await refreshQueuedTurnsGauge(db, observability);
186
193
  // The workflow reaches markSessionIdle exactly when it has decided to stop
187
194
  // for now (no queued turn, no goal continuation): the terminal-for-now
188
195
  // point for a spawned worker, whatever the cause (goal completed, agent or
@@ -199,3 +206,14 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
199
206
  markSessionIdle,
200
207
  };
201
208
  }
209
+
210
+ async function refreshQueuedTurnsGauge(
211
+ db: ActivityServices["db"],
212
+ observability: ActivityServices["observability"],
213
+ ): Promise<void> {
214
+ try {
215
+ recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
216
+ } catch {
217
+ // Best-effort telemetry; session state transitions remain authoritative.
218
+ }
219
+ }
package/src/activities.ts CHANGED
@@ -12,6 +12,7 @@ import { createSandboxLeaseActivities } from "./activities/sandbox-lease";
12
12
  import { createScheduledTaskActivities } from "./activities/scheduled-tasks";
13
13
  import { createSessionStateActivities } from "./activities/session-state";
14
14
  import type { ActivityDependencies, ActivityServices } from "./activities/types";
15
+ import { observabilityEventLogger, runtimeMetricsHooksForObservability } from "./observability-metrics";
15
16
 
16
17
  export type {
17
18
  ActivityDependencies,
@@ -33,6 +34,7 @@ export function createActivities(dependencies: ActivityDependencies = {}) {
33
34
  async function services(): Promise<ActivityServices> {
34
35
  servicesPromise ??= (async () => {
35
36
  const settings = dependencies.settings ?? getSettings();
37
+ const observability = dependencies.observability ?? createObservability(settings, { component: "worker" });
36
38
  // Step I: when not injected, build the standalone handle — searchPath
37
39
  // undefined for standalone (public), scoped to the dedicated schema +
38
40
  // host RLS strategy when embedded config is set. An embedded host injects
@@ -67,11 +69,14 @@ export function createActivities(dependencies: ActivityDependencies = {}) {
67
69
  (await createNatsEventBus(
68
70
  settings.natsUrl,
69
71
  controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : undefined,
72
+ { logger: observabilityEventLogger(observability) },
70
73
  )),
71
- runtime: dependencies.runtime ?? createProductionAgentRuntime(),
74
+ runtime: dependencies.runtime ?? createProductionAgentRuntime({
75
+ metrics: runtimeMetricsHooksForObservability(observability),
76
+ }),
72
77
  objectStorage: dependencies.objectStorage ?? createObjectStorage(settings),
73
78
  documentServices: dependencies.documentServices ?? createDocumentServices(settings),
74
- observability: dependencies.observability ?? createObservability(settings, { component: "worker" }),
79
+ observability,
75
80
  wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
76
81
  // §7.5 P3 — host-entitlements port. No constructed default: standalone
77
82
  // has no host meter, so unset → null → `ensureRunAllowed` reads the
package/src/http.ts ADDED
@@ -0,0 +1,100 @@
1
+ import type { Settings } from "@opengeni/config";
2
+ import { dbSql, type Database } from "@opengeni/db";
3
+ import type { EventBus } from "@opengeni/events";
4
+ import type { Observability } from "@opengeni/observability";
5
+
6
+ export type ReadinessCheckName = "db" | "nats" | "temporal";
7
+ export type ReadinessChecks = Record<ReadinessCheckName, () => Promise<void> | void>;
8
+
9
+ export type ReadinessResult = {
10
+ ok: boolean;
11
+ checks: Record<ReadinessCheckName, { ok: boolean; error?: string }>;
12
+ };
13
+
14
+ export function startWorkerHttpServer(input: {
15
+ settings: Settings;
16
+ observability: Observability;
17
+ checks: ReadinessChecks;
18
+ timeoutMs?: number;
19
+ }): ReturnType<typeof Bun.serve> {
20
+ const { settings, observability, checks } = input;
21
+ const timeoutMs = input.timeoutMs ?? 2_000;
22
+ return Bun.serve({
23
+ hostname: "0.0.0.0",
24
+ port: settings.workerHttpPort,
25
+ async fetch(request) {
26
+ const url = new URL(request.url);
27
+ if (request.method !== "GET") {
28
+ return Response.json({ error: "method_not_allowed" }, { status: 405 });
29
+ }
30
+ if (url.pathname === "/healthz") {
31
+ return Response.json({
32
+ service: settings.serviceName,
33
+ environment: settings.environment,
34
+ deploymentRevision: settings.deploymentRevision,
35
+ ok: true,
36
+ });
37
+ }
38
+ if (url.pathname === "/metrics") {
39
+ return new Response(await observability.prometheusMetrics(), {
40
+ status: 200,
41
+ headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" },
42
+ });
43
+ }
44
+ if (url.pathname === "/readyz") {
45
+ const result = await runReadinessChecks(checks, timeoutMs);
46
+ return Response.json(result, { status: result.ok ? 200 : 503 });
47
+ }
48
+ return Response.json({ error: "not_found" }, { status: 404 });
49
+ },
50
+ });
51
+ }
52
+
53
+ export async function runReadinessChecks(checks: ReadinessChecks, timeoutMs = 2_000): Promise<ReadinessResult> {
54
+ const entries = await Promise.all(
55
+ (Object.entries(checks) as Array<[ReadinessCheckName, () => Promise<void> | void]>)
56
+ .map(async ([name, check]) => {
57
+ try {
58
+ await withTimeout(Promise.resolve().then(check), timeoutMs);
59
+ return [name, { ok: true }] as const;
60
+ } catch (error) {
61
+ return [name, { ok: false, error: error instanceof Error ? error.message : String(error) }] as const;
62
+ }
63
+ }),
64
+ );
65
+ const result = Object.fromEntries(entries) as ReadinessResult["checks"];
66
+ return {
67
+ ok: Object.values(result).every((check) => check.ok),
68
+ checks: result,
69
+ };
70
+ }
71
+
72
+ export function dbReadyCheck(db: Database): () => Promise<void> {
73
+ return async () => {
74
+ await db.execute(dbSql`select 1`);
75
+ };
76
+ }
77
+
78
+ export function natsReadyCheck(bus: EventBus): () => void {
79
+ return () => {
80
+ if (bus.isConnected && !bus.isConnected()) {
81
+ throw new Error("NATS is not connected");
82
+ }
83
+ };
84
+ }
85
+
86
+ async function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
87
+ let timer: ReturnType<typeof setTimeout> | undefined;
88
+ try {
89
+ return await Promise.race([
90
+ promise,
91
+ new Promise<never>((_, reject) => {
92
+ timer = setTimeout(() => reject(new Error(`readiness check timed out after ${timeoutMs}ms`)), timeoutMs);
93
+ }),
94
+ ]);
95
+ } finally {
96
+ if (timer) {
97
+ clearTimeout(timer);
98
+ }
99
+ }
100
+ }