@opengeni/worker-bundle 0.3.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,5 +1,7 @@
1
1
  // src/index.ts
2
- import { getSettings as getSettings2, retryStartupDependency, startupRetryOptions } from "@opengeni/config";
2
+ import { dbSearchPath as dbSearchPath2, getSettings as getSettings2, resolveNatsControlPlaneAuth as resolveNatsControlPlaneAuth2, retryStartupDependency, startupRetryOptions } from "@opengeni/config";
3
+ import { createDb as createDb2 } from "@opengeni/db";
4
+ import { createNatsEventBus as createNatsEventBus2 } from "@opengeni/events";
3
5
  import { createObservability as createObservability2, logStartupDependencyRetry } from "@opengeni/observability";
4
6
  import { Connection, ScheduleAlreadyRunning, ScheduleOverlapPolicy, Client as TemporalClient } from "@temporalio/client";
5
7
  import { NativeConnection, Worker } from "@temporalio/worker";
@@ -1170,6 +1172,7 @@ import {
1170
1172
  failWarmingToCold,
1171
1173
  getSandboxSessionEnvelope as getSandboxSessionEnvelope2,
1172
1174
  readLease,
1175
+ recordWarmingSandboxCreated,
1173
1176
  releaseLeaseHolder,
1174
1177
  SandboxLeaseSupersededError
1175
1178
  } from "@opengeni/db";
@@ -1181,13 +1184,93 @@ import {
1181
1184
  DisplayStackError,
1182
1185
  DisplayStackUnsupportedError,
1183
1186
  buildStreamUrl,
1184
- StreamPortUnavailableError
1187
+ StreamPortUnavailableError,
1188
+ modalSandboxAttributionEnvironment,
1189
+ tagModalSandbox
1185
1190
  } from "@opengeni/runtime";
1186
1191
  import { DESKTOP_STREAM_PORT } from "@opengeni/contracts";
1192
+ var SandboxWarmingTimeoutError = class extends Error {
1193
+ constructor(backend, timeoutMs) {
1194
+ super(
1195
+ `Sandbox backend "${backend}" capacity or creation timed out after ${Math.ceil(timeoutMs / 1e3)}s while warming the sandbox lease. Please try again; if this persists, sandbox capacity may be exhausted.`
1196
+ );
1197
+ this.backend = backend;
1198
+ this.timeoutMs = timeoutMs;
1199
+ this.name = "SandboxWarmingTimeoutError";
1200
+ }
1201
+ backend;
1202
+ timeoutMs;
1203
+ code = "sandbox_warming_timeout";
1204
+ };
1187
1205
  var WARMING_POLL_INTERVAL_MS = 250;
1188
1206
  async function sleep(ms) {
1189
1207
  await new Promise((resolve) => setTimeout(resolve, ms));
1190
1208
  }
1209
+ async function terminateEstablishedSandbox(established) {
1210
+ if (!established) {
1211
+ return true;
1212
+ }
1213
+ const client = established.client;
1214
+ if (typeof client.delete === "function" && established.sessionState !== void 0) {
1215
+ try {
1216
+ await client.delete(established.sessionState);
1217
+ return true;
1218
+ } catch {
1219
+ return false;
1220
+ }
1221
+ }
1222
+ const session = established.session;
1223
+ try {
1224
+ if (session.terminate) {
1225
+ await session.terminate();
1226
+ return true;
1227
+ } else if (session.kill) {
1228
+ await session.kill();
1229
+ return true;
1230
+ } else if (session.close) {
1231
+ if (!session.closed) {
1232
+ await session.close();
1233
+ }
1234
+ return true;
1235
+ }
1236
+ return false;
1237
+ } catch {
1238
+ return false;
1239
+ }
1240
+ }
1241
+ function asSandboxWarmingError(error, backend, timeoutMs) {
1242
+ const message = error instanceof Error ? error.message : String(error);
1243
+ return /sandbox creation timed out|warming timed out|capacity.*timed out/i.test(message) ? new SandboxWarmingTimeoutError(backend, timeoutMs) : error;
1244
+ }
1245
+ function recordSandboxWarmingTimeout(metrics, error) {
1246
+ if (!(error instanceof SandboxWarmingTimeoutError)) {
1247
+ return;
1248
+ }
1249
+ try {
1250
+ metrics?.onSandboxWarmingTimeout?.({ backend: error.backend });
1251
+ } catch {
1252
+ }
1253
+ }
1254
+ function workspaceArchiveFromEnvelope(envelope) {
1255
+ const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
1256
+ const archive = sessionState?.workspaceArchive;
1257
+ return typeof archive === "string" && archive.length > 0 ? archive : null;
1258
+ }
1259
+ function preserveWorkspaceArchiveOnInterimResumeState(resumeState, archiveSource) {
1260
+ const archive = workspaceArchiveFromEnvelope(archiveSource);
1261
+ if (!archive) {
1262
+ return resumeState;
1263
+ }
1264
+ const existingSessionState = resumeState && typeof resumeState.sessionState === "object" && resumeState.sessionState !== null ? resumeState.sessionState : {};
1265
+ return {
1266
+ ...resumeState ?? {},
1267
+ ...resumeState?.backendId === void 0 && archiveSource?.backendId !== void 0 ? { backendId: archiveSource.backendId } : {},
1268
+ sessionState: {
1269
+ ...existingSessionState,
1270
+ workspaceArchive: archive
1271
+ }
1272
+ };
1273
+ }
1191
1274
  async function resumeBoxForTurn(services, ids, kind, holderId) {
1192
1275
  const { db, settings } = services;
1193
1276
  const os = ids.os ?? "linux";
@@ -1229,14 +1312,51 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
1229
1312
  }
1230
1313
  if (acquired.role === "spawner") {
1231
1314
  const expectedEpoch = acquired.lease.leaseEpoch;
1315
+ let createdEstablished = null;
1316
+ if (ids.environment && ids.backend === "modal") {
1317
+ Object.assign(ids.environment, modalSandboxAttributionEnvironment({
1318
+ leaseId: acquired.lease.id,
1319
+ workspaceId: ids.workspaceId,
1320
+ sandboxGroupId: ids.sandboxGroupId
1321
+ }));
1322
+ }
1232
1323
  try {
1233
1324
  const envelope = await getSandboxSessionEnvelope2(db, ids.workspaceId, ids.sessionId);
1234
1325
  const spawnEnvelope = acquired.lease.resumeState ?? envelope;
1235
1326
  const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
1236
1327
  sessionId: ids.sessionId,
1237
1328
  backendOverride: ids.backend,
1238
- ...ids.environment ? { environment: ids.environment } : {}
1329
+ ...ids.environment ? { environment: ids.environment } : {},
1330
+ ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {},
1331
+ onSandboxCreated: async (created) => {
1332
+ createdEstablished = created;
1333
+ const resumeEnvelope2 = preserveWorkspaceArchiveOnInterimResumeState(
1334
+ await serializeEstablishedSandboxEnvelope(created) ?? null,
1335
+ spawnEnvelope
1336
+ );
1337
+ const recorded = await recordWarmingSandboxCreated(db, {
1338
+ accountId: ids.accountId,
1339
+ workspaceId: ids.workspaceId,
1340
+ sandboxGroupId: ids.sandboxGroupId,
1341
+ expectedEpoch,
1342
+ instanceId: created.instanceId,
1343
+ resumeBackendId: created.backendId,
1344
+ resumeState: resumeEnvelope2,
1345
+ leaseTtlMs
1346
+ });
1347
+ if (!recorded.recorded) {
1348
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
1349
+ }
1350
+ if (created.backendId === "modal") {
1351
+ await tagModalSandbox(settings, created.instanceId, {
1352
+ leaseId: acquired.lease.id,
1353
+ workspaceId: ids.workspaceId,
1354
+ sandboxGroupId: ids.sandboxGroupId
1355
+ }).catch(() => void 0);
1356
+ }
1357
+ }
1239
1358
  });
1359
+ createdEstablished = established;
1240
1360
  await ensureDisplayStack(settings, established);
1241
1361
  const endpoint = await exposeStreamPort(settings, established);
1242
1362
  const resumeEnvelope = await serializeEstablishedSandboxEnvelope(established) ?? envelope ?? null;
@@ -1252,41 +1372,52 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
1252
1372
  leaseTtlMs
1253
1373
  });
1254
1374
  if (!committed.committed || !committed.lease) {
1375
+ await terminateEstablishedSandbox(established);
1255
1376
  await release();
1256
1377
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
1257
1378
  }
1258
1379
  return { established, leaseEpoch: committed.lease.leaseEpoch, release };
1259
1380
  } catch (error) {
1260
1381
  if (error instanceof SandboxLeaseSupersededError) {
1382
+ await terminateEstablishedSandbox(createdEstablished);
1383
+ await release();
1261
1384
  throw error;
1262
1385
  }
1263
- await failWarmingToCold(db, {
1264
- accountId: ids.accountId,
1265
- workspaceId: ids.workspaceId,
1266
- sandboxGroupId: ids.sandboxGroupId,
1267
- expectedEpoch
1268
- });
1386
+ const terminated = await terminateEstablishedSandbox(createdEstablished);
1387
+ if (terminated) {
1388
+ await failWarmingToCold(db, {
1389
+ accountId: ids.accountId,
1390
+ workspaceId: ids.workspaceId,
1391
+ sandboxGroupId: ids.sandboxGroupId,
1392
+ expectedEpoch
1393
+ });
1394
+ }
1269
1395
  await release();
1270
- throw error;
1396
+ const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
1397
+ recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
1398
+ throw warmingError;
1271
1399
  }
1272
1400
  }
1273
- let leaseEpoch = acquired.lease.leaseEpoch;
1274
- if (acquired.lease.liveness === "warming") {
1275
- leaseEpoch = (await waitForWarmOrReacquire(services, ids, kind, holderId)).leaseEpoch;
1276
- }
1277
1401
  try {
1402
+ let leaseEpoch = acquired.lease.leaseEpoch;
1403
+ if (acquired.lease.liveness === "warming") {
1404
+ leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
1405
+ }
1278
1406
  const live = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
1279
1407
  const envelope = live?.resumeState ?? await getSandboxSessionEnvelope2(db, ids.workspaceId, ids.sessionId);
1280
1408
  const established = await establishSandboxSessionFromEnvelope(settings, envelope, {
1281
1409
  sessionId: ids.sessionId,
1282
1410
  backendOverride: ids.backend,
1283
- ...ids.environment ? { environment: ids.environment } : {}
1411
+ ...ids.environment ? { environment: ids.environment } : {},
1412
+ ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
1284
1413
  });
1285
1414
  await ensureDisplayStack(settings, established);
1286
1415
  return { established, leaseEpoch, release };
1287
1416
  } catch (error) {
1288
1417
  await release();
1289
- throw error;
1418
+ const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
1419
+ recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
1420
+ throw warmingError;
1290
1421
  }
1291
1422
  }
1292
1423
  async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
@@ -1343,37 +1474,26 @@ async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
1343
1474
  }
1344
1475
  return { leaseEpoch: acquired.lease.leaseEpoch, release };
1345
1476
  }
1346
- async function waitForWarmOrReacquire(services, ids, kind, holderId) {
1477
+ async function waitForWarm(services, ids) {
1347
1478
  const { db, settings } = services;
1348
- const deadline = Date.now() + settings.sandboxLeaseWarmingTtlMs;
1479
+ const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
1349
1480
  while (Date.now() < deadline) {
1350
1481
  await sleep(WARMING_POLL_INTERVAL_MS);
1351
1482
  const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
1352
1483
  if (!lease) {
1353
- break;
1484
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
1485
+ }
1486
+ if (lease.liveness === "warm") {
1487
+ return { leaseEpoch: lease.leaseEpoch };
1354
1488
  }
1355
- if (lease.liveness === "warm" || lease.liveness === "draining") {
1356
- return { liveness: lease.liveness, leaseEpoch: lease.leaseEpoch };
1489
+ if (lease.liveness === "draining") {
1490
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
1357
1491
  }
1358
1492
  if (lease.liveness === "cold") {
1359
- break;
1493
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
1360
1494
  }
1361
1495
  }
1362
- const reacquired = await acquireLease(db, {
1363
- accountId: ids.accountId,
1364
- workspaceId: ids.workspaceId,
1365
- sandboxGroupId: ids.sandboxGroupId,
1366
- kind,
1367
- holderId,
1368
- subjectId: ids.sessionId,
1369
- backend: ids.backend,
1370
- os: ids.os ?? "linux",
1371
- leaseTtlMs: settings.sandboxLeaseTtlMs
1372
- });
1373
- if (reacquired.role === "fenced") {
1374
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, reacquired.lease.leaseEpoch);
1375
- }
1376
- return { liveness: reacquired.lease.liveness, leaseEpoch: reacquired.lease.leaseEpoch };
1496
+ throw new SandboxWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs);
1377
1497
  }
1378
1498
  async function ensureDisplayStack(settings, established) {
1379
1499
  if (!settings.sandboxDesktopEnabled) {
@@ -1524,6 +1644,201 @@ function routingEnabled(settings) {
1524
1644
  return settings.sandboxSelfhostedEnabled === true;
1525
1645
  }
1526
1646
 
1647
+ // src/observability-metrics.ts
1648
+ var turnTrackers = /* @__PURE__ */ new WeakMap();
1649
+ function observabilityEventLogger(observability) {
1650
+ return {
1651
+ debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),
1652
+ warn: (message, attributes) => observability.warn(message, eventAttributes(attributes))
1653
+ };
1654
+ }
1655
+ function eventAttributes(attributes) {
1656
+ if (!attributes) {
1657
+ return void 0;
1658
+ }
1659
+ const sanitized = {};
1660
+ for (const [key, value] of Object.entries(attributes)) {
1661
+ sanitized[key] = eventAttributeValue(value);
1662
+ }
1663
+ return sanitized;
1664
+ }
1665
+ function eventAttributeValue(value) {
1666
+ if (value === null || value === void 0 || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
1667
+ return value;
1668
+ }
1669
+ try {
1670
+ return JSON.stringify(value);
1671
+ } catch {
1672
+ return String(value);
1673
+ }
1674
+ }
1675
+ function runtimeMetricsHooksForObservability(observability) {
1676
+ return {
1677
+ onModelCall: ({ provider, outcome, durationSeconds }) => {
1678
+ observability.incrementCounter({
1679
+ name: "opengeni_model_calls_total",
1680
+ help: "Total model calls by provider and outcome.",
1681
+ labels: { provider, outcome }
1682
+ });
1683
+ observability.observeHistogram({
1684
+ name: "opengeni_model_call_duration_seconds",
1685
+ help: "Model call duration in seconds by provider.",
1686
+ labels: { provider },
1687
+ value: durationSeconds
1688
+ });
1689
+ },
1690
+ onSandboxCreate: ({ backend, outcome, durationSeconds }) => {
1691
+ observability.incrementCounter({
1692
+ name: "opengeni_sandbox_creates_total",
1693
+ help: "Total sandbox create attempts by backend and outcome.",
1694
+ labels: { backend, outcome }
1695
+ });
1696
+ observability.observeHistogram({
1697
+ name: "opengeni_sandbox_create_duration_seconds",
1698
+ help: "Sandbox create duration in seconds by backend.",
1699
+ labels: { backend },
1700
+ value: durationSeconds
1701
+ });
1702
+ },
1703
+ onSandboxWarmingTimeout: () => {
1704
+ observability.incrementCounter({
1705
+ name: "opengeni_sandbox_warming_timeouts_total",
1706
+ help: "Total sandbox warming timeouts."
1707
+ });
1708
+ }
1709
+ };
1710
+ }
1711
+ function turnLifecycleMetricsFor(observability) {
1712
+ const existing = turnTrackers.get(observability);
1713
+ if (existing) {
1714
+ return existing;
1715
+ }
1716
+ const tracker = new TurnLifecycleMetrics(observability);
1717
+ turnTrackers.set(observability, tracker);
1718
+ return tracker;
1719
+ }
1720
+ var TurnLifecycleMetrics = class {
1721
+ constructor(observability, options = {}) {
1722
+ this.observability = observability;
1723
+ this.options = options;
1724
+ }
1725
+ observability;
1726
+ options;
1727
+ startedTurns = /* @__PURE__ */ new Map();
1728
+ timer = null;
1729
+ start(turnId) {
1730
+ this.startedTurns.set(turnId, this.now());
1731
+ this.ensureTimer();
1732
+ this.refreshGauges();
1733
+ }
1734
+ finish(turnId, outcome, durationSeconds) {
1735
+ const startedAt = this.startedTurns.get(turnId);
1736
+ if (startedAt !== void 0) {
1737
+ this.startedTurns.delete(turnId);
1738
+ }
1739
+ if (outcome) {
1740
+ const observedDuration = durationSeconds ?? (startedAt === void 0 ? 0 : Math.max(0, (this.now() - startedAt) / 1e3));
1741
+ this.observability.incrementCounter({
1742
+ name: "opengeni_turns_total",
1743
+ help: "Total agent turns by terminal outcome.",
1744
+ labels: { outcome }
1745
+ });
1746
+ this.observability.observeHistogram({
1747
+ name: "opengeni_turn_duration_seconds",
1748
+ help: "Agent turn duration in seconds by terminal outcome.",
1749
+ labels: { outcome },
1750
+ value: observedDuration
1751
+ });
1752
+ }
1753
+ this.refreshGauges();
1754
+ if (this.startedTurns.size === 0) {
1755
+ this.stopTimer();
1756
+ }
1757
+ }
1758
+ refreshGauges() {
1759
+ this.observability.setGauge({
1760
+ name: "opengeni_turns_inflight",
1761
+ help: "Current number of in-flight agent turns in this worker process.",
1762
+ value: this.startedTurns.size
1763
+ });
1764
+ this.observability.setGauge({
1765
+ name: "opengeni_turn_oldest_inflight_age_seconds",
1766
+ help: "Age in seconds of the oldest in-flight agent turn in this worker process.",
1767
+ value: this.oldestInflightAgeSeconds()
1768
+ });
1769
+ }
1770
+ stop() {
1771
+ this.startedTurns.clear();
1772
+ this.refreshGauges();
1773
+ this.stopTimer();
1774
+ }
1775
+ oldestInflightAgeSeconds() {
1776
+ if (this.startedTurns.size === 0) {
1777
+ return 0;
1778
+ }
1779
+ let oldest = Number.POSITIVE_INFINITY;
1780
+ for (const startedAt of this.startedTurns.values()) {
1781
+ oldest = Math.min(oldest, startedAt);
1782
+ }
1783
+ return Math.max(0, (this.now() - oldest) / 1e3);
1784
+ }
1785
+ ensureTimer() {
1786
+ if (this.timer) {
1787
+ return;
1788
+ }
1789
+ this.timer = setInterval(() => this.refreshGauges(), this.options.refreshIntervalMs ?? 15e3);
1790
+ this.timer.unref?.();
1791
+ }
1792
+ stopTimer() {
1793
+ if (!this.timer) {
1794
+ return;
1795
+ }
1796
+ clearInterval(this.timer);
1797
+ this.timer = null;
1798
+ }
1799
+ now() {
1800
+ return this.options.now?.() ?? Date.now();
1801
+ }
1802
+ };
1803
+ function recordTurnsQueuedGauge(observability, value) {
1804
+ observability.setGauge({
1805
+ name: "opengeni_turns_queued",
1806
+ help: "Current number of queued session turns.",
1807
+ value
1808
+ });
1809
+ }
1810
+ function recordSandboxLeaseGauges(observability, counts) {
1811
+ for (const liveness of ["cold", "warming", "warm", "draining"]) {
1812
+ observability.setGauge({
1813
+ name: "opengeni_sandbox_leases",
1814
+ help: "Current sandbox leases by liveness state.",
1815
+ labels: { liveness },
1816
+ value: counts[liveness] ?? 0
1817
+ });
1818
+ }
1819
+ }
1820
+ function recordSandboxOrphansTerminated(observability, count) {
1821
+ if (count <= 0) {
1822
+ return;
1823
+ }
1824
+ observability.incrementCounter({
1825
+ name: "opengeni_sandbox_orphans_terminated_total",
1826
+ help: "Total provider-side orphan sandboxes terminated by defensive sweeps.",
1827
+ amount: count
1828
+ });
1829
+ }
1830
+ function recordCreditMicros(observability, kind, amountMicros) {
1831
+ if (!observability || amountMicros <= 0) {
1832
+ return;
1833
+ }
1834
+ observability.incrementCounter({
1835
+ name: "opengeni_credit_micros_total",
1836
+ help: "Total credit micros recorded by kind.",
1837
+ labels: { kind },
1838
+ amount: amountMicros
1839
+ });
1840
+ }
1841
+
1527
1842
  // src/activities/recording.ts
1528
1843
  import { deleteRecording, updateRecording } from "@opengeni/db";
1529
1844
  import {
@@ -1817,6 +2132,7 @@ function createRunAgentTurnActivity(services) {
1817
2132
  "opengeni.trigger_event_id": input.triggerEventId
1818
2133
  });
1819
2134
  let activityStatus = "unknown";
2135
+ let turnMetricOutcome = null;
1820
2136
  let activityError;
1821
2137
  let turnId;
1822
2138
  let heartbeatTimer;
@@ -1919,6 +2235,7 @@ function createRunAgentTurnActivity(services) {
1919
2235
  throw new Error(`Session turn not found for trigger: ${input.triggerEventId}`);
1920
2236
  }
1921
2237
  turnId = turn.id;
2238
+ turnLifecycleMetricsFor(observability).start(turnId);
1922
2239
  const isCodexTurn = await isCodexBilledTurn2({ db, settings, workspaceId: input.workspaceId, model: turn.model, active: codexSubscriptionActive });
1923
2240
  await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
1924
2241
  const activityContext = currentActivityContext();
@@ -2019,6 +2336,7 @@ function createRunAgentTurnActivity(services) {
2019
2336
  { type: "session.status.changed", payload: { status: "idle" } }
2020
2337
  ], true);
2021
2338
  await finishTurn(db, input.workspaceId, turnId, "failed");
2339
+ turnMetricOutcome = "failed";
2022
2340
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2023
2341
  activityStatus = "idle";
2024
2342
  return goalActive ? { status: "idle", continueDelayMs: resumeMs, idleUntilReset: true } : { status: "idle" };
@@ -2157,7 +2475,7 @@ function createRunAgentTurnActivity(services) {
2157
2475
  };
2158
2476
  } else {
2159
2477
  resolvedSandbox = await resumeBoxForTurn(
2160
- { db, settings },
2478
+ { db, settings, sandboxMetrics: runtimeMetricsHooksForObservability(observability) },
2161
2479
  {
2162
2480
  accountId: input.accountId,
2163
2481
  workspaceId: input.workspaceId,
@@ -2219,7 +2537,7 @@ function createRunAgentTurnActivity(services) {
2219
2537
  expectedEpoch: heartbeatEpoch,
2220
2538
  warmRateMicrosPerSecond: warmRate,
2221
2539
  subjectId: input.sessionId
2222
- }).catch(() => void 0);
2540
+ }).then((result) => recordCreditMicros(observability, "usage", result.costMicros)).catch(() => void 0);
2223
2541
  }, 1e4);
2224
2542
  if ("unref" in leaseHeartbeatTimer && typeof leaseHeartbeatTimer.unref === "function") {
2225
2543
  leaseHeartbeatTimer.unref();
@@ -2322,6 +2640,10 @@ function createRunAgentTurnActivity(services) {
2322
2640
  } : { computerToolMode: computerToolModeForTurn(null) },
2323
2641
  ...packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {},
2324
2642
  ...workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {},
2643
+ // Per-session persona tier (session > workspace > deployment default).
2644
+ // Composed system-level AFTER the workspace persona so it refines it for
2645
+ // this one session; absent ⇒ byte-identical to today's composition.
2646
+ ...session.instructions ? { sessionInstructions: session.instructions } : {},
2325
2647
  ...workspaceEnvironment ? {
2326
2648
  workspaceEnvironment: {
2327
2649
  name: workspaceEnvironment.name,
@@ -2460,7 +2782,8 @@ function createRunAgentTurnActivity(services) {
2460
2782
  responseId: responseUsage.responseId,
2461
2783
  dispatchId,
2462
2784
  positionalKey: `response-${responseUsageCount}`
2463
- })
2785
+ }),
2786
+ observability
2464
2787
  });
2465
2788
  await reconcileConversationTruth();
2466
2789
  try {
@@ -2506,7 +2829,8 @@ function createRunAgentTurnActivity(services) {
2506
2829
  model: turn.model,
2507
2830
  isCodexTurn,
2508
2831
  usage: stream.state.usage,
2509
- sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" })
2832
+ sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
2833
+ observability
2510
2834
  });
2511
2835
  }
2512
2836
  if (lastInputTokensObserved !== null) {
@@ -2554,6 +2878,7 @@ function createRunAgentTurnActivity(services) {
2554
2878
  { type: "session.status.changed", payload: { status: "idle" } }
2555
2879
  ], true);
2556
2880
  await finishTurn(db, input.workspaceId, activeTurnId, "idle");
2881
+ turnMetricOutcome = "completed";
2557
2882
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2558
2883
  await recordUsageEvent2(db, {
2559
2884
  accountId: input.accountId,
@@ -2634,6 +2959,7 @@ function createRunAgentTurnActivity(services) {
2634
2959
  await requeuePreemptedTurn(db, input.workspaceId, activeTurnId, preemptedEvent ? preemptedEvent.id : input.triggerEventId);
2635
2960
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
2636
2961
  activityStatus = "preempted";
2962
+ turnMetricOutcome = "preempted";
2637
2963
  observability.info("context compaction recovery succeeded by compacting and auto-continuing", {
2638
2964
  sessionId: input.sessionId,
2639
2965
  turnId: activeTurnId,
@@ -2656,6 +2982,7 @@ function createRunAgentTurnActivity(services) {
2656
2982
  { type: "session.status.changed", payload: { status: "idle" } }
2657
2983
  ], true);
2658
2984
  await finishTurn(db, input.workspaceId, activeTurnId, "failed");
2985
+ turnMetricOutcome = "failed";
2659
2986
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2660
2987
  activityStatus = "idle";
2661
2988
  activityError = attemptError;
@@ -2684,6 +3011,7 @@ function createRunAgentTurnActivity(services) {
2684
3011
  await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, input.triggerEventId);
2685
3012
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
2686
3013
  activityStatus = "preempted";
3014
+ turnMetricOutcome = "preempted";
2687
3015
  return { status: "preempted" };
2688
3016
  } catch (requeueError) {
2689
3017
  console.error("sandbox lease supersession requeue failed; falling back", requeueError);
@@ -2731,6 +3059,7 @@ function createRunAgentTurnActivity(services) {
2731
3059
  await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
2732
3060
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
2733
3061
  activityStatus = "preempted";
3062
+ turnMetricOutcome = "preempted";
2734
3063
  return { status: "preempted" };
2735
3064
  } catch (preemptError) {
2736
3065
  console.error("worker-shutdown preemption failed; falling back to cancellation", preemptError);
@@ -2742,6 +3071,7 @@ function createRunAgentTurnActivity(services) {
2742
3071
  await flushRuntimeBatcher();
2743
3072
  if (preemptTurnId) {
2744
3073
  await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => void 0);
3074
+ turnMetricOutcome = "cancelled";
2745
3075
  }
2746
3076
  throw error;
2747
3077
  }
@@ -2766,6 +3096,7 @@ function createRunAgentTurnActivity(services) {
2766
3096
  { type: "session.status.changed", payload: { status: "idle" } }
2767
3097
  ], true);
2768
3098
  await finishTurn(db, input.workspaceId, turnId, "idle");
3099
+ turnMetricOutcome = "completed";
2769
3100
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2770
3101
  await recordUsageEvent2(db, {
2771
3102
  accountId: input.accountId,
@@ -2855,6 +3186,7 @@ function createRunAgentTurnActivity(services) {
2855
3186
  { type: "session.status.changed", payload: { status: "idle" } }
2856
3187
  ], true);
2857
3188
  await finishTurn(db, input.workspaceId, turnId, "failed");
3189
+ turnMetricOutcome = "failed";
2858
3190
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2859
3191
  activityStatus = "idle";
2860
3192
  activityError = error;
@@ -2887,6 +3219,7 @@ function createRunAgentTurnActivity(services) {
2887
3219
  { type: "session.status.changed", payload: { status: "idle" } }
2888
3220
  ], true);
2889
3221
  await finishTurn(db, input.workspaceId, turnId, "idle");
3222
+ turnMetricOutcome = "completed";
2890
3223
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2891
3224
  await recordUsageEvent2(db, {
2892
3225
  accountId: input.accountId,
@@ -2925,6 +3258,7 @@ function createRunAgentTurnActivity(services) {
2925
3258
  { type: "session.status.changed", payload: { status: "idle" } }
2926
3259
  ], true);
2927
3260
  await finishTurn(db, input.workspaceId, turnId, "failed");
3261
+ turnMetricOutcome = "failed";
2928
3262
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2929
3263
  activityStatus = "idle";
2930
3264
  activityError = error;
@@ -2940,6 +3274,7 @@ function createRunAgentTurnActivity(services) {
2940
3274
  { type: "session.status.changed", payload: { status: "failed" } }
2941
3275
  ], true);
2942
3276
  await finishTurn(db, input.workspaceId, turnId, "failed");
3277
+ turnMetricOutcome = "failed";
2943
3278
  await setSessionStatus(db, input.workspaceId, input.sessionId, "failed", null);
2944
3279
  await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed", `turn:${turnId}`);
2945
3280
  return { status: "failed" };
@@ -2950,6 +3285,9 @@ function createRunAgentTurnActivity(services) {
2950
3285
  status: activityStatus,
2951
3286
  durationSeconds
2952
3287
  });
3288
+ if (turnId && activityStatus !== "unknown") {
3289
+ turnLifecycleMetricsFor(observability).finish(turnId, turnMetricOutcome, durationSeconds);
3290
+ }
2953
3291
  activitySpan.end({
2954
3292
  attributes: {
2955
3293
  "opengeni.turn_id": turnId ?? "",
@@ -3172,7 +3510,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
3172
3510
  idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`
3173
3511
  });
3174
3512
  if (costMicros > 0) {
3175
- await applyCreditDebitUpToBalance(db, {
3513
+ const result = await applyCreditDebitUpToBalance(db, {
3176
3514
  accountId: input.accountId,
3177
3515
  workspaceId: input.workspaceId,
3178
3516
  type: "model_usage_debit",
@@ -3190,6 +3528,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
3190
3528
  totalTokens
3191
3529
  }
3192
3530
  });
3531
+ recordCreditMicros(input.observability, "usage", result.debitedMicros);
3193
3532
  }
3194
3533
  }
3195
3534
  function positiveInt(value) {
@@ -3310,8 +3649,11 @@ function startOfUtcMonth3(date = /* @__PURE__ */ new Date()) {
3310
3649
  import {
3311
3650
  accrueWarmSeconds as accrueWarmSeconds2,
3312
3651
  confirmDrainCold,
3652
+ countQueuedTurns,
3653
+ countSandboxLeasesByLiveness,
3313
3654
  forceDrainOverLimitViewerOnlyBoxes,
3314
3655
  getBillingBalance as getBillingBalance4,
3656
+ listLiveModalSandboxLeaseAttributions,
3315
3657
  listMeterableWarmLeases,
3316
3658
  persistDrainSnapshot,
3317
3659
  readLease as readLease2,
@@ -3323,15 +3665,19 @@ import {
3323
3665
  createSandboxClientForBackend,
3324
3666
  deletePriorPersistedSnapshot,
3325
3667
  deserializeSandboxSessionStateEnvelope,
3326
- isProviderSandboxNotFoundError
3668
+ isProviderSandboxNotFoundError,
3669
+ sweepModalOrphanSandboxes,
3670
+ terminateModalSandboxById
3327
3671
  } from "@opengeni/runtime";
3328
3672
  function createSandboxLeaseActivities(services, options = {}) {
3329
3673
  const terminateBox = options.terminateBox ?? terminateProviderBox;
3674
+ const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
3330
3675
  async function reapSandboxLeases2() {
3331
3676
  const { db, settings, observability } = await services();
3332
3677
  if (!settings.sandboxOwnershipEnabled) {
3333
- return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0 };
3678
+ return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
3334
3679
  }
3680
+ await refreshQueueAndLeaseGauges(db, observability);
3335
3681
  const metered = await accrueWarmTick(db, settings, observability);
3336
3682
  const forceDrained = await forceDrainOverLimitWorkspaces(db, settings, metered.workspaceIds, observability);
3337
3683
  const drainable = await reapStaleLeaseHoldersGlobal(db, {
@@ -3340,6 +3686,7 @@ function createSandboxLeaseActivities(services, options = {}) {
3340
3686
  });
3341
3687
  let terminated = 0;
3342
3688
  let skipped = 0;
3689
+ let modalOrphansTerminated = 0;
3343
3690
  for (const row of drainable) {
3344
3691
  try {
3345
3692
  const drainedCold = await terminateDrainableBox(db, settings, row, observability, terminateBox);
@@ -3358,16 +3705,26 @@ function createSandboxLeaseActivities(services, options = {}) {
3358
3705
  });
3359
3706
  }
3360
3707
  }
3361
- if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0) {
3708
+ try {
3709
+ modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
3710
+ recordSandboxOrphansTerminated(observability, modalOrphansTerminated);
3711
+ } catch (error) {
3712
+ observability.warn("sandbox reaper: Modal orphan sweep failed", {
3713
+ error: error instanceof Error ? error.message : String(error)
3714
+ });
3715
+ }
3716
+ await refreshQueueAndLeaseGauges(db, observability);
3717
+ if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
3362
3718
  observability.info("sandbox reaper swept", {
3363
3719
  drainable: drainable.length,
3364
3720
  terminated,
3365
3721
  skipped,
3366
3722
  metered: metered.accrued,
3367
- forceDrained
3723
+ forceDrained,
3724
+ modalOrphansTerminated
3368
3725
  });
3369
3726
  }
3370
- return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained };
3727
+ return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained, modalOrphansTerminated };
3371
3728
  }
3372
3729
  return { reapSandboxLeases: reapSandboxLeases2 };
3373
3730
  }
@@ -3398,6 +3755,7 @@ async function accrueWarmTick(db, settings, observability) {
3398
3755
  if (result.accrued) {
3399
3756
  accrued += 1;
3400
3757
  }
3758
+ recordCreditMicros(observability, "usage", result.costMicros);
3401
3759
  } catch (error) {
3402
3760
  observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
3403
3761
  workspaceId: lease.workspaceId,
@@ -3408,6 +3766,22 @@ async function accrueWarmTick(db, settings, observability) {
3408
3766
  }
3409
3767
  return { accrued, workspaceIds };
3410
3768
  }
3769
+ async function refreshQueueAndLeaseGauges(db, observability) {
3770
+ try {
3771
+ recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
3772
+ } catch (error) {
3773
+ observability.warn("sandbox reaper: queued-turn gauge refresh failed", {
3774
+ error: error instanceof Error ? error.message : String(error)
3775
+ });
3776
+ }
3777
+ try {
3778
+ recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
3779
+ } catch (error) {
3780
+ observability.warn("sandbox reaper: sandbox-lease gauge refresh failed", {
3781
+ error: error instanceof Error ? error.message : String(error)
3782
+ });
3783
+ }
3784
+ }
3411
3785
  async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observability) {
3412
3786
  const enforceBalance = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
3413
3787
  const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
@@ -3443,6 +3817,37 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
3443
3817
  }
3444
3818
  return forceDrained;
3445
3819
  }
3820
+ async function sweepModalOrphansForConfiguredBackend(settings, db, observability) {
3821
+ if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret) {
3822
+ return 0;
3823
+ }
3824
+ let liveLeases;
3825
+ try {
3826
+ liveLeases = await listLiveModalSandboxLeaseAttributions(db);
3827
+ } catch (error) {
3828
+ observability.warn("sandbox reaper: live Modal lease attribution read failed", {
3829
+ error: error instanceof Error ? error.message : String(error)
3830
+ });
3831
+ return 0;
3832
+ }
3833
+ const result = await sweepModalOrphanSandboxes(settings, liveLeases);
3834
+ for (const terminated of result.terminated) {
3835
+ observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
3836
+ sandboxId: terminated.sandboxId,
3837
+ reason: terminated.reason,
3838
+ tags: JSON.stringify(terminated.tags)
3839
+ });
3840
+ }
3841
+ if (result.examined > 0) {
3842
+ observability.info("sandbox reaper: Modal orphan sweep completed", {
3843
+ examined: result.examined,
3844
+ terminated: result.terminated.length,
3845
+ skipped: result.skipped,
3846
+ appName: settings.modalAppName
3847
+ });
3848
+ }
3849
+ return result.terminated.length;
3850
+ }
3446
3851
  async function terminateDrainableBox(db, settings, row, observability, terminateBox) {
3447
3852
  const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
3448
3853
  const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId);
@@ -3483,6 +3888,30 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
3483
3888
  });
3484
3889
  return true;
3485
3890
  }
3891
+ if (backend === "modal" && !lease.resumeState && lease.instanceId) {
3892
+ const { wrote } = await persistArchive(null);
3893
+ if (!wrote) {
3894
+ observability.info("sandbox reaper: Modal lease re-armed before direct terminate \u2014 leaving sandbox RUNNING", {
3895
+ sandboxGroupId: lease.sandboxGroupId,
3896
+ backend,
3897
+ instanceId: lease.instanceId
3898
+ });
3899
+ return false;
3900
+ }
3901
+ try {
3902
+ await terminateModalSandboxById(settings, lease.instanceId);
3903
+ } catch (error) {
3904
+ if (!isProviderSandboxNotFoundError("modal", error)) {
3905
+ throw error;
3906
+ }
3907
+ observability.info("sandbox reaper: Modal sandbox already gone during direct terminate", {
3908
+ sandboxGroupId: lease.sandboxGroupId,
3909
+ backend,
3910
+ instanceId: lease.instanceId
3911
+ });
3912
+ }
3913
+ return true;
3914
+ }
3486
3915
  if (!lease.resumeState) {
3487
3916
  return true;
3488
3917
  }
@@ -3883,6 +4312,7 @@ function startOfUtcMonth4() {
3883
4312
  // src/activities/session-state.ts
3884
4313
  import {
3885
4314
  claimNextQueuedTurn as claimNextQueuedTurnDb2,
4315
+ countQueuedTurns as countQueuedTurns2,
3886
4316
  countTurnSessionHistoryItems,
3887
4317
  finishTurn as finishTurn2,
3888
4318
  getSessionEvent as getSessionEvent4,
@@ -3926,7 +4356,7 @@ function createSessionStateActivities(services) {
3926
4356
  await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed");
3927
4357
  }
3928
4358
  async function interruptActiveTurn2(input) {
3929
- const { db, bus } = await services();
4359
+ const { db, bus, observability } = await services();
3930
4360
  const session = await requireSession4(db, input.workspaceId, input.sessionId);
3931
4361
  const trigger = await getSessionEvent4(db, input.workspaceId, input.triggerEventId);
3932
4362
  if (!isSteerInterrupt(trigger)) {
@@ -3949,9 +4379,10 @@ function createSessionStateActivities(services) {
3949
4379
  ]);
3950
4380
  await finishTurn2(db, input.workspaceId, session.activeTurnId, "cancelled");
3951
4381
  await setSessionStatus2(db, input.workspaceId, input.sessionId, "queued", null);
4382
+ await refreshQueuedTurnsGauge(db, observability);
3952
4383
  }
3953
4384
  async function requeueTurnAfterWorkerDeath2(input) {
3954
- const { settings, db, bus } = await services();
4385
+ const { settings, db, bus, observability } = await services();
3955
4386
  const turn = await getSessionTurn2(db, input.workspaceId, input.turnId);
3956
4387
  if (!turn || turn.status !== "running" && turn.status !== "requires_action") {
3957
4388
  return { action: "stale" };
@@ -3991,11 +4422,14 @@ function createSessionStateActivities(services) {
3991
4422
  throw requeueError;
3992
4423
  }
3993
4424
  await setSessionStatus2(db, input.workspaceId, input.sessionId, "queued", null);
4425
+ await refreshQueuedTurnsGauge(db, observability);
3994
4426
  return { action: "requeued", redispatches };
3995
4427
  }
3996
4428
  async function claimNextQueuedTurn2(input) {
3997
- const { db } = await services();
3998
- return await claimNextQueuedTurnDb2(db, input.workspaceId, input.sessionId, input.workflowId);
4429
+ const { db, observability } = await services();
4430
+ const turn = await claimNextQueuedTurnDb2(db, input.workspaceId, input.sessionId, input.workflowId);
4431
+ await refreshQueuedTurnsGauge(db, observability);
4432
+ return turn;
3999
4433
  }
4000
4434
  async function markSessionIdle2(input) {
4001
4435
  const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
@@ -4003,6 +4437,7 @@ function createSessionStateActivities(services) {
4003
4437
  if (session.status === "queued" || session.status === "running") {
4004
4438
  await setSessionStatus2(db, input.workspaceId, input.sessionId, "idle", null);
4005
4439
  }
4440
+ await refreshQueuedTurnsGauge(db, observability);
4006
4441
  await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "idle");
4007
4442
  }
4008
4443
  return {
@@ -4013,6 +4448,12 @@ function createSessionStateActivities(services) {
4013
4448
  markSessionIdle: markSessionIdle2
4014
4449
  };
4015
4450
  }
4451
+ async function refreshQueuedTurnsGauge(db, observability) {
4452
+ try {
4453
+ recordTurnsQueuedGauge(observability, await countQueuedTurns2(db));
4454
+ } catch {
4455
+ }
4456
+ }
4016
4457
 
4017
4458
  // src/activities.ts
4018
4459
  function createActivities(dependencies = {}) {
@@ -4020,6 +4461,7 @@ function createActivities(dependencies = {}) {
4020
4461
  async function services() {
4021
4462
  servicesPromise ??= (async () => {
4022
4463
  const settings = dependencies.settings ?? getSettings();
4464
+ const observability = dependencies.observability ?? createObservability(settings, { component: "worker" });
4023
4465
  const searchPath = dbSearchPath(settings);
4024
4466
  const dbClient = dependencies.db ? null : createDb(settings.databaseUrl, {
4025
4467
  ...searchPath ? { searchPath } : {},
@@ -4044,12 +4486,15 @@ function createActivities(dependencies = {}) {
4044
4486
  // events by sequence). See `.agents/skills/opengeni/references/eventbus-binding-contract.md`.
4045
4487
  bus: dependencies.bus ?? await createNatsEventBus(
4046
4488
  settings.natsUrl,
4047
- controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0
4489
+ controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
4490
+ { logger: observabilityEventLogger(observability) }
4048
4491
  ),
4049
- runtime: dependencies.runtime ?? createProductionAgentRuntime(),
4492
+ runtime: dependencies.runtime ?? createProductionAgentRuntime({
4493
+ metrics: runtimeMetricsHooksForObservability(observability)
4494
+ }),
4050
4495
  objectStorage: dependencies.objectStorage ?? createObjectStorage2(settings),
4051
4496
  documentServices: dependencies.documentServices ?? createDocumentServices(settings),
4052
- observability: dependencies.observability ?? createObservability(settings, { component: "worker" }),
4497
+ observability,
4053
4498
  wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
4054
4499
  // §7.5 P3 — host-entitlements port. No constructed default: standalone
4055
4500
  // has no host meter, so unset → null → `ensureRunAllowed` reads the
@@ -4098,6 +4543,86 @@ var maybeContinueGoal = defaultActivities.maybeContinueGoal;
4098
4543
  var pauseGoalForInterrupt = defaultActivities.pauseGoalForInterrupt;
4099
4544
  var reapSandboxLeases = defaultActivities.reapSandboxLeases;
4100
4545
 
4546
+ // src/http.ts
4547
+ import { dbSql } from "@opengeni/db";
4548
+ function startWorkerHttpServer(input) {
4549
+ const { settings, observability, checks } = input;
4550
+ const timeoutMs = input.timeoutMs ?? 2e3;
4551
+ return Bun.serve({
4552
+ hostname: "0.0.0.0",
4553
+ port: settings.workerHttpPort,
4554
+ async fetch(request) {
4555
+ const url = new URL(request.url);
4556
+ if (request.method !== "GET") {
4557
+ return Response.json({ error: "method_not_allowed" }, { status: 405 });
4558
+ }
4559
+ if (url.pathname === "/healthz") {
4560
+ return Response.json({
4561
+ service: settings.serviceName,
4562
+ environment: settings.environment,
4563
+ deploymentRevision: settings.deploymentRevision,
4564
+ ok: true
4565
+ });
4566
+ }
4567
+ if (url.pathname === "/metrics") {
4568
+ return new Response(await observability.prometheusMetrics(), {
4569
+ status: 200,
4570
+ headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" }
4571
+ });
4572
+ }
4573
+ if (url.pathname === "/readyz") {
4574
+ const result = await runReadinessChecks(checks, timeoutMs);
4575
+ return Response.json(result, { status: result.ok ? 200 : 503 });
4576
+ }
4577
+ return Response.json({ error: "not_found" }, { status: 404 });
4578
+ }
4579
+ });
4580
+ }
4581
+ async function runReadinessChecks(checks, timeoutMs = 2e3) {
4582
+ const entries = await Promise.all(
4583
+ Object.entries(checks).map(async ([name, check]) => {
4584
+ try {
4585
+ await withTimeout(Promise.resolve().then(check), timeoutMs);
4586
+ return [name, { ok: true }];
4587
+ } catch (error) {
4588
+ return [name, { ok: false, error: error instanceof Error ? error.message : String(error) }];
4589
+ }
4590
+ })
4591
+ );
4592
+ const result = Object.fromEntries(entries);
4593
+ return {
4594
+ ok: Object.values(result).every((check) => check.ok),
4595
+ checks: result
4596
+ };
4597
+ }
4598
+ function dbReadyCheck(db) {
4599
+ return async () => {
4600
+ await db.execute(dbSql`select 1`);
4601
+ };
4602
+ }
4603
+ function natsReadyCheck(bus) {
4604
+ return () => {
4605
+ if (bus.isConnected && !bus.isConnected()) {
4606
+ throw new Error("NATS is not connected");
4607
+ }
4608
+ };
4609
+ }
4610
+ async function withTimeout(promise, timeoutMs) {
4611
+ let timer;
4612
+ try {
4613
+ return await Promise.race([
4614
+ promise,
4615
+ new Promise((_, reject) => {
4616
+ timer = setTimeout(() => reject(new Error(`readiness check timed out after ${timeoutMs}ms`)), timeoutMs);
4617
+ })
4618
+ ]);
4619
+ } finally {
4620
+ if (timer) {
4621
+ clearTimeout(timer);
4622
+ }
4623
+ }
4624
+ }
4625
+
4101
4626
  // src/index.ts
4102
4627
  var SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
4103
4628
  async function createOpenGeniWorker(options = {}) {
@@ -4138,6 +4663,9 @@ async function createWorkerWorkflowSignaler(settings) {
4138
4663
  signal: "queueChanged"
4139
4664
  });
4140
4665
  },
4666
+ check: async () => {
4667
+ await connection.workflowService.getSystemInfo({});
4668
+ },
4141
4669
  close: async () => {
4142
4670
  await connection.close();
4143
4671
  }
@@ -4195,35 +4723,71 @@ async function registerSandboxReaperSchedule(settings, observability) {
4195
4723
  async function startWorker() {
4196
4724
  const settings = getSettings2();
4197
4725
  const observability = createObservability2(settings, { component: "worker" });
4198
- const signaler = await retryStartupDependency(
4199
- "Temporal client",
4200
- () => createWorkerWorkflowSignaler(settings),
4201
- {
4202
- ...startupRetryOptions(settings),
4203
- onRetry: (event) => logStartupDependencyRetry(observability, event)
4204
- }
4205
- );
4206
- const { worker, connection } = await createOpenGeniWorker({
4207
- settings,
4208
- activityDependencies: { observability, wakeSessionWorkflow: signaler.wakeSessionWorkflow }
4209
- });
4210
- const reaperSchedule = await retryStartupDependency(
4211
- "Temporal schedule (sandbox reaper)",
4212
- () => registerSandboxReaperSchedule(settings, observability),
4213
- {
4214
- ...startupRetryOptions(settings),
4215
- onRetry: (event) => logStartupDependencyRetry(observability, event)
4216
- }
4217
- );
4218
- observability.info("OpenGeni worker listening", {
4219
- temporalTaskQueue: settings.temporalTaskQueue
4726
+ const retryOptions = startupRetryOptions(settings);
4727
+ const onRetry = (event) => logStartupDependencyRetry(observability, event);
4728
+ const searchPath = dbSearchPath2(settings);
4729
+ const dbClient = createDb2(settings.databaseUrl, {
4730
+ ...searchPath ? { searchPath } : {},
4731
+ rlsStrategy: settings.rlsStrategy
4220
4732
  });
4733
+ const controlPlaneAuth = resolveNatsControlPlaneAuth2(settings);
4734
+ let bus;
4735
+ let signaler;
4736
+ let workerBundle;
4737
+ let reaperSchedule;
4738
+ let httpServer;
4221
4739
  try {
4222
- await worker.run();
4740
+ bus = await retryStartupDependency(
4741
+ "NATS",
4742
+ () => createNatsEventBus2(
4743
+ settings.natsUrl,
4744
+ controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
4745
+ { logger: observabilityEventLogger(observability) }
4746
+ ),
4747
+ { ...retryOptions, onRetry }
4748
+ );
4749
+ signaler = await retryStartupDependency(
4750
+ "Temporal client",
4751
+ () => createWorkerWorkflowSignaler(settings),
4752
+ { ...retryOptions, onRetry }
4753
+ );
4754
+ workerBundle = await createOpenGeniWorker({
4755
+ settings,
4756
+ activityDependencies: {
4757
+ observability,
4758
+ wakeSessionWorkflow: signaler.wakeSessionWorkflow,
4759
+ db: dbClient.db,
4760
+ bus
4761
+ }
4762
+ });
4763
+ httpServer = startWorkerHttpServer({
4764
+ settings,
4765
+ observability,
4766
+ checks: {
4767
+ db: dbReadyCheck(dbClient.db),
4768
+ nats: natsReadyCheck(bus),
4769
+ temporal: signaler.check
4770
+ }
4771
+ });
4772
+ reaperSchedule = await retryStartupDependency(
4773
+ "Temporal schedule (sandbox reaper)",
4774
+ () => registerSandboxReaperSchedule(settings, observability),
4775
+ { ...retryOptions, onRetry }
4776
+ );
4777
+ observability.info("OpenGeni worker listening", {
4778
+ temporalTaskQueue: settings.temporalTaskQueue,
4779
+ httpPort: settings.workerHttpPort
4780
+ });
4781
+ await workerBundle.worker.run();
4223
4782
  } finally {
4224
- await connection.close();
4225
- await signaler.close();
4226
- await reaperSchedule.close();
4783
+ httpServer?.stop(true);
4784
+ await Promise.allSettled([
4785
+ workerBundle?.connection.close(),
4786
+ signaler?.close(),
4787
+ reaperSchedule?.close(),
4788
+ bus?.close(),
4789
+ dbClient.close()
4790
+ ]);
4227
4791
  }
4228
4792
  }
4229
4793
  if (import.meta.main) {