@opengeni/worker-bundle 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,5 +1,7 @@
1
1
  // src/index.ts
2
- import { getSettings as getSettings2, retryStartupDependency, startupRetryOptions } from "@opengeni/config";
2
+ import { dbSearchPath as dbSearchPath2, getSettings as getSettings2, resolveNatsControlPlaneAuth as resolveNatsControlPlaneAuth2, retryStartupDependency, startupRetryOptions } from "@opengeni/config";
3
+ import { createDb as createDb2 } from "@opengeni/db";
4
+ import { createNatsEventBus as createNatsEventBus2 } from "@opengeni/events";
3
5
  import { createObservability as createObservability2, logStartupDependencyRetry } from "@opengeni/observability";
4
6
  import { Connection, ScheduleAlreadyRunning, ScheduleOverlapPolicy, Client as TemporalClient } from "@temporalio/client";
5
7
  import { NativeConnection, Worker } from "@temporalio/worker";
@@ -1240,6 +1242,15 @@ function asSandboxWarmingError(error, backend, timeoutMs) {
1240
1242
  const message = error instanceof Error ? error.message : String(error);
1241
1243
  return /sandbox creation timed out|warming timed out|capacity.*timed out/i.test(message) ? new SandboxWarmingTimeoutError(backend, timeoutMs) : error;
1242
1244
  }
1245
+ function recordSandboxWarmingTimeout(metrics, error) {
1246
+ if (!(error instanceof SandboxWarmingTimeoutError)) {
1247
+ return;
1248
+ }
1249
+ try {
1250
+ metrics?.onSandboxWarmingTimeout?.({ backend: error.backend });
1251
+ } catch {
1252
+ }
1253
+ }
1243
1254
  function workspaceArchiveFromEnvelope(envelope) {
1244
1255
  const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
1245
1256
  const archive = sessionState?.workspaceArchive;
@@ -1316,6 +1327,7 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
1316
1327
  sessionId: ids.sessionId,
1317
1328
  backendOverride: ids.backend,
1318
1329
  ...ids.environment ? { environment: ids.environment } : {},
1330
+ ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {},
1319
1331
  onSandboxCreated: async (created) => {
1320
1332
  createdEstablished = created;
1321
1333
  const resumeEnvelope2 = preserveWorkspaceArchiveOnInterimResumeState(
@@ -1381,7 +1393,9 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
1381
1393
  });
1382
1394
  }
1383
1395
  await release();
1384
- throw asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
1396
+ const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
1397
+ recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
1398
+ throw warmingError;
1385
1399
  }
1386
1400
  }
1387
1401
  try {
@@ -1394,13 +1408,16 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
1394
1408
  const established = await establishSandboxSessionFromEnvelope(settings, envelope, {
1395
1409
  sessionId: ids.sessionId,
1396
1410
  backendOverride: ids.backend,
1397
- ...ids.environment ? { environment: ids.environment } : {}
1411
+ ...ids.environment ? { environment: ids.environment } : {},
1412
+ ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
1398
1413
  });
1399
1414
  await ensureDisplayStack(settings, established);
1400
1415
  return { established, leaseEpoch, release };
1401
1416
  } catch (error) {
1402
1417
  await release();
1403
- throw asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
1418
+ const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
1419
+ recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
1420
+ throw warmingError;
1404
1421
  }
1405
1422
  }
1406
1423
  async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
@@ -1627,6 +1644,201 @@ function routingEnabled(settings) {
1627
1644
  return settings.sandboxSelfhostedEnabled === true;
1628
1645
  }
1629
1646
 
1647
+ // src/observability-metrics.ts
1648
+ var turnTrackers = /* @__PURE__ */ new WeakMap();
1649
+ function observabilityEventLogger(observability) {
1650
+ return {
1651
+ debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),
1652
+ warn: (message, attributes) => observability.warn(message, eventAttributes(attributes))
1653
+ };
1654
+ }
1655
+ function eventAttributes(attributes) {
1656
+ if (!attributes) {
1657
+ return void 0;
1658
+ }
1659
+ const sanitized = {};
1660
+ for (const [key, value] of Object.entries(attributes)) {
1661
+ sanitized[key] = eventAttributeValue(value);
1662
+ }
1663
+ return sanitized;
1664
+ }
1665
+ function eventAttributeValue(value) {
1666
+ if (value === null || value === void 0 || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
1667
+ return value;
1668
+ }
1669
+ try {
1670
+ return JSON.stringify(value);
1671
+ } catch {
1672
+ return String(value);
1673
+ }
1674
+ }
1675
+ function runtimeMetricsHooksForObservability(observability) {
1676
+ return {
1677
+ onModelCall: ({ provider, outcome, durationSeconds }) => {
1678
+ observability.incrementCounter({
1679
+ name: "opengeni_model_calls_total",
1680
+ help: "Total model calls by provider and outcome.",
1681
+ labels: { provider, outcome }
1682
+ });
1683
+ observability.observeHistogram({
1684
+ name: "opengeni_model_call_duration_seconds",
1685
+ help: "Model call duration in seconds by provider.",
1686
+ labels: { provider },
1687
+ value: durationSeconds
1688
+ });
1689
+ },
1690
+ onSandboxCreate: ({ backend, outcome, durationSeconds }) => {
1691
+ observability.incrementCounter({
1692
+ name: "opengeni_sandbox_creates_total",
1693
+ help: "Total sandbox create attempts by backend and outcome.",
1694
+ labels: { backend, outcome }
1695
+ });
1696
+ observability.observeHistogram({
1697
+ name: "opengeni_sandbox_create_duration_seconds",
1698
+ help: "Sandbox create duration in seconds by backend.",
1699
+ labels: { backend },
1700
+ value: durationSeconds
1701
+ });
1702
+ },
1703
+ onSandboxWarmingTimeout: () => {
1704
+ observability.incrementCounter({
1705
+ name: "opengeni_sandbox_warming_timeouts_total",
1706
+ help: "Total sandbox warming timeouts."
1707
+ });
1708
+ }
1709
+ };
1710
+ }
1711
+ function turnLifecycleMetricsFor(observability) {
1712
+ const existing = turnTrackers.get(observability);
1713
+ if (existing) {
1714
+ return existing;
1715
+ }
1716
+ const tracker = new TurnLifecycleMetrics(observability);
1717
+ turnTrackers.set(observability, tracker);
1718
+ return tracker;
1719
+ }
1720
+ var TurnLifecycleMetrics = class {
1721
+ constructor(observability, options = {}) {
1722
+ this.observability = observability;
1723
+ this.options = options;
1724
+ }
1725
+ observability;
1726
+ options;
1727
+ startedTurns = /* @__PURE__ */ new Map();
1728
+ timer = null;
1729
+ start(turnId) {
1730
+ this.startedTurns.set(turnId, this.now());
1731
+ this.ensureTimer();
1732
+ this.refreshGauges();
1733
+ }
1734
+ finish(turnId, outcome, durationSeconds) {
1735
+ const startedAt = this.startedTurns.get(turnId);
1736
+ if (startedAt !== void 0) {
1737
+ this.startedTurns.delete(turnId);
1738
+ }
1739
+ if (outcome) {
1740
+ const observedDuration = durationSeconds ?? (startedAt === void 0 ? 0 : Math.max(0, (this.now() - startedAt) / 1e3));
1741
+ this.observability.incrementCounter({
1742
+ name: "opengeni_turns_total",
1743
+ help: "Total agent turns by terminal outcome.",
1744
+ labels: { outcome }
1745
+ });
1746
+ this.observability.observeHistogram({
1747
+ name: "opengeni_turn_duration_seconds",
1748
+ help: "Agent turn duration in seconds by terminal outcome.",
1749
+ labels: { outcome },
1750
+ value: observedDuration
1751
+ });
1752
+ }
1753
+ this.refreshGauges();
1754
+ if (this.startedTurns.size === 0) {
1755
+ this.stopTimer();
1756
+ }
1757
+ }
1758
+ refreshGauges() {
1759
+ this.observability.setGauge({
1760
+ name: "opengeni_turns_inflight",
1761
+ help: "Current number of in-flight agent turns in this worker process.",
1762
+ value: this.startedTurns.size
1763
+ });
1764
+ this.observability.setGauge({
1765
+ name: "opengeni_turn_oldest_inflight_age_seconds",
1766
+ help: "Age in seconds of the oldest in-flight agent turn in this worker process.",
1767
+ value: this.oldestInflightAgeSeconds()
1768
+ });
1769
+ }
1770
+ stop() {
1771
+ this.startedTurns.clear();
1772
+ this.refreshGauges();
1773
+ this.stopTimer();
1774
+ }
1775
+ oldestInflightAgeSeconds() {
1776
+ if (this.startedTurns.size === 0) {
1777
+ return 0;
1778
+ }
1779
+ let oldest = Number.POSITIVE_INFINITY;
1780
+ for (const startedAt of this.startedTurns.values()) {
1781
+ oldest = Math.min(oldest, startedAt);
1782
+ }
1783
+ return Math.max(0, (this.now() - oldest) / 1e3);
1784
+ }
1785
+ ensureTimer() {
1786
+ if (this.timer) {
1787
+ return;
1788
+ }
1789
+ this.timer = setInterval(() => this.refreshGauges(), this.options.refreshIntervalMs ?? 15e3);
1790
+ this.timer.unref?.();
1791
+ }
1792
+ stopTimer() {
1793
+ if (!this.timer) {
1794
+ return;
1795
+ }
1796
+ clearInterval(this.timer);
1797
+ this.timer = null;
1798
+ }
1799
+ now() {
1800
+ return this.options.now?.() ?? Date.now();
1801
+ }
1802
+ };
1803
+ function recordTurnsQueuedGauge(observability, value) {
1804
+ observability.setGauge({
1805
+ name: "opengeni_turns_queued",
1806
+ help: "Current number of queued session turns.",
1807
+ value
1808
+ });
1809
+ }
1810
+ function recordSandboxLeaseGauges(observability, counts) {
1811
+ for (const liveness of ["cold", "warming", "warm", "draining"]) {
1812
+ observability.setGauge({
1813
+ name: "opengeni_sandbox_leases",
1814
+ help: "Current sandbox leases by liveness state.",
1815
+ labels: { liveness },
1816
+ value: counts[liveness] ?? 0
1817
+ });
1818
+ }
1819
+ }
1820
+ function recordSandboxOrphansTerminated(observability, count) {
1821
+ if (count <= 0) {
1822
+ return;
1823
+ }
1824
+ observability.incrementCounter({
1825
+ name: "opengeni_sandbox_orphans_terminated_total",
1826
+ help: "Total provider-side orphan sandboxes terminated by defensive sweeps.",
1827
+ amount: count
1828
+ });
1829
+ }
1830
+ function recordCreditMicros(observability, kind, amountMicros) {
1831
+ if (!observability || amountMicros <= 0) {
1832
+ return;
1833
+ }
1834
+ observability.incrementCounter({
1835
+ name: "opengeni_credit_micros_total",
1836
+ help: "Total credit micros recorded by kind.",
1837
+ labels: { kind },
1838
+ amount: amountMicros
1839
+ });
1840
+ }
1841
+
1630
1842
  // src/activities/recording.ts
1631
1843
  import { deleteRecording, updateRecording } from "@opengeni/db";
1632
1844
  import {
@@ -1920,6 +2132,7 @@ function createRunAgentTurnActivity(services) {
1920
2132
  "opengeni.trigger_event_id": input.triggerEventId
1921
2133
  });
1922
2134
  let activityStatus = "unknown";
2135
+ let turnMetricOutcome = null;
1923
2136
  let activityError;
1924
2137
  let turnId;
1925
2138
  let heartbeatTimer;
@@ -2022,6 +2235,7 @@ function createRunAgentTurnActivity(services) {
2022
2235
  throw new Error(`Session turn not found for trigger: ${input.triggerEventId}`);
2023
2236
  }
2024
2237
  turnId = turn.id;
2238
+ turnLifecycleMetricsFor(observability).start(turnId);
2025
2239
  const isCodexTurn = await isCodexBilledTurn2({ db, settings, workspaceId: input.workspaceId, model: turn.model, active: codexSubscriptionActive });
2026
2240
  await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
2027
2241
  const activityContext = currentActivityContext();
@@ -2122,6 +2336,7 @@ function createRunAgentTurnActivity(services) {
2122
2336
  { type: "session.status.changed", payload: { status: "idle" } }
2123
2337
  ], true);
2124
2338
  await finishTurn(db, input.workspaceId, turnId, "failed");
2339
+ turnMetricOutcome = "failed";
2125
2340
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2126
2341
  activityStatus = "idle";
2127
2342
  return goalActive ? { status: "idle", continueDelayMs: resumeMs, idleUntilReset: true } : { status: "idle" };
@@ -2260,7 +2475,7 @@ function createRunAgentTurnActivity(services) {
2260
2475
  };
2261
2476
  } else {
2262
2477
  resolvedSandbox = await resumeBoxForTurn(
2263
- { db, settings },
2478
+ { db, settings, sandboxMetrics: runtimeMetricsHooksForObservability(observability) },
2264
2479
  {
2265
2480
  accountId: input.accountId,
2266
2481
  workspaceId: input.workspaceId,
@@ -2322,7 +2537,7 @@ function createRunAgentTurnActivity(services) {
2322
2537
  expectedEpoch: heartbeatEpoch,
2323
2538
  warmRateMicrosPerSecond: warmRate,
2324
2539
  subjectId: input.sessionId
2325
- }).catch(() => void 0);
2540
+ }).then((result) => recordCreditMicros(observability, "usage", result.costMicros)).catch(() => void 0);
2326
2541
  }, 1e4);
2327
2542
  if ("unref" in leaseHeartbeatTimer && typeof leaseHeartbeatTimer.unref === "function") {
2328
2543
  leaseHeartbeatTimer.unref();
@@ -2567,7 +2782,8 @@ function createRunAgentTurnActivity(services) {
2567
2782
  responseId: responseUsage.responseId,
2568
2783
  dispatchId,
2569
2784
  positionalKey: `response-${responseUsageCount}`
2570
- })
2785
+ }),
2786
+ observability
2571
2787
  });
2572
2788
  await reconcileConversationTruth();
2573
2789
  try {
@@ -2613,7 +2829,8 @@ function createRunAgentTurnActivity(services) {
2613
2829
  model: turn.model,
2614
2830
  isCodexTurn,
2615
2831
  usage: stream.state.usage,
2616
- sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" })
2832
+ sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
2833
+ observability
2617
2834
  });
2618
2835
  }
2619
2836
  if (lastInputTokensObserved !== null) {
@@ -2661,6 +2878,7 @@ function createRunAgentTurnActivity(services) {
2661
2878
  { type: "session.status.changed", payload: { status: "idle" } }
2662
2879
  ], true);
2663
2880
  await finishTurn(db, input.workspaceId, activeTurnId, "idle");
2881
+ turnMetricOutcome = "completed";
2664
2882
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2665
2883
  await recordUsageEvent2(db, {
2666
2884
  accountId: input.accountId,
@@ -2741,6 +2959,7 @@ function createRunAgentTurnActivity(services) {
2741
2959
  await requeuePreemptedTurn(db, input.workspaceId, activeTurnId, preemptedEvent ? preemptedEvent.id : input.triggerEventId);
2742
2960
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
2743
2961
  activityStatus = "preempted";
2962
+ turnMetricOutcome = "preempted";
2744
2963
  observability.info("context compaction recovery succeeded by compacting and auto-continuing", {
2745
2964
  sessionId: input.sessionId,
2746
2965
  turnId: activeTurnId,
@@ -2763,6 +2982,7 @@ function createRunAgentTurnActivity(services) {
2763
2982
  { type: "session.status.changed", payload: { status: "idle" } }
2764
2983
  ], true);
2765
2984
  await finishTurn(db, input.workspaceId, activeTurnId, "failed");
2985
+ turnMetricOutcome = "failed";
2766
2986
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2767
2987
  activityStatus = "idle";
2768
2988
  activityError = attemptError;
@@ -2791,6 +3011,7 @@ function createRunAgentTurnActivity(services) {
2791
3011
  await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, input.triggerEventId);
2792
3012
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
2793
3013
  activityStatus = "preempted";
3014
+ turnMetricOutcome = "preempted";
2794
3015
  return { status: "preempted" };
2795
3016
  } catch (requeueError) {
2796
3017
  console.error("sandbox lease supersession requeue failed; falling back", requeueError);
@@ -2838,6 +3059,7 @@ function createRunAgentTurnActivity(services) {
2838
3059
  await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
2839
3060
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
2840
3061
  activityStatus = "preempted";
3062
+ turnMetricOutcome = "preempted";
2841
3063
  return { status: "preempted" };
2842
3064
  } catch (preemptError) {
2843
3065
  console.error("worker-shutdown preemption failed; falling back to cancellation", preemptError);
@@ -2849,6 +3071,7 @@ function createRunAgentTurnActivity(services) {
2849
3071
  await flushRuntimeBatcher();
2850
3072
  if (preemptTurnId) {
2851
3073
  await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => void 0);
3074
+ turnMetricOutcome = "cancelled";
2852
3075
  }
2853
3076
  throw error;
2854
3077
  }
@@ -2873,6 +3096,7 @@ function createRunAgentTurnActivity(services) {
2873
3096
  { type: "session.status.changed", payload: { status: "idle" } }
2874
3097
  ], true);
2875
3098
  await finishTurn(db, input.workspaceId, turnId, "idle");
3099
+ turnMetricOutcome = "completed";
2876
3100
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2877
3101
  await recordUsageEvent2(db, {
2878
3102
  accountId: input.accountId,
@@ -2962,6 +3186,7 @@ function createRunAgentTurnActivity(services) {
2962
3186
  { type: "session.status.changed", payload: { status: "idle" } }
2963
3187
  ], true);
2964
3188
  await finishTurn(db, input.workspaceId, turnId, "failed");
3189
+ turnMetricOutcome = "failed";
2965
3190
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2966
3191
  activityStatus = "idle";
2967
3192
  activityError = error;
@@ -2994,6 +3219,7 @@ function createRunAgentTurnActivity(services) {
2994
3219
  { type: "session.status.changed", payload: { status: "idle" } }
2995
3220
  ], true);
2996
3221
  await finishTurn(db, input.workspaceId, turnId, "idle");
3222
+ turnMetricOutcome = "completed";
2997
3223
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2998
3224
  await recordUsageEvent2(db, {
2999
3225
  accountId: input.accountId,
@@ -3032,6 +3258,7 @@ function createRunAgentTurnActivity(services) {
3032
3258
  { type: "session.status.changed", payload: { status: "idle" } }
3033
3259
  ], true);
3034
3260
  await finishTurn(db, input.workspaceId, turnId, "failed");
3261
+ turnMetricOutcome = "failed";
3035
3262
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
3036
3263
  activityStatus = "idle";
3037
3264
  activityError = error;
@@ -3047,6 +3274,7 @@ function createRunAgentTurnActivity(services) {
3047
3274
  { type: "session.status.changed", payload: { status: "failed" } }
3048
3275
  ], true);
3049
3276
  await finishTurn(db, input.workspaceId, turnId, "failed");
3277
+ turnMetricOutcome = "failed";
3050
3278
  await setSessionStatus(db, input.workspaceId, input.sessionId, "failed", null);
3051
3279
  await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed", `turn:${turnId}`);
3052
3280
  return { status: "failed" };
@@ -3057,6 +3285,9 @@ function createRunAgentTurnActivity(services) {
3057
3285
  status: activityStatus,
3058
3286
  durationSeconds
3059
3287
  });
3288
+ if (turnId && activityStatus !== "unknown") {
3289
+ turnLifecycleMetricsFor(observability).finish(turnId, turnMetricOutcome, durationSeconds);
3290
+ }
3060
3291
  activitySpan.end({
3061
3292
  attributes: {
3062
3293
  "opengeni.turn_id": turnId ?? "",
@@ -3279,7 +3510,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
3279
3510
  idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`
3280
3511
  });
3281
3512
  if (costMicros > 0) {
3282
- await applyCreditDebitUpToBalance(db, {
3513
+ const result = await applyCreditDebitUpToBalance(db, {
3283
3514
  accountId: input.accountId,
3284
3515
  workspaceId: input.workspaceId,
3285
3516
  type: "model_usage_debit",
@@ -3297,6 +3528,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
3297
3528
  totalTokens
3298
3529
  }
3299
3530
  });
3531
+ recordCreditMicros(input.observability, "usage", result.debitedMicros);
3300
3532
  }
3301
3533
  }
3302
3534
  function positiveInt(value) {
@@ -3417,6 +3649,8 @@ function startOfUtcMonth3(date = /* @__PURE__ */ new Date()) {
3417
3649
  import {
3418
3650
  accrueWarmSeconds as accrueWarmSeconds2,
3419
3651
  confirmDrainCold,
3652
+ countQueuedTurns,
3653
+ countSandboxLeasesByLiveness,
3420
3654
  forceDrainOverLimitViewerOnlyBoxes,
3421
3655
  getBillingBalance as getBillingBalance4,
3422
3656
  listLiveModalSandboxLeaseAttributions,
@@ -3443,6 +3677,7 @@ function createSandboxLeaseActivities(services, options = {}) {
3443
3677
  if (!settings.sandboxOwnershipEnabled) {
3444
3678
  return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
3445
3679
  }
3680
+ await refreshQueueAndLeaseGauges(db, observability);
3446
3681
  const metered = await accrueWarmTick(db, settings, observability);
3447
3682
  const forceDrained = await forceDrainOverLimitWorkspaces(db, settings, metered.workspaceIds, observability);
3448
3683
  const drainable = await reapStaleLeaseHoldersGlobal(db, {
@@ -3472,11 +3707,13 @@ function createSandboxLeaseActivities(services, options = {}) {
3472
3707
  }
3473
3708
  try {
3474
3709
  modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
3710
+ recordSandboxOrphansTerminated(observability, modalOrphansTerminated);
3475
3711
  } catch (error) {
3476
3712
  observability.warn("sandbox reaper: Modal orphan sweep failed", {
3477
3713
  error: error instanceof Error ? error.message : String(error)
3478
3714
  });
3479
3715
  }
3716
+ await refreshQueueAndLeaseGauges(db, observability);
3480
3717
  if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
3481
3718
  observability.info("sandbox reaper swept", {
3482
3719
  drainable: drainable.length,
@@ -3518,6 +3755,7 @@ async function accrueWarmTick(db, settings, observability) {
3518
3755
  if (result.accrued) {
3519
3756
  accrued += 1;
3520
3757
  }
3758
+ recordCreditMicros(observability, "usage", result.costMicros);
3521
3759
  } catch (error) {
3522
3760
  observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
3523
3761
  workspaceId: lease.workspaceId,
@@ -3528,6 +3766,22 @@ async function accrueWarmTick(db, settings, observability) {
3528
3766
  }
3529
3767
  return { accrued, workspaceIds };
3530
3768
  }
3769
+ async function refreshQueueAndLeaseGauges(db, observability) {
3770
+ try {
3771
+ recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
3772
+ } catch (error) {
3773
+ observability.warn("sandbox reaper: queued-turn gauge refresh failed", {
3774
+ error: error instanceof Error ? error.message : String(error)
3775
+ });
3776
+ }
3777
+ try {
3778
+ recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
3779
+ } catch (error) {
3780
+ observability.warn("sandbox reaper: sandbox-lease gauge refresh failed", {
3781
+ error: error instanceof Error ? error.message : String(error)
3782
+ });
3783
+ }
3784
+ }
3531
3785
  async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observability) {
3532
3786
  const enforceBalance = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
3533
3787
  const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
@@ -4058,6 +4312,7 @@ function startOfUtcMonth4() {
4058
4312
  // src/activities/session-state.ts
4059
4313
  import {
4060
4314
  claimNextQueuedTurn as claimNextQueuedTurnDb2,
4315
+ countQueuedTurns as countQueuedTurns2,
4061
4316
  countTurnSessionHistoryItems,
4062
4317
  finishTurn as finishTurn2,
4063
4318
  getSessionEvent as getSessionEvent4,
@@ -4101,7 +4356,7 @@ function createSessionStateActivities(services) {
4101
4356
  await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed");
4102
4357
  }
4103
4358
  async function interruptActiveTurn2(input) {
4104
- const { db, bus } = await services();
4359
+ const { db, bus, observability } = await services();
4105
4360
  const session = await requireSession4(db, input.workspaceId, input.sessionId);
4106
4361
  const trigger = await getSessionEvent4(db, input.workspaceId, input.triggerEventId);
4107
4362
  if (!isSteerInterrupt(trigger)) {
@@ -4124,9 +4379,10 @@ function createSessionStateActivities(services) {
4124
4379
  ]);
4125
4380
  await finishTurn2(db, input.workspaceId, session.activeTurnId, "cancelled");
4126
4381
  await setSessionStatus2(db, input.workspaceId, input.sessionId, "queued", null);
4382
+ await refreshQueuedTurnsGauge(db, observability);
4127
4383
  }
4128
4384
  async function requeueTurnAfterWorkerDeath2(input) {
4129
- const { settings, db, bus } = await services();
4385
+ const { settings, db, bus, observability } = await services();
4130
4386
  const turn = await getSessionTurn2(db, input.workspaceId, input.turnId);
4131
4387
  if (!turn || turn.status !== "running" && turn.status !== "requires_action") {
4132
4388
  return { action: "stale" };
@@ -4166,11 +4422,14 @@ function createSessionStateActivities(services) {
4166
4422
  throw requeueError;
4167
4423
  }
4168
4424
  await setSessionStatus2(db, input.workspaceId, input.sessionId, "queued", null);
4425
+ await refreshQueuedTurnsGauge(db, observability);
4169
4426
  return { action: "requeued", redispatches };
4170
4427
  }
4171
4428
  async function claimNextQueuedTurn2(input) {
4172
- const { db } = await services();
4173
- return await claimNextQueuedTurnDb2(db, input.workspaceId, input.sessionId, input.workflowId);
4429
+ const { db, observability } = await services();
4430
+ const turn = await claimNextQueuedTurnDb2(db, input.workspaceId, input.sessionId, input.workflowId);
4431
+ await refreshQueuedTurnsGauge(db, observability);
4432
+ return turn;
4174
4433
  }
4175
4434
  async function markSessionIdle2(input) {
4176
4435
  const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
@@ -4178,6 +4437,7 @@ function createSessionStateActivities(services) {
4178
4437
  if (session.status === "queued" || session.status === "running") {
4179
4438
  await setSessionStatus2(db, input.workspaceId, input.sessionId, "idle", null);
4180
4439
  }
4440
+ await refreshQueuedTurnsGauge(db, observability);
4181
4441
  await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "idle");
4182
4442
  }
4183
4443
  return {
@@ -4188,6 +4448,12 @@ function createSessionStateActivities(services) {
4188
4448
  markSessionIdle: markSessionIdle2
4189
4449
  };
4190
4450
  }
4451
+ async function refreshQueuedTurnsGauge(db, observability) {
4452
+ try {
4453
+ recordTurnsQueuedGauge(observability, await countQueuedTurns2(db));
4454
+ } catch {
4455
+ }
4456
+ }
4191
4457
 
4192
4458
  // src/activities.ts
4193
4459
  function createActivities(dependencies = {}) {
@@ -4195,6 +4461,7 @@ function createActivities(dependencies = {}) {
4195
4461
  async function services() {
4196
4462
  servicesPromise ??= (async () => {
4197
4463
  const settings = dependencies.settings ?? getSettings();
4464
+ const observability = dependencies.observability ?? createObservability(settings, { component: "worker" });
4198
4465
  const searchPath = dbSearchPath(settings);
4199
4466
  const dbClient = dependencies.db ? null : createDb(settings.databaseUrl, {
4200
4467
  ...searchPath ? { searchPath } : {},
@@ -4219,12 +4486,15 @@ function createActivities(dependencies = {}) {
4219
4486
  // events by sequence). See `.agents/skills/opengeni/references/eventbus-binding-contract.md`.
4220
4487
  bus: dependencies.bus ?? await createNatsEventBus(
4221
4488
  settings.natsUrl,
4222
- controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0
4489
+ controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
4490
+ { logger: observabilityEventLogger(observability) }
4223
4491
  ),
4224
- runtime: dependencies.runtime ?? createProductionAgentRuntime(),
4492
+ runtime: dependencies.runtime ?? createProductionAgentRuntime({
4493
+ metrics: runtimeMetricsHooksForObservability(observability)
4494
+ }),
4225
4495
  objectStorage: dependencies.objectStorage ?? createObjectStorage2(settings),
4226
4496
  documentServices: dependencies.documentServices ?? createDocumentServices(settings),
4227
- observability: dependencies.observability ?? createObservability(settings, { component: "worker" }),
4497
+ observability,
4228
4498
  wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
4229
4499
  // §7.5 P3 — host-entitlements port. No constructed default: standalone
4230
4500
  // has no host meter, so unset → null → `ensureRunAllowed` reads the
@@ -4273,6 +4543,86 @@ var maybeContinueGoal = defaultActivities.maybeContinueGoal;
4273
4543
  var pauseGoalForInterrupt = defaultActivities.pauseGoalForInterrupt;
4274
4544
  var reapSandboxLeases = defaultActivities.reapSandboxLeases;
4275
4545
 
4546
+ // src/http.ts
4547
+ import { dbSql } from "@opengeni/db";
4548
+ function startWorkerHttpServer(input) {
4549
+ const { settings, observability, checks } = input;
4550
+ const timeoutMs = input.timeoutMs ?? 2e3;
4551
+ return Bun.serve({
4552
+ hostname: "0.0.0.0",
4553
+ port: settings.workerHttpPort,
4554
+ async fetch(request) {
4555
+ const url = new URL(request.url);
4556
+ if (request.method !== "GET") {
4557
+ return Response.json({ error: "method_not_allowed" }, { status: 405 });
4558
+ }
4559
+ if (url.pathname === "/healthz") {
4560
+ return Response.json({
4561
+ service: settings.serviceName,
4562
+ environment: settings.environment,
4563
+ deploymentRevision: settings.deploymentRevision,
4564
+ ok: true
4565
+ });
4566
+ }
4567
+ if (url.pathname === "/metrics") {
4568
+ return new Response(await observability.prometheusMetrics(), {
4569
+ status: 200,
4570
+ headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" }
4571
+ });
4572
+ }
4573
+ if (url.pathname === "/readyz") {
4574
+ const result = await runReadinessChecks(checks, timeoutMs);
4575
+ return Response.json(result, { status: result.ok ? 200 : 503 });
4576
+ }
4577
+ return Response.json({ error: "not_found" }, { status: 404 });
4578
+ }
4579
+ });
4580
+ }
4581
+ async function runReadinessChecks(checks, timeoutMs = 2e3) {
4582
+ const entries = await Promise.all(
4583
+ Object.entries(checks).map(async ([name, check]) => {
4584
+ try {
4585
+ await withTimeout(Promise.resolve().then(check), timeoutMs);
4586
+ return [name, { ok: true }];
4587
+ } catch (error) {
4588
+ return [name, { ok: false, error: error instanceof Error ? error.message : String(error) }];
4589
+ }
4590
+ })
4591
+ );
4592
+ const result = Object.fromEntries(entries);
4593
+ return {
4594
+ ok: Object.values(result).every((check) => check.ok),
4595
+ checks: result
4596
+ };
4597
+ }
4598
+ function dbReadyCheck(db) {
4599
+ return async () => {
4600
+ await db.execute(dbSql`select 1`);
4601
+ };
4602
+ }
4603
+ function natsReadyCheck(bus) {
4604
+ return () => {
4605
+ if (bus.isConnected && !bus.isConnected()) {
4606
+ throw new Error("NATS is not connected");
4607
+ }
4608
+ };
4609
+ }
4610
+ async function withTimeout(promise, timeoutMs) {
4611
+ let timer;
4612
+ try {
4613
+ return await Promise.race([
4614
+ promise,
4615
+ new Promise((_, reject) => {
4616
+ timer = setTimeout(() => reject(new Error(`readiness check timed out after ${timeoutMs}ms`)), timeoutMs);
4617
+ })
4618
+ ]);
4619
+ } finally {
4620
+ if (timer) {
4621
+ clearTimeout(timer);
4622
+ }
4623
+ }
4624
+ }
4625
+
4276
4626
  // src/index.ts
4277
4627
  var SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
4278
4628
  async function createOpenGeniWorker(options = {}) {
@@ -4313,6 +4663,9 @@ async function createWorkerWorkflowSignaler(settings) {
4313
4663
  signal: "queueChanged"
4314
4664
  });
4315
4665
  },
4666
+ check: async () => {
4667
+ await connection.workflowService.getSystemInfo({});
4668
+ },
4316
4669
  close: async () => {
4317
4670
  await connection.close();
4318
4671
  }
@@ -4370,35 +4723,71 @@ async function registerSandboxReaperSchedule(settings, observability) {
4370
4723
  async function startWorker() {
4371
4724
  const settings = getSettings2();
4372
4725
  const observability = createObservability2(settings, { component: "worker" });
4373
- const signaler = await retryStartupDependency(
4374
- "Temporal client",
4375
- () => createWorkerWorkflowSignaler(settings),
4376
- {
4377
- ...startupRetryOptions(settings),
4378
- onRetry: (event) => logStartupDependencyRetry(observability, event)
4379
- }
4380
- );
4381
- const { worker, connection } = await createOpenGeniWorker({
4382
- settings,
4383
- activityDependencies: { observability, wakeSessionWorkflow: signaler.wakeSessionWorkflow }
4384
- });
4385
- const reaperSchedule = await retryStartupDependency(
4386
- "Temporal schedule (sandbox reaper)",
4387
- () => registerSandboxReaperSchedule(settings, observability),
4388
- {
4389
- ...startupRetryOptions(settings),
4390
- onRetry: (event) => logStartupDependencyRetry(observability, event)
4391
- }
4392
- );
4393
- observability.info("OpenGeni worker listening", {
4394
- temporalTaskQueue: settings.temporalTaskQueue
4726
+ const retryOptions = startupRetryOptions(settings);
4727
+ const onRetry = (event) => logStartupDependencyRetry(observability, event);
4728
+ const searchPath = dbSearchPath2(settings);
4729
+ const dbClient = createDb2(settings.databaseUrl, {
4730
+ ...searchPath ? { searchPath } : {},
4731
+ rlsStrategy: settings.rlsStrategy
4395
4732
  });
4733
+ const controlPlaneAuth = resolveNatsControlPlaneAuth2(settings);
4734
+ let bus;
4735
+ let signaler;
4736
+ let workerBundle;
4737
+ let reaperSchedule;
4738
+ let httpServer;
4396
4739
  try {
4397
- await worker.run();
4740
+ bus = await retryStartupDependency(
4741
+ "NATS",
4742
+ () => createNatsEventBus2(
4743
+ settings.natsUrl,
4744
+ controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
4745
+ { logger: observabilityEventLogger(observability) }
4746
+ ),
4747
+ { ...retryOptions, onRetry }
4748
+ );
4749
+ signaler = await retryStartupDependency(
4750
+ "Temporal client",
4751
+ () => createWorkerWorkflowSignaler(settings),
4752
+ { ...retryOptions, onRetry }
4753
+ );
4754
+ workerBundle = await createOpenGeniWorker({
4755
+ settings,
4756
+ activityDependencies: {
4757
+ observability,
4758
+ wakeSessionWorkflow: signaler.wakeSessionWorkflow,
4759
+ db: dbClient.db,
4760
+ bus
4761
+ }
4762
+ });
4763
+ httpServer = startWorkerHttpServer({
4764
+ settings,
4765
+ observability,
4766
+ checks: {
4767
+ db: dbReadyCheck(dbClient.db),
4768
+ nats: natsReadyCheck(bus),
4769
+ temporal: signaler.check
4770
+ }
4771
+ });
4772
+ reaperSchedule = await retryStartupDependency(
4773
+ "Temporal schedule (sandbox reaper)",
4774
+ () => registerSandboxReaperSchedule(settings, observability),
4775
+ { ...retryOptions, onRetry }
4776
+ );
4777
+ observability.info("OpenGeni worker listening", {
4778
+ temporalTaskQueue: settings.temporalTaskQueue,
4779
+ httpPort: settings.workerHttpPort
4780
+ });
4781
+ await workerBundle.worker.run();
4398
4782
  } finally {
4399
- await connection.close();
4400
- await signaler.close();
4401
- await reaperSchedule.close();
4783
+ httpServer?.stop(true);
4784
+ await Promise.allSettled([
4785
+ workerBundle?.connection.close(),
4786
+ signaler?.close(),
4787
+ reaperSchedule?.close(),
4788
+ bus?.close(),
4789
+ dbClient.close()
4790
+ ]);
4402
4791
  }
4403
4792
  }
4404
4793
  if (import.meta.main) {