@opengeni/worker-bundle 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,5 +1,7 @@
1
1
  // src/index.ts
2
- import { getSettings as getSettings2, retryStartupDependency, startupRetryOptions } from "@opengeni/config";
2
+ import { dbSearchPath as dbSearchPath2, getSettings as getSettings2, resolveNatsControlPlaneAuth as resolveNatsControlPlaneAuth2, retryStartupDependency, startupRetryOptions } from "@opengeni/config";
3
+ import { createDb as createDb2 } from "@opengeni/db";
4
+ import { createNatsEventBus as createNatsEventBus2 } from "@opengeni/events";
3
5
  import { createObservability as createObservability2, logStartupDependencyRetry } from "@opengeni/observability";
4
6
  import { Connection, ScheduleAlreadyRunning, ScheduleOverlapPolicy, Client as TemporalClient } from "@temporalio/client";
5
7
  import { NativeConnection, Worker } from "@temporalio/worker";
@@ -53,7 +55,8 @@ import {
53
55
  sumUsageQuantity as sumUsageQuantity2,
54
56
  heartbeatLeaseHolder,
55
57
  accrueWarmSeconds,
56
- SandboxLeaseSupersededError as SandboxLeaseSupersededError2
58
+ SandboxLeaseSupersededError as SandboxLeaseSupersededError2,
59
+ buildConnectionTokenResolver
57
60
  } from "@opengeni/db";
58
61
  import { appendAndPublishEvents as appendAndPublishEvents2 } from "@opengeni/events";
59
62
  import {
@@ -72,6 +75,7 @@ import { CancelledFailure } from "@temporalio/activity";
72
75
 
73
76
  // src/activities/capabilities.ts
74
77
  import { environmentsEncryptionKeyBytes, parseModelProvidersJson } from "@opengeni/config";
78
+ import { settingsWithMcpCapabilityServers } from "@opengeni/core";
75
79
  import {
76
80
  CODEX_APPS_MCP_SERVER_ID,
77
81
  CODEX_APPS_MCP_SERVER_NAME,
@@ -83,7 +87,6 @@ import {
83
87
  CODEX_PROVIDER_ID
84
88
  } from "@opengeni/codex";
85
89
  import {
86
- decryptedCapabilityHeaders,
87
90
  listEnabledMcpCapabilityServers,
88
91
  listSessionMcpServerMetadata,
89
92
  listSessionMcpServersForRun,
@@ -120,6 +123,7 @@ function settingsWithSessionMcpServers(settings, servers) {
120
123
  ...server.allowedTools ? { allowedTools: server.allowedTools } : {},
121
124
  ...server.timeoutMs ? { timeoutMs: server.timeoutMs } : {},
122
125
  cacheToolsList: server.cacheToolsList ?? false,
126
+ ...server.requireApproval !== void 0 ? { requireApproval: server.requireApproval } : {},
123
127
  headers: server.headers
124
128
  }))
125
129
  ]
@@ -169,29 +173,6 @@ function withCodexProvider(settings) {
169
173
  };
170
174
  return { ...settings, modelProvidersJson: JSON.stringify([...providers, codexProvider]) };
171
175
  }
172
- function settingsWithMcpCapabilityServers(settings, enabled) {
173
- if (enabled.length === 0) {
174
- return settings;
175
- }
176
- const encryptionKey = environmentsEncryptionKeyBytes(settings);
177
- const existingIds = new Set(settings.mcpServers.map((server) => server.id));
178
- const dynamicServers = enabled.filter((server) => !existingIds.has(server.id)).flatMap((server) => {
179
- const headers = decryptedCapabilityHeaders(server, encryptionKey);
180
- if (headers === "unavailable") {
181
- return [];
182
- }
183
- return [{
184
- id: server.id,
185
- name: server.name,
186
- url: server.url,
187
- ...server.allowedTools ? { allowedTools: server.allowedTools } : {},
188
- ...server.timeoutMs ? { timeoutMs: server.timeoutMs } : {},
189
- cacheToolsList: server.cacheToolsList ?? false,
190
- ...headers ? { headers } : {}
191
- }];
192
- });
193
- return dynamicServers.length ? { ...settings, mcpServers: [...settings.mcpServers, ...dynamicServers] } : settings;
194
- }
195
176
 
196
177
  // src/activities/codex-rotation.ts
197
178
  var MIN_IDLE_MS = 6e4;
@@ -366,13 +347,18 @@ import {
366
347
  import {
367
348
  SUMMARY_BUFFER_TOKENS,
368
349
  buildCompactionPromptInput,
350
+ buildDeterministicFallbackCompactionHistory,
369
351
  buildCompactionReplacementHistory,
370
352
  decideClientCompaction,
353
+ estimateTokens,
371
354
  sanitizeHistoryItemsForModel,
372
355
  summarizeForCompaction
373
356
  } from "@opengeni/runtime";
374
357
  import { resolveContextCompactionMode } from "@opengeni/config";
375
- async function maybeCompactContext(db, settings, scope, lastInputTokens, summarize = (s, m) => summarizeForCompaction(s, m, { maxOutputTokens: SUMMARY_BUFFER_TOKENS }), options = {}) {
358
+ async function maybeCompactContext(db, settings, scope, lastInputTokens, summarize = (s, m, o) => summarizeForCompaction(s, m, {
359
+ maxOutputTokens: SUMMARY_BUFFER_TOKENS,
360
+ ...o?.maxTranscriptTokens ? { maxTranscriptTokens: o.maxTranscriptTokens } : {}
361
+ }), options = {}) {
376
362
  if (resolveContextCompactionMode(settings) !== "client") {
377
363
  return { compacted: false, reason: "mode_not_client" };
378
364
  }
@@ -386,20 +372,67 @@ async function maybeCompactContext(db, settings, scope, lastInputTokens, summari
386
372
  lastInputTokens,
387
373
  contextWindowTokens: settings.contextWindowTokens,
388
374
  contextReservedOutputTokens: settings.contextReservedOutputTokens,
375
+ contextCompactionThresholdRatio: settings.contextCompactionThresholdRatio,
389
376
  ...options.force ? { force: true } : {}
390
377
  });
391
378
  if (!decision.shouldCompact) {
392
379
  return { compacted: false, reason: decision.reason };
393
380
  }
394
381
  const promptInput = buildCompactionPromptInput(items);
395
- const summaryBody = await summarize(settings, promptInput);
396
- if (!summaryBody) {
397
- return { compacted: false, reason: "summarize_failed" };
382
+ const estimatedTokensBefore = estimateTokens(items);
383
+ const shrinkBaselineTokens = Math.max(estimatedTokensBefore, decision.signalTokens);
384
+ const requireShrink = options.requireShrink || decision.reason === "above_threshold";
385
+ let summarizerFailure = "summarizer returned no summary";
386
+ let summaryBody = null;
387
+ const fullAttempt = await runSummarizerAttempt(summarize, settings, promptInput, { attempt: "full" });
388
+ if (fullAttempt.summary) {
389
+ summaryBody = fullAttempt.summary;
390
+ } else {
391
+ summarizerFailure = fullAttempt.failure;
392
+ const hardTrimBudget = Math.max(1, Math.floor(Math.max(1, decision.thresholdTokens) * 0.5));
393
+ const retryAttempt = await runSummarizerAttempt(summarize, settings, promptInput, {
394
+ attempt: "hard_trim",
395
+ maxTranscriptTokens: hardTrimBudget
396
+ });
397
+ if (retryAttempt.summary) {
398
+ summaryBody = retryAttempt.summary;
399
+ } else {
400
+ summarizerFailure = retryAttempt.failure || summarizerFailure;
401
+ }
402
+ }
403
+ let replacementHistory = null;
404
+ let method = "summary";
405
+ if (summaryBody) {
406
+ const summaryReplacement = buildCompactionReplacementHistory(items, summaryBody);
407
+ const summaryEstimate = estimateTokens(summaryReplacement);
408
+ if (!requireShrink || summaryEstimate < shrinkBaselineTokens) {
409
+ replacementHistory = summaryReplacement;
410
+ } else {
411
+ summarizerFailure = `summary replacement did not reduce context signal (${summaryEstimate} >= ${shrinkBaselineTokens})`;
412
+ }
413
+ }
414
+ if (!replacementHistory) {
415
+ method = "fallback";
416
+ const fallbackTargetTokens = Math.max(
417
+ 1,
418
+ Math.min(
419
+ Math.max(1, Math.floor(shrinkBaselineTokens * 0.5)),
420
+ Math.max(1, Math.floor(Math.max(1, decision.thresholdTokens) * 0.5))
421
+ )
422
+ );
423
+ replacementHistory = buildDeterministicFallbackCompactionHistory({
424
+ items,
425
+ cause: summarizerFailure,
426
+ targetTokens: fallbackTargetTokens
427
+ });
428
+ }
429
+ const estimatedTokensAfter = estimateTokens(replacementHistory);
430
+ if (requireShrink && estimatedTokensAfter >= shrinkBaselineTokens) {
431
+ return { compacted: false, reason: `compaction summarization failed: fallback did not reduce context signal (${estimatedTokensAfter} >= ${shrinkBaselineTokens})` };
398
432
  }
399
- const replacementHistory = buildCompactionReplacementHistory(items, summaryBody);
400
433
  const summaryItem = replacementHistory.at(-1);
401
434
  if (!summaryItem) {
402
- return { compacted: false, reason: "empty_replacement_history" };
435
+ return { compacted: false, reason: `compaction summarization failed: ${summarizerFailure}` };
403
436
  }
404
437
  const nextPosition = await nextSessionHistoryPosition(db, scope.workspaceId, scope.sessionId);
405
438
  const replacementItems = replacementHistory.slice(0, -1).map((item, index) => ({
@@ -422,15 +455,33 @@ async function maybeCompactContext(db, settings, scope, lastInputTokens, summari
422
455
  supersededFrom: nextPosition,
423
456
  summaryPosition,
424
457
  signalTokens: decision.signalTokens,
425
- thresholdTokens: decision.thresholdTokens
458
+ thresholdTokens: decision.thresholdTokens,
459
+ method,
460
+ estimatedTokensBefore,
461
+ estimatedTokensAfter
426
462
  };
427
463
  }
464
+ async function runSummarizerAttempt(summarize, settings, promptInput, options) {
465
+ try {
466
+ const summary = await summarize(settings, promptInput, options);
467
+ if (summary && summary.trim().length > 0) {
468
+ return { summary, failure: "" };
469
+ }
470
+ return { summary: null, failure: "summarizer returned no summary" };
471
+ } catch (error) {
472
+ return { summary: null, failure: error instanceof Error ? error.message : String(error) };
473
+ }
474
+ }
428
475
 
429
476
  // src/activities/environment.ts
430
477
  import {
431
478
  applyGitAuthPointerEnvironment,
479
+ firstPartyMcpWorkspaceUrl,
432
480
  stableSandboxEnvironmentForRun
433
481
  } from "@opengeni/config";
482
+ import {
483
+ signDelegatedAccessToken
484
+ } from "@opengeni/contracts";
434
485
  import {
435
486
  loadWorkspaceEnvironmentForRun as loadWorkspaceEnvironmentForRunFromDb
436
487
  } from "@opengeni/db";
@@ -466,10 +517,24 @@ function assertWorkspaceEcho(kind, scope, echoedWorkspaceId) {
466
517
  }
467
518
  }
468
519
  async function sandboxEnvironmentForRun(settings, resources, workspaceEnvironment = {}, options = {}) {
469
- const environment = stableSandboxEnvironmentForRun(settings, workspaceEnvironment);
520
+ const stableOptions = options.scope ? { workspaceId: options.scope.workspaceId } : {};
521
+ const environment = stableSandboxEnvironmentForRun(settings, workspaceEnvironment, stableOptions);
522
+ let toolspaceToken;
523
+ if (settings.toolspaceEnabled && !options.skipToolspaceToken && settings.delegationSecret && options.scope && options.sessionId && options.runId) {
524
+ toolspaceToken = await signDelegatedAccessToken(settings.delegationSecret, {
525
+ accountId: options.scope.accountId,
526
+ workspaceId: options.scope.workspaceId,
527
+ subjectId: `sandbox:${options.runId}`,
528
+ subjectLabel: "sandbox toolspace",
529
+ permissions: ["toolspace:call"],
530
+ sessionId: options.sessionId,
531
+ exp: Math.floor(Date.now() / 1e3) + 60 * 60
532
+ });
533
+ environment.OPENGENI_TOOLSPACE_URL ??= firstPartyMcpWorkspaceUrl(settings, options.scope.workspaceId);
534
+ }
470
535
  const selection = githubRepositorySelection(resources);
471
536
  if (!selection || options.skipGitHubToken) {
472
- return { environment };
537
+ return { environment, ...toolspaceToken ? { toolspaceToken } : {} };
473
538
  }
474
539
  let token;
475
540
  let identity;
@@ -491,7 +556,7 @@ async function sandboxEnvironmentForRun(settings, resources, workspaceEnvironmen
491
556
  identity = githubAppBotIdentity(settings);
492
557
  }
493
558
  applyGitAuthPointerEnvironment(environment, identity);
494
- return { environment, gitToken: token };
559
+ return { environment, gitToken: token, ...toolspaceToken ? { toolspaceToken } : {} };
495
560
  }
496
561
  function githubRepositorySelection(resources) {
497
562
  const selected = resources.flatMap((resource) => {
@@ -1109,6 +1174,7 @@ function createRuntimeBatcher(flushEvents) {
1109
1174
  "agent.toolCall.created",
1110
1175
  "agent.toolCall.output",
1111
1176
  "agent.message.completed",
1177
+ "tool.auth_needed",
1112
1178
  "session.requiresAction",
1113
1179
  "turn.completed",
1114
1180
  "turn.failed",
@@ -1240,6 +1306,15 @@ function asSandboxWarmingError(error, backend, timeoutMs) {
1240
1306
  const message = error instanceof Error ? error.message : String(error);
1241
1307
  return /sandbox creation timed out|warming timed out|capacity.*timed out/i.test(message) ? new SandboxWarmingTimeoutError(backend, timeoutMs) : error;
1242
1308
  }
1309
+ function recordSandboxWarmingTimeout(metrics, error) {
1310
+ if (!(error instanceof SandboxWarmingTimeoutError)) {
1311
+ return;
1312
+ }
1313
+ try {
1314
+ metrics?.onSandboxWarmingTimeout?.({ backend: error.backend });
1315
+ } catch {
1316
+ }
1317
+ }
1243
1318
  function workspaceArchiveFromEnvelope(envelope) {
1244
1319
  const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
1245
1320
  const archive = sessionState?.workspaceArchive;
@@ -1316,6 +1391,7 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
1316
1391
  sessionId: ids.sessionId,
1317
1392
  backendOverride: ids.backend,
1318
1393
  ...ids.environment ? { environment: ids.environment } : {},
1394
+ ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {},
1319
1395
  onSandboxCreated: async (created) => {
1320
1396
  createdEstablished = created;
1321
1397
  const resumeEnvelope2 = preserveWorkspaceArchiveOnInterimResumeState(
@@ -1381,7 +1457,9 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
1381
1457
  });
1382
1458
  }
1383
1459
  await release();
1384
- throw asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
1460
+ const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
1461
+ recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
1462
+ throw warmingError;
1385
1463
  }
1386
1464
  }
1387
1465
  try {
@@ -1394,13 +1472,16 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
1394
1472
  const established = await establishSandboxSessionFromEnvelope(settings, envelope, {
1395
1473
  sessionId: ids.sessionId,
1396
1474
  backendOverride: ids.backend,
1397
- ...ids.environment ? { environment: ids.environment } : {}
1475
+ ...ids.environment ? { environment: ids.environment } : {},
1476
+ ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
1398
1477
  });
1399
1478
  await ensureDisplayStack(settings, established);
1400
1479
  return { established, leaseEpoch, release };
1401
1480
  } catch (error) {
1402
1481
  await release();
1403
- throw asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
1482
+ const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
1483
+ recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
1484
+ throw warmingError;
1404
1485
  }
1405
1486
  }
1406
1487
  async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
@@ -1627,6 +1708,201 @@ function routingEnabled(settings) {
1627
1708
  return settings.sandboxSelfhostedEnabled === true;
1628
1709
  }
1629
1710
 
1711
+ // src/observability-metrics.ts
1712
+ var turnTrackers = /* @__PURE__ */ new WeakMap();
1713
+ function observabilityEventLogger(observability) {
1714
+ return {
1715
+ debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),
1716
+ warn: (message, attributes) => observability.warn(message, eventAttributes(attributes))
1717
+ };
1718
+ }
1719
+ function eventAttributes(attributes) {
1720
+ if (!attributes) {
1721
+ return void 0;
1722
+ }
1723
+ const sanitized = {};
1724
+ for (const [key, value] of Object.entries(attributes)) {
1725
+ sanitized[key] = eventAttributeValue(value);
1726
+ }
1727
+ return sanitized;
1728
+ }
1729
+ function eventAttributeValue(value) {
1730
+ if (value === null || value === void 0 || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
1731
+ return value;
1732
+ }
1733
+ try {
1734
+ return JSON.stringify(value);
1735
+ } catch {
1736
+ return String(value);
1737
+ }
1738
+ }
1739
+ function runtimeMetricsHooksForObservability(observability) {
1740
+ return {
1741
+ onModelCall: ({ provider, outcome, durationSeconds }) => {
1742
+ observability.incrementCounter({
1743
+ name: "opengeni_model_calls_total",
1744
+ help: "Total model calls by provider and outcome.",
1745
+ labels: { provider, outcome }
1746
+ });
1747
+ observability.observeHistogram({
1748
+ name: "opengeni_model_call_duration_seconds",
1749
+ help: "Model call duration in seconds by provider.",
1750
+ labels: { provider },
1751
+ value: durationSeconds
1752
+ });
1753
+ },
1754
+ onSandboxCreate: ({ backend, outcome, durationSeconds }) => {
1755
+ observability.incrementCounter({
1756
+ name: "opengeni_sandbox_creates_total",
1757
+ help: "Total sandbox create attempts by backend and outcome.",
1758
+ labels: { backend, outcome }
1759
+ });
1760
+ observability.observeHistogram({
1761
+ name: "opengeni_sandbox_create_duration_seconds",
1762
+ help: "Sandbox create duration in seconds by backend.",
1763
+ labels: { backend },
1764
+ value: durationSeconds
1765
+ });
1766
+ },
1767
+ onSandboxWarmingTimeout: () => {
1768
+ observability.incrementCounter({
1769
+ name: "opengeni_sandbox_warming_timeouts_total",
1770
+ help: "Total sandbox warming timeouts."
1771
+ });
1772
+ }
1773
+ };
1774
+ }
1775
+ function turnLifecycleMetricsFor(observability) {
1776
+ const existing = turnTrackers.get(observability);
1777
+ if (existing) {
1778
+ return existing;
1779
+ }
1780
+ const tracker = new TurnLifecycleMetrics(observability);
1781
+ turnTrackers.set(observability, tracker);
1782
+ return tracker;
1783
+ }
1784
+ var TurnLifecycleMetrics = class {
1785
+ constructor(observability, options = {}) {
1786
+ this.observability = observability;
1787
+ this.options = options;
1788
+ }
1789
+ observability;
1790
+ options;
1791
+ startedTurns = /* @__PURE__ */ new Map();
1792
+ timer = null;
1793
+ start(turnId) {
1794
+ this.startedTurns.set(turnId, this.now());
1795
+ this.ensureTimer();
1796
+ this.refreshGauges();
1797
+ }
1798
+ finish(turnId, outcome, durationSeconds) {
1799
+ const startedAt = this.startedTurns.get(turnId);
1800
+ if (startedAt !== void 0) {
1801
+ this.startedTurns.delete(turnId);
1802
+ }
1803
+ if (outcome) {
1804
+ const observedDuration = durationSeconds ?? (startedAt === void 0 ? 0 : Math.max(0, (this.now() - startedAt) / 1e3));
1805
+ this.observability.incrementCounter({
1806
+ name: "opengeni_turns_total",
1807
+ help: "Total agent turns by terminal outcome.",
1808
+ labels: { outcome }
1809
+ });
1810
+ this.observability.observeHistogram({
1811
+ name: "opengeni_turn_duration_seconds",
1812
+ help: "Agent turn duration in seconds by terminal outcome.",
1813
+ labels: { outcome },
1814
+ value: observedDuration
1815
+ });
1816
+ }
1817
+ this.refreshGauges();
1818
+ if (this.startedTurns.size === 0) {
1819
+ this.stopTimer();
1820
+ }
1821
+ }
1822
+ refreshGauges() {
1823
+ this.observability.setGauge({
1824
+ name: "opengeni_turns_inflight",
1825
+ help: "Current number of in-flight agent turns in this worker process.",
1826
+ value: this.startedTurns.size
1827
+ });
1828
+ this.observability.setGauge({
1829
+ name: "opengeni_turn_oldest_inflight_age_seconds",
1830
+ help: "Age in seconds of the oldest in-flight agent turn in this worker process.",
1831
+ value: this.oldestInflightAgeSeconds()
1832
+ });
1833
+ }
1834
+ stop() {
1835
+ this.startedTurns.clear();
1836
+ this.refreshGauges();
1837
+ this.stopTimer();
1838
+ }
1839
+ oldestInflightAgeSeconds() {
1840
+ if (this.startedTurns.size === 0) {
1841
+ return 0;
1842
+ }
1843
+ let oldest = Number.POSITIVE_INFINITY;
1844
+ for (const startedAt of this.startedTurns.values()) {
1845
+ oldest = Math.min(oldest, startedAt);
1846
+ }
1847
+ return Math.max(0, (this.now() - oldest) / 1e3);
1848
+ }
1849
+ ensureTimer() {
1850
+ if (this.timer) {
1851
+ return;
1852
+ }
1853
+ this.timer = setInterval(() => this.refreshGauges(), this.options.refreshIntervalMs ?? 15e3);
1854
+ this.timer.unref?.();
1855
+ }
1856
+ stopTimer() {
1857
+ if (!this.timer) {
1858
+ return;
1859
+ }
1860
+ clearInterval(this.timer);
1861
+ this.timer = null;
1862
+ }
1863
+ now() {
1864
+ return this.options.now?.() ?? Date.now();
1865
+ }
1866
+ };
1867
+ function recordTurnsQueuedGauge(observability, value) {
1868
+ observability.setGauge({
1869
+ name: "opengeni_turns_queued",
1870
+ help: "Current number of queued session turns.",
1871
+ value
1872
+ });
1873
+ }
1874
+ function recordSandboxLeaseGauges(observability, counts) {
1875
+ for (const liveness of ["cold", "warming", "warm", "draining"]) {
1876
+ observability.setGauge({
1877
+ name: "opengeni_sandbox_leases",
1878
+ help: "Current sandbox leases by liveness state.",
1879
+ labels: { liveness },
1880
+ value: counts[liveness] ?? 0
1881
+ });
1882
+ }
1883
+ }
1884
+ function recordSandboxOrphansTerminated(observability, count) {
1885
+ if (count <= 0) {
1886
+ return;
1887
+ }
1888
+ observability.incrementCounter({
1889
+ name: "opengeni_sandbox_orphans_terminated_total",
1890
+ help: "Total provider-side orphan sandboxes terminated by defensive sweeps.",
1891
+ amount: count
1892
+ });
1893
+ }
1894
+ function recordCreditMicros(observability, kind, amountMicros) {
1895
+ if (!observability || amountMicros <= 0) {
1896
+ return;
1897
+ }
1898
+ observability.incrementCounter({
1899
+ name: "opengeni_credit_micros_total",
1900
+ help: "Total credit micros recorded by kind.",
1901
+ labels: { kind },
1902
+ amount: amountMicros
1903
+ });
1904
+ }
1905
+
1630
1906
  // src/activities/recording.ts
1631
1907
  import { deleteRecording, updateRecording } from "@opengeni/db";
1632
1908
  import {
@@ -1794,6 +2070,10 @@ function isWorkerShutdownCancellation(error) {
1794
2070
  return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
1795
2071
  }
1796
2072
  var CONTEXT_WINDOW_OVERFLOW_RECOVERY_MESSAGE = "The conversation reached the model context compaction threshold; it has been compacted. Send a message to continue.";
2073
+ var MAX_CONTEXT_COMPACTION_RECOVERY_ATTEMPTS = 3;
2074
+ function compactionFailureReason(reason) {
2075
+ return reason.startsWith("compaction summarization failed:") ? reason : `compaction summarization failed: ${reason}`;
2076
+ }
1797
2077
  function classifyContextWindowOverflowError(error) {
1798
2078
  const fields = collectErrorStrings(error);
1799
2079
  const matched = fields.find(
@@ -1920,6 +2200,7 @@ function createRunAgentTurnActivity(services) {
1920
2200
  "opengeni.trigger_event_id": input.triggerEventId
1921
2201
  });
1922
2202
  let activityStatus = "unknown";
2203
+ let turnMetricOutcome = null;
1923
2204
  let activityError;
1924
2205
  let turnId;
1925
2206
  let heartbeatTimer;
@@ -2022,6 +2303,7 @@ function createRunAgentTurnActivity(services) {
2022
2303
  throw new Error(`Session turn not found for trigger: ${input.triggerEventId}`);
2023
2304
  }
2024
2305
  turnId = turn.id;
2306
+ turnLifecycleMetricsFor(observability).start(turnId);
2025
2307
  const isCodexTurn = await isCodexBilledTurn2({ db, settings, workspaceId: input.workspaceId, model: turn.model, active: codexSubscriptionActive });
2026
2308
  await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
2027
2309
  const activityContext = currentActivityContext();
@@ -2122,6 +2404,7 @@ function createRunAgentTurnActivity(services) {
2122
2404
  { type: "session.status.changed", payload: { status: "idle" } }
2123
2405
  ], true);
2124
2406
  await finishTurn(db, input.workspaceId, turnId, "failed");
2407
+ turnMetricOutcome = "failed";
2125
2408
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2126
2409
  activityStatus = "idle";
2127
2410
  return goalActive ? { status: "idle", continueDelayMs: resumeMs, idleUntilReset: true } : { status: "idle" };
@@ -2199,14 +2482,21 @@ function createRunAgentTurnActivity(services) {
2199
2482
  async () => activeSandboxRecord?.kind ?? null
2200
2483
  );
2201
2484
  const machinePrimary = activeSandboxBackend === "selfhosted" && Boolean(activeSandboxPointer?.activeSandboxId) && Boolean(activeSandboxRecord?.enrollmentId);
2202
- const { environment: sandboxEnvironment, gitToken: sandboxGitToken } = await sandboxEnvironmentForRun(
2485
+ const {
2486
+ environment: sandboxEnvironment,
2487
+ gitToken: sandboxGitToken,
2488
+ toolspaceToken: sandboxToolspaceToken
2489
+ } = await sandboxEnvironmentForRun(
2203
2490
  runSettings,
2204
2491
  turnResources,
2205
2492
  workspaceEnvironment?.values ?? {},
2206
2493
  {
2207
2494
  skipGitHubToken: activeSandboxBackend === "selfhosted",
2495
+ skipToolspaceToken: activeSandboxBackend === "selfhosted",
2208
2496
  scope: connectionScope,
2209
- gitCredentials: connectionCredentials?.gitCredentials
2497
+ gitCredentials: connectionCredentials?.gitCredentials,
2498
+ sessionId: input.sessionId,
2499
+ runId: turnId
2210
2500
  }
2211
2501
  );
2212
2502
  if (settings.sandboxOwnershipEnabled && turn.sandboxBackend !== "none") {
@@ -2260,7 +2550,7 @@ function createRunAgentTurnActivity(services) {
2260
2550
  };
2261
2551
  } else {
2262
2552
  resolvedSandbox = await resumeBoxForTurn(
2263
- { db, settings },
2553
+ { db, settings, sandboxMetrics: runtimeMetricsHooksForObservability(observability) },
2264
2554
  {
2265
2555
  accountId: input.accountId,
2266
2556
  workspaceId: input.workspaceId,
@@ -2322,7 +2612,7 @@ function createRunAgentTurnActivity(services) {
2322
2612
  expectedEpoch: heartbeatEpoch,
2323
2613
  warmRateMicrosPerSecond: warmRate,
2324
2614
  subjectId: input.sessionId
2325
- }).catch(() => void 0);
2615
+ }).then((result) => recordCreditMicros(observability, "usage", result.costMicros)).catch(() => void 0);
2326
2616
  }, 1e4);
2327
2617
  if ("unref" in leaseHeartbeatTimer && typeof leaseHeartbeatTimer.unref === "function") {
2328
2618
  leaseHeartbeatTimer.unref();
@@ -2360,12 +2650,17 @@ function createRunAgentTurnActivity(services) {
2360
2650
  }
2361
2651
  const fileResourceDownloads = await sandboxFileDownloadsForRun(runSettings, db, objectStorage, input.workspaceId, turnResources);
2362
2652
  throwIfWorkerShuttingDown();
2653
+ const resolveCredential = buildConnectionTokenResolver(db, runSettings);
2363
2654
  preparedTools = await withCodex(() => runtime.prepareTools(runSettings, turnTools, {
2364
2655
  accountId: input.accountId,
2365
2656
  workspaceId: input.workspaceId,
2366
2657
  sessionId: input.sessionId,
2367
2658
  subjectId: "worker:first-party-mcp",
2368
2659
  subjectLabel: "OpenGeni worker",
2660
+ resolveCredential,
2661
+ onAuthNeeded: async (payload) => {
2662
+ await publish([{ type: "tool.auth_needed", payload }], true);
2663
+ },
2369
2664
  // Manager-style sessions carry a creation-validated permission set
2370
2665
  // for their first-party MCP token; null keeps the fixed default.
2371
2666
  ...session.firstPartyMcpPermissions?.length ? { firstPartyPermissions: session.firstPartyMcpPermissions } : {}
@@ -2382,6 +2677,7 @@ function createRunAgentTurnActivity(services) {
2382
2677
  // seeds it to the box's token file before the repository-clone runs; it never
2383
2678
  // touches the box/agent manifest env.
2384
2679
  ...activeSandboxBackend !== "selfhosted" && sandboxGitToken ? { gitTokenSeed: sandboxGitToken } : {},
2680
+ ...activeSandboxBackend !== "selfhosted" && sandboxToolspaceToken ? { toolspaceTokenSeed: sandboxToolspaceToken } : {},
2385
2681
  ...activeSandboxBackend ? { activeSandboxBackend } : {},
2386
2682
  fileResourceDownloads,
2387
2683
  mcpServers: preparedTools.mcpServers,
@@ -2437,11 +2733,12 @@ function createRunAgentTurnActivity(services) {
2437
2733
  }
2438
2734
  } : {}
2439
2735
  });
2440
- const compactSummarizer = resolvedModel ? (s, m) => withCodex(() => summarizeForCompaction2(s, m, {
2736
+ const compactSummarizer = resolvedModel ? (s, m, o) => withCodex(() => summarizeForCompaction2(s, m, {
2441
2737
  client: resolvedModel.client,
2442
2738
  api: resolvedModel.provider.api,
2443
2739
  model: resolvedModel.configured.id,
2444
- maxOutputTokens: SUMMARY_BUFFER_TOKENS2
2740
+ maxOutputTokens: SUMMARY_BUFFER_TOKENS2,
2741
+ ...o?.maxTranscriptTokens ? { maxTranscriptTokens: o.maxTranscriptTokens } : {}
2445
2742
  })) : void 0;
2446
2743
  if (triggerType === "user.message" || triggerType === "goal.continuation") {
2447
2744
  try {
@@ -2498,7 +2795,7 @@ function createRunAgentTurnActivity(services) {
2498
2795
  { accountId: input.accountId, workspaceId: input.workspaceId, sessionId: input.sessionId, turnId: activeTurnId },
2499
2796
  session.lastInputTokens,
2500
2797
  compactSummarizer,
2501
- { force: true }
2798
+ { force: true, requireShrink: true }
2502
2799
  );
2503
2800
  if (outcome.compacted) {
2504
2801
  await publish([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, trigger: triggerLabel } }]);
@@ -2567,7 +2864,8 @@ function createRunAgentTurnActivity(services) {
2567
2864
  responseId: responseUsage.responseId,
2568
2865
  dispatchId,
2569
2866
  positionalKey: `response-${responseUsageCount}`
2570
- })
2867
+ }),
2868
+ observability
2571
2869
  });
2572
2870
  await reconcileConversationTruth();
2573
2871
  try {
@@ -2613,7 +2911,8 @@ function createRunAgentTurnActivity(services) {
2613
2911
  model: turn.model,
2614
2912
  isCodexTurn,
2615
2913
  usage: stream.state.usage,
2616
- sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" })
2914
+ sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
2915
+ observability
2617
2916
  });
2618
2917
  }
2619
2918
  if (lastInputTokensObserved !== null) {
@@ -2661,6 +2960,7 @@ function createRunAgentTurnActivity(services) {
2661
2960
  { type: "session.status.changed", payload: { status: "idle" } }
2662
2961
  ], true);
2663
2962
  await finishTurn(db, input.workspaceId, activeTurnId, "idle");
2963
+ turnMetricOutcome = "completed";
2664
2964
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2665
2965
  await recordUsageEvent2(db, {
2666
2966
  accountId: input.accountId,
@@ -2676,7 +2976,7 @@ function createRunAgentTurnActivity(services) {
2676
2976
  return { status: "idle" };
2677
2977
  };
2678
2978
  await prepareRunAttemptInput();
2679
- let compactionRecoveryAttempted = false;
2979
+ let compactionRecoveryAttempts = 0;
2680
2980
  let retriedAfterCompaction = false;
2681
2981
  while (true) {
2682
2982
  try {
@@ -2692,10 +2992,16 @@ function createRunAgentTurnActivity(services) {
2692
2992
  const overflow = classifyContextWindowOverflowError(attemptError);
2693
2993
  const compactionNeeded = findCompactionNeededError(attemptError);
2694
2994
  const recoveryKind = compactionNeeded ? "proactive" : overflow ? "overflow" : null;
2695
- if (!recoveryKind || !publish || !turnStartedPublished || compactionRecoveryAttempted) {
2995
+ if (recoveryKind && compactionRecoveryAttempts >= MAX_CONTEXT_COMPACTION_RECOVERY_ATTEMPTS) {
2996
+ throw new Error(
2997
+ `compaction summarization failed: context compaction recovery attempt cap reached (${MAX_CONTEXT_COMPACTION_RECOVERY_ATTEMPTS})`,
2998
+ { cause: attemptError }
2999
+ );
3000
+ }
3001
+ if (!recoveryKind || !publish || !turnStartedPublished) {
2696
3002
  throw attemptError;
2697
3003
  }
2698
- compactionRecoveryAttempted = true;
3004
+ compactionRecoveryAttempts += 1;
2699
3005
  await flushRuntimeBatcher();
2700
3006
  await reconcileConversationTruth({ skipInputOnlyRows: true });
2701
3007
  const progressPersisted = modelOrToolProgressPersisted;
@@ -2710,16 +3016,46 @@ function createRunAgentTurnActivity(services) {
2710
3016
  thresholdTokens: compactionNeeded?.thresholdTokens
2711
3017
  });
2712
3018
  let compacted = false;
3019
+ let compactionFailureMessage = null;
2713
3020
  try {
2714
3021
  const outcome = await forceContextCompaction(recoveryKind);
2715
3022
  compacted = outcome.compacted;
3023
+ if (!outcome.compacted) {
3024
+ compactionFailureMessage = compactionFailureReason(outcome.reason);
3025
+ }
2716
3026
  } catch (compactError) {
3027
+ compactionFailureMessage = compactionFailureReason(compactError instanceof Error ? compactError.message : String(compactError));
2717
3028
  observability.warn("context compaction recovery compaction failed", {
2718
3029
  sessionId: input.sessionId,
2719
3030
  turnId: activeTurnId,
2720
- error: compactError instanceof Error ? compactError.message : String(compactError)
3031
+ error: compactionFailureMessage
2721
3032
  });
2722
3033
  }
3034
+ if (!compacted) {
3035
+ const errorMessage = compactionFailureMessage ?? "compaction summarization failed: compaction produced no replacement history";
3036
+ if (!progressPersisted) {
3037
+ throw new Error(errorMessage, { cause: attemptError });
3038
+ }
3039
+ await publish([
3040
+ {
3041
+ type: "turn.failed",
3042
+ payload: {
3043
+ error: errorMessage,
3044
+ code: "context_compaction_failed",
3045
+ retryable: false,
3046
+ recovery: "user_message",
3047
+ compacted: false
3048
+ }
3049
+ },
3050
+ { type: "session.status.changed", payload: { status: "idle" } }
3051
+ ], true);
3052
+ await finishTurn(db, input.workspaceId, activeTurnId, "failed");
3053
+ turnMetricOutcome = "failed";
3054
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
3055
+ activityStatus = "idle";
3056
+ activityError = attemptError;
3057
+ return { status: "idle" };
3058
+ }
2723
3059
  if (progressPersisted) {
2724
3060
  const triggerPayload = trigger.payload && typeof trigger.payload === "object" && !Array.isArray(trigger.payload) ? trigger.payload : {};
2725
3061
  const compactionResumedTurn = triggerType === "turn.preempted" && (triggerPayload.reason === "context_compacted" || triggerPayload.reason === "context_overflow_compacted");
@@ -2741,6 +3077,7 @@ function createRunAgentTurnActivity(services) {
2741
3077
  await requeuePreemptedTurn(db, input.workspaceId, activeTurnId, preemptedEvent ? preemptedEvent.id : input.triggerEventId);
2742
3078
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
2743
3079
  activityStatus = "preempted";
3080
+ turnMetricOutcome = "preempted";
2744
3081
  observability.info("context compaction recovery succeeded by compacting and auto-continuing", {
2745
3082
  sessionId: input.sessionId,
2746
3083
  turnId: activeTurnId,
@@ -2763,6 +3100,7 @@ function createRunAgentTurnActivity(services) {
2763
3100
  { type: "session.status.changed", payload: { status: "idle" } }
2764
3101
  ], true);
2765
3102
  await finishTurn(db, input.workspaceId, activeTurnId, "failed");
3103
+ turnMetricOutcome = "failed";
2766
3104
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2767
3105
  activityStatus = "idle";
2768
3106
  activityError = attemptError;
@@ -2791,6 +3129,7 @@ function createRunAgentTurnActivity(services) {
2791
3129
  await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, input.triggerEventId);
2792
3130
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
2793
3131
  activityStatus = "preempted";
3132
+ turnMetricOutcome = "preempted";
2794
3133
  return { status: "preempted" };
2795
3134
  } catch (requeueError) {
2796
3135
  console.error("sandbox lease supersession requeue failed; falling back", requeueError);
@@ -2838,6 +3177,7 @@ function createRunAgentTurnActivity(services) {
2838
3177
  await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
2839
3178
  await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
2840
3179
  activityStatus = "preempted";
3180
+ turnMetricOutcome = "preempted";
2841
3181
  return { status: "preempted" };
2842
3182
  } catch (preemptError) {
2843
3183
  console.error("worker-shutdown preemption failed; falling back to cancellation", preemptError);
@@ -2849,6 +3189,7 @@ function createRunAgentTurnActivity(services) {
2849
3189
  await flushRuntimeBatcher();
2850
3190
  if (preemptTurnId) {
2851
3191
  await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => void 0);
3192
+ turnMetricOutcome = "cancelled";
2852
3193
  }
2853
3194
  throw error;
2854
3195
  }
@@ -2873,6 +3214,7 @@ function createRunAgentTurnActivity(services) {
2873
3214
  { type: "session.status.changed", payload: { status: "idle" } }
2874
3215
  ], true);
2875
3216
  await finishTurn(db, input.workspaceId, turnId, "idle");
3217
+ turnMetricOutcome = "completed";
2876
3218
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2877
3219
  await recordUsageEvent2(db, {
2878
3220
  accountId: input.accountId,
@@ -2962,6 +3304,7 @@ function createRunAgentTurnActivity(services) {
2962
3304
  { type: "session.status.changed", payload: { status: "idle" } }
2963
3305
  ], true);
2964
3306
  await finishTurn(db, input.workspaceId, turnId, "failed");
3307
+ turnMetricOutcome = "failed";
2965
3308
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2966
3309
  activityStatus = "idle";
2967
3310
  activityError = error;
@@ -2994,6 +3337,7 @@ function createRunAgentTurnActivity(services) {
2994
3337
  { type: "session.status.changed", payload: { status: "idle" } }
2995
3338
  ], true);
2996
3339
  await finishTurn(db, input.workspaceId, turnId, "idle");
3340
+ turnMetricOutcome = "completed";
2997
3341
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2998
3342
  await recordUsageEvent2(db, {
2999
3343
  accountId: input.accountId,
@@ -3032,6 +3376,7 @@ function createRunAgentTurnActivity(services) {
3032
3376
  { type: "session.status.changed", payload: { status: "idle" } }
3033
3377
  ], true);
3034
3378
  await finishTurn(db, input.workspaceId, turnId, "failed");
3379
+ turnMetricOutcome = "failed";
3035
3380
  await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
3036
3381
  activityStatus = "idle";
3037
3382
  activityError = error;
@@ -3047,6 +3392,7 @@ function createRunAgentTurnActivity(services) {
3047
3392
  { type: "session.status.changed", payload: { status: "failed" } }
3048
3393
  ], true);
3049
3394
  await finishTurn(db, input.workspaceId, turnId, "failed");
3395
+ turnMetricOutcome = "failed";
3050
3396
  await setSessionStatus(db, input.workspaceId, input.sessionId, "failed", null);
3051
3397
  await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed", `turn:${turnId}`);
3052
3398
  return { status: "failed" };
@@ -3057,6 +3403,9 @@ function createRunAgentTurnActivity(services) {
3057
3403
  status: activityStatus,
3058
3404
  durationSeconds
3059
3405
  });
3406
+ if (turnId && activityStatus !== "unknown") {
3407
+ turnLifecycleMetricsFor(observability).finish(turnId, turnMetricOutcome, durationSeconds);
3408
+ }
3060
3409
  activitySpan.end({
3061
3410
  attributes: {
3062
3411
  "opengeni.turn_id": turnId ?? "",
@@ -3279,7 +3628,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
3279
3628
  idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`
3280
3629
  });
3281
3630
  if (costMicros > 0) {
3282
- await applyCreditDebitUpToBalance(db, {
3631
+ const result = await applyCreditDebitUpToBalance(db, {
3283
3632
  accountId: input.accountId,
3284
3633
  workspaceId: input.workspaceId,
3285
3634
  type: "model_usage_debit",
@@ -3297,6 +3646,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
3297
3646
  totalTokens
3298
3647
  }
3299
3648
  });
3649
+ recordCreditMicros(input.observability, "usage", result.debitedMicros);
3300
3650
  }
3301
3651
  }
3302
3652
  function positiveInt(value) {
@@ -3417,6 +3767,8 @@ function startOfUtcMonth3(date = /* @__PURE__ */ new Date()) {
3417
3767
  import {
3418
3768
  accrueWarmSeconds as accrueWarmSeconds2,
3419
3769
  confirmDrainCold,
3770
+ countQueuedTurns,
3771
+ countSandboxLeasesByLiveness,
3420
3772
  forceDrainOverLimitViewerOnlyBoxes,
3421
3773
  getBillingBalance as getBillingBalance4,
3422
3774
  listLiveModalSandboxLeaseAttributions,
@@ -3443,6 +3795,7 @@ function createSandboxLeaseActivities(services, options = {}) {
3443
3795
  if (!settings.sandboxOwnershipEnabled) {
3444
3796
  return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
3445
3797
  }
3798
+ await refreshQueueAndLeaseGauges(db, observability);
3446
3799
  const metered = await accrueWarmTick(db, settings, observability);
3447
3800
  const forceDrained = await forceDrainOverLimitWorkspaces(db, settings, metered.workspaceIds, observability);
3448
3801
  const drainable = await reapStaleLeaseHoldersGlobal(db, {
@@ -3472,11 +3825,13 @@ function createSandboxLeaseActivities(services, options = {}) {
3472
3825
  }
3473
3826
  try {
3474
3827
  modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
3828
+ recordSandboxOrphansTerminated(observability, modalOrphansTerminated);
3475
3829
  } catch (error) {
3476
3830
  observability.warn("sandbox reaper: Modal orphan sweep failed", {
3477
3831
  error: error instanceof Error ? error.message : String(error)
3478
3832
  });
3479
3833
  }
3834
+ await refreshQueueAndLeaseGauges(db, observability);
3480
3835
  if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
3481
3836
  observability.info("sandbox reaper swept", {
3482
3837
  drainable: drainable.length,
@@ -3518,6 +3873,7 @@ async function accrueWarmTick(db, settings, observability) {
3518
3873
  if (result.accrued) {
3519
3874
  accrued += 1;
3520
3875
  }
3876
+ recordCreditMicros(observability, "usage", result.costMicros);
3521
3877
  } catch (error) {
3522
3878
  observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
3523
3879
  workspaceId: lease.workspaceId,
@@ -3528,6 +3884,22 @@ async function accrueWarmTick(db, settings, observability) {
3528
3884
  }
3529
3885
  return { accrued, workspaceIds };
3530
3886
  }
3887
+ async function refreshQueueAndLeaseGauges(db, observability) {
3888
+ try {
3889
+ recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
3890
+ } catch (error) {
3891
+ observability.warn("sandbox reaper: queued-turn gauge refresh failed", {
3892
+ error: error instanceof Error ? error.message : String(error)
3893
+ });
3894
+ }
3895
+ try {
3896
+ recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
3897
+ } catch (error) {
3898
+ observability.warn("sandbox reaper: sandbox-lease gauge refresh failed", {
3899
+ error: error instanceof Error ? error.message : String(error)
3900
+ });
3901
+ }
3902
+ }
3531
3903
  async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observability) {
3532
3904
  const enforceBalance = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
3533
3905
  const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
@@ -4058,6 +4430,7 @@ function startOfUtcMonth4() {
4058
4430
  // src/activities/session-state.ts
4059
4431
  import {
4060
4432
  claimNextQueuedTurn as claimNextQueuedTurnDb2,
4433
+ countQueuedTurns as countQueuedTurns2,
4061
4434
  countTurnSessionHistoryItems,
4062
4435
  finishTurn as finishTurn2,
4063
4436
  getSessionEvent as getSessionEvent4,
@@ -4101,7 +4474,7 @@ function createSessionStateActivities(services) {
4101
4474
  await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed");
4102
4475
  }
4103
4476
  async function interruptActiveTurn2(input) {
4104
- const { db, bus } = await services();
4477
+ const { db, bus, observability } = await services();
4105
4478
  const session = await requireSession4(db, input.workspaceId, input.sessionId);
4106
4479
  const trigger = await getSessionEvent4(db, input.workspaceId, input.triggerEventId);
4107
4480
  if (!isSteerInterrupt(trigger)) {
@@ -4124,9 +4497,10 @@ function createSessionStateActivities(services) {
4124
4497
  ]);
4125
4498
  await finishTurn2(db, input.workspaceId, session.activeTurnId, "cancelled");
4126
4499
  await setSessionStatus2(db, input.workspaceId, input.sessionId, "queued", null);
4500
+ await refreshQueuedTurnsGauge(db, observability);
4127
4501
  }
4128
4502
  async function requeueTurnAfterWorkerDeath2(input) {
4129
- const { settings, db, bus } = await services();
4503
+ const { settings, db, bus, observability } = await services();
4130
4504
  const turn = await getSessionTurn2(db, input.workspaceId, input.turnId);
4131
4505
  if (!turn || turn.status !== "running" && turn.status !== "requires_action") {
4132
4506
  return { action: "stale" };
@@ -4166,11 +4540,14 @@ function createSessionStateActivities(services) {
4166
4540
  throw requeueError;
4167
4541
  }
4168
4542
  await setSessionStatus2(db, input.workspaceId, input.sessionId, "queued", null);
4543
+ await refreshQueuedTurnsGauge(db, observability);
4169
4544
  return { action: "requeued", redispatches };
4170
4545
  }
4171
4546
  async function claimNextQueuedTurn2(input) {
4172
- const { db } = await services();
4173
- return await claimNextQueuedTurnDb2(db, input.workspaceId, input.sessionId, input.workflowId);
4547
+ const { db, observability } = await services();
4548
+ const turn = await claimNextQueuedTurnDb2(db, input.workspaceId, input.sessionId, input.workflowId);
4549
+ await refreshQueuedTurnsGauge(db, observability);
4550
+ return turn;
4174
4551
  }
4175
4552
  async function markSessionIdle2(input) {
4176
4553
  const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
@@ -4178,6 +4555,7 @@ function createSessionStateActivities(services) {
4178
4555
  if (session.status === "queued" || session.status === "running") {
4179
4556
  await setSessionStatus2(db, input.workspaceId, input.sessionId, "idle", null);
4180
4557
  }
4558
+ await refreshQueuedTurnsGauge(db, observability);
4181
4559
  await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "idle");
4182
4560
  }
4183
4561
  return {
@@ -4188,6 +4566,12 @@ function createSessionStateActivities(services) {
4188
4566
  markSessionIdle: markSessionIdle2
4189
4567
  };
4190
4568
  }
4569
+ async function refreshQueuedTurnsGauge(db, observability) {
4570
+ try {
4571
+ recordTurnsQueuedGauge(observability, await countQueuedTurns2(db));
4572
+ } catch {
4573
+ }
4574
+ }
4191
4575
 
4192
4576
  // src/activities.ts
4193
4577
  function createActivities(dependencies = {}) {
@@ -4195,6 +4579,7 @@ function createActivities(dependencies = {}) {
4195
4579
  async function services() {
4196
4580
  servicesPromise ??= (async () => {
4197
4581
  const settings = dependencies.settings ?? getSettings();
4582
+ const observability = dependencies.observability ?? createObservability(settings, { component: "worker" });
4198
4583
  const searchPath = dbSearchPath(settings);
4199
4584
  const dbClient = dependencies.db ? null : createDb(settings.databaseUrl, {
4200
4585
  ...searchPath ? { searchPath } : {},
@@ -4219,12 +4604,15 @@ function createActivities(dependencies = {}) {
4219
4604
  // events by sequence). See `.agents/skills/opengeni/references/eventbus-binding-contract.md`.
4220
4605
  bus: dependencies.bus ?? await createNatsEventBus(
4221
4606
  settings.natsUrl,
4222
- controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0
4607
+ controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
4608
+ { logger: observabilityEventLogger(observability) }
4223
4609
  ),
4224
- runtime: dependencies.runtime ?? createProductionAgentRuntime(),
4610
+ runtime: dependencies.runtime ?? createProductionAgentRuntime({
4611
+ metrics: runtimeMetricsHooksForObservability(observability)
4612
+ }),
4225
4613
  objectStorage: dependencies.objectStorage ?? createObjectStorage2(settings),
4226
4614
  documentServices: dependencies.documentServices ?? createDocumentServices(settings),
4227
- observability: dependencies.observability ?? createObservability(settings, { component: "worker" }),
4615
+ observability,
4228
4616
  wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
4229
4617
  // §7.5 P3 — host-entitlements port. No constructed default: standalone
4230
4618
  // has no host meter, so unset → null → `ensureRunAllowed` reads the
@@ -4273,6 +4661,86 @@ var maybeContinueGoal = defaultActivities.maybeContinueGoal;
4273
4661
  var pauseGoalForInterrupt = defaultActivities.pauseGoalForInterrupt;
4274
4662
  var reapSandboxLeases = defaultActivities.reapSandboxLeases;
4275
4663
 
4664
+ // src/http.ts
4665
+ import { dbSql } from "@opengeni/db";
4666
+ function startWorkerHttpServer(input) {
4667
+ const { settings, observability, checks } = input;
4668
+ const timeoutMs = input.timeoutMs ?? 2e3;
4669
+ return Bun.serve({
4670
+ hostname: "0.0.0.0",
4671
+ port: settings.workerHttpPort,
4672
+ async fetch(request) {
4673
+ const url = new URL(request.url);
4674
+ if (request.method !== "GET") {
4675
+ return Response.json({ error: "method_not_allowed" }, { status: 405 });
4676
+ }
4677
+ if (url.pathname === "/healthz") {
4678
+ return Response.json({
4679
+ service: settings.serviceName,
4680
+ environment: settings.environment,
4681
+ deploymentRevision: settings.deploymentRevision,
4682
+ ok: true
4683
+ });
4684
+ }
4685
+ if (url.pathname === "/metrics") {
4686
+ return new Response(await observability.prometheusMetrics(), {
4687
+ status: 200,
4688
+ headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" }
4689
+ });
4690
+ }
4691
+ if (url.pathname === "/readyz") {
4692
+ const result = await runReadinessChecks(checks, timeoutMs);
4693
+ return Response.json(result, { status: result.ok ? 200 : 503 });
4694
+ }
4695
+ return Response.json({ error: "not_found" }, { status: 404 });
4696
+ }
4697
+ });
4698
+ }
4699
+ async function runReadinessChecks(checks, timeoutMs = 2e3) {
4700
+ const entries = await Promise.all(
4701
+ Object.entries(checks).map(async ([name, check]) => {
4702
+ try {
4703
+ await withTimeout(Promise.resolve().then(check), timeoutMs);
4704
+ return [name, { ok: true }];
4705
+ } catch (error) {
4706
+ return [name, { ok: false, error: error instanceof Error ? error.message : String(error) }];
4707
+ }
4708
+ })
4709
+ );
4710
+ const result = Object.fromEntries(entries);
4711
+ return {
4712
+ ok: Object.values(result).every((check) => check.ok),
4713
+ checks: result
4714
+ };
4715
+ }
4716
+ function dbReadyCheck(db) {
4717
+ return async () => {
4718
+ await db.execute(dbSql`select 1`);
4719
+ };
4720
+ }
4721
+ function natsReadyCheck(bus) {
4722
+ return () => {
4723
+ if (bus.isConnected && !bus.isConnected()) {
4724
+ throw new Error("NATS is not connected");
4725
+ }
4726
+ };
4727
+ }
4728
+ async function withTimeout(promise, timeoutMs) {
4729
+ let timer;
4730
+ try {
4731
+ return await Promise.race([
4732
+ promise,
4733
+ new Promise((_, reject) => {
4734
+ timer = setTimeout(() => reject(new Error(`readiness check timed out after ${timeoutMs}ms`)), timeoutMs);
4735
+ })
4736
+ ]);
4737
+ } finally {
4738
+ if (timer) {
4739
+ clearTimeout(timer);
4740
+ }
4741
+ }
4742
+ }
4743
+
4276
4744
  // src/index.ts
4277
4745
  var SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
4278
4746
  async function createOpenGeniWorker(options = {}) {
@@ -4313,6 +4781,9 @@ async function createWorkerWorkflowSignaler(settings) {
4313
4781
  signal: "queueChanged"
4314
4782
  });
4315
4783
  },
4784
+ check: async () => {
4785
+ await connection.workflowService.getSystemInfo({});
4786
+ },
4316
4787
  close: async () => {
4317
4788
  await connection.close();
4318
4789
  }
@@ -4370,35 +4841,71 @@ async function registerSandboxReaperSchedule(settings, observability) {
4370
4841
  async function startWorker() {
4371
4842
  const settings = getSettings2();
4372
4843
  const observability = createObservability2(settings, { component: "worker" });
4373
- const signaler = await retryStartupDependency(
4374
- "Temporal client",
4375
- () => createWorkerWorkflowSignaler(settings),
4376
- {
4377
- ...startupRetryOptions(settings),
4378
- onRetry: (event) => logStartupDependencyRetry(observability, event)
4379
- }
4380
- );
4381
- const { worker, connection } = await createOpenGeniWorker({
4382
- settings,
4383
- activityDependencies: { observability, wakeSessionWorkflow: signaler.wakeSessionWorkflow }
4384
- });
4385
- const reaperSchedule = await retryStartupDependency(
4386
- "Temporal schedule (sandbox reaper)",
4387
- () => registerSandboxReaperSchedule(settings, observability),
4388
- {
4389
- ...startupRetryOptions(settings),
4390
- onRetry: (event) => logStartupDependencyRetry(observability, event)
4391
- }
4392
- );
4393
- observability.info("OpenGeni worker listening", {
4394
- temporalTaskQueue: settings.temporalTaskQueue
4844
+ const retryOptions = startupRetryOptions(settings);
4845
+ const onRetry = (event) => logStartupDependencyRetry(observability, event);
4846
+ const searchPath = dbSearchPath2(settings);
4847
+ const dbClient = createDb2(settings.databaseUrl, {
4848
+ ...searchPath ? { searchPath } : {},
4849
+ rlsStrategy: settings.rlsStrategy
4395
4850
  });
4851
+ const controlPlaneAuth = resolveNatsControlPlaneAuth2(settings);
4852
+ let bus;
4853
+ let signaler;
4854
+ let workerBundle;
4855
+ let reaperSchedule;
4856
+ let httpServer;
4396
4857
  try {
4397
- await worker.run();
4858
+ bus = await retryStartupDependency(
4859
+ "NATS",
4860
+ () => createNatsEventBus2(
4861
+ settings.natsUrl,
4862
+ controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
4863
+ { logger: observabilityEventLogger(observability) }
4864
+ ),
4865
+ { ...retryOptions, onRetry }
4866
+ );
4867
+ signaler = await retryStartupDependency(
4868
+ "Temporal client",
4869
+ () => createWorkerWorkflowSignaler(settings),
4870
+ { ...retryOptions, onRetry }
4871
+ );
4872
+ workerBundle = await createOpenGeniWorker({
4873
+ settings,
4874
+ activityDependencies: {
4875
+ observability,
4876
+ wakeSessionWorkflow: signaler.wakeSessionWorkflow,
4877
+ db: dbClient.db,
4878
+ bus
4879
+ }
4880
+ });
4881
+ httpServer = startWorkerHttpServer({
4882
+ settings,
4883
+ observability,
4884
+ checks: {
4885
+ db: dbReadyCheck(dbClient.db),
4886
+ nats: natsReadyCheck(bus),
4887
+ temporal: signaler.check
4888
+ }
4889
+ });
4890
+ reaperSchedule = await retryStartupDependency(
4891
+ "Temporal schedule (sandbox reaper)",
4892
+ () => registerSandboxReaperSchedule(settings, observability),
4893
+ { ...retryOptions, onRetry }
4894
+ );
4895
+ observability.info("OpenGeni worker listening", {
4896
+ temporalTaskQueue: settings.temporalTaskQueue,
4897
+ httpPort: settings.workerHttpPort
4898
+ });
4899
+ await workerBundle.worker.run();
4398
4900
  } finally {
4399
- await connection.close();
4400
- await signaler.close();
4401
- await reaperSchedule.close();
4901
+ httpServer?.stop(true);
4902
+ await Promise.allSettled([
4903
+ workerBundle?.connection.close(),
4904
+ signaler?.close(),
4905
+ reaperSchedule?.close(),
4906
+ bus?.close(),
4907
+ dbClient.close()
4908
+ ]);
4402
4909
  }
4403
4910
  }
4404
4911
  if (import.meta.main) {