@opengeni/worker-bundle 0.22.8 → 0.23.4-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +10 -2
  2. package/dist/activities/agent-turn/agent-build.d.ts +1 -4
  3. package/dist/activities/agent-turn/claim.d.ts +3 -4
  4. package/dist/activities/agent-turn/codex.d.ts +8 -7
  5. package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
  6. package/dist/activities/agent-turn/errors.d.ts +13 -0
  7. package/dist/activities/agent-turn/finalization.d.ts +1 -3
  8. package/dist/activities/agent-turn/list-models.d.ts +16 -0
  9. package/dist/activities/agent-turn/model-usage.d.ts +10 -1
  10. package/dist/activities/agent-turn/quiescence.d.ts +39 -0
  11. package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
  12. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
  13. package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
  14. package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
  15. package/dist/activities/agent-turn/tool-environment.d.ts +1 -0
  16. package/dist/activities/agent-turn/tool-policy.d.ts +7 -5
  17. package/dist/activities/agent-turn/turn-context.d.ts +3 -8
  18. package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
  19. package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
  20. package/dist/activities/automations.d.ts +1 -0
  21. package/dist/activities/capabilities.d.ts +2 -1
  22. package/dist/activities/codex-rotation.d.ts +2 -0
  23. package/dist/activities/goals.d.ts +11 -0
  24. package/dist/activities/image-generation-references.d.ts +2 -1
  25. package/dist/activities/scheduled-tasks.d.ts +1 -0
  26. package/dist/activities/session-state.d.ts +2 -1
  27. package/dist/activities/types.d.ts +8 -0
  28. package/dist/{activities-control-VZIIV7RX.js → activities-control-SGU2WEVB.js} +328 -107
  29. package/dist/activities-control-SGU2WEVB.js.map +1 -0
  30. package/dist/activities-control.d.ts +2 -0
  31. package/dist/{activities-turn-5RGQDBFV.js → activities-turn-7EAQXW6Q.js} +828 -540
  32. package/dist/activities-turn-7EAQXW6Q.js.map +1 -0
  33. package/dist/activities.d.ts +2 -0
  34. package/dist/{chunk-O4HDX5SC.js → chunk-S2QVRVJW.js} +227 -36
  35. package/dist/chunk-S2QVRVJW.js.map +1 -0
  36. package/dist/{chunk-NAVSAZ56.js → chunk-TJIPCHMP.js} +162 -2
  37. package/dist/chunk-TJIPCHMP.js.map +1 -0
  38. package/dist/index.d.ts +1 -0
  39. package/dist/index.js +69 -6
  40. package/dist/index.js.map +1 -1
  41. package/dist/observability-metrics.d.ts +29 -0
  42. package/dist/personal-github-git-credentials.d.ts +1 -0
  43. package/dist/unhandled-rejection-boundary.d.ts +29 -0
  44. package/dist/workflow-bundle.js +26 -3
  45. package/dist/workflows/activities.d.ts +5 -0
  46. package/dist/workflows/automations.d.ts +5 -0
  47. package/package.json +18 -18
  48. package/src/activities/agent-run-admission.ts +7 -1
  49. package/src/activities/agent-turn/admission.ts +28 -6
  50. package/src/activities/agent-turn/agent-build.ts +15 -31
  51. package/src/activities/agent-turn/claim.ts +53 -72
  52. package/src/activities/agent-turn/codex.ts +21 -12
  53. package/src/activities/agent-turn/compaction-prep.ts +52 -1
  54. package/src/activities/agent-turn/errors.ts +78 -0
  55. package/src/activities/agent-turn/failure-settlement.ts +30 -19
  56. package/src/activities/agent-turn/finalization.ts +24 -21
  57. package/src/activities/agent-turn/history.ts +5 -1
  58. package/src/activities/agent-turn/list-models.ts +101 -0
  59. package/src/activities/agent-turn/model-usage.ts +88 -42
  60. package/src/activities/agent-turn/quiescence.ts +86 -0
  61. package/src/activities/agent-turn/run-credentials.ts +12 -1
  62. package/src/activities/agent-turn/run.ts +20 -14
  63. package/src/activities/agent-turn/sandbox-establish.ts +46 -11
  64. package/src/activities/agent-turn/sandbox-route.ts +21 -25
  65. package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
  66. package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
  67. package/src/activities/agent-turn/stream-attempt.ts +106 -14
  68. package/src/activities/agent-turn/tool-environment.ts +72 -14
  69. package/src/activities/agent-turn/tool-policy.ts +7 -6
  70. package/src/activities/agent-turn/turn-context.ts +12 -14
  71. package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
  72. package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
  73. package/src/activities/automations.ts +122 -67
  74. package/src/activities/capabilities.ts +72 -2
  75. package/src/activities/codex-capacity.ts +13 -5
  76. package/src/activities/codex-rotation.ts +17 -0
  77. package/src/activities/goals.ts +117 -35
  78. package/src/activities/image-generation-references.ts +9 -9
  79. package/src/activities/retained-screenshots.ts +104 -21
  80. package/src/activities/sandbox-lease.ts +7 -4
  81. package/src/activities/scheduled-tasks.ts +248 -30
  82. package/src/activities/session-state.ts +8 -1
  83. package/src/activities/site-auth-maintenance.ts +12 -12
  84. package/src/activities/types.ts +6 -0
  85. package/src/activities.ts +1 -0
  86. package/src/activity-services.ts +4 -1
  87. package/src/github-rest-mcp.ts +8 -4
  88. package/src/http.ts +3 -0
  89. package/src/index.ts +38 -4
  90. package/src/observability-metrics.ts +202 -0
  91. package/src/personal-github-git-credentials.ts +45 -4
  92. package/src/sandbox-resume.ts +1 -1
  93. package/src/sandbox-routing.ts +7 -4
  94. package/src/unhandled-rejection-boundary.ts +73 -0
  95. package/src/workflows/activities.ts +15 -0
  96. package/src/workflows/automations.ts +21 -2
  97. package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
  98. package/dist/activities/recording.d.ts +0 -90
  99. package/dist/activities-control-VZIIV7RX.js.map +0 -1
  100. package/dist/activities-turn-5RGQDBFV.js.map +0 -1
  101. package/dist/chunk-NAVSAZ56.js.map +0 -1
  102. package/dist/chunk-O4HDX5SC.js.map +0 -1
  103. package/src/activities/recording.ts +0 -279
package/src/index.ts CHANGED
@@ -9,6 +9,7 @@ import {
9
9
  } from "@opengeni/config";
10
10
  import {
11
11
  assertRuntimeDatabasePosture,
12
+ countSessionRecoveryBacklog,
12
13
  createDb,
13
14
  markSessionWorkflowWakeDelivered,
14
15
  type Database,
@@ -55,12 +56,15 @@ import {
55
56
  type WorkerLifecycleState,
56
57
  } from "./http";
57
58
  import {
59
+ initializeWorkerOutcomeMetrics,
58
60
  normalizeTurnTaskQueueStats,
59
61
  observabilityEventLogger,
62
+ startSessionRecoveryMonitor,
60
63
  startTurnCapacityMonitor,
61
64
  type TurnTaskQueueStats,
62
65
  } from "./observability-metrics";
63
66
  import {
67
+ resolveCatalogSettings,
64
68
  SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
65
69
  SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
66
70
  SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
@@ -86,6 +90,7 @@ import {
86
90
  type TurnWorkerMemoryPressureGuard,
87
91
  } from "./memory-pressure-guard";
88
92
  import { assertSandboxReaperActivityTimeout } from "./sandbox-reaper-timeout";
93
+ import { installWorkerUnhandledRejectionBoundary } from "./unhandled-rejection-boundary";
89
94
  import {
90
95
  SANDBOX_REAPER_V2_WORKFLOW_ID,
91
96
  sandboxLifecycleTaskQueue,
@@ -219,6 +224,7 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
219
224
  const observability =
220
225
  options.activityDependencies?.observability ??
221
226
  createObservability(settings, { component: `worker-${options.role}` });
227
+ initializeWorkerOutcomeMetrics(observability);
222
228
  if (options.role === "turn" && options.workflowBundle) {
223
229
  throw new Error("workflowBundle is valid only for the control worker role");
224
230
  }
@@ -793,11 +799,22 @@ export async function createOpenGeniWorkerService(
793
799
  let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
794
800
  let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
795
801
  let turnCapacityMonitor: ReturnType<typeof startTurnCapacityMonitor> | undefined;
802
+ let sessionRecoveryMonitor: ReturnType<typeof startSessionRecoveryMonitor> | undefined;
796
803
  const schedules: Array<{ close: () => Promise<void> }> = [];
797
804
  let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
798
805
  let memoryPressureGuard: TurnWorkerMemoryPressureGuard | undefined;
799
806
 
800
807
  try {
808
+ const resolvedCatalog = await retryStartupDependency(
809
+ "model catalog",
810
+ () => resolveCatalogSettings(options.activityDependencies.db, settings),
811
+ { ...retryOptions, onRetry },
812
+ );
813
+ observability.info("OpenGeni model catalog resolved", {
814
+ role: options.role,
815
+ catalogSource: resolvedCatalog.source,
816
+ catalogVersion: resolvedCatalog.version,
817
+ });
801
818
  const needsSignaler =
802
819
  options.role === "turn" ||
803
820
  !options.activityDependencies.wakeSessionWorkflow ||
@@ -866,6 +883,11 @@ export async function createOpenGeniWorkerService(
866
883
  observability,
867
884
  read: signaler.getTurnTaskQueueStats,
868
885
  });
886
+ } else {
887
+ sessionRecoveryMonitor = startSessionRecoveryMonitor({
888
+ observability,
889
+ read: async () => await countSessionRecoveryBacklog(options.activityDependencies.db),
890
+ });
869
891
  }
870
892
 
871
893
  if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
@@ -900,14 +922,18 @@ export async function createOpenGeniWorkerService(
900
922
  }
901
923
 
902
924
  if (options.http !== false) {
925
+ const databaseReady = dbReadyCheck(
926
+ options.http?.readinessDb ?? options.activityDependencies.db,
927
+ options.databasePosture,
928
+ );
903
929
  httpServer = startWorkerHttpServer({
904
930
  settings,
905
931
  observability,
906
932
  checks: {
907
- db: dbReadyCheck(
908
- options.http?.readinessDb ?? options.activityDependencies.db,
909
- options.databasePosture,
910
- ),
933
+ db: async () => {
934
+ await databaseReady();
935
+ await resolveCatalogSettings(options.activityDependencies.db, settings);
936
+ },
911
937
  nats: natsReadyCheck(options.activityDependencies.bus),
912
938
  temporal: temporalReadyCheck(workerBundle.connection),
913
939
  },
@@ -919,6 +945,7 @@ export async function createOpenGeniWorkerService(
919
945
  httpServer?.stop(true);
920
946
  await Promise.allSettled([
921
947
  turnCapacityMonitor?.close(),
948
+ sessionRecoveryMonitor?.close(),
922
949
  workerBundle?.connection.close(),
923
950
  signaler?.close(),
924
951
  ...schedules.map((schedule) => schedule.close()),
@@ -941,6 +968,7 @@ export async function createOpenGeniWorkerService(
941
968
  httpServer?.stop(true);
942
969
  await Promise.allSettled([
943
970
  turnCapacityMonitor?.close(),
971
+ sessionRecoveryMonitor?.close(),
944
972
  activeWorkerBundle.connection.close(),
945
973
  activeSignaler?.close(),
946
974
  ...schedules.map((schedule) => schedule.close()),
@@ -1032,6 +1060,11 @@ export async function startWorker() {
1032
1060
  }
1033
1061
  const settings = getSettings();
1034
1062
  const observability = createObservability(settings, { component: `worker-${role}` });
1063
+ // The Agents SDK also listens for unhandled rejections and exits the process.
1064
+ // Keep detached SDK/transport promises observational here: durable turn
1065
+ // activity boundaries own failures, while process exit causes cross-session
1066
+ // lease loss for every turn assigned to this shared worker pod.
1067
+ const disposeUnhandledRejectionBoundary = installWorkerUnhandledRejectionBoundary();
1035
1068
  const retryOptions = startupRetryOptions(settings);
1036
1069
  const onRetry = (event: Parameters<typeof logStartupDependencyRetry>[1]) =>
1037
1070
  logStartupDependencyRetry(observability, event);
@@ -1090,6 +1123,7 @@ export async function startWorker() {
1090
1123
  });
1091
1124
  } finally {
1092
1125
  await Promise.allSettled([bus?.close(), readinessDbClient.close(), dbClient.close()]);
1126
+ disposeUnhandledRejectionBoundary();
1093
1127
  }
1094
1128
  }
1095
1129
 
@@ -1,6 +1,7 @@
1
1
  import { createHash } from "node:crypto";
2
2
  import { errorCodeToJSON } from "@opengeni/agent-proto";
3
3
  import { SandboxBackend, type SessionEventType } from "@opengeni/contracts";
4
+ import type { SessionEventAppendPhaseObservation } from "@opengeni/db";
4
5
  import type { EventLogger } from "@opengeni/events";
5
6
  import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
6
7
  import type { CompanyBrainContributionReceipt } from "./model-context-contributions";
@@ -18,6 +19,7 @@ import {
18
19
  } from "@opengeni/runtime";
19
20
 
20
21
  export type TurnOutcome = "completed" | "failed" | "cancelled" | "recovering";
22
+ export type WorkerDeathRecoveryOutcome = "recovering" | "exhausted";
21
23
  export type CreditMicrosKind = "usage" | "grant" | "topup" | "refund";
22
24
  export type SandboxLeaseLiveness = "cold" | "warming" | "warm" | "draining";
23
25
  export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
@@ -28,6 +30,10 @@ export type TurnTaskQueueStats = {
28
30
  tasksAddRate: number;
29
31
  tasksDispatchRate: number;
30
32
  };
33
+ export type SessionRecoveryBacklog = {
34
+ quiescence_missing: number;
35
+ projection_stale: number;
36
+ };
31
37
 
32
38
  export type TemporalTurnTaskQueueStats = {
33
39
  approximateBacklogCount?: unknown;
@@ -49,6 +55,12 @@ const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>(
49
55
  // before floating-point precision can degrade; a restart resets both the
50
56
  // process-local Prometheus registry and this guard together.
51
57
  const MAX_MODEL_CACHE_COUNTER_TOTAL = 1_000_000_000_000;
58
+ const TURN_OUTCOMES: readonly TurnOutcome[] = ["completed", "failed", "cancelled", "recovering"];
59
+ const WORKER_DEATH_RECOVERY_OUTCOMES: readonly WorkerDeathRecoveryOutcome[] = [
60
+ "recovering",
61
+ "exhausted",
62
+ ];
63
+ const WORKER_DEATH_TIMEOUT_TYPES = ["heartbeat", "schedule_to_start"] as const;
52
64
 
53
65
  export function observabilityEventLogger(observability: Observability): EventLogger {
54
66
  return {
@@ -163,6 +175,19 @@ export function runtimeMetricsHooksForObservability(
163
175
  value: durationSeconds,
164
176
  });
165
177
  },
178
+ onMcpLifecycle: ({ phase, policy, outcome, durationSeconds }) => {
179
+ observability.incrementCounter({
180
+ name: "opengeni_mcp_lifecycle_operations_total",
181
+ help: "Total physical MCP lifecycle operations by bounded phase, policy, and outcome.",
182
+ labels: { phase, policy, outcome },
183
+ });
184
+ observability.observeHistogram({
185
+ name: "opengeni_mcp_lifecycle_operation_duration_seconds",
186
+ help: "MCP lifecycle operation duration in seconds by bounded phase, policy, and outcome.",
187
+ labels: { phase, policy, outcome },
188
+ value: durationSeconds,
189
+ });
190
+ },
166
191
  onSandboxOp: ({ backend, op, outcome, code, healed, durationSeconds, replyBytes }) => {
167
192
  observability.incrementCounter({
168
193
  name: "opengeni_machine_op_total",
@@ -300,6 +325,53 @@ export function turnLifecycleMetricsFor(observability: Observability): TurnLifec
300
325
  return tracker;
301
326
  }
302
327
 
328
+ export function initializeWorkerOutcomeMetrics(observability: Observability): void {
329
+ for (const outcome of TURN_OUTCOMES) {
330
+ observability.incrementCounter({
331
+ name: "opengeni_turns_total",
332
+ help: "Total agent turns by terminal outcome.",
333
+ labels: { outcome },
334
+ amount: 0,
335
+ });
336
+ }
337
+ for (const outcome of WORKER_DEATH_RECOVERY_OUTCOMES) {
338
+ for (const timeoutType of WORKER_DEATH_TIMEOUT_TYPES) {
339
+ observability.incrementCounter({
340
+ name: "opengeni_turn_worker_death_recoveries_total",
341
+ help: "Total durable same-turn worker-death recovery outcomes.",
342
+ labels: { outcome, timeout_type: timeoutType },
343
+ amount: 0,
344
+ });
345
+ }
346
+ }
347
+ }
348
+
349
+ export function recordWorkerDeathRecoveryMetrics(
350
+ observability: Observability,
351
+ input: {
352
+ outcome: WorkerDeathRecoveryOutcome;
353
+ timeoutType: "HEARTBEAT" | "SCHEDULE_TO_START";
354
+ },
355
+ ): void {
356
+ const turnOutcome: TurnOutcome = input.outcome === "exhausted" ? "failed" : "recovering";
357
+ observability.incrementCounter({
358
+ name: "opengeni_turn_worker_death_recoveries_total",
359
+ help: "Total durable same-turn worker-death recovery outcomes.",
360
+ labels: {
361
+ outcome: input.outcome,
362
+ timeout_type: input.timeoutType.toLowerCase(),
363
+ },
364
+ });
365
+ // The turn process that owned this attempt is gone, so its process-local
366
+ // lifecycle tracker cannot publish the outcome. Record it from the fenced
367
+ // control activity only after the durable recovery transaction wins.
368
+ observability.incrementCounter({
369
+ name: "opengeni_turns_total",
370
+ help: "Total agent turns by terminal outcome.",
371
+ labels: { outcome: turnOutcome },
372
+ });
373
+ }
374
+
303
375
  export class TurnLifecycleMetrics {
304
376
  private readonly turns = new Map<string, { startedAt: number; lastProgressAt: number }>();
305
377
  private timer: ReturnType<typeof setInterval> | null = null;
@@ -676,6 +748,88 @@ export function startTurnCapacityMonitor(input: {
676
748
  };
677
749
  }
678
750
 
751
+ export function recordSessionRecoveryBacklogGauges(
752
+ observability: Observability,
753
+ counts: SessionRecoveryBacklog,
754
+ ): void {
755
+ for (const state of ["quiescence_missing", "projection_stale"] as const) {
756
+ observability.setGauge({
757
+ name: "opengeni_session_recovery_backlog",
758
+ help: "Current durable recovering-session obligations with no active attempt, by bounded reconciliation state.",
759
+ labels: { state },
760
+ value: nonnegativeFinite(counts[state]),
761
+ });
762
+ }
763
+ }
764
+
765
+ export function startSessionRecoveryMonitor(input: {
766
+ observability: Observability;
767
+ read: () => Promise<SessionRecoveryBacklog>;
768
+ intervalMs?: number;
769
+ now?: () => number;
770
+ }): { close: () => Promise<void> } {
771
+ const intervalMs = input.intervalMs ?? 60_000;
772
+ const now = input.now ?? Date.now;
773
+ const startedAt = now();
774
+ let lastSuccessAt: number | null = null;
775
+ let lastReadSucceeded = false;
776
+ let stopped = false;
777
+ let running: Promise<void> | null = null;
778
+ const recordStatus = () => {
779
+ const observedAt = now();
780
+ const successAgeMs = observedAt - (lastSuccessAt ?? startedAt);
781
+ const set = (name: string, help: string, value: number) =>
782
+ input.observability.setGauge({ name, help, value });
783
+ set(
784
+ "opengeni_session_recovery_monitor_last_read_success",
785
+ "Whether the latest durable session-recovery aggregate read completed successfully.",
786
+ lastReadSucceeded ? 1 : 0,
787
+ );
788
+ set(
789
+ "opengeni_session_recovery_monitor_last_success_timestamp_seconds",
790
+ "Unix timestamp of the latest successful durable session-recovery aggregate read, or zero before one succeeds.",
791
+ lastSuccessAt === null ? 0 : lastSuccessAt / 1_000,
792
+ );
793
+ set(
794
+ "opengeni_session_recovery_monitor_fresh",
795
+ "Whether durable session-recovery backlog gauges have a successful read within three monitor intervals.",
796
+ lastReadSucceeded && lastSuccessAt !== null && successAgeMs <= intervalMs * 3 ? 1 : 0,
797
+ );
798
+ };
799
+ const refresh = () => {
800
+ recordStatus();
801
+ if (stopped || running) return;
802
+ running = input
803
+ .read()
804
+ .then((counts) => {
805
+ recordSessionRecoveryBacklogGauges(input.observability, counts);
806
+ lastSuccessAt = now();
807
+ lastReadSucceeded = true;
808
+ recordStatus();
809
+ })
810
+ .catch((error) => {
811
+ lastReadSucceeded = false;
812
+ recordStatus();
813
+ input.observability.warn("session recovery monitor: durable aggregate read failed", {
814
+ error: error instanceof Error ? error.message : String(error),
815
+ });
816
+ })
817
+ .finally(() => {
818
+ running = null;
819
+ });
820
+ };
821
+ refresh();
822
+ const timer = setInterval(refresh, intervalMs);
823
+ timer.unref?.();
824
+ return {
825
+ close: async () => {
826
+ stopped = true;
827
+ clearInterval(timer);
828
+ await running;
829
+ },
830
+ };
831
+ }
832
+
679
833
  export function recordSandboxLeaseGauges(
680
834
  observability: Observability,
681
835
  counts: Partial<Record<SandboxLeaseLiveness, number>>,
@@ -869,6 +1023,7 @@ export function recordSandboxRotationBacklogGauges(
869
1023
  turnBlocked: number;
870
1024
  directBlocked: number;
871
1025
  processBlocked: number;
1026
+ interactionBlocked: number;
872
1027
  },
873
1028
  ): void {
874
1029
  const values = {
@@ -877,6 +1032,7 @@ export function recordSandboxRotationBacklogGauges(
877
1032
  turn_blocked: backlog.turnBlocked,
878
1033
  direct_blocked: backlog.directBlocked,
879
1034
  process_blocked: backlog.processBlocked,
1035
+ interaction_blocked: backlog.interactionBlocked,
880
1036
  } as const;
881
1037
  for (const [kind, value] of Object.entries(values)) {
882
1038
  observability.setGauge({
@@ -1349,6 +1505,22 @@ export function recordSandboxLogicalProvision(
1349
1505
  }
1350
1506
  }
1351
1507
 
1508
+ export function recordSandboxSharedPreparation(
1509
+ observability: Observability,
1510
+ measurement: {
1511
+ path: "owner" | "joined" | "reused";
1512
+ outcome: "completed" | "failed";
1513
+ durationSeconds: number;
1514
+ },
1515
+ ): void {
1516
+ observability.observeHistogram({
1517
+ name: "opengeni_sandbox_shared_preparation_duration_seconds",
1518
+ help: "Duration of exact-lease immutable sandbox preparation by durable coordination path and outcome.",
1519
+ labels: { path: measurement.path, outcome: measurement.outcome },
1520
+ value: Math.max(0, measurement.durationSeconds),
1521
+ });
1522
+ }
1523
+
1352
1524
  export function recordTurnWorkerPreparationTotal(
1353
1525
  observability: Observability,
1354
1526
  input: {
@@ -1535,6 +1707,36 @@ export function recordSessionEventAppendLatency(
1535
1707
  });
1536
1708
  }
1537
1709
 
1710
+ export function sessionEventBatchSizeClass(eventCount: number): string {
1711
+ if (!Number.isFinite(eventCount) || eventCount <= 0) return "unknown";
1712
+ if (eventCount === 1) return "1";
1713
+ if (eventCount <= 5) return "2-5";
1714
+ if (eventCount <= 10) return "6-10";
1715
+ if (eventCount <= 25) return "11-25";
1716
+ if (eventCount <= 50) return "26-50";
1717
+ return "51+";
1718
+ }
1719
+
1720
+ /** Bounded database-phase attribution for exact-attempt event appends. */
1721
+ export function recordSessionEventAppendPhase(
1722
+ observability: Observability,
1723
+ observation: SessionEventAppendPhaseObservation,
1724
+ ): void {
1725
+ observability.observeHistogram({
1726
+ name: "opengeni_session_event_append_phase_seconds",
1727
+ help: "Duration of one bounded database phase in an exact-attempt session event append.",
1728
+ buckets: STREAM_IO_BUCKETS,
1729
+ labels: {
1730
+ path: "turn_attempt",
1731
+ event_class: observation.eventClass,
1732
+ batch_size_class: sessionEventBatchSizeClass(observation.eventCount),
1733
+ phase: observation.phase,
1734
+ outcome: observation.outcome,
1735
+ },
1736
+ value: Math.max(0, observation.durationSeconds),
1737
+ });
1738
+ }
1739
+
1538
1740
  /** Latency of the best-effort NATS live fan-out — the delivery path. A p99 climb
1539
1741
  * here (with append healthy) is delivery, not the write path. */
1540
1742
  export function recordSessionEventPublishLatency(
@@ -24,6 +24,8 @@ import {
24
24
  type PersonalGitHubGitBrokerRepositoryClaim,
25
25
  } from "@opengeni/github";
26
26
  import { randomBytes, randomUUID } from "node:crypto";
27
+ import { execFileSync } from "node:child_process";
28
+ import { isIP } from "node:net";
27
29
  import { isDeepStrictEqual } from "node:util";
28
30
 
29
31
  const PERSONAL_GITHUB_SERVER_ID = "github:personal";
@@ -39,7 +41,7 @@ export function buildPersonalGitHubGitCredentials(
39
41
  ): NonNullable<ConnectionCredentialsPort["gitCredentials"]> | null {
40
42
  if (!settings.githubPersonalOauthEnabled) return null;
41
43
  const secret = settings.integrationsStateSecret?.trim();
42
- const publicOrigin = personalGitBrokerOrigin(settings.publicBaseUrl);
44
+ const publicOrigin = personalGitBrokerOrigin(settings);
43
45
  if (!secret || !publicOrigin) {
44
46
  throw new Error("personal GitHub Git broker configuration is unavailable");
45
47
  }
@@ -255,19 +257,58 @@ function personalGitHubCanWrite(permissions: {
255
257
  return permissions.push || permissions.maintain || permissions.admin;
256
258
  }
257
259
 
258
- function personalGitBrokerOrigin(value: string | undefined): string | null {
260
+ let cachedDockerBridgeGateway: string | null | undefined;
261
+
262
+ function dockerBridgeGateway(): string | null {
263
+ if (cachedDockerBridgeGateway !== undefined) return cachedDockerBridgeGateway;
264
+ try {
265
+ const value = execFileSync(
266
+ "docker",
267
+ ["network", "inspect", "bridge", "--format", "{{(index .IPAM.Config 0).Gateway}}"],
268
+ { encoding: "utf8", timeout: 2_000, maxBuffer: 16 * 1024 },
269
+ ).trim();
270
+ cachedDockerBridgeGateway = isIP(value) ? value : null;
271
+ } catch {
272
+ cachedDockerBridgeGateway = null;
273
+ }
274
+ return cachedDockerBridgeGateway;
275
+ }
276
+
277
+ export function personalGitBrokerOrigin(settings: Settings): string | null {
278
+ const value =
279
+ settings.opengeniMcpUrl ??
280
+ (settings.sandboxBackend === "docker"
281
+ ? `http://host.docker.internal:${settings.apiPort}`
282
+ : settings.sandboxBackend === "local" || settings.sandboxBackend === "none"
283
+ ? `http://127.0.0.1:${settings.apiPort}`
284
+ : settings.publicBaseUrl);
259
285
  if (!value) return null;
260
286
  const url = new URL(value);
261
287
  if (
262
- url.protocol !== "https:" ||
288
+ settings.sandboxBackend === "docker" &&
289
+ (url.hostname === "127.0.0.1" || url.hostname === "localhost")
290
+ ) {
291
+ url.hostname = dockerBridgeGateway() ?? "host.docker.internal";
292
+ }
293
+ const dockerPrivateOrigin =
294
+ settings.sandboxBackend === "docker" &&
295
+ (url.hostname === "host.docker.internal" ||
296
+ url.hostname === cachedDockerBridgeGateway ||
297
+ /^[a-z][a-z0-9-]{0,62}$/iu.test(url.hostname));
298
+ const privateExecutionOrigin =
299
+ (url.protocol === "http:" &&
300
+ (url.hostname === "127.0.0.1" || url.hostname === "localhost" || dockerPrivateOrigin)) ||
301
+ ["local", "test"].includes(settings.environment);
302
+ if (
303
+ (url.protocol !== "https:" && !privateExecutionOrigin) ||
263
304
  url.username ||
264
305
  url.password ||
265
- url.pathname !== "/" ||
266
306
  url.search ||
267
307
  url.hash
268
308
  ) {
269
309
  return null;
270
310
  }
311
+ url.pathname = "/";
271
312
  return url.origin;
272
313
  }
273
314
 
@@ -11,7 +11,7 @@
11
11
  // re-enters admission instead of creating a rival.
12
12
  // 3. (spawner) commitWarmingToWarm (the lease_epoch++ fence + folds the resume
13
13
  // envelope onto the lease). Optional desktop work is deliberately absent:
14
- // the viewer and the first actual computer-use action initialize :0 lazily.
14
+ // viewers and managed ComputerSession operations initialize it lazily.
15
15
  // 4. the caller injects {client, session, sessionState} NON-OWNED into the run
16
16
  // (the SDK never reaps it — the keystone), runs, then in `finally` calls the
17
17
  // returned `release()` and drops the in-memory handle. NEVER provider-delete
@@ -672,7 +672,7 @@ function settleRetainedProcessForTurn(
672
672
  ) {
673
673
  throw new Error("Retained-process settlement lost its exact durable backend identity");
674
674
  }
675
- await settleRetainedProcess(services.db, {
675
+ const settlement = await settleRetainedProcess(services.db, {
676
676
  accountId: fence.accountId,
677
677
  workspaceId: ids.workspaceId,
678
678
  sessionId: ids.sessionId,
@@ -683,14 +683,17 @@ function settleRetainedProcessForTurn(
683
683
  reason: proof.reason,
684
684
  idleGraceMs: services.settings.sandboxIdleGraceMs,
685
685
  });
686
+ if (settlement.process.state === "active") {
687
+ throw new Error("Retained-process settlement returned an active durable process");
688
+ }
686
689
  await settleSessionBackgroundCommandForRetainedProcess(services.db, {
687
690
  accountId: fence.accountId,
688
691
  workspaceId: ids.workspaceId,
689
692
  sessionId: ids.sessionId,
690
693
  retainedProcessId: process.id,
691
- outcome: proof.outcome,
692
- exitCode: proof.exitCode,
693
- reason: proof.reason,
694
+ outcome: settlement.process.state,
695
+ exitCode: settlement.process.exitCode,
696
+ reason: settlement.process.settlementReason ?? proof.reason,
694
697
  });
695
698
  };
696
699
  }
@@ -0,0 +1,73 @@
1
+ export type WorkerUnhandledRejectionProcess = {
2
+ on: (event: "unhandledRejection", listener: (reason: unknown) => void) => void;
3
+ off: (event: "unhandledRejection", listener: (reason: unknown) => void) => void;
4
+ };
5
+
6
+ export type WorkerUnhandledRejectionClassification =
7
+ | "detached_runtime_mcp_lifecycle"
8
+ | "detached_unknown";
9
+
10
+ type RuntimeMcpLifecycleError = {
11
+ name: "McpLifecycleError";
12
+ code: "mcp_connect_failed" | "mcp_close_failed";
13
+ origin: "runtime";
14
+ serverId: string;
15
+ };
16
+
17
+ /**
18
+ * The Agents SDK owns MCP connect promises and also installs a process-global
19
+ * unhandled-rejection listener. A best-effort lifecycle rejection can surface
20
+ * twice under concurrent teardown: once through the awaited MCP result and once
21
+ * through the SDK's detached worker promise. The first path already records and
22
+ * degrades the server; the second must not terminate the whole turn-worker pod.
23
+ */
24
+ export function isDetachedRuntimeMcpLifecycleRejection(
25
+ reason: unknown,
26
+ ): reason is RuntimeMcpLifecycleError {
27
+ try {
28
+ if (!reason || typeof reason !== "object") return false;
29
+ const candidate = reason as Partial<RuntimeMcpLifecycleError>;
30
+ return (
31
+ candidate.name === "McpLifecycleError" &&
32
+ (candidate.code === "mcp_connect_failed" || candidate.code === "mcp_close_failed") &&
33
+ candidate.origin === "runtime" &&
34
+ typeof candidate.serverId === "string" &&
35
+ candidate.serverId.length > 0
36
+ );
37
+ } catch {
38
+ return false;
39
+ }
40
+ }
41
+
42
+ function reportIsolatedUnhandledRejection(
43
+ classification: WorkerUnhandledRejectionClassification,
44
+ ): void {
45
+ // Never render the rejection itself: provider and transport errors can carry
46
+ // request bodies or credentials. The durable activity path owns exact errors.
47
+ console.error("[worker] isolated detached unhandled rejection", { classification });
48
+ }
49
+
50
+ /**
51
+ * Keep process-global promise rejection handling observational. Turn activities
52
+ * have their own durable failure/lease boundary; killing the shared worker here
53
+ * converts one detached SDK/transport promise into unrelated lease losses across
54
+ * every session assigned to the pod. The Agents SDK also checks for a consumer
55
+ * listener before applying its own process.exit(1), so this boundary must never
56
+ * reintroduce that fail-fast behavior for an unknown rejection shape.
57
+ */
58
+ export function installWorkerUnhandledRejectionBoundary(
59
+ runtimeProcess: WorkerUnhandledRejectionProcess = process,
60
+ report: (
61
+ classification: WorkerUnhandledRejectionClassification,
62
+ ) => void = reportIsolatedUnhandledRejection,
63
+ ): () => void {
64
+ const onUnhandledRejection = (reason: unknown): void => {
65
+ report(
66
+ isDetachedRuntimeMcpLifecycleRejection(reason)
67
+ ? "detached_runtime_mcp_lifecycle"
68
+ : "detached_unknown",
69
+ );
70
+ };
71
+ runtimeProcess.on("unhandledRejection", onUnhandledRejection);
72
+ return () => runtimeProcess.off("unhandledRejection", onUnhandledRejection);
73
+ }
@@ -68,6 +68,21 @@ export const automationActivity = proxyActivities<Pick<typeof activities, "dispa
68
68
  },
69
69
  );
70
70
 
71
+ /** Dispatch has a bounded retry window, but an accepted automation event must
72
+ * not remain durably `dispatching` after that window closes. This compensating
73
+ * database write is idempotent and retries without a cap until the run reaches
74
+ * a terminal state (or is observed as already dispatched). */
75
+ export const automationFailureActivity = proxyActivities<
76
+ Pick<typeof activities, "settleAutomationRunFailure">
77
+ >({
78
+ startToCloseTimeout: "2 minutes",
79
+ retry: {
80
+ initialInterval: "1 second",
81
+ backoffCoefficient: 2,
82
+ maximumInterval: "30 seconds",
83
+ },
84
+ });
85
+
71
86
  /** Goal continuation evaluates a durable Postgres obligation at an idle
72
87
  * boundary. A transient failure gets a short retry window, then records an
73
88
  * explicit delayed outbox wake instead of relying on an unrelated mutation or
@@ -1,4 +1,4 @@
1
- import { automationActivity } from "./activities";
1
+ import { automationActivity, automationFailureActivity } from "./activities";
2
2
 
3
3
  export type AutomationRunWorkflowInput = {
4
4
  accountId: string;
@@ -6,6 +6,25 @@ export type AutomationRunWorkflowInput = {
6
6
  runId: string;
7
7
  };
8
8
 
9
+ export type AutomationRunWorkflowActivities = {
10
+ dispatchAutomationRun: (input: AutomationRunWorkflowInput) => Promise<unknown>;
11
+ settleAutomationRunFailure: (input: AutomationRunWorkflowInput) => Promise<void>;
12
+ };
13
+
14
+ export async function runAutomationRunWorkflow(
15
+ input: AutomationRunWorkflowInput,
16
+ activities: AutomationRunWorkflowActivities,
17
+ ): Promise<void> {
18
+ try {
19
+ await activities.dispatchAutomationRun(input);
20
+ } catch {
21
+ await activities.settleAutomationRunFailure(input);
22
+ }
23
+ }
24
+
9
25
  export async function automationRunWorkflow(input: AutomationRunWorkflowInput): Promise<void> {
10
- await automationActivity.dispatchAutomationRun(input);
26
+ await runAutomationRunWorkflow(input, {
27
+ dispatchAutomationRun: automationActivity.dispatchAutomationRun,
28
+ settleAutomationRunFailure: automationFailureActivity.settleAutomationRunFailure,
29
+ });
11
30
  }
@@ -1,14 +0,0 @@
1
- import type { ChannelASession } from "@opengeni/runtime/sandbox";
2
- import type { SandboxRuntimeState } from "./turn-context.js";
3
- type ImageReferenceSandboxState = Pick<SandboxRuntimeState, "resolvedSandbox" | "lazyOwnedSandbox" | "turnSandboxProvisioner">;
4
- /**
5
- * Join the turn's canonical lazy sandbox boundary before reading a
6
- * model-supplied /workspace image reference. Durable File and artifact
7
- * references never call this helper because their bytes come from object
8
- * storage instead.
9
- */
10
- export declare function resolveImageReferenceSandboxSession(sandboxState: ImageReferenceSandboxState, sdkOwnedSandboxSession: unknown): Promise<{
11
- session: ChannelASession;
12
- leaseEpoch: number;
13
- }>;
14
- export {};