@opengeni/worker-bundle 0.22.8 → 0.23.3-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/dist/activities/agent-turn/agent-build.d.ts +1 -4
  2. package/dist/activities/agent-turn/claim.d.ts +2 -4
  3. package/dist/activities/agent-turn/codex.d.ts +6 -5
  4. package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
  5. package/dist/activities/agent-turn/errors.d.ts +13 -0
  6. package/dist/activities/agent-turn/finalization.d.ts +1 -3
  7. package/dist/activities/agent-turn/quiescence.d.ts +39 -0
  8. package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
  9. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
  10. package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
  11. package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
  12. package/dist/activities/agent-turn/tool-policy.d.ts +3 -32
  13. package/dist/activities/agent-turn/turn-context.d.ts +1 -8
  14. package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
  15. package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
  16. package/dist/activities/codex-rotation.d.ts +2 -0
  17. package/dist/activities/session-state.d.ts +2 -1
  18. package/dist/{activities-control-VZIIV7RX.js → activities-control-52W5IDWM.js} +125 -13
  19. package/dist/activities-control-52W5IDWM.js.map +1 -0
  20. package/dist/{activities-turn-5RGQDBFV.js → activities-turn-A2G2KLSY.js} +534 -482
  21. package/dist/activities-turn-A2G2KLSY.js.map +1 -0
  22. package/dist/{chunk-NAVSAZ56.js → chunk-GFNXVZVQ.js} +160 -1
  23. package/dist/chunk-GFNXVZVQ.js.map +1 -0
  24. package/dist/{chunk-O4HDX5SC.js → chunk-IPVMCAW5.js} +144 -15
  25. package/dist/chunk-IPVMCAW5.js.map +1 -0
  26. package/dist/index.js +42 -2
  27. package/dist/index.js.map +1 -1
  28. package/dist/observability-metrics.d.ts +28 -0
  29. package/dist/personal-github-git-credentials.d.ts +1 -0
  30. package/dist/unhandled-rejection-boundary.d.ts +29 -0
  31. package/package.json +18 -18
  32. package/src/activities/agent-turn/agent-build.ts +15 -31
  33. package/src/activities/agent-turn/claim.ts +29 -68
  34. package/src/activities/agent-turn/codex.ts +19 -10
  35. package/src/activities/agent-turn/compaction-prep.ts +50 -1
  36. package/src/activities/agent-turn/errors.ts +78 -0
  37. package/src/activities/agent-turn/failure-settlement.ts +30 -19
  38. package/src/activities/agent-turn/finalization.ts +24 -21
  39. package/src/activities/agent-turn/history.ts +5 -1
  40. package/src/activities/agent-turn/quiescence.ts +86 -0
  41. package/src/activities/agent-turn/run-credentials.ts +12 -1
  42. package/src/activities/agent-turn/run.ts +14 -11
  43. package/src/activities/agent-turn/sandbox-establish.ts +46 -11
  44. package/src/activities/agent-turn/sandbox-route.ts +21 -25
  45. package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
  46. package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
  47. package/src/activities/agent-turn/stream-attempt.ts +98 -14
  48. package/src/activities/agent-turn/tool-environment.ts +10 -13
  49. package/src/activities/agent-turn/tool-policy.ts +3 -55
  50. package/src/activities/agent-turn/turn-context.ts +8 -14
  51. package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
  52. package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
  53. package/src/activities/codex-capacity.ts +13 -5
  54. package/src/activities/codex-rotation.ts +17 -0
  55. package/src/activities/goals.ts +10 -4
  56. package/src/activities/retained-screenshots.ts +104 -21
  57. package/src/activities/sandbox-lease.ts +7 -4
  58. package/src/activities/scheduled-tasks.ts +128 -4
  59. package/src/activities/session-state.ts +8 -1
  60. package/src/github-rest-mcp.ts +6 -3
  61. package/src/http.ts +3 -0
  62. package/src/index.ts +19 -0
  63. package/src/observability-metrics.ts +200 -0
  64. package/src/personal-github-git-credentials.ts +45 -4
  65. package/src/sandbox-resume.ts +1 -1
  66. package/src/sandbox-routing.ts +7 -4
  67. package/src/unhandled-rejection-boundary.ts +73 -0
  68. package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
  69. package/dist/activities/recording.d.ts +0 -90
  70. package/dist/activities-control-VZIIV7RX.js.map +0 -1
  71. package/dist/activities-turn-5RGQDBFV.js.map +0 -1
  72. package/dist/chunk-NAVSAZ56.js.map +0 -1
  73. package/dist/chunk-O4HDX5SC.js.map +0 -1
  74. package/src/activities/recording.ts +0 -279
@@ -1,6 +1,7 @@
1
1
  import { createHash } from "node:crypto";
2
2
  import { errorCodeToJSON } from "@opengeni/agent-proto";
3
3
  import { SandboxBackend, type SessionEventType } from "@opengeni/contracts";
4
+ import type { SessionEventAppendPhaseObservation } from "@opengeni/db";
4
5
  import type { EventLogger } from "@opengeni/events";
5
6
  import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
6
7
  import type { CompanyBrainContributionReceipt } from "./model-context-contributions";
@@ -18,6 +19,7 @@ import {
18
19
  } from "@opengeni/runtime";
19
20
 
20
21
  export type TurnOutcome = "completed" | "failed" | "cancelled" | "recovering";
22
+ export type WorkerDeathRecoveryOutcome = "recovering" | "exhausted";
21
23
  export type CreditMicrosKind = "usage" | "grant" | "topup" | "refund";
22
24
  export type SandboxLeaseLiveness = "cold" | "warming" | "warm" | "draining";
23
25
  export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
@@ -28,6 +30,10 @@ export type TurnTaskQueueStats = {
28
30
  tasksAddRate: number;
29
31
  tasksDispatchRate: number;
30
32
  };
33
+ export type SessionRecoveryBacklog = {
34
+ quiescence_missing: number;
35
+ projection_stale: number;
36
+ };
31
37
 
32
38
  export type TemporalTurnTaskQueueStats = {
33
39
  approximateBacklogCount?: unknown;
@@ -49,6 +55,12 @@ const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>(
49
55
  // before floating-point precision can degrade; a restart resets both the
50
56
  // process-local Prometheus registry and this guard together.
51
57
  const MAX_MODEL_CACHE_COUNTER_TOTAL = 1_000_000_000_000;
58
+ const TURN_OUTCOMES: readonly TurnOutcome[] = ["completed", "failed", "cancelled", "recovering"];
59
+ const WORKER_DEATH_RECOVERY_OUTCOMES: readonly WorkerDeathRecoveryOutcome[] = [
60
+ "recovering",
61
+ "exhausted",
62
+ ];
63
+ const WORKER_DEATH_TIMEOUT_TYPES = ["heartbeat", "schedule_to_start"] as const;
52
64
 
53
65
  export function observabilityEventLogger(observability: Observability): EventLogger {
54
66
  return {
@@ -163,6 +175,19 @@ export function runtimeMetricsHooksForObservability(
163
175
  value: durationSeconds,
164
176
  });
165
177
  },
178
+ onMcpLifecycle: ({ phase, policy, outcome, durationSeconds }) => {
179
+ observability.incrementCounter({
180
+ name: "opengeni_mcp_lifecycle_operations_total",
181
+ help: "Total physical MCP lifecycle operations by bounded phase, policy, and outcome.",
182
+ labels: { phase, policy, outcome },
183
+ });
184
+ observability.observeHistogram({
185
+ name: "opengeni_mcp_lifecycle_operation_duration_seconds",
186
+ help: "MCP lifecycle operation duration in seconds by bounded phase, policy, and outcome.",
187
+ labels: { phase, policy, outcome },
188
+ value: durationSeconds,
189
+ });
190
+ },
166
191
  onSandboxOp: ({ backend, op, outcome, code, healed, durationSeconds, replyBytes }) => {
167
192
  observability.incrementCounter({
168
193
  name: "opengeni_machine_op_total",
@@ -300,6 +325,53 @@ export function turnLifecycleMetricsFor(observability: Observability): TurnLifec
300
325
  return tracker;
301
326
  }
302
327
 
328
+ export function initializeWorkerOutcomeMetrics(observability: Observability): void {
329
+ for (const outcome of TURN_OUTCOMES) {
330
+ observability.incrementCounter({
331
+ name: "opengeni_turns_total",
332
+ help: "Total agent turns by terminal outcome.",
333
+ labels: { outcome },
334
+ amount: 0,
335
+ });
336
+ }
337
+ for (const outcome of WORKER_DEATH_RECOVERY_OUTCOMES) {
338
+ for (const timeoutType of WORKER_DEATH_TIMEOUT_TYPES) {
339
+ observability.incrementCounter({
340
+ name: "opengeni_turn_worker_death_recoveries_total",
341
+ help: "Total durable same-turn worker-death recovery outcomes.",
342
+ labels: { outcome, timeout_type: timeoutType },
343
+ amount: 0,
344
+ });
345
+ }
346
+ }
347
+ }
348
+
349
+ export function recordWorkerDeathRecoveryMetrics(
350
+ observability: Observability,
351
+ input: {
352
+ outcome: WorkerDeathRecoveryOutcome;
353
+ timeoutType: "HEARTBEAT" | "SCHEDULE_TO_START";
354
+ },
355
+ ): void {
356
+ const turnOutcome: TurnOutcome = input.outcome === "exhausted" ? "failed" : "recovering";
357
+ observability.incrementCounter({
358
+ name: "opengeni_turn_worker_death_recoveries_total",
359
+ help: "Total durable same-turn worker-death recovery outcomes.",
360
+ labels: {
361
+ outcome: input.outcome,
362
+ timeout_type: input.timeoutType.toLowerCase(),
363
+ },
364
+ });
365
+ // The turn process that owned this attempt is gone, so its process-local
366
+ // lifecycle tracker cannot publish the outcome. Record it from the fenced
367
+ // control activity only after the durable recovery transaction wins.
368
+ observability.incrementCounter({
369
+ name: "opengeni_turns_total",
370
+ help: "Total agent turns by terminal outcome.",
371
+ labels: { outcome: turnOutcome },
372
+ });
373
+ }
374
+
303
375
  export class TurnLifecycleMetrics {
304
376
  private readonly turns = new Map<string, { startedAt: number; lastProgressAt: number }>();
305
377
  private timer: ReturnType<typeof setInterval> | null = null;
@@ -676,6 +748,88 @@ export function startTurnCapacityMonitor(input: {
676
748
  };
677
749
  }
678
750
 
751
+ export function recordSessionRecoveryBacklogGauges(
752
+ observability: Observability,
753
+ counts: SessionRecoveryBacklog,
754
+ ): void {
755
+ for (const state of ["quiescence_missing", "projection_stale"] as const) {
756
+ observability.setGauge({
757
+ name: "opengeni_session_recovery_backlog",
758
+ help: "Current durable recovering-session obligations with no active attempt, by bounded reconciliation state.",
759
+ labels: { state },
760
+ value: nonnegativeFinite(counts[state]),
761
+ });
762
+ }
763
+ }
764
+
765
+ export function startSessionRecoveryMonitor(input: {
766
+ observability: Observability;
767
+ read: () => Promise<SessionRecoveryBacklog>;
768
+ intervalMs?: number;
769
+ now?: () => number;
770
+ }): { close: () => Promise<void> } {
771
+ const intervalMs = input.intervalMs ?? 60_000;
772
+ const now = input.now ?? Date.now;
773
+ const startedAt = now();
774
+ let lastSuccessAt: number | null = null;
775
+ let lastReadSucceeded = false;
776
+ let stopped = false;
777
+ let running: Promise<void> | null = null;
778
+ const recordStatus = () => {
779
+ const observedAt = now();
780
+ const successAgeMs = observedAt - (lastSuccessAt ?? startedAt);
781
+ const set = (name: string, help: string, value: number) =>
782
+ input.observability.setGauge({ name, help, value });
783
+ set(
784
+ "opengeni_session_recovery_monitor_last_read_success",
785
+ "Whether the latest durable session-recovery aggregate read completed successfully.",
786
+ lastReadSucceeded ? 1 : 0,
787
+ );
788
+ set(
789
+ "opengeni_session_recovery_monitor_last_success_timestamp_seconds",
790
+ "Unix timestamp of the latest successful durable session-recovery aggregate read, or zero before one succeeds.",
791
+ lastSuccessAt === null ? 0 : lastSuccessAt / 1_000,
792
+ );
793
+ set(
794
+ "opengeni_session_recovery_monitor_fresh",
795
+ "Whether durable session-recovery backlog gauges have a successful read within three monitor intervals.",
796
+ lastReadSucceeded && lastSuccessAt !== null && successAgeMs <= intervalMs * 3 ? 1 : 0,
797
+ );
798
+ };
799
+ const refresh = () => {
800
+ recordStatus();
801
+ if (stopped || running) return;
802
+ running = input
803
+ .read()
804
+ .then((counts) => {
805
+ recordSessionRecoveryBacklogGauges(input.observability, counts);
806
+ lastSuccessAt = now();
807
+ lastReadSucceeded = true;
808
+ recordStatus();
809
+ })
810
+ .catch((error) => {
811
+ lastReadSucceeded = false;
812
+ recordStatus();
813
+ input.observability.warn("session recovery monitor: durable aggregate read failed", {
814
+ error: error instanceof Error ? error.message : String(error),
815
+ });
816
+ })
817
+ .finally(() => {
818
+ running = null;
819
+ });
820
+ };
821
+ refresh();
822
+ const timer = setInterval(refresh, intervalMs);
823
+ timer.unref?.();
824
+ return {
825
+ close: async () => {
826
+ stopped = true;
827
+ clearInterval(timer);
828
+ await running;
829
+ },
830
+ };
831
+ }
832
+
679
833
  export function recordSandboxLeaseGauges(
680
834
  observability: Observability,
681
835
  counts: Partial<Record<SandboxLeaseLiveness, number>>,
@@ -1349,6 +1503,22 @@ export function recordSandboxLogicalProvision(
1349
1503
  }
1350
1504
  }
1351
1505
 
1506
+ export function recordSandboxSharedPreparation(
1507
+ observability: Observability,
1508
+ measurement: {
1509
+ path: "owner" | "joined" | "reused";
1510
+ outcome: "completed" | "failed";
1511
+ durationSeconds: number;
1512
+ },
1513
+ ): void {
1514
+ observability.observeHistogram({
1515
+ name: "opengeni_sandbox_shared_preparation_duration_seconds",
1516
+ help: "Duration of exact-lease immutable sandbox preparation by durable coordination path and outcome.",
1517
+ labels: { path: measurement.path, outcome: measurement.outcome },
1518
+ value: Math.max(0, measurement.durationSeconds),
1519
+ });
1520
+ }
1521
+
1352
1522
  export function recordTurnWorkerPreparationTotal(
1353
1523
  observability: Observability,
1354
1524
  input: {
@@ -1535,6 +1705,36 @@ export function recordSessionEventAppendLatency(
1535
1705
  });
1536
1706
  }
1537
1707
 
1708
+ export function sessionEventBatchSizeClass(eventCount: number): string {
1709
+ if (!Number.isFinite(eventCount) || eventCount <= 0) return "unknown";
1710
+ if (eventCount === 1) return "1";
1711
+ if (eventCount <= 5) return "2-5";
1712
+ if (eventCount <= 10) return "6-10";
1713
+ if (eventCount <= 25) return "11-25";
1714
+ if (eventCount <= 50) return "26-50";
1715
+ return "51+";
1716
+ }
1717
+
1718
+ /** Bounded database-phase attribution for exact-attempt event appends. */
1719
+ export function recordSessionEventAppendPhase(
1720
+ observability: Observability,
1721
+ observation: SessionEventAppendPhaseObservation,
1722
+ ): void {
1723
+ observability.observeHistogram({
1724
+ name: "opengeni_session_event_append_phase_seconds",
1725
+ help: "Duration of one bounded database phase in an exact-attempt session event append.",
1726
+ buckets: STREAM_IO_BUCKETS,
1727
+ labels: {
1728
+ path: "turn_attempt",
1729
+ event_class: observation.eventClass,
1730
+ batch_size_class: sessionEventBatchSizeClass(observation.eventCount),
1731
+ phase: observation.phase,
1732
+ outcome: observation.outcome,
1733
+ },
1734
+ value: Math.max(0, observation.durationSeconds),
1735
+ });
1736
+ }
1737
+
1538
1738
  /** Latency of the best-effort NATS live fan-out — the delivery path. A p99 climb
1539
1739
  * here (with append healthy) is delivery, not the write path. */
1540
1740
  export function recordSessionEventPublishLatency(
@@ -24,6 +24,8 @@ import {
24
24
  type PersonalGitHubGitBrokerRepositoryClaim,
25
25
  } from "@opengeni/github";
26
26
  import { randomBytes, randomUUID } from "node:crypto";
27
+ import { execFileSync } from "node:child_process";
28
+ import { isIP } from "node:net";
27
29
  import { isDeepStrictEqual } from "node:util";
28
30
 
29
31
  const PERSONAL_GITHUB_SERVER_ID = "github:personal";
@@ -39,7 +41,7 @@ export function buildPersonalGitHubGitCredentials(
39
41
  ): NonNullable<ConnectionCredentialsPort["gitCredentials"]> | null {
40
42
  if (!settings.githubPersonalOauthEnabled) return null;
41
43
  const secret = settings.integrationsStateSecret?.trim();
42
- const publicOrigin = personalGitBrokerOrigin(settings.publicBaseUrl);
44
+ const publicOrigin = personalGitBrokerOrigin(settings);
43
45
  if (!secret || !publicOrigin) {
44
46
  throw new Error("personal GitHub Git broker configuration is unavailable");
45
47
  }
@@ -255,19 +257,58 @@ function personalGitHubCanWrite(permissions: {
255
257
  return permissions.push || permissions.maintain || permissions.admin;
256
258
  }
257
259
 
258
- function personalGitBrokerOrigin(value: string | undefined): string | null {
260
+ let cachedDockerBridgeGateway: string | null | undefined;
261
+
262
+ function dockerBridgeGateway(): string | null {
263
+ if (cachedDockerBridgeGateway !== undefined) return cachedDockerBridgeGateway;
264
+ try {
265
+ const value = execFileSync(
266
+ "docker",
267
+ ["network", "inspect", "bridge", "--format", "{{(index .IPAM.Config 0).Gateway}}"],
268
+ { encoding: "utf8", timeout: 2_000, maxBuffer: 16 * 1024 },
269
+ ).trim();
270
+ cachedDockerBridgeGateway = isIP(value) ? value : null;
271
+ } catch {
272
+ cachedDockerBridgeGateway = null;
273
+ }
274
+ return cachedDockerBridgeGateway;
275
+ }
276
+
277
+ export function personalGitBrokerOrigin(settings: Settings): string | null {
278
+ const value =
279
+ settings.opengeniMcpUrl ??
280
+ (settings.sandboxBackend === "docker"
281
+ ? `http://host.docker.internal:${settings.apiPort}`
282
+ : settings.sandboxBackend === "local" || settings.sandboxBackend === "none"
283
+ ? `http://127.0.0.1:${settings.apiPort}`
284
+ : settings.publicBaseUrl);
259
285
  if (!value) return null;
260
286
  const url = new URL(value);
261
287
  if (
262
- url.protocol !== "https:" ||
288
+ settings.sandboxBackend === "docker" &&
289
+ (url.hostname === "127.0.0.1" || url.hostname === "localhost")
290
+ ) {
291
+ url.hostname = dockerBridgeGateway() ?? "host.docker.internal";
292
+ }
293
+ const dockerPrivateOrigin =
294
+ settings.sandboxBackend === "docker" &&
295
+ (url.hostname === "host.docker.internal" ||
296
+ url.hostname === cachedDockerBridgeGateway ||
297
+ /^[a-z][a-z0-9-]{0,62}$/iu.test(url.hostname));
298
+ const privateExecutionOrigin =
299
+ (url.protocol === "http:" &&
300
+ (url.hostname === "127.0.0.1" || url.hostname === "localhost" || dockerPrivateOrigin)) ||
301
+ ["local", "test"].includes(settings.environment);
302
+ if (
303
+ (url.protocol !== "https:" && !privateExecutionOrigin) ||
263
304
  url.username ||
264
305
  url.password ||
265
- url.pathname !== "/" ||
266
306
  url.search ||
267
307
  url.hash
268
308
  ) {
269
309
  return null;
270
310
  }
311
+ url.pathname = "/";
271
312
  return url.origin;
272
313
  }
273
314
 
@@ -11,7 +11,7 @@
11
11
  // re-enters admission instead of creating a rival.
12
12
  // 3. (spawner) commitWarmingToWarm (the lease_epoch++ fence + folds the resume
13
13
  // envelope onto the lease). Optional desktop work is deliberately absent:
14
- // the viewer and the first actual computer-use action initialize :0 lazily.
14
+ // viewers and managed ComputerSession operations initialize it lazily.
15
15
  // 4. the caller injects {client, session, sessionState} NON-OWNED into the run
16
16
  // (the SDK never reaps it — the keystone), runs, then in `finally` calls the
17
17
  // returned `release()` and drops the in-memory handle. NEVER provider-delete
@@ -672,7 +672,7 @@ function settleRetainedProcessForTurn(
672
672
  ) {
673
673
  throw new Error("Retained-process settlement lost its exact durable backend identity");
674
674
  }
675
- await settleRetainedProcess(services.db, {
675
+ const settlement = await settleRetainedProcess(services.db, {
676
676
  accountId: fence.accountId,
677
677
  workspaceId: ids.workspaceId,
678
678
  sessionId: ids.sessionId,
@@ -683,14 +683,17 @@ function settleRetainedProcessForTurn(
683
683
  reason: proof.reason,
684
684
  idleGraceMs: services.settings.sandboxIdleGraceMs,
685
685
  });
686
+ if (settlement.process.state === "active") {
687
+ throw new Error("Retained-process settlement returned an active durable process");
688
+ }
686
689
  await settleSessionBackgroundCommandForRetainedProcess(services.db, {
687
690
  accountId: fence.accountId,
688
691
  workspaceId: ids.workspaceId,
689
692
  sessionId: ids.sessionId,
690
693
  retainedProcessId: process.id,
691
- outcome: proof.outcome,
692
- exitCode: proof.exitCode,
693
- reason: proof.reason,
694
+ outcome: settlement.process.state,
695
+ exitCode: settlement.process.exitCode,
696
+ reason: settlement.process.settlementReason ?? proof.reason,
694
697
  });
695
698
  };
696
699
  }
@@ -0,0 +1,73 @@
1
+ export type WorkerUnhandledRejectionProcess = {
2
+ on: (event: "unhandledRejection", listener: (reason: unknown) => void) => void;
3
+ off: (event: "unhandledRejection", listener: (reason: unknown) => void) => void;
4
+ };
5
+
6
+ export type WorkerUnhandledRejectionClassification =
7
+ | "detached_runtime_mcp_lifecycle"
8
+ | "detached_unknown";
9
+
10
+ type RuntimeMcpLifecycleError = {
11
+ name: "McpLifecycleError";
12
+ code: "mcp_connect_failed" | "mcp_close_failed";
13
+ origin: "runtime";
14
+ serverId: string;
15
+ };
16
+
17
+ /**
18
+ * The Agents SDK owns MCP connect promises and also installs a process-global
19
+ * unhandled-rejection listener. A best-effort lifecycle rejection can surface
20
+ * twice under concurrent teardown: once through the awaited MCP result and once
21
+ * through the SDK's detached worker promise. The first path already records and
22
+ * degrades the server; the second must not terminate the whole turn-worker pod.
23
+ */
24
+ export function isDetachedRuntimeMcpLifecycleRejection(
25
+ reason: unknown,
26
+ ): reason is RuntimeMcpLifecycleError {
27
+ try {
28
+ if (!reason || typeof reason !== "object") return false;
29
+ const candidate = reason as Partial<RuntimeMcpLifecycleError>;
30
+ return (
31
+ candidate.name === "McpLifecycleError" &&
32
+ (candidate.code === "mcp_connect_failed" || candidate.code === "mcp_close_failed") &&
33
+ candidate.origin === "runtime" &&
34
+ typeof candidate.serverId === "string" &&
35
+ candidate.serverId.length > 0
36
+ );
37
+ } catch {
38
+ return false;
39
+ }
40
+ }
41
+
42
+ function reportIsolatedUnhandledRejection(
43
+ classification: WorkerUnhandledRejectionClassification,
44
+ ): void {
45
+ // Never render the rejection itself: provider and transport errors can carry
46
+ // request bodies or credentials. The durable activity path owns exact errors.
47
+ console.error("[worker] isolated detached unhandled rejection", { classification });
48
+ }
49
+
50
+ /**
51
+ * Keep process-global promise rejection handling observational. Turn activities
52
+ * have their own durable failure/lease boundary; killing the shared worker here
53
+ * converts one detached SDK/transport promise into unrelated lease losses across
54
+ * every session assigned to the pod. The Agents SDK also checks for a consumer
55
+ * listener before applying its own process.exit(1), so this boundary must never
56
+ * reintroduce that fail-fast behavior for an unknown rejection shape.
57
+ */
58
+ export function installWorkerUnhandledRejectionBoundary(
59
+ runtimeProcess: WorkerUnhandledRejectionProcess = process,
60
+ report: (
61
+ classification: WorkerUnhandledRejectionClassification,
62
+ ) => void = reportIsolatedUnhandledRejection,
63
+ ): () => void {
64
+ const onUnhandledRejection = (reason: unknown): void => {
65
+ report(
66
+ isDetachedRuntimeMcpLifecycleRejection(reason)
67
+ ? "detached_runtime_mcp_lifecycle"
68
+ : "detached_unknown",
69
+ );
70
+ };
71
+ runtimeProcess.on("unhandledRejection", onUnhandledRejection);
72
+ return () => runtimeProcess.off("unhandledRejection", onUnhandledRejection);
73
+ }
@@ -1,14 +0,0 @@
1
- import type { ChannelASession } from "@opengeni/runtime/sandbox";
2
- import type { SandboxRuntimeState } from "./turn-context.js";
3
- type ImageReferenceSandboxState = Pick<SandboxRuntimeState, "resolvedSandbox" | "lazyOwnedSandbox" | "turnSandboxProvisioner">;
4
- /**
5
- * Join the turn's canonical lazy sandbox boundary before reading a
6
- * model-supplied /workspace image reference. Durable File and artifact
7
- * references never call this helper because their bytes come from object
8
- * storage instead.
9
- */
10
- export declare function resolveImageReferenceSandboxSession(sandboxState: ImageReferenceSandboxState, sdkOwnedSandboxSession: unknown): Promise<{
11
- session: ChannelASession;
12
- leaseEpoch: number;
13
- }>;
14
- export {};
@@ -1,90 +0,0 @@
1
- import type { Settings } from "@opengeni/config";
2
- import type { Database } from "@opengeni/db";
3
- import type { RecordingFailedReason, RecordingStartedPayload } from "@opengeni/contracts";
4
- import { type RecordingProcess } from "@opengeni/runtime";
5
- import type { ObjectStorage } from "@opengeni/storage";
6
- import { DOWNLOAD_URL_TTL_SECONDS } from "@opengeni/storage";
7
- export type RecordingMode = "manual" | "on-turn" | "on-verify";
8
- export type ActiveRecording = {
9
- recordingId: string;
10
- turnId: string | null;
11
- mode: RecordingMode;
12
- proc: RecordingProcess;
13
- dimensions: [number, number];
14
- framerate: number;
15
- };
16
- export declare const RECORDING_SETTLEMENT_PREP_TIMEOUT_MS = 100000;
17
- export declare function withRecordingPreparationTimeout(preparation: Promise<PreparedRecordingSettlement>, timeoutResult: PreparedRecordingSettlement, timeoutMs?: number): Promise<PreparedRecordingSettlement>;
18
- /**
19
- * Insert the recording row, launch ffmpeg on the box, and return the live handle.
20
- * Emits the `recording.started` payload (the caller publishes it on the spine).
21
- */
22
- export declare function beginRecording(args: {
23
- settings: Settings;
24
- db: Database;
25
- accountId: string;
26
- workspaceId: string;
27
- sessionId: string;
28
- turnId: string | null;
29
- recordingId: string;
30
- mode: RecordingMode;
31
- session: unknown;
32
- runAs?: string | undefined;
33
- reason?: string | null;
34
- }): Promise<{
35
- active: ActiveRecording;
36
- started: RecordingStartedPayload;
37
- }>;
38
- export type PreparedRecordingSettlement = {
39
- mutation: {
40
- action: "available";
41
- recordingId: string;
42
- storageKey: string;
43
- sizeBytes: number;
44
- durationSeconds: number;
45
- };
46
- deleteArtifactsAfterCommit: true;
47
- } | {
48
- mutation: {
49
- action: "failed";
50
- recordingId: string;
51
- reason: RecordingFailedReason;
52
- detail: string | null;
53
- };
54
- deleteArtifactsAfterCommit: false;
55
- } | {
56
- mutation: {
57
- action: "discard";
58
- recordingId: string;
59
- };
60
- deleteArtifactsAfterCommit: true;
61
- };
62
- /**
63
- * Prepare an on-turn recording for the attempt-closing settlement. This performs
64
- * only bounded box/storage work; it does not mutate canonical database state and
65
- * never deletes the box artifact. The caller folds the returned mutation into the
66
- * exact attempt-fenced turn settlement, then deletes artifacts only after commit.
67
- * NEVER throws and never prevents turn truth from settling.
68
- */
69
- export declare function prepareRecordingForSettlement(args: {
70
- settings: Settings;
71
- objectStorage: ObjectStorage | null;
72
- workspaceId: string;
73
- sessionId: string;
74
- active: ActiveRecording;
75
- session: unknown;
76
- didComputerUse: boolean;
77
- }): Promise<PreparedRecordingSettlement>;
78
- /**
79
- * Compensate a start whose fenced `recording.started` event was rejected. Since
80
- * no canonical event was admitted, this exact recording id must leave no row,
81
- * process, or box artifact behind.
82
- */
83
- export declare function discardUnpublishedRecording(args: {
84
- db: Database;
85
- accountId: string;
86
- workspaceId: string;
87
- active: ActiveRecording;
88
- session: unknown;
89
- }): Promise<void>;
90
- export { DOWNLOAD_URL_TTL_SECONDS };