@opengeni/worker-bundle 0.12.7 → 0.13.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/dist/activities/agent-turn.d.ts +778 -0
  2. package/dist/activities/capabilities.d.ts +18 -0
  3. package/dist/activities/codex-auth.d.ts +1 -0
  4. package/dist/activities/codex-capacity.d.ts +31 -0
  5. package/dist/activities/codex-fleet-shadow.d.ts +74 -0
  6. package/dist/activities/codex-rotation.d.ts +261 -0
  7. package/dist/activities/common.d.ts +17 -0
  8. package/dist/activities/context-compaction.d.ts +81 -0
  9. package/dist/activities/documents.d.ts +39 -0
  10. package/dist/activities/environment.d.ts +97 -0
  11. package/dist/activities/expiring-material-renewal.d.ts +38 -0
  12. package/dist/activities/file-upload-reaper.d.ts +25 -0
  13. package/dist/activities/git-credential-renewal.d.ts +41 -0
  14. package/dist/activities/goals.d.ts +16 -0
  15. package/dist/activities/mcp-credentials.d.ts +14 -0
  16. package/dist/activities/packs.d.ts +77 -0
  17. package/dist/activities/parent-wake.d.ts +54 -0
  18. package/dist/activities/recording.d.ts +90 -0
  19. package/dist/activities/redaction.d.ts +1 -0
  20. package/dist/activities/rig-verification.d.ts +131 -0
  21. package/dist/activities/run-credential-renewal.d.ts +28 -0
  22. package/dist/activities/run-credentials.d.ts +36 -0
  23. package/dist/activities/run-input.d.ts +106 -0
  24. package/dist/activities/sandbox-lease.d.ts +128 -0
  25. package/dist/activities/scheduled-tasks.d.ts +4 -0
  26. package/dist/activities/session-state.d.ts +39 -0
  27. package/dist/activities/streaming.d.ts +24 -0
  28. package/dist/activities/toolspace-token-renewal.d.ts +23 -0
  29. package/dist/activities/turn-attempt-fenced.d.ts +11 -0
  30. package/dist/activities/types.d.ts +240 -0
  31. package/dist/activities/workflow-wake.d.ts +15 -0
  32. package/dist/activities/workspace-capture.d.ts +105 -0
  33. package/dist/activities.d.ts +312 -0
  34. package/dist/concurrency.d.ts +19 -0
  35. package/dist/host-export-pump.d.ts +66 -0
  36. package/dist/host-export.d.ts +6 -67
  37. package/dist/http.d.ts +32 -0
  38. package/dist/index.d.ts +27 -388
  39. package/dist/index.js +2730 -1211
  40. package/dist/index.js.map +1 -1
  41. package/dist/observability-metrics.d.ts +200 -0
  42. package/dist/op-journal.d.ts +10 -0
  43. package/dist/sandbox-resume.d.ts +152 -0
  44. package/dist/sandbox-routing.d.ts +159 -0
  45. package/dist/worker-service-lifecycle.d.ts +20 -0
  46. package/dist/workflow-bundle.js +88 -20
  47. package/dist/workflows/activities.d.ts +27 -0
  48. package/dist/workflows/document-index.d.ts +41 -0
  49. package/dist/workflows/file-upload-reaper.d.ts +2 -0
  50. package/dist/workflows/rig-verification.d.ts +3 -0
  51. package/dist/workflows/sandbox-reaper.d.ts +1 -0
  52. package/dist/workflows/scheduled-tasks.d.ts +17 -0
  53. package/dist/workflows/session.d.ts +58 -0
  54. package/dist/workflows/workflow-wake.d.ts +3 -0
  55. package/dist/workflows.d.ts +7 -0
  56. package/package.json +13 -13
  57. package/src/activities/agent-turn.ts +874 -134
  58. package/src/activities/context-compaction.ts +265 -76
  59. package/src/activities/environment.ts +1 -0
  60. package/src/activities/goals.ts +14 -0
  61. package/src/activities/rig-verification.ts +2 -0
  62. package/src/activities/run-input.ts +18 -4
  63. package/src/activities/sandbox-lease.ts +787 -133
  64. package/src/activities/session-state.ts +56 -0
  65. package/src/activities/types.ts +21 -0
  66. package/src/activities/workspace-capture.ts +95 -4
  67. package/src/activities.ts +3 -0
  68. package/src/observability-metrics.ts +94 -0
  69. package/src/sandbox-resume.ts +383 -90
  70. package/src/sandbox-routing.ts +32 -10
  71. package/src/workflows/activities.ts +1 -0
  72. package/src/workflows/session.ts +70 -0
@@ -14,7 +14,7 @@
14
14
  // target lookup), and the selfhosted ControlRpc built over the events bus's NATS
15
15
  // request/reply connection.
16
16
 
17
- import type { Settings } from "@opengeni/config";
17
+ import { sandboxArchiveCaptureTimeoutMs, type Settings } from "@opengeni/config";
18
18
  import {
19
19
  advanceWorkspaceGenerationForRetainedProcess,
20
20
  advanceWorkspaceGeneration,
@@ -43,6 +43,7 @@ import {
43
43
  NatsOpStreamTransport,
44
44
  RoutingBackendRecoveryRequiredError,
45
45
  RoutingSandboxSession,
46
+ resolveModalCheckpointProviderBindingForSession,
46
47
  verifySandboxExecReadiness,
47
48
  type ControlRpc,
48
49
  type EstablishedSandboxSession,
@@ -60,6 +61,13 @@ import {
60
61
  } from "@opengeni/runtime";
61
62
  import { sandboxLeaseHolderIdForAttempt } from "./sandbox-resume";
62
63
 
64
+ type PersistableMutationAdmission = {
65
+ admission: SandboxWorkspaceMutationAdmission;
66
+ providerBinding: Awaited<
67
+ ReturnType<typeof resolveModalCheckpointProviderBindingForSession>
68
+ > | null;
69
+ };
70
+
63
71
  export type RoutingWiringServices = {
64
72
  db: Database;
65
73
  settings: Settings;
@@ -390,12 +398,12 @@ async function resolveCurrentHomeBackend(
390
398
  function beforePersistableHomeMutation(
391
399
  services: RoutingWiringServices,
392
400
  ids: RoutingWiringIds,
393
- home: { accountId: string; sandboxGroupId: string } | undefined,
401
+ home: { accountId: string; sandboxGroupId: string; backend: string } | undefined,
394
402
  ):
395
403
  | ((input: {
396
404
  op: string;
397
405
  backend: ResolvedActiveBackend;
398
- }) => Promise<SandboxWorkspaceMutationAdmission | null>)
406
+ }) => Promise<PersistableMutationAdmission | null>)
399
407
  | undefined {
400
408
  const fence = ids.workspaceMutationFence;
401
409
  if (!home || !fence) return undefined;
@@ -410,7 +418,11 @@ function beforePersistableHomeMutation(
410
418
  ) {
411
419
  return null;
412
420
  }
413
- return await advanceWorkspaceGeneration(services.db, {
421
+ const providerBinding =
422
+ home.backend === "modal"
423
+ ? await resolveModalCheckpointProviderBindingForSession(services.settings, backend.session)
424
+ : null;
425
+ const admission = await advanceWorkspaceGeneration(services.db, {
414
426
  accountId: fence.accountId,
415
427
  workspaceId: ids.workspaceId,
416
428
  sessionId: ids.sessionId,
@@ -422,14 +434,16 @@ function beforePersistableHomeMutation(
422
434
  expectedEpoch: backend.leaseEpoch,
423
435
  expectedInstanceId: backend.providerInstanceId,
424
436
  operation: op,
437
+ captureWaitMs: sandboxArchiveCaptureTimeoutMs(services.settings),
425
438
  });
439
+ return { admission, providerBinding };
426
440
  };
427
441
  }
428
442
 
429
443
  function afterPersistableHomeMutation(
430
444
  services: RoutingWiringServices,
431
445
  ids: RoutingWiringIds,
432
- home: { accountId: string; sandboxGroupId: string } | undefined,
446
+ home: { accountId: string; sandboxGroupId: string; backend: string } | undefined,
433
447
  ):
434
448
  | ((input: {
435
449
  op: string;
@@ -448,14 +462,20 @@ function afterPersistableHomeMutation(
448
462
  backend.leaseEpoch === undefined ||
449
463
  backend.providerInstanceId === undefined ||
450
464
  !admission ||
451
- typeof admission !== "object" ||
452
- typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).id !== "string" ||
453
- typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).workspaceGeneration !==
454
- "number"
465
+ typeof admission !== "object"
455
466
  ) {
456
467
  return;
457
468
  }
458
- const exactAdmission = admission as SandboxWorkspaceMutationAdmission;
469
+ const boundAdmission = admission as Partial<PersistableMutationAdmission>;
470
+ const exactAdmission = boundAdmission.admission;
471
+ if (
472
+ !exactAdmission ||
473
+ typeof exactAdmission.id !== "string" ||
474
+ typeof exactAdmission.workspaceGeneration !== "number" ||
475
+ !("providerBinding" in boundAdmission)
476
+ ) {
477
+ throw new Error("Persistable home mutation settlement lacked its bound admission");
478
+ }
459
479
  if (outcome === "resolved" && retainedProcess) {
460
480
  try {
461
481
  await retainWorkspaceMutationProcess(services.db, {
@@ -467,6 +487,7 @@ function afterPersistableHomeMutation(
467
487
  admissionId: exactAdmission.id,
468
488
  admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
469
489
  operation: op,
490
+ providerBinding: boundAdmission.providerBinding ?? null,
470
491
  owner: {
471
492
  kind: "turn",
472
493
  turnId: fence.turnId,
@@ -548,6 +569,7 @@ function beforeRetainedProcessMutation(
548
569
  sessionId: ids.sessionId,
549
570
  processId: process.id,
550
571
  operation: op,
572
+ captureWaitMs: sandboxArchiveCaptureTimeoutMs(services.settings),
551
573
  });
552
574
  }
553
575
 
@@ -13,6 +13,7 @@ type WorkflowControlActivities = Pick<
13
13
  | "reconcileSessionAttemptQuiescence"
14
14
  | "reconcileCodexCapacityWait"
15
15
  | "recoverDispatch"
16
+ | "recoverEscapedMcpTimeout"
16
17
  | "settleSessionInterruptions"
17
18
  >;
18
19
 
@@ -13,6 +13,11 @@ import {
13
13
  workflowInfo,
14
14
  } from "@temporalio/workflow";
15
15
  import type * as activities from "../activities";
16
+ import {
17
+ ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
18
+ ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
19
+ type EscapedMcpTimeoutRecoveryDetail,
20
+ } from "../activities/types";
16
21
  import {
17
22
  activity,
18
23
  goalActivity,
@@ -36,6 +41,7 @@ import {
36
41
  const TURNS_PER_RUN_BACKSTOP = 2_000;
37
42
  const CODEX_CAPACITY_CHECKS_PER_RUN_BACKSTOP = 512;
38
43
  const HUMAN_INPUT_EXPIRY_STALE_RETRY_MS = 1_000;
44
+ const ESCAPED_MCP_TIMEOUT_RECOVERY_DELAY_MS = 60_000;
39
45
 
40
46
  /**
41
47
  * The minimum hold for a rotation all-capped idle (`idleUntilReset`). A MANDATORY
@@ -119,6 +125,46 @@ function workerDeathFailure(
119
125
  return { timeoutType: cause.timeoutType };
120
126
  }
121
127
 
128
+ /**
129
+ * Parse only the explicit activity wire contract emitted when a recovered
130
+ * turn's MCP setup timeout could not finish its DB checkpoint. Numeric -32001
131
+ * and generic MCP messages remain deliberately insufficient: the activity
132
+ * writes this non-retryable ApplicationFailure only before any model request,
133
+ * with immutable turn identity in payload-converted details.
134
+ */
135
+ export function escapedMcpTimeoutRecoveryDetail(
136
+ error: unknown,
137
+ ): EscapedMcpTimeoutRecoveryDetail | null {
138
+ if (!(error instanceof ActivityFailure) || error.activityType !== "runAgentTurn") {
139
+ return null;
140
+ }
141
+ const cause = error.cause;
142
+ if (
143
+ !(cause instanceof ApplicationFailure) ||
144
+ cause.type !== ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE ||
145
+ cause.message !== ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE
146
+ ) {
147
+ return null;
148
+ }
149
+ const detail = cause.details?.[0] as Partial<EscapedMcpTimeoutRecoveryDetail> | undefined;
150
+ if (
151
+ !detail ||
152
+ typeof detail.turnId !== "string" ||
153
+ detail.turnId.length === 0 ||
154
+ typeof detail.triggerEventId !== "string" ||
155
+ detail.triggerEventId.length === 0 ||
156
+ !Number.isSafeInteger(detail.executionGeneration) ||
157
+ (detail.executionGeneration ?? 0) <= 1
158
+ ) {
159
+ return null;
160
+ }
161
+ return {
162
+ turnId: detail.turnId,
163
+ triggerEventId: detail.triggerEventId,
164
+ executionGeneration: detail.executionGeneration!,
165
+ };
166
+ }
167
+
122
168
  /**
123
169
  * Classify only the exact turn-fence cancellation protocol used to arbitrate a
124
170
  * database control commit that reaches the activity just before its Temporal
@@ -631,6 +677,30 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
631
677
  return true;
632
678
  }
633
679
  }
680
+ // The turn activity already classified a precise MCP request timeout,
681
+ // but its DB checkpoint/fanout path itself failed before any model
682
+ // request. Finish that exact generation-2+ recovery through the bounded
683
+ // control-activity lane instead of terminalizing the turn and arming a
684
+ // fresh goal continuation from unchanged history.
685
+ const escapedMcpTimeout = escapedMcpTimeoutRecoveryDetail(outcome.error);
686
+ if (escapedMcpTimeout) {
687
+ const recovery = await activity.recoverEscapedMcpTimeout({
688
+ accountId,
689
+ workspaceId,
690
+ sessionId,
691
+ attemptId,
692
+ ...escapedMcpTimeout,
693
+ });
694
+ if (recovery.action !== "ineligible") {
695
+ const seenWakeups = wakeups;
696
+ const seenInterruptionWakeups = interruptionWakeups;
697
+ await condition(
698
+ () => interruptionWakeups !== seenInterruptionWakeups || wakeups !== seenWakeups,
699
+ ESCAPED_MCP_TIMEOUT_RECOVERY_DELAY_MS,
700
+ );
701
+ return true;
702
+ }
703
+ }
634
704
  // An ungraceful worker death never reaches the activity's graceful
635
705
  // recovery path — it surfaces here as a heartbeat-timeout failure.
636
706
  // Conversation truth was still dual-written during the turn, so the