@opengeni/worker-bundle 0.20.3 → 0.22.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +52 -0
  2. package/dist/activities/agent-turn/agent-build.d.ts +89 -0
  3. package/dist/activities/agent-turn/claim.d.ts +67 -0
  4. package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
  5. package/dist/activities/agent-turn/codex.d.ts +32 -0
  6. package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
  7. package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
  8. package/dist/activities/agent-turn/errors.d.ts +253 -0
  9. package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
  10. package/dist/activities/agent-turn/file-resources.d.ts +37 -0
  11. package/dist/activities/agent-turn/finalization.d.ts +42 -0
  12. package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
  13. package/dist/activities/agent-turn/governance-model.d.ts +66 -0
  14. package/dist/activities/agent-turn/history-sink.d.ts +52 -0
  15. package/dist/activities/agent-turn/history.d.ts +63 -0
  16. package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
  17. package/dist/activities/agent-turn/model-usage.d.ts +218 -0
  18. package/dist/activities/agent-turn/quiescence.d.ts +135 -0
  19. package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
  20. package/dist/activities/agent-turn/run.d.ts +3 -0
  21. package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
  22. package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
  23. package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
  24. package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
  25. package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
  26. package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
  27. package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
  28. package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
  29. package/dist/activities/agent-turn/turn-context.d.ts +144 -0
  30. package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
  31. package/dist/activities/agent-turn.d.ts +12 -954
  32. package/dist/activities/automations.d.ts +16 -0
  33. package/dist/activities/common.d.ts +1 -1
  34. package/dist/activities/connector-attachments.d.ts +13 -0
  35. package/dist/activities/documents.d.ts +1 -0
  36. package/dist/activities/environment.d.ts +26 -15
  37. package/dist/activities/generated-images.d.ts +1 -1
  38. package/dist/activities/git-credential-renewal.d.ts +3 -1
  39. package/dist/activities/goals.d.ts +4 -1
  40. package/dist/activities/google-drive-provider.d.ts +29 -0
  41. package/dist/activities/google-drive-publication.d.ts +17 -4
  42. package/dist/activities/incident-telemetry-authority.d.ts +52 -0
  43. package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
  44. package/dist/activities/knowledge-source-sync.d.ts +4 -0
  45. package/dist/activities/mcp-credentials.d.ts +9 -3
  46. package/dist/activities/open-suffix-resume.d.ts +77 -0
  47. package/dist/activities/packs.d.ts +3 -11
  48. package/dist/activities/parent-wake.d.ts +19 -0
  49. package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
  50. package/dist/activities/retained-screenshots.d.ts +1 -1
  51. package/dist/activities/rig-verification.d.ts +1 -1
  52. package/dist/activities/run-input.d.ts +9 -10
  53. package/dist/activities/sandbox-images.d.ts +1 -14
  54. package/dist/activities/sandbox-lease.d.ts +21 -3
  55. package/dist/activities/scheduled-tasks.d.ts +47 -0
  56. package/dist/activities/types.d.ts +23 -2
  57. package/dist/activities/video-media-validation.d.ts +1 -1
  58. package/dist/activities/video-output-retention.d.ts +1 -2
  59. package/dist/activities/xai-auth.d.ts +3 -0
  60. package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
  61. package/dist/activities-control-DKG22ONJ.js.map +1 -0
  62. package/dist/activities-control.d.ts +6 -2
  63. package/dist/activities-turn-4IMC3MAA.js +21326 -0
  64. package/dist/activities-turn-4IMC3MAA.js.map +1 -0
  65. package/dist/activities.d.ts +7 -3
  66. package/dist/artifact-materializer-entry.js +41 -22
  67. package/dist/artifact-materializer-entry.js.map +1 -1
  68. package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
  69. package/dist/chunk-CE7TPKBS.js.map +1 -0
  70. package/dist/chunk-CNYGVRKF.js +4698 -0
  71. package/dist/chunk-CNYGVRKF.js.map +1 -0
  72. package/dist/connection-provider-use.d.ts +18 -0
  73. package/dist/editable-artifact-materializer.d.ts +4 -3
  74. package/dist/github-rest-mcp.d.ts +26 -0
  75. package/dist/index.d.ts +10 -2
  76. package/dist/index.js +58 -14
  77. package/dist/index.js.map +1 -1
  78. package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
  79. package/dist/model-context-contributions.d.ts +8 -4
  80. package/dist/observability-metrics.d.ts +63 -4
  81. package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
  82. package/dist/personal-github-git-credentials.d.ts +9 -0
  83. package/dist/pr-review-credentials.d.ts +6 -0
  84. package/dist/sandbox-archive-storage.d.ts +45 -0
  85. package/dist/sandbox-resume.d.ts +65 -6
  86. package/dist/sandbox-routing.d.ts +38 -2
  87. package/dist/scheduled-alert-occurrence.d.ts +33 -0
  88. package/dist/workflow-bundle.js +87 -28
  89. package/dist/workflows/activities.d.ts +1 -0
  90. package/dist/workflows/automations.d.ts +6 -0
  91. package/dist/workflows/document-index.d.ts +1 -0
  92. package/dist/workflows.d.ts +1 -0
  93. package/package.json +17 -17
  94. package/src/activities/agent-turn/admission.ts +292 -0
  95. package/src/activities/agent-turn/agent-build.ts +802 -0
  96. package/src/activities/agent-turn/claim.ts +590 -0
  97. package/src/activities/agent-turn/codex-capacity.ts +720 -0
  98. package/src/activities/agent-turn/codex.ts +94 -0
  99. package/src/activities/agent-turn/compaction-prep.ts +722 -0
  100. package/src/activities/agent-turn/credential-leases.ts +240 -0
  101. package/src/activities/agent-turn/errors.ts +1068 -0
  102. package/src/activities/agent-turn/failure-settlement.ts +1436 -0
  103. package/src/activities/agent-turn/file-resources.ts +227 -0
  104. package/src/activities/agent-turn/finalization.ts +754 -0
  105. package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
  106. package/src/activities/agent-turn/governance-model.ts +465 -0
  107. package/src/activities/agent-turn/history-sink.ts +139 -0
  108. package/src/activities/agent-turn/history.ts +200 -0
  109. package/src/activities/agent-turn/media-artifacts.ts +359 -0
  110. package/src/activities/agent-turn/model-usage.ts +823 -0
  111. package/src/activities/agent-turn/quiescence.ts +353 -0
  112. package/src/activities/agent-turn/run-credentials.ts +761 -0
  113. package/src/activities/agent-turn/run.ts +1530 -0
  114. package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
  115. package/src/activities/agent-turn/sandbox-provision.ts +569 -0
  116. package/src/activities/agent-turn/sandbox-route.ts +358 -0
  117. package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
  118. package/src/activities/agent-turn/stream-attempt.ts +1670 -0
  119. package/src/activities/agent-turn/tool-environment.ts +729 -0
  120. package/src/activities/agent-turn/tool-policy.ts +216 -0
  121. package/src/activities/agent-turn/tool-result-spill.ts +265 -0
  122. package/src/activities/agent-turn/turn-context.ts +281 -0
  123. package/src/activities/agent-turn/xai-capacity.ts +229 -0
  124. package/src/activities/agent-turn.ts +12 -12701
  125. package/src/activities/api-integrations.ts +3 -0
  126. package/src/activities/automations.ts +214 -0
  127. package/src/activities/codemode-dispatcher.ts +12 -1
  128. package/src/activities/common.ts +1 -5
  129. package/src/activities/connector-attachments.ts +168 -0
  130. package/src/activities/context-compaction.ts +18 -0
  131. package/src/activities/environment.ts +151 -55
  132. package/src/activities/generated-images.ts +26 -11
  133. package/src/activities/git-credential-renewal.ts +13 -3
  134. package/src/activities/goals.ts +85 -19
  135. package/src/activities/google-drive-provider.ts +178 -0
  136. package/src/activities/google-drive-publication.ts +147 -49
  137. package/src/activities/incident-telemetry-authority.ts +327 -0
  138. package/src/activities/incident-telemetry-preflight.ts +317 -0
  139. package/src/activities/knowledge-source-sync.ts +178 -37
  140. package/src/activities/mcp-credentials.ts +191 -17
  141. package/src/activities/open-suffix-resume.ts +422 -0
  142. package/src/activities/packs.ts +12 -60
  143. package/src/activities/parent-wake.ts +64 -14
  144. package/src/activities/retained-screenshot-reaper.ts +6 -3
  145. package/src/activities/retained-screenshots.ts +15 -47
  146. package/src/activities/rig-verification.ts +1 -1
  147. package/src/activities/run-input.ts +102 -142
  148. package/src/activities/sandbox-images.ts +7 -166
  149. package/src/activities/sandbox-lease.ts +406 -15
  150. package/src/activities/scheduled-tasks.ts +2103 -411
  151. package/src/activities/session-state.ts +10 -2
  152. package/src/activities/types.ts +32 -2
  153. package/src/activities/video-media-validation.ts +3 -3
  154. package/src/activities/video-output-retention.ts +2 -57
  155. package/src/activities/xai-auth.ts +8 -0
  156. package/src/activities-control.ts +2 -0
  157. package/src/activities.ts +3 -0
  158. package/src/activity-services.ts +13 -4
  159. package/src/connection-provider-use.ts +36 -0
  160. package/src/editable-artifact-materializer-subprocess.ts +25 -14
  161. package/src/editable-artifact-materializer.ts +29 -18
  162. package/src/github-rest-mcp.ts +713 -0
  163. package/src/http.ts +16 -6
  164. package/src/index.ts +40 -5
  165. package/src/knowledge-source-sync-activity-policy.ts +2 -0
  166. package/src/model-context-contributions.ts +33 -10
  167. package/src/observability-metrics.ts +357 -0
  168. package/src/opensandbox-kubernetes-inventory.ts +292 -0
  169. package/src/personal-github-git-credentials.ts +331 -0
  170. package/src/pr-review-credentials.ts +163 -0
  171. package/src/sandbox-archive-storage.ts +159 -0
  172. package/src/sandbox-resume.ts +449 -63
  173. package/src/sandbox-routing.ts +314 -16
  174. package/src/scheduled-alert-occurrence.ts +176 -0
  175. package/src/workflows/activities.ts +18 -2
  176. package/src/workflows/automations.ts +11 -0
  177. package/src/workflows/session.ts +13 -3
  178. package/src/workflows.ts +1 -0
  179. package/dist/activities-control-BJSDRSIP.js.map +0 -1
  180. package/dist/activities-turn-J5YO7XXX.js +0 -17243
  181. package/dist/activities-turn-J5YO7XXX.js.map +0 -1
  182. package/dist/chunk-N5TJIFCE.js.map +0 -1
  183. package/dist/chunk-P6Z2SLAA.js +0 -3092
  184. package/dist/chunk-P6Z2SLAA.js.map +0 -1
@@ -1,3092 +0,0 @@
1
- import {
2
- observabilityEventLogger
3
- } from "./chunk-N5TJIFCE.js";
4
-
5
- // src/activity-services.ts
6
- import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth } from "@opengeni/config";
7
- import { createDb } from "@opengeni/db";
8
- import { createNatsEventBus } from "@opengeni/events";
9
- import { createObservability } from "@opengeni/observability";
10
- import { createObjectStorage } from "@opengeni/storage";
11
- function createSharedActivityServices(dependencies) {
12
- let servicesPromise = null;
13
- return async function services() {
14
- servicesPromise ??= (async () => {
15
- const settings = dependencies.settings ?? getSettings();
16
- const observability = dependencies.observability ?? createObservability(settings, { component: "worker" });
17
- const searchPath = dbSearchPath(settings);
18
- const dbClient = dependencies.db ? null : createDb(settings.databaseUrl, {
19
- ...searchPath ? { searchPath } : {},
20
- rlsStrategy: settings.rlsStrategy
21
- });
22
- const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
23
- return {
24
- settings,
25
- db: dependencies.db ?? dbClient.db,
26
- bus: dependencies.bus ?? await createNatsEventBus(
27
- settings.natsUrl,
28
- controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
29
- { logger: observabilityEventLogger(observability) }
30
- ),
31
- objectStorage: dependencies.objectStorage ?? createObjectStorage(settings),
32
- observability,
33
- wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
34
- signalSessionAttemptQuiesced: dependencies.signalSessionAttemptQuiesced ?? null,
35
- inspectSessionAttemptActivity: dependencies.inspectSessionAttemptActivity ?? null,
36
- signalCodexCapacityWorkflow: dependencies.signalCodexCapacityWorkflow ?? null,
37
- startSandboxReaperWorkflow: dependencies.startSandboxReaperWorkflow ?? null,
38
- startVideoGenerationWorkflow: dependencies.startVideoGenerationWorkflow ?? null,
39
- entitlements: dependencies.entitlements ?? null,
40
- connectionCredentials: dependencies.connectionCredentials ?? null
41
- };
42
- })();
43
- return await servicesPromise;
44
- };
45
- }
46
-
47
- // src/activities/codex-capacity.ts
48
- import {
49
- fetchCodexUsageForAccount,
50
- getCodexCapacityWaitForSession,
51
- getXaiCapacityWaitForSession,
52
- listCodexAccountStatuses,
53
- listPendingCodexCapacityWakeTargets,
54
- reconcileCodexCapacityWait as reconcileCodexCapacityWaitDb,
55
- reconcileXaiCapacityWait as reconcileXaiCapacityWaitDb
56
- } from "@opengeni/db";
57
-
58
- // src/activities/codex-rotation.ts
59
- function effectiveRotationStrategy(_stored) {
60
- return "sharded";
61
- }
62
- function classifyCodexPin(args) {
63
- const { pinnedCredentialId, pinSource, strategy, rotationEnabled } = args;
64
- const pinned = pinnedCredentialId != null;
65
- if (pinned && pinSource !== "policy") {
66
- return "manual";
67
- }
68
- const shardedActive = rotationEnabled && effectiveRotationStrategy(strategy) === "sharded";
69
- if (shardedActive) {
70
- return "sharded";
71
- }
72
- if (pinned && pinSource === "policy") {
73
- return "clearStale";
74
- }
75
- return "unpinned";
76
- }
77
- var MIN_IDLE_MS = 6e4;
78
- var DEFAULT_RESET_COOLDOWN_MS = 6e4;
79
- var CODEX_USAGE_EXHAUSTED_PCT = 100;
80
- function effectiveWindowUsed(usedPercent, resetAt, now) {
81
- return resetAt && resetAt.getTime() <= now.getTime() ? 0 : usedPercent ?? 0;
82
- }
83
- function bindingUsedPct(acct, now) {
84
- return Math.max(
85
- effectiveWindowUsed(acct.primaryUsedPercent, acct.primaryResetAt, now),
86
- effectiveWindowUsed(acct.secondaryUsedPercent, acct.secondaryResetAt, now)
87
- );
88
- }
89
- function bindingRemaining(acct, now) {
90
- const primaryRemaining = 100 - effectiveWindowUsed(acct.primaryUsedPercent, acct.primaryResetAt, now);
91
- const secondaryRemaining = 100 - effectiveWindowUsed(acct.secondaryUsedPercent, acct.secondaryResetAt, now);
92
- return Math.min(primaryRemaining, secondaryRemaining);
93
- }
94
- function cooling(acct, now) {
95
- return acct.exhaustedUntil != null && acct.exhaustedUntil.getTime() > now.getTime();
96
- }
97
- function isCodexCredentialHealthy(acct, now) {
98
- return acct.status === "active" && !cooling(acct, now) && bindingUsedPct(acct, now) < CODEX_USAGE_EXHAUSTED_PCT;
99
- }
100
- function isCodexCredentialEligible(acct, now) {
101
- return acct.allocatorEnabled && isCodexCredentialHealthy(acct, now);
102
- }
103
- function fnv1a32(input) {
104
- let hash = 2166136261;
105
- for (let i = 0; i < input.length; i++) {
106
- hash ^= input.charCodeAt(i);
107
- hash = Math.imul(hash, 16777619);
108
- }
109
- return hash >>> 0;
110
- }
111
- function shardCredentialForSession(args) {
112
- const { sessionId, accounts, now } = args;
113
- const eligibles = accounts.filter((acct) => isCodexCredentialEligible(acct, now));
114
- if (eligibles.length === 0) {
115
- return null;
116
- }
117
- const index = fnv1a32(sessionId) % eligibles.length;
118
- return eligibles[index].id;
119
- }
120
- function earliestCodexReset(accounts, now) {
121
- return earliestReset(accounts, now);
122
- }
123
- function chooseShardedHome(args) {
124
- const { sessionId, currentPolicyPin, accounts, now } = args;
125
- const pinRow = currentPolicyPin ? accounts.find((acct) => acct.id === currentPolicyPin) ?? null : null;
126
- if (pinRow && isCodexCredentialEligible(pinRow, now)) {
127
- return { kind: "home", credentialId: currentPolicyPin, rewritePin: false };
128
- }
129
- const home = shardCredentialForSession({ sessionId, accounts, now });
130
- if (home == null) {
131
- return { kind: "allCapped", earliestResetAt: earliestReset(accounts, now) };
132
- }
133
- return { kind: "home", credentialId: home, rewritePin: true };
134
- }
135
- function availableAt(acct, now) {
136
- const nowMs = now.getTime();
137
- const defaultClear = new Date(nowMs + DEFAULT_RESET_COOLDOWN_MS);
138
- const candidates = [];
139
- if (acct.exhaustedUntil != null && acct.exhaustedUntil.getTime() > nowMs) {
140
- candidates.push(acct.exhaustedUntil);
141
- }
142
- const windowClear = (over, resetAt) => {
143
- if (!over) return;
144
- candidates.push(resetAt != null && resetAt.getTime() > nowMs ? resetAt : defaultClear);
145
- };
146
- windowClear(
147
- effectiveWindowUsed(acct.primaryUsedPercent, acct.primaryResetAt, now) >= CODEX_USAGE_EXHAUSTED_PCT,
148
- acct.primaryResetAt
149
- );
150
- windowClear(
151
- effectiveWindowUsed(acct.secondaryUsedPercent, acct.secondaryResetAt, now) >= CODEX_USAGE_EXHAUSTED_PCT,
152
- acct.secondaryResetAt
153
- );
154
- if (candidates.length === 0) {
155
- return defaultClear;
156
- }
157
- return candidates.reduce((a, b) => b.getTime() > a.getTime() ? b : a);
158
- }
159
- function earliestReset(accounts, now) {
160
- return accounts.map((acct) => availableAt(acct, now)).reduce((a, b) => b.getTime() < a.getTime() ? b : a);
161
- }
162
- function authoritativeCodexCapacityResetAt(accounts, now) {
163
- const future = [];
164
- for (const account of accounts) {
165
- if (!account.allocatorEnabled || account.status !== "active") continue;
166
- if (account.exhaustedUntil && account.exhaustedUntil.getTime() > now.getTime()) {
167
- future.push(account.exhaustedUntil);
168
- }
169
- if ((account.primaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT && account.primaryResetAt && account.primaryResetAt.getTime() > now.getTime()) {
170
- future.push(account.primaryResetAt);
171
- }
172
- if ((account.secondaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT && account.secondaryResetAt && account.secondaryResetAt.getTime() > now.getTime()) {
173
- future.push(account.secondaryResetAt);
174
- }
175
- }
176
- if (future.length === 0) return null;
177
- return future.reduce(
178
- (earliest, candidate) => candidate.getTime() < earliest.getTime() ? candidate : earliest
179
- );
180
- }
181
- function computeIdleDelayMs(earliestResetAt, now, maxMs) {
182
- const delta = earliestResetAt.getTime() - now.getTime();
183
- return Math.min(Math.max(delta, MIN_IDLE_MS), maxMs);
184
- }
185
- var REACTIVE_PERSISTENCE_FAULT_FLOOR_MS = 5e3;
186
- var REACTIVE_ROTATION_MARGIN = 2;
187
- var REACTIVE_CIRCUIT_BREAKER_IDLE_MS = MIN_IDLE_MS;
188
- function computeReactiveRotationResume(args) {
189
- const streak = args.priorConsecutiveRotations + 1;
190
- const bound = args.connectedAccountCount + REACTIVE_ROTATION_MARGIN;
191
- if (streak > bound) {
192
- return { continueDelayMs: REACTIVE_CIRCUIT_BREAKER_IDLE_MS, idleUntilReset: true };
193
- }
194
- if (!args.cooldownPersisted) {
195
- return { continueDelayMs: REACTIVE_PERSISTENCE_FAULT_FLOOR_MS, idleUntilReset: false };
196
- }
197
- return { continueDelayMs: 0, idleUntilReset: false };
198
- }
199
- function chooseRotationActive(args) {
200
- const { activeCredentialId, priorCredentialId, accounts, now } = args;
201
- const rotationStrategy = effectiveRotationStrategy(args.rotationStrategy);
202
- const allocatableAccounts = accounts.filter((account) => account.allocatorEnabled);
203
- if (allocatableAccounts.length === 0) {
204
- return { kind: "none" };
205
- }
206
- const eligibles = allocatableAccounts.filter((acct) => isCodexCredentialEligible(acct, now));
207
- const decide = (chosen2) => {
208
- if (!chosen2) {
209
- return {
210
- kind: "allCapped",
211
- earliestResetAt: earliestReset(allocatableAccounts, now)
212
- };
213
- }
214
- return {
215
- kind: "active",
216
- credentialId: chosen2.id,
217
- moved: chosen2.id !== activeCredentialId
218
- };
219
- };
220
- if (rotationStrategy === "round_robin") {
221
- if (eligibles.length === 0) {
222
- return {
223
- kind: "allCapped",
224
- earliestResetAt: earliestReset(allocatableAccounts, now)
225
- };
226
- }
227
- const priorIdx = priorCredentialId ? allocatableAccounts.findIndex((acct) => acct.id === priorCredentialId) : -1;
228
- const ordered = priorIdx >= 0 ? [
229
- ...allocatableAccounts.slice(priorIdx + 1),
230
- ...allocatableAccounts.slice(0, priorIdx + 1)
231
- ] : allocatableAccounts;
232
- const chosen2 = ordered.find((acct) => isCodexCredentialEligible(acct, now));
233
- return decide(chosen2);
234
- }
235
- if (rotationStrategy === "drain_then_next") {
236
- const priorRow = priorCredentialId ? accounts.find((acct) => acct.id === priorCredentialId) : void 0;
237
- if (priorRow && isCodexCredentialEligible(priorRow, now)) {
238
- return decide(priorRow);
239
- }
240
- return decide(eligibles[0]);
241
- }
242
- const activeLeases = (acct) => "activeLeaseCount" in acct ? acct.activeLeaseCount : 0;
243
- const selections = (acct) => "selectionCount" in acct ? acct.selectionCount : 0;
244
- const lastSelected = (acct) => "lastSelectedAt" in acct && acct.lastSelectedAt ? acct.lastSelectedAt.getTime() : 0;
245
- const chosen = eligibles.reduce((best, acct) => {
246
- if (!best) {
247
- return acct;
248
- }
249
- if (activeLeases(acct) !== activeLeases(best)) {
250
- return activeLeases(acct) < activeLeases(best) ? acct : best;
251
- }
252
- if (bindingRemaining(acct, now) !== bindingRemaining(best, now)) {
253
- return bindingRemaining(acct, now) > bindingRemaining(best, now) ? acct : best;
254
- }
255
- if (selections(acct) !== selections(best)) {
256
- return selections(acct) < selections(best) ? acct : best;
257
- }
258
- return lastSelected(acct) < lastSelected(best) ? acct : best;
259
- }, void 0);
260
- return decide(chosen);
261
- }
262
- function chooseLegacyRotationActive(args) {
263
- if (args.rotationStrategy === "most_remaining") {
264
- const active = args.activeCredentialId ? args.accounts.find((account) => account.id === args.activeCredentialId) : void 0;
265
- if (active && isCodexCredentialEligible(active, args.now)) {
266
- return { kind: "active", credentialId: active.id, moved: false };
267
- }
268
- }
269
- return chooseRotationActive({
270
- ...args,
271
- accounts: args.accounts.map((account) => ({
272
- ...account,
273
- activeLeaseCount: 0,
274
- selectionCount: 0,
275
- lastSelectedAt: null
276
- }))
277
- });
278
- }
279
- function selectCodexCredentialLeaseForTurn(args) {
280
- const {
281
- accounts,
282
- activeCredentialId,
283
- rotationEnabled,
284
- leaseRotationEnabled,
285
- rotationStrategy,
286
- existingCredentialId
287
- } = args.context;
288
- const leasingEnabled = args.leasingEnabled && leaseRotationEnabled;
289
- const strategy = effectiveRotationStrategy(rotationStrategy);
290
- const existing = existingCredentialId ? accounts.find((account) => account.id === existingCredentialId) : void 0;
291
- if (existing && isCodexCredentialHealthy(existing, args.now)) {
292
- return {
293
- credentialId: existing.id,
294
- advanceActivePointer: false,
295
- decision: {
296
- kind: "active",
297
- credentialId: existing.id,
298
- moved: existing.id !== activeCredentialId
299
- }
300
- };
301
- }
302
- const connectedIds = new Set(
303
- accounts.filter((account) => account.allocatorEnabled).map((account) => account.id)
304
- );
305
- const pinDisposition = classifyCodexPin({
306
- pinnedCredentialId: args.sessionPinnedCredentialId,
307
- pinSource: args.sessionPinSource,
308
- strategy,
309
- rotationEnabled
310
- });
311
- if (pinDisposition === "manual" && args.sessionPinnedCredentialId) {
312
- const pinned = accounts.find((account) => account.id === args.sessionPinnedCredentialId);
313
- if (!pinned?.allocatorEnabled) {
314
- return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
315
- }
316
- if (isCodexCredentialHealthy(pinned, args.now)) {
317
- return {
318
- credentialId: pinned.id,
319
- decision: { kind: "active", credentialId: pinned.id, moved: false },
320
- advanceActivePointer: false
321
- };
322
- }
323
- return {
324
- credentialId: null,
325
- decision: {
326
- kind: "allCapped",
327
- earliestResetAt: availableAt(pinned, args.now)
328
- },
329
- advanceActivePointer: false
330
- };
331
- }
332
- if (pinDisposition === "sharded") {
333
- if (accounts.length === 0) {
334
- return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
335
- }
336
- const shard = chooseShardedHome({
337
- sessionId: args.sessionId,
338
- currentPolicyPin: args.sessionPinSource === "policy" ? args.sessionPinnedCredentialId : null,
339
- accounts,
340
- now: args.now
341
- });
342
- if (shard.kind === "allCapped") {
343
- return {
344
- credentialId: null,
345
- decision: shard,
346
- advanceActivePointer: false
347
- };
348
- }
349
- return {
350
- credentialId: shard.credentialId,
351
- decision: {
352
- kind: "active",
353
- credentialId: shard.credentialId,
354
- moved: shard.credentialId !== activeCredentialId
355
- },
356
- advanceActivePointer: false
357
- };
358
- }
359
- if (!rotationEnabled) {
360
- const active = activeCredentialId ? accounts.find((account) => account.id === activeCredentialId) : void 0;
361
- if (!active?.allocatorEnabled) {
362
- return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
363
- }
364
- if (!isCodexCredentialHealthy(active, args.now)) {
365
- return {
366
- credentialId: null,
367
- decision: { kind: "allCapped", earliestResetAt: availableAt(active, args.now) },
368
- advanceActivePointer: false
369
- };
370
- }
371
- return {
372
- credentialId: active.id,
373
- decision: { kind: "active", credentialId: active.id, moved: false },
374
- advanceActivePointer: false
375
- };
376
- }
377
- if (!leasingEnabled && pinDisposition !== "clearStale" && args.sessionPinnedCredentialId) {
378
- const credentialId = connectedIds.has(args.sessionPinnedCredentialId) ? args.sessionPinnedCredentialId : activeCredentialId && connectedIds.has(activeCredentialId) ? activeCredentialId : null;
379
- return {
380
- credentialId,
381
- decision: credentialId ? { kind: "active", credentialId, moved: false } : { kind: "none" }
382
- };
383
- }
384
- const priorId = args.sessionLastCredentialId ?? activeCredentialId;
385
- const choose = leasingEnabled ? chooseRotationActive : chooseLegacyRotationActive;
386
- const decision = choose({
387
- rotationStrategy: strategy,
388
- activeCredentialId,
389
- // Per-session continuity is the round-robin/drain cursor. Falling back to
390
- // the workspace pointer is only correct when this session has never run.
391
- priorCredentialId: priorId,
392
- accounts,
393
- now: args.now
394
- });
395
- return {
396
- credentialId: decision.kind === "active" ? decision.credentialId : null,
397
- decision
398
- };
399
- }
400
-
401
- // src/activities/codex-capacity.ts
402
- async function refreshCodexUsageAndRepairCapacityWaiters(refreshes, repairPendingWakes) {
403
- await Promise.all(refreshes.map((refresh) => refresh().catch(() => void 0)));
404
- await repairPendingWakes();
405
- }
406
- async function signalCodexCapacityWakeTargets(services, targets) {
407
- await Promise.allSettled(
408
- targets.map(
409
- (target) => services.signalCodexCapacityWorkflow ? services.signalCodexCapacityWorkflow({
410
- accountId: target.accountId,
411
- workspaceId: target.workspaceId,
412
- sessionId: target.sessionId,
413
- workflowId: target.workflowId,
414
- wakeRevision: target.wakeRevision
415
- }) : services.wakeSessionWorkflow ? services.wakeSessionWorkflow({
416
- accountId: target.accountId,
417
- workspaceId: target.workspaceId,
418
- sessionId: target.sessionId,
419
- workflowId: target.workflowId,
420
- wakeRevision: target.workflowWakeRevision
421
- }) : Promise.resolve()
422
- )
423
- );
424
- }
425
- async function signalPendingCodexCapacityWakeTargets(services, workspaceId) {
426
- const targets = await listPendingCodexCapacityWakeTargets(services.db, workspaceId).catch(
427
- () => []
428
- );
429
- await signalCodexCapacityWakeTargets(services, targets);
430
- }
431
- function codexCapacityDecision(context, settings) {
432
- const now = /* @__PURE__ */ new Date();
433
- const selected = selectCodexCredentialLeaseForTurn({
434
- context,
435
- leasingEnabled: settings.codexCredentialLeasingEnabled,
436
- sessionId: context.sessionId,
437
- sessionPinnedCredentialId: context.sessionPinnedCredentialId,
438
- sessionPinSource: context.sessionPinSource,
439
- sessionLastCredentialId: context.sessionLastCredentialId,
440
- now
441
- });
442
- const selectedAccount = selected.credentialId ? context.accounts.find((account) => account.id === selected.credentialId) : void 0;
443
- const selectedIsAvailable = selectedAccount !== void 0 && (selectedAccount.id === context.existingCredentialId ? isCodexCredentialHealthy(selectedAccount, now) : isCodexCredentialEligible(selectedAccount, now));
444
- if (selected.credentialId && selectedIsAvailable) {
445
- return {
446
- kind: "available",
447
- credentialId: selected.credentialId,
448
- diagnostic: {
449
- connectedCount: context.accounts.length,
450
- eligibleCount: context.accounts.filter((account) => isCodexCredentialEligible(account, now)).length
451
- }
452
- };
453
- }
454
- const authoritativeReset = authoritativeCodexCapacityResetAt(context.accounts, now);
455
- return {
456
- kind: "unavailable",
457
- earliestResetAt: authoritativeReset,
458
- resetKind: authoritativeReset ? "authoritative" : "bounded_refresh",
459
- diagnostic: {
460
- connectedCount: context.accounts.length,
461
- allocatorEnabledCount: context.accounts.filter((account) => account.allocatorEnabled).length,
462
- policyHash: context.policyHash
463
- }
464
- };
465
- }
466
- async function refreshCapacityMetadata(services, workspaceId) {
467
- const accounts = await listCodexAccountStatuses(services.db, workspaceId).catch(() => []);
468
- const stale = accounts.filter(
469
- (account) => account.allocatorEnabled && account.status === "active" && ((account.primaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT || (account.secondaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT || account.usageCheckedAt === null)
470
- );
471
- await refreshCodexUsageAndRepairCapacityWaiters(
472
- stale.map(
473
- (account) => () => fetchCodexUsageForAccount(services.db, services.settings, workspaceId, account.id)
474
- ),
475
- () => signalPendingCodexCapacityWakeTargets(services, workspaceId)
476
- );
477
- }
478
- function createCodexCapacityActivities(services) {
479
- async function getCodexCapacityWait(input) {
480
- const { db } = await services();
481
- const codexWaiter = await getCodexCapacityWaitForSession(
482
- db,
483
- input.workspaceId,
484
- input.sessionId
485
- );
486
- const xaiWaiter = codexWaiter ? null : await getXaiCapacityWaitForSession(db, input.workspaceId, input.sessionId);
487
- const waiter = codexWaiter ?? xaiWaiter;
488
- return waiter ? {
489
- ...xaiWaiter ? { provider: "xai" } : {},
490
- waiterId: waiter.id,
491
- generation: waiter.generation,
492
- // A capacity mutation may have committed while its Temporal signal
493
- // was lost or while the workflow continued-as-new. Reconstruct that
494
- // outbox edge as an immediate re-evaluation rather than waiting for
495
- // the older timer.
496
- nextCheckAt: waiter.wakeRevision > waiter.observedWakeRevision ? (/* @__PURE__ */ new Date(0)).toISOString() : waiter.nextCheckAt.toISOString(),
497
- wakeRevision: waiter.wakeRevision
498
- } : null;
499
- }
500
- async function reconcileCodexCapacityWait(input) {
501
- const resolved = await services();
502
- if (input.provider === "xai") {
503
- const current2 = await getXaiCapacityWaitForSession(
504
- resolved.db,
505
- input.workspaceId,
506
- input.sessionId
507
- );
508
- if (!current2 || current2.id !== input.waiterId || current2.generation !== input.generation) {
509
- return { action: "stale" };
510
- }
511
- const result2 = await reconcileXaiCapacityWaitDb(resolved.db, {
512
- accountId: input.accountId,
513
- workspaceId: input.workspaceId,
514
- sessionId: input.sessionId,
515
- waiterId: input.waiterId,
516
- generation: input.generation
517
- });
518
- if (result2.events.length > 0) {
519
- try {
520
- await resolved.bus.publish(input.workspaceId, input.sessionId, result2.events);
521
- } catch {
522
- }
523
- }
524
- if (result2.action === "resumed") return { action: "resumed" };
525
- if (result2.action === "waiting") {
526
- return {
527
- action: "waiting",
528
- provider: "xai",
529
- waiterId: result2.waiter.id,
530
- generation: result2.waiter.generation,
531
- nextCheckAt: result2.waiter.nextCheckAt.toISOString(),
532
- wakeRevision: result2.waiter.wakeRevision
533
- };
534
- }
535
- return { action: result2.action };
536
- }
537
- const current = await getCodexCapacityWaitForSession(
538
- resolved.db,
539
- input.workspaceId,
540
- input.sessionId
541
- );
542
- if (!current || current.id !== input.waiterId || current.generation !== input.generation) {
543
- return { action: "stale" };
544
- }
545
- if (input.cause === "timer" && current.nextCheckAt.getTime() <= Date.now()) {
546
- await refreshCapacityMetadata(resolved, input.workspaceId);
547
- }
548
- const result = await reconcileCodexCapacityWaitDb(
549
- resolved.db,
550
- {
551
- accountId: input.accountId,
552
- workspaceId: input.workspaceId,
553
- sessionId: input.sessionId,
554
- waiterId: input.waiterId,
555
- generation: input.generation
556
- },
557
- (context) => codexCapacityDecision(context, resolved.settings)
558
- );
559
- if (result.events.length > 0) {
560
- try {
561
- await resolved.bus.publish(input.workspaceId, input.sessionId, result.events);
562
- } catch {
563
- }
564
- }
565
- if (result.action === "resumed") {
566
- return { action: "resumed" };
567
- }
568
- if (result.action === "waiting") {
569
- return {
570
- action: "waiting",
571
- waiterId: result.waiter.id,
572
- generation: result.waiter.generation,
573
- nextCheckAt: result.waiter.nextCheckAt.toISOString(),
574
- wakeRevision: result.waiter.wakeRevision
575
- };
576
- }
577
- return { action: result.action };
578
- }
579
- return { getCodexCapacityWait, reconcileCodexCapacityWait };
580
- }
581
-
582
- // src/activities/goals.ts
583
- import {
584
- configuredStaticUsageLimits,
585
- policyProviderIdForModel
586
- } from "@opengeni/config";
587
- import {
588
- evaluateWorkspaceModelPolicy,
589
- latencyModeForMetadata,
590
- mergeToolRefs,
591
- reasoningEffortForMetadata
592
- } from "@opengeni/contracts";
593
- import { isCodexBilledModel } from "@opengeni/codex";
594
- import {
595
- enqueueSessionWorkflowWakeIfRunnable,
596
- getBillingBalance,
597
- getLatestStartedSessionTurn,
598
- getWorkspaceModelPolicy,
599
- getSessionGoal,
600
- isCodexBilledTurn,
601
- materializeGoalContinuation,
602
- requireSession,
603
- sumUsageQuantity
604
- } from "@opengeni/db";
605
- function createGoalActivities(services) {
606
- async function enqueueGoalRetryWake(input) {
607
- const { db } = await services();
608
- await enqueueSessionWorkflowWakeIfRunnable(db, {
609
- accountId: input.accountId,
610
- workspaceId: input.workspaceId,
611
- sessionId: input.sessionId,
612
- temporalWorkflowId: input.workflowId,
613
- reason: "goal_retry",
614
- // A permanently invalid goal must not become a tight workflow loop. One
615
- // durable retry after a short delay preserves liveness without scanning.
616
- notBefore: new Date(Date.now() + 3e4)
617
- });
618
- }
619
- async function maybeContinueGoal(input) {
620
- const { settings, db, bus } = await services();
621
- const existingGoal = await getSessionGoal(db, input.workspaceId, input.sessionId);
622
- if (!existingGoal || existingGoal.status !== "active") {
623
- return { action: "none" };
624
- }
625
- const session = await requireSession(db, input.workspaceId, input.sessionId);
626
- const latestStartedTurn = await getLatestStartedSessionTurn(
627
- db,
628
- input.workspaceId,
629
- input.sessionId
630
- );
631
- let continuationModel = latestStartedTurn?.model ?? session.model;
632
- const continuationReasoningEffort = latestStartedTurn?.reasoningEffort ?? reasoningEffortForMetadata(session.metadata, settings.openaiReasoningEffort);
633
- const continuationLatencyMode = latestStartedTurn?.latencyMode ?? latencyModeForMetadata(session.metadata, "standard");
634
- let modelPolicyBlocked = null;
635
- const workspaceModelPolicy = await getWorkspaceModelPolicy(db, input.workspaceId);
636
- if (workspaceModelPolicy) {
637
- const policyBlocks = (modelId) => !evaluateWorkspaceModelPolicy(workspaceModelPolicy, {
638
- providerId: policyProviderIdForModel(settings, modelId),
639
- modelId
640
- }).allowed;
641
- if (policyBlocks(continuationModel)) {
642
- if (continuationModel !== session.model && !policyBlocks(session.model)) {
643
- continuationModel = session.model;
644
- } else {
645
- modelPolicyBlocked = `workspace model policy blocks model "${continuationModel}"; pick an allowed model or change the workspace model policy`;
646
- }
647
- }
648
- }
649
- if (!modelPolicyBlocked && session.codexCompactionMode === "remote_v2" && !isCodexBilledModel(continuationModel)) {
650
- modelPolicyBlocked = `session is locked to Codex remote compaction v2; model "${continuationModel}" is not a Codex subscription model`;
651
- }
652
- const isCodexRun = await isCodexBilledTurn({
653
- db,
654
- settings,
655
- workspaceId: input.workspaceId,
656
- model: continuationModel
657
- });
658
- const budgetBlocked = await goalRunBudgetBlocked(
659
- settings,
660
- db,
661
- input.accountId,
662
- input.workspaceId,
663
- isCodexRun
664
- );
665
- const decision = await materializeGoalContinuation(db, {
666
- accountId: input.accountId,
667
- workspaceId: input.workspaceId,
668
- sessionId: input.sessionId,
669
- workflowId: input.workflowId,
670
- defaultMaxAutoContinuations: settings.goalMaxAutoContinuations ?? null,
671
- noProgressLimit: settings.goalNoProgressLimit,
672
- // A model-policy block takes precedence: it is deterministic (a budget
673
- // pause can clear on its own; a policy pause needs a model/policy change)
674
- // and rides the same visible-pause channel.
675
- budgetBlocked: modelPolicyBlocked ?? budgetBlocked,
676
- policy: {
677
- model: continuationModel,
678
- reasoningEffort: continuationReasoningEffort,
679
- latencyMode: continuationLatencyMode,
680
- tools: withFirstPartyTools(settings, session.tools),
681
- sandboxBackend: session.sandboxBackend
682
- },
683
- prompt: goalContinuationPrompt
684
- });
685
- if (decision.events.length > 0) {
686
- await bus.publish(input.workspaceId, input.sessionId, decision.events);
687
- }
688
- return { action: decision.action };
689
- }
690
- return {
691
- enqueueGoalRetryWake,
692
- maybeContinueGoal
693
- };
694
- }
695
- function goalContinuationPrompt(goal, autoContinuation, cap) {
696
- const counter = cap === null ? `${autoContinuation}` : `${autoContinuation}/${cap}`;
697
- return [
698
- `[GOAL CONTINUATION ${counter}] The session goal is not done. Goal: ${goal.text}.`,
699
- `Success criteria: ${goal.successCriteria ?? "none specified"}.`,
700
- `Objective revision: ${goal.objectiveRevision}. Mutation policy: ${goal.mutationPolicy}.`,
701
- "Continue from the existing conversation state. Do not repeat completed session setup, persistent metadata settings, or context checks merely because this is a new continuation turn.",
702
- "Continue working toward the goal now. If it is actually complete, call opengeni__goal_complete with concrete evidence.",
703
- "If you are blocked or continuing is not productive, call opengeni__goal_pause with your rationale.",
704
- "Record concrete progress with opengeni__goal_progress. Revise the goal only when explicit user direction or material new evidence justifies the declared change kind; use opengeni__goal_update with this objective revision and a rationale. A rewrite alone is not progress. Do not stop without goal_complete, goal_pause, or concrete continued work."
705
- ].join("\n");
706
- }
707
- function withFirstPartyTools(settings, tools) {
708
- if (!settings.mcpServers.some((server) => server.id === "opengeni")) {
709
- return tools;
710
- }
711
- return mergeToolRefs(tools, [{ kind: "mcp", id: "opengeni" }]);
712
- }
713
- async function goalRunBudgetBlocked(settings, db, accountId, workspaceId, isCodexRun) {
714
- if (!isCodexRun && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
715
- const balance = await getBillingBalance(db, accountId);
716
- if (balance.balanceMicros <= 0) {
717
- return "insufficient OpenGeni credits";
718
- }
719
- }
720
- if (settings.usageLimitsMode === "static" || settings.usageLimitsMode === "managed") {
721
- const limits = configuredStaticUsageLimits(settings);
722
- if (!isCodexRun && limits.maxMonthlyCostMicrosPerAccount) {
723
- const used = await sumUsageQuantity(db, {
724
- accountId,
725
- eventType: "model.cost",
726
- since: startOfUtcMonth()
727
- });
728
- if (used >= limits.maxMonthlyCostMicrosPerAccount) {
729
- return `monthly model cost limit reached (${limits.maxMonthlyCostMicrosPerAccount} micros)`;
730
- }
731
- }
732
- if (limits.maxMonthlyAgentRunsPerWorkspace) {
733
- const used = await sumUsageQuantity(db, {
734
- workspaceId,
735
- eventType: "agent_run.created",
736
- since: startOfUtcMonth()
737
- });
738
- if (used + 1 > limits.maxMonthlyAgentRunsPerWorkspace) {
739
- return `monthly agent run limit reached (${limits.maxMonthlyAgentRunsPerWorkspace})`;
740
- }
741
- }
742
- }
743
- return null;
744
- }
745
- function startOfUtcMonth() {
746
- const now = /* @__PURE__ */ new Date();
747
- return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
748
- }
749
-
750
- // src/activities/sandbox-images.ts
751
- import {
752
- rigProviderImageContentHash,
753
- rigProviderImageMatchesDefinition,
754
- rigProviderImageProviderBindingKeyHash
755
- } from "@opengeni/core";
756
- import { resolveModalCheckpointProviderBinding } from "@opengeni/runtime/sandbox";
757
- function settingsWithPackSandboxImage(settings, sandboxImage, sandboxProviderImages = null) {
758
- if (!sandboxImage) return settings;
759
- return {
760
- ...settings,
761
- dockerImage: sandboxImage,
762
- modalImageRef: sandboxImage,
763
- modalImageId: sandboxProviderImages?.modal?.imageId
764
- };
765
- }
766
- function settingsWithRigImage(settings, rigImage) {
767
- if (!rigImage) return settings;
768
- return {
769
- ...settings,
770
- dockerImage: rigImage,
771
- modalImageRef: rigImage,
772
- // A logical rig image invalidates a lower-precedence pack/deployment
773
- // provider ID. A verified version-bound provider ID is applied separately
774
- // after this precedence step.
775
- modalImageId: void 0
776
- };
777
- }
778
- function rigProviderImageSourceImage(settings, backend) {
779
- if (backend === "modal") return settings.modalImageId ?? settings.modalImageRef ?? null;
780
- if (backend === "docker") return settings.dockerImage ?? null;
781
- return null;
782
- }
783
- function resolveRigProviderImageSelection(settings, version, backend, currentProviderBindingKeyHash) {
784
- if (!version) {
785
- return { settings, reason: "missing", contentHash: null, imageId: null };
786
- }
787
- if (backend !== "modal") {
788
- return { settings, reason: "provider_unsupported", contentHash: null, imageId: null };
789
- }
790
- const image = version.providerImages[backend];
791
- if (!image) {
792
- return { settings, reason: "missing", contentHash: null, imageId: null };
793
- }
794
- if (image.status !== "ready" || !image.imageId) {
795
- return {
796
- settings,
797
- reason: "not_ready",
798
- contentHash: image.contentHash,
799
- imageId: image.imageId
800
- };
801
- }
802
- if (image.coldBootValidation?.version !== 1) {
803
- return {
804
- settings,
805
- reason: "not_cold_boot_validated",
806
- contentHash: image.contentHash,
807
- imageId: null
808
- };
809
- }
810
- const sourceImage = rigProviderImageSourceImage(settings, backend);
811
- const expectedContentHash = rigProviderImageContentHash({
812
- backend,
813
- sourceImage,
814
- definition: version
815
- });
816
- if (image.sourceImage !== sourceImage || image.contentHash !== expectedContentHash || !rigProviderImageMatchesDefinition(image, version)) {
817
- return {
818
- settings,
819
- reason: "content_mismatch",
820
- contentHash: expectedContentHash,
821
- imageId: null
822
- };
823
- }
824
- if (!image.providerBindingKeyHash || !currentProviderBindingKeyHash) {
825
- return {
826
- settings,
827
- reason: "provider_binding_unavailable",
828
- contentHash: expectedContentHash,
829
- imageId: null
830
- };
831
- }
832
- if (image.providerBindingKeyHash !== currentProviderBindingKeyHash) {
833
- return {
834
- settings,
835
- reason: "provider_binding_mismatch",
836
- contentHash: expectedContentHash,
837
- imageId: null
838
- };
839
- }
840
- return {
841
- // Keep modalImageRef unchanged: the lease continues to fence the logical
842
- // rig/pack/deployment image, so a newly available build optimization never
843
- // rotates an already-warm or archive-backed box. modalImageId affects only
844
- // genuinely fresh provider creation.
845
- settings: { ...settings, modalImageId: image.imageId },
846
- reason: "selected",
847
- contentHash: expectedContentHash,
848
- imageId: image.imageId
849
- };
850
- }
851
- async function settingsWithRigProviderImage(settings, version, backend, resolveBinding = resolveModalCheckpointProviderBinding) {
852
- const image = backend === "modal" ? version?.providerImages.modal : null;
853
- if (image?.status !== "ready" || !image.providerBindingKeyHash) {
854
- return resolveRigProviderImageSelection(settings, version, backend, null).settings;
855
- }
856
- const structural = resolveRigProviderImageSelection(
857
- settings,
858
- version,
859
- backend,
860
- image.providerBindingKeyHash
861
- );
862
- if (structural.reason !== "selected") return structural.settings;
863
- let currentProviderBindingKeyHash = null;
864
- try {
865
- const binding = await resolveBinding(settings);
866
- currentProviderBindingKeyHash = rigProviderImageProviderBindingKeyHash(binding.key);
867
- } catch {
868
- }
869
- return resolveRigProviderImageSelection(settings, version, backend, currentProviderBindingKeyHash).settings;
870
- }
871
-
872
- // src/activities/packs.ts
873
- import { CapabilityPack } from "@opengeni/contracts";
874
- import {
875
- getWorkspace,
876
- getWorkspacePack,
877
- listInstalledPortableSkills,
878
- listPackInstallations
879
- } from "@opengeni/db";
880
- import {
881
- buildPortableSkillArtifact
882
- } from "@opengeni/runtime";
883
- var emptyPackRuntime = {
884
- sandboxImage: null,
885
- sandboxProviderImages: null,
886
- skillActivations: []
887
- };
888
- async function resolveWorkspacePackRuntime(db, workspaceId) {
889
- const installations = await listPackInstallations(db, workspaceId);
890
- const active = installations.filter((installation) => installation.status === "active");
891
- if (active.length === 0) {
892
- return emptyPackRuntime;
893
- }
894
- const packs = [];
895
- for (const installation of active) {
896
- if (!packInstallationUsesLegacyRuntime(installation)) {
897
- continue;
898
- }
899
- const registration = await getWorkspacePack(db, workspaceId, installation.packId);
900
- if (!registration) {
901
- continue;
902
- }
903
- const parsed = CapabilityPack.safeParse(registration.pack);
904
- if (parsed.success) {
905
- packs.push(parsed.data);
906
- }
907
- }
908
- return workspacePackRuntimeFromPacks(packs);
909
- }
910
- function packInstallationUsesLegacyRuntime(installation) {
911
- return installation.manifestSnapshot === null && installation.manifestDigest === null;
912
- }
913
- async function resolveWorkspaceInstalledSkillRuntime(db, workspaceId) {
914
- const installedSkills = await listInstalledPortableSkills(db, workspaceId);
915
- const activations = [];
916
- for (const installed of installedSkills) {
917
- const artifact = buildPortableSkillArtifact(installed.files);
918
- if (artifact.name !== installed.name || artifact.description !== installed.description || artifact.contentSha256 !== installed.contentSha256) {
919
- throw new Error(
920
- `Installed Skill artifact verification failed for ${installed.capabilityId}; reinstall it from the pinned source`
921
- );
922
- }
923
- activations.push({
924
- source: "installation",
925
- id: installed.capabilityId,
926
- artifact: {
927
- name: artifact.name,
928
- description: artifact.description,
929
- files: artifact.files.map((file) => ({ path: file.path, content: file.content }))
930
- },
931
- version: installed.version,
932
- contentSha256: artifact.contentSha256,
933
- reason: installed.source === "library" ? "installed from the curated Skill library" : installed.source === "pack" ? "installed as a Pack-owned Skill" : `installed from ${installed.sourceUrl}`
934
- });
935
- }
936
- return { activations };
937
- }
938
- function workspacePackRuntimeFromPacks(packs) {
939
- const imagePacks = packs.filter(
940
- (pack) => typeof pack.sandboxImage === "string" && pack.sandboxImage.trim().length > 0
941
- );
942
- if (imagePacks.length > 1) {
943
- const ids = imagePacks.map((pack) => pack.id).sort().join(", ");
944
- throw new Error(
945
- `Multiple enabled packs declare a sandbox image (${ids}). Only one enabled pack per workspace may declare sandboxImage; disable the others and retry.`
946
- );
947
- }
948
- const skillActivations = [];
949
- const skillOwners = /* @__PURE__ */ new Map();
950
- for (const pack of [...packs].sort((a, b) => a.id.localeCompare(b.id))) {
951
- for (const skill of pack.skills) {
952
- const key = skill.name.toLowerCase();
953
- const existingOwner = skillOwners.get(key);
954
- if (existingOwner !== void 0 && existingOwner !== pack.id) {
955
- throw new Error(
956
- `Enabled packs ${existingOwner} and ${pack.id} both declare a skill named "${skill.name}". Pack skill names must be unique across enabled packs; disable one of the packs and retry.`
957
- );
958
- }
959
- skillOwners.set(key, pack.id);
960
- const artifact = {
961
- name: skill.name,
962
- description: skill.description ?? null,
963
- files: skill.files.map((file) => ({ path: file.path, content: file.content }))
964
- };
965
- skillActivations.push({
966
- source: "pack",
967
- id: `pack:${pack.id}:${skill.name}`,
968
- artifact,
969
- reason: `active legacy Pack ${pack.id}`
970
- });
971
- }
972
- }
973
- return {
974
- sandboxImage: imagePacks[0]?.sandboxImage?.trim() ?? null,
975
- sandboxProviderImages: imagePacks[0]?.sandboxProviderImages ?? null,
976
- skillActivations
977
- };
978
- }
979
- function mergeRigDefaultVariableSetEnvironment(rigDefaultValues, sessionValues) {
980
- return { ...rigDefaultValues, ...sessionValues };
981
- }
982
-
983
- // src/activities/streaming.ts
984
- import { TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS } from "@opengeni/core";
985
- import { Context } from "@temporalio/activity";
986
- var TRAILING_FLUSH_MS = 33;
987
- var MAX_BATCH_EVENTS = 50;
988
- var MAX_BUFFERED_EVENTS = 1e3;
989
- function createRuntimeBatcher(flushEvents, options = {}) {
990
- const now = options.now ?? (() => performance.now());
991
- let pending = [];
992
- let lastFlush = Date.now();
993
- let trailingTimer = null;
994
- let activeBatchSize = 0;
995
- let drainPromise = null;
996
- let flushFailed = false;
997
- let flushFailure;
998
- const structural = /* @__PURE__ */ new Set([
999
- "terminal.pty.output.delta",
1000
- "agent.toolCall.created",
1001
- "agent.toolCall.output",
1002
- "agent.message.completed",
1003
- "tool.auth_needed",
1004
- "credential.auth_needed",
1005
- "session.requiresAction",
1006
- "turn.completed",
1007
- "turn.failed",
1008
- "turn.cancelled"
1009
- ]);
1010
- return {
1011
- push: async (event) => {
1012
- throwIfFlushFailed();
1013
- pending.push({ type: event.type, payload: event.payload });
1014
- const elapsed = Date.now() - lastFlush;
1015
- if (structural.has(event.type)) {
1016
- await flush();
1017
- return;
1018
- }
1019
- if (pending.length >= MAX_BATCH_EVENTS || elapsed >= TRAILING_FLUSH_MS) {
1020
- void startDrain().catch(() => void 0);
1021
- }
1022
- if (activeBatchSize + pending.length >= MAX_BUFFERED_EVENTS) {
1023
- await flush();
1024
- return;
1025
- }
1026
- armTrailingTimer();
1027
- },
1028
- flush
1029
- };
1030
- function armTrailingTimer() {
1031
- if (trailingTimer !== null) {
1032
- return;
1033
- }
1034
- trailingTimer = setTimeout(() => {
1035
- trailingTimer = null;
1036
- void flush().catch(() => void 0);
1037
- }, TRAILING_FLUSH_MS);
1038
- if ("unref" in trailingTimer && typeof trailingTimer.unref === "function") {
1039
- trailingTimer.unref();
1040
- }
1041
- }
1042
- function clearTrailingTimer() {
1043
- if (trailingTimer !== null) {
1044
- clearTimeout(trailingTimer);
1045
- trailingTimer = null;
1046
- }
1047
- }
1048
- async function flush() {
1049
- throwIfFlushFailed();
1050
- clearTrailingTimer();
1051
- await startDrain();
1052
- throwIfFlushFailed();
1053
- }
1054
- function startDrain() {
1055
- if (drainPromise) {
1056
- return drainPromise;
1057
- }
1058
- if (flushFailed) {
1059
- return Promise.reject(flushFailure);
1060
- }
1061
- if (pending.length === 0) {
1062
- return Promise.resolve();
1063
- }
1064
- clearTrailingTimer();
1065
- const running = (async () => {
1066
- while (pending.length > 0) {
1067
- const events = pending.splice(0, MAX_BATCH_EVENTS);
1068
- activeBatchSize = events.length;
1069
- lastFlush = Date.now();
1070
- const startedAt = now();
1071
- try {
1072
- await flushEvents(events);
1073
- } catch (error) {
1074
- flushFailed = true;
1075
- flushFailure = error;
1076
- throw error;
1077
- } finally {
1078
- activeBatchSize = 0;
1079
- if (options.onFlush) {
1080
- try {
1081
- options.onFlush({
1082
- events: events.length,
1083
- durationSeconds: Math.max(0, (now() - startedAt) / 1e3)
1084
- });
1085
- } catch {
1086
- }
1087
- }
1088
- }
1089
- }
1090
- })();
1091
- drainPromise = running.finally(() => {
1092
- drainPromise = null;
1093
- });
1094
- return drainPromise;
1095
- }
1096
- function throwIfFlushFailed() {
1097
- if (flushFailed) {
1098
- throw flushFailure;
1099
- }
1100
- }
1101
- }
1102
- function currentActivityContext() {
1103
- try {
1104
- return Context.current();
1105
- } catch {
1106
- return null;
1107
- }
1108
- }
1109
- function startActivityHeartbeat(context, details, schedule = setInterval) {
1110
- if (!context) {
1111
- return null;
1112
- }
1113
- const timer = schedule(() => {
1114
- context.heartbeat({ ...details, at: (/* @__PURE__ */ new Date()).toISOString() });
1115
- }, TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS);
1116
- if ("unref" in timer && typeof timer.unref === "function") {
1117
- timer.unref();
1118
- }
1119
- return timer;
1120
- }
1121
- async function nextStreamEvent(iterator, context) {
1122
- if (!context) {
1123
- return await iterator.next();
1124
- }
1125
- const signal = context.cancellationSignal;
1126
- if (signal.aborted) {
1127
- throw signal.reason;
1128
- }
1129
- return await new Promise((resolve, reject) => {
1130
- let settled = false;
1131
- const settle = (complete) => {
1132
- if (settled) return;
1133
- settled = true;
1134
- signal.removeEventListener("abort", onAbort);
1135
- complete();
1136
- };
1137
- const onAbort = () => settle(() => reject(signal.reason));
1138
- signal.addEventListener("abort", onAbort, { once: true });
1139
- if (signal.aborted) {
1140
- onAbort();
1141
- return;
1142
- }
1143
- let next;
1144
- try {
1145
- next = iterator.next();
1146
- } catch (error) {
1147
- settle(() => reject(error));
1148
- return;
1149
- }
1150
- void Promise.resolve(next).then(
1151
- (result) => settle(() => resolve(result)),
1152
- (error) => settle(() => reject(error))
1153
- );
1154
- });
1155
- }
1156
-
1157
- // src/activities/parent-wake.ts
1158
- import {
1159
- getSession,
1160
- getSessionGoal as getSessionGoal2,
1161
- getSessionParentPersonalConnectionDelegations,
1162
- getSessionParentXaiProviderAccountAuthority,
1163
- addSessionSystemUpdateWithSourceMutation,
1164
- claimPendingSessionSystemUpdateOutbox,
1165
- claimPendingSessionWorkflowWakes,
1166
- getSessionSystemUpdateOutboxByDedupeKey,
1167
- getOrCreateSessionSystemUpdateOutbox,
1168
- markSessionWorkflowWakeFailed,
1169
- markSessionSystemUpdateOutboxDeliveredInTransaction,
1170
- markSessionSystemUpdateOutboxFailed
1171
- } from "@opengeni/db";
1172
- async function notifyParentOfChildIdle(svc, workspaceId, childSessionId, episodeKey) {
1173
- try {
1174
- const child = await getSession(svc.db, workspaceId, childSessionId);
1175
- if (!child || !child.parentSessionId) {
1176
- return;
1177
- }
1178
- const goal = await getSessionGoal2(svc.db, workspaceId, childSessionId);
1179
- const clientEventId = `child-completion:${childSessionId}:${episodeKey}`;
1180
- const payload = childCompletionPayload(child, goal);
1181
- const personalConnectionDelegations = await getSessionParentPersonalConnectionDelegations(
1182
- svc.db,
1183
- workspaceId,
1184
- childSessionId
1185
- );
1186
- const xaiAuthority = await getSessionParentXaiProviderAccountAuthority(
1187
- svc.db,
1188
- workspaceId,
1189
- childSessionId
1190
- );
1191
- const outbox = await getOrCreateSessionSystemUpdateOutbox(svc.db, {
1192
- accountId: child.accountId,
1193
- workspaceId,
1194
- sourceSessionId: child.id,
1195
- targetSessionId: child.parentSessionId,
1196
- kind: "child_terminal_result",
1197
- classification: goal?.status === "paused" ? "action_required" : "success",
1198
- sourceId: child.id,
1199
- dedupeKey: clientEventId,
1200
- summary: childCompletionSummary(child, goal, "idle"),
1201
- payload,
1202
- lineage: {
1203
- childSessionId: child.id,
1204
- parentSessionId: child.parentSessionId,
1205
- ...xaiAuthority.subjectId ? { xaiAuthoritySubjectId: xaiAuthority.subjectId } : {}
1206
- },
1207
- personalConnectionDelegations,
1208
- xaiProviderAccountAuthoritySnapshot: xaiAuthority.snapshot
1209
- });
1210
- if (outbox.status === "delivered") {
1211
- return;
1212
- }
1213
- await deliverParentSystemUpdateOutbox(svc, outbox);
1214
- } catch (error) {
1215
- svc.observability.error("Failed to wake parent session on worker idle boundary", {
1216
- childSessionId,
1217
- error: error instanceof Error ? error.message : String(error)
1218
- });
1219
- }
1220
- }
1221
- async function deliverFailedChildTurnToParent(svc, workspaceId, childSessionId, turnId) {
1222
- try {
1223
- const child = await getSession(svc.db, workspaceId, childSessionId);
1224
- if (!child || !child.parentSessionId) return;
1225
- const dedupeKey = `child-completion:${childSessionId}:turn:${turnId}`;
1226
- const outbox = await getSessionSystemUpdateOutboxByDedupeKey(svc.db, {
1227
- accountId: child.accountId,
1228
- workspaceId,
1229
- dedupeKey
1230
- });
1231
- if (!outbox) {
1232
- throw new Error(`Committed failed-child outbox disappeared: ${dedupeKey}`);
1233
- }
1234
- if (outbox.status === "delivered") return;
1235
- await deliverParentSystemUpdateOutbox(svc, outbox);
1236
- } catch (error) {
1237
- svc.observability.error("Failed to deliver committed child-turn failure", {
1238
- childSessionId,
1239
- turnId,
1240
- error: error instanceof Error ? error.message : String(error)
1241
- });
1242
- }
1243
- }
1244
- async function deliverParentSystemUpdateOutbox(svc, outbox) {
1245
- try {
1246
- const result = await addSessionSystemUpdateWithSourceMutation(
1247
- svc.db,
1248
- {
1249
- accountId: outbox.accountId,
1250
- workspaceId: outbox.workspaceId,
1251
- sessionId: outbox.targetSessionId,
1252
- kind: outbox.kind,
1253
- classification: outbox.classification,
1254
- sourceId: outbox.sourceId,
1255
- dedupeKey: outbox.dedupeKey,
1256
- summary: outbox.summary,
1257
- payload: outbox.payload,
1258
- lineage: outbox.lineage,
1259
- personalConnectionDelegations: outbox.personalConnectionDelegations,
1260
- xaiProviderAccountAuthoritySnapshot: outbox.xaiProviderAccountAuthoritySnapshot
1261
- },
1262
- async (tx) => {
1263
- await markSessionSystemUpdateOutboxDeliveredInTransaction(tx, outbox);
1264
- }
1265
- );
1266
- if (result.reason === "session_cancelled") {
1267
- return;
1268
- }
1269
- if (result.added && result.events.length > 0) {
1270
- await svc.bus.publish(outbox.workspaceId, outbox.targetSessionId, result.events);
1271
- }
1272
- if (result.shouldWake && svc.wakeSessionWorkflow) {
1273
- if (result.workflowWakeRevision === null) {
1274
- throw new Error("Runnable system update has no workflow wake revision");
1275
- }
1276
- await svc.wakeSessionWorkflow({
1277
- accountId: outbox.accountId,
1278
- workspaceId: outbox.workspaceId,
1279
- sessionId: outbox.targetSessionId,
1280
- workflowId: result.temporalWorkflowId ?? `session-${outbox.targetSessionId}`,
1281
- wakeRevision: result.workflowWakeRevision
1282
- });
1283
- }
1284
- svc.observability.info("Woke parent session on worker completion", {
1285
- childSessionId: outbox.sourceSessionId,
1286
- parentSessionId: outbox.targetSessionId,
1287
- dedupeKey: outbox.dedupeKey
1288
- });
1289
- } catch (error) {
1290
- await markSessionSystemUpdateOutboxFailed(
1291
- svc.db,
1292
- outbox,
1293
- error instanceof Error ? error.message : String(error)
1294
- ).catch(() => void 0);
1295
- throw error;
1296
- }
1297
- }
1298
- async function reconcilePendingParentSystemUpdates(svc, limit = 100, overrides = {}) {
1299
- const claimPendingSessionSystemUpdateOutboxFn = overrides.claimPendingSessionSystemUpdateOutbox ?? claimPendingSessionSystemUpdateOutbox;
1300
- const rows = await claimPendingSessionSystemUpdateOutboxFn(svc.db, limit);
1301
- let delivered = 0;
1302
- let failed = 0;
1303
- for (const row of rows) {
1304
- try {
1305
- await deliverParentSystemUpdateOutbox(svc, row);
1306
- delivered += 1;
1307
- } catch {
1308
- failed += 1;
1309
- }
1310
- }
1311
- return { claimed: rows.length, delivered, failed };
1312
- }
1313
- async function reconcilePendingSessionWorkflowWakes(svc, limit = 1e3, overrides = {}) {
1314
- const claimPendingSessionWorkflowWakesFn = overrides.claimPendingSessionWorkflowWakes ?? claimPendingSessionWorkflowWakes;
1315
- if (!svc.wakeSessionWorkflow) {
1316
- return { claimed: 0, delivered: 0, failed: 0 };
1317
- }
1318
- const repairs = await claimPendingSessionWorkflowWakesFn(svc.db, limit);
1319
- let delivered = 0;
1320
- let failed = 0;
1321
- const queue = [...repairs];
1322
- const workers = Array.from({ length: Math.min(20, queue.length) }, async () => {
1323
- for (; ; ) {
1324
- const repair = queue.shift();
1325
- if (!repair) return;
1326
- try {
1327
- await svc.wakeSessionWorkflow({
1328
- accountId: repair.accountId,
1329
- workspaceId: repair.workspaceId,
1330
- sessionId: repair.sessionId,
1331
- workflowId: repair.temporalWorkflowId,
1332
- wakeRevision: repair.wakeRevision,
1333
- ...repair.interruptionRequested ? { interruptionRequested: true } : {}
1334
- });
1335
- delivered += 1;
1336
- } catch (error) {
1337
- failed += 1;
1338
- await markSessionWorkflowWakeFailed(
1339
- svc.db,
1340
- repair,
1341
- error instanceof Error ? error.message : String(error)
1342
- ).catch(() => void 0);
1343
- }
1344
- }
1345
- });
1346
- await Promise.all(workers);
1347
- return { claimed: repairs.length, delivered, failed };
1348
- }
1349
- function childCompletionPayload(child, goal) {
1350
- return {
1351
- type: "child_terminal_result",
1352
- childSessionId: child.id,
1353
- status: "idle",
1354
- ...goal ? {
1355
- goal: {
1356
- status: goal.status,
1357
- text: goal.text,
1358
- ...goal.evidence ? { evidence: goal.evidence } : {},
1359
- ...goal.rationale ? { rationale: goal.rationale } : {},
1360
- ...goal.pausedReason ? { pausedReason: goal.pausedReason } : {}
1361
- }
1362
- } : {}
1363
- };
1364
- }
1365
- function childCompletionSummary(child, goal, terminalStatus) {
1366
- const lines = [];
1367
- if (terminalStatus === "failed") {
1368
- lines.push(`A worker session you spawned has FAILED. Worker session id: ${child.id}.`);
1369
- } else if (goal?.status === "completed") {
1370
- lines.push(
1371
- `A worker session you spawned has COMPLETED its goal. Worker session id: ${child.id}.`
1372
- );
1373
- } else if (goal?.status === "paused") {
1374
- lines.push(
1375
- `A worker session you spawned has PAUSED its goal and gone idle. Worker session id: ${child.id}.`
1376
- );
1377
- } else {
1378
- lines.push(
1379
- `A worker session you spawned has finished its work and gone idle. Worker session id: ${child.id}.`
1380
- );
1381
- }
1382
- if (goal) {
1383
- lines.push(`Worker goal: ${goal.text}`);
1384
- if (goal.status === "completed" && goal.evidence) {
1385
- lines.push(`Completion evidence: ${goal.evidence}`);
1386
- }
1387
- if (goal.status === "paused" && goal.rationale) {
1388
- lines.push(`Pause rationale: ${goal.rationale}`);
1389
- }
1390
- }
1391
- return lines.join("\n");
1392
- }
1393
-
1394
- // src/sandbox-resume.ts
1395
- import {
1396
- sandboxArchiveCaptureTimeoutMs,
1397
- sandboxLifecycleTransitionWaitMs
1398
- } from "@opengeni/config";
1399
- import { randomUUID } from "crypto";
1400
- import {
1401
- acquireLease,
1402
- adoptLegacyModalCheckpointArtifact,
1403
- beginSandboxRematerialization,
1404
- claimWorkspaceArchiveCapture,
1405
- commitWarmingToWarm,
1406
- failSandboxRematerialization,
1407
- failWarmingToCold,
1408
- getSandboxSessionEnvelope,
1409
- heartbeatLeaseHolder,
1410
- markSandboxRestoreVerifying,
1411
- markWarmLeaseInstanceLost,
1412
- markSandboxCheckpointArtifactDeletePending,
1413
- persistWarmSnapshot,
1414
- readLease,
1415
- registerSandboxCheckpointArtifact,
1416
- recordWarmingSandboxCreated,
1417
- releaseWorkspaceArchiveCapture,
1418
- releaseLeaseHolder,
1419
- touchLeaseHolder,
1420
- SandboxLeaseRecoveryBlockedError,
1421
- SandboxLeaseSupersededError,
1422
- SandboxLeaseTransitionError
1423
- } from "@opengeni/db";
1424
- import {
1425
- captureVerifiedWorkspaceArchive,
1426
- describeLegacyNativeSnapshotArchive,
1427
- MODAL_EXEC_READINESS_TIMEOUT_MS,
1428
- SandboxExecReadinessError,
1429
- WorkspaceArchiveIntegrityError,
1430
- establishSandboxSessionFromEnvelope,
1431
- isProviderSandboxNotFoundError,
1432
- parseWorkspaceArchiveDescriptor,
1433
- providerWorkspaceCapturePolicy,
1434
- SandboxProviderContinuityUnavailableError,
1435
- requirePersistableReplacementSandboxEnvelope,
1436
- modalSessionMatchesCheckpointProviderBinding,
1437
- resolveModalCheckpointProviderBindingForSession,
1438
- serializeReplacementSandboxEnvelope,
1439
- tagModalSandbox,
1440
- terminateUnpublishedSandboxSession,
1441
- verifySandboxExecReadiness,
1442
- withoutSandboxProviderIdentity
1443
- } from "@opengeni/runtime";
1444
- function sandboxLeaseHolderIdForAttempt(attemptId) {
1445
- const normalized = attemptId.trim();
1446
- if (!normalized) {
1447
- throw new Error("Sandbox lease holder requires a turn attempt id");
1448
- }
1449
- return `turn-attempt:${normalized}`;
1450
- }
1451
- function isRetryableDegradedRestore(restore) {
1452
- return restore.status === "degraded" && restore.retryable === true;
1453
- }
1454
- var SandboxWarmingTimeoutError = class extends Error {
1455
- constructor(backend, timeoutMs, stage, sandboxGroupId, instanceId) {
1456
- const timeoutSeconds = Math.ceil(timeoutMs / 1e3);
1457
- const identity = [
1458
- sandboxGroupId ? `group ${sandboxGroupId}` : null,
1459
- instanceId ? `sandbox ${instanceId}` : null
1460
- ].filter((part) => part !== null).join(", ");
1461
- const target = identity ? ` (${identity})` : "";
1462
- super(
1463
- stage === "exec_readiness" ? `Sandbox backend "${backend}"${target} did not become command-ready within ${timeoutSeconds}s after creation or restore.` : `Sandbox backend "${backend}"${target} did not finish warming within ${timeoutSeconds}s while waiting for the elected sandbox creator.`
1464
- );
1465
- this.backend = backend;
1466
- this.timeoutMs = timeoutMs;
1467
- this.stage = stage;
1468
- this.sandboxGroupId = sandboxGroupId;
1469
- this.instanceId = instanceId;
1470
- this.name = "SandboxWarmingTimeoutError";
1471
- }
1472
- };
1473
- var SandboxExecReadinessTimeoutError = class extends SandboxWarmingTimeoutError {
1474
- code = "sandbox_exec_readiness_timeout";
1475
- constructor(backend, timeoutMs, identity = {}) {
1476
- super(
1477
- backend,
1478
- timeoutMs,
1479
- "exec_readiness",
1480
- identity.sandboxGroupId ?? null,
1481
- identity.instanceId ?? null
1482
- );
1483
- this.name = "SandboxExecReadinessTimeoutError";
1484
- }
1485
- };
1486
- var SandboxSiblingWarmingTimeoutError = class extends SandboxWarmingTimeoutError {
1487
- code = "sandbox_sibling_warming_timeout";
1488
- constructor(backend, timeoutMs, identity) {
1489
- super(
1490
- backend,
1491
- timeoutMs,
1492
- "sibling_warming",
1493
- identity.sandboxGroupId,
1494
- identity.instanceId ?? null
1495
- );
1496
- this.name = "SandboxSiblingWarmingTimeoutError";
1497
- }
1498
- };
1499
- var SandboxLeaseInstanceLostError = class extends SandboxLeaseSupersededError {
1500
- constructor(sandboxGroupId, leaseEpoch, lostInstanceId) {
1501
- super(sandboxGroupId, leaseEpoch);
1502
- this.lostInstanceId = lostInstanceId;
1503
- this.name = "SandboxLeaseInstanceLostError";
1504
- }
1505
- };
1506
- var WARMING_POLL_INTERVAL_MS = 250;
1507
- async function sleep(ms) {
1508
- await new Promise((resolve) => setTimeout(resolve, ms));
1509
- }
1510
- async function waitForSandboxExecReadiness(established, timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS, identity = {}) {
1511
- try {
1512
- await verifySandboxExecReadiness(established, timeoutMs);
1513
- } catch (error) {
1514
- if (error instanceof SandboxExecReadinessError && error.code === "exec_probe_timeout") {
1515
- throw new SandboxExecReadinessTimeoutError(established.backendId, timeoutMs, {
1516
- sandboxGroupId: identity.sandboxGroupId ?? null,
1517
- instanceId: established.instanceId
1518
- });
1519
- }
1520
- throw error;
1521
- }
1522
- }
1523
- var SnapshotTimeoutError = class extends Error {
1524
- constructor(timeoutMs) {
1525
- super(`workspace snapshot timed out after ${timeoutMs}ms`);
1526
- this.name = "SnapshotTimeoutError";
1527
- }
1528
- };
1529
- function safeSnapshotError(error) {
1530
- const fields = {
1531
- errorClass: "SnapshotOperationError",
1532
- errorCode: "snapshot_operation_failed",
1533
- origin: "sandbox-resume"
1534
- };
1535
- try {
1536
- const rawStatus = error && typeof error === "object" ? error.status ?? error.statusCode : void 0;
1537
- const status = Number(rawStatus);
1538
- if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
1539
- } catch {
1540
- }
1541
- return fields;
1542
- }
1543
- async function waitForWarmSnapshot(snapshot, timeoutMs, signal) {
1544
- if (signal?.aborted) return false;
1545
- let timeout;
1546
- let cancelListener;
1547
- try {
1548
- await Promise.race([
1549
- snapshot,
1550
- ...signal ? [
1551
- new Promise((_resolve, reject) => {
1552
- cancelListener = () => reject(signal.reason ?? new Error("workspace snapshot wait cancelled"));
1553
- signal.addEventListener("abort", cancelListener, { once: true });
1554
- if (signal.aborted) cancelListener();
1555
- })
1556
- ] : [],
1557
- new Promise((_, reject) => {
1558
- timeout = setTimeout(() => reject(new SnapshotTimeoutError(timeoutMs)), timeoutMs);
1559
- if (timeout && "unref" in timeout && typeof timeout.unref === "function") {
1560
- timeout.unref();
1561
- }
1562
- })
1563
- ]);
1564
- return true;
1565
- } catch (error) {
1566
- if (signal?.aborted) return false;
1567
- if (error instanceof SnapshotTimeoutError) {
1568
- console.error(
1569
- "mid-session workspace snapshot wait timed out (turn unaffected)",
1570
- safeSnapshotError(error)
1571
- );
1572
- return false;
1573
- }
1574
- return true;
1575
- } finally {
1576
- if (timeout) {
1577
- clearTimeout(timeout);
1578
- }
1579
- if (cancelListener) signal?.removeEventListener("abort", cancelListener);
1580
- }
1581
- }
1582
- async function resolveModalCheckpointBindingBeforeCapture(settings, session, signal) {
1583
- if (signal?.aborted) {
1584
- throw signal.reason ?? new Error("workspace snapshot identity lookup cancelled");
1585
- }
1586
- const pending = resolveModalCheckpointProviderBindingForSession(settings, session);
1587
- let timeout;
1588
- let cancelListener;
1589
- try {
1590
- return await Promise.race([
1591
- pending,
1592
- ...signal ? [
1593
- new Promise((_resolve, reject) => {
1594
- cancelListener = () => reject(signal.reason ?? new Error("workspace snapshot identity lookup cancelled"));
1595
- signal.addEventListener("abort", cancelListener, { once: true });
1596
- if (signal.aborted) cancelListener();
1597
- })
1598
- ] : [],
1599
- new Promise((_resolve, reject) => {
1600
- timeout = setTimeout(
1601
- () => reject(new SnapshotTimeoutError(settings.sandboxSnapshotTimeoutMs)),
1602
- settings.sandboxSnapshotTimeoutMs
1603
- );
1604
- timeout.unref?.();
1605
- })
1606
- ]);
1607
- } finally {
1608
- if (timeout) clearTimeout(timeout);
1609
- if (cancelListener) signal?.removeEventListener("abort", cancelListener);
1610
- }
1611
- }
1612
- async function terminateEstablishedSandbox(established) {
1613
- if (!established) return true;
1614
- try {
1615
- await terminateUnpublishedSandboxSession(established);
1616
- return true;
1617
- } catch (error) {
1618
- if (isProviderSandboxNotFoundError(established.backendId, error)) {
1619
- return true;
1620
- }
1621
- return false;
1622
- }
1623
- }
1624
- function recordSandboxWarmingTimeout(metrics, error) {
1625
- if (!(error instanceof SandboxWarmingTimeoutError)) {
1626
- return;
1627
- }
1628
- try {
1629
- metrics?.onSandboxWarmingTimeout?.({ backend: error.backend, stage: error.stage });
1630
- } catch {
1631
- }
1632
- }
1633
- function workspaceArchiveFieldsFromEnvelope(envelope) {
1634
- const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
1635
- const archive = sessionState?.workspaceArchive;
1636
- if (typeof archive !== "string" || archive.length === 0) {
1637
- return null;
1638
- }
1639
- const previous = sessionState?.workspaceArchivePrev;
1640
- const metadata = sessionState?.workspaceArchiveMeta;
1641
- const previousMetadata = sessionState?.workspaceArchivePrevMeta;
1642
- const capturedAt = sessionState?.workspaceArchiveAt;
1643
- return {
1644
- workspaceArchive: archive,
1645
- ...metadata !== void 0 ? { workspaceArchiveMeta: metadata } : {},
1646
- ...typeof previous === "string" && previous.length > 0 ? { workspaceArchivePrev: previous } : {},
1647
- ...previousMetadata !== void 0 ? { workspaceArchivePrevMeta: previousMetadata } : {},
1648
- ...typeof capturedAt === "string" && capturedAt.length > 0 ? { workspaceArchiveAt: capturedAt } : {}
1649
- };
1650
- }
1651
- function legacyNativeArchiveFromEnvelope(envelope) {
1652
- const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
1653
- if (!sessionState) return null;
1654
- const existing = parseWorkspaceArchiveDescriptor(sessionState.workspaceArchiveMeta);
1655
- if (existing?.version === 2) return null;
1656
- return describeLegacyNativeSnapshotArchive(
1657
- sessionState.workspaceArchive,
1658
- existing?.version === 1 ? Date.parse(existing.capturedAt) : Date.now()
1659
- );
1660
- }
1661
- async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch, signal, force = false) {
1662
- const { db, settings } = services;
1663
- const intervalMs = settings.sandboxSnapshotIntervalMs;
1664
- if (intervalMs <= 0 && !force) {
1665
- return false;
1666
- }
1667
- const persistable = session;
1668
- if (typeof persistable.persistWorkspace !== "function") {
1669
- return false;
1670
- }
1671
- const workspacePersistence = persistable.state?.workspacePersistence ?? persistable.state?.providerState?.workspacePersistence;
1672
- if (signal?.aborted) {
1673
- return false;
1674
- }
1675
- try {
1676
- const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
1677
- if (!lease || lease.leaseEpoch !== leaseEpoch || lease.liveness !== "warm" || lease.instanceId === null) {
1678
- return false;
1679
- }
1680
- if (lease.archiveComplete) {
1681
- return false;
1682
- }
1683
- const instanceId = lease.instanceId;
1684
- const capturePolicy = providerWorkspaceCapturePolicy(lease.backend, session);
1685
- if (!capturePolicy || capturePolicy.liveInstance !== "preserved") {
1686
- return false;
1687
- }
1688
- const nativeModalPersistence = workspacePersistence === "snapshot_filesystem" || workspacePersistence === "snapshot_directory";
1689
- const checkpointBinding = nativeModalPersistence ? await resolveModalCheckpointBindingBeforeCapture(settings, persistable, signal) : null;
1690
- if (signal?.aborted) return false;
1691
- const captureId = randomUUID();
1692
- const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs(settings);
1693
- const claimed = await claimWorkspaceArchiveCapture(db, {
1694
- accountId: ids.accountId,
1695
- workspaceId: ids.workspaceId,
1696
- sandboxGroupId: ids.sandboxGroupId,
1697
- captureId,
1698
- expectedEpoch: leaseEpoch,
1699
- expectedInstanceId: instanceId,
1700
- liveness: "warm",
1701
- captureTimeoutMs,
1702
- minIntervalMs: force ? 0 : intervalMs,
1703
- providerReplaySafe: capturePolicy.takeover === "same_request",
1704
- takeoverSafe: capturePolicy.takeover !== "exclusive",
1705
- warmAttempt: {
1706
- sessionId: ids.sessionId,
1707
- turnId: ids.turnId,
1708
- attemptId: ids.attemptId,
1709
- holderId: sandboxLeaseHolderIdForAttempt(ids.attemptId)
1710
- }
1711
- });
1712
- if (claimed.status !== "claimed") return false;
1713
- let timeout;
1714
- const capturedAtMs = Date.now();
1715
- const registerCandidate = async (archive) => {
1716
- if (archive.descriptor.version !== 2 || archive.descriptor.provider !== "modal_snapshot_filesystem" && archive.descriptor.provider !== "modal_snapshot_directory") {
1717
- return null;
1718
- }
1719
- if (!checkpointBinding) {
1720
- throw new Error("Modal native snapshot has no exact session provider identity");
1721
- }
1722
- return await registerSandboxCheckpointArtifact(db, {
1723
- accountId: ids.accountId,
1724
- workspaceId: ids.workspaceId,
1725
- sandboxGroupId: ids.sandboxGroupId,
1726
- sourceLeaseId: claimed.claim.leaseId,
1727
- sourceLeaseEpoch: leaseEpoch,
1728
- sourceInstanceId: instanceId,
1729
- sourceWorkspaceGeneration: claimed.claim.workspaceGeneration,
1730
- providerBindingKey: checkpointBinding.key,
1731
- providerBinding: checkpointBinding.binding,
1732
- workspaceArchive: archive.base64,
1733
- workspaceArchiveMeta: archive.descriptor
1734
- });
1735
- };
1736
- const abandonCandidate = async (artifactId, reason) => {
1737
- await markSandboxCheckpointArtifactDeletePending(db, {
1738
- accountId: ids.accountId,
1739
- workspaceId: ids.workspaceId,
1740
- artifactId,
1741
- reason
1742
- }).catch(() => void 0);
1743
- };
1744
- const captureAndPublish = (async () => {
1745
- let candidate = null;
1746
- try {
1747
- const archive = await captureVerifiedWorkspaceArchive(session, capturedAtMs, {
1748
- requestId: claimed.claim.providerRequestId,
1749
- strategy: capturePolicy.strategy
1750
- });
1751
- candidate = await registerCandidate(archive);
1752
- const { wrote } = await persistWarmSnapshot(db, {
1753
- accountId: ids.accountId,
1754
- workspaceId: ids.workspaceId,
1755
- sessionId: ids.sessionId,
1756
- turnId: ids.turnId,
1757
- attemptId: ids.attemptId,
1758
- sandboxGroupId: ids.sandboxGroupId,
1759
- expectedEpoch: leaseEpoch,
1760
- expectedInstanceId: instanceId,
1761
- expectedWorkspaceGeneration: claimed.claim.workspaceGeneration,
1762
- captureId,
1763
- workspaceArchive: archive.base64,
1764
- workspaceArchiveMeta: archive.descriptor,
1765
- checkpointArtifactId: candidate?.id ?? null,
1766
- minIntervalMs: force ? 0 : intervalMs,
1767
- capturedAtMs
1768
- });
1769
- if (!wrote && candidate) {
1770
- await abandonCandidate(candidate.id, "snapshot_publication_fenced");
1771
- }
1772
- return wrote;
1773
- } catch (error) {
1774
- if (candidate) await abandonCandidate(candidate.id, "snapshot_capture_failed");
1775
- throw error;
1776
- } finally {
1777
- await releaseWorkspaceArchiveCapture(db, {
1778
- accountId: ids.accountId,
1779
- workspaceId: ids.workspaceId,
1780
- sandboxGroupId: ids.sandboxGroupId,
1781
- captureId,
1782
- expectedEpoch: leaseEpoch,
1783
- expectedInstanceId: instanceId
1784
- }).catch((error) => {
1785
- console.error(
1786
- "mid-session workspace capture gate release failed",
1787
- safeSnapshotError(error)
1788
- );
1789
- });
1790
- }
1791
- })();
1792
- const settled = captureAndPublish.then(
1793
- (persisted) => ({ kind: "settled", persisted }),
1794
- (error) => {
1795
- console.error(
1796
- "mid-session workspace snapshot failed (turn unaffected)",
1797
- safeSnapshotError(error)
1798
- );
1799
- return { kind: "settled", persisted: false };
1800
- }
1801
- );
1802
- let cancelListener;
1803
- const outcome = await Promise.race([
1804
- settled,
1805
- ...signal ? [
1806
- new Promise((resolve) => {
1807
- cancelListener = () => resolve({ kind: "cancelled" });
1808
- signal.addEventListener("abort", cancelListener, { once: true });
1809
- if (signal.aborted) cancelListener();
1810
- })
1811
- ] : [],
1812
- new Promise((resolve) => {
1813
- timeout = setTimeout(
1814
- () => resolve({ kind: "timed_out" }),
1815
- settings.sandboxSnapshotTimeoutMs
1816
- );
1817
- })
1818
- ]).finally(() => {
1819
- if (timeout) clearTimeout(timeout);
1820
- if (cancelListener) signal?.removeEventListener("abort", cancelListener);
1821
- });
1822
- return outcome.kind === "settled" ? outcome.persisted : false;
1823
- } catch (error) {
1824
- console.error(
1825
- "mid-session workspace snapshot failed (turn unaffected)",
1826
- safeSnapshotError(error)
1827
- );
1828
- return false;
1829
- }
1830
- }
1831
- async function resumeBoxForTurn(services, ids, kind, holderId) {
1832
- const { db, settings } = services;
1833
- const os = ids.os ?? "linux";
1834
- const leaseTtlMs = settings.sandboxLeaseTtlMs;
1835
- const cancellationSignal = services.cancellationSignal;
1836
- let releaseStarted = false;
1837
- let holderRelease = null;
1838
- let quiescedRelease = null;
1839
- let holderLivenessTimer;
1840
- let cancellationListener;
1841
- const release = (options) => {
1842
- const writersQuiesced = options?.workspaceWritersQuiesced === true;
1843
- if (!releaseStarted) {
1844
- releaseStarted = true;
1845
- if (cancellationListener) {
1846
- cancellationSignal?.removeEventListener("abort", cancellationListener);
1847
- cancellationListener = void 0;
1848
- }
1849
- if (holderLivenessTimer) {
1850
- clearInterval(holderLivenessTimer);
1851
- holderLivenessTimer = void 0;
1852
- }
1853
- }
1854
- const persistRelease = async (workspaceWritersQuiesced) => {
1855
- await releaseLeaseHolder(db, {
1856
- accountId: ids.accountId,
1857
- workspaceId: ids.workspaceId,
1858
- sandboxGroupId: ids.sandboxGroupId,
1859
- kind,
1860
- holderId,
1861
- idleGraceMs: settings.sandboxIdleGraceMs,
1862
- ...workspaceWritersQuiesced ? { workspaceWritersQuiesced: true } : {}
1863
- });
1864
- };
1865
- if (writersQuiesced) {
1866
- if (!quiescedRelease) {
1867
- const predecessor = holderRelease;
1868
- const attempt = (predecessor ? predecessor.catch(() => void 0) : Promise.resolve()).then(
1869
- async () => await persistRelease(true)
1870
- );
1871
- const tracked = attempt.catch((error) => {
1872
- if (quiescedRelease === tracked) quiescedRelease = null;
1873
- throw error;
1874
- });
1875
- quiescedRelease = tracked;
1876
- holderRelease = tracked;
1877
- }
1878
- return quiescedRelease;
1879
- }
1880
- if (!holderRelease) {
1881
- const attempt = persistRelease(false);
1882
- const tracked = attempt.catch((error) => {
1883
- if (holderRelease === tracked) holderRelease = null;
1884
- throw error;
1885
- });
1886
- holderRelease = tracked;
1887
- }
1888
- return holderRelease;
1889
- };
1890
- const cancellationError = () => cancellationSignal?.reason instanceof Error ? cancellationSignal.reason : new Error("Sandbox resume was cancelled with its owning turn attempt");
1891
- const throwIfReleasedOrCancelled = () => {
1892
- if (releaseStarted || cancellationSignal?.aborted) {
1893
- throw cancellationError();
1894
- }
1895
- };
1896
- const acquired = await acquireLease(db, {
1897
- accountId: ids.accountId,
1898
- workspaceId: ids.workspaceId,
1899
- sandboxGroupId: ids.sandboxGroupId,
1900
- kind,
1901
- holderId,
1902
- subjectId: ids.sessionId,
1903
- backend: ids.backend,
1904
- os,
1905
- // IMAGE IS SHARED STATE (B3): thread the resolved image so the lease stamps it +
1906
- // conflicts on a live box already running a different image. A
1907
- // SandboxImageConflictError propagates while another holder is active; a
1908
- // solo change requests a capture-and-drain rotation and this attempt retries
1909
- // after the cold successor can safely stamp the new image.
1910
- ...ids.image ? { image: ids.image } : {},
1911
- // RIG IS SHARED STATE (M3): thread the frozen rig version so the lease stamps it
1912
- // + conflicts on a live box under a different rig. A SandboxRigConflictError
1913
- // propagates while another holder is active; a solo change uses the same
1914
- // durable capture-and-drain rotation as an image change.
1915
- ...ids.rigVersionId ? { rigVersionId: ids.rigVersionId } : {},
1916
- leaseTtlMs,
1917
- warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs,
1918
- captureWaitMs: sandboxLifecycleTransitionWaitMs(settings),
1919
- ...cancellationSignal ? { waitSignal: cancellationSignal } : {}
1920
- });
1921
- if (cancellationSignal) {
1922
- cancellationListener = () => {
1923
- void release().catch(() => void 0);
1924
- };
1925
- cancellationSignal.addEventListener("abort", cancellationListener, { once: true });
1926
- }
1927
- if (cancellationSignal?.aborted) {
1928
- await release();
1929
- throw cancellationError();
1930
- }
1931
- let holderLeaseHeartbeat = acquired.role === "spawner" ? {
1932
- expectedEpoch: acquired.lease.leaseEpoch,
1933
- leaseTtlMs: settings.sandboxLeaseWarmingTtlMs
1934
- } : (acquired.role === "attached" || acquired.role === "rearmed") && acquired.lease.liveness === "warm" ? { expectedEpoch: acquired.lease.leaseEpoch, leaseTtlMs } : null;
1935
- holderLivenessTimer = setInterval(() => {
1936
- const heartbeat = holderLeaseHeartbeat;
1937
- const refresh = heartbeat ? heartbeatLeaseHolder(db, {
1938
- accountId: ids.accountId,
1939
- workspaceId: ids.workspaceId,
1940
- sandboxGroupId: ids.sandboxGroupId,
1941
- kind,
1942
- holderId,
1943
- expectedEpoch: heartbeat.expectedEpoch,
1944
- leaseTtlMs: heartbeat.leaseTtlMs
1945
- }) : touchLeaseHolder(db, {
1946
- accountId: ids.accountId,
1947
- workspaceId: ids.workspaceId,
1948
- sandboxGroupId: ids.sandboxGroupId,
1949
- kind,
1950
- holderId
1951
- });
1952
- void refresh.then((touched) => {
1953
- if (!touched && heartbeat === holderLeaseHeartbeat) {
1954
- void release().catch(() => void 0);
1955
- }
1956
- }).catch(() => void 0);
1957
- }, 1e4);
1958
- if ("unref" in holderLivenessTimer && typeof holderLivenessTimer.unref === "function") {
1959
- holderLivenessTimer.unref();
1960
- }
1961
- if (acquired.role === "blocked") {
1962
- await release();
1963
- throw new SandboxLeaseRecoveryBlockedError(
1964
- ids.sandboxGroupId,
1965
- acquired.lease.leaseEpoch,
1966
- acquired.code,
1967
- acquired.lease.recovery
1968
- );
1969
- }
1970
- if (acquired.role === "fenced") {
1971
- await release();
1972
- if (acquired.reason !== "superseded") {
1973
- throw new SandboxLeaseTransitionError(
1974
- ids.sandboxGroupId,
1975
- acquired.lease.leaseEpoch,
1976
- acquired.reason,
1977
- acquired.lease.backend,
1978
- acquired.lease.instanceId,
1979
- acquired.lease.liveness
1980
- );
1981
- }
1982
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch, {
1983
- backend: acquired.lease.backend,
1984
- instanceId: acquired.lease.instanceId,
1985
- liveness: acquired.lease.liveness
1986
- });
1987
- }
1988
- if (acquired.role === "spawner") {
1989
- const expectedEpoch = acquired.lease.leaseEpoch;
1990
- let createdEstablished = null;
1991
- let rematerialization = null;
1992
- try {
1993
- const envelope = await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
1994
- const fallbackArchiveEnvelope = acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(envelope) !== null ? withoutSandboxProviderIdentity(envelope) : null;
1995
- let spawnEnvelope = fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
1996
- const archiveSource = acquired.lease.recovery.archive.status === "none" ? fallbackArchiveEnvelope : acquired.lease.resumeState;
1997
- const continuityRecovery = acquired.lease.recovery.continuity;
1998
- if (acquired.lease.recovery.archive.status === "available" && acquired.lease.archiveComplete || acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(archiveSource) !== null) {
1999
- const rematerializationId = crypto.randomUUID();
2000
- const legacyNativeArchive = legacyNativeArchiveFromEnvelope(archiveSource);
2001
- const begun = await beginSandboxRematerialization(db, {
2002
- accountId: ids.accountId,
2003
- workspaceId: ids.workspaceId,
2004
- sandboxGroupId: ids.sandboxGroupId,
2005
- expectedEpoch,
2006
- rematerializationId,
2007
- archiveSource,
2008
- legacyNativeArchive
2009
- });
2010
- if (begun.status !== "started") {
2011
- if (begun.code === "stale_epoch" || begun.code === "attempt_conflict") {
2012
- throw new SandboxLeaseSupersededError(
2013
- ids.sandboxGroupId,
2014
- begun.lease?.leaseEpoch ?? expectedEpoch
2015
- );
2016
- }
2017
- throw new SandboxLeaseRecoveryBlockedError(
2018
- ids.sandboxGroupId,
2019
- begun.lease?.leaseEpoch ?? expectedEpoch,
2020
- begun.code === "archive_unverified" ? "restore_degraded" : "restore_unrecoverable",
2021
- begun.lease?.recovery ?? acquired.lease.recovery
2022
- );
2023
- }
2024
- spawnEnvelope = begun.lease.resumeState ?? spawnEnvelope;
2025
- const selectedRevision = begun.lease.recovery.restore.selectedRevision;
2026
- if (!selectedRevision) {
2027
- throw new WorkspaceArchiveIntegrityError(
2028
- "archive_metadata_invalid",
2029
- "sandbox rematerialization selected no durable archive revision"
2030
- );
2031
- }
2032
- rematerialization = {
2033
- id: rematerializationId,
2034
- selectedRevision,
2035
- workspaceGeneration: begun.lease.workspaceGeneration,
2036
- providerBindingKey: begun.checkpointArtifact?.providerBindingKey ?? null,
2037
- legacyCheckpoint: begun.checkpointArtifact === null ? legacyNativeArchive : null,
2038
- legacyProviderBinding: null
2039
- };
2040
- } else if (acquired.lease.recovery.archive.status !== "none" && !continuityRecovery) {
2041
- throw new SandboxLeaseRecoveryBlockedError(
2042
- ids.sandboxGroupId,
2043
- expectedEpoch,
2044
- "restore_degraded",
2045
- acquired.lease.recovery
2046
- );
2047
- }
2048
- const providerCreateStartedAt = /* @__PURE__ */ new Date();
2049
- const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
2050
- sessionId: ids.sessionId,
2051
- recovery: "create-or-restore",
2052
- backendOverride: ids.backend,
2053
- ...ids.environment ? { environment: ids.environment } : {},
2054
- ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {},
2055
- ...services.logicalFallbackSettings ? { logicalFallbackSettings: services.logicalFallbackSettings } : {},
2056
- onSandboxCreated: async (created) => {
2057
- createdEstablished = created;
2058
- throwIfReleasedOrCancelled();
2059
- if (rematerialization && (rematerialization.providerBindingKey || rematerialization.legacyCheckpoint)) {
2060
- if (created.backendId !== "modal") {
2061
- throw new WorkspaceArchiveIntegrityError(
2062
- "native_snapshot_reference_invalid",
2063
- "Modal checkpoint restore resolved a non-Modal sandbox backend"
2064
- );
2065
- }
2066
- if (rematerialization.providerBindingKey) {
2067
- if (!await modalSessionMatchesCheckpointProviderBinding(
2068
- settings,
2069
- created.session,
2070
- rematerialization.providerBindingKey
2071
- )) {
2072
- throw new WorkspaceArchiveIntegrityError(
2073
- "native_snapshot_reference_invalid",
2074
- "Modal checkpoint restore refused because the authenticated provider workspace changed"
2075
- );
2076
- }
2077
- } else {
2078
- const resolved = await resolveModalCheckpointProviderBindingForSession(
2079
- settings,
2080
- created.session
2081
- );
2082
- if (rematerialization.legacyProviderBinding && rematerialization.legacyProviderBinding.key !== resolved.key) {
2083
- throw new WorkspaceArchiveIntegrityError(
2084
- "native_snapshot_reference_invalid",
2085
- "Legacy Modal checkpoint restore crossed authenticated provider workspaces"
2086
- );
2087
- }
2088
- rematerialization.legacyProviderBinding = resolved;
2089
- }
2090
- }
2091
- const resumeEnvelope2 = requirePersistableReplacementSandboxEnvelope(
2092
- await serializeReplacementSandboxEnvelope(created, spawnEnvelope),
2093
- created.backendId
2094
- );
2095
- const recorded = await recordWarmingSandboxCreated(db, {
2096
- accountId: ids.accountId,
2097
- workspaceId: ids.workspaceId,
2098
- sandboxGroupId: ids.sandboxGroupId,
2099
- expectedEpoch,
2100
- rematerializationId: rematerialization?.id ?? null,
2101
- ...created.providerContinuity ? { continuityRecovery: created.providerContinuity } : {},
2102
- instanceId: created.instanceId,
2103
- resumeBackendId: created.backendId,
2104
- resumeState: resumeEnvelope2,
2105
- ...created.backendId === "modal" ? {
2106
- providerCreatedAt: providerCreateStartedAt,
2107
- providerDeadlineAt: new Date(
2108
- providerCreateStartedAt.getTime() + settings.modalTimeoutSeconds * 1e3
2109
- )
2110
- } : {},
2111
- leaseTtlMs,
2112
- // Keep the warming budget after create(): manifest setup and
2113
- // commitWarmingToWarm still run, and can exceed the 90s turn TTL.
2114
- warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs
2115
- });
2116
- if (!recorded.recorded) {
2117
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
2118
- }
2119
- if (created.backendId === "modal") {
2120
- await tagModalSandbox(settings, created.instanceId, {
2121
- leaseId: acquired.lease.id,
2122
- workspaceId: ids.workspaceId,
2123
- sandboxGroupId: ids.sandboxGroupId
2124
- }).catch(() => void 0);
2125
- }
2126
- throwIfReleasedOrCancelled();
2127
- },
2128
- onWorkspaceRestoreVerifying: async (descriptor) => {
2129
- if (!rematerialization || descriptor.revision !== rematerialization.selectedRevision) {
2130
- throw new WorkspaceArchiveIntegrityError(
2131
- "archive_metadata_invalid",
2132
- `hydrated archive revision ${descriptor.revision} does not match the selected rematerialization revision`
2133
- );
2134
- }
2135
- if (rematerialization.legacyCheckpoint) {
2136
- const binding = rematerialization.legacyProviderBinding;
2137
- if (!binding) {
2138
- throw new WorkspaceArchiveIntegrityError(
2139
- "native_snapshot_reference_invalid",
2140
- "Legacy Modal checkpoint restore produced no authenticated provider identity"
2141
- );
2142
- }
2143
- const adopted = await adoptLegacyModalCheckpointArtifact(db, {
2144
- accountId: ids.accountId,
2145
- workspaceId: ids.workspaceId,
2146
- sandboxGroupId: ids.sandboxGroupId,
2147
- leaseId: acquired.lease.id,
2148
- leaseEpoch: expectedEpoch,
2149
- workspaceGeneration: rematerialization.workspaceGeneration,
2150
- slot: "current",
2151
- archiveBase64: rematerialization.legacyCheckpoint.archiveBase64,
2152
- descriptor: rematerialization.legacyCheckpoint.descriptor,
2153
- providerBindingKey: binding.key,
2154
- providerBinding: binding.binding,
2155
- rematerializationId: rematerialization.id
2156
- });
2157
- if (!adopted) {
2158
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
2159
- }
2160
- rematerialization.providerBindingKey = binding.key;
2161
- }
2162
- const verifying = await markSandboxRestoreVerifying(db, {
2163
- accountId: ids.accountId,
2164
- workspaceId: ids.workspaceId,
2165
- sandboxGroupId: ids.sandboxGroupId,
2166
- expectedEpoch,
2167
- rematerializationId: rematerialization.id
2168
- });
2169
- if (!verifying.wrote) {
2170
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
2171
- }
2172
- }
2173
- });
2174
- createdEstablished = established;
2175
- throwIfReleasedOrCancelled();
2176
- await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
2177
- sandboxGroupId: ids.sandboxGroupId
2178
- });
2179
- throwIfReleasedOrCancelled();
2180
- const resumeEnvelope = requirePersistableReplacementSandboxEnvelope(
2181
- await serializeReplacementSandboxEnvelope(established, spawnEnvelope),
2182
- established.backendId
2183
- );
2184
- throwIfReleasedOrCancelled();
2185
- if (rematerialization && !established.providerContinuity && established.restoredArchive?.revision !== rematerialization.selectedRevision) {
2186
- throw new WorkspaceArchiveIntegrityError(
2187
- "workspace_fingerprint_mismatch",
2188
- "sandbox restore completed without the exact selected durable archive revision"
2189
- );
2190
- }
2191
- const committed = await commitWarmingToWarm(db, {
2192
- accountId: ids.accountId,
2193
- workspaceId: ids.workspaceId,
2194
- sandboxGroupId: ids.sandboxGroupId,
2195
- expectedEpoch,
2196
- instanceId: established.instanceId,
2197
- // Viewer attach resolves and records the tunnel URL lazily. A headless
2198
- // turn must not touch the optional desktop data plane.
2199
- dataPlaneUrl: null,
2200
- resumeBackendId: established.backendId,
2201
- resumeState: resumeEnvelope,
2202
- ...established.providerContinuity ? { continuityRecovery: established.providerContinuity } : rematerialization ? {
2203
- rematerialization: {
2204
- id: rematerialization.id,
2205
- verifiedRevision: rematerialization.selectedRevision
2206
- }
2207
- } : {},
2208
- leaseTtlMs
2209
- });
2210
- if (!committed.committed || !committed.lease) {
2211
- const terminated = await terminateEstablishedSandbox(established);
2212
- if (terminated && rematerialization && !established.providerContinuity) {
2213
- await failSandboxRematerialization(db, {
2214
- accountId: ids.accountId,
2215
- workspaceId: ids.workspaceId,
2216
- sandboxGroupId: ids.sandboxGroupId,
2217
- expectedEpoch,
2218
- rematerializationId: rematerialization.id,
2219
- failureCode: committed.reason ?? "warm_commit_rejected",
2220
- retryable: false
2221
- });
2222
- } else if (terminated) {
2223
- await failWarmingToCold(db, {
2224
- accountId: ids.accountId,
2225
- workspaceId: ids.workspaceId,
2226
- sandboxGroupId: ids.sandboxGroupId,
2227
- expectedEpoch
2228
- });
2229
- }
2230
- await release();
2231
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
2232
- }
2233
- holderLeaseHeartbeat = {
2234
- expectedEpoch: committed.lease.leaseEpoch,
2235
- leaseTtlMs
2236
- };
2237
- throwIfReleasedOrCancelled();
2238
- return { established, leaseEpoch: committed.lease.leaseEpoch, release };
2239
- } catch (error) {
2240
- if (error instanceof SandboxLeaseSupersededError) {
2241
- await terminateEstablishedSandbox(createdEstablished);
2242
- await release();
2243
- throw error;
2244
- }
2245
- const terminated = await terminateEstablishedSandbox(createdEstablished);
2246
- if (terminated) {
2247
- const continuityUnavailable = error instanceof SandboxProviderContinuityUnavailableError;
2248
- if (rematerialization && !createdEstablished?.providerContinuity) {
2249
- await failSandboxRematerialization(db, {
2250
- accountId: ids.accountId,
2251
- workspaceId: ids.workspaceId,
2252
- sandboxGroupId: ids.sandboxGroupId,
2253
- expectedEpoch,
2254
- rematerializationId: rematerialization.id,
2255
- failureCode: continuityUnavailable ? "provider_continuity_unavailable" : error instanceof WorkspaceArchiveIntegrityError ? error.code : "sandbox_rematerialization_failed",
2256
- retryable: continuityUnavailable ? error.retryable : error instanceof WorkspaceArchiveIntegrityError ? error.retryable : true,
2257
- ...continuityUnavailable ? { discardContinuity: true } : {}
2258
- });
2259
- } else {
2260
- await failWarmingToCold(db, {
2261
- accountId: ids.accountId,
2262
- workspaceId: ids.workspaceId,
2263
- sandboxGroupId: ids.sandboxGroupId,
2264
- expectedEpoch,
2265
- ...continuityUnavailable ? { discardContinuity: true } : {}
2266
- });
2267
- }
2268
- }
2269
- await release();
2270
- recordSandboxWarmingTimeout(services.sandboxMetrics, error);
2271
- throw error;
2272
- }
2273
- }
2274
- try {
2275
- let leaseEpoch = acquired.lease.leaseEpoch;
2276
- if (acquired.lease.liveness === "warming") {
2277
- leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
2278
- holderLeaseHeartbeat = { expectedEpoch: leaseEpoch, leaseTtlMs };
2279
- }
2280
- throwIfReleasedOrCancelled();
2281
- const live = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
2282
- if (!live || live.liveness !== "warm" || live.leaseEpoch !== leaseEpoch || live.instanceId === null) {
2283
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, live?.leaseEpoch ?? leaseEpoch);
2284
- }
2285
- let established;
2286
- try {
2287
- const establish = services.establishAttachedSandbox ?? establishSandboxSessionFromEnvelope;
2288
- established = await establish(settings, live.resumeState, {
2289
- sessionId: ids.sessionId,
2290
- recovery: "resume-only",
2291
- backendOverride: ids.backend,
2292
- ...ids.environment ? { environment: ids.environment } : {},
2293
- ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
2294
- });
2295
- throwIfReleasedOrCancelled();
2296
- if (services.verifyAttachedSandboxReadiness) {
2297
- await services.verifyAttachedSandboxReadiness(established);
2298
- } else {
2299
- await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
2300
- sandboxGroupId: ids.sandboxGroupId
2301
- });
2302
- }
2303
- throwIfReleasedOrCancelled();
2304
- } catch (error) {
2305
- if (!isProviderSandboxNotFoundError(ids.backend, error)) {
2306
- throw error;
2307
- }
2308
- const marked = await markWarmLeaseInstanceLost(db, {
2309
- accountId: ids.accountId,
2310
- workspaceId: ids.workspaceId,
2311
- sandboxGroupId: ids.sandboxGroupId,
2312
- expectedEpoch: leaseEpoch,
2313
- expectedInstanceId: live.instanceId
2314
- });
2315
- if (marked.status === "marked") {
2316
- await services.onSandboxLost?.({
2317
- sandboxGroupId: ids.sandboxGroupId,
2318
- instanceId: live.instanceId,
2319
- leaseEpoch: marked.lease.leaseEpoch
2320
- });
2321
- throw new SandboxLeaseInstanceLostError(
2322
- ids.sandboxGroupId,
2323
- marked.lease.leaseEpoch,
2324
- live.instanceId
2325
- );
2326
- }
2327
- throw new SandboxLeaseSupersededError(
2328
- ids.sandboxGroupId,
2329
- marked.lease?.leaseEpoch ?? leaseEpoch
2330
- );
2331
- }
2332
- throwIfReleasedOrCancelled();
2333
- return { established, leaseEpoch, release };
2334
- } catch (error) {
2335
- await release();
2336
- recordSandboxWarmingTimeout(services.sandboxMetrics, error);
2337
- throw error;
2338
- }
2339
- }
2340
- async function waitForWarm(services, ids) {
2341
- const { db, settings } = services;
2342
- const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
2343
- let instanceId = null;
2344
- while (Date.now() < deadline) {
2345
- await sleep(WARMING_POLL_INTERVAL_MS);
2346
- const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
2347
- if (!lease) {
2348
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
2349
- }
2350
- instanceId = lease.instanceId;
2351
- if (lease.liveness === "warm") {
2352
- return { leaseEpoch: lease.leaseEpoch };
2353
- }
2354
- if (lease.liveness === "draining") {
2355
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
2356
- }
2357
- if (lease.liveness === "cold") {
2358
- if (isRetryableDegradedRestore(lease.recovery.restore)) {
2359
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
2360
- }
2361
- if (lease.recovery.restore.status === "degraded" || lease.recovery.restore.status === "unrecoverable") {
2362
- throw new SandboxLeaseRecoveryBlockedError(
2363
- ids.sandboxGroupId,
2364
- lease.leaseEpoch,
2365
- lease.recovery.restore.status === "degraded" ? "restore_degraded" : "restore_unrecoverable",
2366
- lease.recovery
2367
- );
2368
- }
2369
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
2370
- }
2371
- }
2372
- throw new SandboxSiblingWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs, {
2373
- sandboxGroupId: ids.sandboxGroupId,
2374
- instanceId
2375
- });
2376
- }
2377
-
2378
- // src/sandbox-routing.ts
2379
- import { sandboxLifecycleTransitionWaitMs as sandboxLifecycleTransitionWaitMs2 } from "@opengeni/config";
2380
- import {
2381
- advanceWorkspaceGenerationForRetainedProcess,
2382
- advanceWorkspaceGeneration,
2383
- getRetainedProcess,
2384
- retainWorkspaceMutationProcess,
2385
- retainedProcessSettlementIdentity,
2386
- settleRetainedProcess,
2387
- verifyRetainedProcessMutationSettlement,
2388
- verifyWorkspaceMutationSettlement,
2389
- getSandbox,
2390
- getEnrollment,
2391
- markWarmLeaseInstanceLost as markWarmLeaseInstanceLost2,
2392
- readLease as readLease2,
2393
- readActiveSandbox,
2394
- SandboxRetainedProcessPromotionFencedError
2395
- } from "@opengeni/db";
2396
- import {
2397
- buildSelfhostedBackendSession,
2398
- establishSandboxSessionFromEnvelope as establishSandboxSessionFromEnvelope2,
2399
- isProviderSandboxGoneDuringRoutedOperation,
2400
- isProviderSandboxNotFoundError as isProviderSandboxNotFoundError2,
2401
- makeActiveBackendResolver,
2402
- NatsControlRpc,
2403
- NatsOpStreamTransport,
2404
- RoutingBackendRecoveryRequiredError,
2405
- RoutingSandboxSession,
2406
- resolveModalCheckpointProviderBindingForSession as resolveModalCheckpointProviderBindingForSession2,
2407
- sandboxProviderInstanceIdFromEnvelope,
2408
- sandboxBackendForSdkBackendId,
2409
- selectBackend,
2410
- verifySandboxExecReadiness as verifySandboxExecReadiness2
2411
- } from "@opengeni/runtime";
2412
- function relayConfigFromSettings(settings) {
2413
- const raw = settings.selfhostedRelayUrl?.trim();
2414
- if (!raw) {
2415
- return { host: "relay.opengeni.local", port: 443, tls: true };
2416
- }
2417
- try {
2418
- const url = new URL(raw.includes("://") ? raw : `wss://${raw}`);
2419
- const tls = url.protocol === "wss:" || url.protocol === "https:";
2420
- const port = url.port ? Number(url.port) : tls ? 443 : 80;
2421
- return { host: url.hostname, port, tls };
2422
- } catch {
2423
- return { host: raw, port: 443, tls: true };
2424
- }
2425
- }
2426
- function selfhostedTimeoutsFromSettings(settings) {
2427
- return {
2428
- timeoutMs: settings.sandboxSelfhostedControlTimeoutMs,
2429
- execTimeoutMs: settings.sandboxSelfhostedExecTimeoutMs
2430
- };
2431
- }
2432
- function selfhostedResolverTimeouts(settings) {
2433
- const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
2434
- return {
2435
- selfhostedTimeoutMs: timeoutMs,
2436
- selfhostedExecTimeoutMs: execTimeoutMs
2437
- };
2438
- }
2439
- function homeRouteRecoveryDisposition(lease) {
2440
- if (!lease) return "unrecoverable";
2441
- if (lease.liveness === "warming") return "pending";
2442
- if (lease.liveness === "draining") return "superseded";
2443
- if (lease.recovery.restore.status === "pending" || lease.recovery.restore.status === "restoring" || lease.recovery.restore.status === "verifying") {
2444
- return "pending";
2445
- }
2446
- if (lease.recovery.restore.status === "degraded" || lease.recovery.workspace.status === "degraded" || lease.recovery.archive.status === "unverified" || lease.recovery.archive.status === "invalid") {
2447
- return "degraded";
2448
- }
2449
- return "unrecoverable";
2450
- }
2451
- function homeRouteRecoveryError(lease, fallbackEpoch) {
2452
- return new RoutingBackendRecoveryRequiredError(
2453
- "resolve_home_backend",
2454
- lease?.leaseEpoch ?? fallbackEpoch,
2455
- homeRouteRecoveryDisposition(lease)
2456
- );
2457
- }
2458
- function providerIdentityFromResumeState(resumeState) {
2459
- return sandboxProviderInstanceIdFromEnvelope(resumeState);
2460
- }
2461
- async function resolveCurrentHomeBackend(services, ids, established) {
2462
- const lease = await readLease2(services.db, ids.workspaceId, ids.sandboxGroupId);
2463
- const fallbackEpoch = lease?.leaseEpoch ?? 0;
2464
- if (!lease || lease.liveness !== "warm" || lease.recovery.provider.status !== "exists" || lease.instanceId === null || lease.recovery.provider.instanceId !== lease.instanceId || lease.recovery.workspace.status !== "ready" || lease.recovery.restore.status !== "not_required" && lease.recovery.restore.status !== "ready") {
2465
- throw homeRouteRecoveryError(lease, fallbackEpoch);
2466
- }
2467
- if (lease.instanceId === established.instanceId) {
2468
- services.onHomeSandboxRebound?.({
2469
- established,
2470
- leaseEpoch: lease.leaseEpoch
2471
- });
2472
- return {
2473
- session: established.session,
2474
- sandboxId: null,
2475
- kind: established.backendId,
2476
- leaseEpoch: lease.leaseEpoch,
2477
- providerInstanceId: lease.instanceId
2478
- };
2479
- }
2480
- const resumeState = lease.resumeState;
2481
- const durableResumeBackend = lease.resumeBackendId ?? lease.backend ?? ids.backend;
2482
- const resumeBackend = sandboxBackendForSdkBackendId(durableResumeBackend) ?? durableResumeBackend;
2483
- if (!resumeState || resumeBackend !== ids.backend || providerIdentityFromResumeState(resumeState) !== lease.instanceId) {
2484
- throw homeRouteRecoveryError(lease, lease.leaseEpoch);
2485
- }
2486
- let rebound;
2487
- try {
2488
- rebound = await establishSandboxSessionFromEnvelope2(services.settings, resumeState, {
2489
- sessionId: ids.sessionId,
2490
- recovery: "resume-only",
2491
- backendOverride: resumeBackend
2492
- });
2493
- await verifySandboxExecReadiness2(rebound);
2494
- } catch (error) {
2495
- if (!isProviderSandboxNotFoundError2(resumeBackend, error)) {
2496
- throw new RoutingBackendRecoveryRequiredError(
2497
- "resolve_home_backend",
2498
- lease.leaseEpoch,
2499
- "pending"
2500
- );
2501
- }
2502
- const marked = await markWarmLeaseInstanceLost2(services.db, {
2503
- accountId: ids.accountId,
2504
- workspaceId: ids.workspaceId,
2505
- sandboxGroupId: ids.sandboxGroupId,
2506
- expectedEpoch: lease.leaseEpoch,
2507
- expectedInstanceId: lease.instanceId,
2508
- diagnostic: "provider_not_found_during_home_route_rebind"
2509
- });
2510
- if (marked.status === "marked") {
2511
- await services.onHomeSandboxLost?.({
2512
- sandboxGroupId: ids.sandboxGroupId,
2513
- instanceId: lease.instanceId,
2514
- leaseEpoch: marked.lease.leaseEpoch
2515
- });
2516
- }
2517
- throw new RoutingBackendRecoveryRequiredError(
2518
- "resolve_home_backend",
2519
- marked.lease?.leaseEpoch ?? lease.leaseEpoch,
2520
- marked.status === "stale" ? "superseded" : homeRouteRecoveryDisposition(marked.lease)
2521
- );
2522
- }
2523
- const reboundBackend = sandboxBackendForSdkBackendId(rebound.backendId) ?? rebound.backendId;
2524
- if (rebound.instanceId !== lease.instanceId || reboundBackend !== resumeBackend) {
2525
- throw homeRouteRecoveryError(lease, lease.leaseEpoch);
2526
- }
2527
- services.onHomeSandboxRebound?.({
2528
- established: rebound,
2529
- leaseEpoch: lease.leaseEpoch
2530
- });
2531
- Object.assign(established, rebound);
2532
- return {
2533
- session: rebound.session,
2534
- sandboxId: null,
2535
- kind: rebound.backendId,
2536
- leaseEpoch: lease.leaseEpoch,
2537
- providerInstanceId: lease.instanceId
2538
- };
2539
- }
2540
- function beforePersistableHomeMutation(services, ids, home) {
2541
- const fence = ids.workspaceMutationFence;
2542
- if (!home || !fence) return void 0;
2543
- return async ({ op, backend }) => {
2544
- if (backend.sandboxId !== null || backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0) {
2545
- return null;
2546
- }
2547
- const providerBinding = home.backend === "modal" ? await resolveModalCheckpointProviderBindingForSession2(services.settings, backend.session) : null;
2548
- const admission = await advanceWorkspaceGeneration(services.db, {
2549
- accountId: fence.accountId,
2550
- workspaceId: ids.workspaceId,
2551
- sessionId: ids.sessionId,
2552
- turnId: fence.turnId,
2553
- executionGeneration: fence.executionGeneration,
2554
- attemptId: fence.attemptId,
2555
- holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
2556
- sandboxGroupId: home.sandboxGroupId,
2557
- expectedEpoch: backend.leaseEpoch,
2558
- expectedInstanceId: backend.providerInstanceId,
2559
- operation: op,
2560
- captureWaitMs: sandboxLifecycleTransitionWaitMs2(services.settings),
2561
- ...services.waitSignal ? { waitSignal: services.waitSignal } : {}
2562
- });
2563
- return { admission, providerBinding };
2564
- };
2565
- }
2566
- function afterPersistableHomeMutation(services, ids, home) {
2567
- const fence = ids.workspaceMutationFence;
2568
- if (!home || !fence) return void 0;
2569
- return async ({ op, backend, admission, outcome, retainedProcess }) => {
2570
- if (backend.sandboxId !== null || backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0 || !admission || typeof admission !== "object") {
2571
- return;
2572
- }
2573
- const boundAdmission = admission;
2574
- const exactAdmission = boundAdmission.admission;
2575
- if (!exactAdmission || typeof exactAdmission.id !== "string" || typeof exactAdmission.workspaceGeneration !== "number" || !("providerBinding" in boundAdmission)) {
2576
- throw new Error("Persistable home mutation settlement lacked its bound admission");
2577
- }
2578
- if (outcome === "resolved" && retainedProcess) {
2579
- try {
2580
- await retainWorkspaceMutationProcess(services.db, {
2581
- accountId: fence.accountId,
2582
- workspaceId: ids.workspaceId,
2583
- sessionId: ids.sessionId,
2584
- processId: retainedProcess.id,
2585
- providerSessionId: retainedProcess.providerSessionId,
2586
- admissionId: exactAdmission.id,
2587
- admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
2588
- operation: op,
2589
- providerBinding: boundAdmission.providerBinding ?? null,
2590
- owner: {
2591
- kind: "turn",
2592
- turnId: fence.turnId,
2593
- executionGeneration: fence.executionGeneration,
2594
- attemptId: fence.attemptId,
2595
- holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
2596
- sandboxGroupId: home.sandboxGroupId,
2597
- expectedEpoch: backend.leaseEpoch,
2598
- expectedInstanceId: backend.providerInstanceId,
2599
- routeKind: exactAdmission.routeKind,
2600
- routeTargetId: exactAdmission.routeTargetId,
2601
- routeEpoch: exactAdmission.routeEpoch
2602
- }
2603
- });
2604
- } catch (error) {
2605
- if (!(error instanceof SandboxRetainedProcessPromotionFencedError)) throw error;
2606
- const durable = error.process;
2607
- if (durable.id !== retainedProcess.id || durable.providerSessionId !== retainedProcess.providerSessionId || durable.sessionId !== ids.sessionId || durable.sandboxGroupId !== home.sandboxGroupId || durable.parentAdmissionId !== exactAdmission.id || durable.leaseEpoch !== backend.leaseEpoch || durable.providerInstanceId !== backend.providerInstanceId || durable.routeKind !== exactAdmission.routeKind || durable.routeTargetId !== exactAdmission.routeTargetId || durable.routeEpoch !== exactAdmission.routeEpoch || durable.state !== "active") {
2608
- throw new Error("Durable retained-process promotion returned a mismatched identity", {
2609
- cause: error
2610
- });
2611
- }
2612
- return {
2613
- status: "retained_process_durable_output_rejected",
2614
- retainedProcess: {
2615
- id: durable.id,
2616
- providerSessionId: durable.providerSessionId
2617
- }
2618
- };
2619
- }
2620
- return;
2621
- }
2622
- await verifyWorkspaceMutationSettlement(services.db, {
2623
- accountId: fence.accountId,
2624
- workspaceId: ids.workspaceId,
2625
- sessionId: ids.sessionId,
2626
- turnId: fence.turnId,
2627
- executionGeneration: fence.executionGeneration,
2628
- attemptId: fence.attemptId,
2629
- holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
2630
- sandboxGroupId: home.sandboxGroupId,
2631
- expectedEpoch: backend.leaseEpoch,
2632
- expectedInstanceId: backend.providerInstanceId,
2633
- admission: exactAdmission,
2634
- operation: op,
2635
- outcome
2636
- });
2637
- };
2638
- }
2639
- function beforeRetainedProcessMutation(services, ids) {
2640
- const fence = ids.workspaceMutationFence;
2641
- if (!fence) return void 0;
2642
- return async ({ op, process }) => await advanceWorkspaceGenerationForRetainedProcess(services.db, {
2643
- accountId: fence.accountId,
2644
- workspaceId: ids.workspaceId,
2645
- sessionId: ids.sessionId,
2646
- processId: process.id,
2647
- operation: op,
2648
- captureWaitMs: sandboxLifecycleTransitionWaitMs2(services.settings),
2649
- ...services.waitSignal ? { waitSignal: services.waitSignal } : {}
2650
- });
2651
- }
2652
- function afterRetainedProcessMutation(services, ids) {
2653
- const fence = ids.workspaceMutationFence;
2654
- if (!fence) return void 0;
2655
- return async ({ op, process, admission, outcome }) => {
2656
- if (!admission || typeof admission !== "object" || typeof admission.id !== "string" || typeof admission.workspaceGeneration !== "number") {
2657
- throw new Error("Retained-process mutation settlement lacked its exact admission");
2658
- }
2659
- await verifyRetainedProcessMutationSettlement(services.db, {
2660
- accountId: fence.accountId,
2661
- workspaceId: ids.workspaceId,
2662
- sessionId: ids.sessionId,
2663
- processId: process.id,
2664
- admission,
2665
- operation: op,
2666
- outcome
2667
- });
2668
- };
2669
- }
2670
- function settleRetainedProcessForTurn(services, ids) {
2671
- const fence = ids.workspaceMutationFence;
2672
- if (!fence) return void 0;
2673
- return async ({ backend, process, proof }) => {
2674
- const durable = await getRetainedProcess(services.db, {
2675
- workspaceId: ids.workspaceId,
2676
- sessionId: ids.sessionId,
2677
- processId: process.id
2678
- });
2679
- if (!durable || durable.providerSessionId !== process.providerSessionId || durable.providerBackend !== backend.kind || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch) {
2680
- throw new Error("Retained-process settlement lost its exact durable backend identity");
2681
- }
2682
- await settleRetainedProcess(services.db, {
2683
- accountId: fence.accountId,
2684
- workspaceId: ids.workspaceId,
2685
- sessionId: ids.sessionId,
2686
- processId: process.id,
2687
- expected: retainedProcessSettlementIdentity(durable),
2688
- outcome: proof.outcome,
2689
- exitCode: proof.exitCode,
2690
- reason: proof.reason,
2691
- idleGraceMs: services.settings.sandboxIdleGraceMs
2692
- });
2693
- };
2694
- }
2695
- function controlRpcFactory(bus) {
2696
- return () => new NatsControlRpc(async () => {
2697
- if (!bus) {
2698
- return null;
2699
- }
2700
- return bus.getRequestConnection();
2701
- });
2702
- }
2703
- function wrapTurnBoxWithRouting(services, ids, established) {
2704
- const { db, settings, bus, onOp } = services;
2705
- const beforeMutation = beforePersistableHomeMutation(services, ids, ids.homeLease);
2706
- const afterMutation = afterPersistableHomeMutation(services, ids, ids.homeLease);
2707
- const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
2708
- const afterProcessMutation = afterRetainedProcessMutation(services, ids);
2709
- const settleProcess = settleRetainedProcessForTurn(services, ids);
2710
- const resolver = makeActiveBackendResolver({
2711
- workspaceId: ids.workspaceId,
2712
- defaultBackend: established.session,
2713
- defaultKind: established.backendId,
2714
- getSandbox: async (sandboxId) => {
2715
- const sandbox = await getSandbox(db, ids.workspaceId, sandboxId);
2716
- return sandbox ? {
2717
- id: sandbox.id,
2718
- kind: sandbox.kind,
2719
- name: sandbox.name,
2720
- enrollmentId: sandbox.enrollmentId
2721
- } : null;
2722
- },
2723
- controlRpcFactory: controlRpcFactory(bus),
2724
- resolveSelfhostedOpStream: async (sandbox) => {
2725
- if (!sandbox.enrollmentId) return void 0;
2726
- const enrollment = await getEnrollment(db, ids.workspaceId, sandbox.enrollmentId);
2727
- return opStreamDepsFor(services, enrollment?.opStream === true);
2728
- },
2729
- relay: relayConfigFromSettings(settings),
2730
- // A selfhosted swap target runs real commands too, so give it the same split
2731
- // deadlines the machine-primary establish path uses (short control, long exec).
2732
- ...selfhostedResolverTimeouts(settings),
2733
- ...onOp !== void 0 ? { selfhostedOnOp: onOp } : {},
2734
- // The turn's declared environment → a selfhosted swap target's manifest, so the
2735
- // SDK's per-turn manifest-env delta is empty (no "cannot change manifest
2736
- // environment variables" throw when the turn pins to a vm). Mirrors the group
2737
- // box, which is created WITH this same environment (resumeBoxForTurn).
2738
- ...ids.environment !== void 0 ? { environment: ids.environment } : {},
2739
- ...ids.transientExecEnvironment !== void 0 ? { transientExecEnvironment: ids.transientExecEnvironment } : {},
2740
- // Stage D machine-primary: pin THIS established SelfhostedSession for the machine
2741
- // pointer so the resolver returns the SAME instance (no two-instance manifest
2742
- // divergence). `established.session` is the SelfhostedSession the establish branch
2743
- // bound to (sandboxId, epoch).
2744
- ...ids.pinnedSelfhosted ? {
2745
- pinnedSelfhosted: {
2746
- sandboxId: ids.pinnedSelfhosted.sandboxId,
2747
- epoch: ids.pinnedSelfhosted.epoch,
2748
- session: established.session
2749
- }
2750
- } : {},
2751
- // A modal swap target in the turn path would need its own lease resume-by-id;
2752
- // that is a future cross-group-box concern. Until then a modal swap target is
2753
- // unresolvable (the swap tool validates liveness, so this only triggers if a
2754
- // session points at a sibling modal box the turn cannot resume here) and the
2755
- // op surfaces unresolvable — never a silent wrong-box landing.
2756
- //
2757
- // For a machine-primary turn of a Modal-HOME session (pinned to a machine, no
2758
- // group box established this turn), a mid-turn clear-to-null must NOT fall back to
2759
- // the pinned machine — passing defaultIsHome:false makes the null branch throw typed
2760
- // `home_unavailable_this_turn` instead. Forward the explicit boolean (including false).
2761
- ...ids.defaultIsHome !== void 0 ? { defaultIsHome: ids.defaultIsHome } : {},
2762
- ...ids.homeLease ? {
2763
- resolveDefaultBackend: () => resolveCurrentHomeBackend(
2764
- services,
2765
- {
2766
- workspaceId: ids.workspaceId,
2767
- sessionId: ids.sessionId,
2768
- accountId: ids.homeLease.accountId,
2769
- sandboxGroupId: ids.homeLease.sandboxGroupId,
2770
- backend: ids.homeLease.backend
2771
- },
2772
- established
2773
- )
2774
- } : {}
2775
- });
2776
- const proxy = new RoutingSandboxSession({
2777
- // Seed the DEFAULT backend (the established group box) at construction so
2778
- // `session.state` is the real backend's state object BEFORE the first op. The
2779
- // SDK reads `session.state.manifest` at turn START (and writes it back); an
2780
- // empty `{}` there crashes serializeManifestEnvironment /
2781
- // validateProvidedSessionManifestUpdate. This is byte-identical to what the
2782
- // resolver returns for the default pointer (`activeSandboxId === null`).
2783
- defaultResolved: {
2784
- session: established.session,
2785
- sandboxId: null,
2786
- kind: established.backendId,
2787
- ...ids.homeLease ? {
2788
- leaseEpoch: ids.homeLease.leaseEpoch,
2789
- providerInstanceId: ids.homeLease.instanceId
2790
- } : {}
2791
- },
2792
- readPointer: async () => {
2793
- if (!routingEnabled(settings)) {
2794
- return { activeSandboxId: null, activeEpoch: 0 };
2795
- }
2796
- const pointer = await readActiveSandbox(db, ids.workspaceId, ids.sessionId);
2797
- return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
2798
- },
2799
- resolveActiveBackend: resolver,
2800
- ...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
2801
- ...beforeMutation ? { beforeMutation } : {},
2802
- ...afterMutation ? { afterMutation } : {},
2803
- ...beforeProcessMutation ? { beforeProcessMutation } : {},
2804
- ...afterProcessMutation ? { afterProcessMutation } : {},
2805
- ...settleProcess ? { settleProcess } : {},
2806
- ...ids.homeLease ? {
2807
- onDefaultBackendError: async ({
2808
- error,
2809
- backend
2810
- }) => {
2811
- const home = ids.homeLease;
2812
- if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
2813
- const expectedEpoch = backend.leaseEpoch ?? home.leaseEpoch;
2814
- const expectedInstanceId = backend.providerInstanceId ?? home.instanceId;
2815
- const marked = await markWarmLeaseInstanceLost2(db, {
2816
- accountId: home.accountId,
2817
- workspaceId: ids.workspaceId,
2818
- sandboxGroupId: home.sandboxGroupId,
2819
- expectedEpoch,
2820
- expectedInstanceId,
2821
- diagnostic: "provider_not_found_during_routed_operation"
2822
- });
2823
- if (marked.status === "marked") {
2824
- await services.onHomeSandboxLost?.({
2825
- sandboxGroupId: home.sandboxGroupId,
2826
- instanceId: expectedInstanceId,
2827
- leaseEpoch: marked.lease.leaseEpoch
2828
- });
2829
- }
2830
- const lease = marked.lease;
2831
- const restore = lease?.recovery.restore.status;
2832
- return {
2833
- leaseEpoch: lease?.leaseEpoch ?? expectedEpoch,
2834
- recovery: marked.status === "stale" ? "superseded" : restore === "pending" ? "pending" : restore === "degraded" ? "degraded" : "unrecoverable"
2835
- };
2836
- }
2837
- } : {}
2838
- });
2839
- return { ...established, session: proxy };
2840
- }
2841
- function wrapLazyTurnBoxWithRouting(services, ids, args) {
2842
- const { db, settings, bus, onOp } = services;
2843
- const beforeMutation = beforePersistableHomeMutation(services, ids, args.homeLeaseIdentity);
2844
- const afterMutation = afterPersistableHomeMutation(services, ids, args.homeLeaseIdentity);
2845
- const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
2846
- const afterProcessMutation = afterRetainedProcessMutation(services, ids);
2847
- const settleProcess = settleRetainedProcessForTurn(services, ids);
2848
- const defaultBackend = sandboxBackendForSdkBackendId(args.backendId);
2849
- const defaultSupportsPty = defaultBackend ? selectBackend(defaultBackend).capabilities.Terminal.pty : false;
2850
- const syntheticSession = {
2851
- state: { manifest: args.agentDefaultManifest },
2852
- // The SDK binds shell tools synchronously before lazy provisioning. Preserve
2853
- // the authoritative descriptor's PTY shape so a Modal home exposes
2854
- // write_stdin on its first turn operation; the async cancellation transport
2855
- // below still resolves the exact live route before executing a command.
2856
- supportsPty: () => defaultSupportsPty
2857
- };
2858
- const routedResolver = makeActiveBackendResolver({
2859
- workspaceId: ids.workspaceId,
2860
- defaultBackend: syntheticSession,
2861
- defaultKind: "unprovisioned",
2862
- getSandbox: async (sandboxId) => {
2863
- const sandbox = await getSandbox(db, ids.workspaceId, sandboxId);
2864
- return sandbox ? {
2865
- id: sandbox.id,
2866
- kind: sandbox.kind,
2867
- name: sandbox.name,
2868
- enrollmentId: sandbox.enrollmentId
2869
- } : null;
2870
- },
2871
- controlRpcFactory: controlRpcFactory(bus),
2872
- resolveSelfhostedOpStream: async (sandbox) => {
2873
- if (!sandbox.enrollmentId) return void 0;
2874
- const enrollment = await getEnrollment(db, ids.workspaceId, sandbox.enrollmentId);
2875
- return opStreamDepsFor(services, enrollment?.opStream === true);
2876
- },
2877
- relay: relayConfigFromSettings(settings),
2878
- ...selfhostedResolverTimeouts(settings),
2879
- ...onOp !== void 0 ? { selfhostedOnOp: onOp } : {},
2880
- ...ids.environment !== void 0 ? { environment: ids.environment } : {},
2881
- ...ids.transientExecEnvironment !== void 0 ? { transientExecEnvironment: ids.transientExecEnvironment } : {}
2882
- });
2883
- const proxy = new RoutingSandboxSession({
2884
- // Before the first op the SDK reads `state.manifest`; the synthetic backend
2885
- // points at agent.defaultManifest BY REFERENCE so the provided-session delta is
2886
- // empty. The first default-pointer op resolves the real box through the
2887
- // provisioner and `state` switches to that real backend by reference.
2888
- defaultResolved: {
2889
- session: syntheticSession,
2890
- sandboxId: null,
2891
- kind: "unprovisioned"
2892
- },
2893
- readPointer: async () => {
2894
- const pointer = await readActiveSandbox(db, ids.workspaceId, ids.sessionId);
2895
- return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
2896
- },
2897
- resolveActiveBackend: async (pointer) => {
2898
- if (pointer.activeSandboxId === null || !routingEnabled(settings)) {
2899
- const provisioned = await args.provisioner.get();
2900
- if (args.homeLeaseIdentity && provisioned.leaseEpoch !== void 0) {
2901
- return resolveCurrentHomeBackend(
2902
- services,
2903
- {
2904
- workspaceId: ids.workspaceId,
2905
- sessionId: ids.sessionId,
2906
- accountId: args.homeLeaseIdentity.accountId,
2907
- sandboxGroupId: args.homeLeaseIdentity.sandboxGroupId,
2908
- backend: args.homeLeaseIdentity.backend
2909
- },
2910
- provisioned.established
2911
- );
2912
- }
2913
- return {
2914
- session: provisioned.established.session,
2915
- sandboxId: null,
2916
- kind: provisioned.established.backendId
2917
- };
2918
- }
2919
- return routedResolver(pointer);
2920
- },
2921
- ...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
2922
- ...beforeMutation ? { beforeMutation } : {},
2923
- ...afterMutation ? { afterMutation } : {},
2924
- ...beforeProcessMutation ? { beforeProcessMutation } : {},
2925
- ...afterProcessMutation ? { afterProcessMutation } : {},
2926
- ...settleProcess ? { settleProcess } : {},
2927
- ...args.homeLeaseIdentity ? {
2928
- onDefaultBackendError: async ({
2929
- error,
2930
- backend
2931
- }) => {
2932
- const home = args.homeLeaseIdentity;
2933
- if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
2934
- if (backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0) {
2935
- return null;
2936
- }
2937
- const marked = await markWarmLeaseInstanceLost2(db, {
2938
- accountId: home.accountId,
2939
- workspaceId: ids.workspaceId,
2940
- sandboxGroupId: home.sandboxGroupId,
2941
- expectedEpoch: backend.leaseEpoch,
2942
- expectedInstanceId: backend.providerInstanceId,
2943
- diagnostic: "provider_not_found_during_routed_operation"
2944
- });
2945
- if (marked.status === "marked") {
2946
- await services.onHomeSandboxLost?.({
2947
- sandboxGroupId: home.sandboxGroupId,
2948
- instanceId: backend.providerInstanceId,
2949
- leaseEpoch: marked.lease.leaseEpoch
2950
- });
2951
- }
2952
- const lease = marked.lease;
2953
- const restore = lease?.recovery.restore.status;
2954
- return {
2955
- leaseEpoch: lease?.leaseEpoch ?? backend.leaseEpoch,
2956
- recovery: marked.status === "stale" ? "superseded" : restore === "pending" ? "pending" : restore === "degraded" ? "degraded" : "unrecoverable"
2957
- };
2958
- }
2959
- } : {}
2960
- });
2961
- return {
2962
- client: args.client,
2963
- session: proxy,
2964
- sessionState: void 0,
2965
- instanceId: "unprovisioned",
2966
- backendId: args.backendId
2967
- };
2968
- }
2969
- function opStreamDepsFor(services, machineAdvertisesOpStream) {
2970
- const { settings, bus, opJournal } = services;
2971
- if (!machineAdvertisesOpStream || settings.agentOpStreamEnabled !== true || !bus) {
2972
- return void 0;
2973
- }
2974
- return {
2975
- transport: new NatsOpStreamTransport(async () => bus.getOpStreamConnection?.() ?? null),
2976
- ...opJournal !== void 0 ? { journal: opJournal } : {}
2977
- };
2978
- }
2979
- async function establishSelfhostedTurnSession(services, args) {
2980
- const { settings, bus, onOp } = services;
2981
- const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
2982
- const opStream = opStreamDepsFor(services, args.opStream);
2983
- const { client, session } = await buildSelfhostedBackendSession({
2984
- workspaceId: args.workspaceId,
2985
- agentId: args.agentId,
2986
- relay: relayConfigFromSettings(settings),
2987
- controlRpcFactory: controlRpcFactory(bus),
2988
- epoch: args.epoch,
2989
- environment: args.environment,
2990
- ...args.transientExecEnvironment !== void 0 ? { transientExecEnvironment: args.transientExecEnvironment } : {},
2991
- workingDir: args.workingDir,
2992
- // Give this turn's exec ops the long deadline (control ops stay short) so a real
2993
- // command is not killed at the control wall.
2994
- timeoutMs,
2995
- execTimeoutMs,
2996
- // Meter every control op (out-of-band telemetry) — no-op when unwired.
2997
- ...onOp !== void 0 ? { onOp } : {},
2998
- // The streaming exec transport — present iff the machine advertised the
2999
- // capability AND the server flag is on. It is required when execTimeoutMs=0;
3000
- // legacy request/reply remains available only for explicitly bounded exec.
3001
- ...opStream !== void 0 ? { opStream } : {}
3002
- });
3003
- return {
3004
- client,
3005
- session,
3006
- sessionState: { agentId: args.agentId },
3007
- instanceId: args.agentId,
3008
- backendId: "selfhosted"
3009
- };
3010
- }
3011
- function routingEnabled(settings) {
3012
- return settings.sandboxSelfhostedEnabled === true;
3013
- }
3014
- function lazyProvisionEnabled(settings) {
3015
- return settings.sandboxLazyProvisionEnabled === true && settings.sandboxOwnershipEnabled === true;
3016
- }
3017
-
3018
- // src/activities/common.ts
3019
- import {
3020
- mergeResourceRefs,
3021
- mergeToolRefs as mergeToolRefs2,
3022
- reasoningEffortForMetadata as reasoningEffortForMetadata2
3023
- } from "@opengeni/contracts";
3024
- function assertReusableSessionRevivable(status) {
3025
- if (status === "cancelled") {
3026
- throw new Error("reusable session is cancelled; refusing to revive on scheduled fire");
3027
- }
3028
- }
3029
- function scheduledUserMessagePayload(prompt, resources, tools, taskId, runId) {
3030
- return {
3031
- type: "scheduled_occurrence",
3032
- text: prompt,
3033
- scheduledTaskId: taskId,
3034
- scheduledTaskRunId: runId,
3035
- ...resources.length ? { resources } : {},
3036
- ...tools.length ? { tools } : {}
3037
- };
3038
- }
3039
- function workflowIdForSession(sessionId) {
3040
- return `session-${sessionId}`;
3041
- }
3042
-
3043
- export {
3044
- createSharedActivityServices,
3045
- classifyCodexPin,
3046
- CODEX_USAGE_EXHAUSTED_PCT,
3047
- isCodexCredentialEligible,
3048
- shardCredentialForSession,
3049
- earliestCodexReset,
3050
- authoritativeCodexCapacityResetAt,
3051
- computeIdleDelayMs,
3052
- computeReactiveRotationResume,
3053
- chooseRotationActive,
3054
- selectCodexCredentialLeaseForTurn,
3055
- refreshCodexUsageAndRepairCapacityWaiters,
3056
- signalCodexCapacityWakeTargets,
3057
- signalPendingCodexCapacityWakeTargets,
3058
- createCodexCapacityActivities,
3059
- createGoalActivities,
3060
- withFirstPartyTools,
3061
- settingsWithPackSandboxImage,
3062
- settingsWithRigImage,
3063
- rigProviderImageSourceImage,
3064
- settingsWithRigProviderImage,
3065
- resolveWorkspacePackRuntime,
3066
- resolveWorkspaceInstalledSkillRuntime,
3067
- mergeRigDefaultVariableSetEnvironment,
3068
- createRuntimeBatcher,
3069
- currentActivityContext,
3070
- startActivityHeartbeat,
3071
- nextStreamEvent,
3072
- notifyParentOfChildIdle,
3073
- deliverFailedChildTurnToParent,
3074
- reconcilePendingParentSystemUpdates,
3075
- reconcilePendingSessionWorkflowWakes,
3076
- sandboxLeaseHolderIdForAttempt,
3077
- SandboxWarmingTimeoutError,
3078
- waitForWarmSnapshot,
3079
- maybePersistWarmWorkspaceSnapshot,
3080
- resumeBoxForTurn,
3081
- providerIdentityFromResumeState,
3082
- wrapTurnBoxWithRouting,
3083
- wrapLazyTurnBoxWithRouting,
3084
- establishSelfhostedTurnSession,
3085
- routingEnabled,
3086
- lazyProvisionEnabled,
3087
- assertReusableSessionRevivable,
3088
- scheduledUserMessagePayload,
3089
- workflowIdForSession,
3090
- mergeResourceRefs
3091
- };
3092
- //# sourceMappingURL=chunk-P6Z2SLAA.js.map