@opengeni/worker-bundle 0.20.3 → 0.22.1-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +52 -0
  2. package/dist/activities/agent-turn/agent-build.d.ts +89 -0
  3. package/dist/activities/agent-turn/claim.d.ts +67 -0
  4. package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
  5. package/dist/activities/agent-turn/codex.d.ts +32 -0
  6. package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
  7. package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
  8. package/dist/activities/agent-turn/errors.d.ts +253 -0
  9. package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
  10. package/dist/activities/agent-turn/file-resources.d.ts +37 -0
  11. package/dist/activities/agent-turn/finalization.d.ts +42 -0
  12. package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
  13. package/dist/activities/agent-turn/governance-model.d.ts +66 -0
  14. package/dist/activities/agent-turn/history-sink.d.ts +52 -0
  15. package/dist/activities/agent-turn/history.d.ts +63 -0
  16. package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
  17. package/dist/activities/agent-turn/model-usage.d.ts +218 -0
  18. package/dist/activities/agent-turn/quiescence.d.ts +135 -0
  19. package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
  20. package/dist/activities/agent-turn/run.d.ts +3 -0
  21. package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
  22. package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
  23. package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
  24. package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
  25. package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
  26. package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
  27. package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
  28. package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
  29. package/dist/activities/agent-turn/turn-context.d.ts +144 -0
  30. package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
  31. package/dist/activities/agent-turn.d.ts +12 -954
  32. package/dist/activities/automations.d.ts +16 -0
  33. package/dist/activities/common.d.ts +1 -1
  34. package/dist/activities/connector-attachments.d.ts +13 -0
  35. package/dist/activities/documents.d.ts +1 -0
  36. package/dist/activities/environment.d.ts +26 -15
  37. package/dist/activities/generated-images.d.ts +1 -1
  38. package/dist/activities/git-credential-renewal.d.ts +3 -1
  39. package/dist/activities/goals.d.ts +4 -1
  40. package/dist/activities/google-drive-provider.d.ts +29 -0
  41. package/dist/activities/google-drive-publication.d.ts +17 -4
  42. package/dist/activities/incident-telemetry-authority.d.ts +52 -0
  43. package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
  44. package/dist/activities/knowledge-source-sync.d.ts +4 -0
  45. package/dist/activities/mcp-credentials.d.ts +9 -3
  46. package/dist/activities/open-suffix-resume.d.ts +77 -0
  47. package/dist/activities/packs.d.ts +3 -11
  48. package/dist/activities/parent-wake.d.ts +19 -0
  49. package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
  50. package/dist/activities/retained-screenshots.d.ts +1 -1
  51. package/dist/activities/rig-verification.d.ts +1 -1
  52. package/dist/activities/run-input.d.ts +9 -10
  53. package/dist/activities/sandbox-images.d.ts +1 -14
  54. package/dist/activities/sandbox-lease.d.ts +21 -3
  55. package/dist/activities/scheduled-tasks.d.ts +47 -0
  56. package/dist/activities/types.d.ts +23 -2
  57. package/dist/activities/video-media-validation.d.ts +1 -1
  58. package/dist/activities/video-output-retention.d.ts +1 -2
  59. package/dist/activities/xai-auth.d.ts +3 -0
  60. package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
  61. package/dist/activities-control-DKG22ONJ.js.map +1 -0
  62. package/dist/activities-control.d.ts +6 -2
  63. package/dist/activities-turn-4IMC3MAA.js +21326 -0
  64. package/dist/activities-turn-4IMC3MAA.js.map +1 -0
  65. package/dist/activities.d.ts +7 -3
  66. package/dist/artifact-materializer-entry.js +41 -22
  67. package/dist/artifact-materializer-entry.js.map +1 -1
  68. package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
  69. package/dist/chunk-CE7TPKBS.js.map +1 -0
  70. package/dist/chunk-CNYGVRKF.js +4698 -0
  71. package/dist/chunk-CNYGVRKF.js.map +1 -0
  72. package/dist/connection-provider-use.d.ts +18 -0
  73. package/dist/editable-artifact-materializer.d.ts +4 -3
  74. package/dist/github-rest-mcp.d.ts +26 -0
  75. package/dist/index.d.ts +10 -2
  76. package/dist/index.js +58 -14
  77. package/dist/index.js.map +1 -1
  78. package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
  79. package/dist/model-context-contributions.d.ts +8 -4
  80. package/dist/observability-metrics.d.ts +63 -4
  81. package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
  82. package/dist/personal-github-git-credentials.d.ts +9 -0
  83. package/dist/pr-review-credentials.d.ts +6 -0
  84. package/dist/sandbox-archive-storage.d.ts +45 -0
  85. package/dist/sandbox-resume.d.ts +65 -6
  86. package/dist/sandbox-routing.d.ts +38 -2
  87. package/dist/scheduled-alert-occurrence.d.ts +33 -0
  88. package/dist/workflow-bundle.js +87 -28
  89. package/dist/workflows/activities.d.ts +1 -0
  90. package/dist/workflows/automations.d.ts +6 -0
  91. package/dist/workflows/document-index.d.ts +1 -0
  92. package/dist/workflows.d.ts +1 -0
  93. package/package.json +17 -17
  94. package/src/activities/agent-turn/admission.ts +292 -0
  95. package/src/activities/agent-turn/agent-build.ts +802 -0
  96. package/src/activities/agent-turn/claim.ts +590 -0
  97. package/src/activities/agent-turn/codex-capacity.ts +720 -0
  98. package/src/activities/agent-turn/codex.ts +94 -0
  99. package/src/activities/agent-turn/compaction-prep.ts +722 -0
  100. package/src/activities/agent-turn/credential-leases.ts +240 -0
  101. package/src/activities/agent-turn/errors.ts +1068 -0
  102. package/src/activities/agent-turn/failure-settlement.ts +1436 -0
  103. package/src/activities/agent-turn/file-resources.ts +227 -0
  104. package/src/activities/agent-turn/finalization.ts +754 -0
  105. package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
  106. package/src/activities/agent-turn/governance-model.ts +465 -0
  107. package/src/activities/agent-turn/history-sink.ts +139 -0
  108. package/src/activities/agent-turn/history.ts +200 -0
  109. package/src/activities/agent-turn/media-artifacts.ts +359 -0
  110. package/src/activities/agent-turn/model-usage.ts +823 -0
  111. package/src/activities/agent-turn/quiescence.ts +353 -0
  112. package/src/activities/agent-turn/run-credentials.ts +761 -0
  113. package/src/activities/agent-turn/run.ts +1530 -0
  114. package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
  115. package/src/activities/agent-turn/sandbox-provision.ts +569 -0
  116. package/src/activities/agent-turn/sandbox-route.ts +358 -0
  117. package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
  118. package/src/activities/agent-turn/stream-attempt.ts +1670 -0
  119. package/src/activities/agent-turn/tool-environment.ts +729 -0
  120. package/src/activities/agent-turn/tool-policy.ts +216 -0
  121. package/src/activities/agent-turn/tool-result-spill.ts +265 -0
  122. package/src/activities/agent-turn/turn-context.ts +281 -0
  123. package/src/activities/agent-turn/xai-capacity.ts +229 -0
  124. package/src/activities/agent-turn.ts +12 -12701
  125. package/src/activities/api-integrations.ts +3 -0
  126. package/src/activities/automations.ts +214 -0
  127. package/src/activities/codemode-dispatcher.ts +12 -1
  128. package/src/activities/common.ts +1 -5
  129. package/src/activities/connector-attachments.ts +168 -0
  130. package/src/activities/context-compaction.ts +18 -0
  131. package/src/activities/environment.ts +151 -55
  132. package/src/activities/generated-images.ts +26 -11
  133. package/src/activities/git-credential-renewal.ts +13 -3
  134. package/src/activities/goals.ts +85 -19
  135. package/src/activities/google-drive-provider.ts +178 -0
  136. package/src/activities/google-drive-publication.ts +147 -49
  137. package/src/activities/incident-telemetry-authority.ts +327 -0
  138. package/src/activities/incident-telemetry-preflight.ts +317 -0
  139. package/src/activities/knowledge-source-sync.ts +178 -37
  140. package/src/activities/mcp-credentials.ts +191 -17
  141. package/src/activities/open-suffix-resume.ts +422 -0
  142. package/src/activities/packs.ts +12 -60
  143. package/src/activities/parent-wake.ts +64 -14
  144. package/src/activities/retained-screenshot-reaper.ts +6 -3
  145. package/src/activities/retained-screenshots.ts +15 -47
  146. package/src/activities/rig-verification.ts +1 -1
  147. package/src/activities/run-input.ts +102 -142
  148. package/src/activities/sandbox-images.ts +7 -166
  149. package/src/activities/sandbox-lease.ts +406 -15
  150. package/src/activities/scheduled-tasks.ts +2103 -411
  151. package/src/activities/session-state.ts +10 -2
  152. package/src/activities/types.ts +32 -2
  153. package/src/activities/video-media-validation.ts +3 -3
  154. package/src/activities/video-output-retention.ts +2 -57
  155. package/src/activities/xai-auth.ts +8 -0
  156. package/src/activities-control.ts +2 -0
  157. package/src/activities.ts +3 -0
  158. package/src/activity-services.ts +13 -4
  159. package/src/connection-provider-use.ts +36 -0
  160. package/src/editable-artifact-materializer-subprocess.ts +25 -14
  161. package/src/editable-artifact-materializer.ts +29 -18
  162. package/src/github-rest-mcp.ts +713 -0
  163. package/src/http.ts +16 -6
  164. package/src/index.ts +40 -5
  165. package/src/knowledge-source-sync-activity-policy.ts +2 -0
  166. package/src/model-context-contributions.ts +33 -10
  167. package/src/observability-metrics.ts +357 -0
  168. package/src/opensandbox-kubernetes-inventory.ts +292 -0
  169. package/src/personal-github-git-credentials.ts +331 -0
  170. package/src/pr-review-credentials.ts +163 -0
  171. package/src/sandbox-archive-storage.ts +159 -0
  172. package/src/sandbox-resume.ts +449 -63
  173. package/src/sandbox-routing.ts +314 -16
  174. package/src/scheduled-alert-occurrence.ts +176 -0
  175. package/src/workflows/activities.ts +18 -2
  176. package/src/workflows/automations.ts +11 -0
  177. package/src/workflows/session.ts +13 -3
  178. package/src/workflows.ts +1 -0
  179. package/dist/activities-control-BJSDRSIP.js.map +0 -1
  180. package/dist/activities-turn-J5YO7XXX.js +0 -17243
  181. package/dist/activities-turn-J5YO7XXX.js.map +0 -1
  182. package/dist/chunk-N5TJIFCE.js.map +0 -1
  183. package/dist/chunk-P6Z2SLAA.js +0 -3092
  184. package/dist/chunk-P6Z2SLAA.js.map +0 -1
@@ -0,0 +1,4698 @@
1
+ import {
2
+ observabilityEventLogger
3
+ } from "./chunk-CE7TPKBS.js";
4
+
5
+ // src/activity-services.ts
6
+ import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth } from "@opengeni/config";
7
+ import { configureChildLifecycleNotices, createDb } from "@opengeni/db";
8
+ import { createNatsEventBus } from "@opengeni/events";
9
+ import { createObservability } from "@opengeni/observability";
10
+ import { createObjectStorage } from "@opengeni/storage";
11
+
12
+ // src/personal-github-git-credentials.ts
13
+ import {
14
+ isPersonalGitHubConnection,
15
+ PersonalGitHubConnectionMetadata
16
+ } from "@opengeni/contracts/personal-github";
17
+ import {
18
+ getConnectionMetadata,
19
+ getPersonalGitHubRepositorySelectionState,
20
+ getSessionAuthorityProjection,
21
+ getSessionTurnForAttempt,
22
+ resolveAcceptedConnectionUse
23
+ } from "@opengeni/db";
24
+ import {
25
+ PERSONAL_GITHUB_GIT_BROKER_TOKEN_TTL_SECONDS,
26
+ personalGitHubGitBrokerRouteId,
27
+ sealPersonalGitHubGitBrokerClaims
28
+ } from "@opengeni/github";
29
+ import { randomBytes, randomUUID } from "crypto";
30
+ import { isDeepStrictEqual } from "util";
31
+ var PERSONAL_GITHUB_SERVER_ID = "github:personal";
32
+ function buildPersonalGitHubGitCredentials(db, settings) {
33
+ if (!settings.githubPersonalOauthEnabled) return null;
34
+ const secret = settings.integrationsStateSecret?.trim();
35
+ const publicOrigin = personalGitBrokerOrigin(settings.publicBaseUrl);
36
+ if (!secret || !publicOrigin) {
37
+ throw new Error("personal GitHub Git broker configuration is unavailable");
38
+ }
39
+ return async (request) => {
40
+ if (request.provider && request.provider !== "github") {
41
+ throw new Error("personal GitHub Git broker accepts only GitHub repositories");
42
+ }
43
+ if (!request.credentialBindingId || !request.repositoryRefs?.length) {
44
+ throw new Error("personal GitHub Git broker requires one exact credential binding");
45
+ }
46
+ const turn = await getSessionTurnForAttempt(
47
+ db,
48
+ request.workspaceId,
49
+ request.sessionId,
50
+ request.attemptId
51
+ );
52
+ const session = await getSessionAuthorityProjection(db, request.workspaceId, request.sessionId);
53
+ if (!turn || !session || turn.id !== request.turnId || turn.executionGeneration !== request.executionGeneration || !isDeepStrictEqual(turn.initiator, request.initiator) || !isDeepStrictEqual(turn.initiatorContext, request.initiatorContext) || session.rootSessionId !== request.rootSessionId) {
54
+ throw new Error("personal GitHub Git credential authority is no longer current");
55
+ }
56
+ const delegation = exactPersonalGitHubDelegation(
57
+ turn.personalConnectionDelegations,
58
+ request.credentialBindingId
59
+ );
60
+ const snapshot = delegation.personalGitHubRepositorySelection;
61
+ const repositories = exactRequestedRepositories(
62
+ request.repositoryRefs,
63
+ snapshot.credentialBindingId,
64
+ snapshot.repositories
65
+ );
66
+ const resolution = await resolveAcceptedConnectionUse(db, {
67
+ accountId: request.accountId,
68
+ workspaceId: request.workspaceId,
69
+ sessionId: request.sessionId,
70
+ turnId: request.turnId,
71
+ attemptId: request.attemptId,
72
+ executionGeneration: request.executionGeneration,
73
+ physicalRequestId: randomUUID(),
74
+ usePhase: "credential_resolution",
75
+ serverId: PERSONAL_GITHUB_SERVER_ID,
76
+ connectionId: delegation.connectionId,
77
+ providerDomain: "github.com",
78
+ connectionKind: "oauth2",
79
+ subjectScope: "subject",
80
+ ownerSubjectId: delegation.ownerSubjectId
81
+ });
82
+ if (resolution.status !== "authorized" || resolution.originWorkspaceId !== delegation.originWorkspaceId || resolution.attribution.connectionId !== delegation.connectionId || resolution.attribution.connectionGeneration !== snapshot.connectionAuthorityGeneration || resolution.attribution.ownerSubjectId !== delegation.ownerSubjectId) {
83
+ throw new Error("personal GitHub Git connection authority is no longer current");
84
+ }
85
+ const connection = await getConnectionMetadata(
86
+ db,
87
+ delegation.originWorkspaceId,
88
+ delegation.connectionId,
89
+ delegation.ownerSubjectId
90
+ );
91
+ if (!connection || connection.accountId !== request.accountId || connection.workspaceId !== delegation.originWorkspaceId || connection.subjectId !== delegation.ownerSubjectId || connection.status !== "active" || // connections.version is credential-refresh CAS, not use authority.
92
+ // resolveAcceptedConnectionUse above fences the independent common
93
+ // connection-authority generation carried by the accepted snapshot.
94
+ connection.grantedScopes.length !== 1 || connection.grantedScopes[0] !== "repo" || !isPersonalGitHubConnection(connection)) {
95
+ throw new Error("personal GitHub Git connection is unavailable");
96
+ }
97
+ const metadata = PersonalGitHubConnectionMetadata.parse(connection.metadata);
98
+ const currentSelection = await getPersonalGitHubRepositorySelectionState(db, {
99
+ accountId: request.accountId,
100
+ originWorkspaceId: delegation.originWorkspaceId,
101
+ subjectId: delegation.ownerSubjectId,
102
+ connectionId: delegation.connectionId
103
+ });
104
+ if (!currentSelection || currentSelection.connectionAuthorityGeneration !== snapshot.connectionAuthorityGeneration || currentSelection.selectionGeneration !== snapshot.selectionGeneration || currentSelection.credentialBindingId !== snapshot.credentialBindingId || currentSelection.credentialBindingId !== metadata.credentialBindingId || currentSelection.providerPrincipalId !== metadata.providerPrincipalId || metadata.githubUserId !== metadata.providerPrincipalId) {
105
+ throw new Error("personal GitHub repository authority changed");
106
+ }
107
+ const currentById = new Map(
108
+ currentSelection.repositories.map((repository) => [repository.repositoryId, repository])
109
+ );
110
+ for (const repository of repositories) {
111
+ const current = currentById.get(repository.repositoryId);
112
+ if (!current || current.fullName !== repository.fullName || current.canonicalUrl !== repository.canonicalUrl || current.selectionGeneration !== repository.selectionGeneration || current.disabled || repository.access === "write" && (current.selectedAccess !== "write" || current.archived || !personalGitHubCanWrite(current.permissions)) || !personalGitHubCanRead(current.permissions)) {
113
+ throw new Error("personal GitHub repository authority changed");
114
+ }
115
+ }
116
+ const identity = {
117
+ name: metadata.githubLogin,
118
+ email: `${metadata.githubUserId}+${metadata.githubLogin}@users.noreply.github.com`
119
+ };
120
+ if (request.purpose === "identity") {
121
+ return {
122
+ workspaceId: request.workspaceId,
123
+ credentialBindingId: request.credentialBindingId,
124
+ provider: "github",
125
+ providerHost: "github.com",
126
+ identity
127
+ };
128
+ }
129
+ const now = Math.floor(Date.now() / 1e3);
130
+ const baseClaims = {
131
+ version: 1,
132
+ accountId: request.accountId,
133
+ workspaceId: request.workspaceId,
134
+ sessionId: request.sessionId,
135
+ rootSessionId: request.rootSessionId,
136
+ turnId: request.turnId,
137
+ attemptId: request.attemptId,
138
+ executionGeneration: request.executionGeneration,
139
+ originWorkspaceId: delegation.originWorkspaceId,
140
+ connectionId: delegation.connectionId,
141
+ connectionAuthorityGeneration: snapshot.connectionAuthorityGeneration,
142
+ ownerSubjectId: delegation.ownerSubjectId,
143
+ credentialBindingId: snapshot.credentialBindingId,
144
+ selectionGeneration: snapshot.selectionGeneration
145
+ };
146
+ const repositoryClaims = repositories.map(
147
+ (repository) => ({
148
+ ...repository,
149
+ routeId: personalGitHubGitBrokerRouteId(secret, {
150
+ ...baseClaims,
151
+ repository
152
+ })
153
+ })
154
+ );
155
+ const claims = {
156
+ ...baseClaims,
157
+ nonce: randomBytes(24).toString("base64url"),
158
+ issuedAt: now,
159
+ expiresAt: now + PERSONAL_GITHUB_GIT_BROKER_TOKEN_TTL_SECONDS
160
+ };
161
+ return {
162
+ token: sealPersonalGitHubGitBrokerClaims(secret, claims),
163
+ expiresAt: new Date(claims.expiresAt * 1e3).toISOString(),
164
+ workspaceId: request.workspaceId,
165
+ credentialBindingId: request.credentialBindingId,
166
+ provider: "github",
167
+ providerHost: "github.com",
168
+ identity,
169
+ transport: {
170
+ kind: "http_broker",
171
+ repositories: repositoryClaims.map((repository) => ({
172
+ repositoryUri: repository.canonicalUrl,
173
+ brokerUri: `${publicOrigin}/v1/git/personal/${repository.routeId}`
174
+ }))
175
+ }
176
+ };
177
+ };
178
+ }
179
+ function personalGitHubCanRead(permissions) {
180
+ return permissions.pull || permissions.triage || permissions.push || permissions.maintain || permissions.admin;
181
+ }
182
+ function personalGitHubCanWrite(permissions) {
183
+ return permissions.push || permissions.maintain || permissions.admin;
184
+ }
185
+ function personalGitBrokerOrigin(value) {
186
+ if (!value) return null;
187
+ const url = new URL(value);
188
+ if (url.protocol !== "https:" || url.username || url.password || url.pathname !== "/" || url.search || url.hash) {
189
+ return null;
190
+ }
191
+ return url.origin;
192
+ }
193
+ function exactPersonalGitHubDelegation(delegations, credentialBindingId) {
194
+ const matches = delegations.filter(
195
+ (delegation2) => delegation2.serverId === PERSONAL_GITHUB_SERVER_ID && delegation2.connectionType === "github_personal" && delegation2.personalGitHubRepositorySelection?.credentialBindingId === credentialBindingId
196
+ );
197
+ const delegation = matches[0];
198
+ if (matches.length !== 1 || !delegation || delegation.providerDomain !== "github.com" || delegation.kind !== "oauth2" || !delegation.originWorkspaceId || !delegation.userDelegation || !delegation.personalGitHubRepositorySelection) {
199
+ throw new Error("personal GitHub Git delegation is unavailable");
200
+ }
201
+ return delegation;
202
+ }
203
+ function exactRequestedRepositories(refs, credentialBindingId, snapshots) {
204
+ if (refs.length !== snapshots.length) {
205
+ throw new Error("personal GitHub Git request does not match the accepted repository set");
206
+ }
207
+ const byCanonicalUrl = new Map(
208
+ snapshots.map((repository) => [repository.canonicalUrl, repository])
209
+ );
210
+ const seen = /* @__PURE__ */ new Set();
211
+ return refs.map((ref) => {
212
+ const snapshot = byCanonicalUrl.get(ref.uri);
213
+ if (!snapshot || seen.has(ref.uri) || ref.provider !== "github" || ref.credentialBindingId !== credentialBindingId || ref.connectionId !== void 0 || ref.installationId !== void 0 || String(ref.repositoryId) !== snapshot.repositoryId || ref.ref !== snapshot.ref || ref.access !== snapshot.access) {
214
+ throw new Error("personal GitHub Git request exceeds the accepted repository authority");
215
+ }
216
+ seen.add(ref.uri);
217
+ return snapshot;
218
+ });
219
+ }
220
+
221
+ // src/pr-review-credentials.ts
222
+ import { environmentsEncryptionKeyBytes } from "@opengeni/config";
223
+ import { prReviewRegistrationIdFromCredentialBinding } from "@opengeni/core";
224
+ import { decryptVariableSetValue, resolvePrReviewGitCredential } from "@opengeni/db";
225
+ import {
226
+ createGitHubAppInstallationTokenWithExpiry,
227
+ createGitHubAppInstallationTokenWithSigningSettings
228
+ } from "@opengeni/github";
229
+ function createStandaloneConnectionCredentialsPort(settings, db) {
230
+ return {
231
+ gitCredentials: async (request) => await resolveStandaloneGitCredentials(settings, db, request)
232
+ };
233
+ }
234
+ async function resolveStandaloneGitCredentials(settings, db, request) {
235
+ const provider = request.provider ?? "github";
236
+ const registrationId = request.credentialBindingId ? prReviewRegistrationIdFromCredentialBinding(request.credentialBindingId) : null;
237
+ if (!registrationId) {
238
+ if (provider !== "github") {
239
+ throw new Error(`${provider} Git credentials require an explicit PR Review or host binding`);
240
+ }
241
+ if (request.purpose === "identity") {
242
+ return { workspaceId: request.workspaceId };
243
+ }
244
+ const minted = await createGitHubAppInstallationTokenWithExpiry(settings, {
245
+ installationId: request.installationId,
246
+ repositoryIds: request.repositoryIds
247
+ });
248
+ return {
249
+ token: minted.token,
250
+ workspaceId: request.workspaceId,
251
+ ...minted.expiresAt ? { expiresAt: minted.expiresAt } : {}
252
+ };
253
+ }
254
+ const authority = await resolvePrReviewGitCredential(db, {
255
+ accountId: request.accountId,
256
+ workspaceId: request.workspaceId,
257
+ registrationId,
258
+ provider,
259
+ sessionId: request.sessionId,
260
+ rootSessionId: request.rootSessionId,
261
+ turnId: request.turnId,
262
+ attemptId: request.attemptId,
263
+ executionGeneration: request.executionGeneration,
264
+ repositoryRefs: (request.repositoryRefs ?? []).map((reference) => ({
265
+ uri: reference.uri,
266
+ ...reference.expectedCommitSha !== void 0 ? { expectedCommitSha: reference.expectedCommitSha } : {},
267
+ ...reference.repositoryId !== void 0 ? { repositoryId: reference.repositoryId } : {},
268
+ ...reference.installationId !== void 0 ? { installationId: reference.installationId } : {},
269
+ ...reference.projectId !== void 0 ? { projectId: reference.projectId } : {}
270
+ }))
271
+ });
272
+ const echoes = {
273
+ workspaceId: request.workspaceId,
274
+ ...request.credentialBindingId ? { credentialBindingId: request.credentialBindingId } : {},
275
+ provider,
276
+ ...request.providerHost ? { providerHost: request.providerHost } : {}
277
+ };
278
+ if (request.purpose === "identity") {
279
+ return authority.credentialKind === "github_app" ? {
280
+ ...echoes,
281
+ identity: {
282
+ name: "pr-review[bot]",
283
+ email: `${authority.appId ?? "pr-review"}+pr-review[bot]@users.noreply.github.com`
284
+ }
285
+ } : echoes;
286
+ }
287
+ if (authority.credentialKind === "github_app") {
288
+ assertGitHubMintMatchesRepositoryRefs(request);
289
+ const encryptionKey2 = environmentsEncryptionKeyBytes(settings);
290
+ if (!encryptionKey2 || !authority.appId || !authority.credentialEncrypted) {
291
+ throw new Error("PR Review GitHub App credential is unavailable");
292
+ }
293
+ const minted = await createGitHubAppInstallationTokenWithSigningSettings(
294
+ {
295
+ githubAppId: authority.appId,
296
+ githubAppPrivateKey: decryptVariableSetValue(encryptionKey2, authority.credentialEncrypted)
297
+ },
298
+ {
299
+ installationId: request.installationId,
300
+ repositoryIds: request.repositoryIds
301
+ }
302
+ );
303
+ return {
304
+ ...echoes,
305
+ token: minted.token,
306
+ ...minted.expiresAt ? { expiresAt: minted.expiresAt } : {}
307
+ };
308
+ }
309
+ if (authority.expiresAt && Date.parse(authority.expiresAt) <= Date.now()) {
310
+ throw new Error(`PR Review ${provider} provider credential has expired`);
311
+ }
312
+ const encryptionKey = environmentsEncryptionKeyBytes(settings);
313
+ if (!encryptionKey || !authority.credentialEncrypted) {
314
+ throw new Error(`PR Review ${provider} provider credential is unavailable`);
315
+ }
316
+ return {
317
+ ...echoes,
318
+ token: decryptVariableSetValue(encryptionKey, authority.credentialEncrypted),
319
+ ...authority.expiresAt ? { expiresAt: authority.expiresAt } : {}
320
+ };
321
+ }
322
+ function assertGitHubMintMatchesRepositoryRefs(request) {
323
+ const refs = request.repositoryRefs ?? [];
324
+ const repositoryIds = refs.map((reference) => positiveInteger(reference.repositoryId));
325
+ const installationIds = refs.map((reference) => positiveInteger(reference.installationId));
326
+ if (repositoryIds.some((value) => value === null) || installationIds.some((value) => value === null)) {
327
+ throw new Error("PR Review GitHub credential request lacks exact repository authority");
328
+ }
329
+ const exactRepositoryIds = [...new Set(repositoryIds)].sort(
330
+ (left, right) => left - right
331
+ );
332
+ const requestedRepositoryIds = [...new Set(request.repositoryIds)].sort(
333
+ (left, right) => left - right
334
+ );
335
+ const exactInstallationIds = [...new Set(installationIds)];
336
+ if (exactInstallationIds.length !== 1 || exactInstallationIds[0] !== request.installationId || exactRepositoryIds.length !== request.repositoryIds.length || exactRepositoryIds.some((value, index) => value !== requestedRepositoryIds[index])) {
337
+ throw new Error(
338
+ "PR Review GitHub token mint authority does not match its repository resources"
339
+ );
340
+ }
341
+ }
342
+ function positiveInteger(value) {
343
+ if (typeof value === "number" && Number.isSafeInteger(value) && value > 0) return value;
344
+ if (typeof value === "string" && /^\d+$/.test(value)) {
345
+ const parsed = Number(value);
346
+ return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : null;
347
+ }
348
+ return null;
349
+ }
350
+
351
+ // src/activity-services.ts
352
+ function createSharedActivityServices(dependencies) {
353
+ let servicesPromise = null;
354
+ return async function services() {
355
+ servicesPromise ??= (async () => {
356
+ const settings = dependencies.settings ?? getSettings();
357
+ const observability = dependencies.observability ?? createObservability(settings, { component: "worker" });
358
+ const searchPath = dbSearchPath(settings);
359
+ const dbClient = dependencies.db ? null : createDb(settings.databaseUrl, {
360
+ ...searchPath ? { searchPath } : {},
361
+ rlsStrategy: settings.rlsStrategy
362
+ });
363
+ const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
364
+ const db = dependencies.db ?? dbClient.db;
365
+ configureChildLifecycleNotices({ enabled: settings.childLifecycleNoticesEnabled });
366
+ return {
367
+ settings,
368
+ db,
369
+ bus: dependencies.bus ?? await createNatsEventBus(
370
+ settings.natsUrl,
371
+ controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
372
+ { logger: observabilityEventLogger(observability) }
373
+ ),
374
+ objectStorage: dependencies.objectStorage ?? createObjectStorage(settings),
375
+ observability,
376
+ wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
377
+ signalSessionAttemptQuiesced: dependencies.signalSessionAttemptQuiesced ?? null,
378
+ inspectSessionAttemptActivity: dependencies.inspectSessionAttemptActivity ?? null,
379
+ signalCodexCapacityWorkflow: dependencies.signalCodexCapacityWorkflow ?? null,
380
+ startSandboxReaperWorkflow: dependencies.startSandboxReaperWorkflow ?? null,
381
+ startVideoGenerationWorkflow: dependencies.startVideoGenerationWorkflow ?? null,
382
+ entitlements: dependencies.entitlements ?? null,
383
+ connectionCredentials: dependencies.connectionCredentials ?? createStandaloneConnectionCredentialsPort(settings, db),
384
+ personalGitHubCredentials: dependencies.personalGitHubCredentials ?? buildPersonalGitHubGitCredentials(db, settings)
385
+ };
386
+ })();
387
+ return await servicesPromise;
388
+ };
389
+ }
390
+
391
+ // src/activities/codex-capacity.ts
392
+ import {
393
+ fetchCodexUsageForAccount,
394
+ getCodexCapacityWaitForSession,
395
+ getXaiCapacityWaitForSession,
396
+ listCodexAccountStatuses,
397
+ listPendingCodexCapacityWakeTargets,
398
+ reconcileCodexCapacityWait as reconcileCodexCapacityWaitDb,
399
+ reconcileXaiCapacityWait as reconcileXaiCapacityWaitDb
400
+ } from "@opengeni/db";
401
+
402
+ // src/activities/codex-rotation.ts
403
+ function effectiveRotationStrategy(_stored) {
404
+ return "sharded";
405
+ }
406
+ function classifyCodexPin(args) {
407
+ const { pinnedCredentialId, pinSource, strategy, rotationEnabled } = args;
408
+ const pinned = pinnedCredentialId != null;
409
+ if (pinned && pinSource !== "policy") {
410
+ return "manual";
411
+ }
412
+ const shardedActive = rotationEnabled && effectiveRotationStrategy(strategy) === "sharded";
413
+ if (shardedActive) {
414
+ return "sharded";
415
+ }
416
+ if (pinned && pinSource === "policy") {
417
+ return "clearStale";
418
+ }
419
+ return "unpinned";
420
+ }
421
+ var MIN_IDLE_MS = 6e4;
422
+ var DEFAULT_RESET_COOLDOWN_MS = 6e4;
423
+ var CODEX_USAGE_EXHAUSTED_PCT = 100;
424
+ function effectiveWindowUsed(usedPercent, resetAt, now) {
425
+ return resetAt && resetAt.getTime() <= now.getTime() ? 0 : usedPercent ?? 0;
426
+ }
427
+ function bindingUsedPct(acct, now) {
428
+ return Math.max(
429
+ effectiveWindowUsed(acct.primaryUsedPercent, acct.primaryResetAt, now),
430
+ effectiveWindowUsed(acct.secondaryUsedPercent, acct.secondaryResetAt, now)
431
+ );
432
+ }
433
+ function bindingRemaining(acct, now) {
434
+ const primaryRemaining = 100 - effectiveWindowUsed(acct.primaryUsedPercent, acct.primaryResetAt, now);
435
+ const secondaryRemaining = 100 - effectiveWindowUsed(acct.secondaryUsedPercent, acct.secondaryResetAt, now);
436
+ return Math.min(primaryRemaining, secondaryRemaining);
437
+ }
438
+ function cooling(acct, now) {
439
+ return acct.exhaustedUntil != null && acct.exhaustedUntil.getTime() > now.getTime();
440
+ }
441
+ function isCodexCredentialHealthy(acct, now) {
442
+ return acct.status === "active" && !cooling(acct, now) && bindingUsedPct(acct, now) < CODEX_USAGE_EXHAUSTED_PCT;
443
+ }
444
+ function isCodexCredentialEligible(acct, now) {
445
+ return acct.allocatorEnabled && isCodexCredentialHealthy(acct, now);
446
+ }
447
+ function fnv1a32(input) {
448
+ let hash = 2166136261;
449
+ for (let i = 0; i < input.length; i++) {
450
+ hash ^= input.charCodeAt(i);
451
+ hash = Math.imul(hash, 16777619);
452
+ }
453
+ return hash >>> 0;
454
+ }
455
+ function shardCredentialForSession(args) {
456
+ const { sessionId, accounts, now } = args;
457
+ const eligibles = accounts.filter((acct) => isCodexCredentialEligible(acct, now));
458
+ if (eligibles.length === 0) {
459
+ return null;
460
+ }
461
+ const index = fnv1a32(sessionId) % eligibles.length;
462
+ return eligibles[index].id;
463
+ }
464
+ function earliestCodexReset(accounts, now) {
465
+ return earliestReset(accounts, now);
466
+ }
467
+ function chooseShardedHome(args) {
468
+ const { sessionId, currentPolicyPin, accounts, now } = args;
469
+ const pinRow = currentPolicyPin ? accounts.find((acct) => acct.id === currentPolicyPin) ?? null : null;
470
+ if (pinRow && isCodexCredentialEligible(pinRow, now)) {
471
+ return { kind: "home", credentialId: currentPolicyPin, rewritePin: false };
472
+ }
473
+ const home = shardCredentialForSession({ sessionId, accounts, now });
474
+ if (home == null) {
475
+ return { kind: "allCapped", earliestResetAt: earliestReset(accounts, now) };
476
+ }
477
+ return { kind: "home", credentialId: home, rewritePin: true };
478
+ }
479
+ function availableAt(acct, now) {
480
+ const nowMs = now.getTime();
481
+ const defaultClear = new Date(nowMs + DEFAULT_RESET_COOLDOWN_MS);
482
+ const candidates = [];
483
+ if (acct.exhaustedUntil != null && acct.exhaustedUntil.getTime() > nowMs) {
484
+ candidates.push(acct.exhaustedUntil);
485
+ }
486
+ const windowClear = (over, resetAt) => {
487
+ if (!over) return;
488
+ candidates.push(resetAt != null && resetAt.getTime() > nowMs ? resetAt : defaultClear);
489
+ };
490
+ windowClear(
491
+ effectiveWindowUsed(acct.primaryUsedPercent, acct.primaryResetAt, now) >= CODEX_USAGE_EXHAUSTED_PCT,
492
+ acct.primaryResetAt
493
+ );
494
+ windowClear(
495
+ effectiveWindowUsed(acct.secondaryUsedPercent, acct.secondaryResetAt, now) >= CODEX_USAGE_EXHAUSTED_PCT,
496
+ acct.secondaryResetAt
497
+ );
498
+ if (candidates.length === 0) {
499
+ return defaultClear;
500
+ }
501
+ return candidates.reduce((a, b) => b.getTime() > a.getTime() ? b : a);
502
+ }
503
+ function earliestReset(accounts, now) {
504
+ return accounts.map((acct) => availableAt(acct, now)).reduce((a, b) => b.getTime() < a.getTime() ? b : a);
505
+ }
506
+ function authoritativeCodexCapacityResetAt(accounts, now) {
507
+ const future = [];
508
+ for (const account of accounts) {
509
+ if (!account.allocatorEnabled || account.status !== "active") continue;
510
+ if (account.exhaustedUntil && account.exhaustedUntil.getTime() > now.getTime()) {
511
+ future.push(account.exhaustedUntil);
512
+ }
513
+ if ((account.primaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT && account.primaryResetAt && account.primaryResetAt.getTime() > now.getTime()) {
514
+ future.push(account.primaryResetAt);
515
+ }
516
+ if ((account.secondaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT && account.secondaryResetAt && account.secondaryResetAt.getTime() > now.getTime()) {
517
+ future.push(account.secondaryResetAt);
518
+ }
519
+ }
520
+ if (future.length === 0) return null;
521
+ return future.reduce(
522
+ (earliest, candidate) => candidate.getTime() < earliest.getTime() ? candidate : earliest
523
+ );
524
+ }
525
+ function computeIdleDelayMs(earliestResetAt, now, maxMs) {
526
+ const delta = earliestResetAt.getTime() - now.getTime();
527
+ return Math.min(Math.max(delta, MIN_IDLE_MS), maxMs);
528
+ }
529
+ var REACTIVE_PERSISTENCE_FAULT_FLOOR_MS = 5e3;
530
+ var REACTIVE_ROTATION_MARGIN = 2;
531
+ var REACTIVE_CIRCUIT_BREAKER_IDLE_MS = MIN_IDLE_MS;
532
+ function computeReactiveRotationResume(args) {
533
+ const streak = args.priorConsecutiveRotations + 1;
534
+ const bound = args.connectedAccountCount + REACTIVE_ROTATION_MARGIN;
535
+ if (streak > bound) {
536
+ return { continueDelayMs: REACTIVE_CIRCUIT_BREAKER_IDLE_MS, idleUntilReset: true };
537
+ }
538
+ if (!args.cooldownPersisted) {
539
+ return { continueDelayMs: REACTIVE_PERSISTENCE_FAULT_FLOOR_MS, idleUntilReset: false };
540
+ }
541
+ return { continueDelayMs: 0, idleUntilReset: false };
542
+ }
543
+ function chooseRotationActive(args) {
544
+ const { activeCredentialId, priorCredentialId, accounts, now } = args;
545
+ const rotationStrategy = effectiveRotationStrategy(args.rotationStrategy);
546
+ const allocatableAccounts = accounts.filter((account) => account.allocatorEnabled);
547
+ if (allocatableAccounts.length === 0) {
548
+ return { kind: "none" };
549
+ }
550
+ const eligibles = allocatableAccounts.filter((acct) => isCodexCredentialEligible(acct, now));
551
+ const decide = (chosen2) => {
552
+ if (!chosen2) {
553
+ return {
554
+ kind: "allCapped",
555
+ earliestResetAt: earliestReset(allocatableAccounts, now)
556
+ };
557
+ }
558
+ return {
559
+ kind: "active",
560
+ credentialId: chosen2.id,
561
+ moved: chosen2.id !== activeCredentialId
562
+ };
563
+ };
564
+ if (rotationStrategy === "round_robin") {
565
+ if (eligibles.length === 0) {
566
+ return {
567
+ kind: "allCapped",
568
+ earliestResetAt: earliestReset(allocatableAccounts, now)
569
+ };
570
+ }
571
+ const priorIdx = priorCredentialId ? allocatableAccounts.findIndex((acct) => acct.id === priorCredentialId) : -1;
572
+ const ordered = priorIdx >= 0 ? [
573
+ ...allocatableAccounts.slice(priorIdx + 1),
574
+ ...allocatableAccounts.slice(0, priorIdx + 1)
575
+ ] : allocatableAccounts;
576
+ const chosen2 = ordered.find((acct) => isCodexCredentialEligible(acct, now));
577
+ return decide(chosen2);
578
+ }
579
+ if (rotationStrategy === "drain_then_next") {
580
+ const priorRow = priorCredentialId ? accounts.find((acct) => acct.id === priorCredentialId) : void 0;
581
+ if (priorRow && isCodexCredentialEligible(priorRow, now)) {
582
+ return decide(priorRow);
583
+ }
584
+ return decide(eligibles[0]);
585
+ }
586
+ const activeLeases = (acct) => "activeLeaseCount" in acct ? acct.activeLeaseCount : 0;
587
+ const selections = (acct) => "selectionCount" in acct ? acct.selectionCount : 0;
588
+ const lastSelected = (acct) => "lastSelectedAt" in acct && acct.lastSelectedAt ? acct.lastSelectedAt.getTime() : 0;
589
+ const chosen = eligibles.reduce((best, acct) => {
590
+ if (!best) {
591
+ return acct;
592
+ }
593
+ if (activeLeases(acct) !== activeLeases(best)) {
594
+ return activeLeases(acct) < activeLeases(best) ? acct : best;
595
+ }
596
+ if (bindingRemaining(acct, now) !== bindingRemaining(best, now)) {
597
+ return bindingRemaining(acct, now) > bindingRemaining(best, now) ? acct : best;
598
+ }
599
+ if (selections(acct) !== selections(best)) {
600
+ return selections(acct) < selections(best) ? acct : best;
601
+ }
602
+ return lastSelected(acct) < lastSelected(best) ? acct : best;
603
+ }, void 0);
604
+ return decide(chosen);
605
+ }
606
+ function chooseLegacyRotationActive(args) {
607
+ if (args.rotationStrategy === "most_remaining") {
608
+ const active = args.activeCredentialId ? args.accounts.find((account) => account.id === args.activeCredentialId) : void 0;
609
+ if (active && isCodexCredentialEligible(active, args.now)) {
610
+ return { kind: "active", credentialId: active.id, moved: false };
611
+ }
612
+ }
613
+ return chooseRotationActive({
614
+ ...args,
615
+ accounts: args.accounts.map((account) => ({
616
+ ...account,
617
+ activeLeaseCount: 0,
618
+ selectionCount: 0,
619
+ lastSelectedAt: null
620
+ }))
621
+ });
622
+ }
623
+ function selectCodexCredentialLeaseForTurn(args) {
624
+ const {
625
+ accounts,
626
+ activeCredentialId,
627
+ rotationEnabled,
628
+ leaseRotationEnabled,
629
+ rotationStrategy,
630
+ existingCredentialId
631
+ } = args.context;
632
+ const leasingEnabled = args.leasingEnabled && leaseRotationEnabled;
633
+ const strategy = effectiveRotationStrategy(rotationStrategy);
634
+ const existing = existingCredentialId ? accounts.find((account) => account.id === existingCredentialId) : void 0;
635
+ if (existing && isCodexCredentialHealthy(existing, args.now)) {
636
+ return {
637
+ credentialId: existing.id,
638
+ advanceActivePointer: false,
639
+ decision: {
640
+ kind: "active",
641
+ credentialId: existing.id,
642
+ moved: existing.id !== activeCredentialId
643
+ }
644
+ };
645
+ }
646
+ const connectedIds = new Set(
647
+ accounts.filter((account) => account.allocatorEnabled).map((account) => account.id)
648
+ );
649
+ const pinDisposition = classifyCodexPin({
650
+ pinnedCredentialId: args.sessionPinnedCredentialId,
651
+ pinSource: args.sessionPinSource,
652
+ strategy,
653
+ rotationEnabled
654
+ });
655
+ if (pinDisposition === "manual" && args.sessionPinnedCredentialId) {
656
+ const pinned = accounts.find((account) => account.id === args.sessionPinnedCredentialId);
657
+ if (!pinned?.allocatorEnabled) {
658
+ return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
659
+ }
660
+ if (isCodexCredentialHealthy(pinned, args.now)) {
661
+ return {
662
+ credentialId: pinned.id,
663
+ decision: { kind: "active", credentialId: pinned.id, moved: false },
664
+ advanceActivePointer: false
665
+ };
666
+ }
667
+ return {
668
+ credentialId: null,
669
+ decision: {
670
+ kind: "allCapped",
671
+ earliestResetAt: availableAt(pinned, args.now)
672
+ },
673
+ advanceActivePointer: false
674
+ };
675
+ }
676
+ if (pinDisposition === "sharded") {
677
+ if (accounts.length === 0) {
678
+ return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
679
+ }
680
+ const shard = chooseShardedHome({
681
+ sessionId: args.sessionId,
682
+ currentPolicyPin: args.sessionPinSource === "policy" ? args.sessionPinnedCredentialId : null,
683
+ accounts,
684
+ now: args.now
685
+ });
686
+ if (shard.kind === "allCapped") {
687
+ return {
688
+ credentialId: null,
689
+ decision: shard,
690
+ advanceActivePointer: false
691
+ };
692
+ }
693
+ return {
694
+ credentialId: shard.credentialId,
695
+ decision: {
696
+ kind: "active",
697
+ credentialId: shard.credentialId,
698
+ moved: shard.credentialId !== activeCredentialId
699
+ },
700
+ advanceActivePointer: false
701
+ };
702
+ }
703
+ if (!rotationEnabled) {
704
+ const active = activeCredentialId ? accounts.find((account) => account.id === activeCredentialId) : void 0;
705
+ if (!active?.allocatorEnabled) {
706
+ return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
707
+ }
708
+ if (!isCodexCredentialHealthy(active, args.now)) {
709
+ return {
710
+ credentialId: null,
711
+ decision: { kind: "allCapped", earliestResetAt: availableAt(active, args.now) },
712
+ advanceActivePointer: false
713
+ };
714
+ }
715
+ return {
716
+ credentialId: active.id,
717
+ decision: { kind: "active", credentialId: active.id, moved: false },
718
+ advanceActivePointer: false
719
+ };
720
+ }
721
+ if (!leasingEnabled && pinDisposition !== "clearStale" && args.sessionPinnedCredentialId) {
722
+ const credentialId = connectedIds.has(args.sessionPinnedCredentialId) ? args.sessionPinnedCredentialId : activeCredentialId && connectedIds.has(activeCredentialId) ? activeCredentialId : null;
723
+ return {
724
+ credentialId,
725
+ decision: credentialId ? { kind: "active", credentialId, moved: false } : { kind: "none" }
726
+ };
727
+ }
728
+ const priorId = args.sessionLastCredentialId ?? activeCredentialId;
729
+ const choose = leasingEnabled ? chooseRotationActive : chooseLegacyRotationActive;
730
+ const decision = choose({
731
+ rotationStrategy: strategy,
732
+ activeCredentialId,
733
+ // Per-session continuity is the round-robin/drain cursor. Falling back to
734
+ // the workspace pointer is only correct when this session has never run.
735
+ priorCredentialId: priorId,
736
+ accounts,
737
+ now: args.now
738
+ });
739
+ return {
740
+ credentialId: decision.kind === "active" ? decision.credentialId : null,
741
+ decision
742
+ };
743
+ }
744
+
745
+ // src/activities/codex-capacity.ts
746
+ async function refreshCodexUsageAndRepairCapacityWaiters(refreshes, repairPendingWakes) {
747
+ await Promise.all(refreshes.map((refresh) => refresh().catch(() => void 0)));
748
+ await repairPendingWakes();
749
+ }
750
+ async function signalCodexCapacityWakeTargets(services, targets) {
751
+ await Promise.allSettled(
752
+ targets.map(
753
+ (target) => services.signalCodexCapacityWorkflow ? services.signalCodexCapacityWorkflow({
754
+ accountId: target.accountId,
755
+ workspaceId: target.workspaceId,
756
+ sessionId: target.sessionId,
757
+ workflowId: target.workflowId,
758
+ wakeRevision: target.wakeRevision
759
+ }) : services.wakeSessionWorkflow ? services.wakeSessionWorkflow({
760
+ accountId: target.accountId,
761
+ workspaceId: target.workspaceId,
762
+ sessionId: target.sessionId,
763
+ workflowId: target.workflowId,
764
+ wakeRevision: target.workflowWakeRevision
765
+ }) : Promise.resolve()
766
+ )
767
+ );
768
+ }
769
+ async function signalPendingCodexCapacityWakeTargets(services, workspaceId) {
770
+ const targets = await listPendingCodexCapacityWakeTargets(services.db, workspaceId).catch(
771
+ () => []
772
+ );
773
+ await signalCodexCapacityWakeTargets(services, targets);
774
+ }
775
+ function codexCapacityDecision(context, settings) {
776
+ const now = /* @__PURE__ */ new Date();
777
+ const selected = selectCodexCredentialLeaseForTurn({
778
+ context,
779
+ leasingEnabled: settings.codexCredentialLeasingEnabled,
780
+ sessionId: context.sessionId,
781
+ sessionPinnedCredentialId: context.sessionPinnedCredentialId,
782
+ sessionPinSource: context.sessionPinSource,
783
+ sessionLastCredentialId: context.sessionLastCredentialId,
784
+ now
785
+ });
786
+ const selectedAccount = selected.credentialId ? context.accounts.find((account) => account.id === selected.credentialId) : void 0;
787
+ const selectedIsAvailable = selectedAccount !== void 0 && (selectedAccount.id === context.existingCredentialId ? isCodexCredentialHealthy(selectedAccount, now) : isCodexCredentialEligible(selectedAccount, now));
788
+ if (selected.credentialId && selectedIsAvailable) {
789
+ return {
790
+ kind: "available",
791
+ credentialId: selected.credentialId,
792
+ diagnostic: {
793
+ connectedCount: context.accounts.length,
794
+ eligibleCount: context.accounts.filter((account) => isCodexCredentialEligible(account, now)).length
795
+ }
796
+ };
797
+ }
798
+ const authoritativeReset = authoritativeCodexCapacityResetAt(context.accounts, now);
799
+ return {
800
+ kind: "unavailable",
801
+ earliestResetAt: authoritativeReset,
802
+ resetKind: authoritativeReset ? "authoritative" : "bounded_refresh",
803
+ diagnostic: {
804
+ connectedCount: context.accounts.length,
805
+ allocatorEnabledCount: context.accounts.filter((account) => account.allocatorEnabled).length,
806
+ policyHash: context.policyHash
807
+ }
808
+ };
809
+ }
810
+ async function refreshCapacityMetadata(services, workspaceId) {
811
+ const accounts = await listCodexAccountStatuses(services.db, workspaceId).catch(() => []);
812
+ const stale = accounts.filter(
813
+ (account) => account.allocatorEnabled && account.status === "active" && ((account.primaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT || (account.secondaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT || account.usageCheckedAt === null)
814
+ );
815
+ await refreshCodexUsageAndRepairCapacityWaiters(
816
+ stale.map(
817
+ (account) => () => fetchCodexUsageForAccount(services.db, services.settings, workspaceId, account.id)
818
+ ),
819
+ () => signalPendingCodexCapacityWakeTargets(services, workspaceId)
820
+ );
821
+ }
822
+ function createCodexCapacityActivities(services) {
823
+ async function getCodexCapacityWait(input) {
824
+ const { db } = await services();
825
+ const codexWaiter = await getCodexCapacityWaitForSession(
826
+ db,
827
+ input.workspaceId,
828
+ input.sessionId
829
+ );
830
+ const xaiWaiter = codexWaiter ? null : await getXaiCapacityWaitForSession(db, input.workspaceId, input.sessionId);
831
+ const waiter = codexWaiter ?? xaiWaiter;
832
+ return waiter ? {
833
+ ...xaiWaiter ? { provider: "xai" } : {},
834
+ waiterId: waiter.id,
835
+ generation: waiter.generation,
836
+ // A capacity mutation may have committed while its Temporal signal
837
+ // was lost or while the workflow continued-as-new. Reconstruct that
838
+ // outbox edge as an immediate re-evaluation rather than waiting for
839
+ // the older timer.
840
+ nextCheckAt: waiter.wakeRevision > waiter.observedWakeRevision ? (/* @__PURE__ */ new Date(0)).toISOString() : waiter.nextCheckAt.toISOString(),
841
+ wakeRevision: waiter.wakeRevision
842
+ } : null;
843
+ }
844
+ async function reconcileCodexCapacityWait(input) {
845
+ const resolved = await services();
846
+ if (input.provider === "xai") {
847
+ const current2 = await getXaiCapacityWaitForSession(
848
+ resolved.db,
849
+ input.workspaceId,
850
+ input.sessionId
851
+ );
852
+ if (!current2 || current2.id !== input.waiterId || current2.generation !== input.generation) {
853
+ return { action: "stale" };
854
+ }
855
+ const result2 = await reconcileXaiCapacityWaitDb(resolved.db, {
856
+ accountId: input.accountId,
857
+ workspaceId: input.workspaceId,
858
+ sessionId: input.sessionId,
859
+ waiterId: input.waiterId,
860
+ generation: input.generation
861
+ });
862
+ if (result2.events.length > 0) {
863
+ try {
864
+ await resolved.bus.publish(input.workspaceId, input.sessionId, result2.events);
865
+ } catch {
866
+ }
867
+ }
868
+ if (result2.action === "resumed") return { action: "resumed" };
869
+ if (result2.action === "waiting") {
870
+ return {
871
+ action: "waiting",
872
+ provider: "xai",
873
+ waiterId: result2.waiter.id,
874
+ generation: result2.waiter.generation,
875
+ nextCheckAt: result2.waiter.nextCheckAt.toISOString(),
876
+ wakeRevision: result2.waiter.wakeRevision
877
+ };
878
+ }
879
+ return { action: result2.action };
880
+ }
881
+ const current = await getCodexCapacityWaitForSession(
882
+ resolved.db,
883
+ input.workspaceId,
884
+ input.sessionId
885
+ );
886
+ if (!current || current.id !== input.waiterId || current.generation !== input.generation) {
887
+ return { action: "stale" };
888
+ }
889
+ if (input.cause === "timer" && current.nextCheckAt.getTime() <= Date.now()) {
890
+ await refreshCapacityMetadata(resolved, input.workspaceId);
891
+ }
892
+ const result = await reconcileCodexCapacityWaitDb(
893
+ resolved.db,
894
+ {
895
+ accountId: input.accountId,
896
+ workspaceId: input.workspaceId,
897
+ sessionId: input.sessionId,
898
+ waiterId: input.waiterId,
899
+ generation: input.generation
900
+ },
901
+ (context) => codexCapacityDecision(context, resolved.settings)
902
+ );
903
+ if (result.events.length > 0) {
904
+ try {
905
+ await resolved.bus.publish(input.workspaceId, input.sessionId, result.events);
906
+ } catch {
907
+ }
908
+ }
909
+ if (result.action === "resumed") {
910
+ return { action: "resumed" };
911
+ }
912
+ if (result.action === "waiting") {
913
+ return {
914
+ action: "waiting",
915
+ waiterId: result.waiter.id,
916
+ generation: result.waiter.generation,
917
+ nextCheckAt: result.waiter.nextCheckAt.toISOString(),
918
+ wakeRevision: result.waiter.wakeRevision
919
+ };
920
+ }
921
+ return { action: result.action };
922
+ }
923
+ return { getCodexCapacityWait, reconcileCodexCapacityWait };
924
+ }
925
+
926
+ // src/activities/goals.ts
927
+ import {
928
+ allowedFirstPartyMcpToolsForSession,
929
+ configuredStaticUsageLimits,
930
+ policyProviderIdForModel
931
+ } from "@opengeni/config";
932
+ import {
933
+ evaluateWorkspaceModelPolicy,
934
+ mergeToolRefs
935
+ } from "@opengeni/contracts";
936
+ import { isCodexBilledModel } from "@opengeni/codex";
937
+ import {
938
+ enqueueSessionWorkflowWakeIfRunnable,
939
+ getBillingBalance,
940
+ getLatestStartedSessionTurn,
941
+ getWorkspaceModelPolicy,
942
+ getSessionGoal,
943
+ isCodexBilledTurn,
944
+ materializeGoalContinuation,
945
+ requireSession,
946
+ sumUsageQuantity
947
+ } from "@opengeni/db";
948
+ function createGoalActivities(services) {
949
+ async function enqueueGoalRetryWake(input) {
950
+ const { db } = await services();
951
+ await enqueueSessionWorkflowWakeIfRunnable(db, {
952
+ accountId: input.accountId,
953
+ workspaceId: input.workspaceId,
954
+ sessionId: input.sessionId,
955
+ temporalWorkflowId: input.workflowId,
956
+ reason: "goal_retry",
957
+ // A permanently invalid goal must not become a tight workflow loop. One
958
+ // durable retry after a short delay preserves liveness without scanning.
959
+ notBefore: new Date(Date.now() + 3e4)
960
+ });
961
+ }
962
+ async function maybeContinueGoal(input) {
963
+ const { settings, db, bus } = await services();
964
+ const existingGoal = await getSessionGoal(db, input.workspaceId, input.sessionId);
965
+ if (!existingGoal || existingGoal.status !== "active") {
966
+ return { action: "none" };
967
+ }
968
+ const session = await requireSession(db, input.workspaceId, input.sessionId);
969
+ const latestStartedTurn = await getLatestStartedSessionTurn(
970
+ db,
971
+ input.workspaceId,
972
+ input.sessionId
973
+ );
974
+ let continuationModel = latestStartedTurn?.model ?? session.model;
975
+ const continuationReasoningEffort = latestStartedTurn?.reasoningEffort ?? session.reasoningEffort;
976
+ const continuationLatencyMode = latestStartedTurn?.latencyMode ?? session.latencyMode;
977
+ let modelPolicyBlocked = null;
978
+ const workspaceModelPolicy = await getWorkspaceModelPolicy(db, input.workspaceId);
979
+ if (workspaceModelPolicy) {
980
+ const policyBlocks = (modelId) => !evaluateWorkspaceModelPolicy(workspaceModelPolicy, {
981
+ providerId: policyProviderIdForModel(settings, modelId),
982
+ modelId
983
+ }).allowed;
984
+ if (policyBlocks(continuationModel)) {
985
+ if (continuationModel !== session.model && !policyBlocks(session.model)) {
986
+ continuationModel = session.model;
987
+ } else {
988
+ modelPolicyBlocked = `workspace model policy blocks model "${continuationModel}"; pick an allowed model or change the workspace model policy`;
989
+ }
990
+ }
991
+ }
992
+ if (!modelPolicyBlocked && session.codexCompactionMode === "remote_v2" && !isCodexBilledModel(continuationModel)) {
993
+ modelPolicyBlocked = `session is locked to Codex remote compaction v2; model "${continuationModel}" is not a Codex subscription model`;
994
+ }
995
+ const isCodexRun = await isCodexBilledTurn({
996
+ db,
997
+ settings,
998
+ workspaceId: input.workspaceId,
999
+ model: continuationModel
1000
+ });
1001
+ const budgetBlocked = await goalRunBudgetBlocked(
1002
+ settings,
1003
+ db,
1004
+ input.accountId,
1005
+ input.workspaceId,
1006
+ isCodexRun
1007
+ );
1008
+ const decision = await materializeGoalContinuation(db, {
1009
+ accountId: input.accountId,
1010
+ workspaceId: input.workspaceId,
1011
+ sessionId: input.sessionId,
1012
+ workflowId: input.workflowId,
1013
+ defaultMaxAutoContinuations: settings.goalMaxAutoContinuations ?? null,
1014
+ // Pacing between consecutive no-input continuations (never a cap). The
1015
+ // materializer re-arms a delayed outbox wake and returns `deferred`.
1016
+ idleBackoff: {
1017
+ scheduleMs: settings.goalIdleBackoffMs,
1018
+ maxMs: settings.goalIdleBackoffMaxMs
1019
+ },
1020
+ // A model-policy block takes precedence: it is deterministic (a budget
1021
+ // pause can clear on its own; a policy pause needs a model/policy change)
1022
+ // and rides the same visible-pause channel.
1023
+ budgetBlocked: modelPolicyBlocked ?? budgetBlocked,
1024
+ policy: {
1025
+ model: continuationModel,
1026
+ reasoningEffort: continuationReasoningEffort,
1027
+ latencyMode: continuationLatencyMode,
1028
+ tools: withFirstPartyTools(settings, session.tools),
1029
+ sandboxBackend: session.sandboxBackend
1030
+ },
1031
+ // The hold guidance is only given when `goal_wait` is actually in this
1032
+ // session's effective first-party selection (the same source the worker
1033
+ // signs into the delegated token and the API uses to register tools), so
1034
+ // a pre-existing narrowed selection is never told to call a missing tool.
1035
+ prompt: (goal, autoContinuation, cap) => {
1036
+ const effectiveFirstPartyTools = allowedFirstPartyMcpToolsForSession(
1037
+ settings,
1038
+ session.firstPartyMcpTools
1039
+ );
1040
+ return goalContinuationPrompt(goal, autoContinuation, cap, {
1041
+ goalWaitAvailable: effectiveFirstPartyTools.includes("goal_wait"),
1042
+ humanInputRespondAvailable: effectiveFirstPartyTools.includes(
1043
+ "session_human_input_respond"
1044
+ )
1045
+ });
1046
+ }
1047
+ });
1048
+ if (decision.events.length > 0) {
1049
+ await bus.publish(input.workspaceId, input.sessionId, decision.events);
1050
+ }
1051
+ return { action: decision.action };
1052
+ }
1053
+ return {
1054
+ enqueueGoalRetryWake,
1055
+ maybeContinueGoal
1056
+ };
1057
+ }
1058
+ function goalContinuationPrompt(_goal, _autoContinuation, _cap, options = {}) {
1059
+ const waitingGuidance = options.goalWaitAvailable ? [
1060
+ "Waiting on child sessions or external events:",
1061
+ "- When the next progress depends on child sessions you spawned or on an external event, do not sleep, loop, or poll sessions_list/session_get/session_events to wait for it.",
1062
+ "- Re-check sessions_list or session_get once; if the work is still in flight, call opengeni__goal_wait with a concrete reason and a deadline (untilSeconds), then end your turn immediately. You will be woken by a child result, a message, a human prompt, or at the deadline, and this goal stays active.",
1063
+ "- A hold is for child/external progress only. If you are blocked on a human decision, use opengeni__goal_pause under the blocked audit below instead.",
1064
+ ""
1065
+ ] : [];
1066
+ const childNoticeGuidance = [
1067
+ "Child lifecycle notices:",
1068
+ "- A `child_requires_action` update means a worker you spawned is blocked on a question or a tool approval and will not progress until it is answered. " + (options.humanInputRespondAvailable ? "If you know the answer to its question, answer it with opengeni__session_human_input_respond (pass the worker's sessionId, the requestId from the notice, and a response). " : "") + "Tool approvals can only be decided by a human. If you cannot resolve the blocker yourself, report the exact blocker (worker session id, the question) to the user and wait or pause instead of retrying the worker.",
1069
+ "- `child_requires_action_resolved`, `child_paused`, `child_waiting_capacity`, and `child_progress` updates are informational: a resolved notice means the worker is moving again; a paused worker needs a human or you to resume it; a capacity wait resumes by itself; a progress note needs no action.",
1070
+ ""
1071
+ ];
1072
+ return [
1073
+ "Continue working toward the active session goal.",
1074
+ "",
1075
+ "Continuation behavior:",
1076
+ "- This goal persists across turns. Ending this turn does not require shrinking the objective to what fits now.",
1077
+ "- Keep the full objective intact. If it cannot be finished now, make concrete progress toward the real requested end state, leave the goal active, and do not redefine success around a smaller or easier task.",
1078
+ "- Temporary rough edges are acceptable while the work is moving in the right direction. Completion still requires the requested end state to be true and verified.",
1079
+ "",
1080
+ "Work from evidence:",
1081
+ "Use the current workspace and external state as authoritative. Previous conversation context can help locate relevant work, but inspect the current state before relying on it. Improve, replace, or remove existing work as needed to satisfy the actual objective.",
1082
+ "",
1083
+ "Progress visibility:",
1084
+ "If a planning tool is available and the next work is meaningfully multi-step, use it to show a concise plan tied to the real objective. Keep the plan current as steps complete or the next best action changes. Skip planning overhead for trivial one-step progress, and do not treat a plan update as a substitute for doing the work.",
1085
+ "",
1086
+ "Fidelity:",
1087
+ "- Optimize each turn for movement toward the requested end state, not for the smallest stable-looking subset or easiest passing change.",
1088
+ "- Do not substitute a narrower, safer, smaller, merely compatible, or easier-to-test solution because it is more likely to pass current tests.",
1089
+ "- Treat alignment as movement toward the requested end state. An edit is aligned only if it makes the requested final state more true; useful-looking behavior that preserves a different end state is misaligned.",
1090
+ "",
1091
+ "Completion audit:",
1092
+ "Before deciding that the goal is achieved, treat completion as unproven and verify it against the actual current state:",
1093
+ "- Derive concrete requirements from the objective and any referenced files, plans, specifications, issues, or user instructions.",
1094
+ "- Preserve the original scope; do not redefine success around the work that already exists.",
1095
+ "- For every explicit requirement, named artifact, command, test, gate, invariant, and deliverable, identify and inspect the authoritative evidence that would prove it.",
1096
+ "- Match verification scope to requirement scope. Treat uncertain, indirect, incomplete, or missing evidence as not achieved and continue working.",
1097
+ "- The audit must prove completion, not merely fail to find obvious remaining work.",
1098
+ "",
1099
+ "Do not rely on intent, partial progress, memory of earlier work, or a plausible final answer as proof of completion. Call opengeni__goal_complete with concrete evidence only when the full objective is actually achieved and no required work remains.",
1100
+ "",
1101
+ ...waitingGuidance,
1102
+ ...childNoticeGuidance,
1103
+ "Blocked audit:",
1104
+ "- Do not call opengeni__goal_pause the first time a blocker appears.",
1105
+ "- Pause only when the same blocking condition has repeated for at least three consecutive goal turns and meaningful progress is impossible without user input or an external-state change.",
1106
+ "- Do not pause merely because the work is hard, slow, uncertain, incomplete, or would benefit from clarification.",
1107
+ "- Once that threshold is satisfied, call opengeni__goal_pause with the concrete blocker instead of repeatedly reporting it while leaving the goal active.",
1108
+ "",
1109
+ "Do not call opengeni__goal_complete or opengeni__goal_pause unless the corresponding audit above is satisfied."
1110
+ ].join("\n");
1111
+ }
1112
+ function withFirstPartyTools(settings, tools) {
1113
+ if (!settings.mcpServers.some((server) => server.id === "opengeni")) {
1114
+ return tools;
1115
+ }
1116
+ return mergeToolRefs(tools, [{ kind: "mcp", id: "opengeni" }]);
1117
+ }
1118
+ async function goalRunBudgetBlocked(settings, db, accountId, workspaceId, isCodexRun) {
1119
+ if (!isCodexRun && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
1120
+ const balance = await getBillingBalance(db, accountId);
1121
+ if (balance.balanceMicros <= 0) {
1122
+ return "insufficient OpenGeni credits";
1123
+ }
1124
+ }
1125
+ if (settings.usageLimitsMode === "static" || settings.usageLimitsMode === "managed") {
1126
+ const limits = configuredStaticUsageLimits(settings);
1127
+ if (!isCodexRun && limits.maxMonthlyCostMicrosPerAccount) {
1128
+ const used = await sumUsageQuantity(db, {
1129
+ accountId,
1130
+ eventType: "model.cost",
1131
+ since: startOfUtcMonth()
1132
+ });
1133
+ if (used >= limits.maxMonthlyCostMicrosPerAccount) {
1134
+ return `monthly model cost limit reached (${limits.maxMonthlyCostMicrosPerAccount} micros)`;
1135
+ }
1136
+ }
1137
+ if (limits.maxMonthlyAgentRunsPerWorkspace) {
1138
+ const used = await sumUsageQuantity(db, {
1139
+ workspaceId,
1140
+ eventType: "agent_run.created",
1141
+ since: startOfUtcMonth()
1142
+ });
1143
+ if (used + 1 > limits.maxMonthlyAgentRunsPerWorkspace) {
1144
+ return `monthly agent run limit reached (${limits.maxMonthlyAgentRunsPerWorkspace})`;
1145
+ }
1146
+ }
1147
+ }
1148
+ return null;
1149
+ }
1150
+ function startOfUtcMonth() {
1151
+ const now = /* @__PURE__ */ new Date();
1152
+ return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
1153
+ }
1154
+
1155
+ // src/activities/sandbox-images.ts
1156
+ import {
1157
+ resolveRigProviderImageSelection,
1158
+ rigProviderImageSourceImage,
1159
+ settingsWithPackSandboxImage,
1160
+ settingsWithRigImage,
1161
+ settingsWithRigProviderImage
1162
+ } from "@opengeni/core";
1163
+
1164
+ // src/activities/packs.ts
1165
+ import { getWorkspace, listInstalledPortableSkills } from "@opengeni/db";
1166
+ import {
1167
+ legacySandboxRuntimeFromPacks,
1168
+ packInstallationUsesLegacyRuntime,
1169
+ resolveWorkspaceLegacyRuntimePacks
1170
+ } from "@opengeni/core";
1171
+ import {
1172
+ buildPortableSkillArtifact
1173
+ } from "@opengeni/runtime";
1174
+ var emptyPackRuntime = {
1175
+ sandboxImage: null,
1176
+ sandboxProviderImages: null,
1177
+ skillActivations: []
1178
+ };
1179
+ async function resolveWorkspacePackRuntime(db, workspaceId) {
1180
+ const packs = await resolveWorkspaceLegacyRuntimePacks(db, workspaceId);
1181
+ if (packs.length === 0) return emptyPackRuntime;
1182
+ return workspacePackRuntimeFromPacks(packs);
1183
+ }
1184
+ async function resolveWorkspaceInstalledSkillRuntime(db, workspaceId) {
1185
+ const installedSkills = await listInstalledPortableSkills(db, workspaceId);
1186
+ const activations = [];
1187
+ for (const installed of installedSkills) {
1188
+ const artifact = buildPortableSkillArtifact(installed.files);
1189
+ if (artifact.name !== installed.name || artifact.description !== installed.description || artifact.contentSha256 !== installed.contentSha256) {
1190
+ throw new Error(
1191
+ `Installed Skill artifact verification failed for ${installed.capabilityId}; reinstall it from the pinned source`
1192
+ );
1193
+ }
1194
+ activations.push({
1195
+ source: "installation",
1196
+ id: installed.capabilityId,
1197
+ artifact: {
1198
+ name: artifact.name,
1199
+ description: artifact.description,
1200
+ files: artifact.files.map((file) => ({ path: file.path, content: file.content }))
1201
+ },
1202
+ version: installed.version,
1203
+ contentSha256: artifact.contentSha256,
1204
+ reason: installed.source === "library" ? "installed from the curated Skill library" : installed.source === "pack" ? "installed as a Pack-owned Skill" : `installed from ${installed.sourceUrl}`
1205
+ });
1206
+ }
1207
+ return { activations };
1208
+ }
1209
+ function workspacePackRuntimeFromPacks(packs) {
1210
+ const sandboxRuntime = legacySandboxRuntimeFromPacks(packs);
1211
+ const skillActivations = [];
1212
+ const skillOwners = /* @__PURE__ */ new Map();
1213
+ for (const pack of [...packs].sort((a, b) => a.id.localeCompare(b.id))) {
1214
+ for (const skill of pack.skills) {
1215
+ const key = skill.name.toLowerCase();
1216
+ const existingOwner = skillOwners.get(key);
1217
+ if (existingOwner !== void 0 && existingOwner !== pack.id) {
1218
+ throw new Error(
1219
+ `Enabled packs ${existingOwner} and ${pack.id} both declare a skill named "${skill.name}". Pack skill names must be unique across enabled packs; disable one of the packs and retry.`
1220
+ );
1221
+ }
1222
+ skillOwners.set(key, pack.id);
1223
+ const artifact = {
1224
+ name: skill.name,
1225
+ description: skill.description ?? null,
1226
+ files: skill.files.map((file) => ({ path: file.path, content: file.content }))
1227
+ };
1228
+ skillActivations.push({
1229
+ source: "pack",
1230
+ id: `pack:${pack.id}:${skill.name}`,
1231
+ artifact,
1232
+ reason: `active legacy Pack ${pack.id}`
1233
+ });
1234
+ }
1235
+ }
1236
+ return {
1237
+ sandboxImage: sandboxRuntime.sandboxImage,
1238
+ sandboxProviderImages: sandboxRuntime.sandboxProviderImages,
1239
+ skillActivations
1240
+ };
1241
+ }
1242
+
1243
+ // src/activities/streaming.ts
1244
+ import { TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS } from "@opengeni/core";
1245
+ import { Context } from "@temporalio/activity";
1246
+ var TRAILING_FLUSH_MS = 33;
1247
+ var MAX_BATCH_EVENTS = 50;
1248
+ var MAX_BUFFERED_EVENTS = 1e3;
1249
+ function createRuntimeBatcher(flushEvents, options = {}) {
1250
+ const now = options.now ?? (() => performance.now());
1251
+ let pending = [];
1252
+ let lastFlush = Date.now();
1253
+ let trailingTimer = null;
1254
+ let activeBatchSize = 0;
1255
+ let drainPromise = null;
1256
+ let flushFailed = false;
1257
+ let flushFailure;
1258
+ const structural = /* @__PURE__ */ new Set([
1259
+ "terminal.pty.output.delta",
1260
+ "agent.toolCall.created",
1261
+ "agent.toolCall.output",
1262
+ "agent.message.completed",
1263
+ "tool.auth_needed",
1264
+ "credential.auth_needed",
1265
+ "session.requiresAction",
1266
+ "turn.completed",
1267
+ "turn.failed",
1268
+ "turn.cancelled"
1269
+ ]);
1270
+ return {
1271
+ push: async (event) => {
1272
+ throwIfFlushFailed();
1273
+ pending.push({ type: event.type, payload: event.payload });
1274
+ const elapsed = Date.now() - lastFlush;
1275
+ if (structural.has(event.type)) {
1276
+ await flush();
1277
+ return;
1278
+ }
1279
+ if (pending.length >= MAX_BATCH_EVENTS || elapsed >= TRAILING_FLUSH_MS) {
1280
+ void startDrain().catch(() => void 0);
1281
+ }
1282
+ if (activeBatchSize + pending.length >= MAX_BUFFERED_EVENTS) {
1283
+ await flush();
1284
+ return;
1285
+ }
1286
+ armTrailingTimer();
1287
+ },
1288
+ flush
1289
+ };
1290
+ function armTrailingTimer() {
1291
+ if (trailingTimer !== null) {
1292
+ return;
1293
+ }
1294
+ trailingTimer = setTimeout(() => {
1295
+ trailingTimer = null;
1296
+ void flush().catch(() => void 0);
1297
+ }, TRAILING_FLUSH_MS);
1298
+ if ("unref" in trailingTimer && typeof trailingTimer.unref === "function") {
1299
+ trailingTimer.unref();
1300
+ }
1301
+ }
1302
+ function clearTrailingTimer() {
1303
+ if (trailingTimer !== null) {
1304
+ clearTimeout(trailingTimer);
1305
+ trailingTimer = null;
1306
+ }
1307
+ }
1308
+ async function flush() {
1309
+ throwIfFlushFailed();
1310
+ clearTrailingTimer();
1311
+ await startDrain();
1312
+ throwIfFlushFailed();
1313
+ }
1314
+ function startDrain() {
1315
+ if (drainPromise) {
1316
+ return drainPromise;
1317
+ }
1318
+ if (flushFailed) {
1319
+ return Promise.reject(flushFailure);
1320
+ }
1321
+ if (pending.length === 0) {
1322
+ return Promise.resolve();
1323
+ }
1324
+ clearTrailingTimer();
1325
+ const running = (async () => {
1326
+ while (pending.length > 0) {
1327
+ const events = pending.splice(0, MAX_BATCH_EVENTS);
1328
+ activeBatchSize = events.length;
1329
+ lastFlush = Date.now();
1330
+ const startedAt = now();
1331
+ try {
1332
+ await flushEvents(events);
1333
+ } catch (error) {
1334
+ flushFailed = true;
1335
+ flushFailure = error;
1336
+ throw error;
1337
+ } finally {
1338
+ activeBatchSize = 0;
1339
+ if (options.onFlush) {
1340
+ try {
1341
+ options.onFlush({
1342
+ events: events.length,
1343
+ durationSeconds: Math.max(0, (now() - startedAt) / 1e3)
1344
+ });
1345
+ } catch {
1346
+ }
1347
+ }
1348
+ }
1349
+ }
1350
+ })();
1351
+ drainPromise = running.finally(() => {
1352
+ drainPromise = null;
1353
+ });
1354
+ return drainPromise;
1355
+ }
1356
+ function throwIfFlushFailed() {
1357
+ if (flushFailed) {
1358
+ throw flushFailure;
1359
+ }
1360
+ }
1361
+ }
1362
+ function currentActivityContext() {
1363
+ try {
1364
+ return Context.current();
1365
+ } catch {
1366
+ return null;
1367
+ }
1368
+ }
1369
+ function startActivityHeartbeat(context, details, schedule = setInterval) {
1370
+ if (!context) {
1371
+ return null;
1372
+ }
1373
+ const timer = schedule(() => {
1374
+ context.heartbeat({ ...details, at: (/* @__PURE__ */ new Date()).toISOString() });
1375
+ }, TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS);
1376
+ if ("unref" in timer && typeof timer.unref === "function") {
1377
+ timer.unref();
1378
+ }
1379
+ return timer;
1380
+ }
1381
+ async function nextStreamEvent(iterator, context) {
1382
+ if (!context) {
1383
+ return await iterator.next();
1384
+ }
1385
+ const signal = context.cancellationSignal;
1386
+ if (signal.aborted) {
1387
+ throw signal.reason;
1388
+ }
1389
+ return await new Promise((resolve, reject) => {
1390
+ let settled = false;
1391
+ const settle = (complete) => {
1392
+ if (settled) return;
1393
+ settled = true;
1394
+ signal.removeEventListener("abort", onAbort);
1395
+ complete();
1396
+ };
1397
+ const onAbort = () => settle(() => reject(signal.reason));
1398
+ signal.addEventListener("abort", onAbort, { once: true });
1399
+ if (signal.aborted) {
1400
+ onAbort();
1401
+ return;
1402
+ }
1403
+ let next;
1404
+ try {
1405
+ next = iterator.next();
1406
+ } catch (error) {
1407
+ settle(() => reject(error));
1408
+ return;
1409
+ }
1410
+ void Promise.resolve(next).then(
1411
+ (result) => settle(() => resolve(result)),
1412
+ (error) => settle(() => reject(error))
1413
+ );
1414
+ });
1415
+ }
1416
+
1417
+ // src/activities/parent-wake.ts
1418
+ import {
1419
+ getSession,
1420
+ getSessionGoal as getSessionGoal2,
1421
+ getSessionParentPersonalConnectionDelegations,
1422
+ getSessionParentXaiProviderAccountAuthority,
1423
+ addSessionSystemUpdateWithSourceMutation,
1424
+ claimPendingSessionSystemUpdateOutbox,
1425
+ claimPendingSessionWorkflowWakes,
1426
+ childRequiresActionDedupeKey,
1427
+ getSessionSystemUpdateOutboxByDedupeKey,
1428
+ getOrCreateSessionSystemUpdateOutbox,
1429
+ markSessionWorkflowWakeFailed,
1430
+ markSessionSystemUpdateOutboxDeliveredInTransaction,
1431
+ markSessionSystemUpdateOutboxFailed,
1432
+ sessionSystemUpdateOutboxKindPayload
1433
+ } from "@opengeni/db";
1434
+ async function notifyParentOfChildIdle(svc, workspaceId, childSessionId, episodeKey) {
1435
+ try {
1436
+ const child = await getSession(svc.db, workspaceId, childSessionId);
1437
+ if (!child || !child.parentSessionId) {
1438
+ return;
1439
+ }
1440
+ const goal = await getSessionGoal2(svc.db, workspaceId, childSessionId);
1441
+ const clientEventId = `child-completion:${childSessionId}:${episodeKey}`;
1442
+ const payload = childCompletionPayload(child, goal);
1443
+ const personalConnectionDelegations = await getSessionParentPersonalConnectionDelegations(
1444
+ svc.db,
1445
+ workspaceId,
1446
+ childSessionId
1447
+ );
1448
+ const xaiAuthority = await getSessionParentXaiProviderAccountAuthority(
1449
+ svc.db,
1450
+ workspaceId,
1451
+ childSessionId
1452
+ );
1453
+ const outbox = await getOrCreateSessionSystemUpdateOutbox(svc.db, {
1454
+ accountId: child.accountId,
1455
+ workspaceId,
1456
+ sourceSessionId: child.id,
1457
+ targetSessionId: child.parentSessionId,
1458
+ kind: "child_terminal_result",
1459
+ classification: goal?.status === "paused" ? "action_required" : "success",
1460
+ sourceId: child.id,
1461
+ dedupeKey: clientEventId,
1462
+ summary: childCompletionSummary(child, goal, "idle"),
1463
+ payload,
1464
+ lineage: {
1465
+ childSessionId: child.id,
1466
+ parentSessionId: child.parentSessionId,
1467
+ ...xaiAuthority.subjectId ? { xaiAuthoritySubjectId: xaiAuthority.subjectId } : {}
1468
+ },
1469
+ personalConnectionDelegations,
1470
+ xaiProviderAccountAuthoritySnapshot: xaiAuthority.snapshot
1471
+ });
1472
+ if (outbox.status === "delivered") {
1473
+ return;
1474
+ }
1475
+ await deliverParentSystemUpdateOutbox(svc, outbox);
1476
+ } catch (error) {
1477
+ svc.observability.error("Failed to wake parent session on worker idle boundary", {
1478
+ childSessionId,
1479
+ error: error instanceof Error ? error.message : String(error)
1480
+ });
1481
+ }
1482
+ }
1483
+ async function deliverChildLifecycleOutboxToParent(svc, workspaceId, childSessionId, dedupeKey, options = {}) {
1484
+ try {
1485
+ const child = await getSession(svc.db, workspaceId, childSessionId);
1486
+ if (!child || !child.parentSessionId) return;
1487
+ const outbox = await getSessionSystemUpdateOutboxByDedupeKey(svc.db, {
1488
+ accountId: child.accountId,
1489
+ workspaceId,
1490
+ dedupeKey
1491
+ });
1492
+ if (!outbox) {
1493
+ if (options.requireRow === false) return;
1494
+ throw new Error(`Committed child-lifecycle outbox disappeared: ${dedupeKey}`);
1495
+ }
1496
+ if (outbox.status === "delivered") return;
1497
+ await deliverParentSystemUpdateOutbox(svc, outbox);
1498
+ } catch (error) {
1499
+ svc.observability.error("Failed to deliver committed child-lifecycle notice", {
1500
+ childSessionId,
1501
+ dedupeKey,
1502
+ error: error instanceof Error ? error.message : String(error)
1503
+ });
1504
+ }
1505
+ }
1506
+ async function deliverFailedChildTurnToParent(svc, workspaceId, childSessionId, turnId) {
1507
+ await deliverChildLifecycleOutboxToParent(
1508
+ svc,
1509
+ workspaceId,
1510
+ childSessionId,
1511
+ `child-completion:${childSessionId}:turn:${turnId}`
1512
+ );
1513
+ }
1514
+ async function deliverChildRequiresActionToParent(svc, workspaceId, childSessionId, input) {
1515
+ if (!svc.settings.childLifecycleNoticesEnabled) return;
1516
+ await deliverChildLifecycleOutboxToParent(
1517
+ svc,
1518
+ workspaceId,
1519
+ childSessionId,
1520
+ childRequiresActionDedupeKey({
1521
+ childSessionId,
1522
+ turnId: input.turnId,
1523
+ turnGeneration: input.turnGeneration
1524
+ }),
1525
+ // A requires_action freeze without any human-input/approval request (or a
1526
+ // child without a parent) commits no row; that is not an error.
1527
+ { requireRow: false }
1528
+ );
1529
+ }
1530
+ async function deliverParentSystemUpdateOutbox(svc, outbox) {
1531
+ try {
1532
+ const result = await addSessionSystemUpdateWithSourceMutation(
1533
+ svc.db,
1534
+ {
1535
+ accountId: outbox.accountId,
1536
+ workspaceId: outbox.workspaceId,
1537
+ sessionId: outbox.targetSessionId,
1538
+ ...sessionSystemUpdateOutboxKindPayload(outbox),
1539
+ classification: outbox.classification,
1540
+ sourceId: outbox.sourceId,
1541
+ dedupeKey: outbox.dedupeKey,
1542
+ summary: outbox.summary,
1543
+ lineage: outbox.lineage,
1544
+ personalConnectionDelegations: outbox.personalConnectionDelegations,
1545
+ xaiProviderAccountAuthoritySnapshot: outbox.xaiProviderAccountAuthoritySnapshot
1546
+ },
1547
+ async (tx) => {
1548
+ await markSessionSystemUpdateOutboxDeliveredInTransaction(tx, outbox);
1549
+ }
1550
+ );
1551
+ if (result.reason === "session_cancelled") {
1552
+ return;
1553
+ }
1554
+ if (result.added && result.events.length > 0) {
1555
+ await svc.bus.publish(outbox.workspaceId, outbox.targetSessionId, result.events);
1556
+ }
1557
+ if (result.shouldWake && svc.wakeSessionWorkflow) {
1558
+ if (result.workflowWakeRevision === null) {
1559
+ throw new Error("Runnable system update has no workflow wake revision");
1560
+ }
1561
+ await svc.wakeSessionWorkflow({
1562
+ accountId: outbox.accountId,
1563
+ workspaceId: outbox.workspaceId,
1564
+ sessionId: outbox.targetSessionId,
1565
+ workflowId: result.temporalWorkflowId ?? `session-${outbox.targetSessionId}`,
1566
+ wakeRevision: result.workflowWakeRevision
1567
+ });
1568
+ }
1569
+ svc.observability.info("Delivered child lifecycle notice to parent session", {
1570
+ childSessionId: outbox.sourceSessionId,
1571
+ parentSessionId: outbox.targetSessionId,
1572
+ kind: outbox.kind,
1573
+ dedupeKey: outbox.dedupeKey,
1574
+ woke: result.shouldWake
1575
+ });
1576
+ } catch (error) {
1577
+ await markSessionSystemUpdateOutboxFailed(
1578
+ svc.db,
1579
+ outbox,
1580
+ error instanceof Error ? error.message : String(error)
1581
+ ).catch(() => void 0);
1582
+ throw error;
1583
+ }
1584
+ }
1585
+ async function reconcilePendingParentSystemUpdates(svc, limit = 100, overrides = {}) {
1586
+ const claimPendingSessionSystemUpdateOutboxFn = overrides.claimPendingSessionSystemUpdateOutbox ?? claimPendingSessionSystemUpdateOutbox;
1587
+ const rows = await claimPendingSessionSystemUpdateOutboxFn(svc.db, limit);
1588
+ let delivered = 0;
1589
+ let failed = 0;
1590
+ for (const row of rows) {
1591
+ try {
1592
+ await deliverParentSystemUpdateOutbox(svc, row);
1593
+ delivered += 1;
1594
+ } catch {
1595
+ failed += 1;
1596
+ }
1597
+ }
1598
+ return { claimed: rows.length, delivered, failed };
1599
+ }
1600
+ async function reconcilePendingSessionWorkflowWakes(svc, limit = 1e3, overrides = {}) {
1601
+ const claimPendingSessionWorkflowWakesFn = overrides.claimPendingSessionWorkflowWakes ?? claimPendingSessionWorkflowWakes;
1602
+ if (!svc.wakeSessionWorkflow) {
1603
+ return { claimed: 0, delivered: 0, failed: 0 };
1604
+ }
1605
+ const repairs = await claimPendingSessionWorkflowWakesFn(svc.db, limit);
1606
+ let delivered = 0;
1607
+ let failed = 0;
1608
+ const queue = [...repairs];
1609
+ const workers = Array.from({ length: Math.min(20, queue.length) }, async () => {
1610
+ for (; ; ) {
1611
+ const repair = queue.shift();
1612
+ if (!repair) return;
1613
+ try {
1614
+ await svc.wakeSessionWorkflow({
1615
+ accountId: repair.accountId,
1616
+ workspaceId: repair.workspaceId,
1617
+ sessionId: repair.sessionId,
1618
+ workflowId: repair.temporalWorkflowId,
1619
+ wakeRevision: repair.wakeRevision,
1620
+ ...repair.interruptionRequested ? { interruptionRequested: true } : {}
1621
+ });
1622
+ delivered += 1;
1623
+ } catch (error) {
1624
+ failed += 1;
1625
+ await markSessionWorkflowWakeFailed(
1626
+ svc.db,
1627
+ repair,
1628
+ error instanceof Error ? error.message : String(error)
1629
+ ).catch(() => void 0);
1630
+ }
1631
+ }
1632
+ });
1633
+ await Promise.all(workers);
1634
+ return { claimed: repairs.length, delivered, failed };
1635
+ }
1636
+ function childCompletionPayload(child, goal) {
1637
+ return {
1638
+ type: "child_terminal_result",
1639
+ childSessionId: child.id,
1640
+ status: "idle",
1641
+ ...goal ? {
1642
+ goal: {
1643
+ status: goal.status,
1644
+ text: goal.text,
1645
+ ...goal.evidence ? { evidence: goal.evidence } : {},
1646
+ ...goal.rationale ? { rationale: goal.rationale } : {},
1647
+ ...goal.pausedReason ? { pausedReason: goal.pausedReason } : {}
1648
+ }
1649
+ } : {}
1650
+ };
1651
+ }
1652
+ function childCompletionSummary(child, goal, terminalStatus) {
1653
+ const lines = [];
1654
+ if (terminalStatus === "failed") {
1655
+ lines.push(`A worker session you spawned has FAILED. Worker session id: ${child.id}.`);
1656
+ } else if (goal?.status === "completed") {
1657
+ lines.push(
1658
+ `A worker session you spawned has COMPLETED its goal. Worker session id: ${child.id}.`
1659
+ );
1660
+ } else if (goal?.status === "paused") {
1661
+ lines.push(
1662
+ `A worker session you spawned has PAUSED its goal and gone idle. Worker session id: ${child.id}.`
1663
+ );
1664
+ } else {
1665
+ lines.push(
1666
+ `A worker session you spawned has finished its work and gone idle. Worker session id: ${child.id}.`
1667
+ );
1668
+ }
1669
+ if (goal) {
1670
+ lines.push(`Worker goal: ${goal.text}`);
1671
+ if (goal.status === "completed" && goal.evidence) {
1672
+ lines.push(`Completion evidence: ${goal.evidence}`);
1673
+ }
1674
+ if (goal.status === "paused" && goal.rationale) {
1675
+ lines.push(`Pause rationale: ${goal.rationale}`);
1676
+ }
1677
+ }
1678
+ return lines.join("\n");
1679
+ }
1680
+
1681
+ // src/sandbox-archive-storage.ts
1682
+ import {
1683
+ parseWorkspaceArchiveObjectRef,
1684
+ workspaceArchiveObjectKey
1685
+ } from "@opengeni/contracts";
1686
+ var WorkspaceArchiveObjectStorageRequiredError = class extends Error {
1687
+ code = "workspace_archive_object_storage_required";
1688
+ constructor(backend) {
1689
+ super(`${backend} workspace archives require object storage`);
1690
+ this.name = "WorkspaceArchiveObjectStorageRequiredError";
1691
+ }
1692
+ };
1693
+ function collectWorkspaceArchiveObjectKeys(resumeState) {
1694
+ const sessionState = resumeState?.sessionState && typeof resumeState.sessionState === "object" ? resumeState.sessionState : null;
1695
+ const keys = /* @__PURE__ */ new Set();
1696
+ for (const value of [sessionState?.workspaceArchiveRef, sessionState?.workspaceArchivePrevRef]) {
1697
+ const ref = parseWorkspaceArchiveObjectRef(value);
1698
+ if (ref) keys.add(ref.key);
1699
+ }
1700
+ return keys;
1701
+ }
1702
+ async function putTarWorkspaceArchiveObject(input) {
1703
+ if (input.archive.descriptor.version !== 1) {
1704
+ throw new Error("Object-storage workspace archives are portable tar only");
1705
+ }
1706
+ const key = workspaceArchiveObjectKey({
1707
+ accountId: input.accountId,
1708
+ workspaceId: input.workspaceId,
1709
+ sandboxGroupId: input.sandboxGroupId,
1710
+ revision: input.archive.descriptor.revision
1711
+ });
1712
+ await input.objectStorage.putObject({
1713
+ key,
1714
+ contentType: "application/x-tar",
1715
+ body: input.archive.bytes,
1716
+ sha256: input.archive.descriptor.archiveSha256
1717
+ });
1718
+ return {
1719
+ schema: "sandbox_archive_object_v1",
1720
+ key,
1721
+ sha256: input.archive.descriptor.archiveSha256,
1722
+ bytes: input.archive.descriptor.archiveBytes,
1723
+ backend: input.objectStorage.backend
1724
+ };
1725
+ }
1726
+ async function putVersion1TarArchiveOrInline(input) {
1727
+ if (input.archive.descriptor.version !== 1) {
1728
+ return { workspaceArchive: input.archive.base64 };
1729
+ }
1730
+ if (input.backend === "opensandbox" && !input.objectStorage) {
1731
+ throw new WorkspaceArchiveObjectStorageRequiredError("opensandbox");
1732
+ }
1733
+ if (!input.objectStorage) {
1734
+ return { workspaceArchive: input.archive.base64 };
1735
+ }
1736
+ try {
1737
+ const workspaceArchiveRef = await putTarWorkspaceArchiveObject({
1738
+ objectStorage: input.objectStorage,
1739
+ accountId: input.accountId,
1740
+ workspaceId: input.workspaceId,
1741
+ sandboxGroupId: input.sandboxGroupId,
1742
+ archive: { bytes: input.archive.bytes, descriptor: input.archive.descriptor }
1743
+ });
1744
+ try {
1745
+ input.metrics?.onWorkspaceArchiveObject?.({
1746
+ outcome: "put",
1747
+ backend: input.objectStorage.backend
1748
+ });
1749
+ } catch {
1750
+ }
1751
+ console.info("workspace archive object put", {
1752
+ key: workspaceArchiveRef.key,
1753
+ bytes: workspaceArchiveRef.bytes,
1754
+ sha256: workspaceArchiveRef.sha256
1755
+ });
1756
+ return { workspaceArchiveRef };
1757
+ } catch (error) {
1758
+ try {
1759
+ input.metrics?.onWorkspaceArchiveObject?.({
1760
+ outcome: "put_failed",
1761
+ backend: input.objectStorage.backend
1762
+ });
1763
+ } catch {
1764
+ }
1765
+ throw error;
1766
+ }
1767
+ }
1768
+ async function deleteUnpublishedWorkspaceArchiveObject(objectStorage, ref, metrics) {
1769
+ if (!objectStorage || !ref) return;
1770
+ try {
1771
+ await objectStorage.deleteObject(ref.key);
1772
+ try {
1773
+ metrics?.onWorkspaceArchiveObject?.({
1774
+ outcome: "deleted_unpublished",
1775
+ backend: objectStorage.backend
1776
+ });
1777
+ } catch {
1778
+ }
1779
+ } catch {
1780
+ console.error("unpublished workspace archive object delete failed", { key: ref.key });
1781
+ }
1782
+ }
1783
+ async function deleteWorkspaceArchiveObjectKeys(objectStorage, keys) {
1784
+ for (const key of keys) {
1785
+ await objectStorage.deleteObject(key);
1786
+ }
1787
+ }
1788
+
1789
+ // src/activities/common.ts
1790
+ import { mergeResourceRefs, mergeToolRefs as mergeToolRefs2 } from "@opengeni/contracts";
1791
+ function assertReusableSessionRevivable(status) {
1792
+ if (status === "cancelled") {
1793
+ throw new Error("reusable session is cancelled; refusing to revive on scheduled fire");
1794
+ }
1795
+ }
1796
+ function scheduledUserMessagePayload(prompt, resources, tools, taskId, runId) {
1797
+ return {
1798
+ type: "scheduled_occurrence",
1799
+ text: prompt,
1800
+ scheduledTaskId: taskId,
1801
+ scheduledTaskRunId: runId,
1802
+ ...resources.length ? { resources } : {},
1803
+ ...tools.length ? { tools } : {}
1804
+ };
1805
+ }
1806
+ function workflowIdForSession(sessionId) {
1807
+ return `session-${sessionId}`;
1808
+ }
1809
+
1810
+ // src/activities/incident-telemetry-preflight.ts
1811
+ import {
1812
+ IncidentTelemetryPreflight,
1813
+ stableJson
1814
+ } from "@opengeni/contracts";
1815
+ import { createHash } from "crypto";
1816
+ import { isDeepStrictEqual as isDeepStrictEqual2 } from "util";
1817
+ var INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY = "incidentTelemetryAuthorityFence";
1818
+ function incidentTelemetryPreflightDeclaration(agentConfig, incidentTriggered) {
1819
+ const rawConfig = agentConfig;
1820
+ const executionClass = rawConfig.executionClass;
1821
+ const rawPreflight = rawConfig.incidentTelemetryPreflight;
1822
+ if (executionClass === void 0 && rawPreflight === void 0 && !incidentTriggered) {
1823
+ return { action: "not_required" };
1824
+ }
1825
+ if (executionClass !== "incident_telemetry" || rawPreflight === void 0) {
1826
+ return { action: "blocked", reason: "incident_preflight_metadata_missing" };
1827
+ }
1828
+ const parsed = IncidentTelemetryPreflight.safeParse(rawPreflight);
1829
+ if (!parsed.success) {
1830
+ const dataSourceInvalid = parsed.error.issues.some((issue) => issue.path[0] === "dataSource");
1831
+ return {
1832
+ action: "blocked",
1833
+ reason: dataSourceInvalid ? "incident_data_source_unsuitable" : "incident_preflight_metadata_missing"
1834
+ };
1835
+ }
1836
+ return { action: "required", preflight: parsed.data };
1837
+ }
1838
+ function evaluateIncidentTelemetryPreflight(input) {
1839
+ const declaration = incidentTelemetryPreflightDeclaration(
1840
+ input.agentConfig,
1841
+ input.incidentTriggered
1842
+ );
1843
+ if (declaration.action !== "required") return declaration;
1844
+ const preflight = declaration.preflight;
1845
+ if (!input.responder.metadataComplete) {
1846
+ return blocked("incident_preflight_metadata_missing");
1847
+ }
1848
+ if (preflight.requiredResources.some(
1849
+ (required) => !input.responder.resources.some((selected) => isDeepStrictEqual2(selected, required))
1850
+ )) {
1851
+ return blocked("incident_responder_under_capable");
1852
+ }
1853
+ const selectedMcpServerIds = new Set(input.responder.mcpServerIds);
1854
+ if (preflight.requiredMcpServerIds.some((id) => !selectedMcpServerIds.has(id))) {
1855
+ return blocked("incident_responder_under_capable");
1856
+ }
1857
+ const selectedFirstPartyTools = new Set(input.responder.firstPartyMcpTools);
1858
+ if (preflight.requiredFirstPartyMcpTools.some((tool) => !selectedFirstPartyTools.has(tool))) {
1859
+ return blocked("incident_responder_under_capable");
1860
+ }
1861
+ const selectedFirstPartyPermissions = new Set(input.responder.firstPartyMcpPermissions);
1862
+ if (preflight.requiredFirstPartyMcpPermissions.some(
1863
+ (permission) => !selectedFirstPartyPermissions.has(permission)
1864
+ )) {
1865
+ return blocked("incident_responder_under_capable");
1866
+ }
1867
+ if (preflight.requiredRig && !rigSatisfies(input.responder.rig, preflight.requiredRig)) {
1868
+ return blocked("incident_responder_under_capable");
1869
+ }
1870
+ const variableSetsByName = new Map(input.responder.variableSets.map((set) => [set.name, set]));
1871
+ if (preflight.requiredVariableSetNames.some((name) => !variableSetsByName.has(name))) {
1872
+ return blocked("incident_responder_under_capable");
1873
+ }
1874
+ const attachedVariableNames = new Set(
1875
+ input.responder.variableSets.flatMap((set) => set.variables.map((variable) => variable.name))
1876
+ );
1877
+ if (preflight.requiredVariableNames.some((name) => !attachedVariableNames.has(name))) {
1878
+ return blocked("incident_responder_under_capable");
1879
+ }
1880
+ const route = preflight.dataSource.route;
1881
+ if (route.kind === "mcp" && !selectedMcpServerIds.has(route.serverId)) {
1882
+ return blocked("incident_responder_under_capable");
1883
+ }
1884
+ if (route.kind === "first_party" && !selectedFirstPartyTools.has(route.tool)) {
1885
+ return blocked("incident_responder_under_capable");
1886
+ }
1887
+ if (route.kind === "variable_set") {
1888
+ const selected = variableSetsByName.get(route.variableSetName);
1889
+ const names = new Set(selected?.variables.map((variable) => variable.name) ?? []);
1890
+ if (!selected || route.variableNames.some((name) => !names.has(name))) {
1891
+ return blocked("incident_responder_under_capable");
1892
+ }
1893
+ }
1894
+ if (route.kind === "rig_credential_hook" && !rigSatisfies(input.responder.rig, {
1895
+ name: preflight.requiredRig?.name ?? "",
1896
+ credentialHookIds: [route.credentialHookId]
1897
+ })) {
1898
+ return blocked("incident_responder_under_capable");
1899
+ }
1900
+ if (!seriesMetadataSatisfies(preflight.dataSource)) {
1901
+ return blocked("incident_data_source_unsuitable");
1902
+ }
1903
+ if (!alertSelectorsSatisfy(preflight.dataSource, input.alertOccurrenceLabels)) {
1904
+ return blocked("incident_data_source_unsuitable");
1905
+ }
1906
+ return { action: "ready" };
1907
+ }
1908
+ function incidentTelemetryAuthorityFence(input) {
1909
+ const { task, responder } = input;
1910
+ if (responder.toolPolicyVersion === null) return null;
1911
+ const declaration = incidentTelemetryPreflightDeclaration(
1912
+ task.agentConfig,
1913
+ input.alertOccurrenceLabels !== null
1914
+ );
1915
+ if (declaration.action !== "required" || !input.alertOccurrenceLabels) return null;
1916
+ const selectorEntries = declaration.preflight.dataSource.alertSelectorLabels.map((label) => [label, input.alertOccurrenceLabels?.[label]]).sort(([left], [right]) => left.localeCompare(right));
1917
+ if (selectorEntries.some(([, value]) => typeof value !== "string" || value.length === 0)) {
1918
+ return null;
1919
+ }
1920
+ const normalized = {
1921
+ metadataComplete: responder.metadataComplete,
1922
+ resources: [...responder.resources].map((resource) => stableJson(resource)).sort(),
1923
+ mcpServerIds: [...responder.mcpServerIds].sort(),
1924
+ firstPartyMcpTools: [...responder.firstPartyMcpTools].sort(),
1925
+ firstPartyMcpPermissions: [...responder.firstPartyMcpPermissions].sort(),
1926
+ rig: responder.rig ? {
1927
+ name: responder.rig.name,
1928
+ credentialHooks: [...responder.rig.credentialHooks].sort(),
1929
+ checkHealth: responder.rig.checkHealth
1930
+ } : null,
1931
+ variableSets: [...responder.variableSets].map((set) => ({
1932
+ name: set.name,
1933
+ variables: set.variables.map((variable) => variable.name).sort()
1934
+ })).sort((left, right) => left.name.localeCompare(right.name))
1935
+ };
1936
+ return {
1937
+ version: 2,
1938
+ toolPolicyVersion: responder.toolPolicyVersion,
1939
+ responderDigest: createHash("sha256").update(stableJson(normalized)).digest("hex"),
1940
+ taskDigest: createHash("sha256").update(stableJson(task)).digest("hex"),
1941
+ alertSelectorDigest: createHash("sha256").update(stableJson(selectorEntries)).digest("hex")
1942
+ };
1943
+ }
1944
+ function parseIncidentTelemetryAuthorityFence(lineage) {
1945
+ const raw = lineage[INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY];
1946
+ if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
1947
+ const value = raw;
1948
+ return value.version === 2 && typeof value.toolPolicyVersion === "number" && Number.isInteger(value.toolPolicyVersion) && value.toolPolicyVersion > 0 && typeof value.responderDigest === "string" && /^[0-9a-f]{64}$/.test(value.responderDigest) && typeof value.taskDigest === "string" && /^[0-9a-f]{64}$/.test(value.taskDigest) && typeof value.alertSelectorDigest === "string" && /^[0-9a-f]{64}$/.test(value.alertSelectorDigest) ? {
1949
+ version: 2,
1950
+ toolPolicyVersion: value.toolPolicyVersion,
1951
+ responderDigest: value.responderDigest,
1952
+ taskDigest: value.taskDigest,
1953
+ alertSelectorDigest: value.alertSelectorDigest
1954
+ } : null;
1955
+ }
1956
+ function blocked(reason) {
1957
+ return { action: "blocked", reason };
1958
+ }
1959
+ function rigSatisfies(rig, required) {
1960
+ if (!rig || rig.name !== required.name || rig.checkHealth !== "passing") {
1961
+ return false;
1962
+ }
1963
+ const hookIds = new Set(rig.credentialHooks);
1964
+ return required.credentialHookIds.every((hookId) => hookIds.has(hookId));
1965
+ }
1966
+ function seriesMetadataSatisfies(dataSource) {
1967
+ if (dataSource.queryPath !== "/api/v1/query" && dataSource.queryPath !== "/api/v1/query_range") {
1968
+ return false;
1969
+ }
1970
+ return dataSource.requiredSeries.every(
1971
+ (required) => dataSource.availableSeries.some((available) => {
1972
+ if (available.metric !== required.metric) return false;
1973
+ const labels = new Set(available.labels);
1974
+ return labels.has(dataSource.workspaceLabel) && required.labels.every((label) => labels.has(label));
1975
+ })
1976
+ );
1977
+ }
1978
+ function alertSelectorsSatisfy(dataSource, occurrenceLabels) {
1979
+ if (!occurrenceLabels) return false;
1980
+ return dataSource.alertSelectorLabels.every(
1981
+ (label) => label !== dataSource.workspaceLabel && typeof occurrenceLabels[label] === "string" && occurrenceLabels[label].length > 0 && dataSource.requiredSeries.every((series) => series.labels.includes(label)) && dataSource.availableSeries.some(
1982
+ (series) => dataSource.requiredSeries.some((required) => required.metric === series.metric) && series.labels.includes(dataSource.workspaceLabel) && series.labels.includes(label)
1983
+ )
1984
+ );
1985
+ }
1986
+
1987
+ // src/activities/incident-telemetry-authority.ts
1988
+ import {
1989
+ DEFAULT_FIRST_PARTY_MCP_PERMISSIONS
1990
+ } from "@opengeni/contracts";
1991
+ import { defaultSessionMcpServerIds, resolveSessionToolPolicy } from "@opengeni/core";
1992
+ import {
1993
+ getRig,
1994
+ getScheduledScopedRigVersionMetadata,
1995
+ getScheduledTaskRunAcceptedExecution,
1996
+ getVariableSet,
1997
+ listEnabledMcpCapabilityServerIds,
1998
+ listInstalledApiIntegrationServerIdsForDelegations,
1999
+ markScheduledTaskRunAuthorityRejectedInTransaction,
2000
+ requireSession as requireSession2
2001
+ } from "@opengeni/db";
2002
+ import {
2003
+ allowedFirstPartyMcpToolsForSession as allowedFirstPartyMcpToolsForSession2,
2004
+ resolveFirstPartyMcpToolPolicy
2005
+ } from "@opengeni/config";
2006
+ async function resolveIncidentTelemetryResponderMetadata(input) {
2007
+ const [capabilityServerIds, apiIntegrationServerIds] = await Promise.all([
2008
+ listEnabledMcpCapabilityServerIds(input.db, input.task.workspaceId),
2009
+ listInstalledApiIntegrationServerIdsForDelegations(
2010
+ input.db,
2011
+ input.task.workspaceId,
2012
+ input.personalConnectionDelegations
2013
+ )
2014
+ ]);
2015
+ const sessionTools = input.executionPolicy ? input.executionPolicy.tools : input.session ? input.session.tools : withFirstPartyTools(input.settings, input.task.agentConfig.tools);
2016
+ const resolvedToolPolicy = resolveIncidentTelemetryResponderToolPolicy({
2017
+ settingsMcpServerIds: input.settings.mcpServers.map((server) => server.id),
2018
+ capabilityServerIds,
2019
+ apiIntegrationServerIds,
2020
+ ...input.executionPolicy ? { acceptedSessionMcpServerIds: input.executionPolicy.mcpServerIds } : {},
2021
+ session: input.executionPolicy ? {
2022
+ mcpServers: input.executionPolicy.mcpServerIds.map((id) => ({ id })),
2023
+ tools: input.executionPolicy.tools,
2024
+ toolPolicy: input.executionPolicy.toolPolicy
2025
+ } : input.session,
2026
+ plannedTools: sessionTools
2027
+ });
2028
+ let metadataComplete = true;
2029
+ let rig = null;
2030
+ const variableSetIds = /* @__PURE__ */ new Set();
2031
+ if (input.executionPolicy ?? input.session) {
2032
+ const variableSetId = input.executionPolicy?.variableSetId ?? input.session?.variableSetId;
2033
+ const rigId = input.executionPolicy?.rigId ?? input.session?.rigId ?? null;
2034
+ const rigVersionId = input.executionPolicy?.rigVersionId ?? input.session?.rigVersionId ?? null;
2035
+ if (variableSetId) variableSetIds.add(variableSetId);
2036
+ if (rigId === null !== (rigVersionId === null)) {
2037
+ metadataComplete = false;
2038
+ } else if (rigId && rigVersionId) {
2039
+ const selected = await getScheduledScopedRigVersionMetadata(
2040
+ input.db,
2041
+ {
2042
+ accountId: input.task.accountId,
2043
+ workspaceId: input.task.workspaceId,
2044
+ subjectId: input.personalResourceAuthoritySubjectId ?? input.task.createdBy.subjectId
2045
+ },
2046
+ rigId,
2047
+ rigVersionId
2048
+ );
2049
+ if (!selected) {
2050
+ metadataComplete = false;
2051
+ } else {
2052
+ rig = {
2053
+ name: selected.name,
2054
+ credentialHooks: selected.version.credentialHooks,
2055
+ checkHealth: selected.health.checkHealth
2056
+ };
2057
+ for (const id of selected.version.defaultVariableSetIds) variableSetIds.add(id);
2058
+ }
2059
+ }
2060
+ } else {
2061
+ if (input.task.variableSetId) variableSetIds.add(input.task.variableSetId);
2062
+ if (input.task.rigId) {
2063
+ const selectedRig = await getRig(
2064
+ input.db,
2065
+ {
2066
+ accountId: input.task.accountId,
2067
+ workspaceId: input.task.workspaceId,
2068
+ subjectId: input.personalResourceAuthoritySubjectId ?? input.task.createdBy.subjectId
2069
+ },
2070
+ input.task.rigId
2071
+ );
2072
+ if (!selectedRig?.activeVersion || !selectedRig.activeVersionHealth) {
2073
+ metadataComplete = false;
2074
+ } else {
2075
+ rig = {
2076
+ name: selectedRig.name,
2077
+ credentialHooks: selectedRig.activeVersion.credentialHooks,
2078
+ checkHealth: selectedRig.activeVersionHealth.checkHealth
2079
+ };
2080
+ for (const id of selectedRig.activeVersion.defaultVariableSetIds) variableSetIds.add(id);
2081
+ }
2082
+ }
2083
+ }
2084
+ const variableSets = await Promise.all(
2085
+ [...variableSetIds].map(
2086
+ async (id) => await getVariableSet(
2087
+ input.db,
2088
+ {
2089
+ accountId: input.task.accountId,
2090
+ workspaceId: input.task.workspaceId,
2091
+ subjectId: input.personalResourceAuthoritySubjectId ?? input.task.createdBy.subjectId
2092
+ },
2093
+ id
2094
+ )
2095
+ )
2096
+ );
2097
+ if (variableSets.some((variableSet) => variableSet === null)) {
2098
+ metadataComplete = false;
2099
+ }
2100
+ return {
2101
+ metadataComplete,
2102
+ resources: input.executionPolicy ? input.task.agentConfig.resources : input.session?.resources ?? input.task.agentConfig.resources,
2103
+ mcpServerIds: resolvedToolPolicy.toolRefs.map((tool) => tool.id),
2104
+ firstPartyMcpTools: input.executionPolicy ? input.executionPolicy.firstPartyMcpTools : input.session ? allowedFirstPartyMcpToolsForSession2(input.settings, input.session.firstPartyMcpTools) : resolveFirstPartyMcpToolPolicy(input.settings).default,
2105
+ firstPartyMcpPermissions: input.executionPolicy ? input.executionPolicy.firstPartyMcpPermissions ?? DEFAULT_FIRST_PARTY_MCP_PERMISSIONS : input.session?.firstPartyMcpPermissions ?? DEFAULT_FIRST_PARTY_MCP_PERMISSIONS,
2106
+ rig,
2107
+ variableSets: variableSets.flatMap(
2108
+ (variableSet) => variableSet ? [
2109
+ {
2110
+ name: variableSet.name,
2111
+ variables: variableSet.variables.map((variable) => ({
2112
+ name: variable.name
2113
+ }))
2114
+ }
2115
+ ] : []
2116
+ ),
2117
+ toolPolicyVersion: input.executionPolicy?.toolPolicyVersion ?? input.session?.toolPolicyVersion ?? null
2118
+ };
2119
+ }
2120
+ function resolveIncidentTelemetryResponderToolPolicy(input) {
2121
+ const workspaceDefaultMcpServerIds = new Set(input.settingsMcpServerIds);
2122
+ for (const id of input.capabilityServerIds) workspaceDefaultMcpServerIds.add(id);
2123
+ for (const id of input.apiIntegrationServerIds) workspaceDefaultMcpServerIds.add(id);
2124
+ for (const id of input.acceptedSessionMcpServerIds ?? []) {
2125
+ workspaceDefaultMcpServerIds.add(id);
2126
+ }
2127
+ const availableMcpServerIds = new Set(workspaceDefaultMcpServerIds);
2128
+ for (const server of input.session?.mcpServers ?? []) {
2129
+ availableMcpServerIds.add(server.id);
2130
+ }
2131
+ return resolveSessionToolPolicy({
2132
+ toolPolicy: input.session?.toolPolicy ?? {
2133
+ mode: "explicit",
2134
+ inheritedFromSessionId: null
2135
+ },
2136
+ sessionTools: input.plannedTools,
2137
+ availableMcpServerIds,
2138
+ defaultMcpServerIds: defaultSessionMcpServerIds(
2139
+ [...workspaceDefaultMcpServerIds].map((id) => ({ id }))
2140
+ )
2141
+ });
2142
+ }
2143
+ async function validateIncidentTelemetrySystemUpdateAuthority(input) {
2144
+ if (!(INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY in input.update.lineage)) {
2145
+ return { action: "accept" };
2146
+ }
2147
+ const fence = parseIncidentTelemetryAuthorityFence(input.update.lineage);
2148
+ const payload = input.update.payload;
2149
+ const taskId = input.update.lineage.scheduledTaskId;
2150
+ const runId = payload.type === "scheduled_occurrence" ? payload.scheduledTaskRunId : null;
2151
+ const reject = async () => {
2152
+ if (runId) {
2153
+ await markScheduledTaskRunAuthorityRejectedInTransaction(input.db, {
2154
+ workspaceId: input.workspaceId,
2155
+ sessionId: input.sessionId,
2156
+ runId
2157
+ });
2158
+ }
2159
+ return {
2160
+ action: "reject",
2161
+ reason: "incident_responder_under_capable"
2162
+ };
2163
+ };
2164
+ if (!fence || typeof taskId !== "string" || !runId) return await reject();
2165
+ const session = await requireSession2(input.db, input.workspaceId, input.sessionId);
2166
+ const acceptedExecution = await getScheduledTaskRunAcceptedExecution(input.db, {
2167
+ workspaceId: input.workspaceId,
2168
+ runId
2169
+ });
2170
+ if (!acceptedExecution || acceptedExecution.task.id !== taskId || !acceptedExecution.incidentPreflightRequired) {
2171
+ return await reject();
2172
+ }
2173
+ const task = acceptedExecution.task;
2174
+ const personalConnectionDelegations = acceptedExecution.personalConnectionDelegations;
2175
+ const responder = await resolveIncidentTelemetryResponderMetadata({
2176
+ db: input.db,
2177
+ settings: input.settings,
2178
+ task,
2179
+ session,
2180
+ personalConnectionDelegations,
2181
+ personalResourceAuthoritySubjectId: acceptedExecution.personalResourceAuthoritySubjectId,
2182
+ executionPolicy: acceptedExecution.targetSessionExecution ? {
2183
+ tools: acceptedExecution.targetSessionExecution.tools,
2184
+ firstPartyMcpTools: acceptedExecution.targetSessionExecution.firstPartyMcpTools,
2185
+ firstPartyMcpPermissions: acceptedExecution.targetSessionExecution.firstPartyMcpPermissions,
2186
+ variableSetId: acceptedExecution.targetSessionExecution.variableSetId,
2187
+ rigId: acceptedExecution.targetSessionExecution.rigId,
2188
+ rigVersionId: acceptedExecution.targetSessionExecution.rigVersionId,
2189
+ toolPolicy: acceptedExecution.targetSessionExecution.toolPolicy,
2190
+ mcpServerIds: acceptedExecution.targetSessionExecution.mcpServerIds,
2191
+ toolPolicyVersion: acceptedExecution.targetSessionExecution.toolPolicyVersion
2192
+ } : {
2193
+ tools: acceptedExecution.resolvedTools,
2194
+ firstPartyMcpTools: acceptedExecution.resolvedFirstPartyMcpTools,
2195
+ firstPartyMcpPermissions: acceptedExecution.resolvedFirstPartyMcpPermissions,
2196
+ variableSetId: acceptedExecution.resolvedVariableSet?.id ?? null,
2197
+ rigId: acceptedExecution.resolvedRig?.id ?? null,
2198
+ rigVersionId: acceptedExecution.resolvedRig?.versionId ?? null,
2199
+ toolPolicy: session.toolPolicy,
2200
+ mcpServerIds: [],
2201
+ toolPolicyVersion: session.toolPolicyVersion
2202
+ }
2203
+ });
2204
+ const preflight = evaluateIncidentTelemetryPreflight({
2205
+ agentConfig: task.agentConfig,
2206
+ incidentTriggered: acceptedExecution.alertOccurrenceLabels !== null,
2207
+ alertOccurrenceLabels: acceptedExecution.alertOccurrenceLabels,
2208
+ responder
2209
+ });
2210
+ if (preflight.action !== "ready") return await reject();
2211
+ const current = incidentTelemetryAuthorityFence({
2212
+ task,
2213
+ responder,
2214
+ alertOccurrenceLabels: acceptedExecution.alertOccurrenceLabels
2215
+ });
2216
+ return current && current.toolPolicyVersion === fence.toolPolicyVersion && current.responderDigest === fence.responderDigest && current.taskDigest === fence.taskDigest && current.alertSelectorDigest === fence.alertSelectorDigest ? { action: "accept" } : await reject();
2217
+ }
2218
+
2219
+ // src/activities/agent-turn/sandbox-provision.ts
2220
+ import {
2221
+ SandboxLeaseRecoveryBlockedError as SandboxLeaseRecoveryBlockedError2,
2222
+ SandboxLeaseSupersededError as SandboxLeaseSupersededError2,
2223
+ SandboxLeaseTransitionError as SandboxLeaseTransitionError2,
2224
+ SandboxImageConflictError,
2225
+ SandboxRigConflictError
2226
+ } from "@opengeni/db";
2227
+ import {
2228
+ SandboxConfigError,
2229
+ SandboxExactResumeInstanceUnavailableError,
2230
+ SandboxExactResumeReplacedError,
2231
+ SandboxExecReadinessError as SandboxExecReadinessError2,
2232
+ SandboxProviderContinuityUnavailableError as SandboxProviderContinuityUnavailableError2,
2233
+ SandboxProviderUnavailableError,
2234
+ SandboxResumeIdentityMismatchError,
2235
+ SandboxResumeIdentityUnavailableError,
2236
+ SandboxResumeStateUnavailableError,
2237
+ WorkspaceArchiveIntegrityError as WorkspaceArchiveIntegrityError2,
2238
+ classifyProviderSandboxFailure
2239
+ } from "@opengeni/runtime";
2240
+ import { CancelledFailure } from "@temporalio/activity";
2241
+
2242
+ // src/sandbox-resume.ts
2243
+ import {
2244
+ effectiveSandboxLifecycle,
2245
+ sandboxArchiveCaptureTimeoutMs,
2246
+ sandboxLifecycleTransitionWaitMs
2247
+ } from "@opengeni/config";
2248
+ import { randomUUID as randomUUID2 } from "crypto";
2249
+ import {
2250
+ acquireLease,
2251
+ adoptLegacyModalCheckpointArtifact,
2252
+ beginSandboxRematerialization,
2253
+ claimWorkspaceArchiveCapture,
2254
+ commitWarmingToWarm,
2255
+ failSandboxRematerialization,
2256
+ failWarmingToCold,
2257
+ getSandboxSessionEnvelope,
2258
+ heartbeatLeaseHolderStatus,
2259
+ markSandboxRestoreVerifying,
2260
+ markWarmLeaseInstanceLost,
2261
+ markSandboxCheckpointArtifactDeletePending,
2262
+ persistWarmSnapshot,
2263
+ readLease,
2264
+ registerSandboxCheckpointArtifact,
2265
+ recordWarmingSandboxCreated,
2266
+ reinstateTurnLeaseHolder,
2267
+ releaseWorkspaceArchiveCapture,
2268
+ releaseLeaseHolder,
2269
+ touchLeaseHolder,
2270
+ SandboxLeaseRecoveryBlockedError,
2271
+ SandboxLeaseSupersededError,
2272
+ SandboxLeaseTransitionError
2273
+ } from "@opengeni/db";
2274
+ import {
2275
+ captureVerifiedWorkspaceArchive,
2276
+ describeLegacyNativeSnapshotArchive,
2277
+ inlineWorkspaceArchiveForRestore,
2278
+ MODAL_EXEC_READINESS_TIMEOUT_MS,
2279
+ SandboxExecReadinessError,
2280
+ WorkspaceArchiveIntegrityError,
2281
+ establishSandboxSessionFromEnvelope,
2282
+ isProviderSandboxNotFoundError,
2283
+ parseWorkspaceArchiveDescriptor,
2284
+ providerWorkspaceCapturePolicy,
2285
+ SandboxProviderContinuityUnavailableError,
2286
+ requirePersistableReplacementSandboxEnvelope,
2287
+ renewSandboxProviderExpiration,
2288
+ modalSessionMatchesCheckpointProviderBinding,
2289
+ resolveModalCheckpointProviderBindingForSession,
2290
+ serializeReplacementSandboxEnvelope,
2291
+ tagModalSandbox,
2292
+ terminateUnpublishedSandboxSession,
2293
+ verifySandboxExecReadiness,
2294
+ withoutSandboxProviderIdentity
2295
+ } from "@opengeni/runtime";
2296
+ import { parseWorkspaceArchiveObjectRef as parseWorkspaceArchiveObjectRef2 } from "@opengeni/contracts";
2297
+ function sandboxLeaseHolderIdForAttempt(attemptId) {
2298
+ const normalized = attemptId.trim();
2299
+ if (!normalized) {
2300
+ throw new Error("Sandbox lease holder requires a turn attempt id");
2301
+ }
2302
+ return `turn-attempt:${normalized}`;
2303
+ }
2304
+ function isRetryableDegradedRestore(restore) {
2305
+ return restore.status === "degraded" && restore.retryable === true;
2306
+ }
2307
+ var SandboxProvisionStageError = class extends Error {
2308
+ constructor(stage, source) {
2309
+ super(source instanceof Error ? source.message : String(source), { cause: source });
2310
+ this.stage = stage;
2311
+ this.source = source;
2312
+ }
2313
+ name = "SandboxProvisionStageError";
2314
+ };
2315
+ var sandboxProvisionFailureStages = /* @__PURE__ */ new WeakMap();
2316
+ function sandboxProvisionFailureStage(error) {
2317
+ if (!error || typeof error !== "object") return null;
2318
+ return sandboxProvisionFailureStages.get(error) ?? (error instanceof SandboxProvisionStageError ? error.stage : null);
2319
+ }
2320
+ function sandboxProvisionStageError(stage, error) {
2321
+ if (error && typeof error === "object") {
2322
+ sandboxProvisionFailureStages.set(error, stage);
2323
+ return error;
2324
+ }
2325
+ return new SandboxProvisionStageError(stage, error);
2326
+ }
2327
+ var SandboxWarmingTimeoutError = class extends Error {
2328
+ constructor(backend, timeoutMs, stage, sandboxGroupId, instanceId) {
2329
+ const timeoutSeconds = Math.ceil(timeoutMs / 1e3);
2330
+ const identity = [
2331
+ sandboxGroupId ? `group ${sandboxGroupId}` : null,
2332
+ instanceId ? `sandbox ${instanceId}` : null
2333
+ ].filter((part) => part !== null).join(", ");
2334
+ const target = identity ? ` (${identity})` : "";
2335
+ super(
2336
+ stage === "exec_readiness" ? `Sandbox backend "${backend}"${target} did not become command-ready within ${timeoutSeconds}s after creation or restore.` : `Sandbox backend "${backend}"${target} did not finish warming within ${timeoutSeconds}s while waiting for the elected sandbox creator.`
2337
+ );
2338
+ this.backend = backend;
2339
+ this.timeoutMs = timeoutMs;
2340
+ this.stage = stage;
2341
+ this.sandboxGroupId = sandboxGroupId;
2342
+ this.instanceId = instanceId;
2343
+ this.name = "SandboxWarmingTimeoutError";
2344
+ }
2345
+ };
2346
+ var SandboxExecReadinessTimeoutError = class extends SandboxWarmingTimeoutError {
2347
+ code = "sandbox_exec_readiness_timeout";
2348
+ constructor(backend, timeoutMs, identity = {}) {
2349
+ super(
2350
+ backend,
2351
+ timeoutMs,
2352
+ "exec_readiness",
2353
+ identity.sandboxGroupId ?? null,
2354
+ identity.instanceId ?? null
2355
+ );
2356
+ this.name = "SandboxExecReadinessTimeoutError";
2357
+ }
2358
+ };
2359
+ var SandboxSiblingWarmingTimeoutError = class extends SandboxWarmingTimeoutError {
2360
+ code = "sandbox_sibling_warming_timeout";
2361
+ constructor(backend, timeoutMs, identity) {
2362
+ super(
2363
+ backend,
2364
+ timeoutMs,
2365
+ "sibling_warming",
2366
+ identity.sandboxGroupId,
2367
+ identity.instanceId ?? null
2368
+ );
2369
+ this.name = "SandboxSiblingWarmingTimeoutError";
2370
+ }
2371
+ };
2372
+ var SandboxLeaseInstanceLostError = class extends SandboxLeaseSupersededError {
2373
+ constructor(sandboxGroupId, leaseEpoch, lostInstanceId) {
2374
+ super(sandboxGroupId, leaseEpoch);
2375
+ this.lostInstanceId = lostInstanceId;
2376
+ this.name = "SandboxLeaseInstanceLostError";
2377
+ }
2378
+ };
2379
+ var WARMING_POLL_INTERVAL_MS = 250;
2380
+ async function sleep(ms) {
2381
+ await new Promise((resolve) => setTimeout(resolve, ms));
2382
+ }
2383
+ async function waitForSandboxExecReadiness(established, timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS, identity = {}) {
2384
+ try {
2385
+ await verifySandboxExecReadiness(established, timeoutMs);
2386
+ } catch (error) {
2387
+ if (error instanceof SandboxExecReadinessError && error.code === "exec_probe_timeout") {
2388
+ throw new SandboxExecReadinessTimeoutError(established.backendId, timeoutMs, {
2389
+ sandboxGroupId: identity.sandboxGroupId ?? null,
2390
+ instanceId: established.instanceId
2391
+ });
2392
+ }
2393
+ throw error;
2394
+ }
2395
+ }
2396
+ var SnapshotTimeoutError = class extends Error {
2397
+ constructor(timeoutMs) {
2398
+ super(`workspace snapshot timed out after ${timeoutMs}ms`);
2399
+ this.name = "SnapshotTimeoutError";
2400
+ }
2401
+ };
2402
+ function safeSnapshotError(error) {
2403
+ const fields = {
2404
+ errorClass: "SnapshotOperationError",
2405
+ errorCode: "snapshot_operation_failed",
2406
+ origin: "sandbox-resume"
2407
+ };
2408
+ try {
2409
+ if (error && typeof error === "object") {
2410
+ const candidate = error;
2411
+ if (typeof candidate.name === "string" && /^[A-Z][A-Za-z0-9]{2,62}Error$/u.test(candidate.name)) {
2412
+ fields.causeName = candidate.name;
2413
+ }
2414
+ if (typeof candidate.code === "string" && /^[a-z0-9_]{1,64}$/u.test(candidate.code)) {
2415
+ fields.integrityCode = candidate.code;
2416
+ }
2417
+ const rawStatus = candidate.status ?? candidate.statusCode;
2418
+ const status = Number(rawStatus);
2419
+ if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
2420
+ }
2421
+ } catch {
2422
+ }
2423
+ return fields;
2424
+ }
2425
+ async function waitForWarmSnapshot(snapshot, timeoutMs, signal) {
2426
+ if (signal?.aborted) return false;
2427
+ let timeout;
2428
+ let cancelListener;
2429
+ try {
2430
+ await Promise.race([
2431
+ snapshot,
2432
+ ...signal ? [
2433
+ new Promise((_resolve, reject) => {
2434
+ cancelListener = () => reject(signal.reason ?? new Error("workspace snapshot wait cancelled"));
2435
+ signal.addEventListener("abort", cancelListener, { once: true });
2436
+ if (signal.aborted) cancelListener();
2437
+ })
2438
+ ] : [],
2439
+ new Promise((_, reject) => {
2440
+ timeout = setTimeout(() => reject(new SnapshotTimeoutError(timeoutMs)), timeoutMs);
2441
+ if (timeout && "unref" in timeout && typeof timeout.unref === "function") {
2442
+ timeout.unref();
2443
+ }
2444
+ })
2445
+ ]);
2446
+ return true;
2447
+ } catch (error) {
2448
+ if (signal?.aborted) return false;
2449
+ if (error instanceof SnapshotTimeoutError) {
2450
+ console.error(
2451
+ "mid-session workspace snapshot wait timed out (turn unaffected)",
2452
+ safeSnapshotError(error)
2453
+ );
2454
+ return false;
2455
+ }
2456
+ return true;
2457
+ } finally {
2458
+ if (timeout) {
2459
+ clearTimeout(timeout);
2460
+ }
2461
+ if (cancelListener) signal?.removeEventListener("abort", cancelListener);
2462
+ }
2463
+ }
2464
+ async function resolveModalCheckpointBindingBeforeCapture(settings, session, signal) {
2465
+ if (signal?.aborted) {
2466
+ throw signal.reason ?? new Error("workspace snapshot identity lookup cancelled");
2467
+ }
2468
+ const pending = resolveModalCheckpointProviderBindingForSession(settings, session);
2469
+ let timeout;
2470
+ let cancelListener;
2471
+ try {
2472
+ return await Promise.race([
2473
+ pending,
2474
+ ...signal ? [
2475
+ new Promise((_resolve, reject) => {
2476
+ cancelListener = () => reject(signal.reason ?? new Error("workspace snapshot identity lookup cancelled"));
2477
+ signal.addEventListener("abort", cancelListener, { once: true });
2478
+ if (signal.aborted) cancelListener();
2479
+ })
2480
+ ] : [],
2481
+ new Promise((_resolve, reject) => {
2482
+ timeout = setTimeout(
2483
+ () => reject(new SnapshotTimeoutError(settings.sandboxSnapshotTimeoutMs)),
2484
+ settings.sandboxSnapshotTimeoutMs
2485
+ );
2486
+ timeout.unref?.();
2487
+ })
2488
+ ]);
2489
+ } finally {
2490
+ if (timeout) clearTimeout(timeout);
2491
+ if (cancelListener) signal?.removeEventListener("abort", cancelListener);
2492
+ }
2493
+ }
2494
+ async function terminateEstablishedSandbox(established) {
2495
+ if (!established) return true;
2496
+ try {
2497
+ await terminateUnpublishedSandboxSession(established);
2498
+ return true;
2499
+ } catch (error) {
2500
+ if (isProviderSandboxNotFoundError(established.backendId, error)) {
2501
+ return true;
2502
+ }
2503
+ return false;
2504
+ }
2505
+ }
2506
+ function recordSandboxWarmingTimeout(metrics, error) {
2507
+ if (!(error instanceof SandboxWarmingTimeoutError)) {
2508
+ return;
2509
+ }
2510
+ try {
2511
+ metrics?.onSandboxWarmingTimeout?.({ backend: error.backend, stage: error.stage });
2512
+ } catch {
2513
+ }
2514
+ }
2515
+ function workspaceArchiveFieldsFromEnvelope(envelope) {
2516
+ const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
2517
+ const archive = sessionState?.workspaceArchive;
2518
+ const archiveRef = parseWorkspaceArchiveObjectRef2(sessionState?.workspaceArchiveRef);
2519
+ const hasInline = typeof archive === "string" && archive.length > 0;
2520
+ if (!hasInline && !archiveRef) {
2521
+ return null;
2522
+ }
2523
+ const previous = sessionState?.workspaceArchivePrev;
2524
+ const previousRef = parseWorkspaceArchiveObjectRef2(sessionState?.workspaceArchivePrevRef);
2525
+ const metadata = sessionState?.workspaceArchiveMeta;
2526
+ const previousMetadata = sessionState?.workspaceArchivePrevMeta;
2527
+ const capturedAt = sessionState?.workspaceArchiveAt;
2528
+ return {
2529
+ ...hasInline && !archiveRef ? { workspaceArchive: archive } : {},
2530
+ ...archiveRef ? { workspaceArchiveRef: archiveRef } : {},
2531
+ ...metadata !== void 0 ? { workspaceArchiveMeta: metadata } : {},
2532
+ ...typeof previous === "string" && previous.length > 0 && !previousRef ? { workspaceArchivePrev: previous } : {},
2533
+ ...previousRef ? { workspaceArchivePrevRef: previousRef } : {},
2534
+ ...previousMetadata !== void 0 ? { workspaceArchivePrevMeta: previousMetadata } : {},
2535
+ ...typeof capturedAt === "string" && capturedAt.length > 0 ? { workspaceArchiveAt: capturedAt } : {}
2536
+ };
2537
+ }
2538
+ async function materializeSpawnEnvelopeArchive(envelope, objectStorage) {
2539
+ if (!envelope || typeof envelope !== "object") return envelope;
2540
+ const record = envelope;
2541
+ const sessionState = record.sessionState && typeof record.sessionState === "object" && !Array.isArray(record.sessionState) ? record.sessionState : null;
2542
+ if (!sessionState || !parseWorkspaceArchiveObjectRef2(sessionState.workspaceArchiveRef)) {
2543
+ return envelope;
2544
+ }
2545
+ if (!objectStorage) {
2546
+ throw new WorkspaceArchiveIntegrityError(
2547
+ "archive_base64_invalid",
2548
+ "workspace archive object storage is not configured"
2549
+ );
2550
+ }
2551
+ return {
2552
+ ...record,
2553
+ sessionState: await inlineWorkspaceArchiveForRestore(
2554
+ sessionState,
2555
+ (key) => objectStorage.getObjectBytes(key)
2556
+ )
2557
+ };
2558
+ }
2559
+ function legacyNativeArchiveFromEnvelope(envelope) {
2560
+ const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
2561
+ if (!sessionState) return null;
2562
+ const existing = parseWorkspaceArchiveDescriptor(sessionState.workspaceArchiveMeta);
2563
+ if (existing?.version === 2) return null;
2564
+ return describeLegacyNativeSnapshotArchive(
2565
+ sessionState.workspaceArchive,
2566
+ existing?.version === 1 ? Date.parse(existing.capturedAt) : Date.now()
2567
+ );
2568
+ }
2569
+ async function persistSandboxDeadlineRotationCheckpoint(services, ids, session, expected, signal) {
2570
+ const { db, settings } = services;
2571
+ if (signal?.aborted) {
2572
+ return { checkpointed: false, holder: "attempt_fenced", lease: null };
2573
+ }
2574
+ const reinstated = await reinstateTurnLeaseHolder(db, {
2575
+ accountId: ids.accountId,
2576
+ workspaceId: ids.workspaceId,
2577
+ sandboxGroupId: ids.sandboxGroupId,
2578
+ sessionId: ids.sessionId,
2579
+ attemptId: ids.attemptId,
2580
+ expectedEpoch: expected.leaseEpoch,
2581
+ expectedInstanceId: expected.instanceId,
2582
+ leaseTtlMs: settings.sandboxLeaseTtlMs
2583
+ });
2584
+ if (reinstated.status === "reinstated") {
2585
+ const message = "sandbox rotation checkpoint reinstated the live turn holder";
2586
+ const fields = {
2587
+ workspaceId: ids.workspaceId,
2588
+ sessionId: ids.sessionId,
2589
+ sandboxGroupId: ids.sandboxGroupId,
2590
+ leaseEpoch: expected.leaseEpoch,
2591
+ liveness: reinstated.lease.liveness
2592
+ };
2593
+ if (services.observability) services.observability.warn(message, fields);
2594
+ else console.warn(message, fields);
2595
+ }
2596
+ if (reinstated.status !== "present" && reinstated.status !== "reinstated") {
2597
+ return { checkpointed: false, holder: reinstated.status, lease: null };
2598
+ }
2599
+ await maybePersistWarmWorkspaceSnapshot(
2600
+ services,
2601
+ ids,
2602
+ session,
2603
+ expected.leaseEpoch,
2604
+ signal,
2605
+ true
2606
+ );
2607
+ const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
2608
+ const checkpointed = lease?.leaseEpoch === expected.leaseEpoch && lease.instanceId === expected.instanceId && lease.rotationRequestedAt !== null && lease.archiveComplete && signal?.aborted !== true;
2609
+ return { checkpointed, holder: reinstated.status, lease };
2610
+ }
2611
+ async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch, signal, force = false) {
2612
+ const { db, settings } = services;
2613
+ const intervalMs = settings.sandboxSnapshotIntervalMs;
2614
+ if (intervalMs <= 0 && !force) {
2615
+ return false;
2616
+ }
2617
+ const persistable = session;
2618
+ if (typeof persistable.persistWorkspace !== "function" && typeof persistable.persistWorkspaceTar !== "function") {
2619
+ console.error("mid-session workspace snapshot skipped (no persist primitive)", {
2620
+ backendId: typeof persistable.backendId === "string" ? persistable.backendId : null
2621
+ });
2622
+ return false;
2623
+ }
2624
+ const workspacePersistence = persistable.state?.workspacePersistence ?? persistable.state?.providerState?.workspacePersistence;
2625
+ if (signal?.aborted) {
2626
+ return false;
2627
+ }
2628
+ try {
2629
+ const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
2630
+ const canWarmCapture = lease?.liveness === "warm" && lease.instanceId !== null && lease.leaseEpoch === leaseEpoch;
2631
+ const canForceDrainingCapture = force === true && lease?.liveness === "draining" && lease.instanceId !== null && lease.leaseEpoch === leaseEpoch;
2632
+ if (!lease || !canWarmCapture && !canForceDrainingCapture) {
2633
+ console.error("mid-session workspace snapshot skipped (lease not warm)", {
2634
+ sandboxGroupId: ids.sandboxGroupId,
2635
+ expectedEpoch: leaseEpoch,
2636
+ liveness: lease?.liveness ?? null,
2637
+ leaseEpoch: lease?.leaseEpoch ?? null,
2638
+ hasInstance: lease?.instanceId !== null && lease?.instanceId !== void 0
2639
+ });
2640
+ return false;
2641
+ }
2642
+ if (lease.archiveComplete) {
2643
+ return false;
2644
+ }
2645
+ if (lease.instanceId === null) {
2646
+ return false;
2647
+ }
2648
+ const instanceId = lease.instanceId;
2649
+ const captureLiveness = canWarmCapture ? "warm" : "draining";
2650
+ const capturePolicy = providerWorkspaceCapturePolicy(lease.backend, session);
2651
+ if (!capturePolicy || capturePolicy.liveInstance !== "preserved") {
2652
+ console.error("mid-session workspace snapshot skipped (capture policy)", {
2653
+ sandboxGroupId: ids.sandboxGroupId,
2654
+ backend: lease.backend,
2655
+ liveInstance: capturePolicy?.liveInstance ?? null
2656
+ });
2657
+ return false;
2658
+ }
2659
+ const nativeModalPersistence = workspacePersistence === "snapshot_filesystem" || workspacePersistence === "snapshot_directory";
2660
+ const checkpointBinding = nativeModalPersistence ? await resolveModalCheckpointBindingBeforeCapture(settings, persistable, signal) : null;
2661
+ if (signal?.aborted) return false;
2662
+ const captureId = randomUUID2();
2663
+ const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs(settings);
2664
+ const claimed = await claimWorkspaceArchiveCapture(db, {
2665
+ accountId: ids.accountId,
2666
+ workspaceId: ids.workspaceId,
2667
+ sandboxGroupId: ids.sandboxGroupId,
2668
+ captureId,
2669
+ expectedEpoch: leaseEpoch,
2670
+ expectedInstanceId: instanceId,
2671
+ liveness: captureLiveness,
2672
+ captureTimeoutMs,
2673
+ minIntervalMs: force ? 0 : intervalMs,
2674
+ providerReplaySafe: capturePolicy.takeover === "same_request",
2675
+ takeoverSafe: capturePolicy.takeover !== "exclusive",
2676
+ ...captureLiveness === "warm" ? {
2677
+ warmAttempt: {
2678
+ sessionId: ids.sessionId,
2679
+ turnId: ids.turnId,
2680
+ attemptId: ids.attemptId,
2681
+ holderId: sandboxLeaseHolderIdForAttempt(ids.attemptId)
2682
+ }
2683
+ } : {}
2684
+ });
2685
+ if (claimed.status !== "claimed") {
2686
+ console.error("mid-session workspace snapshot skipped (capture claim)", {
2687
+ sandboxGroupId: ids.sandboxGroupId,
2688
+ status: claimed.status
2689
+ });
2690
+ return false;
2691
+ }
2692
+ let timeout;
2693
+ const capturedAtMs = Date.now();
2694
+ const registerCandidate = async (archive) => {
2695
+ if (archive.descriptor.version !== 2 || archive.descriptor.provider !== "modal_snapshot_filesystem" && archive.descriptor.provider !== "modal_snapshot_directory") {
2696
+ return null;
2697
+ }
2698
+ if (!checkpointBinding) {
2699
+ throw new Error("Modal native snapshot has no exact session provider identity");
2700
+ }
2701
+ return await registerSandboxCheckpointArtifact(db, {
2702
+ accountId: ids.accountId,
2703
+ workspaceId: ids.workspaceId,
2704
+ sandboxGroupId: ids.sandboxGroupId,
2705
+ sourceLeaseId: claimed.claim.leaseId,
2706
+ sourceLeaseEpoch: leaseEpoch,
2707
+ sourceInstanceId: instanceId,
2708
+ sourceWorkspaceGeneration: claimed.claim.workspaceGeneration,
2709
+ providerBindingKey: checkpointBinding.key,
2710
+ providerBinding: checkpointBinding.binding,
2711
+ workspaceArchive: archive.base64,
2712
+ workspaceArchiveMeta: archive.descriptor
2713
+ });
2714
+ };
2715
+ const abandonCandidate = async (artifactId, reason) => {
2716
+ await markSandboxCheckpointArtifactDeletePending(db, {
2717
+ accountId: ids.accountId,
2718
+ workspaceId: ids.workspaceId,
2719
+ artifactId,
2720
+ reason
2721
+ }).catch(() => void 0);
2722
+ };
2723
+ const captureAndPublish = (async () => {
2724
+ let candidate = null;
2725
+ let workspaceArchiveRef;
2726
+ try {
2727
+ const archive = await captureVerifiedWorkspaceArchive(session, capturedAtMs, {
2728
+ requestId: claimed.claim.providerRequestId,
2729
+ strategy: capturePolicy.strategy
2730
+ });
2731
+ candidate = await registerCandidate(archive);
2732
+ const priorLease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
2733
+ const priorKeys = collectWorkspaceArchiveObjectKeys(
2734
+ priorLease?.resumeState ?? null
2735
+ );
2736
+ const published = await putVersion1TarArchiveOrInline({
2737
+ backend: lease.backend,
2738
+ objectStorage: services.objectStorage,
2739
+ accountId: ids.accountId,
2740
+ workspaceId: ids.workspaceId,
2741
+ sandboxGroupId: ids.sandboxGroupId,
2742
+ archive: {
2743
+ bytes: archive.bytes,
2744
+ descriptor: archive.descriptor,
2745
+ base64: archive.base64
2746
+ },
2747
+ ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
2748
+ });
2749
+ workspaceArchiveRef = published.workspaceArchiveRef;
2750
+ const { wrote } = await persistWarmSnapshot(db, {
2751
+ accountId: ids.accountId,
2752
+ workspaceId: ids.workspaceId,
2753
+ sessionId: ids.sessionId,
2754
+ turnId: ids.turnId,
2755
+ attemptId: ids.attemptId,
2756
+ sandboxGroupId: ids.sandboxGroupId,
2757
+ expectedEpoch: leaseEpoch,
2758
+ expectedInstanceId: instanceId,
2759
+ expectedWorkspaceGeneration: claimed.claim.workspaceGeneration,
2760
+ captureId,
2761
+ workspaceArchiveMeta: archive.descriptor,
2762
+ ...published,
2763
+ checkpointArtifactId: candidate?.id ?? null,
2764
+ minIntervalMs: force ? 0 : intervalMs,
2765
+ capturedAtMs
2766
+ });
2767
+ if (published.workspaceArchiveRef && services.objectStorage) {
2768
+ if (!wrote) {
2769
+ await deleteUnpublishedWorkspaceArchiveObject(
2770
+ services.objectStorage,
2771
+ published.workspaceArchiveRef,
2772
+ services.sandboxMetrics
2773
+ );
2774
+ } else {
2775
+ const afterLease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
2776
+ const afterKeys = collectWorkspaceArchiveObjectKeys(
2777
+ afterLease?.resumeState ?? null
2778
+ );
2779
+ await deleteWorkspaceArchiveObjectKeys(
2780
+ services.objectStorage,
2781
+ [...priorKeys].filter((key) => !afterKeys.has(key))
2782
+ ).catch(() => void 0);
2783
+ }
2784
+ }
2785
+ if (!wrote && candidate) {
2786
+ await abandonCandidate(candidate.id, "snapshot_publication_fenced");
2787
+ }
2788
+ return wrote;
2789
+ } catch (error) {
2790
+ await deleteUnpublishedWorkspaceArchiveObject(
2791
+ services.objectStorage,
2792
+ workspaceArchiveRef,
2793
+ services.sandboxMetrics
2794
+ );
2795
+ if (candidate) await abandonCandidate(candidate.id, "snapshot_capture_failed");
2796
+ throw error;
2797
+ } finally {
2798
+ await releaseWorkspaceArchiveCapture(db, {
2799
+ accountId: ids.accountId,
2800
+ workspaceId: ids.workspaceId,
2801
+ sandboxGroupId: ids.sandboxGroupId,
2802
+ captureId,
2803
+ expectedEpoch: leaseEpoch,
2804
+ expectedInstanceId: instanceId
2805
+ }).catch((error) => {
2806
+ console.error(
2807
+ "mid-session workspace capture gate release failed",
2808
+ safeSnapshotError(error)
2809
+ );
2810
+ });
2811
+ }
2812
+ })();
2813
+ const settled = captureAndPublish.then(
2814
+ (persisted) => ({ kind: "settled", persisted }),
2815
+ (error) => {
2816
+ console.error(
2817
+ "mid-session workspace snapshot failed (turn unaffected)",
2818
+ safeSnapshotError(error)
2819
+ );
2820
+ return { kind: "settled", persisted: false };
2821
+ }
2822
+ );
2823
+ let cancelListener;
2824
+ const outcome = await Promise.race([
2825
+ settled,
2826
+ ...signal ? [
2827
+ new Promise((resolve) => {
2828
+ cancelListener = () => resolve({ kind: "cancelled" });
2829
+ signal.addEventListener("abort", cancelListener, { once: true });
2830
+ if (signal.aborted) cancelListener();
2831
+ })
2832
+ ] : [],
2833
+ new Promise((resolve) => {
2834
+ timeout = setTimeout(
2835
+ () => resolve({ kind: "timed_out" }),
2836
+ settings.sandboxSnapshotTimeoutMs
2837
+ );
2838
+ })
2839
+ ]).finally(() => {
2840
+ if (timeout) clearTimeout(timeout);
2841
+ if (cancelListener) signal?.removeEventListener("abort", cancelListener);
2842
+ });
2843
+ return outcome.kind === "settled" ? outcome.persisted : false;
2844
+ } catch (error) {
2845
+ console.error(
2846
+ "mid-session workspace snapshot failed (turn unaffected)",
2847
+ safeSnapshotError(error)
2848
+ );
2849
+ return false;
2850
+ }
2851
+ }
2852
+ async function resumeBoxForTurn(services, ids, kind, holderId) {
2853
+ const { db, settings } = services;
2854
+ const os = ids.os ?? "linux";
2855
+ const leaseTtlMs = settings.sandboxLeaseTtlMs;
2856
+ const cancellationSignal = services.cancellationSignal;
2857
+ let releaseStarted = false;
2858
+ let holderRelease = null;
2859
+ let quiescedRelease = null;
2860
+ let holderLivenessTimer;
2861
+ let cancellationListener;
2862
+ const release = (options) => {
2863
+ const writersQuiesced = options?.workspaceWritersQuiesced === true;
2864
+ if (!releaseStarted) {
2865
+ releaseStarted = true;
2866
+ if (cancellationListener) {
2867
+ cancellationSignal?.removeEventListener("abort", cancellationListener);
2868
+ cancellationListener = void 0;
2869
+ }
2870
+ if (holderLivenessTimer) {
2871
+ clearInterval(holderLivenessTimer);
2872
+ holderLivenessTimer = void 0;
2873
+ }
2874
+ }
2875
+ const persistRelease = async (workspaceWritersQuiesced) => {
2876
+ await releaseLeaseHolder(db, {
2877
+ accountId: ids.accountId,
2878
+ workspaceId: ids.workspaceId,
2879
+ sandboxGroupId: ids.sandboxGroupId,
2880
+ kind,
2881
+ holderId,
2882
+ idleGraceMs: settings.sandboxIdleGraceMs,
2883
+ ...workspaceWritersQuiesced ? { workspaceWritersQuiesced: true } : {}
2884
+ });
2885
+ };
2886
+ if (writersQuiesced) {
2887
+ if (!quiescedRelease) {
2888
+ const predecessor = holderRelease;
2889
+ const attempt = (predecessor ? predecessor.catch(() => void 0) : Promise.resolve()).then(
2890
+ async () => await persistRelease(true)
2891
+ );
2892
+ const tracked = attempt.catch((error) => {
2893
+ if (quiescedRelease === tracked) quiescedRelease = null;
2894
+ throw error;
2895
+ });
2896
+ quiescedRelease = tracked;
2897
+ holderRelease = tracked;
2898
+ }
2899
+ return quiescedRelease;
2900
+ }
2901
+ if (!holderRelease) {
2902
+ const attempt = persistRelease(false);
2903
+ const tracked = attempt.catch((error) => {
2904
+ if (holderRelease === tracked) holderRelease = null;
2905
+ throw error;
2906
+ });
2907
+ holderRelease = tracked;
2908
+ }
2909
+ return holderRelease;
2910
+ };
2911
+ const cancellationError = () => cancellationSignal?.reason instanceof Error ? cancellationSignal.reason : new Error("Sandbox resume was cancelled with its owning turn attempt");
2912
+ const throwIfReleasedOrCancelled = () => {
2913
+ if (releaseStarted || cancellationSignal?.aborted) {
2914
+ throw cancellationError();
2915
+ }
2916
+ };
2917
+ const acquired = await acquireLease(db, {
2918
+ accountId: ids.accountId,
2919
+ workspaceId: ids.workspaceId,
2920
+ sandboxGroupId: ids.sandboxGroupId,
2921
+ kind,
2922
+ holderId,
2923
+ subjectId: ids.sessionId,
2924
+ backend: ids.backend,
2925
+ os,
2926
+ // IMAGE IS SHARED STATE (B3): thread the resolved image so the lease stamps it +
2927
+ // conflicts on a live box already running a different image. A
2928
+ // SandboxImageConflictError propagates while another holder is active; a
2929
+ // solo change requests a capture-and-drain rotation and this attempt retries
2930
+ // after the cold successor can safely stamp the new image.
2931
+ ...ids.image ? { image: ids.image } : {},
2932
+ // RIG IS SHARED STATE (M3): thread the frozen rig version so the lease stamps it
2933
+ // + conflicts on a live box under a different rig. A SandboxRigConflictError
2934
+ // propagates while another holder is active; a solo change uses the same
2935
+ // durable capture-and-drain rotation as an image change.
2936
+ ...ids.rigVersionId ? { rigVersionId: ids.rigVersionId } : {},
2937
+ leaseTtlMs,
2938
+ warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs,
2939
+ captureWaitMs: sandboxLifecycleTransitionWaitMs(settings),
2940
+ ...cancellationSignal ? { waitSignal: cancellationSignal } : {}
2941
+ });
2942
+ if (cancellationSignal) {
2943
+ cancellationListener = () => {
2944
+ void release().catch(() => void 0);
2945
+ };
2946
+ cancellationSignal.addEventListener("abort", cancellationListener, { once: true });
2947
+ }
2948
+ if (cancellationSignal?.aborted) {
2949
+ await release();
2950
+ throw cancellationError();
2951
+ }
2952
+ let holderLeaseHeartbeat = acquired.role === "spawner" ? {
2953
+ expectedEpoch: acquired.lease.leaseEpoch,
2954
+ leaseTtlMs: settings.sandboxLeaseWarmingTtlMs
2955
+ } : (acquired.role === "attached" || acquired.role === "rearmed") && acquired.lease.liveness === "warm" ? { expectedEpoch: acquired.lease.leaseEpoch, leaseTtlMs } : null;
2956
+ let providerRenewalTarget = acquired.lease.liveness === "warm" && acquired.lease.instanceId ? {
2957
+ backend: ids.backend,
2958
+ instanceId: acquired.lease.instanceId
2959
+ } : null;
2960
+ let providerRenewedAtMs = Date.now();
2961
+ let providerRenewalInFlight = null;
2962
+ const maybeRenewProviderExpiration = async (force = false) => {
2963
+ const target = providerRenewalTarget;
2964
+ if (!target || providerRenewalInFlight) return;
2965
+ const ttlSeconds = effectiveSandboxLifecycle(settings, target.backend).renewableTtlSeconds;
2966
+ if (ttlSeconds === null) return;
2967
+ const intervalMs = Math.max(1e4, Math.floor(ttlSeconds * 1e3 / 3));
2968
+ if (!force && Date.now() - providerRenewedAtMs < intervalMs) return;
2969
+ providerRenewalInFlight = renewSandboxProviderExpiration({
2970
+ backend: target.backend,
2971
+ settings,
2972
+ instanceId: target.instanceId,
2973
+ ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
2974
+ }).then(() => {
2975
+ providerRenewedAtMs = Date.now();
2976
+ }).finally(() => {
2977
+ providerRenewalInFlight = null;
2978
+ });
2979
+ await providerRenewalInFlight;
2980
+ };
2981
+ const releaseDeadHolder = (reason) => {
2982
+ const message = "sandbox resume holder released: holder no longer live";
2983
+ const fields = {
2984
+ workspaceId: ids.workspaceId,
2985
+ sessionId: ids.sessionId,
2986
+ sandboxGroupId: ids.sandboxGroupId,
2987
+ holderKind: kind,
2988
+ reason
2989
+ };
2990
+ if (services.observability) services.observability.warn(message, fields);
2991
+ else console.warn(message, fields);
2992
+ void release().catch(() => void 0);
2993
+ };
2994
+ holderLivenessTimer = setInterval(() => {
2995
+ const heartbeat = holderLeaseHeartbeat;
2996
+ if (heartbeat) {
2997
+ void heartbeatLeaseHolderStatus(db, {
2998
+ accountId: ids.accountId,
2999
+ workspaceId: ids.workspaceId,
3000
+ sandboxGroupId: ids.sandboxGroupId,
3001
+ kind,
3002
+ holderId,
3003
+ expectedEpoch: heartbeat.expectedEpoch,
3004
+ leaseTtlMs: heartbeat.leaseTtlMs
3005
+ }).then((status) => {
3006
+ if (heartbeat !== holderLeaseHeartbeat) return;
3007
+ if (!status.holderAlive) {
3008
+ releaseDeadHolder(status.fence);
3009
+ return;
3010
+ }
3011
+ if (status.fence === "epoch") {
3012
+ releaseDeadHolder(status.fence);
3013
+ return;
3014
+ }
3015
+ if (status.leaseExtended || status.fence === "rotation_requested") {
3016
+ void maybeRenewProviderExpiration().catch(() => void 0);
3017
+ }
3018
+ }).catch(() => void 0);
3019
+ return;
3020
+ }
3021
+ void touchLeaseHolder(db, {
3022
+ accountId: ids.accountId,
3023
+ workspaceId: ids.workspaceId,
3024
+ sandboxGroupId: ids.sandboxGroupId,
3025
+ kind,
3026
+ holderId
3027
+ }).then((touched) => {
3028
+ if (heartbeat !== holderLeaseHeartbeat) return;
3029
+ if (!touched) {
3030
+ releaseDeadHolder("holder_gone");
3031
+ return;
3032
+ }
3033
+ void maybeRenewProviderExpiration().catch(() => void 0);
3034
+ }).catch(() => void 0);
3035
+ }, services.holderLivenessIntervalMs ?? 1e4);
3036
+ if ("unref" in holderLivenessTimer && typeof holderLivenessTimer.unref === "function") {
3037
+ holderLivenessTimer.unref();
3038
+ }
3039
+ if (acquired.role === "blocked") {
3040
+ await release();
3041
+ throw new SandboxLeaseRecoveryBlockedError(
3042
+ ids.sandboxGroupId,
3043
+ acquired.lease.leaseEpoch,
3044
+ acquired.code,
3045
+ acquired.lease.recovery
3046
+ );
3047
+ }
3048
+ if (acquired.role === "fenced") {
3049
+ await release();
3050
+ if (acquired.reason !== "superseded") {
3051
+ throw new SandboxLeaseTransitionError(
3052
+ ids.sandboxGroupId,
3053
+ acquired.lease.leaseEpoch,
3054
+ acquired.reason,
3055
+ acquired.lease.backend,
3056
+ acquired.lease.instanceId,
3057
+ acquired.lease.liveness
3058
+ );
3059
+ }
3060
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch, {
3061
+ backend: acquired.lease.backend,
3062
+ instanceId: acquired.lease.instanceId,
3063
+ liveness: acquired.lease.liveness
3064
+ });
3065
+ }
3066
+ if (acquired.role === "spawner") {
3067
+ const expectedEpoch = acquired.lease.leaseEpoch;
3068
+ let createdEstablished = null;
3069
+ let rematerialization = null;
3070
+ try {
3071
+ const envelope = await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
3072
+ const fallbackArchiveEnvelope = acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(envelope) !== null ? withoutSandboxProviderIdentity(envelope) : null;
3073
+ let spawnEnvelope = fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
3074
+ const archiveSource = acquired.lease.recovery.archive.status === "none" ? fallbackArchiveEnvelope : acquired.lease.resumeState;
3075
+ const continuityRecovery = acquired.lease.recovery.continuity;
3076
+ if (acquired.lease.recovery.archive.status === "available" && acquired.lease.archiveComplete || acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(archiveSource) !== null) {
3077
+ const rematerializationId = crypto.randomUUID();
3078
+ const legacyNativeArchive = legacyNativeArchiveFromEnvelope(archiveSource);
3079
+ const begun = await beginSandboxRematerialization(db, {
3080
+ accountId: ids.accountId,
3081
+ workspaceId: ids.workspaceId,
3082
+ sandboxGroupId: ids.sandboxGroupId,
3083
+ expectedEpoch,
3084
+ rematerializationId,
3085
+ archiveSource,
3086
+ legacyNativeArchive
3087
+ });
3088
+ if (begun.status !== "started") {
3089
+ if (begun.code === "stale_epoch" || begun.code === "attempt_conflict") {
3090
+ throw new SandboxLeaseSupersededError(
3091
+ ids.sandboxGroupId,
3092
+ begun.lease?.leaseEpoch ?? expectedEpoch
3093
+ );
3094
+ }
3095
+ throw new SandboxLeaseRecoveryBlockedError(
3096
+ ids.sandboxGroupId,
3097
+ begun.lease?.leaseEpoch ?? expectedEpoch,
3098
+ begun.code === "archive_unverified" ? "restore_degraded" : "restore_unrecoverable",
3099
+ begun.lease?.recovery ?? acquired.lease.recovery
3100
+ );
3101
+ }
3102
+ spawnEnvelope = begun.lease.resumeState ?? spawnEnvelope;
3103
+ const selectedRevision = begun.lease.recovery.restore.selectedRevision;
3104
+ if (!selectedRevision) {
3105
+ throw new WorkspaceArchiveIntegrityError(
3106
+ "archive_metadata_invalid",
3107
+ "sandbox rematerialization selected no durable archive revision"
3108
+ );
3109
+ }
3110
+ rematerialization = {
3111
+ id: rematerializationId,
3112
+ selectedRevision,
3113
+ workspaceGeneration: begun.lease.workspaceGeneration,
3114
+ providerBindingKey: begun.checkpointArtifact?.providerBindingKey ?? null,
3115
+ legacyCheckpoint: begun.checkpointArtifact === null ? legacyNativeArchive : null,
3116
+ legacyProviderBinding: null
3117
+ };
3118
+ } else if (acquired.lease.recovery.archive.status !== "none" && !continuityRecovery) {
3119
+ throw new SandboxLeaseRecoveryBlockedError(
3120
+ ids.sandboxGroupId,
3121
+ expectedEpoch,
3122
+ "restore_degraded",
3123
+ acquired.lease.recovery
3124
+ );
3125
+ }
3126
+ const providerCreateStartedAt = /* @__PURE__ */ new Date();
3127
+ const persistArchiveSource = spawnEnvelope && typeof spawnEnvelope === "object" ? spawnEnvelope : null;
3128
+ const hydrateEnvelopeRaw = await materializeSpawnEnvelopeArchive(
3129
+ persistArchiveSource,
3130
+ services.objectStorage
3131
+ );
3132
+ const hydrateEnvelope = hydrateEnvelopeRaw && typeof hydrateEnvelopeRaw === "object" ? hydrateEnvelopeRaw : null;
3133
+ const established = await establishSandboxSessionFromEnvelope(settings, hydrateEnvelope, {
3134
+ sessionId: ids.sessionId,
3135
+ recovery: "create-or-restore",
3136
+ backendOverride: ids.backend,
3137
+ ...ids.environment ? { environment: ids.environment } : {},
3138
+ ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {},
3139
+ ...services.logicalFallbackSettings ? { logicalFallbackSettings: services.logicalFallbackSettings } : {},
3140
+ onSandboxCreated: async (created) => {
3141
+ createdEstablished = created;
3142
+ providerRenewalTarget = {
3143
+ backend: ids.backend,
3144
+ instanceId: created.instanceId
3145
+ };
3146
+ providerRenewedAtMs = Date.now();
3147
+ throwIfReleasedOrCancelled();
3148
+ if (rematerialization && (rematerialization.providerBindingKey || rematerialization.legacyCheckpoint)) {
3149
+ if (created.backendId !== "modal") {
3150
+ throw new WorkspaceArchiveIntegrityError(
3151
+ "native_snapshot_reference_invalid",
3152
+ "Modal checkpoint restore resolved a non-Modal sandbox backend"
3153
+ );
3154
+ }
3155
+ if (rematerialization.providerBindingKey) {
3156
+ if (!await modalSessionMatchesCheckpointProviderBinding(
3157
+ settings,
3158
+ created.session,
3159
+ rematerialization.providerBindingKey
3160
+ )) {
3161
+ throw new WorkspaceArchiveIntegrityError(
3162
+ "native_snapshot_reference_invalid",
3163
+ "Modal checkpoint restore refused because the authenticated provider workspace changed"
3164
+ );
3165
+ }
3166
+ } else {
3167
+ const resolved = await resolveModalCheckpointProviderBindingForSession(
3168
+ settings,
3169
+ created.session
3170
+ );
3171
+ if (rematerialization.legacyProviderBinding && rematerialization.legacyProviderBinding.key !== resolved.key) {
3172
+ throw new WorkspaceArchiveIntegrityError(
3173
+ "native_snapshot_reference_invalid",
3174
+ "Legacy Modal checkpoint restore crossed authenticated provider workspaces"
3175
+ );
3176
+ }
3177
+ rematerialization.legacyProviderBinding = resolved;
3178
+ }
3179
+ }
3180
+ const resumeEnvelope2 = requirePersistableReplacementSandboxEnvelope(
3181
+ await serializeReplacementSandboxEnvelope(created, persistArchiveSource),
3182
+ created.backendId
3183
+ );
3184
+ const recorded = await recordWarmingSandboxCreated(db, {
3185
+ accountId: ids.accountId,
3186
+ workspaceId: ids.workspaceId,
3187
+ sandboxGroupId: ids.sandboxGroupId,
3188
+ expectedEpoch,
3189
+ rematerializationId: rematerialization?.id ?? null,
3190
+ ...created.providerContinuity ? { continuityRecovery: created.providerContinuity } : {},
3191
+ instanceId: created.instanceId,
3192
+ resumeBackendId: created.backendId,
3193
+ resumeState: resumeEnvelope2,
3194
+ ...created.backendId === "modal" ? {
3195
+ providerCreatedAt: providerCreateStartedAt,
3196
+ providerDeadlineAt: new Date(
3197
+ providerCreateStartedAt.getTime() + settings.modalTimeoutSeconds * 1e3
3198
+ )
3199
+ } : {},
3200
+ leaseTtlMs,
3201
+ // Keep the warming budget after create(): manifest setup and
3202
+ // commitWarmingToWarm still run, and can exceed the 90s turn TTL.
3203
+ warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs
3204
+ });
3205
+ if (!recorded.recorded) {
3206
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
3207
+ }
3208
+ if (created.backendId === "modal") {
3209
+ await tagModalSandbox(settings, created.instanceId, {
3210
+ leaseId: acquired.lease.id,
3211
+ workspaceId: ids.workspaceId,
3212
+ sandboxGroupId: ids.sandboxGroupId
3213
+ }).catch(() => void 0);
3214
+ }
3215
+ throwIfReleasedOrCancelled();
3216
+ },
3217
+ onWorkspaceRestoreVerifying: async (descriptor) => {
3218
+ if (!rematerialization || descriptor.revision !== rematerialization.selectedRevision) {
3219
+ throw new WorkspaceArchiveIntegrityError(
3220
+ "archive_metadata_invalid",
3221
+ `hydrated archive revision ${descriptor.revision} does not match the selected rematerialization revision`
3222
+ );
3223
+ }
3224
+ if (rematerialization.legacyCheckpoint) {
3225
+ const binding = rematerialization.legacyProviderBinding;
3226
+ if (!binding) {
3227
+ throw new WorkspaceArchiveIntegrityError(
3228
+ "native_snapshot_reference_invalid",
3229
+ "Legacy Modal checkpoint restore produced no authenticated provider identity"
3230
+ );
3231
+ }
3232
+ const adopted = await adoptLegacyModalCheckpointArtifact(db, {
3233
+ accountId: ids.accountId,
3234
+ workspaceId: ids.workspaceId,
3235
+ sandboxGroupId: ids.sandboxGroupId,
3236
+ leaseId: acquired.lease.id,
3237
+ leaseEpoch: expectedEpoch,
3238
+ workspaceGeneration: rematerialization.workspaceGeneration,
3239
+ slot: "current",
3240
+ archiveBase64: rematerialization.legacyCheckpoint.archiveBase64,
3241
+ descriptor: rematerialization.legacyCheckpoint.descriptor,
3242
+ providerBindingKey: binding.key,
3243
+ providerBinding: binding.binding,
3244
+ rematerializationId: rematerialization.id
3245
+ });
3246
+ if (!adopted) {
3247
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
3248
+ }
3249
+ rematerialization.providerBindingKey = binding.key;
3250
+ }
3251
+ const verifying = await markSandboxRestoreVerifying(db, {
3252
+ accountId: ids.accountId,
3253
+ workspaceId: ids.workspaceId,
3254
+ sandboxGroupId: ids.sandboxGroupId,
3255
+ expectedEpoch,
3256
+ rematerializationId: rematerialization.id
3257
+ });
3258
+ if (!verifying.wrote) {
3259
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
3260
+ }
3261
+ }
3262
+ });
3263
+ createdEstablished = established;
3264
+ throwIfReleasedOrCancelled();
3265
+ await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
3266
+ sandboxGroupId: ids.sandboxGroupId
3267
+ });
3268
+ await maybeRenewProviderExpiration(true);
3269
+ throwIfReleasedOrCancelled();
3270
+ const resumeEnvelope = requirePersistableReplacementSandboxEnvelope(
3271
+ await serializeReplacementSandboxEnvelope(established, persistArchiveSource),
3272
+ established.backendId
3273
+ );
3274
+ throwIfReleasedOrCancelled();
3275
+ if (rematerialization && !established.providerContinuity && established.restoredArchive?.revision !== rematerialization.selectedRevision) {
3276
+ throw new WorkspaceArchiveIntegrityError(
3277
+ "workspace_fingerprint_mismatch",
3278
+ "sandbox restore completed without the exact selected durable archive revision"
3279
+ );
3280
+ }
3281
+ const committed = await commitWarmingToWarm(db, {
3282
+ accountId: ids.accountId,
3283
+ workspaceId: ids.workspaceId,
3284
+ sandboxGroupId: ids.sandboxGroupId,
3285
+ expectedEpoch,
3286
+ instanceId: established.instanceId,
3287
+ // Viewer attach resolves and records the tunnel URL lazily. A headless
3288
+ // turn must not touch the optional desktop data plane.
3289
+ dataPlaneUrl: null,
3290
+ resumeBackendId: established.backendId,
3291
+ resumeState: resumeEnvelope,
3292
+ ...established.providerContinuity ? { continuityRecovery: established.providerContinuity } : rematerialization ? {
3293
+ rematerialization: {
3294
+ id: rematerialization.id,
3295
+ verifiedRevision: rematerialization.selectedRevision
3296
+ }
3297
+ } : {},
3298
+ leaseTtlMs
3299
+ });
3300
+ if (!committed.committed || !committed.lease) {
3301
+ const terminated = await terminateEstablishedSandbox(established);
3302
+ if (terminated && rematerialization && !established.providerContinuity) {
3303
+ await failSandboxRematerialization(db, {
3304
+ accountId: ids.accountId,
3305
+ workspaceId: ids.workspaceId,
3306
+ sandboxGroupId: ids.sandboxGroupId,
3307
+ expectedEpoch,
3308
+ rematerializationId: rematerialization.id,
3309
+ failureCode: committed.reason ?? "warm_commit_rejected",
3310
+ retryable: false
3311
+ });
3312
+ } else if (terminated) {
3313
+ await failWarmingToCold(db, {
3314
+ accountId: ids.accountId,
3315
+ workspaceId: ids.workspaceId,
3316
+ sandboxGroupId: ids.sandboxGroupId,
3317
+ expectedEpoch
3318
+ });
3319
+ }
3320
+ await release();
3321
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
3322
+ }
3323
+ holderLeaseHeartbeat = {
3324
+ expectedEpoch: committed.lease.leaseEpoch,
3325
+ leaseTtlMs
3326
+ };
3327
+ throwIfReleasedOrCancelled();
3328
+ return { established, leaseEpoch: committed.lease.leaseEpoch, release };
3329
+ } catch (error) {
3330
+ if (error instanceof SandboxLeaseSupersededError) {
3331
+ await terminateEstablishedSandbox(createdEstablished);
3332
+ await release();
3333
+ throw error;
3334
+ }
3335
+ const terminated = await terminateEstablishedSandbox(createdEstablished);
3336
+ if (terminated) {
3337
+ const continuityUnavailable = error instanceof SandboxProviderContinuityUnavailableError;
3338
+ if (rematerialization && !createdEstablished?.providerContinuity) {
3339
+ await failSandboxRematerialization(db, {
3340
+ accountId: ids.accountId,
3341
+ workspaceId: ids.workspaceId,
3342
+ sandboxGroupId: ids.sandboxGroupId,
3343
+ expectedEpoch,
3344
+ rematerializationId: rematerialization.id,
3345
+ failureCode: continuityUnavailable ? "provider_continuity_unavailable" : error instanceof WorkspaceArchiveIntegrityError ? error.code : "sandbox_rematerialization_failed",
3346
+ retryable: continuityUnavailable ? error.retryable : error instanceof WorkspaceArchiveIntegrityError ? error.retryable : true,
3347
+ ...continuityUnavailable ? { discardContinuity: true } : {}
3348
+ });
3349
+ } else {
3350
+ await failWarmingToCold(db, {
3351
+ accountId: ids.accountId,
3352
+ workspaceId: ids.workspaceId,
3353
+ sandboxGroupId: ids.sandboxGroupId,
3354
+ expectedEpoch,
3355
+ ...continuityUnavailable ? { discardContinuity: true } : {}
3356
+ });
3357
+ }
3358
+ }
3359
+ await release();
3360
+ recordSandboxWarmingTimeout(services.sandboxMetrics, error);
3361
+ throw sandboxProvisionStageError(rematerialization ? "archive_recovery" : "create", error);
3362
+ }
3363
+ }
3364
+ try {
3365
+ let leaseEpoch = acquired.lease.leaseEpoch;
3366
+ if (acquired.lease.liveness === "warming") {
3367
+ leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
3368
+ holderLeaseHeartbeat = { expectedEpoch: leaseEpoch, leaseTtlMs };
3369
+ }
3370
+ throwIfReleasedOrCancelled();
3371
+ const live = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
3372
+ if (!live || live.liveness !== "warm" || live.leaseEpoch !== leaseEpoch || live.instanceId === null) {
3373
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, live?.leaseEpoch ?? leaseEpoch);
3374
+ }
3375
+ let established;
3376
+ try {
3377
+ const establish = services.establishAttachedSandbox ?? establishSandboxSessionFromEnvelope;
3378
+ established = await establish(settings, live.resumeState, {
3379
+ sessionId: ids.sessionId,
3380
+ recovery: "resume-only",
3381
+ backendOverride: ids.backend,
3382
+ ...ids.environment ? { environment: ids.environment } : {},
3383
+ ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
3384
+ });
3385
+ throwIfReleasedOrCancelled();
3386
+ if (services.verifyAttachedSandboxReadiness) {
3387
+ await services.verifyAttachedSandboxReadiness(established);
3388
+ } else {
3389
+ await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
3390
+ sandboxGroupId: ids.sandboxGroupId
3391
+ });
3392
+ }
3393
+ providerRenewalTarget = {
3394
+ backend: ids.backend,
3395
+ instanceId: established.instanceId
3396
+ };
3397
+ await maybeRenewProviderExpiration(true);
3398
+ throwIfReleasedOrCancelled();
3399
+ } catch (error) {
3400
+ if (!isProviderSandboxNotFoundError(ids.backend, error)) {
3401
+ throw sandboxProvisionStageError("resume", error);
3402
+ }
3403
+ const marked = await markWarmLeaseInstanceLost(db, {
3404
+ accountId: ids.accountId,
3405
+ workspaceId: ids.workspaceId,
3406
+ sandboxGroupId: ids.sandboxGroupId,
3407
+ expectedEpoch: leaseEpoch,
3408
+ expectedInstanceId: live.instanceId
3409
+ });
3410
+ if (marked.status === "marked") {
3411
+ await services.onSandboxLost?.({
3412
+ sandboxGroupId: ids.sandboxGroupId,
3413
+ instanceId: live.instanceId,
3414
+ leaseEpoch: marked.lease.leaseEpoch
3415
+ });
3416
+ throw new SandboxLeaseInstanceLostError(
3417
+ ids.sandboxGroupId,
3418
+ marked.lease.leaseEpoch,
3419
+ live.instanceId
3420
+ );
3421
+ }
3422
+ throw new SandboxLeaseSupersededError(
3423
+ ids.sandboxGroupId,
3424
+ marked.lease?.leaseEpoch ?? leaseEpoch
3425
+ );
3426
+ }
3427
+ throwIfReleasedOrCancelled();
3428
+ return { established, leaseEpoch, release };
3429
+ } catch (error) {
3430
+ await release();
3431
+ recordSandboxWarmingTimeout(services.sandboxMetrics, error);
3432
+ throw error;
3433
+ }
3434
+ }
3435
+ async function waitForWarm(services, ids) {
3436
+ const { db, settings } = services;
3437
+ const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
3438
+ let instanceId = null;
3439
+ while (Date.now() < deadline) {
3440
+ await sleep(WARMING_POLL_INTERVAL_MS);
3441
+ const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
3442
+ if (!lease) {
3443
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
3444
+ }
3445
+ instanceId = lease.instanceId;
3446
+ if (lease.liveness === "warm") {
3447
+ return { leaseEpoch: lease.leaseEpoch };
3448
+ }
3449
+ if (lease.liveness === "draining") {
3450
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
3451
+ }
3452
+ if (lease.liveness === "cold") {
3453
+ if (isRetryableDegradedRestore(lease.recovery.restore)) {
3454
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
3455
+ }
3456
+ if (lease.recovery.restore.status === "degraded" || lease.recovery.restore.status === "unrecoverable") {
3457
+ throw new SandboxLeaseRecoveryBlockedError(
3458
+ ids.sandboxGroupId,
3459
+ lease.leaseEpoch,
3460
+ lease.recovery.restore.status === "degraded" ? "restore_degraded" : "restore_unrecoverable",
3461
+ lease.recovery
3462
+ );
3463
+ }
3464
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
3465
+ }
3466
+ }
3467
+ throw new SandboxSiblingWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs, {
3468
+ sandboxGroupId: ids.sandboxGroupId,
3469
+ instanceId
3470
+ });
3471
+ }
3472
+
3473
+ // src/activities/agent-turn/sandbox-provision.ts
3474
+ import { randomUUID as randomUUID3 } from "crypto";
3475
+ var TurnOperationCancelledError = class extends Error {
3476
+ constructor(reason) {
3477
+ super("Turn operation was cancelled with its owning turn", {
3478
+ ...reason instanceof Error ? { cause: reason } : {}
3479
+ });
3480
+ this.reason = reason;
3481
+ }
3482
+ name = "TurnOperationCancelledError";
3483
+ };
3484
+ var SandboxDeadlineRotationError = class extends Error {
3485
+ constructor(sandboxGroupId, leaseEpoch) {
3486
+ super(
3487
+ `Sandbox ${sandboxGroupId} reached its provider rotation boundary at lease epoch ${leaseEpoch}`
3488
+ );
3489
+ this.sandboxGroupId = sandboxGroupId;
3490
+ this.leaseEpoch = leaseEpoch;
3491
+ }
3492
+ name = "SandboxDeadlineRotationError";
3493
+ };
3494
+ function turnOperationCancellationFailure(error) {
3495
+ if (error instanceof CancelledFailure) return error;
3496
+ if (!(error instanceof TurnOperationCancelledError)) return null;
3497
+ return error.reason instanceof CancelledFailure ? error.reason : new CancelledFailure("TURN_SANDBOX_PROVISION_CANCELLED", [], error);
3498
+ }
3499
+ function throwIfTurnOperationCancelled(signal) {
3500
+ if (signal?.aborted) {
3501
+ throw new TurnOperationCancelledError(signal.reason);
3502
+ }
3503
+ }
3504
+ async function waitForTurnOperation(operation, signal, disposeLateResult) {
3505
+ if (!signal) return await operation;
3506
+ let rejectCancellation;
3507
+ const cancelled = new Promise((_resolve, reject) => {
3508
+ rejectCancellation = reject;
3509
+ });
3510
+ const cancel = () => {
3511
+ rejectCancellation?.(new TurnOperationCancelledError(signal.reason));
3512
+ };
3513
+ signal.addEventListener("abort", cancel, { once: true });
3514
+ if (signal.aborted) cancel();
3515
+ try {
3516
+ const result = await Promise.race([operation, cancelled]);
3517
+ if (signal.aborted) {
3518
+ throw new TurnOperationCancelledError(signal.reason);
3519
+ }
3520
+ return result;
3521
+ } catch (error) {
3522
+ if (signal.aborted) {
3523
+ void operation.then(async (result) => await disposeLateResult?.(result)).catch(() => void 0);
3524
+ if (!(error instanceof TurnOperationCancelledError)) {
3525
+ throw new TurnOperationCancelledError(signal.reason);
3526
+ }
3527
+ }
3528
+ throw error;
3529
+ } finally {
3530
+ signal.removeEventListener("abort", cancel);
3531
+ }
3532
+ }
3533
+ function sleep2(ms) {
3534
+ return new Promise((resolve) => setTimeout(resolve, ms));
3535
+ }
3536
+ function sandboxProvisionErrorChain(error) {
3537
+ const chain = [];
3538
+ const seen = /* @__PURE__ */ new Set();
3539
+ let current = error;
3540
+ for (let depth = 0; depth < 6; depth += 1) {
3541
+ chain.push(current);
3542
+ if (!current || typeof current !== "object" || seen.has(current)) break;
3543
+ seen.add(current);
3544
+ if (current instanceof SandboxProvisionStageError) {
3545
+ current = current.source;
3546
+ continue;
3547
+ }
3548
+ const cause = Reflect.get(current, "cause");
3549
+ if (cause === void 0 || cause === current) break;
3550
+ current = cause;
3551
+ }
3552
+ return chain;
3553
+ }
3554
+ function firstProvisionError(chain, constructor) {
3555
+ return chain.find((value) => value instanceof constructor) ?? null;
3556
+ }
3557
+ function classifySandboxLogicalProvisionFailure(backend, error) {
3558
+ const chain = sandboxProvisionErrorChain(error);
3559
+ const wrappedStage = chain.map((value) => sandboxProvisionFailureStage(value)).find((stage) => stage !== null) ?? null;
3560
+ const warming = chain.find(
3561
+ (value) => value instanceof SandboxWarmingTimeoutError
3562
+ ) ?? null;
3563
+ if (warming) {
3564
+ const execReadiness = warming.stage === "exec_readiness";
3565
+ return {
3566
+ category: execReadiness ? "exec_readiness" : "sibling_warming",
3567
+ stage: execReadiness ? "exec_readiness" : "sibling_warming",
3568
+ code: warming.code,
3569
+ expected: false,
3570
+ retryable: false
3571
+ };
3572
+ }
3573
+ const readiness = firstProvisionError(chain, SandboxExecReadinessError2);
3574
+ if (readiness) {
3575
+ return {
3576
+ category: "exec_readiness",
3577
+ stage: "exec_readiness",
3578
+ code: readiness.code,
3579
+ expected: false,
3580
+ retryable: false
3581
+ };
3582
+ }
3583
+ const superseded = firstProvisionError(chain, SandboxLeaseSupersededError2);
3584
+ if (superseded) {
3585
+ return {
3586
+ category: "lease_superseded",
3587
+ stage: "lease_admission",
3588
+ code: "lease_superseded",
3589
+ expected: true,
3590
+ retryable: true
3591
+ };
3592
+ }
3593
+ const transition = firstProvisionError(chain, SandboxLeaseTransitionError2);
3594
+ if (transition) {
3595
+ return {
3596
+ category: "drain_capture_wait",
3597
+ stage: "lifecycle_wait",
3598
+ code: transition.reason,
3599
+ expected: true,
3600
+ retryable: true
3601
+ };
3602
+ }
3603
+ const deadlineRotation = firstProvisionError(chain, SandboxDeadlineRotationError);
3604
+ if (deadlineRotation) {
3605
+ return {
3606
+ category: "drain_capture_wait",
3607
+ stage: "lifecycle_wait",
3608
+ code: "provider_deadline_rotation",
3609
+ expected: true,
3610
+ retryable: false
3611
+ };
3612
+ }
3613
+ const recoveryBlocked = firstProvisionError(chain, SandboxLeaseRecoveryBlockedError2);
3614
+ if (recoveryBlocked) {
3615
+ return {
3616
+ category: "archive_recovery",
3617
+ stage: "archive_recovery",
3618
+ code: recoveryBlocked.code,
3619
+ expected: false,
3620
+ retryable: false
3621
+ };
3622
+ }
3623
+ const archive = firstProvisionError(chain, WorkspaceArchiveIntegrityError2);
3624
+ if (archive) {
3625
+ return {
3626
+ category: "archive_recovery",
3627
+ stage: "archive_recovery",
3628
+ code: archive.code,
3629
+ expected: false,
3630
+ retryable: archive.retryable
3631
+ };
3632
+ }
3633
+ const continuity = firstProvisionError(chain, SandboxProviderContinuityUnavailableError2);
3634
+ if (continuity) {
3635
+ return {
3636
+ category: "archive_recovery",
3637
+ stage: "archive_recovery",
3638
+ code: "provider_continuity_unavailable",
3639
+ expected: false,
3640
+ retryable: continuity.retryable
3641
+ };
3642
+ }
3643
+ const configuration = chain.find(
3644
+ (value) => value instanceof SandboxConfigError || value instanceof SandboxProviderUnavailableError || value instanceof SandboxImageConflictError || value instanceof SandboxRigConflictError
3645
+ );
3646
+ if (configuration) {
3647
+ return {
3648
+ category: "configuration",
3649
+ stage: wrappedStage ?? "configuration",
3650
+ code: configuration instanceof SandboxImageConflictError ? "image_conflict" : configuration instanceof SandboxRigConflictError ? "rig_conflict" : configuration instanceof SandboxProviderUnavailableError ? "provider_unavailable" : "sandbox_config",
3651
+ expected: false,
3652
+ retryable: false
3653
+ };
3654
+ }
3655
+ const resumeFailure = chain.find(
3656
+ (value) => value instanceof SandboxResumeStateUnavailableError || value instanceof SandboxResumeIdentityMismatchError || value instanceof SandboxResumeIdentityUnavailableError || value instanceof SandboxExactResumeReplacedError || value instanceof SandboxExactResumeInstanceUnavailableError
3657
+ );
3658
+ if (resumeFailure) {
3659
+ return {
3660
+ category: "resume",
3661
+ stage: "resume",
3662
+ code: resumeFailure instanceof SandboxResumeStateUnavailableError ? "resume_state_unavailable" : resumeFailure instanceof SandboxResumeIdentityMismatchError ? "resume_identity_mismatch" : resumeFailure instanceof SandboxResumeIdentityUnavailableError ? "resume_identity_unavailable" : resumeFailure instanceof SandboxExactResumeReplacedError ? "exact_resume_replaced" : "exact_resume_instance_unavailable",
3663
+ expected: false,
3664
+ retryable: false
3665
+ };
3666
+ }
3667
+ const providerFailure = classifyProviderSandboxFailure(backend, error);
3668
+ if (providerFailure.kind === "transient_transport") {
3669
+ return {
3670
+ category: "provider_transport",
3671
+ stage: wrappedStage ?? "provider_transport",
3672
+ code: providerFailure.diagnostic,
3673
+ expected: false,
3674
+ // A typed transport fault is attributable, but a create outcome may be
3675
+ // ambiguous. Classification never licenses replay on its own.
3676
+ retryable: false
3677
+ };
3678
+ }
3679
+ if (wrappedStage) {
3680
+ return {
3681
+ category: wrappedStage,
3682
+ stage: wrappedStage,
3683
+ code: `${wrappedStage}_failed`,
3684
+ expected: false,
3685
+ retryable: false
3686
+ };
3687
+ }
3688
+ return {
3689
+ category: "unknown",
3690
+ stage: "unknown",
3691
+ code: "unknown",
3692
+ expected: false,
3693
+ retryable: false
3694
+ };
3695
+ }
3696
+ function isLazySandboxProvisionRetryable(error) {
3697
+ const chain = sandboxProvisionErrorChain(error);
3698
+ if (chain.some(
3699
+ (value) => value instanceof SandboxImageConflictError || value instanceof SandboxRigConflictError || value instanceof SandboxLeaseRecoveryBlockedError2
3700
+ )) {
3701
+ return false;
3702
+ }
3703
+ const archive = firstProvisionError(chain, WorkspaceArchiveIntegrityError2);
3704
+ if (archive) {
3705
+ return archive.retryable;
3706
+ }
3707
+ if (chain.some(
3708
+ (value) => value instanceof SandboxLeaseSupersededError2 || value instanceof SandboxLeaseTransitionError2
3709
+ )) {
3710
+ return true;
3711
+ }
3712
+ if (chain.some((value) => value instanceof SandboxWarmingTimeoutError)) {
3713
+ return false;
3714
+ }
3715
+ return false;
3716
+ }
3717
+ function sandboxDeadlineRotationRecoveryDelayMs(settings) {
3718
+ return Math.min(5e3, settings.sandboxLeaseReaperPeriodMs);
3719
+ }
3720
+ function createTurnSandboxProvisioner(establish, options = {}) {
3721
+ const maxRetries = options.maxRetries ?? 2;
3722
+ const backoffMs = options.backoffMs ?? 250;
3723
+ let memo = null;
3724
+ const run = async (provisionId, onAttempt) => {
3725
+ let internalAttempts = 0;
3726
+ while (true) {
3727
+ internalAttempts += 1;
3728
+ onAttempt(internalAttempts);
3729
+ const startedAt = performance.now();
3730
+ try {
3731
+ throwIfTurnOperationCancelled(options.signal);
3732
+ const operation = establish();
3733
+ const result = await waitForTurnOperation(operation, options.signal, options.disposeResult);
3734
+ await options.onAttemptSettled?.({
3735
+ provisionId,
3736
+ attempt: internalAttempts,
3737
+ outcome: "completed",
3738
+ durationMs: performance.now() - startedAt,
3739
+ result
3740
+ });
3741
+ return { result, internalAttempts };
3742
+ } catch (error) {
3743
+ const retrying = !(error instanceof TurnOperationCancelledError || internalAttempts > maxRetries || !isLazySandboxProvisionRetryable(error));
3744
+ await options.onAttemptSettled?.({
3745
+ provisionId,
3746
+ attempt: internalAttempts,
3747
+ outcome: retrying ? "retrying" : "failed",
3748
+ durationMs: performance.now() - startedAt,
3749
+ error
3750
+ });
3751
+ if (!retrying) {
3752
+ throw error;
3753
+ }
3754
+ await sleep2(backoffMs * internalAttempts);
3755
+ }
3756
+ }
3757
+ };
3758
+ return {
3759
+ get() {
3760
+ if (!memo) {
3761
+ const provisionId = options.provisionIdFactory?.() ?? randomUUID3();
3762
+ memo = (async () => {
3763
+ const startedAt = performance.now();
3764
+ throwIfTurnOperationCancelled(options.signal);
3765
+ await options.onStarted?.({ provisionId });
3766
+ throwIfTurnOperationCancelled(options.signal);
3767
+ let result;
3768
+ let internalAttempts = 0;
3769
+ let hasResult = false;
3770
+ try {
3771
+ const runResult = await run(provisionId, (attempt) => {
3772
+ internalAttempts = attempt;
3773
+ });
3774
+ result = runResult.result;
3775
+ internalAttempts = runResult.internalAttempts;
3776
+ hasResult = true;
3777
+ throwIfTurnOperationCancelled(options.signal);
3778
+ await options.beforeCompleted?.(result, {
3779
+ provisionId,
3780
+ internalAttempts,
3781
+ durationMs: performance.now() - startedAt
3782
+ });
3783
+ throwIfTurnOperationCancelled(options.signal);
3784
+ await options.onCompleted?.(result, {
3785
+ provisionId,
3786
+ internalAttempts,
3787
+ durationMs: performance.now() - startedAt
3788
+ });
3789
+ return result;
3790
+ } catch (error) {
3791
+ if (hasResult) {
3792
+ await options.disposeResult?.(result);
3793
+ }
3794
+ if (!(error instanceof TurnOperationCancelledError)) {
3795
+ await options.onFailed?.(error, {
3796
+ provisionId,
3797
+ internalAttempts: Math.max(1, internalAttempts),
3798
+ durationMs: performance.now() - startedAt
3799
+ });
3800
+ }
3801
+ throw error;
3802
+ }
3803
+ })().catch((error) => {
3804
+ if (isLazySandboxProvisionRetryable(error)) {
3805
+ memo = null;
3806
+ }
3807
+ throw error;
3808
+ });
3809
+ }
3810
+ return memo;
3811
+ },
3812
+ hasStarted() {
3813
+ return memo !== null;
3814
+ },
3815
+ async waitForSettled(timeoutMs) {
3816
+ if (!memo) {
3817
+ return null;
3818
+ }
3819
+ let timer;
3820
+ try {
3821
+ return await Promise.race([
3822
+ memo.catch(() => null),
3823
+ new Promise((resolve) => {
3824
+ timer = setTimeout(() => resolve(null), timeoutMs);
3825
+ })
3826
+ ]);
3827
+ } finally {
3828
+ if (timer) {
3829
+ clearTimeout(timer);
3830
+ }
3831
+ }
3832
+ }
3833
+ };
3834
+ }
3835
+
3836
+ // src/activities/agent-turn/quiescence.ts
3837
+ function shouldRunTurnEndWorkspacePersistence(input) {
3838
+ return input.activityStatus !== "cancelled" && !input.cancellationRequested;
3839
+ }
3840
+ function shouldStartPeriodicWorkspaceSnapshot(input) {
3841
+ return input.firstProviderRequestStarted && !input.snapshotInFlight && !input.turnEndCaptureInProgress;
3842
+ }
3843
+ function assertPhysicalToolQuiescenceForCancellation(input) {
3844
+ if (!input.acknowledgeQuiescence || input.physicalToolQuiescenceConfirmed) return;
3845
+ if (input.failure instanceof Error) throw input.failure;
3846
+ throw new Error("Physical sandbox-tool quiescence could not be confirmed", {
3847
+ cause: input.failure
3848
+ });
3849
+ }
3850
+ function assertSessionAttemptQuiescenceRecoveryDurable(input) {
3851
+ if (!input.acknowledgeQuiescence || !input.physicalToolQuiescenceConfirmed || input.receiptOrProofDurable) {
3852
+ return;
3853
+ }
3854
+ if (input.failure instanceof Error) throw input.failure;
3855
+ throw new Error("Physical quiescence had no durable receipt or recovery proof", {
3856
+ cause: input.failure
3857
+ });
3858
+ }
3859
+ var QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS = 250;
3860
+ var QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS = 5e3;
3861
+ async function persistOrSignalSessionAttemptQuiescence(input) {
3862
+ let receipt;
3863
+ try {
3864
+ receipt = await input.persistReceipt();
3865
+ } catch (receiptError) {
3866
+ input.onReceiptFailure?.(receiptError);
3867
+ if (!input.signalProof) {
3868
+ throw new Error("Session-attempt quiescence proof signaler is unavailable", {
3869
+ cause: receiptError
3870
+ });
3871
+ }
3872
+ const delay = input.sleep ?? sleep2;
3873
+ let retryMs = QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS;
3874
+ let attempt = 1;
3875
+ for (; ; ) {
3876
+ try {
3877
+ await input.signalProof(input.proof);
3878
+ return "signal";
3879
+ } catch (signalError) {
3880
+ input.onSignalFailure?.(signalError, attempt, retryMs);
3881
+ try {
3882
+ input.heartbeat?.(attempt, retryMs);
3883
+ } catch {
3884
+ }
3885
+ await delay(retryMs);
3886
+ retryMs = Math.min(retryMs * 2, QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS);
3887
+ attempt += 1;
3888
+ }
3889
+ }
3890
+ }
3891
+ if (receipt.workflowWake && input.deliverWorkflowWake) {
3892
+ try {
3893
+ await input.deliverWorkflowWake(receipt.workflowWake);
3894
+ } catch (wakeError) {
3895
+ input.onWakeFailure?.(wakeError);
3896
+ }
3897
+ }
3898
+ try {
3899
+ await input.publishEvents(receipt.events);
3900
+ } catch (publishError) {
3901
+ input.onPublishFailure?.(publishError);
3902
+ }
3903
+ return "receipt";
3904
+ }
3905
+ async function drainAttemptOwnedSandboxWriters(input) {
3906
+ if (input.toolCancellationFence) {
3907
+ input.toolCancellationFence.cancel(
3908
+ input.cancellationReason ?? new Error("TURN_ATTEMPT_FENCED")
3909
+ );
3910
+ await input.toolCancellationFence.waitForQuiescence();
3911
+ }
3912
+ await Promise.all(input.gitCredentialRenewals.map(async (renewal) => await renewal.stop()));
3913
+ await input.codemodeTokenRenewal?.stop();
3914
+ await input.runCredentialRenewal?.stop();
3915
+ }
3916
+ async function releaseTurnSandboxAfterWriterDrain(sandbox, options = {}) {
3917
+ const maxAttempts = Math.max(1, options.maxAttempts ?? 3);
3918
+ const retryDelayMs = Math.max(0, options.retryDelayMs ?? 50);
3919
+ const wait = options.wait ?? sleep2;
3920
+ let lastError;
3921
+ for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
3922
+ try {
3923
+ await sandbox.release({ workspaceWritersQuiesced: true });
3924
+ return;
3925
+ } catch (error) {
3926
+ lastError = error;
3927
+ if (attempt < maxAttempts) {
3928
+ await wait(retryDelayMs * attempt);
3929
+ }
3930
+ }
3931
+ }
3932
+ throw lastError;
3933
+ }
3934
+ async function waitForTurnFinalizerStep(operation, signal) {
3935
+ if (!signal) return await operation;
3936
+ if (signal.aborted) {
3937
+ void operation.catch(() => void 0);
3938
+ return void 0;
3939
+ }
3940
+ let resolveCancellation;
3941
+ const cancelled = new Promise((resolve) => {
3942
+ resolveCancellation = () => resolve(void 0);
3943
+ });
3944
+ const cancel = () => {
3945
+ void operation.catch(() => void 0);
3946
+ resolveCancellation?.();
3947
+ };
3948
+ signal.addEventListener("abort", cancel, { once: true });
3949
+ try {
3950
+ return await Promise.race([operation, cancelled]);
3951
+ } finally {
3952
+ signal.removeEventListener("abort", cancel);
3953
+ }
3954
+ }
3955
+ async function waitForTurnStreamCleanup(batcherFlush, providerCompleted, signal) {
3956
+ await waitForTurnFinalizerStep(batcherFlush, signal);
3957
+ await waitForTurnFinalizerStep(providerCompleted, signal);
3958
+ }
3959
+ async function clearAttemptCredentialsWithSettledFence(input) {
3960
+ try {
3961
+ await input.runWorkspaceFencedClear();
3962
+ } catch (error) {
3963
+ const settledAttemptFence = error instanceof Error && error.name === "SandboxWorkspaceMutationFencedError" && error.code === "attempt_fenced" && (input.activityStatus === "idle" || input.activityStatus === "failed" || input.activityStatus === "cancelled");
3964
+ if (!settledAttemptFence) throw error;
3965
+ input.onSettledAttemptFence();
3966
+ await input.clearExactAttempt();
3967
+ }
3968
+ }
3969
+ function turnFinalizerCancellationSignal(temporalSignal, activityStatus) {
3970
+ if (activityStatus !== "cancelled" || temporalSignal?.aborted) return temporalSignal;
3971
+ const fenced = new AbortController();
3972
+ fenced.abort(new Error("TURN_ATTEMPT_FENCED"));
3973
+ return fenced.signal;
3974
+ }
3975
+ var MandatoryHistoryPersistenceError = class extends Error {
3976
+ constructor(stage, cause) {
3977
+ super(cause instanceof Error ? cause.message : String(cause));
3978
+ this.stage = stage;
3979
+ this.cause = cause;
3980
+ }
3981
+ name = "MandatoryHistoryPersistenceError";
3982
+ cause;
3983
+ };
3984
+ async function runMandatoryHistoryPersistenceStep(stage, operation) {
3985
+ try {
3986
+ return await operation();
3987
+ } catch (cause) {
3988
+ throw new MandatoryHistoryPersistenceError(stage, cause);
3989
+ }
3990
+ }
3991
+ async function finalizeDurableTurnOpStreams(sessions, fallback) {
3992
+ const candidates = /* @__PURE__ */ new Set();
3993
+ for (const session of sessions) {
3994
+ const candidate = session;
3995
+ if (typeof candidate?.finalizeOpStreamOps === "function") {
3996
+ candidates.add(candidate);
3997
+ }
3998
+ }
3999
+ if (candidates.size === 0 && fallback) {
4000
+ candidates.add(fallback);
4001
+ }
4002
+ for (const candidate of candidates) {
4003
+ try {
4004
+ await candidate.finalizeOpStreamOps();
4005
+ } catch {
4006
+ }
4007
+ }
4008
+ }
4009
+
4010
+ // src/activities/agent-turn/errors.ts
4011
+ import {
4012
+ ActiveSessionHistoryLimitExceededError,
4013
+ ApprovalRunStateLimitExceededError,
4014
+ isRetryableDatabaseTransportFailure,
4015
+ isSessionEventPersistenceError
4016
+ } from "@opengeni/db";
4017
+ import {
4018
+ ActiveBackendUnresolvableError,
4019
+ CompactionProviderResponseError,
4020
+ EmptyCompactionSummaryError,
4021
+ isMcpRequestTimeoutError,
4022
+ isMcpTransportConnectivityError,
4023
+ RoutingWorkspaceRootChangedError,
4024
+ SelfhostedWorkspaceRootChangedError,
4025
+ UNKNOWN_MODEL_FINISH_REASON_CODE
4026
+ } from "@opengeni/runtime";
4027
+ import {
4028
+ mcpTransportRequestFailureDiagnostic
4029
+ } from "@opengeni/runtime/mcp-network";
4030
+ import { ApplicationFailure, CancelledFailure as CancelledFailure2 } from "@temporalio/activity";
4031
+ import {
4032
+ CodexReloginRequired,
4033
+ classifyCodexResponseTimeoutError,
4034
+ classifyCodexUsageLimitError,
4035
+ isCodexTransportError
4036
+ } from "@opengeni/codex";
4037
+ import {
4038
+ classifyXaiSubscriptionStreamingTerminalError,
4039
+ classifyXaiSubscriptionStreamIdleTimeoutError,
4040
+ isXaiSubscriptionHostedToolContinuationError,
4041
+ isXaiSubscriptionRateLimitDiagnostic,
4042
+ isXaiSubscriptionTransportError,
4043
+ XaiSubscriptionReloginRequired
4044
+ } from "@opengeni/xai-subscription";
4045
+
4046
+ // src/activities/types.ts
4047
+ var PRE_CLAIM_FAILURE_TYPE = "OpenGeniPreClaimFailure";
4048
+ var PRE_CLAIM_FAILURE_MESSAGE = "Agent turn admission failed before attempt claim.";
4049
+ var ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE = "EscapedMcpTimeoutRecoveryFailure";
4050
+ var ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE = "MCP request timeout recovery checkpoint failed before model request";
4051
+
4052
+ // src/activities/agent-turn/errors.ts
4053
+ var PROVIDER_BACKPRESSURE_DELAY_MS = 6e4;
4054
+ var PROVIDER_CONNECTIVITY_BACKOFF_MS = [2e3, 5e3, 15e3, 3e4, 6e4];
4055
+ var MAX_AUTOMATIC_PROVIDER_RECOVERIES = PROVIDER_CONNECTIVITY_BACKOFF_MS.length;
4056
+ function providerRecoveryResult(input) {
4057
+ if (input.attemptNumber > MAX_AUTOMATIC_PROVIDER_RECOVERIES) {
4058
+ return {
4059
+ status: "exhausted",
4060
+ providerRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES,
4061
+ maxProviderRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES
4062
+ };
4063
+ }
4064
+ const providerDelay = input.retryAfterMs !== null && input.retryAfterMs !== void 0 && Number.isFinite(input.retryAfterMs) && input.retryAfterMs > 0 ? Math.ceil(input.retryAfterMs) : null;
4065
+ const continueDelayMs = input.failureCode === "provider_rate_limited" ? providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS : input.failureCode === "provider_unavailable" || input.failureCode === "upstream_connectivity_unavailable" || input.failureCode === "mcp_transport_timeout" || input.failureCode === "mcp_transport_unavailable" ? Math.max(
4066
+ providerDelay ?? 0,
4067
+ PROVIDER_CONNECTIVITY_BACKOFF_MS[Math.min(
4068
+ Math.max(Math.trunc(input.attemptNumber) - 1, 0),
4069
+ PROVIDER_CONNECTIVITY_BACKOFF_MS.length - 1
4070
+ )]
4071
+ ) : PROVIDER_BACKPRESSURE_DELAY_MS;
4072
+ return {
4073
+ status: "recovering",
4074
+ continueDelayMs
4075
+ };
4076
+ }
4077
+ function providerRecoveryExhaustedFailure(failure, recovery) {
4078
+ return {
4079
+ ...failure,
4080
+ error: `Automatic same-turn recovery stopped after ${recovery.providerRecoveryCount} retries because the upstream dependency remained unavailable. Send a new message to retry after the dependency recovers.`,
4081
+ retryable: false,
4082
+ recoveryExhausted: true,
4083
+ providerRecoveryCount: recovery.providerRecoveryCount,
4084
+ maxProviderRecoveryCount: recovery.maxProviderRecoveryCount,
4085
+ lastRetryableError: failure.error
4086
+ };
4087
+ }
4088
+ function providerRecoveryCountFromMetadata(metadata) {
4089
+ const value = metadata.providerRecoveryCount;
4090
+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
4091
+ }
4092
+ function headerValue(headers, name) {
4093
+ if (!headers || typeof headers !== "object") return null;
4094
+ const getter = headers.get;
4095
+ if (typeof getter === "function") {
4096
+ const value = getter.call(headers, name);
4097
+ return typeof value === "string" ? value : null;
4098
+ }
4099
+ const target = name.toLowerCase();
4100
+ const entry = Object.entries(headers).find(
4101
+ ([key, value]) => key.toLowerCase() === target && typeof value === "string"
4102
+ );
4103
+ return typeof entry?.[1] === "string" ? entry[1] : null;
4104
+ }
4105
+ function providerRetryAfterMs(error, nowMs = Date.now()) {
4106
+ let current = error;
4107
+ for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
4108
+ const value = current;
4109
+ const body = value.error && typeof value.error === "object" ? value.error : null;
4110
+ const directSeconds = Number(
4111
+ value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds
4112
+ );
4113
+ const header = headerValue(value.headers, "retry-after") ?? headerValue(value.responseHeaders, "retry-after") ?? headerValue(body?.headers, "retry-after");
4114
+ const headerSeconds = header === null ? Number.NaN : Number(header);
4115
+ const headerDate = header !== null && !Number.isFinite(headerSeconds) ? Date.parse(header) : Number.NaN;
4116
+ const seconds = Number.isFinite(directSeconds) ? directSeconds : Number.isFinite(headerSeconds) ? headerSeconds : Number.isFinite(headerDate) ? Math.max(0, (headerDate - nowMs) / 1e3) : Number.NaN;
4117
+ if (Number.isFinite(seconds) && seconds > 0) return Math.ceil(seconds * 1e3);
4118
+ current = value.cause;
4119
+ }
4120
+ return null;
4121
+ }
4122
+ function unavailableMcpOperationalContext(input) {
4123
+ if (input.droppedCount <= 0) {
4124
+ return void 0;
4125
+ }
4126
+ const omittedCount = Math.max(0, input.droppedCount - input.droppedIds.length);
4127
+ const listed = input.droppedIds.map((id) => `"${id}"`).join(", ");
4128
+ const inventory = listed ? `${listed}${omittedCount > 0 ? `, plus ${omittedCount} additional unavailable server(s)` : ""}` : `${input.droppedCount} unavailable server(s)`;
4129
+ return `MCP capability availability for this turn: the following session-selected server(s) are disconnected or no longer registered and were skipped: ${inventory}. Do not claim to have read or updated those systems. If the task depends on one as a source of truth, explain the limitation and ask the user to reconnect it or select another authoritative source; continue with unaffected work only when safe.`;
4130
+ }
4131
+ function escapedMcpTimeoutRecoveryFailure(input) {
4132
+ if (input.failureCode !== "mcp_transport_timeout" || input.modelRequestStarted || !Number.isSafeInteger(input.detail.executionGeneration) || input.detail.executionGeneration <= 1 || !Number.isSafeInteger(input.detail.providerRecoveryCount) || input.detail.providerRecoveryCount <= 0 || input.detail.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES || !Number.isSafeInteger(input.detail.continueDelayMs) || input.detail.continueDelayMs <= 0) {
4133
+ return null;
4134
+ }
4135
+ return ApplicationFailure.create({
4136
+ message: ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
4137
+ type: ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
4138
+ nonRetryable: true,
4139
+ details: [input.detail]
4140
+ });
4141
+ }
4142
+ function preClaimAdmissionFailure(error) {
4143
+ const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
4144
+ const sqlState = persistenceFailure?.details.sqlState ?? null;
4145
+ const rawTransportFailure = sqlState === null && isRetryableDatabaseTransportFailure(error);
4146
+ const retryable = Boolean(
4147
+ rawTransportFailure || persistenceFailure && (sqlState === null || sqlState.startsWith("08") || sqlState.startsWith("40") || sqlState.startsWith("53") || sqlState === "55P03" || sqlState === "57014" || sqlState === "57P01" || sqlState === "57P02" || sqlState === "57P03" || sqlState.startsWith("58"))
4148
+ );
4149
+ const detail = {
4150
+ disposition: retryable ? "retryable" : "permanent",
4151
+ code: persistenceFailure?.details.code ?? (rawTransportFailure ? "db_failure" : "claim_invariant")
4152
+ };
4153
+ return ApplicationFailure.create({
4154
+ message: PRE_CLAIM_FAILURE_MESSAGE,
4155
+ type: PRE_CLAIM_FAILURE_TYPE,
4156
+ nonRetryable: true,
4157
+ details: [detail]
4158
+ });
4159
+ }
4160
+ function isWorkerShutdownCancellation(error) {
4161
+ return error instanceof CancelledFailure2 && error.message === "WORKER_SHUTDOWN";
4162
+ }
4163
+ function sandboxRouteTransitionCode(error) {
4164
+ const pending = [error];
4165
+ const seen = /* @__PURE__ */ new WeakSet();
4166
+ let inspected = 0;
4167
+ while (pending.length > 0 && inspected < 64) {
4168
+ const current = pending.shift();
4169
+ inspected += 1;
4170
+ if (!current || typeof current !== "object" || seen.has(current)) continue;
4171
+ seen.add(current);
4172
+ try {
4173
+ const record = current;
4174
+ if ((current instanceof ActiveBackendUnresolvableError || record.name === "ActiveBackendUnresolvableError") && record.code === "home_unavailable_this_turn") {
4175
+ return "home_unavailable_this_turn";
4176
+ }
4177
+ if (current instanceof RoutingWorkspaceRootChangedError || current instanceof SelfhostedWorkspaceRootChangedError || record.name === "RoutingWorkspaceRootChangedError" || record.name === "SelfhostedWorkspaceRootChangedError") {
4178
+ return "workspace_root_changed_this_turn";
4179
+ }
4180
+ for (const key of ["cause", "error"]) {
4181
+ const nested = record[key];
4182
+ if (nested && typeof nested === "object") pending.push(nested);
4183
+ }
4184
+ if (Array.isArray(record.errors)) {
4185
+ pending.push(...record.errors.slice(0, 32));
4186
+ }
4187
+ } catch {
4188
+ }
4189
+ }
4190
+ return null;
4191
+ }
4192
+ function compactionFailureReason(reason) {
4193
+ return reason.startsWith("compaction summarization failed:") ? reason : `compaction summarization failed: ${reason}`;
4194
+ }
4195
+ function safeErrorDiagnostic(error) {
4196
+ const diagnostic = {
4197
+ errorClass: "WorkerOperationError",
4198
+ errorCode: "worker_operation_failed",
4199
+ origin: "worker"
4200
+ };
4201
+ try {
4202
+ let statusSource = error;
4203
+ if (error instanceof MandatoryHistoryPersistenceError) {
4204
+ diagnostic.historyPersistenceStage = error.stage;
4205
+ statusSource = error.cause;
4206
+ }
4207
+ if (statusSource && typeof statusSource === "object") {
4208
+ const status = Number(
4209
+ statusSource.status ?? statusSource.statusCode
4210
+ );
4211
+ if (Number.isInteger(status) && status >= 100 && status <= 599) {
4212
+ diagnostic.status = status;
4213
+ }
4214
+ }
4215
+ } catch {
4216
+ }
4217
+ return diagnostic;
4218
+ }
4219
+ function safeErrorForTelemetry(error) {
4220
+ const diagnostic = safeErrorDiagnostic(error);
4221
+ const safe = new Error("worker operation failed");
4222
+ safe.name = "WorkerOperationError";
4223
+ safe.code = diagnostic.errorCode;
4224
+ if (diagnostic.status !== void 0) safe.status = diagnostic.status;
4225
+ safe.origin = diagnostic.origin;
4226
+ return safe;
4227
+ }
4228
+ function compactionFailureReasonFromError(error) {
4229
+ if (error instanceof CompactionProviderResponseError || error instanceof EmptyCompactionSummaryError) {
4230
+ return compactionFailureReason(error.message);
4231
+ }
4232
+ const errorName = error instanceof Error && error.name ? error.name : "unknown error";
4233
+ return compactionFailureReason(`unexpected ${errorName}`);
4234
+ }
4235
+ function isCompactionSummaryFailure(error) {
4236
+ return error instanceof CompactionProviderResponseError || error instanceof EmptyCompactionSummaryError;
4237
+ }
4238
+ function shouldRecoverCompactionProviderFailure(error) {
4239
+ if (!(error instanceof CompactionProviderResponseError)) return false;
4240
+ if (isCodexTransportError(error) && classifyCodexUsageLimitError(error)) return true;
4241
+ return agentRunFailurePayload(error).retryable === true;
4242
+ }
4243
+ function classifyContextWindowOverflowError(error) {
4244
+ const fields = collectErrorStrings(error);
4245
+ const matched = fields.find(
4246
+ (value) => /context[_\s-]*length[_\s-]*exceeded/i.test(value) || /exceeds?\s+(?:the\s+)?context\s+window/i.test(value) || /maximum\s+context\s+length/i.test(value) || /context\s+window[^.]*exceed/i.test(value)
4247
+ );
4248
+ if (!matched) {
4249
+ return null;
4250
+ }
4251
+ const message = error instanceof Error ? error.message : String(error);
4252
+ const code = fields.find((value) => /context[_\s-]*length[_\s-]*exceeded/i.test(value));
4253
+ return {
4254
+ message,
4255
+ ...code ? { code } : {},
4256
+ ...matched && matched !== message ? { detail: matched } : {}
4257
+ };
4258
+ }
4259
+ function classifyMcpTransportTimeoutError(error) {
4260
+ const fields = collectErrorStrings(error);
4261
+ const matchedText = fields.find(
4262
+ (value) => /\bmcp\b/i.test(value) && /(?:request\s+timed\s+out|request\s+timeout|\btimed\s+out\b|\btimeout\b|ETIMEDOUT)/i.test(
4263
+ value
4264
+ ) && !/authentication\s+required/i.test(value)
4265
+ );
4266
+ const sanitizedSdkTimeout = isMcpRequestTimeoutError(error);
4267
+ if (!matchedText && !sanitizedSdkTimeout) {
4268
+ return null;
4269
+ }
4270
+ const message = error instanceof Error ? error.message : String(error);
4271
+ const matched = matchedText ?? fields.find((value) => /\bmcp\b/i.test(value));
4272
+ return {
4273
+ message,
4274
+ ...matched && matched !== message ? { detail: matched } : {}
4275
+ };
4276
+ }
4277
+ function collectErrorStrings(value, seen = /* @__PURE__ */ new WeakSet()) {
4278
+ if (typeof value === "string") {
4279
+ return [value];
4280
+ }
4281
+ if (!value || typeof value !== "object") {
4282
+ return [];
4283
+ }
4284
+ if (seen.has(value)) {
4285
+ return [];
4286
+ }
4287
+ seen.add(value);
4288
+ const out = [];
4289
+ const record = value;
4290
+ for (const key of ["message", "code", "type", "name", "param"]) {
4291
+ const field = record[key];
4292
+ if (typeof field === "string" && field.length > 0) {
4293
+ out.push(field);
4294
+ }
4295
+ }
4296
+ for (const key of ["error", "cause", "response", "data"]) {
4297
+ out.push(...collectErrorStrings(record[key], seen));
4298
+ }
4299
+ return out;
4300
+ }
4301
+ var STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE = "unable to connect. is the computer able to access the url?";
4302
+ function isExactStatuslessUpstreamConnectivityMessage(message) {
4303
+ return message.trim().toLowerCase() === STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE;
4304
+ }
4305
+ function isTransientProviderError(error) {
4306
+ const status = typeof error === "object" && error !== null ? Number(
4307
+ error.status ?? error.statusCode
4308
+ ) : void 0;
4309
+ if (status !== void 0 && Number.isFinite(status)) {
4310
+ return status >= 500 && status < 600;
4311
+ }
4312
+ const code = typeof error === "object" && error !== null && "code" in error ? String(error.code) : void 0;
4313
+ if (code && /^(?:ECONNRESET|ETIMEDOUT|EAI_AGAIN|ECONNREFUSED|EPIPE)$/i.test(code)) {
4314
+ return true;
4315
+ }
4316
+ const message = error instanceof Error ? error.message : String(error);
4317
+ if (isExactStatuslessUpstreamConnectivityMessage(message)) {
4318
+ return true;
4319
+ }
4320
+ return /overloaded|an error occurred while processing your request|connection error|service unavailable|bad gateway|gateway timeout/i.test(
4321
+ message
4322
+ );
4323
+ }
4324
+ function classifyXaiCredentialFailure(error) {
4325
+ let relogin = error;
4326
+ for (let depth = 0; depth < 6 && relogin && typeof relogin === "object"; depth += 1) {
4327
+ if (relogin instanceof XaiSubscriptionReloginRequired) {
4328
+ return { kind: "auth", cooldownMs: null };
4329
+ }
4330
+ relogin = relogin.cause;
4331
+ }
4332
+ if (!isXaiSubscriptionTransportError(error)) return null;
4333
+ let current = error;
4334
+ for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
4335
+ const value = current;
4336
+ const body = value.error && typeof value.error === "object" ? value.error : null;
4337
+ const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
4338
+ const code = String(value.code ?? body?.code ?? "").toLowerCase();
4339
+ if (status === 401 || code === "unauthorized" || code === "invalid_token") {
4340
+ return { kind: "auth", cooldownMs: null };
4341
+ }
4342
+ if (status === 403) {
4343
+ return { kind: "forbidden", cooldownMs: null };
4344
+ }
4345
+ const message = String(value.message ?? body?.message ?? "");
4346
+ if (isXaiSubscriptionRateLimitDiagnostic({
4347
+ code,
4348
+ message,
4349
+ status: Number.isInteger(status) ? status : null
4350
+ })) {
4351
+ return {
4352
+ kind: "rate_limit",
4353
+ cooldownMs: providerRetryAfterMs(error) ?? PROVIDER_BACKPRESSURE_DELAY_MS
4354
+ };
4355
+ }
4356
+ current = value.cause;
4357
+ }
4358
+ return null;
4359
+ }
4360
+ function agentRunFailurePayload(error, options = {}) {
4361
+ if (error instanceof MandatoryHistoryPersistenceError) {
4362
+ const underlying = isSessionEventPersistenceError(error.cause) ? agentRunFailurePayload(error.cause, options) : {
4363
+ error: error.cause instanceof Error ? error.cause.message : String(error.cause)
4364
+ };
4365
+ return {
4366
+ ...underlying,
4367
+ historyPersistenceStage: error.stage
4368
+ };
4369
+ }
4370
+ const message = error instanceof Error ? error.message : String(error);
4371
+ const status = typeof error === "object" && error !== null ? Number(
4372
+ error.status ?? error.statusCode
4373
+ ) : void 0;
4374
+ const code = typeof error === "object" && error !== null && "code" in error ? String(error.code) : void 0;
4375
+ if (error instanceof ActiveSessionHistoryLimitExceededError) {
4376
+ return {
4377
+ error: "The session's active conversation history exceeds the worker's safe materialization envelope. Clear the session context before retrying; an oversized history cannot be compacted safely in a serving worker.",
4378
+ code: error.code,
4379
+ retryable: false,
4380
+ detail: error.message
4381
+ };
4382
+ }
4383
+ if (error instanceof ApprovalRunStateLimitExceededError) {
4384
+ return {
4385
+ error: "The saved approval state exceeds the worker's safe materialization envelope. Clear the pending approval context before retrying.",
4386
+ code: error.code,
4387
+ retryable: false,
4388
+ detail: error.message
4389
+ };
4390
+ }
4391
+ if (isCodexTransportError(error) && code === "invalid_sse_terminal") {
4392
+ return {
4393
+ error: "The Codex response stream ended without a terminal response",
4394
+ code: "invalid_sse_terminal",
4395
+ retryable: false
4396
+ };
4397
+ }
4398
+ if (isXaiSubscriptionHostedToolContinuationError(error)) {
4399
+ return {
4400
+ error: "SuperGrok stopped responding after its hosted search completed. Partial output was preserved; automatic replay is disabled because the accepted response may still have provider-side effects.",
4401
+ code: "xai_hosted_tool_continuation_stalled",
4402
+ retryable: false
4403
+ };
4404
+ }
4405
+ const xaiStreamTerminal = classifyXaiSubscriptionStreamingTerminalError(error);
4406
+ if (xaiStreamTerminal) {
4407
+ return {
4408
+ error: xaiStreamTerminal.message,
4409
+ code: xaiStreamTerminal.code,
4410
+ retryable: xaiStreamTerminal.status === 429,
4411
+ lastEventType: xaiStreamTerminal.eventType,
4412
+ ...xaiStreamTerminal.requestId ? { requestId: xaiStreamTerminal.requestId } : {}
4413
+ };
4414
+ }
4415
+ const xaiStreamTimeout = classifyXaiSubscriptionStreamIdleTimeoutError(error);
4416
+ if (xaiStreamTimeout) {
4417
+ return {
4418
+ error: "SuperGrok stopped sending valid response events. Partial output was preserved; automatic replay is disabled because the accepted response may still have provider-side effects.",
4419
+ code: "xai_response_stream_idle_timeout",
4420
+ retryable: false,
4421
+ responseObserved: xaiStreamTimeout.responseObserved,
4422
+ eventCount: xaiStreamTimeout.eventCount,
4423
+ silenceDurationMs: xaiStreamTimeout.silenceDurationMs,
4424
+ ...xaiStreamTimeout.requestId ? { requestId: xaiStreamTimeout.requestId } : {},
4425
+ ...xaiStreamTimeout.lastEventType ? { lastEventType: xaiStreamTimeout.lastEventType } : {}
4426
+ };
4427
+ }
4428
+ if (isSessionEventPersistenceError(error)) {
4429
+ const { details } = error;
4430
+ return {
4431
+ error: error.message,
4432
+ code: details.code,
4433
+ detail: details.retryOutcome === "exhausted" ? `The idempotent persistence transaction failed after ${details.attempts} attempts.` : "The database rejected the idempotent persistence transaction.",
4434
+ correlationId: details.correlationId,
4435
+ stage: details.stage,
4436
+ sqlState: details.sqlState,
4437
+ attempts: details.attempts,
4438
+ retryOutcome: details.retryOutcome,
4439
+ ...Object.keys(details.database).length > 0 ? { database: details.database } : {}
4440
+ };
4441
+ }
4442
+ const usageLimit = classifyCodexUsageLimitError(error);
4443
+ if (usageLimit) {
4444
+ return codexUsageLimitFailurePayload(usageLimit, message);
4445
+ }
4446
+ const codexTimeout = classifyCodexResponseTimeoutError(error, {
4447
+ allowLegacyRequestTimeout: options.isCodexTurn === true
4448
+ });
4449
+ if (codexTimeout) {
4450
+ return {
4451
+ error: codexTimeout.responseObserved ? "The Codex response timed out after streaming began. Observed output was checkpointed; automatic replay is disabled because the upstream operation may still be active." : "The Codex response timed out before any response was observed. Upstream acceptance is unknown, so automatic replay is disabled.",
4452
+ code: "codex_response_timeout",
4453
+ retryable: false,
4454
+ timeoutClass: codexTimeout.timeoutClass,
4455
+ responseObserved: codexTimeout.responseObserved,
4456
+ ...codexTimeout.requestId ? { requestId: codexTimeout.requestId } : {},
4457
+ ...codexTimeout.message ? { detail: codexTimeout.message } : {}
4458
+ };
4459
+ }
4460
+ const mcpTimeout = classifyMcpTransportTimeoutError(error);
4461
+ if (mcpTimeout) {
4462
+ const mcpTransportDiagnostic = mcpTransportRequestFailureDiagnostic(error);
4463
+ return {
4464
+ error: "An MCP server request timed out. Any completed tool output was checkpointed; the session can continue safely.",
4465
+ code: "mcp_transport_timeout",
4466
+ retryable: true,
4467
+ ...mcpTimeout.detail || mcpTimeout.message ? { detail: mcpTimeout.detail ?? mcpTimeout.message } : {},
4468
+ ...mcpTransportDiagnostic ? { mcpTransportDiagnostic } : {}
4469
+ };
4470
+ }
4471
+ if (isMcpTransportConnectivityError(error)) {
4472
+ const mcpTransportDiagnostic = mcpTransportRequestFailureDiagnostic(error);
4473
+ return {
4474
+ error: "A required MCP server was temporarily unreachable. The same turn will retry after a short delay.",
4475
+ code: "mcp_transport_unavailable",
4476
+ retryable: true,
4477
+ detail: message,
4478
+ ...mcpTransportDiagnostic ? { mcpTransportDiagnostic } : {}
4479
+ };
4480
+ }
4481
+ if (code === UNKNOWN_MODEL_FINISH_REASON_CODE) {
4482
+ return {
4483
+ error: "The model provider ended its response ambiguously. Partial output was not accepted as complete; the same turn will retry from durable history.",
4484
+ code: UNKNOWN_MODEL_FINISH_REASON_CODE,
4485
+ retryable: true
4486
+ };
4487
+ }
4488
+ if (status === 429 || code === "rate_limit_exceeded" || /(?:too many requests|rate.?limit|\b429\b)/i.test(message)) {
4489
+ return {
4490
+ error: "Model provider rate limit hit. Try again in a minute or lower the reasoning effort.",
4491
+ code: "provider_rate_limited",
4492
+ retryable: true,
4493
+ ...message && message !== "Too Many Requests" ? { detail: message } : {}
4494
+ };
4495
+ }
4496
+ if (isTransientProviderError(error)) {
4497
+ if (isExactStatuslessUpstreamConnectivityMessage(message)) {
4498
+ return {
4499
+ error: "OpenGeni could not reach an upstream service. The same turn will retry after a short delay.",
4500
+ code: "upstream_connectivity_unavailable",
4501
+ retryable: true
4502
+ };
4503
+ }
4504
+ return { error: message, code: "provider_unavailable", retryable: true };
4505
+ }
4506
+ return { error: message };
4507
+ }
4508
+ var CODEX_ALLOWANCE_FALLBACK_MS = 5 * 60 * 6e4;
4509
+ function codexCredentialCooldownUntil(failure, account, now) {
4510
+ if (failure.kind === "auth" || failure.kind === "forbidden") {
4511
+ return null;
4512
+ }
4513
+ const providerReset = failure.cooldownSeconds !== null && Number.isFinite(failure.cooldownSeconds) && failure.cooldownSeconds > 0 ? new Date(now.getTime() + Math.ceil(failure.cooldownSeconds) * 1e3) : null;
4514
+ if (failure.kind === "rate_limit") {
4515
+ return providerReset ?? new Date(now.getTime() + PROVIDER_BACKPRESSURE_DELAY_MS);
4516
+ }
4517
+ const blockingResets = account ? [
4518
+ { used: account.primaryUsedPercent, reset: account.primaryResetAt },
4519
+ { used: account.secondaryUsedPercent, reset: account.secondaryResetAt }
4520
+ ].filter(
4521
+ (window) => (window.used ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT && window.reset instanceof Date && window.reset.getTime() > now.getTime()
4522
+ ).map((window) => window.reset) : [];
4523
+ const quotaResets = providerReset ? [...blockingResets, providerReset] : blockingResets;
4524
+ if (quotaResets.length === 0) {
4525
+ return new Date(now.getTime() + CODEX_ALLOWANCE_FALLBACK_MS);
4526
+ }
4527
+ return quotaResets.reduce(
4528
+ (latest, reset) => reset.getTime() > latest.getTime() ? reset : latest
4529
+ );
4530
+ }
4531
+ function classifyCodexCredentialFailure(error) {
4532
+ let refreshError = error;
4533
+ for (let depth = 0; depth < 6 && refreshError && typeof refreshError === "object"; depth += 1) {
4534
+ if (refreshError instanceof CodexReloginRequired) {
4535
+ return { kind: "auth", cooldownSeconds: null };
4536
+ }
4537
+ refreshError = refreshError.cause;
4538
+ }
4539
+ if (!isCodexTransportError(error)) {
4540
+ return null;
4541
+ }
4542
+ const usageLimit = classifyCodexUsageLimitError(error);
4543
+ if (usageLimit) {
4544
+ return { kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds };
4545
+ }
4546
+ let cur = error;
4547
+ for (let depth = 0; depth < 6 && cur && typeof cur === "object"; depth++) {
4548
+ const value = cur;
4549
+ const body = value.error && typeof value.error === "object" ? value.error : null;
4550
+ const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
4551
+ const code = String(value.code ?? body?.code ?? "").toLowerCase();
4552
+ const directRetryAfter = Number(
4553
+ value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds
4554
+ );
4555
+ const retryAfterHeader = headerValue(value.headers, "retry-after") ?? headerValue(value.responseHeaders, "retry-after") ?? headerValue(body?.headers, "retry-after");
4556
+ const retryAfterNumber = retryAfterHeader === null ? Number.NaN : Number(retryAfterHeader);
4557
+ const retryAfterDate = retryAfterHeader !== null && !Number.isFinite(retryAfterNumber) ? Date.parse(retryAfterHeader) : Number.NaN;
4558
+ const retryAfter = Number.isFinite(directRetryAfter) ? directRetryAfter : Number.isFinite(retryAfterNumber) ? retryAfterNumber : Number.isFinite(retryAfterDate) ? Math.max(0, (retryAfterDate - Date.now()) / 1e3) : Number.NaN;
4559
+ const cooldownSeconds = Number.isFinite(retryAfter) && retryAfter > 0 ? Math.ceil(retryAfter) : null;
4560
+ if (code === "insufficient_quota" || code === "quota_exceeded" || code === "billing_hard_limit_reached") {
4561
+ return { kind: "quota", cooldownSeconds };
4562
+ }
4563
+ if (status === 401 || code === "unauthorized" || code === "invalid_api_key") {
4564
+ return { kind: "auth", cooldownSeconds };
4565
+ }
4566
+ if (status === 403) {
4567
+ return { kind: "forbidden", cooldownSeconds };
4568
+ }
4569
+ if (status === 429 || code === "rate_limit_exceeded" || code === "too_many_requests") {
4570
+ return { kind: "rate_limit", cooldownSeconds };
4571
+ }
4572
+ cur = value.cause;
4573
+ }
4574
+ return null;
4575
+ }
4576
+ function humanizeResetWindow(resetsInSeconds) {
4577
+ if (resetsInSeconds === null || !Number.isFinite(resetsInSeconds) || resetsInSeconds <= 0) {
4578
+ return "shortly";
4579
+ }
4580
+ const total = Math.ceil(resetsInSeconds);
4581
+ if (total < 60) {
4582
+ return "in under a minute";
4583
+ }
4584
+ const hours = Math.floor(total / 3600);
4585
+ const minutes = Math.round(total % 3600 / 60);
4586
+ if (hours > 0) {
4587
+ return minutes > 0 ? `in about ${hours}h ${minutes}m` : `in about ${hours}h`;
4588
+ }
4589
+ return `in about ${minutes}m`;
4590
+ }
4591
+ function codexUsageLimitFailurePayload(info, detail, opts) {
4592
+ const error = opts?.allAccounts ? `All connected ChatGPT/Codex subscriptions are rate-limited. Access returns ${humanizeResetWindow(info.resetsInSeconds)}. You can switch this session to a different model in the meantime, or wait for a subscription to reset.` : `Your ChatGPT/Codex subscription usage limit has been reached. Access resets ${humanizeResetWindow(info.resetsInSeconds)}. You can switch this session to a different model in the meantime, or wait for the limit to reset.`;
4593
+ return {
4594
+ error,
4595
+ code: "codex_usage_limit_reached",
4596
+ retryable: false,
4597
+ ...detail ? { detail } : {}
4598
+ };
4599
+ }
4600
+ var CODEX_USAGE_LIMIT_MAX_RESUME_MS = 60 * 6e4;
4601
+
4602
+ export {
4603
+ createSharedActivityServices,
4604
+ classifyCodexPin,
4605
+ CODEX_USAGE_EXHAUSTED_PCT,
4606
+ isCodexCredentialEligible,
4607
+ shardCredentialForSession,
4608
+ earliestCodexReset,
4609
+ authoritativeCodexCapacityResetAt,
4610
+ computeIdleDelayMs,
4611
+ computeReactiveRotationResume,
4612
+ chooseRotationActive,
4613
+ selectCodexCredentialLeaseForTurn,
4614
+ refreshCodexUsageAndRepairCapacityWaiters,
4615
+ signalCodexCapacityWakeTargets,
4616
+ signalPendingCodexCapacityWakeTargets,
4617
+ createCodexCapacityActivities,
4618
+ createGoalActivities,
4619
+ withFirstPartyTools,
4620
+ rigProviderImageSourceImage,
4621
+ settingsWithPackSandboxImage,
4622
+ settingsWithRigImage,
4623
+ resolveWorkspacePackRuntime,
4624
+ resolveWorkspaceInstalledSkillRuntime,
4625
+ createRuntimeBatcher,
4626
+ currentActivityContext,
4627
+ startActivityHeartbeat,
4628
+ nextStreamEvent,
4629
+ notifyParentOfChildIdle,
4630
+ deliverFailedChildTurnToParent,
4631
+ deliverChildRequiresActionToParent,
4632
+ reconcilePendingParentSystemUpdates,
4633
+ reconcilePendingSessionWorkflowWakes,
4634
+ collectWorkspaceArchiveObjectKeys,
4635
+ putVersion1TarArchiveOrInline,
4636
+ deleteUnpublishedWorkspaceArchiveObject,
4637
+ deleteWorkspaceArchiveObjectKeys,
4638
+ assertReusableSessionRevivable,
4639
+ scheduledUserMessagePayload,
4640
+ workflowIdForSession,
4641
+ mergeResourceRefs,
4642
+ INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY,
4643
+ incidentTelemetryPreflightDeclaration,
4644
+ evaluateIncidentTelemetryPreflight,
4645
+ incidentTelemetryAuthorityFence,
4646
+ resolveIncidentTelemetryResponderMetadata,
4647
+ validateIncidentTelemetrySystemUpdateAuthority,
4648
+ sandboxLeaseHolderIdForAttempt,
4649
+ waitForWarmSnapshot,
4650
+ persistSandboxDeadlineRotationCheckpoint,
4651
+ maybePersistWarmWorkspaceSnapshot,
4652
+ resumeBoxForTurn,
4653
+ SandboxDeadlineRotationError,
4654
+ turnOperationCancellationFailure,
4655
+ throwIfTurnOperationCancelled,
4656
+ waitForTurnOperation,
4657
+ classifySandboxLogicalProvisionFailure,
4658
+ isLazySandboxProvisionRetryable,
4659
+ sandboxDeadlineRotationRecoveryDelayMs,
4660
+ createTurnSandboxProvisioner,
4661
+ shouldRunTurnEndWorkspacePersistence,
4662
+ shouldStartPeriodicWorkspaceSnapshot,
4663
+ assertPhysicalToolQuiescenceForCancellation,
4664
+ assertSessionAttemptQuiescenceRecoveryDurable,
4665
+ persistOrSignalSessionAttemptQuiescence,
4666
+ drainAttemptOwnedSandboxWriters,
4667
+ releaseTurnSandboxAfterWriterDrain,
4668
+ waitForTurnFinalizerStep,
4669
+ waitForTurnStreamCleanup,
4670
+ clearAttemptCredentialsWithSettledFence,
4671
+ turnFinalizerCancellationSignal,
4672
+ runMandatoryHistoryPersistenceStep,
4673
+ finalizeDurableTurnOpStreams,
4674
+ MAX_AUTOMATIC_PROVIDER_RECOVERIES,
4675
+ providerRecoveryResult,
4676
+ providerRecoveryExhaustedFailure,
4677
+ providerRecoveryCountFromMetadata,
4678
+ providerRetryAfterMs,
4679
+ unavailableMcpOperationalContext,
4680
+ escapedMcpTimeoutRecoveryFailure,
4681
+ preClaimAdmissionFailure,
4682
+ isWorkerShutdownCancellation,
4683
+ sandboxRouteTransitionCode,
4684
+ compactionFailureReason,
4685
+ safeErrorDiagnostic,
4686
+ safeErrorForTelemetry,
4687
+ compactionFailureReasonFromError,
4688
+ isCompactionSummaryFailure,
4689
+ shouldRecoverCompactionProviderFailure,
4690
+ classifyContextWindowOverflowError,
4691
+ classifyXaiCredentialFailure,
4692
+ agentRunFailurePayload,
4693
+ codexCredentialCooldownUntil,
4694
+ classifyCodexCredentialFailure,
4695
+ codexUsageLimitFailurePayload,
4696
+ CODEX_USAGE_LIMIT_MAX_RESUME_MS
4697
+ };
4698
+ //# sourceMappingURL=chunk-CNYGVRKF.js.map