@opengeni/worker-bundle 0.20.3 → 0.22.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +52 -0
  2. package/dist/activities/agent-turn/agent-build.d.ts +89 -0
  3. package/dist/activities/agent-turn/claim.d.ts +67 -0
  4. package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
  5. package/dist/activities/agent-turn/codex.d.ts +32 -0
  6. package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
  7. package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
  8. package/dist/activities/agent-turn/errors.d.ts +253 -0
  9. package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
  10. package/dist/activities/agent-turn/file-resources.d.ts +37 -0
  11. package/dist/activities/agent-turn/finalization.d.ts +42 -0
  12. package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
  13. package/dist/activities/agent-turn/governance-model.d.ts +66 -0
  14. package/dist/activities/agent-turn/history-sink.d.ts +52 -0
  15. package/dist/activities/agent-turn/history.d.ts +63 -0
  16. package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
  17. package/dist/activities/agent-turn/model-usage.d.ts +218 -0
  18. package/dist/activities/agent-turn/quiescence.d.ts +135 -0
  19. package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
  20. package/dist/activities/agent-turn/run.d.ts +3 -0
  21. package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
  22. package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
  23. package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
  24. package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
  25. package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
  26. package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
  27. package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
  28. package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
  29. package/dist/activities/agent-turn/turn-context.d.ts +144 -0
  30. package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
  31. package/dist/activities/agent-turn.d.ts +12 -954
  32. package/dist/activities/automations.d.ts +16 -0
  33. package/dist/activities/common.d.ts +1 -1
  34. package/dist/activities/connector-attachments.d.ts +13 -0
  35. package/dist/activities/documents.d.ts +1 -0
  36. package/dist/activities/environment.d.ts +26 -15
  37. package/dist/activities/generated-images.d.ts +1 -1
  38. package/dist/activities/git-credential-renewal.d.ts +3 -1
  39. package/dist/activities/goals.d.ts +4 -1
  40. package/dist/activities/google-drive-provider.d.ts +29 -0
  41. package/dist/activities/google-drive-publication.d.ts +17 -4
  42. package/dist/activities/incident-telemetry-authority.d.ts +52 -0
  43. package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
  44. package/dist/activities/knowledge-source-sync.d.ts +4 -0
  45. package/dist/activities/mcp-credentials.d.ts +9 -3
  46. package/dist/activities/open-suffix-resume.d.ts +77 -0
  47. package/dist/activities/packs.d.ts +3 -11
  48. package/dist/activities/parent-wake.d.ts +19 -0
  49. package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
  50. package/dist/activities/retained-screenshots.d.ts +1 -1
  51. package/dist/activities/rig-verification.d.ts +1 -1
  52. package/dist/activities/run-input.d.ts +9 -10
  53. package/dist/activities/sandbox-images.d.ts +1 -14
  54. package/dist/activities/sandbox-lease.d.ts +21 -3
  55. package/dist/activities/scheduled-tasks.d.ts +47 -0
  56. package/dist/activities/types.d.ts +23 -2
  57. package/dist/activities/video-media-validation.d.ts +1 -1
  58. package/dist/activities/video-output-retention.d.ts +1 -2
  59. package/dist/activities/xai-auth.d.ts +3 -0
  60. package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
  61. package/dist/activities-control-DKG22ONJ.js.map +1 -0
  62. package/dist/activities-control.d.ts +6 -2
  63. package/dist/activities-turn-4IMC3MAA.js +21326 -0
  64. package/dist/activities-turn-4IMC3MAA.js.map +1 -0
  65. package/dist/activities.d.ts +7 -3
  66. package/dist/artifact-materializer-entry.js +41 -22
  67. package/dist/artifact-materializer-entry.js.map +1 -1
  68. package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
  69. package/dist/chunk-CE7TPKBS.js.map +1 -0
  70. package/dist/chunk-CNYGVRKF.js +4698 -0
  71. package/dist/chunk-CNYGVRKF.js.map +1 -0
  72. package/dist/connection-provider-use.d.ts +18 -0
  73. package/dist/editable-artifact-materializer.d.ts +4 -3
  74. package/dist/github-rest-mcp.d.ts +26 -0
  75. package/dist/index.d.ts +10 -2
  76. package/dist/index.js +58 -14
  77. package/dist/index.js.map +1 -1
  78. package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
  79. package/dist/model-context-contributions.d.ts +8 -4
  80. package/dist/observability-metrics.d.ts +63 -4
  81. package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
  82. package/dist/personal-github-git-credentials.d.ts +9 -0
  83. package/dist/pr-review-credentials.d.ts +6 -0
  84. package/dist/sandbox-archive-storage.d.ts +45 -0
  85. package/dist/sandbox-resume.d.ts +65 -6
  86. package/dist/sandbox-routing.d.ts +38 -2
  87. package/dist/scheduled-alert-occurrence.d.ts +33 -0
  88. package/dist/workflow-bundle.js +87 -28
  89. package/dist/workflows/activities.d.ts +1 -0
  90. package/dist/workflows/automations.d.ts +6 -0
  91. package/dist/workflows/document-index.d.ts +1 -0
  92. package/dist/workflows.d.ts +1 -0
  93. package/package.json +17 -17
  94. package/src/activities/agent-turn/admission.ts +292 -0
  95. package/src/activities/agent-turn/agent-build.ts +802 -0
  96. package/src/activities/agent-turn/claim.ts +590 -0
  97. package/src/activities/agent-turn/codex-capacity.ts +720 -0
  98. package/src/activities/agent-turn/codex.ts +94 -0
  99. package/src/activities/agent-turn/compaction-prep.ts +722 -0
  100. package/src/activities/agent-turn/credential-leases.ts +240 -0
  101. package/src/activities/agent-turn/errors.ts +1068 -0
  102. package/src/activities/agent-turn/failure-settlement.ts +1436 -0
  103. package/src/activities/agent-turn/file-resources.ts +227 -0
  104. package/src/activities/agent-turn/finalization.ts +754 -0
  105. package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
  106. package/src/activities/agent-turn/governance-model.ts +465 -0
  107. package/src/activities/agent-turn/history-sink.ts +139 -0
  108. package/src/activities/agent-turn/history.ts +200 -0
  109. package/src/activities/agent-turn/media-artifacts.ts +359 -0
  110. package/src/activities/agent-turn/model-usage.ts +823 -0
  111. package/src/activities/agent-turn/quiescence.ts +353 -0
  112. package/src/activities/agent-turn/run-credentials.ts +761 -0
  113. package/src/activities/agent-turn/run.ts +1530 -0
  114. package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
  115. package/src/activities/agent-turn/sandbox-provision.ts +569 -0
  116. package/src/activities/agent-turn/sandbox-route.ts +358 -0
  117. package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
  118. package/src/activities/agent-turn/stream-attempt.ts +1670 -0
  119. package/src/activities/agent-turn/tool-environment.ts +729 -0
  120. package/src/activities/agent-turn/tool-policy.ts +216 -0
  121. package/src/activities/agent-turn/tool-result-spill.ts +265 -0
  122. package/src/activities/agent-turn/turn-context.ts +281 -0
  123. package/src/activities/agent-turn/xai-capacity.ts +229 -0
  124. package/src/activities/agent-turn.ts +12 -12701
  125. package/src/activities/api-integrations.ts +3 -0
  126. package/src/activities/automations.ts +214 -0
  127. package/src/activities/codemode-dispatcher.ts +12 -1
  128. package/src/activities/common.ts +1 -5
  129. package/src/activities/connector-attachments.ts +168 -0
  130. package/src/activities/context-compaction.ts +18 -0
  131. package/src/activities/environment.ts +151 -55
  132. package/src/activities/generated-images.ts +26 -11
  133. package/src/activities/git-credential-renewal.ts +13 -3
  134. package/src/activities/goals.ts +85 -19
  135. package/src/activities/google-drive-provider.ts +178 -0
  136. package/src/activities/google-drive-publication.ts +147 -49
  137. package/src/activities/incident-telemetry-authority.ts +327 -0
  138. package/src/activities/incident-telemetry-preflight.ts +317 -0
  139. package/src/activities/knowledge-source-sync.ts +178 -37
  140. package/src/activities/mcp-credentials.ts +191 -17
  141. package/src/activities/open-suffix-resume.ts +422 -0
  142. package/src/activities/packs.ts +12 -60
  143. package/src/activities/parent-wake.ts +64 -14
  144. package/src/activities/retained-screenshot-reaper.ts +6 -3
  145. package/src/activities/retained-screenshots.ts +15 -47
  146. package/src/activities/rig-verification.ts +1 -1
  147. package/src/activities/run-input.ts +102 -142
  148. package/src/activities/sandbox-images.ts +7 -166
  149. package/src/activities/sandbox-lease.ts +406 -15
  150. package/src/activities/scheduled-tasks.ts +2103 -411
  151. package/src/activities/session-state.ts +10 -2
  152. package/src/activities/types.ts +32 -2
  153. package/src/activities/video-media-validation.ts +3 -3
  154. package/src/activities/video-output-retention.ts +2 -57
  155. package/src/activities/xai-auth.ts +8 -0
  156. package/src/activities-control.ts +2 -0
  157. package/src/activities.ts +3 -0
  158. package/src/activity-services.ts +13 -4
  159. package/src/connection-provider-use.ts +36 -0
  160. package/src/editable-artifact-materializer-subprocess.ts +25 -14
  161. package/src/editable-artifact-materializer.ts +29 -18
  162. package/src/github-rest-mcp.ts +713 -0
  163. package/src/http.ts +16 -6
  164. package/src/index.ts +40 -5
  165. package/src/knowledge-source-sync-activity-policy.ts +2 -0
  166. package/src/model-context-contributions.ts +33 -10
  167. package/src/observability-metrics.ts +357 -0
  168. package/src/opensandbox-kubernetes-inventory.ts +292 -0
  169. package/src/personal-github-git-credentials.ts +331 -0
  170. package/src/pr-review-credentials.ts +163 -0
  171. package/src/sandbox-archive-storage.ts +159 -0
  172. package/src/sandbox-resume.ts +449 -63
  173. package/src/sandbox-routing.ts +314 -16
  174. package/src/scheduled-alert-occurrence.ts +176 -0
  175. package/src/workflows/activities.ts +18 -2
  176. package/src/workflows/automations.ts +11 -0
  177. package/src/workflows/session.ts +13 -3
  178. package/src/workflows.ts +1 -0
  179. package/dist/activities-control-BJSDRSIP.js.map +0 -1
  180. package/dist/activities-turn-J5YO7XXX.js +0 -17243
  181. package/dist/activities-turn-J5YO7XXX.js.map +0 -1
  182. package/dist/chunk-N5TJIFCE.js.map +0 -1
  183. package/dist/chunk-P6Z2SLAA.js +0 -3092
  184. package/dist/chunk-P6Z2SLAA.js.map +0 -1
@@ -0,0 +1,754 @@
1
+ import {
2
+ commitSessionAttemptQuiescence,
3
+ recordCodexAccountUsageWithWakeTargets,
4
+ releaseCodexCredentialLease,
5
+ releaseXaiCredentialLease,
6
+ updateXaiQuotaMetadata,
7
+ type SessionAttemptQuiescenceCommit,
8
+ } from "@opengeni/db";
9
+ import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@opengeni/events";
10
+ import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
11
+ import { clearRunCredentialsForAttempt } from "@opengeni/runtime";
12
+ import { fetchXaiSubscriptionQuota } from "@opengeni/xai-subscription";
13
+ import type { Settings } from "@opengeni/config";
14
+ import { signalCodexCapacityWakeTargets } from "../codex-capacity";
15
+ import type { currentActivityContext } from "../streaming";
16
+ import type { CodemodeTokenRenewalController } from "../codemode-token-renewal";
17
+ import type { RunCredentialRenewalController } from "../run-credential-renewal";
18
+ import type {
19
+ RunAgentTurnInput,
20
+ SessionAttemptQuiescenceProof,
21
+ TurnActivityServices as ActivityServices,
22
+ } from "../types";
23
+ import { captureWorkspaceRevision, openFreshWorkspaceCaptureSession } from "../workspace-capture";
24
+ import { type ChannelASession } from "@opengeni/runtime/sandbox";
25
+ import { createModelCheckpointMemoryCollector } from "../../model-checkpoint-memory-collector";
26
+ import { makeMachineOpObserver, turnLifecycleMetricsFor } from "../../observability-metrics";
27
+ import {
28
+ maybePersistWarmWorkspaceSnapshot,
29
+ waitForWarmSnapshot,
30
+ type ResumedTurnSandbox,
31
+ } from "../../sandbox-resume";
32
+ import { createTurnCredentialLeases } from "./credential-leases";
33
+ import { safeErrorDiagnostic, safeErrorForTelemetry } from "./errors";
34
+ import {
35
+ assertPhysicalToolQuiescenceForCancellation,
36
+ assertSessionAttemptQuiescenceRecoveryDurable,
37
+ clearAttemptCredentialsWithSettledFence,
38
+ drainAttemptOwnedSandboxWriters,
39
+ persistOrSignalSessionAttemptQuiescence,
40
+ releaseTurnSandboxAfterWriterDrain,
41
+ shouldRunTurnEndWorkspacePersistence,
42
+ turnFinalizerCancellationSignal,
43
+ waitForTurnFinalizerStep,
44
+ } from "./quiescence";
45
+ import type {
46
+ AttemptIdentityState,
47
+ EventingState,
48
+ ProviderTurnState,
49
+ RecordingState,
50
+ RenewalState,
51
+ SandboxRuntimeState,
52
+ TurnControlState,
53
+ WorkspaceRefState,
54
+ } from "./turn-context";
55
+
56
+ export type TurnFinalizationDeps = {
57
+ input: RunAgentTurnInput;
58
+ settings: Settings;
59
+ db: ActivityServices["db"];
60
+ bus: ActivityServices["bus"];
61
+ objectStorage: ActivityServices["objectStorage"];
62
+ observability: ActivityServices["observability"];
63
+ wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
64
+ signalSessionAttemptQuiesced: ActivityServices["signalSessionAttemptQuiesced"];
65
+ signalCodexCapacityWorkflow: ActivityServices["signalCodexCapacityWorkflow"];
66
+ cancellationSignal: AbortSignal | undefined;
67
+ sandboxResumeController: AbortController;
68
+ activityContext: ReturnType<typeof currentActivityContext>;
69
+ activityStarted: number;
70
+ activitySpan: ReturnType<ActivityServices["observability"]["startSpan"]>;
71
+ dispatchId: string;
72
+ noteCancellationRequested: () => void;
73
+ machineOpObserver: ReturnType<typeof makeMachineOpObserver>;
74
+ leases: ReturnType<typeof createTurnCredentialLeases>;
75
+ control: TurnControlState;
76
+ attempt: AttemptIdentityState;
77
+ sandboxState: SandboxRuntimeState;
78
+ renewals: RenewalState;
79
+ recordingState: RecordingState;
80
+ eventing: EventingState;
81
+ providerTurn: ProviderTurnState;
82
+ workspaceRefs: WorkspaceRefState;
83
+ runWorkspaceMutationForSandbox: <T>(
84
+ sandbox: ResumedTurnSandbox,
85
+ operation: string,
86
+ mutation: () => Promise<T>,
87
+ ) => Promise<T>;
88
+ requireResolvedSandboxForMutation: (message: string) => ResumedTurnSandbox;
89
+ stopLeaseHeartbeat: () => void;
90
+ abandonActiveRecording: (reason: string, disposition?: "failed" | "discard") => Promise<void>;
91
+ turnCompletionMemoryCollector: ReturnType<typeof createModelCheckpointMemoryCollector>;
92
+ };
93
+
94
+ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<void> {
95
+ const {
96
+ input,
97
+ settings,
98
+ db,
99
+ bus,
100
+ objectStorage,
101
+ observability,
102
+ wakeSessionWorkflow,
103
+ signalSessionAttemptQuiesced,
104
+ signalCodexCapacityWorkflow,
105
+ cancellationSignal,
106
+ sandboxResumeController,
107
+ activityContext,
108
+ activityStarted,
109
+ activitySpan,
110
+ dispatchId,
111
+ noteCancellationRequested,
112
+ machineOpObserver,
113
+ leases,
114
+ control,
115
+ attempt,
116
+ sandboxState,
117
+ renewals,
118
+ recordingState,
119
+ eventing,
120
+ providerTurn,
121
+ workspaceRefs,
122
+ runWorkspaceMutationForSandbox,
123
+ requireResolvedSandboxForMutation,
124
+ stopLeaseHeartbeat,
125
+ abandonActiveRecording,
126
+ turnCompletionMemoryCollector,
127
+ } = deps;
128
+ // This is the logical ownership boundary. Abort before any fallible
129
+ // housekeeping so a still-pending provider establish releases its private
130
+ // holder/timer even when Temporal itself never delivered cancellation.
131
+ if (sandboxState.resolvedSandbox === null && !sandboxResumeController.signal.aborted) {
132
+ sandboxResumeController.abort(
133
+ cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
134
+ );
135
+ }
136
+ const finalizationStarted = performance.now();
137
+ let finalizationError: unknown;
138
+ let physicalToolQuiescenceConfirmed = !control.acknowledgeQuiescence;
139
+ let quiescenceReceiptOrProofDurable = !control.acknowledgeQuiescence;
140
+ const finalizerSignal = turnFinalizerCancellationSignal(
141
+ cancellationSignal,
142
+ control.activityStatus,
143
+ );
144
+ try {
145
+ const toolCancellationFence = eventing.toolCancellationFenceRef.current;
146
+ // Every renewal controller is an attempt-owned sandbox writer. Capture
147
+ // and close all of them before the tool/run-writer drain and before the
148
+ // quiescence receipt; none may start another admitted write afterward.
149
+ renewals.gitCredentialRenewalClosed = true;
150
+ const gitRenewalsToStop = renewals.gitCredentialRenewals;
151
+ renewals.gitCredentialRenewals = [];
152
+ renewals.codemodeTokenRenewalClosed = true;
153
+ const codemodeRenewalToStop =
154
+ renewals.codemodeTokenRenewal as CodemodeTokenRenewalController | null;
155
+ renewals.codemodeTokenRenewal = null;
156
+ renewals.runCredentialRenewalClosed = true;
157
+ const runRenewalToStop = renewals.runCredentialRenewal as RunCredentialRenewalController | null;
158
+ renewals.runCredentialRenewal = null;
159
+
160
+ // Capture the attempt-qualified credential target before draining the
161
+ // controllers that can still write a newer generation.
162
+ const credentialSessionToClear = renewals.runCredentialSession;
163
+ renewals.runCredentialSession = null;
164
+ await drainAttemptOwnedSandboxWriters({
165
+ // Normal turn completion owns the same process boundary as
166
+ // Pause/Steer: yielded provider shells must be terminated, polled,
167
+ // and durably settled before workspace capture. Only receipt
168
+ // publication remains conditional on acknowledgeQuiescence.
169
+ toolCancellationFence,
170
+ cancellationReason: cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
171
+ gitCredentialRenewals: gitRenewalsToStop,
172
+ codemodeTokenRenewal: codemodeRenewalToStop,
173
+ runCredentialRenewal: runRenewalToStop,
174
+ });
175
+ // Attempt-qualified credential deletion is also a real workspace write,
176
+ // but it must not be submitted behind the very command being cancelled.
177
+ // Drain tool processes and credential renewals first, then delete only this
178
+ // attempt's generation before publishing physical quiescence. A failure
179
+ // deliberately keeps the receipt closed.
180
+ if (credentialSessionToClear) {
181
+ const clearAttemptCredentials = async (): Promise<void> =>
182
+ await clearRunCredentialsForAttempt(credentialSessionToClear, {
183
+ sessionId: input.sessionId,
184
+ attemptId: input.attemptId,
185
+ executionGeneration: attempt.executionGeneration,
186
+ });
187
+ await clearAttemptCredentialsWithSettledFence({
188
+ activityStatus: control.activityStatus,
189
+ runWorkspaceFencedClear: async () =>
190
+ await runWorkspaceMutationForSandbox(
191
+ requireResolvedSandboxForMutation(
192
+ "Run credential cleanup has no exact sandbox lease target",
193
+ ),
194
+ "runCredentialAttemptClear",
195
+ clearAttemptCredentials,
196
+ ),
197
+ clearExactAttempt: clearAttemptCredentials,
198
+ onSettledAttemptFence: () => {
199
+ // Terminal settlement closes the active attempt before this finally
200
+ // runs, so the ordinary workspace admission correctly rejects it.
201
+ // The attempt-qualified delete is nevertheless successor-safe: it
202
+ // removes only this attempt/generation and clears the pointer only
203
+ // when it still names that exact generation. Finish that cleanup
204
+ // directly so a successful turn can continue into workspace capture,
205
+ // tool teardown, and lease release.
206
+ observability.info("retrying exact run credential cleanup after turn settlement", {
207
+ "opengeni.session_id": input.sessionId,
208
+ "opengeni.turn_id": attempt.turnId ?? "",
209
+ "opengeni.attempt_id": input.attemptId,
210
+ });
211
+ },
212
+ });
213
+ }
214
+ sandboxState.attemptWritersDrained = true;
215
+ if (control.acknowledgeQuiescence) {
216
+ // A cancellation before sandbox-backed capabilities exist still has
217
+ // no tool controller to drain. Sandbox agent construction fails closed
218
+ // when a backend exists but no controller was installed. Renewal
219
+ // writers, when present, were drained above in either case.
220
+ physicalToolQuiescenceConfirmed = true;
221
+ }
222
+ if (control.acknowledgeQuiescence && physicalToolQuiescenceConfirmed) {
223
+ // This receipt is part of the hard cancellation boundary, not
224
+ // housekeeping. Persist it immediately after the sandbox/tool fence
225
+ // and before lease, cache, recording, or provider cleanup. Its
226
+ // transaction also enqueues the exact workflow wake that will admit
227
+ // the replacement; Temporal activity terminalization does neither.
228
+ const proof: SessionAttemptQuiescenceProof = {
229
+ accountId: input.accountId,
230
+ workspaceId: input.workspaceId,
231
+ sessionId: input.sessionId,
232
+ attemptId: input.attemptId,
233
+ workflowId: input.workflowId,
234
+ workflowRunId: input.workflowRunId,
235
+ activityId: dispatchId,
236
+ };
237
+ const recoveryMode = await persistOrSignalSessionAttemptQuiescence({
238
+ proof,
239
+ persistReceipt: async () =>
240
+ await commitSessionAttemptQuiescence(db, {
241
+ accountId: input.accountId,
242
+ workspaceId: input.workspaceId,
243
+ sessionId: input.sessionId,
244
+ attemptId: input.attemptId,
245
+ temporalWorkflowId: input.workflowId,
246
+ temporalWorkflowRunId: input.workflowRunId,
247
+ temporalActivityId: dispatchId,
248
+ allowUninterrupted: true,
249
+ }),
250
+ ...(wakeSessionWorkflow
251
+ ? {
252
+ deliverWorkflowWake: async (
253
+ wake: NonNullable<SessionAttemptQuiescenceCommit["workflowWake"]>,
254
+ ) =>
255
+ await wakeSessionWorkflow({
256
+ accountId: wake.accountId,
257
+ workspaceId: wake.workspaceId,
258
+ sessionId: wake.sessionId,
259
+ workflowId: wake.temporalWorkflowId,
260
+ wakeRevision: wake.wakeRevision,
261
+ interruptionRequested: wake.interruptionRequested,
262
+ }),
263
+ }
264
+ : {}),
265
+ publishEvents: async (events) => {
266
+ await waitForTurnFinalizerStep(
267
+ publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events),
268
+ finalizerSignal,
269
+ );
270
+ },
271
+ signalProof: signalSessionAttemptQuiesced,
272
+ heartbeat: (deliveryAttempt, retryMs) => {
273
+ activityContext?.heartbeat({
274
+ phase: "quiescence-proof-delivery",
275
+ sessionId: input.sessionId,
276
+ attemptId: input.attemptId,
277
+ deliveryAttempt,
278
+ retryMs,
279
+ at: new Date().toISOString(),
280
+ });
281
+ },
282
+ onReceiptFailure: (error) => {
283
+ console.error(
284
+ "agent turn quiescence receipt exhausted; signalling proof",
285
+ safeErrorDiagnostic(error),
286
+ );
287
+ },
288
+ onWakeFailure: (error) => {
289
+ console.error(
290
+ "agent turn quiescence immediate workflow wake failed; outbox repair retained",
291
+ safeErrorDiagnostic(error),
292
+ );
293
+ },
294
+ onPublishFailure: (error) => {
295
+ console.error("agent turn quiescence event fanout failed", safeErrorDiagnostic(error));
296
+ },
297
+ onSignalFailure: (error, deliveryAttempt, retryMs) => {
298
+ console.error("agent turn quiescence proof signal failed; retrying", {
299
+ error: safeErrorDiagnostic(error),
300
+ attempt: deliveryAttempt,
301
+ retryMs,
302
+ });
303
+ },
304
+ });
305
+ quiescenceReceiptOrProofDurable = true;
306
+ if (recoveryMode === "signal") {
307
+ observability.info("agent turn quiescence proof handed to workflow recovery", {
308
+ "opengeni.session_id": input.sessionId,
309
+ "opengeni.attempt_id": input.attemptId,
310
+ "opengeni.workflow_run_id": input.workflowRunId,
311
+ "opengeni.activity_id": dispatchId,
312
+ });
313
+ }
314
+ }
315
+ // Drain the buffered Connected Machine op events (infra failures + healed
316
+ // recoveries) to durable session events — awaited, best-effort, never blocking
317
+ // the turn. Sync observer → buffer → single awaited append here (no unawaited
318
+ // DB write inside the activity). Scoped to this turn; skipped if no turnId
319
+ // (the op ran under a turn, so on the normal path turnId is set).
320
+ const machineOpEvents = machineOpObserver.drainEvents();
321
+ if (machineOpEvents.length > 0 && attempt.turnId && attempt.executionGeneration > 0) {
322
+ await waitForTurnFinalizerStep(
323
+ appendAndPublishTurnEventsFenced(
324
+ db,
325
+ bus,
326
+ input.workspaceId,
327
+ input.sessionId,
328
+ attempt.turnId,
329
+ attempt.executionGeneration,
330
+ input.attemptId,
331
+ machineOpEvents.map((event) => ({
332
+ ...event,
333
+ turnId: attempt.turnId ?? null,
334
+ })),
335
+ ).catch(() => undefined),
336
+ finalizerSignal,
337
+ );
338
+ }
339
+ // Multi-account P4: flush the serving account's free per-turn caches ONCE,
340
+ // best-effort (same discipline as today's usage write). Both writers skip
341
+ // version/updatedAt, so neither can race the token-refresh CAS.
342
+ if (providerTurn.effectiveCodexCredentialId) {
343
+ // Part A: the latest scraped usage-header snapshot → the P2 usage cache. A
344
+ // full both-windows snapshot (parseCodexUsageHeaders gates on both), so this
345
+ // is byte-identical to the /wham/usage write — no partial-window clobber.
346
+ if (providerTurn.latestCodexUsage) {
347
+ const usageMutation = await waitForTurnFinalizerStep(
348
+ recordCodexAccountUsageWithWakeTargets(
349
+ db,
350
+ input.workspaceId,
351
+ providerTurn.effectiveCodexCredentialId,
352
+ providerTurn.latestCodexUsage,
353
+ ).catch(() => null),
354
+ finalizerSignal,
355
+ );
356
+ if (usageMutation) {
357
+ await waitForTurnFinalizerStep(
358
+ signalCodexCapacityWakeTargets(
359
+ { signalCodexCapacityWorkflow, wakeSessionWorkflow },
360
+ usageMutation.wakeTargets,
361
+ ),
362
+ finalizerSignal,
363
+ );
364
+ }
365
+ }
366
+ }
367
+ leases.codex.stopHeartbeat();
368
+ if (
369
+ leases.codex.held &&
370
+ attempt.turnId &&
371
+ leases.codex.holderId &&
372
+ leases.codex.generation !== null
373
+ ) {
374
+ await waitForTurnFinalizerStep(
375
+ releaseCodexCredentialLease(
376
+ db,
377
+ input.accountId,
378
+ input.workspaceId,
379
+ attempt.turnId,
380
+ leases.codex.holderId,
381
+ leases.codex.generation,
382
+ ).catch(() => undefined),
383
+ finalizerSignal,
384
+ );
385
+ leases.codex.held = false;
386
+ }
387
+ if (
388
+ providerTurn.effectiveXaiCredentialId &&
389
+ leases.xai.subjectId &&
390
+ providerTurn.xaiRequestContext &&
391
+ !providerTurn.xaiCredentialQuarantined
392
+ ) {
393
+ const quota = await waitForTurnFinalizerStep(
394
+ fetchXaiSubscriptionQuota({ context: providerTurn.xaiRequestContext }).catch(() => null),
395
+ finalizerSignal,
396
+ );
397
+ if (quota) {
398
+ await waitForTurnFinalizerStep(
399
+ updateXaiQuotaMetadata(db, {
400
+ workspaceId: input.workspaceId,
401
+ subjectId: leases.xai.subjectId,
402
+ credentialId: providerTurn.effectiveXaiCredentialId,
403
+ quotaUsedPercent: quota.usedPercent,
404
+ quotaResetAt: quota.period?.end ?? null,
405
+ quotaCheckedAt: quota.checkedAt,
406
+ exhaustedUntil:
407
+ quota.usedPercent !== null && quota.usedPercent >= 100
408
+ ? (quota.period?.end ?? null)
409
+ : null,
410
+ }).catch(() => undefined),
411
+ finalizerSignal,
412
+ );
413
+ }
414
+ }
415
+ leases.xai.stopHeartbeat();
416
+ if (
417
+ leases.xai.held &&
418
+ attempt.turnId &&
419
+ leases.xai.subjectId &&
420
+ leases.xai.holderId &&
421
+ leases.xai.generation !== null
422
+ ) {
423
+ await waitForTurnFinalizerStep(
424
+ releaseXaiCredentialLease(db, {
425
+ workspaceId: input.workspaceId,
426
+ subjectId: leases.xai.subjectId,
427
+ turnId: attempt.turnId,
428
+ holderId: leases.xai.holderId,
429
+ generation: leases.xai.generation,
430
+ }).catch(() => undefined),
431
+ finalizerSignal,
432
+ );
433
+ leases.xai.held = false;
434
+ }
435
+ // Workbench v2 turn-end workspace capture — runs FIRST in
436
+ // the turn-end finally, while the box is MAXIMALLY ALIVE. The agent's last
437
+ // tool ran before this finally, so /workspace is already final; capture is
438
+ // FS-equivalent to the already-settled recording preparation and the warm
439
+ // snapshot (neither mutates workspace files). Running it here — BEFORE
440
+ // preparedTools.close() (which tears down tools / computer-use / the display
441
+ // stack and is what starts the Modal box exiting a few seconds later) —
442
+ // gives capture the full live-box margin instead of racing the teardown
443
+ // tail, which was dropping 100% of captures on real Modal desktop boxes
444
+ // ("request cancelled due to container exiting", 0 rows). External module:
445
+ // self-capped at 120s, best-effort (never throws past its boundary),
446
+ // epoch-fenced, and it NEVER closes the box. The emitted
447
+ // workspace.revision.captured event is ANNOUNCE-ONLY (metadata, never
448
+ // content).
449
+ if (process.env.OPENGENI_TEST_SCENARIO === "sandbox") {
450
+ console.log(
451
+ `[sandbox-e2e] capture preflight ownership=${settings.sandboxOwnershipEnabled} enabled=${settings.workspaceCaptureEnabled} resolved=${Boolean(sandboxState.resolvedSandbox)} session=${Boolean(sandboxState.setupBoxSession)} group=${Boolean(sandboxState.sandboxGroupId)} storage=${Boolean(objectStorage)}`,
452
+ );
453
+ }
454
+ const runTurnEndPersistence = shouldRunTurnEndWorkspacePersistence({
455
+ activityStatus: control.activityStatus,
456
+ cancellationRequested: finalizerSignal?.aborted === true,
457
+ });
458
+ if (
459
+ runTurnEndPersistence &&
460
+ attempt.turnId &&
461
+ sandboxState.resolvedSandbox &&
462
+ sandboxState.setupBoxSession &&
463
+ sandboxState.sandboxGroupId
464
+ ) {
465
+ // Block new periodic snapshots, then drain any one already in flight.
466
+ // Keep the lease heartbeat itself running: capture may legitimately
467
+ // exceed the 90s holder TTL, and the reaper must remain unable to drain
468
+ // the exact box while this holder still reads it.
469
+ sandboxState.turnEndCaptureInProgress = true;
470
+ if (sandboxState.snapshotInFlight) {
471
+ await waitForWarmSnapshot(
472
+ sandboxState.snapshotInFlight,
473
+ settings.sandboxSnapshotTimeoutMs,
474
+ finalizerSignal,
475
+ );
476
+ }
477
+ const captureEstablished = sandboxState.resolvedSandbox.established;
478
+ await captureWorkspaceRevision({
479
+ db,
480
+ objectStorage,
481
+ settings,
482
+ publish: async (events) => {
483
+ await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events);
484
+ },
485
+ session: sandboxState.setupBoxSession as ChannelASession,
486
+ openReadSession: async () =>
487
+ await openFreshWorkspaceCaptureSession({
488
+ backendId: captureEstablished.backendId,
489
+ client: captureEstablished.client,
490
+ session: sandboxState.setupBoxSession as ChannelASession,
491
+ expectedInstanceId: captureEstablished.instanceId,
492
+ }),
493
+ leaseEpoch: sandboxState.resolvedSandbox.leaseEpoch,
494
+ sandboxGroupId: sandboxState.sandboxGroupId,
495
+ accountId: input.accountId,
496
+ workspaceId: input.workspaceId,
497
+ sessionId: input.sessionId,
498
+ turnId: attempt.turnId,
499
+ attemptId: input.attemptId,
500
+ observability,
501
+ ...(finalizerSignal ? { signal: finalizerSignal } : {}),
502
+ });
503
+ }
504
+ eventing.toolPreparationClosing = true;
505
+ if (eventing.toolPreparationReady) {
506
+ await waitForTurnFinalizerStep(
507
+ eventing.toolPreparationReady.catch(() => undefined),
508
+ finalizerSignal,
509
+ );
510
+ }
511
+ if (eventing.codemodeDispatcher) {
512
+ await waitForTurnFinalizerStep(
513
+ eventing.codemodeDispatcher.close().catch(() => undefined),
514
+ finalizerSignal,
515
+ );
516
+ eventing.codemodeDispatcher = null;
517
+ }
518
+ if (eventing.preparedTools) {
519
+ await waitForTurnFinalizerStep(
520
+ eventing.preparedTools.close().catch(() => undefined),
521
+ finalizerSignal,
522
+ );
523
+ }
524
+ if (eventing.heartbeatTimer) {
525
+ clearInterval(eventing.heartbeatTimer);
526
+ }
527
+ if (sandboxState.turnSandboxProvisioner?.hasStarted()) {
528
+ await waitForTurnFinalizerStep(
529
+ sandboxState.turnSandboxProvisioner.waitForSettled(30_000),
530
+ finalizerSignal,
531
+ );
532
+ } else if (sandboxState.prefetchedManagedBox) {
533
+ // Create finished (or was aborted) before get()/setup. Join so a
534
+ // successful prefetch cannot leak a holder after this attempt ends.
535
+ await waitForTurnFinalizerStep(
536
+ sandboxState.prefetchedManagedBox.then(
537
+ (box) => {
538
+ sandboxState.prefetchedManagedBoxResult = box;
539
+ },
540
+ () => undefined,
541
+ ),
542
+ finalizerSignal,
543
+ );
544
+ }
545
+ // P1.2: stop the lease-TTL refresh, release the turn holder (idempotent
546
+ // delete-my-row; refcount-- and warm->draining if it hit 0 with no turns),
547
+ // and DROP the in-memory handle. Release NEVER stops the box — the reaper
548
+ // (P1.3) issues the provider stop() past the drain grace at refcount 0; the
549
+ // box rides the provider idle-timeout in the meantime. Best-effort: a
550
+ // release failure must never mask the turn's real outcome.
551
+ if (sandboxState.leaseHeartbeatTimer) {
552
+ stopLeaseHeartbeat();
553
+ }
554
+ if (sandboxState.rotationInFlight) {
555
+ await waitForTurnFinalizerStep(sandboxState.rotationInFlight, finalizerSignal).catch(
556
+ () => undefined,
557
+ );
558
+ }
559
+ // A recording normally closes inside the attempt-fenced turn settlement.
560
+ // Reaching finally with one still active means settlement threw, never ran,
561
+ // or lost ownership. Stop ffmpeg and mark only this exact attempt-owned row
562
+ // failed; publish no event and leave the artifact recoverable on the box.
563
+ await waitForTurnFinalizerStep(
564
+ abandonActiveRecording(
565
+ "activity ended without recording settlement",
566
+ recordingState.didComputerUse ? "failed" : "discard",
567
+ ),
568
+ finalizerSignal,
569
+ );
570
+ if (sandboxState.resolvedSandbox) {
571
+ // TURN-END mid-session snapshot (sandbox-file-persistence): fold the
572
+ // turn's finished /workspace onto the lease before releasing the holder,
573
+ // so the work this turn just produced survives any unclean box death in
574
+ // the idle window ahead. Throttled by the same interval as the heartbeat
575
+ // tick (a short turn right after a snapshot skips — bounded-loss contract
576
+ // is the interval, not per-turn). Best-effort and time-capped by the
577
+ // helper's own failure discipline; never delays release on failure.
578
+ const settledTurnId = attempt.turnId;
579
+ if (
580
+ runTurnEndPersistence &&
581
+ sandboxState.setupBoxSession &&
582
+ sandboxState.sandboxGroupId &&
583
+ settledTurnId
584
+ ) {
585
+ // Single-flight vs the heartbeat capture: the timer is already cleared
586
+ // above, but a capture it launched may still be in flight — and that
587
+ // capture predates the turn's final writes. Wait for it, but only up
588
+ // to the snapshot timeout: release must never depend on an unbounded
589
+ // provider capture.
590
+ if (sandboxState.snapshotInFlight) {
591
+ await waitForWarmSnapshot(
592
+ sandboxState.snapshotInFlight,
593
+ settings.sandboxSnapshotTimeoutMs,
594
+ finalizerSignal,
595
+ );
596
+ }
597
+ const persisted = await maybePersistWarmWorkspaceSnapshot(
598
+ { db, settings, objectStorage },
599
+ {
600
+ accountId: input.accountId,
601
+ workspaceId: input.workspaceId,
602
+ sessionId: input.sessionId,
603
+ turnId: settledTurnId,
604
+ attemptId: input.attemptId,
605
+ sandboxGroupId: sandboxState.sandboxGroupId,
606
+ },
607
+ sandboxState.setupBoxSession,
608
+ sandboxState.resolvedSandbox.leaseEpoch,
609
+ finalizerSignal,
610
+ );
611
+ if (persisted && eventing.publish) {
612
+ await eventing
613
+ .publish([
614
+ {
615
+ type: "sandbox.box.snapshot",
616
+ payload: { trigger: "turn-end" },
617
+ },
618
+ ])
619
+ .catch(() => undefined);
620
+ }
621
+ // NB workspace capture no longer runs here — it moved to
622
+ // the TOP of this finally (before preparedTools.close) so it completes
623
+ // while the box is still solidly alive, instead of racing the turn-end
624
+ // teardown that was killing 100% of captures on real Modal desktop boxes.
625
+ }
626
+ }
627
+ } catch (error) {
628
+ finalizationError ??= error;
629
+ console.error(
630
+ "agent turn finalization failed (turn outcome unaffected)",
631
+ safeErrorDiagnostic(error),
632
+ );
633
+ } finally {
634
+ // The writer drain is the only authority that licenses settling an
635
+ // abandoned turn admission. Keep this second-stage release outside all
636
+ // later housekeeping so an event/cache/capture/recording failure cannot
637
+ // strand a null-outcome admission forever. Do not race it against
638
+ // Temporal cancellation: the eager listener may already have dropped
639
+ // the holder, and this idempotent proof-bearing pass still must run.
640
+ stopLeaseHeartbeat();
641
+ const sandboxReleaseTargets = new Set(sandboxState.lateSandboxesAwaitingWriterDrain);
642
+ sandboxState.lateSandboxesAwaitingWriterDrain.clear();
643
+ if (sandboxState.resolvedSandbox) {
644
+ sandboxReleaseTargets.add(sandboxState.resolvedSandbox);
645
+ sandboxState.resolvedSandbox = null;
646
+ } else if (sandboxState.prefetchedManagedBoxResult) {
647
+ sandboxReleaseTargets.add(sandboxState.prefetchedManagedBoxResult);
648
+ }
649
+ sandboxState.prefetchedManagedBoxResult = null;
650
+ if (sandboxState.attemptWritersDrained) {
651
+ for (const sandboxToRelease of sandboxReleaseTargets) {
652
+ try {
653
+ await releaseTurnSandboxAfterWriterDrain(sandboxToRelease);
654
+ } catch (releaseError) {
655
+ finalizationError ??= releaseError;
656
+ console.error(
657
+ "sandbox lease quiesced release failed (turn outcome unaffected)",
658
+ safeErrorDiagnostic(releaseError),
659
+ );
660
+ }
661
+ }
662
+ } else {
663
+ // No proof exists. Ensure every late result still receives the eager
664
+ // leak-prevention release while preserving its admissions as a
665
+ // fail-closed archive-capture fence.
666
+ for (const sandboxToRelease of sandboxReleaseTargets) {
667
+ await sandboxToRelease.release().catch(() => undefined);
668
+ }
669
+ }
670
+ // Close provisioning after consuming the currently known targets. A
671
+ // provider result that races in later is routed through releaseLateSandbox;
672
+ // targets already released above synchronously removed their listener.
673
+ if (!sandboxResumeController.signal.aborted) {
674
+ sandboxResumeController.abort(
675
+ cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
676
+ );
677
+ }
678
+ cancellationSignal?.removeEventListener("abort", noteCancellationRequested);
679
+ const completedAt = performance.now();
680
+ const durationSeconds = (completedAt - activityStarted) / 1000;
681
+ const finalizationDurationSeconds = (completedAt - finalizationStarted) / 1000;
682
+ observability.observeHistogram({
683
+ name: "opengeni_turn_finalization_duration_seconds",
684
+ help: "Agent turn finalization duration, including workspace housekeeping and lease release.",
685
+ labels: {
686
+ cancellation_requested: String(control.cancellationRequestedAt !== null),
687
+ },
688
+ value: finalizationDurationSeconds,
689
+ });
690
+ if (control.cancellationRequestedAt !== null) {
691
+ const physicalCancellationDurationSeconds =
692
+ (completedAt - control.cancellationRequestedAt) / 1000;
693
+ observability.observeHistogram({
694
+ name: "opengeni_turn_physical_cancellation_duration_seconds",
695
+ help: "Time from Temporal cancellation delivery until the activity physically stops.",
696
+ value: physicalCancellationDurationSeconds,
697
+ });
698
+ observability.info("agent turn physical cancellation completed", {
699
+ durationMs: Math.round(physicalCancellationDurationSeconds * 1000),
700
+ ...(sandboxState.sandboxGroupId
701
+ ? {
702
+ sandboxLeaseKey: sandboxLeaseTelemetryKey(
703
+ input.workspaceId,
704
+ sandboxState.sandboxGroupId,
705
+ ),
706
+ }
707
+ : {}),
708
+ });
709
+ }
710
+ observability.recordWorkerActivity({
711
+ activity: "runAgentTurn",
712
+ status: finalizationError ? "cleanup_failed" : control.activityStatus,
713
+ durationSeconds,
714
+ });
715
+ if (attempt.turnId && control.activityStatus !== "unknown") {
716
+ turnLifecycleMetricsFor(observability).finish(
717
+ attempt.turnId,
718
+ control.turnMetricOutcome,
719
+ durationSeconds,
720
+ );
721
+ }
722
+ activitySpan.end({
723
+ attributes: {
724
+ "opengeni.turn_id": attempt.turnId ?? "",
725
+ "opengeni.status": control.activityStatus,
726
+ "opengeni.variable_set_id": workspaceRefs.variableSetId,
727
+ "opengeni.rig_id": workspaceRefs.rigId,
728
+ "opengeni.rig_version_id": workspaceRefs.rigVersionId,
729
+ "opengeni.codex_credential_id": providerTurn.effectiveCodexCredentialId ?? "",
730
+ "opengeni.duration_ms": Math.round(durationSeconds * 1000),
731
+ "opengeni.finalization_duration_ms": Math.round(finalizationDurationSeconds * 1000),
732
+ },
733
+ error:
734
+ finalizationError || control.activityError
735
+ ? safeErrorForTelemetry(finalizationError ?? control.activityError)
736
+ : undefined,
737
+ });
738
+ // This timer runs only after the activity promise and its full turn
739
+ // stack unwind. Checkpoint collection alone cannot reclaim objects that
740
+ // remain strongly reachable until this terminal boundary.
741
+ turnCompletionMemoryCollector.schedule(observability);
742
+ assertPhysicalToolQuiescenceForCancellation({
743
+ acknowledgeQuiescence: control.acknowledgeQuiescence,
744
+ physicalToolQuiescenceConfirmed,
745
+ failure: finalizationError,
746
+ });
747
+ assertSessionAttemptQuiescenceRecoveryDurable({
748
+ acknowledgeQuiescence: control.acknowledgeQuiescence,
749
+ physicalToolQuiescenceConfirmed,
750
+ receiptOrProofDurable: quiescenceReceiptOrProofDurable,
751
+ failure: finalizationError,
752
+ });
753
+ }
754
+ }