@opengeni/worker-bundle 0.20.3 → 0.22.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +52 -0
  2. package/dist/activities/agent-turn/agent-build.d.ts +89 -0
  3. package/dist/activities/agent-turn/claim.d.ts +67 -0
  4. package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
  5. package/dist/activities/agent-turn/codex.d.ts +32 -0
  6. package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
  7. package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
  8. package/dist/activities/agent-turn/errors.d.ts +253 -0
  9. package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
  10. package/dist/activities/agent-turn/file-resources.d.ts +37 -0
  11. package/dist/activities/agent-turn/finalization.d.ts +42 -0
  12. package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
  13. package/dist/activities/agent-turn/governance-model.d.ts +66 -0
  14. package/dist/activities/agent-turn/history-sink.d.ts +52 -0
  15. package/dist/activities/agent-turn/history.d.ts +63 -0
  16. package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
  17. package/dist/activities/agent-turn/model-usage.d.ts +218 -0
  18. package/dist/activities/agent-turn/quiescence.d.ts +135 -0
  19. package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
  20. package/dist/activities/agent-turn/run.d.ts +3 -0
  21. package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
  22. package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
  23. package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
  24. package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
  25. package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
  26. package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
  27. package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
  28. package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
  29. package/dist/activities/agent-turn/turn-context.d.ts +144 -0
  30. package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
  31. package/dist/activities/agent-turn.d.ts +12 -954
  32. package/dist/activities/automations.d.ts +16 -0
  33. package/dist/activities/common.d.ts +1 -1
  34. package/dist/activities/connector-attachments.d.ts +13 -0
  35. package/dist/activities/documents.d.ts +1 -0
  36. package/dist/activities/environment.d.ts +26 -15
  37. package/dist/activities/generated-images.d.ts +1 -1
  38. package/dist/activities/git-credential-renewal.d.ts +3 -1
  39. package/dist/activities/goals.d.ts +4 -1
  40. package/dist/activities/google-drive-provider.d.ts +29 -0
  41. package/dist/activities/google-drive-publication.d.ts +17 -4
  42. package/dist/activities/incident-telemetry-authority.d.ts +52 -0
  43. package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
  44. package/dist/activities/knowledge-source-sync.d.ts +4 -0
  45. package/dist/activities/mcp-credentials.d.ts +9 -3
  46. package/dist/activities/open-suffix-resume.d.ts +77 -0
  47. package/dist/activities/packs.d.ts +3 -11
  48. package/dist/activities/parent-wake.d.ts +19 -0
  49. package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
  50. package/dist/activities/retained-screenshots.d.ts +1 -1
  51. package/dist/activities/rig-verification.d.ts +1 -1
  52. package/dist/activities/run-input.d.ts +9 -10
  53. package/dist/activities/sandbox-images.d.ts +1 -14
  54. package/dist/activities/sandbox-lease.d.ts +21 -3
  55. package/dist/activities/scheduled-tasks.d.ts +47 -0
  56. package/dist/activities/types.d.ts +23 -2
  57. package/dist/activities/video-media-validation.d.ts +1 -1
  58. package/dist/activities/video-output-retention.d.ts +1 -2
  59. package/dist/activities/xai-auth.d.ts +3 -0
  60. package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
  61. package/dist/activities-control-DKG22ONJ.js.map +1 -0
  62. package/dist/activities-control.d.ts +6 -2
  63. package/dist/activities-turn-4IMC3MAA.js +21326 -0
  64. package/dist/activities-turn-4IMC3MAA.js.map +1 -0
  65. package/dist/activities.d.ts +7 -3
  66. package/dist/artifact-materializer-entry.js +41 -22
  67. package/dist/artifact-materializer-entry.js.map +1 -1
  68. package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
  69. package/dist/chunk-CE7TPKBS.js.map +1 -0
  70. package/dist/chunk-CNYGVRKF.js +4698 -0
  71. package/dist/chunk-CNYGVRKF.js.map +1 -0
  72. package/dist/connection-provider-use.d.ts +18 -0
  73. package/dist/editable-artifact-materializer.d.ts +4 -3
  74. package/dist/github-rest-mcp.d.ts +26 -0
  75. package/dist/index.d.ts +10 -2
  76. package/dist/index.js +58 -14
  77. package/dist/index.js.map +1 -1
  78. package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
  79. package/dist/model-context-contributions.d.ts +8 -4
  80. package/dist/observability-metrics.d.ts +63 -4
  81. package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
  82. package/dist/personal-github-git-credentials.d.ts +9 -0
  83. package/dist/pr-review-credentials.d.ts +6 -0
  84. package/dist/sandbox-archive-storage.d.ts +45 -0
  85. package/dist/sandbox-resume.d.ts +65 -6
  86. package/dist/sandbox-routing.d.ts +38 -2
  87. package/dist/scheduled-alert-occurrence.d.ts +33 -0
  88. package/dist/workflow-bundle.js +87 -28
  89. package/dist/workflows/activities.d.ts +1 -0
  90. package/dist/workflows/automations.d.ts +6 -0
  91. package/dist/workflows/document-index.d.ts +1 -0
  92. package/dist/workflows.d.ts +1 -0
  93. package/package.json +17 -17
  94. package/src/activities/agent-turn/admission.ts +292 -0
  95. package/src/activities/agent-turn/agent-build.ts +802 -0
  96. package/src/activities/agent-turn/claim.ts +590 -0
  97. package/src/activities/agent-turn/codex-capacity.ts +720 -0
  98. package/src/activities/agent-turn/codex.ts +94 -0
  99. package/src/activities/agent-turn/compaction-prep.ts +722 -0
  100. package/src/activities/agent-turn/credential-leases.ts +240 -0
  101. package/src/activities/agent-turn/errors.ts +1068 -0
  102. package/src/activities/agent-turn/failure-settlement.ts +1436 -0
  103. package/src/activities/agent-turn/file-resources.ts +227 -0
  104. package/src/activities/agent-turn/finalization.ts +754 -0
  105. package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
  106. package/src/activities/agent-turn/governance-model.ts +465 -0
  107. package/src/activities/agent-turn/history-sink.ts +139 -0
  108. package/src/activities/agent-turn/history.ts +200 -0
  109. package/src/activities/agent-turn/media-artifacts.ts +359 -0
  110. package/src/activities/agent-turn/model-usage.ts +823 -0
  111. package/src/activities/agent-turn/quiescence.ts +353 -0
  112. package/src/activities/agent-turn/run-credentials.ts +761 -0
  113. package/src/activities/agent-turn/run.ts +1530 -0
  114. package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
  115. package/src/activities/agent-turn/sandbox-provision.ts +569 -0
  116. package/src/activities/agent-turn/sandbox-route.ts +358 -0
  117. package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
  118. package/src/activities/agent-turn/stream-attempt.ts +1670 -0
  119. package/src/activities/agent-turn/tool-environment.ts +729 -0
  120. package/src/activities/agent-turn/tool-policy.ts +216 -0
  121. package/src/activities/agent-turn/tool-result-spill.ts +265 -0
  122. package/src/activities/agent-turn/turn-context.ts +281 -0
  123. package/src/activities/agent-turn/xai-capacity.ts +229 -0
  124. package/src/activities/agent-turn.ts +12 -12701
  125. package/src/activities/api-integrations.ts +3 -0
  126. package/src/activities/automations.ts +214 -0
  127. package/src/activities/codemode-dispatcher.ts +12 -1
  128. package/src/activities/common.ts +1 -5
  129. package/src/activities/connector-attachments.ts +168 -0
  130. package/src/activities/context-compaction.ts +18 -0
  131. package/src/activities/environment.ts +151 -55
  132. package/src/activities/generated-images.ts +26 -11
  133. package/src/activities/git-credential-renewal.ts +13 -3
  134. package/src/activities/goals.ts +85 -19
  135. package/src/activities/google-drive-provider.ts +178 -0
  136. package/src/activities/google-drive-publication.ts +147 -49
  137. package/src/activities/incident-telemetry-authority.ts +327 -0
  138. package/src/activities/incident-telemetry-preflight.ts +317 -0
  139. package/src/activities/knowledge-source-sync.ts +178 -37
  140. package/src/activities/mcp-credentials.ts +191 -17
  141. package/src/activities/open-suffix-resume.ts +422 -0
  142. package/src/activities/packs.ts +12 -60
  143. package/src/activities/parent-wake.ts +64 -14
  144. package/src/activities/retained-screenshot-reaper.ts +6 -3
  145. package/src/activities/retained-screenshots.ts +15 -47
  146. package/src/activities/rig-verification.ts +1 -1
  147. package/src/activities/run-input.ts +102 -142
  148. package/src/activities/sandbox-images.ts +7 -166
  149. package/src/activities/sandbox-lease.ts +406 -15
  150. package/src/activities/scheduled-tasks.ts +2103 -411
  151. package/src/activities/session-state.ts +10 -2
  152. package/src/activities/types.ts +32 -2
  153. package/src/activities/video-media-validation.ts +3 -3
  154. package/src/activities/video-output-retention.ts +2 -57
  155. package/src/activities/xai-auth.ts +8 -0
  156. package/src/activities-control.ts +2 -0
  157. package/src/activities.ts +3 -0
  158. package/src/activity-services.ts +13 -4
  159. package/src/connection-provider-use.ts +36 -0
  160. package/src/editable-artifact-materializer-subprocess.ts +25 -14
  161. package/src/editable-artifact-materializer.ts +29 -18
  162. package/src/github-rest-mcp.ts +713 -0
  163. package/src/http.ts +16 -6
  164. package/src/index.ts +40 -5
  165. package/src/knowledge-source-sync-activity-policy.ts +2 -0
  166. package/src/model-context-contributions.ts +33 -10
  167. package/src/observability-metrics.ts +357 -0
  168. package/src/opensandbox-kubernetes-inventory.ts +292 -0
  169. package/src/personal-github-git-credentials.ts +331 -0
  170. package/src/pr-review-credentials.ts +163 -0
  171. package/src/sandbox-archive-storage.ts +159 -0
  172. package/src/sandbox-resume.ts +449 -63
  173. package/src/sandbox-routing.ts +314 -16
  174. package/src/scheduled-alert-occurrence.ts +176 -0
  175. package/src/workflows/activities.ts +18 -2
  176. package/src/workflows/automations.ts +11 -0
  177. package/src/workflows/session.ts +13 -3
  178. package/src/workflows.ts +1 -0
  179. package/dist/activities-control-BJSDRSIP.js.map +0 -1
  180. package/dist/activities-turn-J5YO7XXX.js +0 -17243
  181. package/dist/activities-turn-J5YO7XXX.js.map +0 -1
  182. package/dist/chunk-N5TJIFCE.js.map +0 -1
  183. package/dist/chunk-P6Z2SLAA.js +0 -3092
  184. package/dist/chunk-P6Z2SLAA.js.map +0 -1
@@ -0,0 +1,1670 @@
1
+ import {
2
+ getSessionEvent,
3
+ getHumanInputResumeForEvent,
4
+ getSessionHumanInputRequest,
5
+ getWorkspace,
6
+ recordUsageEvent,
7
+ registerPendingSessionToolCall,
8
+ recordPendingSessionToolCallResult,
9
+ attachOpenSuffixToPendingToolCalls,
10
+ clearDurablePendingSessionToolCalls,
11
+ isSessionCompactionRequested,
12
+ nextSessionHistoryPosition,
13
+ } from "@opengeni/db";
14
+ import {
15
+ normalizeModelCallUsage,
16
+ normalizeSdkEvent,
17
+ extractOpenSuffixFromRunState,
18
+ assertOpenSuffixResumable,
19
+ interruptionKindForCallItem,
20
+ releaseMcpResultCustomDataFromSdkEvent,
21
+ findCompactionNeededError,
22
+ withRunCredentialsSession,
23
+ runOwnedSandboxSetup,
24
+ type SandboxFileDownload,
25
+ type OpenGeniRuntime,
26
+ type HistoryProviderApi,
27
+ type GitCredentialTokenWriterSession,
28
+ type NormalizedRunCredentialMaterial,
29
+ type RunCredentialCommandSession,
30
+ type CodemodeTokenWriterSession,
31
+ } from "@opengeni/runtime";
32
+ import { type Settings } from "@opengeni/config";
33
+ import { maybeCompactContext, settleFailedContextCompactionLandmark } from "../context-compaction";
34
+ import { TurnAttemptFencedError } from "../turn-attempt-fenced";
35
+ import { type MintedRunGitCredentials } from "../environment";
36
+ import { withFirstPartyTools } from "../goals";
37
+ import { turnInput } from "../run-input";
38
+ import { createRuntimeBatcher, currentActivityContext, nextStreamEvent } from "../streaming";
39
+ import type {
40
+ TurnActivityServices as ActivityServices,
41
+ RunAgentTurnInput,
42
+ RunAgentTurnResult,
43
+ } from "../types";
44
+ import { type ResumedTurnSandbox } from "../../sandbox-resume";
45
+ import {
46
+ modelCallAccountContext,
47
+ recordBatchFlush,
48
+ recordContextCompaction,
49
+ recordModelInputTokens,
50
+ recordTurnStartupPhase,
51
+ recordTurnWorkerPreparationTotal,
52
+ StreamTimingMetrics,
53
+ turnLifecycleMetricsFor,
54
+ } from "../../observability-metrics";
55
+ import {
56
+ compactRetainedScreenshotHistory,
57
+ sdkEventContainsInlineImage,
58
+ retainComputerScreenshot,
59
+ typedScreenshotFromSdkEvent,
60
+ unavailableRetainedSessionImage,
61
+ } from "../retained-screenshots";
62
+ import {
63
+ compactGeneratedImageSdkEvent,
64
+ generatedImageFromSdkEvent,
65
+ isCompletedGeneratedImageSdkEvent,
66
+ } from "../generated-images";
67
+ import { ToolResultSpill } from "./tool-result-spill";
68
+ import { createTurnCredentialLeases } from "./credential-leases";
69
+ import { createTurnMediaArtifacts } from "./media-artifacts";
70
+ import { createTurnHistorySink } from "./history-sink";
71
+ import {
72
+ interruptionCallIdsFromPause,
73
+ settleOpenSuffixResumeIfNeeded,
74
+ } from "../open-suffix-resume";
75
+ import {
76
+ OPEN_SUFFIX_RUN_STATE_BLOB,
77
+ resolveWorkspaceAgentHumanInputEnabled,
78
+ type RetainedArtifactMetadata,
79
+ type SessionEvent,
80
+ } from "@opengeni/contracts";
81
+ import { createModelCheckpointMemoryCollector } from "../../model-checkpoint-memory-collector";
82
+
83
+ import {
84
+ assertWorkspaceHumanInputAllowed,
85
+ stableHumanInputRequestId,
86
+ stableInteractionInterventionId,
87
+ stableInteractionInterventionOperationId,
88
+ BudgetExhaustedError,
89
+ ensureRunAllowed,
90
+ } from "./admission";
91
+ import {
92
+ compactionFailureReason,
93
+ safeErrorDiagnostic,
94
+ compactionFailureReasonFromError,
95
+ isCompactionSummaryFailure,
96
+ shouldRecoverCompactionProviderFailure,
97
+ classifyContextWindowOverflowError,
98
+ } from "./errors";
99
+ import {
100
+ pendingToolCallFromSdkEvent,
101
+ toolCallProducesRetainableSessionImage,
102
+ completedToolCallFromSdkEvent,
103
+ } from "./history";
104
+ import {
105
+ modelUsageSourceKey,
106
+ recordCompletedModelCallBeforeOwnershipFences,
107
+ TurnEventPublisher,
108
+ createModelResponseEventState,
109
+ modelResponseContextSignal,
110
+ assertModelResponseLatencyMode,
111
+ processModelResponseTerminalEvent,
112
+ emitModelCallUsage,
113
+ recordModelUsageAndDebitCredits,
114
+ recordAuthoritativeModelCallFact,
115
+ } from "./model-usage";
116
+ import { waitForTurnStreamCleanup } from "./quiescence";
117
+ import { waitForTurnOperation } from "./sandbox-provision";
118
+
119
+ import type { CompactionSummarizer } from "../context-compaction";
120
+ import type { TurnExecutionPolicyV1 } from "@opengeni/contracts";
121
+ import type { BoundRunCredentialResolver } from "../run-credentials";
122
+ import type { ModelHistoryAttachmentProjector } from "../run-input";
123
+ import type {
124
+ AttemptIdentityState,
125
+ BillingState,
126
+ ClaimedResult,
127
+ EventingState,
128
+ ProviderTurnState,
129
+ SandboxRuntimeState,
130
+ TurnControlState,
131
+ } from "./turn-context";
132
+
133
+ export type TurnStreamAttemptDeps = {
134
+ input: RunAgentTurnInput;
135
+ settings: Settings;
136
+ db: ActivityServices["db"];
137
+ runtime: ActivityServices["runtime"];
138
+ objectStorage: ActivityServices["objectStorage"];
139
+ observability: ActivityServices["observability"];
140
+ entitlements: ActivityServices["entitlements"];
141
+ startVideoGenerationWorkflow: ActivityServices["startVideoGenerationWorkflow"];
142
+ cancellationSignal: AbortSignal | undefined;
143
+ runtimeCancellationSignal: AbortSignal;
144
+ activityContext: ReturnType<typeof currentActivityContext>;
145
+ dispatchId: string;
146
+ control: TurnControlState;
147
+ attempt: AttemptIdentityState;
148
+ billingState: BillingState;
149
+ sandboxState: SandboxRuntimeState;
150
+ eventing: EventingState & {
151
+ publish: TurnEventPublisher;
152
+ settle: NonNullable<EventingState["settle"]>;
153
+ };
154
+ providerTurn: ProviderTurnState;
155
+ leases: ReturnType<typeof createTurnCredentialLeases>;
156
+ historySink: ReturnType<typeof createTurnHistorySink>;
157
+ media: ReturnType<typeof createTurnMediaArtifacts>;
158
+ toolResultSpill: ToolResultSpill;
159
+ claimedResult: ClaimedResult;
160
+ flushRuntimeBatcher: () => Promise<void>;
161
+ finalizeTurnOpStreamOps: () => Promise<void>;
162
+ runWorkspaceMutationForSandbox: <T>(
163
+ sandbox: ResumedTurnSandbox,
164
+ operation: string,
165
+ mutation: () => Promise<T>,
166
+ observePhase?: (measurement: {
167
+ phase: "admission" | "provider" | "settlement" | "snapshot_wait";
168
+ outcome: "completed" | "failed";
169
+ durationSeconds: number;
170
+ }) => void,
171
+ ) => Promise<T>;
172
+ throwIfWorkerShuttingDown: () => void;
173
+ throwIfTurnCancelled: () => void;
174
+ setLastInputTokensFenced: (lastInputTokens: number | null) => Promise<void>;
175
+ attachGitCredentialRenewal: (
176
+ tokenSession: GitCredentialTokenWriterSession,
177
+ initial: MintedRunGitCredentials | undefined,
178
+ initialSandbox?: ResumedTurnSandbox,
179
+ ) => Promise<void>;
180
+ attachCodemodeTokenRenewal: (
181
+ tokenSession?: CodemodeTokenWriterSession,
182
+ initialExpiresAt?: Date,
183
+ initialSandbox?: ResumedTurnSandbox,
184
+ ) => Promise<void>;
185
+ attachRunCredentialRenewal: (
186
+ credentialSession: RunCredentialCommandSession,
187
+ initialMaterial: NormalizedRunCredentialMaterial | null,
188
+ initialSandbox?: ResumedTurnSandbox,
189
+ ) => Promise<void>;
190
+ withProviderRequestContext: <T>(fn: () => Promise<T>) => Promise<T>;
191
+ publishCompactionLiveEvents: (events: SessionEvent[]) => Promise<void>;
192
+ publishCompactionOutcomeEvents: (events: SessionEvent[]) => Promise<void>;
193
+ recordCompanyBrainContributionReceiptOnce: () => void;
194
+ modelCheckpointMemoryCollector: ReturnType<typeof createModelCheckpointMemoryCollector>;
195
+ claimedModelUsageSourceKeys: Set<string>;
196
+ emittedModelUsageSourceKeys: Set<string>;
197
+ modelUsageDispatchId: string;
198
+ workerPreparationStartedAt: number;
199
+ fileDownloadsMaterializedForRun: boolean;
200
+ unavailableSandboxFilesNote: string | undefined;
201
+ runCredentialsNote: string | undefined;
202
+ mcpAvailabilityNote: string | undefined;
203
+ fileAuthoritySubjectId: string | null;
204
+ activeSandboxBackend: Settings["sandboxBackend"] | undefined;
205
+ groupBoxBackend: Settings["sandboxBackend"];
206
+ turnExecutionPolicy: TurnExecutionPolicyV1;
207
+ turn: { executionGeneration: number; model: string };
208
+ trigger: NonNullable<Awaited<ReturnType<typeof getSessionEvent>>>;
209
+ humanInputResume: Awaited<ReturnType<typeof getHumanInputResumeForEvent>>;
210
+ agent: ReturnType<ActivityServices["runtime"]["buildAgent"]>;
211
+ resolvedModel: ReturnType<ActivityServices["runtime"]["resolveTurnModel"]>;
212
+ providerApi: HistoryProviderApi;
213
+ runSettings: Settings;
214
+ turnTools: ReturnType<typeof withFirstPartyTools>;
215
+ compactSummarizer: CompactionSummarizer;
216
+ compactionModelHistoryProjector: (
217
+ items: Array<Record<string, unknown>>,
218
+ ) => Promise<Array<Record<string, unknown>>>;
219
+ generatedImageHistoryProjector: (
220
+ items: Array<Record<string, unknown>>,
221
+ ) => Promise<Array<Record<string, unknown>>>;
222
+ modelHistoryProjector: ModelHistoryAttachmentProjector;
223
+ compactionModeOptions: NonNullable<Parameters<typeof maybeCompactContext>[5]>;
224
+ initialRunCredentialMaterial: NormalizedRunCredentialMaterial | null;
225
+ initialGitCredentials: MintedRunGitCredentials | undefined;
226
+ sandboxEnvironment: Record<string, string>;
227
+ sandboxCodemodeToken: string | undefined;
228
+ sandboxCodemodeTokenExpiresAt: Date | string | number | null | undefined;
229
+ fileResourceDownloads: SandboxFileDownload[];
230
+ runCredentialResolver: BoundRunCredentialResolver | null;
231
+ videoGenerationAcceptancesByCallId: Map<string, { operationId: string; requestDigest: string }>;
232
+ };
233
+
234
+ export async function runTurnStreamAttempt(
235
+ deps: TurnStreamAttemptDeps,
236
+ ): Promise<RunAgentTurnResult> {
237
+ const {
238
+ input,
239
+ settings,
240
+ db,
241
+ runtime,
242
+ objectStorage,
243
+ observability,
244
+ entitlements,
245
+ startVideoGenerationWorkflow,
246
+ cancellationSignal,
247
+ runtimeCancellationSignal,
248
+ activityContext,
249
+ dispatchId,
250
+ control,
251
+ attempt,
252
+ billingState,
253
+ sandboxState,
254
+ eventing,
255
+ providerTurn,
256
+ leases,
257
+ historySink,
258
+ media,
259
+ toolResultSpill,
260
+ claimedResult,
261
+ flushRuntimeBatcher,
262
+ finalizeTurnOpStreamOps,
263
+ runWorkspaceMutationForSandbox,
264
+ throwIfWorkerShuttingDown,
265
+ throwIfTurnCancelled,
266
+ setLastInputTokensFenced,
267
+ attachGitCredentialRenewal,
268
+ attachCodemodeTokenRenewal,
269
+ attachRunCredentialRenewal,
270
+ withProviderRequestContext,
271
+ publishCompactionLiveEvents,
272
+ publishCompactionOutcomeEvents,
273
+ recordCompanyBrainContributionReceiptOnce,
274
+ modelCheckpointMemoryCollector,
275
+ claimedModelUsageSourceKeys,
276
+ emittedModelUsageSourceKeys,
277
+ modelUsageDispatchId,
278
+ workerPreparationStartedAt,
279
+ fileDownloadsMaterializedForRun,
280
+ unavailableSandboxFilesNote,
281
+ runCredentialsNote,
282
+ mcpAvailabilityNote,
283
+ fileAuthoritySubjectId,
284
+ activeSandboxBackend,
285
+ groupBoxBackend,
286
+ turnExecutionPolicy,
287
+ turn,
288
+ trigger,
289
+ humanInputResume,
290
+ agent,
291
+ resolvedModel,
292
+ providerApi,
293
+ runSettings,
294
+ turnTools,
295
+ compactSummarizer,
296
+ compactionModelHistoryProjector,
297
+ generatedImageHistoryProjector,
298
+ modelHistoryProjector,
299
+ compactionModeOptions,
300
+ initialRunCredentialMaterial,
301
+ initialGitCredentials,
302
+ sandboxEnvironment,
303
+ sandboxCodemodeToken,
304
+ sandboxCodemodeTokenExpiresAt,
305
+ fileResourceDownloads,
306
+ runCredentialResolver,
307
+ videoGenerationAcceptancesByCallId,
308
+ } = deps;
309
+
310
+ // Build one attempt from canonical history. The provider adapter creates
311
+ // only the temporary wire view required by this turn's selected API;
312
+ // subscription identity never edits or filters durable history.
313
+ const activeTurnId = attempt.turnId;
314
+ if (!activeTurnId) {
315
+ throw new Error("Turn id was not initialized");
316
+ }
317
+ let runInput: Awaited<ReturnType<typeof turnInput>>["input"] | null = null;
318
+ const prepareRunAttemptInput = async () => {
319
+ const historyPreparationStartedAt = performance.now();
320
+ let historyPreparationOutcome: "completed" | "failed" = "completed";
321
+ let preparedHistoryCount: number | null = null;
322
+ try {
323
+ const prepared = await turnInput(db, runtime, agent, trigger, {
324
+ turnId: activeTurnId,
325
+ fileAuthority: {
326
+ accountId: input.accountId,
327
+ subjectId: fileAuthoritySubjectId,
328
+ },
329
+ recovering: turn.executionGeneration > 1,
330
+ ...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
331
+ ...(runCredentialsNote ? { runCredentialsNote } : {}),
332
+ ...(mcpAvailabilityNote ? { mcpAvailabilityNote } : {}),
333
+ providerApi,
334
+ projectCanonicalHistory: generatedImageHistoryProjector,
335
+ materializeModelHistory: media.materializeScreenshotHistory,
336
+ projectModelHistory: modelHistoryProjector,
337
+ onPreparationPhase: (measurement) => {
338
+ recordTurnStartupPhase(observability, {
339
+ phase: `history_${measurement.phase}`,
340
+ provider: turnExecutionPolicy.providerId,
341
+ backend: activeSandboxBackend ?? groupBoxBackend,
342
+ outcome: measurement.outcome,
343
+ durationSeconds: measurement.durationSeconds,
344
+ });
345
+ },
346
+ });
347
+ runInput = prepared.input;
348
+ historySink.providerArtifactCandidates = prepared.providerArtifactCandidates;
349
+ // Slice index = the length of the model-facing (active) history this turn
350
+ // is seeded from; new items beyond it (the trigger message + this turn's
351
+ // generated items) are the ones to persist. After a compaction this is the
352
+ // short [summary, ...tail] active set, NOT the total row count. The
353
+ // absolute write position is tracked separately (next whole number past
354
+ // the max existing position) because the fractional summary row means
355
+ // total rows no longer equal max(position)+1. Pre-compaction both reduce to
356
+ // the old total-count value, so the common path is unchanged.
357
+ //
358
+ // CRITICAL: seed from the structurally repaired active-row length, not the raw active
359
+ // count. `prepareRunInput` builds `state.history` from
360
+ // `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
361
+ // rows (a legacy orphan/dangling pair), the in-memory history this turn
362
+ // starts from is K shorter than the raw row count. The reconcile slices the
363
+ // repaired `state.history` off `historySink.persistedHistoryCount`; seeding it from
364
+ // the raw count (K too high) skips K genuinely-new items, and a
365
+ // `function_call` left in that skipped region can later have its
366
+ // `function_call_result` persisted alone — the orphan that 400s on replay
367
+ // and bricks the session (issue-61). The repaired seed is already
368
+ // orphan-free, so it is a stable prefix of the re-sanitized history and the
369
+ // slice begins exactly at the first genuinely-new item.
370
+ // prepareInput already sanitized the exact durable prefix represented
371
+ // by state.history. Carry its count forward instead of loading and
372
+ // retaining the full active transcript a second time beside runInput.
373
+ historySink.persistedHistoryCount = prepared.persistedHistoryCount;
374
+ preparedHistoryCount = prepared.persistedHistoryCount;
375
+ const historyPositionStartedAt = performance.now();
376
+ let historyPositionOutcome: "completed" | "failed" = "completed";
377
+ try {
378
+ historySink.nextHistoryPosition = await nextSessionHistoryPosition(
379
+ db,
380
+ input.workspaceId,
381
+ input.sessionId,
382
+ );
383
+ } catch (error) {
384
+ historyPositionOutcome = "failed";
385
+ throw error;
386
+ } finally {
387
+ recordTurnStartupPhase(observability, {
388
+ phase: "history_position_load",
389
+ provider: turnExecutionPolicy.providerId,
390
+ backend: activeSandboxBackend ?? groupBoxBackend,
391
+ outcome: historyPositionOutcome,
392
+ durationSeconds: (performance.now() - historyPositionStartedAt) / 1_000,
393
+ });
394
+ }
395
+ } catch (error) {
396
+ historyPreparationOutcome = "failed";
397
+ throw error;
398
+ } finally {
399
+ recordTurnStartupPhase(observability, {
400
+ phase: "history_preparation",
401
+ provider: turnExecutionPolicy.providerId,
402
+ backend: activeSandboxBackend ?? groupBoxBackend,
403
+ outcome: historyPreparationOutcome,
404
+ durationSeconds: (performance.now() - historyPreparationStartedAt) / 1_000,
405
+ count: preparedHistoryCount,
406
+ });
407
+ }
408
+ };
409
+
410
+ const forceContextCompaction = async (
411
+ triggerLabel: "overflow" | "proactive" | "operator",
412
+ recoverySignalTokens: number | null,
413
+ ) => {
414
+ const outcome = await waitForTurnOperation(
415
+ maybeCompactContext(
416
+ db,
417
+ eventing.modelRunSettings,
418
+ {
419
+ accountId: input.accountId,
420
+ workspaceId: input.workspaceId,
421
+ sessionId: input.sessionId,
422
+ turnId: activeTurnId,
423
+ executionGeneration: attempt.executionGeneration,
424
+ attemptId: input.attemptId,
425
+ },
426
+ // Never reuse the persisted prior-turn signal for recovery. Proactive
427
+ // guards provide their exact current provider-based signal; provider
428
+ // overflows do not, so their diagnostic signal stays unknown/zero.
429
+ // Forced recovery proves progress separately by comparing the
430
+ // replacement with the current active-history estimate.
431
+ recoverySignalTokens,
432
+ compactSummarizer,
433
+ {
434
+ force: true,
435
+ ...(triggerLabel === "operator" ? { clearRequestedCompaction: true } : {}),
436
+ trigger: triggerLabel,
437
+ materializeHistory: media.materializeScreenshotHistory,
438
+ projectModelInput: compactionModelHistoryProjector,
439
+ ...compactionModeOptions,
440
+ },
441
+ ),
442
+ cancellationSignal,
443
+ undefined,
444
+ );
445
+ if (outcome.events.length > 0) {
446
+ if (outcome.compacted) {
447
+ recordContextCompaction(observability, triggerLabel);
448
+ }
449
+ await publishCompactionOutcomeEvents(outcome.events);
450
+ }
451
+ return outcome;
452
+ };
453
+
454
+ // Keep response identity across every stream attempt in this
455
+ // activity. Context compaction retries the same logical dispatch by
456
+ // calling runStreamAttempt again; resetting this state there would reuse
457
+ // the first no-response-ID fallback key and suppress a real model call.
458
+ const modelResponseState = createModelResponseEventState(claimedModelUsageSourceKeys);
459
+ let workerPreparationTotalRecorded = false;
460
+ const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
461
+ if (!runInput) {
462
+ throw new Error("Run input was not prepared");
463
+ }
464
+ eventing.stream = undefined;
465
+ eventing.batcher = null;
466
+ // The SDK emits every processed call item for one model response before
467
+ // it emits any result for that response. Keep that response-local batch
468
+ // in memory so an orphan from an older response cannot pin later stable
469
+ // calls. Durable recovery remains call/result based and needs no batch
470
+ // schema or compatibility state.
471
+ let currentToolBatchCallIds = new Set<string>();
472
+ let currentToolBatchCompletedCallIds = new Set<string>();
473
+ let streamSawPerResponseUsage = false;
474
+ // Actual input tokens of the most recent model response this turn; the
475
+ // pre-read trigger for the NEXT turn. Persisted at every turn-end path.
476
+ throwIfWorkerShuttingDown();
477
+ throwIfTurnCancelled();
478
+ const streamProvider = resolvedModel?.provider.id ?? settings.openaiProvider ?? "openai";
479
+ const providerPublishesNativeRequestEvents =
480
+ resolvedModel?.provider.kind === "codex-subscription" ||
481
+ resolvedModel?.provider.kind === "xai-subscription";
482
+ let fallbackProviderRequestStartedAt: number | null = null;
483
+ const recordFallbackProviderDispatchAtWire = async (): Promise<void> => {
484
+ if (
485
+ providerPublishesNativeRequestEvents ||
486
+ eventing.firstModelRequestPreparationRecorded ||
487
+ eventing.firstModelRequestPreparationStartedAt === null
488
+ ) {
489
+ return;
490
+ }
491
+ sandboxState.firstProviderRequestStarted = true;
492
+ eventing.firstModelRequestPreparationRecorded = true;
493
+ const preparationDurationMs = Math.max(
494
+ 0,
495
+ performance.now() - eventing.firstModelRequestPreparationStartedAt,
496
+ );
497
+ recordTurnStartupPhase(observability, {
498
+ phase: "model_request_preparation",
499
+ provider: turnExecutionPolicy.providerId,
500
+ backend: activeSandboxBackend ?? groupBoxBackend,
501
+ outcome: "completed",
502
+ durationSeconds: preparationDurationMs / 1_000,
503
+ count: turnTools.length,
504
+ });
505
+ const auditStartedAt = performance.now();
506
+ let auditOutcome: "completed" | "failed" = "completed";
507
+ try {
508
+ await eventing.publish!([
509
+ {
510
+ type: "turn.startup.phase.completed",
511
+ payload: {
512
+ phase: "model_preparation",
513
+ durationMs: Math.round(preparationDurationMs),
514
+ },
515
+ },
516
+ {
517
+ type: "agent.model.request",
518
+ payload: {
519
+ phase: "started",
520
+ provider: streamProvider,
521
+ turnId: activeTurnId,
522
+ attemptId: input.attemptId,
523
+ dispatchId,
524
+ executionGeneration: attempt.executionGeneration,
525
+ },
526
+ },
527
+ ]);
528
+ fallbackProviderRequestStartedAt = performance.now();
529
+ } catch (error) {
530
+ auditOutcome = "failed";
531
+ throw error;
532
+ } finally {
533
+ recordTurnStartupPhase(observability, {
534
+ phase: "model_request_audit",
535
+ provider: turnExecutionPolicy.providerId,
536
+ backend: activeSandboxBackend ?? groupBoxBackend,
537
+ outcome: auditOutcome,
538
+ durationSeconds: (performance.now() - auditStartedAt) / 1_000,
539
+ count: turnTools.length,
540
+ });
541
+ }
542
+ };
543
+ const ownedEstablished =
544
+ sandboxState.resolvedSandbox?.established ?? sandboxState.lazyOwnedSandbox;
545
+ const runStreamOnce = async (): ReturnType<OpenGeniRuntime["runStream"]> => {
546
+ // Eager owned sessions must settle the exact platform-setup provider
547
+ // promise before the long model stream starts; otherwise one admission
548
+ // would remain in flight for the entire turn and suppress every
549
+ // heartbeat capture. Lazy setup already runs under the same wrapper in
550
+ // its first-operation provisioner above.
551
+ if (sandboxState.resolvedSandbox && !sandboxState.lazyOwnedSandbox && ownedEstablished) {
552
+ const ownedSandboxSetupStartedAt = performance.now();
553
+ let ownedSandboxSetupOutcome: "completed" | "failed" = "completed";
554
+ try {
555
+ const eagerSetupSession = sandboxState.setupBoxSession ?? ownedEstablished.session;
556
+ // `deferredSetup: true` below tells the runtime that the worker owns
557
+ // platform setup, so the runtime intentionally skips its credential
558
+ // session callback. Materialize host-managed run credentials here
559
+ // before any setup command (including provider login hooks), then
560
+ // decorate setup commands so they source the active generation.
561
+ await attachRunCredentialRenewal(
562
+ eagerSetupSession as RunCredentialCommandSession,
563
+ initialRunCredentialMaterial,
564
+ sandboxState.resolvedSandbox,
565
+ );
566
+ const eagerCredentialSetupSession = initialRunCredentialMaterial
567
+ ? withRunCredentialsSession(eagerSetupSession as object, input.sessionId)
568
+ : eagerSetupSession;
569
+ await runWorkspaceMutationForSandbox(
570
+ sandboxState.resolvedSandbox,
571
+ "eagerOwnedSandboxSetup",
572
+ async () =>
573
+ await runOwnedSandboxSetup(
574
+ agent,
575
+ ownedEstablished.session as never,
576
+ eagerCredentialSetupSession as never,
577
+ {
578
+ settings: runSettings,
579
+ environment: sandboxEnvironment,
580
+ preparedInput: runInput!,
581
+ ...(fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {}),
582
+ onRuntimeEvent: async (event) => {
583
+ await leases.renewServing("runtime_event");
584
+ if (leases.servingLost()) {
585
+ throw new Error("Provider credential lease expired during sandbox setup");
586
+ }
587
+ await eventing.publish!([{ type: event.type, payload: event.payload }], true);
588
+ },
589
+ ...(eventing.toolCancellationFenceRef.current
590
+ ? {
591
+ commandRunner:
592
+ eventing.toolCancellationFenceRef.current.runSandboxCommand.bind(
593
+ eventing.toolCancellationFenceRef.current,
594
+ ),
595
+ }
596
+ : {}),
597
+ },
598
+ ),
599
+ );
600
+ await attachGitCredentialRenewal(
601
+ eagerSetupSession as GitCredentialTokenWriterSession,
602
+ initialGitCredentials,
603
+ );
604
+ } catch (error) {
605
+ ownedSandboxSetupOutcome = "failed";
606
+ throw error;
607
+ } finally {
608
+ recordTurnStartupPhase(observability, {
609
+ phase: "owned_sandbox_setup",
610
+ provider: turnExecutionPolicy.providerId,
611
+ backend: activeSandboxBackend ?? groupBoxBackend,
612
+ outcome: ownedSandboxSetupOutcome,
613
+ durationSeconds: (performance.now() - ownedSandboxSetupStartedAt) / 1_000,
614
+ count: fileResourceDownloads.length,
615
+ });
616
+ }
617
+ }
618
+ // Conservative request boundary: once control enters runStream, the
619
+ // provider may have accepted work even if no response/event follows.
620
+ // Escaped MCP setup timeouts are automatically recoverable only
621
+ // before this line.
622
+ recordCompanyBrainContributionReceiptOnce();
623
+ if (!eventing.firstModelRequestPreparationRecorded) {
624
+ await eventing.publish!([
625
+ {
626
+ type: "turn.startup.phase.started",
627
+ payload: { phase: "model_preparation" },
628
+ },
629
+ ]);
630
+ eventing.firstModelRequestPreparationStartedAt = performance.now();
631
+ eventing.firstModelRequestCheckpointAt = eventing.firstModelRequestPreparationStartedAt;
632
+ }
633
+ const providerDispatchStartedAt = performance.now();
634
+ let providerDispatchOutcome: "completed" | "failed" = "completed";
635
+ try {
636
+ // This histogram describes worker preparation until the first entry
637
+ // into the runtime. Lazy SDK request preparation and the durable
638
+ // model-request audit happen after this boundary and have their own
639
+ // phase metrics. Context-compaction recovery re-enters
640
+ // runStreamAttempt; recording again would fold the prior request and
641
+ // compaction into a bogus second startup sample.
642
+ if (!workerPreparationTotalRecorded) {
643
+ workerPreparationTotalRecorded = true;
644
+ recordTurnWorkerPreparationTotal(observability, {
645
+ provider: turnExecutionPolicy.providerId,
646
+ backend: activeSandboxBackend ?? groupBoxBackend,
647
+ outcome: "completed",
648
+ durationSeconds: (performance.now() - workerPreparationStartedAt) / 1_000,
649
+ });
650
+ }
651
+ attempt.modelRequestStarted = true;
652
+ return await runtime.runStream(agent, runInput!, eventing.modelRunSettings, {
653
+ signal: runtimeCancellationSignal,
654
+ sandboxEnvironment,
655
+ onRuntimeEvent: async (event) => {
656
+ await leases.renewServing("runtime_event");
657
+ if (leases.servingLost()) {
658
+ throw new Error("Provider credential lease expired during the active turn");
659
+ }
660
+ await eventing.publish!([{ type: event.type, payload: event.payload }], true);
661
+ },
662
+ // P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
663
+ // keystone). Absent when the flag is off -> legacy build-and-discard.
664
+ ...(ownedEstablished
665
+ ? {
666
+ ownedSandbox: {
667
+ client: ownedEstablished.client,
668
+ session: ownedEstablished.session,
669
+ ...(sandboxState.resolvedSandbox?.established.sessionState
670
+ ? {
671
+ sessionState: sandboxState.resolvedSandbox.established.sessionState,
672
+ }
673
+ : {}),
674
+ // Pin platform setup (hooks + file materialization) to the un-proxied
675
+ // established box — never through the routing proxy, which would
676
+ // re-route those execs onto a machine swapped in mid-turn.
677
+ ...(sandboxState.setupBoxSession
678
+ ? { setupSession: sandboxState.setupBoxSession }
679
+ : {}),
680
+ ...(fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {}),
681
+ // Both owned paths execute setup outside runStream: eager just
682
+ // above under its exact admission, lazy in the provisioner.
683
+ deferredSetup: true,
684
+ },
685
+ }
686
+ : {}),
687
+ ...(activeSandboxBackend !== "selfhosted" &&
688
+ sandboxCodemodeToken &&
689
+ sandboxCodemodeTokenExpiresAt &&
690
+ !sandboxState.lazyOwnedSandbox
691
+ ? {
692
+ onCodemodeTokenSessionReady: async (tokenSession: CodemodeTokenWriterSession) => {
693
+ const renewalSession =
694
+ (sandboxState.setupBoxSession as CodemodeTokenWriterSession | null) ??
695
+ tokenSession;
696
+ await attachCodemodeTokenRenewal(renewalSession);
697
+ },
698
+ }
699
+ : {}),
700
+ ...(runCredentialResolver
701
+ ? {
702
+ runCredentialSessionId: input.sessionId,
703
+ ...(!sandboxState.lazyOwnedSandbox
704
+ ? {
705
+ onRunCredentialSessionReady: async (
706
+ credentialSession: RunCredentialCommandSession,
707
+ ) => {
708
+ const pinnedCredentialSession = sandboxState.setupBoxSession
709
+ ? (sandboxState.setupBoxSession as RunCredentialCommandSession)
710
+ : credentialSession;
711
+ await attachRunCredentialRenewal(
712
+ pinnedCredentialSession,
713
+ initialRunCredentialMaterial,
714
+ );
715
+ },
716
+ }
717
+ : {}),
718
+ }
719
+ : {}),
720
+ ...(eventing.modelRunSettings.sandboxBackend !== "none"
721
+ ? {
722
+ onSandboxSessionReady: async (sandboxSession: CodemodeTokenWriterSession) => {
723
+ media.sdkOwnedSandboxSession = sandboxSession;
724
+ for (const receipt of media.generatedImageReceiptsCreatedThisTurn.values()) {
725
+ await media.materializeGeneratedImageInOwnedSdkSession(receipt, sandboxSession);
726
+ }
727
+ await toolResultSpill.materializeDeferred();
728
+ },
729
+ }
730
+ : {}),
731
+ contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
732
+ contextCompactionRequested: () =>
733
+ isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
734
+ onModelPreparationPhase: (measurement) => {
735
+ recordTurnStartupPhase(observability, {
736
+ phase: `model_prepare_${measurement.phase}`,
737
+ provider: turnExecutionPolicy.providerId,
738
+ backend: activeSandboxBackend ?? groupBoxBackend,
739
+ outcome: measurement.outcome,
740
+ durationSeconds: measurement.durationSeconds,
741
+ ...(measurement.count === undefined ? {} : { count: measurement.count }),
742
+ });
743
+ },
744
+ ...(!providerPublishesNativeRequestEvents
745
+ ? {
746
+ onModelTransportStarted: recordFallbackProviderDispatchAtWire,
747
+ }
748
+ : {}),
749
+ ...(eventing.toolCancellationFenceRef.current
750
+ ? {
751
+ turnToolCancellationFence: eventing.toolCancellationFenceRef.current,
752
+ }
753
+ : {}),
754
+ });
755
+ } catch (error) {
756
+ providerDispatchOutcome = "failed";
757
+ throw error;
758
+ } finally {
759
+ recordTurnStartupPhase(observability, {
760
+ phase: "provider_dispatch",
761
+ provider: turnExecutionPolicy.providerId,
762
+ backend: activeSandboxBackend ?? groupBoxBackend,
763
+ outcome: providerDispatchOutcome,
764
+ durationSeconds: (performance.now() - providerDispatchStartedAt) / 1_000,
765
+ });
766
+ }
767
+ };
768
+ if (leases.codex.lost) {
769
+ throw new Error("Codex credential lease expired before the model run");
770
+ }
771
+ if (leases.xai.lost) {
772
+ throw new Error("xAI credential lease expired before the model run");
773
+ }
774
+ eventing.stream = await withProviderRequestContext(runStreamOnce);
775
+ // Bounded provider label for the streaming SLIs — the resolved registry
776
+ // provider id (or the built-in OpenAI/Azure provider), never a raw
777
+ // user-supplied model string.
778
+ const streamTiming = new StreamTimingMetrics(observability, {
779
+ provider: streamProvider,
780
+ });
781
+ eventing.batcher = createRuntimeBatcher(
782
+ async (events) => {
783
+ await eventing.publish!(events);
784
+ },
785
+ {
786
+ onFlush: ({ events, durationSeconds }) =>
787
+ recordBatchFlush(observability, { events, durationSeconds }),
788
+ },
789
+ );
790
+
791
+ const streamBootstrapStartedAt = performance.now();
792
+ let streamBootstrapRecorded = false;
793
+ const recordStreamBootstrap = (): void => {
794
+ if (streamBootstrapRecorded) return;
795
+ streamBootstrapRecorded = true;
796
+ recordTurnStartupPhase(observability, {
797
+ phase: "stream_bootstrap",
798
+ provider: turnExecutionPolicy.providerId,
799
+ backend: activeSandboxBackend ?? groupBoxBackend,
800
+ outcome: "completed",
801
+ durationSeconds: (performance.now() - streamBootstrapStartedAt) / 1_000,
802
+ count: turnTools.length,
803
+ });
804
+ };
805
+ const settleFallbackProviderFirstByte = async (): Promise<void> => {
806
+ if (fallbackProviderRequestStartedAt === null) return;
807
+ const durationMs = Math.max(0, performance.now() - fallbackProviderRequestStartedAt);
808
+ fallbackProviderRequestStartedAt = null;
809
+ await eventing.publish!([
810
+ {
811
+ type: "agent.model.request",
812
+ payload: {
813
+ phase: "first_byte",
814
+ provider: streamProvider,
815
+ durationMs: Math.round(durationMs),
816
+ turnId: activeTurnId,
817
+ attemptId: input.attemptId,
818
+ dispatchId,
819
+ executionGeneration: attempt.executionGeneration,
820
+ },
821
+ },
822
+ ]);
823
+ };
824
+ const iterator = eventing.stream.toStream()[Symbol.asyncIterator]();
825
+ let streamDone = false;
826
+ try {
827
+ while (true) {
828
+ const next = await nextStreamEvent(iterator, activityContext);
829
+ recordStreamBootstrap();
830
+ if (next.done) {
831
+ streamDone = true;
832
+ break;
833
+ }
834
+ await settleFallbackProviderFirstByte();
835
+ let stableToolCallIdsToClear: string[] | null = null;
836
+ let completedCurrentToolBatch = false;
837
+ let retainedScreenshotMetadata: RetainedArtifactMetadata | null = null;
838
+ let normalizedSdkEvents: ReturnType<typeof normalizeSdkEvent> | null = null;
839
+ const generatedImage = generatedImageFromSdkEvent(next.value);
840
+ if (isCompletedGeneratedImageSdkEvent(next.value) && !generatedImage) {
841
+ throw new Error(
842
+ "Completed native image item could not cross the retained-artifact boundary",
843
+ );
844
+ }
845
+ const generatedImageReceipt = generatedImage
846
+ ? await media.retainNativeGeneratedImage(generatedImage)
847
+ : null;
848
+ const responseResult = await processModelResponseTerminalEvent({
849
+ event: next.value,
850
+ state: modelResponseState,
851
+ dispatchId: modelUsageDispatchId,
852
+ settings,
853
+ db,
854
+ observability,
855
+ publish: eventing.publish,
856
+ accountId: input.accountId,
857
+ workspaceId: input.workspaceId,
858
+ sessionId: input.sessionId,
859
+ turnId: activeTurnId,
860
+ turnAttemptId: input.attemptId,
861
+ provider: resolvedModel?.provider.id ?? settings.openaiProvider,
862
+ providerApi: resolvedModel?.provider.api ?? "responses",
863
+ model: turn.model,
864
+ latencyMode: turnExecutionPolicy.latencyMode,
865
+ metricProvider: streamProvider,
866
+ externallyBilled: billingState.isExternallyBilledTurn,
867
+ servingCredentialId: providerTurn.effectiveCodexCredentialId,
868
+ priorSessionCredentialId: providerTurn.priorSessionCodexCredentialId,
869
+ emittedSourceKeys: emittedModelUsageSourceKeys,
870
+ renewLease: () => leases.renewServing("model_usage"),
871
+ leaseLost: leases.servingLost,
872
+ leaseLostMessage: "Provider credential lease expired during the active turn",
873
+ setLastInputTokens: setLastInputTokensFenced,
874
+ contextContributions: eventing.companyBrainContextContributions,
875
+ });
876
+ assertModelResponseLatencyMode({
877
+ event: next.value,
878
+ requested: turnExecutionPolicy.latencyMode,
879
+ model: turn.model,
880
+ ...(resolvedModel?.provider.id ? { providerId: resolvedModel.provider.id } : {}),
881
+ });
882
+ if (responseResult.status === "processed") {
883
+ const rawStreamHistory = (eventing.stream.state as { history?: unknown[] }).history;
884
+ if (Array.isArray(rawStreamHistory)) {
885
+ // The completed image item is normally retained from its own
886
+ // run-item event above. Scan once at the terminal response as
887
+ // a compatibility backstop before reconciliation can persist
888
+ // provider bytes. Never rescan history for every token delta.
889
+ await media.retainNativeGeneratedImagesFromHistory(
890
+ rawStreamHistory as Array<Record<string, unknown>>,
891
+ );
892
+ }
893
+ streamSawPerResponseUsage ||= responseResult.usageReported;
894
+ currentToolBatchCallIds = new Set<string>();
895
+ currentToolBatchCompletedCallIds = new Set<string>();
896
+ await historySink.reconcileConversationTruth();
897
+ turnLifecycleMetricsFor(observability).progress(attempt.turnId!);
898
+ modelCheckpointMemoryCollector.schedule(observability);
899
+ try {
900
+ await ensureRunAllowed(
901
+ settings,
902
+ db,
903
+ input.accountId,
904
+ input.workspaceId,
905
+ billingState.isExternallyBilledTurn,
906
+ entitlements,
907
+ );
908
+ } catch (limitError) {
909
+ // Capture the run state at the boundary so the budget valve in
910
+ // the outer catch can end this segment gracefully with full
911
+ // conversation context preserved for the post-top-up resume.
912
+ let serializedRunState: string | null = null;
913
+ try {
914
+ serializedRunState = media.compactMediaRunState(
915
+ String(eventing.stream.state.toString()),
916
+ );
917
+ } catch {
918
+ serializedRunState = null;
919
+ }
920
+ throw new BudgetExhaustedError(
921
+ limitError instanceof Error ? limitError.message : String(limitError),
922
+ serializedRunState,
923
+ );
924
+ }
925
+ }
926
+ const durableSdkEvent = generatedImageReceipt
927
+ ? compactGeneratedImageSdkEvent(next.value, generatedImageReceipt)
928
+ : next.value;
929
+ const pendingToolCall = pendingToolCallFromSdkEvent(durableSdkEvent);
930
+ if (pendingToolCall) {
931
+ const registered = await registerPendingSessionToolCall(db, {
932
+ accountId: input.accountId,
933
+ workspaceId: input.workspaceId,
934
+ sessionId: input.sessionId,
935
+ turnId: activeTurnId,
936
+ executionGeneration: attempt.executionGeneration,
937
+ attemptId: input.attemptId,
938
+ modelToolOutputTruncationTokens:
939
+ eventing.modelRunSettings.modelToolOutputTruncationTokens,
940
+ callId: pendingToolCall.callId,
941
+ callType: pendingToolCall.callType,
942
+ callItem: pendingToolCall.callItem as Record<string, unknown>,
943
+ });
944
+ if (!registered.accepted) {
945
+ throw new TurnAttemptFencedError(
946
+ "turn attempt ended while recording an in-flight tool call",
947
+ );
948
+ }
949
+ currentToolBatchCallIds.add(pendingToolCall.callId);
950
+ if (toolCallProducesRetainableSessionImage(pendingToolCall.callName)) {
951
+ media.retainedSessionImageCallIds.add(pendingToolCall.callId);
952
+ }
953
+ }
954
+ const completedToolCall = completedToolCallFromSdkEvent(durableSdkEvent);
955
+ if (completedToolCall) {
956
+ retainedScreenshotMetadata =
957
+ media.retainedScreenshotReceiptsByCallId.get(completedToolCall.callId) ?? null;
958
+ const typedScreenshot = retainedScreenshotMetadata
959
+ ? null
960
+ : typedScreenshotFromSdkEvent(durableSdkEvent);
961
+ if (
962
+ !retainedScreenshotMetadata &&
963
+ media.retainedSessionImageCallIds.has(completedToolCall.callId) &&
964
+ sdkEventContainsInlineImage(durableSdkEvent) &&
965
+ !typedScreenshot
966
+ ) {
967
+ retainedScreenshotMetadata = unavailableRetainedSessionImage({
968
+ sessionId: input.sessionId,
969
+ turnId: activeTurnId,
970
+ attemptId: input.attemptId,
971
+ toolCallId: completedToolCall.callId,
972
+ toolOutputId: completedToolCall.callId,
973
+ reason: "unsupported",
974
+ });
975
+ media.retainedScreenshotReceiptsByCallId.set(
976
+ completedToolCall.callId,
977
+ retainedScreenshotMetadata,
978
+ );
979
+ }
980
+ if (
981
+ !retainedScreenshotMetadata &&
982
+ typedScreenshot &&
983
+ typedScreenshot.callId === completedToolCall.callId &&
984
+ media.retainedSessionImageCallIds.has(completedToolCall.callId)
985
+ ) {
986
+ // Install a compact deterministic fallback before database or
987
+ // object-storage I/O. Unexpected retention failure can then
988
+ // reconcile/serialize without leaking inline bytes.
989
+ retainedScreenshotMetadata = unavailableRetainedSessionImage({
990
+ sessionId: input.sessionId,
991
+ turnId: activeTurnId,
992
+ attemptId: input.attemptId,
993
+ toolCallId: typedScreenshot.callId,
994
+ toolOutputId: typedScreenshot.toolOutputId,
995
+ reason: "pending",
996
+ });
997
+ media.retainedScreenshotReceiptsByCallId.set(
998
+ completedToolCall.callId,
999
+ retainedScreenshotMetadata,
1000
+ );
1001
+ retainedScreenshotMetadata = await retainComputerScreenshot({
1002
+ db,
1003
+ objectStorage,
1004
+ accountId: input.accountId,
1005
+ workspaceId: input.workspaceId,
1006
+ sessionId: input.sessionId,
1007
+ turnId: activeTurnId,
1008
+ attemptId: input.attemptId,
1009
+ output: typedScreenshot,
1010
+ });
1011
+ media.retainedScreenshotReceiptsByCallId.set(
1012
+ completedToolCall.callId,
1013
+ retainedScreenshotMetadata,
1014
+ );
1015
+ }
1016
+ normalizedSdkEvents = normalizeSdkEvent(
1017
+ durableSdkEvent as typeof next.value,
1018
+ retainedScreenshotMetadata
1019
+ ? {
1020
+ toolOutputOverride: retainedScreenshotMetadata,
1021
+ retainedOutputEvidence: retainedScreenshotMetadata.available
1022
+ ? retainedScreenshotMetadata
1023
+ : {
1024
+ available: false,
1025
+ reason: retainedScreenshotMetadata.reason,
1026
+ },
1027
+ }
1028
+ : {},
1029
+ );
1030
+ const normalizedToolOutput = normalizedSdkEvents.find(
1031
+ (event) =>
1032
+ event.type === "agent.toolCall.output" &&
1033
+ (event.payload as { id?: unknown }).id === completedToolCall.callId,
1034
+ )?.payload as { output?: unknown } | undefined;
1035
+ if (!normalizedToolOutput || !Object.hasOwn(normalizedToolOutput, "output")) {
1036
+ throw new Error(
1037
+ `Completed SDK tool call ${completedToolCall.callId} produced no durable output projection`,
1038
+ );
1039
+ }
1040
+ const durableResultItem = retainedScreenshotMetadata
1041
+ ? (compactRetainedScreenshotHistory(
1042
+ [completedToolCall.resultItem],
1043
+ media.retainedScreenshotReceiptsByCallId,
1044
+ )[0] ?? completedToolCall.resultItem)
1045
+ : completedToolCall.resultItem;
1046
+ // Keep every parallel result in the attempt ledger until the full
1047
+ // call batch has settled. The SDK's computed history is
1048
+ // non-monotonic while parallel calls complete (a later call may
1049
+ // appear before an earlier persisted pair), so reconciling a
1050
+ // partial batch through a scalar watermark can create an orphan.
1051
+ const recorded = await recordPendingSessionToolCallResult(db, {
1052
+ accountId: input.accountId,
1053
+ workspaceId: input.workspaceId,
1054
+ sessionId: input.sessionId,
1055
+ turnId: activeTurnId,
1056
+ executionGeneration: attempt.executionGeneration,
1057
+ attemptId: input.attemptId,
1058
+ callId: completedToolCall.callId,
1059
+ modelToolOutputTruncationTokens:
1060
+ eventing.modelRunSettings.modelToolOutputTruncationTokens,
1061
+ resultItem: durableResultItem as Record<string, unknown>,
1062
+ eventOutput: normalizedToolOutput.output,
1063
+ ...(videoGenerationAcceptancesByCallId.has(completedToolCall.callId)
1064
+ ? {
1065
+ videoGenerationAcceptance: videoGenerationAcceptancesByCallId.get(
1066
+ completedToolCall.callId,
1067
+ )!,
1068
+ }
1069
+ : {}),
1070
+ });
1071
+ if (!recorded.accepted) {
1072
+ throw new TurnAttemptFencedError(
1073
+ "turn attempt ended while recording a tool-call result",
1074
+ );
1075
+ }
1076
+ const videoAcceptance = videoGenerationAcceptancesByCallId.get(completedToolCall.callId);
1077
+ if (videoAcceptance && startVideoGenerationWorkflow) {
1078
+ try {
1079
+ await startVideoGenerationWorkflow({
1080
+ accountId: input.accountId,
1081
+ workspaceId: input.workspaceId,
1082
+ operationId: videoAcceptance.operationId,
1083
+ });
1084
+ } catch (error) {
1085
+ // The accepted operation is durable. The recovery sweep starts
1086
+ // the same deterministic workflow ID if this nudge fails.
1087
+ observability.warn("Video generation workflow start deferred", {
1088
+ operationId: videoAcceptance.operationId,
1089
+ errorClass: error instanceof Error ? error.name : "UnknownError",
1090
+ });
1091
+ }
1092
+ }
1093
+ const belongsToCurrentBatch = currentToolBatchCallIds.has(completedToolCall.callId);
1094
+ if (belongsToCurrentBatch) {
1095
+ currentToolBatchCompletedCallIds.add(completedToolCall.callId);
1096
+ }
1097
+ const currentBatchIsStable =
1098
+ belongsToCurrentBatch &&
1099
+ currentToolBatchCallIds.size > 0 &&
1100
+ currentToolBatchCompletedCallIds.size === currentToolBatchCallIds.size;
1101
+ const standaloneStableResult =
1102
+ !belongsToCurrentBatch && currentToolBatchCallIds.size === 0;
1103
+ if (currentBatchIsStable || standaloneStableResult) {
1104
+ // Persist the SDK's now-stable complete call/result batch. Keep
1105
+ // the receipts until the normalized tool-output event below is
1106
+ // durably flushed: recovery then covers every crash boundary
1107
+ // without either losing or duplicating the UI projection.
1108
+ await historySink.reconcileConversationTruth({ requireDurable: true });
1109
+ stableToolCallIdsToClear = currentBatchIsStable
1110
+ ? [...currentToolBatchCallIds]
1111
+ : [completedToolCall.callId];
1112
+ completedCurrentToolBatch = currentBatchIsStable;
1113
+ }
1114
+ }
1115
+ const normalized =
1116
+ normalizedSdkEvents ??
1117
+ normalizeSdkEvent(
1118
+ durableSdkEvent as typeof next.value,
1119
+ retainedScreenshotMetadata
1120
+ ? {
1121
+ toolOutputOverride: retainedScreenshotMetadata,
1122
+ retainedOutputEvidence: retainedScreenshotMetadata.available
1123
+ ? retainedScreenshotMetadata
1124
+ : {
1125
+ available: false,
1126
+ reason: retainedScreenshotMetadata.reason,
1127
+ },
1128
+ }
1129
+ : {},
1130
+ );
1131
+ for (const event of normalized) {
1132
+ streamTiming.onEvent(event.type);
1133
+ await eventing.batcher.push(event);
1134
+ }
1135
+ // Structural tool-output events await their durable append before
1136
+ // push returns. The complete result is now retained in the event
1137
+ // row and pending-call recovery receipt, so release only our
1138
+ // duplicate live SDK marker before a long turn accumulates it.
1139
+ releaseMcpResultCustomDataFromSdkEvent(durableSdkEvent);
1140
+ if (stableToolCallIdsToClear) {
1141
+ const cleared = await clearDurablePendingSessionToolCalls(db, {
1142
+ accountId: input.accountId,
1143
+ workspaceId: input.workspaceId,
1144
+ sessionId: input.sessionId,
1145
+ turnId: activeTurnId,
1146
+ executionGeneration: attempt.executionGeneration,
1147
+ attemptId: input.attemptId,
1148
+ callIds: stableToolCallIdsToClear,
1149
+ });
1150
+ if (!cleared.accepted) {
1151
+ throw new TurnAttemptFencedError(
1152
+ "turn attempt ended while finalizing tool-call results",
1153
+ );
1154
+ }
1155
+ if (completedCurrentToolBatch) {
1156
+ currentToolBatchCallIds = new Set<string>();
1157
+ currentToolBatchCompletedCallIds = new Set<string>();
1158
+ }
1159
+ for (const callId of stableToolCallIdsToClear) {
1160
+ media.retainedSessionImageCallIds.delete(callId);
1161
+ }
1162
+ }
1163
+ }
1164
+ } catch (error) {
1165
+ if (fallbackProviderRequestStartedAt !== null) {
1166
+ const durationMs = Math.max(0, performance.now() - fallbackProviderRequestStartedAt);
1167
+ fallbackProviderRequestStartedAt = null;
1168
+ await eventing.publish!([
1169
+ {
1170
+ type: "agent.model.request",
1171
+ payload: {
1172
+ phase: "failed",
1173
+ provider: streamProvider,
1174
+ durationMs: Math.round(durationMs),
1175
+ turnId: activeTurnId,
1176
+ attemptId: input.attemptId,
1177
+ dispatchId,
1178
+ executionGeneration: attempt.executionGeneration,
1179
+ },
1180
+ },
1181
+ ]);
1182
+ }
1183
+ if (
1184
+ !eventing.firstModelRequestPreparationRecorded &&
1185
+ eventing.firstModelRequestPreparationStartedAt !== null
1186
+ ) {
1187
+ eventing.firstModelRequestPreparationRecorded = true;
1188
+ const durationMs = Math.max(
1189
+ 0,
1190
+ performance.now() - eventing.firstModelRequestPreparationStartedAt,
1191
+ );
1192
+ recordTurnStartupPhase(observability, {
1193
+ phase: "model_request_preparation",
1194
+ provider: turnExecutionPolicy.providerId,
1195
+ backend: activeSandboxBackend ?? groupBoxBackend,
1196
+ outcome: "failed",
1197
+ durationSeconds: durationMs / 1_000,
1198
+ count: turnTools.length,
1199
+ });
1200
+ await eventing.publish!([
1201
+ {
1202
+ type: "turn.startup.phase.failed",
1203
+ payload: {
1204
+ phase: "model_preparation",
1205
+ durationMs: Math.round(durationMs),
1206
+ },
1207
+ },
1208
+ ]);
1209
+ }
1210
+ throw error;
1211
+ } finally {
1212
+ if (!streamDone) {
1213
+ // ReadableStream cancellation synchronously trips the Agents SDK's
1214
+ // abort controller, but its returned promise may wait for an
1215
+ // uncooperative provider producer. Once this attempt is fenced,
1216
+ // awaiting that provider-side cleanup pins the Temporal activity
1217
+ // (and therefore Pause) even though every late write is already
1218
+ // rejected. Start cancellation and detach only its cleanup wait;
1219
+ // the SDK abort signal stops the producer and the durable attempt
1220
+ // fence remains the authority for every callback that arrives late.
1221
+ void iterator.return?.().catch(() => undefined);
1222
+ }
1223
+ }
1224
+ await waitForTurnStreamCleanup(
1225
+ eventing.batcher.flush(),
1226
+ eventing.stream.completed.catch(() => undefined),
1227
+ cancellationSignal,
1228
+ );
1229
+ if (!streamSawPerResponseUsage) {
1230
+ const aggregateUsage = eventing.stream.state.usage;
1231
+ const normalizedAggregateUsage = normalizeModelCallUsage(aggregateUsage);
1232
+ const aggregateInput = normalizedAggregateUsage.telemetry.inputTokens;
1233
+ const aggregateSourceKey = modelUsageSourceKey({
1234
+ responseId: null,
1235
+ dispatchId: modelUsageDispatchId,
1236
+ positionalKey: "aggregate",
1237
+ });
1238
+ if (!claimedModelUsageSourceKeys.has(aggregateSourceKey)) {
1239
+ claimedModelUsageSourceKeys.add(aggregateSourceKey);
1240
+ // The aggregate frame is only a billing fallback when no terminal
1241
+ // response exposed usage. It is not final-request context authority.
1242
+ const aggregateAccountCtx = modelCallAccountContext({
1243
+ servingCredentialId: providerTurn.effectiveCodexCredentialId,
1244
+ priorSessionCredentialId: providerTurn.priorSessionCodexCredentialId,
1245
+ isFirstCallOfTurn: true,
1246
+ });
1247
+ let aggregateAuthoritative = false;
1248
+ await recordCompletedModelCallBeforeOwnershipFences({
1249
+ renewLease: () => leases.renewServing("model_usage"),
1250
+ leaseLost: leases.servingLost,
1251
+ leaseLostMessage: "Provider credential lease expired during the active turn",
1252
+ recordUsage: async () => {
1253
+ const billing = await recordModelUsageAndDebitCredits(settings, db, {
1254
+ accountId: input.accountId,
1255
+ workspaceId: input.workspaceId,
1256
+ sessionId: input.sessionId,
1257
+ turnId: activeTurnId,
1258
+ turnAttemptId: input.attemptId,
1259
+ model: turn.model,
1260
+ externallyBilled: billingState.isExternallyBilledTurn,
1261
+ usage: aggregateUsage,
1262
+ normalizedUsage: normalizedAggregateUsage,
1263
+ sourceKey: aggregateSourceKey,
1264
+ latencyMode: turnExecutionPolicy.latencyMode,
1265
+ observability,
1266
+ });
1267
+ const aggregateProvider = resolvedModel?.provider.id ?? settings.openaiProvider;
1268
+ const aggregateProviderApi = resolvedModel?.provider.api ?? "responses";
1269
+ aggregateAuthoritative = await emitModelCallUsage({
1270
+ observability,
1271
+ publish: eventing.publish,
1272
+ accountId: input.accountId,
1273
+ workspaceId: input.workspaceId,
1274
+ sessionId: input.sessionId,
1275
+ turnId: activeTurnId,
1276
+ provider: aggregateProvider,
1277
+ providerApi: aggregateProviderApi,
1278
+ model: turn.model,
1279
+ sourceKey: aggregateSourceKey,
1280
+ usage: { usage: aggregateUsage },
1281
+ normalizedUsage: normalizedAggregateUsage,
1282
+ servingAccountHash: aggregateAccountCtx.servingAccountHash,
1283
+ accountChangedFromPrevCall: aggregateAccountCtx.accountChangedFromPrevCall,
1284
+ emittedSourceKeys: emittedModelUsageSourceKeys,
1285
+ });
1286
+ if (aggregateAuthoritative && billing) {
1287
+ await recordAuthoritativeModelCallFact({
1288
+ db,
1289
+ observability,
1290
+ accountId: input.accountId,
1291
+ workspaceId: input.workspaceId,
1292
+ sessionId: input.sessionId,
1293
+ turnId: activeTurnId,
1294
+ turnAttemptId: input.attemptId,
1295
+ sourceKey: aggregateSourceKey,
1296
+ provider: aggregateProvider,
1297
+ providerApi: aggregateProviderApi,
1298
+ model: turn.model,
1299
+ billing,
1300
+ contextContributions: eventing.companyBrainContextContributions,
1301
+ });
1302
+ }
1303
+ if (aggregateAuthoritative && aggregateInput !== null && aggregateInput > 0) {
1304
+ recordModelInputTokens(observability, streamProvider, aggregateInput);
1305
+ }
1306
+ },
1307
+ recordAttemptSignals: async () => {
1308
+ if (!aggregateAuthoritative) return;
1309
+ // Stream aggregates can cover multiple requests and therefore
1310
+ // cannot identify the final request that compaction must bind.
1311
+ // They remain billing/telemetry fallback data only.
1312
+ modelResponseState.contextSignal = null;
1313
+ await setLastInputTokensFenced(null);
1314
+ },
1315
+ });
1316
+ }
1317
+ }
1318
+ if (eventing.stream.interruptions.length > 0) {
1319
+ await historySink.reconcileConversationTruth({ requireDurable: true });
1320
+ const approvals = runtime.serializeApprovals(eventing.stream.interruptions);
1321
+ const humanInputInterruptions =
1322
+ runtime.serializeHumanInputRequests?.(eventing.stream.interruptions) ?? [];
1323
+ const interactionInterventionInterruptions =
1324
+ runtime.serializeInteractionInterventionRequests?.(eventing.stream.interruptions) ?? [];
1325
+ const latestWorkspace = await getWorkspace(db, input.workspaceId);
1326
+ if (!latestWorkspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
1327
+ assertWorkspaceHumanInputAllowed(
1328
+ resolveWorkspaceAgentHumanInputEnabled(latestWorkspace.settings),
1329
+ "interruption",
1330
+ humanInputInterruptions.length > 0,
1331
+ );
1332
+ const humanInputRequests = await Promise.all(
1333
+ humanInputInterruptions.map(async (interruption) => {
1334
+ const id = stableHumanInputRequestId(
1335
+ input.sessionId,
1336
+ activeTurnId,
1337
+ interruption.toolCallId,
1338
+ );
1339
+ const existing = await getSessionHumanInputRequest(
1340
+ db,
1341
+ input.workspaceId,
1342
+ input.sessionId,
1343
+ id,
1344
+ );
1345
+ if (existing && existing.status !== "pending") {
1346
+ throw new Error(`Settled human-input request ${id} reappeared as an interruption`);
1347
+ }
1348
+ const expiresAt = existing?.expiresAt
1349
+ ? new Date(existing.expiresAt)
1350
+ : interruption.input.expiresInSeconds
1351
+ ? new Date(Date.now() + interruption.input.expiresInSeconds * 1000)
1352
+ : null;
1353
+ return {
1354
+ id,
1355
+ toolCallId: interruption.toolCallId,
1356
+ questions: interruption.input.questions,
1357
+ allowSkip: interruption.input.allowSkip,
1358
+ expiresAt,
1359
+ isNew: existing === null,
1360
+ };
1361
+ }),
1362
+ );
1363
+ const requestEvents = humanInputRequests
1364
+ .filter((request) => request.isNew)
1365
+ .map((request) => ({
1366
+ type: "session.humanInput.requested" as const,
1367
+ payload: {
1368
+ request: {
1369
+ id: request.id,
1370
+ questions: request.questions,
1371
+ allowSkip: request.allowSkip,
1372
+ expiresAt: request.expiresAt?.toISOString() ?? null,
1373
+ },
1374
+ },
1375
+ }));
1376
+ const interactionInterventionRequests = interactionInterventionInterruptions.map(
1377
+ (interruption) => ({
1378
+ id: stableInteractionInterventionId(
1379
+ input.sessionId,
1380
+ activeTurnId,
1381
+ interruption.toolCallId,
1382
+ ),
1383
+ operationId: stableInteractionInterventionOperationId(
1384
+ input.sessionId,
1385
+ activeTurnId,
1386
+ interruption.toolCallId,
1387
+ ),
1388
+ toolCallId: interruption.toolCallId,
1389
+ input: interruption.input,
1390
+ }),
1391
+ );
1392
+ const pendingApprovals = [
1393
+ ...approvals,
1394
+ ...interactionInterventionInterruptions.map((interruption) => interruption.approval),
1395
+ ];
1396
+ const suffixMembers = extractOpenSuffixFromRunState(eventing.stream.state);
1397
+ const interruptionCallIds = interruptionCallIdsFromPause({
1398
+ humanInputRequests,
1399
+ interactionInterventionRequests,
1400
+ pendingApprovals,
1401
+ });
1402
+ assertOpenSuffixResumable(suffixMembers, interruptionCallIds);
1403
+ const suffixByCallId = new Map(suffixMembers.map((member) => [member.callId, member]));
1404
+ const attached = await attachOpenSuffixToPendingToolCalls(db, {
1405
+ accountId: input.accountId,
1406
+ workspaceId: input.workspaceId,
1407
+ sessionId: input.sessionId,
1408
+ turnId: activeTurnId,
1409
+ executionGeneration: attempt.executionGeneration,
1410
+ attemptId: input.attemptId,
1411
+ members: interruptionCallIds.map((callId) => {
1412
+ const member = suffixByCallId.get(callId)!;
1413
+ return {
1414
+ callId,
1415
+ interruptionKind: interruptionKindForCallItem(
1416
+ member.callItem as Record<string, unknown>,
1417
+ ),
1418
+ reasoningItems: member.reasoningItems as Array<Record<string, unknown>>,
1419
+ };
1420
+ }),
1421
+ });
1422
+ if (!attached.accepted) {
1423
+ return claimedResult({ status: "cancelled" });
1424
+ }
1425
+ if (
1426
+ !(await eventing.settle!({
1427
+ events: [
1428
+ ...requestEvents,
1429
+ ...(approvals.length > 0
1430
+ ? [
1431
+ {
1432
+ type: "session.requiresAction" as const,
1433
+ payload: { approvals },
1434
+ },
1435
+ ]
1436
+ : []),
1437
+ {
1438
+ type: "session.status.changed",
1439
+ payload: { status: "requires_action" },
1440
+ },
1441
+ ],
1442
+ turnStatus: "requires_action",
1443
+ sessionStatus: "requires_action",
1444
+ activeTurnId,
1445
+ runState: {
1446
+ serializedRunState: OPEN_SUFFIX_RUN_STATE_BLOB,
1447
+ pendingApprovals,
1448
+ humanInputRequests: humanInputRequests.map(({ isNew: _isNew, ...request }) => request),
1449
+ interactionInterventionRequests,
1450
+ },
1451
+ }))
1452
+ ) {
1453
+ return claimedResult({ status: "cancelled" });
1454
+ }
1455
+ // The interruption and its preceding tool results are now durable.
1456
+ await finalizeTurnOpStreamOps();
1457
+ control.activityStatus = "requires_action";
1458
+ return claimedResult({ status: "requires_action" });
1459
+ }
1460
+
1461
+ const finalOutput = String(eventing.stream.finalOutput ?? "");
1462
+ await historySink.reconcileConversationTruth({ requireDurable: true });
1463
+ // Op-stream durability fence: the tool outputs are now durably in the
1464
+ // history store (a redispatch would NOT re-execute them), so this
1465
+ // turn's settled ops may advance their acked frontier — journal persist
1466
+ // then wire final ack (licensing the runner to GC its retained
1467
+ // frames). Best-effort: a miss leaves the runner's retention TTL to
1468
+ // reap, never fails a completed turn.
1469
+ await finalizeTurnOpStreamOps();
1470
+ if (
1471
+ !(await eventing.settle!({
1472
+ events: [
1473
+ {
1474
+ type: "agent.message.completed",
1475
+ payload: { text: finalOutput },
1476
+ },
1477
+ { type: "turn.completed", payload: { output: finalOutput } },
1478
+ { type: "session.status.changed", payload: { status: "idle" } },
1479
+ ],
1480
+ turnStatus: "completed",
1481
+ sessionStatus: "idle",
1482
+ activeTurnId: null,
1483
+ }))
1484
+ ) {
1485
+ return claimedResult({ status: "cancelled" });
1486
+ }
1487
+ control.turnMetricOutcome = "completed";
1488
+ await recordUsageEvent(db, {
1489
+ accountId: input.accountId,
1490
+ workspaceId: input.workspaceId,
1491
+ eventType: "agent_run.completed",
1492
+ quantity: 1,
1493
+ unit: "run",
1494
+ sourceResourceType: "session_turn",
1495
+ sourceResourceId: activeTurnId,
1496
+ sessionId: input.sessionId,
1497
+ turnId: activeTurnId,
1498
+ turnAttemptId: input.attemptId,
1499
+ idempotencyKey: `usage:agent_run.completed:${activeTurnId}`,
1500
+ });
1501
+ control.activityStatus = "idle";
1502
+ return claimedResult({ status: "idle" });
1503
+ };
1504
+
1505
+ const openSuffixResume = await settleOpenSuffixResumeIfNeeded({
1506
+ db,
1507
+ agent,
1508
+ accountId: input.accountId,
1509
+ workspaceId: input.workspaceId,
1510
+ sessionId: input.sessionId,
1511
+ turnId: activeTurnId,
1512
+ executionGeneration: attempt.executionGeneration,
1513
+ attemptId: input.attemptId,
1514
+ trigger,
1515
+ humanInputResume,
1516
+ modelToolOutputTruncationTokens: eventing.modelRunSettings.modelToolOutputTruncationTokens,
1517
+ settle: eventing.settle!,
1518
+ publish: eventing.publish,
1519
+ });
1520
+ if (openSuffixResume.action === "cancelled") {
1521
+ return claimedResult({ status: "cancelled" });
1522
+ }
1523
+ if (openSuffixResume.action === "requires_action") {
1524
+ control.activityStatus = "requires_action";
1525
+ return claimedResult({ status: "requires_action" });
1526
+ }
1527
+
1528
+ await prepareRunAttemptInput();
1529
+ let retriedAfterCompaction = false;
1530
+ while (true) {
1531
+ try {
1532
+ const result = await runStreamAttempt();
1533
+ if (retriedAfterCompaction) {
1534
+ observability.info("context compaction recovery succeeded after in-activity retry", {
1535
+ sessionId: input.sessionId,
1536
+ turnId: activeTurnId,
1537
+ });
1538
+ }
1539
+ return result;
1540
+ } catch (attemptError) {
1541
+ const overflow = classifyContextWindowOverflowError(attemptError);
1542
+ const compactionNeeded = findCompactionNeededError(attemptError);
1543
+ const recoveryKind = compactionNeeded
1544
+ ? compactionNeeded.trigger === "operator"
1545
+ ? "operator"
1546
+ : "proactive"
1547
+ : overflow
1548
+ ? "overflow"
1549
+ : null;
1550
+ if (!recoveryKind || !eventing.publish || !eventing.turnStartedPublished) {
1551
+ throw attemptError;
1552
+ }
1553
+ await flushRuntimeBatcher();
1554
+ await historySink.reconcileConversationTruth({ skipInputOnlyRows: true });
1555
+ observability.warn("context compaction recovery attempted", {
1556
+ sessionId: input.sessionId,
1557
+ turnId: activeTurnId,
1558
+ reason: recoveryKind,
1559
+ ...safeErrorDiagnostic(attemptError),
1560
+ signalTokens: compactionNeeded?.signalTokens,
1561
+ thresholdTokens: compactionNeeded?.thresholdTokens,
1562
+ });
1563
+ let compacted = false;
1564
+ let compactionHandled = false;
1565
+ let compactionFailureMessage: string | null = null;
1566
+ let compactionRequestCleared = false;
1567
+ try {
1568
+ const outcome = await forceContextCompaction(
1569
+ recoveryKind,
1570
+ compactionNeeded?.signalTokens ?? null,
1571
+ );
1572
+ compacted = outcome.compacted;
1573
+ if (outcome.compacted) {
1574
+ compactionHandled = true;
1575
+ } else {
1576
+ compactionHandled = recoveryKind === "operator" && outcome.requestConsumed;
1577
+ if (!compactionHandled) {
1578
+ compactionFailureMessage = compactionFailureReason(outcome.reason);
1579
+ }
1580
+ }
1581
+ } catch (compactError) {
1582
+ // Transient checkpoint-provider failures recover this same accepted
1583
+ // turn through the normal provider/capacity path. They are not an
1584
+ // empty summary and must not create a new goal continuation.
1585
+ if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
1586
+ if (compactError instanceof TurnAttemptFencedError) throw compactError;
1587
+ const landmark = await settleFailedContextCompactionLandmark(
1588
+ db,
1589
+ {
1590
+ accountId: input.accountId,
1591
+ workspaceId: input.workspaceId,
1592
+ sessionId: input.sessionId,
1593
+ turnId: activeTurnId!,
1594
+ executionGeneration: attempt.executionGeneration,
1595
+ attemptId: input.attemptId,
1596
+ },
1597
+ {
1598
+ clearRequestedCompaction: recoveryKind === "operator",
1599
+ publishLiveEvents: publishCompactionLiveEvents,
1600
+ },
1601
+ );
1602
+ compactionRequestCleared = landmark.requestConsumed;
1603
+ if (!isCompactionSummaryFailure(compactError)) throw compactError;
1604
+ compactionFailureMessage = String(compactionFailureReasonFromError(compactError));
1605
+ observability.warn("context compaction recovery compaction failed", {
1606
+ sessionId: input.sessionId,
1607
+ turnId: activeTurnId,
1608
+ ...safeErrorDiagnostic(compactError),
1609
+ });
1610
+ }
1611
+ if (!compactionHandled) {
1612
+ const errorMessage =
1613
+ compactionFailureMessage ??
1614
+ "compaction summarization failed: compaction produced no replacement history";
1615
+ if (
1616
+ !(await eventing.settle!({
1617
+ events: [
1618
+ {
1619
+ type: "turn.failed",
1620
+ payload: {
1621
+ error: errorMessage,
1622
+ code: "context_compaction_failed",
1623
+ retryable: false,
1624
+ recovery: "user_message",
1625
+ compacted: false,
1626
+ },
1627
+ },
1628
+ {
1629
+ type: "session.status.changed",
1630
+ payload: { status: "idle" },
1631
+ },
1632
+ ],
1633
+ turnStatus: "failed",
1634
+ sessionStatus: "idle",
1635
+ activeTurnId: null,
1636
+ ...(recoveryKind === "operator" && !compactionRequestCleared
1637
+ ? { consumeRequestedCompactionFailure: true }
1638
+ : {}),
1639
+ }))
1640
+ ) {
1641
+ return claimedResult({ status: "cancelled" });
1642
+ }
1643
+ control.turnMetricOutcome = "failed";
1644
+ control.activityStatus = "idle";
1645
+ control.activityError = attemptError;
1646
+ // The failed turn settlement already defers ordinary internal
1647
+ // updates and makes the delivered goal-continuation receipt
1648
+ // terminal. End this workflow run as well: returning plain idle
1649
+ // would immediately synthesize another goal continuation against
1650
+ // the unchanged active history and repeat the same failed
1651
+ // compaction. A new human/API prompt, Steer, or explicitly requested
1652
+ // Compact remains a durable explicit wake and may retry; ordinary
1653
+ // machine updates stay pending for that actionable wake.
1654
+ return claimedResult({ status: "idle", deferredUntilWake: true });
1655
+ }
1656
+ // Codex parity: compaction remains inside the same logical turn and
1657
+ // the same activity. Rebuild the model-visible history from the
1658
+ // durable replacement and continue the sampling loop; do not create
1659
+ // a recovery event, a queue row, a fake user message, or a sandbox.
1660
+ retriedAfterCompaction = true;
1661
+ observability.info("context compaction recovery retrying turn after compaction", {
1662
+ sessionId: input.sessionId,
1663
+ turnId: activeTurnId,
1664
+ reason: recoveryKind,
1665
+ compacted,
1666
+ });
1667
+ await prepareRunAttemptInput();
1668
+ }
1669
+ }
1670
+ }