@opengeni/worker-bundle 0.20.3 → 0.22.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +52 -0
  2. package/dist/activities/agent-turn/agent-build.d.ts +89 -0
  3. package/dist/activities/agent-turn/claim.d.ts +67 -0
  4. package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
  5. package/dist/activities/agent-turn/codex.d.ts +32 -0
  6. package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
  7. package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
  8. package/dist/activities/agent-turn/errors.d.ts +253 -0
  9. package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
  10. package/dist/activities/agent-turn/file-resources.d.ts +37 -0
  11. package/dist/activities/agent-turn/finalization.d.ts +42 -0
  12. package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
  13. package/dist/activities/agent-turn/governance-model.d.ts +66 -0
  14. package/dist/activities/agent-turn/history-sink.d.ts +52 -0
  15. package/dist/activities/agent-turn/history.d.ts +63 -0
  16. package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
  17. package/dist/activities/agent-turn/model-usage.d.ts +218 -0
  18. package/dist/activities/agent-turn/quiescence.d.ts +135 -0
  19. package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
  20. package/dist/activities/agent-turn/run.d.ts +3 -0
  21. package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
  22. package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
  23. package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
  24. package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
  25. package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
  26. package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
  27. package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
  28. package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
  29. package/dist/activities/agent-turn/turn-context.d.ts +144 -0
  30. package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
  31. package/dist/activities/agent-turn.d.ts +12 -954
  32. package/dist/activities/automations.d.ts +16 -0
  33. package/dist/activities/common.d.ts +1 -1
  34. package/dist/activities/connector-attachments.d.ts +13 -0
  35. package/dist/activities/documents.d.ts +1 -0
  36. package/dist/activities/environment.d.ts +26 -15
  37. package/dist/activities/generated-images.d.ts +1 -1
  38. package/dist/activities/git-credential-renewal.d.ts +3 -1
  39. package/dist/activities/goals.d.ts +4 -1
  40. package/dist/activities/google-drive-provider.d.ts +29 -0
  41. package/dist/activities/google-drive-publication.d.ts +17 -4
  42. package/dist/activities/incident-telemetry-authority.d.ts +52 -0
  43. package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
  44. package/dist/activities/knowledge-source-sync.d.ts +4 -0
  45. package/dist/activities/mcp-credentials.d.ts +9 -3
  46. package/dist/activities/open-suffix-resume.d.ts +77 -0
  47. package/dist/activities/packs.d.ts +3 -11
  48. package/dist/activities/parent-wake.d.ts +19 -0
  49. package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
  50. package/dist/activities/retained-screenshots.d.ts +1 -1
  51. package/dist/activities/rig-verification.d.ts +1 -1
  52. package/dist/activities/run-input.d.ts +9 -10
  53. package/dist/activities/sandbox-images.d.ts +1 -14
  54. package/dist/activities/sandbox-lease.d.ts +21 -3
  55. package/dist/activities/scheduled-tasks.d.ts +47 -0
  56. package/dist/activities/types.d.ts +23 -2
  57. package/dist/activities/video-media-validation.d.ts +1 -1
  58. package/dist/activities/video-output-retention.d.ts +1 -2
  59. package/dist/activities/xai-auth.d.ts +3 -0
  60. package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
  61. package/dist/activities-control-DKG22ONJ.js.map +1 -0
  62. package/dist/activities-control.d.ts +6 -2
  63. package/dist/activities-turn-4IMC3MAA.js +21326 -0
  64. package/dist/activities-turn-4IMC3MAA.js.map +1 -0
  65. package/dist/activities.d.ts +7 -3
  66. package/dist/artifact-materializer-entry.js +41 -22
  67. package/dist/artifact-materializer-entry.js.map +1 -1
  68. package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
  69. package/dist/chunk-CE7TPKBS.js.map +1 -0
  70. package/dist/chunk-CNYGVRKF.js +4698 -0
  71. package/dist/chunk-CNYGVRKF.js.map +1 -0
  72. package/dist/connection-provider-use.d.ts +18 -0
  73. package/dist/editable-artifact-materializer.d.ts +4 -3
  74. package/dist/github-rest-mcp.d.ts +26 -0
  75. package/dist/index.d.ts +10 -2
  76. package/dist/index.js +58 -14
  77. package/dist/index.js.map +1 -1
  78. package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
  79. package/dist/model-context-contributions.d.ts +8 -4
  80. package/dist/observability-metrics.d.ts +63 -4
  81. package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
  82. package/dist/personal-github-git-credentials.d.ts +9 -0
  83. package/dist/pr-review-credentials.d.ts +6 -0
  84. package/dist/sandbox-archive-storage.d.ts +45 -0
  85. package/dist/sandbox-resume.d.ts +65 -6
  86. package/dist/sandbox-routing.d.ts +38 -2
  87. package/dist/scheduled-alert-occurrence.d.ts +33 -0
  88. package/dist/workflow-bundle.js +87 -28
  89. package/dist/workflows/activities.d.ts +1 -0
  90. package/dist/workflows/automations.d.ts +6 -0
  91. package/dist/workflows/document-index.d.ts +1 -0
  92. package/dist/workflows.d.ts +1 -0
  93. package/package.json +17 -17
  94. package/src/activities/agent-turn/admission.ts +292 -0
  95. package/src/activities/agent-turn/agent-build.ts +802 -0
  96. package/src/activities/agent-turn/claim.ts +590 -0
  97. package/src/activities/agent-turn/codex-capacity.ts +720 -0
  98. package/src/activities/agent-turn/codex.ts +94 -0
  99. package/src/activities/agent-turn/compaction-prep.ts +722 -0
  100. package/src/activities/agent-turn/credential-leases.ts +240 -0
  101. package/src/activities/agent-turn/errors.ts +1068 -0
  102. package/src/activities/agent-turn/failure-settlement.ts +1436 -0
  103. package/src/activities/agent-turn/file-resources.ts +227 -0
  104. package/src/activities/agent-turn/finalization.ts +754 -0
  105. package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
  106. package/src/activities/agent-turn/governance-model.ts +465 -0
  107. package/src/activities/agent-turn/history-sink.ts +139 -0
  108. package/src/activities/agent-turn/history.ts +200 -0
  109. package/src/activities/agent-turn/media-artifacts.ts +359 -0
  110. package/src/activities/agent-turn/model-usage.ts +823 -0
  111. package/src/activities/agent-turn/quiescence.ts +353 -0
  112. package/src/activities/agent-turn/run-credentials.ts +761 -0
  113. package/src/activities/agent-turn/run.ts +1530 -0
  114. package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
  115. package/src/activities/agent-turn/sandbox-provision.ts +569 -0
  116. package/src/activities/agent-turn/sandbox-route.ts +358 -0
  117. package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
  118. package/src/activities/agent-turn/stream-attempt.ts +1670 -0
  119. package/src/activities/agent-turn/tool-environment.ts +729 -0
  120. package/src/activities/agent-turn/tool-policy.ts +216 -0
  121. package/src/activities/agent-turn/tool-result-spill.ts +265 -0
  122. package/src/activities/agent-turn/turn-context.ts +281 -0
  123. package/src/activities/agent-turn/xai-capacity.ts +229 -0
  124. package/src/activities/agent-turn.ts +12 -12701
  125. package/src/activities/api-integrations.ts +3 -0
  126. package/src/activities/automations.ts +214 -0
  127. package/src/activities/codemode-dispatcher.ts +12 -1
  128. package/src/activities/common.ts +1 -5
  129. package/src/activities/connector-attachments.ts +168 -0
  130. package/src/activities/context-compaction.ts +18 -0
  131. package/src/activities/environment.ts +151 -55
  132. package/src/activities/generated-images.ts +26 -11
  133. package/src/activities/git-credential-renewal.ts +13 -3
  134. package/src/activities/goals.ts +85 -19
  135. package/src/activities/google-drive-provider.ts +178 -0
  136. package/src/activities/google-drive-publication.ts +147 -49
  137. package/src/activities/incident-telemetry-authority.ts +327 -0
  138. package/src/activities/incident-telemetry-preflight.ts +317 -0
  139. package/src/activities/knowledge-source-sync.ts +178 -37
  140. package/src/activities/mcp-credentials.ts +191 -17
  141. package/src/activities/open-suffix-resume.ts +422 -0
  142. package/src/activities/packs.ts +12 -60
  143. package/src/activities/parent-wake.ts +64 -14
  144. package/src/activities/retained-screenshot-reaper.ts +6 -3
  145. package/src/activities/retained-screenshots.ts +15 -47
  146. package/src/activities/rig-verification.ts +1 -1
  147. package/src/activities/run-input.ts +102 -142
  148. package/src/activities/sandbox-images.ts +7 -166
  149. package/src/activities/sandbox-lease.ts +406 -15
  150. package/src/activities/scheduled-tasks.ts +2103 -411
  151. package/src/activities/session-state.ts +10 -2
  152. package/src/activities/types.ts +32 -2
  153. package/src/activities/video-media-validation.ts +3 -3
  154. package/src/activities/video-output-retention.ts +2 -57
  155. package/src/activities/xai-auth.ts +8 -0
  156. package/src/activities-control.ts +2 -0
  157. package/src/activities.ts +3 -0
  158. package/src/activity-services.ts +13 -4
  159. package/src/connection-provider-use.ts +36 -0
  160. package/src/editable-artifact-materializer-subprocess.ts +25 -14
  161. package/src/editable-artifact-materializer.ts +29 -18
  162. package/src/github-rest-mcp.ts +713 -0
  163. package/src/http.ts +16 -6
  164. package/src/index.ts +40 -5
  165. package/src/knowledge-source-sync-activity-policy.ts +2 -0
  166. package/src/model-context-contributions.ts +33 -10
  167. package/src/observability-metrics.ts +357 -0
  168. package/src/opensandbox-kubernetes-inventory.ts +292 -0
  169. package/src/personal-github-git-credentials.ts +331 -0
  170. package/src/pr-review-credentials.ts +163 -0
  171. package/src/sandbox-archive-storage.ts +159 -0
  172. package/src/sandbox-resume.ts +449 -63
  173. package/src/sandbox-routing.ts +314 -16
  174. package/src/scheduled-alert-occurrence.ts +176 -0
  175. package/src/workflows/activities.ts +18 -2
  176. package/src/workflows/automations.ts +11 -0
  177. package/src/workflows/session.ts +13 -3
  178. package/src/workflows.ts +1 -0
  179. package/dist/activities-control-BJSDRSIP.js.map +0 -1
  180. package/dist/activities-turn-J5YO7XXX.js +0 -17243
  181. package/dist/activities-turn-J5YO7XXX.js.map +0 -1
  182. package/dist/chunk-N5TJIFCE.js.map +0 -1
  183. package/dist/chunk-P6Z2SLAA.js +0 -3092
  184. package/dist/chunk-P6Z2SLAA.js.map +0 -1
@@ -0,0 +1,590 @@
1
+ import {
2
+ applySessionTurnSettlement,
3
+ claimSessionWorkForAttempt,
4
+ getSessionEvent,
5
+ getHumanInputResumeForEvent,
6
+ getInteractionInterventionResumeForEvent,
7
+ installOrReadTurnExecutionPolicyForAttempt,
8
+ workspaceCodexSubscriptionActive,
9
+ requireSession,
10
+ type AppendEventInput,
11
+ type ApiIntegrationRuntime,
12
+ type CanonicalTurnStartupMilestoneReceipt,
13
+ type SessionTurnForExecution,
14
+ type SessionTurnRecordingSettlement,
15
+ } from "@opengeni/db";
16
+ import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@opengeni/events";
17
+ import { deliverChildRequiresActionToParent } from "../parent-wake";
18
+ import { deleteRecordingArtifacts } from "@opengeni/runtime";
19
+ import {
20
+ assertTurnExecutionPolicyMatchesConfigV1,
21
+ resolveTurnExecutionPolicyV1,
22
+ type Settings,
23
+ } from "@opengeni/config";
24
+ import {
25
+ settingsWithCodexCredential,
26
+ settingsWithEnabledCapabilityMcpServers,
27
+ settingsWithWorkspaceGatewayCredential,
28
+ withXaiSubscriptionProvider,
29
+ } from "../capabilities";
30
+ import { validateIncidentTelemetrySystemUpdateAuthority } from "../incident-telemetry-authority";
31
+ import {
32
+ assertSessionAllowsProductModel,
33
+ resolveCodexAppsCredentialIdForRun,
34
+ } from "@opengeni/core";
35
+ import { TurnAttemptFencedError } from "../turn-attempt-fenced";
36
+ import { currentActivityContext, startActivityHeartbeat } from "../streaming";
37
+ import type {
38
+ TurnActivityServices as ActivityServices,
39
+ RunAgentTurnInput,
40
+ RunAgentTurnResult,
41
+ } from "../types";
42
+ import { makeTurnOpJournal, type TurnHeartbeatDetails } from "../../op-journal";
43
+ import {
44
+ recordSessionEventAppendLatency,
45
+ recordSessionEventPublishLatency,
46
+ recordTurnStartupPhase,
47
+ recordTurnStartupMilestone,
48
+ turnLifecycleMetricsFor,
49
+ } from "../../observability-metrics";
50
+ import { prepareRecordingForSettlement, type ActiveRecording } from "../recording";
51
+ import { createTurnCredentialLeases } from "./credential-leases";
52
+ import { createTurnMediaArtifacts } from "./media-artifacts";
53
+ import { readTurnExecutionPolicyV1 } from "@opengeni/contracts";
54
+
55
+ import {
56
+ credentialSubjectIdForTurnInitiator,
57
+ turnExecutionPolicyBillingIdentity,
58
+ legacyTurnExecutionPolicyInput,
59
+ ensureRunAllowed,
60
+ } from "./admission";
61
+ import { providerRecoveryCountFromMetadata, isWorkerShutdownCancellation } from "./errors";
62
+ import { throwIfTurnOperationCancelled, waitForTurnOperation } from "./sandbox-provision";
63
+ import type { TurnExecutionPolicyV1 } from "@opengeni/contracts";
64
+ import type {
65
+ AttemptIdentityState,
66
+ BillingState,
67
+ ClaimedResult,
68
+ EventingState,
69
+ RecordingState,
70
+ SandboxRuntimeState,
71
+ TurnControlState,
72
+ } from "./turn-context";
73
+
74
+ export type ClaimTurnDeps = {
75
+ input: RunAgentTurnInput;
76
+ settings: Settings;
77
+ db: ActivityServices["db"];
78
+ bus: ActivityServices["bus"];
79
+ runtime: ActivityServices["runtime"];
80
+ objectStorage: ActivityServices["objectStorage"];
81
+ observability: ActivityServices["observability"];
82
+ entitlements: ActivityServices["entitlements"];
83
+ wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
84
+ cancellationSignal: AbortSignal | undefined;
85
+ activityContext: ReturnType<typeof currentActivityContext>;
86
+ dispatchId: string;
87
+ activityStarted: number;
88
+ control: TurnControlState;
89
+ attempt: AttemptIdentityState;
90
+ billingState: BillingState;
91
+ sandboxState: SandboxRuntimeState;
92
+ recordingState: RecordingState;
93
+ eventing: EventingState;
94
+ leases: ReturnType<typeof createTurnCredentialLeases>;
95
+ media: ReturnType<typeof createTurnMediaArtifacts>;
96
+ claimedResult: ClaimedResult;
97
+ acknowledgeLostAttemptOwnership: () => void;
98
+ abandonActiveRecording: (reason: string, disposition?: "failed" | "discard") => Promise<void>;
99
+ };
100
+
101
+ export type ClaimTurnOk = {
102
+ turn: SessionTurnForExecution;
103
+ session: Awaited<ReturnType<typeof requireSession>>;
104
+ installedApiIntegrations: readonly ApiIntegrationRuntime[];
105
+ credentialSubjectId: string | undefined;
106
+ fileAuthoritySubjectId: string | null;
107
+ capabilitySettings: Settings;
108
+ codexAppsCredentialId: string | null;
109
+ turnExecutionPolicy: TurnExecutionPolicyV1;
110
+ trigger: NonNullable<Awaited<ReturnType<typeof getSessionEvent>>>;
111
+ humanInputResume: Awaited<ReturnType<typeof getHumanInputResumeForEvent>>;
112
+ interactionInterventionResume: Awaited<
113
+ ReturnType<typeof getInteractionInterventionResumeForEvent>
114
+ >;
115
+ throwIfWorkerShuttingDown: () => void;
116
+ throwIfTurnCancelled: () => void;
117
+ opJournal: ReturnType<typeof makeTurnOpJournal>;
118
+ modelUsageDispatchId: string;
119
+ claimedModelUsageSourceKeys: Set<string>;
120
+ emittedModelUsageSourceKeys: Set<string>;
121
+ };
122
+
123
+ /**
124
+ * Stable for Temporal retries of one scheduled turn attempt, but unique across
125
+ * workflow runs. Temporal activity ids restart at `1` after continue-as-new or
126
+ * workflow restart, so they are not safe durable producer identities by
127
+ * themselves.
128
+ */
129
+ export function turnAttemptProducerId(
130
+ input: Pick<RunAgentTurnInput, "workflowId" | "attemptId">,
131
+ turnId: string,
132
+ ): string {
133
+ return `${input.workflowId}:${turnId}:${input.attemptId}`;
134
+ }
135
+
136
+ export type ClaimTurnOutcome = { exit: RunAgentTurnResult } | { ok: ClaimTurnOk };
137
+
138
+ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOutcome> {
139
+ const {
140
+ input,
141
+ settings,
142
+ db,
143
+ bus,
144
+ runtime,
145
+ objectStorage,
146
+ observability,
147
+ entitlements,
148
+ wakeSessionWorkflow,
149
+ cancellationSignal,
150
+ activityContext,
151
+ dispatchId,
152
+ activityStarted,
153
+ control,
154
+ attempt,
155
+ billingState,
156
+ sandboxState,
157
+ recordingState,
158
+ eventing,
159
+ leases,
160
+ media,
161
+ claimedResult,
162
+ acknowledgeLostAttemptOwnership,
163
+ abandonActiveRecording,
164
+ } = deps;
165
+
166
+ const claim = await claimSessionWorkForAttempt(db, input.workspaceId, {
167
+ sessionId: input.sessionId,
168
+ workflowId: input.workflowId,
169
+ workflowRunId: input.workflowRunId,
170
+ attemptId: input.attemptId,
171
+ dispatchId,
172
+ trigger: input.trigger,
173
+ validatePendingSystemUpdateAuthority: async (tx, update) =>
174
+ await validateIncidentTelemetrySystemUpdateAuthority({
175
+ db: tx,
176
+ settings,
177
+ workspaceId: input.workspaceId,
178
+ sessionId: input.sessionId,
179
+ update,
180
+ }),
181
+ });
182
+ if (claim.action === "unclaimed") {
183
+ control.activityStatus = "unclaimed";
184
+ return { exit: { status: "unclaimed", reason: claim.reason } };
185
+ }
186
+ const turn = claim.turn;
187
+ attempt.turnId = turn.id;
188
+ // Establish durable attempt ownership before any later read can fail.
189
+ // Therefore every failure with no turnId came from the one atomic claim
190
+ // transaction and can be classified without conflating ordinary runtime
191
+ // or transport failures with admission failures.
192
+ const session = await requireSession(db, input.workspaceId, input.sessionId);
193
+ attempt.executionGeneration = turn.executionGeneration;
194
+ attempt.providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
195
+ let installedApiIntegrations: readonly ApiIntegrationRuntime[] = [];
196
+ const credentialSubjectId = credentialSubjectIdForTurnInitiator(turn);
197
+ const fileAuthoritySubjectId = turn.initiatingHumanSubjectId ?? null;
198
+ const mcpSettings = await settingsWithEnabledCapabilityMcpServers(
199
+ db,
200
+ input.workspaceId,
201
+ settings,
202
+ {
203
+ ...(credentialSubjectId
204
+ ? { subjectId: credentialSubjectId }
205
+ : {
206
+ personalConnectionDelegations: turn.personalConnectionDelegations,
207
+ }),
208
+ onResolvedApiIntegrations: (integrations) => {
209
+ installedApiIntegrations = integrations;
210
+ },
211
+ },
212
+ );
213
+ // Read the active-credential flag once for the runtime capability overlay.
214
+ // Accepted billing/provider identity comes from the turn policy below,
215
+ // never from this mutable health snapshot.
216
+ const codexSubscriptionActive = await workspaceCodexSubscriptionActive(
217
+ db,
218
+ mcpSettings,
219
+ input.workspaceId,
220
+ );
221
+ const codexSettings = await settingsWithCodexCredential(
222
+ db,
223
+ input.workspaceId,
224
+ mcpSettings,
225
+ codexSubscriptionActive,
226
+ );
227
+ const xaiSettings = codexSettings.supergrokSubscriptionEnabled
228
+ ? withXaiSubscriptionProvider(codexSettings)
229
+ : codexSettings;
230
+ const capabilitySettings = await settingsWithWorkspaceGatewayCredential(
231
+ db,
232
+ input.workspaceId,
233
+ xaiSettings,
234
+ );
235
+ const codexAppsCredentialId = capabilitySettings.codexConnectedAppsEnabled
236
+ ? await resolveCodexAppsCredentialIdForRun(db, input.workspaceId)
237
+ : null;
238
+ runtime.configure(capabilitySettings);
239
+ const claimedPolicy = readTurnExecutionPolicyV1(turn.metadata);
240
+ const policyForAbsent =
241
+ claimedPolicy.kind === "valid"
242
+ ? claimedPolicy.policy
243
+ : resolveTurnExecutionPolicyV1(capabilitySettings, legacyTurnExecutionPolicyInput(turn));
244
+ const installedPolicy = await installOrReadTurnExecutionPolicyForAttempt(db, {
245
+ accountId: input.accountId,
246
+ workspaceId: input.workspaceId,
247
+ sessionId: input.sessionId,
248
+ turnId: attempt.turnId,
249
+ executionGeneration: attempt.executionGeneration,
250
+ attemptId: input.attemptId,
251
+ policyForAbsent,
252
+ });
253
+ if (!installedPolicy.accepted) {
254
+ throw new TurnAttemptFencedError(`turn execution policy was fenced: ${installedPolicy.reason}`);
255
+ }
256
+ const verifiedExecutionPolicy = assertTurnExecutionPolicyMatchesConfigV1(
257
+ capabilitySettings,
258
+ installedPolicy.policy,
259
+ {
260
+ modelId: turn.model,
261
+ reasoningEffort: turn.reasoningEffort,
262
+ latencyMode: turn.latencyMode,
263
+ },
264
+ );
265
+ const turnExecutionPolicy = verifiedExecutionPolicy.policy;
266
+ assertSessionAllowsProductModel(session, turnExecutionPolicy.productModelId);
267
+ const billingIdentity = turnExecutionPolicyBillingIdentity(turnExecutionPolicy);
268
+ billingState.isExternallyBilledTurn = billingIdentity.externallyBilled;
269
+ billingState.isCodexTurn = billingIdentity.codexSubscription;
270
+ billingState.isXaiTurn = billingIdentity.xaiSubscription;
271
+ attempt.triggerEventId = turn.triggerEventId;
272
+ const trigger = await getSessionEvent(db, input.workspaceId, attempt.triggerEventId);
273
+ if (!trigger) {
274
+ throw new Error(`Trigger event not found: ${attempt.triggerEventId}`);
275
+ }
276
+ const humanInputResume = await getHumanInputResumeForEvent(
277
+ db,
278
+ input.workspaceId,
279
+ input.sessionId,
280
+ trigger,
281
+ );
282
+ const interactionInterventionResume = await getInteractionInterventionResumeForEvent(
283
+ db,
284
+ input.workspaceId,
285
+ input.sessionId,
286
+ trigger,
287
+ );
288
+ attempt.triggerType = trigger.type;
289
+ attempt.redispatchesAtDispatch = Number(
290
+ (turn.metadata as { workerDeathRedispatches?: number } | null)?.workerDeathRedispatches ?? 0,
291
+ );
292
+ turnLifecycleMetricsFor(observability).start(attempt.turnId);
293
+ // §7.5 P3 — pass the accepted billing attribution (externally funded turns
294
+ // bypass OpenGeni credit/token gates)
295
+ // AND the optional host `entitlements` port (when bound, its admitRun replaces
296
+ // the local credit read). Unset port → today's local-ledger path.
297
+ await waitForTurnOperation(
298
+ ensureRunAllowed(
299
+ settings,
300
+ db,
301
+ input.accountId,
302
+ input.workspaceId,
303
+ billingState.isExternallyBilledTurn,
304
+ entitlements,
305
+ ),
306
+ cancellationSignal,
307
+ undefined,
308
+ );
309
+ // Setup (variableSet load, MCP connects, sandbox restore) does not
310
+ // stream and so never observes cancellation on its own; these explicit
311
+ // checks let a graceful shutdown checkpoint the turn before the worker is
312
+ // force-killed instead of riding the setup to a heartbeat timeout.
313
+ const throwIfWorkerShuttingDown = () => {
314
+ const reason = activityContext?.cancellationSignal.reason;
315
+ if (isWorkerShutdownCancellation(reason)) {
316
+ throw reason;
317
+ }
318
+ };
319
+ const throwIfTurnCancelled = () => throwIfTurnOperationCancelled(cancellationSignal);
320
+ // ONE shared details object for every heartbeat this activity sends (each
321
+ // site spreads it + its own phase), so cross-site fields — the op-stream
322
+ // settled roster in particular — survive last-write-wins instead of being
323
+ // clobbered by whichever site heartbeated most recently.
324
+ const heartbeatDetails: TurnHeartbeatDetails = {
325
+ phase: "running",
326
+ sessionId: input.sessionId,
327
+ turnId: attempt.turnId,
328
+ opAcks: {},
329
+ };
330
+ const opJournal = makeTurnOpJournal(activityContext, heartbeatDetails);
331
+ eventing.heartbeatTimer = startActivityHeartbeat(activityContext, heartbeatDetails);
332
+ let producerSeq = 0;
333
+ // One producer per scheduled turn attempt, not per turn. A turn can run
334
+ // again after Pause/Steer/recovery, and each attempt restarts producerSeq at
335
+ // 1. `attemptId` is stable across a genuine Temporal activity retry and
336
+ // unique across workflow runs; `activityId` is not, because Temporal resets
337
+ // it after continue-as-new or a new workflow execution.
338
+ const producerId = turnAttemptProducerId(input, attempt.turnId);
339
+ // Fold the same durable attempt identity into positional usage source keys.
340
+ // A retry dedupes; a re-dispatch cannot collide with the prior attempt.
341
+ leases.codex.holderId = dispatchId;
342
+ const modelUsageDispatchId = input.attemptId;
343
+ const claimedModelUsageSourceKeys = new Set<string>();
344
+ const emittedModelUsageSourceKeys = new Set<string>();
345
+ const recordCanonicalStartupMilestones = (
346
+ receipts: CanonicalTurnStartupMilestoneReceipt[],
347
+ ): void => {
348
+ for (const receipt of receipts) {
349
+ recordTurnStartupMilestone(observability, {
350
+ milestone: receipt.milestone,
351
+ provider: turnExecutionPolicy.providerId,
352
+ backend: sandboxState.startupMilestoneBackend ?? turn.sandboxBackend,
353
+ outcome: receipt.outcome,
354
+ durationSeconds: receipt.durationMs / 1_000,
355
+ });
356
+ }
357
+ };
358
+ eventing.publish = async (
359
+ events: Array<Omit<AppendEventInput, "producerId" | "producerSeq" | "turnId">>,
360
+ immediate = false,
361
+ ) => {
362
+ const inputs = events.map((event) => ({
363
+ ...event,
364
+ payload: event.payload,
365
+ turnId: attempt.turnId!,
366
+ producerId,
367
+ producerSeq: ++producerSeq,
368
+ }));
369
+ const appended = await appendAndPublishTurnEventsFenced(
370
+ db,
371
+ bus,
372
+ input.workspaceId,
373
+ input.sessionId,
374
+ attempt.turnId!,
375
+ attempt.executionGeneration,
376
+ input.attemptId,
377
+ inputs,
378
+ {
379
+ onAppend: ({ durationSeconds }) =>
380
+ recordSessionEventAppendLatency(observability, {
381
+ durationSeconds,
382
+ }),
383
+ onPublish: ({ durationSeconds }) =>
384
+ recordSessionEventPublishLatency(observability, {
385
+ durationSeconds,
386
+ }),
387
+ },
388
+ );
389
+ if (inputs.length > 0 && !appended.accepted) {
390
+ throw new TurnAttemptFencedError("turn execution generation was fenced");
391
+ }
392
+ recordCanonicalStartupMilestones(appended.canonicalStartupMilestones);
393
+ if (inputs.length > 0) {
394
+ turnLifecycleMetricsFor(observability).progress(attempt.turnId!);
395
+ }
396
+ activityContext?.heartbeat({
397
+ ...heartbeatDetails,
398
+ phase: "events_published",
399
+ producerSeq,
400
+ });
401
+ if (immediate) {
402
+ await new Promise<void>((resolve) => setTimeout(resolve, 0));
403
+ }
404
+ return appended;
405
+ };
406
+ eventing.settle = async (inputSettlement) => {
407
+ const attemptClosing = ["completed", "failed", "cancelled", "requires_action"].includes(
408
+ inputSettlement.turnStatus,
409
+ );
410
+ const recordingForSettlement =
411
+ attemptClosing && recordingState.activeRecording && sandboxState.resolvedSandbox
412
+ ? (recordingState.activeRecording as ActiveRecording)
413
+ : null;
414
+ const preparedRecording = recordingForSettlement
415
+ ? await prepareRecordingForSettlement({
416
+ settings,
417
+ objectStorage,
418
+ workspaceId: input.workspaceId,
419
+ sessionId: input.sessionId,
420
+ active: recordingForSettlement,
421
+ session: sandboxState.resolvedSandbox!.established.session,
422
+ didComputerUse: recordingState.didComputerUse,
423
+ })
424
+ : null;
425
+ let recordingMutation: SessionTurnRecordingSettlement | undefined;
426
+ if (preparedRecording) {
427
+ const mutation = preparedRecording.mutation;
428
+ recordingMutation =
429
+ mutation.action === "discard"
430
+ ? mutation
431
+ : {
432
+ ...mutation,
433
+ producerId,
434
+ producerSeq: ++producerSeq,
435
+ };
436
+ }
437
+ const compactionRequestFailure = inputSettlement.consumeRequestedCompactionFailure
438
+ ? {
439
+ reason: "summarization_failed" as const,
440
+ producerId,
441
+ producerSeq: ++producerSeq,
442
+ }
443
+ : undefined;
444
+ const inputs = inputSettlement.events.map((event) => ({
445
+ ...event,
446
+ payload: event.payload,
447
+ turnId: attempt.turnId!,
448
+ producerId,
449
+ producerSeq: ++producerSeq,
450
+ }));
451
+ const runState = inputSettlement.runState
452
+ ? {
453
+ ...inputSettlement.runState,
454
+ serializedRunState: media.compactMediaRunState(
455
+ inputSettlement.runState.serializedRunState,
456
+ ),
457
+ }
458
+ : undefined;
459
+ const result = await applySessionTurnSettlement(db, input.workspaceId, {
460
+ sessionId: input.sessionId,
461
+ turnId: attempt.turnId!,
462
+ triggerEventId: attempt.triggerEventId!,
463
+ attemptId: input.attemptId,
464
+ turnStatus: inputSettlement.turnStatus,
465
+ sessionStatus: inputSettlement.sessionStatus,
466
+ activeTurnId: inputSettlement.activeTurnId,
467
+ events: inputs,
468
+ ...(runState ? { runState } : {}),
469
+ ...(recordingMutation ? { recording: recordingMutation } : {}),
470
+ ...(compactionRequestFailure ? { compactionRequestFailure } : {}),
471
+ });
472
+ if (result.action === "stale") {
473
+ // The terminal write can lose to a control transaction before the
474
+ // workflow delivers Temporal cancellation. That control may settle
475
+ // the already-closed attempt as rejected_stale, so returning without
476
+ // this flag would strand its replacement behind quiesced_at forever.
477
+ // Enter the same hard tool-fence/receipt path as an explicit
478
+ // TurnAttemptFencedError. If ownership was lost for an unrelated
479
+ // reason, allowUninterrupted makes the receipt transaction a no-op.
480
+ acknowledgeLostAttemptOwnership();
481
+ if (recordingForSettlement) {
482
+ await abandonActiveRecording(
483
+ "recording settlement lost attempt ownership",
484
+ preparedRecording?.mutation.action === "discard" ? "discard" : "failed",
485
+ );
486
+ }
487
+ control.activityStatus = "cancelled";
488
+ control.turnMetricOutcome = "cancelled";
489
+ return false;
490
+ }
491
+ recordCanonicalStartupMilestones(result.canonicalStartupMilestones);
492
+ turnLifecycleMetricsFor(observability).progress(attempt.turnId!);
493
+ if (recordingForSettlement && preparedRecording) {
494
+ if (result.recordingMutationApplied) {
495
+ recordingState.activeRecording = null;
496
+ if (preparedRecording.deleteArtifactsAfterCommit) {
497
+ await deleteRecordingArtifacts(
498
+ sandboxState.resolvedSandbox!.established.session,
499
+ recordingForSettlement.proc,
500
+ );
501
+ }
502
+ } else {
503
+ await abandonActiveRecording("recording row was unavailable during turn settlement");
504
+ }
505
+ }
506
+ await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, result.events);
507
+ if (inputSettlement.turnStatus === "requires_action") {
508
+ // The settlement transaction committed the parent's child_requires_action
509
+ // outbox row (when this session has a parent and notices are enabled).
510
+ // Deliver it right away; the reaper covers a crash between the two.
511
+ await deliverChildRequiresActionToParent(
512
+ { db, bus, settings, observability, wakeSessionWorkflow },
513
+ input.workspaceId,
514
+ input.sessionId,
515
+ { turnId: attempt.turnId!, turnGeneration: attempt.executionGeneration },
516
+ );
517
+ }
518
+ activityContext?.heartbeat({
519
+ ...heartbeatDetails,
520
+ phase: "events_published",
521
+ producerSeq,
522
+ });
523
+ return true;
524
+ };
525
+ activityContext?.heartbeat({
526
+ ...heartbeatDetails,
527
+ phase: "turn_started",
528
+ });
529
+
530
+ // A shutdown that landed during claim/billing setup stops before the turn
531
+ // visibly starts: nothing ran yet, so the same inference starts cleanly
532
+ // on a healthy worker.
533
+ throwIfWorkerShuttingDown();
534
+ throwIfTurnCancelled();
535
+ recordTurnStartupPhase(observability, {
536
+ phase: "claim_and_policy",
537
+ provider: turnExecutionPolicy.providerId,
538
+ backend: turn.sandboxBackend,
539
+ outcome: "completed",
540
+ durationSeconds: (performance.now() - activityStarted) / 1_000,
541
+ });
542
+ const turnStartSettlementStartedAt = performance.now();
543
+ if (
544
+ !(await eventing.settle({
545
+ events: [
546
+ { type: "session.status.changed", payload: { status: "running" } },
547
+ {
548
+ type: "turn.started",
549
+ payload: { triggerEventId: attempt.triggerEventId },
550
+ },
551
+ ],
552
+ turnStatus: "running",
553
+ sessionStatus: "running",
554
+ activeTurnId: attempt.turnId,
555
+ }))
556
+ ) {
557
+ return { exit: claimedResult({ status: "cancelled" }) };
558
+ }
559
+ recordTurnStartupPhase(observability, {
560
+ phase: "turn_start_settlement",
561
+ provider: turnExecutionPolicy.providerId,
562
+ backend: turn.sandboxBackend,
563
+ outcome: "completed",
564
+ durationSeconds: (performance.now() - turnStartSettlementStartedAt) / 1_000,
565
+ count: 2,
566
+ });
567
+ eventing.turnStartedPublished = true;
568
+
569
+ return {
570
+ ok: {
571
+ turn,
572
+ session,
573
+ installedApiIntegrations,
574
+ credentialSubjectId,
575
+ fileAuthoritySubjectId,
576
+ capabilitySettings,
577
+ codexAppsCredentialId,
578
+ turnExecutionPolicy,
579
+ trigger,
580
+ humanInputResume,
581
+ interactionInterventionResume,
582
+ throwIfWorkerShuttingDown,
583
+ throwIfTurnCancelled,
584
+ opJournal,
585
+ modelUsageDispatchId,
586
+ claimedModelUsageSourceKeys,
587
+ emittedModelUsageSourceKeys,
588
+ },
589
+ };
590
+ }