@opengeni/worker-bundle 0.17.2 → 0.20.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/dist/activities/agent-run-admission.d.ts +12 -0
  2. package/dist/activities/agent-turn.d.ts +49 -1
  3. package/dist/activities/browser-state-artifact-reaper.d.ts +29 -0
  4. package/dist/activities/capabilities.d.ts +6 -0
  5. package/dist/activities/codex-capacity.d.ts +1 -0
  6. package/dist/activities/environment.d.ts +7 -1
  7. package/dist/activities/packs.d.ts +8 -10
  8. package/dist/activities/run-input.d.ts +8 -0
  9. package/dist/activities/session-state.d.ts +7 -3
  10. package/dist/activities/site-auth-maintenance.d.ts +27 -0
  11. package/dist/activities/types.d.ts +43 -0
  12. package/dist/activities/video-generation-admission.d.ts +9 -4
  13. package/dist/activities/video-generation-credential.d.ts +17 -0
  14. package/dist/activities/video-reference-staging.d.ts +7 -1
  15. package/dist/activities/xai-auth.d.ts +23 -0
  16. package/dist/activities/xai-image-generation.d.ts +26 -0
  17. package/dist/activities/xai-video-generation.d.ts +18 -0
  18. package/dist/{activities-control-44KBWLSM.js → activities-control-SMKF2OBE.js} +438 -55
  19. package/dist/activities-control-SMKF2OBE.js.map +1 -0
  20. package/dist/activities-control.d.ts +10 -2
  21. package/dist/{activities-turn-K4CEDEGJ.js → activities-turn-IDS3WEHA.js} +6704 -4847
  22. package/dist/activities-turn-IDS3WEHA.js.map +1 -0
  23. package/dist/activities.d.ts +11 -3
  24. package/dist/{chunk-E3RI3I33.js → chunk-N5TJIFCE.js} +156 -10
  25. package/dist/chunk-N5TJIFCE.js.map +1 -0
  26. package/dist/{chunk-KKCRGAGZ.js → chunk-P6Z2SLAA.js} +99 -98
  27. package/dist/chunk-P6Z2SLAA.js.map +1 -0
  28. package/dist/index.d.ts +14 -1
  29. package/dist/index.js +49 -2
  30. package/dist/index.js.map +1 -1
  31. package/dist/model-checkpoint-memory-collector.d.ts +1 -0
  32. package/dist/model-context-contributions.d.ts +40 -0
  33. package/dist/observability-metrics.d.ts +57 -1
  34. package/dist/sandbox-routing.d.ts +4 -0
  35. package/dist/workflow-bundle.js +232 -18
  36. package/dist/workflows/activities.d.ts +1 -1
  37. package/dist/workflows/session.d.ts +17 -1
  38. package/dist/workflows/site-auth-maintenance.d.ts +2 -0
  39. package/dist/workflows.d.ts +1 -0
  40. package/package.json +16 -15
  41. package/src/activities/agent-run-admission.ts +76 -0
  42. package/src/activities/agent-turn.ts +2777 -1241
  43. package/src/activities/browser-state-artifact-reaper.ts +120 -0
  44. package/src/activities/capabilities.ts +10 -0
  45. package/src/activities/codex-capacity.ts +48 -1
  46. package/src/activities/environment.ts +14 -8
  47. package/src/activities/goals.ts +2 -1
  48. package/src/activities/packs.ts +45 -122
  49. package/src/activities/parent-wake.ts +13 -1
  50. package/src/activities/run-input.ts +137 -45
  51. package/src/activities/scheduled-tasks.ts +30 -73
  52. package/src/activities/session-state.ts +87 -5
  53. package/src/activities/site-auth-maintenance.ts +269 -0
  54. package/src/activities/types.ts +44 -0
  55. package/src/activities/video-generation-admission.ts +37 -4
  56. package/src/activities/video-generation-credential.ts +84 -6
  57. package/src/activities/video-generation-reconciliation.ts +212 -19
  58. package/src/activities/video-media-validation.ts +8 -2
  59. package/src/activities/video-reference-staging.ts +121 -34
  60. package/src/activities/xai-auth.ts +122 -0
  61. package/src/activities/xai-image-generation.ts +86 -0
  62. package/src/activities/xai-video-generation.ts +69 -0
  63. package/src/activities-control.ts +4 -0
  64. package/src/activities.ts +7 -0
  65. package/src/index.ts +51 -0
  66. package/src/model-checkpoint-memory-collector.ts +8 -1
  67. package/src/model-context-contributions.ts +159 -0
  68. package/src/observability-metrics.ts +282 -12
  69. package/src/sandbox-routing.ts +13 -0
  70. package/src/workflows/activities.ts +1 -0
  71. package/src/workflows/file-upload-reaper.ts +8 -2
  72. package/src/workflows/session.ts +219 -14
  73. package/src/workflows/site-auth-maintenance.ts +14 -0
  74. package/src/workflows.ts +1 -0
  75. package/dist/activities-control-44KBWLSM.js.map +0 -1
  76. package/dist/activities-turn-K4CEDEGJ.js.map +0 -1
  77. package/dist/chunk-E3RI3I33.js.map +0 -1
  78. package/dist/chunk-KKCRGAGZ.js.map +0 -1
@@ -16,12 +16,26 @@ import {
16
16
  getSessionRootId,
17
17
  getSessionGoal,
18
18
  getHumanInputResumeForEvent,
19
+ getInteractionInterventionResumeForEvent,
19
20
  getSessionHumanInputRequest,
20
21
  installOrReadTurnExecutionPolicyForAttempt,
21
22
  persistAttemptToolCatalog,
22
23
  workspaceCodexSubscriptionActive,
24
+ acquireXaiCredentialLease,
25
+ selectXaiCredentialForUse,
26
+ materializeXaiCredentialForRun,
27
+ resolveXaiProviderAccountAuthoritySnapshotForAcceptance,
28
+ heartbeatXaiCredentialLeaseUntil,
29
+ releaseXaiCredentialLease,
30
+ getXaiSessionAccountPin,
31
+ setXaiSessionAccountPin,
32
+ recordXaiSessionLastAccount,
33
+ updateXaiQuotaMetadata,
34
+ XAI_CREDENTIAL_LEASE_TTL_MS,
23
35
  acquireCodexCredentialLease,
24
36
  armCodexCapacityWait,
37
+ armXaiCapacityWait,
38
+ reconcileXaiCapacityWait,
25
39
  heartbeatCodexCredentialLeaseUntil,
26
40
  releaseCodexCredentialLease,
27
41
  CODEX_CREDENTIAL_LEASE_TTL_MS,
@@ -73,6 +87,7 @@ import {
73
87
  SandboxImageConflictError,
74
88
  ActiveSessionHistoryLimitExceededError,
75
89
  ApprovalRunStateLimitExceededError,
90
+ isRetryableDatabaseTransportFailure,
76
91
  isSessionEventPersistenceError,
77
92
  getEnrollment,
78
93
  abandonRecordingForTurnAttempt,
@@ -105,11 +120,14 @@ import {
105
120
  normalizeModelCallUsage,
106
121
  normalizeSdkEvent,
107
122
  normalizeProtocolJsonValue,
123
+ compactMcpResultCustomDataRunState,
124
+ releaseMcpResultCustomDataFromSdkEvent,
108
125
  projectHistoryForProvider,
109
126
  restoreGenericDispatchHistoryItems,
110
127
  sanitizeHistoryItemsForModel,
111
128
  projectModelInputForCapabilities,
112
129
  appendPersistentSessionSettings,
130
+ appendSessionGoal,
113
131
  appendSessionInstructions,
114
132
  appendWorkspaceGovernance,
115
133
  appendWorkspaceMemory,
@@ -166,6 +184,7 @@ import {
166
184
  import { connectionTokenResolverForTurn } from "./mcp-credentials";
167
185
  import { buildApiIntegrationServersForTurn } from "./api-integrations";
168
186
  import {
187
+ allowedFirstPartyMcpToolsForSession,
169
188
  assertTurnExecutionPolicyMatchesConfigV1,
170
189
  calculateGatewayReportedCostBreakdown,
171
190
  calculateModelUsageCostBreakdown,
@@ -175,11 +194,13 @@ import {
175
194
  sandboxLifecycleTransitionWaitMs,
176
195
  sandboxWarmRateMicrosPerSecond,
177
196
  serviceTierForLatencyMode,
197
+ environmentsEncryptionKeyBytes,
178
198
  settingsWithResolvedModelContext,
179
199
  resolveTurnExecutionPolicyV1,
180
200
  OPENGENI_GATEWAY_MODELS,
181
201
  OPENGENI_GATEWAY_PROVIDER_ID,
182
202
  WORKSPACE_GATEWAY_PROVIDER_ID,
203
+ codemodeWorkspaceUrl,
183
204
  isDirectOpenAiApiBaseUrl,
184
205
  resolveModelProvider,
185
206
  type ModelUsageInput,
@@ -193,6 +214,7 @@ import {
193
214
  settingsWithEnabledCapabilityMcpServers,
194
215
  settingsWithSessionMcpServersForRun,
195
216
  settingsWithWorkspaceGatewayCredential,
217
+ withXaiSubscriptionProvider,
196
218
  } from "./capabilities";
197
219
  import {
198
220
  CODEX_USAGE_EXHAUSTED_PCT,
@@ -239,6 +261,15 @@ import {
239
261
  type CodexUsageHeaderSnapshot,
240
262
  } from "@opengeni/codex";
241
263
  import { mergeResourceRefs } from "./common";
264
+ import {
265
+ fetchXaiSubscriptionQuota,
266
+ isXaiSubscriptionTransportError,
267
+ XaiSubscriptionReloginRequired,
268
+ xaiSubscriptionRequestStorage,
269
+ type XaiSubscriptionRequestContext,
270
+ } from "@opengeni/xai-subscription";
271
+ import { buildXaiTurnRequestAuthorization } from "./xai-auth";
272
+ import { executeXaiSubscriptionImageGeneration } from "./xai-image-generation";
242
273
  import {
243
274
  assertSessionAllowsProductModel,
244
275
  defaultSessionMcpServerIds,
@@ -254,8 +285,10 @@ import { TurnAttemptFencedError } from "./turn-attempt-fenced";
254
285
  import {
255
286
  admitVideoGenerationRequest,
256
287
  managedVideoGenerationCredentialLease,
288
+ xaiVideoGenerationCredentialLease,
257
289
  type VideoGenerationCredentialLease,
258
290
  } from "./video-generation-admission";
291
+ import { VideoReferenceInputError } from "./video-reference-staging";
259
292
  import {
260
293
  assertGitCredentialRenewalTransportUnchanged,
261
294
  gitCredentialAuthorityForTurn,
@@ -292,7 +325,7 @@ import {
292
325
  mergeRigDefaultVariableSetEnvironment,
293
326
  rigProviderImageSourceImage,
294
327
  resolveWorkspacePackRuntime,
295
- resolveWorkspaceSkillLibraryRuntime,
328
+ resolveWorkspaceInstalledSkillRuntime,
296
329
  settingsWithPackSandboxImage,
297
330
  settingsWithRigImage,
298
331
  settingsWithRigProviderImage,
@@ -312,6 +345,7 @@ import {
312
345
  import type {
313
346
  TurnActivityServices as ActivityServices,
314
347
  EscapedMcpTimeoutRecoveryDetail,
348
+ PreClaimFailureDetail,
315
349
  RunAgentTurnInput,
316
350
  RunAgentTurnResult,
317
351
  SessionAttemptQuiescenceProof,
@@ -319,6 +353,8 @@ import type {
319
353
  import {
320
354
  ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
321
355
  ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
356
+ PRE_CLAIM_FAILURE_MESSAGE,
357
+ PRE_CLAIM_FAILURE_TYPE,
322
358
  } from "./types";
323
359
  import {
324
360
  resumeBoxForTurn,
@@ -345,13 +381,20 @@ import {
345
381
  recordCreditMicros,
346
382
  recordModelCacheTokens,
347
383
  recordModelInputTokens,
384
+ recordModelRequestPhase,
385
+ recordCompanyBrainContributions,
348
386
  recordSessionEventAppendLatency,
349
387
  recordSessionEventPublishLatency,
388
+ recordTurnSandboxEstablishPolicy,
389
+ recordTurnStartupPhase,
390
+ recordTurnWorkerPreparationTotal,
350
391
  runtimeMetricsHooksForObservability,
351
392
  StreamTimingMetrics,
352
393
  turnLifecycleMetricsFor,
353
394
  type TurnOutcome,
395
+ type TurnSandboxEstablishReason,
354
396
  } from "../observability-metrics";
397
+ import { buildCompanyBrainContributionReceipt } from "../model-context-contributions";
355
398
  import {
356
399
  beginRecording,
357
400
  discardUnpublishedRecording,
@@ -403,11 +446,12 @@ import {
403
446
  } from "@opengeni/runtime";
404
447
  import {
405
448
  CAPABILITY_DESCRIPTORS,
406
- DEFAULT_FIRST_PARTY_MCP_TOOLS,
407
449
  evaluateWorkspaceModelPolicy,
408
450
  readTurnExecutionPolicyV1,
409
451
  resolveWorkspaceAgentHumanInputEnabled,
452
+ resolveWorkspaceMemoryPromptMode,
410
453
  resourceMountPath,
454
+ VideoGenerationRejectedResult,
411
455
  type LatencyMode,
412
456
  type ResourceRef,
413
457
  type RetainedArtifactMetadata,
@@ -418,6 +462,7 @@ import {
418
462
  type SessionTurn,
419
463
  type ToolAuthNeededPayload,
420
464
  type TurnExecutionPolicyV1,
465
+ type XaiProviderAccountAuthoritySnapshotV1,
421
466
  } from "@opengeni/contracts";
422
467
  import { createHash, randomUUID } from "node:crypto";
423
468
  import { createModelCheckpointMemoryCollector } from "../model-checkpoint-memory-collector";
@@ -496,6 +541,7 @@ export function shouldPublishToolAuthNeededForTurn(
496
541
  export function turnExecutionPolicyBillingIdentity(policy: TurnExecutionPolicyV1): {
497
542
  externallyBilled: boolean;
498
543
  codexSubscription: boolean;
544
+ xaiSubscription: boolean;
499
545
  } {
500
546
  return {
501
547
  externallyBilled: policy.billing.metering === "external",
@@ -503,6 +549,10 @@ export function turnExecutionPolicyBillingIdentity(policy: TurnExecutionPolicyV1
503
549
  policy.providerId === "codex-subscription" &&
504
550
  policy.credentialSource.kind === "connected_subscription" &&
505
551
  policy.credentialSource.provider === "codex",
552
+ xaiSubscription:
553
+ policy.providerId === "supergrok-subscription" &&
554
+ policy.credentialSource.kind === "connected_subscription" &&
555
+ policy.credentialSource.provider === "xai",
506
556
  };
507
557
  }
508
558
 
@@ -662,6 +712,52 @@ export function escapedMcpTimeoutRecoveryFailure(input: {
662
712
  });
663
713
  }
664
714
 
715
+ /**
716
+ * Convert the atomic claim transaction's failure into a small, stable
717
+ * Temporal wire contract. The original error remains in activity diagnostics,
718
+ * but SQL text, parameters, and arbitrary invariant messages never enter
719
+ * workflow history. Contention and operational database unavailability are
720
+ * safe to re-read after backoff because the claim transaction contains no
721
+ * model/tool effects; permanent database and state failures require terminal
722
+ * settlement.
723
+ */
724
+ export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
725
+ const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
726
+ const sqlState = persistenceFailure?.details.sqlState ?? null;
727
+ const rawTransportFailure = sqlState === null && isRetryableDatabaseTransportFailure(error);
728
+ // The database transaction itself retries only the two contention failures
729
+ // proven safe for immediate replay. Once the activity has failed, the
730
+ // workflow may also retry operational outages after a durable re-read and
731
+ // bounded delay. Unknown driver/database failures stay recoverable because
732
+ // they commonly represent a lost connection; known constraint, auth, and
733
+ // application SQLSTATEs are permanent and must not create an infinite loop.
734
+ const retryable = Boolean(
735
+ rawTransportFailure ||
736
+ (persistenceFailure &&
737
+ (sqlState === null ||
738
+ sqlState.startsWith("08") ||
739
+ sqlState.startsWith("40") ||
740
+ sqlState.startsWith("53") ||
741
+ sqlState === "55P03" ||
742
+ sqlState === "57014" ||
743
+ sqlState === "57P01" ||
744
+ sqlState === "57P02" ||
745
+ sqlState === "57P03" ||
746
+ sqlState.startsWith("58"))),
747
+ );
748
+ const detail: PreClaimFailureDetail = {
749
+ disposition: retryable ? "retryable" : "permanent",
750
+ code:
751
+ persistenceFailure?.details.code ?? (rawTransportFailure ? "db_failure" : "claim_invariant"),
752
+ };
753
+ return ApplicationFailure.create({
754
+ message: PRE_CLAIM_FAILURE_MESSAGE,
755
+ type: PRE_CLAIM_FAILURE_TYPE,
756
+ nonRetryable: true,
757
+ details: [detail],
758
+ });
759
+ }
760
+
665
761
  /**
666
762
  * Resolve which Codex account a turn runs on (multi-account P1): session-pin >
667
763
  * workspace-active. No rotation in P1. The selected id must still be in the
@@ -687,6 +783,12 @@ export function filterUnmaterializedSandboxFileDownloads(
687
783
  return downloads.filter((download) => !materializedFileIds.has(download.fileId));
688
784
  }
689
785
 
786
+ export function sandboxFileMaterializationOutcome(
787
+ failures: readonly SandboxFileDownloadFailure[],
788
+ ): "completed" | "failed" {
789
+ return failures.length === 0 ? "completed" : "failed";
790
+ }
791
+
690
792
  /** Fixed-length one-way tenant correlation for metrics/alerts; never a raw id. */
691
793
  export function codexWorkspaceMetricKey(workspaceId: string): string {
692
794
  return createHash("sha256").update(workspaceId).digest("hex").slice(0, 12);
@@ -714,6 +816,55 @@ export function stableHumanInputRequestId(
714
816
  return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice(12, 16)}-${value.slice(16, 20)}-${value.slice(20)}`;
715
817
  }
716
818
 
819
+ export function stableInteractionInterventionId(
820
+ sessionId: string,
821
+ turnId: string,
822
+ toolCallId: string,
823
+ ): string {
824
+ return stableInteractionInterventionUuid(
825
+ "opengeni-interaction-intervention-v1",
826
+ sessionId,
827
+ turnId,
828
+ toolCallId,
829
+ );
830
+ }
831
+
832
+ export function stableInteractionInterventionOperationId(
833
+ sessionId: string,
834
+ turnId: string,
835
+ toolCallId: string,
836
+ ): string {
837
+ return stableInteractionInterventionUuid(
838
+ "opengeni-interaction-intervention-operation-v1",
839
+ sessionId,
840
+ turnId,
841
+ toolCallId,
842
+ );
843
+ }
844
+
845
+ function stableInteractionInterventionUuid(
846
+ namespace: string,
847
+ sessionId: string,
848
+ turnId: string,
849
+ toolCallId: string,
850
+ ): string {
851
+ const hex = createHash("sha256")
852
+ .update(namespace)
853
+ .update("\0")
854
+ .update(sessionId)
855
+ .update("\0")
856
+ .update(turnId)
857
+ .update("\0")
858
+ .update(toolCallId)
859
+ .digest("hex")
860
+ .slice(0, 32)
861
+ .split("");
862
+ hex[12] = "5";
863
+ hex[16] = ["8", "9", "a", "b"][Number.parseInt(hex[16] ?? "0", 16) % 4] ?? "8";
864
+ const value = hex.join("");
865
+ return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice(12, 16)}-${value.slice(16, 20)}-${value.slice(20)}`;
866
+ }
867
+
717
868
  /**
718
869
  * True when this activity attempt was cancelled because its hosting worker is
719
870
  * shutting down gracefully (SIGTERM during a deploy), as opposed to a
@@ -1898,6 +2049,42 @@ export function managedSandboxOwnershipForTurn(
1898
2049
  };
1899
2050
  }
1900
2051
 
2052
+ /** A sandboxless home still establishes an explicitly attached Connected Machine. */
2053
+ export function shouldEstablishSandboxForTurn(
2054
+ sandboxOwnershipEnabled: boolean,
2055
+ homeBackend: Settings["sandboxBackend"],
2056
+ machinePrimary: boolean,
2057
+ ): boolean {
2058
+ return sandboxOwnershipEnabled && (homeBackend !== "none" || machinePrimary);
2059
+ }
2060
+
2061
+ export function sandboxEstablishPolicyDecision(input: {
2062
+ lazyEnabled: boolean;
2063
+ machinePrimary: boolean;
2064
+ sandboxBackend: Settings["sandboxBackend"];
2065
+ hasInitialRunCredentialMaterial: boolean;
2066
+ generatedVideoFileCount: number;
2067
+ hasSignedFileResources: boolean;
2068
+ }): { policy: "eager" | "on-demand"; reason: TurnSandboxEstablishReason } {
2069
+ if (!input.lazyEnabled) return { policy: "eager", reason: "lazy_disabled" };
2070
+ if (input.machinePrimary) return { policy: "eager", reason: "machine_primary" };
2071
+ if (input.sandboxBackend === "none") return { policy: "eager", reason: "backend_none" };
2072
+ if (input.hasInitialRunCredentialMaterial) {
2073
+ return { policy: "eager", reason: "initial_run_credentials" };
2074
+ }
2075
+ if (input.generatedVideoFileCount > 0) {
2076
+ return { policy: "eager", reason: "generated_video_files" };
2077
+ }
2078
+ // Signed file downloads must finish before the first model boundary so any
2079
+ // integrity/download failure is present in the prepared input. Deferring the
2080
+ // box until the first tool call would make that first request advertise a
2081
+ // path whose materialization outcome is not known yet.
2082
+ if (input.hasSignedFileResources) {
2083
+ return { policy: "eager", reason: "signed_file_resources" };
2084
+ }
2085
+ return { policy: "on-demand", reason: "eligible" };
2086
+ }
2087
+
1901
2088
  /**
1902
2089
  * Classify a persisted active-sandbox pointer for TURN-START RECONCILE (issue #341
1903
2090
  * invariant B). Returns the typed reason to RESET the pointer to the session HOME,
@@ -2194,9 +2381,12 @@ export function structuredToolTransportForTurn(
2194
2381
  resolvedModel: { provider: { kind: RegistryProviderKind } } | null,
2195
2382
  ): boolean {
2196
2383
  if (!resolvedModel) return true;
2197
- return !["codex-subscription", "vercel-gateway-managed", "vercel-gateway-workspace"].includes(
2198
- resolvedModel.provider.kind,
2199
- );
2384
+ return ![
2385
+ "codex-subscription",
2386
+ "xai-subscription",
2387
+ "vercel-gateway-managed",
2388
+ "vercel-gateway-workspace",
2389
+ ].includes(resolvedModel.provider.kind);
2200
2390
  }
2201
2391
 
2202
2392
  /**
@@ -2243,6 +2433,21 @@ export function hostedWebSearchForTurn(
2243
2433
  return resolvedModel?.configured.hostedWebSearch ?? deploymentWebSearchEnabled;
2244
2434
  }
2245
2435
 
2436
+ /**
2437
+ * Image generation is an optional connected-account capability. A delegated
2438
+ * subscription may still authorize the text model without carrying the local
2439
+ * credential identity required for paid image operations; that must narrow the
2440
+ * tool catalog, not reject an otherwise valid turn.
2441
+ */
2442
+ export function connectedSubscriptionImageGenerationAuthority<T>(
2443
+ credentialContext: T | null | undefined,
2444
+ credentialId: string | null | undefined,
2445
+ ): { credentialContext: T; credentialId: string } | null {
2446
+ if (credentialContext == null || typeof credentialId !== "string" || credentialId.length === 0)
2447
+ return null;
2448
+ return { credentialContext, credentialId };
2449
+ }
2450
+
2246
2451
  /** Direct OpenAI hosted image IDs are reusable only by the exact API credential. */
2247
2452
  export function openAiHostedImageProviderBindingForTurn(
2248
2453
  settings: Pick<Settings, "openaiProvider" | "openaiApiKey" | "openaiBaseUrl">,
@@ -2676,6 +2881,10 @@ export async function finalizeDurableTurnOpStreams(
2676
2881
 
2677
2882
  export function createRunAgentTurnActivity(services: () => Promise<ActivityServices>) {
2678
2883
  const modelCheckpointMemoryCollector = createModelCheckpointMemoryCollector();
2884
+ // Keep a distinct cooldown for terminal collection. A collection at the
2885
+ // final model checkpoint happens while the activity still owns its complete
2886
+ // turn graph and must not suppress collection after that graph is released.
2887
+ const turnCompletionMemoryCollector = createModelCheckpointMemoryCollector();
2679
2888
  return async function runAgentTurn(input: RunAgentTurnInput): Promise<RunAgentTurnResult> {
2680
2889
  const {
2681
2890
  settings,
@@ -2767,6 +2976,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2767
2976
  );
2768
2977
  const sandboxOperationObserver = sandboxOperationMetricObserver(observability);
2769
2978
  let isCodexTurn = false;
2979
+ let isXaiTurn = false;
2770
2980
  let isExternallyBilledTurn = false;
2771
2981
  let executionGeneration = 0;
2772
2982
  let providerRecoveryCount = 0;
@@ -2898,6 +3108,83 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2898
3108
  }, 60_000);
2899
3109
  codexLeaseHeartbeatTimer.unref?.();
2900
3110
  };
3111
+ let xaiLeaseHeld = false;
3112
+ let xaiLeaseLost = false;
3113
+ let xaiLeaseSubjectId: string | null = null;
3114
+ let xaiLeaseHolderId: string | null = null;
3115
+ let xaiLeaseGeneration: number | null = null;
3116
+ let xaiLeaseConfirmedUntilMs: number | null = null;
3117
+ let xaiLeaseHeartbeatInFlight = false;
3118
+ let xaiLeaseHeartbeatTimer: ReturnType<typeof setInterval> | undefined;
3119
+ const markXaiLeaseLost = (reason: "deadline" | "not_found"): void => {
3120
+ if (xaiLeaseLost) return;
3121
+ xaiLeaseLost = true;
3122
+ observability.warn("xAI credential lease was lost during an active turn", {
3123
+ workspaceId: input.workspaceId,
3124
+ turnId,
3125
+ reason,
3126
+ });
3127
+ };
3128
+ const renewXaiLease = async (): Promise<void> => {
3129
+ if (
3130
+ !turnId ||
3131
+ !xaiLeaseHeld ||
3132
+ !xaiLeaseSubjectId ||
3133
+ !xaiLeaseHolderId ||
3134
+ xaiLeaseGeneration === null ||
3135
+ xaiLeaseLost
3136
+ ) {
3137
+ return;
3138
+ }
3139
+ if (codexCredentialLeaseDeadlineExpired(xaiLeaseConfirmedUntilMs)) {
3140
+ markXaiLeaseLost("deadline");
3141
+ return;
3142
+ }
3143
+ if (xaiLeaseHeartbeatInFlight) return;
3144
+ xaiLeaseHeartbeatInFlight = true;
3145
+ const renewalStartedAtMs = performance.now();
3146
+ try {
3147
+ const renewedUntil = await heartbeatXaiCredentialLeaseUntil(db, {
3148
+ workspaceId: input.workspaceId,
3149
+ subjectId: xaiLeaseSubjectId,
3150
+ turnId,
3151
+ holderId: xaiLeaseHolderId,
3152
+ generation: xaiLeaseGeneration,
3153
+ leaseTtlMs: XAI_CREDENTIAL_LEASE_TTL_MS,
3154
+ });
3155
+ if (!renewedUntil) {
3156
+ markXaiLeaseLost("not_found");
3157
+ } else {
3158
+ xaiLeaseConfirmedUntilMs = renewalStartedAtMs + XAI_CREDENTIAL_LEASE_TTL_MS;
3159
+ }
3160
+ } catch (error) {
3161
+ if (codexCredentialLeaseDeadlineExpired(xaiLeaseConfirmedUntilMs)) {
3162
+ markXaiLeaseLost("deadline");
3163
+ return;
3164
+ }
3165
+ observability.warn("xAI credential lease heartbeat failed", {
3166
+ workspaceId: input.workspaceId,
3167
+ turnId,
3168
+ ...safeErrorDiagnostic(error),
3169
+ });
3170
+ } finally {
3171
+ xaiLeaseHeartbeatInFlight = false;
3172
+ }
3173
+ };
3174
+ const startXaiLeaseHeartbeat = (): void => {
3175
+ if (!turnId || xaiLeaseHeartbeatTimer) return;
3176
+ xaiLeaseHeartbeatTimer = setInterval(() => {
3177
+ void renewXaiLease();
3178
+ }, 60_000);
3179
+ xaiLeaseHeartbeatTimer.unref?.();
3180
+ };
3181
+ const renewServingCredentialLease = async (
3182
+ reason: "timer" | "runtime_event" | "model_usage",
3183
+ ): Promise<void> => {
3184
+ await renewCodexLease(reason);
3185
+ await renewXaiLease();
3186
+ };
3187
+ const servingCredentialLeaseLost = (): boolean => codexLeaseLost || xaiLeaseLost;
2901
3188
  // P1.2 ownership inversion: when sandboxOwnershipEnabled, the turn resolves
2902
3189
  // the one box by id from the group lease and injects it NON-OWNED into the
2903
3190
  // run. null when the flag is off (byte-for-byte the legacy build-and-discard
@@ -3670,6 +3957,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3670
3957
  compactRetainedScreenshotRunState(serialized, retainedScreenshotReceiptsByCallId),
3671
3958
  generatedImageReceiptsByProviderItemId,
3672
3959
  );
3960
+ const compactApprovalRunState = (serialized: string): string =>
3961
+ compactMcpResultCustomDataRunState(compactMediaRunState(serialized));
3673
3962
  // Explicit image-producing tools cross the durable session-media boundary.
3674
3963
  // Incidental frames returned by click/scroll actions remain unretained; an
3675
3964
  // intentional computer_screenshot or view_image result must never be lost
@@ -3847,6 +4136,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3847
4136
  // The Codex account this turn runs on (pin > workspace active), resolved once
3848
4137
  // a codex-billed turn is confirmed and threaded into the token resolver below.
3849
4138
  let effectiveCodexCredentialId: string | null = null;
4139
+ let effectiveXaiCredentialId: string | null = null;
4140
+ let xaiRotationEnabled = false;
4141
+ let xaiAuthoritySnapshot: XaiProviderAccountAuthoritySnapshotV1 | null = null;
4142
+ let xaiRequestContext: XaiSubscriptionRequestContext | null = null;
4143
+ let xaiCredentialQuarantined = false;
3850
4144
  // The session's Codex credential BEFORE this turn resolved its own — captured
3851
4145
  // before recordSessionActiveCodexCredential overwrites the durable pointer, so
3852
4146
  // a per-call usage log can report whether the serving account CHANGED since the
@@ -3866,7 +4160,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3866
4160
  // only exists in the RunState blob), never through a swapped trigger.
3867
4161
  let triggerType: string | null = null;
3868
4162
  try {
3869
- const session = await requireSession(db, input.workspaceId, input.sessionId);
3870
4163
  const claim = await claimSessionWorkForAttempt(db, input.workspaceId, {
3871
4164
  sessionId: input.sessionId,
3872
4165
  workflowId: input.workflowId,
@@ -3881,6 +4174,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3881
4174
  }
3882
4175
  const turn = claim.turn;
3883
4176
  turnId = turn.id;
4177
+ // Establish durable attempt ownership before any later read can fail.
4178
+ // Therefore every failure with no turnId came from the one atomic claim
4179
+ // transaction and can be classified without conflating ordinary runtime
4180
+ // or transport failures with admission failures.
4181
+ const session = await requireSession(db, input.workspaceId, input.sessionId);
3884
4182
  executionGeneration = turn.executionGeneration;
3885
4183
  providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
3886
4184
  let installedApiIntegrations: readonly ApiIntegrationRuntime[] = [];
@@ -3914,10 +4212,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3914
4212
  mcpSettings,
3915
4213
  codexSubscriptionActive,
3916
4214
  );
4215
+ const xaiSettings = codexSettings.supergrokSubscriptionEnabled
4216
+ ? withXaiSubscriptionProvider(codexSettings)
4217
+ : codexSettings;
3917
4218
  const capabilitySettings = await settingsWithWorkspaceGatewayCredential(
3918
4219
  db,
3919
4220
  input.workspaceId,
3920
- codexSettings,
4221
+ xaiSettings,
3921
4222
  );
3922
4223
  const codexAppsCredentialId = capabilitySettings.codexConnectedAppsEnabled
3923
4224
  ? await resolveCodexAppsCredentialIdForRun(db, input.workspaceId)
@@ -3956,6 +4257,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3956
4257
  const billingIdentity = turnExecutionPolicyBillingIdentity(turnExecutionPolicy);
3957
4258
  isExternallyBilledTurn = billingIdentity.externallyBilled;
3958
4259
  isCodexTurn = billingIdentity.codexSubscription;
4260
+ isXaiTurn = billingIdentity.xaiSubscription;
3959
4261
  triggerEventId = turn.triggerEventId;
3960
4262
  const trigger = await getSessionEvent(db, input.workspaceId, triggerEventId);
3961
4263
  if (!trigger) {
@@ -3967,6 +4269,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3967
4269
  input.sessionId,
3968
4270
  trigger,
3969
4271
  );
4272
+ const interactionInterventionResume = await getInteractionInterventionResumeForEvent(
4273
+ db,
4274
+ input.workspaceId,
4275
+ input.sessionId,
4276
+ trigger,
4277
+ );
3970
4278
  triggerType = trigger.type;
3971
4279
  redispatchesAtDispatch = Number(
3972
4280
  (turn.metadata as { workerDeathRedispatches?: number } | null)?.workerDeathRedispatches ??
@@ -4065,6 +4373,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4065
4373
  if (inputs.length > 0 && !appended.accepted) {
4066
4374
  throw new TurnAttemptFencedError("turn execution generation was fenced");
4067
4375
  }
4376
+ if (inputs.length > 0) {
4377
+ turnLifecycleMetricsFor(observability).progress(turnId!);
4378
+ }
4068
4379
  activityContext?.heartbeat({
4069
4380
  ...heartbeatDetails,
4070
4381
  phase: "events_published",
@@ -4158,6 +4469,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4158
4469
  turnMetricOutcome = "cancelled";
4159
4470
  return false;
4160
4471
  }
4472
+ turnLifecycleMetricsFor(observability).progress(turnId!);
4161
4473
  if (recordingForSettlement && preparedRecording) {
4162
4474
  if (result.recordingMutationApplied) {
4163
4475
  activeRecording = null;
@@ -4189,6 +4501,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4189
4501
  // on a healthy worker.
4190
4502
  throwIfWorkerShuttingDown();
4191
4503
  throwIfTurnCancelled();
4504
+ recordTurnStartupPhase(observability, {
4505
+ phase: "claim_and_policy",
4506
+ provider: turnExecutionPolicy.providerId,
4507
+ backend: turn.sandboxBackend,
4508
+ outcome: "completed",
4509
+ durationSeconds: (performance.now() - activityStarted) / 1_000,
4510
+ });
4511
+ const turnStartSettlementStartedAt = performance.now();
4192
4512
  if (
4193
4513
  !(await settle({
4194
4514
  events: [
@@ -4205,7 +4525,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4205
4525
  ) {
4206
4526
  return claimedResult({ status: "cancelled" });
4207
4527
  }
4528
+ recordTurnStartupPhase(observability, {
4529
+ phase: "turn_start_settlement",
4530
+ provider: turnExecutionPolicy.providerId,
4531
+ backend: turn.sandboxBackend,
4532
+ outcome: "completed",
4533
+ durationSeconds: (performance.now() - turnStartSettlementStartedAt) / 1_000,
4534
+ count: 2,
4535
+ });
4208
4536
  turnStartedPublished = true;
4537
+ const workerPreparationStartedAt = performance.now();
4209
4538
 
4210
4539
  // Multi-account (P1): resolve the effective Codex account for this turn
4211
4540
  // (session-pin > workspace active) and stamp it on the session so the
@@ -4213,80 +4542,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4213
4542
  // when it changed from the prior run's account so the pill flips live.
4214
4543
  // Gated on the codex-billed predicate — non-codex turns never touch this.
4215
4544
  if (isCodexTurn) {
4216
- const sessionCodex = await getSessionCodexState(db, input.workspaceId, input.sessionId);
4217
- const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
4218
- const sessionPinSource = sessionCodex?.pinSource ?? null;
4219
- const selectForTurn = (context: CodexCredentialLeaseSelectionContext) =>
4220
- selectCodexCredentialLeaseForTurn({
4221
- context,
4222
- leasingEnabled: settings.codexCredentialLeasingEnabled,
4223
- sessionId: input.sessionId,
4224
- sessionPinnedCredentialId: sessionPin,
4225
- sessionPinSource,
4226
- sessionLastCredentialId: sessionCodex?.lastCredentialId ?? null,
4227
- now: new Date(),
4228
- });
4545
+ const credentialSelectionStartedAt = performance.now();
4546
+ let credentialSelectionOutcome: "completed" | "failed" = "completed";
4547
+ try {
4548
+ const sessionCodex = await getSessionCodexState(db, input.workspaceId, input.sessionId);
4549
+ const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
4550
+ const sessionPinSource = sessionCodex?.pinSource ?? null;
4551
+ const selectForTurn = (context: CodexCredentialLeaseSelectionContext) =>
4552
+ selectCodexCredentialLeaseForTurn({
4553
+ context,
4554
+ leasingEnabled: settings.codexCredentialLeasingEnabled,
4555
+ sessionId: input.sessionId,
4556
+ sessionPinnedCredentialId: sessionPin,
4557
+ sessionPinSource,
4558
+ sessionLastCredentialId: sessionCodex?.lastCredentialId ?? null,
4559
+ now: new Date(),
4560
+ });
4229
4561
 
4230
- // Rollout/rollback path is intentionally table-inert. With the flag off,
4231
- // old and new workers both use legacy pin > active-pointer selection and
4232
- // neither reads nor writes the additive lease/cursor schema.
4233
- let leased: CodexCredentialLeaseResult<RotationDecision>;
4234
- let leaseAcquisitionStartedAtMs: number | null = null;
4235
- if (settings.codexCredentialLeasingEnabled) {
4236
- leaseAcquisitionStartedAtMs = performance.now();
4237
- leased = await acquireCodexCredentialLease(
4238
- db,
4239
- {
4240
- accountId: input.accountId,
4241
- workspaceId: input.workspaceId,
4242
- turnId,
4243
- holderId: codexLeaseHolderId,
4244
- advanceActivePointer: sessionPin === null,
4245
- },
4246
- selectForTurn,
4247
- );
4248
- } else {
4249
- const [rotation, accounts] = await Promise.all([
4250
- getCodexRotationSettings(db, input.workspaceId),
4251
- listCodexAccountStatuses(db, input.workspaceId),
4252
- ]);
4253
- const leaseAccounts = accounts.map((account) => ({
4254
- ...account,
4255
- activeLeaseCount: 0,
4256
- selectionCount: 0,
4257
- lastSelectedAt: null,
4258
- }));
4259
- const activeCredentialId = rotation?.activeCredentialId ?? null;
4260
- const selected = selectForTurn({
4261
- accounts: leaseAccounts,
4262
- activeCredentialId,
4263
- rotationEnabled: rotation?.rotationEnabled ?? false,
4264
- leaseRotationEnabled: false,
4265
- rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
4266
- existingCredentialId: null,
4267
- policyScope: null,
4268
- unavailableDiagnostics: [],
4269
- });
4270
- leased = {
4271
- ...selected,
4272
- accounts: leaseAccounts,
4273
- activeCredentialId,
4274
- rotationEnabled: rotation?.rotationEnabled ?? false,
4275
- rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
4276
- reused: false,
4277
- holderId: null,
4278
- generation: null,
4279
- leasedUntil: null,
4280
- unavailableDiagnostics: [],
4281
- advanceActivePointer: selected.advanceActivePointer !== false,
4282
- };
4283
- }
4284
- if (leased.decision.kind === "allCapped") {
4285
- // Bounded self-heal of stale usage cache, then ONE new atomic selection.
4286
- await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
4287
- signalCodexCapacityWorkflow,
4288
- wakeSessionWorkflow,
4289
- });
4562
+ // Rollout/rollback path is intentionally table-inert. With the flag off,
4563
+ // old and new workers both use legacy pin > active-pointer selection and
4564
+ // neither reads nor writes the additive lease/cursor schema.
4565
+ let leased: CodexCredentialLeaseResult<RotationDecision>;
4566
+ let leaseAcquisitionStartedAtMs: number | null = null;
4290
4567
  if (settings.codexCredentialLeasingEnabled) {
4291
4568
  leaseAcquisitionStartedAtMs = performance.now();
4292
4569
  leased = await acquireCodexCredentialLease(
@@ -4311,9 +4588,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4311
4588
  selectionCount: 0,
4312
4589
  lastSelectedAt: null,
4313
4590
  }));
4591
+ const activeCredentialId = rotation?.activeCredentialId ?? null;
4314
4592
  const selected = selectForTurn({
4315
4593
  accounts: leaseAccounts,
4316
- activeCredentialId: rotation?.activeCredentialId ?? null,
4594
+ activeCredentialId,
4317
4595
  rotationEnabled: rotation?.rotationEnabled ?? false,
4318
4596
  leaseRotationEnabled: false,
4319
4597
  rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
@@ -4324,7 +4602,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4324
4602
  leased = {
4325
4603
  ...selected,
4326
4604
  accounts: leaseAccounts,
4327
- activeCredentialId: rotation?.activeCredentialId ?? null,
4605
+ activeCredentialId,
4328
4606
  rotationEnabled: rotation?.rotationEnabled ?? false,
4329
4607
  rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
4330
4608
  reused: false,
@@ -4335,297 +4613,603 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4335
4613
  advanceActivePointer: selected.advanceActivePointer !== false,
4336
4614
  };
4337
4615
  }
4338
- }
4339
- const rotationDecision = leased.decision;
4340
- const selectedPinDisposition = classifyCodexPin({
4341
- pinnedCredentialId: sessionPin,
4342
- pinSource: sessionPinSource,
4343
- strategy: leased.rotationStrategy as CodexRotationStrategy,
4344
- rotationEnabled: leased.rotationEnabled,
4345
- });
4346
- // pin policy pin persistence follows the atomic credential allocator selection. The selector
4347
- // already ran exact-turn reuse before policy filtering and vetoed pointer
4348
- // movement for manual/policy homes; this write only records the NEXT turn's
4349
- // policy home (or clears a policy pin whose strategy is no longer active).
4350
- if (
4351
- selectedPinDisposition === "sharded" &&
4352
- leased.credentialId !== null &&
4353
- (sessionPinSource !== "policy" || sessionPin !== leased.credentialId)
4354
- ) {
4355
- const pinMutation = await withSessionCodexCapacityMutation(
4356
- db,
4357
- {
4358
- workspaceId: input.workspaceId,
4359
- reason: "codex_policy_pin_changed",
4360
- },
4361
- async (tx) => {
4362
- const changed = await setSessionCodexPinInTransaction(
4363
- tx,
4364
- input.workspaceId,
4365
- input.sessionId,
4366
- leased.credentialId,
4367
- "policy",
4368
- {
4369
- expected: {
4370
- pinnedCredentialId: sessionPin,
4371
- pinSource: sessionPinSource,
4372
- },
4373
- },
4374
- );
4375
- return { result: changed, changed };
4376
- },
4377
- );
4378
- await signalCodexCapacityWakeTargets(
4379
- { signalCodexCapacityWorkflow, wakeSessionWorkflow },
4380
- pinMutation.wakeTargets,
4381
- );
4382
- } else if (selectedPinDisposition === "clearStale") {
4383
- const pinMutation = await withSessionCodexCapacityMutation(
4384
- db,
4385
- {
4386
- workspaceId: input.workspaceId,
4387
- reason: "codex_stale_policy_pin_cleared",
4388
- },
4389
- async (tx) => {
4390
- const changed = await setSessionCodexPinInTransaction(
4391
- tx,
4392
- input.workspaceId,
4393
- input.sessionId,
4394
- null,
4395
- "policy",
4616
+ if (leased.decision.kind === "allCapped") {
4617
+ // Bounded self-heal of stale usage cache, then ONE new atomic selection.
4618
+ await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
4619
+ signalCodexCapacityWorkflow,
4620
+ wakeSessionWorkflow,
4621
+ });
4622
+ if (settings.codexCredentialLeasingEnabled) {
4623
+ leaseAcquisitionStartedAtMs = performance.now();
4624
+ leased = await acquireCodexCredentialLease(
4625
+ db,
4396
4626
  {
4397
- expected: {
4398
- pinnedCredentialId: sessionPin,
4399
- pinSource: sessionPinSource,
4400
- },
4627
+ accountId: input.accountId,
4628
+ workspaceId: input.workspaceId,
4629
+ turnId,
4630
+ holderId: codexLeaseHolderId,
4631
+ advanceActivePointer: sessionPin === null,
4401
4632
  },
4633
+ selectForTurn,
4402
4634
  );
4403
- return { result: changed, changed };
4404
- },
4405
- );
4406
- await signalCodexCapacityWakeTargets(
4407
- { signalCodexCapacityWorkflow, wakeSessionWorkflow },
4408
- pinMutation.wakeTargets,
4409
- );
4410
- }
4411
- if (
4412
- !settings.codexCredentialLeasingEnabled &&
4413
- leased.advanceActivePointer &&
4414
- sessionPin === null &&
4415
- rotationDecision.kind === "active" &&
4416
- rotationDecision.moved
4417
- ) {
4418
- await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
4419
- }
4420
- effectiveCodexCredentialId = leased.credentialId;
4421
- codexLeaseGeneration = leased.generation;
4422
- codexLeaseConfirmedUntilMs =
4423
- leased.leasedUntil && leaseAcquisitionStartedAtMs !== null
4424
- ? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS
4425
- : null;
4426
- codexLeaseHeld =
4427
- effectiveCodexCredentialId !== null &&
4428
- leased.holderId !== null &&
4429
- leased.generation !== null &&
4430
- codexLeaseConfirmedUntilMs !== null;
4431
- if (codexLeaseHeld) startCodexLeaseHeartbeat();
4432
-
4433
- const actualOutcome = effectiveCodexCredentialId
4434
- ? "selected"
4435
- : rotationDecision.kind === "allCapped"
4436
- ? "waiting"
4437
- : "none";
4438
- const actualReason = effectiveCodexCredentialId
4439
- ? leased.reused
4440
- ? "lease_reused"
4441
- : sessionPin === effectiveCodexCredentialId
4442
- ? "pin"
4443
- : rotationDecision.kind === "active" && rotationDecision.moved
4444
- ? "rotation"
4445
- : "active"
4446
- : rotationDecision.kind === "allCapped"
4447
- ? "all_capped"
4448
- : "none";
4449
- const fencedInFlight = leased.reused;
4450
- const shadowResult = await publishCodexFleetShadowDecisionV1({
4451
- enabled: settings.codexFleetPolicyShadowEnabled,
4452
- decision: {
4453
- accounts: leased.accounts,
4454
- actualCredentialId: effectiveCodexCredentialId,
4455
- actualOutcome,
4456
- actualReason,
4457
- affinityCredentialId: fencedInFlight
4458
- ? effectiveCodexCredentialId
4459
- : (sessionPin ?? sessionCodex?.lastCredentialId ?? null),
4460
- fencedInFlight,
4461
- nearExhaustionPct: settings.codexRotationNearExhaustionPct,
4462
- now: new Date(),
4463
- aliasSeed: randomUUID(),
4464
- },
4465
- publish,
4466
- });
4467
- if (shadowResult.outcome === "published") {
4468
- const shadowPayload = shadowResult.payload;
4469
- observability.incrementCounter({
4470
- name: "opengeni_codex_fleet_shadow_decisions_total",
4471
- help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
4472
- labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload),
4473
- });
4474
- observability.info("Codex adaptive fleet shadow decision", {
4475
- workspaceId: input.workspaceId,
4476
- policyVersion: shadowPayload.replay.policyVersion,
4477
- inputFingerprint: shadowPayload.replay.inputFingerprint,
4478
- decisionFingerprint: shadowPayload.replay.decisionFingerprint,
4479
- actualOutcome: shadowPayload.actual.outcome,
4480
- shadowOutcome: shadowPayload.replay.decision.outcome,
4481
- comparison: shadowPayload.comparison,
4482
- candidateCount: shadowPayload.replay.input.candidates.length,
4483
- truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
4484
- payloadBytes: shadowResult.payloadBytes,
4485
- });
4486
- } else if (shadowResult.outcome === "failed") {
4487
- // Shadow observability is explicitly non-authoritative. A malformed
4488
- // snapshot or event-write fault must never change the authoritative lease,
4489
- // capacity wait, failover, or the account serving this fenced turn.
4490
- observability.incrementCounter({
4491
- name: "opengeni_codex_fleet_shadow_errors_total",
4492
- help: "Shadow decision build/publication failures.",
4493
- labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
4494
- });
4495
- observability.warn("Codex adaptive fleet shadow decision failed open", {
4496
- stage: shadowResult.stage,
4497
- reason: shadowResult.reason,
4498
- errorClass: "CodexFleetShadowOperationError",
4499
- errorCode: "codex_fleet_shadow_failed",
4500
- origin: "worker",
4501
- payloadBytes: shadowResult.payloadBytes,
4502
- });
4503
- }
4504
-
4505
- const eligibleCount = leased.accounts.filter((account) =>
4506
- isCodexCredentialEligible(account, new Date()),
4507
- ).length;
4508
- const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
4509
- observability.incrementCounter({
4510
- name: "opengeni_codex_pool_observations_total",
4511
- help: "Observed eligible Codex pool depth buckets at turn selection.",
4512
- labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
4513
- });
4514
- if (eligibleCount <= 1) {
4515
- observability.incrementCounter({
4516
- name: "opengeni_codex_pool_low_total",
4517
- help: "Alert signal emitted when the eligible Codex pool is zero or one.",
4518
- labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
4519
- });
4520
- observability.warn("Codex eligible credential pool is low", {
4521
- workspaceId: input.workspaceId,
4522
- eligibleCount,
4523
- connectedCount: leased.accounts.length,
4524
- depth: poolDepth,
4635
+ } else {
4636
+ const [rotation, accounts] = await Promise.all([
4637
+ getCodexRotationSettings(db, input.workspaceId),
4638
+ listCodexAccountStatuses(db, input.workspaceId),
4639
+ ]);
4640
+ const leaseAccounts = accounts.map((account) => ({
4641
+ ...account,
4642
+ activeLeaseCount: 0,
4643
+ selectionCount: 0,
4644
+ lastSelectedAt: null,
4645
+ }));
4646
+ const selected = selectForTurn({
4647
+ accounts: leaseAccounts,
4648
+ activeCredentialId: rotation?.activeCredentialId ?? null,
4649
+ rotationEnabled: rotation?.rotationEnabled ?? false,
4650
+ leaseRotationEnabled: false,
4651
+ rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
4652
+ existingCredentialId: null,
4653
+ policyScope: null,
4654
+ unavailableDiagnostics: [],
4655
+ });
4656
+ leased = {
4657
+ ...selected,
4658
+ accounts: leaseAccounts,
4659
+ activeCredentialId: rotation?.activeCredentialId ?? null,
4660
+ rotationEnabled: rotation?.rotationEnabled ?? false,
4661
+ rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
4662
+ reused: false,
4663
+ holderId: null,
4664
+ generation: null,
4665
+ leasedUntil: null,
4666
+ unavailableDiagnostics: [],
4667
+ advanceActivePointer: selected.advanceActivePointer !== false,
4668
+ };
4669
+ }
4670
+ }
4671
+ const rotationDecision = leased.decision;
4672
+ const selectedPinDisposition = classifyCodexPin({
4673
+ pinnedCredentialId: sessionPin,
4674
+ pinSource: sessionPinSource,
4675
+ strategy: leased.rotationStrategy as CodexRotationStrategy,
4676
+ rotationEnabled: leased.rotationEnabled,
4525
4677
  });
4526
- }
4527
-
4528
- if (
4529
- effectiveCodexCredentialId === null &&
4530
- leased.accounts.length > 0 &&
4531
- leased.accounts.every((account) => !account.allocatorEnabled) &&
4532
- turnId
4533
- ) {
4534
- if (turn.source === "compaction") {
4535
- if (
4536
- !(await settle!({
4537
- events: [
4678
+ // pin policy pin persistence follows the atomic credential allocator selection. The selector
4679
+ // already ran exact-turn reuse before policy filtering and vetoed pointer
4680
+ // movement for manual/policy homes; this write only records the NEXT turn's
4681
+ // policy home (or clears a policy pin whose strategy is no longer active).
4682
+ if (
4683
+ selectedPinDisposition === "sharded" &&
4684
+ leased.credentialId !== null &&
4685
+ (sessionPinSource !== "policy" || sessionPin !== leased.credentialId)
4686
+ ) {
4687
+ const pinMutation = await withSessionCodexCapacityMutation(
4688
+ db,
4689
+ {
4690
+ workspaceId: input.workspaceId,
4691
+ reason: "codex_policy_pin_changed",
4692
+ },
4693
+ async (tx) => {
4694
+ const changed = await setSessionCodexPinInTransaction(
4695
+ tx,
4696
+ input.workspaceId,
4697
+ input.sessionId,
4698
+ leased.credentialId,
4699
+ "policy",
4538
4700
  {
4539
- type: "turn.cancelled",
4540
- payload: {
4541
- maintenance: "context_compaction",
4542
- reason: "codex_allocator_disabled",
4543
- requestPreserved: true,
4701
+ expected: {
4702
+ pinnedCredentialId: sessionPin,
4703
+ pinSource: sessionPinSource,
4544
4704
  },
4545
4705
  },
4706
+ );
4707
+ return { result: changed, changed };
4708
+ },
4709
+ );
4710
+ await signalCodexCapacityWakeTargets(
4711
+ { signalCodexCapacityWorkflow, wakeSessionWorkflow },
4712
+ pinMutation.wakeTargets,
4713
+ );
4714
+ } else if (selectedPinDisposition === "clearStale") {
4715
+ const pinMutation = await withSessionCodexCapacityMutation(
4716
+ db,
4717
+ {
4718
+ workspaceId: input.workspaceId,
4719
+ reason: "codex_stale_policy_pin_cleared",
4720
+ },
4721
+ async (tx) => {
4722
+ const changed = await setSessionCodexPinInTransaction(
4723
+ tx,
4724
+ input.workspaceId,
4725
+ input.sessionId,
4726
+ null,
4727
+ "policy",
4546
4728
  {
4547
- type: "session.status.changed",
4548
- payload: { status: "idle" },
4729
+ expected: {
4730
+ pinnedCredentialId: sessionPin,
4731
+ pinSource: sessionPinSource,
4732
+ },
4549
4733
  },
4550
- ],
4551
- turnStatus: "cancelled",
4552
- sessionStatus: "idle",
4553
- activeTurnId: null,
4554
- }))
4555
- ) {
4556
- return claimedResult({ status: "cancelled" });
4557
- }
4558
- turnMetricOutcome = "cancelled";
4559
- activityStatus = "idle";
4560
- return claimedResult({ status: "idle", deferredUntilWake: true });
4734
+ );
4735
+ return { result: changed, changed };
4736
+ },
4737
+ );
4738
+ await signalCodexCapacityWakeTargets(
4739
+ { signalCodexCapacityWorkflow, wakeSessionWorkflow },
4740
+ pinMutation.wakeTargets,
4741
+ );
4561
4742
  }
4562
- const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
4563
- () => null,
4564
- );
4565
- const activeGoal = goal?.status === "active" ? goal : null;
4566
- const armed = await armCodexCapacityWait(db, {
4567
- accountId: input.accountId,
4568
- workspaceId: input.workspaceId,
4569
- sessionId: input.sessionId,
4570
- turnId,
4571
- attemptId: input.attemptId,
4572
- workflowId: input.workflowId,
4573
- goalId: activeGoal?.id ?? null,
4574
- goalVersion: activeGoal?.version ?? null,
4575
- earliestResetAt: null,
4576
- resetKind: "bounded_refresh",
4577
- failurePayload: {
4578
- error: "All connected Codex subscriptions are disabled for new allocations.",
4579
- code: "codex_allocator_disabled",
4580
- detail: "waiting for a credential to be re-enabled, reconnected, or added",
4743
+ if (
4744
+ !settings.codexCredentialLeasingEnabled &&
4745
+ leased.advanceActivePointer &&
4746
+ sessionPin === null &&
4747
+ rotationDecision.kind === "active" &&
4748
+ rotationDecision.moved
4749
+ ) {
4750
+ await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
4751
+ }
4752
+ effectiveCodexCredentialId = leased.credentialId;
4753
+ codexLeaseGeneration = leased.generation;
4754
+ codexLeaseConfirmedUntilMs =
4755
+ leased.leasedUntil && leaseAcquisitionStartedAtMs !== null
4756
+ ? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS
4757
+ : null;
4758
+ codexLeaseHeld =
4759
+ effectiveCodexCredentialId !== null &&
4760
+ leased.holderId !== null &&
4761
+ leased.generation !== null &&
4762
+ codexLeaseConfirmedUntilMs !== null;
4763
+ if (codexLeaseHeld) startCodexLeaseHeartbeat();
4764
+
4765
+ const actualOutcome = effectiveCodexCredentialId
4766
+ ? "selected"
4767
+ : rotationDecision.kind === "allCapped"
4768
+ ? "waiting"
4769
+ : "none";
4770
+ const actualReason = effectiveCodexCredentialId
4771
+ ? leased.reused
4772
+ ? "lease_reused"
4773
+ : sessionPin === effectiveCodexCredentialId
4774
+ ? "pin"
4775
+ : rotationDecision.kind === "active" && rotationDecision.moved
4776
+ ? "rotation"
4777
+ : "active"
4778
+ : rotationDecision.kind === "allCapped"
4779
+ ? "all_capped"
4780
+ : "none";
4781
+ const fencedInFlight = leased.reused;
4782
+ const shadowResult = await publishCodexFleetShadowDecisionV1({
4783
+ enabled: settings.codexFleetPolicyShadowEnabled,
4784
+ decision: {
4785
+ accounts: leased.accounts,
4786
+ actualCredentialId: effectiveCodexCredentialId,
4787
+ actualOutcome,
4788
+ actualReason,
4789
+ affinityCredentialId: fencedInFlight
4790
+ ? effectiveCodexCredentialId
4791
+ : (sessionPin ?? sessionCodex?.lastCredentialId ?? null),
4792
+ fencedInFlight,
4793
+ nearExhaustionPct: settings.codexRotationNearExhaustionPct,
4794
+ now: new Date(),
4795
+ aliasSeed: randomUUID(),
4581
4796
  },
4797
+ publish,
4582
4798
  });
4583
- if (armed.action === "waiting") {
4584
- await publishDurableSessionEvents(
4585
- bus,
4586
- input.workspaceId,
4587
- input.sessionId,
4588
- armed.events,
4799
+ if (shadowResult.outcome === "published") {
4800
+ const shadowPayload = shadowResult.payload;
4801
+ observability.incrementCounter({
4802
+ name: "opengeni_codex_fleet_shadow_decisions_total",
4803
+ help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
4804
+ labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload),
4805
+ });
4806
+ observability.info("Codex adaptive fleet shadow decision", {
4807
+ workspaceId: input.workspaceId,
4808
+ policyVersion: shadowPayload.replay.policyVersion,
4809
+ inputFingerprint: shadowPayload.replay.inputFingerprint,
4810
+ decisionFingerprint: shadowPayload.replay.decisionFingerprint,
4811
+ actualOutcome: shadowPayload.actual.outcome,
4812
+ shadowOutcome: shadowPayload.replay.decision.outcome,
4813
+ comparison: shadowPayload.comparison,
4814
+ candidateCount: shadowPayload.replay.input.candidates.length,
4815
+ truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
4816
+ payloadBytes: shadowResult.payloadBytes,
4817
+ });
4818
+ } else if (shadowResult.outcome === "failed") {
4819
+ // Shadow observability is explicitly non-authoritative. A malformed
4820
+ // snapshot or event-write fault must never change the authoritative lease,
4821
+ // capacity wait, failover, or the account serving this fenced turn.
4822
+ observability.incrementCounter({
4823
+ name: "opengeni_codex_fleet_shadow_errors_total",
4824
+ help: "Shadow decision build/publication failures.",
4825
+ labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
4826
+ });
4827
+ observability.warn("Codex adaptive fleet shadow decision failed open", {
4828
+ stage: shadowResult.stage,
4829
+ reason: shadowResult.reason,
4830
+ errorClass: "CodexFleetShadowOperationError",
4831
+ errorCode: "codex_fleet_shadow_failed",
4832
+ origin: "worker",
4833
+ payloadBytes: shadowResult.payloadBytes,
4834
+ });
4835
+ }
4836
+
4837
+ const eligibleCount = leased.accounts.filter((account) =>
4838
+ isCodexCredentialEligible(account, new Date()),
4839
+ ).length;
4840
+ const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
4841
+ observability.incrementCounter({
4842
+ name: "opengeni_codex_pool_observations_total",
4843
+ help: "Observed eligible Codex pool depth buckets at turn selection.",
4844
+ labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
4845
+ });
4846
+ if (eligibleCount <= 1) {
4847
+ observability.incrementCounter({
4848
+ name: "opengeni_codex_pool_low_total",
4849
+ help: "Alert signal emitted when the eligible Codex pool is zero or one.",
4850
+ labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
4851
+ });
4852
+ observability.warn("Codex eligible credential pool is low", {
4853
+ workspaceId: input.workspaceId,
4854
+ eligibleCount,
4855
+ connectedCount: leased.accounts.length,
4856
+ depth: poolDepth,
4857
+ });
4858
+ }
4859
+
4860
+ if (
4861
+ effectiveCodexCredentialId === null &&
4862
+ leased.accounts.length > 0 &&
4863
+ leased.accounts.every((account) => !account.allocatorEnabled) &&
4864
+ turnId
4865
+ ) {
4866
+ if (turn.source === "compaction") {
4867
+ if (
4868
+ !(await settle!({
4869
+ events: [
4870
+ {
4871
+ type: "turn.cancelled",
4872
+ payload: {
4873
+ maintenance: "context_compaction",
4874
+ reason: "codex_allocator_disabled",
4875
+ requestPreserved: true,
4876
+ },
4877
+ },
4878
+ {
4879
+ type: "session.status.changed",
4880
+ payload: { status: "idle" },
4881
+ },
4882
+ ],
4883
+ turnStatus: "cancelled",
4884
+ sessionStatus: "idle",
4885
+ activeTurnId: null,
4886
+ }))
4887
+ ) {
4888
+ return claimedResult({ status: "cancelled" });
4889
+ }
4890
+ turnMetricOutcome = "cancelled";
4891
+ activityStatus = "idle";
4892
+ return claimedResult({ status: "idle", deferredUntilWake: true });
4893
+ }
4894
+ const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
4895
+ () => null,
4589
4896
  );
4590
- turnMetricOutcome = "recovering";
4591
- activityStatus = "waiting_capacity";
4592
- return claimedResult({
4593
- status: "waiting_capacity",
4594
- capacityWait: {
4595
- waiterId: armed.waiter.id,
4596
- generation: armed.waiter.generation,
4597
- nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
4598
- wakeRevision: armed.waiter.wakeRevision,
4897
+ const activeGoal = goal?.status === "active" ? goal : null;
4898
+ const armed = await armCodexCapacityWait(db, {
4899
+ accountId: input.accountId,
4900
+ workspaceId: input.workspaceId,
4901
+ sessionId: input.sessionId,
4902
+ turnId,
4903
+ attemptId: input.attemptId,
4904
+ workflowId: input.workflowId,
4905
+ goalId: activeGoal?.id ?? null,
4906
+ goalVersion: activeGoal?.version ?? null,
4907
+ earliestResetAt: null,
4908
+ resetKind: "bounded_refresh",
4909
+ failurePayload: {
4910
+ error: "All connected Codex subscriptions are disabled for new allocations.",
4911
+ code: "codex_allocator_disabled",
4912
+ detail: "waiting for a credential to be re-enabled, reconnected, or added",
4599
4913
  },
4600
4914
  });
4915
+ if (armed.action === "waiting") {
4916
+ await publishDurableSessionEvents(
4917
+ bus,
4918
+ input.workspaceId,
4919
+ input.sessionId,
4920
+ armed.events,
4921
+ );
4922
+ turnMetricOutcome = "recovering";
4923
+ activityStatus = "waiting_capacity";
4924
+ return claimedResult({
4925
+ status: "waiting_capacity",
4926
+ capacityWait: {
4927
+ waiterId: armed.waiter.id,
4928
+ generation: armed.waiter.generation,
4929
+ nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
4930
+ wakeRevision: armed.waiter.wakeRevision,
4931
+ },
4932
+ });
4933
+ }
4934
+ if (
4935
+ !(await settle!({
4936
+ events: [
4937
+ {
4938
+ type: "turn.failed",
4939
+ payload: {
4940
+ error: "All connected Codex subscriptions are disabled for new allocations.",
4941
+ code: "codex_allocator_disabled",
4942
+ retryable: false,
4943
+ recovery: "user_message",
4944
+ },
4945
+ },
4946
+ {
4947
+ type: "session.status.changed",
4948
+ payload: { status: "idle" },
4949
+ },
4950
+ ],
4951
+ turnStatus: "failed",
4952
+ sessionStatus: "idle",
4953
+ activeTurnId: null,
4954
+ }))
4955
+ ) {
4956
+ return claimedResult({ status: "cancelled" });
4957
+ }
4958
+ turnMetricOutcome = "failed";
4959
+ activityStatus = "idle";
4960
+ return claimedResult({ status: "idle" });
4601
4961
  }
4602
- if (
4603
- !(await settle!({
4604
- events: [
4962
+
4963
+ if (rotationDecision.kind === "allCapped" && turnId) {
4964
+ if (turn.source === "compaction") {
4965
+ if (
4966
+ !(await settle!({
4967
+ events: [
4968
+ {
4969
+ type: "turn.cancelled",
4970
+ payload: {
4971
+ maintenance: "context_compaction",
4972
+ reason: "codex_capacity_unavailable",
4973
+ requestPreserved: true,
4974
+ },
4975
+ },
4976
+ {
4977
+ type: "session.status.changed",
4978
+ payload: { status: "idle" },
4979
+ },
4980
+ ],
4981
+ turnStatus: "cancelled",
4982
+ sessionStatus: "idle",
4983
+ activeTurnId: null,
4984
+ }))
4985
+ ) {
4986
+ return claimedResult({ status: "cancelled" });
4987
+ }
4988
+ turnMetricOutcome = "cancelled";
4989
+ activityStatus = "idle";
4990
+ return claimedResult({ status: "idle", deferredUntilWake: true });
4991
+ }
4992
+ // Every eligible account is capped/cooling (and a usage refresh did NOT
4993
+ // surface a reset): idle the turn AT THE BOUNDARY (no wasted model/sandbox
4994
+ // build) until the EARLIEST reset across all accounts — the multi-account
4995
+ // generalization of #143's single-account idle-until-reset. No saveRunState:
4996
+ // no model ran, nothing to freeze.
4997
+ const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
4998
+ () => null,
4999
+ );
5000
+ const goalActive = Boolean(goal && goal.status === "active");
5001
+ // BOUNDED + POSITIVE: clamp to [MIN_IDLE_MS, max] so a null/elapsed/unknown
5002
+ // reset can never yield a 0 (which session.ts would treat as "continue now",
5003
+ // re-entering this path in a tight CPU/DB-hammering loop).
5004
+ const resumeMs = computeIdleDelayMs(
5005
+ rotationDecision.earliestResetAt,
5006
+ new Date(),
5007
+ CODEX_USAGE_LIMIT_MAX_RESUME_MS,
5008
+ );
5009
+ const failurePayload = codexUsageLimitFailurePayload(
5010
+ { resetsInSeconds: Math.ceil(resumeMs / 1000) },
5011
+ "all connected Codex subscriptions are rate-limited",
5012
+ { allAccounts: true },
5013
+ );
5014
+ const authoritativeResetAt = authoritativeCodexCapacityResetAt(
5015
+ leased.accounts,
5016
+ new Date(),
5017
+ );
5018
+ const armed = await armCodexCapacityWait(db, {
5019
+ accountId: input.accountId,
5020
+ workspaceId: input.workspaceId,
5021
+ sessionId: input.sessionId,
5022
+ turnId,
5023
+ attemptId: input.attemptId,
5024
+ workflowId: input.workflowId,
5025
+ goalId: goalActive && goal ? goal.id : null,
5026
+ goalVersion: goalActive && goal ? goal.version : null,
5027
+ earliestResetAt: authoritativeResetAt,
5028
+ resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
5029
+ failurePayload,
5030
+ });
5031
+ if (armed.action === "waiting") {
5032
+ await publishDurableSessionEvents(
5033
+ bus,
5034
+ input.workspaceId,
5035
+ input.sessionId,
5036
+ armed.events,
5037
+ );
5038
+ turnMetricOutcome = "recovering";
5039
+ activityStatus = "waiting_capacity";
5040
+ return claimedResult({
5041
+ status: "waiting_capacity",
5042
+ capacityWait: {
5043
+ waiterId: armed.waiter.id,
5044
+ generation: armed.waiter.generation,
5045
+ nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
5046
+ wakeRevision: armed.waiter.wakeRevision,
5047
+ },
5048
+ });
5049
+ }
5050
+ if (
5051
+ !(await settle!({
5052
+ events: [
5053
+ // `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
5054
+ // rotation-wait state as the reactive all-capped path, so it must freeze
5055
+ // autoContinuations identically (evaluateGoalContinuation reads this marker)
5056
+ // — a goal waiting out a long reset must not burn its continuation budget on
5057
+ // the proactive path while the reactive path spares it.
5058
+ {
5059
+ type: "turn.failed",
5060
+ payload: {
5061
+ ...failurePayload,
5062
+ recovery: goalActive ? "goal_continuation" : "user_message",
5063
+ rotated: true,
5064
+ },
5065
+ },
5066
+ {
5067
+ type: "session.status.changed",
5068
+ payload: { status: "idle" },
5069
+ },
5070
+ ],
5071
+ turnStatus: "failed",
5072
+ sessionStatus: "idle",
5073
+ activeTurnId: null,
5074
+ }))
5075
+ ) {
5076
+ return claimedResult({ status: "cancelled" });
5077
+ }
5078
+ turnMetricOutcome = "failed";
5079
+ activityStatus = "idle";
5080
+ // idleUntilReset marks this a MANDATORY hold: session.ts must wait the full
5081
+ // resumeMs even if a future change made it 0 — never a tight re-dispatch.
5082
+ return claimedResult(
5083
+ goalActive
5084
+ ? {
5085
+ status: "idle",
5086
+ continueDelayMs: resumeMs,
5087
+ idleUntilReset: true,
5088
+ }
5089
+ : { status: "idle" },
5090
+ );
5091
+ }
5092
+ if (effectiveCodexCredentialId) {
5093
+ const priorAccountId = sessionCodex?.lastCredentialId ?? null;
5094
+ await recordSessionActiveCodexCredential(
5095
+ db,
5096
+ input.workspaceId,
5097
+ input.sessionId,
5098
+ effectiveCodexCredentialId,
5099
+ );
5100
+ if (priorAccountId !== effectiveCodexCredentialId) {
5101
+ const rotated = rotationDecision.kind === "active" && rotationDecision.moved;
5102
+ await publish([
4605
5103
  {
4606
- type: "turn.failed",
5104
+ type: "codex.account.switched",
4607
5105
  payload: {
4608
- error: "All connected Codex subscriptions are disabled for new allocations.",
4609
- code: "codex_allocator_disabled",
4610
- retryable: false,
4611
- recovery: "user_message",
5106
+ fromAccountId: priorAccountId,
5107
+ toAccountId: effectiveCodexCredentialId,
5108
+ reason: rotated ? "rotation" : "manual",
4612
5109
  },
4613
5110
  },
4614
- { type: "session.status.changed", payload: { status: "idle" } },
4615
- ],
4616
- turnStatus: "failed",
4617
- sessionStatus: "idle",
4618
- activeTurnId: null,
4619
- }))
4620
- ) {
4621
- return claimedResult({ status: "cancelled" });
5111
+ ]);
5112
+ }
5113
+
5114
+ const selectionReason = leased.reused
5115
+ ? "lease_reused"
5116
+ : sessionPin === effectiveCodexCredentialId
5117
+ ? "pin"
5118
+ : rotationDecision.kind === "active" && rotationDecision.moved
5119
+ ? "rotation"
5120
+ : "active";
5121
+ observability.incrementCounter({
5122
+ name: "opengeni_codex_credential_selections_total",
5123
+ help: "Codex credential selections by strategy and reason.",
5124
+ labels: {
5125
+ workspace_key: codexWorkspaceKey,
5126
+ strategy: leased.rotationStrategy,
5127
+ reason: selectionReason,
5128
+ },
5129
+ });
5130
+ await publish([
5131
+ {
5132
+ type: "codex.credential.selected",
5133
+ payload: {
5134
+ credentialId: effectiveCodexCredentialId,
5135
+ strategy: leased.rotationStrategy,
5136
+ reason: selectionReason,
5137
+ eligibleCount,
5138
+ connectedCount: leased.accounts.length,
5139
+ reused: leased.reused,
5140
+ },
5141
+ },
5142
+ ]);
4622
5143
  }
4623
- turnMetricOutcome = "failed";
4624
- activityStatus = "idle";
4625
- return claimedResult({ status: "idle" });
5144
+ } catch (error) {
5145
+ credentialSelectionOutcome = "failed";
5146
+ throw error;
5147
+ } finally {
5148
+ recordTurnStartupPhase(observability, {
5149
+ phase: "credential_selection",
5150
+ provider: "codex-subscription",
5151
+ backend: turn.sandboxBackend,
5152
+ outcome: credentialSelectionOutcome,
5153
+ durationSeconds: (performance.now() - credentialSelectionStartedAt) / 1_000,
5154
+ });
4626
5155
  }
5156
+ }
4627
5157
 
4628
- if (rotationDecision.kind === "allCapped" && turnId) {
5158
+ if (isXaiTurn) {
5159
+ const authoritySnapshot = turn.xaiProviderAccountAuthoritySnapshot;
5160
+ xaiAuthoritySnapshot = authoritySnapshot;
5161
+ const subjectId =
5162
+ authoritySnapshot.scope === "user"
5163
+ ? turn.initiatingHumanSubjectId
5164
+ : "worker:xai-workspace";
5165
+ if (!subjectId) {
5166
+ throw new Error("User-scoped SuperGrok work has no frozen initiating human");
5167
+ }
5168
+ const sessionPin = await getXaiSessionAccountPin(db, {
5169
+ workspaceId: input.workspaceId,
5170
+ subjectId,
5171
+ sessionId: input.sessionId,
5172
+ authoritySnapshot,
5173
+ });
5174
+ const leaseStartedAtMs = performance.now();
5175
+ const leased = await acquireXaiCredentialLease(db, {
5176
+ accountId: input.accountId,
5177
+ workspaceId: input.workspaceId,
5178
+ subjectId,
5179
+ sessionId: input.sessionId,
5180
+ turnId: turn.id,
5181
+ holderId: dispatchId,
5182
+ authoritySnapshot,
5183
+ pinnedCredentialId: sessionPin?.pinnedCredentialId ?? null,
5184
+ pinSource:
5185
+ sessionPin?.pinSource === "manual" || sessionPin?.pinSource === "policy"
5186
+ ? sessionPin.pinSource
5187
+ : null,
5188
+ });
5189
+ effectiveXaiCredentialId = leased.credentialId;
5190
+ xaiRotationEnabled = leased.rotationEnabled;
5191
+ xaiLeaseSubjectId = subjectId;
5192
+ xaiLeaseHolderId = leased.holderId;
5193
+ xaiLeaseGeneration = leased.generation;
5194
+ xaiLeaseConfirmedUntilMs = leased.leasedUntil
5195
+ ? leaseStartedAtMs + XAI_CREDENTIAL_LEASE_TTL_MS
5196
+ : null;
5197
+ xaiLeaseHeld =
5198
+ effectiveXaiCredentialId !== null &&
5199
+ leased.holderId !== null &&
5200
+ leased.generation !== null &&
5201
+ xaiLeaseConfirmedUntilMs !== null;
5202
+ if (!effectiveXaiCredentialId) {
5203
+ const connected = leased.accounts.length;
5204
+ const allocatorEnabled = leased.accounts.filter(
5205
+ (account) => account.allocatorEnabled,
5206
+ ).length;
5207
+ if (connected === 0) {
5208
+ throw Object.assign(
5209
+ new Error("No SuperGrok subscription account is connected for this authority scope"),
5210
+ { code: "xai_not_connected" },
5211
+ );
5212
+ }
4629
5213
  if (turn.source === "compaction") {
4630
5214
  if (
4631
5215
  !(await settle!({
@@ -4634,7 +5218,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4634
5218
  type: "turn.cancelled",
4635
5219
  payload: {
4636
5220
  maintenance: "context_compaction",
4637
- reason: "codex_capacity_unavailable",
5221
+ reason: "xai_capacity_unavailable",
4638
5222
  requestPreserved: true,
4639
5223
  },
4640
5224
  },
@@ -4654,44 +5238,38 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4654
5238
  activityStatus = "idle";
4655
5239
  return claimedResult({ status: "idle", deferredUntilWake: true });
4656
5240
  }
4657
- // Every eligible account is capped/cooling (and a usage refresh did NOT
4658
- // surface a reset): idle the turn AT THE BOUNDARY (no wasted model/sandbox
4659
- // build) until the EARLIEST reset across all accounts — the multi-account
4660
- // generalization of #143's single-account idle-until-reset. No saveRunState:
4661
- // no model ran, nothing to freeze.
4662
5241
  const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
4663
5242
  () => null,
4664
5243
  );
4665
- const goalActive = Boolean(goal && goal.status === "active");
4666
- // BOUNDED + POSITIVE: clamp to [MIN_IDLE_MS, max] so a null/elapsed/unknown
4667
- // reset can never yield a 0 (which session.ts would treat as "continue now",
4668
- // re-entering this path in a tight CPU/DB-hammering loop).
4669
- const resumeMs = computeIdleDelayMs(
4670
- rotationDecision.earliestResetAt,
4671
- new Date(),
4672
- CODEX_USAGE_LIMIT_MAX_RESUME_MS,
4673
- );
4674
- const failurePayload = codexUsageLimitFailurePayload(
4675
- { resetsInSeconds: Math.ceil(resumeMs / 1000) },
4676
- "all connected Codex subscriptions are rate-limited",
4677
- { allAccounts: true },
4678
- );
4679
- const authoritativeResetAt = authoritativeCodexCapacityResetAt(
4680
- leased.accounts,
4681
- new Date(),
4682
- );
4683
- const armed = await armCodexCapacityWait(db, {
5244
+ const activeGoal = goal?.status === "active" ? goal : null;
5245
+ const now = new Date();
5246
+ const futureResets = leased.accounts
5247
+ .map((account) => account.exhaustedUntil)
5248
+ .filter((date): date is Date => date !== null && date > now);
5249
+ const earliestResetAt = futureResets.length
5250
+ ? new Date(Math.min(...futureResets.map((date) => date.getTime())))
5251
+ : null;
5252
+ const error =
5253
+ allocatorEnabled === 0
5254
+ ? "All connected SuperGrok subscription accounts are disabled for allocation"
5255
+ : "All connected SuperGrok subscription accounts are temporarily unavailable";
5256
+ const armed = await armXaiCapacityWait(db, {
4684
5257
  accountId: input.accountId,
4685
5258
  workspaceId: input.workspaceId,
5259
+ subjectId,
4686
5260
  sessionId: input.sessionId,
4687
- turnId,
5261
+ turnId: turn.id,
4688
5262
  attemptId: input.attemptId,
4689
5263
  workflowId: input.workflowId,
4690
- goalId: goalActive && goal ? goal.id : null,
4691
- goalVersion: goalActive && goal ? goal.version : null,
4692
- earliestResetAt: authoritativeResetAt,
4693
- resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
4694
- failurePayload,
5264
+ authoritySnapshot,
5265
+ goalId: activeGoal?.id ?? null,
5266
+ goalVersion: activeGoal?.version ?? null,
5267
+ earliestResetAt,
5268
+ failurePayload: {
5269
+ error,
5270
+ code: allocatorEnabled === 0 ? "xai_allocator_disabled" : "xai_capacity_unavailable",
5271
+ detail: "waiting for an eligible account, reconnect, pin change, or quota reset",
5272
+ },
4695
5273
  });
4696
5274
  if (armed.action === "waiting") {
4697
5275
  await publishDurableSessionEvents(
@@ -4705,6 +5283,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4705
5283
  return claimedResult({
4706
5284
  status: "waiting_capacity",
4707
5285
  capacityWait: {
5286
+ provider: "xai",
4708
5287
  waiterId: armed.waiter.id,
4709
5288
  generation: armed.waiter.generation,
4710
5289
  nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
@@ -4715,17 +5294,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4715
5294
  if (
4716
5295
  !(await settle!({
4717
5296
  events: [
4718
- // `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
4719
- // rotation-wait state as the reactive all-capped path, so it must freeze
4720
- // autoContinuations identically (evaluateGoalContinuation reads this marker)
4721
- // — a goal waiting out a long reset must not burn its continuation budget on
4722
- // the proactive path while the reactive path spares it.
4723
5297
  {
4724
5298
  type: "turn.failed",
4725
5299
  payload: {
4726
- ...failurePayload,
4727
- recovery: goalActive ? "goal_continuation" : "user_message",
4728
- rotated: true,
5300
+ error,
5301
+ code: "xai_capacity_wait_stale",
5302
+ retryable: false,
5303
+ recovery: "user_message",
4729
5304
  },
4730
5305
  },
4731
5306
  { type: "session.status.changed", payload: { status: "idle" } },
@@ -4739,80 +5314,63 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4739
5314
  }
4740
5315
  turnMetricOutcome = "failed";
4741
5316
  activityStatus = "idle";
4742
- // idleUntilReset marks this a MANDATORY hold: session.ts must wait the full
4743
- // resumeMs even if a future change made it 0 — never a tight re-dispatch.
4744
- return claimedResult(
4745
- goalActive
4746
- ? {
4747
- status: "idle",
4748
- continueDelayMs: resumeMs,
4749
- idleUntilReset: true,
4750
- }
4751
- : { status: "idle" },
4752
- );
5317
+ return claimedResult({ status: "idle" });
4753
5318
  }
4754
- if (effectiveCodexCredentialId) {
4755
- const priorAccountId = sessionCodex?.lastCredentialId ?? null;
4756
- await recordSessionActiveCodexCredential(
4757
- db,
4758
- input.workspaceId,
4759
- input.sessionId,
4760
- effectiveCodexCredentialId,
4761
- );
4762
- if (priorAccountId !== effectiveCodexCredentialId) {
4763
- const rotated = rotationDecision.kind === "active" && rotationDecision.moved;
4764
- await publish([
4765
- {
4766
- type: "codex.account.switched",
4767
- payload: {
4768
- fromAccountId: priorAccountId,
4769
- toAccountId: effectiveCodexCredentialId,
4770
- reason: rotated ? "rotation" : "manual",
4771
- },
4772
- },
4773
- ]);
4774
- }
4775
-
4776
- const selectionReason = leased.reused
4777
- ? "lease_reused"
4778
- : sessionPin === effectiveCodexCredentialId
4779
- ? "pin"
4780
- : rotationDecision.kind === "active" && rotationDecision.moved
4781
- ? "rotation"
4782
- : "active";
4783
- observability.incrementCounter({
4784
- name: "opengeni_codex_credential_selections_total",
4785
- help: "Codex credential selections by strategy and reason.",
4786
- labels: {
4787
- workspace_key: codexWorkspaceKey,
4788
- strategy: leased.rotationStrategy,
4789
- reason: selectionReason,
4790
- },
5319
+ if (xaiLeaseHeld) startXaiLeaseHeartbeat();
5320
+ if (
5321
+ leased.rotationEnabled &&
5322
+ sessionPin?.pinSource !== "manual" &&
5323
+ (sessionPin?.pinnedCredentialId !== effectiveXaiCredentialId ||
5324
+ sessionPin?.pinSource !== "policy")
5325
+ ) {
5326
+ await setXaiSessionAccountPin(db, {
5327
+ accountId: input.accountId,
5328
+ workspaceId: input.workspaceId,
5329
+ subjectId,
5330
+ sessionId: input.sessionId,
5331
+ authoritySnapshot,
5332
+ credentialId: effectiveXaiCredentialId,
5333
+ pinSource: "policy",
5334
+ expectedVersion: sessionPin?.version ?? null,
5335
+ }).catch((error: unknown) => {
5336
+ if (error instanceof Error && error.message === "xAI session pin changed") return;
5337
+ throw error;
5338
+ });
5339
+ } else if (!leased.rotationEnabled && sessionPin?.pinSource === "policy") {
5340
+ await setXaiSessionAccountPin(db, {
5341
+ accountId: input.accountId,
5342
+ workspaceId: input.workspaceId,
5343
+ subjectId,
5344
+ sessionId: input.sessionId,
5345
+ authoritySnapshot,
5346
+ credentialId: null,
5347
+ pinSource: null,
5348
+ expectedVersion: sessionPin.version,
5349
+ }).catch((error: unknown) => {
5350
+ if (error instanceof Error && error.message === "xAI session pin changed") return;
5351
+ throw error;
4791
5352
  });
4792
- await publish([
4793
- {
4794
- type: "codex.credential.selected",
4795
- payload: {
4796
- credentialId: effectiveCodexCredentialId,
4797
- strategy: leased.rotationStrategy,
4798
- reason: selectionReason,
4799
- eligibleCount,
4800
- connectedCount: leased.accounts.length,
4801
- reused: leased.reused,
4802
- },
4803
- },
4804
- ]);
4805
5353
  }
5354
+ await recordXaiSessionLastAccount(db, {
5355
+ accountId: input.accountId,
5356
+ workspaceId: input.workspaceId,
5357
+ subjectId,
5358
+ sessionId: input.sessionId,
5359
+ authoritySnapshot,
5360
+ credentialId: effectiveXaiCredentialId,
5361
+ });
4806
5362
  }
4807
5363
 
5364
+ const runtimePreparationStartedAt = performance.now();
5365
+
4808
5366
  // Pack-scoped runtime: enabled packs may declare the sandbox image this
4809
5367
  // workspace's sessions run in and skills for the sandbox skill index.
4810
5368
  // Resolved after turn.started so a composition conflict (two enabled
4811
5369
  // packs declaring images) fails the turn with its plain error instead
4812
5370
  // of failing the activity opaquely.
4813
- const [packRuntime, skillLibraryRuntime] = await Promise.all([
5371
+ const [packRuntime, installedSkillRuntime] = await Promise.all([
4814
5372
  resolveWorkspacePackRuntime(db, input.workspaceId),
4815
- resolveWorkspaceSkillLibraryRuntime(db, input.workspaceId),
5373
+ resolveWorkspaceInstalledSkillRuntime(db, input.workspaceId),
4816
5374
  ]);
4817
5375
  // RIG BINDING (M3): load the session's FROZEN rig version (resolved+frozen
4818
5376
  // at create). Everything rig-derived below (image precedence, env default
@@ -4860,12 +5418,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4860
5418
  if (!workspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
4861
5419
  const workspaceAgentInstructions = workspace.agentInstructions;
4862
5420
  const agentHumanInputEnabled = resolveWorkspaceAgentHumanInputEnabled(workspace.settings);
5421
+ const memoryPromptMode = resolveWorkspaceMemoryPromptMode(workspace.settings);
4863
5422
  assertWorkspaceHumanInputAllowed(agentHumanInputEnabled, "resume", humanInputResume !== null);
4864
- const workspaceGovernance = renderWorkspaceGovernanceContext({
4865
- companyProfile: companyProfileSnapshot,
4866
- instructionPolicy: instructionPolicySnapshot,
4867
- preferences: preferenceSnapshot,
4868
- });
5423
+ const companyProfileIncluded =
5424
+ memoryPromptMode === "legacy_standing" || session.nestedAgentDepth === 0;
5425
+ const workspaceGovernance = renderWorkspaceGovernanceContext(
5426
+ {
5427
+ companyProfile: companyProfileSnapshot,
5428
+ instructionPolicy: instructionPolicySnapshot,
5429
+ preferences: preferenceSnapshot,
5430
+ },
5431
+ {
5432
+ includeCompanyProfile: companyProfileIncluded,
5433
+ },
5434
+ );
4869
5435
  const structuredWorkspacePolicyActive =
4870
5436
  hasActiveWorkspaceInstructionPolicy(instructionPolicySnapshot);
4871
5437
  const workspaceMemory = await resolveWorkspaceMemoryBlock(db, input.workspaceId);
@@ -5034,6 +5600,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5034
5600
  // call on the same codex client) and the main run; otherwise the summarizer
5035
5601
  // would hit the codex backend unauthenticated.
5036
5602
  let codexModelRequestSequence = 0;
5603
+ let firstModelRequestPreparationStartedAt: number | null = null;
5604
+ let firstModelRequestPreparationRecorded = false;
5605
+ let firstModelRequestAuditRecorded = false;
5606
+ let firstModelRequestCheckpointAt: number | null = null;
5607
+ const firstModelRequestCheckpoints = new Set<string>();
5037
5608
  const codexContext: CodexRequestContext | null =
5038
5609
  resolvedModel?.provider.kind === "codex-subscription"
5039
5610
  ? ((): CodexRequestContext => {
@@ -5064,33 +5635,149 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5064
5635
  onUsageHeaders: (snapshot) => {
5065
5636
  latestCodexUsage = snapshot;
5066
5637
  }, // latest wins; flushed once in finally
5638
+ onRequestPreparationDiagnostic: (phase) => {
5639
+ if (
5640
+ firstModelRequestCheckpointAt === null ||
5641
+ firstModelRequestCheckpoints.has(phase)
5642
+ ) {
5643
+ return;
5644
+ }
5645
+ const now = performance.now();
5646
+ const metricPhase =
5647
+ phase === "transport_entry"
5648
+ ? "model_sdk_serialization"
5649
+ : phase === "credential_ready"
5650
+ ? "model_credential_resolution"
5651
+ : "model_wire_normalization";
5652
+ firstModelRequestCheckpoints.add(phase);
5653
+ recordTurnStartupPhase(observability, {
5654
+ phase: metricPhase,
5655
+ provider: turnExecutionPolicy.providerId,
5656
+ backend: activeSandboxBackend ?? groupBoxBackend,
5657
+ outcome: "completed",
5658
+ durationSeconds: (now - firstModelRequestCheckpointAt) / 1_000,
5659
+ count: turnTools.length,
5660
+ });
5661
+ firstModelRequestCheckpointAt = now;
5662
+ },
5067
5663
  onRequestOpaqueArtifacts: ({ fingerprints }) => {
5068
5664
  lastCodexRequestOpaqueArtifacts = fingerprints;
5069
5665
  },
5666
+ onModelRequestDiagnostic: (event) => {
5667
+ if (
5668
+ event.phase === "started" &&
5669
+ firstModelRequestPreparationStartedAt !== null &&
5670
+ !firstModelRequestPreparationRecorded
5671
+ ) {
5672
+ firstModelRequestPreparationRecorded = true;
5673
+ recordTurnStartupPhase(observability, {
5674
+ phase: "model_request_preparation",
5675
+ provider: turnExecutionPolicy.providerId,
5676
+ backend: activeSandboxBackend ?? groupBoxBackend,
5677
+ outcome: "completed",
5678
+ durationSeconds:
5679
+ (performance.now() - firstModelRequestPreparationStartedAt) / 1_000,
5680
+ count: turnTools.length,
5681
+ });
5682
+ }
5683
+ const phase =
5684
+ event.phase === "headers"
5685
+ ? "headers"
5686
+ : event.phase === "first_byte"
5687
+ ? "first_byte"
5688
+ : event.phase === "completed" ||
5689
+ event.phase === "failed" ||
5690
+ event.phase === "timed_out"
5691
+ ? "terminal"
5692
+ : null;
5693
+ if (!phase) return;
5694
+ recordModelRequestPhase(observability, {
5695
+ provider: "codex-subscription",
5696
+ phase,
5697
+ ...(event.phase === "completed"
5698
+ ? { outcome: "completed" as const }
5699
+ : event.phase === "failed"
5700
+ ? { outcome: "failed" as const }
5701
+ : event.phase === "timed_out"
5702
+ ? { outcome: "timed_out" as const }
5703
+ : {}),
5704
+ durationSeconds: event.durationMs / 1000,
5705
+ });
5706
+ },
5070
5707
  nextRequestId: () => `${dispatchId}:${++codexModelRequestSequence}`,
5071
5708
  onModelRequestEvent: async (event) => {
5072
5709
  if (!publish || !turnId) {
5073
5710
  throw new Error("Codex model request started before the turn event producer");
5074
5711
  }
5075
- await publish([
5076
- {
5077
- type: "agent.model.request",
5078
- payload: {
5079
- ...event,
5080
- provider: "codex-subscription",
5081
- turnId,
5082
- attemptId: input.attemptId,
5083
- dispatchId,
5084
- executionGeneration,
5712
+ const shouldRecordStartedAudit =
5713
+ event.phase === "started" && !firstModelRequestAuditRecorded;
5714
+ if (shouldRecordStartedAudit) {
5715
+ firstModelRequestAuditRecorded = true;
5716
+ }
5717
+ const auditStartedAt = shouldRecordStartedAudit ? performance.now() : null;
5718
+ let auditOutcome: "completed" | "failed" = "completed";
5719
+ try {
5720
+ await publish([
5721
+ {
5722
+ type: "agent.model.request",
5723
+ payload: {
5724
+ ...event,
5725
+ provider: "codex-subscription",
5726
+ turnId,
5727
+ attemptId: input.attemptId,
5728
+ dispatchId,
5729
+ executionGeneration,
5730
+ },
5085
5731
  },
5086
- },
5087
- ]);
5732
+ ]);
5733
+ } catch (error) {
5734
+ auditOutcome = "failed";
5735
+ throw error;
5736
+ } finally {
5737
+ if (auditStartedAt !== null) {
5738
+ recordTurnStartupPhase(observability, {
5739
+ phase: "model_request_audit",
5740
+ provider: turnExecutionPolicy.providerId,
5741
+ backend: activeSandboxBackend ?? groupBoxBackend,
5742
+ outcome: auditOutcome,
5743
+ durationSeconds: (performance.now() - auditStartedAt) / 1_000,
5744
+ count: turnTools.length,
5745
+ });
5746
+ }
5747
+ }
5088
5748
  },
5089
5749
  };
5090
5750
  })()
5091
5751
  : null;
5752
+ if (resolvedModel?.provider.kind === "xai-subscription") {
5753
+ if (!effectiveXaiCredentialId || !xaiLeaseSubjectId) {
5754
+ throw new Error("SuperGrok subscription execution has no leased credential");
5755
+ }
5756
+ let xaiModelRequestSequence = 0;
5757
+ const authorization = await buildXaiTurnRequestAuthorization({
5758
+ db,
5759
+ settings: runSettings,
5760
+ accountId: input.accountId,
5761
+ workspaceId: input.workspaceId,
5762
+ subjectId: xaiLeaseSubjectId,
5763
+ sessionId: input.sessionId,
5764
+ turnId: turn.id,
5765
+ credentialId: effectiveXaiCredentialId,
5766
+ authoritySnapshot: turn.xaiProviderAccountAuthoritySnapshot,
5767
+ hostedSearch: {
5768
+ webSearch: runSettings.webSearchEnabled,
5769
+ xSearch: runSettings.webSearchEnabled,
5770
+ },
5771
+ nextRequestId: () => `${dispatchId}:xai:${++xaiModelRequestSequence}`,
5772
+ });
5773
+ xaiRequestContext = authorization.context;
5774
+ }
5092
5775
  const withCodex = <T>(fn: () => Promise<T>): Promise<T> =>
5093
5776
  codexContext ? codexRequestStorage.run(codexContext, fn) : fn();
5777
+ const withProviderRequestContext = <T>(fn: () => Promise<T>): Promise<T> =>
5778
+ xaiRequestContext
5779
+ ? xaiSubscriptionRequestStorage.run(xaiRequestContext, fn)
5780
+ : withCodex(fn);
5094
5781
  const withCodexRemoteCompaction = <T>(fn: () => Promise<T>): Promise<T> =>
5095
5782
  withCodex(() =>
5096
5783
  withCodexRequestOverrides(
@@ -5134,15 +5821,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5134
5821
  servingCredentialId: effectiveCodexCredentialId,
5135
5822
  priorSessionCredentialId: priorSessionCodexCredentialId,
5136
5823
  emittedSourceKeys: emittedModelUsageSourceKeys,
5137
- renewLease: () => renewCodexLease("model_usage"),
5138
- leaseLost: () => codexLeaseLost,
5139
- leaseLostMessage: "Codex credential lease expired during context compaction",
5824
+ renewLease: () => renewServingCredentialLease("model_usage"),
5825
+ leaseLost: servingCredentialLeaseLost,
5826
+ leaseLostMessage: "Provider credential lease expired during context compaction",
5140
5827
  });
5141
5828
  };
5142
5829
  const compactionSummarizerFor = (systemInstructions?: string) =>
5143
5830
  resolvedModel
5144
5831
  ? (s: Settings, m: Array<Record<string, unknown>>) =>
5145
- withCodex(() =>
5832
+ withProviderRequestContext(() =>
5146
5833
  summarizeContextForCompaction(s, m, {
5147
5834
  client: resolvedModel.client,
5148
5835
  provider: resolvedModel.provider,
@@ -5256,20 +5943,23 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5256
5943
  };
5257
5944
  const compactionInstructions = appendWorkspaceMemory(
5258
5945
  appendPersistentSessionSettings(
5259
- appendSessionInstructions(
5260
- appendWorkspaceGovernance(
5261
- composeAgentInstructions(
5262
- structuredWorkspacePolicyActive
5263
- ? modelRunSettings.agentInstructionsTemplate
5264
- : (workspaceAgentInstructions ?? modelRunSettings.agentInstructionsTemplate),
5265
- undefined,
5266
- rigVersion && rigName
5267
- ? { name: rigName, version: rigVersion.version }
5268
- : undefined,
5946
+ appendSessionGoal(
5947
+ appendSessionInstructions(
5948
+ appendWorkspaceGovernance(
5949
+ composeAgentInstructions(
5950
+ structuredWorkspacePolicyActive
5951
+ ? modelRunSettings.agentInstructionsTemplate
5952
+ : (workspaceAgentInstructions ?? modelRunSettings.agentInstructionsTemplate),
5953
+ undefined,
5954
+ rigVersion && rigName
5955
+ ? { name: rigName, version: rigVersion.version }
5956
+ : undefined,
5957
+ ),
5958
+ workspaceGovernance ?? undefined,
5269
5959
  ),
5270
- workspaceGovernance ?? undefined,
5960
+ session.instructions ?? undefined,
5271
5961
  ),
5272
- session.instructions ?? undefined,
5962
+ turn.goalSnapshot,
5273
5963
  ),
5274
5964
  persistentSessionSettings,
5275
5965
  ),
@@ -5525,6 +6215,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5525
6215
  activeSandboxBackend === "selfhosted" &&
5526
6216
  Boolean(activeSandboxPointer?.activeSandboxId) &&
5527
6217
  Boolean(activeSandboxRecord?.enrollmentId);
6218
+ // `none` describes the durable home, not an explicit per-turn route. Give
6219
+ // the runtime the effective backend so it builds a SandboxAgent for the
6220
+ // attached Connected Machine without mutating the session or turn record.
6221
+ if (machinePrimary && modelRunSettings.sandboxBackend === "none") {
6222
+ modelRunSettings = {
6223
+ ...modelRunSettings,
6224
+ sandboxBackend: "selfhosted",
6225
+ };
6226
+ }
5528
6227
  // The backend that can actually create a sandbox for this turn. In the
5529
6228
  // common path this is runSettings.sandboxBackend. A selfhosted home turn
5530
6229
  // that is NOT machine-primary falls back to the deployment cloud backend
@@ -5586,17 +6285,27 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5586
6285
  cancellationSignal,
5587
6286
  undefined,
5588
6287
  );
5589
- const establishPolicy: "eager" | "on-demand" =
5590
- lazyProvisionEnabled(settings) &&
5591
- !machinePrimary &&
5592
- runSettings.sandboxBackend !== "none" &&
6288
+ const establishDecision = sandboxEstablishPolicyDecision({
6289
+ lazyEnabled: lazyProvisionEnabled(settings),
6290
+ machinePrimary,
6291
+ sandboxBackend: runSettings.sandboxBackend,
5593
6292
  // Resolved run credentials must be written to one exact leased sandbox
5594
6293
  // before agent execution. A warm active pointer can bypass the lazy
5595
6294
  // provisioner entirely, which previously skipped materialization.
5596
- !initialRunCredentialMaterial &&
5597
- requiredGeneratedVideoFiles.length === 0
5598
- ? "on-demand"
5599
- : "eager";
6295
+ hasInitialRunCredentialMaterial: initialRunCredentialMaterial !== null,
6296
+ generatedVideoFileCount: requiredGeneratedVideoFiles.length,
6297
+ hasSignedFileResources:
6298
+ requiresSignedFileResourceDownloads(
6299
+ runSettings,
6300
+ activeSandboxBackend ?? groupBoxBackend,
6301
+ ) && turnResources.some((resource) => resource.kind === "file"),
6302
+ });
6303
+ const establishPolicy = establishDecision.policy;
6304
+ recordTurnSandboxEstablishPolicy(observability, {
6305
+ policy: establishDecision.policy,
6306
+ reason: establishDecision.reason,
6307
+ backend: machinePrimary ? "selfhosted" : groupBoxBackend,
6308
+ });
5600
6309
  // Computed exactly ONCE per turn and reused for BOTH the box manifest
5601
6310
  // (resumeBoxForTurn -> establishSandboxSessionFromEnvelope, below) AND the
5602
6311
  // agent (runtime.buildAgent, below). sandboxEnvironmentForRun mints a FRESH
@@ -5676,7 +6385,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5676
6385
  sandboxWorkspaceEnvironmentValues,
5677
6386
  {
5678
6387
  skipGitHubToken: activeSandboxBackend === "selfhosted",
5679
- skipCodemode: activeSandboxBackend === "selfhosted",
6388
+ codemodeDelivery:
6389
+ activeSandboxBackend === "selfhosted" ? "transient_exec" : "managed_file",
5680
6390
  deferGitHubToken:
5681
6391
  activeSandboxBackend !== "selfhosted" && establishPolicy === "on-demand",
5682
6392
  scope: connectionScope,
@@ -5696,6 +6406,22 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5696
6406
  ? { ...baseSandboxEnvironment, ...sandboxArtifactRuntime.environment }
5697
6407
  : baseSandboxEnvironment;
5698
6408
 
6409
+ // One mutable in-memory bearer cell serves every Connected Machine route
6410
+ // in this attempt. SelfhostedSession snapshots it into each exact exec
6411
+ // request; it never enters the manifest, argv, filesystem, RunState, or
6412
+ // serialized session state. Managed renewal updates the same cell so a
6413
+ // later mid-turn swap sees the fresh bearer too.
6414
+ const codemodeTokenState = sandboxCodemodeToken ? { token: sandboxCodemodeToken } : undefined;
6415
+ const transientCodemodeEnvironment = codemodeTokenState
6416
+ ? (): Readonly<Record<string, string>> => ({
6417
+ OPENGENI_CODEMODE_URL: codemodeWorkspaceUrl(runSettings, input.workspaceId),
6418
+ OPENGENI_CODEMODE_TOKEN: codemodeTokenState.token,
6419
+ ...(runSettings.ogtoolPackageSpec
6420
+ ? { OPENGENI_OGTOOL_PACKAGE_SPEC: runSettings.ogtoolPackageSpec }
6421
+ : {}),
6422
+ })
6423
+ : undefined;
6424
+
5699
6425
  const sandboxCodemodeTokenFile = sandboxCodemodeToken
5700
6426
  ? codemodeTokenFileFromEnvironment(sandboxEnvironment, input.sessionId)
5701
6427
  : undefined;
@@ -5811,11 +6537,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5811
6537
  };
5812
6538
 
5813
6539
  const attachCodemodeTokenRenewal = async (
5814
- tokenSession: CodemodeTokenWriterSession,
6540
+ tokenSession?: CodemodeTokenWriterSession,
5815
6541
  initialExpiresAt = sandboxCodemodeTokenExpiresAt,
5816
6542
  initialSandbox?: ResumedTurnSandbox,
5817
6543
  ): Promise<void> => {
5818
- if (!sandboxCodemodeToken || !initialExpiresAt) return;
6544
+ if (!codemodeTokenState || !initialExpiresAt) return;
5819
6545
  const previous = codemodeTokenRenewal;
5820
6546
  codemodeTokenRenewal = null;
5821
6547
  await previous?.stop();
@@ -5827,37 +6553,38 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5827
6553
  connectionScope,
5828
6554
  codemodeAuthority,
5829
6555
  );
5830
- if (material) {
5831
- }
5832
6556
  return material;
5833
6557
  };
5834
6558
  const write = async (material: NonNullable<Awaited<ReturnType<typeof mint>>>) => {
5835
- const runAs = sandboxRunAs(runSettings);
5836
- const targetSandbox = resolvedSandbox ?? initialSandbox;
5837
- if (!targetSandbox) {
5838
- throw new Error("Codemode token renewal has no exact sandbox lease target");
6559
+ if (tokenSession) {
6560
+ const runAs = sandboxRunAs(runSettings);
6561
+ const targetSandbox = resolvedSandbox ?? initialSandbox;
6562
+ if (!targetSandbox) {
6563
+ throw new Error("Codemode token renewal has no exact sandbox lease target");
6564
+ }
6565
+ await runWorkspaceMutationForSandbox(
6566
+ targetSandbox,
6567
+ "codemodeTokenRenewal",
6568
+ async () =>
6569
+ await refreshCodemodeTokenFile(tokenSession, material.token, {
6570
+ ...(runAs ? { runAs } : {}),
6571
+ ...(sandboxCodemodeTokenFile
6572
+ ? {
6573
+ tokenFile: sandboxCodemodeTokenFile,
6574
+ legacyTokenFile: sandboxEnvironment.OPENGENI_CODEMODE_TOKEN_FILE!,
6575
+ }
6576
+ : {}),
6577
+ ...(toolCancellationFenceRef.current
6578
+ ? {
6579
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
6580
+ toolCancellationFenceRef.current,
6581
+ ),
6582
+ }
6583
+ : {}),
6584
+ }),
6585
+ );
5839
6586
  }
5840
- await runWorkspaceMutationForSandbox(
5841
- targetSandbox,
5842
- "codemodeTokenRenewal",
5843
- async () =>
5844
- await refreshCodemodeTokenFile(tokenSession, material.token, {
5845
- ...(runAs ? { runAs } : {}),
5846
- ...(sandboxCodemodeTokenFile
5847
- ? {
5848
- tokenFile: sandboxCodemodeTokenFile,
5849
- legacyTokenFile: sandboxEnvironment.OPENGENI_CODEMODE_TOKEN_FILE!,
5850
- }
5851
- : {}),
5852
- ...(toolCancellationFenceRef.current
5853
- ? {
5854
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
5855
- toolCancellationFenceRef.current,
5856
- ),
5857
- }
5858
- : {}),
5859
- }),
5860
- );
6587
+ codemodeTokenState.token = material.token;
5861
6588
  };
5862
6589
  let renewalExpiresAt = initialExpiresAt;
5863
6590
  if (renewalExpiresAt.getTime() <= Date.now() + CODEMODE_TOKEN_EXPIRY_LEAD_MS) {
@@ -5900,6 +6627,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5900
6627
  codemodeTokenRenewal = controller;
5901
6628
  };
5902
6629
 
6630
+ // A Connected Machine needs renewal, but renewal is purely worker-local:
6631
+ // starting this loop performs no control-plane or machine operation.
6632
+ if (activeSandboxBackend === "selfhosted") {
6633
+ await attachCodemodeTokenRenewal();
6634
+ }
6635
+
5903
6636
  const attachRunCredentialRenewal = async (
5904
6637
  credentialSession: RunCredentialCommandSession,
5905
6638
  initialSandbox?: ResumedTurnSandbox,
@@ -6045,234 +6778,289 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6045
6778
  // applies the agent's manifest to this provided session and throws on ANY
6046
6779
  // variableSet delta (validateNoEnvironmentDelta). Passing sandboxEnvironment
6047
6780
  // here makes current==target so the delta is empty.
6048
- if (settings.sandboxOwnershipEnabled && turn.sandboxBackend !== "none") {
6049
- const managedOwnership = managedSandboxOwnershipForTurn(
6781
+ recordTurnStartupPhase(observability, {
6782
+ phase: "runtime_preparation",
6783
+ provider: turnExecutionPolicy.providerId,
6784
+ backend: activeSandboxBackend ?? groupBoxBackend,
6785
+ outcome: "completed",
6786
+ durationSeconds: (performance.now() - runtimePreparationStartedAt) / 1_000,
6787
+ });
6788
+ if (
6789
+ shouldEstablishSandboxForTurn(
6790
+ settings.sandboxOwnershipEnabled,
6791
+ turn.sandboxBackend,
6050
6792
  machinePrimary,
6051
- input.attemptId,
6052
- session.sandboxGroupId,
6053
- );
6054
- sandboxHolderId = managedOwnership?.holderId ?? null;
6055
- sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
6056
- // STAGE D honest-label guard: a machine-home session carries
6057
- // turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
6058
- // when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
6059
- // + enrollmentId). When it is NOT primary — the agent swapped back to the
6060
- // group box (sandbox_swap 'session'/'default'/groupId clears the pointer) or
6061
- // selfhosted routing is flag-OFF (the pointer is ignored) — the else-branch
6062
- // must resume a REAL cloud group box, not a "selfhosted" one: the registry
6063
- // SelfhostedSandboxClient has no bound agentId and throws. Fall the group-box
6064
- // backend back to the deployment default cloud backend so swap-away / flag-off
6065
- // degrade to a genuine cloud box exactly like today (home=modal did).
6066
- if (machinePrimary) {
6067
- // STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
6068
- // establish OR lease the managed home box. The active-sandbox pointer is
6069
- // the machine route's own epoch fence; the managed-home lease separately
6070
- // owns cloud recovery, snapshots, billing, and reaping. Mixing those two
6071
- // ownership domains lets an unrelated degraded cloud archive block a
6072
- // healthy explicitly selected machine.
6073
- // Whether the machine's latest Hello advertised the op-stream engine
6074
- // (refreshed on every connect). Read only when the server flag is on —
6075
- // one indexed lookup, and the flag off keeps this path byte-identical.
6076
- const machineOpStream =
6077
- settings.agentOpStreamEnabled === true
6078
- ? (await getEnrollment(db, input.workspaceId, activeSandboxRecord!.enrollmentId!))
6079
- ?.opStream === true
6080
- : false;
6081
- const established = await establishSelfhostedTurnSession(
6082
- {
6083
- db,
6084
- settings,
6085
- bus,
6086
- onOp: machineOpObserver.observer,
6087
- onSandboxOperation: sandboxOperationObserver,
6088
- opJournal,
6089
- },
6090
- {
6091
- workspaceId: input.workspaceId,
6092
- agentId: activeSandboxRecord!.enrollmentId!,
6093
- opStream: machineOpStream,
6094
- epoch: activeSandboxPointer!.activeEpoch,
6095
- environment: sandboxEnvironment,
6096
- workingDir: activeSandboxPointer!.workingDir,
6097
- },
6793
+ )
6794
+ ) {
6795
+ const sandboxEstablishStartedAt = performance.now();
6796
+ let sandboxEstablishOutcome: "completed" | "failed" = "completed";
6797
+ try {
6798
+ const managedOwnership = managedSandboxOwnershipForTurn(
6799
+ machinePrimary,
6800
+ input.attemptId,
6801
+ session.sandboxGroupId,
6098
6802
  );
6099
- // The machine-primary establish narrows `session` to SelfhostedSession
6100
- // (buildSelfhostedBackendSession); EstablishedSandboxSession widens it.
6101
- machinePrimarySession =
6102
- established.session as import("@opengeni/runtime").SelfhostedSession;
6103
- setupBoxSession = established.session;
6104
- resolvedSandbox = {
6105
- // Wrap in the SAME routing proxy so a mid-turn swap (to another machine
6106
- // or back to the group box) still re-routes per op. PIN this established
6107
- // SelfhostedSession for the machine pointer so the turn-start manifest
6108
- // write (via the proxy's `state` getter) and the per-op reads hit ONE
6109
- // instance — no two-instance manifest divergence.
6110
- established: wrapTurnBoxWithRouting(
6803
+ sandboxHolderId = managedOwnership?.holderId ?? null;
6804
+ sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
6805
+ // STAGE D honest-label guard: a machine-home session carries
6806
+ // turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
6807
+ // when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
6808
+ // + enrollmentId). When it is NOT primary — the agent swapped back to the
6809
+ // group box (sandbox_swap 'session'/'default'/groupId clears the pointer) or
6810
+ // selfhosted routing is flag-OFF (the pointer is ignored) — the else-branch
6811
+ // must resume a REAL cloud group box, not a "selfhosted" one: the registry
6812
+ // SelfhostedSandboxClient has no bound agentId and throws. Fall the group-box
6813
+ // backend back to the deployment default cloud backend so swap-away / flag-off
6814
+ // degrade to a genuine cloud box exactly like today (home=modal did).
6815
+ if (machinePrimary) {
6816
+ // STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
6817
+ // establish OR lease the managed home box. The active-sandbox pointer is
6818
+ // the machine route's own epoch fence; the managed-home lease separately
6819
+ // owns cloud recovery, snapshots, billing, and reaping. Mixing those two
6820
+ // ownership domains lets an unrelated degraded cloud archive block a
6821
+ // healthy explicitly selected machine.
6822
+ // Whether the machine's latest Hello advertised the op-stream engine
6823
+ // (refreshed on every connect). Read only when the server flag is on —
6824
+ // one indexed lookup, and the flag off keeps this path byte-identical.
6825
+ const machineOpStream =
6826
+ settings.agentOpStreamEnabled === true
6827
+ ? (await getEnrollment(db, input.workspaceId, activeSandboxRecord!.enrollmentId!))
6828
+ ?.opStream === true
6829
+ : false;
6830
+ const established = await establishSelfhostedTurnSession(
6111
6831
  {
6112
6832
  db,
6113
6833
  settings,
6114
6834
  bus,
6115
- opJournal,
6116
6835
  onOp: machineOpObserver.observer,
6117
6836
  onSandboxOperation: sandboxOperationObserver,
6118
- onHomeSandboxRebound,
6119
- ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
6120
- },
6121
- {
6122
- workspaceId: input.workspaceId,
6123
- sessionId: input.sessionId,
6124
- environment: sandboxEnvironment,
6125
- workspaceMutationFence: {
6126
- accountId: input.accountId,
6127
- turnId: turn.id,
6128
- executionGeneration,
6129
- attemptId: input.attemptId,
6130
- },
6131
- pinnedSelfhosted: {
6132
- sandboxId: activeSandboxPointer!.activeSandboxId!,
6133
- epoch: activeSandboxPointer!.activeEpoch,
6134
- },
6135
- // HOME semantics for a mid-turn clear-to-null: only a genuine
6136
- // machine-HOME session (its home IS this machine, session.sandboxBackend
6137
- // === "selfhosted") resolves null back to the pinned machine. A Modal-HOME
6138
- // session merely PINNED to a machine this turn never established its group
6139
- // box, so defaultIsHome:false makes a clear-to-null fail typed
6140
- // (`home_unavailable_this_turn`) rather than silently serving the machine;
6141
- // the detach takes effect next turn. (Lazy home-box establishment on such a
6142
- // clear is a deferred follow-up; issue #341.)
6143
- defaultIsHome: session.sandboxBackend === "selfhosted",
6144
- },
6145
- established,
6146
- ),
6147
- // For a machine route this is the active-pointer epoch, not a cloud
6148
- // lease epoch. Cloud-only heartbeat/snapshot/capture paths require
6149
- // sandboxGroupId and remain disabled for this branch.
6150
- leaseEpoch: activeSandboxPointer!.activeEpoch,
6151
- release: async () => undefined,
6152
- };
6153
- } else if (establishPolicy === "on-demand") {
6154
- // Lazy sandbox provisioning: holder/group ids are fixed at turn start,
6155
- // but the lease acquire + box establish + setup move behind the routing
6156
- // proxy's first default-pointer op. A chat-only turn never calls it, so
6157
- // no lease row, no provider box, no warm-meter interval.
6158
- } else {
6159
- resolvedSandbox = await waitForTurnOperation(
6160
- resumeBoxForTurn(
6161
- {
6162
- db,
6163
- settings: runSettings,
6164
- logicalFallbackSettings: logicalSandboxSettings,
6165
- cancellationSignal: sandboxResumeSignal,
6166
- sandboxMetrics: runtimeMetricsHooksForObservability(observability),
6167
- onSandboxLost: publishSandboxLost,
6837
+ opJournal,
6168
6838
  },
6169
6839
  {
6170
- accountId: input.accountId,
6171
6840
  workspaceId: input.workspaceId,
6172
- sandboxGroupId: session.sandboxGroupId,
6173
- sessionId: input.sessionId,
6174
- // groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
6175
- // is not machine-primary resumes a real cloud group box (the
6176
- // deployment default), never a "selfhosted" box (which would throw
6177
- // for lack of a bound agentId).
6178
- backend: groupBoxBackend,
6179
- os: session.sandboxOs,
6841
+ agentId: activeSandboxRecord!.enrollmentId!,
6842
+ opStream: machineOpStream,
6843
+ epoch: activeSandboxPointer!.activeEpoch,
6180
6844
  environment: sandboxEnvironment,
6181
- // IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
6182
- // this run resolves. The lease stamps it + conflicts on a live shared box
6183
- // running a DIFFERENT image (solo → durable rotation; N-holders →
6184
- // SandboxImageConflictError surfaced as an actionable turn error). Prefer
6185
- // the explicit Modal image ref, else the docker image. The selfhosted branch
6186
- // (establishSelfhostedTurnSession) NEVER passes
6187
- // an image — B3 lives only on this Modal else-branch.
6188
- ...((runSettings.modalImageRef ?? runSettings.dockerImage)
6189
- ? {
6190
- image: runSettings.modalImageRef ?? runSettings.dockerImage,
6191
- }
6845
+ ...(transientCodemodeEnvironment
6846
+ ? { transientExecEnvironment: transientCodemodeEnvironment }
6192
6847
  : {}),
6193
- // RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
6194
- // conflicts on a live shared box set up under a different rig (solo
6195
- // durable rotation / N-holders SandboxRigConflictError). Omitted for a
6196
- // rig-less turn -> never stamped or enforced (shares exactly as today).
6197
- ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
6198
- },
6199
- "turn",
6200
- managedOwnership!.holderId,
6201
- ),
6202
- sandboxResumeSignal,
6203
- releaseLateSandbox,
6204
- );
6205
- setupBoxSession = resolvedSandbox.established.session;
6206
- // Durable box-lifecycle events (sandbox-file-persistence observability):
6207
- // record every box transition in session_events so the NEXT box loss is
6208
- // attributable from the DB alone — worker logs rotate within hours, which
6209
- // left both 2026-07-06 incidents without a durable trace. Best-effort.
6210
- await publishSandboxLifecycleEvents(resolvedSandbox);
6211
- // M7 hot-swap: when the selfhosted feature is on, wrap the established
6212
- // group box in the STABLE routing proxy before it is injected NON-OWNED
6213
- // into the run. The SDK binds to this ONE object once and calls its
6214
- // methods per tool call; the proxy re-reads (active_sandbox_id,
6215
- // active_epoch) per op and dispatches to the currently-active backend, so
6216
- // a sandbox_swap mid-turn lands the NEXT tool call on the new box. With
6217
- // the flag off the established group box is injected unchanged (today's
6218
- // path). The lease still owns the group box lifecycle — the proxy is a
6219
- // routing veneer, not an owner.
6220
- resolvedSandbox = {
6221
- ...resolvedSandbox,
6222
- established: wrapTurnBoxWithRouting(
6223
- {
6224
- db,
6225
- settings,
6226
- bus,
6227
- opJournal,
6228
- onSandboxOperation: sandboxOperationObserver,
6229
- onHomeSandboxLost: publishSandboxLost,
6230
- onHomeSandboxRebound,
6231
- ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
6848
+ workingDir: activeSandboxPointer!.workingDir,
6232
6849
  },
6233
- // Thread the SAME declared environment the group box was created with
6234
- // (resumeBoxForTurn, above) so a selfhosted swap target's manifest
6235
- // carries it too — the SDK's per-turn manifest-env delta stays empty
6236
- // (no "cannot change manifest environment variables" throw).
6237
- {
6238
- workspaceId: input.workspaceId,
6239
- sessionId: input.sessionId,
6240
- environment: sandboxEnvironment,
6241
- workspaceMutationFence: {
6242
- accountId: input.accountId,
6243
- turnId: turn.id,
6244
- executionGeneration,
6245
- attemptId: input.attemptId,
6850
+ );
6851
+ // The machine-primary establish narrows `session` to SelfhostedSession
6852
+ // (buildSelfhostedBackendSession); EstablishedSandboxSession widens it.
6853
+ machinePrimarySession =
6854
+ established.session as import("@opengeni/runtime").SelfhostedSession;
6855
+ setupBoxSession = established.session;
6856
+ resolvedSandbox = {
6857
+ // Wrap in the SAME routing proxy so a mid-turn swap (to another machine
6858
+ // or back to the group box) still re-routes per op. PIN this established
6859
+ // SelfhostedSession for the machine pointer so the turn-start manifest
6860
+ // write (via the proxy's `state` getter) and the per-op reads hit ONE
6861
+ // instance — no two-instance manifest divergence.
6862
+ established: wrapTurnBoxWithRouting(
6863
+ {
6864
+ db,
6865
+ settings,
6866
+ bus,
6867
+ opJournal,
6868
+ onOp: machineOpObserver.observer,
6869
+ onSandboxOperation: sandboxOperationObserver,
6870
+ onHomeSandboxRebound,
6871
+ ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
6872
+ },
6873
+ {
6874
+ workspaceId: input.workspaceId,
6875
+ sessionId: input.sessionId,
6876
+ environment: sandboxEnvironment,
6877
+ ...(transientCodemodeEnvironment
6878
+ ? { transientExecEnvironment: transientCodemodeEnvironment }
6879
+ : {}),
6880
+ workspaceMutationFence: {
6881
+ accountId: input.accountId,
6882
+ turnId: turn.id,
6883
+ executionGeneration,
6884
+ attemptId: input.attemptId,
6885
+ },
6886
+ pinnedSelfhosted: {
6887
+ sandboxId: activeSandboxPointer!.activeSandboxId!,
6888
+ epoch: activeSandboxPointer!.activeEpoch,
6889
+ },
6890
+ // HOME semantics for a mid-turn clear-to-null: only a genuine
6891
+ // machine-HOME session (its home IS this machine, session.sandboxBackend
6892
+ // === "selfhosted") resolves null back to the pinned machine. A Modal-HOME
6893
+ // session merely PINNED to a machine this turn never established its group
6894
+ // box, so defaultIsHome:false makes a clear-to-null fail typed
6895
+ // (`home_unavailable_this_turn`) rather than silently serving the machine;
6896
+ // the detach takes effect next turn. (Lazy home-box establishment on such a
6897
+ // clear is a deferred follow-up; issue #341.)
6898
+ defaultIsHome: session.sandboxBackend === "selfhosted",
6899
+ },
6900
+ established,
6901
+ ),
6902
+ // For a machine route this is the active-pointer epoch, not a cloud
6903
+ // lease epoch. Cloud-only heartbeat/snapshot/capture paths require
6904
+ // sandboxGroupId and remain disabled for this branch.
6905
+ leaseEpoch: activeSandboxPointer!.activeEpoch,
6906
+ release: async () => undefined,
6907
+ };
6908
+ } else if (establishPolicy === "on-demand") {
6909
+ // Lazy sandbox provisioning: holder/group ids are fixed at turn start,
6910
+ // but the lease acquire + box establish + setup move behind the routing
6911
+ // proxy's first default-pointer op. A chat-only turn never calls it, so
6912
+ // no lease row, no provider box, no warm-meter interval.
6913
+ } else {
6914
+ resolvedSandbox = await waitForTurnOperation(
6915
+ resumeBoxForTurn(
6916
+ {
6917
+ db,
6918
+ settings: runSettings,
6919
+ logicalFallbackSettings: logicalSandboxSettings,
6920
+ cancellationSignal: sandboxResumeSignal,
6921
+ sandboxMetrics: runtimeMetricsHooksForObservability(observability),
6922
+ onSandboxLost: publishSandboxLost,
6246
6923
  },
6247
- homeLease: {
6924
+ {
6248
6925
  accountId: input.accountId,
6926
+ workspaceId: input.workspaceId,
6249
6927
  sandboxGroupId: session.sandboxGroupId,
6250
- leaseEpoch: resolvedSandbox.leaseEpoch,
6251
- instanceId: resolvedSandbox.established.instanceId,
6928
+ sessionId: input.sessionId,
6929
+ // groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
6930
+ // is not machine-primary resumes a real cloud group box (the
6931
+ // deployment default), never a "selfhosted" box (which would throw
6932
+ // for lack of a bound agentId).
6252
6933
  backend: groupBoxBackend,
6934
+ os: session.sandboxOs,
6935
+ environment: sandboxEnvironment,
6936
+ // IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
6937
+ // this run resolves. The lease stamps it + conflicts on a live shared box
6938
+ // running a DIFFERENT image (solo → durable rotation; N-holders →
6939
+ // SandboxImageConflictError surfaced as an actionable turn error). Prefer
6940
+ // the explicit Modal image ref, else the docker image. The selfhosted branch
6941
+ // (establishSelfhostedTurnSession) NEVER passes
6942
+ // an image — B3 lives only on this Modal else-branch.
6943
+ ...((runSettings.modalImageRef ?? runSettings.dockerImage)
6944
+ ? {
6945
+ image: runSettings.modalImageRef ?? runSettings.dockerImage,
6946
+ }
6947
+ : {}),
6948
+ // RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
6949
+ // conflicts on a live shared box set up under a different rig (solo
6950
+ // durable rotation / N-holders SandboxRigConflictError). Omitted for a
6951
+ // rig-less turn -> never stamped or enforced (shares exactly as today).
6952
+ ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
6253
6953
  },
6254
- },
6255
- resolvedSandbox.established,
6256
- ),
6257
- };
6258
- }
6259
- if (resolvedSandbox) {
6260
- startLeaseHeartbeat(resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
6954
+ "turn",
6955
+ managedOwnership!.holderId,
6956
+ ),
6957
+ sandboxResumeSignal,
6958
+ releaseLateSandbox,
6959
+ );
6960
+ setupBoxSession = resolvedSandbox.established.session;
6961
+ // Durable box-lifecycle events (sandbox-file-persistence observability):
6962
+ // record every box transition in session_events so the NEXT box loss is
6963
+ // attributable from the DB alone — worker logs rotate within hours, which
6964
+ // left both 2026-07-06 incidents without a durable trace. Best-effort.
6965
+ await publishSandboxLifecycleEvents(resolvedSandbox);
6966
+ // M7 hot-swap: when the selfhosted feature is on, wrap the established
6967
+ // group box in the STABLE routing proxy before it is injected NON-OWNED
6968
+ // into the run. The SDK binds to this ONE object once and calls its
6969
+ // methods per tool call; the proxy re-reads (active_sandbox_id,
6970
+ // active_epoch) per op and dispatches to the currently-active backend, so
6971
+ // a sandbox_swap mid-turn lands the NEXT tool call on the new box. With
6972
+ // the flag off the established group box is injected unchanged (today's
6973
+ // path). The lease still owns the group box lifecycle — the proxy is a
6974
+ // routing veneer, not an owner.
6975
+ resolvedSandbox = {
6976
+ ...resolvedSandbox,
6977
+ established: wrapTurnBoxWithRouting(
6978
+ {
6979
+ db,
6980
+ settings,
6981
+ bus,
6982
+ opJournal,
6983
+ onSandboxOperation: sandboxOperationObserver,
6984
+ onHomeSandboxLost: publishSandboxLost,
6985
+ onHomeSandboxRebound,
6986
+ ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
6987
+ },
6988
+ // Thread the SAME declared environment the group box was created with
6989
+ // (resumeBoxForTurn, above) so a selfhosted swap target's manifest
6990
+ // carries it too — the SDK's per-turn manifest-env delta stays empty
6991
+ // (no "cannot change manifest environment variables" throw).
6992
+ {
6993
+ workspaceId: input.workspaceId,
6994
+ sessionId: input.sessionId,
6995
+ environment: sandboxEnvironment,
6996
+ ...(transientCodemodeEnvironment
6997
+ ? { transientExecEnvironment: transientCodemodeEnvironment }
6998
+ : {}),
6999
+ workspaceMutationFence: {
7000
+ accountId: input.accountId,
7001
+ turnId: turn.id,
7002
+ executionGeneration,
7003
+ attemptId: input.attemptId,
7004
+ },
7005
+ homeLease: {
7006
+ accountId: input.accountId,
7007
+ sandboxGroupId: session.sandboxGroupId,
7008
+ leaseEpoch: resolvedSandbox.leaseEpoch,
7009
+ instanceId: resolvedSandbox.established.instanceId,
7010
+ backend: groupBoxBackend,
7011
+ },
7012
+ },
7013
+ resolvedSandbox.established,
7014
+ ),
7015
+ };
7016
+ }
7017
+ if (resolvedSandbox) {
7018
+ startLeaseHeartbeat(resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
7019
+ }
7020
+ } catch (error) {
7021
+ sandboxEstablishOutcome = "failed";
7022
+ throw error;
7023
+ } finally {
7024
+ recordTurnStartupPhase(observability, {
7025
+ phase: "sandbox_establish",
7026
+ provider: turnExecutionPolicy.providerId,
7027
+ backend: machinePrimary ? "selfhosted" : groupBoxBackend,
7028
+ outcome: sandboxEstablishOutcome,
7029
+ durationSeconds: (performance.now() - sandboxEstablishStartedAt) / 1_000,
7030
+ });
6261
7031
  }
6262
7032
  }
6263
7033
 
6264
- const ordinaryFileResourceDownloads = await waitForTurnOperation(
6265
- sandboxFileDownloadsForRun(
6266
- runSettings,
6267
- db,
6268
- objectStorage,
6269
- input.workspaceId,
6270
- turnResources,
6271
- activeSandboxBackend ?? groupBoxBackend,
6272
- ),
6273
- cancellationSignal,
6274
- undefined,
6275
- );
7034
+ const ordinaryFileResourceDownloads = await (async () => {
7035
+ const fileResolutionStartedAt = performance.now();
7036
+ let fileResolutionOutcome: "completed" | "failed" = "completed";
7037
+ try {
7038
+ return await waitForTurnOperation(
7039
+ sandboxFileDownloadsForRun(
7040
+ runSettings,
7041
+ db,
7042
+ objectStorage,
7043
+ input.workspaceId,
7044
+ turnResources,
7045
+ activeSandboxBackend ?? groupBoxBackend,
7046
+ ),
7047
+ cancellationSignal,
7048
+ undefined,
7049
+ );
7050
+ } catch (error) {
7051
+ fileResolutionOutcome = "failed";
7052
+ throw error;
7053
+ } finally {
7054
+ recordTurnStartupPhase(observability, {
7055
+ phase: "file_resolution",
7056
+ provider: turnExecutionPolicy.providerId,
7057
+ backend: activeSandboxBackend ?? groupBoxBackend,
7058
+ outcome: fileResolutionOutcome,
7059
+ durationSeconds: (performance.now() - fileResolutionStartedAt) / 1_000,
7060
+ count: turnResources.length,
7061
+ });
7062
+ }
7063
+ })();
6276
7064
  if (requiredGeneratedVideoFiles.length > 0 && !objectStorage) {
6277
7065
  throw new Error("Generated video materialization requires object storage");
6278
7066
  }
@@ -6302,6 +7090,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6302
7090
  ]),
6303
7091
  ).values(),
6304
7092
  ];
7093
+ const toolContextPreparationStartedAt = performance.now();
6305
7094
  throwIfWorkerShuttingDown();
6306
7095
  throwIfTurnCancelled();
6307
7096
  const mcpCredentialRootSessionId =
@@ -6348,11 +7137,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6348
7137
  }
6349
7138
  await publish!([{ type: "tool.auth_needed", payload }], true);
6350
7139
  };
6351
- const selectedApiIntegrationIds = new Set(turnTools.map((tool) => tool.id));
7140
+ const selectedApiIntegrationServerIds = new Set(turnTools.map((tool) => tool.id));
6352
7141
  const localMcpServers = buildApiIntegrationServersForTurn({
6353
7142
  settings: runSettings,
6354
7143
  integrations: installedApiIntegrations.filter((integration) =>
6355
- selectedApiIntegrationIds.has(integration.serverId),
7144
+ selectedApiIntegrationServerIds.has(integration.serverId),
6356
7145
  ),
6357
7146
  authority: {
6358
7147
  accountId: input.accountId,
@@ -6396,54 +7185,98 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6396
7185
  };
6397
7186
  })()
6398
7187
  : undefined;
6399
- preparedTools = await waitForTurnOperation(
6400
- runtime.prepareTools(runSettings, turnTools, {
6401
- accountId: input.accountId,
6402
- workspaceId: input.workspaceId,
6403
- sessionId: input.sessionId,
6404
- // Sign the calling turn into the first-party token so tools classify
6405
- // the caller by its own identity (sacred-pause guard), not the racy
6406
- // live active pointer.
6407
- ...(turnId ? { turnId } : {}),
6408
- attemptId: input.attemptId,
6409
- executionGeneration,
6410
- subjectId: "worker:first-party-mcp",
6411
- subjectLabel: "OpenGeni worker",
6412
- ...(credentialSubjectId ? { credentialSubjectId } : {}),
6413
- ...(codexAppsAuth ? { codexAppsAuth } : {}),
6414
- resolveCredential,
6415
- onAuthNeeded: publishToolAuthNeeded,
6416
- localMcpServers,
6417
- onAttemptToolCatalog: async (catalog) => {
6418
- await persistAttemptToolCatalog(db, catalog);
6419
- },
6420
- // Manager-style sessions carry a creation-validated permission set
6421
- // for their first-party MCP token; null keeps the fixed default.
6422
- ...(session.firstPartyMcpPermissions?.length
6423
- ? { firstPartyPermissions: session.firstPartyMcpPermissions }
6424
- : {}),
6425
- firstPartyTools: session.firstPartyMcpTools ?? [...DEFAULT_FIRST_PARTY_MCP_TOOLS],
6426
- attemptToolDefinitions: createFirstPartyInteractionAttemptToolDefinitions({
6427
- settings: runSettings,
6428
- scope: {
6429
- accountId: input.accountId,
6430
- workspaceId: input.workspaceId,
6431
- sessionId: input.sessionId,
6432
- turnId: turn.id,
6433
- attemptId: input.attemptId,
6434
- executionGeneration,
7188
+ const selectedFirstPartyMcpTools = allowedFirstPartyMcpToolsForSession(
7189
+ runSettings,
7190
+ session.firstPartyMcpTools,
7191
+ );
7192
+ recordTurnStartupPhase(observability, {
7193
+ phase: "tool_context_preparation",
7194
+ provider: turnExecutionPolicy.providerId,
7195
+ backend: activeSandboxBackend ?? groupBoxBackend,
7196
+ outcome: "completed",
7197
+ durationSeconds: (performance.now() - toolContextPreparationStartedAt) / 1_000,
7198
+ count: turnTools.length,
7199
+ });
7200
+ const toolPreparationStartedAt = performance.now();
7201
+ let toolPreparationOutcome: "completed" | "failed" = "completed";
7202
+ try {
7203
+ preparedTools = await waitForTurnOperation(
7204
+ runtime.prepareTools(runSettings, turnTools, {
7205
+ accountId: input.accountId,
7206
+ workspaceId: input.workspaceId,
7207
+ sessionId: input.sessionId,
7208
+ // Sign the calling turn into the first-party token so tools classify
7209
+ // the caller by its own identity (sacred-pause guard), not the racy
7210
+ // live active pointer.
7211
+ ...(turnId ? { turnId } : {}),
7212
+ attemptId: input.attemptId,
7213
+ executionGeneration,
7214
+ subjectId: "worker:first-party-mcp",
7215
+ subjectLabel: "OpenGeni worker",
7216
+ ...(credentialSubjectId ? { credentialSubjectId } : {}),
7217
+ ...(codexAppsAuth ? { codexAppsAuth } : {}),
7218
+ resolveCredential,
7219
+ onAuthNeeded: publishToolAuthNeeded,
7220
+ localMcpServers,
7221
+ onPreparationPhase: (measurement) => {
7222
+ recordTurnStartupPhase(observability, {
7223
+ phase: `tool_${measurement.phase}`,
7224
+ provider: turnExecutionPolicy.providerId,
7225
+ backend: activeSandboxBackend ?? groupBoxBackend,
7226
+ outcome: measurement.outcome,
7227
+ durationSeconds: measurement.durationSeconds,
7228
+ count: turnTools.length,
7229
+ });
6435
7230
  },
7231
+ onAttemptToolCatalog: async (catalog) => {
7232
+ await persistAttemptToolCatalog(db, catalog);
7233
+ },
7234
+ // Manager-style sessions carry a creation-validated permission set
7235
+ // for their first-party MCP token; null keeps the fixed default.
6436
7236
  ...(session.firstPartyMcpPermissions?.length
6437
- ? { permissions: session.firstPartyMcpPermissions }
7237
+ ? { firstPartyPermissions: session.firstPartyMcpPermissions }
6438
7238
  : {}),
6439
- selectedTools: session.firstPartyMcpTools ?? [...DEFAULT_FIRST_PARTY_MCP_TOOLS],
6440
- subjectId: "worker:first-party-mcp",
6441
- subjectLabel: "OpenGeni worker",
7239
+ firstPartyTools: selectedFirstPartyMcpTools,
7240
+ nestedAgentDepth: session.nestedAgentDepth,
7241
+ effectiveMaxNestedAgentDepth: session.effectiveMaxNestedAgentDepth,
7242
+ attemptToolDefinitions: createFirstPartyInteractionAttemptToolDefinitions({
7243
+ settings: runSettings,
7244
+ scope: {
7245
+ accountId: input.accountId,
7246
+ workspaceId: input.workspaceId,
7247
+ sessionId: input.sessionId,
7248
+ turnId: turn.id,
7249
+ attemptId: input.attemptId,
7250
+ executionGeneration,
7251
+ },
7252
+ ...(session.firstPartyMcpPermissions?.length
7253
+ ? { permissions: session.firstPartyMcpPermissions }
7254
+ : {}),
7255
+ selectedTools: selectedFirstPartyMcpTools,
7256
+ subjectId: "worker:first-party-mcp",
7257
+ subjectLabel: "OpenGeni worker",
7258
+ ...(interactionInterventionResume
7259
+ ? { interventionResume: interactionInterventionResume }
7260
+ : {}),
7261
+ }),
6442
7262
  }),
6443
- }),
6444
- cancellationSignal,
6445
- async (latePreparedTools) => await latePreparedTools.close().catch(() => undefined),
6446
- );
7263
+ cancellationSignal,
7264
+ async (latePreparedTools) => await latePreparedTools.close().catch(() => undefined),
7265
+ );
7266
+ } catch (error) {
7267
+ toolPreparationOutcome = "failed";
7268
+ throw error;
7269
+ } finally {
7270
+ recordTurnStartupPhase(observability, {
7271
+ phase: "tool_preparation",
7272
+ provider: turnExecutionPolicy.providerId,
7273
+ backend: activeSandboxBackend ?? groupBoxBackend,
7274
+ outcome: toolPreparationOutcome,
7275
+ durationSeconds: (performance.now() - toolPreparationStartedAt) / 1_000,
7276
+ count: turnTools.length,
7277
+ });
7278
+ }
7279
+ const postToolPreparationStartedAt = performance.now();
6447
7280
  if (turnId && preparedTools.attemptToolEnvironment) {
6448
7281
  codemodeDispatcher = new CodemodeAttemptDispatcher(
6449
7282
  db,
@@ -6546,19 +7379,51 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6546
7379
  }
6547
7380
 
6548
7381
  if (resolvedModel?.provider.kind === "codex-subscription") {
6549
- const codexImageContext = codexContext;
6550
- const codexImageCredentialId = effectiveCodexCredentialId;
6551
- if (!codexImageContext || !codexImageCredentialId) {
6552
- throw new CodexReloginRequired(
6553
- "Codex image generation requires a connected subscription account",
6554
- );
6555
- }
7382
+ const imageAuthority = connectedSubscriptionImageGenerationAuthority(
7383
+ codexContext,
7384
+ effectiveCodexCredentialId,
7385
+ );
7386
+ if (!imageAuthority) return {};
7387
+ return {
7388
+ imageGeneration: {
7389
+ kind: "provider_adapter",
7390
+ execute: async ({ prompt, references }, { toolCallId }) => {
7391
+ const resolvedReferences = await resolveImageReferences(references);
7392
+ const receipt = await executeCodexImageGeneration({
7393
+ db,
7394
+ objectStorage,
7395
+ accountId: input.accountId,
7396
+ workspaceId: input.workspaceId,
7397
+ sessionId: input.sessionId,
7398
+ turnId: turn.id,
7399
+ attemptId: input.attemptId,
7400
+ toolCallId,
7401
+ prompt,
7402
+ references: resolvedReferences,
7403
+ credentialId: imageAuthority.credentialId,
7404
+ codexContext: imageAuthority.credentialContext,
7405
+ ...(runtimeCancellationSignal ? { abortSignal: runtimeCancellationSignal } : {}),
7406
+ });
7407
+ generatedImageReceiptsByArtifactId.set(receipt.artifact.artifactId, receipt);
7408
+ await materializeGeneratedImage(receipt);
7409
+ return receipt;
7410
+ },
7411
+ },
7412
+ };
7413
+ }
7414
+
7415
+ if (resolvedModel?.provider.kind === "xai-subscription") {
7416
+ const imageAuthority = connectedSubscriptionImageGenerationAuthority(
7417
+ xaiRequestContext,
7418
+ effectiveXaiCredentialId,
7419
+ );
7420
+ if (!imageAuthority) return {};
6556
7421
  return {
6557
7422
  imageGeneration: {
6558
7423
  kind: "provider_adapter",
6559
7424
  execute: async ({ prompt, references }, { toolCallId }) => {
6560
7425
  const resolvedReferences = await resolveImageReferences(references);
6561
- const receipt = await executeCodexImageGeneration({
7426
+ const receipt = await executeXaiSubscriptionImageGeneration({
6562
7427
  db,
6563
7428
  objectStorage,
6564
7429
  accountId: input.accountId,
@@ -6569,8 +7434,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6569
7434
  toolCallId,
6570
7435
  prompt,
6571
7436
  references: resolvedReferences,
6572
- credentialId: codexImageCredentialId,
6573
- codexContext: codexImageContext,
7437
+ credentialId: imageAuthority.credentialId,
7438
+ xaiContext: imageAuthority.credentialContext,
6574
7439
  ...(runtimeCancellationSignal ? { abortSignal: runtimeCancellationSignal } : {}),
6575
7440
  });
6576
7441
  generatedImageReceiptsByArtifactId.set(receipt.artifact.artifactId, receipt);
@@ -6623,7 +7488,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6623
7488
  if (objectStorage && videoGenerationEnabled) {
6624
7489
  if (videoGenerationPolicy.fundingSource === "opengeni_credits") {
6625
7490
  videoGenerationCredential = managedVideoGenerationCredentialLease(modelRunSettings);
6626
- } else {
7491
+ } else if (videoGenerationPolicy.fundingSource === "workspace_gateway") {
6627
7492
  const workspaceCredential = await loadWorkspaceVercelAiGatewayCredentialLease(
6628
7493
  db,
6629
7494
  modelRunSettings,
@@ -6635,6 +7500,74 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6635
7500
  ...workspaceCredential,
6636
7501
  };
6637
7502
  }
7503
+ } else if (videoGenerationPolicy.fundingSource === "supergrok_subscription") {
7504
+ const encryptionKey = environmentsEncryptionKeyBytes(modelRunSettings);
7505
+ const subjectId = xaiLeaseSubjectId ?? turn.initiatingHumanSubjectId;
7506
+ if (encryptionKey && subjectId) {
7507
+ const authoritySnapshot =
7508
+ xaiAuthoritySnapshot ??
7509
+ (await resolveXaiProviderAccountAuthoritySnapshotForAcceptance(db, {
7510
+ workspaceId: input.workspaceId,
7511
+ subjectId,
7512
+ }));
7513
+ const pin = await getXaiSessionAccountPin(db, {
7514
+ workspaceId: input.workspaceId,
7515
+ subjectId,
7516
+ sessionId: input.sessionId,
7517
+ authoritySnapshot,
7518
+ });
7519
+ const selected = effectiveXaiCredentialId
7520
+ ? {
7521
+ credentialId: effectiveXaiCredentialId,
7522
+ rotationEnabled: xaiRotationEnabled,
7523
+ }
7524
+ : await selectXaiCredentialForUse(db, {
7525
+ accountId: input.accountId,
7526
+ workspaceId: input.workspaceId,
7527
+ subjectId,
7528
+ authoritySnapshot,
7529
+ shardKey: input.sessionId,
7530
+ pinnedCredentialId: pin?.pinnedCredentialId ?? null,
7531
+ pinSource:
7532
+ pin?.pinSource === "manual" || pin?.pinSource === "policy"
7533
+ ? pin.pinSource
7534
+ : null,
7535
+ });
7536
+ if (selected.credentialId) {
7537
+ if (
7538
+ selected.rotationEnabled &&
7539
+ pin?.pinSource !== "manual" &&
7540
+ pin?.pinnedCredentialId !== selected.credentialId
7541
+ ) {
7542
+ await setXaiSessionAccountPin(db, {
7543
+ accountId: input.accountId,
7544
+ workspaceId: input.workspaceId,
7545
+ subjectId,
7546
+ sessionId: input.sessionId,
7547
+ authoritySnapshot,
7548
+ credentialId: selected.credentialId,
7549
+ pinSource: "policy",
7550
+ expectedVersion: pin?.version ?? null,
7551
+ }).catch((error: unknown) => {
7552
+ if (error instanceof Error && error.message === "xAI session pin changed") return;
7553
+ throw error;
7554
+ });
7555
+ }
7556
+ const credential = await materializeXaiCredentialForRun(db, {
7557
+ workspaceId: input.workspaceId,
7558
+ subjectId,
7559
+ credentialId: selected.credentialId,
7560
+ authoritySnapshot,
7561
+ encryptionKey,
7562
+ });
7563
+ videoGenerationCredential = xaiVideoGenerationCredentialLease({
7564
+ settings: modelRunSettings,
7565
+ credential,
7566
+ subjectId,
7567
+ authoritySnapshot,
7568
+ });
7569
+ }
7570
+ }
6638
7571
  }
6639
7572
  }
6640
7573
  const videoGenerationOption: Pick<BuildAgentOptions, "videoGeneration"> = (() => {
@@ -6661,33 +7594,47 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6661
7594
  resolvedSandbox?.established.session ?? sdkOwnedSandboxSession;
6662
7595
  const fence = toolCancellationFenceRef.current;
6663
7596
  const runAs = sandboxRunAs(modelRunSettings);
6664
- const accepted = await admitVideoGenerationRequest({
6665
- db,
6666
- storage: objectStorage,
6667
- settings: modelRunSettings,
6668
- accountId: input.accountId,
6669
- workspaceId: input.workspaceId,
6670
- sessionId: input.sessionId,
6671
- turnId: turn.id,
6672
- attemptId: input.attemptId,
6673
- toolCallId,
6674
- toolInput,
6675
- policy: videoGenerationPolicy,
6676
- credential: videoGenerationCredential,
6677
- ...(sessionForReference && fence
6678
- ? {
6679
- runCommand: async (command) =>
6680
- await fence.runSandboxCommandStructured(
6681
- sessionForReference as import("@opengeni/runtime").TurnSandboxCommandSession,
6682
- {
6683
- ...command,
6684
- ...(runAs ? { runAs } : {}),
6685
- },
6686
- ),
6687
- }
6688
- : {}),
6689
- ...(runtimeCancellationSignal ? { signal: runtimeCancellationSignal } : {}),
6690
- });
7597
+ let accepted: Awaited<ReturnType<typeof admitVideoGenerationRequest>>;
7598
+ try {
7599
+ accepted = await admitVideoGenerationRequest({
7600
+ db,
7601
+ storage: objectStorage,
7602
+ settings: modelRunSettings,
7603
+ accountId: input.accountId,
7604
+ workspaceId: input.workspaceId,
7605
+ sessionId: input.sessionId,
7606
+ turnId: turn.id,
7607
+ attemptId: input.attemptId,
7608
+ toolCallId,
7609
+ toolInput,
7610
+ policy: videoGenerationPolicy,
7611
+ credential: videoGenerationCredential,
7612
+ ...(sessionForReference && fence
7613
+ ? {
7614
+ runCommand: async (command) =>
7615
+ await fence.runSandboxCommandStructured(
7616
+ sessionForReference as import("@opengeni/runtime").TurnSandboxCommandSession,
7617
+ {
7618
+ ...command,
7619
+ ...(runAs ? { runAs } : {}),
7620
+ },
7621
+ ),
7622
+ }
7623
+ : {}),
7624
+ ...(runtimeCancellationSignal ? { signal: runtimeCancellationSignal } : {}),
7625
+ });
7626
+ } catch (error) {
7627
+ if (error instanceof VideoReferenceInputError) {
7628
+ return VideoGenerationRejectedResult.parse({
7629
+ schemaVersion: 1,
7630
+ status: "rejected",
7631
+ code: error.code,
7632
+ message: error.message,
7633
+ operationCreated: false,
7634
+ });
7635
+ }
7636
+ throw error;
7637
+ }
6691
7638
  videoGenerationAcceptancesByCallId.set(toolCallId, {
6692
7639
  operationId: accepted.operationId,
6693
7640
  requestDigest: accepted.requestDigest,
@@ -6727,179 +7674,218 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6727
7674
  outcome,
6728
7675
  }),
6729
7676
  };
6730
- const agent = runtime.buildAgent(modelRunSettings, turnResources, {
6731
- reasoningEffort: turn.reasoningEffort,
6732
- latencyMode: turnExecutionPolicy.latencyMode,
6733
- ...(serviceTier ? { serviceTier } : {}),
6734
- ...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
6735
- humanInputEnabled: agentHumanInputEnabled,
6736
- genesisTitleHint: isGenesisTurn,
6737
- persistentSessionSettings: {
6738
- titleIsSet: Boolean(session.title?.trim()),
6739
- },
6740
- sandboxEnvironment,
6741
- ...(preparedTools.attemptToolCatalog
6742
- ? { attemptToolCatalog: preparedTools.attemptToolCatalog }
6743
- : {}),
6744
- ...(sandboxArtifactRuntime.available ? { artifactRuntimeAvailable: true } : {}),
6745
- ...(cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {}),
6746
- onToolCancellationFence: (fence) => {
6747
- toolCancellationFenceRef.current = fence;
6748
- },
6749
- // TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
6750
- // seed. ONLY when the effective backend is NOT selfhosted (the connected
6751
- // machine uses its own git creds — mirrors the skipGitHubToken gate above)
6752
- // AND the mint actually produced a token (repo resources present). The runtime
6753
- // seeds it to the box's token file before the repository-clone runs; it never
6754
- // touches the box/agent manifest env.
6755
- ...(activeSandboxBackend !== "selfhosted" && sandboxGitTokens
6756
- ? { gitTokenSeeds: sandboxGitTokens }
6757
- : {}),
6758
- ...(activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings
6759
- ? { gitCredentialBindings: sandboxGitCredentialBindings }
6760
- : {}),
6761
- ...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
6762
- ? { gitTokenSeed: sandboxGitToken }
6763
- : {}),
6764
- // Codemode delivery is managed-sandbox-only. Connected Machines own any
6765
- // manually configured API credentials and must never be contacted during
6766
- // turn admission merely to seed OpenGeni tooling.
6767
- ...(sandboxCodemodeToken
6768
- ? {
6769
- codemodeTokenSeed: sandboxCodemodeToken,
6770
- codemodeTokenSessionId: input.sessionId,
6771
- }
6772
- : {}),
6773
- ...(activeSandboxBackend ? { activeSandboxBackend } : {}),
6774
- fileResourceDownloads,
6775
- mcpServers: preparedTools.mcpServers,
6776
- resolvedMcpConnectionIds: preparedTools.resolvedMcpConnectionIds,
6777
- connectorActionPolicy,
6778
- // LIVE by-reference connector namespaces (fills during this turn's
6779
- // codex_apps tools/list): the codex tool_search description reads it per
6780
- // model call so the model sees the account's real connected sources.
6781
- codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
6782
- // Resolved-model routing + gating (legacy defaults when null). The model
6783
- // is passed as the model *string* (agent.model = runSettings.openaiModel),
6784
- // NOT a Model instance: an instance only survives the in-process
6785
- // ("none") run, whereas the SandboxAgent/Modal path drops it and
6786
- // re-resolves the model *name* through the global MultiProviderModelProvider
6787
- // configureOpenAI installed — so registry models (Fireworks GLM) route to
6788
- // their own client instead of 404ing against the built-in Azure/OpenAI
6789
- // client. The gating still comes from the resolved provider: server-side
6790
- // store/compaction follow the provider's compaction mode (registry
6791
- // providers resolve to "client"); encrypted reasoning is only
6792
- // round-tripped on the Responses wire API; hosted web search is attached
6793
- // whenever the provider declares it runnable and is independent of the
6794
- // session's MCP allow-list; the effective context window drives the
6795
- // compaction threshold.
6796
- hostedWebSearch,
6797
- ...imageGenerationOption,
6798
- ...videoGenerationOption,
6799
- lazyToolTransport,
6800
- supportsImageInput,
6801
- inputFileMediaTypes: modelInputPolicy.inputFileMediaTypes,
6802
- ...(resolvedModel
6803
- ? {
6804
- encryptedReasoning:
6805
- resolvedModel.provider.api === "responses" &&
6806
- runSettings.openaiReasoningEncryptedContent,
6807
- contextWindowTokens:
6808
- resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
6809
- // The ChatGPT/Codex backend rejects the SDK's HOSTED apply_patch
6810
- // tool. Gateway Responses routes likewise expose ordinary function
6811
- // tools, not OpenAI-hosted sandbox tools. Tell buildAgent to use
6812
- // function apply_patch and wrap successful view_image results as
6813
- // typed input_image content. Chat wires have no proven typed image
6814
- // result transport and therefore receive no view_image tool.
6815
- structuredToolTransport: structuredToolTransportForTurn(resolvedModel),
6816
- // EXPLICIT computer-use tool transport, derived from the resolved provider's
6817
- // authoritative wire identity (codex → function-image, chat → disabled,
6818
- // responses → hosted) so the runtime never string-sniffs the model instance's
6819
- // constructor name. See {@link computerToolModeForTurn}.
6820
- computerToolMode: computerToolModeForTurn(resolvedModel),
6821
- ...(promptCacheKey ? { promptCacheKey } : {}),
6822
- }
6823
- : // LEGACY global-client fallback (resolveTurnModel returned null → the model
6824
- // is not in the registry, served by the built-in OpenAI/Azure Responses
6825
- // client). That backend has real hosted support, so pin computerToolMode to
6826
- // "hosted" EXPLICITLY rather than leaving the runtime to sniff the instance.
6827
- {
6828
- computerToolMode: computerToolModeForTurn(null),
6829
- promptCacheKey: input.sessionId,
6830
- }),
6831
- // Lazy computer-use seam: runtime first brings up :0 only after the model
6832
- // selects a computer tool, then this hook begins the optional proof
6833
- // recording. Shell/filesystem turns never invoke either operation.
6834
- onComputerUseReady: async () => {
6835
- if (!resolvedSandbox) {
6836
- throw new Error("Computer-use display became ready without a resolved sandbox");
6837
- }
6838
- // This callback is the authoritative execution boundary. Record the
6839
- // action before async ffmpeg startup so transport-event ordering cannot
6840
- // make settlement misclassify a real computer turn as unused.
6841
- didComputerUse = true;
6842
- await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
6843
- },
6844
- onRetainableSessionImageOutput: retainSessionImageAtToolBoundary,
6845
- ...(packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {}),
6846
- ...(session.skills.length > 0 ? { sessionSkills: session.skills } : {}),
6847
- ...(skillLibraryRuntime.skillLibrarySkills.length > 0
6848
- ? {
6849
- skillLibrarySkills: skillLibraryRuntime.skillLibrarySkills,
6850
- skillLibrarySelections: skillLibraryRuntime.skillLibrarySelections,
6851
- }
6852
- : {}),
6853
- ...(!structuredWorkspacePolicyActive && workspaceAgentInstructions
6854
- ? { instructionsTemplate: workspaceAgentInstructions }
6855
- : {}),
6856
- ...(workspaceGovernance ? { workspaceGovernance } : {}),
6857
- ...(workspaceMemory ? { workspaceMemory } : {}),
6858
- // Per-session persona tier (session > workspace > deployment default).
6859
- // Composed system-level AFTER the workspace persona so it refines it for
6860
- // this one session; absent ⇒ byte-identical to today's composition.
6861
- ...(session.instructions ? { sessionInstructions: session.instructions } : {}),
6862
- // Exact host context captured when this turn was accepted. It is
6863
- // system-level and disappears with the turn rather than entering chat.
6864
- ...(turn.turnInstructions || mcpAvailabilityInstructions
6865
- ? {
6866
- turnInstructions: [turn.turnInstructions, mcpAvailabilityInstructions]
6867
- .filter((value): value is string => Boolean(value))
6868
- .join(" "),
6869
- }
6870
- : {}),
6871
- ...workspaceEnvironmentOption,
6872
- // RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
6873
- // the frozen version carries a non-empty script), and the rig credential
6874
- // hooks. All absent for a rig-less turn (byte-for-byte today).
6875
- ...(rigVersion && rigName
6876
- ? {
6877
- rig: { name: rigName, version: rigVersion.version },
6878
- ...(rigVersion.setupScript && rigVersion.setupScript.trim().length > 0
6879
- ? {
6880
- rigSetup: {
6881
- rigId: session.rigId!,
6882
- versionId: rigVersion.id,
6883
- rigName,
6884
- script: rigVersion.setupScript,
6885
- timeoutMs: runSettings.rigSetupTimeoutMs,
6886
- contentHash: rigProviderImageContentHash({
6887
- backend: turn.sandboxBackend,
6888
- sourceImage: rigProviderImageSourceImage(
6889
- logicalSandboxSettings,
6890
- turn.sandboxBackend,
6891
- ),
6892
- definition: rigVersion,
6893
- }),
6894
- },
6895
- }
6896
- : {}),
6897
- ...(rigVersion.credentialHooks.length > 0
6898
- ? { rigCredentialHookIds: rigVersion.credentialHooks }
6899
- : {}),
6900
- }
6901
- : {}),
7677
+ const runtimeSkillActivations = [
7678
+ ...installedSkillRuntime.activations,
7679
+ ...packRuntime.skillActivations,
7680
+ ...session.skills.map((skill) => ({
7681
+ source: "session" as const,
7682
+ id: `session:${session.id}:${skill.name}`,
7683
+ artifact: {
7684
+ name: skill.name,
7685
+ description: skill.description ?? null,
7686
+ files: skill.files.map((file) => ({
7687
+ path: file.path,
7688
+ content: file.content,
7689
+ })),
7690
+ },
7691
+ reason: "attached to session",
7692
+ })),
7693
+ ];
7694
+ recordTurnStartupPhase(observability, {
7695
+ phase: "post_tool_preparation",
7696
+ provider: turnExecutionPolicy.providerId,
7697
+ backend: activeSandboxBackend ?? groupBoxBackend,
7698
+ outcome: "completed",
7699
+ durationSeconds: (performance.now() - postToolPreparationStartedAt) / 1_000,
6902
7700
  });
7701
+ const agent = (() => {
7702
+ const agentConstructionStartedAt = performance.now();
7703
+ let agentConstructionOutcome: "completed" | "failed" = "completed";
7704
+ try {
7705
+ return runtime.buildAgent(modelRunSettings, turnResources, {
7706
+ reasoningEffort: turn.reasoningEffort,
7707
+ latencyMode: turnExecutionPolicy.latencyMode,
7708
+ ...(serviceTier ? { serviceTier } : {}),
7709
+ ...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
7710
+ humanInputEnabled: agentHumanInputEnabled,
7711
+ genesisTitleHint: isGenesisTurn,
7712
+ persistentSessionSettings: {
7713
+ titleIsSet: Boolean(session.title?.trim()),
7714
+ },
7715
+ sandboxEnvironment,
7716
+ ...(preparedTools.attemptToolCatalog
7717
+ ? { attemptToolCatalog: preparedTools.attemptToolCatalog }
7718
+ : {}),
7719
+ ...(sandboxArtifactRuntime.available ? { artifactRuntimeAvailable: true } : {}),
7720
+ ...(cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {}),
7721
+ onToolCancellationFence: (fence) => {
7722
+ toolCancellationFenceRef.current = fence;
7723
+ },
7724
+ // TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
7725
+ // seed. ONLY when the effective backend is NOT selfhosted (the connected
7726
+ // machine uses its own git creds — mirrors the skipGitHubToken gate above)
7727
+ // AND the mint actually produced a token (repo resources present). The runtime
7728
+ // seeds it to the box's token file before the repository-clone runs; it never
7729
+ // touches the box/agent manifest env.
7730
+ ...(activeSandboxBackend !== "selfhosted" && sandboxGitTokens
7731
+ ? { gitTokenSeeds: sandboxGitTokens }
7732
+ : {}),
7733
+ ...(activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings
7734
+ ? { gitCredentialBindings: sandboxGitCredentialBindings }
7735
+ : {}),
7736
+ ...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
7737
+ ? { gitTokenSeed: sandboxGitToken }
7738
+ : {}),
7739
+ ...(sandboxCodemodeToken ? { codemodeAvailable: true } : {}),
7740
+ // Managed boxes receive the bearer through their protected per-session
7741
+ // token file. Connected Machines use transient per-exec delivery above,
7742
+ // so they must not run the file-seeding lifecycle hook.
7743
+ ...(activeSandboxBackend !== "selfhosted" && sandboxCodemodeToken
7744
+ ? {
7745
+ codemodeTokenSeed: sandboxCodemodeToken,
7746
+ codemodeTokenSessionId: input.sessionId,
7747
+ }
7748
+ : {}),
7749
+ ...(activeSandboxBackend ? { activeSandboxBackend } : {}),
7750
+ fileResourceDownloads,
7751
+ mcpServers: preparedTools.mcpServers,
7752
+ resolvedMcpConnectionIds: preparedTools.resolvedMcpConnectionIds,
7753
+ connectorActionPolicy,
7754
+ // LIVE by-reference connector namespaces (fills during this turn's
7755
+ // codex_apps tools/list): the codex tool_search description reads it per
7756
+ // model call so the model sees the account's real connected sources.
7757
+ codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
7758
+ // Resolved-model routing + gating (legacy defaults when null). The model
7759
+ // is passed as the model *string* (agent.model = runSettings.openaiModel),
7760
+ // NOT a Model instance: an instance only survives the in-process
7761
+ // ("none") run, whereas the SandboxAgent/Modal path drops it and
7762
+ // re-resolves the model *name* through the global MultiProviderModelProvider
7763
+ // configureOpenAI installed — so registry models (Fireworks GLM) route to
7764
+ // their own client instead of 404ing against the built-in Azure/OpenAI
7765
+ // client. The gating still comes from the resolved provider: server-side
7766
+ // store/compaction follow the provider's compaction mode (registry
7767
+ // providers resolve to "client"); encrypted reasoning is only
7768
+ // round-tripped on the Responses wire API; hosted web search is attached
7769
+ // whenever the provider declares it runnable and is independent of the
7770
+ // session's MCP allow-list; the effective context window drives the
7771
+ // compaction threshold.
7772
+ hostedWebSearch,
7773
+ ...imageGenerationOption,
7774
+ ...videoGenerationOption,
7775
+ lazyToolTransport,
7776
+ supportsImageInput,
7777
+ inputFileMediaTypes: modelInputPolicy.inputFileMediaTypes,
7778
+ ...(resolvedModel
7779
+ ? {
7780
+ encryptedReasoning:
7781
+ resolvedModel.provider.api === "responses" &&
7782
+ runSettings.openaiReasoningEncryptedContent,
7783
+ contextWindowTokens:
7784
+ resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
7785
+ // The ChatGPT/Codex backend rejects the SDK's HOSTED apply_patch
7786
+ // tool. Gateway Responses routes likewise expose ordinary function
7787
+ // tools, not OpenAI-hosted sandbox tools. Tell buildAgent to use
7788
+ // function apply_patch and wrap successful view_image results as
7789
+ // typed input_image content. Chat wires have no proven typed image
7790
+ // result transport and therefore receive no view_image tool.
7791
+ structuredToolTransport: structuredToolTransportForTurn(resolvedModel),
7792
+ // EXPLICIT computer-use tool transport, derived from the resolved provider's
7793
+ // authoritative wire identity (codex → function-image, chat → disabled,
7794
+ // responses → hosted) so the runtime never string-sniffs the model instance's
7795
+ // constructor name. See {@link computerToolModeForTurn}.
7796
+ computerToolMode: computerToolModeForTurn(resolvedModel),
7797
+ ...(promptCacheKey ? { promptCacheKey } : {}),
7798
+ }
7799
+ : // LEGACY global-client fallback (resolveTurnModel returned null → the model
7800
+ // is not in the registry, served by the built-in OpenAI/Azure Responses
7801
+ // client). That backend has real hosted support, so pin computerToolMode to
7802
+ // "hosted" EXPLICITLY rather than leaving the runtime to sniff the instance.
7803
+ {
7804
+ computerToolMode: computerToolModeForTurn(null),
7805
+ promptCacheKey: input.sessionId,
7806
+ }),
7807
+ // Lazy computer-use seam: runtime first brings up :0 only after the model
7808
+ // selects a computer tool, then this hook begins the optional proof
7809
+ // recording. Shell/filesystem turns never invoke either operation.
7810
+ onComputerUseReady: async () => {
7811
+ if (!resolvedSandbox) {
7812
+ throw new Error("Computer-use display became ready without a resolved sandbox");
7813
+ }
7814
+ // This callback is the authoritative execution boundary. Record the
7815
+ // action before async ffmpeg startup so transport-event ordering cannot
7816
+ // make settlement misclassify a real computer turn as unused.
7817
+ didComputerUse = true;
7818
+ await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
7819
+ },
7820
+ onRetainableSessionImageOutput: retainSessionImageAtToolBoundary,
7821
+ ...(runtimeSkillActivations.length > 0
7822
+ ? { skillActivations: runtimeSkillActivations }
7823
+ : {}),
7824
+ ...(!structuredWorkspacePolicyActive && workspaceAgentInstructions
7825
+ ? { instructionsTemplate: workspaceAgentInstructions }
7826
+ : {}),
7827
+ ...(workspaceGovernance ? { workspaceGovernance } : {}),
7828
+ ...(workspaceMemory ? { workspaceMemory } : {}),
7829
+ goalSnapshot: turn.goalSnapshot,
7830
+ // Per-session persona tier (session > workspace > deployment default).
7831
+ // Composed system-level AFTER the workspace persona so it refines it for
7832
+ // this one session; absent ⇒ byte-identical to today's composition.
7833
+ ...(session.instructions ? { sessionInstructions: session.instructions } : {}),
7834
+ // Exact host context captured when this turn was accepted. It is
7835
+ // system-level and disappears with the turn rather than entering chat.
7836
+ ...(turn.turnInstructions || mcpAvailabilityInstructions
7837
+ ? {
7838
+ turnInstructions: [turn.turnInstructions, mcpAvailabilityInstructions]
7839
+ .filter((value): value is string => Boolean(value))
7840
+ .join(" "),
7841
+ }
7842
+ : {}),
7843
+ ...workspaceEnvironmentOption,
7844
+ // RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
7845
+ // the frozen version carries a non-empty script), and the rig credential
7846
+ // hooks. All absent for a rig-less turn (byte-for-byte today).
7847
+ ...(rigVersion && rigName
7848
+ ? {
7849
+ rig: { name: rigName, version: rigVersion.version },
7850
+ ...(rigVersion.setupScript && rigVersion.setupScript.trim().length > 0
7851
+ ? {
7852
+ rigSetup: {
7853
+ rigId: session.rigId!,
7854
+ versionId: rigVersion.id,
7855
+ rigName,
7856
+ script: rigVersion.setupScript,
7857
+ timeoutMs: runSettings.rigSetupTimeoutMs,
7858
+ contentHash: rigProviderImageContentHash({
7859
+ backend: turn.sandboxBackend,
7860
+ sourceImage: rigProviderImageSourceImage(
7861
+ logicalSandboxSettings,
7862
+ turn.sandboxBackend,
7863
+ ),
7864
+ definition: rigVersion,
7865
+ }),
7866
+ },
7867
+ }
7868
+ : {}),
7869
+ ...(rigVersion.credentialHooks.length > 0
7870
+ ? { rigCredentialHookIds: rigVersion.credentialHooks }
7871
+ : {}),
7872
+ }
7873
+ : {}),
7874
+ });
7875
+ } catch (error) {
7876
+ agentConstructionOutcome = "failed";
7877
+ throw error;
7878
+ } finally {
7879
+ recordTurnStartupPhase(observability, {
7880
+ phase: "agent_construction",
7881
+ provider: turnExecutionPolicy.providerId,
7882
+ backend: activeSandboxBackend ?? groupBoxBackend,
7883
+ outcome: agentConstructionOutcome,
7884
+ durationSeconds: (performance.now() - agentConstructionStartedAt) / 1_000,
7885
+ });
7886
+ }
7887
+ })();
7888
+ const postAgentPreparationStartedAt = performance.now();
6903
7889
  if (modelRunSettings.sandboxBackend !== "none" && toolCancellationFenceRef.current === null) {
6904
7890
  throw new Error(
6905
7891
  "Sandbox agent construction did not install the mandatory turn tool cancellation fence",
@@ -6954,6 +7940,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6954
7940
  image: runSettings.modalImageRef ?? runSettings.dockerImage,
6955
7941
  }
6956
7942
  : {}),
7943
+ // The lazy acquire must enforce the same frozen rig authority
7944
+ // as the eager acquire; otherwise a warm box for another rig
7945
+ // can bypass the shared-state conflict/rotation fence.
7946
+ ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
6957
7947
  },
6958
7948
  "turn",
6959
7949
  lazyHolderId,
@@ -7103,6 +8093,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7103
8093
  workspaceId: input.workspaceId,
7104
8094
  sessionId: input.sessionId,
7105
8095
  environment: sandboxEnvironment,
8096
+ ...(transientCodemodeEnvironment
8097
+ ? { transientExecEnvironment: transientCodemodeEnvironment }
8098
+ : {}),
7106
8099
  workspaceMutationFence: {
7107
8100
  accountId: input.accountId,
7108
8101
  turnId: turn.id,
@@ -7123,6 +8116,40 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7123
8116
  },
7124
8117
  );
7125
8118
  }
8119
+ let companyBrainContributionReceiptRecorded = false;
8120
+ const recordCompanyBrainContributionReceiptOnce = (): void => {
8121
+ if (companyBrainContributionReceiptRecorded) return;
8122
+ companyBrainContributionReceiptRecorded = true;
8123
+ try {
8124
+ const companyBrainContributionReceipt = buildCompanyBrainContributionReceipt({
8125
+ attemptId: input.attemptId,
8126
+ turnId: turn.id,
8127
+ nestedAgentDepth: session.nestedAgentDepth,
8128
+ memoryPromptMode,
8129
+ instructionPolicy: instructionPolicySnapshot,
8130
+ workspaceAgentInstructions,
8131
+ preferences: preferenceSnapshot,
8132
+ companyProfile: companyProfileSnapshot,
8133
+ companyProfileIncluded,
8134
+ workspaceMemory,
8135
+ skillActivations: runtimeSkillActivations,
8136
+ });
8137
+ recordCompanyBrainContributions(observability, companyBrainContributionReceipt);
8138
+ observability.info("model context contribution receipt", {
8139
+ attemptId: companyBrainContributionReceipt.attemptId,
8140
+ turnId: companyBrainContributionReceipt.turnId,
8141
+ sessionRole: companyBrainContributionReceipt.sessionRole,
8142
+ memoryPromptMode: companyBrainContributionReceipt.memoryPromptMode,
8143
+ instructionPolicySnapshotId:
8144
+ companyBrainContributionReceipt.instructionPolicySnapshotId,
8145
+ preferenceSnapshotId: companyBrainContributionReceipt.preferenceSnapshotId,
8146
+ companyProfileSnapshotId: companyBrainContributionReceipt.companyProfileSnapshotId,
8147
+ contributions: JSON.stringify(companyBrainContributionReceipt.contributions),
8148
+ });
8149
+ } catch {
8150
+ // Observability must never change model execution semantics.
8151
+ }
8152
+ };
7126
8153
  const agentInstructions = typeof agent.instructions === "string" ? agent.instructions : "";
7127
8154
  const compactSummarizer = compactionSummarizerFor(
7128
8155
  agentInstructions.trim() ? agentInstructions : undefined,
@@ -7371,69 +8398,109 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7371
8398
  return claimedResult({ status: "idle" });
7372
8399
  }
7373
8400
  }
8401
+ recordTurnStartupPhase(observability, {
8402
+ phase: "post_agent_preparation",
8403
+ provider: turnExecutionPolicy.providerId,
8404
+ backend: activeSandboxBackend ?? groupBoxBackend,
8405
+ outcome: "completed",
8406
+ durationSeconds: (performance.now() - postAgentPreparationStartedAt) / 1_000,
8407
+ });
7374
8408
  let fileMaterializationFailures: SandboxFileDownloadFailure[] = [];
7375
8409
  let fileDownloadsMaterializedForRun = false;
7376
8410
  if (resolvedSandbox && setupBoxSession && fileResourceDownloads.length > 0) {
7377
- const boxInstanceId = resolvedSandbox.established.instanceId;
7378
- // Managed boxes are immutable platform state, so their durable lease can
7379
- // memoize successful downloads. A connected machine is user-owned: files
7380
- // can be changed or removed between turns, so verify/materialize every
7381
- // attached file each turn instead of trusting the managed-box cache.
8411
+ const fileMaterializationStartedAt = performance.now();
8412
+ let fileMaterializationOutcome: "completed" | "failed" = "completed";
7382
8413
  const cacheMaterialization = resolvedSandbox.established.backendId !== "selfhosted";
7383
- const alreadyMaterialized = cacheMaterialization
7384
- ? await getMaterializedSandboxFileResources(db, {
7385
- accountId: input.accountId,
7386
- workspaceId: input.workspaceId,
7387
- sandboxGroupId: session.sandboxGroupId,
7388
- expectedEpoch: resolvedSandbox.leaseEpoch,
7389
- instanceId: boxInstanceId,
7390
- })
7391
- : new Set<string>();
7392
- const downloadsToMaterialize = filterUnmaterializedSandboxFileDownloads(
7393
- fileResourceDownloads,
7394
- alreadyMaterialized,
7395
- );
7396
- const runAs = sandboxRunAs(runSettings);
7397
- if (downloadsToMaterialize.length > 0) {
7398
- const materialized = await runWorkspaceMutationForSandbox(
7399
- resolvedSandbox,
7400
- "fileMaterialization",
7401
- async () =>
7402
- await materializeSandboxFileDownloads(
7403
- setupBoxSession as CodemodeTokenWriterSession,
7404
- downloadsToMaterialize,
7405
- {
7406
- onRuntimeEvent: async (event) => {
7407
- await publish!([{ type: event.type, payload: event.payload }], true);
7408
- },
7409
- ...(runAs ? { runAs } : {}),
7410
- ...(toolCancellationFenceRef.current
7411
- ? {
7412
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
7413
- toolCancellationFenceRef.current,
7414
- ),
7415
- }
7416
- : {}),
7417
- },
7418
- ),
8414
+ let fileMaterializationCache: "hit" | "miss" | "disabled" = cacheMaterialization
8415
+ ? "miss"
8416
+ : "disabled";
8417
+ try {
8418
+ const boxInstanceId = resolvedSandbox.established.instanceId;
8419
+ // Managed boxes are immutable platform state, so their durable lease can
8420
+ // memoize successful downloads. A connected machine is user-owned: files
8421
+ // can be changed or removed between turns, so verify/materialize every
8422
+ // attached file each turn instead of trusting the managed-box cache.
8423
+ const alreadyMaterialized = cacheMaterialization
8424
+ ? await getMaterializedSandboxFileResources(db, {
8425
+ accountId: input.accountId,
8426
+ workspaceId: input.workspaceId,
8427
+ sandboxGroupId: session.sandboxGroupId,
8428
+ expectedEpoch: resolvedSandbox.leaseEpoch,
8429
+ instanceId: boxInstanceId,
8430
+ })
8431
+ : new Set<string>();
8432
+ const downloadsToMaterialize = filterUnmaterializedSandboxFileDownloads(
8433
+ fileResourceDownloads,
8434
+ alreadyMaterialized,
7419
8435
  );
7420
- fileMaterializationFailures = materialized.failures;
7421
- const failedFileIds = new Set(materialized.failures.map((failure) => failure.fileId));
7422
- const succeededFileIds = downloadsToMaterialize
7423
- .map((download) => download.fileId)
7424
- .filter((fileId) => !failedFileIds.has(fileId));
7425
- if (cacheMaterialization && succeededFileIds.length > 0) {
7426
- await markSandboxFileResourcesMaterialized(db, {
7427
- accountId: input.accountId,
7428
- workspaceId: input.workspaceId,
7429
- sandboxGroupId: session.sandboxGroupId,
7430
- expectedEpoch: resolvedSandbox.leaseEpoch,
7431
- instanceId: boxInstanceId,
7432
- fileIds: succeededFileIds,
7433
- });
8436
+ if (cacheMaterialization && downloadsToMaterialize.length === 0) {
8437
+ fileMaterializationCache = "hit";
8438
+ }
8439
+ const runAs = sandboxRunAs(runSettings);
8440
+ if (downloadsToMaterialize.length > 0) {
8441
+ const materialized = await runWorkspaceMutationForSandbox(
8442
+ resolvedSandbox,
8443
+ "fileMaterialization",
8444
+ async () =>
8445
+ await materializeSandboxFileDownloads(
8446
+ setupBoxSession as CodemodeTokenWriterSession,
8447
+ downloadsToMaterialize,
8448
+ {
8449
+ onRuntimeEvent: async (event) => {
8450
+ await publish!([{ type: event.type, payload: event.payload }], true);
8451
+ },
8452
+ ...(runAs ? { runAs } : {}),
8453
+ ...(toolCancellationFenceRef.current
8454
+ ? {
8455
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
8456
+ toolCancellationFenceRef.current,
8457
+ ),
8458
+ }
8459
+ : {}),
8460
+ },
8461
+ ),
8462
+ );
8463
+ fileMaterializationFailures = materialized.failures;
8464
+ fileMaterializationOutcome = sandboxFileMaterializationOutcome(materialized.failures);
8465
+ const failedFileIds = new Set(materialized.failures.map((failure) => failure.fileId));
8466
+ const succeededFileIds = downloadsToMaterialize
8467
+ .map((download) => download.fileId)
8468
+ .filter((fileId) => !failedFileIds.has(fileId));
8469
+ if (cacheMaterialization && succeededFileIds.length > 0) {
8470
+ await markSandboxFileResourcesMaterialized(db, {
8471
+ accountId: input.accountId,
8472
+ workspaceId: input.workspaceId,
8473
+ sandboxGroupId: session.sandboxGroupId,
8474
+ expectedEpoch: resolvedSandbox.leaseEpoch,
8475
+ instanceId: boxInstanceId,
8476
+ fileIds: succeededFileIds,
8477
+ });
8478
+ }
7434
8479
  }
8480
+ fileDownloadsMaterializedForRun = true;
8481
+ } catch (error) {
8482
+ fileMaterializationOutcome = "failed";
8483
+ throw error;
8484
+ } finally {
8485
+ recordTurnStartupPhase(observability, {
8486
+ phase: "file_materialization",
8487
+ provider: turnExecutionPolicy.providerId,
8488
+ backend: activeSandboxBackend ?? groupBoxBackend,
8489
+ outcome: fileMaterializationOutcome,
8490
+ durationSeconds: (performance.now() - fileMaterializationStartedAt) / 1_000,
8491
+ count: fileResourceDownloads.length,
8492
+ cache: fileMaterializationCache,
8493
+ });
7435
8494
  }
7436
- fileDownloadsMaterializedForRun = true;
8495
+ } else {
8496
+ recordTurnStartupPhase(observability, {
8497
+ phase: "file_materialization",
8498
+ provider: turnExecutionPolicy.providerId,
8499
+ backend: activeSandboxBackend ?? groupBoxBackend,
8500
+ outcome: "completed",
8501
+ durationSeconds: 0,
8502
+ count: 0,
8503
+ });
7437
8504
  }
7438
8505
  const requiredVideoFileIds = new Set(requiredGeneratedVideoFiles.map((file) => file.fileId));
7439
8506
  const failedRequiredVideo = fileMaterializationFailures.find((failure) =>
@@ -7478,52 +8545,110 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7478
8545
  }
7479
8546
  let runInput: Awaited<ReturnType<typeof turnInput>>["input"] | null = null;
7480
8547
  const prepareRunAttemptInput = async () => {
7481
- const prepared = await turnInput(db, runtime, agent, trigger, {
7482
- turnId: activeTurnId,
7483
- recovering: turn.executionGeneration > 1,
7484
- ...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
7485
- ...(runCredentialsNote ? { runCredentialsNote } : {}),
7486
- providerApi,
7487
- projectCanonicalHistory: generatedImageHistoryProjector,
7488
- materializeModelHistory: materializeScreenshotHistory,
7489
- materializeSerializedRunState: materializeScreenshotRunState,
7490
- projectModelHistory: modelHistoryProjector,
7491
- });
7492
- for (const receipt of generatedImageReceiptsByArtifactId.values()) {
7493
- await materializeGeneratedImage(receipt);
8548
+ const historyPreparationStartedAt = performance.now();
8549
+ let historyPreparationOutcome: "completed" | "failed" = "completed";
8550
+ let preparedHistoryCount: number | null = null;
8551
+ try {
8552
+ const prepared = await turnInput(db, runtime, agent, trigger, {
8553
+ turnId: activeTurnId,
8554
+ recovering: turn.executionGeneration > 1,
8555
+ ...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
8556
+ ...(runCredentialsNote ? { runCredentialsNote } : {}),
8557
+ providerApi,
8558
+ projectCanonicalHistory: generatedImageHistoryProjector,
8559
+ materializeModelHistory: materializeScreenshotHistory,
8560
+ materializeSerializedRunState: materializeScreenshotRunState,
8561
+ projectModelHistory: modelHistoryProjector,
8562
+ onPreparationPhase: (measurement) => {
8563
+ recordTurnStartupPhase(observability, {
8564
+ phase: `history_${measurement.phase}`,
8565
+ provider: turnExecutionPolicy.providerId,
8566
+ backend: activeSandboxBackend ?? groupBoxBackend,
8567
+ outcome: measurement.outcome,
8568
+ durationSeconds: measurement.durationSeconds,
8569
+ });
8570
+ },
8571
+ });
8572
+ const generatedImageMaterializationStartedAt = performance.now();
8573
+ let generatedImageMaterializationOutcome: "completed" | "failed" = "completed";
8574
+ try {
8575
+ for (const receipt of generatedImageReceiptsByArtifactId.values()) {
8576
+ await materializeGeneratedImage(receipt);
8577
+ }
8578
+ } catch (error) {
8579
+ generatedImageMaterializationOutcome = "failed";
8580
+ throw error;
8581
+ } finally {
8582
+ recordTurnStartupPhase(observability, {
8583
+ phase: "history_generated_image_materialization",
8584
+ provider: turnExecutionPolicy.providerId,
8585
+ backend: activeSandboxBackend ?? groupBoxBackend,
8586
+ outcome: generatedImageMaterializationOutcome,
8587
+ durationSeconds: (performance.now() - generatedImageMaterializationStartedAt) / 1_000,
8588
+ count: generatedImageReceiptsByArtifactId.size,
8589
+ });
8590
+ }
8591
+ runInput = prepared.input;
8592
+ providerArtifactCandidates = prepared.providerArtifactCandidates;
8593
+ // Slice index = the length of the model-facing (active) history this turn
8594
+ // is seeded from; new items beyond it (the trigger message + this turn's
8595
+ // generated items) are the ones to persist. After a compaction this is the
8596
+ // short [summary, ...tail] active set, NOT the total row count. The
8597
+ // absolute write position is tracked separately (next whole number past
8598
+ // the max existing position) because the fractional summary row means
8599
+ // total rows no longer equal max(position)+1. Pre-compaction both reduce to
8600
+ // the old total-count value, so the common path is unchanged.
8601
+ //
8602
+ // CRITICAL: seed from the structurally repaired active-row length, not the raw active
8603
+ // count. `prepareRunInput` builds `state.history` from
8604
+ // `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
8605
+ // rows (a legacy orphan/dangling pair), the in-memory history this turn
8606
+ // starts from is K shorter than the raw row count. The reconcile slices the
8607
+ // repaired `state.history` off `persistedHistoryCount`; seeding it from
8608
+ // the raw count (K too high) skips K genuinely-new items, and a
8609
+ // `function_call` left in that skipped region can later have its
8610
+ // `function_call_result` persisted alone — the orphan that 400s on replay
8611
+ // and bricks the session (issue-61). The repaired seed is already
8612
+ // orphan-free, so it is a stable prefix of the re-sanitized history and the
8613
+ // slice begins exactly at the first genuinely-new item.
8614
+ // prepareInput already sanitized the exact durable prefix represented
8615
+ // by state.history. Carry its count forward instead of loading and
8616
+ // retaining the full active transcript a second time beside runInput.
8617
+ persistedHistoryCount = prepared.persistedHistoryCount;
8618
+ preparedHistoryCount = prepared.persistedHistoryCount;
8619
+ const historyPositionStartedAt = performance.now();
8620
+ let historyPositionOutcome: "completed" | "failed" = "completed";
8621
+ try {
8622
+ nextHistoryPosition = await nextSessionHistoryPosition(
8623
+ db,
8624
+ input.workspaceId,
8625
+ input.sessionId,
8626
+ );
8627
+ } catch (error) {
8628
+ historyPositionOutcome = "failed";
8629
+ throw error;
8630
+ } finally {
8631
+ recordTurnStartupPhase(observability, {
8632
+ phase: "history_position_load",
8633
+ provider: turnExecutionPolicy.providerId,
8634
+ backend: activeSandboxBackend ?? groupBoxBackend,
8635
+ outcome: historyPositionOutcome,
8636
+ durationSeconds: (performance.now() - historyPositionStartedAt) / 1_000,
8637
+ });
8638
+ }
8639
+ } catch (error) {
8640
+ historyPreparationOutcome = "failed";
8641
+ throw error;
8642
+ } finally {
8643
+ recordTurnStartupPhase(observability, {
8644
+ phase: "history_preparation",
8645
+ provider: turnExecutionPolicy.providerId,
8646
+ backend: activeSandboxBackend ?? groupBoxBackend,
8647
+ outcome: historyPreparationOutcome,
8648
+ durationSeconds: (performance.now() - historyPreparationStartedAt) / 1_000,
8649
+ count: preparedHistoryCount,
8650
+ });
7494
8651
  }
7495
- runInput = prepared.input;
7496
- providerArtifactCandidates = prepared.providerArtifactCandidates;
7497
- // Slice index = the length of the model-facing (active) history this turn
7498
- // is seeded from; new items beyond it (the trigger message + this turn's
7499
- // generated items) are the ones to persist. After a compaction this is the
7500
- // short [summary, ...tail] active set, NOT the total row count. The
7501
- // absolute write position is tracked separately (next whole number past
7502
- // the max existing position) because the fractional summary row means
7503
- // total rows no longer equal max(position)+1. Pre-compaction both reduce to
7504
- // the old total-count value, so the common path is unchanged.
7505
- //
7506
- // CRITICAL: seed from the structurally repaired active-row length, not the raw active
7507
- // count. `prepareRunInput` builds `state.history` from
7508
- // `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
7509
- // rows (a legacy orphan/dangling pair), the in-memory history this turn
7510
- // starts from is K shorter than the raw row count. The reconcile slices the
7511
- // repaired `state.history` off `persistedHistoryCount`; seeding it from
7512
- // the raw count (K too high) skips K genuinely-new items, and a
7513
- // `function_call` left in that skipped region can later have its
7514
- // `function_call_result` persisted alone — the orphan that 400s on replay
7515
- // and bricks the session (issue-61). The repaired seed is already
7516
- // orphan-free, so it is a stable prefix of the re-sanitized history and the
7517
- // slice begins exactly at the first genuinely-new item.
7518
- // prepareInput already sanitized the exact durable prefix represented
7519
- // by state.history. Carry its count forward instead of loading and
7520
- // retaining the full active transcript a second time beside runInput.
7521
- persistedHistoryCount = prepared.persistedHistoryCount;
7522
- nextHistoryPosition = await nextSessionHistoryPosition(
7523
- db,
7524
- input.workspaceId,
7525
- input.sessionId,
7526
- );
7527
8652
  };
7528
8653
 
7529
8654
  const forceContextCompaction = async (
@@ -7575,6 +8700,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7575
8700
  // calling runStreamAttempt again; resetting this state there would reuse
7576
8701
  // the first no-response-ID fallback key and suppress a real model call.
7577
8702
  const modelResponseState = createModelResponseEventState(claimedModelUsageSourceKeys);
8703
+ let workerPreparationTotalRecorded = false;
7578
8704
  const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
7579
8705
  if (!runInput) {
7580
8706
  throw new Error("Run input was not prepared");
@@ -7589,152 +8715,223 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7589
8715
  let currentToolBatchCallIds = new Set<string>();
7590
8716
  let currentToolBatchCompletedCallIds = new Set<string>();
7591
8717
  let streamSawPerResponseUsage = false;
7592
- // Actual input tokens of the most recent model response this turn; the
7593
- // pre-read trigger for the NEXT turn. Persisted at every turn-end path.
7594
- throwIfWorkerShuttingDown();
7595
- throwIfTurnCancelled();
7596
- const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
7597
- const runStreamOnce = async (): ReturnType<OpenGeniRuntime["runStream"]> => {
7598
- // Eager owned sessions must settle the exact platform-setup provider
7599
- // promise before the long model stream starts; otherwise one admission
7600
- // would remain in flight for the entire turn and suppress every
7601
- // heartbeat capture. Lazy setup already runs under the same wrapper in
7602
- // its first-operation provisioner above.
7603
- if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
7604
- const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
7605
- // `deferredSetup: true` below tells the runtime that the worker owns
7606
- // platform setup, so the runtime intentionally skips its credential
7607
- // session callback. Materialize host-managed run credentials here
7608
- // before any setup command (including provider login hooks), then
7609
- // decorate setup commands so they source the active generation.
7610
- await attachRunCredentialRenewal(
7611
- eagerSetupSession as RunCredentialCommandSession,
7612
- resolvedSandbox,
7613
- );
7614
- const eagerCredentialSetupSession = initialRunCredentialMaterial
7615
- ? withRunCredentialsSession(eagerSetupSession as object, input.sessionId)
7616
- : eagerSetupSession;
7617
- await runWorkspaceMutationForSandbox(
7618
- resolvedSandbox,
7619
- "eagerOwnedSandboxSetup",
7620
- async () =>
7621
- await runOwnedSandboxSetup(
7622
- agent,
7623
- ownedEstablished.session as never,
7624
- eagerCredentialSetupSession as never,
7625
- {
7626
- settings: runSettings,
7627
- environment: sandboxEnvironment,
7628
- preparedInput: runInput!,
7629
- ...(fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {}),
7630
- onRuntimeEvent: async (event) => {
7631
- await renewCodexLease("runtime_event");
7632
- if (codexLeaseLost) {
7633
- throw new Error("Codex credential lease expired during sandbox setup");
7634
- }
7635
- await publish!([{ type: event.type, payload: event.payload }], true);
7636
- },
7637
- ...(toolCancellationFenceRef.current
7638
- ? {
7639
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
7640
- toolCancellationFenceRef.current,
7641
- ),
8718
+ // Actual input tokens of the most recent model response this turn; the
8719
+ // pre-read trigger for the NEXT turn. Persisted at every turn-end path.
8720
+ throwIfWorkerShuttingDown();
8721
+ throwIfTurnCancelled();
8722
+ const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
8723
+ const runStreamOnce = async (): ReturnType<OpenGeniRuntime["runStream"]> => {
8724
+ // Eager owned sessions must settle the exact platform-setup provider
8725
+ // promise before the long model stream starts; otherwise one admission
8726
+ // would remain in flight for the entire turn and suppress every
8727
+ // heartbeat capture. Lazy setup already runs under the same wrapper in
8728
+ // its first-operation provisioner above.
8729
+ if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
8730
+ const ownedSandboxSetupStartedAt = performance.now();
8731
+ let ownedSandboxSetupOutcome: "completed" | "failed" = "completed";
8732
+ try {
8733
+ const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
8734
+ // `deferredSetup: true` below tells the runtime that the worker owns
8735
+ // platform setup, so the runtime intentionally skips its credential
8736
+ // session callback. Materialize host-managed run credentials here
8737
+ // before any setup command (including provider login hooks), then
8738
+ // decorate setup commands so they source the active generation.
8739
+ await attachRunCredentialRenewal(
8740
+ eagerSetupSession as RunCredentialCommandSession,
8741
+ resolvedSandbox,
8742
+ );
8743
+ const eagerCredentialSetupSession = initialRunCredentialMaterial
8744
+ ? withRunCredentialsSession(eagerSetupSession as object, input.sessionId)
8745
+ : eagerSetupSession;
8746
+ await runWorkspaceMutationForSandbox(
8747
+ resolvedSandbox,
8748
+ "eagerOwnedSandboxSetup",
8749
+ async () =>
8750
+ await runOwnedSandboxSetup(
8751
+ agent,
8752
+ ownedEstablished.session as never,
8753
+ eagerCredentialSetupSession as never,
8754
+ {
8755
+ settings: runSettings,
8756
+ environment: sandboxEnvironment,
8757
+ preparedInput: runInput!,
8758
+ ...(fileDownloadsMaterializedForRun
8759
+ ? { fileDownloadsMaterialized: true }
8760
+ : {}),
8761
+ onRuntimeEvent: async (event) => {
8762
+ await renewServingCredentialLease("runtime_event");
8763
+ if (servingCredentialLeaseLost()) {
8764
+ throw new Error("Provider credential lease expired during sandbox setup");
7642
8765
  }
7643
- : {}),
7644
- },
7645
- ),
7646
- );
7647
- await attachGitCredentialRenewal(
7648
- eagerSetupSession as GitCredentialTokenWriterSession,
7649
- initialGitCredentials,
7650
- );
8766
+ await publish!([{ type: event.type, payload: event.payload }], true);
8767
+ },
8768
+ ...(toolCancellationFenceRef.current
8769
+ ? {
8770
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
8771
+ toolCancellationFenceRef.current,
8772
+ ),
8773
+ }
8774
+ : {}),
8775
+ },
8776
+ ),
8777
+ );
8778
+ await attachGitCredentialRenewal(
8779
+ eagerSetupSession as GitCredentialTokenWriterSession,
8780
+ initialGitCredentials,
8781
+ );
8782
+ } catch (error) {
8783
+ ownedSandboxSetupOutcome = "failed";
8784
+ throw error;
8785
+ } finally {
8786
+ recordTurnStartupPhase(observability, {
8787
+ phase: "owned_sandbox_setup",
8788
+ provider: turnExecutionPolicy.providerId,
8789
+ backend: activeSandboxBackend ?? groupBoxBackend,
8790
+ outcome: ownedSandboxSetupOutcome,
8791
+ durationSeconds: (performance.now() - ownedSandboxSetupStartedAt) / 1_000,
8792
+ count: fileResourceDownloads.length,
8793
+ });
8794
+ }
7651
8795
  }
7652
8796
  // Conservative request boundary: once control enters runStream, the
7653
8797
  // provider may have accepted work even if no response/event follows.
7654
8798
  // Escaped MCP setup timeouts are automatically recoverable only
7655
8799
  // before this line.
7656
- modelRequestStarted = true;
7657
- return await runtime.runStream(agent, runInput!, modelRunSettings, {
7658
- signal: runtimeCancellationSignal,
7659
- sandboxEnvironment,
7660
- onRuntimeEvent: async (event) => {
7661
- await renewCodexLease("runtime_event");
7662
- if (codexLeaseLost) {
7663
- throw new Error("Codex credential lease expired during the active turn");
7664
- }
7665
- await publish!([{ type: event.type, payload: event.payload }], true);
7666
- },
7667
- // P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
7668
- // keystone). Absent when the flag is off -> legacy build-and-discard.
7669
- ...(ownedEstablished
7670
- ? {
7671
- ownedSandbox: {
7672
- client: ownedEstablished.client,
7673
- session: ownedEstablished.session,
7674
- ...(resolvedSandbox?.established.sessionState
8800
+ recordCompanyBrainContributionReceiptOnce();
8801
+ const providerDispatchStartedAt = performance.now();
8802
+ let providerDispatchOutcome: "completed" | "failed" = "completed";
8803
+ try {
8804
+ // This histogram describes worker preparation until the first entry
8805
+ // into the runtime. Lazy SDK request preparation and the durable
8806
+ // model-request audit happen after this boundary and have their own
8807
+ // phase metrics. Context-compaction recovery re-enters
8808
+ // runStreamAttempt; recording again would fold the prior request and
8809
+ // compaction into a bogus second startup sample.
8810
+ if (!workerPreparationTotalRecorded) {
8811
+ workerPreparationTotalRecorded = true;
8812
+ recordTurnWorkerPreparationTotal(observability, {
8813
+ provider: turnExecutionPolicy.providerId,
8814
+ backend: activeSandboxBackend ?? groupBoxBackend,
8815
+ outcome: "completed",
8816
+ durationSeconds: (performance.now() - workerPreparationStartedAt) / 1_000,
8817
+ });
8818
+ }
8819
+ modelRequestStarted = true;
8820
+ return await runtime.runStream(agent, runInput!, modelRunSettings, {
8821
+ signal: runtimeCancellationSignal,
8822
+ sandboxEnvironment,
8823
+ onRuntimeEvent: async (event) => {
8824
+ await renewServingCredentialLease("runtime_event");
8825
+ if (servingCredentialLeaseLost()) {
8826
+ throw new Error("Provider credential lease expired during the active turn");
8827
+ }
8828
+ await publish!([{ type: event.type, payload: event.payload }], true);
8829
+ },
8830
+ // P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
8831
+ // keystone). Absent when the flag is off -> legacy build-and-discard.
8832
+ ...(ownedEstablished
8833
+ ? {
8834
+ ownedSandbox: {
8835
+ client: ownedEstablished.client,
8836
+ session: ownedEstablished.session,
8837
+ ...(resolvedSandbox?.established.sessionState
8838
+ ? {
8839
+ sessionState: resolvedSandbox.established.sessionState,
8840
+ }
8841
+ : {}),
8842
+ // Pin platform setup (hooks + file materialization) to the un-proxied
8843
+ // established box — never through the routing proxy, which would
8844
+ // re-route those execs onto a machine swapped in mid-turn.
8845
+ ...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
8846
+ ...(fileDownloadsMaterializedForRun
8847
+ ? { fileDownloadsMaterialized: true }
8848
+ : {}),
8849
+ // Both owned paths execute setup outside runStream: eager just
8850
+ // above under its exact admission, lazy in the provisioner.
8851
+ deferredSetup: true,
8852
+ },
8853
+ }
8854
+ : {}),
8855
+ ...(activeSandboxBackend !== "selfhosted" &&
8856
+ sandboxCodemodeToken &&
8857
+ sandboxCodemodeTokenExpiresAt &&
8858
+ !lazyOwnedSandbox
8859
+ ? {
8860
+ onCodemodeTokenSessionReady: async (
8861
+ tokenSession: CodemodeTokenWriterSession,
8862
+ ) => {
8863
+ const renewalSession =
8864
+ (setupBoxSession as CodemodeTokenWriterSession | null) ?? tokenSession;
8865
+ await attachCodemodeTokenRenewal(renewalSession);
8866
+ },
8867
+ }
8868
+ : {}),
8869
+ ...(runCredentialResolver
8870
+ ? {
8871
+ runCredentialSessionId: input.sessionId,
8872
+ ...(!lazyOwnedSandbox
7675
8873
  ? {
7676
- sessionState: resolvedSandbox.established.sessionState,
8874
+ onRunCredentialSessionReady: async (
8875
+ credentialSession: RunCredentialCommandSession,
8876
+ ) => {
8877
+ const pinnedCredentialSession = setupBoxSession
8878
+ ? (setupBoxSession as RunCredentialCommandSession)
8879
+ : credentialSession;
8880
+ await attachRunCredentialRenewal(pinnedCredentialSession);
8881
+ },
7677
8882
  }
7678
8883
  : {}),
7679
- // Pin platform setup (hooks + file materialization) to the un-proxied
7680
- // established box — never through the routing proxy, which would
7681
- // re-route those execs onto a machine swapped in mid-turn.
7682
- ...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
7683
- ...(fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {}),
7684
- // Both owned paths execute setup outside runStream: eager just
7685
- // above under its exact admission, lazy in the provisioner.
7686
- deferredSetup: true,
7687
- },
7688
- }
7689
- : {}),
7690
- ...(sandboxCodemodeToken && sandboxCodemodeTokenExpiresAt && !lazyOwnedSandbox
7691
- ? {
7692
- onCodemodeTokenSessionReady: async (tokenSession: CodemodeTokenWriterSession) => {
7693
- const renewalSession =
7694
- (setupBoxSession as CodemodeTokenWriterSession | null) ?? tokenSession;
7695
- await attachCodemodeTokenRenewal(renewalSession);
7696
- },
7697
- }
7698
- : {}),
7699
- ...(runCredentialResolver
7700
- ? {
7701
- runCredentialSessionId: input.sessionId,
7702
- ...(!lazyOwnedSandbox
7703
- ? {
7704
- onRunCredentialSessionReady: async (
7705
- credentialSession: RunCredentialCommandSession,
7706
- ) => {
7707
- const pinnedCredentialSession = setupBoxSession
7708
- ? (setupBoxSession as RunCredentialCommandSession)
7709
- : credentialSession;
7710
- await attachRunCredentialRenewal(pinnedCredentialSession);
7711
- },
8884
+ }
8885
+ : {}),
8886
+ ...(modelRunSettings.sandboxBackend !== "none"
8887
+ ? {
8888
+ onSandboxSessionReady: async (sandboxSession: CodemodeTokenWriterSession) => {
8889
+ sdkOwnedSandboxSession = sandboxSession;
8890
+ for (const receipt of generatedImageReceiptsByArtifactId.values()) {
8891
+ await materializeGeneratedImageInOwnedSdkSession(receipt, sandboxSession);
7712
8892
  }
7713
- : {}),
7714
- }
7715
- : {}),
7716
- ...(modelRunSettings.sandboxBackend !== "none"
7717
- ? {
7718
- onSandboxSessionReady: async (sandboxSession: CodemodeTokenWriterSession) => {
7719
- sdkOwnedSandboxSession = sandboxSession;
7720
- for (const receipt of generatedImageReceiptsByArtifactId.values()) {
7721
- await materializeGeneratedImageInOwnedSdkSession(receipt, sandboxSession);
7722
- }
7723
- },
7724
- }
7725
- : {}),
7726
- contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
7727
- contextCompactionRequested: () =>
7728
- isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
7729
- ...(toolCancellationFenceRef.current
7730
- ? { turnToolCancellationFence: toolCancellationFenceRef.current }
7731
- : {}),
7732
- });
8893
+ },
8894
+ }
8895
+ : {}),
8896
+ contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
8897
+ contextCompactionRequested: () =>
8898
+ isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
8899
+ onModelPreparationPhase: (measurement) => {
8900
+ recordTurnStartupPhase(observability, {
8901
+ phase: `model_prepare_${measurement.phase}`,
8902
+ provider: turnExecutionPolicy.providerId,
8903
+ backend: activeSandboxBackend ?? groupBoxBackend,
8904
+ outcome: measurement.outcome,
8905
+ durationSeconds: measurement.durationSeconds,
8906
+ ...(measurement.count === undefined ? {} : { count: measurement.count }),
8907
+ });
8908
+ },
8909
+ ...(toolCancellationFenceRef.current
8910
+ ? {
8911
+ turnToolCancellationFence: toolCancellationFenceRef.current,
8912
+ }
8913
+ : {}),
8914
+ });
8915
+ } catch (error) {
8916
+ providerDispatchOutcome = "failed";
8917
+ throw error;
8918
+ } finally {
8919
+ recordTurnStartupPhase(observability, {
8920
+ phase: "provider_dispatch",
8921
+ provider: turnExecutionPolicy.providerId,
8922
+ backend: activeSandboxBackend ?? groupBoxBackend,
8923
+ outcome: providerDispatchOutcome,
8924
+ durationSeconds: (performance.now() - providerDispatchStartedAt) / 1_000,
8925
+ });
8926
+ }
7733
8927
  };
7734
8928
  if (codexLeaseLost) {
7735
8929
  throw new Error("Codex credential lease expired before the model run");
7736
8930
  }
7737
- stream = await withCodex(runStreamOnce);
8931
+ if (xaiLeaseLost) {
8932
+ throw new Error("xAI credential lease expired before the model run");
8933
+ }
8934
+ stream = await withProviderRequestContext(runStreamOnce);
7738
8935
  // Bounded provider label for the streaming SLIs — the resolved registry
7739
8936
  // provider id (or the built-in OpenAI/Azure provider), never a raw
7740
8937
  // user-supplied model string.
@@ -7752,11 +8949,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7752
8949
  },
7753
8950
  );
7754
8951
 
8952
+ const streamBootstrapStartedAt = performance.now();
8953
+ let streamBootstrapRecorded = false;
8954
+ const recordStreamBootstrap = (): void => {
8955
+ if (streamBootstrapRecorded) return;
8956
+ streamBootstrapRecorded = true;
8957
+ recordTurnStartupPhase(observability, {
8958
+ phase: "stream_bootstrap",
8959
+ provider: turnExecutionPolicy.providerId,
8960
+ backend: activeSandboxBackend ?? groupBoxBackend,
8961
+ outcome: "completed",
8962
+ durationSeconds: (performance.now() - streamBootstrapStartedAt) / 1_000,
8963
+ count: turnTools.length,
8964
+ });
8965
+ };
8966
+ if (!firstModelRequestPreparationRecorded) {
8967
+ firstModelRequestPreparationStartedAt = performance.now();
8968
+ firstModelRequestCheckpointAt = firstModelRequestPreparationStartedAt;
8969
+ }
7755
8970
  const iterator = stream.toStream()[Symbol.asyncIterator]();
7756
8971
  let streamDone = false;
7757
8972
  try {
7758
8973
  while (true) {
7759
8974
  const next = await nextStreamEvent(iterator, activityContext);
8975
+ recordStreamBootstrap();
7760
8976
  if (next.done) {
7761
8977
  streamDone = true;
7762
8978
  break;
@@ -7764,6 +8980,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7764
8980
  let stableToolCallIdsToClear: string[] | null = null;
7765
8981
  let completedCurrentToolBatch = false;
7766
8982
  let retainedScreenshotMetadata: RetainedArtifactMetadata | null = null;
8983
+ let normalizedSdkEvents: ReturnType<typeof normalizeSdkEvent> | null = null;
7767
8984
  const generatedImage = generatedImageFromSdkEvent(next.value);
7768
8985
  if (isCompletedGeneratedImageSdkEvent(next.value) && !generatedImage) {
7769
8986
  throw new Error(
@@ -7795,9 +9012,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7795
9012
  servingCredentialId: effectiveCodexCredentialId,
7796
9013
  priorSessionCredentialId: priorSessionCodexCredentialId,
7797
9014
  emittedSourceKeys: emittedModelUsageSourceKeys,
7798
- renewLease: () => renewCodexLease("model_usage"),
7799
- leaseLost: () => codexLeaseLost,
7800
- leaseLostMessage: "Codex credential lease expired during the active turn",
9015
+ renewLease: () => renewServingCredentialLease("model_usage"),
9016
+ leaseLost: servingCredentialLeaseLost,
9017
+ leaseLostMessage: "Provider credential lease expired during the active turn",
7801
9018
  setLastInputTokens: setLastInputTokensFenced,
7802
9019
  });
7803
9020
  assertModelResponseLatencyMode({
@@ -7821,6 +9038,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7821
9038
  currentToolBatchCallIds = new Set<string>();
7822
9039
  currentToolBatchCompletedCallIds = new Set<string>();
7823
9040
  await reconcileConversationTruth();
9041
+ turnLifecycleMetricsFor(observability).progress(turnId!);
7824
9042
  modelCheckpointMemoryCollector.schedule(observability);
7825
9043
  try {
7826
9044
  await ensureRunAllowed(
@@ -7936,6 +9154,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7936
9154
  retainedScreenshotMetadata,
7937
9155
  );
7938
9156
  }
9157
+ normalizedSdkEvents = normalizeSdkEvent(
9158
+ durableSdkEvent as typeof next.value,
9159
+ retainedScreenshotMetadata
9160
+ ? {
9161
+ toolOutputOverride: retainedScreenshotMetadata,
9162
+ retainedOutputEvidence: retainedScreenshotMetadata.available
9163
+ ? retainedScreenshotMetadata
9164
+ : {
9165
+ available: false,
9166
+ reason: retainedScreenshotMetadata.reason,
9167
+ },
9168
+ }
9169
+ : {},
9170
+ );
9171
+ const normalizedToolOutput = normalizedSdkEvents.find(
9172
+ (event) =>
9173
+ event.type === "agent.toolCall.output" &&
9174
+ (event.payload as { id?: unknown }).id === completedToolCall.callId,
9175
+ )?.payload as { output?: unknown } | undefined;
9176
+ if (!normalizedToolOutput || !Object.hasOwn(normalizedToolOutput, "output")) {
9177
+ throw new Error(
9178
+ `Completed SDK tool call ${completedToolCall.callId} produced no durable output projection`,
9179
+ );
9180
+ }
7939
9181
  const durableResultItem = retainedScreenshotMetadata
7940
9182
  ? (compactRetainedScreenshotHistory(
7941
9183
  [completedToolCall.resultItem],
@@ -7957,6 +9199,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7957
9199
  callId: completedToolCall.callId,
7958
9200
  modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
7959
9201
  resultItem: durableResultItem as Record<string, unknown>,
9202
+ eventOutput: normalizedToolOutput.output,
7960
9203
  ...(videoGenerationAcceptancesByCallId.has(completedToolCall.callId)
7961
9204
  ? {
7962
9205
  videoGenerationAcceptance: videoGenerationAcceptancesByCallId.get(
@@ -8011,24 +9254,31 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8011
9254
  completedCurrentToolBatch = currentBatchIsStable;
8012
9255
  }
8013
9256
  }
8014
- const normalized = normalizeSdkEvent(
8015
- durableSdkEvent as typeof next.value,
8016
- retainedScreenshotMetadata
8017
- ? {
8018
- toolOutputOverride: retainedScreenshotMetadata,
8019
- retainedOutputEvidence: retainedScreenshotMetadata.available
8020
- ? retainedScreenshotMetadata
8021
- : {
8022
- available: false,
8023
- reason: retainedScreenshotMetadata.reason,
8024
- },
8025
- }
8026
- : {},
8027
- );
9257
+ const normalized =
9258
+ normalizedSdkEvents ??
9259
+ normalizeSdkEvent(
9260
+ durableSdkEvent as typeof next.value,
9261
+ retainedScreenshotMetadata
9262
+ ? {
9263
+ toolOutputOverride: retainedScreenshotMetadata,
9264
+ retainedOutputEvidence: retainedScreenshotMetadata.available
9265
+ ? retainedScreenshotMetadata
9266
+ : {
9267
+ available: false,
9268
+ reason: retainedScreenshotMetadata.reason,
9269
+ },
9270
+ }
9271
+ : {},
9272
+ );
8028
9273
  for (const event of normalized) {
8029
9274
  streamTiming.onEvent(event.type);
8030
9275
  await batcher.push(event);
8031
9276
  }
9277
+ // Structural tool-output events await their durable append before
9278
+ // push returns. The complete result is now retained in the event
9279
+ // row and pending-call recovery receipt, so release only our
9280
+ // duplicate live SDK marker before a long turn accumulates it.
9281
+ releaseMcpResultCustomDataFromSdkEvent(durableSdkEvent);
8032
9282
  if (stableToolCallIdsToClear) {
8033
9283
  const cleared = await clearDurablePendingSessionToolCalls(db, {
8034
9284
  accountId: input.accountId,
@@ -8091,9 +9341,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8091
9341
  });
8092
9342
  let aggregateAuthoritative = false;
8093
9343
  await recordCompletedModelCallBeforeOwnershipFences({
8094
- renewLease: () => renewCodexLease("model_usage"),
8095
- leaseLost: () => codexLeaseLost,
8096
- leaseLostMessage: "Codex credential lease expired during the active turn",
9344
+ renewLease: () => renewServingCredentialLease("model_usage"),
9345
+ leaseLost: servingCredentialLeaseLost,
9346
+ leaseLostMessage: "Provider credential lease expired during the active turn",
8097
9347
  recordUsage: async () => {
8098
9348
  const billing = await recordModelUsageAndDebitCredits(settings, db, {
8099
9349
  accountId: input.accountId,
@@ -8164,6 +9414,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8164
9414
  const approvals = runtime.serializeApprovals(stream.interruptions);
8165
9415
  const humanInputInterruptions =
8166
9416
  runtime.serializeHumanInputRequests?.(stream.interruptions) ?? [];
9417
+ const interactionInterventionInterruptions =
9418
+ runtime.serializeInteractionInterventionRequests?.(stream.interruptions) ?? [];
8167
9419
  const latestWorkspace = await getWorkspace(db, input.workspaceId);
8168
9420
  if (!latestWorkspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
8169
9421
  assertWorkspaceHumanInputAllowed(
@@ -8215,6 +9467,26 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8215
9467
  },
8216
9468
  },
8217
9469
  }));
9470
+ const interactionInterventionRequests = interactionInterventionInterruptions.map(
9471
+ (interruption) => ({
9472
+ id: stableInteractionInterventionId(
9473
+ input.sessionId,
9474
+ activeTurnId,
9475
+ interruption.toolCallId,
9476
+ ),
9477
+ operationId: stableInteractionInterventionOperationId(
9478
+ input.sessionId,
9479
+ activeTurnId,
9480
+ interruption.toolCallId,
9481
+ ),
9482
+ toolCallId: interruption.toolCallId,
9483
+ input: interruption.input,
9484
+ }),
9485
+ );
9486
+ const pendingApprovals = [
9487
+ ...approvals,
9488
+ ...interactionInterventionInterruptions.map((interruption) => interruption.approval),
9489
+ ];
8218
9490
  if (
8219
9491
  !(await settle!({
8220
9492
  events: [
@@ -8236,11 +9508,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8236
9508
  sessionStatus: "requires_action",
8237
9509
  activeTurnId,
8238
9510
  runState: {
8239
- serializedRunState: compactMediaRunState(stream.state.toString()),
8240
- pendingApprovals: approvals,
9511
+ serializedRunState: compactApprovalRunState(stream.state.toString()),
9512
+ pendingApprovals,
8241
9513
  humanInputRequests: humanInputRequests.map(
8242
9514
  ({ isNew: _isNew, ...request }) => request,
8243
9515
  ),
9516
+ interactionInterventionRequests,
8244
9517
  },
8245
9518
  }))
8246
9519
  ) {
@@ -8798,6 +10071,29 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8798
10071
  ) {
8799
10072
  return await settleLostCodexAttempt(turnId, codexLeaseHolderId, codexLeaseGeneration);
8800
10073
  }
10074
+ if (xaiLeaseLost && isXaiTurn && publish && turnId && turnStartedPublished) {
10075
+ await flushRuntimeBatcher();
10076
+ await reconcileConversationTruth({ requireDurable: true });
10077
+ const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
10078
+ sessionId: input.sessionId,
10079
+ turnId,
10080
+ triggerEventId: triggerEventId!,
10081
+ attemptId: input.attemptId,
10082
+ reason: "xai_lease_lost",
10083
+ detail: { provider: "supergrok-subscription" },
10084
+ });
10085
+ if (recovery.action === "stale") {
10086
+ acknowledgeLostAttemptOwnership();
10087
+ activityStatus = "cancelled";
10088
+ turnMetricOutcome = "cancelled";
10089
+ return claimedResult({ status: "cancelled" });
10090
+ }
10091
+ acknowledgeRecoveryQuiescence();
10092
+ await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
10093
+ activityStatus = "recovering";
10094
+ turnMetricOutcome = "recovering";
10095
+ return claimedResult({ status: "recovering" });
10096
+ }
8801
10097
  // Definitive Codex credential/account refusals are the only provider
8802
10098
  // errors that may walk the pool. This is an explicit checkpoint + SAME
8803
10099
  // turn recovery, never an SDK/Temporal blind retry. A network break,
@@ -8981,6 +10277,143 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8981
10277
  }
8982
10278
  }
8983
10279
  }
10280
+ const xaiCredentialFailure =
10281
+ isXaiTurn && effectiveXaiCredentialId ? classifyXaiCredentialFailure(error) : null;
10282
+ if (
10283
+ xaiCredentialFailure &&
10284
+ effectiveXaiCredentialId &&
10285
+ xaiAuthoritySnapshot &&
10286
+ xaiLeaseSubjectId &&
10287
+ xaiLeaseHolderId &&
10288
+ xaiLeaseGeneration !== null &&
10289
+ publish &&
10290
+ turnId &&
10291
+ turnStartedPublished
10292
+ ) {
10293
+ await flushRuntimeBatcher();
10294
+ await reconcileConversationTruth({ requireDurable: true });
10295
+ const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
10296
+ const activeGoal = goal?.status === "active" ? goal : null;
10297
+ const now = new Date();
10298
+ const cooldownUntil =
10299
+ xaiCredentialFailure.kind === "rate_limit"
10300
+ ? new Date(now.getTime() + Math.max(1, xaiCredentialFailure.cooldownMs ?? 60_000))
10301
+ : null;
10302
+ const failurePayload = {
10303
+ error:
10304
+ xaiCredentialFailure.kind === "auth"
10305
+ ? "The serving SuperGrok account requires reconnection"
10306
+ : xaiCredentialFailure.kind === "forbidden"
10307
+ ? "The serving SuperGrok account is not authorized for this request"
10308
+ : "The serving SuperGrok account is temporarily rate limited",
10309
+ code:
10310
+ xaiCredentialFailure.kind === "auth"
10311
+ ? "xai_relogin_required"
10312
+ : xaiCredentialFailure.kind === "forbidden"
10313
+ ? "xai_account_forbidden"
10314
+ : "xai_account_rate_limited",
10315
+ detail: "the same accepted turn is waiting for another eligible account",
10316
+ };
10317
+ const armed = await armXaiCapacityWait(db, {
10318
+ accountId: input.accountId,
10319
+ workspaceId: input.workspaceId,
10320
+ subjectId: xaiLeaseSubjectId,
10321
+ sessionId: input.sessionId,
10322
+ turnId,
10323
+ attemptId: input.attemptId,
10324
+ workflowId: input.workflowId,
10325
+ authoritySnapshot: xaiAuthoritySnapshot,
10326
+ goalId: activeGoal?.id ?? null,
10327
+ goalVersion: activeGoal?.version ?? null,
10328
+ earliestResetAt: cooldownUntil,
10329
+ failurePayload,
10330
+ leaseFence: {
10331
+ holderId: xaiLeaseHolderId,
10332
+ generation: xaiLeaseGeneration,
10333
+ },
10334
+ credentialQuarantine:
10335
+ xaiCredentialFailure.kind === "auth"
10336
+ ? {
10337
+ kind: "status",
10338
+ status: "needs_relogin",
10339
+ lastError: "model request remained unauthorized after refresh",
10340
+ }
10341
+ : xaiCredentialFailure.kind === "forbidden"
10342
+ ? {
10343
+ kind: "status",
10344
+ status: "error",
10345
+ lastError: "model request was forbidden for this credential",
10346
+ }
10347
+ : { kind: "cooldown", until: cooldownUntil! },
10348
+ now,
10349
+ });
10350
+ if (armed.action === "waiting") {
10351
+ xaiLeaseHeld = false;
10352
+ xaiCredentialQuarantined = true;
10353
+ await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, armed.events);
10354
+ const evaluated = await reconcileXaiCapacityWait(db, {
10355
+ accountId: input.accountId,
10356
+ workspaceId: input.workspaceId,
10357
+ sessionId: input.sessionId,
10358
+ waiterId: armed.waiter.id,
10359
+ generation: armed.waiter.generation,
10360
+ now,
10361
+ });
10362
+ if (evaluated.events.length > 0) {
10363
+ await publishDurableSessionEvents(
10364
+ bus,
10365
+ input.workspaceId,
10366
+ input.sessionId,
10367
+ evaluated.events,
10368
+ );
10369
+ }
10370
+ activityError = error;
10371
+ if (evaluated.action === "resumed") {
10372
+ activityStatus = "recovering";
10373
+ turnMetricOutcome = "recovering";
10374
+ return claimedResult({ status: "recovering" });
10375
+ }
10376
+ if (evaluated.action === "waiting") {
10377
+ activityStatus = "waiting_capacity";
10378
+ turnMetricOutcome = "recovering";
10379
+ return claimedResult({
10380
+ status: "waiting_capacity",
10381
+ capacityWait: {
10382
+ provider: "xai",
10383
+ waiterId: evaluated.waiter.id,
10384
+ generation: evaluated.waiter.generation,
10385
+ nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
10386
+ wakeRevision: evaluated.waiter.wakeRevision,
10387
+ },
10388
+ });
10389
+ }
10390
+ acknowledgeLostAttemptOwnership();
10391
+ activityStatus = "cancelled";
10392
+ turnMetricOutcome = "cancelled";
10393
+ return claimedResult({ status: "cancelled" });
10394
+ }
10395
+
10396
+ const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
10397
+ sessionId: input.sessionId,
10398
+ turnId,
10399
+ triggerEventId: triggerEventId!,
10400
+ attemptId: input.attemptId,
10401
+ reason: "xai_credential_recheck",
10402
+ detail: failurePayload,
10403
+ });
10404
+ if (recovery.action === "stale") {
10405
+ acknowledgeLostAttemptOwnership();
10406
+ activityStatus = "cancelled";
10407
+ turnMetricOutcome = "cancelled";
10408
+ return claimedResult({ status: "cancelled" });
10409
+ }
10410
+ acknowledgeRecoveryQuiescence();
10411
+ await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
10412
+ activityStatus = "recovering";
10413
+ turnMetricOutcome = "recovering";
10414
+ activityError = error;
10415
+ return claimedResult({ status: "recovering" });
10416
+ }
8984
10417
  // A ChatGPT/Codex usage cap (429 usage_limit_reached) is account state,
8985
10418
  // NOT an agent failure: surface the precise, actionable message (so the
8986
10419
  // user sees the reset window) but idle the session — never go terminal,
@@ -9501,7 +10934,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
9501
10934
  }
9502
10935
  activityStatus = "failed";
9503
10936
  activityError = error;
9504
- if (!publish || !turnId || !turnStartedPublished) {
10937
+ if (!turnId) {
10938
+ throw preClaimAdmissionFailure(error);
10939
+ }
10940
+ if (!publish || !turnStartedPublished) {
9505
10941
  throw error;
9506
10942
  }
9507
10943
  // A partial/malformed stream may have emitted assistant/tool items (and
@@ -9792,6 +11228,57 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
9792
11228
  );
9793
11229
  codexLeaseHeld = false;
9794
11230
  }
11231
+ if (
11232
+ effectiveXaiCredentialId &&
11233
+ xaiLeaseSubjectId &&
11234
+ xaiRequestContext &&
11235
+ !xaiCredentialQuarantined
11236
+ ) {
11237
+ const quota = await waitForTurnFinalizerStep(
11238
+ fetchXaiSubscriptionQuota({ context: xaiRequestContext }).catch(() => null),
11239
+ finalizerSignal,
11240
+ );
11241
+ if (quota) {
11242
+ await waitForTurnFinalizerStep(
11243
+ updateXaiQuotaMetadata(db, {
11244
+ workspaceId: input.workspaceId,
11245
+ subjectId: xaiLeaseSubjectId,
11246
+ credentialId: effectiveXaiCredentialId,
11247
+ quotaUsedPercent: quota.usedPercent,
11248
+ quotaResetAt: quota.period?.end ?? null,
11249
+ quotaCheckedAt: quota.checkedAt,
11250
+ exhaustedUntil:
11251
+ quota.usedPercent !== null && quota.usedPercent >= 100
11252
+ ? (quota.period?.end ?? null)
11253
+ : null,
11254
+ }).catch(() => undefined),
11255
+ finalizerSignal,
11256
+ );
11257
+ }
11258
+ }
11259
+ if (xaiLeaseHeartbeatTimer) {
11260
+ clearInterval(xaiLeaseHeartbeatTimer);
11261
+ xaiLeaseHeartbeatTimer = undefined;
11262
+ }
11263
+ if (
11264
+ xaiLeaseHeld &&
11265
+ turnId &&
11266
+ xaiLeaseSubjectId &&
11267
+ xaiLeaseHolderId &&
11268
+ xaiLeaseGeneration !== null
11269
+ ) {
11270
+ await waitForTurnFinalizerStep(
11271
+ releaseXaiCredentialLease(db, {
11272
+ workspaceId: input.workspaceId,
11273
+ subjectId: xaiLeaseSubjectId,
11274
+ turnId,
11275
+ holderId: xaiLeaseHolderId,
11276
+ generation: xaiLeaseGeneration,
11277
+ }).catch(() => undefined),
11278
+ finalizerSignal,
11279
+ );
11280
+ xaiLeaseHeld = false;
11281
+ }
9795
11282
  // Workbench v2 turn-end workspace capture — runs FIRST in
9796
11283
  // the turn-end finally, while the box is MAXIMALLY ALIVE. The agent's last
9797
11284
  // tool ran before this finally, so /workspace is already final; capture is
@@ -10057,6 +11544,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
10057
11544
  ? safeErrorForTelemetry(finalizationError ?? activityError)
10058
11545
  : undefined,
10059
11546
  });
11547
+ // This timer runs only after the activity promise and its full turn
11548
+ // stack unwind. Checkpoint collection alone cannot reclaim objects that
11549
+ // remain strongly reachable until this terminal boundary.
11550
+ turnCompletionMemoryCollector.schedule(observability);
10060
11551
  assertPhysicalToolQuiescenceForCancellation({
10061
11552
  acknowledgeQuiescence,
10062
11553
  physicalToolQuiescenceConfirmed,
@@ -10169,6 +11660,51 @@ export function isTransientProviderError(error: unknown): boolean {
10169
11660
  );
10170
11661
  }
10171
11662
 
11663
+ export type XaiCredentialFailure = {
11664
+ kind: "auth" | "forbidden" | "rate_limit";
11665
+ cooldownMs: number | null;
11666
+ };
11667
+
11668
+ /**
11669
+ * Only definitive SuperGrok account refusals may move the same logical turn to
11670
+ * another credential. The marked transport response proves inference was
11671
+ * rejected before a stream began; refresh relogin is equally definitive.
11672
+ */
11673
+ export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailure | null {
11674
+ let relogin: unknown = error;
11675
+ for (let depth = 0; depth < 6 && relogin && typeof relogin === "object"; depth += 1) {
11676
+ if (relogin instanceof XaiSubscriptionReloginRequired) {
11677
+ return { kind: "auth", cooldownMs: null };
11678
+ }
11679
+ relogin = (relogin as Record<string, unknown>).cause;
11680
+ }
11681
+ if (!isXaiSubscriptionTransportError(error)) return null;
11682
+ let current: unknown = error;
11683
+ for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
11684
+ const value = current as Record<string, unknown>;
11685
+ const body =
11686
+ value.error && typeof value.error === "object"
11687
+ ? (value.error as Record<string, unknown>)
11688
+ : null;
11689
+ const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
11690
+ const code = String(value.code ?? body?.code ?? "").toLowerCase();
11691
+ if (status === 401 || code === "unauthorized" || code === "invalid_token") {
11692
+ return { kind: "auth", cooldownMs: null };
11693
+ }
11694
+ if (status === 403) {
11695
+ return { kind: "forbidden", cooldownMs: null };
11696
+ }
11697
+ if (status === 429 || code === "rate_limit_exceeded" || code === "too_many_requests") {
11698
+ return {
11699
+ kind: "rate_limit",
11700
+ cooldownMs: providerRetryAfterMs(error) ?? PROVIDER_BACKPRESSURE_DELAY_MS,
11701
+ };
11702
+ }
11703
+ current = value.cause;
11704
+ }
11705
+ return null;
11706
+ }
11707
+
10172
11708
  export function agentRunFailurePayload(
10173
11709
  error: unknown,
10174
11710
  options: { isCodexTurn?: boolean } = {},