@opengeni/worker-bundle 0.20.0 → 0.20.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/dist/activities/agent-turn.d.ts +25 -2
  2. package/dist/activities/google-drive-publication.d.ts +53 -0
  3. package/dist/activities/image-generation-references.d.ts +2 -0
  4. package/dist/activities/knowledge-source-sync-driver.d.ts +21 -0
  5. package/dist/activities/knowledge-source-sync.d.ts +29 -1
  6. package/dist/activities/run-input.d.ts +20 -2
  7. package/dist/activities/session-state.d.ts +5 -3
  8. package/dist/activities/types.d.ts +27 -0
  9. package/dist/{activities-control-XURE4VKN.js → activities-control-BJSDRSIP.js} +369 -20
  10. package/dist/activities-control-BJSDRSIP.js.map +1 -0
  11. package/dist/activities-control.d.ts +2 -2
  12. package/dist/{activities-turn-6EBXGGLS.js → activities-turn-J5YO7XXX.js} +2281 -1074
  13. package/dist/activities-turn-J5YO7XXX.js.map +1 -0
  14. package/dist/activities.d.ts +3 -3
  15. package/dist/{chunk-TYNPCG4U.js → chunk-N5TJIFCE.js} +69 -1
  16. package/dist/chunk-N5TJIFCE.js.map +1 -0
  17. package/dist/{chunk-5GRJIP5H.js → chunk-P6Z2SLAA.js} +2 -2
  18. package/dist/index.d.ts +1 -1
  19. package/dist/index.js +2 -2
  20. package/dist/observability-metrics.d.ts +37 -0
  21. package/dist/workflow-bundle.js +174 -4
  22. package/dist/workflows/session.d.ts +17 -1
  23. package/package.json +13 -13
  24. package/src/activities/agent-turn.ts +1801 -1175
  25. package/src/activities/documents.ts +6 -4
  26. package/src/activities/google-drive-publication.ts +840 -0
  27. package/src/activities/image-generation-references.ts +11 -2
  28. package/src/activities/knowledge-source-sync-driver.ts +23 -0
  29. package/src/activities/knowledge-source-sync.ts +384 -9
  30. package/src/activities/run-input.ts +175 -51
  31. package/src/activities/session-state.ts +69 -5
  32. package/src/activities/types.ts +27 -0
  33. package/src/activities.ts +2 -0
  34. package/src/observability-metrics.ts +184 -3
  35. package/src/workflows/session.ts +199 -8
  36. package/dist/activities-control-XURE4VKN.js.map +0 -1
  37. package/dist/activities-turn-6EBXGGLS.js.map +0 -1
  38. package/dist/chunk-TYNPCG4U.js.map +0 -1
  39. /package/dist/{chunk-5GRJIP5H.js.map → chunk-P6Z2SLAA.js.map} +0 -0
@@ -9,7 +9,9 @@ import {
9
9
  getRigName,
10
10
  getRigVersion,
11
11
  getSandbox,
12
+ getFilesForSubject,
12
13
  readActiveSandbox,
14
+ requireFileForSubject,
13
15
  setActiveSandbox,
14
16
  requireFile,
15
17
  getSessionEvent,
@@ -87,6 +89,7 @@ import {
87
89
  SandboxImageConflictError,
88
90
  ActiveSessionHistoryLimitExceededError,
89
91
  ApprovalRunStateLimitExceededError,
92
+ isRetryableDatabaseTransportFailure,
90
93
  isSessionEventPersistenceError,
91
94
  getEnrollment,
92
95
  abandonRecordingForTurnAttempt,
@@ -167,6 +170,7 @@ import {
167
170
  type ModelCallUsageInput,
168
171
  type ModelCallUsageNormalization,
169
172
  type BuildAgentOptions,
173
+ type AttemptConnectorActionBinding,
170
174
  type ConnectorActionPolicyHooks,
171
175
  type TurnToolCancellationFence,
172
176
  type BackendUnresolvableCode,
@@ -180,6 +184,12 @@ import {
180
184
  deleteRecordingArtifacts,
181
185
  stopRecording as stopRecordingOnBox,
182
186
  } from "@opengeni/runtime";
187
+ import {
188
+ authorizeGoogleDrivePublicationAttempt,
189
+ createGoogleDrivePublicationAttemptTool,
190
+ googleDrivePublicationConnectorCall,
191
+ resolveGoogleDrivePublicationTarget,
192
+ } from "./google-drive-publication";
183
193
  import { connectionTokenResolverForTurn } from "./mcp-credentials";
184
194
  import { buildApiIntegrationServersForTurn } from "./api-integrations";
185
195
  import {
@@ -344,6 +354,7 @@ import {
344
354
  import type {
345
355
  TurnActivityServices as ActivityServices,
346
356
  EscapedMcpTimeoutRecoveryDetail,
357
+ PreClaimFailureDetail,
347
358
  RunAgentTurnInput,
348
359
  RunAgentTurnResult,
349
360
  SessionAttemptQuiescenceProof,
@@ -351,6 +362,8 @@ import type {
351
362
  import {
352
363
  ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
353
364
  ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
365
+ PRE_CLAIM_FAILURE_MESSAGE,
366
+ PRE_CLAIM_FAILURE_TYPE,
354
367
  } from "./types";
355
368
  import {
356
369
  resumeBoxForTurn,
@@ -381,10 +394,14 @@ import {
381
394
  recordCompanyBrainContributions,
382
395
  recordSessionEventAppendLatency,
383
396
  recordSessionEventPublishLatency,
397
+ recordTurnSandboxEstablishPolicy,
398
+ recordTurnStartupPhase,
399
+ recordTurnWorkerPreparationTotal,
384
400
  runtimeMetricsHooksForObservability,
385
401
  StreamTimingMetrics,
386
402
  turnLifecycleMetricsFor,
387
403
  type TurnOutcome,
404
+ type TurnSandboxEstablishReason,
388
405
  } from "../observability-metrics";
389
406
  import { buildCompanyBrainContributionReceipt } from "../model-context-contributions";
390
407
  import {
@@ -660,7 +677,7 @@ export function providerRetryAfterMs(error: unknown, nowMs = Date.now()): number
660
677
  * this notice prevents a graceful runtime drop from becoming a silent source-
661
678
  * of-truth substitution or a false claim that the disconnected system was read.
662
679
  */
663
- export function unavailableMcpTurnInstructions(input: {
680
+ export function unavailableMcpOperationalContext(input: {
664
681
  droppedIds: readonly string[];
665
682
  droppedCount: number;
666
683
  }): string | undefined {
@@ -704,6 +721,52 @@ export function escapedMcpTimeoutRecoveryFailure(input: {
704
721
  });
705
722
  }
706
723
 
724
+ /**
725
+ * Convert the atomic claim transaction's failure into a small, stable
726
+ * Temporal wire contract. The original error remains in activity diagnostics,
727
+ * but SQL text, parameters, and arbitrary invariant messages never enter
728
+ * workflow history. Contention and operational database unavailability are
729
+ * safe to re-read after backoff because the claim transaction contains no
730
+ * model/tool effects; permanent database and state failures require terminal
731
+ * settlement.
732
+ */
733
+ export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
734
+ const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
735
+ const sqlState = persistenceFailure?.details.sqlState ?? null;
736
+ const rawTransportFailure = sqlState === null && isRetryableDatabaseTransportFailure(error);
737
+ // The database transaction itself retries only the two contention failures
738
+ // proven safe for immediate replay. Once the activity has failed, the
739
+ // workflow may also retry operational outages after a durable re-read and
740
+ // bounded delay. Unknown driver/database failures stay recoverable because
741
+ // they commonly represent a lost connection; known constraint, auth, and
742
+ // application SQLSTATEs are permanent and must not create an infinite loop.
743
+ const retryable = Boolean(
744
+ rawTransportFailure ||
745
+ (persistenceFailure &&
746
+ (sqlState === null ||
747
+ sqlState.startsWith("08") ||
748
+ sqlState.startsWith("40") ||
749
+ sqlState.startsWith("53") ||
750
+ sqlState === "55P03" ||
751
+ sqlState === "57014" ||
752
+ sqlState === "57P01" ||
753
+ sqlState === "57P02" ||
754
+ sqlState === "57P03" ||
755
+ sqlState.startsWith("58"))),
756
+ );
757
+ const detail: PreClaimFailureDetail = {
758
+ disposition: retryable ? "retryable" : "permanent",
759
+ code:
760
+ persistenceFailure?.details.code ?? (rawTransportFailure ? "db_failure" : "claim_invariant"),
761
+ };
762
+ return ApplicationFailure.create({
763
+ message: PRE_CLAIM_FAILURE_MESSAGE,
764
+ type: PRE_CLAIM_FAILURE_TYPE,
765
+ nonRetryable: true,
766
+ details: [detail],
767
+ });
768
+ }
769
+
707
770
  /**
708
771
  * Resolve which Codex account a turn runs on (multi-account P1): session-pin >
709
772
  * workspace-active. No rotation in P1. The selected id must still be in the
@@ -729,6 +792,12 @@ export function filterUnmaterializedSandboxFileDownloads(
729
792
  return downloads.filter((download) => !materializedFileIds.has(download.fileId));
730
793
  }
731
794
 
795
+ export function sandboxFileMaterializationOutcome(
796
+ failures: readonly SandboxFileDownloadFailure[],
797
+ ): "completed" | "failed" {
798
+ return failures.length === 0 ? "completed" : "failed";
799
+ }
800
+
732
801
  /** Fixed-length one-way tenant correlation for metrics/alerts; never a raw id. */
733
802
  export function codexWorkspaceMetricKey(workspaceId: string): string {
734
803
  return createHash("sha256").update(workspaceId).digest("hex").slice(0, 12);
@@ -1998,6 +2067,33 @@ export function shouldEstablishSandboxForTurn(
1998
2067
  return sandboxOwnershipEnabled && (homeBackend !== "none" || machinePrimary);
1999
2068
  }
2000
2069
 
2070
+ export function sandboxEstablishPolicyDecision(input: {
2071
+ lazyEnabled: boolean;
2072
+ machinePrimary: boolean;
2073
+ sandboxBackend: Settings["sandboxBackend"];
2074
+ hasInitialRunCredentialMaterial: boolean;
2075
+ generatedVideoFileCount: number;
2076
+ hasSignedFileResources: boolean;
2077
+ }): { policy: "eager" | "on-demand"; reason: TurnSandboxEstablishReason } {
2078
+ if (!input.lazyEnabled) return { policy: "eager", reason: "lazy_disabled" };
2079
+ if (input.machinePrimary) return { policy: "eager", reason: "machine_primary" };
2080
+ if (input.sandboxBackend === "none") return { policy: "eager", reason: "backend_none" };
2081
+ if (input.hasInitialRunCredentialMaterial) {
2082
+ return { policy: "eager", reason: "initial_run_credentials" };
2083
+ }
2084
+ if (input.generatedVideoFileCount > 0) {
2085
+ return { policy: "eager", reason: "generated_video_files" };
2086
+ }
2087
+ // Signed file downloads must finish before the first model boundary so any
2088
+ // integrity/download failure is present in the prepared input. Deferring the
2089
+ // box until the first tool call would make that first request advertise a
2090
+ // path whose materialization outcome is not known yet.
2091
+ if (input.hasSignedFileResources) {
2092
+ return { policy: "eager", reason: "signed_file_resources" };
2093
+ }
2094
+ return { policy: "on-demand", reason: "eligible" };
2095
+ }
2096
+
2001
2097
  /**
2002
2098
  * Classify a persisted active-sandbox pointer for TURN-START RECONCILE (issue #341
2003
2099
  * invariant B). Returns the typed reason to RESET the pointer to the session HOME,
@@ -4073,7 +4169,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4073
4169
  // only exists in the RunState blob), never through a swapped trigger.
4074
4170
  let triggerType: string | null = null;
4075
4171
  try {
4076
- const session = await requireSession(db, input.workspaceId, input.sessionId);
4077
4172
  const claim = await claimSessionWorkForAttempt(db, input.workspaceId, {
4078
4173
  sessionId: input.sessionId,
4079
4174
  workflowId: input.workflowId,
@@ -4088,10 +4183,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4088
4183
  }
4089
4184
  const turn = claim.turn;
4090
4185
  turnId = turn.id;
4186
+ // Establish durable attempt ownership before any later read can fail.
4187
+ // Therefore every failure with no turnId came from the one atomic claim
4188
+ // transaction and can be classified without conflating ordinary runtime
4189
+ // or transport failures with admission failures.
4190
+ const session = await requireSession(db, input.workspaceId, input.sessionId);
4091
4191
  executionGeneration = turn.executionGeneration;
4092
4192
  providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
4093
4193
  let installedApiIntegrations: readonly ApiIntegrationRuntime[] = [];
4094
4194
  const credentialSubjectId = credentialSubjectIdForTurnInitiator(turn);
4195
+ const fileAuthoritySubjectId = turn.initiatingHumanSubjectId ?? null;
4095
4196
  const mcpSettings = await settingsWithEnabledCapabilityMcpServers(
4096
4197
  db,
4097
4198
  input.workspaceId,
@@ -4410,6 +4511,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4410
4511
  // on a healthy worker.
4411
4512
  throwIfWorkerShuttingDown();
4412
4513
  throwIfTurnCancelled();
4514
+ recordTurnStartupPhase(observability, {
4515
+ phase: "claim_and_policy",
4516
+ provider: turnExecutionPolicy.providerId,
4517
+ backend: turn.sandboxBackend,
4518
+ outcome: "completed",
4519
+ durationSeconds: (performance.now() - activityStarted) / 1_000,
4520
+ });
4521
+ const turnStartSettlementStartedAt = performance.now();
4413
4522
  if (
4414
4523
  !(await settle({
4415
4524
  events: [
@@ -4426,7 +4535,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4426
4535
  ) {
4427
4536
  return claimedResult({ status: "cancelled" });
4428
4537
  }
4538
+ recordTurnStartupPhase(observability, {
4539
+ phase: "turn_start_settlement",
4540
+ provider: turnExecutionPolicy.providerId,
4541
+ backend: turn.sandboxBackend,
4542
+ outcome: "completed",
4543
+ durationSeconds: (performance.now() - turnStartSettlementStartedAt) / 1_000,
4544
+ count: 2,
4545
+ });
4429
4546
  turnStartedPublished = true;
4547
+ const workerPreparationStartedAt = performance.now();
4430
4548
 
4431
4549
  // Multi-account (P1): resolve the effective Codex account for this turn
4432
4550
  // (session-pin > workspace active) and stamp it on the session so the
@@ -4434,80 +4552,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4434
4552
  // when it changed from the prior run's account so the pill flips live.
4435
4553
  // Gated on the codex-billed predicate — non-codex turns never touch this.
4436
4554
  if (isCodexTurn) {
4437
- const sessionCodex = await getSessionCodexState(db, input.workspaceId, input.sessionId);
4438
- const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
4439
- const sessionPinSource = sessionCodex?.pinSource ?? null;
4440
- const selectForTurn = (context: CodexCredentialLeaseSelectionContext) =>
4441
- selectCodexCredentialLeaseForTurn({
4442
- context,
4443
- leasingEnabled: settings.codexCredentialLeasingEnabled,
4444
- sessionId: input.sessionId,
4445
- sessionPinnedCredentialId: sessionPin,
4446
- sessionPinSource,
4447
- sessionLastCredentialId: sessionCodex?.lastCredentialId ?? null,
4448
- now: new Date(),
4449
- });
4555
+ const credentialSelectionStartedAt = performance.now();
4556
+ let credentialSelectionOutcome: "completed" | "failed" = "completed";
4557
+ try {
4558
+ const sessionCodex = await getSessionCodexState(db, input.workspaceId, input.sessionId);
4559
+ const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
4560
+ const sessionPinSource = sessionCodex?.pinSource ?? null;
4561
+ const selectForTurn = (context: CodexCredentialLeaseSelectionContext) =>
4562
+ selectCodexCredentialLeaseForTurn({
4563
+ context,
4564
+ leasingEnabled: settings.codexCredentialLeasingEnabled,
4565
+ sessionId: input.sessionId,
4566
+ sessionPinnedCredentialId: sessionPin,
4567
+ sessionPinSource,
4568
+ sessionLastCredentialId: sessionCodex?.lastCredentialId ?? null,
4569
+ now: new Date(),
4570
+ });
4450
4571
 
4451
- // Rollout/rollback path is intentionally table-inert. With the flag off,
4452
- // old and new workers both use legacy pin > active-pointer selection and
4453
- // neither reads nor writes the additive lease/cursor schema.
4454
- let leased: CodexCredentialLeaseResult<RotationDecision>;
4455
- let leaseAcquisitionStartedAtMs: number | null = null;
4456
- if (settings.codexCredentialLeasingEnabled) {
4457
- leaseAcquisitionStartedAtMs = performance.now();
4458
- leased = await acquireCodexCredentialLease(
4459
- db,
4460
- {
4461
- accountId: input.accountId,
4462
- workspaceId: input.workspaceId,
4463
- turnId,
4464
- holderId: codexLeaseHolderId,
4465
- advanceActivePointer: sessionPin === null,
4466
- },
4467
- selectForTurn,
4468
- );
4469
- } else {
4470
- const [rotation, accounts] = await Promise.all([
4471
- getCodexRotationSettings(db, input.workspaceId),
4472
- listCodexAccountStatuses(db, input.workspaceId),
4473
- ]);
4474
- const leaseAccounts = accounts.map((account) => ({
4475
- ...account,
4476
- activeLeaseCount: 0,
4477
- selectionCount: 0,
4478
- lastSelectedAt: null,
4479
- }));
4480
- const activeCredentialId = rotation?.activeCredentialId ?? null;
4481
- const selected = selectForTurn({
4482
- accounts: leaseAccounts,
4483
- activeCredentialId,
4484
- rotationEnabled: rotation?.rotationEnabled ?? false,
4485
- leaseRotationEnabled: false,
4486
- rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
4487
- existingCredentialId: null,
4488
- policyScope: null,
4489
- unavailableDiagnostics: [],
4490
- });
4491
- leased = {
4492
- ...selected,
4493
- accounts: leaseAccounts,
4494
- activeCredentialId,
4495
- rotationEnabled: rotation?.rotationEnabled ?? false,
4496
- rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
4497
- reused: false,
4498
- holderId: null,
4499
- generation: null,
4500
- leasedUntil: null,
4501
- unavailableDiagnostics: [],
4502
- advanceActivePointer: selected.advanceActivePointer !== false,
4503
- };
4504
- }
4505
- if (leased.decision.kind === "allCapped") {
4506
- // Bounded self-heal of stale usage cache, then ONE new atomic selection.
4507
- await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
4508
- signalCodexCapacityWorkflow,
4509
- wakeSessionWorkflow,
4510
- });
4572
+ // Rollout/rollback path is intentionally table-inert. With the flag off,
4573
+ // old and new workers both use legacy pin > active-pointer selection and
4574
+ // neither reads nor writes the additive lease/cursor schema.
4575
+ let leased: CodexCredentialLeaseResult<RotationDecision>;
4576
+ let leaseAcquisitionStartedAtMs: number | null = null;
4511
4577
  if (settings.codexCredentialLeasingEnabled) {
4512
4578
  leaseAcquisitionStartedAtMs = performance.now();
4513
4579
  leased = await acquireCodexCredentialLease(
@@ -4532,9 +4598,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4532
4598
  selectionCount: 0,
4533
4599
  lastSelectedAt: null,
4534
4600
  }));
4601
+ const activeCredentialId = rotation?.activeCredentialId ?? null;
4535
4602
  const selected = selectForTurn({
4536
4603
  accounts: leaseAccounts,
4537
- activeCredentialId: rotation?.activeCredentialId ?? null,
4604
+ activeCredentialId,
4538
4605
  rotationEnabled: rotation?.rotationEnabled ?? false,
4539
4606
  leaseRotationEnabled: false,
4540
4607
  rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
@@ -4545,7 +4612,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4545
4612
  leased = {
4546
4613
  ...selected,
4547
4614
  accounts: leaseAccounts,
4548
- activeCredentialId: rotation?.activeCredentialId ?? null,
4615
+ activeCredentialId,
4549
4616
  rotationEnabled: rotation?.rotationEnabled ?? false,
4550
4617
  rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
4551
4618
  reused: false,
@@ -4556,212 +4623,334 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4556
4623
  advanceActivePointer: selected.advanceActivePointer !== false,
4557
4624
  };
4558
4625
  }
4559
- }
4560
- const rotationDecision = leased.decision;
4561
- const selectedPinDisposition = classifyCodexPin({
4562
- pinnedCredentialId: sessionPin,
4563
- pinSource: sessionPinSource,
4564
- strategy: leased.rotationStrategy as CodexRotationStrategy,
4565
- rotationEnabled: leased.rotationEnabled,
4566
- });
4567
- // pin policy pin persistence follows the atomic credential allocator selection. The selector
4568
- // already ran exact-turn reuse before policy filtering and vetoed pointer
4569
- // movement for manual/policy homes; this write only records the NEXT turn's
4570
- // policy home (or clears a policy pin whose strategy is no longer active).
4571
- if (
4572
- selectedPinDisposition === "sharded" &&
4573
- leased.credentialId !== null &&
4574
- (sessionPinSource !== "policy" || sessionPin !== leased.credentialId)
4575
- ) {
4576
- const pinMutation = await withSessionCodexCapacityMutation(
4577
- db,
4578
- {
4579
- workspaceId: input.workspaceId,
4580
- reason: "codex_policy_pin_changed",
4581
- },
4582
- async (tx) => {
4583
- const changed = await setSessionCodexPinInTransaction(
4584
- tx,
4585
- input.workspaceId,
4586
- input.sessionId,
4587
- leased.credentialId,
4588
- "policy",
4626
+ if (leased.decision.kind === "allCapped") {
4627
+ // Bounded self-heal of stale usage cache, then ONE new atomic selection.
4628
+ await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
4629
+ signalCodexCapacityWorkflow,
4630
+ wakeSessionWorkflow,
4631
+ });
4632
+ if (settings.codexCredentialLeasingEnabled) {
4633
+ leaseAcquisitionStartedAtMs = performance.now();
4634
+ leased = await acquireCodexCredentialLease(
4635
+ db,
4589
4636
  {
4590
- expected: {
4591
- pinnedCredentialId: sessionPin,
4592
- pinSource: sessionPinSource,
4593
- },
4637
+ accountId: input.accountId,
4638
+ workspaceId: input.workspaceId,
4639
+ turnId,
4640
+ holderId: codexLeaseHolderId,
4641
+ advanceActivePointer: sessionPin === null,
4594
4642
  },
4643
+ selectForTurn,
4595
4644
  );
4596
- return { result: changed, changed };
4597
- },
4598
- );
4599
- await signalCodexCapacityWakeTargets(
4600
- { signalCodexCapacityWorkflow, wakeSessionWorkflow },
4601
- pinMutation.wakeTargets,
4602
- );
4603
- } else if (selectedPinDisposition === "clearStale") {
4604
- const pinMutation = await withSessionCodexCapacityMutation(
4605
- db,
4606
- {
4607
- workspaceId: input.workspaceId,
4608
- reason: "codex_stale_policy_pin_cleared",
4609
- },
4610
- async (tx) => {
4611
- const changed = await setSessionCodexPinInTransaction(
4612
- tx,
4613
- input.workspaceId,
4614
- input.sessionId,
4615
- null,
4616
- "policy",
4617
- {
4618
- expected: {
4619
- pinnedCredentialId: sessionPin,
4620
- pinSource: sessionPinSource,
4645
+ } else {
4646
+ const [rotation, accounts] = await Promise.all([
4647
+ getCodexRotationSettings(db, input.workspaceId),
4648
+ listCodexAccountStatuses(db, input.workspaceId),
4649
+ ]);
4650
+ const leaseAccounts = accounts.map((account) => ({
4651
+ ...account,
4652
+ activeLeaseCount: 0,
4653
+ selectionCount: 0,
4654
+ lastSelectedAt: null,
4655
+ }));
4656
+ const selected = selectForTurn({
4657
+ accounts: leaseAccounts,
4658
+ activeCredentialId: rotation?.activeCredentialId ?? null,
4659
+ rotationEnabled: rotation?.rotationEnabled ?? false,
4660
+ leaseRotationEnabled: false,
4661
+ rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
4662
+ existingCredentialId: null,
4663
+ policyScope: null,
4664
+ unavailableDiagnostics: [],
4665
+ });
4666
+ leased = {
4667
+ ...selected,
4668
+ accounts: leaseAccounts,
4669
+ activeCredentialId: rotation?.activeCredentialId ?? null,
4670
+ rotationEnabled: rotation?.rotationEnabled ?? false,
4671
+ rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
4672
+ reused: false,
4673
+ holderId: null,
4674
+ generation: null,
4675
+ leasedUntil: null,
4676
+ unavailableDiagnostics: [],
4677
+ advanceActivePointer: selected.advanceActivePointer !== false,
4678
+ };
4679
+ }
4680
+ }
4681
+ const rotationDecision = leased.decision;
4682
+ const selectedPinDisposition = classifyCodexPin({
4683
+ pinnedCredentialId: sessionPin,
4684
+ pinSource: sessionPinSource,
4685
+ strategy: leased.rotationStrategy as CodexRotationStrategy,
4686
+ rotationEnabled: leased.rotationEnabled,
4687
+ });
4688
+ // pin policy pin persistence follows the atomic credential allocator selection. The selector
4689
+ // already ran exact-turn reuse before policy filtering and vetoed pointer
4690
+ // movement for manual/policy homes; this write only records the NEXT turn's
4691
+ // policy home (or clears a policy pin whose strategy is no longer active).
4692
+ if (
4693
+ selectedPinDisposition === "sharded" &&
4694
+ leased.credentialId !== null &&
4695
+ (sessionPinSource !== "policy" || sessionPin !== leased.credentialId)
4696
+ ) {
4697
+ const pinMutation = await withSessionCodexCapacityMutation(
4698
+ db,
4699
+ {
4700
+ workspaceId: input.workspaceId,
4701
+ reason: "codex_policy_pin_changed",
4702
+ },
4703
+ async (tx) => {
4704
+ const changed = await setSessionCodexPinInTransaction(
4705
+ tx,
4706
+ input.workspaceId,
4707
+ input.sessionId,
4708
+ leased.credentialId,
4709
+ "policy",
4710
+ {
4711
+ expected: {
4712
+ pinnedCredentialId: sessionPin,
4713
+ pinSource: sessionPinSource,
4714
+ },
4621
4715
  },
4622
- },
4623
- );
4624
- return { result: changed, changed };
4716
+ );
4717
+ return { result: changed, changed };
4718
+ },
4719
+ );
4720
+ await signalCodexCapacityWakeTargets(
4721
+ { signalCodexCapacityWorkflow, wakeSessionWorkflow },
4722
+ pinMutation.wakeTargets,
4723
+ );
4724
+ } else if (selectedPinDisposition === "clearStale") {
4725
+ const pinMutation = await withSessionCodexCapacityMutation(
4726
+ db,
4727
+ {
4728
+ workspaceId: input.workspaceId,
4729
+ reason: "codex_stale_policy_pin_cleared",
4730
+ },
4731
+ async (tx) => {
4732
+ const changed = await setSessionCodexPinInTransaction(
4733
+ tx,
4734
+ input.workspaceId,
4735
+ input.sessionId,
4736
+ null,
4737
+ "policy",
4738
+ {
4739
+ expected: {
4740
+ pinnedCredentialId: sessionPin,
4741
+ pinSource: sessionPinSource,
4742
+ },
4743
+ },
4744
+ );
4745
+ return { result: changed, changed };
4746
+ },
4747
+ );
4748
+ await signalCodexCapacityWakeTargets(
4749
+ { signalCodexCapacityWorkflow, wakeSessionWorkflow },
4750
+ pinMutation.wakeTargets,
4751
+ );
4752
+ }
4753
+ if (
4754
+ !settings.codexCredentialLeasingEnabled &&
4755
+ leased.advanceActivePointer &&
4756
+ sessionPin === null &&
4757
+ rotationDecision.kind === "active" &&
4758
+ rotationDecision.moved
4759
+ ) {
4760
+ await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
4761
+ }
4762
+ effectiveCodexCredentialId = leased.credentialId;
4763
+ codexLeaseGeneration = leased.generation;
4764
+ codexLeaseConfirmedUntilMs =
4765
+ leased.leasedUntil && leaseAcquisitionStartedAtMs !== null
4766
+ ? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS
4767
+ : null;
4768
+ codexLeaseHeld =
4769
+ effectiveCodexCredentialId !== null &&
4770
+ leased.holderId !== null &&
4771
+ leased.generation !== null &&
4772
+ codexLeaseConfirmedUntilMs !== null;
4773
+ if (codexLeaseHeld) startCodexLeaseHeartbeat();
4774
+
4775
+ const actualOutcome = effectiveCodexCredentialId
4776
+ ? "selected"
4777
+ : rotationDecision.kind === "allCapped"
4778
+ ? "waiting"
4779
+ : "none";
4780
+ const actualReason = effectiveCodexCredentialId
4781
+ ? leased.reused
4782
+ ? "lease_reused"
4783
+ : sessionPin === effectiveCodexCredentialId
4784
+ ? "pin"
4785
+ : rotationDecision.kind === "active" && rotationDecision.moved
4786
+ ? "rotation"
4787
+ : "active"
4788
+ : rotationDecision.kind === "allCapped"
4789
+ ? "all_capped"
4790
+ : "none";
4791
+ const fencedInFlight = leased.reused;
4792
+ const shadowResult = await publishCodexFleetShadowDecisionV1({
4793
+ enabled: settings.codexFleetPolicyShadowEnabled,
4794
+ decision: {
4795
+ accounts: leased.accounts,
4796
+ actualCredentialId: effectiveCodexCredentialId,
4797
+ actualOutcome,
4798
+ actualReason,
4799
+ affinityCredentialId: fencedInFlight
4800
+ ? effectiveCodexCredentialId
4801
+ : (sessionPin ?? sessionCodex?.lastCredentialId ?? null),
4802
+ fencedInFlight,
4803
+ nearExhaustionPct: settings.codexRotationNearExhaustionPct,
4804
+ now: new Date(),
4805
+ aliasSeed: randomUUID(),
4625
4806
  },
4626
- );
4627
- await signalCodexCapacityWakeTargets(
4628
- { signalCodexCapacityWorkflow, wakeSessionWorkflow },
4629
- pinMutation.wakeTargets,
4630
- );
4631
- }
4632
- if (
4633
- !settings.codexCredentialLeasingEnabled &&
4634
- leased.advanceActivePointer &&
4635
- sessionPin === null &&
4636
- rotationDecision.kind === "active" &&
4637
- rotationDecision.moved
4638
- ) {
4639
- await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
4640
- }
4641
- effectiveCodexCredentialId = leased.credentialId;
4642
- codexLeaseGeneration = leased.generation;
4643
- codexLeaseConfirmedUntilMs =
4644
- leased.leasedUntil && leaseAcquisitionStartedAtMs !== null
4645
- ? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS
4646
- : null;
4647
- codexLeaseHeld =
4648
- effectiveCodexCredentialId !== null &&
4649
- leased.holderId !== null &&
4650
- leased.generation !== null &&
4651
- codexLeaseConfirmedUntilMs !== null;
4652
- if (codexLeaseHeld) startCodexLeaseHeartbeat();
4653
-
4654
- const actualOutcome = effectiveCodexCredentialId
4655
- ? "selected"
4656
- : rotationDecision.kind === "allCapped"
4657
- ? "waiting"
4658
- : "none";
4659
- const actualReason = effectiveCodexCredentialId
4660
- ? leased.reused
4661
- ? "lease_reused"
4662
- : sessionPin === effectiveCodexCredentialId
4663
- ? "pin"
4664
- : rotationDecision.kind === "active" && rotationDecision.moved
4665
- ? "rotation"
4666
- : "active"
4667
- : rotationDecision.kind === "allCapped"
4668
- ? "all_capped"
4669
- : "none";
4670
- const fencedInFlight = leased.reused;
4671
- const shadowResult = await publishCodexFleetShadowDecisionV1({
4672
- enabled: settings.codexFleetPolicyShadowEnabled,
4673
- decision: {
4674
- accounts: leased.accounts,
4675
- actualCredentialId: effectiveCodexCredentialId,
4676
- actualOutcome,
4677
- actualReason,
4678
- affinityCredentialId: fencedInFlight
4679
- ? effectiveCodexCredentialId
4680
- : (sessionPin ?? sessionCodex?.lastCredentialId ?? null),
4681
- fencedInFlight,
4682
- nearExhaustionPct: settings.codexRotationNearExhaustionPct,
4683
- now: new Date(),
4684
- aliasSeed: randomUUID(),
4685
- },
4686
- publish,
4687
- });
4688
- if (shadowResult.outcome === "published") {
4689
- const shadowPayload = shadowResult.payload;
4690
- observability.incrementCounter({
4691
- name: "opengeni_codex_fleet_shadow_decisions_total",
4692
- help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
4693
- labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload),
4694
- });
4695
- observability.info("Codex adaptive fleet shadow decision", {
4696
- workspaceId: input.workspaceId,
4697
- policyVersion: shadowPayload.replay.policyVersion,
4698
- inputFingerprint: shadowPayload.replay.inputFingerprint,
4699
- decisionFingerprint: shadowPayload.replay.decisionFingerprint,
4700
- actualOutcome: shadowPayload.actual.outcome,
4701
- shadowOutcome: shadowPayload.replay.decision.outcome,
4702
- comparison: shadowPayload.comparison,
4703
- candidateCount: shadowPayload.replay.input.candidates.length,
4704
- truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
4705
- payloadBytes: shadowResult.payloadBytes,
4706
- });
4707
- } else if (shadowResult.outcome === "failed") {
4708
- // Shadow observability is explicitly non-authoritative. A malformed
4709
- // snapshot or event-write fault must never change the authoritative lease,
4710
- // capacity wait, failover, or the account serving this fenced turn.
4711
- observability.incrementCounter({
4712
- name: "opengeni_codex_fleet_shadow_errors_total",
4713
- help: "Shadow decision build/publication failures.",
4714
- labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
4715
- });
4716
- observability.warn("Codex adaptive fleet shadow decision failed open", {
4717
- stage: shadowResult.stage,
4718
- reason: shadowResult.reason,
4719
- errorClass: "CodexFleetShadowOperationError",
4720
- errorCode: "codex_fleet_shadow_failed",
4721
- origin: "worker",
4722
- payloadBytes: shadowResult.payloadBytes,
4807
+ publish,
4723
4808
  });
4724
- }
4809
+ if (shadowResult.outcome === "published") {
4810
+ const shadowPayload = shadowResult.payload;
4811
+ observability.incrementCounter({
4812
+ name: "opengeni_codex_fleet_shadow_decisions_total",
4813
+ help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
4814
+ labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload),
4815
+ });
4816
+ observability.info("Codex adaptive fleet shadow decision", {
4817
+ workspaceId: input.workspaceId,
4818
+ policyVersion: shadowPayload.replay.policyVersion,
4819
+ inputFingerprint: shadowPayload.replay.inputFingerprint,
4820
+ decisionFingerprint: shadowPayload.replay.decisionFingerprint,
4821
+ actualOutcome: shadowPayload.actual.outcome,
4822
+ shadowOutcome: shadowPayload.replay.decision.outcome,
4823
+ comparison: shadowPayload.comparison,
4824
+ candidateCount: shadowPayload.replay.input.candidates.length,
4825
+ truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
4826
+ payloadBytes: shadowResult.payloadBytes,
4827
+ });
4828
+ } else if (shadowResult.outcome === "failed") {
4829
+ // Shadow observability is explicitly non-authoritative. A malformed
4830
+ // snapshot or event-write fault must never change the authoritative lease,
4831
+ // capacity wait, failover, or the account serving this fenced turn.
4832
+ observability.incrementCounter({
4833
+ name: "opengeni_codex_fleet_shadow_errors_total",
4834
+ help: "Shadow decision build/publication failures.",
4835
+ labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
4836
+ });
4837
+ observability.warn("Codex adaptive fleet shadow decision failed open", {
4838
+ stage: shadowResult.stage,
4839
+ reason: shadowResult.reason,
4840
+ errorClass: "CodexFleetShadowOperationError",
4841
+ errorCode: "codex_fleet_shadow_failed",
4842
+ origin: "worker",
4843
+ payloadBytes: shadowResult.payloadBytes,
4844
+ });
4845
+ }
4725
4846
 
4726
- const eligibleCount = leased.accounts.filter((account) =>
4727
- isCodexCredentialEligible(account, new Date()),
4728
- ).length;
4729
- const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
4730
- observability.incrementCounter({
4731
- name: "opengeni_codex_pool_observations_total",
4732
- help: "Observed eligible Codex pool depth buckets at turn selection.",
4733
- labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
4734
- });
4735
- if (eligibleCount <= 1) {
4847
+ const eligibleCount = leased.accounts.filter((account) =>
4848
+ isCodexCredentialEligible(account, new Date()),
4849
+ ).length;
4850
+ const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
4736
4851
  observability.incrementCounter({
4737
- name: "opengeni_codex_pool_low_total",
4738
- help: "Alert signal emitted when the eligible Codex pool is zero or one.",
4852
+ name: "opengeni_codex_pool_observations_total",
4853
+ help: "Observed eligible Codex pool depth buckets at turn selection.",
4739
4854
  labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
4740
4855
  });
4741
- observability.warn("Codex eligible credential pool is low", {
4742
- workspaceId: input.workspaceId,
4743
- eligibleCount,
4744
- connectedCount: leased.accounts.length,
4745
- depth: poolDepth,
4746
- });
4747
- }
4856
+ if (eligibleCount <= 1) {
4857
+ observability.incrementCounter({
4858
+ name: "opengeni_codex_pool_low_total",
4859
+ help: "Alert signal emitted when the eligible Codex pool is zero or one.",
4860
+ labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
4861
+ });
4862
+ observability.warn("Codex eligible credential pool is low", {
4863
+ workspaceId: input.workspaceId,
4864
+ eligibleCount,
4865
+ connectedCount: leased.accounts.length,
4866
+ depth: poolDepth,
4867
+ });
4868
+ }
4748
4869
 
4749
- if (
4750
- effectiveCodexCredentialId === null &&
4751
- leased.accounts.length > 0 &&
4752
- leased.accounts.every((account) => !account.allocatorEnabled) &&
4753
- turnId
4754
- ) {
4755
- if (turn.source === "compaction") {
4870
+ if (
4871
+ effectiveCodexCredentialId === null &&
4872
+ leased.accounts.length > 0 &&
4873
+ leased.accounts.every((account) => !account.allocatorEnabled) &&
4874
+ turnId
4875
+ ) {
4876
+ if (turn.source === "compaction") {
4877
+ if (
4878
+ !(await settle!({
4879
+ events: [
4880
+ {
4881
+ type: "turn.cancelled",
4882
+ payload: {
4883
+ maintenance: "context_compaction",
4884
+ reason: "codex_allocator_disabled",
4885
+ requestPreserved: true,
4886
+ },
4887
+ },
4888
+ {
4889
+ type: "session.status.changed",
4890
+ payload: { status: "idle" },
4891
+ },
4892
+ ],
4893
+ turnStatus: "cancelled",
4894
+ sessionStatus: "idle",
4895
+ activeTurnId: null,
4896
+ }))
4897
+ ) {
4898
+ return claimedResult({ status: "cancelled" });
4899
+ }
4900
+ turnMetricOutcome = "cancelled";
4901
+ activityStatus = "idle";
4902
+ return claimedResult({ status: "idle", deferredUntilWake: true });
4903
+ }
4904
+ const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
4905
+ () => null,
4906
+ );
4907
+ const activeGoal = goal?.status === "active" ? goal : null;
4908
+ const armed = await armCodexCapacityWait(db, {
4909
+ accountId: input.accountId,
4910
+ workspaceId: input.workspaceId,
4911
+ sessionId: input.sessionId,
4912
+ turnId,
4913
+ attemptId: input.attemptId,
4914
+ workflowId: input.workflowId,
4915
+ goalId: activeGoal?.id ?? null,
4916
+ goalVersion: activeGoal?.version ?? null,
4917
+ earliestResetAt: null,
4918
+ resetKind: "bounded_refresh",
4919
+ failurePayload: {
4920
+ error: "All connected Codex subscriptions are disabled for new allocations.",
4921
+ code: "codex_allocator_disabled",
4922
+ detail: "waiting for a credential to be re-enabled, reconnected, or added",
4923
+ },
4924
+ });
4925
+ if (armed.action === "waiting") {
4926
+ await publishDurableSessionEvents(
4927
+ bus,
4928
+ input.workspaceId,
4929
+ input.sessionId,
4930
+ armed.events,
4931
+ );
4932
+ turnMetricOutcome = "recovering";
4933
+ activityStatus = "waiting_capacity";
4934
+ return claimedResult({
4935
+ status: "waiting_capacity",
4936
+ capacityWait: {
4937
+ waiterId: armed.waiter.id,
4938
+ generation: armed.waiter.generation,
4939
+ nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
4940
+ wakeRevision: armed.waiter.wakeRevision,
4941
+ },
4942
+ });
4943
+ }
4756
4944
  if (
4757
4945
  !(await settle!({
4758
4946
  events: [
4759
4947
  {
4760
- type: "turn.cancelled",
4948
+ type: "turn.failed",
4761
4949
  payload: {
4762
- maintenance: "context_compaction",
4763
- reason: "codex_allocator_disabled",
4764
- requestPreserved: true,
4950
+ error: "All connected Codex subscriptions are disabled for new allocations.",
4951
+ code: "codex_allocator_disabled",
4952
+ retryable: false,
4953
+ recovery: "user_message",
4765
4954
  },
4766
4955
  },
4767
4956
  {
@@ -4769,94 +4958,119 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4769
4958
  payload: { status: "idle" },
4770
4959
  },
4771
4960
  ],
4772
- turnStatus: "cancelled",
4961
+ turnStatus: "failed",
4773
4962
  sessionStatus: "idle",
4774
4963
  activeTurnId: null,
4775
4964
  }))
4776
4965
  ) {
4777
4966
  return claimedResult({ status: "cancelled" });
4778
4967
  }
4779
- turnMetricOutcome = "cancelled";
4968
+ turnMetricOutcome = "failed";
4780
4969
  activityStatus = "idle";
4781
- return claimedResult({ status: "idle", deferredUntilWake: true });
4970
+ return claimedResult({ status: "idle" });
4782
4971
  }
4783
- const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
4784
- () => null,
4785
- );
4786
- const activeGoal = goal?.status === "active" ? goal : null;
4787
- const armed = await armCodexCapacityWait(db, {
4788
- accountId: input.accountId,
4789
- workspaceId: input.workspaceId,
4790
- sessionId: input.sessionId,
4791
- turnId,
4792
- attemptId: input.attemptId,
4793
- workflowId: input.workflowId,
4794
- goalId: activeGoal?.id ?? null,
4795
- goalVersion: activeGoal?.version ?? null,
4796
- earliestResetAt: null,
4797
- resetKind: "bounded_refresh",
4798
- failurePayload: {
4799
- error: "All connected Codex subscriptions are disabled for new allocations.",
4800
- code: "codex_allocator_disabled",
4801
- detail: "waiting for a credential to be re-enabled, reconnected, or added",
4802
- },
4803
- });
4804
- if (armed.action === "waiting") {
4805
- await publishDurableSessionEvents(
4806
- bus,
4807
- input.workspaceId,
4808
- input.sessionId,
4809
- armed.events,
4972
+
4973
+ if (rotationDecision.kind === "allCapped" && turnId) {
4974
+ if (turn.source === "compaction") {
4975
+ if (
4976
+ !(await settle!({
4977
+ events: [
4978
+ {
4979
+ type: "turn.cancelled",
4980
+ payload: {
4981
+ maintenance: "context_compaction",
4982
+ reason: "codex_capacity_unavailable",
4983
+ requestPreserved: true,
4984
+ },
4985
+ },
4986
+ {
4987
+ type: "session.status.changed",
4988
+ payload: { status: "idle" },
4989
+ },
4990
+ ],
4991
+ turnStatus: "cancelled",
4992
+ sessionStatus: "idle",
4993
+ activeTurnId: null,
4994
+ }))
4995
+ ) {
4996
+ return claimedResult({ status: "cancelled" });
4997
+ }
4998
+ turnMetricOutcome = "cancelled";
4999
+ activityStatus = "idle";
5000
+ return claimedResult({ status: "idle", deferredUntilWake: true });
5001
+ }
5002
+ // Every eligible account is capped/cooling (and a usage refresh did NOT
5003
+ // surface a reset): idle the turn AT THE BOUNDARY (no wasted model/sandbox
5004
+ // build) until the EARLIEST reset across all accounts — the multi-account
5005
+ // generalization of #143's single-account idle-until-reset. No saveRunState:
5006
+ // no model ran, nothing to freeze.
5007
+ const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
5008
+ () => null,
4810
5009
  );
4811
- turnMetricOutcome = "recovering";
4812
- activityStatus = "waiting_capacity";
4813
- return claimedResult({
4814
- status: "waiting_capacity",
4815
- capacityWait: {
4816
- waiterId: armed.waiter.id,
4817
- generation: armed.waiter.generation,
4818
- nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
4819
- wakeRevision: armed.waiter.wakeRevision,
4820
- },
5010
+ const goalActive = Boolean(goal && goal.status === "active");
5011
+ // BOUNDED + POSITIVE: clamp to [MIN_IDLE_MS, max] so a null/elapsed/unknown
5012
+ // reset can never yield a 0 (which session.ts would treat as "continue now",
5013
+ // re-entering this path in a tight CPU/DB-hammering loop).
5014
+ const resumeMs = computeIdleDelayMs(
5015
+ rotationDecision.earliestResetAt,
5016
+ new Date(),
5017
+ CODEX_USAGE_LIMIT_MAX_RESUME_MS,
5018
+ );
5019
+ const failurePayload = codexUsageLimitFailurePayload(
5020
+ { resetsInSeconds: Math.ceil(resumeMs / 1000) },
5021
+ "all connected Codex subscriptions are rate-limited",
5022
+ { allAccounts: true },
5023
+ );
5024
+ const authoritativeResetAt = authoritativeCodexCapacityResetAt(
5025
+ leased.accounts,
5026
+ new Date(),
5027
+ );
5028
+ const armed = await armCodexCapacityWait(db, {
5029
+ accountId: input.accountId,
5030
+ workspaceId: input.workspaceId,
5031
+ sessionId: input.sessionId,
5032
+ turnId,
5033
+ attemptId: input.attemptId,
5034
+ workflowId: input.workflowId,
5035
+ goalId: goalActive && goal ? goal.id : null,
5036
+ goalVersion: goalActive && goal ? goal.version : null,
5037
+ earliestResetAt: authoritativeResetAt,
5038
+ resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
5039
+ failurePayload,
4821
5040
  });
4822
- }
4823
- if (
4824
- !(await settle!({
4825
- events: [
4826
- {
4827
- type: "turn.failed",
4828
- payload: {
4829
- error: "All connected Codex subscriptions are disabled for new allocations.",
4830
- code: "codex_allocator_disabled",
4831
- retryable: false,
4832
- recovery: "user_message",
4833
- },
5041
+ if (armed.action === "waiting") {
5042
+ await publishDurableSessionEvents(
5043
+ bus,
5044
+ input.workspaceId,
5045
+ input.sessionId,
5046
+ armed.events,
5047
+ );
5048
+ turnMetricOutcome = "recovering";
5049
+ activityStatus = "waiting_capacity";
5050
+ return claimedResult({
5051
+ status: "waiting_capacity",
5052
+ capacityWait: {
5053
+ waiterId: armed.waiter.id,
5054
+ generation: armed.waiter.generation,
5055
+ nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
5056
+ wakeRevision: armed.waiter.wakeRevision,
4834
5057
  },
4835
- { type: "session.status.changed", payload: { status: "idle" } },
4836
- ],
4837
- turnStatus: "failed",
4838
- sessionStatus: "idle",
4839
- activeTurnId: null,
4840
- }))
4841
- ) {
4842
- return claimedResult({ status: "cancelled" });
4843
- }
4844
- turnMetricOutcome = "failed";
4845
- activityStatus = "idle";
4846
- return claimedResult({ status: "idle" });
4847
- }
4848
-
4849
- if (rotationDecision.kind === "allCapped" && turnId) {
4850
- if (turn.source === "compaction") {
5058
+ });
5059
+ }
4851
5060
  if (
4852
5061
  !(await settle!({
4853
5062
  events: [
5063
+ // `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
5064
+ // rotation-wait state as the reactive all-capped path, so it must freeze
5065
+ // autoContinuations identically (evaluateGoalContinuation reads this marker)
5066
+ // — a goal waiting out a long reset must not burn its continuation budget on
5067
+ // the proactive path while the reactive path spares it.
4854
5068
  {
4855
- type: "turn.cancelled",
5069
+ type: "turn.failed",
4856
5070
  payload: {
4857
- maintenance: "context_compaction",
4858
- reason: "codex_capacity_unavailable",
4859
- requestPreserved: true,
5071
+ ...failurePayload,
5072
+ recovery: goalActive ? "goal_continuation" : "user_message",
5073
+ rotated: true,
4860
5074
  },
4861
5075
  },
4862
5076
  {
@@ -4864,165 +5078,90 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4864
5078
  payload: { status: "idle" },
4865
5079
  },
4866
5080
  ],
4867
- turnStatus: "cancelled",
5081
+ turnStatus: "failed",
4868
5082
  sessionStatus: "idle",
4869
5083
  activeTurnId: null,
4870
5084
  }))
4871
5085
  ) {
4872
5086
  return claimedResult({ status: "cancelled" });
4873
5087
  }
4874
- turnMetricOutcome = "cancelled";
5088
+ turnMetricOutcome = "failed";
4875
5089
  activityStatus = "idle";
4876
- return claimedResult({ status: "idle", deferredUntilWake: true });
5090
+ // idleUntilReset marks this a MANDATORY hold: session.ts must wait the full
5091
+ // resumeMs even if a future change made it 0 — never a tight re-dispatch.
5092
+ return claimedResult(
5093
+ goalActive
5094
+ ? {
5095
+ status: "idle",
5096
+ continueDelayMs: resumeMs,
5097
+ idleUntilReset: true,
5098
+ }
5099
+ : { status: "idle" },
5100
+ );
4877
5101
  }
4878
- // Every eligible account is capped/cooling (and a usage refresh did NOT
4879
- // surface a reset): idle the turn AT THE BOUNDARY (no wasted model/sandbox
4880
- // build) until the EARLIEST reset across all accounts — the multi-account
4881
- // generalization of #143's single-account idle-until-reset. No saveRunState:
4882
- // no model ran, nothing to freeze.
4883
- const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
4884
- () => null,
4885
- );
4886
- const goalActive = Boolean(goal && goal.status === "active");
4887
- // BOUNDED + POSITIVE: clamp to [MIN_IDLE_MS, max] so a null/elapsed/unknown
4888
- // reset can never yield a 0 (which session.ts would treat as "continue now",
4889
- // re-entering this path in a tight CPU/DB-hammering loop).
4890
- const resumeMs = computeIdleDelayMs(
4891
- rotationDecision.earliestResetAt,
4892
- new Date(),
4893
- CODEX_USAGE_LIMIT_MAX_RESUME_MS,
4894
- );
4895
- const failurePayload = codexUsageLimitFailurePayload(
4896
- { resetsInSeconds: Math.ceil(resumeMs / 1000) },
4897
- "all connected Codex subscriptions are rate-limited",
4898
- { allAccounts: true },
4899
- );
4900
- const authoritativeResetAt = authoritativeCodexCapacityResetAt(
4901
- leased.accounts,
4902
- new Date(),
4903
- );
4904
- const armed = await armCodexCapacityWait(db, {
4905
- accountId: input.accountId,
4906
- workspaceId: input.workspaceId,
4907
- sessionId: input.sessionId,
4908
- turnId,
4909
- attemptId: input.attemptId,
4910
- workflowId: input.workflowId,
4911
- goalId: goalActive && goal ? goal.id : null,
4912
- goalVersion: goalActive && goal ? goal.version : null,
4913
- earliestResetAt: authoritativeResetAt,
4914
- resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
4915
- failurePayload,
4916
- });
4917
- if (armed.action === "waiting") {
4918
- await publishDurableSessionEvents(
4919
- bus,
5102
+ if (effectiveCodexCredentialId) {
5103
+ const priorAccountId = sessionCodex?.lastCredentialId ?? null;
5104
+ await recordSessionActiveCodexCredential(
5105
+ db,
4920
5106
  input.workspaceId,
4921
5107
  input.sessionId,
4922
- armed.events,
5108
+ effectiveCodexCredentialId,
4923
5109
  );
4924
- turnMetricOutcome = "recovering";
4925
- activityStatus = "waiting_capacity";
4926
- return claimedResult({
4927
- status: "waiting_capacity",
4928
- capacityWait: {
4929
- waiterId: armed.waiter.id,
4930
- generation: armed.waiter.generation,
4931
- nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
4932
- wakeRevision: armed.waiter.wakeRevision,
4933
- },
4934
- });
4935
- }
4936
- if (
4937
- !(await settle!({
4938
- events: [
4939
- // `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
4940
- // rotation-wait state as the reactive all-capped path, so it must freeze
4941
- // autoContinuations identically (evaluateGoalContinuation reads this marker)
4942
- // — a goal waiting out a long reset must not burn its continuation budget on
4943
- // the proactive path while the reactive path spares it.
5110
+ if (priorAccountId !== effectiveCodexCredentialId) {
5111
+ const rotated = rotationDecision.kind === "active" && rotationDecision.moved;
5112
+ await publish([
4944
5113
  {
4945
- type: "turn.failed",
5114
+ type: "codex.account.switched",
4946
5115
  payload: {
4947
- ...failurePayload,
4948
- recovery: goalActive ? "goal_continuation" : "user_message",
4949
- rotated: true,
5116
+ fromAccountId: priorAccountId,
5117
+ toAccountId: effectiveCodexCredentialId,
5118
+ reason: rotated ? "rotation" : "manual",
4950
5119
  },
4951
5120
  },
4952
- { type: "session.status.changed", payload: { status: "idle" } },
4953
- ],
4954
- turnStatus: "failed",
4955
- sessionStatus: "idle",
4956
- activeTurnId: null,
4957
- }))
4958
- ) {
4959
- return claimedResult({ status: "cancelled" });
4960
- }
4961
- turnMetricOutcome = "failed";
4962
- activityStatus = "idle";
4963
- // idleUntilReset marks this a MANDATORY hold: session.ts must wait the full
4964
- // resumeMs even if a future change made it 0 — never a tight re-dispatch.
4965
- return claimedResult(
4966
- goalActive
4967
- ? {
4968
- status: "idle",
4969
- continueDelayMs: resumeMs,
4970
- idleUntilReset: true,
4971
- }
4972
- : { status: "idle" },
4973
- );
4974
- }
4975
- if (effectiveCodexCredentialId) {
4976
- const priorAccountId = sessionCodex?.lastCredentialId ?? null;
4977
- await recordSessionActiveCodexCredential(
4978
- db,
4979
- input.workspaceId,
4980
- input.sessionId,
4981
- effectiveCodexCredentialId,
4982
- );
4983
- if (priorAccountId !== effectiveCodexCredentialId) {
4984
- const rotated = rotationDecision.kind === "active" && rotationDecision.moved;
5121
+ ]);
5122
+ }
5123
+
5124
+ const selectionReason = leased.reused
5125
+ ? "lease_reused"
5126
+ : sessionPin === effectiveCodexCredentialId
5127
+ ? "pin"
5128
+ : rotationDecision.kind === "active" && rotationDecision.moved
5129
+ ? "rotation"
5130
+ : "active";
5131
+ observability.incrementCounter({
5132
+ name: "opengeni_codex_credential_selections_total",
5133
+ help: "Codex credential selections by strategy and reason.",
5134
+ labels: {
5135
+ workspace_key: codexWorkspaceKey,
5136
+ strategy: leased.rotationStrategy,
5137
+ reason: selectionReason,
5138
+ },
5139
+ });
4985
5140
  await publish([
4986
5141
  {
4987
- type: "codex.account.switched",
5142
+ type: "codex.credential.selected",
4988
5143
  payload: {
4989
- fromAccountId: priorAccountId,
4990
- toAccountId: effectiveCodexCredentialId,
4991
- reason: rotated ? "rotation" : "manual",
5144
+ credentialId: effectiveCodexCredentialId,
5145
+ strategy: leased.rotationStrategy,
5146
+ reason: selectionReason,
5147
+ eligibleCount,
5148
+ connectedCount: leased.accounts.length,
5149
+ reused: leased.reused,
4992
5150
  },
4993
5151
  },
4994
5152
  ]);
4995
5153
  }
4996
-
4997
- const selectionReason = leased.reused
4998
- ? "lease_reused"
4999
- : sessionPin === effectiveCodexCredentialId
5000
- ? "pin"
5001
- : rotationDecision.kind === "active" && rotationDecision.moved
5002
- ? "rotation"
5003
- : "active";
5004
- observability.incrementCounter({
5005
- name: "opengeni_codex_credential_selections_total",
5006
- help: "Codex credential selections by strategy and reason.",
5007
- labels: {
5008
- workspace_key: codexWorkspaceKey,
5009
- strategy: leased.rotationStrategy,
5010
- reason: selectionReason,
5011
- },
5154
+ } catch (error) {
5155
+ credentialSelectionOutcome = "failed";
5156
+ throw error;
5157
+ } finally {
5158
+ recordTurnStartupPhase(observability, {
5159
+ phase: "credential_selection",
5160
+ provider: "codex-subscription",
5161
+ backend: turn.sandboxBackend,
5162
+ outcome: credentialSelectionOutcome,
5163
+ durationSeconds: (performance.now() - credentialSelectionStartedAt) / 1_000,
5012
5164
  });
5013
- await publish([
5014
- {
5015
- type: "codex.credential.selected",
5016
- payload: {
5017
- credentialId: effectiveCodexCredentialId,
5018
- strategy: leased.rotationStrategy,
5019
- reason: selectionReason,
5020
- eligibleCount,
5021
- connectedCount: leased.accounts.length,
5022
- reused: leased.reused,
5023
- },
5024
- },
5025
- ]);
5026
5165
  }
5027
5166
  }
5028
5167
 
@@ -5232,6 +5371,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5232
5371
  });
5233
5372
  }
5234
5373
 
5374
+ const runtimePreparationStartedAt = performance.now();
5375
+
5235
5376
  // Pack-scoped runtime: enabled packs may declare the sandbox image this
5236
5377
  // workspace's sessions run in and skills for the sandbox skill index.
5237
5378
  // Resolved after turn.started so a composition conflict (two enabled
@@ -5369,9 +5510,23 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5369
5510
  modelCanReceiveRetainedSessionImages = supportsImageInput;
5370
5511
  const attachmentProjector = createModelHistoryAttachmentProjector(
5371
5512
  db,
5372
- input.workspaceId,
5513
+ {
5514
+ accountId: input.accountId,
5515
+ workspaceId: input.workspaceId,
5516
+ subjectId: fileAuthoritySubjectId,
5517
+ },
5373
5518
  modelInputPolicy,
5374
- objectStorage ? (file) => objectStorage.getFileBytes(file) : undefined,
5519
+ objectStorage
5520
+ ? async (file) => {
5521
+ await requireFileForSubject(db, {
5522
+ accountId: input.accountId,
5523
+ workspaceId: input.workspaceId,
5524
+ subjectId: fileAuthoritySubjectId,
5525
+ fileId: file.id,
5526
+ });
5527
+ return await objectStorage.getFileBytes(file);
5528
+ }
5529
+ : undefined,
5375
5530
  );
5376
5531
  const modelHistoryProjector = async (items: Array<Record<string, unknown>>) =>
5377
5532
  projectModelInputForCapabilities(await attachmentProjector(items), modelInputPolicy);
@@ -5469,6 +5624,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5469
5624
  // call on the same codex client) and the main run; otherwise the summarizer
5470
5625
  // would hit the codex backend unauthenticated.
5471
5626
  let codexModelRequestSequence = 0;
5627
+ let firstModelRequestPreparationStartedAt: number | null = null;
5628
+ let firstModelRequestPreparationRecorded = false;
5629
+ let firstModelRequestAuditRecorded = false;
5630
+ let firstModelRequestCheckpointAt: number | null = null;
5631
+ const firstModelRequestCheckpoints = new Set<string>();
5472
5632
  const codexContext: CodexRequestContext | null =
5473
5633
  resolvedModel?.provider.kind === "codex-subscription"
5474
5634
  ? ((): CodexRequestContext => {
@@ -5499,10 +5659,51 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5499
5659
  onUsageHeaders: (snapshot) => {
5500
5660
  latestCodexUsage = snapshot;
5501
5661
  }, // latest wins; flushed once in finally
5662
+ onRequestPreparationDiagnostic: (phase) => {
5663
+ if (
5664
+ firstModelRequestCheckpointAt === null ||
5665
+ firstModelRequestCheckpoints.has(phase)
5666
+ ) {
5667
+ return;
5668
+ }
5669
+ const now = performance.now();
5670
+ const metricPhase =
5671
+ phase === "transport_entry"
5672
+ ? "model_sdk_serialization"
5673
+ : phase === "credential_ready"
5674
+ ? "model_credential_resolution"
5675
+ : "model_wire_normalization";
5676
+ firstModelRequestCheckpoints.add(phase);
5677
+ recordTurnStartupPhase(observability, {
5678
+ phase: metricPhase,
5679
+ provider: turnExecutionPolicy.providerId,
5680
+ backend: activeSandboxBackend ?? groupBoxBackend,
5681
+ outcome: "completed",
5682
+ durationSeconds: (now - firstModelRequestCheckpointAt) / 1_000,
5683
+ count: turnTools.length,
5684
+ });
5685
+ firstModelRequestCheckpointAt = now;
5686
+ },
5502
5687
  onRequestOpaqueArtifacts: ({ fingerprints }) => {
5503
5688
  lastCodexRequestOpaqueArtifacts = fingerprints;
5504
5689
  },
5505
5690
  onModelRequestDiagnostic: (event) => {
5691
+ if (
5692
+ event.phase === "started" &&
5693
+ firstModelRequestPreparationStartedAt !== null &&
5694
+ !firstModelRequestPreparationRecorded
5695
+ ) {
5696
+ firstModelRequestPreparationRecorded = true;
5697
+ recordTurnStartupPhase(observability, {
5698
+ phase: "model_request_preparation",
5699
+ provider: turnExecutionPolicy.providerId,
5700
+ backend: activeSandboxBackend ?? groupBoxBackend,
5701
+ outcome: "completed",
5702
+ durationSeconds:
5703
+ (performance.now() - firstModelRequestPreparationStartedAt) / 1_000,
5704
+ count: turnTools.length,
5705
+ });
5706
+ }
5506
5707
  const phase =
5507
5708
  event.phase === "headers"
5508
5709
  ? "headers"
@@ -5532,19 +5733,42 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5532
5733
  if (!publish || !turnId) {
5533
5734
  throw new Error("Codex model request started before the turn event producer");
5534
5735
  }
5535
- await publish([
5536
- {
5537
- type: "agent.model.request",
5538
- payload: {
5539
- ...event,
5540
- provider: "codex-subscription",
5541
- turnId,
5542
- attemptId: input.attemptId,
5543
- dispatchId,
5544
- executionGeneration,
5736
+ const shouldRecordStartedAudit =
5737
+ event.phase === "started" && !firstModelRequestAuditRecorded;
5738
+ if (shouldRecordStartedAudit) {
5739
+ firstModelRequestAuditRecorded = true;
5740
+ }
5741
+ const auditStartedAt = shouldRecordStartedAudit ? performance.now() : null;
5742
+ let auditOutcome: "completed" | "failed" = "completed";
5743
+ try {
5744
+ await publish([
5745
+ {
5746
+ type: "agent.model.request",
5747
+ payload: {
5748
+ ...event,
5749
+ provider: "codex-subscription",
5750
+ turnId,
5751
+ attemptId: input.attemptId,
5752
+ dispatchId,
5753
+ executionGeneration,
5754
+ },
5545
5755
  },
5546
- },
5547
- ]);
5756
+ ]);
5757
+ } catch (error) {
5758
+ auditOutcome = "failed";
5759
+ throw error;
5760
+ } finally {
5761
+ if (auditStartedAt !== null) {
5762
+ recordTurnStartupPhase(observability, {
5763
+ phase: "model_request_audit",
5764
+ provider: turnExecutionPolicy.providerId,
5765
+ backend: activeSandboxBackend ?? groupBoxBackend,
5766
+ outcome: auditOutcome,
5767
+ durationSeconds: (performance.now() - auditStartedAt) / 1_000,
5768
+ count: turnTools.length,
5769
+ });
5770
+ }
5771
+ }
5548
5772
  },
5549
5773
  };
5550
5774
  })()
@@ -5903,7 +6127,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5903
6127
  availableMcpServerIds: runSettings.mcpServers.map((server) => server.id),
5904
6128
  defaultMcpServerIds: defaultSessionMcpServerIds(capabilitySettings.mcpServers),
5905
6129
  });
5906
- const mcpAvailabilityInstructions = unavailableMcpTurnInstructions({
6130
+ const mcpAvailabilityNote = unavailableMcpOperationalContext({
5907
6131
  droppedIds: resolvedToolPolicy.effectivePolicy.droppedIds,
5908
6132
  droppedCount: resolvedToolPolicy.effectivePolicy.counts.dropped,
5909
6133
  });
@@ -6085,17 +6309,27 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6085
6309
  cancellationSignal,
6086
6310
  undefined,
6087
6311
  );
6088
- const establishPolicy: "eager" | "on-demand" =
6089
- lazyProvisionEnabled(settings) &&
6090
- !machinePrimary &&
6091
- runSettings.sandboxBackend !== "none" &&
6312
+ const establishDecision = sandboxEstablishPolicyDecision({
6313
+ lazyEnabled: lazyProvisionEnabled(settings),
6314
+ machinePrimary,
6315
+ sandboxBackend: runSettings.sandboxBackend,
6092
6316
  // Resolved run credentials must be written to one exact leased sandbox
6093
6317
  // before agent execution. A warm active pointer can bypass the lazy
6094
6318
  // provisioner entirely, which previously skipped materialization.
6095
- !initialRunCredentialMaterial &&
6096
- requiredGeneratedVideoFiles.length === 0
6097
- ? "on-demand"
6098
- : "eager";
6319
+ hasInitialRunCredentialMaterial: initialRunCredentialMaterial !== null,
6320
+ generatedVideoFileCount: requiredGeneratedVideoFiles.length,
6321
+ hasSignedFileResources:
6322
+ requiresSignedFileResourceDownloads(
6323
+ runSettings,
6324
+ activeSandboxBackend ?? groupBoxBackend,
6325
+ ) && turnResources.some((resource) => resource.kind === "file"),
6326
+ });
6327
+ const establishPolicy = establishDecision.policy;
6328
+ recordTurnSandboxEstablishPolicy(observability, {
6329
+ policy: establishDecision.policy,
6330
+ reason: establishDecision.reason,
6331
+ backend: machinePrimary ? "selfhosted" : groupBoxBackend,
6332
+ });
6099
6333
  // Computed exactly ONCE per turn and reused for BOTH the box manifest
6100
6334
  // (resumeBoxForTurn -> establishSandboxSessionFromEnvelope, below) AND the
6101
6335
  // agent (runtime.buildAgent, below). sandboxEnvironmentForRun mints a FRESH
@@ -6118,6 +6352,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6118
6352
  if (activeSandboxBackend !== "selfhosted") {
6119
6353
  await assertGitHubResourcesRemainAuthorized(db, input.workspaceId, turnResources);
6120
6354
  }
6355
+ await assertFileResourcesRemainAuthorized(
6356
+ db,
6357
+ input.accountId,
6358
+ input.workspaceId,
6359
+ fileAuthoritySubjectId,
6360
+ turnResources,
6361
+ );
6121
6362
  const authorizeGitHubTokenMint: GitHubTokenMintAuthorization = async (selection) => {
6122
6363
  await assertGitHubTokenMintSelectionAuthorized(
6123
6364
  db,
@@ -6568,6 +6809,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6568
6809
  // applies the agent's manifest to this provided session and throws on ANY
6569
6810
  // variableSet delta (validateNoEnvironmentDelta). Passing sandboxEnvironment
6570
6811
  // here makes current==target so the delta is empty.
6812
+ recordTurnStartupPhase(observability, {
6813
+ phase: "runtime_preparation",
6814
+ provider: turnExecutionPolicy.providerId,
6815
+ backend: activeSandboxBackend ?? groupBoxBackend,
6816
+ outcome: "completed",
6817
+ durationSeconds: (performance.now() - runtimePreparationStartedAt) / 1_000,
6818
+ });
6571
6819
  if (
6572
6820
  shouldEstablishSandboxForTurn(
6573
6821
  settings.sandboxOwnershipEnabled,
@@ -6575,242 +6823,277 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6575
6823
  machinePrimary,
6576
6824
  )
6577
6825
  ) {
6578
- const managedOwnership = managedSandboxOwnershipForTurn(
6579
- machinePrimary,
6580
- input.attemptId,
6581
- session.sandboxGroupId,
6582
- );
6583
- sandboxHolderId = managedOwnership?.holderId ?? null;
6584
- sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
6585
- // STAGE D honest-label guard: a machine-home session carries
6586
- // turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
6587
- // when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
6588
- // + enrollmentId). When it is NOT primary — the agent swapped back to the
6589
- // group box (sandbox_swap 'session'/'default'/groupId clears the pointer) or
6590
- // selfhosted routing is flag-OFF (the pointer is ignored) — the else-branch
6591
- // must resume a REAL cloud group box, not a "selfhosted" one: the registry
6592
- // SelfhostedSandboxClient has no bound agentId and throws. Fall the group-box
6593
- // backend back to the deployment default cloud backend so swap-away / flag-off
6594
- // degrade to a genuine cloud box exactly like today (home=modal did).
6595
- if (machinePrimary) {
6596
- // STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
6597
- // establish OR lease the managed home box. The active-sandbox pointer is
6598
- // the machine route's own epoch fence; the managed-home lease separately
6599
- // owns cloud recovery, snapshots, billing, and reaping. Mixing those two
6600
- // ownership domains lets an unrelated degraded cloud archive block a
6601
- // healthy explicitly selected machine.
6602
- // Whether the machine's latest Hello advertised the op-stream engine
6603
- // (refreshed on every connect). Read only when the server flag is on —
6604
- // one indexed lookup, and the flag off keeps this path byte-identical.
6605
- const machineOpStream =
6606
- settings.agentOpStreamEnabled === true
6607
- ? (await getEnrollment(db, input.workspaceId, activeSandboxRecord!.enrollmentId!))
6608
- ?.opStream === true
6609
- : false;
6610
- const established = await establishSelfhostedTurnSession(
6611
- {
6612
- db,
6613
- settings,
6614
- bus,
6615
- onOp: machineOpObserver.observer,
6616
- onSandboxOperation: sandboxOperationObserver,
6617
- opJournal,
6618
- },
6619
- {
6620
- workspaceId: input.workspaceId,
6621
- agentId: activeSandboxRecord!.enrollmentId!,
6622
- opStream: machineOpStream,
6623
- epoch: activeSandboxPointer!.activeEpoch,
6624
- environment: sandboxEnvironment,
6625
- ...(transientCodemodeEnvironment
6626
- ? { transientExecEnvironment: transientCodemodeEnvironment }
6627
- : {}),
6628
- workingDir: activeSandboxPointer!.workingDir,
6629
- },
6826
+ const sandboxEstablishStartedAt = performance.now();
6827
+ let sandboxEstablishOutcome: "completed" | "failed" = "completed";
6828
+ try {
6829
+ const managedOwnership = managedSandboxOwnershipForTurn(
6830
+ machinePrimary,
6831
+ input.attemptId,
6832
+ session.sandboxGroupId,
6630
6833
  );
6631
- // The machine-primary establish narrows `session` to SelfhostedSession
6632
- // (buildSelfhostedBackendSession); EstablishedSandboxSession widens it.
6633
- machinePrimarySession =
6634
- established.session as import("@opengeni/runtime").SelfhostedSession;
6635
- setupBoxSession = established.session;
6636
- resolvedSandbox = {
6637
- // Wrap in the SAME routing proxy so a mid-turn swap (to another machine
6638
- // or back to the group box) still re-routes per op. PIN this established
6639
- // SelfhostedSession for the machine pointer so the turn-start manifest
6640
- // write (via the proxy's `state` getter) and the per-op reads hit ONE
6641
- // instance — no two-instance manifest divergence.
6642
- established: wrapTurnBoxWithRouting(
6834
+ sandboxHolderId = managedOwnership?.holderId ?? null;
6835
+ sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
6836
+ // STAGE D honest-label guard: a machine-home session carries
6837
+ // turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
6838
+ // when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
6839
+ // + enrollmentId). When it is NOT primary — the agent swapped back to the
6840
+ // group box (sandbox_swap 'session'/'default'/groupId clears the pointer) or
6841
+ // selfhosted routing is flag-OFF (the pointer is ignored) — the else-branch
6842
+ // must resume a REAL cloud group box, not a "selfhosted" one: the registry
6843
+ // SelfhostedSandboxClient has no bound agentId and throws. Fall the group-box
6844
+ // backend back to the deployment default cloud backend so swap-away / flag-off
6845
+ // degrade to a genuine cloud box exactly like today (home=modal did).
6846
+ if (machinePrimary) {
6847
+ // STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
6848
+ // establish OR lease the managed home box. The active-sandbox pointer is
6849
+ // the machine route's own epoch fence; the managed-home lease separately
6850
+ // owns cloud recovery, snapshots, billing, and reaping. Mixing those two
6851
+ // ownership domains lets an unrelated degraded cloud archive block a
6852
+ // healthy explicitly selected machine.
6853
+ // Whether the machine's latest Hello advertised the op-stream engine
6854
+ // (refreshed on every connect). Read only when the server flag is on —
6855
+ // one indexed lookup, and the flag off keeps this path byte-identical.
6856
+ const machineOpStream =
6857
+ settings.agentOpStreamEnabled === true
6858
+ ? (await getEnrollment(db, input.workspaceId, activeSandboxRecord!.enrollmentId!))
6859
+ ?.opStream === true
6860
+ : false;
6861
+ const established = await establishSelfhostedTurnSession(
6643
6862
  {
6644
6863
  db,
6645
6864
  settings,
6646
6865
  bus,
6647
- opJournal,
6648
6866
  onOp: machineOpObserver.observer,
6649
6867
  onSandboxOperation: sandboxOperationObserver,
6650
- onHomeSandboxRebound,
6651
- ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
6868
+ opJournal,
6652
6869
  },
6653
6870
  {
6654
6871
  workspaceId: input.workspaceId,
6655
- sessionId: input.sessionId,
6872
+ agentId: activeSandboxRecord!.enrollmentId!,
6873
+ opStream: machineOpStream,
6874
+ epoch: activeSandboxPointer!.activeEpoch,
6656
6875
  environment: sandboxEnvironment,
6657
6876
  ...(transientCodemodeEnvironment
6658
6877
  ? { transientExecEnvironment: transientCodemodeEnvironment }
6659
6878
  : {}),
6660
- workspaceMutationFence: {
6661
- accountId: input.accountId,
6662
- turnId: turn.id,
6663
- executionGeneration,
6664
- attemptId: input.attemptId,
6879
+ workingDir: activeSandboxPointer!.workingDir,
6880
+ },
6881
+ );
6882
+ // The machine-primary establish narrows `session` to SelfhostedSession
6883
+ // (buildSelfhostedBackendSession); EstablishedSandboxSession widens it.
6884
+ machinePrimarySession =
6885
+ established.session as import("@opengeni/runtime").SelfhostedSession;
6886
+ setupBoxSession = established.session;
6887
+ resolvedSandbox = {
6888
+ // Wrap in the SAME routing proxy so a mid-turn swap (to another machine
6889
+ // or back to the group box) still re-routes per op. PIN this established
6890
+ // SelfhostedSession for the machine pointer so the turn-start manifest
6891
+ // write (via the proxy's `state` getter) and the per-op reads hit ONE
6892
+ // instance — no two-instance manifest divergence.
6893
+ established: wrapTurnBoxWithRouting(
6894
+ {
6895
+ db,
6896
+ settings,
6897
+ bus,
6898
+ opJournal,
6899
+ onOp: machineOpObserver.observer,
6900
+ onSandboxOperation: sandboxOperationObserver,
6901
+ onHomeSandboxRebound,
6902
+ ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
6665
6903
  },
6666
- pinnedSelfhosted: {
6667
- sandboxId: activeSandboxPointer!.activeSandboxId!,
6668
- epoch: activeSandboxPointer!.activeEpoch,
6904
+ {
6905
+ workspaceId: input.workspaceId,
6906
+ sessionId: input.sessionId,
6907
+ environment: sandboxEnvironment,
6908
+ ...(transientCodemodeEnvironment
6909
+ ? { transientExecEnvironment: transientCodemodeEnvironment }
6910
+ : {}),
6911
+ workspaceMutationFence: {
6912
+ accountId: input.accountId,
6913
+ turnId: turn.id,
6914
+ executionGeneration,
6915
+ attemptId: input.attemptId,
6916
+ },
6917
+ pinnedSelfhosted: {
6918
+ sandboxId: activeSandboxPointer!.activeSandboxId!,
6919
+ epoch: activeSandboxPointer!.activeEpoch,
6920
+ },
6921
+ // HOME semantics for a mid-turn clear-to-null: only a genuine
6922
+ // machine-HOME session (its home IS this machine, session.sandboxBackend
6923
+ // === "selfhosted") resolves null back to the pinned machine. A Modal-HOME
6924
+ // session merely PINNED to a machine this turn never established its group
6925
+ // box, so defaultIsHome:false makes a clear-to-null fail typed
6926
+ // (`home_unavailable_this_turn`) rather than silently serving the machine;
6927
+ // the detach takes effect next turn. (Lazy home-box establishment on such a
6928
+ // clear is a deferred follow-up; issue #341.)
6929
+ defaultIsHome: session.sandboxBackend === "selfhosted",
6669
6930
  },
6670
- // HOME semantics for a mid-turn clear-to-null: only a genuine
6671
- // machine-HOME session (its home IS this machine, session.sandboxBackend
6672
- // === "selfhosted") resolves null back to the pinned machine. A Modal-HOME
6673
- // session merely PINNED to a machine this turn never established its group
6674
- // box, so defaultIsHome:false makes a clear-to-null fail typed
6675
- // (`home_unavailable_this_turn`) rather than silently serving the machine;
6676
- // the detach takes effect next turn. (Lazy home-box establishment on such a
6677
- // clear is a deferred follow-up; issue #341.)
6678
- defaultIsHome: session.sandboxBackend === "selfhosted",
6679
- },
6680
- established,
6681
- ),
6682
- // For a machine route this is the active-pointer epoch, not a cloud
6683
- // lease epoch. Cloud-only heartbeat/snapshot/capture paths require
6684
- // sandboxGroupId and remain disabled for this branch.
6685
- leaseEpoch: activeSandboxPointer!.activeEpoch,
6686
- release: async () => undefined,
6687
- };
6688
- } else if (establishPolicy === "on-demand") {
6689
- // Lazy sandbox provisioning: holder/group ids are fixed at turn start,
6690
- // but the lease acquire + box establish + setup move behind the routing
6691
- // proxy's first default-pointer op. A chat-only turn never calls it, so
6692
- // no lease row, no provider box, no warm-meter interval.
6693
- } else {
6694
- resolvedSandbox = await waitForTurnOperation(
6695
- resumeBoxForTurn(
6696
- {
6697
- db,
6698
- settings: runSettings,
6699
- logicalFallbackSettings: logicalSandboxSettings,
6700
- cancellationSignal: sandboxResumeSignal,
6701
- sandboxMetrics: runtimeMetricsHooksForObservability(observability),
6702
- onSandboxLost: publishSandboxLost,
6703
- },
6704
- {
6705
- accountId: input.accountId,
6706
- workspaceId: input.workspaceId,
6707
- sandboxGroupId: session.sandboxGroupId,
6708
- sessionId: input.sessionId,
6709
- // groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
6710
- // is not machine-primary resumes a real cloud group box (the
6711
- // deployment default), never a "selfhosted" box (which would throw
6712
- // for lack of a bound agentId).
6713
- backend: groupBoxBackend,
6714
- os: session.sandboxOs,
6715
- environment: sandboxEnvironment,
6716
- // IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
6717
- // this run resolves. The lease stamps it + conflicts on a live shared box
6718
- // running a DIFFERENT image (solo → durable rotation; N-holders →
6719
- // SandboxImageConflictError surfaced as an actionable turn error). Prefer
6720
- // the explicit Modal image ref, else the docker image. The selfhosted branch
6721
- // (establishSelfhostedTurnSession) NEVER passes
6722
- // an image — B3 lives only on this Modal else-branch.
6723
- ...((runSettings.modalImageRef ?? runSettings.dockerImage)
6724
- ? {
6725
- image: runSettings.modalImageRef ?? runSettings.dockerImage,
6726
- }
6727
- : {}),
6728
- // RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
6729
- // conflicts on a live shared box set up under a different rig (solo
6730
- // durable rotation / N-holders SandboxRigConflictError). Omitted for a
6731
- // rig-less turn -> never stamped or enforced (shares exactly as today).
6732
- ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
6733
- },
6734
- "turn",
6735
- managedOwnership!.holderId,
6736
- ),
6737
- sandboxResumeSignal,
6738
- releaseLateSandbox,
6739
- );
6740
- setupBoxSession = resolvedSandbox.established.session;
6741
- // Durable box-lifecycle events (sandbox-file-persistence observability):
6742
- // record every box transition in session_events so the NEXT box loss is
6743
- // attributable from the DB alone — worker logs rotate within hours, which
6744
- // left both 2026-07-06 incidents without a durable trace. Best-effort.
6745
- await publishSandboxLifecycleEvents(resolvedSandbox);
6746
- // M7 hot-swap: when the selfhosted feature is on, wrap the established
6747
- // group box in the STABLE routing proxy before it is injected NON-OWNED
6748
- // into the run. The SDK binds to this ONE object once and calls its
6749
- // methods per tool call; the proxy re-reads (active_sandbox_id,
6750
- // active_epoch) per op and dispatches to the currently-active backend, so
6751
- // a sandbox_swap mid-turn lands the NEXT tool call on the new box. With
6752
- // the flag off the established group box is injected unchanged (today's
6753
- // path). The lease still owns the group box lifecycle — the proxy is a
6754
- // routing veneer, not an owner.
6755
- resolvedSandbox = {
6756
- ...resolvedSandbox,
6757
- established: wrapTurnBoxWithRouting(
6758
- {
6759
- db,
6760
- settings,
6761
- bus,
6762
- opJournal,
6763
- onSandboxOperation: sandboxOperationObserver,
6764
- onHomeSandboxLost: publishSandboxLost,
6765
- onHomeSandboxRebound,
6766
- ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
6767
- },
6768
- // Thread the SAME declared environment the group box was created with
6769
- // (resumeBoxForTurn, above) so a selfhosted swap target's manifest
6770
- // carries it too — the SDK's per-turn manifest-env delta stays empty
6771
- // (no "cannot change manifest environment variables" throw).
6772
- {
6773
- workspaceId: input.workspaceId,
6774
- sessionId: input.sessionId,
6775
- environment: sandboxEnvironment,
6776
- ...(transientCodemodeEnvironment
6777
- ? { transientExecEnvironment: transientCodemodeEnvironment }
6778
- : {}),
6779
- workspaceMutationFence: {
6780
- accountId: input.accountId,
6781
- turnId: turn.id,
6782
- executionGeneration,
6783
- attemptId: input.attemptId,
6931
+ established,
6932
+ ),
6933
+ // For a machine route this is the active-pointer epoch, not a cloud
6934
+ // lease epoch. Cloud-only heartbeat/snapshot/capture paths require
6935
+ // sandboxGroupId and remain disabled for this branch.
6936
+ leaseEpoch: activeSandboxPointer!.activeEpoch,
6937
+ release: async () => undefined,
6938
+ };
6939
+ } else if (establishPolicy === "on-demand") {
6940
+ // Lazy sandbox provisioning: holder/group ids are fixed at turn start,
6941
+ // but the lease acquire + box establish + setup move behind the routing
6942
+ // proxy's first default-pointer op. A chat-only turn never calls it, so
6943
+ // no lease row, no provider box, no warm-meter interval.
6944
+ } else {
6945
+ resolvedSandbox = await waitForTurnOperation(
6946
+ resumeBoxForTurn(
6947
+ {
6948
+ db,
6949
+ settings: runSettings,
6950
+ logicalFallbackSettings: logicalSandboxSettings,
6951
+ cancellationSignal: sandboxResumeSignal,
6952
+ sandboxMetrics: runtimeMetricsHooksForObservability(observability),
6953
+ onSandboxLost: publishSandboxLost,
6784
6954
  },
6785
- homeLease: {
6955
+ {
6786
6956
  accountId: input.accountId,
6957
+ workspaceId: input.workspaceId,
6787
6958
  sandboxGroupId: session.sandboxGroupId,
6788
- leaseEpoch: resolvedSandbox.leaseEpoch,
6789
- instanceId: resolvedSandbox.established.instanceId,
6959
+ sessionId: input.sessionId,
6960
+ // groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
6961
+ // is not machine-primary resumes a real cloud group box (the
6962
+ // deployment default), never a "selfhosted" box (which would throw
6963
+ // for lack of a bound agentId).
6790
6964
  backend: groupBoxBackend,
6965
+ os: session.sandboxOs,
6966
+ environment: sandboxEnvironment,
6967
+ // IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
6968
+ // this run resolves. The lease stamps it + conflicts on a live shared box
6969
+ // running a DIFFERENT image (solo → durable rotation; N-holders →
6970
+ // SandboxImageConflictError surfaced as an actionable turn error). Prefer
6971
+ // the explicit Modal image ref, else the docker image. The selfhosted branch
6972
+ // (establishSelfhostedTurnSession) NEVER passes
6973
+ // an image — B3 lives only on this Modal else-branch.
6974
+ ...((runSettings.modalImageRef ?? runSettings.dockerImage)
6975
+ ? {
6976
+ image: runSettings.modalImageRef ?? runSettings.dockerImage,
6977
+ }
6978
+ : {}),
6979
+ // RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
6980
+ // conflicts on a live shared box set up under a different rig (solo
6981
+ // durable rotation / N-holders SandboxRigConflictError). Omitted for a
6982
+ // rig-less turn -> never stamped or enforced (shares exactly as today).
6983
+ ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
6791
6984
  },
6792
- },
6793
- resolvedSandbox.established,
6794
- ),
6795
- };
6796
- }
6797
- if (resolvedSandbox) {
6798
- startLeaseHeartbeat(resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
6985
+ "turn",
6986
+ managedOwnership!.holderId,
6987
+ ),
6988
+ sandboxResumeSignal,
6989
+ releaseLateSandbox,
6990
+ );
6991
+ setupBoxSession = resolvedSandbox.established.session;
6992
+ // Durable box-lifecycle events (sandbox-file-persistence observability):
6993
+ // record every box transition in session_events so the NEXT box loss is
6994
+ // attributable from the DB alone — worker logs rotate within hours, which
6995
+ // left both 2026-07-06 incidents without a durable trace. Best-effort.
6996
+ await publishSandboxLifecycleEvents(resolvedSandbox);
6997
+ // M7 hot-swap: when the selfhosted feature is on, wrap the established
6998
+ // group box in the STABLE routing proxy before it is injected NON-OWNED
6999
+ // into the run. The SDK binds to this ONE object once and calls its
7000
+ // methods per tool call; the proxy re-reads (active_sandbox_id,
7001
+ // active_epoch) per op and dispatches to the currently-active backend, so
7002
+ // a sandbox_swap mid-turn lands the NEXT tool call on the new box. With
7003
+ // the flag off the established group box is injected unchanged (today's
7004
+ // path). The lease still owns the group box lifecycle — the proxy is a
7005
+ // routing veneer, not an owner.
7006
+ resolvedSandbox = {
7007
+ ...resolvedSandbox,
7008
+ established: wrapTurnBoxWithRouting(
7009
+ {
7010
+ db,
7011
+ settings,
7012
+ bus,
7013
+ opJournal,
7014
+ onSandboxOperation: sandboxOperationObserver,
7015
+ onHomeSandboxLost: publishSandboxLost,
7016
+ onHomeSandboxRebound,
7017
+ ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
7018
+ },
7019
+ // Thread the SAME declared environment the group box was created with
7020
+ // (resumeBoxForTurn, above) so a selfhosted swap target's manifest
7021
+ // carries it too — the SDK's per-turn manifest-env delta stays empty
7022
+ // (no "cannot change manifest environment variables" throw).
7023
+ {
7024
+ workspaceId: input.workspaceId,
7025
+ sessionId: input.sessionId,
7026
+ environment: sandboxEnvironment,
7027
+ ...(transientCodemodeEnvironment
7028
+ ? { transientExecEnvironment: transientCodemodeEnvironment }
7029
+ : {}),
7030
+ workspaceMutationFence: {
7031
+ accountId: input.accountId,
7032
+ turnId: turn.id,
7033
+ executionGeneration,
7034
+ attemptId: input.attemptId,
7035
+ },
7036
+ homeLease: {
7037
+ accountId: input.accountId,
7038
+ sandboxGroupId: session.sandboxGroupId,
7039
+ leaseEpoch: resolvedSandbox.leaseEpoch,
7040
+ instanceId: resolvedSandbox.established.instanceId,
7041
+ backend: groupBoxBackend,
7042
+ },
7043
+ },
7044
+ resolvedSandbox.established,
7045
+ ),
7046
+ };
7047
+ }
7048
+ if (resolvedSandbox) {
7049
+ startLeaseHeartbeat(resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
7050
+ }
7051
+ } catch (error) {
7052
+ sandboxEstablishOutcome = "failed";
7053
+ throw error;
7054
+ } finally {
7055
+ recordTurnStartupPhase(observability, {
7056
+ phase: "sandbox_establish",
7057
+ provider: turnExecutionPolicy.providerId,
7058
+ backend: machinePrimary ? "selfhosted" : groupBoxBackend,
7059
+ outcome: sandboxEstablishOutcome,
7060
+ durationSeconds: (performance.now() - sandboxEstablishStartedAt) / 1_000,
7061
+ });
6799
7062
  }
6800
7063
  }
6801
7064
 
6802
- const ordinaryFileResourceDownloads = await waitForTurnOperation(
6803
- sandboxFileDownloadsForRun(
6804
- runSettings,
6805
- db,
6806
- objectStorage,
6807
- input.workspaceId,
6808
- turnResources,
6809
- activeSandboxBackend ?? groupBoxBackend,
6810
- ),
6811
- cancellationSignal,
6812
- undefined,
6813
- );
7065
+ const ordinaryFileResourceDownloads = await (async () => {
7066
+ const fileResolutionStartedAt = performance.now();
7067
+ let fileResolutionOutcome: "completed" | "failed" = "completed";
7068
+ try {
7069
+ return await waitForTurnOperation(
7070
+ sandboxFileDownloadsForRun(
7071
+ runSettings,
7072
+ db,
7073
+ objectStorage,
7074
+ input.accountId,
7075
+ input.workspaceId,
7076
+ fileAuthoritySubjectId,
7077
+ turnResources,
7078
+ activeSandboxBackend ?? groupBoxBackend,
7079
+ ),
7080
+ cancellationSignal,
7081
+ undefined,
7082
+ );
7083
+ } catch (error) {
7084
+ fileResolutionOutcome = "failed";
7085
+ throw error;
7086
+ } finally {
7087
+ recordTurnStartupPhase(observability, {
7088
+ phase: "file_resolution",
7089
+ provider: turnExecutionPolicy.providerId,
7090
+ backend: activeSandboxBackend ?? groupBoxBackend,
7091
+ outcome: fileResolutionOutcome,
7092
+ durationSeconds: (performance.now() - fileResolutionStartedAt) / 1_000,
7093
+ count: turnResources.length,
7094
+ });
7095
+ }
7096
+ })();
6814
7097
  if (requiredGeneratedVideoFiles.length > 0 && !objectStorage) {
6815
7098
  throw new Error("Generated video materialization requires object storage");
6816
7099
  }
@@ -6840,6 +7123,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6840
7123
  ]),
6841
7124
  ).values(),
6842
7125
  ];
7126
+ const toolContextPreparationStartedAt = performance.now();
6843
7127
  throwIfWorkerShuttingDown();
6844
7128
  throwIfTurnCancelled();
6845
7129
  const mcpCredentialRootSessionId =
@@ -6880,6 +7164,37 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6880
7164
  }
6881
7165
  return result;
6882
7166
  };
7167
+ const connectorActionIdentity = {
7168
+ accountId: input.accountId,
7169
+ workspaceId: input.workspaceId,
7170
+ sessionId: input.sessionId,
7171
+ turnId: turn.id,
7172
+ attemptId: input.attemptId,
7173
+ executionGeneration,
7174
+ initiator: {
7175
+ kind: turn.initiator.kind,
7176
+ subjectId: turn.initiator.subjectId,
7177
+ },
7178
+ } as const;
7179
+ const googleDrivePublicationTarget = objectStorage
7180
+ ? await resolveGoogleDrivePublicationTarget(
7181
+ db,
7182
+ input.workspaceId,
7183
+ personalConnectionDelegations,
7184
+ )
7185
+ : null;
7186
+ const googleDrivePublicationTool =
7187
+ objectStorage && googleDrivePublicationTarget
7188
+ ? createGoogleDrivePublicationAttemptTool({
7189
+ db,
7190
+ objectStorage,
7191
+ identity: connectorActionIdentity,
7192
+ subjectId: googleDrivePublicationTarget.ownerSubjectId,
7193
+ target: googleDrivePublicationTarget,
7194
+ resolveCredential,
7195
+ ...(runtimeCancellationSignal ? { signal: runtimeCancellationSignal } : {}),
7196
+ })
7197
+ : null;
6883
7198
  const publishToolAuthNeeded = async (payload: ToolAuthNeededPayload): Promise<void> => {
6884
7199
  if (!shouldPublishToolAuthNeededForTurn(payload, trigger, turn)) {
6885
7200
  return;
@@ -6938,59 +7253,140 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6938
7253
  runSettings,
6939
7254
  session.firstPartyMcpTools,
6940
7255
  );
6941
- preparedTools = await waitForTurnOperation(
6942
- runtime.prepareTools(runSettings, turnTools, {
6943
- accountId: input.accountId,
6944
- workspaceId: input.workspaceId,
6945
- sessionId: input.sessionId,
6946
- // Sign the calling turn into the first-party token so tools classify
6947
- // the caller by its own identity (sacred-pause guard), not the racy
6948
- // live active pointer.
6949
- ...(turnId ? { turnId } : {}),
6950
- attemptId: input.attemptId,
6951
- executionGeneration,
6952
- subjectId: "worker:first-party-mcp",
6953
- subjectLabel: "OpenGeni worker",
6954
- ...(credentialSubjectId ? { credentialSubjectId } : {}),
6955
- ...(codexAppsAuth ? { codexAppsAuth } : {}),
6956
- resolveCredential,
6957
- onAuthNeeded: publishToolAuthNeeded,
6958
- localMcpServers,
6959
- onAttemptToolCatalog: async (catalog) => {
6960
- await persistAttemptToolCatalog(db, catalog);
7256
+ const googleDrivePublicationAllowed =
7257
+ selectedFirstPartyMcpTools.includes("editable_artifact_export") &&
7258
+ selectedFirstPartyMcpTools.includes("editable_artifact_export_status") &&
7259
+ (!session.firstPartyMcpPermissions?.length ||
7260
+ (session.firstPartyMcpPermissions.includes("artifacts:read") &&
7261
+ session.firstPartyMcpPermissions.includes("artifacts:publish")));
7262
+ const googleDriveConnectorBindings: readonly AttemptConnectorActionBinding[] =
7263
+ googleDrivePublicationTool && googleDrivePublicationTarget && googleDrivePublicationAllowed
7264
+ ? [
7265
+ {
7266
+ modelName: googleDrivePublicationTool.modelName,
7267
+ call: (approvalId, arguments_) =>
7268
+ googleDrivePublicationConnectorCall(
7269
+ googleDrivePublicationTarget,
7270
+ arguments_,
7271
+ approvalId,
7272
+ ),
7273
+ },
7274
+ ]
7275
+ : [];
7276
+ const attemptToolDefinitions = [
7277
+ ...createFirstPartyInteractionAttemptToolDefinitions({
7278
+ settings: runSettings,
7279
+ scope: {
7280
+ accountId: input.accountId,
7281
+ workspaceId: input.workspaceId,
7282
+ sessionId: input.sessionId,
7283
+ turnId: turn.id,
7284
+ attemptId: input.attemptId,
7285
+ executionGeneration,
6961
7286
  },
6962
- // Manager-style sessions carry a creation-validated permission set
6963
- // for their first-party MCP token; null keeps the fixed default.
6964
7287
  ...(session.firstPartyMcpPermissions?.length
6965
- ? { firstPartyPermissions: session.firstPartyMcpPermissions }
7288
+ ? { permissions: session.firstPartyMcpPermissions }
6966
7289
  : {}),
6967
- firstPartyTools: selectedFirstPartyMcpTools,
6968
- nestedAgentDepth: session.nestedAgentDepth,
6969
- effectiveMaxNestedAgentDepth: session.effectiveMaxNestedAgentDepth,
6970
- attemptToolDefinitions: createFirstPartyInteractionAttemptToolDefinitions({
6971
- settings: runSettings,
6972
- scope: {
6973
- accountId: input.accountId,
6974
- workspaceId: input.workspaceId,
6975
- sessionId: input.sessionId,
6976
- turnId: turn.id,
6977
- attemptId: input.attemptId,
6978
- executionGeneration,
7290
+ selectedTools: selectedFirstPartyMcpTools,
7291
+ subjectId: "worker:first-party-mcp",
7292
+ subjectLabel: "OpenGeni worker",
7293
+ ...(interactionInterventionResume
7294
+ ? { interventionResume: interactionInterventionResume }
7295
+ : {}),
7296
+ }),
7297
+ ...(googleDrivePublicationTool && googleDrivePublicationAllowed
7298
+ ? [googleDrivePublicationTool]
7299
+ : []),
7300
+ ];
7301
+ recordTurnStartupPhase(observability, {
7302
+ phase: "tool_context_preparation",
7303
+ provider: turnExecutionPolicy.providerId,
7304
+ backend: activeSandboxBackend ?? groupBoxBackend,
7305
+ outcome: "completed",
7306
+ durationSeconds: (performance.now() - toolContextPreparationStartedAt) / 1_000,
7307
+ count: turnTools.length,
7308
+ });
7309
+ const toolPreparationStartedAt = performance.now();
7310
+ let toolPreparationOutcome: "completed" | "failed" = "completed";
7311
+ try {
7312
+ preparedTools = await waitForTurnOperation(
7313
+ runtime.prepareTools(runSettings, turnTools, {
7314
+ accountId: input.accountId,
7315
+ workspaceId: input.workspaceId,
7316
+ sessionId: input.sessionId,
7317
+ // Sign the calling turn into the first-party token so tools classify
7318
+ // the caller by its own identity (sacred-pause guard), not the racy
7319
+ // live active pointer.
7320
+ ...(turnId ? { turnId } : {}),
7321
+ attemptId: input.attemptId,
7322
+ executionGeneration,
7323
+ subjectId: "worker:first-party-mcp",
7324
+ subjectLabel: "OpenGeni worker",
7325
+ ...(credentialSubjectId ? { credentialSubjectId } : {}),
7326
+ ...(codexAppsAuth ? { codexAppsAuth } : {}),
7327
+ resolveCredential,
7328
+ onAuthNeeded: publishToolAuthNeeded,
7329
+ localMcpServers,
7330
+ onPreparationPhase: (measurement) => {
7331
+ recordTurnStartupPhase(observability, {
7332
+ phase: `tool_${measurement.phase}`,
7333
+ provider: turnExecutionPolicy.providerId,
7334
+ backend: activeSandboxBackend ?? groupBoxBackend,
7335
+ outcome: measurement.outcome,
7336
+ durationSeconds: measurement.durationSeconds,
7337
+ count: turnTools.length,
7338
+ });
7339
+ },
7340
+ onAttemptToolCatalog: async (catalog) => {
7341
+ await persistAttemptToolCatalog(db, catalog);
6979
7342
  },
7343
+ // Manager-style sessions carry a creation-validated permission set
7344
+ // for their first-party MCP token; null keeps the fixed default.
6980
7345
  ...(session.firstPartyMcpPermissions?.length
6981
- ? { permissions: session.firstPartyMcpPermissions }
7346
+ ? { firstPartyPermissions: session.firstPartyMcpPermissions }
6982
7347
  : {}),
6983
- selectedTools: selectedFirstPartyMcpTools,
6984
- subjectId: "worker:first-party-mcp",
6985
- subjectLabel: "OpenGeni worker",
6986
- ...(interactionInterventionResume
6987
- ? { interventionResume: interactionInterventionResume }
7348
+ firstPartyTools: selectedFirstPartyMcpTools,
7349
+ nestedAgentDepth: session.nestedAgentDepth,
7350
+ effectiveMaxNestedAgentDepth: session.effectiveMaxNestedAgentDepth,
7351
+ attemptToolDefinitions,
7352
+ ...(googleDrivePublicationTarget && googleDrivePublicationAllowed
7353
+ ? {
7354
+ attemptToolAuthorize: async ({ call }) => {
7355
+ if (
7356
+ call.caller.kind !== "codemode" ||
7357
+ call.identity.serverId !== "google-drive-publishing" ||
7358
+ call.identity.toolName !== "google_drive_publish_file"
7359
+ ) {
7360
+ return;
7361
+ }
7362
+ await authorizeGoogleDrivePublicationAttempt({
7363
+ db,
7364
+ identity: connectorActionIdentity,
7365
+ target: googleDrivePublicationTarget,
7366
+ approvalId: call.operationId,
7367
+ arguments: call.arguments,
7368
+ });
7369
+ },
7370
+ }
6988
7371
  : {}),
6989
7372
  }),
6990
- }),
6991
- cancellationSignal,
6992
- async (latePreparedTools) => await latePreparedTools.close().catch(() => undefined),
6993
- );
7373
+ cancellationSignal,
7374
+ async (latePreparedTools) => await latePreparedTools.close().catch(() => undefined),
7375
+ );
7376
+ } catch (error) {
7377
+ toolPreparationOutcome = "failed";
7378
+ throw error;
7379
+ } finally {
7380
+ recordTurnStartupPhase(observability, {
7381
+ phase: "tool_preparation",
7382
+ provider: turnExecutionPolicy.providerId,
7383
+ backend: activeSandboxBackend ?? groupBoxBackend,
7384
+ outcome: toolPreparationOutcome,
7385
+ durationSeconds: (performance.now() - toolPreparationStartedAt) / 1_000,
7386
+ count: turnTools.length,
7387
+ });
7388
+ }
7389
+ const postToolPreparationStartedAt = performance.now();
6994
7390
  if (turnId && preparedTools.attemptToolEnvironment) {
6995
7391
  codemodeDispatcher = new CodemodeAttemptDispatcher(
6996
7392
  db,
@@ -7052,7 +7448,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7052
7448
  await resolveImageGenerationReferences({
7053
7449
  db,
7054
7450
  objectStorage: objectStorage!,
7451
+ accountId: input.accountId,
7055
7452
  workspaceId: input.workspaceId,
7453
+ subjectId: fileAuthoritySubjectId,
7056
7454
  references,
7057
7455
  readSandboxFile: async (path, maxBytes) => {
7058
7456
  const imageReferenceSession = (setupBoxSession ??
@@ -7362,18 +7760,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7362
7760
  turnExecutionPolicy.providerId,
7363
7761
  turnExecutionPolicy.latencyMode,
7364
7762
  );
7365
- const connectorActionIdentity = {
7366
- accountId: input.accountId,
7367
- workspaceId: input.workspaceId,
7368
- sessionId: input.sessionId,
7369
- turnId,
7370
- attemptId: input.attemptId,
7371
- executionGeneration,
7372
- initiator: {
7373
- kind: turn.initiator.kind,
7374
- subjectId: turn.initiator.subjectId,
7375
- },
7376
- } as const;
7377
7763
  const connectorActionPolicy: ConnectorActionPolicyHooks = {
7378
7764
  prepare: async (call) =>
7379
7765
  await prepareConnectorActionApproval(db, connectorActionIdentity, call),
@@ -7391,190 +7777,207 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7391
7777
  const runtimeSkillActivations = [
7392
7778
  ...installedSkillRuntime.activations,
7393
7779
  ...packRuntime.skillActivations,
7394
- ...session.skills.map((skill) => ({
7395
- source: "session" as const,
7396
- id: `session:${session.id}:${skill.name}`,
7397
- artifact: {
7398
- name: skill.name,
7399
- description: skill.description ?? null,
7400
- files: skill.files.map((file) => ({
7401
- path: file.path,
7402
- content: file.content,
7403
- })),
7404
- },
7405
- reason: "attached to session",
7406
- })),
7407
- ];
7408
- const agent = runtime.buildAgent(modelRunSettings, turnResources, {
7409
- reasoningEffort: turn.reasoningEffort,
7410
- latencyMode: turnExecutionPolicy.latencyMode,
7411
- ...(serviceTier ? { serviceTier } : {}),
7412
- ...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
7413
- humanInputEnabled: agentHumanInputEnabled,
7414
- genesisTitleHint: isGenesisTurn,
7415
- persistentSessionSettings: {
7416
- titleIsSet: Boolean(session.title?.trim()),
7417
- },
7418
- sandboxEnvironment,
7419
- ...(preparedTools.attemptToolCatalog
7420
- ? { attemptToolCatalog: preparedTools.attemptToolCatalog }
7421
- : {}),
7422
- ...(sandboxArtifactRuntime.available ? { artifactRuntimeAvailable: true } : {}),
7423
- ...(cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {}),
7424
- onToolCancellationFence: (fence) => {
7425
- toolCancellationFenceRef.current = fence;
7426
- },
7427
- // TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
7428
- // seed. ONLY when the effective backend is NOT selfhosted (the connected
7429
- // machine uses its own git creds — mirrors the skipGitHubToken gate above)
7430
- // AND the mint actually produced a token (repo resources present). The runtime
7431
- // seeds it to the box's token file before the repository-clone runs; it never
7432
- // touches the box/agent manifest env.
7433
- ...(activeSandboxBackend !== "selfhosted" && sandboxGitTokens
7434
- ? { gitTokenSeeds: sandboxGitTokens }
7435
- : {}),
7436
- ...(activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings
7437
- ? { gitCredentialBindings: sandboxGitCredentialBindings }
7438
- : {}),
7439
- ...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
7440
- ? { gitTokenSeed: sandboxGitToken }
7441
- : {}),
7442
- ...(sandboxCodemodeToken ? { codemodeAvailable: true } : {}),
7443
- // Managed boxes receive the bearer through their protected per-session
7444
- // token file. Connected Machines use transient per-exec delivery above,
7445
- // so they must not run the file-seeding lifecycle hook.
7446
- ...(activeSandboxBackend !== "selfhosted" && sandboxCodemodeToken
7447
- ? {
7448
- codemodeTokenSeed: sandboxCodemodeToken,
7449
- codemodeTokenSessionId: input.sessionId,
7450
- }
7451
- : {}),
7452
- ...(activeSandboxBackend ? { activeSandboxBackend } : {}),
7453
- fileResourceDownloads,
7454
- mcpServers: preparedTools.mcpServers,
7455
- resolvedMcpConnectionIds: preparedTools.resolvedMcpConnectionIds,
7456
- connectorActionPolicy,
7457
- // LIVE by-reference connector namespaces (fills during this turn's
7458
- // codex_apps tools/list): the codex tool_search description reads it per
7459
- // model call so the model sees the account's real connected sources.
7460
- codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
7461
- // Resolved-model routing + gating (legacy defaults when null). The model
7462
- // is passed as the model *string* (agent.model = runSettings.openaiModel),
7463
- // NOT a Model instance: an instance only survives the in-process
7464
- // ("none") run, whereas the SandboxAgent/Modal path drops it and
7465
- // re-resolves the model *name* through the global MultiProviderModelProvider
7466
- // configureOpenAI installed — so registry models (Fireworks GLM) route to
7467
- // their own client instead of 404ing against the built-in Azure/OpenAI
7468
- // client. The gating still comes from the resolved provider: server-side
7469
- // store/compaction follow the provider's compaction mode (registry
7470
- // providers resolve to "client"); encrypted reasoning is only
7471
- // round-tripped on the Responses wire API; hosted web search is attached
7472
- // whenever the provider declares it runnable and is independent of the
7473
- // session's MCP allow-list; the effective context window drives the
7474
- // compaction threshold.
7475
- hostedWebSearch,
7476
- ...imageGenerationOption,
7477
- ...videoGenerationOption,
7478
- lazyToolTransport,
7479
- supportsImageInput,
7480
- inputFileMediaTypes: modelInputPolicy.inputFileMediaTypes,
7481
- ...(resolvedModel
7482
- ? {
7483
- encryptedReasoning:
7484
- resolvedModel.provider.api === "responses" &&
7485
- runSettings.openaiReasoningEncryptedContent,
7486
- contextWindowTokens:
7487
- resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
7488
- // The ChatGPT/Codex backend rejects the SDK's HOSTED apply_patch
7489
- // tool. Gateway Responses routes likewise expose ordinary function
7490
- // tools, not OpenAI-hosted sandbox tools. Tell buildAgent to use
7491
- // function apply_patch and wrap successful view_image results as
7492
- // typed input_image content. Chat wires have no proven typed image
7493
- // result transport and therefore receive no view_image tool.
7494
- structuredToolTransport: structuredToolTransportForTurn(resolvedModel),
7495
- // EXPLICIT computer-use tool transport, derived from the resolved provider's
7496
- // authoritative wire identity (codex → function-image, chat → disabled,
7497
- // responses → hosted) so the runtime never string-sniffs the model instance's
7498
- // constructor name. See {@link computerToolModeForTurn}.
7499
- computerToolMode: computerToolModeForTurn(resolvedModel),
7500
- ...(promptCacheKey ? { promptCacheKey } : {}),
7501
- }
7502
- : // LEGACY global-client fallback (resolveTurnModel returned null → the model
7503
- // is not in the registry, served by the built-in OpenAI/Azure Responses
7504
- // client). That backend has real hosted support, so pin computerToolMode to
7505
- // "hosted" EXPLICITLY rather than leaving the runtime to sniff the instance.
7506
- {
7507
- computerToolMode: computerToolModeForTurn(null),
7508
- promptCacheKey: input.sessionId,
7509
- }),
7510
- // Lazy computer-use seam: runtime first brings up :0 only after the model
7511
- // selects a computer tool, then this hook begins the optional proof
7512
- // recording. Shell/filesystem turns never invoke either operation.
7513
- onComputerUseReady: async () => {
7514
- if (!resolvedSandbox) {
7515
- throw new Error("Computer-use display became ready without a resolved sandbox");
7516
- }
7517
- // This callback is the authoritative execution boundary. Record the
7518
- // action before async ffmpeg startup so transport-event ordering cannot
7519
- // make settlement misclassify a real computer turn as unused.
7520
- didComputerUse = true;
7521
- await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
7522
- },
7523
- onRetainableSessionImageOutput: retainSessionImageAtToolBoundary,
7524
- ...(runtimeSkillActivations.length > 0
7525
- ? { skillActivations: runtimeSkillActivations }
7526
- : {}),
7527
- ...(!structuredWorkspacePolicyActive && workspaceAgentInstructions
7528
- ? { instructionsTemplate: workspaceAgentInstructions }
7529
- : {}),
7530
- ...(workspaceGovernance ? { workspaceGovernance } : {}),
7531
- ...(workspaceMemory ? { workspaceMemory } : {}),
7532
- goalSnapshot: turn.goalSnapshot,
7533
- // Per-session persona tier (session > workspace > deployment default).
7534
- // Composed system-level AFTER the workspace persona so it refines it for
7535
- // this one session; absent ⇒ byte-identical to today's composition.
7536
- ...(session.instructions ? { sessionInstructions: session.instructions } : {}),
7537
- // Exact host context captured when this turn was accepted. It is
7538
- // system-level and disappears with the turn rather than entering chat.
7539
- ...(turn.turnInstructions || mcpAvailabilityInstructions
7540
- ? {
7541
- turnInstructions: [turn.turnInstructions, mcpAvailabilityInstructions]
7542
- .filter((value): value is string => Boolean(value))
7543
- .join(" "),
7544
- }
7545
- : {}),
7546
- ...workspaceEnvironmentOption,
7547
- // RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
7548
- // the frozen version carries a non-empty script), and the rig credential
7549
- // hooks. All absent for a rig-less turn (byte-for-byte today).
7550
- ...(rigVersion && rigName
7551
- ? {
7552
- rig: { name: rigName, version: rigVersion.version },
7553
- ...(rigVersion.setupScript && rigVersion.setupScript.trim().length > 0
7554
- ? {
7555
- rigSetup: {
7556
- rigId: session.rigId!,
7557
- versionId: rigVersion.id,
7558
- rigName,
7559
- script: rigVersion.setupScript,
7560
- timeoutMs: runSettings.rigSetupTimeoutMs,
7561
- contentHash: rigProviderImageContentHash({
7562
- backend: turn.sandboxBackend,
7563
- sourceImage: rigProviderImageSourceImage(
7564
- logicalSandboxSettings,
7565
- turn.sandboxBackend,
7566
- ),
7567
- definition: rigVersion,
7568
- }),
7569
- },
7570
- }
7571
- : {}),
7572
- ...(rigVersion.credentialHooks.length > 0
7573
- ? { rigCredentialHookIds: rigVersion.credentialHooks }
7574
- : {}),
7575
- }
7576
- : {}),
7780
+ ...session.skills.map((skill) => ({
7781
+ source: "session" as const,
7782
+ id: `session:${session.id}:${skill.name}`,
7783
+ artifact: {
7784
+ name: skill.name,
7785
+ description: skill.description ?? null,
7786
+ files: skill.files.map((file) => ({
7787
+ path: file.path,
7788
+ content: file.content,
7789
+ })),
7790
+ },
7791
+ reason: "attached to session",
7792
+ })),
7793
+ ];
7794
+ recordTurnStartupPhase(observability, {
7795
+ phase: "post_tool_preparation",
7796
+ provider: turnExecutionPolicy.providerId,
7797
+ backend: activeSandboxBackend ?? groupBoxBackend,
7798
+ outcome: "completed",
7799
+ durationSeconds: (performance.now() - postToolPreparationStartedAt) / 1_000,
7577
7800
  });
7801
+ const agent = (() => {
7802
+ const agentConstructionStartedAt = performance.now();
7803
+ let agentConstructionOutcome: "completed" | "failed" = "completed";
7804
+ try {
7805
+ return runtime.buildAgent(modelRunSettings, turnResources, {
7806
+ reasoningEffort: turn.reasoningEffort,
7807
+ latencyMode: turnExecutionPolicy.latencyMode,
7808
+ ...(serviceTier ? { serviceTier } : {}),
7809
+ ...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
7810
+ humanInputEnabled: agentHumanInputEnabled,
7811
+ genesisTitleHint: isGenesisTurn,
7812
+ persistentSessionSettings: {
7813
+ titleIsSet: Boolean(session.title?.trim()),
7814
+ },
7815
+ sandboxEnvironment,
7816
+ ...(preparedTools.attemptToolCatalog
7817
+ ? { attemptToolCatalog: preparedTools.attemptToolCatalog }
7818
+ : {}),
7819
+ ...(sandboxArtifactRuntime.available ? { artifactRuntimeAvailable: true } : {}),
7820
+ ...(cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {}),
7821
+ onToolCancellationFence: (fence) => {
7822
+ toolCancellationFenceRef.current = fence;
7823
+ },
7824
+ // TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
7825
+ // seed. ONLY when the effective backend is NOT selfhosted (the connected
7826
+ // machine uses its own git creds — mirrors the skipGitHubToken gate above)
7827
+ // AND the mint actually produced a token (repo resources present). The runtime
7828
+ // seeds it to the box's token file before the repository-clone runs; it never
7829
+ // touches the box/agent manifest env.
7830
+ ...(activeSandboxBackend !== "selfhosted" && sandboxGitTokens
7831
+ ? { gitTokenSeeds: sandboxGitTokens }
7832
+ : {}),
7833
+ ...(activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings
7834
+ ? { gitCredentialBindings: sandboxGitCredentialBindings }
7835
+ : {}),
7836
+ ...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
7837
+ ? { gitTokenSeed: sandboxGitToken }
7838
+ : {}),
7839
+ ...(sandboxCodemodeToken ? { codemodeAvailable: true } : {}),
7840
+ // Managed boxes receive the bearer through their protected per-session
7841
+ // token file. Connected Machines use transient per-exec delivery above,
7842
+ // so they must not run the file-seeding lifecycle hook.
7843
+ ...(activeSandboxBackend !== "selfhosted" && sandboxCodemodeToken
7844
+ ? {
7845
+ codemodeTokenSeed: sandboxCodemodeToken,
7846
+ codemodeTokenSessionId: input.sessionId,
7847
+ }
7848
+ : {}),
7849
+ ...(activeSandboxBackend ? { activeSandboxBackend } : {}),
7850
+ fileResourceDownloads,
7851
+ mcpServers: preparedTools.mcpServers,
7852
+ resolvedMcpConnectionIds: preparedTools.resolvedMcpConnectionIds,
7853
+ connectorActionPolicy,
7854
+ attemptConnectorActionBindings: googleDriveConnectorBindings,
7855
+ // LIVE by-reference connector namespaces (fills during this turn's
7856
+ // codex_apps tools/list): the codex tool_search description reads it per
7857
+ // model call so the model sees the account's real connected sources.
7858
+ codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
7859
+ // Resolved-model routing + gating (legacy defaults when null). The model
7860
+ // is passed as the model *string* (agent.model = runSettings.openaiModel),
7861
+ // NOT a Model instance: an instance only survives the in-process
7862
+ // ("none") run, whereas the SandboxAgent/Modal path drops it and
7863
+ // re-resolves the model *name* through the global MultiProviderModelProvider
7864
+ // configureOpenAI installed — so registry models (Fireworks GLM) route to
7865
+ // their own client instead of 404ing against the built-in Azure/OpenAI
7866
+ // client. The gating still comes from the resolved provider: server-side
7867
+ // store/compaction follow the provider's compaction mode (registry
7868
+ // providers resolve to "client"); encrypted reasoning is only
7869
+ // round-tripped on the Responses wire API; hosted web search is attached
7870
+ // whenever the provider declares it runnable and is independent of the
7871
+ // session's MCP allow-list; the effective context window drives the
7872
+ // compaction threshold.
7873
+ hostedWebSearch,
7874
+ ...imageGenerationOption,
7875
+ ...videoGenerationOption,
7876
+ lazyToolTransport,
7877
+ supportsImageInput,
7878
+ inputFileMediaTypes: modelInputPolicy.inputFileMediaTypes,
7879
+ ...(resolvedModel
7880
+ ? {
7881
+ encryptedReasoning:
7882
+ resolvedModel.provider.api === "responses" &&
7883
+ runSettings.openaiReasoningEncryptedContent,
7884
+ contextWindowTokens:
7885
+ resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
7886
+ // The ChatGPT/Codex backend rejects the SDK's HOSTED apply_patch
7887
+ // tool. Gateway Responses routes likewise expose ordinary function
7888
+ // tools, not OpenAI-hosted sandbox tools. Tell buildAgent to use
7889
+ // function apply_patch and wrap successful view_image results as
7890
+ // typed input_image content. Chat wires have no proven typed image
7891
+ // result transport and therefore receive no view_image tool.
7892
+ structuredToolTransport: structuredToolTransportForTurn(resolvedModel),
7893
+ // EXPLICIT computer-use tool transport, derived from the resolved provider's
7894
+ // authoritative wire identity (codex → function-image, chat → disabled,
7895
+ // responses → hosted) so the runtime never string-sniffs the model instance's
7896
+ // constructor name. See {@link computerToolModeForTurn}.
7897
+ computerToolMode: computerToolModeForTurn(resolvedModel),
7898
+ ...(promptCacheKey ? { promptCacheKey } : {}),
7899
+ }
7900
+ : // LEGACY global-client fallback (resolveTurnModel returned null → the model
7901
+ // is not in the registry, served by the built-in OpenAI/Azure Responses
7902
+ // client). That backend has real hosted support, so pin computerToolMode to
7903
+ // "hosted" EXPLICITLY rather than leaving the runtime to sniff the instance.
7904
+ {
7905
+ computerToolMode: computerToolModeForTurn(null),
7906
+ promptCacheKey: input.sessionId,
7907
+ }),
7908
+ // Lazy computer-use seam: runtime first brings up :0 only after the model
7909
+ // selects a computer tool, then this hook begins the optional proof
7910
+ // recording. Shell/filesystem turns never invoke either operation.
7911
+ onComputerUseReady: async () => {
7912
+ if (!resolvedSandbox) {
7913
+ throw new Error("Computer-use display became ready without a resolved sandbox");
7914
+ }
7915
+ // This callback is the authoritative execution boundary. Record the
7916
+ // action before async ffmpeg startup so transport-event ordering cannot
7917
+ // make settlement misclassify a real computer turn as unused.
7918
+ didComputerUse = true;
7919
+ await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
7920
+ },
7921
+ onRetainableSessionImageOutput: retainSessionImageAtToolBoundary,
7922
+ ...(runtimeSkillActivations.length > 0
7923
+ ? { skillActivations: runtimeSkillActivations }
7924
+ : {}),
7925
+ ...(!structuredWorkspacePolicyActive && workspaceAgentInstructions
7926
+ ? { instructionsTemplate: workspaceAgentInstructions }
7927
+ : {}),
7928
+ ...(workspaceGovernance ? { workspaceGovernance } : {}),
7929
+ ...(workspaceMemory ? { workspaceMemory } : {}),
7930
+ goalSnapshot: turn.goalSnapshot,
7931
+ // Per-session persona tier (session > workspace > deployment default).
7932
+ // Composed system-level AFTER the workspace persona so it refines it for
7933
+ // this one session; absent ⇒ byte-identical to today's composition.
7934
+ ...(session.instructions ? { sessionInstructions: session.instructions } : {}),
7935
+ ...workspaceEnvironmentOption,
7936
+ // RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
7937
+ // the frozen version carries a non-empty script), and the rig credential
7938
+ // hooks. All absent for a rig-less turn (byte-for-byte today).
7939
+ ...(rigVersion && rigName
7940
+ ? {
7941
+ rig: { name: rigName, version: rigVersion.version },
7942
+ ...(rigVersion.setupScript && rigVersion.setupScript.trim().length > 0
7943
+ ? {
7944
+ rigSetup: {
7945
+ rigId: session.rigId!,
7946
+ versionId: rigVersion.id,
7947
+ rigName,
7948
+ script: rigVersion.setupScript,
7949
+ timeoutMs: runSettings.rigSetupTimeoutMs,
7950
+ contentHash: rigProviderImageContentHash({
7951
+ backend: turn.sandboxBackend,
7952
+ sourceImage: rigProviderImageSourceImage(
7953
+ logicalSandboxSettings,
7954
+ turn.sandboxBackend,
7955
+ ),
7956
+ definition: rigVersion,
7957
+ }),
7958
+ },
7959
+ }
7960
+ : {}),
7961
+ ...(rigVersion.credentialHooks.length > 0
7962
+ ? { rigCredentialHookIds: rigVersion.credentialHooks }
7963
+ : {}),
7964
+ }
7965
+ : {}),
7966
+ });
7967
+ } catch (error) {
7968
+ agentConstructionOutcome = "failed";
7969
+ throw error;
7970
+ } finally {
7971
+ recordTurnStartupPhase(observability, {
7972
+ phase: "agent_construction",
7973
+ provider: turnExecutionPolicy.providerId,
7974
+ backend: activeSandboxBackend ?? groupBoxBackend,
7975
+ outcome: agentConstructionOutcome,
7976
+ durationSeconds: (performance.now() - agentConstructionStartedAt) / 1_000,
7977
+ });
7978
+ }
7979
+ })();
7980
+ const postAgentPreparationStartedAt = performance.now();
7578
7981
  if (modelRunSettings.sandboxBackend !== "none" && toolCancellationFenceRef.current === null) {
7579
7982
  throw new Error(
7580
7983
  "Sandbox agent construction did not install the mandatory turn tool cancellation fence",
@@ -7629,6 +8032,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7629
8032
  image: runSettings.modalImageRef ?? runSettings.dockerImage,
7630
8033
  }
7631
8034
  : {}),
8035
+ // The lazy acquire must enforce the same frozen rig authority
8036
+ // as the eager acquire; otherwise a warm box for another rig
8037
+ // can bypass the shared-state conflict/rotation fence.
8038
+ ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
7632
8039
  },
7633
8040
  "turn",
7634
8041
  lazyHolderId,
@@ -8083,69 +8490,109 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8083
8490
  return claimedResult({ status: "idle" });
8084
8491
  }
8085
8492
  }
8493
+ recordTurnStartupPhase(observability, {
8494
+ phase: "post_agent_preparation",
8495
+ provider: turnExecutionPolicy.providerId,
8496
+ backend: activeSandboxBackend ?? groupBoxBackend,
8497
+ outcome: "completed",
8498
+ durationSeconds: (performance.now() - postAgentPreparationStartedAt) / 1_000,
8499
+ });
8086
8500
  let fileMaterializationFailures: SandboxFileDownloadFailure[] = [];
8087
8501
  let fileDownloadsMaterializedForRun = false;
8088
8502
  if (resolvedSandbox && setupBoxSession && fileResourceDownloads.length > 0) {
8089
- const boxInstanceId = resolvedSandbox.established.instanceId;
8090
- // Managed boxes are immutable platform state, so their durable lease can
8091
- // memoize successful downloads. A connected machine is user-owned: files
8092
- // can be changed or removed between turns, so verify/materialize every
8093
- // attached file each turn instead of trusting the managed-box cache.
8503
+ const fileMaterializationStartedAt = performance.now();
8504
+ let fileMaterializationOutcome: "completed" | "failed" = "completed";
8094
8505
  const cacheMaterialization = resolvedSandbox.established.backendId !== "selfhosted";
8095
- const alreadyMaterialized = cacheMaterialization
8096
- ? await getMaterializedSandboxFileResources(db, {
8097
- accountId: input.accountId,
8098
- workspaceId: input.workspaceId,
8099
- sandboxGroupId: session.sandboxGroupId,
8100
- expectedEpoch: resolvedSandbox.leaseEpoch,
8101
- instanceId: boxInstanceId,
8102
- })
8103
- : new Set<string>();
8104
- const downloadsToMaterialize = filterUnmaterializedSandboxFileDownloads(
8105
- fileResourceDownloads,
8106
- alreadyMaterialized,
8107
- );
8108
- const runAs = sandboxRunAs(runSettings);
8109
- if (downloadsToMaterialize.length > 0) {
8110
- const materialized = await runWorkspaceMutationForSandbox(
8111
- resolvedSandbox,
8112
- "fileMaterialization",
8113
- async () =>
8114
- await materializeSandboxFileDownloads(
8115
- setupBoxSession as CodemodeTokenWriterSession,
8116
- downloadsToMaterialize,
8117
- {
8118
- onRuntimeEvent: async (event) => {
8119
- await publish!([{ type: event.type, payload: event.payload }], true);
8120
- },
8121
- ...(runAs ? { runAs } : {}),
8122
- ...(toolCancellationFenceRef.current
8123
- ? {
8124
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
8125
- toolCancellationFenceRef.current,
8126
- ),
8127
- }
8128
- : {}),
8129
- },
8130
- ),
8506
+ let fileMaterializationCache: "hit" | "miss" | "disabled" = cacheMaterialization
8507
+ ? "miss"
8508
+ : "disabled";
8509
+ try {
8510
+ const boxInstanceId = resolvedSandbox.established.instanceId;
8511
+ // Managed boxes are immutable platform state, so their durable lease can
8512
+ // memoize successful downloads. A connected machine is user-owned: files
8513
+ // can be changed or removed between turns, so verify/materialize every
8514
+ // attached file each turn instead of trusting the managed-box cache.
8515
+ const alreadyMaterialized = cacheMaterialization
8516
+ ? await getMaterializedSandboxFileResources(db, {
8517
+ accountId: input.accountId,
8518
+ workspaceId: input.workspaceId,
8519
+ sandboxGroupId: session.sandboxGroupId,
8520
+ expectedEpoch: resolvedSandbox.leaseEpoch,
8521
+ instanceId: boxInstanceId,
8522
+ })
8523
+ : new Set<string>();
8524
+ const downloadsToMaterialize = filterUnmaterializedSandboxFileDownloads(
8525
+ fileResourceDownloads,
8526
+ alreadyMaterialized,
8131
8527
  );
8132
- fileMaterializationFailures = materialized.failures;
8133
- const failedFileIds = new Set(materialized.failures.map((failure) => failure.fileId));
8134
- const succeededFileIds = downloadsToMaterialize
8135
- .map((download) => download.fileId)
8136
- .filter((fileId) => !failedFileIds.has(fileId));
8137
- if (cacheMaterialization && succeededFileIds.length > 0) {
8138
- await markSandboxFileResourcesMaterialized(db, {
8139
- accountId: input.accountId,
8140
- workspaceId: input.workspaceId,
8141
- sandboxGroupId: session.sandboxGroupId,
8142
- expectedEpoch: resolvedSandbox.leaseEpoch,
8143
- instanceId: boxInstanceId,
8144
- fileIds: succeededFileIds,
8145
- });
8528
+ if (cacheMaterialization && downloadsToMaterialize.length === 0) {
8529
+ fileMaterializationCache = "hit";
8530
+ }
8531
+ const runAs = sandboxRunAs(runSettings);
8532
+ if (downloadsToMaterialize.length > 0) {
8533
+ const materialized = await runWorkspaceMutationForSandbox(
8534
+ resolvedSandbox,
8535
+ "fileMaterialization",
8536
+ async () =>
8537
+ await materializeSandboxFileDownloads(
8538
+ setupBoxSession as CodemodeTokenWriterSession,
8539
+ downloadsToMaterialize,
8540
+ {
8541
+ onRuntimeEvent: async (event) => {
8542
+ await publish!([{ type: event.type, payload: event.payload }], true);
8543
+ },
8544
+ ...(runAs ? { runAs } : {}),
8545
+ ...(toolCancellationFenceRef.current
8546
+ ? {
8547
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
8548
+ toolCancellationFenceRef.current,
8549
+ ),
8550
+ }
8551
+ : {}),
8552
+ },
8553
+ ),
8554
+ );
8555
+ fileMaterializationFailures = materialized.failures;
8556
+ fileMaterializationOutcome = sandboxFileMaterializationOutcome(materialized.failures);
8557
+ const failedFileIds = new Set(materialized.failures.map((failure) => failure.fileId));
8558
+ const succeededFileIds = downloadsToMaterialize
8559
+ .map((download) => download.fileId)
8560
+ .filter((fileId) => !failedFileIds.has(fileId));
8561
+ if (cacheMaterialization && succeededFileIds.length > 0) {
8562
+ await markSandboxFileResourcesMaterialized(db, {
8563
+ accountId: input.accountId,
8564
+ workspaceId: input.workspaceId,
8565
+ sandboxGroupId: session.sandboxGroupId,
8566
+ expectedEpoch: resolvedSandbox.leaseEpoch,
8567
+ instanceId: boxInstanceId,
8568
+ fileIds: succeededFileIds,
8569
+ });
8570
+ }
8146
8571
  }
8572
+ fileDownloadsMaterializedForRun = true;
8573
+ } catch (error) {
8574
+ fileMaterializationOutcome = "failed";
8575
+ throw error;
8576
+ } finally {
8577
+ recordTurnStartupPhase(observability, {
8578
+ phase: "file_materialization",
8579
+ provider: turnExecutionPolicy.providerId,
8580
+ backend: activeSandboxBackend ?? groupBoxBackend,
8581
+ outcome: fileMaterializationOutcome,
8582
+ durationSeconds: (performance.now() - fileMaterializationStartedAt) / 1_000,
8583
+ count: fileResourceDownloads.length,
8584
+ cache: fileMaterializationCache,
8585
+ });
8147
8586
  }
8148
- fileDownloadsMaterializedForRun = true;
8587
+ } else {
8588
+ recordTurnStartupPhase(observability, {
8589
+ phase: "file_materialization",
8590
+ provider: turnExecutionPolicy.providerId,
8591
+ backend: activeSandboxBackend ?? groupBoxBackend,
8592
+ outcome: "completed",
8593
+ durationSeconds: 0,
8594
+ count: 0,
8595
+ });
8149
8596
  }
8150
8597
  const requiredVideoFileIds = new Set(requiredGeneratedVideoFiles.map((file) => file.fileId));
8151
8598
  const failedRequiredVideo = fileMaterializationFailures.find((failure) =>
@@ -8190,52 +8637,115 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8190
8637
  }
8191
8638
  let runInput: Awaited<ReturnType<typeof turnInput>>["input"] | null = null;
8192
8639
  const prepareRunAttemptInput = async () => {
8193
- const prepared = await turnInput(db, runtime, agent, trigger, {
8194
- turnId: activeTurnId,
8195
- recovering: turn.executionGeneration > 1,
8196
- ...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
8197
- ...(runCredentialsNote ? { runCredentialsNote } : {}),
8198
- providerApi,
8199
- projectCanonicalHistory: generatedImageHistoryProjector,
8200
- materializeModelHistory: materializeScreenshotHistory,
8201
- materializeSerializedRunState: materializeScreenshotRunState,
8202
- projectModelHistory: modelHistoryProjector,
8203
- });
8204
- for (const receipt of generatedImageReceiptsByArtifactId.values()) {
8205
- await materializeGeneratedImage(receipt);
8640
+ const historyPreparationStartedAt = performance.now();
8641
+ let historyPreparationOutcome: "completed" | "failed" = "completed";
8642
+ let preparedHistoryCount: number | null = null;
8643
+ try {
8644
+ const prepared = await turnInput(db, runtime, agent, trigger, {
8645
+ turnId: activeTurnId,
8646
+ fileAuthority: {
8647
+ accountId: input.accountId,
8648
+ subjectId: fileAuthoritySubjectId,
8649
+ },
8650
+ recovering: turn.executionGeneration > 1,
8651
+ ...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
8652
+ ...(runCredentialsNote ? { runCredentialsNote } : {}),
8653
+ ...(mcpAvailabilityNote ? { mcpAvailabilityNote } : {}),
8654
+ providerApi,
8655
+ projectCanonicalHistory: generatedImageHistoryProjector,
8656
+ materializeModelHistory: materializeScreenshotHistory,
8657
+ materializeSerializedRunState: materializeScreenshotRunState,
8658
+ projectModelHistory: modelHistoryProjector,
8659
+ onPreparationPhase: (measurement) => {
8660
+ recordTurnStartupPhase(observability, {
8661
+ phase: `history_${measurement.phase}`,
8662
+ provider: turnExecutionPolicy.providerId,
8663
+ backend: activeSandboxBackend ?? groupBoxBackend,
8664
+ outcome: measurement.outcome,
8665
+ durationSeconds: measurement.durationSeconds,
8666
+ });
8667
+ },
8668
+ });
8669
+ const generatedImageMaterializationStartedAt = performance.now();
8670
+ let generatedImageMaterializationOutcome: "completed" | "failed" = "completed";
8671
+ try {
8672
+ for (const receipt of generatedImageReceiptsByArtifactId.values()) {
8673
+ await materializeGeneratedImage(receipt);
8674
+ }
8675
+ } catch (error) {
8676
+ generatedImageMaterializationOutcome = "failed";
8677
+ throw error;
8678
+ } finally {
8679
+ recordTurnStartupPhase(observability, {
8680
+ phase: "history_generated_image_materialization",
8681
+ provider: turnExecutionPolicy.providerId,
8682
+ backend: activeSandboxBackend ?? groupBoxBackend,
8683
+ outcome: generatedImageMaterializationOutcome,
8684
+ durationSeconds: (performance.now() - generatedImageMaterializationStartedAt) / 1_000,
8685
+ count: generatedImageReceiptsByArtifactId.size,
8686
+ });
8687
+ }
8688
+ runInput = prepared.input;
8689
+ providerArtifactCandidates = prepared.providerArtifactCandidates;
8690
+ // Slice index = the length of the model-facing (active) history this turn
8691
+ // is seeded from; new items beyond it (the trigger message + this turn's
8692
+ // generated items) are the ones to persist. After a compaction this is the
8693
+ // short [summary, ...tail] active set, NOT the total row count. The
8694
+ // absolute write position is tracked separately (next whole number past
8695
+ // the max existing position) because the fractional summary row means
8696
+ // total rows no longer equal max(position)+1. Pre-compaction both reduce to
8697
+ // the old total-count value, so the common path is unchanged.
8698
+ //
8699
+ // CRITICAL: seed from the structurally repaired active-row length, not the raw active
8700
+ // count. `prepareRunInput` builds `state.history` from
8701
+ // `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
8702
+ // rows (a legacy orphan/dangling pair), the in-memory history this turn
8703
+ // starts from is K shorter than the raw row count. The reconcile slices the
8704
+ // repaired `state.history` off `persistedHistoryCount`; seeding it from
8705
+ // the raw count (K too high) skips K genuinely-new items, and a
8706
+ // `function_call` left in that skipped region can later have its
8707
+ // `function_call_result` persisted alone — the orphan that 400s on replay
8708
+ // and bricks the session (issue-61). The repaired seed is already
8709
+ // orphan-free, so it is a stable prefix of the re-sanitized history and the
8710
+ // slice begins exactly at the first genuinely-new item.
8711
+ // prepareInput already sanitized the exact durable prefix represented
8712
+ // by state.history. Carry its count forward instead of loading and
8713
+ // retaining the full active transcript a second time beside runInput.
8714
+ persistedHistoryCount = prepared.persistedHistoryCount;
8715
+ preparedHistoryCount = prepared.persistedHistoryCount;
8716
+ const historyPositionStartedAt = performance.now();
8717
+ let historyPositionOutcome: "completed" | "failed" = "completed";
8718
+ try {
8719
+ nextHistoryPosition = await nextSessionHistoryPosition(
8720
+ db,
8721
+ input.workspaceId,
8722
+ input.sessionId,
8723
+ );
8724
+ } catch (error) {
8725
+ historyPositionOutcome = "failed";
8726
+ throw error;
8727
+ } finally {
8728
+ recordTurnStartupPhase(observability, {
8729
+ phase: "history_position_load",
8730
+ provider: turnExecutionPolicy.providerId,
8731
+ backend: activeSandboxBackend ?? groupBoxBackend,
8732
+ outcome: historyPositionOutcome,
8733
+ durationSeconds: (performance.now() - historyPositionStartedAt) / 1_000,
8734
+ });
8735
+ }
8736
+ } catch (error) {
8737
+ historyPreparationOutcome = "failed";
8738
+ throw error;
8739
+ } finally {
8740
+ recordTurnStartupPhase(observability, {
8741
+ phase: "history_preparation",
8742
+ provider: turnExecutionPolicy.providerId,
8743
+ backend: activeSandboxBackend ?? groupBoxBackend,
8744
+ outcome: historyPreparationOutcome,
8745
+ durationSeconds: (performance.now() - historyPreparationStartedAt) / 1_000,
8746
+ count: preparedHistoryCount,
8747
+ });
8206
8748
  }
8207
- runInput = prepared.input;
8208
- providerArtifactCandidates = prepared.providerArtifactCandidates;
8209
- // Slice index = the length of the model-facing (active) history this turn
8210
- // is seeded from; new items beyond it (the trigger message + this turn's
8211
- // generated items) are the ones to persist. After a compaction this is the
8212
- // short [summary, ...tail] active set, NOT the total row count. The
8213
- // absolute write position is tracked separately (next whole number past
8214
- // the max existing position) because the fractional summary row means
8215
- // total rows no longer equal max(position)+1. Pre-compaction both reduce to
8216
- // the old total-count value, so the common path is unchanged.
8217
- //
8218
- // CRITICAL: seed from the structurally repaired active-row length, not the raw active
8219
- // count. `prepareRunInput` builds `state.history` from
8220
- // `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
8221
- // rows (a legacy orphan/dangling pair), the in-memory history this turn
8222
- // starts from is K shorter than the raw row count. The reconcile slices the
8223
- // repaired `state.history` off `persistedHistoryCount`; seeding it from
8224
- // the raw count (K too high) skips K genuinely-new items, and a
8225
- // `function_call` left in that skipped region can later have its
8226
- // `function_call_result` persisted alone — the orphan that 400s on replay
8227
- // and bricks the session (issue-61). The repaired seed is already
8228
- // orphan-free, so it is a stable prefix of the re-sanitized history and the
8229
- // slice begins exactly at the first genuinely-new item.
8230
- // prepareInput already sanitized the exact durable prefix represented
8231
- // by state.history. Carry its count forward instead of loading and
8232
- // retaining the full active transcript a second time beside runInput.
8233
- persistedHistoryCount = prepared.persistedHistoryCount;
8234
- nextHistoryPosition = await nextSessionHistoryPosition(
8235
- db,
8236
- input.workspaceId,
8237
- input.sessionId,
8238
- );
8239
8749
  };
8240
8750
 
8241
8751
  const forceContextCompaction = async (
@@ -8287,6 +8797,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8287
8797
  // calling runStreamAttempt again; resetting this state there would reuse
8288
8798
  // the first no-response-ID fallback key and suppress a real model call.
8289
8799
  const modelResponseState = createModelResponseEventState(claimedModelUsageSourceKeys);
8800
+ let workerPreparationTotalRecorded = false;
8290
8801
  const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
8291
8802
  if (!runInput) {
8292
8803
  throw new Error("Run input was not prepared");
@@ -8313,139 +8824,203 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8313
8824
  // heartbeat capture. Lazy setup already runs under the same wrapper in
8314
8825
  // its first-operation provisioner above.
8315
8826
  if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
8316
- const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
8317
- // `deferredSetup: true` below tells the runtime that the worker owns
8318
- // platform setup, so the runtime intentionally skips its credential
8319
- // session callback. Materialize host-managed run credentials here
8320
- // before any setup command (including provider login hooks), then
8321
- // decorate setup commands so they source the active generation.
8322
- await attachRunCredentialRenewal(
8323
- eagerSetupSession as RunCredentialCommandSession,
8324
- resolvedSandbox,
8325
- );
8326
- const eagerCredentialSetupSession = initialRunCredentialMaterial
8327
- ? withRunCredentialsSession(eagerSetupSession as object, input.sessionId)
8328
- : eagerSetupSession;
8329
- await runWorkspaceMutationForSandbox(
8330
- resolvedSandbox,
8331
- "eagerOwnedSandboxSetup",
8332
- async () =>
8333
- await runOwnedSandboxSetup(
8334
- agent,
8335
- ownedEstablished.session as never,
8336
- eagerCredentialSetupSession as never,
8337
- {
8338
- settings: runSettings,
8339
- environment: sandboxEnvironment,
8340
- preparedInput: runInput!,
8341
- ...(fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {}),
8342
- onRuntimeEvent: async (event) => {
8343
- await renewServingCredentialLease("runtime_event");
8344
- if (servingCredentialLeaseLost()) {
8345
- throw new Error("Provider credential lease expired during sandbox setup");
8346
- }
8347
- await publish!([{ type: event.type, payload: event.payload }], true);
8348
- },
8349
- ...(toolCancellationFenceRef.current
8350
- ? {
8351
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
8352
- toolCancellationFenceRef.current,
8353
- ),
8827
+ const ownedSandboxSetupStartedAt = performance.now();
8828
+ let ownedSandboxSetupOutcome: "completed" | "failed" = "completed";
8829
+ try {
8830
+ const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
8831
+ // `deferredSetup: true` below tells the runtime that the worker owns
8832
+ // platform setup, so the runtime intentionally skips its credential
8833
+ // session callback. Materialize host-managed run credentials here
8834
+ // before any setup command (including provider login hooks), then
8835
+ // decorate setup commands so they source the active generation.
8836
+ await attachRunCredentialRenewal(
8837
+ eagerSetupSession as RunCredentialCommandSession,
8838
+ resolvedSandbox,
8839
+ );
8840
+ const eagerCredentialSetupSession = initialRunCredentialMaterial
8841
+ ? withRunCredentialsSession(eagerSetupSession as object, input.sessionId)
8842
+ : eagerSetupSession;
8843
+ await runWorkspaceMutationForSandbox(
8844
+ resolvedSandbox,
8845
+ "eagerOwnedSandboxSetup",
8846
+ async () =>
8847
+ await runOwnedSandboxSetup(
8848
+ agent,
8849
+ ownedEstablished.session as never,
8850
+ eagerCredentialSetupSession as never,
8851
+ {
8852
+ settings: runSettings,
8853
+ environment: sandboxEnvironment,
8854
+ preparedInput: runInput!,
8855
+ ...(fileDownloadsMaterializedForRun
8856
+ ? { fileDownloadsMaterialized: true }
8857
+ : {}),
8858
+ onRuntimeEvent: async (event) => {
8859
+ await renewServingCredentialLease("runtime_event");
8860
+ if (servingCredentialLeaseLost()) {
8861
+ throw new Error("Provider credential lease expired during sandbox setup");
8354
8862
  }
8355
- : {}),
8356
- },
8357
- ),
8358
- );
8359
- await attachGitCredentialRenewal(
8360
- eagerSetupSession as GitCredentialTokenWriterSession,
8361
- initialGitCredentials,
8362
- );
8863
+ await publish!([{ type: event.type, payload: event.payload }], true);
8864
+ },
8865
+ ...(toolCancellationFenceRef.current
8866
+ ? {
8867
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
8868
+ toolCancellationFenceRef.current,
8869
+ ),
8870
+ }
8871
+ : {}),
8872
+ },
8873
+ ),
8874
+ );
8875
+ await attachGitCredentialRenewal(
8876
+ eagerSetupSession as GitCredentialTokenWriterSession,
8877
+ initialGitCredentials,
8878
+ );
8879
+ } catch (error) {
8880
+ ownedSandboxSetupOutcome = "failed";
8881
+ throw error;
8882
+ } finally {
8883
+ recordTurnStartupPhase(observability, {
8884
+ phase: "owned_sandbox_setup",
8885
+ provider: turnExecutionPolicy.providerId,
8886
+ backend: activeSandboxBackend ?? groupBoxBackend,
8887
+ outcome: ownedSandboxSetupOutcome,
8888
+ durationSeconds: (performance.now() - ownedSandboxSetupStartedAt) / 1_000,
8889
+ count: fileResourceDownloads.length,
8890
+ });
8891
+ }
8363
8892
  }
8364
8893
  // Conservative request boundary: once control enters runStream, the
8365
8894
  // provider may have accepted work even if no response/event follows.
8366
8895
  // Escaped MCP setup timeouts are automatically recoverable only
8367
8896
  // before this line.
8368
8897
  recordCompanyBrainContributionReceiptOnce();
8369
- modelRequestStarted = true;
8370
- return await runtime.runStream(agent, runInput!, modelRunSettings, {
8371
- signal: runtimeCancellationSignal,
8372
- sandboxEnvironment,
8373
- onRuntimeEvent: async (event) => {
8374
- await renewServingCredentialLease("runtime_event");
8375
- if (servingCredentialLeaseLost()) {
8376
- throw new Error("Provider credential lease expired during the active turn");
8377
- }
8378
- await publish!([{ type: event.type, payload: event.payload }], true);
8379
- },
8380
- // P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
8381
- // keystone). Absent when the flag is off -> legacy build-and-discard.
8382
- ...(ownedEstablished
8383
- ? {
8384
- ownedSandbox: {
8385
- client: ownedEstablished.client,
8386
- session: ownedEstablished.session,
8387
- ...(resolvedSandbox?.established.sessionState
8898
+ const providerDispatchStartedAt = performance.now();
8899
+ let providerDispatchOutcome: "completed" | "failed" = "completed";
8900
+ try {
8901
+ // This histogram describes worker preparation until the first entry
8902
+ // into the runtime. Lazy SDK request preparation and the durable
8903
+ // model-request audit happen after this boundary and have their own
8904
+ // phase metrics. Context-compaction recovery re-enters
8905
+ // runStreamAttempt; recording again would fold the prior request and
8906
+ // compaction into a bogus second startup sample.
8907
+ if (!workerPreparationTotalRecorded) {
8908
+ workerPreparationTotalRecorded = true;
8909
+ recordTurnWorkerPreparationTotal(observability, {
8910
+ provider: turnExecutionPolicy.providerId,
8911
+ backend: activeSandboxBackend ?? groupBoxBackend,
8912
+ outcome: "completed",
8913
+ durationSeconds: (performance.now() - workerPreparationStartedAt) / 1_000,
8914
+ });
8915
+ }
8916
+ modelRequestStarted = true;
8917
+ return await runtime.runStream(agent, runInput!, modelRunSettings, {
8918
+ signal: runtimeCancellationSignal,
8919
+ sandboxEnvironment,
8920
+ onRuntimeEvent: async (event) => {
8921
+ await renewServingCredentialLease("runtime_event");
8922
+ if (servingCredentialLeaseLost()) {
8923
+ throw new Error("Provider credential lease expired during the active turn");
8924
+ }
8925
+ await publish!([{ type: event.type, payload: event.payload }], true);
8926
+ },
8927
+ // P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
8928
+ // keystone). Absent when the flag is off -> legacy build-and-discard.
8929
+ ...(ownedEstablished
8930
+ ? {
8931
+ ownedSandbox: {
8932
+ client: ownedEstablished.client,
8933
+ session: ownedEstablished.session,
8934
+ ...(resolvedSandbox?.established.sessionState
8935
+ ? {
8936
+ sessionState: resolvedSandbox.established.sessionState,
8937
+ }
8938
+ : {}),
8939
+ // Pin platform setup (hooks + file materialization) to the un-proxied
8940
+ // established box — never through the routing proxy, which would
8941
+ // re-route those execs onto a machine swapped in mid-turn.
8942
+ ...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
8943
+ ...(fileDownloadsMaterializedForRun
8944
+ ? { fileDownloadsMaterialized: true }
8945
+ : {}),
8946
+ // Both owned paths execute setup outside runStream: eager just
8947
+ // above under its exact admission, lazy in the provisioner.
8948
+ deferredSetup: true,
8949
+ },
8950
+ }
8951
+ : {}),
8952
+ ...(activeSandboxBackend !== "selfhosted" &&
8953
+ sandboxCodemodeToken &&
8954
+ sandboxCodemodeTokenExpiresAt &&
8955
+ !lazyOwnedSandbox
8956
+ ? {
8957
+ onCodemodeTokenSessionReady: async (
8958
+ tokenSession: CodemodeTokenWriterSession,
8959
+ ) => {
8960
+ const renewalSession =
8961
+ (setupBoxSession as CodemodeTokenWriterSession | null) ?? tokenSession;
8962
+ await attachCodemodeTokenRenewal(renewalSession);
8963
+ },
8964
+ }
8965
+ : {}),
8966
+ ...(runCredentialResolver
8967
+ ? {
8968
+ runCredentialSessionId: input.sessionId,
8969
+ ...(!lazyOwnedSandbox
8388
8970
  ? {
8389
- sessionState: resolvedSandbox.established.sessionState,
8971
+ onRunCredentialSessionReady: async (
8972
+ credentialSession: RunCredentialCommandSession,
8973
+ ) => {
8974
+ const pinnedCredentialSession = setupBoxSession
8975
+ ? (setupBoxSession as RunCredentialCommandSession)
8976
+ : credentialSession;
8977
+ await attachRunCredentialRenewal(pinnedCredentialSession);
8978
+ },
8390
8979
  }
8391
8980
  : {}),
8392
- // Pin platform setup (hooks + file materialization) to the un-proxied
8393
- // established box — never through the routing proxy, which would
8394
- // re-route those execs onto a machine swapped in mid-turn.
8395
- ...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
8396
- ...(fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {}),
8397
- // Both owned paths execute setup outside runStream: eager just
8398
- // above under its exact admission, lazy in the provisioner.
8399
- deferredSetup: true,
8400
- },
8401
- }
8402
- : {}),
8403
- ...(activeSandboxBackend !== "selfhosted" &&
8404
- sandboxCodemodeToken &&
8405
- sandboxCodemodeTokenExpiresAt &&
8406
- !lazyOwnedSandbox
8407
- ? {
8408
- onCodemodeTokenSessionReady: async (tokenSession: CodemodeTokenWriterSession) => {
8409
- const renewalSession =
8410
- (setupBoxSession as CodemodeTokenWriterSession | null) ?? tokenSession;
8411
- await attachCodemodeTokenRenewal(renewalSession);
8412
- },
8413
- }
8414
- : {}),
8415
- ...(runCredentialResolver
8416
- ? {
8417
- runCredentialSessionId: input.sessionId,
8418
- ...(!lazyOwnedSandbox
8419
- ? {
8420
- onRunCredentialSessionReady: async (
8421
- credentialSession: RunCredentialCommandSession,
8422
- ) => {
8423
- const pinnedCredentialSession = setupBoxSession
8424
- ? (setupBoxSession as RunCredentialCommandSession)
8425
- : credentialSession;
8426
- await attachRunCredentialRenewal(pinnedCredentialSession);
8427
- },
8981
+ }
8982
+ : {}),
8983
+ ...(modelRunSettings.sandboxBackend !== "none"
8984
+ ? {
8985
+ onSandboxSessionReady: async (sandboxSession: CodemodeTokenWriterSession) => {
8986
+ sdkOwnedSandboxSession = sandboxSession;
8987
+ for (const receipt of generatedImageReceiptsByArtifactId.values()) {
8988
+ await materializeGeneratedImageInOwnedSdkSession(receipt, sandboxSession);
8428
8989
  }
8429
- : {}),
8430
- }
8431
- : {}),
8432
- ...(modelRunSettings.sandboxBackend !== "none"
8433
- ? {
8434
- onSandboxSessionReady: async (sandboxSession: CodemodeTokenWriterSession) => {
8435
- sdkOwnedSandboxSession = sandboxSession;
8436
- for (const receipt of generatedImageReceiptsByArtifactId.values()) {
8437
- await materializeGeneratedImageInOwnedSdkSession(receipt, sandboxSession);
8438
- }
8439
- },
8440
- }
8441
- : {}),
8442
- contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
8443
- contextCompactionRequested: () =>
8444
- isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
8445
- ...(toolCancellationFenceRef.current
8446
- ? { turnToolCancellationFence: toolCancellationFenceRef.current }
8447
- : {}),
8448
- });
8990
+ },
8991
+ }
8992
+ : {}),
8993
+ contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
8994
+ contextCompactionRequested: () =>
8995
+ isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
8996
+ onModelPreparationPhase: (measurement) => {
8997
+ recordTurnStartupPhase(observability, {
8998
+ phase: `model_prepare_${measurement.phase}`,
8999
+ provider: turnExecutionPolicy.providerId,
9000
+ backend: activeSandboxBackend ?? groupBoxBackend,
9001
+ outcome: measurement.outcome,
9002
+ durationSeconds: measurement.durationSeconds,
9003
+ ...(measurement.count === undefined ? {} : { count: measurement.count }),
9004
+ });
9005
+ },
9006
+ ...(toolCancellationFenceRef.current
9007
+ ? {
9008
+ turnToolCancellationFence: toolCancellationFenceRef.current,
9009
+ }
9010
+ : {}),
9011
+ });
9012
+ } catch (error) {
9013
+ providerDispatchOutcome = "failed";
9014
+ throw error;
9015
+ } finally {
9016
+ recordTurnStartupPhase(observability, {
9017
+ phase: "provider_dispatch",
9018
+ provider: turnExecutionPolicy.providerId,
9019
+ backend: activeSandboxBackend ?? groupBoxBackend,
9020
+ outcome: providerDispatchOutcome,
9021
+ durationSeconds: (performance.now() - providerDispatchStartedAt) / 1_000,
9022
+ });
9023
+ }
8449
9024
  };
8450
9025
  if (codexLeaseLost) {
8451
9026
  throw new Error("Codex credential lease expired before the model run");
@@ -8471,11 +9046,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8471
9046
  },
8472
9047
  );
8473
9048
 
9049
+ const streamBootstrapStartedAt = performance.now();
9050
+ let streamBootstrapRecorded = false;
9051
+ const recordStreamBootstrap = (): void => {
9052
+ if (streamBootstrapRecorded) return;
9053
+ streamBootstrapRecorded = true;
9054
+ recordTurnStartupPhase(observability, {
9055
+ phase: "stream_bootstrap",
9056
+ provider: turnExecutionPolicy.providerId,
9057
+ backend: activeSandboxBackend ?? groupBoxBackend,
9058
+ outcome: "completed",
9059
+ durationSeconds: (performance.now() - streamBootstrapStartedAt) / 1_000,
9060
+ count: turnTools.length,
9061
+ });
9062
+ };
9063
+ if (!firstModelRequestPreparationRecorded) {
9064
+ firstModelRequestPreparationStartedAt = performance.now();
9065
+ firstModelRequestCheckpointAt = firstModelRequestPreparationStartedAt;
9066
+ }
8474
9067
  const iterator = stream.toStream()[Symbol.asyncIterator]();
8475
9068
  let streamDone = false;
8476
9069
  try {
8477
9070
  while (true) {
8478
9071
  const next = await nextStreamEvent(iterator, activityContext);
9072
+ recordStreamBootstrap();
8479
9073
  if (next.done) {
8480
9074
  streamDone = true;
8481
9075
  break;
@@ -10437,7 +11031,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
10437
11031
  }
10438
11032
  activityStatus = "failed";
10439
11033
  activityError = error;
10440
- if (!publish || !turnId || !turnStartedPublished) {
11034
+ if (!turnId) {
11035
+ throw preClaimAdmissionFailure(error);
11036
+ }
11037
+ if (!publish || !turnStartedPublished) {
10441
11038
  throw error;
10442
11039
  }
10443
11040
  // A partial/malformed stream may have emitted assistant/tool items (and
@@ -11081,6 +11678,28 @@ async function assertGitHubResourcesRemainAuthorized(
11081
11678
  }
11082
11679
  }
11083
11680
 
11681
+ async function assertFileResourcesRemainAuthorized(
11682
+ db: ActivityServices["db"],
11683
+ accountId: string,
11684
+ workspaceId: string,
11685
+ subjectId: string | null,
11686
+ resources: ResourceRef[],
11687
+ ): Promise<void> {
11688
+ const fileIds = resources.flatMap((resource) =>
11689
+ resource.kind === "file" ? [resource.fileId] : [],
11690
+ );
11691
+ if (fileIds.length === 0) return;
11692
+ const authorized = await getFilesForSubject(db, {
11693
+ accountId,
11694
+ workspaceId,
11695
+ subjectId,
11696
+ fileIds,
11697
+ });
11698
+ if (authorized.length !== new Set(fileIds).size) {
11699
+ throw new Error("One or more file resources are unavailable or no longer authorized");
11700
+ }
11701
+ }
11702
+
11084
11703
  async function assertGitHubTokenMintSelectionAuthorized(
11085
11704
  db: Parameters<typeof areGitHubRepositoriesAllowedForWorkspace>[0],
11086
11705
  workspaceId: string,
@@ -11994,7 +12613,9 @@ async function sandboxFileDownloadsForRun(
11994
12613
  settings: Settings,
11995
12614
  db: ActivityServices["db"],
11996
12615
  objectStorage: ObjectStorage | null,
12616
+ accountId: string,
11997
12617
  workspaceId: string,
12618
+ subjectId: string | null,
11998
12619
  resources: ResourceRef[],
11999
12620
  activeSandboxBackend: Settings["sandboxBackend"] = settings.sandboxBackend,
12000
12621
  ): Promise<SandboxFileDownload[]> {
@@ -12015,7 +12636,12 @@ async function sandboxFileDownloadsForRun(
12015
12636
  const downloadStorage = objectStorageForSandboxDownloads(settings, objectStorage);
12016
12637
  const downloads: SandboxFileDownload[] = [];
12017
12638
  for (const resource of fileResources) {
12018
- const file = await requireFile(db, workspaceId, resource.fileId);
12639
+ const file = await requireFileForSubject(db, {
12640
+ accountId,
12641
+ workspaceId,
12642
+ subjectId,
12643
+ fileId: resource.fileId,
12644
+ });
12019
12645
  const url = await downloadStorage.createGetUrl({ key: file.objectKey });
12020
12646
  downloads.push({
12021
12647
  fileId: file.id,