@opengeni/worker-bundle 0.20.0 → 0.20.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -87,6 +87,7 @@ import {
87
87
  SandboxImageConflictError,
88
88
  ActiveSessionHistoryLimitExceededError,
89
89
  ApprovalRunStateLimitExceededError,
90
+ isRetryableDatabaseTransportFailure,
90
91
  isSessionEventPersistenceError,
91
92
  getEnrollment,
92
93
  abandonRecordingForTurnAttempt,
@@ -344,6 +345,7 @@ import {
344
345
  import type {
345
346
  TurnActivityServices as ActivityServices,
346
347
  EscapedMcpTimeoutRecoveryDetail,
348
+ PreClaimFailureDetail,
347
349
  RunAgentTurnInput,
348
350
  RunAgentTurnResult,
349
351
  SessionAttemptQuiescenceProof,
@@ -351,6 +353,8 @@ import type {
351
353
  import {
352
354
  ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
353
355
  ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
356
+ PRE_CLAIM_FAILURE_MESSAGE,
357
+ PRE_CLAIM_FAILURE_TYPE,
354
358
  } from "./types";
355
359
  import {
356
360
  resumeBoxForTurn,
@@ -381,10 +385,14 @@ import {
381
385
  recordCompanyBrainContributions,
382
386
  recordSessionEventAppendLatency,
383
387
  recordSessionEventPublishLatency,
388
+ recordTurnSandboxEstablishPolicy,
389
+ recordTurnStartupPhase,
390
+ recordTurnWorkerPreparationTotal,
384
391
  runtimeMetricsHooksForObservability,
385
392
  StreamTimingMetrics,
386
393
  turnLifecycleMetricsFor,
387
394
  type TurnOutcome,
395
+ type TurnSandboxEstablishReason,
388
396
  } from "../observability-metrics";
389
397
  import { buildCompanyBrainContributionReceipt } from "../model-context-contributions";
390
398
  import {
@@ -704,6 +712,52 @@ export function escapedMcpTimeoutRecoveryFailure(input: {
704
712
  });
705
713
  }
706
714
 
715
+ /**
716
+ * Convert the atomic claim transaction's failure into a small, stable
717
+ * Temporal wire contract. The original error remains in activity diagnostics,
718
+ * but SQL text, parameters, and arbitrary invariant messages never enter
719
+ * workflow history. Contention and operational database unavailability are
720
+ * safe to re-read after backoff because the claim transaction contains no
721
+ * model/tool effects; permanent database and state failures require terminal
722
+ * settlement.
723
+ */
724
+ export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
725
+ const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
726
+ const sqlState = persistenceFailure?.details.sqlState ?? null;
727
+ const rawTransportFailure = sqlState === null && isRetryableDatabaseTransportFailure(error);
728
+ // The database transaction itself retries only the two contention failures
729
+ // proven safe for immediate replay. Once the activity has failed, the
730
+ // workflow may also retry operational outages after a durable re-read and
731
+ // bounded delay. Unknown driver/database failures stay recoverable because
732
+ // they commonly represent a lost connection; known constraint, auth, and
733
+ // application SQLSTATEs are permanent and must not create an infinite loop.
734
+ const retryable = Boolean(
735
+ rawTransportFailure ||
736
+ (persistenceFailure &&
737
+ (sqlState === null ||
738
+ sqlState.startsWith("08") ||
739
+ sqlState.startsWith("40") ||
740
+ sqlState.startsWith("53") ||
741
+ sqlState === "55P03" ||
742
+ sqlState === "57014" ||
743
+ sqlState === "57P01" ||
744
+ sqlState === "57P02" ||
745
+ sqlState === "57P03" ||
746
+ sqlState.startsWith("58"))),
747
+ );
748
+ const detail: PreClaimFailureDetail = {
749
+ disposition: retryable ? "retryable" : "permanent",
750
+ code:
751
+ persistenceFailure?.details.code ?? (rawTransportFailure ? "db_failure" : "claim_invariant"),
752
+ };
753
+ return ApplicationFailure.create({
754
+ message: PRE_CLAIM_FAILURE_MESSAGE,
755
+ type: PRE_CLAIM_FAILURE_TYPE,
756
+ nonRetryable: true,
757
+ details: [detail],
758
+ });
759
+ }
760
+
707
761
  /**
708
762
  * Resolve which Codex account a turn runs on (multi-account P1): session-pin >
709
763
  * workspace-active. No rotation in P1. The selected id must still be in the
@@ -729,6 +783,12 @@ export function filterUnmaterializedSandboxFileDownloads(
729
783
  return downloads.filter((download) => !materializedFileIds.has(download.fileId));
730
784
  }
731
785
 
786
+ export function sandboxFileMaterializationOutcome(
787
+ failures: readonly SandboxFileDownloadFailure[],
788
+ ): "completed" | "failed" {
789
+ return failures.length === 0 ? "completed" : "failed";
790
+ }
791
+
732
792
  /** Fixed-length one-way tenant correlation for metrics/alerts; never a raw id. */
733
793
  export function codexWorkspaceMetricKey(workspaceId: string): string {
734
794
  return createHash("sha256").update(workspaceId).digest("hex").slice(0, 12);
@@ -1998,6 +2058,33 @@ export function shouldEstablishSandboxForTurn(
1998
2058
  return sandboxOwnershipEnabled && (homeBackend !== "none" || machinePrimary);
1999
2059
  }
2000
2060
 
2061
+ export function sandboxEstablishPolicyDecision(input: {
2062
+ lazyEnabled: boolean;
2063
+ machinePrimary: boolean;
2064
+ sandboxBackend: Settings["sandboxBackend"];
2065
+ hasInitialRunCredentialMaterial: boolean;
2066
+ generatedVideoFileCount: number;
2067
+ hasSignedFileResources: boolean;
2068
+ }): { policy: "eager" | "on-demand"; reason: TurnSandboxEstablishReason } {
2069
+ if (!input.lazyEnabled) return { policy: "eager", reason: "lazy_disabled" };
2070
+ if (input.machinePrimary) return { policy: "eager", reason: "machine_primary" };
2071
+ if (input.sandboxBackend === "none") return { policy: "eager", reason: "backend_none" };
2072
+ if (input.hasInitialRunCredentialMaterial) {
2073
+ return { policy: "eager", reason: "initial_run_credentials" };
2074
+ }
2075
+ if (input.generatedVideoFileCount > 0) {
2076
+ return { policy: "eager", reason: "generated_video_files" };
2077
+ }
2078
+ // Signed file downloads must finish before the first model boundary so any
2079
+ // integrity/download failure is present in the prepared input. Deferring the
2080
+ // box until the first tool call would make that first request advertise a
2081
+ // path whose materialization outcome is not known yet.
2082
+ if (input.hasSignedFileResources) {
2083
+ return { policy: "eager", reason: "signed_file_resources" };
2084
+ }
2085
+ return { policy: "on-demand", reason: "eligible" };
2086
+ }
2087
+
2001
2088
  /**
2002
2089
  * Classify a persisted active-sandbox pointer for TURN-START RECONCILE (issue #341
2003
2090
  * invariant B). Returns the typed reason to RESET the pointer to the session HOME,
@@ -4073,7 +4160,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4073
4160
  // only exists in the RunState blob), never through a swapped trigger.
4074
4161
  let triggerType: string | null = null;
4075
4162
  try {
4076
- const session = await requireSession(db, input.workspaceId, input.sessionId);
4077
4163
  const claim = await claimSessionWorkForAttempt(db, input.workspaceId, {
4078
4164
  sessionId: input.sessionId,
4079
4165
  workflowId: input.workflowId,
@@ -4088,6 +4174,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4088
4174
  }
4089
4175
  const turn = claim.turn;
4090
4176
  turnId = turn.id;
4177
+ // Establish durable attempt ownership before any later read can fail.
4178
+ // Therefore every failure with no turnId came from the one atomic claim
4179
+ // transaction and can be classified without conflating ordinary runtime
4180
+ // or transport failures with admission failures.
4181
+ const session = await requireSession(db, input.workspaceId, input.sessionId);
4091
4182
  executionGeneration = turn.executionGeneration;
4092
4183
  providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
4093
4184
  let installedApiIntegrations: readonly ApiIntegrationRuntime[] = [];
@@ -4410,6 +4501,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4410
4501
  // on a healthy worker.
4411
4502
  throwIfWorkerShuttingDown();
4412
4503
  throwIfTurnCancelled();
4504
+ recordTurnStartupPhase(observability, {
4505
+ phase: "claim_and_policy",
4506
+ provider: turnExecutionPolicy.providerId,
4507
+ backend: turn.sandboxBackend,
4508
+ outcome: "completed",
4509
+ durationSeconds: (performance.now() - activityStarted) / 1_000,
4510
+ });
4511
+ const turnStartSettlementStartedAt = performance.now();
4413
4512
  if (
4414
4513
  !(await settle({
4415
4514
  events: [
@@ -4426,7 +4525,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4426
4525
  ) {
4427
4526
  return claimedResult({ status: "cancelled" });
4428
4527
  }
4528
+ recordTurnStartupPhase(observability, {
4529
+ phase: "turn_start_settlement",
4530
+ provider: turnExecutionPolicy.providerId,
4531
+ backend: turn.sandboxBackend,
4532
+ outcome: "completed",
4533
+ durationSeconds: (performance.now() - turnStartSettlementStartedAt) / 1_000,
4534
+ count: 2,
4535
+ });
4429
4536
  turnStartedPublished = true;
4537
+ const workerPreparationStartedAt = performance.now();
4430
4538
 
4431
4539
  // Multi-account (P1): resolve the effective Codex account for this turn
4432
4540
  // (session-pin > workspace active) and stamp it on the session so the
@@ -4434,80 +4542,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4434
4542
  // when it changed from the prior run's account so the pill flips live.
4435
4543
  // Gated on the codex-billed predicate — non-codex turns never touch this.
4436
4544
  if (isCodexTurn) {
4437
- const sessionCodex = await getSessionCodexState(db, input.workspaceId, input.sessionId);
4438
- const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
4439
- const sessionPinSource = sessionCodex?.pinSource ?? null;
4440
- const selectForTurn = (context: CodexCredentialLeaseSelectionContext) =>
4441
- selectCodexCredentialLeaseForTurn({
4442
- context,
4443
- leasingEnabled: settings.codexCredentialLeasingEnabled,
4444
- sessionId: input.sessionId,
4445
- sessionPinnedCredentialId: sessionPin,
4446
- sessionPinSource,
4447
- sessionLastCredentialId: sessionCodex?.lastCredentialId ?? null,
4448
- now: new Date(),
4449
- });
4545
+ const credentialSelectionStartedAt = performance.now();
4546
+ let credentialSelectionOutcome: "completed" | "failed" = "completed";
4547
+ try {
4548
+ const sessionCodex = await getSessionCodexState(db, input.workspaceId, input.sessionId);
4549
+ const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
4550
+ const sessionPinSource = sessionCodex?.pinSource ?? null;
4551
+ const selectForTurn = (context: CodexCredentialLeaseSelectionContext) =>
4552
+ selectCodexCredentialLeaseForTurn({
4553
+ context,
4554
+ leasingEnabled: settings.codexCredentialLeasingEnabled,
4555
+ sessionId: input.sessionId,
4556
+ sessionPinnedCredentialId: sessionPin,
4557
+ sessionPinSource,
4558
+ sessionLastCredentialId: sessionCodex?.lastCredentialId ?? null,
4559
+ now: new Date(),
4560
+ });
4450
4561
 
4451
- // Rollout/rollback path is intentionally table-inert. With the flag off,
4452
- // old and new workers both use legacy pin > active-pointer selection and
4453
- // neither reads nor writes the additive lease/cursor schema.
4454
- let leased: CodexCredentialLeaseResult<RotationDecision>;
4455
- let leaseAcquisitionStartedAtMs: number | null = null;
4456
- if (settings.codexCredentialLeasingEnabled) {
4457
- leaseAcquisitionStartedAtMs = performance.now();
4458
- leased = await acquireCodexCredentialLease(
4459
- db,
4460
- {
4461
- accountId: input.accountId,
4462
- workspaceId: input.workspaceId,
4463
- turnId,
4464
- holderId: codexLeaseHolderId,
4465
- advanceActivePointer: sessionPin === null,
4466
- },
4467
- selectForTurn,
4468
- );
4469
- } else {
4470
- const [rotation, accounts] = await Promise.all([
4471
- getCodexRotationSettings(db, input.workspaceId),
4472
- listCodexAccountStatuses(db, input.workspaceId),
4473
- ]);
4474
- const leaseAccounts = accounts.map((account) => ({
4475
- ...account,
4476
- activeLeaseCount: 0,
4477
- selectionCount: 0,
4478
- lastSelectedAt: null,
4479
- }));
4480
- const activeCredentialId = rotation?.activeCredentialId ?? null;
4481
- const selected = selectForTurn({
4482
- accounts: leaseAccounts,
4483
- activeCredentialId,
4484
- rotationEnabled: rotation?.rotationEnabled ?? false,
4485
- leaseRotationEnabled: false,
4486
- rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
4487
- existingCredentialId: null,
4488
- policyScope: null,
4489
- unavailableDiagnostics: [],
4490
- });
4491
- leased = {
4492
- ...selected,
4493
- accounts: leaseAccounts,
4494
- activeCredentialId,
4495
- rotationEnabled: rotation?.rotationEnabled ?? false,
4496
- rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
4497
- reused: false,
4498
- holderId: null,
4499
- generation: null,
4500
- leasedUntil: null,
4501
- unavailableDiagnostics: [],
4502
- advanceActivePointer: selected.advanceActivePointer !== false,
4503
- };
4504
- }
4505
- if (leased.decision.kind === "allCapped") {
4506
- // Bounded self-heal of stale usage cache, then ONE new atomic selection.
4507
- await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
4508
- signalCodexCapacityWorkflow,
4509
- wakeSessionWorkflow,
4510
- });
4562
+ // Rollout/rollback path is intentionally table-inert. With the flag off,
4563
+ // old and new workers both use legacy pin > active-pointer selection and
4564
+ // neither reads nor writes the additive lease/cursor schema.
4565
+ let leased: CodexCredentialLeaseResult<RotationDecision>;
4566
+ let leaseAcquisitionStartedAtMs: number | null = null;
4511
4567
  if (settings.codexCredentialLeasingEnabled) {
4512
4568
  leaseAcquisitionStartedAtMs = performance.now();
4513
4569
  leased = await acquireCodexCredentialLease(
@@ -4532,9 +4588,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4532
4588
  selectionCount: 0,
4533
4589
  lastSelectedAt: null,
4534
4590
  }));
4591
+ const activeCredentialId = rotation?.activeCredentialId ?? null;
4535
4592
  const selected = selectForTurn({
4536
4593
  accounts: leaseAccounts,
4537
- activeCredentialId: rotation?.activeCredentialId ?? null,
4594
+ activeCredentialId,
4538
4595
  rotationEnabled: rotation?.rotationEnabled ?? false,
4539
4596
  leaseRotationEnabled: false,
4540
4597
  rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
@@ -4545,7 +4602,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4545
4602
  leased = {
4546
4603
  ...selected,
4547
4604
  accounts: leaseAccounts,
4548
- activeCredentialId: rotation?.activeCredentialId ?? null,
4605
+ activeCredentialId,
4549
4606
  rotationEnabled: rotation?.rotationEnabled ?? false,
4550
4607
  rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
4551
4608
  reused: false,
@@ -4556,212 +4613,334 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4556
4613
  advanceActivePointer: selected.advanceActivePointer !== false,
4557
4614
  };
4558
4615
  }
4559
- }
4560
- const rotationDecision = leased.decision;
4561
- const selectedPinDisposition = classifyCodexPin({
4562
- pinnedCredentialId: sessionPin,
4563
- pinSource: sessionPinSource,
4564
- strategy: leased.rotationStrategy as CodexRotationStrategy,
4565
- rotationEnabled: leased.rotationEnabled,
4566
- });
4567
- // pin policy pin persistence follows the atomic credential allocator selection. The selector
4568
- // already ran exact-turn reuse before policy filtering and vetoed pointer
4569
- // movement for manual/policy homes; this write only records the NEXT turn's
4570
- // policy home (or clears a policy pin whose strategy is no longer active).
4571
- if (
4572
- selectedPinDisposition === "sharded" &&
4573
- leased.credentialId !== null &&
4574
- (sessionPinSource !== "policy" || sessionPin !== leased.credentialId)
4575
- ) {
4576
- const pinMutation = await withSessionCodexCapacityMutation(
4577
- db,
4578
- {
4579
- workspaceId: input.workspaceId,
4580
- reason: "codex_policy_pin_changed",
4581
- },
4582
- async (tx) => {
4583
- const changed = await setSessionCodexPinInTransaction(
4584
- tx,
4585
- input.workspaceId,
4586
- input.sessionId,
4587
- leased.credentialId,
4588
- "policy",
4616
+ if (leased.decision.kind === "allCapped") {
4617
+ // Bounded self-heal of stale usage cache, then ONE new atomic selection.
4618
+ await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
4619
+ signalCodexCapacityWorkflow,
4620
+ wakeSessionWorkflow,
4621
+ });
4622
+ if (settings.codexCredentialLeasingEnabled) {
4623
+ leaseAcquisitionStartedAtMs = performance.now();
4624
+ leased = await acquireCodexCredentialLease(
4625
+ db,
4589
4626
  {
4590
- expected: {
4591
- pinnedCredentialId: sessionPin,
4592
- pinSource: sessionPinSource,
4593
- },
4627
+ accountId: input.accountId,
4628
+ workspaceId: input.workspaceId,
4629
+ turnId,
4630
+ holderId: codexLeaseHolderId,
4631
+ advanceActivePointer: sessionPin === null,
4594
4632
  },
4633
+ selectForTurn,
4595
4634
  );
4596
- return { result: changed, changed };
4597
- },
4598
- );
4599
- await signalCodexCapacityWakeTargets(
4600
- { signalCodexCapacityWorkflow, wakeSessionWorkflow },
4601
- pinMutation.wakeTargets,
4602
- );
4603
- } else if (selectedPinDisposition === "clearStale") {
4604
- const pinMutation = await withSessionCodexCapacityMutation(
4605
- db,
4606
- {
4607
- workspaceId: input.workspaceId,
4608
- reason: "codex_stale_policy_pin_cleared",
4609
- },
4610
- async (tx) => {
4611
- const changed = await setSessionCodexPinInTransaction(
4612
- tx,
4613
- input.workspaceId,
4614
- input.sessionId,
4615
- null,
4616
- "policy",
4617
- {
4618
- expected: {
4619
- pinnedCredentialId: sessionPin,
4620
- pinSource: sessionPinSource,
4635
+ } else {
4636
+ const [rotation, accounts] = await Promise.all([
4637
+ getCodexRotationSettings(db, input.workspaceId),
4638
+ listCodexAccountStatuses(db, input.workspaceId),
4639
+ ]);
4640
+ const leaseAccounts = accounts.map((account) => ({
4641
+ ...account,
4642
+ activeLeaseCount: 0,
4643
+ selectionCount: 0,
4644
+ lastSelectedAt: null,
4645
+ }));
4646
+ const selected = selectForTurn({
4647
+ accounts: leaseAccounts,
4648
+ activeCredentialId: rotation?.activeCredentialId ?? null,
4649
+ rotationEnabled: rotation?.rotationEnabled ?? false,
4650
+ leaseRotationEnabled: false,
4651
+ rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
4652
+ existingCredentialId: null,
4653
+ policyScope: null,
4654
+ unavailableDiagnostics: [],
4655
+ });
4656
+ leased = {
4657
+ ...selected,
4658
+ accounts: leaseAccounts,
4659
+ activeCredentialId: rotation?.activeCredentialId ?? null,
4660
+ rotationEnabled: rotation?.rotationEnabled ?? false,
4661
+ rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
4662
+ reused: false,
4663
+ holderId: null,
4664
+ generation: null,
4665
+ leasedUntil: null,
4666
+ unavailableDiagnostics: [],
4667
+ advanceActivePointer: selected.advanceActivePointer !== false,
4668
+ };
4669
+ }
4670
+ }
4671
+ const rotationDecision = leased.decision;
4672
+ const selectedPinDisposition = classifyCodexPin({
4673
+ pinnedCredentialId: sessionPin,
4674
+ pinSource: sessionPinSource,
4675
+ strategy: leased.rotationStrategy as CodexRotationStrategy,
4676
+ rotationEnabled: leased.rotationEnabled,
4677
+ });
4678
+ // pin policy pin persistence follows the atomic credential allocator selection. The selector
4679
+ // already ran exact-turn reuse before policy filtering and vetoed pointer
4680
+ // movement for manual/policy homes; this write only records the NEXT turn's
4681
+ // policy home (or clears a policy pin whose strategy is no longer active).
4682
+ if (
4683
+ selectedPinDisposition === "sharded" &&
4684
+ leased.credentialId !== null &&
4685
+ (sessionPinSource !== "policy" || sessionPin !== leased.credentialId)
4686
+ ) {
4687
+ const pinMutation = await withSessionCodexCapacityMutation(
4688
+ db,
4689
+ {
4690
+ workspaceId: input.workspaceId,
4691
+ reason: "codex_policy_pin_changed",
4692
+ },
4693
+ async (tx) => {
4694
+ const changed = await setSessionCodexPinInTransaction(
4695
+ tx,
4696
+ input.workspaceId,
4697
+ input.sessionId,
4698
+ leased.credentialId,
4699
+ "policy",
4700
+ {
4701
+ expected: {
4702
+ pinnedCredentialId: sessionPin,
4703
+ pinSource: sessionPinSource,
4704
+ },
4621
4705
  },
4622
- },
4623
- );
4624
- return { result: changed, changed };
4706
+ );
4707
+ return { result: changed, changed };
4708
+ },
4709
+ );
4710
+ await signalCodexCapacityWakeTargets(
4711
+ { signalCodexCapacityWorkflow, wakeSessionWorkflow },
4712
+ pinMutation.wakeTargets,
4713
+ );
4714
+ } else if (selectedPinDisposition === "clearStale") {
4715
+ const pinMutation = await withSessionCodexCapacityMutation(
4716
+ db,
4717
+ {
4718
+ workspaceId: input.workspaceId,
4719
+ reason: "codex_stale_policy_pin_cleared",
4720
+ },
4721
+ async (tx) => {
4722
+ const changed = await setSessionCodexPinInTransaction(
4723
+ tx,
4724
+ input.workspaceId,
4725
+ input.sessionId,
4726
+ null,
4727
+ "policy",
4728
+ {
4729
+ expected: {
4730
+ pinnedCredentialId: sessionPin,
4731
+ pinSource: sessionPinSource,
4732
+ },
4733
+ },
4734
+ );
4735
+ return { result: changed, changed };
4736
+ },
4737
+ );
4738
+ await signalCodexCapacityWakeTargets(
4739
+ { signalCodexCapacityWorkflow, wakeSessionWorkflow },
4740
+ pinMutation.wakeTargets,
4741
+ );
4742
+ }
4743
+ if (
4744
+ !settings.codexCredentialLeasingEnabled &&
4745
+ leased.advanceActivePointer &&
4746
+ sessionPin === null &&
4747
+ rotationDecision.kind === "active" &&
4748
+ rotationDecision.moved
4749
+ ) {
4750
+ await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
4751
+ }
4752
+ effectiveCodexCredentialId = leased.credentialId;
4753
+ codexLeaseGeneration = leased.generation;
4754
+ codexLeaseConfirmedUntilMs =
4755
+ leased.leasedUntil && leaseAcquisitionStartedAtMs !== null
4756
+ ? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS
4757
+ : null;
4758
+ codexLeaseHeld =
4759
+ effectiveCodexCredentialId !== null &&
4760
+ leased.holderId !== null &&
4761
+ leased.generation !== null &&
4762
+ codexLeaseConfirmedUntilMs !== null;
4763
+ if (codexLeaseHeld) startCodexLeaseHeartbeat();
4764
+
4765
+ const actualOutcome = effectiveCodexCredentialId
4766
+ ? "selected"
4767
+ : rotationDecision.kind === "allCapped"
4768
+ ? "waiting"
4769
+ : "none";
4770
+ const actualReason = effectiveCodexCredentialId
4771
+ ? leased.reused
4772
+ ? "lease_reused"
4773
+ : sessionPin === effectiveCodexCredentialId
4774
+ ? "pin"
4775
+ : rotationDecision.kind === "active" && rotationDecision.moved
4776
+ ? "rotation"
4777
+ : "active"
4778
+ : rotationDecision.kind === "allCapped"
4779
+ ? "all_capped"
4780
+ : "none";
4781
+ const fencedInFlight = leased.reused;
4782
+ const shadowResult = await publishCodexFleetShadowDecisionV1({
4783
+ enabled: settings.codexFleetPolicyShadowEnabled,
4784
+ decision: {
4785
+ accounts: leased.accounts,
4786
+ actualCredentialId: effectiveCodexCredentialId,
4787
+ actualOutcome,
4788
+ actualReason,
4789
+ affinityCredentialId: fencedInFlight
4790
+ ? effectiveCodexCredentialId
4791
+ : (sessionPin ?? sessionCodex?.lastCredentialId ?? null),
4792
+ fencedInFlight,
4793
+ nearExhaustionPct: settings.codexRotationNearExhaustionPct,
4794
+ now: new Date(),
4795
+ aliasSeed: randomUUID(),
4625
4796
  },
4626
- );
4627
- await signalCodexCapacityWakeTargets(
4628
- { signalCodexCapacityWorkflow, wakeSessionWorkflow },
4629
- pinMutation.wakeTargets,
4630
- );
4631
- }
4632
- if (
4633
- !settings.codexCredentialLeasingEnabled &&
4634
- leased.advanceActivePointer &&
4635
- sessionPin === null &&
4636
- rotationDecision.kind === "active" &&
4637
- rotationDecision.moved
4638
- ) {
4639
- await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
4640
- }
4641
- effectiveCodexCredentialId = leased.credentialId;
4642
- codexLeaseGeneration = leased.generation;
4643
- codexLeaseConfirmedUntilMs =
4644
- leased.leasedUntil && leaseAcquisitionStartedAtMs !== null
4645
- ? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS
4646
- : null;
4647
- codexLeaseHeld =
4648
- effectiveCodexCredentialId !== null &&
4649
- leased.holderId !== null &&
4650
- leased.generation !== null &&
4651
- codexLeaseConfirmedUntilMs !== null;
4652
- if (codexLeaseHeld) startCodexLeaseHeartbeat();
4653
-
4654
- const actualOutcome = effectiveCodexCredentialId
4655
- ? "selected"
4656
- : rotationDecision.kind === "allCapped"
4657
- ? "waiting"
4658
- : "none";
4659
- const actualReason = effectiveCodexCredentialId
4660
- ? leased.reused
4661
- ? "lease_reused"
4662
- : sessionPin === effectiveCodexCredentialId
4663
- ? "pin"
4664
- : rotationDecision.kind === "active" && rotationDecision.moved
4665
- ? "rotation"
4666
- : "active"
4667
- : rotationDecision.kind === "allCapped"
4668
- ? "all_capped"
4669
- : "none";
4670
- const fencedInFlight = leased.reused;
4671
- const shadowResult = await publishCodexFleetShadowDecisionV1({
4672
- enabled: settings.codexFleetPolicyShadowEnabled,
4673
- decision: {
4674
- accounts: leased.accounts,
4675
- actualCredentialId: effectiveCodexCredentialId,
4676
- actualOutcome,
4677
- actualReason,
4678
- affinityCredentialId: fencedInFlight
4679
- ? effectiveCodexCredentialId
4680
- : (sessionPin ?? sessionCodex?.lastCredentialId ?? null),
4681
- fencedInFlight,
4682
- nearExhaustionPct: settings.codexRotationNearExhaustionPct,
4683
- now: new Date(),
4684
- aliasSeed: randomUUID(),
4685
- },
4686
- publish,
4687
- });
4688
- if (shadowResult.outcome === "published") {
4689
- const shadowPayload = shadowResult.payload;
4690
- observability.incrementCounter({
4691
- name: "opengeni_codex_fleet_shadow_decisions_total",
4692
- help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
4693
- labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload),
4694
- });
4695
- observability.info("Codex adaptive fleet shadow decision", {
4696
- workspaceId: input.workspaceId,
4697
- policyVersion: shadowPayload.replay.policyVersion,
4698
- inputFingerprint: shadowPayload.replay.inputFingerprint,
4699
- decisionFingerprint: shadowPayload.replay.decisionFingerprint,
4700
- actualOutcome: shadowPayload.actual.outcome,
4701
- shadowOutcome: shadowPayload.replay.decision.outcome,
4702
- comparison: shadowPayload.comparison,
4703
- candidateCount: shadowPayload.replay.input.candidates.length,
4704
- truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
4705
- payloadBytes: shadowResult.payloadBytes,
4797
+ publish,
4706
4798
  });
4707
- } else if (shadowResult.outcome === "failed") {
4708
- // Shadow observability is explicitly non-authoritative. A malformed
4709
- // snapshot or event-write fault must never change the authoritative lease,
4710
- // capacity wait, failover, or the account serving this fenced turn.
4711
- observability.incrementCounter({
4712
- name: "opengeni_codex_fleet_shadow_errors_total",
4713
- help: "Shadow decision build/publication failures.",
4714
- labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
4715
- });
4716
- observability.warn("Codex adaptive fleet shadow decision failed open", {
4717
- stage: shadowResult.stage,
4718
- reason: shadowResult.reason,
4719
- errorClass: "CodexFleetShadowOperationError",
4720
- errorCode: "codex_fleet_shadow_failed",
4721
- origin: "worker",
4722
- payloadBytes: shadowResult.payloadBytes,
4723
- });
4724
- }
4799
+ if (shadowResult.outcome === "published") {
4800
+ const shadowPayload = shadowResult.payload;
4801
+ observability.incrementCounter({
4802
+ name: "opengeni_codex_fleet_shadow_decisions_total",
4803
+ help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
4804
+ labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload),
4805
+ });
4806
+ observability.info("Codex adaptive fleet shadow decision", {
4807
+ workspaceId: input.workspaceId,
4808
+ policyVersion: shadowPayload.replay.policyVersion,
4809
+ inputFingerprint: shadowPayload.replay.inputFingerprint,
4810
+ decisionFingerprint: shadowPayload.replay.decisionFingerprint,
4811
+ actualOutcome: shadowPayload.actual.outcome,
4812
+ shadowOutcome: shadowPayload.replay.decision.outcome,
4813
+ comparison: shadowPayload.comparison,
4814
+ candidateCount: shadowPayload.replay.input.candidates.length,
4815
+ truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
4816
+ payloadBytes: shadowResult.payloadBytes,
4817
+ });
4818
+ } else if (shadowResult.outcome === "failed") {
4819
+ // Shadow observability is explicitly non-authoritative. A malformed
4820
+ // snapshot or event-write fault must never change the authoritative lease,
4821
+ // capacity wait, failover, or the account serving this fenced turn.
4822
+ observability.incrementCounter({
4823
+ name: "opengeni_codex_fleet_shadow_errors_total",
4824
+ help: "Shadow decision build/publication failures.",
4825
+ labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
4826
+ });
4827
+ observability.warn("Codex adaptive fleet shadow decision failed open", {
4828
+ stage: shadowResult.stage,
4829
+ reason: shadowResult.reason,
4830
+ errorClass: "CodexFleetShadowOperationError",
4831
+ errorCode: "codex_fleet_shadow_failed",
4832
+ origin: "worker",
4833
+ payloadBytes: shadowResult.payloadBytes,
4834
+ });
4835
+ }
4725
4836
 
4726
- const eligibleCount = leased.accounts.filter((account) =>
4727
- isCodexCredentialEligible(account, new Date()),
4728
- ).length;
4729
- const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
4730
- observability.incrementCounter({
4731
- name: "opengeni_codex_pool_observations_total",
4732
- help: "Observed eligible Codex pool depth buckets at turn selection.",
4733
- labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
4734
- });
4735
- if (eligibleCount <= 1) {
4837
+ const eligibleCount = leased.accounts.filter((account) =>
4838
+ isCodexCredentialEligible(account, new Date()),
4839
+ ).length;
4840
+ const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
4736
4841
  observability.incrementCounter({
4737
- name: "opengeni_codex_pool_low_total",
4738
- help: "Alert signal emitted when the eligible Codex pool is zero or one.",
4842
+ name: "opengeni_codex_pool_observations_total",
4843
+ help: "Observed eligible Codex pool depth buckets at turn selection.",
4739
4844
  labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
4740
4845
  });
4741
- observability.warn("Codex eligible credential pool is low", {
4742
- workspaceId: input.workspaceId,
4743
- eligibleCount,
4744
- connectedCount: leased.accounts.length,
4745
- depth: poolDepth,
4746
- });
4747
- }
4846
+ if (eligibleCount <= 1) {
4847
+ observability.incrementCounter({
4848
+ name: "opengeni_codex_pool_low_total",
4849
+ help: "Alert signal emitted when the eligible Codex pool is zero or one.",
4850
+ labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
4851
+ });
4852
+ observability.warn("Codex eligible credential pool is low", {
4853
+ workspaceId: input.workspaceId,
4854
+ eligibleCount,
4855
+ connectedCount: leased.accounts.length,
4856
+ depth: poolDepth,
4857
+ });
4858
+ }
4748
4859
 
4749
- if (
4750
- effectiveCodexCredentialId === null &&
4751
- leased.accounts.length > 0 &&
4752
- leased.accounts.every((account) => !account.allocatorEnabled) &&
4753
- turnId
4754
- ) {
4755
- if (turn.source === "compaction") {
4860
+ if (
4861
+ effectiveCodexCredentialId === null &&
4862
+ leased.accounts.length > 0 &&
4863
+ leased.accounts.every((account) => !account.allocatorEnabled) &&
4864
+ turnId
4865
+ ) {
4866
+ if (turn.source === "compaction") {
4867
+ if (
4868
+ !(await settle!({
4869
+ events: [
4870
+ {
4871
+ type: "turn.cancelled",
4872
+ payload: {
4873
+ maintenance: "context_compaction",
4874
+ reason: "codex_allocator_disabled",
4875
+ requestPreserved: true,
4876
+ },
4877
+ },
4878
+ {
4879
+ type: "session.status.changed",
4880
+ payload: { status: "idle" },
4881
+ },
4882
+ ],
4883
+ turnStatus: "cancelled",
4884
+ sessionStatus: "idle",
4885
+ activeTurnId: null,
4886
+ }))
4887
+ ) {
4888
+ return claimedResult({ status: "cancelled" });
4889
+ }
4890
+ turnMetricOutcome = "cancelled";
4891
+ activityStatus = "idle";
4892
+ return claimedResult({ status: "idle", deferredUntilWake: true });
4893
+ }
4894
+ const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
4895
+ () => null,
4896
+ );
4897
+ const activeGoal = goal?.status === "active" ? goal : null;
4898
+ const armed = await armCodexCapacityWait(db, {
4899
+ accountId: input.accountId,
4900
+ workspaceId: input.workspaceId,
4901
+ sessionId: input.sessionId,
4902
+ turnId,
4903
+ attemptId: input.attemptId,
4904
+ workflowId: input.workflowId,
4905
+ goalId: activeGoal?.id ?? null,
4906
+ goalVersion: activeGoal?.version ?? null,
4907
+ earliestResetAt: null,
4908
+ resetKind: "bounded_refresh",
4909
+ failurePayload: {
4910
+ error: "All connected Codex subscriptions are disabled for new allocations.",
4911
+ code: "codex_allocator_disabled",
4912
+ detail: "waiting for a credential to be re-enabled, reconnected, or added",
4913
+ },
4914
+ });
4915
+ if (armed.action === "waiting") {
4916
+ await publishDurableSessionEvents(
4917
+ bus,
4918
+ input.workspaceId,
4919
+ input.sessionId,
4920
+ armed.events,
4921
+ );
4922
+ turnMetricOutcome = "recovering";
4923
+ activityStatus = "waiting_capacity";
4924
+ return claimedResult({
4925
+ status: "waiting_capacity",
4926
+ capacityWait: {
4927
+ waiterId: armed.waiter.id,
4928
+ generation: armed.waiter.generation,
4929
+ nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
4930
+ wakeRevision: armed.waiter.wakeRevision,
4931
+ },
4932
+ });
4933
+ }
4756
4934
  if (
4757
4935
  !(await settle!({
4758
4936
  events: [
4759
4937
  {
4760
- type: "turn.cancelled",
4938
+ type: "turn.failed",
4761
4939
  payload: {
4762
- maintenance: "context_compaction",
4763
- reason: "codex_allocator_disabled",
4764
- requestPreserved: true,
4940
+ error: "All connected Codex subscriptions are disabled for new allocations.",
4941
+ code: "codex_allocator_disabled",
4942
+ retryable: false,
4943
+ recovery: "user_message",
4765
4944
  },
4766
4945
  },
4767
4946
  {
@@ -4769,94 +4948,119 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4769
4948
  payload: { status: "idle" },
4770
4949
  },
4771
4950
  ],
4772
- turnStatus: "cancelled",
4951
+ turnStatus: "failed",
4773
4952
  sessionStatus: "idle",
4774
4953
  activeTurnId: null,
4775
4954
  }))
4776
4955
  ) {
4777
4956
  return claimedResult({ status: "cancelled" });
4778
4957
  }
4779
- turnMetricOutcome = "cancelled";
4958
+ turnMetricOutcome = "failed";
4780
4959
  activityStatus = "idle";
4781
- return claimedResult({ status: "idle", deferredUntilWake: true });
4960
+ return claimedResult({ status: "idle" });
4782
4961
  }
4783
- const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
4784
- () => null,
4785
- );
4786
- const activeGoal = goal?.status === "active" ? goal : null;
4787
- const armed = await armCodexCapacityWait(db, {
4788
- accountId: input.accountId,
4789
- workspaceId: input.workspaceId,
4790
- sessionId: input.sessionId,
4791
- turnId,
4792
- attemptId: input.attemptId,
4793
- workflowId: input.workflowId,
4794
- goalId: activeGoal?.id ?? null,
4795
- goalVersion: activeGoal?.version ?? null,
4796
- earliestResetAt: null,
4797
- resetKind: "bounded_refresh",
4798
- failurePayload: {
4799
- error: "All connected Codex subscriptions are disabled for new allocations.",
4800
- code: "codex_allocator_disabled",
4801
- detail: "waiting for a credential to be re-enabled, reconnected, or added",
4802
- },
4803
- });
4804
- if (armed.action === "waiting") {
4805
- await publishDurableSessionEvents(
4806
- bus,
4807
- input.workspaceId,
4808
- input.sessionId,
4809
- armed.events,
4962
+
4963
+ if (rotationDecision.kind === "allCapped" && turnId) {
4964
+ if (turn.source === "compaction") {
4965
+ if (
4966
+ !(await settle!({
4967
+ events: [
4968
+ {
4969
+ type: "turn.cancelled",
4970
+ payload: {
4971
+ maintenance: "context_compaction",
4972
+ reason: "codex_capacity_unavailable",
4973
+ requestPreserved: true,
4974
+ },
4975
+ },
4976
+ {
4977
+ type: "session.status.changed",
4978
+ payload: { status: "idle" },
4979
+ },
4980
+ ],
4981
+ turnStatus: "cancelled",
4982
+ sessionStatus: "idle",
4983
+ activeTurnId: null,
4984
+ }))
4985
+ ) {
4986
+ return claimedResult({ status: "cancelled" });
4987
+ }
4988
+ turnMetricOutcome = "cancelled";
4989
+ activityStatus = "idle";
4990
+ return claimedResult({ status: "idle", deferredUntilWake: true });
4991
+ }
4992
+ // Every eligible account is capped/cooling (and a usage refresh did NOT
4993
+ // surface a reset): idle the turn AT THE BOUNDARY (no wasted model/sandbox
4994
+ // build) until the EARLIEST reset across all accounts — the multi-account
4995
+ // generalization of #143's single-account idle-until-reset. No saveRunState:
4996
+ // no model ran, nothing to freeze.
4997
+ const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
4998
+ () => null,
4810
4999
  );
4811
- turnMetricOutcome = "recovering";
4812
- activityStatus = "waiting_capacity";
4813
- return claimedResult({
4814
- status: "waiting_capacity",
4815
- capacityWait: {
4816
- waiterId: armed.waiter.id,
4817
- generation: armed.waiter.generation,
4818
- nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
4819
- wakeRevision: armed.waiter.wakeRevision,
4820
- },
5000
+ const goalActive = Boolean(goal && goal.status === "active");
5001
+ // BOUNDED + POSITIVE: clamp to [MIN_IDLE_MS, max] so a null/elapsed/unknown
5002
+ // reset can never yield a 0 (which session.ts would treat as "continue now",
5003
+ // re-entering this path in a tight CPU/DB-hammering loop).
5004
+ const resumeMs = computeIdleDelayMs(
5005
+ rotationDecision.earliestResetAt,
5006
+ new Date(),
5007
+ CODEX_USAGE_LIMIT_MAX_RESUME_MS,
5008
+ );
5009
+ const failurePayload = codexUsageLimitFailurePayload(
5010
+ { resetsInSeconds: Math.ceil(resumeMs / 1000) },
5011
+ "all connected Codex subscriptions are rate-limited",
5012
+ { allAccounts: true },
5013
+ );
5014
+ const authoritativeResetAt = authoritativeCodexCapacityResetAt(
5015
+ leased.accounts,
5016
+ new Date(),
5017
+ );
5018
+ const armed = await armCodexCapacityWait(db, {
5019
+ accountId: input.accountId,
5020
+ workspaceId: input.workspaceId,
5021
+ sessionId: input.sessionId,
5022
+ turnId,
5023
+ attemptId: input.attemptId,
5024
+ workflowId: input.workflowId,
5025
+ goalId: goalActive && goal ? goal.id : null,
5026
+ goalVersion: goalActive && goal ? goal.version : null,
5027
+ earliestResetAt: authoritativeResetAt,
5028
+ resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
5029
+ failurePayload,
4821
5030
  });
4822
- }
4823
- if (
4824
- !(await settle!({
4825
- events: [
4826
- {
4827
- type: "turn.failed",
4828
- payload: {
4829
- error: "All connected Codex subscriptions are disabled for new allocations.",
4830
- code: "codex_allocator_disabled",
4831
- retryable: false,
4832
- recovery: "user_message",
4833
- },
5031
+ if (armed.action === "waiting") {
5032
+ await publishDurableSessionEvents(
5033
+ bus,
5034
+ input.workspaceId,
5035
+ input.sessionId,
5036
+ armed.events,
5037
+ );
5038
+ turnMetricOutcome = "recovering";
5039
+ activityStatus = "waiting_capacity";
5040
+ return claimedResult({
5041
+ status: "waiting_capacity",
5042
+ capacityWait: {
5043
+ waiterId: armed.waiter.id,
5044
+ generation: armed.waiter.generation,
5045
+ nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
5046
+ wakeRevision: armed.waiter.wakeRevision,
4834
5047
  },
4835
- { type: "session.status.changed", payload: { status: "idle" } },
4836
- ],
4837
- turnStatus: "failed",
4838
- sessionStatus: "idle",
4839
- activeTurnId: null,
4840
- }))
4841
- ) {
4842
- return claimedResult({ status: "cancelled" });
4843
- }
4844
- turnMetricOutcome = "failed";
4845
- activityStatus = "idle";
4846
- return claimedResult({ status: "idle" });
4847
- }
4848
-
4849
- if (rotationDecision.kind === "allCapped" && turnId) {
4850
- if (turn.source === "compaction") {
5048
+ });
5049
+ }
4851
5050
  if (
4852
5051
  !(await settle!({
4853
5052
  events: [
5053
+ // `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
5054
+ // rotation-wait state as the reactive all-capped path, so it must freeze
5055
+ // autoContinuations identically (evaluateGoalContinuation reads this marker)
5056
+ // — a goal waiting out a long reset must not burn its continuation budget on
5057
+ // the proactive path while the reactive path spares it.
4854
5058
  {
4855
- type: "turn.cancelled",
5059
+ type: "turn.failed",
4856
5060
  payload: {
4857
- maintenance: "context_compaction",
4858
- reason: "codex_capacity_unavailable",
4859
- requestPreserved: true,
5061
+ ...failurePayload,
5062
+ recovery: goalActive ? "goal_continuation" : "user_message",
5063
+ rotated: true,
4860
5064
  },
4861
5065
  },
4862
5066
  {
@@ -4864,165 +5068,90 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4864
5068
  payload: { status: "idle" },
4865
5069
  },
4866
5070
  ],
4867
- turnStatus: "cancelled",
5071
+ turnStatus: "failed",
4868
5072
  sessionStatus: "idle",
4869
5073
  activeTurnId: null,
4870
5074
  }))
4871
5075
  ) {
4872
5076
  return claimedResult({ status: "cancelled" });
4873
5077
  }
4874
- turnMetricOutcome = "cancelled";
5078
+ turnMetricOutcome = "failed";
4875
5079
  activityStatus = "idle";
4876
- return claimedResult({ status: "idle", deferredUntilWake: true });
5080
+ // idleUntilReset marks this a MANDATORY hold: session.ts must wait the full
5081
+ // resumeMs even if a future change made it 0 — never a tight re-dispatch.
5082
+ return claimedResult(
5083
+ goalActive
5084
+ ? {
5085
+ status: "idle",
5086
+ continueDelayMs: resumeMs,
5087
+ idleUntilReset: true,
5088
+ }
5089
+ : { status: "idle" },
5090
+ );
4877
5091
  }
4878
- // Every eligible account is capped/cooling (and a usage refresh did NOT
4879
- // surface a reset): idle the turn AT THE BOUNDARY (no wasted model/sandbox
4880
- // build) until the EARLIEST reset across all accounts — the multi-account
4881
- // generalization of #143's single-account idle-until-reset. No saveRunState:
4882
- // no model ran, nothing to freeze.
4883
- const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
4884
- () => null,
4885
- );
4886
- const goalActive = Boolean(goal && goal.status === "active");
4887
- // BOUNDED + POSITIVE: clamp to [MIN_IDLE_MS, max] so a null/elapsed/unknown
4888
- // reset can never yield a 0 (which session.ts would treat as "continue now",
4889
- // re-entering this path in a tight CPU/DB-hammering loop).
4890
- const resumeMs = computeIdleDelayMs(
4891
- rotationDecision.earliestResetAt,
4892
- new Date(),
4893
- CODEX_USAGE_LIMIT_MAX_RESUME_MS,
4894
- );
4895
- const failurePayload = codexUsageLimitFailurePayload(
4896
- { resetsInSeconds: Math.ceil(resumeMs / 1000) },
4897
- "all connected Codex subscriptions are rate-limited",
4898
- { allAccounts: true },
4899
- );
4900
- const authoritativeResetAt = authoritativeCodexCapacityResetAt(
4901
- leased.accounts,
4902
- new Date(),
4903
- );
4904
- const armed = await armCodexCapacityWait(db, {
4905
- accountId: input.accountId,
4906
- workspaceId: input.workspaceId,
4907
- sessionId: input.sessionId,
4908
- turnId,
4909
- attemptId: input.attemptId,
4910
- workflowId: input.workflowId,
4911
- goalId: goalActive && goal ? goal.id : null,
4912
- goalVersion: goalActive && goal ? goal.version : null,
4913
- earliestResetAt: authoritativeResetAt,
4914
- resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
4915
- failurePayload,
4916
- });
4917
- if (armed.action === "waiting") {
4918
- await publishDurableSessionEvents(
4919
- bus,
5092
+ if (effectiveCodexCredentialId) {
5093
+ const priorAccountId = sessionCodex?.lastCredentialId ?? null;
5094
+ await recordSessionActiveCodexCredential(
5095
+ db,
4920
5096
  input.workspaceId,
4921
5097
  input.sessionId,
4922
- armed.events,
5098
+ effectiveCodexCredentialId,
4923
5099
  );
4924
- turnMetricOutcome = "recovering";
4925
- activityStatus = "waiting_capacity";
4926
- return claimedResult({
4927
- status: "waiting_capacity",
4928
- capacityWait: {
4929
- waiterId: armed.waiter.id,
4930
- generation: armed.waiter.generation,
4931
- nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
4932
- wakeRevision: armed.waiter.wakeRevision,
4933
- },
4934
- });
4935
- }
4936
- if (
4937
- !(await settle!({
4938
- events: [
4939
- // `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
4940
- // rotation-wait state as the reactive all-capped path, so it must freeze
4941
- // autoContinuations identically (evaluateGoalContinuation reads this marker)
4942
- // — a goal waiting out a long reset must not burn its continuation budget on
4943
- // the proactive path while the reactive path spares it.
5100
+ if (priorAccountId !== effectiveCodexCredentialId) {
5101
+ const rotated = rotationDecision.kind === "active" && rotationDecision.moved;
5102
+ await publish([
4944
5103
  {
4945
- type: "turn.failed",
5104
+ type: "codex.account.switched",
4946
5105
  payload: {
4947
- ...failurePayload,
4948
- recovery: goalActive ? "goal_continuation" : "user_message",
4949
- rotated: true,
5106
+ fromAccountId: priorAccountId,
5107
+ toAccountId: effectiveCodexCredentialId,
5108
+ reason: rotated ? "rotation" : "manual",
4950
5109
  },
4951
5110
  },
4952
- { type: "session.status.changed", payload: { status: "idle" } },
4953
- ],
4954
- turnStatus: "failed",
4955
- sessionStatus: "idle",
4956
- activeTurnId: null,
4957
- }))
4958
- ) {
4959
- return claimedResult({ status: "cancelled" });
4960
- }
4961
- turnMetricOutcome = "failed";
4962
- activityStatus = "idle";
4963
- // idleUntilReset marks this a MANDATORY hold: session.ts must wait the full
4964
- // resumeMs even if a future change made it 0 — never a tight re-dispatch.
4965
- return claimedResult(
4966
- goalActive
4967
- ? {
4968
- status: "idle",
4969
- continueDelayMs: resumeMs,
4970
- idleUntilReset: true,
4971
- }
4972
- : { status: "idle" },
4973
- );
4974
- }
4975
- if (effectiveCodexCredentialId) {
4976
- const priorAccountId = sessionCodex?.lastCredentialId ?? null;
4977
- await recordSessionActiveCodexCredential(
4978
- db,
4979
- input.workspaceId,
4980
- input.sessionId,
4981
- effectiveCodexCredentialId,
4982
- );
4983
- if (priorAccountId !== effectiveCodexCredentialId) {
4984
- const rotated = rotationDecision.kind === "active" && rotationDecision.moved;
5111
+ ]);
5112
+ }
5113
+
5114
+ const selectionReason = leased.reused
5115
+ ? "lease_reused"
5116
+ : sessionPin === effectiveCodexCredentialId
5117
+ ? "pin"
5118
+ : rotationDecision.kind === "active" && rotationDecision.moved
5119
+ ? "rotation"
5120
+ : "active";
5121
+ observability.incrementCounter({
5122
+ name: "opengeni_codex_credential_selections_total",
5123
+ help: "Codex credential selections by strategy and reason.",
5124
+ labels: {
5125
+ workspace_key: codexWorkspaceKey,
5126
+ strategy: leased.rotationStrategy,
5127
+ reason: selectionReason,
5128
+ },
5129
+ });
4985
5130
  await publish([
4986
5131
  {
4987
- type: "codex.account.switched",
5132
+ type: "codex.credential.selected",
4988
5133
  payload: {
4989
- fromAccountId: priorAccountId,
4990
- toAccountId: effectiveCodexCredentialId,
4991
- reason: rotated ? "rotation" : "manual",
5134
+ credentialId: effectiveCodexCredentialId,
5135
+ strategy: leased.rotationStrategy,
5136
+ reason: selectionReason,
5137
+ eligibleCount,
5138
+ connectedCount: leased.accounts.length,
5139
+ reused: leased.reused,
4992
5140
  },
4993
5141
  },
4994
5142
  ]);
4995
5143
  }
4996
-
4997
- const selectionReason = leased.reused
4998
- ? "lease_reused"
4999
- : sessionPin === effectiveCodexCredentialId
5000
- ? "pin"
5001
- : rotationDecision.kind === "active" && rotationDecision.moved
5002
- ? "rotation"
5003
- : "active";
5004
- observability.incrementCounter({
5005
- name: "opengeni_codex_credential_selections_total",
5006
- help: "Codex credential selections by strategy and reason.",
5007
- labels: {
5008
- workspace_key: codexWorkspaceKey,
5009
- strategy: leased.rotationStrategy,
5010
- reason: selectionReason,
5011
- },
5144
+ } catch (error) {
5145
+ credentialSelectionOutcome = "failed";
5146
+ throw error;
5147
+ } finally {
5148
+ recordTurnStartupPhase(observability, {
5149
+ phase: "credential_selection",
5150
+ provider: "codex-subscription",
5151
+ backend: turn.sandboxBackend,
5152
+ outcome: credentialSelectionOutcome,
5153
+ durationSeconds: (performance.now() - credentialSelectionStartedAt) / 1_000,
5012
5154
  });
5013
- await publish([
5014
- {
5015
- type: "codex.credential.selected",
5016
- payload: {
5017
- credentialId: effectiveCodexCredentialId,
5018
- strategy: leased.rotationStrategy,
5019
- reason: selectionReason,
5020
- eligibleCount,
5021
- connectedCount: leased.accounts.length,
5022
- reused: leased.reused,
5023
- },
5024
- },
5025
- ]);
5026
5155
  }
5027
5156
  }
5028
5157
 
@@ -5232,6 +5361,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5232
5361
  });
5233
5362
  }
5234
5363
 
5364
+ const runtimePreparationStartedAt = performance.now();
5365
+
5235
5366
  // Pack-scoped runtime: enabled packs may declare the sandbox image this
5236
5367
  // workspace's sessions run in and skills for the sandbox skill index.
5237
5368
  // Resolved after turn.started so a composition conflict (two enabled
@@ -5469,6 +5600,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5469
5600
  // call on the same codex client) and the main run; otherwise the summarizer
5470
5601
  // would hit the codex backend unauthenticated.
5471
5602
  let codexModelRequestSequence = 0;
5603
+ let firstModelRequestPreparationStartedAt: number | null = null;
5604
+ let firstModelRequestPreparationRecorded = false;
5605
+ let firstModelRequestAuditRecorded = false;
5606
+ let firstModelRequestCheckpointAt: number | null = null;
5607
+ const firstModelRequestCheckpoints = new Set<string>();
5472
5608
  const codexContext: CodexRequestContext | null =
5473
5609
  resolvedModel?.provider.kind === "codex-subscription"
5474
5610
  ? ((): CodexRequestContext => {
@@ -5499,10 +5635,51 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5499
5635
  onUsageHeaders: (snapshot) => {
5500
5636
  latestCodexUsage = snapshot;
5501
5637
  }, // latest wins; flushed once in finally
5638
+ onRequestPreparationDiagnostic: (phase) => {
5639
+ if (
5640
+ firstModelRequestCheckpointAt === null ||
5641
+ firstModelRequestCheckpoints.has(phase)
5642
+ ) {
5643
+ return;
5644
+ }
5645
+ const now = performance.now();
5646
+ const metricPhase =
5647
+ phase === "transport_entry"
5648
+ ? "model_sdk_serialization"
5649
+ : phase === "credential_ready"
5650
+ ? "model_credential_resolution"
5651
+ : "model_wire_normalization";
5652
+ firstModelRequestCheckpoints.add(phase);
5653
+ recordTurnStartupPhase(observability, {
5654
+ phase: metricPhase,
5655
+ provider: turnExecutionPolicy.providerId,
5656
+ backend: activeSandboxBackend ?? groupBoxBackend,
5657
+ outcome: "completed",
5658
+ durationSeconds: (now - firstModelRequestCheckpointAt) / 1_000,
5659
+ count: turnTools.length,
5660
+ });
5661
+ firstModelRequestCheckpointAt = now;
5662
+ },
5502
5663
  onRequestOpaqueArtifacts: ({ fingerprints }) => {
5503
5664
  lastCodexRequestOpaqueArtifacts = fingerprints;
5504
5665
  },
5505
5666
  onModelRequestDiagnostic: (event) => {
5667
+ if (
5668
+ event.phase === "started" &&
5669
+ firstModelRequestPreparationStartedAt !== null &&
5670
+ !firstModelRequestPreparationRecorded
5671
+ ) {
5672
+ firstModelRequestPreparationRecorded = true;
5673
+ recordTurnStartupPhase(observability, {
5674
+ phase: "model_request_preparation",
5675
+ provider: turnExecutionPolicy.providerId,
5676
+ backend: activeSandboxBackend ?? groupBoxBackend,
5677
+ outcome: "completed",
5678
+ durationSeconds:
5679
+ (performance.now() - firstModelRequestPreparationStartedAt) / 1_000,
5680
+ count: turnTools.length,
5681
+ });
5682
+ }
5506
5683
  const phase =
5507
5684
  event.phase === "headers"
5508
5685
  ? "headers"
@@ -5532,19 +5709,42 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5532
5709
  if (!publish || !turnId) {
5533
5710
  throw new Error("Codex model request started before the turn event producer");
5534
5711
  }
5535
- await publish([
5536
- {
5537
- type: "agent.model.request",
5538
- payload: {
5539
- ...event,
5540
- provider: "codex-subscription",
5541
- turnId,
5542
- attemptId: input.attemptId,
5543
- dispatchId,
5544
- executionGeneration,
5712
+ const shouldRecordStartedAudit =
5713
+ event.phase === "started" && !firstModelRequestAuditRecorded;
5714
+ if (shouldRecordStartedAudit) {
5715
+ firstModelRequestAuditRecorded = true;
5716
+ }
5717
+ const auditStartedAt = shouldRecordStartedAudit ? performance.now() : null;
5718
+ let auditOutcome: "completed" | "failed" = "completed";
5719
+ try {
5720
+ await publish([
5721
+ {
5722
+ type: "agent.model.request",
5723
+ payload: {
5724
+ ...event,
5725
+ provider: "codex-subscription",
5726
+ turnId,
5727
+ attemptId: input.attemptId,
5728
+ dispatchId,
5729
+ executionGeneration,
5730
+ },
5545
5731
  },
5546
- },
5547
- ]);
5732
+ ]);
5733
+ } catch (error) {
5734
+ auditOutcome = "failed";
5735
+ throw error;
5736
+ } finally {
5737
+ if (auditStartedAt !== null) {
5738
+ recordTurnStartupPhase(observability, {
5739
+ phase: "model_request_audit",
5740
+ provider: turnExecutionPolicy.providerId,
5741
+ backend: activeSandboxBackend ?? groupBoxBackend,
5742
+ outcome: auditOutcome,
5743
+ durationSeconds: (performance.now() - auditStartedAt) / 1_000,
5744
+ count: turnTools.length,
5745
+ });
5746
+ }
5747
+ }
5548
5748
  },
5549
5749
  };
5550
5750
  })()
@@ -6085,17 +6285,27 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6085
6285
  cancellationSignal,
6086
6286
  undefined,
6087
6287
  );
6088
- const establishPolicy: "eager" | "on-demand" =
6089
- lazyProvisionEnabled(settings) &&
6090
- !machinePrimary &&
6091
- runSettings.sandboxBackend !== "none" &&
6288
+ const establishDecision = sandboxEstablishPolicyDecision({
6289
+ lazyEnabled: lazyProvisionEnabled(settings),
6290
+ machinePrimary,
6291
+ sandboxBackend: runSettings.sandboxBackend,
6092
6292
  // Resolved run credentials must be written to one exact leased sandbox
6093
6293
  // before agent execution. A warm active pointer can bypass the lazy
6094
6294
  // provisioner entirely, which previously skipped materialization.
6095
- !initialRunCredentialMaterial &&
6096
- requiredGeneratedVideoFiles.length === 0
6097
- ? "on-demand"
6098
- : "eager";
6295
+ hasInitialRunCredentialMaterial: initialRunCredentialMaterial !== null,
6296
+ generatedVideoFileCount: requiredGeneratedVideoFiles.length,
6297
+ hasSignedFileResources:
6298
+ requiresSignedFileResourceDownloads(
6299
+ runSettings,
6300
+ activeSandboxBackend ?? groupBoxBackend,
6301
+ ) && turnResources.some((resource) => resource.kind === "file"),
6302
+ });
6303
+ const establishPolicy = establishDecision.policy;
6304
+ recordTurnSandboxEstablishPolicy(observability, {
6305
+ policy: establishDecision.policy,
6306
+ reason: establishDecision.reason,
6307
+ backend: machinePrimary ? "selfhosted" : groupBoxBackend,
6308
+ });
6099
6309
  // Computed exactly ONCE per turn and reused for BOTH the box manifest
6100
6310
  // (resumeBoxForTurn -> establishSandboxSessionFromEnvelope, below) AND the
6101
6311
  // agent (runtime.buildAgent, below). sandboxEnvironmentForRun mints a FRESH
@@ -6568,6 +6778,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6568
6778
  // applies the agent's manifest to this provided session and throws on ANY
6569
6779
  // variableSet delta (validateNoEnvironmentDelta). Passing sandboxEnvironment
6570
6780
  // here makes current==target so the delta is empty.
6781
+ recordTurnStartupPhase(observability, {
6782
+ phase: "runtime_preparation",
6783
+ provider: turnExecutionPolicy.providerId,
6784
+ backend: activeSandboxBackend ?? groupBoxBackend,
6785
+ outcome: "completed",
6786
+ durationSeconds: (performance.now() - runtimePreparationStartedAt) / 1_000,
6787
+ });
6571
6788
  if (
6572
6789
  shouldEstablishSandboxForTurn(
6573
6790
  settings.sandboxOwnershipEnabled,
@@ -6575,242 +6792,275 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6575
6792
  machinePrimary,
6576
6793
  )
6577
6794
  ) {
6578
- const managedOwnership = managedSandboxOwnershipForTurn(
6579
- machinePrimary,
6580
- input.attemptId,
6581
- session.sandboxGroupId,
6582
- );
6583
- sandboxHolderId = managedOwnership?.holderId ?? null;
6584
- sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
6585
- // STAGE D honest-label guard: a machine-home session carries
6586
- // turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
6587
- // when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
6588
- // + enrollmentId). When it is NOT primary — the agent swapped back to the
6589
- // group box (sandbox_swap 'session'/'default'/groupId clears the pointer) or
6590
- // selfhosted routing is flag-OFF (the pointer is ignored) — the else-branch
6591
- // must resume a REAL cloud group box, not a "selfhosted" one: the registry
6592
- // SelfhostedSandboxClient has no bound agentId and throws. Fall the group-box
6593
- // backend back to the deployment default cloud backend so swap-away / flag-off
6594
- // degrade to a genuine cloud box exactly like today (home=modal did).
6595
- if (machinePrimary) {
6596
- // STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
6597
- // establish OR lease the managed home box. The active-sandbox pointer is
6598
- // the machine route's own epoch fence; the managed-home lease separately
6599
- // owns cloud recovery, snapshots, billing, and reaping. Mixing those two
6600
- // ownership domains lets an unrelated degraded cloud archive block a
6601
- // healthy explicitly selected machine.
6602
- // Whether the machine's latest Hello advertised the op-stream engine
6603
- // (refreshed on every connect). Read only when the server flag is on —
6604
- // one indexed lookup, and the flag off keeps this path byte-identical.
6605
- const machineOpStream =
6606
- settings.agentOpStreamEnabled === true
6607
- ? (await getEnrollment(db, input.workspaceId, activeSandboxRecord!.enrollmentId!))
6608
- ?.opStream === true
6609
- : false;
6610
- const established = await establishSelfhostedTurnSession(
6611
- {
6612
- db,
6613
- settings,
6614
- bus,
6615
- onOp: machineOpObserver.observer,
6616
- onSandboxOperation: sandboxOperationObserver,
6617
- opJournal,
6618
- },
6619
- {
6620
- workspaceId: input.workspaceId,
6621
- agentId: activeSandboxRecord!.enrollmentId!,
6622
- opStream: machineOpStream,
6623
- epoch: activeSandboxPointer!.activeEpoch,
6624
- environment: sandboxEnvironment,
6625
- ...(transientCodemodeEnvironment
6626
- ? { transientExecEnvironment: transientCodemodeEnvironment }
6627
- : {}),
6628
- workingDir: activeSandboxPointer!.workingDir,
6629
- },
6795
+ const sandboxEstablishStartedAt = performance.now();
6796
+ let sandboxEstablishOutcome: "completed" | "failed" = "completed";
6797
+ try {
6798
+ const managedOwnership = managedSandboxOwnershipForTurn(
6799
+ machinePrimary,
6800
+ input.attemptId,
6801
+ session.sandboxGroupId,
6630
6802
  );
6631
- // The machine-primary establish narrows `session` to SelfhostedSession
6632
- // (buildSelfhostedBackendSession); EstablishedSandboxSession widens it.
6633
- machinePrimarySession =
6634
- established.session as import("@opengeni/runtime").SelfhostedSession;
6635
- setupBoxSession = established.session;
6636
- resolvedSandbox = {
6637
- // Wrap in the SAME routing proxy so a mid-turn swap (to another machine
6638
- // or back to the group box) still re-routes per op. PIN this established
6639
- // SelfhostedSession for the machine pointer so the turn-start manifest
6640
- // write (via the proxy's `state` getter) and the per-op reads hit ONE
6641
- // instance — no two-instance manifest divergence.
6642
- established: wrapTurnBoxWithRouting(
6803
+ sandboxHolderId = managedOwnership?.holderId ?? null;
6804
+ sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
6805
+ // STAGE D honest-label guard: a machine-home session carries
6806
+ // turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
6807
+ // when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
6808
+ // + enrollmentId). When it is NOT primary — the agent swapped back to the
6809
+ // group box (sandbox_swap 'session'/'default'/groupId clears the pointer) or
6810
+ // selfhosted routing is flag-OFF (the pointer is ignored) — the else-branch
6811
+ // must resume a REAL cloud group box, not a "selfhosted" one: the registry
6812
+ // SelfhostedSandboxClient has no bound agentId and throws. Fall the group-box
6813
+ // backend back to the deployment default cloud backend so swap-away / flag-off
6814
+ // degrade to a genuine cloud box exactly like today (home=modal did).
6815
+ if (machinePrimary) {
6816
+ // STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
6817
+ // establish OR lease the managed home box. The active-sandbox pointer is
6818
+ // the machine route's own epoch fence; the managed-home lease separately
6819
+ // owns cloud recovery, snapshots, billing, and reaping. Mixing those two
6820
+ // ownership domains lets an unrelated degraded cloud archive block a
6821
+ // healthy explicitly selected machine.
6822
+ // Whether the machine's latest Hello advertised the op-stream engine
6823
+ // (refreshed on every connect). Read only when the server flag is on —
6824
+ // one indexed lookup, and the flag off keeps this path byte-identical.
6825
+ const machineOpStream =
6826
+ settings.agentOpStreamEnabled === true
6827
+ ? (await getEnrollment(db, input.workspaceId, activeSandboxRecord!.enrollmentId!))
6828
+ ?.opStream === true
6829
+ : false;
6830
+ const established = await establishSelfhostedTurnSession(
6643
6831
  {
6644
6832
  db,
6645
6833
  settings,
6646
6834
  bus,
6647
- opJournal,
6648
6835
  onOp: machineOpObserver.observer,
6649
6836
  onSandboxOperation: sandboxOperationObserver,
6650
- onHomeSandboxRebound,
6651
- ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
6837
+ opJournal,
6652
6838
  },
6653
6839
  {
6654
6840
  workspaceId: input.workspaceId,
6655
- sessionId: input.sessionId,
6841
+ agentId: activeSandboxRecord!.enrollmentId!,
6842
+ opStream: machineOpStream,
6843
+ epoch: activeSandboxPointer!.activeEpoch,
6656
6844
  environment: sandboxEnvironment,
6657
6845
  ...(transientCodemodeEnvironment
6658
6846
  ? { transientExecEnvironment: transientCodemodeEnvironment }
6659
6847
  : {}),
6660
- workspaceMutationFence: {
6661
- accountId: input.accountId,
6662
- turnId: turn.id,
6663
- executionGeneration,
6664
- attemptId: input.attemptId,
6848
+ workingDir: activeSandboxPointer!.workingDir,
6849
+ },
6850
+ );
6851
+ // The machine-primary establish narrows `session` to SelfhostedSession
6852
+ // (buildSelfhostedBackendSession); EstablishedSandboxSession widens it.
6853
+ machinePrimarySession =
6854
+ established.session as import("@opengeni/runtime").SelfhostedSession;
6855
+ setupBoxSession = established.session;
6856
+ resolvedSandbox = {
6857
+ // Wrap in the SAME routing proxy so a mid-turn swap (to another machine
6858
+ // or back to the group box) still re-routes per op. PIN this established
6859
+ // SelfhostedSession for the machine pointer so the turn-start manifest
6860
+ // write (via the proxy's `state` getter) and the per-op reads hit ONE
6861
+ // instance — no two-instance manifest divergence.
6862
+ established: wrapTurnBoxWithRouting(
6863
+ {
6864
+ db,
6865
+ settings,
6866
+ bus,
6867
+ opJournal,
6868
+ onOp: machineOpObserver.observer,
6869
+ onSandboxOperation: sandboxOperationObserver,
6870
+ onHomeSandboxRebound,
6871
+ ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
6665
6872
  },
6666
- pinnedSelfhosted: {
6667
- sandboxId: activeSandboxPointer!.activeSandboxId!,
6668
- epoch: activeSandboxPointer!.activeEpoch,
6873
+ {
6874
+ workspaceId: input.workspaceId,
6875
+ sessionId: input.sessionId,
6876
+ environment: sandboxEnvironment,
6877
+ ...(transientCodemodeEnvironment
6878
+ ? { transientExecEnvironment: transientCodemodeEnvironment }
6879
+ : {}),
6880
+ workspaceMutationFence: {
6881
+ accountId: input.accountId,
6882
+ turnId: turn.id,
6883
+ executionGeneration,
6884
+ attemptId: input.attemptId,
6885
+ },
6886
+ pinnedSelfhosted: {
6887
+ sandboxId: activeSandboxPointer!.activeSandboxId!,
6888
+ epoch: activeSandboxPointer!.activeEpoch,
6889
+ },
6890
+ // HOME semantics for a mid-turn clear-to-null: only a genuine
6891
+ // machine-HOME session (its home IS this machine, session.sandboxBackend
6892
+ // === "selfhosted") resolves null back to the pinned machine. A Modal-HOME
6893
+ // session merely PINNED to a machine this turn never established its group
6894
+ // box, so defaultIsHome:false makes a clear-to-null fail typed
6895
+ // (`home_unavailable_this_turn`) rather than silently serving the machine;
6896
+ // the detach takes effect next turn. (Lazy home-box establishment on such a
6897
+ // clear is a deferred follow-up; issue #341.)
6898
+ defaultIsHome: session.sandboxBackend === "selfhosted",
6669
6899
  },
6670
- // HOME semantics for a mid-turn clear-to-null: only a genuine
6671
- // machine-HOME session (its home IS this machine, session.sandboxBackend
6672
- // === "selfhosted") resolves null back to the pinned machine. A Modal-HOME
6673
- // session merely PINNED to a machine this turn never established its group
6674
- // box, so defaultIsHome:false makes a clear-to-null fail typed
6675
- // (`home_unavailable_this_turn`) rather than silently serving the machine;
6676
- // the detach takes effect next turn. (Lazy home-box establishment on such a
6677
- // clear is a deferred follow-up; issue #341.)
6678
- defaultIsHome: session.sandboxBackend === "selfhosted",
6679
- },
6680
- established,
6681
- ),
6682
- // For a machine route this is the active-pointer epoch, not a cloud
6683
- // lease epoch. Cloud-only heartbeat/snapshot/capture paths require
6684
- // sandboxGroupId and remain disabled for this branch.
6685
- leaseEpoch: activeSandboxPointer!.activeEpoch,
6686
- release: async () => undefined,
6687
- };
6688
- } else if (establishPolicy === "on-demand") {
6689
- // Lazy sandbox provisioning: holder/group ids are fixed at turn start,
6690
- // but the lease acquire + box establish + setup move behind the routing
6691
- // proxy's first default-pointer op. A chat-only turn never calls it, so
6692
- // no lease row, no provider box, no warm-meter interval.
6693
- } else {
6694
- resolvedSandbox = await waitForTurnOperation(
6695
- resumeBoxForTurn(
6696
- {
6697
- db,
6698
- settings: runSettings,
6699
- logicalFallbackSettings: logicalSandboxSettings,
6700
- cancellationSignal: sandboxResumeSignal,
6701
- sandboxMetrics: runtimeMetricsHooksForObservability(observability),
6702
- onSandboxLost: publishSandboxLost,
6703
- },
6704
- {
6705
- accountId: input.accountId,
6706
- workspaceId: input.workspaceId,
6707
- sandboxGroupId: session.sandboxGroupId,
6708
- sessionId: input.sessionId,
6709
- // groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
6710
- // is not machine-primary resumes a real cloud group box (the
6711
- // deployment default), never a "selfhosted" box (which would throw
6712
- // for lack of a bound agentId).
6713
- backend: groupBoxBackend,
6714
- os: session.sandboxOs,
6715
- environment: sandboxEnvironment,
6716
- // IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
6717
- // this run resolves. The lease stamps it + conflicts on a live shared box
6718
- // running a DIFFERENT image (solo → durable rotation; N-holders →
6719
- // SandboxImageConflictError surfaced as an actionable turn error). Prefer
6720
- // the explicit Modal image ref, else the docker image. The selfhosted branch
6721
- // (establishSelfhostedTurnSession) NEVER passes
6722
- // an image — B3 lives only on this Modal else-branch.
6723
- ...((runSettings.modalImageRef ?? runSettings.dockerImage)
6724
- ? {
6725
- image: runSettings.modalImageRef ?? runSettings.dockerImage,
6726
- }
6727
- : {}),
6728
- // RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
6729
- // conflicts on a live shared box set up under a different rig (solo
6730
- // durable rotation / N-holders SandboxRigConflictError). Omitted for a
6731
- // rig-less turn -> never stamped or enforced (shares exactly as today).
6732
- ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
6733
- },
6734
- "turn",
6735
- managedOwnership!.holderId,
6736
- ),
6737
- sandboxResumeSignal,
6738
- releaseLateSandbox,
6739
- );
6740
- setupBoxSession = resolvedSandbox.established.session;
6741
- // Durable box-lifecycle events (sandbox-file-persistence observability):
6742
- // record every box transition in session_events so the NEXT box loss is
6743
- // attributable from the DB alone — worker logs rotate within hours, which
6744
- // left both 2026-07-06 incidents without a durable trace. Best-effort.
6745
- await publishSandboxLifecycleEvents(resolvedSandbox);
6746
- // M7 hot-swap: when the selfhosted feature is on, wrap the established
6747
- // group box in the STABLE routing proxy before it is injected NON-OWNED
6748
- // into the run. The SDK binds to this ONE object once and calls its
6749
- // methods per tool call; the proxy re-reads (active_sandbox_id,
6750
- // active_epoch) per op and dispatches to the currently-active backend, so
6751
- // a sandbox_swap mid-turn lands the NEXT tool call on the new box. With
6752
- // the flag off the established group box is injected unchanged (today's
6753
- // path). The lease still owns the group box lifecycle — the proxy is a
6754
- // routing veneer, not an owner.
6755
- resolvedSandbox = {
6756
- ...resolvedSandbox,
6757
- established: wrapTurnBoxWithRouting(
6758
- {
6759
- db,
6760
- settings,
6761
- bus,
6762
- opJournal,
6763
- onSandboxOperation: sandboxOperationObserver,
6764
- onHomeSandboxLost: publishSandboxLost,
6765
- onHomeSandboxRebound,
6766
- ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
6767
- },
6768
- // Thread the SAME declared environment the group box was created with
6769
- // (resumeBoxForTurn, above) so a selfhosted swap target's manifest
6770
- // carries it too — the SDK's per-turn manifest-env delta stays empty
6771
- // (no "cannot change manifest environment variables" throw).
6772
- {
6773
- workspaceId: input.workspaceId,
6774
- sessionId: input.sessionId,
6775
- environment: sandboxEnvironment,
6776
- ...(transientCodemodeEnvironment
6777
- ? { transientExecEnvironment: transientCodemodeEnvironment }
6778
- : {}),
6779
- workspaceMutationFence: {
6780
- accountId: input.accountId,
6781
- turnId: turn.id,
6782
- executionGeneration,
6783
- attemptId: input.attemptId,
6900
+ established,
6901
+ ),
6902
+ // For a machine route this is the active-pointer epoch, not a cloud
6903
+ // lease epoch. Cloud-only heartbeat/snapshot/capture paths require
6904
+ // sandboxGroupId and remain disabled for this branch.
6905
+ leaseEpoch: activeSandboxPointer!.activeEpoch,
6906
+ release: async () => undefined,
6907
+ };
6908
+ } else if (establishPolicy === "on-demand") {
6909
+ // Lazy sandbox provisioning: holder/group ids are fixed at turn start,
6910
+ // but the lease acquire + box establish + setup move behind the routing
6911
+ // proxy's first default-pointer op. A chat-only turn never calls it, so
6912
+ // no lease row, no provider box, no warm-meter interval.
6913
+ } else {
6914
+ resolvedSandbox = await waitForTurnOperation(
6915
+ resumeBoxForTurn(
6916
+ {
6917
+ db,
6918
+ settings: runSettings,
6919
+ logicalFallbackSettings: logicalSandboxSettings,
6920
+ cancellationSignal: sandboxResumeSignal,
6921
+ sandboxMetrics: runtimeMetricsHooksForObservability(observability),
6922
+ onSandboxLost: publishSandboxLost,
6784
6923
  },
6785
- homeLease: {
6924
+ {
6786
6925
  accountId: input.accountId,
6926
+ workspaceId: input.workspaceId,
6787
6927
  sandboxGroupId: session.sandboxGroupId,
6788
- leaseEpoch: resolvedSandbox.leaseEpoch,
6789
- instanceId: resolvedSandbox.established.instanceId,
6928
+ sessionId: input.sessionId,
6929
+ // groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
6930
+ // is not machine-primary resumes a real cloud group box (the
6931
+ // deployment default), never a "selfhosted" box (which would throw
6932
+ // for lack of a bound agentId).
6790
6933
  backend: groupBoxBackend,
6934
+ os: session.sandboxOs,
6935
+ environment: sandboxEnvironment,
6936
+ // IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
6937
+ // this run resolves. The lease stamps it + conflicts on a live shared box
6938
+ // running a DIFFERENT image (solo → durable rotation; N-holders →
6939
+ // SandboxImageConflictError surfaced as an actionable turn error). Prefer
6940
+ // the explicit Modal image ref, else the docker image. The selfhosted branch
6941
+ // (establishSelfhostedTurnSession) NEVER passes
6942
+ // an image — B3 lives only on this Modal else-branch.
6943
+ ...((runSettings.modalImageRef ?? runSettings.dockerImage)
6944
+ ? {
6945
+ image: runSettings.modalImageRef ?? runSettings.dockerImage,
6946
+ }
6947
+ : {}),
6948
+ // RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
6949
+ // conflicts on a live shared box set up under a different rig (solo
6950
+ // durable rotation / N-holders SandboxRigConflictError). Omitted for a
6951
+ // rig-less turn -> never stamped or enforced (shares exactly as today).
6952
+ ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
6791
6953
  },
6792
- },
6793
- resolvedSandbox.established,
6794
- ),
6795
- };
6796
- }
6797
- if (resolvedSandbox) {
6798
- startLeaseHeartbeat(resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
6954
+ "turn",
6955
+ managedOwnership!.holderId,
6956
+ ),
6957
+ sandboxResumeSignal,
6958
+ releaseLateSandbox,
6959
+ );
6960
+ setupBoxSession = resolvedSandbox.established.session;
6961
+ // Durable box-lifecycle events (sandbox-file-persistence observability):
6962
+ // record every box transition in session_events so the NEXT box loss is
6963
+ // attributable from the DB alone — worker logs rotate within hours, which
6964
+ // left both 2026-07-06 incidents without a durable trace. Best-effort.
6965
+ await publishSandboxLifecycleEvents(resolvedSandbox);
6966
+ // M7 hot-swap: when the selfhosted feature is on, wrap the established
6967
+ // group box in the STABLE routing proxy before it is injected NON-OWNED
6968
+ // into the run. The SDK binds to this ONE object once and calls its
6969
+ // methods per tool call; the proxy re-reads (active_sandbox_id,
6970
+ // active_epoch) per op and dispatches to the currently-active backend, so
6971
+ // a sandbox_swap mid-turn lands the NEXT tool call on the new box. With
6972
+ // the flag off the established group box is injected unchanged (today's
6973
+ // path). The lease still owns the group box lifecycle — the proxy is a
6974
+ // routing veneer, not an owner.
6975
+ resolvedSandbox = {
6976
+ ...resolvedSandbox,
6977
+ established: wrapTurnBoxWithRouting(
6978
+ {
6979
+ db,
6980
+ settings,
6981
+ bus,
6982
+ opJournal,
6983
+ onSandboxOperation: sandboxOperationObserver,
6984
+ onHomeSandboxLost: publishSandboxLost,
6985
+ onHomeSandboxRebound,
6986
+ ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
6987
+ },
6988
+ // Thread the SAME declared environment the group box was created with
6989
+ // (resumeBoxForTurn, above) so a selfhosted swap target's manifest
6990
+ // carries it too — the SDK's per-turn manifest-env delta stays empty
6991
+ // (no "cannot change manifest environment variables" throw).
6992
+ {
6993
+ workspaceId: input.workspaceId,
6994
+ sessionId: input.sessionId,
6995
+ environment: sandboxEnvironment,
6996
+ ...(transientCodemodeEnvironment
6997
+ ? { transientExecEnvironment: transientCodemodeEnvironment }
6998
+ : {}),
6999
+ workspaceMutationFence: {
7000
+ accountId: input.accountId,
7001
+ turnId: turn.id,
7002
+ executionGeneration,
7003
+ attemptId: input.attemptId,
7004
+ },
7005
+ homeLease: {
7006
+ accountId: input.accountId,
7007
+ sandboxGroupId: session.sandboxGroupId,
7008
+ leaseEpoch: resolvedSandbox.leaseEpoch,
7009
+ instanceId: resolvedSandbox.established.instanceId,
7010
+ backend: groupBoxBackend,
7011
+ },
7012
+ },
7013
+ resolvedSandbox.established,
7014
+ ),
7015
+ };
7016
+ }
7017
+ if (resolvedSandbox) {
7018
+ startLeaseHeartbeat(resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
7019
+ }
7020
+ } catch (error) {
7021
+ sandboxEstablishOutcome = "failed";
7022
+ throw error;
7023
+ } finally {
7024
+ recordTurnStartupPhase(observability, {
7025
+ phase: "sandbox_establish",
7026
+ provider: turnExecutionPolicy.providerId,
7027
+ backend: machinePrimary ? "selfhosted" : groupBoxBackend,
7028
+ outcome: sandboxEstablishOutcome,
7029
+ durationSeconds: (performance.now() - sandboxEstablishStartedAt) / 1_000,
7030
+ });
6799
7031
  }
6800
7032
  }
6801
7033
 
6802
- const ordinaryFileResourceDownloads = await waitForTurnOperation(
6803
- sandboxFileDownloadsForRun(
6804
- runSettings,
6805
- db,
6806
- objectStorage,
6807
- input.workspaceId,
6808
- turnResources,
6809
- activeSandboxBackend ?? groupBoxBackend,
6810
- ),
6811
- cancellationSignal,
6812
- undefined,
6813
- );
7034
+ const ordinaryFileResourceDownloads = await (async () => {
7035
+ const fileResolutionStartedAt = performance.now();
7036
+ let fileResolutionOutcome: "completed" | "failed" = "completed";
7037
+ try {
7038
+ return await waitForTurnOperation(
7039
+ sandboxFileDownloadsForRun(
7040
+ runSettings,
7041
+ db,
7042
+ objectStorage,
7043
+ input.workspaceId,
7044
+ turnResources,
7045
+ activeSandboxBackend ?? groupBoxBackend,
7046
+ ),
7047
+ cancellationSignal,
7048
+ undefined,
7049
+ );
7050
+ } catch (error) {
7051
+ fileResolutionOutcome = "failed";
7052
+ throw error;
7053
+ } finally {
7054
+ recordTurnStartupPhase(observability, {
7055
+ phase: "file_resolution",
7056
+ provider: turnExecutionPolicy.providerId,
7057
+ backend: activeSandboxBackend ?? groupBoxBackend,
7058
+ outcome: fileResolutionOutcome,
7059
+ durationSeconds: (performance.now() - fileResolutionStartedAt) / 1_000,
7060
+ count: turnResources.length,
7061
+ });
7062
+ }
7063
+ })();
6814
7064
  if (requiredGeneratedVideoFiles.length > 0 && !objectStorage) {
6815
7065
  throw new Error("Generated video materialization requires object storage");
6816
7066
  }
@@ -6840,6 +7090,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6840
7090
  ]),
6841
7091
  ).values(),
6842
7092
  ];
7093
+ const toolContextPreparationStartedAt = performance.now();
6843
7094
  throwIfWorkerShuttingDown();
6844
7095
  throwIfTurnCancelled();
6845
7096
  const mcpCredentialRootSessionId =
@@ -6938,59 +7189,94 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6938
7189
  runSettings,
6939
7190
  session.firstPartyMcpTools,
6940
7191
  );
6941
- preparedTools = await waitForTurnOperation(
6942
- runtime.prepareTools(runSettings, turnTools, {
6943
- accountId: input.accountId,
6944
- workspaceId: input.workspaceId,
6945
- sessionId: input.sessionId,
6946
- // Sign the calling turn into the first-party token so tools classify
6947
- // the caller by its own identity (sacred-pause guard), not the racy
6948
- // live active pointer.
6949
- ...(turnId ? { turnId } : {}),
6950
- attemptId: input.attemptId,
6951
- executionGeneration,
6952
- subjectId: "worker:first-party-mcp",
6953
- subjectLabel: "OpenGeni worker",
6954
- ...(credentialSubjectId ? { credentialSubjectId } : {}),
6955
- ...(codexAppsAuth ? { codexAppsAuth } : {}),
6956
- resolveCredential,
6957
- onAuthNeeded: publishToolAuthNeeded,
6958
- localMcpServers,
6959
- onAttemptToolCatalog: async (catalog) => {
6960
- await persistAttemptToolCatalog(db, catalog);
6961
- },
6962
- // Manager-style sessions carry a creation-validated permission set
6963
- // for their first-party MCP token; null keeps the fixed default.
6964
- ...(session.firstPartyMcpPermissions?.length
6965
- ? { firstPartyPermissions: session.firstPartyMcpPermissions }
6966
- : {}),
6967
- firstPartyTools: selectedFirstPartyMcpTools,
6968
- nestedAgentDepth: session.nestedAgentDepth,
6969
- effectiveMaxNestedAgentDepth: session.effectiveMaxNestedAgentDepth,
6970
- attemptToolDefinitions: createFirstPartyInteractionAttemptToolDefinitions({
6971
- settings: runSettings,
6972
- scope: {
6973
- accountId: input.accountId,
6974
- workspaceId: input.workspaceId,
6975
- sessionId: input.sessionId,
6976
- turnId: turn.id,
6977
- attemptId: input.attemptId,
6978
- executionGeneration,
6979
- },
6980
- ...(session.firstPartyMcpPermissions?.length
6981
- ? { permissions: session.firstPartyMcpPermissions }
6982
- : {}),
6983
- selectedTools: selectedFirstPartyMcpTools,
7192
+ recordTurnStartupPhase(observability, {
7193
+ phase: "tool_context_preparation",
7194
+ provider: turnExecutionPolicy.providerId,
7195
+ backend: activeSandboxBackend ?? groupBoxBackend,
7196
+ outcome: "completed",
7197
+ durationSeconds: (performance.now() - toolContextPreparationStartedAt) / 1_000,
7198
+ count: turnTools.length,
7199
+ });
7200
+ const toolPreparationStartedAt = performance.now();
7201
+ let toolPreparationOutcome: "completed" | "failed" = "completed";
7202
+ try {
7203
+ preparedTools = await waitForTurnOperation(
7204
+ runtime.prepareTools(runSettings, turnTools, {
7205
+ accountId: input.accountId,
7206
+ workspaceId: input.workspaceId,
7207
+ sessionId: input.sessionId,
7208
+ // Sign the calling turn into the first-party token so tools classify
7209
+ // the caller by its own identity (sacred-pause guard), not the racy
7210
+ // live active pointer.
7211
+ ...(turnId ? { turnId } : {}),
7212
+ attemptId: input.attemptId,
7213
+ executionGeneration,
6984
7214
  subjectId: "worker:first-party-mcp",
6985
7215
  subjectLabel: "OpenGeni worker",
6986
- ...(interactionInterventionResume
6987
- ? { interventionResume: interactionInterventionResume }
7216
+ ...(credentialSubjectId ? { credentialSubjectId } : {}),
7217
+ ...(codexAppsAuth ? { codexAppsAuth } : {}),
7218
+ resolveCredential,
7219
+ onAuthNeeded: publishToolAuthNeeded,
7220
+ localMcpServers,
7221
+ onPreparationPhase: (measurement) => {
7222
+ recordTurnStartupPhase(observability, {
7223
+ phase: `tool_${measurement.phase}`,
7224
+ provider: turnExecutionPolicy.providerId,
7225
+ backend: activeSandboxBackend ?? groupBoxBackend,
7226
+ outcome: measurement.outcome,
7227
+ durationSeconds: measurement.durationSeconds,
7228
+ count: turnTools.length,
7229
+ });
7230
+ },
7231
+ onAttemptToolCatalog: async (catalog) => {
7232
+ await persistAttemptToolCatalog(db, catalog);
7233
+ },
7234
+ // Manager-style sessions carry a creation-validated permission set
7235
+ // for their first-party MCP token; null keeps the fixed default.
7236
+ ...(session.firstPartyMcpPermissions?.length
7237
+ ? { firstPartyPermissions: session.firstPartyMcpPermissions }
6988
7238
  : {}),
7239
+ firstPartyTools: selectedFirstPartyMcpTools,
7240
+ nestedAgentDepth: session.nestedAgentDepth,
7241
+ effectiveMaxNestedAgentDepth: session.effectiveMaxNestedAgentDepth,
7242
+ attemptToolDefinitions: createFirstPartyInteractionAttemptToolDefinitions({
7243
+ settings: runSettings,
7244
+ scope: {
7245
+ accountId: input.accountId,
7246
+ workspaceId: input.workspaceId,
7247
+ sessionId: input.sessionId,
7248
+ turnId: turn.id,
7249
+ attemptId: input.attemptId,
7250
+ executionGeneration,
7251
+ },
7252
+ ...(session.firstPartyMcpPermissions?.length
7253
+ ? { permissions: session.firstPartyMcpPermissions }
7254
+ : {}),
7255
+ selectedTools: selectedFirstPartyMcpTools,
7256
+ subjectId: "worker:first-party-mcp",
7257
+ subjectLabel: "OpenGeni worker",
7258
+ ...(interactionInterventionResume
7259
+ ? { interventionResume: interactionInterventionResume }
7260
+ : {}),
7261
+ }),
6989
7262
  }),
6990
- }),
6991
- cancellationSignal,
6992
- async (latePreparedTools) => await latePreparedTools.close().catch(() => undefined),
6993
- );
7263
+ cancellationSignal,
7264
+ async (latePreparedTools) => await latePreparedTools.close().catch(() => undefined),
7265
+ );
7266
+ } catch (error) {
7267
+ toolPreparationOutcome = "failed";
7268
+ throw error;
7269
+ } finally {
7270
+ recordTurnStartupPhase(observability, {
7271
+ phase: "tool_preparation",
7272
+ provider: turnExecutionPolicy.providerId,
7273
+ backend: activeSandboxBackend ?? groupBoxBackend,
7274
+ outcome: toolPreparationOutcome,
7275
+ durationSeconds: (performance.now() - toolPreparationStartedAt) / 1_000,
7276
+ count: turnTools.length,
7277
+ });
7278
+ }
7279
+ const postToolPreparationStartedAt = performance.now();
6994
7280
  if (turnId && preparedTools.attemptToolEnvironment) {
6995
7281
  codemodeDispatcher = new CodemodeAttemptDispatcher(
6996
7282
  db,
@@ -7378,203 +7664,228 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7378
7664
  prepare: async (call) =>
7379
7665
  await prepareConnectorActionApproval(db, connectorActionIdentity, call),
7380
7666
  begin: async (call) =>
7381
- await beginConnectorActionExecution(db, connectorActionIdentity, call),
7382
- complete: async ({ requestId, outcome }) =>
7383
- await completeConnectorActionExecution(db, {
7384
- accountId: input.accountId,
7385
- workspaceId: input.workspaceId,
7386
- requestId,
7387
- attemptId: input.attemptId,
7388
- outcome,
7389
- }),
7390
- };
7391
- const runtimeSkillActivations = [
7392
- ...installedSkillRuntime.activations,
7393
- ...packRuntime.skillActivations,
7394
- ...session.skills.map((skill) => ({
7395
- source: "session" as const,
7396
- id: `session:${session.id}:${skill.name}`,
7397
- artifact: {
7398
- name: skill.name,
7399
- description: skill.description ?? null,
7400
- files: skill.files.map((file) => ({
7401
- path: file.path,
7402
- content: file.content,
7403
- })),
7404
- },
7405
- reason: "attached to session",
7406
- })),
7407
- ];
7408
- const agent = runtime.buildAgent(modelRunSettings, turnResources, {
7409
- reasoningEffort: turn.reasoningEffort,
7410
- latencyMode: turnExecutionPolicy.latencyMode,
7411
- ...(serviceTier ? { serviceTier } : {}),
7412
- ...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
7413
- humanInputEnabled: agentHumanInputEnabled,
7414
- genesisTitleHint: isGenesisTurn,
7415
- persistentSessionSettings: {
7416
- titleIsSet: Boolean(session.title?.trim()),
7417
- },
7418
- sandboxEnvironment,
7419
- ...(preparedTools.attemptToolCatalog
7420
- ? { attemptToolCatalog: preparedTools.attemptToolCatalog }
7421
- : {}),
7422
- ...(sandboxArtifactRuntime.available ? { artifactRuntimeAvailable: true } : {}),
7423
- ...(cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {}),
7424
- onToolCancellationFence: (fence) => {
7425
- toolCancellationFenceRef.current = fence;
7426
- },
7427
- // TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
7428
- // seed. ONLY when the effective backend is NOT selfhosted (the connected
7429
- // machine uses its own git creds — mirrors the skipGitHubToken gate above)
7430
- // AND the mint actually produced a token (repo resources present). The runtime
7431
- // seeds it to the box's token file before the repository-clone runs; it never
7432
- // touches the box/agent manifest env.
7433
- ...(activeSandboxBackend !== "selfhosted" && sandboxGitTokens
7434
- ? { gitTokenSeeds: sandboxGitTokens }
7435
- : {}),
7436
- ...(activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings
7437
- ? { gitCredentialBindings: sandboxGitCredentialBindings }
7438
- : {}),
7439
- ...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
7440
- ? { gitTokenSeed: sandboxGitToken }
7441
- : {}),
7442
- ...(sandboxCodemodeToken ? { codemodeAvailable: true } : {}),
7443
- // Managed boxes receive the bearer through their protected per-session
7444
- // token file. Connected Machines use transient per-exec delivery above,
7445
- // so they must not run the file-seeding lifecycle hook.
7446
- ...(activeSandboxBackend !== "selfhosted" && sandboxCodemodeToken
7447
- ? {
7448
- codemodeTokenSeed: sandboxCodemodeToken,
7449
- codemodeTokenSessionId: input.sessionId,
7450
- }
7451
- : {}),
7452
- ...(activeSandboxBackend ? { activeSandboxBackend } : {}),
7453
- fileResourceDownloads,
7454
- mcpServers: preparedTools.mcpServers,
7455
- resolvedMcpConnectionIds: preparedTools.resolvedMcpConnectionIds,
7456
- connectorActionPolicy,
7457
- // LIVE by-reference connector namespaces (fills during this turn's
7458
- // codex_apps tools/list): the codex tool_search description reads it per
7459
- // model call so the model sees the account's real connected sources.
7460
- codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
7461
- // Resolved-model routing + gating (legacy defaults when null). The model
7462
- // is passed as the model *string* (agent.model = runSettings.openaiModel),
7463
- // NOT a Model instance: an instance only survives the in-process
7464
- // ("none") run, whereas the SandboxAgent/Modal path drops it and
7465
- // re-resolves the model *name* through the global MultiProviderModelProvider
7466
- // configureOpenAI installed — so registry models (Fireworks GLM) route to
7467
- // their own client instead of 404ing against the built-in Azure/OpenAI
7468
- // client. The gating still comes from the resolved provider: server-side
7469
- // store/compaction follow the provider's compaction mode (registry
7470
- // providers resolve to "client"); encrypted reasoning is only
7471
- // round-tripped on the Responses wire API; hosted web search is attached
7472
- // whenever the provider declares it runnable and is independent of the
7473
- // session's MCP allow-list; the effective context window drives the
7474
- // compaction threshold.
7475
- hostedWebSearch,
7476
- ...imageGenerationOption,
7477
- ...videoGenerationOption,
7478
- lazyToolTransport,
7479
- supportsImageInput,
7480
- inputFileMediaTypes: modelInputPolicy.inputFileMediaTypes,
7481
- ...(resolvedModel
7482
- ? {
7483
- encryptedReasoning:
7484
- resolvedModel.provider.api === "responses" &&
7485
- runSettings.openaiReasoningEncryptedContent,
7486
- contextWindowTokens:
7487
- resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
7488
- // The ChatGPT/Codex backend rejects the SDK's HOSTED apply_patch
7489
- // tool. Gateway Responses routes likewise expose ordinary function
7490
- // tools, not OpenAI-hosted sandbox tools. Tell buildAgent to use
7491
- // function apply_patch and wrap successful view_image results as
7492
- // typed input_image content. Chat wires have no proven typed image
7493
- // result transport and therefore receive no view_image tool.
7494
- structuredToolTransport: structuredToolTransportForTurn(resolvedModel),
7495
- // EXPLICIT computer-use tool transport, derived from the resolved provider's
7496
- // authoritative wire identity (codex → function-image, chat → disabled,
7497
- // responses → hosted) so the runtime never string-sniffs the model instance's
7498
- // constructor name. See {@link computerToolModeForTurn}.
7499
- computerToolMode: computerToolModeForTurn(resolvedModel),
7500
- ...(promptCacheKey ? { promptCacheKey } : {}),
7501
- }
7502
- : // LEGACY global-client fallback (resolveTurnModel returned null → the model
7503
- // is not in the registry, served by the built-in OpenAI/Azure Responses
7504
- // client). That backend has real hosted support, so pin computerToolMode to
7505
- // "hosted" EXPLICITLY rather than leaving the runtime to sniff the instance.
7506
- {
7507
- computerToolMode: computerToolModeForTurn(null),
7508
- promptCacheKey: input.sessionId,
7509
- }),
7510
- // Lazy computer-use seam: runtime first brings up :0 only after the model
7511
- // selects a computer tool, then this hook begins the optional proof
7512
- // recording. Shell/filesystem turns never invoke either operation.
7513
- onComputerUseReady: async () => {
7514
- if (!resolvedSandbox) {
7515
- throw new Error("Computer-use display became ready without a resolved sandbox");
7516
- }
7517
- // This callback is the authoritative execution boundary. Record the
7518
- // action before async ffmpeg startup so transport-event ordering cannot
7519
- // make settlement misclassify a real computer turn as unused.
7520
- didComputerUse = true;
7521
- await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
7522
- },
7523
- onRetainableSessionImageOutput: retainSessionImageAtToolBoundary,
7524
- ...(runtimeSkillActivations.length > 0
7525
- ? { skillActivations: runtimeSkillActivations }
7526
- : {}),
7527
- ...(!structuredWorkspacePolicyActive && workspaceAgentInstructions
7528
- ? { instructionsTemplate: workspaceAgentInstructions }
7529
- : {}),
7530
- ...(workspaceGovernance ? { workspaceGovernance } : {}),
7531
- ...(workspaceMemory ? { workspaceMemory } : {}),
7532
- goalSnapshot: turn.goalSnapshot,
7533
- // Per-session persona tier (session > workspace > deployment default).
7534
- // Composed system-level AFTER the workspace persona so it refines it for
7535
- // this one session; absent ⇒ byte-identical to today's composition.
7536
- ...(session.instructions ? { sessionInstructions: session.instructions } : {}),
7537
- // Exact host context captured when this turn was accepted. It is
7538
- // system-level and disappears with the turn rather than entering chat.
7539
- ...(turn.turnInstructions || mcpAvailabilityInstructions
7540
- ? {
7541
- turnInstructions: [turn.turnInstructions, mcpAvailabilityInstructions]
7542
- .filter((value): value is string => Boolean(value))
7543
- .join(" "),
7544
- }
7545
- : {}),
7546
- ...workspaceEnvironmentOption,
7547
- // RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
7548
- // the frozen version carries a non-empty script), and the rig credential
7549
- // hooks. All absent for a rig-less turn (byte-for-byte today).
7550
- ...(rigVersion && rigName
7551
- ? {
7552
- rig: { name: rigName, version: rigVersion.version },
7553
- ...(rigVersion.setupScript && rigVersion.setupScript.trim().length > 0
7554
- ? {
7555
- rigSetup: {
7556
- rigId: session.rigId!,
7557
- versionId: rigVersion.id,
7558
- rigName,
7559
- script: rigVersion.setupScript,
7560
- timeoutMs: runSettings.rigSetupTimeoutMs,
7561
- contentHash: rigProviderImageContentHash({
7562
- backend: turn.sandboxBackend,
7563
- sourceImage: rigProviderImageSourceImage(
7564
- logicalSandboxSettings,
7565
- turn.sandboxBackend,
7566
- ),
7567
- definition: rigVersion,
7568
- }),
7569
- },
7570
- }
7571
- : {}),
7572
- ...(rigVersion.credentialHooks.length > 0
7573
- ? { rigCredentialHookIds: rigVersion.credentialHooks }
7574
- : {}),
7575
- }
7576
- : {}),
7667
+ await beginConnectorActionExecution(db, connectorActionIdentity, call),
7668
+ complete: async ({ requestId, outcome }) =>
7669
+ await completeConnectorActionExecution(db, {
7670
+ accountId: input.accountId,
7671
+ workspaceId: input.workspaceId,
7672
+ requestId,
7673
+ attemptId: input.attemptId,
7674
+ outcome,
7675
+ }),
7676
+ };
7677
+ const runtimeSkillActivations = [
7678
+ ...installedSkillRuntime.activations,
7679
+ ...packRuntime.skillActivations,
7680
+ ...session.skills.map((skill) => ({
7681
+ source: "session" as const,
7682
+ id: `session:${session.id}:${skill.name}`,
7683
+ artifact: {
7684
+ name: skill.name,
7685
+ description: skill.description ?? null,
7686
+ files: skill.files.map((file) => ({
7687
+ path: file.path,
7688
+ content: file.content,
7689
+ })),
7690
+ },
7691
+ reason: "attached to session",
7692
+ })),
7693
+ ];
7694
+ recordTurnStartupPhase(observability, {
7695
+ phase: "post_tool_preparation",
7696
+ provider: turnExecutionPolicy.providerId,
7697
+ backend: activeSandboxBackend ?? groupBoxBackend,
7698
+ outcome: "completed",
7699
+ durationSeconds: (performance.now() - postToolPreparationStartedAt) / 1_000,
7577
7700
  });
7701
+ const agent = (() => {
7702
+ const agentConstructionStartedAt = performance.now();
7703
+ let agentConstructionOutcome: "completed" | "failed" = "completed";
7704
+ try {
7705
+ return runtime.buildAgent(modelRunSettings, turnResources, {
7706
+ reasoningEffort: turn.reasoningEffort,
7707
+ latencyMode: turnExecutionPolicy.latencyMode,
7708
+ ...(serviceTier ? { serviceTier } : {}),
7709
+ ...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
7710
+ humanInputEnabled: agentHumanInputEnabled,
7711
+ genesisTitleHint: isGenesisTurn,
7712
+ persistentSessionSettings: {
7713
+ titleIsSet: Boolean(session.title?.trim()),
7714
+ },
7715
+ sandboxEnvironment,
7716
+ ...(preparedTools.attemptToolCatalog
7717
+ ? { attemptToolCatalog: preparedTools.attemptToolCatalog }
7718
+ : {}),
7719
+ ...(sandboxArtifactRuntime.available ? { artifactRuntimeAvailable: true } : {}),
7720
+ ...(cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {}),
7721
+ onToolCancellationFence: (fence) => {
7722
+ toolCancellationFenceRef.current = fence;
7723
+ },
7724
+ // TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
7725
+ // seed. ONLY when the effective backend is NOT selfhosted (the connected
7726
+ // machine uses its own git creds — mirrors the skipGitHubToken gate above)
7727
+ // AND the mint actually produced a token (repo resources present). The runtime
7728
+ // seeds it to the box's token file before the repository-clone runs; it never
7729
+ // touches the box/agent manifest env.
7730
+ ...(activeSandboxBackend !== "selfhosted" && sandboxGitTokens
7731
+ ? { gitTokenSeeds: sandboxGitTokens }
7732
+ : {}),
7733
+ ...(activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings
7734
+ ? { gitCredentialBindings: sandboxGitCredentialBindings }
7735
+ : {}),
7736
+ ...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
7737
+ ? { gitTokenSeed: sandboxGitToken }
7738
+ : {}),
7739
+ ...(sandboxCodemodeToken ? { codemodeAvailable: true } : {}),
7740
+ // Managed boxes receive the bearer through their protected per-session
7741
+ // token file. Connected Machines use transient per-exec delivery above,
7742
+ // so they must not run the file-seeding lifecycle hook.
7743
+ ...(activeSandboxBackend !== "selfhosted" && sandboxCodemodeToken
7744
+ ? {
7745
+ codemodeTokenSeed: sandboxCodemodeToken,
7746
+ codemodeTokenSessionId: input.sessionId,
7747
+ }
7748
+ : {}),
7749
+ ...(activeSandboxBackend ? { activeSandboxBackend } : {}),
7750
+ fileResourceDownloads,
7751
+ mcpServers: preparedTools.mcpServers,
7752
+ resolvedMcpConnectionIds: preparedTools.resolvedMcpConnectionIds,
7753
+ connectorActionPolicy,
7754
+ // LIVE by-reference connector namespaces (fills during this turn's
7755
+ // codex_apps tools/list): the codex tool_search description reads it per
7756
+ // model call so the model sees the account's real connected sources.
7757
+ codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
7758
+ // Resolved-model routing + gating (legacy defaults when null). The model
7759
+ // is passed as the model *string* (agent.model = runSettings.openaiModel),
7760
+ // NOT a Model instance: an instance only survives the in-process
7761
+ // ("none") run, whereas the SandboxAgent/Modal path drops it and
7762
+ // re-resolves the model *name* through the global MultiProviderModelProvider
7763
+ // configureOpenAI installed — so registry models (Fireworks GLM) route to
7764
+ // their own client instead of 404ing against the built-in Azure/OpenAI
7765
+ // client. The gating still comes from the resolved provider: server-side
7766
+ // store/compaction follow the provider's compaction mode (registry
7767
+ // providers resolve to "client"); encrypted reasoning is only
7768
+ // round-tripped on the Responses wire API; hosted web search is attached
7769
+ // whenever the provider declares it runnable and is independent of the
7770
+ // session's MCP allow-list; the effective context window drives the
7771
+ // compaction threshold.
7772
+ hostedWebSearch,
7773
+ ...imageGenerationOption,
7774
+ ...videoGenerationOption,
7775
+ lazyToolTransport,
7776
+ supportsImageInput,
7777
+ inputFileMediaTypes: modelInputPolicy.inputFileMediaTypes,
7778
+ ...(resolvedModel
7779
+ ? {
7780
+ encryptedReasoning:
7781
+ resolvedModel.provider.api === "responses" &&
7782
+ runSettings.openaiReasoningEncryptedContent,
7783
+ contextWindowTokens:
7784
+ resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
7785
+ // The ChatGPT/Codex backend rejects the SDK's HOSTED apply_patch
7786
+ // tool. Gateway Responses routes likewise expose ordinary function
7787
+ // tools, not OpenAI-hosted sandbox tools. Tell buildAgent to use
7788
+ // function apply_patch and wrap successful view_image results as
7789
+ // typed input_image content. Chat wires have no proven typed image
7790
+ // result transport and therefore receive no view_image tool.
7791
+ structuredToolTransport: structuredToolTransportForTurn(resolvedModel),
7792
+ // EXPLICIT computer-use tool transport, derived from the resolved provider's
7793
+ // authoritative wire identity (codex → function-image, chat → disabled,
7794
+ // responses → hosted) so the runtime never string-sniffs the model instance's
7795
+ // constructor name. See {@link computerToolModeForTurn}.
7796
+ computerToolMode: computerToolModeForTurn(resolvedModel),
7797
+ ...(promptCacheKey ? { promptCacheKey } : {}),
7798
+ }
7799
+ : // LEGACY global-client fallback (resolveTurnModel returned null → the model
7800
+ // is not in the registry, served by the built-in OpenAI/Azure Responses
7801
+ // client). That backend has real hosted support, so pin computerToolMode to
7802
+ // "hosted" EXPLICITLY rather than leaving the runtime to sniff the instance.
7803
+ {
7804
+ computerToolMode: computerToolModeForTurn(null),
7805
+ promptCacheKey: input.sessionId,
7806
+ }),
7807
+ // Lazy computer-use seam: runtime first brings up :0 only after the model
7808
+ // selects a computer tool, then this hook begins the optional proof
7809
+ // recording. Shell/filesystem turns never invoke either operation.
7810
+ onComputerUseReady: async () => {
7811
+ if (!resolvedSandbox) {
7812
+ throw new Error("Computer-use display became ready without a resolved sandbox");
7813
+ }
7814
+ // This callback is the authoritative execution boundary. Record the
7815
+ // action before async ffmpeg startup so transport-event ordering cannot
7816
+ // make settlement misclassify a real computer turn as unused.
7817
+ didComputerUse = true;
7818
+ await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
7819
+ },
7820
+ onRetainableSessionImageOutput: retainSessionImageAtToolBoundary,
7821
+ ...(runtimeSkillActivations.length > 0
7822
+ ? { skillActivations: runtimeSkillActivations }
7823
+ : {}),
7824
+ ...(!structuredWorkspacePolicyActive && workspaceAgentInstructions
7825
+ ? { instructionsTemplate: workspaceAgentInstructions }
7826
+ : {}),
7827
+ ...(workspaceGovernance ? { workspaceGovernance } : {}),
7828
+ ...(workspaceMemory ? { workspaceMemory } : {}),
7829
+ goalSnapshot: turn.goalSnapshot,
7830
+ // Per-session persona tier (session > workspace > deployment default).
7831
+ // Composed system-level AFTER the workspace persona so it refines it for
7832
+ // this one session; absent ⇒ byte-identical to today's composition.
7833
+ ...(session.instructions ? { sessionInstructions: session.instructions } : {}),
7834
+ // Exact host context captured when this turn was accepted. It is
7835
+ // system-level and disappears with the turn rather than entering chat.
7836
+ ...(turn.turnInstructions || mcpAvailabilityInstructions
7837
+ ? {
7838
+ turnInstructions: [turn.turnInstructions, mcpAvailabilityInstructions]
7839
+ .filter((value): value is string => Boolean(value))
7840
+ .join(" "),
7841
+ }
7842
+ : {}),
7843
+ ...workspaceEnvironmentOption,
7844
+ // RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
7845
+ // the frozen version carries a non-empty script), and the rig credential
7846
+ // hooks. All absent for a rig-less turn (byte-for-byte today).
7847
+ ...(rigVersion && rigName
7848
+ ? {
7849
+ rig: { name: rigName, version: rigVersion.version },
7850
+ ...(rigVersion.setupScript && rigVersion.setupScript.trim().length > 0
7851
+ ? {
7852
+ rigSetup: {
7853
+ rigId: session.rigId!,
7854
+ versionId: rigVersion.id,
7855
+ rigName,
7856
+ script: rigVersion.setupScript,
7857
+ timeoutMs: runSettings.rigSetupTimeoutMs,
7858
+ contentHash: rigProviderImageContentHash({
7859
+ backend: turn.sandboxBackend,
7860
+ sourceImage: rigProviderImageSourceImage(
7861
+ logicalSandboxSettings,
7862
+ turn.sandboxBackend,
7863
+ ),
7864
+ definition: rigVersion,
7865
+ }),
7866
+ },
7867
+ }
7868
+ : {}),
7869
+ ...(rigVersion.credentialHooks.length > 0
7870
+ ? { rigCredentialHookIds: rigVersion.credentialHooks }
7871
+ : {}),
7872
+ }
7873
+ : {}),
7874
+ });
7875
+ } catch (error) {
7876
+ agentConstructionOutcome = "failed";
7877
+ throw error;
7878
+ } finally {
7879
+ recordTurnStartupPhase(observability, {
7880
+ phase: "agent_construction",
7881
+ provider: turnExecutionPolicy.providerId,
7882
+ backend: activeSandboxBackend ?? groupBoxBackend,
7883
+ outcome: agentConstructionOutcome,
7884
+ durationSeconds: (performance.now() - agentConstructionStartedAt) / 1_000,
7885
+ });
7886
+ }
7887
+ })();
7888
+ const postAgentPreparationStartedAt = performance.now();
7578
7889
  if (modelRunSettings.sandboxBackend !== "none" && toolCancellationFenceRef.current === null) {
7579
7890
  throw new Error(
7580
7891
  "Sandbox agent construction did not install the mandatory turn tool cancellation fence",
@@ -7629,6 +7940,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
7629
7940
  image: runSettings.modalImageRef ?? runSettings.dockerImage,
7630
7941
  }
7631
7942
  : {}),
7943
+ // The lazy acquire must enforce the same frozen rig authority
7944
+ // as the eager acquire; otherwise a warm box for another rig
7945
+ // can bypass the shared-state conflict/rotation fence.
7946
+ ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
7632
7947
  },
7633
7948
  "turn",
7634
7949
  lazyHolderId,
@@ -8083,69 +8398,109 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8083
8398
  return claimedResult({ status: "idle" });
8084
8399
  }
8085
8400
  }
8401
+ recordTurnStartupPhase(observability, {
8402
+ phase: "post_agent_preparation",
8403
+ provider: turnExecutionPolicy.providerId,
8404
+ backend: activeSandboxBackend ?? groupBoxBackend,
8405
+ outcome: "completed",
8406
+ durationSeconds: (performance.now() - postAgentPreparationStartedAt) / 1_000,
8407
+ });
8086
8408
  let fileMaterializationFailures: SandboxFileDownloadFailure[] = [];
8087
8409
  let fileDownloadsMaterializedForRun = false;
8088
8410
  if (resolvedSandbox && setupBoxSession && fileResourceDownloads.length > 0) {
8089
- const boxInstanceId = resolvedSandbox.established.instanceId;
8090
- // Managed boxes are immutable platform state, so their durable lease can
8091
- // memoize successful downloads. A connected machine is user-owned: files
8092
- // can be changed or removed between turns, so verify/materialize every
8093
- // attached file each turn instead of trusting the managed-box cache.
8411
+ const fileMaterializationStartedAt = performance.now();
8412
+ let fileMaterializationOutcome: "completed" | "failed" = "completed";
8094
8413
  const cacheMaterialization = resolvedSandbox.established.backendId !== "selfhosted";
8095
- const alreadyMaterialized = cacheMaterialization
8096
- ? await getMaterializedSandboxFileResources(db, {
8097
- accountId: input.accountId,
8098
- workspaceId: input.workspaceId,
8099
- sandboxGroupId: session.sandboxGroupId,
8100
- expectedEpoch: resolvedSandbox.leaseEpoch,
8101
- instanceId: boxInstanceId,
8102
- })
8103
- : new Set<string>();
8104
- const downloadsToMaterialize = filterUnmaterializedSandboxFileDownloads(
8105
- fileResourceDownloads,
8106
- alreadyMaterialized,
8107
- );
8108
- const runAs = sandboxRunAs(runSettings);
8109
- if (downloadsToMaterialize.length > 0) {
8110
- const materialized = await runWorkspaceMutationForSandbox(
8111
- resolvedSandbox,
8112
- "fileMaterialization",
8113
- async () =>
8114
- await materializeSandboxFileDownloads(
8115
- setupBoxSession as CodemodeTokenWriterSession,
8116
- downloadsToMaterialize,
8117
- {
8118
- onRuntimeEvent: async (event) => {
8119
- await publish!([{ type: event.type, payload: event.payload }], true);
8120
- },
8121
- ...(runAs ? { runAs } : {}),
8122
- ...(toolCancellationFenceRef.current
8123
- ? {
8124
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
8125
- toolCancellationFenceRef.current,
8126
- ),
8127
- }
8128
- : {}),
8129
- },
8130
- ),
8414
+ let fileMaterializationCache: "hit" | "miss" | "disabled" = cacheMaterialization
8415
+ ? "miss"
8416
+ : "disabled";
8417
+ try {
8418
+ const boxInstanceId = resolvedSandbox.established.instanceId;
8419
+ // Managed boxes are immutable platform state, so their durable lease can
8420
+ // memoize successful downloads. A connected machine is user-owned: files
8421
+ // can be changed or removed between turns, so verify/materialize every
8422
+ // attached file each turn instead of trusting the managed-box cache.
8423
+ const alreadyMaterialized = cacheMaterialization
8424
+ ? await getMaterializedSandboxFileResources(db, {
8425
+ accountId: input.accountId,
8426
+ workspaceId: input.workspaceId,
8427
+ sandboxGroupId: session.sandboxGroupId,
8428
+ expectedEpoch: resolvedSandbox.leaseEpoch,
8429
+ instanceId: boxInstanceId,
8430
+ })
8431
+ : new Set<string>();
8432
+ const downloadsToMaterialize = filterUnmaterializedSandboxFileDownloads(
8433
+ fileResourceDownloads,
8434
+ alreadyMaterialized,
8131
8435
  );
8132
- fileMaterializationFailures = materialized.failures;
8133
- const failedFileIds = new Set(materialized.failures.map((failure) => failure.fileId));
8134
- const succeededFileIds = downloadsToMaterialize
8135
- .map((download) => download.fileId)
8136
- .filter((fileId) => !failedFileIds.has(fileId));
8137
- if (cacheMaterialization && succeededFileIds.length > 0) {
8138
- await markSandboxFileResourcesMaterialized(db, {
8139
- accountId: input.accountId,
8140
- workspaceId: input.workspaceId,
8141
- sandboxGroupId: session.sandboxGroupId,
8142
- expectedEpoch: resolvedSandbox.leaseEpoch,
8143
- instanceId: boxInstanceId,
8144
- fileIds: succeededFileIds,
8145
- });
8436
+ if (cacheMaterialization && downloadsToMaterialize.length === 0) {
8437
+ fileMaterializationCache = "hit";
8438
+ }
8439
+ const runAs = sandboxRunAs(runSettings);
8440
+ if (downloadsToMaterialize.length > 0) {
8441
+ const materialized = await runWorkspaceMutationForSandbox(
8442
+ resolvedSandbox,
8443
+ "fileMaterialization",
8444
+ async () =>
8445
+ await materializeSandboxFileDownloads(
8446
+ setupBoxSession as CodemodeTokenWriterSession,
8447
+ downloadsToMaterialize,
8448
+ {
8449
+ onRuntimeEvent: async (event) => {
8450
+ await publish!([{ type: event.type, payload: event.payload }], true);
8451
+ },
8452
+ ...(runAs ? { runAs } : {}),
8453
+ ...(toolCancellationFenceRef.current
8454
+ ? {
8455
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
8456
+ toolCancellationFenceRef.current,
8457
+ ),
8458
+ }
8459
+ : {}),
8460
+ },
8461
+ ),
8462
+ );
8463
+ fileMaterializationFailures = materialized.failures;
8464
+ fileMaterializationOutcome = sandboxFileMaterializationOutcome(materialized.failures);
8465
+ const failedFileIds = new Set(materialized.failures.map((failure) => failure.fileId));
8466
+ const succeededFileIds = downloadsToMaterialize
8467
+ .map((download) => download.fileId)
8468
+ .filter((fileId) => !failedFileIds.has(fileId));
8469
+ if (cacheMaterialization && succeededFileIds.length > 0) {
8470
+ await markSandboxFileResourcesMaterialized(db, {
8471
+ accountId: input.accountId,
8472
+ workspaceId: input.workspaceId,
8473
+ sandboxGroupId: session.sandboxGroupId,
8474
+ expectedEpoch: resolvedSandbox.leaseEpoch,
8475
+ instanceId: boxInstanceId,
8476
+ fileIds: succeededFileIds,
8477
+ });
8478
+ }
8146
8479
  }
8480
+ fileDownloadsMaterializedForRun = true;
8481
+ } catch (error) {
8482
+ fileMaterializationOutcome = "failed";
8483
+ throw error;
8484
+ } finally {
8485
+ recordTurnStartupPhase(observability, {
8486
+ phase: "file_materialization",
8487
+ provider: turnExecutionPolicy.providerId,
8488
+ backend: activeSandboxBackend ?? groupBoxBackend,
8489
+ outcome: fileMaterializationOutcome,
8490
+ durationSeconds: (performance.now() - fileMaterializationStartedAt) / 1_000,
8491
+ count: fileResourceDownloads.length,
8492
+ cache: fileMaterializationCache,
8493
+ });
8147
8494
  }
8148
- fileDownloadsMaterializedForRun = true;
8495
+ } else {
8496
+ recordTurnStartupPhase(observability, {
8497
+ phase: "file_materialization",
8498
+ provider: turnExecutionPolicy.providerId,
8499
+ backend: activeSandboxBackend ?? groupBoxBackend,
8500
+ outcome: "completed",
8501
+ durationSeconds: 0,
8502
+ count: 0,
8503
+ });
8149
8504
  }
8150
8505
  const requiredVideoFileIds = new Set(requiredGeneratedVideoFiles.map((file) => file.fileId));
8151
8506
  const failedRequiredVideo = fileMaterializationFailures.find((failure) =>
@@ -8190,52 +8545,110 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8190
8545
  }
8191
8546
  let runInput: Awaited<ReturnType<typeof turnInput>>["input"] | null = null;
8192
8547
  const prepareRunAttemptInput = async () => {
8193
- const prepared = await turnInput(db, runtime, agent, trigger, {
8194
- turnId: activeTurnId,
8195
- recovering: turn.executionGeneration > 1,
8196
- ...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
8197
- ...(runCredentialsNote ? { runCredentialsNote } : {}),
8198
- providerApi,
8199
- projectCanonicalHistory: generatedImageHistoryProjector,
8200
- materializeModelHistory: materializeScreenshotHistory,
8201
- materializeSerializedRunState: materializeScreenshotRunState,
8202
- projectModelHistory: modelHistoryProjector,
8203
- });
8204
- for (const receipt of generatedImageReceiptsByArtifactId.values()) {
8205
- await materializeGeneratedImage(receipt);
8548
+ const historyPreparationStartedAt = performance.now();
8549
+ let historyPreparationOutcome: "completed" | "failed" = "completed";
8550
+ let preparedHistoryCount: number | null = null;
8551
+ try {
8552
+ const prepared = await turnInput(db, runtime, agent, trigger, {
8553
+ turnId: activeTurnId,
8554
+ recovering: turn.executionGeneration > 1,
8555
+ ...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
8556
+ ...(runCredentialsNote ? { runCredentialsNote } : {}),
8557
+ providerApi,
8558
+ projectCanonicalHistory: generatedImageHistoryProjector,
8559
+ materializeModelHistory: materializeScreenshotHistory,
8560
+ materializeSerializedRunState: materializeScreenshotRunState,
8561
+ projectModelHistory: modelHistoryProjector,
8562
+ onPreparationPhase: (measurement) => {
8563
+ recordTurnStartupPhase(observability, {
8564
+ phase: `history_${measurement.phase}`,
8565
+ provider: turnExecutionPolicy.providerId,
8566
+ backend: activeSandboxBackend ?? groupBoxBackend,
8567
+ outcome: measurement.outcome,
8568
+ durationSeconds: measurement.durationSeconds,
8569
+ });
8570
+ },
8571
+ });
8572
+ const generatedImageMaterializationStartedAt = performance.now();
8573
+ let generatedImageMaterializationOutcome: "completed" | "failed" = "completed";
8574
+ try {
8575
+ for (const receipt of generatedImageReceiptsByArtifactId.values()) {
8576
+ await materializeGeneratedImage(receipt);
8577
+ }
8578
+ } catch (error) {
8579
+ generatedImageMaterializationOutcome = "failed";
8580
+ throw error;
8581
+ } finally {
8582
+ recordTurnStartupPhase(observability, {
8583
+ phase: "history_generated_image_materialization",
8584
+ provider: turnExecutionPolicy.providerId,
8585
+ backend: activeSandboxBackend ?? groupBoxBackend,
8586
+ outcome: generatedImageMaterializationOutcome,
8587
+ durationSeconds: (performance.now() - generatedImageMaterializationStartedAt) / 1_000,
8588
+ count: generatedImageReceiptsByArtifactId.size,
8589
+ });
8590
+ }
8591
+ runInput = prepared.input;
8592
+ providerArtifactCandidates = prepared.providerArtifactCandidates;
8593
+ // Slice index = the length of the model-facing (active) history this turn
8594
+ // is seeded from; new items beyond it (the trigger message + this turn's
8595
+ // generated items) are the ones to persist. After a compaction this is the
8596
+ // short [summary, ...tail] active set, NOT the total row count. The
8597
+ // absolute write position is tracked separately (next whole number past
8598
+ // the max existing position) because the fractional summary row means
8599
+ // total rows no longer equal max(position)+1. Pre-compaction both reduce to
8600
+ // the old total-count value, so the common path is unchanged.
8601
+ //
8602
+ // CRITICAL: seed from the structurally repaired active-row length, not the raw active
8603
+ // count. `prepareRunInput` builds `state.history` from
8604
+ // `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
8605
+ // rows (a legacy orphan/dangling pair), the in-memory history this turn
8606
+ // starts from is K shorter than the raw row count. The reconcile slices the
8607
+ // repaired `state.history` off `persistedHistoryCount`; seeding it from
8608
+ // the raw count (K too high) skips K genuinely-new items, and a
8609
+ // `function_call` left in that skipped region can later have its
8610
+ // `function_call_result` persisted alone — the orphan that 400s on replay
8611
+ // and bricks the session (issue-61). The repaired seed is already
8612
+ // orphan-free, so it is a stable prefix of the re-sanitized history and the
8613
+ // slice begins exactly at the first genuinely-new item.
8614
+ // prepareInput already sanitized the exact durable prefix represented
8615
+ // by state.history. Carry its count forward instead of loading and
8616
+ // retaining the full active transcript a second time beside runInput.
8617
+ persistedHistoryCount = prepared.persistedHistoryCount;
8618
+ preparedHistoryCount = prepared.persistedHistoryCount;
8619
+ const historyPositionStartedAt = performance.now();
8620
+ let historyPositionOutcome: "completed" | "failed" = "completed";
8621
+ try {
8622
+ nextHistoryPosition = await nextSessionHistoryPosition(
8623
+ db,
8624
+ input.workspaceId,
8625
+ input.sessionId,
8626
+ );
8627
+ } catch (error) {
8628
+ historyPositionOutcome = "failed";
8629
+ throw error;
8630
+ } finally {
8631
+ recordTurnStartupPhase(observability, {
8632
+ phase: "history_position_load",
8633
+ provider: turnExecutionPolicy.providerId,
8634
+ backend: activeSandboxBackend ?? groupBoxBackend,
8635
+ outcome: historyPositionOutcome,
8636
+ durationSeconds: (performance.now() - historyPositionStartedAt) / 1_000,
8637
+ });
8638
+ }
8639
+ } catch (error) {
8640
+ historyPreparationOutcome = "failed";
8641
+ throw error;
8642
+ } finally {
8643
+ recordTurnStartupPhase(observability, {
8644
+ phase: "history_preparation",
8645
+ provider: turnExecutionPolicy.providerId,
8646
+ backend: activeSandboxBackend ?? groupBoxBackend,
8647
+ outcome: historyPreparationOutcome,
8648
+ durationSeconds: (performance.now() - historyPreparationStartedAt) / 1_000,
8649
+ count: preparedHistoryCount,
8650
+ });
8206
8651
  }
8207
- runInput = prepared.input;
8208
- providerArtifactCandidates = prepared.providerArtifactCandidates;
8209
- // Slice index = the length of the model-facing (active) history this turn
8210
- // is seeded from; new items beyond it (the trigger message + this turn's
8211
- // generated items) are the ones to persist. After a compaction this is the
8212
- // short [summary, ...tail] active set, NOT the total row count. The
8213
- // absolute write position is tracked separately (next whole number past
8214
- // the max existing position) because the fractional summary row means
8215
- // total rows no longer equal max(position)+1. Pre-compaction both reduce to
8216
- // the old total-count value, so the common path is unchanged.
8217
- //
8218
- // CRITICAL: seed from the structurally repaired active-row length, not the raw active
8219
- // count. `prepareRunInput` builds `state.history` from
8220
- // `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
8221
- // rows (a legacy orphan/dangling pair), the in-memory history this turn
8222
- // starts from is K shorter than the raw row count. The reconcile slices the
8223
- // repaired `state.history` off `persistedHistoryCount`; seeding it from
8224
- // the raw count (K too high) skips K genuinely-new items, and a
8225
- // `function_call` left in that skipped region can later have its
8226
- // `function_call_result` persisted alone — the orphan that 400s on replay
8227
- // and bricks the session (issue-61). The repaired seed is already
8228
- // orphan-free, so it is a stable prefix of the re-sanitized history and the
8229
- // slice begins exactly at the first genuinely-new item.
8230
- // prepareInput already sanitized the exact durable prefix represented
8231
- // by state.history. Carry its count forward instead of loading and
8232
- // retaining the full active transcript a second time beside runInput.
8233
- persistedHistoryCount = prepared.persistedHistoryCount;
8234
- nextHistoryPosition = await nextSessionHistoryPosition(
8235
- db,
8236
- input.workspaceId,
8237
- input.sessionId,
8238
- );
8239
8652
  };
8240
8653
 
8241
8654
  const forceContextCompaction = async (
@@ -8287,6 +8700,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8287
8700
  // calling runStreamAttempt again; resetting this state there would reuse
8288
8701
  // the first no-response-ID fallback key and suppress a real model call.
8289
8702
  const modelResponseState = createModelResponseEventState(claimedModelUsageSourceKeys);
8703
+ let workerPreparationTotalRecorded = false;
8290
8704
  const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
8291
8705
  if (!runInput) {
8292
8706
  throw new Error("Run input was not prepared");
@@ -8313,139 +8727,203 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8313
8727
  // heartbeat capture. Lazy setup already runs under the same wrapper in
8314
8728
  // its first-operation provisioner above.
8315
8729
  if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
8316
- const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
8317
- // `deferredSetup: true` below tells the runtime that the worker owns
8318
- // platform setup, so the runtime intentionally skips its credential
8319
- // session callback. Materialize host-managed run credentials here
8320
- // before any setup command (including provider login hooks), then
8321
- // decorate setup commands so they source the active generation.
8322
- await attachRunCredentialRenewal(
8323
- eagerSetupSession as RunCredentialCommandSession,
8324
- resolvedSandbox,
8325
- );
8326
- const eagerCredentialSetupSession = initialRunCredentialMaterial
8327
- ? withRunCredentialsSession(eagerSetupSession as object, input.sessionId)
8328
- : eagerSetupSession;
8329
- await runWorkspaceMutationForSandbox(
8330
- resolvedSandbox,
8331
- "eagerOwnedSandboxSetup",
8332
- async () =>
8333
- await runOwnedSandboxSetup(
8334
- agent,
8335
- ownedEstablished.session as never,
8336
- eagerCredentialSetupSession as never,
8337
- {
8338
- settings: runSettings,
8339
- environment: sandboxEnvironment,
8340
- preparedInput: runInput!,
8341
- ...(fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {}),
8342
- onRuntimeEvent: async (event) => {
8343
- await renewServingCredentialLease("runtime_event");
8344
- if (servingCredentialLeaseLost()) {
8345
- throw new Error("Provider credential lease expired during sandbox setup");
8346
- }
8347
- await publish!([{ type: event.type, payload: event.payload }], true);
8348
- },
8349
- ...(toolCancellationFenceRef.current
8350
- ? {
8351
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
8352
- toolCancellationFenceRef.current,
8353
- ),
8730
+ const ownedSandboxSetupStartedAt = performance.now();
8731
+ let ownedSandboxSetupOutcome: "completed" | "failed" = "completed";
8732
+ try {
8733
+ const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
8734
+ // `deferredSetup: true` below tells the runtime that the worker owns
8735
+ // platform setup, so the runtime intentionally skips its credential
8736
+ // session callback. Materialize host-managed run credentials here
8737
+ // before any setup command (including provider login hooks), then
8738
+ // decorate setup commands so they source the active generation.
8739
+ await attachRunCredentialRenewal(
8740
+ eagerSetupSession as RunCredentialCommandSession,
8741
+ resolvedSandbox,
8742
+ );
8743
+ const eagerCredentialSetupSession = initialRunCredentialMaterial
8744
+ ? withRunCredentialsSession(eagerSetupSession as object, input.sessionId)
8745
+ : eagerSetupSession;
8746
+ await runWorkspaceMutationForSandbox(
8747
+ resolvedSandbox,
8748
+ "eagerOwnedSandboxSetup",
8749
+ async () =>
8750
+ await runOwnedSandboxSetup(
8751
+ agent,
8752
+ ownedEstablished.session as never,
8753
+ eagerCredentialSetupSession as never,
8754
+ {
8755
+ settings: runSettings,
8756
+ environment: sandboxEnvironment,
8757
+ preparedInput: runInput!,
8758
+ ...(fileDownloadsMaterializedForRun
8759
+ ? { fileDownloadsMaterialized: true }
8760
+ : {}),
8761
+ onRuntimeEvent: async (event) => {
8762
+ await renewServingCredentialLease("runtime_event");
8763
+ if (servingCredentialLeaseLost()) {
8764
+ throw new Error("Provider credential lease expired during sandbox setup");
8354
8765
  }
8355
- : {}),
8356
- },
8357
- ),
8358
- );
8359
- await attachGitCredentialRenewal(
8360
- eagerSetupSession as GitCredentialTokenWriterSession,
8361
- initialGitCredentials,
8362
- );
8766
+ await publish!([{ type: event.type, payload: event.payload }], true);
8767
+ },
8768
+ ...(toolCancellationFenceRef.current
8769
+ ? {
8770
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
8771
+ toolCancellationFenceRef.current,
8772
+ ),
8773
+ }
8774
+ : {}),
8775
+ },
8776
+ ),
8777
+ );
8778
+ await attachGitCredentialRenewal(
8779
+ eagerSetupSession as GitCredentialTokenWriterSession,
8780
+ initialGitCredentials,
8781
+ );
8782
+ } catch (error) {
8783
+ ownedSandboxSetupOutcome = "failed";
8784
+ throw error;
8785
+ } finally {
8786
+ recordTurnStartupPhase(observability, {
8787
+ phase: "owned_sandbox_setup",
8788
+ provider: turnExecutionPolicy.providerId,
8789
+ backend: activeSandboxBackend ?? groupBoxBackend,
8790
+ outcome: ownedSandboxSetupOutcome,
8791
+ durationSeconds: (performance.now() - ownedSandboxSetupStartedAt) / 1_000,
8792
+ count: fileResourceDownloads.length,
8793
+ });
8794
+ }
8363
8795
  }
8364
8796
  // Conservative request boundary: once control enters runStream, the
8365
8797
  // provider may have accepted work even if no response/event follows.
8366
8798
  // Escaped MCP setup timeouts are automatically recoverable only
8367
8799
  // before this line.
8368
8800
  recordCompanyBrainContributionReceiptOnce();
8369
- modelRequestStarted = true;
8370
- return await runtime.runStream(agent, runInput!, modelRunSettings, {
8371
- signal: runtimeCancellationSignal,
8372
- sandboxEnvironment,
8373
- onRuntimeEvent: async (event) => {
8374
- await renewServingCredentialLease("runtime_event");
8375
- if (servingCredentialLeaseLost()) {
8376
- throw new Error("Provider credential lease expired during the active turn");
8377
- }
8378
- await publish!([{ type: event.type, payload: event.payload }], true);
8379
- },
8380
- // P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
8381
- // keystone). Absent when the flag is off -> legacy build-and-discard.
8382
- ...(ownedEstablished
8383
- ? {
8384
- ownedSandbox: {
8385
- client: ownedEstablished.client,
8386
- session: ownedEstablished.session,
8387
- ...(resolvedSandbox?.established.sessionState
8801
+ const providerDispatchStartedAt = performance.now();
8802
+ let providerDispatchOutcome: "completed" | "failed" = "completed";
8803
+ try {
8804
+ // This histogram describes worker preparation until the first entry
8805
+ // into the runtime. Lazy SDK request preparation and the durable
8806
+ // model-request audit happen after this boundary and have their own
8807
+ // phase metrics. Context-compaction recovery re-enters
8808
+ // runStreamAttempt; recording again would fold the prior request and
8809
+ // compaction into a bogus second startup sample.
8810
+ if (!workerPreparationTotalRecorded) {
8811
+ workerPreparationTotalRecorded = true;
8812
+ recordTurnWorkerPreparationTotal(observability, {
8813
+ provider: turnExecutionPolicy.providerId,
8814
+ backend: activeSandboxBackend ?? groupBoxBackend,
8815
+ outcome: "completed",
8816
+ durationSeconds: (performance.now() - workerPreparationStartedAt) / 1_000,
8817
+ });
8818
+ }
8819
+ modelRequestStarted = true;
8820
+ return await runtime.runStream(agent, runInput!, modelRunSettings, {
8821
+ signal: runtimeCancellationSignal,
8822
+ sandboxEnvironment,
8823
+ onRuntimeEvent: async (event) => {
8824
+ await renewServingCredentialLease("runtime_event");
8825
+ if (servingCredentialLeaseLost()) {
8826
+ throw new Error("Provider credential lease expired during the active turn");
8827
+ }
8828
+ await publish!([{ type: event.type, payload: event.payload }], true);
8829
+ },
8830
+ // P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
8831
+ // keystone). Absent when the flag is off -> legacy build-and-discard.
8832
+ ...(ownedEstablished
8833
+ ? {
8834
+ ownedSandbox: {
8835
+ client: ownedEstablished.client,
8836
+ session: ownedEstablished.session,
8837
+ ...(resolvedSandbox?.established.sessionState
8838
+ ? {
8839
+ sessionState: resolvedSandbox.established.sessionState,
8840
+ }
8841
+ : {}),
8842
+ // Pin platform setup (hooks + file materialization) to the un-proxied
8843
+ // established box — never through the routing proxy, which would
8844
+ // re-route those execs onto a machine swapped in mid-turn.
8845
+ ...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
8846
+ ...(fileDownloadsMaterializedForRun
8847
+ ? { fileDownloadsMaterialized: true }
8848
+ : {}),
8849
+ // Both owned paths execute setup outside runStream: eager just
8850
+ // above under its exact admission, lazy in the provisioner.
8851
+ deferredSetup: true,
8852
+ },
8853
+ }
8854
+ : {}),
8855
+ ...(activeSandboxBackend !== "selfhosted" &&
8856
+ sandboxCodemodeToken &&
8857
+ sandboxCodemodeTokenExpiresAt &&
8858
+ !lazyOwnedSandbox
8859
+ ? {
8860
+ onCodemodeTokenSessionReady: async (
8861
+ tokenSession: CodemodeTokenWriterSession,
8862
+ ) => {
8863
+ const renewalSession =
8864
+ (setupBoxSession as CodemodeTokenWriterSession | null) ?? tokenSession;
8865
+ await attachCodemodeTokenRenewal(renewalSession);
8866
+ },
8867
+ }
8868
+ : {}),
8869
+ ...(runCredentialResolver
8870
+ ? {
8871
+ runCredentialSessionId: input.sessionId,
8872
+ ...(!lazyOwnedSandbox
8388
8873
  ? {
8389
- sessionState: resolvedSandbox.established.sessionState,
8874
+ onRunCredentialSessionReady: async (
8875
+ credentialSession: RunCredentialCommandSession,
8876
+ ) => {
8877
+ const pinnedCredentialSession = setupBoxSession
8878
+ ? (setupBoxSession as RunCredentialCommandSession)
8879
+ : credentialSession;
8880
+ await attachRunCredentialRenewal(pinnedCredentialSession);
8881
+ },
8390
8882
  }
8391
8883
  : {}),
8392
- // Pin platform setup (hooks + file materialization) to the un-proxied
8393
- // established box — never through the routing proxy, which would
8394
- // re-route those execs onto a machine swapped in mid-turn.
8395
- ...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
8396
- ...(fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {}),
8397
- // Both owned paths execute setup outside runStream: eager just
8398
- // above under its exact admission, lazy in the provisioner.
8399
- deferredSetup: true,
8400
- },
8401
- }
8402
- : {}),
8403
- ...(activeSandboxBackend !== "selfhosted" &&
8404
- sandboxCodemodeToken &&
8405
- sandboxCodemodeTokenExpiresAt &&
8406
- !lazyOwnedSandbox
8407
- ? {
8408
- onCodemodeTokenSessionReady: async (tokenSession: CodemodeTokenWriterSession) => {
8409
- const renewalSession =
8410
- (setupBoxSession as CodemodeTokenWriterSession | null) ?? tokenSession;
8411
- await attachCodemodeTokenRenewal(renewalSession);
8412
- },
8413
- }
8414
- : {}),
8415
- ...(runCredentialResolver
8416
- ? {
8417
- runCredentialSessionId: input.sessionId,
8418
- ...(!lazyOwnedSandbox
8419
- ? {
8420
- onRunCredentialSessionReady: async (
8421
- credentialSession: RunCredentialCommandSession,
8422
- ) => {
8423
- const pinnedCredentialSession = setupBoxSession
8424
- ? (setupBoxSession as RunCredentialCommandSession)
8425
- : credentialSession;
8426
- await attachRunCredentialRenewal(pinnedCredentialSession);
8427
- },
8884
+ }
8885
+ : {}),
8886
+ ...(modelRunSettings.sandboxBackend !== "none"
8887
+ ? {
8888
+ onSandboxSessionReady: async (sandboxSession: CodemodeTokenWriterSession) => {
8889
+ sdkOwnedSandboxSession = sandboxSession;
8890
+ for (const receipt of generatedImageReceiptsByArtifactId.values()) {
8891
+ await materializeGeneratedImageInOwnedSdkSession(receipt, sandboxSession);
8428
8892
  }
8429
- : {}),
8430
- }
8431
- : {}),
8432
- ...(modelRunSettings.sandboxBackend !== "none"
8433
- ? {
8434
- onSandboxSessionReady: async (sandboxSession: CodemodeTokenWriterSession) => {
8435
- sdkOwnedSandboxSession = sandboxSession;
8436
- for (const receipt of generatedImageReceiptsByArtifactId.values()) {
8437
- await materializeGeneratedImageInOwnedSdkSession(receipt, sandboxSession);
8438
- }
8439
- },
8440
- }
8441
- : {}),
8442
- contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
8443
- contextCompactionRequested: () =>
8444
- isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
8445
- ...(toolCancellationFenceRef.current
8446
- ? { turnToolCancellationFence: toolCancellationFenceRef.current }
8447
- : {}),
8448
- });
8893
+ },
8894
+ }
8895
+ : {}),
8896
+ contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
8897
+ contextCompactionRequested: () =>
8898
+ isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
8899
+ onModelPreparationPhase: (measurement) => {
8900
+ recordTurnStartupPhase(observability, {
8901
+ phase: `model_prepare_${measurement.phase}`,
8902
+ provider: turnExecutionPolicy.providerId,
8903
+ backend: activeSandboxBackend ?? groupBoxBackend,
8904
+ outcome: measurement.outcome,
8905
+ durationSeconds: measurement.durationSeconds,
8906
+ ...(measurement.count === undefined ? {} : { count: measurement.count }),
8907
+ });
8908
+ },
8909
+ ...(toolCancellationFenceRef.current
8910
+ ? {
8911
+ turnToolCancellationFence: toolCancellationFenceRef.current,
8912
+ }
8913
+ : {}),
8914
+ });
8915
+ } catch (error) {
8916
+ providerDispatchOutcome = "failed";
8917
+ throw error;
8918
+ } finally {
8919
+ recordTurnStartupPhase(observability, {
8920
+ phase: "provider_dispatch",
8921
+ provider: turnExecutionPolicy.providerId,
8922
+ backend: activeSandboxBackend ?? groupBoxBackend,
8923
+ outcome: providerDispatchOutcome,
8924
+ durationSeconds: (performance.now() - providerDispatchStartedAt) / 1_000,
8925
+ });
8926
+ }
8449
8927
  };
8450
8928
  if (codexLeaseLost) {
8451
8929
  throw new Error("Codex credential lease expired before the model run");
@@ -8471,11 +8949,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
8471
8949
  },
8472
8950
  );
8473
8951
 
8952
+ const streamBootstrapStartedAt = performance.now();
8953
+ let streamBootstrapRecorded = false;
8954
+ const recordStreamBootstrap = (): void => {
8955
+ if (streamBootstrapRecorded) return;
8956
+ streamBootstrapRecorded = true;
8957
+ recordTurnStartupPhase(observability, {
8958
+ phase: "stream_bootstrap",
8959
+ provider: turnExecutionPolicy.providerId,
8960
+ backend: activeSandboxBackend ?? groupBoxBackend,
8961
+ outcome: "completed",
8962
+ durationSeconds: (performance.now() - streamBootstrapStartedAt) / 1_000,
8963
+ count: turnTools.length,
8964
+ });
8965
+ };
8966
+ if (!firstModelRequestPreparationRecorded) {
8967
+ firstModelRequestPreparationStartedAt = performance.now();
8968
+ firstModelRequestCheckpointAt = firstModelRequestPreparationStartedAt;
8969
+ }
8474
8970
  const iterator = stream.toStream()[Symbol.asyncIterator]();
8475
8971
  let streamDone = false;
8476
8972
  try {
8477
8973
  while (true) {
8478
8974
  const next = await nextStreamEvent(iterator, activityContext);
8975
+ recordStreamBootstrap();
8479
8976
  if (next.done) {
8480
8977
  streamDone = true;
8481
8978
  break;
@@ -10437,7 +10934,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
10437
10934
  }
10438
10935
  activityStatus = "failed";
10439
10936
  activityError = error;
10440
- if (!publish || !turnId || !turnStartedPublished) {
10937
+ if (!turnId) {
10938
+ throw preClaimAdmissionFailure(error);
10939
+ }
10940
+ if (!publish || !turnStartedPublished) {
10441
10941
  throw error;
10442
10942
  }
10443
10943
  // A partial/malformed stream may have emitted assistant/tool items (and