@opengeni/worker-bundle 0.7.5 → 0.10.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist/chunk-BJUW5LCM.js +325 -0
  2. package/dist/chunk-BJUW5LCM.js.map +1 -0
  3. package/dist/host-export.d.ts +70 -0
  4. package/dist/host-export.js +7 -0
  5. package/dist/host-export.js.map +1 -0
  6. package/dist/index.d.ts +98 -6
  7. package/dist/index.js +4097 -1271
  8. package/dist/index.js.map +1 -1
  9. package/dist/workflow-bundle.js +12360 -0
  10. package/package.json +18 -12
  11. package/src/activities/agent-turn.ts +2674 -828
  12. package/src/activities/capabilities.ts +11 -2
  13. package/src/activities/codex-capacity.ts +6 -11
  14. package/src/activities/codex-rotation.ts +75 -86
  15. package/src/activities/context-compaction.ts +104 -24
  16. package/src/activities/environment.ts +503 -97
  17. package/src/activities/expiring-material-renewal.ts +143 -0
  18. package/src/activities/goals.ts +4 -0
  19. package/src/activities/mcp-credentials.ts +38 -0
  20. package/src/activities/packs.ts +115 -2
  21. package/src/activities/rig-verification.ts +897 -244
  22. package/src/activities/run-credential-renewal.ts +70 -0
  23. package/src/activities/run-credentials.ts +132 -0
  24. package/src/activities/run-input.ts +23 -0
  25. package/src/activities/sandbox-lease.ts +33 -2
  26. package/src/activities/scheduled-tasks.ts +21 -0
  27. package/src/activities/session-state.ts +72 -0
  28. package/src/activities/streaming.ts +1 -0
  29. package/src/activities/toolspace-token-renewal.ts +57 -0
  30. package/src/activities/types.ts +44 -6
  31. package/src/activities/workspace-capture.ts +113 -93
  32. package/src/activities.ts +10 -2
  33. package/src/host-export-pump.ts +416 -0
  34. package/src/host-export.ts +22 -0
  35. package/src/http.ts +77 -28
  36. package/src/index.ts +360 -156
  37. package/src/observability-metrics.ts +83 -15
  38. package/src/sandbox-resume.ts +47 -1
  39. package/src/sandbox-routing.ts +1 -1
  40. package/src/worker-service-lifecycle.ts +104 -0
  41. package/src/workflows/activities.ts +13 -7
  42. package/src/workflows/rig-verification.ts +3 -1
  43. package/src/workflows/session.ts +219 -23
  44. package/src/workflows.ts +1 -0
@@ -11,9 +11,12 @@ import {
11
11
  setActiveSandbox,
12
12
  requireFile,
13
13
  getSessionEvent,
14
+ getSessionRootId,
14
15
  getSessionGoal,
15
16
  getLatestRunState,
16
- isCodexBilledTurn,
17
+ getHumanInputResumeForEvent,
18
+ getSessionHumanInputRequest,
19
+ installOrReadTurnExecutionPolicyForAttempt,
17
20
  workspaceCodexSubscriptionActive,
18
21
  acquireCodexCredentialLease,
19
22
  armCodexCapacityWait,
@@ -47,7 +50,6 @@ import {
47
50
  nextSessionHistoryPosition,
48
51
  settleCodexCredentialLeaseLoss,
49
52
  settleCodexCredentialFailover,
50
- saveRunState,
51
53
  upsertSandboxSessionEnvelope,
52
54
  setSessionLastInputTokensForTurnAttempt,
53
55
  sumUsageQuantity,
@@ -55,24 +57,27 @@ import {
55
57
  accrueWarmSeconds,
56
58
  getMaterializedSandboxFileResources,
57
59
  markSandboxFileResourcesMaterialized,
60
+ areGitHubRepositoriesAllowedForWorkspace,
58
61
  SandboxLeaseSupersededError,
59
62
  SandboxImageConflictError,
60
- buildConnectionTokenResolver,
63
+ isSessionEventPersistenceError,
61
64
  getEnrollment,
62
65
  abandonRecordingForTurnAttempt,
66
+ markSessionAttemptQuiesced,
63
67
  type AppendEventInput,
64
68
  type ActiveSandboxPointer,
65
69
  type SandboxRecord,
66
70
  type CodexCredentialLeaseResult,
67
71
  type CodexCredentialLeaseSelectionContext,
72
+ type ApplySessionTurnSettlementInput,
68
73
  type SessionTurnRecordingSettlement,
69
74
  } from "@opengeni/db";
70
75
  import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@opengeni/events";
71
76
  import {
72
77
  sandboxStateEntryFromRunState,
73
78
  maxTurnsExceededRunState,
74
- modelCallUsageTelemetry,
75
79
  modelResponseUsageFromSdkEvent,
80
+ normalizeModelCallUsage,
76
81
  normalizeSdkEvent,
77
82
  sanitizeHistoryItemsForModel,
78
83
  isEphemeralInternalContext,
@@ -81,10 +86,18 @@ import {
81
86
  appendWorkspaceMemory,
82
87
  composeAgentInstructions,
83
88
  summarizeForCompaction,
89
+ CompactionProviderResponseError,
90
+ EmptyCompactionSummaryError,
84
91
  ensureModalRegistryImage,
85
92
  findCompactionNeededError,
86
93
  materializeSandboxFileDownloads,
87
- refreshGitProviderTokenFiles,
94
+ materializeRunCredentials,
95
+ clearRunCredentials,
96
+ clearRunCredentialsForAttempt,
97
+ withRunCredentialsSession,
98
+ refreshGitCredentialBindingTokenFiles,
99
+ refreshToolspaceTokenFile,
100
+ toolspaceTokenFileFromEnvironment,
88
101
  sandboxFileDownloadFailureNote,
89
102
  SUMMARY_BUFFER_TOKENS,
90
103
  runOwnedSandboxSetup,
@@ -94,20 +107,28 @@ import {
94
107
  type OpenGeniRuntime,
95
108
  type ComputerToolMode,
96
109
  type ModelResponseUsage,
110
+ type ModelCallUsageInput,
111
+ type ModelCallUsageNormalization,
97
112
  type BuildAgentOptions,
113
+ type TurnToolCancellationFence,
98
114
  type BackendUnresolvableCode,
99
115
  type EstablishedSandboxSession,
100
116
  type GitCredentialTokenWriterSession,
117
+ type NormalizedRunCredentialMaterial,
118
+ type RunCredentialCommandSession,
119
+ type ToolspaceTokenWriterSession,
101
120
  deleteRecordingArtifacts,
102
121
  stopRecording as stopRecordingOnBox,
103
122
  } from "@opengeni/runtime";
123
+ import { connectionTokenResolverForTurn } from "./mcp-credentials";
104
124
  import {
105
- builtinProviderId,
125
+ assertTurnExecutionPolicyMatchesConfigV1,
106
126
  calculateModelUsageCostMicros,
107
127
  configuredModelPricing,
108
128
  configuredStaticUsageLimits,
109
129
  sandboxWarmRateMicrosPerSecond,
110
130
  settingsWithResolvedModelContext,
131
+ resolveTurnExecutionPolicyV1,
111
132
  type ModelUsageInput,
112
133
  type ModelProviderApi,
113
134
  type RegistryProviderKind,
@@ -120,6 +141,7 @@ import {
120
141
  settingsWithSessionMcpServersForRun,
121
142
  } from "./capabilities";
122
143
  import {
144
+ CODEX_USAGE_EXHAUSTED_PCT,
123
145
  authoritativeCodexCapacityResetAt,
124
146
  chooseRotationActive,
125
147
  classifyCodexPin,
@@ -144,35 +166,59 @@ import {
144
166
  CODEX_CLIENT_VERSION,
145
167
  CODEX_FALLBACK_MODEL_SLUGS,
146
168
  CodexReloginRequired,
169
+ classifyCodexResponseTimeoutError,
147
170
  classifyCodexUsageLimitError,
148
171
  codexRequestStorage,
149
172
  isCodexTransportError,
150
173
  type CodexRequestContext,
151
174
  type CodexUsageHeaderSnapshot,
152
175
  } from "@opengeni/codex";
153
- import { mergeResourceRefs, mergeToolRefs } from "./common";
176
+ import { mergeResourceRefs } from "./common";
177
+ import { enabledCapabilityMcpToolRefs, resolveSessionToolPolicy } from "@opengeni/core";
154
178
  import { maybeCompactContext } from "./context-compaction";
155
179
  import { TurnAttemptFencedError } from "./turn-attempt-fenced";
156
180
  import {
181
+ assertGitCredentialRenewalTransportUnchanged,
182
+ gitCredentialAuthorityForTurn,
183
+ gitHubTokenMintSelections,
157
184
  loadWorkspaceEnvironmentForRunWithCredentials,
185
+ mintSandboxToolspaceToken,
158
186
  mintRunGitCredentials,
187
+ mintRunGitCredentialBinding,
159
188
  sandboxEnvironmentForRun,
189
+ type GitHubTokenMintAuthorization,
160
190
  type MintedRunGitCredentials,
161
191
  } from "./environment";
162
192
  import {
163
193
  startGitCredentialRenewalLoop,
164
194
  type GitCredentialRenewalController,
165
195
  } from "./git-credential-renewal";
196
+ import {
197
+ RUN_CREDENTIAL_EXPIRY_LEAD_MS,
198
+ startRunCredentialRenewalLoop,
199
+ type RunCredentialRenewalController,
200
+ } from "./run-credential-renewal";
201
+ import {
202
+ TOOLSPACE_TOKEN_EXPIRY_LEAD_MS,
203
+ startToolspaceTokenRenewalLoop,
204
+ type ToolspaceTokenRenewalController,
205
+ } from "./toolspace-token-renewal";
206
+ import {
207
+ bindRunCredentialResolver,
208
+ runCredentialAuthNeededPayloads,
209
+ runCredentialModelNote,
210
+ } from "./run-credentials";
166
211
  import { withCodexAppsTool, withFirstPartyTools } from "./goals";
167
212
  import {
168
213
  mergeRigDefaultVariableSetEnvironment,
169
214
  resolveWorkspaceAgentInstructions,
170
215
  resolveWorkspacePackRuntime,
216
+ resolveWorkspaceSkillLibraryRuntime,
171
217
  settingsWithPackSandboxImage,
172
218
  settingsWithRigImage,
173
219
  } from "./packs";
174
220
  import { deliverFailedChildTurnToParent } from "./parent-wake";
175
- import { createSecretRedactor, identityRedactor } from "./redaction";
221
+ import { createSecretRedactor, identityRedactor, type SecretForRedaction } from "./redaction";
176
222
  import { applyCodexHistoryStrip, turnInput, type TurnCodexAccount } from "./run-input";
177
223
  import {
178
224
  createRuntimeBatcher,
@@ -180,7 +226,12 @@ import {
180
226
  nextStreamEvent,
181
227
  startActivityHeartbeat,
182
228
  } from "./streaming";
183
- import type { ActivityServices, RunAgentTurnInput, RunAgentTurnResult } from "./types";
229
+ import type {
230
+ ActivityServices,
231
+ RunAgentTurnInput,
232
+ RunAgentTurnResult,
233
+ SessionAttemptQuiescenceProof,
234
+ } from "./types";
184
235
  import {
185
236
  resumeBoxForTurn,
186
237
  acquireSelfhostedLeaseForTurn,
@@ -229,11 +280,13 @@ import {
229
280
  import {
230
281
  CAPABILITY_DESCRIPTORS,
231
282
  evaluateWorkspaceModelPolicy,
232
- type GitCredentialProvider,
283
+ readTurnExecutionPolicyV1,
233
284
  type ResourceRef,
234
285
  type SessionEvent,
235
286
  type SessionEventType,
236
287
  type SessionStatus,
288
+ type SessionTurn,
289
+ type TurnExecutionPolicyV1,
237
290
  } from "@opengeni/contracts";
238
291
  import { createHash, randomUUID } from "node:crypto";
239
292
 
@@ -243,6 +296,32 @@ import { createHash, randomUUID } from "node:crypto";
243
296
  // budget against the same window.
244
297
  export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
245
298
 
299
+ export function turnExecutionPolicyBillingIdentity(policy: TurnExecutionPolicyV1): {
300
+ externallyBilled: boolean;
301
+ codexSubscription: boolean;
302
+ } {
303
+ return {
304
+ externallyBilled: policy.billing.metering === "external",
305
+ codexSubscription:
306
+ policy.providerId === "codex-subscription" &&
307
+ policy.credentialSource.kind === "connected_subscription" &&
308
+ policy.credentialSource.provider === "codex",
309
+ };
310
+ }
311
+
312
+ export function legacyTurnExecutionPolicyInput(
313
+ turn: Pick<SessionTurn, "source" | "model" | "reasoningEffort">,
314
+ ): Parameters<typeof resolveTurnExecutionPolicyV1>[1] {
315
+ const explicit = turn.source === "user" || turn.source === "api";
316
+ return {
317
+ modelId: turn.model,
318
+ requestedModelId: explicit ? turn.model : null,
319
+ modelSource: explicit ? "explicit" : "continuation",
320
+ reasoningEffort: turn.reasoningEffort,
321
+ reasoningSource: explicit ? "explicit" : "continuation",
322
+ };
323
+ }
324
+
246
325
  /** A retryable provider fault recovers the accepted turn itself. Goal state is
247
326
  * irrelevant: autonomous continuation and infrastructure recovery are separate
248
327
  * concerns. */
@@ -293,6 +372,28 @@ export function codexWorkspaceMetricKey(workspaceId: string): string {
293
372
  return createHash("sha256").update(workspaceId).digest("hex").slice(0, 12);
294
373
  }
295
374
 
375
+ /** Stable public request identity across partial resumes and activity retries. */
376
+ export function stableHumanInputRequestId(
377
+ sessionId: string,
378
+ turnId: string,
379
+ toolCallId: string,
380
+ ): string {
381
+ const hex = createHash("sha256")
382
+ .update("opengeni-human-input-v1\0")
383
+ .update(sessionId)
384
+ .update("\0")
385
+ .update(turnId)
386
+ .update("\0")
387
+ .update(toolCallId)
388
+ .digest("hex")
389
+ .slice(0, 32)
390
+ .split("");
391
+ hex[12] = "5";
392
+ hex[16] = ["8", "9", "a", "b"][Number.parseInt(hex[16] ?? "0", 16) % 4] ?? "8";
393
+ const value = hex.join("");
394
+ return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice(12, 16)}-${value.slice(16, 20)}-${value.slice(20)}`;
395
+ }
396
+
296
397
  /**
297
398
  * True when this activity attempt was cancelled because its hosting worker is
298
399
  * shutting down gracefully (SIGTERM during a deploy), as opposed to a
@@ -302,12 +403,229 @@ export function isWorkerShutdownCancellation(error: unknown): boolean {
302
403
  return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
303
404
  }
304
405
 
406
+ /**
407
+ * Review captures and protective snapshots are cache/persistence housekeeping,
408
+ * never part of cancellation correctness. A control-fenced or Temporal-cancelled
409
+ * attempt must release its physical activity promptly so Steer/Pause can advance.
410
+ */
411
+ export function shouldRunTurnEndWorkspacePersistence(input: {
412
+ activityStatus: RunAgentTurnResult["status"] | "unknown";
413
+ cancellationRequested: boolean;
414
+ }): boolean {
415
+ return input.activityStatus !== "cancelled" && !input.cancellationRequested;
416
+ }
417
+
418
+ /**
419
+ * Temporal cancellation is delivery/transport state, never proof that the
420
+ * dying activity crossed its mandatory sandbox-tool fence. If that fence
421
+ * fails, surface the fence failure instead of retaining a misleading typed
422
+ * cancellation; replacement admission remains closed because no durable
423
+ * quiescence receipt was written.
424
+ */
425
+ export function assertPhysicalToolQuiescenceForCancellation(input: {
426
+ acknowledgeQuiescence: boolean;
427
+ physicalToolQuiescenceConfirmed: boolean;
428
+ failure: unknown;
429
+ }): void {
430
+ if (!input.acknowledgeQuiescence || input.physicalToolQuiescenceConfirmed) return;
431
+ if (input.failure instanceof Error) throw input.failure;
432
+ throw new Error("Physical sandbox-tool quiescence could not be confirmed", {
433
+ cause: input.failure,
434
+ });
435
+ }
436
+
437
+ /** A physically drained attempt must leave one durable recovery producer. The
438
+ * direct Postgres receipt is preferred; after its bounded retries exhaust, an
439
+ * exact Temporal proof signal is sufficient because the workflow persists it
440
+ * through an independently retrying DB-only control activity. */
441
+ export function assertSessionAttemptQuiescenceRecoveryDurable(input: {
442
+ acknowledgeQuiescence: boolean;
443
+ physicalToolQuiescenceConfirmed: boolean;
444
+ receiptOrProofDurable: boolean;
445
+ failure: unknown;
446
+ }): void {
447
+ if (
448
+ !input.acknowledgeQuiescence ||
449
+ !input.physicalToolQuiescenceConfirmed ||
450
+ input.receiptOrProofDurable
451
+ ) {
452
+ return;
453
+ }
454
+ if (input.failure instanceof Error) throw input.failure;
455
+ throw new Error("Physical quiescence had no durable receipt or recovery proof", {
456
+ cause: input.failure,
457
+ });
458
+ }
459
+
460
+ const QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS = 250;
461
+ const QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS = 5_000;
462
+
463
+ /** Persist the authoritative receipt or durably hand the exact physical proof
464
+ * to Temporal. This retries signal delivery, not DB eligibility or workflow
465
+ * state. The proof object never changes between attempts. */
466
+ export async function persistOrSignalSessionAttemptQuiescence(input: {
467
+ proof: SessionAttemptQuiescenceProof;
468
+ persistReceipt: () => Promise<SessionEvent[]>;
469
+ publishEvents: (events: SessionEvent[]) => Promise<unknown>;
470
+ signalProof: ActivityServices["signalSessionAttemptQuiesced"];
471
+ sleep?: (ms: number) => Promise<void>;
472
+ heartbeat?: (attempt: number, delayMs: number) => void;
473
+ onReceiptFailure?: (error: unknown) => void;
474
+ onPublishFailure?: (error: unknown) => void;
475
+ onSignalFailure?: (error: unknown, attempt: number, delayMs: number) => void;
476
+ }): Promise<"receipt" | "signal"> {
477
+ let events: SessionEvent[];
478
+ try {
479
+ events = await input.persistReceipt();
480
+ } catch (receiptError) {
481
+ input.onReceiptFailure?.(receiptError);
482
+ if (!input.signalProof) {
483
+ throw new Error("Session-attempt quiescence proof signaler is unavailable", {
484
+ cause: receiptError,
485
+ });
486
+ }
487
+ const delay = input.sleep ?? sleep;
488
+ let retryMs = QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS;
489
+ let attempt = 1;
490
+ for (;;) {
491
+ try {
492
+ await input.signalProof(input.proof);
493
+ return "signal";
494
+ } catch (signalError) {
495
+ input.onSignalFailure?.(signalError, attempt, retryMs);
496
+ try {
497
+ input.heartbeat?.(attempt, retryMs);
498
+ } catch {
499
+ // Heartbeat telemetry is not proof delivery and cannot replace or
500
+ // interrupt the exact signal retry loop.
501
+ }
502
+ await delay(retryMs);
503
+ retryMs = Math.min(retryMs * 2, QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS);
504
+ attempt += 1;
505
+ }
506
+ }
507
+ }
508
+
509
+ try {
510
+ await input.publishEvents(events);
511
+ } catch (publishError) {
512
+ // Postgres already committed quiesced_at, the queue event, and the wake.
513
+ // NATS is live fanout only; never misclassify its failure as receipt loss.
514
+ input.onPublishFailure?.(publishError);
515
+ }
516
+ return "receipt";
517
+ }
518
+
519
+ /**
520
+ * Cross the non-detachable physical boundary for one dying attempt. Temporal's
521
+ * cancellation signal is intentionally absent: cancellation is transport, not
522
+ * proof that sandbox tools and attempt-owned credential writers have drained.
523
+ * A replacement may be admitted only after this function resolves and the
524
+ * caller durably persists or signals the exact quiescence receipt.
525
+ */
526
+ export async function drainAttemptOwnedSandboxWriters(input: {
527
+ toolCancellationFence: Pick<TurnToolCancellationFence, "cancel" | "waitForQuiescence"> | null;
528
+ cancellationReason?: unknown;
529
+ toolspaceTokenRenewal: Pick<ToolspaceTokenRenewalController, "stop"> | null;
530
+ runCredentialRenewal: Pick<RunCredentialRenewalController, "stop"> | null;
531
+ }): Promise<void> {
532
+ if (input.toolCancellationFence) {
533
+ input.toolCancellationFence.cancel(
534
+ input.cancellationReason ?? new Error("TURN_ATTEMPT_FENCED"),
535
+ );
536
+ await input.toolCancellationFence.waitForQuiescence();
537
+ }
538
+ await input.toolspaceTokenRenewal?.stop();
539
+ await input.runCredentialRenewal?.stop();
540
+ }
541
+
542
+ /**
543
+ * Await a finalizer operation only while this Temporal activity still owns its
544
+ * execution window. Once Pause/Steer cancellation arrives, the operation keeps
545
+ * its own rejection handler and may finish its idempotent, attempt-scoped
546
+ * cleanup in the background, but it cannot pin activity terminalization or
547
+ * delay the separately receipt-gated replacement dispatch.
548
+ */
549
+ export async function waitForTurnFinalizerStep<T>(
550
+ operation: Promise<T>,
551
+ signal: AbortSignal | undefined,
552
+ ): Promise<T | undefined> {
553
+ if (!signal) return await operation;
554
+ if (signal.aborted) {
555
+ void operation.catch(() => undefined);
556
+ return undefined;
557
+ }
558
+
559
+ let resolveCancellation: (() => void) | undefined;
560
+ const cancelled = new Promise<undefined>((resolve) => {
561
+ resolveCancellation = () => resolve(undefined);
562
+ });
563
+ const cancel = (): void => {
564
+ void operation.catch(() => undefined);
565
+ resolveCancellation?.();
566
+ };
567
+ signal.addEventListener("abort", cancel, { once: true });
568
+ try {
569
+ return await Promise.race([operation, cancelled]);
570
+ } finally {
571
+ signal.removeEventListener("abort", cancel);
572
+ }
573
+ }
574
+
575
+ /**
576
+ * Flush provider-facing stream state while the attempt still owns its activity
577
+ * window. On Pause/Steer, both promises are detached with rejection handlers:
578
+ * neither a runtime batcher nor an uncooperative provider completion promise
579
+ * may pin the activity behind cancellation after durable writes are fenced.
580
+ */
581
+ export async function waitForTurnStreamCleanup(
582
+ batcherFlush: Promise<unknown>,
583
+ providerCompleted: Promise<unknown>,
584
+ signal: AbortSignal | undefined,
585
+ ): Promise<void> {
586
+ await waitForTurnFinalizerStep(batcherFlush, signal);
587
+ await waitForTurnFinalizerStep(providerCompleted, signal);
588
+ }
589
+
590
+ function turnFinalizerCancellationSignal(
591
+ temporalSignal: AbortSignal | undefined,
592
+ activityStatus: RunAgentTurnResult["status"] | "unknown",
593
+ ): AbortSignal | undefined {
594
+ if (activityStatus !== "cancelled" || temporalSignal?.aborted) return temporalSignal;
595
+ const fenced = new AbortController();
596
+ fenced.abort(new Error("TURN_ATTEMPT_FENCED"));
597
+ return fenced.signal;
598
+ }
599
+
305
600
  function compactionFailureReason(reason: string): string {
306
601
  return reason.startsWith("compaction summarization failed:")
307
602
  ? reason
308
603
  : `compaction summarization failed: ${reason}`;
309
604
  }
310
605
 
606
+ function compactionFailureReasonFromError(error: unknown): string {
607
+ if (
608
+ error instanceof CompactionProviderResponseError ||
609
+ error instanceof EmptyCompactionSummaryError
610
+ ) {
611
+ return compactionFailureReason(error.message);
612
+ }
613
+ const errorName = error instanceof Error && error.name ? error.name : "unknown error";
614
+ return compactionFailureReason(`unexpected ${errorName}`);
615
+ }
616
+
617
+ function isCompactionSummaryFailure(error: unknown): boolean {
618
+ return (
619
+ error instanceof CompactionProviderResponseError || error instanceof EmptyCompactionSummaryError
620
+ );
621
+ }
622
+
623
+ export function shouldRecoverCompactionProviderFailure(error: unknown): boolean {
624
+ if (!(error instanceof CompactionProviderResponseError)) return false;
625
+ if (isCodexTransportError(error) && classifyCodexUsageLimitError(error)) return true;
626
+ return agentRunFailurePayload(error).retryable === true;
627
+ }
628
+
311
629
  export function classifyContextWindowOverflowError(
312
630
  error: unknown,
313
631
  ): { message: string; code?: string; detail?: string } | null {
@@ -440,6 +758,20 @@ export function modelUsageSourceKey(input: {
440
758
  return input.dispatchId ? `${input.dispatchId}:${input.positionalKey}` : input.positionalKey;
441
759
  }
442
760
 
761
+ export function providerContextTokens(
762
+ usage:
763
+ | {
764
+ inputTokens?: number;
765
+ outputTokens?: number;
766
+ totalTokens?: number;
767
+ }
768
+ | null
769
+ | undefined,
770
+ ): number | null {
771
+ const total = normalizeModelCallUsage(usage).totalTokens;
772
+ return total !== null && total > 0 ? total : null;
773
+ }
774
+
443
775
  /**
444
776
  * A provider call has already consumed tokens by the time its usage frame is
445
777
  * available. Losing the Codex credential lease at the renewal checkpoint must
@@ -468,6 +800,260 @@ type TurnEventPublisher = (
468
800
  immediate?: boolean,
469
801
  ) => Promise<{ events: SessionEvent[]; accepted: boolean }>;
470
802
 
803
+ export type ModelResponseUsageEventState = {
804
+ responseUsageCount: number;
805
+ providerContextRevision: number;
806
+ lastProviderContextTokensObserved: number | null;
807
+ claimedSourceKeys: Set<string>;
808
+ };
809
+
810
+ export type CompactionModelUsageEventState = {
811
+ usageCount: number;
812
+ claimedSourceKeys: Set<string>;
813
+ };
814
+
815
+ export function createModelResponseUsageEventState(
816
+ claimedSourceKeys: Set<string> = new Set<string>(),
817
+ ): ModelResponseUsageEventState {
818
+ return {
819
+ responseUsageCount: 0,
820
+ providerContextRevision: 0,
821
+ lastProviderContextTokensObserved: null,
822
+ claimedSourceKeys,
823
+ };
824
+ }
825
+
826
+ export function createCompactionModelUsageEventState(
827
+ claimedSourceKeys: Set<string> = new Set<string>(),
828
+ ): CompactionModelUsageEventState {
829
+ return { usageCount: 0, claimedSourceKeys };
830
+ }
831
+
832
+ export function modelResponseUsageContextSignal(
833
+ state: ModelResponseUsageEventState,
834
+ ): { revision: number; totalTokens: number } | null {
835
+ return state.lastProviderContextTokensObserved === null
836
+ ? null
837
+ : {
838
+ revision: state.providerContextRevision,
839
+ totalTokens: state.lastProviderContextTokensObserved,
840
+ };
841
+ }
842
+
843
+ /**
844
+ * Process one SDK stream event through the exact production usage path.
845
+ *
846
+ * The pinned Responses SDK mirrors one provider terminal response as both a
847
+ * normalized `response_done` and a raw `model/response.completed` event. Claim
848
+ * the stable response/source key before lease renewal or any usage side effect,
849
+ * and advance the positional ordinal only for a newly claimed response. The
850
+ * durable `agent.model.usage` source-key fence remains the cross-restart
851
+ * authority: a replay may retry the idempotent billing write, but it cannot
852
+ * advance process-local metrics, provider context, or attempt-owned signals.
853
+ */
854
+ export async function processModelResponseUsageEvent(input: {
855
+ event: Parameters<typeof modelResponseUsageFromSdkEvent>[0];
856
+ state: ModelResponseUsageEventState;
857
+ dispatchId: string | null;
858
+ settings: Settings;
859
+ db: ActivityServices["db"];
860
+ observability: ActivityServices["observability"];
861
+ publish: TurnEventPublisher | null;
862
+ accountId: string;
863
+ workspaceId: string;
864
+ sessionId: string;
865
+ turnId: string;
866
+ turnAttemptId: string;
867
+ provider: string;
868
+ providerApi: ModelProviderApi;
869
+ model: string;
870
+ metricProvider: string;
871
+ externallyBilled: boolean;
872
+ servingCredentialId: string | null;
873
+ priorSessionCredentialId: string | null;
874
+ emittedSourceKeys: Set<string>;
875
+ renewLease: () => Promise<void>;
876
+ leaseLost: () => boolean;
877
+ leaseLostMessage: string;
878
+ setLastInputTokens: (tokens: number) => Promise<void>;
879
+ }): Promise<
880
+ | { status: "not_usage" }
881
+ | { status: "duplicate"; sourceKey: string }
882
+ | { status: "processed"; sourceKey: string; authoritative: boolean }
883
+ > {
884
+ const responseUsage = modelResponseUsageFromSdkEvent(input.event);
885
+ if (!responseUsage) {
886
+ return { status: "not_usage" };
887
+ }
888
+
889
+ const responseOrdinal = input.state.responseUsageCount + 1;
890
+ const sourceKey = modelUsageSourceKey({
891
+ responseId: responseUsage.responseId,
892
+ dispatchId: input.dispatchId,
893
+ positionalKey: `response-${responseOrdinal}`,
894
+ });
895
+ if (input.state.claimedSourceKeys.has(sourceKey)) {
896
+ return { status: "duplicate", sourceKey };
897
+ }
898
+ input.state.claimedSourceKeys.add(sourceKey);
899
+ input.state.responseUsageCount = responseOrdinal;
900
+
901
+ const normalizedUsage = normalizeModelCallUsage(responseUsage.usage);
902
+ const accountContext = modelCallAccountContext({
903
+ servingCredentialId: input.servingCredentialId,
904
+ priorSessionCredentialId: input.priorSessionCredentialId,
905
+ isFirstCallOfTurn: responseOrdinal === 1,
906
+ });
907
+ let authoritative = false;
908
+ await recordCompletedModelCallBeforeOwnershipFences({
909
+ renewLease: input.renewLease,
910
+ leaseLost: input.leaseLost,
911
+ leaseLostMessage: input.leaseLostMessage,
912
+ recordUsage: async () => {
913
+ await recordModelUsageAndDebitCredits(input.settings, input.db, {
914
+ accountId: input.accountId,
915
+ workspaceId: input.workspaceId,
916
+ sessionId: input.sessionId,
917
+ turnId: input.turnId,
918
+ turnAttemptId: input.turnAttemptId,
919
+ model: input.model,
920
+ externallyBilled: input.externallyBilled,
921
+ usage: responseUsage.usage,
922
+ normalizedUsage,
923
+ sourceKey,
924
+ observability: input.observability,
925
+ });
926
+ authoritative = await emitModelCallUsage({
927
+ observability: input.observability,
928
+ publish: input.publish,
929
+ accountId: input.accountId,
930
+ workspaceId: input.workspaceId,
931
+ sessionId: input.sessionId,
932
+ turnId: input.turnId,
933
+ provider: input.provider,
934
+ providerApi: input.providerApi,
935
+ model: input.model,
936
+ sourceKey,
937
+ usage: responseUsage,
938
+ normalizedUsage,
939
+ servingAccountHash: accountContext.servingAccountHash,
940
+ accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
941
+ emittedSourceKeys: input.emittedSourceKeys,
942
+ });
943
+ const observedInput = normalizedUsage.telemetry.inputTokens;
944
+ if (authoritative && observedInput !== null && observedInput > 0) {
945
+ recordModelInputTokens(input.observability, input.metricProvider, observedInput);
946
+ }
947
+ },
948
+ recordAttemptSignals: async () => {
949
+ if (!authoritative) return;
950
+ const observedTotal = normalizedUsage.totalTokens;
951
+ if (observedTotal !== null) {
952
+ input.state.lastProviderContextTokensObserved = observedTotal;
953
+ input.state.providerContextRevision += 1;
954
+ }
955
+ const observedInput = normalizedUsage.telemetry.inputTokens;
956
+ if (observedInput !== null && observedInput > 0) {
957
+ await input.setLastInputTokens(observedInput);
958
+ }
959
+ },
960
+ });
961
+ return { status: "processed", sourceKey, authoritative };
962
+ }
963
+
964
+ /**
965
+ * Apply the same source-key authority ordering to the compaction summarizer's
966
+ * usage callback. The summarizer can retry or mirror a terminal response just
967
+ * like the main stream, so claim before lease renewal, billing, durable usage,
968
+ * logging, or cache metrics. Durable source-key idempotency remains the
969
+ * cross-process authority after a worker restart.
970
+ */
971
+ export async function processCompactionModelUsageEvent(input: {
972
+ usage: ModelResponseUsage;
973
+ state: CompactionModelUsageEventState;
974
+ dispatchId: string | null;
975
+ settings: Settings;
976
+ db: ActivityServices["db"];
977
+ observability: ActivityServices["observability"];
978
+ publish: TurnEventPublisher | null;
979
+ accountId: string;
980
+ workspaceId: string;
981
+ sessionId: string;
982
+ turnId: string;
983
+ turnAttemptId: string;
984
+ provider: string;
985
+ providerApi: ModelProviderApi;
986
+ model: string;
987
+ externallyBilled: boolean;
988
+ servingCredentialId: string | null;
989
+ priorSessionCredentialId: string | null;
990
+ emittedSourceKeys: Set<string>;
991
+ renewLease: () => Promise<void>;
992
+ leaseLost: () => boolean;
993
+ leaseLostMessage: string;
994
+ }): Promise<
995
+ | { status: "duplicate"; sourceKey: string }
996
+ | { status: "processed"; sourceKey: string; authoritative: boolean }
997
+ > {
998
+ const usageOrdinal = input.state.usageCount + 1;
999
+ const sourceKey = modelUsageSourceKey({
1000
+ responseId: input.usage.responseId,
1001
+ dispatchId: input.dispatchId,
1002
+ positionalKey: `compaction-${usageOrdinal}`,
1003
+ });
1004
+ if (input.state.claimedSourceKeys.has(sourceKey)) {
1005
+ return { status: "duplicate", sourceKey };
1006
+ }
1007
+ input.state.claimedSourceKeys.add(sourceKey);
1008
+ input.state.usageCount = usageOrdinal;
1009
+
1010
+ const accountContext = modelCallAccountContext({
1011
+ servingCredentialId: input.servingCredentialId,
1012
+ priorSessionCredentialId: input.priorSessionCredentialId,
1013
+ isFirstCallOfTurn: usageOrdinal === 1,
1014
+ });
1015
+ const normalizedUsage = normalizeModelCallUsage(input.usage.usage);
1016
+ let authoritative = false;
1017
+ await recordCompletedModelCallBeforeOwnershipFences({
1018
+ renewLease: input.renewLease,
1019
+ leaseLost: input.leaseLost,
1020
+ leaseLostMessage: input.leaseLostMessage,
1021
+ recordUsage: async () => {
1022
+ await recordModelUsageAndDebitCredits(input.settings, input.db, {
1023
+ accountId: input.accountId,
1024
+ workspaceId: input.workspaceId,
1025
+ sessionId: input.sessionId,
1026
+ turnId: input.turnId,
1027
+ turnAttemptId: input.turnAttemptId,
1028
+ model: input.model,
1029
+ externallyBilled: input.externallyBilled,
1030
+ usage: input.usage.usage,
1031
+ normalizedUsage,
1032
+ sourceKey,
1033
+ observability: input.observability,
1034
+ });
1035
+ authoritative = await emitModelCallUsage({
1036
+ observability: input.observability,
1037
+ publish: input.publish,
1038
+ accountId: input.accountId,
1039
+ workspaceId: input.workspaceId,
1040
+ sessionId: input.sessionId,
1041
+ turnId: input.turnId,
1042
+ provider: input.provider,
1043
+ providerApi: input.providerApi,
1044
+ model: input.model,
1045
+ sourceKey,
1046
+ usage: input.usage,
1047
+ normalizedUsage,
1048
+ servingAccountHash: accountContext.servingAccountHash,
1049
+ accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
1050
+ emittedSourceKeys: input.emittedSourceKeys,
1051
+ });
1052
+ },
1053
+ });
1054
+ return { status: "processed", sourceKey, authoritative };
1055
+ }
1056
+
471
1057
  export async function emitModelCallUsage(input: {
472
1058
  observability: ActivityServices["observability"];
473
1059
  publish: TurnEventPublisher | null;
@@ -480,22 +1066,25 @@ export async function emitModelCallUsage(input: {
480
1066
  model: string;
481
1067
  sourceKey: string;
482
1068
  usage: ModelResponseUsage | { usage?: unknown | null } | null;
1069
+ normalizedUsage?: ModelCallUsageNormalization;
483
1070
  // Prompt-cache research dimensions (log-only; NEVER on a metric label or a
484
1071
  // durable event). The opaque serving-account tag and whether it changed since
485
1072
  // the session's previous call — the account-switch hypothesis for cache misses.
486
1073
  servingAccountHash?: string;
487
1074
  accountChangedFromPrevCall?: boolean;
488
1075
  emittedSourceKeys?: Set<string>;
489
- }): Promise<void> {
1076
+ }): Promise<boolean> {
490
1077
  const usage =
491
1078
  input.usage && typeof input.usage === "object" && "usage" in input.usage
492
1079
  ? (input.usage as { usage?: unknown }).usage
493
1080
  : null;
494
1081
  if (!usage || typeof usage !== "object") {
495
- return;
1082
+ return false;
496
1083
  }
497
- if (input.emittedSourceKeys?.has(input.sourceKey)) return;
498
- const telemetry = modelCallUsageTelemetry(usage as Parameters<typeof modelCallUsageTelemetry>[0]);
1084
+ if (input.emittedSourceKeys?.has(input.sourceKey)) return false;
1085
+ const normalizedUsage =
1086
+ input.normalizedUsage ?? normalizeModelCallUsage(usage as ModelCallUsageInput);
1087
+ const telemetry = normalizedUsage.telemetry;
499
1088
  const appended = await input.publish?.(
500
1089
  [
501
1090
  {
@@ -524,7 +1113,7 @@ export async function emitModelCallUsage(input: {
524
1113
  typeof event.payload === "object" &&
525
1114
  (event.payload as Record<string, unknown>).sourceKey === input.sourceKey,
526
1115
  );
527
- if (!authoritative) return;
1116
+ if (!authoritative) return false;
528
1117
  try {
529
1118
  input.observability.info("model call usage", {
530
1119
  accountId: input.accountId,
@@ -538,6 +1127,7 @@ export async function emitModelCallUsage(input: {
538
1127
  inputTokens: telemetry.inputTokens,
539
1128
  outputTokens: telemetry.outputTokens,
540
1129
  cachedTokens: telemetry.cachedTokens,
1130
+ cacheWriteTokens: telemetry.cacheWriteTokens,
541
1131
  reasoningTokens: telemetry.reasoningTokens,
542
1132
  ...(input.servingAccountHash !== undefined
543
1133
  ? { servingAccountHash: input.servingAccountHash }
@@ -546,9 +1136,41 @@ export async function emitModelCallUsage(input: {
546
1136
  ? { accountChangedFromPrevCall: input.accountChangedFromPrevCall }
547
1137
  : {}),
548
1138
  });
1139
+ if (normalizedUsage.rejectedFields.length > 0) {
1140
+ input.observability.warn("model call usage fields rejected", {
1141
+ provider: input.provider,
1142
+ providerApi: input.providerApi,
1143
+ model: input.model,
1144
+ sourceKey: input.sourceKey,
1145
+ rejectedFields: normalizedUsage.rejectedFields.join(","),
1146
+ });
1147
+ }
549
1148
  } catch {
550
1149
  // Durable event + billing already committed; logging is best-effort only.
551
1150
  }
1151
+ try {
1152
+ applyCodexCacheTelemetry(input.observability, input.provider, normalizedUsage);
1153
+ } catch {
1154
+ // Durable event + billing already committed; metrics are best-effort only.
1155
+ }
1156
+ return true;
1157
+ }
1158
+
1159
+ /**
1160
+ * Apply one authoritative, normalized model-call usage frame to the shared
1161
+ * prompt-cache metrics. The durable source-key fence in `emitModelCallUsage`
1162
+ * owns idempotency; this helper must never receive raw provider values.
1163
+ */
1164
+ export function applyCodexCacheTelemetry(
1165
+ observability: ActivityServices["observability"],
1166
+ provider: string,
1167
+ normalizedUsage: ModelCallUsageNormalization,
1168
+ ): void {
1169
+ recordModelCacheTokens(observability, provider, {
1170
+ cachedTokens: normalizedUsage.telemetry.cachedTokens,
1171
+ cacheWriteTokens: normalizedUsage.telemetry.cacheWriteTokens,
1172
+ promptTokens: normalizedUsage.telemetry.inputTokens,
1173
+ });
552
1174
  }
553
1175
 
554
1176
  export function historyRowsToAppend(
@@ -565,12 +1187,13 @@ export function historyRowsToAppend(
565
1187
  // persistedHistoryCount to preserve the pre-compaction behaviour (contiguous
566
1188
  // positions from 0) when callers do not pass an explicit next position.
567
1189
  nextPosition: number = persistedHistoryCount,
1190
+ toolOutputTruncationTokens?: number,
568
1191
  ): {
569
1192
  rows: Array<{ position: number; item: Record<string, unknown> }>;
570
1193
  nextWatermark: number;
571
1194
  nextPosition: number;
572
1195
  } {
573
- const sanitized = sanitizeHistoryItemsForModel(rawHistory).filter(
1196
+ const sanitized = sanitizeHistoryItemsForModel(rawHistory, toolOutputTruncationTokens).filter(
574
1197
  (item) => !isEphemeralInternalContext(item),
575
1198
  );
576
1199
  if (sanitized.length <= persistedHistoryCount) {
@@ -916,6 +1539,86 @@ export type TurnSandboxProvisioner<T> = {
916
1539
  waitForSettled(timeoutMs: number): Promise<T | null>;
917
1540
  };
918
1541
 
1542
+ export class TurnOperationCancelledError extends Error {
1543
+ readonly name = "TurnOperationCancelledError";
1544
+
1545
+ constructor(readonly reason: unknown) {
1546
+ super("Turn operation was cancelled with its owning turn", {
1547
+ ...(reason instanceof Error ? { cause: reason } : {}),
1548
+ });
1549
+ }
1550
+ }
1551
+
1552
+ /**
1553
+ * Normalize a preparation/provisioning cancellation race back to the Temporal
1554
+ * cancellation that owns the activity. Several provider APIs expose no portable
1555
+ * abort primitive, so the worker stops awaiting them and disposes any late
1556
+ * resource. The wrapper error must never fall through as an ordinary turn
1557
+ * failure: doing so would omit the quiescence receipt and strand a committed
1558
+ * Steer/Pause behind `control-pending`.
1559
+ */
1560
+ export function turnOperationCancellationFailure(error: unknown): CancelledFailure | null {
1561
+ if (error instanceof CancelledFailure) return error;
1562
+ if (!(error instanceof TurnOperationCancelledError)) return null;
1563
+ return error.reason instanceof CancelledFailure
1564
+ ? error.reason
1565
+ : new CancelledFailure("TURN_SANDBOX_PROVISION_CANCELLED", [], error);
1566
+ }
1567
+
1568
+ function throwIfTurnOperationCancelled(signal: AbortSignal | undefined): void {
1569
+ if (signal?.aborted) {
1570
+ throw new TurnOperationCancelledError(signal.reason);
1571
+ }
1572
+ }
1573
+
1574
+ export async function waitForTurnOperation<T>(
1575
+ operation: Promise<T>,
1576
+ signal: AbortSignal | undefined,
1577
+ disposeLateResult: ((result: T) => Promise<void> | void) | undefined,
1578
+ ): Promise<T> {
1579
+ if (!signal) return await operation;
1580
+
1581
+ let rejectCancellation: ((error: TurnOperationCancelledError) => void) | undefined;
1582
+ const cancelled = new Promise<never>((_resolve, reject) => {
1583
+ rejectCancellation = reject;
1584
+ });
1585
+ const cancel = (): void => {
1586
+ rejectCancellation?.(new TurnOperationCancelledError(signal.reason));
1587
+ };
1588
+ signal.addEventListener("abort", cancel, { once: true });
1589
+ if (signal.aborted) cancel();
1590
+
1591
+ try {
1592
+ const result = await Promise.race([operation, cancelled]);
1593
+ // Cancellation owns an exact turn boundary even when the provider result
1594
+ // and AbortSignal settle in the same microtask checkpoint. Never let a
1595
+ // just-resolved lease escape after the control was already committed.
1596
+ if (signal.aborted) {
1597
+ throw new TurnOperationCancelledError(signal.reason);
1598
+ }
1599
+ return result;
1600
+ } catch (error) {
1601
+ if (signal.aborted) {
1602
+ // The provider establish call has no universal cancellation seam. It may
1603
+ // finish after the Temporal activity has correctly stopped; dispose its
1604
+ // late lease instead of letting a cancelled turn resurrect a holder/box.
1605
+ void operation
1606
+ .then(async (result) => await disposeLateResult?.(result))
1607
+ .catch(() => undefined);
1608
+ // A provider failure may settle in the same checkpoint as the committed
1609
+ // control. The control is authoritative; retain its cancellation shape
1610
+ // so the activity publishes quiescence instead of looking like an
1611
+ // unrelated turn failure.
1612
+ if (!(error instanceof TurnOperationCancelledError)) {
1613
+ throw new TurnOperationCancelledError(signal.reason);
1614
+ }
1615
+ }
1616
+ throw error;
1617
+ } finally {
1618
+ signal.removeEventListener("abort", cancel);
1619
+ }
1620
+ }
1621
+
919
1622
  function sleep(ms: number): Promise<void> {
920
1623
  return new Promise((resolve) => setTimeout(resolve, ms));
921
1624
  }
@@ -938,9 +1641,11 @@ export function createTurnSandboxProvisioner<T>(
938
1641
  options: {
939
1642
  maxRetries?: number;
940
1643
  backoffMs?: number;
1644
+ signal?: AbortSignal;
941
1645
  onStarted?: () => Promise<void> | void;
942
- onCompleted?: () => Promise<void> | void;
1646
+ onCompleted?: (result: T) => Promise<void> | void;
943
1647
  onFailed?: (error: unknown) => Promise<void> | void;
1648
+ disposeResult?: (result: T) => Promise<void> | void;
944
1649
  } = {},
945
1650
  ): TurnSandboxProvisioner<T> {
946
1651
  const maxRetries = options.maxRetries ?? 2;
@@ -951,9 +1656,15 @@ export function createTurnSandboxProvisioner<T>(
951
1656
  let attempt = 0;
952
1657
  while (true) {
953
1658
  try {
954
- return await establish();
1659
+ throwIfTurnOperationCancelled(options.signal);
1660
+ const operation = establish();
1661
+ return await waitForTurnOperation(operation, options.signal, options.disposeResult);
955
1662
  } catch (error) {
956
- if (attempt >= maxRetries || !isLazySandboxProvisionRetryable(error)) {
1663
+ if (
1664
+ error instanceof TurnOperationCancelledError ||
1665
+ attempt >= maxRetries ||
1666
+ !isLazySandboxProvisionRetryable(error)
1667
+ ) {
957
1668
  throw error;
958
1669
  }
959
1670
  attempt += 1;
@@ -966,13 +1677,24 @@ export function createTurnSandboxProvisioner<T>(
966
1677
  get(): Promise<T> {
967
1678
  if (!memo) {
968
1679
  memo = (async () => {
1680
+ throwIfTurnOperationCancelled(options.signal);
969
1681
  await options.onStarted?.();
1682
+ throwIfTurnOperationCancelled(options.signal);
1683
+ let result: T | undefined;
1684
+ let hasResult = false;
970
1685
  try {
971
- const result = await run();
972
- await options.onCompleted?.();
1686
+ result = await run();
1687
+ hasResult = true;
1688
+ throwIfTurnOperationCancelled(options.signal);
1689
+ await options.onCompleted?.(result);
973
1690
  return result;
974
1691
  } catch (error) {
975
- await options.onFailed?.(error);
1692
+ if (hasResult) {
1693
+ await options.disposeResult?.(result as T);
1694
+ }
1695
+ if (!(error instanceof TurnOperationCancelledError)) {
1696
+ await options.onFailed?.(error);
1697
+ }
976
1698
  throw error;
977
1699
  }
978
1700
  })().catch((error) => {
@@ -1041,7 +1763,7 @@ export function acceptsPromptCacheKeyForTurn(
1041
1763
  * The turn hot path never refreshes Codex usage — only the usage API route does — so a
1042
1764
  * window that has actually reset still reads OVER-threshold from the stale cache, which
1043
1765
  * would idle-loop forever. Before idling, refresh LIVE usage for every connected account
1044
- * the cache marks over-threshold (bounded to the account count), which re-writes the
1766
+ * the cache marks exhausted (bounded to the account count), which re-writes the
1045
1767
  * cache columns, then return the re-read rows so the ranker can pick up a genuinely-reset
1046
1768
  * window THIS turn. A refresh/read failure is swallowed (fall back to the pre-refresh rows
1047
1769
  * + the bounded idle). Cooling (429'd) accounts are NOT refreshed: their exhaustedUntil
@@ -1051,17 +1773,29 @@ async function refreshCappedCodexUsageRows(
1051
1773
  db: ActivityServices["db"],
1052
1774
  settings: Settings,
1053
1775
  workspaceId: string,
1054
- accounts: CodexAccountStatus[],
1776
+ accounts: Array<
1777
+ Pick<
1778
+ CodexAccountStatus,
1779
+ "id" | "status" | "primaryUsedPercent" | "secondaryUsedPercent" | "exhaustedUntil"
1780
+ >
1781
+ >,
1055
1782
  capacitySignals: {
1056
1783
  signalCodexCapacityWorkflow?: ActivityServices["signalCodexCapacityWorkflow"] | undefined;
1057
1784
  wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
1058
1785
  },
1059
- ): Promise<CodexAccountStatus[]> {
1060
- const nearPct = settings.codexRotationNearExhaustionPct;
1786
+ ): Promise<
1787
+ Array<
1788
+ Pick<
1789
+ CodexAccountStatus,
1790
+ "id" | "status" | "primaryUsedPercent" | "secondaryUsedPercent" | "exhaustedUntil"
1791
+ >
1792
+ >
1793
+ > {
1061
1794
  const stale = accounts.filter(
1062
1795
  (a) =>
1063
1796
  a.status === "active" &&
1064
- ((a.primaryUsedPercent ?? 0) >= nearPct || (a.secondaryUsedPercent ?? 0) >= nearPct),
1797
+ ((a.primaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT ||
1798
+ (a.secondaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT),
1065
1799
  );
1066
1800
  if (stale.length === 0) {
1067
1801
  return accounts;
@@ -1097,11 +1831,22 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1097
1831
  objectStorage,
1098
1832
  observability,
1099
1833
  wakeSessionWorkflow,
1834
+ signalSessionAttemptQuiesced,
1100
1835
  signalCodexCapacityWorkflow,
1101
1836
  entitlements,
1102
1837
  connectionCredentials,
1103
1838
  } = await services();
1104
1839
  const activityContext = currentActivityContext();
1840
+ const cancellationSignal = activityContext?.cancellationSignal;
1841
+ let cancellationRequestedAt: number | null = cancellationSignal?.aborted
1842
+ ? performance.now()
1843
+ : null;
1844
+ const noteCancellationRequested = (): void => {
1845
+ cancellationRequestedAt ??= performance.now();
1846
+ };
1847
+ cancellationSignal?.addEventListener("abort", noteCancellationRequested, {
1848
+ once: true,
1849
+ });
1105
1850
  const dispatchId = activityContext?.info.activityId ?? randomUUID();
1106
1851
  const activityStarted = performance.now();
1107
1852
  const activitySpan = observability.startSpan("worker.run_agent_segment", {
@@ -1112,6 +1857,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1112
1857
  let activityStatus: RunAgentTurnResult["status"] | "unknown" = "unknown";
1113
1858
  let turnMetricOutcome: TurnOutcome | null = null;
1114
1859
  let activityError: unknown;
1860
+ let acknowledgeQuiescence = false;
1861
+ const acknowledgeLostAttemptOwnership = (): void => {
1862
+ // A stale terminal/recovery settlement can lose either to a benign
1863
+ // successor or to Pause/Steer closing this exact attempt. Only the
1864
+ // receipt transaction can distinguish those cases after the hard tool
1865
+ // fence: allowUninterrupted makes the benign case an event-free no-op.
1866
+ acknowledgeQuiescence = true;
1867
+ noteCancellationRequested();
1868
+ };
1115
1869
  let turnId: string | undefined;
1116
1870
  let triggerEventId: string | undefined;
1117
1871
  const claimedResult = (
@@ -1135,17 +1889,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1135
1889
  runtimeMetricsHooksForObservability(observability),
1136
1890
  );
1137
1891
  let isCodexTurn = false;
1892
+ let isExternallyBilledTurn = false;
1138
1893
  let executionGeneration = 0;
1139
1894
  // Still required by credential-loss/capacity settlements, whose own
1140
1895
  // recovery transactions fence against worker-death redispatches.
1141
1896
  let redispatchesAtDispatch = 0;
1142
- const saveRunStateFenced = async (state: Parameters<typeof saveRunState>[1]): Promise<void> => {
1143
- if (!(await saveRunState(db, state))) {
1144
- throw new TurnAttemptFencedError(
1145
- "turn execution generation was fenced while saving run state",
1146
- );
1147
- }
1148
- };
1149
1897
  const setLastInputTokensFenced = async (lastInputTokens: number): Promise<void> => {
1150
1898
  if (!turnId || executionGeneration <= 0) {
1151
1899
  throw new Error("Turn attempt was not initialized before token accounting");
@@ -1164,7 +1912,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1164
1912
  }
1165
1913
  };
1166
1914
  let heartbeatTimer: ReturnType<typeof startActivityHeartbeat> | undefined;
1167
- // OPE-21: one workspace-local idempotent credential holder per running
1915
+ // credential allocator: one workspace-local idempotent credential holder per running
1168
1916
  // Codex turn. The DB row is the cross-replica fairness primitive; this timer
1169
1917
  // only extends its short TTL. A killed worker stops heartbeating and the
1170
1918
  // holder self-expires. Other workspaces never see or share this holder.
@@ -1295,11 +2043,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1295
2043
  // runs it refreshes expires_at epoch-fenced so a legit multi-day turn is
1296
2044
  // never TTL-reaped. Cleared in finally. Only set when the flag resolved a box.
1297
2045
  let leaseHeartbeatTimer: ReturnType<typeof setInterval> | undefined;
1298
- // OPE-41: the worker, not the model, owns renewal of run-scoped Git
2046
+ // credential-renewal policy: the worker, not the model, owns renewal of run-scoped Git
1299
2047
  // credentials for a multi-day turn. The controller is attached only after
1300
2048
  // the initial seed reached a real cloud box and is drained before capture.
1301
- let gitCredentialRenewal: GitCredentialRenewalController | null = null;
2049
+ let gitCredentialRenewals: GitCredentialRenewalController[] = [];
1302
2050
  let gitCredentialRenewalClosed = false;
2051
+ // Generic host-owned run material has its own attempt-scoped renewal and
2052
+ // write handle. It is always drained and wiped before workspace capture.
2053
+ let runCredentialRenewal: RunCredentialRenewalController | null = null;
2054
+ let runCredentialRenewalClosed = false;
2055
+ let runCredentialSession: RunCredentialCommandSession | null = null;
2056
+ // The delegated Toolspace bearer has a one-hour TTL. Renewal is attempt-
2057
+ // owned and attaches only after the initial token file reached a real
2058
+ // sandbox session; finalization drains an in-flight replacement.
2059
+ let toolspaceTokenRenewal: ToolspaceTokenRenewalController | null = null;
2060
+ let toolspaceTokenRenewalClosed = false;
1303
2061
  // MID-SESSION snapshot single-flight guard: the heartbeat tick fires every
1304
2062
  // 10s but a Modal filesystem snapshot can take longer — never overlap two
1305
2063
  // captures on one box. The in-flight capture's promise is held so the
@@ -1349,6 +2107,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1349
2107
  await current?.flush().catch(() => undefined);
1350
2108
  };
1351
2109
  let preparedTools: Awaited<ReturnType<OpenGeniRuntime["prepareTools"]>> | null = null;
2110
+ const toolCancellationFenceRef: {
2111
+ current: TurnToolCancellationFence | null;
2112
+ } = {
2113
+ current: null,
2114
+ };
1352
2115
  let publish: TurnEventPublisher | null = null;
1353
2116
  let settle:
1354
2117
  | ((input: {
@@ -1362,10 +2125,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1362
2125
  | "requires_action";
1363
2126
  sessionStatus: SessionStatus;
1364
2127
  activeTurnId: string | null;
2128
+ consumeRequestedCompactionFailure?: boolean;
2129
+ runState?: ApplySessionTurnSettlementInput["runState"];
1365
2130
  }) => Promise<boolean>)
1366
2131
  | null = null;
1367
2132
  let turnStartedPublished = false;
1368
2133
  let stream: Awaited<ReturnType<OpenGeniRuntime["runStream"]>> | undefined;
2134
+ // Reconciliation is declared before provider routing so every turn-end path
2135
+ // can share one closure. It cannot run until `stream` exists, by which time
2136
+ // this value has been rebound to the turn's resolved model policy.
2137
+ let modelRunSettings: Settings = settings;
1369
2138
  const publishSandboxLifecycleEvents = async (sandbox: ResumedTurnSandbox): Promise<void> => {
1370
2139
  const established = sandbox.established;
1371
2140
  if (publish && established.origin && established.origin !== "resumed") {
@@ -1465,16 +2234,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1465
2234
  // selfhosted target has no persistWorkspace anyway. Best-effort +
1466
2235
  // single-flight; throttling lives in the helper.
1467
2236
  const snapshotSession = setupBoxSession;
1468
- if (snapshotSession && !snapshotInFlight) {
2237
+ const snapshotTurnId = turnId;
2238
+ if (snapshotSession && snapshotTurnId && !snapshotInFlight) {
1469
2239
  snapshotInFlight = maybePersistWarmWorkspaceSnapshot(
1470
2240
  { db, settings },
1471
2241
  {
1472
2242
  accountId: input.accountId,
1473
2243
  workspaceId: input.workspaceId,
2244
+ sessionId: input.sessionId,
2245
+ turnId: snapshotTurnId,
2246
+ attemptId: input.attemptId,
1474
2247
  sandboxGroupId: heartbeatGroupId,
1475
2248
  },
1476
2249
  snapshotSession,
1477
2250
  heartbeatEpoch,
2251
+ activityContext?.cancellationSignal,
1478
2252
  )
1479
2253
  .then(async (persisted) => {
1480
2254
  if (persisted && publish) {
@@ -1602,6 +2376,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1602
2376
  rawHistory as Array<Record<string, unknown>>,
1603
2377
  persistedHistoryCount,
1604
2378
  nextHistoryPosition,
2379
+ modelRunSettings.modelToolOutputTruncationTokens,
1605
2380
  );
1606
2381
  const hasModelOrToolProgress = rows.some((row) =>
1607
2382
  isModelOrToolProgressHistoryItem(row.item),
@@ -1621,6 +2396,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1621
2396
  // runs, so this is the turn's effective account. The read path uses
1622
2397
  // it to strip cross-account reasoning.encrypted_content next turn.
1623
2398
  producerCodexCredentialId: effectiveCodexCredentialId,
2399
+ modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
1624
2400
  items: rows,
1625
2401
  });
1626
2402
  if (!appended) {
@@ -1651,6 +2427,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1651
2427
  // Reassigned after the variable set loads; the publish closure is
1652
2428
  // created (and used for turn.started) before the variableSet is available.
1653
2429
  let redact: (payload: unknown) => unknown = identityRedactor;
2430
+ const secretRedactions = new Map<string, string>();
2431
+ const publishedRunCredentialNotices = new Set<string>();
2432
+ const registerSecretRedactions = (secrets: SecretForRedaction[]): void => {
2433
+ for (const secret of secrets) {
2434
+ if (!secretRedactions.has(secret.value)) secretRedactions.set(secret.value, secret.name);
2435
+ }
2436
+ redact = createSecretRedactor(
2437
+ [...secretRedactions].map(([value, name]) => ({ name, value })),
2438
+ );
2439
+ };
1654
2440
  let variableSetId = "";
1655
2441
  // Rig telemetry (M3): set once the session loads; empty string for a rig-less
1656
2442
  // turn (mirrors variableSetId). Read by the activity span's finally block.
@@ -1682,10 +2468,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1682
2468
  input.workspaceId,
1683
2469
  settings,
1684
2470
  );
1685
- // Read the active-credential flag ONCE (P2-b) and thread it through both the
1686
- // routing overlay (settingsWithCodexCredential) and the billed-turn predicate
1687
- // (isCodexBilledTurn below), so a concurrent disconnect/reconnect cannot make
1688
- // provider-injection and billing disagree about whether this is a codex turn.
2471
+ // Read the active-credential flag once for the runtime capability overlay.
2472
+ // Accepted billing/provider identity comes from the turn policy below,
2473
+ // never from this mutable health snapshot.
1689
2474
  const codexSubscriptionActive = await workspaceCodexSubscriptionActive(
1690
2475
  db,
1691
2476
  mcpSettings,
@@ -1713,13 +2498,50 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1713
2498
  }
1714
2499
  const turn = claim.turn;
1715
2500
  turnId = turn.id;
2501
+ executionGeneration = turn.executionGeneration;
2502
+ const claimedPolicy = readTurnExecutionPolicyV1(turn.metadata);
2503
+ const policyForAbsent =
2504
+ claimedPolicy.kind === "valid"
2505
+ ? claimedPolicy.policy
2506
+ : resolveTurnExecutionPolicyV1(capabilitySettings, legacyTurnExecutionPolicyInput(turn));
2507
+ const installedPolicy = await installOrReadTurnExecutionPolicyForAttempt(db, {
2508
+ accountId: input.accountId,
2509
+ workspaceId: input.workspaceId,
2510
+ sessionId: input.sessionId,
2511
+ turnId,
2512
+ executionGeneration,
2513
+ attemptId: input.attemptId,
2514
+ policyForAbsent,
2515
+ });
2516
+ if (!installedPolicy.accepted) {
2517
+ throw new TurnAttemptFencedError(
2518
+ `turn execution policy was fenced: ${installedPolicy.reason}`,
2519
+ );
2520
+ }
2521
+ const verifiedExecutionPolicy = assertTurnExecutionPolicyMatchesConfigV1(
2522
+ capabilitySettings,
2523
+ installedPolicy.policy,
2524
+ {
2525
+ modelId: turn.model,
2526
+ reasoningEffort: turn.reasoningEffort,
2527
+ },
2528
+ );
2529
+ const turnExecutionPolicy = verifiedExecutionPolicy.policy;
2530
+ const billingIdentity = turnExecutionPolicyBillingIdentity(turnExecutionPolicy);
2531
+ isExternallyBilledTurn = billingIdentity.externallyBilled;
2532
+ isCodexTurn = billingIdentity.codexSubscription;
1716
2533
  triggerEventId = turn.triggerEventId;
1717
2534
  const trigger = await getSessionEvent(db, input.workspaceId, triggerEventId);
1718
2535
  if (!trigger) {
1719
2536
  throw new Error(`Trigger event not found: ${triggerEventId}`);
1720
2537
  }
2538
+ const humanInputResume = await getHumanInputResumeForEvent(
2539
+ db,
2540
+ input.workspaceId,
2541
+ input.sessionId,
2542
+ trigger,
2543
+ );
1721
2544
  triggerType = trigger.type;
1722
- executionGeneration = turn.executionGeneration;
1723
2545
  const latestTurnState = await getLatestRunState(db, input.workspaceId, input.sessionId);
1724
2546
  const continuationCodexCredentialId =
1725
2547
  latestTurnState?.turnId === turnId ? latestTurnState.frozenCodexCredentialId : null;
@@ -1728,30 +2550,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1728
2550
  0,
1729
2551
  );
1730
2552
  turnLifecycleMetricsFor(observability).start(turnId);
1731
- // Canonical codex-billed predicate (codex/<slug> + feature enabled + active
1732
- // workspace credential). Computed once and threaded through every billing
1733
- // gate + the usage recorder so a turn paid by the user's ChatGPT/Codex plan
1734
- // consumes ZERO OpenGeni credits and never feeds an OpenGeni cap. Resolved
1735
- // here (before resolvedModel at the routing step) because the pre-turn gate
1736
- // below needs it; mirrors the same active-credential read the codex provider
1737
- // overlay uses, so billing and routing agree on what "codex" is.
1738
- isCodexTurn = await isCodexBilledTurn({
1739
- db,
1740
- settings,
1741
- workspaceId: input.workspaceId,
1742
- model: turn.model,
1743
- active: codexSubscriptionActive,
1744
- });
1745
- // §7.5 P3 — pass BOTH the codex predicate (codex-plan turns bypass the gate)
2553
+ // §7.5 P3 — pass the accepted billing attribution (externally funded turns
2554
+ // bypass OpenGeni credit/token gates)
1746
2555
  // AND the optional host `entitlements` port (when bound, its admitRun replaces
1747
2556
  // the local credit read). Unset port → today's local-ledger path.
1748
- await ensureRunAllowed(
1749
- settings,
1750
- db,
1751
- input.accountId,
1752
- input.workspaceId,
1753
- isCodexTurn,
1754
- entitlements,
2557
+ await waitForTurnOperation(
2558
+ ensureRunAllowed(
2559
+ settings,
2560
+ db,
2561
+ input.accountId,
2562
+ input.workspaceId,
2563
+ isExternallyBilledTurn,
2564
+ entitlements,
2565
+ ),
2566
+ cancellationSignal,
2567
+ undefined,
1755
2568
  );
1756
2569
  // Setup (variableSet load, MCP connects, sandbox restore) does not
1757
2570
  // stream and so never observes cancellation on its own; these explicit
@@ -1763,6 +2576,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1763
2576
  throw reason;
1764
2577
  }
1765
2578
  };
2579
+ const throwIfTurnCancelled = () => throwIfTurnOperationCancelled(cancellationSignal);
1766
2580
  // ONE shared details object for every heartbeat this activity sends (each
1767
2581
  // site spreads it + its own phase), so cross-site fields — the op-stream
1768
2582
  // settled roster in particular — survive last-write-wins instead of being
@@ -1792,6 +2606,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1792
2606
  // one exactly like production.
1793
2607
  codexLeaseHolderId = dispatchId;
1794
2608
  const modelUsageDispatchId = activityContext?.info.activityId ?? dispatchId;
2609
+ const claimedModelUsageSourceKeys = new Set<string>();
1795
2610
  const emittedModelUsageSourceKeys = new Set<string>();
1796
2611
  publish = async (
1797
2612
  events: Array<Omit<AppendEventInput, "producerId" | "producerSeq" | "turnId">>,
@@ -1858,6 +2673,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1858
2673
  producerSeq: ++producerSeq,
1859
2674
  };
1860
2675
  }
2676
+ const compactionRequestFailure = inputSettlement.consumeRequestedCompactionFailure
2677
+ ? {
2678
+ reason: "summarization_failed" as const,
2679
+ producerId,
2680
+ producerSeq: ++producerSeq,
2681
+ }
2682
+ : undefined;
1861
2683
  const inputs = inputSettlement.events.map((event) => ({
1862
2684
  ...event,
1863
2685
  payload: redact(event.payload),
@@ -1874,9 +2696,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1874
2696
  sessionStatus: inputSettlement.sessionStatus,
1875
2697
  activeTurnId: inputSettlement.activeTurnId,
1876
2698
  events: inputs,
2699
+ ...(inputSettlement.runState ? { runState: inputSettlement.runState } : {}),
1877
2700
  ...(recordingMutation ? { recording: recordingMutation } : {}),
2701
+ ...(compactionRequestFailure ? { compactionRequestFailure } : {}),
1878
2702
  });
1879
2703
  if (result.action === "stale") {
2704
+ // The terminal write can lose to a control transaction before the
2705
+ // workflow delivers Temporal cancellation. That control may settle
2706
+ // the already-closed attempt as rejected_stale, so returning without
2707
+ // this flag would strand its replacement behind quiesced_at forever.
2708
+ // Enter the same hard tool-fence/receipt path as an explicit
2709
+ // TurnAttemptFencedError. If ownership was lost for an unrelated
2710
+ // reason, allowUninterrupted makes the receipt transaction a no-op.
2711
+ acknowledgeLostAttemptOwnership();
1880
2712
  if (recordingForSettlement) {
1881
2713
  await abandonActiveRecording(
1882
2714
  "recording settlement lost attempt ownership",
@@ -1917,6 +2749,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1917
2749
  // visibly starts: nothing ran yet, so the same inference starts cleanly
1918
2750
  // on a healthy worker.
1919
2751
  throwIfWorkerShuttingDown();
2752
+ throwIfTurnCancelled();
1920
2753
  if (
1921
2754
  !(await settle({
1922
2755
  events: [
@@ -1953,7 +2786,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1953
2786
  sessionPinSource,
1954
2787
  sessionLastCredentialId: sessionCodex?.lastCredentialId ?? null,
1955
2788
  continuationCredentialId: continuationCodexCredentialId,
1956
- nearExhaustionPct: settings.codexRotationNearExhaustionPct,
1957
2789
  now: new Date(),
1958
2790
  });
1959
2791
 
@@ -2075,7 +2907,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2075
2907
  strategy: leased.rotationStrategy as CodexRotationStrategy,
2076
2908
  rotationEnabled: leased.rotationEnabled,
2077
2909
  });
2078
- // OPE-31 pin persistence follows the atomic OPE-21 selection. The selector
2910
+ // pin policy pin persistence follows the atomic credential allocator selection. The selector
2079
2911
  // already ran exact-turn reuse before policy filtering and vetoed pointer
2080
2912
  // movement for manual/policy homes; this write only records the NEXT turn's
2081
2913
  // policy home (or clears a policy pin whose strategy is no longer active).
@@ -2163,7 +2995,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2163
2995
  if (codexLeaseHeld) startCodexLeaseHeartbeat();
2164
2996
 
2165
2997
  const eligibleCount = leased.accounts.filter((account) =>
2166
- isCodexCredentialEligible(account, settings.codexRotationNearExhaustionPct, new Date()),
2998
+ isCodexCredentialEligible(account, new Date()),
2167
2999
  ).length;
2168
3000
  const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
2169
3001
  observability.incrementCounter({
@@ -2340,7 +3172,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2340
3172
  if (goalActive && goal) {
2341
3173
  const authoritativeResetAt = authoritativeCodexCapacityResetAt(
2342
3174
  leased.accounts,
2343
- settings.codexRotationNearExhaustionPct,
2344
3175
  new Date(),
2345
3176
  );
2346
3177
  const armed = await armCodexCapacityWait(db, {
@@ -2482,7 +3313,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2482
3313
  // Resolved after turn.started so a composition conflict (two enabled
2483
3314
  // packs declaring images) fails the turn with its plain error instead
2484
3315
  // of failing the activity opaquely.
2485
- const packRuntime = await resolveWorkspacePackRuntime(db, input.workspaceId);
3316
+ const [packRuntime, skillLibraryRuntime] = await Promise.all([
3317
+ resolveWorkspacePackRuntime(db, input.workspaceId),
3318
+ resolveWorkspaceSkillLibraryRuntime(db, input.workspaceId),
3319
+ ]);
2486
3320
  // RIG BINDING (M3): load the session's FROZEN rig version (resolved+frozen
2487
3321
  // at create). Everything rig-derived below (image precedence, env default
2488
3322
  // sets, setup hook, credential hooks, doctrine, lease/telemetry stamps) is
@@ -2530,6 +3364,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2530
3364
  db,
2531
3365
  input.workspaceId,
2532
3366
  input.sessionId,
3367
+ input.attemptId,
2533
3368
  baseRunSettings,
2534
3369
  );
2535
3370
  // Multi-provider per-turn routing → the provider gating (compaction mode,
@@ -2545,14 +3380,17 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2545
3380
  // a chat-only Fireworks model. Resolving against the default-model settings
2546
3381
  // keeps gating consistent with the router. Cost accounting covers registry
2547
3382
  // models via configuredModelPricing.
2548
- const resolvedModel = runtime.resolveTurnModel(capabilitySettings, turn.model);
3383
+ const resolvedModel = runtime.resolveTurnModel(
3384
+ capabilitySettings,
3385
+ turnExecutionPolicy.productModelId,
3386
+ );
2549
3387
  // Bind the provider/model catalog's context policy to every model-facing
2550
3388
  // path for this turn. In particular, Codex subscription turns must not
2551
3389
  // inherit the deployment's OpenAI/Azure mode or 1.05M context defaults:
2552
3390
  // raw window, effective ceiling, and auto-compact limit are distinct live
2553
3391
  // catalog values and must reach pre-turn compaction, history guards, and
2554
3392
  // every model call together.
2555
- const modelRunSettings = resolvedModel
3393
+ modelRunSettings = resolvedModel
2556
3394
  ? settingsWithResolvedModelContext(runSettings, resolvedModel.configured)
2557
3395
  : runSettings;
2558
3396
  // WORKSPACE MODEL POLICY — the authoritative hard gate. Runs immediately
@@ -2560,25 +3398,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2560
3398
  // and the main run both come later in this scope), so a blocked
2561
3399
  // provider/model can never be reached through ANY stamp path: explicit
2562
3400
  // turn model, inherited session default, goal-continuation inheritance,
2563
- // or the legacy null-resolution fallback (null → the built-in
2564
- // OpenAI/Azure client, attributed here via builtinProviderId so a policy
2565
- // blocking the built-in also blocks that fallback — this exact path is
2566
- // how bare-model turns silently spent real Azure money in a
2567
- // codex-intended workspace). Fail-loud, never a silent remap.
3401
+ // or the legacy null-resolution fallback. The frozen execution policy is
3402
+ // the attribution source even if an injected test runtime returns no
3403
+ // concrete resolved model. Fail-loud, never a silent remap.
2568
3404
  {
2569
3405
  const workspaceModelPolicy = await getWorkspaceModelPolicy(db, input.workspaceId);
2570
3406
  if (workspaceModelPolicy) {
2571
- const effectiveProviderId = resolvedModel
2572
- ? resolvedModel.provider.id
2573
- : builtinProviderId(capabilitySettings);
2574
3407
  const verdict = evaluateWorkspaceModelPolicy(workspaceModelPolicy, {
2575
- providerId: effectiveProviderId,
2576
- modelId: turn.model,
3408
+ providerId: turnExecutionPolicy.providerId,
3409
+ modelId: turnExecutionPolicy.productModelId,
2577
3410
  });
2578
3411
  if (!verdict.allowed) {
2579
3412
  throw new WorkspaceModelPolicyBlockedError(
2580
- turn.model,
2581
- effectiveProviderId,
3413
+ turnExecutionPolicy.productModelId,
3414
+ turnExecutionPolicy.providerId,
2582
3415
  verdict.reason,
2583
3416
  );
2584
3417
  }
@@ -2592,6 +3425,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2592
3425
  // Build it once and wrap BOTH the compaction summarizer (a separate model
2593
3426
  // call on the same codex client) and the main run; otherwise the summarizer
2594
3427
  // would hit the codex backend unauthenticated.
3428
+ let codexModelRequestSequence = 0;
2595
3429
  const codexContext: CodexRequestContext | null =
2596
3430
  resolvedModel?.provider.kind === "codex-subscription"
2597
3431
  ? ((): CodexRequestContext => {
@@ -2622,6 +3456,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2622
3456
  onUsageHeaders: (snapshot) => {
2623
3457
  latestCodexUsage = snapshot;
2624
3458
  }, // latest wins; flushed once in finally
3459
+ nextRequestId: () => `${dispatchId}:${++codexModelRequestSequence}`,
3460
+ onModelRequestEvent: async (event) => {
3461
+ if (!publish || !turnId) {
3462
+ throw new Error("Codex model request started before the turn event producer");
3463
+ }
3464
+ await publish([
3465
+ {
3466
+ type: "agent.model.request",
3467
+ payload: {
3468
+ ...event,
3469
+ provider: "codex-subscription",
3470
+ turnId,
3471
+ attemptId: input.attemptId,
3472
+ dispatchId,
3473
+ executionGeneration,
3474
+ },
3475
+ },
3476
+ ]);
3477
+ },
2625
3478
  };
2626
3479
  })()
2627
3480
  : null;
@@ -2630,52 +3483,33 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2630
3483
  const promptCacheKey = acceptsPromptCacheKeyForTurn(resolvedModel)
2631
3484
  ? input.sessionId
2632
3485
  : undefined;
2633
- let compactionUsageCount = 0;
3486
+ const compactionUsageState = createCompactionModelUsageEventState(
3487
+ claimedModelUsageSourceKeys,
3488
+ );
2634
3489
  const recordCompactionUsage = async (usage: ModelResponseUsage) => {
2635
- await recordCompletedModelCallBeforeOwnershipFences({
3490
+ await processCompactionModelUsageEvent({
3491
+ usage,
3492
+ state: compactionUsageState,
3493
+ dispatchId: modelUsageDispatchId,
3494
+ settings,
3495
+ db,
3496
+ observability,
3497
+ publish,
3498
+ accountId: input.accountId,
3499
+ workspaceId: input.workspaceId,
3500
+ sessionId: input.sessionId,
3501
+ turnId: turn.id,
3502
+ provider: resolvedModel?.provider.id ?? settings.openaiProvider,
3503
+ providerApi: resolvedModel?.provider.api ?? "responses",
3504
+ model: resolvedModel?.configured.id ?? turn.model,
3505
+ externallyBilled: isExternallyBilledTurn,
3506
+ turnAttemptId: input.attemptId,
3507
+ servingCredentialId: effectiveCodexCredentialId,
3508
+ priorSessionCredentialId: priorSessionCodexCredentialId,
3509
+ emittedSourceKeys: emittedModelUsageSourceKeys,
2636
3510
  renewLease: () => renewCodexLease("model_usage"),
2637
3511
  leaseLost: () => codexLeaseLost,
2638
3512
  leaseLostMessage: "Codex credential lease expired during context compaction",
2639
- recordUsage: async () => {
2640
- compactionUsageCount += 1;
2641
- const sourceKey = modelUsageSourceKey({
2642
- responseId: usage.responseId,
2643
- dispatchId: modelUsageDispatchId,
2644
- positionalKey: `compaction-${compactionUsageCount}`,
2645
- });
2646
- const responseAccountCtx = modelCallAccountContext({
2647
- servingCredentialId: effectiveCodexCredentialId,
2648
- priorSessionCredentialId: priorSessionCodexCredentialId,
2649
- isFirstCallOfTurn: compactionUsageCount === 1,
2650
- });
2651
- await recordModelUsageAndDebitCredits(settings, db, {
2652
- accountId: input.accountId,
2653
- workspaceId: input.workspaceId,
2654
- sessionId: input.sessionId,
2655
- turnId: turn.id,
2656
- model: resolvedModel?.configured.id ?? turn.model,
2657
- isCodexTurn,
2658
- usage: usage.usage,
2659
- sourceKey,
2660
- observability,
2661
- });
2662
- await emitModelCallUsage({
2663
- observability,
2664
- publish,
2665
- accountId: input.accountId,
2666
- workspaceId: input.workspaceId,
2667
- sessionId: input.sessionId,
2668
- turnId: turn.id,
2669
- provider: resolvedModel?.provider.id ?? settings.openaiProvider,
2670
- providerApi: resolvedModel?.provider.api ?? "responses",
2671
- model: resolvedModel?.configured.id ?? turn.model,
2672
- sourceKey,
2673
- usage,
2674
- servingAccountHash: responseAccountCtx.servingAccountHash,
2675
- accountChangedFromPrevCall: responseAccountCtx.accountChangedFromPrevCall,
2676
- emittedSourceKeys: emittedModelUsageSourceKeys,
2677
- });
2678
- },
2679
3513
  });
2680
3514
  };
2681
3515
  const compactionSummarizerFor = (systemInstructions?: string) =>
@@ -2685,7 +3519,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2685
3519
  summarizeForCompaction(s, m, {
2686
3520
  client: resolvedModel.client,
2687
3521
  api: resolvedModel.provider.api,
2688
- model: resolvedModel.configured.id,
3522
+ model: turnExecutionPolicy.upstreamModelId,
2689
3523
  maxOutputTokens: SUMMARY_BUFFER_TOKENS,
2690
3524
  onUsage: recordCompactionUsage,
2691
3525
  ...(systemInstructions ? { systemInstructions } : {}),
@@ -2694,6 +3528,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2694
3528
  )
2695
3529
  : (s: Settings, m: Array<Record<string, unknown>>) =>
2696
3530
  summarizeForCompaction(s, m, {
3531
+ model: turnExecutionPolicy.upstreamModelId,
2697
3532
  maxOutputTokens: SUMMARY_BUFFER_TOKENS,
2698
3533
  onUsage: recordCompactionUsage,
2699
3534
  ...(systemInstructions ? { systemInstructions } : {}),
@@ -2725,25 +3560,69 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2725
3560
  );
2726
3561
  let outcome: Awaited<ReturnType<typeof maybeCompactContext>> | null = null;
2727
3562
  if (requested) {
2728
- outcome = await maybeCompactContext(
2729
- db,
2730
- modelRunSettings,
2731
- {
2732
- accountId: input.accountId,
2733
- workspaceId: input.workspaceId,
2734
- sessionId: input.sessionId,
2735
- turnId: turn.id,
2736
- executionGeneration,
2737
- attemptId: input.attemptId,
2738
- },
2739
- session.lastInputTokens,
2740
- compactionSummarizerFor(compactionInstructions),
2741
- {
2742
- force: true,
2743
- clearRequestedCompaction: true,
2744
- trigger: "operator",
2745
- },
2746
- );
3563
+ try {
3564
+ outcome = await waitForTurnOperation(
3565
+ maybeCompactContext(
3566
+ db,
3567
+ modelRunSettings,
3568
+ {
3569
+ accountId: input.accountId,
3570
+ workspaceId: input.workspaceId,
3571
+ sessionId: input.sessionId,
3572
+ turnId: turn.id,
3573
+ executionGeneration,
3574
+ attemptId: input.attemptId,
3575
+ },
3576
+ session.lastInputTokens,
3577
+ compactionSummarizerFor(compactionInstructions),
3578
+ {
3579
+ force: true,
3580
+ clearRequestedCompaction: true,
3581
+ trigger: "operator",
3582
+ },
3583
+ ),
3584
+ cancellationSignal,
3585
+ undefined,
3586
+ );
3587
+ } catch (error) {
3588
+ // Codex retries retryable checkpoint-provider failures rather than
3589
+ // treating them as a semantic compaction result. Keep the operator
3590
+ // request pending and let the ordinary same-turn provider/capacity
3591
+ // recovery path re-dispatch this exact maintenance execution.
3592
+ if (shouldRecoverCompactionProviderFailure(error)) throw error;
3593
+ if (!isCompactionSummaryFailure(error)) throw error;
3594
+ const errorMessage = compactionFailureReasonFromError(error);
3595
+ if (
3596
+ !(await settle!({
3597
+ events: [
3598
+ {
3599
+ type: "turn.failed",
3600
+ payload: {
3601
+ error: errorMessage,
3602
+ code: "context_compaction_failed",
3603
+ retryable: false,
3604
+ recovery: "user_message",
3605
+ compacted: false,
3606
+ },
3607
+ },
3608
+ {
3609
+ type: "session.status.changed",
3610
+ payload: { status: "idle" },
3611
+ },
3612
+ ],
3613
+ turnStatus: "failed",
3614
+ sessionStatus: "idle",
3615
+ activeTurnId: null,
3616
+ consumeRequestedCompactionFailure: true,
3617
+ }))
3618
+ ) {
3619
+ return claimedResult({ status: "cancelled" });
3620
+ }
3621
+ turnMetricOutcome = "failed";
3622
+ activityStatus = "idle";
3623
+ activityError = error;
3624
+ return claimedResult({ status: "idle" });
3625
+ }
2747
3626
  if (outcome.events.length > 0) {
2748
3627
  if (outcome.compacted) {
2749
3628
  recordContextCompaction(observability, "operator");
@@ -2790,22 +3669,42 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2790
3669
  // connector scopes); no-op for every other turn. Its refreshing bearer is
2791
3670
  // resolved at connect time from the codex ALS (see the withCodex-wrapped
2792
3671
  // prepareTools call below).
3672
+ // Resolve the durable policy at the turn boundary. Workspace-default
3673
+ // sessions may discover newly enabled capability MCPs, while explicit,
3674
+ // inherited-fixed, and legacy sessions remain narrowed to their stored
3675
+ // materialized allow-list. `withCodexAppsTool` below keeps the existing
3676
+ // single Codex lazy router/connector overlay; it is not a second policy
3677
+ // or discovery path.
3678
+ const effectivePolicyTools = resolveSessionToolPolicy({
3679
+ ...(session.toolPolicy ? { toolPolicy: session.toolPolicy } : {}),
3680
+ sessionTools: session.tools,
3681
+ turnTools: turn.tools,
3682
+ ...(turn.toolsProvided !== undefined ? { turnToolsProvided: turn.toolsProvided } : {}),
3683
+ availableMcpServerIds: runSettings.mcpServers.map((server) => server.id),
3684
+ defaultMcpServerIds: enabledCapabilityMcpToolRefs(settings, mcpSettings).map(
3685
+ (tool) => tool.id,
3686
+ ),
3687
+ }).toolRefs;
2793
3688
  const turnTools = withCodexAppsTool(
2794
3689
  runSettings,
2795
- withFirstPartyTools(runSettings, mergeToolRefs(session.tools, turn.tools)),
3690
+ withFirstPartyTools(runSettings, effectivePolicyTools),
2796
3691
  );
2797
- // §7.6 P4a — load (and decrypt) the variable set via the host
3692
+ // §7.6 connection-credential provider — load (and decrypt) the variable set via the host
2798
3693
  // `sandboxSecrets` provider when bound; unset → today's local decrypt.
2799
3694
  const connectionScope = {
2800
3695
  accountId: input.accountId,
2801
3696
  workspaceId: input.workspaceId,
2802
3697
  };
2803
- const workspaceVariableSet = await loadWorkspaceEnvironmentForRunWithCredentials(
2804
- db,
2805
- runSettings,
2806
- connectionScope,
2807
- session.variableSetId,
2808
- connectionCredentials?.sandboxSecrets,
3698
+ const workspaceVariableSet = await waitForTurnOperation(
3699
+ loadWorkspaceEnvironmentForRunWithCredentials(
3700
+ db,
3701
+ runSettings,
3702
+ connectionScope,
3703
+ session.variableSetId,
3704
+ connectionCredentials?.sandboxSecrets,
3705
+ ),
3706
+ cancellationSignal,
3707
+ undefined,
2809
3708
  );
2810
3709
  variableSetId = workspaceVariableSet?.id ?? "";
2811
3710
  // RIG DEFAULT VARIABLE SETS (M3): decrypt the frozen rig version's default
@@ -2820,12 +3719,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2820
3719
  // set already relies on), keeping validateNoEnvironmentDelta empty.
2821
3720
  const rigDefaultEnvironmentValues: Record<string, string> = {};
2822
3721
  for (const rigDefaultVariableSetId of rigVersion?.defaultVariableSetIds ?? []) {
2823
- const rigDefaultSet = await loadWorkspaceEnvironmentForRunWithCredentials(
2824
- db,
2825
- runSettings,
2826
- connectionScope,
2827
- rigDefaultVariableSetId,
2828
- connectionCredentials?.sandboxSecrets,
3722
+ const rigDefaultSet = await waitForTurnOperation(
3723
+ loadWorkspaceEnvironmentForRunWithCredentials(
3724
+ db,
3725
+ runSettings,
3726
+ connectionScope,
3727
+ rigDefaultVariableSetId,
3728
+ connectionCredentials?.sandboxSecrets,
3729
+ ),
3730
+ cancellationSignal,
3731
+ undefined,
2829
3732
  );
2830
3733
  Object.assign(rigDefaultEnvironmentValues, rigDefaultSet?.values ?? {});
2831
3734
  }
@@ -2836,7 +3739,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2836
3739
  );
2837
3740
  // Redact EVERY exported secret value (rig defaults + session set) from turn
2838
3741
  // output, not just the session set's.
2839
- redact = createSecretRedactor(
3742
+ registerSecretRedactions(
2840
3743
  Object.entries(sandboxWorkspaceEnvironmentValues).map(([name, value]) => ({ name, value })),
2841
3744
  );
2842
3745
  // EFFECTIVE compute backend, resolved ONCE at turn start (Case B + Stage D
@@ -2909,7 +3812,56 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2909
3812
  settings.sandboxOwnershipEnabled && runSettings.sandboxBackend !== "none"
2910
3813
  ? groupBoxBackend
2911
3814
  : runSettings.sandboxBackend;
2912
- await ensureTurnModalRegistryImage(runSettings, sandboxCreationBackend);
3815
+ const effectiveRunCredentialBackend = activeSandboxBackend ?? groupBoxBackend;
3816
+ const runCredentialResolver =
3817
+ effectiveRunCredentialBackend === "none"
3818
+ ? null
3819
+ : await waitForTurnOperation(
3820
+ bindRunCredentialResolver({
3821
+ db,
3822
+ connectionCredentials: connectionCredentials ?? null,
3823
+ accountId: input.accountId,
3824
+ workspaceId: input.workspaceId,
3825
+ session,
3826
+ turn,
3827
+ attemptId: input.attemptId,
3828
+ effectiveSandboxBackend: effectiveRunCredentialBackend,
3829
+ variableSet: workspaceVariableSet
3830
+ ? {
3831
+ id: workspaceVariableSet.id,
3832
+ name: workspaceVariableSet.name,
3833
+ }
3834
+ : null,
3835
+ }),
3836
+ cancellationSignal,
3837
+ undefined,
3838
+ );
3839
+ const initialRunCredentialMaterial = runCredentialResolver
3840
+ ? await waitForTurnOperation(
3841
+ runCredentialResolver.resolve({
3842
+ purpose: "provision",
3843
+ forceRefresh: false,
3844
+ }),
3845
+ cancellationSignal,
3846
+ undefined,
3847
+ )
3848
+ : null;
3849
+ if (initialRunCredentialMaterial) {
3850
+ registerSecretRedactions(initialRunCredentialMaterial.redactions);
3851
+ for (const payload of runCredentialAuthNeededPayloads(initialRunCredentialMaterial)) {
3852
+ publishedRunCredentialNotices.add(JSON.stringify(payload));
3853
+ await publish!([{ type: "credential.auth_needed", payload }], true);
3854
+ }
3855
+ }
3856
+ const runCredentialsNote = initialRunCredentialMaterial
3857
+ ? runCredentialModelNote(initialRunCredentialMaterial)
3858
+ : undefined;
3859
+ throwIfTurnOperationCancelled(cancellationSignal);
3860
+ await waitForTurnOperation(
3861
+ ensureTurnModalRegistryImage(runSettings, sandboxCreationBackend),
3862
+ cancellationSignal,
3863
+ undefined,
3864
+ );
2913
3865
  const establishPolicy: "eager" | "on-demand" =
2914
3866
  lazyProvisionEnabled(settings) && !machinePrimary && runSettings.sandboxBackend !== "none"
2915
3867
  ? "on-demand"
@@ -2933,134 +3885,396 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2933
3885
  // provider may supply it; unset still self-mints GitHub from settings.
2934
3886
  // gitToken/gitTokens are undefined on the selfhosted skip path (the machine
2935
3887
  // uses its own git creds).
3888
+ if (activeSandboxBackend !== "selfhosted") {
3889
+ await assertGitHubResourcesRemainAuthorized(db, input.workspaceId, turnResources);
3890
+ }
3891
+ const authorizeGitHubTokenMint: GitHubTokenMintAuthorization = async (selection) => {
3892
+ await assertGitHubTokenMintSelectionAuthorized(
3893
+ db,
3894
+ input.workspaceId,
3895
+ selection.installationId,
3896
+ selection.repositoryIds,
3897
+ );
3898
+ };
3899
+ // Git and MCP credentials share one lineage snapshot for this turn. A
3900
+ // host that supplies both ports must never see two independently resolved
3901
+ // roots for the same execution merely because the call sites are far apart.
3902
+ const needsHostCredentialRoot = Boolean(
3903
+ connectionCredentials?.gitCredentials || connectionCredentials?.mcpCredentials,
3904
+ );
3905
+ const hostCredentialRootSessionId = needsHostCredentialRoot
3906
+ ? await getSessionRootId(db, input.workspaceId, input.sessionId)
3907
+ : null;
3908
+ if (needsHostCredentialRoot && !hostCredentialRootSessionId) {
3909
+ throw new Error(`cannot resolve host credentials for missing session ${input.sessionId}`);
3910
+ }
3911
+ const gitCredentialAuthority =
3912
+ connectionCredentials?.gitCredentials && hostCredentialRootSessionId
3913
+ ? gitCredentialAuthorityForTurn({
3914
+ sessionId: input.sessionId,
3915
+ rootSessionId: hostCredentialRootSessionId,
3916
+ attemptId: input.attemptId,
3917
+ turn,
3918
+ })
3919
+ : undefined;
3920
+ const toolspaceAuthority = {
3921
+ sessionId: input.sessionId,
3922
+ };
2936
3923
  const {
2937
3924
  environment: sandboxEnvironment,
2938
3925
  gitToken: sandboxGitToken,
2939
3926
  gitTokens: sandboxGitTokens,
2940
3927
  gitTokenExpiresAt: sandboxGitTokenExpiresAt,
3928
+ gitCredentialBindings: sandboxGitCredentialBindings,
2941
3929
  toolspaceToken: sandboxToolspaceToken,
2942
- } = await sandboxEnvironmentForRun(
2943
- runSettings,
2944
- turnResources,
2945
- // Rig default sets merged BELOW the session set (session wins); rig-less
2946
- // turns pass exactly workspaceVariableSet?.values (byte-for-byte today).
2947
- sandboxWorkspaceEnvironmentValues,
2948
- {
2949
- skipGitHubToken: activeSandboxBackend === "selfhosted",
2950
- deferGitHubToken:
2951
- activeSandboxBackend !== "selfhosted" && establishPolicy === "on-demand",
2952
- scope: connectionScope,
2953
- gitCredentials: connectionCredentials?.gitCredentials,
2954
- sessionId: input.sessionId,
2955
- runId: turnId,
2956
- },
3930
+ toolspaceTokenExpiresAt: sandboxToolspaceTokenExpiresAt,
3931
+ } = await waitForTurnOperation(
3932
+ sandboxEnvironmentForRun(
3933
+ runSettings,
3934
+ turnResources,
3935
+ // Rig default sets merged BELOW the session set (session wins); rig-less
3936
+ // turns pass exactly workspaceVariableSet?.values (byte-for-byte today).
3937
+ sandboxWorkspaceEnvironmentValues,
3938
+ {
3939
+ skipGitHubToken: activeSandboxBackend === "selfhosted",
3940
+ deferGitHubToken:
3941
+ activeSandboxBackend !== "selfhosted" && establishPolicy === "on-demand",
3942
+ scope: connectionScope,
3943
+ ...(gitCredentialAuthority ? { authority: gitCredentialAuthority } : {}),
3944
+ gitCredentials: connectionCredentials?.gitCredentials,
3945
+ authorizeGitHubTokenMint,
3946
+ toolspaceAuthority,
3947
+ },
3948
+ ),
3949
+ cancellationSignal,
3950
+ undefined,
2957
3951
  );
2958
-
2959
- const initialGitCredentials: MintedRunGitCredentials | undefined = sandboxGitTokens
2960
- ? {
2961
- gitTokens: sandboxGitTokens,
2962
- expiresAt: sandboxGitTokenExpiresAt ?? {},
2963
- }
3952
+ const sandboxToolspaceTokenFile = sandboxToolspaceToken
3953
+ ? toolspaceTokenFileFromEnvironment(sandboxEnvironment, input.sessionId)
2964
3954
  : undefined;
3955
+
3956
+ const initialGitCredentials: MintedRunGitCredentials | undefined =
3957
+ sandboxGitCredentialBindings
3958
+ ? {
3959
+ bindings: sandboxGitCredentialBindings,
3960
+ gitTokens: sandboxGitTokens ?? {},
3961
+ expiresAt: sandboxGitTokenExpiresAt ?? {},
3962
+ }
3963
+ : undefined;
2965
3964
  const attachGitCredentialRenewal = async (
2966
3965
  tokenSession: GitCredentialTokenWriterSession,
2967
3966
  initial: MintedRunGitCredentials | undefined,
2968
3967
  ): Promise<void> => {
2969
- if (!initial || Object.keys(initial.gitTokens).length === 0) return;
2970
- const previous = gitCredentialRenewal;
2971
- gitCredentialRenewal = null;
2972
- await previous?.stop();
3968
+ if (!initial || initial.bindings.length === 0) return;
3969
+ const previous = gitCredentialRenewals;
3970
+ gitCredentialRenewals = [];
3971
+ await Promise.all(previous.map(async (controller) => await controller.stop()));
2973
3972
  if (gitCredentialRenewalClosed) return;
2974
3973
 
2975
- const providers = Object.keys(initial.gitTokens) as GitCredentialProvider[];
2976
- const controller = startGitCredentialRenewalLoop({
2977
- expectedProviders: providers,
2978
- initialExpiresAt: initial.expiresAt,
2979
- mint: async () =>
2980
- await mintRunGitCredentials(runSettings, turnResources, {
2981
- scope: connectionScope,
2982
- gitCredentials: connectionCredentials?.gitCredentials,
2983
- }),
2984
- write: async (tokens) => {
2985
- const runAs = sandboxRunAs(runSettings);
2986
- await refreshGitProviderTokenFiles(tokenSession, tokens, {
2987
- ...(runAs ? { runAs } : {}),
2988
- });
2989
- },
2990
- onSuccess: ({ providers: renewedProviders }) => {
2991
- for (const provider of renewedProviders) {
2992
- observability.incrementCounter({
2993
- name: "opengeni_git_credential_renewals_total",
2994
- help: "Host-managed Git credential renewal attempts by provider and outcome.",
2995
- labels: { provider, outcome: "completed" },
3974
+ const controllers = initial.bindings.map((initialBinding) => {
3975
+ let pendingBinding: typeof initialBinding | undefined;
3976
+ return startGitCredentialRenewalLoop({
3977
+ expectedProviders: [initialBinding.provider],
3978
+ initialExpiresAt: initialBinding.expiresAt
3979
+ ? { [initialBinding.provider]: initialBinding.expiresAt }
3980
+ : {},
3981
+ mint: async () => {
3982
+ const binding = await mintRunGitCredentialBinding(
3983
+ runSettings,
3984
+ turnResources,
3985
+ initialBinding.provider,
3986
+ initialBinding.credentialBindingId,
3987
+ {
3988
+ scope: connectionScope,
3989
+ ...(gitCredentialAuthority ? { authority: gitCredentialAuthority } : {}),
3990
+ gitCredentials: connectionCredentials?.gitCredentials,
3991
+ authorizeGitHubTokenMint,
3992
+ },
3993
+ );
3994
+ if (binding) {
3995
+ assertGitCredentialRenewalTransportUnchanged(initialBinding, binding);
3996
+ }
3997
+ pendingBinding = binding;
3998
+ return binding
3999
+ ? {
4000
+ bindings: [binding],
4001
+ gitTokens: { [binding.provider]: binding.token },
4002
+ expiresAt: binding.expiresAt ? { [binding.provider]: binding.expiresAt } : {},
4003
+ }
4004
+ : undefined;
4005
+ },
4006
+ write: async () => {
4007
+ if (!pendingBinding) {
4008
+ throw new Error("credential renewal produced no binding token");
4009
+ }
4010
+ const runAs = sandboxRunAs(runSettings);
4011
+ await refreshGitCredentialBindingTokenFiles(tokenSession, [pendingBinding], {
4012
+ ...(runAs ? { runAs } : {}),
4013
+ ...(toolCancellationFenceRef.current
4014
+ ? {
4015
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4016
+ toolCancellationFenceRef.current,
4017
+ ),
4018
+ }
4019
+ : {}),
2996
4020
  });
2997
- }
2998
- },
2999
- onFailure: ({ providers: failedProviders, retryDelayMs, errorClass }) => {
3000
- for (const provider of failedProviders) {
3001
- observability.incrementCounter({
3002
- name: "opengeni_git_credential_renewals_total",
3003
- help: "Host-managed Git credential renewal attempts by provider and outcome.",
3004
- labels: { provider, outcome: "error" },
4021
+ },
4022
+ onSuccess: ({ providers: renewedProviders }) => {
4023
+ for (const provider of renewedProviders) {
4024
+ observability.incrementCounter({
4025
+ name: "opengeni_git_credential_renewals_total",
4026
+ help: "Host-managed Git credential renewal attempts by provider and outcome.",
4027
+ labels: { provider, outcome: "completed" },
4028
+ });
4029
+ }
4030
+ },
4031
+ onFailure: ({ providers: failedProviders, retryDelayMs, errorClass }) => {
4032
+ for (const provider of failedProviders) {
4033
+ observability.incrementCounter({
4034
+ name: "opengeni_git_credential_renewals_total",
4035
+ help: "Host-managed Git credential renewal attempts by provider and outcome.",
4036
+ labels: { provider, outcome: "error" },
4037
+ });
4038
+ }
4039
+ observability.warn("Sandbox Git credential renewal failed; retry scheduled", {
4040
+ sessionId: input.sessionId,
4041
+ turnId,
4042
+ providers: failedProviders.join(","),
4043
+ errorClass,
4044
+ retryDelayMs,
3005
4045
  });
3006
- }
3007
- observability.warn("Sandbox Git credential renewal failed; retry scheduled", {
4046
+ },
4047
+ });
4048
+ });
4049
+ if (gitCredentialRenewalClosed) {
4050
+ await Promise.all(controllers.map(async (controller) => await controller.stop()));
4051
+ return;
4052
+ }
4053
+ gitCredentialRenewals = controllers;
4054
+ };
4055
+
4056
+ const attachToolspaceTokenRenewal = async (
4057
+ tokenSession: ToolspaceTokenWriterSession,
4058
+ initialExpiresAt = sandboxToolspaceTokenExpiresAt,
4059
+ ): Promise<void> => {
4060
+ if (!sandboxToolspaceToken || !initialExpiresAt) return;
4061
+ const previous = toolspaceTokenRenewal;
4062
+ toolspaceTokenRenewal = null;
4063
+ await previous?.stop();
4064
+ if (toolspaceTokenRenewalClosed) return;
4065
+
4066
+ const mint = async () =>
4067
+ await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority);
4068
+ const write = async (material: NonNullable<Awaited<ReturnType<typeof mint>>>) => {
4069
+ const runAs = sandboxRunAs(runSettings);
4070
+ await refreshToolspaceTokenFile(tokenSession, material.token, {
4071
+ ...(runAs ? { runAs } : {}),
4072
+ ...(sandboxToolspaceTokenFile
4073
+ ? {
4074
+ tokenFile: sandboxToolspaceTokenFile,
4075
+ legacyTokenFile: sandboxEnvironment.OPENGENI_TOOLSPACE_TOKEN_FILE!,
4076
+ }
4077
+ : {}),
4078
+ ...(toolCancellationFenceRef.current
4079
+ ? {
4080
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4081
+ toolCancellationFenceRef.current,
4082
+ ),
4083
+ }
4084
+ : {}),
4085
+ });
4086
+ };
4087
+ let renewalExpiresAt = initialExpiresAt;
4088
+ if (renewalExpiresAt.getTime() <= Date.now() + TOOLSPACE_TOKEN_EXPIRY_LEAD_MS) {
4089
+ const fresh = await mint();
4090
+ if (!fresh) {
4091
+ throw new Error("Toolspace token mint became unavailable during sandbox setup");
4092
+ }
4093
+ await write(fresh);
4094
+ renewalExpiresAt = fresh.expiresAt;
4095
+ }
4096
+ const controller = startToolspaceTokenRenewalLoop({
4097
+ initialExpiresAt: renewalExpiresAt,
4098
+ mint,
4099
+ write,
4100
+ onSuccess: () => {
4101
+ observability.incrementCounter({
4102
+ name: "opengeni_toolspace_token_renewals_total",
4103
+ help: "Sandbox Toolspace token renewal attempts by outcome.",
4104
+ labels: { outcome: "completed" },
4105
+ });
4106
+ },
4107
+ onFailure: ({ retryDelayMs, errorClass }) => {
4108
+ observability.incrementCounter({
4109
+ name: "opengeni_toolspace_token_renewals_total",
4110
+ help: "Sandbox Toolspace token renewal attempts by outcome.",
4111
+ labels: { outcome: "error" },
4112
+ });
4113
+ observability.warn("Sandbox Toolspace token renewal failed; retry scheduled", {
3008
4114
  sessionId: input.sessionId,
3009
4115
  turnId,
3010
- providers: failedProviders.join(","),
3011
4116
  errorClass,
3012
4117
  retryDelayMs,
3013
4118
  });
3014
4119
  },
3015
4120
  });
3016
- if (gitCredentialRenewalClosed) {
4121
+ if (toolspaceTokenRenewalClosed) {
3017
4122
  await controller.stop();
3018
4123
  return;
3019
4124
  }
3020
- gitCredentialRenewal = controller;
4125
+ toolspaceTokenRenewal = controller;
3021
4126
  };
3022
4127
 
3023
- // P1.2 ownership inversion (gated, default OFF). With the flag off this
3024
- // block is skipped entirely: resolvedSandbox stays null and runStream
3025
- // takes the legacy per-run build-and-discard path — byte-for-byte today.
3026
- // With it on, acquire the group lease (holder = the durable attempt id),
3027
- // resume the one box by id, and inject it NON-OWNED into the run. The box
3028
- // backend is "none" -> never resolve (no box to touch).
3029
- //
3030
- // Established AFTER sandboxEnvironment is computed (not before) so the box's
3031
- // manifest is created with the SAME variableSet the agent declares — the SDK
3032
- // applies the agent's manifest to this provided session and throws on ANY
3033
- // variableSet delta (validateNoEnvironmentDelta). Passing sandboxEnvironment
3034
- // here makes current==target so the delta is empty.
3035
- if (settings.sandboxOwnershipEnabled && turn.sandboxBackend !== "none") {
3036
- sandboxHolderId = sandboxLeaseHolderIdForAttempt(input.attemptId);
3037
- sandboxGroupId = session.sandboxGroupId;
3038
- // STAGE D honest-label guard: a machine-home session carries
3039
- // turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
3040
- // when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
3041
- // + enrollmentId). When it is NOT primary — the agent swapped back to the
3042
- // group box (sandbox_swap 'session'/'default'/groupId clears the pointer) or
3043
- // selfhosted routing is flag-OFF (the pointer is ignored) — the else-branch
3044
- // must resume a REAL cloud group box, not a "selfhosted" one: the registry
3045
- // SelfhostedSandboxClient has no bound agentId and throws. Fall the group-box
3046
- // backend back to the deployment default cloud backend so swap-away / flag-off
3047
- // degrade to a genuine cloud box exactly like today (home=modal did).
3048
- if (machinePrimary) {
3049
- // STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
3050
- // establish or lease a phantom Modal home box (today's path leased + BILLED
3051
- // a cloud box the turn never touched). Build the SelfhostedSession DIRECTLY
3052
- // (no Modal box created) and take the group lease with backend "selfhosted"
3053
- // (refcount/idle bookkeeping; the reaper drains it cold with NO provider
3054
- // stop, and bills ZERO warm-seconds). The session is a harmless in-memory
3055
- // bind (no NATS round-trip), so build it FIRST; if the lease then fences,
3056
- // there is nothing to clean up.
3057
- // Whether the machine's latest Hello advertised the op-stream engine
3058
- // (refreshed on every connect). Read only when the server flag is on —
3059
- // one indexed lookup, and the flag off keeps this path byte-identical.
3060
- const machineOpStream =
3061
- settings.agentOpStreamEnabled === true
3062
- ? (await getEnrollment(db, input.workspaceId, activeSandboxRecord!.enrollmentId!))
3063
- ?.opStream === true
4128
+ const attachRunCredentialRenewal = async (
4129
+ credentialSession: RunCredentialCommandSession,
4130
+ ): Promise<void> => {
4131
+ if (!runCredentialResolver) return;
4132
+ const previous = runCredentialRenewal;
4133
+ runCredentialRenewal = null;
4134
+ await previous?.stop();
4135
+ if (runCredentialRenewalClosed) return;
4136
+ runCredentialSession = credentialSession;
4137
+
4138
+ if (!initialRunCredentialMaterial) {
4139
+ await clearRunCredentials(
4140
+ credentialSession,
4141
+ input.sessionId,
4142
+ toolCancellationFenceRef.current
4143
+ ? toolCancellationFenceRef.current.runSandboxCommand.bind(
4144
+ toolCancellationFenceRef.current,
4145
+ )
4146
+ : undefined,
4147
+ );
4148
+ return;
4149
+ }
4150
+
4151
+ const write = async (
4152
+ material: NormalizedRunCredentialMaterial | null,
4153
+ pruneOtherAttempts = false,
4154
+ ): Promise<void> => {
4155
+ if (!material) {
4156
+ await clearRunCredentialsForAttempt(credentialSession, {
4157
+ sessionId: input.sessionId,
4158
+ attemptId: input.attemptId,
4159
+ executionGeneration,
4160
+ });
4161
+ return;
4162
+ }
4163
+ registerSecretRedactions(material.redactions);
4164
+ await materializeRunCredentials(credentialSession, material, {
4165
+ sessionId: input.sessionId,
4166
+ attemptId: input.attemptId,
4167
+ executionGeneration,
4168
+ ...(pruneOtherAttempts ? { pruneOtherAttempts: true } : {}),
4169
+ ...(!pruneOtherAttempts ? { pruneSupersededGenerations: true } : {}),
4170
+ ...(material.authNeeded.length > 0 &&
4171
+ Object.keys(material.environment).length === 0 &&
4172
+ material.files.length === 0
4173
+ ? { prunePreviousGenerations: true }
4174
+ : {}),
4175
+ ...(toolCancellationFenceRef.current
4176
+ ? {
4177
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4178
+ toolCancellationFenceRef.current,
4179
+ ),
4180
+ }
4181
+ : {}),
4182
+ });
4183
+ for (const payload of runCredentialAuthNeededPayloads(material)) {
4184
+ const key = JSON.stringify(payload);
4185
+ if (publishedRunCredentialNotices.has(key)) continue;
4186
+ publishedRunCredentialNotices.add(key);
4187
+ await publish!([{ type: "credential.auth_needed", payload }], true);
4188
+ }
4189
+ };
4190
+
4191
+ const initialExpiryMs = initialRunCredentialMaterial.expiresAt?.getTime() ?? null;
4192
+ const seed =
4193
+ initialExpiryMs !== null && initialExpiryMs <= Date.now() + RUN_CREDENTIAL_EXPIRY_LEAD_MS
4194
+ ? await runCredentialResolver.resolve({
4195
+ purpose: "provision",
4196
+ forceRefresh: true,
4197
+ })
4198
+ : initialRunCredentialMaterial;
4199
+ await write(seed, true);
4200
+ if (runCredentialRenewalClosed) return;
4201
+ const controller = startRunCredentialRenewalLoop({
4202
+ initialExpiresAt: seed?.expiresAt ?? null,
4203
+ resolve: async () =>
4204
+ await runCredentialResolver.resolve({
4205
+ purpose: "renewal",
4206
+ forceRefresh: true,
4207
+ }),
4208
+ write: async (material) => await write(material),
4209
+ onSuccess: ({ authNeeded }) => {
4210
+ observability.incrementCounter({
4211
+ name: "opengeni_run_credential_renewals_total",
4212
+ help: "Host-managed run credential renewal attempts by outcome.",
4213
+ labels: { outcome: authNeeded ? "auth_needed" : "completed" },
4214
+ });
4215
+ },
4216
+ onFailure: ({ retryDelayMs, errorClass }) => {
4217
+ observability.incrementCounter({
4218
+ name: "opengeni_run_credential_renewals_total",
4219
+ help: "Host-managed run credential renewal attempts by outcome.",
4220
+ labels: { outcome: "error" },
4221
+ });
4222
+ observability.warn("Host run credential renewal failed; retry scheduled", {
4223
+ sessionId: input.sessionId,
4224
+ turnId,
4225
+ errorClass,
4226
+ retryDelayMs,
4227
+ });
4228
+ },
4229
+ });
4230
+ if (runCredentialRenewalClosed) {
4231
+ await controller.stop();
4232
+ return;
4233
+ }
4234
+ runCredentialRenewal = controller;
4235
+ };
4236
+
4237
+ // P1.2 ownership inversion (gated, default OFF). With the flag off this
4238
+ // block is skipped entirely: resolvedSandbox stays null and runStream
4239
+ // takes the legacy per-run build-and-discard path — byte-for-byte today.
4240
+ // With it on, acquire the group lease (holder = the durable attempt id),
4241
+ // resume the one box by id, and inject it NON-OWNED into the run. The box
4242
+ // backend is "none" -> never resolve (no box to touch).
4243
+ //
4244
+ // Established AFTER sandboxEnvironment is computed (not before) so the box's
4245
+ // manifest is created with the SAME variableSet the agent declares — the SDK
4246
+ // applies the agent's manifest to this provided session and throws on ANY
4247
+ // variableSet delta (validateNoEnvironmentDelta). Passing sandboxEnvironment
4248
+ // here makes current==target so the delta is empty.
4249
+ if (settings.sandboxOwnershipEnabled && turn.sandboxBackend !== "none") {
4250
+ sandboxHolderId = sandboxLeaseHolderIdForAttempt(input.attemptId);
4251
+ sandboxGroupId = session.sandboxGroupId;
4252
+ // STAGE D honest-label guard: a machine-home session carries
4253
+ // turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
4254
+ // when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
4255
+ // + enrollmentId). When it is NOT primary — the agent swapped back to the
4256
+ // group box (sandbox_swap 'session'/'default'/groupId clears the pointer) or
4257
+ // selfhosted routing is flag-OFF (the pointer is ignored) — the else-branch
4258
+ // must resume a REAL cloud group box, not a "selfhosted" one: the registry
4259
+ // SelfhostedSandboxClient has no bound agentId and throws. Fall the group-box
4260
+ // backend back to the deployment default cloud backend so swap-away / flag-off
4261
+ // degrade to a genuine cloud box exactly like today (home=modal did).
4262
+ if (machinePrimary) {
4263
+ // STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
4264
+ // establish or lease a phantom Modal home box (today's path leased + BILLED
4265
+ // a cloud box the turn never touched). Build the SelfhostedSession DIRECTLY
4266
+ // (no Modal box created) and take the group lease with backend "selfhosted"
4267
+ // (refcount/idle bookkeeping; the reaper drains it cold with NO provider
4268
+ // stop, and bills ZERO warm-seconds). The session is a harmless in-memory
4269
+ // bind (no NATS round-trip), so build it FIRST; if the lease then fences,
4270
+ // there is nothing to clean up.
4271
+ // Whether the machine's latest Hello advertised the op-stream engine
4272
+ // (refreshed on every connect). Read only when the server flag is on —
4273
+ // one indexed lookup, and the flag off keeps this path byte-identical.
4274
+ const machineOpStream =
4275
+ settings.agentOpStreamEnabled === true
4276
+ ? (await getEnrollment(db, input.workspaceId, activeSandboxRecord!.enrollmentId!))
4277
+ ?.opStream === true
3064
4278
  : false;
3065
4279
  const established = await establishSelfhostedTurnSession(
3066
4280
  {
@@ -3083,16 +4297,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3083
4297
  // (buildSelfhostedBackendSession); EstablishedSandboxSession widens it.
3084
4298
  machinePrimarySession =
3085
4299
  established.session as import("@opengeni/runtime").SelfhostedSession;
3086
- const lease = await acquireSelfhostedLeaseForTurn(
3087
- { db, settings },
3088
- {
3089
- accountId: input.accountId,
3090
- workspaceId: input.workspaceId,
3091
- sandboxGroupId: session.sandboxGroupId,
3092
- sessionId: input.sessionId,
3093
- },
3094
- "turn",
3095
- sandboxHolderId,
4300
+ const lease = await waitForTurnOperation(
4301
+ acquireSelfhostedLeaseForTurn(
4302
+ { db, settings },
4303
+ {
4304
+ accountId: input.accountId,
4305
+ workspaceId: input.workspaceId,
4306
+ sandboxGroupId: session.sandboxGroupId,
4307
+ sessionId: input.sessionId,
4308
+ },
4309
+ "turn",
4310
+ sandboxHolderId,
4311
+ ),
4312
+ cancellationSignal,
4313
+ async (lateLease) => await lateLease.release(),
3096
4314
  );
3097
4315
  setupBoxSession = established.session;
3098
4316
  resolvedSandbox = {
@@ -3137,45 +4355,49 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3137
4355
  // proxy's first default-pointer op. A chat-only turn never calls it, so
3138
4356
  // no lease row, no provider box, no warm-meter interval.
3139
4357
  } else {
3140
- resolvedSandbox = await resumeBoxForTurn(
3141
- {
3142
- db,
3143
- settings,
3144
- sandboxMetrics: runtimeMetricsHooksForObservability(observability),
3145
- onSandboxLost: publishSandboxLost,
3146
- },
3147
- {
3148
- accountId: input.accountId,
3149
- workspaceId: input.workspaceId,
3150
- sandboxGroupId: session.sandboxGroupId,
3151
- sessionId: input.sessionId,
3152
- // groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
3153
- // is not machine-primary resumes a real cloud group box (the
3154
- // deployment default), never a "selfhosted" box (which would throw
3155
- // for lack of a bound agentId).
3156
- backend: groupBoxBackend,
3157
- os: session.sandboxOs,
3158
- environment: sandboxEnvironment,
3159
- // IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
3160
- // this run resolves. The lease stamps it + conflicts on a live shared box
3161
- // running a DIFFERENT image (solo → recreate on the new image; N-holders →
3162
- // SandboxImageConflictError surfaced as an actionable turn error). Prefer the
3163
- // explicit Modal image ref, else the docker image. The selfhosted branch
3164
- // (establishSelfhostedTurnSession/acquireSelfhostedLeaseForTurn) NEVER passes
3165
- // an image — B3 lives only on this Modal else-branch.
3166
- ...((runSettings.modalImageRef ?? runSettings.dockerImage)
3167
- ? {
3168
- image: runSettings.modalImageRef ?? runSettings.dockerImage,
3169
- }
3170
- : {}),
3171
- // RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
3172
- // conflicts on a live shared box set up under a different rig (solo
3173
- // recreate / N-holders SandboxRigConflictError). Omitted for a rig-less
3174
- // turn -> never stamped or enforced (shares exactly as today).
3175
- ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
3176
- },
3177
- "turn",
3178
- sandboxHolderId,
4358
+ resolvedSandbox = await waitForTurnOperation(
4359
+ resumeBoxForTurn(
4360
+ {
4361
+ db,
4362
+ settings,
4363
+ sandboxMetrics: runtimeMetricsHooksForObservability(observability),
4364
+ onSandboxLost: publishSandboxLost,
4365
+ },
4366
+ {
4367
+ accountId: input.accountId,
4368
+ workspaceId: input.workspaceId,
4369
+ sandboxGroupId: session.sandboxGroupId,
4370
+ sessionId: input.sessionId,
4371
+ // groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
4372
+ // is not machine-primary resumes a real cloud group box (the
4373
+ // deployment default), never a "selfhosted" box (which would throw
4374
+ // for lack of a bound agentId).
4375
+ backend: groupBoxBackend,
4376
+ os: session.sandboxOs,
4377
+ environment: sandboxEnvironment,
4378
+ // IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
4379
+ // this run resolves. The lease stamps it + conflicts on a live shared box
4380
+ // running a DIFFERENT image (solo → recreate on the new image; N-holders →
4381
+ // SandboxImageConflictError surfaced as an actionable turn error). Prefer the
4382
+ // explicit Modal image ref, else the docker image. The selfhosted branch
4383
+ // (establishSelfhostedTurnSession/acquireSelfhostedLeaseForTurn) NEVER passes
4384
+ // an image — B3 lives only on this Modal else-branch.
4385
+ ...((runSettings.modalImageRef ?? runSettings.dockerImage)
4386
+ ? {
4387
+ image: runSettings.modalImageRef ?? runSettings.dockerImage,
4388
+ }
4389
+ : {}),
4390
+ // RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
4391
+ // conflicts on a live shared box set up under a different rig (solo
4392
+ // recreate / N-holders SandboxRigConflictError). Omitted for a rig-less
4393
+ // turn -> never stamped or enforced (shares exactly as today).
4394
+ ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
4395
+ },
4396
+ "turn",
4397
+ sandboxHolderId,
4398
+ ),
4399
+ cancellationSignal,
4400
+ async (lateSandbox) => await lateSandbox.release(),
3179
4401
  );
3180
4402
  setupBoxSession = resolvedSandbox.established.session;
3181
4403
  // Durable box-lifecycle events (sandbox-file-persistence observability):
@@ -3216,42 +4438,65 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3216
4438
  }
3217
4439
  }
3218
4440
 
3219
- const fileResourceDownloads = await sandboxFileDownloadsForRun(
3220
- runSettings,
3221
- db,
3222
- objectStorage,
3223
- input.workspaceId,
3224
- turnResources,
4441
+ const fileResourceDownloads = await waitForTurnOperation(
4442
+ sandboxFileDownloadsForRun(
4443
+ runSettings,
4444
+ db,
4445
+ objectStorage,
4446
+ input.workspaceId,
4447
+ turnResources,
4448
+ ),
4449
+ cancellationSignal,
4450
+ undefined,
3225
4451
  );
3226
4452
  throwIfWorkerShuttingDown();
4453
+ throwIfTurnCancelled();
4454
+ const mcpCredentialRootSessionId =
4455
+ connectionCredentials?.mcpCredentials && hostCredentialRootSessionId
4456
+ ? hostCredentialRootSessionId
4457
+ : input.sessionId;
3227
4458
  // Wrap MCP prep in the codex ALS so the codex_apps connect handshake
3228
4459
  // (initialize + tools/list) can resolve the per-workspace bearer from
3229
4460
  // codexRequestStorage (runtime/codexAppsMcpRequestInit). withCodex is the
3230
4461
  // identity on every non-codex turn, so this is a no-op for existing paths.
3231
- const resolveCredential = buildConnectionTokenResolver(db, runSettings);
3232
- preparedTools = await withCodex(() =>
3233
- runtime.prepareTools(runSettings, turnTools, {
3234
- accountId: input.accountId,
3235
- workspaceId: input.workspaceId,
3236
- sessionId: input.sessionId,
3237
- // Sign the calling turn into the first-party token so tools classify
3238
- // the caller by its own identity (sacred-pause guard), not the racy
3239
- // live active pointer.
3240
- ...(turnId ? { turnId } : {}),
3241
- attemptId: input.attemptId,
3242
- executionGeneration,
3243
- subjectId: "worker:first-party-mcp",
3244
- subjectLabel: "OpenGeni worker",
3245
- resolveCredential,
3246
- onAuthNeeded: async (payload) => {
3247
- await publish!([{ type: "tool.auth_needed", payload }], true);
3248
- },
3249
- // Manager-style sessions carry a creation-validated permission set
3250
- // for their first-party MCP token; null keeps the fixed default.
3251
- ...(session.firstPartyMcpPermissions?.length
3252
- ? { firstPartyPermissions: session.firstPartyMcpPermissions }
3253
- : {}),
3254
- }),
4462
+ const resolveCredential = connectionTokenResolverForTurn({
4463
+ db,
4464
+ settings: runSettings,
4465
+ connectionCredentials: connectionCredentials ?? null,
4466
+ accountId: input.accountId,
4467
+ workspaceId: input.workspaceId,
4468
+ sessionId: input.sessionId,
4469
+ rootSessionId: mcpCredentialRootSessionId,
4470
+ attemptId: input.attemptId,
4471
+ turn,
4472
+ });
4473
+ preparedTools = await waitForTurnOperation(
4474
+ withCodex(() =>
4475
+ runtime.prepareTools(runSettings, turnTools, {
4476
+ accountId: input.accountId,
4477
+ workspaceId: input.workspaceId,
4478
+ sessionId: input.sessionId,
4479
+ // Sign the calling turn into the first-party token so tools classify
4480
+ // the caller by its own identity (sacred-pause guard), not the racy
4481
+ // live active pointer.
4482
+ ...(turnId ? { turnId } : {}),
4483
+ attemptId: input.attemptId,
4484
+ executionGeneration,
4485
+ subjectId: "worker:first-party-mcp",
4486
+ subjectLabel: "OpenGeni worker",
4487
+ resolveCredential,
4488
+ onAuthNeeded: async (payload) => {
4489
+ await publish!([{ type: "tool.auth_needed", payload }], true);
4490
+ },
4491
+ // Manager-style sessions carry a creation-validated permission set
4492
+ // for their first-party MCP token; null keeps the fixed default.
4493
+ ...(session.firstPartyMcpPermissions?.length
4494
+ ? { firstPartyPermissions: session.firstPartyMcpPermissions }
4495
+ : {}),
4496
+ }),
4497
+ ),
4498
+ cancellationSignal,
4499
+ async (latePreparedTools) => await latePreparedTools.close().catch(() => undefined),
3255
4500
  );
3256
4501
  // Genesis turn = the first user turn (no assistant history reconciled
3257
4502
  // yet). Durable Postgres state (countSessionHistoryItems includes
@@ -3292,11 +4537,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3292
4537
  : {};
3293
4538
  const agent = runtime.buildAgent(modelRunSettings, turnResources, {
3294
4539
  reasoningEffort: turn.reasoningEffort,
4540
+ ...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
3295
4541
  genesisTitleHint: isGenesisTurn,
3296
4542
  persistentSessionSettings: {
3297
4543
  titleIsSet: Boolean(session.title?.trim()),
3298
4544
  },
3299
4545
  sandboxEnvironment,
4546
+ ...(cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {}),
4547
+ onToolCancellationFence: (fence) => {
4548
+ toolCancellationFenceRef.current = fence;
4549
+ },
3300
4550
  // TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
3301
4551
  // seed. ONLY when the effective backend is NOT selfhosted (the connected
3302
4552
  // machine uses its own git creds — mirrors the skipGitHubToken gate above)
@@ -3306,6 +4556,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3306
4556
  ...(activeSandboxBackend !== "selfhosted" && sandboxGitTokens
3307
4557
  ? { gitTokenSeeds: sandboxGitTokens }
3308
4558
  : {}),
4559
+ ...(activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings
4560
+ ? { gitCredentialBindings: sandboxGitCredentialBindings }
4561
+ : {}),
3309
4562
  ...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
3310
4563
  ? { gitTokenSeed: sandboxGitToken }
3311
4564
  : {}),
@@ -3313,10 +4566,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3313
4566
  // token skip does NOT transfer: that token is inert on a connected
3314
4567
  // machine (it uses its own git creds), but the toolspace token is the
3315
4568
  // machine's ONLY path to programmatic tool calling and grants no more
3316
- // than toolspace:call for its own session (own-session-bound, turn TTL,
3317
- // budgeted, approval-tools excluded). The runtime seeds it to the box's
3318
- // token file over the same exec channel, off-manifest, on every backend.
3319
- ...(sandboxToolspaceToken ? { toolspaceTokenSeed: sandboxToolspaceToken } : {}),
4569
+ // than toolspace:call for its own session (own-session-bound,
4570
+ // short-lived and renewable, per-active-turn budgeted, approval-tools
4571
+ // excluded). The runtime seeds it to the box's token file over the same
4572
+ // exec channel, off-manifest, on every backend.
4573
+ ...(sandboxToolspaceToken
4574
+ ? {
4575
+ toolspaceTokenSeed: sandboxToolspaceToken,
4576
+ toolspaceTokenSessionId: input.sessionId,
4577
+ }
4578
+ : {}),
3320
4579
  ...(activeSandboxBackend ? { activeSandboxBackend } : {}),
3321
4580
  fileResourceDownloads,
3322
4581
  mcpServers: preparedTools.mcpServers,
@@ -3384,12 +4643,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3384
4643
  await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
3385
4644
  },
3386
4645
  ...(packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {}),
4646
+ ...(skillLibraryRuntime.skillLibrarySkills.length > 0
4647
+ ? {
4648
+ skillLibrarySkills: skillLibraryRuntime.skillLibrarySkills,
4649
+ skillLibrarySelections: skillLibraryRuntime.skillLibrarySelections,
4650
+ }
4651
+ : {}),
3387
4652
  ...(workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {}),
3388
4653
  ...(workspaceMemory ? { workspaceMemory } : {}),
3389
4654
  // Per-session persona tier (session > workspace > deployment default).
3390
4655
  // Composed system-level AFTER the workspace persona so it refines it for
3391
4656
  // this one session; absent ⇒ byte-identical to today's composition.
3392
4657
  ...(session.instructions ? { sessionInstructions: session.instructions } : {}),
4658
+ // Exact host context captured when this turn was accepted. It is
4659
+ // system-level and disappears with the turn rather than entering chat.
4660
+ ...(turn.turnInstructions ? { turnInstructions: turn.turnInstructions } : {}),
3393
4661
  ...workspaceEnvironmentOption,
3394
4662
  // RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
3395
4663
  // the frozen version carries a non-empty script), and the rig credential
@@ -3414,6 +4682,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3414
4682
  }
3415
4683
  : {}),
3416
4684
  });
4685
+ if (modelRunSettings.sandboxBackend !== "none" && toolCancellationFenceRef.current === null) {
4686
+ throw new Error(
4687
+ "Sandbox agent construction did not install the mandatory turn tool cancellation fence",
4688
+ );
4689
+ }
3417
4690
  if (establishPolicy === "on-demand" && sandboxHolderId && sandboxGroupId) {
3418
4691
  const lazyHolderId = sandboxHolderId;
3419
4692
  const lazyGroupId = sandboxGroupId;
@@ -3424,18 +4697,23 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3424
4697
  const lazyClient = {
3425
4698
  backendId: sdkBackendIdForSandboxBackend(groupBoxBackend),
3426
4699
  } as EstablishedSandboxSession["client"];
3427
- let lazyEstablishmentOrigin: EstablishedSandboxSession["origin"] | null = null;
3428
4700
  turnSandboxProvisioner = createTurnSandboxProvisioner<ResumedTurnSandbox>(
3429
4701
  async () => {
3430
4702
  throwIfWorkerShuttingDown();
4703
+ throwIfTurnCancelled();
3431
4704
  const lazyGitCredentials =
3432
4705
  activeSandboxBackend === "selfhosted"
3433
4706
  ? undefined
3434
4707
  : await mintRunGitCredentials(runSettings, turnResources, {
3435
4708
  scope: connectionScope,
4709
+ ...(gitCredentialAuthority ? { authority: gitCredentialAuthority } : {}),
3436
4710
  gitCredentials: connectionCredentials?.gitCredentials,
4711
+ authorizeGitHubTokenMint,
3437
4712
  });
3438
4713
  const lazyGitTokens = lazyGitCredentials?.gitTokens;
4714
+ const lazyToolspaceToken = sandboxToolspaceToken
4715
+ ? await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority)
4716
+ : undefined;
3439
4717
  const provisioned = await resumeBoxForTurn(
3440
4718
  {
3441
4719
  db,
@@ -3460,13 +4738,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3460
4738
  "turn",
3461
4739
  lazyHolderId,
3462
4740
  );
3463
- lazyEstablishmentOrigin = provisioned.established.origin ?? null;
3464
- setupBoxSession = provisioned.established.session;
3465
4741
  await publishSandboxLifecycleEvents(provisioned);
4742
+ await attachRunCredentialRenewal(
4743
+ provisioned.established.session as RunCredentialCommandSession,
4744
+ );
4745
+ const provisionedSetupSession = initialRunCredentialMaterial
4746
+ ? withRunCredentialsSession(
4747
+ provisioned.established.session as object,
4748
+ input.sessionId,
4749
+ )
4750
+ : provisioned.established.session;
3466
4751
  await runOwnedSandboxSetup(
3467
4752
  agent,
3468
4753
  provisioned.established.session as never,
3469
- provisioned.established.session as never,
4754
+ provisionedSetupSession as never,
3470
4755
  {
3471
4756
  settings: runSettings,
3472
4757
  environment: sandboxEnvironment,
@@ -3474,8 +4759,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3474
4759
  await publish?.([{ type: event.type, payload: event.payload }], true);
3475
4760
  },
3476
4761
  ...(lazyGitTokens ? { gitTokenSeedsOverride: lazyGitTokens } : {}),
4762
+ ...(lazyGitCredentials?.bindings
4763
+ ? { gitCredentialBindingsOverride: lazyGitCredentials.bindings }
4764
+ : {}),
4765
+ ...(lazyToolspaceToken
4766
+ ? { toolspaceTokenSeedOverride: lazyToolspaceToken.token }
4767
+ : {}),
4768
+ ...(toolCancellationFenceRef.current
4769
+ ? {
4770
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4771
+ toolCancellationFenceRef.current,
4772
+ ),
4773
+ }
4774
+ : {}),
3477
4775
  },
3478
4776
  );
4777
+ await attachToolspaceTokenRenewal(
4778
+ provisioned.established.session as ToolspaceTokenWriterSession,
4779
+ lazyToolspaceToken?.expiresAt,
4780
+ );
3479
4781
  await attachGitCredentialRenewal(
3480
4782
  provisioned.established.session as GitCredentialTokenWriterSession,
3481
4783
  lazyGitCredentials,
@@ -3488,11 +4790,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3488
4790
  // already holds the proxy directly (injected as lazyOwnedSandbox.session), so it
3489
4791
  // gets per-op routing; the worker-side handle (resolvedSandbox: release,
3490
4792
  // heartbeat, computer-use recording) wants the real box, unproxied.
3491
- resolvedSandbox = provisioned;
3492
- startLeaseHeartbeat(provisioned, activeSandboxBackend ?? groupBoxBackend);
3493
4793
  return provisioned;
3494
4794
  },
3495
4795
  {
4796
+ ...(activityContext ? { signal: activityContext.cancellationSignal } : {}),
3496
4797
  onStarted: async () => {
3497
4798
  await publish?.(
3498
4799
  [
@@ -3504,19 +4805,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3504
4805
  true,
3505
4806
  );
3506
4807
  },
3507
- onCompleted: async () => {
4808
+ onCompleted: async (provisioned) => {
3508
4809
  await publish?.(
3509
4810
  [
3510
4811
  {
3511
4812
  type: "sandbox.operation.completed",
3512
4813
  payload: {
3513
4814
  name: "sandbox.provision",
3514
- ...(lazyEstablishmentOrigin ? { origin: lazyEstablishmentOrigin } : {}),
4815
+ ...(provisioned.established.origin
4816
+ ? { origin: provisioned.established.origin }
4817
+ : {}),
3515
4818
  },
3516
4819
  },
3517
4820
  ],
3518
4821
  true,
3519
4822
  );
4823
+ throwIfTurnOperationCancelled(activityContext?.cancellationSignal);
4824
+ startLeaseHeartbeat(provisioned, activeSandboxBackend ?? groupBoxBackend);
4825
+ setupBoxSession = provisioned.established.session;
4826
+ resolvedSandbox = provisioned;
3520
4827
  },
3521
4828
  onFailed: async (error) => {
3522
4829
  await publish?.(
@@ -3532,6 +4839,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3532
4839
  true,
3533
4840
  );
3534
4841
  },
4842
+ disposeResult: async (provisioned) => {
4843
+ await provisioned.release().catch(() => undefined);
4844
+ },
3535
4845
  },
3536
4846
  );
3537
4847
  lazyOwnedSandbox = wrapLazyTurnBoxWithRouting(
@@ -3566,36 +4876,41 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3566
4876
  // RunState verbatim and recovering attempts already compacted, if needed,
3567
4877
  // before the first attempt's model boundary.
3568
4878
  if (triggerType === "user.message" || triggerType === "system.update.delivered") {
4879
+ let forced = false;
3569
4880
  try {
3570
4881
  // Operator /compact (the slash command) sets a durable request flag;
3571
4882
  // observe it without consuming it so a failed/stale attempt cannot
3572
4883
  // lose the request. The replacement transaction clears it on success.
3573
- const forced = await isSessionCompactionRequested(db, input.workspaceId, input.sessionId);
3574
- const outcome = await maybeCompactContext(
3575
- db,
3576
- modelRunSettings,
3577
- {
3578
- accountId: input.accountId,
3579
- workspaceId: input.workspaceId,
3580
- sessionId: input.sessionId,
3581
- turnId: turnId!,
3582
- executionGeneration,
3583
- attemptId: input.attemptId,
3584
- },
3585
- session.lastInputTokens,
3586
- // Provider-aware summarizer: when the turn's model resolved to a
3587
- // registry provider, summarize on THAT provider's client + wire API
3588
- // (a chat provider can't summarize through OpenAI/Azure). Null
3589
- // resolution uses the built-in Responses summarizer with the same
3590
- // session prompt-cache key as the main model calls.
3591
- compactSummarizer,
3592
- forced
3593
- ? {
3594
- force: true,
3595
- clearRequestedCompaction: true,
3596
- trigger: "operator",
3597
- }
3598
- : { trigger: "auto" },
4884
+ forced = await isSessionCompactionRequested(db, input.workspaceId, input.sessionId);
4885
+ const outcome = await waitForTurnOperation(
4886
+ maybeCompactContext(
4887
+ db,
4888
+ modelRunSettings,
4889
+ {
4890
+ accountId: input.accountId,
4891
+ workspaceId: input.workspaceId,
4892
+ sessionId: input.sessionId,
4893
+ turnId: turnId!,
4894
+ executionGeneration,
4895
+ attemptId: input.attemptId,
4896
+ },
4897
+ session.lastInputTokens,
4898
+ // Provider-aware summarizer: when the turn's model resolved to a
4899
+ // registry provider, summarize on THAT provider's client + wire API
4900
+ // (a chat provider can't summarize through OpenAI/Azure). Null
4901
+ // resolution uses the built-in Responses summarizer with the same
4902
+ // session prompt-cache key as the main model calls.
4903
+ compactSummarizer,
4904
+ forced
4905
+ ? {
4906
+ force: true,
4907
+ clearRequestedCompaction: true,
4908
+ trigger: "operator",
4909
+ }
4910
+ : { trigger: "auto" },
4911
+ ),
4912
+ cancellationSignal,
4913
+ undefined,
3599
4914
  );
3600
4915
  if (outcome.compacted) {
3601
4916
  const compactionTrigger = forced ? "operator" : undefined;
@@ -3608,12 +4923,41 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3608
4923
  );
3609
4924
  }
3610
4925
  } catch (compactError) {
4926
+ if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
4927
+ if (!isCompactionSummaryFailure(compactError)) throw compactError;
4928
+ const errorMessage = compactionFailureReasonFromError(compactError);
3611
4929
  observability.error("context compaction failed", {
3612
4930
  sessionId: input.sessionId,
3613
4931
  turnId,
3614
- error: compactError instanceof Error ? compactError.message : String(compactError),
4932
+ error: errorMessage,
3615
4933
  });
3616
- throw compactError;
4934
+ if (
4935
+ !(await settle!({
4936
+ events: [
4937
+ {
4938
+ type: "turn.failed",
4939
+ payload: {
4940
+ error: errorMessage,
4941
+ code: "context_compaction_failed",
4942
+ retryable: false,
4943
+ recovery: "user_message",
4944
+ compacted: false,
4945
+ },
4946
+ },
4947
+ { type: "session.status.changed", payload: { status: "idle" } },
4948
+ ],
4949
+ turnStatus: "failed",
4950
+ sessionStatus: "idle",
4951
+ activeTurnId: null,
4952
+ ...(forced ? { consumeRequestedCompactionFailure: true } : {}),
4953
+ }))
4954
+ ) {
4955
+ return claimedResult({ status: "cancelled" });
4956
+ }
4957
+ turnMetricOutcome = "failed";
4958
+ activityStatus = "idle";
4959
+ activityError = compactError;
4960
+ return claimedResult({ status: "idle" });
3617
4961
  }
3618
4962
  }
3619
4963
  let fileMaterializationFailures: SandboxFileDownloadFailure[] = [];
@@ -3646,6 +4990,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3646
4990
  await publish!([{ type: event.type, payload: event.payload }], true);
3647
4991
  },
3648
4992
  ...(runAs ? { runAs } : {}),
4993
+ ...(toolCancellationFenceRef.current
4994
+ ? {
4995
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4996
+ toolCancellationFenceRef.current,
4997
+ ),
4998
+ }
4999
+ : {}),
3649
5000
  },
3650
5001
  );
3651
5002
  fileMaterializationFailures = materialized.failures;
@@ -3693,6 +5044,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3693
5044
  turnId: activeTurnId,
3694
5045
  recovering: turn.executionGeneration > 1,
3695
5046
  ...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
5047
+ ...(runCredentialsNote ? { runCredentialsNote } : {}),
3696
5048
  },
3697
5049
  );
3698
5050
  runInput = prepared.input;
@@ -3740,29 +5092,33 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3740
5092
  triggerLabel: "overflow" | "proactive" | "operator",
3741
5093
  recoverySignalTokens: number | null,
3742
5094
  ) => {
3743
- const outcome = await maybeCompactContext(
3744
- db,
3745
- modelRunSettings,
3746
- {
3747
- accountId: input.accountId,
3748
- workspaceId: input.workspaceId,
3749
- sessionId: input.sessionId,
3750
- turnId: activeTurnId,
3751
- executionGeneration,
3752
- attemptId: input.attemptId,
3753
- },
3754
- // Never reuse the persisted prior-turn signal for recovery. Proactive
3755
- // guards provide their exact current signal; provider overflows do not,
3756
- // so null derives decision metadata from active history. Forced
3757
- // recovery proves progress separately by comparing the replacement
3758
- // with the current active-history estimate in maybeCompactContext.
3759
- recoverySignalTokens,
3760
- compactSummarizer,
3761
- {
3762
- force: true,
3763
- ...(triggerLabel === "operator" ? { clearRequestedCompaction: true } : {}),
3764
- trigger: triggerLabel,
3765
- },
5095
+ const outcome = await waitForTurnOperation(
5096
+ maybeCompactContext(
5097
+ db,
5098
+ modelRunSettings,
5099
+ {
5100
+ accountId: input.accountId,
5101
+ workspaceId: input.workspaceId,
5102
+ sessionId: input.sessionId,
5103
+ turnId: activeTurnId,
5104
+ executionGeneration,
5105
+ attemptId: input.attemptId,
5106
+ },
5107
+ // Never reuse the persisted prior-turn signal for recovery. Proactive
5108
+ // guards provide their exact current signal; provider overflows do not,
5109
+ // so null derives decision metadata from active history. Forced
5110
+ // recovery proves progress separately by comparing the replacement
5111
+ // with the current active-history estimate in maybeCompactContext.
5112
+ recoverySignalTokens,
5113
+ compactSummarizer,
5114
+ {
5115
+ force: true,
5116
+ ...(triggerLabel === "operator" ? { clearRequestedCompaction: true } : {}),
5117
+ trigger: triggerLabel,
5118
+ },
5119
+ ),
5120
+ cancellationSignal,
5121
+ undefined,
3766
5122
  );
3767
5123
  if (outcome.events.length > 0) {
3768
5124
  if (outcome.compacted) {
@@ -3778,17 +5134,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3778
5134
  return outcome;
3779
5135
  };
3780
5136
 
5137
+ // Keep response usage identity across every stream attempt in this
5138
+ // activity. Context compaction retries the same logical dispatch by
5139
+ // calling runStreamAttempt again; resetting this state there would reuse
5140
+ // the first no-response-ID fallback key and suppress a real model call.
5141
+ const modelResponseUsageState = createModelResponseUsageEventState(
5142
+ claimedModelUsageSourceKeys,
5143
+ );
3781
5144
  const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
3782
5145
  if (!runInput) {
3783
5146
  throw new Error("Run input was not prepared");
3784
5147
  }
3785
5148
  stream = undefined;
3786
5149
  batcher = null;
3787
- let responseUsageCount = 0;
5150
+ // The SDK emits every processed call item for one model response before
5151
+ // it emits any result for that response. Keep that response-local batch
5152
+ // in memory so an orphan from an older response cannot pin later stable
5153
+ // calls. Durable recovery remains call/result based and needs no batch
5154
+ // schema or compatibility state.
5155
+ let currentToolBatchCallIds = new Set<string>();
5156
+ let currentToolBatchCompletedCallIds = new Set<string>();
3788
5157
  // Actual input tokens of the most recent model response this turn; the
3789
5158
  // pre-read trigger for the NEXT turn. Persisted at every turn-end path.
3790
- let lastInputTokensObserved: number | null = null;
3791
5159
  throwIfWorkerShuttingDown();
5160
+ throwIfTurnCancelled();
3792
5161
  const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
3793
5162
  const runStreamOnce = (): ReturnType<OpenGeniRuntime["runStream"]> =>
3794
5163
  runtime.runStream(agent, runInput!, modelRunSettings, {
@@ -3831,9 +5200,42 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3831
5200
  },
3832
5201
  }
3833
5202
  : {}),
3834
- contextCompactionSignalTokens: () => lastInputTokensObserved,
5203
+ ...(sandboxToolspaceToken && sandboxToolspaceTokenExpiresAt && !lazyOwnedSandbox
5204
+ ? {
5205
+ onToolspaceTokenSessionReady: async (
5206
+ tokenSession: ToolspaceTokenWriterSession,
5207
+ ) => {
5208
+ const renewalSession =
5209
+ activeSandboxBackend === "selfhosted"
5210
+ ? tokenSession
5211
+ : ((setupBoxSession as ToolspaceTokenWriterSession | null) ?? tokenSession);
5212
+ await attachToolspaceTokenRenewal(renewalSession);
5213
+ },
5214
+ }
5215
+ : {}),
5216
+ ...(runCredentialResolver
5217
+ ? {
5218
+ runCredentialSessionId: input.sessionId,
5219
+ ...(!lazyOwnedSandbox
5220
+ ? {
5221
+ onRunCredentialSessionReady: async (
5222
+ credentialSession: RunCredentialCommandSession,
5223
+ ) => {
5224
+ const pinnedCredentialSession = setupBoxSession
5225
+ ? (setupBoxSession as RunCredentialCommandSession)
5226
+ : credentialSession;
5227
+ await attachRunCredentialRenewal(pinnedCredentialSession);
5228
+ },
5229
+ }
5230
+ : {}),
5231
+ }
5232
+ : {}),
5233
+ contextCompactionSignal: () => modelResponseUsageContextSignal(modelResponseUsageState),
3835
5234
  contextCompactionRequested: () =>
3836
5235
  isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
5236
+ ...(toolCancellationFenceRef.current
5237
+ ? { turnToolCancellationFence: toolCancellationFenceRef.current }
5238
+ : {}),
3837
5239
  });
3838
5240
  if (codexLeaseLost) {
3839
5241
  throw new Error("Codex credential lease expired before the model run");
@@ -3865,72 +5267,37 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3865
5267
  streamDone = true;
3866
5268
  break;
3867
5269
  }
3868
- const responseUsage = modelResponseUsageFromSdkEvent(next.value);
3869
- if (responseUsage) {
3870
- await recordCompletedModelCallBeforeOwnershipFences({
3871
- renewLease: () => renewCodexLease("model_usage"),
3872
- leaseLost: () => codexLeaseLost,
3873
- leaseLostMessage: "Codex credential lease expired during the active turn",
3874
- recordUsage: async () => {
3875
- responseUsageCount += 1;
3876
- const responseSourceKey = modelUsageSourceKey({
3877
- responseId: responseUsage.responseId,
3878
- dispatchId: modelUsageDispatchId,
3879
- positionalKey: `response-${responseUsageCount}`,
3880
- });
3881
- // Within a turn the serving credential is fixed, so a switch can only
3882
- // surface on the turn's FIRST model call (vs the session's prior).
3883
- const responseAccountCtx = modelCallAccountContext({
3884
- servingCredentialId: effectiveCodexCredentialId,
3885
- priorSessionCredentialId: priorSessionCodexCredentialId,
3886
- isFirstCallOfTurn: responseUsageCount === 1,
3887
- });
3888
- await recordModelUsageAndDebitCredits(settings, db, {
3889
- accountId: input.accountId,
3890
- workspaceId: input.workspaceId,
3891
- sessionId: input.sessionId,
3892
- turnId: activeTurnId,
3893
- model: turn.model,
3894
- isCodexTurn,
3895
- usage: responseUsage.usage,
3896
- sourceKey: responseSourceKey,
3897
- observability,
3898
- });
3899
- await emitModelCallUsage({
3900
- observability,
3901
- publish,
3902
- accountId: input.accountId,
3903
- workspaceId: input.workspaceId,
3904
- sessionId: input.sessionId,
3905
- turnId: activeTurnId,
3906
- provider: resolvedModel?.provider.id ?? settings.openaiProvider,
3907
- providerApi: resolvedModel?.provider.api ?? "responses",
3908
- model: turn.model,
3909
- sourceKey: responseSourceKey,
3910
- usage: responseUsage,
3911
- servingAccountHash: responseAccountCtx.servingAccountHash,
3912
- accountChangedFromPrevCall: responseAccountCtx.accountChangedFromPrevCall,
3913
- emittedSourceKeys: emittedModelUsageSourceKeys,
3914
- });
3915
- const observed = responseUsage.usage?.inputTokens;
3916
- if (typeof observed === "number" && observed > 0) {
3917
- recordModelInputTokens(observability, streamProvider, observed);
3918
- lastInputTokensObserved = observed;
3919
- }
3920
- // Prompt-cache efficiency for this response — same usage frame as the
3921
- // input-token accounting above, so the two are always consistent.
3922
- recordModelCacheTokens(observability, streamProvider, {
3923
- cachedTokens: modelCallUsageTelemetry(responseUsage.usage).cachedTokens,
3924
- promptTokens: responseUsage.usage?.inputTokens,
3925
- });
3926
- },
3927
- recordAttemptSignals: async () => {
3928
- const observed = responseUsage.usage?.inputTokens;
3929
- if (typeof observed === "number" && observed > 0) {
3930
- await setLastInputTokensFenced(observed);
3931
- }
3932
- },
3933
- });
5270
+ let stableToolCallIdsToClear: string[] | null = null;
5271
+ let completedCurrentToolBatch = false;
5272
+ const responseUsageResult = await processModelResponseUsageEvent({
5273
+ event: next.value,
5274
+ state: modelResponseUsageState,
5275
+ dispatchId: modelUsageDispatchId,
5276
+ settings,
5277
+ db,
5278
+ observability,
5279
+ publish,
5280
+ accountId: input.accountId,
5281
+ workspaceId: input.workspaceId,
5282
+ sessionId: input.sessionId,
5283
+ turnId: activeTurnId,
5284
+ turnAttemptId: input.attemptId,
5285
+ provider: resolvedModel?.provider.id ?? settings.openaiProvider,
5286
+ providerApi: resolvedModel?.provider.api ?? "responses",
5287
+ model: turn.model,
5288
+ metricProvider: streamProvider,
5289
+ externallyBilled: isExternallyBilledTurn,
5290
+ servingCredentialId: effectiveCodexCredentialId,
5291
+ priorSessionCredentialId: priorSessionCodexCredentialId,
5292
+ emittedSourceKeys: emittedModelUsageSourceKeys,
5293
+ renewLease: () => renewCodexLease("model_usage"),
5294
+ leaseLost: () => codexLeaseLost,
5295
+ leaseLostMessage: "Codex credential lease expired during the active turn",
5296
+ setLastInputTokens: setLastInputTokensFenced,
5297
+ });
5298
+ if (responseUsageResult.status === "processed") {
5299
+ currentToolBatchCallIds = new Set<string>();
5300
+ currentToolBatchCompletedCallIds = new Set<string>();
3934
5301
  await reconcileConversationTruth();
3935
5302
  try {
3936
5303
  await ensureRunAllowed(
@@ -3938,7 +5305,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3938
5305
  db,
3939
5306
  input.accountId,
3940
5307
  input.workspaceId,
3941
- isCodexTurn,
5308
+ isExternallyBilledTurn,
3942
5309
  entitlements,
3943
5310
  );
3944
5311
  } catch (limitError) {
@@ -3973,6 +5340,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3973
5340
  "turn attempt ended while recording an in-flight tool call",
3974
5341
  );
3975
5342
  }
5343
+ currentToolBatchCallIds.add(pendingToolCall.callId);
3976
5344
  }
3977
5345
  const completedToolCall = completedToolCallFromSdkEvent(next.value);
3978
5346
  if (completedToolCall) {
@@ -3996,25 +5364,26 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3996
5364
  "turn attempt ended while recording a tool-call result",
3997
5365
  );
3998
5366
  }
3999
- if (recorded.allResultsRecorded) {
4000
- // Persist the SDK's now-stable complete call/result batch before
4001
- // clearing its receipts. A crash between these transactions is
4002
- // harmless: settlement finds both raw results and merely
4003
- // consumes them without fabricating an interruption.
5367
+ const belongsToCurrentBatch = currentToolBatchCallIds.has(completedToolCall.callId);
5368
+ if (belongsToCurrentBatch) {
5369
+ currentToolBatchCompletedCallIds.add(completedToolCall.callId);
5370
+ }
5371
+ const currentBatchIsStable =
5372
+ belongsToCurrentBatch &&
5373
+ currentToolBatchCallIds.size > 0 &&
5374
+ currentToolBatchCompletedCallIds.size === currentToolBatchCallIds.size;
5375
+ const standaloneStableResult =
5376
+ !belongsToCurrentBatch && currentToolBatchCallIds.size === 0;
5377
+ if (currentBatchIsStable || standaloneStableResult) {
5378
+ // Persist the SDK's now-stable complete call/result batch. Keep
5379
+ // the receipts until the normalized tool-output event below is
5380
+ // durably flushed: recovery then covers every crash boundary
5381
+ // without either losing or duplicating the UI projection.
4004
5382
  await reconcileConversationTruth({ requireDurable: true });
4005
- const cleared = await clearDurablePendingSessionToolCalls(db, {
4006
- accountId: input.accountId,
4007
- workspaceId: input.workspaceId,
4008
- sessionId: input.sessionId,
4009
- turnId: activeTurnId,
4010
- executionGeneration,
4011
- attemptId: input.attemptId,
4012
- });
4013
- if (!cleared.accepted) {
4014
- throw new TurnAttemptFencedError(
4015
- "turn attempt ended while finalizing tool-call results",
4016
- );
4017
- }
5383
+ stableToolCallIdsToClear = currentBatchIsStable
5384
+ ? [...currentToolBatchCallIds]
5385
+ : [completedToolCall.callId];
5386
+ completedCurrentToolBatch = currentBatchIsStable;
4018
5387
  }
4019
5388
  }
4020
5389
  const normalized = normalizeSdkEvent(next.value);
@@ -4022,6 +5391,26 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4022
5391
  streamTiming.onEvent(event.type);
4023
5392
  await batcher.push(event);
4024
5393
  }
5394
+ if (stableToolCallIdsToClear) {
5395
+ const cleared = await clearDurablePendingSessionToolCalls(db, {
5396
+ accountId: input.accountId,
5397
+ workspaceId: input.workspaceId,
5398
+ sessionId: input.sessionId,
5399
+ turnId: activeTurnId,
5400
+ executionGeneration,
5401
+ attemptId: input.attemptId,
5402
+ callIds: stableToolCallIdsToClear,
5403
+ });
5404
+ if (!cleared.accepted) {
5405
+ throw new TurnAttemptFencedError(
5406
+ "turn attempt ended while finalizing tool-call results",
5407
+ );
5408
+ }
5409
+ if (completedCurrentToolBatch) {
5410
+ currentToolBatchCallIds = new Set<string>();
5411
+ currentToolBatchCompletedCallIds = new Set<string>();
5412
+ }
5413
+ }
4025
5414
  }
4026
5415
  } finally {
4027
5416
  if (!streamDone) {
@@ -4036,95 +5425,139 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4036
5425
  void iterator.return?.().catch(() => undefined);
4037
5426
  }
4038
5427
  }
4039
- await batcher.flush();
4040
- await stream.completed.catch(() => undefined);
4041
- if (responseUsageCount === 0) {
5428
+ await waitForTurnStreamCleanup(
5429
+ batcher.flush(),
5430
+ stream.completed.catch(() => undefined),
5431
+ cancellationSignal,
5432
+ );
5433
+ if (modelResponseUsageState.responseUsageCount === 0) {
4042
5434
  const aggregateUsage = stream.state.usage;
4043
- const aggregateInput = (aggregateUsage as { inputTokens?: unknown } | undefined)
4044
- ?.inputTokens;
4045
- if (typeof aggregateInput === "number" && aggregateInput > 0) {
4046
- lastInputTokensObserved = aggregateInput;
4047
- }
5435
+ const normalizedAggregateUsage = normalizeModelCallUsage(aggregateUsage);
5436
+ const aggregateInput = normalizedAggregateUsage.telemetry.inputTokens;
4048
5437
  const aggregateSourceKey = modelUsageSourceKey({
4049
5438
  responseId: null,
4050
5439
  dispatchId: modelUsageDispatchId,
4051
5440
  positionalKey: "aggregate",
4052
5441
  });
4053
- // The single aggregate frame is this turn's only model-usage record, so
4054
- // it is the first (account-switch surfaces here just like a first response).
4055
- const aggregateAccountCtx = modelCallAccountContext({
4056
- servingCredentialId: effectiveCodexCredentialId,
4057
- priorSessionCredentialId: priorSessionCodexCredentialId,
4058
- isFirstCallOfTurn: true,
4059
- });
4060
- recordModelCacheTokens(observability, streamProvider, {
4061
- cachedTokens: modelCallUsageTelemetry(
4062
- aggregateUsage as Parameters<typeof modelCallUsageTelemetry>[0],
4063
- ).cachedTokens,
4064
- promptTokens: (aggregateUsage as { inputTokens?: unknown } | undefined)?.inputTokens as
4065
- | number
4066
- | undefined,
4067
- });
4068
- await recordCompletedModelCallBeforeOwnershipFences({
4069
- renewLease: () => renewCodexLease("model_usage"),
4070
- leaseLost: () => codexLeaseLost,
4071
- leaseLostMessage: "Codex credential lease expired during the active turn",
4072
- recordUsage: async () => {
4073
- await recordModelUsageAndDebitCredits(settings, db, {
4074
- accountId: input.accountId,
4075
- workspaceId: input.workspaceId,
4076
- sessionId: input.sessionId,
4077
- turnId: activeTurnId,
4078
- model: turn.model,
4079
- isCodexTurn,
4080
- usage: aggregateUsage,
4081
- sourceKey: aggregateSourceKey,
4082
- observability,
4083
- });
4084
- await emitModelCallUsage({
4085
- observability,
4086
- publish,
4087
- accountId: input.accountId,
4088
- workspaceId: input.workspaceId,
4089
- sessionId: input.sessionId,
4090
- turnId: activeTurnId,
4091
- provider: resolvedModel?.provider.id ?? settings.openaiProvider,
4092
- providerApi: resolvedModel?.provider.api ?? "responses",
4093
- model: turn.model,
4094
- sourceKey: aggregateSourceKey,
4095
- usage: { usage: aggregateUsage },
4096
- servingAccountHash: aggregateAccountCtx.servingAccountHash,
4097
- accountChangedFromPrevCall: aggregateAccountCtx.accountChangedFromPrevCall,
4098
- emittedSourceKeys: emittedModelUsageSourceKeys,
4099
- });
4100
- },
4101
- recordAttemptSignals: async () => {
4102
- if (typeof aggregateInput === "number" && aggregateInput > 0) {
4103
- await setLastInputTokensFenced(aggregateInput);
4104
- }
4105
- },
4106
- });
5442
+ if (!claimedModelUsageSourceKeys.has(aggregateSourceKey)) {
5443
+ claimedModelUsageSourceKeys.add(aggregateSourceKey);
5444
+ // The single aggregate frame is this turn's only model-usage record, so
5445
+ // it is the first (account-switch surfaces here just like a first response).
5446
+ const aggregateAccountCtx = modelCallAccountContext({
5447
+ servingCredentialId: effectiveCodexCredentialId,
5448
+ priorSessionCredentialId: priorSessionCodexCredentialId,
5449
+ isFirstCallOfTurn: true,
5450
+ });
5451
+ let aggregateAuthoritative = false;
5452
+ await recordCompletedModelCallBeforeOwnershipFences({
5453
+ renewLease: () => renewCodexLease("model_usage"),
5454
+ leaseLost: () => codexLeaseLost,
5455
+ leaseLostMessage: "Codex credential lease expired during the active turn",
5456
+ recordUsage: async () => {
5457
+ await recordModelUsageAndDebitCredits(settings, db, {
5458
+ accountId: input.accountId,
5459
+ workspaceId: input.workspaceId,
5460
+ sessionId: input.sessionId,
5461
+ turnId: activeTurnId,
5462
+ turnAttemptId: input.attemptId,
5463
+ model: turn.model,
5464
+ externallyBilled: isExternallyBilledTurn,
5465
+ usage: aggregateUsage,
5466
+ normalizedUsage: normalizedAggregateUsage,
5467
+ sourceKey: aggregateSourceKey,
5468
+ observability,
5469
+ });
5470
+ aggregateAuthoritative = await emitModelCallUsage({
5471
+ observability,
5472
+ publish,
5473
+ accountId: input.accountId,
5474
+ workspaceId: input.workspaceId,
5475
+ sessionId: input.sessionId,
5476
+ turnId: activeTurnId,
5477
+ provider: resolvedModel?.provider.id ?? settings.openaiProvider,
5478
+ providerApi: resolvedModel?.provider.api ?? "responses",
5479
+ model: turn.model,
5480
+ sourceKey: aggregateSourceKey,
5481
+ usage: { usage: aggregateUsage },
5482
+ normalizedUsage: normalizedAggregateUsage,
5483
+ servingAccountHash: aggregateAccountCtx.servingAccountHash,
5484
+ accountChangedFromPrevCall: aggregateAccountCtx.accountChangedFromPrevCall,
5485
+ emittedSourceKeys: emittedModelUsageSourceKeys,
5486
+ });
5487
+ if (aggregateAuthoritative && aggregateInput !== null && aggregateInput > 0) {
5488
+ recordModelInputTokens(observability, streamProvider, aggregateInput);
5489
+ }
5490
+ },
5491
+ recordAttemptSignals: async () => {
5492
+ if (!aggregateAuthoritative) return;
5493
+ if (normalizedAggregateUsage.totalTokens !== null) {
5494
+ modelResponseUsageState.lastProviderContextTokensObserved =
5495
+ normalizedAggregateUsage.totalTokens;
5496
+ modelResponseUsageState.providerContextRevision += 1;
5497
+ }
5498
+ if (aggregateInput !== null && aggregateInput > 0) {
5499
+ await setLastInputTokensFenced(aggregateInput);
5500
+ }
5501
+ },
5502
+ });
5503
+ }
4107
5504
  }
4108
5505
  if (stream.interruptions.length > 0) {
4109
5506
  await reconcileConversationTruth();
4110
5507
  const approvals = runtime.serializeApprovals(stream.interruptions);
4111
- await saveRunStateFenced({
4112
- accountId: input.accountId,
4113
- workspaceId: input.workspaceId,
4114
- sessionId: input.sessionId,
4115
- turnId: activeTurnId,
4116
- expectedExecutionGeneration: executionGeneration,
4117
- expectedAttemptId: input.attemptId,
4118
- serializedRunState: stream.state.toString(),
4119
- pendingApprovals: approvals,
4120
- // Record the account freezing this state so a resume on a DIFFERENT
4121
- // codex account strips its account-bound reasoning before replay (HOLE C).
4122
- frozenCodexCredentialId: effectiveCodexCredentialId,
4123
- });
5508
+ const humanInputInterruptions =
5509
+ runtime.serializeHumanInputRequests?.(stream.interruptions) ?? [];
5510
+ const humanInputRequests = await Promise.all(
5511
+ humanInputInterruptions.map(async (interruption) => {
5512
+ const id = stableHumanInputRequestId(
5513
+ input.sessionId,
5514
+ activeTurnId,
5515
+ interruption.toolCallId,
5516
+ );
5517
+ const existing = await getSessionHumanInputRequest(
5518
+ db,
5519
+ input.workspaceId,
5520
+ input.sessionId,
5521
+ id,
5522
+ );
5523
+ if (existing && existing.status !== "pending") {
5524
+ throw new Error(`Settled human-input request ${id} reappeared as an interruption`);
5525
+ }
5526
+ const expiresAt = existing?.expiresAt
5527
+ ? new Date(existing.expiresAt)
5528
+ : interruption.input.expiresInSeconds
5529
+ ? new Date(Date.now() + interruption.input.expiresInSeconds * 1000)
5530
+ : null;
5531
+ return {
5532
+ id,
5533
+ toolCallId: interruption.toolCallId,
5534
+ questions: interruption.input.questions,
5535
+ allowSkip: interruption.input.allowSkip,
5536
+ expiresAt,
5537
+ isNew: existing === null,
5538
+ };
5539
+ }),
5540
+ );
5541
+ const requestEvents = humanInputRequests
5542
+ .filter((request) => request.isNew)
5543
+ .map((request) => ({
5544
+ type: "session.humanInput.requested" as const,
5545
+ payload: {
5546
+ request: {
5547
+ id: request.id,
5548
+ questions: request.questions,
5549
+ allowSkip: request.allowSkip,
5550
+ expiresAt: request.expiresAt?.toISOString() ?? null,
5551
+ },
5552
+ },
5553
+ }));
4124
5554
  if (
4125
5555
  !(await settle!({
4126
5556
  events: [
4127
- { type: "session.requiresAction", payload: { approvals } },
5557
+ ...requestEvents,
5558
+ ...(approvals.length > 0
5559
+ ? [{ type: "session.requiresAction" as const, payload: { approvals } }]
5560
+ : []),
4128
5561
  {
4129
5562
  type: "session.status.changed",
4130
5563
  payload: { status: "requires_action" },
@@ -4133,6 +5566,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4133
5566
  turnStatus: "requires_action",
4134
5567
  sessionStatus: "requires_action",
4135
5568
  activeTurnId,
5569
+ runState: {
5570
+ serializedRunState: stream.state.toString(),
5571
+ pendingApprovals: approvals,
5572
+ frozenCodexCredentialId: effectiveCodexCredentialId,
5573
+ humanInputRequests: humanInputRequests.map(
5574
+ ({ isNew: _isNew, ...request }) => request,
5575
+ ),
5576
+ },
4136
5577
  }))
4137
5578
  ) {
4138
5579
  return claimedResult({ status: "cancelled" });
@@ -4182,6 +5623,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4182
5623
  unit: "run",
4183
5624
  sourceResourceType: "session_turn",
4184
5625
  sourceResourceId: activeTurnId,
5626
+ sessionId: input.sessionId,
5627
+ turnId: activeTurnId,
5628
+ turnAttemptId: input.attemptId,
4185
5629
  idempotencyKey: `usage:agent_run.completed:${activeTurnId}`,
4186
5630
  });
4187
5631
  activityStatus = "idle";
@@ -4242,9 +5686,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4242
5686
  }
4243
5687
  }
4244
5688
  } catch (compactError) {
4245
- compactionFailureMessage = compactionFailureReason(
4246
- compactError instanceof Error ? compactError.message : String(compactError),
4247
- );
5689
+ // Transient checkpoint-provider failures recover this same accepted
5690
+ // turn through the normal provider/capacity path. They are not an
5691
+ // empty summary and must not create a new goal continuation.
5692
+ if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
5693
+ if (!isCompactionSummaryFailure(compactError)) throw compactError;
5694
+ compactionFailureMessage = compactionFailureReasonFromError(compactError);
4248
5695
  observability.warn("context compaction recovery compaction failed", {
4249
5696
  sessionId: input.sessionId,
4250
5697
  turnId: activeTurnId,
@@ -4276,6 +5723,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4276
5723
  turnStatus: "failed",
4277
5724
  sessionStatus: "idle",
4278
5725
  activeTurnId: null,
5726
+ ...(recoveryKind === "operator" ? { consumeRequestedCompactionFailure: true } : {}),
4279
5727
  }))
4280
5728
  ) {
4281
5729
  return claimedResult({ status: "cancelled" });
@@ -4283,7 +5731,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4283
5731
  turnMetricOutcome = "failed";
4284
5732
  activityStatus = "idle";
4285
5733
  activityError = attemptError;
4286
- return claimedResult({ status: "idle" });
5734
+ // The failed turn settlement already defers ordinary internal
5735
+ // updates and makes the delivered goal-continuation receipt
5736
+ // terminal. End this workflow run as well: returning plain idle
5737
+ // would immediately synthesize another goal continuation against
5738
+ // the unchanged active history and repeat the same failed
5739
+ // compaction. A new human/API prompt, Steer, or explicitly requested
5740
+ // Compact remains a durable explicit wake and may retry; ordinary
5741
+ // machine updates stay pending for that actionable wake.
5742
+ return claimedResult({ status: "idle", deferredUntilWake: true });
4287
5743
  }
4288
5744
  // Codex parity: compaction remains inside the same logical turn and
4289
5745
  // the same activity. Rebuild the model-visible history from the
@@ -4326,6 +5782,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4326
5782
  reason: "sandbox_lease_superseded",
4327
5783
  });
4328
5784
  if (recovery.action === "stale") {
5785
+ acknowledgeLostAttemptOwnership();
4329
5786
  activityStatus = "cancelled";
4330
5787
  turnMetricOutcome = "cancelled";
4331
5788
  return claimedResult({ status: "cancelled" });
@@ -4344,7 +5801,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4344
5801
  throw recoveryError;
4345
5802
  }
4346
5803
  }
4347
- if (isWorkerShutdownCancellation(error) && recoveryTurnId) {
5804
+ const cancellationFailure = turnOperationCancellationFailure(error);
5805
+ if (
5806
+ cancellationFailure &&
5807
+ isWorkerShutdownCancellation(cancellationFailure) &&
5808
+ recoveryTurnId
5809
+ ) {
4348
5810
  try {
4349
5811
  await flushRuntimeBatcher();
4350
5812
  await reconcileConversationTruth();
@@ -4363,6 +5825,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4363
5825
  reason: "worker_shutdown",
4364
5826
  });
4365
5827
  if (recovery.action === "stale") {
5828
+ acknowledgeLostAttemptOwnership();
4366
5829
  activityStatus = "cancelled";
4367
5830
  turnMetricOutcome = "cancelled";
4368
5831
  return claimedResult({ status: "cancelled" });
@@ -4387,26 +5850,35 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4387
5850
  if (error instanceof TurnAttemptFencedError) {
4388
5851
  activityStatus = "cancelled";
4389
5852
  activityError = error;
4390
- await flushRuntimeBatcher();
5853
+ acknowledgeQuiescence = true;
5854
+ noteCancellationRequested();
5855
+ await waitForTurnFinalizerStep(
5856
+ flushRuntimeBatcher(),
5857
+ turnFinalizerCancellationSignal(cancellationSignal, activityStatus),
5858
+ );
4391
5859
  // Ownership already moved to a newer attempt or an authoritative
4392
- // control transaction. Surface a transport cancellation instead of a
4393
- // normal activity result: with WAIT_CANCELLATION_COMPLETED, Temporal
4394
- // must observe a terminal cancellation before the session workflow may
4395
- // close. Returning normally after a cancel request is rejected by the
4396
- // server and leaves the worker task detached indefinitely.
5860
+ // control transaction. Surface the exact transport cancellation rather
5861
+ // than a normal result. Temporal terminalization remains diagnostic
5862
+ // only; replacement admission waits for the activity-owned durable
5863
+ // quiescence receipt written from the hard tool fence below.
4397
5864
  turnMetricOutcome = "cancelled";
4398
5865
  throw new CancelledFailure("TURN_ATTEMPT_FENCED", [], error);
4399
5866
  }
4400
- if (error instanceof CancelledFailure) {
5867
+ if (cancellationFailure) {
4401
5868
  activityStatus = "cancelled";
4402
5869
  activityError = error;
4403
- await flushRuntimeBatcher();
5870
+ acknowledgeQuiescence = true;
5871
+ noteCancellationRequested();
5872
+ await waitForTurnFinalizerStep(
5873
+ flushRuntimeBatcher(),
5874
+ turnFinalizerCancellationSignal(cancellationSignal, activityStatus),
5875
+ );
4404
5876
  // The workflow owns cancellation settlement: Pause/Steer controls use
4405
5877
  // settleSessionControl, and heartbeat timeouts use worker-death
4406
5878
  // recovery. A dying activity must never append a
4407
5879
  // competing cancellation or mutate the turn/session on its own.
4408
5880
  turnMetricOutcome = "cancelled";
4409
- throw error;
5881
+ throw cancellationFailure;
4410
5882
  }
4411
5883
  // The SDK's per-segment turn cap is a pacing valve, not a failure: end
4412
5884
  // the turn gracefully and idle the session so an active goal continues
@@ -4448,6 +5920,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4448
5920
  unit: "run",
4449
5921
  sourceResourceType: "session_turn",
4450
5922
  sourceResourceId: turnId,
5923
+ sessionId: input.sessionId,
5924
+ turnId,
5925
+ turnAttemptId: input.attemptId,
4451
5926
  idempotencyKey: `usage:agent_run.completed:${turnId}`,
4452
5927
  });
4453
5928
  activityStatus = "idle";
@@ -4613,12 +6088,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4613
6088
  : {}),
4614
6089
  }
4615
6090
  : null;
4616
- const cooldownUntil = codexCredentialCooldownUntil(
4617
- codexCredentialFailure,
4618
- serving,
4619
- settings.codexRotationNearExhaustionPct,
4620
- now,
4621
- );
6091
+ const cooldownUntil = codexCredentialCooldownUntil(codexCredentialFailure, serving, now);
4622
6092
  const statePersisted =
4623
6093
  codexLeaseHolderId && codexLeaseGeneration !== null
4624
6094
  ? await quarantineCodexCredentialForLease(db, {
@@ -4663,7 +6133,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4663
6133
  activeCredentialId: rotation.activeCredentialId,
4664
6134
  priorCredentialId: effectiveCodexCredentialId,
4665
6135
  accounts,
4666
- nearExhaustionPct: settings.codexRotationNearExhaustionPct,
4667
6136
  now: new Date(),
4668
6137
  usedConnectors: serving?.connectorNamespaces ?? [],
4669
6138
  })
@@ -4725,9 +6194,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4725
6194
  if (settlement.action === "stale") {
4726
6195
  // One transaction proves both exact-holder recovery (including a
4727
6196
  // just-expired or reaped lease row) and successor/control-gate
4728
- // rejection. A stale activity performs no second settlement.
4729
- activityStatus = "recovering";
4730
- turnMetricOutcome = "recovering";
6197
+ // rejection. Cross the hard tool fence so a control-gate loss can
6198
+ // write its quiescence receipt; a successor-only loss is a no-op.
6199
+ acknowledgeLostAttemptOwnership();
6200
+ activityStatus = "cancelled";
6201
+ turnMetricOutcome = "cancelled";
4731
6202
  return claimedResult({ status: "recovering" });
4732
6203
  }
4733
6204
  }
@@ -4784,7 +6255,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4784
6255
  const until = codexCredentialCooldownUntil(
4785
6256
  { kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds },
4786
6257
  serving,
4787
- settings.codexRotationNearExhaustionPct,
4788
6258
  new Date(),
4789
6259
  )!;
4790
6260
  // Finding 1a: INSPECT the cooldown-write result. A swallowed best-effort
@@ -4822,7 +6292,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4822
6292
  const newHome = shardCredentialForSession({
4823
6293
  sessionId: input.sessionId,
4824
6294
  accounts: fresh,
4825
- nearExhaustionPct: settings.codexRotationNearExhaustionPct,
4826
6295
  now: new Date(),
4827
6296
  });
4828
6297
  if (newHome) {
@@ -4871,16 +6340,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4871
6340
  } else {
4872
6341
  // Every account capped/cooling → idle until the earliest reset across all.
4873
6342
  rotated = true;
4874
- allCappedResetAt = earliestCodexReset(
4875
- fresh,
4876
- settings.codexRotationNearExhaustionPct,
4877
- new Date(),
4878
- );
4879
- capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(
4880
- fresh,
4881
- settings.codexRotationNearExhaustionPct,
4882
- new Date(),
4883
- );
6343
+ allCappedResetAt = earliestCodexReset(fresh, new Date());
6344
+ capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
4884
6345
  }
4885
6346
  } else {
4886
6347
  const decision = chooseRotationActive({
@@ -4888,7 +6349,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4888
6349
  activeCredentialId: rotation.activeCredentialId,
4889
6350
  priorCredentialId: effectiveCodexCredentialId,
4890
6351
  accounts: fresh,
4891
- nearExhaustionPct: settings.codexRotationNearExhaustionPct,
4892
6352
  now: new Date(),
4893
6353
  // P4: the just-capped serving account's connector set is the proxy for
4894
6354
  // "what this session has access to" — prefer a covering failover target.
@@ -4915,11 +6375,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4915
6375
  } else if (decision.kind === "allCapped") {
4916
6376
  rotated = true;
4917
6377
  allCappedResetAt = decision.earliestResetAt;
4918
- capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(
4919
- fresh,
4920
- settings.codexRotationNearExhaustionPct,
4921
- new Date(),
4922
- );
6378
+ capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
4923
6379
  }
4924
6380
  // kind:"none" → fall through to today's single-account idle.
4925
6381
  }
@@ -5094,6 +6550,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5094
6550
  unit: "run",
5095
6551
  sourceResourceType: "session_turn",
5096
6552
  sourceResourceId: turnId,
6553
+ sessionId: input.sessionId,
6554
+ turnId,
6555
+ turnAttemptId: input.attemptId,
5097
6556
  idempotencyKey: `usage:agent_run.completed:${turnId}`,
5098
6557
  });
5099
6558
  activityStatus = "idle";
@@ -5105,7 +6564,35 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5105
6564
  // truth, recover this SAME accepted turn, then let the workflow re-claim
5106
6565
  // it after a pacing delay. This is independent of goal state and never
5107
6566
  // relies on a synthetic continuation prompt.
5108
- const failure = agentRunFailurePayload(error);
6567
+ const failure = agentRunFailurePayload(error, { isCodexTurn });
6568
+ if (isSessionEventPersistenceError(error)) {
6569
+ // Never pass the original Drizzle/postgres-js error to telemetry: its
6570
+ // nested cause may contain raw SQL and bound parameters. The typed DB
6571
+ // boundary retains only SQLSTATE, stage, correlation, and safe catalog
6572
+ // identifiers. Provider inference has already happened and is NOT
6573
+ // retried by this terminal classification.
6574
+ observability.error("session event persistence failed", {
6575
+ accountId: input.accountId,
6576
+ workspaceId: input.workspaceId,
6577
+ sessionId: input.sessionId,
6578
+ turnId,
6579
+ attemptId: input.attemptId,
6580
+ code: error.details.code,
6581
+ sqlState: error.details.sqlState ?? "unknown",
6582
+ stage: error.details.stage,
6583
+ eventTypes: error.details.eventTypes.join(","),
6584
+ correlationId: error.details.correlationId,
6585
+ attempts: error.details.attempts,
6586
+ retryOutcome: error.details.retryOutcome,
6587
+ dbSeverity: error.details.database.severity,
6588
+ dbSchema: error.details.database.schema,
6589
+ dbTable: error.details.database.table,
6590
+ dbColumn: error.details.database.column,
6591
+ dbDataType: error.details.database.dataType,
6592
+ dbConstraint: error.details.database.constraint,
6593
+ dbRoutine: error.details.database.routine,
6594
+ });
6595
+ }
5109
6596
  if (failure.retryable && publish && turnId && turnStartedPublished) {
5110
6597
  const recoveryResult = providerRecoveryResult();
5111
6598
  await flushRuntimeBatcher();
@@ -5122,6 +6609,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5122
6609
  },
5123
6610
  });
5124
6611
  if (recovery.action === "stale") {
6612
+ acknowledgeLostAttemptOwnership();
5125
6613
  activityStatus = "cancelled";
5126
6614
  turnMetricOutcome = "cancelled";
5127
6615
  return claimedResult({ status: "cancelled" });
@@ -5173,215 +6661,490 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5173
6661
  );
5174
6662
  return claimedResult({ status: "failed" });
5175
6663
  } finally {
5176
- gitCredentialRenewalClosed = true;
5177
- const renewalToStop = gitCredentialRenewal as GitCredentialRenewalController | null;
5178
- gitCredentialRenewal = null;
5179
- await renewalToStop?.stop();
5180
- const durationSeconds = (performance.now() - activityStarted) / 1000;
5181
- observability.recordWorkerActivity({
5182
- activity: "runAgentTurn",
5183
- status: activityStatus,
5184
- durationSeconds,
5185
- });
5186
- if (turnId && activityStatus !== "unknown") {
5187
- turnLifecycleMetricsFor(observability).finish(turnId, turnMetricOutcome, durationSeconds);
5188
- }
5189
- activitySpan.end({
5190
- attributes: {
5191
- "opengeni.turn_id": turnId ?? "",
5192
- "opengeni.status": activityStatus,
5193
- "opengeni.variable_set_id": variableSetId,
5194
- "opengeni.rig_id": rigId,
5195
- "opengeni.rig_version_id": rigVersionId,
5196
- "opengeni.codex_credential_id": effectiveCodexCredentialId ?? "",
5197
- "opengeni.duration_ms": Math.round(durationSeconds * 1000),
5198
- },
5199
- error: activityError,
5200
- });
5201
- // Drain the buffered Connected Machine op events (infra failures + healed
5202
- // recoveries) to durable session events — awaited, best-effort, never blocking
5203
- // the turn. Sync observer → buffer → single awaited append here (no unawaited
5204
- // DB write inside the activity). Scoped to this turn; skipped if no turnId
5205
- // (the op ran under a turn, so on the normal path turnId is set).
5206
- const machineOpEvents = machineOpObserver.drainEvents();
5207
- if (machineOpEvents.length > 0 && turnId && executionGeneration > 0) {
5208
- await appendAndPublishTurnEventsFenced(
5209
- db,
5210
- bus,
5211
- input.workspaceId,
5212
- input.sessionId,
5213
- turnId,
5214
- executionGeneration,
5215
- input.attemptId,
5216
- machineOpEvents.map((event) => ({
5217
- ...event,
5218
- turnId: turnId ?? null,
5219
- })),
5220
- ).catch(() => undefined);
5221
- }
5222
- // Multi-account P4: flush the serving account's free per-turn caches ONCE,
5223
- // best-effort (same discipline as today's usage write). Both writers skip
5224
- // version/updatedAt, so neither can race the token-refresh CAS.
5225
- if (effectiveCodexCredentialId) {
5226
- // Part A: the latest scraped usage-header snapshot → the P2 usage cache. A
5227
- // full both-windows snapshot (parseCodexUsageHeaders gates on both), so this
5228
- // is byte-identical to the /wham/usage write — no partial-window clobber.
5229
- if (latestCodexUsage) {
5230
- const usageMutation = await recordCodexAccountUsageWithWakeTargets(
5231
- db,
5232
- input.workspaceId,
5233
- effectiveCodexCredentialId,
5234
- latestCodexUsage,
5235
- ).catch(() => null);
5236
- if (usageMutation) {
5237
- await signalCodexCapacityWakeTargets(
5238
- { signalCodexCapacityWorkflow, wakeSessionWorkflow },
5239
- usageMutation.wakeTargets,
6664
+ const finalizationStarted = performance.now();
6665
+ let finalizationError: unknown;
6666
+ let physicalToolQuiescenceConfirmed = !acknowledgeQuiescence;
6667
+ let quiescenceReceiptOrProofDurable = !acknowledgeQuiescence;
6668
+ const finalizerSignal = turnFinalizerCancellationSignal(cancellationSignal, activityStatus);
6669
+ try {
6670
+ const toolCancellationFence = toolCancellationFenceRef.current;
6671
+ // Toolspace and run-credential renewals are attempt-owned sandbox
6672
+ // writers. Capture and close them before crossing the same authoritative
6673
+ // boundary as tool calls; none of these drains may race the already-
6674
+ // aborted Temporal signal.
6675
+ toolspaceTokenRenewalClosed = true;
6676
+ const toolspaceRenewalToStop =
6677
+ toolspaceTokenRenewal as ToolspaceTokenRenewalController | null;
6678
+ toolspaceTokenRenewal = null;
6679
+ runCredentialRenewalClosed = true;
6680
+ const runRenewalToStop = runCredentialRenewal as RunCredentialRenewalController | null;
6681
+ runCredentialRenewal = null;
6682
+ await drainAttemptOwnedSandboxWriters({
6683
+ toolCancellationFence: acknowledgeQuiescence ? toolCancellationFence : null,
6684
+ cancellationReason: cancellationSignal?.reason,
6685
+ toolspaceTokenRenewal: toolspaceRenewalToStop,
6686
+ runCredentialRenewal: runRenewalToStop,
6687
+ });
6688
+ if (acknowledgeQuiescence) {
6689
+ // A cancellation before sandbox-backed capabilities exist still has
6690
+ // no tool controller to drain. Sandbox agent construction fails closed
6691
+ // when a backend exists but no controller was installed. Renewal
6692
+ // writers, when present, were drained above in either case.
6693
+ physicalToolQuiescenceConfirmed = true;
6694
+ }
6695
+ if (acknowledgeQuiescence && physicalToolQuiescenceConfirmed) {
6696
+ // This receipt is part of the hard cancellation boundary, not
6697
+ // housekeeping. Persist it immediately after the sandbox/tool fence
6698
+ // and before lease, cache, recording, or provider cleanup. Its
6699
+ // transaction also enqueues the exact workflow wake that will admit
6700
+ // the replacement; Temporal activity terminalization does neither.
6701
+ const proof: SessionAttemptQuiescenceProof = {
6702
+ accountId: input.accountId,
6703
+ workspaceId: input.workspaceId,
6704
+ sessionId: input.sessionId,
6705
+ attemptId: input.attemptId,
6706
+ workflowId: input.workflowId,
6707
+ workflowRunId: input.workflowRunId,
6708
+ activityId: dispatchId,
6709
+ };
6710
+ const recoveryMode = await persistOrSignalSessionAttemptQuiescence({
6711
+ proof,
6712
+ persistReceipt: async () =>
6713
+ await markSessionAttemptQuiesced(db, {
6714
+ accountId: input.accountId,
6715
+ workspaceId: input.workspaceId,
6716
+ sessionId: input.sessionId,
6717
+ attemptId: input.attemptId,
6718
+ temporalWorkflowId: input.workflowId,
6719
+ temporalWorkflowRunId: input.workflowRunId,
6720
+ temporalActivityId: dispatchId,
6721
+ allowUninterrupted: true,
6722
+ }),
6723
+ publishEvents: async (events) => {
6724
+ await waitForTurnFinalizerStep(
6725
+ publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events),
6726
+ finalizerSignal,
6727
+ );
6728
+ },
6729
+ signalProof: signalSessionAttemptQuiesced,
6730
+ heartbeat: (attempt, retryMs) => {
6731
+ activityContext?.heartbeat({
6732
+ phase: "quiescence-proof-delivery",
6733
+ sessionId: input.sessionId,
6734
+ attemptId: input.attemptId,
6735
+ deliveryAttempt: attempt,
6736
+ retryMs,
6737
+ at: new Date().toISOString(),
6738
+ });
6739
+ },
6740
+ onReceiptFailure: (error) => {
6741
+ console.error("agent turn quiescence receipt exhausted; signalling proof", error);
6742
+ },
6743
+ onPublishFailure: (error) => {
6744
+ console.error("agent turn quiescence event fanout failed", error);
6745
+ },
6746
+ onSignalFailure: (error, attempt, retryMs) => {
6747
+ console.error("agent turn quiescence proof signal failed; retrying", {
6748
+ error,
6749
+ attempt,
6750
+ retryMs,
6751
+ });
6752
+ },
6753
+ });
6754
+ quiescenceReceiptOrProofDurable = true;
6755
+ if (recoveryMode === "signal") {
6756
+ observability.info("agent turn quiescence proof handed to workflow recovery", {
6757
+ "opengeni.session_id": input.sessionId,
6758
+ "opengeni.attempt_id": input.attemptId,
6759
+ "opengeni.workflow_run_id": input.workflowRunId,
6760
+ "opengeni.activity_id": dispatchId,
6761
+ });
6762
+ }
6763
+ }
6764
+ // Deleting this attempt's credential generation is post-receipt
6765
+ // housekeeping: pointer mutation is serialized with successor
6766
+ // activation and the attempt-qualified check cannot erase a successor.
6767
+ // It may therefore detach on cancellation without weakening the writer
6768
+ // drain that gates the receipt above.
6769
+ const credentialSessionToClear = runCredentialSession;
6770
+ runCredentialSession = null;
6771
+ if (credentialSessionToClear) {
6772
+ const cleanup = clearRunCredentialsForAttempt(credentialSessionToClear, {
6773
+ sessionId: input.sessionId,
6774
+ attemptId: input.attemptId,
6775
+ executionGeneration,
6776
+ }).catch((error: unknown) => {
6777
+ try {
6778
+ observability.incrementCounter({
6779
+ name: "opengeni_run_credential_cleanup_total",
6780
+ help: "Attempt-owned run credential cleanup outcomes.",
6781
+ labels: { outcome: "error" },
6782
+ });
6783
+ observability.warn("Attempt-owned run credential cleanup failed", {
6784
+ sessionId: input.sessionId,
6785
+ turnId,
6786
+ attemptId: input.attemptId,
6787
+ errorClass: error instanceof Error ? error.name : "UnknownError",
6788
+ });
6789
+ } catch {
6790
+ // Cleanup observability must not own finalizer liveness.
6791
+ }
6792
+ });
6793
+ await waitForTurnFinalizerStep(cleanup, finalizerSignal);
6794
+ }
6795
+ gitCredentialRenewalClosed = true;
6796
+ const renewalsToStop = gitCredentialRenewals;
6797
+ gitCredentialRenewals = [];
6798
+ for (const renewal of renewalsToStop) {
6799
+ await waitForTurnFinalizerStep(renewal.stop(), finalizerSignal);
6800
+ }
6801
+ // Drain the buffered Connected Machine op events (infra failures + healed
6802
+ // recoveries) to durable session events — awaited, best-effort, never blocking
6803
+ // the turn. Sync observer → buffer → single awaited append here (no unawaited
6804
+ // DB write inside the activity). Scoped to this turn; skipped if no turnId
6805
+ // (the op ran under a turn, so on the normal path turnId is set).
6806
+ const machineOpEvents = machineOpObserver.drainEvents();
6807
+ if (machineOpEvents.length > 0 && turnId && executionGeneration > 0) {
6808
+ await waitForTurnFinalizerStep(
6809
+ appendAndPublishTurnEventsFenced(
6810
+ db,
6811
+ bus,
6812
+ input.workspaceId,
6813
+ input.sessionId,
6814
+ turnId,
6815
+ executionGeneration,
6816
+ input.attemptId,
6817
+ machineOpEvents.map((event) => ({
6818
+ ...event,
6819
+ turnId: turnId ?? null,
6820
+ })),
6821
+ ).catch(() => undefined),
6822
+ finalizerSignal,
6823
+ );
6824
+ }
6825
+ // Multi-account P4: flush the serving account's free per-turn caches ONCE,
6826
+ // best-effort (same discipline as today's usage write). Both writers skip
6827
+ // version/updatedAt, so neither can race the token-refresh CAS.
6828
+ if (effectiveCodexCredentialId) {
6829
+ // Part A: the latest scraped usage-header snapshot → the P2 usage cache. A
6830
+ // full both-windows snapshot (parseCodexUsageHeaders gates on both), so this
6831
+ // is byte-identical to the /wham/usage write — no partial-window clobber.
6832
+ if (latestCodexUsage) {
6833
+ const usageMutation = await waitForTurnFinalizerStep(
6834
+ recordCodexAccountUsageWithWakeTargets(
6835
+ db,
6836
+ input.workspaceId,
6837
+ effectiveCodexCredentialId,
6838
+ latestCodexUsage,
6839
+ ).catch(() => null),
6840
+ finalizerSignal,
6841
+ );
6842
+ if (usageMutation) {
6843
+ await waitForTurnFinalizerStep(
6844
+ signalCodexCapacityWakeTargets(
6845
+ { signalCodexCapacityWorkflow, wakeSessionWorkflow },
6846
+ usageMutation.wakeTargets,
6847
+ ),
6848
+ finalizerSignal,
6849
+ );
6850
+ }
6851
+ }
6852
+ // Part B.1: the connector namespaces codex_apps listed this turn → the
6853
+ // connector-set cache. NON-EMPTY-only: a flaky/empty tools/list must never
6854
+ // overwrite a known set with [] (false coverage drop). Read by reference
6855
+ // AFTER the run, so every tools/list this turn has accumulated.
6856
+ const connectorNamespaces = preparedTools?.codexConnectorNamespaces;
6857
+ if (connectorNamespaces && connectorNamespaces.size > 0) {
6858
+ await waitForTurnFinalizerStep(
6859
+ recordCodexAccountConnectors(db, input.workspaceId, effectiveCodexCredentialId, [
6860
+ ...connectorNamespaces,
6861
+ ]).catch(() => undefined),
6862
+ finalizerSignal,
5240
6863
  );
5241
6864
  }
5242
6865
  }
5243
- // Part B.1: the connector namespaces codex_apps listed this turn → the
5244
- // connector-set cache. NON-EMPTY-only: a flaky/empty tools/list must never
5245
- // overwrite a known set with [] (false coverage drop). Read by reference
5246
- // AFTER the run, so every tools/list this turn has accumulated.
5247
- const connectorNamespaces = preparedTools?.codexConnectorNamespaces;
5248
- if (connectorNamespaces && connectorNamespaces.size > 0) {
5249
- await recordCodexAccountConnectors(db, input.workspaceId, effectiveCodexCredentialId, [
5250
- ...connectorNamespaces,
5251
- ]).catch(() => undefined);
6866
+ if (codexLeaseHeartbeatTimer) {
6867
+ clearInterval(codexLeaseHeartbeatTimer);
6868
+ codexLeaseHeartbeatTimer = undefined;
5252
6869
  }
5253
- }
5254
- if (codexLeaseHeartbeatTimer) {
5255
- clearInterval(codexLeaseHeartbeatTimer);
5256
- codexLeaseHeartbeatTimer = undefined;
5257
- }
5258
- if (codexLeaseHeld && turnId && codexLeaseHolderId && codexLeaseGeneration !== null) {
5259
- await releaseCodexCredentialLease(
5260
- db,
5261
- input.accountId,
5262
- input.workspaceId,
5263
- turnId,
5264
- codexLeaseHolderId,
5265
- codexLeaseGeneration,
5266
- ).catch(() => undefined);
5267
- codexLeaseHeld = false;
5268
- }
5269
- // Workbench v2 turn-end workspace capture (dossier §10.1) — runs FIRST in
5270
- // the turn-end finally, while the box is MAXIMALLY ALIVE. The agent's last
5271
- // tool ran before this finally, so /workspace is already final; capture is
5272
- // FS-equivalent to the already-settled recording preparation and the warm
5273
- // snapshot (neither mutates workspace files). Running it here — BEFORE
5274
- // preparedTools.close() (which tears down tools / computer-use / the display
5275
- // stack and is what starts the Modal box exiting a few seconds later) —
5276
- // gives capture the full live-box margin instead of racing the teardown
5277
- // tail, which was dropping 100% of captures on real Modal desktop boxes
5278
- // ("request cancelled due to container exiting", 0 rows). External module:
5279
- // self-capped at 60s, best-effort (never throws past its boundary),
5280
- // epoch-fenced, and it NEVER closes the box. The emitted
5281
- // workspace.revision.captured event is ANNOUNCE-ONLY (metadata, never
5282
- // content).
5283
- if (resolvedSandbox && setupBoxSession && sandboxGroupId) {
5284
- // Stop new heartbeat snapshot/meter ticks so a mid-turn snapshot cannot
5285
- // start concurrently with capture, then drain any in-flight snapshot
5286
- // (bounded) — capture and the warm snapshot both exec on the box, so
5287
- // sequence them, exactly as the turn-end snapshot placement did.
5288
- if (leaseHeartbeatTimer) {
5289
- clearInterval(leaseHeartbeatTimer);
5290
- leaseHeartbeatTimer = undefined;
6870
+ if (codexLeaseHeld && turnId && codexLeaseHolderId && codexLeaseGeneration !== null) {
6871
+ await waitForTurnFinalizerStep(
6872
+ releaseCodexCredentialLease(
6873
+ db,
6874
+ input.accountId,
6875
+ input.workspaceId,
6876
+ turnId,
6877
+ codexLeaseHolderId,
6878
+ codexLeaseGeneration,
6879
+ ).catch(() => undefined),
6880
+ finalizerSignal,
6881
+ );
6882
+ codexLeaseHeld = false;
5291
6883
  }
5292
- if (snapshotInFlight) {
5293
- await waitForWarmSnapshot(snapshotInFlight, settings.sandboxSnapshotTimeoutMs);
6884
+ // Workbench v2 turn-end workspace capture — runs FIRST in
6885
+ // the turn-end finally, while the box is MAXIMALLY ALIVE. The agent's last
6886
+ // tool ran before this finally, so /workspace is already final; capture is
6887
+ // FS-equivalent to the already-settled recording preparation and the warm
6888
+ // snapshot (neither mutates workspace files). Running it here — BEFORE
6889
+ // preparedTools.close() (which tears down tools / computer-use / the display
6890
+ // stack and is what starts the Modal box exiting a few seconds later) —
6891
+ // gives capture the full live-box margin instead of racing the teardown
6892
+ // tail, which was dropping 100% of captures on real Modal desktop boxes
6893
+ // ("request cancelled due to container exiting", 0 rows). External module:
6894
+ // self-capped at 60s, best-effort (never throws past its boundary),
6895
+ // epoch-fenced, and it NEVER closes the box. The emitted
6896
+ // workspace.revision.captured event is ANNOUNCE-ONLY (metadata, never
6897
+ // content).
6898
+ if (process.env.OPENGENI_TEST_SCENARIO === "sandbox") {
6899
+ console.log(
6900
+ `[sandbox-e2e] capture preflight ownership=${settings.sandboxOwnershipEnabled} enabled=${settings.workspaceCaptureEnabled} resolved=${Boolean(resolvedSandbox)} session=${Boolean(setupBoxSession)} group=${Boolean(sandboxGroupId)} storage=${Boolean(objectStorage)}`,
6901
+ );
5294
6902
  }
5295
- await captureWorkspaceRevision({
5296
- db,
5297
- objectStorage,
5298
- settings,
5299
- publish: async (events) => {
5300
- await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events);
5301
- },
5302
- session: setupBoxSession as ChannelASession,
5303
- leaseEpoch: resolvedSandbox.leaseEpoch,
5304
- sandboxGroupId,
5305
- accountId: input.accountId,
5306
- workspaceId: input.workspaceId,
5307
- sessionId: input.sessionId,
5308
- turnId: turnId ?? null,
5309
- observability,
6903
+ const runTurnEndPersistence = shouldRunTurnEndWorkspacePersistence({
6904
+ activityStatus,
6905
+ cancellationRequested: finalizerSignal?.aborted === true,
5310
6906
  });
5311
- }
5312
- await preparedTools?.close().catch(() => undefined);
5313
- if (heartbeatTimer) {
5314
- clearInterval(heartbeatTimer);
5315
- }
5316
- if (turnSandboxProvisioner?.hasStarted()) {
5317
- await turnSandboxProvisioner.waitForSettled(30_000);
5318
- }
5319
- // P1.2: stop the lease-TTL refresh, release the turn holder (idempotent
5320
- // delete-my-row; refcount-- and warm->draining if it hit 0 with no turns),
5321
- // and DROP the in-memory handle. Release NEVER stops the box — the reaper
5322
- // (P1.3) issues the provider stop() past the drain grace at refcount 0; the
5323
- // box rides the provider idle-timeout in the meantime. Best-effort: a
5324
- // release failure must never mask the turn's real outcome.
5325
- if (leaseHeartbeatTimer) {
5326
- clearInterval(leaseHeartbeatTimer);
5327
- }
5328
- // A recording normally closes inside the attempt-fenced turn settlement.
5329
- // Reaching finally with one still active means settlement threw, never ran,
5330
- // or lost ownership. Stop ffmpeg and mark only this exact attempt-owned row
5331
- // failed; publish no event and leave the artifact recoverable on the box.
5332
- await abandonActiveRecording(
5333
- "activity ended without recording settlement",
5334
- didComputerUse ? "failed" : "discard",
5335
- );
5336
- if (resolvedSandbox) {
5337
- // TURN-END mid-session snapshot (sandbox-file-persistence): fold the
5338
- // turn's finished /workspace onto the lease before releasing the holder,
5339
- // so the work this turn just produced survives any unclean box death in
5340
- // the idle window ahead. Throttled by the same interval as the heartbeat
5341
- // tick (a short turn right after a snapshot skips — bounded-loss contract
5342
- // is the interval, not per-turn). Best-effort and time-capped by the
5343
- // helper's own failure discipline; never delays release on failure.
5344
- if (setupBoxSession && sandboxGroupId) {
5345
- // Single-flight vs the heartbeat capture: the timer is already cleared
5346
- // above, but a capture it launched may still be in flight — and that
5347
- // capture predates the turn's final writes. Wait for it, but only up
5348
- // to the snapshot timeout: release must never depend on an unbounded
5349
- // provider capture.
6907
+ if (
6908
+ runTurnEndPersistence &&
6909
+ turnId &&
6910
+ resolvedSandbox &&
6911
+ setupBoxSession &&
6912
+ sandboxGroupId
6913
+ ) {
6914
+ // Stop new heartbeat snapshot/meter ticks so a mid-turn snapshot cannot
6915
+ // start concurrently with capture, then drain any in-flight snapshot
6916
+ // (bounded) — capture and the warm snapshot both exec on the box, so
6917
+ // sequence them, exactly as the turn-end snapshot placement did.
6918
+ if (leaseHeartbeatTimer) {
6919
+ clearInterval(leaseHeartbeatTimer);
6920
+ leaseHeartbeatTimer = undefined;
6921
+ }
5350
6922
  if (snapshotInFlight) {
5351
- await waitForWarmSnapshot(snapshotInFlight, settings.sandboxSnapshotTimeoutMs);
6923
+ await waitForWarmSnapshot(
6924
+ snapshotInFlight,
6925
+ settings.sandboxSnapshotTimeoutMs,
6926
+ finalizerSignal,
6927
+ );
5352
6928
  }
5353
- const persisted = await maybePersistWarmWorkspaceSnapshot(
5354
- { db, settings },
5355
- {
5356
- accountId: input.accountId,
5357
- workspaceId: input.workspaceId,
5358
- sandboxGroupId,
6929
+ await captureWorkspaceRevision({
6930
+ db,
6931
+ objectStorage,
6932
+ settings,
6933
+ publish: async (events) => {
6934
+ await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events);
5359
6935
  },
5360
- setupBoxSession,
5361
- resolvedSandbox.leaseEpoch,
6936
+ session: setupBoxSession as ChannelASession,
6937
+ leaseEpoch: resolvedSandbox.leaseEpoch,
6938
+ sandboxGroupId,
6939
+ accountId: input.accountId,
6940
+ workspaceId: input.workspaceId,
6941
+ sessionId: input.sessionId,
6942
+ turnId,
6943
+ attemptId: input.attemptId,
6944
+ observability,
6945
+ ...(finalizerSignal ? { signal: finalizerSignal } : {}),
6946
+ });
6947
+ }
6948
+ if (preparedTools) {
6949
+ await waitForTurnFinalizerStep(
6950
+ preparedTools.close().catch(() => undefined),
6951
+ finalizerSignal,
5362
6952
  );
5363
- if (persisted && publish) {
5364
- await publish([
6953
+ }
6954
+ if (heartbeatTimer) {
6955
+ clearInterval(heartbeatTimer);
6956
+ }
6957
+ if (turnSandboxProvisioner?.hasStarted()) {
6958
+ await waitForTurnFinalizerStep(
6959
+ turnSandboxProvisioner.waitForSettled(30_000),
6960
+ finalizerSignal,
6961
+ );
6962
+ }
6963
+ // P1.2: stop the lease-TTL refresh, release the turn holder (idempotent
6964
+ // delete-my-row; refcount-- and warm->draining if it hit 0 with no turns),
6965
+ // and DROP the in-memory handle. Release NEVER stops the box — the reaper
6966
+ // (P1.3) issues the provider stop() past the drain grace at refcount 0; the
6967
+ // box rides the provider idle-timeout in the meantime. Best-effort: a
6968
+ // release failure must never mask the turn's real outcome.
6969
+ if (leaseHeartbeatTimer) {
6970
+ clearInterval(leaseHeartbeatTimer);
6971
+ }
6972
+ // A recording normally closes inside the attempt-fenced turn settlement.
6973
+ // Reaching finally with one still active means settlement threw, never ran,
6974
+ // or lost ownership. Stop ffmpeg and mark only this exact attempt-owned row
6975
+ // failed; publish no event and leave the artifact recoverable on the box.
6976
+ await waitForTurnFinalizerStep(
6977
+ abandonActiveRecording(
6978
+ "activity ended without recording settlement",
6979
+ didComputerUse ? "failed" : "discard",
6980
+ ),
6981
+ finalizerSignal,
6982
+ );
6983
+ if (resolvedSandbox) {
6984
+ // TURN-END mid-session snapshot (sandbox-file-persistence): fold the
6985
+ // turn's finished /workspace onto the lease before releasing the holder,
6986
+ // so the work this turn just produced survives any unclean box death in
6987
+ // the idle window ahead. Throttled by the same interval as the heartbeat
6988
+ // tick (a short turn right after a snapshot skips — bounded-loss contract
6989
+ // is the interval, not per-turn). Best-effort and time-capped by the
6990
+ // helper's own failure discipline; never delays release on failure.
6991
+ const settledTurnId = turnId;
6992
+ if (runTurnEndPersistence && setupBoxSession && sandboxGroupId && settledTurnId) {
6993
+ // Single-flight vs the heartbeat capture: the timer is already cleared
6994
+ // above, but a capture it launched may still be in flight — and that
6995
+ // capture predates the turn's final writes. Wait for it, but only up
6996
+ // to the snapshot timeout: release must never depend on an unbounded
6997
+ // provider capture.
6998
+ if (snapshotInFlight) {
6999
+ await waitForWarmSnapshot(
7000
+ snapshotInFlight,
7001
+ settings.sandboxSnapshotTimeoutMs,
7002
+ finalizerSignal,
7003
+ );
7004
+ }
7005
+ const persisted = await maybePersistWarmWorkspaceSnapshot(
7006
+ { db, settings },
5365
7007
  {
5366
- type: "sandbox.box.snapshot",
5367
- payload: { trigger: "turn-end" },
7008
+ accountId: input.accountId,
7009
+ workspaceId: input.workspaceId,
7010
+ sessionId: input.sessionId,
7011
+ turnId: settledTurnId,
7012
+ attemptId: input.attemptId,
7013
+ sandboxGroupId,
5368
7014
  },
5369
- ]).catch(() => undefined);
7015
+ setupBoxSession,
7016
+ resolvedSandbox.leaseEpoch,
7017
+ finalizerSignal,
7018
+ );
7019
+ if (persisted && publish) {
7020
+ await publish([
7021
+ {
7022
+ type: "sandbox.box.snapshot",
7023
+ payload: { trigger: "turn-end" },
7024
+ },
7025
+ ]).catch(() => undefined);
7026
+ }
7027
+ // NB workspace capture no longer runs here — it moved to
7028
+ // the TOP of this finally (before preparedTools.close) so it completes
7029
+ // while the box is still solidly alive, instead of racing the turn-end
7030
+ // teardown that was killing 100% of captures on real Modal desktop boxes.
5370
7031
  }
5371
- // NB workspace capture (dossier §10.1) no longer runs here — it moved to
5372
- // the TOP of this finally (before preparedTools.close) so it completes
5373
- // while the box is still solidly alive, instead of racing the turn-end
5374
- // teardown that was killing 100% of captures on real Modal desktop boxes.
7032
+ const sandboxToRelease = resolvedSandbox;
7033
+ resolvedSandbox = null; // drop ownership now; the exact-holder release may finish later
7034
+ await waitForTurnFinalizerStep(
7035
+ sandboxToRelease.release().catch((releaseError) => {
7036
+ console.error("sandbox lease release failed (turn outcome unaffected)", releaseError);
7037
+ }),
7038
+ finalizerSignal,
7039
+ );
7040
+ }
7041
+ } catch (error) {
7042
+ finalizationError ??= error;
7043
+ console.error("agent turn finalization failed (turn outcome unaffected)", error);
7044
+ } finally {
7045
+ cancellationSignal?.removeEventListener("abort", noteCancellationRequested);
7046
+ const completedAt = performance.now();
7047
+ const durationSeconds = (completedAt - activityStarted) / 1000;
7048
+ const finalizationDurationSeconds = (completedAt - finalizationStarted) / 1000;
7049
+ observability.observeHistogram({
7050
+ name: "opengeni_turn_finalization_duration_seconds",
7051
+ help: "Agent turn finalization duration, including workspace housekeeping and lease release.",
7052
+ labels: {
7053
+ cancellation_requested: String(cancellationRequestedAt !== null),
7054
+ },
7055
+ value: finalizationDurationSeconds,
7056
+ });
7057
+ if (cancellationRequestedAt !== null) {
7058
+ const physicalCancellationDurationSeconds =
7059
+ (completedAt - cancellationRequestedAt) / 1000;
7060
+ observability.observeHistogram({
7061
+ name: "opengeni_turn_physical_cancellation_duration_seconds",
7062
+ help: "Time from Temporal cancellation delivery until the activity physically stops.",
7063
+ value: physicalCancellationDurationSeconds,
7064
+ });
7065
+ observability.info("agent turn physical cancellation completed", {
7066
+ "opengeni.session_id": input.sessionId,
7067
+ "opengeni.turn_id": turnId ?? "",
7068
+ "opengeni.attempt_id": input.attemptId,
7069
+ "opengeni.physical_cancellation_duration_ms": Math.round(
7070
+ physicalCancellationDurationSeconds * 1000,
7071
+ ),
7072
+ });
7073
+ }
7074
+ observability.recordWorkerActivity({
7075
+ activity: "runAgentTurn",
7076
+ status: finalizationError ? "cleanup_failed" : activityStatus,
7077
+ durationSeconds,
7078
+ });
7079
+ if (turnId && activityStatus !== "unknown") {
7080
+ turnLifecycleMetricsFor(observability).finish(turnId, turnMetricOutcome, durationSeconds);
5375
7081
  }
5376
- await resolvedSandbox.release().catch((releaseError) => {
5377
- console.error("sandbox lease release failed (turn outcome unaffected)", releaseError);
7082
+ activitySpan.end({
7083
+ attributes: {
7084
+ "opengeni.turn_id": turnId ?? "",
7085
+ "opengeni.status": activityStatus,
7086
+ "opengeni.variable_set_id": variableSetId,
7087
+ "opengeni.rig_id": rigId,
7088
+ "opengeni.rig_version_id": rigVersionId,
7089
+ "opengeni.codex_credential_id": effectiveCodexCredentialId ?? "",
7090
+ "opengeni.duration_ms": Math.round(durationSeconds * 1000),
7091
+ "opengeni.finalization_duration_ms": Math.round(finalizationDurationSeconds * 1000),
7092
+ },
7093
+ error: finalizationError ?? activityError,
7094
+ });
7095
+ assertPhysicalToolQuiescenceForCancellation({
7096
+ acknowledgeQuiescence,
7097
+ physicalToolQuiescenceConfirmed,
7098
+ failure: finalizationError,
7099
+ });
7100
+ assertSessionAttemptQuiescenceRecoveryDurable({
7101
+ acknowledgeQuiescence,
7102
+ physicalToolQuiescenceConfirmed,
7103
+ receiptOrProofDurable: quiescenceReceiptOrProofDurable,
7104
+ failure: finalizationError,
5378
7105
  });
5379
- resolvedSandbox = null; // drop the handle; the box survives the turn
5380
7106
  }
5381
7107
  }
5382
7108
  };
5383
7109
  }
5384
7110
 
7111
+ async function assertGitHubResourcesRemainAuthorized(
7112
+ db: Parameters<typeof areGitHubRepositoriesAllowedForWorkspace>[0],
7113
+ workspaceId: string,
7114
+ resources: import("@opengeni/contracts").ResourceRef[],
7115
+ ): Promise<void> {
7116
+ // Must check exactly what sandboxEnvironmentForRun would mint a token for,
7117
+ // so the selection is derived from the same extraction as the mint path.
7118
+ for (const selection of gitHubTokenMintSelections(resources)) {
7119
+ await assertGitHubTokenMintSelectionAuthorized(
7120
+ db,
7121
+ workspaceId,
7122
+ selection.installationId,
7123
+ selection.repositoryIds,
7124
+ );
7125
+ }
7126
+ }
7127
+
7128
+ async function assertGitHubTokenMintSelectionAuthorized(
7129
+ db: Parameters<typeof areGitHubRepositoriesAllowedForWorkspace>[0],
7130
+ workspaceId: string,
7131
+ installationId: number,
7132
+ repositoryIds: number[],
7133
+ ): Promise<void> {
7134
+ if (
7135
+ !(await areGitHubRepositoriesAllowedForWorkspace(
7136
+ db,
7137
+ workspaceId,
7138
+ installationId,
7139
+ repositoryIds,
7140
+ ))
7141
+ ) {
7142
+ throw new Error(
7143
+ "This workspace no longer authorizes one or more GitHub repositories attached to the session",
7144
+ );
7145
+ }
7146
+ }
7147
+
5385
7148
  /**
5386
7149
  * True when the error is transient upstream backpressure — a model-provider 5xx,
5387
7150
  * a "server had a bad minute" body, or a dropped/again-able network connection —
@@ -5428,11 +7191,23 @@ export function isTransientProviderError(error: unknown): boolean {
5428
7191
  );
5429
7192
  }
5430
7193
 
5431
- export function agentRunFailurePayload(error: unknown): {
7194
+ export function agentRunFailurePayload(
7195
+ error: unknown,
7196
+ options: { isCodexTurn?: boolean } = {},
7197
+ ): {
5432
7198
  error: string;
5433
7199
  code?: string;
5434
7200
  retryable?: boolean;
5435
7201
  detail?: string;
7202
+ timeoutClass?: string;
7203
+ responseObserved?: boolean;
7204
+ requestId?: string;
7205
+ correlationId?: string;
7206
+ stage?: string;
7207
+ sqlState?: string | null;
7208
+ attempts?: number;
7209
+ retryOutcome?: string;
7210
+ database?: Record<string, string>;
5436
7211
  } {
5437
7212
  const message = error instanceof Error ? error.message : String(error);
5438
7213
  const status =
@@ -5443,6 +7218,41 @@ export function agentRunFailurePayload(error: unknown): {
5443
7218
  typeof error === "object" && error !== null && "code" in error
5444
7219
  ? String((error as { code?: unknown }).code)
5445
7220
  : undefined;
7221
+ // An accepted Codex stream with no terminal response is malformed/partial,
7222
+ // not provider backpressure. Replaying the same accepted turn could repeat
7223
+ // model or tool effects, so this marked transport failure must outrank the
7224
+ // generic 5xx retry classifier (CodexStreamingTerminalError uses status 502).
7225
+ if (isCodexTransportError(error) && code === "invalid_sse_terminal") {
7226
+ return {
7227
+ error: "The Codex response stream ended without a terminal response",
7228
+ code: "invalid_sse_terminal",
7229
+ retryable: false,
7230
+ };
7231
+ }
7232
+ if (isSessionEventPersistenceError(error)) {
7233
+ const { details } = error;
7234
+ const eventLabel = details.eventTypes.join(", ") || "session events";
7235
+ const failureLabel =
7236
+ details.code === "db_deadlock"
7237
+ ? "Database deadlock"
7238
+ : details.code === "db_serialization_failure"
7239
+ ? "Database serialization failure"
7240
+ : "Database failure";
7241
+ return {
7242
+ error: `${failureLabel} while persisting ${eventLabel}. The completed provider call and external effects were not retried.`,
7243
+ code: details.code,
7244
+ detail:
7245
+ details.retryOutcome === "exhausted"
7246
+ ? `The idempotent persistence transaction failed after ${details.attempts} attempts.`
7247
+ : "The database rejected the idempotent persistence transaction.",
7248
+ correlationId: details.correlationId,
7249
+ stage: details.stage,
7250
+ sqlState: details.sqlState,
7251
+ attempts: details.attempts,
7252
+ retryOutcome: details.retryOutcome,
7253
+ ...(Object.keys(details.database).length > 0 ? { database: details.database } : {}),
7254
+ };
7255
+ }
5446
7256
  // A ChatGPT/Codex usage cap is a HARD limit, not transient backpressure: it
5447
7257
  // must NOT be reported as a generic, retryable rate-limit (which would loop a
5448
7258
  // goal against a capped backend). Surface a precise, actionable message with
@@ -5457,6 +7267,22 @@ export function agentRunFailurePayload(error: unknown): {
5457
7267
  if (usageLimit) {
5458
7268
  return codexUsageLimitFailurePayload(usageLimit, message);
5459
7269
  }
7270
+ const codexTimeout = classifyCodexResponseTimeoutError(error, {
7271
+ allowLegacyRequestTimeout: options.isCodexTurn === true,
7272
+ });
7273
+ if (codexTimeout) {
7274
+ return {
7275
+ error: codexTimeout.responseObserved
7276
+ ? "The Codex response timed out after streaming began. Observed output was checkpointed; automatic replay is disabled because the upstream operation may still be active."
7277
+ : "The Codex response timed out before any response was observed. Upstream acceptance is unknown, so automatic replay is disabled.",
7278
+ code: "codex_response_timeout",
7279
+ retryable: false,
7280
+ timeoutClass: codexTimeout.timeoutClass,
7281
+ responseObserved: codexTimeout.responseObserved,
7282
+ ...(codexTimeout.requestId ? { requestId: codexTimeout.requestId } : {}),
7283
+ ...(codexTimeout.message ? { detail: codexTimeout.message } : {}),
7284
+ };
7285
+ }
5460
7286
  const mcpTimeout = classifyMcpTransportTimeoutError(error);
5461
7287
  if (mcpTimeout) {
5462
7288
  return {
@@ -5511,7 +7337,6 @@ export function codexCredentialCooldownUntil(
5511
7337
  CodexAccountStatus,
5512
7338
  "primaryUsedPercent" | "primaryResetAt" | "secondaryUsedPercent" | "secondaryResetAt"
5513
7339
  > | null,
5514
- nearExhaustionPct: number,
5515
7340
  now: Date,
5516
7341
  ): Date | null {
5517
7342
  if (failure.kind === "auth" || failure.kind === "forbidden") {
@@ -5533,7 +7358,7 @@ export function codexCredentialCooldownUntil(
5533
7358
  ]
5534
7359
  .filter(
5535
7360
  (window): window is { used: number; reset: Date } =>
5536
- (window.used ?? 0) >= nearExhaustionPct &&
7361
+ (window.used ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT &&
5537
7362
  window.reset instanceof Date &&
5538
7363
  window.reset.getTime() > now.getTime(),
5539
7364
  )
@@ -5737,21 +7562,21 @@ class BudgetExhaustedError extends Error {
5737
7562
  }
5738
7563
  }
5739
7564
 
5740
- // Exported for unit testing the codex-billed bypass (codex-billing.test.ts); not part
5741
- // of the activity surface. Takes BOTH `isCodexTurn` (codex-plan turns bypass the credit
5742
- // and token gates) and the optional §7.5 P3 host `entitlements` port (when bound, its
5743
- // `admitRun` REPLACES the local credit read for a non-codex turn; unset → local ledger).
7565
+ // Exported for unit testing the external-billing bypass (codex-billing.test.ts); not
7566
+ // part of the activity surface. Takes the accepted policy's billing attribution and
7567
+ // the optional §7.5 P3 host `entitlements` port (when bound, its `admitRun` REPLACES
7568
+ // the local credit read for an OpenGeni-metered turn; unset → local ledger).
5744
7569
  export async function ensureRunAllowed(
5745
7570
  settings: Settings,
5746
7571
  db: ActivityServices["db"],
5747
7572
  accountId: string,
5748
7573
  workspaceId: string,
5749
- isCodexTurn: boolean,
7574
+ isExternallyBilledTurn: boolean,
5750
7575
  entitlements?: ActivityServices["entitlements"],
5751
7576
  ): Promise<void> {
5752
- // Codex-billed turns are paid by the user's ChatGPT/Codex plan: skip the
5753
- // credit-balance gate and the monthly token cap. The agent-run COUNT cap below
5754
- // is a volume/fairness quota (not a credit/cost gate) and is intentionally kept.
7577
+ // Externally billed turns are paid outside OpenGeni: skip the credit-balance
7578
+ // gate and monthly token cap. The agent-run COUNT cap below is a
7579
+ // volume/fairness quota (not a credit/cost gate) and is intentionally kept.
5755
7580
  //
5756
7581
  // §7.5 P3 — host-entitlements DELEGATION (the worker half of the same seam the
5757
7582
  // API edge exposes). For a non-codex turn, when the host binds `entitlements`, its
@@ -5765,7 +7590,7 @@ export async function ensureRunAllowed(
5765
7590
  // idempotency-keyed writer at recordModelUsageAndDebitCredits), so a PULL host meter
5766
7591
  // is consulted without ever double-charging.
5767
7592
  if (
5768
- !isCodexTurn &&
7593
+ !isExternallyBilledTurn &&
5769
7594
  entitlements &&
5770
7595
  (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")
5771
7596
  ) {
@@ -5779,7 +7604,7 @@ export async function ensureRunAllowed(
5779
7604
  throw new Error(decision.reason || "insufficient OpenGeni credits");
5780
7605
  }
5781
7606
  } else if (
5782
- !isCodexTurn &&
7607
+ !isExternallyBilledTurn &&
5783
7608
  (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")
5784
7609
  ) {
5785
7610
  const balance = await getBillingBalance(db, accountId);
@@ -5804,7 +7629,7 @@ export async function ensureRunAllowed(
5804
7629
  );
5805
7630
  }
5806
7631
  }
5807
- if (!isCodexTurn && limits.maxMonthlyTokensPerWorkspace) {
7632
+ if (!isExternallyBilledTurn && limits.maxMonthlyTokensPerWorkspace) {
5808
7633
  const used = await sumUsageQuantity(db, {
5809
7634
  workspaceId,
5810
7635
  eventType: "model.tokens",
@@ -5817,7 +7642,7 @@ export async function ensureRunAllowed(
5817
7642
  }
5818
7643
  }
5819
7644
 
5820
- // Exported for unit testing the codex-billed bypass; not part of the activity surface.
7645
+ // Exported for unit testing the external-billing bypass; not part of the activity surface.
5821
7646
  export async function recordModelUsageAndDebitCredits(
5822
7647
  settings: Settings,
5823
7648
  db: ActivityServices["db"],
@@ -5826,9 +7651,11 @@ export async function recordModelUsageAndDebitCredits(
5826
7651
  workspaceId: string;
5827
7652
  sessionId: string;
5828
7653
  turnId: string;
7654
+ turnAttemptId: string;
5829
7655
  model: string;
5830
- isCodexTurn: boolean;
5831
- usage?: ModelUsageInput | null;
7656
+ externallyBilled: boolean;
7657
+ usage?: ModelUsageInput | ModelCallUsageInput | null;
7658
+ normalizedUsage?: ModelCallUsageNormalization;
5832
7659
  sourceKey: string;
5833
7660
  observability?: ActivityServices["observability"];
5834
7661
  },
@@ -5836,10 +7663,12 @@ export async function recordModelUsageAndDebitCredits(
5836
7663
  if (!input.usage) {
5837
7664
  return;
5838
7665
  }
5839
- const inputTokens = positiveInt(input.usage.inputTokens);
5840
- const outputTokens = positiveInt(input.usage.outputTokens);
5841
- const totalTokens = positiveInt(input.usage.totalTokens) || inputTokens + outputTokens;
5842
- // A codex-subscription turn is paid by the user's ChatGPT/Codex plan, so it
7666
+ const normalizedUsage = input.normalizedUsage ?? normalizeModelCallUsage(input.usage);
7667
+ const sanitizedUsage = sanitizedModelUsageInput(normalizedUsage);
7668
+ const inputTokens = sanitizedUsage.inputTokens ?? 0;
7669
+ const outputTokens = sanitizedUsage.outputTokens ?? 0;
7670
+ const totalTokens = sanitizedUsage.totalTokens ?? 0;
7671
+ // An externally billed turn is paid outside OpenGeni, so it
5843
7672
  // consumes ZERO OpenGeni credits and must never feed an OpenGeni cap. A
5844
7673
  // codex/<slug> model has no entry in configuredModelPricing, so the normal path
5845
7674
  // below would throw "Missing model pricing". We:
@@ -5848,7 +7677,7 @@ export async function recordModelUsageAndDebitCredits(
5848
7677
  // any row would count against maxMonthlyTokensPerWorkspace);
5849
7678
  // - record a `model.cost = 0` audit marker (harmless to the monthly cost cap);
5850
7679
  // - never look up pricing and never debit credits.
5851
- if (input.isCodexTurn) {
7680
+ if (input.externallyBilled) {
5852
7681
  await recordUsageEvent(db, {
5853
7682
  accountId: input.accountId,
5854
7683
  workspaceId: input.workspaceId,
@@ -5857,6 +7686,9 @@ export async function recordModelUsageAndDebitCredits(
5857
7686
  unit: "usd_micros",
5858
7687
  sourceResourceType: "model_response",
5859
7688
  sourceResourceId: `${input.turnId}:${input.sourceKey}`,
7689
+ sessionId: input.sessionId,
7690
+ turnId: input.turnId,
7691
+ turnAttemptId: input.turnAttemptId,
5860
7692
  idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`,
5861
7693
  });
5862
7694
  return;
@@ -5870,6 +7702,9 @@ export async function recordModelUsageAndDebitCredits(
5870
7702
  unit: "tokens",
5871
7703
  sourceResourceType: "model_response",
5872
7704
  sourceResourceId: `${input.turnId}:${input.sourceKey}`,
7705
+ sessionId: input.sessionId,
7706
+ turnId: input.turnId,
7707
+ turnAttemptId: input.turnAttemptId,
5873
7708
  idempotencyKey: `usage:model.tokens:${input.turnId}:${input.sourceKey}`,
5874
7709
  });
5875
7710
  }
@@ -5880,7 +7715,7 @@ export async function recordModelUsageAndDebitCredits(
5880
7715
  if (!configuredModelPricing(settings)[input.model]) {
5881
7716
  throw new Error(`Missing model pricing for ${input.model}`);
5882
7717
  }
5883
- const costMicros = calculateModelUsageCostMicros(settings, input.model, input.usage);
7718
+ const costMicros = calculateModelUsageCostMicros(settings, input.model, sanitizedUsage);
5884
7719
  await recordUsageEvent(db, {
5885
7720
  accountId: input.accountId,
5886
7721
  workspaceId: input.workspaceId,
@@ -5889,6 +7724,9 @@ export async function recordModelUsageAndDebitCredits(
5889
7724
  unit: "usd_micros",
5890
7725
  sourceResourceType: "model_response",
5891
7726
  sourceResourceId: `${input.turnId}:${input.sourceKey}`,
7727
+ sessionId: input.sessionId,
7728
+ turnId: input.turnId,
7729
+ turnAttemptId: input.turnAttemptId,
5892
7730
  idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`,
5893
7731
  });
5894
7732
  if (costMicros > 0) {
@@ -5911,18 +7749,26 @@ export async function recordModelUsageAndDebitCredits(
5911
7749
  // Additive: the prompt-cache slice of this call's input tokens, so the
5912
7750
  // per-call debit record carries cache efficiency alongside the token
5913
7751
  // counts. 0 when the provider did not report cached tokens.
5914
- cachedTokens: positiveInt(
5915
- modelCallUsageTelemetry(input.usage as Parameters<typeof modelCallUsageTelemetry>[0])
5916
- .cachedTokens,
5917
- ),
7752
+ cachedTokens: normalizedUsage.telemetry.cachedTokens ?? 0,
5918
7753
  },
5919
7754
  });
5920
7755
  recordCreditMicros(input.observability, "usage", result.debitedMicros);
5921
7756
  }
5922
7757
  }
5923
7758
 
5924
- function positiveInt(value: unknown): number {
5925
- return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
7759
+ function sanitizedModelUsageInput(normalized: ModelCallUsageNormalization): ModelUsageInput {
7760
+ return {
7761
+ ...(normalized.telemetry.inputTokens !== null
7762
+ ? { inputTokens: normalized.telemetry.inputTokens }
7763
+ : {}),
7764
+ ...(normalized.telemetry.outputTokens !== null
7765
+ ? { outputTokens: normalized.telemetry.outputTokens }
7766
+ : {}),
7767
+ ...(normalized.totalTokens !== null ? { totalTokens: normalized.totalTokens } : {}),
7768
+ ...(normalized.telemetry.cachedTokens !== null
7769
+ ? { inputTokensDetails: { cached_tokens: normalized.telemetry.cachedTokens } }
7770
+ : {}),
7771
+ };
5926
7772
  }
5927
7773
 
5928
7774
  function startOfUtcMonth(): Date {