@opengeni/worker-bundle 0.7.5 → 0.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/chunk-BJUW5LCM.js +325 -0
  2. package/dist/chunk-BJUW5LCM.js.map +1 -0
  3. package/dist/host-export.d.ts +70 -0
  4. package/dist/host-export.js +7 -0
  5. package/dist/host-export.js.map +1 -0
  6. package/dist/index.d.ts +98 -6
  7. package/dist/index.js +2812 -813
  8. package/dist/index.js.map +1 -1
  9. package/dist/workflow-bundle.js +12358 -0
  10. package/package.json +18 -12
  11. package/src/activities/agent-turn.ts +2045 -545
  12. package/src/activities/capabilities.ts +9 -2
  13. package/src/activities/codex-capacity.ts +6 -11
  14. package/src/activities/codex-rotation.ts +75 -86
  15. package/src/activities/context-compaction.ts +104 -24
  16. package/src/activities/environment.ts +367 -92
  17. package/src/activities/expiring-material-renewal.ts +143 -0
  18. package/src/activities/goals.ts +4 -0
  19. package/src/activities/mcp-credentials.ts +38 -0
  20. package/src/activities/packs.ts +115 -2
  21. package/src/activities/run-credential-renewal.ts +70 -0
  22. package/src/activities/run-credentials.ts +132 -0
  23. package/src/activities/run-input.ts +23 -0
  24. package/src/activities/sandbox-lease.ts +1 -1
  25. package/src/activities/scheduled-tasks.ts +21 -0
  26. package/src/activities/session-state.ts +72 -0
  27. package/src/activities/streaming.ts +1 -0
  28. package/src/activities/toolspace-token-renewal.ts +57 -0
  29. package/src/activities/types.ts +44 -6
  30. package/src/activities/workspace-capture.ts +113 -93
  31. package/src/activities.ts +10 -2
  32. package/src/host-export-pump.ts +416 -0
  33. package/src/host-export.ts +22 -0
  34. package/src/http.ts +77 -28
  35. package/src/index.ts +360 -156
  36. package/src/sandbox-resume.ts +47 -1
  37. package/src/sandbox-routing.ts +1 -1
  38. package/src/worker-service-lifecycle.ts +104 -0
  39. package/src/workflows/activities.ts +13 -7
  40. package/src/workflows/session.ts +219 -23
  41. package/src/workflows.ts +1 -0
@@ -11,8 +11,11 @@ import {
11
11
  setActiveSandbox,
12
12
  requireFile,
13
13
  getSessionEvent,
14
+ getSessionRootId,
14
15
  getSessionGoal,
15
16
  getLatestRunState,
17
+ getHumanInputResumeForEvent,
18
+ getSessionHumanInputRequest,
16
19
  isCodexBilledTurn,
17
20
  workspaceCodexSubscriptionActive,
18
21
  acquireCodexCredentialLease,
@@ -47,7 +50,6 @@ import {
47
50
  nextSessionHistoryPosition,
48
51
  settleCodexCredentialLeaseLoss,
49
52
  settleCodexCredentialFailover,
50
- saveRunState,
51
53
  upsertSandboxSessionEnvelope,
52
54
  setSessionLastInputTokensForTurnAttempt,
53
55
  sumUsageQuantity,
@@ -55,16 +57,19 @@ import {
55
57
  accrueWarmSeconds,
56
58
  getMaterializedSandboxFileResources,
57
59
  markSandboxFileResourcesMaterialized,
60
+ areGitHubRepositoriesAllowedForWorkspace,
58
61
  SandboxLeaseSupersededError,
59
62
  SandboxImageConflictError,
60
- buildConnectionTokenResolver,
63
+ isSessionEventPersistenceError,
61
64
  getEnrollment,
62
65
  abandonRecordingForTurnAttempt,
66
+ markSessionAttemptQuiesced,
63
67
  type AppendEventInput,
64
68
  type ActiveSandboxPointer,
65
69
  type SandboxRecord,
66
70
  type CodexCredentialLeaseResult,
67
71
  type CodexCredentialLeaseSelectionContext,
72
+ type ApplySessionTurnSettlementInput,
68
73
  type SessionTurnRecordingSettlement,
69
74
  } from "@opengeni/db";
70
75
  import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@opengeni/events";
@@ -81,10 +86,18 @@ import {
81
86
  appendWorkspaceMemory,
82
87
  composeAgentInstructions,
83
88
  summarizeForCompaction,
89
+ CompactionProviderResponseError,
90
+ EmptyCompactionSummaryError,
84
91
  ensureModalRegistryImage,
85
92
  findCompactionNeededError,
86
93
  materializeSandboxFileDownloads,
87
- refreshGitProviderTokenFiles,
94
+ materializeRunCredentials,
95
+ clearRunCredentials,
96
+ clearRunCredentialsForAttempt,
97
+ withRunCredentialsSession,
98
+ refreshGitCredentialBindingTokenFiles,
99
+ refreshToolspaceTokenFile,
100
+ toolspaceTokenFileFromEnvironment,
88
101
  sandboxFileDownloadFailureNote,
89
102
  SUMMARY_BUFFER_TOKENS,
90
103
  runOwnedSandboxSetup,
@@ -95,12 +108,17 @@ import {
95
108
  type ComputerToolMode,
96
109
  type ModelResponseUsage,
97
110
  type BuildAgentOptions,
111
+ type TurnToolCancellationFence,
98
112
  type BackendUnresolvableCode,
99
113
  type EstablishedSandboxSession,
100
114
  type GitCredentialTokenWriterSession,
115
+ type NormalizedRunCredentialMaterial,
116
+ type RunCredentialCommandSession,
117
+ type ToolspaceTokenWriterSession,
101
118
  deleteRecordingArtifacts,
102
119
  stopRecording as stopRecordingOnBox,
103
120
  } from "@opengeni/runtime";
121
+ import { connectionTokenResolverForTurn } from "./mcp-credentials";
104
122
  import {
105
123
  builtinProviderId,
106
124
  calculateModelUsageCostMicros,
@@ -120,6 +138,7 @@ import {
120
138
  settingsWithSessionMcpServersForRun,
121
139
  } from "./capabilities";
122
140
  import {
141
+ CODEX_USAGE_EXHAUSTED_PCT,
123
142
  authoritativeCodexCapacityResetAt,
124
143
  chooseRotationActive,
125
144
  classifyCodexPin,
@@ -154,25 +173,46 @@ import { mergeResourceRefs, mergeToolRefs } from "./common";
154
173
  import { maybeCompactContext } from "./context-compaction";
155
174
  import { TurnAttemptFencedError } from "./turn-attempt-fenced";
156
175
  import {
176
+ gitCredentialAuthorityForTurn,
177
+ gitHubTokenMintSelections,
157
178
  loadWorkspaceEnvironmentForRunWithCredentials,
179
+ mintSandboxToolspaceToken,
158
180
  mintRunGitCredentials,
181
+ mintRunGitCredentialBinding,
159
182
  sandboxEnvironmentForRun,
183
+ type GitHubTokenMintAuthorization,
160
184
  type MintedRunGitCredentials,
161
185
  } from "./environment";
162
186
  import {
163
187
  startGitCredentialRenewalLoop,
164
188
  type GitCredentialRenewalController,
165
189
  } from "./git-credential-renewal";
190
+ import {
191
+ RUN_CREDENTIAL_EXPIRY_LEAD_MS,
192
+ startRunCredentialRenewalLoop,
193
+ type RunCredentialRenewalController,
194
+ } from "./run-credential-renewal";
195
+ import {
196
+ TOOLSPACE_TOKEN_EXPIRY_LEAD_MS,
197
+ startToolspaceTokenRenewalLoop,
198
+ type ToolspaceTokenRenewalController,
199
+ } from "./toolspace-token-renewal";
200
+ import {
201
+ bindRunCredentialResolver,
202
+ runCredentialAuthNeededPayloads,
203
+ runCredentialModelNote,
204
+ } from "./run-credentials";
166
205
  import { withCodexAppsTool, withFirstPartyTools } from "./goals";
167
206
  import {
168
207
  mergeRigDefaultVariableSetEnvironment,
169
208
  resolveWorkspaceAgentInstructions,
170
209
  resolveWorkspacePackRuntime,
210
+ resolveWorkspaceSkillLibraryRuntime,
171
211
  settingsWithPackSandboxImage,
172
212
  settingsWithRigImage,
173
213
  } from "./packs";
174
214
  import { deliverFailedChildTurnToParent } from "./parent-wake";
175
- import { createSecretRedactor, identityRedactor } from "./redaction";
215
+ import { createSecretRedactor, identityRedactor, type SecretForRedaction } from "./redaction";
176
216
  import { applyCodexHistoryStrip, turnInput, type TurnCodexAccount } from "./run-input";
177
217
  import {
178
218
  createRuntimeBatcher,
@@ -180,7 +220,12 @@ import {
180
220
  nextStreamEvent,
181
221
  startActivityHeartbeat,
182
222
  } from "./streaming";
183
- import type { ActivityServices, RunAgentTurnInput, RunAgentTurnResult } from "./types";
223
+ import type {
224
+ ActivityServices,
225
+ RunAgentTurnInput,
226
+ RunAgentTurnResult,
227
+ SessionAttemptQuiescenceProof,
228
+ } from "./types";
184
229
  import {
185
230
  resumeBoxForTurn,
186
231
  acquireSelfhostedLeaseForTurn,
@@ -229,7 +274,6 @@ import {
229
274
  import {
230
275
  CAPABILITY_DESCRIPTORS,
231
276
  evaluateWorkspaceModelPolicy,
232
- type GitCredentialProvider,
233
277
  type ResourceRef,
234
278
  type SessionEvent,
235
279
  type SessionEventType,
@@ -293,6 +337,28 @@ export function codexWorkspaceMetricKey(workspaceId: string): string {
293
337
  return createHash("sha256").update(workspaceId).digest("hex").slice(0, 12);
294
338
  }
295
339
 
340
+ /** Stable public request identity across partial resumes and activity retries. */
341
+ export function stableHumanInputRequestId(
342
+ sessionId: string,
343
+ turnId: string,
344
+ toolCallId: string,
345
+ ): string {
346
+ const hex = createHash("sha256")
347
+ .update("opengeni-human-input-v1\0")
348
+ .update(sessionId)
349
+ .update("\0")
350
+ .update(turnId)
351
+ .update("\0")
352
+ .update(toolCallId)
353
+ .digest("hex")
354
+ .slice(0, 32)
355
+ .split("");
356
+ hex[12] = "5";
357
+ hex[16] = ["8", "9", "a", "b"][Number.parseInt(hex[16] ?? "0", 16) % 4] ?? "8";
358
+ const value = hex.join("");
359
+ return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice(12, 16)}-${value.slice(16, 20)}-${value.slice(20)}`;
360
+ }
361
+
296
362
  /**
297
363
  * True when this activity attempt was cancelled because its hosting worker is
298
364
  * shutting down gracefully (SIGTERM during a deploy), as opposed to a
@@ -302,12 +368,229 @@ export function isWorkerShutdownCancellation(error: unknown): boolean {
302
368
  return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
303
369
  }
304
370
 
371
+ /**
372
+ * Review captures and protective snapshots are cache/persistence housekeeping,
373
+ * never part of cancellation correctness. A control-fenced or Temporal-cancelled
374
+ * attempt must release its physical activity promptly so Steer/Pause can advance.
375
+ */
376
+ export function shouldRunTurnEndWorkspacePersistence(input: {
377
+ activityStatus: RunAgentTurnResult["status"] | "unknown";
378
+ cancellationRequested: boolean;
379
+ }): boolean {
380
+ return input.activityStatus !== "cancelled" && !input.cancellationRequested;
381
+ }
382
+
383
+ /**
384
+ * Temporal cancellation is delivery/transport state, never proof that the
385
+ * dying activity crossed its mandatory sandbox-tool fence. If that fence
386
+ * fails, surface the fence failure instead of retaining a misleading typed
387
+ * cancellation; replacement admission remains closed because no durable
388
+ * quiescence receipt was written.
389
+ */
390
+ export function assertPhysicalToolQuiescenceForCancellation(input: {
391
+ acknowledgeQuiescence: boolean;
392
+ physicalToolQuiescenceConfirmed: boolean;
393
+ failure: unknown;
394
+ }): void {
395
+ if (!input.acknowledgeQuiescence || input.physicalToolQuiescenceConfirmed) return;
396
+ if (input.failure instanceof Error) throw input.failure;
397
+ throw new Error("Physical sandbox-tool quiescence could not be confirmed", {
398
+ cause: input.failure,
399
+ });
400
+ }
401
+
402
+ /** A physically drained attempt must leave one durable recovery producer. The
403
+ * direct Postgres receipt is preferred; after its bounded retries exhaust, an
404
+ * exact Temporal proof signal is sufficient because the workflow persists it
405
+ * through an independently retrying DB-only control activity. */
406
+ export function assertSessionAttemptQuiescenceRecoveryDurable(input: {
407
+ acknowledgeQuiescence: boolean;
408
+ physicalToolQuiescenceConfirmed: boolean;
409
+ receiptOrProofDurable: boolean;
410
+ failure: unknown;
411
+ }): void {
412
+ if (
413
+ !input.acknowledgeQuiescence ||
414
+ !input.physicalToolQuiescenceConfirmed ||
415
+ input.receiptOrProofDurable
416
+ ) {
417
+ return;
418
+ }
419
+ if (input.failure instanceof Error) throw input.failure;
420
+ throw new Error("Physical quiescence had no durable receipt or recovery proof", {
421
+ cause: input.failure,
422
+ });
423
+ }
424
+
425
+ const QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS = 250;
426
+ const QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS = 5_000;
427
+
428
+ /** Persist the authoritative receipt or durably hand the exact physical proof
429
+ * to Temporal. This retries signal delivery, not DB eligibility or workflow
430
+ * state. The proof object never changes between attempts. */
431
+ export async function persistOrSignalSessionAttemptQuiescence(input: {
432
+ proof: SessionAttemptQuiescenceProof;
433
+ persistReceipt: () => Promise<SessionEvent[]>;
434
+ publishEvents: (events: SessionEvent[]) => Promise<unknown>;
435
+ signalProof: ActivityServices["signalSessionAttemptQuiesced"];
436
+ sleep?: (ms: number) => Promise<void>;
437
+ heartbeat?: (attempt: number, delayMs: number) => void;
438
+ onReceiptFailure?: (error: unknown) => void;
439
+ onPublishFailure?: (error: unknown) => void;
440
+ onSignalFailure?: (error: unknown, attempt: number, delayMs: number) => void;
441
+ }): Promise<"receipt" | "signal"> {
442
+ let events: SessionEvent[];
443
+ try {
444
+ events = await input.persistReceipt();
445
+ } catch (receiptError) {
446
+ input.onReceiptFailure?.(receiptError);
447
+ if (!input.signalProof) {
448
+ throw new Error("Session-attempt quiescence proof signaler is unavailable", {
449
+ cause: receiptError,
450
+ });
451
+ }
452
+ const delay = input.sleep ?? sleep;
453
+ let retryMs = QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS;
454
+ let attempt = 1;
455
+ for (;;) {
456
+ try {
457
+ await input.signalProof(input.proof);
458
+ return "signal";
459
+ } catch (signalError) {
460
+ input.onSignalFailure?.(signalError, attempt, retryMs);
461
+ try {
462
+ input.heartbeat?.(attempt, retryMs);
463
+ } catch {
464
+ // Heartbeat telemetry is not proof delivery and cannot replace or
465
+ // interrupt the exact signal retry loop.
466
+ }
467
+ await delay(retryMs);
468
+ retryMs = Math.min(retryMs * 2, QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS);
469
+ attempt += 1;
470
+ }
471
+ }
472
+ }
473
+
474
+ try {
475
+ await input.publishEvents(events);
476
+ } catch (publishError) {
477
+ // Postgres already committed quiesced_at, the queue event, and the wake.
478
+ // NATS is live fanout only; never misclassify its failure as receipt loss.
479
+ input.onPublishFailure?.(publishError);
480
+ }
481
+ return "receipt";
482
+ }
483
+
484
+ /**
485
+ * Cross the non-detachable physical boundary for one dying attempt. Temporal's
486
+ * cancellation signal is intentionally absent: cancellation is transport, not
487
+ * proof that sandbox tools and attempt-owned credential writers have drained.
488
+ * A replacement may be admitted only after this function resolves and the
489
+ * caller durably persists or signals the exact quiescence receipt.
490
+ */
491
+ export async function drainAttemptOwnedSandboxWriters(input: {
492
+ toolCancellationFence: Pick<TurnToolCancellationFence, "cancel" | "waitForQuiescence"> | null;
493
+ cancellationReason?: unknown;
494
+ toolspaceTokenRenewal: Pick<ToolspaceTokenRenewalController, "stop"> | null;
495
+ runCredentialRenewal: Pick<RunCredentialRenewalController, "stop"> | null;
496
+ }): Promise<void> {
497
+ if (input.toolCancellationFence) {
498
+ input.toolCancellationFence.cancel(
499
+ input.cancellationReason ?? new Error("TURN_ATTEMPT_FENCED"),
500
+ );
501
+ await input.toolCancellationFence.waitForQuiescence();
502
+ }
503
+ await input.toolspaceTokenRenewal?.stop();
504
+ await input.runCredentialRenewal?.stop();
505
+ }
506
+
507
+ /**
508
+ * Await a finalizer operation only while this Temporal activity still owns its
509
+ * execution window. Once Pause/Steer cancellation arrives, the operation keeps
510
+ * its own rejection handler and may finish its idempotent, attempt-scoped
511
+ * cleanup in the background, but it cannot pin activity terminalization or
512
+ * delay the separately receipt-gated replacement dispatch.
513
+ */
514
+ export async function waitForTurnFinalizerStep<T>(
515
+ operation: Promise<T>,
516
+ signal: AbortSignal | undefined,
517
+ ): Promise<T | undefined> {
518
+ if (!signal) return await operation;
519
+ if (signal.aborted) {
520
+ void operation.catch(() => undefined);
521
+ return undefined;
522
+ }
523
+
524
+ let resolveCancellation: (() => void) | undefined;
525
+ const cancelled = new Promise<undefined>((resolve) => {
526
+ resolveCancellation = () => resolve(undefined);
527
+ });
528
+ const cancel = (): void => {
529
+ void operation.catch(() => undefined);
530
+ resolveCancellation?.();
531
+ };
532
+ signal.addEventListener("abort", cancel, { once: true });
533
+ try {
534
+ return await Promise.race([operation, cancelled]);
535
+ } finally {
536
+ signal.removeEventListener("abort", cancel);
537
+ }
538
+ }
539
+
540
+ /**
541
+ * Flush provider-facing stream state while the attempt still owns its activity
542
+ * window. On Pause/Steer, both promises are detached with rejection handlers:
543
+ * neither a runtime batcher nor an uncooperative provider completion promise
544
+ * may pin the activity behind cancellation after durable writes are fenced.
545
+ */
546
+ export async function waitForTurnStreamCleanup(
547
+ batcherFlush: Promise<unknown>,
548
+ providerCompleted: Promise<unknown>,
549
+ signal: AbortSignal | undefined,
550
+ ): Promise<void> {
551
+ await waitForTurnFinalizerStep(batcherFlush, signal);
552
+ await waitForTurnFinalizerStep(providerCompleted, signal);
553
+ }
554
+
555
+ function turnFinalizerCancellationSignal(
556
+ temporalSignal: AbortSignal | undefined,
557
+ activityStatus: RunAgentTurnResult["status"] | "unknown",
558
+ ): AbortSignal | undefined {
559
+ if (activityStatus !== "cancelled" || temporalSignal?.aborted) return temporalSignal;
560
+ const fenced = new AbortController();
561
+ fenced.abort(new Error("TURN_ATTEMPT_FENCED"));
562
+ return fenced.signal;
563
+ }
564
+
305
565
  function compactionFailureReason(reason: string): string {
306
566
  return reason.startsWith("compaction summarization failed:")
307
567
  ? reason
308
568
  : `compaction summarization failed: ${reason}`;
309
569
  }
310
570
 
571
+ function compactionFailureReasonFromError(error: unknown): string {
572
+ if (
573
+ error instanceof CompactionProviderResponseError ||
574
+ error instanceof EmptyCompactionSummaryError
575
+ ) {
576
+ return compactionFailureReason(error.message);
577
+ }
578
+ const errorName = error instanceof Error && error.name ? error.name : "unknown error";
579
+ return compactionFailureReason(`unexpected ${errorName}`);
580
+ }
581
+
582
+ function isCompactionSummaryFailure(error: unknown): boolean {
583
+ return (
584
+ error instanceof CompactionProviderResponseError || error instanceof EmptyCompactionSummaryError
585
+ );
586
+ }
587
+
588
+ export function shouldRecoverCompactionProviderFailure(error: unknown): boolean {
589
+ if (!(error instanceof CompactionProviderResponseError)) return false;
590
+ if (isCodexTransportError(error) && classifyCodexUsageLimitError(error)) return true;
591
+ return agentRunFailurePayload(error).retryable === true;
592
+ }
593
+
311
594
  export function classifyContextWindowOverflowError(
312
595
  error: unknown,
313
596
  ): { message: string; code?: string; detail?: string } | null {
@@ -440,6 +723,28 @@ export function modelUsageSourceKey(input: {
440
723
  return input.dispatchId ? `${input.dispatchId}:${input.positionalKey}` : input.positionalKey;
441
724
  }
442
725
 
726
+ export function providerContextTokens(
727
+ usage:
728
+ | {
729
+ inputTokens?: number;
730
+ outputTokens?: number;
731
+ totalTokens?: number;
732
+ }
733
+ | null
734
+ | undefined,
735
+ ): number | null {
736
+ const total = usage?.totalTokens;
737
+ if (typeof total === "number" && Number.isFinite(total) && total > 0) {
738
+ return total;
739
+ }
740
+ const input = usage?.inputTokens;
741
+ if (typeof input !== "number" || !Number.isFinite(input) || input <= 0) {
742
+ return null;
743
+ }
744
+ const output = usage?.outputTokens;
745
+ return input + (typeof output === "number" && Number.isFinite(output) && output > 0 ? output : 0);
746
+ }
747
+
443
748
  /**
444
749
  * A provider call has already consumed tokens by the time its usage frame is
445
750
  * available. Losing the Codex credential lease at the renewal checkpoint must
@@ -565,12 +870,13 @@ export function historyRowsToAppend(
565
870
  // persistedHistoryCount to preserve the pre-compaction behaviour (contiguous
566
871
  // positions from 0) when callers do not pass an explicit next position.
567
872
  nextPosition: number = persistedHistoryCount,
873
+ toolOutputTruncationTokens?: number,
568
874
  ): {
569
875
  rows: Array<{ position: number; item: Record<string, unknown> }>;
570
876
  nextWatermark: number;
571
877
  nextPosition: number;
572
878
  } {
573
- const sanitized = sanitizeHistoryItemsForModel(rawHistory).filter(
879
+ const sanitized = sanitizeHistoryItemsForModel(rawHistory, toolOutputTruncationTokens).filter(
574
880
  (item) => !isEphemeralInternalContext(item),
575
881
  );
576
882
  if (sanitized.length <= persistedHistoryCount) {
@@ -916,6 +1222,86 @@ export type TurnSandboxProvisioner<T> = {
916
1222
  waitForSettled(timeoutMs: number): Promise<T | null>;
917
1223
  };
918
1224
 
1225
+ export class TurnOperationCancelledError extends Error {
1226
+ readonly name = "TurnOperationCancelledError";
1227
+
1228
+ constructor(readonly reason: unknown) {
1229
+ super("Turn operation was cancelled with its owning turn", {
1230
+ ...(reason instanceof Error ? { cause: reason } : {}),
1231
+ });
1232
+ }
1233
+ }
1234
+
1235
+ /**
1236
+ * Normalize a preparation/provisioning cancellation race back to the Temporal
1237
+ * cancellation that owns the activity. Several provider APIs expose no portable
1238
+ * abort primitive, so the worker stops awaiting them and disposes any late
1239
+ * resource. The wrapper error must never fall through as an ordinary turn
1240
+ * failure: doing so would omit the quiescence receipt and strand a committed
1241
+ * Steer/Pause behind `control-pending`.
1242
+ */
1243
+ export function turnOperationCancellationFailure(error: unknown): CancelledFailure | null {
1244
+ if (error instanceof CancelledFailure) return error;
1245
+ if (!(error instanceof TurnOperationCancelledError)) return null;
1246
+ return error.reason instanceof CancelledFailure
1247
+ ? error.reason
1248
+ : new CancelledFailure("TURN_SANDBOX_PROVISION_CANCELLED", [], error);
1249
+ }
1250
+
1251
+ function throwIfTurnOperationCancelled(signal: AbortSignal | undefined): void {
1252
+ if (signal?.aborted) {
1253
+ throw new TurnOperationCancelledError(signal.reason);
1254
+ }
1255
+ }
1256
+
1257
+ export async function waitForTurnOperation<T>(
1258
+ operation: Promise<T>,
1259
+ signal: AbortSignal | undefined,
1260
+ disposeLateResult: ((result: T) => Promise<void> | void) | undefined,
1261
+ ): Promise<T> {
1262
+ if (!signal) return await operation;
1263
+
1264
+ let rejectCancellation: ((error: TurnOperationCancelledError) => void) | undefined;
1265
+ const cancelled = new Promise<never>((_resolve, reject) => {
1266
+ rejectCancellation = reject;
1267
+ });
1268
+ const cancel = (): void => {
1269
+ rejectCancellation?.(new TurnOperationCancelledError(signal.reason));
1270
+ };
1271
+ signal.addEventListener("abort", cancel, { once: true });
1272
+ if (signal.aborted) cancel();
1273
+
1274
+ try {
1275
+ const result = await Promise.race([operation, cancelled]);
1276
+ // Cancellation owns an exact turn boundary even when the provider result
1277
+ // and AbortSignal settle in the same microtask checkpoint. Never let a
1278
+ // just-resolved lease escape after the control was already committed.
1279
+ if (signal.aborted) {
1280
+ throw new TurnOperationCancelledError(signal.reason);
1281
+ }
1282
+ return result;
1283
+ } catch (error) {
1284
+ if (signal.aborted) {
1285
+ // The provider establish call has no universal cancellation seam. It may
1286
+ // finish after the Temporal activity has correctly stopped; dispose its
1287
+ // late lease instead of letting a cancelled turn resurrect a holder/box.
1288
+ void operation
1289
+ .then(async (result) => await disposeLateResult?.(result))
1290
+ .catch(() => undefined);
1291
+ // A provider failure may settle in the same checkpoint as the committed
1292
+ // control. The control is authoritative; retain its cancellation shape
1293
+ // so the activity publishes quiescence instead of looking like an
1294
+ // unrelated turn failure.
1295
+ if (!(error instanceof TurnOperationCancelledError)) {
1296
+ throw new TurnOperationCancelledError(signal.reason);
1297
+ }
1298
+ }
1299
+ throw error;
1300
+ } finally {
1301
+ signal.removeEventListener("abort", cancel);
1302
+ }
1303
+ }
1304
+
919
1305
  function sleep(ms: number): Promise<void> {
920
1306
  return new Promise((resolve) => setTimeout(resolve, ms));
921
1307
  }
@@ -938,9 +1324,11 @@ export function createTurnSandboxProvisioner<T>(
938
1324
  options: {
939
1325
  maxRetries?: number;
940
1326
  backoffMs?: number;
1327
+ signal?: AbortSignal;
941
1328
  onStarted?: () => Promise<void> | void;
942
- onCompleted?: () => Promise<void> | void;
1329
+ onCompleted?: (result: T) => Promise<void> | void;
943
1330
  onFailed?: (error: unknown) => Promise<void> | void;
1331
+ disposeResult?: (result: T) => Promise<void> | void;
944
1332
  } = {},
945
1333
  ): TurnSandboxProvisioner<T> {
946
1334
  const maxRetries = options.maxRetries ?? 2;
@@ -951,9 +1339,15 @@ export function createTurnSandboxProvisioner<T>(
951
1339
  let attempt = 0;
952
1340
  while (true) {
953
1341
  try {
954
- return await establish();
1342
+ throwIfTurnOperationCancelled(options.signal);
1343
+ const operation = establish();
1344
+ return await waitForTurnOperation(operation, options.signal, options.disposeResult);
955
1345
  } catch (error) {
956
- if (attempt >= maxRetries || !isLazySandboxProvisionRetryable(error)) {
1346
+ if (
1347
+ error instanceof TurnOperationCancelledError ||
1348
+ attempt >= maxRetries ||
1349
+ !isLazySandboxProvisionRetryable(error)
1350
+ ) {
957
1351
  throw error;
958
1352
  }
959
1353
  attempt += 1;
@@ -966,13 +1360,24 @@ export function createTurnSandboxProvisioner<T>(
966
1360
  get(): Promise<T> {
967
1361
  if (!memo) {
968
1362
  memo = (async () => {
1363
+ throwIfTurnOperationCancelled(options.signal);
969
1364
  await options.onStarted?.();
1365
+ throwIfTurnOperationCancelled(options.signal);
1366
+ let result: T | undefined;
1367
+ let hasResult = false;
970
1368
  try {
971
- const result = await run();
972
- await options.onCompleted?.();
1369
+ result = await run();
1370
+ hasResult = true;
1371
+ throwIfTurnOperationCancelled(options.signal);
1372
+ await options.onCompleted?.(result);
973
1373
  return result;
974
1374
  } catch (error) {
975
- await options.onFailed?.(error);
1375
+ if (hasResult) {
1376
+ await options.disposeResult?.(result as T);
1377
+ }
1378
+ if (!(error instanceof TurnOperationCancelledError)) {
1379
+ await options.onFailed?.(error);
1380
+ }
976
1381
  throw error;
977
1382
  }
978
1383
  })().catch((error) => {
@@ -1041,7 +1446,7 @@ export function acceptsPromptCacheKeyForTurn(
1041
1446
  * The turn hot path never refreshes Codex usage — only the usage API route does — so a
1042
1447
  * window that has actually reset still reads OVER-threshold from the stale cache, which
1043
1448
  * would idle-loop forever. Before idling, refresh LIVE usage for every connected account
1044
- * the cache marks over-threshold (bounded to the account count), which re-writes the
1449
+ * the cache marks exhausted (bounded to the account count), which re-writes the
1045
1450
  * cache columns, then return the re-read rows so the ranker can pick up a genuinely-reset
1046
1451
  * window THIS turn. A refresh/read failure is swallowed (fall back to the pre-refresh rows
1047
1452
  * + the bounded idle). Cooling (429'd) accounts are NOT refreshed: their exhaustedUntil
@@ -1057,11 +1462,11 @@ async function refreshCappedCodexUsageRows(
1057
1462
  wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
1058
1463
  },
1059
1464
  ): Promise<CodexAccountStatus[]> {
1060
- const nearPct = settings.codexRotationNearExhaustionPct;
1061
1465
  const stale = accounts.filter(
1062
1466
  (a) =>
1063
1467
  a.status === "active" &&
1064
- ((a.primaryUsedPercent ?? 0) >= nearPct || (a.secondaryUsedPercent ?? 0) >= nearPct),
1468
+ ((a.primaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT ||
1469
+ (a.secondaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT),
1065
1470
  );
1066
1471
  if (stale.length === 0) {
1067
1472
  return accounts;
@@ -1097,11 +1502,22 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1097
1502
  objectStorage,
1098
1503
  observability,
1099
1504
  wakeSessionWorkflow,
1505
+ signalSessionAttemptQuiesced,
1100
1506
  signalCodexCapacityWorkflow,
1101
1507
  entitlements,
1102
1508
  connectionCredentials,
1103
1509
  } = await services();
1104
1510
  const activityContext = currentActivityContext();
1511
+ const cancellationSignal = activityContext?.cancellationSignal;
1512
+ let cancellationRequestedAt: number | null = cancellationSignal?.aborted
1513
+ ? performance.now()
1514
+ : null;
1515
+ const noteCancellationRequested = (): void => {
1516
+ cancellationRequestedAt ??= performance.now();
1517
+ };
1518
+ cancellationSignal?.addEventListener("abort", noteCancellationRequested, {
1519
+ once: true,
1520
+ });
1105
1521
  const dispatchId = activityContext?.info.activityId ?? randomUUID();
1106
1522
  const activityStarted = performance.now();
1107
1523
  const activitySpan = observability.startSpan("worker.run_agent_segment", {
@@ -1112,6 +1528,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1112
1528
  let activityStatus: RunAgentTurnResult["status"] | "unknown" = "unknown";
1113
1529
  let turnMetricOutcome: TurnOutcome | null = null;
1114
1530
  let activityError: unknown;
1531
+ let acknowledgeQuiescence = false;
1532
+ const acknowledgeLostAttemptOwnership = (): void => {
1533
+ // A stale terminal/recovery settlement can lose either to a benign
1534
+ // successor or to Pause/Steer closing this exact attempt. Only the
1535
+ // receipt transaction can distinguish those cases after the hard tool
1536
+ // fence: allowUninterrupted makes the benign case an event-free no-op.
1537
+ acknowledgeQuiescence = true;
1538
+ noteCancellationRequested();
1539
+ };
1115
1540
  let turnId: string | undefined;
1116
1541
  let triggerEventId: string | undefined;
1117
1542
  const claimedResult = (
@@ -1139,13 +1564,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1139
1564
  // Still required by credential-loss/capacity settlements, whose own
1140
1565
  // recovery transactions fence against worker-death redispatches.
1141
1566
  let redispatchesAtDispatch = 0;
1142
- const saveRunStateFenced = async (state: Parameters<typeof saveRunState>[1]): Promise<void> => {
1143
- if (!(await saveRunState(db, state))) {
1144
- throw new TurnAttemptFencedError(
1145
- "turn execution generation was fenced while saving run state",
1146
- );
1147
- }
1148
- };
1149
1567
  const setLastInputTokensFenced = async (lastInputTokens: number): Promise<void> => {
1150
1568
  if (!turnId || executionGeneration <= 0) {
1151
1569
  throw new Error("Turn attempt was not initialized before token accounting");
@@ -1164,7 +1582,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1164
1582
  }
1165
1583
  };
1166
1584
  let heartbeatTimer: ReturnType<typeof startActivityHeartbeat> | undefined;
1167
- // OPE-21: one workspace-local idempotent credential holder per running
1585
+ // credential allocator: one workspace-local idempotent credential holder per running
1168
1586
  // Codex turn. The DB row is the cross-replica fairness primitive; this timer
1169
1587
  // only extends its short TTL. A killed worker stops heartbeating and the
1170
1588
  // holder self-expires. Other workspaces never see or share this holder.
@@ -1295,11 +1713,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1295
1713
  // runs it refreshes expires_at epoch-fenced so a legit multi-day turn is
1296
1714
  // never TTL-reaped. Cleared in finally. Only set when the flag resolved a box.
1297
1715
  let leaseHeartbeatTimer: ReturnType<typeof setInterval> | undefined;
1298
- // OPE-41: the worker, not the model, owns renewal of run-scoped Git
1716
+ // credential-renewal policy: the worker, not the model, owns renewal of run-scoped Git
1299
1717
  // credentials for a multi-day turn. The controller is attached only after
1300
1718
  // the initial seed reached a real cloud box and is drained before capture.
1301
- let gitCredentialRenewal: GitCredentialRenewalController | null = null;
1719
+ let gitCredentialRenewals: GitCredentialRenewalController[] = [];
1302
1720
  let gitCredentialRenewalClosed = false;
1721
+ // Generic host-owned run material has its own attempt-scoped renewal and
1722
+ // write handle. It is always drained and wiped before workspace capture.
1723
+ let runCredentialRenewal: RunCredentialRenewalController | null = null;
1724
+ let runCredentialRenewalClosed = false;
1725
+ let runCredentialSession: RunCredentialCommandSession | null = null;
1726
+ // The delegated Toolspace bearer has a one-hour TTL. Renewal is attempt-
1727
+ // owned and attaches only after the initial token file reached a real
1728
+ // sandbox session; finalization drains an in-flight replacement.
1729
+ let toolspaceTokenRenewal: ToolspaceTokenRenewalController | null = null;
1730
+ let toolspaceTokenRenewalClosed = false;
1303
1731
  // MID-SESSION snapshot single-flight guard: the heartbeat tick fires every
1304
1732
  // 10s but a Modal filesystem snapshot can take longer — never overlap two
1305
1733
  // captures on one box. The in-flight capture's promise is held so the
@@ -1349,6 +1777,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1349
1777
  await current?.flush().catch(() => undefined);
1350
1778
  };
1351
1779
  let preparedTools: Awaited<ReturnType<OpenGeniRuntime["prepareTools"]>> | null = null;
1780
+ const toolCancellationFenceRef: {
1781
+ current: TurnToolCancellationFence | null;
1782
+ } = {
1783
+ current: null,
1784
+ };
1352
1785
  let publish: TurnEventPublisher | null = null;
1353
1786
  let settle:
1354
1787
  | ((input: {
@@ -1362,10 +1795,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1362
1795
  | "requires_action";
1363
1796
  sessionStatus: SessionStatus;
1364
1797
  activeTurnId: string | null;
1798
+ consumeRequestedCompactionFailure?: boolean;
1799
+ runState?: ApplySessionTurnSettlementInput["runState"];
1365
1800
  }) => Promise<boolean>)
1366
1801
  | null = null;
1367
1802
  let turnStartedPublished = false;
1368
1803
  let stream: Awaited<ReturnType<OpenGeniRuntime["runStream"]>> | undefined;
1804
+ // Reconciliation is declared before provider routing so every turn-end path
1805
+ // can share one closure. It cannot run until `stream` exists, by which time
1806
+ // this value has been rebound to the turn's resolved model policy.
1807
+ let modelRunSettings: Settings = settings;
1369
1808
  const publishSandboxLifecycleEvents = async (sandbox: ResumedTurnSandbox): Promise<void> => {
1370
1809
  const established = sandbox.established;
1371
1810
  if (publish && established.origin && established.origin !== "resumed") {
@@ -1465,16 +1904,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1465
1904
  // selfhosted target has no persistWorkspace anyway. Best-effort +
1466
1905
  // single-flight; throttling lives in the helper.
1467
1906
  const snapshotSession = setupBoxSession;
1468
- if (snapshotSession && !snapshotInFlight) {
1907
+ const snapshotTurnId = turnId;
1908
+ if (snapshotSession && snapshotTurnId && !snapshotInFlight) {
1469
1909
  snapshotInFlight = maybePersistWarmWorkspaceSnapshot(
1470
1910
  { db, settings },
1471
1911
  {
1472
1912
  accountId: input.accountId,
1473
1913
  workspaceId: input.workspaceId,
1914
+ sessionId: input.sessionId,
1915
+ turnId: snapshotTurnId,
1916
+ attemptId: input.attemptId,
1474
1917
  sandboxGroupId: heartbeatGroupId,
1475
1918
  },
1476
1919
  snapshotSession,
1477
1920
  heartbeatEpoch,
1921
+ activityContext?.cancellationSignal,
1478
1922
  )
1479
1923
  .then(async (persisted) => {
1480
1924
  if (persisted && publish) {
@@ -1602,6 +2046,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1602
2046
  rawHistory as Array<Record<string, unknown>>,
1603
2047
  persistedHistoryCount,
1604
2048
  nextHistoryPosition,
2049
+ modelRunSettings.modelToolOutputTruncationTokens,
1605
2050
  );
1606
2051
  const hasModelOrToolProgress = rows.some((row) =>
1607
2052
  isModelOrToolProgressHistoryItem(row.item),
@@ -1621,6 +2066,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1621
2066
  // runs, so this is the turn's effective account. The read path uses
1622
2067
  // it to strip cross-account reasoning.encrypted_content next turn.
1623
2068
  producerCodexCredentialId: effectiveCodexCredentialId,
2069
+ modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
1624
2070
  items: rows,
1625
2071
  });
1626
2072
  if (!appended) {
@@ -1651,6 +2097,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1651
2097
  // Reassigned after the variable set loads; the publish closure is
1652
2098
  // created (and used for turn.started) before the variableSet is available.
1653
2099
  let redact: (payload: unknown) => unknown = identityRedactor;
2100
+ const secretRedactions = new Map<string, string>();
2101
+ const publishedRunCredentialNotices = new Set<string>();
2102
+ const registerSecretRedactions = (secrets: SecretForRedaction[]): void => {
2103
+ for (const secret of secrets) {
2104
+ if (!secretRedactions.has(secret.value)) secretRedactions.set(secret.value, secret.name);
2105
+ }
2106
+ redact = createSecretRedactor(
2107
+ [...secretRedactions].map(([value, name]) => ({ name, value })),
2108
+ );
2109
+ };
1654
2110
  let variableSetId = "";
1655
2111
  // Rig telemetry (M3): set once the session loads; empty string for a rig-less
1656
2112
  // turn (mirrors variableSetId). Read by the activity span's finally block.
@@ -1718,6 +2174,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1718
2174
  if (!trigger) {
1719
2175
  throw new Error(`Trigger event not found: ${triggerEventId}`);
1720
2176
  }
2177
+ const humanInputResume = await getHumanInputResumeForEvent(
2178
+ db,
2179
+ input.workspaceId,
2180
+ input.sessionId,
2181
+ trigger,
2182
+ );
1721
2183
  triggerType = trigger.type;
1722
2184
  executionGeneration = turn.executionGeneration;
1723
2185
  const latestTurnState = await getLatestRunState(db, input.workspaceId, input.sessionId);
@@ -1745,13 +2207,17 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1745
2207
  // §7.5 P3 — pass BOTH the codex predicate (codex-plan turns bypass the gate)
1746
2208
  // AND the optional host `entitlements` port (when bound, its admitRun replaces
1747
2209
  // the local credit read). Unset port → today's local-ledger path.
1748
- await ensureRunAllowed(
1749
- settings,
1750
- db,
1751
- input.accountId,
1752
- input.workspaceId,
1753
- isCodexTurn,
1754
- entitlements,
2210
+ await waitForTurnOperation(
2211
+ ensureRunAllowed(
2212
+ settings,
2213
+ db,
2214
+ input.accountId,
2215
+ input.workspaceId,
2216
+ isCodexTurn,
2217
+ entitlements,
2218
+ ),
2219
+ cancellationSignal,
2220
+ undefined,
1755
2221
  );
1756
2222
  // Setup (variableSet load, MCP connects, sandbox restore) does not
1757
2223
  // stream and so never observes cancellation on its own; these explicit
@@ -1763,6 +2229,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1763
2229
  throw reason;
1764
2230
  }
1765
2231
  };
2232
+ const throwIfTurnCancelled = () => throwIfTurnOperationCancelled(cancellationSignal);
1766
2233
  // ONE shared details object for every heartbeat this activity sends (each
1767
2234
  // site spreads it + its own phase), so cross-site fields — the op-stream
1768
2235
  // settled roster in particular — survive last-write-wins instead of being
@@ -1858,6 +2325,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1858
2325
  producerSeq: ++producerSeq,
1859
2326
  };
1860
2327
  }
2328
+ const compactionRequestFailure = inputSettlement.consumeRequestedCompactionFailure
2329
+ ? {
2330
+ reason: "summarization_failed" as const,
2331
+ producerId,
2332
+ producerSeq: ++producerSeq,
2333
+ }
2334
+ : undefined;
1861
2335
  const inputs = inputSettlement.events.map((event) => ({
1862
2336
  ...event,
1863
2337
  payload: redact(event.payload),
@@ -1874,9 +2348,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1874
2348
  sessionStatus: inputSettlement.sessionStatus,
1875
2349
  activeTurnId: inputSettlement.activeTurnId,
1876
2350
  events: inputs,
2351
+ ...(inputSettlement.runState ? { runState: inputSettlement.runState } : {}),
1877
2352
  ...(recordingMutation ? { recording: recordingMutation } : {}),
2353
+ ...(compactionRequestFailure ? { compactionRequestFailure } : {}),
1878
2354
  });
1879
2355
  if (result.action === "stale") {
2356
+ // The terminal write can lose to a control transaction before the
2357
+ // workflow delivers Temporal cancellation. That control may settle
2358
+ // the already-closed attempt as rejected_stale, so returning without
2359
+ // this flag would strand its replacement behind quiesced_at forever.
2360
+ // Enter the same hard tool-fence/receipt path as an explicit
2361
+ // TurnAttemptFencedError. If ownership was lost for an unrelated
2362
+ // reason, allowUninterrupted makes the receipt transaction a no-op.
2363
+ acknowledgeLostAttemptOwnership();
1880
2364
  if (recordingForSettlement) {
1881
2365
  await abandonActiveRecording(
1882
2366
  "recording settlement lost attempt ownership",
@@ -1917,6 +2401,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1917
2401
  // visibly starts: nothing ran yet, so the same inference starts cleanly
1918
2402
  // on a healthy worker.
1919
2403
  throwIfWorkerShuttingDown();
2404
+ throwIfTurnCancelled();
1920
2405
  if (
1921
2406
  !(await settle({
1922
2407
  events: [
@@ -1953,7 +2438,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1953
2438
  sessionPinSource,
1954
2439
  sessionLastCredentialId: sessionCodex?.lastCredentialId ?? null,
1955
2440
  continuationCredentialId: continuationCodexCredentialId,
1956
- nearExhaustionPct: settings.codexRotationNearExhaustionPct,
1957
2441
  now: new Date(),
1958
2442
  });
1959
2443
 
@@ -2075,7 +2559,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2075
2559
  strategy: leased.rotationStrategy as CodexRotationStrategy,
2076
2560
  rotationEnabled: leased.rotationEnabled,
2077
2561
  });
2078
- // OPE-31 pin persistence follows the atomic OPE-21 selection. The selector
2562
+ // pin policy pin persistence follows the atomic credential allocator selection. The selector
2079
2563
  // already ran exact-turn reuse before policy filtering and vetoed pointer
2080
2564
  // movement for manual/policy homes; this write only records the NEXT turn's
2081
2565
  // policy home (or clears a policy pin whose strategy is no longer active).
@@ -2163,7 +2647,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2163
2647
  if (codexLeaseHeld) startCodexLeaseHeartbeat();
2164
2648
 
2165
2649
  const eligibleCount = leased.accounts.filter((account) =>
2166
- isCodexCredentialEligible(account, settings.codexRotationNearExhaustionPct, new Date()),
2650
+ isCodexCredentialEligible(account, new Date()),
2167
2651
  ).length;
2168
2652
  const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
2169
2653
  observability.incrementCounter({
@@ -2340,7 +2824,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2340
2824
  if (goalActive && goal) {
2341
2825
  const authoritativeResetAt = authoritativeCodexCapacityResetAt(
2342
2826
  leased.accounts,
2343
- settings.codexRotationNearExhaustionPct,
2344
2827
  new Date(),
2345
2828
  );
2346
2829
  const armed = await armCodexCapacityWait(db, {
@@ -2482,7 +2965,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2482
2965
  // Resolved after turn.started so a composition conflict (two enabled
2483
2966
  // packs declaring images) fails the turn with its plain error instead
2484
2967
  // of failing the activity opaquely.
2485
- const packRuntime = await resolveWorkspacePackRuntime(db, input.workspaceId);
2968
+ const [packRuntime, skillLibraryRuntime] = await Promise.all([
2969
+ resolveWorkspacePackRuntime(db, input.workspaceId),
2970
+ resolveWorkspaceSkillLibraryRuntime(db, input.workspaceId),
2971
+ ]);
2486
2972
  // RIG BINDING (M3): load the session's FROZEN rig version (resolved+frozen
2487
2973
  // at create). Everything rig-derived below (image precedence, env default
2488
2974
  // sets, setup hook, credential hooks, doctrine, lease/telemetry stamps) is
@@ -2552,7 +3038,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2552
3038
  // raw window, effective ceiling, and auto-compact limit are distinct live
2553
3039
  // catalog values and must reach pre-turn compaction, history guards, and
2554
3040
  // every model call together.
2555
- const modelRunSettings = resolvedModel
3041
+ modelRunSettings = resolvedModel
2556
3042
  ? settingsWithResolvedModelContext(runSettings, resolvedModel.configured)
2557
3043
  : runSettings;
2558
3044
  // WORKSPACE MODEL POLICY — the authoritative hard gate. Runs immediately
@@ -2653,6 +3139,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2653
3139
  workspaceId: input.workspaceId,
2654
3140
  sessionId: input.sessionId,
2655
3141
  turnId: turn.id,
3142
+ turnAttemptId: input.attemptId,
2656
3143
  model: resolvedModel?.configured.id ?? turn.model,
2657
3144
  isCodexTurn,
2658
3145
  usage: usage.usage,
@@ -2725,25 +3212,69 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2725
3212
  );
2726
3213
  let outcome: Awaited<ReturnType<typeof maybeCompactContext>> | null = null;
2727
3214
  if (requested) {
2728
- outcome = await maybeCompactContext(
2729
- db,
2730
- modelRunSettings,
2731
- {
2732
- accountId: input.accountId,
2733
- workspaceId: input.workspaceId,
2734
- sessionId: input.sessionId,
2735
- turnId: turn.id,
2736
- executionGeneration,
2737
- attemptId: input.attemptId,
2738
- },
2739
- session.lastInputTokens,
2740
- compactionSummarizerFor(compactionInstructions),
2741
- {
2742
- force: true,
2743
- clearRequestedCompaction: true,
2744
- trigger: "operator",
2745
- },
2746
- );
3215
+ try {
3216
+ outcome = await waitForTurnOperation(
3217
+ maybeCompactContext(
3218
+ db,
3219
+ modelRunSettings,
3220
+ {
3221
+ accountId: input.accountId,
3222
+ workspaceId: input.workspaceId,
3223
+ sessionId: input.sessionId,
3224
+ turnId: turn.id,
3225
+ executionGeneration,
3226
+ attemptId: input.attemptId,
3227
+ },
3228
+ session.lastInputTokens,
3229
+ compactionSummarizerFor(compactionInstructions),
3230
+ {
3231
+ force: true,
3232
+ clearRequestedCompaction: true,
3233
+ trigger: "operator",
3234
+ },
3235
+ ),
3236
+ cancellationSignal,
3237
+ undefined,
3238
+ );
3239
+ } catch (error) {
3240
+ // Codex retries retryable checkpoint-provider failures rather than
3241
+ // treating them as a semantic compaction result. Keep the operator
3242
+ // request pending and let the ordinary same-turn provider/capacity
3243
+ // recovery path re-dispatch this exact maintenance execution.
3244
+ if (shouldRecoverCompactionProviderFailure(error)) throw error;
3245
+ if (!isCompactionSummaryFailure(error)) throw error;
3246
+ const errorMessage = compactionFailureReasonFromError(error);
3247
+ if (
3248
+ !(await settle!({
3249
+ events: [
3250
+ {
3251
+ type: "turn.failed",
3252
+ payload: {
3253
+ error: errorMessage,
3254
+ code: "context_compaction_failed",
3255
+ retryable: false,
3256
+ recovery: "user_message",
3257
+ compacted: false,
3258
+ },
3259
+ },
3260
+ {
3261
+ type: "session.status.changed",
3262
+ payload: { status: "idle" },
3263
+ },
3264
+ ],
3265
+ turnStatus: "failed",
3266
+ sessionStatus: "idle",
3267
+ activeTurnId: null,
3268
+ consumeRequestedCompactionFailure: true,
3269
+ }))
3270
+ ) {
3271
+ return claimedResult({ status: "cancelled" });
3272
+ }
3273
+ turnMetricOutcome = "failed";
3274
+ activityStatus = "idle";
3275
+ activityError = error;
3276
+ return claimedResult({ status: "idle" });
3277
+ }
2747
3278
  if (outcome.events.length > 0) {
2748
3279
  if (outcome.compacted) {
2749
3280
  recordContextCompaction(observability, "operator");
@@ -2794,18 +3325,22 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2794
3325
  runSettings,
2795
3326
  withFirstPartyTools(runSettings, mergeToolRefs(session.tools, turn.tools)),
2796
3327
  );
2797
- // §7.6 P4a — load (and decrypt) the variable set via the host
3328
+ // §7.6 connection-credential provider — load (and decrypt) the variable set via the host
2798
3329
  // `sandboxSecrets` provider when bound; unset → today's local decrypt.
2799
3330
  const connectionScope = {
2800
3331
  accountId: input.accountId,
2801
3332
  workspaceId: input.workspaceId,
2802
3333
  };
2803
- const workspaceVariableSet = await loadWorkspaceEnvironmentForRunWithCredentials(
2804
- db,
2805
- runSettings,
2806
- connectionScope,
2807
- session.variableSetId,
2808
- connectionCredentials?.sandboxSecrets,
3334
+ const workspaceVariableSet = await waitForTurnOperation(
3335
+ loadWorkspaceEnvironmentForRunWithCredentials(
3336
+ db,
3337
+ runSettings,
3338
+ connectionScope,
3339
+ session.variableSetId,
3340
+ connectionCredentials?.sandboxSecrets,
3341
+ ),
3342
+ cancellationSignal,
3343
+ undefined,
2809
3344
  );
2810
3345
  variableSetId = workspaceVariableSet?.id ?? "";
2811
3346
  // RIG DEFAULT VARIABLE SETS (M3): decrypt the frozen rig version's default
@@ -2820,12 +3355,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2820
3355
  // set already relies on), keeping validateNoEnvironmentDelta empty.
2821
3356
  const rigDefaultEnvironmentValues: Record<string, string> = {};
2822
3357
  for (const rigDefaultVariableSetId of rigVersion?.defaultVariableSetIds ?? []) {
2823
- const rigDefaultSet = await loadWorkspaceEnvironmentForRunWithCredentials(
2824
- db,
2825
- runSettings,
2826
- connectionScope,
2827
- rigDefaultVariableSetId,
2828
- connectionCredentials?.sandboxSecrets,
3358
+ const rigDefaultSet = await waitForTurnOperation(
3359
+ loadWorkspaceEnvironmentForRunWithCredentials(
3360
+ db,
3361
+ runSettings,
3362
+ connectionScope,
3363
+ rigDefaultVariableSetId,
3364
+ connectionCredentials?.sandboxSecrets,
3365
+ ),
3366
+ cancellationSignal,
3367
+ undefined,
2829
3368
  );
2830
3369
  Object.assign(rigDefaultEnvironmentValues, rigDefaultSet?.values ?? {});
2831
3370
  }
@@ -2836,7 +3375,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2836
3375
  );
2837
3376
  // Redact EVERY exported secret value (rig defaults + session set) from turn
2838
3377
  // output, not just the session set's.
2839
- redact = createSecretRedactor(
3378
+ registerSecretRedactions(
2840
3379
  Object.entries(sandboxWorkspaceEnvironmentValues).map(([name, value]) => ({ name, value })),
2841
3380
  );
2842
3381
  // EFFECTIVE compute backend, resolved ONCE at turn start (Case B + Stage D
@@ -2909,7 +3448,56 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2909
3448
  settings.sandboxOwnershipEnabled && runSettings.sandboxBackend !== "none"
2910
3449
  ? groupBoxBackend
2911
3450
  : runSettings.sandboxBackend;
2912
- await ensureTurnModalRegistryImage(runSettings, sandboxCreationBackend);
3451
+ const effectiveRunCredentialBackend = activeSandboxBackend ?? groupBoxBackend;
3452
+ const runCredentialResolver =
3453
+ effectiveRunCredentialBackend === "none"
3454
+ ? null
3455
+ : await waitForTurnOperation(
3456
+ bindRunCredentialResolver({
3457
+ db,
3458
+ connectionCredentials: connectionCredentials ?? null,
3459
+ accountId: input.accountId,
3460
+ workspaceId: input.workspaceId,
3461
+ session,
3462
+ turn,
3463
+ attemptId: input.attemptId,
3464
+ effectiveSandboxBackend: effectiveRunCredentialBackend,
3465
+ variableSet: workspaceVariableSet
3466
+ ? {
3467
+ id: workspaceVariableSet.id,
3468
+ name: workspaceVariableSet.name,
3469
+ }
3470
+ : null,
3471
+ }),
3472
+ cancellationSignal,
3473
+ undefined,
3474
+ );
3475
+ const initialRunCredentialMaterial = runCredentialResolver
3476
+ ? await waitForTurnOperation(
3477
+ runCredentialResolver.resolve({
3478
+ purpose: "provision",
3479
+ forceRefresh: false,
3480
+ }),
3481
+ cancellationSignal,
3482
+ undefined,
3483
+ )
3484
+ : null;
3485
+ if (initialRunCredentialMaterial) {
3486
+ registerSecretRedactions(initialRunCredentialMaterial.redactions);
3487
+ for (const payload of runCredentialAuthNeededPayloads(initialRunCredentialMaterial)) {
3488
+ publishedRunCredentialNotices.add(JSON.stringify(payload));
3489
+ await publish!([{ type: "credential.auth_needed", payload }], true);
3490
+ }
3491
+ }
3492
+ const runCredentialsNote = initialRunCredentialMaterial
3493
+ ? runCredentialModelNote(initialRunCredentialMaterial)
3494
+ : undefined;
3495
+ throwIfTurnOperationCancelled(cancellationSignal);
3496
+ await waitForTurnOperation(
3497
+ ensureTurnModalRegistryImage(runSettings, sandboxCreationBackend),
3498
+ cancellationSignal,
3499
+ undefined,
3500
+ );
2913
3501
  const establishPolicy: "eager" | "on-demand" =
2914
3502
  lazyProvisionEnabled(settings) && !machinePrimary && runSettings.sandboxBackend !== "none"
2915
3503
  ? "on-demand"
@@ -2933,91 +3521,348 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2933
3521
  // provider may supply it; unset still self-mints GitHub from settings.
2934
3522
  // gitToken/gitTokens are undefined on the selfhosted skip path (the machine
2935
3523
  // uses its own git creds).
3524
+ if (activeSandboxBackend !== "selfhosted") {
3525
+ await assertGitHubResourcesRemainAuthorized(db, input.workspaceId, turnResources);
3526
+ }
3527
+ const authorizeGitHubTokenMint: GitHubTokenMintAuthorization = async (selection) => {
3528
+ await assertGitHubTokenMintSelectionAuthorized(
3529
+ db,
3530
+ input.workspaceId,
3531
+ selection.installationId,
3532
+ selection.repositoryIds,
3533
+ );
3534
+ };
3535
+ // Git and MCP credentials share one lineage snapshot for this turn. A
3536
+ // host that supplies both ports must never see two independently resolved
3537
+ // roots for the same execution merely because the call sites are far apart.
3538
+ const needsHostCredentialRoot = Boolean(
3539
+ connectionCredentials?.gitCredentials || connectionCredentials?.mcpCredentials,
3540
+ );
3541
+ const hostCredentialRootSessionId = needsHostCredentialRoot
3542
+ ? await getSessionRootId(db, input.workspaceId, input.sessionId)
3543
+ : null;
3544
+ if (needsHostCredentialRoot && !hostCredentialRootSessionId) {
3545
+ throw new Error(`cannot resolve host credentials for missing session ${input.sessionId}`);
3546
+ }
3547
+ const gitCredentialAuthority =
3548
+ connectionCredentials?.gitCredentials && hostCredentialRootSessionId
3549
+ ? gitCredentialAuthorityForTurn({
3550
+ sessionId: input.sessionId,
3551
+ rootSessionId: hostCredentialRootSessionId,
3552
+ attemptId: input.attemptId,
3553
+ turn,
3554
+ })
3555
+ : undefined;
2936
3556
  const {
2937
3557
  environment: sandboxEnvironment,
2938
3558
  gitToken: sandboxGitToken,
2939
3559
  gitTokens: sandboxGitTokens,
2940
3560
  gitTokenExpiresAt: sandboxGitTokenExpiresAt,
3561
+ gitCredentialBindings: sandboxGitCredentialBindings,
2941
3562
  toolspaceToken: sandboxToolspaceToken,
2942
- } = await sandboxEnvironmentForRun(
2943
- runSettings,
2944
- turnResources,
2945
- // Rig default sets merged BELOW the session set (session wins); rig-less
2946
- // turns pass exactly workspaceVariableSet?.values (byte-for-byte today).
2947
- sandboxWorkspaceEnvironmentValues,
2948
- {
2949
- skipGitHubToken: activeSandboxBackend === "selfhosted",
2950
- deferGitHubToken:
2951
- activeSandboxBackend !== "selfhosted" && establishPolicy === "on-demand",
2952
- scope: connectionScope,
2953
- gitCredentials: connectionCredentials?.gitCredentials,
2954
- sessionId: input.sessionId,
2955
- runId: turnId,
2956
- },
3563
+ toolspaceTokenExpiresAt: sandboxToolspaceTokenExpiresAt,
3564
+ } = await waitForTurnOperation(
3565
+ sandboxEnvironmentForRun(
3566
+ runSettings,
3567
+ turnResources,
3568
+ // Rig default sets merged BELOW the session set (session wins); rig-less
3569
+ // turns pass exactly workspaceVariableSet?.values (byte-for-byte today).
3570
+ sandboxWorkspaceEnvironmentValues,
3571
+ {
3572
+ skipGitHubToken: activeSandboxBackend === "selfhosted",
3573
+ deferGitHubToken:
3574
+ activeSandboxBackend !== "selfhosted" && establishPolicy === "on-demand",
3575
+ scope: connectionScope,
3576
+ ...(gitCredentialAuthority ? { authority: gitCredentialAuthority } : {}),
3577
+ gitCredentials: connectionCredentials?.gitCredentials,
3578
+ authorizeGitHubTokenMint,
3579
+ sessionId: input.sessionId,
3580
+ runId: turnId,
3581
+ },
3582
+ ),
3583
+ cancellationSignal,
3584
+ undefined,
2957
3585
  );
2958
-
2959
- const initialGitCredentials: MintedRunGitCredentials | undefined = sandboxGitTokens
2960
- ? {
2961
- gitTokens: sandboxGitTokens,
2962
- expiresAt: sandboxGitTokenExpiresAt ?? {},
2963
- }
3586
+ const sandboxToolspaceTokenFile = sandboxToolspaceToken
3587
+ ? toolspaceTokenFileFromEnvironment(sandboxEnvironment, input.sessionId)
2964
3588
  : undefined;
3589
+
3590
+ const initialGitCredentials: MintedRunGitCredentials | undefined =
3591
+ sandboxGitCredentialBindings
3592
+ ? {
3593
+ bindings: sandboxGitCredentialBindings,
3594
+ gitTokens: sandboxGitTokens ?? {},
3595
+ expiresAt: sandboxGitTokenExpiresAt ?? {},
3596
+ }
3597
+ : undefined;
2965
3598
  const attachGitCredentialRenewal = async (
2966
3599
  tokenSession: GitCredentialTokenWriterSession,
2967
3600
  initial: MintedRunGitCredentials | undefined,
2968
3601
  ): Promise<void> => {
2969
- if (!initial || Object.keys(initial.gitTokens).length === 0) return;
2970
- const previous = gitCredentialRenewal;
2971
- gitCredentialRenewal = null;
2972
- await previous?.stop();
3602
+ if (!initial || initial.bindings.length === 0) return;
3603
+ const previous = gitCredentialRenewals;
3604
+ gitCredentialRenewals = [];
3605
+ await Promise.all(previous.map(async (controller) => await controller.stop()));
2973
3606
  if (gitCredentialRenewalClosed) return;
2974
3607
 
2975
- const providers = Object.keys(initial.gitTokens) as GitCredentialProvider[];
2976
- const controller = startGitCredentialRenewalLoop({
2977
- expectedProviders: providers,
2978
- initialExpiresAt: initial.expiresAt,
2979
- mint: async () =>
2980
- await mintRunGitCredentials(runSettings, turnResources, {
2981
- scope: connectionScope,
2982
- gitCredentials: connectionCredentials?.gitCredentials,
2983
- }),
2984
- write: async (tokens) => {
2985
- const runAs = sandboxRunAs(runSettings);
2986
- await refreshGitProviderTokenFiles(tokenSession, tokens, {
2987
- ...(runAs ? { runAs } : {}),
2988
- });
2989
- },
2990
- onSuccess: ({ providers: renewedProviders }) => {
2991
- for (const provider of renewedProviders) {
2992
- observability.incrementCounter({
2993
- name: "opengeni_git_credential_renewals_total",
2994
- help: "Host-managed Git credential renewal attempts by provider and outcome.",
2995
- labels: { provider, outcome: "completed" },
3608
+ const controllers = initial.bindings.map((initialBinding) => {
3609
+ let pendingBinding: typeof initialBinding | undefined;
3610
+ return startGitCredentialRenewalLoop({
3611
+ expectedProviders: [initialBinding.provider],
3612
+ initialExpiresAt: initialBinding.expiresAt
3613
+ ? { [initialBinding.provider]: initialBinding.expiresAt }
3614
+ : {},
3615
+ mint: async () => {
3616
+ const binding = await mintRunGitCredentialBinding(
3617
+ runSettings,
3618
+ turnResources,
3619
+ initialBinding.provider,
3620
+ initialBinding.credentialBindingId,
3621
+ {
3622
+ scope: connectionScope,
3623
+ ...(gitCredentialAuthority ? { authority: gitCredentialAuthority } : {}),
3624
+ gitCredentials: connectionCredentials?.gitCredentials,
3625
+ authorizeGitHubTokenMint,
3626
+ },
3627
+ );
3628
+ pendingBinding = binding;
3629
+ return binding
3630
+ ? {
3631
+ bindings: [binding],
3632
+ gitTokens: { [binding.provider]: binding.token },
3633
+ expiresAt: binding.expiresAt ? { [binding.provider]: binding.expiresAt } : {},
3634
+ }
3635
+ : undefined;
3636
+ },
3637
+ write: async () => {
3638
+ if (!pendingBinding) {
3639
+ throw new Error("credential renewal produced no binding token");
3640
+ }
3641
+ const runAs = sandboxRunAs(runSettings);
3642
+ await refreshGitCredentialBindingTokenFiles(tokenSession, [pendingBinding], {
3643
+ ...(runAs ? { runAs } : {}),
3644
+ ...(toolCancellationFenceRef.current
3645
+ ? {
3646
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
3647
+ toolCancellationFenceRef.current,
3648
+ ),
3649
+ }
3650
+ : {}),
2996
3651
  });
2997
- }
2998
- },
2999
- onFailure: ({ providers: failedProviders, retryDelayMs, errorClass }) => {
3000
- for (const provider of failedProviders) {
3001
- observability.incrementCounter({
3002
- name: "opengeni_git_credential_renewals_total",
3003
- help: "Host-managed Git credential renewal attempts by provider and outcome.",
3004
- labels: { provider, outcome: "error" },
3652
+ },
3653
+ onSuccess: ({ providers: renewedProviders }) => {
3654
+ for (const provider of renewedProviders) {
3655
+ observability.incrementCounter({
3656
+ name: "opengeni_git_credential_renewals_total",
3657
+ help: "Host-managed Git credential renewal attempts by provider and outcome.",
3658
+ labels: { provider, outcome: "completed" },
3659
+ });
3660
+ }
3661
+ },
3662
+ onFailure: ({ providers: failedProviders, retryDelayMs, errorClass }) => {
3663
+ for (const provider of failedProviders) {
3664
+ observability.incrementCounter({
3665
+ name: "opengeni_git_credential_renewals_total",
3666
+ help: "Host-managed Git credential renewal attempts by provider and outcome.",
3667
+ labels: { provider, outcome: "error" },
3668
+ });
3669
+ }
3670
+ observability.warn("Sandbox Git credential renewal failed; retry scheduled", {
3671
+ sessionId: input.sessionId,
3672
+ turnId,
3673
+ providers: failedProviders.join(","),
3674
+ errorClass,
3675
+ retryDelayMs,
3005
3676
  });
3006
- }
3007
- observability.warn("Sandbox Git credential renewal failed; retry scheduled", {
3677
+ },
3678
+ });
3679
+ });
3680
+ if (gitCredentialRenewalClosed) {
3681
+ await Promise.all(controllers.map(async (controller) => await controller.stop()));
3682
+ return;
3683
+ }
3684
+ gitCredentialRenewals = controllers;
3685
+ };
3686
+
3687
+ const attachToolspaceTokenRenewal = async (
3688
+ tokenSession: ToolspaceTokenWriterSession,
3689
+ initialExpiresAt = sandboxToolspaceTokenExpiresAt,
3690
+ ): Promise<void> => {
3691
+ if (!sandboxToolspaceToken || !initialExpiresAt) return;
3692
+ const previous = toolspaceTokenRenewal;
3693
+ toolspaceTokenRenewal = null;
3694
+ await previous?.stop();
3695
+ if (toolspaceTokenRenewalClosed) return;
3696
+
3697
+ const mint = async () =>
3698
+ await mintSandboxToolspaceToken(runSettings, connectionScope, input.sessionId, turn.id);
3699
+ const write = async (material: NonNullable<Awaited<ReturnType<typeof mint>>>) => {
3700
+ const runAs = sandboxRunAs(runSettings);
3701
+ await refreshToolspaceTokenFile(tokenSession, material.token, {
3702
+ ...(runAs ? { runAs } : {}),
3703
+ ...(sandboxToolspaceTokenFile
3704
+ ? {
3705
+ tokenFile: sandboxToolspaceTokenFile,
3706
+ legacyTokenFile: sandboxEnvironment.OPENGENI_TOOLSPACE_TOKEN_FILE!,
3707
+ }
3708
+ : {}),
3709
+ ...(toolCancellationFenceRef.current
3710
+ ? {
3711
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
3712
+ toolCancellationFenceRef.current,
3713
+ ),
3714
+ }
3715
+ : {}),
3716
+ });
3717
+ };
3718
+ let renewalExpiresAt = initialExpiresAt;
3719
+ if (renewalExpiresAt.getTime() <= Date.now() + TOOLSPACE_TOKEN_EXPIRY_LEAD_MS) {
3720
+ const fresh = await mint();
3721
+ if (!fresh) {
3722
+ throw new Error("Toolspace token mint became unavailable during sandbox setup");
3723
+ }
3724
+ await write(fresh);
3725
+ renewalExpiresAt = fresh.expiresAt;
3726
+ }
3727
+ const controller = startToolspaceTokenRenewalLoop({
3728
+ initialExpiresAt: renewalExpiresAt,
3729
+ mint,
3730
+ write,
3731
+ onSuccess: () => {
3732
+ observability.incrementCounter({
3733
+ name: "opengeni_toolspace_token_renewals_total",
3734
+ help: "Sandbox Toolspace token renewal attempts by outcome.",
3735
+ labels: { outcome: "completed" },
3736
+ });
3737
+ },
3738
+ onFailure: ({ retryDelayMs, errorClass }) => {
3739
+ observability.incrementCounter({
3740
+ name: "opengeni_toolspace_token_renewals_total",
3741
+ help: "Sandbox Toolspace token renewal attempts by outcome.",
3742
+ labels: { outcome: "error" },
3743
+ });
3744
+ observability.warn("Sandbox Toolspace token renewal failed; retry scheduled", {
3008
3745
  sessionId: input.sessionId,
3009
3746
  turnId,
3010
- providers: failedProviders.join(","),
3011
3747
  errorClass,
3012
3748
  retryDelayMs,
3013
3749
  });
3014
3750
  },
3015
3751
  });
3016
- if (gitCredentialRenewalClosed) {
3752
+ if (toolspaceTokenRenewalClosed) {
3753
+ await controller.stop();
3754
+ return;
3755
+ }
3756
+ toolspaceTokenRenewal = controller;
3757
+ };
3758
+
3759
+ const attachRunCredentialRenewal = async (
3760
+ credentialSession: RunCredentialCommandSession,
3761
+ ): Promise<void> => {
3762
+ if (!runCredentialResolver) return;
3763
+ const previous = runCredentialRenewal;
3764
+ runCredentialRenewal = null;
3765
+ await previous?.stop();
3766
+ if (runCredentialRenewalClosed) return;
3767
+ runCredentialSession = credentialSession;
3768
+
3769
+ if (!initialRunCredentialMaterial) {
3770
+ await clearRunCredentials(
3771
+ credentialSession,
3772
+ input.sessionId,
3773
+ toolCancellationFenceRef.current
3774
+ ? toolCancellationFenceRef.current.runSandboxCommand.bind(
3775
+ toolCancellationFenceRef.current,
3776
+ )
3777
+ : undefined,
3778
+ );
3779
+ return;
3780
+ }
3781
+
3782
+ const write = async (
3783
+ material: NormalizedRunCredentialMaterial | null,
3784
+ pruneOtherAttempts = false,
3785
+ ): Promise<void> => {
3786
+ if (!material) {
3787
+ await clearRunCredentialsForAttempt(credentialSession, {
3788
+ sessionId: input.sessionId,
3789
+ attemptId: input.attemptId,
3790
+ executionGeneration,
3791
+ });
3792
+ return;
3793
+ }
3794
+ registerSecretRedactions(material.redactions);
3795
+ await materializeRunCredentials(credentialSession, material, {
3796
+ sessionId: input.sessionId,
3797
+ attemptId: input.attemptId,
3798
+ executionGeneration,
3799
+ ...(pruneOtherAttempts ? { pruneOtherAttempts: true } : {}),
3800
+ ...(!pruneOtherAttempts ? { pruneSupersededGenerations: true } : {}),
3801
+ ...(material.authNeeded.length > 0 &&
3802
+ Object.keys(material.environment).length === 0 &&
3803
+ material.files.length === 0
3804
+ ? { prunePreviousGenerations: true }
3805
+ : {}),
3806
+ ...(toolCancellationFenceRef.current
3807
+ ? {
3808
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
3809
+ toolCancellationFenceRef.current,
3810
+ ),
3811
+ }
3812
+ : {}),
3813
+ });
3814
+ for (const payload of runCredentialAuthNeededPayloads(material)) {
3815
+ const key = JSON.stringify(payload);
3816
+ if (publishedRunCredentialNotices.has(key)) continue;
3817
+ publishedRunCredentialNotices.add(key);
3818
+ await publish!([{ type: "credential.auth_needed", payload }], true);
3819
+ }
3820
+ };
3821
+
3822
+ const initialExpiryMs = initialRunCredentialMaterial.expiresAt?.getTime() ?? null;
3823
+ const seed =
3824
+ initialExpiryMs !== null && initialExpiryMs <= Date.now() + RUN_CREDENTIAL_EXPIRY_LEAD_MS
3825
+ ? await runCredentialResolver.resolve({
3826
+ purpose: "provision",
3827
+ forceRefresh: true,
3828
+ })
3829
+ : initialRunCredentialMaterial;
3830
+ await write(seed, true);
3831
+ if (runCredentialRenewalClosed) return;
3832
+ const controller = startRunCredentialRenewalLoop({
3833
+ initialExpiresAt: seed?.expiresAt ?? null,
3834
+ resolve: async () =>
3835
+ await runCredentialResolver.resolve({
3836
+ purpose: "renewal",
3837
+ forceRefresh: true,
3838
+ }),
3839
+ write: async (material) => await write(material),
3840
+ onSuccess: ({ authNeeded }) => {
3841
+ observability.incrementCounter({
3842
+ name: "opengeni_run_credential_renewals_total",
3843
+ help: "Host-managed run credential renewal attempts by outcome.",
3844
+ labels: { outcome: authNeeded ? "auth_needed" : "completed" },
3845
+ });
3846
+ },
3847
+ onFailure: ({ retryDelayMs, errorClass }) => {
3848
+ observability.incrementCounter({
3849
+ name: "opengeni_run_credential_renewals_total",
3850
+ help: "Host-managed run credential renewal attempts by outcome.",
3851
+ labels: { outcome: "error" },
3852
+ });
3853
+ observability.warn("Host run credential renewal failed; retry scheduled", {
3854
+ sessionId: input.sessionId,
3855
+ turnId,
3856
+ errorClass,
3857
+ retryDelayMs,
3858
+ });
3859
+ },
3860
+ });
3861
+ if (runCredentialRenewalClosed) {
3017
3862
  await controller.stop();
3018
3863
  return;
3019
3864
  }
3020
- gitCredentialRenewal = controller;
3865
+ runCredentialRenewal = controller;
3021
3866
  };
3022
3867
 
3023
3868
  // P1.2 ownership inversion (gated, default OFF). With the flag off this
@@ -3083,16 +3928,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3083
3928
  // (buildSelfhostedBackendSession); EstablishedSandboxSession widens it.
3084
3929
  machinePrimarySession =
3085
3930
  established.session as import("@opengeni/runtime").SelfhostedSession;
3086
- const lease = await acquireSelfhostedLeaseForTurn(
3087
- { db, settings },
3088
- {
3089
- accountId: input.accountId,
3090
- workspaceId: input.workspaceId,
3091
- sandboxGroupId: session.sandboxGroupId,
3092
- sessionId: input.sessionId,
3093
- },
3094
- "turn",
3095
- sandboxHolderId,
3931
+ const lease = await waitForTurnOperation(
3932
+ acquireSelfhostedLeaseForTurn(
3933
+ { db, settings },
3934
+ {
3935
+ accountId: input.accountId,
3936
+ workspaceId: input.workspaceId,
3937
+ sandboxGroupId: session.sandboxGroupId,
3938
+ sessionId: input.sessionId,
3939
+ },
3940
+ "turn",
3941
+ sandboxHolderId,
3942
+ ),
3943
+ cancellationSignal,
3944
+ async (lateLease) => await lateLease.release(),
3096
3945
  );
3097
3946
  setupBoxSession = established.session;
3098
3947
  resolvedSandbox = {
@@ -3137,45 +3986,49 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3137
3986
  // proxy's first default-pointer op. A chat-only turn never calls it, so
3138
3987
  // no lease row, no provider box, no warm-meter interval.
3139
3988
  } else {
3140
- resolvedSandbox = await resumeBoxForTurn(
3141
- {
3142
- db,
3143
- settings,
3144
- sandboxMetrics: runtimeMetricsHooksForObservability(observability),
3145
- onSandboxLost: publishSandboxLost,
3146
- },
3147
- {
3148
- accountId: input.accountId,
3149
- workspaceId: input.workspaceId,
3150
- sandboxGroupId: session.sandboxGroupId,
3151
- sessionId: input.sessionId,
3152
- // groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
3153
- // is not machine-primary resumes a real cloud group box (the
3154
- // deployment default), never a "selfhosted" box (which would throw
3155
- // for lack of a bound agentId).
3156
- backend: groupBoxBackend,
3157
- os: session.sandboxOs,
3158
- environment: sandboxEnvironment,
3159
- // IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
3160
- // this run resolves. The lease stamps it + conflicts on a live shared box
3161
- // running a DIFFERENT image (solo → recreate on the new image; N-holders →
3162
- // SandboxImageConflictError surfaced as an actionable turn error). Prefer the
3163
- // explicit Modal image ref, else the docker image. The selfhosted branch
3164
- // (establishSelfhostedTurnSession/acquireSelfhostedLeaseForTurn) NEVER passes
3165
- // an image — B3 lives only on this Modal else-branch.
3166
- ...((runSettings.modalImageRef ?? runSettings.dockerImage)
3167
- ? {
3168
- image: runSettings.modalImageRef ?? runSettings.dockerImage,
3169
- }
3170
- : {}),
3171
- // RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
3172
- // conflicts on a live shared box set up under a different rig (solo
3173
- // recreate / N-holders SandboxRigConflictError). Omitted for a rig-less
3174
- // turn -> never stamped or enforced (shares exactly as today).
3175
- ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
3176
- },
3177
- "turn",
3178
- sandboxHolderId,
3989
+ resolvedSandbox = await waitForTurnOperation(
3990
+ resumeBoxForTurn(
3991
+ {
3992
+ db,
3993
+ settings,
3994
+ sandboxMetrics: runtimeMetricsHooksForObservability(observability),
3995
+ onSandboxLost: publishSandboxLost,
3996
+ },
3997
+ {
3998
+ accountId: input.accountId,
3999
+ workspaceId: input.workspaceId,
4000
+ sandboxGroupId: session.sandboxGroupId,
4001
+ sessionId: input.sessionId,
4002
+ // groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
4003
+ // is not machine-primary resumes a real cloud group box (the
4004
+ // deployment default), never a "selfhosted" box (which would throw
4005
+ // for lack of a bound agentId).
4006
+ backend: groupBoxBackend,
4007
+ os: session.sandboxOs,
4008
+ environment: sandboxEnvironment,
4009
+ // IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
4010
+ // this run resolves. The lease stamps it + conflicts on a live shared box
4011
+ // running a DIFFERENT image (solo → recreate on the new image; N-holders →
4012
+ // SandboxImageConflictError surfaced as an actionable turn error). Prefer the
4013
+ // explicit Modal image ref, else the docker image. The selfhosted branch
4014
+ // (establishSelfhostedTurnSession/acquireSelfhostedLeaseForTurn) NEVER passes
4015
+ // an image — B3 lives only on this Modal else-branch.
4016
+ ...((runSettings.modalImageRef ?? runSettings.dockerImage)
4017
+ ? {
4018
+ image: runSettings.modalImageRef ?? runSettings.dockerImage,
4019
+ }
4020
+ : {}),
4021
+ // RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
4022
+ // conflicts on a live shared box set up under a different rig (solo
4023
+ // recreate / N-holders SandboxRigConflictError). Omitted for a rig-less
4024
+ // turn -> never stamped or enforced (shares exactly as today).
4025
+ ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
4026
+ },
4027
+ "turn",
4028
+ sandboxHolderId,
4029
+ ),
4030
+ cancellationSignal,
4031
+ async (lateSandbox) => await lateSandbox.release(),
3179
4032
  );
3180
4033
  setupBoxSession = resolvedSandbox.established.session;
3181
4034
  // Durable box-lifecycle events (sandbox-file-persistence observability):
@@ -3216,42 +4069,65 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3216
4069
  }
3217
4070
  }
3218
4071
 
3219
- const fileResourceDownloads = await sandboxFileDownloadsForRun(
3220
- runSettings,
3221
- db,
3222
- objectStorage,
3223
- input.workspaceId,
3224
- turnResources,
4072
+ const fileResourceDownloads = await waitForTurnOperation(
4073
+ sandboxFileDownloadsForRun(
4074
+ runSettings,
4075
+ db,
4076
+ objectStorage,
4077
+ input.workspaceId,
4078
+ turnResources,
4079
+ ),
4080
+ cancellationSignal,
4081
+ undefined,
3225
4082
  );
3226
4083
  throwIfWorkerShuttingDown();
4084
+ throwIfTurnCancelled();
4085
+ const mcpCredentialRootSessionId =
4086
+ connectionCredentials?.mcpCredentials && hostCredentialRootSessionId
4087
+ ? hostCredentialRootSessionId
4088
+ : input.sessionId;
3227
4089
  // Wrap MCP prep in the codex ALS so the codex_apps connect handshake
3228
4090
  // (initialize + tools/list) can resolve the per-workspace bearer from
3229
4091
  // codexRequestStorage (runtime/codexAppsMcpRequestInit). withCodex is the
3230
4092
  // identity on every non-codex turn, so this is a no-op for existing paths.
3231
- const resolveCredential = buildConnectionTokenResolver(db, runSettings);
3232
- preparedTools = await withCodex(() =>
3233
- runtime.prepareTools(runSettings, turnTools, {
3234
- accountId: input.accountId,
3235
- workspaceId: input.workspaceId,
3236
- sessionId: input.sessionId,
3237
- // Sign the calling turn into the first-party token so tools classify
3238
- // the caller by its own identity (sacred-pause guard), not the racy
3239
- // live active pointer.
3240
- ...(turnId ? { turnId } : {}),
3241
- attemptId: input.attemptId,
3242
- executionGeneration,
3243
- subjectId: "worker:first-party-mcp",
3244
- subjectLabel: "OpenGeni worker",
3245
- resolveCredential,
3246
- onAuthNeeded: async (payload) => {
3247
- await publish!([{ type: "tool.auth_needed", payload }], true);
3248
- },
3249
- // Manager-style sessions carry a creation-validated permission set
3250
- // for their first-party MCP token; null keeps the fixed default.
3251
- ...(session.firstPartyMcpPermissions?.length
3252
- ? { firstPartyPermissions: session.firstPartyMcpPermissions }
3253
- : {}),
3254
- }),
4093
+ const resolveCredential = connectionTokenResolverForTurn({
4094
+ db,
4095
+ settings: runSettings,
4096
+ connectionCredentials: connectionCredentials ?? null,
4097
+ accountId: input.accountId,
4098
+ workspaceId: input.workspaceId,
4099
+ sessionId: input.sessionId,
4100
+ rootSessionId: mcpCredentialRootSessionId,
4101
+ attemptId: input.attemptId,
4102
+ turn,
4103
+ });
4104
+ preparedTools = await waitForTurnOperation(
4105
+ withCodex(() =>
4106
+ runtime.prepareTools(runSettings, turnTools, {
4107
+ accountId: input.accountId,
4108
+ workspaceId: input.workspaceId,
4109
+ sessionId: input.sessionId,
4110
+ // Sign the calling turn into the first-party token so tools classify
4111
+ // the caller by its own identity (sacred-pause guard), not the racy
4112
+ // live active pointer.
4113
+ ...(turnId ? { turnId } : {}),
4114
+ attemptId: input.attemptId,
4115
+ executionGeneration,
4116
+ subjectId: "worker:first-party-mcp",
4117
+ subjectLabel: "OpenGeni worker",
4118
+ resolveCredential,
4119
+ onAuthNeeded: async (payload) => {
4120
+ await publish!([{ type: "tool.auth_needed", payload }], true);
4121
+ },
4122
+ // Manager-style sessions carry a creation-validated permission set
4123
+ // for their first-party MCP token; null keeps the fixed default.
4124
+ ...(session.firstPartyMcpPermissions?.length
4125
+ ? { firstPartyPermissions: session.firstPartyMcpPermissions }
4126
+ : {}),
4127
+ }),
4128
+ ),
4129
+ cancellationSignal,
4130
+ async (latePreparedTools) => await latePreparedTools.close().catch(() => undefined),
3255
4131
  );
3256
4132
  // Genesis turn = the first user turn (no assistant history reconciled
3257
4133
  // yet). Durable Postgres state (countSessionHistoryItems includes
@@ -3292,11 +4168,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3292
4168
  : {};
3293
4169
  const agent = runtime.buildAgent(modelRunSettings, turnResources, {
3294
4170
  reasoningEffort: turn.reasoningEffort,
4171
+ ...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
3295
4172
  genesisTitleHint: isGenesisTurn,
3296
4173
  persistentSessionSettings: {
3297
4174
  titleIsSet: Boolean(session.title?.trim()),
3298
4175
  },
3299
4176
  sandboxEnvironment,
4177
+ ...(cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {}),
4178
+ onToolCancellationFence: (fence) => {
4179
+ toolCancellationFenceRef.current = fence;
4180
+ },
3300
4181
  // TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
3301
4182
  // seed. ONLY when the effective backend is NOT selfhosted (the connected
3302
4183
  // machine uses its own git creds — mirrors the skipGitHubToken gate above)
@@ -3306,6 +4187,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3306
4187
  ...(activeSandboxBackend !== "selfhosted" && sandboxGitTokens
3307
4188
  ? { gitTokenSeeds: sandboxGitTokens }
3308
4189
  : {}),
4190
+ ...(activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings
4191
+ ? { gitCredentialBindings: sandboxGitCredentialBindings }
4192
+ : {}),
3309
4193
  ...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
3310
4194
  ? { gitTokenSeed: sandboxGitToken }
3311
4195
  : {}),
@@ -3316,7 +4200,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3316
4200
  // than toolspace:call for its own session (own-session-bound, turn TTL,
3317
4201
  // budgeted, approval-tools excluded). The runtime seeds it to the box's
3318
4202
  // token file over the same exec channel, off-manifest, on every backend.
3319
- ...(sandboxToolspaceToken ? { toolspaceTokenSeed: sandboxToolspaceToken } : {}),
4203
+ ...(sandboxToolspaceToken
4204
+ ? {
4205
+ toolspaceTokenSeed: sandboxToolspaceToken,
4206
+ toolspaceTokenSessionId: input.sessionId,
4207
+ }
4208
+ : {}),
3320
4209
  ...(activeSandboxBackend ? { activeSandboxBackend } : {}),
3321
4210
  fileResourceDownloads,
3322
4211
  mcpServers: preparedTools.mcpServers,
@@ -3384,12 +4273,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3384
4273
  await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
3385
4274
  },
3386
4275
  ...(packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {}),
4276
+ ...(skillLibraryRuntime.skillLibrarySkills.length > 0
4277
+ ? {
4278
+ skillLibrarySkills: skillLibraryRuntime.skillLibrarySkills,
4279
+ skillLibrarySelections: skillLibraryRuntime.skillLibrarySelections,
4280
+ }
4281
+ : {}),
3387
4282
  ...(workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {}),
3388
4283
  ...(workspaceMemory ? { workspaceMemory } : {}),
3389
4284
  // Per-session persona tier (session > workspace > deployment default).
3390
4285
  // Composed system-level AFTER the workspace persona so it refines it for
3391
4286
  // this one session; absent ⇒ byte-identical to today's composition.
3392
4287
  ...(session.instructions ? { sessionInstructions: session.instructions } : {}),
4288
+ // Exact host context captured when this turn was accepted. It is
4289
+ // system-level and disappears with the turn rather than entering chat.
4290
+ ...(turn.turnInstructions ? { turnInstructions: turn.turnInstructions } : {}),
3393
4291
  ...workspaceEnvironmentOption,
3394
4292
  // RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
3395
4293
  // the frozen version carries a non-empty script), and the rig credential
@@ -3414,6 +4312,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3414
4312
  }
3415
4313
  : {}),
3416
4314
  });
4315
+ if (modelRunSettings.sandboxBackend !== "none" && toolCancellationFenceRef.current === null) {
4316
+ throw new Error(
4317
+ "Sandbox agent construction did not install the mandatory turn tool cancellation fence",
4318
+ );
4319
+ }
3417
4320
  if (establishPolicy === "on-demand" && sandboxHolderId && sandboxGroupId) {
3418
4321
  const lazyHolderId = sandboxHolderId;
3419
4322
  const lazyGroupId = sandboxGroupId;
@@ -3424,18 +4327,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3424
4327
  const lazyClient = {
3425
4328
  backendId: sdkBackendIdForSandboxBackend(groupBoxBackend),
3426
4329
  } as EstablishedSandboxSession["client"];
3427
- let lazyEstablishmentOrigin: EstablishedSandboxSession["origin"] | null = null;
3428
4330
  turnSandboxProvisioner = createTurnSandboxProvisioner<ResumedTurnSandbox>(
3429
4331
  async () => {
3430
4332
  throwIfWorkerShuttingDown();
4333
+ throwIfTurnCancelled();
3431
4334
  const lazyGitCredentials =
3432
4335
  activeSandboxBackend === "selfhosted"
3433
4336
  ? undefined
3434
4337
  : await mintRunGitCredentials(runSettings, turnResources, {
3435
4338
  scope: connectionScope,
4339
+ ...(gitCredentialAuthority ? { authority: gitCredentialAuthority } : {}),
3436
4340
  gitCredentials: connectionCredentials?.gitCredentials,
4341
+ authorizeGitHubTokenMint,
3437
4342
  });
3438
4343
  const lazyGitTokens = lazyGitCredentials?.gitTokens;
4344
+ const lazyToolspaceToken = sandboxToolspaceToken
4345
+ ? await mintSandboxToolspaceToken(
4346
+ runSettings,
4347
+ connectionScope,
4348
+ input.sessionId,
4349
+ turn.id,
4350
+ )
4351
+ : undefined;
3439
4352
  const provisioned = await resumeBoxForTurn(
3440
4353
  {
3441
4354
  db,
@@ -3460,13 +4373,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3460
4373
  "turn",
3461
4374
  lazyHolderId,
3462
4375
  );
3463
- lazyEstablishmentOrigin = provisioned.established.origin ?? null;
3464
- setupBoxSession = provisioned.established.session;
3465
4376
  await publishSandboxLifecycleEvents(provisioned);
4377
+ await attachRunCredentialRenewal(
4378
+ provisioned.established.session as RunCredentialCommandSession,
4379
+ );
4380
+ const provisionedSetupSession = initialRunCredentialMaterial
4381
+ ? withRunCredentialsSession(
4382
+ provisioned.established.session as object,
4383
+ input.sessionId,
4384
+ )
4385
+ : provisioned.established.session;
3466
4386
  await runOwnedSandboxSetup(
3467
4387
  agent,
3468
4388
  provisioned.established.session as never,
3469
- provisioned.established.session as never,
4389
+ provisionedSetupSession as never,
3470
4390
  {
3471
4391
  settings: runSettings,
3472
4392
  environment: sandboxEnvironment,
@@ -3474,8 +4394,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3474
4394
  await publish?.([{ type: event.type, payload: event.payload }], true);
3475
4395
  },
3476
4396
  ...(lazyGitTokens ? { gitTokenSeedsOverride: lazyGitTokens } : {}),
4397
+ ...(lazyGitCredentials?.bindings
4398
+ ? { gitCredentialBindingsOverride: lazyGitCredentials.bindings }
4399
+ : {}),
4400
+ ...(lazyToolspaceToken
4401
+ ? { toolspaceTokenSeedOverride: lazyToolspaceToken.token }
4402
+ : {}),
4403
+ ...(toolCancellationFenceRef.current
4404
+ ? {
4405
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4406
+ toolCancellationFenceRef.current,
4407
+ ),
4408
+ }
4409
+ : {}),
3477
4410
  },
3478
4411
  );
4412
+ await attachToolspaceTokenRenewal(
4413
+ provisioned.established.session as ToolspaceTokenWriterSession,
4414
+ lazyToolspaceToken?.expiresAt,
4415
+ );
3479
4416
  await attachGitCredentialRenewal(
3480
4417
  provisioned.established.session as GitCredentialTokenWriterSession,
3481
4418
  lazyGitCredentials,
@@ -3488,11 +4425,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3488
4425
  // already holds the proxy directly (injected as lazyOwnedSandbox.session), so it
3489
4426
  // gets per-op routing; the worker-side handle (resolvedSandbox: release,
3490
4427
  // heartbeat, computer-use recording) wants the real box, unproxied.
3491
- resolvedSandbox = provisioned;
3492
- startLeaseHeartbeat(provisioned, activeSandboxBackend ?? groupBoxBackend);
3493
4428
  return provisioned;
3494
4429
  },
3495
4430
  {
4431
+ ...(activityContext ? { signal: activityContext.cancellationSignal } : {}),
3496
4432
  onStarted: async () => {
3497
4433
  await publish?.(
3498
4434
  [
@@ -3504,19 +4440,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3504
4440
  true,
3505
4441
  );
3506
4442
  },
3507
- onCompleted: async () => {
4443
+ onCompleted: async (provisioned) => {
3508
4444
  await publish?.(
3509
4445
  [
3510
4446
  {
3511
4447
  type: "sandbox.operation.completed",
3512
4448
  payload: {
3513
4449
  name: "sandbox.provision",
3514
- ...(lazyEstablishmentOrigin ? { origin: lazyEstablishmentOrigin } : {}),
4450
+ ...(provisioned.established.origin
4451
+ ? { origin: provisioned.established.origin }
4452
+ : {}),
3515
4453
  },
3516
4454
  },
3517
4455
  ],
3518
4456
  true,
3519
4457
  );
4458
+ throwIfTurnOperationCancelled(activityContext?.cancellationSignal);
4459
+ startLeaseHeartbeat(provisioned, activeSandboxBackend ?? groupBoxBackend);
4460
+ setupBoxSession = provisioned.established.session;
4461
+ resolvedSandbox = provisioned;
3520
4462
  },
3521
4463
  onFailed: async (error) => {
3522
4464
  await publish?.(
@@ -3532,6 +4474,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3532
4474
  true,
3533
4475
  );
3534
4476
  },
4477
+ disposeResult: async (provisioned) => {
4478
+ await provisioned.release().catch(() => undefined);
4479
+ },
3535
4480
  },
3536
4481
  );
3537
4482
  lazyOwnedSandbox = wrapLazyTurnBoxWithRouting(
@@ -3566,36 +4511,41 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3566
4511
  // RunState verbatim and recovering attempts already compacted, if needed,
3567
4512
  // before the first attempt's model boundary.
3568
4513
  if (triggerType === "user.message" || triggerType === "system.update.delivered") {
4514
+ let forced = false;
3569
4515
  try {
3570
4516
  // Operator /compact (the slash command) sets a durable request flag;
3571
4517
  // observe it without consuming it so a failed/stale attempt cannot
3572
4518
  // lose the request. The replacement transaction clears it on success.
3573
- const forced = await isSessionCompactionRequested(db, input.workspaceId, input.sessionId);
3574
- const outcome = await maybeCompactContext(
3575
- db,
3576
- modelRunSettings,
3577
- {
3578
- accountId: input.accountId,
3579
- workspaceId: input.workspaceId,
3580
- sessionId: input.sessionId,
3581
- turnId: turnId!,
3582
- executionGeneration,
3583
- attemptId: input.attemptId,
3584
- },
3585
- session.lastInputTokens,
3586
- // Provider-aware summarizer: when the turn's model resolved to a
3587
- // registry provider, summarize on THAT provider's client + wire API
3588
- // (a chat provider can't summarize through OpenAI/Azure). Null
3589
- // resolution uses the built-in Responses summarizer with the same
3590
- // session prompt-cache key as the main model calls.
3591
- compactSummarizer,
3592
- forced
3593
- ? {
3594
- force: true,
3595
- clearRequestedCompaction: true,
3596
- trigger: "operator",
3597
- }
3598
- : { trigger: "auto" },
4519
+ forced = await isSessionCompactionRequested(db, input.workspaceId, input.sessionId);
4520
+ const outcome = await waitForTurnOperation(
4521
+ maybeCompactContext(
4522
+ db,
4523
+ modelRunSettings,
4524
+ {
4525
+ accountId: input.accountId,
4526
+ workspaceId: input.workspaceId,
4527
+ sessionId: input.sessionId,
4528
+ turnId: turnId!,
4529
+ executionGeneration,
4530
+ attemptId: input.attemptId,
4531
+ },
4532
+ session.lastInputTokens,
4533
+ // Provider-aware summarizer: when the turn's model resolved to a
4534
+ // registry provider, summarize on THAT provider's client + wire API
4535
+ // (a chat provider can't summarize through OpenAI/Azure). Null
4536
+ // resolution uses the built-in Responses summarizer with the same
4537
+ // session prompt-cache key as the main model calls.
4538
+ compactSummarizer,
4539
+ forced
4540
+ ? {
4541
+ force: true,
4542
+ clearRequestedCompaction: true,
4543
+ trigger: "operator",
4544
+ }
4545
+ : { trigger: "auto" },
4546
+ ),
4547
+ cancellationSignal,
4548
+ undefined,
3599
4549
  );
3600
4550
  if (outcome.compacted) {
3601
4551
  const compactionTrigger = forced ? "operator" : undefined;
@@ -3608,12 +4558,41 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3608
4558
  );
3609
4559
  }
3610
4560
  } catch (compactError) {
4561
+ if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
4562
+ if (!isCompactionSummaryFailure(compactError)) throw compactError;
4563
+ const errorMessage = compactionFailureReasonFromError(compactError);
3611
4564
  observability.error("context compaction failed", {
3612
4565
  sessionId: input.sessionId,
3613
4566
  turnId,
3614
- error: compactError instanceof Error ? compactError.message : String(compactError),
4567
+ error: errorMessage,
3615
4568
  });
3616
- throw compactError;
4569
+ if (
4570
+ !(await settle!({
4571
+ events: [
4572
+ {
4573
+ type: "turn.failed",
4574
+ payload: {
4575
+ error: errorMessage,
4576
+ code: "context_compaction_failed",
4577
+ retryable: false,
4578
+ recovery: "user_message",
4579
+ compacted: false,
4580
+ },
4581
+ },
4582
+ { type: "session.status.changed", payload: { status: "idle" } },
4583
+ ],
4584
+ turnStatus: "failed",
4585
+ sessionStatus: "idle",
4586
+ activeTurnId: null,
4587
+ ...(forced ? { consumeRequestedCompactionFailure: true } : {}),
4588
+ }))
4589
+ ) {
4590
+ return claimedResult({ status: "cancelled" });
4591
+ }
4592
+ turnMetricOutcome = "failed";
4593
+ activityStatus = "idle";
4594
+ activityError = compactError;
4595
+ return claimedResult({ status: "idle" });
3617
4596
  }
3618
4597
  }
3619
4598
  let fileMaterializationFailures: SandboxFileDownloadFailure[] = [];
@@ -3646,6 +4625,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3646
4625
  await publish!([{ type: event.type, payload: event.payload }], true);
3647
4626
  },
3648
4627
  ...(runAs ? { runAs } : {}),
4628
+ ...(toolCancellationFenceRef.current
4629
+ ? {
4630
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4631
+ toolCancellationFenceRef.current,
4632
+ ),
4633
+ }
4634
+ : {}),
3649
4635
  },
3650
4636
  );
3651
4637
  fileMaterializationFailures = materialized.failures;
@@ -3693,6 +4679,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3693
4679
  turnId: activeTurnId,
3694
4680
  recovering: turn.executionGeneration > 1,
3695
4681
  ...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
4682
+ ...(runCredentialsNote ? { runCredentialsNote } : {}),
3696
4683
  },
3697
4684
  );
3698
4685
  runInput = prepared.input;
@@ -3740,29 +4727,33 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3740
4727
  triggerLabel: "overflow" | "proactive" | "operator",
3741
4728
  recoverySignalTokens: number | null,
3742
4729
  ) => {
3743
- const outcome = await maybeCompactContext(
3744
- db,
3745
- modelRunSettings,
3746
- {
3747
- accountId: input.accountId,
3748
- workspaceId: input.workspaceId,
3749
- sessionId: input.sessionId,
3750
- turnId: activeTurnId,
3751
- executionGeneration,
3752
- attemptId: input.attemptId,
3753
- },
3754
- // Never reuse the persisted prior-turn signal for recovery. Proactive
3755
- // guards provide their exact current signal; provider overflows do not,
3756
- // so null derives decision metadata from active history. Forced
3757
- // recovery proves progress separately by comparing the replacement
3758
- // with the current active-history estimate in maybeCompactContext.
3759
- recoverySignalTokens,
3760
- compactSummarizer,
3761
- {
3762
- force: true,
3763
- ...(triggerLabel === "operator" ? { clearRequestedCompaction: true } : {}),
3764
- trigger: triggerLabel,
3765
- },
4730
+ const outcome = await waitForTurnOperation(
4731
+ maybeCompactContext(
4732
+ db,
4733
+ modelRunSettings,
4734
+ {
4735
+ accountId: input.accountId,
4736
+ workspaceId: input.workspaceId,
4737
+ sessionId: input.sessionId,
4738
+ turnId: activeTurnId,
4739
+ executionGeneration,
4740
+ attemptId: input.attemptId,
4741
+ },
4742
+ // Never reuse the persisted prior-turn signal for recovery. Proactive
4743
+ // guards provide their exact current signal; provider overflows do not,
4744
+ // so null derives decision metadata from active history. Forced
4745
+ // recovery proves progress separately by comparing the replacement
4746
+ // with the current active-history estimate in maybeCompactContext.
4747
+ recoverySignalTokens,
4748
+ compactSummarizer,
4749
+ {
4750
+ force: true,
4751
+ ...(triggerLabel === "operator" ? { clearRequestedCompaction: true } : {}),
4752
+ trigger: triggerLabel,
4753
+ },
4754
+ ),
4755
+ cancellationSignal,
4756
+ undefined,
3766
4757
  );
3767
4758
  if (outcome.events.length > 0) {
3768
4759
  if (outcome.compacted) {
@@ -3785,10 +4776,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3785
4776
  stream = undefined;
3786
4777
  batcher = null;
3787
4778
  let responseUsageCount = 0;
4779
+ // The SDK emits every processed call item for one model response before
4780
+ // it emits any result for that response. Keep that response-local batch
4781
+ // in memory so an orphan from an older response cannot pin later stable
4782
+ // calls. Durable recovery remains call/result based and needs no batch
4783
+ // schema or compatibility state.
4784
+ let currentToolBatchCallIds = new Set<string>();
4785
+ let currentToolBatchCompletedCallIds = new Set<string>();
3788
4786
  // Actual input tokens of the most recent model response this turn; the
3789
4787
  // pre-read trigger for the NEXT turn. Persisted at every turn-end path.
3790
- let lastInputTokensObserved: number | null = null;
4788
+ let lastProviderContextTokensObserved: number | null = null;
4789
+ let providerContextRevision = 0;
3791
4790
  throwIfWorkerShuttingDown();
4791
+ throwIfTurnCancelled();
3792
4792
  const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
3793
4793
  const runStreamOnce = (): ReturnType<OpenGeniRuntime["runStream"]> =>
3794
4794
  runtime.runStream(agent, runInput!, modelRunSettings, {
@@ -3831,9 +4831,48 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3831
4831
  },
3832
4832
  }
3833
4833
  : {}),
3834
- contextCompactionSignalTokens: () => lastInputTokensObserved,
4834
+ ...(sandboxToolspaceToken && sandboxToolspaceTokenExpiresAt && !lazyOwnedSandbox
4835
+ ? {
4836
+ onToolspaceTokenSessionReady: async (
4837
+ tokenSession: ToolspaceTokenWriterSession,
4838
+ ) => {
4839
+ const renewalSession =
4840
+ activeSandboxBackend === "selfhosted"
4841
+ ? tokenSession
4842
+ : ((setupBoxSession as ToolspaceTokenWriterSession | null) ?? tokenSession);
4843
+ await attachToolspaceTokenRenewal(renewalSession);
4844
+ },
4845
+ }
4846
+ : {}),
4847
+ ...(runCredentialResolver
4848
+ ? {
4849
+ runCredentialSessionId: input.sessionId,
4850
+ ...(!lazyOwnedSandbox
4851
+ ? {
4852
+ onRunCredentialSessionReady: async (
4853
+ credentialSession: RunCredentialCommandSession,
4854
+ ) => {
4855
+ const pinnedCredentialSession = setupBoxSession
4856
+ ? (setupBoxSession as RunCredentialCommandSession)
4857
+ : credentialSession;
4858
+ await attachRunCredentialRenewal(pinnedCredentialSession);
4859
+ },
4860
+ }
4861
+ : {}),
4862
+ }
4863
+ : {}),
4864
+ contextCompactionSignal: () =>
4865
+ lastProviderContextTokensObserved === null
4866
+ ? null
4867
+ : {
4868
+ revision: providerContextRevision,
4869
+ totalTokens: lastProviderContextTokensObserved,
4870
+ },
3835
4871
  contextCompactionRequested: () =>
3836
4872
  isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
4873
+ ...(toolCancellationFenceRef.current
4874
+ ? { turnToolCancellationFence: toolCancellationFenceRef.current }
4875
+ : {}),
3837
4876
  });
3838
4877
  if (codexLeaseLost) {
3839
4878
  throw new Error("Codex credential lease expired before the model run");
@@ -3865,6 +4904,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3865
4904
  streamDone = true;
3866
4905
  break;
3867
4906
  }
4907
+ let stableToolCallIdsToClear: string[] | null = null;
4908
+ let completedCurrentToolBatch = false;
3868
4909
  const responseUsage = modelResponseUsageFromSdkEvent(next.value);
3869
4910
  if (responseUsage) {
3870
4911
  await recordCompletedModelCallBeforeOwnershipFences({
@@ -3890,6 +4931,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3890
4931
  workspaceId: input.workspaceId,
3891
4932
  sessionId: input.sessionId,
3892
4933
  turnId: activeTurnId,
4934
+ turnAttemptId: input.attemptId,
3893
4935
  model: turn.model,
3894
4936
  isCodexTurn,
3895
4937
  usage: responseUsage.usage,
@@ -3915,7 +4957,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3915
4957
  const observed = responseUsage.usage?.inputTokens;
3916
4958
  if (typeof observed === "number" && observed > 0) {
3917
4959
  recordModelInputTokens(observability, streamProvider, observed);
3918
- lastInputTokensObserved = observed;
4960
+ }
4961
+ const observedTotal = providerContextTokens(responseUsage.usage);
4962
+ if (observedTotal !== null) {
4963
+ lastProviderContextTokensObserved = observedTotal;
4964
+ providerContextRevision += 1;
3919
4965
  }
3920
4966
  // Prompt-cache efficiency for this response — same usage frame as the
3921
4967
  // input-token accounting above, so the two are always consistent.
@@ -3931,6 +4977,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3931
4977
  }
3932
4978
  },
3933
4979
  });
4980
+ currentToolBatchCallIds = new Set<string>();
4981
+ currentToolBatchCompletedCallIds = new Set<string>();
3934
4982
  await reconcileConversationTruth();
3935
4983
  try {
3936
4984
  await ensureRunAllowed(
@@ -3973,6 +5021,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3973
5021
  "turn attempt ended while recording an in-flight tool call",
3974
5022
  );
3975
5023
  }
5024
+ currentToolBatchCallIds.add(pendingToolCall.callId);
3976
5025
  }
3977
5026
  const completedToolCall = completedToolCallFromSdkEvent(next.value);
3978
5027
  if (completedToolCall) {
@@ -3994,27 +5043,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3994
5043
  if (!recorded.accepted) {
3995
5044
  throw new TurnAttemptFencedError(
3996
5045
  "turn attempt ended while recording a tool-call result",
3997
- );
3998
- }
3999
- if (recorded.allResultsRecorded) {
4000
- // Persist the SDK's now-stable complete call/result batch before
4001
- // clearing its receipts. A crash between these transactions is
4002
- // harmless: settlement finds both raw results and merely
4003
- // consumes them without fabricating an interruption.
4004
- await reconcileConversationTruth({ requireDurable: true });
4005
- const cleared = await clearDurablePendingSessionToolCalls(db, {
4006
- accountId: input.accountId,
4007
- workspaceId: input.workspaceId,
4008
- sessionId: input.sessionId,
4009
- turnId: activeTurnId,
4010
- executionGeneration,
4011
- attemptId: input.attemptId,
4012
- });
4013
- if (!cleared.accepted) {
4014
- throw new TurnAttemptFencedError(
4015
- "turn attempt ended while finalizing tool-call results",
4016
- );
4017
- }
5046
+ );
5047
+ }
5048
+ const belongsToCurrentBatch = currentToolBatchCallIds.has(completedToolCall.callId);
5049
+ if (belongsToCurrentBatch) {
5050
+ currentToolBatchCompletedCallIds.add(completedToolCall.callId);
5051
+ }
5052
+ const currentBatchIsStable =
5053
+ belongsToCurrentBatch &&
5054
+ currentToolBatchCallIds.size > 0 &&
5055
+ currentToolBatchCompletedCallIds.size === currentToolBatchCallIds.size;
5056
+ const standaloneStableResult =
5057
+ !belongsToCurrentBatch && currentToolBatchCallIds.size === 0;
5058
+ if (currentBatchIsStable || standaloneStableResult) {
5059
+ // Persist the SDK's now-stable complete call/result batch. Keep
5060
+ // the receipts until the normalized tool-output event below is
5061
+ // durably flushed: recovery then covers every crash boundary
5062
+ // without either losing or duplicating the UI projection.
5063
+ await reconcileConversationTruth({ requireDurable: true });
5064
+ stableToolCallIdsToClear = currentBatchIsStable
5065
+ ? [...currentToolBatchCallIds]
5066
+ : [completedToolCall.callId];
5067
+ completedCurrentToolBatch = currentBatchIsStable;
4018
5068
  }
4019
5069
  }
4020
5070
  const normalized = normalizeSdkEvent(next.value);
@@ -4022,6 +5072,26 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4022
5072
  streamTiming.onEvent(event.type);
4023
5073
  await batcher.push(event);
4024
5074
  }
5075
+ if (stableToolCallIdsToClear) {
5076
+ const cleared = await clearDurablePendingSessionToolCalls(db, {
5077
+ accountId: input.accountId,
5078
+ workspaceId: input.workspaceId,
5079
+ sessionId: input.sessionId,
5080
+ turnId: activeTurnId,
5081
+ executionGeneration,
5082
+ attemptId: input.attemptId,
5083
+ callIds: stableToolCallIdsToClear,
5084
+ });
5085
+ if (!cleared.accepted) {
5086
+ throw new TurnAttemptFencedError(
5087
+ "turn attempt ended while finalizing tool-call results",
5088
+ );
5089
+ }
5090
+ if (completedCurrentToolBatch) {
5091
+ currentToolBatchCallIds = new Set<string>();
5092
+ currentToolBatchCompletedCallIds = new Set<string>();
5093
+ }
5094
+ }
4025
5095
  }
4026
5096
  } finally {
4027
5097
  if (!streamDone) {
@@ -4036,14 +5106,27 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4036
5106
  void iterator.return?.().catch(() => undefined);
4037
5107
  }
4038
5108
  }
4039
- await batcher.flush();
4040
- await stream.completed.catch(() => undefined);
5109
+ await waitForTurnStreamCleanup(
5110
+ batcher.flush(),
5111
+ stream.completed.catch(() => undefined),
5112
+ cancellationSignal,
5113
+ );
4041
5114
  if (responseUsageCount === 0) {
4042
5115
  const aggregateUsage = stream.state.usage;
4043
5116
  const aggregateInput = (aggregateUsage as { inputTokens?: unknown } | undefined)
4044
5117
  ?.inputTokens;
4045
5118
  if (typeof aggregateInput === "number" && aggregateInput > 0) {
4046
- lastInputTokensObserved = aggregateInput;
5119
+ const aggregateContext = providerContextTokens(
5120
+ aggregateUsage as {
5121
+ inputTokens?: number;
5122
+ outputTokens?: number;
5123
+ totalTokens?: number;
5124
+ },
5125
+ );
5126
+ if (aggregateContext !== null) {
5127
+ lastProviderContextTokensObserved = aggregateContext;
5128
+ providerContextRevision += 1;
5129
+ }
4047
5130
  }
4048
5131
  const aggregateSourceKey = modelUsageSourceKey({
4049
5132
  responseId: null,
@@ -4075,6 +5158,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4075
5158
  workspaceId: input.workspaceId,
4076
5159
  sessionId: input.sessionId,
4077
5160
  turnId: activeTurnId,
5161
+ turnAttemptId: input.attemptId,
4078
5162
  model: turn.model,
4079
5163
  isCodexTurn,
4080
5164
  usage: aggregateUsage,
@@ -4108,23 +5192,59 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4108
5192
  if (stream.interruptions.length > 0) {
4109
5193
  await reconcileConversationTruth();
4110
5194
  const approvals = runtime.serializeApprovals(stream.interruptions);
4111
- await saveRunStateFenced({
4112
- accountId: input.accountId,
4113
- workspaceId: input.workspaceId,
4114
- sessionId: input.sessionId,
4115
- turnId: activeTurnId,
4116
- expectedExecutionGeneration: executionGeneration,
4117
- expectedAttemptId: input.attemptId,
4118
- serializedRunState: stream.state.toString(),
4119
- pendingApprovals: approvals,
4120
- // Record the account freezing this state so a resume on a DIFFERENT
4121
- // codex account strips its account-bound reasoning before replay (HOLE C).
4122
- frozenCodexCredentialId: effectiveCodexCredentialId,
4123
- });
5195
+ const humanInputInterruptions =
5196
+ runtime.serializeHumanInputRequests?.(stream.interruptions) ?? [];
5197
+ const humanInputRequests = await Promise.all(
5198
+ humanInputInterruptions.map(async (interruption) => {
5199
+ const id = stableHumanInputRequestId(
5200
+ input.sessionId,
5201
+ activeTurnId,
5202
+ interruption.toolCallId,
5203
+ );
5204
+ const existing = await getSessionHumanInputRequest(
5205
+ db,
5206
+ input.workspaceId,
5207
+ input.sessionId,
5208
+ id,
5209
+ );
5210
+ if (existing && existing.status !== "pending") {
5211
+ throw new Error(`Settled human-input request ${id} reappeared as an interruption`);
5212
+ }
5213
+ const expiresAt = existing?.expiresAt
5214
+ ? new Date(existing.expiresAt)
5215
+ : interruption.input.expiresInSeconds
5216
+ ? new Date(Date.now() + interruption.input.expiresInSeconds * 1000)
5217
+ : null;
5218
+ return {
5219
+ id,
5220
+ toolCallId: interruption.toolCallId,
5221
+ questions: interruption.input.questions,
5222
+ allowSkip: interruption.input.allowSkip,
5223
+ expiresAt,
5224
+ isNew: existing === null,
5225
+ };
5226
+ }),
5227
+ );
5228
+ const requestEvents = humanInputRequests
5229
+ .filter((request) => request.isNew)
5230
+ .map((request) => ({
5231
+ type: "session.humanInput.requested" as const,
5232
+ payload: {
5233
+ request: {
5234
+ id: request.id,
5235
+ questions: request.questions,
5236
+ allowSkip: request.allowSkip,
5237
+ expiresAt: request.expiresAt?.toISOString() ?? null,
5238
+ },
5239
+ },
5240
+ }));
4124
5241
  if (
4125
5242
  !(await settle!({
4126
5243
  events: [
4127
- { type: "session.requiresAction", payload: { approvals } },
5244
+ ...requestEvents,
5245
+ ...(approvals.length > 0
5246
+ ? [{ type: "session.requiresAction" as const, payload: { approvals } }]
5247
+ : []),
4128
5248
  {
4129
5249
  type: "session.status.changed",
4130
5250
  payload: { status: "requires_action" },
@@ -4133,6 +5253,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4133
5253
  turnStatus: "requires_action",
4134
5254
  sessionStatus: "requires_action",
4135
5255
  activeTurnId,
5256
+ runState: {
5257
+ serializedRunState: stream.state.toString(),
5258
+ pendingApprovals: approvals,
5259
+ frozenCodexCredentialId: effectiveCodexCredentialId,
5260
+ humanInputRequests: humanInputRequests.map(
5261
+ ({ isNew: _isNew, ...request }) => request,
5262
+ ),
5263
+ },
4136
5264
  }))
4137
5265
  ) {
4138
5266
  return claimedResult({ status: "cancelled" });
@@ -4182,6 +5310,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4182
5310
  unit: "run",
4183
5311
  sourceResourceType: "session_turn",
4184
5312
  sourceResourceId: activeTurnId,
5313
+ sessionId: input.sessionId,
5314
+ turnId: activeTurnId,
5315
+ turnAttemptId: input.attemptId,
4185
5316
  idempotencyKey: `usage:agent_run.completed:${activeTurnId}`,
4186
5317
  });
4187
5318
  activityStatus = "idle";
@@ -4242,9 +5373,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4242
5373
  }
4243
5374
  }
4244
5375
  } catch (compactError) {
4245
- compactionFailureMessage = compactionFailureReason(
4246
- compactError instanceof Error ? compactError.message : String(compactError),
4247
- );
5376
+ // Transient checkpoint-provider failures recover this same accepted
5377
+ // turn through the normal provider/capacity path. They are not an
5378
+ // empty summary and must not create a new goal continuation.
5379
+ if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
5380
+ if (!isCompactionSummaryFailure(compactError)) throw compactError;
5381
+ compactionFailureMessage = compactionFailureReasonFromError(compactError);
4248
5382
  observability.warn("context compaction recovery compaction failed", {
4249
5383
  sessionId: input.sessionId,
4250
5384
  turnId: activeTurnId,
@@ -4276,6 +5410,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4276
5410
  turnStatus: "failed",
4277
5411
  sessionStatus: "idle",
4278
5412
  activeTurnId: null,
5413
+ ...(recoveryKind === "operator" ? { consumeRequestedCompactionFailure: true } : {}),
4279
5414
  }))
4280
5415
  ) {
4281
5416
  return claimedResult({ status: "cancelled" });
@@ -4283,7 +5418,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4283
5418
  turnMetricOutcome = "failed";
4284
5419
  activityStatus = "idle";
4285
5420
  activityError = attemptError;
4286
- return claimedResult({ status: "idle" });
5421
+ // The failed turn settlement already defers ordinary internal
5422
+ // updates and makes the delivered goal-continuation receipt
5423
+ // terminal. End this workflow run as well: returning plain idle
5424
+ // would immediately synthesize another goal continuation against
5425
+ // the unchanged active history and repeat the same failed
5426
+ // compaction. A new human/API prompt, Steer, or explicitly requested
5427
+ // Compact remains a durable explicit wake and may retry; ordinary
5428
+ // machine updates stay pending for that actionable wake.
5429
+ return claimedResult({ status: "idle", deferredUntilWake: true });
4287
5430
  }
4288
5431
  // Codex parity: compaction remains inside the same logical turn and
4289
5432
  // the same activity. Rebuild the model-visible history from the
@@ -4326,6 +5469,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4326
5469
  reason: "sandbox_lease_superseded",
4327
5470
  });
4328
5471
  if (recovery.action === "stale") {
5472
+ acknowledgeLostAttemptOwnership();
4329
5473
  activityStatus = "cancelled";
4330
5474
  turnMetricOutcome = "cancelled";
4331
5475
  return claimedResult({ status: "cancelled" });
@@ -4344,7 +5488,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4344
5488
  throw recoveryError;
4345
5489
  }
4346
5490
  }
4347
- if (isWorkerShutdownCancellation(error) && recoveryTurnId) {
5491
+ const cancellationFailure = turnOperationCancellationFailure(error);
5492
+ if (
5493
+ cancellationFailure &&
5494
+ isWorkerShutdownCancellation(cancellationFailure) &&
5495
+ recoveryTurnId
5496
+ ) {
4348
5497
  try {
4349
5498
  await flushRuntimeBatcher();
4350
5499
  await reconcileConversationTruth();
@@ -4363,6 +5512,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4363
5512
  reason: "worker_shutdown",
4364
5513
  });
4365
5514
  if (recovery.action === "stale") {
5515
+ acknowledgeLostAttemptOwnership();
4366
5516
  activityStatus = "cancelled";
4367
5517
  turnMetricOutcome = "cancelled";
4368
5518
  return claimedResult({ status: "cancelled" });
@@ -4387,26 +5537,35 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4387
5537
  if (error instanceof TurnAttemptFencedError) {
4388
5538
  activityStatus = "cancelled";
4389
5539
  activityError = error;
4390
- await flushRuntimeBatcher();
5540
+ acknowledgeQuiescence = true;
5541
+ noteCancellationRequested();
5542
+ await waitForTurnFinalizerStep(
5543
+ flushRuntimeBatcher(),
5544
+ turnFinalizerCancellationSignal(cancellationSignal, activityStatus),
5545
+ );
4391
5546
  // Ownership already moved to a newer attempt or an authoritative
4392
- // control transaction. Surface a transport cancellation instead of a
4393
- // normal activity result: with WAIT_CANCELLATION_COMPLETED, Temporal
4394
- // must observe a terminal cancellation before the session workflow may
4395
- // close. Returning normally after a cancel request is rejected by the
4396
- // server and leaves the worker task detached indefinitely.
5547
+ // control transaction. Surface the exact transport cancellation rather
5548
+ // than a normal result. Temporal terminalization remains diagnostic
5549
+ // only; replacement admission waits for the activity-owned durable
5550
+ // quiescence receipt written from the hard tool fence below.
4397
5551
  turnMetricOutcome = "cancelled";
4398
5552
  throw new CancelledFailure("TURN_ATTEMPT_FENCED", [], error);
4399
5553
  }
4400
- if (error instanceof CancelledFailure) {
5554
+ if (cancellationFailure) {
4401
5555
  activityStatus = "cancelled";
4402
5556
  activityError = error;
4403
- await flushRuntimeBatcher();
5557
+ acknowledgeQuiescence = true;
5558
+ noteCancellationRequested();
5559
+ await waitForTurnFinalizerStep(
5560
+ flushRuntimeBatcher(),
5561
+ turnFinalizerCancellationSignal(cancellationSignal, activityStatus),
5562
+ );
4404
5563
  // The workflow owns cancellation settlement: Pause/Steer controls use
4405
5564
  // settleSessionControl, and heartbeat timeouts use worker-death
4406
5565
  // recovery. A dying activity must never append a
4407
5566
  // competing cancellation or mutate the turn/session on its own.
4408
5567
  turnMetricOutcome = "cancelled";
4409
- throw error;
5568
+ throw cancellationFailure;
4410
5569
  }
4411
5570
  // The SDK's per-segment turn cap is a pacing valve, not a failure: end
4412
5571
  // the turn gracefully and idle the session so an active goal continues
@@ -4448,6 +5607,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4448
5607
  unit: "run",
4449
5608
  sourceResourceType: "session_turn",
4450
5609
  sourceResourceId: turnId,
5610
+ sessionId: input.sessionId,
5611
+ turnId,
5612
+ turnAttemptId: input.attemptId,
4451
5613
  idempotencyKey: `usage:agent_run.completed:${turnId}`,
4452
5614
  });
4453
5615
  activityStatus = "idle";
@@ -4613,12 +5775,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4613
5775
  : {}),
4614
5776
  }
4615
5777
  : null;
4616
- const cooldownUntil = codexCredentialCooldownUntil(
4617
- codexCredentialFailure,
4618
- serving,
4619
- settings.codexRotationNearExhaustionPct,
4620
- now,
4621
- );
5778
+ const cooldownUntil = codexCredentialCooldownUntil(codexCredentialFailure, serving, now);
4622
5779
  const statePersisted =
4623
5780
  codexLeaseHolderId && codexLeaseGeneration !== null
4624
5781
  ? await quarantineCodexCredentialForLease(db, {
@@ -4663,7 +5820,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4663
5820
  activeCredentialId: rotation.activeCredentialId,
4664
5821
  priorCredentialId: effectiveCodexCredentialId,
4665
5822
  accounts,
4666
- nearExhaustionPct: settings.codexRotationNearExhaustionPct,
4667
5823
  now: new Date(),
4668
5824
  usedConnectors: serving?.connectorNamespaces ?? [],
4669
5825
  })
@@ -4725,9 +5881,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4725
5881
  if (settlement.action === "stale") {
4726
5882
  // One transaction proves both exact-holder recovery (including a
4727
5883
  // just-expired or reaped lease row) and successor/control-gate
4728
- // rejection. A stale activity performs no second settlement.
4729
- activityStatus = "recovering";
4730
- turnMetricOutcome = "recovering";
5884
+ // rejection. Cross the hard tool fence so a control-gate loss can
5885
+ // write its quiescence receipt; a successor-only loss is a no-op.
5886
+ acknowledgeLostAttemptOwnership();
5887
+ activityStatus = "cancelled";
5888
+ turnMetricOutcome = "cancelled";
4731
5889
  return claimedResult({ status: "recovering" });
4732
5890
  }
4733
5891
  }
@@ -4784,7 +5942,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4784
5942
  const until = codexCredentialCooldownUntil(
4785
5943
  { kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds },
4786
5944
  serving,
4787
- settings.codexRotationNearExhaustionPct,
4788
5945
  new Date(),
4789
5946
  )!;
4790
5947
  // Finding 1a: INSPECT the cooldown-write result. A swallowed best-effort
@@ -4822,7 +5979,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4822
5979
  const newHome = shardCredentialForSession({
4823
5980
  sessionId: input.sessionId,
4824
5981
  accounts: fresh,
4825
- nearExhaustionPct: settings.codexRotationNearExhaustionPct,
4826
5982
  now: new Date(),
4827
5983
  });
4828
5984
  if (newHome) {
@@ -4871,16 +6027,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4871
6027
  } else {
4872
6028
  // Every account capped/cooling → idle until the earliest reset across all.
4873
6029
  rotated = true;
4874
- allCappedResetAt = earliestCodexReset(
4875
- fresh,
4876
- settings.codexRotationNearExhaustionPct,
4877
- new Date(),
4878
- );
4879
- capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(
4880
- fresh,
4881
- settings.codexRotationNearExhaustionPct,
4882
- new Date(),
4883
- );
6030
+ allCappedResetAt = earliestCodexReset(fresh, new Date());
6031
+ capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
4884
6032
  }
4885
6033
  } else {
4886
6034
  const decision = chooseRotationActive({
@@ -4888,7 +6036,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4888
6036
  activeCredentialId: rotation.activeCredentialId,
4889
6037
  priorCredentialId: effectiveCodexCredentialId,
4890
6038
  accounts: fresh,
4891
- nearExhaustionPct: settings.codexRotationNearExhaustionPct,
4892
6039
  now: new Date(),
4893
6040
  // P4: the just-capped serving account's connector set is the proxy for
4894
6041
  // "what this session has access to" — prefer a covering failover target.
@@ -4915,11 +6062,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4915
6062
  } else if (decision.kind === "allCapped") {
4916
6063
  rotated = true;
4917
6064
  allCappedResetAt = decision.earliestResetAt;
4918
- capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(
4919
- fresh,
4920
- settings.codexRotationNearExhaustionPct,
4921
- new Date(),
4922
- );
6065
+ capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
4923
6066
  }
4924
6067
  // kind:"none" → fall through to today's single-account idle.
4925
6068
  }
@@ -5094,6 +6237,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5094
6237
  unit: "run",
5095
6238
  sourceResourceType: "session_turn",
5096
6239
  sourceResourceId: turnId,
6240
+ sessionId: input.sessionId,
6241
+ turnId,
6242
+ turnAttemptId: input.attemptId,
5097
6243
  idempotencyKey: `usage:agent_run.completed:${turnId}`,
5098
6244
  });
5099
6245
  activityStatus = "idle";
@@ -5106,6 +6252,34 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5106
6252
  // it after a pacing delay. This is independent of goal state and never
5107
6253
  // relies on a synthetic continuation prompt.
5108
6254
  const failure = agentRunFailurePayload(error);
6255
+ if (isSessionEventPersistenceError(error)) {
6256
+ // Never pass the original Drizzle/postgres-js error to telemetry: its
6257
+ // nested cause may contain raw SQL and bound parameters. The typed DB
6258
+ // boundary retains only SQLSTATE, stage, correlation, and safe catalog
6259
+ // identifiers. Provider inference has already happened and is NOT
6260
+ // retried by this terminal classification.
6261
+ observability.error("session event persistence failed", {
6262
+ accountId: input.accountId,
6263
+ workspaceId: input.workspaceId,
6264
+ sessionId: input.sessionId,
6265
+ turnId,
6266
+ attemptId: input.attemptId,
6267
+ code: error.details.code,
6268
+ sqlState: error.details.sqlState ?? "unknown",
6269
+ stage: error.details.stage,
6270
+ eventTypes: error.details.eventTypes.join(","),
6271
+ correlationId: error.details.correlationId,
6272
+ attempts: error.details.attempts,
6273
+ retryOutcome: error.details.retryOutcome,
6274
+ dbSeverity: error.details.database.severity,
6275
+ dbSchema: error.details.database.schema,
6276
+ dbTable: error.details.database.table,
6277
+ dbColumn: error.details.database.column,
6278
+ dbDataType: error.details.database.dataType,
6279
+ dbConstraint: error.details.database.constraint,
6280
+ dbRoutine: error.details.database.routine,
6281
+ });
6282
+ }
5109
6283
  if (failure.retryable && publish && turnId && turnStartedPublished) {
5110
6284
  const recoveryResult = providerRecoveryResult();
5111
6285
  await flushRuntimeBatcher();
@@ -5122,6 +6296,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5122
6296
  },
5123
6297
  });
5124
6298
  if (recovery.action === "stale") {
6299
+ acknowledgeLostAttemptOwnership();
5125
6300
  activityStatus = "cancelled";
5126
6301
  turnMetricOutcome = "cancelled";
5127
6302
  return claimedResult({ status: "cancelled" });
@@ -5173,215 +6348,490 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5173
6348
  );
5174
6349
  return claimedResult({ status: "failed" });
5175
6350
  } finally {
5176
- gitCredentialRenewalClosed = true;
5177
- const renewalToStop = gitCredentialRenewal as GitCredentialRenewalController | null;
5178
- gitCredentialRenewal = null;
5179
- await renewalToStop?.stop();
5180
- const durationSeconds = (performance.now() - activityStarted) / 1000;
5181
- observability.recordWorkerActivity({
5182
- activity: "runAgentTurn",
5183
- status: activityStatus,
5184
- durationSeconds,
5185
- });
5186
- if (turnId && activityStatus !== "unknown") {
5187
- turnLifecycleMetricsFor(observability).finish(turnId, turnMetricOutcome, durationSeconds);
5188
- }
5189
- activitySpan.end({
5190
- attributes: {
5191
- "opengeni.turn_id": turnId ?? "",
5192
- "opengeni.status": activityStatus,
5193
- "opengeni.variable_set_id": variableSetId,
5194
- "opengeni.rig_id": rigId,
5195
- "opengeni.rig_version_id": rigVersionId,
5196
- "opengeni.codex_credential_id": effectiveCodexCredentialId ?? "",
5197
- "opengeni.duration_ms": Math.round(durationSeconds * 1000),
5198
- },
5199
- error: activityError,
5200
- });
5201
- // Drain the buffered Connected Machine op events (infra failures + healed
5202
- // recoveries) to durable session events — awaited, best-effort, never blocking
5203
- // the turn. Sync observer → buffer → single awaited append here (no unawaited
5204
- // DB write inside the activity). Scoped to this turn; skipped if no turnId
5205
- // (the op ran under a turn, so on the normal path turnId is set).
5206
- const machineOpEvents = machineOpObserver.drainEvents();
5207
- if (machineOpEvents.length > 0 && turnId && executionGeneration > 0) {
5208
- await appendAndPublishTurnEventsFenced(
5209
- db,
5210
- bus,
5211
- input.workspaceId,
5212
- input.sessionId,
5213
- turnId,
5214
- executionGeneration,
5215
- input.attemptId,
5216
- machineOpEvents.map((event) => ({
5217
- ...event,
5218
- turnId: turnId ?? null,
5219
- })),
5220
- ).catch(() => undefined);
5221
- }
5222
- // Multi-account P4: flush the serving account's free per-turn caches ONCE,
5223
- // best-effort (same discipline as today's usage write). Both writers skip
5224
- // version/updatedAt, so neither can race the token-refresh CAS.
5225
- if (effectiveCodexCredentialId) {
5226
- // Part A: the latest scraped usage-header snapshot → the P2 usage cache. A
5227
- // full both-windows snapshot (parseCodexUsageHeaders gates on both), so this
5228
- // is byte-identical to the /wham/usage write — no partial-window clobber.
5229
- if (latestCodexUsage) {
5230
- const usageMutation = await recordCodexAccountUsageWithWakeTargets(
5231
- db,
5232
- input.workspaceId,
5233
- effectiveCodexCredentialId,
5234
- latestCodexUsage,
5235
- ).catch(() => null);
5236
- if (usageMutation) {
5237
- await signalCodexCapacityWakeTargets(
5238
- { signalCodexCapacityWorkflow, wakeSessionWorkflow },
5239
- usageMutation.wakeTargets,
6351
+ const finalizationStarted = performance.now();
6352
+ let finalizationError: unknown;
6353
+ let physicalToolQuiescenceConfirmed = !acknowledgeQuiescence;
6354
+ let quiescenceReceiptOrProofDurable = !acknowledgeQuiescence;
6355
+ const finalizerSignal = turnFinalizerCancellationSignal(cancellationSignal, activityStatus);
6356
+ try {
6357
+ const toolCancellationFence = toolCancellationFenceRef.current;
6358
+ // Toolspace and run-credential renewals are attempt-owned sandbox
6359
+ // writers. Capture and close them before crossing the same authoritative
6360
+ // boundary as tool calls; none of these drains may race the already-
6361
+ // aborted Temporal signal.
6362
+ toolspaceTokenRenewalClosed = true;
6363
+ const toolspaceRenewalToStop =
6364
+ toolspaceTokenRenewal as ToolspaceTokenRenewalController | null;
6365
+ toolspaceTokenRenewal = null;
6366
+ runCredentialRenewalClosed = true;
6367
+ const runRenewalToStop = runCredentialRenewal as RunCredentialRenewalController | null;
6368
+ runCredentialRenewal = null;
6369
+ await drainAttemptOwnedSandboxWriters({
6370
+ toolCancellationFence: acknowledgeQuiescence ? toolCancellationFence : null,
6371
+ cancellationReason: cancellationSignal?.reason,
6372
+ toolspaceTokenRenewal: toolspaceRenewalToStop,
6373
+ runCredentialRenewal: runRenewalToStop,
6374
+ });
6375
+ if (acknowledgeQuiescence) {
6376
+ // A cancellation before sandbox-backed capabilities exist still has
6377
+ // no tool controller to drain. Sandbox agent construction fails closed
6378
+ // when a backend exists but no controller was installed. Renewal
6379
+ // writers, when present, were drained above in either case.
6380
+ physicalToolQuiescenceConfirmed = true;
6381
+ }
6382
+ if (acknowledgeQuiescence && physicalToolQuiescenceConfirmed) {
6383
+ // This receipt is part of the hard cancellation boundary, not
6384
+ // housekeeping. Persist it immediately after the sandbox/tool fence
6385
+ // and before lease, cache, recording, or provider cleanup. Its
6386
+ // transaction also enqueues the exact workflow wake that will admit
6387
+ // the replacement; Temporal activity terminalization does neither.
6388
+ const proof: SessionAttemptQuiescenceProof = {
6389
+ accountId: input.accountId,
6390
+ workspaceId: input.workspaceId,
6391
+ sessionId: input.sessionId,
6392
+ attemptId: input.attemptId,
6393
+ workflowId: input.workflowId,
6394
+ workflowRunId: input.workflowRunId,
6395
+ activityId: dispatchId,
6396
+ };
6397
+ const recoveryMode = await persistOrSignalSessionAttemptQuiescence({
6398
+ proof,
6399
+ persistReceipt: async () =>
6400
+ await markSessionAttemptQuiesced(db, {
6401
+ accountId: input.accountId,
6402
+ workspaceId: input.workspaceId,
6403
+ sessionId: input.sessionId,
6404
+ attemptId: input.attemptId,
6405
+ temporalWorkflowId: input.workflowId,
6406
+ temporalWorkflowRunId: input.workflowRunId,
6407
+ temporalActivityId: dispatchId,
6408
+ allowUninterrupted: true,
6409
+ }),
6410
+ publishEvents: async (events) => {
6411
+ await waitForTurnFinalizerStep(
6412
+ publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events),
6413
+ finalizerSignal,
6414
+ );
6415
+ },
6416
+ signalProof: signalSessionAttemptQuiesced,
6417
+ heartbeat: (attempt, retryMs) => {
6418
+ activityContext?.heartbeat({
6419
+ phase: "quiescence-proof-delivery",
6420
+ sessionId: input.sessionId,
6421
+ attemptId: input.attemptId,
6422
+ deliveryAttempt: attempt,
6423
+ retryMs,
6424
+ at: new Date().toISOString(),
6425
+ });
6426
+ },
6427
+ onReceiptFailure: (error) => {
6428
+ console.error("agent turn quiescence receipt exhausted; signalling proof", error);
6429
+ },
6430
+ onPublishFailure: (error) => {
6431
+ console.error("agent turn quiescence event fanout failed", error);
6432
+ },
6433
+ onSignalFailure: (error, attempt, retryMs) => {
6434
+ console.error("agent turn quiescence proof signal failed; retrying", {
6435
+ error,
6436
+ attempt,
6437
+ retryMs,
6438
+ });
6439
+ },
6440
+ });
6441
+ quiescenceReceiptOrProofDurable = true;
6442
+ if (recoveryMode === "signal") {
6443
+ observability.info("agent turn quiescence proof handed to workflow recovery", {
6444
+ "opengeni.session_id": input.sessionId,
6445
+ "opengeni.attempt_id": input.attemptId,
6446
+ "opengeni.workflow_run_id": input.workflowRunId,
6447
+ "opengeni.activity_id": dispatchId,
6448
+ });
6449
+ }
6450
+ }
6451
+ // Deleting this attempt's credential generation is post-receipt
6452
+ // housekeeping: pointer mutation is serialized with successor
6453
+ // activation and the attempt-qualified check cannot erase a successor.
6454
+ // It may therefore detach on cancellation without weakening the writer
6455
+ // drain that gates the receipt above.
6456
+ const credentialSessionToClear = runCredentialSession;
6457
+ runCredentialSession = null;
6458
+ if (credentialSessionToClear) {
6459
+ const cleanup = clearRunCredentialsForAttempt(credentialSessionToClear, {
6460
+ sessionId: input.sessionId,
6461
+ attemptId: input.attemptId,
6462
+ executionGeneration,
6463
+ }).catch((error: unknown) => {
6464
+ try {
6465
+ observability.incrementCounter({
6466
+ name: "opengeni_run_credential_cleanup_total",
6467
+ help: "Attempt-owned run credential cleanup outcomes.",
6468
+ labels: { outcome: "error" },
6469
+ });
6470
+ observability.warn("Attempt-owned run credential cleanup failed", {
6471
+ sessionId: input.sessionId,
6472
+ turnId,
6473
+ attemptId: input.attemptId,
6474
+ errorClass: error instanceof Error ? error.name : "UnknownError",
6475
+ });
6476
+ } catch {
6477
+ // Cleanup observability must not own finalizer liveness.
6478
+ }
6479
+ });
6480
+ await waitForTurnFinalizerStep(cleanup, finalizerSignal);
6481
+ }
6482
+ gitCredentialRenewalClosed = true;
6483
+ const renewalsToStop = gitCredentialRenewals;
6484
+ gitCredentialRenewals = [];
6485
+ for (const renewal of renewalsToStop) {
6486
+ await waitForTurnFinalizerStep(renewal.stop(), finalizerSignal);
6487
+ }
6488
+ // Drain the buffered Connected Machine op events (infra failures + healed
6489
+ // recoveries) to durable session events — awaited, best-effort, never blocking
6490
+ // the turn. Sync observer → buffer → single awaited append here (no unawaited
6491
+ // DB write inside the activity). Scoped to this turn; skipped if no turnId
6492
+ // (the op ran under a turn, so on the normal path turnId is set).
6493
+ const machineOpEvents = machineOpObserver.drainEvents();
6494
+ if (machineOpEvents.length > 0 && turnId && executionGeneration > 0) {
6495
+ await waitForTurnFinalizerStep(
6496
+ appendAndPublishTurnEventsFenced(
6497
+ db,
6498
+ bus,
6499
+ input.workspaceId,
6500
+ input.sessionId,
6501
+ turnId,
6502
+ executionGeneration,
6503
+ input.attemptId,
6504
+ machineOpEvents.map((event) => ({
6505
+ ...event,
6506
+ turnId: turnId ?? null,
6507
+ })),
6508
+ ).catch(() => undefined),
6509
+ finalizerSignal,
6510
+ );
6511
+ }
6512
+ // Multi-account P4: flush the serving account's free per-turn caches ONCE,
6513
+ // best-effort (same discipline as today's usage write). Both writers skip
6514
+ // version/updatedAt, so neither can race the token-refresh CAS.
6515
+ if (effectiveCodexCredentialId) {
6516
+ // Part A: the latest scraped usage-header snapshot → the P2 usage cache. A
6517
+ // full both-windows snapshot (parseCodexUsageHeaders gates on both), so this
6518
+ // is byte-identical to the /wham/usage write — no partial-window clobber.
6519
+ if (latestCodexUsage) {
6520
+ const usageMutation = await waitForTurnFinalizerStep(
6521
+ recordCodexAccountUsageWithWakeTargets(
6522
+ db,
6523
+ input.workspaceId,
6524
+ effectiveCodexCredentialId,
6525
+ latestCodexUsage,
6526
+ ).catch(() => null),
6527
+ finalizerSignal,
6528
+ );
6529
+ if (usageMutation) {
6530
+ await waitForTurnFinalizerStep(
6531
+ signalCodexCapacityWakeTargets(
6532
+ { signalCodexCapacityWorkflow, wakeSessionWorkflow },
6533
+ usageMutation.wakeTargets,
6534
+ ),
6535
+ finalizerSignal,
6536
+ );
6537
+ }
6538
+ }
6539
+ // Part B.1: the connector namespaces codex_apps listed this turn → the
6540
+ // connector-set cache. NON-EMPTY-only: a flaky/empty tools/list must never
6541
+ // overwrite a known set with [] (false coverage drop). Read by reference
6542
+ // AFTER the run, so every tools/list this turn has accumulated.
6543
+ const connectorNamespaces = preparedTools?.codexConnectorNamespaces;
6544
+ if (connectorNamespaces && connectorNamespaces.size > 0) {
6545
+ await waitForTurnFinalizerStep(
6546
+ recordCodexAccountConnectors(db, input.workspaceId, effectiveCodexCredentialId, [
6547
+ ...connectorNamespaces,
6548
+ ]).catch(() => undefined),
6549
+ finalizerSignal,
5240
6550
  );
5241
6551
  }
5242
6552
  }
5243
- // Part B.1: the connector namespaces codex_apps listed this turn → the
5244
- // connector-set cache. NON-EMPTY-only: a flaky/empty tools/list must never
5245
- // overwrite a known set with [] (false coverage drop). Read by reference
5246
- // AFTER the run, so every tools/list this turn has accumulated.
5247
- const connectorNamespaces = preparedTools?.codexConnectorNamespaces;
5248
- if (connectorNamespaces && connectorNamespaces.size > 0) {
5249
- await recordCodexAccountConnectors(db, input.workspaceId, effectiveCodexCredentialId, [
5250
- ...connectorNamespaces,
5251
- ]).catch(() => undefined);
6553
+ if (codexLeaseHeartbeatTimer) {
6554
+ clearInterval(codexLeaseHeartbeatTimer);
6555
+ codexLeaseHeartbeatTimer = undefined;
5252
6556
  }
5253
- }
5254
- if (codexLeaseHeartbeatTimer) {
5255
- clearInterval(codexLeaseHeartbeatTimer);
5256
- codexLeaseHeartbeatTimer = undefined;
5257
- }
5258
- if (codexLeaseHeld && turnId && codexLeaseHolderId && codexLeaseGeneration !== null) {
5259
- await releaseCodexCredentialLease(
5260
- db,
5261
- input.accountId,
5262
- input.workspaceId,
5263
- turnId,
5264
- codexLeaseHolderId,
5265
- codexLeaseGeneration,
5266
- ).catch(() => undefined);
5267
- codexLeaseHeld = false;
5268
- }
5269
- // Workbench v2 turn-end workspace capture (dossier §10.1) — runs FIRST in
5270
- // the turn-end finally, while the box is MAXIMALLY ALIVE. The agent's last
5271
- // tool ran before this finally, so /workspace is already final; capture is
5272
- // FS-equivalent to the already-settled recording preparation and the warm
5273
- // snapshot (neither mutates workspace files). Running it here — BEFORE
5274
- // preparedTools.close() (which tears down tools / computer-use / the display
5275
- // stack and is what starts the Modal box exiting a few seconds later) —
5276
- // gives capture the full live-box margin instead of racing the teardown
5277
- // tail, which was dropping 100% of captures on real Modal desktop boxes
5278
- // ("request cancelled due to container exiting", 0 rows). External module:
5279
- // self-capped at 60s, best-effort (never throws past its boundary),
5280
- // epoch-fenced, and it NEVER closes the box. The emitted
5281
- // workspace.revision.captured event is ANNOUNCE-ONLY (metadata, never
5282
- // content).
5283
- if (resolvedSandbox && setupBoxSession && sandboxGroupId) {
5284
- // Stop new heartbeat snapshot/meter ticks so a mid-turn snapshot cannot
5285
- // start concurrently with capture, then drain any in-flight snapshot
5286
- // (bounded) — capture and the warm snapshot both exec on the box, so
5287
- // sequence them, exactly as the turn-end snapshot placement did.
5288
- if (leaseHeartbeatTimer) {
5289
- clearInterval(leaseHeartbeatTimer);
5290
- leaseHeartbeatTimer = undefined;
6557
+ if (codexLeaseHeld && turnId && codexLeaseHolderId && codexLeaseGeneration !== null) {
6558
+ await waitForTurnFinalizerStep(
6559
+ releaseCodexCredentialLease(
6560
+ db,
6561
+ input.accountId,
6562
+ input.workspaceId,
6563
+ turnId,
6564
+ codexLeaseHolderId,
6565
+ codexLeaseGeneration,
6566
+ ).catch(() => undefined),
6567
+ finalizerSignal,
6568
+ );
6569
+ codexLeaseHeld = false;
5291
6570
  }
5292
- if (snapshotInFlight) {
5293
- await waitForWarmSnapshot(snapshotInFlight, settings.sandboxSnapshotTimeoutMs);
6571
+ // Workbench v2 turn-end workspace capture — runs FIRST in
6572
+ // the turn-end finally, while the box is MAXIMALLY ALIVE. The agent's last
6573
+ // tool ran before this finally, so /workspace is already final; capture is
6574
+ // FS-equivalent to the already-settled recording preparation and the warm
6575
+ // snapshot (neither mutates workspace files). Running it here — BEFORE
6576
+ // preparedTools.close() (which tears down tools / computer-use / the display
6577
+ // stack and is what starts the Modal box exiting a few seconds later) —
6578
+ // gives capture the full live-box margin instead of racing the teardown
6579
+ // tail, which was dropping 100% of captures on real Modal desktop boxes
6580
+ // ("request cancelled due to container exiting", 0 rows). External module:
6581
+ // self-capped at 60s, best-effort (never throws past its boundary),
6582
+ // epoch-fenced, and it NEVER closes the box. The emitted
6583
+ // workspace.revision.captured event is ANNOUNCE-ONLY (metadata, never
6584
+ // content).
6585
+ if (process.env.OPENGENI_TEST_SCENARIO === "sandbox") {
6586
+ console.log(
6587
+ `[sandbox-e2e] capture preflight ownership=${settings.sandboxOwnershipEnabled} enabled=${settings.workspaceCaptureEnabled} resolved=${Boolean(resolvedSandbox)} session=${Boolean(setupBoxSession)} group=${Boolean(sandboxGroupId)} storage=${Boolean(objectStorage)}`,
6588
+ );
5294
6589
  }
5295
- await captureWorkspaceRevision({
5296
- db,
5297
- objectStorage,
5298
- settings,
5299
- publish: async (events) => {
5300
- await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events);
5301
- },
5302
- session: setupBoxSession as ChannelASession,
5303
- leaseEpoch: resolvedSandbox.leaseEpoch,
5304
- sandboxGroupId,
5305
- accountId: input.accountId,
5306
- workspaceId: input.workspaceId,
5307
- sessionId: input.sessionId,
5308
- turnId: turnId ?? null,
5309
- observability,
6590
+ const runTurnEndPersistence = shouldRunTurnEndWorkspacePersistence({
6591
+ activityStatus,
6592
+ cancellationRequested: finalizerSignal?.aborted === true,
5310
6593
  });
5311
- }
5312
- await preparedTools?.close().catch(() => undefined);
5313
- if (heartbeatTimer) {
5314
- clearInterval(heartbeatTimer);
5315
- }
5316
- if (turnSandboxProvisioner?.hasStarted()) {
5317
- await turnSandboxProvisioner.waitForSettled(30_000);
5318
- }
5319
- // P1.2: stop the lease-TTL refresh, release the turn holder (idempotent
5320
- // delete-my-row; refcount-- and warm->draining if it hit 0 with no turns),
5321
- // and DROP the in-memory handle. Release NEVER stops the box — the reaper
5322
- // (P1.3) issues the provider stop() past the drain grace at refcount 0; the
5323
- // box rides the provider idle-timeout in the meantime. Best-effort: a
5324
- // release failure must never mask the turn's real outcome.
5325
- if (leaseHeartbeatTimer) {
5326
- clearInterval(leaseHeartbeatTimer);
5327
- }
5328
- // A recording normally closes inside the attempt-fenced turn settlement.
5329
- // Reaching finally with one still active means settlement threw, never ran,
5330
- // or lost ownership. Stop ffmpeg and mark only this exact attempt-owned row
5331
- // failed; publish no event and leave the artifact recoverable on the box.
5332
- await abandonActiveRecording(
5333
- "activity ended without recording settlement",
5334
- didComputerUse ? "failed" : "discard",
5335
- );
5336
- if (resolvedSandbox) {
5337
- // TURN-END mid-session snapshot (sandbox-file-persistence): fold the
5338
- // turn's finished /workspace onto the lease before releasing the holder,
5339
- // so the work this turn just produced survives any unclean box death in
5340
- // the idle window ahead. Throttled by the same interval as the heartbeat
5341
- // tick (a short turn right after a snapshot skips — bounded-loss contract
5342
- // is the interval, not per-turn). Best-effort and time-capped by the
5343
- // helper's own failure discipline; never delays release on failure.
5344
- if (setupBoxSession && sandboxGroupId) {
5345
- // Single-flight vs the heartbeat capture: the timer is already cleared
5346
- // above, but a capture it launched may still be in flight — and that
5347
- // capture predates the turn's final writes. Wait for it, but only up
5348
- // to the snapshot timeout: release must never depend on an unbounded
5349
- // provider capture.
6594
+ if (
6595
+ runTurnEndPersistence &&
6596
+ turnId &&
6597
+ resolvedSandbox &&
6598
+ setupBoxSession &&
6599
+ sandboxGroupId
6600
+ ) {
6601
+ // Stop new heartbeat snapshot/meter ticks so a mid-turn snapshot cannot
6602
+ // start concurrently with capture, then drain any in-flight snapshot
6603
+ // (bounded) — capture and the warm snapshot both exec on the box, so
6604
+ // sequence them, exactly as the turn-end snapshot placement did.
6605
+ if (leaseHeartbeatTimer) {
6606
+ clearInterval(leaseHeartbeatTimer);
6607
+ leaseHeartbeatTimer = undefined;
6608
+ }
5350
6609
  if (snapshotInFlight) {
5351
- await waitForWarmSnapshot(snapshotInFlight, settings.sandboxSnapshotTimeoutMs);
6610
+ await waitForWarmSnapshot(
6611
+ snapshotInFlight,
6612
+ settings.sandboxSnapshotTimeoutMs,
6613
+ finalizerSignal,
6614
+ );
5352
6615
  }
5353
- const persisted = await maybePersistWarmWorkspaceSnapshot(
5354
- { db, settings },
5355
- {
5356
- accountId: input.accountId,
5357
- workspaceId: input.workspaceId,
5358
- sandboxGroupId,
6616
+ await captureWorkspaceRevision({
6617
+ db,
6618
+ objectStorage,
6619
+ settings,
6620
+ publish: async (events) => {
6621
+ await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events);
5359
6622
  },
5360
- setupBoxSession,
5361
- resolvedSandbox.leaseEpoch,
6623
+ session: setupBoxSession as ChannelASession,
6624
+ leaseEpoch: resolvedSandbox.leaseEpoch,
6625
+ sandboxGroupId,
6626
+ accountId: input.accountId,
6627
+ workspaceId: input.workspaceId,
6628
+ sessionId: input.sessionId,
6629
+ turnId,
6630
+ attemptId: input.attemptId,
6631
+ observability,
6632
+ ...(finalizerSignal ? { signal: finalizerSignal } : {}),
6633
+ });
6634
+ }
6635
+ if (preparedTools) {
6636
+ await waitForTurnFinalizerStep(
6637
+ preparedTools.close().catch(() => undefined),
6638
+ finalizerSignal,
5362
6639
  );
5363
- if (persisted && publish) {
5364
- await publish([
6640
+ }
6641
+ if (heartbeatTimer) {
6642
+ clearInterval(heartbeatTimer);
6643
+ }
6644
+ if (turnSandboxProvisioner?.hasStarted()) {
6645
+ await waitForTurnFinalizerStep(
6646
+ turnSandboxProvisioner.waitForSettled(30_000),
6647
+ finalizerSignal,
6648
+ );
6649
+ }
6650
+ // P1.2: stop the lease-TTL refresh, release the turn holder (idempotent
6651
+ // delete-my-row; refcount-- and warm->draining if it hit 0 with no turns),
6652
+ // and DROP the in-memory handle. Release NEVER stops the box — the reaper
6653
+ // (P1.3) issues the provider stop() past the drain grace at refcount 0; the
6654
+ // box rides the provider idle-timeout in the meantime. Best-effort: a
6655
+ // release failure must never mask the turn's real outcome.
6656
+ if (leaseHeartbeatTimer) {
6657
+ clearInterval(leaseHeartbeatTimer);
6658
+ }
6659
+ // A recording normally closes inside the attempt-fenced turn settlement.
6660
+ // Reaching finally with one still active means settlement threw, never ran,
6661
+ // or lost ownership. Stop ffmpeg and mark only this exact attempt-owned row
6662
+ // failed; publish no event and leave the artifact recoverable on the box.
6663
+ await waitForTurnFinalizerStep(
6664
+ abandonActiveRecording(
6665
+ "activity ended without recording settlement",
6666
+ didComputerUse ? "failed" : "discard",
6667
+ ),
6668
+ finalizerSignal,
6669
+ );
6670
+ if (resolvedSandbox) {
6671
+ // TURN-END mid-session snapshot (sandbox-file-persistence): fold the
6672
+ // turn's finished /workspace onto the lease before releasing the holder,
6673
+ // so the work this turn just produced survives any unclean box death in
6674
+ // the idle window ahead. Throttled by the same interval as the heartbeat
6675
+ // tick (a short turn right after a snapshot skips — bounded-loss contract
6676
+ // is the interval, not per-turn). Best-effort and time-capped by the
6677
+ // helper's own failure discipline; never delays release on failure.
6678
+ const settledTurnId = turnId;
6679
+ if (runTurnEndPersistence && setupBoxSession && sandboxGroupId && settledTurnId) {
6680
+ // Single-flight vs the heartbeat capture: the timer is already cleared
6681
+ // above, but a capture it launched may still be in flight — and that
6682
+ // capture predates the turn's final writes. Wait for it, but only up
6683
+ // to the snapshot timeout: release must never depend on an unbounded
6684
+ // provider capture.
6685
+ if (snapshotInFlight) {
6686
+ await waitForWarmSnapshot(
6687
+ snapshotInFlight,
6688
+ settings.sandboxSnapshotTimeoutMs,
6689
+ finalizerSignal,
6690
+ );
6691
+ }
6692
+ const persisted = await maybePersistWarmWorkspaceSnapshot(
6693
+ { db, settings },
5365
6694
  {
5366
- type: "sandbox.box.snapshot",
5367
- payload: { trigger: "turn-end" },
6695
+ accountId: input.accountId,
6696
+ workspaceId: input.workspaceId,
6697
+ sessionId: input.sessionId,
6698
+ turnId: settledTurnId,
6699
+ attemptId: input.attemptId,
6700
+ sandboxGroupId,
5368
6701
  },
5369
- ]).catch(() => undefined);
6702
+ setupBoxSession,
6703
+ resolvedSandbox.leaseEpoch,
6704
+ finalizerSignal,
6705
+ );
6706
+ if (persisted && publish) {
6707
+ await publish([
6708
+ {
6709
+ type: "sandbox.box.snapshot",
6710
+ payload: { trigger: "turn-end" },
6711
+ },
6712
+ ]).catch(() => undefined);
6713
+ }
6714
+ // NB workspace capture no longer runs here — it moved to
6715
+ // the TOP of this finally (before preparedTools.close) so it completes
6716
+ // while the box is still solidly alive, instead of racing the turn-end
6717
+ // teardown that was killing 100% of captures on real Modal desktop boxes.
5370
6718
  }
5371
- // NB workspace capture (dossier §10.1) no longer runs here — it moved to
5372
- // the TOP of this finally (before preparedTools.close) so it completes
5373
- // while the box is still solidly alive, instead of racing the turn-end
5374
- // teardown that was killing 100% of captures on real Modal desktop boxes.
6719
+ const sandboxToRelease = resolvedSandbox;
6720
+ resolvedSandbox = null; // drop ownership now; the exact-holder release may finish later
6721
+ await waitForTurnFinalizerStep(
6722
+ sandboxToRelease.release().catch((releaseError) => {
6723
+ console.error("sandbox lease release failed (turn outcome unaffected)", releaseError);
6724
+ }),
6725
+ finalizerSignal,
6726
+ );
6727
+ }
6728
+ } catch (error) {
6729
+ finalizationError ??= error;
6730
+ console.error("agent turn finalization failed (turn outcome unaffected)", error);
6731
+ } finally {
6732
+ cancellationSignal?.removeEventListener("abort", noteCancellationRequested);
6733
+ const completedAt = performance.now();
6734
+ const durationSeconds = (completedAt - activityStarted) / 1000;
6735
+ const finalizationDurationSeconds = (completedAt - finalizationStarted) / 1000;
6736
+ observability.observeHistogram({
6737
+ name: "opengeni_turn_finalization_duration_seconds",
6738
+ help: "Agent turn finalization duration, including workspace housekeeping and lease release.",
6739
+ labels: {
6740
+ cancellation_requested: String(cancellationRequestedAt !== null),
6741
+ },
6742
+ value: finalizationDurationSeconds,
6743
+ });
6744
+ if (cancellationRequestedAt !== null) {
6745
+ const physicalCancellationDurationSeconds =
6746
+ (completedAt - cancellationRequestedAt) / 1000;
6747
+ observability.observeHistogram({
6748
+ name: "opengeni_turn_physical_cancellation_duration_seconds",
6749
+ help: "Time from Temporal cancellation delivery until the activity physically stops.",
6750
+ value: physicalCancellationDurationSeconds,
6751
+ });
6752
+ observability.info("agent turn physical cancellation completed", {
6753
+ "opengeni.session_id": input.sessionId,
6754
+ "opengeni.turn_id": turnId ?? "",
6755
+ "opengeni.attempt_id": input.attemptId,
6756
+ "opengeni.physical_cancellation_duration_ms": Math.round(
6757
+ physicalCancellationDurationSeconds * 1000,
6758
+ ),
6759
+ });
6760
+ }
6761
+ observability.recordWorkerActivity({
6762
+ activity: "runAgentTurn",
6763
+ status: finalizationError ? "cleanup_failed" : activityStatus,
6764
+ durationSeconds,
6765
+ });
6766
+ if (turnId && activityStatus !== "unknown") {
6767
+ turnLifecycleMetricsFor(observability).finish(turnId, turnMetricOutcome, durationSeconds);
5375
6768
  }
5376
- await resolvedSandbox.release().catch((releaseError) => {
5377
- console.error("sandbox lease release failed (turn outcome unaffected)", releaseError);
6769
+ activitySpan.end({
6770
+ attributes: {
6771
+ "opengeni.turn_id": turnId ?? "",
6772
+ "opengeni.status": activityStatus,
6773
+ "opengeni.variable_set_id": variableSetId,
6774
+ "opengeni.rig_id": rigId,
6775
+ "opengeni.rig_version_id": rigVersionId,
6776
+ "opengeni.codex_credential_id": effectiveCodexCredentialId ?? "",
6777
+ "opengeni.duration_ms": Math.round(durationSeconds * 1000),
6778
+ "opengeni.finalization_duration_ms": Math.round(finalizationDurationSeconds * 1000),
6779
+ },
6780
+ error: finalizationError ?? activityError,
6781
+ });
6782
+ assertPhysicalToolQuiescenceForCancellation({
6783
+ acknowledgeQuiescence,
6784
+ physicalToolQuiescenceConfirmed,
6785
+ failure: finalizationError,
6786
+ });
6787
+ assertSessionAttemptQuiescenceRecoveryDurable({
6788
+ acknowledgeQuiescence,
6789
+ physicalToolQuiescenceConfirmed,
6790
+ receiptOrProofDurable: quiescenceReceiptOrProofDurable,
6791
+ failure: finalizationError,
5378
6792
  });
5379
- resolvedSandbox = null; // drop the handle; the box survives the turn
5380
6793
  }
5381
6794
  }
5382
6795
  };
5383
6796
  }
5384
6797
 
6798
+ async function assertGitHubResourcesRemainAuthorized(
6799
+ db: Parameters<typeof areGitHubRepositoriesAllowedForWorkspace>[0],
6800
+ workspaceId: string,
6801
+ resources: import("@opengeni/contracts").ResourceRef[],
6802
+ ): Promise<void> {
6803
+ // Must check exactly what sandboxEnvironmentForRun would mint a token for,
6804
+ // so the selection is derived from the same extraction as the mint path.
6805
+ for (const selection of gitHubTokenMintSelections(resources)) {
6806
+ await assertGitHubTokenMintSelectionAuthorized(
6807
+ db,
6808
+ workspaceId,
6809
+ selection.installationId,
6810
+ selection.repositoryIds,
6811
+ );
6812
+ }
6813
+ }
6814
+
6815
+ async function assertGitHubTokenMintSelectionAuthorized(
6816
+ db: Parameters<typeof areGitHubRepositoriesAllowedForWorkspace>[0],
6817
+ workspaceId: string,
6818
+ installationId: number,
6819
+ repositoryIds: number[],
6820
+ ): Promise<void> {
6821
+ if (
6822
+ !(await areGitHubRepositoriesAllowedForWorkspace(
6823
+ db,
6824
+ workspaceId,
6825
+ installationId,
6826
+ repositoryIds,
6827
+ ))
6828
+ ) {
6829
+ throw new Error(
6830
+ "This workspace no longer authorizes one or more GitHub repositories attached to the session",
6831
+ );
6832
+ }
6833
+ }
6834
+
5385
6835
  /**
5386
6836
  * True when the error is transient upstream backpressure — a model-provider 5xx,
5387
6837
  * a "server had a bad minute" body, or a dropped/again-able network connection —
@@ -5433,6 +6883,12 @@ export function agentRunFailurePayload(error: unknown): {
5433
6883
  code?: string;
5434
6884
  retryable?: boolean;
5435
6885
  detail?: string;
6886
+ correlationId?: string;
6887
+ stage?: string;
6888
+ sqlState?: string | null;
6889
+ attempts?: number;
6890
+ retryOutcome?: string;
6891
+ database?: Record<string, string>;
5436
6892
  } {
5437
6893
  const message = error instanceof Error ? error.message : String(error);
5438
6894
  const status =
@@ -5443,6 +6899,41 @@ export function agentRunFailurePayload(error: unknown): {
5443
6899
  typeof error === "object" && error !== null && "code" in error
5444
6900
  ? String((error as { code?: unknown }).code)
5445
6901
  : undefined;
6902
+ // An accepted Codex stream with no terminal response is malformed/partial,
6903
+ // not provider backpressure. Replaying the same accepted turn could repeat
6904
+ // model or tool effects, so this marked transport failure must outrank the
6905
+ // generic 5xx retry classifier (CodexStreamingTerminalError uses status 502).
6906
+ if (isCodexTransportError(error) && code === "invalid_sse_terminal") {
6907
+ return {
6908
+ error: "The Codex response stream ended without a terminal response",
6909
+ code: "invalid_sse_terminal",
6910
+ retryable: false,
6911
+ };
6912
+ }
6913
+ if (isSessionEventPersistenceError(error)) {
6914
+ const { details } = error;
6915
+ const eventLabel = details.eventTypes.join(", ") || "session events";
6916
+ const failureLabel =
6917
+ details.code === "db_deadlock"
6918
+ ? "Database deadlock"
6919
+ : details.code === "db_serialization_failure"
6920
+ ? "Database serialization failure"
6921
+ : "Database failure";
6922
+ return {
6923
+ error: `${failureLabel} while persisting ${eventLabel}. The completed provider call and external effects were not retried.`,
6924
+ code: details.code,
6925
+ detail:
6926
+ details.retryOutcome === "exhausted"
6927
+ ? `The idempotent persistence transaction failed after ${details.attempts} attempts.`
6928
+ : "The database rejected the idempotent persistence transaction.",
6929
+ correlationId: details.correlationId,
6930
+ stage: details.stage,
6931
+ sqlState: details.sqlState,
6932
+ attempts: details.attempts,
6933
+ retryOutcome: details.retryOutcome,
6934
+ ...(Object.keys(details.database).length > 0 ? { database: details.database } : {}),
6935
+ };
6936
+ }
5446
6937
  // A ChatGPT/Codex usage cap is a HARD limit, not transient backpressure: it
5447
6938
  // must NOT be reported as a generic, retryable rate-limit (which would loop a
5448
6939
  // goal against a capped backend). Surface a precise, actionable message with
@@ -5511,7 +7002,6 @@ export function codexCredentialCooldownUntil(
5511
7002
  CodexAccountStatus,
5512
7003
  "primaryUsedPercent" | "primaryResetAt" | "secondaryUsedPercent" | "secondaryResetAt"
5513
7004
  > | null,
5514
- nearExhaustionPct: number,
5515
7005
  now: Date,
5516
7006
  ): Date | null {
5517
7007
  if (failure.kind === "auth" || failure.kind === "forbidden") {
@@ -5533,7 +7023,7 @@ export function codexCredentialCooldownUntil(
5533
7023
  ]
5534
7024
  .filter(
5535
7025
  (window): window is { used: number; reset: Date } =>
5536
- (window.used ?? 0) >= nearExhaustionPct &&
7026
+ (window.used ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT &&
5537
7027
  window.reset instanceof Date &&
5538
7028
  window.reset.getTime() > now.getTime(),
5539
7029
  )
@@ -5826,6 +7316,7 @@ export async function recordModelUsageAndDebitCredits(
5826
7316
  workspaceId: string;
5827
7317
  sessionId: string;
5828
7318
  turnId: string;
7319
+ turnAttemptId: string;
5829
7320
  model: string;
5830
7321
  isCodexTurn: boolean;
5831
7322
  usage?: ModelUsageInput | null;
@@ -5857,6 +7348,9 @@ export async function recordModelUsageAndDebitCredits(
5857
7348
  unit: "usd_micros",
5858
7349
  sourceResourceType: "model_response",
5859
7350
  sourceResourceId: `${input.turnId}:${input.sourceKey}`,
7351
+ sessionId: input.sessionId,
7352
+ turnId: input.turnId,
7353
+ turnAttemptId: input.turnAttemptId,
5860
7354
  idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`,
5861
7355
  });
5862
7356
  return;
@@ -5870,6 +7364,9 @@ export async function recordModelUsageAndDebitCredits(
5870
7364
  unit: "tokens",
5871
7365
  sourceResourceType: "model_response",
5872
7366
  sourceResourceId: `${input.turnId}:${input.sourceKey}`,
7367
+ sessionId: input.sessionId,
7368
+ turnId: input.turnId,
7369
+ turnAttemptId: input.turnAttemptId,
5873
7370
  idempotencyKey: `usage:model.tokens:${input.turnId}:${input.sourceKey}`,
5874
7371
  });
5875
7372
  }
@@ -5889,6 +7386,9 @@ export async function recordModelUsageAndDebitCredits(
5889
7386
  unit: "usd_micros",
5890
7387
  sourceResourceType: "model_response",
5891
7388
  sourceResourceId: `${input.turnId}:${input.sourceKey}`,
7389
+ sessionId: input.sessionId,
7390
+ turnId: input.turnId,
7391
+ turnAttemptId: input.turnAttemptId,
5892
7392
  idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`,
5893
7393
  });
5894
7394
  if (costMicros > 0) {