@opengeni/worker-bundle 0.22.8 → 0.23.3-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/dist/activities/agent-turn/agent-build.d.ts +1 -4
  2. package/dist/activities/agent-turn/claim.d.ts +2 -4
  3. package/dist/activities/agent-turn/codex.d.ts +6 -5
  4. package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
  5. package/dist/activities/agent-turn/errors.d.ts +13 -0
  6. package/dist/activities/agent-turn/finalization.d.ts +1 -3
  7. package/dist/activities/agent-turn/quiescence.d.ts +39 -0
  8. package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
  9. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
  10. package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
  11. package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
  12. package/dist/activities/agent-turn/tool-policy.d.ts +3 -32
  13. package/dist/activities/agent-turn/turn-context.d.ts +1 -8
  14. package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
  15. package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
  16. package/dist/activities/codex-rotation.d.ts +2 -0
  17. package/dist/activities/session-state.d.ts +2 -1
  18. package/dist/{activities-control-VZIIV7RX.js → activities-control-52W5IDWM.js} +125 -13
  19. package/dist/activities-control-52W5IDWM.js.map +1 -0
  20. package/dist/{activities-turn-5RGQDBFV.js → activities-turn-A2G2KLSY.js} +534 -482
  21. package/dist/activities-turn-A2G2KLSY.js.map +1 -0
  22. package/dist/{chunk-NAVSAZ56.js → chunk-GFNXVZVQ.js} +160 -1
  23. package/dist/chunk-GFNXVZVQ.js.map +1 -0
  24. package/dist/{chunk-O4HDX5SC.js → chunk-IPVMCAW5.js} +144 -15
  25. package/dist/chunk-IPVMCAW5.js.map +1 -0
  26. package/dist/index.js +42 -2
  27. package/dist/index.js.map +1 -1
  28. package/dist/observability-metrics.d.ts +28 -0
  29. package/dist/personal-github-git-credentials.d.ts +1 -0
  30. package/dist/unhandled-rejection-boundary.d.ts +29 -0
  31. package/package.json +18 -18
  32. package/src/activities/agent-turn/agent-build.ts +15 -31
  33. package/src/activities/agent-turn/claim.ts +29 -68
  34. package/src/activities/agent-turn/codex.ts +19 -10
  35. package/src/activities/agent-turn/compaction-prep.ts +50 -1
  36. package/src/activities/agent-turn/errors.ts +78 -0
  37. package/src/activities/agent-turn/failure-settlement.ts +30 -19
  38. package/src/activities/agent-turn/finalization.ts +24 -21
  39. package/src/activities/agent-turn/history.ts +5 -1
  40. package/src/activities/agent-turn/quiescence.ts +86 -0
  41. package/src/activities/agent-turn/run-credentials.ts +12 -1
  42. package/src/activities/agent-turn/run.ts +14 -11
  43. package/src/activities/agent-turn/sandbox-establish.ts +46 -11
  44. package/src/activities/agent-turn/sandbox-route.ts +21 -25
  45. package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
  46. package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
  47. package/src/activities/agent-turn/stream-attempt.ts +98 -14
  48. package/src/activities/agent-turn/tool-environment.ts +10 -13
  49. package/src/activities/agent-turn/tool-policy.ts +3 -55
  50. package/src/activities/agent-turn/turn-context.ts +8 -14
  51. package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
  52. package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
  53. package/src/activities/codex-capacity.ts +13 -5
  54. package/src/activities/codex-rotation.ts +17 -0
  55. package/src/activities/goals.ts +10 -4
  56. package/src/activities/retained-screenshots.ts +104 -21
  57. package/src/activities/sandbox-lease.ts +7 -4
  58. package/src/activities/scheduled-tasks.ts +128 -4
  59. package/src/activities/session-state.ts +8 -1
  60. package/src/github-rest-mcp.ts +6 -3
  61. package/src/http.ts +3 -0
  62. package/src/index.ts +19 -0
  63. package/src/observability-metrics.ts +200 -0
  64. package/src/personal-github-git-credentials.ts +45 -4
  65. package/src/sandbox-resume.ts +1 -1
  66. package/src/sandbox-routing.ts +7 -4
  67. package/src/unhandled-rejection-boundary.ts +73 -0
  68. package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
  69. package/dist/activities/recording.d.ts +0 -90
  70. package/dist/activities-control-VZIIV7RX.js.map +0 -1
  71. package/dist/activities-turn-5RGQDBFV.js.map +0 -1
  72. package/dist/chunk-NAVSAZ56.js.map +0 -1
  73. package/dist/chunk-O4HDX5SC.js.map +0 -1
  74. package/src/activities/recording.ts +0 -279
@@ -3,6 +3,7 @@ import {
3
3
  ApprovalRunStateLimitExceededError,
4
4
  isRetryableDatabaseTransportFailure,
5
5
  isSessionEventPersistenceError,
6
+ SandboxLeaseTransitionError,
6
7
  } from "@opengeni/db";
7
8
  import {
8
9
  ActiveBackendUnresolvableError,
@@ -10,6 +11,7 @@ import {
10
11
  EmptyCompactionSummaryError,
11
12
  isMcpRequestTimeoutError,
12
13
  isMcpTransportConnectivityError,
14
+ isModalTaskExecStartDnsResolutionError,
13
15
  RoutingWorkspaceRootChangedError,
14
16
  SelfhostedWorkspaceRootChangedError,
15
17
  UNKNOWN_MODEL_FINISH_REASON_CODE,
@@ -104,6 +106,7 @@ export function providerRecoveryResult(input: {
104
106
  ? (providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS)
105
107
  : input.failureCode === "provider_unavailable" ||
106
108
  input.failureCode === "upstream_connectivity_unavailable" ||
109
+ input.failureCode === "sandbox_command_start_unavailable" ||
107
110
  input.failureCode === "mcp_transport_timeout" ||
108
111
  input.failureCode === "mcp_transport_unavailable" ||
109
112
  input.failureCode === POST_COMPACTION_CONTINUATION_EMPTY_CODE
@@ -151,6 +154,13 @@ export function providerRecoveryCountFromMetadata(metadata: Record<string, unkno
151
154
  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
152
155
  }
153
156
 
157
+ export function providerRecoveryCountAfterModelRequestPhase(
158
+ currentCount: number,
159
+ phase: string,
160
+ ): number {
161
+ return phase === "completed" ? 0 : currentCount;
162
+ }
163
+
154
164
  export function headerValue(headers: unknown, name: string): string | null {
155
165
  if (!headers || typeof headers !== "object") return null;
156
166
  const getter = (headers as { get?: unknown }).get;
@@ -367,6 +377,66 @@ export function isWorkerShutdownCancellation(error: unknown): boolean {
367
377
  return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
368
378
  }
369
379
 
380
+ export type SandboxLifecycleTransitionDiagnostic = {
381
+ sandboxGroupId: string;
382
+ leaseEpoch: number;
383
+ reason: "capture_in_progress" | "rotation_in_progress" | "provider_recovery_in_progress";
384
+ };
385
+
386
+ /**
387
+ * Recover a typed sandbox lifecycle transition through the structural wrappers
388
+ * used by parallel Agents SDK function-tool execution. Never infer transition
389
+ * truth from message text: only the original class or an exact, fully-shaped
390
+ * cross-package error object is accepted.
391
+ */
392
+ export function sandboxLifecycleTransitionDiagnostic(
393
+ error: unknown,
394
+ ): SandboxLifecycleTransitionDiagnostic | null {
395
+ const pending: unknown[] = [error];
396
+ const seen = new WeakSet<object>();
397
+ let inspected = 0;
398
+
399
+ while (pending.length > 0 && inspected < 64) {
400
+ const current = pending.shift();
401
+ inspected += 1;
402
+ if (!current || typeof current !== "object" || seen.has(current)) continue;
403
+ seen.add(current);
404
+
405
+ try {
406
+ const record = current as Record<string, unknown>;
407
+ const reason = record.reason;
408
+ if (
409
+ (current instanceof SandboxLeaseTransitionError ||
410
+ record.name === "SandboxLeaseTransitionError") &&
411
+ typeof record.sandboxGroupId === "string" &&
412
+ record.sandboxGroupId.length > 0 &&
413
+ typeof record.leaseEpoch === "number" &&
414
+ Number.isSafeInteger(record.leaseEpoch) &&
415
+ record.leaseEpoch >= 0 &&
416
+ (reason === "capture_in_progress" ||
417
+ reason === "rotation_in_progress" ||
418
+ reason === "provider_recovery_in_progress")
419
+ ) {
420
+ return {
421
+ sandboxGroupId: record.sandboxGroupId,
422
+ leaseEpoch: record.leaseEpoch,
423
+ reason,
424
+ };
425
+ }
426
+
427
+ for (const key of ["cause", "error"] as const) {
428
+ const nested = record[key];
429
+ if (nested && typeof nested === "object") pending.push(nested);
430
+ }
431
+ if (Array.isArray(record.errors)) pending.push(...record.errors.slice(0, 32));
432
+ } catch {
433
+ // A hostile proxy or getter is not durable lifecycle evidence.
434
+ }
435
+ }
436
+
437
+ return null;
438
+ }
439
+
370
440
  /**
371
441
  * Recognize the one active-route transition that cannot finish inside its
372
442
  * originating attempt. A Modal-home session may start on a Connected Machine
@@ -812,6 +882,14 @@ export function agentRunFailurePayload(
812
882
  retryable: true,
813
883
  };
814
884
  }
885
+ if (isModalTaskExecStartDnsResolutionError(error)) {
886
+ return {
887
+ error:
888
+ "The managed sandbox command transport was temporarily unreachable before the command started. The same turn will retry after a short delay.",
889
+ code: "sandbox_command_start_unavailable",
890
+ retryable: true,
891
+ };
892
+ }
815
893
  // An accepted Codex stream with no terminal response is malformed/partial,
816
894
  // not provider backpressure. Replaying the same accepted turn could repeat
817
895
  // model or tool effects, so this marked transport failure must outrank the
@@ -18,7 +18,6 @@ import {
18
18
  settleCodexCredentialFailover,
19
19
  readLease,
20
20
  SandboxLeaseSupersededError,
21
- SandboxLeaseTransitionError,
22
21
  isSessionEventPersistenceError,
23
22
  } from "@opengeni/db";
24
23
  import { publishDurableSessionEvents } from "@opengeni/events";
@@ -61,6 +60,7 @@ import {
61
60
  escapedMcpTimeoutRecoveryFailure,
62
61
  preClaimAdmissionFailure,
63
62
  isWorkerShutdownCancellation,
63
+ sandboxLifecycleTransitionDiagnostic,
64
64
  sandboxRouteTransitionCode,
65
65
  safeErrorDiagnostic,
66
66
  classifyXaiCredentialFailure,
@@ -159,20 +159,20 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
159
159
  // recoverable control-plane states, never session failures. The latter is
160
160
  // paced briefly; the next acquire waits on the durable claim and resumes
161
161
  // immediately when it clears.
162
- if (
163
- (error instanceof SandboxLeaseSupersededError ||
164
- error instanceof SandboxLeaseTransitionError) &&
165
- recoveryTurnId
166
- ) {
162
+ const lifecycleTransition = sandboxLifecycleTransitionDiagnostic(error);
163
+ const leaseControlError =
164
+ error instanceof SandboxLeaseSupersededError ? error : lifecycleTransition;
165
+ if (leaseControlError && recoveryTurnId) {
167
166
  try {
168
- const fencedLease = await readLease(db, input.workspaceId, error.sandboxGroupId).catch(
169
- () => null,
170
- );
171
- const lifecycleTransition = error instanceof SandboxLeaseTransitionError;
167
+ const fencedLease = await readLease(
168
+ db,
169
+ input.workspaceId,
170
+ leaseControlError.sandboxGroupId,
171
+ ).catch(() => null);
172
172
  const rotationPending =
173
173
  fencedLease?.rotationRequestedAt != null ||
174
- (lifecycleTransition && error.reason === "rotation_in_progress");
175
- const transitionPending = lifecycleTransition || rotationPending;
174
+ lifecycleTransition?.reason === "rotation_in_progress";
175
+ const transitionPending = lifecycleTransition !== null || rotationPending;
176
176
  const deadlineRotationPending =
177
177
  rotationPending && fencedLease?.rotationReason === "provider_deadline";
178
178
  const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
@@ -182,20 +182,20 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
182
182
  attemptId: input.attemptId,
183
183
  reason: deadlineRotationPending
184
184
  ? "sandbox_deadline_rotation"
185
- : lifecycleTransition
185
+ : lifecycleTransition !== null
186
186
  ? "sandbox_lifecycle_transition"
187
187
  : "sandbox_lease_superseded",
188
188
  ...(transitionPending
189
189
  ? {
190
190
  detail: {
191
- sandboxGroupId: error.sandboxGroupId,
192
- leaseEpoch: error.leaseEpoch,
191
+ sandboxGroupId: leaseControlError.sandboxGroupId,
192
+ leaseEpoch: leaseControlError.leaseEpoch,
193
193
  ...(rotationPending
194
194
  ? {
195
195
  rotationReason: fencedLease?.rotationReason ?? "operator",
196
196
  }
197
197
  : {}),
198
- ...(lifecycleTransition ? { transitionReason: error.reason } : {}),
198
+ ...(lifecycleTransition ? { transitionReason: lifecycleTransition.reason } : {}),
199
199
  },
200
200
  }
201
201
  : {}),
@@ -647,7 +647,11 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
647
647
  status: "error",
648
648
  lastError: "model request was forbidden for this credential",
649
649
  }
650
- : { kind: "cooldown", until: cooldownUntil! },
650
+ : {
651
+ kind: "cooldown",
652
+ until: cooldownUntil!,
653
+ cooldownKind: codexCredentialFailure.kind,
654
+ },
651
655
  })
652
656
  : false;
653
657
  if (!statePersisted && leases.codex.holderId && leases.codex.generation !== null) {
@@ -903,6 +907,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
903
907
  let rotationResumeIdleUntilReset = false; // circuit-breaker fall (Finding 1b) ⇒ MANDATORY hold
904
908
  let allCappedResetAt: Date | null = null; // set ⇒ every account capped; idle until this
905
909
  let capacityAuthoritativeResetAt: Date | null = null;
910
+ let capacityNeedsBoundedRefresh = false;
906
911
  if (providerTurn.effectiveCodexCredentialId) {
907
912
  const [rotation, sessionCodex] = await Promise.all([
908
913
  getCodexRotationSettings(db, input.workspaceId).catch(() => null),
@@ -941,6 +946,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
941
946
  input.workspaceId,
942
947
  providerTurn.effectiveCodexCredentialId,
943
948
  until,
949
+ "quota",
944
950
  ).catch(() => null);
945
951
  const cooldownPersisted = cooldownMutation?.result ?? false;
946
952
  if (cooldownMutation) {
@@ -953,7 +959,9 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
953
959
  // write, so stamp the just-cooled account so the engine excludes it now. The
954
960
  // serving account is thus walked AT MOST ONCE per turn (invariant 4: bounded).
955
961
  const fresh = accounts.map((a) =>
956
- a.id === providerTurn.effectiveCodexCredentialId ? { ...a, exhaustedUntil: until } : a,
962
+ a.id === providerTurn.effectiveCodexCredentialId
963
+ ? { ...a, exhaustedUntil: until, exhaustedKind: "quota" as const }
964
+ : a,
957
965
  );
958
966
  if (reactiveSharded) {
959
967
  // AM-5: RE-SHARD over the healthy survivors (the just-capped serving account is
@@ -1017,6 +1025,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
1017
1025
  rotated = true;
1018
1026
  allCappedResetAt = earliestCodexReset(fresh, new Date());
1019
1027
  capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
1028
+ capacityNeedsBoundedRefresh = true;
1020
1029
  }
1021
1030
  } else {
1022
1031
  const decision = chooseRotationActive({
@@ -1048,6 +1057,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
1048
1057
  rotated = true;
1049
1058
  allCappedResetAt = decision.earliestResetAt;
1050
1059
  capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
1060
+ capacityNeedsBoundedRefresh = true;
1051
1061
  }
1052
1062
  // kind:"none" → fall through to today's single-account idle.
1053
1063
  }
@@ -1088,7 +1098,8 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
1088
1098
  goalId: goalActive && goal ? goal.id : null,
1089
1099
  goalVersion: goalActive && goal ? goal.version : null,
1090
1100
  earliestResetAt: providerResetAt,
1091
- resetKind: providerResetAt ? "authoritative" : "bounded_refresh",
1101
+ resetKind:
1102
+ providerResetAt && !capacityNeedsBoundedRefresh ? "authoritative" : "bounded_refresh",
1092
1103
  failurePayload,
1093
1104
  ...(leases.codex.holderId && leases.codex.generation !== null
1094
1105
  ? {
@@ -23,7 +23,11 @@ import type {
23
23
  import { captureWorkspaceRevision, openFreshWorkspaceCaptureSession } from "../workspace-capture";
24
24
  import { type ChannelASession } from "@opengeni/runtime/sandbox";
25
25
  import { createModelCheckpointMemoryCollector } from "../../model-checkpoint-memory-collector";
26
- import { makeMachineOpObserver, turnLifecycleMetricsFor } from "../../observability-metrics";
26
+ import {
27
+ makeMachineOpObserver,
28
+ recordSessionEventAppendPhase,
29
+ turnLifecycleMetricsFor,
30
+ } from "../../observability-metrics";
27
31
  import {
28
32
  maybePersistWarmWorkspaceSnapshot,
29
33
  waitForWarmSnapshot,
@@ -34,6 +38,7 @@ import { safeErrorDiagnostic, safeErrorForTelemetry } from "./errors";
34
38
  import {
35
39
  assertPhysicalToolQuiescenceForCancellation,
36
40
  assertSessionAttemptQuiescenceRecoveryDurable,
41
+ armTurnQuiescenceWatchdog,
37
42
  clearAttemptCredentialsWithSettledFence,
38
43
  drainAttemptOwnedSandboxWriters,
39
44
  persistOrSignalSessionAttemptQuiescence,
@@ -46,7 +51,6 @@ import type {
46
51
  AttemptIdentityState,
47
52
  EventingState,
48
53
  ProviderTurnState,
49
- RecordingState,
50
54
  RenewalState,
51
55
  SandboxRuntimeState,
52
56
  TurnControlState,
@@ -76,7 +80,6 @@ export type TurnFinalizationDeps = {
76
80
  attempt: AttemptIdentityState;
77
81
  sandboxState: SandboxRuntimeState;
78
82
  renewals: RenewalState;
79
- recordingState: RecordingState;
80
83
  eventing: EventingState;
81
84
  providerTurn: ProviderTurnState;
82
85
  workspaceRefs: WorkspaceRefState;
@@ -87,7 +90,6 @@ export type TurnFinalizationDeps = {
87
90
  ) => Promise<T>;
88
91
  requireResolvedSandboxForMutation: (message: string) => ResumedTurnSandbox;
89
92
  stopLeaseHeartbeat: () => void;
90
- abandonActiveRecording: (reason: string, disposition?: "failed" | "discard") => Promise<void>;
91
93
  turnCompletionMemoryCollector: ReturnType<typeof createModelCheckpointMemoryCollector>;
92
94
  };
93
95
 
@@ -115,14 +117,12 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
115
117
  attempt,
116
118
  sandboxState,
117
119
  renewals,
118
- recordingState,
119
120
  eventing,
120
121
  providerTurn,
121
122
  workspaceRefs,
122
123
  runWorkspaceMutationForSandbox,
123
124
  requireResolvedSandboxForMutation,
124
125
  stopLeaseHeartbeat,
125
- abandonActiveRecording,
126
126
  turnCompletionMemoryCollector,
127
127
  } = deps;
128
128
  // This is the logical ownership boundary. Abort before any fallible
@@ -137,6 +137,17 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
137
137
  let finalizationError: unknown;
138
138
  let physicalToolQuiescenceConfirmed = !control.acknowledgeQuiescence;
139
139
  let quiescenceReceiptOrProofDurable = !control.acknowledgeQuiescence;
140
+ const disarmQuiescenceWatchdog = armTurnQuiescenceWatchdog({
141
+ enabled: control.acknowledgeQuiescence,
142
+ onTimeout: () => {
143
+ observability.error("turn quiescence drain exceeded hard containment deadline", {
144
+ "opengeni.session_id": input.sessionId,
145
+ "opengeni.turn_id": attempt.turnId ?? "",
146
+ "opengeni.attempt_id": input.attemptId,
147
+ });
148
+ },
149
+ terminateWorker: () => process.exit(1),
150
+ });
140
151
  const finalizerSignal = turnFinalizerCancellationSignal(
141
152
  cancellationSignal,
142
153
  control.activityStatus,
@@ -303,6 +314,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
303
314
  },
304
315
  });
305
316
  quiescenceReceiptOrProofDurable = true;
317
+ disarmQuiescenceWatchdog();
306
318
  if (recoveryMode === "signal") {
307
319
  observability.info("agent turn quiescence proof handed to workflow recovery", {
308
320
  "opengeni.session_id": input.sessionId,
@@ -332,6 +344,10 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
332
344
  ...event,
333
345
  turnId: attempt.turnId ?? null,
334
346
  })),
347
+ {
348
+ onAppendPhase: (observation) =>
349
+ recordSessionEventAppendPhase(observability, observation),
350
+ },
335
351
  ).catch(() => undefined),
336
352
  finalizerSignal,
337
353
  );
@@ -435,10 +451,8 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
435
451
  // Workbench v2 turn-end workspace capture — runs FIRST in
436
452
  // the turn-end finally, while the box is MAXIMALLY ALIVE. The agent's last
437
453
  // tool ran before this finally, so /workspace is already final; capture is
438
- // FS-equivalent to the already-settled recording preparation and the warm
439
- // snapshot (neither mutates workspace files). Running it here — BEFORE
440
- // preparedTools.close() (which tears down tools / computer-use / the display
441
- // stack and is what starts the Modal box exiting a few seconds later) —
454
+ // FS-equivalent to the warm snapshot (neither mutates workspace files).
455
+ // Running it here — BEFORE preparedTools.close() tears down tools —
442
456
  // gives capture the full live-box margin instead of racing the teardown
443
457
  // tail, which was dropping 100% of captures on real Modal desktop boxes
444
458
  // ("request cancelled due to container exiting", 0 rows). External module:
@@ -556,17 +570,6 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
556
570
  () => undefined,
557
571
  );
558
572
  }
559
- // A recording normally closes inside the attempt-fenced turn settlement.
560
- // Reaching finally with one still active means settlement threw, never ran,
561
- // or lost ownership. Stop ffmpeg and mark only this exact attempt-owned row
562
- // failed; publish no event and leave the artifact recoverable on the box.
563
- await waitForTurnFinalizerStep(
564
- abandonActiveRecording(
565
- "activity ended without recording settlement",
566
- recordingState.didComputerUse ? "failed" : "discard",
567
- ),
568
- finalizerSignal,
569
- );
570
573
  if (sandboxState.resolvedSandbox) {
571
574
  // TURN-END mid-session snapshot (sandbox-file-persistence): fold the
572
575
  // turn's finished /workspace onto the lease before releasing the holder,
@@ -169,7 +169,11 @@ export function pendingToolCallFromSdkEvent(event: unknown): {
169
169
 
170
170
  /** Tools whose intentional visual result must survive inline-media sanitization. */
171
171
  export function toolCallProducesRetainableSessionImage(name: string | null): boolean {
172
- return name === "computer_screenshot" || name === "view_image";
172
+ return (
173
+ name === "computer_screenshot" ||
174
+ name === "view_image" ||
175
+ name === "interaction__computer_observe"
176
+ );
173
177
  }
174
178
 
175
179
  export function completedToolCallFromSdkEvent(event: unknown): {
@@ -79,6 +79,32 @@ export function assertSessionAttemptQuiescenceRecoveryDurable(input: {
79
79
 
80
80
  export const QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS = 250;
81
81
  export const QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS = 5_000;
82
+ export const TURN_QUIESCENCE_WATCHDOG_MS = 5 * 60_000;
83
+
84
+ /** A fenced activity must never heartbeat forever while its physical writer
85
+ * drain is stuck. The last-resort worker exit stops every in-process writer;
86
+ * Kubernetes replaces the pod and the workflow's heartbeat-lease reconciler
87
+ * then commits the exact quiescence receipt. */
88
+ export function armTurnQuiescenceWatchdog(input: {
89
+ enabled: boolean;
90
+ timeoutMs?: number;
91
+ onTimeout?: () => void;
92
+ terminateWorker: () => void;
93
+ }): () => void {
94
+ if (!input.enabled) return () => undefined;
95
+ let armed = true;
96
+ const timer = setTimeout(() => {
97
+ if (!armed) return;
98
+ input.onTimeout?.();
99
+ input.terminateWorker();
100
+ }, input.timeoutMs ?? TURN_QUIESCENCE_WATCHDOG_MS);
101
+ timer.unref?.();
102
+ return () => {
103
+ if (!armed) return;
104
+ armed = false;
105
+ clearTimeout(timer);
106
+ };
107
+ }
82
108
 
83
109
  /** Persist the authoritative receipt or durably hand the exact physical proof
84
110
  * to Temporal. This retries signal delivery, not DB eligibility or workflow
@@ -253,6 +279,66 @@ export async function waitForTurnStreamCleanup(
253
279
  await waitForTurnFinalizerStep(providerCompleted, signal);
254
280
  }
255
281
 
282
+ export class IncompleteAgentStreamError extends Error {
283
+ readonly name = "IncompleteAgentStreamError";
284
+
285
+ constructor(message = "Agent stream ended without a successful terminal output") {
286
+ super(message);
287
+ }
288
+ }
289
+
290
+ /**
291
+ * Prove that SDK completion authority did not report a failure or runtime
292
+ * cancellation. The Agents SDK closes its ReadableStream immediately when its
293
+ * run signal aborts, so iterator EOF is not completion authority: the separate
294
+ * `completed` promise carries the eventual run failure.
295
+ *
296
+ * Only Temporal cancellation detaches cleanup. Internal runtime cancellation
297
+ * such as a provider-deadline sandbox rotation must escape to the turn failure
298
+ * settlement path, which checkpoints and redispatches the same logical turn.
299
+ */
300
+ export async function assertSuccessfulAgentStreamCompletion(input: {
301
+ batcherFlush: Promise<unknown>;
302
+ stream: {
303
+ completed: Promise<void>;
304
+ error: unknown;
305
+ };
306
+ temporalCancellationSignal: AbortSignal | undefined;
307
+ runtimeCancellationSignal: AbortSignal;
308
+ }): Promise<void> {
309
+ await waitForTurnStreamCleanup(
310
+ input.batcherFlush,
311
+ input.stream.completed,
312
+ input.temporalCancellationSignal,
313
+ );
314
+
315
+ if (input.runtimeCancellationSignal.aborted) {
316
+ throw (
317
+ input.runtimeCancellationSignal.reason ??
318
+ new IncompleteAgentStreamError("Agent stream was cancelled without a terminal reason")
319
+ );
320
+ }
321
+ if (input.stream.error !== null && input.stream.error !== undefined) {
322
+ throw input.stream.error;
323
+ }
324
+ }
325
+
326
+ /** Call after any more-specific empty-stream recovery has had priority. */
327
+ export function assertAgentStreamNotCancelled(cancelled: boolean): void {
328
+ if (cancelled) {
329
+ throw new IncompleteAgentStreamError("Agent stream was cancelled without a terminal result");
330
+ }
331
+ }
332
+
333
+ /** Interruption streams do not produce a normal final output. Call this only
334
+ * after the interruption branch has settled any required action. */
335
+ export function requireAgentStreamFinalOutput(finalOutput: unknown | undefined): unknown {
336
+ if (finalOutput === undefined) {
337
+ throw new IncompleteAgentStreamError();
338
+ }
339
+ return finalOutput;
340
+ }
341
+
256
342
  /**
257
343
  * Terminal settlement closes the active attempt before turn finalization. The
258
344
  * ordinary workspace admission therefore correctly returns `attempt_fenced`
@@ -56,6 +56,7 @@ import {
56
56
  sandboxEstablishPolicyDecision,
57
57
  ensureTurnModalRegistryImage,
58
58
  sandboxArtifactRuntimeAdmission,
59
+ sandboxSettingsForRoute,
59
60
  } from "./sandbox-route";
60
61
  import type { ClaimTurnOk } from "./claim";
61
62
  import type { GovernanceModelOk } from "./governance-model";
@@ -134,6 +135,16 @@ export async function prepareRunCredentials(deps: PrepareRunCredentialsDeps) {
134
135
  connectionScope,
135
136
  runWorkspaceMutationForSandbox,
136
137
  } = deps;
138
+ // A machine-home row carries selfhosted path defaults, but clearing its
139
+ // pointer explicitly selects the deployment-managed group. Build the stable
140
+ // manifest environment with that effective provider just as Channel-A and
141
+ // viewer readiness do; otherwise the verified managed box and the agent can
142
+ // disagree on backend-aware HOME/token-file paths.
143
+ const sandboxEnvironmentSettings = sandboxSettingsForRoute({
144
+ runSettings,
145
+ machinePrimary,
146
+ groupBoxBackend,
147
+ });
137
148
 
138
149
  const runCredentialResolver =
139
150
  effectiveRunCredentialBackend === "none"
@@ -321,7 +332,7 @@ export async function prepareRunCredentials(deps: PrepareRunCredentialsDeps) {
321
332
  codemodeTokenExpiresAt: sandboxCodemodeTokenExpiresAt,
322
333
  } = await waitForTurnOperation(
323
334
  sandboxEnvironmentForRun(
324
- runSettings,
335
+ sandboxEnvironmentSettings,
325
336
  turnResources,
326
337
  // Rig default sets merged BELOW the session set (session wins); rig-less
327
338
  // turns pass exactly workspaceVariableSet?.values (byte-for-byte today).
@@ -45,6 +45,7 @@ import { createTurnCredentialLeases } from "./credential-leases";
45
45
  import { createTurnMediaArtifacts } from "./media-artifacts";
46
46
  import { createTurnHistorySink } from "./history-sink";
47
47
  import { checkpointHistoryBeforeProviderDispatch } from "./provider-dispatch-barrier";
48
+ import { providerRecoveryCountAfterModelRequestPhase } from "./errors";
48
49
  import { sandboxRunAs } from "@opengeni/runtime";
49
50
  import { randomUUID } from "node:crypto";
50
51
  import { createModelCheckpointMemoryCollector } from "../../model-checkpoint-memory-collector";
@@ -124,7 +125,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
124
125
  billingState,
125
126
  sandboxState,
126
127
  renewals,
127
- recordingState,
128
128
  eventing,
129
129
  workspaceRefs,
130
130
  providerTurn,
@@ -222,7 +222,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
222
222
  activityContext,
223
223
  sandboxRotationController,
224
224
  sandboxState,
225
- recordingState,
226
225
  eventing,
227
226
  attempt,
228
227
  });
@@ -230,9 +229,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
230
229
  requireResolvedSandboxForMutation,
231
230
  runWorkspaceMutationForSandbox,
232
231
  stopLeaseHeartbeat,
233
- abandonActiveRecording,
234
232
  flushRuntimeBatcher,
235
- maybeStartOnTurnRecording,
236
233
  finalizeTurnOpStreamOps,
237
234
  } = sandboxRuntime;
238
235
 
@@ -308,7 +305,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
308
305
  db,
309
306
  bus,
310
307
  runtime,
311
- objectStorage,
312
308
  observability,
313
309
  entitlements,
314
310
  wakeSessionWorkflow,
@@ -320,13 +316,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
320
316
  attempt,
321
317
  billingState,
322
318
  sandboxState,
323
- recordingState,
324
319
  eventing,
325
320
  leases,
326
321
  media,
327
322
  claimedResult,
328
323
  acknowledgeLostAttemptOwnership,
329
- abandonActiveRecording,
330
324
  });
331
325
  if ("exit" in claimed) return claimed.exit;
332
326
  if (!eventing.publish || !eventing.settle) {
@@ -350,6 +344,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
350
344
  trigger,
351
345
  humanInputResume,
352
346
  interactionInterventionResume,
347
+ attachPendingUpdatesAfterOpenSuffix,
353
348
  throwIfWorkerShuttingDown,
354
349
  throwIfTurnCancelled,
355
350
  opJournal,
@@ -605,6 +600,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
605
600
  },
606
601
  },
607
602
  ]);
603
+ attempt.providerRecoveryCount = providerRecoveryCountAfterModelRequestPhase(
604
+ attempt.providerRecoveryCount,
605
+ event.phase,
606
+ );
608
607
  } catch (error) {
609
608
  auditOutcome = "failed";
610
609
  throw error;
@@ -750,6 +749,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
750
749
  },
751
750
  },
752
751
  ]);
752
+ attempt.providerRecoveryCount = providerRecoveryCountAfterModelRequestPhase(
753
+ attempt.providerRecoveryCount,
754
+ event.phase,
755
+ );
753
756
  } catch (error) {
754
757
  auditOutcome = "failed";
755
758
  throw error;
@@ -834,6 +837,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
834
837
  compactionModeOptions,
835
838
  compactionOnlyTurn,
836
839
  compactionSummarizerFor,
840
+ settleDeferredSteerAfterCompaction,
837
841
  } = compactionPrep.ok;
838
842
 
839
843
  const toolPolicy = await prepareTurnToolPolicy({
@@ -1112,8 +1116,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1112
1116
  eventing,
1113
1117
  attempt,
1114
1118
  sandboxState,
1115
- recordingState,
1116
- maybeStartOnTurnRecording,
1117
1119
  providerTurn,
1118
1120
  media,
1119
1121
  leases,
@@ -1278,6 +1280,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1278
1280
  compactionModeOptions,
1279
1281
  compactionOnlyTurn,
1280
1282
  compactionSummarizerFor,
1283
+ settleDeferredSteerAfterCompaction,
1281
1284
  agent,
1282
1285
  });
1283
1286
  if ("exit" in postAgentCompaction) return postAgentCompaction.exit;
@@ -1456,12 +1459,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1456
1459
  turn,
1457
1460
  trigger,
1458
1461
  humanInputResume,
1462
+ attachPendingUpdatesAfterOpenSuffix,
1459
1463
  agent,
1460
1464
  resolvedModel,
1461
1465
  providerApi,
1462
1466
  runSettings,
1463
1467
  turnTools,
1464
1468
  compactSummarizer,
1469
+ settleDeferredSteerAfterCompaction,
1465
1470
  compactionModelHistoryProjector,
1466
1471
  generatedImageHistoryProjector,
1467
1472
  modelHistoryProjector,
@@ -1525,14 +1530,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1525
1530
  attempt,
1526
1531
  sandboxState,
1527
1532
  renewals,
1528
- recordingState,
1529
1533
  eventing,
1530
1534
  providerTurn,
1531
1535
  workspaceRefs,
1532
1536
  runWorkspaceMutationForSandbox,
1533
1537
  requireResolvedSandboxForMutation,
1534
1538
  stopLeaseHeartbeat,
1535
- abandonActiveRecording,
1536
1539
  turnCompletionMemoryCollector,
1537
1540
  });
1538
1541
  }