@opengeni/worker-bundle 0.22.8 → 0.23.4-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +10 -2
  2. package/dist/activities/agent-turn/agent-build.d.ts +1 -4
  3. package/dist/activities/agent-turn/claim.d.ts +3 -4
  4. package/dist/activities/agent-turn/codex.d.ts +8 -7
  5. package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
  6. package/dist/activities/agent-turn/errors.d.ts +13 -0
  7. package/dist/activities/agent-turn/finalization.d.ts +1 -3
  8. package/dist/activities/agent-turn/list-models.d.ts +16 -0
  9. package/dist/activities/agent-turn/model-usage.d.ts +10 -1
  10. package/dist/activities/agent-turn/quiescence.d.ts +39 -0
  11. package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
  12. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
  13. package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
  14. package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
  15. package/dist/activities/agent-turn/tool-environment.d.ts +1 -0
  16. package/dist/activities/agent-turn/tool-policy.d.ts +7 -5
  17. package/dist/activities/agent-turn/turn-context.d.ts +3 -8
  18. package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
  19. package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
  20. package/dist/activities/automations.d.ts +1 -0
  21. package/dist/activities/capabilities.d.ts +2 -1
  22. package/dist/activities/codex-rotation.d.ts +2 -0
  23. package/dist/activities/goals.d.ts +11 -0
  24. package/dist/activities/image-generation-references.d.ts +2 -1
  25. package/dist/activities/scheduled-tasks.d.ts +1 -0
  26. package/dist/activities/session-state.d.ts +2 -1
  27. package/dist/activities/types.d.ts +8 -0
  28. package/dist/{activities-control-VZIIV7RX.js → activities-control-SGU2WEVB.js} +328 -107
  29. package/dist/activities-control-SGU2WEVB.js.map +1 -0
  30. package/dist/activities-control.d.ts +2 -0
  31. package/dist/{activities-turn-5RGQDBFV.js → activities-turn-7EAQXW6Q.js} +828 -540
  32. package/dist/activities-turn-7EAQXW6Q.js.map +1 -0
  33. package/dist/activities.d.ts +2 -0
  34. package/dist/{chunk-O4HDX5SC.js → chunk-S2QVRVJW.js} +227 -36
  35. package/dist/chunk-S2QVRVJW.js.map +1 -0
  36. package/dist/{chunk-NAVSAZ56.js → chunk-TJIPCHMP.js} +162 -2
  37. package/dist/chunk-TJIPCHMP.js.map +1 -0
  38. package/dist/index.d.ts +1 -0
  39. package/dist/index.js +69 -6
  40. package/dist/index.js.map +1 -1
  41. package/dist/observability-metrics.d.ts +29 -0
  42. package/dist/personal-github-git-credentials.d.ts +1 -0
  43. package/dist/unhandled-rejection-boundary.d.ts +29 -0
  44. package/dist/workflow-bundle.js +26 -3
  45. package/dist/workflows/activities.d.ts +5 -0
  46. package/dist/workflows/automations.d.ts +5 -0
  47. package/package.json +18 -18
  48. package/src/activities/agent-run-admission.ts +7 -1
  49. package/src/activities/agent-turn/admission.ts +28 -6
  50. package/src/activities/agent-turn/agent-build.ts +15 -31
  51. package/src/activities/agent-turn/claim.ts +53 -72
  52. package/src/activities/agent-turn/codex.ts +21 -12
  53. package/src/activities/agent-turn/compaction-prep.ts +52 -1
  54. package/src/activities/agent-turn/errors.ts +78 -0
  55. package/src/activities/agent-turn/failure-settlement.ts +30 -19
  56. package/src/activities/agent-turn/finalization.ts +24 -21
  57. package/src/activities/agent-turn/history.ts +5 -1
  58. package/src/activities/agent-turn/list-models.ts +101 -0
  59. package/src/activities/agent-turn/model-usage.ts +88 -42
  60. package/src/activities/agent-turn/quiescence.ts +86 -0
  61. package/src/activities/agent-turn/run-credentials.ts +12 -1
  62. package/src/activities/agent-turn/run.ts +20 -14
  63. package/src/activities/agent-turn/sandbox-establish.ts +46 -11
  64. package/src/activities/agent-turn/sandbox-route.ts +21 -25
  65. package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
  66. package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
  67. package/src/activities/agent-turn/stream-attempt.ts +106 -14
  68. package/src/activities/agent-turn/tool-environment.ts +72 -14
  69. package/src/activities/agent-turn/tool-policy.ts +7 -6
  70. package/src/activities/agent-turn/turn-context.ts +12 -14
  71. package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
  72. package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
  73. package/src/activities/automations.ts +122 -67
  74. package/src/activities/capabilities.ts +72 -2
  75. package/src/activities/codex-capacity.ts +13 -5
  76. package/src/activities/codex-rotation.ts +17 -0
  77. package/src/activities/goals.ts +117 -35
  78. package/src/activities/image-generation-references.ts +9 -9
  79. package/src/activities/retained-screenshots.ts +104 -21
  80. package/src/activities/sandbox-lease.ts +7 -4
  81. package/src/activities/scheduled-tasks.ts +248 -30
  82. package/src/activities/session-state.ts +8 -1
  83. package/src/activities/site-auth-maintenance.ts +12 -12
  84. package/src/activities/types.ts +6 -0
  85. package/src/activities.ts +1 -0
  86. package/src/activity-services.ts +4 -1
  87. package/src/github-rest-mcp.ts +8 -4
  88. package/src/http.ts +3 -0
  89. package/src/index.ts +38 -4
  90. package/src/observability-metrics.ts +202 -0
  91. package/src/personal-github-git-credentials.ts +45 -4
  92. package/src/sandbox-resume.ts +1 -1
  93. package/src/sandbox-routing.ts +7 -4
  94. package/src/unhandled-rejection-boundary.ts +73 -0
  95. package/src/workflows/activities.ts +15 -0
  96. package/src/workflows/automations.ts +21 -2
  97. package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
  98. package/dist/activities/recording.d.ts +0 -90
  99. package/dist/activities-control-VZIIV7RX.js.map +0 -1
  100. package/dist/activities-turn-5RGQDBFV.js.map +0 -1
  101. package/dist/chunk-NAVSAZ56.js.map +0 -1
  102. package/dist/chunk-O4HDX5SC.js.map +0 -1
  103. package/src/activities/recording.ts +0 -279
@@ -3,8 +3,8 @@ import {
3
3
  fetchCodexUsageForAccount,
4
4
  type CodexAccountStatus,
5
5
  } from "@opengeni/db";
6
- import { type RegistryProviderKind, type Settings } from "@opengeni/config";
7
- import { CODEX_USAGE_EXHAUSTED_PCT } from "../codex-rotation";
6
+ import { type ResolvedModelProvider, type Settings } from "@opengeni/config";
7
+ import { codexAccountNeedsLiveCapacityRefresh } from "../codex-rotation";
8
8
  import {
9
9
  refreshCodexUsageAndRepairCapacityWaiters,
10
10
  signalPendingCodexCapacityWakeTargets,
@@ -19,7 +19,7 @@ export function codexWorkspaceMetricKey(workspaceId: string): string {
19
19
  /** Stable public request identity across partial resumes and activity retries. */
20
20
  export function acceptsPromptCacheKeyForTurn(
21
21
  resolvedModel: {
22
- provider: { kind: RegistryProviderKind; builtin?: boolean };
22
+ provider: { kind: ResolvedModelProvider["kind"]; builtin?: boolean };
23
23
  } | null,
24
24
  ): boolean {
25
25
  if (!resolvedModel) {
@@ -37,9 +37,10 @@ export function acceptsPromptCacheKeyForTurn(
37
37
  * would idle-loop forever. Before idling, refresh LIVE usage for every connected account
38
38
  * the cache marks exhausted (bounded to the account count), which re-writes the
39
39
  * cache columns, then return the re-read rows so the ranker can pick up a genuinely-reset
40
- * window THIS turn. A refresh/read failure is swallowed (fall back to the pre-refresh rows
41
- * + the bounded idle). Cooling (429'd) accounts are NOT refreshed: their exhaustedUntil
42
- * cooldown is authoritative, and refreshing them would burn a provider call for nothing.
40
+ * window THIS turn. Typed quota cooldowns are also refreshed because the provider may
41
+ * replace an allowance cycle before its original reset; revision fencing makes that
42
+ * repair safe. Generic backpressure and legacy-unknown cooldowns are never reconciled.
43
+ * A refresh/read failure is swallowed (fall back to the pre-refresh rows + bounded idle).
43
44
  */
44
45
  export async function refreshCappedCodexUsageRows(
45
46
  db: ActivityServices["db"],
@@ -48,7 +49,12 @@ export async function refreshCappedCodexUsageRows(
48
49
  accounts: Array<
49
50
  Pick<
50
51
  CodexAccountStatus,
51
- "id" | "status" | "primaryUsedPercent" | "secondaryUsedPercent" | "exhaustedUntil"
52
+ | "id"
53
+ | "status"
54
+ | "primaryUsedPercent"
55
+ | "secondaryUsedPercent"
56
+ | "exhaustedUntil"
57
+ | "exhaustedKind"
52
58
  >
53
59
  >,
54
60
  capacitySignals: {
@@ -59,15 +65,18 @@ export async function refreshCappedCodexUsageRows(
59
65
  Array<
60
66
  Pick<
61
67
  CodexAccountStatus,
62
- "id" | "status" | "primaryUsedPercent" | "secondaryUsedPercent" | "exhaustedUntil"
68
+ | "id"
69
+ | "status"
70
+ | "primaryUsedPercent"
71
+ | "secondaryUsedPercent"
72
+ | "exhaustedUntil"
73
+ | "exhaustedKind"
63
74
  >
64
75
  >
65
76
  > {
77
+ const now = new Date();
66
78
  const stale = accounts.filter(
67
- (a) =>
68
- a.status === "active" &&
69
- ((a.primaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT ||
70
- (a.secondaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT),
79
+ (account) => account.status === "active" && codexAccountNeedsLiveCapacityRefresh(account, now),
71
80
  );
72
81
  if (stale.length === 0) {
73
82
  return accounts;
@@ -1,4 +1,4 @@
1
- import { isSessionCompactionRequested } from "@opengeni/db";
1
+ import { hasPendingSteerAfterContextCompaction, isSessionCompactionRequested } from "@opengeni/db";
2
2
  import { publishDurableSessionEvents } from "@opengeni/events";
3
3
  import {
4
4
  appendSessionInstructions,
@@ -102,6 +102,7 @@ export type CompactionPrepOk = {
102
102
  compactionModeOptions: NonNullable<Parameters<typeof maybeCompactContext>[5]>;
103
103
  compactionOnlyTurn: boolean;
104
104
  compactionSummarizerFor: (systemInstructions?: string) => CompactionSummarizer;
105
+ settleDeferredSteerAfterCompaction: () => Promise<RunAgentTurnResult | null>;
105
106
  };
106
107
 
107
108
  export type CompactionPrepOutcome = { exit: RunAgentTurnResult } | { ok: CompactionPrepOk };
@@ -114,6 +115,7 @@ export type PostAgentCompactionDeps = CompactionPrepDeps & {
114
115
  compactionModeOptions: CompactionPrepOk["compactionModeOptions"];
115
116
  compactionOnlyTurn: boolean;
116
117
  compactionSummarizerFor: CompactionPrepOk["compactionSummarizerFor"];
118
+ settleDeferredSteerAfterCompaction: CompactionPrepOk["settleDeferredSteerAfterCompaction"];
117
119
  agent: ReturnType<ActivityServices["runtime"]["buildAgent"]>;
118
120
  };
119
121
 
@@ -186,6 +188,8 @@ export async function prepareCompaction(deps: CompactionPrepDeps): Promise<Compa
186
188
  providerApi: resolvedModel?.provider.api ?? "responses",
187
189
  model: resolvedModel?.configured.id ?? turn.model,
188
190
  externallyBilled: billingState.isExternallyBilledTurn,
191
+ chargesOpenGeniCredits: billingState.chargesOpenGeniCredits,
192
+ countsTowardTokenCap: billingState.countsTowardTokenCap,
189
193
  turnAttemptId: input.attemptId,
190
194
  servingCredentialId: providerTurn.effectiveCodexCredentialId,
191
195
  priorSessionCredentialId: providerTurn.priorSessionCodexCredentialId,
@@ -298,6 +302,39 @@ export async function prepareCompaction(deps: CompactionPrepDeps): Promise<Compa
298
302
  const compactionOnlyTurn = turn.source === "compaction";
299
303
  const remoteV2CompactionNeedsAgentPrefix =
300
304
  Boolean(remoteCompactionRequester) && session.codexCompactionMode === "remote_v2";
305
+ const settleDeferredSteerAfterCompaction = async (): Promise<RunAgentTurnResult | null> => {
306
+ if (compactionOnlyTurn) return null;
307
+ const pending = await hasPendingSteerAfterContextCompaction(db, {
308
+ accountId: input.accountId,
309
+ workspaceId: input.workspaceId,
310
+ sessionId: input.sessionId,
311
+ turnId: turn.id,
312
+ executionGeneration: attempt.executionGeneration,
313
+ attemptId: input.attemptId,
314
+ });
315
+ if (!pending) return null;
316
+ if (
317
+ !(await eventing.settle({
318
+ events: [
319
+ {
320
+ type: "turn.superseded",
321
+ payload: { reason: "steer", deferredUntilCompaction: true },
322
+ },
323
+ { type: "session.status.changed", payload: { status: "queued" } },
324
+ ],
325
+ turnStatus: "superseded",
326
+ sessionStatus: "queued",
327
+ activeTurnId: null,
328
+ }))
329
+ ) {
330
+ return claimedResult({ status: "cancelled" });
331
+ }
332
+ // Metrics currently group user-directed supersession with cancellation;
333
+ // the durable turn outcome remains the precise `superseded` truth.
334
+ control.turnMetricOutcome = "cancelled";
335
+ control.activityStatus = "idle";
336
+ return claimedResult({ status: "idle" });
337
+ };
301
338
  if (compactionOnlyTurn && !remoteV2CompactionNeedsAgentPrefix) {
302
339
  const compactionInstructions = appendWorkspaceMemory(
303
340
  appendSessionInstructions(
@@ -443,6 +480,7 @@ export async function prepareCompaction(deps: CompactionPrepDeps): Promise<Compa
443
480
  compactionModeOptions,
444
481
  compactionOnlyTurn,
445
482
  compactionSummarizerFor,
483
+ settleDeferredSteerAfterCompaction,
446
484
  },
447
485
  };
448
486
  }
@@ -468,6 +506,7 @@ export async function runPostAgentCompaction(
468
506
  compactionModeOptions,
469
507
  compactionOnlyTurn,
470
508
  compactionSummarizerFor,
509
+ settleDeferredSteerAfterCompaction,
471
510
  compactionModelHistoryProjector,
472
511
  media,
473
512
  agent,
@@ -661,6 +700,16 @@ export async function runPostAgentCompaction(
661
700
  }
662
701
  await publishCompactionOutcomeEvents(outcome.events);
663
702
  }
703
+ if (
704
+ outcome.events.some(
705
+ (event) =>
706
+ event.type === "session.context.compacted" ||
707
+ event.type === "session.context.compaction.skipped",
708
+ )
709
+ ) {
710
+ const deferredSteer = await settleDeferredSteerAfterCompaction();
711
+ if (deferredSteer) return { exit: deferredSteer };
712
+ }
664
713
  } catch (compactError) {
665
714
  if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
666
715
  if (compactError instanceof TurnAttemptFencedError) throw compactError;
@@ -680,6 +729,8 @@ export async function runPostAgentCompaction(
680
729
  },
681
730
  );
682
731
  if (!isCompactionSummaryFailure(compactError)) throw compactError;
732
+ const deferredSteer = await settleDeferredSteerAfterCompaction();
733
+ if (deferredSteer) return { exit: deferredSteer };
683
734
  const errorMessage = String(compactionFailureReasonFromError(compactError));
684
735
  observability.error("context compaction failed", {
685
736
  sessionId: input.sessionId,
@@ -3,6 +3,7 @@ import {
3
3
  ApprovalRunStateLimitExceededError,
4
4
  isRetryableDatabaseTransportFailure,
5
5
  isSessionEventPersistenceError,
6
+ SandboxLeaseTransitionError,
6
7
  } from "@opengeni/db";
7
8
  import {
8
9
  ActiveBackendUnresolvableError,
@@ -10,6 +11,7 @@ import {
10
11
  EmptyCompactionSummaryError,
11
12
  isMcpRequestTimeoutError,
12
13
  isMcpTransportConnectivityError,
14
+ isModalTaskExecStartDnsResolutionError,
13
15
  RoutingWorkspaceRootChangedError,
14
16
  SelfhostedWorkspaceRootChangedError,
15
17
  UNKNOWN_MODEL_FINISH_REASON_CODE,
@@ -104,6 +106,7 @@ export function providerRecoveryResult(input: {
104
106
  ? (providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS)
105
107
  : input.failureCode === "provider_unavailable" ||
106
108
  input.failureCode === "upstream_connectivity_unavailable" ||
109
+ input.failureCode === "sandbox_command_start_unavailable" ||
107
110
  input.failureCode === "mcp_transport_timeout" ||
108
111
  input.failureCode === "mcp_transport_unavailable" ||
109
112
  input.failureCode === POST_COMPACTION_CONTINUATION_EMPTY_CODE
@@ -151,6 +154,13 @@ export function providerRecoveryCountFromMetadata(metadata: Record<string, unkno
151
154
  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
152
155
  }
153
156
 
157
+ export function providerRecoveryCountAfterModelRequestPhase(
158
+ currentCount: number,
159
+ phase: string,
160
+ ): number {
161
+ return phase === "completed" ? 0 : currentCount;
162
+ }
163
+
154
164
  export function headerValue(headers: unknown, name: string): string | null {
155
165
  if (!headers || typeof headers !== "object") return null;
156
166
  const getter = (headers as { get?: unknown }).get;
@@ -367,6 +377,66 @@ export function isWorkerShutdownCancellation(error: unknown): boolean {
367
377
  return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
368
378
  }
369
379
 
380
+ export type SandboxLifecycleTransitionDiagnostic = {
381
+ sandboxGroupId: string;
382
+ leaseEpoch: number;
383
+ reason: "capture_in_progress" | "rotation_in_progress" | "provider_recovery_in_progress";
384
+ };
385
+
386
+ /**
387
+ * Recover a typed sandbox lifecycle transition through the structural wrappers
388
+ * used by parallel Agents SDK function-tool execution. Never infer transition
389
+ * truth from message text: only the original class or an exact, fully-shaped
390
+ * cross-package error object is accepted.
391
+ */
392
+ export function sandboxLifecycleTransitionDiagnostic(
393
+ error: unknown,
394
+ ): SandboxLifecycleTransitionDiagnostic | null {
395
+ const pending: unknown[] = [error];
396
+ const seen = new WeakSet<object>();
397
+ let inspected = 0;
398
+
399
+ while (pending.length > 0 && inspected < 64) {
400
+ const current = pending.shift();
401
+ inspected += 1;
402
+ if (!current || typeof current !== "object" || seen.has(current)) continue;
403
+ seen.add(current);
404
+
405
+ try {
406
+ const record = current as Record<string, unknown>;
407
+ const reason = record.reason;
408
+ if (
409
+ (current instanceof SandboxLeaseTransitionError ||
410
+ record.name === "SandboxLeaseTransitionError") &&
411
+ typeof record.sandboxGroupId === "string" &&
412
+ record.sandboxGroupId.length > 0 &&
413
+ typeof record.leaseEpoch === "number" &&
414
+ Number.isSafeInteger(record.leaseEpoch) &&
415
+ record.leaseEpoch >= 0 &&
416
+ (reason === "capture_in_progress" ||
417
+ reason === "rotation_in_progress" ||
418
+ reason === "provider_recovery_in_progress")
419
+ ) {
420
+ return {
421
+ sandboxGroupId: record.sandboxGroupId,
422
+ leaseEpoch: record.leaseEpoch,
423
+ reason,
424
+ };
425
+ }
426
+
427
+ for (const key of ["cause", "error"] as const) {
428
+ const nested = record[key];
429
+ if (nested && typeof nested === "object") pending.push(nested);
430
+ }
431
+ if (Array.isArray(record.errors)) pending.push(...record.errors.slice(0, 32));
432
+ } catch {
433
+ // A hostile proxy or getter is not durable lifecycle evidence.
434
+ }
435
+ }
436
+
437
+ return null;
438
+ }
439
+
370
440
  /**
371
441
  * Recognize the one active-route transition that cannot finish inside its
372
442
  * originating attempt. A Modal-home session may start on a Connected Machine
@@ -812,6 +882,14 @@ export function agentRunFailurePayload(
812
882
  retryable: true,
813
883
  };
814
884
  }
885
+ if (isModalTaskExecStartDnsResolutionError(error)) {
886
+ return {
887
+ error:
888
+ "The managed sandbox command transport was temporarily unreachable before the command started. The same turn will retry after a short delay.",
889
+ code: "sandbox_command_start_unavailable",
890
+ retryable: true,
891
+ };
892
+ }
815
893
  // An accepted Codex stream with no terminal response is malformed/partial,
816
894
  // not provider backpressure. Replaying the same accepted turn could repeat
817
895
  // model or tool effects, so this marked transport failure must outrank the
@@ -18,7 +18,6 @@ import {
18
18
  settleCodexCredentialFailover,
19
19
  readLease,
20
20
  SandboxLeaseSupersededError,
21
- SandboxLeaseTransitionError,
22
21
  isSessionEventPersistenceError,
23
22
  } from "@opengeni/db";
24
23
  import { publishDurableSessionEvents } from "@opengeni/events";
@@ -61,6 +60,7 @@ import {
61
60
  escapedMcpTimeoutRecoveryFailure,
62
61
  preClaimAdmissionFailure,
63
62
  isWorkerShutdownCancellation,
63
+ sandboxLifecycleTransitionDiagnostic,
64
64
  sandboxRouteTransitionCode,
65
65
  safeErrorDiagnostic,
66
66
  classifyXaiCredentialFailure,
@@ -159,20 +159,20 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
159
159
  // recoverable control-plane states, never session failures. The latter is
160
160
  // paced briefly; the next acquire waits on the durable claim and resumes
161
161
  // immediately when it clears.
162
- if (
163
- (error instanceof SandboxLeaseSupersededError ||
164
- error instanceof SandboxLeaseTransitionError) &&
165
- recoveryTurnId
166
- ) {
162
+ const lifecycleTransition = sandboxLifecycleTransitionDiagnostic(error);
163
+ const leaseControlError =
164
+ error instanceof SandboxLeaseSupersededError ? error : lifecycleTransition;
165
+ if (leaseControlError && recoveryTurnId) {
167
166
  try {
168
- const fencedLease = await readLease(db, input.workspaceId, error.sandboxGroupId).catch(
169
- () => null,
170
- );
171
- const lifecycleTransition = error instanceof SandboxLeaseTransitionError;
167
+ const fencedLease = await readLease(
168
+ db,
169
+ input.workspaceId,
170
+ leaseControlError.sandboxGroupId,
171
+ ).catch(() => null);
172
172
  const rotationPending =
173
173
  fencedLease?.rotationRequestedAt != null ||
174
- (lifecycleTransition && error.reason === "rotation_in_progress");
175
- const transitionPending = lifecycleTransition || rotationPending;
174
+ lifecycleTransition?.reason === "rotation_in_progress";
175
+ const transitionPending = lifecycleTransition !== null || rotationPending;
176
176
  const deadlineRotationPending =
177
177
  rotationPending && fencedLease?.rotationReason === "provider_deadline";
178
178
  const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
@@ -182,20 +182,20 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
182
182
  attemptId: input.attemptId,
183
183
  reason: deadlineRotationPending
184
184
  ? "sandbox_deadline_rotation"
185
- : lifecycleTransition
185
+ : lifecycleTransition !== null
186
186
  ? "sandbox_lifecycle_transition"
187
187
  : "sandbox_lease_superseded",
188
188
  ...(transitionPending
189
189
  ? {
190
190
  detail: {
191
- sandboxGroupId: error.sandboxGroupId,
192
- leaseEpoch: error.leaseEpoch,
191
+ sandboxGroupId: leaseControlError.sandboxGroupId,
192
+ leaseEpoch: leaseControlError.leaseEpoch,
193
193
  ...(rotationPending
194
194
  ? {
195
195
  rotationReason: fencedLease?.rotationReason ?? "operator",
196
196
  }
197
197
  : {}),
198
- ...(lifecycleTransition ? { transitionReason: error.reason } : {}),
198
+ ...(lifecycleTransition ? { transitionReason: lifecycleTransition.reason } : {}),
199
199
  },
200
200
  }
201
201
  : {}),
@@ -647,7 +647,11 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
647
647
  status: "error",
648
648
  lastError: "model request was forbidden for this credential",
649
649
  }
650
- : { kind: "cooldown", until: cooldownUntil! },
650
+ : {
651
+ kind: "cooldown",
652
+ until: cooldownUntil!,
653
+ cooldownKind: codexCredentialFailure.kind,
654
+ },
651
655
  })
652
656
  : false;
653
657
  if (!statePersisted && leases.codex.holderId && leases.codex.generation !== null) {
@@ -903,6 +907,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
903
907
  let rotationResumeIdleUntilReset = false; // circuit-breaker fall (Finding 1b) ⇒ MANDATORY hold
904
908
  let allCappedResetAt: Date | null = null; // set ⇒ every account capped; idle until this
905
909
  let capacityAuthoritativeResetAt: Date | null = null;
910
+ let capacityNeedsBoundedRefresh = false;
906
911
  if (providerTurn.effectiveCodexCredentialId) {
907
912
  const [rotation, sessionCodex] = await Promise.all([
908
913
  getCodexRotationSettings(db, input.workspaceId).catch(() => null),
@@ -941,6 +946,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
941
946
  input.workspaceId,
942
947
  providerTurn.effectiveCodexCredentialId,
943
948
  until,
949
+ "quota",
944
950
  ).catch(() => null);
945
951
  const cooldownPersisted = cooldownMutation?.result ?? false;
946
952
  if (cooldownMutation) {
@@ -953,7 +959,9 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
953
959
  // write, so stamp the just-cooled account so the engine excludes it now. The
954
960
  // serving account is thus walked AT MOST ONCE per turn (invariant 4: bounded).
955
961
  const fresh = accounts.map((a) =>
956
- a.id === providerTurn.effectiveCodexCredentialId ? { ...a, exhaustedUntil: until } : a,
962
+ a.id === providerTurn.effectiveCodexCredentialId
963
+ ? { ...a, exhaustedUntil: until, exhaustedKind: "quota" as const }
964
+ : a,
957
965
  );
958
966
  if (reactiveSharded) {
959
967
  // AM-5: RE-SHARD over the healthy survivors (the just-capped serving account is
@@ -1017,6 +1025,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
1017
1025
  rotated = true;
1018
1026
  allCappedResetAt = earliestCodexReset(fresh, new Date());
1019
1027
  capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
1028
+ capacityNeedsBoundedRefresh = true;
1020
1029
  }
1021
1030
  } else {
1022
1031
  const decision = chooseRotationActive({
@@ -1048,6 +1057,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
1048
1057
  rotated = true;
1049
1058
  allCappedResetAt = decision.earliestResetAt;
1050
1059
  capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
1060
+ capacityNeedsBoundedRefresh = true;
1051
1061
  }
1052
1062
  // kind:"none" → fall through to today's single-account idle.
1053
1063
  }
@@ -1088,7 +1098,8 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
1088
1098
  goalId: goalActive && goal ? goal.id : null,
1089
1099
  goalVersion: goalActive && goal ? goal.version : null,
1090
1100
  earliestResetAt: providerResetAt,
1091
- resetKind: providerResetAt ? "authoritative" : "bounded_refresh",
1101
+ resetKind:
1102
+ providerResetAt && !capacityNeedsBoundedRefresh ? "authoritative" : "bounded_refresh",
1092
1103
  failurePayload,
1093
1104
  ...(leases.codex.holderId && leases.codex.generation !== null
1094
1105
  ? {
@@ -23,7 +23,11 @@ import type {
23
23
  import { captureWorkspaceRevision, openFreshWorkspaceCaptureSession } from "../workspace-capture";
24
24
  import { type ChannelASession } from "@opengeni/runtime/sandbox";
25
25
  import { createModelCheckpointMemoryCollector } from "../../model-checkpoint-memory-collector";
26
- import { makeMachineOpObserver, turnLifecycleMetricsFor } from "../../observability-metrics";
26
+ import {
27
+ makeMachineOpObserver,
28
+ recordSessionEventAppendPhase,
29
+ turnLifecycleMetricsFor,
30
+ } from "../../observability-metrics";
27
31
  import {
28
32
  maybePersistWarmWorkspaceSnapshot,
29
33
  waitForWarmSnapshot,
@@ -34,6 +38,7 @@ import { safeErrorDiagnostic, safeErrorForTelemetry } from "./errors";
34
38
  import {
35
39
  assertPhysicalToolQuiescenceForCancellation,
36
40
  assertSessionAttemptQuiescenceRecoveryDurable,
41
+ armTurnQuiescenceWatchdog,
37
42
  clearAttemptCredentialsWithSettledFence,
38
43
  drainAttemptOwnedSandboxWriters,
39
44
  persistOrSignalSessionAttemptQuiescence,
@@ -46,7 +51,6 @@ import type {
46
51
  AttemptIdentityState,
47
52
  EventingState,
48
53
  ProviderTurnState,
49
- RecordingState,
50
54
  RenewalState,
51
55
  SandboxRuntimeState,
52
56
  TurnControlState,
@@ -76,7 +80,6 @@ export type TurnFinalizationDeps = {
76
80
  attempt: AttemptIdentityState;
77
81
  sandboxState: SandboxRuntimeState;
78
82
  renewals: RenewalState;
79
- recordingState: RecordingState;
80
83
  eventing: EventingState;
81
84
  providerTurn: ProviderTurnState;
82
85
  workspaceRefs: WorkspaceRefState;
@@ -87,7 +90,6 @@ export type TurnFinalizationDeps = {
87
90
  ) => Promise<T>;
88
91
  requireResolvedSandboxForMutation: (message: string) => ResumedTurnSandbox;
89
92
  stopLeaseHeartbeat: () => void;
90
- abandonActiveRecording: (reason: string, disposition?: "failed" | "discard") => Promise<void>;
91
93
  turnCompletionMemoryCollector: ReturnType<typeof createModelCheckpointMemoryCollector>;
92
94
  };
93
95
 
@@ -115,14 +117,12 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
115
117
  attempt,
116
118
  sandboxState,
117
119
  renewals,
118
- recordingState,
119
120
  eventing,
120
121
  providerTurn,
121
122
  workspaceRefs,
122
123
  runWorkspaceMutationForSandbox,
123
124
  requireResolvedSandboxForMutation,
124
125
  stopLeaseHeartbeat,
125
- abandonActiveRecording,
126
126
  turnCompletionMemoryCollector,
127
127
  } = deps;
128
128
  // This is the logical ownership boundary. Abort before any fallible
@@ -137,6 +137,17 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
137
137
  let finalizationError: unknown;
138
138
  let physicalToolQuiescenceConfirmed = !control.acknowledgeQuiescence;
139
139
  let quiescenceReceiptOrProofDurable = !control.acknowledgeQuiescence;
140
+ const disarmQuiescenceWatchdog = armTurnQuiescenceWatchdog({
141
+ enabled: control.acknowledgeQuiescence,
142
+ onTimeout: () => {
143
+ observability.error("turn quiescence drain exceeded hard containment deadline", {
144
+ "opengeni.session_id": input.sessionId,
145
+ "opengeni.turn_id": attempt.turnId ?? "",
146
+ "opengeni.attempt_id": input.attemptId,
147
+ });
148
+ },
149
+ terminateWorker: () => process.exit(1),
150
+ });
140
151
  const finalizerSignal = turnFinalizerCancellationSignal(
141
152
  cancellationSignal,
142
153
  control.activityStatus,
@@ -303,6 +314,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
303
314
  },
304
315
  });
305
316
  quiescenceReceiptOrProofDurable = true;
317
+ disarmQuiescenceWatchdog();
306
318
  if (recoveryMode === "signal") {
307
319
  observability.info("agent turn quiescence proof handed to workflow recovery", {
308
320
  "opengeni.session_id": input.sessionId,
@@ -332,6 +344,10 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
332
344
  ...event,
333
345
  turnId: attempt.turnId ?? null,
334
346
  })),
347
+ {
348
+ onAppendPhase: (observation) =>
349
+ recordSessionEventAppendPhase(observability, observation),
350
+ },
335
351
  ).catch(() => undefined),
336
352
  finalizerSignal,
337
353
  );
@@ -435,10 +451,8 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
435
451
  // Workbench v2 turn-end workspace capture — runs FIRST in
436
452
  // the turn-end finally, while the box is MAXIMALLY ALIVE. The agent's last
437
453
  // tool ran before this finally, so /workspace is already final; capture is
438
- // FS-equivalent to the already-settled recording preparation and the warm
439
- // snapshot (neither mutates workspace files). Running it here — BEFORE
440
- // preparedTools.close() (which tears down tools / computer-use / the display
441
- // stack and is what starts the Modal box exiting a few seconds later) —
454
+ // FS-equivalent to the warm snapshot (neither mutates workspace files).
455
+ // Running it here — BEFORE preparedTools.close() tears down tools —
442
456
  // gives capture the full live-box margin instead of racing the teardown
443
457
  // tail, which was dropping 100% of captures on real Modal desktop boxes
444
458
  // ("request cancelled due to container exiting", 0 rows). External module:
@@ -556,17 +570,6 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
556
570
  () => undefined,
557
571
  );
558
572
  }
559
- // A recording normally closes inside the attempt-fenced turn settlement.
560
- // Reaching finally with one still active means settlement threw, never ran,
561
- // or lost ownership. Stop ffmpeg and mark only this exact attempt-owned row
562
- // failed; publish no event and leave the artifact recoverable on the box.
563
- await waitForTurnFinalizerStep(
564
- abandonActiveRecording(
565
- "activity ended without recording settlement",
566
- recordingState.didComputerUse ? "failed" : "discard",
567
- ),
568
- finalizerSignal,
569
- );
570
573
  if (sandboxState.resolvedSandbox) {
571
574
  // TURN-END mid-session snapshot (sandbox-file-persistence): fold the
572
575
  // turn's finished /workspace onto the lease before releasing the holder,
@@ -169,7 +169,11 @@ export function pendingToolCallFromSdkEvent(event: unknown): {
169
169
 
170
170
  /** Tools whose intentional visual result must survive inline-media sanitization. */
171
171
  export function toolCallProducesRetainableSessionImage(name: string | null): boolean {
172
- return name === "computer_screenshot" || name === "view_image";
172
+ return (
173
+ name === "computer_screenshot" ||
174
+ name === "view_image" ||
175
+ name === "interaction__computer_observe"
176
+ );
173
177
  }
174
178
 
175
179
  export function completedToolCallFromSdkEvent(event: unknown): {
@@ -0,0 +1,101 @@
1
+ import type { AttemptToolDefinition } from "@opengeni/codemode";
2
+ import {
3
+ compareModelPickerOrder,
4
+ modelPickerBillingClassFor,
5
+ } from "@opengeni/contracts/model-picker-order";
6
+ import type { WorkspaceModelSelection } from "@opengeni/core";
7
+
8
+ export const LIST_MODELS_TOOL_NAME = "list_models";
9
+
10
+ export const LIST_MODELS_TOOL_DESCRIPTION =
11
+ "List the models this workspace can select right now, in picker order, with deployment-defined cost and optional guidance. This does not switch the current session model; use an ID with session_create or the human model picker.";
12
+
13
+ function safeLineField(value: string): string {
14
+ return value
15
+ .replace(/[\r\n|]/gu, " ")
16
+ .replace(/\s+/gu, " ")
17
+ .trim();
18
+ }
19
+
20
+ export function renderListModels(input: {
21
+ currentModelId: string | null;
22
+ selections: readonly WorkspaceModelSelection[];
23
+ modelNotes: Readonly<Record<string, string>>;
24
+ }): string {
25
+ const lines: string[] = [];
26
+ if (input.currentModelId) {
27
+ lines.push(`Current: ${safeLineField(input.currentModelId)}`);
28
+ }
29
+
30
+ const selectable = input.selections
31
+ .filter((entry) => entry.availability.selectable)
32
+ .map((selection) => ({
33
+ selection,
34
+ billingClass: modelPickerBillingClassFor(selection.model),
35
+ selectable: selection.availability.selectable,
36
+ label: selection.model.label,
37
+ }))
38
+ .sort(compareModelPickerOrder);
39
+ if (selectable.length === 0) {
40
+ lines.push("No models are available in this workspace.");
41
+ return lines.join("\n");
42
+ }
43
+
44
+ for (const { selection } of selectable) {
45
+ const { model } = selection;
46
+ const fields = [safeLineField(model.id), safeLineField(model.label), model.cost];
47
+ const note = input.modelNotes[model.id];
48
+ if (note) fields.push(safeLineField(note));
49
+ lines.push(`- ${fields.join(" | ")}`);
50
+ }
51
+ return lines.join("\n");
52
+ }
53
+
54
+ export function createListModelsAttemptToolDefinition(input: {
55
+ currentModelId: string | null;
56
+ load: () => Promise<{
57
+ selections: readonly WorkspaceModelSelection[];
58
+ modelNotes: Readonly<Record<string, string>>;
59
+ }>;
60
+ }): AttemptToolDefinition {
61
+ return {
62
+ identity: { serverId: "opengeni", toolName: LIST_MODELS_TOOL_NAME },
63
+ modelName: LIST_MODELS_TOOL_NAME,
64
+ codemodePath: ["opengeni", LIST_MODELS_TOOL_NAME],
65
+ title: "List selectable models",
66
+ description: LIST_MODELS_TOOL_DESCRIPTION,
67
+ inputSchema: {
68
+ type: "object",
69
+ properties: {},
70
+ additionalProperties: false,
71
+ },
72
+ annotations: {
73
+ title: "List selectable models",
74
+ readOnlyHint: true,
75
+ destructiveHint: false,
76
+ idempotentHint: true,
77
+ openWorldHint: false,
78
+ },
79
+ source: "opengeni",
80
+ approval: "none",
81
+ execute: async (args) => {
82
+ if (Object.keys(args).length !== 0) {
83
+ throw new Error("list_models accepts no arguments");
84
+ }
85
+ const catalog = await input.load();
86
+ return {
87
+ isError: false,
88
+ content: [
89
+ {
90
+ type: "text",
91
+ text: renderListModels({
92
+ currentModelId: input.currentModelId,
93
+ selections: catalog.selections,
94
+ modelNotes: catalog.modelNotes,
95
+ }),
96
+ },
97
+ ],
98
+ };
99
+ },
100
+ };
101
+ }