@opengeni/worker-bundle 0.22.8 → 0.23.4-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +10 -2
  2. package/dist/activities/agent-turn/agent-build.d.ts +1 -4
  3. package/dist/activities/agent-turn/claim.d.ts +3 -4
  4. package/dist/activities/agent-turn/codex.d.ts +8 -7
  5. package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
  6. package/dist/activities/agent-turn/errors.d.ts +13 -0
  7. package/dist/activities/agent-turn/finalization.d.ts +1 -3
  8. package/dist/activities/agent-turn/list-models.d.ts +16 -0
  9. package/dist/activities/agent-turn/model-usage.d.ts +10 -1
  10. package/dist/activities/agent-turn/quiescence.d.ts +39 -0
  11. package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
  12. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
  13. package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
  14. package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
  15. package/dist/activities/agent-turn/tool-environment.d.ts +1 -0
  16. package/dist/activities/agent-turn/tool-policy.d.ts +7 -5
  17. package/dist/activities/agent-turn/turn-context.d.ts +3 -8
  18. package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
  19. package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
  20. package/dist/activities/automations.d.ts +1 -0
  21. package/dist/activities/capabilities.d.ts +2 -1
  22. package/dist/activities/codex-rotation.d.ts +2 -0
  23. package/dist/activities/goals.d.ts +11 -0
  24. package/dist/activities/image-generation-references.d.ts +2 -1
  25. package/dist/activities/scheduled-tasks.d.ts +1 -0
  26. package/dist/activities/session-state.d.ts +2 -1
  27. package/dist/activities/types.d.ts +8 -0
  28. package/dist/{activities-control-VZIIV7RX.js → activities-control-SGU2WEVB.js} +328 -107
  29. package/dist/activities-control-SGU2WEVB.js.map +1 -0
  30. package/dist/activities-control.d.ts +2 -0
  31. package/dist/{activities-turn-5RGQDBFV.js → activities-turn-7EAQXW6Q.js} +828 -540
  32. package/dist/activities-turn-7EAQXW6Q.js.map +1 -0
  33. package/dist/activities.d.ts +2 -0
  34. package/dist/{chunk-O4HDX5SC.js → chunk-S2QVRVJW.js} +227 -36
  35. package/dist/chunk-S2QVRVJW.js.map +1 -0
  36. package/dist/{chunk-NAVSAZ56.js → chunk-TJIPCHMP.js} +162 -2
  37. package/dist/chunk-TJIPCHMP.js.map +1 -0
  38. package/dist/index.d.ts +1 -0
  39. package/dist/index.js +69 -6
  40. package/dist/index.js.map +1 -1
  41. package/dist/observability-metrics.d.ts +29 -0
  42. package/dist/personal-github-git-credentials.d.ts +1 -0
  43. package/dist/unhandled-rejection-boundary.d.ts +29 -0
  44. package/dist/workflow-bundle.js +26 -3
  45. package/dist/workflows/activities.d.ts +5 -0
  46. package/dist/workflows/automations.d.ts +5 -0
  47. package/package.json +18 -18
  48. package/src/activities/agent-run-admission.ts +7 -1
  49. package/src/activities/agent-turn/admission.ts +28 -6
  50. package/src/activities/agent-turn/agent-build.ts +15 -31
  51. package/src/activities/agent-turn/claim.ts +53 -72
  52. package/src/activities/agent-turn/codex.ts +21 -12
  53. package/src/activities/agent-turn/compaction-prep.ts +52 -1
  54. package/src/activities/agent-turn/errors.ts +78 -0
  55. package/src/activities/agent-turn/failure-settlement.ts +30 -19
  56. package/src/activities/agent-turn/finalization.ts +24 -21
  57. package/src/activities/agent-turn/history.ts +5 -1
  58. package/src/activities/agent-turn/list-models.ts +101 -0
  59. package/src/activities/agent-turn/model-usage.ts +88 -42
  60. package/src/activities/agent-turn/quiescence.ts +86 -0
  61. package/src/activities/agent-turn/run-credentials.ts +12 -1
  62. package/src/activities/agent-turn/run.ts +20 -14
  63. package/src/activities/agent-turn/sandbox-establish.ts +46 -11
  64. package/src/activities/agent-turn/sandbox-route.ts +21 -25
  65. package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
  66. package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
  67. package/src/activities/agent-turn/stream-attempt.ts +106 -14
  68. package/src/activities/agent-turn/tool-environment.ts +72 -14
  69. package/src/activities/agent-turn/tool-policy.ts +7 -6
  70. package/src/activities/agent-turn/turn-context.ts +12 -14
  71. package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
  72. package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
  73. package/src/activities/automations.ts +122 -67
  74. package/src/activities/capabilities.ts +72 -2
  75. package/src/activities/codex-capacity.ts +13 -5
  76. package/src/activities/codex-rotation.ts +17 -0
  77. package/src/activities/goals.ts +117 -35
  78. package/src/activities/image-generation-references.ts +9 -9
  79. package/src/activities/retained-screenshots.ts +104 -21
  80. package/src/activities/sandbox-lease.ts +7 -4
  81. package/src/activities/scheduled-tasks.ts +248 -30
  82. package/src/activities/session-state.ts +8 -1
  83. package/src/activities/site-auth-maintenance.ts +12 -12
  84. package/src/activities/types.ts +6 -0
  85. package/src/activities.ts +1 -0
  86. package/src/activity-services.ts +4 -1
  87. package/src/github-rest-mcp.ts +8 -4
  88. package/src/http.ts +3 -0
  89. package/src/index.ts +38 -4
  90. package/src/observability-metrics.ts +202 -0
  91. package/src/personal-github-git-credentials.ts +45 -4
  92. package/src/sandbox-resume.ts +1 -1
  93. package/src/sandbox-routing.ts +7 -4
  94. package/src/unhandled-rejection-boundary.ts +73 -0
  95. package/src/workflows/activities.ts +15 -0
  96. package/src/workflows/automations.ts +21 -2
  97. package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
  98. package/dist/activities/recording.d.ts +0 -90
  99. package/dist/activities-control-VZIIV7RX.js.map +0 -1
  100. package/dist/activities-turn-5RGQDBFV.js.map +0 -1
  101. package/dist/chunk-NAVSAZ56.js.map +0 -1
  102. package/dist/chunk-O4HDX5SC.js.map +0 -1
  103. package/src/activities/recording.ts +0 -279
@@ -15,11 +15,13 @@ import {
15
15
  } from "@opengeni/runtime";
16
16
  import {
17
17
  calculateGatewayReportedCostBreakdown,
18
+ calculateGatewayReportedProviderCostMicros,
18
19
  calculateModelUsageCostBreakdown,
19
20
  configuredModelPricingSchedules,
21
+ resolveModelProvider,
20
22
  responseSatisfiesLatencyMode,
21
- OPENGENI_GATEWAY_MODELS,
22
23
  OPENGENI_GATEWAY_PROVIDER_ID,
24
+ WORKSPACE_GATEWAY_PROVIDER_ID,
23
25
  type ModelUsageInput,
24
26
  type ModelProviderApi,
25
27
  type Settings,
@@ -191,6 +193,8 @@ export async function processModelResponseTerminalEvent(input: {
191
193
  latencyMode?: LatencyMode;
192
194
  metricProvider: string;
193
195
  externallyBilled: boolean;
196
+ chargesOpenGeniCredits?: boolean;
197
+ countsTowardTokenCap?: boolean;
194
198
  servingCredentialId: string | null;
195
199
  priorSessionCredentialId: string | null;
196
200
  emittedSourceKeys: Set<string>;
@@ -251,9 +255,14 @@ export async function processModelResponseTerminalEvent(input: {
251
255
  turnAttemptId: input.turnAttemptId,
252
256
  model: input.model,
253
257
  externallyBilled: input.externallyBilled,
258
+ ...(input.chargesOpenGeniCredits !== undefined
259
+ ? { chargesOpenGeniCredits: input.chargesOpenGeniCredits }
260
+ : {}),
261
+ ...(input.countsTowardTokenCap !== undefined
262
+ ? { countsTowardTokenCap: input.countsTowardTokenCap }
263
+ : {}),
254
264
  usage: responseUsage.usage,
255
265
  normalizedUsage,
256
- gatewayManaged: input.provider === OPENGENI_GATEWAY_PROVIDER_ID,
257
266
  gatewayBilling: responseUsage.gatewayBilling,
258
267
  sourceKey,
259
268
  ...(input.latencyMode ? { latencyMode: input.latencyMode } : {}),
@@ -272,6 +281,8 @@ export async function processModelResponseTerminalEvent(input: {
272
281
  sourceKey,
273
282
  usage: responseUsage,
274
283
  normalizedUsage,
284
+ ...(billing ? { billingPath: billing.billingPath } : {}),
285
+ ...(billing?.upstreamProvider ? { upstreamProvider: billing.upstreamProvider } : {}),
275
286
  servingAccountHash: accountContext.servingAccountHash,
276
287
  accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
277
288
  emittedSourceKeys: input.emittedSourceKeys,
@@ -345,6 +356,8 @@ export async function processCompactionModelUsageEvent(input: {
345
356
  providerApi: ModelProviderApi;
346
357
  model: string;
347
358
  externallyBilled: boolean;
359
+ chargesOpenGeniCredits?: boolean;
360
+ countsTowardTokenCap?: boolean;
348
361
  servingCredentialId: string | null;
349
362
  priorSessionCredentialId: string | null;
350
363
  emittedSourceKeys: Set<string>;
@@ -388,9 +401,14 @@ export async function processCompactionModelUsageEvent(input: {
388
401
  turnAttemptId: input.turnAttemptId,
389
402
  model: input.model,
390
403
  externallyBilled: input.externallyBilled,
404
+ ...(input.chargesOpenGeniCredits !== undefined
405
+ ? { chargesOpenGeniCredits: input.chargesOpenGeniCredits }
406
+ : {}),
407
+ ...(input.countsTowardTokenCap !== undefined
408
+ ? { countsTowardTokenCap: input.countsTowardTokenCap }
409
+ : {}),
391
410
  usage: input.usage.usage,
392
411
  normalizedUsage,
393
- gatewayManaged: input.provider === OPENGENI_GATEWAY_PROVIDER_ID,
394
412
  gatewayBilling: input.usage.gatewayBilling,
395
413
  sourceKey,
396
414
  observability: input.observability,
@@ -408,6 +426,8 @@ export async function processCompactionModelUsageEvent(input: {
408
426
  sourceKey,
409
427
  usage: input.usage,
410
428
  normalizedUsage,
429
+ ...(billing ? { billingPath: billing.billingPath } : {}),
430
+ ...(billing?.upstreamProvider ? { upstreamProvider: billing.upstreamProvider } : {}),
411
431
  servingAccountHash: accountContext.servingAccountHash,
412
432
  accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
413
433
  emittedSourceKeys: input.emittedSourceKeys,
@@ -449,6 +469,10 @@ export async function emitModelCallUsage(input: {
449
469
  sourceKey: string;
450
470
  usage: ModelResponseUsage | { usage?: unknown | null } | null;
451
471
  normalizedUsage?: ModelCallUsageNormalization;
472
+ /** Accepted billing authority persisted for Insights repair after a soft fact-write failure. */
473
+ billingPath?: ModelUsageBillingRecord["billingPath"];
474
+ /** Validated Gateway endpoint provider persisted for exact Insights repair. */
475
+ upstreamProvider?: string;
452
476
  // Prompt-cache research dimensions (log-only; NEVER on a metric label or a
453
477
  // durable event). The opaque serving-account tag and whether it changed since
454
478
  // the session's previous call — the account-switch hypothesis for cache misses.
@@ -480,6 +504,8 @@ export async function emitModelCallUsage(input: {
480
504
  providerApi: input.providerApi,
481
505
  model: input.model,
482
506
  sourceKey: input.sourceKey,
507
+ ...(input.billingPath ? { billingPath: input.billingPath } : {}),
508
+ ...(input.upstreamProvider ? { upstreamProvider: input.upstreamProvider } : {}),
483
509
  ...telemetry,
484
510
  },
485
511
  },
@@ -572,7 +598,8 @@ export async function recordModelUsageAndDebitCredits(
572
598
  turnAttemptId: string;
573
599
  model: string;
574
600
  externallyBilled: boolean;
575
- gatewayManaged?: boolean;
601
+ chargesOpenGeniCredits?: boolean;
602
+ countsTowardTokenCap?: boolean;
576
603
  gatewayBilling?: ModelResponseUsage["gatewayBilling"];
577
604
  usage?: ModelUsageInput | ModelCallUsageInput | null;
578
605
  normalizedUsage?: ModelCallUsageNormalization;
@@ -589,19 +616,35 @@ export async function recordModelUsageAndDebitCredits(
589
616
  const inputTokens = sanitizedUsage.inputTokens ?? 0;
590
617
  const outputTokens = sanitizedUsage.outputTokens ?? 0;
591
618
  const totalTokens = sanitizedUsage.totalTokens ?? 0;
592
- const gatewayBilling = input.gatewayManaged ? input.gatewayBilling : undefined;
619
+ const chargesOpenGeniCredits = input.chargesOpenGeniCredits ?? !input.externallyBilled;
620
+ const countsTowardTokenCap = input.countsTowardTokenCap ?? !input.externallyBilled;
621
+ const resolvedGatewayModel = input.gatewayBilling
622
+ ? resolveModelProvider(settings, input.model)
623
+ : undefined;
624
+ const gatewayProviderId = resolvedGatewayModel?.provider.id;
625
+ const gatewayBilling =
626
+ gatewayProviderId === OPENGENI_GATEWAY_PROVIDER_ID ||
627
+ gatewayProviderId === WORKSPACE_GATEWAY_PROVIDER_ID
628
+ ? input.gatewayBilling
629
+ : undefined;
630
+ const allowedProviders = resolvedGatewayModel?.model.requestPolicy?.gateway.only;
631
+ const unpinnedWorkspaceGatewayModel =
632
+ gatewayProviderId === WORKSPACE_GATEWAY_PROVIDER_ID && allowedProviders === undefined;
593
633
  if (gatewayBilling) {
594
- const gatewayModel = Object.values(OPENGENI_GATEWAY_MODELS).find(
595
- (candidate) => candidate.productId === input.model,
596
- );
597
634
  if (
598
- !gatewayModel ||
599
- !(gatewayModel.providers as readonly string[]).includes(gatewayBilling.finalProvider)
635
+ !unpinnedWorkspaceGatewayModel &&
636
+ (!allowedProviders ||
637
+ !(allowedProviders as readonly string[]).includes(gatewayBilling.finalProvider))
600
638
  ) {
601
639
  throw new Error(
602
640
  `AI Gateway reported unapproved provider ${gatewayBilling.finalProvider} for ${input.model}`,
603
641
  );
604
642
  }
643
+ if (unpinnedWorkspaceGatewayModel && chargesOpenGeniCredits) {
644
+ throw new Error(
645
+ `Workspace Gateway custom model ${input.model} cannot charge OpenGeni credits without pinned pricing`,
646
+ );
647
+ }
605
648
  }
606
649
  const pricingSchedules = configuredModelPricingSchedules(settings);
607
650
  const configuredPricingModel = pricingSchedules[input.model]
@@ -610,12 +653,19 @@ export async function recordModelUsageAndDebitCredits(
610
653
  ? input.model.slice("codex/".length)
611
654
  : null;
612
655
  const pricingBreakdown = gatewayBilling
613
- ? calculateGatewayReportedCostBreakdown(
614
- settings,
615
- input.model,
616
- gatewayBilling.inferenceCostUsd,
617
- { inputTokens },
618
- )
656
+ ? unpinnedWorkspaceGatewayModel
657
+ ? {
658
+ providerCostMicros: calculateGatewayReportedProviderCostMicros(
659
+ gatewayBilling.inferenceCostUsd,
660
+ ),
661
+ creditCostMicros: 0,
662
+ }
663
+ : calculateGatewayReportedCostBreakdown(
664
+ settings,
665
+ input.model,
666
+ gatewayBilling.inferenceCostUsd,
667
+ { inputTokens },
668
+ )
619
669
  : configuredPricingModel
620
670
  ? calculateModelUsageCostBreakdown(settings, configuredPricingModel, sanitizedUsage, {
621
671
  latencyMode: input.latencyMode ?? "standard",
@@ -634,17 +684,27 @@ export async function recordModelUsageAndDebitCredits(
634
684
  : estimatedProviderCostMicros !== null
635
685
  ? ("configured_list_price" as const)
636
686
  : null;
637
- // An externally billed turn is paid outside OpenGeni, so it
638
- // consumes ZERO OpenGeni credits and must never feed an OpenGeni cap. A
639
- // codex/<slug> model has no entry in configuredModelPricing, so the normal path
640
- // below would throw "Missing model pricing". We:
641
- // - do NOT emit the cap-feeding `model.tokens` event (ensureRunAllowed and
642
- // the API tokens:consume cap sum `model.tokens` with NO cost dimension, so
643
- // any row would count against maxMonthlyTokensPerWorkspace);
644
- // - record a `model.cost = 0` audit marker (harmless to the monthly cost cap);
645
- // - optionally calculate a non-charging provider estimate for Insights when
646
- // core token telemetry is complete, but never debit credits.
647
- if (input.externallyBilled) {
687
+ // Provider settlement and workspace-facing cost are separate. Externally
688
+ // metered subscription/workspace turns remain exempt from the OpenGeni token
689
+ // cap, while a deployment-funded free model still records model.tokens. Every
690
+ // non-credit path records a zero-cost marker and never consults pricing for a
691
+ // debit.
692
+ if (countsTowardTokenCap && totalTokens > 0) {
693
+ await recordUsageEvent(db, {
694
+ accountId: input.accountId,
695
+ workspaceId: input.workspaceId,
696
+ eventType: "model.tokens",
697
+ quantity: totalTokens,
698
+ unit: "tokens",
699
+ sourceResourceType: "model_response",
700
+ sourceResourceId: `${input.turnId}:${input.sourceKey}`,
701
+ sessionId: input.sessionId,
702
+ turnId: input.turnId,
703
+ turnAttemptId: input.turnAttemptId,
704
+ idempotencyKey: `usage:model.tokens:${input.turnId}:${input.sourceKey}`,
705
+ });
706
+ }
707
+ if (!chargesOpenGeniCredits) {
648
708
  await recordUsageEvent(db, {
649
709
  accountId: input.accountId,
650
710
  workspaceId: input.workspaceId,
@@ -664,23 +724,9 @@ export async function recordModelUsageAndDebitCredits(
664
724
  estimatedProviderCostMicros,
665
725
  pricingSource,
666
726
  normalizedUsage,
727
+ ...(gatewayBilling ? { upstreamProvider: gatewayBilling.finalProvider } : {}),
667
728
  };
668
729
  }
669
- if (totalTokens > 0) {
670
- await recordUsageEvent(db, {
671
- accountId: input.accountId,
672
- workspaceId: input.workspaceId,
673
- eventType: "model.tokens",
674
- quantity: totalTokens,
675
- unit: "tokens",
676
- sourceResourceType: "model_response",
677
- sourceResourceId: `${input.turnId}:${input.sourceKey}`,
678
- sessionId: input.sessionId,
679
- turnId: input.turnId,
680
- turnAttemptId: input.turnAttemptId,
681
- idempotencyKey: `usage:model.tokens:${input.turnId}:${input.sourceKey}`,
682
- });
683
- }
684
730
  const shouldDebit = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
685
731
  if (!shouldDebit || (totalTokens === 0 && !gatewayBilling)) {
686
732
  return {
@@ -79,6 +79,32 @@ export function assertSessionAttemptQuiescenceRecoveryDurable(input: {
79
79
 
80
80
  export const QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS = 250;
81
81
  export const QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS = 5_000;
82
+ export const TURN_QUIESCENCE_WATCHDOG_MS = 5 * 60_000;
83
+
84
+ /** A fenced activity must never heartbeat forever while its physical writer
85
+ * drain is stuck. The last-resort worker exit stops every in-process writer;
86
+ * Kubernetes replaces the pod and the workflow's heartbeat-lease reconciler
87
+ * then commits the exact quiescence receipt. */
88
+ export function armTurnQuiescenceWatchdog(input: {
89
+ enabled: boolean;
90
+ timeoutMs?: number;
91
+ onTimeout?: () => void;
92
+ terminateWorker: () => void;
93
+ }): () => void {
94
+ if (!input.enabled) return () => undefined;
95
+ let armed = true;
96
+ const timer = setTimeout(() => {
97
+ if (!armed) return;
98
+ input.onTimeout?.();
99
+ input.terminateWorker();
100
+ }, input.timeoutMs ?? TURN_QUIESCENCE_WATCHDOG_MS);
101
+ timer.unref?.();
102
+ return () => {
103
+ if (!armed) return;
104
+ armed = false;
105
+ clearTimeout(timer);
106
+ };
107
+ }
82
108
 
83
109
  /** Persist the authoritative receipt or durably hand the exact physical proof
84
110
  * to Temporal. This retries signal delivery, not DB eligibility or workflow
@@ -253,6 +279,66 @@ export async function waitForTurnStreamCleanup(
253
279
  await waitForTurnFinalizerStep(providerCompleted, signal);
254
280
  }
255
281
 
282
+ export class IncompleteAgentStreamError extends Error {
283
+ readonly name = "IncompleteAgentStreamError";
284
+
285
+ constructor(message = "Agent stream ended without a successful terminal output") {
286
+ super(message);
287
+ }
288
+ }
289
+
290
+ /**
291
+ * Prove that SDK completion authority did not report a failure or runtime
292
+ * cancellation. The Agents SDK closes its ReadableStream immediately when its
293
+ * run signal aborts, so iterator EOF is not completion authority: the separate
294
+ * `completed` promise carries the eventual run failure.
295
+ *
296
+ * Only Temporal cancellation detaches cleanup. Internal runtime cancellation
297
+ * such as a provider-deadline sandbox rotation must escape to the turn failure
298
+ * settlement path, which checkpoints and redispatches the same logical turn.
299
+ */
300
+ export async function assertSuccessfulAgentStreamCompletion(input: {
301
+ batcherFlush: Promise<unknown>;
302
+ stream: {
303
+ completed: Promise<void>;
304
+ error: unknown;
305
+ };
306
+ temporalCancellationSignal: AbortSignal | undefined;
307
+ runtimeCancellationSignal: AbortSignal;
308
+ }): Promise<void> {
309
+ await waitForTurnStreamCleanup(
310
+ input.batcherFlush,
311
+ input.stream.completed,
312
+ input.temporalCancellationSignal,
313
+ );
314
+
315
+ if (input.runtimeCancellationSignal.aborted) {
316
+ throw (
317
+ input.runtimeCancellationSignal.reason ??
318
+ new IncompleteAgentStreamError("Agent stream was cancelled without a terminal reason")
319
+ );
320
+ }
321
+ if (input.stream.error !== null && input.stream.error !== undefined) {
322
+ throw input.stream.error;
323
+ }
324
+ }
325
+
326
+ /** Call after any more-specific empty-stream recovery has had priority. */
327
+ export function assertAgentStreamNotCancelled(cancelled: boolean): void {
328
+ if (cancelled) {
329
+ throw new IncompleteAgentStreamError("Agent stream was cancelled without a terminal result");
330
+ }
331
+ }
332
+
333
+ /** Interruption streams do not produce a normal final output. Call this only
334
+ * after the interruption branch has settled any required action. */
335
+ export function requireAgentStreamFinalOutput(finalOutput: unknown | undefined): unknown {
336
+ if (finalOutput === undefined) {
337
+ throw new IncompleteAgentStreamError();
338
+ }
339
+ return finalOutput;
340
+ }
341
+
256
342
  /**
257
343
  * Terminal settlement closes the active attempt before turn finalization. The
258
344
  * ordinary workspace admission therefore correctly returns `attempt_fenced`
@@ -56,6 +56,7 @@ import {
56
56
  sandboxEstablishPolicyDecision,
57
57
  ensureTurnModalRegistryImage,
58
58
  sandboxArtifactRuntimeAdmission,
59
+ sandboxSettingsForRoute,
59
60
  } from "./sandbox-route";
60
61
  import type { ClaimTurnOk } from "./claim";
61
62
  import type { GovernanceModelOk } from "./governance-model";
@@ -134,6 +135,16 @@ export async function prepareRunCredentials(deps: PrepareRunCredentialsDeps) {
134
135
  connectionScope,
135
136
  runWorkspaceMutationForSandbox,
136
137
  } = deps;
138
+ // A machine-home row carries selfhosted path defaults, but clearing its
139
+ // pointer explicitly selects the deployment-managed group. Build the stable
140
+ // manifest environment with that effective provider just as Channel-A and
141
+ // viewer readiness do; otherwise the verified managed box and the agent can
142
+ // disagree on backend-aware HOME/token-file paths.
143
+ const sandboxEnvironmentSettings = sandboxSettingsForRoute({
144
+ runSettings,
145
+ machinePrimary,
146
+ groupBoxBackend,
147
+ });
137
148
 
138
149
  const runCredentialResolver =
139
150
  effectiveRunCredentialBackend === "none"
@@ -321,7 +332,7 @@ export async function prepareRunCredentials(deps: PrepareRunCredentialsDeps) {
321
332
  codemodeTokenExpiresAt: sandboxCodemodeTokenExpiresAt,
322
333
  } = await waitForTurnOperation(
323
334
  sandboxEnvironmentForRun(
324
- runSettings,
335
+ sandboxEnvironmentSettings,
325
336
  turnResources,
326
337
  // Rig default sets merged BELOW the session set (session wins); rig-less
327
338
  // turns pass exactly workspaceVariableSet?.values (byte-for-byte today).
@@ -45,6 +45,7 @@ import { createTurnCredentialLeases } from "./credential-leases";
45
45
  import { createTurnMediaArtifacts } from "./media-artifacts";
46
46
  import { createTurnHistorySink } from "./history-sink";
47
47
  import { checkpointHistoryBeforeProviderDispatch } from "./provider-dispatch-barrier";
48
+ import { providerRecoveryCountAfterModelRequestPhase } from "./errors";
48
49
  import { sandboxRunAs } from "@opengeni/runtime";
49
50
  import { randomUUID } from "node:crypto";
50
51
  import { createModelCheckpointMemoryCollector } from "../../model-checkpoint-memory-collector";
@@ -88,6 +89,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
88
89
  return async function runAgentTurn(input: RunAgentTurnInput): Promise<RunAgentTurnResult> {
89
90
  const {
90
91
  settings,
92
+ catalogSourceSettings = settings,
91
93
  db,
92
94
  bus,
93
95
  runtime,
@@ -124,7 +126,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
124
126
  billingState,
125
127
  sandboxState,
126
128
  renewals,
127
- recordingState,
128
129
  eventing,
129
130
  workspaceRefs,
130
131
  providerTurn,
@@ -222,7 +223,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
222
223
  activityContext,
223
224
  sandboxRotationController,
224
225
  sandboxState,
225
- recordingState,
226
226
  eventing,
227
227
  attempt,
228
228
  });
@@ -230,9 +230,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
230
230
  requireResolvedSandboxForMutation,
231
231
  runWorkspaceMutationForSandbox,
232
232
  stopLeaseHeartbeat,
233
- abandonActiveRecording,
234
233
  flushRuntimeBatcher,
235
- maybeStartOnTurnRecording,
236
234
  finalizeTurnOpStreamOps,
237
235
  } = sandboxRuntime;
238
236
 
@@ -305,10 +303,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
305
303
  const claimed = await claimTurnAttempt({
306
304
  input,
307
305
  settings,
306
+ catalogSourceSettings,
308
307
  db,
309
308
  bus,
310
309
  runtime,
311
- objectStorage,
312
310
  observability,
313
311
  entitlements,
314
312
  wakeSessionWorkflow,
@@ -320,13 +318,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
320
318
  attempt,
321
319
  billingState,
322
320
  sandboxState,
323
- recordingState,
324
321
  eventing,
325
322
  leases,
326
323
  media,
327
324
  claimedResult,
328
325
  acknowledgeLostAttemptOwnership,
329
- abandonActiveRecording,
330
326
  });
331
327
  if ("exit" in claimed) return claimed.exit;
332
328
  if (!eventing.publish || !eventing.settle) {
@@ -350,6 +346,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
350
346
  trigger,
351
347
  humanInputResume,
352
348
  interactionInterventionResume,
349
+ attachPendingUpdatesAfterOpenSuffix,
353
350
  throwIfWorkerShuttingDown,
354
351
  throwIfTurnCancelled,
355
352
  opJournal,
@@ -605,6 +602,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
605
602
  },
606
603
  },
607
604
  ]);
605
+ attempt.providerRecoveryCount = providerRecoveryCountAfterModelRequestPhase(
606
+ attempt.providerRecoveryCount,
607
+ event.phase,
608
+ );
608
609
  } catch (error) {
609
610
  auditOutcome = "failed";
610
611
  throw error;
@@ -750,6 +751,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
750
751
  },
751
752
  },
752
753
  ]);
754
+ attempt.providerRecoveryCount = providerRecoveryCountAfterModelRequestPhase(
755
+ attempt.providerRecoveryCount,
756
+ event.phase,
757
+ );
753
758
  } catch (error) {
754
759
  auditOutcome = "failed";
755
760
  throw error;
@@ -793,7 +798,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
793
798
  );
794
799
  const compactionPrep = await prepareCompaction({
795
800
  input,
796
- settings,
801
+ settings: capabilitySettings,
797
802
  db,
798
803
  bus,
799
804
  observability,
@@ -834,6 +839,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
834
839
  compactionModeOptions,
835
840
  compactionOnlyTurn,
836
841
  compactionSummarizerFor,
842
+ settleDeferredSteerAfterCompaction,
837
843
  } = compactionPrep.ok;
838
844
 
839
845
  const toolPolicy = await prepareTurnToolPolicy({
@@ -1058,6 +1064,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1058
1064
  ];
1059
1065
  const toolRuntime = await prepareTurnToolRuntime({
1060
1066
  input,
1067
+ catalogSourceSettings,
1061
1068
  db,
1062
1069
  bus,
1063
1070
  runtime,
@@ -1112,8 +1119,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1112
1119
  eventing,
1113
1120
  attempt,
1114
1121
  sandboxState,
1115
- recordingState,
1116
- maybeStartOnTurnRecording,
1117
1122
  providerTurn,
1118
1123
  media,
1119
1124
  leases,
@@ -1241,7 +1246,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1241
1246
  };
1242
1247
  const postAgentCompaction = await runPostAgentCompaction({
1243
1248
  input,
1244
- settings,
1249
+ settings: capabilitySettings,
1245
1250
  db,
1246
1251
  bus,
1247
1252
  observability,
@@ -1278,6 +1283,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1278
1283
  compactionModeOptions,
1279
1284
  compactionOnlyTurn,
1280
1285
  compactionSummarizerFor,
1286
+ settleDeferredSteerAfterCompaction,
1281
1287
  agent,
1282
1288
  });
1283
1289
  if ("exit" in postAgentCompaction) return postAgentCompaction.exit;
@@ -1405,7 +1411,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1405
1411
  );
1406
1412
  return await runTurnStreamAttempt({
1407
1413
  input,
1408
- settings,
1414
+ settings: capabilitySettings,
1409
1415
  db,
1410
1416
  runtime,
1411
1417
  objectStorage,
@@ -1456,12 +1462,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1456
1462
  turn,
1457
1463
  trigger,
1458
1464
  humanInputResume,
1465
+ attachPendingUpdatesAfterOpenSuffix,
1459
1466
  agent,
1460
1467
  resolvedModel,
1461
1468
  providerApi,
1462
1469
  runSettings,
1463
1470
  turnTools,
1464
1471
  compactSummarizer,
1472
+ settleDeferredSteerAfterCompaction,
1465
1473
  compactionModelHistoryProjector,
1466
1474
  generatedImageHistoryProjector,
1467
1475
  modelHistoryProjector,
@@ -1525,14 +1533,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1525
1533
  attempt,
1526
1534
  sandboxState,
1527
1535
  renewals,
1528
- recordingState,
1529
1536
  eventing,
1530
1537
  providerTurn,
1531
1538
  workspaceRefs,
1532
1539
  runWorkspaceMutationForSandbox,
1533
1540
  requireResolvedSandboxForMutation,
1534
1541
  stopLeaseHeartbeat,
1535
- abandonActiveRecording,
1536
1542
  turnCompletionMemoryCollector,
1537
1543
  });
1538
1544
  }
@@ -22,6 +22,7 @@ import {
22
22
  resolveConnectedMachineWorkspaceRoot,
23
23
  } from "@opengeni/runtime";
24
24
  import { type Settings } from "@opengeni/config";
25
+ import { managedSessionGroupBackend, managedSessionGroupOs } from "@opengeni/core";
25
26
  import { mergeResourceRefs } from "../common";
26
27
  import {
27
28
  mintSandboxCodemodeToken,
@@ -42,6 +43,7 @@ import {
42
43
  makeMachineOpObserver,
43
44
  recordSandboxLogicalProvision,
44
45
  recordSandboxProvisionAttempt,
46
+ recordSandboxSharedPreparation,
45
47
  recordTurnStartupPhase,
46
48
  runtimeMetricsHooksForObservability,
47
49
  } from "../../observability-metrics";
@@ -62,11 +64,13 @@ import {
62
64
  sandboxEstablishPolicyDecision,
63
65
  shouldPrefetchManagedSandbox,
64
66
  reconcileActiveSandboxPointer,
67
+ sandboxSettingsForRoute,
65
68
  } from "./sandbox-route";
66
69
  import type { ClaimTurnOk } from "./claim";
67
70
  import type { GovernanceModelOk } from "./governance-model";
68
71
  import type { SandboxTurnRuntime } from "./sandbox-runtime";
69
72
  import type { AttemptIdentityState, EventingState, SandboxRuntimeState } from "./turn-context";
73
+ import { createSharedRigSetupCoordinator } from "./sandbox-shared-preparation";
70
74
 
71
75
  export type SandboxRouteDeps = {
72
76
  input: RunAgentTurnInput;
@@ -249,20 +253,32 @@ export async function resolveSandboxRoute(deps: SandboxRouteDeps): Promise<Sandb
249
253
  // common path this is runSettings.sandboxBackend. A selfhosted home turn
250
254
  // that is NOT machine-primary falls back to the deployment cloud backend
251
255
  // so swap-away / flag-off degrade to a real group box.
252
- const groupBoxBackend: Settings["sandboxBackend"] =
253
- runSettings.sandboxBackend === "selfhosted" && !machinePrimary
254
- ? settings.sandboxBackend
255
- : runSettings.sandboxBackend;
256
+ const groupBoxBackend: Settings["sandboxBackend"] = !machinePrimary
257
+ ? (managedSessionGroupBackend(settings.sandboxBackend, runSettings.sandboxBackend) ??
258
+ runSettings.sandboxBackend)
259
+ : runSettings.sandboxBackend;
260
+ const groupRouteSettings = sandboxSettingsForRoute({
261
+ runSettings,
262
+ machinePrimary,
263
+ groupBoxBackend,
264
+ });
265
+ if (groupRouteSettings !== runSettings) {
266
+ // `modelRunSettings` drives SandboxAgent manifest construction and the
267
+ // legacy SDK-owned client path. Keep its provider/model context overrides,
268
+ // but replace the stale durable machine-home label with the route that will
269
+ // actually execute.
270
+ eventing.modelRunSettings = {
271
+ ...eventing.modelRunSettings,
272
+ sandboxBackend: groupRouteSettings.sandboxBackend,
273
+ };
274
+ }
256
275
  sandboxState.startupMilestoneBackend = activeSandboxBackend ?? groupBoxBackend;
257
276
  media.sandboxFileDownloadBackend = sandboxState.startupMilestoneBackend;
258
277
  // Durable lease identity is always the logical OCI/base image. A provider
259
278
  // image id in runSettings is only a physical cold-create optimization and
260
279
  // must never become cross-surface shared-state fencing.
261
280
  const groupBoxImage = rigProviderImageSourceImage(logicalSandboxSettings, groupBoxBackend);
262
- const sandboxCreationBackend: Settings["sandboxBackend"] =
263
- settings.sandboxOwnershipEnabled && runSettings.sandboxBackend !== "none"
264
- ? groupBoxBackend
265
- : runSettings.sandboxBackend;
281
+ const sandboxCreationBackend: Settings["sandboxBackend"] = groupRouteSettings.sandboxBackend;
266
282
  const effectiveRunCredentialBackend = activeSandboxBackend ?? groupBoxBackend;
267
283
 
268
284
  return {
@@ -315,6 +331,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
315
331
  rigVersion,
316
332
  turnResources,
317
333
  } = deps;
334
+ const groupBoxOs = managedSessionGroupOs(runSettings.sandboxBackend, session.sandboxOs);
318
335
  const {
319
336
  releaseLateSandbox,
320
337
  onHomeSandboxRebound,
@@ -577,7 +594,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
577
594
  sandboxGroupId: lazyGroupId,
578
595
  sessionId: input.sessionId,
579
596
  backend: groupBoxBackend,
580
- os: session.sandboxOs,
597
+ os: groupBoxOs,
581
598
  environment: sandboxEnvironment,
582
599
  ...(groupBoxImage
583
600
  ? {
@@ -656,7 +673,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
656
673
  // deployment default), never a "selfhosted" box (which would throw
657
674
  // for lack of a bound agentId).
658
675
  backend: groupBoxBackend,
659
- os: session.sandboxOs,
676
+ os: groupBoxOs,
660
677
  environment: sandboxEnvironment,
661
678
  // IMAGE IS SHARED STATE (B3): the container image
662
679
  // this run resolves. The lease stamps it + conflicts on a live shared box
@@ -881,7 +898,7 @@ export async function bindLazySandboxProvisioner(
881
898
  // recursion that HANGS the turn — caught live on staging 2026-07-08). The SDK
882
899
  // already holds the proxy directly (injected as lazyOwnedSandbox.session), so it
883
900
  // gets per-op routing; the worker-side handle (resolvedSandbox: release,
884
- // heartbeat, computer-use recording) wants the real box, unproxied.
901
+ // heartbeat, workspace capture) wants the real box, unproxied.
885
902
  return provisioned;
886
903
  },
887
904
  {
@@ -1007,6 +1024,24 @@ export async function bindLazySandboxProvisioner(
1007
1024
  ),
1008
1025
  }
1009
1026
  : {}),
1027
+ ...(sandboxState.sandboxGroupId && sandboxState.sandboxHolderId
1028
+ ? {
1029
+ coordinateSharedRigSetup: createSharedRigSetupCoordinator({
1030
+ db,
1031
+ accountId: input.accountId,
1032
+ workspaceId: input.workspaceId,
1033
+ sandboxGroupId: sandboxState.sandboxGroupId,
1034
+ attemptId: input.attemptId,
1035
+ holderId: sandboxState.sandboxHolderId,
1036
+ sandbox: provisioned,
1037
+ ...(runtimeCancellationSignal
1038
+ ? { signal: runtimeCancellationSignal }
1039
+ : {}),
1040
+ observe: (measurement) =>
1041
+ recordSandboxSharedPreparation(observability, measurement),
1042
+ }),
1043
+ }
1044
+ : {}),
1010
1045
  },
1011
1046
  ),
1012
1047
  (measurement) => {