@opengeni/worker-bundle 0.10.8 → 0.11.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,6 @@
1
1
  import {
2
+ advanceWorkspaceGeneration,
3
+ verifyWorkspaceMutationSettlement,
2
4
  applySessionTurnSettlement,
3
5
  requestSessionTurnRecovery,
4
6
  claimSessionWorkForAttempt,
@@ -101,6 +103,7 @@ import {
101
103
  sandboxFileDownloadFailureNote,
102
104
  SUMMARY_BUFFER_TOKENS,
103
105
  runOwnedSandboxSetup,
106
+ RoutingMutationOutcomeUnknownError,
104
107
  swapTargetEstablishability,
105
108
  type SandboxFileDownload,
106
109
  type SandboxFileDownloadFailure,
@@ -154,6 +157,11 @@ import {
154
157
  type CodexRotationStrategy,
155
158
  type RotationDecision,
156
159
  } from "./codex-rotation";
160
+ import {
161
+ codexFleetShadowDecisionMetricLabelsV1,
162
+ codexFleetShadowErrorMetricLabelsV1,
163
+ publishCodexFleetShadowDecisionV1,
164
+ } from "./codex-fleet-shadow";
157
165
  import type { CodexAccountStatus } from "@opengeni/db";
158
166
  import { buildCodexTokenResolver } from "./codex-auth";
159
167
  import {
@@ -526,6 +534,7 @@ export async function persistOrSignalSessionAttemptQuiescence(input: {
526
534
  export async function drainAttemptOwnedSandboxWriters(input: {
527
535
  toolCancellationFence: Pick<TurnToolCancellationFence, "cancel" | "waitForQuiescence"> | null;
528
536
  cancellationReason?: unknown;
537
+ gitCredentialRenewals: readonly Pick<GitCredentialRenewalController, "stop">[];
529
538
  toolspaceTokenRenewal: Pick<ToolspaceTokenRenewalController, "stop"> | null;
530
539
  runCredentialRenewal: Pick<RunCredentialRenewalController, "stop"> | null;
531
540
  }): Promise<void> {
@@ -535,6 +544,7 @@ export async function drainAttemptOwnedSandboxWriters(input: {
535
544
  );
536
545
  await input.toolCancellationFence.waitForQuiescence();
537
546
  }
547
+ await Promise.all(input.gitCredentialRenewals.map(async (renewal) => await renewal.stop()));
538
548
  await input.toolspaceTokenRenewal?.stop();
539
549
  await input.runCredentialRenewal?.stop();
540
550
  }
@@ -1533,6 +1543,19 @@ export function computerToolModeForTurn(
1533
1543
  return "hosted";
1534
1544
  }
1535
1545
 
1546
+ /**
1547
+ * Decide whether this turn's provider accepts typed file/image content in a
1548
+ * user message. The legacy built-in client and registry Responses clients use
1549
+ * the Agents SDK Responses converter, which supports `input_image` and
1550
+ * `input_file`. Chat-completions providers retain the sandbox-path projection
1551
+ * until that transport declares and proves an equivalent capability.
1552
+ */
1553
+ export function modelAcceptsTypedAttachmentContentForTurn(
1554
+ resolvedModel: { provider: { api: ModelProviderApi } } | null,
1555
+ ): boolean {
1556
+ return resolvedModel === null || resolvedModel.provider.api === "responses";
1557
+ }
1558
+
1536
1559
  export type TurnSandboxProvisioner<T> = {
1537
1560
  get(): Promise<T>;
1538
1561
  hasStarted(): boolean;
@@ -2023,6 +2046,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2023
2046
  // run. null when the flag is off (byte-for-byte the legacy build-and-discard
2024
2047
  // path) OR when the backend is "none". Released + dropped in `finally`.
2025
2048
  let resolvedSandbox: ResumedTurnSandbox | null = null;
2049
+ const requireResolvedSandboxForMutation = (message: string): ResumedTurnSandbox => {
2050
+ if (!resolvedSandbox) throw new Error(message);
2051
+ return resolvedSandbox;
2052
+ };
2026
2053
  // The machine-primary SelfhostedSession (the UNWRAPPED backend, not the
2027
2054
  // routing proxy): held so the turn's completion can final-ack this turn's
2028
2055
  // settled op-stream ops AFTER the results are durably persisted.
@@ -2034,11 +2061,102 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2034
2061
  // THIS handle so a mid-turn sandbox_swap can never re-route those execs onto a
2035
2062
  // connected machine (the user's real computer).
2036
2063
  let setupBoxSession: unknown = null;
2064
+ // A same-target API repair can replace the home provider while this turn is
2065
+ // alive. Keep setup/snapshot persistence on the rebound raw session while
2066
+ // preserving the SDK-owned routing proxy for eager turns. Lazy turns hold the
2067
+ // proxy separately, so their worker-side handle may replace its raw session.
2068
+ const onHomeSandboxRebound = (rebound: {
2069
+ established: EstablishedSandboxSession;
2070
+ leaseEpoch: number;
2071
+ }): void => {
2072
+ const current = resolvedSandbox;
2073
+ const previousSession = current?.established.session;
2074
+ const preserveRoutingProxy = current !== null && previousSession !== setupBoxSession;
2075
+ setupBoxSession = rebound.established.session;
2076
+ if (!current) return;
2077
+ current.leaseEpoch = rebound.leaseEpoch;
2078
+ current.established = preserveRoutingProxy
2079
+ ? {
2080
+ ...current.established,
2081
+ client: rebound.established.client,
2082
+ // Keep the stable SDK-facing proxy; only its resolver changes the
2083
+ // underlying backend. The worker's setupBoxSession above is raw.
2084
+ session: previousSession,
2085
+ sessionState: rebound.established.sessionState,
2086
+ instanceId: rebound.established.instanceId,
2087
+ backendId: rebound.established.backendId,
2088
+ ...(rebound.established.origin ? { origin: rebound.established.origin } : {}),
2089
+ ...(rebound.established.restoredArchive
2090
+ ? { restoredArchive: rebound.established.restoredArchive }
2091
+ : {}),
2092
+ }
2093
+ : rebound.established;
2094
+ };
2037
2095
  // The globally unique durable turn-attempt holder id + the group id,
2038
2096
  // captured so the lease heartbeat can refresh the lease TTL epoch-fenced
2039
2097
  // (a superseded owner self-evicts) and finally can release.
2040
2098
  let sandboxHolderId: TurnSandboxLeaseHolderId | null = null;
2041
2099
  let sandboxGroupId: string | null = null;
2100
+ const runWorkspaceMutationForSandbox = async <T>(
2101
+ sandbox: ResumedTurnSandbox,
2102
+ operation: string,
2103
+ mutation: () => Promise<T>,
2104
+ ): Promise<T> => {
2105
+ // Connected machines are the user's own persistence and never dirty the
2106
+ // cloud home archive. Every persistable raw-session write batch is fenced
2107
+ // against the exact current lease/provider before the provider sees it.
2108
+ if (sandbox.established.backendId === "selfhosted") return await mutation();
2109
+ if (!sandboxGroupId || !sandboxHolderId || !turnId || executionGeneration <= 0) {
2110
+ throw new Error("Workspace mutation attempted before exact turn sandbox admission");
2111
+ }
2112
+ const identity = {
2113
+ accountId: input.accountId,
2114
+ workspaceId: input.workspaceId,
2115
+ sessionId: input.sessionId,
2116
+ turnId,
2117
+ executionGeneration,
2118
+ attemptId: input.attemptId,
2119
+ holderId: sandboxHolderId,
2120
+ sandboxGroupId,
2121
+ expectedEpoch: sandbox.leaseEpoch,
2122
+ expectedInstanceId: sandbox.established.instanceId,
2123
+ operation,
2124
+ };
2125
+ const admission = await advanceWorkspaceGeneration(db, identity);
2126
+ let result: T;
2127
+ try {
2128
+ result = await mutation();
2129
+ } catch (providerError) {
2130
+ try {
2131
+ await verifyWorkspaceMutationSettlement(db, {
2132
+ ...identity,
2133
+ admission,
2134
+ outcome: "rejected",
2135
+ });
2136
+ } catch (settlementError) {
2137
+ throw new RoutingMutationOutcomeUnknownError(
2138
+ operation,
2139
+ `Platform workspace mutation "${operation}" rejected at the provider but lost its durable physical settlement; its outcome is unknown and it was not replayed`,
2140
+ { cause: settlementError },
2141
+ );
2142
+ }
2143
+ throw providerError;
2144
+ }
2145
+ try {
2146
+ await verifyWorkspaceMutationSettlement(db, {
2147
+ ...identity,
2148
+ admission,
2149
+ outcome: "resolved",
2150
+ });
2151
+ } catch (settlementError) {
2152
+ throw new RoutingMutationOutcomeUnknownError(
2153
+ operation,
2154
+ `Platform workspace mutation "${operation}" returned from the provider but lost its durable settlement fence; its outcome is unknown and it was not replayed`,
2155
+ { cause: settlementError },
2156
+ );
2157
+ }
2158
+ return result;
2159
+ };
2042
2160
  // Lease-TTL refresh timer (parallels the activity heartbeat): while the turn
2043
2161
  // runs it refreshes expires_at epoch-fenced so a legit multi-day turn is
2044
2162
  // never TTL-reaped. Cleared in finally. Only set when the flag resolved a box.
@@ -2994,6 +3112,80 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2994
3112
  codexLeaseConfirmedUntilMs !== null;
2995
3113
  if (codexLeaseHeld) startCodexLeaseHeartbeat();
2996
3114
 
3115
+ const actualOutcome = effectiveCodexCredentialId
3116
+ ? "selected"
3117
+ : rotationDecision.kind === "allCapped"
3118
+ ? "waiting"
3119
+ : "none";
3120
+ const actualReason = effectiveCodexCredentialId
3121
+ ? leased.reused
3122
+ ? "lease_reused"
3123
+ : sessionPin === effectiveCodexCredentialId
3124
+ ? "pin"
3125
+ : rotationDecision.kind === "active" && rotationDecision.moved
3126
+ ? "rotation"
3127
+ : "active"
3128
+ : rotationDecision.kind === "allCapped"
3129
+ ? "all_capped"
3130
+ : "none";
3131
+ const fencedInFlight =
3132
+ leased.reused ||
3133
+ (continuationCodexCredentialId !== null &&
3134
+ continuationCodexCredentialId === effectiveCodexCredentialId);
3135
+ const shadowResult = await publishCodexFleetShadowDecisionV1({
3136
+ enabled: settings.codexFleetPolicyShadowEnabled,
3137
+ decision: {
3138
+ accounts: leased.accounts,
3139
+ actualCredentialId: effectiveCodexCredentialId,
3140
+ actualOutcome,
3141
+ actualReason,
3142
+ affinityCredentialId: fencedInFlight
3143
+ ? effectiveCodexCredentialId
3144
+ : (sessionPin ?? sessionCodex?.lastCredentialId ?? null),
3145
+ fencedInFlight,
3146
+ nearExhaustionPct: settings.codexRotationNearExhaustionPct,
3147
+ now: new Date(),
3148
+ aliasSeed: randomUUID(),
3149
+ },
3150
+ publish,
3151
+ });
3152
+ if (shadowResult.outcome === "published") {
3153
+ const shadowPayload = shadowResult.payload;
3154
+ observability.incrementCounter({
3155
+ name: "opengeni_codex_fleet_shadow_decisions_total",
3156
+ help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
3157
+ labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload),
3158
+ });
3159
+ observability.info("Codex adaptive fleet shadow decision", {
3160
+ workspaceId: input.workspaceId,
3161
+ policyVersion: shadowPayload.replay.policyVersion,
3162
+ inputFingerprint: shadowPayload.replay.inputFingerprint,
3163
+ decisionFingerprint: shadowPayload.replay.decisionFingerprint,
3164
+ actualOutcome: shadowPayload.actual.outcome,
3165
+ shadowOutcome: shadowPayload.replay.decision.outcome,
3166
+ comparison: shadowPayload.comparison,
3167
+ candidateCount: shadowPayload.replay.input.candidates.length,
3168
+ truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
3169
+ payloadBytes: shadowResult.payloadBytes,
3170
+ });
3171
+ } else if (shadowResult.outcome === "failed") {
3172
+ // Shadow observability is explicitly non-authoritative. A malformed
3173
+ // snapshot or event-write fault must never change the authoritative lease,
3174
+ // capacity wait, failover, or the account serving this fenced turn.
3175
+ observability.incrementCounter({
3176
+ name: "opengeni_codex_fleet_shadow_errors_total",
3177
+ help: "Shadow decision build/publication failures.",
3178
+ labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
3179
+ });
3180
+ observability.warn("Codex adaptive fleet shadow decision failed open", {
3181
+ workspaceId: input.workspaceId,
3182
+ stage: shadowResult.stage,
3183
+ reason: shadowResult.reason,
3184
+ errorName: shadowResult.errorName,
3185
+ payloadBytes: shadowResult.payloadBytes,
3186
+ });
3187
+ }
3188
+
2997
3189
  const eligibleCount = leased.accounts.filter((account) =>
2998
3190
  isCodexCredentialEligible(account, new Date()),
2999
3191
  ).length;
@@ -3964,6 +4156,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3964
4156
  const attachGitCredentialRenewal = async (
3965
4157
  tokenSession: GitCredentialTokenWriterSession,
3966
4158
  initial: MintedRunGitCredentials | undefined,
4159
+ initialSandbox?: ResumedTurnSandbox,
3967
4160
  ): Promise<void> => {
3968
4161
  if (!initial || initial.bindings.length === 0) return;
3969
4162
  const previous = gitCredentialRenewals;
@@ -4008,16 +4201,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4008
4201
  throw new Error("credential renewal produced no binding token");
4009
4202
  }
4010
4203
  const runAs = sandboxRunAs(runSettings);
4011
- await refreshGitCredentialBindingTokenFiles(tokenSession, [pendingBinding], {
4012
- ...(runAs ? { runAs } : {}),
4013
- ...(toolCancellationFenceRef.current
4014
- ? {
4015
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4016
- toolCancellationFenceRef.current,
4017
- ),
4018
- }
4019
- : {}),
4020
- });
4204
+ const targetSandbox = resolvedSandbox ?? initialSandbox;
4205
+ if (!targetSandbox) {
4206
+ throw new Error("Git credential renewal has no exact sandbox lease target");
4207
+ }
4208
+ await runWorkspaceMutationForSandbox(
4209
+ targetSandbox,
4210
+ "gitCredentialRenewal",
4211
+ async () =>
4212
+ await refreshGitCredentialBindingTokenFiles(tokenSession, [pendingBinding!], {
4213
+ ...(runAs ? { runAs } : {}),
4214
+ ...(toolCancellationFenceRef.current
4215
+ ? {
4216
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4217
+ toolCancellationFenceRef.current,
4218
+ ),
4219
+ }
4220
+ : {}),
4221
+ }),
4222
+ );
4021
4223
  },
4022
4224
  onSuccess: ({ providers: renewedProviders }) => {
4023
4225
  for (const provider of renewedProviders) {
@@ -4056,6 +4258,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4056
4258
  const attachToolspaceTokenRenewal = async (
4057
4259
  tokenSession: ToolspaceTokenWriterSession,
4058
4260
  initialExpiresAt = sandboxToolspaceTokenExpiresAt,
4261
+ initialSandbox?: ResumedTurnSandbox,
4059
4262
  ): Promise<void> => {
4060
4263
  if (!sandboxToolspaceToken || !initialExpiresAt) return;
4061
4264
  const previous = toolspaceTokenRenewal;
@@ -4067,22 +4270,31 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4067
4270
  await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority);
4068
4271
  const write = async (material: NonNullable<Awaited<ReturnType<typeof mint>>>) => {
4069
4272
  const runAs = sandboxRunAs(runSettings);
4070
- await refreshToolspaceTokenFile(tokenSession, material.token, {
4071
- ...(runAs ? { runAs } : {}),
4072
- ...(sandboxToolspaceTokenFile
4073
- ? {
4074
- tokenFile: sandboxToolspaceTokenFile,
4075
- legacyTokenFile: sandboxEnvironment.OPENGENI_TOOLSPACE_TOKEN_FILE!,
4076
- }
4077
- : {}),
4078
- ...(toolCancellationFenceRef.current
4079
- ? {
4080
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4081
- toolCancellationFenceRef.current,
4082
- ),
4083
- }
4084
- : {}),
4085
- });
4273
+ const targetSandbox = resolvedSandbox ?? initialSandbox;
4274
+ if (!targetSandbox) {
4275
+ throw new Error("Toolspace token renewal has no exact sandbox lease target");
4276
+ }
4277
+ await runWorkspaceMutationForSandbox(
4278
+ targetSandbox,
4279
+ "toolspaceTokenRenewal",
4280
+ async () =>
4281
+ await refreshToolspaceTokenFile(tokenSession, material.token, {
4282
+ ...(runAs ? { runAs } : {}),
4283
+ ...(sandboxToolspaceTokenFile
4284
+ ? {
4285
+ tokenFile: sandboxToolspaceTokenFile,
4286
+ legacyTokenFile: sandboxEnvironment.OPENGENI_TOOLSPACE_TOKEN_FILE!,
4287
+ }
4288
+ : {}),
4289
+ ...(toolCancellationFenceRef.current
4290
+ ? {
4291
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4292
+ toolCancellationFenceRef.current,
4293
+ ),
4294
+ }
4295
+ : {}),
4296
+ }),
4297
+ );
4086
4298
  };
4087
4299
  let renewalExpiresAt = initialExpiresAt;
4088
4300
  if (renewalExpiresAt.getTime() <= Date.now() + TOOLSPACE_TOKEN_EXPIRY_LEAD_MS) {
@@ -4127,6 +4339,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4127
4339
 
4128
4340
  const attachRunCredentialRenewal = async (
4129
4341
  credentialSession: RunCredentialCommandSession,
4342
+ initialSandbox?: ResumedTurnSandbox,
4130
4343
  ): Promise<void> => {
4131
4344
  if (!runCredentialResolver) return;
4132
4345
  const previous = runCredentialRenewal;
@@ -4135,15 +4348,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4135
4348
  if (runCredentialRenewalClosed) return;
4136
4349
  runCredentialSession = credentialSession;
4137
4350
 
4351
+ const requireTargetSandbox = (): ResumedTurnSandbox => {
4352
+ const targetSandbox = resolvedSandbox ?? initialSandbox;
4353
+ if (!targetSandbox) {
4354
+ throw new Error("Run credential mutation has no exact sandbox lease target");
4355
+ }
4356
+ return targetSandbox;
4357
+ };
4358
+
4138
4359
  if (!initialRunCredentialMaterial) {
4139
- await clearRunCredentials(
4140
- credentialSession,
4141
- input.sessionId,
4142
- toolCancellationFenceRef.current
4143
- ? toolCancellationFenceRef.current.runSandboxCommand.bind(
4144
- toolCancellationFenceRef.current,
4145
- )
4146
- : undefined,
4360
+ await runWorkspaceMutationForSandbox(
4361
+ requireTargetSandbox(),
4362
+ "runCredentialClear",
4363
+ async () =>
4364
+ await clearRunCredentials(
4365
+ credentialSession,
4366
+ input.sessionId,
4367
+ toolCancellationFenceRef.current
4368
+ ? toolCancellationFenceRef.current.runSandboxCommand.bind(
4369
+ toolCancellationFenceRef.current,
4370
+ )
4371
+ : undefined,
4372
+ ),
4147
4373
  );
4148
4374
  return;
4149
4375
  }
@@ -4153,33 +4379,43 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4153
4379
  pruneOtherAttempts = false,
4154
4380
  ): Promise<void> => {
4155
4381
  if (!material) {
4156
- await clearRunCredentialsForAttempt(credentialSession, {
4157
- sessionId: input.sessionId,
4158
- attemptId: input.attemptId,
4159
- executionGeneration,
4160
- });
4382
+ await runWorkspaceMutationForSandbox(
4383
+ requireTargetSandbox(),
4384
+ "runCredentialAttemptClear",
4385
+ async () =>
4386
+ await clearRunCredentialsForAttempt(credentialSession, {
4387
+ sessionId: input.sessionId,
4388
+ attemptId: input.attemptId,
4389
+ executionGeneration,
4390
+ }),
4391
+ );
4161
4392
  return;
4162
4393
  }
4163
4394
  registerSecretRedactions(material.redactions);
4164
- await materializeRunCredentials(credentialSession, material, {
4165
- sessionId: input.sessionId,
4166
- attemptId: input.attemptId,
4167
- executionGeneration,
4168
- ...(pruneOtherAttempts ? { pruneOtherAttempts: true } : {}),
4169
- ...(!pruneOtherAttempts ? { pruneSupersededGenerations: true } : {}),
4170
- ...(material.authNeeded.length > 0 &&
4171
- Object.keys(material.environment).length === 0 &&
4172
- material.files.length === 0
4173
- ? { prunePreviousGenerations: true }
4174
- : {}),
4175
- ...(toolCancellationFenceRef.current
4176
- ? {
4177
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4178
- toolCancellationFenceRef.current,
4179
- ),
4180
- }
4181
- : {}),
4182
- });
4395
+ await runWorkspaceMutationForSandbox(
4396
+ requireTargetSandbox(),
4397
+ "runCredentialMaterialization",
4398
+ async () =>
4399
+ await materializeRunCredentials(credentialSession, material, {
4400
+ sessionId: input.sessionId,
4401
+ attemptId: input.attemptId,
4402
+ executionGeneration,
4403
+ ...(pruneOtherAttempts ? { pruneOtherAttempts: true } : {}),
4404
+ ...(!pruneOtherAttempts ? { pruneSupersededGenerations: true } : {}),
4405
+ ...(material.authNeeded.length > 0 &&
4406
+ Object.keys(material.environment).length === 0 &&
4407
+ material.files.length === 0
4408
+ ? { prunePreviousGenerations: true }
4409
+ : {}),
4410
+ ...(toolCancellationFenceRef.current
4411
+ ? {
4412
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4413
+ toolCancellationFenceRef.current,
4414
+ ),
4415
+ }
4416
+ : {}),
4417
+ }),
4418
+ );
4183
4419
  for (const payload of runCredentialAuthNeededPayloads(material)) {
4184
4420
  const key = JSON.stringify(payload);
4185
4421
  if (publishedRunCredentialNotices.has(key)) continue;
@@ -4325,11 +4561,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4325
4561
  settings,
4326
4562
  bus,
4327
4563
  onOp: machineOpObserver.observer,
4564
+ onHomeSandboxRebound,
4328
4565
  },
4329
4566
  {
4330
4567
  workspaceId: input.workspaceId,
4331
4568
  sessionId: input.sessionId,
4332
4569
  environment: sandboxEnvironment,
4570
+ workspaceMutationFence: {
4571
+ accountId: input.accountId,
4572
+ turnId: turn.id,
4573
+ executionGeneration,
4574
+ attemptId: input.attemptId,
4575
+ },
4333
4576
  pinnedSelfhosted: {
4334
4577
  sandboxId: activeSandboxPointer!.activeSandboxId!,
4335
4578
  epoch: activeSandboxPointer!.activeEpoch,
@@ -4414,24 +4657,41 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4414
4657
  // the flag off the established group box is injected unchanged (today's
4415
4658
  // path). The lease still owns the group box lifecycle — the proxy is a
4416
4659
  // routing veneer, not an owner.
4417
- if (routingEnabled(settings)) {
4418
- resolvedSandbox = {
4419
- ...resolvedSandbox,
4420
- established: wrapTurnBoxWithRouting(
4421
- { db, settings, bus },
4422
- // Thread the SAME declared environment the group box was created with
4423
- // (resumeBoxForTurn, above) so a selfhosted swap target's manifest
4424
- // carries it too — the SDK's per-turn manifest-env delta stays empty
4425
- // (no "cannot change manifest environment variables" throw).
4426
- {
4427
- workspaceId: input.workspaceId,
4428
- sessionId: input.sessionId,
4429
- environment: sandboxEnvironment,
4660
+ resolvedSandbox = {
4661
+ ...resolvedSandbox,
4662
+ established: wrapTurnBoxWithRouting(
4663
+ {
4664
+ db,
4665
+ settings,
4666
+ bus,
4667
+ onHomeSandboxLost: publishSandboxLost,
4668
+ onHomeSandboxRebound,
4669
+ },
4670
+ // Thread the SAME declared environment the group box was created with
4671
+ // (resumeBoxForTurn, above) so a selfhosted swap target's manifest
4672
+ // carries it too — the SDK's per-turn manifest-env delta stays empty
4673
+ // (no "cannot change manifest environment variables" throw).
4674
+ {
4675
+ workspaceId: input.workspaceId,
4676
+ sessionId: input.sessionId,
4677
+ environment: sandboxEnvironment,
4678
+ workspaceMutationFence: {
4679
+ accountId: input.accountId,
4680
+ turnId: turn.id,
4681
+ executionGeneration,
4682
+ attemptId: input.attemptId,
4430
4683
  },
4431
- resolvedSandbox.established,
4432
- ),
4433
- };
4434
- }
4684
+ homeLease: {
4685
+ accountId: input.accountId,
4686
+ sandboxGroupId: session.sandboxGroupId,
4687
+ leaseEpoch: resolvedSandbox.leaseEpoch,
4688
+ instanceId: resolvedSandbox.established.instanceId,
4689
+ backend: groupBoxBackend,
4690
+ },
4691
+ },
4692
+ resolvedSandbox.established,
4693
+ ),
4694
+ };
4435
4695
  }
4436
4696
  if (resolvedSandbox) {
4437
4697
  startLeaseHeartbeat(resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
@@ -4741,6 +5001,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4741
5001
  await publishSandboxLifecycleEvents(provisioned);
4742
5002
  await attachRunCredentialRenewal(
4743
5003
  provisioned.established.session as RunCredentialCommandSession,
5004
+ provisioned,
4744
5005
  );
4745
5006
  const provisionedSetupSession = initialRunCredentialMaterial
4746
5007
  ? withRunCredentialsSession(
@@ -4748,39 +5009,46 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4748
5009
  input.sessionId,
4749
5010
  )
4750
5011
  : provisioned.established.session;
4751
- await runOwnedSandboxSetup(
4752
- agent,
4753
- provisioned.established.session as never,
4754
- provisionedSetupSession as never,
4755
- {
4756
- settings: runSettings,
4757
- environment: sandboxEnvironment,
4758
- onRuntimeEvent: async (event) => {
4759
- await publish?.([{ type: event.type, payload: event.payload }], true);
4760
- },
4761
- ...(lazyGitTokens ? { gitTokenSeedsOverride: lazyGitTokens } : {}),
4762
- ...(lazyGitCredentials?.bindings
4763
- ? { gitCredentialBindingsOverride: lazyGitCredentials.bindings }
4764
- : {}),
4765
- ...(lazyToolspaceToken
4766
- ? { toolspaceTokenSeedOverride: lazyToolspaceToken.token }
4767
- : {}),
4768
- ...(toolCancellationFenceRef.current
4769
- ? {
4770
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4771
- toolCancellationFenceRef.current,
4772
- ),
4773
- }
4774
- : {}),
4775
- },
5012
+ await runWorkspaceMutationForSandbox(
5013
+ provisioned,
5014
+ "lazyOwnedSandboxSetup",
5015
+ async () =>
5016
+ await runOwnedSandboxSetup(
5017
+ agent,
5018
+ provisioned.established.session as never,
5019
+ provisionedSetupSession as never,
5020
+ {
5021
+ settings: runSettings,
5022
+ environment: sandboxEnvironment,
5023
+ onRuntimeEvent: async (event) => {
5024
+ await publish?.([{ type: event.type, payload: event.payload }], true);
5025
+ },
5026
+ ...(lazyGitTokens ? { gitTokenSeedsOverride: lazyGitTokens } : {}),
5027
+ ...(lazyGitCredentials?.bindings
5028
+ ? { gitCredentialBindingsOverride: lazyGitCredentials.bindings }
5029
+ : {}),
5030
+ ...(lazyToolspaceToken
5031
+ ? { toolspaceTokenSeedOverride: lazyToolspaceToken.token }
5032
+ : {}),
5033
+ ...(toolCancellationFenceRef.current
5034
+ ? {
5035
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
5036
+ toolCancellationFenceRef.current,
5037
+ ),
5038
+ }
5039
+ : {}),
5040
+ },
5041
+ ),
4776
5042
  );
4777
5043
  await attachToolspaceTokenRenewal(
4778
5044
  provisioned.established.session as ToolspaceTokenWriterSession,
4779
5045
  lazyToolspaceToken?.expiresAt,
5046
+ provisioned,
4780
5047
  );
4781
5048
  await attachGitCredentialRenewal(
4782
5049
  provisioned.established.session as GitCredentialTokenWriterSession,
4783
5050
  lazyGitCredentials,
5051
+ provisioned,
4784
5052
  );
4785
5053
  // Return the REAL established box (NOT a copy whose session is the routing
4786
5054
  // proxy). resolveActiveBackend dispatches ops to `provisioned.established.session`;
@@ -4850,17 +5118,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4850
5118
  settings,
4851
5119
  bus,
4852
5120
  onOp: machineOpObserver.observer,
5121
+ onHomeSandboxLost: publishSandboxLost,
5122
+ onHomeSandboxRebound,
4853
5123
  },
4854
5124
  {
4855
5125
  workspaceId: input.workspaceId,
4856
5126
  sessionId: input.sessionId,
4857
5127
  environment: sandboxEnvironment,
5128
+ workspaceMutationFence: {
5129
+ accountId: input.accountId,
5130
+ turnId: turn.id,
5131
+ executionGeneration,
5132
+ attemptId: input.attemptId,
5133
+ },
4858
5134
  },
4859
5135
  {
4860
5136
  client: lazyClient,
4861
5137
  backendId: sdkBackendIdForSandboxBackend(groupBoxBackend),
4862
5138
  agentDefaultManifest,
4863
5139
  provisioner: turnSandboxProvisioner,
5140
+ homeLeaseIdentity: {
5141
+ accountId: input.accountId,
5142
+ sandboxGroupId: session.sandboxGroupId,
5143
+ backend: groupBoxBackend,
5144
+ },
4864
5145
  },
4865
5146
  );
4866
5147
  }
@@ -4982,22 +5263,27 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4982
5263
  );
4983
5264
  const runAs = sandboxRunAs(runSettings);
4984
5265
  if (downloadsToMaterialize.length > 0) {
4985
- const materialized = await materializeSandboxFileDownloads(
4986
- setupBoxSession as any,
4987
- downloadsToMaterialize,
4988
- {
4989
- onRuntimeEvent: async (event) => {
4990
- await publish!([{ type: event.type, payload: event.payload }], true);
4991
- },
4992
- ...(runAs ? { runAs } : {}),
4993
- ...(toolCancellationFenceRef.current
4994
- ? {
4995
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4996
- toolCancellationFenceRef.current,
4997
- ),
4998
- }
4999
- : {}),
5000
- },
5266
+ const materialized = await runWorkspaceMutationForSandbox(
5267
+ resolvedSandbox,
5268
+ "fileMaterialization",
5269
+ async () =>
5270
+ await materializeSandboxFileDownloads(
5271
+ setupBoxSession as any,
5272
+ downloadsToMaterialize,
5273
+ {
5274
+ onRuntimeEvent: async (event) => {
5275
+ await publish!([{ type: event.type, payload: event.payload }], true);
5276
+ },
5277
+ ...(runAs ? { runAs } : {}),
5278
+ ...(toolCancellationFenceRef.current
5279
+ ? {
5280
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
5281
+ toolCancellationFenceRef.current,
5282
+ ),
5283
+ }
5284
+ : {}),
5285
+ },
5286
+ ),
5001
5287
  );
5002
5288
  fileMaterializationFailures = materialized.failures;
5003
5289
  const failedFileIds = new Set(materialized.failures.map((failure) => failure.fileId));
@@ -5045,6 +5331,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5045
5331
  recovering: turn.executionGeneration > 1,
5046
5332
  ...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
5047
5333
  ...(runCredentialsNote ? { runCredentialsNote } : {}),
5334
+ ...(modelAcceptsTypedAttachmentContentForTurn(resolvedModel) && objectStorage
5335
+ ? {
5336
+ readFileBytesForModel: (file) => objectStorage.getFileBytes(file),
5337
+ }
5338
+ : {}),
5048
5339
  },
5049
5340
  );
5050
5341
  runInput = prepared.input;
@@ -5159,8 +5450,50 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5159
5450
  throwIfWorkerShuttingDown();
5160
5451
  throwIfTurnCancelled();
5161
5452
  const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
5162
- const runStreamOnce = (): ReturnType<OpenGeniRuntime["runStream"]> =>
5163
- runtime.runStream(agent, runInput!, modelRunSettings, {
5453
+ const runStreamOnce = async (): ReturnType<OpenGeniRuntime["runStream"]> => {
5454
+ // Eager owned sessions must settle the exact platform-setup provider
5455
+ // promise before the long model stream starts; otherwise one admission
5456
+ // would remain in flight for the entire turn and suppress every
5457
+ // heartbeat capture. Lazy setup already runs under the same wrapper in
5458
+ // its first-operation provisioner above.
5459
+ if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
5460
+ const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
5461
+ await runWorkspaceMutationForSandbox(
5462
+ resolvedSandbox,
5463
+ "eagerOwnedSandboxSetup",
5464
+ async () =>
5465
+ await runOwnedSandboxSetup(
5466
+ agent,
5467
+ ownedEstablished.session as never,
5468
+ eagerSetupSession as never,
5469
+ {
5470
+ settings: runSettings,
5471
+ environment: sandboxEnvironment,
5472
+ preparedInput: runInput!,
5473
+ ...(fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {}),
5474
+ onRuntimeEvent: async (event) => {
5475
+ await renewCodexLease("runtime_event");
5476
+ if (codexLeaseLost) {
5477
+ throw new Error("Codex credential lease expired during sandbox setup");
5478
+ }
5479
+ await publish!([{ type: event.type, payload: event.payload }], true);
5480
+ },
5481
+ ...(toolCancellationFenceRef.current
5482
+ ? {
5483
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
5484
+ toolCancellationFenceRef.current,
5485
+ ),
5486
+ }
5487
+ : {}),
5488
+ },
5489
+ ),
5490
+ );
5491
+ await attachGitCredentialRenewal(
5492
+ eagerSetupSession as GitCredentialTokenWriterSession,
5493
+ initialGitCredentials,
5494
+ );
5495
+ }
5496
+ return await runtime.runStream(agent, runInput!, modelRunSettings, {
5164
5497
  ...(activityContext ? { signal: activityContext.cancellationSignal } : {}),
5165
5498
  sandboxEnvironment,
5166
5499
  onRuntimeEvent: async (event) => {
@@ -5187,16 +5520,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5187
5520
  // re-route those execs onto a machine swapped in mid-turn.
5188
5521
  ...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
5189
5522
  ...(fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {}),
5190
- ...(lazyOwnedSandbox ? { deferredSetup: true } : {}),
5191
- },
5192
- }
5193
- : {}),
5194
- ...(initialGitCredentials
5195
- ? {
5196
- onGitCredentialSessionReady: async (
5197
- tokenSession: GitCredentialTokenWriterSession,
5198
- ) => {
5199
- await attachGitCredentialRenewal(tokenSession, initialGitCredentials);
5523
+ // Both owned paths execute setup outside runStream: eager just
5524
+ // above under its exact admission, lazy in the provisioner.
5525
+ deferredSetup: true,
5200
5526
  },
5201
5527
  }
5202
5528
  : {}),
@@ -5237,6 +5563,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5237
5563
  ? { turnToolCancellationFence: toolCancellationFenceRef.current }
5238
5564
  : {}),
5239
5565
  });
5566
+ };
5240
5567
  if (codexLeaseLost) {
5241
5568
  throw new Error("Codex credential lease expired before the model run");
5242
5569
  }
@@ -5333,6 +5660,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5333
5660
  turnId: activeTurnId,
5334
5661
  executionGeneration,
5335
5662
  attemptId: input.attemptId,
5663
+ modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
5336
5664
  ...pendingToolCall,
5337
5665
  });
5338
5666
  if (!registered.accepted) {
@@ -5357,6 +5685,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5357
5685
  executionGeneration,
5358
5686
  attemptId: input.attemptId,
5359
5687
  callId: completedToolCall.callId,
5688
+ modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
5360
5689
  resultItem: completedToolCall.resultItem,
5361
5690
  });
5362
5691
  if (!recorded.accepted) {
@@ -6668,10 +6997,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6668
6997
  const finalizerSignal = turnFinalizerCancellationSignal(cancellationSignal, activityStatus);
6669
6998
  try {
6670
6999
  const toolCancellationFence = toolCancellationFenceRef.current;
6671
- // Toolspace and run-credential renewals are attempt-owned sandbox
6672
- // writers. Capture and close them before crossing the same authoritative
6673
- // boundary as tool calls; none of these drains may race the already-
6674
- // aborted Temporal signal.
7000
+ // Every renewal controller is an attempt-owned sandbox writer. Capture
7001
+ // and close all of them before the tool/run-writer drain and before the
7002
+ // quiescence receipt; none may start another admitted write afterward.
7003
+ gitCredentialRenewalClosed = true;
7004
+ const gitRenewalsToStop = gitCredentialRenewals;
7005
+ gitCredentialRenewals = [];
6675
7006
  toolspaceTokenRenewalClosed = true;
6676
7007
  const toolspaceRenewalToStop =
6677
7008
  toolspaceTokenRenewal as ToolspaceTokenRenewalController | null;
@@ -6679,9 +7010,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6679
7010
  runCredentialRenewalClosed = true;
6680
7011
  const runRenewalToStop = runCredentialRenewal as RunCredentialRenewalController | null;
6681
7012
  runCredentialRenewal = null;
7013
+
7014
+ // Attempt-qualified credential deletion is also a real workspace write.
7015
+ // Perform it under the same admission fence before publishing physical
7016
+ // quiescence; a failure deliberately keeps the receipt closed.
7017
+ const credentialSessionToClear = runCredentialSession;
7018
+ runCredentialSession = null;
7019
+ if (credentialSessionToClear) {
7020
+ await runWorkspaceMutationForSandbox(
7021
+ requireResolvedSandboxForMutation(
7022
+ "Run credential cleanup has no exact sandbox lease target",
7023
+ ),
7024
+ "runCredentialAttemptClear",
7025
+ async () =>
7026
+ await clearRunCredentialsForAttempt(credentialSessionToClear, {
7027
+ sessionId: input.sessionId,
7028
+ attemptId: input.attemptId,
7029
+ executionGeneration,
7030
+ }),
7031
+ );
7032
+ }
6682
7033
  await drainAttemptOwnedSandboxWriters({
6683
7034
  toolCancellationFence: acknowledgeQuiescence ? toolCancellationFence : null,
6684
7035
  cancellationReason: cancellationSignal?.reason,
7036
+ gitCredentialRenewals: gitRenewalsToStop,
6685
7037
  toolspaceTokenRenewal: toolspaceRenewalToStop,
6686
7038
  runCredentialRenewal: runRenewalToStop,
6687
7039
  });
@@ -6761,43 +7113,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6761
7113
  });
6762
7114
  }
6763
7115
  }
6764
- // Deleting this attempt's credential generation is post-receipt
6765
- // housekeeping: pointer mutation is serialized with successor
6766
- // activation and the attempt-qualified check cannot erase a successor.
6767
- // It may therefore detach on cancellation without weakening the writer
6768
- // drain that gates the receipt above.
6769
- const credentialSessionToClear = runCredentialSession;
6770
- runCredentialSession = null;
6771
- if (credentialSessionToClear) {
6772
- const cleanup = clearRunCredentialsForAttempt(credentialSessionToClear, {
6773
- sessionId: input.sessionId,
6774
- attemptId: input.attemptId,
6775
- executionGeneration,
6776
- }).catch((error: unknown) => {
6777
- try {
6778
- observability.incrementCounter({
6779
- name: "opengeni_run_credential_cleanup_total",
6780
- help: "Attempt-owned run credential cleanup outcomes.",
6781
- labels: { outcome: "error" },
6782
- });
6783
- observability.warn("Attempt-owned run credential cleanup failed", {
6784
- sessionId: input.sessionId,
6785
- turnId,
6786
- attemptId: input.attemptId,
6787
- errorClass: error instanceof Error ? error.name : "UnknownError",
6788
- });
6789
- } catch {
6790
- // Cleanup observability must not own finalizer liveness.
6791
- }
6792
- });
6793
- await waitForTurnFinalizerStep(cleanup, finalizerSignal);
6794
- }
6795
- gitCredentialRenewalClosed = true;
6796
- const renewalsToStop = gitCredentialRenewals;
6797
- gitCredentialRenewals = [];
6798
- for (const renewal of renewalsToStop) {
6799
- await waitForTurnFinalizerStep(renewal.stop(), finalizerSignal);
6800
- }
6801
7116
  // Drain the buffered Connected Machine op events (infra failures + healed
6802
7117
  // recoveries) to durable session events — awaited, best-effort, never blocking
6803
7118
  // the turn. Sync observer → buffer → single awaited append here (no unawaited