@opengeni/worker-bundle 0.20.0 → 0.20.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -37,7 +37,7 @@ import {
37
37
  withFirstPartyTools,
38
38
  wrapLazyTurnBoxWithRouting,
39
39
  wrapTurnBoxWithRouting
40
- } from "./chunk-5GRJIP5H.js";
40
+ } from "./chunk-P6Z2SLAA.js";
41
41
  import {
42
42
  StreamTimingMetrics,
43
43
  makeMachineOpObserver,
@@ -51,9 +51,12 @@ import {
51
51
  recordModelRequestPhase,
52
52
  recordSessionEventAppendLatency,
53
53
  recordSessionEventPublishLatency,
54
+ recordTurnSandboxEstablishPolicy,
55
+ recordTurnStartupPhase,
56
+ recordTurnWorkerPreparationTotal,
54
57
  runtimeMetricsHooksForObservability,
55
58
  turnLifecycleMetricsFor
56
- } from "./chunk-TYNPCG4U.js";
59
+ } from "./chunk-N5TJIFCE.js";
57
60
 
58
61
  // src/activities-turn.ts
59
62
  import { createProductionAgentRuntime, summarizeForCompaction as summarizeForCompaction2 } from "@opengeni/runtime";
@@ -148,6 +151,7 @@ import {
148
151
  SandboxImageConflictError,
149
152
  ActiveSessionHistoryLimitExceededError,
150
153
  ApprovalRunStateLimitExceededError,
154
+ isRetryableDatabaseTransportFailure,
151
155
  isSessionEventPersistenceError,
152
156
  getEnrollment,
153
157
  abandonRecordingForTurnAttempt,
@@ -2938,6 +2942,25 @@ function resumeRunState(state) {
2938
2942
  }
2939
2943
  return projectRejectedProviderArtifactsFromSerializedRunState(state.serializedRunState);
2940
2944
  }
2945
+ async function measureHistoryPreparationPhase(options, phase, operation) {
2946
+ const startedAt = performance.now();
2947
+ let outcome = "completed";
2948
+ try {
2949
+ return await operation();
2950
+ } catch (error) {
2951
+ outcome = "failed";
2952
+ throw error;
2953
+ } finally {
2954
+ try {
2955
+ options.onPreparationPhase?.({
2956
+ phase,
2957
+ outcome,
2958
+ durationSeconds: (performance.now() - startedAt) / 1e3
2959
+ });
2960
+ } catch {
2961
+ }
2962
+ }
2963
+ }
2941
2964
  var MAX_INLINE_MODEL_ATTACHMENT_BYTES = 20 * 1024 * 1024;
2942
2965
  var MODEL_IMAGE_CONTENT_TYPES = /* @__PURE__ */ new Set(["image/gif", "image/jpeg", "image/png", "image/webp"]);
2943
2966
  var MODEL_FILE_CONTENT_TYPES = /* @__PURE__ */ new Set([
@@ -3132,11 +3155,15 @@ async function turnInput(db, runtime, agent, trigger, options) {
3132
3155
  if (!trigger) {
3133
3156
  throw new Error("Missing trigger event");
3134
3157
  }
3135
- const updates = await listSessionSystemUpdatesForTurn(
3136
- db,
3137
- trigger.workspaceId,
3138
- trigger.sessionId,
3139
- options.turnId
3158
+ const updates = await measureHistoryPreparationPhase(
3159
+ options,
3160
+ "system_update_load",
3161
+ async () => await listSessionSystemUpdatesForTurn(
3162
+ db,
3163
+ trigger.workspaceId,
3164
+ trigger.sessionId,
3165
+ options.turnId
3166
+ )
3140
3167
  );
3141
3168
  if (updates.length > 0) {
3142
3169
  const historyItemIds = new Set(
@@ -3161,10 +3188,10 @@ async function turnInput(db, runtime, agent, trigger, options) {
3161
3188
  throw new Error("user.message payload is missing text and annotations");
3162
3189
  }
3163
3190
  const resources = Array.isArray(payload.resources) ? payload.resources : [];
3164
- const fileAttachments = await resolveUserMessageFileAttachments(
3165
- db,
3166
- trigger.workspaceId,
3167
- resources
3191
+ const fileAttachments = await measureHistoryPreparationPhase(
3192
+ options,
3193
+ "current_attachment_resolution",
3194
+ async () => await resolveUserMessageFileAttachments(db, trigger.workspaceId, resources)
3168
3195
  );
3169
3196
  const attachmentContext = userMessageAttachmentsContext(fileAttachments);
3170
3197
  return await messageInput(
@@ -3179,7 +3206,8 @@ async function turnInput(db, runtime, agent, trigger, options) {
3179
3206
  options.projectCanonicalHistory,
3180
3207
  options.materializeModelHistory,
3181
3208
  options.projectModelHistory,
3182
- options.loadActiveHistory
3209
+ options.loadActiveHistory,
3210
+ options
3183
3211
  );
3184
3212
  }
3185
3213
  if (trigger.type === "system.update.delivered") {
@@ -3198,7 +3226,8 @@ async function turnInput(db, runtime, agent, trigger, options) {
3198
3226
  options.projectCanonicalHistory,
3199
3227
  options.materializeModelHistory,
3200
3228
  options.projectModelHistory,
3201
- options.loadActiveHistory
3229
+ options.loadActiveHistory,
3230
+ options
3202
3231
  );
3203
3232
  }
3204
3233
  if (trigger.type === "user.approvalDecision") {
@@ -3258,33 +3287,76 @@ function joinInternalContext(...parts) {
3258
3287
  const content = parts.map((part) => part?.trim()).filter((part) => Boolean(part));
3259
3288
  return content.length > 0 ? content.join("\n\n") : void 0;
3260
3289
  }
3261
- async function messageInput(db, runtime, agent, trigger, text, internalContext, currentAttachmentRefs = [], providerApi = "responses", projectCanonicalHistory, materializeModelHistory, projectModelHistory, loadActiveHistory = getActiveSessionHistoryItemsPaged) {
3262
- const stored = await loadActiveHistory(db, trigger.workspaceId, trigger.sessionId);
3263
- const envelope = await getSandboxSessionEnvelope(db, trigger.workspaceId, trigger.sessionId);
3264
- const canonicalView = projectRejectedProviderArtifacts(stored);
3265
- const canonicalProviderView = projectCanonicalHistory ? await projectCanonicalHistory(canonicalView) : canonicalView;
3266
- const providerView = projectHistoryForProvider(canonicalProviderView, providerApi);
3267
- const referencedHistory = withCurrentUserAttachmentRefs(providerView, currentAttachmentRefs);
3268
- const materializedHistory = materializeModelHistory ? await materializeModelHistory(referencedHistory) : referencedHistory;
3269
- const historyItems = projectModelHistory ? await projectModelHistory(materializedHistory) : materializedHistory;
3270
- const prepared = await runtime.prepareInput(agent, {
3271
- kind: "message",
3272
- ...text ? { text } : {},
3273
- ...internalContext ? { internalContext } : {},
3274
- historyItems,
3275
- sandboxEnvelope: envelope,
3276
- ...projectModelHistory ? { modelInputAlreadyProjected: true } : {}
3277
- });
3278
- const preparedItems = Array.isArray(prepared.input) ? new Set(prepared.input) : /* @__PURE__ */ new Set();
3290
+ async function messageInput(db, runtime, agent, trigger, text, internalContext, currentAttachmentRefs = [], providerApi = "responses", projectCanonicalHistory, materializeModelHistory, projectModelHistory, loadActiveHistory = getActiveSessionHistoryItemsPaged, preparationOptions = {}) {
3291
+ const [stored, envelope] = await Promise.all([
3292
+ measureHistoryPreparationPhase(
3293
+ preparationOptions,
3294
+ "durable_history_load",
3295
+ async () => loadActiveHistory(db, trigger.workspaceId, trigger.sessionId)
3296
+ ),
3297
+ measureHistoryPreparationPhase(
3298
+ preparationOptions,
3299
+ "sandbox_envelope_load",
3300
+ async () => getSandboxSessionEnvelope(db, trigger.workspaceId, trigger.sessionId)
3301
+ )
3302
+ ]);
3303
+ const canonicalView = await measureHistoryPreparationPhase(
3304
+ preparationOptions,
3305
+ "canonical_projection",
3306
+ async () => {
3307
+ const active = projectRejectedProviderArtifacts(stored);
3308
+ return projectCanonicalHistory ? await projectCanonicalHistory(active) : active;
3309
+ }
3310
+ );
3311
+ const providerView = await measureHistoryPreparationPhase(
3312
+ preparationOptions,
3313
+ "provider_projection",
3314
+ () => projectHistoryForProvider(canonicalView, providerApi)
3315
+ );
3316
+ const referencedHistory = await measureHistoryPreparationPhase(
3317
+ preparationOptions,
3318
+ "attachment_ref_projection",
3319
+ () => withCurrentUserAttachmentRefs(providerView, currentAttachmentRefs)
3320
+ );
3321
+ const materializedHistory = await measureHistoryPreparationPhase(
3322
+ preparationOptions,
3323
+ "screenshot_materialization",
3324
+ async () => materializeModelHistory ? await materializeModelHistory(referencedHistory) : referencedHistory
3325
+ );
3326
+ const historyItems = await measureHistoryPreparationPhase(
3327
+ preparationOptions,
3328
+ "model_attachment_projection",
3329
+ async () => projectModelHistory ? await projectModelHistory(materializedHistory) : materializedHistory
3330
+ );
3331
+ const prepared = await measureHistoryPreparationPhase(
3332
+ preparationOptions,
3333
+ "runtime_input_assembly",
3334
+ async () => await runtime.prepareInput(agent, {
3335
+ kind: "message",
3336
+ ...text ? { text } : {},
3337
+ ...internalContext ? { internalContext } : {},
3338
+ historyItems,
3339
+ sandboxEnvelope: envelope,
3340
+ ...projectModelHistory ? { modelInputAlreadyProjected: true } : {}
3341
+ })
3342
+ );
3343
+ const providerArtifactCandidates = await measureHistoryPreparationPhase(
3344
+ preparationOptions,
3345
+ "artifact_candidate_scan",
3346
+ () => {
3347
+ const preparedItems = Array.isArray(prepared.input) ? new Set(prepared.input) : /* @__PURE__ */ new Set();
3348
+ return {
3349
+ knownHistoryItemIds: stored.map((row) => row.id),
3350
+ historyItemIds: stored.filter(
3351
+ (row) => row.providerArtifactInvalidatedAt === null && hasOpaqueProviderArtifact(row.item) && preparedItems.has(row.item)
3352
+ ).map((row) => row.id)
3353
+ };
3354
+ }
3355
+ );
3279
3356
  return {
3280
3357
  input: prepared,
3281
3358
  persistedHistoryCount: prepared.persistedHistoryCount,
3282
- providerArtifactCandidates: {
3283
- knownHistoryItemIds: stored.map((row) => row.id),
3284
- historyItemIds: stored.filter(
3285
- (row) => row.providerArtifactInvalidatedAt === null && hasOpaqueProviderArtifact(row.item) && preparedItems.has(row.item)
3286
- ).map((row) => row.id)
3287
- }
3359
+ providerArtifactCandidates
3288
3360
  };
3289
3361
  }
3290
3362
  async function resolveUserMessageFileAttachments(db, workspaceId, resources) {
@@ -5241,6 +5313,8 @@ async function bindRunCredentialResolver(input) {
5241
5313
  }
5242
5314
 
5243
5315
  // src/activities/types.ts
5316
+ var PRE_CLAIM_FAILURE_TYPE = "OpenGeniPreClaimFailure";
5317
+ var PRE_CLAIM_FAILURE_MESSAGE = "Agent turn admission failed before attempt claim.";
5244
5318
  var ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE = "EscapedMcpTimeoutRecoveryFailure";
5245
5319
  var ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE = "MCP request timeout recovery checkpoint failed before model request";
5246
5320
 
@@ -6756,12 +6830,33 @@ function escapedMcpTimeoutRecoveryFailure(input) {
6756
6830
  details: [input.detail]
6757
6831
  });
6758
6832
  }
6833
+ function preClaimAdmissionFailure(error) {
6834
+ const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
6835
+ const sqlState = persistenceFailure?.details.sqlState ?? null;
6836
+ const rawTransportFailure = sqlState === null && isRetryableDatabaseTransportFailure(error);
6837
+ const retryable = Boolean(
6838
+ rawTransportFailure || persistenceFailure && (sqlState === null || sqlState.startsWith("08") || sqlState.startsWith("40") || sqlState.startsWith("53") || sqlState === "55P03" || sqlState === "57014" || sqlState === "57P01" || sqlState === "57P02" || sqlState === "57P03" || sqlState.startsWith("58"))
6839
+ );
6840
+ const detail = {
6841
+ disposition: retryable ? "retryable" : "permanent",
6842
+ code: persistenceFailure?.details.code ?? (rawTransportFailure ? "db_failure" : "claim_invariant")
6843
+ };
6844
+ return ApplicationFailure.create({
6845
+ message: PRE_CLAIM_FAILURE_MESSAGE,
6846
+ type: PRE_CLAIM_FAILURE_TYPE,
6847
+ nonRetryable: true,
6848
+ details: [detail]
6849
+ });
6850
+ }
6759
6851
  function filterUnmaterializedSandboxFileDownloads(downloads, materializedFileIds) {
6760
6852
  if (downloads.length === 0 || materializedFileIds.size === 0) {
6761
6853
  return downloads;
6762
6854
  }
6763
6855
  return downloads.filter((download) => !materializedFileIds.has(download.fileId));
6764
6856
  }
6857
+ function sandboxFileMaterializationOutcome(failures) {
6858
+ return failures.length === 0 ? "completed" : "failed";
6859
+ }
6765
6860
  function codexWorkspaceMetricKey(workspaceId) {
6766
6861
  return createHash8("sha256").update(workspaceId).digest("hex").slice(0, 12);
6767
6862
  }
@@ -7432,6 +7527,21 @@ function managedSandboxOwnershipForTurn(machinePrimary, attemptId, sandboxGroupI
7432
7527
  function shouldEstablishSandboxForTurn(sandboxOwnershipEnabled, homeBackend, machinePrimary) {
7433
7528
  return sandboxOwnershipEnabled && (homeBackend !== "none" || machinePrimary);
7434
7529
  }
7530
+ function sandboxEstablishPolicyDecision(input) {
7531
+ if (!input.lazyEnabled) return { policy: "eager", reason: "lazy_disabled" };
7532
+ if (input.machinePrimary) return { policy: "eager", reason: "machine_primary" };
7533
+ if (input.sandboxBackend === "none") return { policy: "eager", reason: "backend_none" };
7534
+ if (input.hasInitialRunCredentialMaterial) {
7535
+ return { policy: "eager", reason: "initial_run_credentials" };
7536
+ }
7537
+ if (input.generatedVideoFileCount > 0) {
7538
+ return { policy: "eager", reason: "generated_video_files" };
7539
+ }
7540
+ if (input.hasSignedFileResources) {
7541
+ return { policy: "eager", reason: "signed_file_resources" };
7542
+ }
7543
+ return { policy: "on-demand", reason: "eligible" };
7544
+ }
7435
7545
  function pointerReconcileReason(record4) {
7436
7546
  if (!record4) {
7437
7547
  return "stale_pointer";
@@ -8760,7 +8870,6 @@ function createRunAgentTurnActivity(services) {
8760
8870
  let lastCodexRequestOpaqueArtifacts = [];
8761
8871
  let triggerType = null;
8762
8872
  try {
8763
- const session = await requireSession(db, input.workspaceId, input.sessionId);
8764
8873
  const claim = await claimSessionWorkForAttempt(db, input.workspaceId, {
8765
8874
  sessionId: input.sessionId,
8766
8875
  workflowId: input.workflowId,
@@ -8775,6 +8884,7 @@ function createRunAgentTurnActivity(services) {
8775
8884
  }
8776
8885
  const turn = claim.turn;
8777
8886
  turnId = turn.id;
8887
+ const session = await requireSession(db, input.workspaceId, input.sessionId);
8778
8888
  executionGeneration = turn.executionGeneration;
8779
8889
  providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
8780
8890
  let installedApiIntegrations = [];
@@ -9031,6 +9141,14 @@ function createRunAgentTurnActivity(services) {
9031
9141
  });
9032
9142
  throwIfWorkerShuttingDown();
9033
9143
  throwIfTurnCancelled();
9144
+ recordTurnStartupPhase(observability, {
9145
+ phase: "claim_and_policy",
9146
+ provider: turnExecutionPolicy.providerId,
9147
+ backend: turn.sandboxBackend,
9148
+ outcome: "completed",
9149
+ durationSeconds: (performance.now() - activityStarted) / 1e3
9150
+ });
9151
+ const turnStartSettlementStartedAt = performance.now();
9034
9152
  if (!await settle({
9035
9153
  events: [
9036
9154
  { type: "session.status.changed", payload: { status: "running" } },
@@ -9045,76 +9163,34 @@ function createRunAgentTurnActivity(services) {
9045
9163
  })) {
9046
9164
  return claimedResult({ status: "cancelled" });
9047
9165
  }
9166
+ recordTurnStartupPhase(observability, {
9167
+ phase: "turn_start_settlement",
9168
+ provider: turnExecutionPolicy.providerId,
9169
+ backend: turn.sandboxBackend,
9170
+ outcome: "completed",
9171
+ durationSeconds: (performance.now() - turnStartSettlementStartedAt) / 1e3,
9172
+ count: 2
9173
+ });
9048
9174
  turnStartedPublished = true;
9175
+ const workerPreparationStartedAt = performance.now();
9049
9176
  if (isCodexTurn) {
9050
- const sessionCodex = await getSessionCodexState(db, input.workspaceId, input.sessionId);
9051
- const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
9052
- const sessionPinSource = sessionCodex?.pinSource ?? null;
9053
- const selectForTurn = (context) => selectCodexCredentialLeaseForTurn({
9054
- context,
9055
- leasingEnabled: settings.codexCredentialLeasingEnabled,
9056
- sessionId: input.sessionId,
9057
- sessionPinnedCredentialId: sessionPin,
9058
- sessionPinSource,
9059
- sessionLastCredentialId: sessionCodex?.lastCredentialId ?? null,
9060
- now: /* @__PURE__ */ new Date()
9061
- });
9062
- let leased;
9063
- let leaseAcquisitionStartedAtMs = null;
9064
- if (settings.codexCredentialLeasingEnabled) {
9065
- leaseAcquisitionStartedAtMs = performance.now();
9066
- leased = await acquireCodexCredentialLease(
9067
- db,
9068
- {
9069
- accountId: input.accountId,
9070
- workspaceId: input.workspaceId,
9071
- turnId,
9072
- holderId: codexLeaseHolderId,
9073
- advanceActivePointer: sessionPin === null
9074
- },
9075
- selectForTurn
9076
- );
9077
- } else {
9078
- const [rotation, accounts] = await Promise.all([
9079
- getCodexRotationSettings(db, input.workspaceId),
9080
- listCodexAccountStatuses(db, input.workspaceId)
9081
- ]);
9082
- const leaseAccounts = accounts.map((account) => ({
9083
- ...account,
9084
- activeLeaseCount: 0,
9085
- selectionCount: 0,
9086
- lastSelectedAt: null
9087
- }));
9088
- const activeCredentialId = rotation?.activeCredentialId ?? null;
9089
- const selected = selectForTurn({
9090
- accounts: leaseAccounts,
9091
- activeCredentialId,
9092
- rotationEnabled: rotation?.rotationEnabled ?? false,
9093
- leaseRotationEnabled: false,
9094
- rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
9095
- existingCredentialId: null,
9096
- policyScope: null,
9097
- unavailableDiagnostics: []
9098
- });
9099
- leased = {
9100
- ...selected,
9101
- accounts: leaseAccounts,
9102
- activeCredentialId,
9103
- rotationEnabled: rotation?.rotationEnabled ?? false,
9104
- rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
9105
- reused: false,
9106
- holderId: null,
9107
- generation: null,
9108
- leasedUntil: null,
9109
- unavailableDiagnostics: [],
9110
- advanceActivePointer: selected.advanceActivePointer !== false
9111
- };
9112
- }
9113
- if (leased.decision.kind === "allCapped") {
9114
- await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
9115
- signalCodexCapacityWorkflow,
9116
- wakeSessionWorkflow
9177
+ const credentialSelectionStartedAt = performance.now();
9178
+ let credentialSelectionOutcome = "completed";
9179
+ try {
9180
+ const sessionCodex = await getSessionCodexState(db, input.workspaceId, input.sessionId);
9181
+ const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
9182
+ const sessionPinSource = sessionCodex?.pinSource ?? null;
9183
+ const selectForTurn = (context) => selectCodexCredentialLeaseForTurn({
9184
+ context,
9185
+ leasingEnabled: settings.codexCredentialLeasingEnabled,
9186
+ sessionId: input.sessionId,
9187
+ sessionPinnedCredentialId: sessionPin,
9188
+ sessionPinSource,
9189
+ sessionLastCredentialId: sessionCodex?.lastCredentialId ?? null,
9190
+ now: /* @__PURE__ */ new Date()
9117
9191
  });
9192
+ let leased;
9193
+ let leaseAcquisitionStartedAtMs = null;
9118
9194
  if (settings.codexCredentialLeasingEnabled) {
9119
9195
  leaseAcquisitionStartedAtMs = performance.now();
9120
9196
  leased = await acquireCodexCredentialLease(
@@ -9139,9 +9215,10 @@ function createRunAgentTurnActivity(services) {
9139
9215
  selectionCount: 0,
9140
9216
  lastSelectedAt: null
9141
9217
  }));
9218
+ const activeCredentialId = rotation?.activeCredentialId ?? null;
9142
9219
  const selected = selectForTurn({
9143
9220
  accounts: leaseAccounts,
9144
- activeCredentialId: rotation?.activeCredentialId ?? null,
9221
+ activeCredentialId,
9145
9222
  rotationEnabled: rotation?.rotationEnabled ?? false,
9146
9223
  leaseRotationEnabled: false,
9147
9224
  rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
@@ -9152,7 +9229,7 @@ function createRunAgentTurnActivity(services) {
9152
9229
  leased = {
9153
9230
  ...selected,
9154
9231
  accounts: leaseAccounts,
9155
- activeCredentialId: rotation?.activeCredentialId ?? null,
9232
+ activeCredentialId,
9156
9233
  rotationEnabled: rotation?.rotationEnabled ?? false,
9157
9234
  rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
9158
9235
  reused: false,
@@ -9163,163 +9240,389 @@ function createRunAgentTurnActivity(services) {
9163
9240
  advanceActivePointer: selected.advanceActivePointer !== false
9164
9241
  };
9165
9242
  }
9166
- }
9167
- const rotationDecision = leased.decision;
9168
- const selectedPinDisposition = classifyCodexPin({
9169
- pinnedCredentialId: sessionPin,
9170
- pinSource: sessionPinSource,
9171
- strategy: leased.rotationStrategy,
9172
- rotationEnabled: leased.rotationEnabled
9173
- });
9174
- if (selectedPinDisposition === "sharded" && leased.credentialId !== null && (sessionPinSource !== "policy" || sessionPin !== leased.credentialId)) {
9175
- const pinMutation = await withSessionCodexCapacityMutation(
9176
- db,
9177
- {
9178
- workspaceId: input.workspaceId,
9179
- reason: "codex_policy_pin_changed"
9243
+ if (leased.decision.kind === "allCapped") {
9244
+ await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
9245
+ signalCodexCapacityWorkflow,
9246
+ wakeSessionWorkflow
9247
+ });
9248
+ if (settings.codexCredentialLeasingEnabled) {
9249
+ leaseAcquisitionStartedAtMs = performance.now();
9250
+ leased = await acquireCodexCredentialLease(
9251
+ db,
9252
+ {
9253
+ accountId: input.accountId,
9254
+ workspaceId: input.workspaceId,
9255
+ turnId,
9256
+ holderId: codexLeaseHolderId,
9257
+ advanceActivePointer: sessionPin === null
9258
+ },
9259
+ selectForTurn
9260
+ );
9261
+ } else {
9262
+ const [rotation, accounts] = await Promise.all([
9263
+ getCodexRotationSettings(db, input.workspaceId),
9264
+ listCodexAccountStatuses(db, input.workspaceId)
9265
+ ]);
9266
+ const leaseAccounts = accounts.map((account) => ({
9267
+ ...account,
9268
+ activeLeaseCount: 0,
9269
+ selectionCount: 0,
9270
+ lastSelectedAt: null
9271
+ }));
9272
+ const selected = selectForTurn({
9273
+ accounts: leaseAccounts,
9274
+ activeCredentialId: rotation?.activeCredentialId ?? null,
9275
+ rotationEnabled: rotation?.rotationEnabled ?? false,
9276
+ leaseRotationEnabled: false,
9277
+ rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
9278
+ existingCredentialId: null,
9279
+ policyScope: null,
9280
+ unavailableDiagnostics: []
9281
+ });
9282
+ leased = {
9283
+ ...selected,
9284
+ accounts: leaseAccounts,
9285
+ activeCredentialId: rotation?.activeCredentialId ?? null,
9286
+ rotationEnabled: rotation?.rotationEnabled ?? false,
9287
+ rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
9288
+ reused: false,
9289
+ holderId: null,
9290
+ generation: null,
9291
+ leasedUntil: null,
9292
+ unavailableDiagnostics: [],
9293
+ advanceActivePointer: selected.advanceActivePointer !== false
9294
+ };
9295
+ }
9296
+ }
9297
+ const rotationDecision = leased.decision;
9298
+ const selectedPinDisposition = classifyCodexPin({
9299
+ pinnedCredentialId: sessionPin,
9300
+ pinSource: sessionPinSource,
9301
+ strategy: leased.rotationStrategy,
9302
+ rotationEnabled: leased.rotationEnabled
9303
+ });
9304
+ if (selectedPinDisposition === "sharded" && leased.credentialId !== null && (sessionPinSource !== "policy" || sessionPin !== leased.credentialId)) {
9305
+ const pinMutation = await withSessionCodexCapacityMutation(
9306
+ db,
9307
+ {
9308
+ workspaceId: input.workspaceId,
9309
+ reason: "codex_policy_pin_changed"
9310
+ },
9311
+ async (tx) => {
9312
+ const changed = await setSessionCodexPinInTransaction(
9313
+ tx,
9314
+ input.workspaceId,
9315
+ input.sessionId,
9316
+ leased.credentialId,
9317
+ "policy",
9318
+ {
9319
+ expected: {
9320
+ pinnedCredentialId: sessionPin,
9321
+ pinSource: sessionPinSource
9322
+ }
9323
+ }
9324
+ );
9325
+ return { result: changed, changed };
9326
+ }
9327
+ );
9328
+ await signalCodexCapacityWakeTargets(
9329
+ { signalCodexCapacityWorkflow, wakeSessionWorkflow },
9330
+ pinMutation.wakeTargets
9331
+ );
9332
+ } else if (selectedPinDisposition === "clearStale") {
9333
+ const pinMutation = await withSessionCodexCapacityMutation(
9334
+ db,
9335
+ {
9336
+ workspaceId: input.workspaceId,
9337
+ reason: "codex_stale_policy_pin_cleared"
9338
+ },
9339
+ async (tx) => {
9340
+ const changed = await setSessionCodexPinInTransaction(
9341
+ tx,
9342
+ input.workspaceId,
9343
+ input.sessionId,
9344
+ null,
9345
+ "policy",
9346
+ {
9347
+ expected: {
9348
+ pinnedCredentialId: sessionPin,
9349
+ pinSource: sessionPinSource
9350
+ }
9351
+ }
9352
+ );
9353
+ return { result: changed, changed };
9354
+ }
9355
+ );
9356
+ await signalCodexCapacityWakeTargets(
9357
+ { signalCodexCapacityWorkflow, wakeSessionWorkflow },
9358
+ pinMutation.wakeTargets
9359
+ );
9360
+ }
9361
+ if (!settings.codexCredentialLeasingEnabled && leased.advanceActivePointer && sessionPin === null && rotationDecision.kind === "active" && rotationDecision.moved) {
9362
+ await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
9363
+ }
9364
+ effectiveCodexCredentialId = leased.credentialId;
9365
+ codexLeaseGeneration = leased.generation;
9366
+ codexLeaseConfirmedUntilMs = leased.leasedUntil && leaseAcquisitionStartedAtMs !== null ? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS : null;
9367
+ codexLeaseHeld = effectiveCodexCredentialId !== null && leased.holderId !== null && leased.generation !== null && codexLeaseConfirmedUntilMs !== null;
9368
+ if (codexLeaseHeld) startCodexLeaseHeartbeat();
9369
+ const actualOutcome = effectiveCodexCredentialId ? "selected" : rotationDecision.kind === "allCapped" ? "waiting" : "none";
9370
+ const actualReason = effectiveCodexCredentialId ? leased.reused ? "lease_reused" : sessionPin === effectiveCodexCredentialId ? "pin" : rotationDecision.kind === "active" && rotationDecision.moved ? "rotation" : "active" : rotationDecision.kind === "allCapped" ? "all_capped" : "none";
9371
+ const fencedInFlight = leased.reused;
9372
+ const shadowResult = await publishCodexFleetShadowDecisionV1({
9373
+ enabled: settings.codexFleetPolicyShadowEnabled,
9374
+ decision: {
9375
+ accounts: leased.accounts,
9376
+ actualCredentialId: effectiveCodexCredentialId,
9377
+ actualOutcome,
9378
+ actualReason,
9379
+ affinityCredentialId: fencedInFlight ? effectiveCodexCredentialId : sessionPin ?? sessionCodex?.lastCredentialId ?? null,
9380
+ fencedInFlight,
9381
+ nearExhaustionPct: settings.codexRotationNearExhaustionPct,
9382
+ now: /* @__PURE__ */ new Date(),
9383
+ aliasSeed: randomUUID4()
9180
9384
  },
9181
- async (tx) => {
9182
- const changed = await setSessionCodexPinInTransaction(
9183
- tx,
9385
+ publish
9386
+ });
9387
+ if (shadowResult.outcome === "published") {
9388
+ const shadowPayload = shadowResult.payload;
9389
+ observability.incrementCounter({
9390
+ name: "opengeni_codex_fleet_shadow_decisions_total",
9391
+ help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
9392
+ labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload)
9393
+ });
9394
+ observability.info("Codex adaptive fleet shadow decision", {
9395
+ workspaceId: input.workspaceId,
9396
+ policyVersion: shadowPayload.replay.policyVersion,
9397
+ inputFingerprint: shadowPayload.replay.inputFingerprint,
9398
+ decisionFingerprint: shadowPayload.replay.decisionFingerprint,
9399
+ actualOutcome: shadowPayload.actual.outcome,
9400
+ shadowOutcome: shadowPayload.replay.decision.outcome,
9401
+ comparison: shadowPayload.comparison,
9402
+ candidateCount: shadowPayload.replay.input.candidates.length,
9403
+ truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
9404
+ payloadBytes: shadowResult.payloadBytes
9405
+ });
9406
+ } else if (shadowResult.outcome === "failed") {
9407
+ observability.incrementCounter({
9408
+ name: "opengeni_codex_fleet_shadow_errors_total",
9409
+ help: "Shadow decision build/publication failures.",
9410
+ labels: codexFleetShadowErrorMetricLabelsV1(shadowResult)
9411
+ });
9412
+ observability.warn("Codex adaptive fleet shadow decision failed open", {
9413
+ stage: shadowResult.stage,
9414
+ reason: shadowResult.reason,
9415
+ errorClass: "CodexFleetShadowOperationError",
9416
+ errorCode: "codex_fleet_shadow_failed",
9417
+ origin: "worker",
9418
+ payloadBytes: shadowResult.payloadBytes
9419
+ });
9420
+ }
9421
+ const eligibleCount = leased.accounts.filter(
9422
+ (account) => isCodexCredentialEligible(account, /* @__PURE__ */ new Date())
9423
+ ).length;
9424
+ const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
9425
+ observability.incrementCounter({
9426
+ name: "opengeni_codex_pool_observations_total",
9427
+ help: "Observed eligible Codex pool depth buckets at turn selection.",
9428
+ labels: { workspace_key: codexWorkspaceKey, depth: poolDepth }
9429
+ });
9430
+ if (eligibleCount <= 1) {
9431
+ observability.incrementCounter({
9432
+ name: "opengeni_codex_pool_low_total",
9433
+ help: "Alert signal emitted when the eligible Codex pool is zero or one.",
9434
+ labels: { workspace_key: codexWorkspaceKey, depth: poolDepth }
9435
+ });
9436
+ observability.warn("Codex eligible credential pool is low", {
9437
+ workspaceId: input.workspaceId,
9438
+ eligibleCount,
9439
+ connectedCount: leased.accounts.length,
9440
+ depth: poolDepth
9441
+ });
9442
+ }
9443
+ if (effectiveCodexCredentialId === null && leased.accounts.length > 0 && leased.accounts.every((account) => !account.allocatorEnabled) && turnId) {
9444
+ if (turn.source === "compaction") {
9445
+ if (!await settle({
9446
+ events: [
9447
+ {
9448
+ type: "turn.cancelled",
9449
+ payload: {
9450
+ maintenance: "context_compaction",
9451
+ reason: "codex_allocator_disabled",
9452
+ requestPreserved: true
9453
+ }
9454
+ },
9455
+ {
9456
+ type: "session.status.changed",
9457
+ payload: { status: "idle" }
9458
+ }
9459
+ ],
9460
+ turnStatus: "cancelled",
9461
+ sessionStatus: "idle",
9462
+ activeTurnId: null
9463
+ })) {
9464
+ return claimedResult({ status: "cancelled" });
9465
+ }
9466
+ turnMetricOutcome = "cancelled";
9467
+ activityStatus = "idle";
9468
+ return claimedResult({ status: "idle", deferredUntilWake: true });
9469
+ }
9470
+ const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
9471
+ () => null
9472
+ );
9473
+ const activeGoal = goal?.status === "active" ? goal : null;
9474
+ const armed = await armCodexCapacityWait(db, {
9475
+ accountId: input.accountId,
9476
+ workspaceId: input.workspaceId,
9477
+ sessionId: input.sessionId,
9478
+ turnId,
9479
+ attemptId: input.attemptId,
9480
+ workflowId: input.workflowId,
9481
+ goalId: activeGoal?.id ?? null,
9482
+ goalVersion: activeGoal?.version ?? null,
9483
+ earliestResetAt: null,
9484
+ resetKind: "bounded_refresh",
9485
+ failurePayload: {
9486
+ error: "All connected Codex subscriptions are disabled for new allocations.",
9487
+ code: "codex_allocator_disabled",
9488
+ detail: "waiting for a credential to be re-enabled, reconnected, or added"
9489
+ }
9490
+ });
9491
+ if (armed.action === "waiting") {
9492
+ await publishDurableSessionEvents2(
9493
+ bus,
9184
9494
  input.workspaceId,
9185
9495
  input.sessionId,
9186
- leased.credentialId,
9187
- "policy",
9496
+ armed.events
9497
+ );
9498
+ turnMetricOutcome = "recovering";
9499
+ activityStatus = "waiting_capacity";
9500
+ return claimedResult({
9501
+ status: "waiting_capacity",
9502
+ capacityWait: {
9503
+ waiterId: armed.waiter.id,
9504
+ generation: armed.waiter.generation,
9505
+ nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
9506
+ wakeRevision: armed.waiter.wakeRevision
9507
+ }
9508
+ });
9509
+ }
9510
+ if (!await settle({
9511
+ events: [
9188
9512
  {
9189
- expected: {
9190
- pinnedCredentialId: sessionPin,
9191
- pinSource: sessionPinSource
9513
+ type: "turn.failed",
9514
+ payload: {
9515
+ error: "All connected Codex subscriptions are disabled for new allocations.",
9516
+ code: "codex_allocator_disabled",
9517
+ retryable: false,
9518
+ recovery: "user_message"
9192
9519
  }
9520
+ },
9521
+ {
9522
+ type: "session.status.changed",
9523
+ payload: { status: "idle" }
9193
9524
  }
9194
- );
9195
- return { result: changed, changed };
9525
+ ],
9526
+ turnStatus: "failed",
9527
+ sessionStatus: "idle",
9528
+ activeTurnId: null
9529
+ })) {
9530
+ return claimedResult({ status: "cancelled" });
9196
9531
  }
9197
- );
9198
- await signalCodexCapacityWakeTargets(
9199
- { signalCodexCapacityWorkflow, wakeSessionWorkflow },
9200
- pinMutation.wakeTargets
9201
- );
9202
- } else if (selectedPinDisposition === "clearStale") {
9203
- const pinMutation = await withSessionCodexCapacityMutation(
9204
- db,
9205
- {
9532
+ turnMetricOutcome = "failed";
9533
+ activityStatus = "idle";
9534
+ return claimedResult({ status: "idle" });
9535
+ }
9536
+ if (rotationDecision.kind === "allCapped" && turnId) {
9537
+ if (turn.source === "compaction") {
9538
+ if (!await settle({
9539
+ events: [
9540
+ {
9541
+ type: "turn.cancelled",
9542
+ payload: {
9543
+ maintenance: "context_compaction",
9544
+ reason: "codex_capacity_unavailable",
9545
+ requestPreserved: true
9546
+ }
9547
+ },
9548
+ {
9549
+ type: "session.status.changed",
9550
+ payload: { status: "idle" }
9551
+ }
9552
+ ],
9553
+ turnStatus: "cancelled",
9554
+ sessionStatus: "idle",
9555
+ activeTurnId: null
9556
+ })) {
9557
+ return claimedResult({ status: "cancelled" });
9558
+ }
9559
+ turnMetricOutcome = "cancelled";
9560
+ activityStatus = "idle";
9561
+ return claimedResult({ status: "idle", deferredUntilWake: true });
9562
+ }
9563
+ const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
9564
+ () => null
9565
+ );
9566
+ const goalActive = Boolean(goal && goal.status === "active");
9567
+ const resumeMs = computeIdleDelayMs(
9568
+ rotationDecision.earliestResetAt,
9569
+ /* @__PURE__ */ new Date(),
9570
+ CODEX_USAGE_LIMIT_MAX_RESUME_MS
9571
+ );
9572
+ const failurePayload = codexUsageLimitFailurePayload(
9573
+ { resetsInSeconds: Math.ceil(resumeMs / 1e3) },
9574
+ "all connected Codex subscriptions are rate-limited",
9575
+ { allAccounts: true }
9576
+ );
9577
+ const authoritativeResetAt = authoritativeCodexCapacityResetAt(
9578
+ leased.accounts,
9579
+ /* @__PURE__ */ new Date()
9580
+ );
9581
+ const armed = await armCodexCapacityWait(db, {
9582
+ accountId: input.accountId,
9206
9583
  workspaceId: input.workspaceId,
9207
- reason: "codex_stale_policy_pin_cleared"
9208
- },
9209
- async (tx) => {
9210
- const changed = await setSessionCodexPinInTransaction(
9211
- tx,
9584
+ sessionId: input.sessionId,
9585
+ turnId,
9586
+ attemptId: input.attemptId,
9587
+ workflowId: input.workflowId,
9588
+ goalId: goalActive && goal ? goal.id : null,
9589
+ goalVersion: goalActive && goal ? goal.version : null,
9590
+ earliestResetAt: authoritativeResetAt,
9591
+ resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
9592
+ failurePayload
9593
+ });
9594
+ if (armed.action === "waiting") {
9595
+ await publishDurableSessionEvents2(
9596
+ bus,
9212
9597
  input.workspaceId,
9213
9598
  input.sessionId,
9214
- null,
9215
- "policy",
9216
- {
9217
- expected: {
9218
- pinnedCredentialId: sessionPin,
9219
- pinSource: sessionPinSource
9220
- }
9221
- }
9599
+ armed.events
9222
9600
  );
9223
- return { result: changed, changed };
9601
+ turnMetricOutcome = "recovering";
9602
+ activityStatus = "waiting_capacity";
9603
+ return claimedResult({
9604
+ status: "waiting_capacity",
9605
+ capacityWait: {
9606
+ waiterId: armed.waiter.id,
9607
+ generation: armed.waiter.generation,
9608
+ nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
9609
+ wakeRevision: armed.waiter.wakeRevision
9610
+ }
9611
+ });
9224
9612
  }
9225
- );
9226
- await signalCodexCapacityWakeTargets(
9227
- { signalCodexCapacityWorkflow, wakeSessionWorkflow },
9228
- pinMutation.wakeTargets
9229
- );
9230
- }
9231
- if (!settings.codexCredentialLeasingEnabled && leased.advanceActivePointer && sessionPin === null && rotationDecision.kind === "active" && rotationDecision.moved) {
9232
- await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
9233
- }
9234
- effectiveCodexCredentialId = leased.credentialId;
9235
- codexLeaseGeneration = leased.generation;
9236
- codexLeaseConfirmedUntilMs = leased.leasedUntil && leaseAcquisitionStartedAtMs !== null ? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS : null;
9237
- codexLeaseHeld = effectiveCodexCredentialId !== null && leased.holderId !== null && leased.generation !== null && codexLeaseConfirmedUntilMs !== null;
9238
- if (codexLeaseHeld) startCodexLeaseHeartbeat();
9239
- const actualOutcome = effectiveCodexCredentialId ? "selected" : rotationDecision.kind === "allCapped" ? "waiting" : "none";
9240
- const actualReason = effectiveCodexCredentialId ? leased.reused ? "lease_reused" : sessionPin === effectiveCodexCredentialId ? "pin" : rotationDecision.kind === "active" && rotationDecision.moved ? "rotation" : "active" : rotationDecision.kind === "allCapped" ? "all_capped" : "none";
9241
- const fencedInFlight = leased.reused;
9242
- const shadowResult = await publishCodexFleetShadowDecisionV1({
9243
- enabled: settings.codexFleetPolicyShadowEnabled,
9244
- decision: {
9245
- accounts: leased.accounts,
9246
- actualCredentialId: effectiveCodexCredentialId,
9247
- actualOutcome,
9248
- actualReason,
9249
- affinityCredentialId: fencedInFlight ? effectiveCodexCredentialId : sessionPin ?? sessionCodex?.lastCredentialId ?? null,
9250
- fencedInFlight,
9251
- nearExhaustionPct: settings.codexRotationNearExhaustionPct,
9252
- now: /* @__PURE__ */ new Date(),
9253
- aliasSeed: randomUUID4()
9254
- },
9255
- publish
9256
- });
9257
- if (shadowResult.outcome === "published") {
9258
- const shadowPayload = shadowResult.payload;
9259
- observability.incrementCounter({
9260
- name: "opengeni_codex_fleet_shadow_decisions_total",
9261
- help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
9262
- labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload)
9263
- });
9264
- observability.info("Codex adaptive fleet shadow decision", {
9265
- workspaceId: input.workspaceId,
9266
- policyVersion: shadowPayload.replay.policyVersion,
9267
- inputFingerprint: shadowPayload.replay.inputFingerprint,
9268
- decisionFingerprint: shadowPayload.replay.decisionFingerprint,
9269
- actualOutcome: shadowPayload.actual.outcome,
9270
- shadowOutcome: shadowPayload.replay.decision.outcome,
9271
- comparison: shadowPayload.comparison,
9272
- candidateCount: shadowPayload.replay.input.candidates.length,
9273
- truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
9274
- payloadBytes: shadowResult.payloadBytes
9275
- });
9276
- } else if (shadowResult.outcome === "failed") {
9277
- observability.incrementCounter({
9278
- name: "opengeni_codex_fleet_shadow_errors_total",
9279
- help: "Shadow decision build/publication failures.",
9280
- labels: codexFleetShadowErrorMetricLabelsV1(shadowResult)
9281
- });
9282
- observability.warn("Codex adaptive fleet shadow decision failed open", {
9283
- stage: shadowResult.stage,
9284
- reason: shadowResult.reason,
9285
- errorClass: "CodexFleetShadowOperationError",
9286
- errorCode: "codex_fleet_shadow_failed",
9287
- origin: "worker",
9288
- payloadBytes: shadowResult.payloadBytes
9289
- });
9290
- }
9291
- const eligibleCount = leased.accounts.filter(
9292
- (account) => isCodexCredentialEligible(account, /* @__PURE__ */ new Date())
9293
- ).length;
9294
- const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
9295
- observability.incrementCounter({
9296
- name: "opengeni_codex_pool_observations_total",
9297
- help: "Observed eligible Codex pool depth buckets at turn selection.",
9298
- labels: { workspace_key: codexWorkspaceKey, depth: poolDepth }
9299
- });
9300
- if (eligibleCount <= 1) {
9301
- observability.incrementCounter({
9302
- name: "opengeni_codex_pool_low_total",
9303
- help: "Alert signal emitted when the eligible Codex pool is zero or one.",
9304
- labels: { workspace_key: codexWorkspaceKey, depth: poolDepth }
9305
- });
9306
- observability.warn("Codex eligible credential pool is low", {
9307
- workspaceId: input.workspaceId,
9308
- eligibleCount,
9309
- connectedCount: leased.accounts.length,
9310
- depth: poolDepth
9311
- });
9312
- }
9313
- if (effectiveCodexCredentialId === null && leased.accounts.length > 0 && leased.accounts.every((account) => !account.allocatorEnabled) && turnId) {
9314
- if (turn.source === "compaction") {
9315
9613
  if (!await settle({
9316
9614
  events: [
9615
+ // `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
9616
+ // rotation-wait state as the reactive all-capped path, so it must freeze
9617
+ // autoContinuations identically (evaluateGoalContinuation reads this marker)
9618
+ // — a goal waiting out a long reset must not burn its continuation budget on
9619
+ // the proactive path while the reactive path spares it.
9317
9620
  {
9318
- type: "turn.cancelled",
9621
+ type: "turn.failed",
9319
9622
  payload: {
9320
- maintenance: "context_compaction",
9321
- reason: "codex_allocator_disabled",
9322
- requestPreserved: true
9623
+ ...failurePayload,
9624
+ recovery: goalActive ? "goal_continuation" : "user_message",
9625
+ rotated: true
9323
9626
  }
9324
9627
  },
9325
9628
  {
@@ -9327,233 +9630,78 @@ function createRunAgentTurnActivity(services) {
9327
9630
  payload: { status: "idle" }
9328
9631
  }
9329
9632
  ],
9330
- turnStatus: "cancelled",
9633
+ turnStatus: "failed",
9331
9634
  sessionStatus: "idle",
9332
9635
  activeTurnId: null
9333
9636
  })) {
9334
9637
  return claimedResult({ status: "cancelled" });
9335
9638
  }
9336
- turnMetricOutcome = "cancelled";
9639
+ turnMetricOutcome = "failed";
9337
9640
  activityStatus = "idle";
9338
- return claimedResult({ status: "idle", deferredUntilWake: true });
9641
+ return claimedResult(
9642
+ goalActive ? {
9643
+ status: "idle",
9644
+ continueDelayMs: resumeMs,
9645
+ idleUntilReset: true
9646
+ } : { status: "idle" }
9647
+ );
9339
9648
  }
9340
- const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
9341
- () => null
9342
- );
9343
- const activeGoal = goal?.status === "active" ? goal : null;
9344
- const armed = await armCodexCapacityWait(db, {
9345
- accountId: input.accountId,
9346
- workspaceId: input.workspaceId,
9347
- sessionId: input.sessionId,
9348
- turnId,
9349
- attemptId: input.attemptId,
9350
- workflowId: input.workflowId,
9351
- goalId: activeGoal?.id ?? null,
9352
- goalVersion: activeGoal?.version ?? null,
9353
- earliestResetAt: null,
9354
- resetKind: "bounded_refresh",
9355
- failurePayload: {
9356
- error: "All connected Codex subscriptions are disabled for new allocations.",
9357
- code: "codex_allocator_disabled",
9358
- detail: "waiting for a credential to be re-enabled, reconnected, or added"
9359
- }
9360
- });
9361
- if (armed.action === "waiting") {
9362
- await publishDurableSessionEvents2(
9363
- bus,
9649
+ if (effectiveCodexCredentialId) {
9650
+ const priorAccountId = sessionCodex?.lastCredentialId ?? null;
9651
+ await recordSessionActiveCodexCredential(
9652
+ db,
9364
9653
  input.workspaceId,
9365
9654
  input.sessionId,
9366
- armed.events
9655
+ effectiveCodexCredentialId
9367
9656
  );
9368
- turnMetricOutcome = "recovering";
9369
- activityStatus = "waiting_capacity";
9370
- return claimedResult({
9371
- status: "waiting_capacity",
9372
- capacityWait: {
9373
- waiterId: armed.waiter.id,
9374
- generation: armed.waiter.generation,
9375
- nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
9376
- wakeRevision: armed.waiter.wakeRevision
9377
- }
9378
- });
9379
- }
9380
- if (!await settle({
9381
- events: [
9382
- {
9383
- type: "turn.failed",
9384
- payload: {
9385
- error: "All connected Codex subscriptions are disabled for new allocations.",
9386
- code: "codex_allocator_disabled",
9387
- retryable: false,
9388
- recovery: "user_message"
9389
- }
9390
- },
9391
- { type: "session.status.changed", payload: { status: "idle" } }
9392
- ],
9393
- turnStatus: "failed",
9394
- sessionStatus: "idle",
9395
- activeTurnId: null
9396
- })) {
9397
- return claimedResult({ status: "cancelled" });
9398
- }
9399
- turnMetricOutcome = "failed";
9400
- activityStatus = "idle";
9401
- return claimedResult({ status: "idle" });
9402
- }
9403
- if (rotationDecision.kind === "allCapped" && turnId) {
9404
- if (turn.source === "compaction") {
9405
- if (!await settle({
9406
- events: [
9657
+ if (priorAccountId !== effectiveCodexCredentialId) {
9658
+ const rotated = rotationDecision.kind === "active" && rotationDecision.moved;
9659
+ await publish([
9407
9660
  {
9408
- type: "turn.cancelled",
9661
+ type: "codex.account.switched",
9409
9662
  payload: {
9410
- maintenance: "context_compaction",
9411
- reason: "codex_capacity_unavailable",
9412
- requestPreserved: true
9663
+ fromAccountId: priorAccountId,
9664
+ toAccountId: effectiveCodexCredentialId,
9665
+ reason: rotated ? "rotation" : "manual"
9413
9666
  }
9414
- },
9415
- {
9416
- type: "session.status.changed",
9417
- payload: { status: "idle" }
9418
9667
  }
9419
- ],
9420
- turnStatus: "cancelled",
9421
- sessionStatus: "idle",
9422
- activeTurnId: null
9423
- })) {
9424
- return claimedResult({ status: "cancelled" });
9668
+ ]);
9425
9669
  }
9426
- turnMetricOutcome = "cancelled";
9427
- activityStatus = "idle";
9428
- return claimedResult({ status: "idle", deferredUntilWake: true });
9429
- }
9430
- const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
9431
- () => null
9432
- );
9433
- const goalActive = Boolean(goal && goal.status === "active");
9434
- const resumeMs = computeIdleDelayMs(
9435
- rotationDecision.earliestResetAt,
9436
- /* @__PURE__ */ new Date(),
9437
- CODEX_USAGE_LIMIT_MAX_RESUME_MS
9438
- );
9439
- const failurePayload = codexUsageLimitFailurePayload(
9440
- { resetsInSeconds: Math.ceil(resumeMs / 1e3) },
9441
- "all connected Codex subscriptions are rate-limited",
9442
- { allAccounts: true }
9443
- );
9444
- const authoritativeResetAt = authoritativeCodexCapacityResetAt(
9445
- leased.accounts,
9446
- /* @__PURE__ */ new Date()
9447
- );
9448
- const armed = await armCodexCapacityWait(db, {
9449
- accountId: input.accountId,
9450
- workspaceId: input.workspaceId,
9451
- sessionId: input.sessionId,
9452
- turnId,
9453
- attemptId: input.attemptId,
9454
- workflowId: input.workflowId,
9455
- goalId: goalActive && goal ? goal.id : null,
9456
- goalVersion: goalActive && goal ? goal.version : null,
9457
- earliestResetAt: authoritativeResetAt,
9458
- resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
9459
- failurePayload
9460
- });
9461
- if (armed.action === "waiting") {
9462
- await publishDurableSessionEvents2(
9463
- bus,
9464
- input.workspaceId,
9465
- input.sessionId,
9466
- armed.events
9467
- );
9468
- turnMetricOutcome = "recovering";
9469
- activityStatus = "waiting_capacity";
9470
- return claimedResult({
9471
- status: "waiting_capacity",
9472
- capacityWait: {
9473
- waiterId: armed.waiter.id,
9474
- generation: armed.waiter.generation,
9475
- nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
9476
- wakeRevision: armed.waiter.wakeRevision
9670
+ const selectionReason = leased.reused ? "lease_reused" : sessionPin === effectiveCodexCredentialId ? "pin" : rotationDecision.kind === "active" && rotationDecision.moved ? "rotation" : "active";
9671
+ observability.incrementCounter({
9672
+ name: "opengeni_codex_credential_selections_total",
9673
+ help: "Codex credential selections by strategy and reason.",
9674
+ labels: {
9675
+ workspace_key: codexWorkspaceKey,
9676
+ strategy: leased.rotationStrategy,
9677
+ reason: selectionReason
9477
9678
  }
9478
9679
  });
9479
- }
9480
- if (!await settle({
9481
- events: [
9482
- // `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
9483
- // rotation-wait state as the reactive all-capped path, so it must freeze
9484
- // autoContinuations identically (evaluateGoalContinuation reads this marker)
9485
- // — a goal waiting out a long reset must not burn its continuation budget on
9486
- // the proactive path while the reactive path spares it.
9487
- {
9488
- type: "turn.failed",
9489
- payload: {
9490
- ...failurePayload,
9491
- recovery: goalActive ? "goal_continuation" : "user_message",
9492
- rotated: true
9493
- }
9494
- },
9495
- { type: "session.status.changed", payload: { status: "idle" } }
9496
- ],
9497
- turnStatus: "failed",
9498
- sessionStatus: "idle",
9499
- activeTurnId: null
9500
- })) {
9501
- return claimedResult({ status: "cancelled" });
9502
- }
9503
- turnMetricOutcome = "failed";
9504
- activityStatus = "idle";
9505
- return claimedResult(
9506
- goalActive ? {
9507
- status: "idle",
9508
- continueDelayMs: resumeMs,
9509
- idleUntilReset: true
9510
- } : { status: "idle" }
9511
- );
9512
- }
9513
- if (effectiveCodexCredentialId) {
9514
- const priorAccountId = sessionCodex?.lastCredentialId ?? null;
9515
- await recordSessionActiveCodexCredential(
9516
- db,
9517
- input.workspaceId,
9518
- input.sessionId,
9519
- effectiveCodexCredentialId
9520
- );
9521
- if (priorAccountId !== effectiveCodexCredentialId) {
9522
- const rotated = rotationDecision.kind === "active" && rotationDecision.moved;
9523
9680
  await publish([
9524
9681
  {
9525
- type: "codex.account.switched",
9682
+ type: "codex.credential.selected",
9526
9683
  payload: {
9527
- fromAccountId: priorAccountId,
9528
- toAccountId: effectiveCodexCredentialId,
9529
- reason: rotated ? "rotation" : "manual"
9684
+ credentialId: effectiveCodexCredentialId,
9685
+ strategy: leased.rotationStrategy,
9686
+ reason: selectionReason,
9687
+ eligibleCount,
9688
+ connectedCount: leased.accounts.length,
9689
+ reused: leased.reused
9530
9690
  }
9531
9691
  }
9532
9692
  ]);
9533
9693
  }
9534
- const selectionReason = leased.reused ? "lease_reused" : sessionPin === effectiveCodexCredentialId ? "pin" : rotationDecision.kind === "active" && rotationDecision.moved ? "rotation" : "active";
9535
- observability.incrementCounter({
9536
- name: "opengeni_codex_credential_selections_total",
9537
- help: "Codex credential selections by strategy and reason.",
9538
- labels: {
9539
- workspace_key: codexWorkspaceKey,
9540
- strategy: leased.rotationStrategy,
9541
- reason: selectionReason
9542
- }
9694
+ } catch (error) {
9695
+ credentialSelectionOutcome = "failed";
9696
+ throw error;
9697
+ } finally {
9698
+ recordTurnStartupPhase(observability, {
9699
+ phase: "credential_selection",
9700
+ provider: "codex-subscription",
9701
+ backend: turn.sandboxBackend,
9702
+ outcome: credentialSelectionOutcome,
9703
+ durationSeconds: (performance.now() - credentialSelectionStartedAt) / 1e3
9543
9704
  });
9544
- await publish([
9545
- {
9546
- type: "codex.credential.selected",
9547
- payload: {
9548
- credentialId: effectiveCodexCredentialId,
9549
- strategy: leased.rotationStrategy,
9550
- reason: selectionReason,
9551
- eligibleCount,
9552
- connectedCount: leased.accounts.length,
9553
- reused: leased.reused
9554
- }
9555
- }
9556
- ]);
9557
9705
  }
9558
9706
  }
9559
9707
  if (isXaiTurn) {
@@ -9733,6 +9881,7 @@ function createRunAgentTurnActivity(services) {
9733
9881
  credentialId: effectiveXaiCredentialId
9734
9882
  });
9735
9883
  }
9884
+ const runtimePreparationStartedAt = performance.now();
9736
9885
  const [packRuntime, installedSkillRuntime] = await Promise.all([
9737
9886
  resolveWorkspacePackRuntime(db, input.workspaceId),
9738
9887
  resolveWorkspaceInstalledSkillRuntime(db, input.workspaceId)
@@ -9880,6 +10029,11 @@ function createRunAgentTurnActivity(services) {
9880
10029
  });
9881
10030
  }
9882
10031
  let codexModelRequestSequence = 0;
10032
+ let firstModelRequestPreparationStartedAt = null;
10033
+ let firstModelRequestPreparationRecorded = false;
10034
+ let firstModelRequestAuditRecorded = false;
10035
+ let firstModelRequestCheckpointAt = null;
10036
+ const firstModelRequestCheckpoints = /* @__PURE__ */ new Set();
9883
10037
  const codexContext = resolvedModel?.provider.kind === "codex-subscription" ? (() => {
9884
10038
  const resolver = buildCodexTokenResolver(
9885
10039
  db,
@@ -9906,10 +10060,38 @@ function createRunAgentTurnActivity(services) {
9906
10060
  latestCodexUsage = snapshot;
9907
10061
  },
9908
10062
  // latest wins; flushed once in finally
10063
+ onRequestPreparationDiagnostic: (phase) => {
10064
+ if (firstModelRequestCheckpointAt === null || firstModelRequestCheckpoints.has(phase)) {
10065
+ return;
10066
+ }
10067
+ const now = performance.now();
10068
+ const metricPhase = phase === "transport_entry" ? "model_sdk_serialization" : phase === "credential_ready" ? "model_credential_resolution" : "model_wire_normalization";
10069
+ firstModelRequestCheckpoints.add(phase);
10070
+ recordTurnStartupPhase(observability, {
10071
+ phase: metricPhase,
10072
+ provider: turnExecutionPolicy.providerId,
10073
+ backend: activeSandboxBackend ?? groupBoxBackend,
10074
+ outcome: "completed",
10075
+ durationSeconds: (now - firstModelRequestCheckpointAt) / 1e3,
10076
+ count: turnTools.length
10077
+ });
10078
+ firstModelRequestCheckpointAt = now;
10079
+ },
9909
10080
  onRequestOpaqueArtifacts: ({ fingerprints }) => {
9910
10081
  lastCodexRequestOpaqueArtifacts = fingerprints;
9911
10082
  },
9912
10083
  onModelRequestDiagnostic: (event) => {
10084
+ if (event.phase === "started" && firstModelRequestPreparationStartedAt !== null && !firstModelRequestPreparationRecorded) {
10085
+ firstModelRequestPreparationRecorded = true;
10086
+ recordTurnStartupPhase(observability, {
10087
+ phase: "model_request_preparation",
10088
+ provider: turnExecutionPolicy.providerId,
10089
+ backend: activeSandboxBackend ?? groupBoxBackend,
10090
+ outcome: "completed",
10091
+ durationSeconds: (performance.now() - firstModelRequestPreparationStartedAt) / 1e3,
10092
+ count: turnTools.length
10093
+ });
10094
+ }
9913
10095
  const phase = event.phase === "headers" ? "headers" : event.phase === "first_byte" ? "first_byte" : event.phase === "completed" || event.phase === "failed" || event.phase === "timed_out" ? "terminal" : null;
9914
10096
  if (!phase) return;
9915
10097
  recordModelRequestPhase(observability, {
@@ -9924,19 +10106,41 @@ function createRunAgentTurnActivity(services) {
9924
10106
  if (!publish || !turnId) {
9925
10107
  throw new Error("Codex model request started before the turn event producer");
9926
10108
  }
9927
- await publish([
9928
- {
9929
- type: "agent.model.request",
9930
- payload: {
9931
- ...event,
9932
- provider: "codex-subscription",
9933
- turnId,
9934
- attemptId: input.attemptId,
9935
- dispatchId,
9936
- executionGeneration
10109
+ const shouldRecordStartedAudit = event.phase === "started" && !firstModelRequestAuditRecorded;
10110
+ if (shouldRecordStartedAudit) {
10111
+ firstModelRequestAuditRecorded = true;
10112
+ }
10113
+ const auditStartedAt = shouldRecordStartedAudit ? performance.now() : null;
10114
+ let auditOutcome = "completed";
10115
+ try {
10116
+ await publish([
10117
+ {
10118
+ type: "agent.model.request",
10119
+ payload: {
10120
+ ...event,
10121
+ provider: "codex-subscription",
10122
+ turnId,
10123
+ attemptId: input.attemptId,
10124
+ dispatchId,
10125
+ executionGeneration
10126
+ }
9937
10127
  }
10128
+ ]);
10129
+ } catch (error) {
10130
+ auditOutcome = "failed";
10131
+ throw error;
10132
+ } finally {
10133
+ if (auditStartedAt !== null) {
10134
+ recordTurnStartupPhase(observability, {
10135
+ phase: "model_request_audit",
10136
+ provider: turnExecutionPolicy.providerId,
10137
+ backend: activeSandboxBackend ?? groupBoxBackend,
10138
+ outcome: auditOutcome,
10139
+ durationSeconds: (performance.now() - auditStartedAt) / 1e3,
10140
+ count: turnTools.length
10141
+ });
9938
10142
  }
9939
- ]);
10143
+ }
9940
10144
  }
9941
10145
  };
9942
10146
  })() : null;
@@ -10343,10 +10547,26 @@ function createRunAgentTurnActivity(services) {
10343
10547
  cancellationSignal,
10344
10548
  void 0
10345
10549
  );
10346
- const establishPolicy = lazyProvisionEnabled(settings) && !machinePrimary && runSettings.sandboxBackend !== "none" && // Resolved run credentials must be written to one exact leased sandbox
10347
- // before agent execution. A warm active pointer can bypass the lazy
10348
- // provisioner entirely, which previously skipped materialization.
10349
- !initialRunCredentialMaterial && requiredGeneratedVideoFiles.length === 0 ? "on-demand" : "eager";
10550
+ const establishDecision = sandboxEstablishPolicyDecision({
10551
+ lazyEnabled: lazyProvisionEnabled(settings),
10552
+ machinePrimary,
10553
+ sandboxBackend: runSettings.sandboxBackend,
10554
+ // Resolved run credentials must be written to one exact leased sandbox
10555
+ // before agent execution. A warm active pointer can bypass the lazy
10556
+ // provisioner entirely, which previously skipped materialization.
10557
+ hasInitialRunCredentialMaterial: initialRunCredentialMaterial !== null,
10558
+ generatedVideoFileCount: requiredGeneratedVideoFiles.length,
10559
+ hasSignedFileResources: requiresSignedFileResourceDownloads(
10560
+ runSettings,
10561
+ activeSandboxBackend ?? groupBoxBackend
10562
+ ) && turnResources.some((resource) => resource.kind === "file")
10563
+ });
10564
+ const establishPolicy = establishDecision.policy;
10565
+ recordTurnSandboxEstablishPolicy(observability, {
10566
+ policy: establishDecision.policy,
10567
+ reason: establishDecision.reason,
10568
+ backend: machinePrimary ? "selfhosted" : groupBoxBackend
10569
+ });
10350
10570
  if (activeSandboxBackend !== "selfhosted") {
10351
10571
  await assertGitHubResourcesRemainAuthorized(db, input.workspaceId, turnResources);
10352
10572
  }
@@ -10702,195 +10922,235 @@ function createRunAgentTurnActivity(services) {
10702
10922
  }
10703
10923
  runCredentialRenewal = controller;
10704
10924
  };
10925
+ recordTurnStartupPhase(observability, {
10926
+ phase: "runtime_preparation",
10927
+ provider: turnExecutionPolicy.providerId,
10928
+ backend: activeSandboxBackend ?? groupBoxBackend,
10929
+ outcome: "completed",
10930
+ durationSeconds: (performance.now() - runtimePreparationStartedAt) / 1e3
10931
+ });
10705
10932
  if (shouldEstablishSandboxForTurn(
10706
10933
  settings.sandboxOwnershipEnabled,
10707
10934
  turn.sandboxBackend,
10708
10935
  machinePrimary
10709
10936
  )) {
10710
- const managedOwnership = managedSandboxOwnershipForTurn(
10711
- machinePrimary,
10712
- input.attemptId,
10713
- session.sandboxGroupId
10714
- );
10715
- sandboxHolderId = managedOwnership?.holderId ?? null;
10716
- sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
10717
- if (machinePrimary) {
10718
- const machineOpStream = settings.agentOpStreamEnabled === true ? (await getEnrollment(db, input.workspaceId, activeSandboxRecord.enrollmentId))?.opStream === true : false;
10719
- const established = await establishSelfhostedTurnSession(
10720
- {
10721
- db,
10722
- settings,
10723
- bus,
10724
- onOp: machineOpObserver.observer,
10725
- onSandboxOperation: sandboxOperationObserver,
10726
- opJournal
10727
- },
10728
- {
10729
- workspaceId: input.workspaceId,
10730
- agentId: activeSandboxRecord.enrollmentId,
10731
- opStream: machineOpStream,
10732
- epoch: activeSandboxPointer.activeEpoch,
10733
- environment: sandboxEnvironment,
10734
- ...transientCodemodeEnvironment ? { transientExecEnvironment: transientCodemodeEnvironment } : {},
10735
- workingDir: activeSandboxPointer.workingDir
10736
- }
10937
+ const sandboxEstablishStartedAt = performance.now();
10938
+ let sandboxEstablishOutcome = "completed";
10939
+ try {
10940
+ const managedOwnership = managedSandboxOwnershipForTurn(
10941
+ machinePrimary,
10942
+ input.attemptId,
10943
+ session.sandboxGroupId
10737
10944
  );
10738
- machinePrimarySession = established.session;
10739
- setupBoxSession = established.session;
10740
- resolvedSandbox = {
10741
- // Wrap in the SAME routing proxy so a mid-turn swap (to another machine
10742
- // or back to the group box) still re-routes per op. PIN this established
10743
- // SelfhostedSession for the machine pointer so the turn-start manifest
10744
- // write (via the proxy's `state` getter) and the per-op reads hit ONE
10745
- // instance — no two-instance manifest divergence.
10746
- established: wrapTurnBoxWithRouting(
10945
+ sandboxHolderId = managedOwnership?.holderId ?? null;
10946
+ sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
10947
+ if (machinePrimary) {
10948
+ const machineOpStream = settings.agentOpStreamEnabled === true ? (await getEnrollment(db, input.workspaceId, activeSandboxRecord.enrollmentId))?.opStream === true : false;
10949
+ const established = await establishSelfhostedTurnSession(
10747
10950
  {
10748
10951
  db,
10749
10952
  settings,
10750
10953
  bus,
10751
- opJournal,
10752
10954
  onOp: machineOpObserver.observer,
10753
10955
  onSandboxOperation: sandboxOperationObserver,
10754
- onHomeSandboxRebound,
10755
- ...runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}
10956
+ opJournal
10756
10957
  },
10757
10958
  {
10758
10959
  workspaceId: input.workspaceId,
10759
- sessionId: input.sessionId,
10960
+ agentId: activeSandboxRecord.enrollmentId,
10961
+ opStream: machineOpStream,
10962
+ epoch: activeSandboxPointer.activeEpoch,
10760
10963
  environment: sandboxEnvironment,
10761
10964
  ...transientCodemodeEnvironment ? { transientExecEnvironment: transientCodemodeEnvironment } : {},
10762
- workspaceMutationFence: {
10763
- accountId: input.accountId,
10764
- turnId: turn.id,
10765
- executionGeneration,
10766
- attemptId: input.attemptId
10965
+ workingDir: activeSandboxPointer.workingDir
10966
+ }
10967
+ );
10968
+ machinePrimarySession = established.session;
10969
+ setupBoxSession = established.session;
10970
+ resolvedSandbox = {
10971
+ // Wrap in the SAME routing proxy so a mid-turn swap (to another machine
10972
+ // or back to the group box) still re-routes per op. PIN this established
10973
+ // SelfhostedSession for the machine pointer so the turn-start manifest
10974
+ // write (via the proxy's `state` getter) and the per-op reads hit ONE
10975
+ // instance — no two-instance manifest divergence.
10976
+ established: wrapTurnBoxWithRouting(
10977
+ {
10978
+ db,
10979
+ settings,
10980
+ bus,
10981
+ opJournal,
10982
+ onOp: machineOpObserver.observer,
10983
+ onSandboxOperation: sandboxOperationObserver,
10984
+ onHomeSandboxRebound,
10985
+ ...runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}
10767
10986
  },
10768
- pinnedSelfhosted: {
10769
- sandboxId: activeSandboxPointer.activeSandboxId,
10770
- epoch: activeSandboxPointer.activeEpoch
10987
+ {
10988
+ workspaceId: input.workspaceId,
10989
+ sessionId: input.sessionId,
10990
+ environment: sandboxEnvironment,
10991
+ ...transientCodemodeEnvironment ? { transientExecEnvironment: transientCodemodeEnvironment } : {},
10992
+ workspaceMutationFence: {
10993
+ accountId: input.accountId,
10994
+ turnId: turn.id,
10995
+ executionGeneration,
10996
+ attemptId: input.attemptId
10997
+ },
10998
+ pinnedSelfhosted: {
10999
+ sandboxId: activeSandboxPointer.activeSandboxId,
11000
+ epoch: activeSandboxPointer.activeEpoch
11001
+ },
11002
+ // HOME semantics for a mid-turn clear-to-null: only a genuine
11003
+ // machine-HOME session (its home IS this machine, session.sandboxBackend
11004
+ // === "selfhosted") resolves null back to the pinned machine. A Modal-HOME
11005
+ // session merely PINNED to a machine this turn never established its group
11006
+ // box, so defaultIsHome:false makes a clear-to-null fail typed
11007
+ // (`home_unavailable_this_turn`) rather than silently serving the machine;
11008
+ // the detach takes effect next turn. (Lazy home-box establishment on such a
11009
+ // clear is a deferred follow-up; issue #341.)
11010
+ defaultIsHome: session.sandboxBackend === "selfhosted"
10771
11011
  },
10772
- // HOME semantics for a mid-turn clear-to-null: only a genuine
10773
- // machine-HOME session (its home IS this machine, session.sandboxBackend
10774
- // === "selfhosted") resolves null back to the pinned machine. A Modal-HOME
10775
- // session merely PINNED to a machine this turn never established its group
10776
- // box, so defaultIsHome:false makes a clear-to-null fail typed
10777
- // (`home_unavailable_this_turn`) rather than silently serving the machine;
10778
- // the detach takes effect next turn. (Lazy home-box establishment on such a
10779
- // clear is a deferred follow-up; issue #341.)
10780
- defaultIsHome: session.sandboxBackend === "selfhosted"
10781
- },
10782
- established
10783
- ),
10784
- // For a machine route this is the active-pointer epoch, not a cloud
10785
- // lease epoch. Cloud-only heartbeat/snapshot/capture paths require
10786
- // sandboxGroupId and remain disabled for this branch.
10787
- leaseEpoch: activeSandboxPointer.activeEpoch,
10788
- release: async () => void 0
10789
- };
10790
- } else if (establishPolicy === "on-demand") {
10791
- } else {
10792
- resolvedSandbox = await waitForTurnOperation(
10793
- resumeBoxForTurn(
10794
- {
10795
- db,
10796
- settings: runSettings,
10797
- logicalFallbackSettings: logicalSandboxSettings,
10798
- cancellationSignal: sandboxResumeSignal,
10799
- sandboxMetrics: runtimeMetricsHooksForObservability(observability),
10800
- onSandboxLost: publishSandboxLost
10801
- },
10802
- {
10803
- accountId: input.accountId,
10804
- workspaceId: input.workspaceId,
10805
- sandboxGroupId: session.sandboxGroupId,
10806
- sessionId: input.sessionId,
10807
- // groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
10808
- // is not machine-primary resumes a real cloud group box (the
10809
- // deployment default), never a "selfhosted" box (which would throw
10810
- // for lack of a bound agentId).
10811
- backend: groupBoxBackend,
10812
- os: session.sandboxOs,
10813
- environment: sandboxEnvironment,
10814
- // IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
10815
- // this run resolves. The lease stamps it + conflicts on a live shared box
10816
- // running a DIFFERENT image (solo → durable rotation; N-holders →
10817
- // SandboxImageConflictError surfaced as an actionable turn error). Prefer
10818
- // the explicit Modal image ref, else the docker image. The selfhosted branch
10819
- // (establishSelfhostedTurnSession) NEVER passes
10820
- // an image — B3 lives only on this Modal else-branch.
10821
- ...runSettings.modalImageRef ?? runSettings.dockerImage ? {
10822
- image: runSettings.modalImageRef ?? runSettings.dockerImage
10823
- } : {},
10824
- // RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
10825
- // conflicts on a live shared box set up under a different rig (solo
10826
- // durable rotation / N-holders SandboxRigConflictError). Omitted for a
10827
- // rig-less turn -> never stamped or enforced (shares exactly as today).
10828
- ...rigVersion ? { rigVersionId: rigVersion.id } : {}
10829
- },
10830
- "turn",
10831
- managedOwnership.holderId
10832
- ),
10833
- sandboxResumeSignal,
10834
- releaseLateSandbox
10835
- );
10836
- setupBoxSession = resolvedSandbox.established.session;
10837
- await publishSandboxLifecycleEvents(resolvedSandbox);
10838
- resolvedSandbox = {
10839
- ...resolvedSandbox,
10840
- established: wrapTurnBoxWithRouting(
10841
- {
10842
- db,
10843
- settings,
10844
- bus,
10845
- opJournal,
10846
- onSandboxOperation: sandboxOperationObserver,
10847
- onHomeSandboxLost: publishSandboxLost,
10848
- onHomeSandboxRebound,
10849
- ...runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}
10850
- },
10851
- // Thread the SAME declared environment the group box was created with
10852
- // (resumeBoxForTurn, above) so a selfhosted swap target's manifest
10853
- // carries it too — the SDK's per-turn manifest-env delta stays empty
10854
- // (no "cannot change manifest environment variables" throw).
10855
- {
10856
- workspaceId: input.workspaceId,
10857
- sessionId: input.sessionId,
10858
- environment: sandboxEnvironment,
10859
- ...transientCodemodeEnvironment ? { transientExecEnvironment: transientCodemodeEnvironment } : {},
10860
- workspaceMutationFence: {
10861
- accountId: input.accountId,
10862
- turnId: turn.id,
10863
- executionGeneration,
10864
- attemptId: input.attemptId
11012
+ established
11013
+ ),
11014
+ // For a machine route this is the active-pointer epoch, not a cloud
11015
+ // lease epoch. Cloud-only heartbeat/snapshot/capture paths require
11016
+ // sandboxGroupId and remain disabled for this branch.
11017
+ leaseEpoch: activeSandboxPointer.activeEpoch,
11018
+ release: async () => void 0
11019
+ };
11020
+ } else if (establishPolicy === "on-demand") {
11021
+ } else {
11022
+ resolvedSandbox = await waitForTurnOperation(
11023
+ resumeBoxForTurn(
11024
+ {
11025
+ db,
11026
+ settings: runSettings,
11027
+ logicalFallbackSettings: logicalSandboxSettings,
11028
+ cancellationSignal: sandboxResumeSignal,
11029
+ sandboxMetrics: runtimeMetricsHooksForObservability(observability),
11030
+ onSandboxLost: publishSandboxLost
10865
11031
  },
10866
- homeLease: {
11032
+ {
10867
11033
  accountId: input.accountId,
11034
+ workspaceId: input.workspaceId,
10868
11035
  sandboxGroupId: session.sandboxGroupId,
10869
- leaseEpoch: resolvedSandbox.leaseEpoch,
10870
- instanceId: resolvedSandbox.established.instanceId,
10871
- backend: groupBoxBackend
10872
- }
10873
- },
10874
- resolvedSandbox.established
10875
- )
10876
- };
10877
- }
10878
- if (resolvedSandbox) {
10879
- startLeaseHeartbeat(resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
11036
+ sessionId: input.sessionId,
11037
+ // groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
11038
+ // is not machine-primary resumes a real cloud group box (the
11039
+ // deployment default), never a "selfhosted" box (which would throw
11040
+ // for lack of a bound agentId).
11041
+ backend: groupBoxBackend,
11042
+ os: session.sandboxOs,
11043
+ environment: sandboxEnvironment,
11044
+ // IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
11045
+ // this run resolves. The lease stamps it + conflicts on a live shared box
11046
+ // running a DIFFERENT image (solo → durable rotation; N-holders →
11047
+ // SandboxImageConflictError surfaced as an actionable turn error). Prefer
11048
+ // the explicit Modal image ref, else the docker image. The selfhosted branch
11049
+ // (establishSelfhostedTurnSession) NEVER passes
11050
+ // an image — B3 lives only on this Modal else-branch.
11051
+ ...runSettings.modalImageRef ?? runSettings.dockerImage ? {
11052
+ image: runSettings.modalImageRef ?? runSettings.dockerImage
11053
+ } : {},
11054
+ // RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
11055
+ // conflicts on a live shared box set up under a different rig (solo
11056
+ // durable rotation / N-holders SandboxRigConflictError). Omitted for a
11057
+ // rig-less turn -> never stamped or enforced (shares exactly as today).
11058
+ ...rigVersion ? { rigVersionId: rigVersion.id } : {}
11059
+ },
11060
+ "turn",
11061
+ managedOwnership.holderId
11062
+ ),
11063
+ sandboxResumeSignal,
11064
+ releaseLateSandbox
11065
+ );
11066
+ setupBoxSession = resolvedSandbox.established.session;
11067
+ await publishSandboxLifecycleEvents(resolvedSandbox);
11068
+ resolvedSandbox = {
11069
+ ...resolvedSandbox,
11070
+ established: wrapTurnBoxWithRouting(
11071
+ {
11072
+ db,
11073
+ settings,
11074
+ bus,
11075
+ opJournal,
11076
+ onSandboxOperation: sandboxOperationObserver,
11077
+ onHomeSandboxLost: publishSandboxLost,
11078
+ onHomeSandboxRebound,
11079
+ ...runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}
11080
+ },
11081
+ // Thread the SAME declared environment the group box was created with
11082
+ // (resumeBoxForTurn, above) so a selfhosted swap target's manifest
11083
+ // carries it too — the SDK's per-turn manifest-env delta stays empty
11084
+ // (no "cannot change manifest environment variables" throw).
11085
+ {
11086
+ workspaceId: input.workspaceId,
11087
+ sessionId: input.sessionId,
11088
+ environment: sandboxEnvironment,
11089
+ ...transientCodemodeEnvironment ? { transientExecEnvironment: transientCodemodeEnvironment } : {},
11090
+ workspaceMutationFence: {
11091
+ accountId: input.accountId,
11092
+ turnId: turn.id,
11093
+ executionGeneration,
11094
+ attemptId: input.attemptId
11095
+ },
11096
+ homeLease: {
11097
+ accountId: input.accountId,
11098
+ sandboxGroupId: session.sandboxGroupId,
11099
+ leaseEpoch: resolvedSandbox.leaseEpoch,
11100
+ instanceId: resolvedSandbox.established.instanceId,
11101
+ backend: groupBoxBackend
11102
+ }
11103
+ },
11104
+ resolvedSandbox.established
11105
+ )
11106
+ };
11107
+ }
11108
+ if (resolvedSandbox) {
11109
+ startLeaseHeartbeat(resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
11110
+ }
11111
+ } catch (error) {
11112
+ sandboxEstablishOutcome = "failed";
11113
+ throw error;
11114
+ } finally {
11115
+ recordTurnStartupPhase(observability, {
11116
+ phase: "sandbox_establish",
11117
+ provider: turnExecutionPolicy.providerId,
11118
+ backend: machinePrimary ? "selfhosted" : groupBoxBackend,
11119
+ outcome: sandboxEstablishOutcome,
11120
+ durationSeconds: (performance.now() - sandboxEstablishStartedAt) / 1e3
11121
+ });
10880
11122
  }
10881
11123
  }
10882
- const ordinaryFileResourceDownloads = await waitForTurnOperation(
10883
- sandboxFileDownloadsForRun(
10884
- runSettings,
10885
- db,
10886
- objectStorage,
10887
- input.workspaceId,
10888
- turnResources,
10889
- activeSandboxBackend ?? groupBoxBackend
10890
- ),
10891
- cancellationSignal,
10892
- void 0
10893
- );
11124
+ const ordinaryFileResourceDownloads = await (async () => {
11125
+ const fileResolutionStartedAt = performance.now();
11126
+ let fileResolutionOutcome = "completed";
11127
+ try {
11128
+ return await waitForTurnOperation(
11129
+ sandboxFileDownloadsForRun(
11130
+ runSettings,
11131
+ db,
11132
+ objectStorage,
11133
+ input.workspaceId,
11134
+ turnResources,
11135
+ activeSandboxBackend ?? groupBoxBackend
11136
+ ),
11137
+ cancellationSignal,
11138
+ void 0
11139
+ );
11140
+ } catch (error) {
11141
+ fileResolutionOutcome = "failed";
11142
+ throw error;
11143
+ } finally {
11144
+ recordTurnStartupPhase(observability, {
11145
+ phase: "file_resolution",
11146
+ provider: turnExecutionPolicy.providerId,
11147
+ backend: activeSandboxBackend ?? groupBoxBackend,
11148
+ outcome: fileResolutionOutcome,
11149
+ durationSeconds: (performance.now() - fileResolutionStartedAt) / 1e3,
11150
+ count: turnResources.length
11151
+ });
11152
+ }
11153
+ })();
10894
11154
  if (requiredGeneratedVideoFiles.length > 0 && !objectStorage) {
10895
11155
  throw new Error("Generated video materialization requires object storage");
10896
11156
  }
@@ -10920,6 +11180,7 @@ function createRunAgentTurnActivity(services) {
10920
11180
  ])
10921
11181
  ).values()
10922
11182
  ];
11183
+ const toolContextPreparationStartedAt = performance.now();
10923
11184
  throwIfWorkerShuttingDown();
10924
11185
  throwIfTurnCancelled();
10925
11186
  const mcpCredentialRootSessionId = connectionCredentials?.mcpCredentials && hostCredentialRootSessionId ? hostCredentialRootSessionId : input.sessionId;
@@ -11005,53 +11266,88 @@ function createRunAgentTurnActivity(services) {
11005
11266
  runSettings,
11006
11267
  session.firstPartyMcpTools
11007
11268
  );
11008
- preparedTools = await waitForTurnOperation(
11009
- runtime.prepareTools(runSettings, turnTools, {
11010
- accountId: input.accountId,
11011
- workspaceId: input.workspaceId,
11012
- sessionId: input.sessionId,
11013
- // Sign the calling turn into the first-party token so tools classify
11014
- // the caller by its own identity (sacred-pause guard), not the racy
11015
- // live active pointer.
11016
- ...turnId ? { turnId } : {},
11017
- attemptId: input.attemptId,
11018
- executionGeneration,
11019
- subjectId: "worker:first-party-mcp",
11020
- subjectLabel: "OpenGeni worker",
11021
- ...credentialSubjectId ? { credentialSubjectId } : {},
11022
- ...codexAppsAuth ? { codexAppsAuth } : {},
11023
- resolveCredential,
11024
- onAuthNeeded: publishToolAuthNeeded,
11025
- localMcpServers,
11026
- onAttemptToolCatalog: async (catalog) => {
11027
- await persistAttemptToolCatalog(db, catalog);
11028
- },
11029
- // Manager-style sessions carry a creation-validated permission set
11030
- // for their first-party MCP token; null keeps the fixed default.
11031
- ...session.firstPartyMcpPermissions?.length ? { firstPartyPermissions: session.firstPartyMcpPermissions } : {},
11032
- firstPartyTools: selectedFirstPartyMcpTools,
11033
- nestedAgentDepth: session.nestedAgentDepth,
11034
- effectiveMaxNestedAgentDepth: session.effectiveMaxNestedAgentDepth,
11035
- attemptToolDefinitions: createFirstPartyInteractionAttemptToolDefinitions({
11036
- settings: runSettings,
11037
- scope: {
11038
- accountId: input.accountId,
11039
- workspaceId: input.workspaceId,
11040
- sessionId: input.sessionId,
11041
- turnId: turn.id,
11042
- attemptId: input.attemptId,
11043
- executionGeneration
11044
- },
11045
- ...session.firstPartyMcpPermissions?.length ? { permissions: session.firstPartyMcpPermissions } : {},
11046
- selectedTools: selectedFirstPartyMcpTools,
11269
+ recordTurnStartupPhase(observability, {
11270
+ phase: "tool_context_preparation",
11271
+ provider: turnExecutionPolicy.providerId,
11272
+ backend: activeSandboxBackend ?? groupBoxBackend,
11273
+ outcome: "completed",
11274
+ durationSeconds: (performance.now() - toolContextPreparationStartedAt) / 1e3,
11275
+ count: turnTools.length
11276
+ });
11277
+ const toolPreparationStartedAt = performance.now();
11278
+ let toolPreparationOutcome = "completed";
11279
+ try {
11280
+ preparedTools = await waitForTurnOperation(
11281
+ runtime.prepareTools(runSettings, turnTools, {
11282
+ accountId: input.accountId,
11283
+ workspaceId: input.workspaceId,
11284
+ sessionId: input.sessionId,
11285
+ // Sign the calling turn into the first-party token so tools classify
11286
+ // the caller by its own identity (sacred-pause guard), not the racy
11287
+ // live active pointer.
11288
+ ...turnId ? { turnId } : {},
11289
+ attemptId: input.attemptId,
11290
+ executionGeneration,
11047
11291
  subjectId: "worker:first-party-mcp",
11048
11292
  subjectLabel: "OpenGeni worker",
11049
- ...interactionInterventionResume ? { interventionResume: interactionInterventionResume } : {}
11050
- })
11051
- }),
11052
- cancellationSignal,
11053
- async (latePreparedTools) => await latePreparedTools.close().catch(() => void 0)
11054
- );
11293
+ ...credentialSubjectId ? { credentialSubjectId } : {},
11294
+ ...codexAppsAuth ? { codexAppsAuth } : {},
11295
+ resolveCredential,
11296
+ onAuthNeeded: publishToolAuthNeeded,
11297
+ localMcpServers,
11298
+ onPreparationPhase: (measurement) => {
11299
+ recordTurnStartupPhase(observability, {
11300
+ phase: `tool_${measurement.phase}`,
11301
+ provider: turnExecutionPolicy.providerId,
11302
+ backend: activeSandboxBackend ?? groupBoxBackend,
11303
+ outcome: measurement.outcome,
11304
+ durationSeconds: measurement.durationSeconds,
11305
+ count: turnTools.length
11306
+ });
11307
+ },
11308
+ onAttemptToolCatalog: async (catalog) => {
11309
+ await persistAttemptToolCatalog(db, catalog);
11310
+ },
11311
+ // Manager-style sessions carry a creation-validated permission set
11312
+ // for their first-party MCP token; null keeps the fixed default.
11313
+ ...session.firstPartyMcpPermissions?.length ? { firstPartyPermissions: session.firstPartyMcpPermissions } : {},
11314
+ firstPartyTools: selectedFirstPartyMcpTools,
11315
+ nestedAgentDepth: session.nestedAgentDepth,
11316
+ effectiveMaxNestedAgentDepth: session.effectiveMaxNestedAgentDepth,
11317
+ attemptToolDefinitions: createFirstPartyInteractionAttemptToolDefinitions({
11318
+ settings: runSettings,
11319
+ scope: {
11320
+ accountId: input.accountId,
11321
+ workspaceId: input.workspaceId,
11322
+ sessionId: input.sessionId,
11323
+ turnId: turn.id,
11324
+ attemptId: input.attemptId,
11325
+ executionGeneration
11326
+ },
11327
+ ...session.firstPartyMcpPermissions?.length ? { permissions: session.firstPartyMcpPermissions } : {},
11328
+ selectedTools: selectedFirstPartyMcpTools,
11329
+ subjectId: "worker:first-party-mcp",
11330
+ subjectLabel: "OpenGeni worker",
11331
+ ...interactionInterventionResume ? { interventionResume: interactionInterventionResume } : {}
11332
+ })
11333
+ }),
11334
+ cancellationSignal,
11335
+ async (latePreparedTools) => await latePreparedTools.close().catch(() => void 0)
11336
+ );
11337
+ } catch (error) {
11338
+ toolPreparationOutcome = "failed";
11339
+ throw error;
11340
+ } finally {
11341
+ recordTurnStartupPhase(observability, {
11342
+ phase: "tool_preparation",
11343
+ provider: turnExecutionPolicy.providerId,
11344
+ backend: activeSandboxBackend ?? groupBoxBackend,
11345
+ outcome: toolPreparationOutcome,
11346
+ durationSeconds: (performance.now() - toolPreparationStartedAt) / 1e3,
11347
+ count: turnTools.length
11348
+ });
11349
+ }
11350
+ const postToolPreparationStartedAt = performance.now();
11055
11351
  if (turnId && preparedTools.attemptToolEnvironment) {
11056
11352
  codemodeDispatcher = new CodemodeAttemptDispatcher(
11057
11353
  db,
@@ -11399,146 +11695,171 @@ function createRunAgentTurnActivity(services) {
11399
11695
  reason: "attached to session"
11400
11696
  }))
11401
11697
  ];
11402
- const agent = runtime.buildAgent(modelRunSettings, turnResources, {
11403
- reasoningEffort: turn.reasoningEffort,
11404
- latencyMode: turnExecutionPolicy.latencyMode,
11405
- ...serviceTier ? { serviceTier } : {},
11406
- ...humanInputResume ? { humanInputResponse: humanInputResume } : {},
11407
- humanInputEnabled: agentHumanInputEnabled,
11408
- genesisTitleHint: isGenesisTurn,
11409
- persistentSessionSettings: {
11410
- titleIsSet: Boolean(session.title?.trim())
11411
- },
11412
- sandboxEnvironment,
11413
- ...preparedTools.attemptToolCatalog ? { attemptToolCatalog: preparedTools.attemptToolCatalog } : {},
11414
- ...sandboxArtifactRuntime.available ? { artifactRuntimeAvailable: true } : {},
11415
- ...cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {},
11416
- onToolCancellationFence: (fence) => {
11417
- toolCancellationFenceRef.current = fence;
11418
- },
11419
- // TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
11420
- // seed. ONLY when the effective backend is NOT selfhosted (the connected
11421
- // machine uses its own git creds — mirrors the skipGitHubToken gate above)
11422
- // AND the mint actually produced a token (repo resources present). The runtime
11423
- // seeds it to the box's token file before the repository-clone runs; it never
11424
- // touches the box/agent manifest env.
11425
- ...activeSandboxBackend !== "selfhosted" && sandboxGitTokens ? { gitTokenSeeds: sandboxGitTokens } : {},
11426
- ...activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings ? { gitCredentialBindings: sandboxGitCredentialBindings } : {},
11427
- ...activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken ? { gitTokenSeed: sandboxGitToken } : {},
11428
- ...sandboxCodemodeToken ? { codemodeAvailable: true } : {},
11429
- // Managed boxes receive the bearer through their protected per-session
11430
- // token file. Connected Machines use transient per-exec delivery above,
11431
- // so they must not run the file-seeding lifecycle hook.
11432
- ...activeSandboxBackend !== "selfhosted" && sandboxCodemodeToken ? {
11433
- codemodeTokenSeed: sandboxCodemodeToken,
11434
- codemodeTokenSessionId: input.sessionId
11435
- } : {},
11436
- ...activeSandboxBackend ? { activeSandboxBackend } : {},
11437
- fileResourceDownloads,
11438
- mcpServers: preparedTools.mcpServers,
11439
- resolvedMcpConnectionIds: preparedTools.resolvedMcpConnectionIds,
11440
- connectorActionPolicy,
11441
- // LIVE by-reference connector namespaces (fills during this turn's
11442
- // codex_apps tools/list): the codex tool_search description reads it per
11443
- // model call so the model sees the account's real connected sources.
11444
- codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
11445
- // Resolved-model routing + gating (legacy defaults when null). The model
11446
- // is passed as the model *string* (agent.model = runSettings.openaiModel),
11447
- // NOT a Model instance: an instance only survives the in-process
11448
- // ("none") run, whereas the SandboxAgent/Modal path drops it and
11449
- // re-resolves the model *name* through the global MultiProviderModelProvider
11450
- // configureOpenAI installed — so registry models (Fireworks GLM) route to
11451
- // their own client instead of 404ing against the built-in Azure/OpenAI
11452
- // client. The gating still comes from the resolved provider: server-side
11453
- // store/compaction follow the provider's compaction mode (registry
11454
- // providers resolve to "client"); encrypted reasoning is only
11455
- // round-tripped on the Responses wire API; hosted web search is attached
11456
- // whenever the provider declares it runnable and is independent of the
11457
- // session's MCP allow-list; the effective context window drives the
11458
- // compaction threshold.
11459
- hostedWebSearch,
11460
- ...imageGenerationOption,
11461
- ...videoGenerationOption,
11462
- lazyToolTransport,
11463
- supportsImageInput,
11464
- inputFileMediaTypes: modelInputPolicy.inputFileMediaTypes,
11465
- ...resolvedModel ? {
11466
- encryptedReasoning: resolvedModel.provider.api === "responses" && runSettings.openaiReasoningEncryptedContent,
11467
- contextWindowTokens: resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
11468
- // The ChatGPT/Codex backend rejects the SDK's HOSTED apply_patch
11469
- // tool. Gateway Responses routes likewise expose ordinary function
11470
- // tools, not OpenAI-hosted sandbox tools. Tell buildAgent to use
11471
- // function apply_patch and wrap successful view_image results as
11472
- // typed input_image content. Chat wires have no proven typed image
11473
- // result transport and therefore receive no view_image tool.
11474
- structuredToolTransport: structuredToolTransportForTurn(resolvedModel),
11475
- // EXPLICIT computer-use tool transport, derived from the resolved provider's
11476
- // authoritative wire identity (codex → function-image, chat → disabled,
11477
- // responses → hosted) so the runtime never string-sniffs the model instance's
11478
- // constructor name. See {@link computerToolModeForTurn}.
11479
- computerToolMode: computerToolModeForTurn(resolvedModel),
11480
- ...promptCacheKey ? { promptCacheKey } : {}
11481
- } : (
11482
- // LEGACY global-client fallback (resolveTurnModel returned null → the model
11483
- // is not in the registry, served by the built-in OpenAI/Azure Responses
11484
- // client). That backend has real hosted support, so pin computerToolMode to
11485
- // "hosted" EXPLICITLY rather than leaving the runtime to sniff the instance.
11486
- {
11487
- computerToolMode: computerToolModeForTurn(null),
11488
- promptCacheKey: input.sessionId
11489
- }
11490
- ),
11491
- // Lazy computer-use seam: runtime first brings up :0 only after the model
11492
- // selects a computer tool, then this hook begins the optional proof
11493
- // recording. Shell/filesystem turns never invoke either operation.
11494
- onComputerUseReady: async () => {
11495
- if (!resolvedSandbox) {
11496
- throw new Error("Computer-use display became ready without a resolved sandbox");
11497
- }
11498
- didComputerUse = true;
11499
- await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
11500
- },
11501
- onRetainableSessionImageOutput: retainSessionImageAtToolBoundary,
11502
- ...runtimeSkillActivations.length > 0 ? { skillActivations: runtimeSkillActivations } : {},
11503
- ...!structuredWorkspacePolicyActive && workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {},
11504
- ...workspaceGovernance ? { workspaceGovernance } : {},
11505
- ...workspaceMemory ? { workspaceMemory } : {},
11506
- goalSnapshot: turn.goalSnapshot,
11507
- // Per-session persona tier (session > workspace > deployment default).
11508
- // Composed system-level AFTER the workspace persona so it refines it for
11509
- // this one session; absent ⇒ byte-identical to today's composition.
11510
- ...session.instructions ? { sessionInstructions: session.instructions } : {},
11511
- // Exact host context captured when this turn was accepted. It is
11512
- // system-level and disappears with the turn rather than entering chat.
11513
- ...turn.turnInstructions || mcpAvailabilityInstructions ? {
11514
- turnInstructions: [turn.turnInstructions, mcpAvailabilityInstructions].filter((value) => Boolean(value)).join(" ")
11515
- } : {},
11516
- ...workspaceEnvironmentOption,
11517
- // RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
11518
- // the frozen version carries a non-empty script), and the rig credential
11519
- // hooks. All absent for a rig-less turn (byte-for-byte today).
11520
- ...rigVersion && rigName ? {
11521
- rig: { name: rigName, version: rigVersion.version },
11522
- ...rigVersion.setupScript && rigVersion.setupScript.trim().length > 0 ? {
11523
- rigSetup: {
11524
- rigId: session.rigId,
11525
- versionId: rigVersion.id,
11526
- rigName,
11527
- script: rigVersion.setupScript,
11528
- timeoutMs: runSettings.rigSetupTimeoutMs,
11529
- contentHash: rigProviderImageContentHash({
11530
- backend: turn.sandboxBackend,
11531
- sourceImage: rigProviderImageSourceImage(
11532
- logicalSandboxSettings,
11533
- turn.sandboxBackend
11534
- ),
11535
- definition: rigVersion
11536
- })
11537
- }
11538
- } : {},
11539
- ...rigVersion.credentialHooks.length > 0 ? { rigCredentialHookIds: rigVersion.credentialHooks } : {}
11540
- } : {}
11698
+ recordTurnStartupPhase(observability, {
11699
+ phase: "post_tool_preparation",
11700
+ provider: turnExecutionPolicy.providerId,
11701
+ backend: activeSandboxBackend ?? groupBoxBackend,
11702
+ outcome: "completed",
11703
+ durationSeconds: (performance.now() - postToolPreparationStartedAt) / 1e3
11541
11704
  });
11705
+ const agent = (() => {
11706
+ const agentConstructionStartedAt = performance.now();
11707
+ let agentConstructionOutcome = "completed";
11708
+ try {
11709
+ return runtime.buildAgent(modelRunSettings, turnResources, {
11710
+ reasoningEffort: turn.reasoningEffort,
11711
+ latencyMode: turnExecutionPolicy.latencyMode,
11712
+ ...serviceTier ? { serviceTier } : {},
11713
+ ...humanInputResume ? { humanInputResponse: humanInputResume } : {},
11714
+ humanInputEnabled: agentHumanInputEnabled,
11715
+ genesisTitleHint: isGenesisTurn,
11716
+ persistentSessionSettings: {
11717
+ titleIsSet: Boolean(session.title?.trim())
11718
+ },
11719
+ sandboxEnvironment,
11720
+ ...preparedTools.attemptToolCatalog ? { attemptToolCatalog: preparedTools.attemptToolCatalog } : {},
11721
+ ...sandboxArtifactRuntime.available ? { artifactRuntimeAvailable: true } : {},
11722
+ ...cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {},
11723
+ onToolCancellationFence: (fence) => {
11724
+ toolCancellationFenceRef.current = fence;
11725
+ },
11726
+ // TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
11727
+ // seed. ONLY when the effective backend is NOT selfhosted (the connected
11728
+ // machine uses its own git creds — mirrors the skipGitHubToken gate above)
11729
+ // AND the mint actually produced a token (repo resources present). The runtime
11730
+ // seeds it to the box's token file before the repository-clone runs; it never
11731
+ // touches the box/agent manifest env.
11732
+ ...activeSandboxBackend !== "selfhosted" && sandboxGitTokens ? { gitTokenSeeds: sandboxGitTokens } : {},
11733
+ ...activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings ? { gitCredentialBindings: sandboxGitCredentialBindings } : {},
11734
+ ...activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken ? { gitTokenSeed: sandboxGitToken } : {},
11735
+ ...sandboxCodemodeToken ? { codemodeAvailable: true } : {},
11736
+ // Managed boxes receive the bearer through their protected per-session
11737
+ // token file. Connected Machines use transient per-exec delivery above,
11738
+ // so they must not run the file-seeding lifecycle hook.
11739
+ ...activeSandboxBackend !== "selfhosted" && sandboxCodemodeToken ? {
11740
+ codemodeTokenSeed: sandboxCodemodeToken,
11741
+ codemodeTokenSessionId: input.sessionId
11742
+ } : {},
11743
+ ...activeSandboxBackend ? { activeSandboxBackend } : {},
11744
+ fileResourceDownloads,
11745
+ mcpServers: preparedTools.mcpServers,
11746
+ resolvedMcpConnectionIds: preparedTools.resolvedMcpConnectionIds,
11747
+ connectorActionPolicy,
11748
+ // LIVE by-reference connector namespaces (fills during this turn's
11749
+ // codex_apps tools/list): the codex tool_search description reads it per
11750
+ // model call so the model sees the account's real connected sources.
11751
+ codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
11752
+ // Resolved-model routing + gating (legacy defaults when null). The model
11753
+ // is passed as the model *string* (agent.model = runSettings.openaiModel),
11754
+ // NOT a Model instance: an instance only survives the in-process
11755
+ // ("none") run, whereas the SandboxAgent/Modal path drops it and
11756
+ // re-resolves the model *name* through the global MultiProviderModelProvider
11757
+ // configureOpenAI installed — so registry models (Fireworks GLM) route to
11758
+ // their own client instead of 404ing against the built-in Azure/OpenAI
11759
+ // client. The gating still comes from the resolved provider: server-side
11760
+ // store/compaction follow the provider's compaction mode (registry
11761
+ // providers resolve to "client"); encrypted reasoning is only
11762
+ // round-tripped on the Responses wire API; hosted web search is attached
11763
+ // whenever the provider declares it runnable and is independent of the
11764
+ // session's MCP allow-list; the effective context window drives the
11765
+ // compaction threshold.
11766
+ hostedWebSearch,
11767
+ ...imageGenerationOption,
11768
+ ...videoGenerationOption,
11769
+ lazyToolTransport,
11770
+ supportsImageInput,
11771
+ inputFileMediaTypes: modelInputPolicy.inputFileMediaTypes,
11772
+ ...resolvedModel ? {
11773
+ encryptedReasoning: resolvedModel.provider.api === "responses" && runSettings.openaiReasoningEncryptedContent,
11774
+ contextWindowTokens: resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
11775
+ // The ChatGPT/Codex backend rejects the SDK's HOSTED apply_patch
11776
+ // tool. Gateway Responses routes likewise expose ordinary function
11777
+ // tools, not OpenAI-hosted sandbox tools. Tell buildAgent to use
11778
+ // function apply_patch and wrap successful view_image results as
11779
+ // typed input_image content. Chat wires have no proven typed image
11780
+ // result transport and therefore receive no view_image tool.
11781
+ structuredToolTransport: structuredToolTransportForTurn(resolvedModel),
11782
+ // EXPLICIT computer-use tool transport, derived from the resolved provider's
11783
+ // authoritative wire identity (codex → function-image, chat → disabled,
11784
+ // responses → hosted) so the runtime never string-sniffs the model instance's
11785
+ // constructor name. See {@link computerToolModeForTurn}.
11786
+ computerToolMode: computerToolModeForTurn(resolvedModel),
11787
+ ...promptCacheKey ? { promptCacheKey } : {}
11788
+ } : (
11789
+ // LEGACY global-client fallback (resolveTurnModel returned null → the model
11790
+ // is not in the registry, served by the built-in OpenAI/Azure Responses
11791
+ // client). That backend has real hosted support, so pin computerToolMode to
11792
+ // "hosted" EXPLICITLY rather than leaving the runtime to sniff the instance.
11793
+ {
11794
+ computerToolMode: computerToolModeForTurn(null),
11795
+ promptCacheKey: input.sessionId
11796
+ }
11797
+ ),
11798
+ // Lazy computer-use seam: runtime first brings up :0 only after the model
11799
+ // selects a computer tool, then this hook begins the optional proof
11800
+ // recording. Shell/filesystem turns never invoke either operation.
11801
+ onComputerUseReady: async () => {
11802
+ if (!resolvedSandbox) {
11803
+ throw new Error("Computer-use display became ready without a resolved sandbox");
11804
+ }
11805
+ didComputerUse = true;
11806
+ await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
11807
+ },
11808
+ onRetainableSessionImageOutput: retainSessionImageAtToolBoundary,
11809
+ ...runtimeSkillActivations.length > 0 ? { skillActivations: runtimeSkillActivations } : {},
11810
+ ...!structuredWorkspacePolicyActive && workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {},
11811
+ ...workspaceGovernance ? { workspaceGovernance } : {},
11812
+ ...workspaceMemory ? { workspaceMemory } : {},
11813
+ goalSnapshot: turn.goalSnapshot,
11814
+ // Per-session persona tier (session > workspace > deployment default).
11815
+ // Composed system-level AFTER the workspace persona so it refines it for
11816
+ // this one session; absent ⇒ byte-identical to today's composition.
11817
+ ...session.instructions ? { sessionInstructions: session.instructions } : {},
11818
+ // Exact host context captured when this turn was accepted. It is
11819
+ // system-level and disappears with the turn rather than entering chat.
11820
+ ...turn.turnInstructions || mcpAvailabilityInstructions ? {
11821
+ turnInstructions: [turn.turnInstructions, mcpAvailabilityInstructions].filter((value) => Boolean(value)).join(" ")
11822
+ } : {},
11823
+ ...workspaceEnvironmentOption,
11824
+ // RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
11825
+ // the frozen version carries a non-empty script), and the rig credential
11826
+ // hooks. All absent for a rig-less turn (byte-for-byte today).
11827
+ ...rigVersion && rigName ? {
11828
+ rig: { name: rigName, version: rigVersion.version },
11829
+ ...rigVersion.setupScript && rigVersion.setupScript.trim().length > 0 ? {
11830
+ rigSetup: {
11831
+ rigId: session.rigId,
11832
+ versionId: rigVersion.id,
11833
+ rigName,
11834
+ script: rigVersion.setupScript,
11835
+ timeoutMs: runSettings.rigSetupTimeoutMs,
11836
+ contentHash: rigProviderImageContentHash({
11837
+ backend: turn.sandboxBackend,
11838
+ sourceImage: rigProviderImageSourceImage(
11839
+ logicalSandboxSettings,
11840
+ turn.sandboxBackend
11841
+ ),
11842
+ definition: rigVersion
11843
+ })
11844
+ }
11845
+ } : {},
11846
+ ...rigVersion.credentialHooks.length > 0 ? { rigCredentialHookIds: rigVersion.credentialHooks } : {}
11847
+ } : {}
11848
+ });
11849
+ } catch (error) {
11850
+ agentConstructionOutcome = "failed";
11851
+ throw error;
11852
+ } finally {
11853
+ recordTurnStartupPhase(observability, {
11854
+ phase: "agent_construction",
11855
+ provider: turnExecutionPolicy.providerId,
11856
+ backend: activeSandboxBackend ?? groupBoxBackend,
11857
+ outcome: agentConstructionOutcome,
11858
+ durationSeconds: (performance.now() - agentConstructionStartedAt) / 1e3
11859
+ });
11860
+ }
11861
+ })();
11862
+ const postAgentPreparationStartedAt = performance.now();
11542
11863
  if (modelRunSettings.sandboxBackend !== "none" && toolCancellationFenceRef.current === null) {
11543
11864
  throw new Error(
11544
11865
  "Sandbox agent construction did not install the mandatory turn tool cancellation fence"
@@ -11585,7 +11906,11 @@ function createRunAgentTurnActivity(services) {
11585
11906
  environment: sandboxEnvironment,
11586
11907
  ...runSettings.modalImageRef ?? runSettings.dockerImage ? {
11587
11908
  image: runSettings.modalImageRef ?? runSettings.dockerImage
11588
- } : {}
11909
+ } : {},
11910
+ // The lazy acquire must enforce the same frozen rig authority
11911
+ // as the eager acquire; otherwise a warm box for another rig
11912
+ // can bypass the shared-state conflict/rotation fence.
11913
+ ...rigVersion ? { rigVersionId: rigVersion.id } : {}
11589
11914
  },
11590
11915
  "turn",
11591
11916
  lazyHolderId
@@ -11983,58 +12308,96 @@ function createRunAgentTurnActivity(services) {
11983
12308
  return claimedResult({ status: "idle" });
11984
12309
  }
11985
12310
  }
12311
+ recordTurnStartupPhase(observability, {
12312
+ phase: "post_agent_preparation",
12313
+ provider: turnExecutionPolicy.providerId,
12314
+ backend: activeSandboxBackend ?? groupBoxBackend,
12315
+ outcome: "completed",
12316
+ durationSeconds: (performance.now() - postAgentPreparationStartedAt) / 1e3
12317
+ });
11986
12318
  let fileMaterializationFailures = [];
11987
12319
  let fileDownloadsMaterializedForRun = false;
11988
12320
  if (resolvedSandbox && setupBoxSession && fileResourceDownloads.length > 0) {
11989
- const boxInstanceId = resolvedSandbox.established.instanceId;
12321
+ const fileMaterializationStartedAt = performance.now();
12322
+ let fileMaterializationOutcome = "completed";
11990
12323
  const cacheMaterialization = resolvedSandbox.established.backendId !== "selfhosted";
11991
- const alreadyMaterialized = cacheMaterialization ? await getMaterializedSandboxFileResources(db, {
11992
- accountId: input.accountId,
11993
- workspaceId: input.workspaceId,
11994
- sandboxGroupId: session.sandboxGroupId,
11995
- expectedEpoch: resolvedSandbox.leaseEpoch,
11996
- instanceId: boxInstanceId
11997
- }) : /* @__PURE__ */ new Set();
11998
- const downloadsToMaterialize = filterUnmaterializedSandboxFileDownloads(
11999
- fileResourceDownloads,
12000
- alreadyMaterialized
12001
- );
12002
- const runAs = sandboxRunAs(runSettings);
12003
- if (downloadsToMaterialize.length > 0) {
12004
- const materialized = await runWorkspaceMutationForSandbox(
12005
- resolvedSandbox,
12006
- "fileMaterialization",
12007
- async () => await materializeSandboxFileDownloads(
12008
- setupBoxSession,
12009
- downloadsToMaterialize,
12010
- {
12011
- onRuntimeEvent: async (event) => {
12012
- await publish([{ type: event.type, payload: event.payload }], true);
12013
- },
12014
- ...runAs ? { runAs } : {},
12015
- ...toolCancellationFenceRef.current ? {
12016
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
12017
- toolCancellationFenceRef.current
12018
- )
12019
- } : {}
12020
- }
12021
- )
12324
+ let fileMaterializationCache = cacheMaterialization ? "miss" : "disabled";
12325
+ try {
12326
+ const boxInstanceId = resolvedSandbox.established.instanceId;
12327
+ const alreadyMaterialized = cacheMaterialization ? await getMaterializedSandboxFileResources(db, {
12328
+ accountId: input.accountId,
12329
+ workspaceId: input.workspaceId,
12330
+ sandboxGroupId: session.sandboxGroupId,
12331
+ expectedEpoch: resolvedSandbox.leaseEpoch,
12332
+ instanceId: boxInstanceId
12333
+ }) : /* @__PURE__ */ new Set();
12334
+ const downloadsToMaterialize = filterUnmaterializedSandboxFileDownloads(
12335
+ fileResourceDownloads,
12336
+ alreadyMaterialized
12022
12337
  );
12023
- fileMaterializationFailures = materialized.failures;
12024
- const failedFileIds = new Set(materialized.failures.map((failure) => failure.fileId));
12025
- const succeededFileIds = downloadsToMaterialize.map((download) => download.fileId).filter((fileId) => !failedFileIds.has(fileId));
12026
- if (cacheMaterialization && succeededFileIds.length > 0) {
12027
- await markSandboxFileResourcesMaterialized(db, {
12028
- accountId: input.accountId,
12029
- workspaceId: input.workspaceId,
12030
- sandboxGroupId: session.sandboxGroupId,
12031
- expectedEpoch: resolvedSandbox.leaseEpoch,
12032
- instanceId: boxInstanceId,
12033
- fileIds: succeededFileIds
12034
- });
12338
+ if (cacheMaterialization && downloadsToMaterialize.length === 0) {
12339
+ fileMaterializationCache = "hit";
12340
+ }
12341
+ const runAs = sandboxRunAs(runSettings);
12342
+ if (downloadsToMaterialize.length > 0) {
12343
+ const materialized = await runWorkspaceMutationForSandbox(
12344
+ resolvedSandbox,
12345
+ "fileMaterialization",
12346
+ async () => await materializeSandboxFileDownloads(
12347
+ setupBoxSession,
12348
+ downloadsToMaterialize,
12349
+ {
12350
+ onRuntimeEvent: async (event) => {
12351
+ await publish([{ type: event.type, payload: event.payload }], true);
12352
+ },
12353
+ ...runAs ? { runAs } : {},
12354
+ ...toolCancellationFenceRef.current ? {
12355
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
12356
+ toolCancellationFenceRef.current
12357
+ )
12358
+ } : {}
12359
+ }
12360
+ )
12361
+ );
12362
+ fileMaterializationFailures = materialized.failures;
12363
+ fileMaterializationOutcome = sandboxFileMaterializationOutcome(materialized.failures);
12364
+ const failedFileIds = new Set(materialized.failures.map((failure) => failure.fileId));
12365
+ const succeededFileIds = downloadsToMaterialize.map((download) => download.fileId).filter((fileId) => !failedFileIds.has(fileId));
12366
+ if (cacheMaterialization && succeededFileIds.length > 0) {
12367
+ await markSandboxFileResourcesMaterialized(db, {
12368
+ accountId: input.accountId,
12369
+ workspaceId: input.workspaceId,
12370
+ sandboxGroupId: session.sandboxGroupId,
12371
+ expectedEpoch: resolvedSandbox.leaseEpoch,
12372
+ instanceId: boxInstanceId,
12373
+ fileIds: succeededFileIds
12374
+ });
12375
+ }
12035
12376
  }
12377
+ fileDownloadsMaterializedForRun = true;
12378
+ } catch (error) {
12379
+ fileMaterializationOutcome = "failed";
12380
+ throw error;
12381
+ } finally {
12382
+ recordTurnStartupPhase(observability, {
12383
+ phase: "file_materialization",
12384
+ provider: turnExecutionPolicy.providerId,
12385
+ backend: activeSandboxBackend ?? groupBoxBackend,
12386
+ outcome: fileMaterializationOutcome,
12387
+ durationSeconds: (performance.now() - fileMaterializationStartedAt) / 1e3,
12388
+ count: fileResourceDownloads.length,
12389
+ cache: fileMaterializationCache
12390
+ });
12036
12391
  }
12037
- fileDownloadsMaterializedForRun = true;
12392
+ } else {
12393
+ recordTurnStartupPhase(observability, {
12394
+ phase: "file_materialization",
12395
+ provider: turnExecutionPolicy.providerId,
12396
+ backend: activeSandboxBackend ?? groupBoxBackend,
12397
+ outcome: "completed",
12398
+ durationSeconds: 0,
12399
+ count: 0
12400
+ });
12038
12401
  }
12039
12402
  const requiredVideoFileIds = new Set(requiredGeneratedVideoFiles.map((file) => file.fileId));
12040
12403
  const failedRequiredVideo = fileMaterializationFailures.find(
@@ -12076,28 +12439,86 @@ function createRunAgentTurnActivity(services) {
12076
12439
  }
12077
12440
  let runInput = null;
12078
12441
  const prepareRunAttemptInput = async () => {
12079
- const prepared = await turnInput(db, runtime, agent, trigger, {
12080
- turnId: activeTurnId,
12081
- recovering: turn.executionGeneration > 1,
12082
- ...unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {},
12083
- ...runCredentialsNote ? { runCredentialsNote } : {},
12084
- providerApi,
12085
- projectCanonicalHistory: generatedImageHistoryProjector,
12086
- materializeModelHistory: materializeScreenshotHistory,
12087
- materializeSerializedRunState: materializeScreenshotRunState,
12088
- projectModelHistory: modelHistoryProjector
12089
- });
12090
- for (const receipt of generatedImageReceiptsByArtifactId.values()) {
12091
- await materializeGeneratedImage(receipt);
12442
+ const historyPreparationStartedAt = performance.now();
12443
+ let historyPreparationOutcome = "completed";
12444
+ let preparedHistoryCount = null;
12445
+ try {
12446
+ const prepared = await turnInput(db, runtime, agent, trigger, {
12447
+ turnId: activeTurnId,
12448
+ recovering: turn.executionGeneration > 1,
12449
+ ...unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {},
12450
+ ...runCredentialsNote ? { runCredentialsNote } : {},
12451
+ providerApi,
12452
+ projectCanonicalHistory: generatedImageHistoryProjector,
12453
+ materializeModelHistory: materializeScreenshotHistory,
12454
+ materializeSerializedRunState: materializeScreenshotRunState,
12455
+ projectModelHistory: modelHistoryProjector,
12456
+ onPreparationPhase: (measurement) => {
12457
+ recordTurnStartupPhase(observability, {
12458
+ phase: `history_${measurement.phase}`,
12459
+ provider: turnExecutionPolicy.providerId,
12460
+ backend: activeSandboxBackend ?? groupBoxBackend,
12461
+ outcome: measurement.outcome,
12462
+ durationSeconds: measurement.durationSeconds
12463
+ });
12464
+ }
12465
+ });
12466
+ const generatedImageMaterializationStartedAt = performance.now();
12467
+ let generatedImageMaterializationOutcome = "completed";
12468
+ try {
12469
+ for (const receipt of generatedImageReceiptsByArtifactId.values()) {
12470
+ await materializeGeneratedImage(receipt);
12471
+ }
12472
+ } catch (error) {
12473
+ generatedImageMaterializationOutcome = "failed";
12474
+ throw error;
12475
+ } finally {
12476
+ recordTurnStartupPhase(observability, {
12477
+ phase: "history_generated_image_materialization",
12478
+ provider: turnExecutionPolicy.providerId,
12479
+ backend: activeSandboxBackend ?? groupBoxBackend,
12480
+ outcome: generatedImageMaterializationOutcome,
12481
+ durationSeconds: (performance.now() - generatedImageMaterializationStartedAt) / 1e3,
12482
+ count: generatedImageReceiptsByArtifactId.size
12483
+ });
12484
+ }
12485
+ runInput = prepared.input;
12486
+ providerArtifactCandidates = prepared.providerArtifactCandidates;
12487
+ persistedHistoryCount = prepared.persistedHistoryCount;
12488
+ preparedHistoryCount = prepared.persistedHistoryCount;
12489
+ const historyPositionStartedAt = performance.now();
12490
+ let historyPositionOutcome = "completed";
12491
+ try {
12492
+ nextHistoryPosition = await nextSessionHistoryPosition(
12493
+ db,
12494
+ input.workspaceId,
12495
+ input.sessionId
12496
+ );
12497
+ } catch (error) {
12498
+ historyPositionOutcome = "failed";
12499
+ throw error;
12500
+ } finally {
12501
+ recordTurnStartupPhase(observability, {
12502
+ phase: "history_position_load",
12503
+ provider: turnExecutionPolicy.providerId,
12504
+ backend: activeSandboxBackend ?? groupBoxBackend,
12505
+ outcome: historyPositionOutcome,
12506
+ durationSeconds: (performance.now() - historyPositionStartedAt) / 1e3
12507
+ });
12508
+ }
12509
+ } catch (error) {
12510
+ historyPreparationOutcome = "failed";
12511
+ throw error;
12512
+ } finally {
12513
+ recordTurnStartupPhase(observability, {
12514
+ phase: "history_preparation",
12515
+ provider: turnExecutionPolicy.providerId,
12516
+ backend: activeSandboxBackend ?? groupBoxBackend,
12517
+ outcome: historyPreparationOutcome,
12518
+ durationSeconds: (performance.now() - historyPreparationStartedAt) / 1e3,
12519
+ count: preparedHistoryCount
12520
+ });
12092
12521
  }
12093
- runInput = prepared.input;
12094
- providerArtifactCandidates = prepared.providerArtifactCandidates;
12095
- persistedHistoryCount = prepared.persistedHistoryCount;
12096
- nextHistoryPosition = await nextSessionHistoryPosition(
12097
- db,
12098
- input.workspaceId,
12099
- input.sessionId
12100
- );
12101
12522
  };
12102
12523
  const forceContextCompaction = async (triggerLabel, recoverySignalTokens) => {
12103
12524
  const outcome = await waitForTurnOperation(
@@ -12140,6 +12561,7 @@ function createRunAgentTurnActivity(services) {
12140
12561
  return outcome;
12141
12562
  };
12142
12563
  const modelResponseState = createModelResponseEventState(claimedModelUsageSourceKeys);
12564
+ let workerPreparationTotalRecorded = false;
12143
12565
  const runStreamAttempt = async () => {
12144
12566
  if (!runInput) {
12145
12567
  throw new Error("Run input was not prepared");
@@ -12154,102 +12576,154 @@ function createRunAgentTurnActivity(services) {
12154
12576
  const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
12155
12577
  const runStreamOnce = async () => {
12156
12578
  if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
12157
- const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
12158
- await attachRunCredentialRenewal(
12159
- eagerSetupSession,
12160
- resolvedSandbox
12161
- );
12162
- const eagerCredentialSetupSession = initialRunCredentialMaterial ? withRunCredentialsSession(eagerSetupSession, input.sessionId) : eagerSetupSession;
12163
- await runWorkspaceMutationForSandbox(
12164
- resolvedSandbox,
12165
- "eagerOwnedSandboxSetup",
12166
- async () => await runOwnedSandboxSetup(
12167
- agent,
12168
- ownedEstablished.session,
12169
- eagerCredentialSetupSession,
12170
- {
12171
- settings: runSettings,
12172
- environment: sandboxEnvironment,
12173
- preparedInput: runInput,
12174
- ...fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {},
12175
- onRuntimeEvent: async (event) => {
12176
- await renewServingCredentialLease("runtime_event");
12177
- if (servingCredentialLeaseLost()) {
12178
- throw new Error("Provider credential lease expired during sandbox setup");
12179
- }
12180
- await publish([{ type: event.type, payload: event.payload }], true);
12181
- },
12182
- ...toolCancellationFenceRef.current ? {
12183
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
12184
- toolCancellationFenceRef.current
12185
- )
12186
- } : {}
12187
- }
12188
- )
12189
- );
12190
- await attachGitCredentialRenewal(
12191
- eagerSetupSession,
12192
- initialGitCredentials
12193
- );
12579
+ const ownedSandboxSetupStartedAt = performance.now();
12580
+ let ownedSandboxSetupOutcome = "completed";
12581
+ try {
12582
+ const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
12583
+ await attachRunCredentialRenewal(
12584
+ eagerSetupSession,
12585
+ resolvedSandbox
12586
+ );
12587
+ const eagerCredentialSetupSession = initialRunCredentialMaterial ? withRunCredentialsSession(eagerSetupSession, input.sessionId) : eagerSetupSession;
12588
+ await runWorkspaceMutationForSandbox(
12589
+ resolvedSandbox,
12590
+ "eagerOwnedSandboxSetup",
12591
+ async () => await runOwnedSandboxSetup(
12592
+ agent,
12593
+ ownedEstablished.session,
12594
+ eagerCredentialSetupSession,
12595
+ {
12596
+ settings: runSettings,
12597
+ environment: sandboxEnvironment,
12598
+ preparedInput: runInput,
12599
+ ...fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {},
12600
+ onRuntimeEvent: async (event) => {
12601
+ await renewServingCredentialLease("runtime_event");
12602
+ if (servingCredentialLeaseLost()) {
12603
+ throw new Error("Provider credential lease expired during sandbox setup");
12604
+ }
12605
+ await publish([{ type: event.type, payload: event.payload }], true);
12606
+ },
12607
+ ...toolCancellationFenceRef.current ? {
12608
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
12609
+ toolCancellationFenceRef.current
12610
+ )
12611
+ } : {}
12612
+ }
12613
+ )
12614
+ );
12615
+ await attachGitCredentialRenewal(
12616
+ eagerSetupSession,
12617
+ initialGitCredentials
12618
+ );
12619
+ } catch (error) {
12620
+ ownedSandboxSetupOutcome = "failed";
12621
+ throw error;
12622
+ } finally {
12623
+ recordTurnStartupPhase(observability, {
12624
+ phase: "owned_sandbox_setup",
12625
+ provider: turnExecutionPolicy.providerId,
12626
+ backend: activeSandboxBackend ?? groupBoxBackend,
12627
+ outcome: ownedSandboxSetupOutcome,
12628
+ durationSeconds: (performance.now() - ownedSandboxSetupStartedAt) / 1e3,
12629
+ count: fileResourceDownloads.length
12630
+ });
12631
+ }
12194
12632
  }
12195
12633
  recordCompanyBrainContributionReceiptOnce();
12196
- modelRequestStarted = true;
12197
- return await runtime.runStream(agent, runInput, modelRunSettings, {
12198
- signal: runtimeCancellationSignal,
12199
- sandboxEnvironment,
12200
- onRuntimeEvent: async (event) => {
12201
- await renewServingCredentialLease("runtime_event");
12202
- if (servingCredentialLeaseLost()) {
12203
- throw new Error("Provider credential lease expired during the active turn");
12204
- }
12205
- await publish([{ type: event.type, payload: event.payload }], true);
12206
- },
12207
- // P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
12208
- // keystone). Absent when the flag is off -> legacy build-and-discard.
12209
- ...ownedEstablished ? {
12210
- ownedSandbox: {
12211
- client: ownedEstablished.client,
12212
- session: ownedEstablished.session,
12213
- ...resolvedSandbox?.established.sessionState ? {
12214
- sessionState: resolvedSandbox.established.sessionState
12215
- } : {},
12216
- // Pin platform setup (hooks + file materialization) to the un-proxied
12217
- // established box — never through the routing proxy, which would
12218
- // re-route those execs onto a machine swapped in mid-turn.
12219
- ...setupBoxSession ? { setupSession: setupBoxSession } : {},
12220
- ...fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {},
12221
- // Both owned paths execute setup outside runStream: eager just
12222
- // above under its exact admission, lazy in the provisioner.
12223
- deferredSetup: true
12224
- }
12225
- } : {},
12226
- ...activeSandboxBackend !== "selfhosted" && sandboxCodemodeToken && sandboxCodemodeTokenExpiresAt && !lazyOwnedSandbox ? {
12227
- onCodemodeTokenSessionReady: async (tokenSession) => {
12228
- const renewalSession = setupBoxSession ?? tokenSession;
12229
- await attachCodemodeTokenRenewal(renewalSession);
12230
- }
12231
- } : {},
12232
- ...runCredentialResolver ? {
12233
- runCredentialSessionId: input.sessionId,
12234
- ...!lazyOwnedSandbox ? {
12235
- onRunCredentialSessionReady: async (credentialSession) => {
12236
- const pinnedCredentialSession = setupBoxSession ? setupBoxSession : credentialSession;
12237
- await attachRunCredentialRenewal(pinnedCredentialSession);
12634
+ const providerDispatchStartedAt = performance.now();
12635
+ let providerDispatchOutcome = "completed";
12636
+ try {
12637
+ if (!workerPreparationTotalRecorded) {
12638
+ workerPreparationTotalRecorded = true;
12639
+ recordTurnWorkerPreparationTotal(observability, {
12640
+ provider: turnExecutionPolicy.providerId,
12641
+ backend: activeSandboxBackend ?? groupBoxBackend,
12642
+ outcome: "completed",
12643
+ durationSeconds: (performance.now() - workerPreparationStartedAt) / 1e3
12644
+ });
12645
+ }
12646
+ modelRequestStarted = true;
12647
+ return await runtime.runStream(agent, runInput, modelRunSettings, {
12648
+ signal: runtimeCancellationSignal,
12649
+ sandboxEnvironment,
12650
+ onRuntimeEvent: async (event) => {
12651
+ await renewServingCredentialLease("runtime_event");
12652
+ if (servingCredentialLeaseLost()) {
12653
+ throw new Error("Provider credential lease expired during the active turn");
12238
12654
  }
12239
- } : {}
12240
- } : {},
12241
- ...modelRunSettings.sandboxBackend !== "none" ? {
12242
- onSandboxSessionReady: async (sandboxSession) => {
12243
- sdkOwnedSandboxSession = sandboxSession;
12244
- for (const receipt of generatedImageReceiptsByArtifactId.values()) {
12245
- await materializeGeneratedImageInOwnedSdkSession(receipt, sandboxSession);
12655
+ await publish([{ type: event.type, payload: event.payload }], true);
12656
+ },
12657
+ // P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
12658
+ // keystone). Absent when the flag is off -> legacy build-and-discard.
12659
+ ...ownedEstablished ? {
12660
+ ownedSandbox: {
12661
+ client: ownedEstablished.client,
12662
+ session: ownedEstablished.session,
12663
+ ...resolvedSandbox?.established.sessionState ? {
12664
+ sessionState: resolvedSandbox.established.sessionState
12665
+ } : {},
12666
+ // Pin platform setup (hooks + file materialization) to the un-proxied
12667
+ // established box — never through the routing proxy, which would
12668
+ // re-route those execs onto a machine swapped in mid-turn.
12669
+ ...setupBoxSession ? { setupSession: setupBoxSession } : {},
12670
+ ...fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {},
12671
+ // Both owned paths execute setup outside runStream: eager just
12672
+ // above under its exact admission, lazy in the provisioner.
12673
+ deferredSetup: true
12246
12674
  }
12247
- }
12248
- } : {},
12249
- contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
12250
- contextCompactionRequested: () => isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
12251
- ...toolCancellationFenceRef.current ? { turnToolCancellationFence: toolCancellationFenceRef.current } : {}
12252
- });
12675
+ } : {},
12676
+ ...activeSandboxBackend !== "selfhosted" && sandboxCodemodeToken && sandboxCodemodeTokenExpiresAt && !lazyOwnedSandbox ? {
12677
+ onCodemodeTokenSessionReady: async (tokenSession) => {
12678
+ const renewalSession = setupBoxSession ?? tokenSession;
12679
+ await attachCodemodeTokenRenewal(renewalSession);
12680
+ }
12681
+ } : {},
12682
+ ...runCredentialResolver ? {
12683
+ runCredentialSessionId: input.sessionId,
12684
+ ...!lazyOwnedSandbox ? {
12685
+ onRunCredentialSessionReady: async (credentialSession) => {
12686
+ const pinnedCredentialSession = setupBoxSession ? setupBoxSession : credentialSession;
12687
+ await attachRunCredentialRenewal(pinnedCredentialSession);
12688
+ }
12689
+ } : {}
12690
+ } : {},
12691
+ ...modelRunSettings.sandboxBackend !== "none" ? {
12692
+ onSandboxSessionReady: async (sandboxSession) => {
12693
+ sdkOwnedSandboxSession = sandboxSession;
12694
+ for (const receipt of generatedImageReceiptsByArtifactId.values()) {
12695
+ await materializeGeneratedImageInOwnedSdkSession(receipt, sandboxSession);
12696
+ }
12697
+ }
12698
+ } : {},
12699
+ contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
12700
+ contextCompactionRequested: () => isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
12701
+ onModelPreparationPhase: (measurement) => {
12702
+ recordTurnStartupPhase(observability, {
12703
+ phase: `model_prepare_${measurement.phase}`,
12704
+ provider: turnExecutionPolicy.providerId,
12705
+ backend: activeSandboxBackend ?? groupBoxBackend,
12706
+ outcome: measurement.outcome,
12707
+ durationSeconds: measurement.durationSeconds,
12708
+ ...measurement.count === void 0 ? {} : { count: measurement.count }
12709
+ });
12710
+ },
12711
+ ...toolCancellationFenceRef.current ? {
12712
+ turnToolCancellationFence: toolCancellationFenceRef.current
12713
+ } : {}
12714
+ });
12715
+ } catch (error) {
12716
+ providerDispatchOutcome = "failed";
12717
+ throw error;
12718
+ } finally {
12719
+ recordTurnStartupPhase(observability, {
12720
+ phase: "provider_dispatch",
12721
+ provider: turnExecutionPolicy.providerId,
12722
+ backend: activeSandboxBackend ?? groupBoxBackend,
12723
+ outcome: providerDispatchOutcome,
12724
+ durationSeconds: (performance.now() - providerDispatchStartedAt) / 1e3
12725
+ });
12726
+ }
12253
12727
  };
12254
12728
  if (codexLeaseLost) {
12255
12729
  throw new Error("Codex credential lease expired before the model run");
@@ -12270,11 +12744,30 @@ function createRunAgentTurnActivity(services) {
12270
12744
  onFlush: ({ events, durationSeconds }) => recordBatchFlush(observability, { events, durationSeconds })
12271
12745
  }
12272
12746
  );
12747
+ const streamBootstrapStartedAt = performance.now();
12748
+ let streamBootstrapRecorded = false;
12749
+ const recordStreamBootstrap = () => {
12750
+ if (streamBootstrapRecorded) return;
12751
+ streamBootstrapRecorded = true;
12752
+ recordTurnStartupPhase(observability, {
12753
+ phase: "stream_bootstrap",
12754
+ provider: turnExecutionPolicy.providerId,
12755
+ backend: activeSandboxBackend ?? groupBoxBackend,
12756
+ outcome: "completed",
12757
+ durationSeconds: (performance.now() - streamBootstrapStartedAt) / 1e3,
12758
+ count: turnTools.length
12759
+ });
12760
+ };
12761
+ if (!firstModelRequestPreparationRecorded) {
12762
+ firstModelRequestPreparationStartedAt = performance.now();
12763
+ firstModelRequestCheckpointAt = firstModelRequestPreparationStartedAt;
12764
+ }
12273
12765
  const iterator = stream.toStream()[Symbol.asyncIterator]();
12274
12766
  let streamDone = false;
12275
12767
  try {
12276
12768
  while (true) {
12277
12769
  const next = await nextStreamEvent(iterator, activityContext);
12770
+ recordStreamBootstrap();
12278
12771
  if (next.done) {
12279
12772
  streamDone = true;
12280
12773
  break;
@@ -13830,7 +14323,10 @@ function createRunAgentTurnActivity(services) {
13830
14323
  }
13831
14324
  activityStatus = "failed";
13832
14325
  activityError = error;
13833
- if (!publish || !turnId || !turnStartedPublished) {
14326
+ if (!turnId) {
14327
+ throw preClaimAdmissionFailure(error);
14328
+ }
14329
+ if (!publish || !turnStartedPublished) {
13834
14330
  throw error;
13835
14331
  }
13836
14332
  await flushRuntimeBatcher();
@@ -16033,4 +16529,4 @@ export {
16033
16529
  createTurnActivities,
16034
16530
  createTurnActivitiesFromServices
16035
16531
  };
16036
- //# sourceMappingURL=activities-turn-6EBXGGLS.js.map
16532
+ //# sourceMappingURL=activities-turn-IDS3WEHA.js.map