@opengeni/worker-bundle 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2290 @@
1
+ import {
2
+ claimNextQueuedTurn as claimNextQueuedTurnDb,
3
+ createTurn,
4
+ applyCreditDebitUpToBalance,
5
+ finishTurn,
6
+ getBillingBalance,
7
+ getSandbox,
8
+ readActiveSandbox,
9
+ requireFile,
10
+ getSessionEvent,
11
+ getSessionGoal,
12
+ getSessionTurn,
13
+ isCodexBilledTurn,
14
+ workspaceCodexSubscriptionActive,
15
+ getCodexRotationSettings,
16
+ listCodexAccountStatuses,
17
+ fetchCodexUsageForAccount,
18
+ getSessionCodexState,
19
+ recordSessionActiveCodexCredential,
20
+ recordCodexAccountUsage,
21
+ recordCodexAccountConnectors,
22
+ setActiveCodexCredential,
23
+ setCodexCredentialExhausted,
24
+ countConsecutiveReactiveRotations,
25
+ requireSession,
26
+ recordUsageEvent,
27
+ appendSessionHistoryItems,
28
+ consumeSessionCompactionRequest,
29
+ countSessionHistoryItems,
30
+ getActiveSessionHistoryItems,
31
+ nextSessionHistoryPosition,
32
+ requeuePreemptedTurn,
33
+ saveRunState,
34
+ upsertSandboxSessionEnvelope,
35
+ setSessionStatus,
36
+ setSessionLastInputTokens,
37
+ sumUsageQuantity,
38
+ heartbeatLeaseHolder,
39
+ accrueWarmSeconds,
40
+ SandboxLeaseSupersededError,
41
+ type AppendEventInput,
42
+ } from "@opengeni/db";
43
+ import { appendAndPublishEvents } from "@opengeni/events";
44
+ import {
45
+ sandboxStateEntryFromRunState,
46
+ agentsErrorRunState,
47
+ maxTurnsExceededRunState,
48
+ modelResponseUsageFromSdkEvent,
49
+ normalizeSdkEvent,
50
+ sanitizeHistoryItemsForModel,
51
+ summarizeForCompaction,
52
+ type SandboxFileDownload,
53
+ type OpenGeniRuntime,
54
+ } from "@opengeni/runtime";
55
+ import { calculateModelUsageCostMicros, configuredModelPricing, configuredStaticUsageLimits, sandboxWarmRateMicrosPerSecond, type ModelUsageInput, type Settings } from "@opengeni/config";
56
+ import { CancelledFailure } from "@temporalio/activity";
57
+ import { settingsWithCodexCredential, settingsWithEnabledCapabilityMcpServers } from "./capabilities";
58
+ import { chooseRotationActive, computeIdleDelayMs, computeReactiveRotationResume, type CodexRotationStrategy, type RotationDecision } from "./codex-rotation";
59
+ import type { CodexAccountStatus } from "@opengeni/db";
60
+ import { buildCodexTokenResolver } from "./codex-auth";
61
+ import {
62
+ buildModelResolver,
63
+ CODEX_CLIENT_VERSION,
64
+ CODEX_FALLBACK_MODEL_SLUGS,
65
+ classifyCodexUsageLimitError,
66
+ codexRequestStorage,
67
+ type CodexRequestContext,
68
+ type CodexUsageHeaderSnapshot,
69
+ } from "@opengeni/codex";
70
+ import {
71
+ mergeResourceRefs,
72
+ mergeToolRefs,
73
+ } from "./common";
74
+ import { maybeCompactContext } from "./context-compaction";
75
+ import { loadWorkspaceEnvironmentForRunWithCredentials, sandboxEnvironmentForRun } from "./environment";
76
+ import { withCodexAppsTool, withFirstPartyTools } from "./goals";
77
+ import { resolveWorkspaceAgentInstructions, resolveWorkspacePackRuntime, settingsWithPackSandboxImage } from "./packs";
78
+ import { notifyParentOfChildTerminal } from "./parent-wake";
79
+ import { createSecretRedactor, identityRedactor } from "./redaction";
80
+ import { applyCodexHistoryStrip, turnInput, type TurnCodexAccount } from "./run-input";
81
+ import {
82
+ createRuntimeBatcher,
83
+ currentActivityContext,
84
+ nextStreamEvent,
85
+ startActivityHeartbeat,
86
+ } from "./streaming";
87
+ import type {
88
+ ActivityServices,
89
+ RunAgentTurnInput,
90
+ RunAgentTurnResult,
91
+ } from "./types";
92
+ import { resumeBoxForTurn, acquireSelfhostedLeaseForTurn, type ResumedTurnSandbox } from "../sandbox-resume";
93
+ import { wrapTurnBoxWithRouting, establishSelfhostedTurnSession, routingEnabled } from "../sandbox-routing";
94
+ import { beginRecording, discardRecording, finalizeRecording, type ActiveRecording } from "./recording";
95
+ import { createObjectStorage, type ObjectStorage } from "@opengeni/storage";
96
+ import { desktopCapableBackend, sandboxRunAs } from "@opengeni/runtime";
97
+ import { CAPABILITY_DESCRIPTORS, type ResourceRef } from "@opengeni/contracts";
98
+ import { randomUUID } from "node:crypto";
99
+
100
+ // How long the session workflow holds the loop after a retryable provider
101
+ // failure before the goal continuation re-enters the model. Azure/OpenAI TPM
102
+ // throttling is minute-granular; anything shorter mostly burns continuation
103
+ // budget against the same window.
104
+ export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
105
+
106
+ /**
107
+ * Resolve which Codex account a turn runs on (multi-account P1): session-pin >
108
+ * workspace-active. No rotation in P1. The selected id must still be in the
109
+ * connected set — a disconnected pin was FK-nulled, so a stale id can't appear,
110
+ * but we guard anyway. Returns null when there is no usable account (the turn
111
+ * then fails with the existing relogin error path).
112
+ */
113
+ export function selectCodexCredentialForTurn(args: {
114
+ sessionPinnedCredentialId: string | null;
115
+ activeCredentialId: string | null;
116
+ connectedIds: Set<string>;
117
+ }): string | null {
118
+ const { sessionPinnedCredentialId: pin, activeCredentialId: active, connectedIds } = args;
119
+ if (pin && connectedIds.has(pin)) {
120
+ return pin;
121
+ }
122
+ if (active && connectedIds.has(active)) {
123
+ return active;
124
+ }
125
+ return null;
126
+ }
127
+
128
+ // Resume notice fed to the model when a turn re-enters after a graceful
129
+ // worker shutdown (deploy/rollout restart) checkpointed it mid-flight. The
130
+ // agent must not blindly replay side effects it cannot see in its history:
131
+ // at most the single in-flight model step was lost, and anything it started
132
+ // there may or may not have completed.
133
+ export const WORKER_SHUTDOWN_RESUME_TEXT = [
134
+ "[TURN RESUMED AFTER WORKER RESTART] The platform worker running this turn restarted (deploy/rollout) before the turn finished.",
135
+ "Your conversation history above, including this turn's earlier work, is preserved up to the last checkpoint; at most the single in-flight model step after it was lost.",
136
+ "Continue the original task from where it left off. Before repeating any action with side effects, check whether it already happened.",
137
+ ].join("\n");
138
+
139
+ // Resume notice for the harder variant: the worker died WITHOUT running the
140
+ // graceful checkpoint (SIGKILL, OOM, node loss — surfaced to the workflow as
141
+ // a heartbeat-timeout activity failure). Conversation truth is still
142
+ // dual-written after every model response, so the bound is the same order as
143
+ // a graceful preempt, but nothing after the last reconcile was captured and
144
+ // the agent must be even more suspicious about repeated side effects.
145
+ export const WORKER_DEATH_RESUME_TEXT = [
146
+ "[TURN RESUMED AFTER WORKER CRASH] The platform worker running this turn died before the turn finished.",
147
+ "Your conversation history above, including this turn's earlier work, is preserved up to the last checkpoint; anything in flight after that checkpoint was lost.",
148
+ "Continue the original task from where it left off. Before repeating any action with side effects, check whether it already happened.",
149
+ ].join("\n");
150
+
151
+ /**
152
+ * True when this activity attempt was cancelled because its hosting worker is
153
+ * shutting down gracefully (SIGTERM during a deploy), as opposed to a
154
+ * workflow-requested cancellation (user interrupt) or a server-side timeout.
155
+ */
156
+ export function isWorkerShutdownCancellation(error: unknown): boolean {
157
+ return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
158
+ }
159
+
160
+ /**
161
+ * Compute the conversation-truth rows a reconcile pass should append, given the
162
+ * SDK's current `state.history` and the count already persisted.
163
+ *
164
+ * `state.history` is a computed getter that runs the SDK's orphan-tool-call
165
+ * pruning on every access, so it is non-monotonic: a `function_call` with no
166
+ * settling result yet is transiently absent and a later access yields a
167
+ * different, possibly shorter/reordered list. The old code sliced this list by
168
+ * a blind length watermark and appended at fixed positions with
169
+ * onConflictDoNothing, which could freeze a position with one shape and later
170
+ * persist a `function_call_result` whose `function_call` had been pruned away in
171
+ * an earlier slice — the orphaned tool output that 400s the Responses API and
172
+ * bricks the session on every replay.
173
+ *
174
+ * Defending: sanitize the full current history into an API-valid sequence (the
175
+ * same pure rules the read path uses), then append only the new tail beyond the
176
+ * watermark. A trailing dangling call is dropped here and re-evaluated next
177
+ * pass once its result lands, so a call and its result are written together at
178
+ * consecutive positions and a result is never persisted without its call. The
179
+ * watermark advances to the sanitized length — never past anything unwritten —
180
+ * so a non-monotonic history can never desync it. When previously-persisted
181
+ * rows already exceed the sanitized length (e.g. legacy orphans written before
182
+ * this fix), nothing new is appended and the watermark holds steady.
183
+ */
184
+ /**
185
+ * Stable+unique usage source key for one model call, used to build the per-call
186
+ * idempotency key (`usage:model.tokens:${turnId}:${sourceKey}`). The turnId is
187
+ * shared across a re-dispatch of the SAME turn (preemption resume, approval
188
+ * rerun, activity retry), so the sourceKey alone must distinguish calls.
189
+ *
190
+ * - A provider responseId is globally stable+unique, so reuse it verbatim: a
191
+ * true activity retry that re-emits the same responseId correctly DEDUPES
192
+ * (one charge), while two distinct calls get distinct ids.
193
+ * - Without a responseId the only fallback was POSITIONAL ("response-1",
194
+ * "aggregate"), which collides across a re-dispatch — dispatch B's first
195
+ * call reuses dispatch A's "response-1" key and its charge is silently
196
+ * dropped (undercharge). Qualifying the positional fallback with the
197
+ * per-execution dispatch id (the Temporal activityId, unique per scheduled
198
+ * execution) makes re-dispatched calls distinct while still deduping a
199
+ * same-execution retry.
200
+ */
201
+ export function modelUsageSourceKey(input: {
202
+ responseId?: string | null | undefined;
203
+ dispatchId: string | null;
204
+ positionalKey: string;
205
+ }): string {
206
+ if (input.responseId) {
207
+ return input.responseId;
208
+ }
209
+ return input.dispatchId ? `${input.dispatchId}:${input.positionalKey}` : input.positionalKey;
210
+ }
211
+
212
+ export function historyRowsToAppend(
213
+ rawHistory: Array<Record<string, unknown>>,
214
+ // How many items of the CURRENT in-memory history are already persisted (the
215
+ // slice index into `sanitized`). This is the in-memory history length, NOT the
216
+ // total persisted-row count: after a compaction the in-memory history is the
217
+ // short [summary, ...tail, ...new] list, far shorter than the total rows in
218
+ // the table (which still hold the superseded prefix).
219
+ persistedHistoryCount: number,
220
+ // Next free WHOLE-NUMBER absolute position to write at. Decoupled from the
221
+ // slice index because compaction inserts a fractional summary position, so the
222
+ // total-row count no longer equals max(position)+1. Defaults to
223
+ // persistedHistoryCount to preserve the pre-compaction behaviour (contiguous
224
+ // positions from 0) when callers do not pass an explicit next position.
225
+ nextPosition: number = persistedHistoryCount,
226
+ ): { rows: Array<{ position: number; item: Record<string, unknown> }>; nextWatermark: number; nextPosition: number } {
227
+ const sanitized = sanitizeHistoryItemsForModel(rawHistory);
228
+ if (sanitized.length <= persistedHistoryCount) {
229
+ return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
230
+ }
231
+ const rows = sanitized.slice(persistedHistoryCount).map((item, offset) => ({
232
+ position: nextPosition + offset,
233
+ item: item as Record<string, unknown>,
234
+ }));
235
+ return { rows, nextWatermark: sanitized.length, nextPosition: nextPosition + rows.length };
236
+ }
237
+
238
+ /**
239
+ * Seed the turn-end reconcile watermark (`persistedHistoryCount`) from EXACTLY the
240
+ * view `state.history` was seeded from, so the model-input length and the watermark
241
+ * can NEVER disagree. The watermark is the slice index the reconcile cuts the
242
+ * (re-sanitized) `state.history` at to find this turn's genuinely-new items, so it
243
+ * must equal the length of `state.history`'s already-persisted leading prefix.
244
+ *
245
+ * The cross-account reasoning strip drops foreign reasoning items, so the prefix
246
+ * length is PATH-DEPENDENT, captured by `modelHistoryFromItems`:
247
+ *
248
+ * - items read path (`modelHistoryFromItems === true`) — `state.history` was seeded
249
+ * from the cross-account-STRIPPED active items (foreign reasoning DROPPED), so it
250
+ * starts K shorter than the raw active-row count. Seed from the SAME strip
251
+ * (HOLE D); seeding from the un-stripped count would slice K genuinely-new items
252
+ * off the reconcile and silently lose them (incl. the user's switch-turn message).
253
+ *
254
+ * - run-state BLOB path (`modelHistoryFromItems === false`: approval resume, the
255
+ * items-mode run-state fallback, run_state mode) — `state.history` was seeded
256
+ * from the blob, where foreign reasoning is NEUTRALIZED-IN-PLACE (the item is
257
+ * KEPT, only its id/encrypted_content go — see resumeRunStateForCodexAccount), so
258
+ * the blob's history length still COUNTS those items. Applying the strip here
259
+ * under-counts by K and the reconcile re-appends K already-persisted items at
260
+ * fresh positions — HOLE E. So the blob path must NOT strip: count the raw
261
+ * sanitized active length, which mirrors the blob's completed prefix.
262
+ *
263
+ * On a same-account / non-codex turn the strip is a no-op, so both branches reduce
264
+ * to the same raw sanitized count (byte-identical to the pre-strip behaviour).
265
+ * Pure; exported for unit testing the D/E seed invariant.
266
+ */
267
+ export function reconcileSeedCount(
268
+ activeSeedRows: ReadonlyArray<{ item: Record<string, unknown>; producerCodexCredentialId: string | null }>,
269
+ modelHistoryFromItems: boolean,
270
+ current: TurnCodexAccount,
271
+ ): number {
272
+ return sanitizeHistoryItemsForModel(
273
+ modelHistoryFromItems
274
+ ? applyCodexHistoryStrip(activeSeedRows, current)
275
+ : activeSeedRows.map((row) => row.item),
276
+ ).length;
277
+ }
278
+
279
+ /**
280
+ * Resolve the EFFECTIVE/active compute backend a turn should gate
281
+ * filesystem-touching agent lifecycle hooks on (today: the repository clone).
282
+ *
283
+ * WHY (Case B — clone-onto-real-disk hazard): a session keeps its CLOUD HOME
284
+ * backend (`settings.sandboxBackend`, e.g. "modal") but its ACTIVE sandbox may
285
+ * have been swapped to a connected machine (`active_sandbox_id` → a selfhosted
286
+ * lease). `runtime.buildAgent`'s repository-clone hook keys off the backend it is
287
+ * told; if the worker passes nothing it defaults to the HOME backend and the hook
288
+ * would `git clone` a private GitHub-App repo onto the user's REAL disk — a
289
+ * bring-your-own machine owns its own filesystem and must NEVER be cloned onto. So
290
+ * we look at where the agent ACTUALLY runs, not where the session was created.
291
+ *
292
+ * Returns "selfhosted" ONLY when the selfhosted feature is on AND the session has
293
+ * a non-null active pointer whose sandbox `kind` is "selfhosted". Otherwise
294
+ * returns undefined so buildAgent falls back to the home backend — byte-for-byte
295
+ * unchanged cloud behavior.
296
+ *
297
+ * Total + best-effort by contract: it NEVER throws (a lookup failure is logged and
298
+ * falls back to the home default), so wiring it at turn start can't fail the turn.
299
+ * The DB I/O is injected (the real call site passes readActiveSandbox + getSandbox,
300
+ * the same helpers wrapTurnBoxWithRouting reuses) so the gate/decision/safety
301
+ * contract is unit-testable without a live database.
302
+ */
303
+ export async function resolveActiveSandboxBackend(
304
+ routingOn: boolean,
305
+ loadPointer: () => Promise<{ activeSandboxId: string | null } | null>,
306
+ loadSandboxKind: (sandboxId: string) => Promise<string | null>,
307
+ ): Promise<Settings["sandboxBackend"] | undefined> {
308
+ // The active pointer + swap tools only exist when selfhosted routing is on; with
309
+ // the flag off there is nothing to resolve and we keep the home-backend default.
310
+ if (!routingOn) {
311
+ return undefined;
312
+ }
313
+ try {
314
+ const pointer = await loadPointer();
315
+ // A null pointer (no swap) means "use the session's own cloud group box" — the
316
+ // home backend already governs that path, so leave the override unset.
317
+ if (!pointer?.activeSandboxId) {
318
+ return undefined;
319
+ }
320
+ const kind = await loadSandboxKind(pointer.activeSandboxId);
321
+ return kind === "selfhosted" ? "selfhosted" : undefined;
322
+ } catch (error) {
323
+ console.error("active sandbox backend resolution failed (turn proceeds on home backend)", error);
324
+ return undefined;
325
+ }
326
+ }
327
+
328
+ /**
329
+ * SELF-HEAL helper for the all-capped rotation idle (invariant 4: BOUNDED, no thrash).
330
+ * The turn hot path never refreshes Codex usage — only the usage API route does — so a
331
+ * window that has actually reset still reads OVER-threshold from the stale cache, which
332
+ * would idle-loop forever. Before idling, refresh LIVE usage for every connected account
333
+ * the cache marks over-threshold (bounded to the account count), which re-writes the
334
+ * cache columns, then return the re-read rows so the ranker can pick up a genuinely-reset
335
+ * window THIS turn. A refresh/read failure is swallowed (fall back to the pre-refresh rows
336
+ * + the bounded idle). Cooling (429'd) accounts are NOT refreshed: their exhaustedUntil
337
+ * cooldown is authoritative, and refreshing them would burn a provider call for nothing.
338
+ */
339
+ async function refreshCappedCodexUsageRows(
340
+ db: ActivityServices["db"],
341
+ settings: Settings,
342
+ workspaceId: string,
343
+ accounts: CodexAccountStatus[],
344
+ ): Promise<CodexAccountStatus[]> {
345
+ const nearPct = settings.codexRotationNearExhaustionPct;
346
+ const stale = accounts.filter(
347
+ (a) => a.status === "active" && ((a.primaryUsedPercent ?? 0) >= nearPct || (a.secondaryUsedPercent ?? 0) >= nearPct),
348
+ );
349
+ if (stale.length === 0) {
350
+ return accounts;
351
+ }
352
+ await Promise.all(stale.map((a) => fetchCodexUsageForAccount(db, settings, workspaceId, a.id).catch(() => undefined)));
353
+ return listCodexAccountStatuses(db, workspaceId).catch(() => accounts);
354
+ }
355
+
356
+ export function createRunAgentTurnActivity(services: () => Promise<ActivityServices>) {
357
+ return async function runAgentTurn(input: RunAgentTurnInput): Promise<RunAgentTurnResult> {
358
+ const { settings, db, bus, runtime, objectStorage, observability, wakeSessionWorkflow, entitlements, connectionCredentials } = await services();
359
+ const activityStarted = performance.now();
360
+ const activitySpan = observability.startSpan("worker.run_agent_segment", {
361
+ "opengeni.session_id": input.sessionId,
362
+ "opengeni.workflow_id": input.workflowId,
363
+ "opengeni.trigger_event_id": input.triggerEventId,
364
+ });
365
+ let activityStatus = "unknown";
366
+ let activityError: unknown;
367
+ let turnId: string | undefined;
368
+ let heartbeatTimer: ReturnType<typeof startActivityHeartbeat> | undefined;
369
+ // P1.2 ownership inversion: when sandboxOwnershipEnabled, the turn resolves
370
+ // the one box by id from the group lease and injects it NON-OWNED into the
371
+ // run. null when the flag is off (byte-for-byte the legacy build-and-discard
372
+ // path) OR when the backend is "none". Released + dropped in `finally`.
373
+ let resolvedSandbox: ResumedTurnSandbox | null = null;
374
+ // The UN-PROXIED established box session, captured BEFORE wrapTurnBoxWithRouting.
375
+ // Platform setup (beforeAgentStart hooks + file materialization) execs against
376
+ // THIS handle so a mid-turn sandbox_swap can never re-route those execs onto a
377
+ // connected machine (the user's real computer).
378
+ let setupBoxSession: unknown = null;
379
+ // The lease holder id (Temporal activityId, unique per scheduled execution)
380
+ // + the group id, captured so the lease heartbeat can refresh the lease TTL
381
+ // epoch-fenced (a superseded owner self-evicts) and finally can release.
382
+ let sandboxHolderId: string | null = null;
383
+ let sandboxGroupId: string | null = null;
384
+ // Lease-TTL refresh timer (parallels the activity heartbeat): while the turn
385
+ // runs it refreshes expires_at epoch-fenced so a legit multi-day turn is
386
+ // never TTL-reaped. Cleared in finally. Only set when the flag resolved a box.
387
+ let leaseHeartbeatTimer: ReturnType<typeof setInterval> | undefined;
388
+ // P4.3 on-turn recording: when the desktop tier + recording are enabled and
389
+ // the box is desktop-capable, the turn films the SAME :0 the agent's
390
+ // computer-use drives, finalized to storage in this activity's `finally`
391
+ // (read+PUT in-process — never a Temporal payload, F10). null otherwise.
392
+ let activeRecording: ActiveRecording | null = null;
393
+ // P4.3 recording gate: flips true the first time a computer-use/desktop tool
394
+ // ACTUALLY executes this turn (an SDK `computer_call` item streams through). A
395
+ // plain text turn ("hey"/"continue") never flips it, so finalize discards the
396
+ // recording with NO storage PUT and NO recording.failed — a clean no-op (a
397
+ // static frame of an untouched desktop is not worth uploading).
398
+ let didComputerUse = false;
399
+ let batcher: ReturnType<typeof createRuntimeBatcher> | null = null;
400
+ let preparedTools: Awaited<ReturnType<OpenGeniRuntime["prepareTools"]>> | null = null;
401
+ let publish: ((events: Array<Omit<AppendEventInput, "producerId" | "producerSeq" | "turnId">>, immediate?: boolean) => Promise<void>) | null = null;
402
+ let turnStartedPublished = false;
403
+ let stream: Awaited<ReturnType<OpenGeniRuntime["runStream"]>> | undefined;
404
+ // Dual-write of conversation truth (issue #35): completed items are
405
+ // reconciled into session_history_items after every model response and at
406
+ // every turn-end path (idempotent on position), and the sandbox recovery
407
+ // envelope is upserted alongside. Best-effort by design: persistence
408
+ // problems must never fail the run.
409
+ //
410
+ // Orphaned-tool-output guard: `stream.state.history` is NOT a plain
411
+ // append-only array — it is a computed getter
412
+ // (`getTurnInput(originalInput, generatedItems)`) that runs the SDK's
413
+ // `dropOrphanToolCalls` on every access, so a `function_call` with no
414
+ // settling result yet is transiently ABSENT from history and a later
415
+ // reconcile sees a DIFFERENT, shorter/reordered list. A blind length
416
+ // watermark with onConflictDoNothing-on-position then freezes the first
417
+ // shape of a position and can persist a `function_call_result` at a tail
418
+ // position while its `function_call` was pruned away in an earlier slice
419
+ // and never written — the orphan that bricks the session. We defend against
420
+ // it by sanitizing the current history into an API-valid sequence (the same
421
+ // pure function the read path uses) before persisting: a dangling call is
422
+ // never persisted (it is deferred until its result exists, when the pair is
423
+ // written together at consecutive positions), and a result is never
424
+ // persisted without its preceding call. The watermark counts only items we
425
+ // actually persisted, so a non-monotonic history can never desync it.
426
+ let persistedHistoryCount = 0;
427
+ let historyCountAtTurnStart = 0;
428
+ // Next free WHOLE-NUMBER absolute position to append at. Tracked separately
429
+ // from persistedHistoryCount (the in-memory slice index) because a compaction
430
+ // inserts a fractional summary position, so total rows no longer equal
431
+ // max(position)+1 and the slice index can no longer double as the position.
432
+ let nextHistoryPosition = 0;
433
+ const reconcileConversationTruth = async () => {
434
+ if (!stream || !turnId) {
435
+ return;
436
+ }
437
+ try {
438
+ const rawHistory = (stream.state as { history?: unknown[] }).history;
439
+ if (Array.isArray(rawHistory)) {
440
+ const { rows, nextWatermark, nextPosition } = historyRowsToAppend(
441
+ rawHistory as Array<Record<string, unknown>>,
442
+ persistedHistoryCount,
443
+ nextHistoryPosition,
444
+ );
445
+ if (rows.length > 0) {
446
+ await appendSessionHistoryItems(db, {
447
+ accountId: input.accountId,
448
+ workspaceId: input.workspaceId,
449
+ sessionId: input.sessionId,
450
+ turnId,
451
+ // Tag each row with the codex account that produced it (null on the
452
+ // non-codex path). Resolved at line ~504 before any reconcile pass
453
+ // runs, so this is the turn's effective account. The read path uses
454
+ // it to strip cross-account reasoning.encrypted_content next turn.
455
+ producerCodexCredentialId: effectiveCodexCredentialId,
456
+ items: rows,
457
+ });
458
+ }
459
+ persistedHistoryCount = nextWatermark;
460
+ nextHistoryPosition = nextPosition;
461
+ }
462
+ const envelope = sandboxStateEntryFromRunState(stream.state);
463
+ if (envelope) {
464
+ await upsertSandboxSessionEnvelope(db, {
465
+ accountId: input.accountId,
466
+ workspaceId: input.workspaceId,
467
+ sessionId: input.sessionId,
468
+ envelope,
469
+ });
470
+ }
471
+ } catch (persistError) {
472
+ console.error("session history dual-write failed (run unaffected)", persistError);
473
+ }
474
+ };
475
+ // Reassigned after the workspace environment loads; the publish closure is
476
+ // created (and used for turn.started) before the environment is available.
477
+ let redact: (payload: unknown) => unknown = identityRedactor;
478
+ let environmentId = "";
479
+ // The Codex account this turn runs on (pin > workspace active), resolved once
480
+ // a codex-billed turn is confirmed and threaded into the token resolver below.
481
+ let effectiveCodexCredentialId: string | null = null;
482
+ // Multi-account P4 (Part A): the latest usage-header snapshot scraped FOR FREE
483
+ // off this turn's `/codex/responses` responses (a turn issues many model calls;
484
+ // latest wins). Flushed ONCE into the P2 usage cache for the serving account in
485
+ // the `finally` — cheaper than a /wham/usage poll AND it self-heals P3 rotation
486
+ // (the proactive + 429 rankers read these exact columns). null ⇒ nothing scraped.
487
+ // Hoisted to activity scope so the finally flush (below) sees it. The sink is
488
+ // wired into codexContext.onUsageHeaders inside the try.
489
+ let latestCodexUsage: CodexUsageHeaderSnapshot | null = null;
490
+ // Hoisted for the preemption path: an approval-decision rerun must
491
+ // re-enter through the approval resume path (its frozen mid-flight state
492
+ // only exists in the RunState blob), never through a swapped trigger.
493
+ let triggerType: string | null = null;
494
+ try {
495
+ const mcpSettings = await settingsWithEnabledCapabilityMcpServers(db, input.workspaceId, settings);
496
+ // Read the active-credential flag ONCE (P2-b) and thread it through both the
497
+ // routing overlay (settingsWithCodexCredential) and the billed-turn predicate
498
+ // (isCodexBilledTurn below), so a concurrent disconnect/reconnect cannot make
499
+ // provider-injection and billing disagree about whether this is a codex turn.
500
+ const codexSubscriptionActive = await workspaceCodexSubscriptionActive(db, mcpSettings, input.workspaceId);
501
+ const capabilitySettings = await settingsWithCodexCredential(db, input.workspaceId, mcpSettings, codexSubscriptionActive);
502
+ runtime.configure(capabilitySettings);
503
+ const session = await requireSession(db, input.workspaceId, input.sessionId);
504
+ const trigger = await getSessionEvent(db, input.workspaceId, input.triggerEventId);
505
+ if (!trigger) {
506
+ throw new Error(`Trigger event not found: ${input.triggerEventId}`);
507
+ }
508
+ triggerType = trigger.type;
509
+ let turn = input.turnId ? await getSessionTurn(db, input.workspaceId, input.turnId) : await claimNextQueuedTurnDb(db, input.workspaceId, input.sessionId, input.workflowId);
510
+ if (!turn && !input.turnId) {
511
+ const createdTurnId = await createTurn(db, {
512
+ workspaceId: input.workspaceId,
513
+ sessionId: input.sessionId,
514
+ temporalWorkflowId: input.workflowId,
515
+ triggerEventId: input.triggerEventId,
516
+ });
517
+ turn = await getSessionTurn(db, input.workspaceId, createdTurnId);
518
+ }
519
+ if (!turn) {
520
+ throw new Error(`Session turn not found for trigger: ${input.triggerEventId}`);
521
+ }
522
+ turnId = turn.id;
523
+ // Canonical codex-billed predicate (codex/<slug> + feature enabled + active
524
+ // workspace credential). Computed once and threaded through every billing
525
+ // gate + the usage recorder so a turn paid by the user's ChatGPT/Codex plan
526
+ // consumes ZERO OpenGeni credits and never feeds an OpenGeni cap. Resolved
527
+ // here (before resolvedModel at the routing step) because the pre-turn gate
528
+ // below needs it; mirrors the same active-credential read the codex provider
529
+ // overlay uses, so billing and routing agree on what "codex" is.
530
+ const isCodexTurn = await isCodexBilledTurn({ db, settings, workspaceId: input.workspaceId, model: turn.model, active: codexSubscriptionActive });
531
+ // §7.5 P3 — pass BOTH the codex predicate (codex-plan turns bypass the gate)
532
+ // AND the optional host `entitlements` port (when bound, its admitRun replaces
533
+ // the local credit read). Unset port → today's local-ledger path.
534
+ await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
535
+ const activityContext = currentActivityContext();
536
+ // Setup (environment load, MCP connects, sandbox restore) does not
537
+ // stream and so never observes cancellation on its own; these explicit
538
+ // checks let a graceful shutdown preempt the turn before the worker is
539
+ // force-killed instead of riding the setup to a heartbeat timeout.
540
+ const throwIfWorkerShuttingDown = () => {
541
+ const reason = activityContext?.cancellationSignal.reason;
542
+ if (isWorkerShutdownCancellation(reason)) {
543
+ throw reason;
544
+ }
545
+ };
546
+ heartbeatTimer = startActivityHeartbeat(activityContext, {
547
+ phase: "running",
548
+ sessionId: input.sessionId,
549
+ turnId,
550
+ });
551
+ let producerSeq = 0;
552
+ // One producer per activity execution, not per turn: a turn can run
553
+ // again on the same workflow (preemption resume, approval rerun), and
554
+ // each execution restarts producerSeq at 1 — a shared producer id would
555
+ // trip the per-producer uniqueness constraint on the event log. The
556
+ // Temporal activity id is unique per scheduled execution.
557
+ const producerId = `${input.workflowId}:${turnId}${activityContext ? `:${activityContext.info.activityId}` : ""}`;
558
+ // Unique per scheduled activity execution (Temporal activityId). Folded
559
+ // into positional usage source keys so a re-dispatch of this turn does
560
+ // not collide its model-call charges with the prior dispatch's. A genuine
561
+ // activity retry reuses the same activityId, so its re-emitted calls keep
562
+ // deduping (no double charge).
563
+ const dispatchId = activityContext?.info.activityId ?? null;
564
+ publish = async (events: Array<Omit<AppendEventInput, "producerId" | "producerSeq" | "turnId">>, immediate = false) => {
565
+ const inputs = events.map((event) => ({
566
+ ...event,
567
+ payload: redact(event.payload),
568
+ turnId: turnId!,
569
+ producerId,
570
+ producerSeq: ++producerSeq,
571
+ }));
572
+ await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, inputs);
573
+ activityContext?.heartbeat({ phase: "events_published", sessionId: input.sessionId, turnId, producerSeq });
574
+ if (immediate) {
575
+ await Bun.sleep(0);
576
+ }
577
+ };
578
+ activityContext?.heartbeat({ phase: "turn_started", sessionId: input.sessionId, turnId });
579
+
580
+ // A shutdown that landed during claim/billing setup preempts before the
581
+ // turn visibly starts: nothing ran yet, so the requeued turn replays the
582
+ // original trigger cleanly on a healthy worker.
583
+ throwIfWorkerShuttingDown();
584
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "running", turnId);
585
+ await publish([
586
+ { type: "session.status.changed", payload: { status: "running" } },
587
+ { type: "turn.started", payload: { triggerEventId: input.triggerEventId } },
588
+ ], true);
589
+ turnStartedPublished = true;
590
+
591
+ // Multi-account (P1): resolve the effective Codex account for this turn
592
+ // (session-pin > workspace active) and stamp it on the session so the
593
+ // in-session "Running on:" indicator reflects reality. Emit a switch event
594
+ // when it changed from the prior run's account so the pill flips live.
595
+ // Gated on the codex-billed predicate — non-codex turns never touch this.
596
+ if (isCodexTurn) {
597
+ const [rotation, accounts, sessionCodex] = await Promise.all([
598
+ getCodexRotationSettings(db, input.workspaceId),
599
+ listCodexAccountStatuses(db, input.workspaceId),
600
+ getSessionCodexState(db, input.workspaceId, input.sessionId),
601
+ ]);
602
+ const connectedIds = new Set(accounts.map((account) => account.id));
603
+ const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
604
+ // The off-path / P1 default: today's workspace active pointer. Untouched
605
+ // when rotation is off or the session is pinned (byte-identical to P1).
606
+ let chosenActive = rotation?.activeCredentialId ?? null;
607
+ let rotationDecision: RotationDecision | null = null;
608
+
609
+ // P3 auto-rotation: the ONLY new branch, gated on rotation_enabled and
610
+ // pin-guarded (a pinned session NEVER rotates). When skipped, chosenActive
611
+ // stays the active pointer and selectCodexCredentialForTurn is called with
612
+ // byte-identical arguments to today — zero added cost on non-rotation turns.
613
+ if (rotation?.rotationEnabled && sessionPin == null) {
614
+ let rankAccounts = accounts;
615
+ rotationDecision = chooseRotationActive({
616
+ rotationStrategy: rotation.rotationStrategy as CodexRotationStrategy,
617
+ activeCredentialId: rotation.activeCredentialId,
618
+ priorCredentialId: sessionCodex?.lastCredentialId ?? null,
619
+ accounts: rankAccounts,
620
+ nearExhaustionPct: settings.codexRotationNearExhaustionPct,
621
+ now: new Date(),
622
+ // P4: the leaving (active) account's cached connector set is the proxy
623
+ // for "what this session has access to" — prefer a covering target.
624
+ usedConnectors: rankAccounts.find((a) => a.id === rotation.activeCredentialId)?.connectorNamespaces ?? [],
625
+ });
626
+ if (rotationDecision.kind === "allCapped") {
627
+ // SELF-HEAL (invariant 4): the turn hot path NEVER refreshes usage, so a
628
+ // window that has actually reset still reads capped from the stale cache —
629
+ // which would otherwise idle-loop forever (idle → continuation re-dispatch →
630
+ // same stale all-capped → idle …). Before idling, refresh usage for the
631
+ // over-threshold accounts (bounded to the account count) and re-rank ONCE,
632
+ // so a genuinely-reset window is picked up immediately and the cache heals.
633
+ rankAccounts = await refreshCappedCodexUsageRows(db, settings, input.workspaceId, rankAccounts);
634
+ rotationDecision = chooseRotationActive({
635
+ rotationStrategy: rotation.rotationStrategy as CodexRotationStrategy,
636
+ activeCredentialId: rotation.activeCredentialId,
637
+ priorCredentialId: sessionCodex?.lastCredentialId ?? null,
638
+ accounts: rankAccounts,
639
+ nearExhaustionPct: settings.codexRotationNearExhaustionPct,
640
+ now: new Date(),
641
+ // P4: leaving (active) account's connector set (refreshCappedCodexUsageRows
642
+ // only touches usage columns, so connectorNamespaces is preserved here).
643
+ usedConnectors: rankAccounts.find((a) => a.id === rotation.activeCredentialId)?.connectorNamespaces ?? [],
644
+ });
645
+ }
646
+ if (rotationDecision.kind === "active") {
647
+ if (rotationDecision.moved) {
648
+ // The single authoritative pointer-move site: persist the new active.
649
+ await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
650
+ }
651
+ chosenActive = rotationDecision.credentialId;
652
+ } else if (rotationDecision.kind === "allCapped" && publish && turnId) {
653
+ // Every eligible account is capped/cooling (and a usage refresh did NOT
654
+ // surface a reset): idle the turn AT THE BOUNDARY (no wasted model/sandbox
655
+ // build) until the EARLIEST reset across all accounts — the multi-account
656
+ // generalization of #143's single-account idle-until-reset. No saveRunState:
657
+ // no model ran, nothing to freeze.
658
+ const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
659
+ const goalActive = Boolean(goal && goal.status === "active");
660
+ // BOUNDED + POSITIVE: clamp to [MIN_IDLE_MS, max] so a null/elapsed/unknown
661
+ // reset can never yield a 0 (which session.ts would treat as "continue now",
662
+ // re-entering this path in a tight CPU/DB-hammering loop).
663
+ const resumeMs = computeIdleDelayMs(rotationDecision.earliestResetAt, new Date(), CODEX_USAGE_LIMIT_MAX_RESUME_MS);
664
+ const failurePayload = codexUsageLimitFailurePayload(
665
+ { resetsInSeconds: Math.ceil(resumeMs / 1000) },
666
+ "all connected Codex subscriptions are rate-limited",
667
+ { allAccounts: true },
668
+ );
669
+ await publish([
670
+ // `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
671
+ // rotation-wait state as the reactive all-capped path, so it must freeze
672
+ // autoContinuations identically (evaluateGoalContinuation reads this marker)
673
+ // — a goal waiting out a long reset must not burn its continuation budget on
674
+ // the proactive path while the reactive path spares it.
675
+ { type: "turn.failed", payload: { ...failurePayload, recovery: goalActive ? "goal_continuation" : "user_message", runStateSaved: false, rotated: true } },
676
+ { type: "session.status.changed", payload: { status: "idle" } },
677
+ ], true);
678
+ await finishTurn(db, input.workspaceId, turnId, "failed");
679
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
680
+ activityStatus = "idle";
681
+ // idleUntilReset marks this a MANDATORY hold: session.ts must wait the full
682
+ // resumeMs even if a future change made it 0 — never a tight re-dispatch.
683
+ return goalActive ? { status: "idle", continueDelayMs: resumeMs, idleUntilReset: true } : { status: "idle" };
684
+ }
685
+ // kind:"none" (no accounts) → chosenActive stays null → existing relogin path.
686
+ }
687
+
688
+ effectiveCodexCredentialId = selectCodexCredentialForTurn({
689
+ sessionPinnedCredentialId: sessionPin, // pin still wins, structurally
690
+ activeCredentialId: chosenActive, // rotation-choice OR today's active
691
+ connectedIds,
692
+ });
693
+ if (effectiveCodexCredentialId) {
694
+ const priorAccountId = sessionCodex?.lastCredentialId ?? null;
695
+ await recordSessionActiveCodexCredential(db, input.workspaceId, input.sessionId, effectiveCodexCredentialId);
696
+ if (priorAccountId !== effectiveCodexCredentialId) {
697
+ // "rotation" only when the engine actually moved the pointer; otherwise the
698
+ // unchanged P1 "manual" literal (a manual active flip between turns).
699
+ const rotated = rotationDecision?.kind === "active" && rotationDecision.moved;
700
+ // P4: surface the dropped-connector note when this rotation pick couldn't
701
+ // cover the session's used connectors (a Tier-2/unknown failover); the pill
702
+ // renders the badge. Omitted when the switch covered everything (the norm).
703
+ const droppedConnectors = rotationDecision?.kind === "active" ? rotationDecision.droppedConnectors : undefined;
704
+ await publish([{
705
+ type: "codex.account.switched",
706
+ payload: {
707
+ fromAccountId: priorAccountId,
708
+ toAccountId: effectiveCodexCredentialId,
709
+ reason: rotated ? "rotation" : "manual",
710
+ ...(droppedConnectors && droppedConnectors.length > 0 ? { droppedConnectors } : {}),
711
+ },
712
+ }]);
713
+ }
714
+ }
715
+ }
716
+
717
+ // Pack-scoped runtime: enabled packs may declare the sandbox image this
718
+ // workspace's sessions run in and skills for the sandbox skill index.
719
+ // Resolved after turn.started so a composition conflict (two enabled
720
+ // packs declaring images) fails the turn with its plain error instead
721
+ // of failing the activity opaquely.
722
+ const packRuntime = await resolveWorkspacePackRuntime(db, input.workspaceId);
723
+ // Workspace tier of the agent-persona resolution (session > workspace >
724
+ // deployment default). null means the workspace has no override, so the
725
+ // runtime falls back to runSettings.agentInstructionsTemplate (the
726
+ // deployment default, byte-identical to the historical preamble).
727
+ const workspaceAgentInstructions = await resolveWorkspaceAgentInstructions(db, input.workspaceId);
728
+ const runSettings = {
729
+ ...settingsWithPackSandboxImage(capabilitySettings, packRuntime.sandboxImage),
730
+ openaiModel: turn.model,
731
+ openaiReasoningEffort: turn.reasoningEffort,
732
+ sandboxBackend: turn.sandboxBackend,
733
+ };
734
+ // Multi-provider per-turn routing → the provider gating (compaction mode,
735
+ // hosted web search, encrypted reasoning, context window) the agent and
736
+ // compaction summarizer must use; null falls back to the legacy global
737
+ // client. Resolve against `capabilitySettings` (whose openaiModel is the
738
+ // deployment default), NOT `runSettings`: runSettings.openaiModel is the
739
+ // turn's model, so for a turn ON a registry model the built-in provider
740
+ // would otherwise claim that id (configuredModels builds the built-in's
741
+ // models from openaiModel) and shadow the registry entry — resolving the
742
+ // turn to the built-in (Azure) gating while the global model router routes
743
+ // the name to its registry provider. That mismatch attaches web_search to
744
+ // a chat-only Fireworks model. Resolving against the default-model settings
745
+ // keeps gating consistent with the router. Cost accounting covers registry
746
+ // models via configuredModelPricing.
747
+ const resolvedModel = runtime.resolveTurnModel(capabilitySettings, turn.model);
748
+ // A codex-subscription turn resolves the bearer for THIS turn's effective
749
+ // codex account (effectiveCodexCredentialId; pin > workspace-active) at
750
+ // model-call time — multi-account P1 means a workspace can hold N accounts,
751
+ // so the bearer is per-account, not per-workspace. codexSubscriptionFetch
752
+ // (on the provider's OpenAI client) reads this AsyncLocalStorage context.
753
+ // Build it once and wrap BOTH the compaction summarizer (a separate model
754
+ // call on the same codex client) and the main run; otherwise the summarizer
755
+ // would hit the codex backend unauthenticated.
756
+ const codexContext: CodexRequestContext | null = resolvedModel?.provider.kind === "codex-subscription"
757
+ ? ((): CodexRequestContext => {
758
+ // The empty-string fallback yields no row → null credential → the
759
+ // existing CodexReloginRequired path (a codex turn with no usable
760
+ // account fails closed, exactly as before multi-account).
761
+ const resolver = buildCodexTokenResolver(db, runSettings, input.workspaceId, effectiveCodexCredentialId ?? "");
762
+ return {
763
+ clientVersion: CODEX_CLIENT_VERSION,
764
+ getToken: resolver.getToken,
765
+ refresh: resolver.refresh,
766
+ resolveModel: buildModelResolver(CODEX_FALLBACK_MODEL_SLUGS, CODEX_FALLBACK_MODEL_SLUGS[0]),
767
+ onUsageHeaders: (snapshot) => { latestCodexUsage = snapshot; }, // latest wins; flushed once in finally
768
+ };
769
+ })()
770
+ : null;
771
+ const withCodex = <T>(fn: () => Promise<T>): Promise<T> => (codexContext ? codexRequestStorage.run(codexContext, fn) : fn());
772
+ const turnResources = mergeResourceRefs(session.resources, turn.resources);
773
+ // Attach the first-party MCP server to EVERY turn, regardless of how/when
774
+ // the session was created (API, scheduled task, or a pre-existing session
775
+ // whose stored tools predate this) — so set_session_title and the rest are
776
+ // always reachable. Idempotent: mergeToolRefs dedupes if already present.
777
+ // Attach codex_apps (the ChatGPT/Codex connectors MCP) when the codex
778
+ // overlay injected it into runSettings.mcpServers (active subscription +
779
+ // connector scopes); no-op for every other turn. Its refreshing bearer is
780
+ // resolved at connect time from the codex ALS (see the withCodex-wrapped
781
+ // prepareTools call below).
782
+ const turnTools = withCodexAppsTool(runSettings, withFirstPartyTools(runSettings, mergeToolRefs(session.tools, turn.tools)));
783
+ // §7.6 P4a — load (and decrypt) the workspace environment via the host
784
+ // `sandboxSecrets` provider when bound; unset → today's local decrypt.
785
+ const connectionScope = { accountId: input.accountId, workspaceId: input.workspaceId };
786
+ const workspaceEnvironment = await loadWorkspaceEnvironmentForRunWithCredentials(
787
+ db,
788
+ runSettings,
789
+ connectionScope,
790
+ session.environmentId,
791
+ connectionCredentials?.sandboxSecrets,
792
+ );
793
+ environmentId = workspaceEnvironment?.id ?? "";
794
+ redact = createSecretRedactor(
795
+ Object.entries(workspaceEnvironment?.values ?? {}).map(([name, value]) => ({ name, value })),
796
+ );
797
+ // EFFECTIVE compute backend, resolved ONCE at turn start (Case B + Stage D
798
+ // D1-lite) and reused for EVERY downstream decision: the env mint (skip the
799
+ // inert GitHub token for a machine turn), the establish path (no phantom Modal
800
+ // home box for a machine-primary turn), buildAgent (skip the repository clone
801
+ // hook so a private repo is never `git clone`d onto the user's real disk), and
802
+ // the warm-rate (a machine accrues ZERO cloud warm-seconds). The active pointer
803
+ // + its sandbox row are loaded ONCE here (best-effort, never throwing) and the
804
+ // SAME values feed resolveActiveSandboxBackend (the tested gate) AND the
805
+ // machine-primary establish branch (enrollmentId/epoch/workingDir) below — no
806
+ // double read, no read-skew between the gate decision and the establish. With
807
+ // routing OFF this is byte-for-byte the legacy path: no reads, undefined backend.
808
+ const routingOn = routingEnabled(settings);
809
+ const activeSandboxPointer = routingOn
810
+ ? await readActiveSandbox(db, input.workspaceId, input.sessionId).catch(() => null)
811
+ : null;
812
+ const activeSandboxRecord = routingOn && activeSandboxPointer?.activeSandboxId
813
+ ? await getSandbox(db, input.workspaceId, activeSandboxPointer.activeSandboxId).catch(() => null)
814
+ : null;
815
+ const activeSandboxBackend = await resolveActiveSandboxBackend(
816
+ routingOn,
817
+ async () => activeSandboxPointer,
818
+ async () => activeSandboxRecord?.kind ?? null,
819
+ );
820
+ // A machine-primary turn = the effective backend is selfhosted AND we have the
821
+ // machine's enrollment (agent id) + a non-null pointer to bind it. Anything
822
+ // missing (should not happen — the DB enforces selfhosted⇒enrollmentId) falls
823
+ // back to the cloud establish path (a correct, if phantom, box) rather than
824
+ // crashing the turn.
825
+ const machinePrimary =
826
+ activeSandboxBackend === "selfhosted"
827
+ && Boolean(activeSandboxPointer?.activeSandboxId)
828
+ && Boolean(activeSandboxRecord?.enrollmentId);
829
+ // Computed exactly ONCE per turn and reused for BOTH the box manifest
830
+ // (resumeBoxForTurn -> establishSandboxSessionFromEnvelope, below) AND the
831
+ // agent (runtime.buildAgent, below). sandboxEnvironmentForRun mints a FRESH
832
+ // run-scoped GitHub installation token on every call, so a second call would
833
+ // yield a DIFFERENT token value and re-introduce the manifest-env delta the
834
+ // SDK's provided-session guard throws on — the box and the agent MUST share
835
+ // this same object. A machine-primary turn skips the (inert) token mint entirely
836
+ // (the machine uses its own git creds); the SAME base env still feeds the box +
837
+ // the agent, so env-parity holds.
838
+ // TOKEN-BROKER (B1): sandboxEnvironmentForRun now returns the STABLE manifest
839
+ // env (no rotating GH_TOKEN/GITHUB_TOKEN/GIT_CONFIG_* extraheader) PLUS the
840
+ // run-scoped GitHub token minted ONCE per turn as `gitToken`. The env feeds BOTH
841
+ // the box manifest AND the agent (env-parity, as before); the token is threaded
842
+ // OFF-MANIFEST as the clone-seed `gitTokenSeed` to buildAgent (below) so the box
843
+ // never carries a rotating value on its manifest. When the platform token IS
844
+ // minted, the host `gitCredentials` provider may supply it; unset → self-mint
845
+ // from settings. gitToken is undefined on the selfhosted skip path (the machine
846
+ // uses its own git creds).
847
+ const { environment: sandboxEnvironment, gitToken: sandboxGitToken } = await sandboxEnvironmentForRun(
848
+ runSettings,
849
+ turnResources,
850
+ workspaceEnvironment?.values ?? {},
851
+ {
852
+ skipGitHubToken: activeSandboxBackend === "selfhosted",
853
+ scope: connectionScope,
854
+ gitCredentials: connectionCredentials?.gitCredentials,
855
+ },
856
+ );
857
+
858
+ // P1.2 ownership inversion (gated, default OFF). With the flag off this
859
+ // block is skipped entirely: resolvedSandbox stays null and runStream
860
+ // takes the legacy per-run build-and-discard path — byte-for-byte today.
861
+ // With it on, acquire the group lease ('turn' holder = the activityId),
862
+ // resume the one box by id, and inject it NON-OWNED into the run. The box
863
+ // backend is "none" -> never resolve (no box to touch).
864
+ //
865
+ // Established AFTER sandboxEnvironment is computed (not before) so the box's
866
+ // manifest is created with the SAME environment the agent declares — the SDK
867
+ // applies the agent's manifest to this provided session and throws on ANY
868
+ // environment delta (validateNoEnvironmentDelta). Passing sandboxEnvironment
869
+ // here makes current==target so the delta is empty.
870
+ if (settings.sandboxOwnershipEnabled && turn.sandboxBackend !== "none") {
871
+ sandboxHolderId = dispatchId ?? `turn:${turnId}`;
872
+ sandboxGroupId = session.sandboxGroupId;
873
+ // STAGE D honest-label guard: a machine-home session carries
874
+ // turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
875
+ // when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
876
+ // + enrollmentId). When it is NOT primary — the agent swapped back to the
877
+ // group box (sandbox_swap 'session'/'default'/groupId clears the pointer) or
878
+ // selfhosted routing is flag-OFF (the pointer is ignored) — the else-branch
879
+ // must resume a REAL cloud group box, not a "selfhosted" one: the registry
880
+ // SelfhostedSandboxClient has no bound agentId and throws. Fall the group-box
881
+ // backend back to the deployment default cloud backend so swap-away / flag-off
882
+ // degrade to a genuine cloud box exactly like today (home=modal did).
883
+ const groupBoxBackend: Settings["sandboxBackend"] =
884
+ turn.sandboxBackend === "selfhosted" && !machinePrimary
885
+ ? settings.sandboxBackend
886
+ : turn.sandboxBackend;
887
+ if (machinePrimary) {
888
+ // STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
889
+ // establish or lease a phantom Modal home box (today's path leased + BILLED
890
+ // a cloud box the turn never touched). Build the SelfhostedSession DIRECTLY
891
+ // (no Modal box created) and take the group lease with backend "selfhosted"
892
+ // (refcount/idle bookkeeping; the reaper drains it cold with NO provider
893
+ // stop, and bills ZERO warm-seconds). The session is a harmless in-memory
894
+ // bind (no NATS round-trip), so build it FIRST; if the lease then fences,
895
+ // there is nothing to clean up.
896
+ const established = await establishSelfhostedTurnSession(
897
+ { db, settings, bus },
898
+ {
899
+ workspaceId: input.workspaceId,
900
+ agentId: activeSandboxRecord!.enrollmentId!,
901
+ epoch: activeSandboxPointer!.activeEpoch,
902
+ environment: sandboxEnvironment,
903
+ workingDir: activeSandboxPointer!.workingDir,
904
+ },
905
+ );
906
+ const lease = await acquireSelfhostedLeaseForTurn(
907
+ { db, settings },
908
+ {
909
+ accountId: input.accountId,
910
+ workspaceId: input.workspaceId,
911
+ sandboxGroupId: session.sandboxGroupId,
912
+ sessionId: input.sessionId,
913
+ },
914
+ "turn",
915
+ sandboxHolderId,
916
+ );
917
+ setupBoxSession = established.session;
918
+ resolvedSandbox = {
919
+ // Wrap in the SAME routing proxy so a mid-turn swap (to another machine
920
+ // or back to the group box) still re-routes per op. PIN this established
921
+ // SelfhostedSession for the machine pointer so the turn-start manifest
922
+ // write (via the proxy's `state` getter) and the per-op reads hit ONE
923
+ // instance — no two-instance manifest divergence.
924
+ established: wrapTurnBoxWithRouting(
925
+ { db, settings, bus },
926
+ {
927
+ workspaceId: input.workspaceId,
928
+ sessionId: input.sessionId,
929
+ environment: sandboxEnvironment,
930
+ pinnedSelfhosted: {
931
+ sandboxId: activeSandboxPointer!.activeSandboxId!,
932
+ epoch: activeSandboxPointer!.activeEpoch,
933
+ },
934
+ },
935
+ established,
936
+ ),
937
+ leaseEpoch: lease.leaseEpoch,
938
+ release: lease.release,
939
+ };
940
+ } else {
941
+ resolvedSandbox = await resumeBoxForTurn(
942
+ { db, settings },
943
+ {
944
+ accountId: input.accountId,
945
+ workspaceId: input.workspaceId,
946
+ sandboxGroupId: session.sandboxGroupId,
947
+ sessionId: input.sessionId,
948
+ // groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
949
+ // is not machine-primary resumes a real cloud group box (the
950
+ // deployment default), never a "selfhosted" box (which would throw
951
+ // for lack of a bound agentId).
952
+ backend: groupBoxBackend,
953
+ os: session.sandboxOs,
954
+ environment: sandboxEnvironment,
955
+ // IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
956
+ // this run resolves. The lease stamps it + conflicts on a live shared box
957
+ // running a DIFFERENT image (solo → recreate on the new image; N-holders →
958
+ // SandboxImageConflictError surfaced as an actionable turn error). Prefer the
959
+ // explicit Modal image ref, else the docker image. The selfhosted branch
960
+ // (establishSelfhostedTurnSession/acquireSelfhostedLeaseForTurn) NEVER passes
961
+ // an image — B3 lives only on this Modal else-branch.
962
+ ...((runSettings.modalImageRef ?? runSettings.dockerImage)
963
+ ? { image: runSettings.modalImageRef ?? runSettings.dockerImage }
964
+ : {}),
965
+ },
966
+ "turn",
967
+ sandboxHolderId,
968
+ );
969
+ setupBoxSession = resolvedSandbox.established.session;
970
+ // M7 hot-swap: when the selfhosted feature is on, wrap the established
971
+ // group box in the STABLE routing proxy before it is injected NON-OWNED
972
+ // into the run. The SDK binds to this ONE object once and calls its
973
+ // methods per tool call; the proxy re-reads (active_sandbox_id,
974
+ // active_epoch) per op and dispatches to the currently-active backend, so
975
+ // a sandbox_swap mid-turn lands the NEXT tool call on the new box. With
976
+ // the flag off the established group box is injected unchanged (today's
977
+ // path). The lease still owns the group box lifecycle — the proxy is a
978
+ // routing veneer, not an owner.
979
+ if (routingEnabled(settings)) {
980
+ resolvedSandbox = {
981
+ ...resolvedSandbox,
982
+ established: wrapTurnBoxWithRouting(
983
+ { db, settings, bus },
984
+ // Thread the SAME declared environment the group box was created with
985
+ // (resumeBoxForTurn, above) so a selfhosted swap target's manifest
986
+ // carries it too — the SDK's per-turn manifest-env delta stays empty
987
+ // (no "cannot change manifest environment variables" throw).
988
+ { workspaceId: input.workspaceId, sessionId: input.sessionId, environment: sandboxEnvironment },
989
+ resolvedSandbox.established,
990
+ ),
991
+ };
992
+ }
993
+ }
994
+ // Refresh the lease TTL on the activity-heartbeat cadence (10s, well
995
+ // inside the 90s lease TTL). EPOCH-FENCED: a superseded owner's refresh
996
+ // is rejected (returns false) and we stop refreshing — the box rides the
997
+ // provider idle-timeout and the next dispatch re-establishes it. Best-
998
+ // effort: a transient DB error must never fail the turn.
999
+ const heartbeatEpoch = resolvedSandbox.leaseEpoch;
1000
+ const heartbeatHolderId = sandboxHolderId;
1001
+ const heartbeatGroupId = sandboxGroupId;
1002
+ // P2.1 warm-meter (tick A): while a turn runs, the heartbeat is also the
1003
+ // warm-seconds tick. GROUP+epoch+tick keyed (one box = one stream, shared
1004
+ // box metered once); epoch-fenced (a stale tick no-ops). Warm-cost is
1005
+ // metered when a per-backend rate is configured. Best-effort: a metering
1006
+ // failure must never fail the turn.
1007
+ //
1008
+ // Keyed off the EFFECTIVE backend (Stage D): a machine-primary turn has NO
1009
+ // Modal box, so it must accrue ZERO cloud warm-seconds — `selfhosted` has no
1010
+ // configured warm rate (0). Keying off turn.sandboxBackend (modal) would bill
1011
+ // cloud seconds for a box that does not exist (a real money bug). Non-machine
1012
+ // turns fall back to groupBoxBackend (the REAL box that ran): for a machine-
1013
+ // home turn that degraded to the cloud group box (swap-away / flag-off), that
1014
+ // is the deployment default (modal), so the fallback box is warm-metered at
1015
+ // the cloud rate instead of selfhosted's rate-0 (which would under-bill).
1016
+ const warmRate = sandboxWarmRateMicrosPerSecond(settings, activeSandboxBackend ?? groupBoxBackend);
1017
+ leaseHeartbeatTimer = setInterval(() => {
1018
+ void heartbeatLeaseHolder(db, {
1019
+ accountId: input.accountId,
1020
+ workspaceId: input.workspaceId,
1021
+ sandboxGroupId: heartbeatGroupId,
1022
+ kind: "turn",
1023
+ holderId: heartbeatHolderId,
1024
+ leaseTtlMs: settings.sandboxLeaseTtlMs,
1025
+ expectedEpoch: heartbeatEpoch,
1026
+ }).catch(() => undefined);
1027
+ void accrueWarmSeconds(db, {
1028
+ accountId: input.accountId,
1029
+ workspaceId: input.workspaceId,
1030
+ sandboxGroupId: heartbeatGroupId,
1031
+ expectedEpoch: heartbeatEpoch,
1032
+ warmRateMicrosPerSecond: warmRate,
1033
+ subjectId: input.sessionId,
1034
+ }).catch(() => undefined);
1035
+ }, 10_000);
1036
+ if ("unref" in leaseHeartbeatTimer && typeof leaseHeartbeatTimer.unref === "function") {
1037
+ leaseHeartbeatTimer.unref();
1038
+ }
1039
+ }
1040
+
1041
+ // P4.3 on-turn recording. The box's :0 display stack was brought up by
1042
+ // resumeBoxForTurn (spawner path) / is up from a prior turn; film it for
1043
+ // the duration of this turn so the human can watch the agent work and the
1044
+ // agent's computer-use proofs are captured. Best-effort: a recording start
1045
+ // failure NEVER fails the turn (the desktop is a value-add). Finalized in
1046
+ // `finally` (read+PUT in this same activity — never a Temporal payload).
1047
+ if (
1048
+ resolvedSandbox
1049
+ && settings.sandboxDesktopEnabled
1050
+ && settings.recordingEnabled
1051
+ && desktopCapableBackend(resolvedSandbox.established.backendId)
1052
+ ) {
1053
+ try {
1054
+ const begun = await beginRecording({
1055
+ settings,
1056
+ db,
1057
+ accountId: input.accountId,
1058
+ workspaceId: input.workspaceId,
1059
+ sessionId: input.sessionId,
1060
+ turnId: turnId!,
1061
+ recordingId: randomUUID(),
1062
+ mode: "on-turn",
1063
+ session: resolvedSandbox.established.session,
1064
+ runAs: sandboxRunAs(settings),
1065
+ reason: null,
1066
+ });
1067
+ activeRecording = begun.active;
1068
+ await publish([{ type: "recording.started", payload: begun.started }]);
1069
+ } catch (recordingError) {
1070
+ activeRecording = null;
1071
+ console.error("on-turn recording start failed (turn outcome unaffected)", recordingError);
1072
+ }
1073
+ }
1074
+
1075
+ const fileResourceDownloads = await sandboxFileDownloadsForRun(runSettings, db, objectStorage, input.workspaceId, turnResources);
1076
+ throwIfWorkerShuttingDown();
1077
+ // Wrap MCP prep in the codex ALS so the codex_apps connect handshake
1078
+ // (initialize + tools/list) can resolve the per-workspace bearer from
1079
+ // codexRequestStorage (runtime/codexAppsMcpRequestInit). withCodex is the
1080
+ // identity on every non-codex turn, so this is a no-op for existing paths.
1081
+ preparedTools = await withCodex(() => runtime.prepareTools(runSettings, turnTools, {
1082
+ accountId: input.accountId,
1083
+ workspaceId: input.workspaceId,
1084
+ sessionId: input.sessionId,
1085
+ subjectId: "worker:first-party-mcp",
1086
+ subjectLabel: "OpenGeni worker",
1087
+ // Manager-style sessions carry a creation-validated permission set
1088
+ // for their first-party MCP token; null keeps the fixed default.
1089
+ ...(session.firstPartyMcpPermissions?.length ? { firstPartyPermissions: session.firstPartyMcpPermissions } : {}),
1090
+ }));
1091
+ // Genesis turn = the first user turn (no assistant history reconciled
1092
+ // yet). Durable Postgres state (countSessionHistoryItems includes
1093
+ // superseded rows after compaction), NOT a workflow counter (turnsThisRun
1094
+ // resets on continueAsNew). Drives the one-shot title hint appended to the
1095
+ // agent's instructions; continuation/preemption turns never match (their
1096
+ // trigger is goal.continuation/turn.preempted).
1097
+ const isGenesisTurn = triggerType === "user.message"
1098
+ && (await countSessionHistoryItems(db, input.workspaceId, input.sessionId)) === 0;
1099
+ // Clone-onto-real-disk hazard (Case B). A session keeps its CLOUD HOME
1100
+ // backend (runSettings.sandboxBackend, e.g. "modal") but its ACTIVE sandbox
1101
+ // may have been swapped to a connected machine (active_sandbox_id → a
1102
+ // selfhosted lease). buildAgent's repository-clone lifecycle hook keys off
1103
+ // the EFFECTIVE backend; if we let it default to the home backend it would
1104
+ // `git clone` a private GitHub-App repo onto the user's REAL disk. So pass
1105
+ // "selfhosted" through when the active sandbox is a connected machine;
1106
+ // otherwise leave it undefined so buildAgent defaults to the home backend
1107
+ // (byte-for-byte unchanged cloud behavior). `activeSandboxBackend` was
1108
+ // resolved ONCE at turn start (above) via resolveActiveSandboxBackend (the
1109
+ // tested gate) and is reused here — resolving once is correct because the
1110
+ // clone hook runs at beforeAgentStart, so a mid-turn swap can't affect it.
1111
+ const agent = runtime.buildAgent(runSettings, turnResources, {
1112
+ reasoningEffort: turn.reasoningEffort,
1113
+ genesisTitleHint: isGenesisTurn,
1114
+ sandboxEnvironment,
1115
+ // TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
1116
+ // seed. ONLY when the effective backend is NOT selfhosted (the connected
1117
+ // machine uses its own git creds — mirrors the skipGitHubToken gate above)
1118
+ // AND the mint actually produced a token (repo resources present). The runtime
1119
+ // seeds it to the box's token file before the repository-clone runs; it never
1120
+ // touches the box/agent manifest env.
1121
+ ...(activeSandboxBackend !== "selfhosted" && sandboxGitToken ? { gitTokenSeed: sandboxGitToken } : {}),
1122
+ ...(activeSandboxBackend ? { activeSandboxBackend } : {}),
1123
+ fileResourceDownloads,
1124
+ mcpServers: preparedTools.mcpServers,
1125
+ // LIVE by-reference connector namespaces (fills during this turn's
1126
+ // codex_apps tools/list): the codex tool_search description reads it per
1127
+ // model call so the model sees the account's real connected sources.
1128
+ codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
1129
+ // Resolved-model routing + gating (legacy defaults when null). The model
1130
+ // is passed as the model *string* (agent.model = runSettings.openaiModel),
1131
+ // NOT a Model instance: an instance only survives the in-process
1132
+ // ("none") run, whereas the SandboxAgent/Modal path drops it and
1133
+ // re-resolves the model *name* through the global MultiProviderModelProvider
1134
+ // configureOpenAI installed — so registry models (Fireworks GLM) route to
1135
+ // their own client instead of 404ing against the built-in Azure/OpenAI
1136
+ // client. The gating still comes from the resolved provider: server-side
1137
+ // store/compaction follow the provider's compaction mode (registry
1138
+ // providers resolve to "client"); encrypted reasoning is only
1139
+ // round-tripped on the Responses wire API; hosted web search is attached
1140
+ // only when the model opts in; the effective context window drives the
1141
+ // compaction threshold.
1142
+ ...(resolvedModel
1143
+ ? {
1144
+ compactionMode: resolvedModel.provider.compactionMode,
1145
+ hostedWebSearch: resolvedModel.configured.hostedWebSearch,
1146
+ encryptedReasoning: resolvedModel.provider.api === "responses" && runSettings.openaiReasoningEncryptedContent,
1147
+ contextWindowTokens: resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
1148
+ // The ChatGPT/Codex backend rejects the SDK's HOSTED sandbox tools —
1149
+ // the `apply_patch` tool type ("Unsupported tool type: apply_patch")
1150
+ // and structured tool output — which the OpenAIResponsesModel the SDK
1151
+ // binds would otherwise select. Tell buildAgent to emit the function
1152
+ // `apply_patch` + text `view_image` variants the backend accepts. Only
1153
+ // the codex-subscription provider needs this; every other backend
1154
+ // (built-in OpenAI/Azure = real hosted support; registry "chat"
1155
+ // providers = the SDK's own ChatCompletions detection) keeps the SDK
1156
+ // default.
1157
+ structuredToolTransport: resolvedModel.provider.kind !== "codex-subscription",
1158
+ }
1159
+ : {}),
1160
+ ...(packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {}),
1161
+ ...(workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {}),
1162
+ ...(workspaceEnvironment
1163
+ ? {
1164
+ workspaceEnvironment: {
1165
+ name: workspaceEnvironment.name,
1166
+ description: workspaceEnvironment.description,
1167
+ variableNames: Object.keys(workspaceEnvironment.values),
1168
+ },
1169
+ }
1170
+ : {}),
1171
+ });
1172
+ // Pre-turn client-side context compaction (Azure path). When the
1173
+ // resolved mode is "client" and the last turn's input tokens crossed the
1174
+ // soft budget, this summarizes the orphan-safe old prefix into one user
1175
+ // message and supersedes the summarized rows BEFORE the history is read
1176
+ // for this turn, so the model sees [summary, ...recent tail] + new input.
1177
+ // Best-effort: a no-op or failure leaves the un-compacted history intact.
1178
+ // Skipped on approval/preempt resumes (no fresh user turn; the frozen
1179
+ // RunState or in-flight tail must replay verbatim).
1180
+ if (triggerType === "user.message" || triggerType === "goal.continuation") {
1181
+ try {
1182
+ // Operator /compact (the slash command) sets a durable request flag;
1183
+ // consume it atomically so a forced compaction runs now even when the
1184
+ // budget trigger would not fire. Only the turn that observes the flag
1185
+ // runs it, so concurrent turns can't double-compact.
1186
+ const forced = await consumeSessionCompactionRequest(db, input.workspaceId, input.sessionId);
1187
+ const outcome = await maybeCompactContext(
1188
+ db,
1189
+ runSettings,
1190
+ { accountId: input.accountId, workspaceId: input.workspaceId, sessionId: input.sessionId, turnId },
1191
+ session.lastInputTokens,
1192
+ // Provider-aware summarizer: when the turn's model resolved to a
1193
+ // registry provider, summarize on THAT provider's client + wire API
1194
+ // (a chat provider can't summarize through OpenAI/Azure). Null
1195
+ // resolution keeps the default built-in Responses summarizer.
1196
+ resolvedModel
1197
+ ? (s, m) => withCodex(() => summarizeForCompaction(s, m, {
1198
+ client: resolvedModel.client,
1199
+ api: resolvedModel.provider.api,
1200
+ model: resolvedModel.configured.id,
1201
+ maxOutputTokens: s.contextSummaryMaxTokens,
1202
+ }))
1203
+ : undefined,
1204
+ forced ? { force: true } : {},
1205
+ );
1206
+ if (outcome.compacted) {
1207
+ // trigger precedence: an explicit operator /compact wins for the
1208
+ // event label; otherwise a hard-forced (at/over hardFraction*B)
1209
+ // compaction is surfaced as "hard" so the backstop firing is
1210
+ // observable downstream, and the default soft trigger is implicit.
1211
+ const trigger = forced ? "operator" : outcome.hardForced ? "hard" : undefined;
1212
+ await publish([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, ...(trigger ? { trigger } : {}) } }]);
1213
+ }
1214
+ } catch (compactError) {
1215
+ console.error("context compaction failed (turn proceeds un-compacted)", compactError);
1216
+ }
1217
+ }
1218
+ // Cross-account reasoning strip: pass THIS turn's codex account so every
1219
+ // history read path (items + run-state replay) drops reasoning produced by
1220
+ // a DIFFERENT codex account. effectiveCodexCredentialId is the resolved
1221
+ // codex credential on a codex turn (pin > workspace-active) and null on a
1222
+ // non-codex turn OR a codex turn with no usable account — exactly the
1223
+ // "current account" the single strip rule compares against (null is the
1224
+ // built-in/Azure account, so a non-codex turn still drops codex-produced
1225
+ // reasoning, and a no-codex-history session is a byte-for-byte no-op).
1226
+ const { input: runInput, modelHistoryFromItems } = await turnInput(
1227
+ db,
1228
+ runtime,
1229
+ agent,
1230
+ trigger,
1231
+ runSettings,
1232
+ { currentCodexCredentialId: effectiveCodexCredentialId },
1233
+ );
1234
+ // Slice index = the length of the model-facing (active) history this turn
1235
+ // is seeded from; new items beyond it (the trigger message + this turn's
1236
+ // generated items) are the ones to persist. After a compaction this is the
1237
+ // short [summary, ...tail] active set, NOT the total row count. The
1238
+ // absolute write position is tracked separately (next whole number past
1239
+ // the max existing position) because the fractional summary row means
1240
+ // total rows no longer equal max(position)+1. Pre-compaction both reduce to
1241
+ // the old total-count value, so the common path is unchanged.
1242
+ //
1243
+ // CRITICAL: seed from the SANITIZED active-row length, not the raw active
1244
+ // count. `prepareRunInput` builds `state.history` from
1245
+ // `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
1246
+ // rows (a legacy orphan/dangling pair), the in-memory history this turn
1247
+ // starts from is K shorter than the raw row count. The reconcile slices the
1248
+ // re-sanitized `state.history` off `persistedHistoryCount`; seeding it from
1249
+ // the raw count (K too high) skips K genuinely-new items, and a
1250
+ // `function_call` left in that skipped region can later have its
1251
+ // `function_call_result` persisted alone — the orphan that 400s on replay
1252
+ // and bricks the session (issue-61). The sanitized seed is already
1253
+ // orphan-free, so it is a stable prefix of the re-sanitized history and the
1254
+ // slice begins exactly at the first genuinely-new item.
1255
+ const activeSeedRows = await getActiveSessionHistoryItems(db, input.workspaceId, input.sessionId);
1256
+ // Seed the reconcile watermark from EXACTLY the view the model's
1257
+ // `state.history` was seeded from (items strip on the items path = HOLE D; NO
1258
+ // strip on the run-state blob path, where foreign reasoning is neutralized but
1259
+ // KEPT = HOLE E), so the model-input length and the watermark never disagree.
1260
+ persistedHistoryCount = reconcileSeedCount(
1261
+ activeSeedRows,
1262
+ modelHistoryFromItems,
1263
+ { currentCodexCredentialId: effectiveCodexCredentialId },
1264
+ );
1265
+ historyCountAtTurnStart = persistedHistoryCount;
1266
+ nextHistoryPosition = await nextSessionHistoryPosition(db, input.workspaceId, input.sessionId);
1267
+ let responseUsageCount = 0;
1268
+ // Actual input tokens of the most recent model response this turn; the
1269
+ // pre-read trigger for the NEXT turn. Persisted at every turn-end path.
1270
+ let lastInputTokensObserved: number | null = null;
1271
+ throwIfWorkerShuttingDown();
1272
+ const runStreamOnce = (): ReturnType<OpenGeniRuntime["runStream"]> => runtime.runStream(agent, runInput, runSettings, {
1273
+ sandboxEnvironment,
1274
+ onRuntimeEvent: async (event) => {
1275
+ await publish!([{ type: event.type, payload: event.payload }], true);
1276
+ },
1277
+ // P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
1278
+ // keystone). Absent when the flag is off -> legacy build-and-discard.
1279
+ ...(resolvedSandbox
1280
+ ? {
1281
+ ownedSandbox: {
1282
+ client: resolvedSandbox.established.client,
1283
+ session: resolvedSandbox.established.session,
1284
+ sessionState: resolvedSandbox.established.sessionState,
1285
+ // Pin platform setup (hooks + file materialization) to the un-proxied
1286
+ // established box — never through the routing proxy, which would
1287
+ // re-route those execs onto a machine swapped in mid-turn.
1288
+ ...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
1289
+ },
1290
+ }
1291
+ : {}),
1292
+ });
1293
+ stream = await withCodex(runStreamOnce);
1294
+ batcher = createRuntimeBatcher(async (events) => {
1295
+ await publish!(events);
1296
+ });
1297
+
1298
+ const iterator = stream.toStream()[Symbol.asyncIterator]();
1299
+ let streamDone = false;
1300
+ try {
1301
+ while (true) {
1302
+ const next = await nextStreamEvent(iterator, activityContext);
1303
+ if (next.done) {
1304
+ streamDone = true;
1305
+ break;
1306
+ }
1307
+ const responseUsage = modelResponseUsageFromSdkEvent(next.value);
1308
+ if (responseUsage) {
1309
+ responseUsageCount += 1;
1310
+ const observed = responseUsage.usage?.inputTokens;
1311
+ if (typeof observed === "number" && observed > 0) {
1312
+ lastInputTokensObserved = observed;
1313
+ }
1314
+ await recordModelUsageAndDebitCredits(settings, db, {
1315
+ accountId: input.accountId,
1316
+ workspaceId: input.workspaceId,
1317
+ sessionId: input.sessionId,
1318
+ turnId,
1319
+ model: turn.model,
1320
+ isCodexTurn,
1321
+ usage: responseUsage.usage,
1322
+ sourceKey: modelUsageSourceKey({
1323
+ responseId: responseUsage.responseId,
1324
+ dispatchId,
1325
+ positionalKey: `response-${responseUsageCount}`,
1326
+ }),
1327
+ });
1328
+ await reconcileConversationTruth();
1329
+ try {
1330
+ await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
1331
+ } catch (limitError) {
1332
+ // Capture the run state at the boundary so the budget valve in
1333
+ // the outer catch can end this segment gracefully with full
1334
+ // conversation context preserved for the post-top-up resume.
1335
+ let serializedRunState: string | null = null;
1336
+ try {
1337
+ serializedRunState = stream.state.toString();
1338
+ } catch {
1339
+ serializedRunState = null;
1340
+ }
1341
+ throw new BudgetExhaustedError(
1342
+ limitError instanceof Error ? limitError.message : String(limitError),
1343
+ serializedRunState,
1344
+ );
1345
+ }
1346
+ }
1347
+ // Recording gate: a computer-use tool actually ran when an SDK
1348
+ // `tool_call_item` whose rawItem.type is "computer_call" streams through
1349
+ // (screenshot/click/type/scroll/etc. — the first computer action proves
1350
+ // the desktop was driven). Match the raw SDK event (ground truth) BEFORE
1351
+ // normalization. Only meaningful when a recording is live.
1352
+ if (activeRecording && !didComputerUse && isComputerCallStreamEvent(next.value)) {
1353
+ didComputerUse = true;
1354
+ }
1355
+ const normalized = normalizeSdkEvent(next.value);
1356
+ for (const event of normalized) {
1357
+ await batcher.push(event);
1358
+ }
1359
+ }
1360
+ } finally {
1361
+ if (!streamDone) {
1362
+ await iterator.return?.();
1363
+ }
1364
+ }
1365
+ await batcher.flush();
1366
+ await stream.completed.catch(() => undefined);
1367
+ if (responseUsageCount === 0) {
1368
+ const aggregateInput = (stream.state.usage as { inputTokens?: unknown } | undefined)?.inputTokens;
1369
+ if (typeof aggregateInput === "number" && aggregateInput > 0) {
1370
+ lastInputTokensObserved = aggregateInput;
1371
+ }
1372
+ await recordModelUsageAndDebitCredits(settings, db, {
1373
+ accountId: input.accountId,
1374
+ workspaceId: input.workspaceId,
1375
+ sessionId: input.sessionId,
1376
+ turnId,
1377
+ model: turn.model,
1378
+ isCodexTurn,
1379
+ usage: stream.state.usage,
1380
+ sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
1381
+ });
1382
+ }
1383
+ if (lastInputTokensObserved !== null) {
1384
+ await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, lastInputTokensObserved)
1385
+ .catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
1386
+ }
1387
+
1388
+ if (stream.interruptions.length > 0) {
1389
+ await reconcileConversationTruth();
1390
+ const approvals = runtime.serializeApprovals(stream.interruptions);
1391
+ await saveRunState(db, {
1392
+ accountId: input.accountId,
1393
+ workspaceId: input.workspaceId,
1394
+ sessionId: input.sessionId,
1395
+ turnId,
1396
+ serializedRunState: stream.state.toString(),
1397
+ pendingApprovals: approvals,
1398
+ // Record the account freezing this state so a resume on a DIFFERENT
1399
+ // codex account strips its account-bound reasoning before replay (HOLE C).
1400
+ frozenCodexCredentialId: effectiveCodexCredentialId,
1401
+ });
1402
+ await publish([
1403
+ { type: "session.requiresAction", payload: { approvals } },
1404
+ { type: "session.status.changed", payload: { status: "requires_action" } },
1405
+ ], true);
1406
+ await finishTurn(db, input.workspaceId, turnId, "requires_action");
1407
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "requires_action", turnId);
1408
+ activityStatus = "requires_action";
1409
+ return { status: "requires_action" };
1410
+ }
1411
+
1412
+ const finalOutput = String(stream.finalOutput ?? "");
1413
+ await reconcileConversationTruth();
1414
+ if (settings.sessionHistorySource !== "items") {
1415
+ // Legacy conversation memory; in items mode the blob is only written
1416
+ // for requires_action pauses (the one RunState-only resume path).
1417
+ await saveRunState(db, {
1418
+ accountId: input.accountId,
1419
+ workspaceId: input.workspaceId,
1420
+ sessionId: input.sessionId,
1421
+ turnId,
1422
+ serializedRunState: stream.state.toString(),
1423
+ pendingApprovals: [],
1424
+ frozenCodexCredentialId: effectiveCodexCredentialId,
1425
+ });
1426
+ }
1427
+ await publish([
1428
+ { type: "agent.message.completed", payload: { text: finalOutput } },
1429
+ { type: "turn.completed", payload: { output: finalOutput } },
1430
+ { type: "session.status.changed", payload: { status: "idle" } },
1431
+ ], true);
1432
+ await finishTurn(db, input.workspaceId, turnId, "idle");
1433
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
1434
+ await recordUsageEvent(db, {
1435
+ accountId: input.accountId,
1436
+ workspaceId: input.workspaceId,
1437
+ eventType: "agent_run.completed",
1438
+ quantity: 1,
1439
+ unit: "run",
1440
+ sourceResourceType: "session_turn",
1441
+ sourceResourceId: turnId,
1442
+ idempotencyKey: `usage:agent_run.completed:${turnId}`,
1443
+ });
1444
+ activityStatus = "idle";
1445
+ return { status: "idle" };
1446
+ } catch (error) {
1447
+ // Graceful worker shutdown (deploy / rollout restart): checkpoint and
1448
+ // hand the turn back instead of failing the session. Conversation truth
1449
+ // is already dual-written per model response; the final reconcile below
1450
+ // bounds the loss to at most the single in-flight model step. The turn
1451
+ // is re-queued so the session workflow re-dispatches it on a healthy
1452
+ // worker — through a synthesized resume notice when this turn already
1453
+ // persisted progress (replaying the original trigger would duplicate
1454
+ // input the model has already seen and invite it to redo side effects),
1455
+ // or through the original trigger when nothing was persisted yet. This
1456
+ // is an explicit checkpoint/resume, not a blind activity retry: the
1457
+ // resumed attempt sees everything the first attempt did.
1458
+ //
1459
+ // The branch deliberately does NOT require turn.started to have been
1460
+ // published: a shutdown landing during setup (claim/billing, before the
1461
+ // turn visibly started) must also requeue, not fail the session. In
1462
+ // that early case nothing ran and nothing was checkpointed, so the
1463
+ // requeued turn replays the original trigger cleanly. The turn id falls
1464
+ // back to the workflow-claimed turn when the local lookup had not
1465
+ // finished yet.
1466
+ const preemptTurnId = turnId ?? input.turnId;
1467
+ // P1.2: a lease supersession during resume (a newer epoch re-established
1468
+ // the box concurrently) is NOT a session failure — re-dispatch the turn so
1469
+ // it re-resumes by id under the current epoch. Requeue + idle, mirroring
1470
+ // the worker-death re-dispatch (any worker re-resumes by id).
1471
+ if (error instanceof SandboxLeaseSupersededError && preemptTurnId) {
1472
+ try {
1473
+ await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, input.triggerEventId);
1474
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
1475
+ activityStatus = "preempted";
1476
+ return { status: "preempted" };
1477
+ } catch (requeueError) {
1478
+ console.error("sandbox lease supersession requeue failed; falling back", requeueError);
1479
+ }
1480
+ }
1481
+ if (isWorkerShutdownCancellation(error) && preemptTurnId) {
1482
+ try {
1483
+ await batcher?.flush().catch(() => undefined);
1484
+ await reconcileConversationTruth();
1485
+ // An approval-decision rerun always replays its original trigger:
1486
+ // the decision is applied through the approval resume path reading
1487
+ // the frozen RunState blob (the only representation of a turn
1488
+ // paused mid-flight), so swapping the trigger for a resume notice
1489
+ // could drop the user's decision. Re-applying an already-consumed
1490
+ // approval re-executes at most the single approved step — the same
1491
+ // bound every preemption already accepts.
1492
+ const approvalRerun = triggerType === "user.approvalDecision";
1493
+ let resumeWithNotice = !approvalRerun && settings.sessionHistorySource === "items" && persistedHistoryCount > historyCountAtTurnStart;
1494
+ if (settings.sessionHistorySource !== "items" && stream) {
1495
+ // Legacy run-state mode: the resume reads the RunState blob, so
1496
+ // the checkpoint must be captured there — including any pending
1497
+ // approval interruptions, exactly like the requires_action path,
1498
+ // so a shutdown while approvals wait does not erase them. A
1499
+ // failed capture falls back to the previous snapshot and a clean
1500
+ // re-run of the original trigger.
1501
+ try {
1502
+ await saveRunState(db, {
1503
+ accountId: input.accountId,
1504
+ workspaceId: input.workspaceId,
1505
+ sessionId: input.sessionId,
1506
+ turnId: preemptTurnId,
1507
+ serializedRunState: stream.state.toString(),
1508
+ pendingApprovals: runtime.serializeApprovals(stream.interruptions ?? []),
1509
+ frozenCodexCredentialId: effectiveCodexCredentialId,
1510
+ });
1511
+ resumeWithNotice = true;
1512
+ } catch {
1513
+ resumeWithNotice = false;
1514
+ }
1515
+ }
1516
+ const [preemptedEvent] = await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [
1517
+ {
1518
+ turnId: preemptTurnId,
1519
+ type: "turn.preempted",
1520
+ payload: {
1521
+ triggerEventId: input.triggerEventId,
1522
+ reason: "worker_shutdown",
1523
+ resumeWithNotice,
1524
+ ...(resumeWithNotice ? { text: WORKER_SHUTDOWN_RESUME_TEXT } : {}),
1525
+ },
1526
+ },
1527
+ {
1528
+ turnId: preemptTurnId,
1529
+ type: "session.status.changed",
1530
+ payload: { status: "queued" },
1531
+ },
1532
+ ]);
1533
+ await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
1534
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
1535
+ activityStatus = "preempted";
1536
+ return { status: "preempted" };
1537
+ } catch (preemptError) {
1538
+ // A failing checkpoint/requeue must not surface as an arbitrary
1539
+ // activity error around a half-applied preemption (the workflow
1540
+ // would fail the session while a requeued turn lingers). Fall
1541
+ // through to the cancellation path below: the turn is marked
1542
+ // cancelled — also resetting a turn this block already requeued —
1543
+ // and the session fails like an uncheckpointed death.
1544
+ console.error("worker-shutdown preemption failed; falling back to cancellation", preemptError);
1545
+ }
1546
+ }
1547
+ if (error instanceof CancelledFailure) {
1548
+ activityStatus = "cancelled";
1549
+ activityError = error;
1550
+ await batcher?.flush().catch(() => undefined);
1551
+ if (preemptTurnId) {
1552
+ await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => undefined);
1553
+ }
1554
+ throw error;
1555
+ }
1556
+ // The SDK's per-segment turn cap is a pacing valve, not a failure: end
1557
+ // the turn gracefully and idle the session so an active goal continues
1558
+ // via a synthesized continuation turn (or a user message resumes work).
1559
+ // The run state captured at the cap keeps full conversation context for
1560
+ // that resumption.
1561
+ const maxTurns = maxTurnsExceededRunState(error);
1562
+ if (maxTurns && publish && turnId && turnStartedPublished) {
1563
+ await batcher?.flush().catch(() => undefined);
1564
+ // The SDK attaches the run state at the throw site; persisting it lets
1565
+ // the continuation resume with this segment's full context. If capture
1566
+ // ever fails, the continuation falls back to the previous snapshot --
1567
+ // degraded context, flagged on the event, but still strictly better
1568
+ // than a terminal failed session: the sandbox filesystem state
1569
+ // persists independently and the agent re-derives from it.
1570
+ await reconcileConversationTruth();
1571
+ const runStateSaved = Boolean(maxTurns.serializedRunState) && settings.sessionHistorySource !== "items";
1572
+ if (maxTurns.serializedRunState && settings.sessionHistorySource !== "items") {
1573
+ await saveRunState(db, {
1574
+ accountId: input.accountId,
1575
+ workspaceId: input.workspaceId,
1576
+ sessionId: input.sessionId,
1577
+ turnId,
1578
+ serializedRunState: maxTurns.serializedRunState,
1579
+ pendingApprovals: [],
1580
+ frozenCodexCredentialId: effectiveCodexCredentialId,
1581
+ });
1582
+ }
1583
+ await publish([
1584
+ { type: "turn.completed", payload: { output: "", segmentLimit: "max_turns", runStateSaved } },
1585
+ { type: "session.status.changed", payload: { status: "idle" } },
1586
+ ], true);
1587
+ await finishTurn(db, input.workspaceId, turnId, "idle");
1588
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
1589
+ await recordUsageEvent(db, {
1590
+ accountId: input.accountId,
1591
+ workspaceId: input.workspaceId,
1592
+ eventType: "agent_run.completed",
1593
+ quantity: 1,
1594
+ unit: "run",
1595
+ sourceResourceType: "session_turn",
1596
+ sourceResourceId: turnId,
1597
+ idempotencyKey: `usage:agent_run.completed:${turnId}`,
1598
+ });
1599
+ activityStatus = "idle";
1600
+ return { status: "idle" };
1601
+ }
1602
+ // A ChatGPT/Codex usage cap (429 usage_limit_reached) is account state,
1603
+ // NOT an agent failure: surface the precise, actionable message (so the
1604
+ // user sees the reset window) but idle the session — never go terminal,
1605
+ // which would reject the user's next message after the cap lifts. The
1606
+ // payload is retryable:false so the generic provider-backpressure auto-retry
1607
+ // does not loop. For an active goal we hold the continuation for the reported
1608
+ // reset window (capped) so it resumes itself when access returns, instead of
1609
+ // hammering the capped backend.
1610
+ const usageLimit = classifyCodexUsageLimitError(error);
1611
+ if (usageLimit && publish && turnId && turnStartedPublished) {
1612
+ const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
1613
+ const goalActive = Boolean(goal && goal.status === "active");
1614
+ await batcher?.flush().catch(() => undefined);
1615
+ await reconcileConversationTruth();
1616
+ const serializedRunState = agentsErrorRunState(error);
1617
+ const runStateSaved = Boolean(serializedRunState) && settings.sessionHistorySource !== "items";
1618
+ if (serializedRunState && settings.sessionHistorySource !== "items") {
1619
+ await saveRunState(db, {
1620
+ accountId: input.accountId,
1621
+ workspaceId: input.workspaceId,
1622
+ sessionId: input.sessionId,
1623
+ turnId,
1624
+ serializedRunState,
1625
+ pendingApprovals: [],
1626
+ frozenCodexCredentialId: effectiveCodexCredentialId,
1627
+ });
1628
+ }
1629
+ // --- P3 reactive rotation (gated; re-fetch fresh state on this already-failed,
1630
+ // already-idling path). Mark THIS account cooling until its reset, then CONSULT the
1631
+ // engine over fresh accounts to decide continueDelayMs: a fast 0-delay re-dispatch
1632
+ // when another account is available, or idle-until-earliest when all are capped. The
1633
+ // catch deliberately does NOT move the active pointer — the re-dispatched turn's
1634
+ // proactive seam (turn-start) is the single authoritative pointer-move + strip site.
1635
+ let rotated = false;
1636
+ let rotationResumeMs: number | null = null; // 0 ⇒ a candidate is available; re-dispatch now
1637
+ let rotationResumeIdleUntilReset = false; // circuit-breaker fall (Finding 1b) ⇒ MANDATORY hold
1638
+ let allCappedResetAt: Date | null = null; // set ⇒ every account capped; idle until this
1639
+ if (effectiveCodexCredentialId) {
1640
+ const [rotation, sessionCodex] = await Promise.all([
1641
+ getCodexRotationSettings(db, input.workspaceId).catch(() => null),
1642
+ getSessionCodexState(db, input.workspaceId, input.sessionId).catch(() => null),
1643
+ ]);
1644
+ const rotating = Boolean(rotation?.rotationEnabled) && (sessionCodex?.pinnedCredentialId == null);
1645
+ if (rotating && rotation) {
1646
+ const accounts = await listCodexAccountStatuses(db, input.workspaceId).catch(() => []);
1647
+ const serving = accounts.find((a) => a.id === effectiveCodexCredentialId) ?? null;
1648
+ // Cooldown end (invariant 5): authoritative resets_in_seconds from the 429; else
1649
+ // the serving account's soonest cached window reset; else the 1h cap.
1650
+ const cachedReset = [serving?.primaryResetAt, serving?.secondaryResetAt]
1651
+ .filter((d): d is Date => d instanceof Date && d.getTime() > Date.now())
1652
+ .sort((a, b) => a.getTime() - b.getTime())[0] ?? null;
1653
+ const until = usageLimit.resetsInSeconds !== null && Number.isFinite(usageLimit.resetsInSeconds) && usageLimit.resetsInSeconds > 0
1654
+ ? new Date(Date.now() + Math.ceil(usageLimit.resetsInSeconds) * 1000)
1655
+ : (cachedReset ?? new Date(Date.now() + CODEX_USAGE_LIMIT_MAX_RESUME_MS));
1656
+ // Finding 1a: INSPECT the cooldown-write result. A swallowed best-effort
1657
+ // write whose failure went unnoticed is exactly what lets the next proactive
1658
+ // rank re-pick this just-capped account (stale-low cached usedPercent, not
1659
+ // cooling) — so capture whether it PERSISTED and feed it into the resume floor.
1660
+ const cooldownPersisted = await setCodexCredentialExhausted(db, input.workspaceId, effectiveCodexCredentialId, until).catch(() => false);
1661
+ // Re-rank over the fresh accounts; the in-memory list predates the cooldown
1662
+ // write, so stamp the just-cooled account so the engine excludes it now. The
1663
+ // serving account is thus walked AT MOST ONCE per turn (invariant 4: bounded).
1664
+ const fresh = accounts.map((a) => (a.id === effectiveCodexCredentialId ? { ...a, exhaustedUntil: until } : a));
1665
+ const decision = chooseRotationActive({
1666
+ rotationStrategy: rotation.rotationStrategy as CodexRotationStrategy,
1667
+ activeCredentialId: rotation.activeCredentialId,
1668
+ priorCredentialId: effectiveCodexCredentialId,
1669
+ accounts: fresh,
1670
+ nearExhaustionPct: settings.codexRotationNearExhaustionPct,
1671
+ now: new Date(),
1672
+ // P4: the just-capped serving account's connector set is the proxy for
1673
+ // "what this session has access to" — prefer a covering failover target.
1674
+ usedConnectors: serving?.connectorNamespaces ?? [],
1675
+ });
1676
+ if (decision.kind === "active") {
1677
+ rotated = true;
1678
+ // Finding 1: a live candidate normally re-dispatches NOW (0). Two second-order
1679
+ // faults would turn that 0 into a hot loop, so bound it. Count the consecutive
1680
+ // reactive failovers since the last successful turn (this one is not yet
1681
+ // published) and combine with the cooldown-persistence result.
1682
+ const priorConsecutiveRotations = await countConsecutiveReactiveRotations(db, input.workspaceId, input.sessionId).catch(() => 0);
1683
+ const resume = computeReactiveRotationResume({
1684
+ cooldownPersisted,
1685
+ priorConsecutiveRotations,
1686
+ connectedAccountCount: accounts.length,
1687
+ });
1688
+ rotationResumeMs = resume.continueDelayMs; // 0 (happy path), a slow-retry floor, or the circuit-breaker idle
1689
+ rotationResumeIdleUntilReset = resume.idleUntilReset; // true only on the circuit-breaker fall (MANDATORY hold)
1690
+ } else if (decision.kind === "allCapped") {
1691
+ rotated = true;
1692
+ allCappedResetAt = decision.earliestResetAt;
1693
+ }
1694
+ // kind:"none" → fall through to today's single-account idle.
1695
+ }
1696
+ }
1697
+
1698
+ const failurePayload = allCappedResetAt
1699
+ ? codexUsageLimitFailurePayload(
1700
+ { resetsInSeconds: Math.ceil(Math.max(0, allCappedResetAt.getTime() - Date.now()) / 1000) },
1701
+ error instanceof Error ? error.message : String(error),
1702
+ { allAccounts: true },
1703
+ )
1704
+ : codexUsageLimitFailurePayload(usageLimit, error instanceof Error ? error.message : String(error));
1705
+ await publish([
1706
+ // `rotated:true` ONLY on the reactive rotation path tells evaluateGoalContinuation to
1707
+ // freeze autoContinuations (a rotation walk must not burn the goal's continuation budget).
1708
+ { type: "turn.failed", payload: { ...failurePayload, recovery: goalActive ? "goal_continuation" : "user_message", runStateSaved, ...(rotated ? { rotated: true } : {}) } },
1709
+ { type: "session.status.changed", payload: { status: "idle" } },
1710
+ ], true);
1711
+ await finishTurn(db, input.workspaceId, turnId, "failed");
1712
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
1713
+ activityStatus = "idle";
1714
+ activityError = error;
1715
+ if (goalActive) {
1716
+ // Rotation: a candidate is available → continue NOW (0). All-capped → idle until the
1717
+ // earliest reset across all accounts (capped at 1h). Else the unchanged single-account idle.
1718
+ if (rotationResumeMs !== null) {
1719
+ // A candidate IS available. Normally the just-failed account is now cooling so
1720
+ // the ranker cannot re-pick it → 0 (re-dispatch NOW, the legitimate skip-the-hold
1721
+ // case). Finding 1 bounds the two exceptions: a persistence fault yields a positive
1722
+ // slow-retry floor, and once consecutive failovers exceed the account count + margin
1723
+ // the circuit breaker returns a fixed MANDATORY idle (idleUntilReset) — never a 0-delay
1724
+ // hot loop against a capped backend + DB.
1725
+ return { status: "idle", continueDelayMs: rotationResumeMs, ...(rotationResumeIdleUntilReset ? { idleUntilReset: true } : {}) };
1726
+ }
1727
+ // All-capped: clamp to [MIN_IDLE_MS, max] — a POSITIVE, BOUNDED hold (never 0,
1728
+ // so session.ts can never tight-loop). The post-idle continuation re-dispatch
1729
+ // hits the proactive seam, which refreshes usage and self-heals.
1730
+ const resumeMs = allCappedResetAt
1731
+ ? computeIdleDelayMs(allCappedResetAt, new Date(), CODEX_USAGE_LIMIT_MAX_RESUME_MS)
1732
+ : (usageLimit.resetsInSeconds !== null && Number.isFinite(usageLimit.resetsInSeconds) && usageLimit.resetsInSeconds > 0
1733
+ ? Math.min(Math.ceil(usageLimit.resetsInSeconds) * 1000, CODEX_USAGE_LIMIT_MAX_RESUME_MS)
1734
+ : CODEX_USAGE_LIMIT_MAX_RESUME_MS);
1735
+ return { status: "idle", continueDelayMs: resumeMs, ...(allCappedResetAt ? { idleUntilReset: true } : {}) };
1736
+ }
1737
+ return { status: "idle" };
1738
+ }
1739
+ // Budget/limit exhaustion between model calls is account state, not an
1740
+ // agent failure: idle the session for goal-bearing and goal-less runs
1741
+ // alike (a failed session would reject the user's next message after a
1742
+ // top-up). An active goal pauses visibly with reason "limits" at the
1743
+ // next continuation evaluation, without consuming continuation budget.
1744
+ if (error instanceof BudgetExhaustedError && publish && turnId && turnStartedPublished) {
1745
+ await batcher?.flush().catch(() => undefined);
1746
+ await reconcileConversationTruth();
1747
+ const runStateSaved = Boolean(error.serializedRunState) && settings.sessionHistorySource !== "items";
1748
+ if (error.serializedRunState && settings.sessionHistorySource !== "items") {
1749
+ await saveRunState(db, {
1750
+ accountId: input.accountId,
1751
+ workspaceId: input.workspaceId,
1752
+ sessionId: input.sessionId,
1753
+ turnId,
1754
+ serializedRunState: error.serializedRunState,
1755
+ pendingApprovals: [],
1756
+ frozenCodexCredentialId: effectiveCodexCredentialId,
1757
+ });
1758
+ }
1759
+ await publish([
1760
+ { type: "turn.completed", payload: { output: "", segmentLimit: "budget_exhausted", detail: error.message, runStateSaved } },
1761
+ { type: "session.status.changed", payload: { status: "idle" } },
1762
+ ], true);
1763
+ await finishTurn(db, input.workspaceId, turnId, "idle");
1764
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
1765
+ await recordUsageEvent(db, {
1766
+ accountId: input.accountId,
1767
+ workspaceId: input.workspaceId,
1768
+ eventType: "agent_run.completed",
1769
+ quantity: 1,
1770
+ unit: "run",
1771
+ sourceResourceType: "session_turn",
1772
+ sourceResourceId: turnId,
1773
+ idempotencyKey: `usage:agent_run.completed:${turnId}`,
1774
+ });
1775
+ activityStatus = "idle";
1776
+ return { status: "idle" };
1777
+ }
1778
+ // A retryable provider failure (rate limit) is transient backpressure,
1779
+ // not a session failure: the in-client retry budget is already
1780
+ // exhausted by the time the error reaches here, so fail the turn
1781
+ // truthfully but idle the session. With an active goal the continuation
1782
+ // loop resumes after a pacing delay; without one the session simply
1783
+ // waits for the next user message. Long-lived sessions (a per-customer
1784
+ // ops channel between goals) must never go terminal because the
1785
+ // provider had a bad minute -- a failed session would reject the very
1786
+ // retry message the failure asks for.
1787
+ const failure = agentRunFailurePayload(error);
1788
+ if (failure.retryable && publish && turnId && turnStartedPublished) {
1789
+ const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
1790
+ const goalActive = Boolean(goal && goal.status === "active");
1791
+ await batcher?.flush().catch(() => undefined);
1792
+ // Provider errors rarely carry SDK run state; a null falls back to
1793
+ // the previous snapshot, same degraded-context contract as the
1794
+ // max-turns path above.
1795
+ await reconcileConversationTruth();
1796
+ const serializedRunState = agentsErrorRunState(error);
1797
+ const runStateSaved = Boolean(serializedRunState) && settings.sessionHistorySource !== "items";
1798
+ if (serializedRunState && settings.sessionHistorySource !== "items") {
1799
+ await saveRunState(db, {
1800
+ accountId: input.accountId,
1801
+ workspaceId: input.workspaceId,
1802
+ sessionId: input.sessionId,
1803
+ turnId,
1804
+ serializedRunState,
1805
+ pendingApprovals: [],
1806
+ frozenCodexCredentialId: effectiveCodexCredentialId,
1807
+ });
1808
+ }
1809
+ await publish([
1810
+ { type: "turn.failed", payload: { ...failure, recovery: goalActive ? "goal_continuation" : "user_message", runStateSaved } },
1811
+ { type: "session.status.changed", payload: { status: "idle" } },
1812
+ ], true);
1813
+ await finishTurn(db, input.workspaceId, turnId, "failed");
1814
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
1815
+ activityStatus = "idle";
1816
+ activityError = error;
1817
+ return goalActive ? { status: "idle", continueDelayMs: PROVIDER_BACKPRESSURE_DELAY_MS } : { status: "idle" };
1818
+ }
1819
+ activityStatus = "failed";
1820
+ activityError = error;
1821
+ if (!publish || !turnId || !turnStartedPublished) {
1822
+ throw error;
1823
+ }
1824
+ await publish([
1825
+ { type: "turn.failed", payload: failure },
1826
+ { type: "session.status.changed", payload: { status: "failed" } },
1827
+ ], true);
1828
+ await finishTurn(db, input.workspaceId, turnId, "failed");
1829
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "failed", null);
1830
+ // The common failure path ends here: runAgentTurn marks the session
1831
+ // failed and returns "failed", and the session workflow then exits
1832
+ // WITHOUT calling failSession/markSessionIdle. Wake a spawned worker's
1833
+ // parent here too, so a manager learns of a worker that died inside its
1834
+ // turn (not just one failed by the workflow's failSession path). Deduped
1835
+ // per terminal episode by the child's lastSequence, so it never
1836
+ // double-fires with the workflow-level wake.
1837
+ await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed", `turn:${turnId}`);
1838
+ return { status: "failed" };
1839
+ } finally {
1840
+ const durationSeconds = (performance.now() - activityStarted) / 1000;
1841
+ observability.recordWorkerActivity({
1842
+ activity: "runAgentTurn",
1843
+ status: activityStatus,
1844
+ durationSeconds,
1845
+ });
1846
+ activitySpan.end({
1847
+ attributes: {
1848
+ "opengeni.turn_id": turnId ?? "",
1849
+ "opengeni.status": activityStatus,
1850
+ "opengeni.environment_id": environmentId,
1851
+ "opengeni.duration_ms": Math.round(durationSeconds * 1000),
1852
+ },
1853
+ error: activityError,
1854
+ });
1855
+ // Multi-account P4: flush the serving account's free per-turn caches ONCE,
1856
+ // best-effort (same discipline as today's usage write). Both writers skip
1857
+ // version/updatedAt, so neither can race the token-refresh CAS.
1858
+ if (effectiveCodexCredentialId) {
1859
+ // Part A: the latest scraped usage-header snapshot → the P2 usage cache. A
1860
+ // full both-windows snapshot (parseCodexUsageHeaders gates on both), so this
1861
+ // is byte-identical to the /wham/usage write — no partial-window clobber.
1862
+ if (latestCodexUsage) {
1863
+ await recordCodexAccountUsage(db, input.workspaceId, effectiveCodexCredentialId, latestCodexUsage)
1864
+ .catch(() => undefined);
1865
+ }
1866
+ // Part B.1: the connector namespaces codex_apps listed this turn → the
1867
+ // connector-set cache. NON-EMPTY-only: a flaky/empty tools/list must never
1868
+ // overwrite a known set with [] (false coverage drop). Read by reference
1869
+ // AFTER the run, so every tools/list this turn has accumulated.
1870
+ const connectorNamespaces = preparedTools?.codexConnectorNamespaces;
1871
+ if (connectorNamespaces && connectorNamespaces.size > 0) {
1872
+ await recordCodexAccountConnectors(db, input.workspaceId, effectiveCodexCredentialId, [...connectorNamespaces])
1873
+ .catch(() => undefined);
1874
+ }
1875
+ }
1876
+ await preparedTools?.close().catch(() => undefined);
1877
+ if (heartbeatTimer) {
1878
+ clearInterval(heartbeatTimer);
1879
+ }
1880
+ // P1.2: stop the lease-TTL refresh, release the turn holder (idempotent
1881
+ // delete-my-row; refcount-- and warm->draining if it hit 0 with no turns),
1882
+ // and DROP the in-memory handle. Release NEVER stops the box — the reaper
1883
+ // (P1.3) issues the provider stop() past the drain grace at refcount 0; the
1884
+ // box rides the provider idle-timeout in the meantime. Best-effort: a
1885
+ // release failure must never mask the turn's real outcome.
1886
+ if (leaseHeartbeatTimer) {
1887
+ clearInterval(leaseHeartbeatTimer);
1888
+ }
1889
+ // P4.3: finalize the on-turn recording BEFORE releasing the box handle
1890
+ // (the read+PUT needs the live session). read bytes → storage PUT →
1891
+ // updateRecording(available) → publish recording.available; the box file is
1892
+ // deleted only after the PUT confirms (F9). Best-effort: a finalize failure
1893
+ // emits recording.failed and never masks the turn outcome (F10: the bytes
1894
+ // never become a Temporal payload — read+PUT happen here in-process).
1895
+ if (activeRecording && resolvedSandbox) {
1896
+ try {
1897
+ if (!didComputerUse) {
1898
+ // Recording gate (P4.3): the turn never drove the desktop (a plain text
1899
+ // turn), so the film is a static frame of an untouched screen. DISCARD
1900
+ // it — stop ffmpeg, delete the box artifact, drop the row — with NO
1901
+ // storage PUT and NO recording.failed (a clean no-op, not a failure).
1902
+ await discardRecording({
1903
+ db,
1904
+ accountId: input.accountId,
1905
+ workspaceId: input.workspaceId,
1906
+ active: activeRecording,
1907
+ session: resolvedSandbox.established.session,
1908
+ });
1909
+ } else {
1910
+ const outcome = await finalizeRecording({
1911
+ settings,
1912
+ db,
1913
+ objectStorage,
1914
+ accountId: input.accountId,
1915
+ workspaceId: input.workspaceId,
1916
+ sessionId: input.sessionId,
1917
+ active: activeRecording,
1918
+ session: resolvedSandbox.established.session,
1919
+ runAs: sandboxRunAs(settings),
1920
+ });
1921
+ if (publish) {
1922
+ await publish(outcome.ok
1923
+ ? [{ type: "recording.available", payload: outcome.available }]
1924
+ : [{ type: "recording.failed", payload: { recordingId: activeRecording.recordingId, turnId: activeRecording.turnId, reason: outcome.reason, detail: outcome.detail } }]);
1925
+ }
1926
+ }
1927
+ } catch (finalizeError) {
1928
+ console.error("recording finalize failed (turn outcome unaffected)", finalizeError);
1929
+ } finally {
1930
+ activeRecording = null;
1931
+ }
1932
+ }
1933
+ if (resolvedSandbox) {
1934
+ await resolvedSandbox.release().catch((releaseError) => {
1935
+ console.error("sandbox lease release failed (turn outcome unaffected)", releaseError);
1936
+ });
1937
+ resolvedSandbox = null; // drop the handle; the box survives the turn
1938
+ }
1939
+ }
1940
+ };
1941
+ }
1942
+
1943
+ export function agentRunFailurePayload(error: unknown): { error: string; code?: string; retryable?: boolean; detail?: string } {
1944
+ const message = error instanceof Error ? error.message : String(error);
1945
+ const status = typeof error === "object" && error !== null && "status" in error
1946
+ ? Number((error as { status?: unknown }).status)
1947
+ : undefined;
1948
+ const code = typeof error === "object" && error !== null && "code" in error
1949
+ ? String((error as { code?: unknown }).code)
1950
+ : undefined;
1951
+ // A ChatGPT/Codex usage cap is a HARD limit, not transient backpressure: it
1952
+ // must NOT be reported as a generic, retryable rate-limit (which would loop a
1953
+ // goal against a capped backend). Surface a precise, actionable message with
1954
+ // the humanized reset window and code, non-retryable. Checked BEFORE the
1955
+ // generic 429 branch below (a usage cap is also a 429).
1956
+ const usageLimit = classifyCodexUsageLimitError(error);
1957
+ if (usageLimit) {
1958
+ return codexUsageLimitFailurePayload(usageLimit, message);
1959
+ }
1960
+ if (status === 429 || code === "rate_limit_exceeded" || /(?:too many requests|rate.?limit|\b429\b)/i.test(message)) {
1961
+ return {
1962
+ error: "Model provider rate limit hit. Try again in a minute or lower the reasoning effort.",
1963
+ code: "provider_rate_limited",
1964
+ retryable: true,
1965
+ ...(message && message !== "Too Many Requests" ? { detail: message } : {}),
1966
+ };
1967
+ }
1968
+ return { error: message };
1969
+ }
1970
+
1971
+ /** Humanize a seconds duration into a short "2h 5m" / "9m" / "in under a minute" string. */
1972
+ export function humanizeResetWindow(resetsInSeconds: number | null): string {
1973
+ if (resetsInSeconds === null || !Number.isFinite(resetsInSeconds) || resetsInSeconds <= 0) {
1974
+ return "shortly";
1975
+ }
1976
+ const total = Math.ceil(resetsInSeconds);
1977
+ if (total < 60) {
1978
+ return "in under a minute";
1979
+ }
1980
+ const hours = Math.floor(total / 3600);
1981
+ const minutes = Math.round((total % 3600) / 60);
1982
+ if (hours > 0) {
1983
+ return minutes > 0 ? `in about ${hours}h ${minutes}m` : `in about ${hours}h`;
1984
+ }
1985
+ return `in about ${minutes}m`;
1986
+ }
1987
+
1988
+ /**
1989
+ * Build the turn.failed payload for a ChatGPT/Codex usage cap: a precise,
1990
+ * actionable message naming the reset window, the stable `codex_usage_limit_reached`
1991
+ * code, and retryable:false (an auto-retry would just re-hit the cap).
1992
+ */
1993
+ export function codexUsageLimitFailurePayload(
1994
+ info: { resetsInSeconds: number | null },
1995
+ detail: string,
1996
+ opts?: { allAccounts?: boolean },
1997
+ ): { error: string; code: string; retryable: boolean; detail?: string } {
1998
+ // P3: when EVERY connected subscription is rate-limited the message names the
1999
+ // earliest reset across accounts; the single-account message is unchanged.
2000
+ const error = opts?.allAccounts
2001
+ ? `All connected ChatGPT/Codex subscriptions are rate-limited. Access returns ${humanizeResetWindow(info.resetsInSeconds)}. `
2002
+ + `You can switch this session to a different model in the meantime, or wait for a subscription to reset.`
2003
+ : `Your ChatGPT/Codex subscription usage limit has been reached. Access resets ${humanizeResetWindow(info.resetsInSeconds)}. `
2004
+ + `You can switch this session to a different model in the meantime, or wait for the limit to reset.`;
2005
+ return {
2006
+ error,
2007
+ code: "codex_usage_limit_reached",
2008
+ retryable: false,
2009
+ ...(detail ? { detail } : {}),
2010
+ };
2011
+ }
2012
+
2013
+ // A usage cap that won't reset for a long time should not pin a Temporal timer
2014
+ // open indefinitely for a goal-bearing session; cap the continuation hold so the
2015
+ // goal re-evaluates at most this far out (it will re-pause if still capped).
2016
+ const CODEX_USAGE_LIMIT_MAX_RESUME_MS = 60 * 60_000; // 1h
2017
+
2018
+ /**
2019
+ * Recognize an SDK stream event that represents a COMPUTER-USE tool call actually
2020
+ * executing — a `run_item_stream_event` carrying a `tool_call_item` whose
2021
+ * underlying raw item is a `computer_call` (the @openai/agents computer tool's
2022
+ * action: screenshot/click/type/scroll/drag/keypress/move/wait/…). This is the
2023
+ * same shape `normalizeSdkEvent` reads (`event.item.rawItem`), matched here against
2024
+ * the raw SDK type rather than a tool NAME so it is robust to the computer tool's
2025
+ * configured name. Drives the on-turn recording gate (no computer-use → discard).
2026
+ */
2027
+ function isComputerCallStreamEvent(event: unknown): boolean {
2028
+ if (!event || typeof event !== "object") {
2029
+ return false;
2030
+ }
2031
+ if ((event as { type?: unknown }).type !== "run_item_stream_event") {
2032
+ return false;
2033
+ }
2034
+ const item = (event as { item?: { type?: unknown; rawItem?: { type?: unknown } } }).item;
2035
+ if (!item || item.type !== "tool_call_item") {
2036
+ return false;
2037
+ }
2038
+ return item.rawItem?.type === "computer_call";
2039
+ }
2040
+
2041
+ /**
2042
+ * Budget/limit exhaustion detected between model calls. This is account
2043
+ * state, not an agent failure: the segment ends gracefully (session idles,
2044
+ * run state preserved) so a top-up or limit reset lets the same session
2045
+ * continue — a failed session would reject the user's next message. An
2046
+ * active goal pauses visibly (reason "limits") at the next continuation
2047
+ * evaluation without consuming continuation budget.
2048
+ */
2049
+ class BudgetExhaustedError extends Error {
2050
+ constructor(message: string, readonly serializedRunState: string | null) {
2051
+ super(message);
2052
+ this.name = "BudgetExhaustedError";
2053
+ }
2054
+ }
2055
+
2056
+ // Exported for unit testing the codex-billed bypass (codex-billing.test.ts); not part
2057
+ // of the activity surface. Takes BOTH `isCodexTurn` (codex-plan turns bypass the credit
2058
+ // and token gates) and the optional §7.5 P3 host `entitlements` port (when bound, its
2059
+ // `admitRun` REPLACES the local credit read for a non-codex turn; unset → local ledger).
2060
+ export async function ensureRunAllowed(
2061
+ settings: Settings,
2062
+ db: ActivityServices["db"],
2063
+ accountId: string,
2064
+ workspaceId: string,
2065
+ isCodexTurn: boolean,
2066
+ entitlements?: ActivityServices["entitlements"],
2067
+ ): Promise<void> {
2068
+ // Codex-billed turns are paid by the user's ChatGPT/Codex plan: skip the
2069
+ // credit-balance gate and the monthly token cap. The agent-run COUNT cap below
2070
+ // is a volume/fairness quota (not a credit/cost gate) and is intentionally kept.
2071
+ //
2072
+ // §7.5 P3 — host-entitlements DELEGATION (the worker half of the same seam the
2073
+ // API edge exposes). For a non-codex turn, when the host binds `entitlements`, its
2074
+ // `admitRun` decision REPLACES the local credit-balance read below: a host that owns
2075
+ // its ledger/meter is the funding authority. A deny throws the SAME Error the local
2076
+ // read throws, so the mid-stream budget-valve at :727 wraps it in a
2077
+ // `BudgetExhaustedError` and pauses identically — the valve never learns whether the
2078
+ // deny came from the local ledger or the host meter.
2079
+ //
2080
+ // This is an admission READ only; it records NO usage (metering stays the sole,
2081
+ // idempotency-keyed writer at recordModelUsageAndDebitCredits), so a PULL host meter
2082
+ // is consulted without ever double-charging.
2083
+ if (!isCodexTurn && entitlements && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
2084
+ const decision = await entitlements.admitRun({
2085
+ accountId,
2086
+ workspaceId,
2087
+ action: "agent_run:create",
2088
+ quantity: 1,
2089
+ });
2090
+ if (!decision.allowed) {
2091
+ throw new Error(decision.reason || "insufficient OpenGeni credits");
2092
+ }
2093
+ } else if (!isCodexTurn && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
2094
+ const balance = await getBillingBalance(db, accountId);
2095
+ if (balance.balanceMicros <= 0) {
2096
+ throw new Error("insufficient OpenGeni credits");
2097
+ }
2098
+ }
2099
+ if (settings.usageLimitsMode === "static" || settings.usageLimitsMode === "managed") {
2100
+ const limits = configuredStaticUsageLimits(settings);
2101
+ if (limits.maxMonthlyAgentRunsPerWorkspace) {
2102
+ const used = await sumUsageQuantity(db, {
2103
+ workspaceId,
2104
+ eventType: "agent_run.created",
2105
+ since: startOfUtcMonth(),
2106
+ });
2107
+ // Agent turns are admitted and recorded before this worker activity starts.
2108
+ // Equality means this accepted turn is exactly at the cap; greater-than is
2109
+ // the race/backstop case where another admission already exceeded the cap.
2110
+ if (used > limits.maxMonthlyAgentRunsPerWorkspace) {
2111
+ throw new Error(`monthly agent run limit reached (${limits.maxMonthlyAgentRunsPerWorkspace})`);
2112
+ }
2113
+ }
2114
+ if (!isCodexTurn && limits.maxMonthlyTokensPerWorkspace) {
2115
+ const used = await sumUsageQuantity(db, {
2116
+ workspaceId,
2117
+ eventType: "model.tokens",
2118
+ since: startOfUtcMonth(),
2119
+ });
2120
+ if (used >= limits.maxMonthlyTokensPerWorkspace) {
2121
+ throw new Error(`monthly token limit reached (${limits.maxMonthlyTokensPerWorkspace})`);
2122
+ }
2123
+ }
2124
+ }
2125
+ }
2126
+
2127
+ // Exported for unit testing the codex-billed bypass; not part of the activity surface.
2128
+ export async function recordModelUsageAndDebitCredits(settings: Settings, db: ActivityServices["db"], input: {
2129
+ accountId: string;
2130
+ workspaceId: string;
2131
+ sessionId: string;
2132
+ turnId: string;
2133
+ model: string;
2134
+ isCodexTurn: boolean;
2135
+ usage?: ModelUsageInput | null;
2136
+ sourceKey: string;
2137
+ }): Promise<void> {
2138
+ if (!input.usage) {
2139
+ return;
2140
+ }
2141
+ const inputTokens = positiveInt(input.usage.inputTokens);
2142
+ const outputTokens = positiveInt(input.usage.outputTokens);
2143
+ const totalTokens = positiveInt(input.usage.totalTokens) || inputTokens + outputTokens;
2144
+ // A codex-subscription turn is paid by the user's ChatGPT/Codex plan, so it
2145
+ // consumes ZERO OpenGeni credits and must never feed an OpenGeni cap. A
2146
+ // codex/<slug> model has no entry in configuredModelPricing, so the normal path
2147
+ // below would throw "Missing model pricing". We:
2148
+ // - do NOT emit the cap-feeding `model.tokens` event (ensureRunAllowed and
2149
+ // the API tokens:consume cap sum `model.tokens` with NO cost dimension, so
2150
+ // any row would count against maxMonthlyTokensPerWorkspace);
2151
+ // - record a `model.cost = 0` audit marker (harmless to the monthly cost cap);
2152
+ // - never look up pricing and never debit credits.
2153
+ if (input.isCodexTurn) {
2154
+ await recordUsageEvent(db, {
2155
+ accountId: input.accountId,
2156
+ workspaceId: input.workspaceId,
2157
+ eventType: "model.cost",
2158
+ quantity: 0,
2159
+ unit: "usd_micros",
2160
+ sourceResourceType: "model_response",
2161
+ sourceResourceId: `${input.turnId}:${input.sourceKey}`,
2162
+ idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`,
2163
+ });
2164
+ return;
2165
+ }
2166
+ if (totalTokens > 0) {
2167
+ await recordUsageEvent(db, {
2168
+ accountId: input.accountId,
2169
+ workspaceId: input.workspaceId,
2170
+ eventType: "model.tokens",
2171
+ quantity: totalTokens,
2172
+ unit: "tokens",
2173
+ sourceResourceType: "model_response",
2174
+ sourceResourceId: `${input.turnId}:${input.sourceKey}`,
2175
+ idempotencyKey: `usage:model.tokens:${input.turnId}:${input.sourceKey}`,
2176
+ });
2177
+ }
2178
+ const shouldDebit = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
2179
+ if (!shouldDebit || totalTokens === 0) {
2180
+ return;
2181
+ }
2182
+ if (!configuredModelPricing(settings)[input.model]) {
2183
+ throw new Error(`Missing model pricing for ${input.model}`);
2184
+ }
2185
+ const costMicros = calculateModelUsageCostMicros(settings, input.model, input.usage);
2186
+ await recordUsageEvent(db, {
2187
+ accountId: input.accountId,
2188
+ workspaceId: input.workspaceId,
2189
+ eventType: "model.cost",
2190
+ quantity: costMicros,
2191
+ unit: "usd_micros",
2192
+ sourceResourceType: "model_response",
2193
+ sourceResourceId: `${input.turnId}:${input.sourceKey}`,
2194
+ idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`,
2195
+ });
2196
+ if (costMicros > 0) {
2197
+ await applyCreditDebitUpToBalance(db, {
2198
+ accountId: input.accountId,
2199
+ workspaceId: input.workspaceId,
2200
+ type: "model_usage_debit",
2201
+ requestedAmountMicros: costMicros,
2202
+ sourceType: "model_response",
2203
+ sourceId: `${input.turnId}:${input.sourceKey}`,
2204
+ idempotencyKey: `credit:model_usage_debit:${input.turnId}:${input.sourceKey}`,
2205
+ metadata: {
2206
+ model: input.model,
2207
+ sessionId: input.sessionId,
2208
+ turnId: input.turnId,
2209
+ sourceKey: input.sourceKey,
2210
+ inputTokens,
2211
+ outputTokens,
2212
+ totalTokens,
2213
+ },
2214
+ });
2215
+ }
2216
+ }
2217
+
2218
+ function positiveInt(value: unknown): number {
2219
+ return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
2220
+ }
2221
+
2222
+ function startOfUtcMonth(): Date {
2223
+ const now = new Date();
2224
+ return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
2225
+ }
2226
+
2227
+ async function sandboxFileDownloadsForRun(
2228
+ settings: Settings,
2229
+ db: ActivityServices["db"],
2230
+ objectStorage: ObjectStorage | null,
2231
+ workspaceId: string,
2232
+ resources: ResourceRef[],
2233
+ ): Promise<SandboxFileDownload[]> {
2234
+ if (settings.sandboxBackend === "none" || !requiresSignedFileResourceDownloads(settings)) {
2235
+ return [];
2236
+ }
2237
+ const fileResources = resources.filter((resource): resource is Extract<ResourceRef, { kind: "file" }> => resource.kind === "file");
2238
+ if (fileResources.length === 0) {
2239
+ return [];
2240
+ }
2241
+ if (!objectStorage) {
2242
+ throw new Error(`${settings.objectStorageBackend} file resources require configured object storage`);
2243
+ }
2244
+ const downloadStorage = objectStorageForSandboxDownloads(settings, objectStorage);
2245
+ const downloads: SandboxFileDownload[] = [];
2246
+ for (const resource of fileResources) {
2247
+ const file = await requireFile(db, workspaceId, resource.fileId);
2248
+ const url = await downloadStorage.createGetUrl({ key: file.objectKey });
2249
+ downloads.push({
2250
+ fileId: file.id,
2251
+ mountPath: resource.mountPath ?? `files/${file.id}`,
2252
+ filename: file.safeFilename,
2253
+ url: url.url,
2254
+ expiresAt: url.expiresAt,
2255
+ sizeBytes: file.sizeBytes,
2256
+ });
2257
+ }
2258
+ return downloads;
2259
+ }
2260
+
2261
+ function requiresSignedFileResourceDownloads(settings: Settings): boolean {
2262
+ // A selfhosted machine (bring-your-own-compute) can NEVER mount ANY object store
2263
+ // — it is a remote user machine reached only over NATS, so file resources are
2264
+ // ALWAYS delivered by exec-curling a pre-signed URL onto it. Without this a
2265
+ // machine-home turn (sandbox_backend "selfhosted") would silently drop file
2266
+ // resources on an azure-blob / s3-compatible store (nativeBucketMount=false),
2267
+ // a regression from the pre-honest-label path where the same turn ran home=modal
2268
+ // and modal's descriptor forced signed downloads.
2269
+ if (settings.sandboxBackend === "selfhosted") {
2270
+ return true;
2271
+ }
2272
+ // A nativeBucketMount backend (modal) cannot mount Azure Blob entries, so it
2273
+ // needs pre-signed downloads for that store. Keying on the descriptor (not the
2274
+ // "modal" literal) keeps this correct as bucket-mount backends are added.
2275
+ const nativeBucketMount = CAPABILITY_DESCRIPTORS[settings.sandboxBackend].nativeBucketMount;
2276
+ return (settings.sandboxBackend === "docker" && settings.objectStorageBackend === "s3-compatible")
2277
+ || settings.objectStorageBackend === "aws-s3"
2278
+ || settings.objectStorageBackend === "gcs"
2279
+ || (nativeBucketMount && settings.objectStorageBackend === "azure-blob");
2280
+ }
2281
+
2282
+ function objectStorageForSandboxDownloads(settings: Settings, objectStorage: ObjectStorage): ObjectStorage {
2283
+ if (settings.objectStorageBackend !== "s3-compatible" || !settings.objectStorageSandboxEndpoint) {
2284
+ return objectStorage;
2285
+ }
2286
+ return createObjectStorage({
2287
+ ...settings,
2288
+ objectStorageEndpoint: settings.objectStorageSandboxEndpoint,
2289
+ }) ?? objectStorage;
2290
+ }