@opengeni/worker-bundle 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +222 -0
- package/dist/index.js +3960 -0
- package/dist/index.js.map +1 -0
- package/package.json +60 -0
- package/src/activities/agent-turn.ts +2290 -0
- package/src/activities/capabilities.ts +131 -0
- package/src/activities/codex-auth.ts +9 -0
- package/src/activities/codex-rotation.ts +326 -0
- package/src/activities/common.ts +37 -0
- package/src/activities/context-compaction.ts +125 -0
- package/src/activities/documents.ts +67 -0
- package/src/activities/environment.ts +209 -0
- package/src/activities/goals.ts +281 -0
- package/src/activities/packs.ts +116 -0
- package/src/activities/parent-wake.ts +129 -0
- package/src/activities/recording.ts +248 -0
- package/src/activities/redaction.ts +54 -0
- package/src/activities/run-input.ts +331 -0
- package/src/activities/sandbox-lease.ts +612 -0
- package/src/activities/scheduled-tasks.ts +346 -0
- package/src/activities/session-state.ts +201 -0
- package/src/activities/streaming.ts +74 -0
- package/src/activities/types.ts +165 -0
- package/src/activities.ts +124 -0
- package/src/index.ts +184 -0
- package/src/sandbox-resume.ts +558 -0
- package/src/sandbox-routing.ts +232 -0
- package/src/workflows/activities.ts +23 -0
- package/src/workflows/document-index.ts +11 -0
- package/src/workflows/sandbox-reaper.ts +28 -0
- package/src/workflows/scheduled-tasks.ts +38 -0
- package/src/workflows/session.ts +376 -0
- package/src/workflows.ts +19 -0
|
@@ -0,0 +1,2290 @@
|
|
|
1
|
+
import {
|
|
2
|
+
claimNextQueuedTurn as claimNextQueuedTurnDb,
|
|
3
|
+
createTurn,
|
|
4
|
+
applyCreditDebitUpToBalance,
|
|
5
|
+
finishTurn,
|
|
6
|
+
getBillingBalance,
|
|
7
|
+
getSandbox,
|
|
8
|
+
readActiveSandbox,
|
|
9
|
+
requireFile,
|
|
10
|
+
getSessionEvent,
|
|
11
|
+
getSessionGoal,
|
|
12
|
+
getSessionTurn,
|
|
13
|
+
isCodexBilledTurn,
|
|
14
|
+
workspaceCodexSubscriptionActive,
|
|
15
|
+
getCodexRotationSettings,
|
|
16
|
+
listCodexAccountStatuses,
|
|
17
|
+
fetchCodexUsageForAccount,
|
|
18
|
+
getSessionCodexState,
|
|
19
|
+
recordSessionActiveCodexCredential,
|
|
20
|
+
recordCodexAccountUsage,
|
|
21
|
+
recordCodexAccountConnectors,
|
|
22
|
+
setActiveCodexCredential,
|
|
23
|
+
setCodexCredentialExhausted,
|
|
24
|
+
countConsecutiveReactiveRotations,
|
|
25
|
+
requireSession,
|
|
26
|
+
recordUsageEvent,
|
|
27
|
+
appendSessionHistoryItems,
|
|
28
|
+
consumeSessionCompactionRequest,
|
|
29
|
+
countSessionHistoryItems,
|
|
30
|
+
getActiveSessionHistoryItems,
|
|
31
|
+
nextSessionHistoryPosition,
|
|
32
|
+
requeuePreemptedTurn,
|
|
33
|
+
saveRunState,
|
|
34
|
+
upsertSandboxSessionEnvelope,
|
|
35
|
+
setSessionStatus,
|
|
36
|
+
setSessionLastInputTokens,
|
|
37
|
+
sumUsageQuantity,
|
|
38
|
+
heartbeatLeaseHolder,
|
|
39
|
+
accrueWarmSeconds,
|
|
40
|
+
SandboxLeaseSupersededError,
|
|
41
|
+
type AppendEventInput,
|
|
42
|
+
} from "@opengeni/db";
|
|
43
|
+
import { appendAndPublishEvents } from "@opengeni/events";
|
|
44
|
+
import {
|
|
45
|
+
sandboxStateEntryFromRunState,
|
|
46
|
+
agentsErrorRunState,
|
|
47
|
+
maxTurnsExceededRunState,
|
|
48
|
+
modelResponseUsageFromSdkEvent,
|
|
49
|
+
normalizeSdkEvent,
|
|
50
|
+
sanitizeHistoryItemsForModel,
|
|
51
|
+
summarizeForCompaction,
|
|
52
|
+
type SandboxFileDownload,
|
|
53
|
+
type OpenGeniRuntime,
|
|
54
|
+
} from "@opengeni/runtime";
|
|
55
|
+
import { calculateModelUsageCostMicros, configuredModelPricing, configuredStaticUsageLimits, sandboxWarmRateMicrosPerSecond, type ModelUsageInput, type Settings } from "@opengeni/config";
|
|
56
|
+
import { CancelledFailure } from "@temporalio/activity";
|
|
57
|
+
import { settingsWithCodexCredential, settingsWithEnabledCapabilityMcpServers } from "./capabilities";
|
|
58
|
+
import { chooseRotationActive, computeIdleDelayMs, computeReactiveRotationResume, type CodexRotationStrategy, type RotationDecision } from "./codex-rotation";
|
|
59
|
+
import type { CodexAccountStatus } from "@opengeni/db";
|
|
60
|
+
import { buildCodexTokenResolver } from "./codex-auth";
|
|
61
|
+
import {
|
|
62
|
+
buildModelResolver,
|
|
63
|
+
CODEX_CLIENT_VERSION,
|
|
64
|
+
CODEX_FALLBACK_MODEL_SLUGS,
|
|
65
|
+
classifyCodexUsageLimitError,
|
|
66
|
+
codexRequestStorage,
|
|
67
|
+
type CodexRequestContext,
|
|
68
|
+
type CodexUsageHeaderSnapshot,
|
|
69
|
+
} from "@opengeni/codex";
|
|
70
|
+
import {
|
|
71
|
+
mergeResourceRefs,
|
|
72
|
+
mergeToolRefs,
|
|
73
|
+
} from "./common";
|
|
74
|
+
import { maybeCompactContext } from "./context-compaction";
|
|
75
|
+
import { loadWorkspaceEnvironmentForRunWithCredentials, sandboxEnvironmentForRun } from "./environment";
|
|
76
|
+
import { withCodexAppsTool, withFirstPartyTools } from "./goals";
|
|
77
|
+
import { resolveWorkspaceAgentInstructions, resolveWorkspacePackRuntime, settingsWithPackSandboxImage } from "./packs";
|
|
78
|
+
import { notifyParentOfChildTerminal } from "./parent-wake";
|
|
79
|
+
import { createSecretRedactor, identityRedactor } from "./redaction";
|
|
80
|
+
import { applyCodexHistoryStrip, turnInput, type TurnCodexAccount } from "./run-input";
|
|
81
|
+
import {
|
|
82
|
+
createRuntimeBatcher,
|
|
83
|
+
currentActivityContext,
|
|
84
|
+
nextStreamEvent,
|
|
85
|
+
startActivityHeartbeat,
|
|
86
|
+
} from "./streaming";
|
|
87
|
+
import type {
|
|
88
|
+
ActivityServices,
|
|
89
|
+
RunAgentTurnInput,
|
|
90
|
+
RunAgentTurnResult,
|
|
91
|
+
} from "./types";
|
|
92
|
+
import { resumeBoxForTurn, acquireSelfhostedLeaseForTurn, type ResumedTurnSandbox } from "../sandbox-resume";
|
|
93
|
+
import { wrapTurnBoxWithRouting, establishSelfhostedTurnSession, routingEnabled } from "../sandbox-routing";
|
|
94
|
+
import { beginRecording, discardRecording, finalizeRecording, type ActiveRecording } from "./recording";
|
|
95
|
+
import { createObjectStorage, type ObjectStorage } from "@opengeni/storage";
|
|
96
|
+
import { desktopCapableBackend, sandboxRunAs } from "@opengeni/runtime";
|
|
97
|
+
import { CAPABILITY_DESCRIPTORS, type ResourceRef } from "@opengeni/contracts";
|
|
98
|
+
import { randomUUID } from "node:crypto";
|
|
99
|
+
|
|
100
|
+
// How long the session workflow holds the loop after a retryable provider
|
|
101
|
+
// failure before the goal continuation re-enters the model. Azure/OpenAI TPM
|
|
102
|
+
// throttling is minute-granular; anything shorter mostly burns continuation
|
|
103
|
+
// budget against the same window.
|
|
104
|
+
export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Resolve which Codex account a turn runs on (multi-account P1): session-pin >
|
|
108
|
+
* workspace-active. No rotation in P1. The selected id must still be in the
|
|
109
|
+
* connected set — a disconnected pin was FK-nulled, so a stale id can't appear,
|
|
110
|
+
* but we guard anyway. Returns null when there is no usable account (the turn
|
|
111
|
+
* then fails with the existing relogin error path).
|
|
112
|
+
*/
|
|
113
|
+
export function selectCodexCredentialForTurn(args: {
|
|
114
|
+
sessionPinnedCredentialId: string | null;
|
|
115
|
+
activeCredentialId: string | null;
|
|
116
|
+
connectedIds: Set<string>;
|
|
117
|
+
}): string | null {
|
|
118
|
+
const { sessionPinnedCredentialId: pin, activeCredentialId: active, connectedIds } = args;
|
|
119
|
+
if (pin && connectedIds.has(pin)) {
|
|
120
|
+
return pin;
|
|
121
|
+
}
|
|
122
|
+
if (active && connectedIds.has(active)) {
|
|
123
|
+
return active;
|
|
124
|
+
}
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Resume notice fed to the model when a turn re-enters after a graceful
|
|
129
|
+
// worker shutdown (deploy/rollout restart) checkpointed it mid-flight. The
|
|
130
|
+
// agent must not blindly replay side effects it cannot see in its history:
|
|
131
|
+
// at most the single in-flight model step was lost, and anything it started
|
|
132
|
+
// there may or may not have completed.
|
|
133
|
+
export const WORKER_SHUTDOWN_RESUME_TEXT = [
|
|
134
|
+
"[TURN RESUMED AFTER WORKER RESTART] The platform worker running this turn restarted (deploy/rollout) before the turn finished.",
|
|
135
|
+
"Your conversation history above, including this turn's earlier work, is preserved up to the last checkpoint; at most the single in-flight model step after it was lost.",
|
|
136
|
+
"Continue the original task from where it left off. Before repeating any action with side effects, check whether it already happened.",
|
|
137
|
+
].join("\n");
|
|
138
|
+
|
|
139
|
+
// Resume notice for the harder variant: the worker died WITHOUT running the
|
|
140
|
+
// graceful checkpoint (SIGKILL, OOM, node loss — surfaced to the workflow as
|
|
141
|
+
// a heartbeat-timeout activity failure). Conversation truth is still
|
|
142
|
+
// dual-written after every model response, so the bound is the same order as
|
|
143
|
+
// a graceful preempt, but nothing after the last reconcile was captured and
|
|
144
|
+
// the agent must be even more suspicious about repeated side effects.
|
|
145
|
+
export const WORKER_DEATH_RESUME_TEXT = [
|
|
146
|
+
"[TURN RESUMED AFTER WORKER CRASH] The platform worker running this turn died before the turn finished.",
|
|
147
|
+
"Your conversation history above, including this turn's earlier work, is preserved up to the last checkpoint; anything in flight after that checkpoint was lost.",
|
|
148
|
+
"Continue the original task from where it left off. Before repeating any action with side effects, check whether it already happened.",
|
|
149
|
+
].join("\n");
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* True when this activity attempt was cancelled because its hosting worker is
|
|
153
|
+
* shutting down gracefully (SIGTERM during a deploy), as opposed to a
|
|
154
|
+
* workflow-requested cancellation (user interrupt) or a server-side timeout.
|
|
155
|
+
*/
|
|
156
|
+
export function isWorkerShutdownCancellation(error: unknown): boolean {
|
|
157
|
+
return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Compute the conversation-truth rows a reconcile pass should append, given the
|
|
162
|
+
* SDK's current `state.history` and the count already persisted.
|
|
163
|
+
*
|
|
164
|
+
* `state.history` is a computed getter that runs the SDK's orphan-tool-call
|
|
165
|
+
* pruning on every access, so it is non-monotonic: a `function_call` with no
|
|
166
|
+
* settling result yet is transiently absent and a later access yields a
|
|
167
|
+
* different, possibly shorter/reordered list. The old code sliced this list by
|
|
168
|
+
* a blind length watermark and appended at fixed positions with
|
|
169
|
+
* onConflictDoNothing, which could freeze a position with one shape and later
|
|
170
|
+
* persist a `function_call_result` whose `function_call` had been pruned away in
|
|
171
|
+
* an earlier slice — the orphaned tool output that 400s the Responses API and
|
|
172
|
+
* bricks the session on every replay.
|
|
173
|
+
*
|
|
174
|
+
* Defending: sanitize the full current history into an API-valid sequence (the
|
|
175
|
+
* same pure rules the read path uses), then append only the new tail beyond the
|
|
176
|
+
* watermark. A trailing dangling call is dropped here and re-evaluated next
|
|
177
|
+
* pass once its result lands, so a call and its result are written together at
|
|
178
|
+
* consecutive positions and a result is never persisted without its call. The
|
|
179
|
+
* watermark advances to the sanitized length — never past anything unwritten —
|
|
180
|
+
* so a non-monotonic history can never desync it. When previously-persisted
|
|
181
|
+
* rows already exceed the sanitized length (e.g. legacy orphans written before
|
|
182
|
+
* this fix), nothing new is appended and the watermark holds steady.
|
|
183
|
+
*/
|
|
184
|
+
/**
|
|
185
|
+
* Stable+unique usage source key for one model call, used to build the per-call
|
|
186
|
+
* idempotency key (`usage:model.tokens:${turnId}:${sourceKey}`). The turnId is
|
|
187
|
+
* shared across a re-dispatch of the SAME turn (preemption resume, approval
|
|
188
|
+
* rerun, activity retry), so the sourceKey alone must distinguish calls.
|
|
189
|
+
*
|
|
190
|
+
* - A provider responseId is globally stable+unique, so reuse it verbatim: a
|
|
191
|
+
* true activity retry that re-emits the same responseId correctly DEDUPES
|
|
192
|
+
* (one charge), while two distinct calls get distinct ids.
|
|
193
|
+
* - Without a responseId the only fallback was POSITIONAL ("response-1",
|
|
194
|
+
* "aggregate"), which collides across a re-dispatch — dispatch B's first
|
|
195
|
+
* call reuses dispatch A's "response-1" key and its charge is silently
|
|
196
|
+
* dropped (undercharge). Qualifying the positional fallback with the
|
|
197
|
+
* per-execution dispatch id (the Temporal activityId, unique per scheduled
|
|
198
|
+
* execution) makes re-dispatched calls distinct while still deduping a
|
|
199
|
+
* same-execution retry.
|
|
200
|
+
*/
|
|
201
|
+
export function modelUsageSourceKey(input: {
|
|
202
|
+
responseId?: string | null | undefined;
|
|
203
|
+
dispatchId: string | null;
|
|
204
|
+
positionalKey: string;
|
|
205
|
+
}): string {
|
|
206
|
+
if (input.responseId) {
|
|
207
|
+
return input.responseId;
|
|
208
|
+
}
|
|
209
|
+
return input.dispatchId ? `${input.dispatchId}:${input.positionalKey}` : input.positionalKey;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
export function historyRowsToAppend(
|
|
213
|
+
rawHistory: Array<Record<string, unknown>>,
|
|
214
|
+
// How many items of the CURRENT in-memory history are already persisted (the
|
|
215
|
+
// slice index into `sanitized`). This is the in-memory history length, NOT the
|
|
216
|
+
// total persisted-row count: after a compaction the in-memory history is the
|
|
217
|
+
// short [summary, ...tail, ...new] list, far shorter than the total rows in
|
|
218
|
+
// the table (which still hold the superseded prefix).
|
|
219
|
+
persistedHistoryCount: number,
|
|
220
|
+
// Next free WHOLE-NUMBER absolute position to write at. Decoupled from the
|
|
221
|
+
// slice index because compaction inserts a fractional summary position, so the
|
|
222
|
+
// total-row count no longer equals max(position)+1. Defaults to
|
|
223
|
+
// persistedHistoryCount to preserve the pre-compaction behaviour (contiguous
|
|
224
|
+
// positions from 0) when callers do not pass an explicit next position.
|
|
225
|
+
nextPosition: number = persistedHistoryCount,
|
|
226
|
+
): { rows: Array<{ position: number; item: Record<string, unknown> }>; nextWatermark: number; nextPosition: number } {
|
|
227
|
+
const sanitized = sanitizeHistoryItemsForModel(rawHistory);
|
|
228
|
+
if (sanitized.length <= persistedHistoryCount) {
|
|
229
|
+
return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
|
|
230
|
+
}
|
|
231
|
+
const rows = sanitized.slice(persistedHistoryCount).map((item, offset) => ({
|
|
232
|
+
position: nextPosition + offset,
|
|
233
|
+
item: item as Record<string, unknown>,
|
|
234
|
+
}));
|
|
235
|
+
return { rows, nextWatermark: sanitized.length, nextPosition: nextPosition + rows.length };
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Seed the turn-end reconcile watermark (`persistedHistoryCount`) from EXACTLY the
|
|
240
|
+
* view `state.history` was seeded from, so the model-input length and the watermark
|
|
241
|
+
* can NEVER disagree. The watermark is the slice index the reconcile cuts the
|
|
242
|
+
* (re-sanitized) `state.history` at to find this turn's genuinely-new items, so it
|
|
243
|
+
* must equal the length of `state.history`'s already-persisted leading prefix.
|
|
244
|
+
*
|
|
245
|
+
* The cross-account reasoning strip drops foreign reasoning items, so the prefix
|
|
246
|
+
* length is PATH-DEPENDENT, captured by `modelHistoryFromItems`:
|
|
247
|
+
*
|
|
248
|
+
* - items read path (`modelHistoryFromItems === true`) — `state.history` was seeded
|
|
249
|
+
* from the cross-account-STRIPPED active items (foreign reasoning DROPPED), so it
|
|
250
|
+
* starts K shorter than the raw active-row count. Seed from the SAME strip
|
|
251
|
+
* (HOLE D); seeding from the un-stripped count would slice K genuinely-new items
|
|
252
|
+
* off the reconcile and silently lose them (incl. the user's switch-turn message).
|
|
253
|
+
*
|
|
254
|
+
* - run-state BLOB path (`modelHistoryFromItems === false`: approval resume, the
|
|
255
|
+
* items-mode run-state fallback, run_state mode) — `state.history` was seeded
|
|
256
|
+
* from the blob, where foreign reasoning is NEUTRALIZED-IN-PLACE (the item is
|
|
257
|
+
* KEPT, only its id/encrypted_content go — see resumeRunStateForCodexAccount), so
|
|
258
|
+
* the blob's history length still COUNTS those items. Applying the strip here
|
|
259
|
+
* under-counts by K and the reconcile re-appends K already-persisted items at
|
|
260
|
+
* fresh positions — HOLE E. So the blob path must NOT strip: count the raw
|
|
261
|
+
* sanitized active length, which mirrors the blob's completed prefix.
|
|
262
|
+
*
|
|
263
|
+
* On a same-account / non-codex turn the strip is a no-op, so both branches reduce
|
|
264
|
+
* to the same raw sanitized count (byte-identical to the pre-strip behaviour).
|
|
265
|
+
* Pure; exported for unit testing the D/E seed invariant.
|
|
266
|
+
*/
|
|
267
|
+
export function reconcileSeedCount(
|
|
268
|
+
activeSeedRows: ReadonlyArray<{ item: Record<string, unknown>; producerCodexCredentialId: string | null }>,
|
|
269
|
+
modelHistoryFromItems: boolean,
|
|
270
|
+
current: TurnCodexAccount,
|
|
271
|
+
): number {
|
|
272
|
+
return sanitizeHistoryItemsForModel(
|
|
273
|
+
modelHistoryFromItems
|
|
274
|
+
? applyCodexHistoryStrip(activeSeedRows, current)
|
|
275
|
+
: activeSeedRows.map((row) => row.item),
|
|
276
|
+
).length;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* Resolve the EFFECTIVE/active compute backend a turn should gate
|
|
281
|
+
* filesystem-touching agent lifecycle hooks on (today: the repository clone).
|
|
282
|
+
*
|
|
283
|
+
* WHY (Case B — clone-onto-real-disk hazard): a session keeps its CLOUD HOME
|
|
284
|
+
* backend (`settings.sandboxBackend`, e.g. "modal") but its ACTIVE sandbox may
|
|
285
|
+
* have been swapped to a connected machine (`active_sandbox_id` → a selfhosted
|
|
286
|
+
* lease). `runtime.buildAgent`'s repository-clone hook keys off the backend it is
|
|
287
|
+
* told; if the worker passes nothing it defaults to the HOME backend and the hook
|
|
288
|
+
* would `git clone` a private GitHub-App repo onto the user's REAL disk — a
|
|
289
|
+
* bring-your-own machine owns its own filesystem and must NEVER be cloned onto. So
|
|
290
|
+
* we look at where the agent ACTUALLY runs, not where the session was created.
|
|
291
|
+
*
|
|
292
|
+
* Returns "selfhosted" ONLY when the selfhosted feature is on AND the session has
|
|
293
|
+
* a non-null active pointer whose sandbox `kind` is "selfhosted". Otherwise
|
|
294
|
+
* returns undefined so buildAgent falls back to the home backend — byte-for-byte
|
|
295
|
+
* unchanged cloud behavior.
|
|
296
|
+
*
|
|
297
|
+
* Total + best-effort by contract: it NEVER throws (a lookup failure is logged and
|
|
298
|
+
* falls back to the home default), so wiring it at turn start can't fail the turn.
|
|
299
|
+
* The DB I/O is injected (the real call site passes readActiveSandbox + getSandbox,
|
|
300
|
+
* the same helpers wrapTurnBoxWithRouting reuses) so the gate/decision/safety
|
|
301
|
+
* contract is unit-testable without a live database.
|
|
302
|
+
*/
|
|
303
|
+
export async function resolveActiveSandboxBackend(
|
|
304
|
+
routingOn: boolean,
|
|
305
|
+
loadPointer: () => Promise<{ activeSandboxId: string | null } | null>,
|
|
306
|
+
loadSandboxKind: (sandboxId: string) => Promise<string | null>,
|
|
307
|
+
): Promise<Settings["sandboxBackend"] | undefined> {
|
|
308
|
+
// The active pointer + swap tools only exist when selfhosted routing is on; with
|
|
309
|
+
// the flag off there is nothing to resolve and we keep the home-backend default.
|
|
310
|
+
if (!routingOn) {
|
|
311
|
+
return undefined;
|
|
312
|
+
}
|
|
313
|
+
try {
|
|
314
|
+
const pointer = await loadPointer();
|
|
315
|
+
// A null pointer (no swap) means "use the session's own cloud group box" — the
|
|
316
|
+
// home backend already governs that path, so leave the override unset.
|
|
317
|
+
if (!pointer?.activeSandboxId) {
|
|
318
|
+
return undefined;
|
|
319
|
+
}
|
|
320
|
+
const kind = await loadSandboxKind(pointer.activeSandboxId);
|
|
321
|
+
return kind === "selfhosted" ? "selfhosted" : undefined;
|
|
322
|
+
} catch (error) {
|
|
323
|
+
console.error("active sandbox backend resolution failed (turn proceeds on home backend)", error);
|
|
324
|
+
return undefined;
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/**
|
|
329
|
+
* SELF-HEAL helper for the all-capped rotation idle (invariant 4: BOUNDED, no thrash).
|
|
330
|
+
* The turn hot path never refreshes Codex usage — only the usage API route does — so a
|
|
331
|
+
* window that has actually reset still reads OVER-threshold from the stale cache, which
|
|
332
|
+
* would idle-loop forever. Before idling, refresh LIVE usage for every connected account
|
|
333
|
+
* the cache marks over-threshold (bounded to the account count), which re-writes the
|
|
334
|
+
* cache columns, then return the re-read rows so the ranker can pick up a genuinely-reset
|
|
335
|
+
* window THIS turn. A refresh/read failure is swallowed (fall back to the pre-refresh rows
|
|
336
|
+
* + the bounded idle). Cooling (429'd) accounts are NOT refreshed: their exhaustedUntil
|
|
337
|
+
* cooldown is authoritative, and refreshing them would burn a provider call for nothing.
|
|
338
|
+
*/
|
|
339
|
+
async function refreshCappedCodexUsageRows(
|
|
340
|
+
db: ActivityServices["db"],
|
|
341
|
+
settings: Settings,
|
|
342
|
+
workspaceId: string,
|
|
343
|
+
accounts: CodexAccountStatus[],
|
|
344
|
+
): Promise<CodexAccountStatus[]> {
|
|
345
|
+
const nearPct = settings.codexRotationNearExhaustionPct;
|
|
346
|
+
const stale = accounts.filter(
|
|
347
|
+
(a) => a.status === "active" && ((a.primaryUsedPercent ?? 0) >= nearPct || (a.secondaryUsedPercent ?? 0) >= nearPct),
|
|
348
|
+
);
|
|
349
|
+
if (stale.length === 0) {
|
|
350
|
+
return accounts;
|
|
351
|
+
}
|
|
352
|
+
await Promise.all(stale.map((a) => fetchCodexUsageForAccount(db, settings, workspaceId, a.id).catch(() => undefined)));
|
|
353
|
+
return listCodexAccountStatuses(db, workspaceId).catch(() => accounts);
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
export function createRunAgentTurnActivity(services: () => Promise<ActivityServices>) {
|
|
357
|
+
return async function runAgentTurn(input: RunAgentTurnInput): Promise<RunAgentTurnResult> {
|
|
358
|
+
const { settings, db, bus, runtime, objectStorage, observability, wakeSessionWorkflow, entitlements, connectionCredentials } = await services();
|
|
359
|
+
const activityStarted = performance.now();
|
|
360
|
+
const activitySpan = observability.startSpan("worker.run_agent_segment", {
|
|
361
|
+
"opengeni.session_id": input.sessionId,
|
|
362
|
+
"opengeni.workflow_id": input.workflowId,
|
|
363
|
+
"opengeni.trigger_event_id": input.triggerEventId,
|
|
364
|
+
});
|
|
365
|
+
let activityStatus = "unknown";
|
|
366
|
+
let activityError: unknown;
|
|
367
|
+
let turnId: string | undefined;
|
|
368
|
+
let heartbeatTimer: ReturnType<typeof startActivityHeartbeat> | undefined;
|
|
369
|
+
// P1.2 ownership inversion: when sandboxOwnershipEnabled, the turn resolves
|
|
370
|
+
// the one box by id from the group lease and injects it NON-OWNED into the
|
|
371
|
+
// run. null when the flag is off (byte-for-byte the legacy build-and-discard
|
|
372
|
+
// path) OR when the backend is "none". Released + dropped in `finally`.
|
|
373
|
+
let resolvedSandbox: ResumedTurnSandbox | null = null;
|
|
374
|
+
// The UN-PROXIED established box session, captured BEFORE wrapTurnBoxWithRouting.
|
|
375
|
+
// Platform setup (beforeAgentStart hooks + file materialization) execs against
|
|
376
|
+
// THIS handle so a mid-turn sandbox_swap can never re-route those execs onto a
|
|
377
|
+
// connected machine (the user's real computer).
|
|
378
|
+
let setupBoxSession: unknown = null;
|
|
379
|
+
// The lease holder id (Temporal activityId, unique per scheduled execution)
|
|
380
|
+
// + the group id, captured so the lease heartbeat can refresh the lease TTL
|
|
381
|
+
// epoch-fenced (a superseded owner self-evicts) and finally can release.
|
|
382
|
+
let sandboxHolderId: string | null = null;
|
|
383
|
+
let sandboxGroupId: string | null = null;
|
|
384
|
+
// Lease-TTL refresh timer (parallels the activity heartbeat): while the turn
|
|
385
|
+
// runs it refreshes expires_at epoch-fenced so a legit multi-day turn is
|
|
386
|
+
// never TTL-reaped. Cleared in finally. Only set when the flag resolved a box.
|
|
387
|
+
let leaseHeartbeatTimer: ReturnType<typeof setInterval> | undefined;
|
|
388
|
+
// P4.3 on-turn recording: when the desktop tier + recording are enabled and
|
|
389
|
+
// the box is desktop-capable, the turn films the SAME :0 the agent's
|
|
390
|
+
// computer-use drives, finalized to storage in this activity's `finally`
|
|
391
|
+
// (read+PUT in-process — never a Temporal payload, F10). null otherwise.
|
|
392
|
+
let activeRecording: ActiveRecording | null = null;
|
|
393
|
+
// P4.3 recording gate: flips true the first time a computer-use/desktop tool
|
|
394
|
+
// ACTUALLY executes this turn (an SDK `computer_call` item streams through). A
|
|
395
|
+
// plain text turn ("hey"/"continue") never flips it, so finalize discards the
|
|
396
|
+
// recording with NO storage PUT and NO recording.failed — a clean no-op (a
|
|
397
|
+
// static frame of an untouched desktop is not worth uploading).
|
|
398
|
+
let didComputerUse = false;
|
|
399
|
+
let batcher: ReturnType<typeof createRuntimeBatcher> | null = null;
|
|
400
|
+
let preparedTools: Awaited<ReturnType<OpenGeniRuntime["prepareTools"]>> | null = null;
|
|
401
|
+
let publish: ((events: Array<Omit<AppendEventInput, "producerId" | "producerSeq" | "turnId">>, immediate?: boolean) => Promise<void>) | null = null;
|
|
402
|
+
let turnStartedPublished = false;
|
|
403
|
+
let stream: Awaited<ReturnType<OpenGeniRuntime["runStream"]>> | undefined;
|
|
404
|
+
// Dual-write of conversation truth (issue #35): completed items are
|
|
405
|
+
// reconciled into session_history_items after every model response and at
|
|
406
|
+
// every turn-end path (idempotent on position), and the sandbox recovery
|
|
407
|
+
// envelope is upserted alongside. Best-effort by design: persistence
|
|
408
|
+
// problems must never fail the run.
|
|
409
|
+
//
|
|
410
|
+
// Orphaned-tool-output guard: `stream.state.history` is NOT a plain
|
|
411
|
+
// append-only array — it is a computed getter
|
|
412
|
+
// (`getTurnInput(originalInput, generatedItems)`) that runs the SDK's
|
|
413
|
+
// `dropOrphanToolCalls` on every access, so a `function_call` with no
|
|
414
|
+
// settling result yet is transiently ABSENT from history and a later
|
|
415
|
+
// reconcile sees a DIFFERENT, shorter/reordered list. A blind length
|
|
416
|
+
// watermark with onConflictDoNothing-on-position then freezes the first
|
|
417
|
+
// shape of a position and can persist a `function_call_result` at a tail
|
|
418
|
+
// position while its `function_call` was pruned away in an earlier slice
|
|
419
|
+
// and never written — the orphan that bricks the session. We defend against
|
|
420
|
+
// it by sanitizing the current history into an API-valid sequence (the same
|
|
421
|
+
// pure function the read path uses) before persisting: a dangling call is
|
|
422
|
+
// never persisted (it is deferred until its result exists, when the pair is
|
|
423
|
+
// written together at consecutive positions), and a result is never
|
|
424
|
+
// persisted without its preceding call. The watermark counts only items we
|
|
425
|
+
// actually persisted, so a non-monotonic history can never desync it.
|
|
426
|
+
let persistedHistoryCount = 0;
|
|
427
|
+
let historyCountAtTurnStart = 0;
|
|
428
|
+
// Next free WHOLE-NUMBER absolute position to append at. Tracked separately
|
|
429
|
+
// from persistedHistoryCount (the in-memory slice index) because a compaction
|
|
430
|
+
// inserts a fractional summary position, so total rows no longer equal
|
|
431
|
+
// max(position)+1 and the slice index can no longer double as the position.
|
|
432
|
+
let nextHistoryPosition = 0;
|
|
433
|
+
const reconcileConversationTruth = async () => {
|
|
434
|
+
if (!stream || !turnId) {
|
|
435
|
+
return;
|
|
436
|
+
}
|
|
437
|
+
try {
|
|
438
|
+
const rawHistory = (stream.state as { history?: unknown[] }).history;
|
|
439
|
+
if (Array.isArray(rawHistory)) {
|
|
440
|
+
const { rows, nextWatermark, nextPosition } = historyRowsToAppend(
|
|
441
|
+
rawHistory as Array<Record<string, unknown>>,
|
|
442
|
+
persistedHistoryCount,
|
|
443
|
+
nextHistoryPosition,
|
|
444
|
+
);
|
|
445
|
+
if (rows.length > 0) {
|
|
446
|
+
await appendSessionHistoryItems(db, {
|
|
447
|
+
accountId: input.accountId,
|
|
448
|
+
workspaceId: input.workspaceId,
|
|
449
|
+
sessionId: input.sessionId,
|
|
450
|
+
turnId,
|
|
451
|
+
// Tag each row with the codex account that produced it (null on the
|
|
452
|
+
// non-codex path). Resolved at line ~504 before any reconcile pass
|
|
453
|
+
// runs, so this is the turn's effective account. The read path uses
|
|
454
|
+
// it to strip cross-account reasoning.encrypted_content next turn.
|
|
455
|
+
producerCodexCredentialId: effectiveCodexCredentialId,
|
|
456
|
+
items: rows,
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
persistedHistoryCount = nextWatermark;
|
|
460
|
+
nextHistoryPosition = nextPosition;
|
|
461
|
+
}
|
|
462
|
+
const envelope = sandboxStateEntryFromRunState(stream.state);
|
|
463
|
+
if (envelope) {
|
|
464
|
+
await upsertSandboxSessionEnvelope(db, {
|
|
465
|
+
accountId: input.accountId,
|
|
466
|
+
workspaceId: input.workspaceId,
|
|
467
|
+
sessionId: input.sessionId,
|
|
468
|
+
envelope,
|
|
469
|
+
});
|
|
470
|
+
}
|
|
471
|
+
} catch (persistError) {
|
|
472
|
+
console.error("session history dual-write failed (run unaffected)", persistError);
|
|
473
|
+
}
|
|
474
|
+
};
|
|
475
|
+
// Reassigned after the workspace environment loads; the publish closure is
|
|
476
|
+
// created (and used for turn.started) before the environment is available.
|
|
477
|
+
let redact: (payload: unknown) => unknown = identityRedactor;
|
|
478
|
+
let environmentId = "";
|
|
479
|
+
// The Codex account this turn runs on (pin > workspace active), resolved once
|
|
480
|
+
// a codex-billed turn is confirmed and threaded into the token resolver below.
|
|
481
|
+
let effectiveCodexCredentialId: string | null = null;
|
|
482
|
+
// Multi-account P4 (Part A): the latest usage-header snapshot scraped FOR FREE
|
|
483
|
+
// off this turn's `/codex/responses` responses (a turn issues many model calls;
|
|
484
|
+
// latest wins). Flushed ONCE into the P2 usage cache for the serving account in
|
|
485
|
+
// the `finally` — cheaper than a /wham/usage poll AND it self-heals P3 rotation
|
|
486
|
+
// (the proactive + 429 rankers read these exact columns). null ⇒ nothing scraped.
|
|
487
|
+
// Hoisted to activity scope so the finally flush (below) sees it. The sink is
|
|
488
|
+
// wired into codexContext.onUsageHeaders inside the try.
|
|
489
|
+
let latestCodexUsage: CodexUsageHeaderSnapshot | null = null;
|
|
490
|
+
// Hoisted for the preemption path: an approval-decision rerun must
|
|
491
|
+
// re-enter through the approval resume path (its frozen mid-flight state
|
|
492
|
+
// only exists in the RunState blob), never through a swapped trigger.
|
|
493
|
+
let triggerType: string | null = null;
|
|
494
|
+
try {
|
|
495
|
+
const mcpSettings = await settingsWithEnabledCapabilityMcpServers(db, input.workspaceId, settings);
|
|
496
|
+
// Read the active-credential flag ONCE (P2-b) and thread it through both the
|
|
497
|
+
// routing overlay (settingsWithCodexCredential) and the billed-turn predicate
|
|
498
|
+
// (isCodexBilledTurn below), so a concurrent disconnect/reconnect cannot make
|
|
499
|
+
// provider-injection and billing disagree about whether this is a codex turn.
|
|
500
|
+
const codexSubscriptionActive = await workspaceCodexSubscriptionActive(db, mcpSettings, input.workspaceId);
|
|
501
|
+
const capabilitySettings = await settingsWithCodexCredential(db, input.workspaceId, mcpSettings, codexSubscriptionActive);
|
|
502
|
+
runtime.configure(capabilitySettings);
|
|
503
|
+
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
504
|
+
const trigger = await getSessionEvent(db, input.workspaceId, input.triggerEventId);
|
|
505
|
+
if (!trigger) {
|
|
506
|
+
throw new Error(`Trigger event not found: ${input.triggerEventId}`);
|
|
507
|
+
}
|
|
508
|
+
triggerType = trigger.type;
|
|
509
|
+
let turn = input.turnId ? await getSessionTurn(db, input.workspaceId, input.turnId) : await claimNextQueuedTurnDb(db, input.workspaceId, input.sessionId, input.workflowId);
|
|
510
|
+
if (!turn && !input.turnId) {
|
|
511
|
+
const createdTurnId = await createTurn(db, {
|
|
512
|
+
workspaceId: input.workspaceId,
|
|
513
|
+
sessionId: input.sessionId,
|
|
514
|
+
temporalWorkflowId: input.workflowId,
|
|
515
|
+
triggerEventId: input.triggerEventId,
|
|
516
|
+
});
|
|
517
|
+
turn = await getSessionTurn(db, input.workspaceId, createdTurnId);
|
|
518
|
+
}
|
|
519
|
+
if (!turn) {
|
|
520
|
+
throw new Error(`Session turn not found for trigger: ${input.triggerEventId}`);
|
|
521
|
+
}
|
|
522
|
+
turnId = turn.id;
|
|
523
|
+
// Canonical codex-billed predicate (codex/<slug> + feature enabled + active
|
|
524
|
+
// workspace credential). Computed once and threaded through every billing
|
|
525
|
+
// gate + the usage recorder so a turn paid by the user's ChatGPT/Codex plan
|
|
526
|
+
// consumes ZERO OpenGeni credits and never feeds an OpenGeni cap. Resolved
|
|
527
|
+
// here (before resolvedModel at the routing step) because the pre-turn gate
|
|
528
|
+
// below needs it; mirrors the same active-credential read the codex provider
|
|
529
|
+
// overlay uses, so billing and routing agree on what "codex" is.
|
|
530
|
+
const isCodexTurn = await isCodexBilledTurn({ db, settings, workspaceId: input.workspaceId, model: turn.model, active: codexSubscriptionActive });
|
|
531
|
+
// §7.5 P3 — pass BOTH the codex predicate (codex-plan turns bypass the gate)
|
|
532
|
+
// AND the optional host `entitlements` port (when bound, its admitRun replaces
|
|
533
|
+
// the local credit read). Unset port → today's local-ledger path.
|
|
534
|
+
await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
|
|
535
|
+
const activityContext = currentActivityContext();
|
|
536
|
+
// Setup (environment load, MCP connects, sandbox restore) does not
|
|
537
|
+
// stream and so never observes cancellation on its own; these explicit
|
|
538
|
+
// checks let a graceful shutdown preempt the turn before the worker is
|
|
539
|
+
// force-killed instead of riding the setup to a heartbeat timeout.
|
|
540
|
+
const throwIfWorkerShuttingDown = () => {
|
|
541
|
+
const reason = activityContext?.cancellationSignal.reason;
|
|
542
|
+
if (isWorkerShutdownCancellation(reason)) {
|
|
543
|
+
throw reason;
|
|
544
|
+
}
|
|
545
|
+
};
|
|
546
|
+
heartbeatTimer = startActivityHeartbeat(activityContext, {
|
|
547
|
+
phase: "running",
|
|
548
|
+
sessionId: input.sessionId,
|
|
549
|
+
turnId,
|
|
550
|
+
});
|
|
551
|
+
let producerSeq = 0;
|
|
552
|
+
// One producer per activity execution, not per turn: a turn can run
|
|
553
|
+
// again on the same workflow (preemption resume, approval rerun), and
|
|
554
|
+
// each execution restarts producerSeq at 1 — a shared producer id would
|
|
555
|
+
// trip the per-producer uniqueness constraint on the event log. The
|
|
556
|
+
// Temporal activity id is unique per scheduled execution.
|
|
557
|
+
const producerId = `${input.workflowId}:${turnId}${activityContext ? `:${activityContext.info.activityId}` : ""}`;
|
|
558
|
+
// Unique per scheduled activity execution (Temporal activityId). Folded
|
|
559
|
+
// into positional usage source keys so a re-dispatch of this turn does
|
|
560
|
+
// not collide its model-call charges with the prior dispatch's. A genuine
|
|
561
|
+
// activity retry reuses the same activityId, so its re-emitted calls keep
|
|
562
|
+
// deduping (no double charge).
|
|
563
|
+
const dispatchId = activityContext?.info.activityId ?? null;
|
|
564
|
+
publish = async (events: Array<Omit<AppendEventInput, "producerId" | "producerSeq" | "turnId">>, immediate = false) => {
|
|
565
|
+
const inputs = events.map((event) => ({
|
|
566
|
+
...event,
|
|
567
|
+
payload: redact(event.payload),
|
|
568
|
+
turnId: turnId!,
|
|
569
|
+
producerId,
|
|
570
|
+
producerSeq: ++producerSeq,
|
|
571
|
+
}));
|
|
572
|
+
await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, inputs);
|
|
573
|
+
activityContext?.heartbeat({ phase: "events_published", sessionId: input.sessionId, turnId, producerSeq });
|
|
574
|
+
if (immediate) {
|
|
575
|
+
await Bun.sleep(0);
|
|
576
|
+
}
|
|
577
|
+
};
|
|
578
|
+
activityContext?.heartbeat({ phase: "turn_started", sessionId: input.sessionId, turnId });
|
|
579
|
+
|
|
580
|
+
// A shutdown that landed during claim/billing setup preempts before the
|
|
581
|
+
// turn visibly starts: nothing ran yet, so the requeued turn replays the
|
|
582
|
+
// original trigger cleanly on a healthy worker.
|
|
583
|
+
throwIfWorkerShuttingDown();
|
|
584
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "running", turnId);
|
|
585
|
+
await publish([
|
|
586
|
+
{ type: "session.status.changed", payload: { status: "running" } },
|
|
587
|
+
{ type: "turn.started", payload: { triggerEventId: input.triggerEventId } },
|
|
588
|
+
], true);
|
|
589
|
+
turnStartedPublished = true;
|
|
590
|
+
|
|
591
|
+
// Multi-account (P1): resolve the effective Codex account for this turn
|
|
592
|
+
// (session-pin > workspace active) and stamp it on the session so the
|
|
593
|
+
// in-session "Running on:" indicator reflects reality. Emit a switch event
|
|
594
|
+
// when it changed from the prior run's account so the pill flips live.
|
|
595
|
+
// Gated on the codex-billed predicate — non-codex turns never touch this.
|
|
596
|
+
if (isCodexTurn) {
|
|
597
|
+
const [rotation, accounts, sessionCodex] = await Promise.all([
|
|
598
|
+
getCodexRotationSettings(db, input.workspaceId),
|
|
599
|
+
listCodexAccountStatuses(db, input.workspaceId),
|
|
600
|
+
getSessionCodexState(db, input.workspaceId, input.sessionId),
|
|
601
|
+
]);
|
|
602
|
+
const connectedIds = new Set(accounts.map((account) => account.id));
|
|
603
|
+
const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
|
|
604
|
+
// The off-path / P1 default: today's workspace active pointer. Untouched
|
|
605
|
+
// when rotation is off or the session is pinned (byte-identical to P1).
|
|
606
|
+
let chosenActive = rotation?.activeCredentialId ?? null;
|
|
607
|
+
let rotationDecision: RotationDecision | null = null;
|
|
608
|
+
|
|
609
|
+
// P3 auto-rotation: the ONLY new branch, gated on rotation_enabled and
|
|
610
|
+
// pin-guarded (a pinned session NEVER rotates). When skipped, chosenActive
|
|
611
|
+
// stays the active pointer and selectCodexCredentialForTurn is called with
|
|
612
|
+
// byte-identical arguments to today — zero added cost on non-rotation turns.
|
|
613
|
+
if (rotation?.rotationEnabled && sessionPin == null) {
|
|
614
|
+
let rankAccounts = accounts;
|
|
615
|
+
rotationDecision = chooseRotationActive({
|
|
616
|
+
rotationStrategy: rotation.rotationStrategy as CodexRotationStrategy,
|
|
617
|
+
activeCredentialId: rotation.activeCredentialId,
|
|
618
|
+
priorCredentialId: sessionCodex?.lastCredentialId ?? null,
|
|
619
|
+
accounts: rankAccounts,
|
|
620
|
+
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
621
|
+
now: new Date(),
|
|
622
|
+
// P4: the leaving (active) account's cached connector set is the proxy
|
|
623
|
+
// for "what this session has access to" — prefer a covering target.
|
|
624
|
+
usedConnectors: rankAccounts.find((a) => a.id === rotation.activeCredentialId)?.connectorNamespaces ?? [],
|
|
625
|
+
});
|
|
626
|
+
if (rotationDecision.kind === "allCapped") {
|
|
627
|
+
// SELF-HEAL (invariant 4): the turn hot path NEVER refreshes usage, so a
|
|
628
|
+
// window that has actually reset still reads capped from the stale cache —
|
|
629
|
+
// which would otherwise idle-loop forever (idle → continuation re-dispatch →
|
|
630
|
+
// same stale all-capped → idle …). Before idling, refresh usage for the
|
|
631
|
+
// over-threshold accounts (bounded to the account count) and re-rank ONCE,
|
|
632
|
+
// so a genuinely-reset window is picked up immediately and the cache heals.
|
|
633
|
+
rankAccounts = await refreshCappedCodexUsageRows(db, settings, input.workspaceId, rankAccounts);
|
|
634
|
+
rotationDecision = chooseRotationActive({
|
|
635
|
+
rotationStrategy: rotation.rotationStrategy as CodexRotationStrategy,
|
|
636
|
+
activeCredentialId: rotation.activeCredentialId,
|
|
637
|
+
priorCredentialId: sessionCodex?.lastCredentialId ?? null,
|
|
638
|
+
accounts: rankAccounts,
|
|
639
|
+
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
640
|
+
now: new Date(),
|
|
641
|
+
// P4: leaving (active) account's connector set (refreshCappedCodexUsageRows
|
|
642
|
+
// only touches usage columns, so connectorNamespaces is preserved here).
|
|
643
|
+
usedConnectors: rankAccounts.find((a) => a.id === rotation.activeCredentialId)?.connectorNamespaces ?? [],
|
|
644
|
+
});
|
|
645
|
+
}
|
|
646
|
+
if (rotationDecision.kind === "active") {
|
|
647
|
+
if (rotationDecision.moved) {
|
|
648
|
+
// The single authoritative pointer-move site: persist the new active.
|
|
649
|
+
await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
|
|
650
|
+
}
|
|
651
|
+
chosenActive = rotationDecision.credentialId;
|
|
652
|
+
} else if (rotationDecision.kind === "allCapped" && publish && turnId) {
|
|
653
|
+
// Every eligible account is capped/cooling (and a usage refresh did NOT
|
|
654
|
+
// surface a reset): idle the turn AT THE BOUNDARY (no wasted model/sandbox
|
|
655
|
+
// build) until the EARLIEST reset across all accounts — the multi-account
|
|
656
|
+
// generalization of #143's single-account idle-until-reset. No saveRunState:
|
|
657
|
+
// no model ran, nothing to freeze.
|
|
658
|
+
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
659
|
+
const goalActive = Boolean(goal && goal.status === "active");
|
|
660
|
+
// BOUNDED + POSITIVE: clamp to [MIN_IDLE_MS, max] so a null/elapsed/unknown
|
|
661
|
+
// reset can never yield a 0 (which session.ts would treat as "continue now",
|
|
662
|
+
// re-entering this path in a tight CPU/DB-hammering loop).
|
|
663
|
+
const resumeMs = computeIdleDelayMs(rotationDecision.earliestResetAt, new Date(), CODEX_USAGE_LIMIT_MAX_RESUME_MS);
|
|
664
|
+
const failurePayload = codexUsageLimitFailurePayload(
|
|
665
|
+
{ resetsInSeconds: Math.ceil(resumeMs / 1000) },
|
|
666
|
+
"all connected Codex subscriptions are rate-limited",
|
|
667
|
+
{ allAccounts: true },
|
|
668
|
+
);
|
|
669
|
+
await publish([
|
|
670
|
+
// `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
|
|
671
|
+
// rotation-wait state as the reactive all-capped path, so it must freeze
|
|
672
|
+
// autoContinuations identically (evaluateGoalContinuation reads this marker)
|
|
673
|
+
// — a goal waiting out a long reset must not burn its continuation budget on
|
|
674
|
+
// the proactive path while the reactive path spares it.
|
|
675
|
+
{ type: "turn.failed", payload: { ...failurePayload, recovery: goalActive ? "goal_continuation" : "user_message", runStateSaved: false, rotated: true } },
|
|
676
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
677
|
+
], true);
|
|
678
|
+
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
679
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
680
|
+
activityStatus = "idle";
|
|
681
|
+
// idleUntilReset marks this a MANDATORY hold: session.ts must wait the full
|
|
682
|
+
// resumeMs even if a future change made it 0 — never a tight re-dispatch.
|
|
683
|
+
return goalActive ? { status: "idle", continueDelayMs: resumeMs, idleUntilReset: true } : { status: "idle" };
|
|
684
|
+
}
|
|
685
|
+
// kind:"none" (no accounts) → chosenActive stays null → existing relogin path.
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
effectiveCodexCredentialId = selectCodexCredentialForTurn({
|
|
689
|
+
sessionPinnedCredentialId: sessionPin, // pin still wins, structurally
|
|
690
|
+
activeCredentialId: chosenActive, // rotation-choice OR today's active
|
|
691
|
+
connectedIds,
|
|
692
|
+
});
|
|
693
|
+
if (effectiveCodexCredentialId) {
|
|
694
|
+
const priorAccountId = sessionCodex?.lastCredentialId ?? null;
|
|
695
|
+
await recordSessionActiveCodexCredential(db, input.workspaceId, input.sessionId, effectiveCodexCredentialId);
|
|
696
|
+
if (priorAccountId !== effectiveCodexCredentialId) {
|
|
697
|
+
// "rotation" only when the engine actually moved the pointer; otherwise the
|
|
698
|
+
// unchanged P1 "manual" literal (a manual active flip between turns).
|
|
699
|
+
const rotated = rotationDecision?.kind === "active" && rotationDecision.moved;
|
|
700
|
+
// P4: surface the dropped-connector note when this rotation pick couldn't
|
|
701
|
+
// cover the session's used connectors (a Tier-2/unknown failover); the pill
|
|
702
|
+
// renders the badge. Omitted when the switch covered everything (the norm).
|
|
703
|
+
const droppedConnectors = rotationDecision?.kind === "active" ? rotationDecision.droppedConnectors : undefined;
|
|
704
|
+
await publish([{
|
|
705
|
+
type: "codex.account.switched",
|
|
706
|
+
payload: {
|
|
707
|
+
fromAccountId: priorAccountId,
|
|
708
|
+
toAccountId: effectiveCodexCredentialId,
|
|
709
|
+
reason: rotated ? "rotation" : "manual",
|
|
710
|
+
...(droppedConnectors && droppedConnectors.length > 0 ? { droppedConnectors } : {}),
|
|
711
|
+
},
|
|
712
|
+
}]);
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
// Pack-scoped runtime: enabled packs may declare the sandbox image this
|
|
718
|
+
// workspace's sessions run in and skills for the sandbox skill index.
|
|
719
|
+
// Resolved after turn.started so a composition conflict (two enabled
|
|
720
|
+
// packs declaring images) fails the turn with its plain error instead
|
|
721
|
+
// of failing the activity opaquely.
|
|
722
|
+
const packRuntime = await resolveWorkspacePackRuntime(db, input.workspaceId);
|
|
723
|
+
// Workspace tier of the agent-persona resolution (session > workspace >
|
|
724
|
+
// deployment default). null means the workspace has no override, so the
|
|
725
|
+
// runtime falls back to runSettings.agentInstructionsTemplate (the
|
|
726
|
+
// deployment default, byte-identical to the historical preamble).
|
|
727
|
+
const workspaceAgentInstructions = await resolveWorkspaceAgentInstructions(db, input.workspaceId);
|
|
728
|
+
const runSettings = {
|
|
729
|
+
...settingsWithPackSandboxImage(capabilitySettings, packRuntime.sandboxImage),
|
|
730
|
+
openaiModel: turn.model,
|
|
731
|
+
openaiReasoningEffort: turn.reasoningEffort,
|
|
732
|
+
sandboxBackend: turn.sandboxBackend,
|
|
733
|
+
};
|
|
734
|
+
// Multi-provider per-turn routing → the provider gating (compaction mode,
|
|
735
|
+
// hosted web search, encrypted reasoning, context window) the agent and
|
|
736
|
+
// compaction summarizer must use; null falls back to the legacy global
|
|
737
|
+
// client. Resolve against `capabilitySettings` (whose openaiModel is the
|
|
738
|
+
// deployment default), NOT `runSettings`: runSettings.openaiModel is the
|
|
739
|
+
// turn's model, so for a turn ON a registry model the built-in provider
|
|
740
|
+
// would otherwise claim that id (configuredModels builds the built-in's
|
|
741
|
+
// models from openaiModel) and shadow the registry entry — resolving the
|
|
742
|
+
// turn to the built-in (Azure) gating while the global model router routes
|
|
743
|
+
// the name to its registry provider. That mismatch attaches web_search to
|
|
744
|
+
// a chat-only Fireworks model. Resolving against the default-model settings
|
|
745
|
+
// keeps gating consistent with the router. Cost accounting covers registry
|
|
746
|
+
// models via configuredModelPricing.
|
|
747
|
+
const resolvedModel = runtime.resolveTurnModel(capabilitySettings, turn.model);
|
|
748
|
+
// A codex-subscription turn resolves the bearer for THIS turn's effective
|
|
749
|
+
// codex account (effectiveCodexCredentialId; pin > workspace-active) at
|
|
750
|
+
// model-call time — multi-account P1 means a workspace can hold N accounts,
|
|
751
|
+
// so the bearer is per-account, not per-workspace. codexSubscriptionFetch
|
|
752
|
+
// (on the provider's OpenAI client) reads this AsyncLocalStorage context.
|
|
753
|
+
// Build it once and wrap BOTH the compaction summarizer (a separate model
|
|
754
|
+
// call on the same codex client) and the main run; otherwise the summarizer
|
|
755
|
+
// would hit the codex backend unauthenticated.
|
|
756
|
+
const codexContext: CodexRequestContext | null = resolvedModel?.provider.kind === "codex-subscription"
|
|
757
|
+
? ((): CodexRequestContext => {
|
|
758
|
+
// The empty-string fallback yields no row → null credential → the
|
|
759
|
+
// existing CodexReloginRequired path (a codex turn with no usable
|
|
760
|
+
// account fails closed, exactly as before multi-account).
|
|
761
|
+
const resolver = buildCodexTokenResolver(db, runSettings, input.workspaceId, effectiveCodexCredentialId ?? "");
|
|
762
|
+
return {
|
|
763
|
+
clientVersion: CODEX_CLIENT_VERSION,
|
|
764
|
+
getToken: resolver.getToken,
|
|
765
|
+
refresh: resolver.refresh,
|
|
766
|
+
resolveModel: buildModelResolver(CODEX_FALLBACK_MODEL_SLUGS, CODEX_FALLBACK_MODEL_SLUGS[0]),
|
|
767
|
+
onUsageHeaders: (snapshot) => { latestCodexUsage = snapshot; }, // latest wins; flushed once in finally
|
|
768
|
+
};
|
|
769
|
+
})()
|
|
770
|
+
: null;
|
|
771
|
+
const withCodex = <T>(fn: () => Promise<T>): Promise<T> => (codexContext ? codexRequestStorage.run(codexContext, fn) : fn());
|
|
772
|
+
const turnResources = mergeResourceRefs(session.resources, turn.resources);
|
|
773
|
+
// Attach the first-party MCP server to EVERY turn, regardless of how/when
|
|
774
|
+
// the session was created (API, scheduled task, or a pre-existing session
|
|
775
|
+
// whose stored tools predate this) — so set_session_title and the rest are
|
|
776
|
+
// always reachable. Idempotent: mergeToolRefs dedupes if already present.
|
|
777
|
+
// Attach codex_apps (the ChatGPT/Codex connectors MCP) when the codex
|
|
778
|
+
// overlay injected it into runSettings.mcpServers (active subscription +
|
|
779
|
+
// connector scopes); no-op for every other turn. Its refreshing bearer is
|
|
780
|
+
// resolved at connect time from the codex ALS (see the withCodex-wrapped
|
|
781
|
+
// prepareTools call below).
|
|
782
|
+
const turnTools = withCodexAppsTool(runSettings, withFirstPartyTools(runSettings, mergeToolRefs(session.tools, turn.tools)));
|
|
783
|
+
// §7.6 P4a — load (and decrypt) the workspace environment via the host
|
|
784
|
+
// `sandboxSecrets` provider when bound; unset → today's local decrypt.
|
|
785
|
+
const connectionScope = { accountId: input.accountId, workspaceId: input.workspaceId };
|
|
786
|
+
const workspaceEnvironment = await loadWorkspaceEnvironmentForRunWithCredentials(
|
|
787
|
+
db,
|
|
788
|
+
runSettings,
|
|
789
|
+
connectionScope,
|
|
790
|
+
session.environmentId,
|
|
791
|
+
connectionCredentials?.sandboxSecrets,
|
|
792
|
+
);
|
|
793
|
+
environmentId = workspaceEnvironment?.id ?? "";
|
|
794
|
+
redact = createSecretRedactor(
|
|
795
|
+
Object.entries(workspaceEnvironment?.values ?? {}).map(([name, value]) => ({ name, value })),
|
|
796
|
+
);
|
|
797
|
+
// EFFECTIVE compute backend, resolved ONCE at turn start (Case B + Stage D
|
|
798
|
+
// D1-lite) and reused for EVERY downstream decision: the env mint (skip the
|
|
799
|
+
// inert GitHub token for a machine turn), the establish path (no phantom Modal
|
|
800
|
+
// home box for a machine-primary turn), buildAgent (skip the repository clone
|
|
801
|
+
// hook so a private repo is never `git clone`d onto the user's real disk), and
|
|
802
|
+
// the warm-rate (a machine accrues ZERO cloud warm-seconds). The active pointer
|
|
803
|
+
// + its sandbox row are loaded ONCE here (best-effort, never throwing) and the
|
|
804
|
+
// SAME values feed resolveActiveSandboxBackend (the tested gate) AND the
|
|
805
|
+
// machine-primary establish branch (enrollmentId/epoch/workingDir) below — no
|
|
806
|
+
// double read, no read-skew between the gate decision and the establish. With
|
|
807
|
+
// routing OFF this is byte-for-byte the legacy path: no reads, undefined backend.
|
|
808
|
+
const routingOn = routingEnabled(settings);
|
|
809
|
+
const activeSandboxPointer = routingOn
|
|
810
|
+
? await readActiveSandbox(db, input.workspaceId, input.sessionId).catch(() => null)
|
|
811
|
+
: null;
|
|
812
|
+
const activeSandboxRecord = routingOn && activeSandboxPointer?.activeSandboxId
|
|
813
|
+
? await getSandbox(db, input.workspaceId, activeSandboxPointer.activeSandboxId).catch(() => null)
|
|
814
|
+
: null;
|
|
815
|
+
const activeSandboxBackend = await resolveActiveSandboxBackend(
|
|
816
|
+
routingOn,
|
|
817
|
+
async () => activeSandboxPointer,
|
|
818
|
+
async () => activeSandboxRecord?.kind ?? null,
|
|
819
|
+
);
|
|
820
|
+
// A machine-primary turn = the effective backend is selfhosted AND we have the
|
|
821
|
+
// machine's enrollment (agent id) + a non-null pointer to bind it. Anything
|
|
822
|
+
// missing (should not happen — the DB enforces selfhosted⇒enrollmentId) falls
|
|
823
|
+
// back to the cloud establish path (a correct, if phantom, box) rather than
|
|
824
|
+
// crashing the turn.
|
|
825
|
+
const machinePrimary =
|
|
826
|
+
activeSandboxBackend === "selfhosted"
|
|
827
|
+
&& Boolean(activeSandboxPointer?.activeSandboxId)
|
|
828
|
+
&& Boolean(activeSandboxRecord?.enrollmentId);
|
|
829
|
+
// Computed exactly ONCE per turn and reused for BOTH the box manifest
|
|
830
|
+
// (resumeBoxForTurn -> establishSandboxSessionFromEnvelope, below) AND the
|
|
831
|
+
// agent (runtime.buildAgent, below). sandboxEnvironmentForRun mints a FRESH
|
|
832
|
+
// run-scoped GitHub installation token on every call, so a second call would
|
|
833
|
+
// yield a DIFFERENT token value and re-introduce the manifest-env delta the
|
|
834
|
+
// SDK's provided-session guard throws on — the box and the agent MUST share
|
|
835
|
+
// this same object. A machine-primary turn skips the (inert) token mint entirely
|
|
836
|
+
// (the machine uses its own git creds); the SAME base env still feeds the box +
|
|
837
|
+
// the agent, so env-parity holds.
|
|
838
|
+
// TOKEN-BROKER (B1): sandboxEnvironmentForRun now returns the STABLE manifest
|
|
839
|
+
// env (no rotating GH_TOKEN/GITHUB_TOKEN/GIT_CONFIG_* extraheader) PLUS the
|
|
840
|
+
// run-scoped GitHub token minted ONCE per turn as `gitToken`. The env feeds BOTH
|
|
841
|
+
// the box manifest AND the agent (env-parity, as before); the token is threaded
|
|
842
|
+
// OFF-MANIFEST as the clone-seed `gitTokenSeed` to buildAgent (below) so the box
|
|
843
|
+
// never carries a rotating value on its manifest. When the platform token IS
|
|
844
|
+
// minted, the host `gitCredentials` provider may supply it; unset → self-mint
|
|
845
|
+
// from settings. gitToken is undefined on the selfhosted skip path (the machine
|
|
846
|
+
// uses its own git creds).
|
|
847
|
+
const { environment: sandboxEnvironment, gitToken: sandboxGitToken } = await sandboxEnvironmentForRun(
|
|
848
|
+
runSettings,
|
|
849
|
+
turnResources,
|
|
850
|
+
workspaceEnvironment?.values ?? {},
|
|
851
|
+
{
|
|
852
|
+
skipGitHubToken: activeSandboxBackend === "selfhosted",
|
|
853
|
+
scope: connectionScope,
|
|
854
|
+
gitCredentials: connectionCredentials?.gitCredentials,
|
|
855
|
+
},
|
|
856
|
+
);
|
|
857
|
+
|
|
858
|
+
// P1.2 ownership inversion (gated, default OFF). With the flag off this
|
|
859
|
+
// block is skipped entirely: resolvedSandbox stays null and runStream
|
|
860
|
+
// takes the legacy per-run build-and-discard path — byte-for-byte today.
|
|
861
|
+
// With it on, acquire the group lease ('turn' holder = the activityId),
|
|
862
|
+
// resume the one box by id, and inject it NON-OWNED into the run. The box
|
|
863
|
+
// backend is "none" -> never resolve (no box to touch).
|
|
864
|
+
//
|
|
865
|
+
// Established AFTER sandboxEnvironment is computed (not before) so the box's
|
|
866
|
+
// manifest is created with the SAME environment the agent declares — the SDK
|
|
867
|
+
// applies the agent's manifest to this provided session and throws on ANY
|
|
868
|
+
// environment delta (validateNoEnvironmentDelta). Passing sandboxEnvironment
|
|
869
|
+
// here makes current==target so the delta is empty.
|
|
870
|
+
if (settings.sandboxOwnershipEnabled && turn.sandboxBackend !== "none") {
|
|
871
|
+
sandboxHolderId = dispatchId ?? `turn:${turnId}`;
|
|
872
|
+
sandboxGroupId = session.sandboxGroupId;
|
|
873
|
+
// STAGE D honest-label guard: a machine-home session carries
|
|
874
|
+
// turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
|
|
875
|
+
// when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
|
|
876
|
+
// + enrollmentId). When it is NOT primary — the agent swapped back to the
|
|
877
|
+
// group box (sandbox_swap 'session'/'default'/groupId clears the pointer) or
|
|
878
|
+
// selfhosted routing is flag-OFF (the pointer is ignored) — the else-branch
|
|
879
|
+
// must resume a REAL cloud group box, not a "selfhosted" one: the registry
|
|
880
|
+
// SelfhostedSandboxClient has no bound agentId and throws. Fall the group-box
|
|
881
|
+
// backend back to the deployment default cloud backend so swap-away / flag-off
|
|
882
|
+
// degrade to a genuine cloud box exactly like today (home=modal did).
|
|
883
|
+
const groupBoxBackend: Settings["sandboxBackend"] =
|
|
884
|
+
turn.sandboxBackend === "selfhosted" && !machinePrimary
|
|
885
|
+
? settings.sandboxBackend
|
|
886
|
+
: turn.sandboxBackend;
|
|
887
|
+
if (machinePrimary) {
|
|
888
|
+
// STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
|
|
889
|
+
// establish or lease a phantom Modal home box (today's path leased + BILLED
|
|
890
|
+
// a cloud box the turn never touched). Build the SelfhostedSession DIRECTLY
|
|
891
|
+
// (no Modal box created) and take the group lease with backend "selfhosted"
|
|
892
|
+
// (refcount/idle bookkeeping; the reaper drains it cold with NO provider
|
|
893
|
+
// stop, and bills ZERO warm-seconds). The session is a harmless in-memory
|
|
894
|
+
// bind (no NATS round-trip), so build it FIRST; if the lease then fences,
|
|
895
|
+
// there is nothing to clean up.
|
|
896
|
+
const established = await establishSelfhostedTurnSession(
|
|
897
|
+
{ db, settings, bus },
|
|
898
|
+
{
|
|
899
|
+
workspaceId: input.workspaceId,
|
|
900
|
+
agentId: activeSandboxRecord!.enrollmentId!,
|
|
901
|
+
epoch: activeSandboxPointer!.activeEpoch,
|
|
902
|
+
environment: sandboxEnvironment,
|
|
903
|
+
workingDir: activeSandboxPointer!.workingDir,
|
|
904
|
+
},
|
|
905
|
+
);
|
|
906
|
+
const lease = await acquireSelfhostedLeaseForTurn(
|
|
907
|
+
{ db, settings },
|
|
908
|
+
{
|
|
909
|
+
accountId: input.accountId,
|
|
910
|
+
workspaceId: input.workspaceId,
|
|
911
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
912
|
+
sessionId: input.sessionId,
|
|
913
|
+
},
|
|
914
|
+
"turn",
|
|
915
|
+
sandboxHolderId,
|
|
916
|
+
);
|
|
917
|
+
setupBoxSession = established.session;
|
|
918
|
+
resolvedSandbox = {
|
|
919
|
+
// Wrap in the SAME routing proxy so a mid-turn swap (to another machine
|
|
920
|
+
// or back to the group box) still re-routes per op. PIN this established
|
|
921
|
+
// SelfhostedSession for the machine pointer so the turn-start manifest
|
|
922
|
+
// write (via the proxy's `state` getter) and the per-op reads hit ONE
|
|
923
|
+
// instance — no two-instance manifest divergence.
|
|
924
|
+
established: wrapTurnBoxWithRouting(
|
|
925
|
+
{ db, settings, bus },
|
|
926
|
+
{
|
|
927
|
+
workspaceId: input.workspaceId,
|
|
928
|
+
sessionId: input.sessionId,
|
|
929
|
+
environment: sandboxEnvironment,
|
|
930
|
+
pinnedSelfhosted: {
|
|
931
|
+
sandboxId: activeSandboxPointer!.activeSandboxId!,
|
|
932
|
+
epoch: activeSandboxPointer!.activeEpoch,
|
|
933
|
+
},
|
|
934
|
+
},
|
|
935
|
+
established,
|
|
936
|
+
),
|
|
937
|
+
leaseEpoch: lease.leaseEpoch,
|
|
938
|
+
release: lease.release,
|
|
939
|
+
};
|
|
940
|
+
} else {
|
|
941
|
+
resolvedSandbox = await resumeBoxForTurn(
|
|
942
|
+
{ db, settings },
|
|
943
|
+
{
|
|
944
|
+
accountId: input.accountId,
|
|
945
|
+
workspaceId: input.workspaceId,
|
|
946
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
947
|
+
sessionId: input.sessionId,
|
|
948
|
+
// groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
|
|
949
|
+
// is not machine-primary resumes a real cloud group box (the
|
|
950
|
+
// deployment default), never a "selfhosted" box (which would throw
|
|
951
|
+
// for lack of a bound agentId).
|
|
952
|
+
backend: groupBoxBackend,
|
|
953
|
+
os: session.sandboxOs,
|
|
954
|
+
environment: sandboxEnvironment,
|
|
955
|
+
// IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
|
|
956
|
+
// this run resolves. The lease stamps it + conflicts on a live shared box
|
|
957
|
+
// running a DIFFERENT image (solo → recreate on the new image; N-holders →
|
|
958
|
+
// SandboxImageConflictError surfaced as an actionable turn error). Prefer the
|
|
959
|
+
// explicit Modal image ref, else the docker image. The selfhosted branch
|
|
960
|
+
// (establishSelfhostedTurnSession/acquireSelfhostedLeaseForTurn) NEVER passes
|
|
961
|
+
// an image — B3 lives only on this Modal else-branch.
|
|
962
|
+
...((runSettings.modalImageRef ?? runSettings.dockerImage)
|
|
963
|
+
? { image: runSettings.modalImageRef ?? runSettings.dockerImage }
|
|
964
|
+
: {}),
|
|
965
|
+
},
|
|
966
|
+
"turn",
|
|
967
|
+
sandboxHolderId,
|
|
968
|
+
);
|
|
969
|
+
setupBoxSession = resolvedSandbox.established.session;
|
|
970
|
+
// M7 hot-swap: when the selfhosted feature is on, wrap the established
|
|
971
|
+
// group box in the STABLE routing proxy before it is injected NON-OWNED
|
|
972
|
+
// into the run. The SDK binds to this ONE object once and calls its
|
|
973
|
+
// methods per tool call; the proxy re-reads (active_sandbox_id,
|
|
974
|
+
// active_epoch) per op and dispatches to the currently-active backend, so
|
|
975
|
+
// a sandbox_swap mid-turn lands the NEXT tool call on the new box. With
|
|
976
|
+
// the flag off the established group box is injected unchanged (today's
|
|
977
|
+
// path). The lease still owns the group box lifecycle — the proxy is a
|
|
978
|
+
// routing veneer, not an owner.
|
|
979
|
+
if (routingEnabled(settings)) {
|
|
980
|
+
resolvedSandbox = {
|
|
981
|
+
...resolvedSandbox,
|
|
982
|
+
established: wrapTurnBoxWithRouting(
|
|
983
|
+
{ db, settings, bus },
|
|
984
|
+
// Thread the SAME declared environment the group box was created with
|
|
985
|
+
// (resumeBoxForTurn, above) so a selfhosted swap target's manifest
|
|
986
|
+
// carries it too — the SDK's per-turn manifest-env delta stays empty
|
|
987
|
+
// (no "cannot change manifest environment variables" throw).
|
|
988
|
+
{ workspaceId: input.workspaceId, sessionId: input.sessionId, environment: sandboxEnvironment },
|
|
989
|
+
resolvedSandbox.established,
|
|
990
|
+
),
|
|
991
|
+
};
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
// Refresh the lease TTL on the activity-heartbeat cadence (10s, well
|
|
995
|
+
// inside the 90s lease TTL). EPOCH-FENCED: a superseded owner's refresh
|
|
996
|
+
// is rejected (returns false) and we stop refreshing — the box rides the
|
|
997
|
+
// provider idle-timeout and the next dispatch re-establishes it. Best-
|
|
998
|
+
// effort: a transient DB error must never fail the turn.
|
|
999
|
+
const heartbeatEpoch = resolvedSandbox.leaseEpoch;
|
|
1000
|
+
const heartbeatHolderId = sandboxHolderId;
|
|
1001
|
+
const heartbeatGroupId = sandboxGroupId;
|
|
1002
|
+
// P2.1 warm-meter (tick A): while a turn runs, the heartbeat is also the
|
|
1003
|
+
// warm-seconds tick. GROUP+epoch+tick keyed (one box = one stream, shared
|
|
1004
|
+
// box metered once); epoch-fenced (a stale tick no-ops). Warm-cost is
|
|
1005
|
+
// metered when a per-backend rate is configured. Best-effort: a metering
|
|
1006
|
+
// failure must never fail the turn.
|
|
1007
|
+
//
|
|
1008
|
+
// Keyed off the EFFECTIVE backend (Stage D): a machine-primary turn has NO
|
|
1009
|
+
// Modal box, so it must accrue ZERO cloud warm-seconds — `selfhosted` has no
|
|
1010
|
+
// configured warm rate (0). Keying off turn.sandboxBackend (modal) would bill
|
|
1011
|
+
// cloud seconds for a box that does not exist (a real money bug). Non-machine
|
|
1012
|
+
// turns fall back to groupBoxBackend (the REAL box that ran): for a machine-
|
|
1013
|
+
// home turn that degraded to the cloud group box (swap-away / flag-off), that
|
|
1014
|
+
// is the deployment default (modal), so the fallback box is warm-metered at
|
|
1015
|
+
// the cloud rate instead of selfhosted's rate-0 (which would under-bill).
|
|
1016
|
+
const warmRate = sandboxWarmRateMicrosPerSecond(settings, activeSandboxBackend ?? groupBoxBackend);
|
|
1017
|
+
leaseHeartbeatTimer = setInterval(() => {
|
|
1018
|
+
void heartbeatLeaseHolder(db, {
|
|
1019
|
+
accountId: input.accountId,
|
|
1020
|
+
workspaceId: input.workspaceId,
|
|
1021
|
+
sandboxGroupId: heartbeatGroupId,
|
|
1022
|
+
kind: "turn",
|
|
1023
|
+
holderId: heartbeatHolderId,
|
|
1024
|
+
leaseTtlMs: settings.sandboxLeaseTtlMs,
|
|
1025
|
+
expectedEpoch: heartbeatEpoch,
|
|
1026
|
+
}).catch(() => undefined);
|
|
1027
|
+
void accrueWarmSeconds(db, {
|
|
1028
|
+
accountId: input.accountId,
|
|
1029
|
+
workspaceId: input.workspaceId,
|
|
1030
|
+
sandboxGroupId: heartbeatGroupId,
|
|
1031
|
+
expectedEpoch: heartbeatEpoch,
|
|
1032
|
+
warmRateMicrosPerSecond: warmRate,
|
|
1033
|
+
subjectId: input.sessionId,
|
|
1034
|
+
}).catch(() => undefined);
|
|
1035
|
+
}, 10_000);
|
|
1036
|
+
if ("unref" in leaseHeartbeatTimer && typeof leaseHeartbeatTimer.unref === "function") {
|
|
1037
|
+
leaseHeartbeatTimer.unref();
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
// P4.3 on-turn recording. The box's :0 display stack was brought up by
|
|
1042
|
+
// resumeBoxForTurn (spawner path) / is up from a prior turn; film it for
|
|
1043
|
+
// the duration of this turn so the human can watch the agent work and the
|
|
1044
|
+
// agent's computer-use proofs are captured. Best-effort: a recording start
|
|
1045
|
+
// failure NEVER fails the turn (the desktop is a value-add). Finalized in
|
|
1046
|
+
// `finally` (read+PUT in this same activity — never a Temporal payload).
|
|
1047
|
+
if (
|
|
1048
|
+
resolvedSandbox
|
|
1049
|
+
&& settings.sandboxDesktopEnabled
|
|
1050
|
+
&& settings.recordingEnabled
|
|
1051
|
+
&& desktopCapableBackend(resolvedSandbox.established.backendId)
|
|
1052
|
+
) {
|
|
1053
|
+
try {
|
|
1054
|
+
const begun = await beginRecording({
|
|
1055
|
+
settings,
|
|
1056
|
+
db,
|
|
1057
|
+
accountId: input.accountId,
|
|
1058
|
+
workspaceId: input.workspaceId,
|
|
1059
|
+
sessionId: input.sessionId,
|
|
1060
|
+
turnId: turnId!,
|
|
1061
|
+
recordingId: randomUUID(),
|
|
1062
|
+
mode: "on-turn",
|
|
1063
|
+
session: resolvedSandbox.established.session,
|
|
1064
|
+
runAs: sandboxRunAs(settings),
|
|
1065
|
+
reason: null,
|
|
1066
|
+
});
|
|
1067
|
+
activeRecording = begun.active;
|
|
1068
|
+
await publish([{ type: "recording.started", payload: begun.started }]);
|
|
1069
|
+
} catch (recordingError) {
|
|
1070
|
+
activeRecording = null;
|
|
1071
|
+
console.error("on-turn recording start failed (turn outcome unaffected)", recordingError);
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
const fileResourceDownloads = await sandboxFileDownloadsForRun(runSettings, db, objectStorage, input.workspaceId, turnResources);
|
|
1076
|
+
throwIfWorkerShuttingDown();
|
|
1077
|
+
// Wrap MCP prep in the codex ALS so the codex_apps connect handshake
|
|
1078
|
+
// (initialize + tools/list) can resolve the per-workspace bearer from
|
|
1079
|
+
// codexRequestStorage (runtime/codexAppsMcpRequestInit). withCodex is the
|
|
1080
|
+
// identity on every non-codex turn, so this is a no-op for existing paths.
|
|
1081
|
+
preparedTools = await withCodex(() => runtime.prepareTools(runSettings, turnTools, {
|
|
1082
|
+
accountId: input.accountId,
|
|
1083
|
+
workspaceId: input.workspaceId,
|
|
1084
|
+
sessionId: input.sessionId,
|
|
1085
|
+
subjectId: "worker:first-party-mcp",
|
|
1086
|
+
subjectLabel: "OpenGeni worker",
|
|
1087
|
+
// Manager-style sessions carry a creation-validated permission set
|
|
1088
|
+
// for their first-party MCP token; null keeps the fixed default.
|
|
1089
|
+
...(session.firstPartyMcpPermissions?.length ? { firstPartyPermissions: session.firstPartyMcpPermissions } : {}),
|
|
1090
|
+
}));
|
|
1091
|
+
// Genesis turn = the first user turn (no assistant history reconciled
|
|
1092
|
+
// yet). Durable Postgres state (countSessionHistoryItems includes
|
|
1093
|
+
// superseded rows after compaction), NOT a workflow counter (turnsThisRun
|
|
1094
|
+
// resets on continueAsNew). Drives the one-shot title hint appended to the
|
|
1095
|
+
// agent's instructions; continuation/preemption turns never match (their
|
|
1096
|
+
// trigger is goal.continuation/turn.preempted).
|
|
1097
|
+
const isGenesisTurn = triggerType === "user.message"
|
|
1098
|
+
&& (await countSessionHistoryItems(db, input.workspaceId, input.sessionId)) === 0;
|
|
1099
|
+
// Clone-onto-real-disk hazard (Case B). A session keeps its CLOUD HOME
|
|
1100
|
+
// backend (runSettings.sandboxBackend, e.g. "modal") but its ACTIVE sandbox
|
|
1101
|
+
// may have been swapped to a connected machine (active_sandbox_id → a
|
|
1102
|
+
// selfhosted lease). buildAgent's repository-clone lifecycle hook keys off
|
|
1103
|
+
// the EFFECTIVE backend; if we let it default to the home backend it would
|
|
1104
|
+
// `git clone` a private GitHub-App repo onto the user's REAL disk. So pass
|
|
1105
|
+
// "selfhosted" through when the active sandbox is a connected machine;
|
|
1106
|
+
// otherwise leave it undefined so buildAgent defaults to the home backend
|
|
1107
|
+
// (byte-for-byte unchanged cloud behavior). `activeSandboxBackend` was
|
|
1108
|
+
// resolved ONCE at turn start (above) via resolveActiveSandboxBackend (the
|
|
1109
|
+
// tested gate) and is reused here — resolving once is correct because the
|
|
1110
|
+
// clone hook runs at beforeAgentStart, so a mid-turn swap can't affect it.
|
|
1111
|
+
const agent = runtime.buildAgent(runSettings, turnResources, {
|
|
1112
|
+
reasoningEffort: turn.reasoningEffort,
|
|
1113
|
+
genesisTitleHint: isGenesisTurn,
|
|
1114
|
+
sandboxEnvironment,
|
|
1115
|
+
// TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
|
|
1116
|
+
// seed. ONLY when the effective backend is NOT selfhosted (the connected
|
|
1117
|
+
// machine uses its own git creds — mirrors the skipGitHubToken gate above)
|
|
1118
|
+
// AND the mint actually produced a token (repo resources present). The runtime
|
|
1119
|
+
// seeds it to the box's token file before the repository-clone runs; it never
|
|
1120
|
+
// touches the box/agent manifest env.
|
|
1121
|
+
...(activeSandboxBackend !== "selfhosted" && sandboxGitToken ? { gitTokenSeed: sandboxGitToken } : {}),
|
|
1122
|
+
...(activeSandboxBackend ? { activeSandboxBackend } : {}),
|
|
1123
|
+
fileResourceDownloads,
|
|
1124
|
+
mcpServers: preparedTools.mcpServers,
|
|
1125
|
+
// LIVE by-reference connector namespaces (fills during this turn's
|
|
1126
|
+
// codex_apps tools/list): the codex tool_search description reads it per
|
|
1127
|
+
// model call so the model sees the account's real connected sources.
|
|
1128
|
+
codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
|
|
1129
|
+
// Resolved-model routing + gating (legacy defaults when null). The model
|
|
1130
|
+
// is passed as the model *string* (agent.model = runSettings.openaiModel),
|
|
1131
|
+
// NOT a Model instance: an instance only survives the in-process
|
|
1132
|
+
// ("none") run, whereas the SandboxAgent/Modal path drops it and
|
|
1133
|
+
// re-resolves the model *name* through the global MultiProviderModelProvider
|
|
1134
|
+
// configureOpenAI installed — so registry models (Fireworks GLM) route to
|
|
1135
|
+
// their own client instead of 404ing against the built-in Azure/OpenAI
|
|
1136
|
+
// client. The gating still comes from the resolved provider: server-side
|
|
1137
|
+
// store/compaction follow the provider's compaction mode (registry
|
|
1138
|
+
// providers resolve to "client"); encrypted reasoning is only
|
|
1139
|
+
// round-tripped on the Responses wire API; hosted web search is attached
|
|
1140
|
+
// only when the model opts in; the effective context window drives the
|
|
1141
|
+
// compaction threshold.
|
|
1142
|
+
...(resolvedModel
|
|
1143
|
+
? {
|
|
1144
|
+
compactionMode: resolvedModel.provider.compactionMode,
|
|
1145
|
+
hostedWebSearch: resolvedModel.configured.hostedWebSearch,
|
|
1146
|
+
encryptedReasoning: resolvedModel.provider.api === "responses" && runSettings.openaiReasoningEncryptedContent,
|
|
1147
|
+
contextWindowTokens: resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
|
|
1148
|
+
// The ChatGPT/Codex backend rejects the SDK's HOSTED sandbox tools —
|
|
1149
|
+
// the `apply_patch` tool type ("Unsupported tool type: apply_patch")
|
|
1150
|
+
// and structured tool output — which the OpenAIResponsesModel the SDK
|
|
1151
|
+
// binds would otherwise select. Tell buildAgent to emit the function
|
|
1152
|
+
// `apply_patch` + text `view_image` variants the backend accepts. Only
|
|
1153
|
+
// the codex-subscription provider needs this; every other backend
|
|
1154
|
+
// (built-in OpenAI/Azure = real hosted support; registry "chat"
|
|
1155
|
+
// providers = the SDK's own ChatCompletions detection) keeps the SDK
|
|
1156
|
+
// default.
|
|
1157
|
+
structuredToolTransport: resolvedModel.provider.kind !== "codex-subscription",
|
|
1158
|
+
}
|
|
1159
|
+
: {}),
|
|
1160
|
+
...(packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {}),
|
|
1161
|
+
...(workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {}),
|
|
1162
|
+
...(workspaceEnvironment
|
|
1163
|
+
? {
|
|
1164
|
+
workspaceEnvironment: {
|
|
1165
|
+
name: workspaceEnvironment.name,
|
|
1166
|
+
description: workspaceEnvironment.description,
|
|
1167
|
+
variableNames: Object.keys(workspaceEnvironment.values),
|
|
1168
|
+
},
|
|
1169
|
+
}
|
|
1170
|
+
: {}),
|
|
1171
|
+
});
|
|
1172
|
+
// Pre-turn client-side context compaction (Azure path). When the
|
|
1173
|
+
// resolved mode is "client" and the last turn's input tokens crossed the
|
|
1174
|
+
// soft budget, this summarizes the orphan-safe old prefix into one user
|
|
1175
|
+
// message and supersedes the summarized rows BEFORE the history is read
|
|
1176
|
+
// for this turn, so the model sees [summary, ...recent tail] + new input.
|
|
1177
|
+
// Best-effort: a no-op or failure leaves the un-compacted history intact.
|
|
1178
|
+
// Skipped on approval/preempt resumes (no fresh user turn; the frozen
|
|
1179
|
+
// RunState or in-flight tail must replay verbatim).
|
|
1180
|
+
if (triggerType === "user.message" || triggerType === "goal.continuation") {
|
|
1181
|
+
try {
|
|
1182
|
+
// Operator /compact (the slash command) sets a durable request flag;
|
|
1183
|
+
// consume it atomically so a forced compaction runs now even when the
|
|
1184
|
+
// budget trigger would not fire. Only the turn that observes the flag
|
|
1185
|
+
// runs it, so concurrent turns can't double-compact.
|
|
1186
|
+
const forced = await consumeSessionCompactionRequest(db, input.workspaceId, input.sessionId);
|
|
1187
|
+
const outcome = await maybeCompactContext(
|
|
1188
|
+
db,
|
|
1189
|
+
runSettings,
|
|
1190
|
+
{ accountId: input.accountId, workspaceId: input.workspaceId, sessionId: input.sessionId, turnId },
|
|
1191
|
+
session.lastInputTokens,
|
|
1192
|
+
// Provider-aware summarizer: when the turn's model resolved to a
|
|
1193
|
+
// registry provider, summarize on THAT provider's client + wire API
|
|
1194
|
+
// (a chat provider can't summarize through OpenAI/Azure). Null
|
|
1195
|
+
// resolution keeps the default built-in Responses summarizer.
|
|
1196
|
+
resolvedModel
|
|
1197
|
+
? (s, m) => withCodex(() => summarizeForCompaction(s, m, {
|
|
1198
|
+
client: resolvedModel.client,
|
|
1199
|
+
api: resolvedModel.provider.api,
|
|
1200
|
+
model: resolvedModel.configured.id,
|
|
1201
|
+
maxOutputTokens: s.contextSummaryMaxTokens,
|
|
1202
|
+
}))
|
|
1203
|
+
: undefined,
|
|
1204
|
+
forced ? { force: true } : {},
|
|
1205
|
+
);
|
|
1206
|
+
if (outcome.compacted) {
|
|
1207
|
+
// trigger precedence: an explicit operator /compact wins for the
|
|
1208
|
+
// event label; otherwise a hard-forced (at/over hardFraction*B)
|
|
1209
|
+
// compaction is surfaced as "hard" so the backstop firing is
|
|
1210
|
+
// observable downstream, and the default soft trigger is implicit.
|
|
1211
|
+
const trigger = forced ? "operator" : outcome.hardForced ? "hard" : undefined;
|
|
1212
|
+
await publish([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, ...(trigger ? { trigger } : {}) } }]);
|
|
1213
|
+
}
|
|
1214
|
+
} catch (compactError) {
|
|
1215
|
+
console.error("context compaction failed (turn proceeds un-compacted)", compactError);
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
// Cross-account reasoning strip: pass THIS turn's codex account so every
|
|
1219
|
+
// history read path (items + run-state replay) drops reasoning produced by
|
|
1220
|
+
// a DIFFERENT codex account. effectiveCodexCredentialId is the resolved
|
|
1221
|
+
// codex credential on a codex turn (pin > workspace-active) and null on a
|
|
1222
|
+
// non-codex turn OR a codex turn with no usable account — exactly the
|
|
1223
|
+
// "current account" the single strip rule compares against (null is the
|
|
1224
|
+
// built-in/Azure account, so a non-codex turn still drops codex-produced
|
|
1225
|
+
// reasoning, and a no-codex-history session is a byte-for-byte no-op).
|
|
1226
|
+
const { input: runInput, modelHistoryFromItems } = await turnInput(
|
|
1227
|
+
db,
|
|
1228
|
+
runtime,
|
|
1229
|
+
agent,
|
|
1230
|
+
trigger,
|
|
1231
|
+
runSettings,
|
|
1232
|
+
{ currentCodexCredentialId: effectiveCodexCredentialId },
|
|
1233
|
+
);
|
|
1234
|
+
// Slice index = the length of the model-facing (active) history this turn
|
|
1235
|
+
// is seeded from; new items beyond it (the trigger message + this turn's
|
|
1236
|
+
// generated items) are the ones to persist. After a compaction this is the
|
|
1237
|
+
// short [summary, ...tail] active set, NOT the total row count. The
|
|
1238
|
+
// absolute write position is tracked separately (next whole number past
|
|
1239
|
+
// the max existing position) because the fractional summary row means
|
|
1240
|
+
// total rows no longer equal max(position)+1. Pre-compaction both reduce to
|
|
1241
|
+
// the old total-count value, so the common path is unchanged.
|
|
1242
|
+
//
|
|
1243
|
+
// CRITICAL: seed from the SANITIZED active-row length, not the raw active
|
|
1244
|
+
// count. `prepareRunInput` builds `state.history` from
|
|
1245
|
+
// `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
|
|
1246
|
+
// rows (a legacy orphan/dangling pair), the in-memory history this turn
|
|
1247
|
+
// starts from is K shorter than the raw row count. The reconcile slices the
|
|
1248
|
+
// re-sanitized `state.history` off `persistedHistoryCount`; seeding it from
|
|
1249
|
+
// the raw count (K too high) skips K genuinely-new items, and a
|
|
1250
|
+
// `function_call` left in that skipped region can later have its
|
|
1251
|
+
// `function_call_result` persisted alone — the orphan that 400s on replay
|
|
1252
|
+
// and bricks the session (issue-61). The sanitized seed is already
|
|
1253
|
+
// orphan-free, so it is a stable prefix of the re-sanitized history and the
|
|
1254
|
+
// slice begins exactly at the first genuinely-new item.
|
|
1255
|
+
const activeSeedRows = await getActiveSessionHistoryItems(db, input.workspaceId, input.sessionId);
|
|
1256
|
+
// Seed the reconcile watermark from EXACTLY the view the model's
|
|
1257
|
+
// `state.history` was seeded from (items strip on the items path = HOLE D; NO
|
|
1258
|
+
// strip on the run-state blob path, where foreign reasoning is neutralized but
|
|
1259
|
+
// KEPT = HOLE E), so the model-input length and the watermark never disagree.
|
|
1260
|
+
persistedHistoryCount = reconcileSeedCount(
|
|
1261
|
+
activeSeedRows,
|
|
1262
|
+
modelHistoryFromItems,
|
|
1263
|
+
{ currentCodexCredentialId: effectiveCodexCredentialId },
|
|
1264
|
+
);
|
|
1265
|
+
historyCountAtTurnStart = persistedHistoryCount;
|
|
1266
|
+
nextHistoryPosition = await nextSessionHistoryPosition(db, input.workspaceId, input.sessionId);
|
|
1267
|
+
let responseUsageCount = 0;
|
|
1268
|
+
// Actual input tokens of the most recent model response this turn; the
|
|
1269
|
+
// pre-read trigger for the NEXT turn. Persisted at every turn-end path.
|
|
1270
|
+
let lastInputTokensObserved: number | null = null;
|
|
1271
|
+
throwIfWorkerShuttingDown();
|
|
1272
|
+
const runStreamOnce = (): ReturnType<OpenGeniRuntime["runStream"]> => runtime.runStream(agent, runInput, runSettings, {
|
|
1273
|
+
sandboxEnvironment,
|
|
1274
|
+
onRuntimeEvent: async (event) => {
|
|
1275
|
+
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
1276
|
+
},
|
|
1277
|
+
// P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
|
|
1278
|
+
// keystone). Absent when the flag is off -> legacy build-and-discard.
|
|
1279
|
+
...(resolvedSandbox
|
|
1280
|
+
? {
|
|
1281
|
+
ownedSandbox: {
|
|
1282
|
+
client: resolvedSandbox.established.client,
|
|
1283
|
+
session: resolvedSandbox.established.session,
|
|
1284
|
+
sessionState: resolvedSandbox.established.sessionState,
|
|
1285
|
+
// Pin platform setup (hooks + file materialization) to the un-proxied
|
|
1286
|
+
// established box — never through the routing proxy, which would
|
|
1287
|
+
// re-route those execs onto a machine swapped in mid-turn.
|
|
1288
|
+
...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
|
|
1289
|
+
},
|
|
1290
|
+
}
|
|
1291
|
+
: {}),
|
|
1292
|
+
});
|
|
1293
|
+
stream = await withCodex(runStreamOnce);
|
|
1294
|
+
batcher = createRuntimeBatcher(async (events) => {
|
|
1295
|
+
await publish!(events);
|
|
1296
|
+
});
|
|
1297
|
+
|
|
1298
|
+
const iterator = stream.toStream()[Symbol.asyncIterator]();
|
|
1299
|
+
let streamDone = false;
|
|
1300
|
+
try {
|
|
1301
|
+
while (true) {
|
|
1302
|
+
const next = await nextStreamEvent(iterator, activityContext);
|
|
1303
|
+
if (next.done) {
|
|
1304
|
+
streamDone = true;
|
|
1305
|
+
break;
|
|
1306
|
+
}
|
|
1307
|
+
const responseUsage = modelResponseUsageFromSdkEvent(next.value);
|
|
1308
|
+
if (responseUsage) {
|
|
1309
|
+
responseUsageCount += 1;
|
|
1310
|
+
const observed = responseUsage.usage?.inputTokens;
|
|
1311
|
+
if (typeof observed === "number" && observed > 0) {
|
|
1312
|
+
lastInputTokensObserved = observed;
|
|
1313
|
+
}
|
|
1314
|
+
await recordModelUsageAndDebitCredits(settings, db, {
|
|
1315
|
+
accountId: input.accountId,
|
|
1316
|
+
workspaceId: input.workspaceId,
|
|
1317
|
+
sessionId: input.sessionId,
|
|
1318
|
+
turnId,
|
|
1319
|
+
model: turn.model,
|
|
1320
|
+
isCodexTurn,
|
|
1321
|
+
usage: responseUsage.usage,
|
|
1322
|
+
sourceKey: modelUsageSourceKey({
|
|
1323
|
+
responseId: responseUsage.responseId,
|
|
1324
|
+
dispatchId,
|
|
1325
|
+
positionalKey: `response-${responseUsageCount}`,
|
|
1326
|
+
}),
|
|
1327
|
+
});
|
|
1328
|
+
await reconcileConversationTruth();
|
|
1329
|
+
try {
|
|
1330
|
+
await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
|
|
1331
|
+
} catch (limitError) {
|
|
1332
|
+
// Capture the run state at the boundary so the budget valve in
|
|
1333
|
+
// the outer catch can end this segment gracefully with full
|
|
1334
|
+
// conversation context preserved for the post-top-up resume.
|
|
1335
|
+
let serializedRunState: string | null = null;
|
|
1336
|
+
try {
|
|
1337
|
+
serializedRunState = stream.state.toString();
|
|
1338
|
+
} catch {
|
|
1339
|
+
serializedRunState = null;
|
|
1340
|
+
}
|
|
1341
|
+
throw new BudgetExhaustedError(
|
|
1342
|
+
limitError instanceof Error ? limitError.message : String(limitError),
|
|
1343
|
+
serializedRunState,
|
|
1344
|
+
);
|
|
1345
|
+
}
|
|
1346
|
+
}
|
|
1347
|
+
// Recording gate: a computer-use tool actually ran when an SDK
|
|
1348
|
+
// `tool_call_item` whose rawItem.type is "computer_call" streams through
|
|
1349
|
+
// (screenshot/click/type/scroll/etc. — the first computer action proves
|
|
1350
|
+
// the desktop was driven). Match the raw SDK event (ground truth) BEFORE
|
|
1351
|
+
// normalization. Only meaningful when a recording is live.
|
|
1352
|
+
if (activeRecording && !didComputerUse && isComputerCallStreamEvent(next.value)) {
|
|
1353
|
+
didComputerUse = true;
|
|
1354
|
+
}
|
|
1355
|
+
const normalized = normalizeSdkEvent(next.value);
|
|
1356
|
+
for (const event of normalized) {
|
|
1357
|
+
await batcher.push(event);
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1360
|
+
} finally {
|
|
1361
|
+
if (!streamDone) {
|
|
1362
|
+
await iterator.return?.();
|
|
1363
|
+
}
|
|
1364
|
+
}
|
|
1365
|
+
await batcher.flush();
|
|
1366
|
+
await stream.completed.catch(() => undefined);
|
|
1367
|
+
if (responseUsageCount === 0) {
|
|
1368
|
+
const aggregateInput = (stream.state.usage as { inputTokens?: unknown } | undefined)?.inputTokens;
|
|
1369
|
+
if (typeof aggregateInput === "number" && aggregateInput > 0) {
|
|
1370
|
+
lastInputTokensObserved = aggregateInput;
|
|
1371
|
+
}
|
|
1372
|
+
await recordModelUsageAndDebitCredits(settings, db, {
|
|
1373
|
+
accountId: input.accountId,
|
|
1374
|
+
workspaceId: input.workspaceId,
|
|
1375
|
+
sessionId: input.sessionId,
|
|
1376
|
+
turnId,
|
|
1377
|
+
model: turn.model,
|
|
1378
|
+
isCodexTurn,
|
|
1379
|
+
usage: stream.state.usage,
|
|
1380
|
+
sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
|
|
1381
|
+
});
|
|
1382
|
+
}
|
|
1383
|
+
if (lastInputTokensObserved !== null) {
|
|
1384
|
+
await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, lastInputTokensObserved)
|
|
1385
|
+
.catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
|
|
1386
|
+
}
|
|
1387
|
+
|
|
1388
|
+
if (stream.interruptions.length > 0) {
|
|
1389
|
+
await reconcileConversationTruth();
|
|
1390
|
+
const approvals = runtime.serializeApprovals(stream.interruptions);
|
|
1391
|
+
await saveRunState(db, {
|
|
1392
|
+
accountId: input.accountId,
|
|
1393
|
+
workspaceId: input.workspaceId,
|
|
1394
|
+
sessionId: input.sessionId,
|
|
1395
|
+
turnId,
|
|
1396
|
+
serializedRunState: stream.state.toString(),
|
|
1397
|
+
pendingApprovals: approvals,
|
|
1398
|
+
// Record the account freezing this state so a resume on a DIFFERENT
|
|
1399
|
+
// codex account strips its account-bound reasoning before replay (HOLE C).
|
|
1400
|
+
frozenCodexCredentialId: effectiveCodexCredentialId,
|
|
1401
|
+
});
|
|
1402
|
+
await publish([
|
|
1403
|
+
{ type: "session.requiresAction", payload: { approvals } },
|
|
1404
|
+
{ type: "session.status.changed", payload: { status: "requires_action" } },
|
|
1405
|
+
], true);
|
|
1406
|
+
await finishTurn(db, input.workspaceId, turnId, "requires_action");
|
|
1407
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "requires_action", turnId);
|
|
1408
|
+
activityStatus = "requires_action";
|
|
1409
|
+
return { status: "requires_action" };
|
|
1410
|
+
}
|
|
1411
|
+
|
|
1412
|
+
const finalOutput = String(stream.finalOutput ?? "");
|
|
1413
|
+
await reconcileConversationTruth();
|
|
1414
|
+
if (settings.sessionHistorySource !== "items") {
|
|
1415
|
+
// Legacy conversation memory; in items mode the blob is only written
|
|
1416
|
+
// for requires_action pauses (the one RunState-only resume path).
|
|
1417
|
+
await saveRunState(db, {
|
|
1418
|
+
accountId: input.accountId,
|
|
1419
|
+
workspaceId: input.workspaceId,
|
|
1420
|
+
sessionId: input.sessionId,
|
|
1421
|
+
turnId,
|
|
1422
|
+
serializedRunState: stream.state.toString(),
|
|
1423
|
+
pendingApprovals: [],
|
|
1424
|
+
frozenCodexCredentialId: effectiveCodexCredentialId,
|
|
1425
|
+
});
|
|
1426
|
+
}
|
|
1427
|
+
await publish([
|
|
1428
|
+
{ type: "agent.message.completed", payload: { text: finalOutput } },
|
|
1429
|
+
{ type: "turn.completed", payload: { output: finalOutput } },
|
|
1430
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1431
|
+
], true);
|
|
1432
|
+
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
1433
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1434
|
+
await recordUsageEvent(db, {
|
|
1435
|
+
accountId: input.accountId,
|
|
1436
|
+
workspaceId: input.workspaceId,
|
|
1437
|
+
eventType: "agent_run.completed",
|
|
1438
|
+
quantity: 1,
|
|
1439
|
+
unit: "run",
|
|
1440
|
+
sourceResourceType: "session_turn",
|
|
1441
|
+
sourceResourceId: turnId,
|
|
1442
|
+
idempotencyKey: `usage:agent_run.completed:${turnId}`,
|
|
1443
|
+
});
|
|
1444
|
+
activityStatus = "idle";
|
|
1445
|
+
return { status: "idle" };
|
|
1446
|
+
} catch (error) {
|
|
1447
|
+
// Graceful worker shutdown (deploy / rollout restart): checkpoint and
|
|
1448
|
+
// hand the turn back instead of failing the session. Conversation truth
|
|
1449
|
+
// is already dual-written per model response; the final reconcile below
|
|
1450
|
+
// bounds the loss to at most the single in-flight model step. The turn
|
|
1451
|
+
// is re-queued so the session workflow re-dispatches it on a healthy
|
|
1452
|
+
// worker — through a synthesized resume notice when this turn already
|
|
1453
|
+
// persisted progress (replaying the original trigger would duplicate
|
|
1454
|
+
// input the model has already seen and invite it to redo side effects),
|
|
1455
|
+
// or through the original trigger when nothing was persisted yet. This
|
|
1456
|
+
// is an explicit checkpoint/resume, not a blind activity retry: the
|
|
1457
|
+
// resumed attempt sees everything the first attempt did.
|
|
1458
|
+
//
|
|
1459
|
+
// The branch deliberately does NOT require turn.started to have been
|
|
1460
|
+
// published: a shutdown landing during setup (claim/billing, before the
|
|
1461
|
+
// turn visibly started) must also requeue, not fail the session. In
|
|
1462
|
+
// that early case nothing ran and nothing was checkpointed, so the
|
|
1463
|
+
// requeued turn replays the original trigger cleanly. The turn id falls
|
|
1464
|
+
// back to the workflow-claimed turn when the local lookup had not
|
|
1465
|
+
// finished yet.
|
|
1466
|
+
const preemptTurnId = turnId ?? input.turnId;
|
|
1467
|
+
// P1.2: a lease supersession during resume (a newer epoch re-established
|
|
1468
|
+
// the box concurrently) is NOT a session failure — re-dispatch the turn so
|
|
1469
|
+
// it re-resumes by id under the current epoch. Requeue + idle, mirroring
|
|
1470
|
+
// the worker-death re-dispatch (any worker re-resumes by id).
|
|
1471
|
+
if (error instanceof SandboxLeaseSupersededError && preemptTurnId) {
|
|
1472
|
+
try {
|
|
1473
|
+
await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, input.triggerEventId);
|
|
1474
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
|
|
1475
|
+
activityStatus = "preempted";
|
|
1476
|
+
return { status: "preempted" };
|
|
1477
|
+
} catch (requeueError) {
|
|
1478
|
+
console.error("sandbox lease supersession requeue failed; falling back", requeueError);
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
if (isWorkerShutdownCancellation(error) && preemptTurnId) {
|
|
1482
|
+
try {
|
|
1483
|
+
await batcher?.flush().catch(() => undefined);
|
|
1484
|
+
await reconcileConversationTruth();
|
|
1485
|
+
// An approval-decision rerun always replays its original trigger:
|
|
1486
|
+
// the decision is applied through the approval resume path reading
|
|
1487
|
+
// the frozen RunState blob (the only representation of a turn
|
|
1488
|
+
// paused mid-flight), so swapping the trigger for a resume notice
|
|
1489
|
+
// could drop the user's decision. Re-applying an already-consumed
|
|
1490
|
+
// approval re-executes at most the single approved step — the same
|
|
1491
|
+
// bound every preemption already accepts.
|
|
1492
|
+
const approvalRerun = triggerType === "user.approvalDecision";
|
|
1493
|
+
let resumeWithNotice = !approvalRerun && settings.sessionHistorySource === "items" && persistedHistoryCount > historyCountAtTurnStart;
|
|
1494
|
+
if (settings.sessionHistorySource !== "items" && stream) {
|
|
1495
|
+
// Legacy run-state mode: the resume reads the RunState blob, so
|
|
1496
|
+
// the checkpoint must be captured there — including any pending
|
|
1497
|
+
// approval interruptions, exactly like the requires_action path,
|
|
1498
|
+
// so a shutdown while approvals wait does not erase them. A
|
|
1499
|
+
// failed capture falls back to the previous snapshot and a clean
|
|
1500
|
+
// re-run of the original trigger.
|
|
1501
|
+
try {
|
|
1502
|
+
await saveRunState(db, {
|
|
1503
|
+
accountId: input.accountId,
|
|
1504
|
+
workspaceId: input.workspaceId,
|
|
1505
|
+
sessionId: input.sessionId,
|
|
1506
|
+
turnId: preemptTurnId,
|
|
1507
|
+
serializedRunState: stream.state.toString(),
|
|
1508
|
+
pendingApprovals: runtime.serializeApprovals(stream.interruptions ?? []),
|
|
1509
|
+
frozenCodexCredentialId: effectiveCodexCredentialId,
|
|
1510
|
+
});
|
|
1511
|
+
resumeWithNotice = true;
|
|
1512
|
+
} catch {
|
|
1513
|
+
resumeWithNotice = false;
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
1516
|
+
const [preemptedEvent] = await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [
|
|
1517
|
+
{
|
|
1518
|
+
turnId: preemptTurnId,
|
|
1519
|
+
type: "turn.preempted",
|
|
1520
|
+
payload: {
|
|
1521
|
+
triggerEventId: input.triggerEventId,
|
|
1522
|
+
reason: "worker_shutdown",
|
|
1523
|
+
resumeWithNotice,
|
|
1524
|
+
...(resumeWithNotice ? { text: WORKER_SHUTDOWN_RESUME_TEXT } : {}),
|
|
1525
|
+
},
|
|
1526
|
+
},
|
|
1527
|
+
{
|
|
1528
|
+
turnId: preemptTurnId,
|
|
1529
|
+
type: "session.status.changed",
|
|
1530
|
+
payload: { status: "queued" },
|
|
1531
|
+
},
|
|
1532
|
+
]);
|
|
1533
|
+
await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
1534
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
|
|
1535
|
+
activityStatus = "preempted";
|
|
1536
|
+
return { status: "preempted" };
|
|
1537
|
+
} catch (preemptError) {
|
|
1538
|
+
// A failing checkpoint/requeue must not surface as an arbitrary
|
|
1539
|
+
// activity error around a half-applied preemption (the workflow
|
|
1540
|
+
// would fail the session while a requeued turn lingers). Fall
|
|
1541
|
+
// through to the cancellation path below: the turn is marked
|
|
1542
|
+
// cancelled — also resetting a turn this block already requeued —
|
|
1543
|
+
// and the session fails like an uncheckpointed death.
|
|
1544
|
+
console.error("worker-shutdown preemption failed; falling back to cancellation", preemptError);
|
|
1545
|
+
}
|
|
1546
|
+
}
|
|
1547
|
+
if (error instanceof CancelledFailure) {
|
|
1548
|
+
activityStatus = "cancelled";
|
|
1549
|
+
activityError = error;
|
|
1550
|
+
await batcher?.flush().catch(() => undefined);
|
|
1551
|
+
if (preemptTurnId) {
|
|
1552
|
+
await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => undefined);
|
|
1553
|
+
}
|
|
1554
|
+
throw error;
|
|
1555
|
+
}
|
|
1556
|
+
// The SDK's per-segment turn cap is a pacing valve, not a failure: end
|
|
1557
|
+
// the turn gracefully and idle the session so an active goal continues
|
|
1558
|
+
// via a synthesized continuation turn (or a user message resumes work).
|
|
1559
|
+
// The run state captured at the cap keeps full conversation context for
|
|
1560
|
+
// that resumption.
|
|
1561
|
+
const maxTurns = maxTurnsExceededRunState(error);
|
|
1562
|
+
if (maxTurns && publish && turnId && turnStartedPublished) {
|
|
1563
|
+
await batcher?.flush().catch(() => undefined);
|
|
1564
|
+
// The SDK attaches the run state at the throw site; persisting it lets
|
|
1565
|
+
// the continuation resume with this segment's full context. If capture
|
|
1566
|
+
// ever fails, the continuation falls back to the previous snapshot --
|
|
1567
|
+
// degraded context, flagged on the event, but still strictly better
|
|
1568
|
+
// than a terminal failed session: the sandbox filesystem state
|
|
1569
|
+
// persists independently and the agent re-derives from it.
|
|
1570
|
+
await reconcileConversationTruth();
|
|
1571
|
+
const runStateSaved = Boolean(maxTurns.serializedRunState) && settings.sessionHistorySource !== "items";
|
|
1572
|
+
if (maxTurns.serializedRunState && settings.sessionHistorySource !== "items") {
|
|
1573
|
+
await saveRunState(db, {
|
|
1574
|
+
accountId: input.accountId,
|
|
1575
|
+
workspaceId: input.workspaceId,
|
|
1576
|
+
sessionId: input.sessionId,
|
|
1577
|
+
turnId,
|
|
1578
|
+
serializedRunState: maxTurns.serializedRunState,
|
|
1579
|
+
pendingApprovals: [],
|
|
1580
|
+
frozenCodexCredentialId: effectiveCodexCredentialId,
|
|
1581
|
+
});
|
|
1582
|
+
}
|
|
1583
|
+
await publish([
|
|
1584
|
+
{ type: "turn.completed", payload: { output: "", segmentLimit: "max_turns", runStateSaved } },
|
|
1585
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1586
|
+
], true);
|
|
1587
|
+
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
1588
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1589
|
+
await recordUsageEvent(db, {
|
|
1590
|
+
accountId: input.accountId,
|
|
1591
|
+
workspaceId: input.workspaceId,
|
|
1592
|
+
eventType: "agent_run.completed",
|
|
1593
|
+
quantity: 1,
|
|
1594
|
+
unit: "run",
|
|
1595
|
+
sourceResourceType: "session_turn",
|
|
1596
|
+
sourceResourceId: turnId,
|
|
1597
|
+
idempotencyKey: `usage:agent_run.completed:${turnId}`,
|
|
1598
|
+
});
|
|
1599
|
+
activityStatus = "idle";
|
|
1600
|
+
return { status: "idle" };
|
|
1601
|
+
}
|
|
1602
|
+
// A ChatGPT/Codex usage cap (429 usage_limit_reached) is account state,
|
|
1603
|
+
// NOT an agent failure: surface the precise, actionable message (so the
|
|
1604
|
+
// user sees the reset window) but idle the session — never go terminal,
|
|
1605
|
+
// which would reject the user's next message after the cap lifts. The
|
|
1606
|
+
// payload is retryable:false so the generic provider-backpressure auto-retry
|
|
1607
|
+
// does not loop. For an active goal we hold the continuation for the reported
|
|
1608
|
+
// reset window (capped) so it resumes itself when access returns, instead of
|
|
1609
|
+
// hammering the capped backend.
|
|
1610
|
+
const usageLimit = classifyCodexUsageLimitError(error);
|
|
1611
|
+
if (usageLimit && publish && turnId && turnStartedPublished) {
|
|
1612
|
+
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
1613
|
+
const goalActive = Boolean(goal && goal.status === "active");
|
|
1614
|
+
await batcher?.flush().catch(() => undefined);
|
|
1615
|
+
await reconcileConversationTruth();
|
|
1616
|
+
const serializedRunState = agentsErrorRunState(error);
|
|
1617
|
+
const runStateSaved = Boolean(serializedRunState) && settings.sessionHistorySource !== "items";
|
|
1618
|
+
if (serializedRunState && settings.sessionHistorySource !== "items") {
|
|
1619
|
+
await saveRunState(db, {
|
|
1620
|
+
accountId: input.accountId,
|
|
1621
|
+
workspaceId: input.workspaceId,
|
|
1622
|
+
sessionId: input.sessionId,
|
|
1623
|
+
turnId,
|
|
1624
|
+
serializedRunState,
|
|
1625
|
+
pendingApprovals: [],
|
|
1626
|
+
frozenCodexCredentialId: effectiveCodexCredentialId,
|
|
1627
|
+
});
|
|
1628
|
+
}
|
|
1629
|
+
// --- P3 reactive rotation (gated; re-fetch fresh state on this already-failed,
|
|
1630
|
+
// already-idling path). Mark THIS account cooling until its reset, then CONSULT the
|
|
1631
|
+
// engine over fresh accounts to decide continueDelayMs: a fast 0-delay re-dispatch
|
|
1632
|
+
// when another account is available, or idle-until-earliest when all are capped. The
|
|
1633
|
+
// catch deliberately does NOT move the active pointer — the re-dispatched turn's
|
|
1634
|
+
// proactive seam (turn-start) is the single authoritative pointer-move + strip site.
|
|
1635
|
+
let rotated = false;
|
|
1636
|
+
let rotationResumeMs: number | null = null; // 0 ⇒ a candidate is available; re-dispatch now
|
|
1637
|
+
let rotationResumeIdleUntilReset = false; // circuit-breaker fall (Finding 1b) ⇒ MANDATORY hold
|
|
1638
|
+
let allCappedResetAt: Date | null = null; // set ⇒ every account capped; idle until this
|
|
1639
|
+
if (effectiveCodexCredentialId) {
|
|
1640
|
+
const [rotation, sessionCodex] = await Promise.all([
|
|
1641
|
+
getCodexRotationSettings(db, input.workspaceId).catch(() => null),
|
|
1642
|
+
getSessionCodexState(db, input.workspaceId, input.sessionId).catch(() => null),
|
|
1643
|
+
]);
|
|
1644
|
+
const rotating = Boolean(rotation?.rotationEnabled) && (sessionCodex?.pinnedCredentialId == null);
|
|
1645
|
+
if (rotating && rotation) {
|
|
1646
|
+
const accounts = await listCodexAccountStatuses(db, input.workspaceId).catch(() => []);
|
|
1647
|
+
const serving = accounts.find((a) => a.id === effectiveCodexCredentialId) ?? null;
|
|
1648
|
+
// Cooldown end (invariant 5): authoritative resets_in_seconds from the 429; else
|
|
1649
|
+
// the serving account's soonest cached window reset; else the 1h cap.
|
|
1650
|
+
const cachedReset = [serving?.primaryResetAt, serving?.secondaryResetAt]
|
|
1651
|
+
.filter((d): d is Date => d instanceof Date && d.getTime() > Date.now())
|
|
1652
|
+
.sort((a, b) => a.getTime() - b.getTime())[0] ?? null;
|
|
1653
|
+
const until = usageLimit.resetsInSeconds !== null && Number.isFinite(usageLimit.resetsInSeconds) && usageLimit.resetsInSeconds > 0
|
|
1654
|
+
? new Date(Date.now() + Math.ceil(usageLimit.resetsInSeconds) * 1000)
|
|
1655
|
+
: (cachedReset ?? new Date(Date.now() + CODEX_USAGE_LIMIT_MAX_RESUME_MS));
|
|
1656
|
+
// Finding 1a: INSPECT the cooldown-write result. A swallowed best-effort
|
|
1657
|
+
// write whose failure went unnoticed is exactly what lets the next proactive
|
|
1658
|
+
// rank re-pick this just-capped account (stale-low cached usedPercent, not
|
|
1659
|
+
// cooling) — so capture whether it PERSISTED and feed it into the resume floor.
|
|
1660
|
+
const cooldownPersisted = await setCodexCredentialExhausted(db, input.workspaceId, effectiveCodexCredentialId, until).catch(() => false);
|
|
1661
|
+
// Re-rank over the fresh accounts; the in-memory list predates the cooldown
|
|
1662
|
+
// write, so stamp the just-cooled account so the engine excludes it now. The
|
|
1663
|
+
// serving account is thus walked AT MOST ONCE per turn (invariant 4: bounded).
|
|
1664
|
+
const fresh = accounts.map((a) => (a.id === effectiveCodexCredentialId ? { ...a, exhaustedUntil: until } : a));
|
|
1665
|
+
const decision = chooseRotationActive({
|
|
1666
|
+
rotationStrategy: rotation.rotationStrategy as CodexRotationStrategy,
|
|
1667
|
+
activeCredentialId: rotation.activeCredentialId,
|
|
1668
|
+
priorCredentialId: effectiveCodexCredentialId,
|
|
1669
|
+
accounts: fresh,
|
|
1670
|
+
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
1671
|
+
now: new Date(),
|
|
1672
|
+
// P4: the just-capped serving account's connector set is the proxy for
|
|
1673
|
+
// "what this session has access to" — prefer a covering failover target.
|
|
1674
|
+
usedConnectors: serving?.connectorNamespaces ?? [],
|
|
1675
|
+
});
|
|
1676
|
+
if (decision.kind === "active") {
|
|
1677
|
+
rotated = true;
|
|
1678
|
+
// Finding 1: a live candidate normally re-dispatches NOW (0). Two second-order
|
|
1679
|
+
// faults would turn that 0 into a hot loop, so bound it. Count the consecutive
|
|
1680
|
+
// reactive failovers since the last successful turn (this one is not yet
|
|
1681
|
+
// published) and combine with the cooldown-persistence result.
|
|
1682
|
+
const priorConsecutiveRotations = await countConsecutiveReactiveRotations(db, input.workspaceId, input.sessionId).catch(() => 0);
|
|
1683
|
+
const resume = computeReactiveRotationResume({
|
|
1684
|
+
cooldownPersisted,
|
|
1685
|
+
priorConsecutiveRotations,
|
|
1686
|
+
connectedAccountCount: accounts.length,
|
|
1687
|
+
});
|
|
1688
|
+
rotationResumeMs = resume.continueDelayMs; // 0 (happy path), a slow-retry floor, or the circuit-breaker idle
|
|
1689
|
+
rotationResumeIdleUntilReset = resume.idleUntilReset; // true only on the circuit-breaker fall (MANDATORY hold)
|
|
1690
|
+
} else if (decision.kind === "allCapped") {
|
|
1691
|
+
rotated = true;
|
|
1692
|
+
allCappedResetAt = decision.earliestResetAt;
|
|
1693
|
+
}
|
|
1694
|
+
// kind:"none" → fall through to today's single-account idle.
|
|
1695
|
+
}
|
|
1696
|
+
}
|
|
1697
|
+
|
|
1698
|
+
const failurePayload = allCappedResetAt
|
|
1699
|
+
? codexUsageLimitFailurePayload(
|
|
1700
|
+
{ resetsInSeconds: Math.ceil(Math.max(0, allCappedResetAt.getTime() - Date.now()) / 1000) },
|
|
1701
|
+
error instanceof Error ? error.message : String(error),
|
|
1702
|
+
{ allAccounts: true },
|
|
1703
|
+
)
|
|
1704
|
+
: codexUsageLimitFailurePayload(usageLimit, error instanceof Error ? error.message : String(error));
|
|
1705
|
+
await publish([
|
|
1706
|
+
// `rotated:true` ONLY on the reactive rotation path tells evaluateGoalContinuation to
|
|
1707
|
+
// freeze autoContinuations (a rotation walk must not burn the goal's continuation budget).
|
|
1708
|
+
{ type: "turn.failed", payload: { ...failurePayload, recovery: goalActive ? "goal_continuation" : "user_message", runStateSaved, ...(rotated ? { rotated: true } : {}) } },
|
|
1709
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1710
|
+
], true);
|
|
1711
|
+
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
1712
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1713
|
+
activityStatus = "idle";
|
|
1714
|
+
activityError = error;
|
|
1715
|
+
if (goalActive) {
|
|
1716
|
+
// Rotation: a candidate is available → continue NOW (0). All-capped → idle until the
|
|
1717
|
+
// earliest reset across all accounts (capped at 1h). Else the unchanged single-account idle.
|
|
1718
|
+
if (rotationResumeMs !== null) {
|
|
1719
|
+
// A candidate IS available. Normally the just-failed account is now cooling so
|
|
1720
|
+
// the ranker cannot re-pick it → 0 (re-dispatch NOW, the legitimate skip-the-hold
|
|
1721
|
+
// case). Finding 1 bounds the two exceptions: a persistence fault yields a positive
|
|
1722
|
+
// slow-retry floor, and once consecutive failovers exceed the account count + margin
|
|
1723
|
+
// the circuit breaker returns a fixed MANDATORY idle (idleUntilReset) — never a 0-delay
|
|
1724
|
+
// hot loop against a capped backend + DB.
|
|
1725
|
+
return { status: "idle", continueDelayMs: rotationResumeMs, ...(rotationResumeIdleUntilReset ? { idleUntilReset: true } : {}) };
|
|
1726
|
+
}
|
|
1727
|
+
// All-capped: clamp to [MIN_IDLE_MS, max] — a POSITIVE, BOUNDED hold (never 0,
|
|
1728
|
+
// so session.ts can never tight-loop). The post-idle continuation re-dispatch
|
|
1729
|
+
// hits the proactive seam, which refreshes usage and self-heals.
|
|
1730
|
+
const resumeMs = allCappedResetAt
|
|
1731
|
+
? computeIdleDelayMs(allCappedResetAt, new Date(), CODEX_USAGE_LIMIT_MAX_RESUME_MS)
|
|
1732
|
+
: (usageLimit.resetsInSeconds !== null && Number.isFinite(usageLimit.resetsInSeconds) && usageLimit.resetsInSeconds > 0
|
|
1733
|
+
? Math.min(Math.ceil(usageLimit.resetsInSeconds) * 1000, CODEX_USAGE_LIMIT_MAX_RESUME_MS)
|
|
1734
|
+
: CODEX_USAGE_LIMIT_MAX_RESUME_MS);
|
|
1735
|
+
return { status: "idle", continueDelayMs: resumeMs, ...(allCappedResetAt ? { idleUntilReset: true } : {}) };
|
|
1736
|
+
}
|
|
1737
|
+
return { status: "idle" };
|
|
1738
|
+
}
|
|
1739
|
+
// Budget/limit exhaustion between model calls is account state, not an
|
|
1740
|
+
// agent failure: idle the session for goal-bearing and goal-less runs
|
|
1741
|
+
// alike (a failed session would reject the user's next message after a
|
|
1742
|
+
// top-up). An active goal pauses visibly with reason "limits" at the
|
|
1743
|
+
// next continuation evaluation, without consuming continuation budget.
|
|
1744
|
+
if (error instanceof BudgetExhaustedError && publish && turnId && turnStartedPublished) {
|
|
1745
|
+
await batcher?.flush().catch(() => undefined);
|
|
1746
|
+
await reconcileConversationTruth();
|
|
1747
|
+
const runStateSaved = Boolean(error.serializedRunState) && settings.sessionHistorySource !== "items";
|
|
1748
|
+
if (error.serializedRunState && settings.sessionHistorySource !== "items") {
|
|
1749
|
+
await saveRunState(db, {
|
|
1750
|
+
accountId: input.accountId,
|
|
1751
|
+
workspaceId: input.workspaceId,
|
|
1752
|
+
sessionId: input.sessionId,
|
|
1753
|
+
turnId,
|
|
1754
|
+
serializedRunState: error.serializedRunState,
|
|
1755
|
+
pendingApprovals: [],
|
|
1756
|
+
frozenCodexCredentialId: effectiveCodexCredentialId,
|
|
1757
|
+
});
|
|
1758
|
+
}
|
|
1759
|
+
await publish([
|
|
1760
|
+
{ type: "turn.completed", payload: { output: "", segmentLimit: "budget_exhausted", detail: error.message, runStateSaved } },
|
|
1761
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1762
|
+
], true);
|
|
1763
|
+
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
1764
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1765
|
+
await recordUsageEvent(db, {
|
|
1766
|
+
accountId: input.accountId,
|
|
1767
|
+
workspaceId: input.workspaceId,
|
|
1768
|
+
eventType: "agent_run.completed",
|
|
1769
|
+
quantity: 1,
|
|
1770
|
+
unit: "run",
|
|
1771
|
+
sourceResourceType: "session_turn",
|
|
1772
|
+
sourceResourceId: turnId,
|
|
1773
|
+
idempotencyKey: `usage:agent_run.completed:${turnId}`,
|
|
1774
|
+
});
|
|
1775
|
+
activityStatus = "idle";
|
|
1776
|
+
return { status: "idle" };
|
|
1777
|
+
}
|
|
1778
|
+
// A retryable provider failure (rate limit) is transient backpressure,
|
|
1779
|
+
// not a session failure: the in-client retry budget is already
|
|
1780
|
+
// exhausted by the time the error reaches here, so fail the turn
|
|
1781
|
+
// truthfully but idle the session. With an active goal the continuation
|
|
1782
|
+
// loop resumes after a pacing delay; without one the session simply
|
|
1783
|
+
// waits for the next user message. Long-lived sessions (a per-customer
|
|
1784
|
+
// ops channel between goals) must never go terminal because the
|
|
1785
|
+
// provider had a bad minute -- a failed session would reject the very
|
|
1786
|
+
// retry message the failure asks for.
|
|
1787
|
+
const failure = agentRunFailurePayload(error);
|
|
1788
|
+
if (failure.retryable && publish && turnId && turnStartedPublished) {
|
|
1789
|
+
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
1790
|
+
const goalActive = Boolean(goal && goal.status === "active");
|
|
1791
|
+
await batcher?.flush().catch(() => undefined);
|
|
1792
|
+
// Provider errors rarely carry SDK run state; a null falls back to
|
|
1793
|
+
// the previous snapshot, same degraded-context contract as the
|
|
1794
|
+
// max-turns path above.
|
|
1795
|
+
await reconcileConversationTruth();
|
|
1796
|
+
const serializedRunState = agentsErrorRunState(error);
|
|
1797
|
+
const runStateSaved = Boolean(serializedRunState) && settings.sessionHistorySource !== "items";
|
|
1798
|
+
if (serializedRunState && settings.sessionHistorySource !== "items") {
|
|
1799
|
+
await saveRunState(db, {
|
|
1800
|
+
accountId: input.accountId,
|
|
1801
|
+
workspaceId: input.workspaceId,
|
|
1802
|
+
sessionId: input.sessionId,
|
|
1803
|
+
turnId,
|
|
1804
|
+
serializedRunState,
|
|
1805
|
+
pendingApprovals: [],
|
|
1806
|
+
frozenCodexCredentialId: effectiveCodexCredentialId,
|
|
1807
|
+
});
|
|
1808
|
+
}
|
|
1809
|
+
await publish([
|
|
1810
|
+
{ type: "turn.failed", payload: { ...failure, recovery: goalActive ? "goal_continuation" : "user_message", runStateSaved } },
|
|
1811
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1812
|
+
], true);
|
|
1813
|
+
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
1814
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1815
|
+
activityStatus = "idle";
|
|
1816
|
+
activityError = error;
|
|
1817
|
+
return goalActive ? { status: "idle", continueDelayMs: PROVIDER_BACKPRESSURE_DELAY_MS } : { status: "idle" };
|
|
1818
|
+
}
|
|
1819
|
+
activityStatus = "failed";
|
|
1820
|
+
activityError = error;
|
|
1821
|
+
if (!publish || !turnId || !turnStartedPublished) {
|
|
1822
|
+
throw error;
|
|
1823
|
+
}
|
|
1824
|
+
await publish([
|
|
1825
|
+
{ type: "turn.failed", payload: failure },
|
|
1826
|
+
{ type: "session.status.changed", payload: { status: "failed" } },
|
|
1827
|
+
], true);
|
|
1828
|
+
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
1829
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "failed", null);
|
|
1830
|
+
// The common failure path ends here: runAgentTurn marks the session
|
|
1831
|
+
// failed and returns "failed", and the session workflow then exits
|
|
1832
|
+
// WITHOUT calling failSession/markSessionIdle. Wake a spawned worker's
|
|
1833
|
+
// parent here too, so a manager learns of a worker that died inside its
|
|
1834
|
+
// turn (not just one failed by the workflow's failSession path). Deduped
|
|
1835
|
+
// per terminal episode by the child's lastSequence, so it never
|
|
1836
|
+
// double-fires with the workflow-level wake.
|
|
1837
|
+
await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed", `turn:${turnId}`);
|
|
1838
|
+
return { status: "failed" };
|
|
1839
|
+
} finally {
|
|
1840
|
+
const durationSeconds = (performance.now() - activityStarted) / 1000;
|
|
1841
|
+
observability.recordWorkerActivity({
|
|
1842
|
+
activity: "runAgentTurn",
|
|
1843
|
+
status: activityStatus,
|
|
1844
|
+
durationSeconds,
|
|
1845
|
+
});
|
|
1846
|
+
activitySpan.end({
|
|
1847
|
+
attributes: {
|
|
1848
|
+
"opengeni.turn_id": turnId ?? "",
|
|
1849
|
+
"opengeni.status": activityStatus,
|
|
1850
|
+
"opengeni.environment_id": environmentId,
|
|
1851
|
+
"opengeni.duration_ms": Math.round(durationSeconds * 1000),
|
|
1852
|
+
},
|
|
1853
|
+
error: activityError,
|
|
1854
|
+
});
|
|
1855
|
+
// Multi-account P4: flush the serving account's free per-turn caches ONCE,
|
|
1856
|
+
// best-effort (same discipline as today's usage write). Both writers skip
|
|
1857
|
+
// version/updatedAt, so neither can race the token-refresh CAS.
|
|
1858
|
+
if (effectiveCodexCredentialId) {
|
|
1859
|
+
// Part A: the latest scraped usage-header snapshot → the P2 usage cache. A
|
|
1860
|
+
// full both-windows snapshot (parseCodexUsageHeaders gates on both), so this
|
|
1861
|
+
// is byte-identical to the /wham/usage write — no partial-window clobber.
|
|
1862
|
+
if (latestCodexUsage) {
|
|
1863
|
+
await recordCodexAccountUsage(db, input.workspaceId, effectiveCodexCredentialId, latestCodexUsage)
|
|
1864
|
+
.catch(() => undefined);
|
|
1865
|
+
}
|
|
1866
|
+
// Part B.1: the connector namespaces codex_apps listed this turn → the
|
|
1867
|
+
// connector-set cache. NON-EMPTY-only: a flaky/empty tools/list must never
|
|
1868
|
+
// overwrite a known set with [] (false coverage drop). Read by reference
|
|
1869
|
+
// AFTER the run, so every tools/list this turn has accumulated.
|
|
1870
|
+
const connectorNamespaces = preparedTools?.codexConnectorNamespaces;
|
|
1871
|
+
if (connectorNamespaces && connectorNamespaces.size > 0) {
|
|
1872
|
+
await recordCodexAccountConnectors(db, input.workspaceId, effectiveCodexCredentialId, [...connectorNamespaces])
|
|
1873
|
+
.catch(() => undefined);
|
|
1874
|
+
}
|
|
1875
|
+
}
|
|
1876
|
+
await preparedTools?.close().catch(() => undefined);
|
|
1877
|
+
if (heartbeatTimer) {
|
|
1878
|
+
clearInterval(heartbeatTimer);
|
|
1879
|
+
}
|
|
1880
|
+
// P1.2: stop the lease-TTL refresh, release the turn holder (idempotent
|
|
1881
|
+
// delete-my-row; refcount-- and warm->draining if it hit 0 with no turns),
|
|
1882
|
+
// and DROP the in-memory handle. Release NEVER stops the box — the reaper
|
|
1883
|
+
// (P1.3) issues the provider stop() past the drain grace at refcount 0; the
|
|
1884
|
+
// box rides the provider idle-timeout in the meantime. Best-effort: a
|
|
1885
|
+
// release failure must never mask the turn's real outcome.
|
|
1886
|
+
if (leaseHeartbeatTimer) {
|
|
1887
|
+
clearInterval(leaseHeartbeatTimer);
|
|
1888
|
+
}
|
|
1889
|
+
// P4.3: finalize the on-turn recording BEFORE releasing the box handle
|
|
1890
|
+
// (the read+PUT needs the live session). read bytes → storage PUT →
|
|
1891
|
+
// updateRecording(available) → publish recording.available; the box file is
|
|
1892
|
+
// deleted only after the PUT confirms (F9). Best-effort: a finalize failure
|
|
1893
|
+
// emits recording.failed and never masks the turn outcome (F10: the bytes
|
|
1894
|
+
// never become a Temporal payload — read+PUT happen here in-process).
|
|
1895
|
+
if (activeRecording && resolvedSandbox) {
|
|
1896
|
+
try {
|
|
1897
|
+
if (!didComputerUse) {
|
|
1898
|
+
// Recording gate (P4.3): the turn never drove the desktop (a plain text
|
|
1899
|
+
// turn), so the film is a static frame of an untouched screen. DISCARD
|
|
1900
|
+
// it — stop ffmpeg, delete the box artifact, drop the row — with NO
|
|
1901
|
+
// storage PUT and NO recording.failed (a clean no-op, not a failure).
|
|
1902
|
+
await discardRecording({
|
|
1903
|
+
db,
|
|
1904
|
+
accountId: input.accountId,
|
|
1905
|
+
workspaceId: input.workspaceId,
|
|
1906
|
+
active: activeRecording,
|
|
1907
|
+
session: resolvedSandbox.established.session,
|
|
1908
|
+
});
|
|
1909
|
+
} else {
|
|
1910
|
+
const outcome = await finalizeRecording({
|
|
1911
|
+
settings,
|
|
1912
|
+
db,
|
|
1913
|
+
objectStorage,
|
|
1914
|
+
accountId: input.accountId,
|
|
1915
|
+
workspaceId: input.workspaceId,
|
|
1916
|
+
sessionId: input.sessionId,
|
|
1917
|
+
active: activeRecording,
|
|
1918
|
+
session: resolvedSandbox.established.session,
|
|
1919
|
+
runAs: sandboxRunAs(settings),
|
|
1920
|
+
});
|
|
1921
|
+
if (publish) {
|
|
1922
|
+
await publish(outcome.ok
|
|
1923
|
+
? [{ type: "recording.available", payload: outcome.available }]
|
|
1924
|
+
: [{ type: "recording.failed", payload: { recordingId: activeRecording.recordingId, turnId: activeRecording.turnId, reason: outcome.reason, detail: outcome.detail } }]);
|
|
1925
|
+
}
|
|
1926
|
+
}
|
|
1927
|
+
} catch (finalizeError) {
|
|
1928
|
+
console.error("recording finalize failed (turn outcome unaffected)", finalizeError);
|
|
1929
|
+
} finally {
|
|
1930
|
+
activeRecording = null;
|
|
1931
|
+
}
|
|
1932
|
+
}
|
|
1933
|
+
if (resolvedSandbox) {
|
|
1934
|
+
await resolvedSandbox.release().catch((releaseError) => {
|
|
1935
|
+
console.error("sandbox lease release failed (turn outcome unaffected)", releaseError);
|
|
1936
|
+
});
|
|
1937
|
+
resolvedSandbox = null; // drop the handle; the box survives the turn
|
|
1938
|
+
}
|
|
1939
|
+
}
|
|
1940
|
+
};
|
|
1941
|
+
}
|
|
1942
|
+
|
|
1943
|
+
export function agentRunFailurePayload(error: unknown): { error: string; code?: string; retryable?: boolean; detail?: string } {
|
|
1944
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1945
|
+
const status = typeof error === "object" && error !== null && "status" in error
|
|
1946
|
+
? Number((error as { status?: unknown }).status)
|
|
1947
|
+
: undefined;
|
|
1948
|
+
const code = typeof error === "object" && error !== null && "code" in error
|
|
1949
|
+
? String((error as { code?: unknown }).code)
|
|
1950
|
+
: undefined;
|
|
1951
|
+
// A ChatGPT/Codex usage cap is a HARD limit, not transient backpressure: it
|
|
1952
|
+
// must NOT be reported as a generic, retryable rate-limit (which would loop a
|
|
1953
|
+
// goal against a capped backend). Surface a precise, actionable message with
|
|
1954
|
+
// the humanized reset window and code, non-retryable. Checked BEFORE the
|
|
1955
|
+
// generic 429 branch below (a usage cap is also a 429).
|
|
1956
|
+
const usageLimit = classifyCodexUsageLimitError(error);
|
|
1957
|
+
if (usageLimit) {
|
|
1958
|
+
return codexUsageLimitFailurePayload(usageLimit, message);
|
|
1959
|
+
}
|
|
1960
|
+
if (status === 429 || code === "rate_limit_exceeded" || /(?:too many requests|rate.?limit|\b429\b)/i.test(message)) {
|
|
1961
|
+
return {
|
|
1962
|
+
error: "Model provider rate limit hit. Try again in a minute or lower the reasoning effort.",
|
|
1963
|
+
code: "provider_rate_limited",
|
|
1964
|
+
retryable: true,
|
|
1965
|
+
...(message && message !== "Too Many Requests" ? { detail: message } : {}),
|
|
1966
|
+
};
|
|
1967
|
+
}
|
|
1968
|
+
return { error: message };
|
|
1969
|
+
}
|
|
1970
|
+
|
|
1971
|
+
/** Humanize a seconds duration into a short "2h 5m" / "9m" / "in under a minute" string. */
|
|
1972
|
+
export function humanizeResetWindow(resetsInSeconds: number | null): string {
|
|
1973
|
+
if (resetsInSeconds === null || !Number.isFinite(resetsInSeconds) || resetsInSeconds <= 0) {
|
|
1974
|
+
return "shortly";
|
|
1975
|
+
}
|
|
1976
|
+
const total = Math.ceil(resetsInSeconds);
|
|
1977
|
+
if (total < 60) {
|
|
1978
|
+
return "in under a minute";
|
|
1979
|
+
}
|
|
1980
|
+
const hours = Math.floor(total / 3600);
|
|
1981
|
+
const minutes = Math.round((total % 3600) / 60);
|
|
1982
|
+
if (hours > 0) {
|
|
1983
|
+
return minutes > 0 ? `in about ${hours}h ${minutes}m` : `in about ${hours}h`;
|
|
1984
|
+
}
|
|
1985
|
+
return `in about ${minutes}m`;
|
|
1986
|
+
}
|
|
1987
|
+
|
|
1988
|
+
/**
|
|
1989
|
+
* Build the turn.failed payload for a ChatGPT/Codex usage cap: a precise,
|
|
1990
|
+
* actionable message naming the reset window, the stable `codex_usage_limit_reached`
|
|
1991
|
+
* code, and retryable:false (an auto-retry would just re-hit the cap).
|
|
1992
|
+
*/
|
|
1993
|
+
export function codexUsageLimitFailurePayload(
|
|
1994
|
+
info: { resetsInSeconds: number | null },
|
|
1995
|
+
detail: string,
|
|
1996
|
+
opts?: { allAccounts?: boolean },
|
|
1997
|
+
): { error: string; code: string; retryable: boolean; detail?: string } {
|
|
1998
|
+
// P3: when EVERY connected subscription is rate-limited the message names the
|
|
1999
|
+
// earliest reset across accounts; the single-account message is unchanged.
|
|
2000
|
+
const error = opts?.allAccounts
|
|
2001
|
+
? `All connected ChatGPT/Codex subscriptions are rate-limited. Access returns ${humanizeResetWindow(info.resetsInSeconds)}. `
|
|
2002
|
+
+ `You can switch this session to a different model in the meantime, or wait for a subscription to reset.`
|
|
2003
|
+
: `Your ChatGPT/Codex subscription usage limit has been reached. Access resets ${humanizeResetWindow(info.resetsInSeconds)}. `
|
|
2004
|
+
+ `You can switch this session to a different model in the meantime, or wait for the limit to reset.`;
|
|
2005
|
+
return {
|
|
2006
|
+
error,
|
|
2007
|
+
code: "codex_usage_limit_reached",
|
|
2008
|
+
retryable: false,
|
|
2009
|
+
...(detail ? { detail } : {}),
|
|
2010
|
+
};
|
|
2011
|
+
}
|
|
2012
|
+
|
|
2013
|
+
// A usage cap that won't reset for a long time should not pin a Temporal timer
|
|
2014
|
+
// open indefinitely for a goal-bearing session; cap the continuation hold so the
|
|
2015
|
+
// goal re-evaluates at most this far out (it will re-pause if still capped).
|
|
2016
|
+
const CODEX_USAGE_LIMIT_MAX_RESUME_MS = 60 * 60_000; // 1h
|
|
2017
|
+
|
|
2018
|
+
/**
|
|
2019
|
+
* Recognize an SDK stream event that represents a COMPUTER-USE tool call actually
|
|
2020
|
+
* executing — a `run_item_stream_event` carrying a `tool_call_item` whose
|
|
2021
|
+
* underlying raw item is a `computer_call` (the @openai/agents computer tool's
|
|
2022
|
+
* action: screenshot/click/type/scroll/drag/keypress/move/wait/…). This is the
|
|
2023
|
+
* same shape `normalizeSdkEvent` reads (`event.item.rawItem`), matched here against
|
|
2024
|
+
* the raw SDK type rather than a tool NAME so it is robust to the computer tool's
|
|
2025
|
+
* configured name. Drives the on-turn recording gate (no computer-use → discard).
|
|
2026
|
+
*/
|
|
2027
|
+
function isComputerCallStreamEvent(event: unknown): boolean {
|
|
2028
|
+
if (!event || typeof event !== "object") {
|
|
2029
|
+
return false;
|
|
2030
|
+
}
|
|
2031
|
+
if ((event as { type?: unknown }).type !== "run_item_stream_event") {
|
|
2032
|
+
return false;
|
|
2033
|
+
}
|
|
2034
|
+
const item = (event as { item?: { type?: unknown; rawItem?: { type?: unknown } } }).item;
|
|
2035
|
+
if (!item || item.type !== "tool_call_item") {
|
|
2036
|
+
return false;
|
|
2037
|
+
}
|
|
2038
|
+
return item.rawItem?.type === "computer_call";
|
|
2039
|
+
}
|
|
2040
|
+
|
|
2041
|
+
/**
|
|
2042
|
+
* Budget/limit exhaustion detected between model calls. This is account
|
|
2043
|
+
* state, not an agent failure: the segment ends gracefully (session idles,
|
|
2044
|
+
* run state preserved) so a top-up or limit reset lets the same session
|
|
2045
|
+
* continue — a failed session would reject the user's next message. An
|
|
2046
|
+
* active goal pauses visibly (reason "limits") at the next continuation
|
|
2047
|
+
* evaluation without consuming continuation budget.
|
|
2048
|
+
*/
|
|
2049
|
+
class BudgetExhaustedError extends Error {
|
|
2050
|
+
constructor(message: string, readonly serializedRunState: string | null) {
|
|
2051
|
+
super(message);
|
|
2052
|
+
this.name = "BudgetExhaustedError";
|
|
2053
|
+
}
|
|
2054
|
+
}
|
|
2055
|
+
|
|
2056
|
+
// Exported for unit testing the codex-billed bypass (codex-billing.test.ts); not part
|
|
2057
|
+
// of the activity surface. Takes BOTH `isCodexTurn` (codex-plan turns bypass the credit
|
|
2058
|
+
// and token gates) and the optional §7.5 P3 host `entitlements` port (when bound, its
|
|
2059
|
+
// `admitRun` REPLACES the local credit read for a non-codex turn; unset → local ledger).
|
|
2060
|
+
export async function ensureRunAllowed(
|
|
2061
|
+
settings: Settings,
|
|
2062
|
+
db: ActivityServices["db"],
|
|
2063
|
+
accountId: string,
|
|
2064
|
+
workspaceId: string,
|
|
2065
|
+
isCodexTurn: boolean,
|
|
2066
|
+
entitlements?: ActivityServices["entitlements"],
|
|
2067
|
+
): Promise<void> {
|
|
2068
|
+
// Codex-billed turns are paid by the user's ChatGPT/Codex plan: skip the
|
|
2069
|
+
// credit-balance gate and the monthly token cap. The agent-run COUNT cap below
|
|
2070
|
+
// is a volume/fairness quota (not a credit/cost gate) and is intentionally kept.
|
|
2071
|
+
//
|
|
2072
|
+
// §7.5 P3 — host-entitlements DELEGATION (the worker half of the same seam the
|
|
2073
|
+
// API edge exposes). For a non-codex turn, when the host binds `entitlements`, its
|
|
2074
|
+
// `admitRun` decision REPLACES the local credit-balance read below: a host that owns
|
|
2075
|
+
// its ledger/meter is the funding authority. A deny throws the SAME Error the local
|
|
2076
|
+
// read throws, so the mid-stream budget-valve at :727 wraps it in a
|
|
2077
|
+
// `BudgetExhaustedError` and pauses identically — the valve never learns whether the
|
|
2078
|
+
// deny came from the local ledger or the host meter.
|
|
2079
|
+
//
|
|
2080
|
+
// This is an admission READ only; it records NO usage (metering stays the sole,
|
|
2081
|
+
// idempotency-keyed writer at recordModelUsageAndDebitCredits), so a PULL host meter
|
|
2082
|
+
// is consulted without ever double-charging.
|
|
2083
|
+
if (!isCodexTurn && entitlements && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
|
|
2084
|
+
const decision = await entitlements.admitRun({
|
|
2085
|
+
accountId,
|
|
2086
|
+
workspaceId,
|
|
2087
|
+
action: "agent_run:create",
|
|
2088
|
+
quantity: 1,
|
|
2089
|
+
});
|
|
2090
|
+
if (!decision.allowed) {
|
|
2091
|
+
throw new Error(decision.reason || "insufficient OpenGeni credits");
|
|
2092
|
+
}
|
|
2093
|
+
} else if (!isCodexTurn && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
|
|
2094
|
+
const balance = await getBillingBalance(db, accountId);
|
|
2095
|
+
if (balance.balanceMicros <= 0) {
|
|
2096
|
+
throw new Error("insufficient OpenGeni credits");
|
|
2097
|
+
}
|
|
2098
|
+
}
|
|
2099
|
+
if (settings.usageLimitsMode === "static" || settings.usageLimitsMode === "managed") {
|
|
2100
|
+
const limits = configuredStaticUsageLimits(settings);
|
|
2101
|
+
if (limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
2102
|
+
const used = await sumUsageQuantity(db, {
|
|
2103
|
+
workspaceId,
|
|
2104
|
+
eventType: "agent_run.created",
|
|
2105
|
+
since: startOfUtcMonth(),
|
|
2106
|
+
});
|
|
2107
|
+
// Agent turns are admitted and recorded before this worker activity starts.
|
|
2108
|
+
// Equality means this accepted turn is exactly at the cap; greater-than is
|
|
2109
|
+
// the race/backstop case where another admission already exceeded the cap.
|
|
2110
|
+
if (used > limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
2111
|
+
throw new Error(`monthly agent run limit reached (${limits.maxMonthlyAgentRunsPerWorkspace})`);
|
|
2112
|
+
}
|
|
2113
|
+
}
|
|
2114
|
+
if (!isCodexTurn && limits.maxMonthlyTokensPerWorkspace) {
|
|
2115
|
+
const used = await sumUsageQuantity(db, {
|
|
2116
|
+
workspaceId,
|
|
2117
|
+
eventType: "model.tokens",
|
|
2118
|
+
since: startOfUtcMonth(),
|
|
2119
|
+
});
|
|
2120
|
+
if (used >= limits.maxMonthlyTokensPerWorkspace) {
|
|
2121
|
+
throw new Error(`monthly token limit reached (${limits.maxMonthlyTokensPerWorkspace})`);
|
|
2122
|
+
}
|
|
2123
|
+
}
|
|
2124
|
+
}
|
|
2125
|
+
}
|
|
2126
|
+
|
|
2127
|
+
// Exported for unit testing the codex-billed bypass; not part of the activity surface.
|
|
2128
|
+
export async function recordModelUsageAndDebitCredits(settings: Settings, db: ActivityServices["db"], input: {
|
|
2129
|
+
accountId: string;
|
|
2130
|
+
workspaceId: string;
|
|
2131
|
+
sessionId: string;
|
|
2132
|
+
turnId: string;
|
|
2133
|
+
model: string;
|
|
2134
|
+
isCodexTurn: boolean;
|
|
2135
|
+
usage?: ModelUsageInput | null;
|
|
2136
|
+
sourceKey: string;
|
|
2137
|
+
}): Promise<void> {
|
|
2138
|
+
if (!input.usage) {
|
|
2139
|
+
return;
|
|
2140
|
+
}
|
|
2141
|
+
const inputTokens = positiveInt(input.usage.inputTokens);
|
|
2142
|
+
const outputTokens = positiveInt(input.usage.outputTokens);
|
|
2143
|
+
const totalTokens = positiveInt(input.usage.totalTokens) || inputTokens + outputTokens;
|
|
2144
|
+
// A codex-subscription turn is paid by the user's ChatGPT/Codex plan, so it
|
|
2145
|
+
// consumes ZERO OpenGeni credits and must never feed an OpenGeni cap. A
|
|
2146
|
+
// codex/<slug> model has no entry in configuredModelPricing, so the normal path
|
|
2147
|
+
// below would throw "Missing model pricing". We:
|
|
2148
|
+
// - do NOT emit the cap-feeding `model.tokens` event (ensureRunAllowed and
|
|
2149
|
+
// the API tokens:consume cap sum `model.tokens` with NO cost dimension, so
|
|
2150
|
+
// any row would count against maxMonthlyTokensPerWorkspace);
|
|
2151
|
+
// - record a `model.cost = 0` audit marker (harmless to the monthly cost cap);
|
|
2152
|
+
// - never look up pricing and never debit credits.
|
|
2153
|
+
if (input.isCodexTurn) {
|
|
2154
|
+
await recordUsageEvent(db, {
|
|
2155
|
+
accountId: input.accountId,
|
|
2156
|
+
workspaceId: input.workspaceId,
|
|
2157
|
+
eventType: "model.cost",
|
|
2158
|
+
quantity: 0,
|
|
2159
|
+
unit: "usd_micros",
|
|
2160
|
+
sourceResourceType: "model_response",
|
|
2161
|
+
sourceResourceId: `${input.turnId}:${input.sourceKey}`,
|
|
2162
|
+
idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`,
|
|
2163
|
+
});
|
|
2164
|
+
return;
|
|
2165
|
+
}
|
|
2166
|
+
if (totalTokens > 0) {
|
|
2167
|
+
await recordUsageEvent(db, {
|
|
2168
|
+
accountId: input.accountId,
|
|
2169
|
+
workspaceId: input.workspaceId,
|
|
2170
|
+
eventType: "model.tokens",
|
|
2171
|
+
quantity: totalTokens,
|
|
2172
|
+
unit: "tokens",
|
|
2173
|
+
sourceResourceType: "model_response",
|
|
2174
|
+
sourceResourceId: `${input.turnId}:${input.sourceKey}`,
|
|
2175
|
+
idempotencyKey: `usage:model.tokens:${input.turnId}:${input.sourceKey}`,
|
|
2176
|
+
});
|
|
2177
|
+
}
|
|
2178
|
+
const shouldDebit = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
|
|
2179
|
+
if (!shouldDebit || totalTokens === 0) {
|
|
2180
|
+
return;
|
|
2181
|
+
}
|
|
2182
|
+
if (!configuredModelPricing(settings)[input.model]) {
|
|
2183
|
+
throw new Error(`Missing model pricing for ${input.model}`);
|
|
2184
|
+
}
|
|
2185
|
+
const costMicros = calculateModelUsageCostMicros(settings, input.model, input.usage);
|
|
2186
|
+
await recordUsageEvent(db, {
|
|
2187
|
+
accountId: input.accountId,
|
|
2188
|
+
workspaceId: input.workspaceId,
|
|
2189
|
+
eventType: "model.cost",
|
|
2190
|
+
quantity: costMicros,
|
|
2191
|
+
unit: "usd_micros",
|
|
2192
|
+
sourceResourceType: "model_response",
|
|
2193
|
+
sourceResourceId: `${input.turnId}:${input.sourceKey}`,
|
|
2194
|
+
idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`,
|
|
2195
|
+
});
|
|
2196
|
+
if (costMicros > 0) {
|
|
2197
|
+
await applyCreditDebitUpToBalance(db, {
|
|
2198
|
+
accountId: input.accountId,
|
|
2199
|
+
workspaceId: input.workspaceId,
|
|
2200
|
+
type: "model_usage_debit",
|
|
2201
|
+
requestedAmountMicros: costMicros,
|
|
2202
|
+
sourceType: "model_response",
|
|
2203
|
+
sourceId: `${input.turnId}:${input.sourceKey}`,
|
|
2204
|
+
idempotencyKey: `credit:model_usage_debit:${input.turnId}:${input.sourceKey}`,
|
|
2205
|
+
metadata: {
|
|
2206
|
+
model: input.model,
|
|
2207
|
+
sessionId: input.sessionId,
|
|
2208
|
+
turnId: input.turnId,
|
|
2209
|
+
sourceKey: input.sourceKey,
|
|
2210
|
+
inputTokens,
|
|
2211
|
+
outputTokens,
|
|
2212
|
+
totalTokens,
|
|
2213
|
+
},
|
|
2214
|
+
});
|
|
2215
|
+
}
|
|
2216
|
+
}
|
|
2217
|
+
|
|
2218
|
+
function positiveInt(value: unknown): number {
|
|
2219
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
|
|
2220
|
+
}
|
|
2221
|
+
|
|
2222
|
+
function startOfUtcMonth(): Date {
|
|
2223
|
+
const now = new Date();
|
|
2224
|
+
return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
|
|
2225
|
+
}
|
|
2226
|
+
|
|
2227
|
+
async function sandboxFileDownloadsForRun(
|
|
2228
|
+
settings: Settings,
|
|
2229
|
+
db: ActivityServices["db"],
|
|
2230
|
+
objectStorage: ObjectStorage | null,
|
|
2231
|
+
workspaceId: string,
|
|
2232
|
+
resources: ResourceRef[],
|
|
2233
|
+
): Promise<SandboxFileDownload[]> {
|
|
2234
|
+
if (settings.sandboxBackend === "none" || !requiresSignedFileResourceDownloads(settings)) {
|
|
2235
|
+
return [];
|
|
2236
|
+
}
|
|
2237
|
+
const fileResources = resources.filter((resource): resource is Extract<ResourceRef, { kind: "file" }> => resource.kind === "file");
|
|
2238
|
+
if (fileResources.length === 0) {
|
|
2239
|
+
return [];
|
|
2240
|
+
}
|
|
2241
|
+
if (!objectStorage) {
|
|
2242
|
+
throw new Error(`${settings.objectStorageBackend} file resources require configured object storage`);
|
|
2243
|
+
}
|
|
2244
|
+
const downloadStorage = objectStorageForSandboxDownloads(settings, objectStorage);
|
|
2245
|
+
const downloads: SandboxFileDownload[] = [];
|
|
2246
|
+
for (const resource of fileResources) {
|
|
2247
|
+
const file = await requireFile(db, workspaceId, resource.fileId);
|
|
2248
|
+
const url = await downloadStorage.createGetUrl({ key: file.objectKey });
|
|
2249
|
+
downloads.push({
|
|
2250
|
+
fileId: file.id,
|
|
2251
|
+
mountPath: resource.mountPath ?? `files/${file.id}`,
|
|
2252
|
+
filename: file.safeFilename,
|
|
2253
|
+
url: url.url,
|
|
2254
|
+
expiresAt: url.expiresAt,
|
|
2255
|
+
sizeBytes: file.sizeBytes,
|
|
2256
|
+
});
|
|
2257
|
+
}
|
|
2258
|
+
return downloads;
|
|
2259
|
+
}
|
|
2260
|
+
|
|
2261
|
+
function requiresSignedFileResourceDownloads(settings: Settings): boolean {
|
|
2262
|
+
// A selfhosted machine (bring-your-own-compute) can NEVER mount ANY object store
|
|
2263
|
+
// — it is a remote user machine reached only over NATS, so file resources are
|
|
2264
|
+
// ALWAYS delivered by exec-curling a pre-signed URL onto it. Without this a
|
|
2265
|
+
// machine-home turn (sandbox_backend "selfhosted") would silently drop file
|
|
2266
|
+
// resources on an azure-blob / s3-compatible store (nativeBucketMount=false),
|
|
2267
|
+
// a regression from the pre-honest-label path where the same turn ran home=modal
|
|
2268
|
+
// and modal's descriptor forced signed downloads.
|
|
2269
|
+
if (settings.sandboxBackend === "selfhosted") {
|
|
2270
|
+
return true;
|
|
2271
|
+
}
|
|
2272
|
+
// A nativeBucketMount backend (modal) cannot mount Azure Blob entries, so it
|
|
2273
|
+
// needs pre-signed downloads for that store. Keying on the descriptor (not the
|
|
2274
|
+
// "modal" literal) keeps this correct as bucket-mount backends are added.
|
|
2275
|
+
const nativeBucketMount = CAPABILITY_DESCRIPTORS[settings.sandboxBackend].nativeBucketMount;
|
|
2276
|
+
return (settings.sandboxBackend === "docker" && settings.objectStorageBackend === "s3-compatible")
|
|
2277
|
+
|| settings.objectStorageBackend === "aws-s3"
|
|
2278
|
+
|| settings.objectStorageBackend === "gcs"
|
|
2279
|
+
|| (nativeBucketMount && settings.objectStorageBackend === "azure-blob");
|
|
2280
|
+
}
|
|
2281
|
+
|
|
2282
|
+
function objectStorageForSandboxDownloads(settings: Settings, objectStorage: ObjectStorage): ObjectStorage {
|
|
2283
|
+
if (settings.objectStorageBackend !== "s3-compatible" || !settings.objectStorageSandboxEndpoint) {
|
|
2284
|
+
return objectStorage;
|
|
2285
|
+
}
|
|
2286
|
+
return createObjectStorage({
|
|
2287
|
+
...settings,
|
|
2288
|
+
objectStorageEndpoint: settings.objectStorageSandboxEndpoint,
|
|
2289
|
+
}) ?? objectStorage;
|
|
2290
|
+
}
|