@opengeni/worker-bundle 0.2.3 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -49,13 +49,15 @@ import {
49
49
  normalizeSdkEvent,
50
50
  sanitizeHistoryItemsForModel,
51
51
  summarizeForCompaction,
52
+ findCompactionNeededError,
53
+ SUMMARY_BUFFER_TOKENS,
52
54
  type SandboxFileDownload,
53
55
  type OpenGeniRuntime,
54
56
  type ComputerToolMode,
55
57
  } from "@opengeni/runtime";
56
58
  import { calculateModelUsageCostMicros, configuredModelPricing, configuredStaticUsageLimits, sandboxWarmRateMicrosPerSecond, type ModelUsageInput, type ModelProviderApi, type RegistryProviderKind, type Settings } from "@opengeni/config";
57
59
  import { CancelledFailure } from "@temporalio/activity";
58
- import { settingsWithCodexCredential, settingsWithEnabledCapabilityMcpServers } from "./capabilities";
60
+ import { settingsWithCodexCredential, settingsWithEnabledCapabilityMcpServers, settingsWithSessionMcpServersForRun } from "./capabilities";
59
61
  import { chooseRotationActive, computeIdleDelayMs, computeReactiveRotationResume, type CodexRotationStrategy, type RotationDecision } from "./codex-rotation";
60
62
  import type { CodexAccountStatus } from "@opengeni/db";
61
63
  import { buildCodexTokenResolver } from "./codex-auth";
@@ -131,6 +133,14 @@ export function selectCodexCredentialForTurn(args: {
131
133
  // agent must not blindly replay side effects it cannot see in its history:
132
134
  // at most the single in-flight model step was lost, and anything it started
133
135
  // there may or may not have completed.
136
+ // Resume notice after an in-turn context-window overflow was recovered by
137
+ // compaction: the turn's earlier work survives (albeit summarized), so the
138
+ // agent continues instead of the session dying or stalling idle.
139
+ export const CONTEXT_OVERFLOW_RESUME_TEXT = [
140
+ "[TURN RESUMED AFTER CONTEXT COMPACTION] The conversation approached or exceeded the model's context window mid-turn; older history above was compacted into a summary.",
141
+ "Continue the original task from where it left off. Before repeating any action with side effects, check whether it already happened.",
142
+ ].join("\n");
143
+
134
144
  export const WORKER_SHUTDOWN_RESUME_TEXT = [
135
145
  "[TURN RESUMED AFTER WORKER RESTART] The platform worker running this turn restarted (deploy/rollout) before the turn finished.",
136
146
  "Your conversation history above, including this turn's earlier work, is preserved up to the last checkpoint; at most the single in-flight model step after it was lost.",
@@ -158,6 +168,54 @@ export function isWorkerShutdownCancellation(error: unknown): boolean {
158
168
  return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
159
169
  }
160
170
 
171
+ export const CONTEXT_WINDOW_OVERFLOW_RECOVERY_MESSAGE =
172
+ "The conversation reached the model context compaction threshold; it has been compacted. Send a message to continue.";
173
+
174
+ export function classifyContextWindowOverflowError(error: unknown): { message: string; code?: string; detail?: string } | null {
175
+ const fields = collectErrorStrings(error);
176
+ const matched = fields.find((value) =>
177
+ /context[_\s-]*length[_\s-]*exceeded/i.test(value)
178
+ || /exceeds?\s+(?:the\s+)?context\s+window/i.test(value)
179
+ || /maximum\s+context\s+length/i.test(value)
180
+ || /context\s+window[^.]*exceed/i.test(value)
181
+ );
182
+ if (!matched) {
183
+ return null;
184
+ }
185
+ const message = error instanceof Error ? error.message : String(error);
186
+ const code = fields.find((value) => /context[_\s-]*length[_\s-]*exceeded/i.test(value));
187
+ return {
188
+ message,
189
+ ...(code ? { code } : {}),
190
+ ...(matched && matched !== message ? { detail: matched } : {}),
191
+ };
192
+ }
193
+
194
+ function collectErrorStrings(value: unknown, seen = new WeakSet<object>()): string[] {
195
+ if (typeof value === "string") {
196
+ return [value];
197
+ }
198
+ if (!value || typeof value !== "object") {
199
+ return [];
200
+ }
201
+ if (seen.has(value)) {
202
+ return [];
203
+ }
204
+ seen.add(value);
205
+ const out: string[] = [];
206
+ const record = value as Record<string, unknown>;
207
+ for (const key of ["message", "code", "type", "name", "param"]) {
208
+ const field = record[key];
209
+ if (typeof field === "string" && field.length > 0) {
210
+ out.push(field);
211
+ }
212
+ }
213
+ for (const key of ["error", "cause", "response", "data"]) {
214
+ out.push(...collectErrorStrings(record[key], seen));
215
+ }
216
+ return out;
217
+ }
218
+
161
219
  /**
162
220
  * Compute the conversation-truth rows a reconcile pass should append, given the
163
221
  * SDK's current `state.history` and the count already persisted.
@@ -236,6 +294,19 @@ export function historyRowsToAppend(
236
294
  return { rows, nextWatermark: sanitized.length, nextPosition: nextPosition + rows.length };
237
295
  }
238
296
 
297
+ function isModelOrToolProgressHistoryItem(item: Record<string, unknown>): boolean {
298
+ if (item.type === "message") {
299
+ return item.role === "assistant";
300
+ }
301
+ if (item.type === "reasoning" || item.type === "compaction") {
302
+ return true;
303
+ }
304
+ if (typeof item.type === "string") {
305
+ return item.type !== "message";
306
+ }
307
+ return false;
308
+ }
309
+
239
310
  /**
240
311
  * Seed the turn-end reconcile watermark (`persistedHistoryCount`) from EXACTLY the
241
312
  * view `state.history` was seeded from, so the model-input length and the watermark
@@ -474,6 +545,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
474
545
  // static frame of an untouched desktop is not worth uploading).
475
546
  let didComputerUse = false;
476
547
  let batcher: ReturnType<typeof createRuntimeBatcher> | null = null;
548
+ const flushRuntimeBatcher = async () => {
549
+ const current = batcher as ReturnType<typeof createRuntimeBatcher> | null;
550
+ await current?.flush().catch(() => undefined);
551
+ };
477
552
  let preparedTools: Awaited<ReturnType<OpenGeniRuntime["prepareTools"]>> | null = null;
478
553
  let publish: ((events: Array<Omit<AppendEventInput, "producerId" | "producerSeq" | "turnId">>, immediate?: boolean) => Promise<void>) | null = null;
479
554
  let turnStartedPublished = false;
@@ -502,12 +577,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
502
577
  // actually persisted, so a non-monotonic history can never desync it.
503
578
  let persistedHistoryCount = 0;
504
579
  let historyCountAtTurnStart = 0;
580
+ let modelOrToolProgressPersisted = false;
505
581
  // Next free WHOLE-NUMBER absolute position to append at. Tracked separately
506
582
  // from persistedHistoryCount (the in-memory slice index) because a compaction
507
583
  // inserts a fractional summary position, so total rows no longer equal
508
584
  // max(position)+1 and the slice index can no longer double as the position.
509
585
  let nextHistoryPosition = 0;
510
- const reconcileConversationTruth = async () => {
586
+ const reconcileConversationTruth = async (options: { skipInputOnlyRows?: boolean } = {}) => {
511
587
  if (!stream || !turnId) {
512
588
  return;
513
589
  }
@@ -519,7 +595,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
519
595
  persistedHistoryCount,
520
596
  nextHistoryPosition,
521
597
  );
522
- if (rows.length > 0) {
598
+ const hasModelOrToolProgress = rows.some((row) => isModelOrToolProgressHistoryItem(row.item));
599
+ const shouldAppendRows = rows.length > 0 && (!options.skipInputOnlyRows || hasModelOrToolProgress);
600
+ if (shouldAppendRows) {
523
601
  await appendSessionHistoryItems(db, {
524
602
  accountId: input.accountId,
525
603
  workspaceId: input.workspaceId,
@@ -533,8 +611,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
533
611
  items: rows,
534
612
  });
535
613
  }
536
- persistedHistoryCount = nextWatermark;
537
- nextHistoryPosition = nextPosition;
614
+ if (shouldAppendRows || !options.skipInputOnlyRows) {
615
+ persistedHistoryCount = nextWatermark;
616
+ nextHistoryPosition = nextPosition;
617
+ }
618
+ if (hasModelOrToolProgress) {
619
+ modelOrToolProgressPersisted = true;
620
+ }
538
621
  }
539
622
  const envelope = sandboxStateEntryFromRunState(stream.state);
540
623
  if (envelope) {
@@ -802,12 +885,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
802
885
  // runtime falls back to runSettings.agentInstructionsTemplate (the
803
886
  // deployment default, byte-identical to the historical preamble).
804
887
  const workspaceAgentInstructions = await resolveWorkspaceAgentInstructions(db, input.workspaceId);
805
- const runSettings = {
888
+ const baseRunSettings = {
806
889
  ...settingsWithPackSandboxImage(capabilitySettings, packRuntime.sandboxImage),
807
890
  openaiModel: turn.model,
808
891
  openaiReasoningEffort: turn.reasoningEffort,
809
892
  sandboxBackend: turn.sandboxBackend,
810
893
  };
894
+ const runSettings = await settingsWithSessionMcpServersForRun(db, input.workspaceId, input.sessionId, baseRunSettings);
811
895
  // Multi-provider per-turn routing → the provider gating (compaction mode,
812
896
  // hosted web search, encrypted reasoning, context window) the agent and
813
897
  // compaction summarizer must use; null falls back to the legacy global
@@ -1261,11 +1345,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1261
1345
  }
1262
1346
  : {}),
1263
1347
  });
1348
+ const compactSummarizer = resolvedModel
1349
+ ? (s: Settings, m: Array<Record<string, unknown>>) => withCodex(() => summarizeForCompaction(s, m, {
1350
+ client: resolvedModel.client,
1351
+ api: resolvedModel.provider.api,
1352
+ model: resolvedModel.configured.id,
1353
+ maxOutputTokens: SUMMARY_BUFFER_TOKENS,
1354
+ }))
1355
+ : undefined;
1264
1356
  // Pre-turn client-side context compaction (Azure path). When the
1265
- // resolved mode is "client" and the last turn's input tokens crossed the
1266
- // soft budget, this summarizes the orphan-safe old prefix into one user
1267
- // message and supersedes the summarized rows BEFORE the history is read
1268
- // for this turn, so the model sees [summary, ...recent tail] + new input.
1357
+ // resolved mode is "client" and the single Codex-parity threshold is
1358
+ // crossed, this summarizes the current active history and rebuilds active
1359
+ // history as [user messages..., summary] BEFORE the model input is read.
1269
1360
  // Best-effort: a no-op or failure leaves the un-compacted history intact.
1270
1361
  // Skipped on approval/preempt resumes (no fresh user turn; the frozen
1271
1362
  // RunState or in-flight tail must replay verbatim).
@@ -1285,22 +1376,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1285
1376
  // registry provider, summarize on THAT provider's client + wire API
1286
1377
  // (a chat provider can't summarize through OpenAI/Azure). Null
1287
1378
  // resolution keeps the default built-in Responses summarizer.
1288
- resolvedModel
1289
- ? (s, m) => withCodex(() => summarizeForCompaction(s, m, {
1290
- client: resolvedModel.client,
1291
- api: resolvedModel.provider.api,
1292
- model: resolvedModel.configured.id,
1293
- maxOutputTokens: s.contextSummaryMaxTokens,
1294
- }))
1295
- : undefined,
1379
+ compactSummarizer,
1296
1380
  forced ? { force: true } : {},
1297
1381
  );
1298
1382
  if (outcome.compacted) {
1299
- // trigger precedence: an explicit operator /compact wins for the
1300
- // event label; otherwise a hard-forced (at/over hardFraction*B)
1301
- // compaction is surfaced as "hard" so the backstop firing is
1302
- // observable downstream, and the default soft trigger is implicit.
1303
- const trigger = forced ? "operator" : outcome.hardForced ? "hard" : undefined;
1383
+ const trigger = forced ? "operator" : undefined;
1304
1384
  await publish([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, ...(trigger ? { trigger } : {}) } }]);
1305
1385
  }
1306
1386
  } catch (compactError) {
@@ -1315,226 +1395,385 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1315
1395
  // "current account" the single strip rule compares against (null is the
1316
1396
  // built-in/Azure account, so a non-codex turn still drops codex-produced
1317
1397
  // reasoning, and a no-codex-history session is a byte-for-byte no-op).
1318
- const { input: runInput, modelHistoryFromItems } = await turnInput(
1319
- db,
1320
- runtime,
1321
- agent,
1322
- trigger,
1323
- runSettings,
1324
- { currentCodexCredentialId: effectiveCodexCredentialId },
1325
- );
1326
- // Slice index = the length of the model-facing (active) history this turn
1327
- // is seeded from; new items beyond it (the trigger message + this turn's
1328
- // generated items) are the ones to persist. After a compaction this is the
1329
- // short [summary, ...tail] active set, NOT the total row count. The
1330
- // absolute write position is tracked separately (next whole number past
1331
- // the max existing position) because the fractional summary row means
1332
- // total rows no longer equal max(position)+1. Pre-compaction both reduce to
1333
- // the old total-count value, so the common path is unchanged.
1334
- //
1335
- // CRITICAL: seed from the SANITIZED active-row length, not the raw active
1336
- // count. `prepareRunInput` builds `state.history` from
1337
- // `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
1338
- // rows (a legacy orphan/dangling pair), the in-memory history this turn
1339
- // starts from is K shorter than the raw row count. The reconcile slices the
1340
- // re-sanitized `state.history` off `persistedHistoryCount`; seeding it from
1341
- // the raw count (K too high) skips K genuinely-new items, and a
1342
- // `function_call` left in that skipped region can later have its
1343
- // `function_call_result` persisted alone — the orphan that 400s on replay
1344
- // and bricks the session (issue-61). The sanitized seed is already
1345
- // orphan-free, so it is a stable prefix of the re-sanitized history and the
1346
- // slice begins exactly at the first genuinely-new item.
1347
- const activeSeedRows = await getActiveSessionHistoryItems(db, input.workspaceId, input.sessionId);
1348
- // Seed the reconcile watermark from EXACTLY the view the model's
1349
- // `state.history` was seeded from (items strip on the items path = HOLE D; NO
1350
- // strip on the run-state blob path, where foreign reasoning is neutralized but
1351
- // KEPT = HOLE E), so the model-input length and the watermark never disagree.
1352
- persistedHistoryCount = reconcileSeedCount(
1353
- activeSeedRows,
1354
- modelHistoryFromItems,
1355
- { currentCodexCredentialId: effectiveCodexCredentialId },
1356
- );
1357
- historyCountAtTurnStart = persistedHistoryCount;
1358
- nextHistoryPosition = await nextSessionHistoryPosition(db, input.workspaceId, input.sessionId);
1359
- let responseUsageCount = 0;
1360
- // Actual input tokens of the most recent model response this turn; the
1361
- // pre-read trigger for the NEXT turn. Persisted at every turn-end path.
1362
- let lastInputTokensObserved: number | null = null;
1363
- throwIfWorkerShuttingDown();
1364
- const runStreamOnce = (): ReturnType<OpenGeniRuntime["runStream"]> => runtime.runStream(agent, runInput, runSettings, {
1365
- sandboxEnvironment,
1366
- onRuntimeEvent: async (event) => {
1367
- await publish!([{ type: event.type, payload: event.payload }], true);
1368
- },
1369
- // P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
1370
- // keystone). Absent when the flag is off -> legacy build-and-discard.
1371
- ...(resolvedSandbox
1372
- ? {
1373
- ownedSandbox: {
1374
- client: resolvedSandbox.established.client,
1375
- session: resolvedSandbox.established.session,
1376
- sessionState: resolvedSandbox.established.sessionState,
1377
- // Pin platform setup (hooks + file materialization) to the un-proxied
1378
- // established box — never through the routing proxy, which would
1379
- // re-route those execs onto a machine swapped in mid-turn.
1380
- ...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
1381
- },
1382
- }
1383
- : {}),
1384
- });
1385
- stream = await withCodex(runStreamOnce);
1386
- batcher = createRuntimeBatcher(async (events) => {
1387
- await publish!(events);
1388
- });
1398
+ const activeTurnId = turnId;
1399
+ if (!activeTurnId) {
1400
+ throw new Error("Turn id was not initialized");
1401
+ }
1402
+ let runInput: Awaited<ReturnType<typeof turnInput>>["input"] | null = null;
1403
+ const prepareRunAttemptInput = async () => {
1404
+ const prepared = await turnInput(
1405
+ db,
1406
+ runtime,
1407
+ agent,
1408
+ trigger,
1409
+ runSettings,
1410
+ { currentCodexCredentialId: effectiveCodexCredentialId },
1411
+ );
1412
+ runInput = prepared.input;
1413
+ // Slice index = the length of the model-facing (active) history this turn
1414
+ // is seeded from; new items beyond it (the trigger message + this turn's
1415
+ // generated items) are the ones to persist. After a compaction this is the
1416
+ // short [summary, ...tail] active set, NOT the total row count. The
1417
+ // absolute write position is tracked separately (next whole number past
1418
+ // the max existing position) because the fractional summary row means
1419
+ // total rows no longer equal max(position)+1. Pre-compaction both reduce to
1420
+ // the old total-count value, so the common path is unchanged.
1421
+ //
1422
+ // CRITICAL: seed from the SANITIZED active-row length, not the raw active
1423
+ // count. `prepareRunInput` builds `state.history` from
1424
+ // `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
1425
+ // rows (a legacy orphan/dangling pair), the in-memory history this turn
1426
+ // starts from is K shorter than the raw row count. The reconcile slices the
1427
+ // re-sanitized `state.history` off `persistedHistoryCount`; seeding it from
1428
+ // the raw count (K too high) skips K genuinely-new items, and a
1429
+ // `function_call` left in that skipped region can later have its
1430
+ // `function_call_result` persisted alone — the orphan that 400s on replay
1431
+ // and bricks the session (issue-61). The sanitized seed is already
1432
+ // orphan-free, so it is a stable prefix of the re-sanitized history and the
1433
+ // slice begins exactly at the first genuinely-new item.
1434
+ const activeSeedRows = await getActiveSessionHistoryItems(db, input.workspaceId, input.sessionId);
1435
+ // Seed the reconcile watermark from EXACTLY the view the model's
1436
+ // `state.history` was seeded from (items strip on the items path = HOLE D; NO
1437
+ // strip on the run-state blob path, where foreign reasoning is neutralized but
1438
+ // KEPT = HOLE E), so the model-input length and the watermark never disagree.
1439
+ persistedHistoryCount = reconcileSeedCount(
1440
+ activeSeedRows,
1441
+ prepared.modelHistoryFromItems,
1442
+ { currentCodexCredentialId: effectiveCodexCredentialId },
1443
+ );
1444
+ historyCountAtTurnStart = persistedHistoryCount;
1445
+ nextHistoryPosition = await nextSessionHistoryPosition(db, input.workspaceId, input.sessionId);
1446
+ };
1389
1447
 
1390
- const iterator = stream.toStream()[Symbol.asyncIterator]();
1391
- let streamDone = false;
1392
- try {
1393
- while (true) {
1394
- const next = await nextStreamEvent(iterator, activityContext);
1395
- if (next.done) {
1396
- streamDone = true;
1397
- break;
1398
- }
1399
- const responseUsage = modelResponseUsageFromSdkEvent(next.value);
1400
- if (responseUsage) {
1401
- responseUsageCount += 1;
1402
- const observed = responseUsage.usage?.inputTokens;
1403
- if (typeof observed === "number" && observed > 0) {
1404
- lastInputTokensObserved = observed;
1448
+ const forceContextCompaction = async (triggerLabel: "overflow" | "proactive") => {
1449
+ const clientCompactionSettings: Settings = { ...runSettings, contextCompactionMode: "client" };
1450
+ const outcome = await maybeCompactContext(
1451
+ db,
1452
+ clientCompactionSettings,
1453
+ { accountId: input.accountId, workspaceId: input.workspaceId, sessionId: input.sessionId, turnId: activeTurnId },
1454
+ session.lastInputTokens,
1455
+ compactSummarizer,
1456
+ { force: true },
1457
+ );
1458
+ if (outcome.compacted) {
1459
+ await publish!([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, trigger: triggerLabel } }]);
1460
+ }
1461
+ return outcome;
1462
+ };
1463
+
1464
+ const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
1465
+ if (!runInput) {
1466
+ throw new Error("Run input was not prepared");
1467
+ }
1468
+ stream = undefined;
1469
+ batcher = null;
1470
+ let responseUsageCount = 0;
1471
+ // Actual input tokens of the most recent model response this turn; the
1472
+ // pre-read trigger for the NEXT turn. Persisted at every turn-end path.
1473
+ let lastInputTokensObserved: number | null = null;
1474
+ throwIfWorkerShuttingDown();
1475
+ const runStreamOnce = (): ReturnType<OpenGeniRuntime["runStream"]> => runtime.runStream(agent, runInput!, runSettings, {
1476
+ sandboxEnvironment,
1477
+ onRuntimeEvent: async (event) => {
1478
+ await publish!([{ type: event.type, payload: event.payload }], true);
1479
+ },
1480
+ // P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
1481
+ // keystone). Absent when the flag is off -> legacy build-and-discard.
1482
+ ...(resolvedSandbox
1483
+ ? {
1484
+ ownedSandbox: {
1485
+ client: resolvedSandbox.established.client,
1486
+ session: resolvedSandbox.established.session,
1487
+ sessionState: resolvedSandbox.established.sessionState,
1488
+ // Pin platform setup (hooks + file materialization) to the un-proxied
1489
+ // established box — never through the routing proxy, which would
1490
+ // re-route those execs onto a machine swapped in mid-turn.
1491
+ ...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
1492
+ },
1405
1493
  }
1406
- await recordModelUsageAndDebitCredits(settings, db, {
1407
- accountId: input.accountId,
1408
- workspaceId: input.workspaceId,
1409
- sessionId: input.sessionId,
1410
- turnId,
1411
- model: turn.model,
1412
- isCodexTurn,
1413
- usage: responseUsage.usage,
1414
- sourceKey: modelUsageSourceKey({
1415
- responseId: responseUsage.responseId,
1416
- dispatchId,
1417
- positionalKey: `response-${responseUsageCount}`,
1418
- }),
1419
- });
1420
- await reconcileConversationTruth();
1421
- try {
1422
- await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
1423
- } catch (limitError) {
1424
- // Capture the run state at the boundary so the budget valve in
1425
- // the outer catch can end this segment gracefully with full
1426
- // conversation context preserved for the post-top-up resume.
1427
- let serializedRunState: string | null = null;
1494
+ : {}),
1495
+ contextCompactionSignalTokens: () => lastInputTokensObserved,
1496
+ });
1497
+ stream = await withCodex(runStreamOnce);
1498
+ batcher = createRuntimeBatcher(async (events) => {
1499
+ await publish!(events);
1500
+ });
1501
+
1502
+ const iterator = stream.toStream()[Symbol.asyncIterator]();
1503
+ let streamDone = false;
1504
+ try {
1505
+ while (true) {
1506
+ const next = await nextStreamEvent(iterator, activityContext);
1507
+ if (next.done) {
1508
+ streamDone = true;
1509
+ break;
1510
+ }
1511
+ const responseUsage = modelResponseUsageFromSdkEvent(next.value);
1512
+ if (responseUsage) {
1513
+ responseUsageCount += 1;
1514
+ const observed = responseUsage.usage?.inputTokens;
1515
+ if (typeof observed === "number" && observed > 0) {
1516
+ lastInputTokensObserved = observed;
1517
+ await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, observed)
1518
+ .catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
1519
+ }
1520
+ await recordModelUsageAndDebitCredits(settings, db, {
1521
+ accountId: input.accountId,
1522
+ workspaceId: input.workspaceId,
1523
+ sessionId: input.sessionId,
1524
+ turnId: activeTurnId,
1525
+ model: turn.model,
1526
+ isCodexTurn,
1527
+ usage: responseUsage.usage,
1528
+ sourceKey: modelUsageSourceKey({
1529
+ responseId: responseUsage.responseId,
1530
+ dispatchId,
1531
+ positionalKey: `response-${responseUsageCount}`,
1532
+ }),
1533
+ });
1534
+ await reconcileConversationTruth();
1428
1535
  try {
1429
- serializedRunState = stream.state.toString();
1430
- } catch {
1431
- serializedRunState = null;
1536
+ await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
1537
+ } catch (limitError) {
1538
+ // Capture the run state at the boundary so the budget valve in
1539
+ // the outer catch can end this segment gracefully with full
1540
+ // conversation context preserved for the post-top-up resume.
1541
+ let serializedRunState: string | null = null;
1542
+ try {
1543
+ serializedRunState = stream.state.toString();
1544
+ } catch {
1545
+ serializedRunState = null;
1546
+ }
1547
+ throw new BudgetExhaustedError(
1548
+ limitError instanceof Error ? limitError.message : String(limitError),
1549
+ serializedRunState,
1550
+ );
1432
1551
  }
1433
- throw new BudgetExhaustedError(
1434
- limitError instanceof Error ? limitError.message : String(limitError),
1435
- serializedRunState,
1436
- );
1552
+ }
1553
+ // Recording gate: a computer-use tool actually ran when an SDK
1554
+ // `tool_call_item` whose rawItem.type is "computer_call" streams through
1555
+ // (screenshot/click/type/scroll/etc. — the first computer action proves
1556
+ // the desktop was driven). Match the raw SDK event (ground truth) BEFORE
1557
+ // normalization. Only meaningful when a recording is live.
1558
+ if (activeRecording && !didComputerUse && isComputerCallStreamEvent(next.value)) {
1559
+ didComputerUse = true;
1560
+ }
1561
+ const normalized = normalizeSdkEvent(next.value);
1562
+ for (const event of normalized) {
1563
+ await batcher.push(event);
1437
1564
  }
1438
1565
  }
1439
- // Recording gate: a computer-use tool actually ran when an SDK
1440
- // `tool_call_item` whose rawItem.type is "computer_call" streams through
1441
- // (screenshot/click/type/scroll/etc. — the first computer action proves
1442
- // the desktop was driven). Match the raw SDK event (ground truth) BEFORE
1443
- // normalization. Only meaningful when a recording is live.
1444
- if (activeRecording && !didComputerUse && isComputerCallStreamEvent(next.value)) {
1445
- didComputerUse = true;
1566
+ } finally {
1567
+ if (!streamDone) {
1568
+ await iterator.return?.();
1446
1569
  }
1447
- const normalized = normalizeSdkEvent(next.value);
1448
- for (const event of normalized) {
1449
- await batcher.push(event);
1570
+ }
1571
+ await batcher.flush();
1572
+ await stream.completed.catch(() => undefined);
1573
+ if (responseUsageCount === 0) {
1574
+ const aggregateInput = (stream.state.usage as { inputTokens?: unknown } | undefined)?.inputTokens;
1575
+ if (typeof aggregateInput === "number" && aggregateInput > 0) {
1576
+ lastInputTokensObserved = aggregateInput;
1450
1577
  }
1578
+ await recordModelUsageAndDebitCredits(settings, db, {
1579
+ accountId: input.accountId,
1580
+ workspaceId: input.workspaceId,
1581
+ sessionId: input.sessionId,
1582
+ turnId: activeTurnId,
1583
+ model: turn.model,
1584
+ isCodexTurn,
1585
+ usage: stream.state.usage,
1586
+ sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
1587
+ });
1451
1588
  }
1452
- } finally {
1453
- if (!streamDone) {
1454
- await iterator.return?.();
1589
+ if (lastInputTokensObserved !== null) {
1590
+ await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, lastInputTokensObserved)
1591
+ .catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
1455
1592
  }
1456
- }
1457
- await batcher.flush();
1458
- await stream.completed.catch(() => undefined);
1459
- if (responseUsageCount === 0) {
1460
- const aggregateInput = (stream.state.usage as { inputTokens?: unknown } | undefined)?.inputTokens;
1461
- if (typeof aggregateInput === "number" && aggregateInput > 0) {
1462
- lastInputTokensObserved = aggregateInput;
1593
+
1594
+ if (stream.interruptions.length > 0) {
1595
+ await reconcileConversationTruth();
1596
+ const approvals = runtime.serializeApprovals(stream.interruptions);
1597
+ await saveRunState(db, {
1598
+ accountId: input.accountId,
1599
+ workspaceId: input.workspaceId,
1600
+ sessionId: input.sessionId,
1601
+ turnId: activeTurnId,
1602
+ serializedRunState: stream.state.toString(),
1603
+ pendingApprovals: approvals,
1604
+ // Record the account freezing this state so a resume on a DIFFERENT
1605
+ // codex account strips its account-bound reasoning before replay (HOLE C).
1606
+ frozenCodexCredentialId: effectiveCodexCredentialId,
1607
+ });
1608
+ await publish!([
1609
+ { type: "session.requiresAction", payload: { approvals } },
1610
+ { type: "session.status.changed", payload: { status: "requires_action" } },
1611
+ ], true);
1612
+ await finishTurn(db, input.workspaceId, activeTurnId, "requires_action");
1613
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "requires_action", activeTurnId);
1614
+ activityStatus = "requires_action";
1615
+ return { status: "requires_action" };
1463
1616
  }
1464
- await recordModelUsageAndDebitCredits(settings, db, {
1465
- accountId: input.accountId,
1466
- workspaceId: input.workspaceId,
1467
- sessionId: input.sessionId,
1468
- turnId,
1469
- model: turn.model,
1470
- isCodexTurn,
1471
- usage: stream.state.usage,
1472
- sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
1473
- });
1474
- }
1475
- if (lastInputTokensObserved !== null) {
1476
- await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, lastInputTokensObserved)
1477
- .catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
1478
- }
1479
1617
 
1480
- if (stream.interruptions.length > 0) {
1618
+ const finalOutput = String(stream.finalOutput ?? "");
1481
1619
  await reconcileConversationTruth();
1482
- const approvals = runtime.serializeApprovals(stream.interruptions);
1483
- await saveRunState(db, {
1484
- accountId: input.accountId,
1485
- workspaceId: input.workspaceId,
1486
- sessionId: input.sessionId,
1487
- turnId,
1488
- serializedRunState: stream.state.toString(),
1489
- pendingApprovals: approvals,
1490
- // Record the account freezing this state so a resume on a DIFFERENT
1491
- // codex account strips its account-bound reasoning before replay (HOLE C).
1492
- frozenCodexCredentialId: effectiveCodexCredentialId,
1493
- });
1494
- await publish([
1495
- { type: "session.requiresAction", payload: { approvals } },
1496
- { type: "session.status.changed", payload: { status: "requires_action" } },
1620
+ if (settings.sessionHistorySource !== "items") {
1621
+ // Legacy conversation memory; in items mode the blob is only written
1622
+ // for requires_action pauses (the one RunState-only resume path).
1623
+ await saveRunState(db, {
1624
+ accountId: input.accountId,
1625
+ workspaceId: input.workspaceId,
1626
+ sessionId: input.sessionId,
1627
+ turnId: activeTurnId,
1628
+ serializedRunState: stream.state.toString(),
1629
+ pendingApprovals: [],
1630
+ frozenCodexCredentialId: effectiveCodexCredentialId,
1631
+ });
1632
+ }
1633
+ await publish!([
1634
+ { type: "agent.message.completed", payload: { text: finalOutput } },
1635
+ { type: "turn.completed", payload: { output: finalOutput } },
1636
+ { type: "session.status.changed", payload: { status: "idle" } },
1497
1637
  ], true);
1498
- await finishTurn(db, input.workspaceId, turnId, "requires_action");
1499
- await setSessionStatus(db, input.workspaceId, input.sessionId, "requires_action", turnId);
1500
- activityStatus = "requires_action";
1501
- return { status: "requires_action" };
1502
- }
1503
-
1504
- const finalOutput = String(stream.finalOutput ?? "");
1505
- await reconcileConversationTruth();
1506
- if (settings.sessionHistorySource !== "items") {
1507
- // Legacy conversation memory; in items mode the blob is only written
1508
- // for requires_action pauses (the one RunState-only resume path).
1509
- await saveRunState(db, {
1638
+ await finishTurn(db, input.workspaceId, activeTurnId, "idle");
1639
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
1640
+ await recordUsageEvent(db, {
1510
1641
  accountId: input.accountId,
1511
1642
  workspaceId: input.workspaceId,
1512
- sessionId: input.sessionId,
1513
- turnId,
1514
- serializedRunState: stream.state.toString(),
1515
- pendingApprovals: [],
1516
- frozenCodexCredentialId: effectiveCodexCredentialId,
1643
+ eventType: "agent_run.completed",
1644
+ quantity: 1,
1645
+ unit: "run",
1646
+ sourceResourceType: "session_turn",
1647
+ sourceResourceId: activeTurnId,
1648
+ idempotencyKey: `usage:agent_run.completed:${activeTurnId}`,
1517
1649
  });
1650
+ activityStatus = "idle";
1651
+ return { status: "idle" };
1652
+ };
1653
+
1654
+ await prepareRunAttemptInput();
1655
+ let compactionRecoveryAttempted = false;
1656
+ let retriedAfterCompaction = false;
1657
+ while (true) {
1658
+ try {
1659
+ const result = await runStreamAttempt();
1660
+ if (retriedAfterCompaction) {
1661
+ observability.info("context compaction recovery succeeded after in-activity retry", {
1662
+ sessionId: input.sessionId,
1663
+ turnId: activeTurnId,
1664
+ });
1665
+ }
1666
+ return result;
1667
+ } catch (attemptError) {
1668
+ const overflow = classifyContextWindowOverflowError(attemptError);
1669
+ const compactionNeeded = findCompactionNeededError(attemptError);
1670
+ const recoveryKind = compactionNeeded ? "proactive" : overflow ? "overflow" : null;
1671
+ if (!recoveryKind || !publish || !turnStartedPublished || compactionRecoveryAttempted) {
1672
+ throw attemptError;
1673
+ }
1674
+ compactionRecoveryAttempted = true;
1675
+ await flushRuntimeBatcher();
1676
+ await reconcileConversationTruth({ skipInputOnlyRows: true });
1677
+ const progressPersisted = modelOrToolProgressPersisted;
1678
+ observability.warn("context compaction recovery attempted", {
1679
+ sessionId: input.sessionId,
1680
+ turnId: activeTurnId,
1681
+ reason: recoveryKind,
1682
+ progressPersisted,
1683
+ code: overflow?.code,
1684
+ error: overflow?.message ?? compactionNeeded?.message,
1685
+ signalTokens: compactionNeeded?.signalTokens,
1686
+ thresholdTokens: compactionNeeded?.thresholdTokens,
1687
+ });
1688
+ let compacted = false;
1689
+ try {
1690
+ const outcome = await forceContextCompaction(recoveryKind);
1691
+ compacted = outcome.compacted;
1692
+ } catch (compactError) {
1693
+ observability.warn("context compaction recovery compaction failed", {
1694
+ sessionId: input.sessionId,
1695
+ turnId: activeTurnId,
1696
+ error: compactError instanceof Error ? compactError.message : String(compactError),
1697
+ });
1698
+ }
1699
+ if (progressPersisted) {
1700
+ // The user's intent was live mid-turn — after a successful
1701
+ // compaction the turn AUTO-CONTINUES via the same synthesized
1702
+ // resume-notice requeue the worker-shutdown checkpoint uses (never
1703
+ // the original trigger, so side effects are not replayed). Idle is
1704
+ // the fallback, not the default; it remains for: legacy run-state
1705
+ // history (resuming the frozen pre-compaction RunState would
1706
+ // overflow again), a compaction that could not shrink anything,
1707
+ // and a turn that ALREADY resumed from an overflow (loop guard —
1708
+ // a second overflow in the same lineage stops churning).
1709
+ const triggerPayload =
1710
+ trigger.payload && typeof trigger.payload === "object" && !Array.isArray(trigger.payload)
1711
+ ? trigger.payload as Record<string, unknown>
1712
+ : {};
1713
+ const compactionResumedTurn = triggerType === "turn.preempted"
1714
+ && (triggerPayload.reason === "context_compacted" || triggerPayload.reason === "context_overflow_compacted");
1715
+ const canAutoContinue =
1716
+ compacted && !compactionResumedTurn && settings.sessionHistorySource === "items" && activeTurnId != null;
1717
+ if (canAutoContinue) {
1718
+ const [preemptedEvent] = await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [
1719
+ {
1720
+ turnId: activeTurnId,
1721
+ type: "turn.preempted",
1722
+ payload: {
1723
+ triggerEventId: input.triggerEventId,
1724
+ reason: "context_compacted",
1725
+ resumeWithNotice: true,
1726
+ text: CONTEXT_OVERFLOW_RESUME_TEXT,
1727
+ },
1728
+ },
1729
+ { turnId: activeTurnId, type: "session.status.changed", payload: { status: "queued" } },
1730
+ ]);
1731
+ await requeuePreemptedTurn(db, input.workspaceId, activeTurnId, preemptedEvent ? preemptedEvent.id : input.triggerEventId);
1732
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
1733
+ activityStatus = "preempted";
1734
+ observability.info("context compaction recovery succeeded by compacting and auto-continuing", {
1735
+ sessionId: input.sessionId,
1736
+ turnId: activeTurnId,
1737
+ reason: recoveryKind,
1738
+ compacted,
1739
+ });
1740
+ return { status: "preempted" };
1741
+ }
1742
+ await publish!([
1743
+ {
1744
+ type: "turn.failed",
1745
+ payload: {
1746
+ error: CONTEXT_WINDOW_OVERFLOW_RECOVERY_MESSAGE,
1747
+ code: recoveryKind === "overflow" ? "context_window_overflow_compacted" : "context_compacted",
1748
+ retryable: false,
1749
+ recovery: "user_message",
1750
+ compacted,
1751
+ },
1752
+ },
1753
+ { type: "session.status.changed", payload: { status: "idle" } },
1754
+ ], true);
1755
+ await finishTurn(db, input.workspaceId, activeTurnId, "failed");
1756
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
1757
+ activityStatus = "idle";
1758
+ activityError = attemptError;
1759
+ observability.info("context compaction recovery succeeded by compacting and idling", {
1760
+ sessionId: input.sessionId,
1761
+ turnId: activeTurnId,
1762
+ reason: recoveryKind,
1763
+ compacted,
1764
+ });
1765
+ return { status: "idle" };
1766
+ }
1767
+ retriedAfterCompaction = true;
1768
+ observability.info("context compaction recovery retrying turn after compaction", {
1769
+ sessionId: input.sessionId,
1770
+ turnId: activeTurnId,
1771
+ reason: recoveryKind,
1772
+ compacted,
1773
+ });
1774
+ await prepareRunAttemptInput();
1775
+ }
1518
1776
  }
1519
- await publish([
1520
- { type: "agent.message.completed", payload: { text: finalOutput } },
1521
- { type: "turn.completed", payload: { output: finalOutput } },
1522
- { type: "session.status.changed", payload: { status: "idle" } },
1523
- ], true);
1524
- await finishTurn(db, input.workspaceId, turnId, "idle");
1525
- await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
1526
- await recordUsageEvent(db, {
1527
- accountId: input.accountId,
1528
- workspaceId: input.workspaceId,
1529
- eventType: "agent_run.completed",
1530
- quantity: 1,
1531
- unit: "run",
1532
- sourceResourceType: "session_turn",
1533
- sourceResourceId: turnId,
1534
- idempotencyKey: `usage:agent_run.completed:${turnId}`,
1535
- });
1536
- activityStatus = "idle";
1537
- return { status: "idle" };
1538
1777
  } catch (error) {
1539
1778
  // Graceful worker shutdown (deploy / rollout restart): checkpoint and
1540
1779
  // hand the turn back instead of failing the session. Conversation truth
@@ -1572,7 +1811,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1572
1811
  }
1573
1812
  if (isWorkerShutdownCancellation(error) && preemptTurnId) {
1574
1813
  try {
1575
- await batcher?.flush().catch(() => undefined);
1814
+ await flushRuntimeBatcher();
1576
1815
  await reconcileConversationTruth();
1577
1816
  // An approval-decision rerun always replays its original trigger:
1578
1817
  // the decision is applied through the approval resume path reading
@@ -1639,7 +1878,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1639
1878
  if (error instanceof CancelledFailure) {
1640
1879
  activityStatus = "cancelled";
1641
1880
  activityError = error;
1642
- await batcher?.flush().catch(() => undefined);
1881
+ await flushRuntimeBatcher();
1643
1882
  if (preemptTurnId) {
1644
1883
  await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => undefined);
1645
1884
  }
@@ -1652,7 +1891,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1652
1891
  // that resumption.
1653
1892
  const maxTurns = maxTurnsExceededRunState(error);
1654
1893
  if (maxTurns && publish && turnId && turnStartedPublished) {
1655
- await batcher?.flush().catch(() => undefined);
1894
+ await flushRuntimeBatcher();
1656
1895
  // The SDK attaches the run state at the throw site; persisting it lets
1657
1896
  // the continuation resume with this segment's full context. If capture
1658
1897
  // ever fails, the continuation falls back to the previous snapshot --
@@ -1703,7 +1942,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1703
1942
  if (usageLimit && publish && turnId && turnStartedPublished) {
1704
1943
  const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
1705
1944
  const goalActive = Boolean(goal && goal.status === "active");
1706
- await batcher?.flush().catch(() => undefined);
1945
+ await flushRuntimeBatcher();
1707
1946
  await reconcileConversationTruth();
1708
1947
  const serializedRunState = agentsErrorRunState(error);
1709
1948
  const runStateSaved = Boolean(serializedRunState) && settings.sessionHistorySource !== "items";
@@ -1834,7 +2073,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1834
2073
  // top-up). An active goal pauses visibly with reason "limits" at the
1835
2074
  // next continuation evaluation, without consuming continuation budget.
1836
2075
  if (error instanceof BudgetExhaustedError && publish && turnId && turnStartedPublished) {
1837
- await batcher?.flush().catch(() => undefined);
2076
+ await flushRuntimeBatcher();
1838
2077
  await reconcileConversationTruth();
1839
2078
  const runStateSaved = Boolean(error.serializedRunState) && settings.sessionHistorySource !== "items";
1840
2079
  if (error.serializedRunState && settings.sessionHistorySource !== "items") {
@@ -1880,7 +2119,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1880
2119
  if (failure.retryable && publish && turnId && turnStartedPublished) {
1881
2120
  const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
1882
2121
  const goalActive = Boolean(goal && goal.status === "active");
1883
- await batcher?.flush().catch(() => undefined);
2122
+ await flushRuntimeBatcher();
1884
2123
  // Provider errors rarely carry SDK run state; a null falls back to
1885
2124
  // the previous snapshot, same degraded-context contract as the
1886
2125
  // max-turns path above.