@opengeni/worker-bundle 0.2.3 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -44,7 +44,7 @@ import {
44
44
  consumeSessionCompactionRequest,
45
45
  countSessionHistoryItems,
46
46
  getActiveSessionHistoryItems as getActiveSessionHistoryItems3,
47
- nextSessionHistoryPosition,
47
+ nextSessionHistoryPosition as nextSessionHistoryPosition2,
48
48
  requeuePreemptedTurn,
49
49
  saveRunState,
50
50
  upsertSandboxSessionEnvelope,
@@ -62,8 +62,10 @@ import {
62
62
  maxTurnsExceededRunState,
63
63
  modelResponseUsageFromSdkEvent,
64
64
  normalizeSdkEvent,
65
- sanitizeHistoryItemsForModel,
66
- summarizeForCompaction as summarizeForCompaction2
65
+ sanitizeHistoryItemsForModel as sanitizeHistoryItemsForModel2,
66
+ summarizeForCompaction as summarizeForCompaction2,
67
+ findCompactionNeededError,
68
+ SUMMARY_BUFFER_TOKENS as SUMMARY_BUFFER_TOKENS2
67
69
  } from "@opengeni/runtime";
68
70
  import { calculateModelUsageCostMicros, configuredModelPricing, configuredStaticUsageLimits as configuredStaticUsageLimits2, sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
69
71
  import { CancelledFailure } from "@temporalio/activity";
@@ -83,12 +85,46 @@ import {
83
85
  import {
84
86
  decryptedCapabilityHeaders,
85
87
  listEnabledMcpCapabilityServers,
88
+ listSessionMcpServerMetadata,
89
+ listSessionMcpServersForRun,
86
90
  workspaceCodexSubscriptionActive
87
91
  } from "@opengeni/db";
88
92
  async function settingsWithEnabledCapabilityMcpServers(db, workspaceId, settings) {
89
93
  const enabled = await listEnabledMcpCapabilityServers(db, workspaceId);
90
94
  return settingsWithMcpCapabilityServers(settings, enabled);
91
95
  }
96
+ async function settingsWithSessionMcpServersForRun(db, workspaceId, sessionId, settings) {
97
+ const encryptionKey = environmentsEncryptionKeyBytes(settings);
98
+ if (!encryptionKey) {
99
+ const metadata = await listSessionMcpServerMetadata(db, workspaceId, sessionId);
100
+ if (metadata.length === 0) {
101
+ return settings;
102
+ }
103
+ throw new Error("session MCP server credentials require OPENGENI_ENVIRONMENTS_ENCRYPTION_KEY");
104
+ }
105
+ return settingsWithSessionMcpServers(settings, await listSessionMcpServersForRun(db, workspaceId, sessionId, encryptionKey));
106
+ }
107
+ function settingsWithSessionMcpServers(settings, servers) {
108
+ if (servers.length === 0) {
109
+ return settings;
110
+ }
111
+ const sessionIds = new Set(servers.map((server) => server.id));
112
+ return {
113
+ ...settings,
114
+ mcpServers: [
115
+ ...settings.mcpServers.filter((server) => !sessionIds.has(server.id)),
116
+ ...servers.map((server) => ({
117
+ id: server.id,
118
+ ...server.name ? { name: server.name } : {},
119
+ url: server.url,
120
+ ...server.allowedTools ? { allowedTools: server.allowedTools } : {},
121
+ ...server.timeoutMs ? { timeoutMs: server.timeoutMs } : {},
122
+ cacheToolsList: server.cacheToolsList ?? false,
123
+ headers: server.headers
124
+ }))
125
+ ]
126
+ };
127
+ }
92
128
  async function settingsWithCodexCredential(db, workspaceId, settings, activeOverride) {
93
129
  const active = activeOverride ?? await workspaceCodexSubscriptionActive(db, settings, workspaceId);
94
130
  if (!active) {
@@ -324,19 +360,19 @@ function workflowIdForSession(sessionId) {
324
360
  // src/activities/context-compaction.ts
325
361
  import {
326
362
  applyContextCompaction,
327
- getActiveSessionHistoryItems
363
+ getActiveSessionHistoryItems,
364
+ nextSessionHistoryPosition
328
365
  } from "@opengeni/db";
329
366
  import {
330
- buildCompactionMessages,
331
- buildSummaryItem,
332
- planCompaction,
367
+ SUMMARY_BUFFER_TOKENS,
368
+ buildCompactionPromptInput,
369
+ buildCompactionReplacementHistory,
370
+ decideClientCompaction,
371
+ sanitizeHistoryItemsForModel,
333
372
  summarizeForCompaction
334
373
  } from "@opengeni/runtime";
335
- import {
336
- contextInputBudgetTokens,
337
- resolveContextCompactionMode
338
- } from "@opengeni/config";
339
- async function maybeCompactContext(db, settings, scope, lastInputTokens, summarize = (s, m) => summarizeForCompaction(s, m, { maxOutputTokens: s.contextSummaryMaxTokens }), options = {}) {
374
+ import { resolveContextCompactionMode } from "@opengeni/config";
375
+ async function maybeCompactContext(db, settings, scope, lastInputTokens, summarize = (s, m) => summarizeForCompaction(s, m, { maxOutputTokens: SUMMARY_BUFFER_TOKENS }), options = {}) {
340
376
  if (resolveContextCompactionMode(settings) !== "client") {
341
377
  return { compacted: false, reason: "mode_not_client" };
342
378
  }
@@ -344,48 +380,50 @@ async function maybeCompactContext(db, settings, scope, lastInputTokens, summari
344
380
  if (active.length === 0) {
345
381
  return { compacted: false, reason: "no_history" };
346
382
  }
347
- const items = active.map((row) => row.item);
348
- const plan = planCompaction({
383
+ const items = sanitizeHistoryItemsForModel(active.map((row) => row.item));
384
+ const decision = decideClientCompaction({
349
385
  items,
350
386
  lastInputTokens,
351
- inputBudgetTokens: contextInputBudgetTokens(settings),
352
- softFraction: settings.contextCompactSoftFraction,
353
- hardFraction: settings.contextCompactHardFraction,
354
- keepRecentTokens: settings.contextKeepRecentTokens,
387
+ contextWindowTokens: settings.contextWindowTokens,
388
+ contextReservedOutputTokens: settings.contextReservedOutputTokens,
355
389
  ...options.force ? { force: true } : {}
356
390
  });
357
- if (!plan.shouldCompact) {
358
- return { compacted: false, reason: plan.reason };
359
- }
360
- if (plan.hardForced) {
361
- console.warn(
362
- `context compaction HARD-FORCED for session ${scope.sessionId}: signal ${plan.signalTokens} tokens at/over hard ceiling`
363
- );
391
+ if (!decision.shouldCompact) {
392
+ return { compacted: false, reason: decision.reason };
364
393
  }
365
- const messages = buildCompactionMessages(plan);
366
- const summaryBody = await summarize(settings, messages);
394
+ const promptInput = buildCompactionPromptInput(items);
395
+ const summaryBody = await summarize(settings, promptInput);
367
396
  if (!summaryBody) {
368
397
  return { compacted: false, reason: "summarize_failed" };
369
398
  }
370
- const boundaryRow = active[plan.boundaryIndex];
371
- if (!boundaryRow) {
372
- return { compacted: false, reason: "boundary_out_of_range" };
399
+ const replacementHistory = buildCompactionReplacementHistory(items, summaryBody);
400
+ const summaryItem = replacementHistory.at(-1);
401
+ if (!summaryItem) {
402
+ return { compacted: false, reason: "empty_replacement_history" };
373
403
  }
374
- const boundaryPosition = boundaryRow.position;
375
- if (boundaryPosition <= 0) {
376
- return { compacted: false, reason: "boundary_at_origin" };
377
- }
378
- const summaryPosition = boundaryPosition - 0.5;
404
+ const nextPosition = await nextSessionHistoryPosition(db, scope.workspaceId, scope.sessionId);
405
+ const replacementItems = replacementHistory.slice(0, -1).map((item, index) => ({
406
+ position: nextPosition + index,
407
+ item
408
+ }));
409
+ const summaryPosition = nextPosition + replacementItems.length;
379
410
  await applyContextCompaction(db, {
380
411
  accountId: scope.accountId,
381
412
  workspaceId: scope.workspaceId,
382
413
  sessionId: scope.sessionId,
383
414
  turnId: scope.turnId ?? null,
384
- boundaryPosition,
415
+ boundaryPosition: nextPosition,
416
+ replacementItems,
385
417
  summaryPosition,
386
- summaryItem: buildSummaryItem(summaryBody)
418
+ summaryItem
387
419
  });
388
- return { compacted: true, supersededFrom: boundaryPosition, summaryPosition, hardForced: plan.hardForced };
420
+ return {
421
+ compacted: true,
422
+ supersededFrom: nextPosition,
423
+ summaryPosition,
424
+ signalTokens: decision.signalTokens,
425
+ thresholdTokens: decision.thresholdTokens
426
+ };
389
427
  }
390
428
 
391
429
  // src/activities/environment.ts
@@ -886,7 +924,7 @@ function isPlainObject(value) {
886
924
  }
887
925
 
888
926
  // src/activities/run-input.ts
889
- import { contextInputBudgetTokens as contextInputBudgetTokens2, resolveContextCompactionMode as resolveContextCompactionMode2 } from "@opengeni/config";
927
+ import { contextInputBudgetTokens, resolveContextCompactionMode as resolveContextCompactionMode2 } from "@opengeni/config";
890
928
  import {
891
929
  getActiveSessionHistoryItems as getActiveSessionHistoryItems2,
892
930
  getLatestRunState,
@@ -1030,7 +1068,7 @@ function readPathBudgetTokens(settings) {
1030
1068
  if (!settings || resolveContextCompactionMode2(settings) !== "client") {
1031
1069
  return void 0;
1032
1070
  }
1033
- const budget = contextInputBudgetTokens2(settings);
1071
+ const budget = contextInputBudgetTokens(settings);
1034
1072
  return budget > 0 ? budget : void 0;
1035
1073
  }
1036
1074
  async function userMessageTextWithAttachments(db, workspaceId, text, resources) {
@@ -1132,6 +1170,7 @@ import {
1132
1170
  failWarmingToCold,
1133
1171
  getSandboxSessionEnvelope as getSandboxSessionEnvelope2,
1134
1172
  readLease,
1173
+ recordWarmingSandboxCreated,
1135
1174
  releaseLeaseHolder,
1136
1175
  SandboxLeaseSupersededError
1137
1176
  } from "@opengeni/db";
@@ -1143,13 +1182,84 @@ import {
1143
1182
  DisplayStackError,
1144
1183
  DisplayStackUnsupportedError,
1145
1184
  buildStreamUrl,
1146
- StreamPortUnavailableError
1185
+ StreamPortUnavailableError,
1186
+ modalSandboxAttributionEnvironment,
1187
+ tagModalSandbox
1147
1188
  } from "@opengeni/runtime";
1148
1189
  import { DESKTOP_STREAM_PORT } from "@opengeni/contracts";
1190
+ var SandboxWarmingTimeoutError = class extends Error {
1191
+ constructor(backend, timeoutMs) {
1192
+ super(
1193
+ `Sandbox backend "${backend}" capacity or creation timed out after ${Math.ceil(timeoutMs / 1e3)}s while warming the sandbox lease. Please try again; if this persists, sandbox capacity may be exhausted.`
1194
+ );
1195
+ this.backend = backend;
1196
+ this.timeoutMs = timeoutMs;
1197
+ this.name = "SandboxWarmingTimeoutError";
1198
+ }
1199
+ backend;
1200
+ timeoutMs;
1201
+ code = "sandbox_warming_timeout";
1202
+ };
1149
1203
  var WARMING_POLL_INTERVAL_MS = 250;
1150
1204
  async function sleep(ms) {
1151
1205
  await new Promise((resolve) => setTimeout(resolve, ms));
1152
1206
  }
1207
+ async function terminateEstablishedSandbox(established) {
1208
+ if (!established) {
1209
+ return true;
1210
+ }
1211
+ const client = established.client;
1212
+ if (typeof client.delete === "function" && established.sessionState !== void 0) {
1213
+ try {
1214
+ await client.delete(established.sessionState);
1215
+ return true;
1216
+ } catch {
1217
+ return false;
1218
+ }
1219
+ }
1220
+ const session = established.session;
1221
+ try {
1222
+ if (session.terminate) {
1223
+ await session.terminate();
1224
+ return true;
1225
+ } else if (session.kill) {
1226
+ await session.kill();
1227
+ return true;
1228
+ } else if (session.close) {
1229
+ if (!session.closed) {
1230
+ await session.close();
1231
+ }
1232
+ return true;
1233
+ }
1234
+ return false;
1235
+ } catch {
1236
+ return false;
1237
+ }
1238
+ }
1239
+ function asSandboxWarmingError(error, backend, timeoutMs) {
1240
+ const message = error instanceof Error ? error.message : String(error);
1241
+ return /sandbox creation timed out|warming timed out|capacity.*timed out/i.test(message) ? new SandboxWarmingTimeoutError(backend, timeoutMs) : error;
1242
+ }
1243
+ function workspaceArchiveFromEnvelope(envelope) {
1244
+ const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
1245
+ const archive = sessionState?.workspaceArchive;
1246
+ return typeof archive === "string" && archive.length > 0 ? archive : null;
1247
+ }
1248
+ function preserveWorkspaceArchiveOnInterimResumeState(resumeState, archiveSource) {
1249
+ const archive = workspaceArchiveFromEnvelope(archiveSource);
1250
+ if (!archive) {
1251
+ return resumeState;
1252
+ }
1253
+ const existingSessionState = resumeState && typeof resumeState.sessionState === "object" && resumeState.sessionState !== null ? resumeState.sessionState : {};
1254
+ return {
1255
+ ...resumeState ?? {},
1256
+ ...resumeState?.backendId === void 0 && archiveSource?.backendId !== void 0 ? { backendId: archiveSource.backendId } : {},
1257
+ sessionState: {
1258
+ ...existingSessionState,
1259
+ workspaceArchive: archive
1260
+ }
1261
+ };
1262
+ }
1153
1263
  async function resumeBoxForTurn(services, ids, kind, holderId) {
1154
1264
  const { db, settings } = services;
1155
1265
  const os = ids.os ?? "linux";
@@ -1191,14 +1301,50 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
1191
1301
  }
1192
1302
  if (acquired.role === "spawner") {
1193
1303
  const expectedEpoch = acquired.lease.leaseEpoch;
1304
+ let createdEstablished = null;
1305
+ if (ids.environment && ids.backend === "modal") {
1306
+ Object.assign(ids.environment, modalSandboxAttributionEnvironment({
1307
+ leaseId: acquired.lease.id,
1308
+ workspaceId: ids.workspaceId,
1309
+ sandboxGroupId: ids.sandboxGroupId
1310
+ }));
1311
+ }
1194
1312
  try {
1195
1313
  const envelope = await getSandboxSessionEnvelope2(db, ids.workspaceId, ids.sessionId);
1196
1314
  const spawnEnvelope = acquired.lease.resumeState ?? envelope;
1197
1315
  const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
1198
1316
  sessionId: ids.sessionId,
1199
1317
  backendOverride: ids.backend,
1200
- ...ids.environment ? { environment: ids.environment } : {}
1318
+ ...ids.environment ? { environment: ids.environment } : {},
1319
+ onSandboxCreated: async (created) => {
1320
+ createdEstablished = created;
1321
+ const resumeEnvelope2 = preserveWorkspaceArchiveOnInterimResumeState(
1322
+ await serializeEstablishedSandboxEnvelope(created) ?? null,
1323
+ spawnEnvelope
1324
+ );
1325
+ const recorded = await recordWarmingSandboxCreated(db, {
1326
+ accountId: ids.accountId,
1327
+ workspaceId: ids.workspaceId,
1328
+ sandboxGroupId: ids.sandboxGroupId,
1329
+ expectedEpoch,
1330
+ instanceId: created.instanceId,
1331
+ resumeBackendId: created.backendId,
1332
+ resumeState: resumeEnvelope2,
1333
+ leaseTtlMs
1334
+ });
1335
+ if (!recorded.recorded) {
1336
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
1337
+ }
1338
+ if (created.backendId === "modal") {
1339
+ await tagModalSandbox(settings, created.instanceId, {
1340
+ leaseId: acquired.lease.id,
1341
+ workspaceId: ids.workspaceId,
1342
+ sandboxGroupId: ids.sandboxGroupId
1343
+ }).catch(() => void 0);
1344
+ }
1345
+ }
1201
1346
  });
1347
+ createdEstablished = established;
1202
1348
  await ensureDisplayStack(settings, established);
1203
1349
  const endpoint = await exposeStreamPort(settings, established);
1204
1350
  const resumeEnvelope = await serializeEstablishedSandboxEnvelope(established) ?? envelope ?? null;
@@ -1214,29 +1360,35 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
1214
1360
  leaseTtlMs
1215
1361
  });
1216
1362
  if (!committed.committed || !committed.lease) {
1363
+ await terminateEstablishedSandbox(established);
1217
1364
  await release();
1218
1365
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
1219
1366
  }
1220
1367
  return { established, leaseEpoch: committed.lease.leaseEpoch, release };
1221
1368
  } catch (error) {
1222
1369
  if (error instanceof SandboxLeaseSupersededError) {
1370
+ await terminateEstablishedSandbox(createdEstablished);
1371
+ await release();
1223
1372
  throw error;
1224
1373
  }
1225
- await failWarmingToCold(db, {
1226
- accountId: ids.accountId,
1227
- workspaceId: ids.workspaceId,
1228
- sandboxGroupId: ids.sandboxGroupId,
1229
- expectedEpoch
1230
- });
1374
+ const terminated = await terminateEstablishedSandbox(createdEstablished);
1375
+ if (terminated) {
1376
+ await failWarmingToCold(db, {
1377
+ accountId: ids.accountId,
1378
+ workspaceId: ids.workspaceId,
1379
+ sandboxGroupId: ids.sandboxGroupId,
1380
+ expectedEpoch
1381
+ });
1382
+ }
1231
1383
  await release();
1232
- throw error;
1384
+ throw asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
1233
1385
  }
1234
1386
  }
1235
- let leaseEpoch = acquired.lease.leaseEpoch;
1236
- if (acquired.lease.liveness === "warming") {
1237
- leaseEpoch = (await waitForWarmOrReacquire(services, ids, kind, holderId)).leaseEpoch;
1238
- }
1239
1387
  try {
1388
+ let leaseEpoch = acquired.lease.leaseEpoch;
1389
+ if (acquired.lease.liveness === "warming") {
1390
+ leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
1391
+ }
1240
1392
  const live = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
1241
1393
  const envelope = live?.resumeState ?? await getSandboxSessionEnvelope2(db, ids.workspaceId, ids.sessionId);
1242
1394
  const established = await establishSandboxSessionFromEnvelope(settings, envelope, {
@@ -1248,7 +1400,7 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
1248
1400
  return { established, leaseEpoch, release };
1249
1401
  } catch (error) {
1250
1402
  await release();
1251
- throw error;
1403
+ throw asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
1252
1404
  }
1253
1405
  }
1254
1406
  async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
@@ -1305,37 +1457,26 @@ async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
1305
1457
  }
1306
1458
  return { leaseEpoch: acquired.lease.leaseEpoch, release };
1307
1459
  }
1308
- async function waitForWarmOrReacquire(services, ids, kind, holderId) {
1460
+ async function waitForWarm(services, ids) {
1309
1461
  const { db, settings } = services;
1310
- const deadline = Date.now() + settings.sandboxLeaseWarmingTtlMs;
1462
+ const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
1311
1463
  while (Date.now() < deadline) {
1312
1464
  await sleep(WARMING_POLL_INTERVAL_MS);
1313
1465
  const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
1314
1466
  if (!lease) {
1315
- break;
1467
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
1316
1468
  }
1317
- if (lease.liveness === "warm" || lease.liveness === "draining") {
1318
- return { liveness: lease.liveness, leaseEpoch: lease.leaseEpoch };
1469
+ if (lease.liveness === "warm") {
1470
+ return { leaseEpoch: lease.leaseEpoch };
1471
+ }
1472
+ if (lease.liveness === "draining") {
1473
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
1319
1474
  }
1320
1475
  if (lease.liveness === "cold") {
1321
- break;
1476
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
1322
1477
  }
1323
1478
  }
1324
- const reacquired = await acquireLease(db, {
1325
- accountId: ids.accountId,
1326
- workspaceId: ids.workspaceId,
1327
- sandboxGroupId: ids.sandboxGroupId,
1328
- kind,
1329
- holderId,
1330
- subjectId: ids.sessionId,
1331
- backend: ids.backend,
1332
- os: ids.os ?? "linux",
1333
- leaseTtlMs: settings.sandboxLeaseTtlMs
1334
- });
1335
- if (reacquired.role === "fenced") {
1336
- throw new SandboxLeaseSupersededError(ids.sandboxGroupId, reacquired.lease.leaseEpoch);
1337
- }
1338
- return { liveness: reacquired.lease.liveness, leaseEpoch: reacquired.lease.leaseEpoch };
1479
+ throw new SandboxWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs);
1339
1480
  }
1340
1481
  async function ensureDisplayStack(settings, established) {
1341
1482
  if (!settings.sandboxDesktopEnabled) {
@@ -1635,6 +1776,10 @@ function selectCodexCredentialForTurn(args) {
1635
1776
  }
1636
1777
  return null;
1637
1778
  }
1779
+ var CONTEXT_OVERFLOW_RESUME_TEXT = [
1780
+ "[TURN RESUMED AFTER CONTEXT COMPACTION] The conversation approached or exceeded the model's context window mid-turn; older history above was compacted into a summary.",
1781
+ "Continue the original task from where it left off. Before repeating any action with side effects, check whether it already happened."
1782
+ ].join("\n");
1638
1783
  var WORKER_SHUTDOWN_RESUME_TEXT = [
1639
1784
  "[TURN RESUMED AFTER WORKER RESTART] The platform worker running this turn restarted (deploy/rollout) before the turn finished.",
1640
1785
  "Your conversation history above, including this turn's earlier work, is preserved up to the last checkpoint; at most the single in-flight model step after it was lost.",
@@ -1648,6 +1793,47 @@ var WORKER_DEATH_RESUME_TEXT = [
1648
1793
  function isWorkerShutdownCancellation(error) {
1649
1794
  return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
1650
1795
  }
1796
+ var CONTEXT_WINDOW_OVERFLOW_RECOVERY_MESSAGE = "The conversation reached the model context compaction threshold; it has been compacted. Send a message to continue.";
1797
+ function classifyContextWindowOverflowError(error) {
1798
+ const fields = collectErrorStrings(error);
1799
+ const matched = fields.find(
1800
+ (value) => /context[_\s-]*length[_\s-]*exceeded/i.test(value) || /exceeds?\s+(?:the\s+)?context\s+window/i.test(value) || /maximum\s+context\s+length/i.test(value) || /context\s+window[^.]*exceed/i.test(value)
1801
+ );
1802
+ if (!matched) {
1803
+ return null;
1804
+ }
1805
+ const message = error instanceof Error ? error.message : String(error);
1806
+ const code = fields.find((value) => /context[_\s-]*length[_\s-]*exceeded/i.test(value));
1807
+ return {
1808
+ message,
1809
+ ...code ? { code } : {},
1810
+ ...matched && matched !== message ? { detail: matched } : {}
1811
+ };
1812
+ }
1813
+ function collectErrorStrings(value, seen = /* @__PURE__ */ new WeakSet()) {
1814
+ if (typeof value === "string") {
1815
+ return [value];
1816
+ }
1817
+ if (!value || typeof value !== "object") {
1818
+ return [];
1819
+ }
1820
+ if (seen.has(value)) {
1821
+ return [];
1822
+ }
1823
+ seen.add(value);
1824
+ const out = [];
1825
+ const record = value;
1826
+ for (const key of ["message", "code", "type", "name", "param"]) {
1827
+ const field = record[key];
1828
+ if (typeof field === "string" && field.length > 0) {
1829
+ out.push(field);
1830
+ }
1831
+ }
1832
+ for (const key of ["error", "cause", "response", "data"]) {
1833
+ out.push(...collectErrorStrings(record[key], seen));
1834
+ }
1835
+ return out;
1836
+ }
1651
1837
  function modelUsageSourceKey(input) {
1652
1838
  if (input.responseId) {
1653
1839
  return input.responseId;
@@ -1655,7 +1841,7 @@ function modelUsageSourceKey(input) {
1655
1841
  return input.dispatchId ? `${input.dispatchId}:${input.positionalKey}` : input.positionalKey;
1656
1842
  }
1657
1843
  function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = persistedHistoryCount) {
1658
- const sanitized = sanitizeHistoryItemsForModel(rawHistory);
1844
+ const sanitized = sanitizeHistoryItemsForModel2(rawHistory);
1659
1845
  if (sanitized.length <= persistedHistoryCount) {
1660
1846
  return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
1661
1847
  }
@@ -1665,8 +1851,20 @@ function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = p
1665
1851
  }));
1666
1852
  return { rows, nextWatermark: sanitized.length, nextPosition: nextPosition + rows.length };
1667
1853
  }
1854
+ function isModelOrToolProgressHistoryItem(item) {
1855
+ if (item.type === "message") {
1856
+ return item.role === "assistant";
1857
+ }
1858
+ if (item.type === "reasoning" || item.type === "compaction") {
1859
+ return true;
1860
+ }
1861
+ if (typeof item.type === "string") {
1862
+ return item.type !== "message";
1863
+ }
1864
+ return false;
1865
+ }
1668
1866
  function reconcileSeedCount(activeSeedRows, modelHistoryFromItems, current) {
1669
- return sanitizeHistoryItemsForModel(
1867
+ return sanitizeHistoryItemsForModel2(
1670
1868
  modelHistoryFromItems ? applyCodexHistoryStrip(activeSeedRows, current) : activeSeedRows.map((row) => row.item)
1671
1869
  ).length;
1672
1870
  }
@@ -1733,14 +1931,19 @@ function createRunAgentTurnActivity(services) {
1733
1931
  let activeRecording = null;
1734
1932
  let didComputerUse = false;
1735
1933
  let batcher = null;
1934
+ const flushRuntimeBatcher = async () => {
1935
+ const current = batcher;
1936
+ await current?.flush().catch(() => void 0);
1937
+ };
1736
1938
  let preparedTools = null;
1737
1939
  let publish = null;
1738
1940
  let turnStartedPublished = false;
1739
1941
  let stream;
1740
1942
  let persistedHistoryCount = 0;
1741
1943
  let historyCountAtTurnStart = 0;
1944
+ let modelOrToolProgressPersisted = false;
1742
1945
  let nextHistoryPosition = 0;
1743
- const reconcileConversationTruth = async () => {
1946
+ const reconcileConversationTruth = async (options = {}) => {
1744
1947
  if (!stream || !turnId) {
1745
1948
  return;
1746
1949
  }
@@ -1752,7 +1955,9 @@ function createRunAgentTurnActivity(services) {
1752
1955
  persistedHistoryCount,
1753
1956
  nextHistoryPosition
1754
1957
  );
1755
- if (rows.length > 0) {
1958
+ const hasModelOrToolProgress = rows.some((row) => isModelOrToolProgressHistoryItem(row.item));
1959
+ const shouldAppendRows = rows.length > 0 && (!options.skipInputOnlyRows || hasModelOrToolProgress);
1960
+ if (shouldAppendRows) {
1756
1961
  await appendSessionHistoryItems(db, {
1757
1962
  accountId: input.accountId,
1758
1963
  workspaceId: input.workspaceId,
@@ -1766,8 +1971,13 @@ function createRunAgentTurnActivity(services) {
1766
1971
  items: rows
1767
1972
  });
1768
1973
  }
1769
- persistedHistoryCount = nextWatermark;
1770
- nextHistoryPosition = nextPosition;
1974
+ if (shouldAppendRows || !options.skipInputOnlyRows) {
1975
+ persistedHistoryCount = nextWatermark;
1976
+ nextHistoryPosition = nextPosition;
1977
+ }
1978
+ if (hasModelOrToolProgress) {
1979
+ modelOrToolProgressPersisted = true;
1980
+ }
1771
1981
  }
1772
1982
  const envelope = sandboxStateEntryFromRunState(stream.state);
1773
1983
  if (envelope) {
@@ -1944,12 +2154,13 @@ function createRunAgentTurnActivity(services) {
1944
2154
  }
1945
2155
  const packRuntime = await resolveWorkspacePackRuntime(db, input.workspaceId);
1946
2156
  const workspaceAgentInstructions = await resolveWorkspaceAgentInstructions(db, input.workspaceId);
1947
- const runSettings = {
2157
+ const baseRunSettings = {
1948
2158
  ...settingsWithPackSandboxImage(capabilitySettings, packRuntime.sandboxImage),
1949
2159
  openaiModel: turn.model,
1950
2160
  openaiReasoningEffort: turn.reasoningEffort,
1951
2161
  sandboxBackend: turn.sandboxBackend
1952
2162
  };
2163
+ const runSettings = await settingsWithSessionMcpServersForRun(db, input.workspaceId, input.sessionId, baseRunSettings);
1953
2164
  const resolvedModel = runtime.resolveTurnModel(capabilitySettings, turn.model);
1954
2165
  const codexContext = resolvedModel?.provider.kind === "codex-subscription" ? (() => {
1955
2166
  const resolver = buildCodexTokenResolver(db, runSettings, input.workspaceId, effectiveCodexCredentialId ?? "");
@@ -2214,6 +2425,10 @@ function createRunAgentTurnActivity(services) {
2214
2425
  } : { computerToolMode: computerToolModeForTurn(null) },
2215
2426
  ...packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {},
2216
2427
  ...workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {},
2428
+ // Per-session persona tier (session > workspace > deployment default).
2429
+ // Composed system-level AFTER the workspace persona so it refines it for
2430
+ // this one session; absent ⇒ byte-identical to today's composition.
2431
+ ...session.instructions ? { sessionInstructions: session.instructions } : {},
2217
2432
  ...workspaceEnvironment ? {
2218
2433
  workspaceEnvironment: {
2219
2434
  name: workspaceEnvironment.name,
@@ -2222,6 +2437,12 @@ function createRunAgentTurnActivity(services) {
2222
2437
  }
2223
2438
  } : {}
2224
2439
  });
2440
+ const compactSummarizer = resolvedModel ? (s, m) => withCodex(() => summarizeForCompaction2(s, m, {
2441
+ client: resolvedModel.client,
2442
+ api: resolvedModel.provider.api,
2443
+ model: resolvedModel.configured.id,
2444
+ maxOutputTokens: SUMMARY_BUFFER_TOKENS2
2445
+ })) : void 0;
2225
2446
  if (triggerType === "user.message" || triggerType === "goal.continuation") {
2226
2447
  try {
2227
2448
  const forced = await consumeSessionCompactionRequest(db, input.workspaceId, input.sessionId);
@@ -2234,199 +2455,335 @@ function createRunAgentTurnActivity(services) {
2234
2455
  // registry provider, summarize on THAT provider's client + wire API
2235
2456
  // (a chat provider can't summarize through OpenAI/Azure). Null
2236
2457
  // resolution keeps the default built-in Responses summarizer.
2237
- resolvedModel ? (s, m) => withCodex(() => summarizeForCompaction2(s, m, {
2238
- client: resolvedModel.client,
2239
- api: resolvedModel.provider.api,
2240
- model: resolvedModel.configured.id,
2241
- maxOutputTokens: s.contextSummaryMaxTokens
2242
- })) : void 0,
2458
+ compactSummarizer,
2243
2459
  forced ? { force: true } : {}
2244
2460
  );
2245
2461
  if (outcome.compacted) {
2246
- const trigger2 = forced ? "operator" : outcome.hardForced ? "hard" : void 0;
2462
+ const trigger2 = forced ? "operator" : void 0;
2247
2463
  await publish([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, ...trigger2 ? { trigger: trigger2 } : {} } }]);
2248
2464
  }
2249
2465
  } catch (compactError) {
2250
2466
  console.error("context compaction failed (turn proceeds un-compacted)", compactError);
2251
2467
  }
2252
2468
  }
2253
- const { input: runInput, modelHistoryFromItems } = await turnInput(
2254
- db,
2255
- runtime,
2256
- agent,
2257
- trigger,
2258
- runSettings,
2259
- { currentCodexCredentialId: effectiveCodexCredentialId }
2260
- );
2261
- const activeSeedRows = await getActiveSessionHistoryItems3(db, input.workspaceId, input.sessionId);
2262
- persistedHistoryCount = reconcileSeedCount(
2263
- activeSeedRows,
2264
- modelHistoryFromItems,
2265
- { currentCodexCredentialId: effectiveCodexCredentialId }
2266
- );
2267
- historyCountAtTurnStart = persistedHistoryCount;
2268
- nextHistoryPosition = await nextSessionHistoryPosition(db, input.workspaceId, input.sessionId);
2269
- let responseUsageCount = 0;
2270
- let lastInputTokensObserved = null;
2271
- throwIfWorkerShuttingDown();
2272
- const runStreamOnce = () => runtime.runStream(agent, runInput, runSettings, {
2273
- sandboxEnvironment,
2274
- onRuntimeEvent: async (event) => {
2275
- await publish([{ type: event.type, payload: event.payload }], true);
2276
- },
2277
- // P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
2278
- // keystone). Absent when the flag is off -> legacy build-and-discard.
2279
- ...resolvedSandbox ? {
2280
- ownedSandbox: {
2281
- client: resolvedSandbox.established.client,
2282
- session: resolvedSandbox.established.session,
2283
- sessionState: resolvedSandbox.established.sessionState,
2284
- // Pin platform setup (hooks + file materialization) to the un-proxied
2285
- // established box — never through the routing proxy, which would
2286
- // re-route those execs onto a machine swapped in mid-turn.
2287
- ...setupBoxSession ? { setupSession: setupBoxSession } : {}
2288
- }
2289
- } : {}
2290
- });
2291
- stream = await withCodex(runStreamOnce);
2292
- batcher = createRuntimeBatcher(async (events) => {
2293
- await publish(events);
2294
- });
2295
- const iterator = stream.toStream()[Symbol.asyncIterator]();
2296
- let streamDone = false;
2297
- try {
2298
- while (true) {
2299
- const next = await nextStreamEvent(iterator, activityContext);
2300
- if (next.done) {
2301
- streamDone = true;
2302
- break;
2303
- }
2304
- const responseUsage = modelResponseUsageFromSdkEvent(next.value);
2305
- if (responseUsage) {
2306
- responseUsageCount += 1;
2307
- const observed = responseUsage.usage?.inputTokens;
2308
- if (typeof observed === "number" && observed > 0) {
2309
- lastInputTokensObserved = observed;
2469
+ const activeTurnId = turnId;
2470
+ if (!activeTurnId) {
2471
+ throw new Error("Turn id was not initialized");
2472
+ }
2473
+ let runInput = null;
2474
+ const prepareRunAttemptInput = async () => {
2475
+ const prepared = await turnInput(
2476
+ db,
2477
+ runtime,
2478
+ agent,
2479
+ trigger,
2480
+ runSettings,
2481
+ { currentCodexCredentialId: effectiveCodexCredentialId }
2482
+ );
2483
+ runInput = prepared.input;
2484
+ const activeSeedRows = await getActiveSessionHistoryItems3(db, input.workspaceId, input.sessionId);
2485
+ persistedHistoryCount = reconcileSeedCount(
2486
+ activeSeedRows,
2487
+ prepared.modelHistoryFromItems,
2488
+ { currentCodexCredentialId: effectiveCodexCredentialId }
2489
+ );
2490
+ historyCountAtTurnStart = persistedHistoryCount;
2491
+ nextHistoryPosition = await nextSessionHistoryPosition2(db, input.workspaceId, input.sessionId);
2492
+ };
2493
+ const forceContextCompaction = async (triggerLabel) => {
2494
+ const clientCompactionSettings = { ...runSettings, contextCompactionMode: "client" };
2495
+ const outcome = await maybeCompactContext(
2496
+ db,
2497
+ clientCompactionSettings,
2498
+ { accountId: input.accountId, workspaceId: input.workspaceId, sessionId: input.sessionId, turnId: activeTurnId },
2499
+ session.lastInputTokens,
2500
+ compactSummarizer,
2501
+ { force: true }
2502
+ );
2503
+ if (outcome.compacted) {
2504
+ await publish([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, trigger: triggerLabel } }]);
2505
+ }
2506
+ return outcome;
2507
+ };
2508
+ const runStreamAttempt = async () => {
2509
+ if (!runInput) {
2510
+ throw new Error("Run input was not prepared");
2511
+ }
2512
+ stream = void 0;
2513
+ batcher = null;
2514
+ let responseUsageCount = 0;
2515
+ let lastInputTokensObserved = null;
2516
+ throwIfWorkerShuttingDown();
2517
+ const runStreamOnce = () => runtime.runStream(agent, runInput, runSettings, {
2518
+ sandboxEnvironment,
2519
+ onRuntimeEvent: async (event) => {
2520
+ await publish([{ type: event.type, payload: event.payload }], true);
2521
+ },
2522
+ // P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
2523
+ // keystone). Absent when the flag is off -> legacy build-and-discard.
2524
+ ...resolvedSandbox ? {
2525
+ ownedSandbox: {
2526
+ client: resolvedSandbox.established.client,
2527
+ session: resolvedSandbox.established.session,
2528
+ sessionState: resolvedSandbox.established.sessionState,
2529
+ // Pin platform setup (hooks + file materialization) to the un-proxied
2530
+ // established box — never through the routing proxy, which would
2531
+ // re-route those execs onto a machine swapped in mid-turn.
2532
+ ...setupBoxSession ? { setupSession: setupBoxSession } : {}
2310
2533
  }
2311
- await recordModelUsageAndDebitCredits(settings, db, {
2312
- accountId: input.accountId,
2313
- workspaceId: input.workspaceId,
2314
- sessionId: input.sessionId,
2315
- turnId,
2316
- model: turn.model,
2317
- isCodexTurn,
2318
- usage: responseUsage.usage,
2319
- sourceKey: modelUsageSourceKey({
2320
- responseId: responseUsage.responseId,
2321
- dispatchId,
2322
- positionalKey: `response-${responseUsageCount}`
2323
- })
2324
- });
2325
- await reconcileConversationTruth();
2326
- try {
2327
- await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
2328
- } catch (limitError) {
2329
- let serializedRunState = null;
2534
+ } : {},
2535
+ contextCompactionSignalTokens: () => lastInputTokensObserved
2536
+ });
2537
+ stream = await withCodex(runStreamOnce);
2538
+ batcher = createRuntimeBatcher(async (events) => {
2539
+ await publish(events);
2540
+ });
2541
+ const iterator = stream.toStream()[Symbol.asyncIterator]();
2542
+ let streamDone = false;
2543
+ try {
2544
+ while (true) {
2545
+ const next = await nextStreamEvent(iterator, activityContext);
2546
+ if (next.done) {
2547
+ streamDone = true;
2548
+ break;
2549
+ }
2550
+ const responseUsage = modelResponseUsageFromSdkEvent(next.value);
2551
+ if (responseUsage) {
2552
+ responseUsageCount += 1;
2553
+ const observed = responseUsage.usage?.inputTokens;
2554
+ if (typeof observed === "number" && observed > 0) {
2555
+ lastInputTokensObserved = observed;
2556
+ await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, observed).catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
2557
+ }
2558
+ await recordModelUsageAndDebitCredits(settings, db, {
2559
+ accountId: input.accountId,
2560
+ workspaceId: input.workspaceId,
2561
+ sessionId: input.sessionId,
2562
+ turnId: activeTurnId,
2563
+ model: turn.model,
2564
+ isCodexTurn,
2565
+ usage: responseUsage.usage,
2566
+ sourceKey: modelUsageSourceKey({
2567
+ responseId: responseUsage.responseId,
2568
+ dispatchId,
2569
+ positionalKey: `response-${responseUsageCount}`
2570
+ })
2571
+ });
2572
+ await reconcileConversationTruth();
2330
2573
  try {
2331
- serializedRunState = stream.state.toString();
2332
- } catch {
2333
- serializedRunState = null;
2574
+ await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
2575
+ } catch (limitError) {
2576
+ let serializedRunState = null;
2577
+ try {
2578
+ serializedRunState = stream.state.toString();
2579
+ } catch {
2580
+ serializedRunState = null;
2581
+ }
2582
+ throw new BudgetExhaustedError(
2583
+ limitError instanceof Error ? limitError.message : String(limitError),
2584
+ serializedRunState
2585
+ );
2334
2586
  }
2335
- throw new BudgetExhaustedError(
2336
- limitError instanceof Error ? limitError.message : String(limitError),
2337
- serializedRunState
2338
- );
2587
+ }
2588
+ if (activeRecording && !didComputerUse && isComputerCallStreamEvent(next.value)) {
2589
+ didComputerUse = true;
2590
+ }
2591
+ const normalized = normalizeSdkEvent(next.value);
2592
+ for (const event of normalized) {
2593
+ await batcher.push(event);
2339
2594
  }
2340
2595
  }
2341
- if (activeRecording && !didComputerUse && isComputerCallStreamEvent(next.value)) {
2342
- didComputerUse = true;
2596
+ } finally {
2597
+ if (!streamDone) {
2598
+ await iterator.return?.();
2343
2599
  }
2344
- const normalized = normalizeSdkEvent(next.value);
2345
- for (const event of normalized) {
2346
- await batcher.push(event);
2600
+ }
2601
+ await batcher.flush();
2602
+ await stream.completed.catch(() => void 0);
2603
+ if (responseUsageCount === 0) {
2604
+ const aggregateInput = stream.state.usage?.inputTokens;
2605
+ if (typeof aggregateInput === "number" && aggregateInput > 0) {
2606
+ lastInputTokensObserved = aggregateInput;
2347
2607
  }
2608
+ await recordModelUsageAndDebitCredits(settings, db, {
2609
+ accountId: input.accountId,
2610
+ workspaceId: input.workspaceId,
2611
+ sessionId: input.sessionId,
2612
+ turnId: activeTurnId,
2613
+ model: turn.model,
2614
+ isCodexTurn,
2615
+ usage: stream.state.usage,
2616
+ sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" })
2617
+ });
2348
2618
  }
2349
- } finally {
2350
- if (!streamDone) {
2351
- await iterator.return?.();
2619
+ if (lastInputTokensObserved !== null) {
2620
+ await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, lastInputTokensObserved).catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
2352
2621
  }
2353
- }
2354
- await batcher.flush();
2355
- await stream.completed.catch(() => void 0);
2356
- if (responseUsageCount === 0) {
2357
- const aggregateInput = stream.state.usage?.inputTokens;
2358
- if (typeof aggregateInput === "number" && aggregateInput > 0) {
2359
- lastInputTokensObserved = aggregateInput;
2622
+ if (stream.interruptions.length > 0) {
2623
+ await reconcileConversationTruth();
2624
+ const approvals = runtime.serializeApprovals(stream.interruptions);
2625
+ await saveRunState(db, {
2626
+ accountId: input.accountId,
2627
+ workspaceId: input.workspaceId,
2628
+ sessionId: input.sessionId,
2629
+ turnId: activeTurnId,
2630
+ serializedRunState: stream.state.toString(),
2631
+ pendingApprovals: approvals,
2632
+ // Record the account freezing this state so a resume on a DIFFERENT
2633
+ // codex account strips its account-bound reasoning before replay (HOLE C).
2634
+ frozenCodexCredentialId: effectiveCodexCredentialId
2635
+ });
2636
+ await publish([
2637
+ { type: "session.requiresAction", payload: { approvals } },
2638
+ { type: "session.status.changed", payload: { status: "requires_action" } }
2639
+ ], true);
2640
+ await finishTurn(db, input.workspaceId, activeTurnId, "requires_action");
2641
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "requires_action", activeTurnId);
2642
+ activityStatus = "requires_action";
2643
+ return { status: "requires_action" };
2360
2644
  }
2361
- await recordModelUsageAndDebitCredits(settings, db, {
2362
- accountId: input.accountId,
2363
- workspaceId: input.workspaceId,
2364
- sessionId: input.sessionId,
2365
- turnId,
2366
- model: turn.model,
2367
- isCodexTurn,
2368
- usage: stream.state.usage,
2369
- sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" })
2370
- });
2371
- }
2372
- if (lastInputTokensObserved !== null) {
2373
- await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, lastInputTokensObserved).catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
2374
- }
2375
- if (stream.interruptions.length > 0) {
2645
+ const finalOutput = String(stream.finalOutput ?? "");
2376
2646
  await reconcileConversationTruth();
2377
- const approvals = runtime.serializeApprovals(stream.interruptions);
2378
- await saveRunState(db, {
2379
- accountId: input.accountId,
2380
- workspaceId: input.workspaceId,
2381
- sessionId: input.sessionId,
2382
- turnId,
2383
- serializedRunState: stream.state.toString(),
2384
- pendingApprovals: approvals,
2385
- // Record the account freezing this state so a resume on a DIFFERENT
2386
- // codex account strips its account-bound reasoning before replay (HOLE C).
2387
- frozenCodexCredentialId: effectiveCodexCredentialId
2388
- });
2647
+ if (settings.sessionHistorySource !== "items") {
2648
+ await saveRunState(db, {
2649
+ accountId: input.accountId,
2650
+ workspaceId: input.workspaceId,
2651
+ sessionId: input.sessionId,
2652
+ turnId: activeTurnId,
2653
+ serializedRunState: stream.state.toString(),
2654
+ pendingApprovals: [],
2655
+ frozenCodexCredentialId: effectiveCodexCredentialId
2656
+ });
2657
+ }
2389
2658
  await publish([
2390
- { type: "session.requiresAction", payload: { approvals } },
2391
- { type: "session.status.changed", payload: { status: "requires_action" } }
2659
+ { type: "agent.message.completed", payload: { text: finalOutput } },
2660
+ { type: "turn.completed", payload: { output: finalOutput } },
2661
+ { type: "session.status.changed", payload: { status: "idle" } }
2392
2662
  ], true);
2393
- await finishTurn(db, input.workspaceId, turnId, "requires_action");
2394
- await setSessionStatus(db, input.workspaceId, input.sessionId, "requires_action", turnId);
2395
- activityStatus = "requires_action";
2396
- return { status: "requires_action" };
2397
- }
2398
- const finalOutput = String(stream.finalOutput ?? "");
2399
- await reconcileConversationTruth();
2400
- if (settings.sessionHistorySource !== "items") {
2401
- await saveRunState(db, {
2663
+ await finishTurn(db, input.workspaceId, activeTurnId, "idle");
2664
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2665
+ await recordUsageEvent2(db, {
2402
2666
  accountId: input.accountId,
2403
2667
  workspaceId: input.workspaceId,
2404
- sessionId: input.sessionId,
2405
- turnId,
2406
- serializedRunState: stream.state.toString(),
2407
- pendingApprovals: [],
2408
- frozenCodexCredentialId: effectiveCodexCredentialId
2668
+ eventType: "agent_run.completed",
2669
+ quantity: 1,
2670
+ unit: "run",
2671
+ sourceResourceType: "session_turn",
2672
+ sourceResourceId: activeTurnId,
2673
+ idempotencyKey: `usage:agent_run.completed:${activeTurnId}`
2409
2674
  });
2675
+ activityStatus = "idle";
2676
+ return { status: "idle" };
2677
+ };
2678
+ await prepareRunAttemptInput();
2679
+ let compactionRecoveryAttempted = false;
2680
+ let retriedAfterCompaction = false;
2681
+ while (true) {
2682
+ try {
2683
+ const result = await runStreamAttempt();
2684
+ if (retriedAfterCompaction) {
2685
+ observability.info("context compaction recovery succeeded after in-activity retry", {
2686
+ sessionId: input.sessionId,
2687
+ turnId: activeTurnId
2688
+ });
2689
+ }
2690
+ return result;
2691
+ } catch (attemptError) {
2692
+ const overflow = classifyContextWindowOverflowError(attemptError);
2693
+ const compactionNeeded = findCompactionNeededError(attemptError);
2694
+ const recoveryKind = compactionNeeded ? "proactive" : overflow ? "overflow" : null;
2695
+ if (!recoveryKind || !publish || !turnStartedPublished || compactionRecoveryAttempted) {
2696
+ throw attemptError;
2697
+ }
2698
+ compactionRecoveryAttempted = true;
2699
+ await flushRuntimeBatcher();
2700
+ await reconcileConversationTruth({ skipInputOnlyRows: true });
2701
+ const progressPersisted = modelOrToolProgressPersisted;
2702
+ observability.warn("context compaction recovery attempted", {
2703
+ sessionId: input.sessionId,
2704
+ turnId: activeTurnId,
2705
+ reason: recoveryKind,
2706
+ progressPersisted,
2707
+ code: overflow?.code,
2708
+ error: overflow?.message ?? compactionNeeded?.message,
2709
+ signalTokens: compactionNeeded?.signalTokens,
2710
+ thresholdTokens: compactionNeeded?.thresholdTokens
2711
+ });
2712
+ let compacted = false;
2713
+ try {
2714
+ const outcome = await forceContextCompaction(recoveryKind);
2715
+ compacted = outcome.compacted;
2716
+ } catch (compactError) {
2717
+ observability.warn("context compaction recovery compaction failed", {
2718
+ sessionId: input.sessionId,
2719
+ turnId: activeTurnId,
2720
+ error: compactError instanceof Error ? compactError.message : String(compactError)
2721
+ });
2722
+ }
2723
+ if (progressPersisted) {
2724
+ const triggerPayload = trigger.payload && typeof trigger.payload === "object" && !Array.isArray(trigger.payload) ? trigger.payload : {};
2725
+ const compactionResumedTurn = triggerType === "turn.preempted" && (triggerPayload.reason === "context_compacted" || triggerPayload.reason === "context_overflow_compacted");
2726
+ const canAutoContinue = compacted && !compactionResumedTurn && settings.sessionHistorySource === "items" && activeTurnId != null;
2727
+ if (canAutoContinue) {
2728
+ const [preemptedEvent] = await appendAndPublishEvents2(db, bus, input.workspaceId, input.sessionId, [
2729
+ {
2730
+ turnId: activeTurnId,
2731
+ type: "turn.preempted",
2732
+ payload: {
2733
+ triggerEventId: input.triggerEventId,
2734
+ reason: "context_compacted",
2735
+ resumeWithNotice: true,
2736
+ text: CONTEXT_OVERFLOW_RESUME_TEXT
2737
+ }
2738
+ },
2739
+ { turnId: activeTurnId, type: "session.status.changed", payload: { status: "queued" } }
2740
+ ]);
2741
+ await requeuePreemptedTurn(db, input.workspaceId, activeTurnId, preemptedEvent ? preemptedEvent.id : input.triggerEventId);
2742
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
2743
+ activityStatus = "preempted";
2744
+ observability.info("context compaction recovery succeeded by compacting and auto-continuing", {
2745
+ sessionId: input.sessionId,
2746
+ turnId: activeTurnId,
2747
+ reason: recoveryKind,
2748
+ compacted
2749
+ });
2750
+ return { status: "preempted" };
2751
+ }
2752
+ await publish([
2753
+ {
2754
+ type: "turn.failed",
2755
+ payload: {
2756
+ error: CONTEXT_WINDOW_OVERFLOW_RECOVERY_MESSAGE,
2757
+ code: recoveryKind === "overflow" ? "context_window_overflow_compacted" : "context_compacted",
2758
+ retryable: false,
2759
+ recovery: "user_message",
2760
+ compacted
2761
+ }
2762
+ },
2763
+ { type: "session.status.changed", payload: { status: "idle" } }
2764
+ ], true);
2765
+ await finishTurn(db, input.workspaceId, activeTurnId, "failed");
2766
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2767
+ activityStatus = "idle";
2768
+ activityError = attemptError;
2769
+ observability.info("context compaction recovery succeeded by compacting and idling", {
2770
+ sessionId: input.sessionId,
2771
+ turnId: activeTurnId,
2772
+ reason: recoveryKind,
2773
+ compacted
2774
+ });
2775
+ return { status: "idle" };
2776
+ }
2777
+ retriedAfterCompaction = true;
2778
+ observability.info("context compaction recovery retrying turn after compaction", {
2779
+ sessionId: input.sessionId,
2780
+ turnId: activeTurnId,
2781
+ reason: recoveryKind,
2782
+ compacted
2783
+ });
2784
+ await prepareRunAttemptInput();
2785
+ }
2410
2786
  }
2411
- await publish([
2412
- { type: "agent.message.completed", payload: { text: finalOutput } },
2413
- { type: "turn.completed", payload: { output: finalOutput } },
2414
- { type: "session.status.changed", payload: { status: "idle" } }
2415
- ], true);
2416
- await finishTurn(db, input.workspaceId, turnId, "idle");
2417
- await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2418
- await recordUsageEvent2(db, {
2419
- accountId: input.accountId,
2420
- workspaceId: input.workspaceId,
2421
- eventType: "agent_run.completed",
2422
- quantity: 1,
2423
- unit: "run",
2424
- sourceResourceType: "session_turn",
2425
- sourceResourceId: turnId,
2426
- idempotencyKey: `usage:agent_run.completed:${turnId}`
2427
- });
2428
- activityStatus = "idle";
2429
- return { status: "idle" };
2430
2787
  } catch (error) {
2431
2788
  const preemptTurnId = turnId ?? input.turnId;
2432
2789
  if (error instanceof SandboxLeaseSupersededError2 && preemptTurnId) {
@@ -2441,7 +2798,7 @@ function createRunAgentTurnActivity(services) {
2441
2798
  }
2442
2799
  if (isWorkerShutdownCancellation(error) && preemptTurnId) {
2443
2800
  try {
2444
- await batcher?.flush().catch(() => void 0);
2801
+ await flushRuntimeBatcher();
2445
2802
  await reconcileConversationTruth();
2446
2803
  const approvalRerun = triggerType === "user.approvalDecision";
2447
2804
  let resumeWithNotice = !approvalRerun && settings.sessionHistorySource === "items" && persistedHistoryCount > historyCountAtTurnStart;
@@ -2489,7 +2846,7 @@ function createRunAgentTurnActivity(services) {
2489
2846
  if (error instanceof CancelledFailure) {
2490
2847
  activityStatus = "cancelled";
2491
2848
  activityError = error;
2492
- await batcher?.flush().catch(() => void 0);
2849
+ await flushRuntimeBatcher();
2493
2850
  if (preemptTurnId) {
2494
2851
  await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => void 0);
2495
2852
  }
@@ -2497,7 +2854,7 @@ function createRunAgentTurnActivity(services) {
2497
2854
  }
2498
2855
  const maxTurns = maxTurnsExceededRunState(error);
2499
2856
  if (maxTurns && publish && turnId && turnStartedPublished) {
2500
- await batcher?.flush().catch(() => void 0);
2857
+ await flushRuntimeBatcher();
2501
2858
  await reconcileConversationTruth();
2502
2859
  const runStateSaved = Boolean(maxTurns.serializedRunState) && settings.sessionHistorySource !== "items";
2503
2860
  if (maxTurns.serializedRunState && settings.sessionHistorySource !== "items") {
@@ -2534,7 +2891,7 @@ function createRunAgentTurnActivity(services) {
2534
2891
  if (usageLimit && publish && turnId && turnStartedPublished) {
2535
2892
  const goal = await getSessionGoal3(db, input.workspaceId, input.sessionId).catch(() => null);
2536
2893
  const goalActive = Boolean(goal && goal.status === "active");
2537
- await batcher?.flush().catch(() => void 0);
2894
+ await flushRuntimeBatcher();
2538
2895
  await reconcileConversationTruth();
2539
2896
  const serializedRunState = agentsErrorRunState(error);
2540
2897
  const runStateSaved = Boolean(serializedRunState) && settings.sessionHistorySource !== "items";
@@ -2618,7 +2975,7 @@ function createRunAgentTurnActivity(services) {
2618
2975
  return { status: "idle" };
2619
2976
  }
2620
2977
  if (error instanceof BudgetExhaustedError && publish && turnId && turnStartedPublished) {
2621
- await batcher?.flush().catch(() => void 0);
2978
+ await flushRuntimeBatcher();
2622
2979
  await reconcileConversationTruth();
2623
2980
  const runStateSaved = Boolean(error.serializedRunState) && settings.sessionHistorySource !== "items";
2624
2981
  if (error.serializedRunState && settings.sessionHistorySource !== "items") {
@@ -2655,7 +3012,7 @@ function createRunAgentTurnActivity(services) {
2655
3012
  if (failure.retryable && publish && turnId && turnStartedPublished) {
2656
3013
  const goal = await getSessionGoal3(db, input.workspaceId, input.sessionId).catch(() => null);
2657
3014
  const goalActive = Boolean(goal && goal.status === "active");
2658
- await batcher?.flush().catch(() => void 0);
3015
+ await flushRuntimeBatcher();
2659
3016
  await reconcileConversationTruth();
2660
3017
  const serializedRunState = agentsErrorRunState(error);
2661
3018
  const runStateSaved = Boolean(serializedRunState) && settings.sessionHistorySource !== "items";
@@ -3062,6 +3419,7 @@ import {
3062
3419
  confirmDrainCold,
3063
3420
  forceDrainOverLimitViewerOnlyBoxes,
3064
3421
  getBillingBalance as getBillingBalance4,
3422
+ listLiveModalSandboxLeaseAttributions,
3065
3423
  listMeterableWarmLeases,
3066
3424
  persistDrainSnapshot,
3067
3425
  readLease as readLease2,
@@ -3073,14 +3431,17 @@ import {
3073
3431
  createSandboxClientForBackend,
3074
3432
  deletePriorPersistedSnapshot,
3075
3433
  deserializeSandboxSessionStateEnvelope,
3076
- isProviderSandboxNotFoundError
3434
+ isProviderSandboxNotFoundError,
3435
+ sweepModalOrphanSandboxes,
3436
+ terminateModalSandboxById
3077
3437
  } from "@opengeni/runtime";
3078
3438
  function createSandboxLeaseActivities(services, options = {}) {
3079
3439
  const terminateBox = options.terminateBox ?? terminateProviderBox;
3440
+ const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
3080
3441
  async function reapSandboxLeases2() {
3081
3442
  const { db, settings, observability } = await services();
3082
3443
  if (!settings.sandboxOwnershipEnabled) {
3083
- return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0 };
3444
+ return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
3084
3445
  }
3085
3446
  const metered = await accrueWarmTick(db, settings, observability);
3086
3447
  const forceDrained = await forceDrainOverLimitWorkspaces(db, settings, metered.workspaceIds, observability);
@@ -3090,6 +3451,7 @@ function createSandboxLeaseActivities(services, options = {}) {
3090
3451
  });
3091
3452
  let terminated = 0;
3092
3453
  let skipped = 0;
3454
+ let modalOrphansTerminated = 0;
3093
3455
  for (const row of drainable) {
3094
3456
  try {
3095
3457
  const drainedCold = await terminateDrainableBox(db, settings, row, observability, terminateBox);
@@ -3108,16 +3470,24 @@ function createSandboxLeaseActivities(services, options = {}) {
3108
3470
  });
3109
3471
  }
3110
3472
  }
3111
- if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0) {
3473
+ try {
3474
+ modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
3475
+ } catch (error) {
3476
+ observability.warn("sandbox reaper: Modal orphan sweep failed", {
3477
+ error: error instanceof Error ? error.message : String(error)
3478
+ });
3479
+ }
3480
+ if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
3112
3481
  observability.info("sandbox reaper swept", {
3113
3482
  drainable: drainable.length,
3114
3483
  terminated,
3115
3484
  skipped,
3116
3485
  metered: metered.accrued,
3117
- forceDrained
3486
+ forceDrained,
3487
+ modalOrphansTerminated
3118
3488
  });
3119
3489
  }
3120
- return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained };
3490
+ return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained, modalOrphansTerminated };
3121
3491
  }
3122
3492
  return { reapSandboxLeases: reapSandboxLeases2 };
3123
3493
  }
@@ -3193,6 +3563,37 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
3193
3563
  }
3194
3564
  return forceDrained;
3195
3565
  }
3566
+ async function sweepModalOrphansForConfiguredBackend(settings, db, observability) {
3567
+ if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret) {
3568
+ return 0;
3569
+ }
3570
+ let liveLeases;
3571
+ try {
3572
+ liveLeases = await listLiveModalSandboxLeaseAttributions(db);
3573
+ } catch (error) {
3574
+ observability.warn("sandbox reaper: live Modal lease attribution read failed", {
3575
+ error: error instanceof Error ? error.message : String(error)
3576
+ });
3577
+ return 0;
3578
+ }
3579
+ const result = await sweepModalOrphanSandboxes(settings, liveLeases);
3580
+ for (const terminated of result.terminated) {
3581
+ observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
3582
+ sandboxId: terminated.sandboxId,
3583
+ reason: terminated.reason,
3584
+ tags: JSON.stringify(terminated.tags)
3585
+ });
3586
+ }
3587
+ if (result.examined > 0) {
3588
+ observability.info("sandbox reaper: Modal orphan sweep completed", {
3589
+ examined: result.examined,
3590
+ terminated: result.terminated.length,
3591
+ skipped: result.skipped,
3592
+ appName: settings.modalAppName
3593
+ });
3594
+ }
3595
+ return result.terminated.length;
3596
+ }
3196
3597
  async function terminateDrainableBox(db, settings, row, observability, terminateBox) {
3197
3598
  const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
3198
3599
  const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId);
@@ -3233,6 +3634,30 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
3233
3634
  });
3234
3635
  return true;
3235
3636
  }
3637
+ if (backend === "modal" && !lease.resumeState && lease.instanceId) {
3638
+ const { wrote } = await persistArchive(null);
3639
+ if (!wrote) {
3640
+ observability.info("sandbox reaper: Modal lease re-armed before direct terminate \u2014 leaving sandbox RUNNING", {
3641
+ sandboxGroupId: lease.sandboxGroupId,
3642
+ backend,
3643
+ instanceId: lease.instanceId
3644
+ });
3645
+ return false;
3646
+ }
3647
+ try {
3648
+ await terminateModalSandboxById(settings, lease.instanceId);
3649
+ } catch (error) {
3650
+ if (!isProviderSandboxNotFoundError("modal", error)) {
3651
+ throw error;
3652
+ }
3653
+ observability.info("sandbox reaper: Modal sandbox already gone during direct terminate", {
3654
+ sandboxGroupId: lease.sandboxGroupId,
3655
+ backend,
3656
+ instanceId: lease.instanceId
3657
+ });
3658
+ }
3659
+ return true;
3660
+ }
3236
3661
  if (!lease.resumeState) {
3237
3662
  return true;
3238
3663
  }