@opengeni/worker-bundle 0.2.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -0
- package/dist/index.js +685 -260
- package/dist/index.js.map +1 -1
- package/package.json +8 -8
- package/src/activities/agent-turn.ts +474 -231
- package/src/activities/capabilities.ts +42 -0
- package/src/activities/context-compaction.ts +51 -58
- package/src/activities/run-input.ts +2 -2
- package/src/activities/sandbox-lease.ts +97 -3
- package/src/sandbox-resume.ts +178 -59
package/dist/index.js
CHANGED
|
@@ -44,7 +44,7 @@ import {
|
|
|
44
44
|
consumeSessionCompactionRequest,
|
|
45
45
|
countSessionHistoryItems,
|
|
46
46
|
getActiveSessionHistoryItems as getActiveSessionHistoryItems3,
|
|
47
|
-
nextSessionHistoryPosition,
|
|
47
|
+
nextSessionHistoryPosition as nextSessionHistoryPosition2,
|
|
48
48
|
requeuePreemptedTurn,
|
|
49
49
|
saveRunState,
|
|
50
50
|
upsertSandboxSessionEnvelope,
|
|
@@ -62,8 +62,10 @@ import {
|
|
|
62
62
|
maxTurnsExceededRunState,
|
|
63
63
|
modelResponseUsageFromSdkEvent,
|
|
64
64
|
normalizeSdkEvent,
|
|
65
|
-
sanitizeHistoryItemsForModel,
|
|
66
|
-
summarizeForCompaction as summarizeForCompaction2
|
|
65
|
+
sanitizeHistoryItemsForModel as sanitizeHistoryItemsForModel2,
|
|
66
|
+
summarizeForCompaction as summarizeForCompaction2,
|
|
67
|
+
findCompactionNeededError,
|
|
68
|
+
SUMMARY_BUFFER_TOKENS as SUMMARY_BUFFER_TOKENS2
|
|
67
69
|
} from "@opengeni/runtime";
|
|
68
70
|
import { calculateModelUsageCostMicros, configuredModelPricing, configuredStaticUsageLimits as configuredStaticUsageLimits2, sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
|
|
69
71
|
import { CancelledFailure } from "@temporalio/activity";
|
|
@@ -83,12 +85,46 @@ import {
|
|
|
83
85
|
import {
|
|
84
86
|
decryptedCapabilityHeaders,
|
|
85
87
|
listEnabledMcpCapabilityServers,
|
|
88
|
+
listSessionMcpServerMetadata,
|
|
89
|
+
listSessionMcpServersForRun,
|
|
86
90
|
workspaceCodexSubscriptionActive
|
|
87
91
|
} from "@opengeni/db";
|
|
88
92
|
async function settingsWithEnabledCapabilityMcpServers(db, workspaceId, settings) {
|
|
89
93
|
const enabled = await listEnabledMcpCapabilityServers(db, workspaceId);
|
|
90
94
|
return settingsWithMcpCapabilityServers(settings, enabled);
|
|
91
95
|
}
|
|
96
|
+
async function settingsWithSessionMcpServersForRun(db, workspaceId, sessionId, settings) {
|
|
97
|
+
const encryptionKey = environmentsEncryptionKeyBytes(settings);
|
|
98
|
+
if (!encryptionKey) {
|
|
99
|
+
const metadata = await listSessionMcpServerMetadata(db, workspaceId, sessionId);
|
|
100
|
+
if (metadata.length === 0) {
|
|
101
|
+
return settings;
|
|
102
|
+
}
|
|
103
|
+
throw new Error("session MCP server credentials require OPENGENI_ENVIRONMENTS_ENCRYPTION_KEY");
|
|
104
|
+
}
|
|
105
|
+
return settingsWithSessionMcpServers(settings, await listSessionMcpServersForRun(db, workspaceId, sessionId, encryptionKey));
|
|
106
|
+
}
|
|
107
|
+
function settingsWithSessionMcpServers(settings, servers) {
|
|
108
|
+
if (servers.length === 0) {
|
|
109
|
+
return settings;
|
|
110
|
+
}
|
|
111
|
+
const sessionIds = new Set(servers.map((server) => server.id));
|
|
112
|
+
return {
|
|
113
|
+
...settings,
|
|
114
|
+
mcpServers: [
|
|
115
|
+
...settings.mcpServers.filter((server) => !sessionIds.has(server.id)),
|
|
116
|
+
...servers.map((server) => ({
|
|
117
|
+
id: server.id,
|
|
118
|
+
...server.name ? { name: server.name } : {},
|
|
119
|
+
url: server.url,
|
|
120
|
+
...server.allowedTools ? { allowedTools: server.allowedTools } : {},
|
|
121
|
+
...server.timeoutMs ? { timeoutMs: server.timeoutMs } : {},
|
|
122
|
+
cacheToolsList: server.cacheToolsList ?? false,
|
|
123
|
+
headers: server.headers
|
|
124
|
+
}))
|
|
125
|
+
]
|
|
126
|
+
};
|
|
127
|
+
}
|
|
92
128
|
async function settingsWithCodexCredential(db, workspaceId, settings, activeOverride) {
|
|
93
129
|
const active = activeOverride ?? await workspaceCodexSubscriptionActive(db, settings, workspaceId);
|
|
94
130
|
if (!active) {
|
|
@@ -324,19 +360,19 @@ function workflowIdForSession(sessionId) {
|
|
|
324
360
|
// src/activities/context-compaction.ts
|
|
325
361
|
import {
|
|
326
362
|
applyContextCompaction,
|
|
327
|
-
getActiveSessionHistoryItems
|
|
363
|
+
getActiveSessionHistoryItems,
|
|
364
|
+
nextSessionHistoryPosition
|
|
328
365
|
} from "@opengeni/db";
|
|
329
366
|
import {
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
367
|
+
SUMMARY_BUFFER_TOKENS,
|
|
368
|
+
buildCompactionPromptInput,
|
|
369
|
+
buildCompactionReplacementHistory,
|
|
370
|
+
decideClientCompaction,
|
|
371
|
+
sanitizeHistoryItemsForModel,
|
|
333
372
|
summarizeForCompaction
|
|
334
373
|
} from "@opengeni/runtime";
|
|
335
|
-
import {
|
|
336
|
-
|
|
337
|
-
resolveContextCompactionMode
|
|
338
|
-
} from "@opengeni/config";
|
|
339
|
-
async function maybeCompactContext(db, settings, scope, lastInputTokens, summarize = (s, m) => summarizeForCompaction(s, m, { maxOutputTokens: s.contextSummaryMaxTokens }), options = {}) {
|
|
374
|
+
import { resolveContextCompactionMode } from "@opengeni/config";
|
|
375
|
+
async function maybeCompactContext(db, settings, scope, lastInputTokens, summarize = (s, m) => summarizeForCompaction(s, m, { maxOutputTokens: SUMMARY_BUFFER_TOKENS }), options = {}) {
|
|
340
376
|
if (resolveContextCompactionMode(settings) !== "client") {
|
|
341
377
|
return { compacted: false, reason: "mode_not_client" };
|
|
342
378
|
}
|
|
@@ -344,48 +380,50 @@ async function maybeCompactContext(db, settings, scope, lastInputTokens, summari
|
|
|
344
380
|
if (active.length === 0) {
|
|
345
381
|
return { compacted: false, reason: "no_history" };
|
|
346
382
|
}
|
|
347
|
-
const items = active.map((row) => row.item);
|
|
348
|
-
const
|
|
383
|
+
const items = sanitizeHistoryItemsForModel(active.map((row) => row.item));
|
|
384
|
+
const decision = decideClientCompaction({
|
|
349
385
|
items,
|
|
350
386
|
lastInputTokens,
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
hardFraction: settings.contextCompactHardFraction,
|
|
354
|
-
keepRecentTokens: settings.contextKeepRecentTokens,
|
|
387
|
+
contextWindowTokens: settings.contextWindowTokens,
|
|
388
|
+
contextReservedOutputTokens: settings.contextReservedOutputTokens,
|
|
355
389
|
...options.force ? { force: true } : {}
|
|
356
390
|
});
|
|
357
|
-
if (!
|
|
358
|
-
return { compacted: false, reason:
|
|
359
|
-
}
|
|
360
|
-
if (plan.hardForced) {
|
|
361
|
-
console.warn(
|
|
362
|
-
`context compaction HARD-FORCED for session ${scope.sessionId}: signal ${plan.signalTokens} tokens at/over hard ceiling`
|
|
363
|
-
);
|
|
391
|
+
if (!decision.shouldCompact) {
|
|
392
|
+
return { compacted: false, reason: decision.reason };
|
|
364
393
|
}
|
|
365
|
-
const
|
|
366
|
-
const summaryBody = await summarize(settings,
|
|
394
|
+
const promptInput = buildCompactionPromptInput(items);
|
|
395
|
+
const summaryBody = await summarize(settings, promptInput);
|
|
367
396
|
if (!summaryBody) {
|
|
368
397
|
return { compacted: false, reason: "summarize_failed" };
|
|
369
398
|
}
|
|
370
|
-
const
|
|
371
|
-
|
|
372
|
-
|
|
399
|
+
const replacementHistory = buildCompactionReplacementHistory(items, summaryBody);
|
|
400
|
+
const summaryItem = replacementHistory.at(-1);
|
|
401
|
+
if (!summaryItem) {
|
|
402
|
+
return { compacted: false, reason: "empty_replacement_history" };
|
|
373
403
|
}
|
|
374
|
-
const
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
404
|
+
const nextPosition = await nextSessionHistoryPosition(db, scope.workspaceId, scope.sessionId);
|
|
405
|
+
const replacementItems = replacementHistory.slice(0, -1).map((item, index) => ({
|
|
406
|
+
position: nextPosition + index,
|
|
407
|
+
item
|
|
408
|
+
}));
|
|
409
|
+
const summaryPosition = nextPosition + replacementItems.length;
|
|
379
410
|
await applyContextCompaction(db, {
|
|
380
411
|
accountId: scope.accountId,
|
|
381
412
|
workspaceId: scope.workspaceId,
|
|
382
413
|
sessionId: scope.sessionId,
|
|
383
414
|
turnId: scope.turnId ?? null,
|
|
384
|
-
boundaryPosition,
|
|
415
|
+
boundaryPosition: nextPosition,
|
|
416
|
+
replacementItems,
|
|
385
417
|
summaryPosition,
|
|
386
|
-
summaryItem
|
|
418
|
+
summaryItem
|
|
387
419
|
});
|
|
388
|
-
return {
|
|
420
|
+
return {
|
|
421
|
+
compacted: true,
|
|
422
|
+
supersededFrom: nextPosition,
|
|
423
|
+
summaryPosition,
|
|
424
|
+
signalTokens: decision.signalTokens,
|
|
425
|
+
thresholdTokens: decision.thresholdTokens
|
|
426
|
+
};
|
|
389
427
|
}
|
|
390
428
|
|
|
391
429
|
// src/activities/environment.ts
|
|
@@ -886,7 +924,7 @@ function isPlainObject(value) {
|
|
|
886
924
|
}
|
|
887
925
|
|
|
888
926
|
// src/activities/run-input.ts
|
|
889
|
-
import { contextInputBudgetTokens
|
|
927
|
+
import { contextInputBudgetTokens, resolveContextCompactionMode as resolveContextCompactionMode2 } from "@opengeni/config";
|
|
890
928
|
import {
|
|
891
929
|
getActiveSessionHistoryItems as getActiveSessionHistoryItems2,
|
|
892
930
|
getLatestRunState,
|
|
@@ -1030,7 +1068,7 @@ function readPathBudgetTokens(settings) {
|
|
|
1030
1068
|
if (!settings || resolveContextCompactionMode2(settings) !== "client") {
|
|
1031
1069
|
return void 0;
|
|
1032
1070
|
}
|
|
1033
|
-
const budget =
|
|
1071
|
+
const budget = contextInputBudgetTokens(settings);
|
|
1034
1072
|
return budget > 0 ? budget : void 0;
|
|
1035
1073
|
}
|
|
1036
1074
|
async function userMessageTextWithAttachments(db, workspaceId, text, resources) {
|
|
@@ -1132,6 +1170,7 @@ import {
|
|
|
1132
1170
|
failWarmingToCold,
|
|
1133
1171
|
getSandboxSessionEnvelope as getSandboxSessionEnvelope2,
|
|
1134
1172
|
readLease,
|
|
1173
|
+
recordWarmingSandboxCreated,
|
|
1135
1174
|
releaseLeaseHolder,
|
|
1136
1175
|
SandboxLeaseSupersededError
|
|
1137
1176
|
} from "@opengeni/db";
|
|
@@ -1143,13 +1182,84 @@ import {
|
|
|
1143
1182
|
DisplayStackError,
|
|
1144
1183
|
DisplayStackUnsupportedError,
|
|
1145
1184
|
buildStreamUrl,
|
|
1146
|
-
StreamPortUnavailableError
|
|
1185
|
+
StreamPortUnavailableError,
|
|
1186
|
+
modalSandboxAttributionEnvironment,
|
|
1187
|
+
tagModalSandbox
|
|
1147
1188
|
} from "@opengeni/runtime";
|
|
1148
1189
|
import { DESKTOP_STREAM_PORT } from "@opengeni/contracts";
|
|
1190
|
+
var SandboxWarmingTimeoutError = class extends Error {
|
|
1191
|
+
constructor(backend, timeoutMs) {
|
|
1192
|
+
super(
|
|
1193
|
+
`Sandbox backend "${backend}" capacity or creation timed out after ${Math.ceil(timeoutMs / 1e3)}s while warming the sandbox lease. Please try again; if this persists, sandbox capacity may be exhausted.`
|
|
1194
|
+
);
|
|
1195
|
+
this.backend = backend;
|
|
1196
|
+
this.timeoutMs = timeoutMs;
|
|
1197
|
+
this.name = "SandboxWarmingTimeoutError";
|
|
1198
|
+
}
|
|
1199
|
+
backend;
|
|
1200
|
+
timeoutMs;
|
|
1201
|
+
code = "sandbox_warming_timeout";
|
|
1202
|
+
};
|
|
1149
1203
|
var WARMING_POLL_INTERVAL_MS = 250;
|
|
1150
1204
|
async function sleep(ms) {
|
|
1151
1205
|
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
1152
1206
|
}
|
|
1207
|
+
async function terminateEstablishedSandbox(established) {
|
|
1208
|
+
if (!established) {
|
|
1209
|
+
return true;
|
|
1210
|
+
}
|
|
1211
|
+
const client = established.client;
|
|
1212
|
+
if (typeof client.delete === "function" && established.sessionState !== void 0) {
|
|
1213
|
+
try {
|
|
1214
|
+
await client.delete(established.sessionState);
|
|
1215
|
+
return true;
|
|
1216
|
+
} catch {
|
|
1217
|
+
return false;
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1220
|
+
const session = established.session;
|
|
1221
|
+
try {
|
|
1222
|
+
if (session.terminate) {
|
|
1223
|
+
await session.terminate();
|
|
1224
|
+
return true;
|
|
1225
|
+
} else if (session.kill) {
|
|
1226
|
+
await session.kill();
|
|
1227
|
+
return true;
|
|
1228
|
+
} else if (session.close) {
|
|
1229
|
+
if (!session.closed) {
|
|
1230
|
+
await session.close();
|
|
1231
|
+
}
|
|
1232
|
+
return true;
|
|
1233
|
+
}
|
|
1234
|
+
return false;
|
|
1235
|
+
} catch {
|
|
1236
|
+
return false;
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
function asSandboxWarmingError(error, backend, timeoutMs) {
|
|
1240
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1241
|
+
return /sandbox creation timed out|warming timed out|capacity.*timed out/i.test(message) ? new SandboxWarmingTimeoutError(backend, timeoutMs) : error;
|
|
1242
|
+
}
|
|
1243
|
+
function workspaceArchiveFromEnvelope(envelope) {
|
|
1244
|
+
const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
|
|
1245
|
+
const archive = sessionState?.workspaceArchive;
|
|
1246
|
+
return typeof archive === "string" && archive.length > 0 ? archive : null;
|
|
1247
|
+
}
|
|
1248
|
+
function preserveWorkspaceArchiveOnInterimResumeState(resumeState, archiveSource) {
|
|
1249
|
+
const archive = workspaceArchiveFromEnvelope(archiveSource);
|
|
1250
|
+
if (!archive) {
|
|
1251
|
+
return resumeState;
|
|
1252
|
+
}
|
|
1253
|
+
const existingSessionState = resumeState && typeof resumeState.sessionState === "object" && resumeState.sessionState !== null ? resumeState.sessionState : {};
|
|
1254
|
+
return {
|
|
1255
|
+
...resumeState ?? {},
|
|
1256
|
+
...resumeState?.backendId === void 0 && archiveSource?.backendId !== void 0 ? { backendId: archiveSource.backendId } : {},
|
|
1257
|
+
sessionState: {
|
|
1258
|
+
...existingSessionState,
|
|
1259
|
+
workspaceArchive: archive
|
|
1260
|
+
}
|
|
1261
|
+
};
|
|
1262
|
+
}
|
|
1153
1263
|
async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
1154
1264
|
const { db, settings } = services;
|
|
1155
1265
|
const os = ids.os ?? "linux";
|
|
@@ -1191,14 +1301,50 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
1191
1301
|
}
|
|
1192
1302
|
if (acquired.role === "spawner") {
|
|
1193
1303
|
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
1304
|
+
let createdEstablished = null;
|
|
1305
|
+
if (ids.environment && ids.backend === "modal") {
|
|
1306
|
+
Object.assign(ids.environment, modalSandboxAttributionEnvironment({
|
|
1307
|
+
leaseId: acquired.lease.id,
|
|
1308
|
+
workspaceId: ids.workspaceId,
|
|
1309
|
+
sandboxGroupId: ids.sandboxGroupId
|
|
1310
|
+
}));
|
|
1311
|
+
}
|
|
1194
1312
|
try {
|
|
1195
1313
|
const envelope = await getSandboxSessionEnvelope2(db, ids.workspaceId, ids.sessionId);
|
|
1196
1314
|
const spawnEnvelope = acquired.lease.resumeState ?? envelope;
|
|
1197
1315
|
const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
|
|
1198
1316
|
sessionId: ids.sessionId,
|
|
1199
1317
|
backendOverride: ids.backend,
|
|
1200
|
-
...ids.environment ? { environment: ids.environment } : {}
|
|
1318
|
+
...ids.environment ? { environment: ids.environment } : {},
|
|
1319
|
+
onSandboxCreated: async (created) => {
|
|
1320
|
+
createdEstablished = created;
|
|
1321
|
+
const resumeEnvelope2 = preserveWorkspaceArchiveOnInterimResumeState(
|
|
1322
|
+
await serializeEstablishedSandboxEnvelope(created) ?? null,
|
|
1323
|
+
spawnEnvelope
|
|
1324
|
+
);
|
|
1325
|
+
const recorded = await recordWarmingSandboxCreated(db, {
|
|
1326
|
+
accountId: ids.accountId,
|
|
1327
|
+
workspaceId: ids.workspaceId,
|
|
1328
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1329
|
+
expectedEpoch,
|
|
1330
|
+
instanceId: created.instanceId,
|
|
1331
|
+
resumeBackendId: created.backendId,
|
|
1332
|
+
resumeState: resumeEnvelope2,
|
|
1333
|
+
leaseTtlMs
|
|
1334
|
+
});
|
|
1335
|
+
if (!recorded.recorded) {
|
|
1336
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
1337
|
+
}
|
|
1338
|
+
if (created.backendId === "modal") {
|
|
1339
|
+
await tagModalSandbox(settings, created.instanceId, {
|
|
1340
|
+
leaseId: acquired.lease.id,
|
|
1341
|
+
workspaceId: ids.workspaceId,
|
|
1342
|
+
sandboxGroupId: ids.sandboxGroupId
|
|
1343
|
+
}).catch(() => void 0);
|
|
1344
|
+
}
|
|
1345
|
+
}
|
|
1201
1346
|
});
|
|
1347
|
+
createdEstablished = established;
|
|
1202
1348
|
await ensureDisplayStack(settings, established);
|
|
1203
1349
|
const endpoint = await exposeStreamPort(settings, established);
|
|
1204
1350
|
const resumeEnvelope = await serializeEstablishedSandboxEnvelope(established) ?? envelope ?? null;
|
|
@@ -1214,29 +1360,35 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
1214
1360
|
leaseTtlMs
|
|
1215
1361
|
});
|
|
1216
1362
|
if (!committed.committed || !committed.lease) {
|
|
1363
|
+
await terminateEstablishedSandbox(established);
|
|
1217
1364
|
await release();
|
|
1218
1365
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
1219
1366
|
}
|
|
1220
1367
|
return { established, leaseEpoch: committed.lease.leaseEpoch, release };
|
|
1221
1368
|
} catch (error) {
|
|
1222
1369
|
if (error instanceof SandboxLeaseSupersededError) {
|
|
1370
|
+
await terminateEstablishedSandbox(createdEstablished);
|
|
1371
|
+
await release();
|
|
1223
1372
|
throw error;
|
|
1224
1373
|
}
|
|
1225
|
-
await
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1374
|
+
const terminated = await terminateEstablishedSandbox(createdEstablished);
|
|
1375
|
+
if (terminated) {
|
|
1376
|
+
await failWarmingToCold(db, {
|
|
1377
|
+
accountId: ids.accountId,
|
|
1378
|
+
workspaceId: ids.workspaceId,
|
|
1379
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1380
|
+
expectedEpoch
|
|
1381
|
+
});
|
|
1382
|
+
}
|
|
1231
1383
|
await release();
|
|
1232
|
-
throw error;
|
|
1384
|
+
throw asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
1233
1385
|
}
|
|
1234
1386
|
}
|
|
1235
|
-
let leaseEpoch = acquired.lease.leaseEpoch;
|
|
1236
|
-
if (acquired.lease.liveness === "warming") {
|
|
1237
|
-
leaseEpoch = (await waitForWarmOrReacquire(services, ids, kind, holderId)).leaseEpoch;
|
|
1238
|
-
}
|
|
1239
1387
|
try {
|
|
1388
|
+
let leaseEpoch = acquired.lease.leaseEpoch;
|
|
1389
|
+
if (acquired.lease.liveness === "warming") {
|
|
1390
|
+
leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
|
|
1391
|
+
}
|
|
1240
1392
|
const live = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
1241
1393
|
const envelope = live?.resumeState ?? await getSandboxSessionEnvelope2(db, ids.workspaceId, ids.sessionId);
|
|
1242
1394
|
const established = await establishSandboxSessionFromEnvelope(settings, envelope, {
|
|
@@ -1248,7 +1400,7 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
1248
1400
|
return { established, leaseEpoch, release };
|
|
1249
1401
|
} catch (error) {
|
|
1250
1402
|
await release();
|
|
1251
|
-
throw error;
|
|
1403
|
+
throw asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
1252
1404
|
}
|
|
1253
1405
|
}
|
|
1254
1406
|
async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
|
|
@@ -1305,37 +1457,26 @@ async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
|
|
|
1305
1457
|
}
|
|
1306
1458
|
return { leaseEpoch: acquired.lease.leaseEpoch, release };
|
|
1307
1459
|
}
|
|
1308
|
-
async function
|
|
1460
|
+
async function waitForWarm(services, ids) {
|
|
1309
1461
|
const { db, settings } = services;
|
|
1310
|
-
const deadline = Date.now() + settings.
|
|
1462
|
+
const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
|
|
1311
1463
|
while (Date.now() < deadline) {
|
|
1312
1464
|
await sleep(WARMING_POLL_INTERVAL_MS);
|
|
1313
1465
|
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
1314
1466
|
if (!lease) {
|
|
1315
|
-
|
|
1467
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
|
|
1316
1468
|
}
|
|
1317
|
-
if (lease.liveness === "warm"
|
|
1318
|
-
return {
|
|
1469
|
+
if (lease.liveness === "warm") {
|
|
1470
|
+
return { leaseEpoch: lease.leaseEpoch };
|
|
1471
|
+
}
|
|
1472
|
+
if (lease.liveness === "draining") {
|
|
1473
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
1319
1474
|
}
|
|
1320
1475
|
if (lease.liveness === "cold") {
|
|
1321
|
-
|
|
1476
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
1322
1477
|
}
|
|
1323
1478
|
}
|
|
1324
|
-
|
|
1325
|
-
accountId: ids.accountId,
|
|
1326
|
-
workspaceId: ids.workspaceId,
|
|
1327
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
1328
|
-
kind,
|
|
1329
|
-
holderId,
|
|
1330
|
-
subjectId: ids.sessionId,
|
|
1331
|
-
backend: ids.backend,
|
|
1332
|
-
os: ids.os ?? "linux",
|
|
1333
|
-
leaseTtlMs: settings.sandboxLeaseTtlMs
|
|
1334
|
-
});
|
|
1335
|
-
if (reacquired.role === "fenced") {
|
|
1336
|
-
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, reacquired.lease.leaseEpoch);
|
|
1337
|
-
}
|
|
1338
|
-
return { liveness: reacquired.lease.liveness, leaseEpoch: reacquired.lease.leaseEpoch };
|
|
1479
|
+
throw new SandboxWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
1339
1480
|
}
|
|
1340
1481
|
async function ensureDisplayStack(settings, established) {
|
|
1341
1482
|
if (!settings.sandboxDesktopEnabled) {
|
|
@@ -1635,6 +1776,10 @@ function selectCodexCredentialForTurn(args) {
|
|
|
1635
1776
|
}
|
|
1636
1777
|
return null;
|
|
1637
1778
|
}
|
|
1779
|
+
var CONTEXT_OVERFLOW_RESUME_TEXT = [
|
|
1780
|
+
"[TURN RESUMED AFTER CONTEXT COMPACTION] The conversation approached or exceeded the model's context window mid-turn; older history above was compacted into a summary.",
|
|
1781
|
+
"Continue the original task from where it left off. Before repeating any action with side effects, check whether it already happened."
|
|
1782
|
+
].join("\n");
|
|
1638
1783
|
var WORKER_SHUTDOWN_RESUME_TEXT = [
|
|
1639
1784
|
"[TURN RESUMED AFTER WORKER RESTART] The platform worker running this turn restarted (deploy/rollout) before the turn finished.",
|
|
1640
1785
|
"Your conversation history above, including this turn's earlier work, is preserved up to the last checkpoint; at most the single in-flight model step after it was lost.",
|
|
@@ -1648,6 +1793,47 @@ var WORKER_DEATH_RESUME_TEXT = [
|
|
|
1648
1793
|
function isWorkerShutdownCancellation(error) {
|
|
1649
1794
|
return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
|
|
1650
1795
|
}
|
|
1796
|
+
var CONTEXT_WINDOW_OVERFLOW_RECOVERY_MESSAGE = "The conversation reached the model context compaction threshold; it has been compacted. Send a message to continue.";
|
|
1797
|
+
function classifyContextWindowOverflowError(error) {
|
|
1798
|
+
const fields = collectErrorStrings(error);
|
|
1799
|
+
const matched = fields.find(
|
|
1800
|
+
(value) => /context[_\s-]*length[_\s-]*exceeded/i.test(value) || /exceeds?\s+(?:the\s+)?context\s+window/i.test(value) || /maximum\s+context\s+length/i.test(value) || /context\s+window[^.]*exceed/i.test(value)
|
|
1801
|
+
);
|
|
1802
|
+
if (!matched) {
|
|
1803
|
+
return null;
|
|
1804
|
+
}
|
|
1805
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1806
|
+
const code = fields.find((value) => /context[_\s-]*length[_\s-]*exceeded/i.test(value));
|
|
1807
|
+
return {
|
|
1808
|
+
message,
|
|
1809
|
+
...code ? { code } : {},
|
|
1810
|
+
...matched && matched !== message ? { detail: matched } : {}
|
|
1811
|
+
};
|
|
1812
|
+
}
|
|
1813
|
+
function collectErrorStrings(value, seen = /* @__PURE__ */ new WeakSet()) {
|
|
1814
|
+
if (typeof value === "string") {
|
|
1815
|
+
return [value];
|
|
1816
|
+
}
|
|
1817
|
+
if (!value || typeof value !== "object") {
|
|
1818
|
+
return [];
|
|
1819
|
+
}
|
|
1820
|
+
if (seen.has(value)) {
|
|
1821
|
+
return [];
|
|
1822
|
+
}
|
|
1823
|
+
seen.add(value);
|
|
1824
|
+
const out = [];
|
|
1825
|
+
const record = value;
|
|
1826
|
+
for (const key of ["message", "code", "type", "name", "param"]) {
|
|
1827
|
+
const field = record[key];
|
|
1828
|
+
if (typeof field === "string" && field.length > 0) {
|
|
1829
|
+
out.push(field);
|
|
1830
|
+
}
|
|
1831
|
+
}
|
|
1832
|
+
for (const key of ["error", "cause", "response", "data"]) {
|
|
1833
|
+
out.push(...collectErrorStrings(record[key], seen));
|
|
1834
|
+
}
|
|
1835
|
+
return out;
|
|
1836
|
+
}
|
|
1651
1837
|
function modelUsageSourceKey(input) {
|
|
1652
1838
|
if (input.responseId) {
|
|
1653
1839
|
return input.responseId;
|
|
@@ -1655,7 +1841,7 @@ function modelUsageSourceKey(input) {
|
|
|
1655
1841
|
return input.dispatchId ? `${input.dispatchId}:${input.positionalKey}` : input.positionalKey;
|
|
1656
1842
|
}
|
|
1657
1843
|
function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = persistedHistoryCount) {
|
|
1658
|
-
const sanitized =
|
|
1844
|
+
const sanitized = sanitizeHistoryItemsForModel2(rawHistory);
|
|
1659
1845
|
if (sanitized.length <= persistedHistoryCount) {
|
|
1660
1846
|
return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
|
|
1661
1847
|
}
|
|
@@ -1665,8 +1851,20 @@ function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = p
|
|
|
1665
1851
|
}));
|
|
1666
1852
|
return { rows, nextWatermark: sanitized.length, nextPosition: nextPosition + rows.length };
|
|
1667
1853
|
}
|
|
1854
|
+
function isModelOrToolProgressHistoryItem(item) {
|
|
1855
|
+
if (item.type === "message") {
|
|
1856
|
+
return item.role === "assistant";
|
|
1857
|
+
}
|
|
1858
|
+
if (item.type === "reasoning" || item.type === "compaction") {
|
|
1859
|
+
return true;
|
|
1860
|
+
}
|
|
1861
|
+
if (typeof item.type === "string") {
|
|
1862
|
+
return item.type !== "message";
|
|
1863
|
+
}
|
|
1864
|
+
return false;
|
|
1865
|
+
}
|
|
1668
1866
|
function reconcileSeedCount(activeSeedRows, modelHistoryFromItems, current) {
|
|
1669
|
-
return
|
|
1867
|
+
return sanitizeHistoryItemsForModel2(
|
|
1670
1868
|
modelHistoryFromItems ? applyCodexHistoryStrip(activeSeedRows, current) : activeSeedRows.map((row) => row.item)
|
|
1671
1869
|
).length;
|
|
1672
1870
|
}
|
|
@@ -1733,14 +1931,19 @@ function createRunAgentTurnActivity(services) {
|
|
|
1733
1931
|
let activeRecording = null;
|
|
1734
1932
|
let didComputerUse = false;
|
|
1735
1933
|
let batcher = null;
|
|
1934
|
+
const flushRuntimeBatcher = async () => {
|
|
1935
|
+
const current = batcher;
|
|
1936
|
+
await current?.flush().catch(() => void 0);
|
|
1937
|
+
};
|
|
1736
1938
|
let preparedTools = null;
|
|
1737
1939
|
let publish = null;
|
|
1738
1940
|
let turnStartedPublished = false;
|
|
1739
1941
|
let stream;
|
|
1740
1942
|
let persistedHistoryCount = 0;
|
|
1741
1943
|
let historyCountAtTurnStart = 0;
|
|
1944
|
+
let modelOrToolProgressPersisted = false;
|
|
1742
1945
|
let nextHistoryPosition = 0;
|
|
1743
|
-
const reconcileConversationTruth = async () => {
|
|
1946
|
+
const reconcileConversationTruth = async (options = {}) => {
|
|
1744
1947
|
if (!stream || !turnId) {
|
|
1745
1948
|
return;
|
|
1746
1949
|
}
|
|
@@ -1752,7 +1955,9 @@ function createRunAgentTurnActivity(services) {
|
|
|
1752
1955
|
persistedHistoryCount,
|
|
1753
1956
|
nextHistoryPosition
|
|
1754
1957
|
);
|
|
1755
|
-
|
|
1958
|
+
const hasModelOrToolProgress = rows.some((row) => isModelOrToolProgressHistoryItem(row.item));
|
|
1959
|
+
const shouldAppendRows = rows.length > 0 && (!options.skipInputOnlyRows || hasModelOrToolProgress);
|
|
1960
|
+
if (shouldAppendRows) {
|
|
1756
1961
|
await appendSessionHistoryItems(db, {
|
|
1757
1962
|
accountId: input.accountId,
|
|
1758
1963
|
workspaceId: input.workspaceId,
|
|
@@ -1766,8 +1971,13 @@ function createRunAgentTurnActivity(services) {
|
|
|
1766
1971
|
items: rows
|
|
1767
1972
|
});
|
|
1768
1973
|
}
|
|
1769
|
-
|
|
1770
|
-
|
|
1974
|
+
if (shouldAppendRows || !options.skipInputOnlyRows) {
|
|
1975
|
+
persistedHistoryCount = nextWatermark;
|
|
1976
|
+
nextHistoryPosition = nextPosition;
|
|
1977
|
+
}
|
|
1978
|
+
if (hasModelOrToolProgress) {
|
|
1979
|
+
modelOrToolProgressPersisted = true;
|
|
1980
|
+
}
|
|
1771
1981
|
}
|
|
1772
1982
|
const envelope = sandboxStateEntryFromRunState(stream.state);
|
|
1773
1983
|
if (envelope) {
|
|
@@ -1944,12 +2154,13 @@ function createRunAgentTurnActivity(services) {
|
|
|
1944
2154
|
}
|
|
1945
2155
|
const packRuntime = await resolveWorkspacePackRuntime(db, input.workspaceId);
|
|
1946
2156
|
const workspaceAgentInstructions = await resolveWorkspaceAgentInstructions(db, input.workspaceId);
|
|
1947
|
-
const
|
|
2157
|
+
const baseRunSettings = {
|
|
1948
2158
|
...settingsWithPackSandboxImage(capabilitySettings, packRuntime.sandboxImage),
|
|
1949
2159
|
openaiModel: turn.model,
|
|
1950
2160
|
openaiReasoningEffort: turn.reasoningEffort,
|
|
1951
2161
|
sandboxBackend: turn.sandboxBackend
|
|
1952
2162
|
};
|
|
2163
|
+
const runSettings = await settingsWithSessionMcpServersForRun(db, input.workspaceId, input.sessionId, baseRunSettings);
|
|
1953
2164
|
const resolvedModel = runtime.resolveTurnModel(capabilitySettings, turn.model);
|
|
1954
2165
|
const codexContext = resolvedModel?.provider.kind === "codex-subscription" ? (() => {
|
|
1955
2166
|
const resolver = buildCodexTokenResolver(db, runSettings, input.workspaceId, effectiveCodexCredentialId ?? "");
|
|
@@ -2214,6 +2425,10 @@ function createRunAgentTurnActivity(services) {
|
|
|
2214
2425
|
} : { computerToolMode: computerToolModeForTurn(null) },
|
|
2215
2426
|
...packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {},
|
|
2216
2427
|
...workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {},
|
|
2428
|
+
// Per-session persona tier (session > workspace > deployment default).
|
|
2429
|
+
// Composed system-level AFTER the workspace persona so it refines it for
|
|
2430
|
+
// this one session; absent ⇒ byte-identical to today's composition.
|
|
2431
|
+
...session.instructions ? { sessionInstructions: session.instructions } : {},
|
|
2217
2432
|
...workspaceEnvironment ? {
|
|
2218
2433
|
workspaceEnvironment: {
|
|
2219
2434
|
name: workspaceEnvironment.name,
|
|
@@ -2222,6 +2437,12 @@ function createRunAgentTurnActivity(services) {
|
|
|
2222
2437
|
}
|
|
2223
2438
|
} : {}
|
|
2224
2439
|
});
|
|
2440
|
+
const compactSummarizer = resolvedModel ? (s, m) => withCodex(() => summarizeForCompaction2(s, m, {
|
|
2441
|
+
client: resolvedModel.client,
|
|
2442
|
+
api: resolvedModel.provider.api,
|
|
2443
|
+
model: resolvedModel.configured.id,
|
|
2444
|
+
maxOutputTokens: SUMMARY_BUFFER_TOKENS2
|
|
2445
|
+
})) : void 0;
|
|
2225
2446
|
if (triggerType === "user.message" || triggerType === "goal.continuation") {
|
|
2226
2447
|
try {
|
|
2227
2448
|
const forced = await consumeSessionCompactionRequest(db, input.workspaceId, input.sessionId);
|
|
@@ -2234,199 +2455,335 @@ function createRunAgentTurnActivity(services) {
|
|
|
2234
2455
|
// registry provider, summarize on THAT provider's client + wire API
|
|
2235
2456
|
// (a chat provider can't summarize through OpenAI/Azure). Null
|
|
2236
2457
|
// resolution keeps the default built-in Responses summarizer.
|
|
2237
|
-
|
|
2238
|
-
client: resolvedModel.client,
|
|
2239
|
-
api: resolvedModel.provider.api,
|
|
2240
|
-
model: resolvedModel.configured.id,
|
|
2241
|
-
maxOutputTokens: s.contextSummaryMaxTokens
|
|
2242
|
-
})) : void 0,
|
|
2458
|
+
compactSummarizer,
|
|
2243
2459
|
forced ? { force: true } : {}
|
|
2244
2460
|
);
|
|
2245
2461
|
if (outcome.compacted) {
|
|
2246
|
-
const trigger2 = forced ? "operator" :
|
|
2462
|
+
const trigger2 = forced ? "operator" : void 0;
|
|
2247
2463
|
await publish([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, ...trigger2 ? { trigger: trigger2 } : {} } }]);
|
|
2248
2464
|
}
|
|
2249
2465
|
} catch (compactError) {
|
|
2250
2466
|
console.error("context compaction failed (turn proceeds un-compacted)", compactError);
|
|
2251
2467
|
}
|
|
2252
2468
|
}
|
|
2253
|
-
const
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
}
|
|
2289
|
-
}
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2469
|
+
const activeTurnId = turnId;
|
|
2470
|
+
if (!activeTurnId) {
|
|
2471
|
+
throw new Error("Turn id was not initialized");
|
|
2472
|
+
}
|
|
2473
|
+
let runInput = null;
|
|
2474
|
+
const prepareRunAttemptInput = async () => {
|
|
2475
|
+
const prepared = await turnInput(
|
|
2476
|
+
db,
|
|
2477
|
+
runtime,
|
|
2478
|
+
agent,
|
|
2479
|
+
trigger,
|
|
2480
|
+
runSettings,
|
|
2481
|
+
{ currentCodexCredentialId: effectiveCodexCredentialId }
|
|
2482
|
+
);
|
|
2483
|
+
runInput = prepared.input;
|
|
2484
|
+
const activeSeedRows = await getActiveSessionHistoryItems3(db, input.workspaceId, input.sessionId);
|
|
2485
|
+
persistedHistoryCount = reconcileSeedCount(
|
|
2486
|
+
activeSeedRows,
|
|
2487
|
+
prepared.modelHistoryFromItems,
|
|
2488
|
+
{ currentCodexCredentialId: effectiveCodexCredentialId }
|
|
2489
|
+
);
|
|
2490
|
+
historyCountAtTurnStart = persistedHistoryCount;
|
|
2491
|
+
nextHistoryPosition = await nextSessionHistoryPosition2(db, input.workspaceId, input.sessionId);
|
|
2492
|
+
};
|
|
2493
|
+
const forceContextCompaction = async (triggerLabel) => {
|
|
2494
|
+
const clientCompactionSettings = { ...runSettings, contextCompactionMode: "client" };
|
|
2495
|
+
const outcome = await maybeCompactContext(
|
|
2496
|
+
db,
|
|
2497
|
+
clientCompactionSettings,
|
|
2498
|
+
{ accountId: input.accountId, workspaceId: input.workspaceId, sessionId: input.sessionId, turnId: activeTurnId },
|
|
2499
|
+
session.lastInputTokens,
|
|
2500
|
+
compactSummarizer,
|
|
2501
|
+
{ force: true }
|
|
2502
|
+
);
|
|
2503
|
+
if (outcome.compacted) {
|
|
2504
|
+
await publish([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, trigger: triggerLabel } }]);
|
|
2505
|
+
}
|
|
2506
|
+
return outcome;
|
|
2507
|
+
};
|
|
2508
|
+
const runStreamAttempt = async () => {
|
|
2509
|
+
if (!runInput) {
|
|
2510
|
+
throw new Error("Run input was not prepared");
|
|
2511
|
+
}
|
|
2512
|
+
stream = void 0;
|
|
2513
|
+
batcher = null;
|
|
2514
|
+
let responseUsageCount = 0;
|
|
2515
|
+
let lastInputTokensObserved = null;
|
|
2516
|
+
throwIfWorkerShuttingDown();
|
|
2517
|
+
const runStreamOnce = () => runtime.runStream(agent, runInput, runSettings, {
|
|
2518
|
+
sandboxEnvironment,
|
|
2519
|
+
onRuntimeEvent: async (event) => {
|
|
2520
|
+
await publish([{ type: event.type, payload: event.payload }], true);
|
|
2521
|
+
},
|
|
2522
|
+
// P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
|
|
2523
|
+
// keystone). Absent when the flag is off -> legacy build-and-discard.
|
|
2524
|
+
...resolvedSandbox ? {
|
|
2525
|
+
ownedSandbox: {
|
|
2526
|
+
client: resolvedSandbox.established.client,
|
|
2527
|
+
session: resolvedSandbox.established.session,
|
|
2528
|
+
sessionState: resolvedSandbox.established.sessionState,
|
|
2529
|
+
// Pin platform setup (hooks + file materialization) to the un-proxied
|
|
2530
|
+
// established box — never through the routing proxy, which would
|
|
2531
|
+
// re-route those execs onto a machine swapped in mid-turn.
|
|
2532
|
+
...setupBoxSession ? { setupSession: setupBoxSession } : {}
|
|
2310
2533
|
}
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2534
|
+
} : {},
|
|
2535
|
+
contextCompactionSignalTokens: () => lastInputTokensObserved
|
|
2536
|
+
});
|
|
2537
|
+
stream = await withCodex(runStreamOnce);
|
|
2538
|
+
batcher = createRuntimeBatcher(async (events) => {
|
|
2539
|
+
await publish(events);
|
|
2540
|
+
});
|
|
2541
|
+
const iterator = stream.toStream()[Symbol.asyncIterator]();
|
|
2542
|
+
let streamDone = false;
|
|
2543
|
+
try {
|
|
2544
|
+
while (true) {
|
|
2545
|
+
const next = await nextStreamEvent(iterator, activityContext);
|
|
2546
|
+
if (next.done) {
|
|
2547
|
+
streamDone = true;
|
|
2548
|
+
break;
|
|
2549
|
+
}
|
|
2550
|
+
const responseUsage = modelResponseUsageFromSdkEvent(next.value);
|
|
2551
|
+
if (responseUsage) {
|
|
2552
|
+
responseUsageCount += 1;
|
|
2553
|
+
const observed = responseUsage.usage?.inputTokens;
|
|
2554
|
+
if (typeof observed === "number" && observed > 0) {
|
|
2555
|
+
lastInputTokensObserved = observed;
|
|
2556
|
+
await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, observed).catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
|
|
2557
|
+
}
|
|
2558
|
+
await recordModelUsageAndDebitCredits(settings, db, {
|
|
2559
|
+
accountId: input.accountId,
|
|
2560
|
+
workspaceId: input.workspaceId,
|
|
2561
|
+
sessionId: input.sessionId,
|
|
2562
|
+
turnId: activeTurnId,
|
|
2563
|
+
model: turn.model,
|
|
2564
|
+
isCodexTurn,
|
|
2565
|
+
usage: responseUsage.usage,
|
|
2566
|
+
sourceKey: modelUsageSourceKey({
|
|
2567
|
+
responseId: responseUsage.responseId,
|
|
2568
|
+
dispatchId,
|
|
2569
|
+
positionalKey: `response-${responseUsageCount}`
|
|
2570
|
+
})
|
|
2571
|
+
});
|
|
2572
|
+
await reconcileConversationTruth();
|
|
2330
2573
|
try {
|
|
2331
|
-
|
|
2332
|
-
} catch {
|
|
2333
|
-
serializedRunState = null;
|
|
2574
|
+
await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
|
|
2575
|
+
} catch (limitError) {
|
|
2576
|
+
let serializedRunState = null;
|
|
2577
|
+
try {
|
|
2578
|
+
serializedRunState = stream.state.toString();
|
|
2579
|
+
} catch {
|
|
2580
|
+
serializedRunState = null;
|
|
2581
|
+
}
|
|
2582
|
+
throw new BudgetExhaustedError(
|
|
2583
|
+
limitError instanceof Error ? limitError.message : String(limitError),
|
|
2584
|
+
serializedRunState
|
|
2585
|
+
);
|
|
2334
2586
|
}
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2587
|
+
}
|
|
2588
|
+
if (activeRecording && !didComputerUse && isComputerCallStreamEvent(next.value)) {
|
|
2589
|
+
didComputerUse = true;
|
|
2590
|
+
}
|
|
2591
|
+
const normalized = normalizeSdkEvent(next.value);
|
|
2592
|
+
for (const event of normalized) {
|
|
2593
|
+
await batcher.push(event);
|
|
2339
2594
|
}
|
|
2340
2595
|
}
|
|
2341
|
-
|
|
2342
|
-
|
|
2596
|
+
} finally {
|
|
2597
|
+
if (!streamDone) {
|
|
2598
|
+
await iterator.return?.();
|
|
2343
2599
|
}
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2600
|
+
}
|
|
2601
|
+
await batcher.flush();
|
|
2602
|
+
await stream.completed.catch(() => void 0);
|
|
2603
|
+
if (responseUsageCount === 0) {
|
|
2604
|
+
const aggregateInput = stream.state.usage?.inputTokens;
|
|
2605
|
+
if (typeof aggregateInput === "number" && aggregateInput > 0) {
|
|
2606
|
+
lastInputTokensObserved = aggregateInput;
|
|
2347
2607
|
}
|
|
2608
|
+
await recordModelUsageAndDebitCredits(settings, db, {
|
|
2609
|
+
accountId: input.accountId,
|
|
2610
|
+
workspaceId: input.workspaceId,
|
|
2611
|
+
sessionId: input.sessionId,
|
|
2612
|
+
turnId: activeTurnId,
|
|
2613
|
+
model: turn.model,
|
|
2614
|
+
isCodexTurn,
|
|
2615
|
+
usage: stream.state.usage,
|
|
2616
|
+
sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" })
|
|
2617
|
+
});
|
|
2348
2618
|
}
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
await iterator.return?.();
|
|
2619
|
+
if (lastInputTokensObserved !== null) {
|
|
2620
|
+
await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, lastInputTokensObserved).catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
|
|
2352
2621
|
}
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2622
|
+
if (stream.interruptions.length > 0) {
|
|
2623
|
+
await reconcileConversationTruth();
|
|
2624
|
+
const approvals = runtime.serializeApprovals(stream.interruptions);
|
|
2625
|
+
await saveRunState(db, {
|
|
2626
|
+
accountId: input.accountId,
|
|
2627
|
+
workspaceId: input.workspaceId,
|
|
2628
|
+
sessionId: input.sessionId,
|
|
2629
|
+
turnId: activeTurnId,
|
|
2630
|
+
serializedRunState: stream.state.toString(),
|
|
2631
|
+
pendingApprovals: approvals,
|
|
2632
|
+
// Record the account freezing this state so a resume on a DIFFERENT
|
|
2633
|
+
// codex account strips its account-bound reasoning before replay (HOLE C).
|
|
2634
|
+
frozenCodexCredentialId: effectiveCodexCredentialId
|
|
2635
|
+
});
|
|
2636
|
+
await publish([
|
|
2637
|
+
{ type: "session.requiresAction", payload: { approvals } },
|
|
2638
|
+
{ type: "session.status.changed", payload: { status: "requires_action" } }
|
|
2639
|
+
], true);
|
|
2640
|
+
await finishTurn(db, input.workspaceId, activeTurnId, "requires_action");
|
|
2641
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "requires_action", activeTurnId);
|
|
2642
|
+
activityStatus = "requires_action";
|
|
2643
|
+
return { status: "requires_action" };
|
|
2360
2644
|
}
|
|
2361
|
-
|
|
2362
|
-
accountId: input.accountId,
|
|
2363
|
-
workspaceId: input.workspaceId,
|
|
2364
|
-
sessionId: input.sessionId,
|
|
2365
|
-
turnId,
|
|
2366
|
-
model: turn.model,
|
|
2367
|
-
isCodexTurn,
|
|
2368
|
-
usage: stream.state.usage,
|
|
2369
|
-
sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" })
|
|
2370
|
-
});
|
|
2371
|
-
}
|
|
2372
|
-
if (lastInputTokensObserved !== null) {
|
|
2373
|
-
await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, lastInputTokensObserved).catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
|
|
2374
|
-
}
|
|
2375
|
-
if (stream.interruptions.length > 0) {
|
|
2645
|
+
const finalOutput = String(stream.finalOutput ?? "");
|
|
2376
2646
|
await reconcileConversationTruth();
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
});
|
|
2647
|
+
if (settings.sessionHistorySource !== "items") {
|
|
2648
|
+
await saveRunState(db, {
|
|
2649
|
+
accountId: input.accountId,
|
|
2650
|
+
workspaceId: input.workspaceId,
|
|
2651
|
+
sessionId: input.sessionId,
|
|
2652
|
+
turnId: activeTurnId,
|
|
2653
|
+
serializedRunState: stream.state.toString(),
|
|
2654
|
+
pendingApprovals: [],
|
|
2655
|
+
frozenCodexCredentialId: effectiveCodexCredentialId
|
|
2656
|
+
});
|
|
2657
|
+
}
|
|
2389
2658
|
await publish([
|
|
2390
|
-
{ type: "
|
|
2391
|
-
{ type: "
|
|
2659
|
+
{ type: "agent.message.completed", payload: { text: finalOutput } },
|
|
2660
|
+
{ type: "turn.completed", payload: { output: finalOutput } },
|
|
2661
|
+
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2392
2662
|
], true);
|
|
2393
|
-
await finishTurn(db, input.workspaceId,
|
|
2394
|
-
await setSessionStatus(db, input.workspaceId, input.sessionId, "
|
|
2395
|
-
|
|
2396
|
-
return { status: "requires_action" };
|
|
2397
|
-
}
|
|
2398
|
-
const finalOutput = String(stream.finalOutput ?? "");
|
|
2399
|
-
await reconcileConversationTruth();
|
|
2400
|
-
if (settings.sessionHistorySource !== "items") {
|
|
2401
|
-
await saveRunState(db, {
|
|
2663
|
+
await finishTurn(db, input.workspaceId, activeTurnId, "idle");
|
|
2664
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2665
|
+
await recordUsageEvent2(db, {
|
|
2402
2666
|
accountId: input.accountId,
|
|
2403
2667
|
workspaceId: input.workspaceId,
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2668
|
+
eventType: "agent_run.completed",
|
|
2669
|
+
quantity: 1,
|
|
2670
|
+
unit: "run",
|
|
2671
|
+
sourceResourceType: "session_turn",
|
|
2672
|
+
sourceResourceId: activeTurnId,
|
|
2673
|
+
idempotencyKey: `usage:agent_run.completed:${activeTurnId}`
|
|
2409
2674
|
});
|
|
2675
|
+
activityStatus = "idle";
|
|
2676
|
+
return { status: "idle" };
|
|
2677
|
+
};
|
|
2678
|
+
await prepareRunAttemptInput();
|
|
2679
|
+
let compactionRecoveryAttempted = false;
|
|
2680
|
+
let retriedAfterCompaction = false;
|
|
2681
|
+
while (true) {
|
|
2682
|
+
try {
|
|
2683
|
+
const result = await runStreamAttempt();
|
|
2684
|
+
if (retriedAfterCompaction) {
|
|
2685
|
+
observability.info("context compaction recovery succeeded after in-activity retry", {
|
|
2686
|
+
sessionId: input.sessionId,
|
|
2687
|
+
turnId: activeTurnId
|
|
2688
|
+
});
|
|
2689
|
+
}
|
|
2690
|
+
return result;
|
|
2691
|
+
} catch (attemptError) {
|
|
2692
|
+
const overflow = classifyContextWindowOverflowError(attemptError);
|
|
2693
|
+
const compactionNeeded = findCompactionNeededError(attemptError);
|
|
2694
|
+
const recoveryKind = compactionNeeded ? "proactive" : overflow ? "overflow" : null;
|
|
2695
|
+
if (!recoveryKind || !publish || !turnStartedPublished || compactionRecoveryAttempted) {
|
|
2696
|
+
throw attemptError;
|
|
2697
|
+
}
|
|
2698
|
+
compactionRecoveryAttempted = true;
|
|
2699
|
+
await flushRuntimeBatcher();
|
|
2700
|
+
await reconcileConversationTruth({ skipInputOnlyRows: true });
|
|
2701
|
+
const progressPersisted = modelOrToolProgressPersisted;
|
|
2702
|
+
observability.warn("context compaction recovery attempted", {
|
|
2703
|
+
sessionId: input.sessionId,
|
|
2704
|
+
turnId: activeTurnId,
|
|
2705
|
+
reason: recoveryKind,
|
|
2706
|
+
progressPersisted,
|
|
2707
|
+
code: overflow?.code,
|
|
2708
|
+
error: overflow?.message ?? compactionNeeded?.message,
|
|
2709
|
+
signalTokens: compactionNeeded?.signalTokens,
|
|
2710
|
+
thresholdTokens: compactionNeeded?.thresholdTokens
|
|
2711
|
+
});
|
|
2712
|
+
let compacted = false;
|
|
2713
|
+
try {
|
|
2714
|
+
const outcome = await forceContextCompaction(recoveryKind);
|
|
2715
|
+
compacted = outcome.compacted;
|
|
2716
|
+
} catch (compactError) {
|
|
2717
|
+
observability.warn("context compaction recovery compaction failed", {
|
|
2718
|
+
sessionId: input.sessionId,
|
|
2719
|
+
turnId: activeTurnId,
|
|
2720
|
+
error: compactError instanceof Error ? compactError.message : String(compactError)
|
|
2721
|
+
});
|
|
2722
|
+
}
|
|
2723
|
+
if (progressPersisted) {
|
|
2724
|
+
const triggerPayload = trigger.payload && typeof trigger.payload === "object" && !Array.isArray(trigger.payload) ? trigger.payload : {};
|
|
2725
|
+
const compactionResumedTurn = triggerType === "turn.preempted" && (triggerPayload.reason === "context_compacted" || triggerPayload.reason === "context_overflow_compacted");
|
|
2726
|
+
const canAutoContinue = compacted && !compactionResumedTurn && settings.sessionHistorySource === "items" && activeTurnId != null;
|
|
2727
|
+
if (canAutoContinue) {
|
|
2728
|
+
const [preemptedEvent] = await appendAndPublishEvents2(db, bus, input.workspaceId, input.sessionId, [
|
|
2729
|
+
{
|
|
2730
|
+
turnId: activeTurnId,
|
|
2731
|
+
type: "turn.preempted",
|
|
2732
|
+
payload: {
|
|
2733
|
+
triggerEventId: input.triggerEventId,
|
|
2734
|
+
reason: "context_compacted",
|
|
2735
|
+
resumeWithNotice: true,
|
|
2736
|
+
text: CONTEXT_OVERFLOW_RESUME_TEXT
|
|
2737
|
+
}
|
|
2738
|
+
},
|
|
2739
|
+
{ turnId: activeTurnId, type: "session.status.changed", payload: { status: "queued" } }
|
|
2740
|
+
]);
|
|
2741
|
+
await requeuePreemptedTurn(db, input.workspaceId, activeTurnId, preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
2742
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
|
|
2743
|
+
activityStatus = "preempted";
|
|
2744
|
+
observability.info("context compaction recovery succeeded by compacting and auto-continuing", {
|
|
2745
|
+
sessionId: input.sessionId,
|
|
2746
|
+
turnId: activeTurnId,
|
|
2747
|
+
reason: recoveryKind,
|
|
2748
|
+
compacted
|
|
2749
|
+
});
|
|
2750
|
+
return { status: "preempted" };
|
|
2751
|
+
}
|
|
2752
|
+
await publish([
|
|
2753
|
+
{
|
|
2754
|
+
type: "turn.failed",
|
|
2755
|
+
payload: {
|
|
2756
|
+
error: CONTEXT_WINDOW_OVERFLOW_RECOVERY_MESSAGE,
|
|
2757
|
+
code: recoveryKind === "overflow" ? "context_window_overflow_compacted" : "context_compacted",
|
|
2758
|
+
retryable: false,
|
|
2759
|
+
recovery: "user_message",
|
|
2760
|
+
compacted
|
|
2761
|
+
}
|
|
2762
|
+
},
|
|
2763
|
+
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2764
|
+
], true);
|
|
2765
|
+
await finishTurn(db, input.workspaceId, activeTurnId, "failed");
|
|
2766
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2767
|
+
activityStatus = "idle";
|
|
2768
|
+
activityError = attemptError;
|
|
2769
|
+
observability.info("context compaction recovery succeeded by compacting and idling", {
|
|
2770
|
+
sessionId: input.sessionId,
|
|
2771
|
+
turnId: activeTurnId,
|
|
2772
|
+
reason: recoveryKind,
|
|
2773
|
+
compacted
|
|
2774
|
+
});
|
|
2775
|
+
return { status: "idle" };
|
|
2776
|
+
}
|
|
2777
|
+
retriedAfterCompaction = true;
|
|
2778
|
+
observability.info("context compaction recovery retrying turn after compaction", {
|
|
2779
|
+
sessionId: input.sessionId,
|
|
2780
|
+
turnId: activeTurnId,
|
|
2781
|
+
reason: recoveryKind,
|
|
2782
|
+
compacted
|
|
2783
|
+
});
|
|
2784
|
+
await prepareRunAttemptInput();
|
|
2785
|
+
}
|
|
2410
2786
|
}
|
|
2411
|
-
await publish([
|
|
2412
|
-
{ type: "agent.message.completed", payload: { text: finalOutput } },
|
|
2413
|
-
{ type: "turn.completed", payload: { output: finalOutput } },
|
|
2414
|
-
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2415
|
-
], true);
|
|
2416
|
-
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
2417
|
-
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2418
|
-
await recordUsageEvent2(db, {
|
|
2419
|
-
accountId: input.accountId,
|
|
2420
|
-
workspaceId: input.workspaceId,
|
|
2421
|
-
eventType: "agent_run.completed",
|
|
2422
|
-
quantity: 1,
|
|
2423
|
-
unit: "run",
|
|
2424
|
-
sourceResourceType: "session_turn",
|
|
2425
|
-
sourceResourceId: turnId,
|
|
2426
|
-
idempotencyKey: `usage:agent_run.completed:${turnId}`
|
|
2427
|
-
});
|
|
2428
|
-
activityStatus = "idle";
|
|
2429
|
-
return { status: "idle" };
|
|
2430
2787
|
} catch (error) {
|
|
2431
2788
|
const preemptTurnId = turnId ?? input.turnId;
|
|
2432
2789
|
if (error instanceof SandboxLeaseSupersededError2 && preemptTurnId) {
|
|
@@ -2441,7 +2798,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2441
2798
|
}
|
|
2442
2799
|
if (isWorkerShutdownCancellation(error) && preemptTurnId) {
|
|
2443
2800
|
try {
|
|
2444
|
-
await
|
|
2801
|
+
await flushRuntimeBatcher();
|
|
2445
2802
|
await reconcileConversationTruth();
|
|
2446
2803
|
const approvalRerun = triggerType === "user.approvalDecision";
|
|
2447
2804
|
let resumeWithNotice = !approvalRerun && settings.sessionHistorySource === "items" && persistedHistoryCount > historyCountAtTurnStart;
|
|
@@ -2489,7 +2846,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2489
2846
|
if (error instanceof CancelledFailure) {
|
|
2490
2847
|
activityStatus = "cancelled";
|
|
2491
2848
|
activityError = error;
|
|
2492
|
-
await
|
|
2849
|
+
await flushRuntimeBatcher();
|
|
2493
2850
|
if (preemptTurnId) {
|
|
2494
2851
|
await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => void 0);
|
|
2495
2852
|
}
|
|
@@ -2497,7 +2854,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2497
2854
|
}
|
|
2498
2855
|
const maxTurns = maxTurnsExceededRunState(error);
|
|
2499
2856
|
if (maxTurns && publish && turnId && turnStartedPublished) {
|
|
2500
|
-
await
|
|
2857
|
+
await flushRuntimeBatcher();
|
|
2501
2858
|
await reconcileConversationTruth();
|
|
2502
2859
|
const runStateSaved = Boolean(maxTurns.serializedRunState) && settings.sessionHistorySource !== "items";
|
|
2503
2860
|
if (maxTurns.serializedRunState && settings.sessionHistorySource !== "items") {
|
|
@@ -2534,7 +2891,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2534
2891
|
if (usageLimit && publish && turnId && turnStartedPublished) {
|
|
2535
2892
|
const goal = await getSessionGoal3(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
2536
2893
|
const goalActive = Boolean(goal && goal.status === "active");
|
|
2537
|
-
await
|
|
2894
|
+
await flushRuntimeBatcher();
|
|
2538
2895
|
await reconcileConversationTruth();
|
|
2539
2896
|
const serializedRunState = agentsErrorRunState(error);
|
|
2540
2897
|
const runStateSaved = Boolean(serializedRunState) && settings.sessionHistorySource !== "items";
|
|
@@ -2618,7 +2975,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2618
2975
|
return { status: "idle" };
|
|
2619
2976
|
}
|
|
2620
2977
|
if (error instanceof BudgetExhaustedError && publish && turnId && turnStartedPublished) {
|
|
2621
|
-
await
|
|
2978
|
+
await flushRuntimeBatcher();
|
|
2622
2979
|
await reconcileConversationTruth();
|
|
2623
2980
|
const runStateSaved = Boolean(error.serializedRunState) && settings.sessionHistorySource !== "items";
|
|
2624
2981
|
if (error.serializedRunState && settings.sessionHistorySource !== "items") {
|
|
@@ -2655,7 +3012,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2655
3012
|
if (failure.retryable && publish && turnId && turnStartedPublished) {
|
|
2656
3013
|
const goal = await getSessionGoal3(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
2657
3014
|
const goalActive = Boolean(goal && goal.status === "active");
|
|
2658
|
-
await
|
|
3015
|
+
await flushRuntimeBatcher();
|
|
2659
3016
|
await reconcileConversationTruth();
|
|
2660
3017
|
const serializedRunState = agentsErrorRunState(error);
|
|
2661
3018
|
const runStateSaved = Boolean(serializedRunState) && settings.sessionHistorySource !== "items";
|
|
@@ -3062,6 +3419,7 @@ import {
|
|
|
3062
3419
|
confirmDrainCold,
|
|
3063
3420
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
3064
3421
|
getBillingBalance as getBillingBalance4,
|
|
3422
|
+
listLiveModalSandboxLeaseAttributions,
|
|
3065
3423
|
listMeterableWarmLeases,
|
|
3066
3424
|
persistDrainSnapshot,
|
|
3067
3425
|
readLease as readLease2,
|
|
@@ -3073,14 +3431,17 @@ import {
|
|
|
3073
3431
|
createSandboxClientForBackend,
|
|
3074
3432
|
deletePriorPersistedSnapshot,
|
|
3075
3433
|
deserializeSandboxSessionStateEnvelope,
|
|
3076
|
-
isProviderSandboxNotFoundError
|
|
3434
|
+
isProviderSandboxNotFoundError,
|
|
3435
|
+
sweepModalOrphanSandboxes,
|
|
3436
|
+
terminateModalSandboxById
|
|
3077
3437
|
} from "@opengeni/runtime";
|
|
3078
3438
|
function createSandboxLeaseActivities(services, options = {}) {
|
|
3079
3439
|
const terminateBox = options.terminateBox ?? terminateProviderBox;
|
|
3440
|
+
const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
3080
3441
|
async function reapSandboxLeases2() {
|
|
3081
3442
|
const { db, settings, observability } = await services();
|
|
3082
3443
|
if (!settings.sandboxOwnershipEnabled) {
|
|
3083
|
-
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0 };
|
|
3444
|
+
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
|
|
3084
3445
|
}
|
|
3085
3446
|
const metered = await accrueWarmTick(db, settings, observability);
|
|
3086
3447
|
const forceDrained = await forceDrainOverLimitWorkspaces(db, settings, metered.workspaceIds, observability);
|
|
@@ -3090,6 +3451,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
3090
3451
|
});
|
|
3091
3452
|
let terminated = 0;
|
|
3092
3453
|
let skipped = 0;
|
|
3454
|
+
let modalOrphansTerminated = 0;
|
|
3093
3455
|
for (const row of drainable) {
|
|
3094
3456
|
try {
|
|
3095
3457
|
const drainedCold = await terminateDrainableBox(db, settings, row, observability, terminateBox);
|
|
@@ -3108,16 +3470,24 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
3108
3470
|
});
|
|
3109
3471
|
}
|
|
3110
3472
|
}
|
|
3111
|
-
|
|
3473
|
+
try {
|
|
3474
|
+
modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
|
|
3475
|
+
} catch (error) {
|
|
3476
|
+
observability.warn("sandbox reaper: Modal orphan sweep failed", {
|
|
3477
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3478
|
+
});
|
|
3479
|
+
}
|
|
3480
|
+
if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
|
|
3112
3481
|
observability.info("sandbox reaper swept", {
|
|
3113
3482
|
drainable: drainable.length,
|
|
3114
3483
|
terminated,
|
|
3115
3484
|
skipped,
|
|
3116
3485
|
metered: metered.accrued,
|
|
3117
|
-
forceDrained
|
|
3486
|
+
forceDrained,
|
|
3487
|
+
modalOrphansTerminated
|
|
3118
3488
|
});
|
|
3119
3489
|
}
|
|
3120
|
-
return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained };
|
|
3490
|
+
return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained, modalOrphansTerminated };
|
|
3121
3491
|
}
|
|
3122
3492
|
return { reapSandboxLeases: reapSandboxLeases2 };
|
|
3123
3493
|
}
|
|
@@ -3193,6 +3563,37 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
|
|
|
3193
3563
|
}
|
|
3194
3564
|
return forceDrained;
|
|
3195
3565
|
}
|
|
3566
|
+
async function sweepModalOrphansForConfiguredBackend(settings, db, observability) {
|
|
3567
|
+
if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret) {
|
|
3568
|
+
return 0;
|
|
3569
|
+
}
|
|
3570
|
+
let liveLeases;
|
|
3571
|
+
try {
|
|
3572
|
+
liveLeases = await listLiveModalSandboxLeaseAttributions(db);
|
|
3573
|
+
} catch (error) {
|
|
3574
|
+
observability.warn("sandbox reaper: live Modal lease attribution read failed", {
|
|
3575
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3576
|
+
});
|
|
3577
|
+
return 0;
|
|
3578
|
+
}
|
|
3579
|
+
const result = await sweepModalOrphanSandboxes(settings, liveLeases);
|
|
3580
|
+
for (const terminated of result.terminated) {
|
|
3581
|
+
observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
|
|
3582
|
+
sandboxId: terminated.sandboxId,
|
|
3583
|
+
reason: terminated.reason,
|
|
3584
|
+
tags: JSON.stringify(terminated.tags)
|
|
3585
|
+
});
|
|
3586
|
+
}
|
|
3587
|
+
if (result.examined > 0) {
|
|
3588
|
+
observability.info("sandbox reaper: Modal orphan sweep completed", {
|
|
3589
|
+
examined: result.examined,
|
|
3590
|
+
terminated: result.terminated.length,
|
|
3591
|
+
skipped: result.skipped,
|
|
3592
|
+
appName: settings.modalAppName
|
|
3593
|
+
});
|
|
3594
|
+
}
|
|
3595
|
+
return result.terminated.length;
|
|
3596
|
+
}
|
|
3196
3597
|
async function terminateDrainableBox(db, settings, row, observability, terminateBox) {
|
|
3197
3598
|
const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
|
|
3198
3599
|
const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId);
|
|
@@ -3233,6 +3634,30 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
3233
3634
|
});
|
|
3234
3635
|
return true;
|
|
3235
3636
|
}
|
|
3637
|
+
if (backend === "modal" && !lease.resumeState && lease.instanceId) {
|
|
3638
|
+
const { wrote } = await persistArchive(null);
|
|
3639
|
+
if (!wrote) {
|
|
3640
|
+
observability.info("sandbox reaper: Modal lease re-armed before direct terminate \u2014 leaving sandbox RUNNING", {
|
|
3641
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
3642
|
+
backend,
|
|
3643
|
+
instanceId: lease.instanceId
|
|
3644
|
+
});
|
|
3645
|
+
return false;
|
|
3646
|
+
}
|
|
3647
|
+
try {
|
|
3648
|
+
await terminateModalSandboxById(settings, lease.instanceId);
|
|
3649
|
+
} catch (error) {
|
|
3650
|
+
if (!isProviderSandboxNotFoundError("modal", error)) {
|
|
3651
|
+
throw error;
|
|
3652
|
+
}
|
|
3653
|
+
observability.info("sandbox reaper: Modal sandbox already gone during direct terminate", {
|
|
3654
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
3655
|
+
backend,
|
|
3656
|
+
instanceId: lease.instanceId
|
|
3657
|
+
});
|
|
3658
|
+
}
|
|
3659
|
+
return true;
|
|
3660
|
+
}
|
|
3236
3661
|
if (!lease.resumeState) {
|
|
3237
3662
|
return true;
|
|
3238
3663
|
}
|