@opengeni/worker-bundle 0.2.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -0
- package/dist/index.js +685 -260
- package/dist/index.js.map +1 -1
- package/package.json +8 -8
- package/src/activities/agent-turn.ts +474 -231
- package/src/activities/capabilities.ts +42 -0
- package/src/activities/context-compaction.ts +51 -58
- package/src/activities/run-input.ts +2 -2
- package/src/activities/sandbox-lease.ts +97 -3
- package/src/sandbox-resume.ts +178 -59
|
@@ -49,13 +49,15 @@ import {
|
|
|
49
49
|
normalizeSdkEvent,
|
|
50
50
|
sanitizeHistoryItemsForModel,
|
|
51
51
|
summarizeForCompaction,
|
|
52
|
+
findCompactionNeededError,
|
|
53
|
+
SUMMARY_BUFFER_TOKENS,
|
|
52
54
|
type SandboxFileDownload,
|
|
53
55
|
type OpenGeniRuntime,
|
|
54
56
|
type ComputerToolMode,
|
|
55
57
|
} from "@opengeni/runtime";
|
|
56
58
|
import { calculateModelUsageCostMicros, configuredModelPricing, configuredStaticUsageLimits, sandboxWarmRateMicrosPerSecond, type ModelUsageInput, type ModelProviderApi, type RegistryProviderKind, type Settings } from "@opengeni/config";
|
|
57
59
|
import { CancelledFailure } from "@temporalio/activity";
|
|
58
|
-
import { settingsWithCodexCredential, settingsWithEnabledCapabilityMcpServers } from "./capabilities";
|
|
60
|
+
import { settingsWithCodexCredential, settingsWithEnabledCapabilityMcpServers, settingsWithSessionMcpServersForRun } from "./capabilities";
|
|
59
61
|
import { chooseRotationActive, computeIdleDelayMs, computeReactiveRotationResume, type CodexRotationStrategy, type RotationDecision } from "./codex-rotation";
|
|
60
62
|
import type { CodexAccountStatus } from "@opengeni/db";
|
|
61
63
|
import { buildCodexTokenResolver } from "./codex-auth";
|
|
@@ -131,6 +133,14 @@ export function selectCodexCredentialForTurn(args: {
|
|
|
131
133
|
// agent must not blindly replay side effects it cannot see in its history:
|
|
132
134
|
// at most the single in-flight model step was lost, and anything it started
|
|
133
135
|
// there may or may not have completed.
|
|
136
|
+
// Resume notice after an in-turn context-window overflow was recovered by
|
|
137
|
+
// compaction: the turn's earlier work survives (albeit summarized), so the
|
|
138
|
+
// agent continues instead of the session dying or stalling idle.
|
|
139
|
+
export const CONTEXT_OVERFLOW_RESUME_TEXT = [
|
|
140
|
+
"[TURN RESUMED AFTER CONTEXT COMPACTION] The conversation approached or exceeded the model's context window mid-turn; older history above was compacted into a summary.",
|
|
141
|
+
"Continue the original task from where it left off. Before repeating any action with side effects, check whether it already happened.",
|
|
142
|
+
].join("\n");
|
|
143
|
+
|
|
134
144
|
export const WORKER_SHUTDOWN_RESUME_TEXT = [
|
|
135
145
|
"[TURN RESUMED AFTER WORKER RESTART] The platform worker running this turn restarted (deploy/rollout) before the turn finished.",
|
|
136
146
|
"Your conversation history above, including this turn's earlier work, is preserved up to the last checkpoint; at most the single in-flight model step after it was lost.",
|
|
@@ -158,6 +168,54 @@ export function isWorkerShutdownCancellation(error: unknown): boolean {
|
|
|
158
168
|
return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
|
|
159
169
|
}
|
|
160
170
|
|
|
171
|
+
export const CONTEXT_WINDOW_OVERFLOW_RECOVERY_MESSAGE =
|
|
172
|
+
"The conversation reached the model context compaction threshold; it has been compacted. Send a message to continue.";
|
|
173
|
+
|
|
174
|
+
export function classifyContextWindowOverflowError(error: unknown): { message: string; code?: string; detail?: string } | null {
|
|
175
|
+
const fields = collectErrorStrings(error);
|
|
176
|
+
const matched = fields.find((value) =>
|
|
177
|
+
/context[_\s-]*length[_\s-]*exceeded/i.test(value)
|
|
178
|
+
|| /exceeds?\s+(?:the\s+)?context\s+window/i.test(value)
|
|
179
|
+
|| /maximum\s+context\s+length/i.test(value)
|
|
180
|
+
|| /context\s+window[^.]*exceed/i.test(value)
|
|
181
|
+
);
|
|
182
|
+
if (!matched) {
|
|
183
|
+
return null;
|
|
184
|
+
}
|
|
185
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
186
|
+
const code = fields.find((value) => /context[_\s-]*length[_\s-]*exceeded/i.test(value));
|
|
187
|
+
return {
|
|
188
|
+
message,
|
|
189
|
+
...(code ? { code } : {}),
|
|
190
|
+
...(matched && matched !== message ? { detail: matched } : {}),
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function collectErrorStrings(value: unknown, seen = new WeakSet<object>()): string[] {
|
|
195
|
+
if (typeof value === "string") {
|
|
196
|
+
return [value];
|
|
197
|
+
}
|
|
198
|
+
if (!value || typeof value !== "object") {
|
|
199
|
+
return [];
|
|
200
|
+
}
|
|
201
|
+
if (seen.has(value)) {
|
|
202
|
+
return [];
|
|
203
|
+
}
|
|
204
|
+
seen.add(value);
|
|
205
|
+
const out: string[] = [];
|
|
206
|
+
const record = value as Record<string, unknown>;
|
|
207
|
+
for (const key of ["message", "code", "type", "name", "param"]) {
|
|
208
|
+
const field = record[key];
|
|
209
|
+
if (typeof field === "string" && field.length > 0) {
|
|
210
|
+
out.push(field);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
for (const key of ["error", "cause", "response", "data"]) {
|
|
214
|
+
out.push(...collectErrorStrings(record[key], seen));
|
|
215
|
+
}
|
|
216
|
+
return out;
|
|
217
|
+
}
|
|
218
|
+
|
|
161
219
|
/**
|
|
162
220
|
* Compute the conversation-truth rows a reconcile pass should append, given the
|
|
163
221
|
* SDK's current `state.history` and the count already persisted.
|
|
@@ -236,6 +294,19 @@ export function historyRowsToAppend(
|
|
|
236
294
|
return { rows, nextWatermark: sanitized.length, nextPosition: nextPosition + rows.length };
|
|
237
295
|
}
|
|
238
296
|
|
|
297
|
+
function isModelOrToolProgressHistoryItem(item: Record<string, unknown>): boolean {
|
|
298
|
+
if (item.type === "message") {
|
|
299
|
+
return item.role === "assistant";
|
|
300
|
+
}
|
|
301
|
+
if (item.type === "reasoning" || item.type === "compaction") {
|
|
302
|
+
return true;
|
|
303
|
+
}
|
|
304
|
+
if (typeof item.type === "string") {
|
|
305
|
+
return item.type !== "message";
|
|
306
|
+
}
|
|
307
|
+
return false;
|
|
308
|
+
}
|
|
309
|
+
|
|
239
310
|
/**
|
|
240
311
|
* Seed the turn-end reconcile watermark (`persistedHistoryCount`) from EXACTLY the
|
|
241
312
|
* view `state.history` was seeded from, so the model-input length and the watermark
|
|
@@ -474,6 +545,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
474
545
|
// static frame of an untouched desktop is not worth uploading).
|
|
475
546
|
let didComputerUse = false;
|
|
476
547
|
let batcher: ReturnType<typeof createRuntimeBatcher> | null = null;
|
|
548
|
+
const flushRuntimeBatcher = async () => {
|
|
549
|
+
const current = batcher as ReturnType<typeof createRuntimeBatcher> | null;
|
|
550
|
+
await current?.flush().catch(() => undefined);
|
|
551
|
+
};
|
|
477
552
|
let preparedTools: Awaited<ReturnType<OpenGeniRuntime["prepareTools"]>> | null = null;
|
|
478
553
|
let publish: ((events: Array<Omit<AppendEventInput, "producerId" | "producerSeq" | "turnId">>, immediate?: boolean) => Promise<void>) | null = null;
|
|
479
554
|
let turnStartedPublished = false;
|
|
@@ -502,12 +577,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
502
577
|
// actually persisted, so a non-monotonic history can never desync it.
|
|
503
578
|
let persistedHistoryCount = 0;
|
|
504
579
|
let historyCountAtTurnStart = 0;
|
|
580
|
+
let modelOrToolProgressPersisted = false;
|
|
505
581
|
// Next free WHOLE-NUMBER absolute position to append at. Tracked separately
|
|
506
582
|
// from persistedHistoryCount (the in-memory slice index) because a compaction
|
|
507
583
|
// inserts a fractional summary position, so total rows no longer equal
|
|
508
584
|
// max(position)+1 and the slice index can no longer double as the position.
|
|
509
585
|
let nextHistoryPosition = 0;
|
|
510
|
-
const reconcileConversationTruth = async () => {
|
|
586
|
+
const reconcileConversationTruth = async (options: { skipInputOnlyRows?: boolean } = {}) => {
|
|
511
587
|
if (!stream || !turnId) {
|
|
512
588
|
return;
|
|
513
589
|
}
|
|
@@ -519,7 +595,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
519
595
|
persistedHistoryCount,
|
|
520
596
|
nextHistoryPosition,
|
|
521
597
|
);
|
|
522
|
-
|
|
598
|
+
const hasModelOrToolProgress = rows.some((row) => isModelOrToolProgressHistoryItem(row.item));
|
|
599
|
+
const shouldAppendRows = rows.length > 0 && (!options.skipInputOnlyRows || hasModelOrToolProgress);
|
|
600
|
+
if (shouldAppendRows) {
|
|
523
601
|
await appendSessionHistoryItems(db, {
|
|
524
602
|
accountId: input.accountId,
|
|
525
603
|
workspaceId: input.workspaceId,
|
|
@@ -533,8 +611,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
533
611
|
items: rows,
|
|
534
612
|
});
|
|
535
613
|
}
|
|
536
|
-
|
|
537
|
-
|
|
614
|
+
if (shouldAppendRows || !options.skipInputOnlyRows) {
|
|
615
|
+
persistedHistoryCount = nextWatermark;
|
|
616
|
+
nextHistoryPosition = nextPosition;
|
|
617
|
+
}
|
|
618
|
+
if (hasModelOrToolProgress) {
|
|
619
|
+
modelOrToolProgressPersisted = true;
|
|
620
|
+
}
|
|
538
621
|
}
|
|
539
622
|
const envelope = sandboxStateEntryFromRunState(stream.state);
|
|
540
623
|
if (envelope) {
|
|
@@ -802,12 +885,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
802
885
|
// runtime falls back to runSettings.agentInstructionsTemplate (the
|
|
803
886
|
// deployment default, byte-identical to the historical preamble).
|
|
804
887
|
const workspaceAgentInstructions = await resolveWorkspaceAgentInstructions(db, input.workspaceId);
|
|
805
|
-
const
|
|
888
|
+
const baseRunSettings = {
|
|
806
889
|
...settingsWithPackSandboxImage(capabilitySettings, packRuntime.sandboxImage),
|
|
807
890
|
openaiModel: turn.model,
|
|
808
891
|
openaiReasoningEffort: turn.reasoningEffort,
|
|
809
892
|
sandboxBackend: turn.sandboxBackend,
|
|
810
893
|
};
|
|
894
|
+
const runSettings = await settingsWithSessionMcpServersForRun(db, input.workspaceId, input.sessionId, baseRunSettings);
|
|
811
895
|
// Multi-provider per-turn routing → the provider gating (compaction mode,
|
|
812
896
|
// hosted web search, encrypted reasoning, context window) the agent and
|
|
813
897
|
// compaction summarizer must use; null falls back to the legacy global
|
|
@@ -1251,6 +1335,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1251
1335
|
: { computerToolMode: computerToolModeForTurn(null) }),
|
|
1252
1336
|
...(packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {}),
|
|
1253
1337
|
...(workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {}),
|
|
1338
|
+
// Per-session persona tier (session > workspace > deployment default).
|
|
1339
|
+
// Composed system-level AFTER the workspace persona so it refines it for
|
|
1340
|
+
// this one session; absent ⇒ byte-identical to today's composition.
|
|
1341
|
+
...(session.instructions ? { sessionInstructions: session.instructions } : {}),
|
|
1254
1342
|
...(workspaceEnvironment
|
|
1255
1343
|
? {
|
|
1256
1344
|
workspaceEnvironment: {
|
|
@@ -1261,11 +1349,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1261
1349
|
}
|
|
1262
1350
|
: {}),
|
|
1263
1351
|
});
|
|
1352
|
+
const compactSummarizer = resolvedModel
|
|
1353
|
+
? (s: Settings, m: Array<Record<string, unknown>>) => withCodex(() => summarizeForCompaction(s, m, {
|
|
1354
|
+
client: resolvedModel.client,
|
|
1355
|
+
api: resolvedModel.provider.api,
|
|
1356
|
+
model: resolvedModel.configured.id,
|
|
1357
|
+
maxOutputTokens: SUMMARY_BUFFER_TOKENS,
|
|
1358
|
+
}))
|
|
1359
|
+
: undefined;
|
|
1264
1360
|
// Pre-turn client-side context compaction (Azure path). When the
|
|
1265
|
-
// resolved mode is "client" and the
|
|
1266
|
-
//
|
|
1267
|
-
//
|
|
1268
|
-
// for this turn, so the model sees [summary, ...recent tail] + new input.
|
|
1361
|
+
// resolved mode is "client" and the single Codex-parity threshold is
|
|
1362
|
+
// crossed, this summarizes the current active history and rebuilds active
|
|
1363
|
+
// history as [user messages..., summary] BEFORE the model input is read.
|
|
1269
1364
|
// Best-effort: a no-op or failure leaves the un-compacted history intact.
|
|
1270
1365
|
// Skipped on approval/preempt resumes (no fresh user turn; the frozen
|
|
1271
1366
|
// RunState or in-flight tail must replay verbatim).
|
|
@@ -1285,22 +1380,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1285
1380
|
// registry provider, summarize on THAT provider's client + wire API
|
|
1286
1381
|
// (a chat provider can't summarize through OpenAI/Azure). Null
|
|
1287
1382
|
// resolution keeps the default built-in Responses summarizer.
|
|
1288
|
-
|
|
1289
|
-
? (s, m) => withCodex(() => summarizeForCompaction(s, m, {
|
|
1290
|
-
client: resolvedModel.client,
|
|
1291
|
-
api: resolvedModel.provider.api,
|
|
1292
|
-
model: resolvedModel.configured.id,
|
|
1293
|
-
maxOutputTokens: s.contextSummaryMaxTokens,
|
|
1294
|
-
}))
|
|
1295
|
-
: undefined,
|
|
1383
|
+
compactSummarizer,
|
|
1296
1384
|
forced ? { force: true } : {},
|
|
1297
1385
|
);
|
|
1298
1386
|
if (outcome.compacted) {
|
|
1299
|
-
|
|
1300
|
-
// event label; otherwise a hard-forced (at/over hardFraction*B)
|
|
1301
|
-
// compaction is surfaced as "hard" so the backstop firing is
|
|
1302
|
-
// observable downstream, and the default soft trigger is implicit.
|
|
1303
|
-
const trigger = forced ? "operator" : outcome.hardForced ? "hard" : undefined;
|
|
1387
|
+
const trigger = forced ? "operator" : undefined;
|
|
1304
1388
|
await publish([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, ...(trigger ? { trigger } : {}) } }]);
|
|
1305
1389
|
}
|
|
1306
1390
|
} catch (compactError) {
|
|
@@ -1315,226 +1399,385 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1315
1399
|
// "current account" the single strip rule compares against (null is the
|
|
1316
1400
|
// built-in/Azure account, so a non-codex turn still drops codex-produced
|
|
1317
1401
|
// reasoning, and a no-codex-history session is a byte-for-byte no-op).
|
|
1318
|
-
const
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
1368
|
-
},
|
|
1369
|
-
// P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
|
|
1370
|
-
// keystone). Absent when the flag is off -> legacy build-and-discard.
|
|
1371
|
-
...(resolvedSandbox
|
|
1372
|
-
? {
|
|
1373
|
-
ownedSandbox: {
|
|
1374
|
-
client: resolvedSandbox.established.client,
|
|
1375
|
-
session: resolvedSandbox.established.session,
|
|
1376
|
-
sessionState: resolvedSandbox.established.sessionState,
|
|
1377
|
-
// Pin platform setup (hooks + file materialization) to the un-proxied
|
|
1378
|
-
// established box — never through the routing proxy, which would
|
|
1379
|
-
// re-route those execs onto a machine swapped in mid-turn.
|
|
1380
|
-
...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
|
|
1381
|
-
},
|
|
1382
|
-
}
|
|
1383
|
-
: {}),
|
|
1384
|
-
});
|
|
1385
|
-
stream = await withCodex(runStreamOnce);
|
|
1386
|
-
batcher = createRuntimeBatcher(async (events) => {
|
|
1387
|
-
await publish!(events);
|
|
1388
|
-
});
|
|
1402
|
+
const activeTurnId = turnId;
|
|
1403
|
+
if (!activeTurnId) {
|
|
1404
|
+
throw new Error("Turn id was not initialized");
|
|
1405
|
+
}
|
|
1406
|
+
let runInput: Awaited<ReturnType<typeof turnInput>>["input"] | null = null;
|
|
1407
|
+
const prepareRunAttemptInput = async () => {
|
|
1408
|
+
const prepared = await turnInput(
|
|
1409
|
+
db,
|
|
1410
|
+
runtime,
|
|
1411
|
+
agent,
|
|
1412
|
+
trigger,
|
|
1413
|
+
runSettings,
|
|
1414
|
+
{ currentCodexCredentialId: effectiveCodexCredentialId },
|
|
1415
|
+
);
|
|
1416
|
+
runInput = prepared.input;
|
|
1417
|
+
// Slice index = the length of the model-facing (active) history this turn
|
|
1418
|
+
// is seeded from; new items beyond it (the trigger message + this turn's
|
|
1419
|
+
// generated items) are the ones to persist. After a compaction this is the
|
|
1420
|
+
// short [summary, ...tail] active set, NOT the total row count. The
|
|
1421
|
+
// absolute write position is tracked separately (next whole number past
|
|
1422
|
+
// the max existing position) because the fractional summary row means
|
|
1423
|
+
// total rows no longer equal max(position)+1. Pre-compaction both reduce to
|
|
1424
|
+
// the old total-count value, so the common path is unchanged.
|
|
1425
|
+
//
|
|
1426
|
+
// CRITICAL: seed from the SANITIZED active-row length, not the raw active
|
|
1427
|
+
// count. `prepareRunInput` builds `state.history` from
|
|
1428
|
+
// `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
|
|
1429
|
+
// rows (a legacy orphan/dangling pair), the in-memory history this turn
|
|
1430
|
+
// starts from is K shorter than the raw row count. The reconcile slices the
|
|
1431
|
+
// re-sanitized `state.history` off `persistedHistoryCount`; seeding it from
|
|
1432
|
+
// the raw count (K too high) skips K genuinely-new items, and a
|
|
1433
|
+
// `function_call` left in that skipped region can later have its
|
|
1434
|
+
// `function_call_result` persisted alone — the orphan that 400s on replay
|
|
1435
|
+
// and bricks the session (issue-61). The sanitized seed is already
|
|
1436
|
+
// orphan-free, so it is a stable prefix of the re-sanitized history and the
|
|
1437
|
+
// slice begins exactly at the first genuinely-new item.
|
|
1438
|
+
const activeSeedRows = await getActiveSessionHistoryItems(db, input.workspaceId, input.sessionId);
|
|
1439
|
+
// Seed the reconcile watermark from EXACTLY the view the model's
|
|
1440
|
+
// `state.history` was seeded from (items strip on the items path = HOLE D; NO
|
|
1441
|
+
// strip on the run-state blob path, where foreign reasoning is neutralized but
|
|
1442
|
+
// KEPT = HOLE E), so the model-input length and the watermark never disagree.
|
|
1443
|
+
persistedHistoryCount = reconcileSeedCount(
|
|
1444
|
+
activeSeedRows,
|
|
1445
|
+
prepared.modelHistoryFromItems,
|
|
1446
|
+
{ currentCodexCredentialId: effectiveCodexCredentialId },
|
|
1447
|
+
);
|
|
1448
|
+
historyCountAtTurnStart = persistedHistoryCount;
|
|
1449
|
+
nextHistoryPosition = await nextSessionHistoryPosition(db, input.workspaceId, input.sessionId);
|
|
1450
|
+
};
|
|
1389
1451
|
|
|
1390
|
-
const
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
}
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1452
|
+
const forceContextCompaction = async (triggerLabel: "overflow" | "proactive") => {
|
|
1453
|
+
const clientCompactionSettings: Settings = { ...runSettings, contextCompactionMode: "client" };
|
|
1454
|
+
const outcome = await maybeCompactContext(
|
|
1455
|
+
db,
|
|
1456
|
+
clientCompactionSettings,
|
|
1457
|
+
{ accountId: input.accountId, workspaceId: input.workspaceId, sessionId: input.sessionId, turnId: activeTurnId },
|
|
1458
|
+
session.lastInputTokens,
|
|
1459
|
+
compactSummarizer,
|
|
1460
|
+
{ force: true },
|
|
1461
|
+
);
|
|
1462
|
+
if (outcome.compacted) {
|
|
1463
|
+
await publish!([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, trigger: triggerLabel } }]);
|
|
1464
|
+
}
|
|
1465
|
+
return outcome;
|
|
1466
|
+
};
|
|
1467
|
+
|
|
1468
|
+
const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
|
|
1469
|
+
if (!runInput) {
|
|
1470
|
+
throw new Error("Run input was not prepared");
|
|
1471
|
+
}
|
|
1472
|
+
stream = undefined;
|
|
1473
|
+
batcher = null;
|
|
1474
|
+
let responseUsageCount = 0;
|
|
1475
|
+
// Actual input tokens of the most recent model response this turn; the
|
|
1476
|
+
// pre-read trigger for the NEXT turn. Persisted at every turn-end path.
|
|
1477
|
+
let lastInputTokensObserved: number | null = null;
|
|
1478
|
+
throwIfWorkerShuttingDown();
|
|
1479
|
+
const runStreamOnce = (): ReturnType<OpenGeniRuntime["runStream"]> => runtime.runStream(agent, runInput!, runSettings, {
|
|
1480
|
+
sandboxEnvironment,
|
|
1481
|
+
onRuntimeEvent: async (event) => {
|
|
1482
|
+
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
1483
|
+
},
|
|
1484
|
+
// P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
|
|
1485
|
+
// keystone). Absent when the flag is off -> legacy build-and-discard.
|
|
1486
|
+
...(resolvedSandbox
|
|
1487
|
+
? {
|
|
1488
|
+
ownedSandbox: {
|
|
1489
|
+
client: resolvedSandbox.established.client,
|
|
1490
|
+
session: resolvedSandbox.established.session,
|
|
1491
|
+
sessionState: resolvedSandbox.established.sessionState,
|
|
1492
|
+
// Pin platform setup (hooks + file materialization) to the un-proxied
|
|
1493
|
+
// established box — never through the routing proxy, which would
|
|
1494
|
+
// re-route those execs onto a machine swapped in mid-turn.
|
|
1495
|
+
...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
|
|
1496
|
+
},
|
|
1405
1497
|
}
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1498
|
+
: {}),
|
|
1499
|
+
contextCompactionSignalTokens: () => lastInputTokensObserved,
|
|
1500
|
+
});
|
|
1501
|
+
stream = await withCodex(runStreamOnce);
|
|
1502
|
+
batcher = createRuntimeBatcher(async (events) => {
|
|
1503
|
+
await publish!(events);
|
|
1504
|
+
});
|
|
1505
|
+
|
|
1506
|
+
const iterator = stream.toStream()[Symbol.asyncIterator]();
|
|
1507
|
+
let streamDone = false;
|
|
1508
|
+
try {
|
|
1509
|
+
while (true) {
|
|
1510
|
+
const next = await nextStreamEvent(iterator, activityContext);
|
|
1511
|
+
if (next.done) {
|
|
1512
|
+
streamDone = true;
|
|
1513
|
+
break;
|
|
1514
|
+
}
|
|
1515
|
+
const responseUsage = modelResponseUsageFromSdkEvent(next.value);
|
|
1516
|
+
if (responseUsage) {
|
|
1517
|
+
responseUsageCount += 1;
|
|
1518
|
+
const observed = responseUsage.usage?.inputTokens;
|
|
1519
|
+
if (typeof observed === "number" && observed > 0) {
|
|
1520
|
+
lastInputTokensObserved = observed;
|
|
1521
|
+
await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, observed)
|
|
1522
|
+
.catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
|
|
1523
|
+
}
|
|
1524
|
+
await recordModelUsageAndDebitCredits(settings, db, {
|
|
1525
|
+
accountId: input.accountId,
|
|
1526
|
+
workspaceId: input.workspaceId,
|
|
1527
|
+
sessionId: input.sessionId,
|
|
1528
|
+
turnId: activeTurnId,
|
|
1529
|
+
model: turn.model,
|
|
1530
|
+
isCodexTurn,
|
|
1531
|
+
usage: responseUsage.usage,
|
|
1532
|
+
sourceKey: modelUsageSourceKey({
|
|
1533
|
+
responseId: responseUsage.responseId,
|
|
1534
|
+
dispatchId,
|
|
1535
|
+
positionalKey: `response-${responseUsageCount}`,
|
|
1536
|
+
}),
|
|
1537
|
+
});
|
|
1538
|
+
await reconcileConversationTruth();
|
|
1428
1539
|
try {
|
|
1429
|
-
|
|
1430
|
-
} catch {
|
|
1431
|
-
|
|
1540
|
+
await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
|
|
1541
|
+
} catch (limitError) {
|
|
1542
|
+
// Capture the run state at the boundary so the budget valve in
|
|
1543
|
+
// the outer catch can end this segment gracefully with full
|
|
1544
|
+
// conversation context preserved for the post-top-up resume.
|
|
1545
|
+
let serializedRunState: string | null = null;
|
|
1546
|
+
try {
|
|
1547
|
+
serializedRunState = stream.state.toString();
|
|
1548
|
+
} catch {
|
|
1549
|
+
serializedRunState = null;
|
|
1550
|
+
}
|
|
1551
|
+
throw new BudgetExhaustedError(
|
|
1552
|
+
limitError instanceof Error ? limitError.message : String(limitError),
|
|
1553
|
+
serializedRunState,
|
|
1554
|
+
);
|
|
1432
1555
|
}
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1556
|
+
}
|
|
1557
|
+
// Recording gate: a computer-use tool actually ran when an SDK
|
|
1558
|
+
// `tool_call_item` whose rawItem.type is "computer_call" streams through
|
|
1559
|
+
// (screenshot/click/type/scroll/etc. — the first computer action proves
|
|
1560
|
+
// the desktop was driven). Match the raw SDK event (ground truth) BEFORE
|
|
1561
|
+
// normalization. Only meaningful when a recording is live.
|
|
1562
|
+
if (activeRecording && !didComputerUse && isComputerCallStreamEvent(next.value)) {
|
|
1563
|
+
didComputerUse = true;
|
|
1564
|
+
}
|
|
1565
|
+
const normalized = normalizeSdkEvent(next.value);
|
|
1566
|
+
for (const event of normalized) {
|
|
1567
|
+
await batcher.push(event);
|
|
1437
1568
|
}
|
|
1438
1569
|
}
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
// the desktop was driven). Match the raw SDK event (ground truth) BEFORE
|
|
1443
|
-
// normalization. Only meaningful when a recording is live.
|
|
1444
|
-
if (activeRecording && !didComputerUse && isComputerCallStreamEvent(next.value)) {
|
|
1445
|
-
didComputerUse = true;
|
|
1570
|
+
} finally {
|
|
1571
|
+
if (!streamDone) {
|
|
1572
|
+
await iterator.return?.();
|
|
1446
1573
|
}
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1574
|
+
}
|
|
1575
|
+
await batcher.flush();
|
|
1576
|
+
await stream.completed.catch(() => undefined);
|
|
1577
|
+
if (responseUsageCount === 0) {
|
|
1578
|
+
const aggregateInput = (stream.state.usage as { inputTokens?: unknown } | undefined)?.inputTokens;
|
|
1579
|
+
if (typeof aggregateInput === "number" && aggregateInput > 0) {
|
|
1580
|
+
lastInputTokensObserved = aggregateInput;
|
|
1450
1581
|
}
|
|
1582
|
+
await recordModelUsageAndDebitCredits(settings, db, {
|
|
1583
|
+
accountId: input.accountId,
|
|
1584
|
+
workspaceId: input.workspaceId,
|
|
1585
|
+
sessionId: input.sessionId,
|
|
1586
|
+
turnId: activeTurnId,
|
|
1587
|
+
model: turn.model,
|
|
1588
|
+
isCodexTurn,
|
|
1589
|
+
usage: stream.state.usage,
|
|
1590
|
+
sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
|
|
1591
|
+
});
|
|
1451
1592
|
}
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1593
|
+
if (lastInputTokensObserved !== null) {
|
|
1594
|
+
await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, lastInputTokensObserved)
|
|
1595
|
+
.catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
|
|
1455
1596
|
}
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1597
|
+
|
|
1598
|
+
if (stream.interruptions.length > 0) {
|
|
1599
|
+
await reconcileConversationTruth();
|
|
1600
|
+
const approvals = runtime.serializeApprovals(stream.interruptions);
|
|
1601
|
+
await saveRunState(db, {
|
|
1602
|
+
accountId: input.accountId,
|
|
1603
|
+
workspaceId: input.workspaceId,
|
|
1604
|
+
sessionId: input.sessionId,
|
|
1605
|
+
turnId: activeTurnId,
|
|
1606
|
+
serializedRunState: stream.state.toString(),
|
|
1607
|
+
pendingApprovals: approvals,
|
|
1608
|
+
// Record the account freezing this state so a resume on a DIFFERENT
|
|
1609
|
+
// codex account strips its account-bound reasoning before replay (HOLE C).
|
|
1610
|
+
frozenCodexCredentialId: effectiveCodexCredentialId,
|
|
1611
|
+
});
|
|
1612
|
+
await publish!([
|
|
1613
|
+
{ type: "session.requiresAction", payload: { approvals } },
|
|
1614
|
+
{ type: "session.status.changed", payload: { status: "requires_action" } },
|
|
1615
|
+
], true);
|
|
1616
|
+
await finishTurn(db, input.workspaceId, activeTurnId, "requires_action");
|
|
1617
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "requires_action", activeTurnId);
|
|
1618
|
+
activityStatus = "requires_action";
|
|
1619
|
+
return { status: "requires_action" };
|
|
1463
1620
|
}
|
|
1464
|
-
await recordModelUsageAndDebitCredits(settings, db, {
|
|
1465
|
-
accountId: input.accountId,
|
|
1466
|
-
workspaceId: input.workspaceId,
|
|
1467
|
-
sessionId: input.sessionId,
|
|
1468
|
-
turnId,
|
|
1469
|
-
model: turn.model,
|
|
1470
|
-
isCodexTurn,
|
|
1471
|
-
usage: stream.state.usage,
|
|
1472
|
-
sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
|
|
1473
|
-
});
|
|
1474
|
-
}
|
|
1475
|
-
if (lastInputTokensObserved !== null) {
|
|
1476
|
-
await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, lastInputTokensObserved)
|
|
1477
|
-
.catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
|
|
1478
|
-
}
|
|
1479
1621
|
|
|
1480
|
-
|
|
1622
|
+
const finalOutput = String(stream.finalOutput ?? "");
|
|
1481
1623
|
await reconcileConversationTruth();
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
{ type: "
|
|
1624
|
+
if (settings.sessionHistorySource !== "items") {
|
|
1625
|
+
// Legacy conversation memory; in items mode the blob is only written
|
|
1626
|
+
// for requires_action pauses (the one RunState-only resume path).
|
|
1627
|
+
await saveRunState(db, {
|
|
1628
|
+
accountId: input.accountId,
|
|
1629
|
+
workspaceId: input.workspaceId,
|
|
1630
|
+
sessionId: input.sessionId,
|
|
1631
|
+
turnId: activeTurnId,
|
|
1632
|
+
serializedRunState: stream.state.toString(),
|
|
1633
|
+
pendingApprovals: [],
|
|
1634
|
+
frozenCodexCredentialId: effectiveCodexCredentialId,
|
|
1635
|
+
});
|
|
1636
|
+
}
|
|
1637
|
+
await publish!([
|
|
1638
|
+
{ type: "agent.message.completed", payload: { text: finalOutput } },
|
|
1639
|
+
{ type: "turn.completed", payload: { output: finalOutput } },
|
|
1640
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1497
1641
|
], true);
|
|
1498
|
-
await finishTurn(db, input.workspaceId,
|
|
1499
|
-
await setSessionStatus(db, input.workspaceId, input.sessionId, "
|
|
1500
|
-
|
|
1501
|
-
return { status: "requires_action" };
|
|
1502
|
-
}
|
|
1503
|
-
|
|
1504
|
-
const finalOutput = String(stream.finalOutput ?? "");
|
|
1505
|
-
await reconcileConversationTruth();
|
|
1506
|
-
if (settings.sessionHistorySource !== "items") {
|
|
1507
|
-
// Legacy conversation memory; in items mode the blob is only written
|
|
1508
|
-
// for requires_action pauses (the one RunState-only resume path).
|
|
1509
|
-
await saveRunState(db, {
|
|
1642
|
+
await finishTurn(db, input.workspaceId, activeTurnId, "idle");
|
|
1643
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1644
|
+
await recordUsageEvent(db, {
|
|
1510
1645
|
accountId: input.accountId,
|
|
1511
1646
|
workspaceId: input.workspaceId,
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1647
|
+
eventType: "agent_run.completed",
|
|
1648
|
+
quantity: 1,
|
|
1649
|
+
unit: "run",
|
|
1650
|
+
sourceResourceType: "session_turn",
|
|
1651
|
+
sourceResourceId: activeTurnId,
|
|
1652
|
+
idempotencyKey: `usage:agent_run.completed:${activeTurnId}`,
|
|
1517
1653
|
});
|
|
1654
|
+
activityStatus = "idle";
|
|
1655
|
+
return { status: "idle" };
|
|
1656
|
+
};
|
|
1657
|
+
|
|
1658
|
+
await prepareRunAttemptInput();
|
|
1659
|
+
let compactionRecoveryAttempted = false;
|
|
1660
|
+
let retriedAfterCompaction = false;
|
|
1661
|
+
while (true) {
|
|
1662
|
+
try {
|
|
1663
|
+
const result = await runStreamAttempt();
|
|
1664
|
+
if (retriedAfterCompaction) {
|
|
1665
|
+
observability.info("context compaction recovery succeeded after in-activity retry", {
|
|
1666
|
+
sessionId: input.sessionId,
|
|
1667
|
+
turnId: activeTurnId,
|
|
1668
|
+
});
|
|
1669
|
+
}
|
|
1670
|
+
return result;
|
|
1671
|
+
} catch (attemptError) {
|
|
1672
|
+
const overflow = classifyContextWindowOverflowError(attemptError);
|
|
1673
|
+
const compactionNeeded = findCompactionNeededError(attemptError);
|
|
1674
|
+
const recoveryKind = compactionNeeded ? "proactive" : overflow ? "overflow" : null;
|
|
1675
|
+
if (!recoveryKind || !publish || !turnStartedPublished || compactionRecoveryAttempted) {
|
|
1676
|
+
throw attemptError;
|
|
1677
|
+
}
|
|
1678
|
+
compactionRecoveryAttempted = true;
|
|
1679
|
+
await flushRuntimeBatcher();
|
|
1680
|
+
await reconcileConversationTruth({ skipInputOnlyRows: true });
|
|
1681
|
+
const progressPersisted = modelOrToolProgressPersisted;
|
|
1682
|
+
observability.warn("context compaction recovery attempted", {
|
|
1683
|
+
sessionId: input.sessionId,
|
|
1684
|
+
turnId: activeTurnId,
|
|
1685
|
+
reason: recoveryKind,
|
|
1686
|
+
progressPersisted,
|
|
1687
|
+
code: overflow?.code,
|
|
1688
|
+
error: overflow?.message ?? compactionNeeded?.message,
|
|
1689
|
+
signalTokens: compactionNeeded?.signalTokens,
|
|
1690
|
+
thresholdTokens: compactionNeeded?.thresholdTokens,
|
|
1691
|
+
});
|
|
1692
|
+
let compacted = false;
|
|
1693
|
+
try {
|
|
1694
|
+
const outcome = await forceContextCompaction(recoveryKind);
|
|
1695
|
+
compacted = outcome.compacted;
|
|
1696
|
+
} catch (compactError) {
|
|
1697
|
+
observability.warn("context compaction recovery compaction failed", {
|
|
1698
|
+
sessionId: input.sessionId,
|
|
1699
|
+
turnId: activeTurnId,
|
|
1700
|
+
error: compactError instanceof Error ? compactError.message : String(compactError),
|
|
1701
|
+
});
|
|
1702
|
+
}
|
|
1703
|
+
if (progressPersisted) {
|
|
1704
|
+
// The user's intent was live mid-turn — after a successful
|
|
1705
|
+
// compaction the turn AUTO-CONTINUES via the same synthesized
|
|
1706
|
+
// resume-notice requeue the worker-shutdown checkpoint uses (never
|
|
1707
|
+
// the original trigger, so side effects are not replayed). Idle is
|
|
1708
|
+
// the fallback, not the default; it remains for: legacy run-state
|
|
1709
|
+
// history (resuming the frozen pre-compaction RunState would
|
|
1710
|
+
// overflow again), a compaction that could not shrink anything,
|
|
1711
|
+
// and a turn that ALREADY resumed from an overflow (loop guard —
|
|
1712
|
+
// a second overflow in the same lineage stops churning).
|
|
1713
|
+
const triggerPayload =
|
|
1714
|
+
trigger.payload && typeof trigger.payload === "object" && !Array.isArray(trigger.payload)
|
|
1715
|
+
? trigger.payload as Record<string, unknown>
|
|
1716
|
+
: {};
|
|
1717
|
+
const compactionResumedTurn = triggerType === "turn.preempted"
|
|
1718
|
+
&& (triggerPayload.reason === "context_compacted" || triggerPayload.reason === "context_overflow_compacted");
|
|
1719
|
+
const canAutoContinue =
|
|
1720
|
+
compacted && !compactionResumedTurn && settings.sessionHistorySource === "items" && activeTurnId != null;
|
|
1721
|
+
if (canAutoContinue) {
|
|
1722
|
+
const [preemptedEvent] = await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [
|
|
1723
|
+
{
|
|
1724
|
+
turnId: activeTurnId,
|
|
1725
|
+
type: "turn.preempted",
|
|
1726
|
+
payload: {
|
|
1727
|
+
triggerEventId: input.triggerEventId,
|
|
1728
|
+
reason: "context_compacted",
|
|
1729
|
+
resumeWithNotice: true,
|
|
1730
|
+
text: CONTEXT_OVERFLOW_RESUME_TEXT,
|
|
1731
|
+
},
|
|
1732
|
+
},
|
|
1733
|
+
{ turnId: activeTurnId, type: "session.status.changed", payload: { status: "queued" } },
|
|
1734
|
+
]);
|
|
1735
|
+
await requeuePreemptedTurn(db, input.workspaceId, activeTurnId, preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
1736
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
|
|
1737
|
+
activityStatus = "preempted";
|
|
1738
|
+
observability.info("context compaction recovery succeeded by compacting and auto-continuing", {
|
|
1739
|
+
sessionId: input.sessionId,
|
|
1740
|
+
turnId: activeTurnId,
|
|
1741
|
+
reason: recoveryKind,
|
|
1742
|
+
compacted,
|
|
1743
|
+
});
|
|
1744
|
+
return { status: "preempted" };
|
|
1745
|
+
}
|
|
1746
|
+
await publish!([
|
|
1747
|
+
{
|
|
1748
|
+
type: "turn.failed",
|
|
1749
|
+
payload: {
|
|
1750
|
+
error: CONTEXT_WINDOW_OVERFLOW_RECOVERY_MESSAGE,
|
|
1751
|
+
code: recoveryKind === "overflow" ? "context_window_overflow_compacted" : "context_compacted",
|
|
1752
|
+
retryable: false,
|
|
1753
|
+
recovery: "user_message",
|
|
1754
|
+
compacted,
|
|
1755
|
+
},
|
|
1756
|
+
},
|
|
1757
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1758
|
+
], true);
|
|
1759
|
+
await finishTurn(db, input.workspaceId, activeTurnId, "failed");
|
|
1760
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1761
|
+
activityStatus = "idle";
|
|
1762
|
+
activityError = attemptError;
|
|
1763
|
+
observability.info("context compaction recovery succeeded by compacting and idling", {
|
|
1764
|
+
sessionId: input.sessionId,
|
|
1765
|
+
turnId: activeTurnId,
|
|
1766
|
+
reason: recoveryKind,
|
|
1767
|
+
compacted,
|
|
1768
|
+
});
|
|
1769
|
+
return { status: "idle" };
|
|
1770
|
+
}
|
|
1771
|
+
retriedAfterCompaction = true;
|
|
1772
|
+
observability.info("context compaction recovery retrying turn after compaction", {
|
|
1773
|
+
sessionId: input.sessionId,
|
|
1774
|
+
turnId: activeTurnId,
|
|
1775
|
+
reason: recoveryKind,
|
|
1776
|
+
compacted,
|
|
1777
|
+
});
|
|
1778
|
+
await prepareRunAttemptInput();
|
|
1779
|
+
}
|
|
1518
1780
|
}
|
|
1519
|
-
await publish([
|
|
1520
|
-
{ type: "agent.message.completed", payload: { text: finalOutput } },
|
|
1521
|
-
{ type: "turn.completed", payload: { output: finalOutput } },
|
|
1522
|
-
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1523
|
-
], true);
|
|
1524
|
-
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
1525
|
-
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1526
|
-
await recordUsageEvent(db, {
|
|
1527
|
-
accountId: input.accountId,
|
|
1528
|
-
workspaceId: input.workspaceId,
|
|
1529
|
-
eventType: "agent_run.completed",
|
|
1530
|
-
quantity: 1,
|
|
1531
|
-
unit: "run",
|
|
1532
|
-
sourceResourceType: "session_turn",
|
|
1533
|
-
sourceResourceId: turnId,
|
|
1534
|
-
idempotencyKey: `usage:agent_run.completed:${turnId}`,
|
|
1535
|
-
});
|
|
1536
|
-
activityStatus = "idle";
|
|
1537
|
-
return { status: "idle" };
|
|
1538
1781
|
} catch (error) {
|
|
1539
1782
|
// Graceful worker shutdown (deploy / rollout restart): checkpoint and
|
|
1540
1783
|
// hand the turn back instead of failing the session. Conversation truth
|
|
@@ -1572,7 +1815,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1572
1815
|
}
|
|
1573
1816
|
if (isWorkerShutdownCancellation(error) && preemptTurnId) {
|
|
1574
1817
|
try {
|
|
1575
|
-
await
|
|
1818
|
+
await flushRuntimeBatcher();
|
|
1576
1819
|
await reconcileConversationTruth();
|
|
1577
1820
|
// An approval-decision rerun always replays its original trigger:
|
|
1578
1821
|
// the decision is applied through the approval resume path reading
|
|
@@ -1639,7 +1882,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1639
1882
|
if (error instanceof CancelledFailure) {
|
|
1640
1883
|
activityStatus = "cancelled";
|
|
1641
1884
|
activityError = error;
|
|
1642
|
-
await
|
|
1885
|
+
await flushRuntimeBatcher();
|
|
1643
1886
|
if (preemptTurnId) {
|
|
1644
1887
|
await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => undefined);
|
|
1645
1888
|
}
|
|
@@ -1652,7 +1895,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1652
1895
|
// that resumption.
|
|
1653
1896
|
const maxTurns = maxTurnsExceededRunState(error);
|
|
1654
1897
|
if (maxTurns && publish && turnId && turnStartedPublished) {
|
|
1655
|
-
await
|
|
1898
|
+
await flushRuntimeBatcher();
|
|
1656
1899
|
// The SDK attaches the run state at the throw site; persisting it lets
|
|
1657
1900
|
// the continuation resume with this segment's full context. If capture
|
|
1658
1901
|
// ever fails, the continuation falls back to the previous snapshot --
|
|
@@ -1703,7 +1946,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1703
1946
|
if (usageLimit && publish && turnId && turnStartedPublished) {
|
|
1704
1947
|
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
1705
1948
|
const goalActive = Boolean(goal && goal.status === "active");
|
|
1706
|
-
await
|
|
1949
|
+
await flushRuntimeBatcher();
|
|
1707
1950
|
await reconcileConversationTruth();
|
|
1708
1951
|
const serializedRunState = agentsErrorRunState(error);
|
|
1709
1952
|
const runStateSaved = Boolean(serializedRunState) && settings.sessionHistorySource !== "items";
|
|
@@ -1834,7 +2077,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1834
2077
|
// top-up). An active goal pauses visibly with reason "limits" at the
|
|
1835
2078
|
// next continuation evaluation, without consuming continuation budget.
|
|
1836
2079
|
if (error instanceof BudgetExhaustedError && publish && turnId && turnStartedPublished) {
|
|
1837
|
-
await
|
|
2080
|
+
await flushRuntimeBatcher();
|
|
1838
2081
|
await reconcileConversationTruth();
|
|
1839
2082
|
const runStateSaved = Boolean(error.serializedRunState) && settings.sessionHistorySource !== "items";
|
|
1840
2083
|
if (error.serializedRunState && settings.sessionHistorySource !== "items") {
|
|
@@ -1880,7 +2123,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1880
2123
|
if (failure.retryable && publish && turnId && turnStartedPublished) {
|
|
1881
2124
|
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
1882
2125
|
const goalActive = Boolean(goal && goal.status === "active");
|
|
1883
|
-
await
|
|
2126
|
+
await flushRuntimeBatcher();
|
|
1884
2127
|
// Provider errors rarely carry SDK run state; a null falls back to
|
|
1885
2128
|
// the previous snapshot, same degraded-context contract as the
|
|
1886
2129
|
// max-turns path above.
|