@opengeni/worker-bundle 0.2.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +470 -220
- package/dist/index.js.map +1 -1
- package/package.json +8 -8
- package/src/activities/agent-turn.ts +470 -231
- package/src/activities/capabilities.ts +42 -0
- package/src/activities/context-compaction.ts +51 -58
- package/src/activities/run-input.ts +2 -2
|
@@ -49,13 +49,15 @@ import {
|
|
|
49
49
|
normalizeSdkEvent,
|
|
50
50
|
sanitizeHistoryItemsForModel,
|
|
51
51
|
summarizeForCompaction,
|
|
52
|
+
findCompactionNeededError,
|
|
53
|
+
SUMMARY_BUFFER_TOKENS,
|
|
52
54
|
type SandboxFileDownload,
|
|
53
55
|
type OpenGeniRuntime,
|
|
54
56
|
type ComputerToolMode,
|
|
55
57
|
} from "@opengeni/runtime";
|
|
56
58
|
import { calculateModelUsageCostMicros, configuredModelPricing, configuredStaticUsageLimits, sandboxWarmRateMicrosPerSecond, type ModelUsageInput, type ModelProviderApi, type RegistryProviderKind, type Settings } from "@opengeni/config";
|
|
57
59
|
import { CancelledFailure } from "@temporalio/activity";
|
|
58
|
-
import { settingsWithCodexCredential, settingsWithEnabledCapabilityMcpServers } from "./capabilities";
|
|
60
|
+
import { settingsWithCodexCredential, settingsWithEnabledCapabilityMcpServers, settingsWithSessionMcpServersForRun } from "./capabilities";
|
|
59
61
|
import { chooseRotationActive, computeIdleDelayMs, computeReactiveRotationResume, type CodexRotationStrategy, type RotationDecision } from "./codex-rotation";
|
|
60
62
|
import type { CodexAccountStatus } from "@opengeni/db";
|
|
61
63
|
import { buildCodexTokenResolver } from "./codex-auth";
|
|
@@ -131,6 +133,14 @@ export function selectCodexCredentialForTurn(args: {
|
|
|
131
133
|
// agent must not blindly replay side effects it cannot see in its history:
|
|
132
134
|
// at most the single in-flight model step was lost, and anything it started
|
|
133
135
|
// there may or may not have completed.
|
|
136
|
+
// Resume notice after an in-turn context-window overflow was recovered by
|
|
137
|
+
// compaction: the turn's earlier work survives (albeit summarized), so the
|
|
138
|
+
// agent continues instead of the session dying or stalling idle.
|
|
139
|
+
export const CONTEXT_OVERFLOW_RESUME_TEXT = [
|
|
140
|
+
"[TURN RESUMED AFTER CONTEXT COMPACTION] The conversation approached or exceeded the model's context window mid-turn; older history above was compacted into a summary.",
|
|
141
|
+
"Continue the original task from where it left off. Before repeating any action with side effects, check whether it already happened.",
|
|
142
|
+
].join("\n");
|
|
143
|
+
|
|
134
144
|
export const WORKER_SHUTDOWN_RESUME_TEXT = [
|
|
135
145
|
"[TURN RESUMED AFTER WORKER RESTART] The platform worker running this turn restarted (deploy/rollout) before the turn finished.",
|
|
136
146
|
"Your conversation history above, including this turn's earlier work, is preserved up to the last checkpoint; at most the single in-flight model step after it was lost.",
|
|
@@ -158,6 +168,54 @@ export function isWorkerShutdownCancellation(error: unknown): boolean {
|
|
|
158
168
|
return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
|
|
159
169
|
}
|
|
160
170
|
|
|
171
|
+
export const CONTEXT_WINDOW_OVERFLOW_RECOVERY_MESSAGE =
|
|
172
|
+
"The conversation reached the model context compaction threshold; it has been compacted. Send a message to continue.";
|
|
173
|
+
|
|
174
|
+
export function classifyContextWindowOverflowError(error: unknown): { message: string; code?: string; detail?: string } | null {
|
|
175
|
+
const fields = collectErrorStrings(error);
|
|
176
|
+
const matched = fields.find((value) =>
|
|
177
|
+
/context[_\s-]*length[_\s-]*exceeded/i.test(value)
|
|
178
|
+
|| /exceeds?\s+(?:the\s+)?context\s+window/i.test(value)
|
|
179
|
+
|| /maximum\s+context\s+length/i.test(value)
|
|
180
|
+
|| /context\s+window[^.]*exceed/i.test(value)
|
|
181
|
+
);
|
|
182
|
+
if (!matched) {
|
|
183
|
+
return null;
|
|
184
|
+
}
|
|
185
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
186
|
+
const code = fields.find((value) => /context[_\s-]*length[_\s-]*exceeded/i.test(value));
|
|
187
|
+
return {
|
|
188
|
+
message,
|
|
189
|
+
...(code ? { code } : {}),
|
|
190
|
+
...(matched && matched !== message ? { detail: matched } : {}),
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function collectErrorStrings(value: unknown, seen = new WeakSet<object>()): string[] {
|
|
195
|
+
if (typeof value === "string") {
|
|
196
|
+
return [value];
|
|
197
|
+
}
|
|
198
|
+
if (!value || typeof value !== "object") {
|
|
199
|
+
return [];
|
|
200
|
+
}
|
|
201
|
+
if (seen.has(value)) {
|
|
202
|
+
return [];
|
|
203
|
+
}
|
|
204
|
+
seen.add(value);
|
|
205
|
+
const out: string[] = [];
|
|
206
|
+
const record = value as Record<string, unknown>;
|
|
207
|
+
for (const key of ["message", "code", "type", "name", "param"]) {
|
|
208
|
+
const field = record[key];
|
|
209
|
+
if (typeof field === "string" && field.length > 0) {
|
|
210
|
+
out.push(field);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
for (const key of ["error", "cause", "response", "data"]) {
|
|
214
|
+
out.push(...collectErrorStrings(record[key], seen));
|
|
215
|
+
}
|
|
216
|
+
return out;
|
|
217
|
+
}
|
|
218
|
+
|
|
161
219
|
/**
|
|
162
220
|
* Compute the conversation-truth rows a reconcile pass should append, given the
|
|
163
221
|
* SDK's current `state.history` and the count already persisted.
|
|
@@ -236,6 +294,19 @@ export function historyRowsToAppend(
|
|
|
236
294
|
return { rows, nextWatermark: sanitized.length, nextPosition: nextPosition + rows.length };
|
|
237
295
|
}
|
|
238
296
|
|
|
297
|
+
function isModelOrToolProgressHistoryItem(item: Record<string, unknown>): boolean {
|
|
298
|
+
if (item.type === "message") {
|
|
299
|
+
return item.role === "assistant";
|
|
300
|
+
}
|
|
301
|
+
if (item.type === "reasoning" || item.type === "compaction") {
|
|
302
|
+
return true;
|
|
303
|
+
}
|
|
304
|
+
if (typeof item.type === "string") {
|
|
305
|
+
return item.type !== "message";
|
|
306
|
+
}
|
|
307
|
+
return false;
|
|
308
|
+
}
|
|
309
|
+
|
|
239
310
|
/**
|
|
240
311
|
* Seed the turn-end reconcile watermark (`persistedHistoryCount`) from EXACTLY the
|
|
241
312
|
* view `state.history` was seeded from, so the model-input length and the watermark
|
|
@@ -474,6 +545,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
474
545
|
// static frame of an untouched desktop is not worth uploading).
|
|
475
546
|
let didComputerUse = false;
|
|
476
547
|
let batcher: ReturnType<typeof createRuntimeBatcher> | null = null;
|
|
548
|
+
const flushRuntimeBatcher = async () => {
|
|
549
|
+
const current = batcher as ReturnType<typeof createRuntimeBatcher> | null;
|
|
550
|
+
await current?.flush().catch(() => undefined);
|
|
551
|
+
};
|
|
477
552
|
let preparedTools: Awaited<ReturnType<OpenGeniRuntime["prepareTools"]>> | null = null;
|
|
478
553
|
let publish: ((events: Array<Omit<AppendEventInput, "producerId" | "producerSeq" | "turnId">>, immediate?: boolean) => Promise<void>) | null = null;
|
|
479
554
|
let turnStartedPublished = false;
|
|
@@ -502,12 +577,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
502
577
|
// actually persisted, so a non-monotonic history can never desync it.
|
|
503
578
|
let persistedHistoryCount = 0;
|
|
504
579
|
let historyCountAtTurnStart = 0;
|
|
580
|
+
let modelOrToolProgressPersisted = false;
|
|
505
581
|
// Next free WHOLE-NUMBER absolute position to append at. Tracked separately
|
|
506
582
|
// from persistedHistoryCount (the in-memory slice index) because a compaction
|
|
507
583
|
// inserts a fractional summary position, so total rows no longer equal
|
|
508
584
|
// max(position)+1 and the slice index can no longer double as the position.
|
|
509
585
|
let nextHistoryPosition = 0;
|
|
510
|
-
const reconcileConversationTruth = async () => {
|
|
586
|
+
const reconcileConversationTruth = async (options: { skipInputOnlyRows?: boolean } = {}) => {
|
|
511
587
|
if (!stream || !turnId) {
|
|
512
588
|
return;
|
|
513
589
|
}
|
|
@@ -519,7 +595,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
519
595
|
persistedHistoryCount,
|
|
520
596
|
nextHistoryPosition,
|
|
521
597
|
);
|
|
522
|
-
|
|
598
|
+
const hasModelOrToolProgress = rows.some((row) => isModelOrToolProgressHistoryItem(row.item));
|
|
599
|
+
const shouldAppendRows = rows.length > 0 && (!options.skipInputOnlyRows || hasModelOrToolProgress);
|
|
600
|
+
if (shouldAppendRows) {
|
|
523
601
|
await appendSessionHistoryItems(db, {
|
|
524
602
|
accountId: input.accountId,
|
|
525
603
|
workspaceId: input.workspaceId,
|
|
@@ -533,8 +611,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
533
611
|
items: rows,
|
|
534
612
|
});
|
|
535
613
|
}
|
|
536
|
-
|
|
537
|
-
|
|
614
|
+
if (shouldAppendRows || !options.skipInputOnlyRows) {
|
|
615
|
+
persistedHistoryCount = nextWatermark;
|
|
616
|
+
nextHistoryPosition = nextPosition;
|
|
617
|
+
}
|
|
618
|
+
if (hasModelOrToolProgress) {
|
|
619
|
+
modelOrToolProgressPersisted = true;
|
|
620
|
+
}
|
|
538
621
|
}
|
|
539
622
|
const envelope = sandboxStateEntryFromRunState(stream.state);
|
|
540
623
|
if (envelope) {
|
|
@@ -802,12 +885,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
802
885
|
// runtime falls back to runSettings.agentInstructionsTemplate (the
|
|
803
886
|
// deployment default, byte-identical to the historical preamble).
|
|
804
887
|
const workspaceAgentInstructions = await resolveWorkspaceAgentInstructions(db, input.workspaceId);
|
|
805
|
-
const
|
|
888
|
+
const baseRunSettings = {
|
|
806
889
|
...settingsWithPackSandboxImage(capabilitySettings, packRuntime.sandboxImage),
|
|
807
890
|
openaiModel: turn.model,
|
|
808
891
|
openaiReasoningEffort: turn.reasoningEffort,
|
|
809
892
|
sandboxBackend: turn.sandboxBackend,
|
|
810
893
|
};
|
|
894
|
+
const runSettings = await settingsWithSessionMcpServersForRun(db, input.workspaceId, input.sessionId, baseRunSettings);
|
|
811
895
|
// Multi-provider per-turn routing → the provider gating (compaction mode,
|
|
812
896
|
// hosted web search, encrypted reasoning, context window) the agent and
|
|
813
897
|
// compaction summarizer must use; null falls back to the legacy global
|
|
@@ -1261,11 +1345,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1261
1345
|
}
|
|
1262
1346
|
: {}),
|
|
1263
1347
|
});
|
|
1348
|
+
const compactSummarizer = resolvedModel
|
|
1349
|
+
? (s: Settings, m: Array<Record<string, unknown>>) => withCodex(() => summarizeForCompaction(s, m, {
|
|
1350
|
+
client: resolvedModel.client,
|
|
1351
|
+
api: resolvedModel.provider.api,
|
|
1352
|
+
model: resolvedModel.configured.id,
|
|
1353
|
+
maxOutputTokens: SUMMARY_BUFFER_TOKENS,
|
|
1354
|
+
}))
|
|
1355
|
+
: undefined;
|
|
1264
1356
|
// Pre-turn client-side context compaction (Azure path). When the
|
|
1265
|
-
// resolved mode is "client" and the
|
|
1266
|
-
//
|
|
1267
|
-
//
|
|
1268
|
-
// for this turn, so the model sees [summary, ...recent tail] + new input.
|
|
1357
|
+
// resolved mode is "client" and the single Codex-parity threshold is
|
|
1358
|
+
// crossed, this summarizes the current active history and rebuilds active
|
|
1359
|
+
// history as [user messages..., summary] BEFORE the model input is read.
|
|
1269
1360
|
// Best-effort: a no-op or failure leaves the un-compacted history intact.
|
|
1270
1361
|
// Skipped on approval/preempt resumes (no fresh user turn; the frozen
|
|
1271
1362
|
// RunState or in-flight tail must replay verbatim).
|
|
@@ -1285,22 +1376,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1285
1376
|
// registry provider, summarize on THAT provider's client + wire API
|
|
1286
1377
|
// (a chat provider can't summarize through OpenAI/Azure). Null
|
|
1287
1378
|
// resolution keeps the default built-in Responses summarizer.
|
|
1288
|
-
|
|
1289
|
-
? (s, m) => withCodex(() => summarizeForCompaction(s, m, {
|
|
1290
|
-
client: resolvedModel.client,
|
|
1291
|
-
api: resolvedModel.provider.api,
|
|
1292
|
-
model: resolvedModel.configured.id,
|
|
1293
|
-
maxOutputTokens: s.contextSummaryMaxTokens,
|
|
1294
|
-
}))
|
|
1295
|
-
: undefined,
|
|
1379
|
+
compactSummarizer,
|
|
1296
1380
|
forced ? { force: true } : {},
|
|
1297
1381
|
);
|
|
1298
1382
|
if (outcome.compacted) {
|
|
1299
|
-
|
|
1300
|
-
// event label; otherwise a hard-forced (at/over hardFraction*B)
|
|
1301
|
-
// compaction is surfaced as "hard" so the backstop firing is
|
|
1302
|
-
// observable downstream, and the default soft trigger is implicit.
|
|
1303
|
-
const trigger = forced ? "operator" : outcome.hardForced ? "hard" : undefined;
|
|
1383
|
+
const trigger = forced ? "operator" : undefined;
|
|
1304
1384
|
await publish([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, ...(trigger ? { trigger } : {}) } }]);
|
|
1305
1385
|
}
|
|
1306
1386
|
} catch (compactError) {
|
|
@@ -1315,226 +1395,385 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1315
1395
|
// "current account" the single strip rule compares against (null is the
|
|
1316
1396
|
// built-in/Azure account, so a non-codex turn still drops codex-produced
|
|
1317
1397
|
// reasoning, and a no-codex-history session is a byte-for-byte no-op).
|
|
1318
|
-
const
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
1368
|
-
},
|
|
1369
|
-
// P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
|
|
1370
|
-
// keystone). Absent when the flag is off -> legacy build-and-discard.
|
|
1371
|
-
...(resolvedSandbox
|
|
1372
|
-
? {
|
|
1373
|
-
ownedSandbox: {
|
|
1374
|
-
client: resolvedSandbox.established.client,
|
|
1375
|
-
session: resolvedSandbox.established.session,
|
|
1376
|
-
sessionState: resolvedSandbox.established.sessionState,
|
|
1377
|
-
// Pin platform setup (hooks + file materialization) to the un-proxied
|
|
1378
|
-
// established box — never through the routing proxy, which would
|
|
1379
|
-
// re-route those execs onto a machine swapped in mid-turn.
|
|
1380
|
-
...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
|
|
1381
|
-
},
|
|
1382
|
-
}
|
|
1383
|
-
: {}),
|
|
1384
|
-
});
|
|
1385
|
-
stream = await withCodex(runStreamOnce);
|
|
1386
|
-
batcher = createRuntimeBatcher(async (events) => {
|
|
1387
|
-
await publish!(events);
|
|
1388
|
-
});
|
|
1398
|
+
const activeTurnId = turnId;
|
|
1399
|
+
if (!activeTurnId) {
|
|
1400
|
+
throw new Error("Turn id was not initialized");
|
|
1401
|
+
}
|
|
1402
|
+
let runInput: Awaited<ReturnType<typeof turnInput>>["input"] | null = null;
|
|
1403
|
+
const prepareRunAttemptInput = async () => {
|
|
1404
|
+
const prepared = await turnInput(
|
|
1405
|
+
db,
|
|
1406
|
+
runtime,
|
|
1407
|
+
agent,
|
|
1408
|
+
trigger,
|
|
1409
|
+
runSettings,
|
|
1410
|
+
{ currentCodexCredentialId: effectiveCodexCredentialId },
|
|
1411
|
+
);
|
|
1412
|
+
runInput = prepared.input;
|
|
1413
|
+
// Slice index = the length of the model-facing (active) history this turn
|
|
1414
|
+
// is seeded from; new items beyond it (the trigger message + this turn's
|
|
1415
|
+
// generated items) are the ones to persist. After a compaction this is the
|
|
1416
|
+
// short [summary, ...tail] active set, NOT the total row count. The
|
|
1417
|
+
// absolute write position is tracked separately (next whole number past
|
|
1418
|
+
// the max existing position) because the fractional summary row means
|
|
1419
|
+
// total rows no longer equal max(position)+1. Pre-compaction both reduce to
|
|
1420
|
+
// the old total-count value, so the common path is unchanged.
|
|
1421
|
+
//
|
|
1422
|
+
// CRITICAL: seed from the SANITIZED active-row length, not the raw active
|
|
1423
|
+
// count. `prepareRunInput` builds `state.history` from
|
|
1424
|
+
// `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
|
|
1425
|
+
// rows (a legacy orphan/dangling pair), the in-memory history this turn
|
|
1426
|
+
// starts from is K shorter than the raw row count. The reconcile slices the
|
|
1427
|
+
// re-sanitized `state.history` off `persistedHistoryCount`; seeding it from
|
|
1428
|
+
// the raw count (K too high) skips K genuinely-new items, and a
|
|
1429
|
+
// `function_call` left in that skipped region can later have its
|
|
1430
|
+
// `function_call_result` persisted alone — the orphan that 400s on replay
|
|
1431
|
+
// and bricks the session (issue-61). The sanitized seed is already
|
|
1432
|
+
// orphan-free, so it is a stable prefix of the re-sanitized history and the
|
|
1433
|
+
// slice begins exactly at the first genuinely-new item.
|
|
1434
|
+
const activeSeedRows = await getActiveSessionHistoryItems(db, input.workspaceId, input.sessionId);
|
|
1435
|
+
// Seed the reconcile watermark from EXACTLY the view the model's
|
|
1436
|
+
// `state.history` was seeded from (items strip on the items path = HOLE D; NO
|
|
1437
|
+
// strip on the run-state blob path, where foreign reasoning is neutralized but
|
|
1438
|
+
// KEPT = HOLE E), so the model-input length and the watermark never disagree.
|
|
1439
|
+
persistedHistoryCount = reconcileSeedCount(
|
|
1440
|
+
activeSeedRows,
|
|
1441
|
+
prepared.modelHistoryFromItems,
|
|
1442
|
+
{ currentCodexCredentialId: effectiveCodexCredentialId },
|
|
1443
|
+
);
|
|
1444
|
+
historyCountAtTurnStart = persistedHistoryCount;
|
|
1445
|
+
nextHistoryPosition = await nextSessionHistoryPosition(db, input.workspaceId, input.sessionId);
|
|
1446
|
+
};
|
|
1389
1447
|
|
|
1390
|
-
const
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
}
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1448
|
+
const forceContextCompaction = async (triggerLabel: "overflow" | "proactive") => {
|
|
1449
|
+
const clientCompactionSettings: Settings = { ...runSettings, contextCompactionMode: "client" };
|
|
1450
|
+
const outcome = await maybeCompactContext(
|
|
1451
|
+
db,
|
|
1452
|
+
clientCompactionSettings,
|
|
1453
|
+
{ accountId: input.accountId, workspaceId: input.workspaceId, sessionId: input.sessionId, turnId: activeTurnId },
|
|
1454
|
+
session.lastInputTokens,
|
|
1455
|
+
compactSummarizer,
|
|
1456
|
+
{ force: true },
|
|
1457
|
+
);
|
|
1458
|
+
if (outcome.compacted) {
|
|
1459
|
+
await publish!([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, trigger: triggerLabel } }]);
|
|
1460
|
+
}
|
|
1461
|
+
return outcome;
|
|
1462
|
+
};
|
|
1463
|
+
|
|
1464
|
+
const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
|
|
1465
|
+
if (!runInput) {
|
|
1466
|
+
throw new Error("Run input was not prepared");
|
|
1467
|
+
}
|
|
1468
|
+
stream = undefined;
|
|
1469
|
+
batcher = null;
|
|
1470
|
+
let responseUsageCount = 0;
|
|
1471
|
+
// Actual input tokens of the most recent model response this turn; the
|
|
1472
|
+
// pre-read trigger for the NEXT turn. Persisted at every turn-end path.
|
|
1473
|
+
let lastInputTokensObserved: number | null = null;
|
|
1474
|
+
throwIfWorkerShuttingDown();
|
|
1475
|
+
const runStreamOnce = (): ReturnType<OpenGeniRuntime["runStream"]> => runtime.runStream(agent, runInput!, runSettings, {
|
|
1476
|
+
sandboxEnvironment,
|
|
1477
|
+
onRuntimeEvent: async (event) => {
|
|
1478
|
+
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
1479
|
+
},
|
|
1480
|
+
// P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
|
|
1481
|
+
// keystone). Absent when the flag is off -> legacy build-and-discard.
|
|
1482
|
+
...(resolvedSandbox
|
|
1483
|
+
? {
|
|
1484
|
+
ownedSandbox: {
|
|
1485
|
+
client: resolvedSandbox.established.client,
|
|
1486
|
+
session: resolvedSandbox.established.session,
|
|
1487
|
+
sessionState: resolvedSandbox.established.sessionState,
|
|
1488
|
+
// Pin platform setup (hooks + file materialization) to the un-proxied
|
|
1489
|
+
// established box — never through the routing proxy, which would
|
|
1490
|
+
// re-route those execs onto a machine swapped in mid-turn.
|
|
1491
|
+
...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
|
|
1492
|
+
},
|
|
1405
1493
|
}
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1494
|
+
: {}),
|
|
1495
|
+
contextCompactionSignalTokens: () => lastInputTokensObserved,
|
|
1496
|
+
});
|
|
1497
|
+
stream = await withCodex(runStreamOnce);
|
|
1498
|
+
batcher = createRuntimeBatcher(async (events) => {
|
|
1499
|
+
await publish!(events);
|
|
1500
|
+
});
|
|
1501
|
+
|
|
1502
|
+
const iterator = stream.toStream()[Symbol.asyncIterator]();
|
|
1503
|
+
let streamDone = false;
|
|
1504
|
+
try {
|
|
1505
|
+
while (true) {
|
|
1506
|
+
const next = await nextStreamEvent(iterator, activityContext);
|
|
1507
|
+
if (next.done) {
|
|
1508
|
+
streamDone = true;
|
|
1509
|
+
break;
|
|
1510
|
+
}
|
|
1511
|
+
const responseUsage = modelResponseUsageFromSdkEvent(next.value);
|
|
1512
|
+
if (responseUsage) {
|
|
1513
|
+
responseUsageCount += 1;
|
|
1514
|
+
const observed = responseUsage.usage?.inputTokens;
|
|
1515
|
+
if (typeof observed === "number" && observed > 0) {
|
|
1516
|
+
lastInputTokensObserved = observed;
|
|
1517
|
+
await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, observed)
|
|
1518
|
+
.catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
|
|
1519
|
+
}
|
|
1520
|
+
await recordModelUsageAndDebitCredits(settings, db, {
|
|
1521
|
+
accountId: input.accountId,
|
|
1522
|
+
workspaceId: input.workspaceId,
|
|
1523
|
+
sessionId: input.sessionId,
|
|
1524
|
+
turnId: activeTurnId,
|
|
1525
|
+
model: turn.model,
|
|
1526
|
+
isCodexTurn,
|
|
1527
|
+
usage: responseUsage.usage,
|
|
1528
|
+
sourceKey: modelUsageSourceKey({
|
|
1529
|
+
responseId: responseUsage.responseId,
|
|
1530
|
+
dispatchId,
|
|
1531
|
+
positionalKey: `response-${responseUsageCount}`,
|
|
1532
|
+
}),
|
|
1533
|
+
});
|
|
1534
|
+
await reconcileConversationTruth();
|
|
1428
1535
|
try {
|
|
1429
|
-
|
|
1430
|
-
} catch {
|
|
1431
|
-
|
|
1536
|
+
await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
|
|
1537
|
+
} catch (limitError) {
|
|
1538
|
+
// Capture the run state at the boundary so the budget valve in
|
|
1539
|
+
// the outer catch can end this segment gracefully with full
|
|
1540
|
+
// conversation context preserved for the post-top-up resume.
|
|
1541
|
+
let serializedRunState: string | null = null;
|
|
1542
|
+
try {
|
|
1543
|
+
serializedRunState = stream.state.toString();
|
|
1544
|
+
} catch {
|
|
1545
|
+
serializedRunState = null;
|
|
1546
|
+
}
|
|
1547
|
+
throw new BudgetExhaustedError(
|
|
1548
|
+
limitError instanceof Error ? limitError.message : String(limitError),
|
|
1549
|
+
serializedRunState,
|
|
1550
|
+
);
|
|
1432
1551
|
}
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1552
|
+
}
|
|
1553
|
+
// Recording gate: a computer-use tool actually ran when an SDK
|
|
1554
|
+
// `tool_call_item` whose rawItem.type is "computer_call" streams through
|
|
1555
|
+
// (screenshot/click/type/scroll/etc. — the first computer action proves
|
|
1556
|
+
// the desktop was driven). Match the raw SDK event (ground truth) BEFORE
|
|
1557
|
+
// normalization. Only meaningful when a recording is live.
|
|
1558
|
+
if (activeRecording && !didComputerUse && isComputerCallStreamEvent(next.value)) {
|
|
1559
|
+
didComputerUse = true;
|
|
1560
|
+
}
|
|
1561
|
+
const normalized = normalizeSdkEvent(next.value);
|
|
1562
|
+
for (const event of normalized) {
|
|
1563
|
+
await batcher.push(event);
|
|
1437
1564
|
}
|
|
1438
1565
|
}
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
// the desktop was driven). Match the raw SDK event (ground truth) BEFORE
|
|
1443
|
-
// normalization. Only meaningful when a recording is live.
|
|
1444
|
-
if (activeRecording && !didComputerUse && isComputerCallStreamEvent(next.value)) {
|
|
1445
|
-
didComputerUse = true;
|
|
1566
|
+
} finally {
|
|
1567
|
+
if (!streamDone) {
|
|
1568
|
+
await iterator.return?.();
|
|
1446
1569
|
}
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1570
|
+
}
|
|
1571
|
+
await batcher.flush();
|
|
1572
|
+
await stream.completed.catch(() => undefined);
|
|
1573
|
+
if (responseUsageCount === 0) {
|
|
1574
|
+
const aggregateInput = (stream.state.usage as { inputTokens?: unknown } | undefined)?.inputTokens;
|
|
1575
|
+
if (typeof aggregateInput === "number" && aggregateInput > 0) {
|
|
1576
|
+
lastInputTokensObserved = aggregateInput;
|
|
1450
1577
|
}
|
|
1578
|
+
await recordModelUsageAndDebitCredits(settings, db, {
|
|
1579
|
+
accountId: input.accountId,
|
|
1580
|
+
workspaceId: input.workspaceId,
|
|
1581
|
+
sessionId: input.sessionId,
|
|
1582
|
+
turnId: activeTurnId,
|
|
1583
|
+
model: turn.model,
|
|
1584
|
+
isCodexTurn,
|
|
1585
|
+
usage: stream.state.usage,
|
|
1586
|
+
sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
|
|
1587
|
+
});
|
|
1451
1588
|
}
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1589
|
+
if (lastInputTokensObserved !== null) {
|
|
1590
|
+
await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, lastInputTokensObserved)
|
|
1591
|
+
.catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
|
|
1455
1592
|
}
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1593
|
+
|
|
1594
|
+
if (stream.interruptions.length > 0) {
|
|
1595
|
+
await reconcileConversationTruth();
|
|
1596
|
+
const approvals = runtime.serializeApprovals(stream.interruptions);
|
|
1597
|
+
await saveRunState(db, {
|
|
1598
|
+
accountId: input.accountId,
|
|
1599
|
+
workspaceId: input.workspaceId,
|
|
1600
|
+
sessionId: input.sessionId,
|
|
1601
|
+
turnId: activeTurnId,
|
|
1602
|
+
serializedRunState: stream.state.toString(),
|
|
1603
|
+
pendingApprovals: approvals,
|
|
1604
|
+
// Record the account freezing this state so a resume on a DIFFERENT
|
|
1605
|
+
// codex account strips its account-bound reasoning before replay (HOLE C).
|
|
1606
|
+
frozenCodexCredentialId: effectiveCodexCredentialId,
|
|
1607
|
+
});
|
|
1608
|
+
await publish!([
|
|
1609
|
+
{ type: "session.requiresAction", payload: { approvals } },
|
|
1610
|
+
{ type: "session.status.changed", payload: { status: "requires_action" } },
|
|
1611
|
+
], true);
|
|
1612
|
+
await finishTurn(db, input.workspaceId, activeTurnId, "requires_action");
|
|
1613
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "requires_action", activeTurnId);
|
|
1614
|
+
activityStatus = "requires_action";
|
|
1615
|
+
return { status: "requires_action" };
|
|
1463
1616
|
}
|
|
1464
|
-
await recordModelUsageAndDebitCredits(settings, db, {
|
|
1465
|
-
accountId: input.accountId,
|
|
1466
|
-
workspaceId: input.workspaceId,
|
|
1467
|
-
sessionId: input.sessionId,
|
|
1468
|
-
turnId,
|
|
1469
|
-
model: turn.model,
|
|
1470
|
-
isCodexTurn,
|
|
1471
|
-
usage: stream.state.usage,
|
|
1472
|
-
sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
|
|
1473
|
-
});
|
|
1474
|
-
}
|
|
1475
|
-
if (lastInputTokensObserved !== null) {
|
|
1476
|
-
await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, lastInputTokensObserved)
|
|
1477
|
-
.catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
|
|
1478
|
-
}
|
|
1479
1617
|
|
|
1480
|
-
|
|
1618
|
+
const finalOutput = String(stream.finalOutput ?? "");
|
|
1481
1619
|
await reconcileConversationTruth();
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
{ type: "
|
|
1620
|
+
if (settings.sessionHistorySource !== "items") {
|
|
1621
|
+
// Legacy conversation memory; in items mode the blob is only written
|
|
1622
|
+
// for requires_action pauses (the one RunState-only resume path).
|
|
1623
|
+
await saveRunState(db, {
|
|
1624
|
+
accountId: input.accountId,
|
|
1625
|
+
workspaceId: input.workspaceId,
|
|
1626
|
+
sessionId: input.sessionId,
|
|
1627
|
+
turnId: activeTurnId,
|
|
1628
|
+
serializedRunState: stream.state.toString(),
|
|
1629
|
+
pendingApprovals: [],
|
|
1630
|
+
frozenCodexCredentialId: effectiveCodexCredentialId,
|
|
1631
|
+
});
|
|
1632
|
+
}
|
|
1633
|
+
await publish!([
|
|
1634
|
+
{ type: "agent.message.completed", payload: { text: finalOutput } },
|
|
1635
|
+
{ type: "turn.completed", payload: { output: finalOutput } },
|
|
1636
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1497
1637
|
], true);
|
|
1498
|
-
await finishTurn(db, input.workspaceId,
|
|
1499
|
-
await setSessionStatus(db, input.workspaceId, input.sessionId, "
|
|
1500
|
-
|
|
1501
|
-
return { status: "requires_action" };
|
|
1502
|
-
}
|
|
1503
|
-
|
|
1504
|
-
const finalOutput = String(stream.finalOutput ?? "");
|
|
1505
|
-
await reconcileConversationTruth();
|
|
1506
|
-
if (settings.sessionHistorySource !== "items") {
|
|
1507
|
-
// Legacy conversation memory; in items mode the blob is only written
|
|
1508
|
-
// for requires_action pauses (the one RunState-only resume path).
|
|
1509
|
-
await saveRunState(db, {
|
|
1638
|
+
await finishTurn(db, input.workspaceId, activeTurnId, "idle");
|
|
1639
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1640
|
+
await recordUsageEvent(db, {
|
|
1510
1641
|
accountId: input.accountId,
|
|
1511
1642
|
workspaceId: input.workspaceId,
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1643
|
+
eventType: "agent_run.completed",
|
|
1644
|
+
quantity: 1,
|
|
1645
|
+
unit: "run",
|
|
1646
|
+
sourceResourceType: "session_turn",
|
|
1647
|
+
sourceResourceId: activeTurnId,
|
|
1648
|
+
idempotencyKey: `usage:agent_run.completed:${activeTurnId}`,
|
|
1517
1649
|
});
|
|
1650
|
+
activityStatus = "idle";
|
|
1651
|
+
return { status: "idle" };
|
|
1652
|
+
};
|
|
1653
|
+
|
|
1654
|
+
await prepareRunAttemptInput();
|
|
1655
|
+
let compactionRecoveryAttempted = false;
|
|
1656
|
+
let retriedAfterCompaction = false;
|
|
1657
|
+
while (true) {
|
|
1658
|
+
try {
|
|
1659
|
+
const result = await runStreamAttempt();
|
|
1660
|
+
if (retriedAfterCompaction) {
|
|
1661
|
+
observability.info("context compaction recovery succeeded after in-activity retry", {
|
|
1662
|
+
sessionId: input.sessionId,
|
|
1663
|
+
turnId: activeTurnId,
|
|
1664
|
+
});
|
|
1665
|
+
}
|
|
1666
|
+
return result;
|
|
1667
|
+
} catch (attemptError) {
|
|
1668
|
+
const overflow = classifyContextWindowOverflowError(attemptError);
|
|
1669
|
+
const compactionNeeded = findCompactionNeededError(attemptError);
|
|
1670
|
+
const recoveryKind = compactionNeeded ? "proactive" : overflow ? "overflow" : null;
|
|
1671
|
+
if (!recoveryKind || !publish || !turnStartedPublished || compactionRecoveryAttempted) {
|
|
1672
|
+
throw attemptError;
|
|
1673
|
+
}
|
|
1674
|
+
compactionRecoveryAttempted = true;
|
|
1675
|
+
await flushRuntimeBatcher();
|
|
1676
|
+
await reconcileConversationTruth({ skipInputOnlyRows: true });
|
|
1677
|
+
const progressPersisted = modelOrToolProgressPersisted;
|
|
1678
|
+
observability.warn("context compaction recovery attempted", {
|
|
1679
|
+
sessionId: input.sessionId,
|
|
1680
|
+
turnId: activeTurnId,
|
|
1681
|
+
reason: recoveryKind,
|
|
1682
|
+
progressPersisted,
|
|
1683
|
+
code: overflow?.code,
|
|
1684
|
+
error: overflow?.message ?? compactionNeeded?.message,
|
|
1685
|
+
signalTokens: compactionNeeded?.signalTokens,
|
|
1686
|
+
thresholdTokens: compactionNeeded?.thresholdTokens,
|
|
1687
|
+
});
|
|
1688
|
+
let compacted = false;
|
|
1689
|
+
try {
|
|
1690
|
+
const outcome = await forceContextCompaction(recoveryKind);
|
|
1691
|
+
compacted = outcome.compacted;
|
|
1692
|
+
} catch (compactError) {
|
|
1693
|
+
observability.warn("context compaction recovery compaction failed", {
|
|
1694
|
+
sessionId: input.sessionId,
|
|
1695
|
+
turnId: activeTurnId,
|
|
1696
|
+
error: compactError instanceof Error ? compactError.message : String(compactError),
|
|
1697
|
+
});
|
|
1698
|
+
}
|
|
1699
|
+
if (progressPersisted) {
|
|
1700
|
+
// The user's intent was live mid-turn — after a successful
|
|
1701
|
+
// compaction the turn AUTO-CONTINUES via the same synthesized
|
|
1702
|
+
// resume-notice requeue the worker-shutdown checkpoint uses (never
|
|
1703
|
+
// the original trigger, so side effects are not replayed). Idle is
|
|
1704
|
+
// the fallback, not the default; it remains for: legacy run-state
|
|
1705
|
+
// history (resuming the frozen pre-compaction RunState would
|
|
1706
|
+
// overflow again), a compaction that could not shrink anything,
|
|
1707
|
+
// and a turn that ALREADY resumed from an overflow (loop guard —
|
|
1708
|
+
// a second overflow in the same lineage stops churning).
|
|
1709
|
+
const triggerPayload =
|
|
1710
|
+
trigger.payload && typeof trigger.payload === "object" && !Array.isArray(trigger.payload)
|
|
1711
|
+
? trigger.payload as Record<string, unknown>
|
|
1712
|
+
: {};
|
|
1713
|
+
const compactionResumedTurn = triggerType === "turn.preempted"
|
|
1714
|
+
&& (triggerPayload.reason === "context_compacted" || triggerPayload.reason === "context_overflow_compacted");
|
|
1715
|
+
const canAutoContinue =
|
|
1716
|
+
compacted && !compactionResumedTurn && settings.sessionHistorySource === "items" && activeTurnId != null;
|
|
1717
|
+
if (canAutoContinue) {
|
|
1718
|
+
const [preemptedEvent] = await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [
|
|
1719
|
+
{
|
|
1720
|
+
turnId: activeTurnId,
|
|
1721
|
+
type: "turn.preempted",
|
|
1722
|
+
payload: {
|
|
1723
|
+
triggerEventId: input.triggerEventId,
|
|
1724
|
+
reason: "context_compacted",
|
|
1725
|
+
resumeWithNotice: true,
|
|
1726
|
+
text: CONTEXT_OVERFLOW_RESUME_TEXT,
|
|
1727
|
+
},
|
|
1728
|
+
},
|
|
1729
|
+
{ turnId: activeTurnId, type: "session.status.changed", payload: { status: "queued" } },
|
|
1730
|
+
]);
|
|
1731
|
+
await requeuePreemptedTurn(db, input.workspaceId, activeTurnId, preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
1732
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
|
|
1733
|
+
activityStatus = "preempted";
|
|
1734
|
+
observability.info("context compaction recovery succeeded by compacting and auto-continuing", {
|
|
1735
|
+
sessionId: input.sessionId,
|
|
1736
|
+
turnId: activeTurnId,
|
|
1737
|
+
reason: recoveryKind,
|
|
1738
|
+
compacted,
|
|
1739
|
+
});
|
|
1740
|
+
return { status: "preempted" };
|
|
1741
|
+
}
|
|
1742
|
+
await publish!([
|
|
1743
|
+
{
|
|
1744
|
+
type: "turn.failed",
|
|
1745
|
+
payload: {
|
|
1746
|
+
error: CONTEXT_WINDOW_OVERFLOW_RECOVERY_MESSAGE,
|
|
1747
|
+
code: recoveryKind === "overflow" ? "context_window_overflow_compacted" : "context_compacted",
|
|
1748
|
+
retryable: false,
|
|
1749
|
+
recovery: "user_message",
|
|
1750
|
+
compacted,
|
|
1751
|
+
},
|
|
1752
|
+
},
|
|
1753
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1754
|
+
], true);
|
|
1755
|
+
await finishTurn(db, input.workspaceId, activeTurnId, "failed");
|
|
1756
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1757
|
+
activityStatus = "idle";
|
|
1758
|
+
activityError = attemptError;
|
|
1759
|
+
observability.info("context compaction recovery succeeded by compacting and idling", {
|
|
1760
|
+
sessionId: input.sessionId,
|
|
1761
|
+
turnId: activeTurnId,
|
|
1762
|
+
reason: recoveryKind,
|
|
1763
|
+
compacted,
|
|
1764
|
+
});
|
|
1765
|
+
return { status: "idle" };
|
|
1766
|
+
}
|
|
1767
|
+
retriedAfterCompaction = true;
|
|
1768
|
+
observability.info("context compaction recovery retrying turn after compaction", {
|
|
1769
|
+
sessionId: input.sessionId,
|
|
1770
|
+
turnId: activeTurnId,
|
|
1771
|
+
reason: recoveryKind,
|
|
1772
|
+
compacted,
|
|
1773
|
+
});
|
|
1774
|
+
await prepareRunAttemptInput();
|
|
1775
|
+
}
|
|
1518
1776
|
}
|
|
1519
|
-
await publish([
|
|
1520
|
-
{ type: "agent.message.completed", payload: { text: finalOutput } },
|
|
1521
|
-
{ type: "turn.completed", payload: { output: finalOutput } },
|
|
1522
|
-
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1523
|
-
], true);
|
|
1524
|
-
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
1525
|
-
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1526
|
-
await recordUsageEvent(db, {
|
|
1527
|
-
accountId: input.accountId,
|
|
1528
|
-
workspaceId: input.workspaceId,
|
|
1529
|
-
eventType: "agent_run.completed",
|
|
1530
|
-
quantity: 1,
|
|
1531
|
-
unit: "run",
|
|
1532
|
-
sourceResourceType: "session_turn",
|
|
1533
|
-
sourceResourceId: turnId,
|
|
1534
|
-
idempotencyKey: `usage:agent_run.completed:${turnId}`,
|
|
1535
|
-
});
|
|
1536
|
-
activityStatus = "idle";
|
|
1537
|
-
return { status: "idle" };
|
|
1538
1777
|
} catch (error) {
|
|
1539
1778
|
// Graceful worker shutdown (deploy / rollout restart): checkpoint and
|
|
1540
1779
|
// hand the turn back instead of failing the session. Conversation truth
|
|
@@ -1572,7 +1811,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1572
1811
|
}
|
|
1573
1812
|
if (isWorkerShutdownCancellation(error) && preemptTurnId) {
|
|
1574
1813
|
try {
|
|
1575
|
-
await
|
|
1814
|
+
await flushRuntimeBatcher();
|
|
1576
1815
|
await reconcileConversationTruth();
|
|
1577
1816
|
// An approval-decision rerun always replays its original trigger:
|
|
1578
1817
|
// the decision is applied through the approval resume path reading
|
|
@@ -1639,7 +1878,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1639
1878
|
if (error instanceof CancelledFailure) {
|
|
1640
1879
|
activityStatus = "cancelled";
|
|
1641
1880
|
activityError = error;
|
|
1642
|
-
await
|
|
1881
|
+
await flushRuntimeBatcher();
|
|
1643
1882
|
if (preemptTurnId) {
|
|
1644
1883
|
await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => undefined);
|
|
1645
1884
|
}
|
|
@@ -1652,7 +1891,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1652
1891
|
// that resumption.
|
|
1653
1892
|
const maxTurns = maxTurnsExceededRunState(error);
|
|
1654
1893
|
if (maxTurns && publish && turnId && turnStartedPublished) {
|
|
1655
|
-
await
|
|
1894
|
+
await flushRuntimeBatcher();
|
|
1656
1895
|
// The SDK attaches the run state at the throw site; persisting it lets
|
|
1657
1896
|
// the continuation resume with this segment's full context. If capture
|
|
1658
1897
|
// ever fails, the continuation falls back to the previous snapshot --
|
|
@@ -1703,7 +1942,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1703
1942
|
if (usageLimit && publish && turnId && turnStartedPublished) {
|
|
1704
1943
|
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
1705
1944
|
const goalActive = Boolean(goal && goal.status === "active");
|
|
1706
|
-
await
|
|
1945
|
+
await flushRuntimeBatcher();
|
|
1707
1946
|
await reconcileConversationTruth();
|
|
1708
1947
|
const serializedRunState = agentsErrorRunState(error);
|
|
1709
1948
|
const runStateSaved = Boolean(serializedRunState) && settings.sessionHistorySource !== "items";
|
|
@@ -1834,7 +2073,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1834
2073
|
// top-up). An active goal pauses visibly with reason "limits" at the
|
|
1835
2074
|
// next continuation evaluation, without consuming continuation budget.
|
|
1836
2075
|
if (error instanceof BudgetExhaustedError && publish && turnId && turnStartedPublished) {
|
|
1837
|
-
await
|
|
2076
|
+
await flushRuntimeBatcher();
|
|
1838
2077
|
await reconcileConversationTruth();
|
|
1839
2078
|
const runStateSaved = Boolean(error.serializedRunState) && settings.sessionHistorySource !== "items";
|
|
1840
2079
|
if (error.serializedRunState && settings.sessionHistorySource !== "items") {
|
|
@@ -1880,7 +2119,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1880
2119
|
if (failure.retryable && publish && turnId && turnStartedPublished) {
|
|
1881
2120
|
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
1882
2121
|
const goalActive = Boolean(goal && goal.status === "active");
|
|
1883
|
-
await
|
|
2122
|
+
await flushRuntimeBatcher();
|
|
1884
2123
|
// Provider errors rarely carry SDK run state; a null falls back to
|
|
1885
2124
|
// the previous snapshot, same degraded-context contract as the
|
|
1886
2125
|
// max-turns path above.
|