@opengeni/worker-bundle 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -0
- package/dist/index.js +490 -223
- package/dist/index.js.map +1 -1
- package/package.json +10 -10
- package/src/activities/agent-turn.ts +519 -233
- package/src/activities/capabilities.ts +42 -0
- package/src/activities/context-compaction.ts +51 -58
- package/src/activities/run-input.ts +2 -2
- package/src/index.ts +6 -1
|
@@ -49,12 +49,15 @@ import {
|
|
|
49
49
|
normalizeSdkEvent,
|
|
50
50
|
sanitizeHistoryItemsForModel,
|
|
51
51
|
summarizeForCompaction,
|
|
52
|
+
findCompactionNeededError,
|
|
53
|
+
SUMMARY_BUFFER_TOKENS,
|
|
52
54
|
type SandboxFileDownload,
|
|
53
55
|
type OpenGeniRuntime,
|
|
56
|
+
type ComputerToolMode,
|
|
54
57
|
} from "@opengeni/runtime";
|
|
55
|
-
import { calculateModelUsageCostMicros, configuredModelPricing, configuredStaticUsageLimits, sandboxWarmRateMicrosPerSecond, type ModelUsageInput, type Settings } from "@opengeni/config";
|
|
58
|
+
import { calculateModelUsageCostMicros, configuredModelPricing, configuredStaticUsageLimits, sandboxWarmRateMicrosPerSecond, type ModelUsageInput, type ModelProviderApi, type RegistryProviderKind, type Settings } from "@opengeni/config";
|
|
56
59
|
import { CancelledFailure } from "@temporalio/activity";
|
|
57
|
-
import { settingsWithCodexCredential, settingsWithEnabledCapabilityMcpServers } from "./capabilities";
|
|
60
|
+
import { settingsWithCodexCredential, settingsWithEnabledCapabilityMcpServers, settingsWithSessionMcpServersForRun } from "./capabilities";
|
|
58
61
|
import { chooseRotationActive, computeIdleDelayMs, computeReactiveRotationResume, type CodexRotationStrategy, type RotationDecision } from "./codex-rotation";
|
|
59
62
|
import type { CodexAccountStatus } from "@opengeni/db";
|
|
60
63
|
import { buildCodexTokenResolver } from "./codex-auth";
|
|
@@ -130,6 +133,14 @@ export function selectCodexCredentialForTurn(args: {
|
|
|
130
133
|
// agent must not blindly replay side effects it cannot see in its history:
|
|
131
134
|
// at most the single in-flight model step was lost, and anything it started
|
|
132
135
|
// there may or may not have completed.
|
|
136
|
+
// Resume notice after an in-turn context-window overflow was recovered by
|
|
137
|
+
// compaction: the turn's earlier work survives (albeit summarized), so the
|
|
138
|
+
// agent continues instead of the session dying or stalling idle.
|
|
139
|
+
export const CONTEXT_OVERFLOW_RESUME_TEXT = [
|
|
140
|
+
"[TURN RESUMED AFTER CONTEXT COMPACTION] The conversation approached or exceeded the model's context window mid-turn; older history above was compacted into a summary.",
|
|
141
|
+
"Continue the original task from where it left off. Before repeating any action with side effects, check whether it already happened.",
|
|
142
|
+
].join("\n");
|
|
143
|
+
|
|
133
144
|
export const WORKER_SHUTDOWN_RESUME_TEXT = [
|
|
134
145
|
"[TURN RESUMED AFTER WORKER RESTART] The platform worker running this turn restarted (deploy/rollout) before the turn finished.",
|
|
135
146
|
"Your conversation history above, including this turn's earlier work, is preserved up to the last checkpoint; at most the single in-flight model step after it was lost.",
|
|
@@ -157,6 +168,54 @@ export function isWorkerShutdownCancellation(error: unknown): boolean {
|
|
|
157
168
|
return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
|
|
158
169
|
}
|
|
159
170
|
|
|
171
|
+
export const CONTEXT_WINDOW_OVERFLOW_RECOVERY_MESSAGE =
|
|
172
|
+
"The conversation reached the model context compaction threshold; it has been compacted. Send a message to continue.";
|
|
173
|
+
|
|
174
|
+
export function classifyContextWindowOverflowError(error: unknown): { message: string; code?: string; detail?: string } | null {
|
|
175
|
+
const fields = collectErrorStrings(error);
|
|
176
|
+
const matched = fields.find((value) =>
|
|
177
|
+
/context[_\s-]*length[_\s-]*exceeded/i.test(value)
|
|
178
|
+
|| /exceeds?\s+(?:the\s+)?context\s+window/i.test(value)
|
|
179
|
+
|| /maximum\s+context\s+length/i.test(value)
|
|
180
|
+
|| /context\s+window[^.]*exceed/i.test(value)
|
|
181
|
+
);
|
|
182
|
+
if (!matched) {
|
|
183
|
+
return null;
|
|
184
|
+
}
|
|
185
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
186
|
+
const code = fields.find((value) => /context[_\s-]*length[_\s-]*exceeded/i.test(value));
|
|
187
|
+
return {
|
|
188
|
+
message,
|
|
189
|
+
...(code ? { code } : {}),
|
|
190
|
+
...(matched && matched !== message ? { detail: matched } : {}),
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function collectErrorStrings(value: unknown, seen = new WeakSet<object>()): string[] {
|
|
195
|
+
if (typeof value === "string") {
|
|
196
|
+
return [value];
|
|
197
|
+
}
|
|
198
|
+
if (!value || typeof value !== "object") {
|
|
199
|
+
return [];
|
|
200
|
+
}
|
|
201
|
+
if (seen.has(value)) {
|
|
202
|
+
return [];
|
|
203
|
+
}
|
|
204
|
+
seen.add(value);
|
|
205
|
+
const out: string[] = [];
|
|
206
|
+
const record = value as Record<string, unknown>;
|
|
207
|
+
for (const key of ["message", "code", "type", "name", "param"]) {
|
|
208
|
+
const field = record[key];
|
|
209
|
+
if (typeof field === "string" && field.length > 0) {
|
|
210
|
+
out.push(field);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
for (const key of ["error", "cause", "response", "data"]) {
|
|
214
|
+
out.push(...collectErrorStrings(record[key], seen));
|
|
215
|
+
}
|
|
216
|
+
return out;
|
|
217
|
+
}
|
|
218
|
+
|
|
160
219
|
/**
|
|
161
220
|
* Compute the conversation-truth rows a reconcile pass should append, given the
|
|
162
221
|
* SDK's current `state.history` and the count already persisted.
|
|
@@ -235,6 +294,19 @@ export function historyRowsToAppend(
|
|
|
235
294
|
return { rows, nextWatermark: sanitized.length, nextPosition: nextPosition + rows.length };
|
|
236
295
|
}
|
|
237
296
|
|
|
297
|
+
function isModelOrToolProgressHistoryItem(item: Record<string, unknown>): boolean {
|
|
298
|
+
if (item.type === "message") {
|
|
299
|
+
return item.role === "assistant";
|
|
300
|
+
}
|
|
301
|
+
if (item.type === "reasoning" || item.type === "compaction") {
|
|
302
|
+
return true;
|
|
303
|
+
}
|
|
304
|
+
if (typeof item.type === "string") {
|
|
305
|
+
return item.type !== "message";
|
|
306
|
+
}
|
|
307
|
+
return false;
|
|
308
|
+
}
|
|
309
|
+
|
|
238
310
|
/**
|
|
239
311
|
* Seed the turn-end reconcile watermark (`persistedHistoryCount`) from EXACTLY the
|
|
240
312
|
* view `state.history` was seeded from, so the model-input length and the watermark
|
|
@@ -364,6 +436,43 @@ export function shouldStartOnTurnRecording(params: {
|
|
|
364
436
|
);
|
|
365
437
|
}
|
|
366
438
|
|
|
439
|
+
/**
|
|
440
|
+
* Decide the EXPLICIT computer-use tool transport for THIS turn.
|
|
441
|
+
*
|
|
442
|
+
* The runtime's SDK-mirrored capability would otherwise pick hosted-vs-function
|
|
443
|
+
* tools by string-sniffing the bound model instance's constructor name for
|
|
444
|
+
* "ChatCompletions" (supportsStructuredToolOutputTransport). That is fragile: a
|
|
445
|
+
* wrapped / proxied / minified model instance defeats the sniff and a
|
|
446
|
+
* chat-completions provider would silently get the HOSTED `computer_use_preview`
|
|
447
|
+
* tool it 400s on every turn. So the mode is decided HERE — the worker's model
|
|
448
|
+
* resolution is the ONE place a provider's true wire identity is authoritative —
|
|
449
|
+
* and threaded to the runtime as an explicit flag (buildAgent → computerToolMode):
|
|
450
|
+
* • codex-subscription → "function-image": the ChatGPT/Codex backend rejects
|
|
451
|
+
* hosted tool types but SEES structured `input_image` tool results.
|
|
452
|
+
* • a "chat" (OpenAIChatCompletionsModel wire) provider → "function-text": it takes
|
|
453
|
+
* function tools but can't read structured image results, so screenshots render
|
|
454
|
+
* as a text data-URL.
|
|
455
|
+
* • everything else — built-in Azure/OpenAI responses, registry "responses"
|
|
456
|
+
* providers, AND the LEGACY global-client fallback (resolveTurnModel returned
|
|
457
|
+
* null) — → "hosted": real Responses hosted-tool support.
|
|
458
|
+
*
|
|
459
|
+
* Pure + exported so the mapping is unit-testable without a live turn.
|
|
460
|
+
*/
|
|
461
|
+
export function computerToolModeForTurn(
|
|
462
|
+
resolvedModel: { provider: { kind: RegistryProviderKind; api: ModelProviderApi } } | null,
|
|
463
|
+
): ComputerToolMode {
|
|
464
|
+
if (!resolvedModel) {
|
|
465
|
+
return "hosted"; // legacy built-in Responses client — real hosted support
|
|
466
|
+
}
|
|
467
|
+
if (resolvedModel.provider.kind === "codex-subscription") {
|
|
468
|
+
return "function-image";
|
|
469
|
+
}
|
|
470
|
+
if (resolvedModel.provider.api === "chat") {
|
|
471
|
+
return "function-text";
|
|
472
|
+
}
|
|
473
|
+
return "hosted";
|
|
474
|
+
}
|
|
475
|
+
|
|
367
476
|
/**
|
|
368
477
|
* SELF-HEAL helper for the all-capped rotation idle (invariant 4: BOUNDED, no thrash).
|
|
369
478
|
* The turn hot path never refreshes Codex usage — only the usage API route does — so a
|
|
@@ -436,6 +545,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
436
545
|
// static frame of an untouched desktop is not worth uploading).
|
|
437
546
|
let didComputerUse = false;
|
|
438
547
|
let batcher: ReturnType<typeof createRuntimeBatcher> | null = null;
|
|
548
|
+
const flushRuntimeBatcher = async () => {
|
|
549
|
+
const current = batcher as ReturnType<typeof createRuntimeBatcher> | null;
|
|
550
|
+
await current?.flush().catch(() => undefined);
|
|
551
|
+
};
|
|
439
552
|
let preparedTools: Awaited<ReturnType<OpenGeniRuntime["prepareTools"]>> | null = null;
|
|
440
553
|
let publish: ((events: Array<Omit<AppendEventInput, "producerId" | "producerSeq" | "turnId">>, immediate?: boolean) => Promise<void>) | null = null;
|
|
441
554
|
let turnStartedPublished = false;
|
|
@@ -464,12 +577,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
464
577
|
// actually persisted, so a non-monotonic history can never desync it.
|
|
465
578
|
let persistedHistoryCount = 0;
|
|
466
579
|
let historyCountAtTurnStart = 0;
|
|
580
|
+
let modelOrToolProgressPersisted = false;
|
|
467
581
|
// Next free WHOLE-NUMBER absolute position to append at. Tracked separately
|
|
468
582
|
// from persistedHistoryCount (the in-memory slice index) because a compaction
|
|
469
583
|
// inserts a fractional summary position, so total rows no longer equal
|
|
470
584
|
// max(position)+1 and the slice index can no longer double as the position.
|
|
471
585
|
let nextHistoryPosition = 0;
|
|
472
|
-
const reconcileConversationTruth = async () => {
|
|
586
|
+
const reconcileConversationTruth = async (options: { skipInputOnlyRows?: boolean } = {}) => {
|
|
473
587
|
if (!stream || !turnId) {
|
|
474
588
|
return;
|
|
475
589
|
}
|
|
@@ -481,7 +595,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
481
595
|
persistedHistoryCount,
|
|
482
596
|
nextHistoryPosition,
|
|
483
597
|
);
|
|
484
|
-
|
|
598
|
+
const hasModelOrToolProgress = rows.some((row) => isModelOrToolProgressHistoryItem(row.item));
|
|
599
|
+
const shouldAppendRows = rows.length > 0 && (!options.skipInputOnlyRows || hasModelOrToolProgress);
|
|
600
|
+
if (shouldAppendRows) {
|
|
485
601
|
await appendSessionHistoryItems(db, {
|
|
486
602
|
accountId: input.accountId,
|
|
487
603
|
workspaceId: input.workspaceId,
|
|
@@ -495,8 +611,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
495
611
|
items: rows,
|
|
496
612
|
});
|
|
497
613
|
}
|
|
498
|
-
|
|
499
|
-
|
|
614
|
+
if (shouldAppendRows || !options.skipInputOnlyRows) {
|
|
615
|
+
persistedHistoryCount = nextWatermark;
|
|
616
|
+
nextHistoryPosition = nextPosition;
|
|
617
|
+
}
|
|
618
|
+
if (hasModelOrToolProgress) {
|
|
619
|
+
modelOrToolProgressPersisted = true;
|
|
620
|
+
}
|
|
500
621
|
}
|
|
501
622
|
const envelope = sandboxStateEntryFromRunState(stream.state);
|
|
502
623
|
if (envelope) {
|
|
@@ -764,12 +885,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
764
885
|
// runtime falls back to runSettings.agentInstructionsTemplate (the
|
|
765
886
|
// deployment default, byte-identical to the historical preamble).
|
|
766
887
|
const workspaceAgentInstructions = await resolveWorkspaceAgentInstructions(db, input.workspaceId);
|
|
767
|
-
const
|
|
888
|
+
const baseRunSettings = {
|
|
768
889
|
...settingsWithPackSandboxImage(capabilitySettings, packRuntime.sandboxImage),
|
|
769
890
|
openaiModel: turn.model,
|
|
770
891
|
openaiReasoningEffort: turn.reasoningEffort,
|
|
771
892
|
sandboxBackend: turn.sandboxBackend,
|
|
772
893
|
};
|
|
894
|
+
const runSettings = await settingsWithSessionMcpServersForRun(db, input.workspaceId, input.sessionId, baseRunSettings);
|
|
773
895
|
// Multi-provider per-turn routing → the provider gating (compaction mode,
|
|
774
896
|
// hosted web search, encrypted reasoning, context window) the agent and
|
|
775
897
|
// compaction summarizer must use; null falls back to the legacy global
|
|
@@ -1200,8 +1322,17 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1200
1322
|
// providers = the SDK's own ChatCompletions detection) keeps the SDK
|
|
1201
1323
|
// default.
|
|
1202
1324
|
structuredToolTransport: resolvedModel.provider.kind !== "codex-subscription",
|
|
1325
|
+
// EXPLICIT computer-use tool transport, derived from the resolved provider's
|
|
1326
|
+
// authoritative wire identity (codex → function-image, chat → function-text,
|
|
1327
|
+
// responses → hosted) so the runtime never string-sniffs the model instance's
|
|
1328
|
+
// constructor name. See {@link computerToolModeForTurn}.
|
|
1329
|
+
computerToolMode: computerToolModeForTurn(resolvedModel),
|
|
1203
1330
|
}
|
|
1204
|
-
|
|
1331
|
+
// LEGACY global-client fallback (resolveTurnModel returned null → the model
|
|
1332
|
+
// is not in the registry, served by the built-in OpenAI/Azure Responses
|
|
1333
|
+
// client). That backend has real hosted support, so pin computerToolMode to
|
|
1334
|
+
// "hosted" EXPLICITLY rather than leaving the runtime to sniff the instance.
|
|
1335
|
+
: { computerToolMode: computerToolModeForTurn(null) }),
|
|
1205
1336
|
...(packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {}),
|
|
1206
1337
|
...(workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {}),
|
|
1207
1338
|
...(workspaceEnvironment
|
|
@@ -1214,11 +1345,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1214
1345
|
}
|
|
1215
1346
|
: {}),
|
|
1216
1347
|
});
|
|
1348
|
+
const compactSummarizer = resolvedModel
|
|
1349
|
+
? (s: Settings, m: Array<Record<string, unknown>>) => withCodex(() => summarizeForCompaction(s, m, {
|
|
1350
|
+
client: resolvedModel.client,
|
|
1351
|
+
api: resolvedModel.provider.api,
|
|
1352
|
+
model: resolvedModel.configured.id,
|
|
1353
|
+
maxOutputTokens: SUMMARY_BUFFER_TOKENS,
|
|
1354
|
+
}))
|
|
1355
|
+
: undefined;
|
|
1217
1356
|
// Pre-turn client-side context compaction (Azure path). When the
|
|
1218
|
-
// resolved mode is "client" and the
|
|
1219
|
-
//
|
|
1220
|
-
//
|
|
1221
|
-
// for this turn, so the model sees [summary, ...recent tail] + new input.
|
|
1357
|
+
// resolved mode is "client" and the single Codex-parity threshold is
|
|
1358
|
+
// crossed, this summarizes the current active history and rebuilds active
|
|
1359
|
+
// history as [user messages..., summary] BEFORE the model input is read.
|
|
1222
1360
|
// Best-effort: a no-op or failure leaves the un-compacted history intact.
|
|
1223
1361
|
// Skipped on approval/preempt resumes (no fresh user turn; the frozen
|
|
1224
1362
|
// RunState or in-flight tail must replay verbatim).
|
|
@@ -1238,22 +1376,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1238
1376
|
// registry provider, summarize on THAT provider's client + wire API
|
|
1239
1377
|
// (a chat provider can't summarize through OpenAI/Azure). Null
|
|
1240
1378
|
// resolution keeps the default built-in Responses summarizer.
|
|
1241
|
-
|
|
1242
|
-
? (s, m) => withCodex(() => summarizeForCompaction(s, m, {
|
|
1243
|
-
client: resolvedModel.client,
|
|
1244
|
-
api: resolvedModel.provider.api,
|
|
1245
|
-
model: resolvedModel.configured.id,
|
|
1246
|
-
maxOutputTokens: s.contextSummaryMaxTokens,
|
|
1247
|
-
}))
|
|
1248
|
-
: undefined,
|
|
1379
|
+
compactSummarizer,
|
|
1249
1380
|
forced ? { force: true } : {},
|
|
1250
1381
|
);
|
|
1251
1382
|
if (outcome.compacted) {
|
|
1252
|
-
|
|
1253
|
-
// event label; otherwise a hard-forced (at/over hardFraction*B)
|
|
1254
|
-
// compaction is surfaced as "hard" so the backstop firing is
|
|
1255
|
-
// observable downstream, and the default soft trigger is implicit.
|
|
1256
|
-
const trigger = forced ? "operator" : outcome.hardForced ? "hard" : undefined;
|
|
1383
|
+
const trigger = forced ? "operator" : undefined;
|
|
1257
1384
|
await publish([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, ...(trigger ? { trigger } : {}) } }]);
|
|
1258
1385
|
}
|
|
1259
1386
|
} catch (compactError) {
|
|
@@ -1268,226 +1395,385 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1268
1395
|
// "current account" the single strip rule compares against (null is the
|
|
1269
1396
|
// built-in/Azure account, so a non-codex turn still drops codex-produced
|
|
1270
1397
|
// reasoning, and a no-codex-history session is a byte-for-byte no-op).
|
|
1271
|
-
const
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
1321
|
-
},
|
|
1322
|
-
// P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
|
|
1323
|
-
// keystone). Absent when the flag is off -> legacy build-and-discard.
|
|
1324
|
-
...(resolvedSandbox
|
|
1325
|
-
? {
|
|
1326
|
-
ownedSandbox: {
|
|
1327
|
-
client: resolvedSandbox.established.client,
|
|
1328
|
-
session: resolvedSandbox.established.session,
|
|
1329
|
-
sessionState: resolvedSandbox.established.sessionState,
|
|
1330
|
-
// Pin platform setup (hooks + file materialization) to the un-proxied
|
|
1331
|
-
// established box — never through the routing proxy, which would
|
|
1332
|
-
// re-route those execs onto a machine swapped in mid-turn.
|
|
1333
|
-
...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
|
|
1334
|
-
},
|
|
1335
|
-
}
|
|
1336
|
-
: {}),
|
|
1337
|
-
});
|
|
1338
|
-
stream = await withCodex(runStreamOnce);
|
|
1339
|
-
batcher = createRuntimeBatcher(async (events) => {
|
|
1340
|
-
await publish!(events);
|
|
1341
|
-
});
|
|
1398
|
+
const activeTurnId = turnId;
|
|
1399
|
+
if (!activeTurnId) {
|
|
1400
|
+
throw new Error("Turn id was not initialized");
|
|
1401
|
+
}
|
|
1402
|
+
let runInput: Awaited<ReturnType<typeof turnInput>>["input"] | null = null;
|
|
1403
|
+
const prepareRunAttemptInput = async () => {
|
|
1404
|
+
const prepared = await turnInput(
|
|
1405
|
+
db,
|
|
1406
|
+
runtime,
|
|
1407
|
+
agent,
|
|
1408
|
+
trigger,
|
|
1409
|
+
runSettings,
|
|
1410
|
+
{ currentCodexCredentialId: effectiveCodexCredentialId },
|
|
1411
|
+
);
|
|
1412
|
+
runInput = prepared.input;
|
|
1413
|
+
// Slice index = the length of the model-facing (active) history this turn
|
|
1414
|
+
// is seeded from; new items beyond it (the trigger message + this turn's
|
|
1415
|
+
// generated items) are the ones to persist. After a compaction this is the
|
|
1416
|
+
// short [summary, ...tail] active set, NOT the total row count. The
|
|
1417
|
+
// absolute write position is tracked separately (next whole number past
|
|
1418
|
+
// the max existing position) because the fractional summary row means
|
|
1419
|
+
// total rows no longer equal max(position)+1. Pre-compaction both reduce to
|
|
1420
|
+
// the old total-count value, so the common path is unchanged.
|
|
1421
|
+
//
|
|
1422
|
+
// CRITICAL: seed from the SANITIZED active-row length, not the raw active
|
|
1423
|
+
// count. `prepareRunInput` builds `state.history` from
|
|
1424
|
+
// `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
|
|
1425
|
+
// rows (a legacy orphan/dangling pair), the in-memory history this turn
|
|
1426
|
+
// starts from is K shorter than the raw row count. The reconcile slices the
|
|
1427
|
+
// re-sanitized `state.history` off `persistedHistoryCount`; seeding it from
|
|
1428
|
+
// the raw count (K too high) skips K genuinely-new items, and a
|
|
1429
|
+
// `function_call` left in that skipped region can later have its
|
|
1430
|
+
// `function_call_result` persisted alone — the orphan that 400s on replay
|
|
1431
|
+
// and bricks the session (issue-61). The sanitized seed is already
|
|
1432
|
+
// orphan-free, so it is a stable prefix of the re-sanitized history and the
|
|
1433
|
+
// slice begins exactly at the first genuinely-new item.
|
|
1434
|
+
const activeSeedRows = await getActiveSessionHistoryItems(db, input.workspaceId, input.sessionId);
|
|
1435
|
+
// Seed the reconcile watermark from EXACTLY the view the model's
|
|
1436
|
+
// `state.history` was seeded from (items strip on the items path = HOLE D; NO
|
|
1437
|
+
// strip on the run-state blob path, where foreign reasoning is neutralized but
|
|
1438
|
+
// KEPT = HOLE E), so the model-input length and the watermark never disagree.
|
|
1439
|
+
persistedHistoryCount = reconcileSeedCount(
|
|
1440
|
+
activeSeedRows,
|
|
1441
|
+
prepared.modelHistoryFromItems,
|
|
1442
|
+
{ currentCodexCredentialId: effectiveCodexCredentialId },
|
|
1443
|
+
);
|
|
1444
|
+
historyCountAtTurnStart = persistedHistoryCount;
|
|
1445
|
+
nextHistoryPosition = await nextSessionHistoryPosition(db, input.workspaceId, input.sessionId);
|
|
1446
|
+
};
|
|
1342
1447
|
|
|
1343
|
-
const
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
}
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1448
|
+
const forceContextCompaction = async (triggerLabel: "overflow" | "proactive") => {
|
|
1449
|
+
const clientCompactionSettings: Settings = { ...runSettings, contextCompactionMode: "client" };
|
|
1450
|
+
const outcome = await maybeCompactContext(
|
|
1451
|
+
db,
|
|
1452
|
+
clientCompactionSettings,
|
|
1453
|
+
{ accountId: input.accountId, workspaceId: input.workspaceId, sessionId: input.sessionId, turnId: activeTurnId },
|
|
1454
|
+
session.lastInputTokens,
|
|
1455
|
+
compactSummarizer,
|
|
1456
|
+
{ force: true },
|
|
1457
|
+
);
|
|
1458
|
+
if (outcome.compacted) {
|
|
1459
|
+
await publish!([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, trigger: triggerLabel } }]);
|
|
1460
|
+
}
|
|
1461
|
+
return outcome;
|
|
1462
|
+
};
|
|
1463
|
+
|
|
1464
|
+
const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
|
|
1465
|
+
if (!runInput) {
|
|
1466
|
+
throw new Error("Run input was not prepared");
|
|
1467
|
+
}
|
|
1468
|
+
stream = undefined;
|
|
1469
|
+
batcher = null;
|
|
1470
|
+
let responseUsageCount = 0;
|
|
1471
|
+
// Actual input tokens of the most recent model response this turn; the
|
|
1472
|
+
// pre-read trigger for the NEXT turn. Persisted at every turn-end path.
|
|
1473
|
+
let lastInputTokensObserved: number | null = null;
|
|
1474
|
+
throwIfWorkerShuttingDown();
|
|
1475
|
+
const runStreamOnce = (): ReturnType<OpenGeniRuntime["runStream"]> => runtime.runStream(agent, runInput!, runSettings, {
|
|
1476
|
+
sandboxEnvironment,
|
|
1477
|
+
onRuntimeEvent: async (event) => {
|
|
1478
|
+
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
1479
|
+
},
|
|
1480
|
+
// P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
|
|
1481
|
+
// keystone). Absent when the flag is off -> legacy build-and-discard.
|
|
1482
|
+
...(resolvedSandbox
|
|
1483
|
+
? {
|
|
1484
|
+
ownedSandbox: {
|
|
1485
|
+
client: resolvedSandbox.established.client,
|
|
1486
|
+
session: resolvedSandbox.established.session,
|
|
1487
|
+
sessionState: resolvedSandbox.established.sessionState,
|
|
1488
|
+
// Pin platform setup (hooks + file materialization) to the un-proxied
|
|
1489
|
+
// established box — never through the routing proxy, which would
|
|
1490
|
+
// re-route those execs onto a machine swapped in mid-turn.
|
|
1491
|
+
...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
|
|
1492
|
+
},
|
|
1358
1493
|
}
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1494
|
+
: {}),
|
|
1495
|
+
contextCompactionSignalTokens: () => lastInputTokensObserved,
|
|
1496
|
+
});
|
|
1497
|
+
stream = await withCodex(runStreamOnce);
|
|
1498
|
+
batcher = createRuntimeBatcher(async (events) => {
|
|
1499
|
+
await publish!(events);
|
|
1500
|
+
});
|
|
1501
|
+
|
|
1502
|
+
const iterator = stream.toStream()[Symbol.asyncIterator]();
|
|
1503
|
+
let streamDone = false;
|
|
1504
|
+
try {
|
|
1505
|
+
while (true) {
|
|
1506
|
+
const next = await nextStreamEvent(iterator, activityContext);
|
|
1507
|
+
if (next.done) {
|
|
1508
|
+
streamDone = true;
|
|
1509
|
+
break;
|
|
1510
|
+
}
|
|
1511
|
+
const responseUsage = modelResponseUsageFromSdkEvent(next.value);
|
|
1512
|
+
if (responseUsage) {
|
|
1513
|
+
responseUsageCount += 1;
|
|
1514
|
+
const observed = responseUsage.usage?.inputTokens;
|
|
1515
|
+
if (typeof observed === "number" && observed > 0) {
|
|
1516
|
+
lastInputTokensObserved = observed;
|
|
1517
|
+
await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, observed)
|
|
1518
|
+
.catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
|
|
1519
|
+
}
|
|
1520
|
+
await recordModelUsageAndDebitCredits(settings, db, {
|
|
1521
|
+
accountId: input.accountId,
|
|
1522
|
+
workspaceId: input.workspaceId,
|
|
1523
|
+
sessionId: input.sessionId,
|
|
1524
|
+
turnId: activeTurnId,
|
|
1525
|
+
model: turn.model,
|
|
1526
|
+
isCodexTurn,
|
|
1527
|
+
usage: responseUsage.usage,
|
|
1528
|
+
sourceKey: modelUsageSourceKey({
|
|
1529
|
+
responseId: responseUsage.responseId,
|
|
1530
|
+
dispatchId,
|
|
1531
|
+
positionalKey: `response-${responseUsageCount}`,
|
|
1532
|
+
}),
|
|
1533
|
+
});
|
|
1534
|
+
await reconcileConversationTruth();
|
|
1381
1535
|
try {
|
|
1382
|
-
|
|
1383
|
-
} catch {
|
|
1384
|
-
|
|
1536
|
+
await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
|
|
1537
|
+
} catch (limitError) {
|
|
1538
|
+
// Capture the run state at the boundary so the budget valve in
|
|
1539
|
+
// the outer catch can end this segment gracefully with full
|
|
1540
|
+
// conversation context preserved for the post-top-up resume.
|
|
1541
|
+
let serializedRunState: string | null = null;
|
|
1542
|
+
try {
|
|
1543
|
+
serializedRunState = stream.state.toString();
|
|
1544
|
+
} catch {
|
|
1545
|
+
serializedRunState = null;
|
|
1546
|
+
}
|
|
1547
|
+
throw new BudgetExhaustedError(
|
|
1548
|
+
limitError instanceof Error ? limitError.message : String(limitError),
|
|
1549
|
+
serializedRunState,
|
|
1550
|
+
);
|
|
1385
1551
|
}
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1552
|
+
}
|
|
1553
|
+
// Recording gate: a computer-use tool actually ran when an SDK
|
|
1554
|
+
// `tool_call_item` whose rawItem.type is "computer_call" streams through
|
|
1555
|
+
// (screenshot/click/type/scroll/etc. — the first computer action proves
|
|
1556
|
+
// the desktop was driven). Match the raw SDK event (ground truth) BEFORE
|
|
1557
|
+
// normalization. Only meaningful when a recording is live.
|
|
1558
|
+
if (activeRecording && !didComputerUse && isComputerCallStreamEvent(next.value)) {
|
|
1559
|
+
didComputerUse = true;
|
|
1560
|
+
}
|
|
1561
|
+
const normalized = normalizeSdkEvent(next.value);
|
|
1562
|
+
for (const event of normalized) {
|
|
1563
|
+
await batcher.push(event);
|
|
1390
1564
|
}
|
|
1391
1565
|
}
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
// the desktop was driven). Match the raw SDK event (ground truth) BEFORE
|
|
1396
|
-
// normalization. Only meaningful when a recording is live.
|
|
1397
|
-
if (activeRecording && !didComputerUse && isComputerCallStreamEvent(next.value)) {
|
|
1398
|
-
didComputerUse = true;
|
|
1566
|
+
} finally {
|
|
1567
|
+
if (!streamDone) {
|
|
1568
|
+
await iterator.return?.();
|
|
1399
1569
|
}
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1570
|
+
}
|
|
1571
|
+
await batcher.flush();
|
|
1572
|
+
await stream.completed.catch(() => undefined);
|
|
1573
|
+
if (responseUsageCount === 0) {
|
|
1574
|
+
const aggregateInput = (stream.state.usage as { inputTokens?: unknown } | undefined)?.inputTokens;
|
|
1575
|
+
if (typeof aggregateInput === "number" && aggregateInput > 0) {
|
|
1576
|
+
lastInputTokensObserved = aggregateInput;
|
|
1403
1577
|
}
|
|
1578
|
+
await recordModelUsageAndDebitCredits(settings, db, {
|
|
1579
|
+
accountId: input.accountId,
|
|
1580
|
+
workspaceId: input.workspaceId,
|
|
1581
|
+
sessionId: input.sessionId,
|
|
1582
|
+
turnId: activeTurnId,
|
|
1583
|
+
model: turn.model,
|
|
1584
|
+
isCodexTurn,
|
|
1585
|
+
usage: stream.state.usage,
|
|
1586
|
+
sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
|
|
1587
|
+
});
|
|
1404
1588
|
}
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1589
|
+
if (lastInputTokensObserved !== null) {
|
|
1590
|
+
await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, lastInputTokensObserved)
|
|
1591
|
+
.catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
|
|
1408
1592
|
}
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1593
|
+
|
|
1594
|
+
if (stream.interruptions.length > 0) {
|
|
1595
|
+
await reconcileConversationTruth();
|
|
1596
|
+
const approvals = runtime.serializeApprovals(stream.interruptions);
|
|
1597
|
+
await saveRunState(db, {
|
|
1598
|
+
accountId: input.accountId,
|
|
1599
|
+
workspaceId: input.workspaceId,
|
|
1600
|
+
sessionId: input.sessionId,
|
|
1601
|
+
turnId: activeTurnId,
|
|
1602
|
+
serializedRunState: stream.state.toString(),
|
|
1603
|
+
pendingApprovals: approvals,
|
|
1604
|
+
// Record the account freezing this state so a resume on a DIFFERENT
|
|
1605
|
+
// codex account strips its account-bound reasoning before replay (HOLE C).
|
|
1606
|
+
frozenCodexCredentialId: effectiveCodexCredentialId,
|
|
1607
|
+
});
|
|
1608
|
+
await publish!([
|
|
1609
|
+
{ type: "session.requiresAction", payload: { approvals } },
|
|
1610
|
+
{ type: "session.status.changed", payload: { status: "requires_action" } },
|
|
1611
|
+
], true);
|
|
1612
|
+
await finishTurn(db, input.workspaceId, activeTurnId, "requires_action");
|
|
1613
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "requires_action", activeTurnId);
|
|
1614
|
+
activityStatus = "requires_action";
|
|
1615
|
+
return { status: "requires_action" };
|
|
1416
1616
|
}
|
|
1417
|
-
await recordModelUsageAndDebitCredits(settings, db, {
|
|
1418
|
-
accountId: input.accountId,
|
|
1419
|
-
workspaceId: input.workspaceId,
|
|
1420
|
-
sessionId: input.sessionId,
|
|
1421
|
-
turnId,
|
|
1422
|
-
model: turn.model,
|
|
1423
|
-
isCodexTurn,
|
|
1424
|
-
usage: stream.state.usage,
|
|
1425
|
-
sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
|
|
1426
|
-
});
|
|
1427
|
-
}
|
|
1428
|
-
if (lastInputTokensObserved !== null) {
|
|
1429
|
-
await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, lastInputTokensObserved)
|
|
1430
|
-
.catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
|
|
1431
|
-
}
|
|
1432
1617
|
|
|
1433
|
-
|
|
1618
|
+
const finalOutput = String(stream.finalOutput ?? "");
|
|
1434
1619
|
await reconcileConversationTruth();
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
{ type: "
|
|
1620
|
+
if (settings.sessionHistorySource !== "items") {
|
|
1621
|
+
// Legacy conversation memory; in items mode the blob is only written
|
|
1622
|
+
// for requires_action pauses (the one RunState-only resume path).
|
|
1623
|
+
await saveRunState(db, {
|
|
1624
|
+
accountId: input.accountId,
|
|
1625
|
+
workspaceId: input.workspaceId,
|
|
1626
|
+
sessionId: input.sessionId,
|
|
1627
|
+
turnId: activeTurnId,
|
|
1628
|
+
serializedRunState: stream.state.toString(),
|
|
1629
|
+
pendingApprovals: [],
|
|
1630
|
+
frozenCodexCredentialId: effectiveCodexCredentialId,
|
|
1631
|
+
});
|
|
1632
|
+
}
|
|
1633
|
+
await publish!([
|
|
1634
|
+
{ type: "agent.message.completed", payload: { text: finalOutput } },
|
|
1635
|
+
{ type: "turn.completed", payload: { output: finalOutput } },
|
|
1636
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1450
1637
|
], true);
|
|
1451
|
-
await finishTurn(db, input.workspaceId,
|
|
1452
|
-
await setSessionStatus(db, input.workspaceId, input.sessionId, "
|
|
1453
|
-
|
|
1454
|
-
return { status: "requires_action" };
|
|
1455
|
-
}
|
|
1456
|
-
|
|
1457
|
-
const finalOutput = String(stream.finalOutput ?? "");
|
|
1458
|
-
await reconcileConversationTruth();
|
|
1459
|
-
if (settings.sessionHistorySource !== "items") {
|
|
1460
|
-
// Legacy conversation memory; in items mode the blob is only written
|
|
1461
|
-
// for requires_action pauses (the one RunState-only resume path).
|
|
1462
|
-
await saveRunState(db, {
|
|
1638
|
+
await finishTurn(db, input.workspaceId, activeTurnId, "idle");
|
|
1639
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1640
|
+
await recordUsageEvent(db, {
|
|
1463
1641
|
accountId: input.accountId,
|
|
1464
1642
|
workspaceId: input.workspaceId,
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1643
|
+
eventType: "agent_run.completed",
|
|
1644
|
+
quantity: 1,
|
|
1645
|
+
unit: "run",
|
|
1646
|
+
sourceResourceType: "session_turn",
|
|
1647
|
+
sourceResourceId: activeTurnId,
|
|
1648
|
+
idempotencyKey: `usage:agent_run.completed:${activeTurnId}`,
|
|
1470
1649
|
});
|
|
1650
|
+
activityStatus = "idle";
|
|
1651
|
+
return { status: "idle" };
|
|
1652
|
+
};
|
|
1653
|
+
|
|
1654
|
+
await prepareRunAttemptInput();
|
|
1655
|
+
let compactionRecoveryAttempted = false;
|
|
1656
|
+
let retriedAfterCompaction = false;
|
|
1657
|
+
while (true) {
|
|
1658
|
+
try {
|
|
1659
|
+
const result = await runStreamAttempt();
|
|
1660
|
+
if (retriedAfterCompaction) {
|
|
1661
|
+
observability.info("context compaction recovery succeeded after in-activity retry", {
|
|
1662
|
+
sessionId: input.sessionId,
|
|
1663
|
+
turnId: activeTurnId,
|
|
1664
|
+
});
|
|
1665
|
+
}
|
|
1666
|
+
return result;
|
|
1667
|
+
} catch (attemptError) {
|
|
1668
|
+
const overflow = classifyContextWindowOverflowError(attemptError);
|
|
1669
|
+
const compactionNeeded = findCompactionNeededError(attemptError);
|
|
1670
|
+
const recoveryKind = compactionNeeded ? "proactive" : overflow ? "overflow" : null;
|
|
1671
|
+
if (!recoveryKind || !publish || !turnStartedPublished || compactionRecoveryAttempted) {
|
|
1672
|
+
throw attemptError;
|
|
1673
|
+
}
|
|
1674
|
+
compactionRecoveryAttempted = true;
|
|
1675
|
+
await flushRuntimeBatcher();
|
|
1676
|
+
await reconcileConversationTruth({ skipInputOnlyRows: true });
|
|
1677
|
+
const progressPersisted = modelOrToolProgressPersisted;
|
|
1678
|
+
observability.warn("context compaction recovery attempted", {
|
|
1679
|
+
sessionId: input.sessionId,
|
|
1680
|
+
turnId: activeTurnId,
|
|
1681
|
+
reason: recoveryKind,
|
|
1682
|
+
progressPersisted,
|
|
1683
|
+
code: overflow?.code,
|
|
1684
|
+
error: overflow?.message ?? compactionNeeded?.message,
|
|
1685
|
+
signalTokens: compactionNeeded?.signalTokens,
|
|
1686
|
+
thresholdTokens: compactionNeeded?.thresholdTokens,
|
|
1687
|
+
});
|
|
1688
|
+
let compacted = false;
|
|
1689
|
+
try {
|
|
1690
|
+
const outcome = await forceContextCompaction(recoveryKind);
|
|
1691
|
+
compacted = outcome.compacted;
|
|
1692
|
+
} catch (compactError) {
|
|
1693
|
+
observability.warn("context compaction recovery compaction failed", {
|
|
1694
|
+
sessionId: input.sessionId,
|
|
1695
|
+
turnId: activeTurnId,
|
|
1696
|
+
error: compactError instanceof Error ? compactError.message : String(compactError),
|
|
1697
|
+
});
|
|
1698
|
+
}
|
|
1699
|
+
if (progressPersisted) {
|
|
1700
|
+
// The user's intent was live mid-turn — after a successful
|
|
1701
|
+
// compaction the turn AUTO-CONTINUES via the same synthesized
|
|
1702
|
+
// resume-notice requeue the worker-shutdown checkpoint uses (never
|
|
1703
|
+
// the original trigger, so side effects are not replayed). Idle is
|
|
1704
|
+
// the fallback, not the default; it remains for: legacy run-state
|
|
1705
|
+
// history (resuming the frozen pre-compaction RunState would
|
|
1706
|
+
// overflow again), a compaction that could not shrink anything,
|
|
1707
|
+
// and a turn that ALREADY resumed from an overflow (loop guard —
|
|
1708
|
+
// a second overflow in the same lineage stops churning).
|
|
1709
|
+
const triggerPayload =
|
|
1710
|
+
trigger.payload && typeof trigger.payload === "object" && !Array.isArray(trigger.payload)
|
|
1711
|
+
? trigger.payload as Record<string, unknown>
|
|
1712
|
+
: {};
|
|
1713
|
+
const compactionResumedTurn = triggerType === "turn.preempted"
|
|
1714
|
+
&& (triggerPayload.reason === "context_compacted" || triggerPayload.reason === "context_overflow_compacted");
|
|
1715
|
+
const canAutoContinue =
|
|
1716
|
+
compacted && !compactionResumedTurn && settings.sessionHistorySource === "items" && activeTurnId != null;
|
|
1717
|
+
if (canAutoContinue) {
|
|
1718
|
+
const [preemptedEvent] = await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [
|
|
1719
|
+
{
|
|
1720
|
+
turnId: activeTurnId,
|
|
1721
|
+
type: "turn.preempted",
|
|
1722
|
+
payload: {
|
|
1723
|
+
triggerEventId: input.triggerEventId,
|
|
1724
|
+
reason: "context_compacted",
|
|
1725
|
+
resumeWithNotice: true,
|
|
1726
|
+
text: CONTEXT_OVERFLOW_RESUME_TEXT,
|
|
1727
|
+
},
|
|
1728
|
+
},
|
|
1729
|
+
{ turnId: activeTurnId, type: "session.status.changed", payload: { status: "queued" } },
|
|
1730
|
+
]);
|
|
1731
|
+
await requeuePreemptedTurn(db, input.workspaceId, activeTurnId, preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
1732
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => undefined);
|
|
1733
|
+
activityStatus = "preempted";
|
|
1734
|
+
observability.info("context compaction recovery succeeded by compacting and auto-continuing", {
|
|
1735
|
+
sessionId: input.sessionId,
|
|
1736
|
+
turnId: activeTurnId,
|
|
1737
|
+
reason: recoveryKind,
|
|
1738
|
+
compacted,
|
|
1739
|
+
});
|
|
1740
|
+
return { status: "preempted" };
|
|
1741
|
+
}
|
|
1742
|
+
await publish!([
|
|
1743
|
+
{
|
|
1744
|
+
type: "turn.failed",
|
|
1745
|
+
payload: {
|
|
1746
|
+
error: CONTEXT_WINDOW_OVERFLOW_RECOVERY_MESSAGE,
|
|
1747
|
+
code: recoveryKind === "overflow" ? "context_window_overflow_compacted" : "context_compacted",
|
|
1748
|
+
retryable: false,
|
|
1749
|
+
recovery: "user_message",
|
|
1750
|
+
compacted,
|
|
1751
|
+
},
|
|
1752
|
+
},
|
|
1753
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1754
|
+
], true);
|
|
1755
|
+
await finishTurn(db, input.workspaceId, activeTurnId, "failed");
|
|
1756
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1757
|
+
activityStatus = "idle";
|
|
1758
|
+
activityError = attemptError;
|
|
1759
|
+
observability.info("context compaction recovery succeeded by compacting and idling", {
|
|
1760
|
+
sessionId: input.sessionId,
|
|
1761
|
+
turnId: activeTurnId,
|
|
1762
|
+
reason: recoveryKind,
|
|
1763
|
+
compacted,
|
|
1764
|
+
});
|
|
1765
|
+
return { status: "idle" };
|
|
1766
|
+
}
|
|
1767
|
+
retriedAfterCompaction = true;
|
|
1768
|
+
observability.info("context compaction recovery retrying turn after compaction", {
|
|
1769
|
+
sessionId: input.sessionId,
|
|
1770
|
+
turnId: activeTurnId,
|
|
1771
|
+
reason: recoveryKind,
|
|
1772
|
+
compacted,
|
|
1773
|
+
});
|
|
1774
|
+
await prepareRunAttemptInput();
|
|
1775
|
+
}
|
|
1471
1776
|
}
|
|
1472
|
-
await publish([
|
|
1473
|
-
{ type: "agent.message.completed", payload: { text: finalOutput } },
|
|
1474
|
-
{ type: "turn.completed", payload: { output: finalOutput } },
|
|
1475
|
-
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1476
|
-
], true);
|
|
1477
|
-
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
1478
|
-
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1479
|
-
await recordUsageEvent(db, {
|
|
1480
|
-
accountId: input.accountId,
|
|
1481
|
-
workspaceId: input.workspaceId,
|
|
1482
|
-
eventType: "agent_run.completed",
|
|
1483
|
-
quantity: 1,
|
|
1484
|
-
unit: "run",
|
|
1485
|
-
sourceResourceType: "session_turn",
|
|
1486
|
-
sourceResourceId: turnId,
|
|
1487
|
-
idempotencyKey: `usage:agent_run.completed:${turnId}`,
|
|
1488
|
-
});
|
|
1489
|
-
activityStatus = "idle";
|
|
1490
|
-
return { status: "idle" };
|
|
1491
1777
|
} catch (error) {
|
|
1492
1778
|
// Graceful worker shutdown (deploy / rollout restart): checkpoint and
|
|
1493
1779
|
// hand the turn back instead of failing the session. Conversation truth
|
|
@@ -1525,7 +1811,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1525
1811
|
}
|
|
1526
1812
|
if (isWorkerShutdownCancellation(error) && preemptTurnId) {
|
|
1527
1813
|
try {
|
|
1528
|
-
await
|
|
1814
|
+
await flushRuntimeBatcher();
|
|
1529
1815
|
await reconcileConversationTruth();
|
|
1530
1816
|
// An approval-decision rerun always replays its original trigger:
|
|
1531
1817
|
// the decision is applied through the approval resume path reading
|
|
@@ -1592,7 +1878,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1592
1878
|
if (error instanceof CancelledFailure) {
|
|
1593
1879
|
activityStatus = "cancelled";
|
|
1594
1880
|
activityError = error;
|
|
1595
|
-
await
|
|
1881
|
+
await flushRuntimeBatcher();
|
|
1596
1882
|
if (preemptTurnId) {
|
|
1597
1883
|
await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => undefined);
|
|
1598
1884
|
}
|
|
@@ -1605,7 +1891,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1605
1891
|
// that resumption.
|
|
1606
1892
|
const maxTurns = maxTurnsExceededRunState(error);
|
|
1607
1893
|
if (maxTurns && publish && turnId && turnStartedPublished) {
|
|
1608
|
-
await
|
|
1894
|
+
await flushRuntimeBatcher();
|
|
1609
1895
|
// The SDK attaches the run state at the throw site; persisting it lets
|
|
1610
1896
|
// the continuation resume with this segment's full context. If capture
|
|
1611
1897
|
// ever fails, the continuation falls back to the previous snapshot --
|
|
@@ -1656,7 +1942,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1656
1942
|
if (usageLimit && publish && turnId && turnStartedPublished) {
|
|
1657
1943
|
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
1658
1944
|
const goalActive = Boolean(goal && goal.status === "active");
|
|
1659
|
-
await
|
|
1945
|
+
await flushRuntimeBatcher();
|
|
1660
1946
|
await reconcileConversationTruth();
|
|
1661
1947
|
const serializedRunState = agentsErrorRunState(error);
|
|
1662
1948
|
const runStateSaved = Boolean(serializedRunState) && settings.sessionHistorySource !== "items";
|
|
@@ -1787,7 +2073,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1787
2073
|
// top-up). An active goal pauses visibly with reason "limits" at the
|
|
1788
2074
|
// next continuation evaluation, without consuming continuation budget.
|
|
1789
2075
|
if (error instanceof BudgetExhaustedError && publish && turnId && turnStartedPublished) {
|
|
1790
|
-
await
|
|
2076
|
+
await flushRuntimeBatcher();
|
|
1791
2077
|
await reconcileConversationTruth();
|
|
1792
2078
|
const runStateSaved = Boolean(error.serializedRunState) && settings.sessionHistorySource !== "items";
|
|
1793
2079
|
if (error.serializedRunState && settings.sessionHistorySource !== "items") {
|
|
@@ -1833,7 +2119,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1833
2119
|
if (failure.retryable && publish && turnId && turnStartedPublished) {
|
|
1834
2120
|
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
1835
2121
|
const goalActive = Boolean(goal && goal.status === "active");
|
|
1836
|
-
await
|
|
2122
|
+
await flushRuntimeBatcher();
|
|
1837
2123
|
// Provider errors rarely carry SDK run state; a null falls back to
|
|
1838
2124
|
// the previous snapshot, same degraded-context contract as the
|
|
1839
2125
|
// max-turns path above.
|