@opengeni/worker-bundle 0.23.4 → 0.26.0-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +17 -2
- package/dist/activities/agent-turn/agent-build.d.ts +3 -10
- package/dist/activities/agent-turn/claim.d.ts +8 -0
- package/dist/activities/agent-turn/codex.d.ts +2 -2
- package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
- package/dist/activities/agent-turn/errors.d.ts +1 -1
- package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
- package/dist/activities/agent-turn/list-models.d.ts +16 -0
- package/dist/activities/agent-turn/model-usage.d.ts +15 -1
- package/dist/activities/agent-turn/quiescence.d.ts +11 -6
- package/dist/activities/agent-turn/run.d.ts +9 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +15 -0
- package/dist/activities/agent-turn/session-title.d.ts +21 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +4 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +4 -13
- package/dist/activities/agent-turn/tool-policy.d.ts +40 -5
- package/dist/activities/agent-turn/turn-context.d.ts +13 -2
- package/dist/activities/agent-turn.d.ts +1 -1
- package/dist/activities/automations.d.ts +1 -0
- package/dist/activities/capabilities.d.ts +3 -1
- package/dist/activities/codemode-dispatcher.d.ts +10 -1
- package/dist/activities/codex-capacity.d.ts +32 -3
- package/dist/activities/codex-fleet-shadow.d.ts +1 -1
- package/dist/activities/codex-image-generation.d.ts +1 -1
- package/dist/activities/codex-rotation.d.ts +8 -67
- package/dist/activities/context-compaction.d.ts +2 -0
- package/dist/activities/goals.d.ts +12 -1
- package/dist/activities/google-drive-publication.d.ts +2 -13
- package/dist/activities/image-generation-operation.d.ts +13 -4
- package/dist/activities/image-generation-references.d.ts +2 -1
- package/dist/activities/scheduled-tasks.d.ts +1 -0
- package/dist/activities/session-state.d.ts +4 -2
- package/dist/activities/types.d.ts +19 -1
- package/dist/{activities-control-52W5IDWM.js → activities-control-CTEMLTHG.js} +339 -195
- package/dist/activities-control-CTEMLTHG.js.map +1 -0
- package/dist/activities-control.d.ts +4 -0
- package/dist/{activities-turn-A2G2KLSY.js → activities-turn-S5IU347A.js} +1846 -1136
- package/dist/activities-turn-S5IU347A.js.map +1 -0
- package/dist/activities.d.ts +4 -0
- package/dist/{chunk-GFNXVZVQ.js → chunk-D4ZEFOYH.js} +107 -7
- package/dist/chunk-D4ZEFOYH.js.map +1 -0
- package/dist/{chunk-GRFQTTSQ.js → chunk-GDCLDKSH.js} +7 -3
- package/dist/{chunk-GRFQTTSQ.js.map → chunk-GDCLDKSH.js.map} +1 -1
- package/dist/{chunk-IPVMCAW5.js → chunk-TAUE3GYT.js} +225 -154
- package/dist/chunk-TAUE3GYT.js.map +1 -0
- package/dist/github-rest-mcp.d.ts +0 -2
- package/dist/index.d.ts +2 -0
- package/dist/index.js +43 -7
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +35 -8
- package/dist/sandbox-reaper-timeout.d.ts +2 -2
- package/dist/sandbox-resume.d.ts +6 -1
- package/dist/workflow-bundle.js +96 -10
- package/dist/workflows/activities.d.ts +6 -1
- package/dist/workflows/automations.d.ts +5 -0
- package/dist/workflows/session.d.ts +3 -0
- package/package.json +17 -17
- package/src/activities/agent-run-admission.ts +7 -1
- package/src/activities/agent-turn/admission.ts +47 -6
- package/src/activities/agent-turn/agent-build.ts +20 -26
- package/src/activities/agent-turn/claim.ts +59 -6
- package/src/activities/agent-turn/codex-capacity.ts +311 -234
- package/src/activities/agent-turn/codex.ts +2 -2
- package/src/activities/agent-turn/compaction-prep.ts +8 -1
- package/src/activities/agent-turn/credential-leases.ts +36 -0
- package/src/activities/agent-turn/errors.ts +29 -1
- package/src/activities/agent-turn/failure-settlement.ts +451 -323
- package/src/activities/agent-turn/finalization.ts +31 -17
- package/src/activities/agent-turn/list-models.ts +101 -0
- package/src/activities/agent-turn/model-usage.ts +120 -45
- package/src/activities/agent-turn/quiescence.ts +32 -17
- package/src/activities/agent-turn/run.ts +98 -9
- package/src/activities/agent-turn/sandbox-runtime.ts +78 -81
- package/src/activities/agent-turn/session-title.ts +55 -2
- package/src/activities/agent-turn/stream-attempt.ts +281 -136
- package/src/activities/agent-turn/tool-environment.ts +132 -39
- package/src/activities/agent-turn/tool-policy.ts +71 -12
- package/src/activities/agent-turn/turn-context.ts +21 -2
- package/src/activities/agent-turn.ts +5 -1
- package/src/activities/automations.ts +122 -67
- package/src/activities/capabilities.ts +129 -2
- package/src/activities/codemode-dispatcher.ts +235 -59
- package/src/activities/codex-capacity.ts +89 -10
- package/src/activities/codex-fleet-shadow.ts +8 -1
- package/src/activities/codex-image-generation.ts +19 -2
- package/src/activities/codex-rotation.ts +45 -142
- package/src/activities/context-compaction.ts +3 -0
- package/src/activities/goals.ts +116 -39
- package/src/activities/google-drive-publication.ts +4 -81
- package/src/activities/image-generation-operation.ts +28 -3
- package/src/activities/image-generation-references.ts +9 -9
- package/src/activities/mcp-credentials.ts +53 -0
- package/src/activities/rig-verification.ts +5 -3
- package/src/activities/sandbox-lease.ts +29 -12
- package/src/activities/scheduled-tasks.ts +217 -110
- package/src/activities/session-state.ts +17 -0
- package/src/activities/site-auth-maintenance.ts +12 -12
- package/src/activities/types.ts +22 -8
- package/src/activities.ts +2 -0
- package/src/activity-services.ts +4 -1
- package/src/github-rest-mcp.ts +2 -83
- package/src/index.ts +34 -4
- package/src/observability-metrics.ts +158 -14
- package/src/sandbox-reaper-timeout.ts +10 -4
- package/src/sandbox-resume.ts +58 -8
- package/src/sandbox-routing.ts +71 -16
- package/src/workflows/activities.ts +16 -0
- package/src/workflows/automations.ts +21 -2
- package/src/workflows/session.ts +85 -7
- package/dist/activities/api-integrations.d.ts +0 -21
- package/dist/activities-control-52W5IDWM.js.map +0 -1
- package/dist/activities-turn-A2G2KLSY.js.map +0 -1
- package/dist/chunk-GFNXVZVQ.js.map +0 -1
- package/dist/chunk-IPVMCAW5.js.map +0 -1
- package/src/activities/api-integrations.ts +0 -141
|
@@ -10,7 +10,10 @@ import {
|
|
|
10
10
|
clearDurablePendingSessionToolCalls,
|
|
11
11
|
isSessionCompactionRequested,
|
|
12
12
|
nextSessionHistoryPosition,
|
|
13
|
+
persistModelContextSnapshot,
|
|
14
|
+
updateSessionTitleWithEvent,
|
|
13
15
|
} from "@opengeni/db";
|
|
16
|
+
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
14
17
|
import {
|
|
15
18
|
normalizeModelCallUsage,
|
|
16
19
|
normalizeSdkEvent,
|
|
@@ -29,7 +32,7 @@ import {
|
|
|
29
32
|
type RunCredentialCommandSession,
|
|
30
33
|
type CodemodeTokenWriterSession,
|
|
31
34
|
} from "@opengeni/runtime";
|
|
32
|
-
import { type Settings } from "@opengeni/config";
|
|
35
|
+
import { serviceTierForLatencyMode, type Settings } from "@opengeni/config";
|
|
33
36
|
import { maybeCompactContext, settleFailedContextCompactionLandmark } from "../context-compaction";
|
|
34
37
|
import { TurnAttemptFencedError } from "../turn-attempt-fenced";
|
|
35
38
|
import { type MintedRunGitCredentials } from "../environment";
|
|
@@ -109,13 +112,16 @@ import {
|
|
|
109
112
|
recordCompletedModelCallBeforeOwnershipFences,
|
|
110
113
|
TurnEventPublisher,
|
|
111
114
|
createModelResponseEventState,
|
|
115
|
+
createSessionTitleModelUsageEventState,
|
|
112
116
|
modelResponseContextSignal,
|
|
113
117
|
assertModelResponseLatencyMode,
|
|
114
118
|
processModelResponseTerminalEvent,
|
|
119
|
+
processSessionTitleModelUsageEvent,
|
|
115
120
|
emitModelCallUsage,
|
|
116
121
|
recordModelUsageAndDebitCredits,
|
|
117
122
|
recordAuthoritativeModelCallFact,
|
|
118
123
|
} from "./model-usage";
|
|
124
|
+
import { startParallelSessionTitleGeneration } from "./session-title";
|
|
119
125
|
import {
|
|
120
126
|
assertAgentStreamNotCancelled,
|
|
121
127
|
assertSuccessfulAgentStreamCompletion,
|
|
@@ -142,6 +148,7 @@ export type TurnStreamAttemptDeps = {
|
|
|
142
148
|
input: RunAgentTurnInput;
|
|
143
149
|
settings: Settings;
|
|
144
150
|
db: ActivityServices["db"];
|
|
151
|
+
bus: ActivityServices["bus"];
|
|
145
152
|
runtime: ActivityServices["runtime"];
|
|
146
153
|
objectStorage: ActivityServices["objectStorage"];
|
|
147
154
|
observability: ActivityServices["observability"];
|
|
@@ -196,6 +203,7 @@ export type TurnStreamAttemptDeps = {
|
|
|
196
203
|
initialSandbox?: ResumedTurnSandbox,
|
|
197
204
|
) => Promise<void>;
|
|
198
205
|
withProviderRequestContext: <T>(fn: () => Promise<T>) => Promise<T>;
|
|
206
|
+
withSessionTitleProviderRequestContext: <T>(fn: () => Promise<T>) => Promise<T>;
|
|
199
207
|
publishCompactionLiveEvents: (events: SessionEvent[]) => Promise<void>;
|
|
200
208
|
publishCompactionOutcomeEvents: (events: SessionEvent[]) => Promise<void>;
|
|
201
209
|
recordCompanyBrainContributionReceiptOnce: () => void;
|
|
@@ -221,6 +229,8 @@ export type TurnStreamAttemptDeps = {
|
|
|
221
229
|
providerApi: HistoryProviderApi;
|
|
222
230
|
runSettings: Settings;
|
|
223
231
|
turnTools: ReturnType<typeof withFirstPartyTools>;
|
|
232
|
+
generateSessionTitleInParallel: boolean;
|
|
233
|
+
sessionTitlePrompt: string;
|
|
224
234
|
compactSummarizer: CompactionSummarizer;
|
|
225
235
|
settleDeferredSteerAfterCompaction: () => Promise<RunAgentTurnResult | null>;
|
|
226
236
|
compactionModelHistoryProjector: (
|
|
@@ -258,6 +268,7 @@ export async function runTurnStreamAttempt(
|
|
|
258
268
|
input,
|
|
259
269
|
settings,
|
|
260
270
|
db,
|
|
271
|
+
bus,
|
|
261
272
|
runtime,
|
|
262
273
|
objectStorage,
|
|
263
274
|
observability,
|
|
@@ -288,6 +299,7 @@ export async function runTurnStreamAttempt(
|
|
|
288
299
|
attachCodemodeTokenRenewal,
|
|
289
300
|
attachRunCredentialRenewal,
|
|
290
301
|
withProviderRequestContext,
|
|
302
|
+
withSessionTitleProviderRequestContext,
|
|
291
303
|
publishCompactionLiveEvents,
|
|
292
304
|
publishCompactionOutcomeEvents,
|
|
293
305
|
recordCompanyBrainContributionReceiptOnce,
|
|
@@ -313,6 +325,8 @@ export async function runTurnStreamAttempt(
|
|
|
313
325
|
providerApi,
|
|
314
326
|
runSettings,
|
|
315
327
|
turnTools,
|
|
328
|
+
generateSessionTitleInParallel,
|
|
329
|
+
sessionTitlePrompt,
|
|
316
330
|
compactSummarizer,
|
|
317
331
|
settleDeferredSteerAfterCompaction,
|
|
318
332
|
compactionModelHistoryProjector,
|
|
@@ -336,6 +350,71 @@ export async function runTurnStreamAttempt(
|
|
|
336
350
|
if (!activeTurnId) {
|
|
337
351
|
throw new Error("Turn id was not initialized");
|
|
338
352
|
}
|
|
353
|
+
const sessionTitleUsageState = createSessionTitleModelUsageEventState(
|
|
354
|
+
claimedModelUsageSourceKeys,
|
|
355
|
+
);
|
|
356
|
+
let parallelSessionTitle: ReturnType<typeof startParallelSessionTitleGeneration> | null = null;
|
|
357
|
+
let parallelSessionTitleFinished = false;
|
|
358
|
+
const finishParallelSessionTitle = async (): Promise<void> => {
|
|
359
|
+
if (parallelSessionTitleFinished) return;
|
|
360
|
+
parallelSessionTitleFinished = true;
|
|
361
|
+
const generated = await parallelSessionTitle?.finish();
|
|
362
|
+
if (!generated) return;
|
|
363
|
+
|
|
364
|
+
if (generated.usage) {
|
|
365
|
+
await processSessionTitleModelUsageEvent({
|
|
366
|
+
usage: generated.usage,
|
|
367
|
+
state: sessionTitleUsageState,
|
|
368
|
+
dispatchId: modelUsageDispatchId,
|
|
369
|
+
settings,
|
|
370
|
+
db,
|
|
371
|
+
observability,
|
|
372
|
+
publish: eventing.publish,
|
|
373
|
+
accountId: input.accountId,
|
|
374
|
+
workspaceId: input.workspaceId,
|
|
375
|
+
sessionId: input.sessionId,
|
|
376
|
+
turnId: activeTurnId,
|
|
377
|
+
turnAttemptId: input.attemptId,
|
|
378
|
+
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
379
|
+
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
380
|
+
model: resolvedModel?.configured.id ?? turn.model,
|
|
381
|
+
externallyBilled: billingState.isExternallyBilledTurn,
|
|
382
|
+
chargesOpenGeniCredits: billingState.chargesOpenGeniCredits,
|
|
383
|
+
countsTowardTokenCap: billingState.countsTowardTokenCap,
|
|
384
|
+
servingCredentialId: providerTurn.effectiveCodexCredentialId,
|
|
385
|
+
priorSessionCredentialId: providerTurn.priorSessionCodexCredentialId,
|
|
386
|
+
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
387
|
+
renewLease: () => leases.renewServing("model_usage"),
|
|
388
|
+
leaseLost: leases.servingLost,
|
|
389
|
+
leaseLostMessage: "Provider credential lease expired during session title generation",
|
|
390
|
+
});
|
|
391
|
+
}
|
|
392
|
+
if (!generated.title) return;
|
|
393
|
+
|
|
394
|
+
try {
|
|
395
|
+
const result = await updateSessionTitleWithEvent(db, {
|
|
396
|
+
workspaceId: input.workspaceId,
|
|
397
|
+
sessionId: input.sessionId,
|
|
398
|
+
title: generated.title,
|
|
399
|
+
source: "agent",
|
|
400
|
+
});
|
|
401
|
+
if (result.events.length > 0) {
|
|
402
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, result.events);
|
|
403
|
+
}
|
|
404
|
+
} catch (error) {
|
|
405
|
+
observability.warn("parallel session title persistence failed", {
|
|
406
|
+
...safeErrorDiagnostic(error),
|
|
407
|
+
errorClass: "ParallelSessionTitlePersistenceError",
|
|
408
|
+
errorCode: "parallel_session_title_persistence_failed",
|
|
409
|
+
origin: "worker",
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
};
|
|
413
|
+
const cancelParallelSessionTitle = async (): Promise<void> => {
|
|
414
|
+
if (parallelSessionTitleFinished) return;
|
|
415
|
+
parallelSessionTitleFinished = true;
|
|
416
|
+
await parallelSessionTitle?.cancel();
|
|
417
|
+
};
|
|
339
418
|
let runInput: Awaited<ReturnType<typeof turnInput>>["input"] | null = null;
|
|
340
419
|
const prepareRunAttemptInput = async () => {
|
|
341
420
|
const historyPreparationStartedAt = performance.now();
|
|
@@ -699,6 +778,17 @@ export async function runTurnStreamAttempt(
|
|
|
699
778
|
return await runtime.runStream(agent, runInput!, eventing.modelRunSettings, {
|
|
700
779
|
signal: runtimeCancellationSignal,
|
|
701
780
|
sandboxEnvironment,
|
|
781
|
+
onModelVisibleContext: async (snapshot) => {
|
|
782
|
+
await persistModelContextSnapshot(db, {
|
|
783
|
+
accountId: input.accountId,
|
|
784
|
+
workspaceId: input.workspaceId,
|
|
785
|
+
sessionId: input.sessionId,
|
|
786
|
+
turnId: activeTurnId,
|
|
787
|
+
attemptId: input.attemptId,
|
|
788
|
+
executionGeneration: attempt.executionGeneration,
|
|
789
|
+
snapshot,
|
|
790
|
+
});
|
|
791
|
+
},
|
|
702
792
|
onRuntimeEvent: async (event) => {
|
|
703
793
|
await leases.renewServing("runtime_event");
|
|
704
794
|
if (leases.servingLost()) {
|
|
@@ -812,7 +902,9 @@ export async function runTurnStreamAttempt(
|
|
|
812
902
|
});
|
|
813
903
|
}
|
|
814
904
|
};
|
|
815
|
-
if (
|
|
905
|
+
if (billingState.isCodexTurn) {
|
|
906
|
+
leases.codex.assertUsable();
|
|
907
|
+
} else if (leases.codex.lost) {
|
|
816
908
|
throw new Error("Codex credential lease expired before the model run");
|
|
817
909
|
}
|
|
818
910
|
if (leases.xai.lost) {
|
|
@@ -911,6 +1003,8 @@ export async function runTurnStreamAttempt(
|
|
|
911
1003
|
latencyMode: turnExecutionPolicy.latencyMode,
|
|
912
1004
|
metricProvider: streamProvider,
|
|
913
1005
|
externallyBilled: billingState.isExternallyBilledTurn,
|
|
1006
|
+
chargesOpenGeniCredits: billingState.chargesOpenGeniCredits,
|
|
1007
|
+
countsTowardTokenCap: billingState.countsTowardTokenCap,
|
|
914
1008
|
servingCredentialId: providerTurn.effectiveCodexCredentialId,
|
|
915
1009
|
priorSessionCredentialId: providerTurn.priorSessionCodexCredentialId,
|
|
916
1010
|
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
@@ -976,6 +1070,8 @@ export async function runTurnStreamAttempt(
|
|
|
976
1070
|
input.workspaceId,
|
|
977
1071
|
billingState.isExternallyBilledTurn,
|
|
978
1072
|
entitlements,
|
|
1073
|
+
billingState.chargesOpenGeniCredits,
|
|
1074
|
+
billingState.countsTowardTokenCap,
|
|
979
1075
|
);
|
|
980
1076
|
} catch (limitError) {
|
|
981
1077
|
// Capture the run state at the boundary so the budget valve in
|
|
@@ -1351,6 +1447,8 @@ export async function runTurnStreamAttempt(
|
|
|
1351
1447
|
turnAttemptId: input.attemptId,
|
|
1352
1448
|
model: turn.model,
|
|
1353
1449
|
externallyBilled: billingState.isExternallyBilledTurn,
|
|
1450
|
+
chargesOpenGeniCredits: billingState.chargesOpenGeniCredits,
|
|
1451
|
+
countsTowardTokenCap: billingState.countsTowardTokenCap,
|
|
1354
1452
|
usage: aggregateUsage,
|
|
1355
1453
|
normalizedUsage: normalizedAggregateUsage,
|
|
1356
1454
|
sourceKey: aggregateSourceKey,
|
|
@@ -1372,6 +1470,8 @@ export async function runTurnStreamAttempt(
|
|
|
1372
1470
|
sourceKey: aggregateSourceKey,
|
|
1373
1471
|
usage: { usage: aggregateUsage },
|
|
1374
1472
|
normalizedUsage: normalizedAggregateUsage,
|
|
1473
|
+
...(billing ? { billingPath: billing.billingPath } : {}),
|
|
1474
|
+
...(billing?.upstreamProvider ? { upstreamProvider: billing.upstreamProvider } : {}),
|
|
1375
1475
|
servingAccountHash: aggregateAccountCtx.servingAccountHash,
|
|
1376
1476
|
accountChangedFromPrevCall: aggregateAccountCtx.accountChangedFromPrevCall,
|
|
1377
1477
|
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
@@ -1515,6 +1615,7 @@ export async function runTurnStreamAttempt(
|
|
|
1515
1615
|
if (!attached.accepted) {
|
|
1516
1616
|
return claimedResult({ status: "cancelled" });
|
|
1517
1617
|
}
|
|
1618
|
+
await finishParallelSessionTitle();
|
|
1518
1619
|
if (
|
|
1519
1620
|
!(await eventing.settle!({
|
|
1520
1621
|
events: [
|
|
@@ -1560,6 +1661,7 @@ export async function runTurnStreamAttempt(
|
|
|
1560
1661
|
// frames). Best-effort: a miss leaves the runner's retention TTL to
|
|
1561
1662
|
// reap, never fails a completed turn.
|
|
1562
1663
|
await finalizeTurnOpStreamOps();
|
|
1664
|
+
await finishParallelSessionTitle();
|
|
1563
1665
|
if (
|
|
1564
1666
|
!(await eventing.settle!({
|
|
1565
1667
|
events: [
|
|
@@ -1627,151 +1729,194 @@ export async function runTurnStreamAttempt(
|
|
|
1627
1729
|
) {
|
|
1628
1730
|
return claimedResult({ status: "cancelled" });
|
|
1629
1731
|
}
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1732
|
+
if (
|
|
1733
|
+
generateSessionTitleInParallel &&
|
|
1734
|
+
runtime.generateSessionTitle &&
|
|
1735
|
+
!runtimeCancellationSignal.aborted
|
|
1736
|
+
) {
|
|
1737
|
+
const serviceTier = serviceTierForLatencyMode(
|
|
1738
|
+
turnExecutionPolicy.providerId,
|
|
1739
|
+
turnExecutionPolicy.latencyMode,
|
|
1740
|
+
);
|
|
1741
|
+
parallelSessionTitle = startParallelSessionTitleGeneration({
|
|
1742
|
+
signal: runtimeCancellationSignal,
|
|
1743
|
+
generate: async (signal) =>
|
|
1744
|
+
await withSessionTitleProviderRequestContext(() =>
|
|
1745
|
+
runtime.generateSessionTitle!(runSettings, sessionTitlePrompt, {
|
|
1746
|
+
...(resolvedModel
|
|
1747
|
+
? {
|
|
1748
|
+
client: resolvedModel.client,
|
|
1749
|
+
provider: resolvedModel.provider,
|
|
1750
|
+
model: resolvedModel.model,
|
|
1751
|
+
}
|
|
1752
|
+
: {}),
|
|
1753
|
+
modelName: turnExecutionPolicy.upstreamModelId,
|
|
1754
|
+
...(serviceTier ? { serviceTier } : {}),
|
|
1755
|
+
signal,
|
|
1756
|
+
}),
|
|
1757
|
+
),
|
|
1758
|
+
onError: (error) => {
|
|
1759
|
+
observability.warn("parallel session title generation failed", {
|
|
1760
|
+
...safeErrorDiagnostic(error),
|
|
1761
|
+
errorClass: "ParallelSessionTitleGenerationError",
|
|
1762
|
+
errorCode: "parallel_session_title_generation_failed",
|
|
1763
|
+
origin: "worker",
|
|
1764
|
+
});
|
|
1765
|
+
},
|
|
1766
|
+
});
|
|
1767
|
+
}
|
|
1768
|
+
try {
|
|
1769
|
+
let retriedAfterCompaction = false;
|
|
1770
|
+
while (true) {
|
|
1771
|
+
try {
|
|
1772
|
+
const result = await runStreamAttempt({
|
|
1773
|
+
requireTerminalModelResponse: retriedAfterCompaction,
|
|
1774
|
+
});
|
|
1775
|
+
if (retriedAfterCompaction) {
|
|
1776
|
+
observability.info("context compaction recovery succeeded after in-activity retry", {
|
|
1777
|
+
sessionId: input.sessionId,
|
|
1778
|
+
turnId: activeTurnId,
|
|
1779
|
+
});
|
|
1780
|
+
}
|
|
1781
|
+
return result;
|
|
1782
|
+
} catch (attemptError) {
|
|
1783
|
+
const overflow = classifyContextWindowOverflowError(attemptError);
|
|
1784
|
+
const compactionNeeded = findCompactionNeededError(attemptError);
|
|
1785
|
+
const recoveryKind = compactionNeeded
|
|
1786
|
+
? compactionNeeded.trigger === "operator"
|
|
1787
|
+
? "operator"
|
|
1788
|
+
: "proactive"
|
|
1789
|
+
: overflow
|
|
1790
|
+
? "overflow"
|
|
1791
|
+
: null;
|
|
1792
|
+
if (!recoveryKind || !eventing.publish || !eventing.turnStartedPublished) {
|
|
1793
|
+
throw attemptError;
|
|
1794
|
+
}
|
|
1795
|
+
await flushRuntimeBatcher();
|
|
1796
|
+
await historySink.reconcileConversationTruth({ skipInputOnlyRows: true });
|
|
1797
|
+
observability.warn("context compaction recovery attempted", {
|
|
1638
1798
|
sessionId: input.sessionId,
|
|
1639
1799
|
turnId: activeTurnId,
|
|
1800
|
+
reason: recoveryKind,
|
|
1801
|
+
...safeErrorDiagnostic(attemptError),
|
|
1802
|
+
signalTokens: compactionNeeded?.signalTokens,
|
|
1803
|
+
thresholdTokens: compactionNeeded?.thresholdTokens,
|
|
1640
1804
|
});
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
observability.warn("context compaction recovery attempted", {
|
|
1659
|
-
sessionId: input.sessionId,
|
|
1660
|
-
turnId: activeTurnId,
|
|
1661
|
-
reason: recoveryKind,
|
|
1662
|
-
...safeErrorDiagnostic(attemptError),
|
|
1663
|
-
signalTokens: compactionNeeded?.signalTokens,
|
|
1664
|
-
thresholdTokens: compactionNeeded?.thresholdTokens,
|
|
1665
|
-
});
|
|
1666
|
-
let compacted = false;
|
|
1667
|
-
let compactionHandled = false;
|
|
1668
|
-
let compactionFailureMessage: string | null = null;
|
|
1669
|
-
let compactionRequestCleared = false;
|
|
1670
|
-
try {
|
|
1671
|
-
const outcome = await forceContextCompaction(
|
|
1672
|
-
recoveryKind,
|
|
1673
|
-
compactionNeeded?.signalTokens ?? null,
|
|
1674
|
-
);
|
|
1675
|
-
compacted = outcome.compacted;
|
|
1676
|
-
if (outcome.compacted) {
|
|
1677
|
-
compactionHandled = true;
|
|
1678
|
-
} else {
|
|
1679
|
-
compactionHandled = recoveryKind === "operator" && outcome.requestConsumed;
|
|
1680
|
-
if (!compactionHandled) {
|
|
1681
|
-
compactionFailureMessage = compactionFailureReason(outcome.reason);
|
|
1805
|
+
let compacted = false;
|
|
1806
|
+
let compactionHandled = false;
|
|
1807
|
+
let compactionFailureMessage: string | null = null;
|
|
1808
|
+
let compactionRequestCleared = false;
|
|
1809
|
+
try {
|
|
1810
|
+
const outcome = await forceContextCompaction(
|
|
1811
|
+
recoveryKind,
|
|
1812
|
+
compactionNeeded?.signalTokens ?? null,
|
|
1813
|
+
);
|
|
1814
|
+
compacted = outcome.compacted;
|
|
1815
|
+
if (outcome.compacted) {
|
|
1816
|
+
compactionHandled = true;
|
|
1817
|
+
} else {
|
|
1818
|
+
compactionHandled = recoveryKind === "operator" && outcome.requestConsumed;
|
|
1819
|
+
if (!compactionHandled) {
|
|
1820
|
+
compactionFailureMessage = compactionFailureReason(outcome.reason);
|
|
1821
|
+
}
|
|
1682
1822
|
}
|
|
1683
|
-
}
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1823
|
+
} catch (compactError) {
|
|
1824
|
+
// Transient checkpoint-provider failures recover this same accepted
|
|
1825
|
+
// turn through the normal provider/capacity path. They are not an
|
|
1826
|
+
// empty summary and must not create a new goal continuation.
|
|
1827
|
+
if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
|
|
1828
|
+
if (compactError instanceof TurnAttemptFencedError) throw compactError;
|
|
1829
|
+
const landmark = await settleFailedContextCompactionLandmark(
|
|
1830
|
+
db,
|
|
1831
|
+
{
|
|
1832
|
+
accountId: input.accountId,
|
|
1833
|
+
workspaceId: input.workspaceId,
|
|
1834
|
+
sessionId: input.sessionId,
|
|
1835
|
+
turnId: activeTurnId!,
|
|
1836
|
+
executionGeneration: attempt.executionGeneration,
|
|
1837
|
+
attemptId: input.attemptId,
|
|
1838
|
+
},
|
|
1839
|
+
{
|
|
1840
|
+
clearRequestedCompaction: recoveryKind === "operator",
|
|
1841
|
+
publishLiveEvents: publishCompactionLiveEvents,
|
|
1842
|
+
},
|
|
1843
|
+
);
|
|
1844
|
+
compactionRequestCleared = landmark.requestConsumed;
|
|
1845
|
+
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
1846
|
+
await finishParallelSessionTitle();
|
|
1847
|
+
const deferredSteer = await settleDeferredSteerAfterCompaction();
|
|
1848
|
+
if (deferredSteer) return deferredSteer;
|
|
1849
|
+
compactionFailureMessage = String(compactionFailureReasonFromError(compactError));
|
|
1850
|
+
observability.warn("context compaction recovery compaction failed", {
|
|
1695
1851
|
sessionId: input.sessionId,
|
|
1696
|
-
turnId: activeTurnId
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1852
|
+
turnId: activeTurnId,
|
|
1853
|
+
...safeErrorDiagnostic(compactError),
|
|
1854
|
+
});
|
|
1855
|
+
}
|
|
1856
|
+
if (!compactionHandled) {
|
|
1857
|
+
const errorMessage =
|
|
1858
|
+
compactionFailureMessage ??
|
|
1859
|
+
"compaction summarization failed: compaction produced no replacement history";
|
|
1860
|
+
await finishParallelSessionTitle();
|
|
1861
|
+
if (
|
|
1862
|
+
!(await eventing.settle!({
|
|
1863
|
+
events: [
|
|
1864
|
+
{
|
|
1865
|
+
type: "turn.failed",
|
|
1866
|
+
payload: {
|
|
1867
|
+
error: errorMessage,
|
|
1868
|
+
code: "context_compaction_failed",
|
|
1869
|
+
retryable: false,
|
|
1870
|
+
recovery: "user_message",
|
|
1871
|
+
compacted: false,
|
|
1872
|
+
},
|
|
1873
|
+
},
|
|
1874
|
+
{
|
|
1875
|
+
type: "session.status.changed",
|
|
1876
|
+
payload: { status: "idle" },
|
|
1877
|
+
},
|
|
1878
|
+
],
|
|
1879
|
+
turnStatus: "failed",
|
|
1880
|
+
sessionStatus: "idle",
|
|
1881
|
+
activeTurnId: null,
|
|
1882
|
+
...(recoveryKind === "operator" && !compactionRequestCleared
|
|
1883
|
+
? { consumeRequestedCompactionFailure: true }
|
|
1884
|
+
: {}),
|
|
1885
|
+
}))
|
|
1886
|
+
) {
|
|
1887
|
+
return claimedResult({ status: "cancelled" });
|
|
1888
|
+
}
|
|
1889
|
+
control.turnMetricOutcome = "failed";
|
|
1890
|
+
control.activityStatus = "idle";
|
|
1891
|
+
control.activityError = attemptError;
|
|
1892
|
+
// The failed turn settlement already defers ordinary internal
|
|
1893
|
+
// updates and makes the delivered goal-continuation receipt
|
|
1894
|
+
// terminal. End this workflow run as well: returning plain idle
|
|
1895
|
+
// would immediately synthesize another goal continuation against
|
|
1896
|
+
// the unchanged active history and repeat the same failed
|
|
1897
|
+
// compaction. A new human/API prompt, Steer, or explicitly requested
|
|
1898
|
+
// Compact remains a durable explicit wake and may retry; ordinary
|
|
1899
|
+
// machine updates stay pending for that actionable wake.
|
|
1900
|
+
return claimedResult({ status: "idle", deferredUntilWake: true });
|
|
1901
|
+
}
|
|
1902
|
+
await finishParallelSessionTitle();
|
|
1707
1903
|
const deferredSteer = await settleDeferredSteerAfterCompaction();
|
|
1708
1904
|
if (deferredSteer) return deferredSteer;
|
|
1709
|
-
|
|
1710
|
-
|
|
1905
|
+
// Codex parity: compaction remains inside the same logical turn and
|
|
1906
|
+
// the same activity. Rebuild the model-visible history from the
|
|
1907
|
+
// durable replacement and continue the sampling loop; do not create
|
|
1908
|
+
// a recovery event, a queue row, a fake user message, or a sandbox.
|
|
1909
|
+
retriedAfterCompaction = true;
|
|
1910
|
+
observability.info("context compaction recovery retrying turn after compaction", {
|
|
1711
1911
|
sessionId: input.sessionId,
|
|
1712
1912
|
turnId: activeTurnId,
|
|
1713
|
-
|
|
1913
|
+
reason: recoveryKind,
|
|
1914
|
+
compacted,
|
|
1714
1915
|
});
|
|
1916
|
+
await prepareRunAttemptInput();
|
|
1715
1917
|
}
|
|
1716
|
-
if (!compactionHandled) {
|
|
1717
|
-
const errorMessage =
|
|
1718
|
-
compactionFailureMessage ??
|
|
1719
|
-
"compaction summarization failed: compaction produced no replacement history";
|
|
1720
|
-
if (
|
|
1721
|
-
!(await eventing.settle!({
|
|
1722
|
-
events: [
|
|
1723
|
-
{
|
|
1724
|
-
type: "turn.failed",
|
|
1725
|
-
payload: {
|
|
1726
|
-
error: errorMessage,
|
|
1727
|
-
code: "context_compaction_failed",
|
|
1728
|
-
retryable: false,
|
|
1729
|
-
recovery: "user_message",
|
|
1730
|
-
compacted: false,
|
|
1731
|
-
},
|
|
1732
|
-
},
|
|
1733
|
-
{
|
|
1734
|
-
type: "session.status.changed",
|
|
1735
|
-
payload: { status: "idle" },
|
|
1736
|
-
},
|
|
1737
|
-
],
|
|
1738
|
-
turnStatus: "failed",
|
|
1739
|
-
sessionStatus: "idle",
|
|
1740
|
-
activeTurnId: null,
|
|
1741
|
-
...(recoveryKind === "operator" && !compactionRequestCleared
|
|
1742
|
-
? { consumeRequestedCompactionFailure: true }
|
|
1743
|
-
: {}),
|
|
1744
|
-
}))
|
|
1745
|
-
) {
|
|
1746
|
-
return claimedResult({ status: "cancelled" });
|
|
1747
|
-
}
|
|
1748
|
-
control.turnMetricOutcome = "failed";
|
|
1749
|
-
control.activityStatus = "idle";
|
|
1750
|
-
control.activityError = attemptError;
|
|
1751
|
-
// The failed turn settlement already defers ordinary internal
|
|
1752
|
-
// updates and makes the delivered goal-continuation receipt
|
|
1753
|
-
// terminal. End this workflow run as well: returning plain idle
|
|
1754
|
-
// would immediately synthesize another goal continuation against
|
|
1755
|
-
// the unchanged active history and repeat the same failed
|
|
1756
|
-
// compaction. A new human/API prompt, Steer, or explicitly requested
|
|
1757
|
-
// Compact remains a durable explicit wake and may retry; ordinary
|
|
1758
|
-
// machine updates stay pending for that actionable wake.
|
|
1759
|
-
return claimedResult({ status: "idle", deferredUntilWake: true });
|
|
1760
|
-
}
|
|
1761
|
-
const deferredSteer = await settleDeferredSteerAfterCompaction();
|
|
1762
|
-
if (deferredSteer) return deferredSteer;
|
|
1763
|
-
// Codex parity: compaction remains inside the same logical turn and
|
|
1764
|
-
// the same activity. Rebuild the model-visible history from the
|
|
1765
|
-
// durable replacement and continue the sampling loop; do not create
|
|
1766
|
-
// a recovery event, a queue row, a fake user message, or a sandbox.
|
|
1767
|
-
retriedAfterCompaction = true;
|
|
1768
|
-
observability.info("context compaction recovery retrying turn after compaction", {
|
|
1769
|
-
sessionId: input.sessionId,
|
|
1770
|
-
turnId: activeTurnId,
|
|
1771
|
-
reason: recoveryKind,
|
|
1772
|
-
compacted,
|
|
1773
|
-
});
|
|
1774
|
-
await prepareRunAttemptInput();
|
|
1775
1918
|
}
|
|
1919
|
+
} finally {
|
|
1920
|
+
await cancelParallelSessionTitle();
|
|
1776
1921
|
}
|
|
1777
1922
|
}
|