@opengeni/worker-bundle 0.20.3 → 0.22.1-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +52 -0
- package/dist/activities/agent-turn/agent-build.d.ts +89 -0
- package/dist/activities/agent-turn/claim.d.ts +67 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
- package/dist/activities/agent-turn/codex.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
- package/dist/activities/agent-turn/errors.d.ts +253 -0
- package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
- package/dist/activities/agent-turn/file-resources.d.ts +37 -0
- package/dist/activities/agent-turn/finalization.d.ts +42 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
- package/dist/activities/agent-turn/governance-model.d.ts +66 -0
- package/dist/activities/agent-turn/history-sink.d.ts +52 -0
- package/dist/activities/agent-turn/history.d.ts +63 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
- package/dist/activities/agent-turn/model-usage.d.ts +218 -0
- package/dist/activities/agent-turn/quiescence.d.ts +135 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
- package/dist/activities/agent-turn/run.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
- package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
- package/dist/activities/agent-turn/turn-context.d.ts +144 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
- package/dist/activities/agent-turn.d.ts +12 -954
- package/dist/activities/automations.d.ts +16 -0
- package/dist/activities/common.d.ts +1 -1
- package/dist/activities/connector-attachments.d.ts +13 -0
- package/dist/activities/documents.d.ts +1 -0
- package/dist/activities/environment.d.ts +26 -15
- package/dist/activities/generated-images.d.ts +1 -1
- package/dist/activities/git-credential-renewal.d.ts +3 -1
- package/dist/activities/goals.d.ts +4 -1
- package/dist/activities/google-drive-provider.d.ts +29 -0
- package/dist/activities/google-drive-publication.d.ts +17 -4
- package/dist/activities/incident-telemetry-authority.d.ts +52 -0
- package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
- package/dist/activities/knowledge-source-sync.d.ts +4 -0
- package/dist/activities/mcp-credentials.d.ts +9 -3
- package/dist/activities/open-suffix-resume.d.ts +77 -0
- package/dist/activities/packs.d.ts +3 -11
- package/dist/activities/parent-wake.d.ts +19 -0
- package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
- package/dist/activities/retained-screenshots.d.ts +1 -1
- package/dist/activities/rig-verification.d.ts +1 -1
- package/dist/activities/run-input.d.ts +9 -10
- package/dist/activities/sandbox-images.d.ts +1 -14
- package/dist/activities/sandbox-lease.d.ts +21 -3
- package/dist/activities/scheduled-tasks.d.ts +47 -0
- package/dist/activities/types.d.ts +23 -2
- package/dist/activities/video-media-validation.d.ts +1 -1
- package/dist/activities/video-output-retention.d.ts +1 -2
- package/dist/activities/xai-auth.d.ts +3 -0
- package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
- package/dist/activities-control-DKG22ONJ.js.map +1 -0
- package/dist/activities-control.d.ts +6 -2
- package/dist/activities-turn-4IMC3MAA.js +21326 -0
- package/dist/activities-turn-4IMC3MAA.js.map +1 -0
- package/dist/activities.d.ts +7 -3
- package/dist/artifact-materializer-entry.js +41 -22
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
- package/dist/chunk-CE7TPKBS.js.map +1 -0
- package/dist/chunk-CNYGVRKF.js +4698 -0
- package/dist/chunk-CNYGVRKF.js.map +1 -0
- package/dist/connection-provider-use.d.ts +18 -0
- package/dist/editable-artifact-materializer.d.ts +4 -3
- package/dist/github-rest-mcp.d.ts +26 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +58 -14
- package/dist/index.js.map +1 -1
- package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
- package/dist/model-context-contributions.d.ts +8 -4
- package/dist/observability-metrics.d.ts +63 -4
- package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
- package/dist/personal-github-git-credentials.d.ts +9 -0
- package/dist/pr-review-credentials.d.ts +6 -0
- package/dist/sandbox-archive-storage.d.ts +45 -0
- package/dist/sandbox-resume.d.ts +65 -6
- package/dist/sandbox-routing.d.ts +38 -2
- package/dist/scheduled-alert-occurrence.d.ts +33 -0
- package/dist/workflow-bundle.js +87 -28
- package/dist/workflows/activities.d.ts +1 -0
- package/dist/workflows/automations.d.ts +6 -0
- package/dist/workflows/document-index.d.ts +1 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/admission.ts +292 -0
- package/src/activities/agent-turn/agent-build.ts +802 -0
- package/src/activities/agent-turn/claim.ts +590 -0
- package/src/activities/agent-turn/codex-capacity.ts +720 -0
- package/src/activities/agent-turn/codex.ts +94 -0
- package/src/activities/agent-turn/compaction-prep.ts +722 -0
- package/src/activities/agent-turn/credential-leases.ts +240 -0
- package/src/activities/agent-turn/errors.ts +1068 -0
- package/src/activities/agent-turn/failure-settlement.ts +1436 -0
- package/src/activities/agent-turn/file-resources.ts +227 -0
- package/src/activities/agent-turn/finalization.ts +754 -0
- package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
- package/src/activities/agent-turn/governance-model.ts +465 -0
- package/src/activities/agent-turn/history-sink.ts +139 -0
- package/src/activities/agent-turn/history.ts +200 -0
- package/src/activities/agent-turn/media-artifacts.ts +359 -0
- package/src/activities/agent-turn/model-usage.ts +823 -0
- package/src/activities/agent-turn/quiescence.ts +353 -0
- package/src/activities/agent-turn/run-credentials.ts +761 -0
- package/src/activities/agent-turn/run.ts +1530 -0
- package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
- package/src/activities/agent-turn/sandbox-provision.ts +569 -0
- package/src/activities/agent-turn/sandbox-route.ts +358 -0
- package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
- package/src/activities/agent-turn/stream-attempt.ts +1670 -0
- package/src/activities/agent-turn/tool-environment.ts +729 -0
- package/src/activities/agent-turn/tool-policy.ts +216 -0
- package/src/activities/agent-turn/tool-result-spill.ts +265 -0
- package/src/activities/agent-turn/turn-context.ts +281 -0
- package/src/activities/agent-turn/xai-capacity.ts +229 -0
- package/src/activities/agent-turn.ts +12 -12701
- package/src/activities/api-integrations.ts +3 -0
- package/src/activities/automations.ts +214 -0
- package/src/activities/codemode-dispatcher.ts +12 -1
- package/src/activities/common.ts +1 -5
- package/src/activities/connector-attachments.ts +168 -0
- package/src/activities/context-compaction.ts +18 -0
- package/src/activities/environment.ts +151 -55
- package/src/activities/generated-images.ts +26 -11
- package/src/activities/git-credential-renewal.ts +13 -3
- package/src/activities/goals.ts +85 -19
- package/src/activities/google-drive-provider.ts +178 -0
- package/src/activities/google-drive-publication.ts +147 -49
- package/src/activities/incident-telemetry-authority.ts +327 -0
- package/src/activities/incident-telemetry-preflight.ts +317 -0
- package/src/activities/knowledge-source-sync.ts +178 -37
- package/src/activities/mcp-credentials.ts +191 -17
- package/src/activities/open-suffix-resume.ts +422 -0
- package/src/activities/packs.ts +12 -60
- package/src/activities/parent-wake.ts +64 -14
- package/src/activities/retained-screenshot-reaper.ts +6 -3
- package/src/activities/retained-screenshots.ts +15 -47
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +102 -142
- package/src/activities/sandbox-images.ts +7 -166
- package/src/activities/sandbox-lease.ts +406 -15
- package/src/activities/scheduled-tasks.ts +2103 -411
- package/src/activities/session-state.ts +10 -2
- package/src/activities/types.ts +32 -2
- package/src/activities/video-media-validation.ts +3 -3
- package/src/activities/video-output-retention.ts +2 -57
- package/src/activities/xai-auth.ts +8 -0
- package/src/activities-control.ts +2 -0
- package/src/activities.ts +3 -0
- package/src/activity-services.ts +13 -4
- package/src/connection-provider-use.ts +36 -0
- package/src/editable-artifact-materializer-subprocess.ts +25 -14
- package/src/editable-artifact-materializer.ts +29 -18
- package/src/github-rest-mcp.ts +713 -0
- package/src/http.ts +16 -6
- package/src/index.ts +40 -5
- package/src/knowledge-source-sync-activity-policy.ts +2 -0
- package/src/model-context-contributions.ts +33 -10
- package/src/observability-metrics.ts +357 -0
- package/src/opensandbox-kubernetes-inventory.ts +292 -0
- package/src/personal-github-git-credentials.ts +331 -0
- package/src/pr-review-credentials.ts +163 -0
- package/src/sandbox-archive-storage.ts +159 -0
- package/src/sandbox-resume.ts +449 -63
- package/src/sandbox-routing.ts +314 -16
- package/src/scheduled-alert-occurrence.ts +176 -0
- package/src/workflows/activities.ts +18 -2
- package/src/workflows/automations.ts +11 -0
- package/src/workflows/session.ts +13 -3
- package/src/workflows.ts +1 -0
- package/dist/activities-control-BJSDRSIP.js.map +0 -1
- package/dist/activities-turn-J5YO7XXX.js +0 -17243
- package/dist/activities-turn-J5YO7XXX.js.map +0 -1
- package/dist/chunk-N5TJIFCE.js.map +0 -1
- package/dist/chunk-P6Z2SLAA.js +0 -3092
- package/dist/chunk-P6Z2SLAA.js.map +0 -1
|
@@ -0,0 +1,1670 @@
|
|
|
1
|
+
import {
|
|
2
|
+
getSessionEvent,
|
|
3
|
+
getHumanInputResumeForEvent,
|
|
4
|
+
getSessionHumanInputRequest,
|
|
5
|
+
getWorkspace,
|
|
6
|
+
recordUsageEvent,
|
|
7
|
+
registerPendingSessionToolCall,
|
|
8
|
+
recordPendingSessionToolCallResult,
|
|
9
|
+
attachOpenSuffixToPendingToolCalls,
|
|
10
|
+
clearDurablePendingSessionToolCalls,
|
|
11
|
+
isSessionCompactionRequested,
|
|
12
|
+
nextSessionHistoryPosition,
|
|
13
|
+
} from "@opengeni/db";
|
|
14
|
+
import {
|
|
15
|
+
normalizeModelCallUsage,
|
|
16
|
+
normalizeSdkEvent,
|
|
17
|
+
extractOpenSuffixFromRunState,
|
|
18
|
+
assertOpenSuffixResumable,
|
|
19
|
+
interruptionKindForCallItem,
|
|
20
|
+
releaseMcpResultCustomDataFromSdkEvent,
|
|
21
|
+
findCompactionNeededError,
|
|
22
|
+
withRunCredentialsSession,
|
|
23
|
+
runOwnedSandboxSetup,
|
|
24
|
+
type SandboxFileDownload,
|
|
25
|
+
type OpenGeniRuntime,
|
|
26
|
+
type HistoryProviderApi,
|
|
27
|
+
type GitCredentialTokenWriterSession,
|
|
28
|
+
type NormalizedRunCredentialMaterial,
|
|
29
|
+
type RunCredentialCommandSession,
|
|
30
|
+
type CodemodeTokenWriterSession,
|
|
31
|
+
} from "@opengeni/runtime";
|
|
32
|
+
import { type Settings } from "@opengeni/config";
|
|
33
|
+
import { maybeCompactContext, settleFailedContextCompactionLandmark } from "../context-compaction";
|
|
34
|
+
import { TurnAttemptFencedError } from "../turn-attempt-fenced";
|
|
35
|
+
import { type MintedRunGitCredentials } from "../environment";
|
|
36
|
+
import { withFirstPartyTools } from "../goals";
|
|
37
|
+
import { turnInput } from "../run-input";
|
|
38
|
+
import { createRuntimeBatcher, currentActivityContext, nextStreamEvent } from "../streaming";
|
|
39
|
+
import type {
|
|
40
|
+
TurnActivityServices as ActivityServices,
|
|
41
|
+
RunAgentTurnInput,
|
|
42
|
+
RunAgentTurnResult,
|
|
43
|
+
} from "../types";
|
|
44
|
+
import { type ResumedTurnSandbox } from "../../sandbox-resume";
|
|
45
|
+
import {
|
|
46
|
+
modelCallAccountContext,
|
|
47
|
+
recordBatchFlush,
|
|
48
|
+
recordContextCompaction,
|
|
49
|
+
recordModelInputTokens,
|
|
50
|
+
recordTurnStartupPhase,
|
|
51
|
+
recordTurnWorkerPreparationTotal,
|
|
52
|
+
StreamTimingMetrics,
|
|
53
|
+
turnLifecycleMetricsFor,
|
|
54
|
+
} from "../../observability-metrics";
|
|
55
|
+
import {
|
|
56
|
+
compactRetainedScreenshotHistory,
|
|
57
|
+
sdkEventContainsInlineImage,
|
|
58
|
+
retainComputerScreenshot,
|
|
59
|
+
typedScreenshotFromSdkEvent,
|
|
60
|
+
unavailableRetainedSessionImage,
|
|
61
|
+
} from "../retained-screenshots";
|
|
62
|
+
import {
|
|
63
|
+
compactGeneratedImageSdkEvent,
|
|
64
|
+
generatedImageFromSdkEvent,
|
|
65
|
+
isCompletedGeneratedImageSdkEvent,
|
|
66
|
+
} from "../generated-images";
|
|
67
|
+
import { ToolResultSpill } from "./tool-result-spill";
|
|
68
|
+
import { createTurnCredentialLeases } from "./credential-leases";
|
|
69
|
+
import { createTurnMediaArtifacts } from "./media-artifacts";
|
|
70
|
+
import { createTurnHistorySink } from "./history-sink";
|
|
71
|
+
import {
|
|
72
|
+
interruptionCallIdsFromPause,
|
|
73
|
+
settleOpenSuffixResumeIfNeeded,
|
|
74
|
+
} from "../open-suffix-resume";
|
|
75
|
+
import {
|
|
76
|
+
OPEN_SUFFIX_RUN_STATE_BLOB,
|
|
77
|
+
resolveWorkspaceAgentHumanInputEnabled,
|
|
78
|
+
type RetainedArtifactMetadata,
|
|
79
|
+
type SessionEvent,
|
|
80
|
+
} from "@opengeni/contracts";
|
|
81
|
+
import { createModelCheckpointMemoryCollector } from "../../model-checkpoint-memory-collector";
|
|
82
|
+
|
|
83
|
+
import {
|
|
84
|
+
assertWorkspaceHumanInputAllowed,
|
|
85
|
+
stableHumanInputRequestId,
|
|
86
|
+
stableInteractionInterventionId,
|
|
87
|
+
stableInteractionInterventionOperationId,
|
|
88
|
+
BudgetExhaustedError,
|
|
89
|
+
ensureRunAllowed,
|
|
90
|
+
} from "./admission";
|
|
91
|
+
import {
|
|
92
|
+
compactionFailureReason,
|
|
93
|
+
safeErrorDiagnostic,
|
|
94
|
+
compactionFailureReasonFromError,
|
|
95
|
+
isCompactionSummaryFailure,
|
|
96
|
+
shouldRecoverCompactionProviderFailure,
|
|
97
|
+
classifyContextWindowOverflowError,
|
|
98
|
+
} from "./errors";
|
|
99
|
+
import {
|
|
100
|
+
pendingToolCallFromSdkEvent,
|
|
101
|
+
toolCallProducesRetainableSessionImage,
|
|
102
|
+
completedToolCallFromSdkEvent,
|
|
103
|
+
} from "./history";
|
|
104
|
+
import {
|
|
105
|
+
modelUsageSourceKey,
|
|
106
|
+
recordCompletedModelCallBeforeOwnershipFences,
|
|
107
|
+
TurnEventPublisher,
|
|
108
|
+
createModelResponseEventState,
|
|
109
|
+
modelResponseContextSignal,
|
|
110
|
+
assertModelResponseLatencyMode,
|
|
111
|
+
processModelResponseTerminalEvent,
|
|
112
|
+
emitModelCallUsage,
|
|
113
|
+
recordModelUsageAndDebitCredits,
|
|
114
|
+
recordAuthoritativeModelCallFact,
|
|
115
|
+
} from "./model-usage";
|
|
116
|
+
import { waitForTurnStreamCleanup } from "./quiescence";
|
|
117
|
+
import { waitForTurnOperation } from "./sandbox-provision";
|
|
118
|
+
|
|
119
|
+
import type { CompactionSummarizer } from "../context-compaction";
|
|
120
|
+
import type { TurnExecutionPolicyV1 } from "@opengeni/contracts";
|
|
121
|
+
import type { BoundRunCredentialResolver } from "../run-credentials";
|
|
122
|
+
import type { ModelHistoryAttachmentProjector } from "../run-input";
|
|
123
|
+
import type {
|
|
124
|
+
AttemptIdentityState,
|
|
125
|
+
BillingState,
|
|
126
|
+
ClaimedResult,
|
|
127
|
+
EventingState,
|
|
128
|
+
ProviderTurnState,
|
|
129
|
+
SandboxRuntimeState,
|
|
130
|
+
TurnControlState,
|
|
131
|
+
} from "./turn-context";
|
|
132
|
+
|
|
133
|
+
export type TurnStreamAttemptDeps = {
|
|
134
|
+
input: RunAgentTurnInput;
|
|
135
|
+
settings: Settings;
|
|
136
|
+
db: ActivityServices["db"];
|
|
137
|
+
runtime: ActivityServices["runtime"];
|
|
138
|
+
objectStorage: ActivityServices["objectStorage"];
|
|
139
|
+
observability: ActivityServices["observability"];
|
|
140
|
+
entitlements: ActivityServices["entitlements"];
|
|
141
|
+
startVideoGenerationWorkflow: ActivityServices["startVideoGenerationWorkflow"];
|
|
142
|
+
cancellationSignal: AbortSignal | undefined;
|
|
143
|
+
runtimeCancellationSignal: AbortSignal;
|
|
144
|
+
activityContext: ReturnType<typeof currentActivityContext>;
|
|
145
|
+
dispatchId: string;
|
|
146
|
+
control: TurnControlState;
|
|
147
|
+
attempt: AttemptIdentityState;
|
|
148
|
+
billingState: BillingState;
|
|
149
|
+
sandboxState: SandboxRuntimeState;
|
|
150
|
+
eventing: EventingState & {
|
|
151
|
+
publish: TurnEventPublisher;
|
|
152
|
+
settle: NonNullable<EventingState["settle"]>;
|
|
153
|
+
};
|
|
154
|
+
providerTurn: ProviderTurnState;
|
|
155
|
+
leases: ReturnType<typeof createTurnCredentialLeases>;
|
|
156
|
+
historySink: ReturnType<typeof createTurnHistorySink>;
|
|
157
|
+
media: ReturnType<typeof createTurnMediaArtifacts>;
|
|
158
|
+
toolResultSpill: ToolResultSpill;
|
|
159
|
+
claimedResult: ClaimedResult;
|
|
160
|
+
flushRuntimeBatcher: () => Promise<void>;
|
|
161
|
+
finalizeTurnOpStreamOps: () => Promise<void>;
|
|
162
|
+
runWorkspaceMutationForSandbox: <T>(
|
|
163
|
+
sandbox: ResumedTurnSandbox,
|
|
164
|
+
operation: string,
|
|
165
|
+
mutation: () => Promise<T>,
|
|
166
|
+
observePhase?: (measurement: {
|
|
167
|
+
phase: "admission" | "provider" | "settlement" | "snapshot_wait";
|
|
168
|
+
outcome: "completed" | "failed";
|
|
169
|
+
durationSeconds: number;
|
|
170
|
+
}) => void,
|
|
171
|
+
) => Promise<T>;
|
|
172
|
+
throwIfWorkerShuttingDown: () => void;
|
|
173
|
+
throwIfTurnCancelled: () => void;
|
|
174
|
+
setLastInputTokensFenced: (lastInputTokens: number | null) => Promise<void>;
|
|
175
|
+
attachGitCredentialRenewal: (
|
|
176
|
+
tokenSession: GitCredentialTokenWriterSession,
|
|
177
|
+
initial: MintedRunGitCredentials | undefined,
|
|
178
|
+
initialSandbox?: ResumedTurnSandbox,
|
|
179
|
+
) => Promise<void>;
|
|
180
|
+
attachCodemodeTokenRenewal: (
|
|
181
|
+
tokenSession?: CodemodeTokenWriterSession,
|
|
182
|
+
initialExpiresAt?: Date,
|
|
183
|
+
initialSandbox?: ResumedTurnSandbox,
|
|
184
|
+
) => Promise<void>;
|
|
185
|
+
attachRunCredentialRenewal: (
|
|
186
|
+
credentialSession: RunCredentialCommandSession,
|
|
187
|
+
initialMaterial: NormalizedRunCredentialMaterial | null,
|
|
188
|
+
initialSandbox?: ResumedTurnSandbox,
|
|
189
|
+
) => Promise<void>;
|
|
190
|
+
withProviderRequestContext: <T>(fn: () => Promise<T>) => Promise<T>;
|
|
191
|
+
publishCompactionLiveEvents: (events: SessionEvent[]) => Promise<void>;
|
|
192
|
+
publishCompactionOutcomeEvents: (events: SessionEvent[]) => Promise<void>;
|
|
193
|
+
recordCompanyBrainContributionReceiptOnce: () => void;
|
|
194
|
+
modelCheckpointMemoryCollector: ReturnType<typeof createModelCheckpointMemoryCollector>;
|
|
195
|
+
claimedModelUsageSourceKeys: Set<string>;
|
|
196
|
+
emittedModelUsageSourceKeys: Set<string>;
|
|
197
|
+
modelUsageDispatchId: string;
|
|
198
|
+
workerPreparationStartedAt: number;
|
|
199
|
+
fileDownloadsMaterializedForRun: boolean;
|
|
200
|
+
unavailableSandboxFilesNote: string | undefined;
|
|
201
|
+
runCredentialsNote: string | undefined;
|
|
202
|
+
mcpAvailabilityNote: string | undefined;
|
|
203
|
+
fileAuthoritySubjectId: string | null;
|
|
204
|
+
activeSandboxBackend: Settings["sandboxBackend"] | undefined;
|
|
205
|
+
groupBoxBackend: Settings["sandboxBackend"];
|
|
206
|
+
turnExecutionPolicy: TurnExecutionPolicyV1;
|
|
207
|
+
turn: { executionGeneration: number; model: string };
|
|
208
|
+
trigger: NonNullable<Awaited<ReturnType<typeof getSessionEvent>>>;
|
|
209
|
+
humanInputResume: Awaited<ReturnType<typeof getHumanInputResumeForEvent>>;
|
|
210
|
+
agent: ReturnType<ActivityServices["runtime"]["buildAgent"]>;
|
|
211
|
+
resolvedModel: ReturnType<ActivityServices["runtime"]["resolveTurnModel"]>;
|
|
212
|
+
providerApi: HistoryProviderApi;
|
|
213
|
+
runSettings: Settings;
|
|
214
|
+
turnTools: ReturnType<typeof withFirstPartyTools>;
|
|
215
|
+
compactSummarizer: CompactionSummarizer;
|
|
216
|
+
compactionModelHistoryProjector: (
|
|
217
|
+
items: Array<Record<string, unknown>>,
|
|
218
|
+
) => Promise<Array<Record<string, unknown>>>;
|
|
219
|
+
generatedImageHistoryProjector: (
|
|
220
|
+
items: Array<Record<string, unknown>>,
|
|
221
|
+
) => Promise<Array<Record<string, unknown>>>;
|
|
222
|
+
modelHistoryProjector: ModelHistoryAttachmentProjector;
|
|
223
|
+
compactionModeOptions: NonNullable<Parameters<typeof maybeCompactContext>[5]>;
|
|
224
|
+
initialRunCredentialMaterial: NormalizedRunCredentialMaterial | null;
|
|
225
|
+
initialGitCredentials: MintedRunGitCredentials | undefined;
|
|
226
|
+
sandboxEnvironment: Record<string, string>;
|
|
227
|
+
sandboxCodemodeToken: string | undefined;
|
|
228
|
+
sandboxCodemodeTokenExpiresAt: Date | string | number | null | undefined;
|
|
229
|
+
fileResourceDownloads: SandboxFileDownload[];
|
|
230
|
+
runCredentialResolver: BoundRunCredentialResolver | null;
|
|
231
|
+
videoGenerationAcceptancesByCallId: Map<string, { operationId: string; requestDigest: string }>;
|
|
232
|
+
};
|
|
233
|
+
|
|
234
|
+
export async function runTurnStreamAttempt(
|
|
235
|
+
deps: TurnStreamAttemptDeps,
|
|
236
|
+
): Promise<RunAgentTurnResult> {
|
|
237
|
+
const {
|
|
238
|
+
input,
|
|
239
|
+
settings,
|
|
240
|
+
db,
|
|
241
|
+
runtime,
|
|
242
|
+
objectStorage,
|
|
243
|
+
observability,
|
|
244
|
+
entitlements,
|
|
245
|
+
startVideoGenerationWorkflow,
|
|
246
|
+
cancellationSignal,
|
|
247
|
+
runtimeCancellationSignal,
|
|
248
|
+
activityContext,
|
|
249
|
+
dispatchId,
|
|
250
|
+
control,
|
|
251
|
+
attempt,
|
|
252
|
+
billingState,
|
|
253
|
+
sandboxState,
|
|
254
|
+
eventing,
|
|
255
|
+
providerTurn,
|
|
256
|
+
leases,
|
|
257
|
+
historySink,
|
|
258
|
+
media,
|
|
259
|
+
toolResultSpill,
|
|
260
|
+
claimedResult,
|
|
261
|
+
flushRuntimeBatcher,
|
|
262
|
+
finalizeTurnOpStreamOps,
|
|
263
|
+
runWorkspaceMutationForSandbox,
|
|
264
|
+
throwIfWorkerShuttingDown,
|
|
265
|
+
throwIfTurnCancelled,
|
|
266
|
+
setLastInputTokensFenced,
|
|
267
|
+
attachGitCredentialRenewal,
|
|
268
|
+
attachCodemodeTokenRenewal,
|
|
269
|
+
attachRunCredentialRenewal,
|
|
270
|
+
withProviderRequestContext,
|
|
271
|
+
publishCompactionLiveEvents,
|
|
272
|
+
publishCompactionOutcomeEvents,
|
|
273
|
+
recordCompanyBrainContributionReceiptOnce,
|
|
274
|
+
modelCheckpointMemoryCollector,
|
|
275
|
+
claimedModelUsageSourceKeys,
|
|
276
|
+
emittedModelUsageSourceKeys,
|
|
277
|
+
modelUsageDispatchId,
|
|
278
|
+
workerPreparationStartedAt,
|
|
279
|
+
fileDownloadsMaterializedForRun,
|
|
280
|
+
unavailableSandboxFilesNote,
|
|
281
|
+
runCredentialsNote,
|
|
282
|
+
mcpAvailabilityNote,
|
|
283
|
+
fileAuthoritySubjectId,
|
|
284
|
+
activeSandboxBackend,
|
|
285
|
+
groupBoxBackend,
|
|
286
|
+
turnExecutionPolicy,
|
|
287
|
+
turn,
|
|
288
|
+
trigger,
|
|
289
|
+
humanInputResume,
|
|
290
|
+
agent,
|
|
291
|
+
resolvedModel,
|
|
292
|
+
providerApi,
|
|
293
|
+
runSettings,
|
|
294
|
+
turnTools,
|
|
295
|
+
compactSummarizer,
|
|
296
|
+
compactionModelHistoryProjector,
|
|
297
|
+
generatedImageHistoryProjector,
|
|
298
|
+
modelHistoryProjector,
|
|
299
|
+
compactionModeOptions,
|
|
300
|
+
initialRunCredentialMaterial,
|
|
301
|
+
initialGitCredentials,
|
|
302
|
+
sandboxEnvironment,
|
|
303
|
+
sandboxCodemodeToken,
|
|
304
|
+
sandboxCodemodeTokenExpiresAt,
|
|
305
|
+
fileResourceDownloads,
|
|
306
|
+
runCredentialResolver,
|
|
307
|
+
videoGenerationAcceptancesByCallId,
|
|
308
|
+
} = deps;
|
|
309
|
+
|
|
310
|
+
// Build one attempt from canonical history. The provider adapter creates
|
|
311
|
+
// only the temporary wire view required by this turn's selected API;
|
|
312
|
+
// subscription identity never edits or filters durable history.
|
|
313
|
+
const activeTurnId = attempt.turnId;
|
|
314
|
+
if (!activeTurnId) {
|
|
315
|
+
throw new Error("Turn id was not initialized");
|
|
316
|
+
}
|
|
317
|
+
let runInput: Awaited<ReturnType<typeof turnInput>>["input"] | null = null;
|
|
318
|
+
const prepareRunAttemptInput = async () => {
|
|
319
|
+
const historyPreparationStartedAt = performance.now();
|
|
320
|
+
let historyPreparationOutcome: "completed" | "failed" = "completed";
|
|
321
|
+
let preparedHistoryCount: number | null = null;
|
|
322
|
+
try {
|
|
323
|
+
const prepared = await turnInput(db, runtime, agent, trigger, {
|
|
324
|
+
turnId: activeTurnId,
|
|
325
|
+
fileAuthority: {
|
|
326
|
+
accountId: input.accountId,
|
|
327
|
+
subjectId: fileAuthoritySubjectId,
|
|
328
|
+
},
|
|
329
|
+
recovering: turn.executionGeneration > 1,
|
|
330
|
+
...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
|
|
331
|
+
...(runCredentialsNote ? { runCredentialsNote } : {}),
|
|
332
|
+
...(mcpAvailabilityNote ? { mcpAvailabilityNote } : {}),
|
|
333
|
+
providerApi,
|
|
334
|
+
projectCanonicalHistory: generatedImageHistoryProjector,
|
|
335
|
+
materializeModelHistory: media.materializeScreenshotHistory,
|
|
336
|
+
projectModelHistory: modelHistoryProjector,
|
|
337
|
+
onPreparationPhase: (measurement) => {
|
|
338
|
+
recordTurnStartupPhase(observability, {
|
|
339
|
+
phase: `history_${measurement.phase}`,
|
|
340
|
+
provider: turnExecutionPolicy.providerId,
|
|
341
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
342
|
+
outcome: measurement.outcome,
|
|
343
|
+
durationSeconds: measurement.durationSeconds,
|
|
344
|
+
});
|
|
345
|
+
},
|
|
346
|
+
});
|
|
347
|
+
runInput = prepared.input;
|
|
348
|
+
historySink.providerArtifactCandidates = prepared.providerArtifactCandidates;
|
|
349
|
+
// Slice index = the length of the model-facing (active) history this turn
|
|
350
|
+
// is seeded from; new items beyond it (the trigger message + this turn's
|
|
351
|
+
// generated items) are the ones to persist. After a compaction this is the
|
|
352
|
+
// short [summary, ...tail] active set, NOT the total row count. The
|
|
353
|
+
// absolute write position is tracked separately (next whole number past
|
|
354
|
+
// the max existing position) because the fractional summary row means
|
|
355
|
+
// total rows no longer equal max(position)+1. Pre-compaction both reduce to
|
|
356
|
+
// the old total-count value, so the common path is unchanged.
|
|
357
|
+
//
|
|
358
|
+
// CRITICAL: seed from the structurally repaired active-row length, not the raw active
|
|
359
|
+
// count. `prepareRunInput` builds `state.history` from
|
|
360
|
+
// `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
|
|
361
|
+
// rows (a legacy orphan/dangling pair), the in-memory history this turn
|
|
362
|
+
// starts from is K shorter than the raw row count. The reconcile slices the
|
|
363
|
+
// repaired `state.history` off `historySink.persistedHistoryCount`; seeding it from
|
|
364
|
+
// the raw count (K too high) skips K genuinely-new items, and a
|
|
365
|
+
// `function_call` left in that skipped region can later have its
|
|
366
|
+
// `function_call_result` persisted alone — the orphan that 400s on replay
|
|
367
|
+
// and bricks the session (issue-61). The repaired seed is already
|
|
368
|
+
// orphan-free, so it is a stable prefix of the re-sanitized history and the
|
|
369
|
+
// slice begins exactly at the first genuinely-new item.
|
|
370
|
+
// prepareInput already sanitized the exact durable prefix represented
|
|
371
|
+
// by state.history. Carry its count forward instead of loading and
|
|
372
|
+
// retaining the full active transcript a second time beside runInput.
|
|
373
|
+
historySink.persistedHistoryCount = prepared.persistedHistoryCount;
|
|
374
|
+
preparedHistoryCount = prepared.persistedHistoryCount;
|
|
375
|
+
const historyPositionStartedAt = performance.now();
|
|
376
|
+
let historyPositionOutcome: "completed" | "failed" = "completed";
|
|
377
|
+
try {
|
|
378
|
+
historySink.nextHistoryPosition = await nextSessionHistoryPosition(
|
|
379
|
+
db,
|
|
380
|
+
input.workspaceId,
|
|
381
|
+
input.sessionId,
|
|
382
|
+
);
|
|
383
|
+
} catch (error) {
|
|
384
|
+
historyPositionOutcome = "failed";
|
|
385
|
+
throw error;
|
|
386
|
+
} finally {
|
|
387
|
+
recordTurnStartupPhase(observability, {
|
|
388
|
+
phase: "history_position_load",
|
|
389
|
+
provider: turnExecutionPolicy.providerId,
|
|
390
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
391
|
+
outcome: historyPositionOutcome,
|
|
392
|
+
durationSeconds: (performance.now() - historyPositionStartedAt) / 1_000,
|
|
393
|
+
});
|
|
394
|
+
}
|
|
395
|
+
} catch (error) {
|
|
396
|
+
historyPreparationOutcome = "failed";
|
|
397
|
+
throw error;
|
|
398
|
+
} finally {
|
|
399
|
+
recordTurnStartupPhase(observability, {
|
|
400
|
+
phase: "history_preparation",
|
|
401
|
+
provider: turnExecutionPolicy.providerId,
|
|
402
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
403
|
+
outcome: historyPreparationOutcome,
|
|
404
|
+
durationSeconds: (performance.now() - historyPreparationStartedAt) / 1_000,
|
|
405
|
+
count: preparedHistoryCount,
|
|
406
|
+
});
|
|
407
|
+
}
|
|
408
|
+
};
|
|
409
|
+
|
|
410
|
+
const forceContextCompaction = async (
|
|
411
|
+
triggerLabel: "overflow" | "proactive" | "operator",
|
|
412
|
+
recoverySignalTokens: number | null,
|
|
413
|
+
) => {
|
|
414
|
+
const outcome = await waitForTurnOperation(
|
|
415
|
+
maybeCompactContext(
|
|
416
|
+
db,
|
|
417
|
+
eventing.modelRunSettings,
|
|
418
|
+
{
|
|
419
|
+
accountId: input.accountId,
|
|
420
|
+
workspaceId: input.workspaceId,
|
|
421
|
+
sessionId: input.sessionId,
|
|
422
|
+
turnId: activeTurnId,
|
|
423
|
+
executionGeneration: attempt.executionGeneration,
|
|
424
|
+
attemptId: input.attemptId,
|
|
425
|
+
},
|
|
426
|
+
// Never reuse the persisted prior-turn signal for recovery. Proactive
|
|
427
|
+
// guards provide their exact current provider-based signal; provider
|
|
428
|
+
// overflows do not, so their diagnostic signal stays unknown/zero.
|
|
429
|
+
// Forced recovery proves progress separately by comparing the
|
|
430
|
+
// replacement with the current active-history estimate.
|
|
431
|
+
recoverySignalTokens,
|
|
432
|
+
compactSummarizer,
|
|
433
|
+
{
|
|
434
|
+
force: true,
|
|
435
|
+
...(triggerLabel === "operator" ? { clearRequestedCompaction: true } : {}),
|
|
436
|
+
trigger: triggerLabel,
|
|
437
|
+
materializeHistory: media.materializeScreenshotHistory,
|
|
438
|
+
projectModelInput: compactionModelHistoryProjector,
|
|
439
|
+
...compactionModeOptions,
|
|
440
|
+
},
|
|
441
|
+
),
|
|
442
|
+
cancellationSignal,
|
|
443
|
+
undefined,
|
|
444
|
+
);
|
|
445
|
+
if (outcome.events.length > 0) {
|
|
446
|
+
if (outcome.compacted) {
|
|
447
|
+
recordContextCompaction(observability, triggerLabel);
|
|
448
|
+
}
|
|
449
|
+
await publishCompactionOutcomeEvents(outcome.events);
|
|
450
|
+
}
|
|
451
|
+
return outcome;
|
|
452
|
+
};
|
|
453
|
+
|
|
454
|
+
// Keep response identity across every stream attempt in this
|
|
455
|
+
// activity. Context compaction retries the same logical dispatch by
|
|
456
|
+
// calling runStreamAttempt again; resetting this state there would reuse
|
|
457
|
+
// the first no-response-ID fallback key and suppress a real model call.
|
|
458
|
+
const modelResponseState = createModelResponseEventState(claimedModelUsageSourceKeys);
|
|
459
|
+
let workerPreparationTotalRecorded = false;
|
|
460
|
+
const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
|
|
461
|
+
if (!runInput) {
|
|
462
|
+
throw new Error("Run input was not prepared");
|
|
463
|
+
}
|
|
464
|
+
eventing.stream = undefined;
|
|
465
|
+
eventing.batcher = null;
|
|
466
|
+
// The SDK emits every processed call item for one model response before
|
|
467
|
+
// it emits any result for that response. Keep that response-local batch
|
|
468
|
+
// in memory so an orphan from an older response cannot pin later stable
|
|
469
|
+
// calls. Durable recovery remains call/result based and needs no batch
|
|
470
|
+
// schema or compatibility state.
|
|
471
|
+
let currentToolBatchCallIds = new Set<string>();
|
|
472
|
+
let currentToolBatchCompletedCallIds = new Set<string>();
|
|
473
|
+
let streamSawPerResponseUsage = false;
|
|
474
|
+
// Actual input tokens of the most recent model response this turn; the
|
|
475
|
+
// pre-read trigger for the NEXT turn. Persisted at every turn-end path.
|
|
476
|
+
throwIfWorkerShuttingDown();
|
|
477
|
+
throwIfTurnCancelled();
|
|
478
|
+
const streamProvider = resolvedModel?.provider.id ?? settings.openaiProvider ?? "openai";
|
|
479
|
+
const providerPublishesNativeRequestEvents =
|
|
480
|
+
resolvedModel?.provider.kind === "codex-subscription" ||
|
|
481
|
+
resolvedModel?.provider.kind === "xai-subscription";
|
|
482
|
+
let fallbackProviderRequestStartedAt: number | null = null;
|
|
483
|
+
const recordFallbackProviderDispatchAtWire = async (): Promise<void> => {
|
|
484
|
+
if (
|
|
485
|
+
providerPublishesNativeRequestEvents ||
|
|
486
|
+
eventing.firstModelRequestPreparationRecorded ||
|
|
487
|
+
eventing.firstModelRequestPreparationStartedAt === null
|
|
488
|
+
) {
|
|
489
|
+
return;
|
|
490
|
+
}
|
|
491
|
+
sandboxState.firstProviderRequestStarted = true;
|
|
492
|
+
eventing.firstModelRequestPreparationRecorded = true;
|
|
493
|
+
const preparationDurationMs = Math.max(
|
|
494
|
+
0,
|
|
495
|
+
performance.now() - eventing.firstModelRequestPreparationStartedAt,
|
|
496
|
+
);
|
|
497
|
+
recordTurnStartupPhase(observability, {
|
|
498
|
+
phase: "model_request_preparation",
|
|
499
|
+
provider: turnExecutionPolicy.providerId,
|
|
500
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
501
|
+
outcome: "completed",
|
|
502
|
+
durationSeconds: preparationDurationMs / 1_000,
|
|
503
|
+
count: turnTools.length,
|
|
504
|
+
});
|
|
505
|
+
const auditStartedAt = performance.now();
|
|
506
|
+
let auditOutcome: "completed" | "failed" = "completed";
|
|
507
|
+
try {
|
|
508
|
+
await eventing.publish!([
|
|
509
|
+
{
|
|
510
|
+
type: "turn.startup.phase.completed",
|
|
511
|
+
payload: {
|
|
512
|
+
phase: "model_preparation",
|
|
513
|
+
durationMs: Math.round(preparationDurationMs),
|
|
514
|
+
},
|
|
515
|
+
},
|
|
516
|
+
{
|
|
517
|
+
type: "agent.model.request",
|
|
518
|
+
payload: {
|
|
519
|
+
phase: "started",
|
|
520
|
+
provider: streamProvider,
|
|
521
|
+
turnId: activeTurnId,
|
|
522
|
+
attemptId: input.attemptId,
|
|
523
|
+
dispatchId,
|
|
524
|
+
executionGeneration: attempt.executionGeneration,
|
|
525
|
+
},
|
|
526
|
+
},
|
|
527
|
+
]);
|
|
528
|
+
fallbackProviderRequestStartedAt = performance.now();
|
|
529
|
+
} catch (error) {
|
|
530
|
+
auditOutcome = "failed";
|
|
531
|
+
throw error;
|
|
532
|
+
} finally {
|
|
533
|
+
recordTurnStartupPhase(observability, {
|
|
534
|
+
phase: "model_request_audit",
|
|
535
|
+
provider: turnExecutionPolicy.providerId,
|
|
536
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
537
|
+
outcome: auditOutcome,
|
|
538
|
+
durationSeconds: (performance.now() - auditStartedAt) / 1_000,
|
|
539
|
+
count: turnTools.length,
|
|
540
|
+
});
|
|
541
|
+
}
|
|
542
|
+
};
|
|
543
|
+
const ownedEstablished =
|
|
544
|
+
sandboxState.resolvedSandbox?.established ?? sandboxState.lazyOwnedSandbox;
|
|
545
|
+
const runStreamOnce = async (): ReturnType<OpenGeniRuntime["runStream"]> => {
|
|
546
|
+
// Eager owned sessions must settle the exact platform-setup provider
|
|
547
|
+
// promise before the long model stream starts; otherwise one admission
|
|
548
|
+
// would remain in flight for the entire turn and suppress every
|
|
549
|
+
// heartbeat capture. Lazy setup already runs under the same wrapper in
|
|
550
|
+
// its first-operation provisioner above.
|
|
551
|
+
if (sandboxState.resolvedSandbox && !sandboxState.lazyOwnedSandbox && ownedEstablished) {
|
|
552
|
+
const ownedSandboxSetupStartedAt = performance.now();
|
|
553
|
+
let ownedSandboxSetupOutcome: "completed" | "failed" = "completed";
|
|
554
|
+
try {
|
|
555
|
+
const eagerSetupSession = sandboxState.setupBoxSession ?? ownedEstablished.session;
|
|
556
|
+
// `deferredSetup: true` below tells the runtime that the worker owns
|
|
557
|
+
// platform setup, so the runtime intentionally skips its credential
|
|
558
|
+
// session callback. Materialize host-managed run credentials here
|
|
559
|
+
// before any setup command (including provider login hooks), then
|
|
560
|
+
// decorate setup commands so they source the active generation.
|
|
561
|
+
await attachRunCredentialRenewal(
|
|
562
|
+
eagerSetupSession as RunCredentialCommandSession,
|
|
563
|
+
initialRunCredentialMaterial,
|
|
564
|
+
sandboxState.resolvedSandbox,
|
|
565
|
+
);
|
|
566
|
+
const eagerCredentialSetupSession = initialRunCredentialMaterial
|
|
567
|
+
? withRunCredentialsSession(eagerSetupSession as object, input.sessionId)
|
|
568
|
+
: eagerSetupSession;
|
|
569
|
+
await runWorkspaceMutationForSandbox(
|
|
570
|
+
sandboxState.resolvedSandbox,
|
|
571
|
+
"eagerOwnedSandboxSetup",
|
|
572
|
+
async () =>
|
|
573
|
+
await runOwnedSandboxSetup(
|
|
574
|
+
agent,
|
|
575
|
+
ownedEstablished.session as never,
|
|
576
|
+
eagerCredentialSetupSession as never,
|
|
577
|
+
{
|
|
578
|
+
settings: runSettings,
|
|
579
|
+
environment: sandboxEnvironment,
|
|
580
|
+
preparedInput: runInput!,
|
|
581
|
+
...(fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {}),
|
|
582
|
+
onRuntimeEvent: async (event) => {
|
|
583
|
+
await leases.renewServing("runtime_event");
|
|
584
|
+
if (leases.servingLost()) {
|
|
585
|
+
throw new Error("Provider credential lease expired during sandbox setup");
|
|
586
|
+
}
|
|
587
|
+
await eventing.publish!([{ type: event.type, payload: event.payload }], true);
|
|
588
|
+
},
|
|
589
|
+
...(eventing.toolCancellationFenceRef.current
|
|
590
|
+
? {
|
|
591
|
+
commandRunner:
|
|
592
|
+
eventing.toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
593
|
+
eventing.toolCancellationFenceRef.current,
|
|
594
|
+
),
|
|
595
|
+
}
|
|
596
|
+
: {}),
|
|
597
|
+
},
|
|
598
|
+
),
|
|
599
|
+
);
|
|
600
|
+
await attachGitCredentialRenewal(
|
|
601
|
+
eagerSetupSession as GitCredentialTokenWriterSession,
|
|
602
|
+
initialGitCredentials,
|
|
603
|
+
);
|
|
604
|
+
} catch (error) {
|
|
605
|
+
ownedSandboxSetupOutcome = "failed";
|
|
606
|
+
throw error;
|
|
607
|
+
} finally {
|
|
608
|
+
recordTurnStartupPhase(observability, {
|
|
609
|
+
phase: "owned_sandbox_setup",
|
|
610
|
+
provider: turnExecutionPolicy.providerId,
|
|
611
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
612
|
+
outcome: ownedSandboxSetupOutcome,
|
|
613
|
+
durationSeconds: (performance.now() - ownedSandboxSetupStartedAt) / 1_000,
|
|
614
|
+
count: fileResourceDownloads.length,
|
|
615
|
+
});
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
// Conservative request boundary: once control enters runStream, the
|
|
619
|
+
// provider may have accepted work even if no response/event follows.
|
|
620
|
+
// Escaped MCP setup timeouts are automatically recoverable only
|
|
621
|
+
// before this line.
|
|
622
|
+
recordCompanyBrainContributionReceiptOnce();
|
|
623
|
+
if (!eventing.firstModelRequestPreparationRecorded) {
|
|
624
|
+
await eventing.publish!([
|
|
625
|
+
{
|
|
626
|
+
type: "turn.startup.phase.started",
|
|
627
|
+
payload: { phase: "model_preparation" },
|
|
628
|
+
},
|
|
629
|
+
]);
|
|
630
|
+
eventing.firstModelRequestPreparationStartedAt = performance.now();
|
|
631
|
+
eventing.firstModelRequestCheckpointAt = eventing.firstModelRequestPreparationStartedAt;
|
|
632
|
+
}
|
|
633
|
+
const providerDispatchStartedAt = performance.now();
|
|
634
|
+
let providerDispatchOutcome: "completed" | "failed" = "completed";
|
|
635
|
+
try {
|
|
636
|
+
// This histogram describes worker preparation until the first entry
|
|
637
|
+
// into the runtime. Lazy SDK request preparation and the durable
|
|
638
|
+
// model-request audit happen after this boundary and have their own
|
|
639
|
+
// phase metrics. Context-compaction recovery re-enters
|
|
640
|
+
// runStreamAttempt; recording again would fold the prior request and
|
|
641
|
+
// compaction into a bogus second startup sample.
|
|
642
|
+
if (!workerPreparationTotalRecorded) {
|
|
643
|
+
workerPreparationTotalRecorded = true;
|
|
644
|
+
recordTurnWorkerPreparationTotal(observability, {
|
|
645
|
+
provider: turnExecutionPolicy.providerId,
|
|
646
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
647
|
+
outcome: "completed",
|
|
648
|
+
durationSeconds: (performance.now() - workerPreparationStartedAt) / 1_000,
|
|
649
|
+
});
|
|
650
|
+
}
|
|
651
|
+
attempt.modelRequestStarted = true;
|
|
652
|
+
return await runtime.runStream(agent, runInput!, eventing.modelRunSettings, {
|
|
653
|
+
signal: runtimeCancellationSignal,
|
|
654
|
+
sandboxEnvironment,
|
|
655
|
+
onRuntimeEvent: async (event) => {
|
|
656
|
+
await leases.renewServing("runtime_event");
|
|
657
|
+
if (leases.servingLost()) {
|
|
658
|
+
throw new Error("Provider credential lease expired during the active turn");
|
|
659
|
+
}
|
|
660
|
+
await eventing.publish!([{ type: event.type, payload: event.payload }], true);
|
|
661
|
+
},
|
|
662
|
+
// P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
|
|
663
|
+
// keystone). Absent when the flag is off -> legacy build-and-discard.
|
|
664
|
+
...(ownedEstablished
|
|
665
|
+
? {
|
|
666
|
+
ownedSandbox: {
|
|
667
|
+
client: ownedEstablished.client,
|
|
668
|
+
session: ownedEstablished.session,
|
|
669
|
+
...(sandboxState.resolvedSandbox?.established.sessionState
|
|
670
|
+
? {
|
|
671
|
+
sessionState: sandboxState.resolvedSandbox.established.sessionState,
|
|
672
|
+
}
|
|
673
|
+
: {}),
|
|
674
|
+
// Pin platform setup (hooks + file materialization) to the un-proxied
|
|
675
|
+
// established box — never through the routing proxy, which would
|
|
676
|
+
// re-route those execs onto a machine swapped in mid-turn.
|
|
677
|
+
...(sandboxState.setupBoxSession
|
|
678
|
+
? { setupSession: sandboxState.setupBoxSession }
|
|
679
|
+
: {}),
|
|
680
|
+
...(fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {}),
|
|
681
|
+
// Both owned paths execute setup outside runStream: eager just
|
|
682
|
+
// above under its exact admission, lazy in the provisioner.
|
|
683
|
+
deferredSetup: true,
|
|
684
|
+
},
|
|
685
|
+
}
|
|
686
|
+
: {}),
|
|
687
|
+
...(activeSandboxBackend !== "selfhosted" &&
|
|
688
|
+
sandboxCodemodeToken &&
|
|
689
|
+
sandboxCodemodeTokenExpiresAt &&
|
|
690
|
+
!sandboxState.lazyOwnedSandbox
|
|
691
|
+
? {
|
|
692
|
+
onCodemodeTokenSessionReady: async (tokenSession: CodemodeTokenWriterSession) => {
|
|
693
|
+
const renewalSession =
|
|
694
|
+
(sandboxState.setupBoxSession as CodemodeTokenWriterSession | null) ??
|
|
695
|
+
tokenSession;
|
|
696
|
+
await attachCodemodeTokenRenewal(renewalSession);
|
|
697
|
+
},
|
|
698
|
+
}
|
|
699
|
+
: {}),
|
|
700
|
+
...(runCredentialResolver
|
|
701
|
+
? {
|
|
702
|
+
runCredentialSessionId: input.sessionId,
|
|
703
|
+
...(!sandboxState.lazyOwnedSandbox
|
|
704
|
+
? {
|
|
705
|
+
onRunCredentialSessionReady: async (
|
|
706
|
+
credentialSession: RunCredentialCommandSession,
|
|
707
|
+
) => {
|
|
708
|
+
const pinnedCredentialSession = sandboxState.setupBoxSession
|
|
709
|
+
? (sandboxState.setupBoxSession as RunCredentialCommandSession)
|
|
710
|
+
: credentialSession;
|
|
711
|
+
await attachRunCredentialRenewal(
|
|
712
|
+
pinnedCredentialSession,
|
|
713
|
+
initialRunCredentialMaterial,
|
|
714
|
+
);
|
|
715
|
+
},
|
|
716
|
+
}
|
|
717
|
+
: {}),
|
|
718
|
+
}
|
|
719
|
+
: {}),
|
|
720
|
+
...(eventing.modelRunSettings.sandboxBackend !== "none"
|
|
721
|
+
? {
|
|
722
|
+
onSandboxSessionReady: async (sandboxSession: CodemodeTokenWriterSession) => {
|
|
723
|
+
media.sdkOwnedSandboxSession = sandboxSession;
|
|
724
|
+
for (const receipt of media.generatedImageReceiptsCreatedThisTurn.values()) {
|
|
725
|
+
await media.materializeGeneratedImageInOwnedSdkSession(receipt, sandboxSession);
|
|
726
|
+
}
|
|
727
|
+
await toolResultSpill.materializeDeferred();
|
|
728
|
+
},
|
|
729
|
+
}
|
|
730
|
+
: {}),
|
|
731
|
+
contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
|
|
732
|
+
contextCompactionRequested: () =>
|
|
733
|
+
isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
734
|
+
onModelPreparationPhase: (measurement) => {
|
|
735
|
+
recordTurnStartupPhase(observability, {
|
|
736
|
+
phase: `model_prepare_${measurement.phase}`,
|
|
737
|
+
provider: turnExecutionPolicy.providerId,
|
|
738
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
739
|
+
outcome: measurement.outcome,
|
|
740
|
+
durationSeconds: measurement.durationSeconds,
|
|
741
|
+
...(measurement.count === undefined ? {} : { count: measurement.count }),
|
|
742
|
+
});
|
|
743
|
+
},
|
|
744
|
+
...(!providerPublishesNativeRequestEvents
|
|
745
|
+
? {
|
|
746
|
+
onModelTransportStarted: recordFallbackProviderDispatchAtWire,
|
|
747
|
+
}
|
|
748
|
+
: {}),
|
|
749
|
+
...(eventing.toolCancellationFenceRef.current
|
|
750
|
+
? {
|
|
751
|
+
turnToolCancellationFence: eventing.toolCancellationFenceRef.current,
|
|
752
|
+
}
|
|
753
|
+
: {}),
|
|
754
|
+
});
|
|
755
|
+
} catch (error) {
|
|
756
|
+
providerDispatchOutcome = "failed";
|
|
757
|
+
throw error;
|
|
758
|
+
} finally {
|
|
759
|
+
recordTurnStartupPhase(observability, {
|
|
760
|
+
phase: "provider_dispatch",
|
|
761
|
+
provider: turnExecutionPolicy.providerId,
|
|
762
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
763
|
+
outcome: providerDispatchOutcome,
|
|
764
|
+
durationSeconds: (performance.now() - providerDispatchStartedAt) / 1_000,
|
|
765
|
+
});
|
|
766
|
+
}
|
|
767
|
+
};
|
|
768
|
+
if (leases.codex.lost) {
|
|
769
|
+
throw new Error("Codex credential lease expired before the model run");
|
|
770
|
+
}
|
|
771
|
+
if (leases.xai.lost) {
|
|
772
|
+
throw new Error("xAI credential lease expired before the model run");
|
|
773
|
+
}
|
|
774
|
+
eventing.stream = await withProviderRequestContext(runStreamOnce);
|
|
775
|
+
// Bounded provider label for the streaming SLIs — the resolved registry
|
|
776
|
+
// provider id (or the built-in OpenAI/Azure provider), never a raw
|
|
777
|
+
// user-supplied model string.
|
|
778
|
+
const streamTiming = new StreamTimingMetrics(observability, {
|
|
779
|
+
provider: streamProvider,
|
|
780
|
+
});
|
|
781
|
+
eventing.batcher = createRuntimeBatcher(
|
|
782
|
+
async (events) => {
|
|
783
|
+
await eventing.publish!(events);
|
|
784
|
+
},
|
|
785
|
+
{
|
|
786
|
+
onFlush: ({ events, durationSeconds }) =>
|
|
787
|
+
recordBatchFlush(observability, { events, durationSeconds }),
|
|
788
|
+
},
|
|
789
|
+
);
|
|
790
|
+
|
|
791
|
+
const streamBootstrapStartedAt = performance.now();
|
|
792
|
+
let streamBootstrapRecorded = false;
|
|
793
|
+
const recordStreamBootstrap = (): void => {
|
|
794
|
+
if (streamBootstrapRecorded) return;
|
|
795
|
+
streamBootstrapRecorded = true;
|
|
796
|
+
recordTurnStartupPhase(observability, {
|
|
797
|
+
phase: "stream_bootstrap",
|
|
798
|
+
provider: turnExecutionPolicy.providerId,
|
|
799
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
800
|
+
outcome: "completed",
|
|
801
|
+
durationSeconds: (performance.now() - streamBootstrapStartedAt) / 1_000,
|
|
802
|
+
count: turnTools.length,
|
|
803
|
+
});
|
|
804
|
+
};
|
|
805
|
+
const settleFallbackProviderFirstByte = async (): Promise<void> => {
|
|
806
|
+
if (fallbackProviderRequestStartedAt === null) return;
|
|
807
|
+
const durationMs = Math.max(0, performance.now() - fallbackProviderRequestStartedAt);
|
|
808
|
+
fallbackProviderRequestStartedAt = null;
|
|
809
|
+
await eventing.publish!([
|
|
810
|
+
{
|
|
811
|
+
type: "agent.model.request",
|
|
812
|
+
payload: {
|
|
813
|
+
phase: "first_byte",
|
|
814
|
+
provider: streamProvider,
|
|
815
|
+
durationMs: Math.round(durationMs),
|
|
816
|
+
turnId: activeTurnId,
|
|
817
|
+
attemptId: input.attemptId,
|
|
818
|
+
dispatchId,
|
|
819
|
+
executionGeneration: attempt.executionGeneration,
|
|
820
|
+
},
|
|
821
|
+
},
|
|
822
|
+
]);
|
|
823
|
+
};
|
|
824
|
+
const iterator = eventing.stream.toStream()[Symbol.asyncIterator]();
|
|
825
|
+
let streamDone = false;
|
|
826
|
+
try {
|
|
827
|
+
while (true) {
|
|
828
|
+
const next = await nextStreamEvent(iterator, activityContext);
|
|
829
|
+
recordStreamBootstrap();
|
|
830
|
+
if (next.done) {
|
|
831
|
+
streamDone = true;
|
|
832
|
+
break;
|
|
833
|
+
}
|
|
834
|
+
await settleFallbackProviderFirstByte();
|
|
835
|
+
let stableToolCallIdsToClear: string[] | null = null;
|
|
836
|
+
let completedCurrentToolBatch = false;
|
|
837
|
+
let retainedScreenshotMetadata: RetainedArtifactMetadata | null = null;
|
|
838
|
+
let normalizedSdkEvents: ReturnType<typeof normalizeSdkEvent> | null = null;
|
|
839
|
+
const generatedImage = generatedImageFromSdkEvent(next.value);
|
|
840
|
+
if (isCompletedGeneratedImageSdkEvent(next.value) && !generatedImage) {
|
|
841
|
+
throw new Error(
|
|
842
|
+
"Completed native image item could not cross the retained-artifact boundary",
|
|
843
|
+
);
|
|
844
|
+
}
|
|
845
|
+
const generatedImageReceipt = generatedImage
|
|
846
|
+
? await media.retainNativeGeneratedImage(generatedImage)
|
|
847
|
+
: null;
|
|
848
|
+
const responseResult = await processModelResponseTerminalEvent({
|
|
849
|
+
event: next.value,
|
|
850
|
+
state: modelResponseState,
|
|
851
|
+
dispatchId: modelUsageDispatchId,
|
|
852
|
+
settings,
|
|
853
|
+
db,
|
|
854
|
+
observability,
|
|
855
|
+
publish: eventing.publish,
|
|
856
|
+
accountId: input.accountId,
|
|
857
|
+
workspaceId: input.workspaceId,
|
|
858
|
+
sessionId: input.sessionId,
|
|
859
|
+
turnId: activeTurnId,
|
|
860
|
+
turnAttemptId: input.attemptId,
|
|
861
|
+
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
862
|
+
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
863
|
+
model: turn.model,
|
|
864
|
+
latencyMode: turnExecutionPolicy.latencyMode,
|
|
865
|
+
metricProvider: streamProvider,
|
|
866
|
+
externallyBilled: billingState.isExternallyBilledTurn,
|
|
867
|
+
servingCredentialId: providerTurn.effectiveCodexCredentialId,
|
|
868
|
+
priorSessionCredentialId: providerTurn.priorSessionCodexCredentialId,
|
|
869
|
+
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
870
|
+
renewLease: () => leases.renewServing("model_usage"),
|
|
871
|
+
leaseLost: leases.servingLost,
|
|
872
|
+
leaseLostMessage: "Provider credential lease expired during the active turn",
|
|
873
|
+
setLastInputTokens: setLastInputTokensFenced,
|
|
874
|
+
contextContributions: eventing.companyBrainContextContributions,
|
|
875
|
+
});
|
|
876
|
+
assertModelResponseLatencyMode({
|
|
877
|
+
event: next.value,
|
|
878
|
+
requested: turnExecutionPolicy.latencyMode,
|
|
879
|
+
model: turn.model,
|
|
880
|
+
...(resolvedModel?.provider.id ? { providerId: resolvedModel.provider.id } : {}),
|
|
881
|
+
});
|
|
882
|
+
if (responseResult.status === "processed") {
|
|
883
|
+
const rawStreamHistory = (eventing.stream.state as { history?: unknown[] }).history;
|
|
884
|
+
if (Array.isArray(rawStreamHistory)) {
|
|
885
|
+
// The completed image item is normally retained from its own
|
|
886
|
+
// run-item event above. Scan once at the terminal response as
|
|
887
|
+
// a compatibility backstop before reconciliation can persist
|
|
888
|
+
// provider bytes. Never rescan history for every token delta.
|
|
889
|
+
await media.retainNativeGeneratedImagesFromHistory(
|
|
890
|
+
rawStreamHistory as Array<Record<string, unknown>>,
|
|
891
|
+
);
|
|
892
|
+
}
|
|
893
|
+
streamSawPerResponseUsage ||= responseResult.usageReported;
|
|
894
|
+
currentToolBatchCallIds = new Set<string>();
|
|
895
|
+
currentToolBatchCompletedCallIds = new Set<string>();
|
|
896
|
+
await historySink.reconcileConversationTruth();
|
|
897
|
+
turnLifecycleMetricsFor(observability).progress(attempt.turnId!);
|
|
898
|
+
modelCheckpointMemoryCollector.schedule(observability);
|
|
899
|
+
try {
|
|
900
|
+
await ensureRunAllowed(
|
|
901
|
+
settings,
|
|
902
|
+
db,
|
|
903
|
+
input.accountId,
|
|
904
|
+
input.workspaceId,
|
|
905
|
+
billingState.isExternallyBilledTurn,
|
|
906
|
+
entitlements,
|
|
907
|
+
);
|
|
908
|
+
} catch (limitError) {
|
|
909
|
+
// Capture the run state at the boundary so the budget valve in
|
|
910
|
+
// the outer catch can end this segment gracefully with full
|
|
911
|
+
// conversation context preserved for the post-top-up resume.
|
|
912
|
+
let serializedRunState: string | null = null;
|
|
913
|
+
try {
|
|
914
|
+
serializedRunState = media.compactMediaRunState(
|
|
915
|
+
String(eventing.stream.state.toString()),
|
|
916
|
+
);
|
|
917
|
+
} catch {
|
|
918
|
+
serializedRunState = null;
|
|
919
|
+
}
|
|
920
|
+
throw new BudgetExhaustedError(
|
|
921
|
+
limitError instanceof Error ? limitError.message : String(limitError),
|
|
922
|
+
serializedRunState,
|
|
923
|
+
);
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
const durableSdkEvent = generatedImageReceipt
|
|
927
|
+
? compactGeneratedImageSdkEvent(next.value, generatedImageReceipt)
|
|
928
|
+
: next.value;
|
|
929
|
+
const pendingToolCall = pendingToolCallFromSdkEvent(durableSdkEvent);
|
|
930
|
+
if (pendingToolCall) {
|
|
931
|
+
const registered = await registerPendingSessionToolCall(db, {
|
|
932
|
+
accountId: input.accountId,
|
|
933
|
+
workspaceId: input.workspaceId,
|
|
934
|
+
sessionId: input.sessionId,
|
|
935
|
+
turnId: activeTurnId,
|
|
936
|
+
executionGeneration: attempt.executionGeneration,
|
|
937
|
+
attemptId: input.attemptId,
|
|
938
|
+
modelToolOutputTruncationTokens:
|
|
939
|
+
eventing.modelRunSettings.modelToolOutputTruncationTokens,
|
|
940
|
+
callId: pendingToolCall.callId,
|
|
941
|
+
callType: pendingToolCall.callType,
|
|
942
|
+
callItem: pendingToolCall.callItem as Record<string, unknown>,
|
|
943
|
+
});
|
|
944
|
+
if (!registered.accepted) {
|
|
945
|
+
throw new TurnAttemptFencedError(
|
|
946
|
+
"turn attempt ended while recording an in-flight tool call",
|
|
947
|
+
);
|
|
948
|
+
}
|
|
949
|
+
currentToolBatchCallIds.add(pendingToolCall.callId);
|
|
950
|
+
if (toolCallProducesRetainableSessionImage(pendingToolCall.callName)) {
|
|
951
|
+
media.retainedSessionImageCallIds.add(pendingToolCall.callId);
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
const completedToolCall = completedToolCallFromSdkEvent(durableSdkEvent);
|
|
955
|
+
if (completedToolCall) {
|
|
956
|
+
retainedScreenshotMetadata =
|
|
957
|
+
media.retainedScreenshotReceiptsByCallId.get(completedToolCall.callId) ?? null;
|
|
958
|
+
const typedScreenshot = retainedScreenshotMetadata
|
|
959
|
+
? null
|
|
960
|
+
: typedScreenshotFromSdkEvent(durableSdkEvent);
|
|
961
|
+
if (
|
|
962
|
+
!retainedScreenshotMetadata &&
|
|
963
|
+
media.retainedSessionImageCallIds.has(completedToolCall.callId) &&
|
|
964
|
+
sdkEventContainsInlineImage(durableSdkEvent) &&
|
|
965
|
+
!typedScreenshot
|
|
966
|
+
) {
|
|
967
|
+
retainedScreenshotMetadata = unavailableRetainedSessionImage({
|
|
968
|
+
sessionId: input.sessionId,
|
|
969
|
+
turnId: activeTurnId,
|
|
970
|
+
attemptId: input.attemptId,
|
|
971
|
+
toolCallId: completedToolCall.callId,
|
|
972
|
+
toolOutputId: completedToolCall.callId,
|
|
973
|
+
reason: "unsupported",
|
|
974
|
+
});
|
|
975
|
+
media.retainedScreenshotReceiptsByCallId.set(
|
|
976
|
+
completedToolCall.callId,
|
|
977
|
+
retainedScreenshotMetadata,
|
|
978
|
+
);
|
|
979
|
+
}
|
|
980
|
+
if (
|
|
981
|
+
!retainedScreenshotMetadata &&
|
|
982
|
+
typedScreenshot &&
|
|
983
|
+
typedScreenshot.callId === completedToolCall.callId &&
|
|
984
|
+
media.retainedSessionImageCallIds.has(completedToolCall.callId)
|
|
985
|
+
) {
|
|
986
|
+
// Install a compact deterministic fallback before database or
|
|
987
|
+
// object-storage I/O. Unexpected retention failure can then
|
|
988
|
+
// reconcile/serialize without leaking inline bytes.
|
|
989
|
+
retainedScreenshotMetadata = unavailableRetainedSessionImage({
|
|
990
|
+
sessionId: input.sessionId,
|
|
991
|
+
turnId: activeTurnId,
|
|
992
|
+
attemptId: input.attemptId,
|
|
993
|
+
toolCallId: typedScreenshot.callId,
|
|
994
|
+
toolOutputId: typedScreenshot.toolOutputId,
|
|
995
|
+
reason: "pending",
|
|
996
|
+
});
|
|
997
|
+
media.retainedScreenshotReceiptsByCallId.set(
|
|
998
|
+
completedToolCall.callId,
|
|
999
|
+
retainedScreenshotMetadata,
|
|
1000
|
+
);
|
|
1001
|
+
retainedScreenshotMetadata = await retainComputerScreenshot({
|
|
1002
|
+
db,
|
|
1003
|
+
objectStorage,
|
|
1004
|
+
accountId: input.accountId,
|
|
1005
|
+
workspaceId: input.workspaceId,
|
|
1006
|
+
sessionId: input.sessionId,
|
|
1007
|
+
turnId: activeTurnId,
|
|
1008
|
+
attemptId: input.attemptId,
|
|
1009
|
+
output: typedScreenshot,
|
|
1010
|
+
});
|
|
1011
|
+
media.retainedScreenshotReceiptsByCallId.set(
|
|
1012
|
+
completedToolCall.callId,
|
|
1013
|
+
retainedScreenshotMetadata,
|
|
1014
|
+
);
|
|
1015
|
+
}
|
|
1016
|
+
normalizedSdkEvents = normalizeSdkEvent(
|
|
1017
|
+
durableSdkEvent as typeof next.value,
|
|
1018
|
+
retainedScreenshotMetadata
|
|
1019
|
+
? {
|
|
1020
|
+
toolOutputOverride: retainedScreenshotMetadata,
|
|
1021
|
+
retainedOutputEvidence: retainedScreenshotMetadata.available
|
|
1022
|
+
? retainedScreenshotMetadata
|
|
1023
|
+
: {
|
|
1024
|
+
available: false,
|
|
1025
|
+
reason: retainedScreenshotMetadata.reason,
|
|
1026
|
+
},
|
|
1027
|
+
}
|
|
1028
|
+
: {},
|
|
1029
|
+
);
|
|
1030
|
+
const normalizedToolOutput = normalizedSdkEvents.find(
|
|
1031
|
+
(event) =>
|
|
1032
|
+
event.type === "agent.toolCall.output" &&
|
|
1033
|
+
(event.payload as { id?: unknown }).id === completedToolCall.callId,
|
|
1034
|
+
)?.payload as { output?: unknown } | undefined;
|
|
1035
|
+
if (!normalizedToolOutput || !Object.hasOwn(normalizedToolOutput, "output")) {
|
|
1036
|
+
throw new Error(
|
|
1037
|
+
`Completed SDK tool call ${completedToolCall.callId} produced no durable output projection`,
|
|
1038
|
+
);
|
|
1039
|
+
}
|
|
1040
|
+
const durableResultItem = retainedScreenshotMetadata
|
|
1041
|
+
? (compactRetainedScreenshotHistory(
|
|
1042
|
+
[completedToolCall.resultItem],
|
|
1043
|
+
media.retainedScreenshotReceiptsByCallId,
|
|
1044
|
+
)[0] ?? completedToolCall.resultItem)
|
|
1045
|
+
: completedToolCall.resultItem;
|
|
1046
|
+
// Keep every parallel result in the attempt ledger until the full
|
|
1047
|
+
// call batch has settled. The SDK's computed history is
|
|
1048
|
+
// non-monotonic while parallel calls complete (a later call may
|
|
1049
|
+
// appear before an earlier persisted pair), so reconciling a
|
|
1050
|
+
// partial batch through a scalar watermark can create an orphan.
|
|
1051
|
+
const recorded = await recordPendingSessionToolCallResult(db, {
|
|
1052
|
+
accountId: input.accountId,
|
|
1053
|
+
workspaceId: input.workspaceId,
|
|
1054
|
+
sessionId: input.sessionId,
|
|
1055
|
+
turnId: activeTurnId,
|
|
1056
|
+
executionGeneration: attempt.executionGeneration,
|
|
1057
|
+
attemptId: input.attemptId,
|
|
1058
|
+
callId: completedToolCall.callId,
|
|
1059
|
+
modelToolOutputTruncationTokens:
|
|
1060
|
+
eventing.modelRunSettings.modelToolOutputTruncationTokens,
|
|
1061
|
+
resultItem: durableResultItem as Record<string, unknown>,
|
|
1062
|
+
eventOutput: normalizedToolOutput.output,
|
|
1063
|
+
...(videoGenerationAcceptancesByCallId.has(completedToolCall.callId)
|
|
1064
|
+
? {
|
|
1065
|
+
videoGenerationAcceptance: videoGenerationAcceptancesByCallId.get(
|
|
1066
|
+
completedToolCall.callId,
|
|
1067
|
+
)!,
|
|
1068
|
+
}
|
|
1069
|
+
: {}),
|
|
1070
|
+
});
|
|
1071
|
+
if (!recorded.accepted) {
|
|
1072
|
+
throw new TurnAttemptFencedError(
|
|
1073
|
+
"turn attempt ended while recording a tool-call result",
|
|
1074
|
+
);
|
|
1075
|
+
}
|
|
1076
|
+
const videoAcceptance = videoGenerationAcceptancesByCallId.get(completedToolCall.callId);
|
|
1077
|
+
if (videoAcceptance && startVideoGenerationWorkflow) {
|
|
1078
|
+
try {
|
|
1079
|
+
await startVideoGenerationWorkflow({
|
|
1080
|
+
accountId: input.accountId,
|
|
1081
|
+
workspaceId: input.workspaceId,
|
|
1082
|
+
operationId: videoAcceptance.operationId,
|
|
1083
|
+
});
|
|
1084
|
+
} catch (error) {
|
|
1085
|
+
// The accepted operation is durable. The recovery sweep starts
|
|
1086
|
+
// the same deterministic workflow ID if this nudge fails.
|
|
1087
|
+
observability.warn("Video generation workflow start deferred", {
|
|
1088
|
+
operationId: videoAcceptance.operationId,
|
|
1089
|
+
errorClass: error instanceof Error ? error.name : "UnknownError",
|
|
1090
|
+
});
|
|
1091
|
+
}
|
|
1092
|
+
}
|
|
1093
|
+
const belongsToCurrentBatch = currentToolBatchCallIds.has(completedToolCall.callId);
|
|
1094
|
+
if (belongsToCurrentBatch) {
|
|
1095
|
+
currentToolBatchCompletedCallIds.add(completedToolCall.callId);
|
|
1096
|
+
}
|
|
1097
|
+
const currentBatchIsStable =
|
|
1098
|
+
belongsToCurrentBatch &&
|
|
1099
|
+
currentToolBatchCallIds.size > 0 &&
|
|
1100
|
+
currentToolBatchCompletedCallIds.size === currentToolBatchCallIds.size;
|
|
1101
|
+
const standaloneStableResult =
|
|
1102
|
+
!belongsToCurrentBatch && currentToolBatchCallIds.size === 0;
|
|
1103
|
+
if (currentBatchIsStable || standaloneStableResult) {
|
|
1104
|
+
// Persist the SDK's now-stable complete call/result batch. Keep
|
|
1105
|
+
// the receipts until the normalized tool-output event below is
|
|
1106
|
+
// durably flushed: recovery then covers every crash boundary
|
|
1107
|
+
// without either losing or duplicating the UI projection.
|
|
1108
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
1109
|
+
stableToolCallIdsToClear = currentBatchIsStable
|
|
1110
|
+
? [...currentToolBatchCallIds]
|
|
1111
|
+
: [completedToolCall.callId];
|
|
1112
|
+
completedCurrentToolBatch = currentBatchIsStable;
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1115
|
+
const normalized =
|
|
1116
|
+
normalizedSdkEvents ??
|
|
1117
|
+
normalizeSdkEvent(
|
|
1118
|
+
durableSdkEvent as typeof next.value,
|
|
1119
|
+
retainedScreenshotMetadata
|
|
1120
|
+
? {
|
|
1121
|
+
toolOutputOverride: retainedScreenshotMetadata,
|
|
1122
|
+
retainedOutputEvidence: retainedScreenshotMetadata.available
|
|
1123
|
+
? retainedScreenshotMetadata
|
|
1124
|
+
: {
|
|
1125
|
+
available: false,
|
|
1126
|
+
reason: retainedScreenshotMetadata.reason,
|
|
1127
|
+
},
|
|
1128
|
+
}
|
|
1129
|
+
: {},
|
|
1130
|
+
);
|
|
1131
|
+
for (const event of normalized) {
|
|
1132
|
+
streamTiming.onEvent(event.type);
|
|
1133
|
+
await eventing.batcher.push(event);
|
|
1134
|
+
}
|
|
1135
|
+
// Structural tool-output events await their durable append before
|
|
1136
|
+
// push returns. The complete result is now retained in the event
|
|
1137
|
+
// row and pending-call recovery receipt, so release only our
|
|
1138
|
+
// duplicate live SDK marker before a long turn accumulates it.
|
|
1139
|
+
releaseMcpResultCustomDataFromSdkEvent(durableSdkEvent);
|
|
1140
|
+
if (stableToolCallIdsToClear) {
|
|
1141
|
+
const cleared = await clearDurablePendingSessionToolCalls(db, {
|
|
1142
|
+
accountId: input.accountId,
|
|
1143
|
+
workspaceId: input.workspaceId,
|
|
1144
|
+
sessionId: input.sessionId,
|
|
1145
|
+
turnId: activeTurnId,
|
|
1146
|
+
executionGeneration: attempt.executionGeneration,
|
|
1147
|
+
attemptId: input.attemptId,
|
|
1148
|
+
callIds: stableToolCallIdsToClear,
|
|
1149
|
+
});
|
|
1150
|
+
if (!cleared.accepted) {
|
|
1151
|
+
throw new TurnAttemptFencedError(
|
|
1152
|
+
"turn attempt ended while finalizing tool-call results",
|
|
1153
|
+
);
|
|
1154
|
+
}
|
|
1155
|
+
if (completedCurrentToolBatch) {
|
|
1156
|
+
currentToolBatchCallIds = new Set<string>();
|
|
1157
|
+
currentToolBatchCompletedCallIds = new Set<string>();
|
|
1158
|
+
}
|
|
1159
|
+
for (const callId of stableToolCallIdsToClear) {
|
|
1160
|
+
media.retainedSessionImageCallIds.delete(callId);
|
|
1161
|
+
}
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1164
|
+
} catch (error) {
|
|
1165
|
+
if (fallbackProviderRequestStartedAt !== null) {
|
|
1166
|
+
const durationMs = Math.max(0, performance.now() - fallbackProviderRequestStartedAt);
|
|
1167
|
+
fallbackProviderRequestStartedAt = null;
|
|
1168
|
+
await eventing.publish!([
|
|
1169
|
+
{
|
|
1170
|
+
type: "agent.model.request",
|
|
1171
|
+
payload: {
|
|
1172
|
+
phase: "failed",
|
|
1173
|
+
provider: streamProvider,
|
|
1174
|
+
durationMs: Math.round(durationMs),
|
|
1175
|
+
turnId: activeTurnId,
|
|
1176
|
+
attemptId: input.attemptId,
|
|
1177
|
+
dispatchId,
|
|
1178
|
+
executionGeneration: attempt.executionGeneration,
|
|
1179
|
+
},
|
|
1180
|
+
},
|
|
1181
|
+
]);
|
|
1182
|
+
}
|
|
1183
|
+
if (
|
|
1184
|
+
!eventing.firstModelRequestPreparationRecorded &&
|
|
1185
|
+
eventing.firstModelRequestPreparationStartedAt !== null
|
|
1186
|
+
) {
|
|
1187
|
+
eventing.firstModelRequestPreparationRecorded = true;
|
|
1188
|
+
const durationMs = Math.max(
|
|
1189
|
+
0,
|
|
1190
|
+
performance.now() - eventing.firstModelRequestPreparationStartedAt,
|
|
1191
|
+
);
|
|
1192
|
+
recordTurnStartupPhase(observability, {
|
|
1193
|
+
phase: "model_request_preparation",
|
|
1194
|
+
provider: turnExecutionPolicy.providerId,
|
|
1195
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
1196
|
+
outcome: "failed",
|
|
1197
|
+
durationSeconds: durationMs / 1_000,
|
|
1198
|
+
count: turnTools.length,
|
|
1199
|
+
});
|
|
1200
|
+
await eventing.publish!([
|
|
1201
|
+
{
|
|
1202
|
+
type: "turn.startup.phase.failed",
|
|
1203
|
+
payload: {
|
|
1204
|
+
phase: "model_preparation",
|
|
1205
|
+
durationMs: Math.round(durationMs),
|
|
1206
|
+
},
|
|
1207
|
+
},
|
|
1208
|
+
]);
|
|
1209
|
+
}
|
|
1210
|
+
throw error;
|
|
1211
|
+
} finally {
|
|
1212
|
+
if (!streamDone) {
|
|
1213
|
+
// ReadableStream cancellation synchronously trips the Agents SDK's
|
|
1214
|
+
// abort controller, but its returned promise may wait for an
|
|
1215
|
+
// uncooperative provider producer. Once this attempt is fenced,
|
|
1216
|
+
// awaiting that provider-side cleanup pins the Temporal activity
|
|
1217
|
+
// (and therefore Pause) even though every late write is already
|
|
1218
|
+
// rejected. Start cancellation and detach only its cleanup wait;
|
|
1219
|
+
// the SDK abort signal stops the producer and the durable attempt
|
|
1220
|
+
// fence remains the authority for every callback that arrives late.
|
|
1221
|
+
void iterator.return?.().catch(() => undefined);
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
await waitForTurnStreamCleanup(
|
|
1225
|
+
eventing.batcher.flush(),
|
|
1226
|
+
eventing.stream.completed.catch(() => undefined),
|
|
1227
|
+
cancellationSignal,
|
|
1228
|
+
);
|
|
1229
|
+
if (!streamSawPerResponseUsage) {
|
|
1230
|
+
const aggregateUsage = eventing.stream.state.usage;
|
|
1231
|
+
const normalizedAggregateUsage = normalizeModelCallUsage(aggregateUsage);
|
|
1232
|
+
const aggregateInput = normalizedAggregateUsage.telemetry.inputTokens;
|
|
1233
|
+
const aggregateSourceKey = modelUsageSourceKey({
|
|
1234
|
+
responseId: null,
|
|
1235
|
+
dispatchId: modelUsageDispatchId,
|
|
1236
|
+
positionalKey: "aggregate",
|
|
1237
|
+
});
|
|
1238
|
+
if (!claimedModelUsageSourceKeys.has(aggregateSourceKey)) {
|
|
1239
|
+
claimedModelUsageSourceKeys.add(aggregateSourceKey);
|
|
1240
|
+
// The aggregate frame is only a billing fallback when no terminal
|
|
1241
|
+
// response exposed usage. It is not final-request context authority.
|
|
1242
|
+
const aggregateAccountCtx = modelCallAccountContext({
|
|
1243
|
+
servingCredentialId: providerTurn.effectiveCodexCredentialId,
|
|
1244
|
+
priorSessionCredentialId: providerTurn.priorSessionCodexCredentialId,
|
|
1245
|
+
isFirstCallOfTurn: true,
|
|
1246
|
+
});
|
|
1247
|
+
let aggregateAuthoritative = false;
|
|
1248
|
+
await recordCompletedModelCallBeforeOwnershipFences({
|
|
1249
|
+
renewLease: () => leases.renewServing("model_usage"),
|
|
1250
|
+
leaseLost: leases.servingLost,
|
|
1251
|
+
leaseLostMessage: "Provider credential lease expired during the active turn",
|
|
1252
|
+
recordUsage: async () => {
|
|
1253
|
+
const billing = await recordModelUsageAndDebitCredits(settings, db, {
|
|
1254
|
+
accountId: input.accountId,
|
|
1255
|
+
workspaceId: input.workspaceId,
|
|
1256
|
+
sessionId: input.sessionId,
|
|
1257
|
+
turnId: activeTurnId,
|
|
1258
|
+
turnAttemptId: input.attemptId,
|
|
1259
|
+
model: turn.model,
|
|
1260
|
+
externallyBilled: billingState.isExternallyBilledTurn,
|
|
1261
|
+
usage: aggregateUsage,
|
|
1262
|
+
normalizedUsage: normalizedAggregateUsage,
|
|
1263
|
+
sourceKey: aggregateSourceKey,
|
|
1264
|
+
latencyMode: turnExecutionPolicy.latencyMode,
|
|
1265
|
+
observability,
|
|
1266
|
+
});
|
|
1267
|
+
const aggregateProvider = resolvedModel?.provider.id ?? settings.openaiProvider;
|
|
1268
|
+
const aggregateProviderApi = resolvedModel?.provider.api ?? "responses";
|
|
1269
|
+
aggregateAuthoritative = await emitModelCallUsage({
|
|
1270
|
+
observability,
|
|
1271
|
+
publish: eventing.publish,
|
|
1272
|
+
accountId: input.accountId,
|
|
1273
|
+
workspaceId: input.workspaceId,
|
|
1274
|
+
sessionId: input.sessionId,
|
|
1275
|
+
turnId: activeTurnId,
|
|
1276
|
+
provider: aggregateProvider,
|
|
1277
|
+
providerApi: aggregateProviderApi,
|
|
1278
|
+
model: turn.model,
|
|
1279
|
+
sourceKey: aggregateSourceKey,
|
|
1280
|
+
usage: { usage: aggregateUsage },
|
|
1281
|
+
normalizedUsage: normalizedAggregateUsage,
|
|
1282
|
+
servingAccountHash: aggregateAccountCtx.servingAccountHash,
|
|
1283
|
+
accountChangedFromPrevCall: aggregateAccountCtx.accountChangedFromPrevCall,
|
|
1284
|
+
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
1285
|
+
});
|
|
1286
|
+
if (aggregateAuthoritative && billing) {
|
|
1287
|
+
await recordAuthoritativeModelCallFact({
|
|
1288
|
+
db,
|
|
1289
|
+
observability,
|
|
1290
|
+
accountId: input.accountId,
|
|
1291
|
+
workspaceId: input.workspaceId,
|
|
1292
|
+
sessionId: input.sessionId,
|
|
1293
|
+
turnId: activeTurnId,
|
|
1294
|
+
turnAttemptId: input.attemptId,
|
|
1295
|
+
sourceKey: aggregateSourceKey,
|
|
1296
|
+
provider: aggregateProvider,
|
|
1297
|
+
providerApi: aggregateProviderApi,
|
|
1298
|
+
model: turn.model,
|
|
1299
|
+
billing,
|
|
1300
|
+
contextContributions: eventing.companyBrainContextContributions,
|
|
1301
|
+
});
|
|
1302
|
+
}
|
|
1303
|
+
if (aggregateAuthoritative && aggregateInput !== null && aggregateInput > 0) {
|
|
1304
|
+
recordModelInputTokens(observability, streamProvider, aggregateInput);
|
|
1305
|
+
}
|
|
1306
|
+
},
|
|
1307
|
+
recordAttemptSignals: async () => {
|
|
1308
|
+
if (!aggregateAuthoritative) return;
|
|
1309
|
+
// Stream aggregates can cover multiple requests and therefore
|
|
1310
|
+
// cannot identify the final request that compaction must bind.
|
|
1311
|
+
// They remain billing/telemetry fallback data only.
|
|
1312
|
+
modelResponseState.contextSignal = null;
|
|
1313
|
+
await setLastInputTokensFenced(null);
|
|
1314
|
+
},
|
|
1315
|
+
});
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
if (eventing.stream.interruptions.length > 0) {
|
|
1319
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
1320
|
+
const approvals = runtime.serializeApprovals(eventing.stream.interruptions);
|
|
1321
|
+
const humanInputInterruptions =
|
|
1322
|
+
runtime.serializeHumanInputRequests?.(eventing.stream.interruptions) ?? [];
|
|
1323
|
+
const interactionInterventionInterruptions =
|
|
1324
|
+
runtime.serializeInteractionInterventionRequests?.(eventing.stream.interruptions) ?? [];
|
|
1325
|
+
const latestWorkspace = await getWorkspace(db, input.workspaceId);
|
|
1326
|
+
if (!latestWorkspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
|
|
1327
|
+
assertWorkspaceHumanInputAllowed(
|
|
1328
|
+
resolveWorkspaceAgentHumanInputEnabled(latestWorkspace.settings),
|
|
1329
|
+
"interruption",
|
|
1330
|
+
humanInputInterruptions.length > 0,
|
|
1331
|
+
);
|
|
1332
|
+
const humanInputRequests = await Promise.all(
|
|
1333
|
+
humanInputInterruptions.map(async (interruption) => {
|
|
1334
|
+
const id = stableHumanInputRequestId(
|
|
1335
|
+
input.sessionId,
|
|
1336
|
+
activeTurnId,
|
|
1337
|
+
interruption.toolCallId,
|
|
1338
|
+
);
|
|
1339
|
+
const existing = await getSessionHumanInputRequest(
|
|
1340
|
+
db,
|
|
1341
|
+
input.workspaceId,
|
|
1342
|
+
input.sessionId,
|
|
1343
|
+
id,
|
|
1344
|
+
);
|
|
1345
|
+
if (existing && existing.status !== "pending") {
|
|
1346
|
+
throw new Error(`Settled human-input request ${id} reappeared as an interruption`);
|
|
1347
|
+
}
|
|
1348
|
+
const expiresAt = existing?.expiresAt
|
|
1349
|
+
? new Date(existing.expiresAt)
|
|
1350
|
+
: interruption.input.expiresInSeconds
|
|
1351
|
+
? new Date(Date.now() + interruption.input.expiresInSeconds * 1000)
|
|
1352
|
+
: null;
|
|
1353
|
+
return {
|
|
1354
|
+
id,
|
|
1355
|
+
toolCallId: interruption.toolCallId,
|
|
1356
|
+
questions: interruption.input.questions,
|
|
1357
|
+
allowSkip: interruption.input.allowSkip,
|
|
1358
|
+
expiresAt,
|
|
1359
|
+
isNew: existing === null,
|
|
1360
|
+
};
|
|
1361
|
+
}),
|
|
1362
|
+
);
|
|
1363
|
+
const requestEvents = humanInputRequests
|
|
1364
|
+
.filter((request) => request.isNew)
|
|
1365
|
+
.map((request) => ({
|
|
1366
|
+
type: "session.humanInput.requested" as const,
|
|
1367
|
+
payload: {
|
|
1368
|
+
request: {
|
|
1369
|
+
id: request.id,
|
|
1370
|
+
questions: request.questions,
|
|
1371
|
+
allowSkip: request.allowSkip,
|
|
1372
|
+
expiresAt: request.expiresAt?.toISOString() ?? null,
|
|
1373
|
+
},
|
|
1374
|
+
},
|
|
1375
|
+
}));
|
|
1376
|
+
const interactionInterventionRequests = interactionInterventionInterruptions.map(
|
|
1377
|
+
(interruption) => ({
|
|
1378
|
+
id: stableInteractionInterventionId(
|
|
1379
|
+
input.sessionId,
|
|
1380
|
+
activeTurnId,
|
|
1381
|
+
interruption.toolCallId,
|
|
1382
|
+
),
|
|
1383
|
+
operationId: stableInteractionInterventionOperationId(
|
|
1384
|
+
input.sessionId,
|
|
1385
|
+
activeTurnId,
|
|
1386
|
+
interruption.toolCallId,
|
|
1387
|
+
),
|
|
1388
|
+
toolCallId: interruption.toolCallId,
|
|
1389
|
+
input: interruption.input,
|
|
1390
|
+
}),
|
|
1391
|
+
);
|
|
1392
|
+
const pendingApprovals = [
|
|
1393
|
+
...approvals,
|
|
1394
|
+
...interactionInterventionInterruptions.map((interruption) => interruption.approval),
|
|
1395
|
+
];
|
|
1396
|
+
const suffixMembers = extractOpenSuffixFromRunState(eventing.stream.state);
|
|
1397
|
+
const interruptionCallIds = interruptionCallIdsFromPause({
|
|
1398
|
+
humanInputRequests,
|
|
1399
|
+
interactionInterventionRequests,
|
|
1400
|
+
pendingApprovals,
|
|
1401
|
+
});
|
|
1402
|
+
assertOpenSuffixResumable(suffixMembers, interruptionCallIds);
|
|
1403
|
+
const suffixByCallId = new Map(suffixMembers.map((member) => [member.callId, member]));
|
|
1404
|
+
const attached = await attachOpenSuffixToPendingToolCalls(db, {
|
|
1405
|
+
accountId: input.accountId,
|
|
1406
|
+
workspaceId: input.workspaceId,
|
|
1407
|
+
sessionId: input.sessionId,
|
|
1408
|
+
turnId: activeTurnId,
|
|
1409
|
+
executionGeneration: attempt.executionGeneration,
|
|
1410
|
+
attemptId: input.attemptId,
|
|
1411
|
+
members: interruptionCallIds.map((callId) => {
|
|
1412
|
+
const member = suffixByCallId.get(callId)!;
|
|
1413
|
+
return {
|
|
1414
|
+
callId,
|
|
1415
|
+
interruptionKind: interruptionKindForCallItem(
|
|
1416
|
+
member.callItem as Record<string, unknown>,
|
|
1417
|
+
),
|
|
1418
|
+
reasoningItems: member.reasoningItems as Array<Record<string, unknown>>,
|
|
1419
|
+
};
|
|
1420
|
+
}),
|
|
1421
|
+
});
|
|
1422
|
+
if (!attached.accepted) {
|
|
1423
|
+
return claimedResult({ status: "cancelled" });
|
|
1424
|
+
}
|
|
1425
|
+
if (
|
|
1426
|
+
!(await eventing.settle!({
|
|
1427
|
+
events: [
|
|
1428
|
+
...requestEvents,
|
|
1429
|
+
...(approvals.length > 0
|
|
1430
|
+
? [
|
|
1431
|
+
{
|
|
1432
|
+
type: "session.requiresAction" as const,
|
|
1433
|
+
payload: { approvals },
|
|
1434
|
+
},
|
|
1435
|
+
]
|
|
1436
|
+
: []),
|
|
1437
|
+
{
|
|
1438
|
+
type: "session.status.changed",
|
|
1439
|
+
payload: { status: "requires_action" },
|
|
1440
|
+
},
|
|
1441
|
+
],
|
|
1442
|
+
turnStatus: "requires_action",
|
|
1443
|
+
sessionStatus: "requires_action",
|
|
1444
|
+
activeTurnId,
|
|
1445
|
+
runState: {
|
|
1446
|
+
serializedRunState: OPEN_SUFFIX_RUN_STATE_BLOB,
|
|
1447
|
+
pendingApprovals,
|
|
1448
|
+
humanInputRequests: humanInputRequests.map(({ isNew: _isNew, ...request }) => request),
|
|
1449
|
+
interactionInterventionRequests,
|
|
1450
|
+
},
|
|
1451
|
+
}))
|
|
1452
|
+
) {
|
|
1453
|
+
return claimedResult({ status: "cancelled" });
|
|
1454
|
+
}
|
|
1455
|
+
// The interruption and its preceding tool results are now durable.
|
|
1456
|
+
await finalizeTurnOpStreamOps();
|
|
1457
|
+
control.activityStatus = "requires_action";
|
|
1458
|
+
return claimedResult({ status: "requires_action" });
|
|
1459
|
+
}
|
|
1460
|
+
|
|
1461
|
+
const finalOutput = String(eventing.stream.finalOutput ?? "");
|
|
1462
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
1463
|
+
// Op-stream durability fence: the tool outputs are now durably in the
|
|
1464
|
+
// history store (a redispatch would NOT re-execute them), so this
|
|
1465
|
+
// turn's settled ops may advance their acked frontier — journal persist
|
|
1466
|
+
// then wire final ack (licensing the runner to GC its retained
|
|
1467
|
+
// frames). Best-effort: a miss leaves the runner's retention TTL to
|
|
1468
|
+
// reap, never fails a completed turn.
|
|
1469
|
+
await finalizeTurnOpStreamOps();
|
|
1470
|
+
if (
|
|
1471
|
+
!(await eventing.settle!({
|
|
1472
|
+
events: [
|
|
1473
|
+
{
|
|
1474
|
+
type: "agent.message.completed",
|
|
1475
|
+
payload: { text: finalOutput },
|
|
1476
|
+
},
|
|
1477
|
+
{ type: "turn.completed", payload: { output: finalOutput } },
|
|
1478
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1479
|
+
],
|
|
1480
|
+
turnStatus: "completed",
|
|
1481
|
+
sessionStatus: "idle",
|
|
1482
|
+
activeTurnId: null,
|
|
1483
|
+
}))
|
|
1484
|
+
) {
|
|
1485
|
+
return claimedResult({ status: "cancelled" });
|
|
1486
|
+
}
|
|
1487
|
+
control.turnMetricOutcome = "completed";
|
|
1488
|
+
await recordUsageEvent(db, {
|
|
1489
|
+
accountId: input.accountId,
|
|
1490
|
+
workspaceId: input.workspaceId,
|
|
1491
|
+
eventType: "agent_run.completed",
|
|
1492
|
+
quantity: 1,
|
|
1493
|
+
unit: "run",
|
|
1494
|
+
sourceResourceType: "session_turn",
|
|
1495
|
+
sourceResourceId: activeTurnId,
|
|
1496
|
+
sessionId: input.sessionId,
|
|
1497
|
+
turnId: activeTurnId,
|
|
1498
|
+
turnAttemptId: input.attemptId,
|
|
1499
|
+
idempotencyKey: `usage:agent_run.completed:${activeTurnId}`,
|
|
1500
|
+
});
|
|
1501
|
+
control.activityStatus = "idle";
|
|
1502
|
+
return claimedResult({ status: "idle" });
|
|
1503
|
+
};
|
|
1504
|
+
|
|
1505
|
+
const openSuffixResume = await settleOpenSuffixResumeIfNeeded({
|
|
1506
|
+
db,
|
|
1507
|
+
agent,
|
|
1508
|
+
accountId: input.accountId,
|
|
1509
|
+
workspaceId: input.workspaceId,
|
|
1510
|
+
sessionId: input.sessionId,
|
|
1511
|
+
turnId: activeTurnId,
|
|
1512
|
+
executionGeneration: attempt.executionGeneration,
|
|
1513
|
+
attemptId: input.attemptId,
|
|
1514
|
+
trigger,
|
|
1515
|
+
humanInputResume,
|
|
1516
|
+
modelToolOutputTruncationTokens: eventing.modelRunSettings.modelToolOutputTruncationTokens,
|
|
1517
|
+
settle: eventing.settle!,
|
|
1518
|
+
publish: eventing.publish,
|
|
1519
|
+
});
|
|
1520
|
+
if (openSuffixResume.action === "cancelled") {
|
|
1521
|
+
return claimedResult({ status: "cancelled" });
|
|
1522
|
+
}
|
|
1523
|
+
if (openSuffixResume.action === "requires_action") {
|
|
1524
|
+
control.activityStatus = "requires_action";
|
|
1525
|
+
return claimedResult({ status: "requires_action" });
|
|
1526
|
+
}
|
|
1527
|
+
|
|
1528
|
+
await prepareRunAttemptInput();
|
|
1529
|
+
let retriedAfterCompaction = false;
|
|
1530
|
+
while (true) {
|
|
1531
|
+
try {
|
|
1532
|
+
const result = await runStreamAttempt();
|
|
1533
|
+
if (retriedAfterCompaction) {
|
|
1534
|
+
observability.info("context compaction recovery succeeded after in-activity retry", {
|
|
1535
|
+
sessionId: input.sessionId,
|
|
1536
|
+
turnId: activeTurnId,
|
|
1537
|
+
});
|
|
1538
|
+
}
|
|
1539
|
+
return result;
|
|
1540
|
+
} catch (attemptError) {
|
|
1541
|
+
const overflow = classifyContextWindowOverflowError(attemptError);
|
|
1542
|
+
const compactionNeeded = findCompactionNeededError(attemptError);
|
|
1543
|
+
const recoveryKind = compactionNeeded
|
|
1544
|
+
? compactionNeeded.trigger === "operator"
|
|
1545
|
+
? "operator"
|
|
1546
|
+
: "proactive"
|
|
1547
|
+
: overflow
|
|
1548
|
+
? "overflow"
|
|
1549
|
+
: null;
|
|
1550
|
+
if (!recoveryKind || !eventing.publish || !eventing.turnStartedPublished) {
|
|
1551
|
+
throw attemptError;
|
|
1552
|
+
}
|
|
1553
|
+
await flushRuntimeBatcher();
|
|
1554
|
+
await historySink.reconcileConversationTruth({ skipInputOnlyRows: true });
|
|
1555
|
+
observability.warn("context compaction recovery attempted", {
|
|
1556
|
+
sessionId: input.sessionId,
|
|
1557
|
+
turnId: activeTurnId,
|
|
1558
|
+
reason: recoveryKind,
|
|
1559
|
+
...safeErrorDiagnostic(attemptError),
|
|
1560
|
+
signalTokens: compactionNeeded?.signalTokens,
|
|
1561
|
+
thresholdTokens: compactionNeeded?.thresholdTokens,
|
|
1562
|
+
});
|
|
1563
|
+
let compacted = false;
|
|
1564
|
+
let compactionHandled = false;
|
|
1565
|
+
let compactionFailureMessage: string | null = null;
|
|
1566
|
+
let compactionRequestCleared = false;
|
|
1567
|
+
try {
|
|
1568
|
+
const outcome = await forceContextCompaction(
|
|
1569
|
+
recoveryKind,
|
|
1570
|
+
compactionNeeded?.signalTokens ?? null,
|
|
1571
|
+
);
|
|
1572
|
+
compacted = outcome.compacted;
|
|
1573
|
+
if (outcome.compacted) {
|
|
1574
|
+
compactionHandled = true;
|
|
1575
|
+
} else {
|
|
1576
|
+
compactionHandled = recoveryKind === "operator" && outcome.requestConsumed;
|
|
1577
|
+
if (!compactionHandled) {
|
|
1578
|
+
compactionFailureMessage = compactionFailureReason(outcome.reason);
|
|
1579
|
+
}
|
|
1580
|
+
}
|
|
1581
|
+
} catch (compactError) {
|
|
1582
|
+
// Transient checkpoint-provider failures recover this same accepted
|
|
1583
|
+
// turn through the normal provider/capacity path. They are not an
|
|
1584
|
+
// empty summary and must not create a new goal continuation.
|
|
1585
|
+
if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
|
|
1586
|
+
if (compactError instanceof TurnAttemptFencedError) throw compactError;
|
|
1587
|
+
const landmark = await settleFailedContextCompactionLandmark(
|
|
1588
|
+
db,
|
|
1589
|
+
{
|
|
1590
|
+
accountId: input.accountId,
|
|
1591
|
+
workspaceId: input.workspaceId,
|
|
1592
|
+
sessionId: input.sessionId,
|
|
1593
|
+
turnId: activeTurnId!,
|
|
1594
|
+
executionGeneration: attempt.executionGeneration,
|
|
1595
|
+
attemptId: input.attemptId,
|
|
1596
|
+
},
|
|
1597
|
+
{
|
|
1598
|
+
clearRequestedCompaction: recoveryKind === "operator",
|
|
1599
|
+
publishLiveEvents: publishCompactionLiveEvents,
|
|
1600
|
+
},
|
|
1601
|
+
);
|
|
1602
|
+
compactionRequestCleared = landmark.requestConsumed;
|
|
1603
|
+
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
1604
|
+
compactionFailureMessage = String(compactionFailureReasonFromError(compactError));
|
|
1605
|
+
observability.warn("context compaction recovery compaction failed", {
|
|
1606
|
+
sessionId: input.sessionId,
|
|
1607
|
+
turnId: activeTurnId,
|
|
1608
|
+
...safeErrorDiagnostic(compactError),
|
|
1609
|
+
});
|
|
1610
|
+
}
|
|
1611
|
+
if (!compactionHandled) {
|
|
1612
|
+
const errorMessage =
|
|
1613
|
+
compactionFailureMessage ??
|
|
1614
|
+
"compaction summarization failed: compaction produced no replacement history";
|
|
1615
|
+
if (
|
|
1616
|
+
!(await eventing.settle!({
|
|
1617
|
+
events: [
|
|
1618
|
+
{
|
|
1619
|
+
type: "turn.failed",
|
|
1620
|
+
payload: {
|
|
1621
|
+
error: errorMessage,
|
|
1622
|
+
code: "context_compaction_failed",
|
|
1623
|
+
retryable: false,
|
|
1624
|
+
recovery: "user_message",
|
|
1625
|
+
compacted: false,
|
|
1626
|
+
},
|
|
1627
|
+
},
|
|
1628
|
+
{
|
|
1629
|
+
type: "session.status.changed",
|
|
1630
|
+
payload: { status: "idle" },
|
|
1631
|
+
},
|
|
1632
|
+
],
|
|
1633
|
+
turnStatus: "failed",
|
|
1634
|
+
sessionStatus: "idle",
|
|
1635
|
+
activeTurnId: null,
|
|
1636
|
+
...(recoveryKind === "operator" && !compactionRequestCleared
|
|
1637
|
+
? { consumeRequestedCompactionFailure: true }
|
|
1638
|
+
: {}),
|
|
1639
|
+
}))
|
|
1640
|
+
) {
|
|
1641
|
+
return claimedResult({ status: "cancelled" });
|
|
1642
|
+
}
|
|
1643
|
+
control.turnMetricOutcome = "failed";
|
|
1644
|
+
control.activityStatus = "idle";
|
|
1645
|
+
control.activityError = attemptError;
|
|
1646
|
+
// The failed turn settlement already defers ordinary internal
|
|
1647
|
+
// updates and makes the delivered goal-continuation receipt
|
|
1648
|
+
// terminal. End this workflow run as well: returning plain idle
|
|
1649
|
+
// would immediately synthesize another goal continuation against
|
|
1650
|
+
// the unchanged active history and repeat the same failed
|
|
1651
|
+
// compaction. A new human/API prompt, Steer, or explicitly requested
|
|
1652
|
+
// Compact remains a durable explicit wake and may retry; ordinary
|
|
1653
|
+
// machine updates stay pending for that actionable wake.
|
|
1654
|
+
return claimedResult({ status: "idle", deferredUntilWake: true });
|
|
1655
|
+
}
|
|
1656
|
+
// Codex parity: compaction remains inside the same logical turn and
|
|
1657
|
+
// the same activity. Rebuild the model-visible history from the
|
|
1658
|
+
// durable replacement and continue the sampling loop; do not create
|
|
1659
|
+
// a recovery event, a queue row, a fake user message, or a sandbox.
|
|
1660
|
+
retriedAfterCompaction = true;
|
|
1661
|
+
observability.info("context compaction recovery retrying turn after compaction", {
|
|
1662
|
+
sessionId: input.sessionId,
|
|
1663
|
+
turnId: activeTurnId,
|
|
1664
|
+
reason: recoveryKind,
|
|
1665
|
+
compacted,
|
|
1666
|
+
});
|
|
1667
|
+
await prepareRunAttemptInput();
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
1670
|
+
}
|