@opengeni/worker-bundle 0.16.21 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +145 -32
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +5 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +3 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +11 -7
- package/dist/activities/expiring-material-renewal.d.ts +5 -3
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +2 -2
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +13 -26
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
- package/dist/activities/retained-screenshots.d.ts +129 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +2 -2
- package/dist/activities/run-input.d.ts +5 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +4 -4
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +71 -5
- package/dist/activities/video-generation-admission.d.ts +38 -0
- package/dist/activities/video-generation-credential.d.ts +7 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +49 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +11 -1
- package/dist/activities-control-44KBWLSM.js +6885 -0
- package/dist/activities-control-44KBWLSM.js.map +1 -0
- package/dist/activities-control.d.ts +289 -0
- package/dist/activities-turn-K4CEDEGJ.js +14675 -0
- package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +130 -147
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-E3RI3I33.js +828 -0
- package/dist/chunk-E3RI3I33.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-KKCRGAGZ.js +3091 -0
- package/dist/chunk-KKCRGAGZ.js.map +1 -0
- package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
- package/dist/chunk-RAMNYSAW.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/host-export.js +1 -1
- package/dist/index.d.ts +176 -10
- package/dist/index.js +675 -16460
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +22 -0
- package/dist/observability-metrics.d.ts +33 -0
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +40 -7
- package/dist/sandbox-routing.d.ts +13 -6
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1044 -363
- package/dist/workflows/activities.d.ts +7 -1
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +9 -7
- package/package.json +34 -15
- package/src/activities/agent-turn.ts +1905 -559
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +16 -7
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/codex-rotation.ts +19 -5
- package/src/activities/context-compaction.ts +47 -6
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +52 -60
- package/src/activities/expiring-material-renewal.ts +16 -7
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/git-credential-renewal.ts +3 -3
- package/src/activities/goals.ts +6 -2
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +55 -55
- package/src/activities/recording.ts +4 -5
- package/src/activities/retained-screenshot-reaper.ts +202 -0
- package/src/activities/retained-screenshots.ts +1317 -0
- package/src/activities/rig-verification.ts +567 -77
- package/src/activities/run-credential-renewal.ts +9 -2
- package/src/activities/run-input.ts +40 -10
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +96 -7
- package/src/activities/session-state.ts +4 -4
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +87 -5
- package/src/activities/video-generation-admission.ts +270 -0
- package/src/activities/video-generation-credential.ts +34 -0
- package/src/activities/video-generation-reconciliation.ts +578 -0
- package/src/activities/video-media-validation.ts +326 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +263 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +77 -13
- package/src/activities-control.ts +54 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +50 -118
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/host-export-pump.ts +28 -11
- package/src/http.ts +3 -6
- package/src/index.ts +277 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +116 -0
- package/src/observability-metrics.ts +202 -7
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +331 -157
- package/src/sandbox-routing.ts +49 -39
- package/src/worker-service-lifecycle.ts +82 -12
- package/src/workflows/activities.ts +30 -0
- package/src/workflows/file-upload-reaper.ts +6 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +17 -1
- package/dist/activities/redaction.d.ts +0 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/dist/chunk-BJUW5LCM.js.map +0 -1
- package/src/activities/redaction.ts +0 -9
- package/src/activities/toolspace-token-renewal.ts +0 -57
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
getHumanInputResumeForEvent,
|
|
19
19
|
getSessionHumanInputRequest,
|
|
20
20
|
installOrReadTurnExecutionPolicyForAttempt,
|
|
21
|
+
persistAttemptToolCatalog,
|
|
21
22
|
workspaceCodexSubscriptionActive,
|
|
22
23
|
acquireCodexCredentialLease,
|
|
23
24
|
armCodexCapacityWait,
|
|
@@ -27,17 +28,18 @@ import {
|
|
|
27
28
|
getCodexRotationSettings,
|
|
28
29
|
getWorkspaceModelPolicy,
|
|
29
30
|
getWorkspaceGrant,
|
|
31
|
+
getWorkspace,
|
|
30
32
|
listCodexAccountStatuses,
|
|
31
33
|
fetchCodexUsageForAccount,
|
|
32
34
|
getSessionCodexState,
|
|
33
35
|
recordSessionActiveCodexCredential,
|
|
34
|
-
|
|
36
|
+
setSessionCodexPinInTransaction,
|
|
35
37
|
recordCodexAccountUsageWithWakeTargets,
|
|
36
38
|
quarantineCodexCredentialForLease,
|
|
37
39
|
setActiveCodexCredential,
|
|
38
40
|
resolveWorkspaceMemoryBlock,
|
|
39
41
|
setCodexCredentialExhaustedWithWakeTargets,
|
|
40
|
-
|
|
42
|
+
withSessionCodexCapacityMutation,
|
|
41
43
|
countConsecutiveReactiveRotations,
|
|
42
44
|
requireSession,
|
|
43
45
|
recordUsageEvent,
|
|
@@ -47,8 +49,8 @@ import {
|
|
|
47
49
|
clearDurablePendingSessionToolCalls,
|
|
48
50
|
appendSessionHistoryItems,
|
|
49
51
|
isSessionCompactionRequested,
|
|
52
|
+
getActiveSessionHistoryItemsPaged,
|
|
50
53
|
countSessionHistoryItems,
|
|
51
|
-
getActiveSessionHistoryItems,
|
|
52
54
|
nextSessionHistoryPosition,
|
|
53
55
|
settleCodexCredentialLeaseLoss,
|
|
54
56
|
settleCodexCredentialFailover,
|
|
@@ -60,14 +62,22 @@ import {
|
|
|
60
62
|
accrueWarmSeconds,
|
|
61
63
|
getMaterializedSandboxFileResources,
|
|
62
64
|
markSandboxFileResourcesMaterialized,
|
|
65
|
+
getGeneratedVideoArtifact,
|
|
66
|
+
listSessionSystemUpdatesForTurn,
|
|
67
|
+
getWorkspaceVideoGenerationPolicy,
|
|
68
|
+
loadWorkspaceVercelAiGatewayCredentialLease,
|
|
63
69
|
areGitHubRepositoriesAllowedForWorkspace,
|
|
64
70
|
SandboxLeaseRecoveryBlockedError,
|
|
65
71
|
SandboxLeaseSupersededError,
|
|
72
|
+
SandboxLeaseTransitionError,
|
|
66
73
|
SandboxImageConflictError,
|
|
74
|
+
ActiveSessionHistoryLimitExceededError,
|
|
75
|
+
ApprovalRunStateLimitExceededError,
|
|
67
76
|
isSessionEventPersistenceError,
|
|
68
77
|
getEnrollment,
|
|
69
78
|
abandonRecordingForTurnAttempt,
|
|
70
|
-
|
|
79
|
+
commitSessionAttemptQuiescence,
|
|
80
|
+
getOrCreateCompanyProfileSnapshot,
|
|
71
81
|
getOrCreatePreferenceRegistrySnapshot,
|
|
72
82
|
getOrCreateWorkspaceInstructionPolicySnapshot,
|
|
73
83
|
PreferenceRegistryInitiatorError,
|
|
@@ -81,10 +91,12 @@ import {
|
|
|
81
91
|
type CodexCredentialLeaseResult,
|
|
82
92
|
type CodexCredentialLeaseSelectionContext,
|
|
83
93
|
type ApplySessionTurnSettlementInput,
|
|
94
|
+
type ApiIntegrationRuntime,
|
|
95
|
+
type SessionAttemptQuiescenceCommit,
|
|
84
96
|
type SessionTurnRecordingSettlement,
|
|
85
97
|
} from "@opengeni/db";
|
|
86
98
|
import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@opengeni/events";
|
|
87
|
-
import { sandboxOperationMetricObserver } from "@opengeni/observability";
|
|
99
|
+
import { sandboxLeaseTelemetryKey, sandboxOperationMetricObserver } from "@opengeni/observability";
|
|
88
100
|
import {
|
|
89
101
|
sandboxStateEntryFromRunState,
|
|
90
102
|
maxTurnsExceededRunState,
|
|
@@ -92,7 +104,9 @@ import {
|
|
|
92
104
|
modelTerminalResponseFromSdkEvent,
|
|
93
105
|
normalizeModelCallUsage,
|
|
94
106
|
normalizeSdkEvent,
|
|
107
|
+
normalizeProtocolJsonValue,
|
|
95
108
|
projectHistoryForProvider,
|
|
109
|
+
restoreGenericDispatchHistoryItems,
|
|
96
110
|
sanitizeHistoryItemsForModel,
|
|
97
111
|
projectModelInputForCapabilities,
|
|
98
112
|
appendPersistentSessionSettings,
|
|
@@ -102,7 +116,6 @@ import {
|
|
|
102
116
|
composeAgentInstructions,
|
|
103
117
|
hasActiveWorkspaceInstructionPolicy,
|
|
104
118
|
renderWorkspaceGovernanceContext,
|
|
105
|
-
summarizeForCompaction,
|
|
106
119
|
requestRemoteCompactionV2,
|
|
107
120
|
serializedToolsForRemoteCompaction,
|
|
108
121
|
REMOTE_COMPACTION_V2_BETA_FEATURE,
|
|
@@ -117,8 +130,8 @@ import {
|
|
|
117
130
|
clearRunCredentialsForAttempt,
|
|
118
131
|
withRunCredentialsSession,
|
|
119
132
|
refreshGitCredentialBindingTokenFiles,
|
|
120
|
-
|
|
121
|
-
|
|
133
|
+
refreshCodemodeTokenFile,
|
|
134
|
+
codemodeTokenFileFromEnvironment,
|
|
122
135
|
sandboxFileDownloadFailureNote,
|
|
123
136
|
SUMMARY_BUFFER_TOKENS,
|
|
124
137
|
isMcpRequestTimeoutError,
|
|
@@ -132,37 +145,43 @@ import {
|
|
|
132
145
|
type SandboxFileDownloadFailure,
|
|
133
146
|
type OpenGeniRuntime,
|
|
134
147
|
type ComputerToolMode,
|
|
148
|
+
type RetainableSessionImageOutputHook,
|
|
135
149
|
type ModelResponseUsage,
|
|
136
150
|
type ModelCallUsageInput,
|
|
137
151
|
type ModelCallUsageNormalization,
|
|
138
152
|
type BuildAgentOptions,
|
|
139
153
|
type ConnectorActionPolicyHooks,
|
|
140
|
-
type RunAgentStreamOptions,
|
|
141
154
|
type TurnToolCancellationFence,
|
|
142
155
|
type BackendUnresolvableCode,
|
|
143
156
|
type EstablishedSandboxSession,
|
|
144
157
|
type GitCredentialTokenWriterSession,
|
|
158
|
+
type LazyToolTransport,
|
|
145
159
|
type NormalizedRunCredentialMaterial,
|
|
146
160
|
type RunCredentialCommandSession,
|
|
147
|
-
type
|
|
161
|
+
type CodemodeTokenWriterSession,
|
|
162
|
+
createFirstPartyInteractionAttemptToolDefinitions,
|
|
148
163
|
deleteRecordingArtifacts,
|
|
149
164
|
stopRecording as stopRecordingOnBox,
|
|
150
165
|
} from "@opengeni/runtime";
|
|
151
166
|
import { connectionTokenResolverForTurn } from "./mcp-credentials";
|
|
167
|
+
import { buildApiIntegrationServersForTurn } from "./api-integrations";
|
|
152
168
|
import {
|
|
153
169
|
assertTurnExecutionPolicyMatchesConfigV1,
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
170
|
+
calculateGatewayReportedCostBreakdown,
|
|
171
|
+
calculateModelUsageCostBreakdown,
|
|
172
|
+
configuredModelPricingSchedules,
|
|
157
173
|
configuredStaticUsageLimits,
|
|
158
174
|
responseSatisfiesLatencyMode,
|
|
159
|
-
|
|
175
|
+
sandboxLifecycleTransitionWaitMs,
|
|
160
176
|
sandboxWarmRateMicrosPerSecond,
|
|
161
177
|
serviceTierForLatencyMode,
|
|
162
178
|
settingsWithResolvedModelContext,
|
|
163
179
|
resolveTurnExecutionPolicyV1,
|
|
164
180
|
OPENGENI_GATEWAY_MODELS,
|
|
165
181
|
OPENGENI_GATEWAY_PROVIDER_ID,
|
|
182
|
+
WORKSPACE_GATEWAY_PROVIDER_ID,
|
|
183
|
+
isDirectOpenAiApiBaseUrl,
|
|
184
|
+
resolveModelProvider,
|
|
166
185
|
type ModelUsageInput,
|
|
167
186
|
type ModelProviderApi,
|
|
168
187
|
type RegistryProviderKind,
|
|
@@ -195,6 +214,7 @@ import {
|
|
|
195
214
|
publishCodexFleetShadowDecisionV1,
|
|
196
215
|
} from "./codex-fleet-shadow";
|
|
197
216
|
import type { CodexAccountStatus } from "@opengeni/db";
|
|
217
|
+
import { CodemodeAttemptDispatcher } from "./codemode-dispatcher";
|
|
198
218
|
import { buildCodexTokenResolver } from "./codex-auth";
|
|
199
219
|
import {
|
|
200
220
|
refreshCodexUsageAndRepairCapacityWaiters,
|
|
@@ -223,18 +243,25 @@ import {
|
|
|
223
243
|
assertSessionAllowsProductModel,
|
|
224
244
|
defaultSessionMcpServerIds,
|
|
225
245
|
directPersonalConnectionSubjectId,
|
|
246
|
+
rigProviderImageContentHash,
|
|
226
247
|
resolveCodexAppsCredentialIdForRun,
|
|
227
248
|
withFrozenPersonalConnectionDelegations,
|
|
228
249
|
resolveSessionToolPolicy,
|
|
250
|
+
videoGenerationCapabilitiesForPolicy,
|
|
229
251
|
} from "@opengeni/core";
|
|
230
252
|
import { maybeCompactContext, settleFailedContextCompactionLandmark } from "./context-compaction";
|
|
231
253
|
import { TurnAttemptFencedError } from "./turn-attempt-fenced";
|
|
254
|
+
import {
|
|
255
|
+
admitVideoGenerationRequest,
|
|
256
|
+
managedVideoGenerationCredentialLease,
|
|
257
|
+
type VideoGenerationCredentialLease,
|
|
258
|
+
} from "./video-generation-admission";
|
|
232
259
|
import {
|
|
233
260
|
assertGitCredentialRenewalTransportUnchanged,
|
|
234
261
|
gitCredentialAuthorityForTurn,
|
|
235
262
|
gitHubTokenMintSelections,
|
|
236
263
|
loadWorkspaceEnvironmentForRunWithCredentials,
|
|
237
|
-
|
|
264
|
+
mintSandboxCodemodeToken,
|
|
238
265
|
mintRunGitCredentials,
|
|
239
266
|
mintRunGitCredentialBinding,
|
|
240
267
|
sandboxEnvironmentForRun,
|
|
@@ -251,10 +278,10 @@ import {
|
|
|
251
278
|
type RunCredentialRenewalController,
|
|
252
279
|
} from "./run-credential-renewal";
|
|
253
280
|
import {
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
type
|
|
257
|
-
} from "./
|
|
281
|
+
CODEMODE_TOKEN_EXPIRY_LEAD_MS,
|
|
282
|
+
startCodemodeTokenRenewalLoop,
|
|
283
|
+
type CodemodeTokenRenewalController,
|
|
284
|
+
} from "./codemode-token-renewal";
|
|
258
285
|
import {
|
|
259
286
|
bindRunCredentialResolver,
|
|
260
287
|
runCredentialAuthNeededPayloads,
|
|
@@ -263,19 +290,14 @@ import {
|
|
|
263
290
|
import { withFirstPartyTools } from "./goals";
|
|
264
291
|
import {
|
|
265
292
|
mergeRigDefaultVariableSetEnvironment,
|
|
266
|
-
|
|
293
|
+
rigProviderImageSourceImage,
|
|
267
294
|
resolveWorkspacePackRuntime,
|
|
268
295
|
resolveWorkspaceSkillLibraryRuntime,
|
|
269
296
|
settingsWithPackSandboxImage,
|
|
270
297
|
settingsWithRigImage,
|
|
298
|
+
settingsWithRigProviderImage,
|
|
271
299
|
} from "./packs";
|
|
272
300
|
import { deliverFailedChildTurnToParent } from "./parent-wake";
|
|
273
|
-
import {
|
|
274
|
-
createSecretRedactor,
|
|
275
|
-
isCredentialHeaderName,
|
|
276
|
-
redactSensitiveText,
|
|
277
|
-
type SecretForRedaction,
|
|
278
|
-
} from "./redaction";
|
|
279
301
|
import {
|
|
280
302
|
createModelHistoryAttachmentProjector,
|
|
281
303
|
turnInput,
|
|
@@ -288,7 +310,7 @@ import {
|
|
|
288
310
|
startActivityHeartbeat,
|
|
289
311
|
} from "./streaming";
|
|
290
312
|
import type {
|
|
291
|
-
ActivityServices,
|
|
313
|
+
TurnActivityServices as ActivityServices,
|
|
292
314
|
EscapedMcpTimeoutRecoveryDetail,
|
|
293
315
|
RunAgentTurnInput,
|
|
294
316
|
RunAgentTurnResult,
|
|
@@ -323,6 +345,8 @@ import {
|
|
|
323
345
|
recordCreditMicros,
|
|
324
346
|
recordModelCacheTokens,
|
|
325
347
|
recordModelInputTokens,
|
|
348
|
+
recordSessionEventAppendLatency,
|
|
349
|
+
recordSessionEventPublishLatency,
|
|
326
350
|
runtimeMetricsHooksForObservability,
|
|
327
351
|
StreamTimingMetrics,
|
|
328
352
|
turnLifecycleMetricsFor,
|
|
@@ -334,8 +358,43 @@ import {
|
|
|
334
358
|
prepareRecordingForSettlement,
|
|
335
359
|
type ActiveRecording,
|
|
336
360
|
} from "./recording";
|
|
361
|
+
import {
|
|
362
|
+
collectRetainedScreenshotReceipts,
|
|
363
|
+
collectRetainedScreenshotRunStateReceipts,
|
|
364
|
+
compactRetainedScreenshotHistory,
|
|
365
|
+
compactRetainedScreenshotRunState,
|
|
366
|
+
materializeRetainedScreenshotHistory,
|
|
367
|
+
materializeRetainedScreenshotRunState,
|
|
368
|
+
sdkEventContainsInlineImage,
|
|
369
|
+
retainComputerScreenshot,
|
|
370
|
+
toolOutputContainsInlineImage,
|
|
371
|
+
typedScreenshotFromSdkEvent,
|
|
372
|
+
typedScreenshotFromToolOutput,
|
|
373
|
+
unavailableRetainedSessionImage,
|
|
374
|
+
} from "./retained-screenshots";
|
|
375
|
+
import {
|
|
376
|
+
assertGeneratedImageHistoryRetained,
|
|
377
|
+
collectGeneratedImageReceipts,
|
|
378
|
+
collectGeneratedImageArtifactReceipts,
|
|
379
|
+
collectGeneratedImageRunStateArtifactReceipts,
|
|
380
|
+
compactGeneratedImageHistory,
|
|
381
|
+
compactGeneratedImageRunState,
|
|
382
|
+
compactGeneratedImageSdkEvent,
|
|
383
|
+
generatedImageFromSdkEvent,
|
|
384
|
+
generatedImagesFromHistory,
|
|
385
|
+
isCompletedGeneratedImageSdkEvent,
|
|
386
|
+
projectGeneratedImageHistoryForModel,
|
|
387
|
+
projectGeneratedImageRunStateForModel,
|
|
388
|
+
retainGeneratedImage,
|
|
389
|
+
type GeneratedImageReceipt,
|
|
390
|
+
type GeneratedImageOutput,
|
|
391
|
+
} from "./generated-images";
|
|
392
|
+
import { executeGatewayImageGeneration } from "./gateway-image-generation";
|
|
393
|
+
import { executeCodexImageGeneration } from "./codex-image-generation";
|
|
394
|
+
import { imageProviderBindingHash } from "./image-generation-operation";
|
|
395
|
+
import { resolveImageGenerationReferences } from "./image-generation-references";
|
|
337
396
|
import { captureWorkspaceRevision, openFreshWorkspaceCaptureSession } from "./workspace-capture";
|
|
338
|
-
import type
|
|
397
|
+
import { SandboxChannelAService, type ChannelASession } from "@opengeni/runtime/sandbox";
|
|
339
398
|
import { createObjectStorage, type ObjectStorage } from "@opengeni/storage";
|
|
340
399
|
import {
|
|
341
400
|
desktopCapableBackend,
|
|
@@ -347,9 +406,12 @@ import {
|
|
|
347
406
|
DEFAULT_FIRST_PARTY_MCP_TOOLS,
|
|
348
407
|
evaluateWorkspaceModelPolicy,
|
|
349
408
|
readTurnExecutionPolicyV1,
|
|
409
|
+
resolveWorkspaceAgentHumanInputEnabled,
|
|
350
410
|
resourceMountPath,
|
|
351
411
|
type LatencyMode,
|
|
352
412
|
type ResourceRef,
|
|
413
|
+
type RetainedArtifactMetadata,
|
|
414
|
+
type MediaGenerationResult,
|
|
353
415
|
type SessionEvent,
|
|
354
416
|
type SessionEventType,
|
|
355
417
|
type SessionStatus,
|
|
@@ -358,12 +420,28 @@ import {
|
|
|
358
420
|
type TurnExecutionPolicyV1,
|
|
359
421
|
} from "@opengeni/contracts";
|
|
360
422
|
import { createHash, randomUUID } from "node:crypto";
|
|
423
|
+
import { createModelCheckpointMemoryCollector } from "../model-checkpoint-memory-collector";
|
|
361
424
|
|
|
362
425
|
// Retryable provider connectivity/5xx failures start quickly and back off to
|
|
363
426
|
// this ceiling. Explicit rate limits retain the minute-granular fallback.
|
|
364
427
|
export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
|
|
365
428
|
const PROVIDER_CONNECTIVITY_BACKOFF_MS = [2_000, 5_000, 15_000, 30_000, 60_000] as const;
|
|
366
429
|
|
|
430
|
+
export class WorkspaceHumanInputDisabledError extends Error {
|
|
431
|
+
constructor(kind: "resume" | "interruption") {
|
|
432
|
+
super(`Workspace policy rejects structured human-input ${kind}`);
|
|
433
|
+
this.name = "WorkspaceHumanInputDisabledError";
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
export function assertWorkspaceHumanInputAllowed(
|
|
438
|
+
enabled: boolean,
|
|
439
|
+
kind: "resume" | "interruption",
|
|
440
|
+
attempted: boolean,
|
|
441
|
+
): void {
|
|
442
|
+
if (!enabled && attempted) throw new WorkspaceHumanInputDisabledError(kind);
|
|
443
|
+
}
|
|
444
|
+
|
|
367
445
|
/** Broad personal lookup is allowed only for a direct human/API command. */
|
|
368
446
|
export function credentialSubjectIdForTurnInitiator(
|
|
369
447
|
turn: Pick<SessionTurn, "source" | "initiator" | "initiatorContext">,
|
|
@@ -707,18 +785,22 @@ const QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS = 5_000;
|
|
|
707
785
|
* state. The proof object never changes between attempts. */
|
|
708
786
|
export async function persistOrSignalSessionAttemptQuiescence(input: {
|
|
709
787
|
proof: SessionAttemptQuiescenceProof;
|
|
710
|
-
persistReceipt: () => Promise<
|
|
788
|
+
persistReceipt: () => Promise<SessionAttemptQuiescenceCommit>;
|
|
789
|
+
deliverWorkflowWake?: (
|
|
790
|
+
wake: NonNullable<SessionAttemptQuiescenceCommit["workflowWake"]>,
|
|
791
|
+
) => Promise<unknown>;
|
|
711
792
|
publishEvents: (events: SessionEvent[]) => Promise<unknown>;
|
|
712
793
|
signalProof: ActivityServices["signalSessionAttemptQuiesced"];
|
|
713
794
|
sleep?: (ms: number) => Promise<void>;
|
|
714
795
|
heartbeat?: (attempt: number, delayMs: number) => void;
|
|
715
796
|
onReceiptFailure?: (error: unknown) => void;
|
|
797
|
+
onWakeFailure?: (error: unknown) => void;
|
|
716
798
|
onPublishFailure?: (error: unknown) => void;
|
|
717
799
|
onSignalFailure?: (error: unknown, attempt: number, delayMs: number) => void;
|
|
718
800
|
}): Promise<"receipt" | "signal"> {
|
|
719
|
-
let
|
|
801
|
+
let receipt: SessionAttemptQuiescenceCommit;
|
|
720
802
|
try {
|
|
721
|
-
|
|
803
|
+
receipt = await input.persistReceipt();
|
|
722
804
|
} catch (receiptError) {
|
|
723
805
|
input.onReceiptFailure?.(receiptError);
|
|
724
806
|
if (!input.signalProof) {
|
|
@@ -748,8 +830,17 @@ export async function persistOrSignalSessionAttemptQuiescence(input: {
|
|
|
748
830
|
}
|
|
749
831
|
}
|
|
750
832
|
|
|
833
|
+
if (receipt.workflowWake && input.deliverWorkflowWake) {
|
|
834
|
+
try {
|
|
835
|
+
await input.deliverWorkflowWake(receipt.workflowWake);
|
|
836
|
+
} catch (wakeError) {
|
|
837
|
+
// The exact revision is already durable. Immediate delivery is latency
|
|
838
|
+
// optimization only; the outbox dispatcher remains the repair path.
|
|
839
|
+
input.onWakeFailure?.(wakeError);
|
|
840
|
+
}
|
|
841
|
+
}
|
|
751
842
|
try {
|
|
752
|
-
await input.publishEvents(events);
|
|
843
|
+
await input.publishEvents(receipt.events);
|
|
753
844
|
} catch (publishError) {
|
|
754
845
|
// Postgres already committed quiesced_at, the queue event, and the wake.
|
|
755
846
|
// NATS is live fanout only; never misclassify its failure as receipt loss.
|
|
@@ -769,7 +860,7 @@ export async function drainAttemptOwnedSandboxWriters(input: {
|
|
|
769
860
|
toolCancellationFence: Pick<TurnToolCancellationFence, "cancel" | "waitForQuiescence"> | null;
|
|
770
861
|
cancellationReason?: unknown;
|
|
771
862
|
gitCredentialRenewals: readonly Pick<GitCredentialRenewalController, "stop">[];
|
|
772
|
-
|
|
863
|
+
codemodeTokenRenewal: Pick<CodemodeTokenRenewalController, "stop"> | null;
|
|
773
864
|
runCredentialRenewal: Pick<RunCredentialRenewalController, "stop"> | null;
|
|
774
865
|
}): Promise<void> {
|
|
775
866
|
if (input.toolCancellationFence) {
|
|
@@ -779,10 +870,41 @@ export async function drainAttemptOwnedSandboxWriters(input: {
|
|
|
779
870
|
await input.toolCancellationFence.waitForQuiescence();
|
|
780
871
|
}
|
|
781
872
|
await Promise.all(input.gitCredentialRenewals.map(async (renewal) => await renewal.stop()));
|
|
782
|
-
await input.
|
|
873
|
+
await input.codemodeTokenRenewal?.stop();
|
|
783
874
|
await input.runCredentialRenewal?.stop();
|
|
784
875
|
}
|
|
785
876
|
|
|
877
|
+
/** Persist the exact turn's second-stage release only after the caller has
|
|
878
|
+
* crossed the physical writer boundary. This wait is intentionally independent
|
|
879
|
+
* of Temporal cancellation: it is a short idempotent DB transaction that closes
|
|
880
|
+
* the durable archive-capture fence. Bounded retries cover a rollout connection
|
|
881
|
+
* reset without turning ordinary finalizer housekeeping into lifecycle state. */
|
|
882
|
+
export async function releaseTurnSandboxAfterWriterDrain(
|
|
883
|
+
sandbox: Pick<ResumedTurnSandbox, "release">,
|
|
884
|
+
options: {
|
|
885
|
+
maxAttempts?: number;
|
|
886
|
+
retryDelayMs?: number;
|
|
887
|
+
wait?: (delayMs: number) => Promise<void>;
|
|
888
|
+
} = {},
|
|
889
|
+
): Promise<void> {
|
|
890
|
+
const maxAttempts = Math.max(1, options.maxAttempts ?? 3);
|
|
891
|
+
const retryDelayMs = Math.max(0, options.retryDelayMs ?? 50);
|
|
892
|
+
const wait = options.wait ?? sleep;
|
|
893
|
+
let lastError: unknown;
|
|
894
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
895
|
+
try {
|
|
896
|
+
await sandbox.release({ workspaceWritersQuiesced: true });
|
|
897
|
+
return;
|
|
898
|
+
} catch (error) {
|
|
899
|
+
lastError = error;
|
|
900
|
+
if (attempt < maxAttempts) {
|
|
901
|
+
await wait(retryDelayMs * attempt);
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
throw lastError;
|
|
906
|
+
}
|
|
907
|
+
|
|
786
908
|
/**
|
|
787
909
|
* Await a finalizer operation only while this Temporal activity still owns its
|
|
788
910
|
* execution window. Once Pause/Steer cancellation arrives, the operation keeps
|
|
@@ -831,6 +953,33 @@ export async function waitForTurnStreamCleanup(
|
|
|
831
953
|
await waitForTurnFinalizerStep(providerCompleted, signal);
|
|
832
954
|
}
|
|
833
955
|
|
|
956
|
+
/**
|
|
957
|
+
* Terminal settlement closes the active attempt before turn finalization. The
|
|
958
|
+
* ordinary workspace admission therefore correctly returns `attempt_fenced`
|
|
959
|
+
* for the final attempt-qualified credential deletion. Retry only that exact
|
|
960
|
+
* settled-attempt case directly; every other fence/status and the direct
|
|
961
|
+
* deletion itself remain fail-closed.
|
|
962
|
+
*/
|
|
963
|
+
export async function clearAttemptCredentialsWithSettledFence(input: {
|
|
964
|
+
activityStatus: RunAgentTurnResult["status"] | "unknown";
|
|
965
|
+
runWorkspaceFencedClear: () => Promise<void>;
|
|
966
|
+
clearExactAttempt: () => Promise<void>;
|
|
967
|
+
onSettledAttemptFence: () => void;
|
|
968
|
+
}): Promise<void> {
|
|
969
|
+
try {
|
|
970
|
+
await input.runWorkspaceFencedClear();
|
|
971
|
+
} catch (error) {
|
|
972
|
+
const settledAttemptFence =
|
|
973
|
+
error instanceof Error &&
|
|
974
|
+
error.name === "SandboxWorkspaceMutationFencedError" &&
|
|
975
|
+
(error as Error & { code?: unknown }).code === "attempt_fenced" &&
|
|
976
|
+
(input.activityStatus === "idle" || input.activityStatus === "failed");
|
|
977
|
+
if (!settledAttemptFence) throw error;
|
|
978
|
+
input.onSettledAttemptFence();
|
|
979
|
+
await input.clearExactAttempt();
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
|
|
834
983
|
function turnFinalizerCancellationSignal(
|
|
835
984
|
temporalSignal: AbortSignal | undefined,
|
|
836
985
|
activityStatus: RunAgentTurnResult["status"] | "unknown",
|
|
@@ -847,84 +996,94 @@ function compactionFailureReason(reason: string): string {
|
|
|
847
996
|
: `compaction summarization failed: ${reason}`;
|
|
848
997
|
}
|
|
849
998
|
|
|
999
|
+
export type MandatoryHistoryPersistenceStage = "history_append" | "sandbox_envelope";
|
|
1000
|
+
|
|
1001
|
+
/**
|
|
1002
|
+
* Preserve the exact storage failure for the permission-controlled turn event
|
|
1003
|
+
* while attaching only a stable, non-secret operation token to public worker
|
|
1004
|
+
* diagnostics. Mandatory history failures remain terminal and never inherit a
|
|
1005
|
+
* provider-retry classification from coincidental source text or status.
|
|
1006
|
+
*/
|
|
1007
|
+
export class MandatoryHistoryPersistenceError extends Error {
|
|
1008
|
+
readonly name = "MandatoryHistoryPersistenceError";
|
|
1009
|
+
readonly cause: unknown;
|
|
1010
|
+
|
|
1011
|
+
constructor(
|
|
1012
|
+
readonly stage: MandatoryHistoryPersistenceStage,
|
|
1013
|
+
cause: unknown,
|
|
1014
|
+
) {
|
|
1015
|
+
super(cause instanceof Error ? cause.message : String(cause));
|
|
1016
|
+
this.cause = cause;
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
export async function runMandatoryHistoryPersistenceStep<T>(
|
|
1021
|
+
stage: MandatoryHistoryPersistenceStage,
|
|
1022
|
+
operation: () => Promise<T>,
|
|
1023
|
+
): Promise<T> {
|
|
1024
|
+
try {
|
|
1025
|
+
return await operation();
|
|
1026
|
+
} catch (cause) {
|
|
1027
|
+
throw new MandatoryHistoryPersistenceError(stage, cause);
|
|
1028
|
+
}
|
|
1029
|
+
}
|
|
1030
|
+
|
|
850
1031
|
export type SafeErrorDiagnostic = {
|
|
851
|
-
|
|
852
|
-
|
|
1032
|
+
errorClass: "WorkerOperationError";
|
|
1033
|
+
errorCode: "worker_operation_failed";
|
|
853
1034
|
status?: number;
|
|
854
|
-
|
|
1035
|
+
origin: "worker";
|
|
1036
|
+
historyPersistenceStage?: MandatoryHistoryPersistenceStage;
|
|
855
1037
|
};
|
|
856
1038
|
|
|
857
1039
|
/**
|
|
858
1040
|
* Produce the only exception shape allowed in worker logs. It deliberately
|
|
859
|
-
* excludes stack, cause, response/request
|
|
860
|
-
* properties
|
|
1041
|
+
* excludes the arbitrary source message, stack, cause, response/request
|
|
1042
|
+
* bodies, and enumerable properties. Exact failure content belongs in the
|
|
1043
|
+
* permission-controlled session event, not stdout or telemetry.
|
|
861
1044
|
*/
|
|
862
|
-
export function safeErrorDiagnostic(
|
|
863
|
-
error: unknown,
|
|
864
|
-
redactText: (value: string) => string = redactSensitiveText,
|
|
865
|
-
): SafeErrorDiagnostic {
|
|
866
|
-
const rawName = error instanceof Error ? error.name : "Error";
|
|
867
|
-
const name = /^[A-Za-z][A-Za-z0-9_.-]{0,79}$/.test(rawName) ? rawName : "Error";
|
|
868
|
-
const rawMessage = error instanceof Error ? error.message : String(error);
|
|
1045
|
+
export function safeErrorDiagnostic(error: unknown): SafeErrorDiagnostic {
|
|
869
1046
|
const diagnostic: SafeErrorDiagnostic = {
|
|
870
|
-
|
|
871
|
-
|
|
1047
|
+
errorClass: "WorkerOperationError",
|
|
1048
|
+
errorCode: "worker_operation_failed",
|
|
1049
|
+
origin: "worker",
|
|
872
1050
|
};
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
if (Number.isInteger(status) && status >= 100 && status <= 599) {
|
|
879
|
-
diagnostic.status = status;
|
|
1051
|
+
try {
|
|
1052
|
+
let statusSource = error;
|
|
1053
|
+
if (error instanceof MandatoryHistoryPersistenceError) {
|
|
1054
|
+
diagnostic.historyPersistenceStage = error.stage;
|
|
1055
|
+
statusSource = error.cause;
|
|
880
1056
|
}
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
1057
|
+
if (statusSource && typeof statusSource === "object") {
|
|
1058
|
+
const status = Number(
|
|
1059
|
+
(statusSource as { status?: unknown; statusCode?: unknown }).status ??
|
|
1060
|
+
(statusSource as { statusCode?: unknown }).statusCode,
|
|
1061
|
+
);
|
|
1062
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) {
|
|
1063
|
+
diagnostic.status = status;
|
|
1064
|
+
}
|
|
885
1065
|
}
|
|
1066
|
+
} catch {
|
|
1067
|
+
// Public diagnostics are best-effort and must never replace the exact
|
|
1068
|
+
// internal worker failure.
|
|
886
1069
|
}
|
|
887
1070
|
return diagnostic;
|
|
888
1071
|
}
|
|
889
1072
|
|
|
890
|
-
function safeErrorForTelemetry(error: unknown
|
|
891
|
-
const diagnostic = safeErrorDiagnostic(error
|
|
892
|
-
const safe = new Error(
|
|
893
|
-
|
|
1073
|
+
function safeErrorForTelemetry(error: unknown): Error {
|
|
1074
|
+
const diagnostic = safeErrorDiagnostic(error);
|
|
1075
|
+
const safe = new Error("worker operation failed") as Error & {
|
|
1076
|
+
code?: string;
|
|
1077
|
+
status?: number;
|
|
1078
|
+
origin?: string;
|
|
1079
|
+
};
|
|
1080
|
+
safe.name = "WorkerOperationError";
|
|
1081
|
+
safe.code = diagnostic.errorCode;
|
|
1082
|
+
if (diagnostic.status !== undefined) safe.status = diagnostic.status;
|
|
1083
|
+
safe.origin = diagnostic.origin;
|
|
894
1084
|
return safe;
|
|
895
1085
|
}
|
|
896
1086
|
|
|
897
|
-
export function headerSecretRedactions(
|
|
898
|
-
prefix: string,
|
|
899
|
-
headers: Readonly<Record<string, string>> | undefined,
|
|
900
|
-
credentialHeaderNames?: Iterable<string>,
|
|
901
|
-
): SecretForRedaction[] {
|
|
902
|
-
const discovered: SecretForRedaction[] = [];
|
|
903
|
-
const explicitCredentialHeaders = credentialHeaderNames
|
|
904
|
-
? new Set([...credentialHeaderNames].map((name) => name.toLowerCase()))
|
|
905
|
-
: null;
|
|
906
|
-
for (const [headerName, value] of Object.entries(headers ?? {})) {
|
|
907
|
-
if (
|
|
908
|
-
!explicitCredentialHeaders?.has(headerName.toLowerCase()) &&
|
|
909
|
-
!isCredentialHeaderName(headerName)
|
|
910
|
-
) {
|
|
911
|
-
continue;
|
|
912
|
-
}
|
|
913
|
-
const safeHeaderName = headerName.toUpperCase().replace(/[^A-Z0-9]+/g, "_");
|
|
914
|
-
discovered.push({ name: `${prefix}_${safeHeaderName || "HEADER"}`, value });
|
|
915
|
-
if (/^(?:proxy-)?authorization$/i.test(headerName)) {
|
|
916
|
-
const credential = value.match(/^\s*[A-Za-z][A-Za-z0-9_-]*\s+(.+)\s*$/)?.[1];
|
|
917
|
-
if (credential) {
|
|
918
|
-
discovered.push({
|
|
919
|
-
name: `${prefix}_${safeHeaderName || "AUTHORIZATION"}_CREDENTIAL`,
|
|
920
|
-
value: credential,
|
|
921
|
-
});
|
|
922
|
-
}
|
|
923
|
-
}
|
|
924
|
-
}
|
|
925
|
-
return discovered;
|
|
926
|
-
}
|
|
927
|
-
|
|
928
1087
|
function compactionFailureReasonFromError(error: unknown): string {
|
|
929
1088
|
if (
|
|
930
1089
|
error instanceof CompactionProviderResponseError ||
|
|
@@ -1044,14 +1203,14 @@ function collectErrorStrings(value: unknown, seen = new WeakSet<object>()): stri
|
|
|
1044
1203
|
* an earlier slice — the orphaned tool output that 400s the Responses API and
|
|
1045
1204
|
* bricks the session on every replay.
|
|
1046
1205
|
*
|
|
1047
|
-
* Defending:
|
|
1206
|
+
* Defending: structurally repair the full current history into an API-valid sequence (the
|
|
1048
1207
|
* same pure rules the read path uses), then append only the new tail beyond the
|
|
1049
1208
|
* watermark. A trailing dangling call is dropped here and re-evaluated next
|
|
1050
1209
|
* pass once its result lands, so a call and its result are written together at
|
|
1051
1210
|
* consecutive positions and a result is never persisted without its call. The
|
|
1052
|
-
* watermark advances to the
|
|
1211
|
+
* watermark advances to the repaired length — never past anything unwritten —
|
|
1053
1212
|
* so a non-monotonic history can never desync it. When previously-persisted
|
|
1054
|
-
* rows already exceed the
|
|
1213
|
+
* rows already exceed the repaired length (e.g. legacy orphans written before
|
|
1055
1214
|
* this fix), nothing new is appended and the watermark holds steady.
|
|
1056
1215
|
*/
|
|
1057
1216
|
/**
|
|
@@ -1518,14 +1677,9 @@ export async function emitModelCallUsage(input: {
|
|
|
1518
1677
|
if (!authoritative) return false;
|
|
1519
1678
|
try {
|
|
1520
1679
|
input.observability.info("model call usage", {
|
|
1521
|
-
accountId: input.accountId,
|
|
1522
|
-
workspaceId: input.workspaceId,
|
|
1523
|
-
sessionId: input.sessionId,
|
|
1524
|
-
turnId: input.turnId,
|
|
1525
1680
|
provider: input.provider,
|
|
1526
1681
|
providerApi: input.providerApi,
|
|
1527
1682
|
model: input.model,
|
|
1528
|
-
sourceKey: input.sourceKey,
|
|
1529
1683
|
inputTokens: telemetry.inputTokens,
|
|
1530
1684
|
outputTokens: telemetry.outputTokens,
|
|
1531
1685
|
cachedTokens: telemetry.cachedTokens,
|
|
@@ -1543,7 +1697,6 @@ export async function emitModelCallUsage(input: {
|
|
|
1543
1697
|
provider: input.provider,
|
|
1544
1698
|
providerApi: input.providerApi,
|
|
1545
1699
|
model: input.model,
|
|
1546
|
-
sourceKey: input.sourceKey,
|
|
1547
1700
|
rejectedFields: normalizedUsage.rejectedFields.join(","),
|
|
1548
1701
|
});
|
|
1549
1702
|
}
|
|
@@ -1590,16 +1743,13 @@ export function historyRowsToAppend(
|
|
|
1590
1743
|
// positions from 0) when callers do not pass an explicit next position.
|
|
1591
1744
|
nextPosition: number = persistedHistoryCount,
|
|
1592
1745
|
toolOutputTruncationTokens?: number,
|
|
1593
|
-
redactValue: (value: unknown) => unknown = (value) => value,
|
|
1594
1746
|
): {
|
|
1595
1747
|
rows: Array<{ position: number; item: Record<string, unknown> }>;
|
|
1596
1748
|
nextWatermark: number;
|
|
1597
1749
|
nextPosition: number;
|
|
1598
1750
|
} {
|
|
1599
|
-
const
|
|
1600
|
-
|
|
1601
|
-
);
|
|
1602
|
-
if (sanitized.length <= persistedHistoryCount) {
|
|
1751
|
+
const modelReady = sanitizeHistoryItemsForModel(rawHistory, toolOutputTruncationTokens);
|
|
1752
|
+
if (modelReady.length <= persistedHistoryCount) {
|
|
1603
1753
|
return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
|
|
1604
1754
|
}
|
|
1605
1755
|
// Canonical model-facing inputs (including machine-input batches) are
|
|
@@ -1609,16 +1759,22 @@ export function historyRowsToAppend(
|
|
|
1609
1759
|
// notes—must not accidentally become conversation memory during reconciliation.
|
|
1610
1760
|
// Advance the in-memory watermark past them, but only allocate durable
|
|
1611
1761
|
// positions to actual model/tool output.
|
|
1612
|
-
const rows =
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1762
|
+
const rows: Array<{ position: number; item: Record<string, unknown> }> = [];
|
|
1763
|
+
for (const [offset, item] of modelReady.slice(persistedHistoryCount).entries()) {
|
|
1764
|
+
if (item.type === "message" && item.role === "system") {
|
|
1765
|
+
continue;
|
|
1766
|
+
}
|
|
1767
|
+
rows.push({
|
|
1768
|
+
position: nextPosition + rows.length,
|
|
1769
|
+
item: normalizeProtocolJsonValue(
|
|
1770
|
+
item as Record<string, unknown>,
|
|
1771
|
+
`$[${persistedHistoryCount + offset}]`,
|
|
1772
|
+
),
|
|
1773
|
+
});
|
|
1774
|
+
}
|
|
1619
1775
|
return {
|
|
1620
1776
|
rows,
|
|
1621
|
-
nextWatermark:
|
|
1777
|
+
nextWatermark: modelReady.length,
|
|
1622
1778
|
nextPosition: nextPosition + rows.length,
|
|
1623
1779
|
};
|
|
1624
1780
|
}
|
|
@@ -1660,16 +1816,7 @@ export function selectRejectedProviderArtifactHistoryIds(
|
|
|
1660
1816
|
}
|
|
1661
1817
|
|
|
1662
1818
|
/** Literal final per-call boundary for replayed and current-turn model input. */
|
|
1663
|
-
export function
|
|
1664
|
-
redactValue: (value: unknown) => unknown,
|
|
1665
|
-
): NonNullable<RunAgentStreamOptions["callModelInputFilter"]> {
|
|
1666
|
-
return ({ modelData }) => ({
|
|
1667
|
-
...modelData,
|
|
1668
|
-
input: redactValue(modelData.input) as typeof modelData.input,
|
|
1669
|
-
});
|
|
1670
|
-
}
|
|
1671
|
-
|
|
1672
|
-
function isModelOrToolProgressHistoryItem(item: Record<string, unknown>): boolean {
|
|
1819
|
+
export function isModelOrToolProgressHistoryItem(item: Record<string, unknown>): boolean {
|
|
1673
1820
|
if (item.type === "message") {
|
|
1674
1821
|
return item.role === "assistant";
|
|
1675
1822
|
}
|
|
@@ -1894,6 +2041,65 @@ export async function ensureTurnModalRegistryImage(
|
|
|
1894
2041
|
await ensureRegistryImage(runSettings);
|
|
1895
2042
|
}
|
|
1896
2043
|
|
|
2044
|
+
const SANDBOX_ARTIFACT_RUNTIME_MANIFEST = "/opt/opengeni/artifact-runtime/installation.json";
|
|
2045
|
+
const SANDBOX_ARTIFACT_TOOL_ENTRY = "/opt/opengeni/artifact-runtime/skill-facade-entry.mjs";
|
|
2046
|
+
|
|
2047
|
+
export type SandboxArtifactRuntimeAdmission = Readonly<{
|
|
2048
|
+
available: boolean;
|
|
2049
|
+
environment: Readonly<Record<string, string>>;
|
|
2050
|
+
}>;
|
|
2051
|
+
|
|
2052
|
+
/**
|
|
2053
|
+
* Admit the optional native standalone-file runtime only for the deployment's
|
|
2054
|
+
* exact base sandbox image contract. A pack/rig image override is an
|
|
2055
|
+
* independent filesystem and therefore fails closed even when the deployment
|
|
2056
|
+
* base image is capable. Collaborative artifact skills are admitted separately
|
|
2057
|
+
* from the frozen canonical tool catalog.
|
|
2058
|
+
*
|
|
2059
|
+
* This keeps lazy provisioning intact: CI/release proves the image closure,
|
|
2060
|
+
* while a before-agent-start doctor verifies the actual box before any model
|
|
2061
|
+
* call. No speculative sandbox is created merely to populate the skill index.
|
|
2062
|
+
*/
|
|
2063
|
+
export function sandboxArtifactRuntimeAdmission(
|
|
2064
|
+
deploymentSettings: Settings,
|
|
2065
|
+
runSettings: Settings,
|
|
2066
|
+
backend: Settings["sandboxBackend"] | undefined,
|
|
2067
|
+
options: Readonly<{ production?: boolean }> = {},
|
|
2068
|
+
): SandboxArtifactRuntimeAdmission {
|
|
2069
|
+
if (!deploymentSettings.sandboxArtifactRuntimeEnabled) {
|
|
2070
|
+
return { available: false, environment: {} };
|
|
2071
|
+
}
|
|
2072
|
+
const production = options.production ?? process.env.NODE_ENV === "production";
|
|
2073
|
+
if (backend === "docker") {
|
|
2074
|
+
const image = deploymentSettings.dockerImage;
|
|
2075
|
+
const localDevelopmentImage = /^opengeni-sandbox:local-[0-9a-f]{12}$/u.test(image);
|
|
2076
|
+
if (
|
|
2077
|
+
runSettings.dockerImage !== image ||
|
|
2078
|
+
(!/@sha256:[0-9a-f]{64}$/u.test(image) && (production || !localDevelopmentImage))
|
|
2079
|
+
) {
|
|
2080
|
+
return { available: false, environment: {} };
|
|
2081
|
+
}
|
|
2082
|
+
} else if (backend === "modal") {
|
|
2083
|
+
if (
|
|
2084
|
+
!deploymentSettings.modalImageRef ||
|
|
2085
|
+
!/@sha256:[0-9a-f]{64}$/u.test(deploymentSettings.modalImageRef) ||
|
|
2086
|
+
runSettings.modalImageRef !== deploymentSettings.modalImageRef ||
|
|
2087
|
+
runSettings.modalImageId !== deploymentSettings.modalImageId
|
|
2088
|
+
) {
|
|
2089
|
+
return { available: false, environment: {} };
|
|
2090
|
+
}
|
|
2091
|
+
} else {
|
|
2092
|
+
return { available: false, environment: {} };
|
|
2093
|
+
}
|
|
2094
|
+
return {
|
|
2095
|
+
available: true,
|
|
2096
|
+
environment: {
|
|
2097
|
+
OPENGENI_ARTIFACT_RUNTIME_MANIFEST: SANDBOX_ARTIFACT_RUNTIME_MANIFEST,
|
|
2098
|
+
OPENGENI_ARTIFACT_TOOL_ENTRY: SANDBOX_ARTIFACT_TOOL_ENTRY,
|
|
2099
|
+
},
|
|
2100
|
+
};
|
|
2101
|
+
}
|
|
2102
|
+
|
|
1897
2103
|
/**
|
|
1898
2104
|
* Decide whether the first actual computer action may start a proof recording.
|
|
1899
2105
|
*
|
|
@@ -1993,6 +2199,36 @@ export function structuredToolTransportForTurn(
|
|
|
1993
2199
|
);
|
|
1994
2200
|
}
|
|
1995
2201
|
|
|
2202
|
+
/**
|
|
2203
|
+
* Progressive tool disclosure is universal for supported OpenGeni turns; only
|
|
2204
|
+
* its contained transport differs. Codex keeps its native path, built-in direct
|
|
2205
|
+
* OpenAI/Azure Responses use native client tool search, and every other ordinary
|
|
2206
|
+
* function-calling provider uses OpenGeni's stable search/invoke dispatcher.
|
|
2207
|
+
*/
|
|
2208
|
+
export function lazyToolTransportForTurn(
|
|
2209
|
+
resolvedModel: {
|
|
2210
|
+
provider: {
|
|
2211
|
+
id: string;
|
|
2212
|
+
kind: RegistryProviderKind;
|
|
2213
|
+
api: ModelProviderApi;
|
|
2214
|
+
builtin: boolean;
|
|
2215
|
+
baseUrl?: string | undefined;
|
|
2216
|
+
};
|
|
2217
|
+
} | null,
|
|
2218
|
+
): LazyToolTransport {
|
|
2219
|
+
if (!resolvedModel) return "openai_native";
|
|
2220
|
+
const provider = resolvedModel.provider;
|
|
2221
|
+
if (provider.kind === "codex-subscription") return "codex_native";
|
|
2222
|
+
const isDirectBuiltinResponses =
|
|
2223
|
+
provider.builtin &&
|
|
2224
|
+
provider.api === "responses" &&
|
|
2225
|
+
(provider.id === "azure" || (provider.id === "openai" && provider.baseUrl === undefined));
|
|
2226
|
+
if (isDirectBuiltinResponses) {
|
|
2227
|
+
return "openai_native";
|
|
2228
|
+
}
|
|
2229
|
+
return "generic_dispatch";
|
|
2230
|
+
}
|
|
2231
|
+
|
|
1996
2232
|
/**
|
|
1997
2233
|
* Native web search is a runtime capability, not part of the session's MCP
|
|
1998
2234
|
* allow-list. Attach it whenever the resolved provider advertises runnable
|
|
@@ -2007,6 +2243,39 @@ export function hostedWebSearchForTurn(
|
|
|
2007
2243
|
return resolvedModel?.configured.hostedWebSearch ?? deploymentWebSearchEnabled;
|
|
2008
2244
|
}
|
|
2009
2245
|
|
|
2246
|
+
/** Direct OpenAI hosted image IDs are reusable only by the exact API credential. */
|
|
2247
|
+
export function openAiHostedImageProviderBindingForTurn(
|
|
2248
|
+
settings: Pick<Settings, "openaiProvider" | "openaiApiKey" | "openaiBaseUrl">,
|
|
2249
|
+
resolvedModel: {
|
|
2250
|
+
provider: {
|
|
2251
|
+
id: string;
|
|
2252
|
+
kind: RegistryProviderKind;
|
|
2253
|
+
builtin: boolean;
|
|
2254
|
+
baseUrl?: string | undefined;
|
|
2255
|
+
};
|
|
2256
|
+
configured: {
|
|
2257
|
+
capabilities: {
|
|
2258
|
+
hostedTools: { imageGeneration: { runnable: boolean } };
|
|
2259
|
+
};
|
|
2260
|
+
};
|
|
2261
|
+
} | null,
|
|
2262
|
+
): { providerId: string; providerBindingHash: string } | null {
|
|
2263
|
+
if (!resolvedModel?.configured.capabilities.hostedTools.imageGeneration.runnable) return null;
|
|
2264
|
+
const providerId = resolvedModel.provider.id;
|
|
2265
|
+
const isDirectOpenAi =
|
|
2266
|
+
resolvedModel.provider.builtin &&
|
|
2267
|
+
resolvedModel.provider.id === "openai" &&
|
|
2268
|
+
isDirectOpenAiApiBaseUrl(resolvedModel.provider.baseUrl) &&
|
|
2269
|
+
settings.openaiProvider === "openai" &&
|
|
2270
|
+
isDirectOpenAiApiBaseUrl(settings.openaiBaseUrl);
|
|
2271
|
+
const bindingIdentity = isDirectOpenAi ? settings.openaiApiKey : null;
|
|
2272
|
+
if (!bindingIdentity) return null;
|
|
2273
|
+
return {
|
|
2274
|
+
providerId,
|
|
2275
|
+
providerBindingHash: imageProviderBindingHash(providerId, bindingIdentity),
|
|
2276
|
+
};
|
|
2277
|
+
}
|
|
2278
|
+
|
|
2010
2279
|
/** Exact model-catalog modality gate; null preserves the legacy built-in path. */
|
|
2011
2280
|
export function modelSupportsImageInputForTurn(
|
|
2012
2281
|
resolvedModel: {
|
|
@@ -2032,7 +2301,10 @@ export function modelAttachmentInputPolicyForTurn(
|
|
|
2032
2301
|
resolvedModel: {
|
|
2033
2302
|
provider: { api: ModelProviderApi };
|
|
2034
2303
|
configured: {
|
|
2035
|
-
capabilities?: {
|
|
2304
|
+
capabilities?: {
|
|
2305
|
+
inputModalities: string[];
|
|
2306
|
+
inputFileMediaTypes?: string[];
|
|
2307
|
+
};
|
|
2036
2308
|
};
|
|
2037
2309
|
} | null,
|
|
2038
2310
|
): ModelAttachmentInputPolicy {
|
|
@@ -2159,27 +2431,30 @@ export function isLazySandboxProvisionRetryable(error: unknown): boolean {
|
|
|
2159
2431
|
if (error instanceof WorkspaceArchiveIntegrityError) {
|
|
2160
2432
|
return error.retryable;
|
|
2161
2433
|
}
|
|
2162
|
-
if (
|
|
2434
|
+
if (
|
|
2435
|
+
error instanceof SandboxLeaseSupersededError ||
|
|
2436
|
+
error instanceof SandboxLeaseTransitionError
|
|
2437
|
+
) {
|
|
2163
2438
|
return true;
|
|
2164
2439
|
}
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2440
|
+
if (error instanceof SandboxWarmingTimeoutError) {
|
|
2441
|
+
return false;
|
|
2442
|
+
}
|
|
2443
|
+
// Provider/transport text is not durable evidence that creation never
|
|
2444
|
+
// happened. Retrying an ambiguous unknown here can create a second box; only
|
|
2445
|
+
// typed, ownership-fenced lifecycle outcomes above are safe to replay.
|
|
2446
|
+
return false;
|
|
2169
2447
|
}
|
|
2170
2448
|
|
|
2171
|
-
/**
|
|
2172
|
-
*
|
|
2173
|
-
*
|
|
2174
|
-
* provider box, and committed the lease cold. Give that full sequence one
|
|
2175
|
-
* snapshot budget plus two schedule periods. This is workflow-visible pacing,
|
|
2176
|
-
* not an in-memory sleep, so Pause/Steer/new user intent can still wake the
|
|
2177
|
-
* durable session immediately.
|
|
2178
|
-
*/
|
|
2449
|
+
/** Short workflow-visible anti-churn pacing after a lifecycle transition. The
|
|
2450
|
+
* next acquire waits on the exact durable claim itself, so this is deliberately
|
|
2451
|
+
* not a guessed snapshot-plus-schedules completion time. */
|
|
2179
2452
|
export function sandboxDeadlineRotationRecoveryDelayMs(
|
|
2180
|
-
settings: Pick<Settings, "sandboxLeaseReaperPeriodMs"
|
|
2453
|
+
settings: Pick<Settings, "sandboxLeaseReaperPeriodMs">,
|
|
2181
2454
|
): number {
|
|
2182
|
-
|
|
2455
|
+
// The next attempt waits on the durable lifecycle claim itself. This delay is
|
|
2456
|
+
// only anti-churn pacing, not an estimate of snapshot + future schedules.
|
|
2457
|
+
return Math.min(5_000, settings.sandboxLeaseReaperPeriodMs);
|
|
2183
2458
|
}
|
|
2184
2459
|
|
|
2185
2460
|
export function createTurnSandboxProvisioner<T>(
|
|
@@ -2371,13 +2646,43 @@ export function codexCredentialLeaseDeadlineExpired(
|
|
|
2371
2646
|
);
|
|
2372
2647
|
}
|
|
2373
2648
|
|
|
2649
|
+
type OpStreamFinalizer = {
|
|
2650
|
+
finalizeOpStreamOps(): Promise<void>;
|
|
2651
|
+
};
|
|
2652
|
+
|
|
2653
|
+
/** Release retained Connected Machine output only after the turn is durable. */
|
|
2654
|
+
export async function finalizeDurableTurnOpStreams(
|
|
2655
|
+
sessions: readonly unknown[],
|
|
2656
|
+
fallback: OpStreamFinalizer | null,
|
|
2657
|
+
): Promise<void> {
|
|
2658
|
+
const candidates = new Set<OpStreamFinalizer>();
|
|
2659
|
+
for (const session of sessions) {
|
|
2660
|
+
const candidate = session as Partial<OpStreamFinalizer> | null;
|
|
2661
|
+
if (typeof candidate?.finalizeOpStreamOps === "function") {
|
|
2662
|
+
candidates.add(candidate as OpStreamFinalizer);
|
|
2663
|
+
}
|
|
2664
|
+
}
|
|
2665
|
+
if (candidates.size === 0 && fallback) {
|
|
2666
|
+
candidates.add(fallback);
|
|
2667
|
+
}
|
|
2668
|
+
for (const candidate of candidates) {
|
|
2669
|
+
try {
|
|
2670
|
+
await candidate.finalizeOpStreamOps();
|
|
2671
|
+
} catch {
|
|
2672
|
+
// The runner's retention TTL owns the fallback.
|
|
2673
|
+
}
|
|
2674
|
+
}
|
|
2675
|
+
}
|
|
2676
|
+
|
|
2374
2677
|
export function createRunAgentTurnActivity(services: () => Promise<ActivityServices>) {
|
|
2678
|
+
const modelCheckpointMemoryCollector = createModelCheckpointMemoryCollector();
|
|
2375
2679
|
return async function runAgentTurn(input: RunAgentTurnInput): Promise<RunAgentTurnResult> {
|
|
2376
2680
|
const {
|
|
2377
2681
|
settings,
|
|
2378
2682
|
db,
|
|
2379
2683
|
bus,
|
|
2380
2684
|
runtime,
|
|
2685
|
+
summarizeContextForCompaction,
|
|
2381
2686
|
objectStorage,
|
|
2382
2687
|
observability,
|
|
2383
2688
|
wakeSessionWorkflow,
|
|
@@ -2385,6 +2690,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2385
2690
|
signalCodexCapacityWorkflow,
|
|
2386
2691
|
entitlements,
|
|
2387
2692
|
connectionCredentials,
|
|
2693
|
+
startVideoGenerationWorkflow,
|
|
2388
2694
|
} = await services();
|
|
2389
2695
|
const activityContext = currentActivityContext();
|
|
2390
2696
|
const cancellationSignal = activityContext?.cancellationSignal;
|
|
@@ -2429,6 +2735,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2429
2735
|
acknowledgeQuiescence = true;
|
|
2430
2736
|
noteCancellationRequested();
|
|
2431
2737
|
};
|
|
2738
|
+
const acknowledgeRecoveryQuiescence = (): void => {
|
|
2739
|
+
// requestSessionTurnRecovery closed this exact attempt and durably
|
|
2740
|
+
// recorded the replacement cause. The activity still owns the physical
|
|
2741
|
+
// tool/credential boundary until finally drains it and publishes the
|
|
2742
|
+
// exact quiescence receipt; a workflow result alone is never that proof.
|
|
2743
|
+
acknowledgeQuiescence = true;
|
|
2744
|
+
noteCancellationRequested();
|
|
2745
|
+
};
|
|
2432
2746
|
let turnId: string | undefined;
|
|
2433
2747
|
let triggerEventId: string | undefined;
|
|
2434
2748
|
const claimedResult = (
|
|
@@ -2562,7 +2876,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2562
2876
|
workspaceId: input.workspaceId,
|
|
2563
2877
|
turnId,
|
|
2564
2878
|
reason,
|
|
2565
|
-
|
|
2879
|
+
...safeErrorDiagnostic(error),
|
|
2566
2880
|
});
|
|
2567
2881
|
observability.incrementCounter({
|
|
2568
2882
|
name: "opengeni_codex_lease_renewals_total",
|
|
@@ -2589,13 +2903,32 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2589
2903
|
// run. null when the flag is off (byte-for-byte the legacy build-and-discard
|
|
2590
2904
|
// path) OR when the backend is "none". Released + dropped in `finally`.
|
|
2591
2905
|
let resolvedSandbox: ResumedTurnSandbox | null = null;
|
|
2906
|
+
let attemptWritersDrained = false;
|
|
2907
|
+
const lateSandboxesAwaitingWriterDrain = new Set<ResumedTurnSandbox>();
|
|
2908
|
+
const releaseLateSandbox = async (sandbox: ResumedTurnSandbox): Promise<void> => {
|
|
2909
|
+
lateSandboxesAwaitingWriterDrain.add(sandbox);
|
|
2910
|
+
// Drop the holder/timer immediately, but keep its null-outcome admissions
|
|
2911
|
+
// fenced until the shared attempt writer drain completes. The staged
|
|
2912
|
+
// release serializes a later proof-bearing call behind this one.
|
|
2913
|
+
await sandbox.release().catch(() => undefined);
|
|
2914
|
+
if (!attemptWritersDrained) return;
|
|
2915
|
+
try {
|
|
2916
|
+
await releaseTurnSandboxAfterWriterDrain(sandbox);
|
|
2917
|
+
lateSandboxesAwaitingWriterDrain.delete(sandbox);
|
|
2918
|
+
} catch (error) {
|
|
2919
|
+
console.error(
|
|
2920
|
+
"late sandbox quiesced release failed (turn outcome unaffected)",
|
|
2921
|
+
safeErrorDiagnostic(error),
|
|
2922
|
+
);
|
|
2923
|
+
}
|
|
2924
|
+
};
|
|
2592
2925
|
const requireResolvedSandboxForMutation = (message: string): ResumedTurnSandbox => {
|
|
2593
2926
|
if (!resolvedSandbox) throw new Error(message);
|
|
2594
2927
|
return resolvedSandbox;
|
|
2595
2928
|
};
|
|
2596
2929
|
// The machine-primary SelfhostedSession (the UNWRAPPED backend, not the
|
|
2597
|
-
// routing proxy)
|
|
2598
|
-
//
|
|
2930
|
+
// routing proxy). Kept as a fallback finalizer; the routing proxy normally
|
|
2931
|
+
// aggregates it together with every machine reached after a mid-turn swap.
|
|
2599
2932
|
let machinePrimarySession: import("@opengeni/runtime").SelfhostedSession | null = null;
|
|
2600
2933
|
let lazyOwnedSandbox: EstablishedSandboxSession | null = null;
|
|
2601
2934
|
let turnSandboxProvisioner: TurnSandboxProvisioner<ResumedTurnSandbox> | null = null;
|
|
@@ -2604,6 +2937,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2604
2937
|
// THIS handle so a mid-turn sandbox_swap can never re-route those execs onto a
|
|
2605
2938
|
// connected machine (the user's real computer).
|
|
2606
2939
|
let setupBoxSession: unknown = null;
|
|
2940
|
+
const finalizeTurnOpStreamOps = async (): Promise<void> => {
|
|
2941
|
+
await finalizeDurableTurnOpStreams(
|
|
2942
|
+
[lazyOwnedSandbox?.session, resolvedSandbox?.established.session],
|
|
2943
|
+
machinePrimarySession,
|
|
2944
|
+
);
|
|
2945
|
+
};
|
|
2607
2946
|
// A same-target API repair can replace the home provider while this turn is
|
|
2608
2947
|
// alive. Keep setup/snapshot persistence on the rebound raw session while
|
|
2609
2948
|
// preserving the SDK-owned routing proxy for eager turns. Lazy turns hold the
|
|
@@ -2664,7 +3003,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2664
3003
|
expectedEpoch: sandbox.leaseEpoch,
|
|
2665
3004
|
expectedInstanceId: sandbox.established.instanceId,
|
|
2666
3005
|
operation,
|
|
2667
|
-
captureWaitMs:
|
|
3006
|
+
captureWaitMs: sandboxLifecycleTransitionWaitMs(settings),
|
|
3007
|
+
...(cancellationSignal ? { waitSignal: cancellationSignal } : {}),
|
|
2668
3008
|
};
|
|
2669
3009
|
const admission = await advanceWorkspaceGeneration(db, identity);
|
|
2670
3010
|
let result: T;
|
|
@@ -2721,11 +3061,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2721
3061
|
let runCredentialRenewal: RunCredentialRenewalController | null = null;
|
|
2722
3062
|
let runCredentialRenewalClosed = false;
|
|
2723
3063
|
let runCredentialSession: RunCredentialCommandSession | null = null;
|
|
2724
|
-
// The delegated
|
|
3064
|
+
// The delegated Codemode bearer has a one-hour TTL. Renewal is attempt-
|
|
2725
3065
|
// owned and attaches only after the initial token file reached a real
|
|
2726
3066
|
// sandbox session; finalization drains an in-flight replacement.
|
|
2727
|
-
let
|
|
2728
|
-
let
|
|
3067
|
+
let codemodeTokenRenewal: CodemodeTokenRenewalController | null = null;
|
|
3068
|
+
let codemodeTokenRenewalClosed = false;
|
|
2729
3069
|
// MID-SESSION snapshot single-flight guard: the heartbeat tick fires every
|
|
2730
3070
|
// 10s but a Modal filesystem snapshot can take longer — never overlap two
|
|
2731
3071
|
// captures on one box. The in-flight capture's promise is held so the
|
|
@@ -2779,6 +3119,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2779
3119
|
await current?.flush().catch(() => undefined);
|
|
2780
3120
|
};
|
|
2781
3121
|
let preparedTools: Awaited<ReturnType<OpenGeniRuntime["prepareTools"]>> | null = null;
|
|
3122
|
+
let codemodeDispatcher: CodemodeAttemptDispatcher | null = null;
|
|
2782
3123
|
const toolCancellationFenceRef: {
|
|
2783
3124
|
current: TurnToolCancellationFence | null;
|
|
2784
3125
|
} = {
|
|
@@ -2951,7 +3292,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2951
3292
|
sessionId: input.sessionId,
|
|
2952
3293
|
sandboxGroupId: heartbeatGroupId,
|
|
2953
3294
|
leaseEpoch: heartbeatEpoch,
|
|
2954
|
-
|
|
3295
|
+
...safeErrorDiagnostic(error),
|
|
2955
3296
|
});
|
|
2956
3297
|
})
|
|
2957
3298
|
.finally(() => {
|
|
@@ -3104,6 +3445,317 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3104
3445
|
// stable and this scalar append watermark is valid again. The sanitizer
|
|
3105
3446
|
// remains the final call/result pairing guard for every other reconcile.
|
|
3106
3447
|
let persistedHistoryCount = 0;
|
|
3448
|
+
const retainedScreenshotReceiptsByCallId = new Map<string, RetainedArtifactMetadata>();
|
|
3449
|
+
// Set from the resolved provider wire before any model history is prepared.
|
|
3450
|
+
// Text-only/chat wires keep compact receipts and must never allocate image
|
|
3451
|
+
// bytes merely for the following request-local projection to remove them.
|
|
3452
|
+
let modelCanReceiveRetainedSessionImages = true;
|
|
3453
|
+
const generatedImageReceiptsByProviderItemId = new Map<string, GeneratedImageReceipt>();
|
|
3454
|
+
const generatedImageReceiptsByArtifactId = new Map<string, GeneratedImageReceipt>();
|
|
3455
|
+
const videoGenerationAcceptancesByCallId = new Map<
|
|
3456
|
+
string,
|
|
3457
|
+
{ operationId: string; requestDigest: string }
|
|
3458
|
+
>();
|
|
3459
|
+
const requiredGeneratedVideoFiles: Array<{
|
|
3460
|
+
operationId: string;
|
|
3461
|
+
artifactId: string;
|
|
3462
|
+
fileId: string;
|
|
3463
|
+
objectKey: string;
|
|
3464
|
+
sizeBytes: number;
|
|
3465
|
+
sha256: string;
|
|
3466
|
+
filename: string;
|
|
3467
|
+
}> = [];
|
|
3468
|
+
let generatedImageMaterializationCache: {
|
|
3469
|
+
instanceId: string;
|
|
3470
|
+
fileIds: Set<string>;
|
|
3471
|
+
} | null = null;
|
|
3472
|
+
// Legacy ownership mode lets the Agents SDK create the sandbox inside
|
|
3473
|
+
// run(). Keep that exact session once the runtime exposes it so an image
|
|
3474
|
+
// generated later in the same model/tool loop can be copied immediately.
|
|
3475
|
+
let sdkOwnedSandboxSession: CodemodeTokenWriterSession | null = null;
|
|
3476
|
+
const prepareGeneratedImageDownload = async (receipt: GeneratedImageReceipt) => {
|
|
3477
|
+
if (!objectStorage) {
|
|
3478
|
+
throw new Error("Generated image sandbox materialization requires object storage");
|
|
3479
|
+
}
|
|
3480
|
+
const file = await requireFile(db, input.workspaceId, receipt.artifact.artifactId);
|
|
3481
|
+
const downloadStorage = objectStorageForSandboxDownloads(modelRunSettings, objectStorage);
|
|
3482
|
+
const signed = await downloadStorage.createGetUrl({
|
|
3483
|
+
key: file.objectKey,
|
|
3484
|
+
});
|
|
3485
|
+
return {
|
|
3486
|
+
file,
|
|
3487
|
+
download: {
|
|
3488
|
+
fileId: file.id,
|
|
3489
|
+
mountPath: "generated-images",
|
|
3490
|
+
filename: file.safeFilename,
|
|
3491
|
+
url: signed.url,
|
|
3492
|
+
expiresAt: signed.expiresAt,
|
|
3493
|
+
sizeBytes: file.sizeBytes,
|
|
3494
|
+
...(file.sha256 ? { sha256: file.sha256 } : {}),
|
|
3495
|
+
} satisfies SandboxFileDownload,
|
|
3496
|
+
};
|
|
3497
|
+
};
|
|
3498
|
+
const writeGeneratedImageDownload = async (
|
|
3499
|
+
sessionForDownload: CodemodeTokenWriterSession,
|
|
3500
|
+
download: SandboxFileDownload,
|
|
3501
|
+
): Promise<void> => {
|
|
3502
|
+
const runAs = sandboxRunAs(modelRunSettings);
|
|
3503
|
+
const materialized = await materializeSandboxFileDownloads(sessionForDownload, [download], {
|
|
3504
|
+
onRuntimeEvent: async (event) => {
|
|
3505
|
+
await publish?.([{ type: event.type, payload: event.payload }], true);
|
|
3506
|
+
},
|
|
3507
|
+
...(runAs ? { runAs } : {}),
|
|
3508
|
+
...(toolCancellationFenceRef.current
|
|
3509
|
+
? {
|
|
3510
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
3511
|
+
toolCancellationFenceRef.current,
|
|
3512
|
+
),
|
|
3513
|
+
}
|
|
3514
|
+
: {}),
|
|
3515
|
+
});
|
|
3516
|
+
if (materialized.failures.length > 0) {
|
|
3517
|
+
throw new Error(materialized.failures[0]!.reason);
|
|
3518
|
+
}
|
|
3519
|
+
};
|
|
3520
|
+
const warnGeneratedImageMaterializationDeferred = (error: unknown): void => {
|
|
3521
|
+
// Generation and permanent retention are already durable. Never turn a
|
|
3522
|
+
// transient sandbox copy failure into a replay of paid provider work;
|
|
3523
|
+
// an unmaterialized resource is retried on the next real sandbox.
|
|
3524
|
+
observability.warn("Generated image sandbox materialization deferred", {
|
|
3525
|
+
errorClass: error instanceof Error ? error.name : "UnknownError",
|
|
3526
|
+
errorCode: "generated_image_materialization_deferred",
|
|
3527
|
+
origin: "worker",
|
|
3528
|
+
});
|
|
3529
|
+
};
|
|
3530
|
+
const materializeGeneratedImageInSandbox = async (
|
|
3531
|
+
receipt: GeneratedImageReceipt,
|
|
3532
|
+
sandbox: ResumedTurnSandbox,
|
|
3533
|
+
sessionForDownload: unknown,
|
|
3534
|
+
): Promise<boolean> => {
|
|
3535
|
+
try {
|
|
3536
|
+
const { file, download } = await prepareGeneratedImageDownload(receipt);
|
|
3537
|
+
const cacheable = sandbox.established.backendId !== "selfhosted";
|
|
3538
|
+
if (generatedImageMaterializationCache?.instanceId !== sandbox.established.instanceId) {
|
|
3539
|
+
const fileIds =
|
|
3540
|
+
cacheable && sandboxGroupId
|
|
3541
|
+
? await getMaterializedSandboxFileResources(db, {
|
|
3542
|
+
accountId: input.accountId,
|
|
3543
|
+
workspaceId: input.workspaceId,
|
|
3544
|
+
sandboxGroupId,
|
|
3545
|
+
expectedEpoch: sandbox.leaseEpoch,
|
|
3546
|
+
instanceId: sandbox.established.instanceId,
|
|
3547
|
+
})
|
|
3548
|
+
: new Set<string>();
|
|
3549
|
+
generatedImageMaterializationCache = {
|
|
3550
|
+
instanceId: sandbox.established.instanceId,
|
|
3551
|
+
fileIds,
|
|
3552
|
+
};
|
|
3553
|
+
}
|
|
3554
|
+
if (generatedImageMaterializationCache.fileIds.has(file.id)) return true;
|
|
3555
|
+
await runWorkspaceMutationForSandbox(
|
|
3556
|
+
sandbox,
|
|
3557
|
+
"generatedImageMaterialization",
|
|
3558
|
+
async () =>
|
|
3559
|
+
await writeGeneratedImageDownload(
|
|
3560
|
+
sessionForDownload as CodemodeTokenWriterSession,
|
|
3561
|
+
download,
|
|
3562
|
+
),
|
|
3563
|
+
);
|
|
3564
|
+
generatedImageMaterializationCache.fileIds.add(file.id);
|
|
3565
|
+
if (cacheable && sandboxGroupId) {
|
|
3566
|
+
await markSandboxFileResourcesMaterialized(db, {
|
|
3567
|
+
accountId: input.accountId,
|
|
3568
|
+
workspaceId: input.workspaceId,
|
|
3569
|
+
sandboxGroupId,
|
|
3570
|
+
expectedEpoch: sandbox.leaseEpoch,
|
|
3571
|
+
instanceId: sandbox.established.instanceId,
|
|
3572
|
+
fileIds: [file.id],
|
|
3573
|
+
});
|
|
3574
|
+
}
|
|
3575
|
+
return true;
|
|
3576
|
+
} catch (error) {
|
|
3577
|
+
warnGeneratedImageMaterializationDeferred(error);
|
|
3578
|
+
return false;
|
|
3579
|
+
}
|
|
3580
|
+
};
|
|
3581
|
+
const materializeGeneratedImageInOwnedSdkSession = async (
|
|
3582
|
+
receipt: GeneratedImageReceipt,
|
|
3583
|
+
sessionForDownload: unknown,
|
|
3584
|
+
): Promise<boolean> => {
|
|
3585
|
+
try {
|
|
3586
|
+
const { download } = await prepareGeneratedImageDownload(receipt);
|
|
3587
|
+
await writeGeneratedImageDownload(
|
|
3588
|
+
sessionForDownload as CodemodeTokenWriterSession,
|
|
3589
|
+
download,
|
|
3590
|
+
);
|
|
3591
|
+
return true;
|
|
3592
|
+
} catch (error) {
|
|
3593
|
+
warnGeneratedImageMaterializationDeferred(error);
|
|
3594
|
+
return false;
|
|
3595
|
+
}
|
|
3596
|
+
};
|
|
3597
|
+
const materializeGeneratedImage = async (receipt: GeneratedImageReceipt): Promise<boolean> => {
|
|
3598
|
+
if (modelRunSettings.sandboxBackend === "none") return false;
|
|
3599
|
+
// Never allocate a cloud sandbox merely because durable history contains
|
|
3600
|
+
// an image. If this turn later needs one, its ready hook copies every
|
|
3601
|
+
// known receipt before releasing the first operation.
|
|
3602
|
+
if (resolvedSandbox && setupBoxSession) {
|
|
3603
|
+
return await materializeGeneratedImageInSandbox(receipt, resolvedSandbox, setupBoxSession);
|
|
3604
|
+
}
|
|
3605
|
+
if (sdkOwnedSandboxSession) {
|
|
3606
|
+
return await materializeGeneratedImageInOwnedSdkSession(receipt, sdkOwnedSandboxSession);
|
|
3607
|
+
}
|
|
3608
|
+
return false;
|
|
3609
|
+
};
|
|
3610
|
+
let nativeImageGenerationRetention: {
|
|
3611
|
+
providerId: string;
|
|
3612
|
+
providerBindingHash: string;
|
|
3613
|
+
sessionId: string;
|
|
3614
|
+
turnId: string;
|
|
3615
|
+
attemptId: string;
|
|
3616
|
+
} | null = null;
|
|
3617
|
+
const retainNativeGeneratedImage = async (
|
|
3618
|
+
output: GeneratedImageOutput,
|
|
3619
|
+
): Promise<GeneratedImageReceipt> => {
|
|
3620
|
+
if (output.providerItemId) {
|
|
3621
|
+
const existing = generatedImageReceiptsByProviderItemId.get(output.providerItemId);
|
|
3622
|
+
if (existing) return existing;
|
|
3623
|
+
}
|
|
3624
|
+
const context = nativeImageGenerationRetention;
|
|
3625
|
+
if (!context || !output.providerItemId) {
|
|
3626
|
+
throw new Error("Native generated image arrived without provider retention identity");
|
|
3627
|
+
}
|
|
3628
|
+
const retained = await retainGeneratedImage({
|
|
3629
|
+
db,
|
|
3630
|
+
objectStorage,
|
|
3631
|
+
accountId: input.accountId,
|
|
3632
|
+
workspaceId: input.workspaceId,
|
|
3633
|
+
sessionId: context.sessionId,
|
|
3634
|
+
turnId: context.turnId,
|
|
3635
|
+
attemptId: context.attemptId,
|
|
3636
|
+
sourceStrategy: "native_hosted",
|
|
3637
|
+
providerId: context.providerId,
|
|
3638
|
+
providerBindingHash: context.providerBindingHash,
|
|
3639
|
+
output,
|
|
3640
|
+
});
|
|
3641
|
+
generatedImageReceiptsByProviderItemId.set(output.providerItemId, retained.receipt);
|
|
3642
|
+
generatedImageReceiptsByArtifactId.set(
|
|
3643
|
+
retained.receipt.artifact.artifactId,
|
|
3644
|
+
retained.receipt,
|
|
3645
|
+
);
|
|
3646
|
+
await materializeGeneratedImage(retained.receipt);
|
|
3647
|
+
return retained.receipt;
|
|
3648
|
+
};
|
|
3649
|
+
const retainNativeGeneratedImagesFromHistory = async (
|
|
3650
|
+
history: Array<Record<string, unknown>>,
|
|
3651
|
+
): Promise<void> => {
|
|
3652
|
+
collectGeneratedImageReceipts(history, generatedImageReceiptsByProviderItemId);
|
|
3653
|
+
collectGeneratedImageArtifactReceipts(history, generatedImageReceiptsByArtifactId);
|
|
3654
|
+
for (const output of generatedImagesFromHistory(
|
|
3655
|
+
history,
|
|
3656
|
+
generatedImageReceiptsByProviderItemId,
|
|
3657
|
+
)) {
|
|
3658
|
+
if (
|
|
3659
|
+
!output.providerItemId ||
|
|
3660
|
+
generatedImageReceiptsByProviderItemId.has(output.providerItemId)
|
|
3661
|
+
) {
|
|
3662
|
+
continue;
|
|
3663
|
+
}
|
|
3664
|
+
await retainNativeGeneratedImage(output);
|
|
3665
|
+
}
|
|
3666
|
+
assertGeneratedImageHistoryRetained(history, generatedImageReceiptsByProviderItemId);
|
|
3667
|
+
};
|
|
3668
|
+
const compactMediaRunState = (serialized: string): string =>
|
|
3669
|
+
compactGeneratedImageRunState(
|
|
3670
|
+
compactRetainedScreenshotRunState(serialized, retainedScreenshotReceiptsByCallId),
|
|
3671
|
+
generatedImageReceiptsByProviderItemId,
|
|
3672
|
+
);
|
|
3673
|
+
// Explicit image-producing tools cross the durable session-media boundary.
|
|
3674
|
+
// Incidental frames returned by click/scroll actions remain unretained; an
|
|
3675
|
+
// intentional computer_screenshot or view_image result must never be lost
|
|
3676
|
+
// when the event/history sanitizer removes its inline bytes.
|
|
3677
|
+
const retainedSessionImageCallIds = new Set<string>();
|
|
3678
|
+
const retainSessionImageAtToolBoundary: RetainableSessionImageOutputHook = async ({
|
|
3679
|
+
toolCallId,
|
|
3680
|
+
output,
|
|
3681
|
+
}) => {
|
|
3682
|
+
const activeTurnId = turnId;
|
|
3683
|
+
if (!activeTurnId) {
|
|
3684
|
+
throw new Error("Session image tool completed before turn initialization");
|
|
3685
|
+
}
|
|
3686
|
+
const typedScreenshot = typedScreenshotFromToolOutput({
|
|
3687
|
+
callId: toolCallId,
|
|
3688
|
+
output,
|
|
3689
|
+
});
|
|
3690
|
+
retainedSessionImageCallIds.add(toolCallId);
|
|
3691
|
+
if (!typedScreenshot) {
|
|
3692
|
+
if (toolOutputContainsInlineImage(output)) {
|
|
3693
|
+
retainedScreenshotReceiptsByCallId.set(
|
|
3694
|
+
toolCallId,
|
|
3695
|
+
unavailableRetainedSessionImage({
|
|
3696
|
+
sessionId: input.sessionId,
|
|
3697
|
+
turnId: activeTurnId,
|
|
3698
|
+
attemptId: input.attemptId,
|
|
3699
|
+
toolCallId,
|
|
3700
|
+
toolOutputId: toolCallId,
|
|
3701
|
+
reason: "unsupported",
|
|
3702
|
+
}),
|
|
3703
|
+
);
|
|
3704
|
+
}
|
|
3705
|
+
return;
|
|
3706
|
+
}
|
|
3707
|
+
retainedScreenshotReceiptsByCallId.set(
|
|
3708
|
+
toolCallId,
|
|
3709
|
+
unavailableRetainedSessionImage({
|
|
3710
|
+
sessionId: input.sessionId,
|
|
3711
|
+
turnId: activeTurnId,
|
|
3712
|
+
attemptId: input.attemptId,
|
|
3713
|
+
toolCallId,
|
|
3714
|
+
toolOutputId: toolCallId,
|
|
3715
|
+
reason: "pending",
|
|
3716
|
+
}),
|
|
3717
|
+
);
|
|
3718
|
+
retainedScreenshotReceiptsByCallId.set(
|
|
3719
|
+
toolCallId,
|
|
3720
|
+
await retainComputerScreenshot({
|
|
3721
|
+
db,
|
|
3722
|
+
objectStorage,
|
|
3723
|
+
accountId: input.accountId,
|
|
3724
|
+
workspaceId: input.workspaceId,
|
|
3725
|
+
sessionId: input.sessionId,
|
|
3726
|
+
turnId: activeTurnId,
|
|
3727
|
+
attemptId: input.attemptId,
|
|
3728
|
+
output: typedScreenshot,
|
|
3729
|
+
}),
|
|
3730
|
+
);
|
|
3731
|
+
};
|
|
3732
|
+
const materializeScreenshotHistory = async (history: Array<Record<string, unknown>>) => {
|
|
3733
|
+
collectRetainedScreenshotReceipts(history, retainedScreenshotReceiptsByCallId);
|
|
3734
|
+
if (!modelCanReceiveRetainedSessionImages) return history;
|
|
3735
|
+
return await materializeRetainedScreenshotHistory({
|
|
3736
|
+
db,
|
|
3737
|
+
objectStorage,
|
|
3738
|
+
workspaceId: input.workspaceId,
|
|
3739
|
+
sessionId: input.sessionId,
|
|
3740
|
+
history,
|
|
3741
|
+
});
|
|
3742
|
+
};
|
|
3743
|
+
const materializeScreenshotRunState = async (serialized: string) => {
|
|
3744
|
+
collectRetainedScreenshotRunStateReceipts(serialized, retainedScreenshotReceiptsByCallId);
|
|
3745
|
+
collectGeneratedImageRunStateArtifactReceipts(serialized, generatedImageReceiptsByArtifactId);
|
|
3746
|
+
const withScreenshots = modelCanReceiveRetainedSessionImages
|
|
3747
|
+
? await materializeRetainedScreenshotRunState({
|
|
3748
|
+
db,
|
|
3749
|
+
objectStorage,
|
|
3750
|
+
workspaceId: input.workspaceId,
|
|
3751
|
+
sessionId: input.sessionId,
|
|
3752
|
+
serialized,
|
|
3753
|
+
})
|
|
3754
|
+
: serialized;
|
|
3755
|
+
return projectGeneratedImageRunStateForModel(
|
|
3756
|
+
compactGeneratedImageRunState(withScreenshots, generatedImageReceiptsByProviderItemId),
|
|
3757
|
+
);
|
|
3758
|
+
};
|
|
3107
3759
|
let providerArtifactCandidates: Awaited<
|
|
3108
3760
|
ReturnType<typeof turnInput>
|
|
3109
3761
|
>["providerArtifactCandidates"] = {
|
|
@@ -3121,15 +3773,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3121
3773
|
if (!stream || !turnId) {
|
|
3122
3774
|
return;
|
|
3123
3775
|
}
|
|
3776
|
+
const durableTurnId = turnId;
|
|
3124
3777
|
try {
|
|
3125
3778
|
const rawHistory = (stream.state as { history?: unknown[] }).history;
|
|
3126
3779
|
if (Array.isArray(rawHistory)) {
|
|
3780
|
+
const typedHistory = rawHistory as Array<Record<string, unknown>>;
|
|
3781
|
+
await retainNativeGeneratedImagesFromHistory(typedHistory);
|
|
3782
|
+
const durableHistory = compactGeneratedImageHistory(
|
|
3783
|
+
compactRetainedScreenshotHistory(typedHistory, retainedScreenshotReceiptsByCallId),
|
|
3784
|
+
generatedImageReceiptsByProviderItemId,
|
|
3785
|
+
);
|
|
3127
3786
|
const { rows, nextWatermark, nextPosition } = historyRowsToAppend(
|
|
3128
|
-
|
|
3787
|
+
durableHistory,
|
|
3129
3788
|
persistedHistoryCount,
|
|
3130
3789
|
nextHistoryPosition,
|
|
3131
3790
|
modelRunSettings.modelToolOutputTruncationTokens,
|
|
3132
|
-
(value) => redact(value),
|
|
3133
3791
|
);
|
|
3134
3792
|
const hasModelOrToolProgress = rows.some((row) =>
|
|
3135
3793
|
isModelOrToolProgressHistoryItem(row.item),
|
|
@@ -3137,21 +3795,23 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3137
3795
|
const shouldAppendRows =
|
|
3138
3796
|
rows.length > 0 && (!options.skipInputOnlyRows || hasModelOrToolProgress);
|
|
3139
3797
|
if (shouldAppendRows) {
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
|
|
3143
|
-
|
|
3144
|
-
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3148
|
-
|
|
3798
|
+
await runMandatoryHistoryPersistenceStep("history_append", async () => {
|
|
3799
|
+
const appended = await appendSessionHistoryItems(db, {
|
|
3800
|
+
accountId: input.accountId,
|
|
3801
|
+
workspaceId: input.workspaceId,
|
|
3802
|
+
sessionId: input.sessionId,
|
|
3803
|
+
turnId: durableTurnId,
|
|
3804
|
+
expectedExecutionGeneration: executionGeneration,
|
|
3805
|
+
expectedAttemptId: input.attemptId,
|
|
3806
|
+
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
3807
|
+
items: rows,
|
|
3808
|
+
});
|
|
3809
|
+
if (!appended) {
|
|
3810
|
+
throw new TurnAttemptFencedError(
|
|
3811
|
+
"turn execution generation was fenced while saving conversation history",
|
|
3812
|
+
);
|
|
3813
|
+
}
|
|
3149
3814
|
});
|
|
3150
|
-
if (!appended) {
|
|
3151
|
-
throw new TurnAttemptFencedError(
|
|
3152
|
-
"turn execution generation was fenced while saving conversation history",
|
|
3153
|
-
);
|
|
3154
|
-
}
|
|
3155
3815
|
}
|
|
3156
3816
|
if (shouldAppendRows || !options.skipInputOnlyRows) {
|
|
3157
3817
|
persistedHistoryCount = nextWatermark;
|
|
@@ -3160,37 +3820,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3160
3820
|
}
|
|
3161
3821
|
const envelope = sandboxStateEntryFromRunState(stream.state);
|
|
3162
3822
|
if (envelope) {
|
|
3163
|
-
await
|
|
3164
|
-
|
|
3165
|
-
|
|
3166
|
-
|
|
3167
|
-
|
|
3168
|
-
|
|
3823
|
+
await runMandatoryHistoryPersistenceStep("sandbox_envelope", () =>
|
|
3824
|
+
upsertSandboxSessionEnvelope(db, {
|
|
3825
|
+
accountId: input.accountId,
|
|
3826
|
+
workspaceId: input.workspaceId,
|
|
3827
|
+
sessionId: input.sessionId,
|
|
3828
|
+
envelope,
|
|
3829
|
+
}),
|
|
3830
|
+
);
|
|
3169
3831
|
}
|
|
3170
3832
|
} catch (persistError) {
|
|
3171
3833
|
console.error(
|
|
3172
3834
|
"session history dual-write failed (run unaffected)",
|
|
3173
|
-
safeErrorDiagnostic(persistError
|
|
3835
|
+
safeErrorDiagnostic(persistError),
|
|
3174
3836
|
);
|
|
3175
3837
|
if (options.requireDurable) throw persistError;
|
|
3176
3838
|
}
|
|
3177
3839
|
};
|
|
3178
|
-
// Reassigned after the variable set loads; the publish closure is
|
|
3179
|
-
// created (and used for turn.started) before the variableSet is available.
|
|
3180
|
-
// Generic auth/cookie/signed-URL/token classification is active before any
|
|
3181
|
-
// exact run-secret provenance is available. Registered values strengthen
|
|
3182
|
-
// this boundary as credentials are resolved and renewed during the turn.
|
|
3183
|
-
let redact: (payload: unknown) => unknown = createSecretRedactor([]);
|
|
3184
|
-
const secretRedactions = new Map<string, string>();
|
|
3185
3840
|
const publishedRunCredentialNotices = new Set<string>();
|
|
3186
|
-
|
|
3187
|
-
for (const secret of secrets) {
|
|
3188
|
-
if (!secretRedactions.has(secret.value)) secretRedactions.set(secret.value, secret.name);
|
|
3189
|
-
}
|
|
3190
|
-
redact = createSecretRedactor(
|
|
3191
|
-
[...secretRedactions].map(([value, name]) => ({ name, value })),
|
|
3192
|
-
);
|
|
3193
|
-
};
|
|
3841
|
+
|
|
3194
3842
|
let variableSetId = "";
|
|
3195
3843
|
// Rig telemetry (M3): set once the session loads; empty string for a rig-less
|
|
3196
3844
|
// turn (mirrors variableSetId). Read by the activity span's finally block.
|
|
@@ -3218,10 +3866,39 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3218
3866
|
// only exists in the RunState blob), never through a swapped trigger.
|
|
3219
3867
|
let triggerType: string | null = null;
|
|
3220
3868
|
try {
|
|
3869
|
+
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
3870
|
+
const claim = await claimSessionWorkForAttempt(db, input.workspaceId, {
|
|
3871
|
+
sessionId: input.sessionId,
|
|
3872
|
+
workflowId: input.workflowId,
|
|
3873
|
+
workflowRunId: input.workflowRunId,
|
|
3874
|
+
attemptId: input.attemptId,
|
|
3875
|
+
dispatchId,
|
|
3876
|
+
trigger: input.trigger,
|
|
3877
|
+
});
|
|
3878
|
+
if (claim.action === "unclaimed") {
|
|
3879
|
+
activityStatus = "unclaimed";
|
|
3880
|
+
return { status: "unclaimed", reason: claim.reason };
|
|
3881
|
+
}
|
|
3882
|
+
const turn = claim.turn;
|
|
3883
|
+
turnId = turn.id;
|
|
3884
|
+
executionGeneration = turn.executionGeneration;
|
|
3885
|
+
providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
|
|
3886
|
+
let installedApiIntegrations: readonly ApiIntegrationRuntime[] = [];
|
|
3887
|
+
const credentialSubjectId = credentialSubjectIdForTurnInitiator(turn);
|
|
3221
3888
|
const mcpSettings = await settingsWithEnabledCapabilityMcpServers(
|
|
3222
3889
|
db,
|
|
3223
3890
|
input.workspaceId,
|
|
3224
3891
|
settings,
|
|
3892
|
+
{
|
|
3893
|
+
...(credentialSubjectId
|
|
3894
|
+
? { subjectId: credentialSubjectId }
|
|
3895
|
+
: {
|
|
3896
|
+
personalConnectionDelegations: turn.personalConnectionDelegations,
|
|
3897
|
+
}),
|
|
3898
|
+
onResolvedApiIntegrations: (integrations) => {
|
|
3899
|
+
installedApiIntegrations = integrations;
|
|
3900
|
+
},
|
|
3901
|
+
},
|
|
3225
3902
|
);
|
|
3226
3903
|
// Read the active-credential flag once for the runtime capability overlay.
|
|
3227
3904
|
// Accepted billing/provider identity comes from the turn policy below,
|
|
@@ -3246,23 +3923,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3246
3923
|
? await resolveCodexAppsCredentialIdForRun(db, input.workspaceId)
|
|
3247
3924
|
: null;
|
|
3248
3925
|
runtime.configure(capabilitySettings);
|
|
3249
|
-
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
3250
|
-
const claim = await claimSessionWorkForAttempt(db, input.workspaceId, {
|
|
3251
|
-
sessionId: input.sessionId,
|
|
3252
|
-
workflowId: input.workflowId,
|
|
3253
|
-
workflowRunId: input.workflowRunId,
|
|
3254
|
-
attemptId: input.attemptId,
|
|
3255
|
-
dispatchId,
|
|
3256
|
-
trigger: input.trigger,
|
|
3257
|
-
});
|
|
3258
|
-
if (claim.action === "unclaimed") {
|
|
3259
|
-
activityStatus = "unclaimed";
|
|
3260
|
-
return { status: "unclaimed", reason: claim.reason };
|
|
3261
|
-
}
|
|
3262
|
-
const turn = claim.turn;
|
|
3263
|
-
turnId = turn.id;
|
|
3264
|
-
executionGeneration = turn.executionGeneration;
|
|
3265
|
-
providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
|
|
3266
3926
|
const claimedPolicy = readTurnExecutionPolicyV1(turn.metadata);
|
|
3267
3927
|
const policyForAbsent =
|
|
3268
3928
|
claimedPolicy.kind === "valid"
|
|
@@ -3377,7 +4037,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3377
4037
|
) => {
|
|
3378
4038
|
const inputs = events.map((event) => ({
|
|
3379
4039
|
...event,
|
|
3380
|
-
payload:
|
|
4040
|
+
payload: event.payload,
|
|
3381
4041
|
turnId: turnId!,
|
|
3382
4042
|
producerId,
|
|
3383
4043
|
producerSeq: ++producerSeq,
|
|
@@ -3391,6 +4051,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3391
4051
|
executionGeneration,
|
|
3392
4052
|
input.attemptId,
|
|
3393
4053
|
inputs,
|
|
4054
|
+
{
|
|
4055
|
+
onAppend: ({ durationSeconds }) =>
|
|
4056
|
+
recordSessionEventAppendLatency(observability, {
|
|
4057
|
+
durationSeconds,
|
|
4058
|
+
}),
|
|
4059
|
+
onPublish: ({ durationSeconds }) =>
|
|
4060
|
+
recordSessionEventPublishLatency(observability, {
|
|
4061
|
+
durationSeconds,
|
|
4062
|
+
}),
|
|
4063
|
+
},
|
|
3394
4064
|
);
|
|
3395
4065
|
if (inputs.length > 0 && !appended.accepted) {
|
|
3396
4066
|
throw new TurnAttemptFencedError("turn execution generation was fenced");
|
|
@@ -3445,13 +4115,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3445
4115
|
: undefined;
|
|
3446
4116
|
const inputs = inputSettlement.events.map((event) => ({
|
|
3447
4117
|
...event,
|
|
3448
|
-
payload:
|
|
4118
|
+
payload: event.payload,
|
|
3449
4119
|
turnId: turnId!,
|
|
3450
4120
|
producerId,
|
|
3451
4121
|
producerSeq: ++producerSeq,
|
|
3452
4122
|
}));
|
|
3453
|
-
const
|
|
3454
|
-
?
|
|
4123
|
+
const runState = inputSettlement.runState
|
|
4124
|
+
? {
|
|
4125
|
+
...inputSettlement.runState,
|
|
4126
|
+
serializedRunState: compactMediaRunState(inputSettlement.runState.serializedRunState),
|
|
4127
|
+
}
|
|
3455
4128
|
: undefined;
|
|
3456
4129
|
const result = await applySessionTurnSettlement(db, input.workspaceId, {
|
|
3457
4130
|
sessionId: input.sessionId,
|
|
@@ -3462,7 +4135,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3462
4135
|
sessionStatus: inputSettlement.sessionStatus,
|
|
3463
4136
|
activeTurnId: inputSettlement.activeTurnId,
|
|
3464
4137
|
events: inputs,
|
|
3465
|
-
...(
|
|
4138
|
+
...(runState ? { runState } : {}),
|
|
3466
4139
|
...(recordingMutation ? { recording: recordingMutation } : {}),
|
|
3467
4140
|
...(compactionRequestFailure ? { compactionRequestFailure } : {}),
|
|
3468
4141
|
});
|
|
@@ -3679,14 +4352,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3679
4352
|
leased.credentialId !== null &&
|
|
3680
4353
|
(sessionPinSource !== "policy" || sessionPin !== leased.credentialId)
|
|
3681
4354
|
) {
|
|
3682
|
-
const pinMutation = await
|
|
4355
|
+
const pinMutation = await withSessionCodexCapacityMutation(
|
|
3683
4356
|
db,
|
|
3684
4357
|
{
|
|
3685
4358
|
workspaceId: input.workspaceId,
|
|
3686
4359
|
reason: "codex_policy_pin_changed",
|
|
3687
4360
|
},
|
|
3688
4361
|
async (tx) => {
|
|
3689
|
-
const changed = await
|
|
4362
|
+
const changed = await setSessionCodexPinInTransaction(
|
|
3690
4363
|
tx,
|
|
3691
4364
|
input.workspaceId,
|
|
3692
4365
|
input.sessionId,
|
|
@@ -3707,14 +4380,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3707
4380
|
pinMutation.wakeTargets,
|
|
3708
4381
|
);
|
|
3709
4382
|
} else if (selectedPinDisposition === "clearStale") {
|
|
3710
|
-
const pinMutation = await
|
|
4383
|
+
const pinMutation = await withSessionCodexCapacityMutation(
|
|
3711
4384
|
db,
|
|
3712
4385
|
{
|
|
3713
4386
|
workspaceId: input.workspaceId,
|
|
3714
4387
|
reason: "codex_stale_policy_pin_cleared",
|
|
3715
4388
|
},
|
|
3716
4389
|
async (tx) => {
|
|
3717
|
-
const changed = await
|
|
4390
|
+
const changed = await setSessionCodexPinInTransaction(
|
|
3718
4391
|
tx,
|
|
3719
4392
|
input.workspaceId,
|
|
3720
4393
|
input.sessionId,
|
|
@@ -3820,10 +4493,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3820
4493
|
labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
|
|
3821
4494
|
});
|
|
3822
4495
|
observability.warn("Codex adaptive fleet shadow decision failed open", {
|
|
3823
|
-
workspaceId: input.workspaceId,
|
|
3824
4496
|
stage: shadowResult.stage,
|
|
3825
4497
|
reason: shadowResult.reason,
|
|
3826
|
-
|
|
4498
|
+
errorClass: "CodexFleetShadowOperationError",
|
|
4499
|
+
errorCode: "codex_fleet_shadow_failed",
|
|
4500
|
+
origin: "worker",
|
|
3827
4501
|
payloadBytes: shadowResult.payloadBytes,
|
|
3828
4502
|
});
|
|
3829
4503
|
}
|
|
@@ -4173,66 +4847,61 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4173
4847
|
attemptId: input.attemptId,
|
|
4174
4848
|
executionGeneration: turn.executionGeneration,
|
|
4175
4849
|
};
|
|
4176
|
-
const [
|
|
4850
|
+
const [workspace, companyProfileSnapshot, instructionPolicySnapshot, preferenceSnapshot] =
|
|
4177
4851
|
await Promise.all([
|
|
4178
|
-
|
|
4852
|
+
getWorkspace(db, input.workspaceId),
|
|
4853
|
+
getOrCreateCompanyProfileSnapshot(db, governanceClaims),
|
|
4179
4854
|
getOrCreateWorkspaceInstructionPolicySnapshot(db, governanceClaims),
|
|
4180
4855
|
getOrCreatePreferenceRegistrySnapshot(db, governanceClaims).catch((error) => {
|
|
4181
4856
|
if (error instanceof PreferenceRegistryInitiatorError) return null;
|
|
4182
4857
|
throw error;
|
|
4183
4858
|
}),
|
|
4184
4859
|
]);
|
|
4860
|
+
if (!workspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
|
|
4861
|
+
const workspaceAgentInstructions = workspace.agentInstructions;
|
|
4862
|
+
const agentHumanInputEnabled = resolveWorkspaceAgentHumanInputEnabled(workspace.settings);
|
|
4863
|
+
assertWorkspaceHumanInputAllowed(agentHumanInputEnabled, "resume", humanInputResume !== null);
|
|
4185
4864
|
const workspaceGovernance = renderWorkspaceGovernanceContext({
|
|
4865
|
+
companyProfile: companyProfileSnapshot,
|
|
4186
4866
|
instructionPolicy: instructionPolicySnapshot,
|
|
4187
4867
|
preferences: preferenceSnapshot,
|
|
4188
4868
|
});
|
|
4189
4869
|
const structuredWorkspacePolicyActive =
|
|
4190
4870
|
hasActiveWorkspaceInstructionPolicy(instructionPolicySnapshot);
|
|
4191
4871
|
const workspaceMemory = await resolveWorkspaceMemoryBlock(db, input.workspaceId);
|
|
4192
|
-
const
|
|
4193
|
-
|
|
4194
|
-
|
|
4195
|
-
|
|
4196
|
-
|
|
4197
|
-
...settingsWithRigImage(
|
|
4198
|
-
settingsWithPackSandboxImage(
|
|
4199
|
-
capabilitySettings,
|
|
4200
|
-
packRuntime.sandboxImage,
|
|
4201
|
-
packRuntime.sandboxProviderImages,
|
|
4202
|
-
),
|
|
4203
|
-
rigVersion?.image ?? null,
|
|
4872
|
+
const logicalSandboxSettings = settingsWithRigImage(
|
|
4873
|
+
settingsWithPackSandboxImage(
|
|
4874
|
+
capabilitySettings,
|
|
4875
|
+
packRuntime.sandboxImage,
|
|
4876
|
+
packRuntime.sandboxProviderImages,
|
|
4204
4877
|
),
|
|
4878
|
+
rigVersion?.image ?? null,
|
|
4879
|
+
);
|
|
4880
|
+
const providerImageSettings = await settingsWithRigProviderImage(
|
|
4881
|
+
logicalSandboxSettings,
|
|
4882
|
+
rigVersion,
|
|
4883
|
+
turn.sandboxBackend,
|
|
4884
|
+
);
|
|
4885
|
+
const baseRunSettings = {
|
|
4886
|
+
// IMAGE PRECEDENCE: rig > pre-V2 Pack compatibility > deployment.
|
|
4887
|
+
// resolveWorkspacePackRuntime returns no image for V2 Pack rows, so
|
|
4888
|
+
// settingsWithRigImage runs outermost over only the intentionally
|
|
4889
|
+
// retained legacy fallback. A matching verified provider-native ID is
|
|
4890
|
+
// then applied only to fresh creation without changing the logical
|
|
4891
|
+
// lease image.
|
|
4892
|
+
...providerImageSettings,
|
|
4205
4893
|
openaiModel: turn.model,
|
|
4206
4894
|
openaiReasoningEffort: turn.reasoningEffort,
|
|
4207
4895
|
sandboxBackend: turn.sandboxBackend,
|
|
4208
4896
|
};
|
|
4209
|
-
const sessionMcpHeaderNames = new Map<string, readonly string[]>();
|
|
4210
4897
|
const runSettings = await settingsWithSessionMcpServersForRun(
|
|
4211
4898
|
db,
|
|
4212
4899
|
input.workspaceId,
|
|
4213
4900
|
input.sessionId,
|
|
4214
4901
|
input.attemptId,
|
|
4215
4902
|
baseRunSettings,
|
|
4216
|
-
{
|
|
4217
|
-
onResolvedServers: (servers) => {
|
|
4218
|
-
for (const server of servers) {
|
|
4219
|
-
sessionMcpHeaderNames.set(server.id, server.headerNames);
|
|
4220
|
-
}
|
|
4221
|
-
},
|
|
4222
|
-
},
|
|
4223
4903
|
);
|
|
4224
|
-
|
|
4225
|
-
...(runSettings.accessKey
|
|
4226
|
-
? [{ name: "OPENGENI_ACCESS_KEY", value: runSettings.accessKey }]
|
|
4227
|
-
: []),
|
|
4228
|
-
...runSettings.mcpServers.flatMap((server) => {
|
|
4229
|
-
return headerSecretRedactions(
|
|
4230
|
-
`MCP_${server.id.toUpperCase()}_STATIC`,
|
|
4231
|
-
server.headers,
|
|
4232
|
-
sessionMcpHeaderNames.get(server.id),
|
|
4233
|
-
);
|
|
4234
|
-
}),
|
|
4235
|
-
]);
|
|
4904
|
+
|
|
4236
4905
|
// Multi-provider per-turn routing → the provider gating (compaction mode,
|
|
4237
4906
|
// hosted web search, encrypted reasoning, context window) the agent and
|
|
4238
4907
|
// compaction summarizer must use; null falls back to the legacy global
|
|
@@ -4250,9 +4919,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4250
4919
|
capabilitySettings,
|
|
4251
4920
|
turnExecutionPolicy.productModelId,
|
|
4252
4921
|
);
|
|
4253
|
-
const supportsImageInput = modelSupportsImageInputForTurn(resolvedModel);
|
|
4254
4922
|
const providerApi = resolvedModel?.provider.api ?? "responses";
|
|
4923
|
+
const nativeImageProviderBinding =
|
|
4924
|
+
providerApi === "responses"
|
|
4925
|
+
? openAiHostedImageProviderBindingForTurn(capabilitySettings, resolvedModel)
|
|
4926
|
+
: null;
|
|
4927
|
+
const lazyToolTransport = lazyToolTransportForTurn(resolvedModel);
|
|
4255
4928
|
const modelInputPolicy = modelAttachmentInputPolicyForTurn(resolvedModel);
|
|
4929
|
+
// Use the proven wire capability, not the catalogue modality alone. Chat
|
|
4930
|
+
// providers may advertise vision, but OpenGeni intentionally has no typed
|
|
4931
|
+
// image transport for that wire yet; exposing view_image there would turn
|
|
4932
|
+
// pixels into a multi-megabyte text/base64 function result.
|
|
4933
|
+
const supportsImageInput = modelInputPolicy.supportsImageInput;
|
|
4934
|
+
modelCanReceiveRetainedSessionImages = supportsImageInput;
|
|
4256
4935
|
const attachmentProjector = createModelHistoryAttachmentProjector(
|
|
4257
4936
|
db,
|
|
4258
4937
|
input.workspaceId,
|
|
@@ -4261,51 +4940,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4261
4940
|
);
|
|
4262
4941
|
const modelHistoryProjector = async (items: Array<Record<string, unknown>>) =>
|
|
4263
4942
|
projectModelInputForCapabilities(await attachmentProjector(items), modelInputPolicy);
|
|
4943
|
+
const generatedImageHistoryProjector = async (
|
|
4944
|
+
items: Array<Record<string, unknown>>,
|
|
4945
|
+
): Promise<Array<Record<string, unknown>>> => {
|
|
4946
|
+
collectGeneratedImageReceipts(items, generatedImageReceiptsByProviderItemId);
|
|
4947
|
+
collectGeneratedImageArtifactReceipts(items, generatedImageReceiptsByArtifactId);
|
|
4948
|
+
return projectGeneratedImageHistoryForModel(items);
|
|
4949
|
+
};
|
|
4264
4950
|
const compactionModelHistoryProjector = async (items: Array<Record<string, unknown>>) =>
|
|
4265
|
-
await modelHistoryProjector(
|
|
4266
|
-
|
|
4267
|
-
|
|
4268
|
-
|
|
4269
|
-
|
|
4270
|
-
|
|
4271
|
-
selectedProvider?.publicDefaultQueryNames?.map((name) => name.toLowerCase()) ?? [],
|
|
4272
|
-
);
|
|
4273
|
-
const fallbackProviderApiKey =
|
|
4274
|
-
capabilitySettings.openaiProvider === "azure"
|
|
4275
|
-
? (capabilitySettings.azureOpenaiApiKey ?? capabilitySettings.azureOpenaiAdToken)
|
|
4276
|
-
: capabilitySettings.openaiApiKey;
|
|
4277
|
-
registerSecretRedactions([
|
|
4278
|
-
...(selectedProvider?.apiKey
|
|
4279
|
-
? [
|
|
4280
|
-
{
|
|
4281
|
-
name: `MODEL_${selectedProvider.id.toUpperCase()}_API_KEY`,
|
|
4282
|
-
value: selectedProvider.apiKey,
|
|
4283
|
-
},
|
|
4284
|
-
]
|
|
4285
|
-
: !resolvedModel && fallbackProviderApiKey
|
|
4286
|
-
? [
|
|
4287
|
-
{
|
|
4288
|
-
name: "MODEL_PROVIDER_API_KEY",
|
|
4289
|
-
value: fallbackProviderApiKey,
|
|
4290
|
-
},
|
|
4291
|
-
]
|
|
4292
|
-
: []),
|
|
4293
|
-
...Object.entries(selectedProvider?.defaultHeaders ?? {}).flatMap(([name, value]) =>
|
|
4294
|
-
publicProviderHeaders.has(name.toLowerCase())
|
|
4295
|
-
? []
|
|
4296
|
-
: [
|
|
4297
|
-
{
|
|
4298
|
-
name: `MODEL_PROVIDER_HEADER_${name.toUpperCase()}`,
|
|
4299
|
-
value,
|
|
4300
|
-
},
|
|
4301
|
-
],
|
|
4302
|
-
),
|
|
4303
|
-
...Object.entries(selectedProvider?.defaultQuery ?? {}).flatMap(([name, value]) =>
|
|
4304
|
-
publicProviderQueries.has(name.toLowerCase())
|
|
4305
|
-
? []
|
|
4306
|
-
: [{ name: `MODEL_PROVIDER_QUERY_${name.toUpperCase()}`, value }],
|
|
4307
|
-
),
|
|
4308
|
-
]);
|
|
4951
|
+
await modelHistoryProjector(
|
|
4952
|
+
projectHistoryForProvider(
|
|
4953
|
+
await generatedImageHistoryProjector(restoreGenericDispatchHistoryItems(items)),
|
|
4954
|
+
providerApi,
|
|
4955
|
+
),
|
|
4956
|
+
);
|
|
4309
4957
|
// Bind the provider/model catalog's context policy to every model-facing
|
|
4310
4958
|
// path for this turn. In particular, Codex subscription turns must not
|
|
4311
4959
|
// inherit the deployment's OpenAI/Azure mode or 1.05M context defaults:
|
|
@@ -4339,6 +4987,44 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4339
4987
|
}
|
|
4340
4988
|
}
|
|
4341
4989
|
}
|
|
4990
|
+
// A recovered asynchronous video completion is model-visible only after
|
|
4991
|
+
// its durable File is present at the exact sandbox path carried by the
|
|
4992
|
+
// receipt. Resolve and verify the claimed update before sandbox policy is
|
|
4993
|
+
// chosen so this turn cannot remain lazy and expose an absent path.
|
|
4994
|
+
for (const update of await listSessionSystemUpdatesForTurn(
|
|
4995
|
+
db,
|
|
4996
|
+
input.workspaceId,
|
|
4997
|
+
input.sessionId,
|
|
4998
|
+
turn.id,
|
|
4999
|
+
)) {
|
|
5000
|
+
const payload = update.payload as MediaGenerationResult;
|
|
5001
|
+
if (payload.type !== "media_generation_result" || payload.status !== "ready") continue;
|
|
5002
|
+
const retained = await getGeneratedVideoArtifact(
|
|
5003
|
+
db,
|
|
5004
|
+
input.workspaceId,
|
|
5005
|
+
payload.receipt.artifact.artifactId,
|
|
5006
|
+
);
|
|
5007
|
+
if (
|
|
5008
|
+
!retained ||
|
|
5009
|
+
retained.artifact.deletedAt ||
|
|
5010
|
+
retained.file.status !== "ready" ||
|
|
5011
|
+
retained.file.contentType !== "video/mp4" ||
|
|
5012
|
+
retained.file.sizeBytes !== payload.receipt.artifact.originalBytes ||
|
|
5013
|
+
retained.file.sha256 !== payload.receipt.artifact.sha256 ||
|
|
5014
|
+
retained.artifact.sandboxFilename !== `generated-video-${retained.artifact.id}.mp4`
|
|
5015
|
+
) {
|
|
5016
|
+
throw new Error("Generated video completion does not match its retained File");
|
|
5017
|
+
}
|
|
5018
|
+
requiredGeneratedVideoFiles.push({
|
|
5019
|
+
operationId: payload.operationId,
|
|
5020
|
+
artifactId: retained.artifact.id,
|
|
5021
|
+
fileId: retained.file.id,
|
|
5022
|
+
objectKey: retained.file.objectKey,
|
|
5023
|
+
sizeBytes: retained.file.sizeBytes,
|
|
5024
|
+
sha256: retained.file.sha256,
|
|
5025
|
+
filename: retained.artifact.sandboxFilename,
|
|
5026
|
+
});
|
|
5027
|
+
}
|
|
4342
5028
|
// A codex-subscription turn resolves the bearer for THIS turn's effective
|
|
4343
5029
|
// codex account (effectiveCodexCredentialId; pin > workspace-active) at
|
|
4344
5030
|
// model-call time — multi-account P1 means a workspace can hold N accounts,
|
|
@@ -4360,15 +5046,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4360
5046
|
input.workspaceId,
|
|
4361
5047
|
effectiveCodexCredentialId ?? "",
|
|
4362
5048
|
);
|
|
4363
|
-
const registerSnapshot = async (
|
|
4364
|
-
snapshotPromise: ReturnType<typeof resolver.getToken>,
|
|
4365
|
-
) => {
|
|
4366
|
-
const snapshot = await snapshotPromise;
|
|
4367
|
-
registerSecretRedactions([
|
|
4368
|
-
{ name: "CODEX_ACCESS_TOKEN", value: snapshot.accessToken },
|
|
4369
|
-
]);
|
|
4370
|
-
return snapshot;
|
|
4371
|
-
};
|
|
4372
5049
|
return {
|
|
4373
5050
|
clientVersion: CODEX_CLIENT_VERSION,
|
|
4374
5051
|
// Backend sticky cache-routing key — the SAME id as the body's
|
|
@@ -4378,8 +5055,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4378
5055
|
// (per-request shard lottery = prod's measured 48.6% on sol);
|
|
4379
5056
|
// with it, resends pin to the warm shard (Codex CLI parity).
|
|
4380
5057
|
sessionId: input.sessionId,
|
|
4381
|
-
getToken: () =>
|
|
4382
|
-
refresh: () =>
|
|
5058
|
+
getToken: () => resolver.getToken(),
|
|
5059
|
+
refresh: () => resolver.refresh(),
|
|
4383
5060
|
resolveModel: buildModelResolver(
|
|
4384
5061
|
CODEX_FALLBACK_MODEL_SLUGS,
|
|
4385
5062
|
CODEX_FALLBACK_MODEL_SLUGS[0],
|
|
@@ -4466,8 +5143,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4466
5143
|
resolvedModel
|
|
4467
5144
|
? (s: Settings, m: Array<Record<string, unknown>>) =>
|
|
4468
5145
|
withCodex(() =>
|
|
4469
|
-
|
|
5146
|
+
summarizeContextForCompaction(s, m, {
|
|
4470
5147
|
client: resolvedModel.client,
|
|
5148
|
+
provider: resolvedModel.provider,
|
|
4471
5149
|
api: resolvedModel.provider.api,
|
|
4472
5150
|
model: turnExecutionPolicy.upstreamModelId,
|
|
4473
5151
|
maxOutputTokens: SUMMARY_BUFFER_TOKENS,
|
|
@@ -4477,7 +5155,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4477
5155
|
}),
|
|
4478
5156
|
)
|
|
4479
5157
|
: (s: Settings, m: Array<Record<string, unknown>>) =>
|
|
4480
|
-
|
|
5158
|
+
summarizeContextForCompaction(s, m, {
|
|
4481
5159
|
model: turnExecutionPolicy.upstreamModelId,
|
|
4482
5160
|
maxOutputTokens: SUMMARY_BUFFER_TOKENS,
|
|
4483
5161
|
onUsage: recordCompactionUsage,
|
|
@@ -4532,6 +5210,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4532
5210
|
}
|
|
4533
5211
|
return requestRemoteCompactionV2(s, m, {
|
|
4534
5212
|
client: resolvedModel.client,
|
|
5213
|
+
provider: resolvedModel.provider,
|
|
4535
5214
|
model: turnExecutionPolicy.upstreamModelId,
|
|
4536
5215
|
systemInstructions: remoteCompactionInstructions,
|
|
4537
5216
|
onUsage: recordCompactionUsage,
|
|
@@ -4622,6 +5301,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4622
5301
|
force: true,
|
|
4623
5302
|
clearRequestedCompaction: true,
|
|
4624
5303
|
trigger: "operator",
|
|
5304
|
+
materializeHistory: materializeScreenshotHistory,
|
|
4625
5305
|
projectModelInput: compactionModelHistoryProjector,
|
|
4626
5306
|
...compactionModeOptions,
|
|
4627
5307
|
},
|
|
@@ -4654,7 +5334,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4654
5334
|
},
|
|
4655
5335
|
);
|
|
4656
5336
|
if (!isCompactionSummaryFailure(error)) throw error;
|
|
4657
|
-
const errorMessage = String(
|
|
5337
|
+
const errorMessage = String(compactionFailureReasonFromError(error));
|
|
4658
5338
|
if (
|
|
4659
5339
|
!(await settle!({
|
|
4660
5340
|
events: [
|
|
@@ -4787,11 +5467,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4787
5467
|
rigDefaultEnvironmentValues,
|
|
4788
5468
|
workspaceVariableSet?.values ?? {},
|
|
4789
5469
|
);
|
|
4790
|
-
// Redact EVERY exported secret value (rig defaults + session set) from turn
|
|
4791
|
-
// output, not just the session set's.
|
|
4792
|
-
registerSecretRedactions(
|
|
4793
|
-
Object.entries(sandboxWorkspaceEnvironmentValues).map(([name, value]) => ({ name, value })),
|
|
4794
|
-
);
|
|
4795
5470
|
// EFFECTIVE compute backend, resolved ONCE at turn start (Case B + Stage D
|
|
4796
5471
|
// D1-lite) and reused for EVERY downstream decision: the env mint (skip
|
|
4797
5472
|
// inert platform git tokens for a machine turn), the establish path (no phantom Modal
|
|
@@ -4897,7 +5572,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4897
5572
|
)
|
|
4898
5573
|
: null;
|
|
4899
5574
|
if (initialRunCredentialMaterial) {
|
|
4900
|
-
registerSecretRedactions(initialRunCredentialMaterial.redactions);
|
|
4901
5575
|
for (const payload of runCredentialAuthNeededPayloads(initialRunCredentialMaterial)) {
|
|
4902
5576
|
publishedRunCredentialNotices.add(JSON.stringify(payload));
|
|
4903
5577
|
await publish!([{ type: "credential.auth_needed", payload }], true);
|
|
@@ -4913,7 +5587,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4913
5587
|
undefined,
|
|
4914
5588
|
);
|
|
4915
5589
|
const establishPolicy: "eager" | "on-demand" =
|
|
4916
|
-
lazyProvisionEnabled(settings) &&
|
|
5590
|
+
lazyProvisionEnabled(settings) &&
|
|
5591
|
+
!machinePrimary &&
|
|
5592
|
+
runSettings.sandboxBackend !== "none" &&
|
|
5593
|
+
// Resolved run credentials must be written to one exact leased sandbox
|
|
5594
|
+
// before agent execution. A warm active pointer can bypass the lazy
|
|
5595
|
+
// provisioner entirely, which previously skipped materialization.
|
|
5596
|
+
!initialRunCredentialMaterial &&
|
|
5597
|
+
requiredGeneratedVideoFiles.length === 0
|
|
4917
5598
|
? "on-demand"
|
|
4918
5599
|
: "eager";
|
|
4919
5600
|
// Computed exactly ONCE per turn and reused for BOTH the box manifest
|
|
@@ -4967,17 +5648,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4967
5648
|
turn,
|
|
4968
5649
|
})
|
|
4969
5650
|
: undefined;
|
|
4970
|
-
const
|
|
5651
|
+
const codemodeAuthority = {
|
|
4971
5652
|
sessionId: input.sessionId,
|
|
5653
|
+
turnId: turn.id,
|
|
5654
|
+
attemptId: input.attemptId,
|
|
5655
|
+
executionGeneration: turn.executionGeneration,
|
|
4972
5656
|
};
|
|
5657
|
+
const sandboxArtifactRuntime = sandboxArtifactRuntimeAdmission(
|
|
5658
|
+
settings,
|
|
5659
|
+
runSettings,
|
|
5660
|
+
activeSandboxBackend ?? groupBoxBackend,
|
|
5661
|
+
);
|
|
4973
5662
|
const {
|
|
4974
|
-
environment:
|
|
5663
|
+
environment: baseSandboxEnvironment,
|
|
4975
5664
|
gitToken: sandboxGitToken,
|
|
4976
5665
|
gitTokens: sandboxGitTokens,
|
|
4977
5666
|
gitTokenExpiresAt: sandboxGitTokenExpiresAt,
|
|
4978
5667
|
gitCredentialBindings: sandboxGitCredentialBindings,
|
|
4979
|
-
|
|
4980
|
-
|
|
5668
|
+
codemodeToken: sandboxCodemodeToken,
|
|
5669
|
+
codemodeTokenExpiresAt: sandboxCodemodeTokenExpiresAt,
|
|
4981
5670
|
} = await waitForTurnOperation(
|
|
4982
5671
|
sandboxEnvironmentForRun(
|
|
4983
5672
|
runSettings,
|
|
@@ -4987,33 +5676,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4987
5676
|
sandboxWorkspaceEnvironmentValues,
|
|
4988
5677
|
{
|
|
4989
5678
|
skipGitHubToken: activeSandboxBackend === "selfhosted",
|
|
5679
|
+
skipCodemode: activeSandboxBackend === "selfhosted",
|
|
4990
5680
|
deferGitHubToken:
|
|
4991
5681
|
activeSandboxBackend !== "selfhosted" && establishPolicy === "on-demand",
|
|
4992
5682
|
scope: connectionScope,
|
|
4993
5683
|
...(gitCredentialAuthority ? { authority: gitCredentialAuthority } : {}),
|
|
4994
5684
|
gitCredentials: connectionCredentials?.gitCredentials,
|
|
4995
5685
|
authorizeGitHubTokenMint,
|
|
4996
|
-
|
|
5686
|
+
codemodeAuthority,
|
|
4997
5687
|
},
|
|
4998
5688
|
),
|
|
4999
5689
|
cancellationSignal,
|
|
5000
5690
|
undefined,
|
|
5001
5691
|
);
|
|
5002
|
-
|
|
5003
|
-
|
|
5004
|
-
|
|
5005
|
-
|
|
5006
|
-
|
|
5007
|
-
|
|
5008
|
-
|
|
5009
|
-
|
|
5010
|
-
|
|
5011
|
-
...(sandboxToolspaceToken
|
|
5012
|
-
? [{ name: "TOOLSPACE_TOKEN", value: sandboxToolspaceToken }]
|
|
5013
|
-
: []),
|
|
5014
|
-
]);
|
|
5015
|
-
const sandboxToolspaceTokenFile = sandboxToolspaceToken
|
|
5016
|
-
? toolspaceTokenFileFromEnvironment(sandboxEnvironment, input.sessionId)
|
|
5692
|
+
// Reserved, image-owned paths win over workspace/session variables. The
|
|
5693
|
+
// exact same merged object feeds both box manifest and agent declaration,
|
|
5694
|
+
// preserving the no-environment-delta invariant.
|
|
5695
|
+
const sandboxEnvironment = sandboxArtifactRuntime.available
|
|
5696
|
+
? { ...baseSandboxEnvironment, ...sandboxArtifactRuntime.environment }
|
|
5697
|
+
: baseSandboxEnvironment;
|
|
5698
|
+
|
|
5699
|
+
const sandboxCodemodeTokenFile = sandboxCodemodeToken
|
|
5700
|
+
? codemodeTokenFileFromEnvironment(sandboxEnvironment, input.sessionId)
|
|
5017
5701
|
: undefined;
|
|
5018
5702
|
|
|
5019
5703
|
const initialGitCredentials: MintedRunGitCredentials | undefined =
|
|
@@ -5057,12 +5741,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5057
5741
|
);
|
|
5058
5742
|
if (binding) {
|
|
5059
5743
|
assertGitCredentialRenewalTransportUnchanged(initialBinding, binding);
|
|
5060
|
-
registerSecretRedactions([
|
|
5061
|
-
{
|
|
5062
|
-
name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
|
|
5063
|
-
value: binding.token,
|
|
5064
|
-
},
|
|
5065
|
-
]);
|
|
5066
5744
|
}
|
|
5067
5745
|
pendingBinding = binding;
|
|
5068
5746
|
return binding
|
|
@@ -5132,25 +5810,24 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5132
5810
|
gitCredentialRenewals = controllers;
|
|
5133
5811
|
};
|
|
5134
5812
|
|
|
5135
|
-
const
|
|
5136
|
-
tokenSession:
|
|
5137
|
-
initialExpiresAt =
|
|
5813
|
+
const attachCodemodeTokenRenewal = async (
|
|
5814
|
+
tokenSession: CodemodeTokenWriterSession,
|
|
5815
|
+
initialExpiresAt = sandboxCodemodeTokenExpiresAt,
|
|
5138
5816
|
initialSandbox?: ResumedTurnSandbox,
|
|
5139
5817
|
): Promise<void> => {
|
|
5140
|
-
if (!
|
|
5141
|
-
const previous =
|
|
5142
|
-
|
|
5818
|
+
if (!sandboxCodemodeToken || !initialExpiresAt) return;
|
|
5819
|
+
const previous = codemodeTokenRenewal;
|
|
5820
|
+
codemodeTokenRenewal = null;
|
|
5143
5821
|
await previous?.stop();
|
|
5144
|
-
if (
|
|
5822
|
+
if (codemodeTokenRenewalClosed) return;
|
|
5145
5823
|
|
|
5146
5824
|
const mint = async () => {
|
|
5147
|
-
const material = await
|
|
5825
|
+
const material = await mintSandboxCodemodeToken(
|
|
5148
5826
|
runSettings,
|
|
5149
5827
|
connectionScope,
|
|
5150
|
-
|
|
5828
|
+
codemodeAuthority,
|
|
5151
5829
|
);
|
|
5152
5830
|
if (material) {
|
|
5153
|
-
registerSecretRedactions([{ name: "TOOLSPACE_TOKEN", value: material.token }]);
|
|
5154
5831
|
}
|
|
5155
5832
|
return material;
|
|
5156
5833
|
};
|
|
@@ -5158,18 +5835,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5158
5835
|
const runAs = sandboxRunAs(runSettings);
|
|
5159
5836
|
const targetSandbox = resolvedSandbox ?? initialSandbox;
|
|
5160
5837
|
if (!targetSandbox) {
|
|
5161
|
-
throw new Error("
|
|
5838
|
+
throw new Error("Codemode token renewal has no exact sandbox lease target");
|
|
5162
5839
|
}
|
|
5163
5840
|
await runWorkspaceMutationForSandbox(
|
|
5164
5841
|
targetSandbox,
|
|
5165
|
-
"
|
|
5842
|
+
"codemodeTokenRenewal",
|
|
5166
5843
|
async () =>
|
|
5167
|
-
await
|
|
5844
|
+
await refreshCodemodeTokenFile(tokenSession, material.token, {
|
|
5168
5845
|
...(runAs ? { runAs } : {}),
|
|
5169
|
-
...(
|
|
5846
|
+
...(sandboxCodemodeTokenFile
|
|
5170
5847
|
? {
|
|
5171
|
-
tokenFile:
|
|
5172
|
-
legacyTokenFile: sandboxEnvironment.
|
|
5848
|
+
tokenFile: sandboxCodemodeTokenFile,
|
|
5849
|
+
legacyTokenFile: sandboxEnvironment.OPENGENI_CODEMODE_TOKEN_FILE!,
|
|
5173
5850
|
}
|
|
5174
5851
|
: {}),
|
|
5175
5852
|
...(toolCancellationFenceRef.current
|
|
@@ -5183,32 +5860,32 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5183
5860
|
);
|
|
5184
5861
|
};
|
|
5185
5862
|
let renewalExpiresAt = initialExpiresAt;
|
|
5186
|
-
if (renewalExpiresAt.getTime() <= Date.now() +
|
|
5863
|
+
if (renewalExpiresAt.getTime() <= Date.now() + CODEMODE_TOKEN_EXPIRY_LEAD_MS) {
|
|
5187
5864
|
const fresh = await mint();
|
|
5188
5865
|
if (!fresh) {
|
|
5189
|
-
throw new Error("
|
|
5866
|
+
throw new Error("Codemode token mint became unavailable during sandbox setup");
|
|
5190
5867
|
}
|
|
5191
5868
|
await write(fresh);
|
|
5192
5869
|
renewalExpiresAt = fresh.expiresAt;
|
|
5193
5870
|
}
|
|
5194
|
-
const controller =
|
|
5871
|
+
const controller = startCodemodeTokenRenewalLoop({
|
|
5195
5872
|
initialExpiresAt: renewalExpiresAt,
|
|
5196
5873
|
mint,
|
|
5197
5874
|
write,
|
|
5198
5875
|
onSuccess: () => {
|
|
5199
5876
|
observability.incrementCounter({
|
|
5200
|
-
name: "
|
|
5201
|
-
help: "Sandbox
|
|
5877
|
+
name: "opengeni_codemode_token_renewals_total",
|
|
5878
|
+
help: "Sandbox Codemode token renewal attempts by outcome.",
|
|
5202
5879
|
labels: { outcome: "completed" },
|
|
5203
5880
|
});
|
|
5204
5881
|
},
|
|
5205
5882
|
onFailure: ({ retryDelayMs, errorClass }) => {
|
|
5206
5883
|
observability.incrementCounter({
|
|
5207
|
-
name: "
|
|
5208
|
-
help: "Sandbox
|
|
5884
|
+
name: "opengeni_codemode_token_renewals_total",
|
|
5885
|
+
help: "Sandbox Codemode token renewal attempts by outcome.",
|
|
5209
5886
|
labels: { outcome: "error" },
|
|
5210
5887
|
});
|
|
5211
|
-
observability.warn("Sandbox
|
|
5888
|
+
observability.warn("Sandbox Codemode token renewal failed; retry scheduled", {
|
|
5212
5889
|
sessionId: input.sessionId,
|
|
5213
5890
|
turnId,
|
|
5214
5891
|
errorClass,
|
|
@@ -5216,11 +5893,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5216
5893
|
});
|
|
5217
5894
|
},
|
|
5218
5895
|
});
|
|
5219
|
-
if (
|
|
5896
|
+
if (codemodeTokenRenewalClosed) {
|
|
5220
5897
|
await controller.stop();
|
|
5221
5898
|
return;
|
|
5222
5899
|
}
|
|
5223
|
-
|
|
5900
|
+
codemodeTokenRenewal = controller;
|
|
5224
5901
|
};
|
|
5225
5902
|
|
|
5226
5903
|
const attachRunCredentialRenewal = async (
|
|
@@ -5277,7 +5954,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5277
5954
|
);
|
|
5278
5955
|
return;
|
|
5279
5956
|
}
|
|
5280
|
-
|
|
5957
|
+
|
|
5281
5958
|
await runWorkspaceMutationForSandbox(
|
|
5282
5959
|
requireTargetSandbox(),
|
|
5283
5960
|
"runCredentialMaterialization",
|
|
@@ -5435,9 +6112,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5435
6112
|
db,
|
|
5436
6113
|
settings,
|
|
5437
6114
|
bus,
|
|
6115
|
+
opJournal,
|
|
5438
6116
|
onOp: machineOpObserver.observer,
|
|
5439
6117
|
onSandboxOperation: sandboxOperationObserver,
|
|
5440
6118
|
onHomeSandboxRebound,
|
|
6119
|
+
...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
|
|
5441
6120
|
},
|
|
5442
6121
|
{
|
|
5443
6122
|
workspaceId: input.workspaceId,
|
|
@@ -5482,6 +6161,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5482
6161
|
{
|
|
5483
6162
|
db,
|
|
5484
6163
|
settings: runSettings,
|
|
6164
|
+
logicalFallbackSettings: logicalSandboxSettings,
|
|
5485
6165
|
cancellationSignal: sandboxResumeSignal,
|
|
5486
6166
|
sandboxMetrics: runtimeMetricsHooksForObservability(observability),
|
|
5487
6167
|
onSandboxLost: publishSandboxLost,
|
|
@@ -5520,7 +6200,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5520
6200
|
managedOwnership!.holderId,
|
|
5521
6201
|
),
|
|
5522
6202
|
sandboxResumeSignal,
|
|
5523
|
-
|
|
6203
|
+
releaseLateSandbox,
|
|
5524
6204
|
);
|
|
5525
6205
|
setupBoxSession = resolvedSandbox.established.session;
|
|
5526
6206
|
// Durable box-lifecycle events (sandbox-file-persistence observability):
|
|
@@ -5544,9 +6224,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5544
6224
|
db,
|
|
5545
6225
|
settings,
|
|
5546
6226
|
bus,
|
|
6227
|
+
opJournal,
|
|
5547
6228
|
onSandboxOperation: sandboxOperationObserver,
|
|
5548
6229
|
onHomeSandboxLost: publishSandboxLost,
|
|
5549
6230
|
onHomeSandboxRebound,
|
|
6231
|
+
...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
|
|
5550
6232
|
},
|
|
5551
6233
|
// Thread the SAME declared environment the group box was created with
|
|
5552
6234
|
// (resumeBoxForTurn, above) so a selfhosted swap target's manifest
|
|
@@ -5579,7 +6261,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5579
6261
|
}
|
|
5580
6262
|
}
|
|
5581
6263
|
|
|
5582
|
-
const
|
|
6264
|
+
const ordinaryFileResourceDownloads = await waitForTurnOperation(
|
|
5583
6265
|
sandboxFileDownloadsForRun(
|
|
5584
6266
|
runSettings,
|
|
5585
6267
|
db,
|
|
@@ -5591,6 +6273,35 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5591
6273
|
cancellationSignal,
|
|
5592
6274
|
undefined,
|
|
5593
6275
|
);
|
|
6276
|
+
if (requiredGeneratedVideoFiles.length > 0 && !objectStorage) {
|
|
6277
|
+
throw new Error("Generated video materialization requires object storage");
|
|
6278
|
+
}
|
|
6279
|
+
const generatedVideoDownloads: SandboxFileDownload[] = [];
|
|
6280
|
+
if (objectStorage && requiredGeneratedVideoFiles.length > 0) {
|
|
6281
|
+
const downloadStorage = objectStorageForSandboxDownloads(modelRunSettings, objectStorage);
|
|
6282
|
+
for (const file of requiredGeneratedVideoFiles) {
|
|
6283
|
+
const signed = await downloadStorage.createGetUrl({
|
|
6284
|
+
key: file.objectKey,
|
|
6285
|
+
});
|
|
6286
|
+
generatedVideoDownloads.push({
|
|
6287
|
+
fileId: file.fileId,
|
|
6288
|
+
mountPath: "generated-videos",
|
|
6289
|
+
filename: file.filename,
|
|
6290
|
+
url: signed.url,
|
|
6291
|
+
expiresAt: signed.expiresAt,
|
|
6292
|
+
sizeBytes: file.sizeBytes,
|
|
6293
|
+
sha256: file.sha256,
|
|
6294
|
+
});
|
|
6295
|
+
}
|
|
6296
|
+
}
|
|
6297
|
+
const fileResourceDownloads = [
|
|
6298
|
+
...new Map(
|
|
6299
|
+
[...ordinaryFileResourceDownloads, ...generatedVideoDownloads].map((download) => [
|
|
6300
|
+
download.fileId,
|
|
6301
|
+
download,
|
|
6302
|
+
]),
|
|
6303
|
+
).values(),
|
|
6304
|
+
];
|
|
5594
6305
|
throwIfWorkerShuttingDown();
|
|
5595
6306
|
throwIfTurnCancelled();
|
|
5596
6307
|
const mcpCredentialRootSessionId =
|
|
@@ -5628,13 +6339,33 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5628
6339
|
const resolveCredential: typeof rawResolveCredential = async (request) => {
|
|
5629
6340
|
const result = await resolveFrozenCredential(request);
|
|
5630
6341
|
if (result.status === "ok") {
|
|
5631
|
-
registerSecretRedactions(
|
|
5632
|
-
headerSecretRedactions("MCP", result.headers, Object.keys(result.headers)),
|
|
5633
|
-
);
|
|
5634
6342
|
}
|
|
5635
6343
|
return result;
|
|
5636
6344
|
};
|
|
5637
|
-
const
|
|
6345
|
+
const publishToolAuthNeeded = async (payload: ToolAuthNeededPayload): Promise<void> => {
|
|
6346
|
+
if (!shouldPublishToolAuthNeededForTurn(payload, trigger, turn)) {
|
|
6347
|
+
return;
|
|
6348
|
+
}
|
|
6349
|
+
await publish!([{ type: "tool.auth_needed", payload }], true);
|
|
6350
|
+
};
|
|
6351
|
+
const selectedApiIntegrationIds = new Set(turnTools.map((tool) => tool.id));
|
|
6352
|
+
const localMcpServers = buildApiIntegrationServersForTurn({
|
|
6353
|
+
settings: runSettings,
|
|
6354
|
+
integrations: installedApiIntegrations.filter((integration) =>
|
|
6355
|
+
selectedApiIntegrationIds.has(integration.serverId),
|
|
6356
|
+
),
|
|
6357
|
+
authority: {
|
|
6358
|
+
accountId: input.accountId,
|
|
6359
|
+
workspaceId: input.workspaceId,
|
|
6360
|
+
sessionId: input.sessionId,
|
|
6361
|
+
rootSessionId: session.rootSessionId,
|
|
6362
|
+
turnId: turn.id,
|
|
6363
|
+
attemptId: input.attemptId,
|
|
6364
|
+
...(credentialSubjectId ? { initiatingSubjectId: credentialSubjectId } : {}),
|
|
6365
|
+
},
|
|
6366
|
+
resolveCredential,
|
|
6367
|
+
onAuthNeeded: publishToolAuthNeeded,
|
|
6368
|
+
});
|
|
5638
6369
|
const codexAppsAuth = codexAppsCredentialId
|
|
5639
6370
|
? (() => {
|
|
5640
6371
|
const resolver = buildCodexTokenResolver(
|
|
@@ -5652,12 +6383,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5652
6383
|
}) => Promise<T>,
|
|
5653
6384
|
): Promise<T> => {
|
|
5654
6385
|
const snapshot = await resolver.getToken();
|
|
5655
|
-
|
|
5656
|
-
{ name: "CODEX_APPS_ACCESS_TOKEN", value: snapshot.accessToken },
|
|
5657
|
-
]);
|
|
6386
|
+
|
|
5658
6387
|
return await withCodexAppsRequestAuthorization(
|
|
5659
6388
|
db,
|
|
5660
|
-
{
|
|
6389
|
+
{
|
|
6390
|
+
workspaceId: input.workspaceId,
|
|
6391
|
+
credentialId: codexAppsCredentialId,
|
|
6392
|
+
},
|
|
5661
6393
|
async () => await use(snapshot),
|
|
5662
6394
|
);
|
|
5663
6395
|
},
|
|
@@ -5680,11 +6412,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5680
6412
|
...(credentialSubjectId ? { credentialSubjectId } : {}),
|
|
5681
6413
|
...(codexAppsAuth ? { codexAppsAuth } : {}),
|
|
5682
6414
|
resolveCredential,
|
|
5683
|
-
onAuthNeeded:
|
|
5684
|
-
|
|
5685
|
-
|
|
5686
|
-
|
|
5687
|
-
await publish!([{ type: "tool.auth_needed", payload }], true);
|
|
6415
|
+
onAuthNeeded: publishToolAuthNeeded,
|
|
6416
|
+
localMcpServers,
|
|
6417
|
+
onAttemptToolCatalog: async (catalog) => {
|
|
6418
|
+
await persistAttemptToolCatalog(db, catalog);
|
|
5688
6419
|
},
|
|
5689
6420
|
// Manager-style sessions carry a creation-validated permission set
|
|
5690
6421
|
// for their first-party MCP token; null keeps the fixed default.
|
|
@@ -5692,10 +6423,44 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5692
6423
|
? { firstPartyPermissions: session.firstPartyMcpPermissions }
|
|
5693
6424
|
: {}),
|
|
5694
6425
|
firstPartyTools: session.firstPartyMcpTools ?? [...DEFAULT_FIRST_PARTY_MCP_TOOLS],
|
|
6426
|
+
attemptToolDefinitions: createFirstPartyInteractionAttemptToolDefinitions({
|
|
6427
|
+
settings: runSettings,
|
|
6428
|
+
scope: {
|
|
6429
|
+
accountId: input.accountId,
|
|
6430
|
+
workspaceId: input.workspaceId,
|
|
6431
|
+
sessionId: input.sessionId,
|
|
6432
|
+
turnId: turn.id,
|
|
6433
|
+
attemptId: input.attemptId,
|
|
6434
|
+
executionGeneration,
|
|
6435
|
+
},
|
|
6436
|
+
...(session.firstPartyMcpPermissions?.length
|
|
6437
|
+
? { permissions: session.firstPartyMcpPermissions }
|
|
6438
|
+
: {}),
|
|
6439
|
+
selectedTools: session.firstPartyMcpTools ?? [...DEFAULT_FIRST_PARTY_MCP_TOOLS],
|
|
6440
|
+
subjectId: "worker:first-party-mcp",
|
|
6441
|
+
subjectLabel: "OpenGeni worker",
|
|
6442
|
+
}),
|
|
5695
6443
|
}),
|
|
5696
6444
|
cancellationSignal,
|
|
5697
6445
|
async (latePreparedTools) => await latePreparedTools.close().catch(() => undefined),
|
|
5698
6446
|
);
|
|
6447
|
+
if (turnId && preparedTools.attemptToolEnvironment) {
|
|
6448
|
+
codemodeDispatcher = new CodemodeAttemptDispatcher(
|
|
6449
|
+
db,
|
|
6450
|
+
bus,
|
|
6451
|
+
preparedTools.attemptToolEnvironment,
|
|
6452
|
+
{
|
|
6453
|
+
accountId: input.accountId,
|
|
6454
|
+
workspaceId: input.workspaceId,
|
|
6455
|
+
sessionId: input.sessionId,
|
|
6456
|
+
turnId,
|
|
6457
|
+
attemptId: input.attemptId,
|
|
6458
|
+
executionGeneration,
|
|
6459
|
+
},
|
|
6460
|
+
cancellationSignal,
|
|
6461
|
+
);
|
|
6462
|
+
codemodeDispatcher.start();
|
|
6463
|
+
}
|
|
5699
6464
|
// Genesis turn = the first user turn (no assistant history reconciled
|
|
5700
6465
|
// yet). Durable Postgres state (countSessionHistoryItems includes
|
|
5701
6466
|
// superseded rows after compaction), NOT a workflow counter (turnsThisRun
|
|
@@ -5734,6 +6499,204 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5734
6499
|
}
|
|
5735
6500
|
: {};
|
|
5736
6501
|
const hostedWebSearch = hostedWebSearchForTurn(resolvedModel, runSettings.webSearchEnabled);
|
|
6502
|
+
const resolveImageReferences = async (
|
|
6503
|
+
references: Parameters<typeof resolveImageGenerationReferences>[0]["references"],
|
|
6504
|
+
) =>
|
|
6505
|
+
await resolveImageGenerationReferences({
|
|
6506
|
+
db,
|
|
6507
|
+
objectStorage: objectStorage!,
|
|
6508
|
+
workspaceId: input.workspaceId,
|
|
6509
|
+
references,
|
|
6510
|
+
readSandboxFile: async (path, maxBytes) => {
|
|
6511
|
+
const imageReferenceSession = (setupBoxSession ??
|
|
6512
|
+
sdkOwnedSandboxSession) as ChannelASession | null;
|
|
6513
|
+
if (!imageReferenceSession) {
|
|
6514
|
+
throw new Error("Sandbox image reference is unavailable");
|
|
6515
|
+
}
|
|
6516
|
+
const relativePath = path.slice("/workspace/".length);
|
|
6517
|
+
const referenceRunAs = sandboxRunAs(modelRunSettings);
|
|
6518
|
+
const channel = new SandboxChannelAService({
|
|
6519
|
+
session: imageReferenceSession,
|
|
6520
|
+
workspaceRoot: "/workspace",
|
|
6521
|
+
leaseEpoch: resolvedSandbox?.leaseEpoch ?? 0,
|
|
6522
|
+
...(referenceRunAs ? { runAs: referenceRunAs } : {}),
|
|
6523
|
+
});
|
|
6524
|
+
const read = await channel.fsRead({
|
|
6525
|
+
path: relativePath,
|
|
6526
|
+
encoding: "base64",
|
|
6527
|
+
maxBytes,
|
|
6528
|
+
});
|
|
6529
|
+
if (read.truncated) throw new Error("Sandbox image reference exceeds the byte limit");
|
|
6530
|
+
return Uint8Array.from(Buffer.from(read.content, "base64"));
|
|
6531
|
+
},
|
|
6532
|
+
});
|
|
6533
|
+
const imageGenerationOption: Pick<BuildAgentOptions, "imageGeneration"> = (() => {
|
|
6534
|
+
// Never expose a paid image operation unless its permanent artifact can
|
|
6535
|
+
// be committed. Failing after provider execution would leave an
|
|
6536
|
+
// unrecoverable outcome-unknown operation with no user-visible image.
|
|
6537
|
+
if (!objectStorage) return {};
|
|
6538
|
+
if (nativeImageProviderBinding) {
|
|
6539
|
+
nativeImageGenerationRetention = {
|
|
6540
|
+
...nativeImageProviderBinding,
|
|
6541
|
+
sessionId: input.sessionId,
|
|
6542
|
+
turnId: turn.id,
|
|
6543
|
+
attemptId: input.attemptId,
|
|
6544
|
+
};
|
|
6545
|
+
return { imageGeneration: { kind: "native_hosted" } };
|
|
6546
|
+
}
|
|
6547
|
+
|
|
6548
|
+
if (resolvedModel?.provider.kind === "codex-subscription") {
|
|
6549
|
+
const codexImageContext = codexContext;
|
|
6550
|
+
const codexImageCredentialId = effectiveCodexCredentialId;
|
|
6551
|
+
if (!codexImageContext || !codexImageCredentialId) {
|
|
6552
|
+
throw new CodexReloginRequired(
|
|
6553
|
+
"Codex image generation requires a connected subscription account",
|
|
6554
|
+
);
|
|
6555
|
+
}
|
|
6556
|
+
return {
|
|
6557
|
+
imageGeneration: {
|
|
6558
|
+
kind: "provider_adapter",
|
|
6559
|
+
execute: async ({ prompt, references }, { toolCallId }) => {
|
|
6560
|
+
const resolvedReferences = await resolveImageReferences(references);
|
|
6561
|
+
const receipt = await executeCodexImageGeneration({
|
|
6562
|
+
db,
|
|
6563
|
+
objectStorage,
|
|
6564
|
+
accountId: input.accountId,
|
|
6565
|
+
workspaceId: input.workspaceId,
|
|
6566
|
+
sessionId: input.sessionId,
|
|
6567
|
+
turnId: turn.id,
|
|
6568
|
+
attemptId: input.attemptId,
|
|
6569
|
+
toolCallId,
|
|
6570
|
+
prompt,
|
|
6571
|
+
references: resolvedReferences,
|
|
6572
|
+
credentialId: codexImageCredentialId,
|
|
6573
|
+
codexContext: codexImageContext,
|
|
6574
|
+
...(runtimeCancellationSignal ? { abortSignal: runtimeCancellationSignal } : {}),
|
|
6575
|
+
});
|
|
6576
|
+
generatedImageReceiptsByArtifactId.set(receipt.artifact.artifactId, receipt);
|
|
6577
|
+
await materializeGeneratedImage(receipt);
|
|
6578
|
+
return receipt;
|
|
6579
|
+
},
|
|
6580
|
+
},
|
|
6581
|
+
};
|
|
6582
|
+
}
|
|
6583
|
+
|
|
6584
|
+
const gatewayResolution = resolveModelProvider(
|
|
6585
|
+
capabilitySettings,
|
|
6586
|
+
WORKSPACE_GATEWAY_PROVIDER_ID,
|
|
6587
|
+
);
|
|
6588
|
+
const gateway = gatewayResolution?.provider;
|
|
6589
|
+
if (gateway?.kind !== "vercel-gateway-workspace" || !gateway.apiKey) return {};
|
|
6590
|
+
const gatewayApiKey = gateway.apiKey;
|
|
6591
|
+
return {
|
|
6592
|
+
imageGeneration: {
|
|
6593
|
+
kind: "provider_adapter",
|
|
6594
|
+
execute: async ({ prompt, references }, { toolCallId }) => {
|
|
6595
|
+
const resolvedReferences = await resolveImageReferences(references);
|
|
6596
|
+
const receipt = await executeGatewayImageGeneration({
|
|
6597
|
+
db,
|
|
6598
|
+
objectStorage,
|
|
6599
|
+
accountId: input.accountId,
|
|
6600
|
+
workspaceId: input.workspaceId,
|
|
6601
|
+
sessionId: input.sessionId,
|
|
6602
|
+
turnId: turn.id,
|
|
6603
|
+
attemptId: input.attemptId,
|
|
6604
|
+
apiKey: gatewayApiKey,
|
|
6605
|
+
modelId: capabilitySettings.imageGenerationModel,
|
|
6606
|
+
prompt,
|
|
6607
|
+
references: resolvedReferences,
|
|
6608
|
+
toolCallId,
|
|
6609
|
+
...(runtimeCancellationSignal ? { abortSignal: runtimeCancellationSignal } : {}),
|
|
6610
|
+
});
|
|
6611
|
+
generatedImageReceiptsByArtifactId.set(receipt.artifact.artifactId, receipt);
|
|
6612
|
+
await materializeGeneratedImage(receipt);
|
|
6613
|
+
return receipt;
|
|
6614
|
+
},
|
|
6615
|
+
},
|
|
6616
|
+
};
|
|
6617
|
+
})();
|
|
6618
|
+
const videoGenerationPolicy = await getWorkspaceVideoGenerationPolicy(db, input.workspaceId);
|
|
6619
|
+
const videoGenerationEnabled =
|
|
6620
|
+
videoGenerationPolicy.defaultModelId !== null &&
|
|
6621
|
+
videoGenerationPolicy.enabledModelIds.length > 0;
|
|
6622
|
+
let videoGenerationCredential: VideoGenerationCredentialLease | null = null;
|
|
6623
|
+
if (objectStorage && videoGenerationEnabled) {
|
|
6624
|
+
if (videoGenerationPolicy.fundingSource === "opengeni_credits") {
|
|
6625
|
+
videoGenerationCredential = managedVideoGenerationCredentialLease(modelRunSettings);
|
|
6626
|
+
} else {
|
|
6627
|
+
const workspaceCredential = await loadWorkspaceVercelAiGatewayCredentialLease(
|
|
6628
|
+
db,
|
|
6629
|
+
modelRunSettings,
|
|
6630
|
+
input.workspaceId,
|
|
6631
|
+
);
|
|
6632
|
+
if (workspaceCredential) {
|
|
6633
|
+
videoGenerationCredential = {
|
|
6634
|
+
fundingSource: "workspace_gateway",
|
|
6635
|
+
...workspaceCredential,
|
|
6636
|
+
};
|
|
6637
|
+
}
|
|
6638
|
+
}
|
|
6639
|
+
}
|
|
6640
|
+
const videoGenerationOption: Pick<BuildAgentOptions, "videoGeneration"> = (() => {
|
|
6641
|
+
if (
|
|
6642
|
+
!objectStorage ||
|
|
6643
|
+
modelRunSettings.sandboxBackend === "none" ||
|
|
6644
|
+
!videoGenerationCredential ||
|
|
6645
|
+
!videoGenerationEnabled
|
|
6646
|
+
) {
|
|
6647
|
+
return {};
|
|
6648
|
+
}
|
|
6649
|
+
// Parse the frozen capability snapshot before advertising either tool.
|
|
6650
|
+
// Invalid or unsupported workspace policy therefore fails closed before
|
|
6651
|
+
// it can perturb the model's tool list.
|
|
6652
|
+
const capabilities = videoGenerationCapabilitiesForPolicy({
|
|
6653
|
+
policy: videoGenerationPolicy,
|
|
6654
|
+
credentialVersion: videoGenerationCredential.version,
|
|
6655
|
+
});
|
|
6656
|
+
return {
|
|
6657
|
+
videoGeneration: {
|
|
6658
|
+
capabilities: async () => capabilities,
|
|
6659
|
+
execute: async (toolInput, { toolCallId }) => {
|
|
6660
|
+
const sessionForReference =
|
|
6661
|
+
resolvedSandbox?.established.session ?? sdkOwnedSandboxSession;
|
|
6662
|
+
const fence = toolCancellationFenceRef.current;
|
|
6663
|
+
const runAs = sandboxRunAs(modelRunSettings);
|
|
6664
|
+
const accepted = await admitVideoGenerationRequest({
|
|
6665
|
+
db,
|
|
6666
|
+
storage: objectStorage,
|
|
6667
|
+
settings: modelRunSettings,
|
|
6668
|
+
accountId: input.accountId,
|
|
6669
|
+
workspaceId: input.workspaceId,
|
|
6670
|
+
sessionId: input.sessionId,
|
|
6671
|
+
turnId: turn.id,
|
|
6672
|
+
attemptId: input.attemptId,
|
|
6673
|
+
toolCallId,
|
|
6674
|
+
toolInput,
|
|
6675
|
+
policy: videoGenerationPolicy,
|
|
6676
|
+
credential: videoGenerationCredential,
|
|
6677
|
+
...(sessionForReference && fence
|
|
6678
|
+
? {
|
|
6679
|
+
runCommand: async (command) =>
|
|
6680
|
+
await fence.runSandboxCommandStructured(
|
|
6681
|
+
sessionForReference as import("@opengeni/runtime").TurnSandboxCommandSession,
|
|
6682
|
+
{
|
|
6683
|
+
...command,
|
|
6684
|
+
...(runAs ? { runAs } : {}),
|
|
6685
|
+
},
|
|
6686
|
+
),
|
|
6687
|
+
}
|
|
6688
|
+
: {}),
|
|
6689
|
+
...(runtimeCancellationSignal ? { signal: runtimeCancellationSignal } : {}),
|
|
6690
|
+
});
|
|
6691
|
+
videoGenerationAcceptancesByCallId.set(toolCallId, {
|
|
6692
|
+
operationId: accepted.operationId,
|
|
6693
|
+
requestDigest: accepted.requestDigest,
|
|
6694
|
+
});
|
|
6695
|
+
return accepted.receipt;
|
|
6696
|
+
},
|
|
6697
|
+
},
|
|
6698
|
+
};
|
|
6699
|
+
})();
|
|
5737
6700
|
const serviceTier = serviceTierForLatencyMode(
|
|
5738
6701
|
turnExecutionPolicy.providerId,
|
|
5739
6702
|
turnExecutionPolicy.latencyMode,
|
|
@@ -5769,11 +6732,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5769
6732
|
latencyMode: turnExecutionPolicy.latencyMode,
|
|
5770
6733
|
...(serviceTier ? { serviceTier } : {}),
|
|
5771
6734
|
...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
|
|
6735
|
+
humanInputEnabled: agentHumanInputEnabled,
|
|
5772
6736
|
genesisTitleHint: isGenesisTurn,
|
|
5773
6737
|
persistentSessionSettings: {
|
|
5774
6738
|
titleIsSet: Boolean(session.title?.trim()),
|
|
5775
6739
|
},
|
|
5776
6740
|
sandboxEnvironment,
|
|
6741
|
+
...(preparedTools.attemptToolCatalog
|
|
6742
|
+
? { attemptToolCatalog: preparedTools.attemptToolCatalog }
|
|
6743
|
+
: {}),
|
|
6744
|
+
...(sandboxArtifactRuntime.available ? { artifactRuntimeAvailable: true } : {}),
|
|
5777
6745
|
...(cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {}),
|
|
5778
6746
|
onToolCancellationFence: (fence) => {
|
|
5779
6747
|
toolCancellationFenceRef.current = fence;
|
|
@@ -5793,18 +6761,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5793
6761
|
...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
|
|
5794
6762
|
? { gitTokenSeed: sandboxGitToken }
|
|
5795
6763
|
: {}),
|
|
5796
|
-
//
|
|
5797
|
-
//
|
|
5798
|
-
//
|
|
5799
|
-
|
|
5800
|
-
// than toolspace:call for its own session (own-session-bound,
|
|
5801
|
-
// short-lived and renewable, per-active-turn budgeted, approval-tools
|
|
5802
|
-
// excluded). The runtime seeds it to the box's token file over the same
|
|
5803
|
-
// exec channel, off-manifest, on every backend.
|
|
5804
|
-
...(sandboxToolspaceToken
|
|
6764
|
+
// Codemode delivery is managed-sandbox-only. Connected Machines own any
|
|
6765
|
+
// manually configured API credentials and must never be contacted during
|
|
6766
|
+
// turn admission merely to seed OpenGeni tooling.
|
|
6767
|
+
...(sandboxCodemodeToken
|
|
5805
6768
|
? {
|
|
5806
|
-
|
|
5807
|
-
|
|
6769
|
+
codemodeTokenSeed: sandboxCodemodeToken,
|
|
6770
|
+
codemodeTokenSessionId: input.sessionId,
|
|
5808
6771
|
}
|
|
5809
6772
|
: {}),
|
|
5810
6773
|
...(activeSandboxBackend ? { activeSandboxBackend } : {}),
|
|
@@ -5831,6 +6794,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5831
6794
|
// session's MCP allow-list; the effective context window drives the
|
|
5832
6795
|
// compaction threshold.
|
|
5833
6796
|
hostedWebSearch,
|
|
6797
|
+
...imageGenerationOption,
|
|
6798
|
+
...videoGenerationOption,
|
|
6799
|
+
lazyToolTransport,
|
|
5834
6800
|
supportsImageInput,
|
|
5835
6801
|
inputFileMediaTypes: modelInputPolicy.inputFileMediaTypes,
|
|
5836
6802
|
...(resolvedModel
|
|
@@ -5840,15 +6806,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5840
6806
|
runSettings.openaiReasoningEncryptedContent,
|
|
5841
6807
|
contextWindowTokens:
|
|
5842
6808
|
resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
|
|
5843
|
-
// The ChatGPT/Codex backend rejects the SDK's HOSTED
|
|
5844
|
-
//
|
|
5845
|
-
//
|
|
5846
|
-
//
|
|
5847
|
-
//
|
|
5848
|
-
//
|
|
5849
|
-
// (built-in OpenAI/Azure = real hosted support; registry "chat"
|
|
5850
|
-
// providers = the SDK's own ChatCompletions detection) keeps the SDK
|
|
5851
|
-
// default.
|
|
6809
|
+
// The ChatGPT/Codex backend rejects the SDK's HOSTED apply_patch
|
|
6810
|
+
// tool. Gateway Responses routes likewise expose ordinary function
|
|
6811
|
+
// tools, not OpenAI-hosted sandbox tools. Tell buildAgent to use
|
|
6812
|
+
// function apply_patch and wrap successful view_image results as
|
|
6813
|
+
// typed input_image content. Chat wires have no proven typed image
|
|
6814
|
+
// result transport and therefore receive no view_image tool.
|
|
5852
6815
|
structuredToolTransport: structuredToolTransportForTurn(resolvedModel),
|
|
5853
6816
|
// EXPLICIT computer-use tool transport, derived from the resolved provider's
|
|
5854
6817
|
// authoritative wire identity (codex → function-image, chat → disabled,
|
|
@@ -5878,6 +6841,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5878
6841
|
didComputerUse = true;
|
|
5879
6842
|
await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
|
|
5880
6843
|
},
|
|
6844
|
+
onRetainableSessionImageOutput: retainSessionImageAtToolBoundary,
|
|
5881
6845
|
...(packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {}),
|
|
5882
6846
|
...(session.skills.length > 0 ? { sessionSkills: session.skills } : {}),
|
|
5883
6847
|
...(skillLibraryRuntime.skillLibrarySkills.length > 0
|
|
@@ -5919,6 +6883,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5919
6883
|
rigName,
|
|
5920
6884
|
script: rigVersion.setupScript,
|
|
5921
6885
|
timeoutMs: runSettings.rigSetupTimeoutMs,
|
|
6886
|
+
contentHash: rigProviderImageContentHash({
|
|
6887
|
+
backend: turn.sandboxBackend,
|
|
6888
|
+
sourceImage: rigProviderImageSourceImage(
|
|
6889
|
+
logicalSandboxSettings,
|
|
6890
|
+
turn.sandboxBackend,
|
|
6891
|
+
),
|
|
6892
|
+
definition: rigVersion,
|
|
6893
|
+
}),
|
|
5922
6894
|
},
|
|
5923
6895
|
}
|
|
5924
6896
|
: {}),
|
|
@@ -5957,13 +6929,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5957
6929
|
authorizeGitHubTokenMint,
|
|
5958
6930
|
});
|
|
5959
6931
|
const lazyGitTokens = lazyGitCredentials?.gitTokens;
|
|
5960
|
-
const
|
|
5961
|
-
? await
|
|
6932
|
+
const lazyCodemodeToken = sandboxCodemodeToken
|
|
6933
|
+
? await mintSandboxCodemodeToken(runSettings, connectionScope, codemodeAuthority)
|
|
5962
6934
|
: undefined;
|
|
5963
6935
|
const provisioned = await resumeBoxForTurn(
|
|
5964
6936
|
{
|
|
5965
6937
|
db,
|
|
5966
6938
|
settings: runSettings,
|
|
6939
|
+
logicalFallbackSettings: logicalSandboxSettings,
|
|
5967
6940
|
cancellationSignal: sandboxResumeSignal,
|
|
5968
6941
|
sandboxMetrics: runtimeMetricsHooksForObservability(observability),
|
|
5969
6942
|
onSandboxLost: publishSandboxLost,
|
|
@@ -6016,9 +6989,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6016
6989
|
gitCredentialBindingsOverride: lazyGitCredentials.bindings,
|
|
6017
6990
|
}
|
|
6018
6991
|
: {}),
|
|
6019
|
-
...(
|
|
6992
|
+
...(lazyCodemodeToken
|
|
6020
6993
|
? {
|
|
6021
|
-
|
|
6994
|
+
codemodeTokenSeedOverride: lazyCodemodeToken.token,
|
|
6022
6995
|
}
|
|
6023
6996
|
: {}),
|
|
6024
6997
|
...(toolCancellationFenceRef.current
|
|
@@ -6031,9 +7004,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6031
7004
|
},
|
|
6032
7005
|
),
|
|
6033
7006
|
);
|
|
6034
|
-
await
|
|
6035
|
-
provisioned.established.session as
|
|
6036
|
-
|
|
7007
|
+
await attachCodemodeTokenRenewal(
|
|
7008
|
+
provisioned.established.session as CodemodeTokenWriterSession,
|
|
7009
|
+
lazyCodemodeToken?.expiresAt,
|
|
6037
7010
|
provisioned,
|
|
6038
7011
|
);
|
|
6039
7012
|
await attachGitCredentialRenewal(
|
|
@@ -6083,6 +7056,17 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6083
7056
|
startLeaseHeartbeat(provisioned, activeSandboxBackend ?? groupBoxBackend);
|
|
6084
7057
|
setupBoxSession = provisioned.established.session;
|
|
6085
7058
|
resolvedSandbox = provisioned;
|
|
7059
|
+
// `get()` does not release the first routed sandbox operation
|
|
7060
|
+
// until this hook returns. Materialize durable generated images
|
|
7061
|
+
// here so that operation can use their advertised paths without
|
|
7062
|
+
// racing a best-effort background copy.
|
|
7063
|
+
for (const receipt of generatedImageReceiptsByArtifactId.values()) {
|
|
7064
|
+
await materializeGeneratedImageInSandbox(
|
|
7065
|
+
receipt,
|
|
7066
|
+
provisioned,
|
|
7067
|
+
provisioned.established.session,
|
|
7068
|
+
);
|
|
7069
|
+
}
|
|
6086
7070
|
},
|
|
6087
7071
|
onFailed: async (error) => {
|
|
6088
7072
|
await publish?.(
|
|
@@ -6099,7 +7083,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6099
7083
|
);
|
|
6100
7084
|
},
|
|
6101
7085
|
disposeResult: async (provisioned) => {
|
|
6102
|
-
await provisioned
|
|
7086
|
+
await releaseLateSandbox(provisioned);
|
|
6103
7087
|
},
|
|
6104
7088
|
},
|
|
6105
7089
|
);
|
|
@@ -6108,10 +7092,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6108
7092
|
db,
|
|
6109
7093
|
settings,
|
|
6110
7094
|
bus,
|
|
7095
|
+
opJournal,
|
|
6111
7096
|
onOp: machineOpObserver.observer,
|
|
6112
7097
|
onSandboxOperation: sandboxOperationObserver,
|
|
6113
7098
|
onHomeSandboxLost: publishSandboxLost,
|
|
6114
7099
|
onHomeSandboxRebound,
|
|
7100
|
+
...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
|
|
6115
7101
|
},
|
|
6116
7102
|
{
|
|
6117
7103
|
workspaceId: input.workspaceId,
|
|
@@ -6206,7 +7192,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6206
7192
|
},
|
|
6207
7193
|
);
|
|
6208
7194
|
if (!isCompactionSummaryFailure(error)) throw error;
|
|
6209
|
-
const errorMessage = String(
|
|
7195
|
+
const errorMessage = String(compactionFailureReasonFromError(error));
|
|
6210
7196
|
if (
|
|
6211
7197
|
!(await settle!({
|
|
6212
7198
|
events: [
|
|
@@ -6308,11 +7294,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6308
7294
|
force: true,
|
|
6309
7295
|
clearRequestedCompaction: true,
|
|
6310
7296
|
trigger: "operator",
|
|
7297
|
+
materializeHistory: materializeScreenshotHistory,
|
|
6311
7298
|
projectModelInput: compactionModelHistoryProjector,
|
|
6312
7299
|
...compactionModeOptions,
|
|
6313
7300
|
}
|
|
6314
7301
|
: {
|
|
6315
7302
|
trigger: "auto",
|
|
7303
|
+
materializeHistory: materializeScreenshotHistory,
|
|
6316
7304
|
projectModelInput: compactionModelHistoryProjector,
|
|
6317
7305
|
...compactionModeOptions,
|
|
6318
7306
|
},
|
|
@@ -6346,11 +7334,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6346
7334
|
},
|
|
6347
7335
|
);
|
|
6348
7336
|
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
6349
|
-
const errorMessage = String(
|
|
7337
|
+
const errorMessage = String(compactionFailureReasonFromError(compactError));
|
|
6350
7338
|
observability.error("context compaction failed", {
|
|
6351
7339
|
sessionId: input.sessionId,
|
|
6352
7340
|
turnId,
|
|
6353
|
-
|
|
7341
|
+
...safeErrorDiagnostic(compactError),
|
|
6354
7342
|
});
|
|
6355
7343
|
if (
|
|
6356
7344
|
!(await settle!({
|
|
@@ -6412,7 +7400,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6412
7400
|
"fileMaterialization",
|
|
6413
7401
|
async () =>
|
|
6414
7402
|
await materializeSandboxFileDownloads(
|
|
6415
|
-
setupBoxSession as
|
|
7403
|
+
setupBoxSession as CodemodeTokenWriterSession,
|
|
6416
7404
|
downloadsToMaterialize,
|
|
6417
7405
|
{
|
|
6418
7406
|
onRuntimeEvent: async (event) => {
|
|
@@ -6447,6 +7435,37 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6447
7435
|
}
|
|
6448
7436
|
fileDownloadsMaterializedForRun = true;
|
|
6449
7437
|
}
|
|
7438
|
+
const requiredVideoFileIds = new Set(requiredGeneratedVideoFiles.map((file) => file.fileId));
|
|
7439
|
+
const failedRequiredVideo = fileMaterializationFailures.find((failure) =>
|
|
7440
|
+
requiredVideoFileIds.has(failure.fileId),
|
|
7441
|
+
);
|
|
7442
|
+
if (failedRequiredVideo) {
|
|
7443
|
+
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
7444
|
+
sessionId: input.sessionId,
|
|
7445
|
+
turnId: turn.id,
|
|
7446
|
+
triggerEventId: triggerEventId!,
|
|
7447
|
+
attemptId: input.attemptId,
|
|
7448
|
+
reason: "generated_video_materialization_failed",
|
|
7449
|
+
detail: {
|
|
7450
|
+
retryable: true,
|
|
7451
|
+
operationIds: requiredGeneratedVideoFiles.map((file) => file.operationId),
|
|
7452
|
+
},
|
|
7453
|
+
});
|
|
7454
|
+
if (recovery.action === "stale") {
|
|
7455
|
+
acknowledgeLostAttemptOwnership();
|
|
7456
|
+
activityStatus = "cancelled";
|
|
7457
|
+
turnMetricOutcome = "cancelled";
|
|
7458
|
+
return claimedResult({ status: "cancelled" });
|
|
7459
|
+
}
|
|
7460
|
+
if (recovery.action !== "recovering") {
|
|
7461
|
+
throw new Error("Generated video materialization could not enter recovery");
|
|
7462
|
+
}
|
|
7463
|
+
acknowledgeRecoveryQuiescence();
|
|
7464
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
7465
|
+
activityStatus = "recovering";
|
|
7466
|
+
turnMetricOutcome = "recovering";
|
|
7467
|
+
return claimedResult({ status: "recovering", continueDelayMs: 1_000 });
|
|
7468
|
+
}
|
|
6450
7469
|
const unavailableSandboxFilesNote = sandboxFileDownloadFailureNote(
|
|
6451
7470
|
fileMaterializationFailures,
|
|
6452
7471
|
);
|
|
@@ -6465,8 +7484,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6465
7484
|
...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
|
|
6466
7485
|
...(runCredentialsNote ? { runCredentialsNote } : {}),
|
|
6467
7486
|
providerApi,
|
|
7487
|
+
projectCanonicalHistory: generatedImageHistoryProjector,
|
|
7488
|
+
materializeModelHistory: materializeScreenshotHistory,
|
|
7489
|
+
materializeSerializedRunState: materializeScreenshotRunState,
|
|
6468
7490
|
projectModelHistory: modelHistoryProjector,
|
|
6469
7491
|
});
|
|
7492
|
+
for (const receipt of generatedImageReceiptsByArtifactId.values()) {
|
|
7493
|
+
await materializeGeneratedImage(receipt);
|
|
7494
|
+
}
|
|
6470
7495
|
runInput = prepared.input;
|
|
6471
7496
|
providerArtifactCandidates = prepared.providerArtifactCandidates;
|
|
6472
7497
|
// Slice index = the length of the model-facing (active) history this turn
|
|
@@ -6478,18 +7503,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6478
7503
|
// total rows no longer equal max(position)+1. Pre-compaction both reduce to
|
|
6479
7504
|
// the old total-count value, so the common path is unchanged.
|
|
6480
7505
|
//
|
|
6481
|
-
// CRITICAL: seed from the
|
|
7506
|
+
// CRITICAL: seed from the structurally repaired active-row length, not the raw active
|
|
6482
7507
|
// count. `prepareRunInput` builds `state.history` from
|
|
6483
7508
|
// `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
|
|
6484
7509
|
// rows (a legacy orphan/dangling pair), the in-memory history this turn
|
|
6485
7510
|
// starts from is K shorter than the raw row count. The reconcile slices the
|
|
6486
|
-
//
|
|
7511
|
+
// repaired `state.history` off `persistedHistoryCount`; seeding it from
|
|
6487
7512
|
// the raw count (K too high) skips K genuinely-new items, and a
|
|
6488
7513
|
// `function_call` left in that skipped region can later have its
|
|
6489
7514
|
// `function_call_result` persisted alone — the orphan that 400s on replay
|
|
6490
|
-
// and bricks the session (issue-61). The
|
|
7515
|
+
// and bricks the session (issue-61). The repaired seed is already
|
|
6491
7516
|
// orphan-free, so it is a stable prefix of the re-sanitized history and the
|
|
6492
7517
|
// slice begins exactly at the first genuinely-new item.
|
|
7518
|
+
// prepareInput already sanitized the exact durable prefix represented
|
|
7519
|
+
// by state.history. Carry its count forward instead of loading and
|
|
7520
|
+
// retaining the full active transcript a second time beside runInput.
|
|
6493
7521
|
persistedHistoryCount = prepared.persistedHistoryCount;
|
|
6494
7522
|
nextHistoryPosition = await nextSessionHistoryPosition(
|
|
6495
7523
|
db,
|
|
@@ -6525,6 +7553,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6525
7553
|
force: true,
|
|
6526
7554
|
...(triggerLabel === "operator" ? { clearRequestedCompaction: true } : {}),
|
|
6527
7555
|
trigger: triggerLabel,
|
|
7556
|
+
materializeHistory: materializeScreenshotHistory,
|
|
6528
7557
|
projectModelInput: compactionModelHistoryProjector,
|
|
6529
7558
|
...compactionModeOptions,
|
|
6530
7559
|
},
|
|
@@ -6573,6 +7602,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6573
7602
|
// its first-operation provisioner above.
|
|
6574
7603
|
if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
|
|
6575
7604
|
const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
|
|
7605
|
+
// `deferredSetup: true` below tells the runtime that the worker owns
|
|
7606
|
+
// platform setup, so the runtime intentionally skips its credential
|
|
7607
|
+
// session callback. Materialize host-managed run credentials here
|
|
7608
|
+
// before any setup command (including provider login hooks), then
|
|
7609
|
+
// decorate setup commands so they source the active generation.
|
|
7610
|
+
await attachRunCredentialRenewal(
|
|
7611
|
+
eagerSetupSession as RunCredentialCommandSession,
|
|
7612
|
+
resolvedSandbox,
|
|
7613
|
+
);
|
|
7614
|
+
const eagerCredentialSetupSession = initialRunCredentialMaterial
|
|
7615
|
+
? withRunCredentialsSession(eagerSetupSession as object, input.sessionId)
|
|
7616
|
+
: eagerSetupSession;
|
|
6576
7617
|
await runWorkspaceMutationForSandbox(
|
|
6577
7618
|
resolvedSandbox,
|
|
6578
7619
|
"eagerOwnedSandboxSetup",
|
|
@@ -6580,7 +7621,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6580
7621
|
await runOwnedSandboxSetup(
|
|
6581
7622
|
agent,
|
|
6582
7623
|
ownedEstablished.session as never,
|
|
6583
|
-
|
|
7624
|
+
eagerCredentialSetupSession as never,
|
|
6584
7625
|
{
|
|
6585
7626
|
settings: runSettings,
|
|
6586
7627
|
environment: sandboxEnvironment,
|
|
@@ -6646,16 +7687,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6646
7687
|
},
|
|
6647
7688
|
}
|
|
6648
7689
|
: {}),
|
|
6649
|
-
...(
|
|
7690
|
+
...(sandboxCodemodeToken && sandboxCodemodeTokenExpiresAt && !lazyOwnedSandbox
|
|
6650
7691
|
? {
|
|
6651
|
-
|
|
6652
|
-
tokenSession: ToolspaceTokenWriterSession,
|
|
6653
|
-
) => {
|
|
7692
|
+
onCodemodeTokenSessionReady: async (tokenSession: CodemodeTokenWriterSession) => {
|
|
6654
7693
|
const renewalSession =
|
|
6655
|
-
|
|
6656
|
-
|
|
6657
|
-
: ((setupBoxSession as ToolspaceTokenWriterSession | null) ?? tokenSession);
|
|
6658
|
-
await attachToolspaceTokenRenewal(renewalSession);
|
|
7694
|
+
(setupBoxSession as CodemodeTokenWriterSession | null) ?? tokenSession;
|
|
7695
|
+
await attachCodemodeTokenRenewal(renewalSession);
|
|
6659
7696
|
},
|
|
6660
7697
|
}
|
|
6661
7698
|
: {}),
|
|
@@ -6676,10 +7713,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6676
7713
|
: {}),
|
|
6677
7714
|
}
|
|
6678
7715
|
: {}),
|
|
7716
|
+
...(modelRunSettings.sandboxBackend !== "none"
|
|
7717
|
+
? {
|
|
7718
|
+
onSandboxSessionReady: async (sandboxSession: CodemodeTokenWriterSession) => {
|
|
7719
|
+
sdkOwnedSandboxSession = sandboxSession;
|
|
7720
|
+
for (const receipt of generatedImageReceiptsByArtifactId.values()) {
|
|
7721
|
+
await materializeGeneratedImageInOwnedSdkSession(receipt, sandboxSession);
|
|
7722
|
+
}
|
|
7723
|
+
},
|
|
7724
|
+
}
|
|
7725
|
+
: {}),
|
|
6679
7726
|
contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
|
|
6680
7727
|
contextCompactionRequested: () =>
|
|
6681
7728
|
isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
6682
|
-
callModelInputFilter: secretRedactionModelInputFilter((value) => redact(value)),
|
|
6683
7729
|
...(toolCancellationFenceRef.current
|
|
6684
7730
|
? { turnToolCancellationFence: toolCancellationFenceRef.current }
|
|
6685
7731
|
: {}),
|
|
@@ -6717,6 +7763,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6717
7763
|
}
|
|
6718
7764
|
let stableToolCallIdsToClear: string[] | null = null;
|
|
6719
7765
|
let completedCurrentToolBatch = false;
|
|
7766
|
+
let retainedScreenshotMetadata: RetainedArtifactMetadata | null = null;
|
|
7767
|
+
const generatedImage = generatedImageFromSdkEvent(next.value);
|
|
7768
|
+
if (isCompletedGeneratedImageSdkEvent(next.value) && !generatedImage) {
|
|
7769
|
+
throw new Error(
|
|
7770
|
+
"Completed native image item could not cross the retained-artifact boundary",
|
|
7771
|
+
);
|
|
7772
|
+
}
|
|
7773
|
+
const generatedImageReceipt = generatedImage
|
|
7774
|
+
? await retainNativeGeneratedImage(generatedImage)
|
|
7775
|
+
: null;
|
|
6720
7776
|
const responseResult = await processModelResponseTerminalEvent({
|
|
6721
7777
|
event: next.value,
|
|
6722
7778
|
state: modelResponseState,
|
|
@@ -6751,10 +7807,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6751
7807
|
...(resolvedModel?.provider.id ? { providerId: resolvedModel.provider.id } : {}),
|
|
6752
7808
|
});
|
|
6753
7809
|
if (responseResult.status === "processed") {
|
|
7810
|
+
const rawStreamHistory = (stream.state as { history?: unknown[] }).history;
|
|
7811
|
+
if (Array.isArray(rawStreamHistory)) {
|
|
7812
|
+
// The completed image item is normally retained from its own
|
|
7813
|
+
// run-item event above. Scan once at the terminal response as
|
|
7814
|
+
// a compatibility backstop before reconciliation can persist
|
|
7815
|
+
// provider bytes. Never rescan history for every token delta.
|
|
7816
|
+
await retainNativeGeneratedImagesFromHistory(
|
|
7817
|
+
rawStreamHistory as Array<Record<string, unknown>>,
|
|
7818
|
+
);
|
|
7819
|
+
}
|
|
6754
7820
|
streamSawPerResponseUsage ||= responseResult.usageReported;
|
|
6755
7821
|
currentToolBatchCallIds = new Set<string>();
|
|
6756
7822
|
currentToolBatchCompletedCallIds = new Set<string>();
|
|
6757
7823
|
await reconcileConversationTruth();
|
|
7824
|
+
modelCheckpointMemoryCollector.schedule(observability);
|
|
6758
7825
|
try {
|
|
6759
7826
|
await ensureRunAllowed(
|
|
6760
7827
|
settings,
|
|
@@ -6770,7 +7837,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6770
7837
|
// conversation context preserved for the post-top-up resume.
|
|
6771
7838
|
let serializedRunState: string | null = null;
|
|
6772
7839
|
try {
|
|
6773
|
-
serializedRunState = String(
|
|
7840
|
+
serializedRunState = compactMediaRunState(String(stream.state.toString()));
|
|
6774
7841
|
} catch {
|
|
6775
7842
|
serializedRunState = null;
|
|
6776
7843
|
}
|
|
@@ -6780,7 +7847,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6780
7847
|
);
|
|
6781
7848
|
}
|
|
6782
7849
|
}
|
|
6783
|
-
const
|
|
7850
|
+
const durableSdkEvent = generatedImageReceipt
|
|
7851
|
+
? compactGeneratedImageSdkEvent(next.value, generatedImageReceipt)
|
|
7852
|
+
: next.value;
|
|
7853
|
+
const pendingToolCall = pendingToolCallFromSdkEvent(durableSdkEvent);
|
|
6784
7854
|
if (pendingToolCall) {
|
|
6785
7855
|
const registered = await registerPendingSessionToolCall(db, {
|
|
6786
7856
|
accountId: input.accountId,
|
|
@@ -6792,7 +7862,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6792
7862
|
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
6793
7863
|
callId: pendingToolCall.callId,
|
|
6794
7864
|
callType: pendingToolCall.callType,
|
|
6795
|
-
callItem:
|
|
7865
|
+
callItem: pendingToolCall.callItem as Record<string, unknown>,
|
|
6796
7866
|
});
|
|
6797
7867
|
if (!registered.accepted) {
|
|
6798
7868
|
throw new TurnAttemptFencedError(
|
|
@@ -6800,9 +7870,78 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6800
7870
|
);
|
|
6801
7871
|
}
|
|
6802
7872
|
currentToolBatchCallIds.add(pendingToolCall.callId);
|
|
7873
|
+
if (toolCallProducesRetainableSessionImage(pendingToolCall.callName)) {
|
|
7874
|
+
retainedSessionImageCallIds.add(pendingToolCall.callId);
|
|
7875
|
+
}
|
|
6803
7876
|
}
|
|
6804
|
-
const completedToolCall = completedToolCallFromSdkEvent(
|
|
7877
|
+
const completedToolCall = completedToolCallFromSdkEvent(durableSdkEvent);
|
|
6805
7878
|
if (completedToolCall) {
|
|
7879
|
+
retainedScreenshotMetadata =
|
|
7880
|
+
retainedScreenshotReceiptsByCallId.get(completedToolCall.callId) ?? null;
|
|
7881
|
+
const typedScreenshot = retainedScreenshotMetadata
|
|
7882
|
+
? null
|
|
7883
|
+
: typedScreenshotFromSdkEvent(durableSdkEvent);
|
|
7884
|
+
if (
|
|
7885
|
+
!retainedScreenshotMetadata &&
|
|
7886
|
+
retainedSessionImageCallIds.has(completedToolCall.callId) &&
|
|
7887
|
+
sdkEventContainsInlineImage(durableSdkEvent) &&
|
|
7888
|
+
!typedScreenshot
|
|
7889
|
+
) {
|
|
7890
|
+
retainedScreenshotMetadata = unavailableRetainedSessionImage({
|
|
7891
|
+
sessionId: input.sessionId,
|
|
7892
|
+
turnId: activeTurnId,
|
|
7893
|
+
attemptId: input.attemptId,
|
|
7894
|
+
toolCallId: completedToolCall.callId,
|
|
7895
|
+
toolOutputId: completedToolCall.callId,
|
|
7896
|
+
reason: "unsupported",
|
|
7897
|
+
});
|
|
7898
|
+
retainedScreenshotReceiptsByCallId.set(
|
|
7899
|
+
completedToolCall.callId,
|
|
7900
|
+
retainedScreenshotMetadata,
|
|
7901
|
+
);
|
|
7902
|
+
}
|
|
7903
|
+
if (
|
|
7904
|
+
!retainedScreenshotMetadata &&
|
|
7905
|
+
typedScreenshot &&
|
|
7906
|
+
typedScreenshot.callId === completedToolCall.callId &&
|
|
7907
|
+
retainedSessionImageCallIds.has(completedToolCall.callId)
|
|
7908
|
+
) {
|
|
7909
|
+
// Install a compact deterministic fallback before database or
|
|
7910
|
+
// object-storage I/O. Unexpected retention failure can then
|
|
7911
|
+
// reconcile/serialize without leaking inline bytes.
|
|
7912
|
+
retainedScreenshotMetadata = unavailableRetainedSessionImage({
|
|
7913
|
+
sessionId: input.sessionId,
|
|
7914
|
+
turnId: activeTurnId,
|
|
7915
|
+
attemptId: input.attemptId,
|
|
7916
|
+
toolCallId: typedScreenshot.callId,
|
|
7917
|
+
toolOutputId: typedScreenshot.toolOutputId,
|
|
7918
|
+
reason: "pending",
|
|
7919
|
+
});
|
|
7920
|
+
retainedScreenshotReceiptsByCallId.set(
|
|
7921
|
+
completedToolCall.callId,
|
|
7922
|
+
retainedScreenshotMetadata,
|
|
7923
|
+
);
|
|
7924
|
+
retainedScreenshotMetadata = await retainComputerScreenshot({
|
|
7925
|
+
db,
|
|
7926
|
+
objectStorage,
|
|
7927
|
+
accountId: input.accountId,
|
|
7928
|
+
workspaceId: input.workspaceId,
|
|
7929
|
+
sessionId: input.sessionId,
|
|
7930
|
+
turnId: activeTurnId,
|
|
7931
|
+
attemptId: input.attemptId,
|
|
7932
|
+
output: typedScreenshot,
|
|
7933
|
+
});
|
|
7934
|
+
retainedScreenshotReceiptsByCallId.set(
|
|
7935
|
+
completedToolCall.callId,
|
|
7936
|
+
retainedScreenshotMetadata,
|
|
7937
|
+
);
|
|
7938
|
+
}
|
|
7939
|
+
const durableResultItem = retainedScreenshotMetadata
|
|
7940
|
+
? (compactRetainedScreenshotHistory(
|
|
7941
|
+
[completedToolCall.resultItem],
|
|
7942
|
+
retainedScreenshotReceiptsByCallId,
|
|
7943
|
+
)[0] ?? completedToolCall.resultItem)
|
|
7944
|
+
: completedToolCall.resultItem;
|
|
6806
7945
|
// Keep every parallel result in the attempt ledger until the full
|
|
6807
7946
|
// call batch has settled. The SDK's computed history is
|
|
6808
7947
|
// non-monotonic while parallel calls complete (a later call may
|
|
@@ -6817,13 +7956,39 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6817
7956
|
attemptId: input.attemptId,
|
|
6818
7957
|
callId: completedToolCall.callId,
|
|
6819
7958
|
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
6820
|
-
resultItem:
|
|
7959
|
+
resultItem: durableResultItem as Record<string, unknown>,
|
|
7960
|
+
...(videoGenerationAcceptancesByCallId.has(completedToolCall.callId)
|
|
7961
|
+
? {
|
|
7962
|
+
videoGenerationAcceptance: videoGenerationAcceptancesByCallId.get(
|
|
7963
|
+
completedToolCall.callId,
|
|
7964
|
+
)!,
|
|
7965
|
+
}
|
|
7966
|
+
: {}),
|
|
6821
7967
|
});
|
|
6822
7968
|
if (!recorded.accepted) {
|
|
6823
7969
|
throw new TurnAttemptFencedError(
|
|
6824
7970
|
"turn attempt ended while recording a tool-call result",
|
|
6825
7971
|
);
|
|
6826
7972
|
}
|
|
7973
|
+
const videoAcceptance = videoGenerationAcceptancesByCallId.get(
|
|
7974
|
+
completedToolCall.callId,
|
|
7975
|
+
);
|
|
7976
|
+
if (videoAcceptance && startVideoGenerationWorkflow) {
|
|
7977
|
+
try {
|
|
7978
|
+
await startVideoGenerationWorkflow({
|
|
7979
|
+
accountId: input.accountId,
|
|
7980
|
+
workspaceId: input.workspaceId,
|
|
7981
|
+
operationId: videoAcceptance.operationId,
|
|
7982
|
+
});
|
|
7983
|
+
} catch (error) {
|
|
7984
|
+
// The accepted operation is durable. The recovery sweep starts
|
|
7985
|
+
// the same deterministic workflow ID if this nudge fails.
|
|
7986
|
+
observability.warn("Video generation workflow start deferred", {
|
|
7987
|
+
operationId: videoAcceptance.operationId,
|
|
7988
|
+
errorClass: error instanceof Error ? error.name : "UnknownError",
|
|
7989
|
+
});
|
|
7990
|
+
}
|
|
7991
|
+
}
|
|
6827
7992
|
const belongsToCurrentBatch = currentToolBatchCallIds.has(completedToolCall.callId);
|
|
6828
7993
|
if (belongsToCurrentBatch) {
|
|
6829
7994
|
currentToolBatchCompletedCallIds.add(completedToolCall.callId);
|
|
@@ -6846,7 +8011,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6846
8011
|
completedCurrentToolBatch = currentBatchIsStable;
|
|
6847
8012
|
}
|
|
6848
8013
|
}
|
|
6849
|
-
const normalized = normalizeSdkEvent(
|
|
8014
|
+
const normalized = normalizeSdkEvent(
|
|
8015
|
+
durableSdkEvent as typeof next.value,
|
|
8016
|
+
retainedScreenshotMetadata
|
|
8017
|
+
? {
|
|
8018
|
+
toolOutputOverride: retainedScreenshotMetadata,
|
|
8019
|
+
retainedOutputEvidence: retainedScreenshotMetadata.available
|
|
8020
|
+
? retainedScreenshotMetadata
|
|
8021
|
+
: {
|
|
8022
|
+
available: false,
|
|
8023
|
+
reason: retainedScreenshotMetadata.reason,
|
|
8024
|
+
},
|
|
8025
|
+
}
|
|
8026
|
+
: {},
|
|
8027
|
+
);
|
|
6850
8028
|
for (const event of normalized) {
|
|
6851
8029
|
streamTiming.onEvent(event.type);
|
|
6852
8030
|
await batcher.push(event);
|
|
@@ -6870,6 +8048,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6870
8048
|
currentToolBatchCallIds = new Set<string>();
|
|
6871
8049
|
currentToolBatchCompletedCallIds = new Set<string>();
|
|
6872
8050
|
}
|
|
8051
|
+
for (const callId of stableToolCallIdsToClear) {
|
|
8052
|
+
retainedSessionImageCallIds.delete(callId);
|
|
8053
|
+
}
|
|
6873
8054
|
}
|
|
6874
8055
|
}
|
|
6875
8056
|
} finally {
|
|
@@ -6983,6 +8164,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6983
8164
|
const approvals = runtime.serializeApprovals(stream.interruptions);
|
|
6984
8165
|
const humanInputInterruptions =
|
|
6985
8166
|
runtime.serializeHumanInputRequests?.(stream.interruptions) ?? [];
|
|
8167
|
+
const latestWorkspace = await getWorkspace(db, input.workspaceId);
|
|
8168
|
+
if (!latestWorkspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
|
|
8169
|
+
assertWorkspaceHumanInputAllowed(
|
|
8170
|
+
resolveWorkspaceAgentHumanInputEnabled(latestWorkspace.settings),
|
|
8171
|
+
"interruption",
|
|
8172
|
+
humanInputInterruptions.length > 0,
|
|
8173
|
+
);
|
|
6986
8174
|
const humanInputRequests = await Promise.all(
|
|
6987
8175
|
humanInputInterruptions.map(async (interruption) => {
|
|
6988
8176
|
const id = stableHumanInputRequestId(
|
|
@@ -7048,7 +8236,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7048
8236
|
sessionStatus: "requires_action",
|
|
7049
8237
|
activeTurnId,
|
|
7050
8238
|
runState: {
|
|
7051
|
-
serializedRunState: stream.state.toString(),
|
|
8239
|
+
serializedRunState: compactMediaRunState(stream.state.toString()),
|
|
7052
8240
|
pendingApprovals: approvals,
|
|
7053
8241
|
humanInputRequests: humanInputRequests.map(
|
|
7054
8242
|
({ isNew: _isNew, ...request }) => request,
|
|
@@ -7058,25 +8246,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7058
8246
|
) {
|
|
7059
8247
|
return claimedResult({ status: "cancelled" });
|
|
7060
8248
|
}
|
|
8249
|
+
// The interruption and its preceding tool results are now durable.
|
|
8250
|
+
await finalizeTurnOpStreamOps();
|
|
7061
8251
|
activityStatus = "requires_action";
|
|
7062
8252
|
return claimedResult({ status: "requires_action" });
|
|
7063
8253
|
}
|
|
7064
8254
|
|
|
7065
|
-
const finalOutput = String(
|
|
7066
|
-
await reconcileConversationTruth();
|
|
8255
|
+
const finalOutput = String(stream.finalOutput ?? "");
|
|
8256
|
+
await reconcileConversationTruth({ requireDurable: true });
|
|
7067
8257
|
// Op-stream durability fence: the tool outputs are now durably in the
|
|
7068
8258
|
// history store (a redispatch would NOT re-execute them), so this
|
|
7069
8259
|
// turn's settled ops may advance their acked frontier — journal persist
|
|
7070
8260
|
// then wire final ack (licensing the runner to GC its retained
|
|
7071
8261
|
// frames). Best-effort: a miss leaves the runner's retention TTL to
|
|
7072
8262
|
// reap, never fails a completed turn.
|
|
7073
|
-
|
|
7074
|
-
try {
|
|
7075
|
-
await machinePrimarySession.finalizeOpStreamOps();
|
|
7076
|
-
} catch {
|
|
7077
|
-
// The runner's retention TTL owns the fallback.
|
|
7078
|
-
}
|
|
7079
|
-
}
|
|
8263
|
+
await finalizeTurnOpStreamOps();
|
|
7080
8264
|
if (
|
|
7081
8265
|
!(await settle!({
|
|
7082
8266
|
events: [
|
|
@@ -7143,8 +8327,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7143
8327
|
sessionId: input.sessionId,
|
|
7144
8328
|
turnId: activeTurnId,
|
|
7145
8329
|
reason: recoveryKind,
|
|
7146
|
-
|
|
7147
|
-
error: String(redact(overflow?.message ?? compactionNeeded?.message ?? "")),
|
|
8330
|
+
...safeErrorDiagnostic(attemptError),
|
|
7148
8331
|
signalTokens: compactionNeeded?.signalTokens,
|
|
7149
8332
|
thresholdTokens: compactionNeeded?.thresholdTokens,
|
|
7150
8333
|
});
|
|
@@ -7189,13 +8372,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7189
8372
|
);
|
|
7190
8373
|
compactionRequestCleared = landmark.requestConsumed;
|
|
7191
8374
|
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
7192
|
-
compactionFailureMessage = String(
|
|
7193
|
-
redact(compactionFailureReasonFromError(compactError)),
|
|
7194
|
-
);
|
|
8375
|
+
compactionFailureMessage = String(compactionFailureReasonFromError(compactError));
|
|
7195
8376
|
observability.warn("context compaction recovery compaction failed", {
|
|
7196
8377
|
sessionId: input.sessionId,
|
|
7197
8378
|
turnId: activeTurnId,
|
|
7198
|
-
|
|
8379
|
+
...safeErrorDiagnostic(compactError),
|
|
7199
8380
|
});
|
|
7200
8381
|
}
|
|
7201
8382
|
if (!compactionHandled) {
|
|
@@ -7271,15 +8452,24 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7271
8452
|
// back to the workflow-claimed turn when the local lookup had not
|
|
7272
8453
|
// finished yet.
|
|
7273
8454
|
const recoveryTurnId = turnId;
|
|
7274
|
-
//
|
|
7275
|
-
//
|
|
7276
|
-
//
|
|
7277
|
-
|
|
8455
|
+
// A true epoch supersession and a provider lifecycle transition are both
|
|
8456
|
+
// recoverable control-plane states, never session failures. The latter is
|
|
8457
|
+
// paced briefly; the next acquire waits on the durable claim and resumes
|
|
8458
|
+
// immediately when it clears.
|
|
8459
|
+
if (
|
|
8460
|
+
(error instanceof SandboxLeaseSupersededError ||
|
|
8461
|
+
error instanceof SandboxLeaseTransitionError) &&
|
|
8462
|
+
recoveryTurnId
|
|
8463
|
+
) {
|
|
7278
8464
|
try {
|
|
7279
8465
|
const fencedLease = await readLease(db, input.workspaceId, error.sandboxGroupId).catch(
|
|
7280
8466
|
() => null,
|
|
7281
8467
|
);
|
|
7282
|
-
const
|
|
8468
|
+
const lifecycleTransition = error instanceof SandboxLeaseTransitionError;
|
|
8469
|
+
const rotationPending =
|
|
8470
|
+
fencedLease?.rotationRequestedAt != null ||
|
|
8471
|
+
(lifecycleTransition && error.reason === "rotation_in_progress");
|
|
8472
|
+
const transitionPending = lifecycleTransition || rotationPending;
|
|
7283
8473
|
const deadlineRotationPending =
|
|
7284
8474
|
rotationPending && fencedLease?.rotationReason === "provider_deadline";
|
|
7285
8475
|
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
@@ -7289,13 +8479,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7289
8479
|
attemptId: input.attemptId,
|
|
7290
8480
|
reason: deadlineRotationPending
|
|
7291
8481
|
? "sandbox_deadline_rotation"
|
|
7292
|
-
:
|
|
7293
|
-
|
|
8482
|
+
: lifecycleTransition
|
|
8483
|
+
? "sandbox_lifecycle_transition"
|
|
8484
|
+
: "sandbox_lease_superseded",
|
|
8485
|
+
...(transitionPending
|
|
7294
8486
|
? {
|
|
7295
8487
|
detail: {
|
|
7296
8488
|
sandboxGroupId: error.sandboxGroupId,
|
|
7297
8489
|
leaseEpoch: error.leaseEpoch,
|
|
7298
|
-
|
|
8490
|
+
...(rotationPending
|
|
8491
|
+
? {
|
|
8492
|
+
rotationReason: fencedLease?.rotationReason ?? "operator",
|
|
8493
|
+
}
|
|
8494
|
+
: {}),
|
|
8495
|
+
...(lifecycleTransition ? { transitionReason: error.reason } : {}),
|
|
7299
8496
|
},
|
|
7300
8497
|
}
|
|
7301
8498
|
: {}),
|
|
@@ -7306,6 +8503,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7306
8503
|
turnMetricOutcome = "cancelled";
|
|
7307
8504
|
return claimedResult({ status: "cancelled" });
|
|
7308
8505
|
}
|
|
8506
|
+
acknowledgeRecoveryQuiescence();
|
|
7309
8507
|
await publishDurableSessionEvents(
|
|
7310
8508
|
bus,
|
|
7311
8509
|
input.workspaceId,
|
|
@@ -7316,17 +8514,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7316
8514
|
turnMetricOutcome = "recovering";
|
|
7317
8515
|
return claimedResult({
|
|
7318
8516
|
status: "recovering",
|
|
7319
|
-
...(
|
|
8517
|
+
...(transitionPending
|
|
7320
8518
|
? {
|
|
7321
8519
|
continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings),
|
|
7322
8520
|
}
|
|
7323
8521
|
: {}),
|
|
7324
8522
|
});
|
|
7325
8523
|
} catch (recoveryError) {
|
|
7326
|
-
console.error(
|
|
7327
|
-
"sandbox lease supersession recovery failed",
|
|
7328
|
-
safeErrorDiagnostic(recoveryError, (value) => String(redact(value))),
|
|
7329
|
-
);
|
|
8524
|
+
console.error("sandbox lifecycle recovery failed", safeErrorDiagnostic(recoveryError));
|
|
7330
8525
|
throw recoveryError;
|
|
7331
8526
|
}
|
|
7332
8527
|
}
|
|
@@ -7357,6 +8552,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7357
8552
|
turnMetricOutcome = "cancelled";
|
|
7358
8553
|
return claimedResult({ status: "cancelled" });
|
|
7359
8554
|
}
|
|
8555
|
+
acknowledgeRecoveryQuiescence();
|
|
7360
8556
|
await publishDurableSessionEvents(
|
|
7361
8557
|
bus,
|
|
7362
8558
|
input.workspaceId,
|
|
@@ -7372,7 +8568,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7372
8568
|
} catch (recoveryError) {
|
|
7373
8569
|
console.error(
|
|
7374
8570
|
"sandbox deadline rotation recovery failed",
|
|
7375
|
-
safeErrorDiagnostic(recoveryError
|
|
8571
|
+
safeErrorDiagnostic(recoveryError),
|
|
7376
8572
|
);
|
|
7377
8573
|
throw recoveryError;
|
|
7378
8574
|
}
|
|
@@ -7391,8 +8587,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7391
8587
|
// the frozen RunState blob (the only representation of a turn
|
|
7392
8588
|
// paused mid-flight), so swapping the trigger for a resume notice
|
|
7393
8589
|
// could drop the user's decision. Re-applying an already-consumed
|
|
7394
|
-
// approval re-
|
|
7395
|
-
//
|
|
8590
|
+
// approval re-enters at most the single approved step. Every
|
|
8591
|
+
// approval-gated MCP action crosses the durable execution admission
|
|
8592
|
+
// fence before its provider invocation, so a consumed step resumes
|
|
8593
|
+
// as already-executed or outcome-unknown rather than calling MCP again.
|
|
7396
8594
|
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
7397
8595
|
sessionId: input.sessionId,
|
|
7398
8596
|
turnId: recoveryTurnId,
|
|
@@ -7406,6 +8604,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7406
8604
|
turnMetricOutcome = "cancelled";
|
|
7407
8605
|
return claimedResult({ status: "cancelled" });
|
|
7408
8606
|
}
|
|
8607
|
+
acknowledgeRecoveryQuiescence();
|
|
7409
8608
|
await publishDurableSessionEvents(
|
|
7410
8609
|
bus,
|
|
7411
8610
|
input.workspaceId,
|
|
@@ -7421,7 +8620,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7421
8620
|
// the turn through a second cancellation path.
|
|
7422
8621
|
console.error(
|
|
7423
8622
|
"worker-shutdown recovery checkpoint failed",
|
|
7424
|
-
safeErrorDiagnostic(recoveryError
|
|
8623
|
+
safeErrorDiagnostic(recoveryError),
|
|
7425
8624
|
);
|
|
7426
8625
|
throw recoveryError;
|
|
7427
8626
|
}
|
|
@@ -7520,11 +8719,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7520
8719
|
await reconcileConversationTruth({ requireDurable: true });
|
|
7521
8720
|
}
|
|
7522
8721
|
checkpointDurable = true;
|
|
7523
|
-
} catch
|
|
8722
|
+
} catch {
|
|
7524
8723
|
observability.warn("Codex lease-loss checkpoint failed; refusing automatic turn replay", {
|
|
7525
|
-
|
|
7526
|
-
|
|
7527
|
-
|
|
8724
|
+
errorClass: "CodexCheckpointOperationError",
|
|
8725
|
+
errorCode: "codex_lease_loss_checkpoint_failed",
|
|
8726
|
+
origin: "worker",
|
|
7528
8727
|
});
|
|
7529
8728
|
}
|
|
7530
8729
|
|
|
@@ -7631,16 +8830,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7631
8830
|
await flushRuntimeBatcher();
|
|
7632
8831
|
await reconcileConversationTruth({ requireDurable: true });
|
|
7633
8832
|
checkpointDurable = true;
|
|
7634
|
-
} catch
|
|
8833
|
+
} catch {
|
|
7635
8834
|
observability.incrementCounter({
|
|
7636
8835
|
name: "opengeni_codex_failover_checkpoints_total",
|
|
7637
8836
|
help: "Durable Codex failover checkpoint attempts by outcome.",
|
|
7638
8837
|
labels: { workspace_key: codexWorkspaceKey, outcome: "failed" },
|
|
7639
8838
|
});
|
|
7640
8839
|
observability.warn("Codex failover checkpoint failed; refusing automatic replay", {
|
|
7641
|
-
|
|
7642
|
-
|
|
7643
|
-
|
|
8840
|
+
errorClass: "CodexCheckpointOperationError",
|
|
8841
|
+
errorCode: "codex_failover_checkpoint_failed",
|
|
8842
|
+
origin: "worker",
|
|
7644
8843
|
});
|
|
7645
8844
|
}
|
|
7646
8845
|
|
|
@@ -7874,14 +9073,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7874
9073
|
});
|
|
7875
9074
|
if (newHome) {
|
|
7876
9075
|
rotated = true;
|
|
7877
|
-
const pinMutation = await
|
|
9076
|
+
const pinMutation = await withSessionCodexCapacityMutation(
|
|
7878
9077
|
db,
|
|
7879
9078
|
{
|
|
7880
9079
|
workspaceId: input.workspaceId,
|
|
7881
9080
|
reason: "codex_policy_pin_resharded",
|
|
7882
9081
|
},
|
|
7883
9082
|
async (tx) => {
|
|
7884
|
-
const changed = await
|
|
9083
|
+
const changed = await setSessionCodexPinInTransaction(
|
|
7885
9084
|
tx,
|
|
7886
9085
|
input.workspaceId,
|
|
7887
9086
|
input.sessionId,
|
|
@@ -8156,7 +9355,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8156
9355
|
) {
|
|
8157
9356
|
await flushRuntimeBatcher();
|
|
8158
9357
|
await reconcileConversationTruth({ requireDurable: true });
|
|
8159
|
-
const activeHistory = await
|
|
9358
|
+
const activeHistory = await getActiveSessionHistoryItemsPaged(
|
|
8160
9359
|
db,
|
|
8161
9360
|
input.workspaceId,
|
|
8162
9361
|
input.sessionId,
|
|
@@ -8193,6 +9392,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8193
9392
|
return claimedResult({ status: "cancelled" });
|
|
8194
9393
|
}
|
|
8195
9394
|
if (recovery.action === "recovering") {
|
|
9395
|
+
acknowledgeRecoveryQuiescence();
|
|
8196
9396
|
await publishDurableSessionEvents(
|
|
8197
9397
|
bus,
|
|
8198
9398
|
input.workspaceId,
|
|
@@ -8211,15 +9411,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8211
9411
|
// truth, recover this SAME accepted turn, then let the workflow re-claim
|
|
8212
9412
|
// it after a pacing delay. This is independent of goal state and never
|
|
8213
9413
|
// relies on a synthetic continuation prompt.
|
|
8214
|
-
const failure =
|
|
8215
|
-
|
|
8216
|
-
>;
|
|
9414
|
+
const failure = agentRunFailurePayload(error, {
|
|
9415
|
+
isCodexTurn,
|
|
9416
|
+
}) as ReturnType<typeof agentRunFailurePayload>;
|
|
8217
9417
|
if (isSessionEventPersistenceError(error)) {
|
|
8218
|
-
//
|
|
8219
|
-
//
|
|
8220
|
-
// boundary retains only SQLSTATE, stage, correlation, and safe catalog
|
|
8221
|
-
// identifiers. Provider inference has already happened and is NOT
|
|
8222
|
-
// retried by this terminal classification.
|
|
9418
|
+
// Preserve the exact source message in the internal runtime diagnostic;
|
|
9419
|
+
// SQLSTATE/catalog facts remain separate classification attributes.
|
|
8223
9420
|
observability.error("session event persistence failed", {
|
|
8224
9421
|
accountId: input.accountId,
|
|
8225
9422
|
workspaceId: input.workspaceId,
|
|
@@ -8240,6 +9437,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8240
9437
|
dbDataType: error.details.database.dataType,
|
|
8241
9438
|
dbConstraint: error.details.database.constraint,
|
|
8242
9439
|
dbRoutine: error.details.database.routine,
|
|
9440
|
+
error: error.message,
|
|
8243
9441
|
});
|
|
8244
9442
|
}
|
|
8245
9443
|
if (failure.retryable && publish && turnId && turnStartedPublished) {
|
|
@@ -8271,6 +9469,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8271
9469
|
turnMetricOutcome = "cancelled";
|
|
8272
9470
|
return claimedResult({ status: "cancelled" });
|
|
8273
9471
|
}
|
|
9472
|
+
acknowledgeRecoveryQuiescence();
|
|
8274
9473
|
await publishDurableSessionEvents(
|
|
8275
9474
|
bus,
|
|
8276
9475
|
input.workspaceId,
|
|
@@ -8362,10 +9561,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8362
9561
|
gitCredentialRenewalClosed = true;
|
|
8363
9562
|
const gitRenewalsToStop = gitCredentialRenewals;
|
|
8364
9563
|
gitCredentialRenewals = [];
|
|
8365
|
-
|
|
8366
|
-
const
|
|
8367
|
-
|
|
8368
|
-
toolspaceTokenRenewal = null;
|
|
9564
|
+
codemodeTokenRenewalClosed = true;
|
|
9565
|
+
const codemodeRenewalToStop = codemodeTokenRenewal as CodemodeTokenRenewalController | null;
|
|
9566
|
+
codemodeTokenRenewal = null;
|
|
8369
9567
|
runCredentialRenewalClosed = true;
|
|
8370
9568
|
const runRenewalToStop = runCredentialRenewal as RunCredentialRenewalController | null;
|
|
8371
9569
|
runCredentialRenewal = null;
|
|
@@ -8376,18 +9574,38 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8376
9574
|
const credentialSessionToClear = runCredentialSession;
|
|
8377
9575
|
runCredentialSession = null;
|
|
8378
9576
|
if (credentialSessionToClear) {
|
|
8379
|
-
|
|
8380
|
-
|
|
8381
|
-
|
|
8382
|
-
|
|
8383
|
-
|
|
8384
|
-
|
|
8385
|
-
|
|
8386
|
-
|
|
8387
|
-
|
|
8388
|
-
|
|
8389
|
-
|
|
8390
|
-
|
|
9577
|
+
const clearAttemptCredentials = async (): Promise<void> =>
|
|
9578
|
+
await clearRunCredentialsForAttempt(credentialSessionToClear, {
|
|
9579
|
+
sessionId: input.sessionId,
|
|
9580
|
+
attemptId: input.attemptId,
|
|
9581
|
+
executionGeneration,
|
|
9582
|
+
});
|
|
9583
|
+
await clearAttemptCredentialsWithSettledFence({
|
|
9584
|
+
activityStatus,
|
|
9585
|
+
runWorkspaceFencedClear: async () =>
|
|
9586
|
+
await runWorkspaceMutationForSandbox(
|
|
9587
|
+
requireResolvedSandboxForMutation(
|
|
9588
|
+
"Run credential cleanup has no exact sandbox lease target",
|
|
9589
|
+
),
|
|
9590
|
+
"runCredentialAttemptClear",
|
|
9591
|
+
clearAttemptCredentials,
|
|
9592
|
+
),
|
|
9593
|
+
clearExactAttempt: clearAttemptCredentials,
|
|
9594
|
+
onSettledAttemptFence: () => {
|
|
9595
|
+
// Terminal settlement closes the active attempt before this finally
|
|
9596
|
+
// runs, so the ordinary workspace admission correctly rejects it.
|
|
9597
|
+
// The attempt-qualified delete is nevertheless successor-safe: it
|
|
9598
|
+
// removes only this attempt/generation and clears the pointer only
|
|
9599
|
+
// when it still names that exact generation. Finish that cleanup
|
|
9600
|
+
// directly so a successful turn can continue into workspace capture,
|
|
9601
|
+
// tool teardown, and lease release.
|
|
9602
|
+
observability.info("retrying exact run credential cleanup after turn settlement", {
|
|
9603
|
+
"opengeni.session_id": input.sessionId,
|
|
9604
|
+
"opengeni.turn_id": turnId ?? "",
|
|
9605
|
+
"opengeni.attempt_id": input.attemptId,
|
|
9606
|
+
});
|
|
9607
|
+
},
|
|
9608
|
+
});
|
|
8391
9609
|
}
|
|
8392
9610
|
await drainAttemptOwnedSandboxWriters({
|
|
8393
9611
|
// Normal turn completion owns the same process boundary as
|
|
@@ -8397,9 +9615,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8397
9615
|
toolCancellationFence,
|
|
8398
9616
|
cancellationReason: cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
|
|
8399
9617
|
gitCredentialRenewals: gitRenewalsToStop,
|
|
8400
|
-
|
|
9618
|
+
codemodeTokenRenewal: codemodeRenewalToStop,
|
|
8401
9619
|
runCredentialRenewal: runRenewalToStop,
|
|
8402
9620
|
});
|
|
9621
|
+
attemptWritersDrained = true;
|
|
8403
9622
|
if (acknowledgeQuiescence) {
|
|
8404
9623
|
// A cancellation before sandbox-backed capabilities exist still has
|
|
8405
9624
|
// no tool controller to drain. Sandbox agent construction fails closed
|
|
@@ -8425,7 +9644,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8425
9644
|
const recoveryMode = await persistOrSignalSessionAttemptQuiescence({
|
|
8426
9645
|
proof,
|
|
8427
9646
|
persistReceipt: async () =>
|
|
8428
|
-
await
|
|
9647
|
+
await commitSessionAttemptQuiescence(db, {
|
|
8429
9648
|
accountId: input.accountId,
|
|
8430
9649
|
workspaceId: input.workspaceId,
|
|
8431
9650
|
sessionId: input.sessionId,
|
|
@@ -8435,6 +9654,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8435
9654
|
temporalActivityId: dispatchId,
|
|
8436
9655
|
allowUninterrupted: true,
|
|
8437
9656
|
}),
|
|
9657
|
+
...(wakeSessionWorkflow
|
|
9658
|
+
? {
|
|
9659
|
+
deliverWorkflowWake: async (
|
|
9660
|
+
wake: NonNullable<SessionAttemptQuiescenceCommit["workflowWake"]>,
|
|
9661
|
+
) =>
|
|
9662
|
+
await wakeSessionWorkflow({
|
|
9663
|
+
accountId: wake.accountId,
|
|
9664
|
+
workspaceId: wake.workspaceId,
|
|
9665
|
+
sessionId: wake.sessionId,
|
|
9666
|
+
workflowId: wake.temporalWorkflowId,
|
|
9667
|
+
wakeRevision: wake.wakeRevision,
|
|
9668
|
+
interruptionRequested: wake.interruptionRequested,
|
|
9669
|
+
}),
|
|
9670
|
+
}
|
|
9671
|
+
: {}),
|
|
8438
9672
|
publishEvents: async (events) => {
|
|
8439
9673
|
await waitForTurnFinalizerStep(
|
|
8440
9674
|
publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events),
|
|
@@ -8455,18 +9689,24 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8455
9689
|
onReceiptFailure: (error) => {
|
|
8456
9690
|
console.error(
|
|
8457
9691
|
"agent turn quiescence receipt exhausted; signalling proof",
|
|
8458
|
-
safeErrorDiagnostic(error
|
|
9692
|
+
safeErrorDiagnostic(error),
|
|
9693
|
+
);
|
|
9694
|
+
},
|
|
9695
|
+
onWakeFailure: (error) => {
|
|
9696
|
+
console.error(
|
|
9697
|
+
"agent turn quiescence immediate workflow wake failed; outbox repair retained",
|
|
9698
|
+
safeErrorDiagnostic(error),
|
|
8459
9699
|
);
|
|
8460
9700
|
},
|
|
8461
9701
|
onPublishFailure: (error) => {
|
|
8462
9702
|
console.error(
|
|
8463
9703
|
"agent turn quiescence event fanout failed",
|
|
8464
|
-
safeErrorDiagnostic(error
|
|
9704
|
+
safeErrorDiagnostic(error),
|
|
8465
9705
|
);
|
|
8466
9706
|
},
|
|
8467
9707
|
onSignalFailure: (error, attempt, retryMs) => {
|
|
8468
9708
|
console.error("agent turn quiescence proof signal failed; retrying", {
|
|
8469
|
-
error: safeErrorDiagnostic(error
|
|
9709
|
+
error: safeErrorDiagnostic(error),
|
|
8470
9710
|
attempt,
|
|
8471
9711
|
retryMs,
|
|
8472
9712
|
});
|
|
@@ -8621,6 +9861,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8621
9861
|
...(finalizerSignal ? { signal: finalizerSignal } : {}),
|
|
8622
9862
|
});
|
|
8623
9863
|
}
|
|
9864
|
+
if (codemodeDispatcher) {
|
|
9865
|
+
await waitForTurnFinalizerStep(
|
|
9866
|
+
codemodeDispatcher.close().catch(() => undefined),
|
|
9867
|
+
finalizerSignal,
|
|
9868
|
+
);
|
|
9869
|
+
codemodeDispatcher = null;
|
|
9870
|
+
}
|
|
8624
9871
|
if (preparedTools) {
|
|
8625
9872
|
await waitForTurnFinalizerStep(
|
|
8626
9873
|
preparedTools.close().catch(() => undefined),
|
|
@@ -8708,28 +9955,50 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8708
9955
|
// while the box is still solidly alive, instead of racing the turn-end
|
|
8709
9956
|
// teardown that was killing 100% of captures on real Modal desktop boxes.
|
|
8710
9957
|
}
|
|
8711
|
-
const sandboxToRelease = resolvedSandbox;
|
|
8712
|
-
resolvedSandbox = null; // drop ownership now; the exact-holder release may finish later
|
|
8713
|
-
await waitForTurnFinalizerStep(
|
|
8714
|
-
sandboxToRelease.release().catch((releaseError) => {
|
|
8715
|
-
console.error(
|
|
8716
|
-
"sandbox lease release failed (turn outcome unaffected)",
|
|
8717
|
-
safeErrorDiagnostic(releaseError, (value) => String(redact(value))),
|
|
8718
|
-
);
|
|
8719
|
-
}),
|
|
8720
|
-
finalizerSignal,
|
|
8721
|
-
);
|
|
8722
9958
|
}
|
|
8723
9959
|
} catch (error) {
|
|
8724
9960
|
finalizationError ??= error;
|
|
8725
9961
|
console.error(
|
|
8726
9962
|
"agent turn finalization failed (turn outcome unaffected)",
|
|
8727
|
-
safeErrorDiagnostic(error
|
|
9963
|
+
safeErrorDiagnostic(error),
|
|
8728
9964
|
);
|
|
8729
9965
|
} finally {
|
|
8730
|
-
//
|
|
8731
|
-
//
|
|
8732
|
-
//
|
|
9966
|
+
// The writer drain is the only authority that licenses settling an
|
|
9967
|
+
// abandoned turn admission. Keep this second-stage release outside all
|
|
9968
|
+
// later housekeeping so an event/cache/capture/recording failure cannot
|
|
9969
|
+
// strand a null-outcome admission forever. Do not race it against
|
|
9970
|
+
// Temporal cancellation: the eager listener may already have dropped
|
|
9971
|
+
// the holder, and this idempotent proof-bearing pass still must run.
|
|
9972
|
+
stopLeaseHeartbeat();
|
|
9973
|
+
const sandboxReleaseTargets = new Set(lateSandboxesAwaitingWriterDrain);
|
|
9974
|
+
lateSandboxesAwaitingWriterDrain.clear();
|
|
9975
|
+
if (resolvedSandbox) {
|
|
9976
|
+
sandboxReleaseTargets.add(resolvedSandbox);
|
|
9977
|
+
resolvedSandbox = null;
|
|
9978
|
+
}
|
|
9979
|
+
if (attemptWritersDrained) {
|
|
9980
|
+
for (const sandboxToRelease of sandboxReleaseTargets) {
|
|
9981
|
+
try {
|
|
9982
|
+
await releaseTurnSandboxAfterWriterDrain(sandboxToRelease);
|
|
9983
|
+
} catch (releaseError) {
|
|
9984
|
+
finalizationError ??= releaseError;
|
|
9985
|
+
console.error(
|
|
9986
|
+
"sandbox lease quiesced release failed (turn outcome unaffected)",
|
|
9987
|
+
safeErrorDiagnostic(releaseError),
|
|
9988
|
+
);
|
|
9989
|
+
}
|
|
9990
|
+
}
|
|
9991
|
+
} else {
|
|
9992
|
+
// No proof exists. Ensure every late result still receives the eager
|
|
9993
|
+
// leak-prevention release while preserving its admissions as a
|
|
9994
|
+
// fail-closed archive-capture fence.
|
|
9995
|
+
for (const sandboxToRelease of sandboxReleaseTargets) {
|
|
9996
|
+
await sandboxToRelease.release().catch(() => undefined);
|
|
9997
|
+
}
|
|
9998
|
+
}
|
|
9999
|
+
// Close provisioning after consuming the currently known targets. A
|
|
10000
|
+
// provider result that races in later is routed through releaseLateSandbox;
|
|
10001
|
+
// targets already released above synchronously removed their listener.
|
|
8733
10002
|
if (!sandboxResumeController.signal.aborted) {
|
|
8734
10003
|
sandboxResumeController.abort(
|
|
8735
10004
|
cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
|
|
@@ -8756,12 +10025,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8756
10025
|
value: physicalCancellationDurationSeconds,
|
|
8757
10026
|
});
|
|
8758
10027
|
observability.info("agent turn physical cancellation completed", {
|
|
8759
|
-
|
|
8760
|
-
|
|
8761
|
-
|
|
8762
|
-
|
|
8763
|
-
|
|
8764
|
-
|
|
10028
|
+
durationMs: Math.round(physicalCancellationDurationSeconds * 1000),
|
|
10029
|
+
...(sandboxGroupId
|
|
10030
|
+
? {
|
|
10031
|
+
sandboxLeaseKey: sandboxLeaseTelemetryKey(input.workspaceId, sandboxGroupId),
|
|
10032
|
+
}
|
|
10033
|
+
: {}),
|
|
8765
10034
|
});
|
|
8766
10035
|
}
|
|
8767
10036
|
observability.recordWorkerActivity({
|
|
@@ -8785,9 +10054,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8785
10054
|
},
|
|
8786
10055
|
error:
|
|
8787
10056
|
finalizationError || activityError
|
|
8788
|
-
? safeErrorForTelemetry(finalizationError ?? activityError
|
|
8789
|
-
String(redact(value)),
|
|
8790
|
-
)
|
|
10057
|
+
? safeErrorForTelemetry(finalizationError ?? activityError)
|
|
8791
10058
|
: undefined,
|
|
8792
10059
|
});
|
|
8793
10060
|
assertPhysicalToolQuiescenceForCancellation({
|
|
@@ -8919,7 +10186,19 @@ export function agentRunFailurePayload(
|
|
|
8919
10186
|
attempts?: number;
|
|
8920
10187
|
retryOutcome?: string;
|
|
8921
10188
|
database?: Record<string, string>;
|
|
10189
|
+
historyPersistenceStage?: MandatoryHistoryPersistenceStage;
|
|
8922
10190
|
} {
|
|
10191
|
+
if (error instanceof MandatoryHistoryPersistenceError) {
|
|
10192
|
+
const underlying = isSessionEventPersistenceError(error.cause)
|
|
10193
|
+
? agentRunFailurePayload(error.cause, options)
|
|
10194
|
+
: {
|
|
10195
|
+
error: error.cause instanceof Error ? error.cause.message : String(error.cause),
|
|
10196
|
+
};
|
|
10197
|
+
return {
|
|
10198
|
+
...underlying,
|
|
10199
|
+
historyPersistenceStage: error.stage,
|
|
10200
|
+
};
|
|
10201
|
+
}
|
|
8923
10202
|
const message = error instanceof Error ? error.message : String(error);
|
|
8924
10203
|
const status =
|
|
8925
10204
|
typeof error === "object" && error !== null
|
|
@@ -8932,6 +10211,24 @@ export function agentRunFailurePayload(
|
|
|
8932
10211
|
typeof error === "object" && error !== null && "code" in error
|
|
8933
10212
|
? String((error as { code?: unknown }).code)
|
|
8934
10213
|
: undefined;
|
|
10214
|
+
if (error instanceof ActiveSessionHistoryLimitExceededError) {
|
|
10215
|
+
return {
|
|
10216
|
+
error:
|
|
10217
|
+
"The session's active conversation history exceeds the worker's safe materialization envelope. Clear the session context before retrying; an oversized history cannot be compacted safely in a serving worker.",
|
|
10218
|
+
code: error.code,
|
|
10219
|
+
retryable: false,
|
|
10220
|
+
detail: error.message,
|
|
10221
|
+
};
|
|
10222
|
+
}
|
|
10223
|
+
if (error instanceof ApprovalRunStateLimitExceededError) {
|
|
10224
|
+
return {
|
|
10225
|
+
error:
|
|
10226
|
+
"The saved approval state exceeds the worker's safe materialization envelope. Clear the pending approval context before retrying.",
|
|
10227
|
+
code: error.code,
|
|
10228
|
+
retryable: false,
|
|
10229
|
+
detail: error.message,
|
|
10230
|
+
};
|
|
10231
|
+
}
|
|
8935
10232
|
// An accepted Codex stream with no terminal response is malformed/partial,
|
|
8936
10233
|
// not provider backpressure. Replaying the same accepted turn could repeat
|
|
8937
10234
|
// model or tool effects, so this marked transport failure must outrank the
|
|
@@ -8945,15 +10242,8 @@ export function agentRunFailurePayload(
|
|
|
8945
10242
|
}
|
|
8946
10243
|
if (isSessionEventPersistenceError(error)) {
|
|
8947
10244
|
const { details } = error;
|
|
8948
|
-
const eventLabel = details.eventTypes.join(", ") || "session events";
|
|
8949
|
-
const failureLabel =
|
|
8950
|
-
details.code === "db_deadlock"
|
|
8951
|
-
? "Database deadlock"
|
|
8952
|
-
: details.code === "db_serialization_failure"
|
|
8953
|
-
? "Database serialization failure"
|
|
8954
|
-
: "Database failure";
|
|
8955
10245
|
return {
|
|
8956
|
-
error:
|
|
10246
|
+
error: error.message,
|
|
8957
10247
|
code: details.code,
|
|
8958
10248
|
detail:
|
|
8959
10249
|
details.retryOutcome === "exhausted"
|
|
@@ -9015,6 +10305,7 @@ export function agentRunFailurePayload(
|
|
|
9015
10305
|
"A required MCP server was temporarily unreachable. The same turn will retry after a short delay.",
|
|
9016
10306
|
code: "mcp_transport_unavailable",
|
|
9017
10307
|
retryable: true,
|
|
10308
|
+
detail: message,
|
|
9018
10309
|
};
|
|
9019
10310
|
}
|
|
9020
10311
|
if (
|
|
@@ -9232,9 +10523,10 @@ export function codexUsageLimitFailurePayload(
|
|
|
9232
10523
|
// goal re-evaluates at most this far out (it will re-pause if still capped).
|
|
9233
10524
|
const CODEX_USAGE_LIMIT_MAX_RESUME_MS = 60 * 60_000; // 1h
|
|
9234
10525
|
|
|
9235
|
-
function pendingToolCallFromSdkEvent(event: unknown): {
|
|
10526
|
+
export function pendingToolCallFromSdkEvent(event: unknown): {
|
|
9236
10527
|
callId: string;
|
|
9237
10528
|
callType: string;
|
|
10529
|
+
callName: string | null;
|
|
9238
10530
|
callItem: Record<string, unknown>;
|
|
9239
10531
|
} | null {
|
|
9240
10532
|
if (!event || typeof event !== "object") return null;
|
|
@@ -9250,15 +10542,30 @@ function pendingToolCallFromSdkEvent(event: unknown): {
|
|
|
9250
10542
|
return null;
|
|
9251
10543
|
}
|
|
9252
10544
|
const raw = item.rawItem as Record<string, unknown>;
|
|
10545
|
+
// The hosted image call is a complete provider fact carried by one item; it
|
|
10546
|
+
// never receives a separate function result and therefore must not enter the
|
|
10547
|
+
// pending function-call ledger.
|
|
10548
|
+
if (raw.type === "hosted_tool_call" && raw.name === "image_generation_call") return null;
|
|
9253
10549
|
const callId = raw.callId ?? raw.call_id ?? raw.id;
|
|
9254
10550
|
const callType = raw.type;
|
|
9255
10551
|
if (typeof callId !== "string" || callId.length === 0 || typeof callType !== "string") {
|
|
9256
10552
|
return null;
|
|
9257
10553
|
}
|
|
9258
|
-
|
|
10554
|
+
const callItem = normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]');
|
|
10555
|
+
return {
|
|
10556
|
+
callId,
|
|
10557
|
+
callType,
|
|
10558
|
+
callName: typeof raw.name === "string" ? raw.name : null,
|
|
10559
|
+
callItem,
|
|
10560
|
+
};
|
|
9259
10561
|
}
|
|
9260
10562
|
|
|
9261
|
-
|
|
10563
|
+
/** Tools whose intentional visual result must survive inline-media sanitization. */
|
|
10564
|
+
export function toolCallProducesRetainableSessionImage(name: string | null): boolean {
|
|
10565
|
+
return name === "computer_screenshot" || name === "view_image";
|
|
10566
|
+
}
|
|
10567
|
+
|
|
10568
|
+
export function completedToolCallFromSdkEvent(event: unknown): {
|
|
9262
10569
|
callId: string;
|
|
9263
10570
|
resultItem: Record<string, unknown>;
|
|
9264
10571
|
} | null {
|
|
@@ -9273,7 +10580,11 @@ function completedToolCallFromSdkEvent(event: unknown): {
|
|
|
9273
10580
|
? (item.rawItem as Record<string, unknown>)
|
|
9274
10581
|
: {};
|
|
9275
10582
|
const callId = raw.callId ?? raw.call_id ?? item.id;
|
|
9276
|
-
|
|
10583
|
+
if (typeof callId !== "string" || callId.length === 0) return null;
|
|
10584
|
+
return {
|
|
10585
|
+
callId,
|
|
10586
|
+
resultItem: normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]'),
|
|
10587
|
+
};
|
|
9277
10588
|
}
|
|
9278
10589
|
|
|
9279
10590
|
/**
|
|
@@ -9378,6 +10689,9 @@ export type ModelUsageBillingRecord = {
|
|
|
9378
10689
|
billingPath: "opengeni_credits" | "external";
|
|
9379
10690
|
/** Same quantity written to usage_events.model.cost when present; else 0. */
|
|
9380
10691
|
pricedCostMicros: number;
|
|
10692
|
+
/** Hypothetical provider-rate USD micros; never an OpenGeni charge. */
|
|
10693
|
+
estimatedProviderCostMicros: number | null;
|
|
10694
|
+
pricingSource: "configured_list_price" | "gateway_reported" | null;
|
|
9381
10695
|
normalizedUsage: ModelCallUsageNormalization;
|
|
9382
10696
|
upstreamProvider?: string;
|
|
9383
10697
|
};
|
|
@@ -9425,6 +10739,37 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
9425
10739
|
);
|
|
9426
10740
|
}
|
|
9427
10741
|
}
|
|
10742
|
+
const pricingSchedules = configuredModelPricingSchedules(settings);
|
|
10743
|
+
const configuredPricingModel = pricingSchedules[input.model]
|
|
10744
|
+
? input.model
|
|
10745
|
+
: input.model.startsWith("codex/") && pricingSchedules[input.model.slice("codex/".length)]
|
|
10746
|
+
? input.model.slice("codex/".length)
|
|
10747
|
+
: null;
|
|
10748
|
+
const pricingBreakdown = gatewayBilling
|
|
10749
|
+
? calculateGatewayReportedCostBreakdown(
|
|
10750
|
+
settings,
|
|
10751
|
+
input.model,
|
|
10752
|
+
gatewayBilling.inferenceCostUsd,
|
|
10753
|
+
{ inputTokens },
|
|
10754
|
+
)
|
|
10755
|
+
: configuredPricingModel
|
|
10756
|
+
? calculateModelUsageCostBreakdown(settings, configuredPricingModel, sanitizedUsage, {
|
|
10757
|
+
latencyMode: input.latencyMode ?? "standard",
|
|
10758
|
+
})
|
|
10759
|
+
: null;
|
|
10760
|
+
const hasCompleteCoreTokenTelemetry =
|
|
10761
|
+
normalizedUsage.telemetry.inputTokens !== null &&
|
|
10762
|
+
normalizedUsage.telemetry.outputTokens !== null;
|
|
10763
|
+
const estimatedProviderCostMicros = gatewayBilling
|
|
10764
|
+
? (pricingBreakdown?.providerCostMicros ?? null)
|
|
10765
|
+
: hasCompleteCoreTokenTelemetry
|
|
10766
|
+
? (pricingBreakdown?.providerCostMicros ?? null)
|
|
10767
|
+
: null;
|
|
10768
|
+
const pricingSource = gatewayBilling
|
|
10769
|
+
? ("gateway_reported" as const)
|
|
10770
|
+
: estimatedProviderCostMicros !== null
|
|
10771
|
+
? ("configured_list_price" as const)
|
|
10772
|
+
: null;
|
|
9428
10773
|
// An externally billed turn is paid outside OpenGeni, so it
|
|
9429
10774
|
// consumes ZERO OpenGeni credits and must never feed an OpenGeni cap. A
|
|
9430
10775
|
// codex/<slug> model has no entry in configuredModelPricing, so the normal path
|
|
@@ -9433,7 +10778,8 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
9433
10778
|
// the API tokens:consume cap sum `model.tokens` with NO cost dimension, so
|
|
9434
10779
|
// any row would count against maxMonthlyTokensPerWorkspace);
|
|
9435
10780
|
// - record a `model.cost = 0` audit marker (harmless to the monthly cost cap);
|
|
9436
|
-
// -
|
|
10781
|
+
// - optionally calculate a non-charging provider estimate for Insights when
|
|
10782
|
+
// core token telemetry is complete, but never debit credits.
|
|
9437
10783
|
if (input.externallyBilled) {
|
|
9438
10784
|
await recordUsageEvent(db, {
|
|
9439
10785
|
accountId: input.accountId,
|
|
@@ -9451,6 +10797,8 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
9451
10797
|
return {
|
|
9452
10798
|
billingPath: "external",
|
|
9453
10799
|
pricedCostMicros: 0,
|
|
10800
|
+
estimatedProviderCostMicros,
|
|
10801
|
+
pricingSource,
|
|
9454
10802
|
normalizedUsage,
|
|
9455
10803
|
};
|
|
9456
10804
|
}
|
|
@@ -9474,20 +10822,16 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
9474
10822
|
return {
|
|
9475
10823
|
billingPath: "opengeni_credits",
|
|
9476
10824
|
pricedCostMicros: 0,
|
|
10825
|
+
estimatedProviderCostMicros,
|
|
10826
|
+
pricingSource,
|
|
9477
10827
|
normalizedUsage,
|
|
9478
10828
|
...(gatewayBilling ? { upstreamProvider: gatewayBilling.finalProvider } : {}),
|
|
9479
10829
|
};
|
|
9480
10830
|
}
|
|
9481
|
-
if (!
|
|
10831
|
+
if (!pricingBreakdown) {
|
|
9482
10832
|
throw new Error(`Missing model pricing for ${input.model}`);
|
|
9483
10833
|
}
|
|
9484
|
-
const costMicros =
|
|
9485
|
-
? calculateGatewayReportedCostMicros(settings, input.model, gatewayBilling.inferenceCostUsd, {
|
|
9486
|
-
inputTokens,
|
|
9487
|
-
})
|
|
9488
|
-
: calculateModelUsageCostMicros(settings, input.model, sanitizedUsage, {
|
|
9489
|
-
latencyMode: input.latencyMode ?? "standard",
|
|
9490
|
-
});
|
|
10834
|
+
const costMicros = pricingBreakdown.creditCostMicros;
|
|
9491
10835
|
await recordUsageEvent(db, {
|
|
9492
10836
|
accountId: input.accountId,
|
|
9493
10837
|
workspaceId: input.workspaceId,
|
|
@@ -9531,6 +10875,8 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
9531
10875
|
return {
|
|
9532
10876
|
billingPath: "opengeni_credits",
|
|
9533
10877
|
pricedCostMicros: costMicros,
|
|
10878
|
+
estimatedProviderCostMicros,
|
|
10879
|
+
pricingSource,
|
|
9534
10880
|
normalizedUsage,
|
|
9535
10881
|
...(gatewayBilling ? { upstreamProvider: gatewayBilling.finalProvider } : {}),
|
|
9536
10882
|
};
|
|
@@ -9565,6 +10911,8 @@ export async function recordAuthoritativeModelCallFact(input: {
|
|
|
9565
10911
|
model: input.model,
|
|
9566
10912
|
billingPath: input.billing.billingPath,
|
|
9567
10913
|
pricedCostMicros: input.billing.pricedCostMicros,
|
|
10914
|
+
estimatedProviderCostMicros: input.billing.estimatedProviderCostMicros,
|
|
10915
|
+
pricingSource: input.billing.pricingSource,
|
|
9568
10916
|
inputTokens: telemetry.inputTokens,
|
|
9569
10917
|
outputTokens: telemetry.outputTokens,
|
|
9570
10918
|
cachedTokens: telemetry.cachedTokens,
|
|
@@ -9574,12 +10922,7 @@ export async function recordAuthoritativeModelCallFact(input: {
|
|
|
9574
10922
|
});
|
|
9575
10923
|
} catch (error) {
|
|
9576
10924
|
input.observability.warn("model call fact persist failed", {
|
|
9577
|
-
|
|
9578
|
-
workspaceId: input.workspaceId,
|
|
9579
|
-
sessionId: input.sessionId,
|
|
9580
|
-
turnId: input.turnId,
|
|
9581
|
-
sourceKey: input.sourceKey,
|
|
9582
|
-
error: error instanceof Error ? error.message : String(error),
|
|
10925
|
+
...safeErrorDiagnostic(error),
|
|
9583
10926
|
});
|
|
9584
10927
|
}
|
|
9585
10928
|
}
|
|
@@ -9600,6 +10943,9 @@ function sanitizedModelUsageInput(normalized: ModelCallUsageNormalization): Mode
|
|
|
9600
10943
|
},
|
|
9601
10944
|
}
|
|
9602
10945
|
: {}),
|
|
10946
|
+
...(normalized.requestUsageEntries
|
|
10947
|
+
? { requestUsageEntries: normalized.requestUsageEntries }
|
|
10948
|
+
: {}),
|
|
9603
10949
|
};
|
|
9604
10950
|
}
|
|
9605
10951
|
|