@opengeni/worker-bundle 0.17.2 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +12 -0
- package/dist/activities/agent-turn.d.ts +49 -1
- package/dist/activities/browser-state-artifact-reaper.d.ts +29 -0
- package/dist/activities/capabilities.d.ts +6 -0
- package/dist/activities/codex-capacity.d.ts +1 -0
- package/dist/activities/environment.d.ts +7 -1
- package/dist/activities/packs.d.ts +8 -10
- package/dist/activities/run-input.d.ts +8 -0
- package/dist/activities/session-state.d.ts +7 -3
- package/dist/activities/site-auth-maintenance.d.ts +27 -0
- package/dist/activities/types.d.ts +43 -0
- package/dist/activities/video-generation-admission.d.ts +9 -4
- package/dist/activities/video-generation-credential.d.ts +17 -0
- package/dist/activities/video-reference-staging.d.ts +7 -1
- package/dist/activities/xai-auth.d.ts +23 -0
- package/dist/activities/xai-image-generation.d.ts +26 -0
- package/dist/activities/xai-video-generation.d.ts +18 -0
- package/dist/{activities-control-44KBWLSM.js → activities-control-SMKF2OBE.js} +438 -55
- package/dist/activities-control-SMKF2OBE.js.map +1 -0
- package/dist/activities-control.d.ts +10 -2
- package/dist/{activities-turn-K4CEDEGJ.js → activities-turn-IDS3WEHA.js} +6704 -4847
- package/dist/activities-turn-IDS3WEHA.js.map +1 -0
- package/dist/activities.d.ts +11 -3
- package/dist/{chunk-E3RI3I33.js → chunk-N5TJIFCE.js} +156 -10
- package/dist/chunk-N5TJIFCE.js.map +1 -0
- package/dist/{chunk-KKCRGAGZ.js → chunk-P6Z2SLAA.js} +99 -98
- package/dist/chunk-P6Z2SLAA.js.map +1 -0
- package/dist/index.d.ts +14 -1
- package/dist/index.js +49 -2
- package/dist/index.js.map +1 -1
- package/dist/model-checkpoint-memory-collector.d.ts +1 -0
- package/dist/model-context-contributions.d.ts +40 -0
- package/dist/observability-metrics.d.ts +57 -1
- package/dist/sandbox-routing.d.ts +4 -0
- package/dist/workflow-bundle.js +232 -18
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/session.d.ts +17 -1
- package/dist/workflows/site-auth-maintenance.d.ts +2 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +16 -15
- package/src/activities/agent-run-admission.ts +76 -0
- package/src/activities/agent-turn.ts +2777 -1241
- package/src/activities/browser-state-artifact-reaper.ts +120 -0
- package/src/activities/capabilities.ts +10 -0
- package/src/activities/codex-capacity.ts +48 -1
- package/src/activities/environment.ts +14 -8
- package/src/activities/goals.ts +2 -1
- package/src/activities/packs.ts +45 -122
- package/src/activities/parent-wake.ts +13 -1
- package/src/activities/run-input.ts +137 -45
- package/src/activities/scheduled-tasks.ts +30 -73
- package/src/activities/session-state.ts +87 -5
- package/src/activities/site-auth-maintenance.ts +269 -0
- package/src/activities/types.ts +44 -0
- package/src/activities/video-generation-admission.ts +37 -4
- package/src/activities/video-generation-credential.ts +84 -6
- package/src/activities/video-generation-reconciliation.ts +212 -19
- package/src/activities/video-media-validation.ts +8 -2
- package/src/activities/video-reference-staging.ts +121 -34
- package/src/activities/xai-auth.ts +122 -0
- package/src/activities/xai-image-generation.ts +86 -0
- package/src/activities/xai-video-generation.ts +69 -0
- package/src/activities-control.ts +4 -0
- package/src/activities.ts +7 -0
- package/src/index.ts +51 -0
- package/src/model-checkpoint-memory-collector.ts +8 -1
- package/src/model-context-contributions.ts +159 -0
- package/src/observability-metrics.ts +282 -12
- package/src/sandbox-routing.ts +13 -0
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/file-upload-reaper.ts +8 -2
- package/src/workflows/session.ts +219 -14
- package/src/workflows/site-auth-maintenance.ts +14 -0
- package/src/workflows.ts +1 -0
- package/dist/activities-control-44KBWLSM.js.map +0 -1
- package/dist/activities-turn-K4CEDEGJ.js.map +0 -1
- package/dist/chunk-E3RI3I33.js.map +0 -1
- package/dist/chunk-KKCRGAGZ.js.map +0 -1
|
@@ -16,12 +16,26 @@ import {
|
|
|
16
16
|
getSessionRootId,
|
|
17
17
|
getSessionGoal,
|
|
18
18
|
getHumanInputResumeForEvent,
|
|
19
|
+
getInteractionInterventionResumeForEvent,
|
|
19
20
|
getSessionHumanInputRequest,
|
|
20
21
|
installOrReadTurnExecutionPolicyForAttempt,
|
|
21
22
|
persistAttemptToolCatalog,
|
|
22
23
|
workspaceCodexSubscriptionActive,
|
|
24
|
+
acquireXaiCredentialLease,
|
|
25
|
+
selectXaiCredentialForUse,
|
|
26
|
+
materializeXaiCredentialForRun,
|
|
27
|
+
resolveXaiProviderAccountAuthoritySnapshotForAcceptance,
|
|
28
|
+
heartbeatXaiCredentialLeaseUntil,
|
|
29
|
+
releaseXaiCredentialLease,
|
|
30
|
+
getXaiSessionAccountPin,
|
|
31
|
+
setXaiSessionAccountPin,
|
|
32
|
+
recordXaiSessionLastAccount,
|
|
33
|
+
updateXaiQuotaMetadata,
|
|
34
|
+
XAI_CREDENTIAL_LEASE_TTL_MS,
|
|
23
35
|
acquireCodexCredentialLease,
|
|
24
36
|
armCodexCapacityWait,
|
|
37
|
+
armXaiCapacityWait,
|
|
38
|
+
reconcileXaiCapacityWait,
|
|
25
39
|
heartbeatCodexCredentialLeaseUntil,
|
|
26
40
|
releaseCodexCredentialLease,
|
|
27
41
|
CODEX_CREDENTIAL_LEASE_TTL_MS,
|
|
@@ -73,6 +87,7 @@ import {
|
|
|
73
87
|
SandboxImageConflictError,
|
|
74
88
|
ActiveSessionHistoryLimitExceededError,
|
|
75
89
|
ApprovalRunStateLimitExceededError,
|
|
90
|
+
isRetryableDatabaseTransportFailure,
|
|
76
91
|
isSessionEventPersistenceError,
|
|
77
92
|
getEnrollment,
|
|
78
93
|
abandonRecordingForTurnAttempt,
|
|
@@ -105,11 +120,14 @@ import {
|
|
|
105
120
|
normalizeModelCallUsage,
|
|
106
121
|
normalizeSdkEvent,
|
|
107
122
|
normalizeProtocolJsonValue,
|
|
123
|
+
compactMcpResultCustomDataRunState,
|
|
124
|
+
releaseMcpResultCustomDataFromSdkEvent,
|
|
108
125
|
projectHistoryForProvider,
|
|
109
126
|
restoreGenericDispatchHistoryItems,
|
|
110
127
|
sanitizeHistoryItemsForModel,
|
|
111
128
|
projectModelInputForCapabilities,
|
|
112
129
|
appendPersistentSessionSettings,
|
|
130
|
+
appendSessionGoal,
|
|
113
131
|
appendSessionInstructions,
|
|
114
132
|
appendWorkspaceGovernance,
|
|
115
133
|
appendWorkspaceMemory,
|
|
@@ -166,6 +184,7 @@ import {
|
|
|
166
184
|
import { connectionTokenResolverForTurn } from "./mcp-credentials";
|
|
167
185
|
import { buildApiIntegrationServersForTurn } from "./api-integrations";
|
|
168
186
|
import {
|
|
187
|
+
allowedFirstPartyMcpToolsForSession,
|
|
169
188
|
assertTurnExecutionPolicyMatchesConfigV1,
|
|
170
189
|
calculateGatewayReportedCostBreakdown,
|
|
171
190
|
calculateModelUsageCostBreakdown,
|
|
@@ -175,11 +194,13 @@ import {
|
|
|
175
194
|
sandboxLifecycleTransitionWaitMs,
|
|
176
195
|
sandboxWarmRateMicrosPerSecond,
|
|
177
196
|
serviceTierForLatencyMode,
|
|
197
|
+
environmentsEncryptionKeyBytes,
|
|
178
198
|
settingsWithResolvedModelContext,
|
|
179
199
|
resolveTurnExecutionPolicyV1,
|
|
180
200
|
OPENGENI_GATEWAY_MODELS,
|
|
181
201
|
OPENGENI_GATEWAY_PROVIDER_ID,
|
|
182
202
|
WORKSPACE_GATEWAY_PROVIDER_ID,
|
|
203
|
+
codemodeWorkspaceUrl,
|
|
183
204
|
isDirectOpenAiApiBaseUrl,
|
|
184
205
|
resolveModelProvider,
|
|
185
206
|
type ModelUsageInput,
|
|
@@ -193,6 +214,7 @@ import {
|
|
|
193
214
|
settingsWithEnabledCapabilityMcpServers,
|
|
194
215
|
settingsWithSessionMcpServersForRun,
|
|
195
216
|
settingsWithWorkspaceGatewayCredential,
|
|
217
|
+
withXaiSubscriptionProvider,
|
|
196
218
|
} from "./capabilities";
|
|
197
219
|
import {
|
|
198
220
|
CODEX_USAGE_EXHAUSTED_PCT,
|
|
@@ -239,6 +261,15 @@ import {
|
|
|
239
261
|
type CodexUsageHeaderSnapshot,
|
|
240
262
|
} from "@opengeni/codex";
|
|
241
263
|
import { mergeResourceRefs } from "./common";
|
|
264
|
+
import {
|
|
265
|
+
fetchXaiSubscriptionQuota,
|
|
266
|
+
isXaiSubscriptionTransportError,
|
|
267
|
+
XaiSubscriptionReloginRequired,
|
|
268
|
+
xaiSubscriptionRequestStorage,
|
|
269
|
+
type XaiSubscriptionRequestContext,
|
|
270
|
+
} from "@opengeni/xai-subscription";
|
|
271
|
+
import { buildXaiTurnRequestAuthorization } from "./xai-auth";
|
|
272
|
+
import { executeXaiSubscriptionImageGeneration } from "./xai-image-generation";
|
|
242
273
|
import {
|
|
243
274
|
assertSessionAllowsProductModel,
|
|
244
275
|
defaultSessionMcpServerIds,
|
|
@@ -254,8 +285,10 @@ import { TurnAttemptFencedError } from "./turn-attempt-fenced";
|
|
|
254
285
|
import {
|
|
255
286
|
admitVideoGenerationRequest,
|
|
256
287
|
managedVideoGenerationCredentialLease,
|
|
288
|
+
xaiVideoGenerationCredentialLease,
|
|
257
289
|
type VideoGenerationCredentialLease,
|
|
258
290
|
} from "./video-generation-admission";
|
|
291
|
+
import { VideoReferenceInputError } from "./video-reference-staging";
|
|
259
292
|
import {
|
|
260
293
|
assertGitCredentialRenewalTransportUnchanged,
|
|
261
294
|
gitCredentialAuthorityForTurn,
|
|
@@ -292,7 +325,7 @@ import {
|
|
|
292
325
|
mergeRigDefaultVariableSetEnvironment,
|
|
293
326
|
rigProviderImageSourceImage,
|
|
294
327
|
resolveWorkspacePackRuntime,
|
|
295
|
-
|
|
328
|
+
resolveWorkspaceInstalledSkillRuntime,
|
|
296
329
|
settingsWithPackSandboxImage,
|
|
297
330
|
settingsWithRigImage,
|
|
298
331
|
settingsWithRigProviderImage,
|
|
@@ -312,6 +345,7 @@ import {
|
|
|
312
345
|
import type {
|
|
313
346
|
TurnActivityServices as ActivityServices,
|
|
314
347
|
EscapedMcpTimeoutRecoveryDetail,
|
|
348
|
+
PreClaimFailureDetail,
|
|
315
349
|
RunAgentTurnInput,
|
|
316
350
|
RunAgentTurnResult,
|
|
317
351
|
SessionAttemptQuiescenceProof,
|
|
@@ -319,6 +353,8 @@ import type {
|
|
|
319
353
|
import {
|
|
320
354
|
ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
|
|
321
355
|
ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
|
|
356
|
+
PRE_CLAIM_FAILURE_MESSAGE,
|
|
357
|
+
PRE_CLAIM_FAILURE_TYPE,
|
|
322
358
|
} from "./types";
|
|
323
359
|
import {
|
|
324
360
|
resumeBoxForTurn,
|
|
@@ -345,13 +381,20 @@ import {
|
|
|
345
381
|
recordCreditMicros,
|
|
346
382
|
recordModelCacheTokens,
|
|
347
383
|
recordModelInputTokens,
|
|
384
|
+
recordModelRequestPhase,
|
|
385
|
+
recordCompanyBrainContributions,
|
|
348
386
|
recordSessionEventAppendLatency,
|
|
349
387
|
recordSessionEventPublishLatency,
|
|
388
|
+
recordTurnSandboxEstablishPolicy,
|
|
389
|
+
recordTurnStartupPhase,
|
|
390
|
+
recordTurnWorkerPreparationTotal,
|
|
350
391
|
runtimeMetricsHooksForObservability,
|
|
351
392
|
StreamTimingMetrics,
|
|
352
393
|
turnLifecycleMetricsFor,
|
|
353
394
|
type TurnOutcome,
|
|
395
|
+
type TurnSandboxEstablishReason,
|
|
354
396
|
} from "../observability-metrics";
|
|
397
|
+
import { buildCompanyBrainContributionReceipt } from "../model-context-contributions";
|
|
355
398
|
import {
|
|
356
399
|
beginRecording,
|
|
357
400
|
discardUnpublishedRecording,
|
|
@@ -403,11 +446,12 @@ import {
|
|
|
403
446
|
} from "@opengeni/runtime";
|
|
404
447
|
import {
|
|
405
448
|
CAPABILITY_DESCRIPTORS,
|
|
406
|
-
DEFAULT_FIRST_PARTY_MCP_TOOLS,
|
|
407
449
|
evaluateWorkspaceModelPolicy,
|
|
408
450
|
readTurnExecutionPolicyV1,
|
|
409
451
|
resolveWorkspaceAgentHumanInputEnabled,
|
|
452
|
+
resolveWorkspaceMemoryPromptMode,
|
|
410
453
|
resourceMountPath,
|
|
454
|
+
VideoGenerationRejectedResult,
|
|
411
455
|
type LatencyMode,
|
|
412
456
|
type ResourceRef,
|
|
413
457
|
type RetainedArtifactMetadata,
|
|
@@ -418,6 +462,7 @@ import {
|
|
|
418
462
|
type SessionTurn,
|
|
419
463
|
type ToolAuthNeededPayload,
|
|
420
464
|
type TurnExecutionPolicyV1,
|
|
465
|
+
type XaiProviderAccountAuthoritySnapshotV1,
|
|
421
466
|
} from "@opengeni/contracts";
|
|
422
467
|
import { createHash, randomUUID } from "node:crypto";
|
|
423
468
|
import { createModelCheckpointMemoryCollector } from "../model-checkpoint-memory-collector";
|
|
@@ -496,6 +541,7 @@ export function shouldPublishToolAuthNeededForTurn(
|
|
|
496
541
|
export function turnExecutionPolicyBillingIdentity(policy: TurnExecutionPolicyV1): {
|
|
497
542
|
externallyBilled: boolean;
|
|
498
543
|
codexSubscription: boolean;
|
|
544
|
+
xaiSubscription: boolean;
|
|
499
545
|
} {
|
|
500
546
|
return {
|
|
501
547
|
externallyBilled: policy.billing.metering === "external",
|
|
@@ -503,6 +549,10 @@ export function turnExecutionPolicyBillingIdentity(policy: TurnExecutionPolicyV1
|
|
|
503
549
|
policy.providerId === "codex-subscription" &&
|
|
504
550
|
policy.credentialSource.kind === "connected_subscription" &&
|
|
505
551
|
policy.credentialSource.provider === "codex",
|
|
552
|
+
xaiSubscription:
|
|
553
|
+
policy.providerId === "supergrok-subscription" &&
|
|
554
|
+
policy.credentialSource.kind === "connected_subscription" &&
|
|
555
|
+
policy.credentialSource.provider === "xai",
|
|
506
556
|
};
|
|
507
557
|
}
|
|
508
558
|
|
|
@@ -662,6 +712,52 @@ export function escapedMcpTimeoutRecoveryFailure(input: {
|
|
|
662
712
|
});
|
|
663
713
|
}
|
|
664
714
|
|
|
715
|
+
/**
|
|
716
|
+
* Convert the atomic claim transaction's failure into a small, stable
|
|
717
|
+
* Temporal wire contract. The original error remains in activity diagnostics,
|
|
718
|
+
* but SQL text, parameters, and arbitrary invariant messages never enter
|
|
719
|
+
* workflow history. Contention and operational database unavailability are
|
|
720
|
+
* safe to re-read after backoff because the claim transaction contains no
|
|
721
|
+
* model/tool effects; permanent database and state failures require terminal
|
|
722
|
+
* settlement.
|
|
723
|
+
*/
|
|
724
|
+
export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
|
|
725
|
+
const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
|
|
726
|
+
const sqlState = persistenceFailure?.details.sqlState ?? null;
|
|
727
|
+
const rawTransportFailure = sqlState === null && isRetryableDatabaseTransportFailure(error);
|
|
728
|
+
// The database transaction itself retries only the two contention failures
|
|
729
|
+
// proven safe for immediate replay. Once the activity has failed, the
|
|
730
|
+
// workflow may also retry operational outages after a durable re-read and
|
|
731
|
+
// bounded delay. Unknown driver/database failures stay recoverable because
|
|
732
|
+
// they commonly represent a lost connection; known constraint, auth, and
|
|
733
|
+
// application SQLSTATEs are permanent and must not create an infinite loop.
|
|
734
|
+
const retryable = Boolean(
|
|
735
|
+
rawTransportFailure ||
|
|
736
|
+
(persistenceFailure &&
|
|
737
|
+
(sqlState === null ||
|
|
738
|
+
sqlState.startsWith("08") ||
|
|
739
|
+
sqlState.startsWith("40") ||
|
|
740
|
+
sqlState.startsWith("53") ||
|
|
741
|
+
sqlState === "55P03" ||
|
|
742
|
+
sqlState === "57014" ||
|
|
743
|
+
sqlState === "57P01" ||
|
|
744
|
+
sqlState === "57P02" ||
|
|
745
|
+
sqlState === "57P03" ||
|
|
746
|
+
sqlState.startsWith("58"))),
|
|
747
|
+
);
|
|
748
|
+
const detail: PreClaimFailureDetail = {
|
|
749
|
+
disposition: retryable ? "retryable" : "permanent",
|
|
750
|
+
code:
|
|
751
|
+
persistenceFailure?.details.code ?? (rawTransportFailure ? "db_failure" : "claim_invariant"),
|
|
752
|
+
};
|
|
753
|
+
return ApplicationFailure.create({
|
|
754
|
+
message: PRE_CLAIM_FAILURE_MESSAGE,
|
|
755
|
+
type: PRE_CLAIM_FAILURE_TYPE,
|
|
756
|
+
nonRetryable: true,
|
|
757
|
+
details: [detail],
|
|
758
|
+
});
|
|
759
|
+
}
|
|
760
|
+
|
|
665
761
|
/**
|
|
666
762
|
* Resolve which Codex account a turn runs on (multi-account P1): session-pin >
|
|
667
763
|
* workspace-active. No rotation in P1. The selected id must still be in the
|
|
@@ -687,6 +783,12 @@ export function filterUnmaterializedSandboxFileDownloads(
|
|
|
687
783
|
return downloads.filter((download) => !materializedFileIds.has(download.fileId));
|
|
688
784
|
}
|
|
689
785
|
|
|
786
|
+
export function sandboxFileMaterializationOutcome(
|
|
787
|
+
failures: readonly SandboxFileDownloadFailure[],
|
|
788
|
+
): "completed" | "failed" {
|
|
789
|
+
return failures.length === 0 ? "completed" : "failed";
|
|
790
|
+
}
|
|
791
|
+
|
|
690
792
|
/** Fixed-length one-way tenant correlation for metrics/alerts; never a raw id. */
|
|
691
793
|
export function codexWorkspaceMetricKey(workspaceId: string): string {
|
|
692
794
|
return createHash("sha256").update(workspaceId).digest("hex").slice(0, 12);
|
|
@@ -714,6 +816,55 @@ export function stableHumanInputRequestId(
|
|
|
714
816
|
return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice(12, 16)}-${value.slice(16, 20)}-${value.slice(20)}`;
|
|
715
817
|
}
|
|
716
818
|
|
|
819
|
+
export function stableInteractionInterventionId(
|
|
820
|
+
sessionId: string,
|
|
821
|
+
turnId: string,
|
|
822
|
+
toolCallId: string,
|
|
823
|
+
): string {
|
|
824
|
+
return stableInteractionInterventionUuid(
|
|
825
|
+
"opengeni-interaction-intervention-v1",
|
|
826
|
+
sessionId,
|
|
827
|
+
turnId,
|
|
828
|
+
toolCallId,
|
|
829
|
+
);
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
export function stableInteractionInterventionOperationId(
|
|
833
|
+
sessionId: string,
|
|
834
|
+
turnId: string,
|
|
835
|
+
toolCallId: string,
|
|
836
|
+
): string {
|
|
837
|
+
return stableInteractionInterventionUuid(
|
|
838
|
+
"opengeni-interaction-intervention-operation-v1",
|
|
839
|
+
sessionId,
|
|
840
|
+
turnId,
|
|
841
|
+
toolCallId,
|
|
842
|
+
);
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
function stableInteractionInterventionUuid(
|
|
846
|
+
namespace: string,
|
|
847
|
+
sessionId: string,
|
|
848
|
+
turnId: string,
|
|
849
|
+
toolCallId: string,
|
|
850
|
+
): string {
|
|
851
|
+
const hex = createHash("sha256")
|
|
852
|
+
.update(namespace)
|
|
853
|
+
.update("\0")
|
|
854
|
+
.update(sessionId)
|
|
855
|
+
.update("\0")
|
|
856
|
+
.update(turnId)
|
|
857
|
+
.update("\0")
|
|
858
|
+
.update(toolCallId)
|
|
859
|
+
.digest("hex")
|
|
860
|
+
.slice(0, 32)
|
|
861
|
+
.split("");
|
|
862
|
+
hex[12] = "5";
|
|
863
|
+
hex[16] = ["8", "9", "a", "b"][Number.parseInt(hex[16] ?? "0", 16) % 4] ?? "8";
|
|
864
|
+
const value = hex.join("");
|
|
865
|
+
return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice(12, 16)}-${value.slice(16, 20)}-${value.slice(20)}`;
|
|
866
|
+
}
|
|
867
|
+
|
|
717
868
|
/**
|
|
718
869
|
* True when this activity attempt was cancelled because its hosting worker is
|
|
719
870
|
* shutting down gracefully (SIGTERM during a deploy), as opposed to a
|
|
@@ -1898,6 +2049,42 @@ export function managedSandboxOwnershipForTurn(
|
|
|
1898
2049
|
};
|
|
1899
2050
|
}
|
|
1900
2051
|
|
|
2052
|
+
/** A sandboxless home still establishes an explicitly attached Connected Machine. */
|
|
2053
|
+
export function shouldEstablishSandboxForTurn(
|
|
2054
|
+
sandboxOwnershipEnabled: boolean,
|
|
2055
|
+
homeBackend: Settings["sandboxBackend"],
|
|
2056
|
+
machinePrimary: boolean,
|
|
2057
|
+
): boolean {
|
|
2058
|
+
return sandboxOwnershipEnabled && (homeBackend !== "none" || machinePrimary);
|
|
2059
|
+
}
|
|
2060
|
+
|
|
2061
|
+
export function sandboxEstablishPolicyDecision(input: {
|
|
2062
|
+
lazyEnabled: boolean;
|
|
2063
|
+
machinePrimary: boolean;
|
|
2064
|
+
sandboxBackend: Settings["sandboxBackend"];
|
|
2065
|
+
hasInitialRunCredentialMaterial: boolean;
|
|
2066
|
+
generatedVideoFileCount: number;
|
|
2067
|
+
hasSignedFileResources: boolean;
|
|
2068
|
+
}): { policy: "eager" | "on-demand"; reason: TurnSandboxEstablishReason } {
|
|
2069
|
+
if (!input.lazyEnabled) return { policy: "eager", reason: "lazy_disabled" };
|
|
2070
|
+
if (input.machinePrimary) return { policy: "eager", reason: "machine_primary" };
|
|
2071
|
+
if (input.sandboxBackend === "none") return { policy: "eager", reason: "backend_none" };
|
|
2072
|
+
if (input.hasInitialRunCredentialMaterial) {
|
|
2073
|
+
return { policy: "eager", reason: "initial_run_credentials" };
|
|
2074
|
+
}
|
|
2075
|
+
if (input.generatedVideoFileCount > 0) {
|
|
2076
|
+
return { policy: "eager", reason: "generated_video_files" };
|
|
2077
|
+
}
|
|
2078
|
+
// Signed file downloads must finish before the first model boundary so any
|
|
2079
|
+
// integrity/download failure is present in the prepared input. Deferring the
|
|
2080
|
+
// box until the first tool call would make that first request advertise a
|
|
2081
|
+
// path whose materialization outcome is not known yet.
|
|
2082
|
+
if (input.hasSignedFileResources) {
|
|
2083
|
+
return { policy: "eager", reason: "signed_file_resources" };
|
|
2084
|
+
}
|
|
2085
|
+
return { policy: "on-demand", reason: "eligible" };
|
|
2086
|
+
}
|
|
2087
|
+
|
|
1901
2088
|
/**
|
|
1902
2089
|
* Classify a persisted active-sandbox pointer for TURN-START RECONCILE (issue #341
|
|
1903
2090
|
* invariant B). Returns the typed reason to RESET the pointer to the session HOME,
|
|
@@ -2194,9 +2381,12 @@ export function structuredToolTransportForTurn(
|
|
|
2194
2381
|
resolvedModel: { provider: { kind: RegistryProviderKind } } | null,
|
|
2195
2382
|
): boolean {
|
|
2196
2383
|
if (!resolvedModel) return true;
|
|
2197
|
-
return ![
|
|
2198
|
-
|
|
2199
|
-
|
|
2384
|
+
return ![
|
|
2385
|
+
"codex-subscription",
|
|
2386
|
+
"xai-subscription",
|
|
2387
|
+
"vercel-gateway-managed",
|
|
2388
|
+
"vercel-gateway-workspace",
|
|
2389
|
+
].includes(resolvedModel.provider.kind);
|
|
2200
2390
|
}
|
|
2201
2391
|
|
|
2202
2392
|
/**
|
|
@@ -2243,6 +2433,21 @@ export function hostedWebSearchForTurn(
|
|
|
2243
2433
|
return resolvedModel?.configured.hostedWebSearch ?? deploymentWebSearchEnabled;
|
|
2244
2434
|
}
|
|
2245
2435
|
|
|
2436
|
+
/**
|
|
2437
|
+
* Image generation is an optional connected-account capability. A delegated
|
|
2438
|
+
* subscription may still authorize the text model without carrying the local
|
|
2439
|
+
* credential identity required for paid image operations; that must narrow the
|
|
2440
|
+
* tool catalog, not reject an otherwise valid turn.
|
|
2441
|
+
*/
|
|
2442
|
+
export function connectedSubscriptionImageGenerationAuthority<T>(
|
|
2443
|
+
credentialContext: T | null | undefined,
|
|
2444
|
+
credentialId: string | null | undefined,
|
|
2445
|
+
): { credentialContext: T; credentialId: string } | null {
|
|
2446
|
+
if (credentialContext == null || typeof credentialId !== "string" || credentialId.length === 0)
|
|
2447
|
+
return null;
|
|
2448
|
+
return { credentialContext, credentialId };
|
|
2449
|
+
}
|
|
2450
|
+
|
|
2246
2451
|
/** Direct OpenAI hosted image IDs are reusable only by the exact API credential. */
|
|
2247
2452
|
export function openAiHostedImageProviderBindingForTurn(
|
|
2248
2453
|
settings: Pick<Settings, "openaiProvider" | "openaiApiKey" | "openaiBaseUrl">,
|
|
@@ -2676,6 +2881,10 @@ export async function finalizeDurableTurnOpStreams(
|
|
|
2676
2881
|
|
|
2677
2882
|
export function createRunAgentTurnActivity(services: () => Promise<ActivityServices>) {
|
|
2678
2883
|
const modelCheckpointMemoryCollector = createModelCheckpointMemoryCollector();
|
|
2884
|
+
// Keep a distinct cooldown for terminal collection. A collection at the
|
|
2885
|
+
// final model checkpoint happens while the activity still owns its complete
|
|
2886
|
+
// turn graph and must not suppress collection after that graph is released.
|
|
2887
|
+
const turnCompletionMemoryCollector = createModelCheckpointMemoryCollector();
|
|
2679
2888
|
return async function runAgentTurn(input: RunAgentTurnInput): Promise<RunAgentTurnResult> {
|
|
2680
2889
|
const {
|
|
2681
2890
|
settings,
|
|
@@ -2767,6 +2976,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2767
2976
|
);
|
|
2768
2977
|
const sandboxOperationObserver = sandboxOperationMetricObserver(observability);
|
|
2769
2978
|
let isCodexTurn = false;
|
|
2979
|
+
let isXaiTurn = false;
|
|
2770
2980
|
let isExternallyBilledTurn = false;
|
|
2771
2981
|
let executionGeneration = 0;
|
|
2772
2982
|
let providerRecoveryCount = 0;
|
|
@@ -2898,6 +3108,83 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2898
3108
|
}, 60_000);
|
|
2899
3109
|
codexLeaseHeartbeatTimer.unref?.();
|
|
2900
3110
|
};
|
|
3111
|
+
let xaiLeaseHeld = false;
|
|
3112
|
+
let xaiLeaseLost = false;
|
|
3113
|
+
let xaiLeaseSubjectId: string | null = null;
|
|
3114
|
+
let xaiLeaseHolderId: string | null = null;
|
|
3115
|
+
let xaiLeaseGeneration: number | null = null;
|
|
3116
|
+
let xaiLeaseConfirmedUntilMs: number | null = null;
|
|
3117
|
+
let xaiLeaseHeartbeatInFlight = false;
|
|
3118
|
+
let xaiLeaseHeartbeatTimer: ReturnType<typeof setInterval> | undefined;
|
|
3119
|
+
const markXaiLeaseLost = (reason: "deadline" | "not_found"): void => {
|
|
3120
|
+
if (xaiLeaseLost) return;
|
|
3121
|
+
xaiLeaseLost = true;
|
|
3122
|
+
observability.warn("xAI credential lease was lost during an active turn", {
|
|
3123
|
+
workspaceId: input.workspaceId,
|
|
3124
|
+
turnId,
|
|
3125
|
+
reason,
|
|
3126
|
+
});
|
|
3127
|
+
};
|
|
3128
|
+
const renewXaiLease = async (): Promise<void> => {
|
|
3129
|
+
if (
|
|
3130
|
+
!turnId ||
|
|
3131
|
+
!xaiLeaseHeld ||
|
|
3132
|
+
!xaiLeaseSubjectId ||
|
|
3133
|
+
!xaiLeaseHolderId ||
|
|
3134
|
+
xaiLeaseGeneration === null ||
|
|
3135
|
+
xaiLeaseLost
|
|
3136
|
+
) {
|
|
3137
|
+
return;
|
|
3138
|
+
}
|
|
3139
|
+
if (codexCredentialLeaseDeadlineExpired(xaiLeaseConfirmedUntilMs)) {
|
|
3140
|
+
markXaiLeaseLost("deadline");
|
|
3141
|
+
return;
|
|
3142
|
+
}
|
|
3143
|
+
if (xaiLeaseHeartbeatInFlight) return;
|
|
3144
|
+
xaiLeaseHeartbeatInFlight = true;
|
|
3145
|
+
const renewalStartedAtMs = performance.now();
|
|
3146
|
+
try {
|
|
3147
|
+
const renewedUntil = await heartbeatXaiCredentialLeaseUntil(db, {
|
|
3148
|
+
workspaceId: input.workspaceId,
|
|
3149
|
+
subjectId: xaiLeaseSubjectId,
|
|
3150
|
+
turnId,
|
|
3151
|
+
holderId: xaiLeaseHolderId,
|
|
3152
|
+
generation: xaiLeaseGeneration,
|
|
3153
|
+
leaseTtlMs: XAI_CREDENTIAL_LEASE_TTL_MS,
|
|
3154
|
+
});
|
|
3155
|
+
if (!renewedUntil) {
|
|
3156
|
+
markXaiLeaseLost("not_found");
|
|
3157
|
+
} else {
|
|
3158
|
+
xaiLeaseConfirmedUntilMs = renewalStartedAtMs + XAI_CREDENTIAL_LEASE_TTL_MS;
|
|
3159
|
+
}
|
|
3160
|
+
} catch (error) {
|
|
3161
|
+
if (codexCredentialLeaseDeadlineExpired(xaiLeaseConfirmedUntilMs)) {
|
|
3162
|
+
markXaiLeaseLost("deadline");
|
|
3163
|
+
return;
|
|
3164
|
+
}
|
|
3165
|
+
observability.warn("xAI credential lease heartbeat failed", {
|
|
3166
|
+
workspaceId: input.workspaceId,
|
|
3167
|
+
turnId,
|
|
3168
|
+
...safeErrorDiagnostic(error),
|
|
3169
|
+
});
|
|
3170
|
+
} finally {
|
|
3171
|
+
xaiLeaseHeartbeatInFlight = false;
|
|
3172
|
+
}
|
|
3173
|
+
};
|
|
3174
|
+
const startXaiLeaseHeartbeat = (): void => {
|
|
3175
|
+
if (!turnId || xaiLeaseHeartbeatTimer) return;
|
|
3176
|
+
xaiLeaseHeartbeatTimer = setInterval(() => {
|
|
3177
|
+
void renewXaiLease();
|
|
3178
|
+
}, 60_000);
|
|
3179
|
+
xaiLeaseHeartbeatTimer.unref?.();
|
|
3180
|
+
};
|
|
3181
|
+
const renewServingCredentialLease = async (
|
|
3182
|
+
reason: "timer" | "runtime_event" | "model_usage",
|
|
3183
|
+
): Promise<void> => {
|
|
3184
|
+
await renewCodexLease(reason);
|
|
3185
|
+
await renewXaiLease();
|
|
3186
|
+
};
|
|
3187
|
+
const servingCredentialLeaseLost = (): boolean => codexLeaseLost || xaiLeaseLost;
|
|
2901
3188
|
// P1.2 ownership inversion: when sandboxOwnershipEnabled, the turn resolves
|
|
2902
3189
|
// the one box by id from the group lease and injects it NON-OWNED into the
|
|
2903
3190
|
// run. null when the flag is off (byte-for-byte the legacy build-and-discard
|
|
@@ -3670,6 +3957,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3670
3957
|
compactRetainedScreenshotRunState(serialized, retainedScreenshotReceiptsByCallId),
|
|
3671
3958
|
generatedImageReceiptsByProviderItemId,
|
|
3672
3959
|
);
|
|
3960
|
+
const compactApprovalRunState = (serialized: string): string =>
|
|
3961
|
+
compactMcpResultCustomDataRunState(compactMediaRunState(serialized));
|
|
3673
3962
|
// Explicit image-producing tools cross the durable session-media boundary.
|
|
3674
3963
|
// Incidental frames returned by click/scroll actions remain unretained; an
|
|
3675
3964
|
// intentional computer_screenshot or view_image result must never be lost
|
|
@@ -3847,6 +4136,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3847
4136
|
// The Codex account this turn runs on (pin > workspace active), resolved once
|
|
3848
4137
|
// a codex-billed turn is confirmed and threaded into the token resolver below.
|
|
3849
4138
|
let effectiveCodexCredentialId: string | null = null;
|
|
4139
|
+
let effectiveXaiCredentialId: string | null = null;
|
|
4140
|
+
let xaiRotationEnabled = false;
|
|
4141
|
+
let xaiAuthoritySnapshot: XaiProviderAccountAuthoritySnapshotV1 | null = null;
|
|
4142
|
+
let xaiRequestContext: XaiSubscriptionRequestContext | null = null;
|
|
4143
|
+
let xaiCredentialQuarantined = false;
|
|
3850
4144
|
// The session's Codex credential BEFORE this turn resolved its own — captured
|
|
3851
4145
|
// before recordSessionActiveCodexCredential overwrites the durable pointer, so
|
|
3852
4146
|
// a per-call usage log can report whether the serving account CHANGED since the
|
|
@@ -3866,7 +4160,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3866
4160
|
// only exists in the RunState blob), never through a swapped trigger.
|
|
3867
4161
|
let triggerType: string | null = null;
|
|
3868
4162
|
try {
|
|
3869
|
-
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
3870
4163
|
const claim = await claimSessionWorkForAttempt(db, input.workspaceId, {
|
|
3871
4164
|
sessionId: input.sessionId,
|
|
3872
4165
|
workflowId: input.workflowId,
|
|
@@ -3881,6 +4174,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3881
4174
|
}
|
|
3882
4175
|
const turn = claim.turn;
|
|
3883
4176
|
turnId = turn.id;
|
|
4177
|
+
// Establish durable attempt ownership before any later read can fail.
|
|
4178
|
+
// Therefore every failure with no turnId came from the one atomic claim
|
|
4179
|
+
// transaction and can be classified without conflating ordinary runtime
|
|
4180
|
+
// or transport failures with admission failures.
|
|
4181
|
+
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
3884
4182
|
executionGeneration = turn.executionGeneration;
|
|
3885
4183
|
providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
|
|
3886
4184
|
let installedApiIntegrations: readonly ApiIntegrationRuntime[] = [];
|
|
@@ -3914,10 +4212,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3914
4212
|
mcpSettings,
|
|
3915
4213
|
codexSubscriptionActive,
|
|
3916
4214
|
);
|
|
4215
|
+
const xaiSettings = codexSettings.supergrokSubscriptionEnabled
|
|
4216
|
+
? withXaiSubscriptionProvider(codexSettings)
|
|
4217
|
+
: codexSettings;
|
|
3917
4218
|
const capabilitySettings = await settingsWithWorkspaceGatewayCredential(
|
|
3918
4219
|
db,
|
|
3919
4220
|
input.workspaceId,
|
|
3920
|
-
|
|
4221
|
+
xaiSettings,
|
|
3921
4222
|
);
|
|
3922
4223
|
const codexAppsCredentialId = capabilitySettings.codexConnectedAppsEnabled
|
|
3923
4224
|
? await resolveCodexAppsCredentialIdForRun(db, input.workspaceId)
|
|
@@ -3956,6 +4257,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3956
4257
|
const billingIdentity = turnExecutionPolicyBillingIdentity(turnExecutionPolicy);
|
|
3957
4258
|
isExternallyBilledTurn = billingIdentity.externallyBilled;
|
|
3958
4259
|
isCodexTurn = billingIdentity.codexSubscription;
|
|
4260
|
+
isXaiTurn = billingIdentity.xaiSubscription;
|
|
3959
4261
|
triggerEventId = turn.triggerEventId;
|
|
3960
4262
|
const trigger = await getSessionEvent(db, input.workspaceId, triggerEventId);
|
|
3961
4263
|
if (!trigger) {
|
|
@@ -3967,6 +4269,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3967
4269
|
input.sessionId,
|
|
3968
4270
|
trigger,
|
|
3969
4271
|
);
|
|
4272
|
+
const interactionInterventionResume = await getInteractionInterventionResumeForEvent(
|
|
4273
|
+
db,
|
|
4274
|
+
input.workspaceId,
|
|
4275
|
+
input.sessionId,
|
|
4276
|
+
trigger,
|
|
4277
|
+
);
|
|
3970
4278
|
triggerType = trigger.type;
|
|
3971
4279
|
redispatchesAtDispatch = Number(
|
|
3972
4280
|
(turn.metadata as { workerDeathRedispatches?: number } | null)?.workerDeathRedispatches ??
|
|
@@ -4065,6 +4373,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4065
4373
|
if (inputs.length > 0 && !appended.accepted) {
|
|
4066
4374
|
throw new TurnAttemptFencedError("turn execution generation was fenced");
|
|
4067
4375
|
}
|
|
4376
|
+
if (inputs.length > 0) {
|
|
4377
|
+
turnLifecycleMetricsFor(observability).progress(turnId!);
|
|
4378
|
+
}
|
|
4068
4379
|
activityContext?.heartbeat({
|
|
4069
4380
|
...heartbeatDetails,
|
|
4070
4381
|
phase: "events_published",
|
|
@@ -4158,6 +4469,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4158
4469
|
turnMetricOutcome = "cancelled";
|
|
4159
4470
|
return false;
|
|
4160
4471
|
}
|
|
4472
|
+
turnLifecycleMetricsFor(observability).progress(turnId!);
|
|
4161
4473
|
if (recordingForSettlement && preparedRecording) {
|
|
4162
4474
|
if (result.recordingMutationApplied) {
|
|
4163
4475
|
activeRecording = null;
|
|
@@ -4189,6 +4501,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4189
4501
|
// on a healthy worker.
|
|
4190
4502
|
throwIfWorkerShuttingDown();
|
|
4191
4503
|
throwIfTurnCancelled();
|
|
4504
|
+
recordTurnStartupPhase(observability, {
|
|
4505
|
+
phase: "claim_and_policy",
|
|
4506
|
+
provider: turnExecutionPolicy.providerId,
|
|
4507
|
+
backend: turn.sandboxBackend,
|
|
4508
|
+
outcome: "completed",
|
|
4509
|
+
durationSeconds: (performance.now() - activityStarted) / 1_000,
|
|
4510
|
+
});
|
|
4511
|
+
const turnStartSettlementStartedAt = performance.now();
|
|
4192
4512
|
if (
|
|
4193
4513
|
!(await settle({
|
|
4194
4514
|
events: [
|
|
@@ -4205,7 +4525,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4205
4525
|
) {
|
|
4206
4526
|
return claimedResult({ status: "cancelled" });
|
|
4207
4527
|
}
|
|
4528
|
+
recordTurnStartupPhase(observability, {
|
|
4529
|
+
phase: "turn_start_settlement",
|
|
4530
|
+
provider: turnExecutionPolicy.providerId,
|
|
4531
|
+
backend: turn.sandboxBackend,
|
|
4532
|
+
outcome: "completed",
|
|
4533
|
+
durationSeconds: (performance.now() - turnStartSettlementStartedAt) / 1_000,
|
|
4534
|
+
count: 2,
|
|
4535
|
+
});
|
|
4208
4536
|
turnStartedPublished = true;
|
|
4537
|
+
const workerPreparationStartedAt = performance.now();
|
|
4209
4538
|
|
|
4210
4539
|
// Multi-account (P1): resolve the effective Codex account for this turn
|
|
4211
4540
|
// (session-pin > workspace active) and stamp it on the session so the
|
|
@@ -4213,80 +4542,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4213
4542
|
// when it changed from the prior run's account so the pill flips live.
|
|
4214
4543
|
// Gated on the codex-billed predicate — non-codex turns never touch this.
|
|
4215
4544
|
if (isCodexTurn) {
|
|
4216
|
-
const
|
|
4217
|
-
|
|
4218
|
-
|
|
4219
|
-
|
|
4220
|
-
|
|
4221
|
-
|
|
4222
|
-
|
|
4223
|
-
|
|
4224
|
-
|
|
4225
|
-
|
|
4226
|
-
|
|
4227
|
-
|
|
4228
|
-
|
|
4545
|
+
const credentialSelectionStartedAt = performance.now();
|
|
4546
|
+
let credentialSelectionOutcome: "completed" | "failed" = "completed";
|
|
4547
|
+
try {
|
|
4548
|
+
const sessionCodex = await getSessionCodexState(db, input.workspaceId, input.sessionId);
|
|
4549
|
+
const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
|
|
4550
|
+
const sessionPinSource = sessionCodex?.pinSource ?? null;
|
|
4551
|
+
const selectForTurn = (context: CodexCredentialLeaseSelectionContext) =>
|
|
4552
|
+
selectCodexCredentialLeaseForTurn({
|
|
4553
|
+
context,
|
|
4554
|
+
leasingEnabled: settings.codexCredentialLeasingEnabled,
|
|
4555
|
+
sessionId: input.sessionId,
|
|
4556
|
+
sessionPinnedCredentialId: sessionPin,
|
|
4557
|
+
sessionPinSource,
|
|
4558
|
+
sessionLastCredentialId: sessionCodex?.lastCredentialId ?? null,
|
|
4559
|
+
now: new Date(),
|
|
4560
|
+
});
|
|
4229
4561
|
|
|
4230
|
-
|
|
4231
|
-
|
|
4232
|
-
|
|
4233
|
-
|
|
4234
|
-
|
|
4235
|
-
if (settings.codexCredentialLeasingEnabled) {
|
|
4236
|
-
leaseAcquisitionStartedAtMs = performance.now();
|
|
4237
|
-
leased = await acquireCodexCredentialLease(
|
|
4238
|
-
db,
|
|
4239
|
-
{
|
|
4240
|
-
accountId: input.accountId,
|
|
4241
|
-
workspaceId: input.workspaceId,
|
|
4242
|
-
turnId,
|
|
4243
|
-
holderId: codexLeaseHolderId,
|
|
4244
|
-
advanceActivePointer: sessionPin === null,
|
|
4245
|
-
},
|
|
4246
|
-
selectForTurn,
|
|
4247
|
-
);
|
|
4248
|
-
} else {
|
|
4249
|
-
const [rotation, accounts] = await Promise.all([
|
|
4250
|
-
getCodexRotationSettings(db, input.workspaceId),
|
|
4251
|
-
listCodexAccountStatuses(db, input.workspaceId),
|
|
4252
|
-
]);
|
|
4253
|
-
const leaseAccounts = accounts.map((account) => ({
|
|
4254
|
-
...account,
|
|
4255
|
-
activeLeaseCount: 0,
|
|
4256
|
-
selectionCount: 0,
|
|
4257
|
-
lastSelectedAt: null,
|
|
4258
|
-
}));
|
|
4259
|
-
const activeCredentialId = rotation?.activeCredentialId ?? null;
|
|
4260
|
-
const selected = selectForTurn({
|
|
4261
|
-
accounts: leaseAccounts,
|
|
4262
|
-
activeCredentialId,
|
|
4263
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
4264
|
-
leaseRotationEnabled: false,
|
|
4265
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
4266
|
-
existingCredentialId: null,
|
|
4267
|
-
policyScope: null,
|
|
4268
|
-
unavailableDiagnostics: [],
|
|
4269
|
-
});
|
|
4270
|
-
leased = {
|
|
4271
|
-
...selected,
|
|
4272
|
-
accounts: leaseAccounts,
|
|
4273
|
-
activeCredentialId,
|
|
4274
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
4275
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
4276
|
-
reused: false,
|
|
4277
|
-
holderId: null,
|
|
4278
|
-
generation: null,
|
|
4279
|
-
leasedUntil: null,
|
|
4280
|
-
unavailableDiagnostics: [],
|
|
4281
|
-
advanceActivePointer: selected.advanceActivePointer !== false,
|
|
4282
|
-
};
|
|
4283
|
-
}
|
|
4284
|
-
if (leased.decision.kind === "allCapped") {
|
|
4285
|
-
// Bounded self-heal of stale usage cache, then ONE new atomic selection.
|
|
4286
|
-
await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
|
|
4287
|
-
signalCodexCapacityWorkflow,
|
|
4288
|
-
wakeSessionWorkflow,
|
|
4289
|
-
});
|
|
4562
|
+
// Rollout/rollback path is intentionally table-inert. With the flag off,
|
|
4563
|
+
// old and new workers both use legacy pin > active-pointer selection and
|
|
4564
|
+
// neither reads nor writes the additive lease/cursor schema.
|
|
4565
|
+
let leased: CodexCredentialLeaseResult<RotationDecision>;
|
|
4566
|
+
let leaseAcquisitionStartedAtMs: number | null = null;
|
|
4290
4567
|
if (settings.codexCredentialLeasingEnabled) {
|
|
4291
4568
|
leaseAcquisitionStartedAtMs = performance.now();
|
|
4292
4569
|
leased = await acquireCodexCredentialLease(
|
|
@@ -4311,9 +4588,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4311
4588
|
selectionCount: 0,
|
|
4312
4589
|
lastSelectedAt: null,
|
|
4313
4590
|
}));
|
|
4591
|
+
const activeCredentialId = rotation?.activeCredentialId ?? null;
|
|
4314
4592
|
const selected = selectForTurn({
|
|
4315
4593
|
accounts: leaseAccounts,
|
|
4316
|
-
activeCredentialId
|
|
4594
|
+
activeCredentialId,
|
|
4317
4595
|
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
4318
4596
|
leaseRotationEnabled: false,
|
|
4319
4597
|
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
@@ -4324,7 +4602,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4324
4602
|
leased = {
|
|
4325
4603
|
...selected,
|
|
4326
4604
|
accounts: leaseAccounts,
|
|
4327
|
-
activeCredentialId
|
|
4605
|
+
activeCredentialId,
|
|
4328
4606
|
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
4329
4607
|
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
4330
4608
|
reused: false,
|
|
@@ -4335,297 +4613,603 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4335
4613
|
advanceActivePointer: selected.advanceActivePointer !== false,
|
|
4336
4614
|
};
|
|
4337
4615
|
}
|
|
4338
|
-
|
|
4339
|
-
|
|
4340
|
-
|
|
4341
|
-
|
|
4342
|
-
|
|
4343
|
-
|
|
4344
|
-
|
|
4345
|
-
|
|
4346
|
-
|
|
4347
|
-
|
|
4348
|
-
// movement for manual/policy homes; this write only records the NEXT turn's
|
|
4349
|
-
// policy home (or clears a policy pin whose strategy is no longer active).
|
|
4350
|
-
if (
|
|
4351
|
-
selectedPinDisposition === "sharded" &&
|
|
4352
|
-
leased.credentialId !== null &&
|
|
4353
|
-
(sessionPinSource !== "policy" || sessionPin !== leased.credentialId)
|
|
4354
|
-
) {
|
|
4355
|
-
const pinMutation = await withSessionCodexCapacityMutation(
|
|
4356
|
-
db,
|
|
4357
|
-
{
|
|
4358
|
-
workspaceId: input.workspaceId,
|
|
4359
|
-
reason: "codex_policy_pin_changed",
|
|
4360
|
-
},
|
|
4361
|
-
async (tx) => {
|
|
4362
|
-
const changed = await setSessionCodexPinInTransaction(
|
|
4363
|
-
tx,
|
|
4364
|
-
input.workspaceId,
|
|
4365
|
-
input.sessionId,
|
|
4366
|
-
leased.credentialId,
|
|
4367
|
-
"policy",
|
|
4368
|
-
{
|
|
4369
|
-
expected: {
|
|
4370
|
-
pinnedCredentialId: sessionPin,
|
|
4371
|
-
pinSource: sessionPinSource,
|
|
4372
|
-
},
|
|
4373
|
-
},
|
|
4374
|
-
);
|
|
4375
|
-
return { result: changed, changed };
|
|
4376
|
-
},
|
|
4377
|
-
);
|
|
4378
|
-
await signalCodexCapacityWakeTargets(
|
|
4379
|
-
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
4380
|
-
pinMutation.wakeTargets,
|
|
4381
|
-
);
|
|
4382
|
-
} else if (selectedPinDisposition === "clearStale") {
|
|
4383
|
-
const pinMutation = await withSessionCodexCapacityMutation(
|
|
4384
|
-
db,
|
|
4385
|
-
{
|
|
4386
|
-
workspaceId: input.workspaceId,
|
|
4387
|
-
reason: "codex_stale_policy_pin_cleared",
|
|
4388
|
-
},
|
|
4389
|
-
async (tx) => {
|
|
4390
|
-
const changed = await setSessionCodexPinInTransaction(
|
|
4391
|
-
tx,
|
|
4392
|
-
input.workspaceId,
|
|
4393
|
-
input.sessionId,
|
|
4394
|
-
null,
|
|
4395
|
-
"policy",
|
|
4616
|
+
if (leased.decision.kind === "allCapped") {
|
|
4617
|
+
// Bounded self-heal of stale usage cache, then ONE new atomic selection.
|
|
4618
|
+
await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
|
|
4619
|
+
signalCodexCapacityWorkflow,
|
|
4620
|
+
wakeSessionWorkflow,
|
|
4621
|
+
});
|
|
4622
|
+
if (settings.codexCredentialLeasingEnabled) {
|
|
4623
|
+
leaseAcquisitionStartedAtMs = performance.now();
|
|
4624
|
+
leased = await acquireCodexCredentialLease(
|
|
4625
|
+
db,
|
|
4396
4626
|
{
|
|
4397
|
-
|
|
4398
|
-
|
|
4399
|
-
|
|
4400
|
-
|
|
4627
|
+
accountId: input.accountId,
|
|
4628
|
+
workspaceId: input.workspaceId,
|
|
4629
|
+
turnId,
|
|
4630
|
+
holderId: codexLeaseHolderId,
|
|
4631
|
+
advanceActivePointer: sessionPin === null,
|
|
4401
4632
|
},
|
|
4633
|
+
selectForTurn,
|
|
4402
4634
|
);
|
|
4403
|
-
|
|
4404
|
-
|
|
4405
|
-
|
|
4406
|
-
|
|
4407
|
-
|
|
4408
|
-
|
|
4409
|
-
|
|
4410
|
-
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
|
|
4414
|
-
|
|
4415
|
-
|
|
4416
|
-
|
|
4417
|
-
|
|
4418
|
-
|
|
4419
|
-
|
|
4420
|
-
|
|
4421
|
-
|
|
4422
|
-
|
|
4423
|
-
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
|
|
4429
|
-
|
|
4430
|
-
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4437
|
-
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
|
|
4441
|
-
: sessionPin
|
|
4442
|
-
|
|
4443
|
-
|
|
4444
|
-
|
|
4445
|
-
: "active"
|
|
4446
|
-
: rotationDecision.kind === "allCapped"
|
|
4447
|
-
? "all_capped"
|
|
4448
|
-
: "none";
|
|
4449
|
-
const fencedInFlight = leased.reused;
|
|
4450
|
-
const shadowResult = await publishCodexFleetShadowDecisionV1({
|
|
4451
|
-
enabled: settings.codexFleetPolicyShadowEnabled,
|
|
4452
|
-
decision: {
|
|
4453
|
-
accounts: leased.accounts,
|
|
4454
|
-
actualCredentialId: effectiveCodexCredentialId,
|
|
4455
|
-
actualOutcome,
|
|
4456
|
-
actualReason,
|
|
4457
|
-
affinityCredentialId: fencedInFlight
|
|
4458
|
-
? effectiveCodexCredentialId
|
|
4459
|
-
: (sessionPin ?? sessionCodex?.lastCredentialId ?? null),
|
|
4460
|
-
fencedInFlight,
|
|
4461
|
-
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
4462
|
-
now: new Date(),
|
|
4463
|
-
aliasSeed: randomUUID(),
|
|
4464
|
-
},
|
|
4465
|
-
publish,
|
|
4466
|
-
});
|
|
4467
|
-
if (shadowResult.outcome === "published") {
|
|
4468
|
-
const shadowPayload = shadowResult.payload;
|
|
4469
|
-
observability.incrementCounter({
|
|
4470
|
-
name: "opengeni_codex_fleet_shadow_decisions_total",
|
|
4471
|
-
help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
|
|
4472
|
-
labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload),
|
|
4473
|
-
});
|
|
4474
|
-
observability.info("Codex adaptive fleet shadow decision", {
|
|
4475
|
-
workspaceId: input.workspaceId,
|
|
4476
|
-
policyVersion: shadowPayload.replay.policyVersion,
|
|
4477
|
-
inputFingerprint: shadowPayload.replay.inputFingerprint,
|
|
4478
|
-
decisionFingerprint: shadowPayload.replay.decisionFingerprint,
|
|
4479
|
-
actualOutcome: shadowPayload.actual.outcome,
|
|
4480
|
-
shadowOutcome: shadowPayload.replay.decision.outcome,
|
|
4481
|
-
comparison: shadowPayload.comparison,
|
|
4482
|
-
candidateCount: shadowPayload.replay.input.candidates.length,
|
|
4483
|
-
truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
|
|
4484
|
-
payloadBytes: shadowResult.payloadBytes,
|
|
4485
|
-
});
|
|
4486
|
-
} else if (shadowResult.outcome === "failed") {
|
|
4487
|
-
// Shadow observability is explicitly non-authoritative. A malformed
|
|
4488
|
-
// snapshot or event-write fault must never change the authoritative lease,
|
|
4489
|
-
// capacity wait, failover, or the account serving this fenced turn.
|
|
4490
|
-
observability.incrementCounter({
|
|
4491
|
-
name: "opengeni_codex_fleet_shadow_errors_total",
|
|
4492
|
-
help: "Shadow decision build/publication failures.",
|
|
4493
|
-
labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
|
|
4494
|
-
});
|
|
4495
|
-
observability.warn("Codex adaptive fleet shadow decision failed open", {
|
|
4496
|
-
stage: shadowResult.stage,
|
|
4497
|
-
reason: shadowResult.reason,
|
|
4498
|
-
errorClass: "CodexFleetShadowOperationError",
|
|
4499
|
-
errorCode: "codex_fleet_shadow_failed",
|
|
4500
|
-
origin: "worker",
|
|
4501
|
-
payloadBytes: shadowResult.payloadBytes,
|
|
4502
|
-
});
|
|
4503
|
-
}
|
|
4504
|
-
|
|
4505
|
-
const eligibleCount = leased.accounts.filter((account) =>
|
|
4506
|
-
isCodexCredentialEligible(account, new Date()),
|
|
4507
|
-
).length;
|
|
4508
|
-
const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
|
|
4509
|
-
observability.incrementCounter({
|
|
4510
|
-
name: "opengeni_codex_pool_observations_total",
|
|
4511
|
-
help: "Observed eligible Codex pool depth buckets at turn selection.",
|
|
4512
|
-
labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
|
|
4513
|
-
});
|
|
4514
|
-
if (eligibleCount <= 1) {
|
|
4515
|
-
observability.incrementCounter({
|
|
4516
|
-
name: "opengeni_codex_pool_low_total",
|
|
4517
|
-
help: "Alert signal emitted when the eligible Codex pool is zero or one.",
|
|
4518
|
-
labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
|
|
4519
|
-
});
|
|
4520
|
-
observability.warn("Codex eligible credential pool is low", {
|
|
4521
|
-
workspaceId: input.workspaceId,
|
|
4522
|
-
eligibleCount,
|
|
4523
|
-
connectedCount: leased.accounts.length,
|
|
4524
|
-
depth: poolDepth,
|
|
4635
|
+
} else {
|
|
4636
|
+
const [rotation, accounts] = await Promise.all([
|
|
4637
|
+
getCodexRotationSettings(db, input.workspaceId),
|
|
4638
|
+
listCodexAccountStatuses(db, input.workspaceId),
|
|
4639
|
+
]);
|
|
4640
|
+
const leaseAccounts = accounts.map((account) => ({
|
|
4641
|
+
...account,
|
|
4642
|
+
activeLeaseCount: 0,
|
|
4643
|
+
selectionCount: 0,
|
|
4644
|
+
lastSelectedAt: null,
|
|
4645
|
+
}));
|
|
4646
|
+
const selected = selectForTurn({
|
|
4647
|
+
accounts: leaseAccounts,
|
|
4648
|
+
activeCredentialId: rotation?.activeCredentialId ?? null,
|
|
4649
|
+
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
4650
|
+
leaseRotationEnabled: false,
|
|
4651
|
+
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
4652
|
+
existingCredentialId: null,
|
|
4653
|
+
policyScope: null,
|
|
4654
|
+
unavailableDiagnostics: [],
|
|
4655
|
+
});
|
|
4656
|
+
leased = {
|
|
4657
|
+
...selected,
|
|
4658
|
+
accounts: leaseAccounts,
|
|
4659
|
+
activeCredentialId: rotation?.activeCredentialId ?? null,
|
|
4660
|
+
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
4661
|
+
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
4662
|
+
reused: false,
|
|
4663
|
+
holderId: null,
|
|
4664
|
+
generation: null,
|
|
4665
|
+
leasedUntil: null,
|
|
4666
|
+
unavailableDiagnostics: [],
|
|
4667
|
+
advanceActivePointer: selected.advanceActivePointer !== false,
|
|
4668
|
+
};
|
|
4669
|
+
}
|
|
4670
|
+
}
|
|
4671
|
+
const rotationDecision = leased.decision;
|
|
4672
|
+
const selectedPinDisposition = classifyCodexPin({
|
|
4673
|
+
pinnedCredentialId: sessionPin,
|
|
4674
|
+
pinSource: sessionPinSource,
|
|
4675
|
+
strategy: leased.rotationStrategy as CodexRotationStrategy,
|
|
4676
|
+
rotationEnabled: leased.rotationEnabled,
|
|
4525
4677
|
});
|
|
4526
|
-
|
|
4527
|
-
|
|
4528
|
-
|
|
4529
|
-
|
|
4530
|
-
|
|
4531
|
-
|
|
4532
|
-
|
|
4533
|
-
|
|
4534
|
-
|
|
4535
|
-
|
|
4536
|
-
|
|
4537
|
-
|
|
4678
|
+
// pin policy pin persistence follows the atomic credential allocator selection. The selector
|
|
4679
|
+
// already ran exact-turn reuse before policy filtering and vetoed pointer
|
|
4680
|
+
// movement for manual/policy homes; this write only records the NEXT turn's
|
|
4681
|
+
// policy home (or clears a policy pin whose strategy is no longer active).
|
|
4682
|
+
if (
|
|
4683
|
+
selectedPinDisposition === "sharded" &&
|
|
4684
|
+
leased.credentialId !== null &&
|
|
4685
|
+
(sessionPinSource !== "policy" || sessionPin !== leased.credentialId)
|
|
4686
|
+
) {
|
|
4687
|
+
const pinMutation = await withSessionCodexCapacityMutation(
|
|
4688
|
+
db,
|
|
4689
|
+
{
|
|
4690
|
+
workspaceId: input.workspaceId,
|
|
4691
|
+
reason: "codex_policy_pin_changed",
|
|
4692
|
+
},
|
|
4693
|
+
async (tx) => {
|
|
4694
|
+
const changed = await setSessionCodexPinInTransaction(
|
|
4695
|
+
tx,
|
|
4696
|
+
input.workspaceId,
|
|
4697
|
+
input.sessionId,
|
|
4698
|
+
leased.credentialId,
|
|
4699
|
+
"policy",
|
|
4538
4700
|
{
|
|
4539
|
-
|
|
4540
|
-
|
|
4541
|
-
|
|
4542
|
-
reason: "codex_allocator_disabled",
|
|
4543
|
-
requestPreserved: true,
|
|
4701
|
+
expected: {
|
|
4702
|
+
pinnedCredentialId: sessionPin,
|
|
4703
|
+
pinSource: sessionPinSource,
|
|
4544
4704
|
},
|
|
4545
4705
|
},
|
|
4706
|
+
);
|
|
4707
|
+
return { result: changed, changed };
|
|
4708
|
+
},
|
|
4709
|
+
);
|
|
4710
|
+
await signalCodexCapacityWakeTargets(
|
|
4711
|
+
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
4712
|
+
pinMutation.wakeTargets,
|
|
4713
|
+
);
|
|
4714
|
+
} else if (selectedPinDisposition === "clearStale") {
|
|
4715
|
+
const pinMutation = await withSessionCodexCapacityMutation(
|
|
4716
|
+
db,
|
|
4717
|
+
{
|
|
4718
|
+
workspaceId: input.workspaceId,
|
|
4719
|
+
reason: "codex_stale_policy_pin_cleared",
|
|
4720
|
+
},
|
|
4721
|
+
async (tx) => {
|
|
4722
|
+
const changed = await setSessionCodexPinInTransaction(
|
|
4723
|
+
tx,
|
|
4724
|
+
input.workspaceId,
|
|
4725
|
+
input.sessionId,
|
|
4726
|
+
null,
|
|
4727
|
+
"policy",
|
|
4546
4728
|
{
|
|
4547
|
-
|
|
4548
|
-
|
|
4729
|
+
expected: {
|
|
4730
|
+
pinnedCredentialId: sessionPin,
|
|
4731
|
+
pinSource: sessionPinSource,
|
|
4732
|
+
},
|
|
4549
4733
|
},
|
|
4550
|
-
|
|
4551
|
-
|
|
4552
|
-
|
|
4553
|
-
|
|
4554
|
-
|
|
4555
|
-
|
|
4556
|
-
|
|
4557
|
-
|
|
4558
|
-
turnMetricOutcome = "cancelled";
|
|
4559
|
-
activityStatus = "idle";
|
|
4560
|
-
return claimedResult({ status: "idle", deferredUntilWake: true });
|
|
4734
|
+
);
|
|
4735
|
+
return { result: changed, changed };
|
|
4736
|
+
},
|
|
4737
|
+
);
|
|
4738
|
+
await signalCodexCapacityWakeTargets(
|
|
4739
|
+
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
4740
|
+
pinMutation.wakeTargets,
|
|
4741
|
+
);
|
|
4561
4742
|
}
|
|
4562
|
-
|
|
4563
|
-
|
|
4564
|
-
|
|
4565
|
-
|
|
4566
|
-
|
|
4567
|
-
|
|
4568
|
-
|
|
4569
|
-
|
|
4570
|
-
|
|
4571
|
-
|
|
4572
|
-
|
|
4573
|
-
|
|
4574
|
-
|
|
4575
|
-
|
|
4576
|
-
|
|
4577
|
-
|
|
4578
|
-
|
|
4579
|
-
|
|
4580
|
-
|
|
4743
|
+
if (
|
|
4744
|
+
!settings.codexCredentialLeasingEnabled &&
|
|
4745
|
+
leased.advanceActivePointer &&
|
|
4746
|
+
sessionPin === null &&
|
|
4747
|
+
rotationDecision.kind === "active" &&
|
|
4748
|
+
rotationDecision.moved
|
|
4749
|
+
) {
|
|
4750
|
+
await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
|
|
4751
|
+
}
|
|
4752
|
+
effectiveCodexCredentialId = leased.credentialId;
|
|
4753
|
+
codexLeaseGeneration = leased.generation;
|
|
4754
|
+
codexLeaseConfirmedUntilMs =
|
|
4755
|
+
leased.leasedUntil && leaseAcquisitionStartedAtMs !== null
|
|
4756
|
+
? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS
|
|
4757
|
+
: null;
|
|
4758
|
+
codexLeaseHeld =
|
|
4759
|
+
effectiveCodexCredentialId !== null &&
|
|
4760
|
+
leased.holderId !== null &&
|
|
4761
|
+
leased.generation !== null &&
|
|
4762
|
+
codexLeaseConfirmedUntilMs !== null;
|
|
4763
|
+
if (codexLeaseHeld) startCodexLeaseHeartbeat();
|
|
4764
|
+
|
|
4765
|
+
const actualOutcome = effectiveCodexCredentialId
|
|
4766
|
+
? "selected"
|
|
4767
|
+
: rotationDecision.kind === "allCapped"
|
|
4768
|
+
? "waiting"
|
|
4769
|
+
: "none";
|
|
4770
|
+
const actualReason = effectiveCodexCredentialId
|
|
4771
|
+
? leased.reused
|
|
4772
|
+
? "lease_reused"
|
|
4773
|
+
: sessionPin === effectiveCodexCredentialId
|
|
4774
|
+
? "pin"
|
|
4775
|
+
: rotationDecision.kind === "active" && rotationDecision.moved
|
|
4776
|
+
? "rotation"
|
|
4777
|
+
: "active"
|
|
4778
|
+
: rotationDecision.kind === "allCapped"
|
|
4779
|
+
? "all_capped"
|
|
4780
|
+
: "none";
|
|
4781
|
+
const fencedInFlight = leased.reused;
|
|
4782
|
+
const shadowResult = await publishCodexFleetShadowDecisionV1({
|
|
4783
|
+
enabled: settings.codexFleetPolicyShadowEnabled,
|
|
4784
|
+
decision: {
|
|
4785
|
+
accounts: leased.accounts,
|
|
4786
|
+
actualCredentialId: effectiveCodexCredentialId,
|
|
4787
|
+
actualOutcome,
|
|
4788
|
+
actualReason,
|
|
4789
|
+
affinityCredentialId: fencedInFlight
|
|
4790
|
+
? effectiveCodexCredentialId
|
|
4791
|
+
: (sessionPin ?? sessionCodex?.lastCredentialId ?? null),
|
|
4792
|
+
fencedInFlight,
|
|
4793
|
+
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
4794
|
+
now: new Date(),
|
|
4795
|
+
aliasSeed: randomUUID(),
|
|
4581
4796
|
},
|
|
4797
|
+
publish,
|
|
4582
4798
|
});
|
|
4583
|
-
if (
|
|
4584
|
-
|
|
4585
|
-
|
|
4586
|
-
|
|
4587
|
-
|
|
4588
|
-
|
|
4799
|
+
if (shadowResult.outcome === "published") {
|
|
4800
|
+
const shadowPayload = shadowResult.payload;
|
|
4801
|
+
observability.incrementCounter({
|
|
4802
|
+
name: "opengeni_codex_fleet_shadow_decisions_total",
|
|
4803
|
+
help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
|
|
4804
|
+
labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload),
|
|
4805
|
+
});
|
|
4806
|
+
observability.info("Codex adaptive fleet shadow decision", {
|
|
4807
|
+
workspaceId: input.workspaceId,
|
|
4808
|
+
policyVersion: shadowPayload.replay.policyVersion,
|
|
4809
|
+
inputFingerprint: shadowPayload.replay.inputFingerprint,
|
|
4810
|
+
decisionFingerprint: shadowPayload.replay.decisionFingerprint,
|
|
4811
|
+
actualOutcome: shadowPayload.actual.outcome,
|
|
4812
|
+
shadowOutcome: shadowPayload.replay.decision.outcome,
|
|
4813
|
+
comparison: shadowPayload.comparison,
|
|
4814
|
+
candidateCount: shadowPayload.replay.input.candidates.length,
|
|
4815
|
+
truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
|
|
4816
|
+
payloadBytes: shadowResult.payloadBytes,
|
|
4817
|
+
});
|
|
4818
|
+
} else if (shadowResult.outcome === "failed") {
|
|
4819
|
+
// Shadow observability is explicitly non-authoritative. A malformed
|
|
4820
|
+
// snapshot or event-write fault must never change the authoritative lease,
|
|
4821
|
+
// capacity wait, failover, or the account serving this fenced turn.
|
|
4822
|
+
observability.incrementCounter({
|
|
4823
|
+
name: "opengeni_codex_fleet_shadow_errors_total",
|
|
4824
|
+
help: "Shadow decision build/publication failures.",
|
|
4825
|
+
labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
|
|
4826
|
+
});
|
|
4827
|
+
observability.warn("Codex adaptive fleet shadow decision failed open", {
|
|
4828
|
+
stage: shadowResult.stage,
|
|
4829
|
+
reason: shadowResult.reason,
|
|
4830
|
+
errorClass: "CodexFleetShadowOperationError",
|
|
4831
|
+
errorCode: "codex_fleet_shadow_failed",
|
|
4832
|
+
origin: "worker",
|
|
4833
|
+
payloadBytes: shadowResult.payloadBytes,
|
|
4834
|
+
});
|
|
4835
|
+
}
|
|
4836
|
+
|
|
4837
|
+
const eligibleCount = leased.accounts.filter((account) =>
|
|
4838
|
+
isCodexCredentialEligible(account, new Date()),
|
|
4839
|
+
).length;
|
|
4840
|
+
const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
|
|
4841
|
+
observability.incrementCounter({
|
|
4842
|
+
name: "opengeni_codex_pool_observations_total",
|
|
4843
|
+
help: "Observed eligible Codex pool depth buckets at turn selection.",
|
|
4844
|
+
labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
|
|
4845
|
+
});
|
|
4846
|
+
if (eligibleCount <= 1) {
|
|
4847
|
+
observability.incrementCounter({
|
|
4848
|
+
name: "opengeni_codex_pool_low_total",
|
|
4849
|
+
help: "Alert signal emitted when the eligible Codex pool is zero or one.",
|
|
4850
|
+
labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
|
|
4851
|
+
});
|
|
4852
|
+
observability.warn("Codex eligible credential pool is low", {
|
|
4853
|
+
workspaceId: input.workspaceId,
|
|
4854
|
+
eligibleCount,
|
|
4855
|
+
connectedCount: leased.accounts.length,
|
|
4856
|
+
depth: poolDepth,
|
|
4857
|
+
});
|
|
4858
|
+
}
|
|
4859
|
+
|
|
4860
|
+
if (
|
|
4861
|
+
effectiveCodexCredentialId === null &&
|
|
4862
|
+
leased.accounts.length > 0 &&
|
|
4863
|
+
leased.accounts.every((account) => !account.allocatorEnabled) &&
|
|
4864
|
+
turnId
|
|
4865
|
+
) {
|
|
4866
|
+
if (turn.source === "compaction") {
|
|
4867
|
+
if (
|
|
4868
|
+
!(await settle!({
|
|
4869
|
+
events: [
|
|
4870
|
+
{
|
|
4871
|
+
type: "turn.cancelled",
|
|
4872
|
+
payload: {
|
|
4873
|
+
maintenance: "context_compaction",
|
|
4874
|
+
reason: "codex_allocator_disabled",
|
|
4875
|
+
requestPreserved: true,
|
|
4876
|
+
},
|
|
4877
|
+
},
|
|
4878
|
+
{
|
|
4879
|
+
type: "session.status.changed",
|
|
4880
|
+
payload: { status: "idle" },
|
|
4881
|
+
},
|
|
4882
|
+
],
|
|
4883
|
+
turnStatus: "cancelled",
|
|
4884
|
+
sessionStatus: "idle",
|
|
4885
|
+
activeTurnId: null,
|
|
4886
|
+
}))
|
|
4887
|
+
) {
|
|
4888
|
+
return claimedResult({ status: "cancelled" });
|
|
4889
|
+
}
|
|
4890
|
+
turnMetricOutcome = "cancelled";
|
|
4891
|
+
activityStatus = "idle";
|
|
4892
|
+
return claimedResult({ status: "idle", deferredUntilWake: true });
|
|
4893
|
+
}
|
|
4894
|
+
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
|
|
4895
|
+
() => null,
|
|
4589
4896
|
);
|
|
4590
|
-
|
|
4591
|
-
|
|
4592
|
-
|
|
4593
|
-
|
|
4594
|
-
|
|
4595
|
-
|
|
4596
|
-
|
|
4597
|
-
|
|
4598
|
-
|
|
4897
|
+
const activeGoal = goal?.status === "active" ? goal : null;
|
|
4898
|
+
const armed = await armCodexCapacityWait(db, {
|
|
4899
|
+
accountId: input.accountId,
|
|
4900
|
+
workspaceId: input.workspaceId,
|
|
4901
|
+
sessionId: input.sessionId,
|
|
4902
|
+
turnId,
|
|
4903
|
+
attemptId: input.attemptId,
|
|
4904
|
+
workflowId: input.workflowId,
|
|
4905
|
+
goalId: activeGoal?.id ?? null,
|
|
4906
|
+
goalVersion: activeGoal?.version ?? null,
|
|
4907
|
+
earliestResetAt: null,
|
|
4908
|
+
resetKind: "bounded_refresh",
|
|
4909
|
+
failurePayload: {
|
|
4910
|
+
error: "All connected Codex subscriptions are disabled for new allocations.",
|
|
4911
|
+
code: "codex_allocator_disabled",
|
|
4912
|
+
detail: "waiting for a credential to be re-enabled, reconnected, or added",
|
|
4599
4913
|
},
|
|
4600
4914
|
});
|
|
4915
|
+
if (armed.action === "waiting") {
|
|
4916
|
+
await publishDurableSessionEvents(
|
|
4917
|
+
bus,
|
|
4918
|
+
input.workspaceId,
|
|
4919
|
+
input.sessionId,
|
|
4920
|
+
armed.events,
|
|
4921
|
+
);
|
|
4922
|
+
turnMetricOutcome = "recovering";
|
|
4923
|
+
activityStatus = "waiting_capacity";
|
|
4924
|
+
return claimedResult({
|
|
4925
|
+
status: "waiting_capacity",
|
|
4926
|
+
capacityWait: {
|
|
4927
|
+
waiterId: armed.waiter.id,
|
|
4928
|
+
generation: armed.waiter.generation,
|
|
4929
|
+
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
4930
|
+
wakeRevision: armed.waiter.wakeRevision,
|
|
4931
|
+
},
|
|
4932
|
+
});
|
|
4933
|
+
}
|
|
4934
|
+
if (
|
|
4935
|
+
!(await settle!({
|
|
4936
|
+
events: [
|
|
4937
|
+
{
|
|
4938
|
+
type: "turn.failed",
|
|
4939
|
+
payload: {
|
|
4940
|
+
error: "All connected Codex subscriptions are disabled for new allocations.",
|
|
4941
|
+
code: "codex_allocator_disabled",
|
|
4942
|
+
retryable: false,
|
|
4943
|
+
recovery: "user_message",
|
|
4944
|
+
},
|
|
4945
|
+
},
|
|
4946
|
+
{
|
|
4947
|
+
type: "session.status.changed",
|
|
4948
|
+
payload: { status: "idle" },
|
|
4949
|
+
},
|
|
4950
|
+
],
|
|
4951
|
+
turnStatus: "failed",
|
|
4952
|
+
sessionStatus: "idle",
|
|
4953
|
+
activeTurnId: null,
|
|
4954
|
+
}))
|
|
4955
|
+
) {
|
|
4956
|
+
return claimedResult({ status: "cancelled" });
|
|
4957
|
+
}
|
|
4958
|
+
turnMetricOutcome = "failed";
|
|
4959
|
+
activityStatus = "idle";
|
|
4960
|
+
return claimedResult({ status: "idle" });
|
|
4601
4961
|
}
|
|
4602
|
-
|
|
4603
|
-
|
|
4604
|
-
|
|
4962
|
+
|
|
4963
|
+
if (rotationDecision.kind === "allCapped" && turnId) {
|
|
4964
|
+
if (turn.source === "compaction") {
|
|
4965
|
+
if (
|
|
4966
|
+
!(await settle!({
|
|
4967
|
+
events: [
|
|
4968
|
+
{
|
|
4969
|
+
type: "turn.cancelled",
|
|
4970
|
+
payload: {
|
|
4971
|
+
maintenance: "context_compaction",
|
|
4972
|
+
reason: "codex_capacity_unavailable",
|
|
4973
|
+
requestPreserved: true,
|
|
4974
|
+
},
|
|
4975
|
+
},
|
|
4976
|
+
{
|
|
4977
|
+
type: "session.status.changed",
|
|
4978
|
+
payload: { status: "idle" },
|
|
4979
|
+
},
|
|
4980
|
+
],
|
|
4981
|
+
turnStatus: "cancelled",
|
|
4982
|
+
sessionStatus: "idle",
|
|
4983
|
+
activeTurnId: null,
|
|
4984
|
+
}))
|
|
4985
|
+
) {
|
|
4986
|
+
return claimedResult({ status: "cancelled" });
|
|
4987
|
+
}
|
|
4988
|
+
turnMetricOutcome = "cancelled";
|
|
4989
|
+
activityStatus = "idle";
|
|
4990
|
+
return claimedResult({ status: "idle", deferredUntilWake: true });
|
|
4991
|
+
}
|
|
4992
|
+
// Every eligible account is capped/cooling (and a usage refresh did NOT
|
|
4993
|
+
// surface a reset): idle the turn AT THE BOUNDARY (no wasted model/sandbox
|
|
4994
|
+
// build) until the EARLIEST reset across all accounts — the multi-account
|
|
4995
|
+
// generalization of #143's single-account idle-until-reset. No saveRunState:
|
|
4996
|
+
// no model ran, nothing to freeze.
|
|
4997
|
+
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
|
|
4998
|
+
() => null,
|
|
4999
|
+
);
|
|
5000
|
+
const goalActive = Boolean(goal && goal.status === "active");
|
|
5001
|
+
// BOUNDED + POSITIVE: clamp to [MIN_IDLE_MS, max] so a null/elapsed/unknown
|
|
5002
|
+
// reset can never yield a 0 (which session.ts would treat as "continue now",
|
|
5003
|
+
// re-entering this path in a tight CPU/DB-hammering loop).
|
|
5004
|
+
const resumeMs = computeIdleDelayMs(
|
|
5005
|
+
rotationDecision.earliestResetAt,
|
|
5006
|
+
new Date(),
|
|
5007
|
+
CODEX_USAGE_LIMIT_MAX_RESUME_MS,
|
|
5008
|
+
);
|
|
5009
|
+
const failurePayload = codexUsageLimitFailurePayload(
|
|
5010
|
+
{ resetsInSeconds: Math.ceil(resumeMs / 1000) },
|
|
5011
|
+
"all connected Codex subscriptions are rate-limited",
|
|
5012
|
+
{ allAccounts: true },
|
|
5013
|
+
);
|
|
5014
|
+
const authoritativeResetAt = authoritativeCodexCapacityResetAt(
|
|
5015
|
+
leased.accounts,
|
|
5016
|
+
new Date(),
|
|
5017
|
+
);
|
|
5018
|
+
const armed = await armCodexCapacityWait(db, {
|
|
5019
|
+
accountId: input.accountId,
|
|
5020
|
+
workspaceId: input.workspaceId,
|
|
5021
|
+
sessionId: input.sessionId,
|
|
5022
|
+
turnId,
|
|
5023
|
+
attemptId: input.attemptId,
|
|
5024
|
+
workflowId: input.workflowId,
|
|
5025
|
+
goalId: goalActive && goal ? goal.id : null,
|
|
5026
|
+
goalVersion: goalActive && goal ? goal.version : null,
|
|
5027
|
+
earliestResetAt: authoritativeResetAt,
|
|
5028
|
+
resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
|
|
5029
|
+
failurePayload,
|
|
5030
|
+
});
|
|
5031
|
+
if (armed.action === "waiting") {
|
|
5032
|
+
await publishDurableSessionEvents(
|
|
5033
|
+
bus,
|
|
5034
|
+
input.workspaceId,
|
|
5035
|
+
input.sessionId,
|
|
5036
|
+
armed.events,
|
|
5037
|
+
);
|
|
5038
|
+
turnMetricOutcome = "recovering";
|
|
5039
|
+
activityStatus = "waiting_capacity";
|
|
5040
|
+
return claimedResult({
|
|
5041
|
+
status: "waiting_capacity",
|
|
5042
|
+
capacityWait: {
|
|
5043
|
+
waiterId: armed.waiter.id,
|
|
5044
|
+
generation: armed.waiter.generation,
|
|
5045
|
+
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
5046
|
+
wakeRevision: armed.waiter.wakeRevision,
|
|
5047
|
+
},
|
|
5048
|
+
});
|
|
5049
|
+
}
|
|
5050
|
+
if (
|
|
5051
|
+
!(await settle!({
|
|
5052
|
+
events: [
|
|
5053
|
+
// `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
|
|
5054
|
+
// rotation-wait state as the reactive all-capped path, so it must freeze
|
|
5055
|
+
// autoContinuations identically (evaluateGoalContinuation reads this marker)
|
|
5056
|
+
// — a goal waiting out a long reset must not burn its continuation budget on
|
|
5057
|
+
// the proactive path while the reactive path spares it.
|
|
5058
|
+
{
|
|
5059
|
+
type: "turn.failed",
|
|
5060
|
+
payload: {
|
|
5061
|
+
...failurePayload,
|
|
5062
|
+
recovery: goalActive ? "goal_continuation" : "user_message",
|
|
5063
|
+
rotated: true,
|
|
5064
|
+
},
|
|
5065
|
+
},
|
|
5066
|
+
{
|
|
5067
|
+
type: "session.status.changed",
|
|
5068
|
+
payload: { status: "idle" },
|
|
5069
|
+
},
|
|
5070
|
+
],
|
|
5071
|
+
turnStatus: "failed",
|
|
5072
|
+
sessionStatus: "idle",
|
|
5073
|
+
activeTurnId: null,
|
|
5074
|
+
}))
|
|
5075
|
+
) {
|
|
5076
|
+
return claimedResult({ status: "cancelled" });
|
|
5077
|
+
}
|
|
5078
|
+
turnMetricOutcome = "failed";
|
|
5079
|
+
activityStatus = "idle";
|
|
5080
|
+
// idleUntilReset marks this a MANDATORY hold: session.ts must wait the full
|
|
5081
|
+
// resumeMs even if a future change made it 0 — never a tight re-dispatch.
|
|
5082
|
+
return claimedResult(
|
|
5083
|
+
goalActive
|
|
5084
|
+
? {
|
|
5085
|
+
status: "idle",
|
|
5086
|
+
continueDelayMs: resumeMs,
|
|
5087
|
+
idleUntilReset: true,
|
|
5088
|
+
}
|
|
5089
|
+
: { status: "idle" },
|
|
5090
|
+
);
|
|
5091
|
+
}
|
|
5092
|
+
if (effectiveCodexCredentialId) {
|
|
5093
|
+
const priorAccountId = sessionCodex?.lastCredentialId ?? null;
|
|
5094
|
+
await recordSessionActiveCodexCredential(
|
|
5095
|
+
db,
|
|
5096
|
+
input.workspaceId,
|
|
5097
|
+
input.sessionId,
|
|
5098
|
+
effectiveCodexCredentialId,
|
|
5099
|
+
);
|
|
5100
|
+
if (priorAccountId !== effectiveCodexCredentialId) {
|
|
5101
|
+
const rotated = rotationDecision.kind === "active" && rotationDecision.moved;
|
|
5102
|
+
await publish([
|
|
4605
5103
|
{
|
|
4606
|
-
type: "
|
|
5104
|
+
type: "codex.account.switched",
|
|
4607
5105
|
payload: {
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
|
|
4611
|
-
recovery: "user_message",
|
|
5106
|
+
fromAccountId: priorAccountId,
|
|
5107
|
+
toAccountId: effectiveCodexCredentialId,
|
|
5108
|
+
reason: rotated ? "rotation" : "manual",
|
|
4612
5109
|
},
|
|
4613
5110
|
},
|
|
4614
|
-
|
|
4615
|
-
|
|
4616
|
-
|
|
4617
|
-
|
|
4618
|
-
|
|
4619
|
-
|
|
4620
|
-
|
|
4621
|
-
|
|
5111
|
+
]);
|
|
5112
|
+
}
|
|
5113
|
+
|
|
5114
|
+
const selectionReason = leased.reused
|
|
5115
|
+
? "lease_reused"
|
|
5116
|
+
: sessionPin === effectiveCodexCredentialId
|
|
5117
|
+
? "pin"
|
|
5118
|
+
: rotationDecision.kind === "active" && rotationDecision.moved
|
|
5119
|
+
? "rotation"
|
|
5120
|
+
: "active";
|
|
5121
|
+
observability.incrementCounter({
|
|
5122
|
+
name: "opengeni_codex_credential_selections_total",
|
|
5123
|
+
help: "Codex credential selections by strategy and reason.",
|
|
5124
|
+
labels: {
|
|
5125
|
+
workspace_key: codexWorkspaceKey,
|
|
5126
|
+
strategy: leased.rotationStrategy,
|
|
5127
|
+
reason: selectionReason,
|
|
5128
|
+
},
|
|
5129
|
+
});
|
|
5130
|
+
await publish([
|
|
5131
|
+
{
|
|
5132
|
+
type: "codex.credential.selected",
|
|
5133
|
+
payload: {
|
|
5134
|
+
credentialId: effectiveCodexCredentialId,
|
|
5135
|
+
strategy: leased.rotationStrategy,
|
|
5136
|
+
reason: selectionReason,
|
|
5137
|
+
eligibleCount,
|
|
5138
|
+
connectedCount: leased.accounts.length,
|
|
5139
|
+
reused: leased.reused,
|
|
5140
|
+
},
|
|
5141
|
+
},
|
|
5142
|
+
]);
|
|
4622
5143
|
}
|
|
4623
|
-
|
|
4624
|
-
|
|
4625
|
-
|
|
5144
|
+
} catch (error) {
|
|
5145
|
+
credentialSelectionOutcome = "failed";
|
|
5146
|
+
throw error;
|
|
5147
|
+
} finally {
|
|
5148
|
+
recordTurnStartupPhase(observability, {
|
|
5149
|
+
phase: "credential_selection",
|
|
5150
|
+
provider: "codex-subscription",
|
|
5151
|
+
backend: turn.sandboxBackend,
|
|
5152
|
+
outcome: credentialSelectionOutcome,
|
|
5153
|
+
durationSeconds: (performance.now() - credentialSelectionStartedAt) / 1_000,
|
|
5154
|
+
});
|
|
4626
5155
|
}
|
|
5156
|
+
}
|
|
4627
5157
|
|
|
4628
|
-
|
|
5158
|
+
if (isXaiTurn) {
|
|
5159
|
+
const authoritySnapshot = turn.xaiProviderAccountAuthoritySnapshot;
|
|
5160
|
+
xaiAuthoritySnapshot = authoritySnapshot;
|
|
5161
|
+
const subjectId =
|
|
5162
|
+
authoritySnapshot.scope === "user"
|
|
5163
|
+
? turn.initiatingHumanSubjectId
|
|
5164
|
+
: "worker:xai-workspace";
|
|
5165
|
+
if (!subjectId) {
|
|
5166
|
+
throw new Error("User-scoped SuperGrok work has no frozen initiating human");
|
|
5167
|
+
}
|
|
5168
|
+
const sessionPin = await getXaiSessionAccountPin(db, {
|
|
5169
|
+
workspaceId: input.workspaceId,
|
|
5170
|
+
subjectId,
|
|
5171
|
+
sessionId: input.sessionId,
|
|
5172
|
+
authoritySnapshot,
|
|
5173
|
+
});
|
|
5174
|
+
const leaseStartedAtMs = performance.now();
|
|
5175
|
+
const leased = await acquireXaiCredentialLease(db, {
|
|
5176
|
+
accountId: input.accountId,
|
|
5177
|
+
workspaceId: input.workspaceId,
|
|
5178
|
+
subjectId,
|
|
5179
|
+
sessionId: input.sessionId,
|
|
5180
|
+
turnId: turn.id,
|
|
5181
|
+
holderId: dispatchId,
|
|
5182
|
+
authoritySnapshot,
|
|
5183
|
+
pinnedCredentialId: sessionPin?.pinnedCredentialId ?? null,
|
|
5184
|
+
pinSource:
|
|
5185
|
+
sessionPin?.pinSource === "manual" || sessionPin?.pinSource === "policy"
|
|
5186
|
+
? sessionPin.pinSource
|
|
5187
|
+
: null,
|
|
5188
|
+
});
|
|
5189
|
+
effectiveXaiCredentialId = leased.credentialId;
|
|
5190
|
+
xaiRotationEnabled = leased.rotationEnabled;
|
|
5191
|
+
xaiLeaseSubjectId = subjectId;
|
|
5192
|
+
xaiLeaseHolderId = leased.holderId;
|
|
5193
|
+
xaiLeaseGeneration = leased.generation;
|
|
5194
|
+
xaiLeaseConfirmedUntilMs = leased.leasedUntil
|
|
5195
|
+
? leaseStartedAtMs + XAI_CREDENTIAL_LEASE_TTL_MS
|
|
5196
|
+
: null;
|
|
5197
|
+
xaiLeaseHeld =
|
|
5198
|
+
effectiveXaiCredentialId !== null &&
|
|
5199
|
+
leased.holderId !== null &&
|
|
5200
|
+
leased.generation !== null &&
|
|
5201
|
+
xaiLeaseConfirmedUntilMs !== null;
|
|
5202
|
+
if (!effectiveXaiCredentialId) {
|
|
5203
|
+
const connected = leased.accounts.length;
|
|
5204
|
+
const allocatorEnabled = leased.accounts.filter(
|
|
5205
|
+
(account) => account.allocatorEnabled,
|
|
5206
|
+
).length;
|
|
5207
|
+
if (connected === 0) {
|
|
5208
|
+
throw Object.assign(
|
|
5209
|
+
new Error("No SuperGrok subscription account is connected for this authority scope"),
|
|
5210
|
+
{ code: "xai_not_connected" },
|
|
5211
|
+
);
|
|
5212
|
+
}
|
|
4629
5213
|
if (turn.source === "compaction") {
|
|
4630
5214
|
if (
|
|
4631
5215
|
!(await settle!({
|
|
@@ -4634,7 +5218,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4634
5218
|
type: "turn.cancelled",
|
|
4635
5219
|
payload: {
|
|
4636
5220
|
maintenance: "context_compaction",
|
|
4637
|
-
reason: "
|
|
5221
|
+
reason: "xai_capacity_unavailable",
|
|
4638
5222
|
requestPreserved: true,
|
|
4639
5223
|
},
|
|
4640
5224
|
},
|
|
@@ -4654,44 +5238,38 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4654
5238
|
activityStatus = "idle";
|
|
4655
5239
|
return claimedResult({ status: "idle", deferredUntilWake: true });
|
|
4656
5240
|
}
|
|
4657
|
-
// Every eligible account is capped/cooling (and a usage refresh did NOT
|
|
4658
|
-
// surface a reset): idle the turn AT THE BOUNDARY (no wasted model/sandbox
|
|
4659
|
-
// build) until the EARLIEST reset across all accounts — the multi-account
|
|
4660
|
-
// generalization of #143's single-account idle-until-reset. No saveRunState:
|
|
4661
|
-
// no model ran, nothing to freeze.
|
|
4662
5241
|
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
|
|
4663
5242
|
() => null,
|
|
4664
5243
|
);
|
|
4665
|
-
const
|
|
4666
|
-
|
|
4667
|
-
|
|
4668
|
-
|
|
4669
|
-
|
|
4670
|
-
|
|
4671
|
-
new Date()
|
|
4672
|
-
|
|
4673
|
-
|
|
4674
|
-
|
|
4675
|
-
|
|
4676
|
-
|
|
4677
|
-
|
|
4678
|
-
);
|
|
4679
|
-
const authoritativeResetAt = authoritativeCodexCapacityResetAt(
|
|
4680
|
-
leased.accounts,
|
|
4681
|
-
new Date(),
|
|
4682
|
-
);
|
|
4683
|
-
const armed = await armCodexCapacityWait(db, {
|
|
5244
|
+
const activeGoal = goal?.status === "active" ? goal : null;
|
|
5245
|
+
const now = new Date();
|
|
5246
|
+
const futureResets = leased.accounts
|
|
5247
|
+
.map((account) => account.exhaustedUntil)
|
|
5248
|
+
.filter((date): date is Date => date !== null && date > now);
|
|
5249
|
+
const earliestResetAt = futureResets.length
|
|
5250
|
+
? new Date(Math.min(...futureResets.map((date) => date.getTime())))
|
|
5251
|
+
: null;
|
|
5252
|
+
const error =
|
|
5253
|
+
allocatorEnabled === 0
|
|
5254
|
+
? "All connected SuperGrok subscription accounts are disabled for allocation"
|
|
5255
|
+
: "All connected SuperGrok subscription accounts are temporarily unavailable";
|
|
5256
|
+
const armed = await armXaiCapacityWait(db, {
|
|
4684
5257
|
accountId: input.accountId,
|
|
4685
5258
|
workspaceId: input.workspaceId,
|
|
5259
|
+
subjectId,
|
|
4686
5260
|
sessionId: input.sessionId,
|
|
4687
|
-
turnId,
|
|
5261
|
+
turnId: turn.id,
|
|
4688
5262
|
attemptId: input.attemptId,
|
|
4689
5263
|
workflowId: input.workflowId,
|
|
4690
|
-
|
|
4691
|
-
|
|
4692
|
-
|
|
4693
|
-
|
|
4694
|
-
failurePayload
|
|
5264
|
+
authoritySnapshot,
|
|
5265
|
+
goalId: activeGoal?.id ?? null,
|
|
5266
|
+
goalVersion: activeGoal?.version ?? null,
|
|
5267
|
+
earliestResetAt,
|
|
5268
|
+
failurePayload: {
|
|
5269
|
+
error,
|
|
5270
|
+
code: allocatorEnabled === 0 ? "xai_allocator_disabled" : "xai_capacity_unavailable",
|
|
5271
|
+
detail: "waiting for an eligible account, reconnect, pin change, or quota reset",
|
|
5272
|
+
},
|
|
4695
5273
|
});
|
|
4696
5274
|
if (armed.action === "waiting") {
|
|
4697
5275
|
await publishDurableSessionEvents(
|
|
@@ -4705,6 +5283,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4705
5283
|
return claimedResult({
|
|
4706
5284
|
status: "waiting_capacity",
|
|
4707
5285
|
capacityWait: {
|
|
5286
|
+
provider: "xai",
|
|
4708
5287
|
waiterId: armed.waiter.id,
|
|
4709
5288
|
generation: armed.waiter.generation,
|
|
4710
5289
|
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
@@ -4715,17 +5294,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4715
5294
|
if (
|
|
4716
5295
|
!(await settle!({
|
|
4717
5296
|
events: [
|
|
4718
|
-
// `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
|
|
4719
|
-
// rotation-wait state as the reactive all-capped path, so it must freeze
|
|
4720
|
-
// autoContinuations identically (evaluateGoalContinuation reads this marker)
|
|
4721
|
-
// — a goal waiting out a long reset must not burn its continuation budget on
|
|
4722
|
-
// the proactive path while the reactive path spares it.
|
|
4723
5297
|
{
|
|
4724
5298
|
type: "turn.failed",
|
|
4725
5299
|
payload: {
|
|
4726
|
-
|
|
4727
|
-
|
|
4728
|
-
|
|
5300
|
+
error,
|
|
5301
|
+
code: "xai_capacity_wait_stale",
|
|
5302
|
+
retryable: false,
|
|
5303
|
+
recovery: "user_message",
|
|
4729
5304
|
},
|
|
4730
5305
|
},
|
|
4731
5306
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
@@ -4739,80 +5314,63 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4739
5314
|
}
|
|
4740
5315
|
turnMetricOutcome = "failed";
|
|
4741
5316
|
activityStatus = "idle";
|
|
4742
|
-
|
|
4743
|
-
// resumeMs even if a future change made it 0 — never a tight re-dispatch.
|
|
4744
|
-
return claimedResult(
|
|
4745
|
-
goalActive
|
|
4746
|
-
? {
|
|
4747
|
-
status: "idle",
|
|
4748
|
-
continueDelayMs: resumeMs,
|
|
4749
|
-
idleUntilReset: true,
|
|
4750
|
-
}
|
|
4751
|
-
: { status: "idle" },
|
|
4752
|
-
);
|
|
5317
|
+
return claimedResult({ status: "idle" });
|
|
4753
5318
|
}
|
|
4754
|
-
if (
|
|
4755
|
-
|
|
4756
|
-
|
|
4757
|
-
|
|
4758
|
-
|
|
4759
|
-
|
|
4760
|
-
|
|
4761
|
-
|
|
4762
|
-
|
|
4763
|
-
|
|
4764
|
-
|
|
4765
|
-
|
|
4766
|
-
|
|
4767
|
-
|
|
4768
|
-
|
|
4769
|
-
|
|
4770
|
-
|
|
4771
|
-
|
|
4772
|
-
|
|
4773
|
-
|
|
4774
|
-
|
|
4775
|
-
|
|
4776
|
-
|
|
4777
|
-
|
|
4778
|
-
|
|
4779
|
-
|
|
4780
|
-
|
|
4781
|
-
|
|
4782
|
-
|
|
4783
|
-
|
|
4784
|
-
|
|
4785
|
-
|
|
4786
|
-
|
|
4787
|
-
workspace_key: codexWorkspaceKey,
|
|
4788
|
-
strategy: leased.rotationStrategy,
|
|
4789
|
-
reason: selectionReason,
|
|
4790
|
-
},
|
|
5319
|
+
if (xaiLeaseHeld) startXaiLeaseHeartbeat();
|
|
5320
|
+
if (
|
|
5321
|
+
leased.rotationEnabled &&
|
|
5322
|
+
sessionPin?.pinSource !== "manual" &&
|
|
5323
|
+
(sessionPin?.pinnedCredentialId !== effectiveXaiCredentialId ||
|
|
5324
|
+
sessionPin?.pinSource !== "policy")
|
|
5325
|
+
) {
|
|
5326
|
+
await setXaiSessionAccountPin(db, {
|
|
5327
|
+
accountId: input.accountId,
|
|
5328
|
+
workspaceId: input.workspaceId,
|
|
5329
|
+
subjectId,
|
|
5330
|
+
sessionId: input.sessionId,
|
|
5331
|
+
authoritySnapshot,
|
|
5332
|
+
credentialId: effectiveXaiCredentialId,
|
|
5333
|
+
pinSource: "policy",
|
|
5334
|
+
expectedVersion: sessionPin?.version ?? null,
|
|
5335
|
+
}).catch((error: unknown) => {
|
|
5336
|
+
if (error instanceof Error && error.message === "xAI session pin changed") return;
|
|
5337
|
+
throw error;
|
|
5338
|
+
});
|
|
5339
|
+
} else if (!leased.rotationEnabled && sessionPin?.pinSource === "policy") {
|
|
5340
|
+
await setXaiSessionAccountPin(db, {
|
|
5341
|
+
accountId: input.accountId,
|
|
5342
|
+
workspaceId: input.workspaceId,
|
|
5343
|
+
subjectId,
|
|
5344
|
+
sessionId: input.sessionId,
|
|
5345
|
+
authoritySnapshot,
|
|
5346
|
+
credentialId: null,
|
|
5347
|
+
pinSource: null,
|
|
5348
|
+
expectedVersion: sessionPin.version,
|
|
5349
|
+
}).catch((error: unknown) => {
|
|
5350
|
+
if (error instanceof Error && error.message === "xAI session pin changed") return;
|
|
5351
|
+
throw error;
|
|
4791
5352
|
});
|
|
4792
|
-
await publish([
|
|
4793
|
-
{
|
|
4794
|
-
type: "codex.credential.selected",
|
|
4795
|
-
payload: {
|
|
4796
|
-
credentialId: effectiveCodexCredentialId,
|
|
4797
|
-
strategy: leased.rotationStrategy,
|
|
4798
|
-
reason: selectionReason,
|
|
4799
|
-
eligibleCount,
|
|
4800
|
-
connectedCount: leased.accounts.length,
|
|
4801
|
-
reused: leased.reused,
|
|
4802
|
-
},
|
|
4803
|
-
},
|
|
4804
|
-
]);
|
|
4805
5353
|
}
|
|
5354
|
+
await recordXaiSessionLastAccount(db, {
|
|
5355
|
+
accountId: input.accountId,
|
|
5356
|
+
workspaceId: input.workspaceId,
|
|
5357
|
+
subjectId,
|
|
5358
|
+
sessionId: input.sessionId,
|
|
5359
|
+
authoritySnapshot,
|
|
5360
|
+
credentialId: effectiveXaiCredentialId,
|
|
5361
|
+
});
|
|
4806
5362
|
}
|
|
4807
5363
|
|
|
5364
|
+
const runtimePreparationStartedAt = performance.now();
|
|
5365
|
+
|
|
4808
5366
|
// Pack-scoped runtime: enabled packs may declare the sandbox image this
|
|
4809
5367
|
// workspace's sessions run in and skills for the sandbox skill index.
|
|
4810
5368
|
// Resolved after turn.started so a composition conflict (two enabled
|
|
4811
5369
|
// packs declaring images) fails the turn with its plain error instead
|
|
4812
5370
|
// of failing the activity opaquely.
|
|
4813
|
-
const [packRuntime,
|
|
5371
|
+
const [packRuntime, installedSkillRuntime] = await Promise.all([
|
|
4814
5372
|
resolveWorkspacePackRuntime(db, input.workspaceId),
|
|
4815
|
-
|
|
5373
|
+
resolveWorkspaceInstalledSkillRuntime(db, input.workspaceId),
|
|
4816
5374
|
]);
|
|
4817
5375
|
// RIG BINDING (M3): load the session's FROZEN rig version (resolved+frozen
|
|
4818
5376
|
// at create). Everything rig-derived below (image precedence, env default
|
|
@@ -4860,12 +5418,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4860
5418
|
if (!workspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
|
|
4861
5419
|
const workspaceAgentInstructions = workspace.agentInstructions;
|
|
4862
5420
|
const agentHumanInputEnabled = resolveWorkspaceAgentHumanInputEnabled(workspace.settings);
|
|
5421
|
+
const memoryPromptMode = resolveWorkspaceMemoryPromptMode(workspace.settings);
|
|
4863
5422
|
assertWorkspaceHumanInputAllowed(agentHumanInputEnabled, "resume", humanInputResume !== null);
|
|
4864
|
-
const
|
|
4865
|
-
|
|
4866
|
-
|
|
4867
|
-
|
|
4868
|
-
|
|
5423
|
+
const companyProfileIncluded =
|
|
5424
|
+
memoryPromptMode === "legacy_standing" || session.nestedAgentDepth === 0;
|
|
5425
|
+
const workspaceGovernance = renderWorkspaceGovernanceContext(
|
|
5426
|
+
{
|
|
5427
|
+
companyProfile: companyProfileSnapshot,
|
|
5428
|
+
instructionPolicy: instructionPolicySnapshot,
|
|
5429
|
+
preferences: preferenceSnapshot,
|
|
5430
|
+
},
|
|
5431
|
+
{
|
|
5432
|
+
includeCompanyProfile: companyProfileIncluded,
|
|
5433
|
+
},
|
|
5434
|
+
);
|
|
4869
5435
|
const structuredWorkspacePolicyActive =
|
|
4870
5436
|
hasActiveWorkspaceInstructionPolicy(instructionPolicySnapshot);
|
|
4871
5437
|
const workspaceMemory = await resolveWorkspaceMemoryBlock(db, input.workspaceId);
|
|
@@ -5034,6 +5600,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5034
5600
|
// call on the same codex client) and the main run; otherwise the summarizer
|
|
5035
5601
|
// would hit the codex backend unauthenticated.
|
|
5036
5602
|
let codexModelRequestSequence = 0;
|
|
5603
|
+
let firstModelRequestPreparationStartedAt: number | null = null;
|
|
5604
|
+
let firstModelRequestPreparationRecorded = false;
|
|
5605
|
+
let firstModelRequestAuditRecorded = false;
|
|
5606
|
+
let firstModelRequestCheckpointAt: number | null = null;
|
|
5607
|
+
const firstModelRequestCheckpoints = new Set<string>();
|
|
5037
5608
|
const codexContext: CodexRequestContext | null =
|
|
5038
5609
|
resolvedModel?.provider.kind === "codex-subscription"
|
|
5039
5610
|
? ((): CodexRequestContext => {
|
|
@@ -5064,33 +5635,149 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5064
5635
|
onUsageHeaders: (snapshot) => {
|
|
5065
5636
|
latestCodexUsage = snapshot;
|
|
5066
5637
|
}, // latest wins; flushed once in finally
|
|
5638
|
+
onRequestPreparationDiagnostic: (phase) => {
|
|
5639
|
+
if (
|
|
5640
|
+
firstModelRequestCheckpointAt === null ||
|
|
5641
|
+
firstModelRequestCheckpoints.has(phase)
|
|
5642
|
+
) {
|
|
5643
|
+
return;
|
|
5644
|
+
}
|
|
5645
|
+
const now = performance.now();
|
|
5646
|
+
const metricPhase =
|
|
5647
|
+
phase === "transport_entry"
|
|
5648
|
+
? "model_sdk_serialization"
|
|
5649
|
+
: phase === "credential_ready"
|
|
5650
|
+
? "model_credential_resolution"
|
|
5651
|
+
: "model_wire_normalization";
|
|
5652
|
+
firstModelRequestCheckpoints.add(phase);
|
|
5653
|
+
recordTurnStartupPhase(observability, {
|
|
5654
|
+
phase: metricPhase,
|
|
5655
|
+
provider: turnExecutionPolicy.providerId,
|
|
5656
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
5657
|
+
outcome: "completed",
|
|
5658
|
+
durationSeconds: (now - firstModelRequestCheckpointAt) / 1_000,
|
|
5659
|
+
count: turnTools.length,
|
|
5660
|
+
});
|
|
5661
|
+
firstModelRequestCheckpointAt = now;
|
|
5662
|
+
},
|
|
5067
5663
|
onRequestOpaqueArtifacts: ({ fingerprints }) => {
|
|
5068
5664
|
lastCodexRequestOpaqueArtifacts = fingerprints;
|
|
5069
5665
|
},
|
|
5666
|
+
onModelRequestDiagnostic: (event) => {
|
|
5667
|
+
if (
|
|
5668
|
+
event.phase === "started" &&
|
|
5669
|
+
firstModelRequestPreparationStartedAt !== null &&
|
|
5670
|
+
!firstModelRequestPreparationRecorded
|
|
5671
|
+
) {
|
|
5672
|
+
firstModelRequestPreparationRecorded = true;
|
|
5673
|
+
recordTurnStartupPhase(observability, {
|
|
5674
|
+
phase: "model_request_preparation",
|
|
5675
|
+
provider: turnExecutionPolicy.providerId,
|
|
5676
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
5677
|
+
outcome: "completed",
|
|
5678
|
+
durationSeconds:
|
|
5679
|
+
(performance.now() - firstModelRequestPreparationStartedAt) / 1_000,
|
|
5680
|
+
count: turnTools.length,
|
|
5681
|
+
});
|
|
5682
|
+
}
|
|
5683
|
+
const phase =
|
|
5684
|
+
event.phase === "headers"
|
|
5685
|
+
? "headers"
|
|
5686
|
+
: event.phase === "first_byte"
|
|
5687
|
+
? "first_byte"
|
|
5688
|
+
: event.phase === "completed" ||
|
|
5689
|
+
event.phase === "failed" ||
|
|
5690
|
+
event.phase === "timed_out"
|
|
5691
|
+
? "terminal"
|
|
5692
|
+
: null;
|
|
5693
|
+
if (!phase) return;
|
|
5694
|
+
recordModelRequestPhase(observability, {
|
|
5695
|
+
provider: "codex-subscription",
|
|
5696
|
+
phase,
|
|
5697
|
+
...(event.phase === "completed"
|
|
5698
|
+
? { outcome: "completed" as const }
|
|
5699
|
+
: event.phase === "failed"
|
|
5700
|
+
? { outcome: "failed" as const }
|
|
5701
|
+
: event.phase === "timed_out"
|
|
5702
|
+
? { outcome: "timed_out" as const }
|
|
5703
|
+
: {}),
|
|
5704
|
+
durationSeconds: event.durationMs / 1000,
|
|
5705
|
+
});
|
|
5706
|
+
},
|
|
5070
5707
|
nextRequestId: () => `${dispatchId}:${++codexModelRequestSequence}`,
|
|
5071
5708
|
onModelRequestEvent: async (event) => {
|
|
5072
5709
|
if (!publish || !turnId) {
|
|
5073
5710
|
throw new Error("Codex model request started before the turn event producer");
|
|
5074
5711
|
}
|
|
5075
|
-
|
|
5076
|
-
|
|
5077
|
-
|
|
5078
|
-
|
|
5079
|
-
|
|
5080
|
-
|
|
5081
|
-
|
|
5082
|
-
|
|
5083
|
-
|
|
5084
|
-
|
|
5712
|
+
const shouldRecordStartedAudit =
|
|
5713
|
+
event.phase === "started" && !firstModelRequestAuditRecorded;
|
|
5714
|
+
if (shouldRecordStartedAudit) {
|
|
5715
|
+
firstModelRequestAuditRecorded = true;
|
|
5716
|
+
}
|
|
5717
|
+
const auditStartedAt = shouldRecordStartedAudit ? performance.now() : null;
|
|
5718
|
+
let auditOutcome: "completed" | "failed" = "completed";
|
|
5719
|
+
try {
|
|
5720
|
+
await publish([
|
|
5721
|
+
{
|
|
5722
|
+
type: "agent.model.request",
|
|
5723
|
+
payload: {
|
|
5724
|
+
...event,
|
|
5725
|
+
provider: "codex-subscription",
|
|
5726
|
+
turnId,
|
|
5727
|
+
attemptId: input.attemptId,
|
|
5728
|
+
dispatchId,
|
|
5729
|
+
executionGeneration,
|
|
5730
|
+
},
|
|
5085
5731
|
},
|
|
5086
|
-
|
|
5087
|
-
|
|
5732
|
+
]);
|
|
5733
|
+
} catch (error) {
|
|
5734
|
+
auditOutcome = "failed";
|
|
5735
|
+
throw error;
|
|
5736
|
+
} finally {
|
|
5737
|
+
if (auditStartedAt !== null) {
|
|
5738
|
+
recordTurnStartupPhase(observability, {
|
|
5739
|
+
phase: "model_request_audit",
|
|
5740
|
+
provider: turnExecutionPolicy.providerId,
|
|
5741
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
5742
|
+
outcome: auditOutcome,
|
|
5743
|
+
durationSeconds: (performance.now() - auditStartedAt) / 1_000,
|
|
5744
|
+
count: turnTools.length,
|
|
5745
|
+
});
|
|
5746
|
+
}
|
|
5747
|
+
}
|
|
5088
5748
|
},
|
|
5089
5749
|
};
|
|
5090
5750
|
})()
|
|
5091
5751
|
: null;
|
|
5752
|
+
if (resolvedModel?.provider.kind === "xai-subscription") {
|
|
5753
|
+
if (!effectiveXaiCredentialId || !xaiLeaseSubjectId) {
|
|
5754
|
+
throw new Error("SuperGrok subscription execution has no leased credential");
|
|
5755
|
+
}
|
|
5756
|
+
let xaiModelRequestSequence = 0;
|
|
5757
|
+
const authorization = await buildXaiTurnRequestAuthorization({
|
|
5758
|
+
db,
|
|
5759
|
+
settings: runSettings,
|
|
5760
|
+
accountId: input.accountId,
|
|
5761
|
+
workspaceId: input.workspaceId,
|
|
5762
|
+
subjectId: xaiLeaseSubjectId,
|
|
5763
|
+
sessionId: input.sessionId,
|
|
5764
|
+
turnId: turn.id,
|
|
5765
|
+
credentialId: effectiveXaiCredentialId,
|
|
5766
|
+
authoritySnapshot: turn.xaiProviderAccountAuthoritySnapshot,
|
|
5767
|
+
hostedSearch: {
|
|
5768
|
+
webSearch: runSettings.webSearchEnabled,
|
|
5769
|
+
xSearch: runSettings.webSearchEnabled,
|
|
5770
|
+
},
|
|
5771
|
+
nextRequestId: () => `${dispatchId}:xai:${++xaiModelRequestSequence}`,
|
|
5772
|
+
});
|
|
5773
|
+
xaiRequestContext = authorization.context;
|
|
5774
|
+
}
|
|
5092
5775
|
const withCodex = <T>(fn: () => Promise<T>): Promise<T> =>
|
|
5093
5776
|
codexContext ? codexRequestStorage.run(codexContext, fn) : fn();
|
|
5777
|
+
const withProviderRequestContext = <T>(fn: () => Promise<T>): Promise<T> =>
|
|
5778
|
+
xaiRequestContext
|
|
5779
|
+
? xaiSubscriptionRequestStorage.run(xaiRequestContext, fn)
|
|
5780
|
+
: withCodex(fn);
|
|
5094
5781
|
const withCodexRemoteCompaction = <T>(fn: () => Promise<T>): Promise<T> =>
|
|
5095
5782
|
withCodex(() =>
|
|
5096
5783
|
withCodexRequestOverrides(
|
|
@@ -5134,15 +5821,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5134
5821
|
servingCredentialId: effectiveCodexCredentialId,
|
|
5135
5822
|
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
5136
5823
|
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
5137
|
-
renewLease: () =>
|
|
5138
|
-
leaseLost:
|
|
5139
|
-
leaseLostMessage: "
|
|
5824
|
+
renewLease: () => renewServingCredentialLease("model_usage"),
|
|
5825
|
+
leaseLost: servingCredentialLeaseLost,
|
|
5826
|
+
leaseLostMessage: "Provider credential lease expired during context compaction",
|
|
5140
5827
|
});
|
|
5141
5828
|
};
|
|
5142
5829
|
const compactionSummarizerFor = (systemInstructions?: string) =>
|
|
5143
5830
|
resolvedModel
|
|
5144
5831
|
? (s: Settings, m: Array<Record<string, unknown>>) =>
|
|
5145
|
-
|
|
5832
|
+
withProviderRequestContext(() =>
|
|
5146
5833
|
summarizeContextForCompaction(s, m, {
|
|
5147
5834
|
client: resolvedModel.client,
|
|
5148
5835
|
provider: resolvedModel.provider,
|
|
@@ -5256,20 +5943,23 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5256
5943
|
};
|
|
5257
5944
|
const compactionInstructions = appendWorkspaceMemory(
|
|
5258
5945
|
appendPersistentSessionSettings(
|
|
5259
|
-
|
|
5260
|
-
|
|
5261
|
-
|
|
5262
|
-
|
|
5263
|
-
|
|
5264
|
-
|
|
5265
|
-
|
|
5266
|
-
|
|
5267
|
-
|
|
5268
|
-
|
|
5946
|
+
appendSessionGoal(
|
|
5947
|
+
appendSessionInstructions(
|
|
5948
|
+
appendWorkspaceGovernance(
|
|
5949
|
+
composeAgentInstructions(
|
|
5950
|
+
structuredWorkspacePolicyActive
|
|
5951
|
+
? modelRunSettings.agentInstructionsTemplate
|
|
5952
|
+
: (workspaceAgentInstructions ?? modelRunSettings.agentInstructionsTemplate),
|
|
5953
|
+
undefined,
|
|
5954
|
+
rigVersion && rigName
|
|
5955
|
+
? { name: rigName, version: rigVersion.version }
|
|
5956
|
+
: undefined,
|
|
5957
|
+
),
|
|
5958
|
+
workspaceGovernance ?? undefined,
|
|
5269
5959
|
),
|
|
5270
|
-
|
|
5960
|
+
session.instructions ?? undefined,
|
|
5271
5961
|
),
|
|
5272
|
-
|
|
5962
|
+
turn.goalSnapshot,
|
|
5273
5963
|
),
|
|
5274
5964
|
persistentSessionSettings,
|
|
5275
5965
|
),
|
|
@@ -5525,6 +6215,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5525
6215
|
activeSandboxBackend === "selfhosted" &&
|
|
5526
6216
|
Boolean(activeSandboxPointer?.activeSandboxId) &&
|
|
5527
6217
|
Boolean(activeSandboxRecord?.enrollmentId);
|
|
6218
|
+
// `none` describes the durable home, not an explicit per-turn route. Give
|
|
6219
|
+
// the runtime the effective backend so it builds a SandboxAgent for the
|
|
6220
|
+
// attached Connected Machine without mutating the session or turn record.
|
|
6221
|
+
if (machinePrimary && modelRunSettings.sandboxBackend === "none") {
|
|
6222
|
+
modelRunSettings = {
|
|
6223
|
+
...modelRunSettings,
|
|
6224
|
+
sandboxBackend: "selfhosted",
|
|
6225
|
+
};
|
|
6226
|
+
}
|
|
5528
6227
|
// The backend that can actually create a sandbox for this turn. In the
|
|
5529
6228
|
// common path this is runSettings.sandboxBackend. A selfhosted home turn
|
|
5530
6229
|
// that is NOT machine-primary falls back to the deployment cloud backend
|
|
@@ -5586,17 +6285,27 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5586
6285
|
cancellationSignal,
|
|
5587
6286
|
undefined,
|
|
5588
6287
|
);
|
|
5589
|
-
const
|
|
5590
|
-
lazyProvisionEnabled(settings)
|
|
5591
|
-
|
|
5592
|
-
runSettings.sandboxBackend
|
|
6288
|
+
const establishDecision = sandboxEstablishPolicyDecision({
|
|
6289
|
+
lazyEnabled: lazyProvisionEnabled(settings),
|
|
6290
|
+
machinePrimary,
|
|
6291
|
+
sandboxBackend: runSettings.sandboxBackend,
|
|
5593
6292
|
// Resolved run credentials must be written to one exact leased sandbox
|
|
5594
6293
|
// before agent execution. A warm active pointer can bypass the lazy
|
|
5595
6294
|
// provisioner entirely, which previously skipped materialization.
|
|
5596
|
-
|
|
5597
|
-
requiredGeneratedVideoFiles.length
|
|
5598
|
-
|
|
5599
|
-
|
|
6295
|
+
hasInitialRunCredentialMaterial: initialRunCredentialMaterial !== null,
|
|
6296
|
+
generatedVideoFileCount: requiredGeneratedVideoFiles.length,
|
|
6297
|
+
hasSignedFileResources:
|
|
6298
|
+
requiresSignedFileResourceDownloads(
|
|
6299
|
+
runSettings,
|
|
6300
|
+
activeSandboxBackend ?? groupBoxBackend,
|
|
6301
|
+
) && turnResources.some((resource) => resource.kind === "file"),
|
|
6302
|
+
});
|
|
6303
|
+
const establishPolicy = establishDecision.policy;
|
|
6304
|
+
recordTurnSandboxEstablishPolicy(observability, {
|
|
6305
|
+
policy: establishDecision.policy,
|
|
6306
|
+
reason: establishDecision.reason,
|
|
6307
|
+
backend: machinePrimary ? "selfhosted" : groupBoxBackend,
|
|
6308
|
+
});
|
|
5600
6309
|
// Computed exactly ONCE per turn and reused for BOTH the box manifest
|
|
5601
6310
|
// (resumeBoxForTurn -> establishSandboxSessionFromEnvelope, below) AND the
|
|
5602
6311
|
// agent (runtime.buildAgent, below). sandboxEnvironmentForRun mints a FRESH
|
|
@@ -5676,7 +6385,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5676
6385
|
sandboxWorkspaceEnvironmentValues,
|
|
5677
6386
|
{
|
|
5678
6387
|
skipGitHubToken: activeSandboxBackend === "selfhosted",
|
|
5679
|
-
|
|
6388
|
+
codemodeDelivery:
|
|
6389
|
+
activeSandboxBackend === "selfhosted" ? "transient_exec" : "managed_file",
|
|
5680
6390
|
deferGitHubToken:
|
|
5681
6391
|
activeSandboxBackend !== "selfhosted" && establishPolicy === "on-demand",
|
|
5682
6392
|
scope: connectionScope,
|
|
@@ -5696,6 +6406,22 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5696
6406
|
? { ...baseSandboxEnvironment, ...sandboxArtifactRuntime.environment }
|
|
5697
6407
|
: baseSandboxEnvironment;
|
|
5698
6408
|
|
|
6409
|
+
// One mutable in-memory bearer cell serves every Connected Machine route
|
|
6410
|
+
// in this attempt. SelfhostedSession snapshots it into each exact exec
|
|
6411
|
+
// request; it never enters the manifest, argv, filesystem, RunState, or
|
|
6412
|
+
// serialized session state. Managed renewal updates the same cell so a
|
|
6413
|
+
// later mid-turn swap sees the fresh bearer too.
|
|
6414
|
+
const codemodeTokenState = sandboxCodemodeToken ? { token: sandboxCodemodeToken } : undefined;
|
|
6415
|
+
const transientCodemodeEnvironment = codemodeTokenState
|
|
6416
|
+
? (): Readonly<Record<string, string>> => ({
|
|
6417
|
+
OPENGENI_CODEMODE_URL: codemodeWorkspaceUrl(runSettings, input.workspaceId),
|
|
6418
|
+
OPENGENI_CODEMODE_TOKEN: codemodeTokenState.token,
|
|
6419
|
+
...(runSettings.ogtoolPackageSpec
|
|
6420
|
+
? { OPENGENI_OGTOOL_PACKAGE_SPEC: runSettings.ogtoolPackageSpec }
|
|
6421
|
+
: {}),
|
|
6422
|
+
})
|
|
6423
|
+
: undefined;
|
|
6424
|
+
|
|
5699
6425
|
const sandboxCodemodeTokenFile = sandboxCodemodeToken
|
|
5700
6426
|
? codemodeTokenFileFromEnvironment(sandboxEnvironment, input.sessionId)
|
|
5701
6427
|
: undefined;
|
|
@@ -5811,11 +6537,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5811
6537
|
};
|
|
5812
6538
|
|
|
5813
6539
|
const attachCodemodeTokenRenewal = async (
|
|
5814
|
-
tokenSession
|
|
6540
|
+
tokenSession?: CodemodeTokenWriterSession,
|
|
5815
6541
|
initialExpiresAt = sandboxCodemodeTokenExpiresAt,
|
|
5816
6542
|
initialSandbox?: ResumedTurnSandbox,
|
|
5817
6543
|
): Promise<void> => {
|
|
5818
|
-
if (!
|
|
6544
|
+
if (!codemodeTokenState || !initialExpiresAt) return;
|
|
5819
6545
|
const previous = codemodeTokenRenewal;
|
|
5820
6546
|
codemodeTokenRenewal = null;
|
|
5821
6547
|
await previous?.stop();
|
|
@@ -5827,37 +6553,38 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5827
6553
|
connectionScope,
|
|
5828
6554
|
codemodeAuthority,
|
|
5829
6555
|
);
|
|
5830
|
-
if (material) {
|
|
5831
|
-
}
|
|
5832
6556
|
return material;
|
|
5833
6557
|
};
|
|
5834
6558
|
const write = async (material: NonNullable<Awaited<ReturnType<typeof mint>>>) => {
|
|
5835
|
-
|
|
5836
|
-
|
|
5837
|
-
|
|
5838
|
-
|
|
6559
|
+
if (tokenSession) {
|
|
6560
|
+
const runAs = sandboxRunAs(runSettings);
|
|
6561
|
+
const targetSandbox = resolvedSandbox ?? initialSandbox;
|
|
6562
|
+
if (!targetSandbox) {
|
|
6563
|
+
throw new Error("Codemode token renewal has no exact sandbox lease target");
|
|
6564
|
+
}
|
|
6565
|
+
await runWorkspaceMutationForSandbox(
|
|
6566
|
+
targetSandbox,
|
|
6567
|
+
"codemodeTokenRenewal",
|
|
6568
|
+
async () =>
|
|
6569
|
+
await refreshCodemodeTokenFile(tokenSession, material.token, {
|
|
6570
|
+
...(runAs ? { runAs } : {}),
|
|
6571
|
+
...(sandboxCodemodeTokenFile
|
|
6572
|
+
? {
|
|
6573
|
+
tokenFile: sandboxCodemodeTokenFile,
|
|
6574
|
+
legacyTokenFile: sandboxEnvironment.OPENGENI_CODEMODE_TOKEN_FILE!,
|
|
6575
|
+
}
|
|
6576
|
+
: {}),
|
|
6577
|
+
...(toolCancellationFenceRef.current
|
|
6578
|
+
? {
|
|
6579
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
6580
|
+
toolCancellationFenceRef.current,
|
|
6581
|
+
),
|
|
6582
|
+
}
|
|
6583
|
+
: {}),
|
|
6584
|
+
}),
|
|
6585
|
+
);
|
|
5839
6586
|
}
|
|
5840
|
-
|
|
5841
|
-
targetSandbox,
|
|
5842
|
-
"codemodeTokenRenewal",
|
|
5843
|
-
async () =>
|
|
5844
|
-
await refreshCodemodeTokenFile(tokenSession, material.token, {
|
|
5845
|
-
...(runAs ? { runAs } : {}),
|
|
5846
|
-
...(sandboxCodemodeTokenFile
|
|
5847
|
-
? {
|
|
5848
|
-
tokenFile: sandboxCodemodeTokenFile,
|
|
5849
|
-
legacyTokenFile: sandboxEnvironment.OPENGENI_CODEMODE_TOKEN_FILE!,
|
|
5850
|
-
}
|
|
5851
|
-
: {}),
|
|
5852
|
-
...(toolCancellationFenceRef.current
|
|
5853
|
-
? {
|
|
5854
|
-
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
5855
|
-
toolCancellationFenceRef.current,
|
|
5856
|
-
),
|
|
5857
|
-
}
|
|
5858
|
-
: {}),
|
|
5859
|
-
}),
|
|
5860
|
-
);
|
|
6587
|
+
codemodeTokenState.token = material.token;
|
|
5861
6588
|
};
|
|
5862
6589
|
let renewalExpiresAt = initialExpiresAt;
|
|
5863
6590
|
if (renewalExpiresAt.getTime() <= Date.now() + CODEMODE_TOKEN_EXPIRY_LEAD_MS) {
|
|
@@ -5900,6 +6627,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5900
6627
|
codemodeTokenRenewal = controller;
|
|
5901
6628
|
};
|
|
5902
6629
|
|
|
6630
|
+
// A Connected Machine needs renewal, but renewal is purely worker-local:
|
|
6631
|
+
// starting this loop performs no control-plane or machine operation.
|
|
6632
|
+
if (activeSandboxBackend === "selfhosted") {
|
|
6633
|
+
await attachCodemodeTokenRenewal();
|
|
6634
|
+
}
|
|
6635
|
+
|
|
5903
6636
|
const attachRunCredentialRenewal = async (
|
|
5904
6637
|
credentialSession: RunCredentialCommandSession,
|
|
5905
6638
|
initialSandbox?: ResumedTurnSandbox,
|
|
@@ -6045,234 +6778,289 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6045
6778
|
// applies the agent's manifest to this provided session and throws on ANY
|
|
6046
6779
|
// variableSet delta (validateNoEnvironmentDelta). Passing sandboxEnvironment
|
|
6047
6780
|
// here makes current==target so the delta is empty.
|
|
6048
|
-
|
|
6049
|
-
|
|
6781
|
+
recordTurnStartupPhase(observability, {
|
|
6782
|
+
phase: "runtime_preparation",
|
|
6783
|
+
provider: turnExecutionPolicy.providerId,
|
|
6784
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
6785
|
+
outcome: "completed",
|
|
6786
|
+
durationSeconds: (performance.now() - runtimePreparationStartedAt) / 1_000,
|
|
6787
|
+
});
|
|
6788
|
+
if (
|
|
6789
|
+
shouldEstablishSandboxForTurn(
|
|
6790
|
+
settings.sandboxOwnershipEnabled,
|
|
6791
|
+
turn.sandboxBackend,
|
|
6050
6792
|
machinePrimary,
|
|
6051
|
-
|
|
6052
|
-
|
|
6053
|
-
);
|
|
6054
|
-
|
|
6055
|
-
|
|
6056
|
-
|
|
6057
|
-
|
|
6058
|
-
|
|
6059
|
-
|
|
6060
|
-
// group box (sandbox_swap 'session'/'default'/groupId clears the pointer) or
|
|
6061
|
-
// selfhosted routing is flag-OFF (the pointer is ignored) — the else-branch
|
|
6062
|
-
// must resume a REAL cloud group box, not a "selfhosted" one: the registry
|
|
6063
|
-
// SelfhostedSandboxClient has no bound agentId and throws. Fall the group-box
|
|
6064
|
-
// backend back to the deployment default cloud backend so swap-away / flag-off
|
|
6065
|
-
// degrade to a genuine cloud box exactly like today (home=modal did).
|
|
6066
|
-
if (machinePrimary) {
|
|
6067
|
-
// STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
|
|
6068
|
-
// establish OR lease the managed home box. The active-sandbox pointer is
|
|
6069
|
-
// the machine route's own epoch fence; the managed-home lease separately
|
|
6070
|
-
// owns cloud recovery, snapshots, billing, and reaping. Mixing those two
|
|
6071
|
-
// ownership domains lets an unrelated degraded cloud archive block a
|
|
6072
|
-
// healthy explicitly selected machine.
|
|
6073
|
-
// Whether the machine's latest Hello advertised the op-stream engine
|
|
6074
|
-
// (refreshed on every connect). Read only when the server flag is on —
|
|
6075
|
-
// one indexed lookup, and the flag off keeps this path byte-identical.
|
|
6076
|
-
const machineOpStream =
|
|
6077
|
-
settings.agentOpStreamEnabled === true
|
|
6078
|
-
? (await getEnrollment(db, input.workspaceId, activeSandboxRecord!.enrollmentId!))
|
|
6079
|
-
?.opStream === true
|
|
6080
|
-
: false;
|
|
6081
|
-
const established = await establishSelfhostedTurnSession(
|
|
6082
|
-
{
|
|
6083
|
-
db,
|
|
6084
|
-
settings,
|
|
6085
|
-
bus,
|
|
6086
|
-
onOp: machineOpObserver.observer,
|
|
6087
|
-
onSandboxOperation: sandboxOperationObserver,
|
|
6088
|
-
opJournal,
|
|
6089
|
-
},
|
|
6090
|
-
{
|
|
6091
|
-
workspaceId: input.workspaceId,
|
|
6092
|
-
agentId: activeSandboxRecord!.enrollmentId!,
|
|
6093
|
-
opStream: machineOpStream,
|
|
6094
|
-
epoch: activeSandboxPointer!.activeEpoch,
|
|
6095
|
-
environment: sandboxEnvironment,
|
|
6096
|
-
workingDir: activeSandboxPointer!.workingDir,
|
|
6097
|
-
},
|
|
6793
|
+
)
|
|
6794
|
+
) {
|
|
6795
|
+
const sandboxEstablishStartedAt = performance.now();
|
|
6796
|
+
let sandboxEstablishOutcome: "completed" | "failed" = "completed";
|
|
6797
|
+
try {
|
|
6798
|
+
const managedOwnership = managedSandboxOwnershipForTurn(
|
|
6799
|
+
machinePrimary,
|
|
6800
|
+
input.attemptId,
|
|
6801
|
+
session.sandboxGroupId,
|
|
6098
6802
|
);
|
|
6099
|
-
|
|
6100
|
-
|
|
6101
|
-
|
|
6102
|
-
|
|
6103
|
-
|
|
6104
|
-
|
|
6105
|
-
|
|
6106
|
-
|
|
6107
|
-
|
|
6108
|
-
|
|
6109
|
-
|
|
6110
|
-
|
|
6803
|
+
sandboxHolderId = managedOwnership?.holderId ?? null;
|
|
6804
|
+
sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
|
|
6805
|
+
// STAGE D honest-label guard: a machine-home session carries
|
|
6806
|
+
// turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
|
|
6807
|
+
// when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
|
|
6808
|
+
// + enrollmentId). When it is NOT primary — the agent swapped back to the
|
|
6809
|
+
// group box (sandbox_swap 'session'/'default'/groupId clears the pointer) or
|
|
6810
|
+
// selfhosted routing is flag-OFF (the pointer is ignored) — the else-branch
|
|
6811
|
+
// must resume a REAL cloud group box, not a "selfhosted" one: the registry
|
|
6812
|
+
// SelfhostedSandboxClient has no bound agentId and throws. Fall the group-box
|
|
6813
|
+
// backend back to the deployment default cloud backend so swap-away / flag-off
|
|
6814
|
+
// degrade to a genuine cloud box exactly like today (home=modal did).
|
|
6815
|
+
if (machinePrimary) {
|
|
6816
|
+
// STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
|
|
6817
|
+
// establish OR lease the managed home box. The active-sandbox pointer is
|
|
6818
|
+
// the machine route's own epoch fence; the managed-home lease separately
|
|
6819
|
+
// owns cloud recovery, snapshots, billing, and reaping. Mixing those two
|
|
6820
|
+
// ownership domains lets an unrelated degraded cloud archive block a
|
|
6821
|
+
// healthy explicitly selected machine.
|
|
6822
|
+
// Whether the machine's latest Hello advertised the op-stream engine
|
|
6823
|
+
// (refreshed on every connect). Read only when the server flag is on —
|
|
6824
|
+
// one indexed lookup, and the flag off keeps this path byte-identical.
|
|
6825
|
+
const machineOpStream =
|
|
6826
|
+
settings.agentOpStreamEnabled === true
|
|
6827
|
+
? (await getEnrollment(db, input.workspaceId, activeSandboxRecord!.enrollmentId!))
|
|
6828
|
+
?.opStream === true
|
|
6829
|
+
: false;
|
|
6830
|
+
const established = await establishSelfhostedTurnSession(
|
|
6111
6831
|
{
|
|
6112
6832
|
db,
|
|
6113
6833
|
settings,
|
|
6114
6834
|
bus,
|
|
6115
|
-
opJournal,
|
|
6116
6835
|
onOp: machineOpObserver.observer,
|
|
6117
6836
|
onSandboxOperation: sandboxOperationObserver,
|
|
6118
|
-
|
|
6119
|
-
...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
|
|
6120
|
-
},
|
|
6121
|
-
{
|
|
6122
|
-
workspaceId: input.workspaceId,
|
|
6123
|
-
sessionId: input.sessionId,
|
|
6124
|
-
environment: sandboxEnvironment,
|
|
6125
|
-
workspaceMutationFence: {
|
|
6126
|
-
accountId: input.accountId,
|
|
6127
|
-
turnId: turn.id,
|
|
6128
|
-
executionGeneration,
|
|
6129
|
-
attemptId: input.attemptId,
|
|
6130
|
-
},
|
|
6131
|
-
pinnedSelfhosted: {
|
|
6132
|
-
sandboxId: activeSandboxPointer!.activeSandboxId!,
|
|
6133
|
-
epoch: activeSandboxPointer!.activeEpoch,
|
|
6134
|
-
},
|
|
6135
|
-
// HOME semantics for a mid-turn clear-to-null: only a genuine
|
|
6136
|
-
// machine-HOME session (its home IS this machine, session.sandboxBackend
|
|
6137
|
-
// === "selfhosted") resolves null back to the pinned machine. A Modal-HOME
|
|
6138
|
-
// session merely PINNED to a machine this turn never established its group
|
|
6139
|
-
// box, so defaultIsHome:false makes a clear-to-null fail typed
|
|
6140
|
-
// (`home_unavailable_this_turn`) rather than silently serving the machine;
|
|
6141
|
-
// the detach takes effect next turn. (Lazy home-box establishment on such a
|
|
6142
|
-
// clear is a deferred follow-up; issue #341.)
|
|
6143
|
-
defaultIsHome: session.sandboxBackend === "selfhosted",
|
|
6144
|
-
},
|
|
6145
|
-
established,
|
|
6146
|
-
),
|
|
6147
|
-
// For a machine route this is the active-pointer epoch, not a cloud
|
|
6148
|
-
// lease epoch. Cloud-only heartbeat/snapshot/capture paths require
|
|
6149
|
-
// sandboxGroupId and remain disabled for this branch.
|
|
6150
|
-
leaseEpoch: activeSandboxPointer!.activeEpoch,
|
|
6151
|
-
release: async () => undefined,
|
|
6152
|
-
};
|
|
6153
|
-
} else if (establishPolicy === "on-demand") {
|
|
6154
|
-
// Lazy sandbox provisioning: holder/group ids are fixed at turn start,
|
|
6155
|
-
// but the lease acquire + box establish + setup move behind the routing
|
|
6156
|
-
// proxy's first default-pointer op. A chat-only turn never calls it, so
|
|
6157
|
-
// no lease row, no provider box, no warm-meter interval.
|
|
6158
|
-
} else {
|
|
6159
|
-
resolvedSandbox = await waitForTurnOperation(
|
|
6160
|
-
resumeBoxForTurn(
|
|
6161
|
-
{
|
|
6162
|
-
db,
|
|
6163
|
-
settings: runSettings,
|
|
6164
|
-
logicalFallbackSettings: logicalSandboxSettings,
|
|
6165
|
-
cancellationSignal: sandboxResumeSignal,
|
|
6166
|
-
sandboxMetrics: runtimeMetricsHooksForObservability(observability),
|
|
6167
|
-
onSandboxLost: publishSandboxLost,
|
|
6837
|
+
opJournal,
|
|
6168
6838
|
},
|
|
6169
6839
|
{
|
|
6170
|
-
accountId: input.accountId,
|
|
6171
6840
|
workspaceId: input.workspaceId,
|
|
6172
|
-
|
|
6173
|
-
|
|
6174
|
-
|
|
6175
|
-
// is not machine-primary resumes a real cloud group box (the
|
|
6176
|
-
// deployment default), never a "selfhosted" box (which would throw
|
|
6177
|
-
// for lack of a bound agentId).
|
|
6178
|
-
backend: groupBoxBackend,
|
|
6179
|
-
os: session.sandboxOs,
|
|
6841
|
+
agentId: activeSandboxRecord!.enrollmentId!,
|
|
6842
|
+
opStream: machineOpStream,
|
|
6843
|
+
epoch: activeSandboxPointer!.activeEpoch,
|
|
6180
6844
|
environment: sandboxEnvironment,
|
|
6181
|
-
|
|
6182
|
-
|
|
6183
|
-
// running a DIFFERENT image (solo → durable rotation; N-holders →
|
|
6184
|
-
// SandboxImageConflictError surfaced as an actionable turn error). Prefer
|
|
6185
|
-
// the explicit Modal image ref, else the docker image. The selfhosted branch
|
|
6186
|
-
// (establishSelfhostedTurnSession) NEVER passes
|
|
6187
|
-
// an image — B3 lives only on this Modal else-branch.
|
|
6188
|
-
...((runSettings.modalImageRef ?? runSettings.dockerImage)
|
|
6189
|
-
? {
|
|
6190
|
-
image: runSettings.modalImageRef ?? runSettings.dockerImage,
|
|
6191
|
-
}
|
|
6845
|
+
...(transientCodemodeEnvironment
|
|
6846
|
+
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
6192
6847
|
: {}),
|
|
6193
|
-
|
|
6194
|
-
// conflicts on a live shared box set up under a different rig (solo
|
|
6195
|
-
// durable rotation / N-holders SandboxRigConflictError). Omitted for a
|
|
6196
|
-
// rig-less turn -> never stamped or enforced (shares exactly as today).
|
|
6197
|
-
...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
|
|
6198
|
-
},
|
|
6199
|
-
"turn",
|
|
6200
|
-
managedOwnership!.holderId,
|
|
6201
|
-
),
|
|
6202
|
-
sandboxResumeSignal,
|
|
6203
|
-
releaseLateSandbox,
|
|
6204
|
-
);
|
|
6205
|
-
setupBoxSession = resolvedSandbox.established.session;
|
|
6206
|
-
// Durable box-lifecycle events (sandbox-file-persistence observability):
|
|
6207
|
-
// record every box transition in session_events so the NEXT box loss is
|
|
6208
|
-
// attributable from the DB alone — worker logs rotate within hours, which
|
|
6209
|
-
// left both 2026-07-06 incidents without a durable trace. Best-effort.
|
|
6210
|
-
await publishSandboxLifecycleEvents(resolvedSandbox);
|
|
6211
|
-
// M7 hot-swap: when the selfhosted feature is on, wrap the established
|
|
6212
|
-
// group box in the STABLE routing proxy before it is injected NON-OWNED
|
|
6213
|
-
// into the run. The SDK binds to this ONE object once and calls its
|
|
6214
|
-
// methods per tool call; the proxy re-reads (active_sandbox_id,
|
|
6215
|
-
// active_epoch) per op and dispatches to the currently-active backend, so
|
|
6216
|
-
// a sandbox_swap mid-turn lands the NEXT tool call on the new box. With
|
|
6217
|
-
// the flag off the established group box is injected unchanged (today's
|
|
6218
|
-
// path). The lease still owns the group box lifecycle — the proxy is a
|
|
6219
|
-
// routing veneer, not an owner.
|
|
6220
|
-
resolvedSandbox = {
|
|
6221
|
-
...resolvedSandbox,
|
|
6222
|
-
established: wrapTurnBoxWithRouting(
|
|
6223
|
-
{
|
|
6224
|
-
db,
|
|
6225
|
-
settings,
|
|
6226
|
-
bus,
|
|
6227
|
-
opJournal,
|
|
6228
|
-
onSandboxOperation: sandboxOperationObserver,
|
|
6229
|
-
onHomeSandboxLost: publishSandboxLost,
|
|
6230
|
-
onHomeSandboxRebound,
|
|
6231
|
-
...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
|
|
6848
|
+
workingDir: activeSandboxPointer!.workingDir,
|
|
6232
6849
|
},
|
|
6233
|
-
|
|
6234
|
-
|
|
6235
|
-
|
|
6236
|
-
|
|
6237
|
-
|
|
6238
|
-
|
|
6239
|
-
|
|
6240
|
-
|
|
6241
|
-
|
|
6242
|
-
|
|
6243
|
-
|
|
6244
|
-
|
|
6245
|
-
|
|
6850
|
+
);
|
|
6851
|
+
// The machine-primary establish narrows `session` to SelfhostedSession
|
|
6852
|
+
// (buildSelfhostedBackendSession); EstablishedSandboxSession widens it.
|
|
6853
|
+
machinePrimarySession =
|
|
6854
|
+
established.session as import("@opengeni/runtime").SelfhostedSession;
|
|
6855
|
+
setupBoxSession = established.session;
|
|
6856
|
+
resolvedSandbox = {
|
|
6857
|
+
// Wrap in the SAME routing proxy so a mid-turn swap (to another machine
|
|
6858
|
+
// or back to the group box) still re-routes per op. PIN this established
|
|
6859
|
+
// SelfhostedSession for the machine pointer so the turn-start manifest
|
|
6860
|
+
// write (via the proxy's `state` getter) and the per-op reads hit ONE
|
|
6861
|
+
// instance — no two-instance manifest divergence.
|
|
6862
|
+
established: wrapTurnBoxWithRouting(
|
|
6863
|
+
{
|
|
6864
|
+
db,
|
|
6865
|
+
settings,
|
|
6866
|
+
bus,
|
|
6867
|
+
opJournal,
|
|
6868
|
+
onOp: machineOpObserver.observer,
|
|
6869
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
6870
|
+
onHomeSandboxRebound,
|
|
6871
|
+
...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
|
|
6872
|
+
},
|
|
6873
|
+
{
|
|
6874
|
+
workspaceId: input.workspaceId,
|
|
6875
|
+
sessionId: input.sessionId,
|
|
6876
|
+
environment: sandboxEnvironment,
|
|
6877
|
+
...(transientCodemodeEnvironment
|
|
6878
|
+
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
6879
|
+
: {}),
|
|
6880
|
+
workspaceMutationFence: {
|
|
6881
|
+
accountId: input.accountId,
|
|
6882
|
+
turnId: turn.id,
|
|
6883
|
+
executionGeneration,
|
|
6884
|
+
attemptId: input.attemptId,
|
|
6885
|
+
},
|
|
6886
|
+
pinnedSelfhosted: {
|
|
6887
|
+
sandboxId: activeSandboxPointer!.activeSandboxId!,
|
|
6888
|
+
epoch: activeSandboxPointer!.activeEpoch,
|
|
6889
|
+
},
|
|
6890
|
+
// HOME semantics for a mid-turn clear-to-null: only a genuine
|
|
6891
|
+
// machine-HOME session (its home IS this machine, session.sandboxBackend
|
|
6892
|
+
// === "selfhosted") resolves null back to the pinned machine. A Modal-HOME
|
|
6893
|
+
// session merely PINNED to a machine this turn never established its group
|
|
6894
|
+
// box, so defaultIsHome:false makes a clear-to-null fail typed
|
|
6895
|
+
// (`home_unavailable_this_turn`) rather than silently serving the machine;
|
|
6896
|
+
// the detach takes effect next turn. (Lazy home-box establishment on such a
|
|
6897
|
+
// clear is a deferred follow-up; issue #341.)
|
|
6898
|
+
defaultIsHome: session.sandboxBackend === "selfhosted",
|
|
6899
|
+
},
|
|
6900
|
+
established,
|
|
6901
|
+
),
|
|
6902
|
+
// For a machine route this is the active-pointer epoch, not a cloud
|
|
6903
|
+
// lease epoch. Cloud-only heartbeat/snapshot/capture paths require
|
|
6904
|
+
// sandboxGroupId and remain disabled for this branch.
|
|
6905
|
+
leaseEpoch: activeSandboxPointer!.activeEpoch,
|
|
6906
|
+
release: async () => undefined,
|
|
6907
|
+
};
|
|
6908
|
+
} else if (establishPolicy === "on-demand") {
|
|
6909
|
+
// Lazy sandbox provisioning: holder/group ids are fixed at turn start,
|
|
6910
|
+
// but the lease acquire + box establish + setup move behind the routing
|
|
6911
|
+
// proxy's first default-pointer op. A chat-only turn never calls it, so
|
|
6912
|
+
// no lease row, no provider box, no warm-meter interval.
|
|
6913
|
+
} else {
|
|
6914
|
+
resolvedSandbox = await waitForTurnOperation(
|
|
6915
|
+
resumeBoxForTurn(
|
|
6916
|
+
{
|
|
6917
|
+
db,
|
|
6918
|
+
settings: runSettings,
|
|
6919
|
+
logicalFallbackSettings: logicalSandboxSettings,
|
|
6920
|
+
cancellationSignal: sandboxResumeSignal,
|
|
6921
|
+
sandboxMetrics: runtimeMetricsHooksForObservability(observability),
|
|
6922
|
+
onSandboxLost: publishSandboxLost,
|
|
6246
6923
|
},
|
|
6247
|
-
|
|
6924
|
+
{
|
|
6248
6925
|
accountId: input.accountId,
|
|
6926
|
+
workspaceId: input.workspaceId,
|
|
6249
6927
|
sandboxGroupId: session.sandboxGroupId,
|
|
6250
|
-
|
|
6251
|
-
|
|
6928
|
+
sessionId: input.sessionId,
|
|
6929
|
+
// groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
|
|
6930
|
+
// is not machine-primary resumes a real cloud group box (the
|
|
6931
|
+
// deployment default), never a "selfhosted" box (which would throw
|
|
6932
|
+
// for lack of a bound agentId).
|
|
6252
6933
|
backend: groupBoxBackend,
|
|
6934
|
+
os: session.sandboxOs,
|
|
6935
|
+
environment: sandboxEnvironment,
|
|
6936
|
+
// IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
|
|
6937
|
+
// this run resolves. The lease stamps it + conflicts on a live shared box
|
|
6938
|
+
// running a DIFFERENT image (solo → durable rotation; N-holders →
|
|
6939
|
+
// SandboxImageConflictError surfaced as an actionable turn error). Prefer
|
|
6940
|
+
// the explicit Modal image ref, else the docker image. The selfhosted branch
|
|
6941
|
+
// (establishSelfhostedTurnSession) NEVER passes
|
|
6942
|
+
// an image — B3 lives only on this Modal else-branch.
|
|
6943
|
+
...((runSettings.modalImageRef ?? runSettings.dockerImage)
|
|
6944
|
+
? {
|
|
6945
|
+
image: runSettings.modalImageRef ?? runSettings.dockerImage,
|
|
6946
|
+
}
|
|
6947
|
+
: {}),
|
|
6948
|
+
// RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
|
|
6949
|
+
// conflicts on a live shared box set up under a different rig (solo
|
|
6950
|
+
// durable rotation / N-holders SandboxRigConflictError). Omitted for a
|
|
6951
|
+
// rig-less turn -> never stamped or enforced (shares exactly as today).
|
|
6952
|
+
...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
|
|
6253
6953
|
},
|
|
6254
|
-
|
|
6255
|
-
|
|
6256
|
-
|
|
6257
|
-
|
|
6258
|
-
|
|
6259
|
-
|
|
6260
|
-
|
|
6954
|
+
"turn",
|
|
6955
|
+
managedOwnership!.holderId,
|
|
6956
|
+
),
|
|
6957
|
+
sandboxResumeSignal,
|
|
6958
|
+
releaseLateSandbox,
|
|
6959
|
+
);
|
|
6960
|
+
setupBoxSession = resolvedSandbox.established.session;
|
|
6961
|
+
// Durable box-lifecycle events (sandbox-file-persistence observability):
|
|
6962
|
+
// record every box transition in session_events so the NEXT box loss is
|
|
6963
|
+
// attributable from the DB alone — worker logs rotate within hours, which
|
|
6964
|
+
// left both 2026-07-06 incidents without a durable trace. Best-effort.
|
|
6965
|
+
await publishSandboxLifecycleEvents(resolvedSandbox);
|
|
6966
|
+
// M7 hot-swap: when the selfhosted feature is on, wrap the established
|
|
6967
|
+
// group box in the STABLE routing proxy before it is injected NON-OWNED
|
|
6968
|
+
// into the run. The SDK binds to this ONE object once and calls its
|
|
6969
|
+
// methods per tool call; the proxy re-reads (active_sandbox_id,
|
|
6970
|
+
// active_epoch) per op and dispatches to the currently-active backend, so
|
|
6971
|
+
// a sandbox_swap mid-turn lands the NEXT tool call on the new box. With
|
|
6972
|
+
// the flag off the established group box is injected unchanged (today's
|
|
6973
|
+
// path). The lease still owns the group box lifecycle — the proxy is a
|
|
6974
|
+
// routing veneer, not an owner.
|
|
6975
|
+
resolvedSandbox = {
|
|
6976
|
+
...resolvedSandbox,
|
|
6977
|
+
established: wrapTurnBoxWithRouting(
|
|
6978
|
+
{
|
|
6979
|
+
db,
|
|
6980
|
+
settings,
|
|
6981
|
+
bus,
|
|
6982
|
+
opJournal,
|
|
6983
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
6984
|
+
onHomeSandboxLost: publishSandboxLost,
|
|
6985
|
+
onHomeSandboxRebound,
|
|
6986
|
+
...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
|
|
6987
|
+
},
|
|
6988
|
+
// Thread the SAME declared environment the group box was created with
|
|
6989
|
+
// (resumeBoxForTurn, above) so a selfhosted swap target's manifest
|
|
6990
|
+
// carries it too — the SDK's per-turn manifest-env delta stays empty
|
|
6991
|
+
// (no "cannot change manifest environment variables" throw).
|
|
6992
|
+
{
|
|
6993
|
+
workspaceId: input.workspaceId,
|
|
6994
|
+
sessionId: input.sessionId,
|
|
6995
|
+
environment: sandboxEnvironment,
|
|
6996
|
+
...(transientCodemodeEnvironment
|
|
6997
|
+
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
6998
|
+
: {}),
|
|
6999
|
+
workspaceMutationFence: {
|
|
7000
|
+
accountId: input.accountId,
|
|
7001
|
+
turnId: turn.id,
|
|
7002
|
+
executionGeneration,
|
|
7003
|
+
attemptId: input.attemptId,
|
|
7004
|
+
},
|
|
7005
|
+
homeLease: {
|
|
7006
|
+
accountId: input.accountId,
|
|
7007
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
7008
|
+
leaseEpoch: resolvedSandbox.leaseEpoch,
|
|
7009
|
+
instanceId: resolvedSandbox.established.instanceId,
|
|
7010
|
+
backend: groupBoxBackend,
|
|
7011
|
+
},
|
|
7012
|
+
},
|
|
7013
|
+
resolvedSandbox.established,
|
|
7014
|
+
),
|
|
7015
|
+
};
|
|
7016
|
+
}
|
|
7017
|
+
if (resolvedSandbox) {
|
|
7018
|
+
startLeaseHeartbeat(resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
|
|
7019
|
+
}
|
|
7020
|
+
} catch (error) {
|
|
7021
|
+
sandboxEstablishOutcome = "failed";
|
|
7022
|
+
throw error;
|
|
7023
|
+
} finally {
|
|
7024
|
+
recordTurnStartupPhase(observability, {
|
|
7025
|
+
phase: "sandbox_establish",
|
|
7026
|
+
provider: turnExecutionPolicy.providerId,
|
|
7027
|
+
backend: machinePrimary ? "selfhosted" : groupBoxBackend,
|
|
7028
|
+
outcome: sandboxEstablishOutcome,
|
|
7029
|
+
durationSeconds: (performance.now() - sandboxEstablishStartedAt) / 1_000,
|
|
7030
|
+
});
|
|
6261
7031
|
}
|
|
6262
7032
|
}
|
|
6263
7033
|
|
|
6264
|
-
const ordinaryFileResourceDownloads = await
|
|
6265
|
-
|
|
6266
|
-
|
|
6267
|
-
|
|
6268
|
-
|
|
6269
|
-
|
|
6270
|
-
|
|
6271
|
-
|
|
6272
|
-
|
|
6273
|
-
|
|
6274
|
-
|
|
6275
|
-
|
|
7034
|
+
const ordinaryFileResourceDownloads = await (async () => {
|
|
7035
|
+
const fileResolutionStartedAt = performance.now();
|
|
7036
|
+
let fileResolutionOutcome: "completed" | "failed" = "completed";
|
|
7037
|
+
try {
|
|
7038
|
+
return await waitForTurnOperation(
|
|
7039
|
+
sandboxFileDownloadsForRun(
|
|
7040
|
+
runSettings,
|
|
7041
|
+
db,
|
|
7042
|
+
objectStorage,
|
|
7043
|
+
input.workspaceId,
|
|
7044
|
+
turnResources,
|
|
7045
|
+
activeSandboxBackend ?? groupBoxBackend,
|
|
7046
|
+
),
|
|
7047
|
+
cancellationSignal,
|
|
7048
|
+
undefined,
|
|
7049
|
+
);
|
|
7050
|
+
} catch (error) {
|
|
7051
|
+
fileResolutionOutcome = "failed";
|
|
7052
|
+
throw error;
|
|
7053
|
+
} finally {
|
|
7054
|
+
recordTurnStartupPhase(observability, {
|
|
7055
|
+
phase: "file_resolution",
|
|
7056
|
+
provider: turnExecutionPolicy.providerId,
|
|
7057
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
7058
|
+
outcome: fileResolutionOutcome,
|
|
7059
|
+
durationSeconds: (performance.now() - fileResolutionStartedAt) / 1_000,
|
|
7060
|
+
count: turnResources.length,
|
|
7061
|
+
});
|
|
7062
|
+
}
|
|
7063
|
+
})();
|
|
6276
7064
|
if (requiredGeneratedVideoFiles.length > 0 && !objectStorage) {
|
|
6277
7065
|
throw new Error("Generated video materialization requires object storage");
|
|
6278
7066
|
}
|
|
@@ -6302,6 +7090,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6302
7090
|
]),
|
|
6303
7091
|
).values(),
|
|
6304
7092
|
];
|
|
7093
|
+
const toolContextPreparationStartedAt = performance.now();
|
|
6305
7094
|
throwIfWorkerShuttingDown();
|
|
6306
7095
|
throwIfTurnCancelled();
|
|
6307
7096
|
const mcpCredentialRootSessionId =
|
|
@@ -6348,11 +7137,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6348
7137
|
}
|
|
6349
7138
|
await publish!([{ type: "tool.auth_needed", payload }], true);
|
|
6350
7139
|
};
|
|
6351
|
-
const
|
|
7140
|
+
const selectedApiIntegrationServerIds = new Set(turnTools.map((tool) => tool.id));
|
|
6352
7141
|
const localMcpServers = buildApiIntegrationServersForTurn({
|
|
6353
7142
|
settings: runSettings,
|
|
6354
7143
|
integrations: installedApiIntegrations.filter((integration) =>
|
|
6355
|
-
|
|
7144
|
+
selectedApiIntegrationServerIds.has(integration.serverId),
|
|
6356
7145
|
),
|
|
6357
7146
|
authority: {
|
|
6358
7147
|
accountId: input.accountId,
|
|
@@ -6396,54 +7185,98 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6396
7185
|
};
|
|
6397
7186
|
})()
|
|
6398
7187
|
: undefined;
|
|
6399
|
-
|
|
6400
|
-
|
|
6401
|
-
|
|
6402
|
-
|
|
6403
|
-
|
|
6404
|
-
|
|
6405
|
-
|
|
6406
|
-
|
|
6407
|
-
|
|
6408
|
-
|
|
6409
|
-
|
|
6410
|
-
|
|
6411
|
-
|
|
6412
|
-
|
|
6413
|
-
|
|
6414
|
-
|
|
6415
|
-
|
|
6416
|
-
|
|
6417
|
-
|
|
6418
|
-
|
|
6419
|
-
|
|
6420
|
-
|
|
6421
|
-
|
|
6422
|
-
|
|
6423
|
-
|
|
6424
|
-
|
|
6425
|
-
|
|
6426
|
-
|
|
6427
|
-
|
|
6428
|
-
|
|
6429
|
-
|
|
6430
|
-
|
|
6431
|
-
|
|
6432
|
-
|
|
6433
|
-
|
|
6434
|
-
|
|
7188
|
+
const selectedFirstPartyMcpTools = allowedFirstPartyMcpToolsForSession(
|
|
7189
|
+
runSettings,
|
|
7190
|
+
session.firstPartyMcpTools,
|
|
7191
|
+
);
|
|
7192
|
+
recordTurnStartupPhase(observability, {
|
|
7193
|
+
phase: "tool_context_preparation",
|
|
7194
|
+
provider: turnExecutionPolicy.providerId,
|
|
7195
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
7196
|
+
outcome: "completed",
|
|
7197
|
+
durationSeconds: (performance.now() - toolContextPreparationStartedAt) / 1_000,
|
|
7198
|
+
count: turnTools.length,
|
|
7199
|
+
});
|
|
7200
|
+
const toolPreparationStartedAt = performance.now();
|
|
7201
|
+
let toolPreparationOutcome: "completed" | "failed" = "completed";
|
|
7202
|
+
try {
|
|
7203
|
+
preparedTools = await waitForTurnOperation(
|
|
7204
|
+
runtime.prepareTools(runSettings, turnTools, {
|
|
7205
|
+
accountId: input.accountId,
|
|
7206
|
+
workspaceId: input.workspaceId,
|
|
7207
|
+
sessionId: input.sessionId,
|
|
7208
|
+
// Sign the calling turn into the first-party token so tools classify
|
|
7209
|
+
// the caller by its own identity (sacred-pause guard), not the racy
|
|
7210
|
+
// live active pointer.
|
|
7211
|
+
...(turnId ? { turnId } : {}),
|
|
7212
|
+
attemptId: input.attemptId,
|
|
7213
|
+
executionGeneration,
|
|
7214
|
+
subjectId: "worker:first-party-mcp",
|
|
7215
|
+
subjectLabel: "OpenGeni worker",
|
|
7216
|
+
...(credentialSubjectId ? { credentialSubjectId } : {}),
|
|
7217
|
+
...(codexAppsAuth ? { codexAppsAuth } : {}),
|
|
7218
|
+
resolveCredential,
|
|
7219
|
+
onAuthNeeded: publishToolAuthNeeded,
|
|
7220
|
+
localMcpServers,
|
|
7221
|
+
onPreparationPhase: (measurement) => {
|
|
7222
|
+
recordTurnStartupPhase(observability, {
|
|
7223
|
+
phase: `tool_${measurement.phase}`,
|
|
7224
|
+
provider: turnExecutionPolicy.providerId,
|
|
7225
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
7226
|
+
outcome: measurement.outcome,
|
|
7227
|
+
durationSeconds: measurement.durationSeconds,
|
|
7228
|
+
count: turnTools.length,
|
|
7229
|
+
});
|
|
6435
7230
|
},
|
|
7231
|
+
onAttemptToolCatalog: async (catalog) => {
|
|
7232
|
+
await persistAttemptToolCatalog(db, catalog);
|
|
7233
|
+
},
|
|
7234
|
+
// Manager-style sessions carry a creation-validated permission set
|
|
7235
|
+
// for their first-party MCP token; null keeps the fixed default.
|
|
6436
7236
|
...(session.firstPartyMcpPermissions?.length
|
|
6437
|
-
? {
|
|
7237
|
+
? { firstPartyPermissions: session.firstPartyMcpPermissions }
|
|
6438
7238
|
: {}),
|
|
6439
|
-
|
|
6440
|
-
|
|
6441
|
-
|
|
7239
|
+
firstPartyTools: selectedFirstPartyMcpTools,
|
|
7240
|
+
nestedAgentDepth: session.nestedAgentDepth,
|
|
7241
|
+
effectiveMaxNestedAgentDepth: session.effectiveMaxNestedAgentDepth,
|
|
7242
|
+
attemptToolDefinitions: createFirstPartyInteractionAttemptToolDefinitions({
|
|
7243
|
+
settings: runSettings,
|
|
7244
|
+
scope: {
|
|
7245
|
+
accountId: input.accountId,
|
|
7246
|
+
workspaceId: input.workspaceId,
|
|
7247
|
+
sessionId: input.sessionId,
|
|
7248
|
+
turnId: turn.id,
|
|
7249
|
+
attemptId: input.attemptId,
|
|
7250
|
+
executionGeneration,
|
|
7251
|
+
},
|
|
7252
|
+
...(session.firstPartyMcpPermissions?.length
|
|
7253
|
+
? { permissions: session.firstPartyMcpPermissions }
|
|
7254
|
+
: {}),
|
|
7255
|
+
selectedTools: selectedFirstPartyMcpTools,
|
|
7256
|
+
subjectId: "worker:first-party-mcp",
|
|
7257
|
+
subjectLabel: "OpenGeni worker",
|
|
7258
|
+
...(interactionInterventionResume
|
|
7259
|
+
? { interventionResume: interactionInterventionResume }
|
|
7260
|
+
: {}),
|
|
7261
|
+
}),
|
|
6442
7262
|
}),
|
|
6443
|
-
|
|
6444
|
-
|
|
6445
|
-
|
|
6446
|
-
)
|
|
7263
|
+
cancellationSignal,
|
|
7264
|
+
async (latePreparedTools) => await latePreparedTools.close().catch(() => undefined),
|
|
7265
|
+
);
|
|
7266
|
+
} catch (error) {
|
|
7267
|
+
toolPreparationOutcome = "failed";
|
|
7268
|
+
throw error;
|
|
7269
|
+
} finally {
|
|
7270
|
+
recordTurnStartupPhase(observability, {
|
|
7271
|
+
phase: "tool_preparation",
|
|
7272
|
+
provider: turnExecutionPolicy.providerId,
|
|
7273
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
7274
|
+
outcome: toolPreparationOutcome,
|
|
7275
|
+
durationSeconds: (performance.now() - toolPreparationStartedAt) / 1_000,
|
|
7276
|
+
count: turnTools.length,
|
|
7277
|
+
});
|
|
7278
|
+
}
|
|
7279
|
+
const postToolPreparationStartedAt = performance.now();
|
|
6447
7280
|
if (turnId && preparedTools.attemptToolEnvironment) {
|
|
6448
7281
|
codemodeDispatcher = new CodemodeAttemptDispatcher(
|
|
6449
7282
|
db,
|
|
@@ -6546,19 +7379,51 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6546
7379
|
}
|
|
6547
7380
|
|
|
6548
7381
|
if (resolvedModel?.provider.kind === "codex-subscription") {
|
|
6549
|
-
const
|
|
6550
|
-
|
|
6551
|
-
|
|
6552
|
-
|
|
6553
|
-
|
|
6554
|
-
|
|
6555
|
-
|
|
7382
|
+
const imageAuthority = connectedSubscriptionImageGenerationAuthority(
|
|
7383
|
+
codexContext,
|
|
7384
|
+
effectiveCodexCredentialId,
|
|
7385
|
+
);
|
|
7386
|
+
if (!imageAuthority) return {};
|
|
7387
|
+
return {
|
|
7388
|
+
imageGeneration: {
|
|
7389
|
+
kind: "provider_adapter",
|
|
7390
|
+
execute: async ({ prompt, references }, { toolCallId }) => {
|
|
7391
|
+
const resolvedReferences = await resolveImageReferences(references);
|
|
7392
|
+
const receipt = await executeCodexImageGeneration({
|
|
7393
|
+
db,
|
|
7394
|
+
objectStorage,
|
|
7395
|
+
accountId: input.accountId,
|
|
7396
|
+
workspaceId: input.workspaceId,
|
|
7397
|
+
sessionId: input.sessionId,
|
|
7398
|
+
turnId: turn.id,
|
|
7399
|
+
attemptId: input.attemptId,
|
|
7400
|
+
toolCallId,
|
|
7401
|
+
prompt,
|
|
7402
|
+
references: resolvedReferences,
|
|
7403
|
+
credentialId: imageAuthority.credentialId,
|
|
7404
|
+
codexContext: imageAuthority.credentialContext,
|
|
7405
|
+
...(runtimeCancellationSignal ? { abortSignal: runtimeCancellationSignal } : {}),
|
|
7406
|
+
});
|
|
7407
|
+
generatedImageReceiptsByArtifactId.set(receipt.artifact.artifactId, receipt);
|
|
7408
|
+
await materializeGeneratedImage(receipt);
|
|
7409
|
+
return receipt;
|
|
7410
|
+
},
|
|
7411
|
+
},
|
|
7412
|
+
};
|
|
7413
|
+
}
|
|
7414
|
+
|
|
7415
|
+
if (resolvedModel?.provider.kind === "xai-subscription") {
|
|
7416
|
+
const imageAuthority = connectedSubscriptionImageGenerationAuthority(
|
|
7417
|
+
xaiRequestContext,
|
|
7418
|
+
effectiveXaiCredentialId,
|
|
7419
|
+
);
|
|
7420
|
+
if (!imageAuthority) return {};
|
|
6556
7421
|
return {
|
|
6557
7422
|
imageGeneration: {
|
|
6558
7423
|
kind: "provider_adapter",
|
|
6559
7424
|
execute: async ({ prompt, references }, { toolCallId }) => {
|
|
6560
7425
|
const resolvedReferences = await resolveImageReferences(references);
|
|
6561
|
-
const receipt = await
|
|
7426
|
+
const receipt = await executeXaiSubscriptionImageGeneration({
|
|
6562
7427
|
db,
|
|
6563
7428
|
objectStorage,
|
|
6564
7429
|
accountId: input.accountId,
|
|
@@ -6569,8 +7434,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6569
7434
|
toolCallId,
|
|
6570
7435
|
prompt,
|
|
6571
7436
|
references: resolvedReferences,
|
|
6572
|
-
credentialId:
|
|
6573
|
-
|
|
7437
|
+
credentialId: imageAuthority.credentialId,
|
|
7438
|
+
xaiContext: imageAuthority.credentialContext,
|
|
6574
7439
|
...(runtimeCancellationSignal ? { abortSignal: runtimeCancellationSignal } : {}),
|
|
6575
7440
|
});
|
|
6576
7441
|
generatedImageReceiptsByArtifactId.set(receipt.artifact.artifactId, receipt);
|
|
@@ -6623,7 +7488,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6623
7488
|
if (objectStorage && videoGenerationEnabled) {
|
|
6624
7489
|
if (videoGenerationPolicy.fundingSource === "opengeni_credits") {
|
|
6625
7490
|
videoGenerationCredential = managedVideoGenerationCredentialLease(modelRunSettings);
|
|
6626
|
-
} else {
|
|
7491
|
+
} else if (videoGenerationPolicy.fundingSource === "workspace_gateway") {
|
|
6627
7492
|
const workspaceCredential = await loadWorkspaceVercelAiGatewayCredentialLease(
|
|
6628
7493
|
db,
|
|
6629
7494
|
modelRunSettings,
|
|
@@ -6635,6 +7500,74 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6635
7500
|
...workspaceCredential,
|
|
6636
7501
|
};
|
|
6637
7502
|
}
|
|
7503
|
+
} else if (videoGenerationPolicy.fundingSource === "supergrok_subscription") {
|
|
7504
|
+
const encryptionKey = environmentsEncryptionKeyBytes(modelRunSettings);
|
|
7505
|
+
const subjectId = xaiLeaseSubjectId ?? turn.initiatingHumanSubjectId;
|
|
7506
|
+
if (encryptionKey && subjectId) {
|
|
7507
|
+
const authoritySnapshot =
|
|
7508
|
+
xaiAuthoritySnapshot ??
|
|
7509
|
+
(await resolveXaiProviderAccountAuthoritySnapshotForAcceptance(db, {
|
|
7510
|
+
workspaceId: input.workspaceId,
|
|
7511
|
+
subjectId,
|
|
7512
|
+
}));
|
|
7513
|
+
const pin = await getXaiSessionAccountPin(db, {
|
|
7514
|
+
workspaceId: input.workspaceId,
|
|
7515
|
+
subjectId,
|
|
7516
|
+
sessionId: input.sessionId,
|
|
7517
|
+
authoritySnapshot,
|
|
7518
|
+
});
|
|
7519
|
+
const selected = effectiveXaiCredentialId
|
|
7520
|
+
? {
|
|
7521
|
+
credentialId: effectiveXaiCredentialId,
|
|
7522
|
+
rotationEnabled: xaiRotationEnabled,
|
|
7523
|
+
}
|
|
7524
|
+
: await selectXaiCredentialForUse(db, {
|
|
7525
|
+
accountId: input.accountId,
|
|
7526
|
+
workspaceId: input.workspaceId,
|
|
7527
|
+
subjectId,
|
|
7528
|
+
authoritySnapshot,
|
|
7529
|
+
shardKey: input.sessionId,
|
|
7530
|
+
pinnedCredentialId: pin?.pinnedCredentialId ?? null,
|
|
7531
|
+
pinSource:
|
|
7532
|
+
pin?.pinSource === "manual" || pin?.pinSource === "policy"
|
|
7533
|
+
? pin.pinSource
|
|
7534
|
+
: null,
|
|
7535
|
+
});
|
|
7536
|
+
if (selected.credentialId) {
|
|
7537
|
+
if (
|
|
7538
|
+
selected.rotationEnabled &&
|
|
7539
|
+
pin?.pinSource !== "manual" &&
|
|
7540
|
+
pin?.pinnedCredentialId !== selected.credentialId
|
|
7541
|
+
) {
|
|
7542
|
+
await setXaiSessionAccountPin(db, {
|
|
7543
|
+
accountId: input.accountId,
|
|
7544
|
+
workspaceId: input.workspaceId,
|
|
7545
|
+
subjectId,
|
|
7546
|
+
sessionId: input.sessionId,
|
|
7547
|
+
authoritySnapshot,
|
|
7548
|
+
credentialId: selected.credentialId,
|
|
7549
|
+
pinSource: "policy",
|
|
7550
|
+
expectedVersion: pin?.version ?? null,
|
|
7551
|
+
}).catch((error: unknown) => {
|
|
7552
|
+
if (error instanceof Error && error.message === "xAI session pin changed") return;
|
|
7553
|
+
throw error;
|
|
7554
|
+
});
|
|
7555
|
+
}
|
|
7556
|
+
const credential = await materializeXaiCredentialForRun(db, {
|
|
7557
|
+
workspaceId: input.workspaceId,
|
|
7558
|
+
subjectId,
|
|
7559
|
+
credentialId: selected.credentialId,
|
|
7560
|
+
authoritySnapshot,
|
|
7561
|
+
encryptionKey,
|
|
7562
|
+
});
|
|
7563
|
+
videoGenerationCredential = xaiVideoGenerationCredentialLease({
|
|
7564
|
+
settings: modelRunSettings,
|
|
7565
|
+
credential,
|
|
7566
|
+
subjectId,
|
|
7567
|
+
authoritySnapshot,
|
|
7568
|
+
});
|
|
7569
|
+
}
|
|
7570
|
+
}
|
|
6638
7571
|
}
|
|
6639
7572
|
}
|
|
6640
7573
|
const videoGenerationOption: Pick<BuildAgentOptions, "videoGeneration"> = (() => {
|
|
@@ -6661,33 +7594,47 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6661
7594
|
resolvedSandbox?.established.session ?? sdkOwnedSandboxSession;
|
|
6662
7595
|
const fence = toolCancellationFenceRef.current;
|
|
6663
7596
|
const runAs = sandboxRunAs(modelRunSettings);
|
|
6664
|
-
|
|
6665
|
-
|
|
6666
|
-
|
|
6667
|
-
|
|
6668
|
-
|
|
6669
|
-
|
|
6670
|
-
|
|
6671
|
-
|
|
6672
|
-
|
|
6673
|
-
|
|
6674
|
-
|
|
6675
|
-
|
|
6676
|
-
|
|
6677
|
-
|
|
6678
|
-
|
|
6679
|
-
|
|
6680
|
-
|
|
6681
|
-
|
|
6682
|
-
|
|
6683
|
-
|
|
6684
|
-
|
|
6685
|
-
|
|
6686
|
-
|
|
6687
|
-
|
|
6688
|
-
|
|
6689
|
-
|
|
6690
|
-
|
|
7597
|
+
let accepted: Awaited<ReturnType<typeof admitVideoGenerationRequest>>;
|
|
7598
|
+
try {
|
|
7599
|
+
accepted = await admitVideoGenerationRequest({
|
|
7600
|
+
db,
|
|
7601
|
+
storage: objectStorage,
|
|
7602
|
+
settings: modelRunSettings,
|
|
7603
|
+
accountId: input.accountId,
|
|
7604
|
+
workspaceId: input.workspaceId,
|
|
7605
|
+
sessionId: input.sessionId,
|
|
7606
|
+
turnId: turn.id,
|
|
7607
|
+
attemptId: input.attemptId,
|
|
7608
|
+
toolCallId,
|
|
7609
|
+
toolInput,
|
|
7610
|
+
policy: videoGenerationPolicy,
|
|
7611
|
+
credential: videoGenerationCredential,
|
|
7612
|
+
...(sessionForReference && fence
|
|
7613
|
+
? {
|
|
7614
|
+
runCommand: async (command) =>
|
|
7615
|
+
await fence.runSandboxCommandStructured(
|
|
7616
|
+
sessionForReference as import("@opengeni/runtime").TurnSandboxCommandSession,
|
|
7617
|
+
{
|
|
7618
|
+
...command,
|
|
7619
|
+
...(runAs ? { runAs } : {}),
|
|
7620
|
+
},
|
|
7621
|
+
),
|
|
7622
|
+
}
|
|
7623
|
+
: {}),
|
|
7624
|
+
...(runtimeCancellationSignal ? { signal: runtimeCancellationSignal } : {}),
|
|
7625
|
+
});
|
|
7626
|
+
} catch (error) {
|
|
7627
|
+
if (error instanceof VideoReferenceInputError) {
|
|
7628
|
+
return VideoGenerationRejectedResult.parse({
|
|
7629
|
+
schemaVersion: 1,
|
|
7630
|
+
status: "rejected",
|
|
7631
|
+
code: error.code,
|
|
7632
|
+
message: error.message,
|
|
7633
|
+
operationCreated: false,
|
|
7634
|
+
});
|
|
7635
|
+
}
|
|
7636
|
+
throw error;
|
|
7637
|
+
}
|
|
6691
7638
|
videoGenerationAcceptancesByCallId.set(toolCallId, {
|
|
6692
7639
|
operationId: accepted.operationId,
|
|
6693
7640
|
requestDigest: accepted.requestDigest,
|
|
@@ -6727,179 +7674,218 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6727
7674
|
outcome,
|
|
6728
7675
|
}),
|
|
6729
7676
|
};
|
|
6730
|
-
const
|
|
6731
|
-
|
|
6732
|
-
|
|
6733
|
-
...(
|
|
6734
|
-
|
|
6735
|
-
|
|
6736
|
-
|
|
6737
|
-
|
|
6738
|
-
|
|
6739
|
-
|
|
6740
|
-
|
|
6741
|
-
|
|
6742
|
-
|
|
6743
|
-
|
|
6744
|
-
|
|
6745
|
-
|
|
6746
|
-
|
|
6747
|
-
|
|
6748
|
-
|
|
6749
|
-
|
|
6750
|
-
|
|
6751
|
-
|
|
6752
|
-
|
|
6753
|
-
// seeds it to the box's token file before the repository-clone runs; it never
|
|
6754
|
-
// touches the box/agent manifest env.
|
|
6755
|
-
...(activeSandboxBackend !== "selfhosted" && sandboxGitTokens
|
|
6756
|
-
? { gitTokenSeeds: sandboxGitTokens }
|
|
6757
|
-
: {}),
|
|
6758
|
-
...(activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings
|
|
6759
|
-
? { gitCredentialBindings: sandboxGitCredentialBindings }
|
|
6760
|
-
: {}),
|
|
6761
|
-
...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
|
|
6762
|
-
? { gitTokenSeed: sandboxGitToken }
|
|
6763
|
-
: {}),
|
|
6764
|
-
// Codemode delivery is managed-sandbox-only. Connected Machines own any
|
|
6765
|
-
// manually configured API credentials and must never be contacted during
|
|
6766
|
-
// turn admission merely to seed OpenGeni tooling.
|
|
6767
|
-
...(sandboxCodemodeToken
|
|
6768
|
-
? {
|
|
6769
|
-
codemodeTokenSeed: sandboxCodemodeToken,
|
|
6770
|
-
codemodeTokenSessionId: input.sessionId,
|
|
6771
|
-
}
|
|
6772
|
-
: {}),
|
|
6773
|
-
...(activeSandboxBackend ? { activeSandboxBackend } : {}),
|
|
6774
|
-
fileResourceDownloads,
|
|
6775
|
-
mcpServers: preparedTools.mcpServers,
|
|
6776
|
-
resolvedMcpConnectionIds: preparedTools.resolvedMcpConnectionIds,
|
|
6777
|
-
connectorActionPolicy,
|
|
6778
|
-
// LIVE by-reference connector namespaces (fills during this turn's
|
|
6779
|
-
// codex_apps tools/list): the codex tool_search description reads it per
|
|
6780
|
-
// model call so the model sees the account's real connected sources.
|
|
6781
|
-
codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
|
|
6782
|
-
// Resolved-model routing + gating (legacy defaults when null). The model
|
|
6783
|
-
// is passed as the model *string* (agent.model = runSettings.openaiModel),
|
|
6784
|
-
// NOT a Model instance: an instance only survives the in-process
|
|
6785
|
-
// ("none") run, whereas the SandboxAgent/Modal path drops it and
|
|
6786
|
-
// re-resolves the model *name* through the global MultiProviderModelProvider
|
|
6787
|
-
// configureOpenAI installed — so registry models (Fireworks GLM) route to
|
|
6788
|
-
// their own client instead of 404ing against the built-in Azure/OpenAI
|
|
6789
|
-
// client. The gating still comes from the resolved provider: server-side
|
|
6790
|
-
// store/compaction follow the provider's compaction mode (registry
|
|
6791
|
-
// providers resolve to "client"); encrypted reasoning is only
|
|
6792
|
-
// round-tripped on the Responses wire API; hosted web search is attached
|
|
6793
|
-
// whenever the provider declares it runnable and is independent of the
|
|
6794
|
-
// session's MCP allow-list; the effective context window drives the
|
|
6795
|
-
// compaction threshold.
|
|
6796
|
-
hostedWebSearch,
|
|
6797
|
-
...imageGenerationOption,
|
|
6798
|
-
...videoGenerationOption,
|
|
6799
|
-
lazyToolTransport,
|
|
6800
|
-
supportsImageInput,
|
|
6801
|
-
inputFileMediaTypes: modelInputPolicy.inputFileMediaTypes,
|
|
6802
|
-
...(resolvedModel
|
|
6803
|
-
? {
|
|
6804
|
-
encryptedReasoning:
|
|
6805
|
-
resolvedModel.provider.api === "responses" &&
|
|
6806
|
-
runSettings.openaiReasoningEncryptedContent,
|
|
6807
|
-
contextWindowTokens:
|
|
6808
|
-
resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
|
|
6809
|
-
// The ChatGPT/Codex backend rejects the SDK's HOSTED apply_patch
|
|
6810
|
-
// tool. Gateway Responses routes likewise expose ordinary function
|
|
6811
|
-
// tools, not OpenAI-hosted sandbox tools. Tell buildAgent to use
|
|
6812
|
-
// function apply_patch and wrap successful view_image results as
|
|
6813
|
-
// typed input_image content. Chat wires have no proven typed image
|
|
6814
|
-
// result transport and therefore receive no view_image tool.
|
|
6815
|
-
structuredToolTransport: structuredToolTransportForTurn(resolvedModel),
|
|
6816
|
-
// EXPLICIT computer-use tool transport, derived from the resolved provider's
|
|
6817
|
-
// authoritative wire identity (codex → function-image, chat → disabled,
|
|
6818
|
-
// responses → hosted) so the runtime never string-sniffs the model instance's
|
|
6819
|
-
// constructor name. See {@link computerToolModeForTurn}.
|
|
6820
|
-
computerToolMode: computerToolModeForTurn(resolvedModel),
|
|
6821
|
-
...(promptCacheKey ? { promptCacheKey } : {}),
|
|
6822
|
-
}
|
|
6823
|
-
: // LEGACY global-client fallback (resolveTurnModel returned null → the model
|
|
6824
|
-
// is not in the registry, served by the built-in OpenAI/Azure Responses
|
|
6825
|
-
// client). That backend has real hosted support, so pin computerToolMode to
|
|
6826
|
-
// "hosted" EXPLICITLY rather than leaving the runtime to sniff the instance.
|
|
6827
|
-
{
|
|
6828
|
-
computerToolMode: computerToolModeForTurn(null),
|
|
6829
|
-
promptCacheKey: input.sessionId,
|
|
6830
|
-
}),
|
|
6831
|
-
// Lazy computer-use seam: runtime first brings up :0 only after the model
|
|
6832
|
-
// selects a computer tool, then this hook begins the optional proof
|
|
6833
|
-
// recording. Shell/filesystem turns never invoke either operation.
|
|
6834
|
-
onComputerUseReady: async () => {
|
|
6835
|
-
if (!resolvedSandbox) {
|
|
6836
|
-
throw new Error("Computer-use display became ready without a resolved sandbox");
|
|
6837
|
-
}
|
|
6838
|
-
// This callback is the authoritative execution boundary. Record the
|
|
6839
|
-
// action before async ffmpeg startup so transport-event ordering cannot
|
|
6840
|
-
// make settlement misclassify a real computer turn as unused.
|
|
6841
|
-
didComputerUse = true;
|
|
6842
|
-
await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
|
|
6843
|
-
},
|
|
6844
|
-
onRetainableSessionImageOutput: retainSessionImageAtToolBoundary,
|
|
6845
|
-
...(packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {}),
|
|
6846
|
-
...(session.skills.length > 0 ? { sessionSkills: session.skills } : {}),
|
|
6847
|
-
...(skillLibraryRuntime.skillLibrarySkills.length > 0
|
|
6848
|
-
? {
|
|
6849
|
-
skillLibrarySkills: skillLibraryRuntime.skillLibrarySkills,
|
|
6850
|
-
skillLibrarySelections: skillLibraryRuntime.skillLibrarySelections,
|
|
6851
|
-
}
|
|
6852
|
-
: {}),
|
|
6853
|
-
...(!structuredWorkspacePolicyActive && workspaceAgentInstructions
|
|
6854
|
-
? { instructionsTemplate: workspaceAgentInstructions }
|
|
6855
|
-
: {}),
|
|
6856
|
-
...(workspaceGovernance ? { workspaceGovernance } : {}),
|
|
6857
|
-
...(workspaceMemory ? { workspaceMemory } : {}),
|
|
6858
|
-
// Per-session persona tier (session > workspace > deployment default).
|
|
6859
|
-
// Composed system-level AFTER the workspace persona so it refines it for
|
|
6860
|
-
// this one session; absent ⇒ byte-identical to today's composition.
|
|
6861
|
-
...(session.instructions ? { sessionInstructions: session.instructions } : {}),
|
|
6862
|
-
// Exact host context captured when this turn was accepted. It is
|
|
6863
|
-
// system-level and disappears with the turn rather than entering chat.
|
|
6864
|
-
...(turn.turnInstructions || mcpAvailabilityInstructions
|
|
6865
|
-
? {
|
|
6866
|
-
turnInstructions: [turn.turnInstructions, mcpAvailabilityInstructions]
|
|
6867
|
-
.filter((value): value is string => Boolean(value))
|
|
6868
|
-
.join(" "),
|
|
6869
|
-
}
|
|
6870
|
-
: {}),
|
|
6871
|
-
...workspaceEnvironmentOption,
|
|
6872
|
-
// RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
|
|
6873
|
-
// the frozen version carries a non-empty script), and the rig credential
|
|
6874
|
-
// hooks. All absent for a rig-less turn (byte-for-byte today).
|
|
6875
|
-
...(rigVersion && rigName
|
|
6876
|
-
? {
|
|
6877
|
-
rig: { name: rigName, version: rigVersion.version },
|
|
6878
|
-
...(rigVersion.setupScript && rigVersion.setupScript.trim().length > 0
|
|
6879
|
-
? {
|
|
6880
|
-
rigSetup: {
|
|
6881
|
-
rigId: session.rigId!,
|
|
6882
|
-
versionId: rigVersion.id,
|
|
6883
|
-
rigName,
|
|
6884
|
-
script: rigVersion.setupScript,
|
|
6885
|
-
timeoutMs: runSettings.rigSetupTimeoutMs,
|
|
6886
|
-
contentHash: rigProviderImageContentHash({
|
|
6887
|
-
backend: turn.sandboxBackend,
|
|
6888
|
-
sourceImage: rigProviderImageSourceImage(
|
|
6889
|
-
logicalSandboxSettings,
|
|
6890
|
-
turn.sandboxBackend,
|
|
6891
|
-
),
|
|
6892
|
-
definition: rigVersion,
|
|
6893
|
-
}),
|
|
6894
|
-
},
|
|
6895
|
-
}
|
|
6896
|
-
: {}),
|
|
6897
|
-
...(rigVersion.credentialHooks.length > 0
|
|
6898
|
-
? { rigCredentialHookIds: rigVersion.credentialHooks }
|
|
6899
|
-
: {}),
|
|
6900
|
-
}
|
|
6901
|
-
: {}),
|
|
7677
|
+
const runtimeSkillActivations = [
|
|
7678
|
+
...installedSkillRuntime.activations,
|
|
7679
|
+
...packRuntime.skillActivations,
|
|
7680
|
+
...session.skills.map((skill) => ({
|
|
7681
|
+
source: "session" as const,
|
|
7682
|
+
id: `session:${session.id}:${skill.name}`,
|
|
7683
|
+
artifact: {
|
|
7684
|
+
name: skill.name,
|
|
7685
|
+
description: skill.description ?? null,
|
|
7686
|
+
files: skill.files.map((file) => ({
|
|
7687
|
+
path: file.path,
|
|
7688
|
+
content: file.content,
|
|
7689
|
+
})),
|
|
7690
|
+
},
|
|
7691
|
+
reason: "attached to session",
|
|
7692
|
+
})),
|
|
7693
|
+
];
|
|
7694
|
+
recordTurnStartupPhase(observability, {
|
|
7695
|
+
phase: "post_tool_preparation",
|
|
7696
|
+
provider: turnExecutionPolicy.providerId,
|
|
7697
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
7698
|
+
outcome: "completed",
|
|
7699
|
+
durationSeconds: (performance.now() - postToolPreparationStartedAt) / 1_000,
|
|
6902
7700
|
});
|
|
7701
|
+
const agent = (() => {
|
|
7702
|
+
const agentConstructionStartedAt = performance.now();
|
|
7703
|
+
let agentConstructionOutcome: "completed" | "failed" = "completed";
|
|
7704
|
+
try {
|
|
7705
|
+
return runtime.buildAgent(modelRunSettings, turnResources, {
|
|
7706
|
+
reasoningEffort: turn.reasoningEffort,
|
|
7707
|
+
latencyMode: turnExecutionPolicy.latencyMode,
|
|
7708
|
+
...(serviceTier ? { serviceTier } : {}),
|
|
7709
|
+
...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
|
|
7710
|
+
humanInputEnabled: agentHumanInputEnabled,
|
|
7711
|
+
genesisTitleHint: isGenesisTurn,
|
|
7712
|
+
persistentSessionSettings: {
|
|
7713
|
+
titleIsSet: Boolean(session.title?.trim()),
|
|
7714
|
+
},
|
|
7715
|
+
sandboxEnvironment,
|
|
7716
|
+
...(preparedTools.attemptToolCatalog
|
|
7717
|
+
? { attemptToolCatalog: preparedTools.attemptToolCatalog }
|
|
7718
|
+
: {}),
|
|
7719
|
+
...(sandboxArtifactRuntime.available ? { artifactRuntimeAvailable: true } : {}),
|
|
7720
|
+
...(cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {}),
|
|
7721
|
+
onToolCancellationFence: (fence) => {
|
|
7722
|
+
toolCancellationFenceRef.current = fence;
|
|
7723
|
+
},
|
|
7724
|
+
// TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
|
|
7725
|
+
// seed. ONLY when the effective backend is NOT selfhosted (the connected
|
|
7726
|
+
// machine uses its own git creds — mirrors the skipGitHubToken gate above)
|
|
7727
|
+
// AND the mint actually produced a token (repo resources present). The runtime
|
|
7728
|
+
// seeds it to the box's token file before the repository-clone runs; it never
|
|
7729
|
+
// touches the box/agent manifest env.
|
|
7730
|
+
...(activeSandboxBackend !== "selfhosted" && sandboxGitTokens
|
|
7731
|
+
? { gitTokenSeeds: sandboxGitTokens }
|
|
7732
|
+
: {}),
|
|
7733
|
+
...(activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings
|
|
7734
|
+
? { gitCredentialBindings: sandboxGitCredentialBindings }
|
|
7735
|
+
: {}),
|
|
7736
|
+
...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
|
|
7737
|
+
? { gitTokenSeed: sandboxGitToken }
|
|
7738
|
+
: {}),
|
|
7739
|
+
...(sandboxCodemodeToken ? { codemodeAvailable: true } : {}),
|
|
7740
|
+
// Managed boxes receive the bearer through their protected per-session
|
|
7741
|
+
// token file. Connected Machines use transient per-exec delivery above,
|
|
7742
|
+
// so they must not run the file-seeding lifecycle hook.
|
|
7743
|
+
...(activeSandboxBackend !== "selfhosted" && sandboxCodemodeToken
|
|
7744
|
+
? {
|
|
7745
|
+
codemodeTokenSeed: sandboxCodemodeToken,
|
|
7746
|
+
codemodeTokenSessionId: input.sessionId,
|
|
7747
|
+
}
|
|
7748
|
+
: {}),
|
|
7749
|
+
...(activeSandboxBackend ? { activeSandboxBackend } : {}),
|
|
7750
|
+
fileResourceDownloads,
|
|
7751
|
+
mcpServers: preparedTools.mcpServers,
|
|
7752
|
+
resolvedMcpConnectionIds: preparedTools.resolvedMcpConnectionIds,
|
|
7753
|
+
connectorActionPolicy,
|
|
7754
|
+
// LIVE by-reference connector namespaces (fills during this turn's
|
|
7755
|
+
// codex_apps tools/list): the codex tool_search description reads it per
|
|
7756
|
+
// model call so the model sees the account's real connected sources.
|
|
7757
|
+
codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
|
|
7758
|
+
// Resolved-model routing + gating (legacy defaults when null). The model
|
|
7759
|
+
// is passed as the model *string* (agent.model = runSettings.openaiModel),
|
|
7760
|
+
// NOT a Model instance: an instance only survives the in-process
|
|
7761
|
+
// ("none") run, whereas the SandboxAgent/Modal path drops it and
|
|
7762
|
+
// re-resolves the model *name* through the global MultiProviderModelProvider
|
|
7763
|
+
// configureOpenAI installed — so registry models (Fireworks GLM) route to
|
|
7764
|
+
// their own client instead of 404ing against the built-in Azure/OpenAI
|
|
7765
|
+
// client. The gating still comes from the resolved provider: server-side
|
|
7766
|
+
// store/compaction follow the provider's compaction mode (registry
|
|
7767
|
+
// providers resolve to "client"); encrypted reasoning is only
|
|
7768
|
+
// round-tripped on the Responses wire API; hosted web search is attached
|
|
7769
|
+
// whenever the provider declares it runnable and is independent of the
|
|
7770
|
+
// session's MCP allow-list; the effective context window drives the
|
|
7771
|
+
// compaction threshold.
|
|
7772
|
+
hostedWebSearch,
|
|
7773
|
+
...imageGenerationOption,
|
|
7774
|
+
...videoGenerationOption,
|
|
7775
|
+
lazyToolTransport,
|
|
7776
|
+
supportsImageInput,
|
|
7777
|
+
inputFileMediaTypes: modelInputPolicy.inputFileMediaTypes,
|
|
7778
|
+
...(resolvedModel
|
|
7779
|
+
? {
|
|
7780
|
+
encryptedReasoning:
|
|
7781
|
+
resolvedModel.provider.api === "responses" &&
|
|
7782
|
+
runSettings.openaiReasoningEncryptedContent,
|
|
7783
|
+
contextWindowTokens:
|
|
7784
|
+
resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
|
|
7785
|
+
// The ChatGPT/Codex backend rejects the SDK's HOSTED apply_patch
|
|
7786
|
+
// tool. Gateway Responses routes likewise expose ordinary function
|
|
7787
|
+
// tools, not OpenAI-hosted sandbox tools. Tell buildAgent to use
|
|
7788
|
+
// function apply_patch and wrap successful view_image results as
|
|
7789
|
+
// typed input_image content. Chat wires have no proven typed image
|
|
7790
|
+
// result transport and therefore receive no view_image tool.
|
|
7791
|
+
structuredToolTransport: structuredToolTransportForTurn(resolvedModel),
|
|
7792
|
+
// EXPLICIT computer-use tool transport, derived from the resolved provider's
|
|
7793
|
+
// authoritative wire identity (codex → function-image, chat → disabled,
|
|
7794
|
+
// responses → hosted) so the runtime never string-sniffs the model instance's
|
|
7795
|
+
// constructor name. See {@link computerToolModeForTurn}.
|
|
7796
|
+
computerToolMode: computerToolModeForTurn(resolvedModel),
|
|
7797
|
+
...(promptCacheKey ? { promptCacheKey } : {}),
|
|
7798
|
+
}
|
|
7799
|
+
: // LEGACY global-client fallback (resolveTurnModel returned null → the model
|
|
7800
|
+
// is not in the registry, served by the built-in OpenAI/Azure Responses
|
|
7801
|
+
// client). That backend has real hosted support, so pin computerToolMode to
|
|
7802
|
+
// "hosted" EXPLICITLY rather than leaving the runtime to sniff the instance.
|
|
7803
|
+
{
|
|
7804
|
+
computerToolMode: computerToolModeForTurn(null),
|
|
7805
|
+
promptCacheKey: input.sessionId,
|
|
7806
|
+
}),
|
|
7807
|
+
// Lazy computer-use seam: runtime first brings up :0 only after the model
|
|
7808
|
+
// selects a computer tool, then this hook begins the optional proof
|
|
7809
|
+
// recording. Shell/filesystem turns never invoke either operation.
|
|
7810
|
+
onComputerUseReady: async () => {
|
|
7811
|
+
if (!resolvedSandbox) {
|
|
7812
|
+
throw new Error("Computer-use display became ready without a resolved sandbox");
|
|
7813
|
+
}
|
|
7814
|
+
// This callback is the authoritative execution boundary. Record the
|
|
7815
|
+
// action before async ffmpeg startup so transport-event ordering cannot
|
|
7816
|
+
// make settlement misclassify a real computer turn as unused.
|
|
7817
|
+
didComputerUse = true;
|
|
7818
|
+
await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
|
|
7819
|
+
},
|
|
7820
|
+
onRetainableSessionImageOutput: retainSessionImageAtToolBoundary,
|
|
7821
|
+
...(runtimeSkillActivations.length > 0
|
|
7822
|
+
? { skillActivations: runtimeSkillActivations }
|
|
7823
|
+
: {}),
|
|
7824
|
+
...(!structuredWorkspacePolicyActive && workspaceAgentInstructions
|
|
7825
|
+
? { instructionsTemplate: workspaceAgentInstructions }
|
|
7826
|
+
: {}),
|
|
7827
|
+
...(workspaceGovernance ? { workspaceGovernance } : {}),
|
|
7828
|
+
...(workspaceMemory ? { workspaceMemory } : {}),
|
|
7829
|
+
goalSnapshot: turn.goalSnapshot,
|
|
7830
|
+
// Per-session persona tier (session > workspace > deployment default).
|
|
7831
|
+
// Composed system-level AFTER the workspace persona so it refines it for
|
|
7832
|
+
// this one session; absent ⇒ byte-identical to today's composition.
|
|
7833
|
+
...(session.instructions ? { sessionInstructions: session.instructions } : {}),
|
|
7834
|
+
// Exact host context captured when this turn was accepted. It is
|
|
7835
|
+
// system-level and disappears with the turn rather than entering chat.
|
|
7836
|
+
...(turn.turnInstructions || mcpAvailabilityInstructions
|
|
7837
|
+
? {
|
|
7838
|
+
turnInstructions: [turn.turnInstructions, mcpAvailabilityInstructions]
|
|
7839
|
+
.filter((value): value is string => Boolean(value))
|
|
7840
|
+
.join(" "),
|
|
7841
|
+
}
|
|
7842
|
+
: {}),
|
|
7843
|
+
...workspaceEnvironmentOption,
|
|
7844
|
+
// RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
|
|
7845
|
+
// the frozen version carries a non-empty script), and the rig credential
|
|
7846
|
+
// hooks. All absent for a rig-less turn (byte-for-byte today).
|
|
7847
|
+
...(rigVersion && rigName
|
|
7848
|
+
? {
|
|
7849
|
+
rig: { name: rigName, version: rigVersion.version },
|
|
7850
|
+
...(rigVersion.setupScript && rigVersion.setupScript.trim().length > 0
|
|
7851
|
+
? {
|
|
7852
|
+
rigSetup: {
|
|
7853
|
+
rigId: session.rigId!,
|
|
7854
|
+
versionId: rigVersion.id,
|
|
7855
|
+
rigName,
|
|
7856
|
+
script: rigVersion.setupScript,
|
|
7857
|
+
timeoutMs: runSettings.rigSetupTimeoutMs,
|
|
7858
|
+
contentHash: rigProviderImageContentHash({
|
|
7859
|
+
backend: turn.sandboxBackend,
|
|
7860
|
+
sourceImage: rigProviderImageSourceImage(
|
|
7861
|
+
logicalSandboxSettings,
|
|
7862
|
+
turn.sandboxBackend,
|
|
7863
|
+
),
|
|
7864
|
+
definition: rigVersion,
|
|
7865
|
+
}),
|
|
7866
|
+
},
|
|
7867
|
+
}
|
|
7868
|
+
: {}),
|
|
7869
|
+
...(rigVersion.credentialHooks.length > 0
|
|
7870
|
+
? { rigCredentialHookIds: rigVersion.credentialHooks }
|
|
7871
|
+
: {}),
|
|
7872
|
+
}
|
|
7873
|
+
: {}),
|
|
7874
|
+
});
|
|
7875
|
+
} catch (error) {
|
|
7876
|
+
agentConstructionOutcome = "failed";
|
|
7877
|
+
throw error;
|
|
7878
|
+
} finally {
|
|
7879
|
+
recordTurnStartupPhase(observability, {
|
|
7880
|
+
phase: "agent_construction",
|
|
7881
|
+
provider: turnExecutionPolicy.providerId,
|
|
7882
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
7883
|
+
outcome: agentConstructionOutcome,
|
|
7884
|
+
durationSeconds: (performance.now() - agentConstructionStartedAt) / 1_000,
|
|
7885
|
+
});
|
|
7886
|
+
}
|
|
7887
|
+
})();
|
|
7888
|
+
const postAgentPreparationStartedAt = performance.now();
|
|
6903
7889
|
if (modelRunSettings.sandboxBackend !== "none" && toolCancellationFenceRef.current === null) {
|
|
6904
7890
|
throw new Error(
|
|
6905
7891
|
"Sandbox agent construction did not install the mandatory turn tool cancellation fence",
|
|
@@ -6954,6 +7940,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6954
7940
|
image: runSettings.modalImageRef ?? runSettings.dockerImage,
|
|
6955
7941
|
}
|
|
6956
7942
|
: {}),
|
|
7943
|
+
// The lazy acquire must enforce the same frozen rig authority
|
|
7944
|
+
// as the eager acquire; otherwise a warm box for another rig
|
|
7945
|
+
// can bypass the shared-state conflict/rotation fence.
|
|
7946
|
+
...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
|
|
6957
7947
|
},
|
|
6958
7948
|
"turn",
|
|
6959
7949
|
lazyHolderId,
|
|
@@ -7103,6 +8093,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7103
8093
|
workspaceId: input.workspaceId,
|
|
7104
8094
|
sessionId: input.sessionId,
|
|
7105
8095
|
environment: sandboxEnvironment,
|
|
8096
|
+
...(transientCodemodeEnvironment
|
|
8097
|
+
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
8098
|
+
: {}),
|
|
7106
8099
|
workspaceMutationFence: {
|
|
7107
8100
|
accountId: input.accountId,
|
|
7108
8101
|
turnId: turn.id,
|
|
@@ -7123,6 +8116,40 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7123
8116
|
},
|
|
7124
8117
|
);
|
|
7125
8118
|
}
|
|
8119
|
+
let companyBrainContributionReceiptRecorded = false;
|
|
8120
|
+
const recordCompanyBrainContributionReceiptOnce = (): void => {
|
|
8121
|
+
if (companyBrainContributionReceiptRecorded) return;
|
|
8122
|
+
companyBrainContributionReceiptRecorded = true;
|
|
8123
|
+
try {
|
|
8124
|
+
const companyBrainContributionReceipt = buildCompanyBrainContributionReceipt({
|
|
8125
|
+
attemptId: input.attemptId,
|
|
8126
|
+
turnId: turn.id,
|
|
8127
|
+
nestedAgentDepth: session.nestedAgentDepth,
|
|
8128
|
+
memoryPromptMode,
|
|
8129
|
+
instructionPolicy: instructionPolicySnapshot,
|
|
8130
|
+
workspaceAgentInstructions,
|
|
8131
|
+
preferences: preferenceSnapshot,
|
|
8132
|
+
companyProfile: companyProfileSnapshot,
|
|
8133
|
+
companyProfileIncluded,
|
|
8134
|
+
workspaceMemory,
|
|
8135
|
+
skillActivations: runtimeSkillActivations,
|
|
8136
|
+
});
|
|
8137
|
+
recordCompanyBrainContributions(observability, companyBrainContributionReceipt);
|
|
8138
|
+
observability.info("model context contribution receipt", {
|
|
8139
|
+
attemptId: companyBrainContributionReceipt.attemptId,
|
|
8140
|
+
turnId: companyBrainContributionReceipt.turnId,
|
|
8141
|
+
sessionRole: companyBrainContributionReceipt.sessionRole,
|
|
8142
|
+
memoryPromptMode: companyBrainContributionReceipt.memoryPromptMode,
|
|
8143
|
+
instructionPolicySnapshotId:
|
|
8144
|
+
companyBrainContributionReceipt.instructionPolicySnapshotId,
|
|
8145
|
+
preferenceSnapshotId: companyBrainContributionReceipt.preferenceSnapshotId,
|
|
8146
|
+
companyProfileSnapshotId: companyBrainContributionReceipt.companyProfileSnapshotId,
|
|
8147
|
+
contributions: JSON.stringify(companyBrainContributionReceipt.contributions),
|
|
8148
|
+
});
|
|
8149
|
+
} catch {
|
|
8150
|
+
// Observability must never change model execution semantics.
|
|
8151
|
+
}
|
|
8152
|
+
};
|
|
7126
8153
|
const agentInstructions = typeof agent.instructions === "string" ? agent.instructions : "";
|
|
7127
8154
|
const compactSummarizer = compactionSummarizerFor(
|
|
7128
8155
|
agentInstructions.trim() ? agentInstructions : undefined,
|
|
@@ -7371,69 +8398,109 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7371
8398
|
return claimedResult({ status: "idle" });
|
|
7372
8399
|
}
|
|
7373
8400
|
}
|
|
8401
|
+
recordTurnStartupPhase(observability, {
|
|
8402
|
+
phase: "post_agent_preparation",
|
|
8403
|
+
provider: turnExecutionPolicy.providerId,
|
|
8404
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8405
|
+
outcome: "completed",
|
|
8406
|
+
durationSeconds: (performance.now() - postAgentPreparationStartedAt) / 1_000,
|
|
8407
|
+
});
|
|
7374
8408
|
let fileMaterializationFailures: SandboxFileDownloadFailure[] = [];
|
|
7375
8409
|
let fileDownloadsMaterializedForRun = false;
|
|
7376
8410
|
if (resolvedSandbox && setupBoxSession && fileResourceDownloads.length > 0) {
|
|
7377
|
-
const
|
|
7378
|
-
|
|
7379
|
-
// memoize successful downloads. A connected machine is user-owned: files
|
|
7380
|
-
// can be changed or removed between turns, so verify/materialize every
|
|
7381
|
-
// attached file each turn instead of trusting the managed-box cache.
|
|
8411
|
+
const fileMaterializationStartedAt = performance.now();
|
|
8412
|
+
let fileMaterializationOutcome: "completed" | "failed" = "completed";
|
|
7382
8413
|
const cacheMaterialization = resolvedSandbox.established.backendId !== "selfhosted";
|
|
7383
|
-
|
|
7384
|
-
?
|
|
7385
|
-
|
|
7386
|
-
|
|
7387
|
-
|
|
7388
|
-
|
|
7389
|
-
|
|
7390
|
-
|
|
7391
|
-
|
|
7392
|
-
|
|
7393
|
-
|
|
7394
|
-
|
|
7395
|
-
|
|
7396
|
-
|
|
7397
|
-
|
|
7398
|
-
|
|
7399
|
-
|
|
7400
|
-
|
|
7401
|
-
|
|
7402
|
-
|
|
7403
|
-
|
|
7404
|
-
downloadsToMaterialize,
|
|
7405
|
-
{
|
|
7406
|
-
onRuntimeEvent: async (event) => {
|
|
7407
|
-
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
7408
|
-
},
|
|
7409
|
-
...(runAs ? { runAs } : {}),
|
|
7410
|
-
...(toolCancellationFenceRef.current
|
|
7411
|
-
? {
|
|
7412
|
-
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
7413
|
-
toolCancellationFenceRef.current,
|
|
7414
|
-
),
|
|
7415
|
-
}
|
|
7416
|
-
: {}),
|
|
7417
|
-
},
|
|
7418
|
-
),
|
|
8414
|
+
let fileMaterializationCache: "hit" | "miss" | "disabled" = cacheMaterialization
|
|
8415
|
+
? "miss"
|
|
8416
|
+
: "disabled";
|
|
8417
|
+
try {
|
|
8418
|
+
const boxInstanceId = resolvedSandbox.established.instanceId;
|
|
8419
|
+
// Managed boxes are immutable platform state, so their durable lease can
|
|
8420
|
+
// memoize successful downloads. A connected machine is user-owned: files
|
|
8421
|
+
// can be changed or removed between turns, so verify/materialize every
|
|
8422
|
+
// attached file each turn instead of trusting the managed-box cache.
|
|
8423
|
+
const alreadyMaterialized = cacheMaterialization
|
|
8424
|
+
? await getMaterializedSandboxFileResources(db, {
|
|
8425
|
+
accountId: input.accountId,
|
|
8426
|
+
workspaceId: input.workspaceId,
|
|
8427
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
8428
|
+
expectedEpoch: resolvedSandbox.leaseEpoch,
|
|
8429
|
+
instanceId: boxInstanceId,
|
|
8430
|
+
})
|
|
8431
|
+
: new Set<string>();
|
|
8432
|
+
const downloadsToMaterialize = filterUnmaterializedSandboxFileDownloads(
|
|
8433
|
+
fileResourceDownloads,
|
|
8434
|
+
alreadyMaterialized,
|
|
7419
8435
|
);
|
|
7420
|
-
|
|
7421
|
-
|
|
7422
|
-
|
|
7423
|
-
|
|
7424
|
-
|
|
7425
|
-
|
|
7426
|
-
|
|
7427
|
-
|
|
7428
|
-
|
|
7429
|
-
|
|
7430
|
-
|
|
7431
|
-
|
|
7432
|
-
|
|
7433
|
-
|
|
8436
|
+
if (cacheMaterialization && downloadsToMaterialize.length === 0) {
|
|
8437
|
+
fileMaterializationCache = "hit";
|
|
8438
|
+
}
|
|
8439
|
+
const runAs = sandboxRunAs(runSettings);
|
|
8440
|
+
if (downloadsToMaterialize.length > 0) {
|
|
8441
|
+
const materialized = await runWorkspaceMutationForSandbox(
|
|
8442
|
+
resolvedSandbox,
|
|
8443
|
+
"fileMaterialization",
|
|
8444
|
+
async () =>
|
|
8445
|
+
await materializeSandboxFileDownloads(
|
|
8446
|
+
setupBoxSession as CodemodeTokenWriterSession,
|
|
8447
|
+
downloadsToMaterialize,
|
|
8448
|
+
{
|
|
8449
|
+
onRuntimeEvent: async (event) => {
|
|
8450
|
+
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
8451
|
+
},
|
|
8452
|
+
...(runAs ? { runAs } : {}),
|
|
8453
|
+
...(toolCancellationFenceRef.current
|
|
8454
|
+
? {
|
|
8455
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
8456
|
+
toolCancellationFenceRef.current,
|
|
8457
|
+
),
|
|
8458
|
+
}
|
|
8459
|
+
: {}),
|
|
8460
|
+
},
|
|
8461
|
+
),
|
|
8462
|
+
);
|
|
8463
|
+
fileMaterializationFailures = materialized.failures;
|
|
8464
|
+
fileMaterializationOutcome = sandboxFileMaterializationOutcome(materialized.failures);
|
|
8465
|
+
const failedFileIds = new Set(materialized.failures.map((failure) => failure.fileId));
|
|
8466
|
+
const succeededFileIds = downloadsToMaterialize
|
|
8467
|
+
.map((download) => download.fileId)
|
|
8468
|
+
.filter((fileId) => !failedFileIds.has(fileId));
|
|
8469
|
+
if (cacheMaterialization && succeededFileIds.length > 0) {
|
|
8470
|
+
await markSandboxFileResourcesMaterialized(db, {
|
|
8471
|
+
accountId: input.accountId,
|
|
8472
|
+
workspaceId: input.workspaceId,
|
|
8473
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
8474
|
+
expectedEpoch: resolvedSandbox.leaseEpoch,
|
|
8475
|
+
instanceId: boxInstanceId,
|
|
8476
|
+
fileIds: succeededFileIds,
|
|
8477
|
+
});
|
|
8478
|
+
}
|
|
7434
8479
|
}
|
|
8480
|
+
fileDownloadsMaterializedForRun = true;
|
|
8481
|
+
} catch (error) {
|
|
8482
|
+
fileMaterializationOutcome = "failed";
|
|
8483
|
+
throw error;
|
|
8484
|
+
} finally {
|
|
8485
|
+
recordTurnStartupPhase(observability, {
|
|
8486
|
+
phase: "file_materialization",
|
|
8487
|
+
provider: turnExecutionPolicy.providerId,
|
|
8488
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8489
|
+
outcome: fileMaterializationOutcome,
|
|
8490
|
+
durationSeconds: (performance.now() - fileMaterializationStartedAt) / 1_000,
|
|
8491
|
+
count: fileResourceDownloads.length,
|
|
8492
|
+
cache: fileMaterializationCache,
|
|
8493
|
+
});
|
|
7435
8494
|
}
|
|
7436
|
-
|
|
8495
|
+
} else {
|
|
8496
|
+
recordTurnStartupPhase(observability, {
|
|
8497
|
+
phase: "file_materialization",
|
|
8498
|
+
provider: turnExecutionPolicy.providerId,
|
|
8499
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8500
|
+
outcome: "completed",
|
|
8501
|
+
durationSeconds: 0,
|
|
8502
|
+
count: 0,
|
|
8503
|
+
});
|
|
7437
8504
|
}
|
|
7438
8505
|
const requiredVideoFileIds = new Set(requiredGeneratedVideoFiles.map((file) => file.fileId));
|
|
7439
8506
|
const failedRequiredVideo = fileMaterializationFailures.find((failure) =>
|
|
@@ -7478,52 +8545,110 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7478
8545
|
}
|
|
7479
8546
|
let runInput: Awaited<ReturnType<typeof turnInput>>["input"] | null = null;
|
|
7480
8547
|
const prepareRunAttemptInput = async () => {
|
|
7481
|
-
const
|
|
7482
|
-
|
|
7483
|
-
|
|
7484
|
-
|
|
7485
|
-
|
|
7486
|
-
|
|
7487
|
-
|
|
7488
|
-
|
|
7489
|
-
|
|
7490
|
-
|
|
7491
|
-
|
|
7492
|
-
|
|
7493
|
-
|
|
8548
|
+
const historyPreparationStartedAt = performance.now();
|
|
8549
|
+
let historyPreparationOutcome: "completed" | "failed" = "completed";
|
|
8550
|
+
let preparedHistoryCount: number | null = null;
|
|
8551
|
+
try {
|
|
8552
|
+
const prepared = await turnInput(db, runtime, agent, trigger, {
|
|
8553
|
+
turnId: activeTurnId,
|
|
8554
|
+
recovering: turn.executionGeneration > 1,
|
|
8555
|
+
...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
|
|
8556
|
+
...(runCredentialsNote ? { runCredentialsNote } : {}),
|
|
8557
|
+
providerApi,
|
|
8558
|
+
projectCanonicalHistory: generatedImageHistoryProjector,
|
|
8559
|
+
materializeModelHistory: materializeScreenshotHistory,
|
|
8560
|
+
materializeSerializedRunState: materializeScreenshotRunState,
|
|
8561
|
+
projectModelHistory: modelHistoryProjector,
|
|
8562
|
+
onPreparationPhase: (measurement) => {
|
|
8563
|
+
recordTurnStartupPhase(observability, {
|
|
8564
|
+
phase: `history_${measurement.phase}`,
|
|
8565
|
+
provider: turnExecutionPolicy.providerId,
|
|
8566
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8567
|
+
outcome: measurement.outcome,
|
|
8568
|
+
durationSeconds: measurement.durationSeconds,
|
|
8569
|
+
});
|
|
8570
|
+
},
|
|
8571
|
+
});
|
|
8572
|
+
const generatedImageMaterializationStartedAt = performance.now();
|
|
8573
|
+
let generatedImageMaterializationOutcome: "completed" | "failed" = "completed";
|
|
8574
|
+
try {
|
|
8575
|
+
for (const receipt of generatedImageReceiptsByArtifactId.values()) {
|
|
8576
|
+
await materializeGeneratedImage(receipt);
|
|
8577
|
+
}
|
|
8578
|
+
} catch (error) {
|
|
8579
|
+
generatedImageMaterializationOutcome = "failed";
|
|
8580
|
+
throw error;
|
|
8581
|
+
} finally {
|
|
8582
|
+
recordTurnStartupPhase(observability, {
|
|
8583
|
+
phase: "history_generated_image_materialization",
|
|
8584
|
+
provider: turnExecutionPolicy.providerId,
|
|
8585
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8586
|
+
outcome: generatedImageMaterializationOutcome,
|
|
8587
|
+
durationSeconds: (performance.now() - generatedImageMaterializationStartedAt) / 1_000,
|
|
8588
|
+
count: generatedImageReceiptsByArtifactId.size,
|
|
8589
|
+
});
|
|
8590
|
+
}
|
|
8591
|
+
runInput = prepared.input;
|
|
8592
|
+
providerArtifactCandidates = prepared.providerArtifactCandidates;
|
|
8593
|
+
// Slice index = the length of the model-facing (active) history this turn
|
|
8594
|
+
// is seeded from; new items beyond it (the trigger message + this turn's
|
|
8595
|
+
// generated items) are the ones to persist. After a compaction this is the
|
|
8596
|
+
// short [summary, ...tail] active set, NOT the total row count. The
|
|
8597
|
+
// absolute write position is tracked separately (next whole number past
|
|
8598
|
+
// the max existing position) because the fractional summary row means
|
|
8599
|
+
// total rows no longer equal max(position)+1. Pre-compaction both reduce to
|
|
8600
|
+
// the old total-count value, so the common path is unchanged.
|
|
8601
|
+
//
|
|
8602
|
+
// CRITICAL: seed from the structurally repaired active-row length, not the raw active
|
|
8603
|
+
// count. `prepareRunInput` builds `state.history` from
|
|
8604
|
+
// `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
|
|
8605
|
+
// rows (a legacy orphan/dangling pair), the in-memory history this turn
|
|
8606
|
+
// starts from is K shorter than the raw row count. The reconcile slices the
|
|
8607
|
+
// repaired `state.history` off `persistedHistoryCount`; seeding it from
|
|
8608
|
+
// the raw count (K too high) skips K genuinely-new items, and a
|
|
8609
|
+
// `function_call` left in that skipped region can later have its
|
|
8610
|
+
// `function_call_result` persisted alone — the orphan that 400s on replay
|
|
8611
|
+
// and bricks the session (issue-61). The repaired seed is already
|
|
8612
|
+
// orphan-free, so it is a stable prefix of the re-sanitized history and the
|
|
8613
|
+
// slice begins exactly at the first genuinely-new item.
|
|
8614
|
+
// prepareInput already sanitized the exact durable prefix represented
|
|
8615
|
+
// by state.history. Carry its count forward instead of loading and
|
|
8616
|
+
// retaining the full active transcript a second time beside runInput.
|
|
8617
|
+
persistedHistoryCount = prepared.persistedHistoryCount;
|
|
8618
|
+
preparedHistoryCount = prepared.persistedHistoryCount;
|
|
8619
|
+
const historyPositionStartedAt = performance.now();
|
|
8620
|
+
let historyPositionOutcome: "completed" | "failed" = "completed";
|
|
8621
|
+
try {
|
|
8622
|
+
nextHistoryPosition = await nextSessionHistoryPosition(
|
|
8623
|
+
db,
|
|
8624
|
+
input.workspaceId,
|
|
8625
|
+
input.sessionId,
|
|
8626
|
+
);
|
|
8627
|
+
} catch (error) {
|
|
8628
|
+
historyPositionOutcome = "failed";
|
|
8629
|
+
throw error;
|
|
8630
|
+
} finally {
|
|
8631
|
+
recordTurnStartupPhase(observability, {
|
|
8632
|
+
phase: "history_position_load",
|
|
8633
|
+
provider: turnExecutionPolicy.providerId,
|
|
8634
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8635
|
+
outcome: historyPositionOutcome,
|
|
8636
|
+
durationSeconds: (performance.now() - historyPositionStartedAt) / 1_000,
|
|
8637
|
+
});
|
|
8638
|
+
}
|
|
8639
|
+
} catch (error) {
|
|
8640
|
+
historyPreparationOutcome = "failed";
|
|
8641
|
+
throw error;
|
|
8642
|
+
} finally {
|
|
8643
|
+
recordTurnStartupPhase(observability, {
|
|
8644
|
+
phase: "history_preparation",
|
|
8645
|
+
provider: turnExecutionPolicy.providerId,
|
|
8646
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8647
|
+
outcome: historyPreparationOutcome,
|
|
8648
|
+
durationSeconds: (performance.now() - historyPreparationStartedAt) / 1_000,
|
|
8649
|
+
count: preparedHistoryCount,
|
|
8650
|
+
});
|
|
7494
8651
|
}
|
|
7495
|
-
runInput = prepared.input;
|
|
7496
|
-
providerArtifactCandidates = prepared.providerArtifactCandidates;
|
|
7497
|
-
// Slice index = the length of the model-facing (active) history this turn
|
|
7498
|
-
// is seeded from; new items beyond it (the trigger message + this turn's
|
|
7499
|
-
// generated items) are the ones to persist. After a compaction this is the
|
|
7500
|
-
// short [summary, ...tail] active set, NOT the total row count. The
|
|
7501
|
-
// absolute write position is tracked separately (next whole number past
|
|
7502
|
-
// the max existing position) because the fractional summary row means
|
|
7503
|
-
// total rows no longer equal max(position)+1. Pre-compaction both reduce to
|
|
7504
|
-
// the old total-count value, so the common path is unchanged.
|
|
7505
|
-
//
|
|
7506
|
-
// CRITICAL: seed from the structurally repaired active-row length, not the raw active
|
|
7507
|
-
// count. `prepareRunInput` builds `state.history` from
|
|
7508
|
-
// `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
|
|
7509
|
-
// rows (a legacy orphan/dangling pair), the in-memory history this turn
|
|
7510
|
-
// starts from is K shorter than the raw row count. The reconcile slices the
|
|
7511
|
-
// repaired `state.history` off `persistedHistoryCount`; seeding it from
|
|
7512
|
-
// the raw count (K too high) skips K genuinely-new items, and a
|
|
7513
|
-
// `function_call` left in that skipped region can later have its
|
|
7514
|
-
// `function_call_result` persisted alone — the orphan that 400s on replay
|
|
7515
|
-
// and bricks the session (issue-61). The repaired seed is already
|
|
7516
|
-
// orphan-free, so it is a stable prefix of the re-sanitized history and the
|
|
7517
|
-
// slice begins exactly at the first genuinely-new item.
|
|
7518
|
-
// prepareInput already sanitized the exact durable prefix represented
|
|
7519
|
-
// by state.history. Carry its count forward instead of loading and
|
|
7520
|
-
// retaining the full active transcript a second time beside runInput.
|
|
7521
|
-
persistedHistoryCount = prepared.persistedHistoryCount;
|
|
7522
|
-
nextHistoryPosition = await nextSessionHistoryPosition(
|
|
7523
|
-
db,
|
|
7524
|
-
input.workspaceId,
|
|
7525
|
-
input.sessionId,
|
|
7526
|
-
);
|
|
7527
8652
|
};
|
|
7528
8653
|
|
|
7529
8654
|
const forceContextCompaction = async (
|
|
@@ -7575,6 +8700,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7575
8700
|
// calling runStreamAttempt again; resetting this state there would reuse
|
|
7576
8701
|
// the first no-response-ID fallback key and suppress a real model call.
|
|
7577
8702
|
const modelResponseState = createModelResponseEventState(claimedModelUsageSourceKeys);
|
|
8703
|
+
let workerPreparationTotalRecorded = false;
|
|
7578
8704
|
const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
|
|
7579
8705
|
if (!runInput) {
|
|
7580
8706
|
throw new Error("Run input was not prepared");
|
|
@@ -7589,152 +8715,223 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7589
8715
|
let currentToolBatchCallIds = new Set<string>();
|
|
7590
8716
|
let currentToolBatchCompletedCallIds = new Set<string>();
|
|
7591
8717
|
let streamSawPerResponseUsage = false;
|
|
7592
|
-
// Actual input tokens of the most recent model response this turn; the
|
|
7593
|
-
// pre-read trigger for the NEXT turn. Persisted at every turn-end path.
|
|
7594
|
-
throwIfWorkerShuttingDown();
|
|
7595
|
-
throwIfTurnCancelled();
|
|
7596
|
-
const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
|
|
7597
|
-
const runStreamOnce = async (): ReturnType<OpenGeniRuntime["runStream"]> => {
|
|
7598
|
-
// Eager owned sessions must settle the exact platform-setup provider
|
|
7599
|
-
// promise before the long model stream starts; otherwise one admission
|
|
7600
|
-
// would remain in flight for the entire turn and suppress every
|
|
7601
|
-
// heartbeat capture. Lazy setup already runs under the same wrapper in
|
|
7602
|
-
// its first-operation provisioner above.
|
|
7603
|
-
if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
|
|
7604
|
-
const
|
|
7605
|
-
|
|
7606
|
-
|
|
7607
|
-
|
|
7608
|
-
|
|
7609
|
-
|
|
7610
|
-
|
|
7611
|
-
|
|
7612
|
-
|
|
7613
|
-
|
|
7614
|
-
|
|
7615
|
-
|
|
7616
|
-
|
|
7617
|
-
|
|
7618
|
-
|
|
7619
|
-
|
|
7620
|
-
|
|
7621
|
-
|
|
7622
|
-
|
|
7623
|
-
|
|
7624
|
-
|
|
7625
|
-
|
|
7626
|
-
|
|
7627
|
-
|
|
7628
|
-
|
|
7629
|
-
|
|
7630
|
-
|
|
7631
|
-
|
|
7632
|
-
|
|
7633
|
-
|
|
7634
|
-
|
|
7635
|
-
|
|
7636
|
-
|
|
7637
|
-
|
|
7638
|
-
|
|
7639
|
-
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
7640
|
-
toolCancellationFenceRef.current,
|
|
7641
|
-
),
|
|
8718
|
+
// Actual input tokens of the most recent model response this turn; the
|
|
8719
|
+
// pre-read trigger for the NEXT turn. Persisted at every turn-end path.
|
|
8720
|
+
throwIfWorkerShuttingDown();
|
|
8721
|
+
throwIfTurnCancelled();
|
|
8722
|
+
const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
|
|
8723
|
+
const runStreamOnce = async (): ReturnType<OpenGeniRuntime["runStream"]> => {
|
|
8724
|
+
// Eager owned sessions must settle the exact platform-setup provider
|
|
8725
|
+
// promise before the long model stream starts; otherwise one admission
|
|
8726
|
+
// would remain in flight for the entire turn and suppress every
|
|
8727
|
+
// heartbeat capture. Lazy setup already runs under the same wrapper in
|
|
8728
|
+
// its first-operation provisioner above.
|
|
8729
|
+
if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
|
|
8730
|
+
const ownedSandboxSetupStartedAt = performance.now();
|
|
8731
|
+
let ownedSandboxSetupOutcome: "completed" | "failed" = "completed";
|
|
8732
|
+
try {
|
|
8733
|
+
const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
|
|
8734
|
+
// `deferredSetup: true` below tells the runtime that the worker owns
|
|
8735
|
+
// platform setup, so the runtime intentionally skips its credential
|
|
8736
|
+
// session callback. Materialize host-managed run credentials here
|
|
8737
|
+
// before any setup command (including provider login hooks), then
|
|
8738
|
+
// decorate setup commands so they source the active generation.
|
|
8739
|
+
await attachRunCredentialRenewal(
|
|
8740
|
+
eagerSetupSession as RunCredentialCommandSession,
|
|
8741
|
+
resolvedSandbox,
|
|
8742
|
+
);
|
|
8743
|
+
const eagerCredentialSetupSession = initialRunCredentialMaterial
|
|
8744
|
+
? withRunCredentialsSession(eagerSetupSession as object, input.sessionId)
|
|
8745
|
+
: eagerSetupSession;
|
|
8746
|
+
await runWorkspaceMutationForSandbox(
|
|
8747
|
+
resolvedSandbox,
|
|
8748
|
+
"eagerOwnedSandboxSetup",
|
|
8749
|
+
async () =>
|
|
8750
|
+
await runOwnedSandboxSetup(
|
|
8751
|
+
agent,
|
|
8752
|
+
ownedEstablished.session as never,
|
|
8753
|
+
eagerCredentialSetupSession as never,
|
|
8754
|
+
{
|
|
8755
|
+
settings: runSettings,
|
|
8756
|
+
environment: sandboxEnvironment,
|
|
8757
|
+
preparedInput: runInput!,
|
|
8758
|
+
...(fileDownloadsMaterializedForRun
|
|
8759
|
+
? { fileDownloadsMaterialized: true }
|
|
8760
|
+
: {}),
|
|
8761
|
+
onRuntimeEvent: async (event) => {
|
|
8762
|
+
await renewServingCredentialLease("runtime_event");
|
|
8763
|
+
if (servingCredentialLeaseLost()) {
|
|
8764
|
+
throw new Error("Provider credential lease expired during sandbox setup");
|
|
7642
8765
|
}
|
|
7643
|
-
|
|
7644
|
-
|
|
7645
|
-
|
|
7646
|
-
|
|
7647
|
-
|
|
7648
|
-
|
|
7649
|
-
|
|
7650
|
-
|
|
8766
|
+
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
8767
|
+
},
|
|
8768
|
+
...(toolCancellationFenceRef.current
|
|
8769
|
+
? {
|
|
8770
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
8771
|
+
toolCancellationFenceRef.current,
|
|
8772
|
+
),
|
|
8773
|
+
}
|
|
8774
|
+
: {}),
|
|
8775
|
+
},
|
|
8776
|
+
),
|
|
8777
|
+
);
|
|
8778
|
+
await attachGitCredentialRenewal(
|
|
8779
|
+
eagerSetupSession as GitCredentialTokenWriterSession,
|
|
8780
|
+
initialGitCredentials,
|
|
8781
|
+
);
|
|
8782
|
+
} catch (error) {
|
|
8783
|
+
ownedSandboxSetupOutcome = "failed";
|
|
8784
|
+
throw error;
|
|
8785
|
+
} finally {
|
|
8786
|
+
recordTurnStartupPhase(observability, {
|
|
8787
|
+
phase: "owned_sandbox_setup",
|
|
8788
|
+
provider: turnExecutionPolicy.providerId,
|
|
8789
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8790
|
+
outcome: ownedSandboxSetupOutcome,
|
|
8791
|
+
durationSeconds: (performance.now() - ownedSandboxSetupStartedAt) / 1_000,
|
|
8792
|
+
count: fileResourceDownloads.length,
|
|
8793
|
+
});
|
|
8794
|
+
}
|
|
7651
8795
|
}
|
|
7652
8796
|
// Conservative request boundary: once control enters runStream, the
|
|
7653
8797
|
// provider may have accepted work even if no response/event follows.
|
|
7654
8798
|
// Escaped MCP setup timeouts are automatically recoverable only
|
|
7655
8799
|
// before this line.
|
|
7656
|
-
|
|
7657
|
-
|
|
7658
|
-
|
|
7659
|
-
|
|
7660
|
-
|
|
7661
|
-
|
|
7662
|
-
|
|
7663
|
-
|
|
7664
|
-
|
|
7665
|
-
|
|
7666
|
-
|
|
7667
|
-
|
|
7668
|
-
|
|
7669
|
-
|
|
7670
|
-
|
|
7671
|
-
|
|
7672
|
-
|
|
7673
|
-
|
|
7674
|
-
|
|
8800
|
+
recordCompanyBrainContributionReceiptOnce();
|
|
8801
|
+
const providerDispatchStartedAt = performance.now();
|
|
8802
|
+
let providerDispatchOutcome: "completed" | "failed" = "completed";
|
|
8803
|
+
try {
|
|
8804
|
+
// This histogram describes worker preparation until the first entry
|
|
8805
|
+
// into the runtime. Lazy SDK request preparation and the durable
|
|
8806
|
+
// model-request audit happen after this boundary and have their own
|
|
8807
|
+
// phase metrics. Context-compaction recovery re-enters
|
|
8808
|
+
// runStreamAttempt; recording again would fold the prior request and
|
|
8809
|
+
// compaction into a bogus second startup sample.
|
|
8810
|
+
if (!workerPreparationTotalRecorded) {
|
|
8811
|
+
workerPreparationTotalRecorded = true;
|
|
8812
|
+
recordTurnWorkerPreparationTotal(observability, {
|
|
8813
|
+
provider: turnExecutionPolicy.providerId,
|
|
8814
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8815
|
+
outcome: "completed",
|
|
8816
|
+
durationSeconds: (performance.now() - workerPreparationStartedAt) / 1_000,
|
|
8817
|
+
});
|
|
8818
|
+
}
|
|
8819
|
+
modelRequestStarted = true;
|
|
8820
|
+
return await runtime.runStream(agent, runInput!, modelRunSettings, {
|
|
8821
|
+
signal: runtimeCancellationSignal,
|
|
8822
|
+
sandboxEnvironment,
|
|
8823
|
+
onRuntimeEvent: async (event) => {
|
|
8824
|
+
await renewServingCredentialLease("runtime_event");
|
|
8825
|
+
if (servingCredentialLeaseLost()) {
|
|
8826
|
+
throw new Error("Provider credential lease expired during the active turn");
|
|
8827
|
+
}
|
|
8828
|
+
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
8829
|
+
},
|
|
8830
|
+
// P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
|
|
8831
|
+
// keystone). Absent when the flag is off -> legacy build-and-discard.
|
|
8832
|
+
...(ownedEstablished
|
|
8833
|
+
? {
|
|
8834
|
+
ownedSandbox: {
|
|
8835
|
+
client: ownedEstablished.client,
|
|
8836
|
+
session: ownedEstablished.session,
|
|
8837
|
+
...(resolvedSandbox?.established.sessionState
|
|
8838
|
+
? {
|
|
8839
|
+
sessionState: resolvedSandbox.established.sessionState,
|
|
8840
|
+
}
|
|
8841
|
+
: {}),
|
|
8842
|
+
// Pin platform setup (hooks + file materialization) to the un-proxied
|
|
8843
|
+
// established box — never through the routing proxy, which would
|
|
8844
|
+
// re-route those execs onto a machine swapped in mid-turn.
|
|
8845
|
+
...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
|
|
8846
|
+
...(fileDownloadsMaterializedForRun
|
|
8847
|
+
? { fileDownloadsMaterialized: true }
|
|
8848
|
+
: {}),
|
|
8849
|
+
// Both owned paths execute setup outside runStream: eager just
|
|
8850
|
+
// above under its exact admission, lazy in the provisioner.
|
|
8851
|
+
deferredSetup: true,
|
|
8852
|
+
},
|
|
8853
|
+
}
|
|
8854
|
+
: {}),
|
|
8855
|
+
...(activeSandboxBackend !== "selfhosted" &&
|
|
8856
|
+
sandboxCodemodeToken &&
|
|
8857
|
+
sandboxCodemodeTokenExpiresAt &&
|
|
8858
|
+
!lazyOwnedSandbox
|
|
8859
|
+
? {
|
|
8860
|
+
onCodemodeTokenSessionReady: async (
|
|
8861
|
+
tokenSession: CodemodeTokenWriterSession,
|
|
8862
|
+
) => {
|
|
8863
|
+
const renewalSession =
|
|
8864
|
+
(setupBoxSession as CodemodeTokenWriterSession | null) ?? tokenSession;
|
|
8865
|
+
await attachCodemodeTokenRenewal(renewalSession);
|
|
8866
|
+
},
|
|
8867
|
+
}
|
|
8868
|
+
: {}),
|
|
8869
|
+
...(runCredentialResolver
|
|
8870
|
+
? {
|
|
8871
|
+
runCredentialSessionId: input.sessionId,
|
|
8872
|
+
...(!lazyOwnedSandbox
|
|
7675
8873
|
? {
|
|
7676
|
-
|
|
8874
|
+
onRunCredentialSessionReady: async (
|
|
8875
|
+
credentialSession: RunCredentialCommandSession,
|
|
8876
|
+
) => {
|
|
8877
|
+
const pinnedCredentialSession = setupBoxSession
|
|
8878
|
+
? (setupBoxSession as RunCredentialCommandSession)
|
|
8879
|
+
: credentialSession;
|
|
8880
|
+
await attachRunCredentialRenewal(pinnedCredentialSession);
|
|
8881
|
+
},
|
|
7677
8882
|
}
|
|
7678
8883
|
: {}),
|
|
7679
|
-
|
|
7680
|
-
|
|
7681
|
-
|
|
7682
|
-
|
|
7683
|
-
|
|
7684
|
-
|
|
7685
|
-
|
|
7686
|
-
|
|
7687
|
-
},
|
|
7688
|
-
}
|
|
7689
|
-
: {}),
|
|
7690
|
-
...(sandboxCodemodeToken && sandboxCodemodeTokenExpiresAt && !lazyOwnedSandbox
|
|
7691
|
-
? {
|
|
7692
|
-
onCodemodeTokenSessionReady: async (tokenSession: CodemodeTokenWriterSession) => {
|
|
7693
|
-
const renewalSession =
|
|
7694
|
-
(setupBoxSession as CodemodeTokenWriterSession | null) ?? tokenSession;
|
|
7695
|
-
await attachCodemodeTokenRenewal(renewalSession);
|
|
7696
|
-
},
|
|
7697
|
-
}
|
|
7698
|
-
: {}),
|
|
7699
|
-
...(runCredentialResolver
|
|
7700
|
-
? {
|
|
7701
|
-
runCredentialSessionId: input.sessionId,
|
|
7702
|
-
...(!lazyOwnedSandbox
|
|
7703
|
-
? {
|
|
7704
|
-
onRunCredentialSessionReady: async (
|
|
7705
|
-
credentialSession: RunCredentialCommandSession,
|
|
7706
|
-
) => {
|
|
7707
|
-
const pinnedCredentialSession = setupBoxSession
|
|
7708
|
-
? (setupBoxSession as RunCredentialCommandSession)
|
|
7709
|
-
: credentialSession;
|
|
7710
|
-
await attachRunCredentialRenewal(pinnedCredentialSession);
|
|
7711
|
-
},
|
|
8884
|
+
}
|
|
8885
|
+
: {}),
|
|
8886
|
+
...(modelRunSettings.sandboxBackend !== "none"
|
|
8887
|
+
? {
|
|
8888
|
+
onSandboxSessionReady: async (sandboxSession: CodemodeTokenWriterSession) => {
|
|
8889
|
+
sdkOwnedSandboxSession = sandboxSession;
|
|
8890
|
+
for (const receipt of generatedImageReceiptsByArtifactId.values()) {
|
|
8891
|
+
await materializeGeneratedImageInOwnedSdkSession(receipt, sandboxSession);
|
|
7712
8892
|
}
|
|
7713
|
-
|
|
7714
|
-
|
|
7715
|
-
|
|
7716
|
-
|
|
7717
|
-
|
|
7718
|
-
|
|
7719
|
-
|
|
7720
|
-
|
|
7721
|
-
|
|
7722
|
-
|
|
7723
|
-
|
|
7724
|
-
|
|
7725
|
-
|
|
7726
|
-
|
|
7727
|
-
|
|
7728
|
-
|
|
7729
|
-
|
|
7730
|
-
|
|
7731
|
-
|
|
7732
|
-
|
|
8893
|
+
},
|
|
8894
|
+
}
|
|
8895
|
+
: {}),
|
|
8896
|
+
contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
|
|
8897
|
+
contextCompactionRequested: () =>
|
|
8898
|
+
isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
8899
|
+
onModelPreparationPhase: (measurement) => {
|
|
8900
|
+
recordTurnStartupPhase(observability, {
|
|
8901
|
+
phase: `model_prepare_${measurement.phase}`,
|
|
8902
|
+
provider: turnExecutionPolicy.providerId,
|
|
8903
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8904
|
+
outcome: measurement.outcome,
|
|
8905
|
+
durationSeconds: measurement.durationSeconds,
|
|
8906
|
+
...(measurement.count === undefined ? {} : { count: measurement.count }),
|
|
8907
|
+
});
|
|
8908
|
+
},
|
|
8909
|
+
...(toolCancellationFenceRef.current
|
|
8910
|
+
? {
|
|
8911
|
+
turnToolCancellationFence: toolCancellationFenceRef.current,
|
|
8912
|
+
}
|
|
8913
|
+
: {}),
|
|
8914
|
+
});
|
|
8915
|
+
} catch (error) {
|
|
8916
|
+
providerDispatchOutcome = "failed";
|
|
8917
|
+
throw error;
|
|
8918
|
+
} finally {
|
|
8919
|
+
recordTurnStartupPhase(observability, {
|
|
8920
|
+
phase: "provider_dispatch",
|
|
8921
|
+
provider: turnExecutionPolicy.providerId,
|
|
8922
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8923
|
+
outcome: providerDispatchOutcome,
|
|
8924
|
+
durationSeconds: (performance.now() - providerDispatchStartedAt) / 1_000,
|
|
8925
|
+
});
|
|
8926
|
+
}
|
|
7733
8927
|
};
|
|
7734
8928
|
if (codexLeaseLost) {
|
|
7735
8929
|
throw new Error("Codex credential lease expired before the model run");
|
|
7736
8930
|
}
|
|
7737
|
-
|
|
8931
|
+
if (xaiLeaseLost) {
|
|
8932
|
+
throw new Error("xAI credential lease expired before the model run");
|
|
8933
|
+
}
|
|
8934
|
+
stream = await withProviderRequestContext(runStreamOnce);
|
|
7738
8935
|
// Bounded provider label for the streaming SLIs — the resolved registry
|
|
7739
8936
|
// provider id (or the built-in OpenAI/Azure provider), never a raw
|
|
7740
8937
|
// user-supplied model string.
|
|
@@ -7752,11 +8949,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7752
8949
|
},
|
|
7753
8950
|
);
|
|
7754
8951
|
|
|
8952
|
+
const streamBootstrapStartedAt = performance.now();
|
|
8953
|
+
let streamBootstrapRecorded = false;
|
|
8954
|
+
const recordStreamBootstrap = (): void => {
|
|
8955
|
+
if (streamBootstrapRecorded) return;
|
|
8956
|
+
streamBootstrapRecorded = true;
|
|
8957
|
+
recordTurnStartupPhase(observability, {
|
|
8958
|
+
phase: "stream_bootstrap",
|
|
8959
|
+
provider: turnExecutionPolicy.providerId,
|
|
8960
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8961
|
+
outcome: "completed",
|
|
8962
|
+
durationSeconds: (performance.now() - streamBootstrapStartedAt) / 1_000,
|
|
8963
|
+
count: turnTools.length,
|
|
8964
|
+
});
|
|
8965
|
+
};
|
|
8966
|
+
if (!firstModelRequestPreparationRecorded) {
|
|
8967
|
+
firstModelRequestPreparationStartedAt = performance.now();
|
|
8968
|
+
firstModelRequestCheckpointAt = firstModelRequestPreparationStartedAt;
|
|
8969
|
+
}
|
|
7755
8970
|
const iterator = stream.toStream()[Symbol.asyncIterator]();
|
|
7756
8971
|
let streamDone = false;
|
|
7757
8972
|
try {
|
|
7758
8973
|
while (true) {
|
|
7759
8974
|
const next = await nextStreamEvent(iterator, activityContext);
|
|
8975
|
+
recordStreamBootstrap();
|
|
7760
8976
|
if (next.done) {
|
|
7761
8977
|
streamDone = true;
|
|
7762
8978
|
break;
|
|
@@ -7764,6 +8980,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7764
8980
|
let stableToolCallIdsToClear: string[] | null = null;
|
|
7765
8981
|
let completedCurrentToolBatch = false;
|
|
7766
8982
|
let retainedScreenshotMetadata: RetainedArtifactMetadata | null = null;
|
|
8983
|
+
let normalizedSdkEvents: ReturnType<typeof normalizeSdkEvent> | null = null;
|
|
7767
8984
|
const generatedImage = generatedImageFromSdkEvent(next.value);
|
|
7768
8985
|
if (isCompletedGeneratedImageSdkEvent(next.value) && !generatedImage) {
|
|
7769
8986
|
throw new Error(
|
|
@@ -7795,9 +9012,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7795
9012
|
servingCredentialId: effectiveCodexCredentialId,
|
|
7796
9013
|
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
7797
9014
|
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
7798
|
-
renewLease: () =>
|
|
7799
|
-
leaseLost:
|
|
7800
|
-
leaseLostMessage: "
|
|
9015
|
+
renewLease: () => renewServingCredentialLease("model_usage"),
|
|
9016
|
+
leaseLost: servingCredentialLeaseLost,
|
|
9017
|
+
leaseLostMessage: "Provider credential lease expired during the active turn",
|
|
7801
9018
|
setLastInputTokens: setLastInputTokensFenced,
|
|
7802
9019
|
});
|
|
7803
9020
|
assertModelResponseLatencyMode({
|
|
@@ -7821,6 +9038,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7821
9038
|
currentToolBatchCallIds = new Set<string>();
|
|
7822
9039
|
currentToolBatchCompletedCallIds = new Set<string>();
|
|
7823
9040
|
await reconcileConversationTruth();
|
|
9041
|
+
turnLifecycleMetricsFor(observability).progress(turnId!);
|
|
7824
9042
|
modelCheckpointMemoryCollector.schedule(observability);
|
|
7825
9043
|
try {
|
|
7826
9044
|
await ensureRunAllowed(
|
|
@@ -7936,6 +9154,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7936
9154
|
retainedScreenshotMetadata,
|
|
7937
9155
|
);
|
|
7938
9156
|
}
|
|
9157
|
+
normalizedSdkEvents = normalizeSdkEvent(
|
|
9158
|
+
durableSdkEvent as typeof next.value,
|
|
9159
|
+
retainedScreenshotMetadata
|
|
9160
|
+
? {
|
|
9161
|
+
toolOutputOverride: retainedScreenshotMetadata,
|
|
9162
|
+
retainedOutputEvidence: retainedScreenshotMetadata.available
|
|
9163
|
+
? retainedScreenshotMetadata
|
|
9164
|
+
: {
|
|
9165
|
+
available: false,
|
|
9166
|
+
reason: retainedScreenshotMetadata.reason,
|
|
9167
|
+
},
|
|
9168
|
+
}
|
|
9169
|
+
: {},
|
|
9170
|
+
);
|
|
9171
|
+
const normalizedToolOutput = normalizedSdkEvents.find(
|
|
9172
|
+
(event) =>
|
|
9173
|
+
event.type === "agent.toolCall.output" &&
|
|
9174
|
+
(event.payload as { id?: unknown }).id === completedToolCall.callId,
|
|
9175
|
+
)?.payload as { output?: unknown } | undefined;
|
|
9176
|
+
if (!normalizedToolOutput || !Object.hasOwn(normalizedToolOutput, "output")) {
|
|
9177
|
+
throw new Error(
|
|
9178
|
+
`Completed SDK tool call ${completedToolCall.callId} produced no durable output projection`,
|
|
9179
|
+
);
|
|
9180
|
+
}
|
|
7939
9181
|
const durableResultItem = retainedScreenshotMetadata
|
|
7940
9182
|
? (compactRetainedScreenshotHistory(
|
|
7941
9183
|
[completedToolCall.resultItem],
|
|
@@ -7957,6 +9199,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7957
9199
|
callId: completedToolCall.callId,
|
|
7958
9200
|
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
7959
9201
|
resultItem: durableResultItem as Record<string, unknown>,
|
|
9202
|
+
eventOutput: normalizedToolOutput.output,
|
|
7960
9203
|
...(videoGenerationAcceptancesByCallId.has(completedToolCall.callId)
|
|
7961
9204
|
? {
|
|
7962
9205
|
videoGenerationAcceptance: videoGenerationAcceptancesByCallId.get(
|
|
@@ -8011,24 +9254,31 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8011
9254
|
completedCurrentToolBatch = currentBatchIsStable;
|
|
8012
9255
|
}
|
|
8013
9256
|
}
|
|
8014
|
-
const normalized =
|
|
8015
|
-
|
|
8016
|
-
|
|
8017
|
-
|
|
8018
|
-
|
|
8019
|
-
|
|
8020
|
-
|
|
8021
|
-
:
|
|
8022
|
-
|
|
8023
|
-
|
|
8024
|
-
|
|
8025
|
-
|
|
8026
|
-
|
|
8027
|
-
|
|
9257
|
+
const normalized =
|
|
9258
|
+
normalizedSdkEvents ??
|
|
9259
|
+
normalizeSdkEvent(
|
|
9260
|
+
durableSdkEvent as typeof next.value,
|
|
9261
|
+
retainedScreenshotMetadata
|
|
9262
|
+
? {
|
|
9263
|
+
toolOutputOverride: retainedScreenshotMetadata,
|
|
9264
|
+
retainedOutputEvidence: retainedScreenshotMetadata.available
|
|
9265
|
+
? retainedScreenshotMetadata
|
|
9266
|
+
: {
|
|
9267
|
+
available: false,
|
|
9268
|
+
reason: retainedScreenshotMetadata.reason,
|
|
9269
|
+
},
|
|
9270
|
+
}
|
|
9271
|
+
: {},
|
|
9272
|
+
);
|
|
8028
9273
|
for (const event of normalized) {
|
|
8029
9274
|
streamTiming.onEvent(event.type);
|
|
8030
9275
|
await batcher.push(event);
|
|
8031
9276
|
}
|
|
9277
|
+
// Structural tool-output events await their durable append before
|
|
9278
|
+
// push returns. The complete result is now retained in the event
|
|
9279
|
+
// row and pending-call recovery receipt, so release only our
|
|
9280
|
+
// duplicate live SDK marker before a long turn accumulates it.
|
|
9281
|
+
releaseMcpResultCustomDataFromSdkEvent(durableSdkEvent);
|
|
8032
9282
|
if (stableToolCallIdsToClear) {
|
|
8033
9283
|
const cleared = await clearDurablePendingSessionToolCalls(db, {
|
|
8034
9284
|
accountId: input.accountId,
|
|
@@ -8091,9 +9341,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8091
9341
|
});
|
|
8092
9342
|
let aggregateAuthoritative = false;
|
|
8093
9343
|
await recordCompletedModelCallBeforeOwnershipFences({
|
|
8094
|
-
renewLease: () =>
|
|
8095
|
-
leaseLost:
|
|
8096
|
-
leaseLostMessage: "
|
|
9344
|
+
renewLease: () => renewServingCredentialLease("model_usage"),
|
|
9345
|
+
leaseLost: servingCredentialLeaseLost,
|
|
9346
|
+
leaseLostMessage: "Provider credential lease expired during the active turn",
|
|
8097
9347
|
recordUsage: async () => {
|
|
8098
9348
|
const billing = await recordModelUsageAndDebitCredits(settings, db, {
|
|
8099
9349
|
accountId: input.accountId,
|
|
@@ -8164,6 +9414,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8164
9414
|
const approvals = runtime.serializeApprovals(stream.interruptions);
|
|
8165
9415
|
const humanInputInterruptions =
|
|
8166
9416
|
runtime.serializeHumanInputRequests?.(stream.interruptions) ?? [];
|
|
9417
|
+
const interactionInterventionInterruptions =
|
|
9418
|
+
runtime.serializeInteractionInterventionRequests?.(stream.interruptions) ?? [];
|
|
8167
9419
|
const latestWorkspace = await getWorkspace(db, input.workspaceId);
|
|
8168
9420
|
if (!latestWorkspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
|
|
8169
9421
|
assertWorkspaceHumanInputAllowed(
|
|
@@ -8215,6 +9467,26 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8215
9467
|
},
|
|
8216
9468
|
},
|
|
8217
9469
|
}));
|
|
9470
|
+
const interactionInterventionRequests = interactionInterventionInterruptions.map(
|
|
9471
|
+
(interruption) => ({
|
|
9472
|
+
id: stableInteractionInterventionId(
|
|
9473
|
+
input.sessionId,
|
|
9474
|
+
activeTurnId,
|
|
9475
|
+
interruption.toolCallId,
|
|
9476
|
+
),
|
|
9477
|
+
operationId: stableInteractionInterventionOperationId(
|
|
9478
|
+
input.sessionId,
|
|
9479
|
+
activeTurnId,
|
|
9480
|
+
interruption.toolCallId,
|
|
9481
|
+
),
|
|
9482
|
+
toolCallId: interruption.toolCallId,
|
|
9483
|
+
input: interruption.input,
|
|
9484
|
+
}),
|
|
9485
|
+
);
|
|
9486
|
+
const pendingApprovals = [
|
|
9487
|
+
...approvals,
|
|
9488
|
+
...interactionInterventionInterruptions.map((interruption) => interruption.approval),
|
|
9489
|
+
];
|
|
8218
9490
|
if (
|
|
8219
9491
|
!(await settle!({
|
|
8220
9492
|
events: [
|
|
@@ -8236,11 +9508,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8236
9508
|
sessionStatus: "requires_action",
|
|
8237
9509
|
activeTurnId,
|
|
8238
9510
|
runState: {
|
|
8239
|
-
serializedRunState:
|
|
8240
|
-
pendingApprovals
|
|
9511
|
+
serializedRunState: compactApprovalRunState(stream.state.toString()),
|
|
9512
|
+
pendingApprovals,
|
|
8241
9513
|
humanInputRequests: humanInputRequests.map(
|
|
8242
9514
|
({ isNew: _isNew, ...request }) => request,
|
|
8243
9515
|
),
|
|
9516
|
+
interactionInterventionRequests,
|
|
8244
9517
|
},
|
|
8245
9518
|
}))
|
|
8246
9519
|
) {
|
|
@@ -8798,6 +10071,29 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8798
10071
|
) {
|
|
8799
10072
|
return await settleLostCodexAttempt(turnId, codexLeaseHolderId, codexLeaseGeneration);
|
|
8800
10073
|
}
|
|
10074
|
+
if (xaiLeaseLost && isXaiTurn && publish && turnId && turnStartedPublished) {
|
|
10075
|
+
await flushRuntimeBatcher();
|
|
10076
|
+
await reconcileConversationTruth({ requireDurable: true });
|
|
10077
|
+
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
10078
|
+
sessionId: input.sessionId,
|
|
10079
|
+
turnId,
|
|
10080
|
+
triggerEventId: triggerEventId!,
|
|
10081
|
+
attemptId: input.attemptId,
|
|
10082
|
+
reason: "xai_lease_lost",
|
|
10083
|
+
detail: { provider: "supergrok-subscription" },
|
|
10084
|
+
});
|
|
10085
|
+
if (recovery.action === "stale") {
|
|
10086
|
+
acknowledgeLostAttemptOwnership();
|
|
10087
|
+
activityStatus = "cancelled";
|
|
10088
|
+
turnMetricOutcome = "cancelled";
|
|
10089
|
+
return claimedResult({ status: "cancelled" });
|
|
10090
|
+
}
|
|
10091
|
+
acknowledgeRecoveryQuiescence();
|
|
10092
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
10093
|
+
activityStatus = "recovering";
|
|
10094
|
+
turnMetricOutcome = "recovering";
|
|
10095
|
+
return claimedResult({ status: "recovering" });
|
|
10096
|
+
}
|
|
8801
10097
|
// Definitive Codex credential/account refusals are the only provider
|
|
8802
10098
|
// errors that may walk the pool. This is an explicit checkpoint + SAME
|
|
8803
10099
|
// turn recovery, never an SDK/Temporal blind retry. A network break,
|
|
@@ -8981,6 +10277,143 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8981
10277
|
}
|
|
8982
10278
|
}
|
|
8983
10279
|
}
|
|
10280
|
+
const xaiCredentialFailure =
|
|
10281
|
+
isXaiTurn && effectiveXaiCredentialId ? classifyXaiCredentialFailure(error) : null;
|
|
10282
|
+
if (
|
|
10283
|
+
xaiCredentialFailure &&
|
|
10284
|
+
effectiveXaiCredentialId &&
|
|
10285
|
+
xaiAuthoritySnapshot &&
|
|
10286
|
+
xaiLeaseSubjectId &&
|
|
10287
|
+
xaiLeaseHolderId &&
|
|
10288
|
+
xaiLeaseGeneration !== null &&
|
|
10289
|
+
publish &&
|
|
10290
|
+
turnId &&
|
|
10291
|
+
turnStartedPublished
|
|
10292
|
+
) {
|
|
10293
|
+
await flushRuntimeBatcher();
|
|
10294
|
+
await reconcileConversationTruth({ requireDurable: true });
|
|
10295
|
+
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
10296
|
+
const activeGoal = goal?.status === "active" ? goal : null;
|
|
10297
|
+
const now = new Date();
|
|
10298
|
+
const cooldownUntil =
|
|
10299
|
+
xaiCredentialFailure.kind === "rate_limit"
|
|
10300
|
+
? new Date(now.getTime() + Math.max(1, xaiCredentialFailure.cooldownMs ?? 60_000))
|
|
10301
|
+
: null;
|
|
10302
|
+
const failurePayload = {
|
|
10303
|
+
error:
|
|
10304
|
+
xaiCredentialFailure.kind === "auth"
|
|
10305
|
+
? "The serving SuperGrok account requires reconnection"
|
|
10306
|
+
: xaiCredentialFailure.kind === "forbidden"
|
|
10307
|
+
? "The serving SuperGrok account is not authorized for this request"
|
|
10308
|
+
: "The serving SuperGrok account is temporarily rate limited",
|
|
10309
|
+
code:
|
|
10310
|
+
xaiCredentialFailure.kind === "auth"
|
|
10311
|
+
? "xai_relogin_required"
|
|
10312
|
+
: xaiCredentialFailure.kind === "forbidden"
|
|
10313
|
+
? "xai_account_forbidden"
|
|
10314
|
+
: "xai_account_rate_limited",
|
|
10315
|
+
detail: "the same accepted turn is waiting for another eligible account",
|
|
10316
|
+
};
|
|
10317
|
+
const armed = await armXaiCapacityWait(db, {
|
|
10318
|
+
accountId: input.accountId,
|
|
10319
|
+
workspaceId: input.workspaceId,
|
|
10320
|
+
subjectId: xaiLeaseSubjectId,
|
|
10321
|
+
sessionId: input.sessionId,
|
|
10322
|
+
turnId,
|
|
10323
|
+
attemptId: input.attemptId,
|
|
10324
|
+
workflowId: input.workflowId,
|
|
10325
|
+
authoritySnapshot: xaiAuthoritySnapshot,
|
|
10326
|
+
goalId: activeGoal?.id ?? null,
|
|
10327
|
+
goalVersion: activeGoal?.version ?? null,
|
|
10328
|
+
earliestResetAt: cooldownUntil,
|
|
10329
|
+
failurePayload,
|
|
10330
|
+
leaseFence: {
|
|
10331
|
+
holderId: xaiLeaseHolderId,
|
|
10332
|
+
generation: xaiLeaseGeneration,
|
|
10333
|
+
},
|
|
10334
|
+
credentialQuarantine:
|
|
10335
|
+
xaiCredentialFailure.kind === "auth"
|
|
10336
|
+
? {
|
|
10337
|
+
kind: "status",
|
|
10338
|
+
status: "needs_relogin",
|
|
10339
|
+
lastError: "model request remained unauthorized after refresh",
|
|
10340
|
+
}
|
|
10341
|
+
: xaiCredentialFailure.kind === "forbidden"
|
|
10342
|
+
? {
|
|
10343
|
+
kind: "status",
|
|
10344
|
+
status: "error",
|
|
10345
|
+
lastError: "model request was forbidden for this credential",
|
|
10346
|
+
}
|
|
10347
|
+
: { kind: "cooldown", until: cooldownUntil! },
|
|
10348
|
+
now,
|
|
10349
|
+
});
|
|
10350
|
+
if (armed.action === "waiting") {
|
|
10351
|
+
xaiLeaseHeld = false;
|
|
10352
|
+
xaiCredentialQuarantined = true;
|
|
10353
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, armed.events);
|
|
10354
|
+
const evaluated = await reconcileXaiCapacityWait(db, {
|
|
10355
|
+
accountId: input.accountId,
|
|
10356
|
+
workspaceId: input.workspaceId,
|
|
10357
|
+
sessionId: input.sessionId,
|
|
10358
|
+
waiterId: armed.waiter.id,
|
|
10359
|
+
generation: armed.waiter.generation,
|
|
10360
|
+
now,
|
|
10361
|
+
});
|
|
10362
|
+
if (evaluated.events.length > 0) {
|
|
10363
|
+
await publishDurableSessionEvents(
|
|
10364
|
+
bus,
|
|
10365
|
+
input.workspaceId,
|
|
10366
|
+
input.sessionId,
|
|
10367
|
+
evaluated.events,
|
|
10368
|
+
);
|
|
10369
|
+
}
|
|
10370
|
+
activityError = error;
|
|
10371
|
+
if (evaluated.action === "resumed") {
|
|
10372
|
+
activityStatus = "recovering";
|
|
10373
|
+
turnMetricOutcome = "recovering";
|
|
10374
|
+
return claimedResult({ status: "recovering" });
|
|
10375
|
+
}
|
|
10376
|
+
if (evaluated.action === "waiting") {
|
|
10377
|
+
activityStatus = "waiting_capacity";
|
|
10378
|
+
turnMetricOutcome = "recovering";
|
|
10379
|
+
return claimedResult({
|
|
10380
|
+
status: "waiting_capacity",
|
|
10381
|
+
capacityWait: {
|
|
10382
|
+
provider: "xai",
|
|
10383
|
+
waiterId: evaluated.waiter.id,
|
|
10384
|
+
generation: evaluated.waiter.generation,
|
|
10385
|
+
nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
|
|
10386
|
+
wakeRevision: evaluated.waiter.wakeRevision,
|
|
10387
|
+
},
|
|
10388
|
+
});
|
|
10389
|
+
}
|
|
10390
|
+
acknowledgeLostAttemptOwnership();
|
|
10391
|
+
activityStatus = "cancelled";
|
|
10392
|
+
turnMetricOutcome = "cancelled";
|
|
10393
|
+
return claimedResult({ status: "cancelled" });
|
|
10394
|
+
}
|
|
10395
|
+
|
|
10396
|
+
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
10397
|
+
sessionId: input.sessionId,
|
|
10398
|
+
turnId,
|
|
10399
|
+
triggerEventId: triggerEventId!,
|
|
10400
|
+
attemptId: input.attemptId,
|
|
10401
|
+
reason: "xai_credential_recheck",
|
|
10402
|
+
detail: failurePayload,
|
|
10403
|
+
});
|
|
10404
|
+
if (recovery.action === "stale") {
|
|
10405
|
+
acknowledgeLostAttemptOwnership();
|
|
10406
|
+
activityStatus = "cancelled";
|
|
10407
|
+
turnMetricOutcome = "cancelled";
|
|
10408
|
+
return claimedResult({ status: "cancelled" });
|
|
10409
|
+
}
|
|
10410
|
+
acknowledgeRecoveryQuiescence();
|
|
10411
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
10412
|
+
activityStatus = "recovering";
|
|
10413
|
+
turnMetricOutcome = "recovering";
|
|
10414
|
+
activityError = error;
|
|
10415
|
+
return claimedResult({ status: "recovering" });
|
|
10416
|
+
}
|
|
8984
10417
|
// A ChatGPT/Codex usage cap (429 usage_limit_reached) is account state,
|
|
8985
10418
|
// NOT an agent failure: surface the precise, actionable message (so the
|
|
8986
10419
|
// user sees the reset window) but idle the session — never go terminal,
|
|
@@ -9501,7 +10934,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
9501
10934
|
}
|
|
9502
10935
|
activityStatus = "failed";
|
|
9503
10936
|
activityError = error;
|
|
9504
|
-
if (!
|
|
10937
|
+
if (!turnId) {
|
|
10938
|
+
throw preClaimAdmissionFailure(error);
|
|
10939
|
+
}
|
|
10940
|
+
if (!publish || !turnStartedPublished) {
|
|
9505
10941
|
throw error;
|
|
9506
10942
|
}
|
|
9507
10943
|
// A partial/malformed stream may have emitted assistant/tool items (and
|
|
@@ -9792,6 +11228,57 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
9792
11228
|
);
|
|
9793
11229
|
codexLeaseHeld = false;
|
|
9794
11230
|
}
|
|
11231
|
+
if (
|
|
11232
|
+
effectiveXaiCredentialId &&
|
|
11233
|
+
xaiLeaseSubjectId &&
|
|
11234
|
+
xaiRequestContext &&
|
|
11235
|
+
!xaiCredentialQuarantined
|
|
11236
|
+
) {
|
|
11237
|
+
const quota = await waitForTurnFinalizerStep(
|
|
11238
|
+
fetchXaiSubscriptionQuota({ context: xaiRequestContext }).catch(() => null),
|
|
11239
|
+
finalizerSignal,
|
|
11240
|
+
);
|
|
11241
|
+
if (quota) {
|
|
11242
|
+
await waitForTurnFinalizerStep(
|
|
11243
|
+
updateXaiQuotaMetadata(db, {
|
|
11244
|
+
workspaceId: input.workspaceId,
|
|
11245
|
+
subjectId: xaiLeaseSubjectId,
|
|
11246
|
+
credentialId: effectiveXaiCredentialId,
|
|
11247
|
+
quotaUsedPercent: quota.usedPercent,
|
|
11248
|
+
quotaResetAt: quota.period?.end ?? null,
|
|
11249
|
+
quotaCheckedAt: quota.checkedAt,
|
|
11250
|
+
exhaustedUntil:
|
|
11251
|
+
quota.usedPercent !== null && quota.usedPercent >= 100
|
|
11252
|
+
? (quota.period?.end ?? null)
|
|
11253
|
+
: null,
|
|
11254
|
+
}).catch(() => undefined),
|
|
11255
|
+
finalizerSignal,
|
|
11256
|
+
);
|
|
11257
|
+
}
|
|
11258
|
+
}
|
|
11259
|
+
if (xaiLeaseHeartbeatTimer) {
|
|
11260
|
+
clearInterval(xaiLeaseHeartbeatTimer);
|
|
11261
|
+
xaiLeaseHeartbeatTimer = undefined;
|
|
11262
|
+
}
|
|
11263
|
+
if (
|
|
11264
|
+
xaiLeaseHeld &&
|
|
11265
|
+
turnId &&
|
|
11266
|
+
xaiLeaseSubjectId &&
|
|
11267
|
+
xaiLeaseHolderId &&
|
|
11268
|
+
xaiLeaseGeneration !== null
|
|
11269
|
+
) {
|
|
11270
|
+
await waitForTurnFinalizerStep(
|
|
11271
|
+
releaseXaiCredentialLease(db, {
|
|
11272
|
+
workspaceId: input.workspaceId,
|
|
11273
|
+
subjectId: xaiLeaseSubjectId,
|
|
11274
|
+
turnId,
|
|
11275
|
+
holderId: xaiLeaseHolderId,
|
|
11276
|
+
generation: xaiLeaseGeneration,
|
|
11277
|
+
}).catch(() => undefined),
|
|
11278
|
+
finalizerSignal,
|
|
11279
|
+
);
|
|
11280
|
+
xaiLeaseHeld = false;
|
|
11281
|
+
}
|
|
9795
11282
|
// Workbench v2 turn-end workspace capture — runs FIRST in
|
|
9796
11283
|
// the turn-end finally, while the box is MAXIMALLY ALIVE. The agent's last
|
|
9797
11284
|
// tool ran before this finally, so /workspace is already final; capture is
|
|
@@ -10057,6 +11544,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
10057
11544
|
? safeErrorForTelemetry(finalizationError ?? activityError)
|
|
10058
11545
|
: undefined,
|
|
10059
11546
|
});
|
|
11547
|
+
// This timer runs only after the activity promise and its full turn
|
|
11548
|
+
// stack unwind. Checkpoint collection alone cannot reclaim objects that
|
|
11549
|
+
// remain strongly reachable until this terminal boundary.
|
|
11550
|
+
turnCompletionMemoryCollector.schedule(observability);
|
|
10060
11551
|
assertPhysicalToolQuiescenceForCancellation({
|
|
10061
11552
|
acknowledgeQuiescence,
|
|
10062
11553
|
physicalToolQuiescenceConfirmed,
|
|
@@ -10169,6 +11660,51 @@ export function isTransientProviderError(error: unknown): boolean {
|
|
|
10169
11660
|
);
|
|
10170
11661
|
}
|
|
10171
11662
|
|
|
11663
|
+
export type XaiCredentialFailure = {
|
|
11664
|
+
kind: "auth" | "forbidden" | "rate_limit";
|
|
11665
|
+
cooldownMs: number | null;
|
|
11666
|
+
};
|
|
11667
|
+
|
|
11668
|
+
/**
|
|
11669
|
+
* Only definitive SuperGrok account refusals may move the same logical turn to
|
|
11670
|
+
* another credential. The marked transport response proves inference was
|
|
11671
|
+
* rejected before a stream began; refresh relogin is equally definitive.
|
|
11672
|
+
*/
|
|
11673
|
+
export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailure | null {
|
|
11674
|
+
let relogin: unknown = error;
|
|
11675
|
+
for (let depth = 0; depth < 6 && relogin && typeof relogin === "object"; depth += 1) {
|
|
11676
|
+
if (relogin instanceof XaiSubscriptionReloginRequired) {
|
|
11677
|
+
return { kind: "auth", cooldownMs: null };
|
|
11678
|
+
}
|
|
11679
|
+
relogin = (relogin as Record<string, unknown>).cause;
|
|
11680
|
+
}
|
|
11681
|
+
if (!isXaiSubscriptionTransportError(error)) return null;
|
|
11682
|
+
let current: unknown = error;
|
|
11683
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
11684
|
+
const value = current as Record<string, unknown>;
|
|
11685
|
+
const body =
|
|
11686
|
+
value.error && typeof value.error === "object"
|
|
11687
|
+
? (value.error as Record<string, unknown>)
|
|
11688
|
+
: null;
|
|
11689
|
+
const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
|
|
11690
|
+
const code = String(value.code ?? body?.code ?? "").toLowerCase();
|
|
11691
|
+
if (status === 401 || code === "unauthorized" || code === "invalid_token") {
|
|
11692
|
+
return { kind: "auth", cooldownMs: null };
|
|
11693
|
+
}
|
|
11694
|
+
if (status === 403) {
|
|
11695
|
+
return { kind: "forbidden", cooldownMs: null };
|
|
11696
|
+
}
|
|
11697
|
+
if (status === 429 || code === "rate_limit_exceeded" || code === "too_many_requests") {
|
|
11698
|
+
return {
|
|
11699
|
+
kind: "rate_limit",
|
|
11700
|
+
cooldownMs: providerRetryAfterMs(error) ?? PROVIDER_BACKPRESSURE_DELAY_MS,
|
|
11701
|
+
};
|
|
11702
|
+
}
|
|
11703
|
+
current = value.cause;
|
|
11704
|
+
}
|
|
11705
|
+
return null;
|
|
11706
|
+
}
|
|
11707
|
+
|
|
10172
11708
|
export function agentRunFailurePayload(
|
|
10173
11709
|
error: unknown,
|
|
10174
11710
|
options: { isCodexTurn?: boolean } = {},
|