@opengeni/worker-bundle 0.17.2 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +12 -0
- package/dist/activities/agent-turn.d.ts +25 -0
- package/dist/activities/browser-state-artifact-reaper.d.ts +29 -0
- package/dist/activities/capabilities.d.ts +6 -0
- package/dist/activities/codex-capacity.d.ts +1 -0
- package/dist/activities/environment.d.ts +7 -1
- package/dist/activities/packs.d.ts +8 -10
- package/dist/activities/session-state.d.ts +4 -2
- package/dist/activities/site-auth-maintenance.d.ts +27 -0
- package/dist/activities/types.d.ts +16 -0
- package/dist/activities/video-generation-admission.d.ts +9 -4
- package/dist/activities/video-generation-credential.d.ts +17 -0
- package/dist/activities/video-reference-staging.d.ts +7 -1
- package/dist/activities/xai-auth.d.ts +23 -0
- package/dist/activities/xai-image-generation.d.ts +26 -0
- package/dist/activities/xai-video-generation.d.ts +18 -0
- package/dist/{activities-control-44KBWLSM.js → activities-control-XURE4VKN.js} +387 -51
- package/dist/activities-control-XURE4VKN.js.map +1 -0
- package/dist/activities-control.d.ts +8 -0
- package/dist/{activities-turn-K4CEDEGJ.js → activities-turn-6EBXGGLS.js} +5312 -3951
- package/dist/activities-turn-6EBXGGLS.js.map +1 -0
- package/dist/activities.d.ts +9 -1
- package/dist/{chunk-KKCRGAGZ.js → chunk-5GRJIP5H.js} +99 -98
- package/dist/chunk-5GRJIP5H.js.map +1 -0
- package/dist/{chunk-E3RI3I33.js → chunk-TYNPCG4U.js} +88 -10
- package/dist/chunk-TYNPCG4U.js.map +1 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.js +49 -2
- package/dist/index.js.map +1 -1
- package/dist/model-checkpoint-memory-collector.d.ts +1 -0
- package/dist/model-context-contributions.d.ts +40 -0
- package/dist/observability-metrics.d.ts +20 -1
- package/dist/sandbox-routing.d.ts +4 -0
- package/dist/workflow-bundle.js +59 -15
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/site-auth-maintenance.d.ts +2 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +16 -15
- package/src/activities/agent-run-admission.ts +76 -0
- package/src/activities/agent-turn.ts +1178 -142
- package/src/activities/browser-state-artifact-reaper.ts +120 -0
- package/src/activities/capabilities.ts +10 -0
- package/src/activities/codex-capacity.ts +48 -1
- package/src/activities/environment.ts +14 -8
- package/src/activities/goals.ts +2 -1
- package/src/activities/packs.ts +45 -122
- package/src/activities/parent-wake.ts +13 -1
- package/src/activities/scheduled-tasks.ts +30 -73
- package/src/activities/session-state.ts +18 -0
- package/src/activities/site-auth-maintenance.ts +269 -0
- package/src/activities/types.ts +17 -0
- package/src/activities/video-generation-admission.ts +37 -4
- package/src/activities/video-generation-credential.ts +84 -6
- package/src/activities/video-generation-reconciliation.ts +212 -19
- package/src/activities/video-media-validation.ts +8 -2
- package/src/activities/video-reference-staging.ts +121 -34
- package/src/activities/xai-auth.ts +122 -0
- package/src/activities/xai-image-generation.ts +86 -0
- package/src/activities/xai-video-generation.ts +69 -0
- package/src/activities-control.ts +4 -0
- package/src/activities.ts +5 -0
- package/src/index.ts +51 -0
- package/src/model-checkpoint-memory-collector.ts +8 -1
- package/src/model-context-contributions.ts +159 -0
- package/src/observability-metrics.ts +98 -9
- package/src/sandbox-routing.ts +13 -0
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/file-upload-reaper.ts +8 -2
- package/src/workflows/session.ts +20 -6
- package/src/workflows/site-auth-maintenance.ts +14 -0
- package/src/workflows.ts +1 -0
- package/dist/activities-control-44KBWLSM.js.map +0 -1
- package/dist/activities-turn-K4CEDEGJ.js.map +0 -1
- package/dist/chunk-E3RI3I33.js.map +0 -1
- package/dist/chunk-KKCRGAGZ.js.map +0 -1
|
@@ -16,12 +16,26 @@ import {
|
|
|
16
16
|
getSessionRootId,
|
|
17
17
|
getSessionGoal,
|
|
18
18
|
getHumanInputResumeForEvent,
|
|
19
|
+
getInteractionInterventionResumeForEvent,
|
|
19
20
|
getSessionHumanInputRequest,
|
|
20
21
|
installOrReadTurnExecutionPolicyForAttempt,
|
|
21
22
|
persistAttemptToolCatalog,
|
|
22
23
|
workspaceCodexSubscriptionActive,
|
|
24
|
+
acquireXaiCredentialLease,
|
|
25
|
+
selectXaiCredentialForUse,
|
|
26
|
+
materializeXaiCredentialForRun,
|
|
27
|
+
resolveXaiProviderAccountAuthoritySnapshotForAcceptance,
|
|
28
|
+
heartbeatXaiCredentialLeaseUntil,
|
|
29
|
+
releaseXaiCredentialLease,
|
|
30
|
+
getXaiSessionAccountPin,
|
|
31
|
+
setXaiSessionAccountPin,
|
|
32
|
+
recordXaiSessionLastAccount,
|
|
33
|
+
updateXaiQuotaMetadata,
|
|
34
|
+
XAI_CREDENTIAL_LEASE_TTL_MS,
|
|
23
35
|
acquireCodexCredentialLease,
|
|
24
36
|
armCodexCapacityWait,
|
|
37
|
+
armXaiCapacityWait,
|
|
38
|
+
reconcileXaiCapacityWait,
|
|
25
39
|
heartbeatCodexCredentialLeaseUntil,
|
|
26
40
|
releaseCodexCredentialLease,
|
|
27
41
|
CODEX_CREDENTIAL_LEASE_TTL_MS,
|
|
@@ -105,11 +119,14 @@ import {
|
|
|
105
119
|
normalizeModelCallUsage,
|
|
106
120
|
normalizeSdkEvent,
|
|
107
121
|
normalizeProtocolJsonValue,
|
|
122
|
+
compactMcpResultCustomDataRunState,
|
|
123
|
+
releaseMcpResultCustomDataFromSdkEvent,
|
|
108
124
|
projectHistoryForProvider,
|
|
109
125
|
restoreGenericDispatchHistoryItems,
|
|
110
126
|
sanitizeHistoryItemsForModel,
|
|
111
127
|
projectModelInputForCapabilities,
|
|
112
128
|
appendPersistentSessionSettings,
|
|
129
|
+
appendSessionGoal,
|
|
113
130
|
appendSessionInstructions,
|
|
114
131
|
appendWorkspaceGovernance,
|
|
115
132
|
appendWorkspaceMemory,
|
|
@@ -166,6 +183,7 @@ import {
|
|
|
166
183
|
import { connectionTokenResolverForTurn } from "./mcp-credentials";
|
|
167
184
|
import { buildApiIntegrationServersForTurn } from "./api-integrations";
|
|
168
185
|
import {
|
|
186
|
+
allowedFirstPartyMcpToolsForSession,
|
|
169
187
|
assertTurnExecutionPolicyMatchesConfigV1,
|
|
170
188
|
calculateGatewayReportedCostBreakdown,
|
|
171
189
|
calculateModelUsageCostBreakdown,
|
|
@@ -175,11 +193,13 @@ import {
|
|
|
175
193
|
sandboxLifecycleTransitionWaitMs,
|
|
176
194
|
sandboxWarmRateMicrosPerSecond,
|
|
177
195
|
serviceTierForLatencyMode,
|
|
196
|
+
environmentsEncryptionKeyBytes,
|
|
178
197
|
settingsWithResolvedModelContext,
|
|
179
198
|
resolveTurnExecutionPolicyV1,
|
|
180
199
|
OPENGENI_GATEWAY_MODELS,
|
|
181
200
|
OPENGENI_GATEWAY_PROVIDER_ID,
|
|
182
201
|
WORKSPACE_GATEWAY_PROVIDER_ID,
|
|
202
|
+
codemodeWorkspaceUrl,
|
|
183
203
|
isDirectOpenAiApiBaseUrl,
|
|
184
204
|
resolveModelProvider,
|
|
185
205
|
type ModelUsageInput,
|
|
@@ -193,6 +213,7 @@ import {
|
|
|
193
213
|
settingsWithEnabledCapabilityMcpServers,
|
|
194
214
|
settingsWithSessionMcpServersForRun,
|
|
195
215
|
settingsWithWorkspaceGatewayCredential,
|
|
216
|
+
withXaiSubscriptionProvider,
|
|
196
217
|
} from "./capabilities";
|
|
197
218
|
import {
|
|
198
219
|
CODEX_USAGE_EXHAUSTED_PCT,
|
|
@@ -239,6 +260,15 @@ import {
|
|
|
239
260
|
type CodexUsageHeaderSnapshot,
|
|
240
261
|
} from "@opengeni/codex";
|
|
241
262
|
import { mergeResourceRefs } from "./common";
|
|
263
|
+
import {
|
|
264
|
+
fetchXaiSubscriptionQuota,
|
|
265
|
+
isXaiSubscriptionTransportError,
|
|
266
|
+
XaiSubscriptionReloginRequired,
|
|
267
|
+
xaiSubscriptionRequestStorage,
|
|
268
|
+
type XaiSubscriptionRequestContext,
|
|
269
|
+
} from "@opengeni/xai-subscription";
|
|
270
|
+
import { buildXaiTurnRequestAuthorization } from "./xai-auth";
|
|
271
|
+
import { executeXaiSubscriptionImageGeneration } from "./xai-image-generation";
|
|
242
272
|
import {
|
|
243
273
|
assertSessionAllowsProductModel,
|
|
244
274
|
defaultSessionMcpServerIds,
|
|
@@ -254,8 +284,10 @@ import { TurnAttemptFencedError } from "./turn-attempt-fenced";
|
|
|
254
284
|
import {
|
|
255
285
|
admitVideoGenerationRequest,
|
|
256
286
|
managedVideoGenerationCredentialLease,
|
|
287
|
+
xaiVideoGenerationCredentialLease,
|
|
257
288
|
type VideoGenerationCredentialLease,
|
|
258
289
|
} from "./video-generation-admission";
|
|
290
|
+
import { VideoReferenceInputError } from "./video-reference-staging";
|
|
259
291
|
import {
|
|
260
292
|
assertGitCredentialRenewalTransportUnchanged,
|
|
261
293
|
gitCredentialAuthorityForTurn,
|
|
@@ -292,7 +324,7 @@ import {
|
|
|
292
324
|
mergeRigDefaultVariableSetEnvironment,
|
|
293
325
|
rigProviderImageSourceImage,
|
|
294
326
|
resolveWorkspacePackRuntime,
|
|
295
|
-
|
|
327
|
+
resolveWorkspaceInstalledSkillRuntime,
|
|
296
328
|
settingsWithPackSandboxImage,
|
|
297
329
|
settingsWithRigImage,
|
|
298
330
|
settingsWithRigProviderImage,
|
|
@@ -345,6 +377,8 @@ import {
|
|
|
345
377
|
recordCreditMicros,
|
|
346
378
|
recordModelCacheTokens,
|
|
347
379
|
recordModelInputTokens,
|
|
380
|
+
recordModelRequestPhase,
|
|
381
|
+
recordCompanyBrainContributions,
|
|
348
382
|
recordSessionEventAppendLatency,
|
|
349
383
|
recordSessionEventPublishLatency,
|
|
350
384
|
runtimeMetricsHooksForObservability,
|
|
@@ -352,6 +386,7 @@ import {
|
|
|
352
386
|
turnLifecycleMetricsFor,
|
|
353
387
|
type TurnOutcome,
|
|
354
388
|
} from "../observability-metrics";
|
|
389
|
+
import { buildCompanyBrainContributionReceipt } from "../model-context-contributions";
|
|
355
390
|
import {
|
|
356
391
|
beginRecording,
|
|
357
392
|
discardUnpublishedRecording,
|
|
@@ -403,11 +438,12 @@ import {
|
|
|
403
438
|
} from "@opengeni/runtime";
|
|
404
439
|
import {
|
|
405
440
|
CAPABILITY_DESCRIPTORS,
|
|
406
|
-
DEFAULT_FIRST_PARTY_MCP_TOOLS,
|
|
407
441
|
evaluateWorkspaceModelPolicy,
|
|
408
442
|
readTurnExecutionPolicyV1,
|
|
409
443
|
resolveWorkspaceAgentHumanInputEnabled,
|
|
444
|
+
resolveWorkspaceMemoryPromptMode,
|
|
410
445
|
resourceMountPath,
|
|
446
|
+
VideoGenerationRejectedResult,
|
|
411
447
|
type LatencyMode,
|
|
412
448
|
type ResourceRef,
|
|
413
449
|
type RetainedArtifactMetadata,
|
|
@@ -418,6 +454,7 @@ import {
|
|
|
418
454
|
type SessionTurn,
|
|
419
455
|
type ToolAuthNeededPayload,
|
|
420
456
|
type TurnExecutionPolicyV1,
|
|
457
|
+
type XaiProviderAccountAuthoritySnapshotV1,
|
|
421
458
|
} from "@opengeni/contracts";
|
|
422
459
|
import { createHash, randomUUID } from "node:crypto";
|
|
423
460
|
import { createModelCheckpointMemoryCollector } from "../model-checkpoint-memory-collector";
|
|
@@ -496,6 +533,7 @@ export function shouldPublishToolAuthNeededForTurn(
|
|
|
496
533
|
export function turnExecutionPolicyBillingIdentity(policy: TurnExecutionPolicyV1): {
|
|
497
534
|
externallyBilled: boolean;
|
|
498
535
|
codexSubscription: boolean;
|
|
536
|
+
xaiSubscription: boolean;
|
|
499
537
|
} {
|
|
500
538
|
return {
|
|
501
539
|
externallyBilled: policy.billing.metering === "external",
|
|
@@ -503,6 +541,10 @@ export function turnExecutionPolicyBillingIdentity(policy: TurnExecutionPolicyV1
|
|
|
503
541
|
policy.providerId === "codex-subscription" &&
|
|
504
542
|
policy.credentialSource.kind === "connected_subscription" &&
|
|
505
543
|
policy.credentialSource.provider === "codex",
|
|
544
|
+
xaiSubscription:
|
|
545
|
+
policy.providerId === "supergrok-subscription" &&
|
|
546
|
+
policy.credentialSource.kind === "connected_subscription" &&
|
|
547
|
+
policy.credentialSource.provider === "xai",
|
|
506
548
|
};
|
|
507
549
|
}
|
|
508
550
|
|
|
@@ -714,6 +756,55 @@ export function stableHumanInputRequestId(
|
|
|
714
756
|
return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice(12, 16)}-${value.slice(16, 20)}-${value.slice(20)}`;
|
|
715
757
|
}
|
|
716
758
|
|
|
759
|
+
export function stableInteractionInterventionId(
|
|
760
|
+
sessionId: string,
|
|
761
|
+
turnId: string,
|
|
762
|
+
toolCallId: string,
|
|
763
|
+
): string {
|
|
764
|
+
return stableInteractionInterventionUuid(
|
|
765
|
+
"opengeni-interaction-intervention-v1",
|
|
766
|
+
sessionId,
|
|
767
|
+
turnId,
|
|
768
|
+
toolCallId,
|
|
769
|
+
);
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
export function stableInteractionInterventionOperationId(
|
|
773
|
+
sessionId: string,
|
|
774
|
+
turnId: string,
|
|
775
|
+
toolCallId: string,
|
|
776
|
+
): string {
|
|
777
|
+
return stableInteractionInterventionUuid(
|
|
778
|
+
"opengeni-interaction-intervention-operation-v1",
|
|
779
|
+
sessionId,
|
|
780
|
+
turnId,
|
|
781
|
+
toolCallId,
|
|
782
|
+
);
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
function stableInteractionInterventionUuid(
|
|
786
|
+
namespace: string,
|
|
787
|
+
sessionId: string,
|
|
788
|
+
turnId: string,
|
|
789
|
+
toolCallId: string,
|
|
790
|
+
): string {
|
|
791
|
+
const hex = createHash("sha256")
|
|
792
|
+
.update(namespace)
|
|
793
|
+
.update("\0")
|
|
794
|
+
.update(sessionId)
|
|
795
|
+
.update("\0")
|
|
796
|
+
.update(turnId)
|
|
797
|
+
.update("\0")
|
|
798
|
+
.update(toolCallId)
|
|
799
|
+
.digest("hex")
|
|
800
|
+
.slice(0, 32)
|
|
801
|
+
.split("");
|
|
802
|
+
hex[12] = "5";
|
|
803
|
+
hex[16] = ["8", "9", "a", "b"][Number.parseInt(hex[16] ?? "0", 16) % 4] ?? "8";
|
|
804
|
+
const value = hex.join("");
|
|
805
|
+
return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice(12, 16)}-${value.slice(16, 20)}-${value.slice(20)}`;
|
|
806
|
+
}
|
|
807
|
+
|
|
717
808
|
/**
|
|
718
809
|
* True when this activity attempt was cancelled because its hosting worker is
|
|
719
810
|
* shutting down gracefully (SIGTERM during a deploy), as opposed to a
|
|
@@ -1898,6 +1989,15 @@ export function managedSandboxOwnershipForTurn(
|
|
|
1898
1989
|
};
|
|
1899
1990
|
}
|
|
1900
1991
|
|
|
1992
|
+
/** A sandboxless home still establishes an explicitly attached Connected Machine. */
|
|
1993
|
+
export function shouldEstablishSandboxForTurn(
|
|
1994
|
+
sandboxOwnershipEnabled: boolean,
|
|
1995
|
+
homeBackend: Settings["sandboxBackend"],
|
|
1996
|
+
machinePrimary: boolean,
|
|
1997
|
+
): boolean {
|
|
1998
|
+
return sandboxOwnershipEnabled && (homeBackend !== "none" || machinePrimary);
|
|
1999
|
+
}
|
|
2000
|
+
|
|
1901
2001
|
/**
|
|
1902
2002
|
* Classify a persisted active-sandbox pointer for TURN-START RECONCILE (issue #341
|
|
1903
2003
|
* invariant B). Returns the typed reason to RESET the pointer to the session HOME,
|
|
@@ -2194,9 +2294,12 @@ export function structuredToolTransportForTurn(
|
|
|
2194
2294
|
resolvedModel: { provider: { kind: RegistryProviderKind } } | null,
|
|
2195
2295
|
): boolean {
|
|
2196
2296
|
if (!resolvedModel) return true;
|
|
2197
|
-
return ![
|
|
2198
|
-
|
|
2199
|
-
|
|
2297
|
+
return ![
|
|
2298
|
+
"codex-subscription",
|
|
2299
|
+
"xai-subscription",
|
|
2300
|
+
"vercel-gateway-managed",
|
|
2301
|
+
"vercel-gateway-workspace",
|
|
2302
|
+
].includes(resolvedModel.provider.kind);
|
|
2200
2303
|
}
|
|
2201
2304
|
|
|
2202
2305
|
/**
|
|
@@ -2243,6 +2346,21 @@ export function hostedWebSearchForTurn(
|
|
|
2243
2346
|
return resolvedModel?.configured.hostedWebSearch ?? deploymentWebSearchEnabled;
|
|
2244
2347
|
}
|
|
2245
2348
|
|
|
2349
|
+
/**
|
|
2350
|
+
* Image generation is an optional connected-account capability. A delegated
|
|
2351
|
+
* subscription may still authorize the text model without carrying the local
|
|
2352
|
+
* credential identity required for paid image operations; that must narrow the
|
|
2353
|
+
* tool catalog, not reject an otherwise valid turn.
|
|
2354
|
+
*/
|
|
2355
|
+
export function connectedSubscriptionImageGenerationAuthority<T>(
|
|
2356
|
+
credentialContext: T | null | undefined,
|
|
2357
|
+
credentialId: string | null | undefined,
|
|
2358
|
+
): { credentialContext: T; credentialId: string } | null {
|
|
2359
|
+
if (credentialContext == null || typeof credentialId !== "string" || credentialId.length === 0)
|
|
2360
|
+
return null;
|
|
2361
|
+
return { credentialContext, credentialId };
|
|
2362
|
+
}
|
|
2363
|
+
|
|
2246
2364
|
/** Direct OpenAI hosted image IDs are reusable only by the exact API credential. */
|
|
2247
2365
|
export function openAiHostedImageProviderBindingForTurn(
|
|
2248
2366
|
settings: Pick<Settings, "openaiProvider" | "openaiApiKey" | "openaiBaseUrl">,
|
|
@@ -2676,6 +2794,10 @@ export async function finalizeDurableTurnOpStreams(
|
|
|
2676
2794
|
|
|
2677
2795
|
export function createRunAgentTurnActivity(services: () => Promise<ActivityServices>) {
|
|
2678
2796
|
const modelCheckpointMemoryCollector = createModelCheckpointMemoryCollector();
|
|
2797
|
+
// Keep a distinct cooldown for terminal collection. A collection at the
|
|
2798
|
+
// final model checkpoint happens while the activity still owns its complete
|
|
2799
|
+
// turn graph and must not suppress collection after that graph is released.
|
|
2800
|
+
const turnCompletionMemoryCollector = createModelCheckpointMemoryCollector();
|
|
2679
2801
|
return async function runAgentTurn(input: RunAgentTurnInput): Promise<RunAgentTurnResult> {
|
|
2680
2802
|
const {
|
|
2681
2803
|
settings,
|
|
@@ -2767,6 +2889,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2767
2889
|
);
|
|
2768
2890
|
const sandboxOperationObserver = sandboxOperationMetricObserver(observability);
|
|
2769
2891
|
let isCodexTurn = false;
|
|
2892
|
+
let isXaiTurn = false;
|
|
2770
2893
|
let isExternallyBilledTurn = false;
|
|
2771
2894
|
let executionGeneration = 0;
|
|
2772
2895
|
let providerRecoveryCount = 0;
|
|
@@ -2898,6 +3021,83 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2898
3021
|
}, 60_000);
|
|
2899
3022
|
codexLeaseHeartbeatTimer.unref?.();
|
|
2900
3023
|
};
|
|
3024
|
+
let xaiLeaseHeld = false;
|
|
3025
|
+
let xaiLeaseLost = false;
|
|
3026
|
+
let xaiLeaseSubjectId: string | null = null;
|
|
3027
|
+
let xaiLeaseHolderId: string | null = null;
|
|
3028
|
+
let xaiLeaseGeneration: number | null = null;
|
|
3029
|
+
let xaiLeaseConfirmedUntilMs: number | null = null;
|
|
3030
|
+
let xaiLeaseHeartbeatInFlight = false;
|
|
3031
|
+
let xaiLeaseHeartbeatTimer: ReturnType<typeof setInterval> | undefined;
|
|
3032
|
+
const markXaiLeaseLost = (reason: "deadline" | "not_found"): void => {
|
|
3033
|
+
if (xaiLeaseLost) return;
|
|
3034
|
+
xaiLeaseLost = true;
|
|
3035
|
+
observability.warn("xAI credential lease was lost during an active turn", {
|
|
3036
|
+
workspaceId: input.workspaceId,
|
|
3037
|
+
turnId,
|
|
3038
|
+
reason,
|
|
3039
|
+
});
|
|
3040
|
+
};
|
|
3041
|
+
const renewXaiLease = async (): Promise<void> => {
|
|
3042
|
+
if (
|
|
3043
|
+
!turnId ||
|
|
3044
|
+
!xaiLeaseHeld ||
|
|
3045
|
+
!xaiLeaseSubjectId ||
|
|
3046
|
+
!xaiLeaseHolderId ||
|
|
3047
|
+
xaiLeaseGeneration === null ||
|
|
3048
|
+
xaiLeaseLost
|
|
3049
|
+
) {
|
|
3050
|
+
return;
|
|
3051
|
+
}
|
|
3052
|
+
if (codexCredentialLeaseDeadlineExpired(xaiLeaseConfirmedUntilMs)) {
|
|
3053
|
+
markXaiLeaseLost("deadline");
|
|
3054
|
+
return;
|
|
3055
|
+
}
|
|
3056
|
+
if (xaiLeaseHeartbeatInFlight) return;
|
|
3057
|
+
xaiLeaseHeartbeatInFlight = true;
|
|
3058
|
+
const renewalStartedAtMs = performance.now();
|
|
3059
|
+
try {
|
|
3060
|
+
const renewedUntil = await heartbeatXaiCredentialLeaseUntil(db, {
|
|
3061
|
+
workspaceId: input.workspaceId,
|
|
3062
|
+
subjectId: xaiLeaseSubjectId,
|
|
3063
|
+
turnId,
|
|
3064
|
+
holderId: xaiLeaseHolderId,
|
|
3065
|
+
generation: xaiLeaseGeneration,
|
|
3066
|
+
leaseTtlMs: XAI_CREDENTIAL_LEASE_TTL_MS,
|
|
3067
|
+
});
|
|
3068
|
+
if (!renewedUntil) {
|
|
3069
|
+
markXaiLeaseLost("not_found");
|
|
3070
|
+
} else {
|
|
3071
|
+
xaiLeaseConfirmedUntilMs = renewalStartedAtMs + XAI_CREDENTIAL_LEASE_TTL_MS;
|
|
3072
|
+
}
|
|
3073
|
+
} catch (error) {
|
|
3074
|
+
if (codexCredentialLeaseDeadlineExpired(xaiLeaseConfirmedUntilMs)) {
|
|
3075
|
+
markXaiLeaseLost("deadline");
|
|
3076
|
+
return;
|
|
3077
|
+
}
|
|
3078
|
+
observability.warn("xAI credential lease heartbeat failed", {
|
|
3079
|
+
workspaceId: input.workspaceId,
|
|
3080
|
+
turnId,
|
|
3081
|
+
...safeErrorDiagnostic(error),
|
|
3082
|
+
});
|
|
3083
|
+
} finally {
|
|
3084
|
+
xaiLeaseHeartbeatInFlight = false;
|
|
3085
|
+
}
|
|
3086
|
+
};
|
|
3087
|
+
const startXaiLeaseHeartbeat = (): void => {
|
|
3088
|
+
if (!turnId || xaiLeaseHeartbeatTimer) return;
|
|
3089
|
+
xaiLeaseHeartbeatTimer = setInterval(() => {
|
|
3090
|
+
void renewXaiLease();
|
|
3091
|
+
}, 60_000);
|
|
3092
|
+
xaiLeaseHeartbeatTimer.unref?.();
|
|
3093
|
+
};
|
|
3094
|
+
const renewServingCredentialLease = async (
|
|
3095
|
+
reason: "timer" | "runtime_event" | "model_usage",
|
|
3096
|
+
): Promise<void> => {
|
|
3097
|
+
await renewCodexLease(reason);
|
|
3098
|
+
await renewXaiLease();
|
|
3099
|
+
};
|
|
3100
|
+
const servingCredentialLeaseLost = (): boolean => codexLeaseLost || xaiLeaseLost;
|
|
2901
3101
|
// P1.2 ownership inversion: when sandboxOwnershipEnabled, the turn resolves
|
|
2902
3102
|
// the one box by id from the group lease and injects it NON-OWNED into the
|
|
2903
3103
|
// run. null when the flag is off (byte-for-byte the legacy build-and-discard
|
|
@@ -3670,6 +3870,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3670
3870
|
compactRetainedScreenshotRunState(serialized, retainedScreenshotReceiptsByCallId),
|
|
3671
3871
|
generatedImageReceiptsByProviderItemId,
|
|
3672
3872
|
);
|
|
3873
|
+
const compactApprovalRunState = (serialized: string): string =>
|
|
3874
|
+
compactMcpResultCustomDataRunState(compactMediaRunState(serialized));
|
|
3673
3875
|
// Explicit image-producing tools cross the durable session-media boundary.
|
|
3674
3876
|
// Incidental frames returned by click/scroll actions remain unretained; an
|
|
3675
3877
|
// intentional computer_screenshot or view_image result must never be lost
|
|
@@ -3847,6 +4049,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3847
4049
|
// The Codex account this turn runs on (pin > workspace active), resolved once
|
|
3848
4050
|
// a codex-billed turn is confirmed and threaded into the token resolver below.
|
|
3849
4051
|
let effectiveCodexCredentialId: string | null = null;
|
|
4052
|
+
let effectiveXaiCredentialId: string | null = null;
|
|
4053
|
+
let xaiRotationEnabled = false;
|
|
4054
|
+
let xaiAuthoritySnapshot: XaiProviderAccountAuthoritySnapshotV1 | null = null;
|
|
4055
|
+
let xaiRequestContext: XaiSubscriptionRequestContext | null = null;
|
|
4056
|
+
let xaiCredentialQuarantined = false;
|
|
3850
4057
|
// The session's Codex credential BEFORE this turn resolved its own — captured
|
|
3851
4058
|
// before recordSessionActiveCodexCredential overwrites the durable pointer, so
|
|
3852
4059
|
// a per-call usage log can report whether the serving account CHANGED since the
|
|
@@ -3914,10 +4121,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3914
4121
|
mcpSettings,
|
|
3915
4122
|
codexSubscriptionActive,
|
|
3916
4123
|
);
|
|
4124
|
+
const xaiSettings = codexSettings.supergrokSubscriptionEnabled
|
|
4125
|
+
? withXaiSubscriptionProvider(codexSettings)
|
|
4126
|
+
: codexSettings;
|
|
3917
4127
|
const capabilitySettings = await settingsWithWorkspaceGatewayCredential(
|
|
3918
4128
|
db,
|
|
3919
4129
|
input.workspaceId,
|
|
3920
|
-
|
|
4130
|
+
xaiSettings,
|
|
3921
4131
|
);
|
|
3922
4132
|
const codexAppsCredentialId = capabilitySettings.codexConnectedAppsEnabled
|
|
3923
4133
|
? await resolveCodexAppsCredentialIdForRun(db, input.workspaceId)
|
|
@@ -3956,6 +4166,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3956
4166
|
const billingIdentity = turnExecutionPolicyBillingIdentity(turnExecutionPolicy);
|
|
3957
4167
|
isExternallyBilledTurn = billingIdentity.externallyBilled;
|
|
3958
4168
|
isCodexTurn = billingIdentity.codexSubscription;
|
|
4169
|
+
isXaiTurn = billingIdentity.xaiSubscription;
|
|
3959
4170
|
triggerEventId = turn.triggerEventId;
|
|
3960
4171
|
const trigger = await getSessionEvent(db, input.workspaceId, triggerEventId);
|
|
3961
4172
|
if (!trigger) {
|
|
@@ -3967,6 +4178,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3967
4178
|
input.sessionId,
|
|
3968
4179
|
trigger,
|
|
3969
4180
|
);
|
|
4181
|
+
const interactionInterventionResume = await getInteractionInterventionResumeForEvent(
|
|
4182
|
+
db,
|
|
4183
|
+
input.workspaceId,
|
|
4184
|
+
input.sessionId,
|
|
4185
|
+
trigger,
|
|
4186
|
+
);
|
|
3970
4187
|
triggerType = trigger.type;
|
|
3971
4188
|
redispatchesAtDispatch = Number(
|
|
3972
4189
|
(turn.metadata as { workerDeathRedispatches?: number } | null)?.workerDeathRedispatches ??
|
|
@@ -4065,6 +4282,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4065
4282
|
if (inputs.length > 0 && !appended.accepted) {
|
|
4066
4283
|
throw new TurnAttemptFencedError("turn execution generation was fenced");
|
|
4067
4284
|
}
|
|
4285
|
+
if (inputs.length > 0) {
|
|
4286
|
+
turnLifecycleMetricsFor(observability).progress(turnId!);
|
|
4287
|
+
}
|
|
4068
4288
|
activityContext?.heartbeat({
|
|
4069
4289
|
...heartbeatDetails,
|
|
4070
4290
|
phase: "events_published",
|
|
@@ -4158,6 +4378,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4158
4378
|
turnMetricOutcome = "cancelled";
|
|
4159
4379
|
return false;
|
|
4160
4380
|
}
|
|
4381
|
+
turnLifecycleMetricsFor(observability).progress(turnId!);
|
|
4161
4382
|
if (recordingForSettlement && preparedRecording) {
|
|
4162
4383
|
if (result.recordingMutationApplied) {
|
|
4163
4384
|
activeRecording = null;
|
|
@@ -4805,14 +5026,220 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4805
5026
|
}
|
|
4806
5027
|
}
|
|
4807
5028
|
|
|
5029
|
+
if (isXaiTurn) {
|
|
5030
|
+
const authoritySnapshot = turn.xaiProviderAccountAuthoritySnapshot;
|
|
5031
|
+
xaiAuthoritySnapshot = authoritySnapshot;
|
|
5032
|
+
const subjectId =
|
|
5033
|
+
authoritySnapshot.scope === "user"
|
|
5034
|
+
? turn.initiatingHumanSubjectId
|
|
5035
|
+
: "worker:xai-workspace";
|
|
5036
|
+
if (!subjectId) {
|
|
5037
|
+
throw new Error("User-scoped SuperGrok work has no frozen initiating human");
|
|
5038
|
+
}
|
|
5039
|
+
const sessionPin = await getXaiSessionAccountPin(db, {
|
|
5040
|
+
workspaceId: input.workspaceId,
|
|
5041
|
+
subjectId,
|
|
5042
|
+
sessionId: input.sessionId,
|
|
5043
|
+
authoritySnapshot,
|
|
5044
|
+
});
|
|
5045
|
+
const leaseStartedAtMs = performance.now();
|
|
5046
|
+
const leased = await acquireXaiCredentialLease(db, {
|
|
5047
|
+
accountId: input.accountId,
|
|
5048
|
+
workspaceId: input.workspaceId,
|
|
5049
|
+
subjectId,
|
|
5050
|
+
sessionId: input.sessionId,
|
|
5051
|
+
turnId: turn.id,
|
|
5052
|
+
holderId: dispatchId,
|
|
5053
|
+
authoritySnapshot,
|
|
5054
|
+
pinnedCredentialId: sessionPin?.pinnedCredentialId ?? null,
|
|
5055
|
+
pinSource:
|
|
5056
|
+
sessionPin?.pinSource === "manual" || sessionPin?.pinSource === "policy"
|
|
5057
|
+
? sessionPin.pinSource
|
|
5058
|
+
: null,
|
|
5059
|
+
});
|
|
5060
|
+
effectiveXaiCredentialId = leased.credentialId;
|
|
5061
|
+
xaiRotationEnabled = leased.rotationEnabled;
|
|
5062
|
+
xaiLeaseSubjectId = subjectId;
|
|
5063
|
+
xaiLeaseHolderId = leased.holderId;
|
|
5064
|
+
xaiLeaseGeneration = leased.generation;
|
|
5065
|
+
xaiLeaseConfirmedUntilMs = leased.leasedUntil
|
|
5066
|
+
? leaseStartedAtMs + XAI_CREDENTIAL_LEASE_TTL_MS
|
|
5067
|
+
: null;
|
|
5068
|
+
xaiLeaseHeld =
|
|
5069
|
+
effectiveXaiCredentialId !== null &&
|
|
5070
|
+
leased.holderId !== null &&
|
|
5071
|
+
leased.generation !== null &&
|
|
5072
|
+
xaiLeaseConfirmedUntilMs !== null;
|
|
5073
|
+
if (!effectiveXaiCredentialId) {
|
|
5074
|
+
const connected = leased.accounts.length;
|
|
5075
|
+
const allocatorEnabled = leased.accounts.filter(
|
|
5076
|
+
(account) => account.allocatorEnabled,
|
|
5077
|
+
).length;
|
|
5078
|
+
if (connected === 0) {
|
|
5079
|
+
throw Object.assign(
|
|
5080
|
+
new Error("No SuperGrok subscription account is connected for this authority scope"),
|
|
5081
|
+
{ code: "xai_not_connected" },
|
|
5082
|
+
);
|
|
5083
|
+
}
|
|
5084
|
+
if (turn.source === "compaction") {
|
|
5085
|
+
if (
|
|
5086
|
+
!(await settle!({
|
|
5087
|
+
events: [
|
|
5088
|
+
{
|
|
5089
|
+
type: "turn.cancelled",
|
|
5090
|
+
payload: {
|
|
5091
|
+
maintenance: "context_compaction",
|
|
5092
|
+
reason: "xai_capacity_unavailable",
|
|
5093
|
+
requestPreserved: true,
|
|
5094
|
+
},
|
|
5095
|
+
},
|
|
5096
|
+
{
|
|
5097
|
+
type: "session.status.changed",
|
|
5098
|
+
payload: { status: "idle" },
|
|
5099
|
+
},
|
|
5100
|
+
],
|
|
5101
|
+
turnStatus: "cancelled",
|
|
5102
|
+
sessionStatus: "idle",
|
|
5103
|
+
activeTurnId: null,
|
|
5104
|
+
}))
|
|
5105
|
+
) {
|
|
5106
|
+
return claimedResult({ status: "cancelled" });
|
|
5107
|
+
}
|
|
5108
|
+
turnMetricOutcome = "cancelled";
|
|
5109
|
+
activityStatus = "idle";
|
|
5110
|
+
return claimedResult({ status: "idle", deferredUntilWake: true });
|
|
5111
|
+
}
|
|
5112
|
+
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
|
|
5113
|
+
() => null,
|
|
5114
|
+
);
|
|
5115
|
+
const activeGoal = goal?.status === "active" ? goal : null;
|
|
5116
|
+
const now = new Date();
|
|
5117
|
+
const futureResets = leased.accounts
|
|
5118
|
+
.map((account) => account.exhaustedUntil)
|
|
5119
|
+
.filter((date): date is Date => date !== null && date > now);
|
|
5120
|
+
const earliestResetAt = futureResets.length
|
|
5121
|
+
? new Date(Math.min(...futureResets.map((date) => date.getTime())))
|
|
5122
|
+
: null;
|
|
5123
|
+
const error =
|
|
5124
|
+
allocatorEnabled === 0
|
|
5125
|
+
? "All connected SuperGrok subscription accounts are disabled for allocation"
|
|
5126
|
+
: "All connected SuperGrok subscription accounts are temporarily unavailable";
|
|
5127
|
+
const armed = await armXaiCapacityWait(db, {
|
|
5128
|
+
accountId: input.accountId,
|
|
5129
|
+
workspaceId: input.workspaceId,
|
|
5130
|
+
subjectId,
|
|
5131
|
+
sessionId: input.sessionId,
|
|
5132
|
+
turnId: turn.id,
|
|
5133
|
+
attemptId: input.attemptId,
|
|
5134
|
+
workflowId: input.workflowId,
|
|
5135
|
+
authoritySnapshot,
|
|
5136
|
+
goalId: activeGoal?.id ?? null,
|
|
5137
|
+
goalVersion: activeGoal?.version ?? null,
|
|
5138
|
+
earliestResetAt,
|
|
5139
|
+
failurePayload: {
|
|
5140
|
+
error,
|
|
5141
|
+
code: allocatorEnabled === 0 ? "xai_allocator_disabled" : "xai_capacity_unavailable",
|
|
5142
|
+
detail: "waiting for an eligible account, reconnect, pin change, or quota reset",
|
|
5143
|
+
},
|
|
5144
|
+
});
|
|
5145
|
+
if (armed.action === "waiting") {
|
|
5146
|
+
await publishDurableSessionEvents(
|
|
5147
|
+
bus,
|
|
5148
|
+
input.workspaceId,
|
|
5149
|
+
input.sessionId,
|
|
5150
|
+
armed.events,
|
|
5151
|
+
);
|
|
5152
|
+
turnMetricOutcome = "recovering";
|
|
5153
|
+
activityStatus = "waiting_capacity";
|
|
5154
|
+
return claimedResult({
|
|
5155
|
+
status: "waiting_capacity",
|
|
5156
|
+
capacityWait: {
|
|
5157
|
+
provider: "xai",
|
|
5158
|
+
waiterId: armed.waiter.id,
|
|
5159
|
+
generation: armed.waiter.generation,
|
|
5160
|
+
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
5161
|
+
wakeRevision: armed.waiter.wakeRevision,
|
|
5162
|
+
},
|
|
5163
|
+
});
|
|
5164
|
+
}
|
|
5165
|
+
if (
|
|
5166
|
+
!(await settle!({
|
|
5167
|
+
events: [
|
|
5168
|
+
{
|
|
5169
|
+
type: "turn.failed",
|
|
5170
|
+
payload: {
|
|
5171
|
+
error,
|
|
5172
|
+
code: "xai_capacity_wait_stale",
|
|
5173
|
+
retryable: false,
|
|
5174
|
+
recovery: "user_message",
|
|
5175
|
+
},
|
|
5176
|
+
},
|
|
5177
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
5178
|
+
],
|
|
5179
|
+
turnStatus: "failed",
|
|
5180
|
+
sessionStatus: "idle",
|
|
5181
|
+
activeTurnId: null,
|
|
5182
|
+
}))
|
|
5183
|
+
) {
|
|
5184
|
+
return claimedResult({ status: "cancelled" });
|
|
5185
|
+
}
|
|
5186
|
+
turnMetricOutcome = "failed";
|
|
5187
|
+
activityStatus = "idle";
|
|
5188
|
+
return claimedResult({ status: "idle" });
|
|
5189
|
+
}
|
|
5190
|
+
if (xaiLeaseHeld) startXaiLeaseHeartbeat();
|
|
5191
|
+
if (
|
|
5192
|
+
leased.rotationEnabled &&
|
|
5193
|
+
sessionPin?.pinSource !== "manual" &&
|
|
5194
|
+
(sessionPin?.pinnedCredentialId !== effectiveXaiCredentialId ||
|
|
5195
|
+
sessionPin?.pinSource !== "policy")
|
|
5196
|
+
) {
|
|
5197
|
+
await setXaiSessionAccountPin(db, {
|
|
5198
|
+
accountId: input.accountId,
|
|
5199
|
+
workspaceId: input.workspaceId,
|
|
5200
|
+
subjectId,
|
|
5201
|
+
sessionId: input.sessionId,
|
|
5202
|
+
authoritySnapshot,
|
|
5203
|
+
credentialId: effectiveXaiCredentialId,
|
|
5204
|
+
pinSource: "policy",
|
|
5205
|
+
expectedVersion: sessionPin?.version ?? null,
|
|
5206
|
+
}).catch((error: unknown) => {
|
|
5207
|
+
if (error instanceof Error && error.message === "xAI session pin changed") return;
|
|
5208
|
+
throw error;
|
|
5209
|
+
});
|
|
5210
|
+
} else if (!leased.rotationEnabled && sessionPin?.pinSource === "policy") {
|
|
5211
|
+
await setXaiSessionAccountPin(db, {
|
|
5212
|
+
accountId: input.accountId,
|
|
5213
|
+
workspaceId: input.workspaceId,
|
|
5214
|
+
subjectId,
|
|
5215
|
+
sessionId: input.sessionId,
|
|
5216
|
+
authoritySnapshot,
|
|
5217
|
+
credentialId: null,
|
|
5218
|
+
pinSource: null,
|
|
5219
|
+
expectedVersion: sessionPin.version,
|
|
5220
|
+
}).catch((error: unknown) => {
|
|
5221
|
+
if (error instanceof Error && error.message === "xAI session pin changed") return;
|
|
5222
|
+
throw error;
|
|
5223
|
+
});
|
|
5224
|
+
}
|
|
5225
|
+
await recordXaiSessionLastAccount(db, {
|
|
5226
|
+
accountId: input.accountId,
|
|
5227
|
+
workspaceId: input.workspaceId,
|
|
5228
|
+
subjectId,
|
|
5229
|
+
sessionId: input.sessionId,
|
|
5230
|
+
authoritySnapshot,
|
|
5231
|
+
credentialId: effectiveXaiCredentialId,
|
|
5232
|
+
});
|
|
5233
|
+
}
|
|
5234
|
+
|
|
4808
5235
|
// Pack-scoped runtime: enabled packs may declare the sandbox image this
|
|
4809
5236
|
// workspace's sessions run in and skills for the sandbox skill index.
|
|
4810
5237
|
// Resolved after turn.started so a composition conflict (two enabled
|
|
4811
5238
|
// packs declaring images) fails the turn with its plain error instead
|
|
4812
5239
|
// of failing the activity opaquely.
|
|
4813
|
-
const [packRuntime,
|
|
5240
|
+
const [packRuntime, installedSkillRuntime] = await Promise.all([
|
|
4814
5241
|
resolveWorkspacePackRuntime(db, input.workspaceId),
|
|
4815
|
-
|
|
5242
|
+
resolveWorkspaceInstalledSkillRuntime(db, input.workspaceId),
|
|
4816
5243
|
]);
|
|
4817
5244
|
// RIG BINDING (M3): load the session's FROZEN rig version (resolved+frozen
|
|
4818
5245
|
// at create). Everything rig-derived below (image precedence, env default
|
|
@@ -4860,12 +5287,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4860
5287
|
if (!workspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
|
|
4861
5288
|
const workspaceAgentInstructions = workspace.agentInstructions;
|
|
4862
5289
|
const agentHumanInputEnabled = resolveWorkspaceAgentHumanInputEnabled(workspace.settings);
|
|
5290
|
+
const memoryPromptMode = resolveWorkspaceMemoryPromptMode(workspace.settings);
|
|
4863
5291
|
assertWorkspaceHumanInputAllowed(agentHumanInputEnabled, "resume", humanInputResume !== null);
|
|
4864
|
-
const
|
|
4865
|
-
|
|
4866
|
-
|
|
4867
|
-
|
|
4868
|
-
|
|
5292
|
+
const companyProfileIncluded =
|
|
5293
|
+
memoryPromptMode === "legacy_standing" || session.nestedAgentDepth === 0;
|
|
5294
|
+
const workspaceGovernance = renderWorkspaceGovernanceContext(
|
|
5295
|
+
{
|
|
5296
|
+
companyProfile: companyProfileSnapshot,
|
|
5297
|
+
instructionPolicy: instructionPolicySnapshot,
|
|
5298
|
+
preferences: preferenceSnapshot,
|
|
5299
|
+
},
|
|
5300
|
+
{
|
|
5301
|
+
includeCompanyProfile: companyProfileIncluded,
|
|
5302
|
+
},
|
|
5303
|
+
);
|
|
4869
5304
|
const structuredWorkspacePolicyActive =
|
|
4870
5305
|
hasActiveWorkspaceInstructionPolicy(instructionPolicySnapshot);
|
|
4871
5306
|
const workspaceMemory = await resolveWorkspaceMemoryBlock(db, input.workspaceId);
|
|
@@ -5067,6 +5502,31 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5067
5502
|
onRequestOpaqueArtifacts: ({ fingerprints }) => {
|
|
5068
5503
|
lastCodexRequestOpaqueArtifacts = fingerprints;
|
|
5069
5504
|
},
|
|
5505
|
+
onModelRequestDiagnostic: (event) => {
|
|
5506
|
+
const phase =
|
|
5507
|
+
event.phase === "headers"
|
|
5508
|
+
? "headers"
|
|
5509
|
+
: event.phase === "first_byte"
|
|
5510
|
+
? "first_byte"
|
|
5511
|
+
: event.phase === "completed" ||
|
|
5512
|
+
event.phase === "failed" ||
|
|
5513
|
+
event.phase === "timed_out"
|
|
5514
|
+
? "terminal"
|
|
5515
|
+
: null;
|
|
5516
|
+
if (!phase) return;
|
|
5517
|
+
recordModelRequestPhase(observability, {
|
|
5518
|
+
provider: "codex-subscription",
|
|
5519
|
+
phase,
|
|
5520
|
+
...(event.phase === "completed"
|
|
5521
|
+
? { outcome: "completed" as const }
|
|
5522
|
+
: event.phase === "failed"
|
|
5523
|
+
? { outcome: "failed" as const }
|
|
5524
|
+
: event.phase === "timed_out"
|
|
5525
|
+
? { outcome: "timed_out" as const }
|
|
5526
|
+
: {}),
|
|
5527
|
+
durationSeconds: event.durationMs / 1000,
|
|
5528
|
+
});
|
|
5529
|
+
},
|
|
5070
5530
|
nextRequestId: () => `${dispatchId}:${++codexModelRequestSequence}`,
|
|
5071
5531
|
onModelRequestEvent: async (event) => {
|
|
5072
5532
|
if (!publish || !turnId) {
|
|
@@ -5089,8 +5549,35 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5089
5549
|
};
|
|
5090
5550
|
})()
|
|
5091
5551
|
: null;
|
|
5552
|
+
if (resolvedModel?.provider.kind === "xai-subscription") {
|
|
5553
|
+
if (!effectiveXaiCredentialId || !xaiLeaseSubjectId) {
|
|
5554
|
+
throw new Error("SuperGrok subscription execution has no leased credential");
|
|
5555
|
+
}
|
|
5556
|
+
let xaiModelRequestSequence = 0;
|
|
5557
|
+
const authorization = await buildXaiTurnRequestAuthorization({
|
|
5558
|
+
db,
|
|
5559
|
+
settings: runSettings,
|
|
5560
|
+
accountId: input.accountId,
|
|
5561
|
+
workspaceId: input.workspaceId,
|
|
5562
|
+
subjectId: xaiLeaseSubjectId,
|
|
5563
|
+
sessionId: input.sessionId,
|
|
5564
|
+
turnId: turn.id,
|
|
5565
|
+
credentialId: effectiveXaiCredentialId,
|
|
5566
|
+
authoritySnapshot: turn.xaiProviderAccountAuthoritySnapshot,
|
|
5567
|
+
hostedSearch: {
|
|
5568
|
+
webSearch: runSettings.webSearchEnabled,
|
|
5569
|
+
xSearch: runSettings.webSearchEnabled,
|
|
5570
|
+
},
|
|
5571
|
+
nextRequestId: () => `${dispatchId}:xai:${++xaiModelRequestSequence}`,
|
|
5572
|
+
});
|
|
5573
|
+
xaiRequestContext = authorization.context;
|
|
5574
|
+
}
|
|
5092
5575
|
const withCodex = <T>(fn: () => Promise<T>): Promise<T> =>
|
|
5093
5576
|
codexContext ? codexRequestStorage.run(codexContext, fn) : fn();
|
|
5577
|
+
const withProviderRequestContext = <T>(fn: () => Promise<T>): Promise<T> =>
|
|
5578
|
+
xaiRequestContext
|
|
5579
|
+
? xaiSubscriptionRequestStorage.run(xaiRequestContext, fn)
|
|
5580
|
+
: withCodex(fn);
|
|
5094
5581
|
const withCodexRemoteCompaction = <T>(fn: () => Promise<T>): Promise<T> =>
|
|
5095
5582
|
withCodex(() =>
|
|
5096
5583
|
withCodexRequestOverrides(
|
|
@@ -5134,15 +5621,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5134
5621
|
servingCredentialId: effectiveCodexCredentialId,
|
|
5135
5622
|
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
5136
5623
|
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
5137
|
-
renewLease: () =>
|
|
5138
|
-
leaseLost:
|
|
5139
|
-
leaseLostMessage: "
|
|
5624
|
+
renewLease: () => renewServingCredentialLease("model_usage"),
|
|
5625
|
+
leaseLost: servingCredentialLeaseLost,
|
|
5626
|
+
leaseLostMessage: "Provider credential lease expired during context compaction",
|
|
5140
5627
|
});
|
|
5141
5628
|
};
|
|
5142
5629
|
const compactionSummarizerFor = (systemInstructions?: string) =>
|
|
5143
5630
|
resolvedModel
|
|
5144
5631
|
? (s: Settings, m: Array<Record<string, unknown>>) =>
|
|
5145
|
-
|
|
5632
|
+
withProviderRequestContext(() =>
|
|
5146
5633
|
summarizeContextForCompaction(s, m, {
|
|
5147
5634
|
client: resolvedModel.client,
|
|
5148
5635
|
provider: resolvedModel.provider,
|
|
@@ -5256,20 +5743,23 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5256
5743
|
};
|
|
5257
5744
|
const compactionInstructions = appendWorkspaceMemory(
|
|
5258
5745
|
appendPersistentSessionSettings(
|
|
5259
|
-
|
|
5260
|
-
|
|
5261
|
-
|
|
5262
|
-
|
|
5263
|
-
|
|
5264
|
-
|
|
5265
|
-
|
|
5266
|
-
|
|
5267
|
-
|
|
5268
|
-
|
|
5746
|
+
appendSessionGoal(
|
|
5747
|
+
appendSessionInstructions(
|
|
5748
|
+
appendWorkspaceGovernance(
|
|
5749
|
+
composeAgentInstructions(
|
|
5750
|
+
structuredWorkspacePolicyActive
|
|
5751
|
+
? modelRunSettings.agentInstructionsTemplate
|
|
5752
|
+
: (workspaceAgentInstructions ?? modelRunSettings.agentInstructionsTemplate),
|
|
5753
|
+
undefined,
|
|
5754
|
+
rigVersion && rigName
|
|
5755
|
+
? { name: rigName, version: rigVersion.version }
|
|
5756
|
+
: undefined,
|
|
5757
|
+
),
|
|
5758
|
+
workspaceGovernance ?? undefined,
|
|
5269
5759
|
),
|
|
5270
|
-
|
|
5760
|
+
session.instructions ?? undefined,
|
|
5271
5761
|
),
|
|
5272
|
-
|
|
5762
|
+
turn.goalSnapshot,
|
|
5273
5763
|
),
|
|
5274
5764
|
persistentSessionSettings,
|
|
5275
5765
|
),
|
|
@@ -5525,6 +6015,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5525
6015
|
activeSandboxBackend === "selfhosted" &&
|
|
5526
6016
|
Boolean(activeSandboxPointer?.activeSandboxId) &&
|
|
5527
6017
|
Boolean(activeSandboxRecord?.enrollmentId);
|
|
6018
|
+
// `none` describes the durable home, not an explicit per-turn route. Give
|
|
6019
|
+
// the runtime the effective backend so it builds a SandboxAgent for the
|
|
6020
|
+
// attached Connected Machine without mutating the session or turn record.
|
|
6021
|
+
if (machinePrimary && modelRunSettings.sandboxBackend === "none") {
|
|
6022
|
+
modelRunSettings = {
|
|
6023
|
+
...modelRunSettings,
|
|
6024
|
+
sandboxBackend: "selfhosted",
|
|
6025
|
+
};
|
|
6026
|
+
}
|
|
5528
6027
|
// The backend that can actually create a sandbox for this turn. In the
|
|
5529
6028
|
// common path this is runSettings.sandboxBackend. A selfhosted home turn
|
|
5530
6029
|
// that is NOT machine-primary falls back to the deployment cloud backend
|
|
@@ -5676,7 +6175,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5676
6175
|
sandboxWorkspaceEnvironmentValues,
|
|
5677
6176
|
{
|
|
5678
6177
|
skipGitHubToken: activeSandboxBackend === "selfhosted",
|
|
5679
|
-
|
|
6178
|
+
codemodeDelivery:
|
|
6179
|
+
activeSandboxBackend === "selfhosted" ? "transient_exec" : "managed_file",
|
|
5680
6180
|
deferGitHubToken:
|
|
5681
6181
|
activeSandboxBackend !== "selfhosted" && establishPolicy === "on-demand",
|
|
5682
6182
|
scope: connectionScope,
|
|
@@ -5696,6 +6196,22 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5696
6196
|
? { ...baseSandboxEnvironment, ...sandboxArtifactRuntime.environment }
|
|
5697
6197
|
: baseSandboxEnvironment;
|
|
5698
6198
|
|
|
6199
|
+
// One mutable in-memory bearer cell serves every Connected Machine route
|
|
6200
|
+
// in this attempt. SelfhostedSession snapshots it into each exact exec
|
|
6201
|
+
// request; it never enters the manifest, argv, filesystem, RunState, or
|
|
6202
|
+
// serialized session state. Managed renewal updates the same cell so a
|
|
6203
|
+
// later mid-turn swap sees the fresh bearer too.
|
|
6204
|
+
const codemodeTokenState = sandboxCodemodeToken ? { token: sandboxCodemodeToken } : undefined;
|
|
6205
|
+
const transientCodemodeEnvironment = codemodeTokenState
|
|
6206
|
+
? (): Readonly<Record<string, string>> => ({
|
|
6207
|
+
OPENGENI_CODEMODE_URL: codemodeWorkspaceUrl(runSettings, input.workspaceId),
|
|
6208
|
+
OPENGENI_CODEMODE_TOKEN: codemodeTokenState.token,
|
|
6209
|
+
...(runSettings.ogtoolPackageSpec
|
|
6210
|
+
? { OPENGENI_OGTOOL_PACKAGE_SPEC: runSettings.ogtoolPackageSpec }
|
|
6211
|
+
: {}),
|
|
6212
|
+
})
|
|
6213
|
+
: undefined;
|
|
6214
|
+
|
|
5699
6215
|
const sandboxCodemodeTokenFile = sandboxCodemodeToken
|
|
5700
6216
|
? codemodeTokenFileFromEnvironment(sandboxEnvironment, input.sessionId)
|
|
5701
6217
|
: undefined;
|
|
@@ -5811,11 +6327,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5811
6327
|
};
|
|
5812
6328
|
|
|
5813
6329
|
const attachCodemodeTokenRenewal = async (
|
|
5814
|
-
tokenSession
|
|
6330
|
+
tokenSession?: CodemodeTokenWriterSession,
|
|
5815
6331
|
initialExpiresAt = sandboxCodemodeTokenExpiresAt,
|
|
5816
6332
|
initialSandbox?: ResumedTurnSandbox,
|
|
5817
6333
|
): Promise<void> => {
|
|
5818
|
-
if (!
|
|
6334
|
+
if (!codemodeTokenState || !initialExpiresAt) return;
|
|
5819
6335
|
const previous = codemodeTokenRenewal;
|
|
5820
6336
|
codemodeTokenRenewal = null;
|
|
5821
6337
|
await previous?.stop();
|
|
@@ -5827,37 +6343,38 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5827
6343
|
connectionScope,
|
|
5828
6344
|
codemodeAuthority,
|
|
5829
6345
|
);
|
|
5830
|
-
if (material) {
|
|
5831
|
-
}
|
|
5832
6346
|
return material;
|
|
5833
6347
|
};
|
|
5834
6348
|
const write = async (material: NonNullable<Awaited<ReturnType<typeof mint>>>) => {
|
|
5835
|
-
|
|
5836
|
-
|
|
5837
|
-
|
|
5838
|
-
|
|
6349
|
+
if (tokenSession) {
|
|
6350
|
+
const runAs = sandboxRunAs(runSettings);
|
|
6351
|
+
const targetSandbox = resolvedSandbox ?? initialSandbox;
|
|
6352
|
+
if (!targetSandbox) {
|
|
6353
|
+
throw new Error("Codemode token renewal has no exact sandbox lease target");
|
|
6354
|
+
}
|
|
6355
|
+
await runWorkspaceMutationForSandbox(
|
|
6356
|
+
targetSandbox,
|
|
6357
|
+
"codemodeTokenRenewal",
|
|
6358
|
+
async () =>
|
|
6359
|
+
await refreshCodemodeTokenFile(tokenSession, material.token, {
|
|
6360
|
+
...(runAs ? { runAs } : {}),
|
|
6361
|
+
...(sandboxCodemodeTokenFile
|
|
6362
|
+
? {
|
|
6363
|
+
tokenFile: sandboxCodemodeTokenFile,
|
|
6364
|
+
legacyTokenFile: sandboxEnvironment.OPENGENI_CODEMODE_TOKEN_FILE!,
|
|
6365
|
+
}
|
|
6366
|
+
: {}),
|
|
6367
|
+
...(toolCancellationFenceRef.current
|
|
6368
|
+
? {
|
|
6369
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
6370
|
+
toolCancellationFenceRef.current,
|
|
6371
|
+
),
|
|
6372
|
+
}
|
|
6373
|
+
: {}),
|
|
6374
|
+
}),
|
|
6375
|
+
);
|
|
5839
6376
|
}
|
|
5840
|
-
|
|
5841
|
-
targetSandbox,
|
|
5842
|
-
"codemodeTokenRenewal",
|
|
5843
|
-
async () =>
|
|
5844
|
-
await refreshCodemodeTokenFile(tokenSession, material.token, {
|
|
5845
|
-
...(runAs ? { runAs } : {}),
|
|
5846
|
-
...(sandboxCodemodeTokenFile
|
|
5847
|
-
? {
|
|
5848
|
-
tokenFile: sandboxCodemodeTokenFile,
|
|
5849
|
-
legacyTokenFile: sandboxEnvironment.OPENGENI_CODEMODE_TOKEN_FILE!,
|
|
5850
|
-
}
|
|
5851
|
-
: {}),
|
|
5852
|
-
...(toolCancellationFenceRef.current
|
|
5853
|
-
? {
|
|
5854
|
-
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
5855
|
-
toolCancellationFenceRef.current,
|
|
5856
|
-
),
|
|
5857
|
-
}
|
|
5858
|
-
: {}),
|
|
5859
|
-
}),
|
|
5860
|
-
);
|
|
6377
|
+
codemodeTokenState.token = material.token;
|
|
5861
6378
|
};
|
|
5862
6379
|
let renewalExpiresAt = initialExpiresAt;
|
|
5863
6380
|
if (renewalExpiresAt.getTime() <= Date.now() + CODEMODE_TOKEN_EXPIRY_LEAD_MS) {
|
|
@@ -5900,6 +6417,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5900
6417
|
codemodeTokenRenewal = controller;
|
|
5901
6418
|
};
|
|
5902
6419
|
|
|
6420
|
+
// A Connected Machine needs renewal, but renewal is purely worker-local:
|
|
6421
|
+
// starting this loop performs no control-plane or machine operation.
|
|
6422
|
+
if (activeSandboxBackend === "selfhosted") {
|
|
6423
|
+
await attachCodemodeTokenRenewal();
|
|
6424
|
+
}
|
|
6425
|
+
|
|
5903
6426
|
const attachRunCredentialRenewal = async (
|
|
5904
6427
|
credentialSession: RunCredentialCommandSession,
|
|
5905
6428
|
initialSandbox?: ResumedTurnSandbox,
|
|
@@ -6045,7 +6568,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6045
6568
|
// applies the agent's manifest to this provided session and throws on ANY
|
|
6046
6569
|
// variableSet delta (validateNoEnvironmentDelta). Passing sandboxEnvironment
|
|
6047
6570
|
// here makes current==target so the delta is empty.
|
|
6048
|
-
if (
|
|
6571
|
+
if (
|
|
6572
|
+
shouldEstablishSandboxForTurn(
|
|
6573
|
+
settings.sandboxOwnershipEnabled,
|
|
6574
|
+
turn.sandboxBackend,
|
|
6575
|
+
machinePrimary,
|
|
6576
|
+
)
|
|
6577
|
+
) {
|
|
6049
6578
|
const managedOwnership = managedSandboxOwnershipForTurn(
|
|
6050
6579
|
machinePrimary,
|
|
6051
6580
|
input.attemptId,
|
|
@@ -6093,6 +6622,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6093
6622
|
opStream: machineOpStream,
|
|
6094
6623
|
epoch: activeSandboxPointer!.activeEpoch,
|
|
6095
6624
|
environment: sandboxEnvironment,
|
|
6625
|
+
...(transientCodemodeEnvironment
|
|
6626
|
+
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
6627
|
+
: {}),
|
|
6096
6628
|
workingDir: activeSandboxPointer!.workingDir,
|
|
6097
6629
|
},
|
|
6098
6630
|
);
|
|
@@ -6122,6 +6654,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6122
6654
|
workspaceId: input.workspaceId,
|
|
6123
6655
|
sessionId: input.sessionId,
|
|
6124
6656
|
environment: sandboxEnvironment,
|
|
6657
|
+
...(transientCodemodeEnvironment
|
|
6658
|
+
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
6659
|
+
: {}),
|
|
6125
6660
|
workspaceMutationFence: {
|
|
6126
6661
|
accountId: input.accountId,
|
|
6127
6662
|
turnId: turn.id,
|
|
@@ -6238,6 +6773,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6238
6773
|
workspaceId: input.workspaceId,
|
|
6239
6774
|
sessionId: input.sessionId,
|
|
6240
6775
|
environment: sandboxEnvironment,
|
|
6776
|
+
...(transientCodemodeEnvironment
|
|
6777
|
+
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
6778
|
+
: {}),
|
|
6241
6779
|
workspaceMutationFence: {
|
|
6242
6780
|
accountId: input.accountId,
|
|
6243
6781
|
turnId: turn.id,
|
|
@@ -6348,11 +6886,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6348
6886
|
}
|
|
6349
6887
|
await publish!([{ type: "tool.auth_needed", payload }], true);
|
|
6350
6888
|
};
|
|
6351
|
-
const
|
|
6889
|
+
const selectedApiIntegrationServerIds = new Set(turnTools.map((tool) => tool.id));
|
|
6352
6890
|
const localMcpServers = buildApiIntegrationServersForTurn({
|
|
6353
6891
|
settings: runSettings,
|
|
6354
6892
|
integrations: installedApiIntegrations.filter((integration) =>
|
|
6355
|
-
|
|
6893
|
+
selectedApiIntegrationServerIds.has(integration.serverId),
|
|
6356
6894
|
),
|
|
6357
6895
|
authority: {
|
|
6358
6896
|
accountId: input.accountId,
|
|
@@ -6396,6 +6934,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6396
6934
|
};
|
|
6397
6935
|
})()
|
|
6398
6936
|
: undefined;
|
|
6937
|
+
const selectedFirstPartyMcpTools = allowedFirstPartyMcpToolsForSession(
|
|
6938
|
+
runSettings,
|
|
6939
|
+
session.firstPartyMcpTools,
|
|
6940
|
+
);
|
|
6399
6941
|
preparedTools = await waitForTurnOperation(
|
|
6400
6942
|
runtime.prepareTools(runSettings, turnTools, {
|
|
6401
6943
|
accountId: input.accountId,
|
|
@@ -6422,7 +6964,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6422
6964
|
...(session.firstPartyMcpPermissions?.length
|
|
6423
6965
|
? { firstPartyPermissions: session.firstPartyMcpPermissions }
|
|
6424
6966
|
: {}),
|
|
6425
|
-
firstPartyTools:
|
|
6967
|
+
firstPartyTools: selectedFirstPartyMcpTools,
|
|
6968
|
+
nestedAgentDepth: session.nestedAgentDepth,
|
|
6969
|
+
effectiveMaxNestedAgentDepth: session.effectiveMaxNestedAgentDepth,
|
|
6426
6970
|
attemptToolDefinitions: createFirstPartyInteractionAttemptToolDefinitions({
|
|
6427
6971
|
settings: runSettings,
|
|
6428
6972
|
scope: {
|
|
@@ -6436,9 +6980,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6436
6980
|
...(session.firstPartyMcpPermissions?.length
|
|
6437
6981
|
? { permissions: session.firstPartyMcpPermissions }
|
|
6438
6982
|
: {}),
|
|
6439
|
-
selectedTools:
|
|
6983
|
+
selectedTools: selectedFirstPartyMcpTools,
|
|
6440
6984
|
subjectId: "worker:first-party-mcp",
|
|
6441
6985
|
subjectLabel: "OpenGeni worker",
|
|
6986
|
+
...(interactionInterventionResume
|
|
6987
|
+
? { interventionResume: interactionInterventionResume }
|
|
6988
|
+
: {}),
|
|
6442
6989
|
}),
|
|
6443
6990
|
}),
|
|
6444
6991
|
cancellationSignal,
|
|
@@ -6546,13 +7093,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6546
7093
|
}
|
|
6547
7094
|
|
|
6548
7095
|
if (resolvedModel?.provider.kind === "codex-subscription") {
|
|
6549
|
-
const
|
|
6550
|
-
|
|
6551
|
-
|
|
6552
|
-
|
|
6553
|
-
|
|
6554
|
-
);
|
|
6555
|
-
}
|
|
7096
|
+
const imageAuthority = connectedSubscriptionImageGenerationAuthority(
|
|
7097
|
+
codexContext,
|
|
7098
|
+
effectiveCodexCredentialId,
|
|
7099
|
+
);
|
|
7100
|
+
if (!imageAuthority) return {};
|
|
6556
7101
|
return {
|
|
6557
7102
|
imageGeneration: {
|
|
6558
7103
|
kind: "provider_adapter",
|
|
@@ -6569,8 +7114,42 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6569
7114
|
toolCallId,
|
|
6570
7115
|
prompt,
|
|
6571
7116
|
references: resolvedReferences,
|
|
6572
|
-
credentialId:
|
|
6573
|
-
codexContext:
|
|
7117
|
+
credentialId: imageAuthority.credentialId,
|
|
7118
|
+
codexContext: imageAuthority.credentialContext,
|
|
7119
|
+
...(runtimeCancellationSignal ? { abortSignal: runtimeCancellationSignal } : {}),
|
|
7120
|
+
});
|
|
7121
|
+
generatedImageReceiptsByArtifactId.set(receipt.artifact.artifactId, receipt);
|
|
7122
|
+
await materializeGeneratedImage(receipt);
|
|
7123
|
+
return receipt;
|
|
7124
|
+
},
|
|
7125
|
+
},
|
|
7126
|
+
};
|
|
7127
|
+
}
|
|
7128
|
+
|
|
7129
|
+
if (resolvedModel?.provider.kind === "xai-subscription") {
|
|
7130
|
+
const imageAuthority = connectedSubscriptionImageGenerationAuthority(
|
|
7131
|
+
xaiRequestContext,
|
|
7132
|
+
effectiveXaiCredentialId,
|
|
7133
|
+
);
|
|
7134
|
+
if (!imageAuthority) return {};
|
|
7135
|
+
return {
|
|
7136
|
+
imageGeneration: {
|
|
7137
|
+
kind: "provider_adapter",
|
|
7138
|
+
execute: async ({ prompt, references }, { toolCallId }) => {
|
|
7139
|
+
const resolvedReferences = await resolveImageReferences(references);
|
|
7140
|
+
const receipt = await executeXaiSubscriptionImageGeneration({
|
|
7141
|
+
db,
|
|
7142
|
+
objectStorage,
|
|
7143
|
+
accountId: input.accountId,
|
|
7144
|
+
workspaceId: input.workspaceId,
|
|
7145
|
+
sessionId: input.sessionId,
|
|
7146
|
+
turnId: turn.id,
|
|
7147
|
+
attemptId: input.attemptId,
|
|
7148
|
+
toolCallId,
|
|
7149
|
+
prompt,
|
|
7150
|
+
references: resolvedReferences,
|
|
7151
|
+
credentialId: imageAuthority.credentialId,
|
|
7152
|
+
xaiContext: imageAuthority.credentialContext,
|
|
6574
7153
|
...(runtimeCancellationSignal ? { abortSignal: runtimeCancellationSignal } : {}),
|
|
6575
7154
|
});
|
|
6576
7155
|
generatedImageReceiptsByArtifactId.set(receipt.artifact.artifactId, receipt);
|
|
@@ -6623,7 +7202,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6623
7202
|
if (objectStorage && videoGenerationEnabled) {
|
|
6624
7203
|
if (videoGenerationPolicy.fundingSource === "opengeni_credits") {
|
|
6625
7204
|
videoGenerationCredential = managedVideoGenerationCredentialLease(modelRunSettings);
|
|
6626
|
-
} else {
|
|
7205
|
+
} else if (videoGenerationPolicy.fundingSource === "workspace_gateway") {
|
|
6627
7206
|
const workspaceCredential = await loadWorkspaceVercelAiGatewayCredentialLease(
|
|
6628
7207
|
db,
|
|
6629
7208
|
modelRunSettings,
|
|
@@ -6635,6 +7214,74 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6635
7214
|
...workspaceCredential,
|
|
6636
7215
|
};
|
|
6637
7216
|
}
|
|
7217
|
+
} else if (videoGenerationPolicy.fundingSource === "supergrok_subscription") {
|
|
7218
|
+
const encryptionKey = environmentsEncryptionKeyBytes(modelRunSettings);
|
|
7219
|
+
const subjectId = xaiLeaseSubjectId ?? turn.initiatingHumanSubjectId;
|
|
7220
|
+
if (encryptionKey && subjectId) {
|
|
7221
|
+
const authoritySnapshot =
|
|
7222
|
+
xaiAuthoritySnapshot ??
|
|
7223
|
+
(await resolveXaiProviderAccountAuthoritySnapshotForAcceptance(db, {
|
|
7224
|
+
workspaceId: input.workspaceId,
|
|
7225
|
+
subjectId,
|
|
7226
|
+
}));
|
|
7227
|
+
const pin = await getXaiSessionAccountPin(db, {
|
|
7228
|
+
workspaceId: input.workspaceId,
|
|
7229
|
+
subjectId,
|
|
7230
|
+
sessionId: input.sessionId,
|
|
7231
|
+
authoritySnapshot,
|
|
7232
|
+
});
|
|
7233
|
+
const selected = effectiveXaiCredentialId
|
|
7234
|
+
? {
|
|
7235
|
+
credentialId: effectiveXaiCredentialId,
|
|
7236
|
+
rotationEnabled: xaiRotationEnabled,
|
|
7237
|
+
}
|
|
7238
|
+
: await selectXaiCredentialForUse(db, {
|
|
7239
|
+
accountId: input.accountId,
|
|
7240
|
+
workspaceId: input.workspaceId,
|
|
7241
|
+
subjectId,
|
|
7242
|
+
authoritySnapshot,
|
|
7243
|
+
shardKey: input.sessionId,
|
|
7244
|
+
pinnedCredentialId: pin?.pinnedCredentialId ?? null,
|
|
7245
|
+
pinSource:
|
|
7246
|
+
pin?.pinSource === "manual" || pin?.pinSource === "policy"
|
|
7247
|
+
? pin.pinSource
|
|
7248
|
+
: null,
|
|
7249
|
+
});
|
|
7250
|
+
if (selected.credentialId) {
|
|
7251
|
+
if (
|
|
7252
|
+
selected.rotationEnabled &&
|
|
7253
|
+
pin?.pinSource !== "manual" &&
|
|
7254
|
+
pin?.pinnedCredentialId !== selected.credentialId
|
|
7255
|
+
) {
|
|
7256
|
+
await setXaiSessionAccountPin(db, {
|
|
7257
|
+
accountId: input.accountId,
|
|
7258
|
+
workspaceId: input.workspaceId,
|
|
7259
|
+
subjectId,
|
|
7260
|
+
sessionId: input.sessionId,
|
|
7261
|
+
authoritySnapshot,
|
|
7262
|
+
credentialId: selected.credentialId,
|
|
7263
|
+
pinSource: "policy",
|
|
7264
|
+
expectedVersion: pin?.version ?? null,
|
|
7265
|
+
}).catch((error: unknown) => {
|
|
7266
|
+
if (error instanceof Error && error.message === "xAI session pin changed") return;
|
|
7267
|
+
throw error;
|
|
7268
|
+
});
|
|
7269
|
+
}
|
|
7270
|
+
const credential = await materializeXaiCredentialForRun(db, {
|
|
7271
|
+
workspaceId: input.workspaceId,
|
|
7272
|
+
subjectId,
|
|
7273
|
+
credentialId: selected.credentialId,
|
|
7274
|
+
authoritySnapshot,
|
|
7275
|
+
encryptionKey,
|
|
7276
|
+
});
|
|
7277
|
+
videoGenerationCredential = xaiVideoGenerationCredentialLease({
|
|
7278
|
+
settings: modelRunSettings,
|
|
7279
|
+
credential,
|
|
7280
|
+
subjectId,
|
|
7281
|
+
authoritySnapshot,
|
|
7282
|
+
});
|
|
7283
|
+
}
|
|
7284
|
+
}
|
|
6638
7285
|
}
|
|
6639
7286
|
}
|
|
6640
7287
|
const videoGenerationOption: Pick<BuildAgentOptions, "videoGeneration"> = (() => {
|
|
@@ -6661,33 +7308,47 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6661
7308
|
resolvedSandbox?.established.session ?? sdkOwnedSandboxSession;
|
|
6662
7309
|
const fence = toolCancellationFenceRef.current;
|
|
6663
7310
|
const runAs = sandboxRunAs(modelRunSettings);
|
|
6664
|
-
|
|
6665
|
-
|
|
6666
|
-
|
|
6667
|
-
|
|
6668
|
-
|
|
6669
|
-
|
|
6670
|
-
|
|
6671
|
-
|
|
6672
|
-
|
|
6673
|
-
|
|
6674
|
-
|
|
6675
|
-
|
|
6676
|
-
|
|
6677
|
-
|
|
6678
|
-
|
|
6679
|
-
|
|
6680
|
-
|
|
6681
|
-
|
|
6682
|
-
|
|
6683
|
-
|
|
6684
|
-
|
|
6685
|
-
|
|
6686
|
-
|
|
6687
|
-
|
|
6688
|
-
|
|
6689
|
-
|
|
6690
|
-
|
|
7311
|
+
let accepted: Awaited<ReturnType<typeof admitVideoGenerationRequest>>;
|
|
7312
|
+
try {
|
|
7313
|
+
accepted = await admitVideoGenerationRequest({
|
|
7314
|
+
db,
|
|
7315
|
+
storage: objectStorage,
|
|
7316
|
+
settings: modelRunSettings,
|
|
7317
|
+
accountId: input.accountId,
|
|
7318
|
+
workspaceId: input.workspaceId,
|
|
7319
|
+
sessionId: input.sessionId,
|
|
7320
|
+
turnId: turn.id,
|
|
7321
|
+
attemptId: input.attemptId,
|
|
7322
|
+
toolCallId,
|
|
7323
|
+
toolInput,
|
|
7324
|
+
policy: videoGenerationPolicy,
|
|
7325
|
+
credential: videoGenerationCredential,
|
|
7326
|
+
...(sessionForReference && fence
|
|
7327
|
+
? {
|
|
7328
|
+
runCommand: async (command) =>
|
|
7329
|
+
await fence.runSandboxCommandStructured(
|
|
7330
|
+
sessionForReference as import("@opengeni/runtime").TurnSandboxCommandSession,
|
|
7331
|
+
{
|
|
7332
|
+
...command,
|
|
7333
|
+
...(runAs ? { runAs } : {}),
|
|
7334
|
+
},
|
|
7335
|
+
),
|
|
7336
|
+
}
|
|
7337
|
+
: {}),
|
|
7338
|
+
...(runtimeCancellationSignal ? { signal: runtimeCancellationSignal } : {}),
|
|
7339
|
+
});
|
|
7340
|
+
} catch (error) {
|
|
7341
|
+
if (error instanceof VideoReferenceInputError) {
|
|
7342
|
+
return VideoGenerationRejectedResult.parse({
|
|
7343
|
+
schemaVersion: 1,
|
|
7344
|
+
status: "rejected",
|
|
7345
|
+
code: error.code,
|
|
7346
|
+
message: error.message,
|
|
7347
|
+
operationCreated: false,
|
|
7348
|
+
});
|
|
7349
|
+
}
|
|
7350
|
+
throw error;
|
|
7351
|
+
}
|
|
6691
7352
|
videoGenerationAcceptancesByCallId.set(toolCallId, {
|
|
6692
7353
|
operationId: accepted.operationId,
|
|
6693
7354
|
requestDigest: accepted.requestDigest,
|
|
@@ -6727,6 +7388,23 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6727
7388
|
outcome,
|
|
6728
7389
|
}),
|
|
6729
7390
|
};
|
|
7391
|
+
const runtimeSkillActivations = [
|
|
7392
|
+
...installedSkillRuntime.activations,
|
|
7393
|
+
...packRuntime.skillActivations,
|
|
7394
|
+
...session.skills.map((skill) => ({
|
|
7395
|
+
source: "session" as const,
|
|
7396
|
+
id: `session:${session.id}:${skill.name}`,
|
|
7397
|
+
artifact: {
|
|
7398
|
+
name: skill.name,
|
|
7399
|
+
description: skill.description ?? null,
|
|
7400
|
+
files: skill.files.map((file) => ({
|
|
7401
|
+
path: file.path,
|
|
7402
|
+
content: file.content,
|
|
7403
|
+
})),
|
|
7404
|
+
},
|
|
7405
|
+
reason: "attached to session",
|
|
7406
|
+
})),
|
|
7407
|
+
];
|
|
6730
7408
|
const agent = runtime.buildAgent(modelRunSettings, turnResources, {
|
|
6731
7409
|
reasoningEffort: turn.reasoningEffort,
|
|
6732
7410
|
latencyMode: turnExecutionPolicy.latencyMode,
|
|
@@ -6761,10 +7439,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6761
7439
|
...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
|
|
6762
7440
|
? { gitTokenSeed: sandboxGitToken }
|
|
6763
7441
|
: {}),
|
|
6764
|
-
|
|
6765
|
-
//
|
|
6766
|
-
//
|
|
6767
|
-
|
|
7442
|
+
...(sandboxCodemodeToken ? { codemodeAvailable: true } : {}),
|
|
7443
|
+
// Managed boxes receive the bearer through their protected per-session
|
|
7444
|
+
// token file. Connected Machines use transient per-exec delivery above,
|
|
7445
|
+
// so they must not run the file-seeding lifecycle hook.
|
|
7446
|
+
...(activeSandboxBackend !== "selfhosted" && sandboxCodemodeToken
|
|
6768
7447
|
? {
|
|
6769
7448
|
codemodeTokenSeed: sandboxCodemodeToken,
|
|
6770
7449
|
codemodeTokenSessionId: input.sessionId,
|
|
@@ -6842,19 +7521,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6842
7521
|
await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
|
|
6843
7522
|
},
|
|
6844
7523
|
onRetainableSessionImageOutput: retainSessionImageAtToolBoundary,
|
|
6845
|
-
...(
|
|
6846
|
-
|
|
6847
|
-
...(skillLibraryRuntime.skillLibrarySkills.length > 0
|
|
6848
|
-
? {
|
|
6849
|
-
skillLibrarySkills: skillLibraryRuntime.skillLibrarySkills,
|
|
6850
|
-
skillLibrarySelections: skillLibraryRuntime.skillLibrarySelections,
|
|
6851
|
-
}
|
|
7524
|
+
...(runtimeSkillActivations.length > 0
|
|
7525
|
+
? { skillActivations: runtimeSkillActivations }
|
|
6852
7526
|
: {}),
|
|
6853
7527
|
...(!structuredWorkspacePolicyActive && workspaceAgentInstructions
|
|
6854
7528
|
? { instructionsTemplate: workspaceAgentInstructions }
|
|
6855
7529
|
: {}),
|
|
6856
7530
|
...(workspaceGovernance ? { workspaceGovernance } : {}),
|
|
6857
7531
|
...(workspaceMemory ? { workspaceMemory } : {}),
|
|
7532
|
+
goalSnapshot: turn.goalSnapshot,
|
|
6858
7533
|
// Per-session persona tier (session > workspace > deployment default).
|
|
6859
7534
|
// Composed system-level AFTER the workspace persona so it refines it for
|
|
6860
7535
|
// this one session; absent ⇒ byte-identical to today's composition.
|
|
@@ -7103,6 +7778,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7103
7778
|
workspaceId: input.workspaceId,
|
|
7104
7779
|
sessionId: input.sessionId,
|
|
7105
7780
|
environment: sandboxEnvironment,
|
|
7781
|
+
...(transientCodemodeEnvironment
|
|
7782
|
+
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
7783
|
+
: {}),
|
|
7106
7784
|
workspaceMutationFence: {
|
|
7107
7785
|
accountId: input.accountId,
|
|
7108
7786
|
turnId: turn.id,
|
|
@@ -7123,6 +7801,40 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7123
7801
|
},
|
|
7124
7802
|
);
|
|
7125
7803
|
}
|
|
7804
|
+
let companyBrainContributionReceiptRecorded = false;
|
|
7805
|
+
const recordCompanyBrainContributionReceiptOnce = (): void => {
|
|
7806
|
+
if (companyBrainContributionReceiptRecorded) return;
|
|
7807
|
+
companyBrainContributionReceiptRecorded = true;
|
|
7808
|
+
try {
|
|
7809
|
+
const companyBrainContributionReceipt = buildCompanyBrainContributionReceipt({
|
|
7810
|
+
attemptId: input.attemptId,
|
|
7811
|
+
turnId: turn.id,
|
|
7812
|
+
nestedAgentDepth: session.nestedAgentDepth,
|
|
7813
|
+
memoryPromptMode,
|
|
7814
|
+
instructionPolicy: instructionPolicySnapshot,
|
|
7815
|
+
workspaceAgentInstructions,
|
|
7816
|
+
preferences: preferenceSnapshot,
|
|
7817
|
+
companyProfile: companyProfileSnapshot,
|
|
7818
|
+
companyProfileIncluded,
|
|
7819
|
+
workspaceMemory,
|
|
7820
|
+
skillActivations: runtimeSkillActivations,
|
|
7821
|
+
});
|
|
7822
|
+
recordCompanyBrainContributions(observability, companyBrainContributionReceipt);
|
|
7823
|
+
observability.info("model context contribution receipt", {
|
|
7824
|
+
attemptId: companyBrainContributionReceipt.attemptId,
|
|
7825
|
+
turnId: companyBrainContributionReceipt.turnId,
|
|
7826
|
+
sessionRole: companyBrainContributionReceipt.sessionRole,
|
|
7827
|
+
memoryPromptMode: companyBrainContributionReceipt.memoryPromptMode,
|
|
7828
|
+
instructionPolicySnapshotId:
|
|
7829
|
+
companyBrainContributionReceipt.instructionPolicySnapshotId,
|
|
7830
|
+
preferenceSnapshotId: companyBrainContributionReceipt.preferenceSnapshotId,
|
|
7831
|
+
companyProfileSnapshotId: companyBrainContributionReceipt.companyProfileSnapshotId,
|
|
7832
|
+
contributions: JSON.stringify(companyBrainContributionReceipt.contributions),
|
|
7833
|
+
});
|
|
7834
|
+
} catch {
|
|
7835
|
+
// Observability must never change model execution semantics.
|
|
7836
|
+
}
|
|
7837
|
+
};
|
|
7126
7838
|
const agentInstructions = typeof agent.instructions === "string" ? agent.instructions : "";
|
|
7127
7839
|
const compactSummarizer = compactionSummarizerFor(
|
|
7128
7840
|
agentInstructions.trim() ? agentInstructions : undefined,
|
|
@@ -7628,9 +8340,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7628
8340
|
preparedInput: runInput!,
|
|
7629
8341
|
...(fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {}),
|
|
7630
8342
|
onRuntimeEvent: async (event) => {
|
|
7631
|
-
await
|
|
7632
|
-
if (
|
|
7633
|
-
throw new Error("
|
|
8343
|
+
await renewServingCredentialLease("runtime_event");
|
|
8344
|
+
if (servingCredentialLeaseLost()) {
|
|
8345
|
+
throw new Error("Provider credential lease expired during sandbox setup");
|
|
7634
8346
|
}
|
|
7635
8347
|
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
7636
8348
|
},
|
|
@@ -7653,14 +8365,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7653
8365
|
// provider may have accepted work even if no response/event follows.
|
|
7654
8366
|
// Escaped MCP setup timeouts are automatically recoverable only
|
|
7655
8367
|
// before this line.
|
|
8368
|
+
recordCompanyBrainContributionReceiptOnce();
|
|
7656
8369
|
modelRequestStarted = true;
|
|
7657
8370
|
return await runtime.runStream(agent, runInput!, modelRunSettings, {
|
|
7658
8371
|
signal: runtimeCancellationSignal,
|
|
7659
8372
|
sandboxEnvironment,
|
|
7660
8373
|
onRuntimeEvent: async (event) => {
|
|
7661
|
-
await
|
|
7662
|
-
if (
|
|
7663
|
-
throw new Error("
|
|
8374
|
+
await renewServingCredentialLease("runtime_event");
|
|
8375
|
+
if (servingCredentialLeaseLost()) {
|
|
8376
|
+
throw new Error("Provider credential lease expired during the active turn");
|
|
7664
8377
|
}
|
|
7665
8378
|
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
7666
8379
|
},
|
|
@@ -7687,7 +8400,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7687
8400
|
},
|
|
7688
8401
|
}
|
|
7689
8402
|
: {}),
|
|
7690
|
-
...(
|
|
8403
|
+
...(activeSandboxBackend !== "selfhosted" &&
|
|
8404
|
+
sandboxCodemodeToken &&
|
|
8405
|
+
sandboxCodemodeTokenExpiresAt &&
|
|
8406
|
+
!lazyOwnedSandbox
|
|
7691
8407
|
? {
|
|
7692
8408
|
onCodemodeTokenSessionReady: async (tokenSession: CodemodeTokenWriterSession) => {
|
|
7693
8409
|
const renewalSession =
|
|
@@ -7734,7 +8450,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7734
8450
|
if (codexLeaseLost) {
|
|
7735
8451
|
throw new Error("Codex credential lease expired before the model run");
|
|
7736
8452
|
}
|
|
7737
|
-
|
|
8453
|
+
if (xaiLeaseLost) {
|
|
8454
|
+
throw new Error("xAI credential lease expired before the model run");
|
|
8455
|
+
}
|
|
8456
|
+
stream = await withProviderRequestContext(runStreamOnce);
|
|
7738
8457
|
// Bounded provider label for the streaming SLIs — the resolved registry
|
|
7739
8458
|
// provider id (or the built-in OpenAI/Azure provider), never a raw
|
|
7740
8459
|
// user-supplied model string.
|
|
@@ -7764,6 +8483,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7764
8483
|
let stableToolCallIdsToClear: string[] | null = null;
|
|
7765
8484
|
let completedCurrentToolBatch = false;
|
|
7766
8485
|
let retainedScreenshotMetadata: RetainedArtifactMetadata | null = null;
|
|
8486
|
+
let normalizedSdkEvents: ReturnType<typeof normalizeSdkEvent> | null = null;
|
|
7767
8487
|
const generatedImage = generatedImageFromSdkEvent(next.value);
|
|
7768
8488
|
if (isCompletedGeneratedImageSdkEvent(next.value) && !generatedImage) {
|
|
7769
8489
|
throw new Error(
|
|
@@ -7795,9 +8515,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7795
8515
|
servingCredentialId: effectiveCodexCredentialId,
|
|
7796
8516
|
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
7797
8517
|
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
7798
|
-
renewLease: () =>
|
|
7799
|
-
leaseLost:
|
|
7800
|
-
leaseLostMessage: "
|
|
8518
|
+
renewLease: () => renewServingCredentialLease("model_usage"),
|
|
8519
|
+
leaseLost: servingCredentialLeaseLost,
|
|
8520
|
+
leaseLostMessage: "Provider credential lease expired during the active turn",
|
|
7801
8521
|
setLastInputTokens: setLastInputTokensFenced,
|
|
7802
8522
|
});
|
|
7803
8523
|
assertModelResponseLatencyMode({
|
|
@@ -7821,6 +8541,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7821
8541
|
currentToolBatchCallIds = new Set<string>();
|
|
7822
8542
|
currentToolBatchCompletedCallIds = new Set<string>();
|
|
7823
8543
|
await reconcileConversationTruth();
|
|
8544
|
+
turnLifecycleMetricsFor(observability).progress(turnId!);
|
|
7824
8545
|
modelCheckpointMemoryCollector.schedule(observability);
|
|
7825
8546
|
try {
|
|
7826
8547
|
await ensureRunAllowed(
|
|
@@ -7936,6 +8657,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7936
8657
|
retainedScreenshotMetadata,
|
|
7937
8658
|
);
|
|
7938
8659
|
}
|
|
8660
|
+
normalizedSdkEvents = normalizeSdkEvent(
|
|
8661
|
+
durableSdkEvent as typeof next.value,
|
|
8662
|
+
retainedScreenshotMetadata
|
|
8663
|
+
? {
|
|
8664
|
+
toolOutputOverride: retainedScreenshotMetadata,
|
|
8665
|
+
retainedOutputEvidence: retainedScreenshotMetadata.available
|
|
8666
|
+
? retainedScreenshotMetadata
|
|
8667
|
+
: {
|
|
8668
|
+
available: false,
|
|
8669
|
+
reason: retainedScreenshotMetadata.reason,
|
|
8670
|
+
},
|
|
8671
|
+
}
|
|
8672
|
+
: {},
|
|
8673
|
+
);
|
|
8674
|
+
const normalizedToolOutput = normalizedSdkEvents.find(
|
|
8675
|
+
(event) =>
|
|
8676
|
+
event.type === "agent.toolCall.output" &&
|
|
8677
|
+
(event.payload as { id?: unknown }).id === completedToolCall.callId,
|
|
8678
|
+
)?.payload as { output?: unknown } | undefined;
|
|
8679
|
+
if (!normalizedToolOutput || !Object.hasOwn(normalizedToolOutput, "output")) {
|
|
8680
|
+
throw new Error(
|
|
8681
|
+
`Completed SDK tool call ${completedToolCall.callId} produced no durable output projection`,
|
|
8682
|
+
);
|
|
8683
|
+
}
|
|
7939
8684
|
const durableResultItem = retainedScreenshotMetadata
|
|
7940
8685
|
? (compactRetainedScreenshotHistory(
|
|
7941
8686
|
[completedToolCall.resultItem],
|
|
@@ -7957,6 +8702,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7957
8702
|
callId: completedToolCall.callId,
|
|
7958
8703
|
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
7959
8704
|
resultItem: durableResultItem as Record<string, unknown>,
|
|
8705
|
+
eventOutput: normalizedToolOutput.output,
|
|
7960
8706
|
...(videoGenerationAcceptancesByCallId.has(completedToolCall.callId)
|
|
7961
8707
|
? {
|
|
7962
8708
|
videoGenerationAcceptance: videoGenerationAcceptancesByCallId.get(
|
|
@@ -8011,24 +8757,31 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8011
8757
|
completedCurrentToolBatch = currentBatchIsStable;
|
|
8012
8758
|
}
|
|
8013
8759
|
}
|
|
8014
|
-
const normalized =
|
|
8015
|
-
|
|
8016
|
-
|
|
8017
|
-
|
|
8018
|
-
|
|
8019
|
-
|
|
8020
|
-
|
|
8021
|
-
:
|
|
8022
|
-
|
|
8023
|
-
|
|
8024
|
-
|
|
8025
|
-
|
|
8026
|
-
|
|
8027
|
-
|
|
8760
|
+
const normalized =
|
|
8761
|
+
normalizedSdkEvents ??
|
|
8762
|
+
normalizeSdkEvent(
|
|
8763
|
+
durableSdkEvent as typeof next.value,
|
|
8764
|
+
retainedScreenshotMetadata
|
|
8765
|
+
? {
|
|
8766
|
+
toolOutputOverride: retainedScreenshotMetadata,
|
|
8767
|
+
retainedOutputEvidence: retainedScreenshotMetadata.available
|
|
8768
|
+
? retainedScreenshotMetadata
|
|
8769
|
+
: {
|
|
8770
|
+
available: false,
|
|
8771
|
+
reason: retainedScreenshotMetadata.reason,
|
|
8772
|
+
},
|
|
8773
|
+
}
|
|
8774
|
+
: {},
|
|
8775
|
+
);
|
|
8028
8776
|
for (const event of normalized) {
|
|
8029
8777
|
streamTiming.onEvent(event.type);
|
|
8030
8778
|
await batcher.push(event);
|
|
8031
8779
|
}
|
|
8780
|
+
// Structural tool-output events await their durable append before
|
|
8781
|
+
// push returns. The complete result is now retained in the event
|
|
8782
|
+
// row and pending-call recovery receipt, so release only our
|
|
8783
|
+
// duplicate live SDK marker before a long turn accumulates it.
|
|
8784
|
+
releaseMcpResultCustomDataFromSdkEvent(durableSdkEvent);
|
|
8032
8785
|
if (stableToolCallIdsToClear) {
|
|
8033
8786
|
const cleared = await clearDurablePendingSessionToolCalls(db, {
|
|
8034
8787
|
accountId: input.accountId,
|
|
@@ -8091,9 +8844,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8091
8844
|
});
|
|
8092
8845
|
let aggregateAuthoritative = false;
|
|
8093
8846
|
await recordCompletedModelCallBeforeOwnershipFences({
|
|
8094
|
-
renewLease: () =>
|
|
8095
|
-
leaseLost:
|
|
8096
|
-
leaseLostMessage: "
|
|
8847
|
+
renewLease: () => renewServingCredentialLease("model_usage"),
|
|
8848
|
+
leaseLost: servingCredentialLeaseLost,
|
|
8849
|
+
leaseLostMessage: "Provider credential lease expired during the active turn",
|
|
8097
8850
|
recordUsage: async () => {
|
|
8098
8851
|
const billing = await recordModelUsageAndDebitCredits(settings, db, {
|
|
8099
8852
|
accountId: input.accountId,
|
|
@@ -8164,6 +8917,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8164
8917
|
const approvals = runtime.serializeApprovals(stream.interruptions);
|
|
8165
8918
|
const humanInputInterruptions =
|
|
8166
8919
|
runtime.serializeHumanInputRequests?.(stream.interruptions) ?? [];
|
|
8920
|
+
const interactionInterventionInterruptions =
|
|
8921
|
+
runtime.serializeInteractionInterventionRequests?.(stream.interruptions) ?? [];
|
|
8167
8922
|
const latestWorkspace = await getWorkspace(db, input.workspaceId);
|
|
8168
8923
|
if (!latestWorkspace) throw new Error(`Workspace not found: ${input.workspaceId}`);
|
|
8169
8924
|
assertWorkspaceHumanInputAllowed(
|
|
@@ -8215,6 +8970,26 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8215
8970
|
},
|
|
8216
8971
|
},
|
|
8217
8972
|
}));
|
|
8973
|
+
const interactionInterventionRequests = interactionInterventionInterruptions.map(
|
|
8974
|
+
(interruption) => ({
|
|
8975
|
+
id: stableInteractionInterventionId(
|
|
8976
|
+
input.sessionId,
|
|
8977
|
+
activeTurnId,
|
|
8978
|
+
interruption.toolCallId,
|
|
8979
|
+
),
|
|
8980
|
+
operationId: stableInteractionInterventionOperationId(
|
|
8981
|
+
input.sessionId,
|
|
8982
|
+
activeTurnId,
|
|
8983
|
+
interruption.toolCallId,
|
|
8984
|
+
),
|
|
8985
|
+
toolCallId: interruption.toolCallId,
|
|
8986
|
+
input: interruption.input,
|
|
8987
|
+
}),
|
|
8988
|
+
);
|
|
8989
|
+
const pendingApprovals = [
|
|
8990
|
+
...approvals,
|
|
8991
|
+
...interactionInterventionInterruptions.map((interruption) => interruption.approval),
|
|
8992
|
+
];
|
|
8218
8993
|
if (
|
|
8219
8994
|
!(await settle!({
|
|
8220
8995
|
events: [
|
|
@@ -8236,11 +9011,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8236
9011
|
sessionStatus: "requires_action",
|
|
8237
9012
|
activeTurnId,
|
|
8238
9013
|
runState: {
|
|
8239
|
-
serializedRunState:
|
|
8240
|
-
pendingApprovals
|
|
9014
|
+
serializedRunState: compactApprovalRunState(stream.state.toString()),
|
|
9015
|
+
pendingApprovals,
|
|
8241
9016
|
humanInputRequests: humanInputRequests.map(
|
|
8242
9017
|
({ isNew: _isNew, ...request }) => request,
|
|
8243
9018
|
),
|
|
9019
|
+
interactionInterventionRequests,
|
|
8244
9020
|
},
|
|
8245
9021
|
}))
|
|
8246
9022
|
) {
|
|
@@ -8798,6 +9574,29 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8798
9574
|
) {
|
|
8799
9575
|
return await settleLostCodexAttempt(turnId, codexLeaseHolderId, codexLeaseGeneration);
|
|
8800
9576
|
}
|
|
9577
|
+
if (xaiLeaseLost && isXaiTurn && publish && turnId && turnStartedPublished) {
|
|
9578
|
+
await flushRuntimeBatcher();
|
|
9579
|
+
await reconcileConversationTruth({ requireDurable: true });
|
|
9580
|
+
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
9581
|
+
sessionId: input.sessionId,
|
|
9582
|
+
turnId,
|
|
9583
|
+
triggerEventId: triggerEventId!,
|
|
9584
|
+
attemptId: input.attemptId,
|
|
9585
|
+
reason: "xai_lease_lost",
|
|
9586
|
+
detail: { provider: "supergrok-subscription" },
|
|
9587
|
+
});
|
|
9588
|
+
if (recovery.action === "stale") {
|
|
9589
|
+
acknowledgeLostAttemptOwnership();
|
|
9590
|
+
activityStatus = "cancelled";
|
|
9591
|
+
turnMetricOutcome = "cancelled";
|
|
9592
|
+
return claimedResult({ status: "cancelled" });
|
|
9593
|
+
}
|
|
9594
|
+
acknowledgeRecoveryQuiescence();
|
|
9595
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
9596
|
+
activityStatus = "recovering";
|
|
9597
|
+
turnMetricOutcome = "recovering";
|
|
9598
|
+
return claimedResult({ status: "recovering" });
|
|
9599
|
+
}
|
|
8801
9600
|
// Definitive Codex credential/account refusals are the only provider
|
|
8802
9601
|
// errors that may walk the pool. This is an explicit checkpoint + SAME
|
|
8803
9602
|
// turn recovery, never an SDK/Temporal blind retry. A network break,
|
|
@@ -8981,6 +9780,143 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8981
9780
|
}
|
|
8982
9781
|
}
|
|
8983
9782
|
}
|
|
9783
|
+
const xaiCredentialFailure =
|
|
9784
|
+
isXaiTurn && effectiveXaiCredentialId ? classifyXaiCredentialFailure(error) : null;
|
|
9785
|
+
if (
|
|
9786
|
+
xaiCredentialFailure &&
|
|
9787
|
+
effectiveXaiCredentialId &&
|
|
9788
|
+
xaiAuthoritySnapshot &&
|
|
9789
|
+
xaiLeaseSubjectId &&
|
|
9790
|
+
xaiLeaseHolderId &&
|
|
9791
|
+
xaiLeaseGeneration !== null &&
|
|
9792
|
+
publish &&
|
|
9793
|
+
turnId &&
|
|
9794
|
+
turnStartedPublished
|
|
9795
|
+
) {
|
|
9796
|
+
await flushRuntimeBatcher();
|
|
9797
|
+
await reconcileConversationTruth({ requireDurable: true });
|
|
9798
|
+
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
9799
|
+
const activeGoal = goal?.status === "active" ? goal : null;
|
|
9800
|
+
const now = new Date();
|
|
9801
|
+
const cooldownUntil =
|
|
9802
|
+
xaiCredentialFailure.kind === "rate_limit"
|
|
9803
|
+
? new Date(now.getTime() + Math.max(1, xaiCredentialFailure.cooldownMs ?? 60_000))
|
|
9804
|
+
: null;
|
|
9805
|
+
const failurePayload = {
|
|
9806
|
+
error:
|
|
9807
|
+
xaiCredentialFailure.kind === "auth"
|
|
9808
|
+
? "The serving SuperGrok account requires reconnection"
|
|
9809
|
+
: xaiCredentialFailure.kind === "forbidden"
|
|
9810
|
+
? "The serving SuperGrok account is not authorized for this request"
|
|
9811
|
+
: "The serving SuperGrok account is temporarily rate limited",
|
|
9812
|
+
code:
|
|
9813
|
+
xaiCredentialFailure.kind === "auth"
|
|
9814
|
+
? "xai_relogin_required"
|
|
9815
|
+
: xaiCredentialFailure.kind === "forbidden"
|
|
9816
|
+
? "xai_account_forbidden"
|
|
9817
|
+
: "xai_account_rate_limited",
|
|
9818
|
+
detail: "the same accepted turn is waiting for another eligible account",
|
|
9819
|
+
};
|
|
9820
|
+
const armed = await armXaiCapacityWait(db, {
|
|
9821
|
+
accountId: input.accountId,
|
|
9822
|
+
workspaceId: input.workspaceId,
|
|
9823
|
+
subjectId: xaiLeaseSubjectId,
|
|
9824
|
+
sessionId: input.sessionId,
|
|
9825
|
+
turnId,
|
|
9826
|
+
attemptId: input.attemptId,
|
|
9827
|
+
workflowId: input.workflowId,
|
|
9828
|
+
authoritySnapshot: xaiAuthoritySnapshot,
|
|
9829
|
+
goalId: activeGoal?.id ?? null,
|
|
9830
|
+
goalVersion: activeGoal?.version ?? null,
|
|
9831
|
+
earliestResetAt: cooldownUntil,
|
|
9832
|
+
failurePayload,
|
|
9833
|
+
leaseFence: {
|
|
9834
|
+
holderId: xaiLeaseHolderId,
|
|
9835
|
+
generation: xaiLeaseGeneration,
|
|
9836
|
+
},
|
|
9837
|
+
credentialQuarantine:
|
|
9838
|
+
xaiCredentialFailure.kind === "auth"
|
|
9839
|
+
? {
|
|
9840
|
+
kind: "status",
|
|
9841
|
+
status: "needs_relogin",
|
|
9842
|
+
lastError: "model request remained unauthorized after refresh",
|
|
9843
|
+
}
|
|
9844
|
+
: xaiCredentialFailure.kind === "forbidden"
|
|
9845
|
+
? {
|
|
9846
|
+
kind: "status",
|
|
9847
|
+
status: "error",
|
|
9848
|
+
lastError: "model request was forbidden for this credential",
|
|
9849
|
+
}
|
|
9850
|
+
: { kind: "cooldown", until: cooldownUntil! },
|
|
9851
|
+
now,
|
|
9852
|
+
});
|
|
9853
|
+
if (armed.action === "waiting") {
|
|
9854
|
+
xaiLeaseHeld = false;
|
|
9855
|
+
xaiCredentialQuarantined = true;
|
|
9856
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, armed.events);
|
|
9857
|
+
const evaluated = await reconcileXaiCapacityWait(db, {
|
|
9858
|
+
accountId: input.accountId,
|
|
9859
|
+
workspaceId: input.workspaceId,
|
|
9860
|
+
sessionId: input.sessionId,
|
|
9861
|
+
waiterId: armed.waiter.id,
|
|
9862
|
+
generation: armed.waiter.generation,
|
|
9863
|
+
now,
|
|
9864
|
+
});
|
|
9865
|
+
if (evaluated.events.length > 0) {
|
|
9866
|
+
await publishDurableSessionEvents(
|
|
9867
|
+
bus,
|
|
9868
|
+
input.workspaceId,
|
|
9869
|
+
input.sessionId,
|
|
9870
|
+
evaluated.events,
|
|
9871
|
+
);
|
|
9872
|
+
}
|
|
9873
|
+
activityError = error;
|
|
9874
|
+
if (evaluated.action === "resumed") {
|
|
9875
|
+
activityStatus = "recovering";
|
|
9876
|
+
turnMetricOutcome = "recovering";
|
|
9877
|
+
return claimedResult({ status: "recovering" });
|
|
9878
|
+
}
|
|
9879
|
+
if (evaluated.action === "waiting") {
|
|
9880
|
+
activityStatus = "waiting_capacity";
|
|
9881
|
+
turnMetricOutcome = "recovering";
|
|
9882
|
+
return claimedResult({
|
|
9883
|
+
status: "waiting_capacity",
|
|
9884
|
+
capacityWait: {
|
|
9885
|
+
provider: "xai",
|
|
9886
|
+
waiterId: evaluated.waiter.id,
|
|
9887
|
+
generation: evaluated.waiter.generation,
|
|
9888
|
+
nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
|
|
9889
|
+
wakeRevision: evaluated.waiter.wakeRevision,
|
|
9890
|
+
},
|
|
9891
|
+
});
|
|
9892
|
+
}
|
|
9893
|
+
acknowledgeLostAttemptOwnership();
|
|
9894
|
+
activityStatus = "cancelled";
|
|
9895
|
+
turnMetricOutcome = "cancelled";
|
|
9896
|
+
return claimedResult({ status: "cancelled" });
|
|
9897
|
+
}
|
|
9898
|
+
|
|
9899
|
+
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
9900
|
+
sessionId: input.sessionId,
|
|
9901
|
+
turnId,
|
|
9902
|
+
triggerEventId: triggerEventId!,
|
|
9903
|
+
attemptId: input.attemptId,
|
|
9904
|
+
reason: "xai_credential_recheck",
|
|
9905
|
+
detail: failurePayload,
|
|
9906
|
+
});
|
|
9907
|
+
if (recovery.action === "stale") {
|
|
9908
|
+
acknowledgeLostAttemptOwnership();
|
|
9909
|
+
activityStatus = "cancelled";
|
|
9910
|
+
turnMetricOutcome = "cancelled";
|
|
9911
|
+
return claimedResult({ status: "cancelled" });
|
|
9912
|
+
}
|
|
9913
|
+
acknowledgeRecoveryQuiescence();
|
|
9914
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
9915
|
+
activityStatus = "recovering";
|
|
9916
|
+
turnMetricOutcome = "recovering";
|
|
9917
|
+
activityError = error;
|
|
9918
|
+
return claimedResult({ status: "recovering" });
|
|
9919
|
+
}
|
|
8984
9920
|
// A ChatGPT/Codex usage cap (429 usage_limit_reached) is account state,
|
|
8985
9921
|
// NOT an agent failure: surface the precise, actionable message (so the
|
|
8986
9922
|
// user sees the reset window) but idle the session — never go terminal,
|
|
@@ -9792,6 +10728,57 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
9792
10728
|
);
|
|
9793
10729
|
codexLeaseHeld = false;
|
|
9794
10730
|
}
|
|
10731
|
+
if (
|
|
10732
|
+
effectiveXaiCredentialId &&
|
|
10733
|
+
xaiLeaseSubjectId &&
|
|
10734
|
+
xaiRequestContext &&
|
|
10735
|
+
!xaiCredentialQuarantined
|
|
10736
|
+
) {
|
|
10737
|
+
const quota = await waitForTurnFinalizerStep(
|
|
10738
|
+
fetchXaiSubscriptionQuota({ context: xaiRequestContext }).catch(() => null),
|
|
10739
|
+
finalizerSignal,
|
|
10740
|
+
);
|
|
10741
|
+
if (quota) {
|
|
10742
|
+
await waitForTurnFinalizerStep(
|
|
10743
|
+
updateXaiQuotaMetadata(db, {
|
|
10744
|
+
workspaceId: input.workspaceId,
|
|
10745
|
+
subjectId: xaiLeaseSubjectId,
|
|
10746
|
+
credentialId: effectiveXaiCredentialId,
|
|
10747
|
+
quotaUsedPercent: quota.usedPercent,
|
|
10748
|
+
quotaResetAt: quota.period?.end ?? null,
|
|
10749
|
+
quotaCheckedAt: quota.checkedAt,
|
|
10750
|
+
exhaustedUntil:
|
|
10751
|
+
quota.usedPercent !== null && quota.usedPercent >= 100
|
|
10752
|
+
? (quota.period?.end ?? null)
|
|
10753
|
+
: null,
|
|
10754
|
+
}).catch(() => undefined),
|
|
10755
|
+
finalizerSignal,
|
|
10756
|
+
);
|
|
10757
|
+
}
|
|
10758
|
+
}
|
|
10759
|
+
if (xaiLeaseHeartbeatTimer) {
|
|
10760
|
+
clearInterval(xaiLeaseHeartbeatTimer);
|
|
10761
|
+
xaiLeaseHeartbeatTimer = undefined;
|
|
10762
|
+
}
|
|
10763
|
+
if (
|
|
10764
|
+
xaiLeaseHeld &&
|
|
10765
|
+
turnId &&
|
|
10766
|
+
xaiLeaseSubjectId &&
|
|
10767
|
+
xaiLeaseHolderId &&
|
|
10768
|
+
xaiLeaseGeneration !== null
|
|
10769
|
+
) {
|
|
10770
|
+
await waitForTurnFinalizerStep(
|
|
10771
|
+
releaseXaiCredentialLease(db, {
|
|
10772
|
+
workspaceId: input.workspaceId,
|
|
10773
|
+
subjectId: xaiLeaseSubjectId,
|
|
10774
|
+
turnId,
|
|
10775
|
+
holderId: xaiLeaseHolderId,
|
|
10776
|
+
generation: xaiLeaseGeneration,
|
|
10777
|
+
}).catch(() => undefined),
|
|
10778
|
+
finalizerSignal,
|
|
10779
|
+
);
|
|
10780
|
+
xaiLeaseHeld = false;
|
|
10781
|
+
}
|
|
9795
10782
|
// Workbench v2 turn-end workspace capture — runs FIRST in
|
|
9796
10783
|
// the turn-end finally, while the box is MAXIMALLY ALIVE. The agent's last
|
|
9797
10784
|
// tool ran before this finally, so /workspace is already final; capture is
|
|
@@ -10057,6 +11044,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
10057
11044
|
? safeErrorForTelemetry(finalizationError ?? activityError)
|
|
10058
11045
|
: undefined,
|
|
10059
11046
|
});
|
|
11047
|
+
// This timer runs only after the activity promise and its full turn
|
|
11048
|
+
// stack unwind. Checkpoint collection alone cannot reclaim objects that
|
|
11049
|
+
// remain strongly reachable until this terminal boundary.
|
|
11050
|
+
turnCompletionMemoryCollector.schedule(observability);
|
|
10060
11051
|
assertPhysicalToolQuiescenceForCancellation({
|
|
10061
11052
|
acknowledgeQuiescence,
|
|
10062
11053
|
physicalToolQuiescenceConfirmed,
|
|
@@ -10169,6 +11160,51 @@ export function isTransientProviderError(error: unknown): boolean {
|
|
|
10169
11160
|
);
|
|
10170
11161
|
}
|
|
10171
11162
|
|
|
11163
|
+
export type XaiCredentialFailure = {
|
|
11164
|
+
kind: "auth" | "forbidden" | "rate_limit";
|
|
11165
|
+
cooldownMs: number | null;
|
|
11166
|
+
};
|
|
11167
|
+
|
|
11168
|
+
/**
|
|
11169
|
+
* Only definitive SuperGrok account refusals may move the same logical turn to
|
|
11170
|
+
* another credential. The marked transport response proves inference was
|
|
11171
|
+
* rejected before a stream began; refresh relogin is equally definitive.
|
|
11172
|
+
*/
|
|
11173
|
+
export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailure | null {
|
|
11174
|
+
let relogin: unknown = error;
|
|
11175
|
+
for (let depth = 0; depth < 6 && relogin && typeof relogin === "object"; depth += 1) {
|
|
11176
|
+
if (relogin instanceof XaiSubscriptionReloginRequired) {
|
|
11177
|
+
return { kind: "auth", cooldownMs: null };
|
|
11178
|
+
}
|
|
11179
|
+
relogin = (relogin as Record<string, unknown>).cause;
|
|
11180
|
+
}
|
|
11181
|
+
if (!isXaiSubscriptionTransportError(error)) return null;
|
|
11182
|
+
let current: unknown = error;
|
|
11183
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
11184
|
+
const value = current as Record<string, unknown>;
|
|
11185
|
+
const body =
|
|
11186
|
+
value.error && typeof value.error === "object"
|
|
11187
|
+
? (value.error as Record<string, unknown>)
|
|
11188
|
+
: null;
|
|
11189
|
+
const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
|
|
11190
|
+
const code = String(value.code ?? body?.code ?? "").toLowerCase();
|
|
11191
|
+
if (status === 401 || code === "unauthorized" || code === "invalid_token") {
|
|
11192
|
+
return { kind: "auth", cooldownMs: null };
|
|
11193
|
+
}
|
|
11194
|
+
if (status === 403) {
|
|
11195
|
+
return { kind: "forbidden", cooldownMs: null };
|
|
11196
|
+
}
|
|
11197
|
+
if (status === 429 || code === "rate_limit_exceeded" || code === "too_many_requests") {
|
|
11198
|
+
return {
|
|
11199
|
+
kind: "rate_limit",
|
|
11200
|
+
cooldownMs: providerRetryAfterMs(error) ?? PROVIDER_BACKPRESSURE_DELAY_MS,
|
|
11201
|
+
};
|
|
11202
|
+
}
|
|
11203
|
+
current = value.cause;
|
|
11204
|
+
}
|
|
11205
|
+
return null;
|
|
11206
|
+
}
|
|
11207
|
+
|
|
10172
11208
|
export function agentRunFailurePayload(
|
|
10173
11209
|
error: unknown,
|
|
10174
11210
|
options: { isCodexTurn?: boolean } = {},
|