@opengeni/worker-bundle 0.20.1 → 0.22.0-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +52 -0
- package/dist/activities/agent-turn/agent-build.d.ts +89 -0
- package/dist/activities/agent-turn/claim.d.ts +67 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
- package/dist/activities/agent-turn/codex.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
- package/dist/activities/agent-turn/errors.d.ts +253 -0
- package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
- package/dist/activities/agent-turn/file-resources.d.ts +37 -0
- package/dist/activities/agent-turn/finalization.d.ts +42 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
- package/dist/activities/agent-turn/governance-model.d.ts +66 -0
- package/dist/activities/agent-turn/history-sink.d.ts +52 -0
- package/dist/activities/agent-turn/history.d.ts +63 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
- package/dist/activities/agent-turn/model-usage.d.ts +218 -0
- package/dist/activities/agent-turn/quiescence.d.ts +135 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
- package/dist/activities/agent-turn/run.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
- package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
- package/dist/activities/agent-turn/turn-context.d.ts +144 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
- package/dist/activities/agent-turn.d.ts +12 -954
- package/dist/activities/automations.d.ts +16 -0
- package/dist/activities/common.d.ts +1 -1
- package/dist/activities/connector-attachments.d.ts +13 -0
- package/dist/activities/documents.d.ts +1 -0
- package/dist/activities/environment.d.ts +26 -15
- package/dist/activities/generated-images.d.ts +1 -1
- package/dist/activities/git-credential-renewal.d.ts +3 -1
- package/dist/activities/goals.d.ts +4 -1
- package/dist/activities/google-drive-provider.d.ts +29 -0
- package/dist/activities/google-drive-publication.d.ts +66 -0
- package/dist/activities/image-generation-references.d.ts +2 -0
- package/dist/activities/incident-telemetry-authority.d.ts +52 -0
- package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +21 -0
- package/dist/activities/knowledge-source-sync.d.ts +33 -1
- package/dist/activities/mcp-credentials.d.ts +9 -3
- package/dist/activities/open-suffix-resume.d.ts +77 -0
- package/dist/activities/packs.d.ts +3 -11
- package/dist/activities/parent-wake.d.ts +19 -0
- package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
- package/dist/activities/retained-screenshots.d.ts +1 -1
- package/dist/activities/rig-verification.d.ts +1 -1
- package/dist/activities/run-input.d.ts +16 -7
- package/dist/activities/sandbox-images.d.ts +1 -14
- package/dist/activities/sandbox-lease.d.ts +21 -3
- package/dist/activities/scheduled-tasks.d.ts +47 -0
- package/dist/activities/types.d.ts +23 -2
- package/dist/activities/video-media-validation.d.ts +1 -1
- package/dist/activities/video-output-retention.d.ts +1 -2
- package/dist/activities/xai-auth.d.ts +3 -0
- package/dist/{activities-control-SMKF2OBE.js → activities-control-DKG22ONJ.js} +2946 -549
- package/dist/activities-control-DKG22ONJ.js.map +1 -0
- package/dist/activities-control.d.ts +6 -2
- package/dist/activities-turn-4IMC3MAA.js +21326 -0
- package/dist/activities-turn-4IMC3MAA.js.map +1 -0
- package/dist/activities.d.ts +7 -3
- package/dist/artifact-materializer-entry.js +41 -22
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
- package/dist/chunk-CE7TPKBS.js.map +1 -0
- package/dist/chunk-CNYGVRKF.js +4698 -0
- package/dist/chunk-CNYGVRKF.js.map +1 -0
- package/dist/connection-provider-use.d.ts +18 -0
- package/dist/editable-artifact-materializer.d.ts +4 -3
- package/dist/github-rest-mcp.d.ts +26 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +58 -14
- package/dist/index.js.map +1 -1
- package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
- package/dist/model-context-contributions.d.ts +8 -4
- package/dist/observability-metrics.d.ts +63 -4
- package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
- package/dist/personal-github-git-credentials.d.ts +9 -0
- package/dist/pr-review-credentials.d.ts +6 -0
- package/dist/sandbox-archive-storage.d.ts +45 -0
- package/dist/sandbox-resume.d.ts +65 -6
- package/dist/sandbox-routing.d.ts +38 -2
- package/dist/scheduled-alert-occurrence.d.ts +33 -0
- package/dist/workflow-bundle.js +87 -28
- package/dist/workflows/activities.d.ts +1 -0
- package/dist/workflows/automations.d.ts +6 -0
- package/dist/workflows/document-index.d.ts +1 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/admission.ts +292 -0
- package/src/activities/agent-turn/agent-build.ts +802 -0
- package/src/activities/agent-turn/claim.ts +590 -0
- package/src/activities/agent-turn/codex-capacity.ts +720 -0
- package/src/activities/agent-turn/codex.ts +94 -0
- package/src/activities/agent-turn/compaction-prep.ts +722 -0
- package/src/activities/agent-turn/credential-leases.ts +240 -0
- package/src/activities/agent-turn/errors.ts +1068 -0
- package/src/activities/agent-turn/failure-settlement.ts +1436 -0
- package/src/activities/agent-turn/file-resources.ts +227 -0
- package/src/activities/agent-turn/finalization.ts +754 -0
- package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
- package/src/activities/agent-turn/governance-model.ts +465 -0
- package/src/activities/agent-turn/history-sink.ts +139 -0
- package/src/activities/agent-turn/history.ts +200 -0
- package/src/activities/agent-turn/media-artifacts.ts +359 -0
- package/src/activities/agent-turn/model-usage.ts +823 -0
- package/src/activities/agent-turn/quiescence.ts +353 -0
- package/src/activities/agent-turn/run-credentials.ts +761 -0
- package/src/activities/agent-turn/run.ts +1530 -0
- package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
- package/src/activities/agent-turn/sandbox-provision.ts +569 -0
- package/src/activities/agent-turn/sandbox-route.ts +358 -0
- package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
- package/src/activities/agent-turn/stream-attempt.ts +1670 -0
- package/src/activities/agent-turn/tool-environment.ts +729 -0
- package/src/activities/agent-turn/tool-policy.ts +216 -0
- package/src/activities/agent-turn/tool-result-spill.ts +265 -0
- package/src/activities/agent-turn/turn-context.ts +281 -0
- package/src/activities/agent-turn/xai-capacity.ts +229 -0
- package/src/activities/agent-turn.ts +12 -12575
- package/src/activities/api-integrations.ts +3 -0
- package/src/activities/automations.ts +214 -0
- package/src/activities/codemode-dispatcher.ts +12 -1
- package/src/activities/common.ts +1 -5
- package/src/activities/connector-attachments.ts +168 -0
- package/src/activities/context-compaction.ts +18 -0
- package/src/activities/documents.ts +6 -4
- package/src/activities/environment.ts +151 -55
- package/src/activities/generated-images.ts +26 -11
- package/src/activities/git-credential-renewal.ts +13 -3
- package/src/activities/goals.ts +85 -19
- package/src/activities/google-drive-provider.ts +178 -0
- package/src/activities/google-drive-publication.ts +938 -0
- package/src/activities/image-generation-references.ts +11 -2
- package/src/activities/incident-telemetry-authority.ts +327 -0
- package/src/activities/incident-telemetry-preflight.ts +317 -0
- package/src/activities/knowledge-source-sync-driver.ts +23 -0
- package/src/activities/knowledge-source-sync.ts +556 -40
- package/src/activities/mcp-credentials.ts +191 -17
- package/src/activities/open-suffix-resume.ts +422 -0
- package/src/activities/packs.ts +12 -60
- package/src/activities/parent-wake.ts +64 -14
- package/src/activities/retained-screenshot-reaper.ts +6 -3
- package/src/activities/retained-screenshots.ts +15 -47
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +128 -136
- package/src/activities/sandbox-images.ts +7 -166
- package/src/activities/sandbox-lease.ts +406 -15
- package/src/activities/scheduled-tasks.ts +2103 -411
- package/src/activities/session-state.ts +10 -2
- package/src/activities/types.ts +32 -2
- package/src/activities/video-media-validation.ts +3 -3
- package/src/activities/video-output-retention.ts +2 -57
- package/src/activities/xai-auth.ts +8 -0
- package/src/activities-control.ts +2 -0
- package/src/activities.ts +3 -0
- package/src/activity-services.ts +13 -4
- package/src/connection-provider-use.ts +36 -0
- package/src/editable-artifact-materializer-subprocess.ts +25 -14
- package/src/editable-artifact-materializer.ts +29 -18
- package/src/github-rest-mcp.ts +713 -0
- package/src/http.ts +16 -6
- package/src/index.ts +40 -5
- package/src/knowledge-source-sync-activity-policy.ts +2 -0
- package/src/model-context-contributions.ts +33 -10
- package/src/observability-metrics.ts +357 -0
- package/src/opensandbox-kubernetes-inventory.ts +292 -0
- package/src/personal-github-git-credentials.ts +331 -0
- package/src/pr-review-credentials.ts +163 -0
- package/src/sandbox-archive-storage.ts +159 -0
- package/src/sandbox-resume.ts +449 -63
- package/src/sandbox-routing.ts +314 -16
- package/src/scheduled-alert-occurrence.ts +176 -0
- package/src/workflows/activities.ts +18 -2
- package/src/workflows/automations.ts +11 -0
- package/src/workflows/session.ts +13 -3
- package/src/workflows.ts +1 -0
- package/dist/activities-control-SMKF2OBE.js.map +0 -1
- package/dist/activities-turn-IDS3WEHA.js +0 -16532
- package/dist/activities-turn-IDS3WEHA.js.map +0 -1
- package/dist/chunk-N5TJIFCE.js.map +0 -1
- package/dist/chunk-P6Z2SLAA.js +0 -3092
- package/dist/chunk-P6Z2SLAA.js.map +0 -1
|
@@ -0,0 +1,1530 @@
|
|
|
1
|
+
import {
|
|
2
|
+
setSessionLastInputTokensForTurnAttempt,
|
|
3
|
+
getMaterializedSandboxFileResources,
|
|
4
|
+
markSandboxFileResourcesMaterialized,
|
|
5
|
+
} from "@opengeni/db";
|
|
6
|
+
import { sandboxOperationMetricObserver } from "@opengeni/observability";
|
|
7
|
+
import {
|
|
8
|
+
REMOTE_COMPACTION_V2_BETA_FEATURE,
|
|
9
|
+
REMOTE_COMPACTION_V2_IMPLEMENTATION,
|
|
10
|
+
materializeSandboxFileDownloads,
|
|
11
|
+
sandboxFileDownloadFailureNote,
|
|
12
|
+
type SandboxFileDownload,
|
|
13
|
+
type SandboxFileDownloadFailure,
|
|
14
|
+
type CodemodeTokenWriterSession,
|
|
15
|
+
} from "@opengeni/runtime";
|
|
16
|
+
import { buildCodexTokenResolver } from "../codex-auth";
|
|
17
|
+
import {
|
|
18
|
+
buildModelResolver,
|
|
19
|
+
CODEX_CLIENT_VERSION,
|
|
20
|
+
CODEX_FALLBACK_MODEL_SLUGS,
|
|
21
|
+
codexRequestStorage,
|
|
22
|
+
withCodexRequestOverrides,
|
|
23
|
+
type CodexRequestContext,
|
|
24
|
+
} from "@opengeni/codex";
|
|
25
|
+
import { xaiSubscriptionRequestStorage } from "@opengeni/xai-subscription";
|
|
26
|
+
import { buildXaiTurnRequestAuthorization } from "../xai-auth";
|
|
27
|
+
import { TurnAttemptFencedError } from "../turn-attempt-fenced";
|
|
28
|
+
import { currentActivityContext } from "../streaming";
|
|
29
|
+
import type {
|
|
30
|
+
TurnActivityServices as ActivityServices,
|
|
31
|
+
RunAgentTurnInput,
|
|
32
|
+
RunAgentTurnResult,
|
|
33
|
+
} from "../types";
|
|
34
|
+
import {
|
|
35
|
+
makeMachineOpObserver,
|
|
36
|
+
modelRequestLifecycleMetricsFor,
|
|
37
|
+
recordModelRequestPhase,
|
|
38
|
+
recordCompanyBrainContributions,
|
|
39
|
+
recordTurnStartupPhase,
|
|
40
|
+
runtimeMetricsHooksForObservability,
|
|
41
|
+
} from "../../observability-metrics";
|
|
42
|
+
import { summarizeCompanyBrainContributions } from "../../model-context-contributions";
|
|
43
|
+
import { ToolResultSpill } from "./tool-result-spill";
|
|
44
|
+
import { createTurnCredentialLeases } from "./credential-leases";
|
|
45
|
+
import { createTurnMediaArtifacts } from "./media-artifacts";
|
|
46
|
+
import { createTurnHistorySink } from "./history-sink";
|
|
47
|
+
import { sandboxRunAs } from "@opengeni/runtime";
|
|
48
|
+
import { randomUUID } from "node:crypto";
|
|
49
|
+
import { createModelCheckpointMemoryCollector } from "../../model-checkpoint-memory-collector";
|
|
50
|
+
|
|
51
|
+
import { codexWorkspaceMetricKey } from "./codex";
|
|
52
|
+
import {
|
|
53
|
+
filterUnmaterializedSandboxFileDownloads,
|
|
54
|
+
sandboxFileMaterializationOutcome,
|
|
55
|
+
sandboxFileDownloadsForRun,
|
|
56
|
+
objectStorageForSandboxDownloads,
|
|
57
|
+
} from "./file-resources";
|
|
58
|
+
import { TurnEventPublisher } from "./model-usage";
|
|
59
|
+
import { waitForTurnOperation } from "./sandbox-provision";
|
|
60
|
+
import { createTurnContext, type EventingState } from "./turn-context";
|
|
61
|
+
import { finalizeTurnAttempt } from "./finalization";
|
|
62
|
+
import { settleTurnFailure } from "./failure-settlement";
|
|
63
|
+
import { runTurnStreamAttempt } from "./stream-attempt";
|
|
64
|
+
import { claimTurnAttempt } from "./claim";
|
|
65
|
+
import { selectCodexTurnCapacity, type CapacityPhaseDeps } from "./codex-capacity";
|
|
66
|
+
import { prepareGovernanceAndModel } from "./governance-model";
|
|
67
|
+
import { prepareCompaction, runPostAgentCompaction } from "./compaction-prep";
|
|
68
|
+
import { createSandboxTurnRuntime } from "./sandbox-runtime";
|
|
69
|
+
import {
|
|
70
|
+
resolveSandboxRoute,
|
|
71
|
+
establishTurnSandbox,
|
|
72
|
+
bindLazySandboxProvisioner,
|
|
73
|
+
} from "./sandbox-establish";
|
|
74
|
+
import { selectXaiTurnCapacity } from "./xai-capacity";
|
|
75
|
+
import { prepareRunCredentials } from "./run-credentials";
|
|
76
|
+
import { prepareTurnToolPolicy, prepareTurnToolRuntime } from "./tool-environment";
|
|
77
|
+
import { applyTurnGitHubRepositoryBindings } from "./github-repository-bindings";
|
|
78
|
+
import { buildTurnAgent } from "./agent-build";
|
|
79
|
+
|
|
80
|
+
/** Lifecycle orchestrator: claim → capacity → governance → sandbox → tools → stream. */
|
|
81
|
+
export function createRunAgentTurnActivity(services: () => Promise<ActivityServices>) {
|
|
82
|
+
const modelCheckpointMemoryCollector = createModelCheckpointMemoryCollector();
|
|
83
|
+
// Keep a distinct cooldown for terminal collection. A collection at the
|
|
84
|
+
// final model checkpoint happens while the activity still owns its complete
|
|
85
|
+
// turn graph and must not suppress collection after that graph is released.
|
|
86
|
+
const turnCompletionMemoryCollector = createModelCheckpointMemoryCollector();
|
|
87
|
+
return async function runAgentTurn(input: RunAgentTurnInput): Promise<RunAgentTurnResult> {
|
|
88
|
+
const {
|
|
89
|
+
settings,
|
|
90
|
+
db,
|
|
91
|
+
bus,
|
|
92
|
+
runtime,
|
|
93
|
+
summarizeContextForCompaction,
|
|
94
|
+
objectStorage,
|
|
95
|
+
observability,
|
|
96
|
+
wakeSessionWorkflow,
|
|
97
|
+
signalSessionAttemptQuiesced,
|
|
98
|
+
signalCodexCapacityWorkflow,
|
|
99
|
+
entitlements,
|
|
100
|
+
connectionCredentials,
|
|
101
|
+
personalGitHubCredentials,
|
|
102
|
+
startVideoGenerationWorkflow,
|
|
103
|
+
} = await services();
|
|
104
|
+
const activityContext = currentActivityContext();
|
|
105
|
+
const cancellationSignal = activityContext?.cancellationSignal;
|
|
106
|
+
// Temporal cancellation is not the only way an activity loses ownership:
|
|
107
|
+
// a database attempt fence or workflow abandonment can finalize the logical
|
|
108
|
+
// turn while the SDK cancellation signal remains open. Bind sandbox
|
|
109
|
+
// provisioning to both lifetimes so an uninterruptible, late provider
|
|
110
|
+
// promise cannot resurrect a holder after this activity has finalized.
|
|
111
|
+
const sandboxResumeController = new AbortController();
|
|
112
|
+
const sandboxResumeSignal = cancellationSignal
|
|
113
|
+
? AbortSignal.any([cancellationSignal, sandboxResumeController.signal])
|
|
114
|
+
: sandboxResumeController.signal;
|
|
115
|
+
const sandboxRotationController = new AbortController();
|
|
116
|
+
const runtimeCancellationSignal = cancellationSignal
|
|
117
|
+
? AbortSignal.any([cancellationSignal, sandboxRotationController.signal])
|
|
118
|
+
: sandboxRotationController.signal;
|
|
119
|
+
|
|
120
|
+
const {
|
|
121
|
+
control,
|
|
122
|
+
attempt,
|
|
123
|
+
billingState,
|
|
124
|
+
sandboxState,
|
|
125
|
+
renewals,
|
|
126
|
+
recordingState,
|
|
127
|
+
eventing,
|
|
128
|
+
workspaceRefs,
|
|
129
|
+
providerTurn,
|
|
130
|
+
} = createTurnContext({
|
|
131
|
+
settings,
|
|
132
|
+
cancellationRequestedAt: cancellationSignal?.aborted ? performance.now() : null,
|
|
133
|
+
});
|
|
134
|
+
const noteCancellationRequested = (): void => {
|
|
135
|
+
control.cancellationRequestedAt ??= performance.now();
|
|
136
|
+
};
|
|
137
|
+
cancellationSignal?.addEventListener("abort", noteCancellationRequested, {
|
|
138
|
+
once: true,
|
|
139
|
+
});
|
|
140
|
+
const dispatchId = activityContext?.info.activityId ?? randomUUID();
|
|
141
|
+
const activityStarted = performance.now();
|
|
142
|
+
const activitySpan = observability.startSpan("worker.run_agent_segment", {
|
|
143
|
+
"opengeni.session_id": input.sessionId,
|
|
144
|
+
"opengeni.workflow_id": input.workflowId,
|
|
145
|
+
"opengeni.trigger_kind": input.trigger.kind,
|
|
146
|
+
});
|
|
147
|
+
const acknowledgeLostAttemptOwnership = (): void => {
|
|
148
|
+
// A stale terminal/recovery settlement can lose either to a benign
|
|
149
|
+
// successor or to Pause/Steer closing this exact attempt. Only the
|
|
150
|
+
// receipt transaction can distinguish those cases after the hard tool
|
|
151
|
+
// fence: allowUninterrupted makes the benign case an event-free no-op.
|
|
152
|
+
control.acknowledgeQuiescence = true;
|
|
153
|
+
noteCancellationRequested();
|
|
154
|
+
};
|
|
155
|
+
const acknowledgeRecoveryQuiescence = (): void => {
|
|
156
|
+
// requestSessionTurnRecovery closed this exact attempt and durably
|
|
157
|
+
// recorded the replacement cause. The activity still owns the physical
|
|
158
|
+
// tool/credential boundary until finally drains it and publishes the
|
|
159
|
+
// exact quiescence receipt; a workflow result alone is never that proof.
|
|
160
|
+
control.acknowledgeQuiescence = true;
|
|
161
|
+
noteCancellationRequested();
|
|
162
|
+
};
|
|
163
|
+
const claimedResult = (
|
|
164
|
+
result: Omit<
|
|
165
|
+
Extract<RunAgentTurnResult, { status: Exclude<RunAgentTurnResult["status"], "unclaimed"> }>,
|
|
166
|
+
"turnId" | "attemptId"
|
|
167
|
+
>,
|
|
168
|
+
): RunAgentTurnResult => {
|
|
169
|
+
if (!attempt.turnId)
|
|
170
|
+
throw new Error("Claimed activity result produced before turn admission");
|
|
171
|
+
return {
|
|
172
|
+
...result,
|
|
173
|
+
turnId: attempt.turnId,
|
|
174
|
+
attemptId: input.attemptId,
|
|
175
|
+
} as RunAgentTurnResult;
|
|
176
|
+
};
|
|
177
|
+
// The Connected Machine op observer for this turn: meters every op AND buffers
|
|
178
|
+
// the eventable ones (infra failures + healed recoveries) as machine.op.* session
|
|
179
|
+
// events, drained (awaited) at turn end in the finally below. ONE instance shared
|
|
180
|
+
// by the machine-primary establish + both routing wraps.
|
|
181
|
+
const machineOpObserver = makeMachineOpObserver(
|
|
182
|
+
runtimeMetricsHooksForObservability(observability),
|
|
183
|
+
);
|
|
184
|
+
const sandboxOperationObserver = sandboxOperationMetricObserver(observability);
|
|
185
|
+
// Still required by credential-loss/capacity settlements, whose own
|
|
186
|
+
// recovery transactions fence against worker-death redispatches.
|
|
187
|
+
const setLastInputTokensFenced = async (lastInputTokens: number | null): Promise<void> => {
|
|
188
|
+
if (!attempt.turnId || attempt.executionGeneration <= 0) {
|
|
189
|
+
throw new Error("Turn attempt was not initialized before token accounting");
|
|
190
|
+
}
|
|
191
|
+
if (
|
|
192
|
+
!(await setSessionLastInputTokensForTurnAttempt(db, {
|
|
193
|
+
workspaceId: input.workspaceId,
|
|
194
|
+
sessionId: input.sessionId,
|
|
195
|
+
turnId: attempt.turnId,
|
|
196
|
+
expectedExecutionGeneration: attempt.executionGeneration,
|
|
197
|
+
expectedAttemptId: input.attemptId,
|
|
198
|
+
lastInputTokens,
|
|
199
|
+
}))
|
|
200
|
+
) {
|
|
201
|
+
throw new TurnAttemptFencedError("turn attempt was fenced while recording input tokens");
|
|
202
|
+
}
|
|
203
|
+
};
|
|
204
|
+
const codexWorkspaceKey = codexWorkspaceMetricKey(input.workspaceId);
|
|
205
|
+
const leases = createTurnCredentialLeases({
|
|
206
|
+
db,
|
|
207
|
+
observability,
|
|
208
|
+
accountId: input.accountId,
|
|
209
|
+
workspaceId: input.workspaceId,
|
|
210
|
+
codexWorkspaceKey,
|
|
211
|
+
getTurnId: () => attempt.turnId,
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
const sandboxRuntime = createSandboxTurnRuntime({
|
|
215
|
+
input,
|
|
216
|
+
settings,
|
|
217
|
+
db,
|
|
218
|
+
objectStorage,
|
|
219
|
+
observability,
|
|
220
|
+
cancellationSignal,
|
|
221
|
+
activityContext,
|
|
222
|
+
sandboxRotationController,
|
|
223
|
+
sandboxState,
|
|
224
|
+
recordingState,
|
|
225
|
+
eventing,
|
|
226
|
+
attempt,
|
|
227
|
+
});
|
|
228
|
+
const {
|
|
229
|
+
requireResolvedSandboxForMutation,
|
|
230
|
+
runWorkspaceMutationForSandbox,
|
|
231
|
+
stopLeaseHeartbeat,
|
|
232
|
+
abandonActiveRecording,
|
|
233
|
+
flushRuntimeBatcher,
|
|
234
|
+
maybeStartOnTurnRecording,
|
|
235
|
+
finalizeTurnOpStreamOps,
|
|
236
|
+
} = sandboxRuntime;
|
|
237
|
+
|
|
238
|
+
const media = createTurnMediaArtifacts({
|
|
239
|
+
db,
|
|
240
|
+
objectStorage,
|
|
241
|
+
observability,
|
|
242
|
+
accountId: input.accountId,
|
|
243
|
+
workspaceId: input.workspaceId,
|
|
244
|
+
sessionId: input.sessionId,
|
|
245
|
+
attemptId: input.attemptId,
|
|
246
|
+
getTurnId: () => attempt.turnId,
|
|
247
|
+
getModelRunSettings: () => eventing.modelRunSettings,
|
|
248
|
+
getPublish: () => eventing.publish,
|
|
249
|
+
toolCancellationFenceRef: eventing.toolCancellationFenceRef,
|
|
250
|
+
getResolvedSandbox: () => sandboxState.resolvedSandbox,
|
|
251
|
+
getSetupBoxSession: () => sandboxState.setupBoxSession,
|
|
252
|
+
getSandboxGroupId: () => sandboxState.sandboxGroupId,
|
|
253
|
+
runWorkspaceMutation: runWorkspaceMutationForSandbox,
|
|
254
|
+
});
|
|
255
|
+
const videoGenerationAcceptancesByCallId = new Map<
|
|
256
|
+
string,
|
|
257
|
+
{ operationId: string; requestDigest: string }
|
|
258
|
+
>();
|
|
259
|
+
const requiredGeneratedVideoFiles: Array<{
|
|
260
|
+
operationId: string;
|
|
261
|
+
artifactId: string;
|
|
262
|
+
fileId: string;
|
|
263
|
+
objectKey: string;
|
|
264
|
+
sizeBytes: number;
|
|
265
|
+
sha256: string;
|
|
266
|
+
filename: string;
|
|
267
|
+
}> = [];
|
|
268
|
+
const toolResultSpill = new ToolResultSpill({
|
|
269
|
+
db,
|
|
270
|
+
objectStorage,
|
|
271
|
+
observability,
|
|
272
|
+
accountId: input.accountId,
|
|
273
|
+
workspaceId: input.workspaceId,
|
|
274
|
+
attemptId: input.attemptId,
|
|
275
|
+
getModelRunSettings: () => eventing.modelRunSettings,
|
|
276
|
+
getSandboxFileDownloadBackend: () => media.sandboxFileDownloadBackend,
|
|
277
|
+
getPublish: () => {
|
|
278
|
+
const current = eventing.publish;
|
|
279
|
+
if (!current) return null;
|
|
280
|
+
return async (events, immediate) =>
|
|
281
|
+
await current(events as Parameters<TurnEventPublisher>[0], immediate);
|
|
282
|
+
},
|
|
283
|
+
toolCancellationFenceRef: eventing.toolCancellationFenceRef,
|
|
284
|
+
getResolvedSandbox: () => sandboxState.resolvedSandbox,
|
|
285
|
+
getSetupBoxSession: () => sandboxState.setupBoxSession,
|
|
286
|
+
getSdkOwnedSandboxSession: () => media.sdkOwnedSandboxSession,
|
|
287
|
+
getSandboxGroupId: () => sandboxState.sandboxGroupId,
|
|
288
|
+
runWorkspaceMutation: runWorkspaceMutationForSandbox,
|
|
289
|
+
});
|
|
290
|
+
const historySink = createTurnHistorySink({
|
|
291
|
+
db,
|
|
292
|
+
accountId: input.accountId,
|
|
293
|
+
workspaceId: input.workspaceId,
|
|
294
|
+
sessionId: input.sessionId,
|
|
295
|
+
attemptId: input.attemptId,
|
|
296
|
+
media,
|
|
297
|
+
getTurnId: () => attempt.turnId,
|
|
298
|
+
getStream: () => eventing.stream,
|
|
299
|
+
getModelRunSettings: () => eventing.modelRunSettings,
|
|
300
|
+
getExecutionGeneration: () => attempt.executionGeneration,
|
|
301
|
+
});
|
|
302
|
+
|
|
303
|
+
try {
|
|
304
|
+
const claimed = await claimTurnAttempt({
|
|
305
|
+
input,
|
|
306
|
+
settings,
|
|
307
|
+
db,
|
|
308
|
+
bus,
|
|
309
|
+
runtime,
|
|
310
|
+
objectStorage,
|
|
311
|
+
observability,
|
|
312
|
+
entitlements,
|
|
313
|
+
wakeSessionWorkflow,
|
|
314
|
+
cancellationSignal,
|
|
315
|
+
activityContext,
|
|
316
|
+
dispatchId,
|
|
317
|
+
activityStarted,
|
|
318
|
+
control,
|
|
319
|
+
attempt,
|
|
320
|
+
billingState,
|
|
321
|
+
sandboxState,
|
|
322
|
+
recordingState,
|
|
323
|
+
eventing,
|
|
324
|
+
leases,
|
|
325
|
+
media,
|
|
326
|
+
claimedResult,
|
|
327
|
+
acknowledgeLostAttemptOwnership,
|
|
328
|
+
abandonActiveRecording,
|
|
329
|
+
});
|
|
330
|
+
if ("exit" in claimed) return claimed.exit;
|
|
331
|
+
if (!eventing.publish || !eventing.settle) {
|
|
332
|
+
throw new Error("turn eventing was not wired during claim");
|
|
333
|
+
}
|
|
334
|
+
// Same object, narrowed type: every post-claim phase mutates this exact
|
|
335
|
+
// context, so this must stay an assertion and never become a copy.
|
|
336
|
+
const wiredEventing = eventing as EventingState & {
|
|
337
|
+
publish: NonNullable<EventingState["publish"]>;
|
|
338
|
+
settle: NonNullable<EventingState["settle"]>;
|
|
339
|
+
};
|
|
340
|
+
const {
|
|
341
|
+
turn,
|
|
342
|
+
session,
|
|
343
|
+
installedApiIntegrations,
|
|
344
|
+
credentialSubjectId,
|
|
345
|
+
fileAuthoritySubjectId,
|
|
346
|
+
capabilitySettings,
|
|
347
|
+
codexAppsCredentialId,
|
|
348
|
+
turnExecutionPolicy,
|
|
349
|
+
trigger,
|
|
350
|
+
humanInputResume,
|
|
351
|
+
interactionInterventionResume,
|
|
352
|
+
throwIfWorkerShuttingDown,
|
|
353
|
+
throwIfTurnCancelled,
|
|
354
|
+
opJournal,
|
|
355
|
+
modelUsageDispatchId,
|
|
356
|
+
claimedModelUsageSourceKeys,
|
|
357
|
+
emittedModelUsageSourceKeys,
|
|
358
|
+
} = claimed.ok;
|
|
359
|
+
if (!attempt.turnId) {
|
|
360
|
+
throw new Error("Turn id was not initialized");
|
|
361
|
+
}
|
|
362
|
+
if (!leases.codex.holderId) {
|
|
363
|
+
throw new Error("Codex lease holder was not initialized");
|
|
364
|
+
}
|
|
365
|
+
const workerPreparationStartedAt = performance.now();
|
|
366
|
+
|
|
367
|
+
// Multi-account (P1): resolve the effective Codex account for this turn
|
|
368
|
+
// (session-pin > workspace active) and stamp it on the session so the
|
|
369
|
+
// in-session "Running on:" indicator reflects reality. Emit a switch event
|
|
370
|
+
// when it changed from the prior run's account so the pill flips live.
|
|
371
|
+
// Gated on the codex-billed predicate — non-codex turns never touch this.
|
|
372
|
+
{
|
|
373
|
+
const capacityDeps: CapacityPhaseDeps = {
|
|
374
|
+
input,
|
|
375
|
+
settings,
|
|
376
|
+
db,
|
|
377
|
+
bus,
|
|
378
|
+
observability,
|
|
379
|
+
wakeSessionWorkflow,
|
|
380
|
+
signalCodexCapacityWorkflow,
|
|
381
|
+
cancellationSignal,
|
|
382
|
+
dispatchId,
|
|
383
|
+
control,
|
|
384
|
+
attempt,
|
|
385
|
+
billingState,
|
|
386
|
+
eventing: wiredEventing,
|
|
387
|
+
providerTurn,
|
|
388
|
+
leases,
|
|
389
|
+
claimedResult,
|
|
390
|
+
acknowledgeLostAttemptOwnership,
|
|
391
|
+
acknowledgeRecoveryQuiescence,
|
|
392
|
+
setLastInputTokensFenced,
|
|
393
|
+
turn,
|
|
394
|
+
session,
|
|
395
|
+
turnExecutionPolicy,
|
|
396
|
+
trigger,
|
|
397
|
+
codexWorkspaceKey,
|
|
398
|
+
};
|
|
399
|
+
const codexCapacity = await selectCodexTurnCapacity(capacityDeps);
|
|
400
|
+
if ("exit" in codexCapacity) return codexCapacity.exit;
|
|
401
|
+
const xaiCapacity = await selectXaiTurnCapacity(capacityDeps);
|
|
402
|
+
if ("exit" in xaiCapacity) return xaiCapacity.exit;
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
const governance = await prepareGovernanceAndModel({
|
|
406
|
+
input,
|
|
407
|
+
db,
|
|
408
|
+
runtime,
|
|
409
|
+
objectStorage,
|
|
410
|
+
eventing,
|
|
411
|
+
workspaceRefs,
|
|
412
|
+
media,
|
|
413
|
+
turn,
|
|
414
|
+
session,
|
|
415
|
+
capabilitySettings,
|
|
416
|
+
fileAuthoritySubjectId,
|
|
417
|
+
humanInputResume,
|
|
418
|
+
turnExecutionPolicy,
|
|
419
|
+
requiredGeneratedVideoFiles,
|
|
420
|
+
});
|
|
421
|
+
if ("exit" in governance) return governance.exit;
|
|
422
|
+
const {
|
|
423
|
+
runtimePreparationStartedAt,
|
|
424
|
+
packRuntime,
|
|
425
|
+
installedSkillRuntime,
|
|
426
|
+
rigVersion,
|
|
427
|
+
rigName,
|
|
428
|
+
agentHumanInputEnabled,
|
|
429
|
+
workspaceAgentInstructions,
|
|
430
|
+
workspaceGovernance,
|
|
431
|
+
structuredWorkspacePolicyActive,
|
|
432
|
+
workspaceMemory,
|
|
433
|
+
buildCompanyBrainContributionReceiptFor,
|
|
434
|
+
logicalSandboxSettings,
|
|
435
|
+
verifiedRigProviderImageId,
|
|
436
|
+
runSettings,
|
|
437
|
+
resolvedModel,
|
|
438
|
+
providerApi,
|
|
439
|
+
nativeImageProviderBinding,
|
|
440
|
+
lazyToolTransport,
|
|
441
|
+
modelInputPolicy,
|
|
442
|
+
supportsImageInput,
|
|
443
|
+
modelHistoryProjector,
|
|
444
|
+
generatedImageHistoryProjector,
|
|
445
|
+
compactionModelHistoryProjector,
|
|
446
|
+
} = governance.ok;
|
|
447
|
+
|
|
448
|
+
// A codex-subscription turn resolves the bearer for THIS turn's effective
|
|
449
|
+
// codex account (effectiveCodexCredentialId; pin > workspace-active) at
|
|
450
|
+
// model-call time — multi-account P1 means a workspace can hold N accounts,
|
|
451
|
+
// so the bearer is per-account, not per-workspace. codexSubscriptionFetch
|
|
452
|
+
// (on the provider's OpenAI client) reads this AsyncLocalStorage context.
|
|
453
|
+
// Build it once and wrap BOTH the compaction summarizer (a separate model
|
|
454
|
+
// call on the same codex client) and the main run; otherwise the summarizer
|
|
455
|
+
// would hit the codex backend unauthenticated.
|
|
456
|
+
let codexModelRequestSequence = 0;
|
|
457
|
+
let firstModelRequestAuditRecorded = false;
|
|
458
|
+
const firstModelRequestCheckpoints = new Set<string>();
|
|
459
|
+
const codexContext: CodexRequestContext | null =
|
|
460
|
+
resolvedModel?.provider.kind === "codex-subscription"
|
|
461
|
+
? ((): CodexRequestContext => {
|
|
462
|
+
// The empty-string fallback yields no row → null credential → the
|
|
463
|
+
// existing CodexReloginRequired path (a codex turn with no usable
|
|
464
|
+
// account fails closed, exactly as before multi-account).
|
|
465
|
+
const resolver = buildCodexTokenResolver(
|
|
466
|
+
db,
|
|
467
|
+
runSettings,
|
|
468
|
+
input.workspaceId,
|
|
469
|
+
providerTurn.effectiveCodexCredentialId ?? "",
|
|
470
|
+
);
|
|
471
|
+
return {
|
|
472
|
+
clientVersion: CODEX_CLIENT_VERSION,
|
|
473
|
+
// Backend sticky cache-routing key — the SAME id as the body's
|
|
474
|
+
// prompt_cache_key (set from input.sessionId for codex turns),
|
|
475
|
+
// so routing and cache key agree. Without this header on the
|
|
476
|
+
// wire, byte-identical resends hit the prompt cache ~50%
|
|
477
|
+
// (per-request shard lottery = prod's measured 48.6% on sol);
|
|
478
|
+
// with it, resends pin to the warm shard (Codex CLI parity).
|
|
479
|
+
sessionId: input.sessionId,
|
|
480
|
+
getToken: () => resolver.getToken(),
|
|
481
|
+
refresh: () => resolver.refresh(),
|
|
482
|
+
resolveModel: buildModelResolver(
|
|
483
|
+
CODEX_FALLBACK_MODEL_SLUGS,
|
|
484
|
+
CODEX_FALLBACK_MODEL_SLUGS[0],
|
|
485
|
+
),
|
|
486
|
+
onUsageHeaders: (snapshot) => {
|
|
487
|
+
providerTurn.latestCodexUsage = snapshot;
|
|
488
|
+
}, // latest wins; flushed once in finally
|
|
489
|
+
onRequestPreparationDiagnostic: (phase) => {
|
|
490
|
+
if (
|
|
491
|
+
eventing.firstModelRequestCheckpointAt === null ||
|
|
492
|
+
firstModelRequestCheckpoints.has(phase)
|
|
493
|
+
) {
|
|
494
|
+
return;
|
|
495
|
+
}
|
|
496
|
+
const now = performance.now();
|
|
497
|
+
const metricPhase =
|
|
498
|
+
phase === "transport_entry"
|
|
499
|
+
? "model_sdk_serialization"
|
|
500
|
+
: phase === "credential_ready"
|
|
501
|
+
? "model_credential_resolution"
|
|
502
|
+
: "model_wire_normalization";
|
|
503
|
+
firstModelRequestCheckpoints.add(phase);
|
|
504
|
+
recordTurnStartupPhase(observability, {
|
|
505
|
+
phase: metricPhase,
|
|
506
|
+
provider: turnExecutionPolicy.providerId,
|
|
507
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
508
|
+
outcome: "completed",
|
|
509
|
+
durationSeconds: (now - eventing.firstModelRequestCheckpointAt) / 1_000,
|
|
510
|
+
count: turnTools.length,
|
|
511
|
+
});
|
|
512
|
+
eventing.firstModelRequestCheckpointAt = now;
|
|
513
|
+
},
|
|
514
|
+
onRequestOpaqueArtifacts: ({ fingerprints }) => {
|
|
515
|
+
providerTurn.lastCodexRequestOpaqueArtifacts = fingerprints;
|
|
516
|
+
},
|
|
517
|
+
onModelRequestDiagnostic: (event) => {
|
|
518
|
+
if (event.phase === "started") sandboxState.firstProviderRequestStarted = true;
|
|
519
|
+
if (
|
|
520
|
+
event.phase === "started" &&
|
|
521
|
+
eventing.firstModelRequestPreparationStartedAt !== null &&
|
|
522
|
+
!eventing.firstModelRequestPreparationRecorded
|
|
523
|
+
) {
|
|
524
|
+
eventing.firstModelRequestPreparationRecorded = true;
|
|
525
|
+
recordTurnStartupPhase(observability, {
|
|
526
|
+
phase: "model_request_preparation",
|
|
527
|
+
provider: turnExecutionPolicy.providerId,
|
|
528
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
529
|
+
outcome: "completed",
|
|
530
|
+
durationSeconds:
|
|
531
|
+
(performance.now() - eventing.firstModelRequestPreparationStartedAt) /
|
|
532
|
+
1_000,
|
|
533
|
+
count: turnTools.length,
|
|
534
|
+
});
|
|
535
|
+
}
|
|
536
|
+
const phase =
|
|
537
|
+
event.phase === "headers"
|
|
538
|
+
? "headers"
|
|
539
|
+
: event.phase === "first_byte"
|
|
540
|
+
? "first_byte"
|
|
541
|
+
: event.phase === "completed" ||
|
|
542
|
+
event.phase === "failed" ||
|
|
543
|
+
event.phase === "timed_out"
|
|
544
|
+
? "terminal"
|
|
545
|
+
: null;
|
|
546
|
+
if (!phase) return;
|
|
547
|
+
recordModelRequestPhase(observability, {
|
|
548
|
+
provider: "codex-subscription",
|
|
549
|
+
phase,
|
|
550
|
+
...(event.phase === "completed"
|
|
551
|
+
? { outcome: "completed" as const }
|
|
552
|
+
: event.phase === "failed"
|
|
553
|
+
? { outcome: "failed" as const }
|
|
554
|
+
: event.phase === "timed_out"
|
|
555
|
+
? { outcome: "timed_out" as const }
|
|
556
|
+
: {}),
|
|
557
|
+
durationSeconds: event.durationMs / 1000,
|
|
558
|
+
});
|
|
559
|
+
},
|
|
560
|
+
nextRequestId: () => `${dispatchId}:${++codexModelRequestSequence}`,
|
|
561
|
+
onModelRequestEvent: async (event) => {
|
|
562
|
+
if (!eventing.publish || !attempt.turnId) {
|
|
563
|
+
throw new Error("Codex model request started before the turn event producer");
|
|
564
|
+
}
|
|
565
|
+
const shouldRecordStartedAudit =
|
|
566
|
+
event.phase === "started" && !firstModelRequestAuditRecorded;
|
|
567
|
+
if (shouldRecordStartedAudit) {
|
|
568
|
+
firstModelRequestAuditRecorded = true;
|
|
569
|
+
}
|
|
570
|
+
const auditStartedAt = shouldRecordStartedAudit ? performance.now() : null;
|
|
571
|
+
let auditOutcome: "completed" | "failed" = "completed";
|
|
572
|
+
try {
|
|
573
|
+
await eventing.publish([
|
|
574
|
+
...(shouldRecordStartedAudit &&
|
|
575
|
+
eventing.firstModelRequestPreparationStartedAt !== null
|
|
576
|
+
? [
|
|
577
|
+
{
|
|
578
|
+
type: "turn.startup.phase.completed" as const,
|
|
579
|
+
payload: {
|
|
580
|
+
phase: "model_preparation",
|
|
581
|
+
durationMs: Math.max(
|
|
582
|
+
0,
|
|
583
|
+
Math.round(
|
|
584
|
+
performance.now() -
|
|
585
|
+
eventing.firstModelRequestPreparationStartedAt,
|
|
586
|
+
),
|
|
587
|
+
),
|
|
588
|
+
},
|
|
589
|
+
},
|
|
590
|
+
]
|
|
591
|
+
: []),
|
|
592
|
+
{
|
|
593
|
+
type: "agent.model.request",
|
|
594
|
+
payload: {
|
|
595
|
+
...event,
|
|
596
|
+
provider: "codex-subscription",
|
|
597
|
+
turnId: attempt.turnId,
|
|
598
|
+
attemptId: input.attemptId,
|
|
599
|
+
dispatchId,
|
|
600
|
+
executionGeneration: attempt.executionGeneration,
|
|
601
|
+
},
|
|
602
|
+
},
|
|
603
|
+
]);
|
|
604
|
+
} catch (error) {
|
|
605
|
+
auditOutcome = "failed";
|
|
606
|
+
throw error;
|
|
607
|
+
} finally {
|
|
608
|
+
if (auditStartedAt !== null) {
|
|
609
|
+
recordTurnStartupPhase(observability, {
|
|
610
|
+
phase: "model_request_audit",
|
|
611
|
+
provider: turnExecutionPolicy.providerId,
|
|
612
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
613
|
+
outcome: auditOutcome,
|
|
614
|
+
durationSeconds: (performance.now() - auditStartedAt) / 1_000,
|
|
615
|
+
count: turnTools.length,
|
|
616
|
+
});
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
},
|
|
620
|
+
};
|
|
621
|
+
})()
|
|
622
|
+
: null;
|
|
623
|
+
if (resolvedModel?.provider.kind === "xai-subscription") {
|
|
624
|
+
if (!providerTurn.effectiveXaiCredentialId || !leases.xai.subjectId) {
|
|
625
|
+
throw new Error("SuperGrok subscription execution has no leased credential");
|
|
626
|
+
}
|
|
627
|
+
let xaiModelRequestSequence = 0;
|
|
628
|
+
const authorization = await buildXaiTurnRequestAuthorization({
|
|
629
|
+
db,
|
|
630
|
+
settings: runSettings,
|
|
631
|
+
accountId: input.accountId,
|
|
632
|
+
workspaceId: input.workspaceId,
|
|
633
|
+
subjectId: leases.xai.subjectId,
|
|
634
|
+
sessionId: input.sessionId,
|
|
635
|
+
turnId: turn.id,
|
|
636
|
+
credentialId: providerTurn.effectiveXaiCredentialId,
|
|
637
|
+
authoritySnapshot: turn.xaiProviderAccountAuthoritySnapshot,
|
|
638
|
+
hostedSearch: {
|
|
639
|
+
webSearch: runSettings.webSearchEnabled,
|
|
640
|
+
xSearch: runSettings.webSearchEnabled,
|
|
641
|
+
},
|
|
642
|
+
streamIdleTimeoutMs: runSettings.supergrokResponseStreamIdleTimeoutMs,
|
|
643
|
+
nextRequestId: () => `${dispatchId}:xai:${++xaiModelRequestSequence}`,
|
|
644
|
+
onModelRequestDiagnostic: (event) => {
|
|
645
|
+
const requestKey = `${event.requestId}:${event.transportAttempt}`;
|
|
646
|
+
if (event.phase === "started") {
|
|
647
|
+
sandboxState.firstProviderRequestStarted = true;
|
|
648
|
+
modelRequestLifecycleMetricsFor(observability).start(
|
|
649
|
+
requestKey,
|
|
650
|
+
"supergrok-subscription",
|
|
651
|
+
);
|
|
652
|
+
if (
|
|
653
|
+
eventing.firstModelRequestPreparationStartedAt !== null &&
|
|
654
|
+
!eventing.firstModelRequestPreparationRecorded
|
|
655
|
+
) {
|
|
656
|
+
eventing.firstModelRequestPreparationRecorded = true;
|
|
657
|
+
recordTurnStartupPhase(observability, {
|
|
658
|
+
phase: "model_request_preparation",
|
|
659
|
+
provider: turnExecutionPolicy.providerId,
|
|
660
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
661
|
+
outcome: "completed",
|
|
662
|
+
durationSeconds:
|
|
663
|
+
(performance.now() - eventing.firstModelRequestPreparationStartedAt) / 1_000,
|
|
664
|
+
count: turnTools.length,
|
|
665
|
+
});
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
if (event.phase === "first_event" || event.phase === "progress") {
|
|
669
|
+
modelRequestLifecycleMetricsFor(observability).event(
|
|
670
|
+
requestKey,
|
|
671
|
+
event.phase === "progress" ? event.interEventGapMs : undefined,
|
|
672
|
+
);
|
|
673
|
+
}
|
|
674
|
+
const terminal =
|
|
675
|
+
event.phase === "completed" ||
|
|
676
|
+
event.phase === "failed" ||
|
|
677
|
+
event.phase === "timed_out";
|
|
678
|
+
if (terminal) {
|
|
679
|
+
modelRequestLifecycleMetricsFor(observability).finish(requestKey);
|
|
680
|
+
}
|
|
681
|
+
const phase =
|
|
682
|
+
event.phase === "headers"
|
|
683
|
+
? "headers"
|
|
684
|
+
: event.phase === "first_event"
|
|
685
|
+
? "first_byte"
|
|
686
|
+
: terminal
|
|
687
|
+
? "terminal"
|
|
688
|
+
: null;
|
|
689
|
+
if (!phase) return;
|
|
690
|
+
recordModelRequestPhase(observability, {
|
|
691
|
+
provider: "supergrok-subscription",
|
|
692
|
+
phase,
|
|
693
|
+
...(terminal
|
|
694
|
+
? {
|
|
695
|
+
outcome:
|
|
696
|
+
event.phase === "completed"
|
|
697
|
+
? ("completed" as const)
|
|
698
|
+
: event.phase === "timed_out"
|
|
699
|
+
? ("timed_out" as const)
|
|
700
|
+
: ("failed" as const),
|
|
701
|
+
}
|
|
702
|
+
: {}),
|
|
703
|
+
durationSeconds: event.durationMs / 1_000,
|
|
704
|
+
});
|
|
705
|
+
},
|
|
706
|
+
onModelRequestEvent: async (event) => {
|
|
707
|
+
if (!eventing.publish || !attempt.turnId) {
|
|
708
|
+
throw new Error("SuperGrok model request started before the turn event producer");
|
|
709
|
+
}
|
|
710
|
+
const shouldRecordStartedAudit =
|
|
711
|
+
event.phase === "started" && !firstModelRequestAuditRecorded;
|
|
712
|
+
if (shouldRecordStartedAudit) firstModelRequestAuditRecorded = true;
|
|
713
|
+
const auditStartedAt = shouldRecordStartedAudit ? performance.now() : null;
|
|
714
|
+
let auditOutcome: "completed" | "failed" = "completed";
|
|
715
|
+
try {
|
|
716
|
+
await eventing.publish([
|
|
717
|
+
...(shouldRecordStartedAudit &&
|
|
718
|
+
eventing.firstModelRequestPreparationStartedAt !== null
|
|
719
|
+
? [
|
|
720
|
+
{
|
|
721
|
+
type: "turn.startup.phase.completed" as const,
|
|
722
|
+
payload: {
|
|
723
|
+
phase: "model_preparation",
|
|
724
|
+
durationMs: Math.max(
|
|
725
|
+
0,
|
|
726
|
+
Math.round(
|
|
727
|
+
performance.now() - eventing.firstModelRequestPreparationStartedAt,
|
|
728
|
+
),
|
|
729
|
+
),
|
|
730
|
+
},
|
|
731
|
+
},
|
|
732
|
+
]
|
|
733
|
+
: []),
|
|
734
|
+
{
|
|
735
|
+
type: "agent.model.request",
|
|
736
|
+
payload: {
|
|
737
|
+
...event,
|
|
738
|
+
provider: "supergrok-subscription",
|
|
739
|
+
turnId: attempt.turnId,
|
|
740
|
+
attemptId: input.attemptId,
|
|
741
|
+
dispatchId,
|
|
742
|
+
executionGeneration: attempt.executionGeneration,
|
|
743
|
+
},
|
|
744
|
+
},
|
|
745
|
+
]);
|
|
746
|
+
} catch (error) {
|
|
747
|
+
auditOutcome = "failed";
|
|
748
|
+
throw error;
|
|
749
|
+
} finally {
|
|
750
|
+
if (auditStartedAt !== null) {
|
|
751
|
+
recordTurnStartupPhase(observability, {
|
|
752
|
+
phase: "model_request_audit",
|
|
753
|
+
provider: turnExecutionPolicy.providerId,
|
|
754
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
755
|
+
outcome: auditOutcome,
|
|
756
|
+
durationSeconds: (performance.now() - auditStartedAt) / 1_000,
|
|
757
|
+
count: turnTools.length,
|
|
758
|
+
});
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
},
|
|
762
|
+
});
|
|
763
|
+
providerTurn.xaiRequestContext = authorization.context;
|
|
764
|
+
}
|
|
765
|
+
const withCodex = <T>(fn: () => Promise<T>): Promise<T> =>
|
|
766
|
+
codexContext ? codexRequestStorage.run(codexContext, fn) : fn();
|
|
767
|
+
const withProviderRequestContext = <T>(fn: () => Promise<T>): Promise<T> =>
|
|
768
|
+
providerTurn.xaiRequestContext
|
|
769
|
+
? xaiSubscriptionRequestStorage.run(providerTurn.xaiRequestContext, fn)
|
|
770
|
+
: withCodex(fn);
|
|
771
|
+
const withCodexRemoteCompaction = <T>(fn: () => Promise<T>): Promise<T> =>
|
|
772
|
+
withCodex(() =>
|
|
773
|
+
withCodexRequestOverrides(
|
|
774
|
+
{
|
|
775
|
+
betaFeatures: [REMOTE_COMPACTION_V2_BETA_FEATURE],
|
|
776
|
+
turnMetadata: {
|
|
777
|
+
request_kind: "compaction",
|
|
778
|
+
compaction: {
|
|
779
|
+
implementation: REMOTE_COMPACTION_V2_IMPLEMENTATION,
|
|
780
|
+
strategy: "memento",
|
|
781
|
+
},
|
|
782
|
+
},
|
|
783
|
+
},
|
|
784
|
+
fn,
|
|
785
|
+
),
|
|
786
|
+
);
|
|
787
|
+
const compactionPrep = await prepareCompaction({
|
|
788
|
+
input,
|
|
789
|
+
settings,
|
|
790
|
+
db,
|
|
791
|
+
bus,
|
|
792
|
+
observability,
|
|
793
|
+
cancellationSignal,
|
|
794
|
+
control,
|
|
795
|
+
attempt,
|
|
796
|
+
billingState,
|
|
797
|
+
eventing: wiredEventing,
|
|
798
|
+
providerTurn,
|
|
799
|
+
leases,
|
|
800
|
+
media,
|
|
801
|
+
claimedResult,
|
|
802
|
+
claimedModelUsageSourceKeys,
|
|
803
|
+
emittedModelUsageSourceKeys,
|
|
804
|
+
modelUsageDispatchId,
|
|
805
|
+
turn,
|
|
806
|
+
session,
|
|
807
|
+
turnExecutionPolicy,
|
|
808
|
+
resolvedModel,
|
|
809
|
+
workspaceAgentInstructions,
|
|
810
|
+
workspaceGovernance,
|
|
811
|
+
structuredWorkspacePolicyActive,
|
|
812
|
+
workspaceMemory,
|
|
813
|
+
rigVersion,
|
|
814
|
+
rigName,
|
|
815
|
+
compactionModelHistoryProjector,
|
|
816
|
+
summarizeContextForCompaction,
|
|
817
|
+
withProviderRequestContext,
|
|
818
|
+
withCodexRemoteCompaction,
|
|
819
|
+
});
|
|
820
|
+
if ("exit" in compactionPrep) return compactionPrep.exit;
|
|
821
|
+
const {
|
|
822
|
+
promptCacheKey,
|
|
823
|
+
remotePrefix,
|
|
824
|
+
remoteCompactionRequester,
|
|
825
|
+
publishCompactionLiveEvents,
|
|
826
|
+
publishCompactionOutcomeEvents,
|
|
827
|
+
compactionModeOptions,
|
|
828
|
+
compactionOnlyTurn,
|
|
829
|
+
compactionSummarizerFor,
|
|
830
|
+
} = compactionPrep.ok;
|
|
831
|
+
|
|
832
|
+
const toolPolicy = await prepareTurnToolPolicy({
|
|
833
|
+
input,
|
|
834
|
+
db,
|
|
835
|
+
cancellationSignal,
|
|
836
|
+
connectionCredentials,
|
|
837
|
+
turn,
|
|
838
|
+
session,
|
|
839
|
+
fileAuthoritySubjectId,
|
|
840
|
+
capabilitySettings,
|
|
841
|
+
runSettings,
|
|
842
|
+
rigVersion,
|
|
843
|
+
workspaceRefs,
|
|
844
|
+
});
|
|
845
|
+
const {
|
|
846
|
+
turnResources: claimedTurnResources,
|
|
847
|
+
runtimeResources: claimedRuntimeResources,
|
|
848
|
+
mcpAvailabilityNote,
|
|
849
|
+
turnTools,
|
|
850
|
+
connectionScope,
|
|
851
|
+
workspaceVariableSet,
|
|
852
|
+
sandboxWorkspaceEnvironmentValues,
|
|
853
|
+
} = toolPolicy;
|
|
854
|
+
|
|
855
|
+
const sandboxRoute = await resolveSandboxRoute({
|
|
856
|
+
input,
|
|
857
|
+
settings,
|
|
858
|
+
db,
|
|
859
|
+
eventing,
|
|
860
|
+
sandboxState,
|
|
861
|
+
media,
|
|
862
|
+
fileAuthoritySubjectId,
|
|
863
|
+
runSettings,
|
|
864
|
+
});
|
|
865
|
+
const {
|
|
866
|
+
routingOn,
|
|
867
|
+
activeSandboxBackend,
|
|
868
|
+
machinePrimary,
|
|
869
|
+
groupBoxBackend,
|
|
870
|
+
sandboxCreationBackend,
|
|
871
|
+
effectiveRunCredentialBackend,
|
|
872
|
+
} = sandboxRoute;
|
|
873
|
+
|
|
874
|
+
// A bare github.com repository URI (API caller, older session, or an
|
|
875
|
+
// agent-spawned child inheriting its parent's resources) resolves to
|
|
876
|
+
// the workspace's GitHub App binding here, before credential minting
|
|
877
|
+
// and the runtime clone plan derive binding ids from the same resource
|
|
878
|
+
// set. A Connected Machine receives no platform Git credential, so its
|
|
879
|
+
// resources stay exactly as stored. Resolution never fails the turn;
|
|
880
|
+
// an unusable bound repository stays bare and is reported visibly.
|
|
881
|
+
const { turnResources, runtimeResources } = await waitForTurnOperation(
|
|
882
|
+
applyTurnGitHubRepositoryBindings({
|
|
883
|
+
db,
|
|
884
|
+
settings: runSettings,
|
|
885
|
+
workspaceId: input.workspaceId,
|
|
886
|
+
sessionId: input.sessionId,
|
|
887
|
+
activeSandboxBackend,
|
|
888
|
+
claimedTurnResources,
|
|
889
|
+
claimedRuntimeResources,
|
|
890
|
+
publish: async (events) => {
|
|
891
|
+
await eventing.publish!(events, true);
|
|
892
|
+
},
|
|
893
|
+
warn: (message, fields) => observability.warn(message, fields),
|
|
894
|
+
}),
|
|
895
|
+
cancellationSignal,
|
|
896
|
+
undefined,
|
|
897
|
+
);
|
|
898
|
+
|
|
899
|
+
const runCredentials = await prepareRunCredentials({
|
|
900
|
+
input,
|
|
901
|
+
settings,
|
|
902
|
+
db,
|
|
903
|
+
observability,
|
|
904
|
+
cancellationSignal,
|
|
905
|
+
connectionCredentials,
|
|
906
|
+
personalGitHubCredentials,
|
|
907
|
+
eventing,
|
|
908
|
+
attempt,
|
|
909
|
+
renewals,
|
|
910
|
+
sandboxState,
|
|
911
|
+
sandboxRuntime,
|
|
912
|
+
turn,
|
|
913
|
+
session,
|
|
914
|
+
fileAuthoritySubjectId,
|
|
915
|
+
runSettings,
|
|
916
|
+
workspaceVariableSet,
|
|
917
|
+
turnResources,
|
|
918
|
+
requiredGeneratedVideoFiles,
|
|
919
|
+
machinePrimary,
|
|
920
|
+
activeSandboxBackend,
|
|
921
|
+
groupBoxBackend,
|
|
922
|
+
sandboxCreationBackend,
|
|
923
|
+
effectiveRunCredentialBackend,
|
|
924
|
+
sandboxWorkspaceEnvironmentValues,
|
|
925
|
+
connectionScope,
|
|
926
|
+
runWorkspaceMutationForSandbox,
|
|
927
|
+
});
|
|
928
|
+
const {
|
|
929
|
+
runCredentialResolver,
|
|
930
|
+
establishPolicy,
|
|
931
|
+
initialRunCredentialMaterial,
|
|
932
|
+
runCredentialsNote,
|
|
933
|
+
hostCredentialRootSessionId,
|
|
934
|
+
codemodeAuthority,
|
|
935
|
+
sandboxArtifactRuntime,
|
|
936
|
+
sandboxEnvironment,
|
|
937
|
+
sandboxGitToken,
|
|
938
|
+
sandboxGitTokens,
|
|
939
|
+
sandboxGitCredentialBindings,
|
|
940
|
+
sandboxCodemodeToken,
|
|
941
|
+
sandboxCodemodeTokenExpiresAt,
|
|
942
|
+
transientCodemodeEnvironment,
|
|
943
|
+
initialGitCredentials,
|
|
944
|
+
startRunGitCredentialsMint,
|
|
945
|
+
attachGitCredentialRenewal,
|
|
946
|
+
attachCodemodeTokenRenewal,
|
|
947
|
+
attachRunCredentialRenewal,
|
|
948
|
+
} = runCredentials;
|
|
949
|
+
|
|
950
|
+
await establishTurnSandbox({
|
|
951
|
+
...sandboxRoute,
|
|
952
|
+
input,
|
|
953
|
+
settings,
|
|
954
|
+
db,
|
|
955
|
+
bus,
|
|
956
|
+
objectStorage,
|
|
957
|
+
observability,
|
|
958
|
+
cancellationSignal,
|
|
959
|
+
runtimeCancellationSignal,
|
|
960
|
+
sandboxResumeSignal,
|
|
961
|
+
activityContext,
|
|
962
|
+
opJournal,
|
|
963
|
+
sandboxState,
|
|
964
|
+
eventing,
|
|
965
|
+
attempt,
|
|
966
|
+
turn,
|
|
967
|
+
session,
|
|
968
|
+
fileAuthoritySubjectId,
|
|
969
|
+
capabilitySettings,
|
|
970
|
+
turnExecutionPolicy,
|
|
971
|
+
runSettings,
|
|
972
|
+
logicalSandboxSettings,
|
|
973
|
+
verifiedRigProviderImageId,
|
|
974
|
+
runtimePreparationStartedAt,
|
|
975
|
+
establishPolicy,
|
|
976
|
+
sandboxEnvironment,
|
|
977
|
+
startRunGitCredentialsMint,
|
|
978
|
+
machineOpObserver,
|
|
979
|
+
sandboxOperationObserver,
|
|
980
|
+
sandboxRuntime,
|
|
981
|
+
transientCodemodeEnvironment,
|
|
982
|
+
rigVersion,
|
|
983
|
+
turnResources,
|
|
984
|
+
});
|
|
985
|
+
|
|
986
|
+
const ordinaryFileResourceDownloads = await (async () => {
|
|
987
|
+
const fileResolutionStartedAt = performance.now();
|
|
988
|
+
let fileResolutionOutcome: "completed" | "failed" = "completed";
|
|
989
|
+
try {
|
|
990
|
+
return await waitForTurnOperation(
|
|
991
|
+
sandboxFileDownloadsForRun(
|
|
992
|
+
runSettings,
|
|
993
|
+
db,
|
|
994
|
+
objectStorage,
|
|
995
|
+
input.accountId,
|
|
996
|
+
input.workspaceId,
|
|
997
|
+
fileAuthoritySubjectId,
|
|
998
|
+
turn.resources,
|
|
999
|
+
activeSandboxBackend ?? groupBoxBackend,
|
|
1000
|
+
),
|
|
1001
|
+
cancellationSignal,
|
|
1002
|
+
undefined,
|
|
1003
|
+
);
|
|
1004
|
+
} catch (error) {
|
|
1005
|
+
fileResolutionOutcome = "failed";
|
|
1006
|
+
throw error;
|
|
1007
|
+
} finally {
|
|
1008
|
+
recordTurnStartupPhase(observability, {
|
|
1009
|
+
phase: "file_resolution",
|
|
1010
|
+
provider: turnExecutionPolicy.providerId,
|
|
1011
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
1012
|
+
outcome: fileResolutionOutcome,
|
|
1013
|
+
durationSeconds: (performance.now() - fileResolutionStartedAt) / 1_000,
|
|
1014
|
+
count: turn.resources.length,
|
|
1015
|
+
});
|
|
1016
|
+
}
|
|
1017
|
+
})();
|
|
1018
|
+
if (requiredGeneratedVideoFiles.length > 0 && !objectStorage) {
|
|
1019
|
+
throw new Error("Generated video materialization requires object storage");
|
|
1020
|
+
}
|
|
1021
|
+
const generatedVideoDownloads: SandboxFileDownload[] = [];
|
|
1022
|
+
if (objectStorage && requiredGeneratedVideoFiles.length > 0) {
|
|
1023
|
+
const downloadStorage = objectStorageForSandboxDownloads(
|
|
1024
|
+
eventing.modelRunSettings,
|
|
1025
|
+
objectStorage,
|
|
1026
|
+
media.sandboxFileDownloadBackend,
|
|
1027
|
+
);
|
|
1028
|
+
for (const file of requiredGeneratedVideoFiles) {
|
|
1029
|
+
const signed = await downloadStorage.createGetUrl({
|
|
1030
|
+
key: file.objectKey,
|
|
1031
|
+
});
|
|
1032
|
+
generatedVideoDownloads.push({
|
|
1033
|
+
fileId: file.fileId,
|
|
1034
|
+
mountPath: "generated-videos",
|
|
1035
|
+
filename: file.filename,
|
|
1036
|
+
url: signed.url,
|
|
1037
|
+
expiresAt: signed.expiresAt,
|
|
1038
|
+
sizeBytes: file.sizeBytes,
|
|
1039
|
+
sha256: file.sha256,
|
|
1040
|
+
});
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
const fileResourceDownloads = [
|
|
1044
|
+
...new Map(
|
|
1045
|
+
[...ordinaryFileResourceDownloads, ...generatedVideoDownloads].map((download) => [
|
|
1046
|
+
download.fileId,
|
|
1047
|
+
download,
|
|
1048
|
+
]),
|
|
1049
|
+
).values(),
|
|
1050
|
+
];
|
|
1051
|
+
const toolRuntime = await prepareTurnToolRuntime({
|
|
1052
|
+
input,
|
|
1053
|
+
db,
|
|
1054
|
+
bus,
|
|
1055
|
+
runtime,
|
|
1056
|
+
objectStorage,
|
|
1057
|
+
observability,
|
|
1058
|
+
cancellationSignal,
|
|
1059
|
+
connectionCredentials,
|
|
1060
|
+
eventing,
|
|
1061
|
+
attempt,
|
|
1062
|
+
sandboxState,
|
|
1063
|
+
media,
|
|
1064
|
+
toolResultSpill,
|
|
1065
|
+
turn,
|
|
1066
|
+
session,
|
|
1067
|
+
fileAuthoritySubjectId,
|
|
1068
|
+
capabilitySettings,
|
|
1069
|
+
installedApiIntegrations,
|
|
1070
|
+
codexAppsCredentialId,
|
|
1071
|
+
turnExecutionPolicy,
|
|
1072
|
+
trigger,
|
|
1073
|
+
runSettings,
|
|
1074
|
+
lazyToolTransport,
|
|
1075
|
+
turnTools,
|
|
1076
|
+
connectionScope,
|
|
1077
|
+
hostCredentialRootSessionId,
|
|
1078
|
+
sandboxArtifactRuntime,
|
|
1079
|
+
activeSandboxBackend,
|
|
1080
|
+
groupBoxBackend,
|
|
1081
|
+
routingOn,
|
|
1082
|
+
runtimeCancellationSignal,
|
|
1083
|
+
credentialSubjectId,
|
|
1084
|
+
interactionInterventionResume,
|
|
1085
|
+
runWorkspaceMutationForSandbox,
|
|
1086
|
+
throwIfWorkerShuttingDown,
|
|
1087
|
+
throwIfTurnCancelled,
|
|
1088
|
+
});
|
|
1089
|
+
const {
|
|
1090
|
+
attemptConnectorActionBindings,
|
|
1091
|
+
connectorActionIdentity,
|
|
1092
|
+
postToolPreparationStartedAt,
|
|
1093
|
+
} = toolRuntime;
|
|
1094
|
+
|
|
1095
|
+
const builtAgent = await buildTurnAgent({
|
|
1096
|
+
input,
|
|
1097
|
+
db,
|
|
1098
|
+
runtime,
|
|
1099
|
+
objectStorage,
|
|
1100
|
+
observability,
|
|
1101
|
+
cancellationSignal,
|
|
1102
|
+
runtimeCancellationSignal,
|
|
1103
|
+
eventing,
|
|
1104
|
+
attempt,
|
|
1105
|
+
sandboxState,
|
|
1106
|
+
recordingState,
|
|
1107
|
+
maybeStartOnTurnRecording,
|
|
1108
|
+
providerTurn,
|
|
1109
|
+
media,
|
|
1110
|
+
leases,
|
|
1111
|
+
turn,
|
|
1112
|
+
session,
|
|
1113
|
+
fileAuthoritySubjectId,
|
|
1114
|
+
capabilitySettings,
|
|
1115
|
+
humanInputResume,
|
|
1116
|
+
turnExecutionPolicy,
|
|
1117
|
+
runSettings,
|
|
1118
|
+
logicalSandboxSettings,
|
|
1119
|
+
verifiedRigProviderImageId,
|
|
1120
|
+
resolvedModel,
|
|
1121
|
+
nativeImageProviderBinding,
|
|
1122
|
+
lazyToolTransport,
|
|
1123
|
+
modelInputPolicy,
|
|
1124
|
+
supportsImageInput,
|
|
1125
|
+
agentHumanInputEnabled,
|
|
1126
|
+
workspaceAgentInstructions,
|
|
1127
|
+
workspaceGovernance,
|
|
1128
|
+
structuredWorkspacePolicyActive,
|
|
1129
|
+
workspaceMemory,
|
|
1130
|
+
rigVersion,
|
|
1131
|
+
rigName,
|
|
1132
|
+
packRuntime,
|
|
1133
|
+
installedSkillRuntime,
|
|
1134
|
+
buildCompanyBrainContributionReceiptFor,
|
|
1135
|
+
promptCacheKey,
|
|
1136
|
+
workspaceVariableSet,
|
|
1137
|
+
runtimeResources,
|
|
1138
|
+
sandboxEnvironment,
|
|
1139
|
+
sandboxArtifactRuntime,
|
|
1140
|
+
sandboxGitToken,
|
|
1141
|
+
sandboxGitTokens,
|
|
1142
|
+
sandboxGitCredentialBindings,
|
|
1143
|
+
sandboxCodemodeToken,
|
|
1144
|
+
fileResourceDownloads,
|
|
1145
|
+
attemptConnectorActionBindings,
|
|
1146
|
+
connectorActionIdentity,
|
|
1147
|
+
videoGenerationAcceptancesByCallId,
|
|
1148
|
+
activeSandboxBackend,
|
|
1149
|
+
groupBoxBackend,
|
|
1150
|
+
postToolPreparationStartedAt,
|
|
1151
|
+
codexContext,
|
|
1152
|
+
});
|
|
1153
|
+
const { agent, modelVisibleRuntimeSkillActivations, postAgentPreparationStartedAt } =
|
|
1154
|
+
builtAgent;
|
|
1155
|
+
|
|
1156
|
+
await bindLazySandboxProvisioner({
|
|
1157
|
+
...sandboxRoute,
|
|
1158
|
+
input,
|
|
1159
|
+
settings,
|
|
1160
|
+
db,
|
|
1161
|
+
bus,
|
|
1162
|
+
objectStorage,
|
|
1163
|
+
observability,
|
|
1164
|
+
cancellationSignal,
|
|
1165
|
+
runtimeCancellationSignal,
|
|
1166
|
+
sandboxResumeSignal,
|
|
1167
|
+
activityContext,
|
|
1168
|
+
opJournal,
|
|
1169
|
+
sandboxState,
|
|
1170
|
+
eventing,
|
|
1171
|
+
attempt,
|
|
1172
|
+
turn,
|
|
1173
|
+
session,
|
|
1174
|
+
fileAuthoritySubjectId,
|
|
1175
|
+
capabilitySettings,
|
|
1176
|
+
turnExecutionPolicy,
|
|
1177
|
+
runSettings,
|
|
1178
|
+
logicalSandboxSettings,
|
|
1179
|
+
verifiedRigProviderImageId,
|
|
1180
|
+
runtimePreparationStartedAt,
|
|
1181
|
+
establishPolicy,
|
|
1182
|
+
sandboxEnvironment,
|
|
1183
|
+
startRunGitCredentialsMint,
|
|
1184
|
+
machineOpObserver,
|
|
1185
|
+
sandboxOperationObserver,
|
|
1186
|
+
sandboxRuntime,
|
|
1187
|
+
transientCodemodeEnvironment,
|
|
1188
|
+
agent,
|
|
1189
|
+
attachRunCredentialRenewal,
|
|
1190
|
+
attachGitCredentialRenewal,
|
|
1191
|
+
attachCodemodeTokenRenewal,
|
|
1192
|
+
throwIfWorkerShuttingDown,
|
|
1193
|
+
throwIfTurnCancelled,
|
|
1194
|
+
initialRunCredentialMaterial,
|
|
1195
|
+
sandboxCodemodeToken,
|
|
1196
|
+
media,
|
|
1197
|
+
toolResultSpill,
|
|
1198
|
+
connectionScope,
|
|
1199
|
+
codemodeAuthority,
|
|
1200
|
+
rigVersion,
|
|
1201
|
+
turnResources,
|
|
1202
|
+
});
|
|
1203
|
+
|
|
1204
|
+
let companyBrainContributionReceiptRecorded = false;
|
|
1205
|
+
const recordCompanyBrainContributionReceiptOnce = (): void => {
|
|
1206
|
+
if (companyBrainContributionReceiptRecorded) return;
|
|
1207
|
+
companyBrainContributionReceiptRecorded = true;
|
|
1208
|
+
try {
|
|
1209
|
+
const companyBrainContributionReceipt = buildCompanyBrainContributionReceiptFor(
|
|
1210
|
+
modelVisibleRuntimeSkillActivations,
|
|
1211
|
+
);
|
|
1212
|
+
eventing.companyBrainContextContributions = summarizeCompanyBrainContributions(
|
|
1213
|
+
companyBrainContributionReceipt,
|
|
1214
|
+
);
|
|
1215
|
+
recordCompanyBrainContributions(observability, companyBrainContributionReceipt);
|
|
1216
|
+
observability.info("model context contribution receipt", {
|
|
1217
|
+
attemptId: companyBrainContributionReceipt.attemptId,
|
|
1218
|
+
turnId: companyBrainContributionReceipt.turnId,
|
|
1219
|
+
contextSelectionReceiptId: companyBrainContributionReceipt.contextSelectionReceiptId,
|
|
1220
|
+
sessionRole: companyBrainContributionReceipt.sessionRole,
|
|
1221
|
+
memoryPromptMode: companyBrainContributionReceipt.memoryPromptMode,
|
|
1222
|
+
instructionPolicySnapshotId:
|
|
1223
|
+
companyBrainContributionReceipt.instructionPolicySnapshotId,
|
|
1224
|
+
preferenceSnapshotId: companyBrainContributionReceipt.preferenceSnapshotId,
|
|
1225
|
+
companyProfileSnapshotId: companyBrainContributionReceipt.companyProfileSnapshotId,
|
|
1226
|
+
contributions: JSON.stringify(companyBrainContributionReceipt.contributions),
|
|
1227
|
+
});
|
|
1228
|
+
} catch {
|
|
1229
|
+
// Observability must never change model execution semantics.
|
|
1230
|
+
}
|
|
1231
|
+
};
|
|
1232
|
+
const postAgentCompaction = await runPostAgentCompaction({
|
|
1233
|
+
input,
|
|
1234
|
+
settings,
|
|
1235
|
+
db,
|
|
1236
|
+
bus,
|
|
1237
|
+
observability,
|
|
1238
|
+
cancellationSignal,
|
|
1239
|
+
control,
|
|
1240
|
+
attempt,
|
|
1241
|
+
billingState,
|
|
1242
|
+
eventing: wiredEventing,
|
|
1243
|
+
providerTurn,
|
|
1244
|
+
leases,
|
|
1245
|
+
media,
|
|
1246
|
+
claimedResult,
|
|
1247
|
+
claimedModelUsageSourceKeys,
|
|
1248
|
+
emittedModelUsageSourceKeys,
|
|
1249
|
+
modelUsageDispatchId,
|
|
1250
|
+
turn,
|
|
1251
|
+
session,
|
|
1252
|
+
turnExecutionPolicy,
|
|
1253
|
+
resolvedModel,
|
|
1254
|
+
workspaceAgentInstructions,
|
|
1255
|
+
workspaceGovernance,
|
|
1256
|
+
structuredWorkspacePolicyActive,
|
|
1257
|
+
workspaceMemory,
|
|
1258
|
+
rigVersion,
|
|
1259
|
+
rigName,
|
|
1260
|
+
compactionModelHistoryProjector,
|
|
1261
|
+
summarizeContextForCompaction,
|
|
1262
|
+
withProviderRequestContext,
|
|
1263
|
+
withCodexRemoteCompaction,
|
|
1264
|
+
remotePrefix,
|
|
1265
|
+
remoteCompactionRequester,
|
|
1266
|
+
publishCompactionLiveEvents,
|
|
1267
|
+
publishCompactionOutcomeEvents,
|
|
1268
|
+
compactionModeOptions,
|
|
1269
|
+
compactionOnlyTurn,
|
|
1270
|
+
compactionSummarizerFor,
|
|
1271
|
+
agent,
|
|
1272
|
+
});
|
|
1273
|
+
if ("exit" in postAgentCompaction) return postAgentCompaction.exit;
|
|
1274
|
+
const { compactSummarizer } = postAgentCompaction.ok;
|
|
1275
|
+
|
|
1276
|
+
recordTurnStartupPhase(observability, {
|
|
1277
|
+
phase: "post_agent_preparation",
|
|
1278
|
+
provider: turnExecutionPolicy.providerId,
|
|
1279
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
1280
|
+
outcome: "completed",
|
|
1281
|
+
durationSeconds: (performance.now() - postAgentPreparationStartedAt) / 1_000,
|
|
1282
|
+
});
|
|
1283
|
+
let fileMaterializationFailures: SandboxFileDownloadFailure[] = [];
|
|
1284
|
+
let fileDownloadsMaterializedForRun = false;
|
|
1285
|
+
if (
|
|
1286
|
+
sandboxState.resolvedSandbox &&
|
|
1287
|
+
sandboxState.setupBoxSession &&
|
|
1288
|
+
fileResourceDownloads.length > 0
|
|
1289
|
+
) {
|
|
1290
|
+
const fileMaterializationStartedAt = performance.now();
|
|
1291
|
+
let fileMaterializationOutcome: "completed" | "failed" = "completed";
|
|
1292
|
+
let fileMaterializationCache: "hit" | "miss" = "miss";
|
|
1293
|
+
try {
|
|
1294
|
+
const boxInstanceId = sandboxState.resolvedSandbox.established.instanceId;
|
|
1295
|
+
// A successful transfer is durable for this exact filesystem instance.
|
|
1296
|
+
// Do not turn later model startup into an integrity scan. If an owner or
|
|
1297
|
+
// agent removes the file, it can be restored explicitly through the
|
|
1298
|
+
// existing Files MCP using the durable file id carried in model history.
|
|
1299
|
+
const alreadyMaterialized = await getMaterializedSandboxFileResources(db, {
|
|
1300
|
+
accountId: input.accountId,
|
|
1301
|
+
workspaceId: input.workspaceId,
|
|
1302
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
1303
|
+
expectedEpoch: sandboxState.resolvedSandbox.leaseEpoch,
|
|
1304
|
+
instanceId: boxInstanceId,
|
|
1305
|
+
});
|
|
1306
|
+
const downloadsToMaterialize = filterUnmaterializedSandboxFileDownloads(
|
|
1307
|
+
fileResourceDownloads,
|
|
1308
|
+
alreadyMaterialized,
|
|
1309
|
+
);
|
|
1310
|
+
if (downloadsToMaterialize.length === 0) {
|
|
1311
|
+
fileMaterializationCache = "hit";
|
|
1312
|
+
}
|
|
1313
|
+
const runAs = sandboxRunAs(runSettings);
|
|
1314
|
+
if (downloadsToMaterialize.length > 0) {
|
|
1315
|
+
const materialized = await runWorkspaceMutationForSandbox(
|
|
1316
|
+
sandboxState.resolvedSandbox,
|
|
1317
|
+
"fileMaterialization",
|
|
1318
|
+
async () =>
|
|
1319
|
+
await materializeSandboxFileDownloads(
|
|
1320
|
+
sandboxState.setupBoxSession as CodemodeTokenWriterSession,
|
|
1321
|
+
downloadsToMaterialize,
|
|
1322
|
+
{
|
|
1323
|
+
onRuntimeEvent: async (event) => {
|
|
1324
|
+
await eventing.publish!([{ type: event.type, payload: event.payload }], true);
|
|
1325
|
+
},
|
|
1326
|
+
...(runAs ? { runAs } : {}),
|
|
1327
|
+
...(eventing.toolCancellationFenceRef.current
|
|
1328
|
+
? {
|
|
1329
|
+
commandRunner:
|
|
1330
|
+
eventing.toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
1331
|
+
eventing.toolCancellationFenceRef.current,
|
|
1332
|
+
),
|
|
1333
|
+
}
|
|
1334
|
+
: {}),
|
|
1335
|
+
},
|
|
1336
|
+
),
|
|
1337
|
+
);
|
|
1338
|
+
fileMaterializationFailures = materialized.failures;
|
|
1339
|
+
fileMaterializationOutcome = sandboxFileMaterializationOutcome(materialized.failures);
|
|
1340
|
+
const failedFileIds = new Set(materialized.failures.map((failure) => failure.fileId));
|
|
1341
|
+
const succeededFileIds = downloadsToMaterialize
|
|
1342
|
+
.map((download) => download.fileId)
|
|
1343
|
+
.filter((fileId) => !failedFileIds.has(fileId));
|
|
1344
|
+
if (succeededFileIds.length > 0) {
|
|
1345
|
+
await markSandboxFileResourcesMaterialized(db, {
|
|
1346
|
+
accountId: input.accountId,
|
|
1347
|
+
workspaceId: input.workspaceId,
|
|
1348
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
1349
|
+
expectedEpoch: sandboxState.resolvedSandbox.leaseEpoch,
|
|
1350
|
+
instanceId: boxInstanceId,
|
|
1351
|
+
fileIds: succeededFileIds,
|
|
1352
|
+
});
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
fileDownloadsMaterializedForRun = true;
|
|
1356
|
+
} catch (error) {
|
|
1357
|
+
fileMaterializationOutcome = "failed";
|
|
1358
|
+
throw error;
|
|
1359
|
+
} finally {
|
|
1360
|
+
recordTurnStartupPhase(observability, {
|
|
1361
|
+
phase: "file_materialization",
|
|
1362
|
+
provider: turnExecutionPolicy.providerId,
|
|
1363
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
1364
|
+
outcome: fileMaterializationOutcome,
|
|
1365
|
+
durationSeconds: (performance.now() - fileMaterializationStartedAt) / 1_000,
|
|
1366
|
+
count: fileResourceDownloads.length,
|
|
1367
|
+
cache: fileMaterializationCache,
|
|
1368
|
+
});
|
|
1369
|
+
}
|
|
1370
|
+
} else {
|
|
1371
|
+
recordTurnStartupPhase(observability, {
|
|
1372
|
+
phase: "file_materialization",
|
|
1373
|
+
provider: turnExecutionPolicy.providerId,
|
|
1374
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
1375
|
+
outcome: "completed",
|
|
1376
|
+
durationSeconds: 0,
|
|
1377
|
+
count: 0,
|
|
1378
|
+
});
|
|
1379
|
+
}
|
|
1380
|
+
const requiredVideoFileIds = new Set(requiredGeneratedVideoFiles.map((file) => file.fileId));
|
|
1381
|
+
const failedRequiredVideo = fileMaterializationFailures.find((failure) =>
|
|
1382
|
+
requiredVideoFileIds.has(failure.fileId),
|
|
1383
|
+
);
|
|
1384
|
+
if (failedRequiredVideo) {
|
|
1385
|
+
// Retention is already durable. A sandbox copy miss must not fail the
|
|
1386
|
+
// turn or replay generation; the File remains retrievable via Files MCP.
|
|
1387
|
+
observability.warn("Generated video sandbox materialization deferred", {
|
|
1388
|
+
errorClass: "SandboxFileDownloadFailure",
|
|
1389
|
+
errorCode: "generated_video_materialization_deferred",
|
|
1390
|
+
origin: "worker",
|
|
1391
|
+
});
|
|
1392
|
+
}
|
|
1393
|
+
const unavailableSandboxFilesNote = sandboxFileDownloadFailureNote(
|
|
1394
|
+
fileMaterializationFailures,
|
|
1395
|
+
);
|
|
1396
|
+
return await runTurnStreamAttempt({
|
|
1397
|
+
input,
|
|
1398
|
+
settings,
|
|
1399
|
+
db,
|
|
1400
|
+
runtime,
|
|
1401
|
+
objectStorage,
|
|
1402
|
+
observability,
|
|
1403
|
+
entitlements,
|
|
1404
|
+
startVideoGenerationWorkflow,
|
|
1405
|
+
cancellationSignal,
|
|
1406
|
+
runtimeCancellationSignal,
|
|
1407
|
+
activityContext,
|
|
1408
|
+
dispatchId,
|
|
1409
|
+
control,
|
|
1410
|
+
attempt,
|
|
1411
|
+
billingState,
|
|
1412
|
+
sandboxState,
|
|
1413
|
+
eventing: wiredEventing,
|
|
1414
|
+
providerTurn,
|
|
1415
|
+
leases,
|
|
1416
|
+
historySink,
|
|
1417
|
+
media,
|
|
1418
|
+
toolResultSpill,
|
|
1419
|
+
claimedResult,
|
|
1420
|
+
flushRuntimeBatcher,
|
|
1421
|
+
finalizeTurnOpStreamOps,
|
|
1422
|
+
runWorkspaceMutationForSandbox,
|
|
1423
|
+
throwIfWorkerShuttingDown,
|
|
1424
|
+
throwIfTurnCancelled,
|
|
1425
|
+
setLastInputTokensFenced,
|
|
1426
|
+
attachGitCredentialRenewal,
|
|
1427
|
+
attachCodemodeTokenRenewal,
|
|
1428
|
+
attachRunCredentialRenewal,
|
|
1429
|
+
withProviderRequestContext,
|
|
1430
|
+
publishCompactionLiveEvents,
|
|
1431
|
+
publishCompactionOutcomeEvents,
|
|
1432
|
+
recordCompanyBrainContributionReceiptOnce,
|
|
1433
|
+
modelCheckpointMemoryCollector,
|
|
1434
|
+
claimedModelUsageSourceKeys,
|
|
1435
|
+
emittedModelUsageSourceKeys,
|
|
1436
|
+
modelUsageDispatchId,
|
|
1437
|
+
workerPreparationStartedAt,
|
|
1438
|
+
fileDownloadsMaterializedForRun,
|
|
1439
|
+
unavailableSandboxFilesNote,
|
|
1440
|
+
runCredentialsNote,
|
|
1441
|
+
mcpAvailabilityNote,
|
|
1442
|
+
fileAuthoritySubjectId,
|
|
1443
|
+
activeSandboxBackend,
|
|
1444
|
+
groupBoxBackend,
|
|
1445
|
+
turnExecutionPolicy,
|
|
1446
|
+
turn,
|
|
1447
|
+
trigger,
|
|
1448
|
+
humanInputResume,
|
|
1449
|
+
agent,
|
|
1450
|
+
resolvedModel,
|
|
1451
|
+
providerApi,
|
|
1452
|
+
runSettings,
|
|
1453
|
+
turnTools,
|
|
1454
|
+
compactSummarizer,
|
|
1455
|
+
compactionModelHistoryProjector,
|
|
1456
|
+
generatedImageHistoryProjector,
|
|
1457
|
+
modelHistoryProjector,
|
|
1458
|
+
compactionModeOptions,
|
|
1459
|
+
initialRunCredentialMaterial,
|
|
1460
|
+
initialGitCredentials,
|
|
1461
|
+
sandboxEnvironment,
|
|
1462
|
+
sandboxCodemodeToken,
|
|
1463
|
+
sandboxCodemodeTokenExpiresAt,
|
|
1464
|
+
fileResourceDownloads,
|
|
1465
|
+
runCredentialResolver,
|
|
1466
|
+
videoGenerationAcceptancesByCallId,
|
|
1467
|
+
});
|
|
1468
|
+
} catch (error) {
|
|
1469
|
+
return await settleTurnFailure({
|
|
1470
|
+
error,
|
|
1471
|
+
input,
|
|
1472
|
+
settings,
|
|
1473
|
+
db,
|
|
1474
|
+
bus,
|
|
1475
|
+
observability,
|
|
1476
|
+
wakeSessionWorkflow,
|
|
1477
|
+
signalCodexCapacityWorkflow,
|
|
1478
|
+
cancellationSignal,
|
|
1479
|
+
sandboxRotationController,
|
|
1480
|
+
noteCancellationRequested,
|
|
1481
|
+
codexWorkspaceKey,
|
|
1482
|
+
control,
|
|
1483
|
+
attempt,
|
|
1484
|
+
billingState,
|
|
1485
|
+
eventing,
|
|
1486
|
+
providerTurn,
|
|
1487
|
+
leases,
|
|
1488
|
+
historySink,
|
|
1489
|
+
claimedResult,
|
|
1490
|
+
flushRuntimeBatcher,
|
|
1491
|
+
acknowledgeLostAttemptOwnership,
|
|
1492
|
+
acknowledgeRecoveryQuiescence,
|
|
1493
|
+
});
|
|
1494
|
+
} finally {
|
|
1495
|
+
await finalizeTurnAttempt({
|
|
1496
|
+
input,
|
|
1497
|
+
settings,
|
|
1498
|
+
db,
|
|
1499
|
+
bus,
|
|
1500
|
+
objectStorage,
|
|
1501
|
+
observability,
|
|
1502
|
+
wakeSessionWorkflow,
|
|
1503
|
+
signalSessionAttemptQuiesced,
|
|
1504
|
+
signalCodexCapacityWorkflow,
|
|
1505
|
+
cancellationSignal,
|
|
1506
|
+
sandboxResumeController,
|
|
1507
|
+
activityContext,
|
|
1508
|
+
activityStarted,
|
|
1509
|
+
activitySpan,
|
|
1510
|
+
dispatchId,
|
|
1511
|
+
noteCancellationRequested,
|
|
1512
|
+
machineOpObserver,
|
|
1513
|
+
leases,
|
|
1514
|
+
control,
|
|
1515
|
+
attempt,
|
|
1516
|
+
sandboxState,
|
|
1517
|
+
renewals,
|
|
1518
|
+
recordingState,
|
|
1519
|
+
eventing,
|
|
1520
|
+
providerTurn,
|
|
1521
|
+
workspaceRefs,
|
|
1522
|
+
runWorkspaceMutationForSandbox,
|
|
1523
|
+
requireResolvedSandboxForMutation,
|
|
1524
|
+
stopLeaseHeartbeat,
|
|
1525
|
+
abandonActiveRecording,
|
|
1526
|
+
turnCompletionMemoryCollector,
|
|
1527
|
+
});
|
|
1528
|
+
}
|
|
1529
|
+
};
|
|
1530
|
+
}
|