@opengeni/worker-bundle 0.7.5 → 0.10.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-BJUW5LCM.js +325 -0
- package/dist/chunk-BJUW5LCM.js.map +1 -0
- package/dist/host-export.d.ts +70 -0
- package/dist/host-export.js +7 -0
- package/dist/host-export.js.map +1 -0
- package/dist/index.d.ts +98 -6
- package/dist/index.js +4097 -1271
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +12360 -0
- package/package.json +18 -12
- package/src/activities/agent-turn.ts +2674 -828
- package/src/activities/capabilities.ts +11 -2
- package/src/activities/codex-capacity.ts +6 -11
- package/src/activities/codex-rotation.ts +75 -86
- package/src/activities/context-compaction.ts +104 -24
- package/src/activities/environment.ts +503 -97
- package/src/activities/expiring-material-renewal.ts +143 -0
- package/src/activities/goals.ts +4 -0
- package/src/activities/mcp-credentials.ts +38 -0
- package/src/activities/packs.ts +115 -2
- package/src/activities/rig-verification.ts +897 -244
- package/src/activities/run-credential-renewal.ts +70 -0
- package/src/activities/run-credentials.ts +132 -0
- package/src/activities/run-input.ts +23 -0
- package/src/activities/sandbox-lease.ts +33 -2
- package/src/activities/scheduled-tasks.ts +21 -0
- package/src/activities/session-state.ts +72 -0
- package/src/activities/streaming.ts +1 -0
- package/src/activities/toolspace-token-renewal.ts +57 -0
- package/src/activities/types.ts +44 -6
- package/src/activities/workspace-capture.ts +113 -93
- package/src/activities.ts +10 -2
- package/src/host-export-pump.ts +416 -0
- package/src/host-export.ts +22 -0
- package/src/http.ts +77 -28
- package/src/index.ts +360 -156
- package/src/observability-metrics.ts +83 -15
- package/src/sandbox-resume.ts +47 -1
- package/src/sandbox-routing.ts +1 -1
- package/src/worker-service-lifecycle.ts +104 -0
- package/src/workflows/activities.ts +13 -7
- package/src/workflows/rig-verification.ts +3 -1
- package/src/workflows/session.ts +219 -23
- package/src/workflows.ts +1 -0
|
@@ -11,9 +11,12 @@ import {
|
|
|
11
11
|
setActiveSandbox,
|
|
12
12
|
requireFile,
|
|
13
13
|
getSessionEvent,
|
|
14
|
+
getSessionRootId,
|
|
14
15
|
getSessionGoal,
|
|
15
16
|
getLatestRunState,
|
|
16
|
-
|
|
17
|
+
getHumanInputResumeForEvent,
|
|
18
|
+
getSessionHumanInputRequest,
|
|
19
|
+
installOrReadTurnExecutionPolicyForAttempt,
|
|
17
20
|
workspaceCodexSubscriptionActive,
|
|
18
21
|
acquireCodexCredentialLease,
|
|
19
22
|
armCodexCapacityWait,
|
|
@@ -47,7 +50,6 @@ import {
|
|
|
47
50
|
nextSessionHistoryPosition,
|
|
48
51
|
settleCodexCredentialLeaseLoss,
|
|
49
52
|
settleCodexCredentialFailover,
|
|
50
|
-
saveRunState,
|
|
51
53
|
upsertSandboxSessionEnvelope,
|
|
52
54
|
setSessionLastInputTokensForTurnAttempt,
|
|
53
55
|
sumUsageQuantity,
|
|
@@ -55,24 +57,27 @@ import {
|
|
|
55
57
|
accrueWarmSeconds,
|
|
56
58
|
getMaterializedSandboxFileResources,
|
|
57
59
|
markSandboxFileResourcesMaterialized,
|
|
60
|
+
areGitHubRepositoriesAllowedForWorkspace,
|
|
58
61
|
SandboxLeaseSupersededError,
|
|
59
62
|
SandboxImageConflictError,
|
|
60
|
-
|
|
63
|
+
isSessionEventPersistenceError,
|
|
61
64
|
getEnrollment,
|
|
62
65
|
abandonRecordingForTurnAttempt,
|
|
66
|
+
markSessionAttemptQuiesced,
|
|
63
67
|
type AppendEventInput,
|
|
64
68
|
type ActiveSandboxPointer,
|
|
65
69
|
type SandboxRecord,
|
|
66
70
|
type CodexCredentialLeaseResult,
|
|
67
71
|
type CodexCredentialLeaseSelectionContext,
|
|
72
|
+
type ApplySessionTurnSettlementInput,
|
|
68
73
|
type SessionTurnRecordingSettlement,
|
|
69
74
|
} from "@opengeni/db";
|
|
70
75
|
import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@opengeni/events";
|
|
71
76
|
import {
|
|
72
77
|
sandboxStateEntryFromRunState,
|
|
73
78
|
maxTurnsExceededRunState,
|
|
74
|
-
modelCallUsageTelemetry,
|
|
75
79
|
modelResponseUsageFromSdkEvent,
|
|
80
|
+
normalizeModelCallUsage,
|
|
76
81
|
normalizeSdkEvent,
|
|
77
82
|
sanitizeHistoryItemsForModel,
|
|
78
83
|
isEphemeralInternalContext,
|
|
@@ -81,10 +86,18 @@ import {
|
|
|
81
86
|
appendWorkspaceMemory,
|
|
82
87
|
composeAgentInstructions,
|
|
83
88
|
summarizeForCompaction,
|
|
89
|
+
CompactionProviderResponseError,
|
|
90
|
+
EmptyCompactionSummaryError,
|
|
84
91
|
ensureModalRegistryImage,
|
|
85
92
|
findCompactionNeededError,
|
|
86
93
|
materializeSandboxFileDownloads,
|
|
87
|
-
|
|
94
|
+
materializeRunCredentials,
|
|
95
|
+
clearRunCredentials,
|
|
96
|
+
clearRunCredentialsForAttempt,
|
|
97
|
+
withRunCredentialsSession,
|
|
98
|
+
refreshGitCredentialBindingTokenFiles,
|
|
99
|
+
refreshToolspaceTokenFile,
|
|
100
|
+
toolspaceTokenFileFromEnvironment,
|
|
88
101
|
sandboxFileDownloadFailureNote,
|
|
89
102
|
SUMMARY_BUFFER_TOKENS,
|
|
90
103
|
runOwnedSandboxSetup,
|
|
@@ -94,20 +107,28 @@ import {
|
|
|
94
107
|
type OpenGeniRuntime,
|
|
95
108
|
type ComputerToolMode,
|
|
96
109
|
type ModelResponseUsage,
|
|
110
|
+
type ModelCallUsageInput,
|
|
111
|
+
type ModelCallUsageNormalization,
|
|
97
112
|
type BuildAgentOptions,
|
|
113
|
+
type TurnToolCancellationFence,
|
|
98
114
|
type BackendUnresolvableCode,
|
|
99
115
|
type EstablishedSandboxSession,
|
|
100
116
|
type GitCredentialTokenWriterSession,
|
|
117
|
+
type NormalizedRunCredentialMaterial,
|
|
118
|
+
type RunCredentialCommandSession,
|
|
119
|
+
type ToolspaceTokenWriterSession,
|
|
101
120
|
deleteRecordingArtifacts,
|
|
102
121
|
stopRecording as stopRecordingOnBox,
|
|
103
122
|
} from "@opengeni/runtime";
|
|
123
|
+
import { connectionTokenResolverForTurn } from "./mcp-credentials";
|
|
104
124
|
import {
|
|
105
|
-
|
|
125
|
+
assertTurnExecutionPolicyMatchesConfigV1,
|
|
106
126
|
calculateModelUsageCostMicros,
|
|
107
127
|
configuredModelPricing,
|
|
108
128
|
configuredStaticUsageLimits,
|
|
109
129
|
sandboxWarmRateMicrosPerSecond,
|
|
110
130
|
settingsWithResolvedModelContext,
|
|
131
|
+
resolveTurnExecutionPolicyV1,
|
|
111
132
|
type ModelUsageInput,
|
|
112
133
|
type ModelProviderApi,
|
|
113
134
|
type RegistryProviderKind,
|
|
@@ -120,6 +141,7 @@ import {
|
|
|
120
141
|
settingsWithSessionMcpServersForRun,
|
|
121
142
|
} from "./capabilities";
|
|
122
143
|
import {
|
|
144
|
+
CODEX_USAGE_EXHAUSTED_PCT,
|
|
123
145
|
authoritativeCodexCapacityResetAt,
|
|
124
146
|
chooseRotationActive,
|
|
125
147
|
classifyCodexPin,
|
|
@@ -144,35 +166,59 @@ import {
|
|
|
144
166
|
CODEX_CLIENT_VERSION,
|
|
145
167
|
CODEX_FALLBACK_MODEL_SLUGS,
|
|
146
168
|
CodexReloginRequired,
|
|
169
|
+
classifyCodexResponseTimeoutError,
|
|
147
170
|
classifyCodexUsageLimitError,
|
|
148
171
|
codexRequestStorage,
|
|
149
172
|
isCodexTransportError,
|
|
150
173
|
type CodexRequestContext,
|
|
151
174
|
type CodexUsageHeaderSnapshot,
|
|
152
175
|
} from "@opengeni/codex";
|
|
153
|
-
import { mergeResourceRefs
|
|
176
|
+
import { mergeResourceRefs } from "./common";
|
|
177
|
+
import { enabledCapabilityMcpToolRefs, resolveSessionToolPolicy } from "@opengeni/core";
|
|
154
178
|
import { maybeCompactContext } from "./context-compaction";
|
|
155
179
|
import { TurnAttemptFencedError } from "./turn-attempt-fenced";
|
|
156
180
|
import {
|
|
181
|
+
assertGitCredentialRenewalTransportUnchanged,
|
|
182
|
+
gitCredentialAuthorityForTurn,
|
|
183
|
+
gitHubTokenMintSelections,
|
|
157
184
|
loadWorkspaceEnvironmentForRunWithCredentials,
|
|
185
|
+
mintSandboxToolspaceToken,
|
|
158
186
|
mintRunGitCredentials,
|
|
187
|
+
mintRunGitCredentialBinding,
|
|
159
188
|
sandboxEnvironmentForRun,
|
|
189
|
+
type GitHubTokenMintAuthorization,
|
|
160
190
|
type MintedRunGitCredentials,
|
|
161
191
|
} from "./environment";
|
|
162
192
|
import {
|
|
163
193
|
startGitCredentialRenewalLoop,
|
|
164
194
|
type GitCredentialRenewalController,
|
|
165
195
|
} from "./git-credential-renewal";
|
|
196
|
+
import {
|
|
197
|
+
RUN_CREDENTIAL_EXPIRY_LEAD_MS,
|
|
198
|
+
startRunCredentialRenewalLoop,
|
|
199
|
+
type RunCredentialRenewalController,
|
|
200
|
+
} from "./run-credential-renewal";
|
|
201
|
+
import {
|
|
202
|
+
TOOLSPACE_TOKEN_EXPIRY_LEAD_MS,
|
|
203
|
+
startToolspaceTokenRenewalLoop,
|
|
204
|
+
type ToolspaceTokenRenewalController,
|
|
205
|
+
} from "./toolspace-token-renewal";
|
|
206
|
+
import {
|
|
207
|
+
bindRunCredentialResolver,
|
|
208
|
+
runCredentialAuthNeededPayloads,
|
|
209
|
+
runCredentialModelNote,
|
|
210
|
+
} from "./run-credentials";
|
|
166
211
|
import { withCodexAppsTool, withFirstPartyTools } from "./goals";
|
|
167
212
|
import {
|
|
168
213
|
mergeRigDefaultVariableSetEnvironment,
|
|
169
214
|
resolveWorkspaceAgentInstructions,
|
|
170
215
|
resolveWorkspacePackRuntime,
|
|
216
|
+
resolveWorkspaceSkillLibraryRuntime,
|
|
171
217
|
settingsWithPackSandboxImage,
|
|
172
218
|
settingsWithRigImage,
|
|
173
219
|
} from "./packs";
|
|
174
220
|
import { deliverFailedChildTurnToParent } from "./parent-wake";
|
|
175
|
-
import { createSecretRedactor, identityRedactor } from "./redaction";
|
|
221
|
+
import { createSecretRedactor, identityRedactor, type SecretForRedaction } from "./redaction";
|
|
176
222
|
import { applyCodexHistoryStrip, turnInput, type TurnCodexAccount } from "./run-input";
|
|
177
223
|
import {
|
|
178
224
|
createRuntimeBatcher,
|
|
@@ -180,7 +226,12 @@ import {
|
|
|
180
226
|
nextStreamEvent,
|
|
181
227
|
startActivityHeartbeat,
|
|
182
228
|
} from "./streaming";
|
|
183
|
-
import type {
|
|
229
|
+
import type {
|
|
230
|
+
ActivityServices,
|
|
231
|
+
RunAgentTurnInput,
|
|
232
|
+
RunAgentTurnResult,
|
|
233
|
+
SessionAttemptQuiescenceProof,
|
|
234
|
+
} from "./types";
|
|
184
235
|
import {
|
|
185
236
|
resumeBoxForTurn,
|
|
186
237
|
acquireSelfhostedLeaseForTurn,
|
|
@@ -229,11 +280,13 @@ import {
|
|
|
229
280
|
import {
|
|
230
281
|
CAPABILITY_DESCRIPTORS,
|
|
231
282
|
evaluateWorkspaceModelPolicy,
|
|
232
|
-
|
|
283
|
+
readTurnExecutionPolicyV1,
|
|
233
284
|
type ResourceRef,
|
|
234
285
|
type SessionEvent,
|
|
235
286
|
type SessionEventType,
|
|
236
287
|
type SessionStatus,
|
|
288
|
+
type SessionTurn,
|
|
289
|
+
type TurnExecutionPolicyV1,
|
|
237
290
|
} from "@opengeni/contracts";
|
|
238
291
|
import { createHash, randomUUID } from "node:crypto";
|
|
239
292
|
|
|
@@ -243,6 +296,32 @@ import { createHash, randomUUID } from "node:crypto";
|
|
|
243
296
|
// budget against the same window.
|
|
244
297
|
export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
|
|
245
298
|
|
|
299
|
+
export function turnExecutionPolicyBillingIdentity(policy: TurnExecutionPolicyV1): {
|
|
300
|
+
externallyBilled: boolean;
|
|
301
|
+
codexSubscription: boolean;
|
|
302
|
+
} {
|
|
303
|
+
return {
|
|
304
|
+
externallyBilled: policy.billing.metering === "external",
|
|
305
|
+
codexSubscription:
|
|
306
|
+
policy.providerId === "codex-subscription" &&
|
|
307
|
+
policy.credentialSource.kind === "connected_subscription" &&
|
|
308
|
+
policy.credentialSource.provider === "codex",
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
export function legacyTurnExecutionPolicyInput(
|
|
313
|
+
turn: Pick<SessionTurn, "source" | "model" | "reasoningEffort">,
|
|
314
|
+
): Parameters<typeof resolveTurnExecutionPolicyV1>[1] {
|
|
315
|
+
const explicit = turn.source === "user" || turn.source === "api";
|
|
316
|
+
return {
|
|
317
|
+
modelId: turn.model,
|
|
318
|
+
requestedModelId: explicit ? turn.model : null,
|
|
319
|
+
modelSource: explicit ? "explicit" : "continuation",
|
|
320
|
+
reasoningEffort: turn.reasoningEffort,
|
|
321
|
+
reasoningSource: explicit ? "explicit" : "continuation",
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
|
|
246
325
|
/** A retryable provider fault recovers the accepted turn itself. Goal state is
|
|
247
326
|
* irrelevant: autonomous continuation and infrastructure recovery are separate
|
|
248
327
|
* concerns. */
|
|
@@ -293,6 +372,28 @@ export function codexWorkspaceMetricKey(workspaceId: string): string {
|
|
|
293
372
|
return createHash("sha256").update(workspaceId).digest("hex").slice(0, 12);
|
|
294
373
|
}
|
|
295
374
|
|
|
375
|
+
/** Stable public request identity across partial resumes and activity retries. */
|
|
376
|
+
export function stableHumanInputRequestId(
|
|
377
|
+
sessionId: string,
|
|
378
|
+
turnId: string,
|
|
379
|
+
toolCallId: string,
|
|
380
|
+
): string {
|
|
381
|
+
const hex = createHash("sha256")
|
|
382
|
+
.update("opengeni-human-input-v1\0")
|
|
383
|
+
.update(sessionId)
|
|
384
|
+
.update("\0")
|
|
385
|
+
.update(turnId)
|
|
386
|
+
.update("\0")
|
|
387
|
+
.update(toolCallId)
|
|
388
|
+
.digest("hex")
|
|
389
|
+
.slice(0, 32)
|
|
390
|
+
.split("");
|
|
391
|
+
hex[12] = "5";
|
|
392
|
+
hex[16] = ["8", "9", "a", "b"][Number.parseInt(hex[16] ?? "0", 16) % 4] ?? "8";
|
|
393
|
+
const value = hex.join("");
|
|
394
|
+
return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice(12, 16)}-${value.slice(16, 20)}-${value.slice(20)}`;
|
|
395
|
+
}
|
|
396
|
+
|
|
296
397
|
/**
|
|
297
398
|
* True when this activity attempt was cancelled because its hosting worker is
|
|
298
399
|
* shutting down gracefully (SIGTERM during a deploy), as opposed to a
|
|
@@ -302,12 +403,229 @@ export function isWorkerShutdownCancellation(error: unknown): boolean {
|
|
|
302
403
|
return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
|
|
303
404
|
}
|
|
304
405
|
|
|
406
|
+
/**
|
|
407
|
+
* Review captures and protective snapshots are cache/persistence housekeeping,
|
|
408
|
+
* never part of cancellation correctness. A control-fenced or Temporal-cancelled
|
|
409
|
+
* attempt must release its physical activity promptly so Steer/Pause can advance.
|
|
410
|
+
*/
|
|
411
|
+
export function shouldRunTurnEndWorkspacePersistence(input: {
|
|
412
|
+
activityStatus: RunAgentTurnResult["status"] | "unknown";
|
|
413
|
+
cancellationRequested: boolean;
|
|
414
|
+
}): boolean {
|
|
415
|
+
return input.activityStatus !== "cancelled" && !input.cancellationRequested;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* Temporal cancellation is delivery/transport state, never proof that the
|
|
420
|
+
* dying activity crossed its mandatory sandbox-tool fence. If that fence
|
|
421
|
+
* fails, surface the fence failure instead of retaining a misleading typed
|
|
422
|
+
* cancellation; replacement admission remains closed because no durable
|
|
423
|
+
* quiescence receipt was written.
|
|
424
|
+
*/
|
|
425
|
+
export function assertPhysicalToolQuiescenceForCancellation(input: {
|
|
426
|
+
acknowledgeQuiescence: boolean;
|
|
427
|
+
physicalToolQuiescenceConfirmed: boolean;
|
|
428
|
+
failure: unknown;
|
|
429
|
+
}): void {
|
|
430
|
+
if (!input.acknowledgeQuiescence || input.physicalToolQuiescenceConfirmed) return;
|
|
431
|
+
if (input.failure instanceof Error) throw input.failure;
|
|
432
|
+
throw new Error("Physical sandbox-tool quiescence could not be confirmed", {
|
|
433
|
+
cause: input.failure,
|
|
434
|
+
});
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/** A physically drained attempt must leave one durable recovery producer. The
|
|
438
|
+
* direct Postgres receipt is preferred; after its bounded retries exhaust, an
|
|
439
|
+
* exact Temporal proof signal is sufficient because the workflow persists it
|
|
440
|
+
* through an independently retrying DB-only control activity. */
|
|
441
|
+
export function assertSessionAttemptQuiescenceRecoveryDurable(input: {
|
|
442
|
+
acknowledgeQuiescence: boolean;
|
|
443
|
+
physicalToolQuiescenceConfirmed: boolean;
|
|
444
|
+
receiptOrProofDurable: boolean;
|
|
445
|
+
failure: unknown;
|
|
446
|
+
}): void {
|
|
447
|
+
if (
|
|
448
|
+
!input.acknowledgeQuiescence ||
|
|
449
|
+
!input.physicalToolQuiescenceConfirmed ||
|
|
450
|
+
input.receiptOrProofDurable
|
|
451
|
+
) {
|
|
452
|
+
return;
|
|
453
|
+
}
|
|
454
|
+
if (input.failure instanceof Error) throw input.failure;
|
|
455
|
+
throw new Error("Physical quiescence had no durable receipt or recovery proof", {
|
|
456
|
+
cause: input.failure,
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
const QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS = 250;
|
|
461
|
+
const QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS = 5_000;
|
|
462
|
+
|
|
463
|
+
/** Persist the authoritative receipt or durably hand the exact physical proof
|
|
464
|
+
* to Temporal. This retries signal delivery, not DB eligibility or workflow
|
|
465
|
+
* state. The proof object never changes between attempts. */
|
|
466
|
+
export async function persistOrSignalSessionAttemptQuiescence(input: {
|
|
467
|
+
proof: SessionAttemptQuiescenceProof;
|
|
468
|
+
persistReceipt: () => Promise<SessionEvent[]>;
|
|
469
|
+
publishEvents: (events: SessionEvent[]) => Promise<unknown>;
|
|
470
|
+
signalProof: ActivityServices["signalSessionAttemptQuiesced"];
|
|
471
|
+
sleep?: (ms: number) => Promise<void>;
|
|
472
|
+
heartbeat?: (attempt: number, delayMs: number) => void;
|
|
473
|
+
onReceiptFailure?: (error: unknown) => void;
|
|
474
|
+
onPublishFailure?: (error: unknown) => void;
|
|
475
|
+
onSignalFailure?: (error: unknown, attempt: number, delayMs: number) => void;
|
|
476
|
+
}): Promise<"receipt" | "signal"> {
|
|
477
|
+
let events: SessionEvent[];
|
|
478
|
+
try {
|
|
479
|
+
events = await input.persistReceipt();
|
|
480
|
+
} catch (receiptError) {
|
|
481
|
+
input.onReceiptFailure?.(receiptError);
|
|
482
|
+
if (!input.signalProof) {
|
|
483
|
+
throw new Error("Session-attempt quiescence proof signaler is unavailable", {
|
|
484
|
+
cause: receiptError,
|
|
485
|
+
});
|
|
486
|
+
}
|
|
487
|
+
const delay = input.sleep ?? sleep;
|
|
488
|
+
let retryMs = QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS;
|
|
489
|
+
let attempt = 1;
|
|
490
|
+
for (;;) {
|
|
491
|
+
try {
|
|
492
|
+
await input.signalProof(input.proof);
|
|
493
|
+
return "signal";
|
|
494
|
+
} catch (signalError) {
|
|
495
|
+
input.onSignalFailure?.(signalError, attempt, retryMs);
|
|
496
|
+
try {
|
|
497
|
+
input.heartbeat?.(attempt, retryMs);
|
|
498
|
+
} catch {
|
|
499
|
+
// Heartbeat telemetry is not proof delivery and cannot replace or
|
|
500
|
+
// interrupt the exact signal retry loop.
|
|
501
|
+
}
|
|
502
|
+
await delay(retryMs);
|
|
503
|
+
retryMs = Math.min(retryMs * 2, QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS);
|
|
504
|
+
attempt += 1;
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
try {
|
|
510
|
+
await input.publishEvents(events);
|
|
511
|
+
} catch (publishError) {
|
|
512
|
+
// Postgres already committed quiesced_at, the queue event, and the wake.
|
|
513
|
+
// NATS is live fanout only; never misclassify its failure as receipt loss.
|
|
514
|
+
input.onPublishFailure?.(publishError);
|
|
515
|
+
}
|
|
516
|
+
return "receipt";
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
/**
|
|
520
|
+
* Cross the non-detachable physical boundary for one dying attempt. Temporal's
|
|
521
|
+
* cancellation signal is intentionally absent: cancellation is transport, not
|
|
522
|
+
* proof that sandbox tools and attempt-owned credential writers have drained.
|
|
523
|
+
* A replacement may be admitted only after this function resolves and the
|
|
524
|
+
* caller durably persists or signals the exact quiescence receipt.
|
|
525
|
+
*/
|
|
526
|
+
export async function drainAttemptOwnedSandboxWriters(input: {
|
|
527
|
+
toolCancellationFence: Pick<TurnToolCancellationFence, "cancel" | "waitForQuiescence"> | null;
|
|
528
|
+
cancellationReason?: unknown;
|
|
529
|
+
toolspaceTokenRenewal: Pick<ToolspaceTokenRenewalController, "stop"> | null;
|
|
530
|
+
runCredentialRenewal: Pick<RunCredentialRenewalController, "stop"> | null;
|
|
531
|
+
}): Promise<void> {
|
|
532
|
+
if (input.toolCancellationFence) {
|
|
533
|
+
input.toolCancellationFence.cancel(
|
|
534
|
+
input.cancellationReason ?? new Error("TURN_ATTEMPT_FENCED"),
|
|
535
|
+
);
|
|
536
|
+
await input.toolCancellationFence.waitForQuiescence();
|
|
537
|
+
}
|
|
538
|
+
await input.toolspaceTokenRenewal?.stop();
|
|
539
|
+
await input.runCredentialRenewal?.stop();
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
/**
|
|
543
|
+
* Await a finalizer operation only while this Temporal activity still owns its
|
|
544
|
+
* execution window. Once Pause/Steer cancellation arrives, the operation keeps
|
|
545
|
+
* its own rejection handler and may finish its idempotent, attempt-scoped
|
|
546
|
+
* cleanup in the background, but it cannot pin activity terminalization or
|
|
547
|
+
* delay the separately receipt-gated replacement dispatch.
|
|
548
|
+
*/
|
|
549
|
+
export async function waitForTurnFinalizerStep<T>(
|
|
550
|
+
operation: Promise<T>,
|
|
551
|
+
signal: AbortSignal | undefined,
|
|
552
|
+
): Promise<T | undefined> {
|
|
553
|
+
if (!signal) return await operation;
|
|
554
|
+
if (signal.aborted) {
|
|
555
|
+
void operation.catch(() => undefined);
|
|
556
|
+
return undefined;
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
let resolveCancellation: (() => void) | undefined;
|
|
560
|
+
const cancelled = new Promise<undefined>((resolve) => {
|
|
561
|
+
resolveCancellation = () => resolve(undefined);
|
|
562
|
+
});
|
|
563
|
+
const cancel = (): void => {
|
|
564
|
+
void operation.catch(() => undefined);
|
|
565
|
+
resolveCancellation?.();
|
|
566
|
+
};
|
|
567
|
+
signal.addEventListener("abort", cancel, { once: true });
|
|
568
|
+
try {
|
|
569
|
+
return await Promise.race([operation, cancelled]);
|
|
570
|
+
} finally {
|
|
571
|
+
signal.removeEventListener("abort", cancel);
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
/**
|
|
576
|
+
* Flush provider-facing stream state while the attempt still owns its activity
|
|
577
|
+
* window. On Pause/Steer, both promises are detached with rejection handlers:
|
|
578
|
+
* neither a runtime batcher nor an uncooperative provider completion promise
|
|
579
|
+
* may pin the activity behind cancellation after durable writes are fenced.
|
|
580
|
+
*/
|
|
581
|
+
export async function waitForTurnStreamCleanup(
|
|
582
|
+
batcherFlush: Promise<unknown>,
|
|
583
|
+
providerCompleted: Promise<unknown>,
|
|
584
|
+
signal: AbortSignal | undefined,
|
|
585
|
+
): Promise<void> {
|
|
586
|
+
await waitForTurnFinalizerStep(batcherFlush, signal);
|
|
587
|
+
await waitForTurnFinalizerStep(providerCompleted, signal);
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
function turnFinalizerCancellationSignal(
|
|
591
|
+
temporalSignal: AbortSignal | undefined,
|
|
592
|
+
activityStatus: RunAgentTurnResult["status"] | "unknown",
|
|
593
|
+
): AbortSignal | undefined {
|
|
594
|
+
if (activityStatus !== "cancelled" || temporalSignal?.aborted) return temporalSignal;
|
|
595
|
+
const fenced = new AbortController();
|
|
596
|
+
fenced.abort(new Error("TURN_ATTEMPT_FENCED"));
|
|
597
|
+
return fenced.signal;
|
|
598
|
+
}
|
|
599
|
+
|
|
305
600
|
function compactionFailureReason(reason: string): string {
|
|
306
601
|
return reason.startsWith("compaction summarization failed:")
|
|
307
602
|
? reason
|
|
308
603
|
: `compaction summarization failed: ${reason}`;
|
|
309
604
|
}
|
|
310
605
|
|
|
606
|
+
function compactionFailureReasonFromError(error: unknown): string {
|
|
607
|
+
if (
|
|
608
|
+
error instanceof CompactionProviderResponseError ||
|
|
609
|
+
error instanceof EmptyCompactionSummaryError
|
|
610
|
+
) {
|
|
611
|
+
return compactionFailureReason(error.message);
|
|
612
|
+
}
|
|
613
|
+
const errorName = error instanceof Error && error.name ? error.name : "unknown error";
|
|
614
|
+
return compactionFailureReason(`unexpected ${errorName}`);
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
function isCompactionSummaryFailure(error: unknown): boolean {
|
|
618
|
+
return (
|
|
619
|
+
error instanceof CompactionProviderResponseError || error instanceof EmptyCompactionSummaryError
|
|
620
|
+
);
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
export function shouldRecoverCompactionProviderFailure(error: unknown): boolean {
|
|
624
|
+
if (!(error instanceof CompactionProviderResponseError)) return false;
|
|
625
|
+
if (isCodexTransportError(error) && classifyCodexUsageLimitError(error)) return true;
|
|
626
|
+
return agentRunFailurePayload(error).retryable === true;
|
|
627
|
+
}
|
|
628
|
+
|
|
311
629
|
export function classifyContextWindowOverflowError(
|
|
312
630
|
error: unknown,
|
|
313
631
|
): { message: string; code?: string; detail?: string } | null {
|
|
@@ -440,6 +758,20 @@ export function modelUsageSourceKey(input: {
|
|
|
440
758
|
return input.dispatchId ? `${input.dispatchId}:${input.positionalKey}` : input.positionalKey;
|
|
441
759
|
}
|
|
442
760
|
|
|
761
|
+
export function providerContextTokens(
|
|
762
|
+
usage:
|
|
763
|
+
| {
|
|
764
|
+
inputTokens?: number;
|
|
765
|
+
outputTokens?: number;
|
|
766
|
+
totalTokens?: number;
|
|
767
|
+
}
|
|
768
|
+
| null
|
|
769
|
+
| undefined,
|
|
770
|
+
): number | null {
|
|
771
|
+
const total = normalizeModelCallUsage(usage).totalTokens;
|
|
772
|
+
return total !== null && total > 0 ? total : null;
|
|
773
|
+
}
|
|
774
|
+
|
|
443
775
|
/**
|
|
444
776
|
* A provider call has already consumed tokens by the time its usage frame is
|
|
445
777
|
* available. Losing the Codex credential lease at the renewal checkpoint must
|
|
@@ -468,6 +800,260 @@ type TurnEventPublisher = (
|
|
|
468
800
|
immediate?: boolean,
|
|
469
801
|
) => Promise<{ events: SessionEvent[]; accepted: boolean }>;
|
|
470
802
|
|
|
803
|
+
export type ModelResponseUsageEventState = {
|
|
804
|
+
responseUsageCount: number;
|
|
805
|
+
providerContextRevision: number;
|
|
806
|
+
lastProviderContextTokensObserved: number | null;
|
|
807
|
+
claimedSourceKeys: Set<string>;
|
|
808
|
+
};
|
|
809
|
+
|
|
810
|
+
export type CompactionModelUsageEventState = {
|
|
811
|
+
usageCount: number;
|
|
812
|
+
claimedSourceKeys: Set<string>;
|
|
813
|
+
};
|
|
814
|
+
|
|
815
|
+
export function createModelResponseUsageEventState(
|
|
816
|
+
claimedSourceKeys: Set<string> = new Set<string>(),
|
|
817
|
+
): ModelResponseUsageEventState {
|
|
818
|
+
return {
|
|
819
|
+
responseUsageCount: 0,
|
|
820
|
+
providerContextRevision: 0,
|
|
821
|
+
lastProviderContextTokensObserved: null,
|
|
822
|
+
claimedSourceKeys,
|
|
823
|
+
};
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
export function createCompactionModelUsageEventState(
|
|
827
|
+
claimedSourceKeys: Set<string> = new Set<string>(),
|
|
828
|
+
): CompactionModelUsageEventState {
|
|
829
|
+
return { usageCount: 0, claimedSourceKeys };
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
export function modelResponseUsageContextSignal(
|
|
833
|
+
state: ModelResponseUsageEventState,
|
|
834
|
+
): { revision: number; totalTokens: number } | null {
|
|
835
|
+
return state.lastProviderContextTokensObserved === null
|
|
836
|
+
? null
|
|
837
|
+
: {
|
|
838
|
+
revision: state.providerContextRevision,
|
|
839
|
+
totalTokens: state.lastProviderContextTokensObserved,
|
|
840
|
+
};
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
/**
|
|
844
|
+
* Process one SDK stream event through the exact production usage path.
|
|
845
|
+
*
|
|
846
|
+
* The pinned Responses SDK mirrors one provider terminal response as both a
|
|
847
|
+
* normalized `response_done` and a raw `model/response.completed` event. Claim
|
|
848
|
+
* the stable response/source key before lease renewal or any usage side effect,
|
|
849
|
+
* and advance the positional ordinal only for a newly claimed response. The
|
|
850
|
+
* durable `agent.model.usage` source-key fence remains the cross-restart
|
|
851
|
+
* authority: a replay may retry the idempotent billing write, but it cannot
|
|
852
|
+
* advance process-local metrics, provider context, or attempt-owned signals.
|
|
853
|
+
*/
|
|
854
|
+
export async function processModelResponseUsageEvent(input: {
|
|
855
|
+
event: Parameters<typeof modelResponseUsageFromSdkEvent>[0];
|
|
856
|
+
state: ModelResponseUsageEventState;
|
|
857
|
+
dispatchId: string | null;
|
|
858
|
+
settings: Settings;
|
|
859
|
+
db: ActivityServices["db"];
|
|
860
|
+
observability: ActivityServices["observability"];
|
|
861
|
+
publish: TurnEventPublisher | null;
|
|
862
|
+
accountId: string;
|
|
863
|
+
workspaceId: string;
|
|
864
|
+
sessionId: string;
|
|
865
|
+
turnId: string;
|
|
866
|
+
turnAttemptId: string;
|
|
867
|
+
provider: string;
|
|
868
|
+
providerApi: ModelProviderApi;
|
|
869
|
+
model: string;
|
|
870
|
+
metricProvider: string;
|
|
871
|
+
externallyBilled: boolean;
|
|
872
|
+
servingCredentialId: string | null;
|
|
873
|
+
priorSessionCredentialId: string | null;
|
|
874
|
+
emittedSourceKeys: Set<string>;
|
|
875
|
+
renewLease: () => Promise<void>;
|
|
876
|
+
leaseLost: () => boolean;
|
|
877
|
+
leaseLostMessage: string;
|
|
878
|
+
setLastInputTokens: (tokens: number) => Promise<void>;
|
|
879
|
+
}): Promise<
|
|
880
|
+
| { status: "not_usage" }
|
|
881
|
+
| { status: "duplicate"; sourceKey: string }
|
|
882
|
+
| { status: "processed"; sourceKey: string; authoritative: boolean }
|
|
883
|
+
> {
|
|
884
|
+
const responseUsage = modelResponseUsageFromSdkEvent(input.event);
|
|
885
|
+
if (!responseUsage) {
|
|
886
|
+
return { status: "not_usage" };
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
const responseOrdinal = input.state.responseUsageCount + 1;
|
|
890
|
+
const sourceKey = modelUsageSourceKey({
|
|
891
|
+
responseId: responseUsage.responseId,
|
|
892
|
+
dispatchId: input.dispatchId,
|
|
893
|
+
positionalKey: `response-${responseOrdinal}`,
|
|
894
|
+
});
|
|
895
|
+
if (input.state.claimedSourceKeys.has(sourceKey)) {
|
|
896
|
+
return { status: "duplicate", sourceKey };
|
|
897
|
+
}
|
|
898
|
+
input.state.claimedSourceKeys.add(sourceKey);
|
|
899
|
+
input.state.responseUsageCount = responseOrdinal;
|
|
900
|
+
|
|
901
|
+
const normalizedUsage = normalizeModelCallUsage(responseUsage.usage);
|
|
902
|
+
const accountContext = modelCallAccountContext({
|
|
903
|
+
servingCredentialId: input.servingCredentialId,
|
|
904
|
+
priorSessionCredentialId: input.priorSessionCredentialId,
|
|
905
|
+
isFirstCallOfTurn: responseOrdinal === 1,
|
|
906
|
+
});
|
|
907
|
+
let authoritative = false;
|
|
908
|
+
await recordCompletedModelCallBeforeOwnershipFences({
|
|
909
|
+
renewLease: input.renewLease,
|
|
910
|
+
leaseLost: input.leaseLost,
|
|
911
|
+
leaseLostMessage: input.leaseLostMessage,
|
|
912
|
+
recordUsage: async () => {
|
|
913
|
+
await recordModelUsageAndDebitCredits(input.settings, input.db, {
|
|
914
|
+
accountId: input.accountId,
|
|
915
|
+
workspaceId: input.workspaceId,
|
|
916
|
+
sessionId: input.sessionId,
|
|
917
|
+
turnId: input.turnId,
|
|
918
|
+
turnAttemptId: input.turnAttemptId,
|
|
919
|
+
model: input.model,
|
|
920
|
+
externallyBilled: input.externallyBilled,
|
|
921
|
+
usage: responseUsage.usage,
|
|
922
|
+
normalizedUsage,
|
|
923
|
+
sourceKey,
|
|
924
|
+
observability: input.observability,
|
|
925
|
+
});
|
|
926
|
+
authoritative = await emitModelCallUsage({
|
|
927
|
+
observability: input.observability,
|
|
928
|
+
publish: input.publish,
|
|
929
|
+
accountId: input.accountId,
|
|
930
|
+
workspaceId: input.workspaceId,
|
|
931
|
+
sessionId: input.sessionId,
|
|
932
|
+
turnId: input.turnId,
|
|
933
|
+
provider: input.provider,
|
|
934
|
+
providerApi: input.providerApi,
|
|
935
|
+
model: input.model,
|
|
936
|
+
sourceKey,
|
|
937
|
+
usage: responseUsage,
|
|
938
|
+
normalizedUsage,
|
|
939
|
+
servingAccountHash: accountContext.servingAccountHash,
|
|
940
|
+
accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
|
|
941
|
+
emittedSourceKeys: input.emittedSourceKeys,
|
|
942
|
+
});
|
|
943
|
+
const observedInput = normalizedUsage.telemetry.inputTokens;
|
|
944
|
+
if (authoritative && observedInput !== null && observedInput > 0) {
|
|
945
|
+
recordModelInputTokens(input.observability, input.metricProvider, observedInput);
|
|
946
|
+
}
|
|
947
|
+
},
|
|
948
|
+
recordAttemptSignals: async () => {
|
|
949
|
+
if (!authoritative) return;
|
|
950
|
+
const observedTotal = normalizedUsage.totalTokens;
|
|
951
|
+
if (observedTotal !== null) {
|
|
952
|
+
input.state.lastProviderContextTokensObserved = observedTotal;
|
|
953
|
+
input.state.providerContextRevision += 1;
|
|
954
|
+
}
|
|
955
|
+
const observedInput = normalizedUsage.telemetry.inputTokens;
|
|
956
|
+
if (observedInput !== null && observedInput > 0) {
|
|
957
|
+
await input.setLastInputTokens(observedInput);
|
|
958
|
+
}
|
|
959
|
+
},
|
|
960
|
+
});
|
|
961
|
+
return { status: "processed", sourceKey, authoritative };
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
/**
|
|
965
|
+
* Apply the same source-key authority ordering to the compaction summarizer's
|
|
966
|
+
* usage callback. The summarizer can retry or mirror a terminal response just
|
|
967
|
+
* like the main stream, so claim before lease renewal, billing, durable usage,
|
|
968
|
+
* logging, or cache metrics. Durable source-key idempotency remains the
|
|
969
|
+
* cross-process authority after a worker restart.
|
|
970
|
+
*/
|
|
971
|
+
export async function processCompactionModelUsageEvent(input: {
|
|
972
|
+
usage: ModelResponseUsage;
|
|
973
|
+
state: CompactionModelUsageEventState;
|
|
974
|
+
dispatchId: string | null;
|
|
975
|
+
settings: Settings;
|
|
976
|
+
db: ActivityServices["db"];
|
|
977
|
+
observability: ActivityServices["observability"];
|
|
978
|
+
publish: TurnEventPublisher | null;
|
|
979
|
+
accountId: string;
|
|
980
|
+
workspaceId: string;
|
|
981
|
+
sessionId: string;
|
|
982
|
+
turnId: string;
|
|
983
|
+
turnAttemptId: string;
|
|
984
|
+
provider: string;
|
|
985
|
+
providerApi: ModelProviderApi;
|
|
986
|
+
model: string;
|
|
987
|
+
externallyBilled: boolean;
|
|
988
|
+
servingCredentialId: string | null;
|
|
989
|
+
priorSessionCredentialId: string | null;
|
|
990
|
+
emittedSourceKeys: Set<string>;
|
|
991
|
+
renewLease: () => Promise<void>;
|
|
992
|
+
leaseLost: () => boolean;
|
|
993
|
+
leaseLostMessage: string;
|
|
994
|
+
}): Promise<
|
|
995
|
+
| { status: "duplicate"; sourceKey: string }
|
|
996
|
+
| { status: "processed"; sourceKey: string; authoritative: boolean }
|
|
997
|
+
> {
|
|
998
|
+
const usageOrdinal = input.state.usageCount + 1;
|
|
999
|
+
const sourceKey = modelUsageSourceKey({
|
|
1000
|
+
responseId: input.usage.responseId,
|
|
1001
|
+
dispatchId: input.dispatchId,
|
|
1002
|
+
positionalKey: `compaction-${usageOrdinal}`,
|
|
1003
|
+
});
|
|
1004
|
+
if (input.state.claimedSourceKeys.has(sourceKey)) {
|
|
1005
|
+
return { status: "duplicate", sourceKey };
|
|
1006
|
+
}
|
|
1007
|
+
input.state.claimedSourceKeys.add(sourceKey);
|
|
1008
|
+
input.state.usageCount = usageOrdinal;
|
|
1009
|
+
|
|
1010
|
+
const accountContext = modelCallAccountContext({
|
|
1011
|
+
servingCredentialId: input.servingCredentialId,
|
|
1012
|
+
priorSessionCredentialId: input.priorSessionCredentialId,
|
|
1013
|
+
isFirstCallOfTurn: usageOrdinal === 1,
|
|
1014
|
+
});
|
|
1015
|
+
const normalizedUsage = normalizeModelCallUsage(input.usage.usage);
|
|
1016
|
+
let authoritative = false;
|
|
1017
|
+
await recordCompletedModelCallBeforeOwnershipFences({
|
|
1018
|
+
renewLease: input.renewLease,
|
|
1019
|
+
leaseLost: input.leaseLost,
|
|
1020
|
+
leaseLostMessage: input.leaseLostMessage,
|
|
1021
|
+
recordUsage: async () => {
|
|
1022
|
+
await recordModelUsageAndDebitCredits(input.settings, input.db, {
|
|
1023
|
+
accountId: input.accountId,
|
|
1024
|
+
workspaceId: input.workspaceId,
|
|
1025
|
+
sessionId: input.sessionId,
|
|
1026
|
+
turnId: input.turnId,
|
|
1027
|
+
turnAttemptId: input.turnAttemptId,
|
|
1028
|
+
model: input.model,
|
|
1029
|
+
externallyBilled: input.externallyBilled,
|
|
1030
|
+
usage: input.usage.usage,
|
|
1031
|
+
normalizedUsage,
|
|
1032
|
+
sourceKey,
|
|
1033
|
+
observability: input.observability,
|
|
1034
|
+
});
|
|
1035
|
+
authoritative = await emitModelCallUsage({
|
|
1036
|
+
observability: input.observability,
|
|
1037
|
+
publish: input.publish,
|
|
1038
|
+
accountId: input.accountId,
|
|
1039
|
+
workspaceId: input.workspaceId,
|
|
1040
|
+
sessionId: input.sessionId,
|
|
1041
|
+
turnId: input.turnId,
|
|
1042
|
+
provider: input.provider,
|
|
1043
|
+
providerApi: input.providerApi,
|
|
1044
|
+
model: input.model,
|
|
1045
|
+
sourceKey,
|
|
1046
|
+
usage: input.usage,
|
|
1047
|
+
normalizedUsage,
|
|
1048
|
+
servingAccountHash: accountContext.servingAccountHash,
|
|
1049
|
+
accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
|
|
1050
|
+
emittedSourceKeys: input.emittedSourceKeys,
|
|
1051
|
+
});
|
|
1052
|
+
},
|
|
1053
|
+
});
|
|
1054
|
+
return { status: "processed", sourceKey, authoritative };
|
|
1055
|
+
}
|
|
1056
|
+
|
|
471
1057
|
export async function emitModelCallUsage(input: {
|
|
472
1058
|
observability: ActivityServices["observability"];
|
|
473
1059
|
publish: TurnEventPublisher | null;
|
|
@@ -480,22 +1066,25 @@ export async function emitModelCallUsage(input: {
|
|
|
480
1066
|
model: string;
|
|
481
1067
|
sourceKey: string;
|
|
482
1068
|
usage: ModelResponseUsage | { usage?: unknown | null } | null;
|
|
1069
|
+
normalizedUsage?: ModelCallUsageNormalization;
|
|
483
1070
|
// Prompt-cache research dimensions (log-only; NEVER on a metric label or a
|
|
484
1071
|
// durable event). The opaque serving-account tag and whether it changed since
|
|
485
1072
|
// the session's previous call — the account-switch hypothesis for cache misses.
|
|
486
1073
|
servingAccountHash?: string;
|
|
487
1074
|
accountChangedFromPrevCall?: boolean;
|
|
488
1075
|
emittedSourceKeys?: Set<string>;
|
|
489
|
-
}): Promise<
|
|
1076
|
+
}): Promise<boolean> {
|
|
490
1077
|
const usage =
|
|
491
1078
|
input.usage && typeof input.usage === "object" && "usage" in input.usage
|
|
492
1079
|
? (input.usage as { usage?: unknown }).usage
|
|
493
1080
|
: null;
|
|
494
1081
|
if (!usage || typeof usage !== "object") {
|
|
495
|
-
return;
|
|
1082
|
+
return false;
|
|
496
1083
|
}
|
|
497
|
-
if (input.emittedSourceKeys?.has(input.sourceKey)) return;
|
|
498
|
-
const
|
|
1084
|
+
if (input.emittedSourceKeys?.has(input.sourceKey)) return false;
|
|
1085
|
+
const normalizedUsage =
|
|
1086
|
+
input.normalizedUsage ?? normalizeModelCallUsage(usage as ModelCallUsageInput);
|
|
1087
|
+
const telemetry = normalizedUsage.telemetry;
|
|
499
1088
|
const appended = await input.publish?.(
|
|
500
1089
|
[
|
|
501
1090
|
{
|
|
@@ -524,7 +1113,7 @@ export async function emitModelCallUsage(input: {
|
|
|
524
1113
|
typeof event.payload === "object" &&
|
|
525
1114
|
(event.payload as Record<string, unknown>).sourceKey === input.sourceKey,
|
|
526
1115
|
);
|
|
527
|
-
if (!authoritative) return;
|
|
1116
|
+
if (!authoritative) return false;
|
|
528
1117
|
try {
|
|
529
1118
|
input.observability.info("model call usage", {
|
|
530
1119
|
accountId: input.accountId,
|
|
@@ -538,6 +1127,7 @@ export async function emitModelCallUsage(input: {
|
|
|
538
1127
|
inputTokens: telemetry.inputTokens,
|
|
539
1128
|
outputTokens: telemetry.outputTokens,
|
|
540
1129
|
cachedTokens: telemetry.cachedTokens,
|
|
1130
|
+
cacheWriteTokens: telemetry.cacheWriteTokens,
|
|
541
1131
|
reasoningTokens: telemetry.reasoningTokens,
|
|
542
1132
|
...(input.servingAccountHash !== undefined
|
|
543
1133
|
? { servingAccountHash: input.servingAccountHash }
|
|
@@ -546,9 +1136,41 @@ export async function emitModelCallUsage(input: {
|
|
|
546
1136
|
? { accountChangedFromPrevCall: input.accountChangedFromPrevCall }
|
|
547
1137
|
: {}),
|
|
548
1138
|
});
|
|
1139
|
+
if (normalizedUsage.rejectedFields.length > 0) {
|
|
1140
|
+
input.observability.warn("model call usage fields rejected", {
|
|
1141
|
+
provider: input.provider,
|
|
1142
|
+
providerApi: input.providerApi,
|
|
1143
|
+
model: input.model,
|
|
1144
|
+
sourceKey: input.sourceKey,
|
|
1145
|
+
rejectedFields: normalizedUsage.rejectedFields.join(","),
|
|
1146
|
+
});
|
|
1147
|
+
}
|
|
549
1148
|
} catch {
|
|
550
1149
|
// Durable event + billing already committed; logging is best-effort only.
|
|
551
1150
|
}
|
|
1151
|
+
try {
|
|
1152
|
+
applyCodexCacheTelemetry(input.observability, input.provider, normalizedUsage);
|
|
1153
|
+
} catch {
|
|
1154
|
+
// Durable event + billing already committed; metrics are best-effort only.
|
|
1155
|
+
}
|
|
1156
|
+
return true;
|
|
1157
|
+
}
|
|
1158
|
+
|
|
1159
|
+
/**
|
|
1160
|
+
* Apply one authoritative, normalized model-call usage frame to the shared
|
|
1161
|
+
* prompt-cache metrics. The durable source-key fence in `emitModelCallUsage`
|
|
1162
|
+
* owns idempotency; this helper must never receive raw provider values.
|
|
1163
|
+
*/
|
|
1164
|
+
export function applyCodexCacheTelemetry(
|
|
1165
|
+
observability: ActivityServices["observability"],
|
|
1166
|
+
provider: string,
|
|
1167
|
+
normalizedUsage: ModelCallUsageNormalization,
|
|
1168
|
+
): void {
|
|
1169
|
+
recordModelCacheTokens(observability, provider, {
|
|
1170
|
+
cachedTokens: normalizedUsage.telemetry.cachedTokens,
|
|
1171
|
+
cacheWriteTokens: normalizedUsage.telemetry.cacheWriteTokens,
|
|
1172
|
+
promptTokens: normalizedUsage.telemetry.inputTokens,
|
|
1173
|
+
});
|
|
552
1174
|
}
|
|
553
1175
|
|
|
554
1176
|
export function historyRowsToAppend(
|
|
@@ -565,12 +1187,13 @@ export function historyRowsToAppend(
|
|
|
565
1187
|
// persistedHistoryCount to preserve the pre-compaction behaviour (contiguous
|
|
566
1188
|
// positions from 0) when callers do not pass an explicit next position.
|
|
567
1189
|
nextPosition: number = persistedHistoryCount,
|
|
1190
|
+
toolOutputTruncationTokens?: number,
|
|
568
1191
|
): {
|
|
569
1192
|
rows: Array<{ position: number; item: Record<string, unknown> }>;
|
|
570
1193
|
nextWatermark: number;
|
|
571
1194
|
nextPosition: number;
|
|
572
1195
|
} {
|
|
573
|
-
const sanitized = sanitizeHistoryItemsForModel(rawHistory).filter(
|
|
1196
|
+
const sanitized = sanitizeHistoryItemsForModel(rawHistory, toolOutputTruncationTokens).filter(
|
|
574
1197
|
(item) => !isEphemeralInternalContext(item),
|
|
575
1198
|
);
|
|
576
1199
|
if (sanitized.length <= persistedHistoryCount) {
|
|
@@ -916,6 +1539,86 @@ export type TurnSandboxProvisioner<T> = {
|
|
|
916
1539
|
waitForSettled(timeoutMs: number): Promise<T | null>;
|
|
917
1540
|
};
|
|
918
1541
|
|
|
1542
|
+
export class TurnOperationCancelledError extends Error {
|
|
1543
|
+
readonly name = "TurnOperationCancelledError";
|
|
1544
|
+
|
|
1545
|
+
constructor(readonly reason: unknown) {
|
|
1546
|
+
super("Turn operation was cancelled with its owning turn", {
|
|
1547
|
+
...(reason instanceof Error ? { cause: reason } : {}),
|
|
1548
|
+
});
|
|
1549
|
+
}
|
|
1550
|
+
}
|
|
1551
|
+
|
|
1552
|
+
/**
|
|
1553
|
+
* Normalize a preparation/provisioning cancellation race back to the Temporal
|
|
1554
|
+
* cancellation that owns the activity. Several provider APIs expose no portable
|
|
1555
|
+
* abort primitive, so the worker stops awaiting them and disposes any late
|
|
1556
|
+
* resource. The wrapper error must never fall through as an ordinary turn
|
|
1557
|
+
* failure: doing so would omit the quiescence receipt and strand a committed
|
|
1558
|
+
* Steer/Pause behind `control-pending`.
|
|
1559
|
+
*/
|
|
1560
|
+
export function turnOperationCancellationFailure(error: unknown): CancelledFailure | null {
|
|
1561
|
+
if (error instanceof CancelledFailure) return error;
|
|
1562
|
+
if (!(error instanceof TurnOperationCancelledError)) return null;
|
|
1563
|
+
return error.reason instanceof CancelledFailure
|
|
1564
|
+
? error.reason
|
|
1565
|
+
: new CancelledFailure("TURN_SANDBOX_PROVISION_CANCELLED", [], error);
|
|
1566
|
+
}
|
|
1567
|
+
|
|
1568
|
+
function throwIfTurnOperationCancelled(signal: AbortSignal | undefined): void {
|
|
1569
|
+
if (signal?.aborted) {
|
|
1570
|
+
throw new TurnOperationCancelledError(signal.reason);
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1573
|
+
|
|
1574
|
+
export async function waitForTurnOperation<T>(
|
|
1575
|
+
operation: Promise<T>,
|
|
1576
|
+
signal: AbortSignal | undefined,
|
|
1577
|
+
disposeLateResult: ((result: T) => Promise<void> | void) | undefined,
|
|
1578
|
+
): Promise<T> {
|
|
1579
|
+
if (!signal) return await operation;
|
|
1580
|
+
|
|
1581
|
+
let rejectCancellation: ((error: TurnOperationCancelledError) => void) | undefined;
|
|
1582
|
+
const cancelled = new Promise<never>((_resolve, reject) => {
|
|
1583
|
+
rejectCancellation = reject;
|
|
1584
|
+
});
|
|
1585
|
+
const cancel = (): void => {
|
|
1586
|
+
rejectCancellation?.(new TurnOperationCancelledError(signal.reason));
|
|
1587
|
+
};
|
|
1588
|
+
signal.addEventListener("abort", cancel, { once: true });
|
|
1589
|
+
if (signal.aborted) cancel();
|
|
1590
|
+
|
|
1591
|
+
try {
|
|
1592
|
+
const result = await Promise.race([operation, cancelled]);
|
|
1593
|
+
// Cancellation owns an exact turn boundary even when the provider result
|
|
1594
|
+
// and AbortSignal settle in the same microtask checkpoint. Never let a
|
|
1595
|
+
// just-resolved lease escape after the control was already committed.
|
|
1596
|
+
if (signal.aborted) {
|
|
1597
|
+
throw new TurnOperationCancelledError(signal.reason);
|
|
1598
|
+
}
|
|
1599
|
+
return result;
|
|
1600
|
+
} catch (error) {
|
|
1601
|
+
if (signal.aborted) {
|
|
1602
|
+
// The provider establish call has no universal cancellation seam. It may
|
|
1603
|
+
// finish after the Temporal activity has correctly stopped; dispose its
|
|
1604
|
+
// late lease instead of letting a cancelled turn resurrect a holder/box.
|
|
1605
|
+
void operation
|
|
1606
|
+
.then(async (result) => await disposeLateResult?.(result))
|
|
1607
|
+
.catch(() => undefined);
|
|
1608
|
+
// A provider failure may settle in the same checkpoint as the committed
|
|
1609
|
+
// control. The control is authoritative; retain its cancellation shape
|
|
1610
|
+
// so the activity publishes quiescence instead of looking like an
|
|
1611
|
+
// unrelated turn failure.
|
|
1612
|
+
if (!(error instanceof TurnOperationCancelledError)) {
|
|
1613
|
+
throw new TurnOperationCancelledError(signal.reason);
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
throw error;
|
|
1617
|
+
} finally {
|
|
1618
|
+
signal.removeEventListener("abort", cancel);
|
|
1619
|
+
}
|
|
1620
|
+
}
|
|
1621
|
+
|
|
919
1622
|
function sleep(ms: number): Promise<void> {
|
|
920
1623
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
921
1624
|
}
|
|
@@ -938,9 +1641,11 @@ export function createTurnSandboxProvisioner<T>(
|
|
|
938
1641
|
options: {
|
|
939
1642
|
maxRetries?: number;
|
|
940
1643
|
backoffMs?: number;
|
|
1644
|
+
signal?: AbortSignal;
|
|
941
1645
|
onStarted?: () => Promise<void> | void;
|
|
942
|
-
onCompleted?: () => Promise<void> | void;
|
|
1646
|
+
onCompleted?: (result: T) => Promise<void> | void;
|
|
943
1647
|
onFailed?: (error: unknown) => Promise<void> | void;
|
|
1648
|
+
disposeResult?: (result: T) => Promise<void> | void;
|
|
944
1649
|
} = {},
|
|
945
1650
|
): TurnSandboxProvisioner<T> {
|
|
946
1651
|
const maxRetries = options.maxRetries ?? 2;
|
|
@@ -951,9 +1656,15 @@ export function createTurnSandboxProvisioner<T>(
|
|
|
951
1656
|
let attempt = 0;
|
|
952
1657
|
while (true) {
|
|
953
1658
|
try {
|
|
954
|
-
|
|
1659
|
+
throwIfTurnOperationCancelled(options.signal);
|
|
1660
|
+
const operation = establish();
|
|
1661
|
+
return await waitForTurnOperation(operation, options.signal, options.disposeResult);
|
|
955
1662
|
} catch (error) {
|
|
956
|
-
if (
|
|
1663
|
+
if (
|
|
1664
|
+
error instanceof TurnOperationCancelledError ||
|
|
1665
|
+
attempt >= maxRetries ||
|
|
1666
|
+
!isLazySandboxProvisionRetryable(error)
|
|
1667
|
+
) {
|
|
957
1668
|
throw error;
|
|
958
1669
|
}
|
|
959
1670
|
attempt += 1;
|
|
@@ -966,13 +1677,24 @@ export function createTurnSandboxProvisioner<T>(
|
|
|
966
1677
|
get(): Promise<T> {
|
|
967
1678
|
if (!memo) {
|
|
968
1679
|
memo = (async () => {
|
|
1680
|
+
throwIfTurnOperationCancelled(options.signal);
|
|
969
1681
|
await options.onStarted?.();
|
|
1682
|
+
throwIfTurnOperationCancelled(options.signal);
|
|
1683
|
+
let result: T | undefined;
|
|
1684
|
+
let hasResult = false;
|
|
970
1685
|
try {
|
|
971
|
-
|
|
972
|
-
|
|
1686
|
+
result = await run();
|
|
1687
|
+
hasResult = true;
|
|
1688
|
+
throwIfTurnOperationCancelled(options.signal);
|
|
1689
|
+
await options.onCompleted?.(result);
|
|
973
1690
|
return result;
|
|
974
1691
|
} catch (error) {
|
|
975
|
-
|
|
1692
|
+
if (hasResult) {
|
|
1693
|
+
await options.disposeResult?.(result as T);
|
|
1694
|
+
}
|
|
1695
|
+
if (!(error instanceof TurnOperationCancelledError)) {
|
|
1696
|
+
await options.onFailed?.(error);
|
|
1697
|
+
}
|
|
976
1698
|
throw error;
|
|
977
1699
|
}
|
|
978
1700
|
})().catch((error) => {
|
|
@@ -1041,7 +1763,7 @@ export function acceptsPromptCacheKeyForTurn(
|
|
|
1041
1763
|
* The turn hot path never refreshes Codex usage — only the usage API route does — so a
|
|
1042
1764
|
* window that has actually reset still reads OVER-threshold from the stale cache, which
|
|
1043
1765
|
* would idle-loop forever. Before idling, refresh LIVE usage for every connected account
|
|
1044
|
-
* the cache marks
|
|
1766
|
+
* the cache marks exhausted (bounded to the account count), which re-writes the
|
|
1045
1767
|
* cache columns, then return the re-read rows so the ranker can pick up a genuinely-reset
|
|
1046
1768
|
* window THIS turn. A refresh/read failure is swallowed (fall back to the pre-refresh rows
|
|
1047
1769
|
* + the bounded idle). Cooling (429'd) accounts are NOT refreshed: their exhaustedUntil
|
|
@@ -1051,17 +1773,29 @@ async function refreshCappedCodexUsageRows(
|
|
|
1051
1773
|
db: ActivityServices["db"],
|
|
1052
1774
|
settings: Settings,
|
|
1053
1775
|
workspaceId: string,
|
|
1054
|
-
accounts:
|
|
1776
|
+
accounts: Array<
|
|
1777
|
+
Pick<
|
|
1778
|
+
CodexAccountStatus,
|
|
1779
|
+
"id" | "status" | "primaryUsedPercent" | "secondaryUsedPercent" | "exhaustedUntil"
|
|
1780
|
+
>
|
|
1781
|
+
>,
|
|
1055
1782
|
capacitySignals: {
|
|
1056
1783
|
signalCodexCapacityWorkflow?: ActivityServices["signalCodexCapacityWorkflow"] | undefined;
|
|
1057
1784
|
wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
|
|
1058
1785
|
},
|
|
1059
|
-
): Promise<
|
|
1060
|
-
|
|
1786
|
+
): Promise<
|
|
1787
|
+
Array<
|
|
1788
|
+
Pick<
|
|
1789
|
+
CodexAccountStatus,
|
|
1790
|
+
"id" | "status" | "primaryUsedPercent" | "secondaryUsedPercent" | "exhaustedUntil"
|
|
1791
|
+
>
|
|
1792
|
+
>
|
|
1793
|
+
> {
|
|
1061
1794
|
const stale = accounts.filter(
|
|
1062
1795
|
(a) =>
|
|
1063
1796
|
a.status === "active" &&
|
|
1064
|
-
((a.primaryUsedPercent ?? 0) >=
|
|
1797
|
+
((a.primaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT ||
|
|
1798
|
+
(a.secondaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT),
|
|
1065
1799
|
);
|
|
1066
1800
|
if (stale.length === 0) {
|
|
1067
1801
|
return accounts;
|
|
@@ -1097,11 +1831,22 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1097
1831
|
objectStorage,
|
|
1098
1832
|
observability,
|
|
1099
1833
|
wakeSessionWorkflow,
|
|
1834
|
+
signalSessionAttemptQuiesced,
|
|
1100
1835
|
signalCodexCapacityWorkflow,
|
|
1101
1836
|
entitlements,
|
|
1102
1837
|
connectionCredentials,
|
|
1103
1838
|
} = await services();
|
|
1104
1839
|
const activityContext = currentActivityContext();
|
|
1840
|
+
const cancellationSignal = activityContext?.cancellationSignal;
|
|
1841
|
+
let cancellationRequestedAt: number | null = cancellationSignal?.aborted
|
|
1842
|
+
? performance.now()
|
|
1843
|
+
: null;
|
|
1844
|
+
const noteCancellationRequested = (): void => {
|
|
1845
|
+
cancellationRequestedAt ??= performance.now();
|
|
1846
|
+
};
|
|
1847
|
+
cancellationSignal?.addEventListener("abort", noteCancellationRequested, {
|
|
1848
|
+
once: true,
|
|
1849
|
+
});
|
|
1105
1850
|
const dispatchId = activityContext?.info.activityId ?? randomUUID();
|
|
1106
1851
|
const activityStarted = performance.now();
|
|
1107
1852
|
const activitySpan = observability.startSpan("worker.run_agent_segment", {
|
|
@@ -1112,6 +1857,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1112
1857
|
let activityStatus: RunAgentTurnResult["status"] | "unknown" = "unknown";
|
|
1113
1858
|
let turnMetricOutcome: TurnOutcome | null = null;
|
|
1114
1859
|
let activityError: unknown;
|
|
1860
|
+
let acknowledgeQuiescence = false;
|
|
1861
|
+
const acknowledgeLostAttemptOwnership = (): void => {
|
|
1862
|
+
// A stale terminal/recovery settlement can lose either to a benign
|
|
1863
|
+
// successor or to Pause/Steer closing this exact attempt. Only the
|
|
1864
|
+
// receipt transaction can distinguish those cases after the hard tool
|
|
1865
|
+
// fence: allowUninterrupted makes the benign case an event-free no-op.
|
|
1866
|
+
acknowledgeQuiescence = true;
|
|
1867
|
+
noteCancellationRequested();
|
|
1868
|
+
};
|
|
1115
1869
|
let turnId: string | undefined;
|
|
1116
1870
|
let triggerEventId: string | undefined;
|
|
1117
1871
|
const claimedResult = (
|
|
@@ -1135,17 +1889,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1135
1889
|
runtimeMetricsHooksForObservability(observability),
|
|
1136
1890
|
);
|
|
1137
1891
|
let isCodexTurn = false;
|
|
1892
|
+
let isExternallyBilledTurn = false;
|
|
1138
1893
|
let executionGeneration = 0;
|
|
1139
1894
|
// Still required by credential-loss/capacity settlements, whose own
|
|
1140
1895
|
// recovery transactions fence against worker-death redispatches.
|
|
1141
1896
|
let redispatchesAtDispatch = 0;
|
|
1142
|
-
const saveRunStateFenced = async (state: Parameters<typeof saveRunState>[1]): Promise<void> => {
|
|
1143
|
-
if (!(await saveRunState(db, state))) {
|
|
1144
|
-
throw new TurnAttemptFencedError(
|
|
1145
|
-
"turn execution generation was fenced while saving run state",
|
|
1146
|
-
);
|
|
1147
|
-
}
|
|
1148
|
-
};
|
|
1149
1897
|
const setLastInputTokensFenced = async (lastInputTokens: number): Promise<void> => {
|
|
1150
1898
|
if (!turnId || executionGeneration <= 0) {
|
|
1151
1899
|
throw new Error("Turn attempt was not initialized before token accounting");
|
|
@@ -1164,7 +1912,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1164
1912
|
}
|
|
1165
1913
|
};
|
|
1166
1914
|
let heartbeatTimer: ReturnType<typeof startActivityHeartbeat> | undefined;
|
|
1167
|
-
//
|
|
1915
|
+
// credential allocator: one workspace-local idempotent credential holder per running
|
|
1168
1916
|
// Codex turn. The DB row is the cross-replica fairness primitive; this timer
|
|
1169
1917
|
// only extends its short TTL. A killed worker stops heartbeating and the
|
|
1170
1918
|
// holder self-expires. Other workspaces never see or share this holder.
|
|
@@ -1295,11 +2043,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1295
2043
|
// runs it refreshes expires_at epoch-fenced so a legit multi-day turn is
|
|
1296
2044
|
// never TTL-reaped. Cleared in finally. Only set when the flag resolved a box.
|
|
1297
2045
|
let leaseHeartbeatTimer: ReturnType<typeof setInterval> | undefined;
|
|
1298
|
-
//
|
|
2046
|
+
// credential-renewal policy: the worker, not the model, owns renewal of run-scoped Git
|
|
1299
2047
|
// credentials for a multi-day turn. The controller is attached only after
|
|
1300
2048
|
// the initial seed reached a real cloud box and is drained before capture.
|
|
1301
|
-
let
|
|
2049
|
+
let gitCredentialRenewals: GitCredentialRenewalController[] = [];
|
|
1302
2050
|
let gitCredentialRenewalClosed = false;
|
|
2051
|
+
// Generic host-owned run material has its own attempt-scoped renewal and
|
|
2052
|
+
// write handle. It is always drained and wiped before workspace capture.
|
|
2053
|
+
let runCredentialRenewal: RunCredentialRenewalController | null = null;
|
|
2054
|
+
let runCredentialRenewalClosed = false;
|
|
2055
|
+
let runCredentialSession: RunCredentialCommandSession | null = null;
|
|
2056
|
+
// The delegated Toolspace bearer has a one-hour TTL. Renewal is attempt-
|
|
2057
|
+
// owned and attaches only after the initial token file reached a real
|
|
2058
|
+
// sandbox session; finalization drains an in-flight replacement.
|
|
2059
|
+
let toolspaceTokenRenewal: ToolspaceTokenRenewalController | null = null;
|
|
2060
|
+
let toolspaceTokenRenewalClosed = false;
|
|
1303
2061
|
// MID-SESSION snapshot single-flight guard: the heartbeat tick fires every
|
|
1304
2062
|
// 10s but a Modal filesystem snapshot can take longer — never overlap two
|
|
1305
2063
|
// captures on one box. The in-flight capture's promise is held so the
|
|
@@ -1349,6 +2107,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1349
2107
|
await current?.flush().catch(() => undefined);
|
|
1350
2108
|
};
|
|
1351
2109
|
let preparedTools: Awaited<ReturnType<OpenGeniRuntime["prepareTools"]>> | null = null;
|
|
2110
|
+
const toolCancellationFenceRef: {
|
|
2111
|
+
current: TurnToolCancellationFence | null;
|
|
2112
|
+
} = {
|
|
2113
|
+
current: null,
|
|
2114
|
+
};
|
|
1352
2115
|
let publish: TurnEventPublisher | null = null;
|
|
1353
2116
|
let settle:
|
|
1354
2117
|
| ((input: {
|
|
@@ -1362,10 +2125,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1362
2125
|
| "requires_action";
|
|
1363
2126
|
sessionStatus: SessionStatus;
|
|
1364
2127
|
activeTurnId: string | null;
|
|
2128
|
+
consumeRequestedCompactionFailure?: boolean;
|
|
2129
|
+
runState?: ApplySessionTurnSettlementInput["runState"];
|
|
1365
2130
|
}) => Promise<boolean>)
|
|
1366
2131
|
| null = null;
|
|
1367
2132
|
let turnStartedPublished = false;
|
|
1368
2133
|
let stream: Awaited<ReturnType<OpenGeniRuntime["runStream"]>> | undefined;
|
|
2134
|
+
// Reconciliation is declared before provider routing so every turn-end path
|
|
2135
|
+
// can share one closure. It cannot run until `stream` exists, by which time
|
|
2136
|
+
// this value has been rebound to the turn's resolved model policy.
|
|
2137
|
+
let modelRunSettings: Settings = settings;
|
|
1369
2138
|
const publishSandboxLifecycleEvents = async (sandbox: ResumedTurnSandbox): Promise<void> => {
|
|
1370
2139
|
const established = sandbox.established;
|
|
1371
2140
|
if (publish && established.origin && established.origin !== "resumed") {
|
|
@@ -1465,16 +2234,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1465
2234
|
// selfhosted target has no persistWorkspace anyway. Best-effort +
|
|
1466
2235
|
// single-flight; throttling lives in the helper.
|
|
1467
2236
|
const snapshotSession = setupBoxSession;
|
|
1468
|
-
|
|
2237
|
+
const snapshotTurnId = turnId;
|
|
2238
|
+
if (snapshotSession && snapshotTurnId && !snapshotInFlight) {
|
|
1469
2239
|
snapshotInFlight = maybePersistWarmWorkspaceSnapshot(
|
|
1470
2240
|
{ db, settings },
|
|
1471
2241
|
{
|
|
1472
2242
|
accountId: input.accountId,
|
|
1473
2243
|
workspaceId: input.workspaceId,
|
|
2244
|
+
sessionId: input.sessionId,
|
|
2245
|
+
turnId: snapshotTurnId,
|
|
2246
|
+
attemptId: input.attemptId,
|
|
1474
2247
|
sandboxGroupId: heartbeatGroupId,
|
|
1475
2248
|
},
|
|
1476
2249
|
snapshotSession,
|
|
1477
2250
|
heartbeatEpoch,
|
|
2251
|
+
activityContext?.cancellationSignal,
|
|
1478
2252
|
)
|
|
1479
2253
|
.then(async (persisted) => {
|
|
1480
2254
|
if (persisted && publish) {
|
|
@@ -1602,6 +2376,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1602
2376
|
rawHistory as Array<Record<string, unknown>>,
|
|
1603
2377
|
persistedHistoryCount,
|
|
1604
2378
|
nextHistoryPosition,
|
|
2379
|
+
modelRunSettings.modelToolOutputTruncationTokens,
|
|
1605
2380
|
);
|
|
1606
2381
|
const hasModelOrToolProgress = rows.some((row) =>
|
|
1607
2382
|
isModelOrToolProgressHistoryItem(row.item),
|
|
@@ -1621,6 +2396,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1621
2396
|
// runs, so this is the turn's effective account. The read path uses
|
|
1622
2397
|
// it to strip cross-account reasoning.encrypted_content next turn.
|
|
1623
2398
|
producerCodexCredentialId: effectiveCodexCredentialId,
|
|
2399
|
+
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
1624
2400
|
items: rows,
|
|
1625
2401
|
});
|
|
1626
2402
|
if (!appended) {
|
|
@@ -1651,6 +2427,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1651
2427
|
// Reassigned after the variable set loads; the publish closure is
|
|
1652
2428
|
// created (and used for turn.started) before the variableSet is available.
|
|
1653
2429
|
let redact: (payload: unknown) => unknown = identityRedactor;
|
|
2430
|
+
const secretRedactions = new Map<string, string>();
|
|
2431
|
+
const publishedRunCredentialNotices = new Set<string>();
|
|
2432
|
+
const registerSecretRedactions = (secrets: SecretForRedaction[]): void => {
|
|
2433
|
+
for (const secret of secrets) {
|
|
2434
|
+
if (!secretRedactions.has(secret.value)) secretRedactions.set(secret.value, secret.name);
|
|
2435
|
+
}
|
|
2436
|
+
redact = createSecretRedactor(
|
|
2437
|
+
[...secretRedactions].map(([value, name]) => ({ name, value })),
|
|
2438
|
+
);
|
|
2439
|
+
};
|
|
1654
2440
|
let variableSetId = "";
|
|
1655
2441
|
// Rig telemetry (M3): set once the session loads; empty string for a rig-less
|
|
1656
2442
|
// turn (mirrors variableSetId). Read by the activity span's finally block.
|
|
@@ -1682,10 +2468,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1682
2468
|
input.workspaceId,
|
|
1683
2469
|
settings,
|
|
1684
2470
|
);
|
|
1685
|
-
// Read the active-credential flag
|
|
1686
|
-
//
|
|
1687
|
-
//
|
|
1688
|
-
// provider-injection and billing disagree about whether this is a codex turn.
|
|
2471
|
+
// Read the active-credential flag once for the runtime capability overlay.
|
|
2472
|
+
// Accepted billing/provider identity comes from the turn policy below,
|
|
2473
|
+
// never from this mutable health snapshot.
|
|
1689
2474
|
const codexSubscriptionActive = await workspaceCodexSubscriptionActive(
|
|
1690
2475
|
db,
|
|
1691
2476
|
mcpSettings,
|
|
@@ -1713,13 +2498,50 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1713
2498
|
}
|
|
1714
2499
|
const turn = claim.turn;
|
|
1715
2500
|
turnId = turn.id;
|
|
2501
|
+
executionGeneration = turn.executionGeneration;
|
|
2502
|
+
const claimedPolicy = readTurnExecutionPolicyV1(turn.metadata);
|
|
2503
|
+
const policyForAbsent =
|
|
2504
|
+
claimedPolicy.kind === "valid"
|
|
2505
|
+
? claimedPolicy.policy
|
|
2506
|
+
: resolveTurnExecutionPolicyV1(capabilitySettings, legacyTurnExecutionPolicyInput(turn));
|
|
2507
|
+
const installedPolicy = await installOrReadTurnExecutionPolicyForAttempt(db, {
|
|
2508
|
+
accountId: input.accountId,
|
|
2509
|
+
workspaceId: input.workspaceId,
|
|
2510
|
+
sessionId: input.sessionId,
|
|
2511
|
+
turnId,
|
|
2512
|
+
executionGeneration,
|
|
2513
|
+
attemptId: input.attemptId,
|
|
2514
|
+
policyForAbsent,
|
|
2515
|
+
});
|
|
2516
|
+
if (!installedPolicy.accepted) {
|
|
2517
|
+
throw new TurnAttemptFencedError(
|
|
2518
|
+
`turn execution policy was fenced: ${installedPolicy.reason}`,
|
|
2519
|
+
);
|
|
2520
|
+
}
|
|
2521
|
+
const verifiedExecutionPolicy = assertTurnExecutionPolicyMatchesConfigV1(
|
|
2522
|
+
capabilitySettings,
|
|
2523
|
+
installedPolicy.policy,
|
|
2524
|
+
{
|
|
2525
|
+
modelId: turn.model,
|
|
2526
|
+
reasoningEffort: turn.reasoningEffort,
|
|
2527
|
+
},
|
|
2528
|
+
);
|
|
2529
|
+
const turnExecutionPolicy = verifiedExecutionPolicy.policy;
|
|
2530
|
+
const billingIdentity = turnExecutionPolicyBillingIdentity(turnExecutionPolicy);
|
|
2531
|
+
isExternallyBilledTurn = billingIdentity.externallyBilled;
|
|
2532
|
+
isCodexTurn = billingIdentity.codexSubscription;
|
|
1716
2533
|
triggerEventId = turn.triggerEventId;
|
|
1717
2534
|
const trigger = await getSessionEvent(db, input.workspaceId, triggerEventId);
|
|
1718
2535
|
if (!trigger) {
|
|
1719
2536
|
throw new Error(`Trigger event not found: ${triggerEventId}`);
|
|
1720
2537
|
}
|
|
2538
|
+
const humanInputResume = await getHumanInputResumeForEvent(
|
|
2539
|
+
db,
|
|
2540
|
+
input.workspaceId,
|
|
2541
|
+
input.sessionId,
|
|
2542
|
+
trigger,
|
|
2543
|
+
);
|
|
1721
2544
|
triggerType = trigger.type;
|
|
1722
|
-
executionGeneration = turn.executionGeneration;
|
|
1723
2545
|
const latestTurnState = await getLatestRunState(db, input.workspaceId, input.sessionId);
|
|
1724
2546
|
const continuationCodexCredentialId =
|
|
1725
2547
|
latestTurnState?.turnId === turnId ? latestTurnState.frozenCodexCredentialId : null;
|
|
@@ -1728,30 +2550,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1728
2550
|
0,
|
|
1729
2551
|
);
|
|
1730
2552
|
turnLifecycleMetricsFor(observability).start(turnId);
|
|
1731
|
-
//
|
|
1732
|
-
//
|
|
1733
|
-
// gate + the usage recorder so a turn paid by the user's ChatGPT/Codex plan
|
|
1734
|
-
// consumes ZERO OpenGeni credits and never feeds an OpenGeni cap. Resolved
|
|
1735
|
-
// here (before resolvedModel at the routing step) because the pre-turn gate
|
|
1736
|
-
// below needs it; mirrors the same active-credential read the codex provider
|
|
1737
|
-
// overlay uses, so billing and routing agree on what "codex" is.
|
|
1738
|
-
isCodexTurn = await isCodexBilledTurn({
|
|
1739
|
-
db,
|
|
1740
|
-
settings,
|
|
1741
|
-
workspaceId: input.workspaceId,
|
|
1742
|
-
model: turn.model,
|
|
1743
|
-
active: codexSubscriptionActive,
|
|
1744
|
-
});
|
|
1745
|
-
// §7.5 P3 — pass BOTH the codex predicate (codex-plan turns bypass the gate)
|
|
2553
|
+
// §7.5 P3 — pass the accepted billing attribution (externally funded turns
|
|
2554
|
+
// bypass OpenGeni credit/token gates)
|
|
1746
2555
|
// AND the optional host `entitlements` port (when bound, its admitRun replaces
|
|
1747
2556
|
// the local credit read). Unset port → today's local-ledger path.
|
|
1748
|
-
await
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
2557
|
+
await waitForTurnOperation(
|
|
2558
|
+
ensureRunAllowed(
|
|
2559
|
+
settings,
|
|
2560
|
+
db,
|
|
2561
|
+
input.accountId,
|
|
2562
|
+
input.workspaceId,
|
|
2563
|
+
isExternallyBilledTurn,
|
|
2564
|
+
entitlements,
|
|
2565
|
+
),
|
|
2566
|
+
cancellationSignal,
|
|
2567
|
+
undefined,
|
|
1755
2568
|
);
|
|
1756
2569
|
// Setup (variableSet load, MCP connects, sandbox restore) does not
|
|
1757
2570
|
// stream and so never observes cancellation on its own; these explicit
|
|
@@ -1763,6 +2576,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1763
2576
|
throw reason;
|
|
1764
2577
|
}
|
|
1765
2578
|
};
|
|
2579
|
+
const throwIfTurnCancelled = () => throwIfTurnOperationCancelled(cancellationSignal);
|
|
1766
2580
|
// ONE shared details object for every heartbeat this activity sends (each
|
|
1767
2581
|
// site spreads it + its own phase), so cross-site fields — the op-stream
|
|
1768
2582
|
// settled roster in particular — survive last-write-wins instead of being
|
|
@@ -1792,6 +2606,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1792
2606
|
// one exactly like production.
|
|
1793
2607
|
codexLeaseHolderId = dispatchId;
|
|
1794
2608
|
const modelUsageDispatchId = activityContext?.info.activityId ?? dispatchId;
|
|
2609
|
+
const claimedModelUsageSourceKeys = new Set<string>();
|
|
1795
2610
|
const emittedModelUsageSourceKeys = new Set<string>();
|
|
1796
2611
|
publish = async (
|
|
1797
2612
|
events: Array<Omit<AppendEventInput, "producerId" | "producerSeq" | "turnId">>,
|
|
@@ -1858,6 +2673,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1858
2673
|
producerSeq: ++producerSeq,
|
|
1859
2674
|
};
|
|
1860
2675
|
}
|
|
2676
|
+
const compactionRequestFailure = inputSettlement.consumeRequestedCompactionFailure
|
|
2677
|
+
? {
|
|
2678
|
+
reason: "summarization_failed" as const,
|
|
2679
|
+
producerId,
|
|
2680
|
+
producerSeq: ++producerSeq,
|
|
2681
|
+
}
|
|
2682
|
+
: undefined;
|
|
1861
2683
|
const inputs = inputSettlement.events.map((event) => ({
|
|
1862
2684
|
...event,
|
|
1863
2685
|
payload: redact(event.payload),
|
|
@@ -1874,9 +2696,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1874
2696
|
sessionStatus: inputSettlement.sessionStatus,
|
|
1875
2697
|
activeTurnId: inputSettlement.activeTurnId,
|
|
1876
2698
|
events: inputs,
|
|
2699
|
+
...(inputSettlement.runState ? { runState: inputSettlement.runState } : {}),
|
|
1877
2700
|
...(recordingMutation ? { recording: recordingMutation } : {}),
|
|
2701
|
+
...(compactionRequestFailure ? { compactionRequestFailure } : {}),
|
|
1878
2702
|
});
|
|
1879
2703
|
if (result.action === "stale") {
|
|
2704
|
+
// The terminal write can lose to a control transaction before the
|
|
2705
|
+
// workflow delivers Temporal cancellation. That control may settle
|
|
2706
|
+
// the already-closed attempt as rejected_stale, so returning without
|
|
2707
|
+
// this flag would strand its replacement behind quiesced_at forever.
|
|
2708
|
+
// Enter the same hard tool-fence/receipt path as an explicit
|
|
2709
|
+
// TurnAttemptFencedError. If ownership was lost for an unrelated
|
|
2710
|
+
// reason, allowUninterrupted makes the receipt transaction a no-op.
|
|
2711
|
+
acknowledgeLostAttemptOwnership();
|
|
1880
2712
|
if (recordingForSettlement) {
|
|
1881
2713
|
await abandonActiveRecording(
|
|
1882
2714
|
"recording settlement lost attempt ownership",
|
|
@@ -1917,6 +2749,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1917
2749
|
// visibly starts: nothing ran yet, so the same inference starts cleanly
|
|
1918
2750
|
// on a healthy worker.
|
|
1919
2751
|
throwIfWorkerShuttingDown();
|
|
2752
|
+
throwIfTurnCancelled();
|
|
1920
2753
|
if (
|
|
1921
2754
|
!(await settle({
|
|
1922
2755
|
events: [
|
|
@@ -1953,7 +2786,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1953
2786
|
sessionPinSource,
|
|
1954
2787
|
sessionLastCredentialId: sessionCodex?.lastCredentialId ?? null,
|
|
1955
2788
|
continuationCredentialId: continuationCodexCredentialId,
|
|
1956
|
-
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
1957
2789
|
now: new Date(),
|
|
1958
2790
|
});
|
|
1959
2791
|
|
|
@@ -2075,7 +2907,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2075
2907
|
strategy: leased.rotationStrategy as CodexRotationStrategy,
|
|
2076
2908
|
rotationEnabled: leased.rotationEnabled,
|
|
2077
2909
|
});
|
|
2078
|
-
//
|
|
2910
|
+
// pin policy pin persistence follows the atomic credential allocator selection. The selector
|
|
2079
2911
|
// already ran exact-turn reuse before policy filtering and vetoed pointer
|
|
2080
2912
|
// movement for manual/policy homes; this write only records the NEXT turn's
|
|
2081
2913
|
// policy home (or clears a policy pin whose strategy is no longer active).
|
|
@@ -2163,7 +2995,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2163
2995
|
if (codexLeaseHeld) startCodexLeaseHeartbeat();
|
|
2164
2996
|
|
|
2165
2997
|
const eligibleCount = leased.accounts.filter((account) =>
|
|
2166
|
-
isCodexCredentialEligible(account,
|
|
2998
|
+
isCodexCredentialEligible(account, new Date()),
|
|
2167
2999
|
).length;
|
|
2168
3000
|
const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
|
|
2169
3001
|
observability.incrementCounter({
|
|
@@ -2340,7 +3172,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2340
3172
|
if (goalActive && goal) {
|
|
2341
3173
|
const authoritativeResetAt = authoritativeCodexCapacityResetAt(
|
|
2342
3174
|
leased.accounts,
|
|
2343
|
-
settings.codexRotationNearExhaustionPct,
|
|
2344
3175
|
new Date(),
|
|
2345
3176
|
);
|
|
2346
3177
|
const armed = await armCodexCapacityWait(db, {
|
|
@@ -2482,7 +3313,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2482
3313
|
// Resolved after turn.started so a composition conflict (two enabled
|
|
2483
3314
|
// packs declaring images) fails the turn with its plain error instead
|
|
2484
3315
|
// of failing the activity opaquely.
|
|
2485
|
-
const packRuntime = await
|
|
3316
|
+
const [packRuntime, skillLibraryRuntime] = await Promise.all([
|
|
3317
|
+
resolveWorkspacePackRuntime(db, input.workspaceId),
|
|
3318
|
+
resolveWorkspaceSkillLibraryRuntime(db, input.workspaceId),
|
|
3319
|
+
]);
|
|
2486
3320
|
// RIG BINDING (M3): load the session's FROZEN rig version (resolved+frozen
|
|
2487
3321
|
// at create). Everything rig-derived below (image precedence, env default
|
|
2488
3322
|
// sets, setup hook, credential hooks, doctrine, lease/telemetry stamps) is
|
|
@@ -2530,6 +3364,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2530
3364
|
db,
|
|
2531
3365
|
input.workspaceId,
|
|
2532
3366
|
input.sessionId,
|
|
3367
|
+
input.attemptId,
|
|
2533
3368
|
baseRunSettings,
|
|
2534
3369
|
);
|
|
2535
3370
|
// Multi-provider per-turn routing → the provider gating (compaction mode,
|
|
@@ -2545,14 +3380,17 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2545
3380
|
// a chat-only Fireworks model. Resolving against the default-model settings
|
|
2546
3381
|
// keeps gating consistent with the router. Cost accounting covers registry
|
|
2547
3382
|
// models via configuredModelPricing.
|
|
2548
|
-
const resolvedModel = runtime.resolveTurnModel(
|
|
3383
|
+
const resolvedModel = runtime.resolveTurnModel(
|
|
3384
|
+
capabilitySettings,
|
|
3385
|
+
turnExecutionPolicy.productModelId,
|
|
3386
|
+
);
|
|
2549
3387
|
// Bind the provider/model catalog's context policy to every model-facing
|
|
2550
3388
|
// path for this turn. In particular, Codex subscription turns must not
|
|
2551
3389
|
// inherit the deployment's OpenAI/Azure mode or 1.05M context defaults:
|
|
2552
3390
|
// raw window, effective ceiling, and auto-compact limit are distinct live
|
|
2553
3391
|
// catalog values and must reach pre-turn compaction, history guards, and
|
|
2554
3392
|
// every model call together.
|
|
2555
|
-
|
|
3393
|
+
modelRunSettings = resolvedModel
|
|
2556
3394
|
? settingsWithResolvedModelContext(runSettings, resolvedModel.configured)
|
|
2557
3395
|
: runSettings;
|
|
2558
3396
|
// WORKSPACE MODEL POLICY — the authoritative hard gate. Runs immediately
|
|
@@ -2560,25 +3398,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2560
3398
|
// and the main run both come later in this scope), so a blocked
|
|
2561
3399
|
// provider/model can never be reached through ANY stamp path: explicit
|
|
2562
3400
|
// turn model, inherited session default, goal-continuation inheritance,
|
|
2563
|
-
// or the legacy null-resolution fallback
|
|
2564
|
-
//
|
|
2565
|
-
//
|
|
2566
|
-
// how bare-model turns silently spent real Azure money in a
|
|
2567
|
-
// codex-intended workspace). Fail-loud, never a silent remap.
|
|
3401
|
+
// or the legacy null-resolution fallback. The frozen execution policy is
|
|
3402
|
+
// the attribution source even if an injected test runtime returns no
|
|
3403
|
+
// concrete resolved model. Fail-loud, never a silent remap.
|
|
2568
3404
|
{
|
|
2569
3405
|
const workspaceModelPolicy = await getWorkspaceModelPolicy(db, input.workspaceId);
|
|
2570
3406
|
if (workspaceModelPolicy) {
|
|
2571
|
-
const effectiveProviderId = resolvedModel
|
|
2572
|
-
? resolvedModel.provider.id
|
|
2573
|
-
: builtinProviderId(capabilitySettings);
|
|
2574
3407
|
const verdict = evaluateWorkspaceModelPolicy(workspaceModelPolicy, {
|
|
2575
|
-
providerId:
|
|
2576
|
-
modelId:
|
|
3408
|
+
providerId: turnExecutionPolicy.providerId,
|
|
3409
|
+
modelId: turnExecutionPolicy.productModelId,
|
|
2577
3410
|
});
|
|
2578
3411
|
if (!verdict.allowed) {
|
|
2579
3412
|
throw new WorkspaceModelPolicyBlockedError(
|
|
2580
|
-
|
|
2581
|
-
|
|
3413
|
+
turnExecutionPolicy.productModelId,
|
|
3414
|
+
turnExecutionPolicy.providerId,
|
|
2582
3415
|
verdict.reason,
|
|
2583
3416
|
);
|
|
2584
3417
|
}
|
|
@@ -2592,6 +3425,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2592
3425
|
// Build it once and wrap BOTH the compaction summarizer (a separate model
|
|
2593
3426
|
// call on the same codex client) and the main run; otherwise the summarizer
|
|
2594
3427
|
// would hit the codex backend unauthenticated.
|
|
3428
|
+
let codexModelRequestSequence = 0;
|
|
2595
3429
|
const codexContext: CodexRequestContext | null =
|
|
2596
3430
|
resolvedModel?.provider.kind === "codex-subscription"
|
|
2597
3431
|
? ((): CodexRequestContext => {
|
|
@@ -2622,6 +3456,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2622
3456
|
onUsageHeaders: (snapshot) => {
|
|
2623
3457
|
latestCodexUsage = snapshot;
|
|
2624
3458
|
}, // latest wins; flushed once in finally
|
|
3459
|
+
nextRequestId: () => `${dispatchId}:${++codexModelRequestSequence}`,
|
|
3460
|
+
onModelRequestEvent: async (event) => {
|
|
3461
|
+
if (!publish || !turnId) {
|
|
3462
|
+
throw new Error("Codex model request started before the turn event producer");
|
|
3463
|
+
}
|
|
3464
|
+
await publish([
|
|
3465
|
+
{
|
|
3466
|
+
type: "agent.model.request",
|
|
3467
|
+
payload: {
|
|
3468
|
+
...event,
|
|
3469
|
+
provider: "codex-subscription",
|
|
3470
|
+
turnId,
|
|
3471
|
+
attemptId: input.attemptId,
|
|
3472
|
+
dispatchId,
|
|
3473
|
+
executionGeneration,
|
|
3474
|
+
},
|
|
3475
|
+
},
|
|
3476
|
+
]);
|
|
3477
|
+
},
|
|
2625
3478
|
};
|
|
2626
3479
|
})()
|
|
2627
3480
|
: null;
|
|
@@ -2630,52 +3483,33 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2630
3483
|
const promptCacheKey = acceptsPromptCacheKeyForTurn(resolvedModel)
|
|
2631
3484
|
? input.sessionId
|
|
2632
3485
|
: undefined;
|
|
2633
|
-
|
|
3486
|
+
const compactionUsageState = createCompactionModelUsageEventState(
|
|
3487
|
+
claimedModelUsageSourceKeys,
|
|
3488
|
+
);
|
|
2634
3489
|
const recordCompactionUsage = async (usage: ModelResponseUsage) => {
|
|
2635
|
-
await
|
|
3490
|
+
await processCompactionModelUsageEvent({
|
|
3491
|
+
usage,
|
|
3492
|
+
state: compactionUsageState,
|
|
3493
|
+
dispatchId: modelUsageDispatchId,
|
|
3494
|
+
settings,
|
|
3495
|
+
db,
|
|
3496
|
+
observability,
|
|
3497
|
+
publish,
|
|
3498
|
+
accountId: input.accountId,
|
|
3499
|
+
workspaceId: input.workspaceId,
|
|
3500
|
+
sessionId: input.sessionId,
|
|
3501
|
+
turnId: turn.id,
|
|
3502
|
+
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
3503
|
+
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
3504
|
+
model: resolvedModel?.configured.id ?? turn.model,
|
|
3505
|
+
externallyBilled: isExternallyBilledTurn,
|
|
3506
|
+
turnAttemptId: input.attemptId,
|
|
3507
|
+
servingCredentialId: effectiveCodexCredentialId,
|
|
3508
|
+
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
3509
|
+
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
2636
3510
|
renewLease: () => renewCodexLease("model_usage"),
|
|
2637
3511
|
leaseLost: () => codexLeaseLost,
|
|
2638
3512
|
leaseLostMessage: "Codex credential lease expired during context compaction",
|
|
2639
|
-
recordUsage: async () => {
|
|
2640
|
-
compactionUsageCount += 1;
|
|
2641
|
-
const sourceKey = modelUsageSourceKey({
|
|
2642
|
-
responseId: usage.responseId,
|
|
2643
|
-
dispatchId: modelUsageDispatchId,
|
|
2644
|
-
positionalKey: `compaction-${compactionUsageCount}`,
|
|
2645
|
-
});
|
|
2646
|
-
const responseAccountCtx = modelCallAccountContext({
|
|
2647
|
-
servingCredentialId: effectiveCodexCredentialId,
|
|
2648
|
-
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
2649
|
-
isFirstCallOfTurn: compactionUsageCount === 1,
|
|
2650
|
-
});
|
|
2651
|
-
await recordModelUsageAndDebitCredits(settings, db, {
|
|
2652
|
-
accountId: input.accountId,
|
|
2653
|
-
workspaceId: input.workspaceId,
|
|
2654
|
-
sessionId: input.sessionId,
|
|
2655
|
-
turnId: turn.id,
|
|
2656
|
-
model: resolvedModel?.configured.id ?? turn.model,
|
|
2657
|
-
isCodexTurn,
|
|
2658
|
-
usage: usage.usage,
|
|
2659
|
-
sourceKey,
|
|
2660
|
-
observability,
|
|
2661
|
-
});
|
|
2662
|
-
await emitModelCallUsage({
|
|
2663
|
-
observability,
|
|
2664
|
-
publish,
|
|
2665
|
-
accountId: input.accountId,
|
|
2666
|
-
workspaceId: input.workspaceId,
|
|
2667
|
-
sessionId: input.sessionId,
|
|
2668
|
-
turnId: turn.id,
|
|
2669
|
-
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
2670
|
-
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
2671
|
-
model: resolvedModel?.configured.id ?? turn.model,
|
|
2672
|
-
sourceKey,
|
|
2673
|
-
usage,
|
|
2674
|
-
servingAccountHash: responseAccountCtx.servingAccountHash,
|
|
2675
|
-
accountChangedFromPrevCall: responseAccountCtx.accountChangedFromPrevCall,
|
|
2676
|
-
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
2677
|
-
});
|
|
2678
|
-
},
|
|
2679
3513
|
});
|
|
2680
3514
|
};
|
|
2681
3515
|
const compactionSummarizerFor = (systemInstructions?: string) =>
|
|
@@ -2685,7 +3519,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2685
3519
|
summarizeForCompaction(s, m, {
|
|
2686
3520
|
client: resolvedModel.client,
|
|
2687
3521
|
api: resolvedModel.provider.api,
|
|
2688
|
-
model:
|
|
3522
|
+
model: turnExecutionPolicy.upstreamModelId,
|
|
2689
3523
|
maxOutputTokens: SUMMARY_BUFFER_TOKENS,
|
|
2690
3524
|
onUsage: recordCompactionUsage,
|
|
2691
3525
|
...(systemInstructions ? { systemInstructions } : {}),
|
|
@@ -2694,6 +3528,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2694
3528
|
)
|
|
2695
3529
|
: (s: Settings, m: Array<Record<string, unknown>>) =>
|
|
2696
3530
|
summarizeForCompaction(s, m, {
|
|
3531
|
+
model: turnExecutionPolicy.upstreamModelId,
|
|
2697
3532
|
maxOutputTokens: SUMMARY_BUFFER_TOKENS,
|
|
2698
3533
|
onUsage: recordCompactionUsage,
|
|
2699
3534
|
...(systemInstructions ? { systemInstructions } : {}),
|
|
@@ -2725,25 +3560,69 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2725
3560
|
);
|
|
2726
3561
|
let outcome: Awaited<ReturnType<typeof maybeCompactContext>> | null = null;
|
|
2727
3562
|
if (requested) {
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
3563
|
+
try {
|
|
3564
|
+
outcome = await waitForTurnOperation(
|
|
3565
|
+
maybeCompactContext(
|
|
3566
|
+
db,
|
|
3567
|
+
modelRunSettings,
|
|
3568
|
+
{
|
|
3569
|
+
accountId: input.accountId,
|
|
3570
|
+
workspaceId: input.workspaceId,
|
|
3571
|
+
sessionId: input.sessionId,
|
|
3572
|
+
turnId: turn.id,
|
|
3573
|
+
executionGeneration,
|
|
3574
|
+
attemptId: input.attemptId,
|
|
3575
|
+
},
|
|
3576
|
+
session.lastInputTokens,
|
|
3577
|
+
compactionSummarizerFor(compactionInstructions),
|
|
3578
|
+
{
|
|
3579
|
+
force: true,
|
|
3580
|
+
clearRequestedCompaction: true,
|
|
3581
|
+
trigger: "operator",
|
|
3582
|
+
},
|
|
3583
|
+
),
|
|
3584
|
+
cancellationSignal,
|
|
3585
|
+
undefined,
|
|
3586
|
+
);
|
|
3587
|
+
} catch (error) {
|
|
3588
|
+
// Codex retries retryable checkpoint-provider failures rather than
|
|
3589
|
+
// treating them as a semantic compaction result. Keep the operator
|
|
3590
|
+
// request pending and let the ordinary same-turn provider/capacity
|
|
3591
|
+
// recovery path re-dispatch this exact maintenance execution.
|
|
3592
|
+
if (shouldRecoverCompactionProviderFailure(error)) throw error;
|
|
3593
|
+
if (!isCompactionSummaryFailure(error)) throw error;
|
|
3594
|
+
const errorMessage = compactionFailureReasonFromError(error);
|
|
3595
|
+
if (
|
|
3596
|
+
!(await settle!({
|
|
3597
|
+
events: [
|
|
3598
|
+
{
|
|
3599
|
+
type: "turn.failed",
|
|
3600
|
+
payload: {
|
|
3601
|
+
error: errorMessage,
|
|
3602
|
+
code: "context_compaction_failed",
|
|
3603
|
+
retryable: false,
|
|
3604
|
+
recovery: "user_message",
|
|
3605
|
+
compacted: false,
|
|
3606
|
+
},
|
|
3607
|
+
},
|
|
3608
|
+
{
|
|
3609
|
+
type: "session.status.changed",
|
|
3610
|
+
payload: { status: "idle" },
|
|
3611
|
+
},
|
|
3612
|
+
],
|
|
3613
|
+
turnStatus: "failed",
|
|
3614
|
+
sessionStatus: "idle",
|
|
3615
|
+
activeTurnId: null,
|
|
3616
|
+
consumeRequestedCompactionFailure: true,
|
|
3617
|
+
}))
|
|
3618
|
+
) {
|
|
3619
|
+
return claimedResult({ status: "cancelled" });
|
|
3620
|
+
}
|
|
3621
|
+
turnMetricOutcome = "failed";
|
|
3622
|
+
activityStatus = "idle";
|
|
3623
|
+
activityError = error;
|
|
3624
|
+
return claimedResult({ status: "idle" });
|
|
3625
|
+
}
|
|
2747
3626
|
if (outcome.events.length > 0) {
|
|
2748
3627
|
if (outcome.compacted) {
|
|
2749
3628
|
recordContextCompaction(observability, "operator");
|
|
@@ -2790,22 +3669,42 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2790
3669
|
// connector scopes); no-op for every other turn. Its refreshing bearer is
|
|
2791
3670
|
// resolved at connect time from the codex ALS (see the withCodex-wrapped
|
|
2792
3671
|
// prepareTools call below).
|
|
3672
|
+
// Resolve the durable policy at the turn boundary. Workspace-default
|
|
3673
|
+
// sessions may discover newly enabled capability MCPs, while explicit,
|
|
3674
|
+
// inherited-fixed, and legacy sessions remain narrowed to their stored
|
|
3675
|
+
// materialized allow-list. `withCodexAppsTool` below keeps the existing
|
|
3676
|
+
// single Codex lazy router/connector overlay; it is not a second policy
|
|
3677
|
+
// or discovery path.
|
|
3678
|
+
const effectivePolicyTools = resolveSessionToolPolicy({
|
|
3679
|
+
...(session.toolPolicy ? { toolPolicy: session.toolPolicy } : {}),
|
|
3680
|
+
sessionTools: session.tools,
|
|
3681
|
+
turnTools: turn.tools,
|
|
3682
|
+
...(turn.toolsProvided !== undefined ? { turnToolsProvided: turn.toolsProvided } : {}),
|
|
3683
|
+
availableMcpServerIds: runSettings.mcpServers.map((server) => server.id),
|
|
3684
|
+
defaultMcpServerIds: enabledCapabilityMcpToolRefs(settings, mcpSettings).map(
|
|
3685
|
+
(tool) => tool.id,
|
|
3686
|
+
),
|
|
3687
|
+
}).toolRefs;
|
|
2793
3688
|
const turnTools = withCodexAppsTool(
|
|
2794
3689
|
runSettings,
|
|
2795
|
-
withFirstPartyTools(runSettings,
|
|
3690
|
+
withFirstPartyTools(runSettings, effectivePolicyTools),
|
|
2796
3691
|
);
|
|
2797
|
-
// §7.6
|
|
3692
|
+
// §7.6 connection-credential provider — load (and decrypt) the variable set via the host
|
|
2798
3693
|
// `sandboxSecrets` provider when bound; unset → today's local decrypt.
|
|
2799
3694
|
const connectionScope = {
|
|
2800
3695
|
accountId: input.accountId,
|
|
2801
3696
|
workspaceId: input.workspaceId,
|
|
2802
3697
|
};
|
|
2803
|
-
const workspaceVariableSet = await
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
|
|
3698
|
+
const workspaceVariableSet = await waitForTurnOperation(
|
|
3699
|
+
loadWorkspaceEnvironmentForRunWithCredentials(
|
|
3700
|
+
db,
|
|
3701
|
+
runSettings,
|
|
3702
|
+
connectionScope,
|
|
3703
|
+
session.variableSetId,
|
|
3704
|
+
connectionCredentials?.sandboxSecrets,
|
|
3705
|
+
),
|
|
3706
|
+
cancellationSignal,
|
|
3707
|
+
undefined,
|
|
2809
3708
|
);
|
|
2810
3709
|
variableSetId = workspaceVariableSet?.id ?? "";
|
|
2811
3710
|
// RIG DEFAULT VARIABLE SETS (M3): decrypt the frozen rig version's default
|
|
@@ -2820,12 +3719,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2820
3719
|
// set already relies on), keeping validateNoEnvironmentDelta empty.
|
|
2821
3720
|
const rigDefaultEnvironmentValues: Record<string, string> = {};
|
|
2822
3721
|
for (const rigDefaultVariableSetId of rigVersion?.defaultVariableSetIds ?? []) {
|
|
2823
|
-
const rigDefaultSet = await
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
|
|
2827
|
-
|
|
2828
|
-
|
|
3722
|
+
const rigDefaultSet = await waitForTurnOperation(
|
|
3723
|
+
loadWorkspaceEnvironmentForRunWithCredentials(
|
|
3724
|
+
db,
|
|
3725
|
+
runSettings,
|
|
3726
|
+
connectionScope,
|
|
3727
|
+
rigDefaultVariableSetId,
|
|
3728
|
+
connectionCredentials?.sandboxSecrets,
|
|
3729
|
+
),
|
|
3730
|
+
cancellationSignal,
|
|
3731
|
+
undefined,
|
|
2829
3732
|
);
|
|
2830
3733
|
Object.assign(rigDefaultEnvironmentValues, rigDefaultSet?.values ?? {});
|
|
2831
3734
|
}
|
|
@@ -2836,7 +3739,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2836
3739
|
);
|
|
2837
3740
|
// Redact EVERY exported secret value (rig defaults + session set) from turn
|
|
2838
3741
|
// output, not just the session set's.
|
|
2839
|
-
|
|
3742
|
+
registerSecretRedactions(
|
|
2840
3743
|
Object.entries(sandboxWorkspaceEnvironmentValues).map(([name, value]) => ({ name, value })),
|
|
2841
3744
|
);
|
|
2842
3745
|
// EFFECTIVE compute backend, resolved ONCE at turn start (Case B + Stage D
|
|
@@ -2909,7 +3812,56 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2909
3812
|
settings.sandboxOwnershipEnabled && runSettings.sandboxBackend !== "none"
|
|
2910
3813
|
? groupBoxBackend
|
|
2911
3814
|
: runSettings.sandboxBackend;
|
|
2912
|
-
|
|
3815
|
+
const effectiveRunCredentialBackend = activeSandboxBackend ?? groupBoxBackend;
|
|
3816
|
+
const runCredentialResolver =
|
|
3817
|
+
effectiveRunCredentialBackend === "none"
|
|
3818
|
+
? null
|
|
3819
|
+
: await waitForTurnOperation(
|
|
3820
|
+
bindRunCredentialResolver({
|
|
3821
|
+
db,
|
|
3822
|
+
connectionCredentials: connectionCredentials ?? null,
|
|
3823
|
+
accountId: input.accountId,
|
|
3824
|
+
workspaceId: input.workspaceId,
|
|
3825
|
+
session,
|
|
3826
|
+
turn,
|
|
3827
|
+
attemptId: input.attemptId,
|
|
3828
|
+
effectiveSandboxBackend: effectiveRunCredentialBackend,
|
|
3829
|
+
variableSet: workspaceVariableSet
|
|
3830
|
+
? {
|
|
3831
|
+
id: workspaceVariableSet.id,
|
|
3832
|
+
name: workspaceVariableSet.name,
|
|
3833
|
+
}
|
|
3834
|
+
: null,
|
|
3835
|
+
}),
|
|
3836
|
+
cancellationSignal,
|
|
3837
|
+
undefined,
|
|
3838
|
+
);
|
|
3839
|
+
const initialRunCredentialMaterial = runCredentialResolver
|
|
3840
|
+
? await waitForTurnOperation(
|
|
3841
|
+
runCredentialResolver.resolve({
|
|
3842
|
+
purpose: "provision",
|
|
3843
|
+
forceRefresh: false,
|
|
3844
|
+
}),
|
|
3845
|
+
cancellationSignal,
|
|
3846
|
+
undefined,
|
|
3847
|
+
)
|
|
3848
|
+
: null;
|
|
3849
|
+
if (initialRunCredentialMaterial) {
|
|
3850
|
+
registerSecretRedactions(initialRunCredentialMaterial.redactions);
|
|
3851
|
+
for (const payload of runCredentialAuthNeededPayloads(initialRunCredentialMaterial)) {
|
|
3852
|
+
publishedRunCredentialNotices.add(JSON.stringify(payload));
|
|
3853
|
+
await publish!([{ type: "credential.auth_needed", payload }], true);
|
|
3854
|
+
}
|
|
3855
|
+
}
|
|
3856
|
+
const runCredentialsNote = initialRunCredentialMaterial
|
|
3857
|
+
? runCredentialModelNote(initialRunCredentialMaterial)
|
|
3858
|
+
: undefined;
|
|
3859
|
+
throwIfTurnOperationCancelled(cancellationSignal);
|
|
3860
|
+
await waitForTurnOperation(
|
|
3861
|
+
ensureTurnModalRegistryImage(runSettings, sandboxCreationBackend),
|
|
3862
|
+
cancellationSignal,
|
|
3863
|
+
undefined,
|
|
3864
|
+
);
|
|
2913
3865
|
const establishPolicy: "eager" | "on-demand" =
|
|
2914
3866
|
lazyProvisionEnabled(settings) && !machinePrimary && runSettings.sandboxBackend !== "none"
|
|
2915
3867
|
? "on-demand"
|
|
@@ -2933,134 +3885,396 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2933
3885
|
// provider may supply it; unset still self-mints GitHub from settings.
|
|
2934
3886
|
// gitToken/gitTokens are undefined on the selfhosted skip path (the machine
|
|
2935
3887
|
// uses its own git creds).
|
|
3888
|
+
if (activeSandboxBackend !== "selfhosted") {
|
|
3889
|
+
await assertGitHubResourcesRemainAuthorized(db, input.workspaceId, turnResources);
|
|
3890
|
+
}
|
|
3891
|
+
const authorizeGitHubTokenMint: GitHubTokenMintAuthorization = async (selection) => {
|
|
3892
|
+
await assertGitHubTokenMintSelectionAuthorized(
|
|
3893
|
+
db,
|
|
3894
|
+
input.workspaceId,
|
|
3895
|
+
selection.installationId,
|
|
3896
|
+
selection.repositoryIds,
|
|
3897
|
+
);
|
|
3898
|
+
};
|
|
3899
|
+
// Git and MCP credentials share one lineage snapshot for this turn. A
|
|
3900
|
+
// host that supplies both ports must never see two independently resolved
|
|
3901
|
+
// roots for the same execution merely because the call sites are far apart.
|
|
3902
|
+
const needsHostCredentialRoot = Boolean(
|
|
3903
|
+
connectionCredentials?.gitCredentials || connectionCredentials?.mcpCredentials,
|
|
3904
|
+
);
|
|
3905
|
+
const hostCredentialRootSessionId = needsHostCredentialRoot
|
|
3906
|
+
? await getSessionRootId(db, input.workspaceId, input.sessionId)
|
|
3907
|
+
: null;
|
|
3908
|
+
if (needsHostCredentialRoot && !hostCredentialRootSessionId) {
|
|
3909
|
+
throw new Error(`cannot resolve host credentials for missing session ${input.sessionId}`);
|
|
3910
|
+
}
|
|
3911
|
+
const gitCredentialAuthority =
|
|
3912
|
+
connectionCredentials?.gitCredentials && hostCredentialRootSessionId
|
|
3913
|
+
? gitCredentialAuthorityForTurn({
|
|
3914
|
+
sessionId: input.sessionId,
|
|
3915
|
+
rootSessionId: hostCredentialRootSessionId,
|
|
3916
|
+
attemptId: input.attemptId,
|
|
3917
|
+
turn,
|
|
3918
|
+
})
|
|
3919
|
+
: undefined;
|
|
3920
|
+
const toolspaceAuthority = {
|
|
3921
|
+
sessionId: input.sessionId,
|
|
3922
|
+
};
|
|
2936
3923
|
const {
|
|
2937
3924
|
environment: sandboxEnvironment,
|
|
2938
3925
|
gitToken: sandboxGitToken,
|
|
2939
3926
|
gitTokens: sandboxGitTokens,
|
|
2940
3927
|
gitTokenExpiresAt: sandboxGitTokenExpiresAt,
|
|
3928
|
+
gitCredentialBindings: sandboxGitCredentialBindings,
|
|
2941
3929
|
toolspaceToken: sandboxToolspaceToken,
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
|
|
2948
|
-
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
activeSandboxBackend
|
|
2952
|
-
|
|
2953
|
-
|
|
2954
|
-
|
|
2955
|
-
|
|
2956
|
-
|
|
3930
|
+
toolspaceTokenExpiresAt: sandboxToolspaceTokenExpiresAt,
|
|
3931
|
+
} = await waitForTurnOperation(
|
|
3932
|
+
sandboxEnvironmentForRun(
|
|
3933
|
+
runSettings,
|
|
3934
|
+
turnResources,
|
|
3935
|
+
// Rig default sets merged BELOW the session set (session wins); rig-less
|
|
3936
|
+
// turns pass exactly workspaceVariableSet?.values (byte-for-byte today).
|
|
3937
|
+
sandboxWorkspaceEnvironmentValues,
|
|
3938
|
+
{
|
|
3939
|
+
skipGitHubToken: activeSandboxBackend === "selfhosted",
|
|
3940
|
+
deferGitHubToken:
|
|
3941
|
+
activeSandboxBackend !== "selfhosted" && establishPolicy === "on-demand",
|
|
3942
|
+
scope: connectionScope,
|
|
3943
|
+
...(gitCredentialAuthority ? { authority: gitCredentialAuthority } : {}),
|
|
3944
|
+
gitCredentials: connectionCredentials?.gitCredentials,
|
|
3945
|
+
authorizeGitHubTokenMint,
|
|
3946
|
+
toolspaceAuthority,
|
|
3947
|
+
},
|
|
3948
|
+
),
|
|
3949
|
+
cancellationSignal,
|
|
3950
|
+
undefined,
|
|
2957
3951
|
);
|
|
2958
|
-
|
|
2959
|
-
|
|
2960
|
-
? {
|
|
2961
|
-
gitTokens: sandboxGitTokens,
|
|
2962
|
-
expiresAt: sandboxGitTokenExpiresAt ?? {},
|
|
2963
|
-
}
|
|
3952
|
+
const sandboxToolspaceTokenFile = sandboxToolspaceToken
|
|
3953
|
+
? toolspaceTokenFileFromEnvironment(sandboxEnvironment, input.sessionId)
|
|
2964
3954
|
: undefined;
|
|
3955
|
+
|
|
3956
|
+
const initialGitCredentials: MintedRunGitCredentials | undefined =
|
|
3957
|
+
sandboxGitCredentialBindings
|
|
3958
|
+
? {
|
|
3959
|
+
bindings: sandboxGitCredentialBindings,
|
|
3960
|
+
gitTokens: sandboxGitTokens ?? {},
|
|
3961
|
+
expiresAt: sandboxGitTokenExpiresAt ?? {},
|
|
3962
|
+
}
|
|
3963
|
+
: undefined;
|
|
2965
3964
|
const attachGitCredentialRenewal = async (
|
|
2966
3965
|
tokenSession: GitCredentialTokenWriterSession,
|
|
2967
3966
|
initial: MintedRunGitCredentials | undefined,
|
|
2968
3967
|
): Promise<void> => {
|
|
2969
|
-
if (!initial ||
|
|
2970
|
-
const previous =
|
|
2971
|
-
|
|
2972
|
-
await previous
|
|
3968
|
+
if (!initial || initial.bindings.length === 0) return;
|
|
3969
|
+
const previous = gitCredentialRenewals;
|
|
3970
|
+
gitCredentialRenewals = [];
|
|
3971
|
+
await Promise.all(previous.map(async (controller) => await controller.stop()));
|
|
2973
3972
|
if (gitCredentialRenewalClosed) return;
|
|
2974
3973
|
|
|
2975
|
-
const
|
|
2976
|
-
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
3974
|
+
const controllers = initial.bindings.map((initialBinding) => {
|
|
3975
|
+
let pendingBinding: typeof initialBinding | undefined;
|
|
3976
|
+
return startGitCredentialRenewalLoop({
|
|
3977
|
+
expectedProviders: [initialBinding.provider],
|
|
3978
|
+
initialExpiresAt: initialBinding.expiresAt
|
|
3979
|
+
? { [initialBinding.provider]: initialBinding.expiresAt }
|
|
3980
|
+
: {},
|
|
3981
|
+
mint: async () => {
|
|
3982
|
+
const binding = await mintRunGitCredentialBinding(
|
|
3983
|
+
runSettings,
|
|
3984
|
+
turnResources,
|
|
3985
|
+
initialBinding.provider,
|
|
3986
|
+
initialBinding.credentialBindingId,
|
|
3987
|
+
{
|
|
3988
|
+
scope: connectionScope,
|
|
3989
|
+
...(gitCredentialAuthority ? { authority: gitCredentialAuthority } : {}),
|
|
3990
|
+
gitCredentials: connectionCredentials?.gitCredentials,
|
|
3991
|
+
authorizeGitHubTokenMint,
|
|
3992
|
+
},
|
|
3993
|
+
);
|
|
3994
|
+
if (binding) {
|
|
3995
|
+
assertGitCredentialRenewalTransportUnchanged(initialBinding, binding);
|
|
3996
|
+
}
|
|
3997
|
+
pendingBinding = binding;
|
|
3998
|
+
return binding
|
|
3999
|
+
? {
|
|
4000
|
+
bindings: [binding],
|
|
4001
|
+
gitTokens: { [binding.provider]: binding.token },
|
|
4002
|
+
expiresAt: binding.expiresAt ? { [binding.provider]: binding.expiresAt } : {},
|
|
4003
|
+
}
|
|
4004
|
+
: undefined;
|
|
4005
|
+
},
|
|
4006
|
+
write: async () => {
|
|
4007
|
+
if (!pendingBinding) {
|
|
4008
|
+
throw new Error("credential renewal produced no binding token");
|
|
4009
|
+
}
|
|
4010
|
+
const runAs = sandboxRunAs(runSettings);
|
|
4011
|
+
await refreshGitCredentialBindingTokenFiles(tokenSession, [pendingBinding], {
|
|
4012
|
+
...(runAs ? { runAs } : {}),
|
|
4013
|
+
...(toolCancellationFenceRef.current
|
|
4014
|
+
? {
|
|
4015
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
4016
|
+
toolCancellationFenceRef.current,
|
|
4017
|
+
),
|
|
4018
|
+
}
|
|
4019
|
+
: {}),
|
|
2996
4020
|
});
|
|
2997
|
-
}
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
4021
|
+
},
|
|
4022
|
+
onSuccess: ({ providers: renewedProviders }) => {
|
|
4023
|
+
for (const provider of renewedProviders) {
|
|
4024
|
+
observability.incrementCounter({
|
|
4025
|
+
name: "opengeni_git_credential_renewals_total",
|
|
4026
|
+
help: "Host-managed Git credential renewal attempts by provider and outcome.",
|
|
4027
|
+
labels: { provider, outcome: "completed" },
|
|
4028
|
+
});
|
|
4029
|
+
}
|
|
4030
|
+
},
|
|
4031
|
+
onFailure: ({ providers: failedProviders, retryDelayMs, errorClass }) => {
|
|
4032
|
+
for (const provider of failedProviders) {
|
|
4033
|
+
observability.incrementCounter({
|
|
4034
|
+
name: "opengeni_git_credential_renewals_total",
|
|
4035
|
+
help: "Host-managed Git credential renewal attempts by provider and outcome.",
|
|
4036
|
+
labels: { provider, outcome: "error" },
|
|
4037
|
+
});
|
|
4038
|
+
}
|
|
4039
|
+
observability.warn("Sandbox Git credential renewal failed; retry scheduled", {
|
|
4040
|
+
sessionId: input.sessionId,
|
|
4041
|
+
turnId,
|
|
4042
|
+
providers: failedProviders.join(","),
|
|
4043
|
+
errorClass,
|
|
4044
|
+
retryDelayMs,
|
|
3005
4045
|
});
|
|
3006
|
-
}
|
|
3007
|
-
|
|
4046
|
+
},
|
|
4047
|
+
});
|
|
4048
|
+
});
|
|
4049
|
+
if (gitCredentialRenewalClosed) {
|
|
4050
|
+
await Promise.all(controllers.map(async (controller) => await controller.stop()));
|
|
4051
|
+
return;
|
|
4052
|
+
}
|
|
4053
|
+
gitCredentialRenewals = controllers;
|
|
4054
|
+
};
|
|
4055
|
+
|
|
4056
|
+
const attachToolspaceTokenRenewal = async (
|
|
4057
|
+
tokenSession: ToolspaceTokenWriterSession,
|
|
4058
|
+
initialExpiresAt = sandboxToolspaceTokenExpiresAt,
|
|
4059
|
+
): Promise<void> => {
|
|
4060
|
+
if (!sandboxToolspaceToken || !initialExpiresAt) return;
|
|
4061
|
+
const previous = toolspaceTokenRenewal;
|
|
4062
|
+
toolspaceTokenRenewal = null;
|
|
4063
|
+
await previous?.stop();
|
|
4064
|
+
if (toolspaceTokenRenewalClosed) return;
|
|
4065
|
+
|
|
4066
|
+
const mint = async () =>
|
|
4067
|
+
await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority);
|
|
4068
|
+
const write = async (material: NonNullable<Awaited<ReturnType<typeof mint>>>) => {
|
|
4069
|
+
const runAs = sandboxRunAs(runSettings);
|
|
4070
|
+
await refreshToolspaceTokenFile(tokenSession, material.token, {
|
|
4071
|
+
...(runAs ? { runAs } : {}),
|
|
4072
|
+
...(sandboxToolspaceTokenFile
|
|
4073
|
+
? {
|
|
4074
|
+
tokenFile: sandboxToolspaceTokenFile,
|
|
4075
|
+
legacyTokenFile: sandboxEnvironment.OPENGENI_TOOLSPACE_TOKEN_FILE!,
|
|
4076
|
+
}
|
|
4077
|
+
: {}),
|
|
4078
|
+
...(toolCancellationFenceRef.current
|
|
4079
|
+
? {
|
|
4080
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
4081
|
+
toolCancellationFenceRef.current,
|
|
4082
|
+
),
|
|
4083
|
+
}
|
|
4084
|
+
: {}),
|
|
4085
|
+
});
|
|
4086
|
+
};
|
|
4087
|
+
let renewalExpiresAt = initialExpiresAt;
|
|
4088
|
+
if (renewalExpiresAt.getTime() <= Date.now() + TOOLSPACE_TOKEN_EXPIRY_LEAD_MS) {
|
|
4089
|
+
const fresh = await mint();
|
|
4090
|
+
if (!fresh) {
|
|
4091
|
+
throw new Error("Toolspace token mint became unavailable during sandbox setup");
|
|
4092
|
+
}
|
|
4093
|
+
await write(fresh);
|
|
4094
|
+
renewalExpiresAt = fresh.expiresAt;
|
|
4095
|
+
}
|
|
4096
|
+
const controller = startToolspaceTokenRenewalLoop({
|
|
4097
|
+
initialExpiresAt: renewalExpiresAt,
|
|
4098
|
+
mint,
|
|
4099
|
+
write,
|
|
4100
|
+
onSuccess: () => {
|
|
4101
|
+
observability.incrementCounter({
|
|
4102
|
+
name: "opengeni_toolspace_token_renewals_total",
|
|
4103
|
+
help: "Sandbox Toolspace token renewal attempts by outcome.",
|
|
4104
|
+
labels: { outcome: "completed" },
|
|
4105
|
+
});
|
|
4106
|
+
},
|
|
4107
|
+
onFailure: ({ retryDelayMs, errorClass }) => {
|
|
4108
|
+
observability.incrementCounter({
|
|
4109
|
+
name: "opengeni_toolspace_token_renewals_total",
|
|
4110
|
+
help: "Sandbox Toolspace token renewal attempts by outcome.",
|
|
4111
|
+
labels: { outcome: "error" },
|
|
4112
|
+
});
|
|
4113
|
+
observability.warn("Sandbox Toolspace token renewal failed; retry scheduled", {
|
|
3008
4114
|
sessionId: input.sessionId,
|
|
3009
4115
|
turnId,
|
|
3010
|
-
providers: failedProviders.join(","),
|
|
3011
4116
|
errorClass,
|
|
3012
4117
|
retryDelayMs,
|
|
3013
4118
|
});
|
|
3014
4119
|
},
|
|
3015
4120
|
});
|
|
3016
|
-
if (
|
|
4121
|
+
if (toolspaceTokenRenewalClosed) {
|
|
3017
4122
|
await controller.stop();
|
|
3018
4123
|
return;
|
|
3019
4124
|
}
|
|
3020
|
-
|
|
4125
|
+
toolspaceTokenRenewal = controller;
|
|
3021
4126
|
};
|
|
3022
4127
|
|
|
3023
|
-
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
|
|
3039
|
-
|
|
3040
|
-
|
|
3041
|
-
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
|
|
3052
|
-
|
|
3053
|
-
|
|
3054
|
-
|
|
3055
|
-
|
|
3056
|
-
|
|
3057
|
-
|
|
3058
|
-
|
|
3059
|
-
|
|
3060
|
-
|
|
3061
|
-
|
|
3062
|
-
|
|
3063
|
-
|
|
4128
|
+
const attachRunCredentialRenewal = async (
|
|
4129
|
+
credentialSession: RunCredentialCommandSession,
|
|
4130
|
+
): Promise<void> => {
|
|
4131
|
+
if (!runCredentialResolver) return;
|
|
4132
|
+
const previous = runCredentialRenewal;
|
|
4133
|
+
runCredentialRenewal = null;
|
|
4134
|
+
await previous?.stop();
|
|
4135
|
+
if (runCredentialRenewalClosed) return;
|
|
4136
|
+
runCredentialSession = credentialSession;
|
|
4137
|
+
|
|
4138
|
+
if (!initialRunCredentialMaterial) {
|
|
4139
|
+
await clearRunCredentials(
|
|
4140
|
+
credentialSession,
|
|
4141
|
+
input.sessionId,
|
|
4142
|
+
toolCancellationFenceRef.current
|
|
4143
|
+
? toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
4144
|
+
toolCancellationFenceRef.current,
|
|
4145
|
+
)
|
|
4146
|
+
: undefined,
|
|
4147
|
+
);
|
|
4148
|
+
return;
|
|
4149
|
+
}
|
|
4150
|
+
|
|
4151
|
+
const write = async (
|
|
4152
|
+
material: NormalizedRunCredentialMaterial | null,
|
|
4153
|
+
pruneOtherAttempts = false,
|
|
4154
|
+
): Promise<void> => {
|
|
4155
|
+
if (!material) {
|
|
4156
|
+
await clearRunCredentialsForAttempt(credentialSession, {
|
|
4157
|
+
sessionId: input.sessionId,
|
|
4158
|
+
attemptId: input.attemptId,
|
|
4159
|
+
executionGeneration,
|
|
4160
|
+
});
|
|
4161
|
+
return;
|
|
4162
|
+
}
|
|
4163
|
+
registerSecretRedactions(material.redactions);
|
|
4164
|
+
await materializeRunCredentials(credentialSession, material, {
|
|
4165
|
+
sessionId: input.sessionId,
|
|
4166
|
+
attemptId: input.attemptId,
|
|
4167
|
+
executionGeneration,
|
|
4168
|
+
...(pruneOtherAttempts ? { pruneOtherAttempts: true } : {}),
|
|
4169
|
+
...(!pruneOtherAttempts ? { pruneSupersededGenerations: true } : {}),
|
|
4170
|
+
...(material.authNeeded.length > 0 &&
|
|
4171
|
+
Object.keys(material.environment).length === 0 &&
|
|
4172
|
+
material.files.length === 0
|
|
4173
|
+
? { prunePreviousGenerations: true }
|
|
4174
|
+
: {}),
|
|
4175
|
+
...(toolCancellationFenceRef.current
|
|
4176
|
+
? {
|
|
4177
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
4178
|
+
toolCancellationFenceRef.current,
|
|
4179
|
+
),
|
|
4180
|
+
}
|
|
4181
|
+
: {}),
|
|
4182
|
+
});
|
|
4183
|
+
for (const payload of runCredentialAuthNeededPayloads(material)) {
|
|
4184
|
+
const key = JSON.stringify(payload);
|
|
4185
|
+
if (publishedRunCredentialNotices.has(key)) continue;
|
|
4186
|
+
publishedRunCredentialNotices.add(key);
|
|
4187
|
+
await publish!([{ type: "credential.auth_needed", payload }], true);
|
|
4188
|
+
}
|
|
4189
|
+
};
|
|
4190
|
+
|
|
4191
|
+
const initialExpiryMs = initialRunCredentialMaterial.expiresAt?.getTime() ?? null;
|
|
4192
|
+
const seed =
|
|
4193
|
+
initialExpiryMs !== null && initialExpiryMs <= Date.now() + RUN_CREDENTIAL_EXPIRY_LEAD_MS
|
|
4194
|
+
? await runCredentialResolver.resolve({
|
|
4195
|
+
purpose: "provision",
|
|
4196
|
+
forceRefresh: true,
|
|
4197
|
+
})
|
|
4198
|
+
: initialRunCredentialMaterial;
|
|
4199
|
+
await write(seed, true);
|
|
4200
|
+
if (runCredentialRenewalClosed) return;
|
|
4201
|
+
const controller = startRunCredentialRenewalLoop({
|
|
4202
|
+
initialExpiresAt: seed?.expiresAt ?? null,
|
|
4203
|
+
resolve: async () =>
|
|
4204
|
+
await runCredentialResolver.resolve({
|
|
4205
|
+
purpose: "renewal",
|
|
4206
|
+
forceRefresh: true,
|
|
4207
|
+
}),
|
|
4208
|
+
write: async (material) => await write(material),
|
|
4209
|
+
onSuccess: ({ authNeeded }) => {
|
|
4210
|
+
observability.incrementCounter({
|
|
4211
|
+
name: "opengeni_run_credential_renewals_total",
|
|
4212
|
+
help: "Host-managed run credential renewal attempts by outcome.",
|
|
4213
|
+
labels: { outcome: authNeeded ? "auth_needed" : "completed" },
|
|
4214
|
+
});
|
|
4215
|
+
},
|
|
4216
|
+
onFailure: ({ retryDelayMs, errorClass }) => {
|
|
4217
|
+
observability.incrementCounter({
|
|
4218
|
+
name: "opengeni_run_credential_renewals_total",
|
|
4219
|
+
help: "Host-managed run credential renewal attempts by outcome.",
|
|
4220
|
+
labels: { outcome: "error" },
|
|
4221
|
+
});
|
|
4222
|
+
observability.warn("Host run credential renewal failed; retry scheduled", {
|
|
4223
|
+
sessionId: input.sessionId,
|
|
4224
|
+
turnId,
|
|
4225
|
+
errorClass,
|
|
4226
|
+
retryDelayMs,
|
|
4227
|
+
});
|
|
4228
|
+
},
|
|
4229
|
+
});
|
|
4230
|
+
if (runCredentialRenewalClosed) {
|
|
4231
|
+
await controller.stop();
|
|
4232
|
+
return;
|
|
4233
|
+
}
|
|
4234
|
+
runCredentialRenewal = controller;
|
|
4235
|
+
};
|
|
4236
|
+
|
|
4237
|
+
// P1.2 ownership inversion (gated, default OFF). With the flag off this
|
|
4238
|
+
// block is skipped entirely: resolvedSandbox stays null and runStream
|
|
4239
|
+
// takes the legacy per-run build-and-discard path — byte-for-byte today.
|
|
4240
|
+
// With it on, acquire the group lease (holder = the durable attempt id),
|
|
4241
|
+
// resume the one box by id, and inject it NON-OWNED into the run. The box
|
|
4242
|
+
// backend is "none" -> never resolve (no box to touch).
|
|
4243
|
+
//
|
|
4244
|
+
// Established AFTER sandboxEnvironment is computed (not before) so the box's
|
|
4245
|
+
// manifest is created with the SAME variableSet the agent declares — the SDK
|
|
4246
|
+
// applies the agent's manifest to this provided session and throws on ANY
|
|
4247
|
+
// variableSet delta (validateNoEnvironmentDelta). Passing sandboxEnvironment
|
|
4248
|
+
// here makes current==target so the delta is empty.
|
|
4249
|
+
if (settings.sandboxOwnershipEnabled && turn.sandboxBackend !== "none") {
|
|
4250
|
+
sandboxHolderId = sandboxLeaseHolderIdForAttempt(input.attemptId);
|
|
4251
|
+
sandboxGroupId = session.sandboxGroupId;
|
|
4252
|
+
// STAGE D honest-label guard: a machine-home session carries
|
|
4253
|
+
// turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
|
|
4254
|
+
// when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
|
|
4255
|
+
// + enrollmentId). When it is NOT primary — the agent swapped back to the
|
|
4256
|
+
// group box (sandbox_swap 'session'/'default'/groupId clears the pointer) or
|
|
4257
|
+
// selfhosted routing is flag-OFF (the pointer is ignored) — the else-branch
|
|
4258
|
+
// must resume a REAL cloud group box, not a "selfhosted" one: the registry
|
|
4259
|
+
// SelfhostedSandboxClient has no bound agentId and throws. Fall the group-box
|
|
4260
|
+
// backend back to the deployment default cloud backend so swap-away / flag-off
|
|
4261
|
+
// degrade to a genuine cloud box exactly like today (home=modal did).
|
|
4262
|
+
if (machinePrimary) {
|
|
4263
|
+
// STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
|
|
4264
|
+
// establish or lease a phantom Modal home box (today's path leased + BILLED
|
|
4265
|
+
// a cloud box the turn never touched). Build the SelfhostedSession DIRECTLY
|
|
4266
|
+
// (no Modal box created) and take the group lease with backend "selfhosted"
|
|
4267
|
+
// (refcount/idle bookkeeping; the reaper drains it cold with NO provider
|
|
4268
|
+
// stop, and bills ZERO warm-seconds). The session is a harmless in-memory
|
|
4269
|
+
// bind (no NATS round-trip), so build it FIRST; if the lease then fences,
|
|
4270
|
+
// there is nothing to clean up.
|
|
4271
|
+
// Whether the machine's latest Hello advertised the op-stream engine
|
|
4272
|
+
// (refreshed on every connect). Read only when the server flag is on —
|
|
4273
|
+
// one indexed lookup, and the flag off keeps this path byte-identical.
|
|
4274
|
+
const machineOpStream =
|
|
4275
|
+
settings.agentOpStreamEnabled === true
|
|
4276
|
+
? (await getEnrollment(db, input.workspaceId, activeSandboxRecord!.enrollmentId!))
|
|
4277
|
+
?.opStream === true
|
|
3064
4278
|
: false;
|
|
3065
4279
|
const established = await establishSelfhostedTurnSession(
|
|
3066
4280
|
{
|
|
@@ -3083,16 +4297,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3083
4297
|
// (buildSelfhostedBackendSession); EstablishedSandboxSession widens it.
|
|
3084
4298
|
machinePrimarySession =
|
|
3085
4299
|
established.session as import("@opengeni/runtime").SelfhostedSession;
|
|
3086
|
-
const lease = await
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
3090
|
-
|
|
3091
|
-
|
|
3092
|
-
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
|
|
4300
|
+
const lease = await waitForTurnOperation(
|
|
4301
|
+
acquireSelfhostedLeaseForTurn(
|
|
4302
|
+
{ db, settings },
|
|
4303
|
+
{
|
|
4304
|
+
accountId: input.accountId,
|
|
4305
|
+
workspaceId: input.workspaceId,
|
|
4306
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
4307
|
+
sessionId: input.sessionId,
|
|
4308
|
+
},
|
|
4309
|
+
"turn",
|
|
4310
|
+
sandboxHolderId,
|
|
4311
|
+
),
|
|
4312
|
+
cancellationSignal,
|
|
4313
|
+
async (lateLease) => await lateLease.release(),
|
|
3096
4314
|
);
|
|
3097
4315
|
setupBoxSession = established.session;
|
|
3098
4316
|
resolvedSandbox = {
|
|
@@ -3137,45 +4355,49 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3137
4355
|
// proxy's first default-pointer op. A chat-only turn never calls it, so
|
|
3138
4356
|
// no lease row, no provider box, no warm-meter interval.
|
|
3139
4357
|
} else {
|
|
3140
|
-
resolvedSandbox = await
|
|
3141
|
-
|
|
3142
|
-
|
|
3143
|
-
|
|
3144
|
-
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
|
|
3156
|
-
|
|
3157
|
-
|
|
3158
|
-
|
|
3159
|
-
|
|
3160
|
-
|
|
3161
|
-
|
|
3162
|
-
|
|
3163
|
-
|
|
3164
|
-
|
|
3165
|
-
|
|
3166
|
-
|
|
3167
|
-
|
|
3168
|
-
|
|
3169
|
-
|
|
3170
|
-
|
|
3171
|
-
|
|
3172
|
-
|
|
3173
|
-
|
|
3174
|
-
|
|
3175
|
-
|
|
3176
|
-
|
|
3177
|
-
|
|
3178
|
-
|
|
4358
|
+
resolvedSandbox = await waitForTurnOperation(
|
|
4359
|
+
resumeBoxForTurn(
|
|
4360
|
+
{
|
|
4361
|
+
db,
|
|
4362
|
+
settings,
|
|
4363
|
+
sandboxMetrics: runtimeMetricsHooksForObservability(observability),
|
|
4364
|
+
onSandboxLost: publishSandboxLost,
|
|
4365
|
+
},
|
|
4366
|
+
{
|
|
4367
|
+
accountId: input.accountId,
|
|
4368
|
+
workspaceId: input.workspaceId,
|
|
4369
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
4370
|
+
sessionId: input.sessionId,
|
|
4371
|
+
// groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
|
|
4372
|
+
// is not machine-primary resumes a real cloud group box (the
|
|
4373
|
+
// deployment default), never a "selfhosted" box (which would throw
|
|
4374
|
+
// for lack of a bound agentId).
|
|
4375
|
+
backend: groupBoxBackend,
|
|
4376
|
+
os: session.sandboxOs,
|
|
4377
|
+
environment: sandboxEnvironment,
|
|
4378
|
+
// IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
|
|
4379
|
+
// this run resolves. The lease stamps it + conflicts on a live shared box
|
|
4380
|
+
// running a DIFFERENT image (solo → recreate on the new image; N-holders →
|
|
4381
|
+
// SandboxImageConflictError surfaced as an actionable turn error). Prefer the
|
|
4382
|
+
// explicit Modal image ref, else the docker image. The selfhosted branch
|
|
4383
|
+
// (establishSelfhostedTurnSession/acquireSelfhostedLeaseForTurn) NEVER passes
|
|
4384
|
+
// an image — B3 lives only on this Modal else-branch.
|
|
4385
|
+
...((runSettings.modalImageRef ?? runSettings.dockerImage)
|
|
4386
|
+
? {
|
|
4387
|
+
image: runSettings.modalImageRef ?? runSettings.dockerImage,
|
|
4388
|
+
}
|
|
4389
|
+
: {}),
|
|
4390
|
+
// RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
|
|
4391
|
+
// conflicts on a live shared box set up under a different rig (solo
|
|
4392
|
+
// recreate / N-holders SandboxRigConflictError). Omitted for a rig-less
|
|
4393
|
+
// turn -> never stamped or enforced (shares exactly as today).
|
|
4394
|
+
...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
|
|
4395
|
+
},
|
|
4396
|
+
"turn",
|
|
4397
|
+
sandboxHolderId,
|
|
4398
|
+
),
|
|
4399
|
+
cancellationSignal,
|
|
4400
|
+
async (lateSandbox) => await lateSandbox.release(),
|
|
3179
4401
|
);
|
|
3180
4402
|
setupBoxSession = resolvedSandbox.established.session;
|
|
3181
4403
|
// Durable box-lifecycle events (sandbox-file-persistence observability):
|
|
@@ -3216,42 +4438,65 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3216
4438
|
}
|
|
3217
4439
|
}
|
|
3218
4440
|
|
|
3219
|
-
const fileResourceDownloads = await
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
4441
|
+
const fileResourceDownloads = await waitForTurnOperation(
|
|
4442
|
+
sandboxFileDownloadsForRun(
|
|
4443
|
+
runSettings,
|
|
4444
|
+
db,
|
|
4445
|
+
objectStorage,
|
|
4446
|
+
input.workspaceId,
|
|
4447
|
+
turnResources,
|
|
4448
|
+
),
|
|
4449
|
+
cancellationSignal,
|
|
4450
|
+
undefined,
|
|
3225
4451
|
);
|
|
3226
4452
|
throwIfWorkerShuttingDown();
|
|
4453
|
+
throwIfTurnCancelled();
|
|
4454
|
+
const mcpCredentialRootSessionId =
|
|
4455
|
+
connectionCredentials?.mcpCredentials && hostCredentialRootSessionId
|
|
4456
|
+
? hostCredentialRootSessionId
|
|
4457
|
+
: input.sessionId;
|
|
3227
4458
|
// Wrap MCP prep in the codex ALS so the codex_apps connect handshake
|
|
3228
4459
|
// (initialize + tools/list) can resolve the per-workspace bearer from
|
|
3229
4460
|
// codexRequestStorage (runtime/codexAppsMcpRequestInit). withCodex is the
|
|
3230
4461
|
// identity on every non-codex turn, so this is a no-op for existing paths.
|
|
3231
|
-
const resolveCredential =
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
|
|
3241
|
-
|
|
3242
|
-
|
|
3243
|
-
|
|
3244
|
-
|
|
3245
|
-
|
|
3246
|
-
|
|
3247
|
-
|
|
3248
|
-
|
|
3249
|
-
|
|
3250
|
-
|
|
3251
|
-
|
|
3252
|
-
|
|
3253
|
-
|
|
3254
|
-
|
|
4462
|
+
const resolveCredential = connectionTokenResolverForTurn({
|
|
4463
|
+
db,
|
|
4464
|
+
settings: runSettings,
|
|
4465
|
+
connectionCredentials: connectionCredentials ?? null,
|
|
4466
|
+
accountId: input.accountId,
|
|
4467
|
+
workspaceId: input.workspaceId,
|
|
4468
|
+
sessionId: input.sessionId,
|
|
4469
|
+
rootSessionId: mcpCredentialRootSessionId,
|
|
4470
|
+
attemptId: input.attemptId,
|
|
4471
|
+
turn,
|
|
4472
|
+
});
|
|
4473
|
+
preparedTools = await waitForTurnOperation(
|
|
4474
|
+
withCodex(() =>
|
|
4475
|
+
runtime.prepareTools(runSettings, turnTools, {
|
|
4476
|
+
accountId: input.accountId,
|
|
4477
|
+
workspaceId: input.workspaceId,
|
|
4478
|
+
sessionId: input.sessionId,
|
|
4479
|
+
// Sign the calling turn into the first-party token so tools classify
|
|
4480
|
+
// the caller by its own identity (sacred-pause guard), not the racy
|
|
4481
|
+
// live active pointer.
|
|
4482
|
+
...(turnId ? { turnId } : {}),
|
|
4483
|
+
attemptId: input.attemptId,
|
|
4484
|
+
executionGeneration,
|
|
4485
|
+
subjectId: "worker:first-party-mcp",
|
|
4486
|
+
subjectLabel: "OpenGeni worker",
|
|
4487
|
+
resolveCredential,
|
|
4488
|
+
onAuthNeeded: async (payload) => {
|
|
4489
|
+
await publish!([{ type: "tool.auth_needed", payload }], true);
|
|
4490
|
+
},
|
|
4491
|
+
// Manager-style sessions carry a creation-validated permission set
|
|
4492
|
+
// for their first-party MCP token; null keeps the fixed default.
|
|
4493
|
+
...(session.firstPartyMcpPermissions?.length
|
|
4494
|
+
? { firstPartyPermissions: session.firstPartyMcpPermissions }
|
|
4495
|
+
: {}),
|
|
4496
|
+
}),
|
|
4497
|
+
),
|
|
4498
|
+
cancellationSignal,
|
|
4499
|
+
async (latePreparedTools) => await latePreparedTools.close().catch(() => undefined),
|
|
3255
4500
|
);
|
|
3256
4501
|
// Genesis turn = the first user turn (no assistant history reconciled
|
|
3257
4502
|
// yet). Durable Postgres state (countSessionHistoryItems includes
|
|
@@ -3292,11 +4537,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3292
4537
|
: {};
|
|
3293
4538
|
const agent = runtime.buildAgent(modelRunSettings, turnResources, {
|
|
3294
4539
|
reasoningEffort: turn.reasoningEffort,
|
|
4540
|
+
...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
|
|
3295
4541
|
genesisTitleHint: isGenesisTurn,
|
|
3296
4542
|
persistentSessionSettings: {
|
|
3297
4543
|
titleIsSet: Boolean(session.title?.trim()),
|
|
3298
4544
|
},
|
|
3299
4545
|
sandboxEnvironment,
|
|
4546
|
+
...(cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {}),
|
|
4547
|
+
onToolCancellationFence: (fence) => {
|
|
4548
|
+
toolCancellationFenceRef.current = fence;
|
|
4549
|
+
},
|
|
3300
4550
|
// TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
|
|
3301
4551
|
// seed. ONLY when the effective backend is NOT selfhosted (the connected
|
|
3302
4552
|
// machine uses its own git creds — mirrors the skipGitHubToken gate above)
|
|
@@ -3306,6 +4556,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3306
4556
|
...(activeSandboxBackend !== "selfhosted" && sandboxGitTokens
|
|
3307
4557
|
? { gitTokenSeeds: sandboxGitTokens }
|
|
3308
4558
|
: {}),
|
|
4559
|
+
...(activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings
|
|
4560
|
+
? { gitCredentialBindings: sandboxGitCredentialBindings }
|
|
4561
|
+
: {}),
|
|
3309
4562
|
...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
|
|
3310
4563
|
? { gitTokenSeed: sandboxGitToken }
|
|
3311
4564
|
: {}),
|
|
@@ -3313,10 +4566,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3313
4566
|
// token skip does NOT transfer: that token is inert on a connected
|
|
3314
4567
|
// machine (it uses its own git creds), but the toolspace token is the
|
|
3315
4568
|
// machine's ONLY path to programmatic tool calling and grants no more
|
|
3316
|
-
// than toolspace:call for its own session (own-session-bound,
|
|
3317
|
-
// budgeted, approval-tools
|
|
3318
|
-
//
|
|
3319
|
-
|
|
4569
|
+
// than toolspace:call for its own session (own-session-bound,
|
|
4570
|
+
// short-lived and renewable, per-active-turn budgeted, approval-tools
|
|
4571
|
+
// excluded). The runtime seeds it to the box's token file over the same
|
|
4572
|
+
// exec channel, off-manifest, on every backend.
|
|
4573
|
+
...(sandboxToolspaceToken
|
|
4574
|
+
? {
|
|
4575
|
+
toolspaceTokenSeed: sandboxToolspaceToken,
|
|
4576
|
+
toolspaceTokenSessionId: input.sessionId,
|
|
4577
|
+
}
|
|
4578
|
+
: {}),
|
|
3320
4579
|
...(activeSandboxBackend ? { activeSandboxBackend } : {}),
|
|
3321
4580
|
fileResourceDownloads,
|
|
3322
4581
|
mcpServers: preparedTools.mcpServers,
|
|
@@ -3384,12 +4643,21 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3384
4643
|
await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
|
|
3385
4644
|
},
|
|
3386
4645
|
...(packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {}),
|
|
4646
|
+
...(skillLibraryRuntime.skillLibrarySkills.length > 0
|
|
4647
|
+
? {
|
|
4648
|
+
skillLibrarySkills: skillLibraryRuntime.skillLibrarySkills,
|
|
4649
|
+
skillLibrarySelections: skillLibraryRuntime.skillLibrarySelections,
|
|
4650
|
+
}
|
|
4651
|
+
: {}),
|
|
3387
4652
|
...(workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {}),
|
|
3388
4653
|
...(workspaceMemory ? { workspaceMemory } : {}),
|
|
3389
4654
|
// Per-session persona tier (session > workspace > deployment default).
|
|
3390
4655
|
// Composed system-level AFTER the workspace persona so it refines it for
|
|
3391
4656
|
// this one session; absent ⇒ byte-identical to today's composition.
|
|
3392
4657
|
...(session.instructions ? { sessionInstructions: session.instructions } : {}),
|
|
4658
|
+
// Exact host context captured when this turn was accepted. It is
|
|
4659
|
+
// system-level and disappears with the turn rather than entering chat.
|
|
4660
|
+
...(turn.turnInstructions ? { turnInstructions: turn.turnInstructions } : {}),
|
|
3393
4661
|
...workspaceEnvironmentOption,
|
|
3394
4662
|
// RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
|
|
3395
4663
|
// the frozen version carries a non-empty script), and the rig credential
|
|
@@ -3414,6 +4682,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3414
4682
|
}
|
|
3415
4683
|
: {}),
|
|
3416
4684
|
});
|
|
4685
|
+
if (modelRunSettings.sandboxBackend !== "none" && toolCancellationFenceRef.current === null) {
|
|
4686
|
+
throw new Error(
|
|
4687
|
+
"Sandbox agent construction did not install the mandatory turn tool cancellation fence",
|
|
4688
|
+
);
|
|
4689
|
+
}
|
|
3417
4690
|
if (establishPolicy === "on-demand" && sandboxHolderId && sandboxGroupId) {
|
|
3418
4691
|
const lazyHolderId = sandboxHolderId;
|
|
3419
4692
|
const lazyGroupId = sandboxGroupId;
|
|
@@ -3424,18 +4697,23 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3424
4697
|
const lazyClient = {
|
|
3425
4698
|
backendId: sdkBackendIdForSandboxBackend(groupBoxBackend),
|
|
3426
4699
|
} as EstablishedSandboxSession["client"];
|
|
3427
|
-
let lazyEstablishmentOrigin: EstablishedSandboxSession["origin"] | null = null;
|
|
3428
4700
|
turnSandboxProvisioner = createTurnSandboxProvisioner<ResumedTurnSandbox>(
|
|
3429
4701
|
async () => {
|
|
3430
4702
|
throwIfWorkerShuttingDown();
|
|
4703
|
+
throwIfTurnCancelled();
|
|
3431
4704
|
const lazyGitCredentials =
|
|
3432
4705
|
activeSandboxBackend === "selfhosted"
|
|
3433
4706
|
? undefined
|
|
3434
4707
|
: await mintRunGitCredentials(runSettings, turnResources, {
|
|
3435
4708
|
scope: connectionScope,
|
|
4709
|
+
...(gitCredentialAuthority ? { authority: gitCredentialAuthority } : {}),
|
|
3436
4710
|
gitCredentials: connectionCredentials?.gitCredentials,
|
|
4711
|
+
authorizeGitHubTokenMint,
|
|
3437
4712
|
});
|
|
3438
4713
|
const lazyGitTokens = lazyGitCredentials?.gitTokens;
|
|
4714
|
+
const lazyToolspaceToken = sandboxToolspaceToken
|
|
4715
|
+
? await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority)
|
|
4716
|
+
: undefined;
|
|
3439
4717
|
const provisioned = await resumeBoxForTurn(
|
|
3440
4718
|
{
|
|
3441
4719
|
db,
|
|
@@ -3460,13 +4738,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3460
4738
|
"turn",
|
|
3461
4739
|
lazyHolderId,
|
|
3462
4740
|
);
|
|
3463
|
-
lazyEstablishmentOrigin = provisioned.established.origin ?? null;
|
|
3464
|
-
setupBoxSession = provisioned.established.session;
|
|
3465
4741
|
await publishSandboxLifecycleEvents(provisioned);
|
|
4742
|
+
await attachRunCredentialRenewal(
|
|
4743
|
+
provisioned.established.session as RunCredentialCommandSession,
|
|
4744
|
+
);
|
|
4745
|
+
const provisionedSetupSession = initialRunCredentialMaterial
|
|
4746
|
+
? withRunCredentialsSession(
|
|
4747
|
+
provisioned.established.session as object,
|
|
4748
|
+
input.sessionId,
|
|
4749
|
+
)
|
|
4750
|
+
: provisioned.established.session;
|
|
3466
4751
|
await runOwnedSandboxSetup(
|
|
3467
4752
|
agent,
|
|
3468
4753
|
provisioned.established.session as never,
|
|
3469
|
-
|
|
4754
|
+
provisionedSetupSession as never,
|
|
3470
4755
|
{
|
|
3471
4756
|
settings: runSettings,
|
|
3472
4757
|
environment: sandboxEnvironment,
|
|
@@ -3474,8 +4759,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3474
4759
|
await publish?.([{ type: event.type, payload: event.payload }], true);
|
|
3475
4760
|
},
|
|
3476
4761
|
...(lazyGitTokens ? { gitTokenSeedsOverride: lazyGitTokens } : {}),
|
|
4762
|
+
...(lazyGitCredentials?.bindings
|
|
4763
|
+
? { gitCredentialBindingsOverride: lazyGitCredentials.bindings }
|
|
4764
|
+
: {}),
|
|
4765
|
+
...(lazyToolspaceToken
|
|
4766
|
+
? { toolspaceTokenSeedOverride: lazyToolspaceToken.token }
|
|
4767
|
+
: {}),
|
|
4768
|
+
...(toolCancellationFenceRef.current
|
|
4769
|
+
? {
|
|
4770
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
4771
|
+
toolCancellationFenceRef.current,
|
|
4772
|
+
),
|
|
4773
|
+
}
|
|
4774
|
+
: {}),
|
|
3477
4775
|
},
|
|
3478
4776
|
);
|
|
4777
|
+
await attachToolspaceTokenRenewal(
|
|
4778
|
+
provisioned.established.session as ToolspaceTokenWriterSession,
|
|
4779
|
+
lazyToolspaceToken?.expiresAt,
|
|
4780
|
+
);
|
|
3479
4781
|
await attachGitCredentialRenewal(
|
|
3480
4782
|
provisioned.established.session as GitCredentialTokenWriterSession,
|
|
3481
4783
|
lazyGitCredentials,
|
|
@@ -3488,11 +4790,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3488
4790
|
// already holds the proxy directly (injected as lazyOwnedSandbox.session), so it
|
|
3489
4791
|
// gets per-op routing; the worker-side handle (resolvedSandbox: release,
|
|
3490
4792
|
// heartbeat, computer-use recording) wants the real box, unproxied.
|
|
3491
|
-
resolvedSandbox = provisioned;
|
|
3492
|
-
startLeaseHeartbeat(provisioned, activeSandboxBackend ?? groupBoxBackend);
|
|
3493
4793
|
return provisioned;
|
|
3494
4794
|
},
|
|
3495
4795
|
{
|
|
4796
|
+
...(activityContext ? { signal: activityContext.cancellationSignal } : {}),
|
|
3496
4797
|
onStarted: async () => {
|
|
3497
4798
|
await publish?.(
|
|
3498
4799
|
[
|
|
@@ -3504,19 +4805,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3504
4805
|
true,
|
|
3505
4806
|
);
|
|
3506
4807
|
},
|
|
3507
|
-
onCompleted: async () => {
|
|
4808
|
+
onCompleted: async (provisioned) => {
|
|
3508
4809
|
await publish?.(
|
|
3509
4810
|
[
|
|
3510
4811
|
{
|
|
3511
4812
|
type: "sandbox.operation.completed",
|
|
3512
4813
|
payload: {
|
|
3513
4814
|
name: "sandbox.provision",
|
|
3514
|
-
...(
|
|
4815
|
+
...(provisioned.established.origin
|
|
4816
|
+
? { origin: provisioned.established.origin }
|
|
4817
|
+
: {}),
|
|
3515
4818
|
},
|
|
3516
4819
|
},
|
|
3517
4820
|
],
|
|
3518
4821
|
true,
|
|
3519
4822
|
);
|
|
4823
|
+
throwIfTurnOperationCancelled(activityContext?.cancellationSignal);
|
|
4824
|
+
startLeaseHeartbeat(provisioned, activeSandboxBackend ?? groupBoxBackend);
|
|
4825
|
+
setupBoxSession = provisioned.established.session;
|
|
4826
|
+
resolvedSandbox = provisioned;
|
|
3520
4827
|
},
|
|
3521
4828
|
onFailed: async (error) => {
|
|
3522
4829
|
await publish?.(
|
|
@@ -3532,6 +4839,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3532
4839
|
true,
|
|
3533
4840
|
);
|
|
3534
4841
|
},
|
|
4842
|
+
disposeResult: async (provisioned) => {
|
|
4843
|
+
await provisioned.release().catch(() => undefined);
|
|
4844
|
+
},
|
|
3535
4845
|
},
|
|
3536
4846
|
);
|
|
3537
4847
|
lazyOwnedSandbox = wrapLazyTurnBoxWithRouting(
|
|
@@ -3566,36 +4876,41 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3566
4876
|
// RunState verbatim and recovering attempts already compacted, if needed,
|
|
3567
4877
|
// before the first attempt's model boundary.
|
|
3568
4878
|
if (triggerType === "user.message" || triggerType === "system.update.delivered") {
|
|
4879
|
+
let forced = false;
|
|
3569
4880
|
try {
|
|
3570
4881
|
// Operator /compact (the slash command) sets a durable request flag;
|
|
3571
4882
|
// observe it without consuming it so a failed/stale attempt cannot
|
|
3572
4883
|
// lose the request. The replacement transaction clears it on success.
|
|
3573
|
-
|
|
3574
|
-
const outcome = await
|
|
3575
|
-
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
|
|
3581
|
-
|
|
3582
|
-
|
|
3583
|
-
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
|
|
3587
|
-
|
|
3588
|
-
|
|
3589
|
-
|
|
3590
|
-
|
|
3591
|
-
|
|
3592
|
-
|
|
3593
|
-
|
|
3594
|
-
|
|
3595
|
-
|
|
3596
|
-
|
|
3597
|
-
|
|
3598
|
-
|
|
4884
|
+
forced = await isSessionCompactionRequested(db, input.workspaceId, input.sessionId);
|
|
4885
|
+
const outcome = await waitForTurnOperation(
|
|
4886
|
+
maybeCompactContext(
|
|
4887
|
+
db,
|
|
4888
|
+
modelRunSettings,
|
|
4889
|
+
{
|
|
4890
|
+
accountId: input.accountId,
|
|
4891
|
+
workspaceId: input.workspaceId,
|
|
4892
|
+
sessionId: input.sessionId,
|
|
4893
|
+
turnId: turnId!,
|
|
4894
|
+
executionGeneration,
|
|
4895
|
+
attemptId: input.attemptId,
|
|
4896
|
+
},
|
|
4897
|
+
session.lastInputTokens,
|
|
4898
|
+
// Provider-aware summarizer: when the turn's model resolved to a
|
|
4899
|
+
// registry provider, summarize on THAT provider's client + wire API
|
|
4900
|
+
// (a chat provider can't summarize through OpenAI/Azure). Null
|
|
4901
|
+
// resolution uses the built-in Responses summarizer with the same
|
|
4902
|
+
// session prompt-cache key as the main model calls.
|
|
4903
|
+
compactSummarizer,
|
|
4904
|
+
forced
|
|
4905
|
+
? {
|
|
4906
|
+
force: true,
|
|
4907
|
+
clearRequestedCompaction: true,
|
|
4908
|
+
trigger: "operator",
|
|
4909
|
+
}
|
|
4910
|
+
: { trigger: "auto" },
|
|
4911
|
+
),
|
|
4912
|
+
cancellationSignal,
|
|
4913
|
+
undefined,
|
|
3599
4914
|
);
|
|
3600
4915
|
if (outcome.compacted) {
|
|
3601
4916
|
const compactionTrigger = forced ? "operator" : undefined;
|
|
@@ -3608,12 +4923,41 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3608
4923
|
);
|
|
3609
4924
|
}
|
|
3610
4925
|
} catch (compactError) {
|
|
4926
|
+
if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
|
|
4927
|
+
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
4928
|
+
const errorMessage = compactionFailureReasonFromError(compactError);
|
|
3611
4929
|
observability.error("context compaction failed", {
|
|
3612
4930
|
sessionId: input.sessionId,
|
|
3613
4931
|
turnId,
|
|
3614
|
-
error:
|
|
4932
|
+
error: errorMessage,
|
|
3615
4933
|
});
|
|
3616
|
-
|
|
4934
|
+
if (
|
|
4935
|
+
!(await settle!({
|
|
4936
|
+
events: [
|
|
4937
|
+
{
|
|
4938
|
+
type: "turn.failed",
|
|
4939
|
+
payload: {
|
|
4940
|
+
error: errorMessage,
|
|
4941
|
+
code: "context_compaction_failed",
|
|
4942
|
+
retryable: false,
|
|
4943
|
+
recovery: "user_message",
|
|
4944
|
+
compacted: false,
|
|
4945
|
+
},
|
|
4946
|
+
},
|
|
4947
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
4948
|
+
],
|
|
4949
|
+
turnStatus: "failed",
|
|
4950
|
+
sessionStatus: "idle",
|
|
4951
|
+
activeTurnId: null,
|
|
4952
|
+
...(forced ? { consumeRequestedCompactionFailure: true } : {}),
|
|
4953
|
+
}))
|
|
4954
|
+
) {
|
|
4955
|
+
return claimedResult({ status: "cancelled" });
|
|
4956
|
+
}
|
|
4957
|
+
turnMetricOutcome = "failed";
|
|
4958
|
+
activityStatus = "idle";
|
|
4959
|
+
activityError = compactError;
|
|
4960
|
+
return claimedResult({ status: "idle" });
|
|
3617
4961
|
}
|
|
3618
4962
|
}
|
|
3619
4963
|
let fileMaterializationFailures: SandboxFileDownloadFailure[] = [];
|
|
@@ -3646,6 +4990,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3646
4990
|
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
3647
4991
|
},
|
|
3648
4992
|
...(runAs ? { runAs } : {}),
|
|
4993
|
+
...(toolCancellationFenceRef.current
|
|
4994
|
+
? {
|
|
4995
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
4996
|
+
toolCancellationFenceRef.current,
|
|
4997
|
+
),
|
|
4998
|
+
}
|
|
4999
|
+
: {}),
|
|
3649
5000
|
},
|
|
3650
5001
|
);
|
|
3651
5002
|
fileMaterializationFailures = materialized.failures;
|
|
@@ -3693,6 +5044,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3693
5044
|
turnId: activeTurnId,
|
|
3694
5045
|
recovering: turn.executionGeneration > 1,
|
|
3695
5046
|
...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
|
|
5047
|
+
...(runCredentialsNote ? { runCredentialsNote } : {}),
|
|
3696
5048
|
},
|
|
3697
5049
|
);
|
|
3698
5050
|
runInput = prepared.input;
|
|
@@ -3740,29 +5092,33 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3740
5092
|
triggerLabel: "overflow" | "proactive" | "operator",
|
|
3741
5093
|
recoverySignalTokens: number | null,
|
|
3742
5094
|
) => {
|
|
3743
|
-
const outcome = await
|
|
3744
|
-
|
|
3745
|
-
|
|
3746
|
-
|
|
3747
|
-
|
|
3748
|
-
|
|
3749
|
-
|
|
3750
|
-
|
|
3751
|
-
|
|
3752
|
-
|
|
3753
|
-
|
|
3754
|
-
|
|
3755
|
-
|
|
3756
|
-
|
|
3757
|
-
|
|
3758
|
-
|
|
3759
|
-
|
|
3760
|
-
|
|
3761
|
-
|
|
3762
|
-
|
|
3763
|
-
|
|
3764
|
-
|
|
3765
|
-
|
|
5095
|
+
const outcome = await waitForTurnOperation(
|
|
5096
|
+
maybeCompactContext(
|
|
5097
|
+
db,
|
|
5098
|
+
modelRunSettings,
|
|
5099
|
+
{
|
|
5100
|
+
accountId: input.accountId,
|
|
5101
|
+
workspaceId: input.workspaceId,
|
|
5102
|
+
sessionId: input.sessionId,
|
|
5103
|
+
turnId: activeTurnId,
|
|
5104
|
+
executionGeneration,
|
|
5105
|
+
attemptId: input.attemptId,
|
|
5106
|
+
},
|
|
5107
|
+
// Never reuse the persisted prior-turn signal for recovery. Proactive
|
|
5108
|
+
// guards provide their exact current signal; provider overflows do not,
|
|
5109
|
+
// so null derives decision metadata from active history. Forced
|
|
5110
|
+
// recovery proves progress separately by comparing the replacement
|
|
5111
|
+
// with the current active-history estimate in maybeCompactContext.
|
|
5112
|
+
recoverySignalTokens,
|
|
5113
|
+
compactSummarizer,
|
|
5114
|
+
{
|
|
5115
|
+
force: true,
|
|
5116
|
+
...(triggerLabel === "operator" ? { clearRequestedCompaction: true } : {}),
|
|
5117
|
+
trigger: triggerLabel,
|
|
5118
|
+
},
|
|
5119
|
+
),
|
|
5120
|
+
cancellationSignal,
|
|
5121
|
+
undefined,
|
|
3766
5122
|
);
|
|
3767
5123
|
if (outcome.events.length > 0) {
|
|
3768
5124
|
if (outcome.compacted) {
|
|
@@ -3778,17 +5134,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3778
5134
|
return outcome;
|
|
3779
5135
|
};
|
|
3780
5136
|
|
|
5137
|
+
// Keep response usage identity across every stream attempt in this
|
|
5138
|
+
// activity. Context compaction retries the same logical dispatch by
|
|
5139
|
+
// calling runStreamAttempt again; resetting this state there would reuse
|
|
5140
|
+
// the first no-response-ID fallback key and suppress a real model call.
|
|
5141
|
+
const modelResponseUsageState = createModelResponseUsageEventState(
|
|
5142
|
+
claimedModelUsageSourceKeys,
|
|
5143
|
+
);
|
|
3781
5144
|
const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
|
|
3782
5145
|
if (!runInput) {
|
|
3783
5146
|
throw new Error("Run input was not prepared");
|
|
3784
5147
|
}
|
|
3785
5148
|
stream = undefined;
|
|
3786
5149
|
batcher = null;
|
|
3787
|
-
|
|
5150
|
+
// The SDK emits every processed call item for one model response before
|
|
5151
|
+
// it emits any result for that response. Keep that response-local batch
|
|
5152
|
+
// in memory so an orphan from an older response cannot pin later stable
|
|
5153
|
+
// calls. Durable recovery remains call/result based and needs no batch
|
|
5154
|
+
// schema or compatibility state.
|
|
5155
|
+
let currentToolBatchCallIds = new Set<string>();
|
|
5156
|
+
let currentToolBatchCompletedCallIds = new Set<string>();
|
|
3788
5157
|
// Actual input tokens of the most recent model response this turn; the
|
|
3789
5158
|
// pre-read trigger for the NEXT turn. Persisted at every turn-end path.
|
|
3790
|
-
let lastInputTokensObserved: number | null = null;
|
|
3791
5159
|
throwIfWorkerShuttingDown();
|
|
5160
|
+
throwIfTurnCancelled();
|
|
3792
5161
|
const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
|
|
3793
5162
|
const runStreamOnce = (): ReturnType<OpenGeniRuntime["runStream"]> =>
|
|
3794
5163
|
runtime.runStream(agent, runInput!, modelRunSettings, {
|
|
@@ -3831,9 +5200,42 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3831
5200
|
},
|
|
3832
5201
|
}
|
|
3833
5202
|
: {}),
|
|
3834
|
-
|
|
5203
|
+
...(sandboxToolspaceToken && sandboxToolspaceTokenExpiresAt && !lazyOwnedSandbox
|
|
5204
|
+
? {
|
|
5205
|
+
onToolspaceTokenSessionReady: async (
|
|
5206
|
+
tokenSession: ToolspaceTokenWriterSession,
|
|
5207
|
+
) => {
|
|
5208
|
+
const renewalSession =
|
|
5209
|
+
activeSandboxBackend === "selfhosted"
|
|
5210
|
+
? tokenSession
|
|
5211
|
+
: ((setupBoxSession as ToolspaceTokenWriterSession | null) ?? tokenSession);
|
|
5212
|
+
await attachToolspaceTokenRenewal(renewalSession);
|
|
5213
|
+
},
|
|
5214
|
+
}
|
|
5215
|
+
: {}),
|
|
5216
|
+
...(runCredentialResolver
|
|
5217
|
+
? {
|
|
5218
|
+
runCredentialSessionId: input.sessionId,
|
|
5219
|
+
...(!lazyOwnedSandbox
|
|
5220
|
+
? {
|
|
5221
|
+
onRunCredentialSessionReady: async (
|
|
5222
|
+
credentialSession: RunCredentialCommandSession,
|
|
5223
|
+
) => {
|
|
5224
|
+
const pinnedCredentialSession = setupBoxSession
|
|
5225
|
+
? (setupBoxSession as RunCredentialCommandSession)
|
|
5226
|
+
: credentialSession;
|
|
5227
|
+
await attachRunCredentialRenewal(pinnedCredentialSession);
|
|
5228
|
+
},
|
|
5229
|
+
}
|
|
5230
|
+
: {}),
|
|
5231
|
+
}
|
|
5232
|
+
: {}),
|
|
5233
|
+
contextCompactionSignal: () => modelResponseUsageContextSignal(modelResponseUsageState),
|
|
3835
5234
|
contextCompactionRequested: () =>
|
|
3836
5235
|
isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
5236
|
+
...(toolCancellationFenceRef.current
|
|
5237
|
+
? { turnToolCancellationFence: toolCancellationFenceRef.current }
|
|
5238
|
+
: {}),
|
|
3837
5239
|
});
|
|
3838
5240
|
if (codexLeaseLost) {
|
|
3839
5241
|
throw new Error("Codex credential lease expired before the model run");
|
|
@@ -3865,72 +5267,37 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3865
5267
|
streamDone = true;
|
|
3866
5268
|
break;
|
|
3867
5269
|
}
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
3871
|
-
|
|
3872
|
-
|
|
3873
|
-
|
|
3874
|
-
|
|
3875
|
-
|
|
3876
|
-
|
|
3877
|
-
|
|
3878
|
-
|
|
3879
|
-
|
|
3880
|
-
|
|
3881
|
-
|
|
3882
|
-
|
|
3883
|
-
|
|
3884
|
-
|
|
3885
|
-
|
|
3886
|
-
|
|
3887
|
-
|
|
3888
|
-
|
|
3889
|
-
|
|
3890
|
-
|
|
3891
|
-
|
|
3892
|
-
|
|
3893
|
-
|
|
3894
|
-
|
|
3895
|
-
|
|
3896
|
-
|
|
3897
|
-
|
|
3898
|
-
|
|
3899
|
-
await emitModelCallUsage({
|
|
3900
|
-
observability,
|
|
3901
|
-
publish,
|
|
3902
|
-
accountId: input.accountId,
|
|
3903
|
-
workspaceId: input.workspaceId,
|
|
3904
|
-
sessionId: input.sessionId,
|
|
3905
|
-
turnId: activeTurnId,
|
|
3906
|
-
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
3907
|
-
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
3908
|
-
model: turn.model,
|
|
3909
|
-
sourceKey: responseSourceKey,
|
|
3910
|
-
usage: responseUsage,
|
|
3911
|
-
servingAccountHash: responseAccountCtx.servingAccountHash,
|
|
3912
|
-
accountChangedFromPrevCall: responseAccountCtx.accountChangedFromPrevCall,
|
|
3913
|
-
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
3914
|
-
});
|
|
3915
|
-
const observed = responseUsage.usage?.inputTokens;
|
|
3916
|
-
if (typeof observed === "number" && observed > 0) {
|
|
3917
|
-
recordModelInputTokens(observability, streamProvider, observed);
|
|
3918
|
-
lastInputTokensObserved = observed;
|
|
3919
|
-
}
|
|
3920
|
-
// Prompt-cache efficiency for this response — same usage frame as the
|
|
3921
|
-
// input-token accounting above, so the two are always consistent.
|
|
3922
|
-
recordModelCacheTokens(observability, streamProvider, {
|
|
3923
|
-
cachedTokens: modelCallUsageTelemetry(responseUsage.usage).cachedTokens,
|
|
3924
|
-
promptTokens: responseUsage.usage?.inputTokens,
|
|
3925
|
-
});
|
|
3926
|
-
},
|
|
3927
|
-
recordAttemptSignals: async () => {
|
|
3928
|
-
const observed = responseUsage.usage?.inputTokens;
|
|
3929
|
-
if (typeof observed === "number" && observed > 0) {
|
|
3930
|
-
await setLastInputTokensFenced(observed);
|
|
3931
|
-
}
|
|
3932
|
-
},
|
|
3933
|
-
});
|
|
5270
|
+
let stableToolCallIdsToClear: string[] | null = null;
|
|
5271
|
+
let completedCurrentToolBatch = false;
|
|
5272
|
+
const responseUsageResult = await processModelResponseUsageEvent({
|
|
5273
|
+
event: next.value,
|
|
5274
|
+
state: modelResponseUsageState,
|
|
5275
|
+
dispatchId: modelUsageDispatchId,
|
|
5276
|
+
settings,
|
|
5277
|
+
db,
|
|
5278
|
+
observability,
|
|
5279
|
+
publish,
|
|
5280
|
+
accountId: input.accountId,
|
|
5281
|
+
workspaceId: input.workspaceId,
|
|
5282
|
+
sessionId: input.sessionId,
|
|
5283
|
+
turnId: activeTurnId,
|
|
5284
|
+
turnAttemptId: input.attemptId,
|
|
5285
|
+
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
5286
|
+
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
5287
|
+
model: turn.model,
|
|
5288
|
+
metricProvider: streamProvider,
|
|
5289
|
+
externallyBilled: isExternallyBilledTurn,
|
|
5290
|
+
servingCredentialId: effectiveCodexCredentialId,
|
|
5291
|
+
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
5292
|
+
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
5293
|
+
renewLease: () => renewCodexLease("model_usage"),
|
|
5294
|
+
leaseLost: () => codexLeaseLost,
|
|
5295
|
+
leaseLostMessage: "Codex credential lease expired during the active turn",
|
|
5296
|
+
setLastInputTokens: setLastInputTokensFenced,
|
|
5297
|
+
});
|
|
5298
|
+
if (responseUsageResult.status === "processed") {
|
|
5299
|
+
currentToolBatchCallIds = new Set<string>();
|
|
5300
|
+
currentToolBatchCompletedCallIds = new Set<string>();
|
|
3934
5301
|
await reconcileConversationTruth();
|
|
3935
5302
|
try {
|
|
3936
5303
|
await ensureRunAllowed(
|
|
@@ -3938,7 +5305,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3938
5305
|
db,
|
|
3939
5306
|
input.accountId,
|
|
3940
5307
|
input.workspaceId,
|
|
3941
|
-
|
|
5308
|
+
isExternallyBilledTurn,
|
|
3942
5309
|
entitlements,
|
|
3943
5310
|
);
|
|
3944
5311
|
} catch (limitError) {
|
|
@@ -3973,6 +5340,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3973
5340
|
"turn attempt ended while recording an in-flight tool call",
|
|
3974
5341
|
);
|
|
3975
5342
|
}
|
|
5343
|
+
currentToolBatchCallIds.add(pendingToolCall.callId);
|
|
3976
5344
|
}
|
|
3977
5345
|
const completedToolCall = completedToolCallFromSdkEvent(next.value);
|
|
3978
5346
|
if (completedToolCall) {
|
|
@@ -3996,25 +5364,26 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3996
5364
|
"turn attempt ended while recording a tool-call result",
|
|
3997
5365
|
);
|
|
3998
5366
|
}
|
|
3999
|
-
|
|
4000
|
-
|
|
4001
|
-
|
|
4002
|
-
|
|
4003
|
-
|
|
5367
|
+
const belongsToCurrentBatch = currentToolBatchCallIds.has(completedToolCall.callId);
|
|
5368
|
+
if (belongsToCurrentBatch) {
|
|
5369
|
+
currentToolBatchCompletedCallIds.add(completedToolCall.callId);
|
|
5370
|
+
}
|
|
5371
|
+
const currentBatchIsStable =
|
|
5372
|
+
belongsToCurrentBatch &&
|
|
5373
|
+
currentToolBatchCallIds.size > 0 &&
|
|
5374
|
+
currentToolBatchCompletedCallIds.size === currentToolBatchCallIds.size;
|
|
5375
|
+
const standaloneStableResult =
|
|
5376
|
+
!belongsToCurrentBatch && currentToolBatchCallIds.size === 0;
|
|
5377
|
+
if (currentBatchIsStable || standaloneStableResult) {
|
|
5378
|
+
// Persist the SDK's now-stable complete call/result batch. Keep
|
|
5379
|
+
// the receipts until the normalized tool-output event below is
|
|
5380
|
+
// durably flushed: recovery then covers every crash boundary
|
|
5381
|
+
// without either losing or duplicating the UI projection.
|
|
4004
5382
|
await reconcileConversationTruth({ requireDurable: true });
|
|
4005
|
-
|
|
4006
|
-
|
|
4007
|
-
|
|
4008
|
-
|
|
4009
|
-
turnId: activeTurnId,
|
|
4010
|
-
executionGeneration,
|
|
4011
|
-
attemptId: input.attemptId,
|
|
4012
|
-
});
|
|
4013
|
-
if (!cleared.accepted) {
|
|
4014
|
-
throw new TurnAttemptFencedError(
|
|
4015
|
-
"turn attempt ended while finalizing tool-call results",
|
|
4016
|
-
);
|
|
4017
|
-
}
|
|
5383
|
+
stableToolCallIdsToClear = currentBatchIsStable
|
|
5384
|
+
? [...currentToolBatchCallIds]
|
|
5385
|
+
: [completedToolCall.callId];
|
|
5386
|
+
completedCurrentToolBatch = currentBatchIsStable;
|
|
4018
5387
|
}
|
|
4019
5388
|
}
|
|
4020
5389
|
const normalized = normalizeSdkEvent(next.value);
|
|
@@ -4022,6 +5391,26 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4022
5391
|
streamTiming.onEvent(event.type);
|
|
4023
5392
|
await batcher.push(event);
|
|
4024
5393
|
}
|
|
5394
|
+
if (stableToolCallIdsToClear) {
|
|
5395
|
+
const cleared = await clearDurablePendingSessionToolCalls(db, {
|
|
5396
|
+
accountId: input.accountId,
|
|
5397
|
+
workspaceId: input.workspaceId,
|
|
5398
|
+
sessionId: input.sessionId,
|
|
5399
|
+
turnId: activeTurnId,
|
|
5400
|
+
executionGeneration,
|
|
5401
|
+
attemptId: input.attemptId,
|
|
5402
|
+
callIds: stableToolCallIdsToClear,
|
|
5403
|
+
});
|
|
5404
|
+
if (!cleared.accepted) {
|
|
5405
|
+
throw new TurnAttemptFencedError(
|
|
5406
|
+
"turn attempt ended while finalizing tool-call results",
|
|
5407
|
+
);
|
|
5408
|
+
}
|
|
5409
|
+
if (completedCurrentToolBatch) {
|
|
5410
|
+
currentToolBatchCallIds = new Set<string>();
|
|
5411
|
+
currentToolBatchCompletedCallIds = new Set<string>();
|
|
5412
|
+
}
|
|
5413
|
+
}
|
|
4025
5414
|
}
|
|
4026
5415
|
} finally {
|
|
4027
5416
|
if (!streamDone) {
|
|
@@ -4036,95 +5425,139 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4036
5425
|
void iterator.return?.().catch(() => undefined);
|
|
4037
5426
|
}
|
|
4038
5427
|
}
|
|
4039
|
-
await
|
|
4040
|
-
|
|
4041
|
-
|
|
5428
|
+
await waitForTurnStreamCleanup(
|
|
5429
|
+
batcher.flush(),
|
|
5430
|
+
stream.completed.catch(() => undefined),
|
|
5431
|
+
cancellationSignal,
|
|
5432
|
+
);
|
|
5433
|
+
if (modelResponseUsageState.responseUsageCount === 0) {
|
|
4042
5434
|
const aggregateUsage = stream.state.usage;
|
|
4043
|
-
const
|
|
4044
|
-
|
|
4045
|
-
if (typeof aggregateInput === "number" && aggregateInput > 0) {
|
|
4046
|
-
lastInputTokensObserved = aggregateInput;
|
|
4047
|
-
}
|
|
5435
|
+
const normalizedAggregateUsage = normalizeModelCallUsage(aggregateUsage);
|
|
5436
|
+
const aggregateInput = normalizedAggregateUsage.telemetry.inputTokens;
|
|
4048
5437
|
const aggregateSourceKey = modelUsageSourceKey({
|
|
4049
5438
|
responseId: null,
|
|
4050
5439
|
dispatchId: modelUsageDispatchId,
|
|
4051
5440
|
positionalKey: "aggregate",
|
|
4052
5441
|
});
|
|
4053
|
-
|
|
4054
|
-
|
|
4055
|
-
|
|
4056
|
-
|
|
4057
|
-
|
|
4058
|
-
|
|
4059
|
-
|
|
4060
|
-
|
|
4061
|
-
|
|
4062
|
-
|
|
4063
|
-
|
|
4064
|
-
|
|
4065
|
-
|
|
4066
|
-
|
|
4067
|
-
|
|
4068
|
-
|
|
4069
|
-
|
|
4070
|
-
|
|
4071
|
-
|
|
4072
|
-
|
|
4073
|
-
|
|
4074
|
-
|
|
4075
|
-
|
|
4076
|
-
|
|
4077
|
-
|
|
4078
|
-
|
|
4079
|
-
|
|
4080
|
-
|
|
4081
|
-
|
|
4082
|
-
|
|
4083
|
-
|
|
4084
|
-
|
|
4085
|
-
|
|
4086
|
-
|
|
4087
|
-
|
|
4088
|
-
|
|
4089
|
-
|
|
4090
|
-
|
|
4091
|
-
|
|
4092
|
-
|
|
4093
|
-
|
|
4094
|
-
|
|
4095
|
-
|
|
4096
|
-
|
|
4097
|
-
|
|
4098
|
-
|
|
4099
|
-
|
|
4100
|
-
|
|
4101
|
-
|
|
4102
|
-
|
|
4103
|
-
|
|
4104
|
-
|
|
4105
|
-
|
|
4106
|
-
|
|
5442
|
+
if (!claimedModelUsageSourceKeys.has(aggregateSourceKey)) {
|
|
5443
|
+
claimedModelUsageSourceKeys.add(aggregateSourceKey);
|
|
5444
|
+
// The single aggregate frame is this turn's only model-usage record, so
|
|
5445
|
+
// it is the first (account-switch surfaces here just like a first response).
|
|
5446
|
+
const aggregateAccountCtx = modelCallAccountContext({
|
|
5447
|
+
servingCredentialId: effectiveCodexCredentialId,
|
|
5448
|
+
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
5449
|
+
isFirstCallOfTurn: true,
|
|
5450
|
+
});
|
|
5451
|
+
let aggregateAuthoritative = false;
|
|
5452
|
+
await recordCompletedModelCallBeforeOwnershipFences({
|
|
5453
|
+
renewLease: () => renewCodexLease("model_usage"),
|
|
5454
|
+
leaseLost: () => codexLeaseLost,
|
|
5455
|
+
leaseLostMessage: "Codex credential lease expired during the active turn",
|
|
5456
|
+
recordUsage: async () => {
|
|
5457
|
+
await recordModelUsageAndDebitCredits(settings, db, {
|
|
5458
|
+
accountId: input.accountId,
|
|
5459
|
+
workspaceId: input.workspaceId,
|
|
5460
|
+
sessionId: input.sessionId,
|
|
5461
|
+
turnId: activeTurnId,
|
|
5462
|
+
turnAttemptId: input.attemptId,
|
|
5463
|
+
model: turn.model,
|
|
5464
|
+
externallyBilled: isExternallyBilledTurn,
|
|
5465
|
+
usage: aggregateUsage,
|
|
5466
|
+
normalizedUsage: normalizedAggregateUsage,
|
|
5467
|
+
sourceKey: aggregateSourceKey,
|
|
5468
|
+
observability,
|
|
5469
|
+
});
|
|
5470
|
+
aggregateAuthoritative = await emitModelCallUsage({
|
|
5471
|
+
observability,
|
|
5472
|
+
publish,
|
|
5473
|
+
accountId: input.accountId,
|
|
5474
|
+
workspaceId: input.workspaceId,
|
|
5475
|
+
sessionId: input.sessionId,
|
|
5476
|
+
turnId: activeTurnId,
|
|
5477
|
+
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
5478
|
+
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
5479
|
+
model: turn.model,
|
|
5480
|
+
sourceKey: aggregateSourceKey,
|
|
5481
|
+
usage: { usage: aggregateUsage },
|
|
5482
|
+
normalizedUsage: normalizedAggregateUsage,
|
|
5483
|
+
servingAccountHash: aggregateAccountCtx.servingAccountHash,
|
|
5484
|
+
accountChangedFromPrevCall: aggregateAccountCtx.accountChangedFromPrevCall,
|
|
5485
|
+
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
5486
|
+
});
|
|
5487
|
+
if (aggregateAuthoritative && aggregateInput !== null && aggregateInput > 0) {
|
|
5488
|
+
recordModelInputTokens(observability, streamProvider, aggregateInput);
|
|
5489
|
+
}
|
|
5490
|
+
},
|
|
5491
|
+
recordAttemptSignals: async () => {
|
|
5492
|
+
if (!aggregateAuthoritative) return;
|
|
5493
|
+
if (normalizedAggregateUsage.totalTokens !== null) {
|
|
5494
|
+
modelResponseUsageState.lastProviderContextTokensObserved =
|
|
5495
|
+
normalizedAggregateUsage.totalTokens;
|
|
5496
|
+
modelResponseUsageState.providerContextRevision += 1;
|
|
5497
|
+
}
|
|
5498
|
+
if (aggregateInput !== null && aggregateInput > 0) {
|
|
5499
|
+
await setLastInputTokensFenced(aggregateInput);
|
|
5500
|
+
}
|
|
5501
|
+
},
|
|
5502
|
+
});
|
|
5503
|
+
}
|
|
4107
5504
|
}
|
|
4108
5505
|
if (stream.interruptions.length > 0) {
|
|
4109
5506
|
await reconcileConversationTruth();
|
|
4110
5507
|
const approvals = runtime.serializeApprovals(stream.interruptions);
|
|
4111
|
-
|
|
4112
|
-
|
|
4113
|
-
|
|
4114
|
-
|
|
4115
|
-
|
|
4116
|
-
|
|
4117
|
-
|
|
4118
|
-
|
|
4119
|
-
|
|
4120
|
-
|
|
4121
|
-
|
|
4122
|
-
|
|
4123
|
-
|
|
5508
|
+
const humanInputInterruptions =
|
|
5509
|
+
runtime.serializeHumanInputRequests?.(stream.interruptions) ?? [];
|
|
5510
|
+
const humanInputRequests = await Promise.all(
|
|
5511
|
+
humanInputInterruptions.map(async (interruption) => {
|
|
5512
|
+
const id = stableHumanInputRequestId(
|
|
5513
|
+
input.sessionId,
|
|
5514
|
+
activeTurnId,
|
|
5515
|
+
interruption.toolCallId,
|
|
5516
|
+
);
|
|
5517
|
+
const existing = await getSessionHumanInputRequest(
|
|
5518
|
+
db,
|
|
5519
|
+
input.workspaceId,
|
|
5520
|
+
input.sessionId,
|
|
5521
|
+
id,
|
|
5522
|
+
);
|
|
5523
|
+
if (existing && existing.status !== "pending") {
|
|
5524
|
+
throw new Error(`Settled human-input request ${id} reappeared as an interruption`);
|
|
5525
|
+
}
|
|
5526
|
+
const expiresAt = existing?.expiresAt
|
|
5527
|
+
? new Date(existing.expiresAt)
|
|
5528
|
+
: interruption.input.expiresInSeconds
|
|
5529
|
+
? new Date(Date.now() + interruption.input.expiresInSeconds * 1000)
|
|
5530
|
+
: null;
|
|
5531
|
+
return {
|
|
5532
|
+
id,
|
|
5533
|
+
toolCallId: interruption.toolCallId,
|
|
5534
|
+
questions: interruption.input.questions,
|
|
5535
|
+
allowSkip: interruption.input.allowSkip,
|
|
5536
|
+
expiresAt,
|
|
5537
|
+
isNew: existing === null,
|
|
5538
|
+
};
|
|
5539
|
+
}),
|
|
5540
|
+
);
|
|
5541
|
+
const requestEvents = humanInputRequests
|
|
5542
|
+
.filter((request) => request.isNew)
|
|
5543
|
+
.map((request) => ({
|
|
5544
|
+
type: "session.humanInput.requested" as const,
|
|
5545
|
+
payload: {
|
|
5546
|
+
request: {
|
|
5547
|
+
id: request.id,
|
|
5548
|
+
questions: request.questions,
|
|
5549
|
+
allowSkip: request.allowSkip,
|
|
5550
|
+
expiresAt: request.expiresAt?.toISOString() ?? null,
|
|
5551
|
+
},
|
|
5552
|
+
},
|
|
5553
|
+
}));
|
|
4124
5554
|
if (
|
|
4125
5555
|
!(await settle!({
|
|
4126
5556
|
events: [
|
|
4127
|
-
|
|
5557
|
+
...requestEvents,
|
|
5558
|
+
...(approvals.length > 0
|
|
5559
|
+
? [{ type: "session.requiresAction" as const, payload: { approvals } }]
|
|
5560
|
+
: []),
|
|
4128
5561
|
{
|
|
4129
5562
|
type: "session.status.changed",
|
|
4130
5563
|
payload: { status: "requires_action" },
|
|
@@ -4133,6 +5566,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4133
5566
|
turnStatus: "requires_action",
|
|
4134
5567
|
sessionStatus: "requires_action",
|
|
4135
5568
|
activeTurnId,
|
|
5569
|
+
runState: {
|
|
5570
|
+
serializedRunState: stream.state.toString(),
|
|
5571
|
+
pendingApprovals: approvals,
|
|
5572
|
+
frozenCodexCredentialId: effectiveCodexCredentialId,
|
|
5573
|
+
humanInputRequests: humanInputRequests.map(
|
|
5574
|
+
({ isNew: _isNew, ...request }) => request,
|
|
5575
|
+
),
|
|
5576
|
+
},
|
|
4136
5577
|
}))
|
|
4137
5578
|
) {
|
|
4138
5579
|
return claimedResult({ status: "cancelled" });
|
|
@@ -4182,6 +5623,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4182
5623
|
unit: "run",
|
|
4183
5624
|
sourceResourceType: "session_turn",
|
|
4184
5625
|
sourceResourceId: activeTurnId,
|
|
5626
|
+
sessionId: input.sessionId,
|
|
5627
|
+
turnId: activeTurnId,
|
|
5628
|
+
turnAttemptId: input.attemptId,
|
|
4185
5629
|
idempotencyKey: `usage:agent_run.completed:${activeTurnId}`,
|
|
4186
5630
|
});
|
|
4187
5631
|
activityStatus = "idle";
|
|
@@ -4242,9 +5686,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4242
5686
|
}
|
|
4243
5687
|
}
|
|
4244
5688
|
} catch (compactError) {
|
|
4245
|
-
|
|
4246
|
-
|
|
4247
|
-
|
|
5689
|
+
// Transient checkpoint-provider failures recover this same accepted
|
|
5690
|
+
// turn through the normal provider/capacity path. They are not an
|
|
5691
|
+
// empty summary and must not create a new goal continuation.
|
|
5692
|
+
if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
|
|
5693
|
+
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
5694
|
+
compactionFailureMessage = compactionFailureReasonFromError(compactError);
|
|
4248
5695
|
observability.warn("context compaction recovery compaction failed", {
|
|
4249
5696
|
sessionId: input.sessionId,
|
|
4250
5697
|
turnId: activeTurnId,
|
|
@@ -4276,6 +5723,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4276
5723
|
turnStatus: "failed",
|
|
4277
5724
|
sessionStatus: "idle",
|
|
4278
5725
|
activeTurnId: null,
|
|
5726
|
+
...(recoveryKind === "operator" ? { consumeRequestedCompactionFailure: true } : {}),
|
|
4279
5727
|
}))
|
|
4280
5728
|
) {
|
|
4281
5729
|
return claimedResult({ status: "cancelled" });
|
|
@@ -4283,7 +5731,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4283
5731
|
turnMetricOutcome = "failed";
|
|
4284
5732
|
activityStatus = "idle";
|
|
4285
5733
|
activityError = attemptError;
|
|
4286
|
-
|
|
5734
|
+
// The failed turn settlement already defers ordinary internal
|
|
5735
|
+
// updates and makes the delivered goal-continuation receipt
|
|
5736
|
+
// terminal. End this workflow run as well: returning plain idle
|
|
5737
|
+
// would immediately synthesize another goal continuation against
|
|
5738
|
+
// the unchanged active history and repeat the same failed
|
|
5739
|
+
// compaction. A new human/API prompt, Steer, or explicitly requested
|
|
5740
|
+
// Compact remains a durable explicit wake and may retry; ordinary
|
|
5741
|
+
// machine updates stay pending for that actionable wake.
|
|
5742
|
+
return claimedResult({ status: "idle", deferredUntilWake: true });
|
|
4287
5743
|
}
|
|
4288
5744
|
// Codex parity: compaction remains inside the same logical turn and
|
|
4289
5745
|
// the same activity. Rebuild the model-visible history from the
|
|
@@ -4326,6 +5782,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4326
5782
|
reason: "sandbox_lease_superseded",
|
|
4327
5783
|
});
|
|
4328
5784
|
if (recovery.action === "stale") {
|
|
5785
|
+
acknowledgeLostAttemptOwnership();
|
|
4329
5786
|
activityStatus = "cancelled";
|
|
4330
5787
|
turnMetricOutcome = "cancelled";
|
|
4331
5788
|
return claimedResult({ status: "cancelled" });
|
|
@@ -4344,7 +5801,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4344
5801
|
throw recoveryError;
|
|
4345
5802
|
}
|
|
4346
5803
|
}
|
|
4347
|
-
|
|
5804
|
+
const cancellationFailure = turnOperationCancellationFailure(error);
|
|
5805
|
+
if (
|
|
5806
|
+
cancellationFailure &&
|
|
5807
|
+
isWorkerShutdownCancellation(cancellationFailure) &&
|
|
5808
|
+
recoveryTurnId
|
|
5809
|
+
) {
|
|
4348
5810
|
try {
|
|
4349
5811
|
await flushRuntimeBatcher();
|
|
4350
5812
|
await reconcileConversationTruth();
|
|
@@ -4363,6 +5825,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4363
5825
|
reason: "worker_shutdown",
|
|
4364
5826
|
});
|
|
4365
5827
|
if (recovery.action === "stale") {
|
|
5828
|
+
acknowledgeLostAttemptOwnership();
|
|
4366
5829
|
activityStatus = "cancelled";
|
|
4367
5830
|
turnMetricOutcome = "cancelled";
|
|
4368
5831
|
return claimedResult({ status: "cancelled" });
|
|
@@ -4387,26 +5850,35 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4387
5850
|
if (error instanceof TurnAttemptFencedError) {
|
|
4388
5851
|
activityStatus = "cancelled";
|
|
4389
5852
|
activityError = error;
|
|
4390
|
-
|
|
5853
|
+
acknowledgeQuiescence = true;
|
|
5854
|
+
noteCancellationRequested();
|
|
5855
|
+
await waitForTurnFinalizerStep(
|
|
5856
|
+
flushRuntimeBatcher(),
|
|
5857
|
+
turnFinalizerCancellationSignal(cancellationSignal, activityStatus),
|
|
5858
|
+
);
|
|
4391
5859
|
// Ownership already moved to a newer attempt or an authoritative
|
|
4392
|
-
// control transaction. Surface
|
|
4393
|
-
// normal
|
|
4394
|
-
//
|
|
4395
|
-
//
|
|
4396
|
-
// server and leaves the worker task detached indefinitely.
|
|
5860
|
+
// control transaction. Surface the exact transport cancellation rather
|
|
5861
|
+
// than a normal result. Temporal terminalization remains diagnostic
|
|
5862
|
+
// only; replacement admission waits for the activity-owned durable
|
|
5863
|
+
// quiescence receipt written from the hard tool fence below.
|
|
4397
5864
|
turnMetricOutcome = "cancelled";
|
|
4398
5865
|
throw new CancelledFailure("TURN_ATTEMPT_FENCED", [], error);
|
|
4399
5866
|
}
|
|
4400
|
-
if (
|
|
5867
|
+
if (cancellationFailure) {
|
|
4401
5868
|
activityStatus = "cancelled";
|
|
4402
5869
|
activityError = error;
|
|
4403
|
-
|
|
5870
|
+
acknowledgeQuiescence = true;
|
|
5871
|
+
noteCancellationRequested();
|
|
5872
|
+
await waitForTurnFinalizerStep(
|
|
5873
|
+
flushRuntimeBatcher(),
|
|
5874
|
+
turnFinalizerCancellationSignal(cancellationSignal, activityStatus),
|
|
5875
|
+
);
|
|
4404
5876
|
// The workflow owns cancellation settlement: Pause/Steer controls use
|
|
4405
5877
|
// settleSessionControl, and heartbeat timeouts use worker-death
|
|
4406
5878
|
// recovery. A dying activity must never append a
|
|
4407
5879
|
// competing cancellation or mutate the turn/session on its own.
|
|
4408
5880
|
turnMetricOutcome = "cancelled";
|
|
4409
|
-
throw
|
|
5881
|
+
throw cancellationFailure;
|
|
4410
5882
|
}
|
|
4411
5883
|
// The SDK's per-segment turn cap is a pacing valve, not a failure: end
|
|
4412
5884
|
// the turn gracefully and idle the session so an active goal continues
|
|
@@ -4448,6 +5920,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4448
5920
|
unit: "run",
|
|
4449
5921
|
sourceResourceType: "session_turn",
|
|
4450
5922
|
sourceResourceId: turnId,
|
|
5923
|
+
sessionId: input.sessionId,
|
|
5924
|
+
turnId,
|
|
5925
|
+
turnAttemptId: input.attemptId,
|
|
4451
5926
|
idempotencyKey: `usage:agent_run.completed:${turnId}`,
|
|
4452
5927
|
});
|
|
4453
5928
|
activityStatus = "idle";
|
|
@@ -4613,12 +6088,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4613
6088
|
: {}),
|
|
4614
6089
|
}
|
|
4615
6090
|
: null;
|
|
4616
|
-
const cooldownUntil = codexCredentialCooldownUntil(
|
|
4617
|
-
codexCredentialFailure,
|
|
4618
|
-
serving,
|
|
4619
|
-
settings.codexRotationNearExhaustionPct,
|
|
4620
|
-
now,
|
|
4621
|
-
);
|
|
6091
|
+
const cooldownUntil = codexCredentialCooldownUntil(codexCredentialFailure, serving, now);
|
|
4622
6092
|
const statePersisted =
|
|
4623
6093
|
codexLeaseHolderId && codexLeaseGeneration !== null
|
|
4624
6094
|
? await quarantineCodexCredentialForLease(db, {
|
|
@@ -4663,7 +6133,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4663
6133
|
activeCredentialId: rotation.activeCredentialId,
|
|
4664
6134
|
priorCredentialId: effectiveCodexCredentialId,
|
|
4665
6135
|
accounts,
|
|
4666
|
-
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
4667
6136
|
now: new Date(),
|
|
4668
6137
|
usedConnectors: serving?.connectorNamespaces ?? [],
|
|
4669
6138
|
})
|
|
@@ -4725,9 +6194,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4725
6194
|
if (settlement.action === "stale") {
|
|
4726
6195
|
// One transaction proves both exact-holder recovery (including a
|
|
4727
6196
|
// just-expired or reaped lease row) and successor/control-gate
|
|
4728
|
-
// rejection.
|
|
4729
|
-
|
|
4730
|
-
|
|
6197
|
+
// rejection. Cross the hard tool fence so a control-gate loss can
|
|
6198
|
+
// write its quiescence receipt; a successor-only loss is a no-op.
|
|
6199
|
+
acknowledgeLostAttemptOwnership();
|
|
6200
|
+
activityStatus = "cancelled";
|
|
6201
|
+
turnMetricOutcome = "cancelled";
|
|
4731
6202
|
return claimedResult({ status: "recovering" });
|
|
4732
6203
|
}
|
|
4733
6204
|
}
|
|
@@ -4784,7 +6255,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4784
6255
|
const until = codexCredentialCooldownUntil(
|
|
4785
6256
|
{ kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds },
|
|
4786
6257
|
serving,
|
|
4787
|
-
settings.codexRotationNearExhaustionPct,
|
|
4788
6258
|
new Date(),
|
|
4789
6259
|
)!;
|
|
4790
6260
|
// Finding 1a: INSPECT the cooldown-write result. A swallowed best-effort
|
|
@@ -4822,7 +6292,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4822
6292
|
const newHome = shardCredentialForSession({
|
|
4823
6293
|
sessionId: input.sessionId,
|
|
4824
6294
|
accounts: fresh,
|
|
4825
|
-
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
4826
6295
|
now: new Date(),
|
|
4827
6296
|
});
|
|
4828
6297
|
if (newHome) {
|
|
@@ -4871,16 +6340,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4871
6340
|
} else {
|
|
4872
6341
|
// Every account capped/cooling → idle until the earliest reset across all.
|
|
4873
6342
|
rotated = true;
|
|
4874
|
-
allCappedResetAt = earliestCodexReset(
|
|
4875
|
-
|
|
4876
|
-
settings.codexRotationNearExhaustionPct,
|
|
4877
|
-
new Date(),
|
|
4878
|
-
);
|
|
4879
|
-
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(
|
|
4880
|
-
fresh,
|
|
4881
|
-
settings.codexRotationNearExhaustionPct,
|
|
4882
|
-
new Date(),
|
|
4883
|
-
);
|
|
6343
|
+
allCappedResetAt = earliestCodexReset(fresh, new Date());
|
|
6344
|
+
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
|
|
4884
6345
|
}
|
|
4885
6346
|
} else {
|
|
4886
6347
|
const decision = chooseRotationActive({
|
|
@@ -4888,7 +6349,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4888
6349
|
activeCredentialId: rotation.activeCredentialId,
|
|
4889
6350
|
priorCredentialId: effectiveCodexCredentialId,
|
|
4890
6351
|
accounts: fresh,
|
|
4891
|
-
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
4892
6352
|
now: new Date(),
|
|
4893
6353
|
// P4: the just-capped serving account's connector set is the proxy for
|
|
4894
6354
|
// "what this session has access to" — prefer a covering failover target.
|
|
@@ -4915,11 +6375,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4915
6375
|
} else if (decision.kind === "allCapped") {
|
|
4916
6376
|
rotated = true;
|
|
4917
6377
|
allCappedResetAt = decision.earliestResetAt;
|
|
4918
|
-
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(
|
|
4919
|
-
fresh,
|
|
4920
|
-
settings.codexRotationNearExhaustionPct,
|
|
4921
|
-
new Date(),
|
|
4922
|
-
);
|
|
6378
|
+
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
|
|
4923
6379
|
}
|
|
4924
6380
|
// kind:"none" → fall through to today's single-account idle.
|
|
4925
6381
|
}
|
|
@@ -5094,6 +6550,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5094
6550
|
unit: "run",
|
|
5095
6551
|
sourceResourceType: "session_turn",
|
|
5096
6552
|
sourceResourceId: turnId,
|
|
6553
|
+
sessionId: input.sessionId,
|
|
6554
|
+
turnId,
|
|
6555
|
+
turnAttemptId: input.attemptId,
|
|
5097
6556
|
idempotencyKey: `usage:agent_run.completed:${turnId}`,
|
|
5098
6557
|
});
|
|
5099
6558
|
activityStatus = "idle";
|
|
@@ -5105,7 +6564,35 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5105
6564
|
// truth, recover this SAME accepted turn, then let the workflow re-claim
|
|
5106
6565
|
// it after a pacing delay. This is independent of goal state and never
|
|
5107
6566
|
// relies on a synthetic continuation prompt.
|
|
5108
|
-
const failure = agentRunFailurePayload(error);
|
|
6567
|
+
const failure = agentRunFailurePayload(error, { isCodexTurn });
|
|
6568
|
+
if (isSessionEventPersistenceError(error)) {
|
|
6569
|
+
// Never pass the original Drizzle/postgres-js error to telemetry: its
|
|
6570
|
+
// nested cause may contain raw SQL and bound parameters. The typed DB
|
|
6571
|
+
// boundary retains only SQLSTATE, stage, correlation, and safe catalog
|
|
6572
|
+
// identifiers. Provider inference has already happened and is NOT
|
|
6573
|
+
// retried by this terminal classification.
|
|
6574
|
+
observability.error("session event persistence failed", {
|
|
6575
|
+
accountId: input.accountId,
|
|
6576
|
+
workspaceId: input.workspaceId,
|
|
6577
|
+
sessionId: input.sessionId,
|
|
6578
|
+
turnId,
|
|
6579
|
+
attemptId: input.attemptId,
|
|
6580
|
+
code: error.details.code,
|
|
6581
|
+
sqlState: error.details.sqlState ?? "unknown",
|
|
6582
|
+
stage: error.details.stage,
|
|
6583
|
+
eventTypes: error.details.eventTypes.join(","),
|
|
6584
|
+
correlationId: error.details.correlationId,
|
|
6585
|
+
attempts: error.details.attempts,
|
|
6586
|
+
retryOutcome: error.details.retryOutcome,
|
|
6587
|
+
dbSeverity: error.details.database.severity,
|
|
6588
|
+
dbSchema: error.details.database.schema,
|
|
6589
|
+
dbTable: error.details.database.table,
|
|
6590
|
+
dbColumn: error.details.database.column,
|
|
6591
|
+
dbDataType: error.details.database.dataType,
|
|
6592
|
+
dbConstraint: error.details.database.constraint,
|
|
6593
|
+
dbRoutine: error.details.database.routine,
|
|
6594
|
+
});
|
|
6595
|
+
}
|
|
5109
6596
|
if (failure.retryable && publish && turnId && turnStartedPublished) {
|
|
5110
6597
|
const recoveryResult = providerRecoveryResult();
|
|
5111
6598
|
await flushRuntimeBatcher();
|
|
@@ -5122,6 +6609,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5122
6609
|
},
|
|
5123
6610
|
});
|
|
5124
6611
|
if (recovery.action === "stale") {
|
|
6612
|
+
acknowledgeLostAttemptOwnership();
|
|
5125
6613
|
activityStatus = "cancelled";
|
|
5126
6614
|
turnMetricOutcome = "cancelled";
|
|
5127
6615
|
return claimedResult({ status: "cancelled" });
|
|
@@ -5173,215 +6661,490 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5173
6661
|
);
|
|
5174
6662
|
return claimedResult({ status: "failed" });
|
|
5175
6663
|
} finally {
|
|
5176
|
-
|
|
5177
|
-
|
|
5178
|
-
|
|
5179
|
-
|
|
5180
|
-
const
|
|
5181
|
-
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
5185
|
-
|
|
5186
|
-
|
|
5187
|
-
|
|
5188
|
-
|
|
5189
|
-
|
|
5190
|
-
|
|
5191
|
-
|
|
5192
|
-
|
|
5193
|
-
|
|
5194
|
-
|
|
5195
|
-
|
|
5196
|
-
|
|
5197
|
-
|
|
5198
|
-
|
|
5199
|
-
|
|
5200
|
-
|
|
5201
|
-
|
|
5202
|
-
|
|
5203
|
-
|
|
5204
|
-
|
|
5205
|
-
|
|
5206
|
-
|
|
5207
|
-
|
|
5208
|
-
|
|
5209
|
-
|
|
5210
|
-
|
|
5211
|
-
|
|
5212
|
-
|
|
5213
|
-
|
|
5214
|
-
|
|
5215
|
-
|
|
5216
|
-
|
|
5217
|
-
|
|
5218
|
-
|
|
5219
|
-
|
|
5220
|
-
|
|
5221
|
-
|
|
5222
|
-
|
|
5223
|
-
|
|
5224
|
-
|
|
5225
|
-
|
|
5226
|
-
|
|
5227
|
-
|
|
5228
|
-
|
|
5229
|
-
|
|
5230
|
-
|
|
5231
|
-
|
|
5232
|
-
|
|
5233
|
-
|
|
5234
|
-
|
|
5235
|
-
|
|
5236
|
-
|
|
5237
|
-
|
|
5238
|
-
|
|
5239
|
-
|
|
6664
|
+
const finalizationStarted = performance.now();
|
|
6665
|
+
let finalizationError: unknown;
|
|
6666
|
+
let physicalToolQuiescenceConfirmed = !acknowledgeQuiescence;
|
|
6667
|
+
let quiescenceReceiptOrProofDurable = !acknowledgeQuiescence;
|
|
6668
|
+
const finalizerSignal = turnFinalizerCancellationSignal(cancellationSignal, activityStatus);
|
|
6669
|
+
try {
|
|
6670
|
+
const toolCancellationFence = toolCancellationFenceRef.current;
|
|
6671
|
+
// Toolspace and run-credential renewals are attempt-owned sandbox
|
|
6672
|
+
// writers. Capture and close them before crossing the same authoritative
|
|
6673
|
+
// boundary as tool calls; none of these drains may race the already-
|
|
6674
|
+
// aborted Temporal signal.
|
|
6675
|
+
toolspaceTokenRenewalClosed = true;
|
|
6676
|
+
const toolspaceRenewalToStop =
|
|
6677
|
+
toolspaceTokenRenewal as ToolspaceTokenRenewalController | null;
|
|
6678
|
+
toolspaceTokenRenewal = null;
|
|
6679
|
+
runCredentialRenewalClosed = true;
|
|
6680
|
+
const runRenewalToStop = runCredentialRenewal as RunCredentialRenewalController | null;
|
|
6681
|
+
runCredentialRenewal = null;
|
|
6682
|
+
await drainAttemptOwnedSandboxWriters({
|
|
6683
|
+
toolCancellationFence: acknowledgeQuiescence ? toolCancellationFence : null,
|
|
6684
|
+
cancellationReason: cancellationSignal?.reason,
|
|
6685
|
+
toolspaceTokenRenewal: toolspaceRenewalToStop,
|
|
6686
|
+
runCredentialRenewal: runRenewalToStop,
|
|
6687
|
+
});
|
|
6688
|
+
if (acknowledgeQuiescence) {
|
|
6689
|
+
// A cancellation before sandbox-backed capabilities exist still has
|
|
6690
|
+
// no tool controller to drain. Sandbox agent construction fails closed
|
|
6691
|
+
// when a backend exists but no controller was installed. Renewal
|
|
6692
|
+
// writers, when present, were drained above in either case.
|
|
6693
|
+
physicalToolQuiescenceConfirmed = true;
|
|
6694
|
+
}
|
|
6695
|
+
if (acknowledgeQuiescence && physicalToolQuiescenceConfirmed) {
|
|
6696
|
+
// This receipt is part of the hard cancellation boundary, not
|
|
6697
|
+
// housekeeping. Persist it immediately after the sandbox/tool fence
|
|
6698
|
+
// and before lease, cache, recording, or provider cleanup. Its
|
|
6699
|
+
// transaction also enqueues the exact workflow wake that will admit
|
|
6700
|
+
// the replacement; Temporal activity terminalization does neither.
|
|
6701
|
+
const proof: SessionAttemptQuiescenceProof = {
|
|
6702
|
+
accountId: input.accountId,
|
|
6703
|
+
workspaceId: input.workspaceId,
|
|
6704
|
+
sessionId: input.sessionId,
|
|
6705
|
+
attemptId: input.attemptId,
|
|
6706
|
+
workflowId: input.workflowId,
|
|
6707
|
+
workflowRunId: input.workflowRunId,
|
|
6708
|
+
activityId: dispatchId,
|
|
6709
|
+
};
|
|
6710
|
+
const recoveryMode = await persistOrSignalSessionAttemptQuiescence({
|
|
6711
|
+
proof,
|
|
6712
|
+
persistReceipt: async () =>
|
|
6713
|
+
await markSessionAttemptQuiesced(db, {
|
|
6714
|
+
accountId: input.accountId,
|
|
6715
|
+
workspaceId: input.workspaceId,
|
|
6716
|
+
sessionId: input.sessionId,
|
|
6717
|
+
attemptId: input.attemptId,
|
|
6718
|
+
temporalWorkflowId: input.workflowId,
|
|
6719
|
+
temporalWorkflowRunId: input.workflowRunId,
|
|
6720
|
+
temporalActivityId: dispatchId,
|
|
6721
|
+
allowUninterrupted: true,
|
|
6722
|
+
}),
|
|
6723
|
+
publishEvents: async (events) => {
|
|
6724
|
+
await waitForTurnFinalizerStep(
|
|
6725
|
+
publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events),
|
|
6726
|
+
finalizerSignal,
|
|
6727
|
+
);
|
|
6728
|
+
},
|
|
6729
|
+
signalProof: signalSessionAttemptQuiesced,
|
|
6730
|
+
heartbeat: (attempt, retryMs) => {
|
|
6731
|
+
activityContext?.heartbeat({
|
|
6732
|
+
phase: "quiescence-proof-delivery",
|
|
6733
|
+
sessionId: input.sessionId,
|
|
6734
|
+
attemptId: input.attemptId,
|
|
6735
|
+
deliveryAttempt: attempt,
|
|
6736
|
+
retryMs,
|
|
6737
|
+
at: new Date().toISOString(),
|
|
6738
|
+
});
|
|
6739
|
+
},
|
|
6740
|
+
onReceiptFailure: (error) => {
|
|
6741
|
+
console.error("agent turn quiescence receipt exhausted; signalling proof", error);
|
|
6742
|
+
},
|
|
6743
|
+
onPublishFailure: (error) => {
|
|
6744
|
+
console.error("agent turn quiescence event fanout failed", error);
|
|
6745
|
+
},
|
|
6746
|
+
onSignalFailure: (error, attempt, retryMs) => {
|
|
6747
|
+
console.error("agent turn quiescence proof signal failed; retrying", {
|
|
6748
|
+
error,
|
|
6749
|
+
attempt,
|
|
6750
|
+
retryMs,
|
|
6751
|
+
});
|
|
6752
|
+
},
|
|
6753
|
+
});
|
|
6754
|
+
quiescenceReceiptOrProofDurable = true;
|
|
6755
|
+
if (recoveryMode === "signal") {
|
|
6756
|
+
observability.info("agent turn quiescence proof handed to workflow recovery", {
|
|
6757
|
+
"opengeni.session_id": input.sessionId,
|
|
6758
|
+
"opengeni.attempt_id": input.attemptId,
|
|
6759
|
+
"opengeni.workflow_run_id": input.workflowRunId,
|
|
6760
|
+
"opengeni.activity_id": dispatchId,
|
|
6761
|
+
});
|
|
6762
|
+
}
|
|
6763
|
+
}
|
|
6764
|
+
// Deleting this attempt's credential generation is post-receipt
|
|
6765
|
+
// housekeeping: pointer mutation is serialized with successor
|
|
6766
|
+
// activation and the attempt-qualified check cannot erase a successor.
|
|
6767
|
+
// It may therefore detach on cancellation without weakening the writer
|
|
6768
|
+
// drain that gates the receipt above.
|
|
6769
|
+
const credentialSessionToClear = runCredentialSession;
|
|
6770
|
+
runCredentialSession = null;
|
|
6771
|
+
if (credentialSessionToClear) {
|
|
6772
|
+
const cleanup = clearRunCredentialsForAttempt(credentialSessionToClear, {
|
|
6773
|
+
sessionId: input.sessionId,
|
|
6774
|
+
attemptId: input.attemptId,
|
|
6775
|
+
executionGeneration,
|
|
6776
|
+
}).catch((error: unknown) => {
|
|
6777
|
+
try {
|
|
6778
|
+
observability.incrementCounter({
|
|
6779
|
+
name: "opengeni_run_credential_cleanup_total",
|
|
6780
|
+
help: "Attempt-owned run credential cleanup outcomes.",
|
|
6781
|
+
labels: { outcome: "error" },
|
|
6782
|
+
});
|
|
6783
|
+
observability.warn("Attempt-owned run credential cleanup failed", {
|
|
6784
|
+
sessionId: input.sessionId,
|
|
6785
|
+
turnId,
|
|
6786
|
+
attemptId: input.attemptId,
|
|
6787
|
+
errorClass: error instanceof Error ? error.name : "UnknownError",
|
|
6788
|
+
});
|
|
6789
|
+
} catch {
|
|
6790
|
+
// Cleanup observability must not own finalizer liveness.
|
|
6791
|
+
}
|
|
6792
|
+
});
|
|
6793
|
+
await waitForTurnFinalizerStep(cleanup, finalizerSignal);
|
|
6794
|
+
}
|
|
6795
|
+
gitCredentialRenewalClosed = true;
|
|
6796
|
+
const renewalsToStop = gitCredentialRenewals;
|
|
6797
|
+
gitCredentialRenewals = [];
|
|
6798
|
+
for (const renewal of renewalsToStop) {
|
|
6799
|
+
await waitForTurnFinalizerStep(renewal.stop(), finalizerSignal);
|
|
6800
|
+
}
|
|
6801
|
+
// Drain the buffered Connected Machine op events (infra failures + healed
|
|
6802
|
+
// recoveries) to durable session events — awaited, best-effort, never blocking
|
|
6803
|
+
// the turn. Sync observer → buffer → single awaited append here (no unawaited
|
|
6804
|
+
// DB write inside the activity). Scoped to this turn; skipped if no turnId
|
|
6805
|
+
// (the op ran under a turn, so on the normal path turnId is set).
|
|
6806
|
+
const machineOpEvents = machineOpObserver.drainEvents();
|
|
6807
|
+
if (machineOpEvents.length > 0 && turnId && executionGeneration > 0) {
|
|
6808
|
+
await waitForTurnFinalizerStep(
|
|
6809
|
+
appendAndPublishTurnEventsFenced(
|
|
6810
|
+
db,
|
|
6811
|
+
bus,
|
|
6812
|
+
input.workspaceId,
|
|
6813
|
+
input.sessionId,
|
|
6814
|
+
turnId,
|
|
6815
|
+
executionGeneration,
|
|
6816
|
+
input.attemptId,
|
|
6817
|
+
machineOpEvents.map((event) => ({
|
|
6818
|
+
...event,
|
|
6819
|
+
turnId: turnId ?? null,
|
|
6820
|
+
})),
|
|
6821
|
+
).catch(() => undefined),
|
|
6822
|
+
finalizerSignal,
|
|
6823
|
+
);
|
|
6824
|
+
}
|
|
6825
|
+
// Multi-account P4: flush the serving account's free per-turn caches ONCE,
|
|
6826
|
+
// best-effort (same discipline as today's usage write). Both writers skip
|
|
6827
|
+
// version/updatedAt, so neither can race the token-refresh CAS.
|
|
6828
|
+
if (effectiveCodexCredentialId) {
|
|
6829
|
+
// Part A: the latest scraped usage-header snapshot → the P2 usage cache. A
|
|
6830
|
+
// full both-windows snapshot (parseCodexUsageHeaders gates on both), so this
|
|
6831
|
+
// is byte-identical to the /wham/usage write — no partial-window clobber.
|
|
6832
|
+
if (latestCodexUsage) {
|
|
6833
|
+
const usageMutation = await waitForTurnFinalizerStep(
|
|
6834
|
+
recordCodexAccountUsageWithWakeTargets(
|
|
6835
|
+
db,
|
|
6836
|
+
input.workspaceId,
|
|
6837
|
+
effectiveCodexCredentialId,
|
|
6838
|
+
latestCodexUsage,
|
|
6839
|
+
).catch(() => null),
|
|
6840
|
+
finalizerSignal,
|
|
6841
|
+
);
|
|
6842
|
+
if (usageMutation) {
|
|
6843
|
+
await waitForTurnFinalizerStep(
|
|
6844
|
+
signalCodexCapacityWakeTargets(
|
|
6845
|
+
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
6846
|
+
usageMutation.wakeTargets,
|
|
6847
|
+
),
|
|
6848
|
+
finalizerSignal,
|
|
6849
|
+
);
|
|
6850
|
+
}
|
|
6851
|
+
}
|
|
6852
|
+
// Part B.1: the connector namespaces codex_apps listed this turn → the
|
|
6853
|
+
// connector-set cache. NON-EMPTY-only: a flaky/empty tools/list must never
|
|
6854
|
+
// overwrite a known set with [] (false coverage drop). Read by reference
|
|
6855
|
+
// AFTER the run, so every tools/list this turn has accumulated.
|
|
6856
|
+
const connectorNamespaces = preparedTools?.codexConnectorNamespaces;
|
|
6857
|
+
if (connectorNamespaces && connectorNamespaces.size > 0) {
|
|
6858
|
+
await waitForTurnFinalizerStep(
|
|
6859
|
+
recordCodexAccountConnectors(db, input.workspaceId, effectiveCodexCredentialId, [
|
|
6860
|
+
...connectorNamespaces,
|
|
6861
|
+
]).catch(() => undefined),
|
|
6862
|
+
finalizerSignal,
|
|
5240
6863
|
);
|
|
5241
6864
|
}
|
|
5242
6865
|
}
|
|
5243
|
-
|
|
5244
|
-
|
|
5245
|
-
|
|
5246
|
-
// AFTER the run, so every tools/list this turn has accumulated.
|
|
5247
|
-
const connectorNamespaces = preparedTools?.codexConnectorNamespaces;
|
|
5248
|
-
if (connectorNamespaces && connectorNamespaces.size > 0) {
|
|
5249
|
-
await recordCodexAccountConnectors(db, input.workspaceId, effectiveCodexCredentialId, [
|
|
5250
|
-
...connectorNamespaces,
|
|
5251
|
-
]).catch(() => undefined);
|
|
6866
|
+
if (codexLeaseHeartbeatTimer) {
|
|
6867
|
+
clearInterval(codexLeaseHeartbeatTimer);
|
|
6868
|
+
codexLeaseHeartbeatTimer = undefined;
|
|
5252
6869
|
}
|
|
5253
|
-
|
|
5254
|
-
|
|
5255
|
-
|
|
5256
|
-
|
|
5257
|
-
|
|
5258
|
-
|
|
5259
|
-
|
|
5260
|
-
|
|
5261
|
-
|
|
5262
|
-
|
|
5263
|
-
|
|
5264
|
-
|
|
5265
|
-
|
|
5266
|
-
).catch(() => undefined);
|
|
5267
|
-
codexLeaseHeld = false;
|
|
5268
|
-
}
|
|
5269
|
-
// Workbench v2 turn-end workspace capture (dossier §10.1) — runs FIRST in
|
|
5270
|
-
// the turn-end finally, while the box is MAXIMALLY ALIVE. The agent's last
|
|
5271
|
-
// tool ran before this finally, so /workspace is already final; capture is
|
|
5272
|
-
// FS-equivalent to the already-settled recording preparation and the warm
|
|
5273
|
-
// snapshot (neither mutates workspace files). Running it here — BEFORE
|
|
5274
|
-
// preparedTools.close() (which tears down tools / computer-use / the display
|
|
5275
|
-
// stack and is what starts the Modal box exiting a few seconds later) —
|
|
5276
|
-
// gives capture the full live-box margin instead of racing the teardown
|
|
5277
|
-
// tail, which was dropping 100% of captures on real Modal desktop boxes
|
|
5278
|
-
// ("request cancelled due to container exiting", 0 rows). External module:
|
|
5279
|
-
// self-capped at 60s, best-effort (never throws past its boundary),
|
|
5280
|
-
// epoch-fenced, and it NEVER closes the box. The emitted
|
|
5281
|
-
// workspace.revision.captured event is ANNOUNCE-ONLY (metadata, never
|
|
5282
|
-
// content).
|
|
5283
|
-
if (resolvedSandbox && setupBoxSession && sandboxGroupId) {
|
|
5284
|
-
// Stop new heartbeat snapshot/meter ticks so a mid-turn snapshot cannot
|
|
5285
|
-
// start concurrently with capture, then drain any in-flight snapshot
|
|
5286
|
-
// (bounded) — capture and the warm snapshot both exec on the box, so
|
|
5287
|
-
// sequence them, exactly as the turn-end snapshot placement did.
|
|
5288
|
-
if (leaseHeartbeatTimer) {
|
|
5289
|
-
clearInterval(leaseHeartbeatTimer);
|
|
5290
|
-
leaseHeartbeatTimer = undefined;
|
|
6870
|
+
if (codexLeaseHeld && turnId && codexLeaseHolderId && codexLeaseGeneration !== null) {
|
|
6871
|
+
await waitForTurnFinalizerStep(
|
|
6872
|
+
releaseCodexCredentialLease(
|
|
6873
|
+
db,
|
|
6874
|
+
input.accountId,
|
|
6875
|
+
input.workspaceId,
|
|
6876
|
+
turnId,
|
|
6877
|
+
codexLeaseHolderId,
|
|
6878
|
+
codexLeaseGeneration,
|
|
6879
|
+
).catch(() => undefined),
|
|
6880
|
+
finalizerSignal,
|
|
6881
|
+
);
|
|
6882
|
+
codexLeaseHeld = false;
|
|
5291
6883
|
}
|
|
5292
|
-
|
|
5293
|
-
|
|
6884
|
+
// Workbench v2 turn-end workspace capture — runs FIRST in
|
|
6885
|
+
// the turn-end finally, while the box is MAXIMALLY ALIVE. The agent's last
|
|
6886
|
+
// tool ran before this finally, so /workspace is already final; capture is
|
|
6887
|
+
// FS-equivalent to the already-settled recording preparation and the warm
|
|
6888
|
+
// snapshot (neither mutates workspace files). Running it here — BEFORE
|
|
6889
|
+
// preparedTools.close() (which tears down tools / computer-use / the display
|
|
6890
|
+
// stack and is what starts the Modal box exiting a few seconds later) —
|
|
6891
|
+
// gives capture the full live-box margin instead of racing the teardown
|
|
6892
|
+
// tail, which was dropping 100% of captures on real Modal desktop boxes
|
|
6893
|
+
// ("request cancelled due to container exiting", 0 rows). External module:
|
|
6894
|
+
// self-capped at 60s, best-effort (never throws past its boundary),
|
|
6895
|
+
// epoch-fenced, and it NEVER closes the box. The emitted
|
|
6896
|
+
// workspace.revision.captured event is ANNOUNCE-ONLY (metadata, never
|
|
6897
|
+
// content).
|
|
6898
|
+
if (process.env.OPENGENI_TEST_SCENARIO === "sandbox") {
|
|
6899
|
+
console.log(
|
|
6900
|
+
`[sandbox-e2e] capture preflight ownership=${settings.sandboxOwnershipEnabled} enabled=${settings.workspaceCaptureEnabled} resolved=${Boolean(resolvedSandbox)} session=${Boolean(setupBoxSession)} group=${Boolean(sandboxGroupId)} storage=${Boolean(objectStorage)}`,
|
|
6901
|
+
);
|
|
5294
6902
|
}
|
|
5295
|
-
|
|
5296
|
-
|
|
5297
|
-
|
|
5298
|
-
settings,
|
|
5299
|
-
publish: async (events) => {
|
|
5300
|
-
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events);
|
|
5301
|
-
},
|
|
5302
|
-
session: setupBoxSession as ChannelASession,
|
|
5303
|
-
leaseEpoch: resolvedSandbox.leaseEpoch,
|
|
5304
|
-
sandboxGroupId,
|
|
5305
|
-
accountId: input.accountId,
|
|
5306
|
-
workspaceId: input.workspaceId,
|
|
5307
|
-
sessionId: input.sessionId,
|
|
5308
|
-
turnId: turnId ?? null,
|
|
5309
|
-
observability,
|
|
6903
|
+
const runTurnEndPersistence = shouldRunTurnEndWorkspacePersistence({
|
|
6904
|
+
activityStatus,
|
|
6905
|
+
cancellationRequested: finalizerSignal?.aborted === true,
|
|
5310
6906
|
});
|
|
5311
|
-
|
|
5312
|
-
|
|
5313
|
-
|
|
5314
|
-
|
|
5315
|
-
|
|
5316
|
-
|
|
5317
|
-
|
|
5318
|
-
|
|
5319
|
-
|
|
5320
|
-
|
|
5321
|
-
|
|
5322
|
-
|
|
5323
|
-
|
|
5324
|
-
|
|
5325
|
-
|
|
5326
|
-
clearInterval(leaseHeartbeatTimer);
|
|
5327
|
-
}
|
|
5328
|
-
// A recording normally closes inside the attempt-fenced turn settlement.
|
|
5329
|
-
// Reaching finally with one still active means settlement threw, never ran,
|
|
5330
|
-
// or lost ownership. Stop ffmpeg and mark only this exact attempt-owned row
|
|
5331
|
-
// failed; publish no event and leave the artifact recoverable on the box.
|
|
5332
|
-
await abandonActiveRecording(
|
|
5333
|
-
"activity ended without recording settlement",
|
|
5334
|
-
didComputerUse ? "failed" : "discard",
|
|
5335
|
-
);
|
|
5336
|
-
if (resolvedSandbox) {
|
|
5337
|
-
// TURN-END mid-session snapshot (sandbox-file-persistence): fold the
|
|
5338
|
-
// turn's finished /workspace onto the lease before releasing the holder,
|
|
5339
|
-
// so the work this turn just produced survives any unclean box death in
|
|
5340
|
-
// the idle window ahead. Throttled by the same interval as the heartbeat
|
|
5341
|
-
// tick (a short turn right after a snapshot skips — bounded-loss contract
|
|
5342
|
-
// is the interval, not per-turn). Best-effort and time-capped by the
|
|
5343
|
-
// helper's own failure discipline; never delays release on failure.
|
|
5344
|
-
if (setupBoxSession && sandboxGroupId) {
|
|
5345
|
-
// Single-flight vs the heartbeat capture: the timer is already cleared
|
|
5346
|
-
// above, but a capture it launched may still be in flight — and that
|
|
5347
|
-
// capture predates the turn's final writes. Wait for it, but only up
|
|
5348
|
-
// to the snapshot timeout: release must never depend on an unbounded
|
|
5349
|
-
// provider capture.
|
|
6907
|
+
if (
|
|
6908
|
+
runTurnEndPersistence &&
|
|
6909
|
+
turnId &&
|
|
6910
|
+
resolvedSandbox &&
|
|
6911
|
+
setupBoxSession &&
|
|
6912
|
+
sandboxGroupId
|
|
6913
|
+
) {
|
|
6914
|
+
// Stop new heartbeat snapshot/meter ticks so a mid-turn snapshot cannot
|
|
6915
|
+
// start concurrently with capture, then drain any in-flight snapshot
|
|
6916
|
+
// (bounded) — capture and the warm snapshot both exec on the box, so
|
|
6917
|
+
// sequence them, exactly as the turn-end snapshot placement did.
|
|
6918
|
+
if (leaseHeartbeatTimer) {
|
|
6919
|
+
clearInterval(leaseHeartbeatTimer);
|
|
6920
|
+
leaseHeartbeatTimer = undefined;
|
|
6921
|
+
}
|
|
5350
6922
|
if (snapshotInFlight) {
|
|
5351
|
-
await waitForWarmSnapshot(
|
|
6923
|
+
await waitForWarmSnapshot(
|
|
6924
|
+
snapshotInFlight,
|
|
6925
|
+
settings.sandboxSnapshotTimeoutMs,
|
|
6926
|
+
finalizerSignal,
|
|
6927
|
+
);
|
|
5352
6928
|
}
|
|
5353
|
-
|
|
5354
|
-
|
|
5355
|
-
|
|
5356
|
-
|
|
5357
|
-
|
|
5358
|
-
|
|
6929
|
+
await captureWorkspaceRevision({
|
|
6930
|
+
db,
|
|
6931
|
+
objectStorage,
|
|
6932
|
+
settings,
|
|
6933
|
+
publish: async (events) => {
|
|
6934
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events);
|
|
5359
6935
|
},
|
|
5360
|
-
setupBoxSession,
|
|
5361
|
-
resolvedSandbox.leaseEpoch,
|
|
6936
|
+
session: setupBoxSession as ChannelASession,
|
|
6937
|
+
leaseEpoch: resolvedSandbox.leaseEpoch,
|
|
6938
|
+
sandboxGroupId,
|
|
6939
|
+
accountId: input.accountId,
|
|
6940
|
+
workspaceId: input.workspaceId,
|
|
6941
|
+
sessionId: input.sessionId,
|
|
6942
|
+
turnId,
|
|
6943
|
+
attemptId: input.attemptId,
|
|
6944
|
+
observability,
|
|
6945
|
+
...(finalizerSignal ? { signal: finalizerSignal } : {}),
|
|
6946
|
+
});
|
|
6947
|
+
}
|
|
6948
|
+
if (preparedTools) {
|
|
6949
|
+
await waitForTurnFinalizerStep(
|
|
6950
|
+
preparedTools.close().catch(() => undefined),
|
|
6951
|
+
finalizerSignal,
|
|
5362
6952
|
);
|
|
5363
|
-
|
|
5364
|
-
|
|
6953
|
+
}
|
|
6954
|
+
if (heartbeatTimer) {
|
|
6955
|
+
clearInterval(heartbeatTimer);
|
|
6956
|
+
}
|
|
6957
|
+
if (turnSandboxProvisioner?.hasStarted()) {
|
|
6958
|
+
await waitForTurnFinalizerStep(
|
|
6959
|
+
turnSandboxProvisioner.waitForSettled(30_000),
|
|
6960
|
+
finalizerSignal,
|
|
6961
|
+
);
|
|
6962
|
+
}
|
|
6963
|
+
// P1.2: stop the lease-TTL refresh, release the turn holder (idempotent
|
|
6964
|
+
// delete-my-row; refcount-- and warm->draining if it hit 0 with no turns),
|
|
6965
|
+
// and DROP the in-memory handle. Release NEVER stops the box — the reaper
|
|
6966
|
+
// (P1.3) issues the provider stop() past the drain grace at refcount 0; the
|
|
6967
|
+
// box rides the provider idle-timeout in the meantime. Best-effort: a
|
|
6968
|
+
// release failure must never mask the turn's real outcome.
|
|
6969
|
+
if (leaseHeartbeatTimer) {
|
|
6970
|
+
clearInterval(leaseHeartbeatTimer);
|
|
6971
|
+
}
|
|
6972
|
+
// A recording normally closes inside the attempt-fenced turn settlement.
|
|
6973
|
+
// Reaching finally with one still active means settlement threw, never ran,
|
|
6974
|
+
// or lost ownership. Stop ffmpeg and mark only this exact attempt-owned row
|
|
6975
|
+
// failed; publish no event and leave the artifact recoverable on the box.
|
|
6976
|
+
await waitForTurnFinalizerStep(
|
|
6977
|
+
abandonActiveRecording(
|
|
6978
|
+
"activity ended without recording settlement",
|
|
6979
|
+
didComputerUse ? "failed" : "discard",
|
|
6980
|
+
),
|
|
6981
|
+
finalizerSignal,
|
|
6982
|
+
);
|
|
6983
|
+
if (resolvedSandbox) {
|
|
6984
|
+
// TURN-END mid-session snapshot (sandbox-file-persistence): fold the
|
|
6985
|
+
// turn's finished /workspace onto the lease before releasing the holder,
|
|
6986
|
+
// so the work this turn just produced survives any unclean box death in
|
|
6987
|
+
// the idle window ahead. Throttled by the same interval as the heartbeat
|
|
6988
|
+
// tick (a short turn right after a snapshot skips — bounded-loss contract
|
|
6989
|
+
// is the interval, not per-turn). Best-effort and time-capped by the
|
|
6990
|
+
// helper's own failure discipline; never delays release on failure.
|
|
6991
|
+
const settledTurnId = turnId;
|
|
6992
|
+
if (runTurnEndPersistence && setupBoxSession && sandboxGroupId && settledTurnId) {
|
|
6993
|
+
// Single-flight vs the heartbeat capture: the timer is already cleared
|
|
6994
|
+
// above, but a capture it launched may still be in flight — and that
|
|
6995
|
+
// capture predates the turn's final writes. Wait for it, but only up
|
|
6996
|
+
// to the snapshot timeout: release must never depend on an unbounded
|
|
6997
|
+
// provider capture.
|
|
6998
|
+
if (snapshotInFlight) {
|
|
6999
|
+
await waitForWarmSnapshot(
|
|
7000
|
+
snapshotInFlight,
|
|
7001
|
+
settings.sandboxSnapshotTimeoutMs,
|
|
7002
|
+
finalizerSignal,
|
|
7003
|
+
);
|
|
7004
|
+
}
|
|
7005
|
+
const persisted = await maybePersistWarmWorkspaceSnapshot(
|
|
7006
|
+
{ db, settings },
|
|
5365
7007
|
{
|
|
5366
|
-
|
|
5367
|
-
|
|
7008
|
+
accountId: input.accountId,
|
|
7009
|
+
workspaceId: input.workspaceId,
|
|
7010
|
+
sessionId: input.sessionId,
|
|
7011
|
+
turnId: settledTurnId,
|
|
7012
|
+
attemptId: input.attemptId,
|
|
7013
|
+
sandboxGroupId,
|
|
5368
7014
|
},
|
|
5369
|
-
|
|
7015
|
+
setupBoxSession,
|
|
7016
|
+
resolvedSandbox.leaseEpoch,
|
|
7017
|
+
finalizerSignal,
|
|
7018
|
+
);
|
|
7019
|
+
if (persisted && publish) {
|
|
7020
|
+
await publish([
|
|
7021
|
+
{
|
|
7022
|
+
type: "sandbox.box.snapshot",
|
|
7023
|
+
payload: { trigger: "turn-end" },
|
|
7024
|
+
},
|
|
7025
|
+
]).catch(() => undefined);
|
|
7026
|
+
}
|
|
7027
|
+
// NB workspace capture no longer runs here — it moved to
|
|
7028
|
+
// the TOP of this finally (before preparedTools.close) so it completes
|
|
7029
|
+
// while the box is still solidly alive, instead of racing the turn-end
|
|
7030
|
+
// teardown that was killing 100% of captures on real Modal desktop boxes.
|
|
5370
7031
|
}
|
|
5371
|
-
|
|
5372
|
-
//
|
|
5373
|
-
|
|
5374
|
-
|
|
7032
|
+
const sandboxToRelease = resolvedSandbox;
|
|
7033
|
+
resolvedSandbox = null; // drop ownership now; the exact-holder release may finish later
|
|
7034
|
+
await waitForTurnFinalizerStep(
|
|
7035
|
+
sandboxToRelease.release().catch((releaseError) => {
|
|
7036
|
+
console.error("sandbox lease release failed (turn outcome unaffected)", releaseError);
|
|
7037
|
+
}),
|
|
7038
|
+
finalizerSignal,
|
|
7039
|
+
);
|
|
7040
|
+
}
|
|
7041
|
+
} catch (error) {
|
|
7042
|
+
finalizationError ??= error;
|
|
7043
|
+
console.error("agent turn finalization failed (turn outcome unaffected)", error);
|
|
7044
|
+
} finally {
|
|
7045
|
+
cancellationSignal?.removeEventListener("abort", noteCancellationRequested);
|
|
7046
|
+
const completedAt = performance.now();
|
|
7047
|
+
const durationSeconds = (completedAt - activityStarted) / 1000;
|
|
7048
|
+
const finalizationDurationSeconds = (completedAt - finalizationStarted) / 1000;
|
|
7049
|
+
observability.observeHistogram({
|
|
7050
|
+
name: "opengeni_turn_finalization_duration_seconds",
|
|
7051
|
+
help: "Agent turn finalization duration, including workspace housekeeping and lease release.",
|
|
7052
|
+
labels: {
|
|
7053
|
+
cancellation_requested: String(cancellationRequestedAt !== null),
|
|
7054
|
+
},
|
|
7055
|
+
value: finalizationDurationSeconds,
|
|
7056
|
+
});
|
|
7057
|
+
if (cancellationRequestedAt !== null) {
|
|
7058
|
+
const physicalCancellationDurationSeconds =
|
|
7059
|
+
(completedAt - cancellationRequestedAt) / 1000;
|
|
7060
|
+
observability.observeHistogram({
|
|
7061
|
+
name: "opengeni_turn_physical_cancellation_duration_seconds",
|
|
7062
|
+
help: "Time from Temporal cancellation delivery until the activity physically stops.",
|
|
7063
|
+
value: physicalCancellationDurationSeconds,
|
|
7064
|
+
});
|
|
7065
|
+
observability.info("agent turn physical cancellation completed", {
|
|
7066
|
+
"opengeni.session_id": input.sessionId,
|
|
7067
|
+
"opengeni.turn_id": turnId ?? "",
|
|
7068
|
+
"opengeni.attempt_id": input.attemptId,
|
|
7069
|
+
"opengeni.physical_cancellation_duration_ms": Math.round(
|
|
7070
|
+
physicalCancellationDurationSeconds * 1000,
|
|
7071
|
+
),
|
|
7072
|
+
});
|
|
7073
|
+
}
|
|
7074
|
+
observability.recordWorkerActivity({
|
|
7075
|
+
activity: "runAgentTurn",
|
|
7076
|
+
status: finalizationError ? "cleanup_failed" : activityStatus,
|
|
7077
|
+
durationSeconds,
|
|
7078
|
+
});
|
|
7079
|
+
if (turnId && activityStatus !== "unknown") {
|
|
7080
|
+
turnLifecycleMetricsFor(observability).finish(turnId, turnMetricOutcome, durationSeconds);
|
|
5375
7081
|
}
|
|
5376
|
-
|
|
5377
|
-
|
|
7082
|
+
activitySpan.end({
|
|
7083
|
+
attributes: {
|
|
7084
|
+
"opengeni.turn_id": turnId ?? "",
|
|
7085
|
+
"opengeni.status": activityStatus,
|
|
7086
|
+
"opengeni.variable_set_id": variableSetId,
|
|
7087
|
+
"opengeni.rig_id": rigId,
|
|
7088
|
+
"opengeni.rig_version_id": rigVersionId,
|
|
7089
|
+
"opengeni.codex_credential_id": effectiveCodexCredentialId ?? "",
|
|
7090
|
+
"opengeni.duration_ms": Math.round(durationSeconds * 1000),
|
|
7091
|
+
"opengeni.finalization_duration_ms": Math.round(finalizationDurationSeconds * 1000),
|
|
7092
|
+
},
|
|
7093
|
+
error: finalizationError ?? activityError,
|
|
7094
|
+
});
|
|
7095
|
+
assertPhysicalToolQuiescenceForCancellation({
|
|
7096
|
+
acknowledgeQuiescence,
|
|
7097
|
+
physicalToolQuiescenceConfirmed,
|
|
7098
|
+
failure: finalizationError,
|
|
7099
|
+
});
|
|
7100
|
+
assertSessionAttemptQuiescenceRecoveryDurable({
|
|
7101
|
+
acknowledgeQuiescence,
|
|
7102
|
+
physicalToolQuiescenceConfirmed,
|
|
7103
|
+
receiptOrProofDurable: quiescenceReceiptOrProofDurable,
|
|
7104
|
+
failure: finalizationError,
|
|
5378
7105
|
});
|
|
5379
|
-
resolvedSandbox = null; // drop the handle; the box survives the turn
|
|
5380
7106
|
}
|
|
5381
7107
|
}
|
|
5382
7108
|
};
|
|
5383
7109
|
}
|
|
5384
7110
|
|
|
7111
|
+
async function assertGitHubResourcesRemainAuthorized(
|
|
7112
|
+
db: Parameters<typeof areGitHubRepositoriesAllowedForWorkspace>[0],
|
|
7113
|
+
workspaceId: string,
|
|
7114
|
+
resources: import("@opengeni/contracts").ResourceRef[],
|
|
7115
|
+
): Promise<void> {
|
|
7116
|
+
// Must check exactly what sandboxEnvironmentForRun would mint a token for,
|
|
7117
|
+
// so the selection is derived from the same extraction as the mint path.
|
|
7118
|
+
for (const selection of gitHubTokenMintSelections(resources)) {
|
|
7119
|
+
await assertGitHubTokenMintSelectionAuthorized(
|
|
7120
|
+
db,
|
|
7121
|
+
workspaceId,
|
|
7122
|
+
selection.installationId,
|
|
7123
|
+
selection.repositoryIds,
|
|
7124
|
+
);
|
|
7125
|
+
}
|
|
7126
|
+
}
|
|
7127
|
+
|
|
7128
|
+
async function assertGitHubTokenMintSelectionAuthorized(
|
|
7129
|
+
db: Parameters<typeof areGitHubRepositoriesAllowedForWorkspace>[0],
|
|
7130
|
+
workspaceId: string,
|
|
7131
|
+
installationId: number,
|
|
7132
|
+
repositoryIds: number[],
|
|
7133
|
+
): Promise<void> {
|
|
7134
|
+
if (
|
|
7135
|
+
!(await areGitHubRepositoriesAllowedForWorkspace(
|
|
7136
|
+
db,
|
|
7137
|
+
workspaceId,
|
|
7138
|
+
installationId,
|
|
7139
|
+
repositoryIds,
|
|
7140
|
+
))
|
|
7141
|
+
) {
|
|
7142
|
+
throw new Error(
|
|
7143
|
+
"This workspace no longer authorizes one or more GitHub repositories attached to the session",
|
|
7144
|
+
);
|
|
7145
|
+
}
|
|
7146
|
+
}
|
|
7147
|
+
|
|
5385
7148
|
/**
|
|
5386
7149
|
* True when the error is transient upstream backpressure — a model-provider 5xx,
|
|
5387
7150
|
* a "server had a bad minute" body, or a dropped/again-able network connection —
|
|
@@ -5428,11 +7191,23 @@ export function isTransientProviderError(error: unknown): boolean {
|
|
|
5428
7191
|
);
|
|
5429
7192
|
}
|
|
5430
7193
|
|
|
5431
|
-
export function agentRunFailurePayload(
|
|
7194
|
+
export function agentRunFailurePayload(
|
|
7195
|
+
error: unknown,
|
|
7196
|
+
options: { isCodexTurn?: boolean } = {},
|
|
7197
|
+
): {
|
|
5432
7198
|
error: string;
|
|
5433
7199
|
code?: string;
|
|
5434
7200
|
retryable?: boolean;
|
|
5435
7201
|
detail?: string;
|
|
7202
|
+
timeoutClass?: string;
|
|
7203
|
+
responseObserved?: boolean;
|
|
7204
|
+
requestId?: string;
|
|
7205
|
+
correlationId?: string;
|
|
7206
|
+
stage?: string;
|
|
7207
|
+
sqlState?: string | null;
|
|
7208
|
+
attempts?: number;
|
|
7209
|
+
retryOutcome?: string;
|
|
7210
|
+
database?: Record<string, string>;
|
|
5436
7211
|
} {
|
|
5437
7212
|
const message = error instanceof Error ? error.message : String(error);
|
|
5438
7213
|
const status =
|
|
@@ -5443,6 +7218,41 @@ export function agentRunFailurePayload(error: unknown): {
|
|
|
5443
7218
|
typeof error === "object" && error !== null && "code" in error
|
|
5444
7219
|
? String((error as { code?: unknown }).code)
|
|
5445
7220
|
: undefined;
|
|
7221
|
+
// An accepted Codex stream with no terminal response is malformed/partial,
|
|
7222
|
+
// not provider backpressure. Replaying the same accepted turn could repeat
|
|
7223
|
+
// model or tool effects, so this marked transport failure must outrank the
|
|
7224
|
+
// generic 5xx retry classifier (CodexStreamingTerminalError uses status 502).
|
|
7225
|
+
if (isCodexTransportError(error) && code === "invalid_sse_terminal") {
|
|
7226
|
+
return {
|
|
7227
|
+
error: "The Codex response stream ended without a terminal response",
|
|
7228
|
+
code: "invalid_sse_terminal",
|
|
7229
|
+
retryable: false,
|
|
7230
|
+
};
|
|
7231
|
+
}
|
|
7232
|
+
if (isSessionEventPersistenceError(error)) {
|
|
7233
|
+
const { details } = error;
|
|
7234
|
+
const eventLabel = details.eventTypes.join(", ") || "session events";
|
|
7235
|
+
const failureLabel =
|
|
7236
|
+
details.code === "db_deadlock"
|
|
7237
|
+
? "Database deadlock"
|
|
7238
|
+
: details.code === "db_serialization_failure"
|
|
7239
|
+
? "Database serialization failure"
|
|
7240
|
+
: "Database failure";
|
|
7241
|
+
return {
|
|
7242
|
+
error: `${failureLabel} while persisting ${eventLabel}. The completed provider call and external effects were not retried.`,
|
|
7243
|
+
code: details.code,
|
|
7244
|
+
detail:
|
|
7245
|
+
details.retryOutcome === "exhausted"
|
|
7246
|
+
? `The idempotent persistence transaction failed after ${details.attempts} attempts.`
|
|
7247
|
+
: "The database rejected the idempotent persistence transaction.",
|
|
7248
|
+
correlationId: details.correlationId,
|
|
7249
|
+
stage: details.stage,
|
|
7250
|
+
sqlState: details.sqlState,
|
|
7251
|
+
attempts: details.attempts,
|
|
7252
|
+
retryOutcome: details.retryOutcome,
|
|
7253
|
+
...(Object.keys(details.database).length > 0 ? { database: details.database } : {}),
|
|
7254
|
+
};
|
|
7255
|
+
}
|
|
5446
7256
|
// A ChatGPT/Codex usage cap is a HARD limit, not transient backpressure: it
|
|
5447
7257
|
// must NOT be reported as a generic, retryable rate-limit (which would loop a
|
|
5448
7258
|
// goal against a capped backend). Surface a precise, actionable message with
|
|
@@ -5457,6 +7267,22 @@ export function agentRunFailurePayload(error: unknown): {
|
|
|
5457
7267
|
if (usageLimit) {
|
|
5458
7268
|
return codexUsageLimitFailurePayload(usageLimit, message);
|
|
5459
7269
|
}
|
|
7270
|
+
const codexTimeout = classifyCodexResponseTimeoutError(error, {
|
|
7271
|
+
allowLegacyRequestTimeout: options.isCodexTurn === true,
|
|
7272
|
+
});
|
|
7273
|
+
if (codexTimeout) {
|
|
7274
|
+
return {
|
|
7275
|
+
error: codexTimeout.responseObserved
|
|
7276
|
+
? "The Codex response timed out after streaming began. Observed output was checkpointed; automatic replay is disabled because the upstream operation may still be active."
|
|
7277
|
+
: "The Codex response timed out before any response was observed. Upstream acceptance is unknown, so automatic replay is disabled.",
|
|
7278
|
+
code: "codex_response_timeout",
|
|
7279
|
+
retryable: false,
|
|
7280
|
+
timeoutClass: codexTimeout.timeoutClass,
|
|
7281
|
+
responseObserved: codexTimeout.responseObserved,
|
|
7282
|
+
...(codexTimeout.requestId ? { requestId: codexTimeout.requestId } : {}),
|
|
7283
|
+
...(codexTimeout.message ? { detail: codexTimeout.message } : {}),
|
|
7284
|
+
};
|
|
7285
|
+
}
|
|
5460
7286
|
const mcpTimeout = classifyMcpTransportTimeoutError(error);
|
|
5461
7287
|
if (mcpTimeout) {
|
|
5462
7288
|
return {
|
|
@@ -5511,7 +7337,6 @@ export function codexCredentialCooldownUntil(
|
|
|
5511
7337
|
CodexAccountStatus,
|
|
5512
7338
|
"primaryUsedPercent" | "primaryResetAt" | "secondaryUsedPercent" | "secondaryResetAt"
|
|
5513
7339
|
> | null,
|
|
5514
|
-
nearExhaustionPct: number,
|
|
5515
7340
|
now: Date,
|
|
5516
7341
|
): Date | null {
|
|
5517
7342
|
if (failure.kind === "auth" || failure.kind === "forbidden") {
|
|
@@ -5533,7 +7358,7 @@ export function codexCredentialCooldownUntil(
|
|
|
5533
7358
|
]
|
|
5534
7359
|
.filter(
|
|
5535
7360
|
(window): window is { used: number; reset: Date } =>
|
|
5536
|
-
(window.used ?? 0) >=
|
|
7361
|
+
(window.used ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT &&
|
|
5537
7362
|
window.reset instanceof Date &&
|
|
5538
7363
|
window.reset.getTime() > now.getTime(),
|
|
5539
7364
|
)
|
|
@@ -5737,21 +7562,21 @@ class BudgetExhaustedError extends Error {
|
|
|
5737
7562
|
}
|
|
5738
7563
|
}
|
|
5739
7564
|
|
|
5740
|
-
// Exported for unit testing the
|
|
5741
|
-
// of the activity surface. Takes
|
|
5742
|
-
//
|
|
5743
|
-
//
|
|
7565
|
+
// Exported for unit testing the external-billing bypass (codex-billing.test.ts); not
|
|
7566
|
+
// part of the activity surface. Takes the accepted policy's billing attribution and
|
|
7567
|
+
// the optional §7.5 P3 host `entitlements` port (when bound, its `admitRun` REPLACES
|
|
7568
|
+
// the local credit read for an OpenGeni-metered turn; unset → local ledger).
|
|
5744
7569
|
export async function ensureRunAllowed(
|
|
5745
7570
|
settings: Settings,
|
|
5746
7571
|
db: ActivityServices["db"],
|
|
5747
7572
|
accountId: string,
|
|
5748
7573
|
workspaceId: string,
|
|
5749
|
-
|
|
7574
|
+
isExternallyBilledTurn: boolean,
|
|
5750
7575
|
entitlements?: ActivityServices["entitlements"],
|
|
5751
7576
|
): Promise<void> {
|
|
5752
|
-
//
|
|
5753
|
-
//
|
|
5754
|
-
//
|
|
7577
|
+
// Externally billed turns are paid outside OpenGeni: skip the credit-balance
|
|
7578
|
+
// gate and monthly token cap. The agent-run COUNT cap below is a
|
|
7579
|
+
// volume/fairness quota (not a credit/cost gate) and is intentionally kept.
|
|
5755
7580
|
//
|
|
5756
7581
|
// §7.5 P3 — host-entitlements DELEGATION (the worker half of the same seam the
|
|
5757
7582
|
// API edge exposes). For a non-codex turn, when the host binds `entitlements`, its
|
|
@@ -5765,7 +7590,7 @@ export async function ensureRunAllowed(
|
|
|
5765
7590
|
// idempotency-keyed writer at recordModelUsageAndDebitCredits), so a PULL host meter
|
|
5766
7591
|
// is consulted without ever double-charging.
|
|
5767
7592
|
if (
|
|
5768
|
-
!
|
|
7593
|
+
!isExternallyBilledTurn &&
|
|
5769
7594
|
entitlements &&
|
|
5770
7595
|
(settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")
|
|
5771
7596
|
) {
|
|
@@ -5779,7 +7604,7 @@ export async function ensureRunAllowed(
|
|
|
5779
7604
|
throw new Error(decision.reason || "insufficient OpenGeni credits");
|
|
5780
7605
|
}
|
|
5781
7606
|
} else if (
|
|
5782
|
-
!
|
|
7607
|
+
!isExternallyBilledTurn &&
|
|
5783
7608
|
(settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")
|
|
5784
7609
|
) {
|
|
5785
7610
|
const balance = await getBillingBalance(db, accountId);
|
|
@@ -5804,7 +7629,7 @@ export async function ensureRunAllowed(
|
|
|
5804
7629
|
);
|
|
5805
7630
|
}
|
|
5806
7631
|
}
|
|
5807
|
-
if (!
|
|
7632
|
+
if (!isExternallyBilledTurn && limits.maxMonthlyTokensPerWorkspace) {
|
|
5808
7633
|
const used = await sumUsageQuantity(db, {
|
|
5809
7634
|
workspaceId,
|
|
5810
7635
|
eventType: "model.tokens",
|
|
@@ -5817,7 +7642,7 @@ export async function ensureRunAllowed(
|
|
|
5817
7642
|
}
|
|
5818
7643
|
}
|
|
5819
7644
|
|
|
5820
|
-
// Exported for unit testing the
|
|
7645
|
+
// Exported for unit testing the external-billing bypass; not part of the activity surface.
|
|
5821
7646
|
export async function recordModelUsageAndDebitCredits(
|
|
5822
7647
|
settings: Settings,
|
|
5823
7648
|
db: ActivityServices["db"],
|
|
@@ -5826,9 +7651,11 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
5826
7651
|
workspaceId: string;
|
|
5827
7652
|
sessionId: string;
|
|
5828
7653
|
turnId: string;
|
|
7654
|
+
turnAttemptId: string;
|
|
5829
7655
|
model: string;
|
|
5830
|
-
|
|
5831
|
-
usage?: ModelUsageInput | null;
|
|
7656
|
+
externallyBilled: boolean;
|
|
7657
|
+
usage?: ModelUsageInput | ModelCallUsageInput | null;
|
|
7658
|
+
normalizedUsage?: ModelCallUsageNormalization;
|
|
5832
7659
|
sourceKey: string;
|
|
5833
7660
|
observability?: ActivityServices["observability"];
|
|
5834
7661
|
},
|
|
@@ -5836,10 +7663,12 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
5836
7663
|
if (!input.usage) {
|
|
5837
7664
|
return;
|
|
5838
7665
|
}
|
|
5839
|
-
const
|
|
5840
|
-
const
|
|
5841
|
-
const
|
|
5842
|
-
|
|
7666
|
+
const normalizedUsage = input.normalizedUsage ?? normalizeModelCallUsage(input.usage);
|
|
7667
|
+
const sanitizedUsage = sanitizedModelUsageInput(normalizedUsage);
|
|
7668
|
+
const inputTokens = sanitizedUsage.inputTokens ?? 0;
|
|
7669
|
+
const outputTokens = sanitizedUsage.outputTokens ?? 0;
|
|
7670
|
+
const totalTokens = sanitizedUsage.totalTokens ?? 0;
|
|
7671
|
+
// An externally billed turn is paid outside OpenGeni, so it
|
|
5843
7672
|
// consumes ZERO OpenGeni credits and must never feed an OpenGeni cap. A
|
|
5844
7673
|
// codex/<slug> model has no entry in configuredModelPricing, so the normal path
|
|
5845
7674
|
// below would throw "Missing model pricing". We:
|
|
@@ -5848,7 +7677,7 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
5848
7677
|
// any row would count against maxMonthlyTokensPerWorkspace);
|
|
5849
7678
|
// - record a `model.cost = 0` audit marker (harmless to the monthly cost cap);
|
|
5850
7679
|
// - never look up pricing and never debit credits.
|
|
5851
|
-
if (input.
|
|
7680
|
+
if (input.externallyBilled) {
|
|
5852
7681
|
await recordUsageEvent(db, {
|
|
5853
7682
|
accountId: input.accountId,
|
|
5854
7683
|
workspaceId: input.workspaceId,
|
|
@@ -5857,6 +7686,9 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
5857
7686
|
unit: "usd_micros",
|
|
5858
7687
|
sourceResourceType: "model_response",
|
|
5859
7688
|
sourceResourceId: `${input.turnId}:${input.sourceKey}`,
|
|
7689
|
+
sessionId: input.sessionId,
|
|
7690
|
+
turnId: input.turnId,
|
|
7691
|
+
turnAttemptId: input.turnAttemptId,
|
|
5860
7692
|
idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`,
|
|
5861
7693
|
});
|
|
5862
7694
|
return;
|
|
@@ -5870,6 +7702,9 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
5870
7702
|
unit: "tokens",
|
|
5871
7703
|
sourceResourceType: "model_response",
|
|
5872
7704
|
sourceResourceId: `${input.turnId}:${input.sourceKey}`,
|
|
7705
|
+
sessionId: input.sessionId,
|
|
7706
|
+
turnId: input.turnId,
|
|
7707
|
+
turnAttemptId: input.turnAttemptId,
|
|
5873
7708
|
idempotencyKey: `usage:model.tokens:${input.turnId}:${input.sourceKey}`,
|
|
5874
7709
|
});
|
|
5875
7710
|
}
|
|
@@ -5880,7 +7715,7 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
5880
7715
|
if (!configuredModelPricing(settings)[input.model]) {
|
|
5881
7716
|
throw new Error(`Missing model pricing for ${input.model}`);
|
|
5882
7717
|
}
|
|
5883
|
-
const costMicros = calculateModelUsageCostMicros(settings, input.model,
|
|
7718
|
+
const costMicros = calculateModelUsageCostMicros(settings, input.model, sanitizedUsage);
|
|
5884
7719
|
await recordUsageEvent(db, {
|
|
5885
7720
|
accountId: input.accountId,
|
|
5886
7721
|
workspaceId: input.workspaceId,
|
|
@@ -5889,6 +7724,9 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
5889
7724
|
unit: "usd_micros",
|
|
5890
7725
|
sourceResourceType: "model_response",
|
|
5891
7726
|
sourceResourceId: `${input.turnId}:${input.sourceKey}`,
|
|
7727
|
+
sessionId: input.sessionId,
|
|
7728
|
+
turnId: input.turnId,
|
|
7729
|
+
turnAttemptId: input.turnAttemptId,
|
|
5892
7730
|
idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`,
|
|
5893
7731
|
});
|
|
5894
7732
|
if (costMicros > 0) {
|
|
@@ -5911,18 +7749,26 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
5911
7749
|
// Additive: the prompt-cache slice of this call's input tokens, so the
|
|
5912
7750
|
// per-call debit record carries cache efficiency alongside the token
|
|
5913
7751
|
// counts. 0 when the provider did not report cached tokens.
|
|
5914
|
-
cachedTokens:
|
|
5915
|
-
modelCallUsageTelemetry(input.usage as Parameters<typeof modelCallUsageTelemetry>[0])
|
|
5916
|
-
.cachedTokens,
|
|
5917
|
-
),
|
|
7752
|
+
cachedTokens: normalizedUsage.telemetry.cachedTokens ?? 0,
|
|
5918
7753
|
},
|
|
5919
7754
|
});
|
|
5920
7755
|
recordCreditMicros(input.observability, "usage", result.debitedMicros);
|
|
5921
7756
|
}
|
|
5922
7757
|
}
|
|
5923
7758
|
|
|
5924
|
-
function
|
|
5925
|
-
return
|
|
7759
|
+
function sanitizedModelUsageInput(normalized: ModelCallUsageNormalization): ModelUsageInput {
|
|
7760
|
+
return {
|
|
7761
|
+
...(normalized.telemetry.inputTokens !== null
|
|
7762
|
+
? { inputTokens: normalized.telemetry.inputTokens }
|
|
7763
|
+
: {}),
|
|
7764
|
+
...(normalized.telemetry.outputTokens !== null
|
|
7765
|
+
? { outputTokens: normalized.telemetry.outputTokens }
|
|
7766
|
+
: {}),
|
|
7767
|
+
...(normalized.totalTokens !== null ? { totalTokens: normalized.totalTokens } : {}),
|
|
7768
|
+
...(normalized.telemetry.cachedTokens !== null
|
|
7769
|
+
? { inputTokensDetails: { cached_tokens: normalized.telemetry.cachedTokens } }
|
|
7770
|
+
: {}),
|
|
7771
|
+
};
|
|
5926
7772
|
}
|
|
5927
7773
|
|
|
5928
7774
|
function startOfUtcMonth(): Date {
|