@opengeni/worker-bundle 0.12.10 → 0.13.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +778 -0
- package/dist/activities/capabilities.d.ts +18 -0
- package/dist/activities/codex-auth.d.ts +1 -0
- package/dist/activities/codex-capacity.d.ts +31 -0
- package/dist/activities/codex-fleet-shadow.d.ts +74 -0
- package/dist/activities/codex-rotation.d.ts +261 -0
- package/dist/activities/common.d.ts +17 -0
- package/dist/activities/context-compaction.d.ts +81 -0
- package/dist/activities/documents.d.ts +39 -0
- package/dist/activities/environment.d.ts +97 -0
- package/dist/activities/expiring-material-renewal.d.ts +38 -0
- package/dist/activities/file-upload-reaper.d.ts +25 -0
- package/dist/activities/git-credential-renewal.d.ts +41 -0
- package/dist/activities/goals.d.ts +16 -0
- package/dist/activities/mcp-credentials.d.ts +14 -0
- package/dist/activities/packs.d.ts +77 -0
- package/dist/activities/parent-wake.d.ts +54 -0
- package/dist/activities/recording.d.ts +90 -0
- package/dist/activities/redaction.d.ts +1 -0
- package/dist/activities/rig-verification.d.ts +131 -0
- package/dist/activities/run-credential-renewal.d.ts +28 -0
- package/dist/activities/run-credentials.d.ts +36 -0
- package/dist/activities/run-input.d.ts +106 -0
- package/dist/activities/sandbox-lease.d.ts +128 -0
- package/dist/activities/scheduled-tasks.d.ts +4 -0
- package/dist/activities/session-state.d.ts +39 -0
- package/dist/activities/streaming.d.ts +24 -0
- package/dist/activities/toolspace-token-renewal.d.ts +23 -0
- package/dist/activities/turn-attempt-fenced.d.ts +11 -0
- package/dist/activities/types.d.ts +240 -0
- package/dist/activities/workflow-wake.d.ts +15 -0
- package/dist/activities/workspace-capture.d.ts +105 -0
- package/dist/activities.d.ts +312 -0
- package/dist/concurrency.d.ts +19 -0
- package/dist/host-export-pump.d.ts +66 -0
- package/dist/host-export.d.ts +6 -67
- package/dist/http.d.ts +32 -0
- package/dist/index.d.ts +27 -388
- package/dist/index.js +2729 -1208
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +200 -0
- package/dist/op-journal.d.ts +10 -0
- package/dist/sandbox-resume.d.ts +152 -0
- package/dist/sandbox-routing.d.ts +159 -0
- package/dist/worker-service-lifecycle.d.ts +20 -0
- package/dist/workflow-bundle.js +88 -20
- package/dist/workflows/activities.d.ts +27 -0
- package/dist/workflows/document-index.d.ts +41 -0
- package/dist/workflows/file-upload-reaper.d.ts +2 -0
- package/dist/workflows/rig-verification.d.ts +3 -0
- package/dist/workflows/sandbox-reaper.d.ts +1 -0
- package/dist/workflows/scheduled-tasks.d.ts +17 -0
- package/dist/workflows/session.d.ts +58 -0
- package/dist/workflows/workflow-wake.d.ts +3 -0
- package/dist/workflows.d.ts +7 -0
- package/package.json +13 -13
- package/src/activities/agent-turn.ts +873 -131
- package/src/activities/context-compaction.ts +265 -76
- package/src/activities/environment.ts +1 -0
- package/src/activities/goals.ts +14 -0
- package/src/activities/rig-verification.ts +2 -0
- package/src/activities/run-input.ts +18 -4
- package/src/activities/sandbox-lease.ts +787 -133
- package/src/activities/session-state.ts +56 -0
- package/src/activities/types.ts +21 -0
- package/src/activities/workspace-capture.ts +95 -4
- package/src/activities.ts +3 -0
- package/src/observability-metrics.ts +94 -0
- package/src/sandbox-resume.ts +383 -90
- package/src/sandbox-routing.ts +32 -10
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/session.ts +70 -0
|
@@ -42,6 +42,7 @@ import {
|
|
|
42
42
|
countConsecutiveReactiveRotations,
|
|
43
43
|
requireSession,
|
|
44
44
|
recordUsageEvent,
|
|
45
|
+
recordModelCallFact,
|
|
45
46
|
registerPendingSessionToolCall,
|
|
46
47
|
recordPendingSessionToolCallResult,
|
|
47
48
|
clearDurablePendingSessionToolCalls,
|
|
@@ -56,10 +57,12 @@ import {
|
|
|
56
57
|
setSessionLastInputTokensForTurnAttempt,
|
|
57
58
|
sumUsageQuantity,
|
|
58
59
|
heartbeatLeaseHolder,
|
|
60
|
+
readLease,
|
|
59
61
|
accrueWarmSeconds,
|
|
60
62
|
getMaterializedSandboxFileResources,
|
|
61
63
|
markSandboxFileResourcesMaterialized,
|
|
62
64
|
areGitHubRepositoriesAllowedForWorkspace,
|
|
65
|
+
SandboxLeaseRecoveryBlockedError,
|
|
63
66
|
SandboxLeaseSupersededError,
|
|
64
67
|
SandboxImageConflictError,
|
|
65
68
|
isSessionEventPersistenceError,
|
|
@@ -78,6 +81,7 @@ import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@
|
|
|
78
81
|
import {
|
|
79
82
|
sandboxStateEntryFromRunState,
|
|
80
83
|
maxTurnsExceededRunState,
|
|
84
|
+
modelResponseServiceTierFromSdkEvent,
|
|
81
85
|
modelResponseUsageFromSdkEvent,
|
|
82
86
|
normalizeModelCallUsage,
|
|
83
87
|
normalizeSdkEvent,
|
|
@@ -87,6 +91,9 @@ import {
|
|
|
87
91
|
appendWorkspaceMemory,
|
|
88
92
|
composeAgentInstructions,
|
|
89
93
|
summarizeForCompaction,
|
|
94
|
+
requestRemoteCompactionV2,
|
|
95
|
+
REMOTE_COMPACTION_V2_BETA_FEATURE,
|
|
96
|
+
REMOTE_COMPACTION_V2_IMPLEMENTATION,
|
|
90
97
|
CompactionProviderResponseError,
|
|
91
98
|
EmptyCompactionSummaryError,
|
|
92
99
|
ensureModalRegistryImage,
|
|
@@ -101,6 +108,7 @@ import {
|
|
|
101
108
|
toolspaceTokenFileFromEnvironment,
|
|
102
109
|
sandboxFileDownloadFailureNote,
|
|
103
110
|
SUMMARY_BUFFER_TOKENS,
|
|
111
|
+
isMcpRequestTimeoutError,
|
|
104
112
|
runOwnedSandboxSetup,
|
|
105
113
|
RoutingMutationOutcomeUnknownError,
|
|
106
114
|
WorkspaceArchiveIntegrityError,
|
|
@@ -130,7 +138,10 @@ import {
|
|
|
130
138
|
calculateModelUsageCostMicros,
|
|
131
139
|
configuredModelPricing,
|
|
132
140
|
configuredStaticUsageLimits,
|
|
141
|
+
responseSatisfiesLatencyMode,
|
|
142
|
+
sandboxArchiveCaptureTimeoutMs,
|
|
133
143
|
sandboxWarmRateMicrosPerSecond,
|
|
144
|
+
serviceTierForLatencyMode,
|
|
134
145
|
settingsWithResolvedModelContext,
|
|
135
146
|
resolveTurnExecutionPolicyV1,
|
|
136
147
|
type ModelUsageInput,
|
|
@@ -138,7 +149,7 @@ import {
|
|
|
138
149
|
type RegistryProviderKind,
|
|
139
150
|
type Settings,
|
|
140
151
|
} from "@opengeni/config";
|
|
141
|
-
import { CancelledFailure } from "@temporalio/activity";
|
|
152
|
+
import { ApplicationFailure, CancelledFailure } from "@temporalio/activity";
|
|
142
153
|
import {
|
|
143
154
|
settingsWithCodexCredential,
|
|
144
155
|
settingsWithEnabledCapabilityMcpServers,
|
|
@@ -174,21 +185,24 @@ import {
|
|
|
174
185
|
buildModelResolver,
|
|
175
186
|
CODEX_CLIENT_VERSION,
|
|
176
187
|
CODEX_FALLBACK_MODEL_SLUGS,
|
|
188
|
+
CODEX_PROVIDER_ID,
|
|
177
189
|
CodexReloginRequired,
|
|
190
|
+
classifyCodexEncryptedArtifactRejection,
|
|
178
191
|
classifyCodexResponseTimeoutError,
|
|
179
192
|
classifyCodexUsageLimitError,
|
|
180
193
|
codexRequestStorage,
|
|
181
194
|
isCodexTransportError,
|
|
195
|
+
withCodexRequestOverrides,
|
|
182
196
|
type CodexRequestContext,
|
|
183
197
|
type CodexUsageHeaderSnapshot,
|
|
184
198
|
} from "@opengeni/codex";
|
|
185
199
|
import { mergeResourceRefs } from "./common";
|
|
186
200
|
import {
|
|
187
|
-
|
|
201
|
+
assertSessionAllowsProductModel,
|
|
202
|
+
defaultSessionMcpServerIds,
|
|
188
203
|
resolveSessionToolPolicy,
|
|
189
|
-
sessionToolPolicyAllowsDefaultNativeTools,
|
|
190
204
|
} from "@opengeni/core";
|
|
191
|
-
import { maybeCompactContext } from "./context-compaction";
|
|
205
|
+
import { maybeCompactContext, settleFailedContextCompactionLandmark } from "./context-compaction";
|
|
192
206
|
import { TurnAttemptFencedError } from "./turn-attempt-fenced";
|
|
193
207
|
import {
|
|
194
208
|
assertGitCredentialRenewalTransportUnchanged,
|
|
@@ -246,10 +260,15 @@ import {
|
|
|
246
260
|
} from "./streaming";
|
|
247
261
|
import type {
|
|
248
262
|
ActivityServices,
|
|
263
|
+
EscapedMcpTimeoutRecoveryDetail,
|
|
249
264
|
RunAgentTurnInput,
|
|
250
265
|
RunAgentTurnResult,
|
|
251
266
|
SessionAttemptQuiescenceProof,
|
|
252
267
|
} from "./types";
|
|
268
|
+
import {
|
|
269
|
+
ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
|
|
270
|
+
ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
|
|
271
|
+
} from "./types";
|
|
253
272
|
import {
|
|
254
273
|
resumeBoxForTurn,
|
|
255
274
|
sandboxLeaseHolderIdForAttempt,
|
|
@@ -286,7 +305,7 @@ import {
|
|
|
286
305
|
prepareRecordingForSettlement,
|
|
287
306
|
type ActiveRecording,
|
|
288
307
|
} from "./recording";
|
|
289
|
-
import { captureWorkspaceRevision } from "./workspace-capture";
|
|
308
|
+
import { captureWorkspaceRevision, openFreshWorkspaceCaptureSession } from "./workspace-capture";
|
|
290
309
|
import type { ChannelASession } from "@opengeni/runtime/sandbox";
|
|
291
310
|
import { createObjectStorage, type ObjectStorage } from "@opengeni/storage";
|
|
292
311
|
import {
|
|
@@ -299,11 +318,13 @@ import {
|
|
|
299
318
|
DEFAULT_FIRST_PARTY_MCP_TOOLS,
|
|
300
319
|
evaluateWorkspaceModelPolicy,
|
|
301
320
|
readTurnExecutionPolicyV1,
|
|
321
|
+
type LatencyMode,
|
|
302
322
|
type ResourceRef,
|
|
303
323
|
type SessionEvent,
|
|
304
324
|
type SessionEventType,
|
|
305
325
|
type SessionStatus,
|
|
306
326
|
type SessionTurn,
|
|
327
|
+
type ToolAuthNeededPayload,
|
|
307
328
|
type TurnExecutionPolicyV1,
|
|
308
329
|
type TurnInitiator,
|
|
309
330
|
} from "@opengeni/contracts";
|
|
@@ -322,6 +343,50 @@ export function credentialSubjectIdForTurnInitiator(
|
|
|
322
343
|
return initiator.kind === "subject" ? initiator.subjectId : undefined;
|
|
323
344
|
}
|
|
324
345
|
|
|
346
|
+
/**
|
|
347
|
+
* Direct authenticated-human work is the persisted human source with subject
|
|
348
|
+
* authority and no inherited agent or legacy provenance. Use this complete
|
|
349
|
+
* immutable turn authority instead of inferring causality from a
|
|
350
|
+
* `user.message` event shape or the root initiator alone.
|
|
351
|
+
*/
|
|
352
|
+
export function isDirectHumanTurnInitiation(
|
|
353
|
+
turn: Pick<SessionTurn, "source" | "initiator" | "initiatorContext">,
|
|
354
|
+
): boolean {
|
|
355
|
+
if (turn.source !== "user" || turn.initiator.kind !== "subject") {
|
|
356
|
+
return false;
|
|
357
|
+
}
|
|
358
|
+
return !["via", "viaTruncated", "provenanceError", "backfill"].some((key) =>
|
|
359
|
+
Object.prototype.hasOwnProperty.call(turn.initiatorContext, key),
|
|
360
|
+
);
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* A disconnected personal Slack server is prepared best-effort before the model
|
|
365
|
+
* runs. Its initialize/tools-list credential miss is setup state, not evidence
|
|
366
|
+
* that an unrelated turn wants Slack. Keep concrete tool-call failures
|
|
367
|
+
* actionable, but gate setup-time Slack prompts to a human message that names
|
|
368
|
+
* Slack explicitly. Other providers retain the existing generic behavior.
|
|
369
|
+
*/
|
|
370
|
+
export function shouldPublishToolAuthNeededForTurn(
|
|
371
|
+
payload: Pick<ToolAuthNeededPayload, "providerDomain" | "toolName">,
|
|
372
|
+
trigger: Pick<SessionEvent, "type" | "payload">,
|
|
373
|
+
turn: Pick<SessionTurn, "source" | "initiator" | "initiatorContext">,
|
|
374
|
+
): boolean {
|
|
375
|
+
if (typeof payload.toolName === "string" && payload.toolName.trim().length > 0) {
|
|
376
|
+
return true;
|
|
377
|
+
}
|
|
378
|
+
const providerDomain = payload.providerDomain.trim().toLowerCase();
|
|
379
|
+
const isSlack = providerDomain === "slack.com" || providerDomain.endsWith(".slack.com");
|
|
380
|
+
if (!isSlack) {
|
|
381
|
+
return true;
|
|
382
|
+
}
|
|
383
|
+
if (!isDirectHumanTurnInitiation(turn) || trigger.type !== "user.message") {
|
|
384
|
+
return false;
|
|
385
|
+
}
|
|
386
|
+
const text = (trigger.payload as { text?: unknown }).text;
|
|
387
|
+
return typeof text === "string" && /\bslack\b/i.test(text);
|
|
388
|
+
}
|
|
389
|
+
|
|
325
390
|
export function turnExecutionPolicyBillingIdentity(policy: TurnExecutionPolicyV1): {
|
|
326
391
|
externallyBilled: boolean;
|
|
327
392
|
codexSubscription: boolean;
|
|
@@ -336,7 +401,7 @@ export function turnExecutionPolicyBillingIdentity(policy: TurnExecutionPolicyV1
|
|
|
336
401
|
}
|
|
337
402
|
|
|
338
403
|
export function legacyTurnExecutionPolicyInput(
|
|
339
|
-
turn: Pick<SessionTurn, "source" | "model" | "reasoningEffort">,
|
|
404
|
+
turn: Pick<SessionTurn, "source" | "model" | "reasoningEffort" | "latencyMode">,
|
|
340
405
|
): Parameters<typeof resolveTurnExecutionPolicyV1>[1] {
|
|
341
406
|
const explicit = turn.source === "user" || turn.source === "api";
|
|
342
407
|
return {
|
|
@@ -345,6 +410,8 @@ export function legacyTurnExecutionPolicyInput(
|
|
|
345
410
|
modelSource: explicit ? "explicit" : "continuation",
|
|
346
411
|
reasoningEffort: turn.reasoningEffort,
|
|
347
412
|
reasoningSource: explicit ? "explicit" : "continuation",
|
|
413
|
+
latencyMode: turn.latencyMode,
|
|
414
|
+
latencyModeSource: explicit ? "explicit" : "continuation",
|
|
348
415
|
};
|
|
349
416
|
}
|
|
350
417
|
|
|
@@ -361,6 +428,56 @@ export function providerRecoveryResult(): {
|
|
|
361
428
|
};
|
|
362
429
|
}
|
|
363
430
|
|
|
431
|
+
/**
|
|
432
|
+
* Tell the model when durable MCP policy references are unavailable for this
|
|
433
|
+
* exact turn. The policy projection already bounds and validates these ids;
|
|
434
|
+
* this notice prevents a graceful runtime drop from becoming a silent source-
|
|
435
|
+
* of-truth substitution or a false claim that the disconnected system was read.
|
|
436
|
+
*/
|
|
437
|
+
export function unavailableMcpTurnInstructions(input: {
|
|
438
|
+
droppedIds: readonly string[];
|
|
439
|
+
droppedCount: number;
|
|
440
|
+
}): string | undefined {
|
|
441
|
+
if (input.droppedCount <= 0) {
|
|
442
|
+
return undefined;
|
|
443
|
+
}
|
|
444
|
+
const omittedCount = Math.max(0, input.droppedCount - input.droppedIds.length);
|
|
445
|
+
const listed = input.droppedIds.map((id) => `"${id}"`).join(", ");
|
|
446
|
+
const inventory = listed
|
|
447
|
+
? `${listed}${omittedCount > 0 ? `, plus ${omittedCount} additional unavailable server(s)` : ""}`
|
|
448
|
+
: `${input.droppedCount} unavailable server(s)`;
|
|
449
|
+
return `MCP capability availability for this turn: the following session-selected server(s) are disconnected or no longer registered and were skipped: ${inventory}. Do not claim to have read or updated those systems. If the task depends on one as a source of truth, explain the limitation and ask the user to reconnect it or select another authoritative source; continue with unaffected work only when safe.`;
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/**
|
|
453
|
+
* Preserve one precise recovery obligation across Temporal's activity boundary.
|
|
454
|
+
* This is intentionally narrower than the ordinary retryable-provider path:
|
|
455
|
+
* only a recovered turn (generation > 1) whose MCP timeout happened before a
|
|
456
|
+
* model request may ask the workflow's DB-only control activity to finish the
|
|
457
|
+
* same-turn checkpoint. The original transport/recovery errors are excluded
|
|
458
|
+
* from details so raw MCP response data can never enter workflow history.
|
|
459
|
+
*/
|
|
460
|
+
export function escapedMcpTimeoutRecoveryFailure(input: {
|
|
461
|
+
failureCode: string | undefined;
|
|
462
|
+
modelRequestStarted: boolean;
|
|
463
|
+
detail: EscapedMcpTimeoutRecoveryDetail;
|
|
464
|
+
}): ApplicationFailure | null {
|
|
465
|
+
if (
|
|
466
|
+
input.failureCode !== "mcp_transport_timeout" ||
|
|
467
|
+
input.modelRequestStarted ||
|
|
468
|
+
!Number.isSafeInteger(input.detail.executionGeneration) ||
|
|
469
|
+
input.detail.executionGeneration <= 1
|
|
470
|
+
) {
|
|
471
|
+
return null;
|
|
472
|
+
}
|
|
473
|
+
return ApplicationFailure.create({
|
|
474
|
+
message: ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
|
|
475
|
+
type: ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
|
|
476
|
+
nonRetryable: true,
|
|
477
|
+
details: [input.detail],
|
|
478
|
+
});
|
|
479
|
+
}
|
|
480
|
+
|
|
364
481
|
/**
|
|
365
482
|
* Resolve which Codex account a turn runs on (multi-account P1): session-pin >
|
|
366
483
|
* workspace-active. No rotation in P1. The selected id must still be in the
|
|
@@ -762,14 +879,14 @@ export function classifyContextWindowOverflowError(
|
|
|
762
879
|
* next-loop transport work can reject the stream iterator after a prior tool
|
|
763
880
|
* output was already published. That is transient external backpressure, not a
|
|
764
881
|
* terminal session error. Match MCP-qualified timeout text only: an unrelated
|
|
765
|
-
* sandbox/model timeout and MCP's
|
|
882
|
+
* sandbox/model timeout and MCP's application-defined Authentication required signal must
|
|
766
883
|
* retain their existing semantics.
|
|
767
884
|
*/
|
|
768
885
|
export function classifyMcpTransportTimeoutError(
|
|
769
886
|
error: unknown,
|
|
770
887
|
): { message: string; detail?: string } | null {
|
|
771
888
|
const fields = collectErrorStrings(error);
|
|
772
|
-
const
|
|
889
|
+
const matchedText = fields.find(
|
|
773
890
|
(value) =>
|
|
774
891
|
/\bmcp\b/i.test(value) &&
|
|
775
892
|
/(?:request\s+timed\s+out|request\s+timeout|\btimed\s+out\b|\btimeout\b|ETIMEDOUT)/i.test(
|
|
@@ -777,13 +894,15 @@ export function classifyMcpTransportTimeoutError(
|
|
|
777
894
|
) &&
|
|
778
895
|
!/authentication\s+required/i.test(value),
|
|
779
896
|
);
|
|
780
|
-
|
|
897
|
+
const sanitizedSdkTimeout = isMcpRequestTimeoutError(error);
|
|
898
|
+
if (!matchedText && !sanitizedSdkTimeout) {
|
|
781
899
|
return null;
|
|
782
900
|
}
|
|
783
901
|
const message = error instanceof Error ? error.message : String(error);
|
|
902
|
+
const matched = matchedText ?? fields.find((value) => /\bmcp\b/i.test(value));
|
|
784
903
|
return {
|
|
785
904
|
message,
|
|
786
|
-
...(matched !== message ? { detail: matched } : {}),
|
|
905
|
+
...(matched && matched !== message ? { detail: matched } : {}),
|
|
787
906
|
};
|
|
788
907
|
}
|
|
789
908
|
|
|
@@ -946,6 +1065,37 @@ export function modelResponseUsageContextSignal(
|
|
|
946
1065
|
};
|
|
947
1066
|
}
|
|
948
1067
|
|
|
1068
|
+
export function assertModelResponseLatencyMode(input: {
|
|
1069
|
+
event: Parameters<typeof modelResponseServiceTierFromSdkEvent>[0];
|
|
1070
|
+
requested: LatencyMode;
|
|
1071
|
+
model: string;
|
|
1072
|
+
/** When set to Codex ChatGPT auth, response `service_tier` is not an honor signal. */
|
|
1073
|
+
providerId?: string;
|
|
1074
|
+
}): void {
|
|
1075
|
+
if (input.requested === "standard") {
|
|
1076
|
+
return;
|
|
1077
|
+
}
|
|
1078
|
+
// ChatGPT-auth Codex (subscription): Fast maps to request `service_tier=priority`
|
|
1079
|
+
// (see openai/codex ServiceTier::Fast.request_value). The backend may still return
|
|
1080
|
+
// `response.service_tier=default`; OpenAI maintainers document that this does not
|
|
1081
|
+
// mean Fast was ignored. Native CLI also does not fail closed on that field.
|
|
1082
|
+
if (input.providerId === CODEX_PROVIDER_ID) {
|
|
1083
|
+
return;
|
|
1084
|
+
}
|
|
1085
|
+
const serviceTierEvent = modelResponseServiceTierFromSdkEvent(input.event);
|
|
1086
|
+
if (
|
|
1087
|
+
!serviceTierEvent ||
|
|
1088
|
+
(serviceTierEvent.source === "normalized" && serviceTierEvent.serviceTier === null)
|
|
1089
|
+
) {
|
|
1090
|
+
return;
|
|
1091
|
+
}
|
|
1092
|
+
if (!responseSatisfiesLatencyMode(input.requested, serviceTierEvent.serviceTier)) {
|
|
1093
|
+
throw new Error(
|
|
1094
|
+
`Provider did not honor ${input.requested} latency mode for ${input.model}: response service_tier=${serviceTierEvent.serviceTier ?? "missing"}`,
|
|
1095
|
+
);
|
|
1096
|
+
}
|
|
1097
|
+
}
|
|
1098
|
+
|
|
949
1099
|
/**
|
|
950
1100
|
* Process one SDK stream event through the exact production usage path.
|
|
951
1101
|
*
|
|
@@ -973,6 +1123,7 @@ export async function processModelResponseUsageEvent(input: {
|
|
|
973
1123
|
provider: string;
|
|
974
1124
|
providerApi: ModelProviderApi;
|
|
975
1125
|
model: string;
|
|
1126
|
+
latencyMode?: LatencyMode;
|
|
976
1127
|
metricProvider: string;
|
|
977
1128
|
externallyBilled: boolean;
|
|
978
1129
|
servingCredentialId: string | null;
|
|
@@ -1016,7 +1167,7 @@ export async function processModelResponseUsageEvent(input: {
|
|
|
1016
1167
|
leaseLost: input.leaseLost,
|
|
1017
1168
|
leaseLostMessage: input.leaseLostMessage,
|
|
1018
1169
|
recordUsage: async () => {
|
|
1019
|
-
await recordModelUsageAndDebitCredits(input.settings, input.db, {
|
|
1170
|
+
const billing = await recordModelUsageAndDebitCredits(input.settings, input.db, {
|
|
1020
1171
|
accountId: input.accountId,
|
|
1021
1172
|
workspaceId: input.workspaceId,
|
|
1022
1173
|
sessionId: input.sessionId,
|
|
@@ -1027,6 +1178,7 @@ export async function processModelResponseUsageEvent(input: {
|
|
|
1027
1178
|
usage: responseUsage.usage,
|
|
1028
1179
|
normalizedUsage,
|
|
1029
1180
|
sourceKey,
|
|
1181
|
+
...(input.latencyMode ? { latencyMode: input.latencyMode } : {}),
|
|
1030
1182
|
observability: input.observability,
|
|
1031
1183
|
});
|
|
1032
1184
|
authoritative = await emitModelCallUsage({
|
|
@@ -1046,6 +1198,22 @@ export async function processModelResponseUsageEvent(input: {
|
|
|
1046
1198
|
accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
|
|
1047
1199
|
emittedSourceKeys: input.emittedSourceKeys,
|
|
1048
1200
|
});
|
|
1201
|
+
if (authoritative && billing) {
|
|
1202
|
+
await recordAuthoritativeModelCallFact({
|
|
1203
|
+
db: input.db,
|
|
1204
|
+
observability: input.observability,
|
|
1205
|
+
accountId: input.accountId,
|
|
1206
|
+
workspaceId: input.workspaceId,
|
|
1207
|
+
sessionId: input.sessionId,
|
|
1208
|
+
turnId: input.turnId,
|
|
1209
|
+
turnAttemptId: input.turnAttemptId,
|
|
1210
|
+
sourceKey,
|
|
1211
|
+
provider: input.provider,
|
|
1212
|
+
providerApi: input.providerApi,
|
|
1213
|
+
model: input.model,
|
|
1214
|
+
billing,
|
|
1215
|
+
});
|
|
1216
|
+
}
|
|
1049
1217
|
const observedInput = normalizedUsage.telemetry.inputTokens;
|
|
1050
1218
|
if (authoritative && observedInput !== null && observedInput > 0) {
|
|
1051
1219
|
recordModelInputTokens(input.observability, input.metricProvider, observedInput);
|
|
@@ -1125,7 +1293,7 @@ export async function processCompactionModelUsageEvent(input: {
|
|
|
1125
1293
|
leaseLost: input.leaseLost,
|
|
1126
1294
|
leaseLostMessage: input.leaseLostMessage,
|
|
1127
1295
|
recordUsage: async () => {
|
|
1128
|
-
await recordModelUsageAndDebitCredits(input.settings, input.db, {
|
|
1296
|
+
const billing = await recordModelUsageAndDebitCredits(input.settings, input.db, {
|
|
1129
1297
|
accountId: input.accountId,
|
|
1130
1298
|
workspaceId: input.workspaceId,
|
|
1131
1299
|
sessionId: input.sessionId,
|
|
@@ -1155,6 +1323,22 @@ export async function processCompactionModelUsageEvent(input: {
|
|
|
1155
1323
|
accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
|
|
1156
1324
|
emittedSourceKeys: input.emittedSourceKeys,
|
|
1157
1325
|
});
|
|
1326
|
+
if (authoritative && billing) {
|
|
1327
|
+
await recordAuthoritativeModelCallFact({
|
|
1328
|
+
db: input.db,
|
|
1329
|
+
observability: input.observability,
|
|
1330
|
+
accountId: input.accountId,
|
|
1331
|
+
workspaceId: input.workspaceId,
|
|
1332
|
+
sessionId: input.sessionId,
|
|
1333
|
+
turnId: input.turnId,
|
|
1334
|
+
turnAttemptId: input.turnAttemptId,
|
|
1335
|
+
sourceKey,
|
|
1336
|
+
provider: input.provider,
|
|
1337
|
+
providerApi: input.providerApi,
|
|
1338
|
+
model: input.model,
|
|
1339
|
+
billing,
|
|
1340
|
+
});
|
|
1341
|
+
}
|
|
1158
1342
|
},
|
|
1159
1343
|
});
|
|
1160
1344
|
return { status: "processed", sourceKey, authoritative };
|
|
@@ -1651,9 +1835,9 @@ export function shouldStartOnTurnRecording(params: {
|
|
|
1651
1835
|
* and threaded to the runtime as an explicit flag (buildAgent → computerToolMode):
|
|
1652
1836
|
* • codex-subscription → "function-image": the ChatGPT/Codex backend rejects
|
|
1653
1837
|
* hosted tool types but SEES structured `input_image` tool results.
|
|
1654
|
-
* • a "chat" (OpenAIChatCompletionsModel wire) provider → "
|
|
1655
|
-
*
|
|
1656
|
-
* as
|
|
1838
|
+
* • a "chat" (OpenAIChatCompletionsModel wire) provider → "disabled": it cannot
|
|
1839
|
+
* receive a screenshot through a proven visual transport, so computer use fails
|
|
1840
|
+
* closed instead of serializing the image as text.
|
|
1657
1841
|
* • everything else — built-in Azure/OpenAI responses, registry "responses"
|
|
1658
1842
|
* providers, AND the LEGACY global-client fallback (resolveTurnModel returned
|
|
1659
1843
|
* null) — → "hosted": real Responses hosted-tool support.
|
|
@@ -1672,24 +1856,22 @@ export function computerToolModeForTurn(
|
|
|
1672
1856
|
return "function-image";
|
|
1673
1857
|
}
|
|
1674
1858
|
if (resolvedModel.provider.api === "chat") {
|
|
1675
|
-
return "
|
|
1859
|
+
return "disabled";
|
|
1676
1860
|
}
|
|
1677
1861
|
return "hosted";
|
|
1678
1862
|
}
|
|
1679
1863
|
|
|
1680
1864
|
/**
|
|
1681
|
-
* Native web search
|
|
1682
|
-
*
|
|
1683
|
-
*
|
|
1684
|
-
*
|
|
1685
|
-
*
|
|
1865
|
+
* Native web search is a runtime capability, not part of the session's MCP
|
|
1866
|
+
* allow-list. Attach it whenever the resolved provider advertises runnable
|
|
1867
|
+
* support. The null model is the legacy built-in Responses path, whose
|
|
1868
|
+
* deployment flag is its provider capability gate. There is deliberately no
|
|
1869
|
+
* cross-provider or sandbox/curl fallback.
|
|
1686
1870
|
*/
|
|
1687
1871
|
export function hostedWebSearchForTurn(
|
|
1688
1872
|
resolvedModel: { configured: { hostedWebSearch: boolean } } | null,
|
|
1689
1873
|
deploymentWebSearchEnabled: boolean,
|
|
1690
|
-
defaultNativeToolsAllowed: boolean,
|
|
1691
1874
|
): boolean {
|
|
1692
|
-
if (!defaultNativeToolsAllowed) return false;
|
|
1693
1875
|
return resolvedModel?.configured.hostedWebSearch ?? deploymentWebSearchEnabled;
|
|
1694
1876
|
}
|
|
1695
1877
|
|
|
@@ -1722,6 +1904,19 @@ export class TurnOperationCancelledError extends Error {
|
|
|
1722
1904
|
}
|
|
1723
1905
|
}
|
|
1724
1906
|
|
|
1907
|
+
export class SandboxDeadlineRotationError extends Error {
|
|
1908
|
+
readonly name = "SandboxDeadlineRotationError";
|
|
1909
|
+
|
|
1910
|
+
constructor(
|
|
1911
|
+
readonly sandboxGroupId: string,
|
|
1912
|
+
readonly leaseEpoch: number,
|
|
1913
|
+
) {
|
|
1914
|
+
super(
|
|
1915
|
+
`Sandbox ${sandboxGroupId} reached its provider rotation boundary at lease epoch ${leaseEpoch}`,
|
|
1916
|
+
);
|
|
1917
|
+
}
|
|
1918
|
+
}
|
|
1919
|
+
|
|
1725
1920
|
/**
|
|
1726
1921
|
* Normalize a preparation/provisioning cancellation race back to the Temporal
|
|
1727
1922
|
* cancellation that owns the activity. Several provider APIs expose no portable
|
|
@@ -1797,7 +1992,10 @@ function sleep(ms: number): Promise<void> {
|
|
|
1797
1992
|
}
|
|
1798
1993
|
|
|
1799
1994
|
export function isLazySandboxProvisionRetryable(error: unknown): boolean {
|
|
1800
|
-
if (
|
|
1995
|
+
if (
|
|
1996
|
+
error instanceof SandboxImageConflictError ||
|
|
1997
|
+
error instanceof SandboxLeaseRecoveryBlockedError
|
|
1998
|
+
) {
|
|
1801
1999
|
return false;
|
|
1802
2000
|
}
|
|
1803
2001
|
if (error instanceof WorkspaceArchiveIntegrityError) {
|
|
@@ -1812,6 +2010,20 @@ export function isLazySandboxProvisionRetryable(error: unknown): boolean {
|
|
|
1812
2010
|
);
|
|
1813
2011
|
}
|
|
1814
2012
|
|
|
2013
|
+
/**
|
|
2014
|
+
* Deadline recovery cannot re-admit the same turn until the global reaper has
|
|
2015
|
+
* observed the released holder, captured the final archive, terminated the old
|
|
2016
|
+
* provider box, and committed the lease cold. Give that full sequence one
|
|
2017
|
+
* snapshot budget plus two schedule periods. This is workflow-visible pacing,
|
|
2018
|
+
* not an in-memory sleep, so Pause/Steer/new user intent can still wake the
|
|
2019
|
+
* durable session immediately.
|
|
2020
|
+
*/
|
|
2021
|
+
export function sandboxDeadlineRotationRecoveryDelayMs(
|
|
2022
|
+
settings: Pick<Settings, "sandboxLeaseReaperPeriodMs" | "sandboxSnapshotTimeoutMs">,
|
|
2023
|
+
): number {
|
|
2024
|
+
return settings.sandboxSnapshotTimeoutMs + settings.sandboxLeaseReaperPeriodMs * 2;
|
|
2025
|
+
}
|
|
2026
|
+
|
|
1815
2027
|
export function createTurnSandboxProvisioner<T>(
|
|
1816
2028
|
establish: () => Promise<T>,
|
|
1817
2029
|
options: {
|
|
@@ -1874,7 +2086,15 @@ export function createTurnSandboxProvisioner<T>(
|
|
|
1874
2086
|
throw error;
|
|
1875
2087
|
}
|
|
1876
2088
|
})().catch((error) => {
|
|
1877
|
-
|
|
2089
|
+
// A terminal lease/archive/configuration result cannot change within
|
|
2090
|
+
// this frozen turn. Keep its rejected promise memoized so later model
|
|
2091
|
+
// tool calls observe the same typed result without re-entering
|
|
2092
|
+
// provisioning, emitting another failure event, or creating another
|
|
2093
|
+
// provider attempt. Only a retryable failure releases the single-flight
|
|
2094
|
+
// for a later operation to re-read the lease/provider state.
|
|
2095
|
+
if (isLazySandboxProvisionRetryable(error)) {
|
|
2096
|
+
memo = null;
|
|
2097
|
+
}
|
|
1878
2098
|
throw error;
|
|
1879
2099
|
});
|
|
1880
2100
|
}
|
|
@@ -2014,6 +2234,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2014
2234
|
} = await services();
|
|
2015
2235
|
const activityContext = currentActivityContext();
|
|
2016
2236
|
const cancellationSignal = activityContext?.cancellationSignal;
|
|
2237
|
+
// Temporal cancellation is not the only way an activity loses ownership:
|
|
2238
|
+
// a database attempt fence or workflow abandonment can finalize the logical
|
|
2239
|
+
// turn while the SDK cancellation signal remains open. Bind sandbox
|
|
2240
|
+
// provisioning to both lifetimes so an uninterruptible, late provider
|
|
2241
|
+
// promise cannot resurrect a holder after this activity has finalized.
|
|
2242
|
+
const sandboxResumeController = new AbortController();
|
|
2243
|
+
const sandboxResumeSignal = cancellationSignal
|
|
2244
|
+
? AbortSignal.any([cancellationSignal, sandboxResumeController.signal])
|
|
2245
|
+
: sandboxResumeController.signal;
|
|
2246
|
+
const sandboxRotationController = new AbortController();
|
|
2247
|
+
const runtimeCancellationSignal = cancellationSignal
|
|
2248
|
+
? AbortSignal.any([cancellationSignal, sandboxRotationController.signal])
|
|
2249
|
+
: sandboxRotationController.signal;
|
|
2017
2250
|
let cancellationRequestedAt: number | null = cancellationSignal?.aborted
|
|
2018
2251
|
? performance.now()
|
|
2019
2252
|
: null;
|
|
@@ -2067,6 +2300,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2067
2300
|
let isCodexTurn = false;
|
|
2068
2301
|
let isExternallyBilledTurn = false;
|
|
2069
2302
|
let executionGeneration = 0;
|
|
2303
|
+
let modelRequestStarted = false;
|
|
2070
2304
|
// Still required by credential-loss/capacity settlements, whose own
|
|
2071
2305
|
// recovery transactions fence against worker-death redispatches.
|
|
2072
2306
|
let redispatchesAtDispatch = 0;
|
|
@@ -2274,6 +2508,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2274
2508
|
expectedEpoch: sandbox.leaseEpoch,
|
|
2275
2509
|
expectedInstanceId: sandbox.established.instanceId,
|
|
2276
2510
|
operation,
|
|
2511
|
+
captureWaitMs: sandboxArchiveCaptureTimeoutMs(settings),
|
|
2277
2512
|
};
|
|
2278
2513
|
const admission = await advanceWorkspaceGeneration(db, identity);
|
|
2279
2514
|
let result: T;
|
|
@@ -2314,6 +2549,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2314
2549
|
// runs it refreshes expires_at epoch-fenced so a legit multi-day turn is
|
|
2315
2550
|
// never TTL-reaped. Cleared in finally. Only set when the flag resolved a box.
|
|
2316
2551
|
let leaseHeartbeatTimer: ReturnType<typeof setInterval> | undefined;
|
|
2552
|
+
const stopLeaseHeartbeat = (): void => {
|
|
2553
|
+
if (!leaseHeartbeatTimer) return;
|
|
2554
|
+
clearInterval(leaseHeartbeatTimer);
|
|
2555
|
+
leaseHeartbeatTimer = undefined;
|
|
2556
|
+
};
|
|
2557
|
+
let rotationInFlight: Promise<void> | null = null;
|
|
2317
2558
|
// credential-renewal policy: the worker, not the model, owns renewal of run-scoped Git
|
|
2318
2559
|
// credentials for a multi-day turn. The controller is attached only after
|
|
2319
2560
|
// the initial seed reached a real cloud box and is drained before capture.
|
|
@@ -2337,6 +2578,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2337
2578
|
// the atomic DB throttle discard the fresher one). Interval throttling
|
|
2338
2579
|
// itself lives in maybePersistWarmWorkspaceSnapshot / persistWarmSnapshot.
|
|
2339
2580
|
let snapshotInFlight: Promise<void> | null = null;
|
|
2581
|
+
// Turn-end capture needs the lease heartbeat to keep its holder alive, but
|
|
2582
|
+
// must prevent that same timer from starting another periodic snapshot
|
|
2583
|
+
// while it reads. This gate separates those two responsibilities.
|
|
2584
|
+
let turnEndCaptureInProgress = false;
|
|
2340
2585
|
// Computer-use-only recording. Ordinary shell/filesystem turns leave this
|
|
2341
2586
|
// null; the first actual computer action starts it after :0 is ready.
|
|
2342
2587
|
let activeRecording: ActiveRecording | null = null;
|
|
@@ -2378,10 +2623,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2378
2623
|
await current?.flush().catch(() => undefined);
|
|
2379
2624
|
};
|
|
2380
2625
|
let preparedTools: Awaited<ReturnType<OpenGeniRuntime["prepareTools"]>> | null = null;
|
|
2381
|
-
// Resolved with the MCP policy at the immutable turn boundary. Provider
|
|
2382
|
-
// capability is checked separately when buildAgent is called; this flag is
|
|
2383
|
-
// only the durable omission/narrowing entitlement.
|
|
2384
|
-
let defaultNativeToolsAllowed = false;
|
|
2385
2626
|
const toolCancellationFenceRef: {
|
|
2386
2627
|
current: TurnToolCancellationFence | null;
|
|
2387
2628
|
} = {
|
|
@@ -2488,7 +2729,81 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2488
2729
|
holderId: heartbeatHolderId,
|
|
2489
2730
|
leaseTtlMs: settings.sandboxLeaseTtlMs,
|
|
2490
2731
|
expectedEpoch: heartbeatEpoch,
|
|
2491
|
-
})
|
|
2732
|
+
})
|
|
2733
|
+
.then(async (alive) => {
|
|
2734
|
+
if (alive || rotationInFlight || sandboxRotationController.signal.aborted) return;
|
|
2735
|
+
const rotatingLease = await readLease(db, input.workspaceId, heartbeatGroupId).catch(
|
|
2736
|
+
() => null,
|
|
2737
|
+
);
|
|
2738
|
+
if (
|
|
2739
|
+
!rotatingLease ||
|
|
2740
|
+
rotatingLease.leaseEpoch !== heartbeatEpoch ||
|
|
2741
|
+
rotatingLease.instanceId !== sandbox.established.instanceId ||
|
|
2742
|
+
rotatingLease.rotationRequestedAt === null
|
|
2743
|
+
) {
|
|
2744
|
+
// The holder was reaped, the exact attempt closed, the epoch was
|
|
2745
|
+
// superseded, or the lease began draining. Do not leave a dead
|
|
2746
|
+
// interval issuing DB writes and snapshot probes forever.
|
|
2747
|
+
stopLeaseHeartbeat();
|
|
2748
|
+
return;
|
|
2749
|
+
}
|
|
2750
|
+
rotationInFlight = (async () => {
|
|
2751
|
+
// Rotation admission already fenced all new workspace mutations.
|
|
2752
|
+
// Wait for an earlier periodic capture, then produce the exact
|
|
2753
|
+
// generation-complete checkpoint that licenses aborting this run.
|
|
2754
|
+
if (snapshotInFlight) {
|
|
2755
|
+
await waitForWarmSnapshot(
|
|
2756
|
+
snapshotInFlight,
|
|
2757
|
+
settings.sandboxSnapshotTimeoutMs,
|
|
2758
|
+
cancellationSignal,
|
|
2759
|
+
);
|
|
2760
|
+
}
|
|
2761
|
+
const snapshotSession = setupBoxSession;
|
|
2762
|
+
const snapshotTurnId = turnId;
|
|
2763
|
+
if (snapshotSession && snapshotTurnId) {
|
|
2764
|
+
await maybePersistWarmWorkspaceSnapshot(
|
|
2765
|
+
{ db, settings },
|
|
2766
|
+
{
|
|
2767
|
+
accountId: input.accountId,
|
|
2768
|
+
workspaceId: input.workspaceId,
|
|
2769
|
+
sessionId: input.sessionId,
|
|
2770
|
+
turnId: snapshotTurnId,
|
|
2771
|
+
attemptId: input.attemptId,
|
|
2772
|
+
sandboxGroupId: heartbeatGroupId,
|
|
2773
|
+
},
|
|
2774
|
+
snapshotSession,
|
|
2775
|
+
heartbeatEpoch,
|
|
2776
|
+
cancellationSignal,
|
|
2777
|
+
true,
|
|
2778
|
+
);
|
|
2779
|
+
}
|
|
2780
|
+
const checkpointed = await readLease(db, input.workspaceId, heartbeatGroupId);
|
|
2781
|
+
if (
|
|
2782
|
+
checkpointed?.leaseEpoch === heartbeatEpoch &&
|
|
2783
|
+
checkpointed.rotationRequestedAt !== null &&
|
|
2784
|
+
checkpointed.archiveComplete &&
|
|
2785
|
+
!cancellationSignal?.aborted
|
|
2786
|
+
) {
|
|
2787
|
+
sandboxRotationController.abort(
|
|
2788
|
+
new SandboxDeadlineRotationError(heartbeatGroupId, heartbeatEpoch),
|
|
2789
|
+
);
|
|
2790
|
+
}
|
|
2791
|
+
})()
|
|
2792
|
+
.catch((error) => {
|
|
2793
|
+
observability.warn("sandbox deadline rotation checkpoint failed; retrying", {
|
|
2794
|
+
workspaceId: input.workspaceId,
|
|
2795
|
+
sessionId: input.sessionId,
|
|
2796
|
+
sandboxGroupId: heartbeatGroupId,
|
|
2797
|
+
leaseEpoch: heartbeatEpoch,
|
|
2798
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2799
|
+
});
|
|
2800
|
+
})
|
|
2801
|
+
.finally(() => {
|
|
2802
|
+
rotationInFlight = null;
|
|
2803
|
+
});
|
|
2804
|
+
await rotationInFlight;
|
|
2805
|
+
})
|
|
2806
|
+
.catch(() => undefined);
|
|
2492
2807
|
void accrueWarmSeconds(db, {
|
|
2493
2808
|
accountId: input.accountId,
|
|
2494
2809
|
workspaceId: input.workspaceId,
|
|
@@ -2510,7 +2825,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2510
2825
|
// single-flight; throttling lives in the helper.
|
|
2511
2826
|
const snapshotSession = setupBoxSession;
|
|
2512
2827
|
const snapshotTurnId = turnId;
|
|
2513
|
-
if (snapshotSession && snapshotTurnId && !snapshotInFlight) {
|
|
2828
|
+
if (snapshotSession && snapshotTurnId && !snapshotInFlight && !turnEndCaptureInProgress) {
|
|
2514
2829
|
snapshotInFlight = maybePersistWarmWorkspaceSnapshot(
|
|
2515
2830
|
{ db, settings },
|
|
2516
2831
|
{
|
|
@@ -2806,9 +3121,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2806
3121
|
{
|
|
2807
3122
|
modelId: turn.model,
|
|
2808
3123
|
reasoningEffort: turn.reasoningEffort,
|
|
3124
|
+
latencyMode: turn.latencyMode,
|
|
2809
3125
|
},
|
|
2810
3126
|
);
|
|
2811
3127
|
const turnExecutionPolicy = verifiedExecutionPolicy.policy;
|
|
3128
|
+
assertSessionAllowsProductModel(session, turnExecutionPolicy.productModelId);
|
|
2812
3129
|
const billingIdentity = turnExecutionPolicyBillingIdentity(turnExecutionPolicy);
|
|
2813
3130
|
isExternallyBilledTurn = billingIdentity.externallyBilled;
|
|
2814
3131
|
isCodexTurn = billingIdentity.codexSubscription;
|
|
@@ -3780,12 +4097,22 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3780
4097
|
},
|
|
3781
4098
|
]
|
|
3782
4099
|
: !resolvedModel && fallbackProviderApiKey
|
|
3783
|
-
? [
|
|
4100
|
+
? [
|
|
4101
|
+
{
|
|
4102
|
+
name: "MODEL_PROVIDER_API_KEY",
|
|
4103
|
+
value: fallbackProviderApiKey,
|
|
4104
|
+
},
|
|
4105
|
+
]
|
|
3784
4106
|
: []),
|
|
3785
4107
|
...Object.entries(selectedProvider?.defaultHeaders ?? {}).flatMap(([name, value]) =>
|
|
3786
4108
|
publicProviderHeaders.has(name.toLowerCase())
|
|
3787
4109
|
? []
|
|
3788
|
-
: [
|
|
4110
|
+
: [
|
|
4111
|
+
{
|
|
4112
|
+
name: `MODEL_PROVIDER_HEADER_${name.toUpperCase()}`,
|
|
4113
|
+
value,
|
|
4114
|
+
},
|
|
4115
|
+
],
|
|
3789
4116
|
),
|
|
3790
4117
|
...Object.entries(selectedProvider?.defaultQuery ?? {}).flatMap(([name, value]) =>
|
|
3791
4118
|
publicProviderQueries.has(name.toLowerCase())
|
|
@@ -3898,6 +4225,22 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3898
4225
|
: null;
|
|
3899
4226
|
const withCodex = <T>(fn: () => Promise<T>): Promise<T> =>
|
|
3900
4227
|
codexContext ? codexRequestStorage.run(codexContext, fn) : fn();
|
|
4228
|
+
const withCodexRemoteCompaction = <T>(fn: () => Promise<T>): Promise<T> =>
|
|
4229
|
+
withCodex(() =>
|
|
4230
|
+
withCodexRequestOverrides(
|
|
4231
|
+
{
|
|
4232
|
+
betaFeatures: [REMOTE_COMPACTION_V2_BETA_FEATURE],
|
|
4233
|
+
turnMetadata: {
|
|
4234
|
+
request_kind: "compaction",
|
|
4235
|
+
compaction: {
|
|
4236
|
+
implementation: REMOTE_COMPACTION_V2_IMPLEMENTATION,
|
|
4237
|
+
strategy: "memento",
|
|
4238
|
+
},
|
|
4239
|
+
},
|
|
4240
|
+
},
|
|
4241
|
+
fn,
|
|
4242
|
+
),
|
|
4243
|
+
);
|
|
3901
4244
|
const promptCacheKey = acceptsPromptCacheKeyForTurn(resolvedModel)
|
|
3902
4245
|
? input.sessionId
|
|
3903
4246
|
: undefined;
|
|
@@ -3952,6 +4295,41 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3952
4295
|
...(systemInstructions ? { systemInstructions } : {}),
|
|
3953
4296
|
...(promptCacheKey ? { promptCacheKey } : {}),
|
|
3954
4297
|
});
|
|
4298
|
+
const remoteCompactionRequester =
|
|
4299
|
+
resolvedModel && isCodexTurn
|
|
4300
|
+
? (s: Settings, m: Array<Record<string, unknown>>) =>
|
|
4301
|
+
withCodexRemoteCompaction(() =>
|
|
4302
|
+
requestRemoteCompactionV2(s, m, {
|
|
4303
|
+
client: resolvedModel.client,
|
|
4304
|
+
model: turnExecutionPolicy.upstreamModelId,
|
|
4305
|
+
onUsage: recordCompactionUsage,
|
|
4306
|
+
...(promptCacheKey ? { promptCacheKey } : {}),
|
|
4307
|
+
}),
|
|
4308
|
+
)
|
|
4309
|
+
: undefined;
|
|
4310
|
+
const publishCompactionLiveEvents = async (events: SessionEvent[]) => {
|
|
4311
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events);
|
|
4312
|
+
};
|
|
4313
|
+
const publishCompactionOutcomeEvents = async (events: SessionEvent[]) => {
|
|
4314
|
+
// `compaction.started` was already fanout via publishCompactionLiveEvents.
|
|
4315
|
+
await publishDurableSessionEvents(
|
|
4316
|
+
bus,
|
|
4317
|
+
input.workspaceId,
|
|
4318
|
+
input.sessionId,
|
|
4319
|
+
events.filter((event) => event.type !== "session.context.compaction.started"),
|
|
4320
|
+
);
|
|
4321
|
+
};
|
|
4322
|
+
const compactionModeOptions = {
|
|
4323
|
+
codexCompactionMode: session.codexCompactionMode,
|
|
4324
|
+
isCodexSubscriptionTurn: isCodexTurn,
|
|
4325
|
+
publishLiveEvents: publishCompactionLiveEvents,
|
|
4326
|
+
...(remoteCompactionRequester
|
|
4327
|
+
? { requestRemoteCompactionV2: remoteCompactionRequester }
|
|
4328
|
+
: {}),
|
|
4329
|
+
...(effectiveCodexCredentialId
|
|
4330
|
+
? { producerCodexCredentialId: effectiveCodexCredentialId }
|
|
4331
|
+
: {}),
|
|
4332
|
+
} as const;
|
|
3955
4333
|
|
|
3956
4334
|
if (turn.source === "compaction") {
|
|
3957
4335
|
const persistentSessionSettings = {
|
|
@@ -3997,6 +4375,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3997
4375
|
force: true,
|
|
3998
4376
|
clearRequestedCompaction: true,
|
|
3999
4377
|
trigger: "operator",
|
|
4378
|
+
codexAccount: {
|
|
4379
|
+
currentCodexCredentialId: effectiveCodexCredentialId,
|
|
4380
|
+
},
|
|
4381
|
+
...compactionModeOptions,
|
|
4000
4382
|
},
|
|
4001
4383
|
),
|
|
4002
4384
|
cancellationSignal,
|
|
@@ -4008,6 +4390,24 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4008
4390
|
// request pending and let the ordinary same-turn provider/capacity
|
|
4009
4391
|
// recovery path re-dispatch this exact maintenance execution.
|
|
4010
4392
|
if (shouldRecoverCompactionProviderFailure(error)) throw error;
|
|
4393
|
+
if (error instanceof TurnAttemptFencedError) throw error;
|
|
4394
|
+
// `compaction.started` may already be live — settle skipped before
|
|
4395
|
+
// failing the turn so the timeline cannot stick on Compacting…
|
|
4396
|
+
const landmark = await settleFailedContextCompactionLandmark(
|
|
4397
|
+
db,
|
|
4398
|
+
{
|
|
4399
|
+
accountId: input.accountId,
|
|
4400
|
+
workspaceId: input.workspaceId,
|
|
4401
|
+
sessionId: input.sessionId,
|
|
4402
|
+
turnId: turn.id,
|
|
4403
|
+
executionGeneration,
|
|
4404
|
+
attemptId: input.attemptId,
|
|
4405
|
+
},
|
|
4406
|
+
{
|
|
4407
|
+
clearRequestedCompaction: true,
|
|
4408
|
+
publishLiveEvents: publishCompactionLiveEvents,
|
|
4409
|
+
},
|
|
4410
|
+
);
|
|
4011
4411
|
if (!isCompactionSummaryFailure(error)) throw error;
|
|
4012
4412
|
const errorMessage = String(redact(compactionFailureReasonFromError(error)));
|
|
4013
4413
|
if (
|
|
@@ -4031,7 +4431,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4031
4431
|
turnStatus: "failed",
|
|
4032
4432
|
sessionStatus: "idle",
|
|
4033
4433
|
activeTurnId: null,
|
|
4034
|
-
consumeRequestedCompactionFailure: true,
|
|
4434
|
+
...(landmark.requestConsumed ? {} : { consumeRequestedCompactionFailure: true }),
|
|
4035
4435
|
}))
|
|
4036
4436
|
) {
|
|
4037
4437
|
return claimedResult({ status: "cancelled" });
|
|
@@ -4045,12 +4445,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4045
4445
|
if (outcome.compacted) {
|
|
4046
4446
|
recordContextCompaction(observability, "operator");
|
|
4047
4447
|
}
|
|
4048
|
-
await
|
|
4049
|
-
bus,
|
|
4050
|
-
input.workspaceId,
|
|
4051
|
-
input.sessionId,
|
|
4052
|
-
outcome.events,
|
|
4053
|
-
);
|
|
4448
|
+
await publishCompactionOutcomeEvents(outcome.events);
|
|
4054
4449
|
}
|
|
4055
4450
|
}
|
|
4056
4451
|
if (
|
|
@@ -4084,21 +4479,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4084
4479
|
// narrowed by the session's exact firstPartyMcpTools selection and
|
|
4085
4480
|
// authorization. Idempotent: mergeToolRefs dedupes if already present.
|
|
4086
4481
|
// Resolve the durable policy at the turn boundary. Workspace-default
|
|
4087
|
-
// sessions
|
|
4482
|
+
// sessions follow the current configured MCP set,
|
|
4088
4483
|
// while explicit, inherited-fixed, and legacy sessions remain narrowed
|
|
4089
4484
|
// to their stored materialized allow-list.
|
|
4090
4485
|
const resolvedToolPolicy = resolveSessionToolPolicy({
|
|
4091
4486
|
toolPolicy: session.toolPolicy,
|
|
4092
4487
|
sessionTools: session.tools,
|
|
4093
4488
|
availableMcpServerIds: runSettings.mcpServers.map((server) => server.id),
|
|
4094
|
-
defaultMcpServerIds:
|
|
4095
|
-
|
|
4096
|
-
|
|
4489
|
+
defaultMcpServerIds: defaultSessionMcpServerIds(capabilitySettings.mcpServers),
|
|
4490
|
+
});
|
|
4491
|
+
const mcpAvailabilityInstructions = unavailableMcpTurnInstructions({
|
|
4492
|
+
droppedIds: resolvedToolPolicy.effectivePolicy.droppedIds,
|
|
4493
|
+
droppedCount: resolvedToolPolicy.effectivePolicy.counts.dropped,
|
|
4097
4494
|
});
|
|
4098
4495
|
const effectivePolicyTools = resolvedToolPolicy.toolRefs;
|
|
4099
|
-
defaultNativeToolsAllowed = sessionToolPolicyAllowsDefaultNativeTools(
|
|
4100
|
-
resolvedToolPolicy.effectivePolicy,
|
|
4101
|
-
);
|
|
4102
4496
|
const turnTools = withFirstPartyTools(runSettings, effectivePolicyTools);
|
|
4103
4497
|
// §7.6 connection-credential provider — load (and decrypt) the variable set via the host
|
|
4104
4498
|
// `sandboxSecrets` provider when bound; unset → today's local decrypt.
|
|
@@ -4841,6 +5235,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4841
5235
|
{
|
|
4842
5236
|
db,
|
|
4843
5237
|
settings,
|
|
5238
|
+
cancellationSignal: sandboxResumeSignal,
|
|
4844
5239
|
sandboxMetrics: runtimeMetricsHooksForObservability(observability),
|
|
4845
5240
|
onSandboxLost: publishSandboxLost,
|
|
4846
5241
|
},
|
|
@@ -4858,9 +5253,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4858
5253
|
environment: sandboxEnvironment,
|
|
4859
5254
|
// IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
|
|
4860
5255
|
// this run resolves. The lease stamps it + conflicts on a live shared box
|
|
4861
|
-
// running a DIFFERENT image (solo →
|
|
4862
|
-
// SandboxImageConflictError surfaced as an actionable turn error). Prefer
|
|
4863
|
-
// explicit Modal image ref, else the docker image. The selfhosted branch
|
|
5256
|
+
// running a DIFFERENT image (solo → durable rotation; N-holders →
|
|
5257
|
+
// SandboxImageConflictError surfaced as an actionable turn error). Prefer
|
|
5258
|
+
// the explicit Modal image ref, else the docker image. The selfhosted branch
|
|
4864
5259
|
// (establishSelfhostedTurnSession) NEVER passes
|
|
4865
5260
|
// an image — B3 lives only on this Modal else-branch.
|
|
4866
5261
|
...((runSettings.modalImageRef ?? runSettings.dockerImage)
|
|
@@ -4870,14 +5265,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4870
5265
|
: {}),
|
|
4871
5266
|
// RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
|
|
4872
5267
|
// conflicts on a live shared box set up under a different rig (solo
|
|
4873
|
-
//
|
|
4874
|
-
// turn -> never stamped or enforced (shares exactly as today).
|
|
5268
|
+
// durable rotation / N-holders SandboxRigConflictError). Omitted for a
|
|
5269
|
+
// rig-less turn -> never stamped or enforced (shares exactly as today).
|
|
4875
5270
|
...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
|
|
4876
5271
|
},
|
|
4877
5272
|
"turn",
|
|
4878
5273
|
managedOwnership!.holderId,
|
|
4879
5274
|
),
|
|
4880
|
-
|
|
5275
|
+
sandboxResumeSignal,
|
|
4881
5276
|
async (lateSandbox) => await lateSandbox.release(),
|
|
4882
5277
|
);
|
|
4883
5278
|
setupBoxSession = resolvedSandbox.established.session;
|
|
@@ -4995,6 +5390,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4995
5390
|
...(credentialSubjectId ? { credentialSubjectId } : {}),
|
|
4996
5391
|
resolveCredential,
|
|
4997
5392
|
onAuthNeeded: async (payload) => {
|
|
5393
|
+
if (!shouldPublishToolAuthNeededForTurn(payload, trigger, turn)) {
|
|
5394
|
+
return;
|
|
5395
|
+
}
|
|
4998
5396
|
await publish!([{ type: "tool.auth_needed", payload }], true);
|
|
4999
5397
|
},
|
|
5000
5398
|
// Manager-style sessions carry a creation-validated permission set
|
|
@@ -5045,13 +5443,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5045
5443
|
},
|
|
5046
5444
|
}
|
|
5047
5445
|
: {};
|
|
5048
|
-
const hostedWebSearch = hostedWebSearchForTurn(
|
|
5049
|
-
|
|
5050
|
-
|
|
5051
|
-
|
|
5446
|
+
const hostedWebSearch = hostedWebSearchForTurn(resolvedModel, runSettings.webSearchEnabled);
|
|
5447
|
+
const serviceTier = serviceTierForLatencyMode(
|
|
5448
|
+
turnExecutionPolicy.providerId,
|
|
5449
|
+
turnExecutionPolicy.latencyMode,
|
|
5052
5450
|
);
|
|
5053
5451
|
const agent = runtime.buildAgent(modelRunSettings, turnResources, {
|
|
5054
5452
|
reasoningEffort: turn.reasoningEffort,
|
|
5453
|
+
latencyMode: turnExecutionPolicy.latencyMode,
|
|
5454
|
+
...(serviceTier ? { serviceTier } : {}),
|
|
5055
5455
|
...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
|
|
5056
5456
|
genesisTitleHint: isGenesisTurn,
|
|
5057
5457
|
persistentSessionSettings: {
|
|
@@ -5109,9 +5509,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5109
5509
|
// store/compaction follow the provider's compaction mode (registry
|
|
5110
5510
|
// providers resolve to "client"); encrypted reasoning is only
|
|
5111
5511
|
// round-tripped on the Responses wire API; hosted web search is attached
|
|
5112
|
-
//
|
|
5113
|
-
//
|
|
5114
|
-
//
|
|
5512
|
+
// whenever the provider declares it runnable and is independent of the
|
|
5513
|
+
// session's MCP allow-list; the effective context window drives the
|
|
5514
|
+
// compaction threshold.
|
|
5115
5515
|
hostedWebSearch,
|
|
5116
5516
|
...(resolvedModel
|
|
5117
5517
|
? {
|
|
@@ -5131,7 +5531,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5131
5531
|
// default.
|
|
5132
5532
|
structuredToolTransport: resolvedModel.provider.kind !== "codex-subscription",
|
|
5133
5533
|
// EXPLICIT computer-use tool transport, derived from the resolved provider's
|
|
5134
|
-
// authoritative wire identity (codex → function-image, chat →
|
|
5534
|
+
// authoritative wire identity (codex → function-image, chat → disabled,
|
|
5135
5535
|
// responses → hosted) so the runtime never string-sniffs the model instance's
|
|
5136
5536
|
// constructor name. See {@link computerToolModeForTurn}.
|
|
5137
5537
|
computerToolMode: computerToolModeForTurn(resolvedModel),
|
|
@@ -5174,7 +5574,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5174
5574
|
...(session.instructions ? { sessionInstructions: session.instructions } : {}),
|
|
5175
5575
|
// Exact host context captured when this turn was accepted. It is
|
|
5176
5576
|
// system-level and disappears with the turn rather than entering chat.
|
|
5177
|
-
...(turn.turnInstructions
|
|
5577
|
+
...(turn.turnInstructions || mcpAvailabilityInstructions
|
|
5578
|
+
? {
|
|
5579
|
+
turnInstructions: [turn.turnInstructions, mcpAvailabilityInstructions]
|
|
5580
|
+
.filter((value): value is string => Boolean(value))
|
|
5581
|
+
.join(" "),
|
|
5582
|
+
}
|
|
5583
|
+
: {}),
|
|
5178
5584
|
...workspaceEnvironmentOption,
|
|
5179
5585
|
// RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
|
|
5180
5586
|
// the frozen version carries a non-empty script), and the rig credential
|
|
@@ -5235,6 +5641,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5235
5641
|
{
|
|
5236
5642
|
db,
|
|
5237
5643
|
settings,
|
|
5644
|
+
cancellationSignal: sandboxResumeSignal,
|
|
5238
5645
|
sandboxMetrics: runtimeMetricsHooksForObservability(observability),
|
|
5239
5646
|
onSandboxLost: publishSandboxLost,
|
|
5240
5647
|
},
|
|
@@ -5282,10 +5689,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5282
5689
|
},
|
|
5283
5690
|
...(lazyGitTokens ? { gitTokenSeedsOverride: lazyGitTokens } : {}),
|
|
5284
5691
|
...(lazyGitCredentials?.bindings
|
|
5285
|
-
? {
|
|
5692
|
+
? {
|
|
5693
|
+
gitCredentialBindingsOverride: lazyGitCredentials.bindings,
|
|
5694
|
+
}
|
|
5286
5695
|
: {}),
|
|
5287
5696
|
...(lazyToolspaceToken
|
|
5288
|
-
? {
|
|
5697
|
+
? {
|
|
5698
|
+
toolspaceTokenSeedOverride: lazyToolspaceToken.token,
|
|
5699
|
+
}
|
|
5289
5700
|
: {}),
|
|
5290
5701
|
...(toolCancellationFenceRef.current
|
|
5291
5702
|
? {
|
|
@@ -5318,7 +5729,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5318
5729
|
return provisioned;
|
|
5319
5730
|
},
|
|
5320
5731
|
{
|
|
5321
|
-
|
|
5732
|
+
signal: sandboxResumeSignal,
|
|
5322
5733
|
onStarted: async () => {
|
|
5323
5734
|
await publish?.(
|
|
5324
5735
|
[
|
|
@@ -5345,7 +5756,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5345
5756
|
],
|
|
5346
5757
|
true,
|
|
5347
5758
|
);
|
|
5348
|
-
throwIfTurnOperationCancelled(
|
|
5759
|
+
throwIfTurnOperationCancelled(sandboxResumeSignal);
|
|
5349
5760
|
startLeaseHeartbeat(provisioned, activeSandboxBackend ?? groupBoxBackend);
|
|
5350
5761
|
setupBoxSession = provisioned.established.session;
|
|
5351
5762
|
resolvedSandbox = provisioned;
|
|
@@ -5444,24 +5855,47 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5444
5855
|
force: true,
|
|
5445
5856
|
clearRequestedCompaction: true,
|
|
5446
5857
|
trigger: "operator",
|
|
5858
|
+
codexAccount: {
|
|
5859
|
+
currentCodexCredentialId: effectiveCodexCredentialId,
|
|
5860
|
+
},
|
|
5861
|
+
...compactionModeOptions,
|
|
5447
5862
|
}
|
|
5448
|
-
: {
|
|
5863
|
+
: {
|
|
5864
|
+
trigger: "auto",
|
|
5865
|
+
codexAccount: {
|
|
5866
|
+
currentCodexCredentialId: effectiveCodexCredentialId,
|
|
5867
|
+
},
|
|
5868
|
+
...compactionModeOptions,
|
|
5869
|
+
},
|
|
5449
5870
|
),
|
|
5450
5871
|
cancellationSignal,
|
|
5451
5872
|
undefined,
|
|
5452
5873
|
);
|
|
5453
|
-
if (outcome.
|
|
5454
|
-
|
|
5455
|
-
|
|
5456
|
-
|
|
5457
|
-
|
|
5458
|
-
|
|
5459
|
-
input.sessionId,
|
|
5460
|
-
outcome.events,
|
|
5461
|
-
);
|
|
5874
|
+
if (outcome.events.length > 0) {
|
|
5875
|
+
if (outcome.compacted) {
|
|
5876
|
+
const compactionTrigger = forced ? "operator" : undefined;
|
|
5877
|
+
recordContextCompaction(observability, compactionTrigger ?? "auto");
|
|
5878
|
+
}
|
|
5879
|
+
await publishCompactionOutcomeEvents(outcome.events);
|
|
5462
5880
|
}
|
|
5463
5881
|
} catch (compactError) {
|
|
5464
5882
|
if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
|
|
5883
|
+
if (compactError instanceof TurnAttemptFencedError) throw compactError;
|
|
5884
|
+
const landmark = await settleFailedContextCompactionLandmark(
|
|
5885
|
+
db,
|
|
5886
|
+
{
|
|
5887
|
+
accountId: input.accountId,
|
|
5888
|
+
workspaceId: input.workspaceId,
|
|
5889
|
+
sessionId: input.sessionId,
|
|
5890
|
+
turnId: turnId!,
|
|
5891
|
+
executionGeneration,
|
|
5892
|
+
attemptId: input.attemptId,
|
|
5893
|
+
},
|
|
5894
|
+
{
|
|
5895
|
+
clearRequestedCompaction: forced,
|
|
5896
|
+
publishLiveEvents: publishCompactionLiveEvents,
|
|
5897
|
+
},
|
|
5898
|
+
);
|
|
5465
5899
|
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
5466
5900
|
const errorMessage = String(redact(compactionFailureReasonFromError(compactError)));
|
|
5467
5901
|
observability.error("context compaction failed", {
|
|
@@ -5487,7 +5921,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5487
5921
|
turnStatus: "failed",
|
|
5488
5922
|
sessionStatus: "idle",
|
|
5489
5923
|
activeTurnId: null,
|
|
5490
|
-
...(forced
|
|
5924
|
+
...(forced && !landmark.requestConsumed
|
|
5925
|
+
? { consumeRequestedCompactionFailure: true }
|
|
5926
|
+
: {}),
|
|
5491
5927
|
}))
|
|
5492
5928
|
) {
|
|
5493
5929
|
return claimedResult({ status: "cancelled" });
|
|
@@ -5663,6 +6099,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5663
6099
|
force: true,
|
|
5664
6100
|
...(triggerLabel === "operator" ? { clearRequestedCompaction: true } : {}),
|
|
5665
6101
|
trigger: triggerLabel,
|
|
6102
|
+
codexAccount: {
|
|
6103
|
+
currentCodexCredentialId: effectiveCodexCredentialId,
|
|
6104
|
+
},
|
|
6105
|
+
...compactionModeOptions,
|
|
5666
6106
|
},
|
|
5667
6107
|
),
|
|
5668
6108
|
cancellationSignal,
|
|
@@ -5672,12 +6112,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5672
6112
|
if (outcome.compacted) {
|
|
5673
6113
|
recordContextCompaction(observability, triggerLabel);
|
|
5674
6114
|
}
|
|
5675
|
-
await
|
|
5676
|
-
bus,
|
|
5677
|
-
input.workspaceId,
|
|
5678
|
-
input.sessionId,
|
|
5679
|
-
outcome.events,
|
|
5680
|
-
);
|
|
6115
|
+
await publishCompactionOutcomeEvents(outcome.events);
|
|
5681
6116
|
}
|
|
5682
6117
|
return outcome;
|
|
5683
6118
|
};
|
|
@@ -5750,8 +6185,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5750
6185
|
initialGitCredentials,
|
|
5751
6186
|
);
|
|
5752
6187
|
}
|
|
6188
|
+
// Conservative request boundary: once control enters runStream, the
|
|
6189
|
+
// provider may have accepted work even if no response/event follows.
|
|
6190
|
+
// Escaped MCP setup timeouts are automatically recoverable only
|
|
6191
|
+
// before this line.
|
|
6192
|
+
modelRequestStarted = true;
|
|
5753
6193
|
return await runtime.runStream(agent, runInput!, modelRunSettings, {
|
|
5754
|
-
|
|
6194
|
+
signal: runtimeCancellationSignal,
|
|
5755
6195
|
sandboxEnvironment,
|
|
5756
6196
|
onRuntimeEvent: async (event) => {
|
|
5757
6197
|
await renewCodexLease("runtime_event");
|
|
@@ -5870,6 +6310,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5870
6310
|
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
5871
6311
|
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
5872
6312
|
model: turn.model,
|
|
6313
|
+
latencyMode: turnExecutionPolicy.latencyMode,
|
|
5873
6314
|
metricProvider: streamProvider,
|
|
5874
6315
|
externallyBilled: isExternallyBilledTurn,
|
|
5875
6316
|
servingCredentialId: effectiveCodexCredentialId,
|
|
@@ -5880,6 +6321,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5880
6321
|
leaseLostMessage: "Codex credential lease expired during the active turn",
|
|
5881
6322
|
setLastInputTokens: setLastInputTokensFenced,
|
|
5882
6323
|
});
|
|
6324
|
+
assertModelResponseLatencyMode({
|
|
6325
|
+
event: next.value,
|
|
6326
|
+
requested: turnExecutionPolicy.latencyMode,
|
|
6327
|
+
model: turn.model,
|
|
6328
|
+
...(resolvedModel?.provider.id ? { providerId: resolvedModel.provider.id } : {}),
|
|
6329
|
+
});
|
|
5883
6330
|
if (responseUsageResult.status === "processed") {
|
|
5884
6331
|
currentToolBatchCallIds = new Set<string>();
|
|
5885
6332
|
currentToolBatchCompletedCallIds = new Set<string>();
|
|
@@ -6043,7 +6490,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6043
6490
|
leaseLost: () => codexLeaseLost,
|
|
6044
6491
|
leaseLostMessage: "Codex credential lease expired during the active turn",
|
|
6045
6492
|
recordUsage: async () => {
|
|
6046
|
-
await recordModelUsageAndDebitCredits(settings, db, {
|
|
6493
|
+
const billing = await recordModelUsageAndDebitCredits(settings, db, {
|
|
6047
6494
|
accountId: input.accountId,
|
|
6048
6495
|
workspaceId: input.workspaceId,
|
|
6049
6496
|
sessionId: input.sessionId,
|
|
@@ -6054,8 +6501,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6054
6501
|
usage: aggregateUsage,
|
|
6055
6502
|
normalizedUsage: normalizedAggregateUsage,
|
|
6056
6503
|
sourceKey: aggregateSourceKey,
|
|
6504
|
+
latencyMode: turnExecutionPolicy.latencyMode,
|
|
6057
6505
|
observability,
|
|
6058
6506
|
});
|
|
6507
|
+
const aggregateProvider = resolvedModel?.provider.id ?? settings.openaiProvider;
|
|
6508
|
+
const aggregateProviderApi = resolvedModel?.provider.api ?? "responses";
|
|
6059
6509
|
aggregateAuthoritative = await emitModelCallUsage({
|
|
6060
6510
|
observability,
|
|
6061
6511
|
publish,
|
|
@@ -6063,8 +6513,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6063
6513
|
workspaceId: input.workspaceId,
|
|
6064
6514
|
sessionId: input.sessionId,
|
|
6065
6515
|
turnId: activeTurnId,
|
|
6066
|
-
provider:
|
|
6067
|
-
providerApi:
|
|
6516
|
+
provider: aggregateProvider,
|
|
6517
|
+
providerApi: aggregateProviderApi,
|
|
6068
6518
|
model: turn.model,
|
|
6069
6519
|
sourceKey: aggregateSourceKey,
|
|
6070
6520
|
usage: { usage: aggregateUsage },
|
|
@@ -6073,6 +6523,22 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6073
6523
|
accountChangedFromPrevCall: aggregateAccountCtx.accountChangedFromPrevCall,
|
|
6074
6524
|
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
6075
6525
|
});
|
|
6526
|
+
if (aggregateAuthoritative && billing) {
|
|
6527
|
+
await recordAuthoritativeModelCallFact({
|
|
6528
|
+
db,
|
|
6529
|
+
observability,
|
|
6530
|
+
accountId: input.accountId,
|
|
6531
|
+
workspaceId: input.workspaceId,
|
|
6532
|
+
sessionId: input.sessionId,
|
|
6533
|
+
turnId: activeTurnId,
|
|
6534
|
+
turnAttemptId: input.attemptId,
|
|
6535
|
+
sourceKey: aggregateSourceKey,
|
|
6536
|
+
provider: aggregateProvider,
|
|
6537
|
+
providerApi: aggregateProviderApi,
|
|
6538
|
+
model: turn.model,
|
|
6539
|
+
billing,
|
|
6540
|
+
});
|
|
6541
|
+
}
|
|
6076
6542
|
if (aggregateAuthoritative && aggregateInput !== null && aggregateInput > 0) {
|
|
6077
6543
|
recordModelInputTokens(observability, streamProvider, aggregateInput);
|
|
6078
6544
|
}
|
|
@@ -6145,7 +6611,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6145
6611
|
events: [
|
|
6146
6612
|
...requestEvents,
|
|
6147
6613
|
...(approvals.length > 0
|
|
6148
|
-
? [
|
|
6614
|
+
? [
|
|
6615
|
+
{
|
|
6616
|
+
type: "session.requiresAction" as const,
|
|
6617
|
+
payload: { approvals },
|
|
6618
|
+
},
|
|
6619
|
+
]
|
|
6149
6620
|
: []),
|
|
6150
6621
|
{
|
|
6151
6622
|
type: "session.status.changed",
|
|
@@ -6260,6 +6731,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6260
6731
|
let compacted = false;
|
|
6261
6732
|
let compactionHandled = false;
|
|
6262
6733
|
let compactionFailureMessage: string | null = null;
|
|
6734
|
+
let compactionRequestCleared = false;
|
|
6263
6735
|
try {
|
|
6264
6736
|
const outcome = await forceContextCompaction(
|
|
6265
6737
|
recoveryKind,
|
|
@@ -6279,6 +6751,23 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6279
6751
|
// turn through the normal provider/capacity path. They are not an
|
|
6280
6752
|
// empty summary and must not create a new goal continuation.
|
|
6281
6753
|
if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
|
|
6754
|
+
if (compactError instanceof TurnAttemptFencedError) throw compactError;
|
|
6755
|
+
const landmark = await settleFailedContextCompactionLandmark(
|
|
6756
|
+
db,
|
|
6757
|
+
{
|
|
6758
|
+
accountId: input.accountId,
|
|
6759
|
+
workspaceId: input.workspaceId,
|
|
6760
|
+
sessionId: input.sessionId,
|
|
6761
|
+
turnId: activeTurnId!,
|
|
6762
|
+
executionGeneration,
|
|
6763
|
+
attemptId: input.attemptId,
|
|
6764
|
+
},
|
|
6765
|
+
{
|
|
6766
|
+
clearRequestedCompaction: recoveryKind === "operator",
|
|
6767
|
+
publishLiveEvents: publishCompactionLiveEvents,
|
|
6768
|
+
},
|
|
6769
|
+
);
|
|
6770
|
+
compactionRequestCleared = landmark.requestConsumed;
|
|
6282
6771
|
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
6283
6772
|
compactionFailureMessage = String(
|
|
6284
6773
|
redact(compactionFailureReasonFromError(compactError)),
|
|
@@ -6314,7 +6803,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6314
6803
|
turnStatus: "failed",
|
|
6315
6804
|
sessionStatus: "idle",
|
|
6316
6805
|
activeTurnId: null,
|
|
6317
|
-
...(recoveryKind === "operator"
|
|
6806
|
+
...(recoveryKind === "operator" && !compactionRequestCleared
|
|
6807
|
+
? { consumeRequestedCompactionFailure: true }
|
|
6808
|
+
: {}),
|
|
6318
6809
|
}))
|
|
6319
6810
|
) {
|
|
6320
6811
|
return claimedResult({ status: "cancelled" });
|
|
@@ -6365,12 +6856,29 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6365
6856
|
// so its next attempt reattaches under the current epoch.
|
|
6366
6857
|
if (error instanceof SandboxLeaseSupersededError && recoveryTurnId) {
|
|
6367
6858
|
try {
|
|
6859
|
+
const fencedLease = await readLease(db, input.workspaceId, error.sandboxGroupId).catch(
|
|
6860
|
+
() => null,
|
|
6861
|
+
);
|
|
6862
|
+
const rotationPending = fencedLease?.rotationRequestedAt != null;
|
|
6863
|
+
const deadlineRotationPending =
|
|
6864
|
+
rotationPending && fencedLease?.rotationReason === "provider_deadline";
|
|
6368
6865
|
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
6369
6866
|
sessionId: input.sessionId,
|
|
6370
6867
|
turnId: recoveryTurnId,
|
|
6371
6868
|
triggerEventId: triggerEventId!,
|
|
6372
6869
|
attemptId: input.attemptId,
|
|
6373
|
-
reason:
|
|
6870
|
+
reason: deadlineRotationPending
|
|
6871
|
+
? "sandbox_deadline_rotation"
|
|
6872
|
+
: "sandbox_lease_superseded",
|
|
6873
|
+
...(rotationPending
|
|
6874
|
+
? {
|
|
6875
|
+
detail: {
|
|
6876
|
+
sandboxGroupId: error.sandboxGroupId,
|
|
6877
|
+
leaseEpoch: error.leaseEpoch,
|
|
6878
|
+
rotationReason: fencedLease?.rotationReason ?? "operator",
|
|
6879
|
+
},
|
|
6880
|
+
}
|
|
6881
|
+
: {}),
|
|
6374
6882
|
});
|
|
6375
6883
|
if (recovery.action === "stale") {
|
|
6376
6884
|
acknowledgeLostAttemptOwnership();
|
|
@@ -6386,7 +6894,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6386
6894
|
);
|
|
6387
6895
|
activityStatus = "recovering";
|
|
6388
6896
|
turnMetricOutcome = "recovering";
|
|
6389
|
-
return claimedResult({
|
|
6897
|
+
return claimedResult({
|
|
6898
|
+
status: "recovering",
|
|
6899
|
+
...(rotationPending
|
|
6900
|
+
? {
|
|
6901
|
+
continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings),
|
|
6902
|
+
}
|
|
6903
|
+
: {}),
|
|
6904
|
+
});
|
|
6390
6905
|
} catch (recoveryError) {
|
|
6391
6906
|
console.error(
|
|
6392
6907
|
"sandbox lease supersession recovery failed",
|
|
@@ -6395,6 +6910,53 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6395
6910
|
throw recoveryError;
|
|
6396
6911
|
}
|
|
6397
6912
|
}
|
|
6913
|
+
if (
|
|
6914
|
+
sandboxRotationController.signal.aborted &&
|
|
6915
|
+
sandboxRotationController.signal.reason instanceof SandboxDeadlineRotationError &&
|
|
6916
|
+
!cancellationSignal?.aborted &&
|
|
6917
|
+
recoveryTurnId
|
|
6918
|
+
) {
|
|
6919
|
+
try {
|
|
6920
|
+
await flushRuntimeBatcher();
|
|
6921
|
+
await reconcileConversationTruth({ requireDurable: true });
|
|
6922
|
+
const rotation = sandboxRotationController.signal.reason;
|
|
6923
|
+
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
6924
|
+
sessionId: input.sessionId,
|
|
6925
|
+
turnId: recoveryTurnId,
|
|
6926
|
+
triggerEventId: triggerEventId!,
|
|
6927
|
+
attemptId: input.attemptId,
|
|
6928
|
+
reason: "sandbox_deadline_rotation",
|
|
6929
|
+
detail: {
|
|
6930
|
+
sandboxGroupId: rotation.sandboxGroupId,
|
|
6931
|
+
leaseEpoch: rotation.leaseEpoch,
|
|
6932
|
+
},
|
|
6933
|
+
});
|
|
6934
|
+
if (recovery.action === "stale") {
|
|
6935
|
+
acknowledgeLostAttemptOwnership();
|
|
6936
|
+
activityStatus = "cancelled";
|
|
6937
|
+
turnMetricOutcome = "cancelled";
|
|
6938
|
+
return claimedResult({ status: "cancelled" });
|
|
6939
|
+
}
|
|
6940
|
+
await publishDurableSessionEvents(
|
|
6941
|
+
bus,
|
|
6942
|
+
input.workspaceId,
|
|
6943
|
+
input.sessionId,
|
|
6944
|
+
recovery.events,
|
|
6945
|
+
);
|
|
6946
|
+
activityStatus = "recovering";
|
|
6947
|
+
turnMetricOutcome = "recovering";
|
|
6948
|
+
return claimedResult({
|
|
6949
|
+
status: "recovering",
|
|
6950
|
+
continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings),
|
|
6951
|
+
});
|
|
6952
|
+
} catch (recoveryError) {
|
|
6953
|
+
console.error(
|
|
6954
|
+
"sandbox deadline rotation recovery failed",
|
|
6955
|
+
safeErrorDiagnostic(recoveryError, (value) => String(redact(value))),
|
|
6956
|
+
);
|
|
6957
|
+
throw recoveryError;
|
|
6958
|
+
}
|
|
6959
|
+
}
|
|
6398
6960
|
const cancellationFailure = turnOperationCancellationFailure(error);
|
|
6399
6961
|
if (
|
|
6400
6962
|
cancellationFailure &&
|
|
@@ -7155,6 +7717,62 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7155
7717
|
activityStatus = "idle";
|
|
7156
7718
|
return claimedResult({ status: "idle" });
|
|
7157
7719
|
}
|
|
7720
|
+
// The Codex backend can reject an opaque reasoning artifact that it
|
|
7721
|
+
// minted on the immediately preceding successful request, even when the
|
|
7722
|
+
// credential-row UUID is unchanged. HTTP 400 + the exact provider
|
|
7723
|
+
// semantic proves this request never entered inference. Atomically mark
|
|
7724
|
+
// the active opaque artifacts rejected and recover the SAME logical turn
|
|
7725
|
+
// from durable history; messages, tool calls/results, summaries, and the
|
|
7726
|
+
// original audit rows remain intact. If no artifact can be invalidated,
|
|
7727
|
+
// fall through to the terminal path rather than resend an equivalent
|
|
7728
|
+
// request forever.
|
|
7729
|
+
const encryptedArtifactRejection =
|
|
7730
|
+
isCodexTurn && effectiveCodexCredentialId
|
|
7731
|
+
? classifyCodexEncryptedArtifactRejection(error)
|
|
7732
|
+
: null;
|
|
7733
|
+
if (
|
|
7734
|
+
encryptedArtifactRejection &&
|
|
7735
|
+
effectiveCodexCredentialId &&
|
|
7736
|
+
publish &&
|
|
7737
|
+
turnId &&
|
|
7738
|
+
turnStartedPublished
|
|
7739
|
+
) {
|
|
7740
|
+
await flushRuntimeBatcher();
|
|
7741
|
+
await reconcileConversationTruth({ requireDurable: true });
|
|
7742
|
+
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
7743
|
+
sessionId: input.sessionId,
|
|
7744
|
+
turnId,
|
|
7745
|
+
triggerEventId: triggerEventId!,
|
|
7746
|
+
attemptId: input.attemptId,
|
|
7747
|
+
reason: encryptedArtifactRejection.kind,
|
|
7748
|
+
detail: {
|
|
7749
|
+
code: encryptedArtifactRejection.kind,
|
|
7750
|
+
retryable: true,
|
|
7751
|
+
},
|
|
7752
|
+
providerArtifactInvalidation: {
|
|
7753
|
+
codexCredentialId: effectiveCodexCredentialId,
|
|
7754
|
+
reason: encryptedArtifactRejection.kind,
|
|
7755
|
+
},
|
|
7756
|
+
});
|
|
7757
|
+
if (recovery.action === "stale") {
|
|
7758
|
+
acknowledgeLostAttemptOwnership();
|
|
7759
|
+
activityStatus = "cancelled";
|
|
7760
|
+
turnMetricOutcome = "cancelled";
|
|
7761
|
+
return claimedResult({ status: "cancelled" });
|
|
7762
|
+
}
|
|
7763
|
+
if (recovery.action === "recovering") {
|
|
7764
|
+
await publishDurableSessionEvents(
|
|
7765
|
+
bus,
|
|
7766
|
+
input.workspaceId,
|
|
7767
|
+
input.sessionId,
|
|
7768
|
+
recovery.events,
|
|
7769
|
+
);
|
|
7770
|
+
turnMetricOutcome = "recovering";
|
|
7771
|
+
activityStatus = "recovering";
|
|
7772
|
+
activityError = error;
|
|
7773
|
+
return claimedResult({ status: "recovering" });
|
|
7774
|
+
}
|
|
7775
|
+
}
|
|
7158
7776
|
// A retryable provider/MCP failure is transient external backpressure,
|
|
7159
7777
|
// not a session or goal failure. The in-client retry budget is already
|
|
7160
7778
|
// exhausted by the time the error reaches here. Checkpoint conversation
|
|
@@ -7193,31 +7811,55 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7193
7811
|
});
|
|
7194
7812
|
}
|
|
7195
7813
|
if (failure.retryable && publish && turnId && turnStartedPublished) {
|
|
7196
|
-
|
|
7197
|
-
|
|
7198
|
-
|
|
7199
|
-
|
|
7200
|
-
|
|
7201
|
-
|
|
7202
|
-
|
|
7203
|
-
|
|
7204
|
-
|
|
7205
|
-
|
|
7206
|
-
|
|
7207
|
-
|
|
7208
|
-
|
|
7209
|
-
|
|
7210
|
-
|
|
7211
|
-
|
|
7212
|
-
|
|
7213
|
-
|
|
7214
|
-
|
|
7814
|
+
try {
|
|
7815
|
+
const recoveryResult = providerRecoveryResult();
|
|
7816
|
+
await flushRuntimeBatcher();
|
|
7817
|
+
await reconcileConversationTruth({ requireDurable: true });
|
|
7818
|
+
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
7819
|
+
sessionId: input.sessionId,
|
|
7820
|
+
turnId,
|
|
7821
|
+
triggerEventId: triggerEventId!,
|
|
7822
|
+
attemptId: input.attemptId,
|
|
7823
|
+
reason: failure.code ?? "provider_unavailable",
|
|
7824
|
+
detail: {
|
|
7825
|
+
...failure,
|
|
7826
|
+
continueDelayMs: recoveryResult.continueDelayMs,
|
|
7827
|
+
},
|
|
7828
|
+
});
|
|
7829
|
+
if (recovery.action === "stale") {
|
|
7830
|
+
acknowledgeLostAttemptOwnership();
|
|
7831
|
+
activityStatus = "cancelled";
|
|
7832
|
+
turnMetricOutcome = "cancelled";
|
|
7833
|
+
return claimedResult({ status: "cancelled" });
|
|
7834
|
+
}
|
|
7835
|
+
await publishDurableSessionEvents(
|
|
7836
|
+
bus,
|
|
7837
|
+
input.workspaceId,
|
|
7838
|
+
input.sessionId,
|
|
7839
|
+
recovery.events,
|
|
7840
|
+
);
|
|
7841
|
+
turnMetricOutcome = "recovering";
|
|
7842
|
+
activityStatus = "recovering";
|
|
7843
|
+
activityError = error;
|
|
7844
|
+
return claimedResult(recoveryResult);
|
|
7845
|
+
} catch (recoveryError) {
|
|
7846
|
+
const escaped = escapedMcpTimeoutRecoveryFailure({
|
|
7847
|
+
failureCode: failure.code,
|
|
7848
|
+
modelRequestStarted,
|
|
7849
|
+
detail: {
|
|
7850
|
+
turnId,
|
|
7851
|
+
triggerEventId: triggerEventId!,
|
|
7852
|
+
executionGeneration,
|
|
7853
|
+
},
|
|
7854
|
+
});
|
|
7855
|
+
if (escaped) {
|
|
7856
|
+
activityStatus = "recovering";
|
|
7857
|
+
turnMetricOutcome = "recovering";
|
|
7858
|
+
activityError = error;
|
|
7859
|
+
throw escaped;
|
|
7860
|
+
}
|
|
7861
|
+
throw recoveryError;
|
|
7215
7862
|
}
|
|
7216
|
-
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
7217
|
-
turnMetricOutcome = "recovering";
|
|
7218
|
-
activityStatus = "recovering";
|
|
7219
|
-
activityError = error;
|
|
7220
|
-
return claimedResult(recoveryResult);
|
|
7221
7863
|
}
|
|
7222
7864
|
activityStatus = "failed";
|
|
7223
7865
|
activityError = error;
|
|
@@ -7260,6 +7902,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7260
7902
|
);
|
|
7261
7903
|
return claimedResult({ status: "failed" });
|
|
7262
7904
|
} finally {
|
|
7905
|
+
// This is the logical ownership boundary. Abort before any fallible
|
|
7906
|
+
// housekeeping so a still-pending provider establish releases its private
|
|
7907
|
+
// holder/timer even when Temporal itself never delivered cancellation.
|
|
7908
|
+
if (resolvedSandbox === null && !sandboxResumeController.signal.aborted) {
|
|
7909
|
+
sandboxResumeController.abort(
|
|
7910
|
+
cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
|
|
7911
|
+
);
|
|
7912
|
+
}
|
|
7263
7913
|
const finalizationStarted = performance.now();
|
|
7264
7914
|
let finalizationError: unknown;
|
|
7265
7915
|
let physicalToolQuiescenceConfirmed = !acknowledgeQuiescence;
|
|
@@ -7486,7 +8136,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7486
8136
|
// gives capture the full live-box margin instead of racing the teardown
|
|
7487
8137
|
// tail, which was dropping 100% of captures on real Modal desktop boxes
|
|
7488
8138
|
// ("request cancelled due to container exiting", 0 rows). External module:
|
|
7489
|
-
// self-capped at
|
|
8139
|
+
// self-capped at 120s, best-effort (never throws past its boundary),
|
|
7490
8140
|
// epoch-fenced, and it NEVER closes the box. The emitted
|
|
7491
8141
|
// workspace.revision.captured event is ANNOUNCE-ONLY (metadata, never
|
|
7492
8142
|
// content).
|
|
@@ -7506,14 +8156,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7506
8156
|
setupBoxSession &&
|
|
7507
8157
|
sandboxGroupId
|
|
7508
8158
|
) {
|
|
7509
|
-
//
|
|
7510
|
-
//
|
|
7511
|
-
//
|
|
7512
|
-
//
|
|
7513
|
-
|
|
7514
|
-
clearInterval(leaseHeartbeatTimer);
|
|
7515
|
-
leaseHeartbeatTimer = undefined;
|
|
7516
|
-
}
|
|
8159
|
+
// Block new periodic snapshots, then drain any one already in flight.
|
|
8160
|
+
// Keep the lease heartbeat itself running: capture may legitimately
|
|
8161
|
+
// exceed the 90s holder TTL, and the reaper must remain unable to drain
|
|
8162
|
+
// the exact box while this holder still reads it.
|
|
8163
|
+
turnEndCaptureInProgress = true;
|
|
7517
8164
|
if (snapshotInFlight) {
|
|
7518
8165
|
await waitForWarmSnapshot(
|
|
7519
8166
|
snapshotInFlight,
|
|
@@ -7521,6 +8168,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7521
8168
|
finalizerSignal,
|
|
7522
8169
|
);
|
|
7523
8170
|
}
|
|
8171
|
+
const captureEstablished = resolvedSandbox.established;
|
|
7524
8172
|
await captureWorkspaceRevision({
|
|
7525
8173
|
db,
|
|
7526
8174
|
objectStorage,
|
|
@@ -7529,6 +8177,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7529
8177
|
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events);
|
|
7530
8178
|
},
|
|
7531
8179
|
session: setupBoxSession as ChannelASession,
|
|
8180
|
+
openReadSession: async () =>
|
|
8181
|
+
await openFreshWorkspaceCaptureSession({
|
|
8182
|
+
backendId: captureEstablished.backendId,
|
|
8183
|
+
client: captureEstablished.client,
|
|
8184
|
+
session: setupBoxSession as ChannelASession,
|
|
8185
|
+
expectedInstanceId: captureEstablished.instanceId,
|
|
8186
|
+
}),
|
|
7532
8187
|
leaseEpoch: resolvedSandbox.leaseEpoch,
|
|
7533
8188
|
sandboxGroupId,
|
|
7534
8189
|
accountId: input.accountId,
|
|
@@ -7562,7 +8217,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7562
8217
|
// box rides the provider idle-timeout in the meantime. Best-effort: a
|
|
7563
8218
|
// release failure must never mask the turn's real outcome.
|
|
7564
8219
|
if (leaseHeartbeatTimer) {
|
|
7565
|
-
|
|
8220
|
+
stopLeaseHeartbeat();
|
|
8221
|
+
}
|
|
8222
|
+
if (rotationInFlight) {
|
|
8223
|
+
await waitForTurnFinalizerStep(rotationInFlight, finalizerSignal).catch(() => undefined);
|
|
7566
8224
|
}
|
|
7567
8225
|
// A recording normally closes inside the attempt-fenced turn settlement.
|
|
7568
8226
|
// Reaching finally with one still active means settlement threw, never ran,
|
|
@@ -7643,6 +8301,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7643
8301
|
safeErrorDiagnostic(error, (value) => String(redact(value))),
|
|
7644
8302
|
);
|
|
7645
8303
|
} finally {
|
|
8304
|
+
// If fallible finalization exited before the ordinary exact-holder
|
|
8305
|
+
// release, the resume signal is the unconditional last line of defense.
|
|
8306
|
+
// Normal finalization already removed its listener in release().
|
|
8307
|
+
if (!sandboxResumeController.signal.aborted) {
|
|
8308
|
+
sandboxResumeController.abort(
|
|
8309
|
+
cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
|
|
8310
|
+
);
|
|
8311
|
+
}
|
|
7646
8312
|
cancellationSignal?.removeEventListener("abort", noteCancellationRequested);
|
|
7647
8313
|
const completedAt = performance.now();
|
|
7648
8314
|
const durationSeconds = (completedAt - activityStarted) / 1000;
|
|
@@ -8266,6 +8932,13 @@ export async function ensureRunAllowed(
|
|
|
8266
8932
|
}
|
|
8267
8933
|
}
|
|
8268
8934
|
|
|
8935
|
+
export type ModelUsageBillingRecord = {
|
|
8936
|
+
billingPath: "opengeni_credits" | "external";
|
|
8937
|
+
/** Same quantity written to usage_events.model.cost when present; else 0. */
|
|
8938
|
+
pricedCostMicros: number;
|
|
8939
|
+
normalizedUsage: ModelCallUsageNormalization;
|
|
8940
|
+
};
|
|
8941
|
+
|
|
8269
8942
|
// Exported for unit testing the external-billing bypass; not part of the activity surface.
|
|
8270
8943
|
export async function recordModelUsageAndDebitCredits(
|
|
8271
8944
|
settings: Settings,
|
|
@@ -8281,11 +8954,12 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
8281
8954
|
usage?: ModelUsageInput | ModelCallUsageInput | null;
|
|
8282
8955
|
normalizedUsage?: ModelCallUsageNormalization;
|
|
8283
8956
|
sourceKey: string;
|
|
8957
|
+
latencyMode?: LatencyMode;
|
|
8284
8958
|
observability?: ActivityServices["observability"];
|
|
8285
8959
|
},
|
|
8286
|
-
): Promise<
|
|
8960
|
+
): Promise<ModelUsageBillingRecord | null> {
|
|
8287
8961
|
if (!input.usage) {
|
|
8288
|
-
return;
|
|
8962
|
+
return null;
|
|
8289
8963
|
}
|
|
8290
8964
|
const normalizedUsage = input.normalizedUsage ?? normalizeModelCallUsage(input.usage);
|
|
8291
8965
|
const sanitizedUsage = sanitizedModelUsageInput(normalizedUsage);
|
|
@@ -8315,7 +8989,11 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
8315
8989
|
turnAttemptId: input.turnAttemptId,
|
|
8316
8990
|
idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`,
|
|
8317
8991
|
});
|
|
8318
|
-
return
|
|
8992
|
+
return {
|
|
8993
|
+
billingPath: "external",
|
|
8994
|
+
pricedCostMicros: 0,
|
|
8995
|
+
normalizedUsage,
|
|
8996
|
+
};
|
|
8319
8997
|
}
|
|
8320
8998
|
if (totalTokens > 0) {
|
|
8321
8999
|
await recordUsageEvent(db, {
|
|
@@ -8334,12 +9012,18 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
8334
9012
|
}
|
|
8335
9013
|
const shouldDebit = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
|
|
8336
9014
|
if (!shouldDebit || totalTokens === 0) {
|
|
8337
|
-
return
|
|
9015
|
+
return {
|
|
9016
|
+
billingPath: "opengeni_credits",
|
|
9017
|
+
pricedCostMicros: 0,
|
|
9018
|
+
normalizedUsage,
|
|
9019
|
+
};
|
|
8338
9020
|
}
|
|
8339
9021
|
if (!configuredModelPricing(settings)[input.model]) {
|
|
8340
9022
|
throw new Error(`Missing model pricing for ${input.model}`);
|
|
8341
9023
|
}
|
|
8342
|
-
const costMicros = calculateModelUsageCostMicros(settings, input.model, sanitizedUsage
|
|
9024
|
+
const costMicros = calculateModelUsageCostMicros(settings, input.model, sanitizedUsage, {
|
|
9025
|
+
latencyMode: input.latencyMode ?? "standard",
|
|
9026
|
+
});
|
|
8343
9027
|
await recordUsageEvent(db, {
|
|
8344
9028
|
accountId: input.accountId,
|
|
8345
9029
|
workspaceId: input.workspaceId,
|
|
@@ -8367,6 +9051,7 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
8367
9051
|
sessionId: input.sessionId,
|
|
8368
9052
|
turnId: input.turnId,
|
|
8369
9053
|
sourceKey: input.sourceKey,
|
|
9054
|
+
latencyMode: input.latencyMode ?? "standard",
|
|
8370
9055
|
inputTokens,
|
|
8371
9056
|
outputTokens,
|
|
8372
9057
|
totalTokens,
|
|
@@ -8378,6 +9063,59 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
8378
9063
|
});
|
|
8379
9064
|
recordCreditMicros(input.observability, "usage", result.debitedMicros);
|
|
8380
9065
|
}
|
|
9066
|
+
return {
|
|
9067
|
+
billingPath: "opengeni_credits",
|
|
9068
|
+
pricedCostMicros: costMicros,
|
|
9069
|
+
normalizedUsage,
|
|
9070
|
+
};
|
|
9071
|
+
}
|
|
9072
|
+
|
|
9073
|
+
/** Soft-fail Insights fact write — never throws into the billing/emit path. */
|
|
9074
|
+
export async function recordAuthoritativeModelCallFact(input: {
|
|
9075
|
+
db: ActivityServices["db"];
|
|
9076
|
+
observability: ActivityServices["observability"];
|
|
9077
|
+
accountId: string;
|
|
9078
|
+
workspaceId: string;
|
|
9079
|
+
sessionId: string;
|
|
9080
|
+
turnId: string;
|
|
9081
|
+
turnAttemptId: string;
|
|
9082
|
+
sourceKey: string;
|
|
9083
|
+
provider: string;
|
|
9084
|
+
providerApi: ModelProviderApi;
|
|
9085
|
+
model: string;
|
|
9086
|
+
billing: ModelUsageBillingRecord;
|
|
9087
|
+
}): Promise<void> {
|
|
9088
|
+
try {
|
|
9089
|
+
const telemetry = input.billing.normalizedUsage.telemetry;
|
|
9090
|
+
await recordModelCallFact(input.db, {
|
|
9091
|
+
accountId: input.accountId,
|
|
9092
|
+
workspaceId: input.workspaceId,
|
|
9093
|
+
sessionId: input.sessionId,
|
|
9094
|
+
turnId: input.turnId,
|
|
9095
|
+
turnAttemptId: input.turnAttemptId,
|
|
9096
|
+
sourceKey: input.sourceKey,
|
|
9097
|
+
provider: input.provider,
|
|
9098
|
+
providerApi: input.providerApi,
|
|
9099
|
+
model: input.model,
|
|
9100
|
+
billingPath: input.billing.billingPath,
|
|
9101
|
+
pricedCostMicros: input.billing.pricedCostMicros,
|
|
9102
|
+
inputTokens: telemetry.inputTokens,
|
|
9103
|
+
outputTokens: telemetry.outputTokens,
|
|
9104
|
+
cachedTokens: telemetry.cachedTokens,
|
|
9105
|
+
cacheWriteTokens: telemetry.cacheWriteTokens,
|
|
9106
|
+
reasoningTokens: telemetry.reasoningTokens,
|
|
9107
|
+
totalTokens: input.billing.normalizedUsage.totalTokens,
|
|
9108
|
+
});
|
|
9109
|
+
} catch (error) {
|
|
9110
|
+
input.observability.warn("model call fact persist failed", {
|
|
9111
|
+
accountId: input.accountId,
|
|
9112
|
+
workspaceId: input.workspaceId,
|
|
9113
|
+
sessionId: input.sessionId,
|
|
9114
|
+
turnId: input.turnId,
|
|
9115
|
+
sourceKey: input.sourceKey,
|
|
9116
|
+
error: error instanceof Error ? error.message : String(error),
|
|
9117
|
+
});
|
|
9118
|
+
}
|
|
8381
9119
|
}
|
|
8382
9120
|
|
|
8383
9121
|
function sanitizedModelUsageInput(normalized: ModelCallUsageNormalization): ModelUsageInput {
|
|
@@ -8390,7 +9128,11 @@ function sanitizedModelUsageInput(normalized: ModelCallUsageNormalization): Mode
|
|
|
8390
9128
|
: {}),
|
|
8391
9129
|
...(normalized.totalTokens !== null ? { totalTokens: normalized.totalTokens } : {}),
|
|
8392
9130
|
...(normalized.telemetry.cachedTokens !== null
|
|
8393
|
-
? {
|
|
9131
|
+
? {
|
|
9132
|
+
inputTokensDetails: {
|
|
9133
|
+
cached_tokens: normalized.telemetry.cachedTokens,
|
|
9134
|
+
},
|
|
9135
|
+
}
|
|
8394
9136
|
: {}),
|
|
8395
9137
|
};
|
|
8396
9138
|
}
|