@opengeni/worker-bundle 0.8.3 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,6 @@
1
1
  import {
2
+ advanceWorkspaceGeneration,
3
+ verifyWorkspaceMutationSettlement,
2
4
  applySessionTurnSettlement,
3
5
  requestSessionTurnRecovery,
4
6
  claimSessionWorkForAttempt,
@@ -16,7 +18,7 @@ import {
16
18
  getLatestRunState,
17
19
  getHumanInputResumeForEvent,
18
20
  getSessionHumanInputRequest,
19
- isCodexBilledTurn,
21
+ installOrReadTurnExecutionPolicyForAttempt,
20
22
  workspaceCodexSubscriptionActive,
21
23
  acquireCodexCredentialLease,
22
24
  armCodexCapacityWait,
@@ -76,8 +78,8 @@ import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@
76
78
  import {
77
79
  sandboxStateEntryFromRunState,
78
80
  maxTurnsExceededRunState,
79
- modelCallUsageTelemetry,
80
81
  modelResponseUsageFromSdkEvent,
82
+ normalizeModelCallUsage,
81
83
  normalizeSdkEvent,
82
84
  sanitizeHistoryItemsForModel,
83
85
  isEphemeralInternalContext,
@@ -101,12 +103,15 @@ import {
101
103
  sandboxFileDownloadFailureNote,
102
104
  SUMMARY_BUFFER_TOKENS,
103
105
  runOwnedSandboxSetup,
106
+ RoutingMutationOutcomeUnknownError,
104
107
  swapTargetEstablishability,
105
108
  type SandboxFileDownload,
106
109
  type SandboxFileDownloadFailure,
107
110
  type OpenGeniRuntime,
108
111
  type ComputerToolMode,
109
112
  type ModelResponseUsage,
113
+ type ModelCallUsageInput,
114
+ type ModelCallUsageNormalization,
110
115
  type BuildAgentOptions,
111
116
  type TurnToolCancellationFence,
112
117
  type BackendUnresolvableCode,
@@ -120,12 +125,13 @@ import {
120
125
  } from "@opengeni/runtime";
121
126
  import { connectionTokenResolverForTurn } from "./mcp-credentials";
122
127
  import {
123
- builtinProviderId,
128
+ assertTurnExecutionPolicyMatchesConfigV1,
124
129
  calculateModelUsageCostMicros,
125
130
  configuredModelPricing,
126
131
  configuredStaticUsageLimits,
127
132
  sandboxWarmRateMicrosPerSecond,
128
133
  settingsWithResolvedModelContext,
134
+ resolveTurnExecutionPolicyV1,
129
135
  type ModelUsageInput,
130
136
  type ModelProviderApi,
131
137
  type RegistryProviderKind,
@@ -151,6 +157,11 @@ import {
151
157
  type CodexRotationStrategy,
152
158
  type RotationDecision,
153
159
  } from "./codex-rotation";
160
+ import {
161
+ codexFleetShadowDecisionMetricLabelsV1,
162
+ codexFleetShadowErrorMetricLabelsV1,
163
+ publishCodexFleetShadowDecisionV1,
164
+ } from "./codex-fleet-shadow";
154
165
  import type { CodexAccountStatus } from "@opengeni/db";
155
166
  import { buildCodexTokenResolver } from "./codex-auth";
156
167
  import {
@@ -163,16 +174,19 @@ import {
163
174
  CODEX_CLIENT_VERSION,
164
175
  CODEX_FALLBACK_MODEL_SLUGS,
165
176
  CodexReloginRequired,
177
+ classifyCodexResponseTimeoutError,
166
178
  classifyCodexUsageLimitError,
167
179
  codexRequestStorage,
168
180
  isCodexTransportError,
169
181
  type CodexRequestContext,
170
182
  type CodexUsageHeaderSnapshot,
171
183
  } from "@opengeni/codex";
172
- import { mergeResourceRefs, mergeToolRefs } from "./common";
184
+ import { mergeResourceRefs } from "./common";
185
+ import { enabledCapabilityMcpToolRefs, resolveSessionToolPolicy } from "@opengeni/core";
173
186
  import { maybeCompactContext } from "./context-compaction";
174
187
  import { TurnAttemptFencedError } from "./turn-attempt-fenced";
175
188
  import {
189
+ assertGitCredentialRenewalTransportUnchanged,
176
190
  gitCredentialAuthorityForTurn,
177
191
  gitHubTokenMintSelections,
178
192
  loadWorkspaceEnvironmentForRunWithCredentials,
@@ -274,10 +288,13 @@ import {
274
288
  import {
275
289
  CAPABILITY_DESCRIPTORS,
276
290
  evaluateWorkspaceModelPolicy,
291
+ readTurnExecutionPolicyV1,
277
292
  type ResourceRef,
278
293
  type SessionEvent,
279
294
  type SessionEventType,
280
295
  type SessionStatus,
296
+ type SessionTurn,
297
+ type TurnExecutionPolicyV1,
281
298
  } from "@opengeni/contracts";
282
299
  import { createHash, randomUUID } from "node:crypto";
283
300
 
@@ -287,6 +304,32 @@ import { createHash, randomUUID } from "node:crypto";
287
304
  // budget against the same window.
288
305
  export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
289
306
 
307
+ export function turnExecutionPolicyBillingIdentity(policy: TurnExecutionPolicyV1): {
308
+ externallyBilled: boolean;
309
+ codexSubscription: boolean;
310
+ } {
311
+ return {
312
+ externallyBilled: policy.billing.metering === "external",
313
+ codexSubscription:
314
+ policy.providerId === "codex-subscription" &&
315
+ policy.credentialSource.kind === "connected_subscription" &&
316
+ policy.credentialSource.provider === "codex",
317
+ };
318
+ }
319
+
320
+ export function legacyTurnExecutionPolicyInput(
321
+ turn: Pick<SessionTurn, "source" | "model" | "reasoningEffort">,
322
+ ): Parameters<typeof resolveTurnExecutionPolicyV1>[1] {
323
+ const explicit = turn.source === "user" || turn.source === "api";
324
+ return {
325
+ modelId: turn.model,
326
+ requestedModelId: explicit ? turn.model : null,
327
+ modelSource: explicit ? "explicit" : "continuation",
328
+ reasoningEffort: turn.reasoningEffort,
329
+ reasoningSource: explicit ? "explicit" : "continuation",
330
+ };
331
+ }
332
+
290
333
  /** A retryable provider fault recovers the accepted turn itself. Goal state is
291
334
  * irrelevant: autonomous continuation and infrastructure recovery are separate
292
335
  * concerns. */
@@ -491,6 +534,7 @@ export async function persistOrSignalSessionAttemptQuiescence(input: {
491
534
  export async function drainAttemptOwnedSandboxWriters(input: {
492
535
  toolCancellationFence: Pick<TurnToolCancellationFence, "cancel" | "waitForQuiescence"> | null;
493
536
  cancellationReason?: unknown;
537
+ gitCredentialRenewals: readonly Pick<GitCredentialRenewalController, "stop">[];
494
538
  toolspaceTokenRenewal: Pick<ToolspaceTokenRenewalController, "stop"> | null;
495
539
  runCredentialRenewal: Pick<RunCredentialRenewalController, "stop"> | null;
496
540
  }): Promise<void> {
@@ -500,6 +544,7 @@ export async function drainAttemptOwnedSandboxWriters(input: {
500
544
  );
501
545
  await input.toolCancellationFence.waitForQuiescence();
502
546
  }
547
+ await Promise.all(input.gitCredentialRenewals.map(async (renewal) => await renewal.stop()));
503
548
  await input.toolspaceTokenRenewal?.stop();
504
549
  await input.runCredentialRenewal?.stop();
505
550
  }
@@ -733,16 +778,8 @@ export function providerContextTokens(
733
778
  | null
734
779
  | undefined,
735
780
  ): number | null {
736
- const total = usage?.totalTokens;
737
- if (typeof total === "number" && Number.isFinite(total) && total > 0) {
738
- return total;
739
- }
740
- const input = usage?.inputTokens;
741
- if (typeof input !== "number" || !Number.isFinite(input) || input <= 0) {
742
- return null;
743
- }
744
- const output = usage?.outputTokens;
745
- return input + (typeof output === "number" && Number.isFinite(output) && output > 0 ? output : 0);
781
+ const total = normalizeModelCallUsage(usage).totalTokens;
782
+ return total !== null && total > 0 ? total : null;
746
783
  }
747
784
 
748
785
  /**
@@ -773,6 +810,260 @@ type TurnEventPublisher = (
773
810
  immediate?: boolean,
774
811
  ) => Promise<{ events: SessionEvent[]; accepted: boolean }>;
775
812
 
813
+ export type ModelResponseUsageEventState = {
814
+ responseUsageCount: number;
815
+ providerContextRevision: number;
816
+ lastProviderContextTokensObserved: number | null;
817
+ claimedSourceKeys: Set<string>;
818
+ };
819
+
820
+ export type CompactionModelUsageEventState = {
821
+ usageCount: number;
822
+ claimedSourceKeys: Set<string>;
823
+ };
824
+
825
+ export function createModelResponseUsageEventState(
826
+ claimedSourceKeys: Set<string> = new Set<string>(),
827
+ ): ModelResponseUsageEventState {
828
+ return {
829
+ responseUsageCount: 0,
830
+ providerContextRevision: 0,
831
+ lastProviderContextTokensObserved: null,
832
+ claimedSourceKeys,
833
+ };
834
+ }
835
+
836
+ export function createCompactionModelUsageEventState(
837
+ claimedSourceKeys: Set<string> = new Set<string>(),
838
+ ): CompactionModelUsageEventState {
839
+ return { usageCount: 0, claimedSourceKeys };
840
+ }
841
+
842
+ export function modelResponseUsageContextSignal(
843
+ state: ModelResponseUsageEventState,
844
+ ): { revision: number; totalTokens: number } | null {
845
+ return state.lastProviderContextTokensObserved === null
846
+ ? null
847
+ : {
848
+ revision: state.providerContextRevision,
849
+ totalTokens: state.lastProviderContextTokensObserved,
850
+ };
851
+ }
852
+
853
+ /**
854
+ * Process one SDK stream event through the exact production usage path.
855
+ *
856
+ * The pinned Responses SDK mirrors one provider terminal response as both a
857
+ * normalized `response_done` and a raw `model/response.completed` event. Claim
858
+ * the stable response/source key before lease renewal or any usage side effect,
859
+ * and advance the positional ordinal only for a newly claimed response. The
860
+ * durable `agent.model.usage` source-key fence remains the cross-restart
861
+ * authority: a replay may retry the idempotent billing write, but it cannot
862
+ * advance process-local metrics, provider context, or attempt-owned signals.
863
+ */
864
+ export async function processModelResponseUsageEvent(input: {
865
+ event: Parameters<typeof modelResponseUsageFromSdkEvent>[0];
866
+ state: ModelResponseUsageEventState;
867
+ dispatchId: string | null;
868
+ settings: Settings;
869
+ db: ActivityServices["db"];
870
+ observability: ActivityServices["observability"];
871
+ publish: TurnEventPublisher | null;
872
+ accountId: string;
873
+ workspaceId: string;
874
+ sessionId: string;
875
+ turnId: string;
876
+ turnAttemptId: string;
877
+ provider: string;
878
+ providerApi: ModelProviderApi;
879
+ model: string;
880
+ metricProvider: string;
881
+ externallyBilled: boolean;
882
+ servingCredentialId: string | null;
883
+ priorSessionCredentialId: string | null;
884
+ emittedSourceKeys: Set<string>;
885
+ renewLease: () => Promise<void>;
886
+ leaseLost: () => boolean;
887
+ leaseLostMessage: string;
888
+ setLastInputTokens: (tokens: number) => Promise<void>;
889
+ }): Promise<
890
+ | { status: "not_usage" }
891
+ | { status: "duplicate"; sourceKey: string }
892
+ | { status: "processed"; sourceKey: string; authoritative: boolean }
893
+ > {
894
+ const responseUsage = modelResponseUsageFromSdkEvent(input.event);
895
+ if (!responseUsage) {
896
+ return { status: "not_usage" };
897
+ }
898
+
899
+ const responseOrdinal = input.state.responseUsageCount + 1;
900
+ const sourceKey = modelUsageSourceKey({
901
+ responseId: responseUsage.responseId,
902
+ dispatchId: input.dispatchId,
903
+ positionalKey: `response-${responseOrdinal}`,
904
+ });
905
+ if (input.state.claimedSourceKeys.has(sourceKey)) {
906
+ return { status: "duplicate", sourceKey };
907
+ }
908
+ input.state.claimedSourceKeys.add(sourceKey);
909
+ input.state.responseUsageCount = responseOrdinal;
910
+
911
+ const normalizedUsage = normalizeModelCallUsage(responseUsage.usage);
912
+ const accountContext = modelCallAccountContext({
913
+ servingCredentialId: input.servingCredentialId,
914
+ priorSessionCredentialId: input.priorSessionCredentialId,
915
+ isFirstCallOfTurn: responseOrdinal === 1,
916
+ });
917
+ let authoritative = false;
918
+ await recordCompletedModelCallBeforeOwnershipFences({
919
+ renewLease: input.renewLease,
920
+ leaseLost: input.leaseLost,
921
+ leaseLostMessage: input.leaseLostMessage,
922
+ recordUsage: async () => {
923
+ await recordModelUsageAndDebitCredits(input.settings, input.db, {
924
+ accountId: input.accountId,
925
+ workspaceId: input.workspaceId,
926
+ sessionId: input.sessionId,
927
+ turnId: input.turnId,
928
+ turnAttemptId: input.turnAttemptId,
929
+ model: input.model,
930
+ externallyBilled: input.externallyBilled,
931
+ usage: responseUsage.usage,
932
+ normalizedUsage,
933
+ sourceKey,
934
+ observability: input.observability,
935
+ });
936
+ authoritative = await emitModelCallUsage({
937
+ observability: input.observability,
938
+ publish: input.publish,
939
+ accountId: input.accountId,
940
+ workspaceId: input.workspaceId,
941
+ sessionId: input.sessionId,
942
+ turnId: input.turnId,
943
+ provider: input.provider,
944
+ providerApi: input.providerApi,
945
+ model: input.model,
946
+ sourceKey,
947
+ usage: responseUsage,
948
+ normalizedUsage,
949
+ servingAccountHash: accountContext.servingAccountHash,
950
+ accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
951
+ emittedSourceKeys: input.emittedSourceKeys,
952
+ });
953
+ const observedInput = normalizedUsage.telemetry.inputTokens;
954
+ if (authoritative && observedInput !== null && observedInput > 0) {
955
+ recordModelInputTokens(input.observability, input.metricProvider, observedInput);
956
+ }
957
+ },
958
+ recordAttemptSignals: async () => {
959
+ if (!authoritative) return;
960
+ const observedTotal = normalizedUsage.totalTokens;
961
+ if (observedTotal !== null) {
962
+ input.state.lastProviderContextTokensObserved = observedTotal;
963
+ input.state.providerContextRevision += 1;
964
+ }
965
+ const observedInput = normalizedUsage.telemetry.inputTokens;
966
+ if (observedInput !== null && observedInput > 0) {
967
+ await input.setLastInputTokens(observedInput);
968
+ }
969
+ },
970
+ });
971
+ return { status: "processed", sourceKey, authoritative };
972
+ }
973
+
974
+ /**
975
+ * Apply the same source-key authority ordering to the compaction summarizer's
976
+ * usage callback. The summarizer can retry or mirror a terminal response just
977
+ * like the main stream, so claim before lease renewal, billing, durable usage,
978
+ * logging, or cache metrics. Durable source-key idempotency remains the
979
+ * cross-process authority after a worker restart.
980
+ */
981
+ export async function processCompactionModelUsageEvent(input: {
982
+ usage: ModelResponseUsage;
983
+ state: CompactionModelUsageEventState;
984
+ dispatchId: string | null;
985
+ settings: Settings;
986
+ db: ActivityServices["db"];
987
+ observability: ActivityServices["observability"];
988
+ publish: TurnEventPublisher | null;
989
+ accountId: string;
990
+ workspaceId: string;
991
+ sessionId: string;
992
+ turnId: string;
993
+ turnAttemptId: string;
994
+ provider: string;
995
+ providerApi: ModelProviderApi;
996
+ model: string;
997
+ externallyBilled: boolean;
998
+ servingCredentialId: string | null;
999
+ priorSessionCredentialId: string | null;
1000
+ emittedSourceKeys: Set<string>;
1001
+ renewLease: () => Promise<void>;
1002
+ leaseLost: () => boolean;
1003
+ leaseLostMessage: string;
1004
+ }): Promise<
1005
+ | { status: "duplicate"; sourceKey: string }
1006
+ | { status: "processed"; sourceKey: string; authoritative: boolean }
1007
+ > {
1008
+ const usageOrdinal = input.state.usageCount + 1;
1009
+ const sourceKey = modelUsageSourceKey({
1010
+ responseId: input.usage.responseId,
1011
+ dispatchId: input.dispatchId,
1012
+ positionalKey: `compaction-${usageOrdinal}`,
1013
+ });
1014
+ if (input.state.claimedSourceKeys.has(sourceKey)) {
1015
+ return { status: "duplicate", sourceKey };
1016
+ }
1017
+ input.state.claimedSourceKeys.add(sourceKey);
1018
+ input.state.usageCount = usageOrdinal;
1019
+
1020
+ const accountContext = modelCallAccountContext({
1021
+ servingCredentialId: input.servingCredentialId,
1022
+ priorSessionCredentialId: input.priorSessionCredentialId,
1023
+ isFirstCallOfTurn: usageOrdinal === 1,
1024
+ });
1025
+ const normalizedUsage = normalizeModelCallUsage(input.usage.usage);
1026
+ let authoritative = false;
1027
+ await recordCompletedModelCallBeforeOwnershipFences({
1028
+ renewLease: input.renewLease,
1029
+ leaseLost: input.leaseLost,
1030
+ leaseLostMessage: input.leaseLostMessage,
1031
+ recordUsage: async () => {
1032
+ await recordModelUsageAndDebitCredits(input.settings, input.db, {
1033
+ accountId: input.accountId,
1034
+ workspaceId: input.workspaceId,
1035
+ sessionId: input.sessionId,
1036
+ turnId: input.turnId,
1037
+ turnAttemptId: input.turnAttemptId,
1038
+ model: input.model,
1039
+ externallyBilled: input.externallyBilled,
1040
+ usage: input.usage.usage,
1041
+ normalizedUsage,
1042
+ sourceKey,
1043
+ observability: input.observability,
1044
+ });
1045
+ authoritative = await emitModelCallUsage({
1046
+ observability: input.observability,
1047
+ publish: input.publish,
1048
+ accountId: input.accountId,
1049
+ workspaceId: input.workspaceId,
1050
+ sessionId: input.sessionId,
1051
+ turnId: input.turnId,
1052
+ provider: input.provider,
1053
+ providerApi: input.providerApi,
1054
+ model: input.model,
1055
+ sourceKey,
1056
+ usage: input.usage,
1057
+ normalizedUsage,
1058
+ servingAccountHash: accountContext.servingAccountHash,
1059
+ accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
1060
+ emittedSourceKeys: input.emittedSourceKeys,
1061
+ });
1062
+ },
1063
+ });
1064
+ return { status: "processed", sourceKey, authoritative };
1065
+ }
1066
+
776
1067
  export async function emitModelCallUsage(input: {
777
1068
  observability: ActivityServices["observability"];
778
1069
  publish: TurnEventPublisher | null;
@@ -785,22 +1076,25 @@ export async function emitModelCallUsage(input: {
785
1076
  model: string;
786
1077
  sourceKey: string;
787
1078
  usage: ModelResponseUsage | { usage?: unknown | null } | null;
1079
+ normalizedUsage?: ModelCallUsageNormalization;
788
1080
  // Prompt-cache research dimensions (log-only; NEVER on a metric label or a
789
1081
  // durable event). The opaque serving-account tag and whether it changed since
790
1082
  // the session's previous call — the account-switch hypothesis for cache misses.
791
1083
  servingAccountHash?: string;
792
1084
  accountChangedFromPrevCall?: boolean;
793
1085
  emittedSourceKeys?: Set<string>;
794
- }): Promise<void> {
1086
+ }): Promise<boolean> {
795
1087
  const usage =
796
1088
  input.usage && typeof input.usage === "object" && "usage" in input.usage
797
1089
  ? (input.usage as { usage?: unknown }).usage
798
1090
  : null;
799
1091
  if (!usage || typeof usage !== "object") {
800
- return;
1092
+ return false;
801
1093
  }
802
- if (input.emittedSourceKeys?.has(input.sourceKey)) return;
803
- const telemetry = modelCallUsageTelemetry(usage as Parameters<typeof modelCallUsageTelemetry>[0]);
1094
+ if (input.emittedSourceKeys?.has(input.sourceKey)) return false;
1095
+ const normalizedUsage =
1096
+ input.normalizedUsage ?? normalizeModelCallUsage(usage as ModelCallUsageInput);
1097
+ const telemetry = normalizedUsage.telemetry;
804
1098
  const appended = await input.publish?.(
805
1099
  [
806
1100
  {
@@ -829,7 +1123,7 @@ export async function emitModelCallUsage(input: {
829
1123
  typeof event.payload === "object" &&
830
1124
  (event.payload as Record<string, unknown>).sourceKey === input.sourceKey,
831
1125
  );
832
- if (!authoritative) return;
1126
+ if (!authoritative) return false;
833
1127
  try {
834
1128
  input.observability.info("model call usage", {
835
1129
  accountId: input.accountId,
@@ -843,6 +1137,7 @@ export async function emitModelCallUsage(input: {
843
1137
  inputTokens: telemetry.inputTokens,
844
1138
  outputTokens: telemetry.outputTokens,
845
1139
  cachedTokens: telemetry.cachedTokens,
1140
+ cacheWriteTokens: telemetry.cacheWriteTokens,
846
1141
  reasoningTokens: telemetry.reasoningTokens,
847
1142
  ...(input.servingAccountHash !== undefined
848
1143
  ? { servingAccountHash: input.servingAccountHash }
@@ -851,9 +1146,41 @@ export async function emitModelCallUsage(input: {
851
1146
  ? { accountChangedFromPrevCall: input.accountChangedFromPrevCall }
852
1147
  : {}),
853
1148
  });
1149
+ if (normalizedUsage.rejectedFields.length > 0) {
1150
+ input.observability.warn("model call usage fields rejected", {
1151
+ provider: input.provider,
1152
+ providerApi: input.providerApi,
1153
+ model: input.model,
1154
+ sourceKey: input.sourceKey,
1155
+ rejectedFields: normalizedUsage.rejectedFields.join(","),
1156
+ });
1157
+ }
854
1158
  } catch {
855
1159
  // Durable event + billing already committed; logging is best-effort only.
856
1160
  }
1161
+ try {
1162
+ applyCodexCacheTelemetry(input.observability, input.provider, normalizedUsage);
1163
+ } catch {
1164
+ // Durable event + billing already committed; metrics are best-effort only.
1165
+ }
1166
+ return true;
1167
+ }
1168
+
1169
+ /**
1170
+ * Apply one authoritative, normalized model-call usage frame to the shared
1171
+ * prompt-cache metrics. The durable source-key fence in `emitModelCallUsage`
1172
+ * owns idempotency; this helper must never receive raw provider values.
1173
+ */
1174
+ export function applyCodexCacheTelemetry(
1175
+ observability: ActivityServices["observability"],
1176
+ provider: string,
1177
+ normalizedUsage: ModelCallUsageNormalization,
1178
+ ): void {
1179
+ recordModelCacheTokens(observability, provider, {
1180
+ cachedTokens: normalizedUsage.telemetry.cachedTokens,
1181
+ cacheWriteTokens: normalizedUsage.telemetry.cacheWriteTokens,
1182
+ promptTokens: normalizedUsage.telemetry.inputTokens,
1183
+ });
857
1184
  }
858
1185
 
859
1186
  export function historyRowsToAppend(
@@ -1216,6 +1543,19 @@ export function computerToolModeForTurn(
1216
1543
  return "hosted";
1217
1544
  }
1218
1545
 
1546
+ /**
1547
+ * Decide whether this turn's provider accepts typed file/image content in a
1548
+ * user message. The legacy built-in client and registry Responses clients use
1549
+ * the Agents SDK Responses converter, which supports `input_image` and
1550
+ * `input_file`. Chat-completions providers retain the sandbox-path projection
1551
+ * until that transport declares and proves an equivalent capability.
1552
+ */
1553
+ export function modelAcceptsTypedAttachmentContentForTurn(
1554
+ resolvedModel: { provider: { api: ModelProviderApi } } | null,
1555
+ ): boolean {
1556
+ return resolvedModel === null || resolvedModel.provider.api === "responses";
1557
+ }
1558
+
1219
1559
  export type TurnSandboxProvisioner<T> = {
1220
1560
  get(): Promise<T>;
1221
1561
  hasStarted(): boolean;
@@ -1456,12 +1796,24 @@ async function refreshCappedCodexUsageRows(
1456
1796
  db: ActivityServices["db"],
1457
1797
  settings: Settings,
1458
1798
  workspaceId: string,
1459
- accounts: CodexAccountStatus[],
1799
+ accounts: Array<
1800
+ Pick<
1801
+ CodexAccountStatus,
1802
+ "id" | "status" | "primaryUsedPercent" | "secondaryUsedPercent" | "exhaustedUntil"
1803
+ >
1804
+ >,
1460
1805
  capacitySignals: {
1461
1806
  signalCodexCapacityWorkflow?: ActivityServices["signalCodexCapacityWorkflow"] | undefined;
1462
1807
  wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
1463
1808
  },
1464
- ): Promise<CodexAccountStatus[]> {
1809
+ ): Promise<
1810
+ Array<
1811
+ Pick<
1812
+ CodexAccountStatus,
1813
+ "id" | "status" | "primaryUsedPercent" | "secondaryUsedPercent" | "exhaustedUntil"
1814
+ >
1815
+ >
1816
+ > {
1465
1817
  const stale = accounts.filter(
1466
1818
  (a) =>
1467
1819
  a.status === "active" &&
@@ -1560,6 +1912,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1560
1912
  runtimeMetricsHooksForObservability(observability),
1561
1913
  );
1562
1914
  let isCodexTurn = false;
1915
+ let isExternallyBilledTurn = false;
1563
1916
  let executionGeneration = 0;
1564
1917
  // Still required by credential-loss/capacity settlements, whose own
1565
1918
  // recovery transactions fence against worker-death redispatches.
@@ -1693,6 +2046,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1693
2046
  // run. null when the flag is off (byte-for-byte the legacy build-and-discard
1694
2047
  // path) OR when the backend is "none". Released + dropped in `finally`.
1695
2048
  let resolvedSandbox: ResumedTurnSandbox | null = null;
2049
+ const requireResolvedSandboxForMutation = (message: string): ResumedTurnSandbox => {
2050
+ if (!resolvedSandbox) throw new Error(message);
2051
+ return resolvedSandbox;
2052
+ };
1696
2053
  // The machine-primary SelfhostedSession (the UNWRAPPED backend, not the
1697
2054
  // routing proxy): held so the turn's completion can final-ack this turn's
1698
2055
  // settled op-stream ops AFTER the results are durably persisted.
@@ -1704,11 +2061,102 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1704
2061
  // THIS handle so a mid-turn sandbox_swap can never re-route those execs onto a
1705
2062
  // connected machine (the user's real computer).
1706
2063
  let setupBoxSession: unknown = null;
2064
+ // A same-target API repair can replace the home provider while this turn is
2065
+ // alive. Keep setup/snapshot persistence on the rebound raw session while
2066
+ // preserving the SDK-owned routing proxy for eager turns. Lazy turns hold the
2067
+ // proxy separately, so their worker-side handle may replace its raw session.
2068
+ const onHomeSandboxRebound = (rebound: {
2069
+ established: EstablishedSandboxSession;
2070
+ leaseEpoch: number;
2071
+ }): void => {
2072
+ const current = resolvedSandbox;
2073
+ const previousSession = current?.established.session;
2074
+ const preserveRoutingProxy = current !== null && previousSession !== setupBoxSession;
2075
+ setupBoxSession = rebound.established.session;
2076
+ if (!current) return;
2077
+ current.leaseEpoch = rebound.leaseEpoch;
2078
+ current.established = preserveRoutingProxy
2079
+ ? {
2080
+ ...current.established,
2081
+ client: rebound.established.client,
2082
+ // Keep the stable SDK-facing proxy; only its resolver changes the
2083
+ // underlying backend. The worker's setupBoxSession above is raw.
2084
+ session: previousSession,
2085
+ sessionState: rebound.established.sessionState,
2086
+ instanceId: rebound.established.instanceId,
2087
+ backendId: rebound.established.backendId,
2088
+ ...(rebound.established.origin ? { origin: rebound.established.origin } : {}),
2089
+ ...(rebound.established.restoredArchive
2090
+ ? { restoredArchive: rebound.established.restoredArchive }
2091
+ : {}),
2092
+ }
2093
+ : rebound.established;
2094
+ };
1707
2095
  // The globally unique durable turn-attempt holder id + the group id,
1708
2096
  // captured so the lease heartbeat can refresh the lease TTL epoch-fenced
1709
2097
  // (a superseded owner self-evicts) and finally can release.
1710
2098
  let sandboxHolderId: TurnSandboxLeaseHolderId | null = null;
1711
2099
  let sandboxGroupId: string | null = null;
2100
+ const runWorkspaceMutationForSandbox = async <T>(
2101
+ sandbox: ResumedTurnSandbox,
2102
+ operation: string,
2103
+ mutation: () => Promise<T>,
2104
+ ): Promise<T> => {
2105
+ // Connected machines are the user's own persistence and never dirty the
2106
+ // cloud home archive. Every persistable raw-session write batch is fenced
2107
+ // against the exact current lease/provider before the provider sees it.
2108
+ if (sandbox.established.backendId === "selfhosted") return await mutation();
2109
+ if (!sandboxGroupId || !sandboxHolderId || !turnId || executionGeneration <= 0) {
2110
+ throw new Error("Workspace mutation attempted before exact turn sandbox admission");
2111
+ }
2112
+ const identity = {
2113
+ accountId: input.accountId,
2114
+ workspaceId: input.workspaceId,
2115
+ sessionId: input.sessionId,
2116
+ turnId,
2117
+ executionGeneration,
2118
+ attemptId: input.attemptId,
2119
+ holderId: sandboxHolderId,
2120
+ sandboxGroupId,
2121
+ expectedEpoch: sandbox.leaseEpoch,
2122
+ expectedInstanceId: sandbox.established.instanceId,
2123
+ operation,
2124
+ };
2125
+ const admission = await advanceWorkspaceGeneration(db, identity);
2126
+ let result: T;
2127
+ try {
2128
+ result = await mutation();
2129
+ } catch (providerError) {
2130
+ try {
2131
+ await verifyWorkspaceMutationSettlement(db, {
2132
+ ...identity,
2133
+ admission,
2134
+ outcome: "rejected",
2135
+ });
2136
+ } catch (settlementError) {
2137
+ throw new RoutingMutationOutcomeUnknownError(
2138
+ operation,
2139
+ `Platform workspace mutation "${operation}" rejected at the provider but lost its durable physical settlement; its outcome is unknown and it was not replayed`,
2140
+ { cause: settlementError },
2141
+ );
2142
+ }
2143
+ throw providerError;
2144
+ }
2145
+ try {
2146
+ await verifyWorkspaceMutationSettlement(db, {
2147
+ ...identity,
2148
+ admission,
2149
+ outcome: "resolved",
2150
+ });
2151
+ } catch (settlementError) {
2152
+ throw new RoutingMutationOutcomeUnknownError(
2153
+ operation,
2154
+ `Platform workspace mutation "${operation}" returned from the provider but lost its durable settlement fence; its outcome is unknown and it was not replayed`,
2155
+ { cause: settlementError },
2156
+ );
2157
+ }
2158
+ return result;
2159
+ };
1712
2160
  // Lease-TTL refresh timer (parallels the activity heartbeat): while the turn
1713
2161
  // runs it refreshes expires_at epoch-fenced so a legit multi-day turn is
1714
2162
  // never TTL-reaped. Cleared in finally. Only set when the flag resolved a box.
@@ -2138,10 +2586,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2138
2586
  input.workspaceId,
2139
2587
  settings,
2140
2588
  );
2141
- // Read the active-credential flag ONCE (P2-b) and thread it through both the
2142
- // routing overlay (settingsWithCodexCredential) and the billed-turn predicate
2143
- // (isCodexBilledTurn below), so a concurrent disconnect/reconnect cannot make
2144
- // provider-injection and billing disagree about whether this is a codex turn.
2589
+ // Read the active-credential flag once for the runtime capability overlay.
2590
+ // Accepted billing/provider identity comes from the turn policy below,
2591
+ // never from this mutable health snapshot.
2145
2592
  const codexSubscriptionActive = await workspaceCodexSubscriptionActive(
2146
2593
  db,
2147
2594
  mcpSettings,
@@ -2169,6 +2616,38 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2169
2616
  }
2170
2617
  const turn = claim.turn;
2171
2618
  turnId = turn.id;
2619
+ executionGeneration = turn.executionGeneration;
2620
+ const claimedPolicy = readTurnExecutionPolicyV1(turn.metadata);
2621
+ const policyForAbsent =
2622
+ claimedPolicy.kind === "valid"
2623
+ ? claimedPolicy.policy
2624
+ : resolveTurnExecutionPolicyV1(capabilitySettings, legacyTurnExecutionPolicyInput(turn));
2625
+ const installedPolicy = await installOrReadTurnExecutionPolicyForAttempt(db, {
2626
+ accountId: input.accountId,
2627
+ workspaceId: input.workspaceId,
2628
+ sessionId: input.sessionId,
2629
+ turnId,
2630
+ executionGeneration,
2631
+ attemptId: input.attemptId,
2632
+ policyForAbsent,
2633
+ });
2634
+ if (!installedPolicy.accepted) {
2635
+ throw new TurnAttemptFencedError(
2636
+ `turn execution policy was fenced: ${installedPolicy.reason}`,
2637
+ );
2638
+ }
2639
+ const verifiedExecutionPolicy = assertTurnExecutionPolicyMatchesConfigV1(
2640
+ capabilitySettings,
2641
+ installedPolicy.policy,
2642
+ {
2643
+ modelId: turn.model,
2644
+ reasoningEffort: turn.reasoningEffort,
2645
+ },
2646
+ );
2647
+ const turnExecutionPolicy = verifiedExecutionPolicy.policy;
2648
+ const billingIdentity = turnExecutionPolicyBillingIdentity(turnExecutionPolicy);
2649
+ isExternallyBilledTurn = billingIdentity.externallyBilled;
2650
+ isCodexTurn = billingIdentity.codexSubscription;
2172
2651
  triggerEventId = turn.triggerEventId;
2173
2652
  const trigger = await getSessionEvent(db, input.workspaceId, triggerEventId);
2174
2653
  if (!trigger) {
@@ -2181,7 +2660,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2181
2660
  trigger,
2182
2661
  );
2183
2662
  triggerType = trigger.type;
2184
- executionGeneration = turn.executionGeneration;
2185
2663
  const latestTurnState = await getLatestRunState(db, input.workspaceId, input.sessionId);
2186
2664
  const continuationCodexCredentialId =
2187
2665
  latestTurnState?.turnId === turnId ? latestTurnState.frozenCodexCredentialId : null;
@@ -2190,21 +2668,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2190
2668
  0,
2191
2669
  );
2192
2670
  turnLifecycleMetricsFor(observability).start(turnId);
2193
- // Canonical codex-billed predicate (codex/<slug> + feature enabled + active
2194
- // workspace credential). Computed once and threaded through every billing
2195
- // gate + the usage recorder so a turn paid by the user's ChatGPT/Codex plan
2196
- // consumes ZERO OpenGeni credits and never feeds an OpenGeni cap. Resolved
2197
- // here (before resolvedModel at the routing step) because the pre-turn gate
2198
- // below needs it; mirrors the same active-credential read the codex provider
2199
- // overlay uses, so billing and routing agree on what "codex" is.
2200
- isCodexTurn = await isCodexBilledTurn({
2201
- db,
2202
- settings,
2203
- workspaceId: input.workspaceId,
2204
- model: turn.model,
2205
- active: codexSubscriptionActive,
2206
- });
2207
- // §7.5 P3 — pass BOTH the codex predicate (codex-plan turns bypass the gate)
2671
+ // §7.5 P3 — pass the accepted billing attribution (externally funded turns
2672
+ // bypass OpenGeni credit/token gates)
2208
2673
  // AND the optional host `entitlements` port (when bound, its admitRun replaces
2209
2674
  // the local credit read). Unset port → today's local-ledger path.
2210
2675
  await waitForTurnOperation(
@@ -2213,7 +2678,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2213
2678
  db,
2214
2679
  input.accountId,
2215
2680
  input.workspaceId,
2216
- isCodexTurn,
2681
+ isExternallyBilledTurn,
2217
2682
  entitlements,
2218
2683
  ),
2219
2684
  cancellationSignal,
@@ -2259,6 +2724,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2259
2724
  // one exactly like production.
2260
2725
  codexLeaseHolderId = dispatchId;
2261
2726
  const modelUsageDispatchId = activityContext?.info.activityId ?? dispatchId;
2727
+ const claimedModelUsageSourceKeys = new Set<string>();
2262
2728
  const emittedModelUsageSourceKeys = new Set<string>();
2263
2729
  publish = async (
2264
2730
  events: Array<Omit<AppendEventInput, "producerId" | "producerSeq" | "turnId">>,
@@ -2646,6 +3112,80 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2646
3112
  codexLeaseConfirmedUntilMs !== null;
2647
3113
  if (codexLeaseHeld) startCodexLeaseHeartbeat();
2648
3114
 
3115
+ const actualOutcome = effectiveCodexCredentialId
3116
+ ? "selected"
3117
+ : rotationDecision.kind === "allCapped"
3118
+ ? "waiting"
3119
+ : "none";
3120
+ const actualReason = effectiveCodexCredentialId
3121
+ ? leased.reused
3122
+ ? "lease_reused"
3123
+ : sessionPin === effectiveCodexCredentialId
3124
+ ? "pin"
3125
+ : rotationDecision.kind === "active" && rotationDecision.moved
3126
+ ? "rotation"
3127
+ : "active"
3128
+ : rotationDecision.kind === "allCapped"
3129
+ ? "all_capped"
3130
+ : "none";
3131
+ const fencedInFlight =
3132
+ leased.reused ||
3133
+ (continuationCodexCredentialId !== null &&
3134
+ continuationCodexCredentialId === effectiveCodexCredentialId);
3135
+ const shadowResult = await publishCodexFleetShadowDecisionV1({
3136
+ enabled: settings.codexFleetPolicyShadowEnabled,
3137
+ decision: {
3138
+ accounts: leased.accounts,
3139
+ actualCredentialId: effectiveCodexCredentialId,
3140
+ actualOutcome,
3141
+ actualReason,
3142
+ affinityCredentialId: fencedInFlight
3143
+ ? effectiveCodexCredentialId
3144
+ : (sessionPin ?? sessionCodex?.lastCredentialId ?? null),
3145
+ fencedInFlight,
3146
+ nearExhaustionPct: settings.codexRotationNearExhaustionPct,
3147
+ now: new Date(),
3148
+ aliasSeed: randomUUID(),
3149
+ },
3150
+ publish,
3151
+ });
3152
+ if (shadowResult.outcome === "published") {
3153
+ const shadowPayload = shadowResult.payload;
3154
+ observability.incrementCounter({
3155
+ name: "opengeni_codex_fleet_shadow_decisions_total",
3156
+ help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
3157
+ labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload),
3158
+ });
3159
+ observability.info("Codex adaptive fleet shadow decision", {
3160
+ workspaceId: input.workspaceId,
3161
+ policyVersion: shadowPayload.replay.policyVersion,
3162
+ inputFingerprint: shadowPayload.replay.inputFingerprint,
3163
+ decisionFingerprint: shadowPayload.replay.decisionFingerprint,
3164
+ actualOutcome: shadowPayload.actual.outcome,
3165
+ shadowOutcome: shadowPayload.replay.decision.outcome,
3166
+ comparison: shadowPayload.comparison,
3167
+ candidateCount: shadowPayload.replay.input.candidates.length,
3168
+ truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
3169
+ payloadBytes: shadowResult.payloadBytes,
3170
+ });
3171
+ } else if (shadowResult.outcome === "failed") {
3172
+ // Shadow observability is explicitly non-authoritative. A malformed
3173
+ // snapshot or event-write fault must never change the authoritative lease,
3174
+ // capacity wait, failover, or the account serving this fenced turn.
3175
+ observability.incrementCounter({
3176
+ name: "opengeni_codex_fleet_shadow_errors_total",
3177
+ help: "Shadow decision build/publication failures.",
3178
+ labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
3179
+ });
3180
+ observability.warn("Codex adaptive fleet shadow decision failed open", {
3181
+ workspaceId: input.workspaceId,
3182
+ stage: shadowResult.stage,
3183
+ reason: shadowResult.reason,
3184
+ errorName: shadowResult.errorName,
3185
+ payloadBytes: shadowResult.payloadBytes,
3186
+ });
3187
+ }
3188
+
2649
3189
  const eligibleCount = leased.accounts.filter((account) =>
2650
3190
  isCodexCredentialEligible(account, new Date()),
2651
3191
  ).length;
@@ -3016,6 +3556,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3016
3556
  db,
3017
3557
  input.workspaceId,
3018
3558
  input.sessionId,
3559
+ input.attemptId,
3019
3560
  baseRunSettings,
3020
3561
  );
3021
3562
  // Multi-provider per-turn routing → the provider gating (compaction mode,
@@ -3031,7 +3572,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3031
3572
  // a chat-only Fireworks model. Resolving against the default-model settings
3032
3573
  // keeps gating consistent with the router. Cost accounting covers registry
3033
3574
  // models via configuredModelPricing.
3034
- const resolvedModel = runtime.resolveTurnModel(capabilitySettings, turn.model);
3575
+ const resolvedModel = runtime.resolveTurnModel(
3576
+ capabilitySettings,
3577
+ turnExecutionPolicy.productModelId,
3578
+ );
3035
3579
  // Bind the provider/model catalog's context policy to every model-facing
3036
3580
  // path for this turn. In particular, Codex subscription turns must not
3037
3581
  // inherit the deployment's OpenAI/Azure mode or 1.05M context defaults:
@@ -3046,25 +3590,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3046
3590
  // and the main run both come later in this scope), so a blocked
3047
3591
  // provider/model can never be reached through ANY stamp path: explicit
3048
3592
  // turn model, inherited session default, goal-continuation inheritance,
3049
- // or the legacy null-resolution fallback (null → the built-in
3050
- // OpenAI/Azure client, attributed here via builtinProviderId so a policy
3051
- // blocking the built-in also blocks that fallback — this exact path is
3052
- // how bare-model turns silently spent real Azure money in a
3053
- // codex-intended workspace). Fail-loud, never a silent remap.
3593
+ // or the legacy null-resolution fallback. The frozen execution policy is
3594
+ // the attribution source even if an injected test runtime returns no
3595
+ // concrete resolved model. Fail-loud, never a silent remap.
3054
3596
  {
3055
3597
  const workspaceModelPolicy = await getWorkspaceModelPolicy(db, input.workspaceId);
3056
3598
  if (workspaceModelPolicy) {
3057
- const effectiveProviderId = resolvedModel
3058
- ? resolvedModel.provider.id
3059
- : builtinProviderId(capabilitySettings);
3060
3599
  const verdict = evaluateWorkspaceModelPolicy(workspaceModelPolicy, {
3061
- providerId: effectiveProviderId,
3062
- modelId: turn.model,
3600
+ providerId: turnExecutionPolicy.providerId,
3601
+ modelId: turnExecutionPolicy.productModelId,
3063
3602
  });
3064
3603
  if (!verdict.allowed) {
3065
3604
  throw new WorkspaceModelPolicyBlockedError(
3066
- turn.model,
3067
- effectiveProviderId,
3605
+ turnExecutionPolicy.productModelId,
3606
+ turnExecutionPolicy.providerId,
3068
3607
  verdict.reason,
3069
3608
  );
3070
3609
  }
@@ -3078,6 +3617,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3078
3617
  // Build it once and wrap BOTH the compaction summarizer (a separate model
3079
3618
  // call on the same codex client) and the main run; otherwise the summarizer
3080
3619
  // would hit the codex backend unauthenticated.
3620
+ let codexModelRequestSequence = 0;
3081
3621
  const codexContext: CodexRequestContext | null =
3082
3622
  resolvedModel?.provider.kind === "codex-subscription"
3083
3623
  ? ((): CodexRequestContext => {
@@ -3108,6 +3648,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3108
3648
  onUsageHeaders: (snapshot) => {
3109
3649
  latestCodexUsage = snapshot;
3110
3650
  }, // latest wins; flushed once in finally
3651
+ nextRequestId: () => `${dispatchId}:${++codexModelRequestSequence}`,
3652
+ onModelRequestEvent: async (event) => {
3653
+ if (!publish || !turnId) {
3654
+ throw new Error("Codex model request started before the turn event producer");
3655
+ }
3656
+ await publish([
3657
+ {
3658
+ type: "agent.model.request",
3659
+ payload: {
3660
+ ...event,
3661
+ provider: "codex-subscription",
3662
+ turnId,
3663
+ attemptId: input.attemptId,
3664
+ dispatchId,
3665
+ executionGeneration,
3666
+ },
3667
+ },
3668
+ ]);
3669
+ },
3111
3670
  };
3112
3671
  })()
3113
3672
  : null;
@@ -3116,53 +3675,33 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3116
3675
  const promptCacheKey = acceptsPromptCacheKeyForTurn(resolvedModel)
3117
3676
  ? input.sessionId
3118
3677
  : undefined;
3119
- let compactionUsageCount = 0;
3678
+ const compactionUsageState = createCompactionModelUsageEventState(
3679
+ claimedModelUsageSourceKeys,
3680
+ );
3120
3681
  const recordCompactionUsage = async (usage: ModelResponseUsage) => {
3121
- await recordCompletedModelCallBeforeOwnershipFences({
3682
+ await processCompactionModelUsageEvent({
3683
+ usage,
3684
+ state: compactionUsageState,
3685
+ dispatchId: modelUsageDispatchId,
3686
+ settings,
3687
+ db,
3688
+ observability,
3689
+ publish,
3690
+ accountId: input.accountId,
3691
+ workspaceId: input.workspaceId,
3692
+ sessionId: input.sessionId,
3693
+ turnId: turn.id,
3694
+ provider: resolvedModel?.provider.id ?? settings.openaiProvider,
3695
+ providerApi: resolvedModel?.provider.api ?? "responses",
3696
+ model: resolvedModel?.configured.id ?? turn.model,
3697
+ externallyBilled: isExternallyBilledTurn,
3698
+ turnAttemptId: input.attemptId,
3699
+ servingCredentialId: effectiveCodexCredentialId,
3700
+ priorSessionCredentialId: priorSessionCodexCredentialId,
3701
+ emittedSourceKeys: emittedModelUsageSourceKeys,
3122
3702
  renewLease: () => renewCodexLease("model_usage"),
3123
3703
  leaseLost: () => codexLeaseLost,
3124
3704
  leaseLostMessage: "Codex credential lease expired during context compaction",
3125
- recordUsage: async () => {
3126
- compactionUsageCount += 1;
3127
- const sourceKey = modelUsageSourceKey({
3128
- responseId: usage.responseId,
3129
- dispatchId: modelUsageDispatchId,
3130
- positionalKey: `compaction-${compactionUsageCount}`,
3131
- });
3132
- const responseAccountCtx = modelCallAccountContext({
3133
- servingCredentialId: effectiveCodexCredentialId,
3134
- priorSessionCredentialId: priorSessionCodexCredentialId,
3135
- isFirstCallOfTurn: compactionUsageCount === 1,
3136
- });
3137
- await recordModelUsageAndDebitCredits(settings, db, {
3138
- accountId: input.accountId,
3139
- workspaceId: input.workspaceId,
3140
- sessionId: input.sessionId,
3141
- turnId: turn.id,
3142
- turnAttemptId: input.attemptId,
3143
- model: resolvedModel?.configured.id ?? turn.model,
3144
- isCodexTurn,
3145
- usage: usage.usage,
3146
- sourceKey,
3147
- observability,
3148
- });
3149
- await emitModelCallUsage({
3150
- observability,
3151
- publish,
3152
- accountId: input.accountId,
3153
- workspaceId: input.workspaceId,
3154
- sessionId: input.sessionId,
3155
- turnId: turn.id,
3156
- provider: resolvedModel?.provider.id ?? settings.openaiProvider,
3157
- providerApi: resolvedModel?.provider.api ?? "responses",
3158
- model: resolvedModel?.configured.id ?? turn.model,
3159
- sourceKey,
3160
- usage,
3161
- servingAccountHash: responseAccountCtx.servingAccountHash,
3162
- accountChangedFromPrevCall: responseAccountCtx.accountChangedFromPrevCall,
3163
- emittedSourceKeys: emittedModelUsageSourceKeys,
3164
- });
3165
- },
3166
3705
  });
3167
3706
  };
3168
3707
  const compactionSummarizerFor = (systemInstructions?: string) =>
@@ -3172,7 +3711,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3172
3711
  summarizeForCompaction(s, m, {
3173
3712
  client: resolvedModel.client,
3174
3713
  api: resolvedModel.provider.api,
3175
- model: resolvedModel.configured.id,
3714
+ model: turnExecutionPolicy.upstreamModelId,
3176
3715
  maxOutputTokens: SUMMARY_BUFFER_TOKENS,
3177
3716
  onUsage: recordCompactionUsage,
3178
3717
  ...(systemInstructions ? { systemInstructions } : {}),
@@ -3181,6 +3720,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3181
3720
  )
3182
3721
  : (s: Settings, m: Array<Record<string, unknown>>) =>
3183
3722
  summarizeForCompaction(s, m, {
3723
+ model: turnExecutionPolicy.upstreamModelId,
3184
3724
  maxOutputTokens: SUMMARY_BUFFER_TOKENS,
3185
3725
  onUsage: recordCompactionUsage,
3186
3726
  ...(systemInstructions ? { systemInstructions } : {}),
@@ -3321,9 +3861,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3321
3861
  // connector scopes); no-op for every other turn. Its refreshing bearer is
3322
3862
  // resolved at connect time from the codex ALS (see the withCodex-wrapped
3323
3863
  // prepareTools call below).
3864
+ // Resolve the durable policy at the turn boundary. Workspace-default
3865
+ // sessions may discover newly enabled capability MCPs, while explicit,
3866
+ // inherited-fixed, and legacy sessions remain narrowed to their stored
3867
+ // materialized allow-list. `withCodexAppsTool` below keeps the existing
3868
+ // single Codex lazy router/connector overlay; it is not a second policy
3869
+ // or discovery path.
3870
+ const effectivePolicyTools = resolveSessionToolPolicy({
3871
+ ...(session.toolPolicy ? { toolPolicy: session.toolPolicy } : {}),
3872
+ sessionTools: session.tools,
3873
+ turnTools: turn.tools,
3874
+ ...(turn.toolsProvided !== undefined ? { turnToolsProvided: turn.toolsProvided } : {}),
3875
+ availableMcpServerIds: runSettings.mcpServers.map((server) => server.id),
3876
+ defaultMcpServerIds: enabledCapabilityMcpToolRefs(settings, mcpSettings).map(
3877
+ (tool) => tool.id,
3878
+ ),
3879
+ }).toolRefs;
3324
3880
  const turnTools = withCodexAppsTool(
3325
3881
  runSettings,
3326
- withFirstPartyTools(runSettings, mergeToolRefs(session.tools, turn.tools)),
3882
+ withFirstPartyTools(runSettings, effectivePolicyTools),
3327
3883
  );
3328
3884
  // §7.6 connection-credential provider — load (and decrypt) the variable set via the host
3329
3885
  // `sandboxSecrets` provider when bound; unset → today's local decrypt.
@@ -3553,6 +4109,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3553
4109
  turn,
3554
4110
  })
3555
4111
  : undefined;
4112
+ const toolspaceAuthority = {
4113
+ sessionId: input.sessionId,
4114
+ };
3556
4115
  const {
3557
4116
  environment: sandboxEnvironment,
3558
4117
  gitToken: sandboxGitToken,
@@ -3576,8 +4135,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3576
4135
  ...(gitCredentialAuthority ? { authority: gitCredentialAuthority } : {}),
3577
4136
  gitCredentials: connectionCredentials?.gitCredentials,
3578
4137
  authorizeGitHubTokenMint,
3579
- sessionId: input.sessionId,
3580
- runId: turnId,
4138
+ toolspaceAuthority,
3581
4139
  },
3582
4140
  ),
3583
4141
  cancellationSignal,
@@ -3598,6 +4156,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3598
4156
  const attachGitCredentialRenewal = async (
3599
4157
  tokenSession: GitCredentialTokenWriterSession,
3600
4158
  initial: MintedRunGitCredentials | undefined,
4159
+ initialSandbox?: ResumedTurnSandbox,
3601
4160
  ): Promise<void> => {
3602
4161
  if (!initial || initial.bindings.length === 0) return;
3603
4162
  const previous = gitCredentialRenewals;
@@ -3625,6 +4184,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3625
4184
  authorizeGitHubTokenMint,
3626
4185
  },
3627
4186
  );
4187
+ if (binding) {
4188
+ assertGitCredentialRenewalTransportUnchanged(initialBinding, binding);
4189
+ }
3628
4190
  pendingBinding = binding;
3629
4191
  return binding
3630
4192
  ? {
@@ -3639,16 +4201,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3639
4201
  throw new Error("credential renewal produced no binding token");
3640
4202
  }
3641
4203
  const runAs = sandboxRunAs(runSettings);
3642
- await refreshGitCredentialBindingTokenFiles(tokenSession, [pendingBinding], {
3643
- ...(runAs ? { runAs } : {}),
3644
- ...(toolCancellationFenceRef.current
3645
- ? {
3646
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
3647
- toolCancellationFenceRef.current,
3648
- ),
3649
- }
3650
- : {}),
3651
- });
4204
+ const targetSandbox = resolvedSandbox ?? initialSandbox;
4205
+ if (!targetSandbox) {
4206
+ throw new Error("Git credential renewal has no exact sandbox lease target");
4207
+ }
4208
+ await runWorkspaceMutationForSandbox(
4209
+ targetSandbox,
4210
+ "gitCredentialRenewal",
4211
+ async () =>
4212
+ await refreshGitCredentialBindingTokenFiles(tokenSession, [pendingBinding!], {
4213
+ ...(runAs ? { runAs } : {}),
4214
+ ...(toolCancellationFenceRef.current
4215
+ ? {
4216
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4217
+ toolCancellationFenceRef.current,
4218
+ ),
4219
+ }
4220
+ : {}),
4221
+ }),
4222
+ );
3652
4223
  },
3653
4224
  onSuccess: ({ providers: renewedProviders }) => {
3654
4225
  for (const provider of renewedProviders) {
@@ -3687,6 +4258,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3687
4258
  const attachToolspaceTokenRenewal = async (
3688
4259
  tokenSession: ToolspaceTokenWriterSession,
3689
4260
  initialExpiresAt = sandboxToolspaceTokenExpiresAt,
4261
+ initialSandbox?: ResumedTurnSandbox,
3690
4262
  ): Promise<void> => {
3691
4263
  if (!sandboxToolspaceToken || !initialExpiresAt) return;
3692
4264
  const previous = toolspaceTokenRenewal;
@@ -3695,25 +4267,34 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3695
4267
  if (toolspaceTokenRenewalClosed) return;
3696
4268
 
3697
4269
  const mint = async () =>
3698
- await mintSandboxToolspaceToken(runSettings, connectionScope, input.sessionId, turn.id);
4270
+ await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority);
3699
4271
  const write = async (material: NonNullable<Awaited<ReturnType<typeof mint>>>) => {
3700
4272
  const runAs = sandboxRunAs(runSettings);
3701
- await refreshToolspaceTokenFile(tokenSession, material.token, {
3702
- ...(runAs ? { runAs } : {}),
3703
- ...(sandboxToolspaceTokenFile
3704
- ? {
3705
- tokenFile: sandboxToolspaceTokenFile,
3706
- legacyTokenFile: sandboxEnvironment.OPENGENI_TOOLSPACE_TOKEN_FILE!,
3707
- }
3708
- : {}),
3709
- ...(toolCancellationFenceRef.current
3710
- ? {
3711
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
3712
- toolCancellationFenceRef.current,
3713
- ),
3714
- }
3715
- : {}),
3716
- });
4273
+ const targetSandbox = resolvedSandbox ?? initialSandbox;
4274
+ if (!targetSandbox) {
4275
+ throw new Error("Toolspace token renewal has no exact sandbox lease target");
4276
+ }
4277
+ await runWorkspaceMutationForSandbox(
4278
+ targetSandbox,
4279
+ "toolspaceTokenRenewal",
4280
+ async () =>
4281
+ await refreshToolspaceTokenFile(tokenSession, material.token, {
4282
+ ...(runAs ? { runAs } : {}),
4283
+ ...(sandboxToolspaceTokenFile
4284
+ ? {
4285
+ tokenFile: sandboxToolspaceTokenFile,
4286
+ legacyTokenFile: sandboxEnvironment.OPENGENI_TOOLSPACE_TOKEN_FILE!,
4287
+ }
4288
+ : {}),
4289
+ ...(toolCancellationFenceRef.current
4290
+ ? {
4291
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4292
+ toolCancellationFenceRef.current,
4293
+ ),
4294
+ }
4295
+ : {}),
4296
+ }),
4297
+ );
3717
4298
  };
3718
4299
  let renewalExpiresAt = initialExpiresAt;
3719
4300
  if (renewalExpiresAt.getTime() <= Date.now() + TOOLSPACE_TOKEN_EXPIRY_LEAD_MS) {
@@ -3758,6 +4339,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3758
4339
 
3759
4340
  const attachRunCredentialRenewal = async (
3760
4341
  credentialSession: RunCredentialCommandSession,
4342
+ initialSandbox?: ResumedTurnSandbox,
3761
4343
  ): Promise<void> => {
3762
4344
  if (!runCredentialResolver) return;
3763
4345
  const previous = runCredentialRenewal;
@@ -3766,15 +4348,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3766
4348
  if (runCredentialRenewalClosed) return;
3767
4349
  runCredentialSession = credentialSession;
3768
4350
 
4351
+ const requireTargetSandbox = (): ResumedTurnSandbox => {
4352
+ const targetSandbox = resolvedSandbox ?? initialSandbox;
4353
+ if (!targetSandbox) {
4354
+ throw new Error("Run credential mutation has no exact sandbox lease target");
4355
+ }
4356
+ return targetSandbox;
4357
+ };
4358
+
3769
4359
  if (!initialRunCredentialMaterial) {
3770
- await clearRunCredentials(
3771
- credentialSession,
3772
- input.sessionId,
3773
- toolCancellationFenceRef.current
3774
- ? toolCancellationFenceRef.current.runSandboxCommand.bind(
3775
- toolCancellationFenceRef.current,
3776
- )
3777
- : undefined,
4360
+ await runWorkspaceMutationForSandbox(
4361
+ requireTargetSandbox(),
4362
+ "runCredentialClear",
4363
+ async () =>
4364
+ await clearRunCredentials(
4365
+ credentialSession,
4366
+ input.sessionId,
4367
+ toolCancellationFenceRef.current
4368
+ ? toolCancellationFenceRef.current.runSandboxCommand.bind(
4369
+ toolCancellationFenceRef.current,
4370
+ )
4371
+ : undefined,
4372
+ ),
3778
4373
  );
3779
4374
  return;
3780
4375
  }
@@ -3784,33 +4379,43 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3784
4379
  pruneOtherAttempts = false,
3785
4380
  ): Promise<void> => {
3786
4381
  if (!material) {
3787
- await clearRunCredentialsForAttempt(credentialSession, {
3788
- sessionId: input.sessionId,
3789
- attemptId: input.attemptId,
3790
- executionGeneration,
3791
- });
4382
+ await runWorkspaceMutationForSandbox(
4383
+ requireTargetSandbox(),
4384
+ "runCredentialAttemptClear",
4385
+ async () =>
4386
+ await clearRunCredentialsForAttempt(credentialSession, {
4387
+ sessionId: input.sessionId,
4388
+ attemptId: input.attemptId,
4389
+ executionGeneration,
4390
+ }),
4391
+ );
3792
4392
  return;
3793
4393
  }
3794
4394
  registerSecretRedactions(material.redactions);
3795
- await materializeRunCredentials(credentialSession, material, {
3796
- sessionId: input.sessionId,
3797
- attemptId: input.attemptId,
3798
- executionGeneration,
3799
- ...(pruneOtherAttempts ? { pruneOtherAttempts: true } : {}),
3800
- ...(!pruneOtherAttempts ? { pruneSupersededGenerations: true } : {}),
3801
- ...(material.authNeeded.length > 0 &&
3802
- Object.keys(material.environment).length === 0 &&
3803
- material.files.length === 0
3804
- ? { prunePreviousGenerations: true }
3805
- : {}),
3806
- ...(toolCancellationFenceRef.current
3807
- ? {
3808
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
3809
- toolCancellationFenceRef.current,
3810
- ),
3811
- }
3812
- : {}),
3813
- });
4395
+ await runWorkspaceMutationForSandbox(
4396
+ requireTargetSandbox(),
4397
+ "runCredentialMaterialization",
4398
+ async () =>
4399
+ await materializeRunCredentials(credentialSession, material, {
4400
+ sessionId: input.sessionId,
4401
+ attemptId: input.attemptId,
4402
+ executionGeneration,
4403
+ ...(pruneOtherAttempts ? { pruneOtherAttempts: true } : {}),
4404
+ ...(!pruneOtherAttempts ? { pruneSupersededGenerations: true } : {}),
4405
+ ...(material.authNeeded.length > 0 &&
4406
+ Object.keys(material.environment).length === 0 &&
4407
+ material.files.length === 0
4408
+ ? { prunePreviousGenerations: true }
4409
+ : {}),
4410
+ ...(toolCancellationFenceRef.current
4411
+ ? {
4412
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4413
+ toolCancellationFenceRef.current,
4414
+ ),
4415
+ }
4416
+ : {}),
4417
+ }),
4418
+ );
3814
4419
  for (const payload of runCredentialAuthNeededPayloads(material)) {
3815
4420
  const key = JSON.stringify(payload);
3816
4421
  if (publishedRunCredentialNotices.has(key)) continue;
@@ -3956,11 +4561,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
3956
4561
  settings,
3957
4562
  bus,
3958
4563
  onOp: machineOpObserver.observer,
4564
+ onHomeSandboxRebound,
3959
4565
  },
3960
4566
  {
3961
4567
  workspaceId: input.workspaceId,
3962
4568
  sessionId: input.sessionId,
3963
4569
  environment: sandboxEnvironment,
4570
+ workspaceMutationFence: {
4571
+ accountId: input.accountId,
4572
+ turnId: turn.id,
4573
+ executionGeneration,
4574
+ attemptId: input.attemptId,
4575
+ },
3964
4576
  pinnedSelfhosted: {
3965
4577
  sandboxId: activeSandboxPointer!.activeSandboxId!,
3966
4578
  epoch: activeSandboxPointer!.activeEpoch,
@@ -4045,24 +4657,41 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4045
4657
  // the flag off the established group box is injected unchanged (today's
4046
4658
  // path). The lease still owns the group box lifecycle — the proxy is a
4047
4659
  // routing veneer, not an owner.
4048
- if (routingEnabled(settings)) {
4049
- resolvedSandbox = {
4050
- ...resolvedSandbox,
4051
- established: wrapTurnBoxWithRouting(
4052
- { db, settings, bus },
4053
- // Thread the SAME declared environment the group box was created with
4054
- // (resumeBoxForTurn, above) so a selfhosted swap target's manifest
4055
- // carries it too — the SDK's per-turn manifest-env delta stays empty
4056
- // (no "cannot change manifest environment variables" throw).
4057
- {
4058
- workspaceId: input.workspaceId,
4059
- sessionId: input.sessionId,
4060
- environment: sandboxEnvironment,
4660
+ resolvedSandbox = {
4661
+ ...resolvedSandbox,
4662
+ established: wrapTurnBoxWithRouting(
4663
+ {
4664
+ db,
4665
+ settings,
4666
+ bus,
4667
+ onHomeSandboxLost: publishSandboxLost,
4668
+ onHomeSandboxRebound,
4669
+ },
4670
+ // Thread the SAME declared environment the group box was created with
4671
+ // (resumeBoxForTurn, above) so a selfhosted swap target's manifest
4672
+ // carries it too — the SDK's per-turn manifest-env delta stays empty
4673
+ // (no "cannot change manifest environment variables" throw).
4674
+ {
4675
+ workspaceId: input.workspaceId,
4676
+ sessionId: input.sessionId,
4677
+ environment: sandboxEnvironment,
4678
+ workspaceMutationFence: {
4679
+ accountId: input.accountId,
4680
+ turnId: turn.id,
4681
+ executionGeneration,
4682
+ attemptId: input.attemptId,
4061
4683
  },
4062
- resolvedSandbox.established,
4063
- ),
4064
- };
4065
- }
4684
+ homeLease: {
4685
+ accountId: input.accountId,
4686
+ sandboxGroupId: session.sandboxGroupId,
4687
+ leaseEpoch: resolvedSandbox.leaseEpoch,
4688
+ instanceId: resolvedSandbox.established.instanceId,
4689
+ backend: groupBoxBackend,
4690
+ },
4691
+ },
4692
+ resolvedSandbox.established,
4693
+ ),
4694
+ };
4066
4695
  }
4067
4696
  if (resolvedSandbox) {
4068
4697
  startLeaseHeartbeat(resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
@@ -4197,9 +4826,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4197
4826
  // token skip does NOT transfer: that token is inert on a connected
4198
4827
  // machine (it uses its own git creds), but the toolspace token is the
4199
4828
  // machine's ONLY path to programmatic tool calling and grants no more
4200
- // than toolspace:call for its own session (own-session-bound, turn TTL,
4201
- // budgeted, approval-tools excluded). The runtime seeds it to the box's
4202
- // token file over the same exec channel, off-manifest, on every backend.
4829
+ // than toolspace:call for its own session (own-session-bound,
4830
+ // short-lived and renewable, per-active-turn budgeted, approval-tools
4831
+ // excluded). The runtime seeds it to the box's token file over the same
4832
+ // exec channel, off-manifest, on every backend.
4203
4833
  ...(sandboxToolspaceToken
4204
4834
  ? {
4205
4835
  toolspaceTokenSeed: sandboxToolspaceToken,
@@ -4342,12 +4972,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4342
4972
  });
4343
4973
  const lazyGitTokens = lazyGitCredentials?.gitTokens;
4344
4974
  const lazyToolspaceToken = sandboxToolspaceToken
4345
- ? await mintSandboxToolspaceToken(
4346
- runSettings,
4347
- connectionScope,
4348
- input.sessionId,
4349
- turn.id,
4350
- )
4975
+ ? await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority)
4351
4976
  : undefined;
4352
4977
  const provisioned = await resumeBoxForTurn(
4353
4978
  {
@@ -4376,6 +5001,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4376
5001
  await publishSandboxLifecycleEvents(provisioned);
4377
5002
  await attachRunCredentialRenewal(
4378
5003
  provisioned.established.session as RunCredentialCommandSession,
5004
+ provisioned,
4379
5005
  );
4380
5006
  const provisionedSetupSession = initialRunCredentialMaterial
4381
5007
  ? withRunCredentialsSession(
@@ -4383,39 +5009,46 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4383
5009
  input.sessionId,
4384
5010
  )
4385
5011
  : provisioned.established.session;
4386
- await runOwnedSandboxSetup(
4387
- agent,
4388
- provisioned.established.session as never,
4389
- provisionedSetupSession as never,
4390
- {
4391
- settings: runSettings,
4392
- environment: sandboxEnvironment,
4393
- onRuntimeEvent: async (event) => {
4394
- await publish?.([{ type: event.type, payload: event.payload }], true);
4395
- },
4396
- ...(lazyGitTokens ? { gitTokenSeedsOverride: lazyGitTokens } : {}),
4397
- ...(lazyGitCredentials?.bindings
4398
- ? { gitCredentialBindingsOverride: lazyGitCredentials.bindings }
4399
- : {}),
4400
- ...(lazyToolspaceToken
4401
- ? { toolspaceTokenSeedOverride: lazyToolspaceToken.token }
4402
- : {}),
4403
- ...(toolCancellationFenceRef.current
4404
- ? {
4405
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4406
- toolCancellationFenceRef.current,
4407
- ),
4408
- }
4409
- : {}),
4410
- },
5012
+ await runWorkspaceMutationForSandbox(
5013
+ provisioned,
5014
+ "lazyOwnedSandboxSetup",
5015
+ async () =>
5016
+ await runOwnedSandboxSetup(
5017
+ agent,
5018
+ provisioned.established.session as never,
5019
+ provisionedSetupSession as never,
5020
+ {
5021
+ settings: runSettings,
5022
+ environment: sandboxEnvironment,
5023
+ onRuntimeEvent: async (event) => {
5024
+ await publish?.([{ type: event.type, payload: event.payload }], true);
5025
+ },
5026
+ ...(lazyGitTokens ? { gitTokenSeedsOverride: lazyGitTokens } : {}),
5027
+ ...(lazyGitCredentials?.bindings
5028
+ ? { gitCredentialBindingsOverride: lazyGitCredentials.bindings }
5029
+ : {}),
5030
+ ...(lazyToolspaceToken
5031
+ ? { toolspaceTokenSeedOverride: lazyToolspaceToken.token }
5032
+ : {}),
5033
+ ...(toolCancellationFenceRef.current
5034
+ ? {
5035
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
5036
+ toolCancellationFenceRef.current,
5037
+ ),
5038
+ }
5039
+ : {}),
5040
+ },
5041
+ ),
4411
5042
  );
4412
5043
  await attachToolspaceTokenRenewal(
4413
5044
  provisioned.established.session as ToolspaceTokenWriterSession,
4414
5045
  lazyToolspaceToken?.expiresAt,
5046
+ provisioned,
4415
5047
  );
4416
5048
  await attachGitCredentialRenewal(
4417
5049
  provisioned.established.session as GitCredentialTokenWriterSession,
4418
5050
  lazyGitCredentials,
5051
+ provisioned,
4419
5052
  );
4420
5053
  // Return the REAL established box (NOT a copy whose session is the routing
4421
5054
  // proxy). resolveActiveBackend dispatches ops to `provisioned.established.session`;
@@ -4485,17 +5118,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4485
5118
  settings,
4486
5119
  bus,
4487
5120
  onOp: machineOpObserver.observer,
5121
+ onHomeSandboxLost: publishSandboxLost,
5122
+ onHomeSandboxRebound,
4488
5123
  },
4489
5124
  {
4490
5125
  workspaceId: input.workspaceId,
4491
5126
  sessionId: input.sessionId,
4492
5127
  environment: sandboxEnvironment,
5128
+ workspaceMutationFence: {
5129
+ accountId: input.accountId,
5130
+ turnId: turn.id,
5131
+ executionGeneration,
5132
+ attemptId: input.attemptId,
5133
+ },
4493
5134
  },
4494
5135
  {
4495
5136
  client: lazyClient,
4496
5137
  backendId: sdkBackendIdForSandboxBackend(groupBoxBackend),
4497
5138
  agentDefaultManifest,
4498
5139
  provisioner: turnSandboxProvisioner,
5140
+ homeLeaseIdentity: {
5141
+ accountId: input.accountId,
5142
+ sandboxGroupId: session.sandboxGroupId,
5143
+ backend: groupBoxBackend,
5144
+ },
4499
5145
  },
4500
5146
  );
4501
5147
  }
@@ -4617,22 +5263,27 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4617
5263
  );
4618
5264
  const runAs = sandboxRunAs(runSettings);
4619
5265
  if (downloadsToMaterialize.length > 0) {
4620
- const materialized = await materializeSandboxFileDownloads(
4621
- setupBoxSession as any,
4622
- downloadsToMaterialize,
4623
- {
4624
- onRuntimeEvent: async (event) => {
4625
- await publish!([{ type: event.type, payload: event.payload }], true);
4626
- },
4627
- ...(runAs ? { runAs } : {}),
4628
- ...(toolCancellationFenceRef.current
4629
- ? {
4630
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
4631
- toolCancellationFenceRef.current,
4632
- ),
4633
- }
4634
- : {}),
4635
- },
5266
+ const materialized = await runWorkspaceMutationForSandbox(
5267
+ resolvedSandbox,
5268
+ "fileMaterialization",
5269
+ async () =>
5270
+ await materializeSandboxFileDownloads(
5271
+ setupBoxSession as any,
5272
+ downloadsToMaterialize,
5273
+ {
5274
+ onRuntimeEvent: async (event) => {
5275
+ await publish!([{ type: event.type, payload: event.payload }], true);
5276
+ },
5277
+ ...(runAs ? { runAs } : {}),
5278
+ ...(toolCancellationFenceRef.current
5279
+ ? {
5280
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
5281
+ toolCancellationFenceRef.current,
5282
+ ),
5283
+ }
5284
+ : {}),
5285
+ },
5286
+ ),
4636
5287
  );
4637
5288
  fileMaterializationFailures = materialized.failures;
4638
5289
  const failedFileIds = new Set(materialized.failures.map((failure) => failure.fileId));
@@ -4680,6 +5331,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4680
5331
  recovering: turn.executionGeneration > 1,
4681
5332
  ...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
4682
5333
  ...(runCredentialsNote ? { runCredentialsNote } : {}),
5334
+ ...(modelAcceptsTypedAttachmentContentForTurn(resolvedModel) && objectStorage
5335
+ ? {
5336
+ readFileBytesForModel: (file) => objectStorage.getFileBytes(file),
5337
+ }
5338
+ : {}),
4683
5339
  },
4684
5340
  );
4685
5341
  runInput = prepared.input;
@@ -4769,13 +5425,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4769
5425
  return outcome;
4770
5426
  };
4771
5427
 
5428
+ // Keep response usage identity across every stream attempt in this
5429
+ // activity. Context compaction retries the same logical dispatch by
5430
+ // calling runStreamAttempt again; resetting this state there would reuse
5431
+ // the first no-response-ID fallback key and suppress a real model call.
5432
+ const modelResponseUsageState = createModelResponseUsageEventState(
5433
+ claimedModelUsageSourceKeys,
5434
+ );
4772
5435
  const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
4773
5436
  if (!runInput) {
4774
5437
  throw new Error("Run input was not prepared");
4775
5438
  }
4776
5439
  stream = undefined;
4777
5440
  batcher = null;
4778
- let responseUsageCount = 0;
4779
5441
  // The SDK emits every processed call item for one model response before
4780
5442
  // it emits any result for that response. Keep that response-local batch
4781
5443
  // in memory so an orphan from an older response cannot pin later stable
@@ -4785,13 +5447,53 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4785
5447
  let currentToolBatchCompletedCallIds = new Set<string>();
4786
5448
  // Actual input tokens of the most recent model response this turn; the
4787
5449
  // pre-read trigger for the NEXT turn. Persisted at every turn-end path.
4788
- let lastProviderContextTokensObserved: number | null = null;
4789
- let providerContextRevision = 0;
4790
5450
  throwIfWorkerShuttingDown();
4791
5451
  throwIfTurnCancelled();
4792
5452
  const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
4793
- const runStreamOnce = (): ReturnType<OpenGeniRuntime["runStream"]> =>
4794
- runtime.runStream(agent, runInput!, modelRunSettings, {
5453
+ const runStreamOnce = async (): ReturnType<OpenGeniRuntime["runStream"]> => {
5454
+ // Eager owned sessions must settle the exact platform-setup provider
5455
+ // promise before the long model stream starts; otherwise one admission
5456
+ // would remain in flight for the entire turn and suppress every
5457
+ // heartbeat capture. Lazy setup already runs under the same wrapper in
5458
+ // its first-operation provisioner above.
5459
+ if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
5460
+ const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
5461
+ await runWorkspaceMutationForSandbox(
5462
+ resolvedSandbox,
5463
+ "eagerOwnedSandboxSetup",
5464
+ async () =>
5465
+ await runOwnedSandboxSetup(
5466
+ agent,
5467
+ ownedEstablished.session as never,
5468
+ eagerSetupSession as never,
5469
+ {
5470
+ settings: runSettings,
5471
+ environment: sandboxEnvironment,
5472
+ preparedInput: runInput!,
5473
+ ...(fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {}),
5474
+ onRuntimeEvent: async (event) => {
5475
+ await renewCodexLease("runtime_event");
5476
+ if (codexLeaseLost) {
5477
+ throw new Error("Codex credential lease expired during sandbox setup");
5478
+ }
5479
+ await publish!([{ type: event.type, payload: event.payload }], true);
5480
+ },
5481
+ ...(toolCancellationFenceRef.current
5482
+ ? {
5483
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
5484
+ toolCancellationFenceRef.current,
5485
+ ),
5486
+ }
5487
+ : {}),
5488
+ },
5489
+ ),
5490
+ );
5491
+ await attachGitCredentialRenewal(
5492
+ eagerSetupSession as GitCredentialTokenWriterSession,
5493
+ initialGitCredentials,
5494
+ );
5495
+ }
5496
+ return await runtime.runStream(agent, runInput!, modelRunSettings, {
4795
5497
  ...(activityContext ? { signal: activityContext.cancellationSignal } : {}),
4796
5498
  sandboxEnvironment,
4797
5499
  onRuntimeEvent: async (event) => {
@@ -4818,16 +5520,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4818
5520
  // re-route those execs onto a machine swapped in mid-turn.
4819
5521
  ...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
4820
5522
  ...(fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {}),
4821
- ...(lazyOwnedSandbox ? { deferredSetup: true } : {}),
4822
- },
4823
- }
4824
- : {}),
4825
- ...(initialGitCredentials
4826
- ? {
4827
- onGitCredentialSessionReady: async (
4828
- tokenSession: GitCredentialTokenWriterSession,
4829
- ) => {
4830
- await attachGitCredentialRenewal(tokenSession, initialGitCredentials);
5523
+ // Both owned paths execute setup outside runStream: eager just
5524
+ // above under its exact admission, lazy in the provisioner.
5525
+ deferredSetup: true,
4831
5526
  },
4832
5527
  }
4833
5528
  : {}),
@@ -4861,19 +5556,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4861
5556
  : {}),
4862
5557
  }
4863
5558
  : {}),
4864
- contextCompactionSignal: () =>
4865
- lastProviderContextTokensObserved === null
4866
- ? null
4867
- : {
4868
- revision: providerContextRevision,
4869
- totalTokens: lastProviderContextTokensObserved,
4870
- },
5559
+ contextCompactionSignal: () => modelResponseUsageContextSignal(modelResponseUsageState),
4871
5560
  contextCompactionRequested: () =>
4872
5561
  isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
4873
5562
  ...(toolCancellationFenceRef.current
4874
5563
  ? { turnToolCancellationFence: toolCancellationFenceRef.current }
4875
5564
  : {}),
4876
5565
  });
5566
+ };
4877
5567
  if (codexLeaseLost) {
4878
5568
  throw new Error("Codex credential lease expired before the model run");
4879
5569
  }
@@ -4906,77 +5596,33 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4906
5596
  }
4907
5597
  let stableToolCallIdsToClear: string[] | null = null;
4908
5598
  let completedCurrentToolBatch = false;
4909
- const responseUsage = modelResponseUsageFromSdkEvent(next.value);
4910
- if (responseUsage) {
4911
- await recordCompletedModelCallBeforeOwnershipFences({
4912
- renewLease: () => renewCodexLease("model_usage"),
4913
- leaseLost: () => codexLeaseLost,
4914
- leaseLostMessage: "Codex credential lease expired during the active turn",
4915
- recordUsage: async () => {
4916
- responseUsageCount += 1;
4917
- const responseSourceKey = modelUsageSourceKey({
4918
- responseId: responseUsage.responseId,
4919
- dispatchId: modelUsageDispatchId,
4920
- positionalKey: `response-${responseUsageCount}`,
4921
- });
4922
- // Within a turn the serving credential is fixed, so a switch can only
4923
- // surface on the turn's FIRST model call (vs the session's prior).
4924
- const responseAccountCtx = modelCallAccountContext({
4925
- servingCredentialId: effectiveCodexCredentialId,
4926
- priorSessionCredentialId: priorSessionCodexCredentialId,
4927
- isFirstCallOfTurn: responseUsageCount === 1,
4928
- });
4929
- await recordModelUsageAndDebitCredits(settings, db, {
4930
- accountId: input.accountId,
4931
- workspaceId: input.workspaceId,
4932
- sessionId: input.sessionId,
4933
- turnId: activeTurnId,
4934
- turnAttemptId: input.attemptId,
4935
- model: turn.model,
4936
- isCodexTurn,
4937
- usage: responseUsage.usage,
4938
- sourceKey: responseSourceKey,
4939
- observability,
4940
- });
4941
- await emitModelCallUsage({
4942
- observability,
4943
- publish,
4944
- accountId: input.accountId,
4945
- workspaceId: input.workspaceId,
4946
- sessionId: input.sessionId,
4947
- turnId: activeTurnId,
4948
- provider: resolvedModel?.provider.id ?? settings.openaiProvider,
4949
- providerApi: resolvedModel?.provider.api ?? "responses",
4950
- model: turn.model,
4951
- sourceKey: responseSourceKey,
4952
- usage: responseUsage,
4953
- servingAccountHash: responseAccountCtx.servingAccountHash,
4954
- accountChangedFromPrevCall: responseAccountCtx.accountChangedFromPrevCall,
4955
- emittedSourceKeys: emittedModelUsageSourceKeys,
4956
- });
4957
- const observed = responseUsage.usage?.inputTokens;
4958
- if (typeof observed === "number" && observed > 0) {
4959
- recordModelInputTokens(observability, streamProvider, observed);
4960
- }
4961
- const observedTotal = providerContextTokens(responseUsage.usage);
4962
- if (observedTotal !== null) {
4963
- lastProviderContextTokensObserved = observedTotal;
4964
- providerContextRevision += 1;
4965
- }
4966
- // Prompt-cache efficiency for this response — same usage frame as the
4967
- // input-token accounting above, so the two are always consistent.
4968
- recordModelCacheTokens(observability, streamProvider, {
4969
- cachedTokens: modelCallUsageTelemetry(responseUsage.usage).cachedTokens,
4970
- promptTokens: responseUsage.usage?.inputTokens,
4971
- });
4972
- },
4973
- recordAttemptSignals: async () => {
4974
- const observed = responseUsage.usage?.inputTokens;
4975
- if (typeof observed === "number" && observed > 0) {
4976
- await setLastInputTokensFenced(observed);
4977
- }
4978
- },
4979
- });
5599
+ const responseUsageResult = await processModelResponseUsageEvent({
5600
+ event: next.value,
5601
+ state: modelResponseUsageState,
5602
+ dispatchId: modelUsageDispatchId,
5603
+ settings,
5604
+ db,
5605
+ observability,
5606
+ publish,
5607
+ accountId: input.accountId,
5608
+ workspaceId: input.workspaceId,
5609
+ sessionId: input.sessionId,
5610
+ turnId: activeTurnId,
5611
+ turnAttemptId: input.attemptId,
5612
+ provider: resolvedModel?.provider.id ?? settings.openaiProvider,
5613
+ providerApi: resolvedModel?.provider.api ?? "responses",
5614
+ model: turn.model,
5615
+ metricProvider: streamProvider,
5616
+ externallyBilled: isExternallyBilledTurn,
5617
+ servingCredentialId: effectiveCodexCredentialId,
5618
+ priorSessionCredentialId: priorSessionCodexCredentialId,
5619
+ emittedSourceKeys: emittedModelUsageSourceKeys,
5620
+ renewLease: () => renewCodexLease("model_usage"),
5621
+ leaseLost: () => codexLeaseLost,
5622
+ leaseLostMessage: "Codex credential lease expired during the active turn",
5623
+ setLastInputTokens: setLastInputTokensFenced,
5624
+ });
5625
+ if (responseUsageResult.status === "processed") {
4980
5626
  currentToolBatchCallIds = new Set<string>();
4981
5627
  currentToolBatchCompletedCallIds = new Set<string>();
4982
5628
  await reconcileConversationTruth();
@@ -4986,7 +5632,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
4986
5632
  db,
4987
5633
  input.accountId,
4988
5634
  input.workspaceId,
4989
- isCodexTurn,
5635
+ isExternallyBilledTurn,
4990
5636
  entitlements,
4991
5637
  );
4992
5638
  } catch (limitError) {
@@ -5014,6 +5660,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5014
5660
  turnId: activeTurnId,
5015
5661
  executionGeneration,
5016
5662
  attemptId: input.attemptId,
5663
+ modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
5017
5664
  ...pendingToolCall,
5018
5665
  });
5019
5666
  if (!registered.accepted) {
@@ -5038,6 +5685,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5038
5685
  executionGeneration,
5039
5686
  attemptId: input.attemptId,
5040
5687
  callId: completedToolCall.callId,
5688
+ modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
5041
5689
  resultItem: completedToolCall.resultItem,
5042
5690
  });
5043
5691
  if (!recorded.accepted) {
@@ -5111,83 +5759,77 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5111
5759
  stream.completed.catch(() => undefined),
5112
5760
  cancellationSignal,
5113
5761
  );
5114
- if (responseUsageCount === 0) {
5762
+ if (modelResponseUsageState.responseUsageCount === 0) {
5115
5763
  const aggregateUsage = stream.state.usage;
5116
- const aggregateInput = (aggregateUsage as { inputTokens?: unknown } | undefined)
5117
- ?.inputTokens;
5118
- if (typeof aggregateInput === "number" && aggregateInput > 0) {
5119
- const aggregateContext = providerContextTokens(
5120
- aggregateUsage as {
5121
- inputTokens?: number;
5122
- outputTokens?: number;
5123
- totalTokens?: number;
5124
- },
5125
- );
5126
- if (aggregateContext !== null) {
5127
- lastProviderContextTokensObserved = aggregateContext;
5128
- providerContextRevision += 1;
5129
- }
5130
- }
5764
+ const normalizedAggregateUsage = normalizeModelCallUsage(aggregateUsage);
5765
+ const aggregateInput = normalizedAggregateUsage.telemetry.inputTokens;
5131
5766
  const aggregateSourceKey = modelUsageSourceKey({
5132
5767
  responseId: null,
5133
5768
  dispatchId: modelUsageDispatchId,
5134
5769
  positionalKey: "aggregate",
5135
5770
  });
5136
- // The single aggregate frame is this turn's only model-usage record, so
5137
- // it is the first (account-switch surfaces here just like a first response).
5138
- const aggregateAccountCtx = modelCallAccountContext({
5139
- servingCredentialId: effectiveCodexCredentialId,
5140
- priorSessionCredentialId: priorSessionCodexCredentialId,
5141
- isFirstCallOfTurn: true,
5142
- });
5143
- recordModelCacheTokens(observability, streamProvider, {
5144
- cachedTokens: modelCallUsageTelemetry(
5145
- aggregateUsage as Parameters<typeof modelCallUsageTelemetry>[0],
5146
- ).cachedTokens,
5147
- promptTokens: (aggregateUsage as { inputTokens?: unknown } | undefined)?.inputTokens as
5148
- | number
5149
- | undefined,
5150
- });
5151
- await recordCompletedModelCallBeforeOwnershipFences({
5152
- renewLease: () => renewCodexLease("model_usage"),
5153
- leaseLost: () => codexLeaseLost,
5154
- leaseLostMessage: "Codex credential lease expired during the active turn",
5155
- recordUsage: async () => {
5156
- await recordModelUsageAndDebitCredits(settings, db, {
5157
- accountId: input.accountId,
5158
- workspaceId: input.workspaceId,
5159
- sessionId: input.sessionId,
5160
- turnId: activeTurnId,
5161
- turnAttemptId: input.attemptId,
5162
- model: turn.model,
5163
- isCodexTurn,
5164
- usage: aggregateUsage,
5165
- sourceKey: aggregateSourceKey,
5166
- observability,
5167
- });
5168
- await emitModelCallUsage({
5169
- observability,
5170
- publish,
5171
- accountId: input.accountId,
5172
- workspaceId: input.workspaceId,
5173
- sessionId: input.sessionId,
5174
- turnId: activeTurnId,
5175
- provider: resolvedModel?.provider.id ?? settings.openaiProvider,
5176
- providerApi: resolvedModel?.provider.api ?? "responses",
5177
- model: turn.model,
5178
- sourceKey: aggregateSourceKey,
5179
- usage: { usage: aggregateUsage },
5180
- servingAccountHash: aggregateAccountCtx.servingAccountHash,
5181
- accountChangedFromPrevCall: aggregateAccountCtx.accountChangedFromPrevCall,
5182
- emittedSourceKeys: emittedModelUsageSourceKeys,
5183
- });
5184
- },
5185
- recordAttemptSignals: async () => {
5186
- if (typeof aggregateInput === "number" && aggregateInput > 0) {
5187
- await setLastInputTokensFenced(aggregateInput);
5188
- }
5189
- },
5190
- });
5771
+ if (!claimedModelUsageSourceKeys.has(aggregateSourceKey)) {
5772
+ claimedModelUsageSourceKeys.add(aggregateSourceKey);
5773
+ // The single aggregate frame is this turn's only model-usage record, so
5774
+ // it is the first (account-switch surfaces here just like a first response).
5775
+ const aggregateAccountCtx = modelCallAccountContext({
5776
+ servingCredentialId: effectiveCodexCredentialId,
5777
+ priorSessionCredentialId: priorSessionCodexCredentialId,
5778
+ isFirstCallOfTurn: true,
5779
+ });
5780
+ let aggregateAuthoritative = false;
5781
+ await recordCompletedModelCallBeforeOwnershipFences({
5782
+ renewLease: () => renewCodexLease("model_usage"),
5783
+ leaseLost: () => codexLeaseLost,
5784
+ leaseLostMessage: "Codex credential lease expired during the active turn",
5785
+ recordUsage: async () => {
5786
+ await recordModelUsageAndDebitCredits(settings, db, {
5787
+ accountId: input.accountId,
5788
+ workspaceId: input.workspaceId,
5789
+ sessionId: input.sessionId,
5790
+ turnId: activeTurnId,
5791
+ turnAttemptId: input.attemptId,
5792
+ model: turn.model,
5793
+ externallyBilled: isExternallyBilledTurn,
5794
+ usage: aggregateUsage,
5795
+ normalizedUsage: normalizedAggregateUsage,
5796
+ sourceKey: aggregateSourceKey,
5797
+ observability,
5798
+ });
5799
+ aggregateAuthoritative = await emitModelCallUsage({
5800
+ observability,
5801
+ publish,
5802
+ accountId: input.accountId,
5803
+ workspaceId: input.workspaceId,
5804
+ sessionId: input.sessionId,
5805
+ turnId: activeTurnId,
5806
+ provider: resolvedModel?.provider.id ?? settings.openaiProvider,
5807
+ providerApi: resolvedModel?.provider.api ?? "responses",
5808
+ model: turn.model,
5809
+ sourceKey: aggregateSourceKey,
5810
+ usage: { usage: aggregateUsage },
5811
+ normalizedUsage: normalizedAggregateUsage,
5812
+ servingAccountHash: aggregateAccountCtx.servingAccountHash,
5813
+ accountChangedFromPrevCall: aggregateAccountCtx.accountChangedFromPrevCall,
5814
+ emittedSourceKeys: emittedModelUsageSourceKeys,
5815
+ });
5816
+ if (aggregateAuthoritative && aggregateInput !== null && aggregateInput > 0) {
5817
+ recordModelInputTokens(observability, streamProvider, aggregateInput);
5818
+ }
5819
+ },
5820
+ recordAttemptSignals: async () => {
5821
+ if (!aggregateAuthoritative) return;
5822
+ if (normalizedAggregateUsage.totalTokens !== null) {
5823
+ modelResponseUsageState.lastProviderContextTokensObserved =
5824
+ normalizedAggregateUsage.totalTokens;
5825
+ modelResponseUsageState.providerContextRevision += 1;
5826
+ }
5827
+ if (aggregateInput !== null && aggregateInput > 0) {
5828
+ await setLastInputTokensFenced(aggregateInput);
5829
+ }
5830
+ },
5831
+ });
5832
+ }
5191
5833
  }
5192
5834
  if (stream.interruptions.length > 0) {
5193
5835
  await reconcileConversationTruth();
@@ -6251,7 +6893,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6251
6893
  // truth, recover this SAME accepted turn, then let the workflow re-claim
6252
6894
  // it after a pacing delay. This is independent of goal state and never
6253
6895
  // relies on a synthetic continuation prompt.
6254
- const failure = agentRunFailurePayload(error);
6896
+ const failure = agentRunFailurePayload(error, { isCodexTurn });
6255
6897
  if (isSessionEventPersistenceError(error)) {
6256
6898
  // Never pass the original Drizzle/postgres-js error to telemetry: its
6257
6899
  // nested cause may contain raw SQL and bound parameters. The typed DB
@@ -6355,10 +6997,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6355
6997
  const finalizerSignal = turnFinalizerCancellationSignal(cancellationSignal, activityStatus);
6356
6998
  try {
6357
6999
  const toolCancellationFence = toolCancellationFenceRef.current;
6358
- // Toolspace and run-credential renewals are attempt-owned sandbox
6359
- // writers. Capture and close them before crossing the same authoritative
6360
- // boundary as tool calls; none of these drains may race the already-
6361
- // aborted Temporal signal.
7000
+ // Every renewal controller is an attempt-owned sandbox writer. Capture
7001
+ // and close all of them before the tool/run-writer drain and before the
7002
+ // quiescence receipt; none may start another admitted write afterward.
7003
+ gitCredentialRenewalClosed = true;
7004
+ const gitRenewalsToStop = gitCredentialRenewals;
7005
+ gitCredentialRenewals = [];
6362
7006
  toolspaceTokenRenewalClosed = true;
6363
7007
  const toolspaceRenewalToStop =
6364
7008
  toolspaceTokenRenewal as ToolspaceTokenRenewalController | null;
@@ -6366,9 +7010,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6366
7010
  runCredentialRenewalClosed = true;
6367
7011
  const runRenewalToStop = runCredentialRenewal as RunCredentialRenewalController | null;
6368
7012
  runCredentialRenewal = null;
7013
+
7014
+ // Attempt-qualified credential deletion is also a real workspace write.
7015
+ // Perform it under the same admission fence before publishing physical
7016
+ // quiescence; a failure deliberately keeps the receipt closed.
7017
+ const credentialSessionToClear = runCredentialSession;
7018
+ runCredentialSession = null;
7019
+ if (credentialSessionToClear) {
7020
+ await runWorkspaceMutationForSandbox(
7021
+ requireResolvedSandboxForMutation(
7022
+ "Run credential cleanup has no exact sandbox lease target",
7023
+ ),
7024
+ "runCredentialAttemptClear",
7025
+ async () =>
7026
+ await clearRunCredentialsForAttempt(credentialSessionToClear, {
7027
+ sessionId: input.sessionId,
7028
+ attemptId: input.attemptId,
7029
+ executionGeneration,
7030
+ }),
7031
+ );
7032
+ }
6369
7033
  await drainAttemptOwnedSandboxWriters({
6370
7034
  toolCancellationFence: acknowledgeQuiescence ? toolCancellationFence : null,
6371
7035
  cancellationReason: cancellationSignal?.reason,
7036
+ gitCredentialRenewals: gitRenewalsToStop,
6372
7037
  toolspaceTokenRenewal: toolspaceRenewalToStop,
6373
7038
  runCredentialRenewal: runRenewalToStop,
6374
7039
  });
@@ -6448,43 +7113,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6448
7113
  });
6449
7114
  }
6450
7115
  }
6451
- // Deleting this attempt's credential generation is post-receipt
6452
- // housekeeping: pointer mutation is serialized with successor
6453
- // activation and the attempt-qualified check cannot erase a successor.
6454
- // It may therefore detach on cancellation without weakening the writer
6455
- // drain that gates the receipt above.
6456
- const credentialSessionToClear = runCredentialSession;
6457
- runCredentialSession = null;
6458
- if (credentialSessionToClear) {
6459
- const cleanup = clearRunCredentialsForAttempt(credentialSessionToClear, {
6460
- sessionId: input.sessionId,
6461
- attemptId: input.attemptId,
6462
- executionGeneration,
6463
- }).catch((error: unknown) => {
6464
- try {
6465
- observability.incrementCounter({
6466
- name: "opengeni_run_credential_cleanup_total",
6467
- help: "Attempt-owned run credential cleanup outcomes.",
6468
- labels: { outcome: "error" },
6469
- });
6470
- observability.warn("Attempt-owned run credential cleanup failed", {
6471
- sessionId: input.sessionId,
6472
- turnId,
6473
- attemptId: input.attemptId,
6474
- errorClass: error instanceof Error ? error.name : "UnknownError",
6475
- });
6476
- } catch {
6477
- // Cleanup observability must not own finalizer liveness.
6478
- }
6479
- });
6480
- await waitForTurnFinalizerStep(cleanup, finalizerSignal);
6481
- }
6482
- gitCredentialRenewalClosed = true;
6483
- const renewalsToStop = gitCredentialRenewals;
6484
- gitCredentialRenewals = [];
6485
- for (const renewal of renewalsToStop) {
6486
- await waitForTurnFinalizerStep(renewal.stop(), finalizerSignal);
6487
- }
6488
7116
  // Drain the buffered Connected Machine op events (infra failures + healed
6489
7117
  // recoveries) to durable session events — awaited, best-effort, never blocking
6490
7118
  // the turn. Sync observer → buffer → single awaited append here (no unawaited
@@ -6878,11 +7506,17 @@ export function isTransientProviderError(error: unknown): boolean {
6878
7506
  );
6879
7507
  }
6880
7508
 
6881
- export function agentRunFailurePayload(error: unknown): {
7509
+ export function agentRunFailurePayload(
7510
+ error: unknown,
7511
+ options: { isCodexTurn?: boolean } = {},
7512
+ ): {
6882
7513
  error: string;
6883
7514
  code?: string;
6884
7515
  retryable?: boolean;
6885
7516
  detail?: string;
7517
+ timeoutClass?: string;
7518
+ responseObserved?: boolean;
7519
+ requestId?: string;
6886
7520
  correlationId?: string;
6887
7521
  stage?: string;
6888
7522
  sqlState?: string | null;
@@ -6948,6 +7582,22 @@ export function agentRunFailurePayload(error: unknown): {
6948
7582
  if (usageLimit) {
6949
7583
  return codexUsageLimitFailurePayload(usageLimit, message);
6950
7584
  }
7585
+ const codexTimeout = classifyCodexResponseTimeoutError(error, {
7586
+ allowLegacyRequestTimeout: options.isCodexTurn === true,
7587
+ });
7588
+ if (codexTimeout) {
7589
+ return {
7590
+ error: codexTimeout.responseObserved
7591
+ ? "The Codex response timed out after streaming began. Observed output was checkpointed; automatic replay is disabled because the upstream operation may still be active."
7592
+ : "The Codex response timed out before any response was observed. Upstream acceptance is unknown, so automatic replay is disabled.",
7593
+ code: "codex_response_timeout",
7594
+ retryable: false,
7595
+ timeoutClass: codexTimeout.timeoutClass,
7596
+ responseObserved: codexTimeout.responseObserved,
7597
+ ...(codexTimeout.requestId ? { requestId: codexTimeout.requestId } : {}),
7598
+ ...(codexTimeout.message ? { detail: codexTimeout.message } : {}),
7599
+ };
7600
+ }
6951
7601
  const mcpTimeout = classifyMcpTransportTimeoutError(error);
6952
7602
  if (mcpTimeout) {
6953
7603
  return {
@@ -7227,21 +7877,21 @@ class BudgetExhaustedError extends Error {
7227
7877
  }
7228
7878
  }
7229
7879
 
7230
- // Exported for unit testing the codex-billed bypass (codex-billing.test.ts); not part
7231
- // of the activity surface. Takes BOTH `isCodexTurn` (codex-plan turns bypass the credit
7232
- // and token gates) and the optional §7.5 P3 host `entitlements` port (when bound, its
7233
- // `admitRun` REPLACES the local credit read for a non-codex turn; unset → local ledger).
7880
+ // Exported for unit testing the external-billing bypass (codex-billing.test.ts); not
7881
+ // part of the activity surface. Takes the accepted policy's billing attribution and
7882
+ // the optional §7.5 P3 host `entitlements` port (when bound, its `admitRun` REPLACES
7883
+ // the local credit read for an OpenGeni-metered turn; unset → local ledger).
7234
7884
  export async function ensureRunAllowed(
7235
7885
  settings: Settings,
7236
7886
  db: ActivityServices["db"],
7237
7887
  accountId: string,
7238
7888
  workspaceId: string,
7239
- isCodexTurn: boolean,
7889
+ isExternallyBilledTurn: boolean,
7240
7890
  entitlements?: ActivityServices["entitlements"],
7241
7891
  ): Promise<void> {
7242
- // Codex-billed turns are paid by the user's ChatGPT/Codex plan: skip the
7243
- // credit-balance gate and the monthly token cap. The agent-run COUNT cap below
7244
- // is a volume/fairness quota (not a credit/cost gate) and is intentionally kept.
7892
+ // Externally billed turns are paid outside OpenGeni: skip the credit-balance
7893
+ // gate and monthly token cap. The agent-run COUNT cap below is a
7894
+ // volume/fairness quota (not a credit/cost gate) and is intentionally kept.
7245
7895
  //
7246
7896
  // §7.5 P3 — host-entitlements DELEGATION (the worker half of the same seam the
7247
7897
  // API edge exposes). For a non-codex turn, when the host binds `entitlements`, its
@@ -7255,7 +7905,7 @@ export async function ensureRunAllowed(
7255
7905
  // idempotency-keyed writer at recordModelUsageAndDebitCredits), so a PULL host meter
7256
7906
  // is consulted without ever double-charging.
7257
7907
  if (
7258
- !isCodexTurn &&
7908
+ !isExternallyBilledTurn &&
7259
7909
  entitlements &&
7260
7910
  (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")
7261
7911
  ) {
@@ -7269,7 +7919,7 @@ export async function ensureRunAllowed(
7269
7919
  throw new Error(decision.reason || "insufficient OpenGeni credits");
7270
7920
  }
7271
7921
  } else if (
7272
- !isCodexTurn &&
7922
+ !isExternallyBilledTurn &&
7273
7923
  (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")
7274
7924
  ) {
7275
7925
  const balance = await getBillingBalance(db, accountId);
@@ -7294,7 +7944,7 @@ export async function ensureRunAllowed(
7294
7944
  );
7295
7945
  }
7296
7946
  }
7297
- if (!isCodexTurn && limits.maxMonthlyTokensPerWorkspace) {
7947
+ if (!isExternallyBilledTurn && limits.maxMonthlyTokensPerWorkspace) {
7298
7948
  const used = await sumUsageQuantity(db, {
7299
7949
  workspaceId,
7300
7950
  eventType: "model.tokens",
@@ -7307,7 +7957,7 @@ export async function ensureRunAllowed(
7307
7957
  }
7308
7958
  }
7309
7959
 
7310
- // Exported for unit testing the codex-billed bypass; not part of the activity surface.
7960
+ // Exported for unit testing the external-billing bypass; not part of the activity surface.
7311
7961
  export async function recordModelUsageAndDebitCredits(
7312
7962
  settings: Settings,
7313
7963
  db: ActivityServices["db"],
@@ -7318,8 +7968,9 @@ export async function recordModelUsageAndDebitCredits(
7318
7968
  turnId: string;
7319
7969
  turnAttemptId: string;
7320
7970
  model: string;
7321
- isCodexTurn: boolean;
7322
- usage?: ModelUsageInput | null;
7971
+ externallyBilled: boolean;
7972
+ usage?: ModelUsageInput | ModelCallUsageInput | null;
7973
+ normalizedUsage?: ModelCallUsageNormalization;
7323
7974
  sourceKey: string;
7324
7975
  observability?: ActivityServices["observability"];
7325
7976
  },
@@ -7327,10 +7978,12 @@ export async function recordModelUsageAndDebitCredits(
7327
7978
  if (!input.usage) {
7328
7979
  return;
7329
7980
  }
7330
- const inputTokens = positiveInt(input.usage.inputTokens);
7331
- const outputTokens = positiveInt(input.usage.outputTokens);
7332
- const totalTokens = positiveInt(input.usage.totalTokens) || inputTokens + outputTokens;
7333
- // A codex-subscription turn is paid by the user's ChatGPT/Codex plan, so it
7981
+ const normalizedUsage = input.normalizedUsage ?? normalizeModelCallUsage(input.usage);
7982
+ const sanitizedUsage = sanitizedModelUsageInput(normalizedUsage);
7983
+ const inputTokens = sanitizedUsage.inputTokens ?? 0;
7984
+ const outputTokens = sanitizedUsage.outputTokens ?? 0;
7985
+ const totalTokens = sanitizedUsage.totalTokens ?? 0;
7986
+ // An externally billed turn is paid outside OpenGeni, so it
7334
7987
  // consumes ZERO OpenGeni credits and must never feed an OpenGeni cap. A
7335
7988
  // codex/<slug> model has no entry in configuredModelPricing, so the normal path
7336
7989
  // below would throw "Missing model pricing". We:
@@ -7339,7 +7992,7 @@ export async function recordModelUsageAndDebitCredits(
7339
7992
  // any row would count against maxMonthlyTokensPerWorkspace);
7340
7993
  // - record a `model.cost = 0` audit marker (harmless to the monthly cost cap);
7341
7994
  // - never look up pricing and never debit credits.
7342
- if (input.isCodexTurn) {
7995
+ if (input.externallyBilled) {
7343
7996
  await recordUsageEvent(db, {
7344
7997
  accountId: input.accountId,
7345
7998
  workspaceId: input.workspaceId,
@@ -7377,7 +8030,7 @@ export async function recordModelUsageAndDebitCredits(
7377
8030
  if (!configuredModelPricing(settings)[input.model]) {
7378
8031
  throw new Error(`Missing model pricing for ${input.model}`);
7379
8032
  }
7380
- const costMicros = calculateModelUsageCostMicros(settings, input.model, input.usage);
8033
+ const costMicros = calculateModelUsageCostMicros(settings, input.model, sanitizedUsage);
7381
8034
  await recordUsageEvent(db, {
7382
8035
  accountId: input.accountId,
7383
8036
  workspaceId: input.workspaceId,
@@ -7411,18 +8064,26 @@ export async function recordModelUsageAndDebitCredits(
7411
8064
  // Additive: the prompt-cache slice of this call's input tokens, so the
7412
8065
  // per-call debit record carries cache efficiency alongside the token
7413
8066
  // counts. 0 when the provider did not report cached tokens.
7414
- cachedTokens: positiveInt(
7415
- modelCallUsageTelemetry(input.usage as Parameters<typeof modelCallUsageTelemetry>[0])
7416
- .cachedTokens,
7417
- ),
8067
+ cachedTokens: normalizedUsage.telemetry.cachedTokens ?? 0,
7418
8068
  },
7419
8069
  });
7420
8070
  recordCreditMicros(input.observability, "usage", result.debitedMicros);
7421
8071
  }
7422
8072
  }
7423
8073
 
7424
- function positiveInt(value: unknown): number {
7425
- return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
8074
+ function sanitizedModelUsageInput(normalized: ModelCallUsageNormalization): ModelUsageInput {
8075
+ return {
8076
+ ...(normalized.telemetry.inputTokens !== null
8077
+ ? { inputTokens: normalized.telemetry.inputTokens }
8078
+ : {}),
8079
+ ...(normalized.telemetry.outputTokens !== null
8080
+ ? { outputTokens: normalized.telemetry.outputTokens }
8081
+ : {}),
8082
+ ...(normalized.totalTokens !== null ? { totalTokens: normalized.totalTokens } : {}),
8083
+ ...(normalized.telemetry.cachedTokens !== null
8084
+ ? { inputTokensDetails: { cached_tokens: normalized.telemetry.cachedTokens } }
8085
+ : {}),
8086
+ };
7426
8087
  }
7427
8088
 
7428
8089
  function startOfUtcMonth(): Date {