@opengeni/worker-bundle 0.8.3 → 0.10.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1261 -434
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +3 -1
- package/package.json +11 -11
- package/src/activities/agent-turn.ts +639 -293
- package/src/activities/capabilities.ts +3 -1
- package/src/activities/environment.ts +142 -11
- package/src/activities/rig-verification.ts +897 -244
- package/src/activities/sandbox-lease.ts +32 -1
- package/src/observability-metrics.ts +83 -15
- package/src/workflows/rig-verification.ts +3 -1
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
getLatestRunState,
|
|
17
17
|
getHumanInputResumeForEvent,
|
|
18
18
|
getSessionHumanInputRequest,
|
|
19
|
-
|
|
19
|
+
installOrReadTurnExecutionPolicyForAttempt,
|
|
20
20
|
workspaceCodexSubscriptionActive,
|
|
21
21
|
acquireCodexCredentialLease,
|
|
22
22
|
armCodexCapacityWait,
|
|
@@ -76,8 +76,8 @@ import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@
|
|
|
76
76
|
import {
|
|
77
77
|
sandboxStateEntryFromRunState,
|
|
78
78
|
maxTurnsExceededRunState,
|
|
79
|
-
modelCallUsageTelemetry,
|
|
80
79
|
modelResponseUsageFromSdkEvent,
|
|
80
|
+
normalizeModelCallUsage,
|
|
81
81
|
normalizeSdkEvent,
|
|
82
82
|
sanitizeHistoryItemsForModel,
|
|
83
83
|
isEphemeralInternalContext,
|
|
@@ -107,6 +107,8 @@ import {
|
|
|
107
107
|
type OpenGeniRuntime,
|
|
108
108
|
type ComputerToolMode,
|
|
109
109
|
type ModelResponseUsage,
|
|
110
|
+
type ModelCallUsageInput,
|
|
111
|
+
type ModelCallUsageNormalization,
|
|
110
112
|
type BuildAgentOptions,
|
|
111
113
|
type TurnToolCancellationFence,
|
|
112
114
|
type BackendUnresolvableCode,
|
|
@@ -120,12 +122,13 @@ import {
|
|
|
120
122
|
} from "@opengeni/runtime";
|
|
121
123
|
import { connectionTokenResolverForTurn } from "./mcp-credentials";
|
|
122
124
|
import {
|
|
123
|
-
|
|
125
|
+
assertTurnExecutionPolicyMatchesConfigV1,
|
|
124
126
|
calculateModelUsageCostMicros,
|
|
125
127
|
configuredModelPricing,
|
|
126
128
|
configuredStaticUsageLimits,
|
|
127
129
|
sandboxWarmRateMicrosPerSecond,
|
|
128
130
|
settingsWithResolvedModelContext,
|
|
131
|
+
resolveTurnExecutionPolicyV1,
|
|
129
132
|
type ModelUsageInput,
|
|
130
133
|
type ModelProviderApi,
|
|
131
134
|
type RegistryProviderKind,
|
|
@@ -163,16 +166,19 @@ import {
|
|
|
163
166
|
CODEX_CLIENT_VERSION,
|
|
164
167
|
CODEX_FALLBACK_MODEL_SLUGS,
|
|
165
168
|
CodexReloginRequired,
|
|
169
|
+
classifyCodexResponseTimeoutError,
|
|
166
170
|
classifyCodexUsageLimitError,
|
|
167
171
|
codexRequestStorage,
|
|
168
172
|
isCodexTransportError,
|
|
169
173
|
type CodexRequestContext,
|
|
170
174
|
type CodexUsageHeaderSnapshot,
|
|
171
175
|
} from "@opengeni/codex";
|
|
172
|
-
import { mergeResourceRefs
|
|
176
|
+
import { mergeResourceRefs } from "./common";
|
|
177
|
+
import { enabledCapabilityMcpToolRefs, resolveSessionToolPolicy } from "@opengeni/core";
|
|
173
178
|
import { maybeCompactContext } from "./context-compaction";
|
|
174
179
|
import { TurnAttemptFencedError } from "./turn-attempt-fenced";
|
|
175
180
|
import {
|
|
181
|
+
assertGitCredentialRenewalTransportUnchanged,
|
|
176
182
|
gitCredentialAuthorityForTurn,
|
|
177
183
|
gitHubTokenMintSelections,
|
|
178
184
|
loadWorkspaceEnvironmentForRunWithCredentials,
|
|
@@ -274,10 +280,13 @@ import {
|
|
|
274
280
|
import {
|
|
275
281
|
CAPABILITY_DESCRIPTORS,
|
|
276
282
|
evaluateWorkspaceModelPolicy,
|
|
283
|
+
readTurnExecutionPolicyV1,
|
|
277
284
|
type ResourceRef,
|
|
278
285
|
type SessionEvent,
|
|
279
286
|
type SessionEventType,
|
|
280
287
|
type SessionStatus,
|
|
288
|
+
type SessionTurn,
|
|
289
|
+
type TurnExecutionPolicyV1,
|
|
281
290
|
} from "@opengeni/contracts";
|
|
282
291
|
import { createHash, randomUUID } from "node:crypto";
|
|
283
292
|
|
|
@@ -287,6 +296,32 @@ import { createHash, randomUUID } from "node:crypto";
|
|
|
287
296
|
// budget against the same window.
|
|
288
297
|
export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
|
|
289
298
|
|
|
299
|
+
export function turnExecutionPolicyBillingIdentity(policy: TurnExecutionPolicyV1): {
|
|
300
|
+
externallyBilled: boolean;
|
|
301
|
+
codexSubscription: boolean;
|
|
302
|
+
} {
|
|
303
|
+
return {
|
|
304
|
+
externallyBilled: policy.billing.metering === "external",
|
|
305
|
+
codexSubscription:
|
|
306
|
+
policy.providerId === "codex-subscription" &&
|
|
307
|
+
policy.credentialSource.kind === "connected_subscription" &&
|
|
308
|
+
policy.credentialSource.provider === "codex",
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
export function legacyTurnExecutionPolicyInput(
|
|
313
|
+
turn: Pick<SessionTurn, "source" | "model" | "reasoningEffort">,
|
|
314
|
+
): Parameters<typeof resolveTurnExecutionPolicyV1>[1] {
|
|
315
|
+
const explicit = turn.source === "user" || turn.source === "api";
|
|
316
|
+
return {
|
|
317
|
+
modelId: turn.model,
|
|
318
|
+
requestedModelId: explicit ? turn.model : null,
|
|
319
|
+
modelSource: explicit ? "explicit" : "continuation",
|
|
320
|
+
reasoningEffort: turn.reasoningEffort,
|
|
321
|
+
reasoningSource: explicit ? "explicit" : "continuation",
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
|
|
290
325
|
/** A retryable provider fault recovers the accepted turn itself. Goal state is
|
|
291
326
|
* irrelevant: autonomous continuation and infrastructure recovery are separate
|
|
292
327
|
* concerns. */
|
|
@@ -733,16 +768,8 @@ export function providerContextTokens(
|
|
|
733
768
|
| null
|
|
734
769
|
| undefined,
|
|
735
770
|
): number | null {
|
|
736
|
-
const total = usage
|
|
737
|
-
|
|
738
|
-
return total;
|
|
739
|
-
}
|
|
740
|
-
const input = usage?.inputTokens;
|
|
741
|
-
if (typeof input !== "number" || !Number.isFinite(input) || input <= 0) {
|
|
742
|
-
return null;
|
|
743
|
-
}
|
|
744
|
-
const output = usage?.outputTokens;
|
|
745
|
-
return input + (typeof output === "number" && Number.isFinite(output) && output > 0 ? output : 0);
|
|
771
|
+
const total = normalizeModelCallUsage(usage).totalTokens;
|
|
772
|
+
return total !== null && total > 0 ? total : null;
|
|
746
773
|
}
|
|
747
774
|
|
|
748
775
|
/**
|
|
@@ -773,6 +800,260 @@ type TurnEventPublisher = (
|
|
|
773
800
|
immediate?: boolean,
|
|
774
801
|
) => Promise<{ events: SessionEvent[]; accepted: boolean }>;
|
|
775
802
|
|
|
803
|
+
export type ModelResponseUsageEventState = {
|
|
804
|
+
responseUsageCount: number;
|
|
805
|
+
providerContextRevision: number;
|
|
806
|
+
lastProviderContextTokensObserved: number | null;
|
|
807
|
+
claimedSourceKeys: Set<string>;
|
|
808
|
+
};
|
|
809
|
+
|
|
810
|
+
export type CompactionModelUsageEventState = {
|
|
811
|
+
usageCount: number;
|
|
812
|
+
claimedSourceKeys: Set<string>;
|
|
813
|
+
};
|
|
814
|
+
|
|
815
|
+
export function createModelResponseUsageEventState(
|
|
816
|
+
claimedSourceKeys: Set<string> = new Set<string>(),
|
|
817
|
+
): ModelResponseUsageEventState {
|
|
818
|
+
return {
|
|
819
|
+
responseUsageCount: 0,
|
|
820
|
+
providerContextRevision: 0,
|
|
821
|
+
lastProviderContextTokensObserved: null,
|
|
822
|
+
claimedSourceKeys,
|
|
823
|
+
};
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
export function createCompactionModelUsageEventState(
|
|
827
|
+
claimedSourceKeys: Set<string> = new Set<string>(),
|
|
828
|
+
): CompactionModelUsageEventState {
|
|
829
|
+
return { usageCount: 0, claimedSourceKeys };
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
export function modelResponseUsageContextSignal(
|
|
833
|
+
state: ModelResponseUsageEventState,
|
|
834
|
+
): { revision: number; totalTokens: number } | null {
|
|
835
|
+
return state.lastProviderContextTokensObserved === null
|
|
836
|
+
? null
|
|
837
|
+
: {
|
|
838
|
+
revision: state.providerContextRevision,
|
|
839
|
+
totalTokens: state.lastProviderContextTokensObserved,
|
|
840
|
+
};
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
/**
|
|
844
|
+
* Process one SDK stream event through the exact production usage path.
|
|
845
|
+
*
|
|
846
|
+
* The pinned Responses SDK mirrors one provider terminal response as both a
|
|
847
|
+
* normalized `response_done` and a raw `model/response.completed` event. Claim
|
|
848
|
+
* the stable response/source key before lease renewal or any usage side effect,
|
|
849
|
+
* and advance the positional ordinal only for a newly claimed response. The
|
|
850
|
+
* durable `agent.model.usage` source-key fence remains the cross-restart
|
|
851
|
+
* authority: a replay may retry the idempotent billing write, but it cannot
|
|
852
|
+
* advance process-local metrics, provider context, or attempt-owned signals.
|
|
853
|
+
*/
|
|
854
|
+
export async function processModelResponseUsageEvent(input: {
|
|
855
|
+
event: Parameters<typeof modelResponseUsageFromSdkEvent>[0];
|
|
856
|
+
state: ModelResponseUsageEventState;
|
|
857
|
+
dispatchId: string | null;
|
|
858
|
+
settings: Settings;
|
|
859
|
+
db: ActivityServices["db"];
|
|
860
|
+
observability: ActivityServices["observability"];
|
|
861
|
+
publish: TurnEventPublisher | null;
|
|
862
|
+
accountId: string;
|
|
863
|
+
workspaceId: string;
|
|
864
|
+
sessionId: string;
|
|
865
|
+
turnId: string;
|
|
866
|
+
turnAttemptId: string;
|
|
867
|
+
provider: string;
|
|
868
|
+
providerApi: ModelProviderApi;
|
|
869
|
+
model: string;
|
|
870
|
+
metricProvider: string;
|
|
871
|
+
externallyBilled: boolean;
|
|
872
|
+
servingCredentialId: string | null;
|
|
873
|
+
priorSessionCredentialId: string | null;
|
|
874
|
+
emittedSourceKeys: Set<string>;
|
|
875
|
+
renewLease: () => Promise<void>;
|
|
876
|
+
leaseLost: () => boolean;
|
|
877
|
+
leaseLostMessage: string;
|
|
878
|
+
setLastInputTokens: (tokens: number) => Promise<void>;
|
|
879
|
+
}): Promise<
|
|
880
|
+
| { status: "not_usage" }
|
|
881
|
+
| { status: "duplicate"; sourceKey: string }
|
|
882
|
+
| { status: "processed"; sourceKey: string; authoritative: boolean }
|
|
883
|
+
> {
|
|
884
|
+
const responseUsage = modelResponseUsageFromSdkEvent(input.event);
|
|
885
|
+
if (!responseUsage) {
|
|
886
|
+
return { status: "not_usage" };
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
const responseOrdinal = input.state.responseUsageCount + 1;
|
|
890
|
+
const sourceKey = modelUsageSourceKey({
|
|
891
|
+
responseId: responseUsage.responseId,
|
|
892
|
+
dispatchId: input.dispatchId,
|
|
893
|
+
positionalKey: `response-${responseOrdinal}`,
|
|
894
|
+
});
|
|
895
|
+
if (input.state.claimedSourceKeys.has(sourceKey)) {
|
|
896
|
+
return { status: "duplicate", sourceKey };
|
|
897
|
+
}
|
|
898
|
+
input.state.claimedSourceKeys.add(sourceKey);
|
|
899
|
+
input.state.responseUsageCount = responseOrdinal;
|
|
900
|
+
|
|
901
|
+
const normalizedUsage = normalizeModelCallUsage(responseUsage.usage);
|
|
902
|
+
const accountContext = modelCallAccountContext({
|
|
903
|
+
servingCredentialId: input.servingCredentialId,
|
|
904
|
+
priorSessionCredentialId: input.priorSessionCredentialId,
|
|
905
|
+
isFirstCallOfTurn: responseOrdinal === 1,
|
|
906
|
+
});
|
|
907
|
+
let authoritative = false;
|
|
908
|
+
await recordCompletedModelCallBeforeOwnershipFences({
|
|
909
|
+
renewLease: input.renewLease,
|
|
910
|
+
leaseLost: input.leaseLost,
|
|
911
|
+
leaseLostMessage: input.leaseLostMessage,
|
|
912
|
+
recordUsage: async () => {
|
|
913
|
+
await recordModelUsageAndDebitCredits(input.settings, input.db, {
|
|
914
|
+
accountId: input.accountId,
|
|
915
|
+
workspaceId: input.workspaceId,
|
|
916
|
+
sessionId: input.sessionId,
|
|
917
|
+
turnId: input.turnId,
|
|
918
|
+
turnAttemptId: input.turnAttemptId,
|
|
919
|
+
model: input.model,
|
|
920
|
+
externallyBilled: input.externallyBilled,
|
|
921
|
+
usage: responseUsage.usage,
|
|
922
|
+
normalizedUsage,
|
|
923
|
+
sourceKey,
|
|
924
|
+
observability: input.observability,
|
|
925
|
+
});
|
|
926
|
+
authoritative = await emitModelCallUsage({
|
|
927
|
+
observability: input.observability,
|
|
928
|
+
publish: input.publish,
|
|
929
|
+
accountId: input.accountId,
|
|
930
|
+
workspaceId: input.workspaceId,
|
|
931
|
+
sessionId: input.sessionId,
|
|
932
|
+
turnId: input.turnId,
|
|
933
|
+
provider: input.provider,
|
|
934
|
+
providerApi: input.providerApi,
|
|
935
|
+
model: input.model,
|
|
936
|
+
sourceKey,
|
|
937
|
+
usage: responseUsage,
|
|
938
|
+
normalizedUsage,
|
|
939
|
+
servingAccountHash: accountContext.servingAccountHash,
|
|
940
|
+
accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
|
|
941
|
+
emittedSourceKeys: input.emittedSourceKeys,
|
|
942
|
+
});
|
|
943
|
+
const observedInput = normalizedUsage.telemetry.inputTokens;
|
|
944
|
+
if (authoritative && observedInput !== null && observedInput > 0) {
|
|
945
|
+
recordModelInputTokens(input.observability, input.metricProvider, observedInput);
|
|
946
|
+
}
|
|
947
|
+
},
|
|
948
|
+
recordAttemptSignals: async () => {
|
|
949
|
+
if (!authoritative) return;
|
|
950
|
+
const observedTotal = normalizedUsage.totalTokens;
|
|
951
|
+
if (observedTotal !== null) {
|
|
952
|
+
input.state.lastProviderContextTokensObserved = observedTotal;
|
|
953
|
+
input.state.providerContextRevision += 1;
|
|
954
|
+
}
|
|
955
|
+
const observedInput = normalizedUsage.telemetry.inputTokens;
|
|
956
|
+
if (observedInput !== null && observedInput > 0) {
|
|
957
|
+
await input.setLastInputTokens(observedInput);
|
|
958
|
+
}
|
|
959
|
+
},
|
|
960
|
+
});
|
|
961
|
+
return { status: "processed", sourceKey, authoritative };
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
/**
|
|
965
|
+
* Apply the same source-key authority ordering to the compaction summarizer's
|
|
966
|
+
* usage callback. The summarizer can retry or mirror a terminal response just
|
|
967
|
+
* like the main stream, so claim before lease renewal, billing, durable usage,
|
|
968
|
+
* logging, or cache metrics. Durable source-key idempotency remains the
|
|
969
|
+
* cross-process authority after a worker restart.
|
|
970
|
+
*/
|
|
971
|
+
export async function processCompactionModelUsageEvent(input: {
|
|
972
|
+
usage: ModelResponseUsage;
|
|
973
|
+
state: CompactionModelUsageEventState;
|
|
974
|
+
dispatchId: string | null;
|
|
975
|
+
settings: Settings;
|
|
976
|
+
db: ActivityServices["db"];
|
|
977
|
+
observability: ActivityServices["observability"];
|
|
978
|
+
publish: TurnEventPublisher | null;
|
|
979
|
+
accountId: string;
|
|
980
|
+
workspaceId: string;
|
|
981
|
+
sessionId: string;
|
|
982
|
+
turnId: string;
|
|
983
|
+
turnAttemptId: string;
|
|
984
|
+
provider: string;
|
|
985
|
+
providerApi: ModelProviderApi;
|
|
986
|
+
model: string;
|
|
987
|
+
externallyBilled: boolean;
|
|
988
|
+
servingCredentialId: string | null;
|
|
989
|
+
priorSessionCredentialId: string | null;
|
|
990
|
+
emittedSourceKeys: Set<string>;
|
|
991
|
+
renewLease: () => Promise<void>;
|
|
992
|
+
leaseLost: () => boolean;
|
|
993
|
+
leaseLostMessage: string;
|
|
994
|
+
}): Promise<
|
|
995
|
+
| { status: "duplicate"; sourceKey: string }
|
|
996
|
+
| { status: "processed"; sourceKey: string; authoritative: boolean }
|
|
997
|
+
> {
|
|
998
|
+
const usageOrdinal = input.state.usageCount + 1;
|
|
999
|
+
const sourceKey = modelUsageSourceKey({
|
|
1000
|
+
responseId: input.usage.responseId,
|
|
1001
|
+
dispatchId: input.dispatchId,
|
|
1002
|
+
positionalKey: `compaction-${usageOrdinal}`,
|
|
1003
|
+
});
|
|
1004
|
+
if (input.state.claimedSourceKeys.has(sourceKey)) {
|
|
1005
|
+
return { status: "duplicate", sourceKey };
|
|
1006
|
+
}
|
|
1007
|
+
input.state.claimedSourceKeys.add(sourceKey);
|
|
1008
|
+
input.state.usageCount = usageOrdinal;
|
|
1009
|
+
|
|
1010
|
+
const accountContext = modelCallAccountContext({
|
|
1011
|
+
servingCredentialId: input.servingCredentialId,
|
|
1012
|
+
priorSessionCredentialId: input.priorSessionCredentialId,
|
|
1013
|
+
isFirstCallOfTurn: usageOrdinal === 1,
|
|
1014
|
+
});
|
|
1015
|
+
const normalizedUsage = normalizeModelCallUsage(input.usage.usage);
|
|
1016
|
+
let authoritative = false;
|
|
1017
|
+
await recordCompletedModelCallBeforeOwnershipFences({
|
|
1018
|
+
renewLease: input.renewLease,
|
|
1019
|
+
leaseLost: input.leaseLost,
|
|
1020
|
+
leaseLostMessage: input.leaseLostMessage,
|
|
1021
|
+
recordUsage: async () => {
|
|
1022
|
+
await recordModelUsageAndDebitCredits(input.settings, input.db, {
|
|
1023
|
+
accountId: input.accountId,
|
|
1024
|
+
workspaceId: input.workspaceId,
|
|
1025
|
+
sessionId: input.sessionId,
|
|
1026
|
+
turnId: input.turnId,
|
|
1027
|
+
turnAttemptId: input.turnAttemptId,
|
|
1028
|
+
model: input.model,
|
|
1029
|
+
externallyBilled: input.externallyBilled,
|
|
1030
|
+
usage: input.usage.usage,
|
|
1031
|
+
normalizedUsage,
|
|
1032
|
+
sourceKey,
|
|
1033
|
+
observability: input.observability,
|
|
1034
|
+
});
|
|
1035
|
+
authoritative = await emitModelCallUsage({
|
|
1036
|
+
observability: input.observability,
|
|
1037
|
+
publish: input.publish,
|
|
1038
|
+
accountId: input.accountId,
|
|
1039
|
+
workspaceId: input.workspaceId,
|
|
1040
|
+
sessionId: input.sessionId,
|
|
1041
|
+
turnId: input.turnId,
|
|
1042
|
+
provider: input.provider,
|
|
1043
|
+
providerApi: input.providerApi,
|
|
1044
|
+
model: input.model,
|
|
1045
|
+
sourceKey,
|
|
1046
|
+
usage: input.usage,
|
|
1047
|
+
normalizedUsage,
|
|
1048
|
+
servingAccountHash: accountContext.servingAccountHash,
|
|
1049
|
+
accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
|
|
1050
|
+
emittedSourceKeys: input.emittedSourceKeys,
|
|
1051
|
+
});
|
|
1052
|
+
},
|
|
1053
|
+
});
|
|
1054
|
+
return { status: "processed", sourceKey, authoritative };
|
|
1055
|
+
}
|
|
1056
|
+
|
|
776
1057
|
export async function emitModelCallUsage(input: {
|
|
777
1058
|
observability: ActivityServices["observability"];
|
|
778
1059
|
publish: TurnEventPublisher | null;
|
|
@@ -785,22 +1066,25 @@ export async function emitModelCallUsage(input: {
|
|
|
785
1066
|
model: string;
|
|
786
1067
|
sourceKey: string;
|
|
787
1068
|
usage: ModelResponseUsage | { usage?: unknown | null } | null;
|
|
1069
|
+
normalizedUsage?: ModelCallUsageNormalization;
|
|
788
1070
|
// Prompt-cache research dimensions (log-only; NEVER on a metric label or a
|
|
789
1071
|
// durable event). The opaque serving-account tag and whether it changed since
|
|
790
1072
|
// the session's previous call — the account-switch hypothesis for cache misses.
|
|
791
1073
|
servingAccountHash?: string;
|
|
792
1074
|
accountChangedFromPrevCall?: boolean;
|
|
793
1075
|
emittedSourceKeys?: Set<string>;
|
|
794
|
-
}): Promise<
|
|
1076
|
+
}): Promise<boolean> {
|
|
795
1077
|
const usage =
|
|
796
1078
|
input.usage && typeof input.usage === "object" && "usage" in input.usage
|
|
797
1079
|
? (input.usage as { usage?: unknown }).usage
|
|
798
1080
|
: null;
|
|
799
1081
|
if (!usage || typeof usage !== "object") {
|
|
800
|
-
return;
|
|
1082
|
+
return false;
|
|
801
1083
|
}
|
|
802
|
-
if (input.emittedSourceKeys?.has(input.sourceKey)) return;
|
|
803
|
-
const
|
|
1084
|
+
if (input.emittedSourceKeys?.has(input.sourceKey)) return false;
|
|
1085
|
+
const normalizedUsage =
|
|
1086
|
+
input.normalizedUsage ?? normalizeModelCallUsage(usage as ModelCallUsageInput);
|
|
1087
|
+
const telemetry = normalizedUsage.telemetry;
|
|
804
1088
|
const appended = await input.publish?.(
|
|
805
1089
|
[
|
|
806
1090
|
{
|
|
@@ -829,7 +1113,7 @@ export async function emitModelCallUsage(input: {
|
|
|
829
1113
|
typeof event.payload === "object" &&
|
|
830
1114
|
(event.payload as Record<string, unknown>).sourceKey === input.sourceKey,
|
|
831
1115
|
);
|
|
832
|
-
if (!authoritative) return;
|
|
1116
|
+
if (!authoritative) return false;
|
|
833
1117
|
try {
|
|
834
1118
|
input.observability.info("model call usage", {
|
|
835
1119
|
accountId: input.accountId,
|
|
@@ -843,6 +1127,7 @@ export async function emitModelCallUsage(input: {
|
|
|
843
1127
|
inputTokens: telemetry.inputTokens,
|
|
844
1128
|
outputTokens: telemetry.outputTokens,
|
|
845
1129
|
cachedTokens: telemetry.cachedTokens,
|
|
1130
|
+
cacheWriteTokens: telemetry.cacheWriteTokens,
|
|
846
1131
|
reasoningTokens: telemetry.reasoningTokens,
|
|
847
1132
|
...(input.servingAccountHash !== undefined
|
|
848
1133
|
? { servingAccountHash: input.servingAccountHash }
|
|
@@ -851,9 +1136,41 @@ export async function emitModelCallUsage(input: {
|
|
|
851
1136
|
? { accountChangedFromPrevCall: input.accountChangedFromPrevCall }
|
|
852
1137
|
: {}),
|
|
853
1138
|
});
|
|
1139
|
+
if (normalizedUsage.rejectedFields.length > 0) {
|
|
1140
|
+
input.observability.warn("model call usage fields rejected", {
|
|
1141
|
+
provider: input.provider,
|
|
1142
|
+
providerApi: input.providerApi,
|
|
1143
|
+
model: input.model,
|
|
1144
|
+
sourceKey: input.sourceKey,
|
|
1145
|
+
rejectedFields: normalizedUsage.rejectedFields.join(","),
|
|
1146
|
+
});
|
|
1147
|
+
}
|
|
854
1148
|
} catch {
|
|
855
1149
|
// Durable event + billing already committed; logging is best-effort only.
|
|
856
1150
|
}
|
|
1151
|
+
try {
|
|
1152
|
+
applyCodexCacheTelemetry(input.observability, input.provider, normalizedUsage);
|
|
1153
|
+
} catch {
|
|
1154
|
+
// Durable event + billing already committed; metrics are best-effort only.
|
|
1155
|
+
}
|
|
1156
|
+
return true;
|
|
1157
|
+
}
|
|
1158
|
+
|
|
1159
|
+
/**
|
|
1160
|
+
* Apply one authoritative, normalized model-call usage frame to the shared
|
|
1161
|
+
* prompt-cache metrics. The durable source-key fence in `emitModelCallUsage`
|
|
1162
|
+
* owns idempotency; this helper must never receive raw provider values.
|
|
1163
|
+
*/
|
|
1164
|
+
export function applyCodexCacheTelemetry(
|
|
1165
|
+
observability: ActivityServices["observability"],
|
|
1166
|
+
provider: string,
|
|
1167
|
+
normalizedUsage: ModelCallUsageNormalization,
|
|
1168
|
+
): void {
|
|
1169
|
+
recordModelCacheTokens(observability, provider, {
|
|
1170
|
+
cachedTokens: normalizedUsage.telemetry.cachedTokens,
|
|
1171
|
+
cacheWriteTokens: normalizedUsage.telemetry.cacheWriteTokens,
|
|
1172
|
+
promptTokens: normalizedUsage.telemetry.inputTokens,
|
|
1173
|
+
});
|
|
857
1174
|
}
|
|
858
1175
|
|
|
859
1176
|
export function historyRowsToAppend(
|
|
@@ -1456,12 +1773,24 @@ async function refreshCappedCodexUsageRows(
|
|
|
1456
1773
|
db: ActivityServices["db"],
|
|
1457
1774
|
settings: Settings,
|
|
1458
1775
|
workspaceId: string,
|
|
1459
|
-
accounts:
|
|
1776
|
+
accounts: Array<
|
|
1777
|
+
Pick<
|
|
1778
|
+
CodexAccountStatus,
|
|
1779
|
+
"id" | "status" | "primaryUsedPercent" | "secondaryUsedPercent" | "exhaustedUntil"
|
|
1780
|
+
>
|
|
1781
|
+
>,
|
|
1460
1782
|
capacitySignals: {
|
|
1461
1783
|
signalCodexCapacityWorkflow?: ActivityServices["signalCodexCapacityWorkflow"] | undefined;
|
|
1462
1784
|
wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
|
|
1463
1785
|
},
|
|
1464
|
-
): Promise<
|
|
1786
|
+
): Promise<
|
|
1787
|
+
Array<
|
|
1788
|
+
Pick<
|
|
1789
|
+
CodexAccountStatus,
|
|
1790
|
+
"id" | "status" | "primaryUsedPercent" | "secondaryUsedPercent" | "exhaustedUntil"
|
|
1791
|
+
>
|
|
1792
|
+
>
|
|
1793
|
+
> {
|
|
1465
1794
|
const stale = accounts.filter(
|
|
1466
1795
|
(a) =>
|
|
1467
1796
|
a.status === "active" &&
|
|
@@ -1560,6 +1889,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1560
1889
|
runtimeMetricsHooksForObservability(observability),
|
|
1561
1890
|
);
|
|
1562
1891
|
let isCodexTurn = false;
|
|
1892
|
+
let isExternallyBilledTurn = false;
|
|
1563
1893
|
let executionGeneration = 0;
|
|
1564
1894
|
// Still required by credential-loss/capacity settlements, whose own
|
|
1565
1895
|
// recovery transactions fence against worker-death redispatches.
|
|
@@ -2138,10 +2468,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2138
2468
|
input.workspaceId,
|
|
2139
2469
|
settings,
|
|
2140
2470
|
);
|
|
2141
|
-
// Read the active-credential flag
|
|
2142
|
-
//
|
|
2143
|
-
//
|
|
2144
|
-
// provider-injection and billing disagree about whether this is a codex turn.
|
|
2471
|
+
// Read the active-credential flag once for the runtime capability overlay.
|
|
2472
|
+
// Accepted billing/provider identity comes from the turn policy below,
|
|
2473
|
+
// never from this mutable health snapshot.
|
|
2145
2474
|
const codexSubscriptionActive = await workspaceCodexSubscriptionActive(
|
|
2146
2475
|
db,
|
|
2147
2476
|
mcpSettings,
|
|
@@ -2169,6 +2498,38 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2169
2498
|
}
|
|
2170
2499
|
const turn = claim.turn;
|
|
2171
2500
|
turnId = turn.id;
|
|
2501
|
+
executionGeneration = turn.executionGeneration;
|
|
2502
|
+
const claimedPolicy = readTurnExecutionPolicyV1(turn.metadata);
|
|
2503
|
+
const policyForAbsent =
|
|
2504
|
+
claimedPolicy.kind === "valid"
|
|
2505
|
+
? claimedPolicy.policy
|
|
2506
|
+
: resolveTurnExecutionPolicyV1(capabilitySettings, legacyTurnExecutionPolicyInput(turn));
|
|
2507
|
+
const installedPolicy = await installOrReadTurnExecutionPolicyForAttempt(db, {
|
|
2508
|
+
accountId: input.accountId,
|
|
2509
|
+
workspaceId: input.workspaceId,
|
|
2510
|
+
sessionId: input.sessionId,
|
|
2511
|
+
turnId,
|
|
2512
|
+
executionGeneration,
|
|
2513
|
+
attemptId: input.attemptId,
|
|
2514
|
+
policyForAbsent,
|
|
2515
|
+
});
|
|
2516
|
+
if (!installedPolicy.accepted) {
|
|
2517
|
+
throw new TurnAttemptFencedError(
|
|
2518
|
+
`turn execution policy was fenced: ${installedPolicy.reason}`,
|
|
2519
|
+
);
|
|
2520
|
+
}
|
|
2521
|
+
const verifiedExecutionPolicy = assertTurnExecutionPolicyMatchesConfigV1(
|
|
2522
|
+
capabilitySettings,
|
|
2523
|
+
installedPolicy.policy,
|
|
2524
|
+
{
|
|
2525
|
+
modelId: turn.model,
|
|
2526
|
+
reasoningEffort: turn.reasoningEffort,
|
|
2527
|
+
},
|
|
2528
|
+
);
|
|
2529
|
+
const turnExecutionPolicy = verifiedExecutionPolicy.policy;
|
|
2530
|
+
const billingIdentity = turnExecutionPolicyBillingIdentity(turnExecutionPolicy);
|
|
2531
|
+
isExternallyBilledTurn = billingIdentity.externallyBilled;
|
|
2532
|
+
isCodexTurn = billingIdentity.codexSubscription;
|
|
2172
2533
|
triggerEventId = turn.triggerEventId;
|
|
2173
2534
|
const trigger = await getSessionEvent(db, input.workspaceId, triggerEventId);
|
|
2174
2535
|
if (!trigger) {
|
|
@@ -2181,7 +2542,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2181
2542
|
trigger,
|
|
2182
2543
|
);
|
|
2183
2544
|
triggerType = trigger.type;
|
|
2184
|
-
executionGeneration = turn.executionGeneration;
|
|
2185
2545
|
const latestTurnState = await getLatestRunState(db, input.workspaceId, input.sessionId);
|
|
2186
2546
|
const continuationCodexCredentialId =
|
|
2187
2547
|
latestTurnState?.turnId === turnId ? latestTurnState.frozenCodexCredentialId : null;
|
|
@@ -2190,21 +2550,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2190
2550
|
0,
|
|
2191
2551
|
);
|
|
2192
2552
|
turnLifecycleMetricsFor(observability).start(turnId);
|
|
2193
|
-
//
|
|
2194
|
-
//
|
|
2195
|
-
// gate + the usage recorder so a turn paid by the user's ChatGPT/Codex plan
|
|
2196
|
-
// consumes ZERO OpenGeni credits and never feeds an OpenGeni cap. Resolved
|
|
2197
|
-
// here (before resolvedModel at the routing step) because the pre-turn gate
|
|
2198
|
-
// below needs it; mirrors the same active-credential read the codex provider
|
|
2199
|
-
// overlay uses, so billing and routing agree on what "codex" is.
|
|
2200
|
-
isCodexTurn = await isCodexBilledTurn({
|
|
2201
|
-
db,
|
|
2202
|
-
settings,
|
|
2203
|
-
workspaceId: input.workspaceId,
|
|
2204
|
-
model: turn.model,
|
|
2205
|
-
active: codexSubscriptionActive,
|
|
2206
|
-
});
|
|
2207
|
-
// §7.5 P3 — pass BOTH the codex predicate (codex-plan turns bypass the gate)
|
|
2553
|
+
// §7.5 P3 — pass the accepted billing attribution (externally funded turns
|
|
2554
|
+
// bypass OpenGeni credit/token gates)
|
|
2208
2555
|
// AND the optional host `entitlements` port (when bound, its admitRun replaces
|
|
2209
2556
|
// the local credit read). Unset port → today's local-ledger path.
|
|
2210
2557
|
await waitForTurnOperation(
|
|
@@ -2213,7 +2560,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2213
2560
|
db,
|
|
2214
2561
|
input.accountId,
|
|
2215
2562
|
input.workspaceId,
|
|
2216
|
-
|
|
2563
|
+
isExternallyBilledTurn,
|
|
2217
2564
|
entitlements,
|
|
2218
2565
|
),
|
|
2219
2566
|
cancellationSignal,
|
|
@@ -2259,6 +2606,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2259
2606
|
// one exactly like production.
|
|
2260
2607
|
codexLeaseHolderId = dispatchId;
|
|
2261
2608
|
const modelUsageDispatchId = activityContext?.info.activityId ?? dispatchId;
|
|
2609
|
+
const claimedModelUsageSourceKeys = new Set<string>();
|
|
2262
2610
|
const emittedModelUsageSourceKeys = new Set<string>();
|
|
2263
2611
|
publish = async (
|
|
2264
2612
|
events: Array<Omit<AppendEventInput, "producerId" | "producerSeq" | "turnId">>,
|
|
@@ -3016,6 +3364,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3016
3364
|
db,
|
|
3017
3365
|
input.workspaceId,
|
|
3018
3366
|
input.sessionId,
|
|
3367
|
+
input.attemptId,
|
|
3019
3368
|
baseRunSettings,
|
|
3020
3369
|
);
|
|
3021
3370
|
// Multi-provider per-turn routing → the provider gating (compaction mode,
|
|
@@ -3031,7 +3380,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3031
3380
|
// a chat-only Fireworks model. Resolving against the default-model settings
|
|
3032
3381
|
// keeps gating consistent with the router. Cost accounting covers registry
|
|
3033
3382
|
// models via configuredModelPricing.
|
|
3034
|
-
const resolvedModel = runtime.resolveTurnModel(
|
|
3383
|
+
const resolvedModel = runtime.resolveTurnModel(
|
|
3384
|
+
capabilitySettings,
|
|
3385
|
+
turnExecutionPolicy.productModelId,
|
|
3386
|
+
);
|
|
3035
3387
|
// Bind the provider/model catalog's context policy to every model-facing
|
|
3036
3388
|
// path for this turn. In particular, Codex subscription turns must not
|
|
3037
3389
|
// inherit the deployment's OpenAI/Azure mode or 1.05M context defaults:
|
|
@@ -3046,25 +3398,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3046
3398
|
// and the main run both come later in this scope), so a blocked
|
|
3047
3399
|
// provider/model can never be reached through ANY stamp path: explicit
|
|
3048
3400
|
// turn model, inherited session default, goal-continuation inheritance,
|
|
3049
|
-
// or the legacy null-resolution fallback
|
|
3050
|
-
//
|
|
3051
|
-
//
|
|
3052
|
-
// how bare-model turns silently spent real Azure money in a
|
|
3053
|
-
// codex-intended workspace). Fail-loud, never a silent remap.
|
|
3401
|
+
// or the legacy null-resolution fallback. The frozen execution policy is
|
|
3402
|
+
// the attribution source even if an injected test runtime returns no
|
|
3403
|
+
// concrete resolved model. Fail-loud, never a silent remap.
|
|
3054
3404
|
{
|
|
3055
3405
|
const workspaceModelPolicy = await getWorkspaceModelPolicy(db, input.workspaceId);
|
|
3056
3406
|
if (workspaceModelPolicy) {
|
|
3057
|
-
const effectiveProviderId = resolvedModel
|
|
3058
|
-
? resolvedModel.provider.id
|
|
3059
|
-
: builtinProviderId(capabilitySettings);
|
|
3060
3407
|
const verdict = evaluateWorkspaceModelPolicy(workspaceModelPolicy, {
|
|
3061
|
-
providerId:
|
|
3062
|
-
modelId:
|
|
3408
|
+
providerId: turnExecutionPolicy.providerId,
|
|
3409
|
+
modelId: turnExecutionPolicy.productModelId,
|
|
3063
3410
|
});
|
|
3064
3411
|
if (!verdict.allowed) {
|
|
3065
3412
|
throw new WorkspaceModelPolicyBlockedError(
|
|
3066
|
-
|
|
3067
|
-
|
|
3413
|
+
turnExecutionPolicy.productModelId,
|
|
3414
|
+
turnExecutionPolicy.providerId,
|
|
3068
3415
|
verdict.reason,
|
|
3069
3416
|
);
|
|
3070
3417
|
}
|
|
@@ -3078,6 +3425,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3078
3425
|
// Build it once and wrap BOTH the compaction summarizer (a separate model
|
|
3079
3426
|
// call on the same codex client) and the main run; otherwise the summarizer
|
|
3080
3427
|
// would hit the codex backend unauthenticated.
|
|
3428
|
+
let codexModelRequestSequence = 0;
|
|
3081
3429
|
const codexContext: CodexRequestContext | null =
|
|
3082
3430
|
resolvedModel?.provider.kind === "codex-subscription"
|
|
3083
3431
|
? ((): CodexRequestContext => {
|
|
@@ -3108,6 +3456,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3108
3456
|
onUsageHeaders: (snapshot) => {
|
|
3109
3457
|
latestCodexUsage = snapshot;
|
|
3110
3458
|
}, // latest wins; flushed once in finally
|
|
3459
|
+
nextRequestId: () => `${dispatchId}:${++codexModelRequestSequence}`,
|
|
3460
|
+
onModelRequestEvent: async (event) => {
|
|
3461
|
+
if (!publish || !turnId) {
|
|
3462
|
+
throw new Error("Codex model request started before the turn event producer");
|
|
3463
|
+
}
|
|
3464
|
+
await publish([
|
|
3465
|
+
{
|
|
3466
|
+
type: "agent.model.request",
|
|
3467
|
+
payload: {
|
|
3468
|
+
...event,
|
|
3469
|
+
provider: "codex-subscription",
|
|
3470
|
+
turnId,
|
|
3471
|
+
attemptId: input.attemptId,
|
|
3472
|
+
dispatchId,
|
|
3473
|
+
executionGeneration,
|
|
3474
|
+
},
|
|
3475
|
+
},
|
|
3476
|
+
]);
|
|
3477
|
+
},
|
|
3111
3478
|
};
|
|
3112
3479
|
})()
|
|
3113
3480
|
: null;
|
|
@@ -3116,53 +3483,33 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3116
3483
|
const promptCacheKey = acceptsPromptCacheKeyForTurn(resolvedModel)
|
|
3117
3484
|
? input.sessionId
|
|
3118
3485
|
: undefined;
|
|
3119
|
-
|
|
3486
|
+
const compactionUsageState = createCompactionModelUsageEventState(
|
|
3487
|
+
claimedModelUsageSourceKeys,
|
|
3488
|
+
);
|
|
3120
3489
|
const recordCompactionUsage = async (usage: ModelResponseUsage) => {
|
|
3121
|
-
await
|
|
3490
|
+
await processCompactionModelUsageEvent({
|
|
3491
|
+
usage,
|
|
3492
|
+
state: compactionUsageState,
|
|
3493
|
+
dispatchId: modelUsageDispatchId,
|
|
3494
|
+
settings,
|
|
3495
|
+
db,
|
|
3496
|
+
observability,
|
|
3497
|
+
publish,
|
|
3498
|
+
accountId: input.accountId,
|
|
3499
|
+
workspaceId: input.workspaceId,
|
|
3500
|
+
sessionId: input.sessionId,
|
|
3501
|
+
turnId: turn.id,
|
|
3502
|
+
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
3503
|
+
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
3504
|
+
model: resolvedModel?.configured.id ?? turn.model,
|
|
3505
|
+
externallyBilled: isExternallyBilledTurn,
|
|
3506
|
+
turnAttemptId: input.attemptId,
|
|
3507
|
+
servingCredentialId: effectiveCodexCredentialId,
|
|
3508
|
+
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
3509
|
+
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
3122
3510
|
renewLease: () => renewCodexLease("model_usage"),
|
|
3123
3511
|
leaseLost: () => codexLeaseLost,
|
|
3124
3512
|
leaseLostMessage: "Codex credential lease expired during context compaction",
|
|
3125
|
-
recordUsage: async () => {
|
|
3126
|
-
compactionUsageCount += 1;
|
|
3127
|
-
const sourceKey = modelUsageSourceKey({
|
|
3128
|
-
responseId: usage.responseId,
|
|
3129
|
-
dispatchId: modelUsageDispatchId,
|
|
3130
|
-
positionalKey: `compaction-${compactionUsageCount}`,
|
|
3131
|
-
});
|
|
3132
|
-
const responseAccountCtx = modelCallAccountContext({
|
|
3133
|
-
servingCredentialId: effectiveCodexCredentialId,
|
|
3134
|
-
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
3135
|
-
isFirstCallOfTurn: compactionUsageCount === 1,
|
|
3136
|
-
});
|
|
3137
|
-
await recordModelUsageAndDebitCredits(settings, db, {
|
|
3138
|
-
accountId: input.accountId,
|
|
3139
|
-
workspaceId: input.workspaceId,
|
|
3140
|
-
sessionId: input.sessionId,
|
|
3141
|
-
turnId: turn.id,
|
|
3142
|
-
turnAttemptId: input.attemptId,
|
|
3143
|
-
model: resolvedModel?.configured.id ?? turn.model,
|
|
3144
|
-
isCodexTurn,
|
|
3145
|
-
usage: usage.usage,
|
|
3146
|
-
sourceKey,
|
|
3147
|
-
observability,
|
|
3148
|
-
});
|
|
3149
|
-
await emitModelCallUsage({
|
|
3150
|
-
observability,
|
|
3151
|
-
publish,
|
|
3152
|
-
accountId: input.accountId,
|
|
3153
|
-
workspaceId: input.workspaceId,
|
|
3154
|
-
sessionId: input.sessionId,
|
|
3155
|
-
turnId: turn.id,
|
|
3156
|
-
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
3157
|
-
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
3158
|
-
model: resolvedModel?.configured.id ?? turn.model,
|
|
3159
|
-
sourceKey,
|
|
3160
|
-
usage,
|
|
3161
|
-
servingAccountHash: responseAccountCtx.servingAccountHash,
|
|
3162
|
-
accountChangedFromPrevCall: responseAccountCtx.accountChangedFromPrevCall,
|
|
3163
|
-
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
3164
|
-
});
|
|
3165
|
-
},
|
|
3166
3513
|
});
|
|
3167
3514
|
};
|
|
3168
3515
|
const compactionSummarizerFor = (systemInstructions?: string) =>
|
|
@@ -3172,7 +3519,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3172
3519
|
summarizeForCompaction(s, m, {
|
|
3173
3520
|
client: resolvedModel.client,
|
|
3174
3521
|
api: resolvedModel.provider.api,
|
|
3175
|
-
model:
|
|
3522
|
+
model: turnExecutionPolicy.upstreamModelId,
|
|
3176
3523
|
maxOutputTokens: SUMMARY_BUFFER_TOKENS,
|
|
3177
3524
|
onUsage: recordCompactionUsage,
|
|
3178
3525
|
...(systemInstructions ? { systemInstructions } : {}),
|
|
@@ -3181,6 +3528,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3181
3528
|
)
|
|
3182
3529
|
: (s: Settings, m: Array<Record<string, unknown>>) =>
|
|
3183
3530
|
summarizeForCompaction(s, m, {
|
|
3531
|
+
model: turnExecutionPolicy.upstreamModelId,
|
|
3184
3532
|
maxOutputTokens: SUMMARY_BUFFER_TOKENS,
|
|
3185
3533
|
onUsage: recordCompactionUsage,
|
|
3186
3534
|
...(systemInstructions ? { systemInstructions } : {}),
|
|
@@ -3321,9 +3669,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3321
3669
|
// connector scopes); no-op for every other turn. Its refreshing bearer is
|
|
3322
3670
|
// resolved at connect time from the codex ALS (see the withCodex-wrapped
|
|
3323
3671
|
// prepareTools call below).
|
|
3672
|
+
// Resolve the durable policy at the turn boundary. Workspace-default
|
|
3673
|
+
// sessions may discover newly enabled capability MCPs, while explicit,
|
|
3674
|
+
// inherited-fixed, and legacy sessions remain narrowed to their stored
|
|
3675
|
+
// materialized allow-list. `withCodexAppsTool` below keeps the existing
|
|
3676
|
+
// single Codex lazy router/connector overlay; it is not a second policy
|
|
3677
|
+
// or discovery path.
|
|
3678
|
+
const effectivePolicyTools = resolveSessionToolPolicy({
|
|
3679
|
+
...(session.toolPolicy ? { toolPolicy: session.toolPolicy } : {}),
|
|
3680
|
+
sessionTools: session.tools,
|
|
3681
|
+
turnTools: turn.tools,
|
|
3682
|
+
...(turn.toolsProvided !== undefined ? { turnToolsProvided: turn.toolsProvided } : {}),
|
|
3683
|
+
availableMcpServerIds: runSettings.mcpServers.map((server) => server.id),
|
|
3684
|
+
defaultMcpServerIds: enabledCapabilityMcpToolRefs(settings, mcpSettings).map(
|
|
3685
|
+
(tool) => tool.id,
|
|
3686
|
+
),
|
|
3687
|
+
}).toolRefs;
|
|
3324
3688
|
const turnTools = withCodexAppsTool(
|
|
3325
3689
|
runSettings,
|
|
3326
|
-
withFirstPartyTools(runSettings,
|
|
3690
|
+
withFirstPartyTools(runSettings, effectivePolicyTools),
|
|
3327
3691
|
);
|
|
3328
3692
|
// §7.6 connection-credential provider — load (and decrypt) the variable set via the host
|
|
3329
3693
|
// `sandboxSecrets` provider when bound; unset → today's local decrypt.
|
|
@@ -3553,6 +3917,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3553
3917
|
turn,
|
|
3554
3918
|
})
|
|
3555
3919
|
: undefined;
|
|
3920
|
+
const toolspaceAuthority = {
|
|
3921
|
+
sessionId: input.sessionId,
|
|
3922
|
+
};
|
|
3556
3923
|
const {
|
|
3557
3924
|
environment: sandboxEnvironment,
|
|
3558
3925
|
gitToken: sandboxGitToken,
|
|
@@ -3576,8 +3943,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3576
3943
|
...(gitCredentialAuthority ? { authority: gitCredentialAuthority } : {}),
|
|
3577
3944
|
gitCredentials: connectionCredentials?.gitCredentials,
|
|
3578
3945
|
authorizeGitHubTokenMint,
|
|
3579
|
-
|
|
3580
|
-
runId: turnId,
|
|
3946
|
+
toolspaceAuthority,
|
|
3581
3947
|
},
|
|
3582
3948
|
),
|
|
3583
3949
|
cancellationSignal,
|
|
@@ -3625,6 +3991,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3625
3991
|
authorizeGitHubTokenMint,
|
|
3626
3992
|
},
|
|
3627
3993
|
);
|
|
3994
|
+
if (binding) {
|
|
3995
|
+
assertGitCredentialRenewalTransportUnchanged(initialBinding, binding);
|
|
3996
|
+
}
|
|
3628
3997
|
pendingBinding = binding;
|
|
3629
3998
|
return binding
|
|
3630
3999
|
? {
|
|
@@ -3695,7 +4064,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3695
4064
|
if (toolspaceTokenRenewalClosed) return;
|
|
3696
4065
|
|
|
3697
4066
|
const mint = async () =>
|
|
3698
|
-
await mintSandboxToolspaceToken(runSettings, connectionScope,
|
|
4067
|
+
await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority);
|
|
3699
4068
|
const write = async (material: NonNullable<Awaited<ReturnType<typeof mint>>>) => {
|
|
3700
4069
|
const runAs = sandboxRunAs(runSettings);
|
|
3701
4070
|
await refreshToolspaceTokenFile(tokenSession, material.token, {
|
|
@@ -4197,9 +4566,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4197
4566
|
// token skip does NOT transfer: that token is inert on a connected
|
|
4198
4567
|
// machine (it uses its own git creds), but the toolspace token is the
|
|
4199
4568
|
// machine's ONLY path to programmatic tool calling and grants no more
|
|
4200
|
-
// than toolspace:call for its own session (own-session-bound,
|
|
4201
|
-
// budgeted, approval-tools
|
|
4202
|
-
//
|
|
4569
|
+
// than toolspace:call for its own session (own-session-bound,
|
|
4570
|
+
// short-lived and renewable, per-active-turn budgeted, approval-tools
|
|
4571
|
+
// excluded). The runtime seeds it to the box's token file over the same
|
|
4572
|
+
// exec channel, off-manifest, on every backend.
|
|
4203
4573
|
...(sandboxToolspaceToken
|
|
4204
4574
|
? {
|
|
4205
4575
|
toolspaceTokenSeed: sandboxToolspaceToken,
|
|
@@ -4342,12 +4712,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4342
4712
|
});
|
|
4343
4713
|
const lazyGitTokens = lazyGitCredentials?.gitTokens;
|
|
4344
4714
|
const lazyToolspaceToken = sandboxToolspaceToken
|
|
4345
|
-
? await mintSandboxToolspaceToken(
|
|
4346
|
-
runSettings,
|
|
4347
|
-
connectionScope,
|
|
4348
|
-
input.sessionId,
|
|
4349
|
-
turn.id,
|
|
4350
|
-
)
|
|
4715
|
+
? await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority)
|
|
4351
4716
|
: undefined;
|
|
4352
4717
|
const provisioned = await resumeBoxForTurn(
|
|
4353
4718
|
{
|
|
@@ -4769,13 +5134,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4769
5134
|
return outcome;
|
|
4770
5135
|
};
|
|
4771
5136
|
|
|
5137
|
+
// Keep response usage identity across every stream attempt in this
|
|
5138
|
+
// activity. Context compaction retries the same logical dispatch by
|
|
5139
|
+
// calling runStreamAttempt again; resetting this state there would reuse
|
|
5140
|
+
// the first no-response-ID fallback key and suppress a real model call.
|
|
5141
|
+
const modelResponseUsageState = createModelResponseUsageEventState(
|
|
5142
|
+
claimedModelUsageSourceKeys,
|
|
5143
|
+
);
|
|
4772
5144
|
const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
|
|
4773
5145
|
if (!runInput) {
|
|
4774
5146
|
throw new Error("Run input was not prepared");
|
|
4775
5147
|
}
|
|
4776
5148
|
stream = undefined;
|
|
4777
5149
|
batcher = null;
|
|
4778
|
-
let responseUsageCount = 0;
|
|
4779
5150
|
// The SDK emits every processed call item for one model response before
|
|
4780
5151
|
// it emits any result for that response. Keep that response-local batch
|
|
4781
5152
|
// in memory so an orphan from an older response cannot pin later stable
|
|
@@ -4785,8 +5156,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4785
5156
|
let currentToolBatchCompletedCallIds = new Set<string>();
|
|
4786
5157
|
// Actual input tokens of the most recent model response this turn; the
|
|
4787
5158
|
// pre-read trigger for the NEXT turn. Persisted at every turn-end path.
|
|
4788
|
-
let lastProviderContextTokensObserved: number | null = null;
|
|
4789
|
-
let providerContextRevision = 0;
|
|
4790
5159
|
throwIfWorkerShuttingDown();
|
|
4791
5160
|
throwIfTurnCancelled();
|
|
4792
5161
|
const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
|
|
@@ -4861,13 +5230,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4861
5230
|
: {}),
|
|
4862
5231
|
}
|
|
4863
5232
|
: {}),
|
|
4864
|
-
contextCompactionSignal: () =>
|
|
4865
|
-
lastProviderContextTokensObserved === null
|
|
4866
|
-
? null
|
|
4867
|
-
: {
|
|
4868
|
-
revision: providerContextRevision,
|
|
4869
|
-
totalTokens: lastProviderContextTokensObserved,
|
|
4870
|
-
},
|
|
5233
|
+
contextCompactionSignal: () => modelResponseUsageContextSignal(modelResponseUsageState),
|
|
4871
5234
|
contextCompactionRequested: () =>
|
|
4872
5235
|
isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
4873
5236
|
...(toolCancellationFenceRef.current
|
|
@@ -4906,77 +5269,33 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4906
5269
|
}
|
|
4907
5270
|
let stableToolCallIdsToClear: string[] | null = null;
|
|
4908
5271
|
let completedCurrentToolBatch = false;
|
|
4909
|
-
const
|
|
4910
|
-
|
|
4911
|
-
|
|
4912
|
-
|
|
4913
|
-
|
|
4914
|
-
|
|
4915
|
-
|
|
4916
|
-
|
|
4917
|
-
|
|
4918
|
-
|
|
4919
|
-
|
|
4920
|
-
|
|
4921
|
-
|
|
4922
|
-
|
|
4923
|
-
|
|
4924
|
-
|
|
4925
|
-
|
|
4926
|
-
|
|
4927
|
-
|
|
4928
|
-
|
|
4929
|
-
|
|
4930
|
-
|
|
4931
|
-
|
|
4932
|
-
|
|
4933
|
-
|
|
4934
|
-
|
|
4935
|
-
|
|
4936
|
-
isCodexTurn,
|
|
4937
|
-
usage: responseUsage.usage,
|
|
4938
|
-
sourceKey: responseSourceKey,
|
|
4939
|
-
observability,
|
|
4940
|
-
});
|
|
4941
|
-
await emitModelCallUsage({
|
|
4942
|
-
observability,
|
|
4943
|
-
publish,
|
|
4944
|
-
accountId: input.accountId,
|
|
4945
|
-
workspaceId: input.workspaceId,
|
|
4946
|
-
sessionId: input.sessionId,
|
|
4947
|
-
turnId: activeTurnId,
|
|
4948
|
-
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
4949
|
-
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
4950
|
-
model: turn.model,
|
|
4951
|
-
sourceKey: responseSourceKey,
|
|
4952
|
-
usage: responseUsage,
|
|
4953
|
-
servingAccountHash: responseAccountCtx.servingAccountHash,
|
|
4954
|
-
accountChangedFromPrevCall: responseAccountCtx.accountChangedFromPrevCall,
|
|
4955
|
-
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
4956
|
-
});
|
|
4957
|
-
const observed = responseUsage.usage?.inputTokens;
|
|
4958
|
-
if (typeof observed === "number" && observed > 0) {
|
|
4959
|
-
recordModelInputTokens(observability, streamProvider, observed);
|
|
4960
|
-
}
|
|
4961
|
-
const observedTotal = providerContextTokens(responseUsage.usage);
|
|
4962
|
-
if (observedTotal !== null) {
|
|
4963
|
-
lastProviderContextTokensObserved = observedTotal;
|
|
4964
|
-
providerContextRevision += 1;
|
|
4965
|
-
}
|
|
4966
|
-
// Prompt-cache efficiency for this response — same usage frame as the
|
|
4967
|
-
// input-token accounting above, so the two are always consistent.
|
|
4968
|
-
recordModelCacheTokens(observability, streamProvider, {
|
|
4969
|
-
cachedTokens: modelCallUsageTelemetry(responseUsage.usage).cachedTokens,
|
|
4970
|
-
promptTokens: responseUsage.usage?.inputTokens,
|
|
4971
|
-
});
|
|
4972
|
-
},
|
|
4973
|
-
recordAttemptSignals: async () => {
|
|
4974
|
-
const observed = responseUsage.usage?.inputTokens;
|
|
4975
|
-
if (typeof observed === "number" && observed > 0) {
|
|
4976
|
-
await setLastInputTokensFenced(observed);
|
|
4977
|
-
}
|
|
4978
|
-
},
|
|
4979
|
-
});
|
|
5272
|
+
const responseUsageResult = await processModelResponseUsageEvent({
|
|
5273
|
+
event: next.value,
|
|
5274
|
+
state: modelResponseUsageState,
|
|
5275
|
+
dispatchId: modelUsageDispatchId,
|
|
5276
|
+
settings,
|
|
5277
|
+
db,
|
|
5278
|
+
observability,
|
|
5279
|
+
publish,
|
|
5280
|
+
accountId: input.accountId,
|
|
5281
|
+
workspaceId: input.workspaceId,
|
|
5282
|
+
sessionId: input.sessionId,
|
|
5283
|
+
turnId: activeTurnId,
|
|
5284
|
+
turnAttemptId: input.attemptId,
|
|
5285
|
+
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
5286
|
+
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
5287
|
+
model: turn.model,
|
|
5288
|
+
metricProvider: streamProvider,
|
|
5289
|
+
externallyBilled: isExternallyBilledTurn,
|
|
5290
|
+
servingCredentialId: effectiveCodexCredentialId,
|
|
5291
|
+
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
5292
|
+
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
5293
|
+
renewLease: () => renewCodexLease("model_usage"),
|
|
5294
|
+
leaseLost: () => codexLeaseLost,
|
|
5295
|
+
leaseLostMessage: "Codex credential lease expired during the active turn",
|
|
5296
|
+
setLastInputTokens: setLastInputTokensFenced,
|
|
5297
|
+
});
|
|
5298
|
+
if (responseUsageResult.status === "processed") {
|
|
4980
5299
|
currentToolBatchCallIds = new Set<string>();
|
|
4981
5300
|
currentToolBatchCompletedCallIds = new Set<string>();
|
|
4982
5301
|
await reconcileConversationTruth();
|
|
@@ -4986,7 +5305,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4986
5305
|
db,
|
|
4987
5306
|
input.accountId,
|
|
4988
5307
|
input.workspaceId,
|
|
4989
|
-
|
|
5308
|
+
isExternallyBilledTurn,
|
|
4990
5309
|
entitlements,
|
|
4991
5310
|
);
|
|
4992
5311
|
} catch (limitError) {
|
|
@@ -5111,83 +5430,77 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5111
5430
|
stream.completed.catch(() => undefined),
|
|
5112
5431
|
cancellationSignal,
|
|
5113
5432
|
);
|
|
5114
|
-
if (responseUsageCount === 0) {
|
|
5433
|
+
if (modelResponseUsageState.responseUsageCount === 0) {
|
|
5115
5434
|
const aggregateUsage = stream.state.usage;
|
|
5116
|
-
const
|
|
5117
|
-
|
|
5118
|
-
if (typeof aggregateInput === "number" && aggregateInput > 0) {
|
|
5119
|
-
const aggregateContext = providerContextTokens(
|
|
5120
|
-
aggregateUsage as {
|
|
5121
|
-
inputTokens?: number;
|
|
5122
|
-
outputTokens?: number;
|
|
5123
|
-
totalTokens?: number;
|
|
5124
|
-
},
|
|
5125
|
-
);
|
|
5126
|
-
if (aggregateContext !== null) {
|
|
5127
|
-
lastProviderContextTokensObserved = aggregateContext;
|
|
5128
|
-
providerContextRevision += 1;
|
|
5129
|
-
}
|
|
5130
|
-
}
|
|
5435
|
+
const normalizedAggregateUsage = normalizeModelCallUsage(aggregateUsage);
|
|
5436
|
+
const aggregateInput = normalizedAggregateUsage.telemetry.inputTokens;
|
|
5131
5437
|
const aggregateSourceKey = modelUsageSourceKey({
|
|
5132
5438
|
responseId: null,
|
|
5133
5439
|
dispatchId: modelUsageDispatchId,
|
|
5134
5440
|
positionalKey: "aggregate",
|
|
5135
5441
|
});
|
|
5136
|
-
|
|
5137
|
-
|
|
5138
|
-
|
|
5139
|
-
|
|
5140
|
-
|
|
5141
|
-
|
|
5142
|
-
|
|
5143
|
-
|
|
5144
|
-
|
|
5145
|
-
|
|
5146
|
-
|
|
5147
|
-
|
|
5148
|
-
|
|
5149
|
-
|
|
5150
|
-
|
|
5151
|
-
|
|
5152
|
-
|
|
5153
|
-
|
|
5154
|
-
|
|
5155
|
-
|
|
5156
|
-
|
|
5157
|
-
|
|
5158
|
-
|
|
5159
|
-
|
|
5160
|
-
|
|
5161
|
-
|
|
5162
|
-
|
|
5163
|
-
|
|
5164
|
-
|
|
5165
|
-
|
|
5166
|
-
|
|
5167
|
-
|
|
5168
|
-
|
|
5169
|
-
|
|
5170
|
-
|
|
5171
|
-
|
|
5172
|
-
|
|
5173
|
-
|
|
5174
|
-
|
|
5175
|
-
|
|
5176
|
-
|
|
5177
|
-
|
|
5178
|
-
|
|
5179
|
-
|
|
5180
|
-
|
|
5181
|
-
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
5185
|
-
|
|
5186
|
-
|
|
5187
|
-
|
|
5188
|
-
|
|
5189
|
-
|
|
5190
|
-
|
|
5442
|
+
if (!claimedModelUsageSourceKeys.has(aggregateSourceKey)) {
|
|
5443
|
+
claimedModelUsageSourceKeys.add(aggregateSourceKey);
|
|
5444
|
+
// The single aggregate frame is this turn's only model-usage record, so
|
|
5445
|
+
// it is the first (account-switch surfaces here just like a first response).
|
|
5446
|
+
const aggregateAccountCtx = modelCallAccountContext({
|
|
5447
|
+
servingCredentialId: effectiveCodexCredentialId,
|
|
5448
|
+
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
5449
|
+
isFirstCallOfTurn: true,
|
|
5450
|
+
});
|
|
5451
|
+
let aggregateAuthoritative = false;
|
|
5452
|
+
await recordCompletedModelCallBeforeOwnershipFences({
|
|
5453
|
+
renewLease: () => renewCodexLease("model_usage"),
|
|
5454
|
+
leaseLost: () => codexLeaseLost,
|
|
5455
|
+
leaseLostMessage: "Codex credential lease expired during the active turn",
|
|
5456
|
+
recordUsage: async () => {
|
|
5457
|
+
await recordModelUsageAndDebitCredits(settings, db, {
|
|
5458
|
+
accountId: input.accountId,
|
|
5459
|
+
workspaceId: input.workspaceId,
|
|
5460
|
+
sessionId: input.sessionId,
|
|
5461
|
+
turnId: activeTurnId,
|
|
5462
|
+
turnAttemptId: input.attemptId,
|
|
5463
|
+
model: turn.model,
|
|
5464
|
+
externallyBilled: isExternallyBilledTurn,
|
|
5465
|
+
usage: aggregateUsage,
|
|
5466
|
+
normalizedUsage: normalizedAggregateUsage,
|
|
5467
|
+
sourceKey: aggregateSourceKey,
|
|
5468
|
+
observability,
|
|
5469
|
+
});
|
|
5470
|
+
aggregateAuthoritative = await emitModelCallUsage({
|
|
5471
|
+
observability,
|
|
5472
|
+
publish,
|
|
5473
|
+
accountId: input.accountId,
|
|
5474
|
+
workspaceId: input.workspaceId,
|
|
5475
|
+
sessionId: input.sessionId,
|
|
5476
|
+
turnId: activeTurnId,
|
|
5477
|
+
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
5478
|
+
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
5479
|
+
model: turn.model,
|
|
5480
|
+
sourceKey: aggregateSourceKey,
|
|
5481
|
+
usage: { usage: aggregateUsage },
|
|
5482
|
+
normalizedUsage: normalizedAggregateUsage,
|
|
5483
|
+
servingAccountHash: aggregateAccountCtx.servingAccountHash,
|
|
5484
|
+
accountChangedFromPrevCall: aggregateAccountCtx.accountChangedFromPrevCall,
|
|
5485
|
+
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
5486
|
+
});
|
|
5487
|
+
if (aggregateAuthoritative && aggregateInput !== null && aggregateInput > 0) {
|
|
5488
|
+
recordModelInputTokens(observability, streamProvider, aggregateInput);
|
|
5489
|
+
}
|
|
5490
|
+
},
|
|
5491
|
+
recordAttemptSignals: async () => {
|
|
5492
|
+
if (!aggregateAuthoritative) return;
|
|
5493
|
+
if (normalizedAggregateUsage.totalTokens !== null) {
|
|
5494
|
+
modelResponseUsageState.lastProviderContextTokensObserved =
|
|
5495
|
+
normalizedAggregateUsage.totalTokens;
|
|
5496
|
+
modelResponseUsageState.providerContextRevision += 1;
|
|
5497
|
+
}
|
|
5498
|
+
if (aggregateInput !== null && aggregateInput > 0) {
|
|
5499
|
+
await setLastInputTokensFenced(aggregateInput);
|
|
5500
|
+
}
|
|
5501
|
+
},
|
|
5502
|
+
});
|
|
5503
|
+
}
|
|
5191
5504
|
}
|
|
5192
5505
|
if (stream.interruptions.length > 0) {
|
|
5193
5506
|
await reconcileConversationTruth();
|
|
@@ -6251,7 +6564,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6251
6564
|
// truth, recover this SAME accepted turn, then let the workflow re-claim
|
|
6252
6565
|
// it after a pacing delay. This is independent of goal state and never
|
|
6253
6566
|
// relies on a synthetic continuation prompt.
|
|
6254
|
-
const failure = agentRunFailurePayload(error);
|
|
6567
|
+
const failure = agentRunFailurePayload(error, { isCodexTurn });
|
|
6255
6568
|
if (isSessionEventPersistenceError(error)) {
|
|
6256
6569
|
// Never pass the original Drizzle/postgres-js error to telemetry: its
|
|
6257
6570
|
// nested cause may contain raw SQL and bound parameters. The typed DB
|
|
@@ -6878,11 +7191,17 @@ export function isTransientProviderError(error: unknown): boolean {
|
|
|
6878
7191
|
);
|
|
6879
7192
|
}
|
|
6880
7193
|
|
|
6881
|
-
export function agentRunFailurePayload(
|
|
7194
|
+
export function agentRunFailurePayload(
|
|
7195
|
+
error: unknown,
|
|
7196
|
+
options: { isCodexTurn?: boolean } = {},
|
|
7197
|
+
): {
|
|
6882
7198
|
error: string;
|
|
6883
7199
|
code?: string;
|
|
6884
7200
|
retryable?: boolean;
|
|
6885
7201
|
detail?: string;
|
|
7202
|
+
timeoutClass?: string;
|
|
7203
|
+
responseObserved?: boolean;
|
|
7204
|
+
requestId?: string;
|
|
6886
7205
|
correlationId?: string;
|
|
6887
7206
|
stage?: string;
|
|
6888
7207
|
sqlState?: string | null;
|
|
@@ -6948,6 +7267,22 @@ export function agentRunFailurePayload(error: unknown): {
|
|
|
6948
7267
|
if (usageLimit) {
|
|
6949
7268
|
return codexUsageLimitFailurePayload(usageLimit, message);
|
|
6950
7269
|
}
|
|
7270
|
+
const codexTimeout = classifyCodexResponseTimeoutError(error, {
|
|
7271
|
+
allowLegacyRequestTimeout: options.isCodexTurn === true,
|
|
7272
|
+
});
|
|
7273
|
+
if (codexTimeout) {
|
|
7274
|
+
return {
|
|
7275
|
+
error: codexTimeout.responseObserved
|
|
7276
|
+
? "The Codex response timed out after streaming began. Observed output was checkpointed; automatic replay is disabled because the upstream operation may still be active."
|
|
7277
|
+
: "The Codex response timed out before any response was observed. Upstream acceptance is unknown, so automatic replay is disabled.",
|
|
7278
|
+
code: "codex_response_timeout",
|
|
7279
|
+
retryable: false,
|
|
7280
|
+
timeoutClass: codexTimeout.timeoutClass,
|
|
7281
|
+
responseObserved: codexTimeout.responseObserved,
|
|
7282
|
+
...(codexTimeout.requestId ? { requestId: codexTimeout.requestId } : {}),
|
|
7283
|
+
...(codexTimeout.message ? { detail: codexTimeout.message } : {}),
|
|
7284
|
+
};
|
|
7285
|
+
}
|
|
6951
7286
|
const mcpTimeout = classifyMcpTransportTimeoutError(error);
|
|
6952
7287
|
if (mcpTimeout) {
|
|
6953
7288
|
return {
|
|
@@ -7227,21 +7562,21 @@ class BudgetExhaustedError extends Error {
|
|
|
7227
7562
|
}
|
|
7228
7563
|
}
|
|
7229
7564
|
|
|
7230
|
-
// Exported for unit testing the
|
|
7231
|
-
// of the activity surface. Takes
|
|
7232
|
-
//
|
|
7233
|
-
//
|
|
7565
|
+
// Exported for unit testing the external-billing bypass (codex-billing.test.ts); not
|
|
7566
|
+
// part of the activity surface. Takes the accepted policy's billing attribution and
|
|
7567
|
+
// the optional §7.5 P3 host `entitlements` port (when bound, its `admitRun` REPLACES
|
|
7568
|
+
// the local credit read for an OpenGeni-metered turn; unset → local ledger).
|
|
7234
7569
|
export async function ensureRunAllowed(
|
|
7235
7570
|
settings: Settings,
|
|
7236
7571
|
db: ActivityServices["db"],
|
|
7237
7572
|
accountId: string,
|
|
7238
7573
|
workspaceId: string,
|
|
7239
|
-
|
|
7574
|
+
isExternallyBilledTurn: boolean,
|
|
7240
7575
|
entitlements?: ActivityServices["entitlements"],
|
|
7241
7576
|
): Promise<void> {
|
|
7242
|
-
//
|
|
7243
|
-
//
|
|
7244
|
-
//
|
|
7577
|
+
// Externally billed turns are paid outside OpenGeni: skip the credit-balance
|
|
7578
|
+
// gate and monthly token cap. The agent-run COUNT cap below is a
|
|
7579
|
+
// volume/fairness quota (not a credit/cost gate) and is intentionally kept.
|
|
7245
7580
|
//
|
|
7246
7581
|
// §7.5 P3 — host-entitlements DELEGATION (the worker half of the same seam the
|
|
7247
7582
|
// API edge exposes). For a non-codex turn, when the host binds `entitlements`, its
|
|
@@ -7255,7 +7590,7 @@ export async function ensureRunAllowed(
|
|
|
7255
7590
|
// idempotency-keyed writer at recordModelUsageAndDebitCredits), so a PULL host meter
|
|
7256
7591
|
// is consulted without ever double-charging.
|
|
7257
7592
|
if (
|
|
7258
|
-
!
|
|
7593
|
+
!isExternallyBilledTurn &&
|
|
7259
7594
|
entitlements &&
|
|
7260
7595
|
(settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")
|
|
7261
7596
|
) {
|
|
@@ -7269,7 +7604,7 @@ export async function ensureRunAllowed(
|
|
|
7269
7604
|
throw new Error(decision.reason || "insufficient OpenGeni credits");
|
|
7270
7605
|
}
|
|
7271
7606
|
} else if (
|
|
7272
|
-
!
|
|
7607
|
+
!isExternallyBilledTurn &&
|
|
7273
7608
|
(settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")
|
|
7274
7609
|
) {
|
|
7275
7610
|
const balance = await getBillingBalance(db, accountId);
|
|
@@ -7294,7 +7629,7 @@ export async function ensureRunAllowed(
|
|
|
7294
7629
|
);
|
|
7295
7630
|
}
|
|
7296
7631
|
}
|
|
7297
|
-
if (!
|
|
7632
|
+
if (!isExternallyBilledTurn && limits.maxMonthlyTokensPerWorkspace) {
|
|
7298
7633
|
const used = await sumUsageQuantity(db, {
|
|
7299
7634
|
workspaceId,
|
|
7300
7635
|
eventType: "model.tokens",
|
|
@@ -7307,7 +7642,7 @@ export async function ensureRunAllowed(
|
|
|
7307
7642
|
}
|
|
7308
7643
|
}
|
|
7309
7644
|
|
|
7310
|
-
// Exported for unit testing the
|
|
7645
|
+
// Exported for unit testing the external-billing bypass; not part of the activity surface.
|
|
7311
7646
|
export async function recordModelUsageAndDebitCredits(
|
|
7312
7647
|
settings: Settings,
|
|
7313
7648
|
db: ActivityServices["db"],
|
|
@@ -7318,8 +7653,9 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
7318
7653
|
turnId: string;
|
|
7319
7654
|
turnAttemptId: string;
|
|
7320
7655
|
model: string;
|
|
7321
|
-
|
|
7322
|
-
usage?: ModelUsageInput | null;
|
|
7656
|
+
externallyBilled: boolean;
|
|
7657
|
+
usage?: ModelUsageInput | ModelCallUsageInput | null;
|
|
7658
|
+
normalizedUsage?: ModelCallUsageNormalization;
|
|
7323
7659
|
sourceKey: string;
|
|
7324
7660
|
observability?: ActivityServices["observability"];
|
|
7325
7661
|
},
|
|
@@ -7327,10 +7663,12 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
7327
7663
|
if (!input.usage) {
|
|
7328
7664
|
return;
|
|
7329
7665
|
}
|
|
7330
|
-
const
|
|
7331
|
-
const
|
|
7332
|
-
const
|
|
7333
|
-
|
|
7666
|
+
const normalizedUsage = input.normalizedUsage ?? normalizeModelCallUsage(input.usage);
|
|
7667
|
+
const sanitizedUsage = sanitizedModelUsageInput(normalizedUsage);
|
|
7668
|
+
const inputTokens = sanitizedUsage.inputTokens ?? 0;
|
|
7669
|
+
const outputTokens = sanitizedUsage.outputTokens ?? 0;
|
|
7670
|
+
const totalTokens = sanitizedUsage.totalTokens ?? 0;
|
|
7671
|
+
// An externally billed turn is paid outside OpenGeni, so it
|
|
7334
7672
|
// consumes ZERO OpenGeni credits and must never feed an OpenGeni cap. A
|
|
7335
7673
|
// codex/<slug> model has no entry in configuredModelPricing, so the normal path
|
|
7336
7674
|
// below would throw "Missing model pricing". We:
|
|
@@ -7339,7 +7677,7 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
7339
7677
|
// any row would count against maxMonthlyTokensPerWorkspace);
|
|
7340
7678
|
// - record a `model.cost = 0` audit marker (harmless to the monthly cost cap);
|
|
7341
7679
|
// - never look up pricing and never debit credits.
|
|
7342
|
-
if (input.
|
|
7680
|
+
if (input.externallyBilled) {
|
|
7343
7681
|
await recordUsageEvent(db, {
|
|
7344
7682
|
accountId: input.accountId,
|
|
7345
7683
|
workspaceId: input.workspaceId,
|
|
@@ -7377,7 +7715,7 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
7377
7715
|
if (!configuredModelPricing(settings)[input.model]) {
|
|
7378
7716
|
throw new Error(`Missing model pricing for ${input.model}`);
|
|
7379
7717
|
}
|
|
7380
|
-
const costMicros = calculateModelUsageCostMicros(settings, input.model,
|
|
7718
|
+
const costMicros = calculateModelUsageCostMicros(settings, input.model, sanitizedUsage);
|
|
7381
7719
|
await recordUsageEvent(db, {
|
|
7382
7720
|
accountId: input.accountId,
|
|
7383
7721
|
workspaceId: input.workspaceId,
|
|
@@ -7411,18 +7749,26 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
7411
7749
|
// Additive: the prompt-cache slice of this call's input tokens, so the
|
|
7412
7750
|
// per-call debit record carries cache efficiency alongside the token
|
|
7413
7751
|
// counts. 0 when the provider did not report cached tokens.
|
|
7414
|
-
cachedTokens:
|
|
7415
|
-
modelCallUsageTelemetry(input.usage as Parameters<typeof modelCallUsageTelemetry>[0])
|
|
7416
|
-
.cachedTokens,
|
|
7417
|
-
),
|
|
7752
|
+
cachedTokens: normalizedUsage.telemetry.cachedTokens ?? 0,
|
|
7418
7753
|
},
|
|
7419
7754
|
});
|
|
7420
7755
|
recordCreditMicros(input.observability, "usage", result.debitedMicros);
|
|
7421
7756
|
}
|
|
7422
7757
|
}
|
|
7423
7758
|
|
|
7424
|
-
function
|
|
7425
|
-
return
|
|
7759
|
+
function sanitizedModelUsageInput(normalized: ModelCallUsageNormalization): ModelUsageInput {
|
|
7760
|
+
return {
|
|
7761
|
+
...(normalized.telemetry.inputTokens !== null
|
|
7762
|
+
? { inputTokens: normalized.telemetry.inputTokens }
|
|
7763
|
+
: {}),
|
|
7764
|
+
...(normalized.telemetry.outputTokens !== null
|
|
7765
|
+
? { outputTokens: normalized.telemetry.outputTokens }
|
|
7766
|
+
: {}),
|
|
7767
|
+
...(normalized.totalTokens !== null ? { totalTokens: normalized.totalTokens } : {}),
|
|
7768
|
+
...(normalized.telemetry.cachedTokens !== null
|
|
7769
|
+
? { inputTokensDetails: { cached_tokens: normalized.telemetry.cachedTokens } }
|
|
7770
|
+
: {}),
|
|
7771
|
+
};
|
|
7426
7772
|
}
|
|
7427
7773
|
|
|
7428
7774
|
function startOfUtcMonth(): Date {
|