@opengeni/worker-bundle 0.8.3 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +2735 -905
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +7 -4
- package/package.json +11 -11
- package/src/activities/agent-turn.ts +1123 -462
- package/src/activities/capabilities.ts +3 -1
- package/src/activities/codex-fleet-shadow.ts +296 -0
- package/src/activities/environment.ts +142 -11
- package/src/activities/goals.ts +13 -79
- package/src/activities/rig-verification.ts +897 -244
- package/src/activities/run-input.ts +186 -13
- package/src/activities/sandbox-lease.ts +162 -84
- package/src/activities/scheduled-tasks.ts +42 -27
- package/src/observability-metrics.ts +83 -15
- package/src/sandbox-resume.ts +249 -94
- package/src/sandbox-routing.ts +597 -2
- package/src/workflows/activities.ts +4 -3
- package/src/workflows/rig-verification.ts +3 -1
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
|
+
advanceWorkspaceGeneration,
|
|
3
|
+
verifyWorkspaceMutationSettlement,
|
|
2
4
|
applySessionTurnSettlement,
|
|
3
5
|
requestSessionTurnRecovery,
|
|
4
6
|
claimSessionWorkForAttempt,
|
|
@@ -16,7 +18,7 @@ import {
|
|
|
16
18
|
getLatestRunState,
|
|
17
19
|
getHumanInputResumeForEvent,
|
|
18
20
|
getSessionHumanInputRequest,
|
|
19
|
-
|
|
21
|
+
installOrReadTurnExecutionPolicyForAttempt,
|
|
20
22
|
workspaceCodexSubscriptionActive,
|
|
21
23
|
acquireCodexCredentialLease,
|
|
22
24
|
armCodexCapacityWait,
|
|
@@ -76,8 +78,8 @@ import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@
|
|
|
76
78
|
import {
|
|
77
79
|
sandboxStateEntryFromRunState,
|
|
78
80
|
maxTurnsExceededRunState,
|
|
79
|
-
modelCallUsageTelemetry,
|
|
80
81
|
modelResponseUsageFromSdkEvent,
|
|
82
|
+
normalizeModelCallUsage,
|
|
81
83
|
normalizeSdkEvent,
|
|
82
84
|
sanitizeHistoryItemsForModel,
|
|
83
85
|
isEphemeralInternalContext,
|
|
@@ -101,12 +103,15 @@ import {
|
|
|
101
103
|
sandboxFileDownloadFailureNote,
|
|
102
104
|
SUMMARY_BUFFER_TOKENS,
|
|
103
105
|
runOwnedSandboxSetup,
|
|
106
|
+
RoutingMutationOutcomeUnknownError,
|
|
104
107
|
swapTargetEstablishability,
|
|
105
108
|
type SandboxFileDownload,
|
|
106
109
|
type SandboxFileDownloadFailure,
|
|
107
110
|
type OpenGeniRuntime,
|
|
108
111
|
type ComputerToolMode,
|
|
109
112
|
type ModelResponseUsage,
|
|
113
|
+
type ModelCallUsageInput,
|
|
114
|
+
type ModelCallUsageNormalization,
|
|
110
115
|
type BuildAgentOptions,
|
|
111
116
|
type TurnToolCancellationFence,
|
|
112
117
|
type BackendUnresolvableCode,
|
|
@@ -120,12 +125,13 @@ import {
|
|
|
120
125
|
} from "@opengeni/runtime";
|
|
121
126
|
import { connectionTokenResolverForTurn } from "./mcp-credentials";
|
|
122
127
|
import {
|
|
123
|
-
|
|
128
|
+
assertTurnExecutionPolicyMatchesConfigV1,
|
|
124
129
|
calculateModelUsageCostMicros,
|
|
125
130
|
configuredModelPricing,
|
|
126
131
|
configuredStaticUsageLimits,
|
|
127
132
|
sandboxWarmRateMicrosPerSecond,
|
|
128
133
|
settingsWithResolvedModelContext,
|
|
134
|
+
resolveTurnExecutionPolicyV1,
|
|
129
135
|
type ModelUsageInput,
|
|
130
136
|
type ModelProviderApi,
|
|
131
137
|
type RegistryProviderKind,
|
|
@@ -151,6 +157,11 @@ import {
|
|
|
151
157
|
type CodexRotationStrategy,
|
|
152
158
|
type RotationDecision,
|
|
153
159
|
} from "./codex-rotation";
|
|
160
|
+
import {
|
|
161
|
+
codexFleetShadowDecisionMetricLabelsV1,
|
|
162
|
+
codexFleetShadowErrorMetricLabelsV1,
|
|
163
|
+
publishCodexFleetShadowDecisionV1,
|
|
164
|
+
} from "./codex-fleet-shadow";
|
|
154
165
|
import type { CodexAccountStatus } from "@opengeni/db";
|
|
155
166
|
import { buildCodexTokenResolver } from "./codex-auth";
|
|
156
167
|
import {
|
|
@@ -163,16 +174,19 @@ import {
|
|
|
163
174
|
CODEX_CLIENT_VERSION,
|
|
164
175
|
CODEX_FALLBACK_MODEL_SLUGS,
|
|
165
176
|
CodexReloginRequired,
|
|
177
|
+
classifyCodexResponseTimeoutError,
|
|
166
178
|
classifyCodexUsageLimitError,
|
|
167
179
|
codexRequestStorage,
|
|
168
180
|
isCodexTransportError,
|
|
169
181
|
type CodexRequestContext,
|
|
170
182
|
type CodexUsageHeaderSnapshot,
|
|
171
183
|
} from "@opengeni/codex";
|
|
172
|
-
import { mergeResourceRefs
|
|
184
|
+
import { mergeResourceRefs } from "./common";
|
|
185
|
+
import { enabledCapabilityMcpToolRefs, resolveSessionToolPolicy } from "@opengeni/core";
|
|
173
186
|
import { maybeCompactContext } from "./context-compaction";
|
|
174
187
|
import { TurnAttemptFencedError } from "./turn-attempt-fenced";
|
|
175
188
|
import {
|
|
189
|
+
assertGitCredentialRenewalTransportUnchanged,
|
|
176
190
|
gitCredentialAuthorityForTurn,
|
|
177
191
|
gitHubTokenMintSelections,
|
|
178
192
|
loadWorkspaceEnvironmentForRunWithCredentials,
|
|
@@ -274,10 +288,13 @@ import {
|
|
|
274
288
|
import {
|
|
275
289
|
CAPABILITY_DESCRIPTORS,
|
|
276
290
|
evaluateWorkspaceModelPolicy,
|
|
291
|
+
readTurnExecutionPolicyV1,
|
|
277
292
|
type ResourceRef,
|
|
278
293
|
type SessionEvent,
|
|
279
294
|
type SessionEventType,
|
|
280
295
|
type SessionStatus,
|
|
296
|
+
type SessionTurn,
|
|
297
|
+
type TurnExecutionPolicyV1,
|
|
281
298
|
} from "@opengeni/contracts";
|
|
282
299
|
import { createHash, randomUUID } from "node:crypto";
|
|
283
300
|
|
|
@@ -287,6 +304,32 @@ import { createHash, randomUUID } from "node:crypto";
|
|
|
287
304
|
// budget against the same window.
|
|
288
305
|
export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
|
|
289
306
|
|
|
307
|
+
export function turnExecutionPolicyBillingIdentity(policy: TurnExecutionPolicyV1): {
|
|
308
|
+
externallyBilled: boolean;
|
|
309
|
+
codexSubscription: boolean;
|
|
310
|
+
} {
|
|
311
|
+
return {
|
|
312
|
+
externallyBilled: policy.billing.metering === "external",
|
|
313
|
+
codexSubscription:
|
|
314
|
+
policy.providerId === "codex-subscription" &&
|
|
315
|
+
policy.credentialSource.kind === "connected_subscription" &&
|
|
316
|
+
policy.credentialSource.provider === "codex",
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
export function legacyTurnExecutionPolicyInput(
|
|
321
|
+
turn: Pick<SessionTurn, "source" | "model" | "reasoningEffort">,
|
|
322
|
+
): Parameters<typeof resolveTurnExecutionPolicyV1>[1] {
|
|
323
|
+
const explicit = turn.source === "user" || turn.source === "api";
|
|
324
|
+
return {
|
|
325
|
+
modelId: turn.model,
|
|
326
|
+
requestedModelId: explicit ? turn.model : null,
|
|
327
|
+
modelSource: explicit ? "explicit" : "continuation",
|
|
328
|
+
reasoningEffort: turn.reasoningEffort,
|
|
329
|
+
reasoningSource: explicit ? "explicit" : "continuation",
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
|
|
290
333
|
/** A retryable provider fault recovers the accepted turn itself. Goal state is
|
|
291
334
|
* irrelevant: autonomous continuation and infrastructure recovery are separate
|
|
292
335
|
* concerns. */
|
|
@@ -491,6 +534,7 @@ export async function persistOrSignalSessionAttemptQuiescence(input: {
|
|
|
491
534
|
export async function drainAttemptOwnedSandboxWriters(input: {
|
|
492
535
|
toolCancellationFence: Pick<TurnToolCancellationFence, "cancel" | "waitForQuiescence"> | null;
|
|
493
536
|
cancellationReason?: unknown;
|
|
537
|
+
gitCredentialRenewals: readonly Pick<GitCredentialRenewalController, "stop">[];
|
|
494
538
|
toolspaceTokenRenewal: Pick<ToolspaceTokenRenewalController, "stop"> | null;
|
|
495
539
|
runCredentialRenewal: Pick<RunCredentialRenewalController, "stop"> | null;
|
|
496
540
|
}): Promise<void> {
|
|
@@ -500,6 +544,7 @@ export async function drainAttemptOwnedSandboxWriters(input: {
|
|
|
500
544
|
);
|
|
501
545
|
await input.toolCancellationFence.waitForQuiescence();
|
|
502
546
|
}
|
|
547
|
+
await Promise.all(input.gitCredentialRenewals.map(async (renewal) => await renewal.stop()));
|
|
503
548
|
await input.toolspaceTokenRenewal?.stop();
|
|
504
549
|
await input.runCredentialRenewal?.stop();
|
|
505
550
|
}
|
|
@@ -733,16 +778,8 @@ export function providerContextTokens(
|
|
|
733
778
|
| null
|
|
734
779
|
| undefined,
|
|
735
780
|
): number | null {
|
|
736
|
-
const total = usage
|
|
737
|
-
|
|
738
|
-
return total;
|
|
739
|
-
}
|
|
740
|
-
const input = usage?.inputTokens;
|
|
741
|
-
if (typeof input !== "number" || !Number.isFinite(input) || input <= 0) {
|
|
742
|
-
return null;
|
|
743
|
-
}
|
|
744
|
-
const output = usage?.outputTokens;
|
|
745
|
-
return input + (typeof output === "number" && Number.isFinite(output) && output > 0 ? output : 0);
|
|
781
|
+
const total = normalizeModelCallUsage(usage).totalTokens;
|
|
782
|
+
return total !== null && total > 0 ? total : null;
|
|
746
783
|
}
|
|
747
784
|
|
|
748
785
|
/**
|
|
@@ -773,6 +810,260 @@ type TurnEventPublisher = (
|
|
|
773
810
|
immediate?: boolean,
|
|
774
811
|
) => Promise<{ events: SessionEvent[]; accepted: boolean }>;
|
|
775
812
|
|
|
813
|
+
export type ModelResponseUsageEventState = {
|
|
814
|
+
responseUsageCount: number;
|
|
815
|
+
providerContextRevision: number;
|
|
816
|
+
lastProviderContextTokensObserved: number | null;
|
|
817
|
+
claimedSourceKeys: Set<string>;
|
|
818
|
+
};
|
|
819
|
+
|
|
820
|
+
export type CompactionModelUsageEventState = {
|
|
821
|
+
usageCount: number;
|
|
822
|
+
claimedSourceKeys: Set<string>;
|
|
823
|
+
};
|
|
824
|
+
|
|
825
|
+
export function createModelResponseUsageEventState(
|
|
826
|
+
claimedSourceKeys: Set<string> = new Set<string>(),
|
|
827
|
+
): ModelResponseUsageEventState {
|
|
828
|
+
return {
|
|
829
|
+
responseUsageCount: 0,
|
|
830
|
+
providerContextRevision: 0,
|
|
831
|
+
lastProviderContextTokensObserved: null,
|
|
832
|
+
claimedSourceKeys,
|
|
833
|
+
};
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
export function createCompactionModelUsageEventState(
|
|
837
|
+
claimedSourceKeys: Set<string> = new Set<string>(),
|
|
838
|
+
): CompactionModelUsageEventState {
|
|
839
|
+
return { usageCount: 0, claimedSourceKeys };
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
export function modelResponseUsageContextSignal(
|
|
843
|
+
state: ModelResponseUsageEventState,
|
|
844
|
+
): { revision: number; totalTokens: number } | null {
|
|
845
|
+
return state.lastProviderContextTokensObserved === null
|
|
846
|
+
? null
|
|
847
|
+
: {
|
|
848
|
+
revision: state.providerContextRevision,
|
|
849
|
+
totalTokens: state.lastProviderContextTokensObserved,
|
|
850
|
+
};
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
/**
|
|
854
|
+
* Process one SDK stream event through the exact production usage path.
|
|
855
|
+
*
|
|
856
|
+
* The pinned Responses SDK mirrors one provider terminal response as both a
|
|
857
|
+
* normalized `response_done` and a raw `model/response.completed` event. Claim
|
|
858
|
+
* the stable response/source key before lease renewal or any usage side effect,
|
|
859
|
+
* and advance the positional ordinal only for a newly claimed response. The
|
|
860
|
+
* durable `agent.model.usage` source-key fence remains the cross-restart
|
|
861
|
+
* authority: a replay may retry the idempotent billing write, but it cannot
|
|
862
|
+
* advance process-local metrics, provider context, or attempt-owned signals.
|
|
863
|
+
*/
|
|
864
|
+
export async function processModelResponseUsageEvent(input: {
|
|
865
|
+
event: Parameters<typeof modelResponseUsageFromSdkEvent>[0];
|
|
866
|
+
state: ModelResponseUsageEventState;
|
|
867
|
+
dispatchId: string | null;
|
|
868
|
+
settings: Settings;
|
|
869
|
+
db: ActivityServices["db"];
|
|
870
|
+
observability: ActivityServices["observability"];
|
|
871
|
+
publish: TurnEventPublisher | null;
|
|
872
|
+
accountId: string;
|
|
873
|
+
workspaceId: string;
|
|
874
|
+
sessionId: string;
|
|
875
|
+
turnId: string;
|
|
876
|
+
turnAttemptId: string;
|
|
877
|
+
provider: string;
|
|
878
|
+
providerApi: ModelProviderApi;
|
|
879
|
+
model: string;
|
|
880
|
+
metricProvider: string;
|
|
881
|
+
externallyBilled: boolean;
|
|
882
|
+
servingCredentialId: string | null;
|
|
883
|
+
priorSessionCredentialId: string | null;
|
|
884
|
+
emittedSourceKeys: Set<string>;
|
|
885
|
+
renewLease: () => Promise<void>;
|
|
886
|
+
leaseLost: () => boolean;
|
|
887
|
+
leaseLostMessage: string;
|
|
888
|
+
setLastInputTokens: (tokens: number) => Promise<void>;
|
|
889
|
+
}): Promise<
|
|
890
|
+
| { status: "not_usage" }
|
|
891
|
+
| { status: "duplicate"; sourceKey: string }
|
|
892
|
+
| { status: "processed"; sourceKey: string; authoritative: boolean }
|
|
893
|
+
> {
|
|
894
|
+
const responseUsage = modelResponseUsageFromSdkEvent(input.event);
|
|
895
|
+
if (!responseUsage) {
|
|
896
|
+
return { status: "not_usage" };
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
const responseOrdinal = input.state.responseUsageCount + 1;
|
|
900
|
+
const sourceKey = modelUsageSourceKey({
|
|
901
|
+
responseId: responseUsage.responseId,
|
|
902
|
+
dispatchId: input.dispatchId,
|
|
903
|
+
positionalKey: `response-${responseOrdinal}`,
|
|
904
|
+
});
|
|
905
|
+
if (input.state.claimedSourceKeys.has(sourceKey)) {
|
|
906
|
+
return { status: "duplicate", sourceKey };
|
|
907
|
+
}
|
|
908
|
+
input.state.claimedSourceKeys.add(sourceKey);
|
|
909
|
+
input.state.responseUsageCount = responseOrdinal;
|
|
910
|
+
|
|
911
|
+
const normalizedUsage = normalizeModelCallUsage(responseUsage.usage);
|
|
912
|
+
const accountContext = modelCallAccountContext({
|
|
913
|
+
servingCredentialId: input.servingCredentialId,
|
|
914
|
+
priorSessionCredentialId: input.priorSessionCredentialId,
|
|
915
|
+
isFirstCallOfTurn: responseOrdinal === 1,
|
|
916
|
+
});
|
|
917
|
+
let authoritative = false;
|
|
918
|
+
await recordCompletedModelCallBeforeOwnershipFences({
|
|
919
|
+
renewLease: input.renewLease,
|
|
920
|
+
leaseLost: input.leaseLost,
|
|
921
|
+
leaseLostMessage: input.leaseLostMessage,
|
|
922
|
+
recordUsage: async () => {
|
|
923
|
+
await recordModelUsageAndDebitCredits(input.settings, input.db, {
|
|
924
|
+
accountId: input.accountId,
|
|
925
|
+
workspaceId: input.workspaceId,
|
|
926
|
+
sessionId: input.sessionId,
|
|
927
|
+
turnId: input.turnId,
|
|
928
|
+
turnAttemptId: input.turnAttemptId,
|
|
929
|
+
model: input.model,
|
|
930
|
+
externallyBilled: input.externallyBilled,
|
|
931
|
+
usage: responseUsage.usage,
|
|
932
|
+
normalizedUsage,
|
|
933
|
+
sourceKey,
|
|
934
|
+
observability: input.observability,
|
|
935
|
+
});
|
|
936
|
+
authoritative = await emitModelCallUsage({
|
|
937
|
+
observability: input.observability,
|
|
938
|
+
publish: input.publish,
|
|
939
|
+
accountId: input.accountId,
|
|
940
|
+
workspaceId: input.workspaceId,
|
|
941
|
+
sessionId: input.sessionId,
|
|
942
|
+
turnId: input.turnId,
|
|
943
|
+
provider: input.provider,
|
|
944
|
+
providerApi: input.providerApi,
|
|
945
|
+
model: input.model,
|
|
946
|
+
sourceKey,
|
|
947
|
+
usage: responseUsage,
|
|
948
|
+
normalizedUsage,
|
|
949
|
+
servingAccountHash: accountContext.servingAccountHash,
|
|
950
|
+
accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
|
|
951
|
+
emittedSourceKeys: input.emittedSourceKeys,
|
|
952
|
+
});
|
|
953
|
+
const observedInput = normalizedUsage.telemetry.inputTokens;
|
|
954
|
+
if (authoritative && observedInput !== null && observedInput > 0) {
|
|
955
|
+
recordModelInputTokens(input.observability, input.metricProvider, observedInput);
|
|
956
|
+
}
|
|
957
|
+
},
|
|
958
|
+
recordAttemptSignals: async () => {
|
|
959
|
+
if (!authoritative) return;
|
|
960
|
+
const observedTotal = normalizedUsage.totalTokens;
|
|
961
|
+
if (observedTotal !== null) {
|
|
962
|
+
input.state.lastProviderContextTokensObserved = observedTotal;
|
|
963
|
+
input.state.providerContextRevision += 1;
|
|
964
|
+
}
|
|
965
|
+
const observedInput = normalizedUsage.telemetry.inputTokens;
|
|
966
|
+
if (observedInput !== null && observedInput > 0) {
|
|
967
|
+
await input.setLastInputTokens(observedInput);
|
|
968
|
+
}
|
|
969
|
+
},
|
|
970
|
+
});
|
|
971
|
+
return { status: "processed", sourceKey, authoritative };
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
/**
|
|
975
|
+
* Apply the same source-key authority ordering to the compaction summarizer's
|
|
976
|
+
* usage callback. The summarizer can retry or mirror a terminal response just
|
|
977
|
+
* like the main stream, so claim before lease renewal, billing, durable usage,
|
|
978
|
+
* logging, or cache metrics. Durable source-key idempotency remains the
|
|
979
|
+
* cross-process authority after a worker restart.
|
|
980
|
+
*/
|
|
981
|
+
export async function processCompactionModelUsageEvent(input: {
|
|
982
|
+
usage: ModelResponseUsage;
|
|
983
|
+
state: CompactionModelUsageEventState;
|
|
984
|
+
dispatchId: string | null;
|
|
985
|
+
settings: Settings;
|
|
986
|
+
db: ActivityServices["db"];
|
|
987
|
+
observability: ActivityServices["observability"];
|
|
988
|
+
publish: TurnEventPublisher | null;
|
|
989
|
+
accountId: string;
|
|
990
|
+
workspaceId: string;
|
|
991
|
+
sessionId: string;
|
|
992
|
+
turnId: string;
|
|
993
|
+
turnAttemptId: string;
|
|
994
|
+
provider: string;
|
|
995
|
+
providerApi: ModelProviderApi;
|
|
996
|
+
model: string;
|
|
997
|
+
externallyBilled: boolean;
|
|
998
|
+
servingCredentialId: string | null;
|
|
999
|
+
priorSessionCredentialId: string | null;
|
|
1000
|
+
emittedSourceKeys: Set<string>;
|
|
1001
|
+
renewLease: () => Promise<void>;
|
|
1002
|
+
leaseLost: () => boolean;
|
|
1003
|
+
leaseLostMessage: string;
|
|
1004
|
+
}): Promise<
|
|
1005
|
+
| { status: "duplicate"; sourceKey: string }
|
|
1006
|
+
| { status: "processed"; sourceKey: string; authoritative: boolean }
|
|
1007
|
+
> {
|
|
1008
|
+
const usageOrdinal = input.state.usageCount + 1;
|
|
1009
|
+
const sourceKey = modelUsageSourceKey({
|
|
1010
|
+
responseId: input.usage.responseId,
|
|
1011
|
+
dispatchId: input.dispatchId,
|
|
1012
|
+
positionalKey: `compaction-${usageOrdinal}`,
|
|
1013
|
+
});
|
|
1014
|
+
if (input.state.claimedSourceKeys.has(sourceKey)) {
|
|
1015
|
+
return { status: "duplicate", sourceKey };
|
|
1016
|
+
}
|
|
1017
|
+
input.state.claimedSourceKeys.add(sourceKey);
|
|
1018
|
+
input.state.usageCount = usageOrdinal;
|
|
1019
|
+
|
|
1020
|
+
const accountContext = modelCallAccountContext({
|
|
1021
|
+
servingCredentialId: input.servingCredentialId,
|
|
1022
|
+
priorSessionCredentialId: input.priorSessionCredentialId,
|
|
1023
|
+
isFirstCallOfTurn: usageOrdinal === 1,
|
|
1024
|
+
});
|
|
1025
|
+
const normalizedUsage = normalizeModelCallUsage(input.usage.usage);
|
|
1026
|
+
let authoritative = false;
|
|
1027
|
+
await recordCompletedModelCallBeforeOwnershipFences({
|
|
1028
|
+
renewLease: input.renewLease,
|
|
1029
|
+
leaseLost: input.leaseLost,
|
|
1030
|
+
leaseLostMessage: input.leaseLostMessage,
|
|
1031
|
+
recordUsage: async () => {
|
|
1032
|
+
await recordModelUsageAndDebitCredits(input.settings, input.db, {
|
|
1033
|
+
accountId: input.accountId,
|
|
1034
|
+
workspaceId: input.workspaceId,
|
|
1035
|
+
sessionId: input.sessionId,
|
|
1036
|
+
turnId: input.turnId,
|
|
1037
|
+
turnAttemptId: input.turnAttemptId,
|
|
1038
|
+
model: input.model,
|
|
1039
|
+
externallyBilled: input.externallyBilled,
|
|
1040
|
+
usage: input.usage.usage,
|
|
1041
|
+
normalizedUsage,
|
|
1042
|
+
sourceKey,
|
|
1043
|
+
observability: input.observability,
|
|
1044
|
+
});
|
|
1045
|
+
authoritative = await emitModelCallUsage({
|
|
1046
|
+
observability: input.observability,
|
|
1047
|
+
publish: input.publish,
|
|
1048
|
+
accountId: input.accountId,
|
|
1049
|
+
workspaceId: input.workspaceId,
|
|
1050
|
+
sessionId: input.sessionId,
|
|
1051
|
+
turnId: input.turnId,
|
|
1052
|
+
provider: input.provider,
|
|
1053
|
+
providerApi: input.providerApi,
|
|
1054
|
+
model: input.model,
|
|
1055
|
+
sourceKey,
|
|
1056
|
+
usage: input.usage,
|
|
1057
|
+
normalizedUsage,
|
|
1058
|
+
servingAccountHash: accountContext.servingAccountHash,
|
|
1059
|
+
accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
|
|
1060
|
+
emittedSourceKeys: input.emittedSourceKeys,
|
|
1061
|
+
});
|
|
1062
|
+
},
|
|
1063
|
+
});
|
|
1064
|
+
return { status: "processed", sourceKey, authoritative };
|
|
1065
|
+
}
|
|
1066
|
+
|
|
776
1067
|
export async function emitModelCallUsage(input: {
|
|
777
1068
|
observability: ActivityServices["observability"];
|
|
778
1069
|
publish: TurnEventPublisher | null;
|
|
@@ -785,22 +1076,25 @@ export async function emitModelCallUsage(input: {
|
|
|
785
1076
|
model: string;
|
|
786
1077
|
sourceKey: string;
|
|
787
1078
|
usage: ModelResponseUsage | { usage?: unknown | null } | null;
|
|
1079
|
+
normalizedUsage?: ModelCallUsageNormalization;
|
|
788
1080
|
// Prompt-cache research dimensions (log-only; NEVER on a metric label or a
|
|
789
1081
|
// durable event). The opaque serving-account tag and whether it changed since
|
|
790
1082
|
// the session's previous call — the account-switch hypothesis for cache misses.
|
|
791
1083
|
servingAccountHash?: string;
|
|
792
1084
|
accountChangedFromPrevCall?: boolean;
|
|
793
1085
|
emittedSourceKeys?: Set<string>;
|
|
794
|
-
}): Promise<
|
|
1086
|
+
}): Promise<boolean> {
|
|
795
1087
|
const usage =
|
|
796
1088
|
input.usage && typeof input.usage === "object" && "usage" in input.usage
|
|
797
1089
|
? (input.usage as { usage?: unknown }).usage
|
|
798
1090
|
: null;
|
|
799
1091
|
if (!usage || typeof usage !== "object") {
|
|
800
|
-
return;
|
|
1092
|
+
return false;
|
|
801
1093
|
}
|
|
802
|
-
if (input.emittedSourceKeys?.has(input.sourceKey)) return;
|
|
803
|
-
const
|
|
1094
|
+
if (input.emittedSourceKeys?.has(input.sourceKey)) return false;
|
|
1095
|
+
const normalizedUsage =
|
|
1096
|
+
input.normalizedUsage ?? normalizeModelCallUsage(usage as ModelCallUsageInput);
|
|
1097
|
+
const telemetry = normalizedUsage.telemetry;
|
|
804
1098
|
const appended = await input.publish?.(
|
|
805
1099
|
[
|
|
806
1100
|
{
|
|
@@ -829,7 +1123,7 @@ export async function emitModelCallUsage(input: {
|
|
|
829
1123
|
typeof event.payload === "object" &&
|
|
830
1124
|
(event.payload as Record<string, unknown>).sourceKey === input.sourceKey,
|
|
831
1125
|
);
|
|
832
|
-
if (!authoritative) return;
|
|
1126
|
+
if (!authoritative) return false;
|
|
833
1127
|
try {
|
|
834
1128
|
input.observability.info("model call usage", {
|
|
835
1129
|
accountId: input.accountId,
|
|
@@ -843,6 +1137,7 @@ export async function emitModelCallUsage(input: {
|
|
|
843
1137
|
inputTokens: telemetry.inputTokens,
|
|
844
1138
|
outputTokens: telemetry.outputTokens,
|
|
845
1139
|
cachedTokens: telemetry.cachedTokens,
|
|
1140
|
+
cacheWriteTokens: telemetry.cacheWriteTokens,
|
|
846
1141
|
reasoningTokens: telemetry.reasoningTokens,
|
|
847
1142
|
...(input.servingAccountHash !== undefined
|
|
848
1143
|
? { servingAccountHash: input.servingAccountHash }
|
|
@@ -851,9 +1146,41 @@ export async function emitModelCallUsage(input: {
|
|
|
851
1146
|
? { accountChangedFromPrevCall: input.accountChangedFromPrevCall }
|
|
852
1147
|
: {}),
|
|
853
1148
|
});
|
|
1149
|
+
if (normalizedUsage.rejectedFields.length > 0) {
|
|
1150
|
+
input.observability.warn("model call usage fields rejected", {
|
|
1151
|
+
provider: input.provider,
|
|
1152
|
+
providerApi: input.providerApi,
|
|
1153
|
+
model: input.model,
|
|
1154
|
+
sourceKey: input.sourceKey,
|
|
1155
|
+
rejectedFields: normalizedUsage.rejectedFields.join(","),
|
|
1156
|
+
});
|
|
1157
|
+
}
|
|
854
1158
|
} catch {
|
|
855
1159
|
// Durable event + billing already committed; logging is best-effort only.
|
|
856
1160
|
}
|
|
1161
|
+
try {
|
|
1162
|
+
applyCodexCacheTelemetry(input.observability, input.provider, normalizedUsage);
|
|
1163
|
+
} catch {
|
|
1164
|
+
// Durable event + billing already committed; metrics are best-effort only.
|
|
1165
|
+
}
|
|
1166
|
+
return true;
|
|
1167
|
+
}
|
|
1168
|
+
|
|
1169
|
+
/**
|
|
1170
|
+
* Apply one authoritative, normalized model-call usage frame to the shared
|
|
1171
|
+
* prompt-cache metrics. The durable source-key fence in `emitModelCallUsage`
|
|
1172
|
+
* owns idempotency; this helper must never receive raw provider values.
|
|
1173
|
+
*/
|
|
1174
|
+
export function applyCodexCacheTelemetry(
|
|
1175
|
+
observability: ActivityServices["observability"],
|
|
1176
|
+
provider: string,
|
|
1177
|
+
normalizedUsage: ModelCallUsageNormalization,
|
|
1178
|
+
): void {
|
|
1179
|
+
recordModelCacheTokens(observability, provider, {
|
|
1180
|
+
cachedTokens: normalizedUsage.telemetry.cachedTokens,
|
|
1181
|
+
cacheWriteTokens: normalizedUsage.telemetry.cacheWriteTokens,
|
|
1182
|
+
promptTokens: normalizedUsage.telemetry.inputTokens,
|
|
1183
|
+
});
|
|
857
1184
|
}
|
|
858
1185
|
|
|
859
1186
|
export function historyRowsToAppend(
|
|
@@ -1216,6 +1543,19 @@ export function computerToolModeForTurn(
|
|
|
1216
1543
|
return "hosted";
|
|
1217
1544
|
}
|
|
1218
1545
|
|
|
1546
|
+
/**
|
|
1547
|
+
* Decide whether this turn's provider accepts typed file/image content in a
|
|
1548
|
+
* user message. The legacy built-in client and registry Responses clients use
|
|
1549
|
+
* the Agents SDK Responses converter, which supports `input_image` and
|
|
1550
|
+
* `input_file`. Chat-completions providers retain the sandbox-path projection
|
|
1551
|
+
* until that transport declares and proves an equivalent capability.
|
|
1552
|
+
*/
|
|
1553
|
+
export function modelAcceptsTypedAttachmentContentForTurn(
|
|
1554
|
+
resolvedModel: { provider: { api: ModelProviderApi } } | null,
|
|
1555
|
+
): boolean {
|
|
1556
|
+
return resolvedModel === null || resolvedModel.provider.api === "responses";
|
|
1557
|
+
}
|
|
1558
|
+
|
|
1219
1559
|
export type TurnSandboxProvisioner<T> = {
|
|
1220
1560
|
get(): Promise<T>;
|
|
1221
1561
|
hasStarted(): boolean;
|
|
@@ -1456,12 +1796,24 @@ async function refreshCappedCodexUsageRows(
|
|
|
1456
1796
|
db: ActivityServices["db"],
|
|
1457
1797
|
settings: Settings,
|
|
1458
1798
|
workspaceId: string,
|
|
1459
|
-
accounts:
|
|
1799
|
+
accounts: Array<
|
|
1800
|
+
Pick<
|
|
1801
|
+
CodexAccountStatus,
|
|
1802
|
+
"id" | "status" | "primaryUsedPercent" | "secondaryUsedPercent" | "exhaustedUntil"
|
|
1803
|
+
>
|
|
1804
|
+
>,
|
|
1460
1805
|
capacitySignals: {
|
|
1461
1806
|
signalCodexCapacityWorkflow?: ActivityServices["signalCodexCapacityWorkflow"] | undefined;
|
|
1462
1807
|
wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
|
|
1463
1808
|
},
|
|
1464
|
-
): Promise<
|
|
1809
|
+
): Promise<
|
|
1810
|
+
Array<
|
|
1811
|
+
Pick<
|
|
1812
|
+
CodexAccountStatus,
|
|
1813
|
+
"id" | "status" | "primaryUsedPercent" | "secondaryUsedPercent" | "exhaustedUntil"
|
|
1814
|
+
>
|
|
1815
|
+
>
|
|
1816
|
+
> {
|
|
1465
1817
|
const stale = accounts.filter(
|
|
1466
1818
|
(a) =>
|
|
1467
1819
|
a.status === "active" &&
|
|
@@ -1560,6 +1912,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1560
1912
|
runtimeMetricsHooksForObservability(observability),
|
|
1561
1913
|
);
|
|
1562
1914
|
let isCodexTurn = false;
|
|
1915
|
+
let isExternallyBilledTurn = false;
|
|
1563
1916
|
let executionGeneration = 0;
|
|
1564
1917
|
// Still required by credential-loss/capacity settlements, whose own
|
|
1565
1918
|
// recovery transactions fence against worker-death redispatches.
|
|
@@ -1693,6 +2046,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1693
2046
|
// run. null when the flag is off (byte-for-byte the legacy build-and-discard
|
|
1694
2047
|
// path) OR when the backend is "none". Released + dropped in `finally`.
|
|
1695
2048
|
let resolvedSandbox: ResumedTurnSandbox | null = null;
|
|
2049
|
+
const requireResolvedSandboxForMutation = (message: string): ResumedTurnSandbox => {
|
|
2050
|
+
if (!resolvedSandbox) throw new Error(message);
|
|
2051
|
+
return resolvedSandbox;
|
|
2052
|
+
};
|
|
1696
2053
|
// The machine-primary SelfhostedSession (the UNWRAPPED backend, not the
|
|
1697
2054
|
// routing proxy): held so the turn's completion can final-ack this turn's
|
|
1698
2055
|
// settled op-stream ops AFTER the results are durably persisted.
|
|
@@ -1704,11 +2061,102 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1704
2061
|
// THIS handle so a mid-turn sandbox_swap can never re-route those execs onto a
|
|
1705
2062
|
// connected machine (the user's real computer).
|
|
1706
2063
|
let setupBoxSession: unknown = null;
|
|
2064
|
+
// A same-target API repair can replace the home provider while this turn is
|
|
2065
|
+
// alive. Keep setup/snapshot persistence on the rebound raw session while
|
|
2066
|
+
// preserving the SDK-owned routing proxy for eager turns. Lazy turns hold the
|
|
2067
|
+
// proxy separately, so their worker-side handle may replace its raw session.
|
|
2068
|
+
const onHomeSandboxRebound = (rebound: {
|
|
2069
|
+
established: EstablishedSandboxSession;
|
|
2070
|
+
leaseEpoch: number;
|
|
2071
|
+
}): void => {
|
|
2072
|
+
const current = resolvedSandbox;
|
|
2073
|
+
const previousSession = current?.established.session;
|
|
2074
|
+
const preserveRoutingProxy = current !== null && previousSession !== setupBoxSession;
|
|
2075
|
+
setupBoxSession = rebound.established.session;
|
|
2076
|
+
if (!current) return;
|
|
2077
|
+
current.leaseEpoch = rebound.leaseEpoch;
|
|
2078
|
+
current.established = preserveRoutingProxy
|
|
2079
|
+
? {
|
|
2080
|
+
...current.established,
|
|
2081
|
+
client: rebound.established.client,
|
|
2082
|
+
// Keep the stable SDK-facing proxy; only its resolver changes the
|
|
2083
|
+
// underlying backend. The worker's setupBoxSession above is raw.
|
|
2084
|
+
session: previousSession,
|
|
2085
|
+
sessionState: rebound.established.sessionState,
|
|
2086
|
+
instanceId: rebound.established.instanceId,
|
|
2087
|
+
backendId: rebound.established.backendId,
|
|
2088
|
+
...(rebound.established.origin ? { origin: rebound.established.origin } : {}),
|
|
2089
|
+
...(rebound.established.restoredArchive
|
|
2090
|
+
? { restoredArchive: rebound.established.restoredArchive }
|
|
2091
|
+
: {}),
|
|
2092
|
+
}
|
|
2093
|
+
: rebound.established;
|
|
2094
|
+
};
|
|
1707
2095
|
// The globally unique durable turn-attempt holder id + the group id,
|
|
1708
2096
|
// captured so the lease heartbeat can refresh the lease TTL epoch-fenced
|
|
1709
2097
|
// (a superseded owner self-evicts) and finally can release.
|
|
1710
2098
|
let sandboxHolderId: TurnSandboxLeaseHolderId | null = null;
|
|
1711
2099
|
let sandboxGroupId: string | null = null;
|
|
2100
|
+
const runWorkspaceMutationForSandbox = async <T>(
|
|
2101
|
+
sandbox: ResumedTurnSandbox,
|
|
2102
|
+
operation: string,
|
|
2103
|
+
mutation: () => Promise<T>,
|
|
2104
|
+
): Promise<T> => {
|
|
2105
|
+
// Connected machines are the user's own persistence and never dirty the
|
|
2106
|
+
// cloud home archive. Every persistable raw-session write batch is fenced
|
|
2107
|
+
// against the exact current lease/provider before the provider sees it.
|
|
2108
|
+
if (sandbox.established.backendId === "selfhosted") return await mutation();
|
|
2109
|
+
if (!sandboxGroupId || !sandboxHolderId || !turnId || executionGeneration <= 0) {
|
|
2110
|
+
throw new Error("Workspace mutation attempted before exact turn sandbox admission");
|
|
2111
|
+
}
|
|
2112
|
+
const identity = {
|
|
2113
|
+
accountId: input.accountId,
|
|
2114
|
+
workspaceId: input.workspaceId,
|
|
2115
|
+
sessionId: input.sessionId,
|
|
2116
|
+
turnId,
|
|
2117
|
+
executionGeneration,
|
|
2118
|
+
attemptId: input.attemptId,
|
|
2119
|
+
holderId: sandboxHolderId,
|
|
2120
|
+
sandboxGroupId,
|
|
2121
|
+
expectedEpoch: sandbox.leaseEpoch,
|
|
2122
|
+
expectedInstanceId: sandbox.established.instanceId,
|
|
2123
|
+
operation,
|
|
2124
|
+
};
|
|
2125
|
+
const admission = await advanceWorkspaceGeneration(db, identity);
|
|
2126
|
+
let result: T;
|
|
2127
|
+
try {
|
|
2128
|
+
result = await mutation();
|
|
2129
|
+
} catch (providerError) {
|
|
2130
|
+
try {
|
|
2131
|
+
await verifyWorkspaceMutationSettlement(db, {
|
|
2132
|
+
...identity,
|
|
2133
|
+
admission,
|
|
2134
|
+
outcome: "rejected",
|
|
2135
|
+
});
|
|
2136
|
+
} catch (settlementError) {
|
|
2137
|
+
throw new RoutingMutationOutcomeUnknownError(
|
|
2138
|
+
operation,
|
|
2139
|
+
`Platform workspace mutation "${operation}" rejected at the provider but lost its durable physical settlement; its outcome is unknown and it was not replayed`,
|
|
2140
|
+
{ cause: settlementError },
|
|
2141
|
+
);
|
|
2142
|
+
}
|
|
2143
|
+
throw providerError;
|
|
2144
|
+
}
|
|
2145
|
+
try {
|
|
2146
|
+
await verifyWorkspaceMutationSettlement(db, {
|
|
2147
|
+
...identity,
|
|
2148
|
+
admission,
|
|
2149
|
+
outcome: "resolved",
|
|
2150
|
+
});
|
|
2151
|
+
} catch (settlementError) {
|
|
2152
|
+
throw new RoutingMutationOutcomeUnknownError(
|
|
2153
|
+
operation,
|
|
2154
|
+
`Platform workspace mutation "${operation}" returned from the provider but lost its durable settlement fence; its outcome is unknown and it was not replayed`,
|
|
2155
|
+
{ cause: settlementError },
|
|
2156
|
+
);
|
|
2157
|
+
}
|
|
2158
|
+
return result;
|
|
2159
|
+
};
|
|
1712
2160
|
// Lease-TTL refresh timer (parallels the activity heartbeat): while the turn
|
|
1713
2161
|
// runs it refreshes expires_at epoch-fenced so a legit multi-day turn is
|
|
1714
2162
|
// never TTL-reaped. Cleared in finally. Only set when the flag resolved a box.
|
|
@@ -2138,10 +2586,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2138
2586
|
input.workspaceId,
|
|
2139
2587
|
settings,
|
|
2140
2588
|
);
|
|
2141
|
-
// Read the active-credential flag
|
|
2142
|
-
//
|
|
2143
|
-
//
|
|
2144
|
-
// provider-injection and billing disagree about whether this is a codex turn.
|
|
2589
|
+
// Read the active-credential flag once for the runtime capability overlay.
|
|
2590
|
+
// Accepted billing/provider identity comes from the turn policy below,
|
|
2591
|
+
// never from this mutable health snapshot.
|
|
2145
2592
|
const codexSubscriptionActive = await workspaceCodexSubscriptionActive(
|
|
2146
2593
|
db,
|
|
2147
2594
|
mcpSettings,
|
|
@@ -2169,6 +2616,38 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2169
2616
|
}
|
|
2170
2617
|
const turn = claim.turn;
|
|
2171
2618
|
turnId = turn.id;
|
|
2619
|
+
executionGeneration = turn.executionGeneration;
|
|
2620
|
+
const claimedPolicy = readTurnExecutionPolicyV1(turn.metadata);
|
|
2621
|
+
const policyForAbsent =
|
|
2622
|
+
claimedPolicy.kind === "valid"
|
|
2623
|
+
? claimedPolicy.policy
|
|
2624
|
+
: resolveTurnExecutionPolicyV1(capabilitySettings, legacyTurnExecutionPolicyInput(turn));
|
|
2625
|
+
const installedPolicy = await installOrReadTurnExecutionPolicyForAttempt(db, {
|
|
2626
|
+
accountId: input.accountId,
|
|
2627
|
+
workspaceId: input.workspaceId,
|
|
2628
|
+
sessionId: input.sessionId,
|
|
2629
|
+
turnId,
|
|
2630
|
+
executionGeneration,
|
|
2631
|
+
attemptId: input.attemptId,
|
|
2632
|
+
policyForAbsent,
|
|
2633
|
+
});
|
|
2634
|
+
if (!installedPolicy.accepted) {
|
|
2635
|
+
throw new TurnAttemptFencedError(
|
|
2636
|
+
`turn execution policy was fenced: ${installedPolicy.reason}`,
|
|
2637
|
+
);
|
|
2638
|
+
}
|
|
2639
|
+
const verifiedExecutionPolicy = assertTurnExecutionPolicyMatchesConfigV1(
|
|
2640
|
+
capabilitySettings,
|
|
2641
|
+
installedPolicy.policy,
|
|
2642
|
+
{
|
|
2643
|
+
modelId: turn.model,
|
|
2644
|
+
reasoningEffort: turn.reasoningEffort,
|
|
2645
|
+
},
|
|
2646
|
+
);
|
|
2647
|
+
const turnExecutionPolicy = verifiedExecutionPolicy.policy;
|
|
2648
|
+
const billingIdentity = turnExecutionPolicyBillingIdentity(turnExecutionPolicy);
|
|
2649
|
+
isExternallyBilledTurn = billingIdentity.externallyBilled;
|
|
2650
|
+
isCodexTurn = billingIdentity.codexSubscription;
|
|
2172
2651
|
triggerEventId = turn.triggerEventId;
|
|
2173
2652
|
const trigger = await getSessionEvent(db, input.workspaceId, triggerEventId);
|
|
2174
2653
|
if (!trigger) {
|
|
@@ -2181,7 +2660,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2181
2660
|
trigger,
|
|
2182
2661
|
);
|
|
2183
2662
|
triggerType = trigger.type;
|
|
2184
|
-
executionGeneration = turn.executionGeneration;
|
|
2185
2663
|
const latestTurnState = await getLatestRunState(db, input.workspaceId, input.sessionId);
|
|
2186
2664
|
const continuationCodexCredentialId =
|
|
2187
2665
|
latestTurnState?.turnId === turnId ? latestTurnState.frozenCodexCredentialId : null;
|
|
@@ -2190,21 +2668,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2190
2668
|
0,
|
|
2191
2669
|
);
|
|
2192
2670
|
turnLifecycleMetricsFor(observability).start(turnId);
|
|
2193
|
-
//
|
|
2194
|
-
//
|
|
2195
|
-
// gate + the usage recorder so a turn paid by the user's ChatGPT/Codex plan
|
|
2196
|
-
// consumes ZERO OpenGeni credits and never feeds an OpenGeni cap. Resolved
|
|
2197
|
-
// here (before resolvedModel at the routing step) because the pre-turn gate
|
|
2198
|
-
// below needs it; mirrors the same active-credential read the codex provider
|
|
2199
|
-
// overlay uses, so billing and routing agree on what "codex" is.
|
|
2200
|
-
isCodexTurn = await isCodexBilledTurn({
|
|
2201
|
-
db,
|
|
2202
|
-
settings,
|
|
2203
|
-
workspaceId: input.workspaceId,
|
|
2204
|
-
model: turn.model,
|
|
2205
|
-
active: codexSubscriptionActive,
|
|
2206
|
-
});
|
|
2207
|
-
// §7.5 P3 — pass BOTH the codex predicate (codex-plan turns bypass the gate)
|
|
2671
|
+
// §7.5 P3 — pass the accepted billing attribution (externally funded turns
|
|
2672
|
+
// bypass OpenGeni credit/token gates)
|
|
2208
2673
|
// AND the optional host `entitlements` port (when bound, its admitRun replaces
|
|
2209
2674
|
// the local credit read). Unset port → today's local-ledger path.
|
|
2210
2675
|
await waitForTurnOperation(
|
|
@@ -2213,7 +2678,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2213
2678
|
db,
|
|
2214
2679
|
input.accountId,
|
|
2215
2680
|
input.workspaceId,
|
|
2216
|
-
|
|
2681
|
+
isExternallyBilledTurn,
|
|
2217
2682
|
entitlements,
|
|
2218
2683
|
),
|
|
2219
2684
|
cancellationSignal,
|
|
@@ -2259,6 +2724,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2259
2724
|
// one exactly like production.
|
|
2260
2725
|
codexLeaseHolderId = dispatchId;
|
|
2261
2726
|
const modelUsageDispatchId = activityContext?.info.activityId ?? dispatchId;
|
|
2727
|
+
const claimedModelUsageSourceKeys = new Set<string>();
|
|
2262
2728
|
const emittedModelUsageSourceKeys = new Set<string>();
|
|
2263
2729
|
publish = async (
|
|
2264
2730
|
events: Array<Omit<AppendEventInput, "producerId" | "producerSeq" | "turnId">>,
|
|
@@ -2646,6 +3112,80 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2646
3112
|
codexLeaseConfirmedUntilMs !== null;
|
|
2647
3113
|
if (codexLeaseHeld) startCodexLeaseHeartbeat();
|
|
2648
3114
|
|
|
3115
|
+
const actualOutcome = effectiveCodexCredentialId
|
|
3116
|
+
? "selected"
|
|
3117
|
+
: rotationDecision.kind === "allCapped"
|
|
3118
|
+
? "waiting"
|
|
3119
|
+
: "none";
|
|
3120
|
+
const actualReason = effectiveCodexCredentialId
|
|
3121
|
+
? leased.reused
|
|
3122
|
+
? "lease_reused"
|
|
3123
|
+
: sessionPin === effectiveCodexCredentialId
|
|
3124
|
+
? "pin"
|
|
3125
|
+
: rotationDecision.kind === "active" && rotationDecision.moved
|
|
3126
|
+
? "rotation"
|
|
3127
|
+
: "active"
|
|
3128
|
+
: rotationDecision.kind === "allCapped"
|
|
3129
|
+
? "all_capped"
|
|
3130
|
+
: "none";
|
|
3131
|
+
const fencedInFlight =
|
|
3132
|
+
leased.reused ||
|
|
3133
|
+
(continuationCodexCredentialId !== null &&
|
|
3134
|
+
continuationCodexCredentialId === effectiveCodexCredentialId);
|
|
3135
|
+
const shadowResult = await publishCodexFleetShadowDecisionV1({
|
|
3136
|
+
enabled: settings.codexFleetPolicyShadowEnabled,
|
|
3137
|
+
decision: {
|
|
3138
|
+
accounts: leased.accounts,
|
|
3139
|
+
actualCredentialId: effectiveCodexCredentialId,
|
|
3140
|
+
actualOutcome,
|
|
3141
|
+
actualReason,
|
|
3142
|
+
affinityCredentialId: fencedInFlight
|
|
3143
|
+
? effectiveCodexCredentialId
|
|
3144
|
+
: (sessionPin ?? sessionCodex?.lastCredentialId ?? null),
|
|
3145
|
+
fencedInFlight,
|
|
3146
|
+
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
3147
|
+
now: new Date(),
|
|
3148
|
+
aliasSeed: randomUUID(),
|
|
3149
|
+
},
|
|
3150
|
+
publish,
|
|
3151
|
+
});
|
|
3152
|
+
if (shadowResult.outcome === "published") {
|
|
3153
|
+
const shadowPayload = shadowResult.payload;
|
|
3154
|
+
observability.incrementCounter({
|
|
3155
|
+
name: "opengeni_codex_fleet_shadow_decisions_total",
|
|
3156
|
+
help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
|
|
3157
|
+
labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload),
|
|
3158
|
+
});
|
|
3159
|
+
observability.info("Codex adaptive fleet shadow decision", {
|
|
3160
|
+
workspaceId: input.workspaceId,
|
|
3161
|
+
policyVersion: shadowPayload.replay.policyVersion,
|
|
3162
|
+
inputFingerprint: shadowPayload.replay.inputFingerprint,
|
|
3163
|
+
decisionFingerprint: shadowPayload.replay.decisionFingerprint,
|
|
3164
|
+
actualOutcome: shadowPayload.actual.outcome,
|
|
3165
|
+
shadowOutcome: shadowPayload.replay.decision.outcome,
|
|
3166
|
+
comparison: shadowPayload.comparison,
|
|
3167
|
+
candidateCount: shadowPayload.replay.input.candidates.length,
|
|
3168
|
+
truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
|
|
3169
|
+
payloadBytes: shadowResult.payloadBytes,
|
|
3170
|
+
});
|
|
3171
|
+
} else if (shadowResult.outcome === "failed") {
|
|
3172
|
+
// Shadow observability is explicitly non-authoritative. A malformed
|
|
3173
|
+
// snapshot or event-write fault must never change the authoritative lease,
|
|
3174
|
+
// capacity wait, failover, or the account serving this fenced turn.
|
|
3175
|
+
observability.incrementCounter({
|
|
3176
|
+
name: "opengeni_codex_fleet_shadow_errors_total",
|
|
3177
|
+
help: "Shadow decision build/publication failures.",
|
|
3178
|
+
labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
|
|
3179
|
+
});
|
|
3180
|
+
observability.warn("Codex adaptive fleet shadow decision failed open", {
|
|
3181
|
+
workspaceId: input.workspaceId,
|
|
3182
|
+
stage: shadowResult.stage,
|
|
3183
|
+
reason: shadowResult.reason,
|
|
3184
|
+
errorName: shadowResult.errorName,
|
|
3185
|
+
payloadBytes: shadowResult.payloadBytes,
|
|
3186
|
+
});
|
|
3187
|
+
}
|
|
3188
|
+
|
|
2649
3189
|
const eligibleCount = leased.accounts.filter((account) =>
|
|
2650
3190
|
isCodexCredentialEligible(account, new Date()),
|
|
2651
3191
|
).length;
|
|
@@ -3016,6 +3556,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3016
3556
|
db,
|
|
3017
3557
|
input.workspaceId,
|
|
3018
3558
|
input.sessionId,
|
|
3559
|
+
input.attemptId,
|
|
3019
3560
|
baseRunSettings,
|
|
3020
3561
|
);
|
|
3021
3562
|
// Multi-provider per-turn routing → the provider gating (compaction mode,
|
|
@@ -3031,7 +3572,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3031
3572
|
// a chat-only Fireworks model. Resolving against the default-model settings
|
|
3032
3573
|
// keeps gating consistent with the router. Cost accounting covers registry
|
|
3033
3574
|
// models via configuredModelPricing.
|
|
3034
|
-
const resolvedModel = runtime.resolveTurnModel(
|
|
3575
|
+
const resolvedModel = runtime.resolveTurnModel(
|
|
3576
|
+
capabilitySettings,
|
|
3577
|
+
turnExecutionPolicy.productModelId,
|
|
3578
|
+
);
|
|
3035
3579
|
// Bind the provider/model catalog's context policy to every model-facing
|
|
3036
3580
|
// path for this turn. In particular, Codex subscription turns must not
|
|
3037
3581
|
// inherit the deployment's OpenAI/Azure mode or 1.05M context defaults:
|
|
@@ -3046,25 +3590,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3046
3590
|
// and the main run both come later in this scope), so a blocked
|
|
3047
3591
|
// provider/model can never be reached through ANY stamp path: explicit
|
|
3048
3592
|
// turn model, inherited session default, goal-continuation inheritance,
|
|
3049
|
-
// or the legacy null-resolution fallback
|
|
3050
|
-
//
|
|
3051
|
-
//
|
|
3052
|
-
// how bare-model turns silently spent real Azure money in a
|
|
3053
|
-
// codex-intended workspace). Fail-loud, never a silent remap.
|
|
3593
|
+
// or the legacy null-resolution fallback. The frozen execution policy is
|
|
3594
|
+
// the attribution source even if an injected test runtime returns no
|
|
3595
|
+
// concrete resolved model. Fail-loud, never a silent remap.
|
|
3054
3596
|
{
|
|
3055
3597
|
const workspaceModelPolicy = await getWorkspaceModelPolicy(db, input.workspaceId);
|
|
3056
3598
|
if (workspaceModelPolicy) {
|
|
3057
|
-
const effectiveProviderId = resolvedModel
|
|
3058
|
-
? resolvedModel.provider.id
|
|
3059
|
-
: builtinProviderId(capabilitySettings);
|
|
3060
3599
|
const verdict = evaluateWorkspaceModelPolicy(workspaceModelPolicy, {
|
|
3061
|
-
providerId:
|
|
3062
|
-
modelId:
|
|
3600
|
+
providerId: turnExecutionPolicy.providerId,
|
|
3601
|
+
modelId: turnExecutionPolicy.productModelId,
|
|
3063
3602
|
});
|
|
3064
3603
|
if (!verdict.allowed) {
|
|
3065
3604
|
throw new WorkspaceModelPolicyBlockedError(
|
|
3066
|
-
|
|
3067
|
-
|
|
3605
|
+
turnExecutionPolicy.productModelId,
|
|
3606
|
+
turnExecutionPolicy.providerId,
|
|
3068
3607
|
verdict.reason,
|
|
3069
3608
|
);
|
|
3070
3609
|
}
|
|
@@ -3078,6 +3617,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3078
3617
|
// Build it once and wrap BOTH the compaction summarizer (a separate model
|
|
3079
3618
|
// call on the same codex client) and the main run; otherwise the summarizer
|
|
3080
3619
|
// would hit the codex backend unauthenticated.
|
|
3620
|
+
let codexModelRequestSequence = 0;
|
|
3081
3621
|
const codexContext: CodexRequestContext | null =
|
|
3082
3622
|
resolvedModel?.provider.kind === "codex-subscription"
|
|
3083
3623
|
? ((): CodexRequestContext => {
|
|
@@ -3108,6 +3648,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3108
3648
|
onUsageHeaders: (snapshot) => {
|
|
3109
3649
|
latestCodexUsage = snapshot;
|
|
3110
3650
|
}, // latest wins; flushed once in finally
|
|
3651
|
+
nextRequestId: () => `${dispatchId}:${++codexModelRequestSequence}`,
|
|
3652
|
+
onModelRequestEvent: async (event) => {
|
|
3653
|
+
if (!publish || !turnId) {
|
|
3654
|
+
throw new Error("Codex model request started before the turn event producer");
|
|
3655
|
+
}
|
|
3656
|
+
await publish([
|
|
3657
|
+
{
|
|
3658
|
+
type: "agent.model.request",
|
|
3659
|
+
payload: {
|
|
3660
|
+
...event,
|
|
3661
|
+
provider: "codex-subscription",
|
|
3662
|
+
turnId,
|
|
3663
|
+
attemptId: input.attemptId,
|
|
3664
|
+
dispatchId,
|
|
3665
|
+
executionGeneration,
|
|
3666
|
+
},
|
|
3667
|
+
},
|
|
3668
|
+
]);
|
|
3669
|
+
},
|
|
3111
3670
|
};
|
|
3112
3671
|
})()
|
|
3113
3672
|
: null;
|
|
@@ -3116,53 +3675,33 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3116
3675
|
const promptCacheKey = acceptsPromptCacheKeyForTurn(resolvedModel)
|
|
3117
3676
|
? input.sessionId
|
|
3118
3677
|
: undefined;
|
|
3119
|
-
|
|
3678
|
+
const compactionUsageState = createCompactionModelUsageEventState(
|
|
3679
|
+
claimedModelUsageSourceKeys,
|
|
3680
|
+
);
|
|
3120
3681
|
const recordCompactionUsage = async (usage: ModelResponseUsage) => {
|
|
3121
|
-
await
|
|
3682
|
+
await processCompactionModelUsageEvent({
|
|
3683
|
+
usage,
|
|
3684
|
+
state: compactionUsageState,
|
|
3685
|
+
dispatchId: modelUsageDispatchId,
|
|
3686
|
+
settings,
|
|
3687
|
+
db,
|
|
3688
|
+
observability,
|
|
3689
|
+
publish,
|
|
3690
|
+
accountId: input.accountId,
|
|
3691
|
+
workspaceId: input.workspaceId,
|
|
3692
|
+
sessionId: input.sessionId,
|
|
3693
|
+
turnId: turn.id,
|
|
3694
|
+
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
3695
|
+
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
3696
|
+
model: resolvedModel?.configured.id ?? turn.model,
|
|
3697
|
+
externallyBilled: isExternallyBilledTurn,
|
|
3698
|
+
turnAttemptId: input.attemptId,
|
|
3699
|
+
servingCredentialId: effectiveCodexCredentialId,
|
|
3700
|
+
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
3701
|
+
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
3122
3702
|
renewLease: () => renewCodexLease("model_usage"),
|
|
3123
3703
|
leaseLost: () => codexLeaseLost,
|
|
3124
3704
|
leaseLostMessage: "Codex credential lease expired during context compaction",
|
|
3125
|
-
recordUsage: async () => {
|
|
3126
|
-
compactionUsageCount += 1;
|
|
3127
|
-
const sourceKey = modelUsageSourceKey({
|
|
3128
|
-
responseId: usage.responseId,
|
|
3129
|
-
dispatchId: modelUsageDispatchId,
|
|
3130
|
-
positionalKey: `compaction-${compactionUsageCount}`,
|
|
3131
|
-
});
|
|
3132
|
-
const responseAccountCtx = modelCallAccountContext({
|
|
3133
|
-
servingCredentialId: effectiveCodexCredentialId,
|
|
3134
|
-
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
3135
|
-
isFirstCallOfTurn: compactionUsageCount === 1,
|
|
3136
|
-
});
|
|
3137
|
-
await recordModelUsageAndDebitCredits(settings, db, {
|
|
3138
|
-
accountId: input.accountId,
|
|
3139
|
-
workspaceId: input.workspaceId,
|
|
3140
|
-
sessionId: input.sessionId,
|
|
3141
|
-
turnId: turn.id,
|
|
3142
|
-
turnAttemptId: input.attemptId,
|
|
3143
|
-
model: resolvedModel?.configured.id ?? turn.model,
|
|
3144
|
-
isCodexTurn,
|
|
3145
|
-
usage: usage.usage,
|
|
3146
|
-
sourceKey,
|
|
3147
|
-
observability,
|
|
3148
|
-
});
|
|
3149
|
-
await emitModelCallUsage({
|
|
3150
|
-
observability,
|
|
3151
|
-
publish,
|
|
3152
|
-
accountId: input.accountId,
|
|
3153
|
-
workspaceId: input.workspaceId,
|
|
3154
|
-
sessionId: input.sessionId,
|
|
3155
|
-
turnId: turn.id,
|
|
3156
|
-
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
3157
|
-
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
3158
|
-
model: resolvedModel?.configured.id ?? turn.model,
|
|
3159
|
-
sourceKey,
|
|
3160
|
-
usage,
|
|
3161
|
-
servingAccountHash: responseAccountCtx.servingAccountHash,
|
|
3162
|
-
accountChangedFromPrevCall: responseAccountCtx.accountChangedFromPrevCall,
|
|
3163
|
-
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
3164
|
-
});
|
|
3165
|
-
},
|
|
3166
3705
|
});
|
|
3167
3706
|
};
|
|
3168
3707
|
const compactionSummarizerFor = (systemInstructions?: string) =>
|
|
@@ -3172,7 +3711,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3172
3711
|
summarizeForCompaction(s, m, {
|
|
3173
3712
|
client: resolvedModel.client,
|
|
3174
3713
|
api: resolvedModel.provider.api,
|
|
3175
|
-
model:
|
|
3714
|
+
model: turnExecutionPolicy.upstreamModelId,
|
|
3176
3715
|
maxOutputTokens: SUMMARY_BUFFER_TOKENS,
|
|
3177
3716
|
onUsage: recordCompactionUsage,
|
|
3178
3717
|
...(systemInstructions ? { systemInstructions } : {}),
|
|
@@ -3181,6 +3720,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3181
3720
|
)
|
|
3182
3721
|
: (s: Settings, m: Array<Record<string, unknown>>) =>
|
|
3183
3722
|
summarizeForCompaction(s, m, {
|
|
3723
|
+
model: turnExecutionPolicy.upstreamModelId,
|
|
3184
3724
|
maxOutputTokens: SUMMARY_BUFFER_TOKENS,
|
|
3185
3725
|
onUsage: recordCompactionUsage,
|
|
3186
3726
|
...(systemInstructions ? { systemInstructions } : {}),
|
|
@@ -3321,9 +3861,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3321
3861
|
// connector scopes); no-op for every other turn. Its refreshing bearer is
|
|
3322
3862
|
// resolved at connect time from the codex ALS (see the withCodex-wrapped
|
|
3323
3863
|
// prepareTools call below).
|
|
3864
|
+
// Resolve the durable policy at the turn boundary. Workspace-default
|
|
3865
|
+
// sessions may discover newly enabled capability MCPs, while explicit,
|
|
3866
|
+
// inherited-fixed, and legacy sessions remain narrowed to their stored
|
|
3867
|
+
// materialized allow-list. `withCodexAppsTool` below keeps the existing
|
|
3868
|
+
// single Codex lazy router/connector overlay; it is not a second policy
|
|
3869
|
+
// or discovery path.
|
|
3870
|
+
const effectivePolicyTools = resolveSessionToolPolicy({
|
|
3871
|
+
...(session.toolPolicy ? { toolPolicy: session.toolPolicy } : {}),
|
|
3872
|
+
sessionTools: session.tools,
|
|
3873
|
+
turnTools: turn.tools,
|
|
3874
|
+
...(turn.toolsProvided !== undefined ? { turnToolsProvided: turn.toolsProvided } : {}),
|
|
3875
|
+
availableMcpServerIds: runSettings.mcpServers.map((server) => server.id),
|
|
3876
|
+
defaultMcpServerIds: enabledCapabilityMcpToolRefs(settings, mcpSettings).map(
|
|
3877
|
+
(tool) => tool.id,
|
|
3878
|
+
),
|
|
3879
|
+
}).toolRefs;
|
|
3324
3880
|
const turnTools = withCodexAppsTool(
|
|
3325
3881
|
runSettings,
|
|
3326
|
-
withFirstPartyTools(runSettings,
|
|
3882
|
+
withFirstPartyTools(runSettings, effectivePolicyTools),
|
|
3327
3883
|
);
|
|
3328
3884
|
// §7.6 connection-credential provider — load (and decrypt) the variable set via the host
|
|
3329
3885
|
// `sandboxSecrets` provider when bound; unset → today's local decrypt.
|
|
@@ -3553,6 +4109,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3553
4109
|
turn,
|
|
3554
4110
|
})
|
|
3555
4111
|
: undefined;
|
|
4112
|
+
const toolspaceAuthority = {
|
|
4113
|
+
sessionId: input.sessionId,
|
|
4114
|
+
};
|
|
3556
4115
|
const {
|
|
3557
4116
|
environment: sandboxEnvironment,
|
|
3558
4117
|
gitToken: sandboxGitToken,
|
|
@@ -3576,8 +4135,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3576
4135
|
...(gitCredentialAuthority ? { authority: gitCredentialAuthority } : {}),
|
|
3577
4136
|
gitCredentials: connectionCredentials?.gitCredentials,
|
|
3578
4137
|
authorizeGitHubTokenMint,
|
|
3579
|
-
|
|
3580
|
-
runId: turnId,
|
|
4138
|
+
toolspaceAuthority,
|
|
3581
4139
|
},
|
|
3582
4140
|
),
|
|
3583
4141
|
cancellationSignal,
|
|
@@ -3598,6 +4156,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3598
4156
|
const attachGitCredentialRenewal = async (
|
|
3599
4157
|
tokenSession: GitCredentialTokenWriterSession,
|
|
3600
4158
|
initial: MintedRunGitCredentials | undefined,
|
|
4159
|
+
initialSandbox?: ResumedTurnSandbox,
|
|
3601
4160
|
): Promise<void> => {
|
|
3602
4161
|
if (!initial || initial.bindings.length === 0) return;
|
|
3603
4162
|
const previous = gitCredentialRenewals;
|
|
@@ -3625,6 +4184,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3625
4184
|
authorizeGitHubTokenMint,
|
|
3626
4185
|
},
|
|
3627
4186
|
);
|
|
4187
|
+
if (binding) {
|
|
4188
|
+
assertGitCredentialRenewalTransportUnchanged(initialBinding, binding);
|
|
4189
|
+
}
|
|
3628
4190
|
pendingBinding = binding;
|
|
3629
4191
|
return binding
|
|
3630
4192
|
? {
|
|
@@ -3639,16 +4201,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3639
4201
|
throw new Error("credential renewal produced no binding token");
|
|
3640
4202
|
}
|
|
3641
4203
|
const runAs = sandboxRunAs(runSettings);
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
4204
|
+
const targetSandbox = resolvedSandbox ?? initialSandbox;
|
|
4205
|
+
if (!targetSandbox) {
|
|
4206
|
+
throw new Error("Git credential renewal has no exact sandbox lease target");
|
|
4207
|
+
}
|
|
4208
|
+
await runWorkspaceMutationForSandbox(
|
|
4209
|
+
targetSandbox,
|
|
4210
|
+
"gitCredentialRenewal",
|
|
4211
|
+
async () =>
|
|
4212
|
+
await refreshGitCredentialBindingTokenFiles(tokenSession, [pendingBinding!], {
|
|
4213
|
+
...(runAs ? { runAs } : {}),
|
|
4214
|
+
...(toolCancellationFenceRef.current
|
|
4215
|
+
? {
|
|
4216
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
4217
|
+
toolCancellationFenceRef.current,
|
|
4218
|
+
),
|
|
4219
|
+
}
|
|
4220
|
+
: {}),
|
|
4221
|
+
}),
|
|
4222
|
+
);
|
|
3652
4223
|
},
|
|
3653
4224
|
onSuccess: ({ providers: renewedProviders }) => {
|
|
3654
4225
|
for (const provider of renewedProviders) {
|
|
@@ -3687,6 +4258,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3687
4258
|
const attachToolspaceTokenRenewal = async (
|
|
3688
4259
|
tokenSession: ToolspaceTokenWriterSession,
|
|
3689
4260
|
initialExpiresAt = sandboxToolspaceTokenExpiresAt,
|
|
4261
|
+
initialSandbox?: ResumedTurnSandbox,
|
|
3690
4262
|
): Promise<void> => {
|
|
3691
4263
|
if (!sandboxToolspaceToken || !initialExpiresAt) return;
|
|
3692
4264
|
const previous = toolspaceTokenRenewal;
|
|
@@ -3695,25 +4267,34 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3695
4267
|
if (toolspaceTokenRenewalClosed) return;
|
|
3696
4268
|
|
|
3697
4269
|
const mint = async () =>
|
|
3698
|
-
await mintSandboxToolspaceToken(runSettings, connectionScope,
|
|
4270
|
+
await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority);
|
|
3699
4271
|
const write = async (material: NonNullable<Awaited<ReturnType<typeof mint>>>) => {
|
|
3700
4272
|
const runAs = sandboxRunAs(runSettings);
|
|
3701
|
-
|
|
3702
|
-
|
|
3703
|
-
|
|
3704
|
-
|
|
3705
|
-
|
|
3706
|
-
|
|
3707
|
-
|
|
3708
|
-
|
|
3709
|
-
|
|
3710
|
-
|
|
3711
|
-
|
|
3712
|
-
|
|
3713
|
-
|
|
3714
|
-
|
|
3715
|
-
|
|
3716
|
-
|
|
4273
|
+
const targetSandbox = resolvedSandbox ?? initialSandbox;
|
|
4274
|
+
if (!targetSandbox) {
|
|
4275
|
+
throw new Error("Toolspace token renewal has no exact sandbox lease target");
|
|
4276
|
+
}
|
|
4277
|
+
await runWorkspaceMutationForSandbox(
|
|
4278
|
+
targetSandbox,
|
|
4279
|
+
"toolspaceTokenRenewal",
|
|
4280
|
+
async () =>
|
|
4281
|
+
await refreshToolspaceTokenFile(tokenSession, material.token, {
|
|
4282
|
+
...(runAs ? { runAs } : {}),
|
|
4283
|
+
...(sandboxToolspaceTokenFile
|
|
4284
|
+
? {
|
|
4285
|
+
tokenFile: sandboxToolspaceTokenFile,
|
|
4286
|
+
legacyTokenFile: sandboxEnvironment.OPENGENI_TOOLSPACE_TOKEN_FILE!,
|
|
4287
|
+
}
|
|
4288
|
+
: {}),
|
|
4289
|
+
...(toolCancellationFenceRef.current
|
|
4290
|
+
? {
|
|
4291
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
4292
|
+
toolCancellationFenceRef.current,
|
|
4293
|
+
),
|
|
4294
|
+
}
|
|
4295
|
+
: {}),
|
|
4296
|
+
}),
|
|
4297
|
+
);
|
|
3717
4298
|
};
|
|
3718
4299
|
let renewalExpiresAt = initialExpiresAt;
|
|
3719
4300
|
if (renewalExpiresAt.getTime() <= Date.now() + TOOLSPACE_TOKEN_EXPIRY_LEAD_MS) {
|
|
@@ -3758,6 +4339,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3758
4339
|
|
|
3759
4340
|
const attachRunCredentialRenewal = async (
|
|
3760
4341
|
credentialSession: RunCredentialCommandSession,
|
|
4342
|
+
initialSandbox?: ResumedTurnSandbox,
|
|
3761
4343
|
): Promise<void> => {
|
|
3762
4344
|
if (!runCredentialResolver) return;
|
|
3763
4345
|
const previous = runCredentialRenewal;
|
|
@@ -3766,15 +4348,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3766
4348
|
if (runCredentialRenewalClosed) return;
|
|
3767
4349
|
runCredentialSession = credentialSession;
|
|
3768
4350
|
|
|
4351
|
+
const requireTargetSandbox = (): ResumedTurnSandbox => {
|
|
4352
|
+
const targetSandbox = resolvedSandbox ?? initialSandbox;
|
|
4353
|
+
if (!targetSandbox) {
|
|
4354
|
+
throw new Error("Run credential mutation has no exact sandbox lease target");
|
|
4355
|
+
}
|
|
4356
|
+
return targetSandbox;
|
|
4357
|
+
};
|
|
4358
|
+
|
|
3769
4359
|
if (!initialRunCredentialMaterial) {
|
|
3770
|
-
await
|
|
3771
|
-
|
|
3772
|
-
|
|
3773
|
-
|
|
3774
|
-
|
|
3775
|
-
|
|
3776
|
-
|
|
3777
|
-
|
|
4360
|
+
await runWorkspaceMutationForSandbox(
|
|
4361
|
+
requireTargetSandbox(),
|
|
4362
|
+
"runCredentialClear",
|
|
4363
|
+
async () =>
|
|
4364
|
+
await clearRunCredentials(
|
|
4365
|
+
credentialSession,
|
|
4366
|
+
input.sessionId,
|
|
4367
|
+
toolCancellationFenceRef.current
|
|
4368
|
+
? toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
4369
|
+
toolCancellationFenceRef.current,
|
|
4370
|
+
)
|
|
4371
|
+
: undefined,
|
|
4372
|
+
),
|
|
3778
4373
|
);
|
|
3779
4374
|
return;
|
|
3780
4375
|
}
|
|
@@ -3784,33 +4379,43 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3784
4379
|
pruneOtherAttempts = false,
|
|
3785
4380
|
): Promise<void> => {
|
|
3786
4381
|
if (!material) {
|
|
3787
|
-
await
|
|
3788
|
-
|
|
3789
|
-
|
|
3790
|
-
|
|
3791
|
-
|
|
4382
|
+
await runWorkspaceMutationForSandbox(
|
|
4383
|
+
requireTargetSandbox(),
|
|
4384
|
+
"runCredentialAttemptClear",
|
|
4385
|
+
async () =>
|
|
4386
|
+
await clearRunCredentialsForAttempt(credentialSession, {
|
|
4387
|
+
sessionId: input.sessionId,
|
|
4388
|
+
attemptId: input.attemptId,
|
|
4389
|
+
executionGeneration,
|
|
4390
|
+
}),
|
|
4391
|
+
);
|
|
3792
4392
|
return;
|
|
3793
4393
|
}
|
|
3794
4394
|
registerSecretRedactions(material.redactions);
|
|
3795
|
-
await
|
|
3796
|
-
|
|
3797
|
-
|
|
3798
|
-
|
|
3799
|
-
|
|
3800
|
-
|
|
3801
|
-
|
|
3802
|
-
|
|
3803
|
-
|
|
3804
|
-
|
|
3805
|
-
|
|
3806
|
-
|
|
3807
|
-
|
|
3808
|
-
|
|
3809
|
-
|
|
3810
|
-
|
|
3811
|
-
|
|
3812
|
-
|
|
3813
|
-
|
|
4395
|
+
await runWorkspaceMutationForSandbox(
|
|
4396
|
+
requireTargetSandbox(),
|
|
4397
|
+
"runCredentialMaterialization",
|
|
4398
|
+
async () =>
|
|
4399
|
+
await materializeRunCredentials(credentialSession, material, {
|
|
4400
|
+
sessionId: input.sessionId,
|
|
4401
|
+
attemptId: input.attemptId,
|
|
4402
|
+
executionGeneration,
|
|
4403
|
+
...(pruneOtherAttempts ? { pruneOtherAttempts: true } : {}),
|
|
4404
|
+
...(!pruneOtherAttempts ? { pruneSupersededGenerations: true } : {}),
|
|
4405
|
+
...(material.authNeeded.length > 0 &&
|
|
4406
|
+
Object.keys(material.environment).length === 0 &&
|
|
4407
|
+
material.files.length === 0
|
|
4408
|
+
? { prunePreviousGenerations: true }
|
|
4409
|
+
: {}),
|
|
4410
|
+
...(toolCancellationFenceRef.current
|
|
4411
|
+
? {
|
|
4412
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
4413
|
+
toolCancellationFenceRef.current,
|
|
4414
|
+
),
|
|
4415
|
+
}
|
|
4416
|
+
: {}),
|
|
4417
|
+
}),
|
|
4418
|
+
);
|
|
3814
4419
|
for (const payload of runCredentialAuthNeededPayloads(material)) {
|
|
3815
4420
|
const key = JSON.stringify(payload);
|
|
3816
4421
|
if (publishedRunCredentialNotices.has(key)) continue;
|
|
@@ -3956,11 +4561,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3956
4561
|
settings,
|
|
3957
4562
|
bus,
|
|
3958
4563
|
onOp: machineOpObserver.observer,
|
|
4564
|
+
onHomeSandboxRebound,
|
|
3959
4565
|
},
|
|
3960
4566
|
{
|
|
3961
4567
|
workspaceId: input.workspaceId,
|
|
3962
4568
|
sessionId: input.sessionId,
|
|
3963
4569
|
environment: sandboxEnvironment,
|
|
4570
|
+
workspaceMutationFence: {
|
|
4571
|
+
accountId: input.accountId,
|
|
4572
|
+
turnId: turn.id,
|
|
4573
|
+
executionGeneration,
|
|
4574
|
+
attemptId: input.attemptId,
|
|
4575
|
+
},
|
|
3964
4576
|
pinnedSelfhosted: {
|
|
3965
4577
|
sandboxId: activeSandboxPointer!.activeSandboxId!,
|
|
3966
4578
|
epoch: activeSandboxPointer!.activeEpoch,
|
|
@@ -4045,24 +4657,41 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4045
4657
|
// the flag off the established group box is injected unchanged (today's
|
|
4046
4658
|
// path). The lease still owns the group box lifecycle — the proxy is a
|
|
4047
4659
|
// routing veneer, not an owner.
|
|
4048
|
-
|
|
4049
|
-
resolvedSandbox
|
|
4050
|
-
|
|
4051
|
-
|
|
4052
|
-
|
|
4053
|
-
|
|
4054
|
-
|
|
4055
|
-
|
|
4056
|
-
|
|
4057
|
-
|
|
4058
|
-
|
|
4059
|
-
|
|
4060
|
-
|
|
4660
|
+
resolvedSandbox = {
|
|
4661
|
+
...resolvedSandbox,
|
|
4662
|
+
established: wrapTurnBoxWithRouting(
|
|
4663
|
+
{
|
|
4664
|
+
db,
|
|
4665
|
+
settings,
|
|
4666
|
+
bus,
|
|
4667
|
+
onHomeSandboxLost: publishSandboxLost,
|
|
4668
|
+
onHomeSandboxRebound,
|
|
4669
|
+
},
|
|
4670
|
+
// Thread the SAME declared environment the group box was created with
|
|
4671
|
+
// (resumeBoxForTurn, above) so a selfhosted swap target's manifest
|
|
4672
|
+
// carries it too — the SDK's per-turn manifest-env delta stays empty
|
|
4673
|
+
// (no "cannot change manifest environment variables" throw).
|
|
4674
|
+
{
|
|
4675
|
+
workspaceId: input.workspaceId,
|
|
4676
|
+
sessionId: input.sessionId,
|
|
4677
|
+
environment: sandboxEnvironment,
|
|
4678
|
+
workspaceMutationFence: {
|
|
4679
|
+
accountId: input.accountId,
|
|
4680
|
+
turnId: turn.id,
|
|
4681
|
+
executionGeneration,
|
|
4682
|
+
attemptId: input.attemptId,
|
|
4061
4683
|
},
|
|
4062
|
-
|
|
4063
|
-
|
|
4064
|
-
|
|
4065
|
-
|
|
4684
|
+
homeLease: {
|
|
4685
|
+
accountId: input.accountId,
|
|
4686
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
4687
|
+
leaseEpoch: resolvedSandbox.leaseEpoch,
|
|
4688
|
+
instanceId: resolvedSandbox.established.instanceId,
|
|
4689
|
+
backend: groupBoxBackend,
|
|
4690
|
+
},
|
|
4691
|
+
},
|
|
4692
|
+
resolvedSandbox.established,
|
|
4693
|
+
),
|
|
4694
|
+
};
|
|
4066
4695
|
}
|
|
4067
4696
|
if (resolvedSandbox) {
|
|
4068
4697
|
startLeaseHeartbeat(resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
|
|
@@ -4197,9 +4826,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4197
4826
|
// token skip does NOT transfer: that token is inert on a connected
|
|
4198
4827
|
// machine (it uses its own git creds), but the toolspace token is the
|
|
4199
4828
|
// machine's ONLY path to programmatic tool calling and grants no more
|
|
4200
|
-
// than toolspace:call for its own session (own-session-bound,
|
|
4201
|
-
// budgeted, approval-tools
|
|
4202
|
-
//
|
|
4829
|
+
// than toolspace:call for its own session (own-session-bound,
|
|
4830
|
+
// short-lived and renewable, per-active-turn budgeted, approval-tools
|
|
4831
|
+
// excluded). The runtime seeds it to the box's token file over the same
|
|
4832
|
+
// exec channel, off-manifest, on every backend.
|
|
4203
4833
|
...(sandboxToolspaceToken
|
|
4204
4834
|
? {
|
|
4205
4835
|
toolspaceTokenSeed: sandboxToolspaceToken,
|
|
@@ -4342,12 +4972,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4342
4972
|
});
|
|
4343
4973
|
const lazyGitTokens = lazyGitCredentials?.gitTokens;
|
|
4344
4974
|
const lazyToolspaceToken = sandboxToolspaceToken
|
|
4345
|
-
? await mintSandboxToolspaceToken(
|
|
4346
|
-
runSettings,
|
|
4347
|
-
connectionScope,
|
|
4348
|
-
input.sessionId,
|
|
4349
|
-
turn.id,
|
|
4350
|
-
)
|
|
4975
|
+
? await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority)
|
|
4351
4976
|
: undefined;
|
|
4352
4977
|
const provisioned = await resumeBoxForTurn(
|
|
4353
4978
|
{
|
|
@@ -4376,6 +5001,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4376
5001
|
await publishSandboxLifecycleEvents(provisioned);
|
|
4377
5002
|
await attachRunCredentialRenewal(
|
|
4378
5003
|
provisioned.established.session as RunCredentialCommandSession,
|
|
5004
|
+
provisioned,
|
|
4379
5005
|
);
|
|
4380
5006
|
const provisionedSetupSession = initialRunCredentialMaterial
|
|
4381
5007
|
? withRunCredentialsSession(
|
|
@@ -4383,39 +5009,46 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4383
5009
|
input.sessionId,
|
|
4384
5010
|
)
|
|
4385
5011
|
: provisioned.established.session;
|
|
4386
|
-
await
|
|
4387
|
-
|
|
4388
|
-
|
|
4389
|
-
|
|
4390
|
-
|
|
4391
|
-
|
|
4392
|
-
|
|
4393
|
-
|
|
4394
|
-
|
|
4395
|
-
|
|
4396
|
-
|
|
4397
|
-
|
|
4398
|
-
|
|
4399
|
-
|
|
4400
|
-
|
|
4401
|
-
|
|
4402
|
-
|
|
4403
|
-
|
|
4404
|
-
|
|
4405
|
-
|
|
4406
|
-
|
|
4407
|
-
|
|
4408
|
-
|
|
4409
|
-
|
|
4410
|
-
|
|
5012
|
+
await runWorkspaceMutationForSandbox(
|
|
5013
|
+
provisioned,
|
|
5014
|
+
"lazyOwnedSandboxSetup",
|
|
5015
|
+
async () =>
|
|
5016
|
+
await runOwnedSandboxSetup(
|
|
5017
|
+
agent,
|
|
5018
|
+
provisioned.established.session as never,
|
|
5019
|
+
provisionedSetupSession as never,
|
|
5020
|
+
{
|
|
5021
|
+
settings: runSettings,
|
|
5022
|
+
environment: sandboxEnvironment,
|
|
5023
|
+
onRuntimeEvent: async (event) => {
|
|
5024
|
+
await publish?.([{ type: event.type, payload: event.payload }], true);
|
|
5025
|
+
},
|
|
5026
|
+
...(lazyGitTokens ? { gitTokenSeedsOverride: lazyGitTokens } : {}),
|
|
5027
|
+
...(lazyGitCredentials?.bindings
|
|
5028
|
+
? { gitCredentialBindingsOverride: lazyGitCredentials.bindings }
|
|
5029
|
+
: {}),
|
|
5030
|
+
...(lazyToolspaceToken
|
|
5031
|
+
? { toolspaceTokenSeedOverride: lazyToolspaceToken.token }
|
|
5032
|
+
: {}),
|
|
5033
|
+
...(toolCancellationFenceRef.current
|
|
5034
|
+
? {
|
|
5035
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
5036
|
+
toolCancellationFenceRef.current,
|
|
5037
|
+
),
|
|
5038
|
+
}
|
|
5039
|
+
: {}),
|
|
5040
|
+
},
|
|
5041
|
+
),
|
|
4411
5042
|
);
|
|
4412
5043
|
await attachToolspaceTokenRenewal(
|
|
4413
5044
|
provisioned.established.session as ToolspaceTokenWriterSession,
|
|
4414
5045
|
lazyToolspaceToken?.expiresAt,
|
|
5046
|
+
provisioned,
|
|
4415
5047
|
);
|
|
4416
5048
|
await attachGitCredentialRenewal(
|
|
4417
5049
|
provisioned.established.session as GitCredentialTokenWriterSession,
|
|
4418
5050
|
lazyGitCredentials,
|
|
5051
|
+
provisioned,
|
|
4419
5052
|
);
|
|
4420
5053
|
// Return the REAL established box (NOT a copy whose session is the routing
|
|
4421
5054
|
// proxy). resolveActiveBackend dispatches ops to `provisioned.established.session`;
|
|
@@ -4485,17 +5118,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4485
5118
|
settings,
|
|
4486
5119
|
bus,
|
|
4487
5120
|
onOp: machineOpObserver.observer,
|
|
5121
|
+
onHomeSandboxLost: publishSandboxLost,
|
|
5122
|
+
onHomeSandboxRebound,
|
|
4488
5123
|
},
|
|
4489
5124
|
{
|
|
4490
5125
|
workspaceId: input.workspaceId,
|
|
4491
5126
|
sessionId: input.sessionId,
|
|
4492
5127
|
environment: sandboxEnvironment,
|
|
5128
|
+
workspaceMutationFence: {
|
|
5129
|
+
accountId: input.accountId,
|
|
5130
|
+
turnId: turn.id,
|
|
5131
|
+
executionGeneration,
|
|
5132
|
+
attemptId: input.attemptId,
|
|
5133
|
+
},
|
|
4493
5134
|
},
|
|
4494
5135
|
{
|
|
4495
5136
|
client: lazyClient,
|
|
4496
5137
|
backendId: sdkBackendIdForSandboxBackend(groupBoxBackend),
|
|
4497
5138
|
agentDefaultManifest,
|
|
4498
5139
|
provisioner: turnSandboxProvisioner,
|
|
5140
|
+
homeLeaseIdentity: {
|
|
5141
|
+
accountId: input.accountId,
|
|
5142
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
5143
|
+
backend: groupBoxBackend,
|
|
5144
|
+
},
|
|
4499
5145
|
},
|
|
4500
5146
|
);
|
|
4501
5147
|
}
|
|
@@ -4617,22 +5263,27 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4617
5263
|
);
|
|
4618
5264
|
const runAs = sandboxRunAs(runSettings);
|
|
4619
5265
|
if (downloadsToMaterialize.length > 0) {
|
|
4620
|
-
const materialized = await
|
|
4621
|
-
|
|
4622
|
-
|
|
4623
|
-
|
|
4624
|
-
|
|
4625
|
-
|
|
4626
|
-
|
|
4627
|
-
|
|
4628
|
-
|
|
4629
|
-
|
|
4630
|
-
|
|
4631
|
-
|
|
4632
|
-
|
|
4633
|
-
|
|
4634
|
-
|
|
4635
|
-
|
|
5266
|
+
const materialized = await runWorkspaceMutationForSandbox(
|
|
5267
|
+
resolvedSandbox,
|
|
5268
|
+
"fileMaterialization",
|
|
5269
|
+
async () =>
|
|
5270
|
+
await materializeSandboxFileDownloads(
|
|
5271
|
+
setupBoxSession as any,
|
|
5272
|
+
downloadsToMaterialize,
|
|
5273
|
+
{
|
|
5274
|
+
onRuntimeEvent: async (event) => {
|
|
5275
|
+
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
5276
|
+
},
|
|
5277
|
+
...(runAs ? { runAs } : {}),
|
|
5278
|
+
...(toolCancellationFenceRef.current
|
|
5279
|
+
? {
|
|
5280
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
5281
|
+
toolCancellationFenceRef.current,
|
|
5282
|
+
),
|
|
5283
|
+
}
|
|
5284
|
+
: {}),
|
|
5285
|
+
},
|
|
5286
|
+
),
|
|
4636
5287
|
);
|
|
4637
5288
|
fileMaterializationFailures = materialized.failures;
|
|
4638
5289
|
const failedFileIds = new Set(materialized.failures.map((failure) => failure.fileId));
|
|
@@ -4680,6 +5331,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4680
5331
|
recovering: turn.executionGeneration > 1,
|
|
4681
5332
|
...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
|
|
4682
5333
|
...(runCredentialsNote ? { runCredentialsNote } : {}),
|
|
5334
|
+
...(modelAcceptsTypedAttachmentContentForTurn(resolvedModel) && objectStorage
|
|
5335
|
+
? {
|
|
5336
|
+
readFileBytesForModel: (file) => objectStorage.getFileBytes(file),
|
|
5337
|
+
}
|
|
5338
|
+
: {}),
|
|
4683
5339
|
},
|
|
4684
5340
|
);
|
|
4685
5341
|
runInput = prepared.input;
|
|
@@ -4769,13 +5425,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4769
5425
|
return outcome;
|
|
4770
5426
|
};
|
|
4771
5427
|
|
|
5428
|
+
// Keep response usage identity across every stream attempt in this
|
|
5429
|
+
// activity. Context compaction retries the same logical dispatch by
|
|
5430
|
+
// calling runStreamAttempt again; resetting this state there would reuse
|
|
5431
|
+
// the first no-response-ID fallback key and suppress a real model call.
|
|
5432
|
+
const modelResponseUsageState = createModelResponseUsageEventState(
|
|
5433
|
+
claimedModelUsageSourceKeys,
|
|
5434
|
+
);
|
|
4772
5435
|
const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
|
|
4773
5436
|
if (!runInput) {
|
|
4774
5437
|
throw new Error("Run input was not prepared");
|
|
4775
5438
|
}
|
|
4776
5439
|
stream = undefined;
|
|
4777
5440
|
batcher = null;
|
|
4778
|
-
let responseUsageCount = 0;
|
|
4779
5441
|
// The SDK emits every processed call item for one model response before
|
|
4780
5442
|
// it emits any result for that response. Keep that response-local batch
|
|
4781
5443
|
// in memory so an orphan from an older response cannot pin later stable
|
|
@@ -4785,13 +5447,53 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4785
5447
|
let currentToolBatchCompletedCallIds = new Set<string>();
|
|
4786
5448
|
// Actual input tokens of the most recent model response this turn; the
|
|
4787
5449
|
// pre-read trigger for the NEXT turn. Persisted at every turn-end path.
|
|
4788
|
-
let lastProviderContextTokensObserved: number | null = null;
|
|
4789
|
-
let providerContextRevision = 0;
|
|
4790
5450
|
throwIfWorkerShuttingDown();
|
|
4791
5451
|
throwIfTurnCancelled();
|
|
4792
5452
|
const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
|
|
4793
|
-
const runStreamOnce = (): ReturnType<OpenGeniRuntime["runStream"]> =>
|
|
4794
|
-
|
|
5453
|
+
const runStreamOnce = async (): ReturnType<OpenGeniRuntime["runStream"]> => {
|
|
5454
|
+
// Eager owned sessions must settle the exact platform-setup provider
|
|
5455
|
+
// promise before the long model stream starts; otherwise one admission
|
|
5456
|
+
// would remain in flight for the entire turn and suppress every
|
|
5457
|
+
// heartbeat capture. Lazy setup already runs under the same wrapper in
|
|
5458
|
+
// its first-operation provisioner above.
|
|
5459
|
+
if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
|
|
5460
|
+
const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
|
|
5461
|
+
await runWorkspaceMutationForSandbox(
|
|
5462
|
+
resolvedSandbox,
|
|
5463
|
+
"eagerOwnedSandboxSetup",
|
|
5464
|
+
async () =>
|
|
5465
|
+
await runOwnedSandboxSetup(
|
|
5466
|
+
agent,
|
|
5467
|
+
ownedEstablished.session as never,
|
|
5468
|
+
eagerSetupSession as never,
|
|
5469
|
+
{
|
|
5470
|
+
settings: runSettings,
|
|
5471
|
+
environment: sandboxEnvironment,
|
|
5472
|
+
preparedInput: runInput!,
|
|
5473
|
+
...(fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {}),
|
|
5474
|
+
onRuntimeEvent: async (event) => {
|
|
5475
|
+
await renewCodexLease("runtime_event");
|
|
5476
|
+
if (codexLeaseLost) {
|
|
5477
|
+
throw new Error("Codex credential lease expired during sandbox setup");
|
|
5478
|
+
}
|
|
5479
|
+
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
5480
|
+
},
|
|
5481
|
+
...(toolCancellationFenceRef.current
|
|
5482
|
+
? {
|
|
5483
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
5484
|
+
toolCancellationFenceRef.current,
|
|
5485
|
+
),
|
|
5486
|
+
}
|
|
5487
|
+
: {}),
|
|
5488
|
+
},
|
|
5489
|
+
),
|
|
5490
|
+
);
|
|
5491
|
+
await attachGitCredentialRenewal(
|
|
5492
|
+
eagerSetupSession as GitCredentialTokenWriterSession,
|
|
5493
|
+
initialGitCredentials,
|
|
5494
|
+
);
|
|
5495
|
+
}
|
|
5496
|
+
return await runtime.runStream(agent, runInput!, modelRunSettings, {
|
|
4795
5497
|
...(activityContext ? { signal: activityContext.cancellationSignal } : {}),
|
|
4796
5498
|
sandboxEnvironment,
|
|
4797
5499
|
onRuntimeEvent: async (event) => {
|
|
@@ -4818,16 +5520,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4818
5520
|
// re-route those execs onto a machine swapped in mid-turn.
|
|
4819
5521
|
...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
|
|
4820
5522
|
...(fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {}),
|
|
4821
|
-
|
|
4822
|
-
|
|
4823
|
-
|
|
4824
|
-
: {}),
|
|
4825
|
-
...(initialGitCredentials
|
|
4826
|
-
? {
|
|
4827
|
-
onGitCredentialSessionReady: async (
|
|
4828
|
-
tokenSession: GitCredentialTokenWriterSession,
|
|
4829
|
-
) => {
|
|
4830
|
-
await attachGitCredentialRenewal(tokenSession, initialGitCredentials);
|
|
5523
|
+
// Both owned paths execute setup outside runStream: eager just
|
|
5524
|
+
// above under its exact admission, lazy in the provisioner.
|
|
5525
|
+
deferredSetup: true,
|
|
4831
5526
|
},
|
|
4832
5527
|
}
|
|
4833
5528
|
: {}),
|
|
@@ -4861,19 +5556,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4861
5556
|
: {}),
|
|
4862
5557
|
}
|
|
4863
5558
|
: {}),
|
|
4864
|
-
contextCompactionSignal: () =>
|
|
4865
|
-
lastProviderContextTokensObserved === null
|
|
4866
|
-
? null
|
|
4867
|
-
: {
|
|
4868
|
-
revision: providerContextRevision,
|
|
4869
|
-
totalTokens: lastProviderContextTokensObserved,
|
|
4870
|
-
},
|
|
5559
|
+
contextCompactionSignal: () => modelResponseUsageContextSignal(modelResponseUsageState),
|
|
4871
5560
|
contextCompactionRequested: () =>
|
|
4872
5561
|
isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
4873
5562
|
...(toolCancellationFenceRef.current
|
|
4874
5563
|
? { turnToolCancellationFence: toolCancellationFenceRef.current }
|
|
4875
5564
|
: {}),
|
|
4876
5565
|
});
|
|
5566
|
+
};
|
|
4877
5567
|
if (codexLeaseLost) {
|
|
4878
5568
|
throw new Error("Codex credential lease expired before the model run");
|
|
4879
5569
|
}
|
|
@@ -4906,77 +5596,33 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4906
5596
|
}
|
|
4907
5597
|
let stableToolCallIdsToClear: string[] | null = null;
|
|
4908
5598
|
let completedCurrentToolBatch = false;
|
|
4909
|
-
const
|
|
4910
|
-
|
|
4911
|
-
|
|
4912
|
-
|
|
4913
|
-
|
|
4914
|
-
|
|
4915
|
-
|
|
4916
|
-
|
|
4917
|
-
|
|
4918
|
-
|
|
4919
|
-
|
|
4920
|
-
|
|
4921
|
-
|
|
4922
|
-
|
|
4923
|
-
|
|
4924
|
-
|
|
4925
|
-
|
|
4926
|
-
|
|
4927
|
-
|
|
4928
|
-
|
|
4929
|
-
|
|
4930
|
-
|
|
4931
|
-
|
|
4932
|
-
|
|
4933
|
-
|
|
4934
|
-
|
|
4935
|
-
|
|
4936
|
-
isCodexTurn,
|
|
4937
|
-
usage: responseUsage.usage,
|
|
4938
|
-
sourceKey: responseSourceKey,
|
|
4939
|
-
observability,
|
|
4940
|
-
});
|
|
4941
|
-
await emitModelCallUsage({
|
|
4942
|
-
observability,
|
|
4943
|
-
publish,
|
|
4944
|
-
accountId: input.accountId,
|
|
4945
|
-
workspaceId: input.workspaceId,
|
|
4946
|
-
sessionId: input.sessionId,
|
|
4947
|
-
turnId: activeTurnId,
|
|
4948
|
-
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
4949
|
-
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
4950
|
-
model: turn.model,
|
|
4951
|
-
sourceKey: responseSourceKey,
|
|
4952
|
-
usage: responseUsage,
|
|
4953
|
-
servingAccountHash: responseAccountCtx.servingAccountHash,
|
|
4954
|
-
accountChangedFromPrevCall: responseAccountCtx.accountChangedFromPrevCall,
|
|
4955
|
-
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
4956
|
-
});
|
|
4957
|
-
const observed = responseUsage.usage?.inputTokens;
|
|
4958
|
-
if (typeof observed === "number" && observed > 0) {
|
|
4959
|
-
recordModelInputTokens(observability, streamProvider, observed);
|
|
4960
|
-
}
|
|
4961
|
-
const observedTotal = providerContextTokens(responseUsage.usage);
|
|
4962
|
-
if (observedTotal !== null) {
|
|
4963
|
-
lastProviderContextTokensObserved = observedTotal;
|
|
4964
|
-
providerContextRevision += 1;
|
|
4965
|
-
}
|
|
4966
|
-
// Prompt-cache efficiency for this response — same usage frame as the
|
|
4967
|
-
// input-token accounting above, so the two are always consistent.
|
|
4968
|
-
recordModelCacheTokens(observability, streamProvider, {
|
|
4969
|
-
cachedTokens: modelCallUsageTelemetry(responseUsage.usage).cachedTokens,
|
|
4970
|
-
promptTokens: responseUsage.usage?.inputTokens,
|
|
4971
|
-
});
|
|
4972
|
-
},
|
|
4973
|
-
recordAttemptSignals: async () => {
|
|
4974
|
-
const observed = responseUsage.usage?.inputTokens;
|
|
4975
|
-
if (typeof observed === "number" && observed > 0) {
|
|
4976
|
-
await setLastInputTokensFenced(observed);
|
|
4977
|
-
}
|
|
4978
|
-
},
|
|
4979
|
-
});
|
|
5599
|
+
const responseUsageResult = await processModelResponseUsageEvent({
|
|
5600
|
+
event: next.value,
|
|
5601
|
+
state: modelResponseUsageState,
|
|
5602
|
+
dispatchId: modelUsageDispatchId,
|
|
5603
|
+
settings,
|
|
5604
|
+
db,
|
|
5605
|
+
observability,
|
|
5606
|
+
publish,
|
|
5607
|
+
accountId: input.accountId,
|
|
5608
|
+
workspaceId: input.workspaceId,
|
|
5609
|
+
sessionId: input.sessionId,
|
|
5610
|
+
turnId: activeTurnId,
|
|
5611
|
+
turnAttemptId: input.attemptId,
|
|
5612
|
+
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
5613
|
+
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
5614
|
+
model: turn.model,
|
|
5615
|
+
metricProvider: streamProvider,
|
|
5616
|
+
externallyBilled: isExternallyBilledTurn,
|
|
5617
|
+
servingCredentialId: effectiveCodexCredentialId,
|
|
5618
|
+
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
5619
|
+
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
5620
|
+
renewLease: () => renewCodexLease("model_usage"),
|
|
5621
|
+
leaseLost: () => codexLeaseLost,
|
|
5622
|
+
leaseLostMessage: "Codex credential lease expired during the active turn",
|
|
5623
|
+
setLastInputTokens: setLastInputTokensFenced,
|
|
5624
|
+
});
|
|
5625
|
+
if (responseUsageResult.status === "processed") {
|
|
4980
5626
|
currentToolBatchCallIds = new Set<string>();
|
|
4981
5627
|
currentToolBatchCompletedCallIds = new Set<string>();
|
|
4982
5628
|
await reconcileConversationTruth();
|
|
@@ -4986,7 +5632,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4986
5632
|
db,
|
|
4987
5633
|
input.accountId,
|
|
4988
5634
|
input.workspaceId,
|
|
4989
|
-
|
|
5635
|
+
isExternallyBilledTurn,
|
|
4990
5636
|
entitlements,
|
|
4991
5637
|
);
|
|
4992
5638
|
} catch (limitError) {
|
|
@@ -5014,6 +5660,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5014
5660
|
turnId: activeTurnId,
|
|
5015
5661
|
executionGeneration,
|
|
5016
5662
|
attemptId: input.attemptId,
|
|
5663
|
+
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
5017
5664
|
...pendingToolCall,
|
|
5018
5665
|
});
|
|
5019
5666
|
if (!registered.accepted) {
|
|
@@ -5038,6 +5685,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5038
5685
|
executionGeneration,
|
|
5039
5686
|
attemptId: input.attemptId,
|
|
5040
5687
|
callId: completedToolCall.callId,
|
|
5688
|
+
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
5041
5689
|
resultItem: completedToolCall.resultItem,
|
|
5042
5690
|
});
|
|
5043
5691
|
if (!recorded.accepted) {
|
|
@@ -5111,83 +5759,77 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5111
5759
|
stream.completed.catch(() => undefined),
|
|
5112
5760
|
cancellationSignal,
|
|
5113
5761
|
);
|
|
5114
|
-
if (responseUsageCount === 0) {
|
|
5762
|
+
if (modelResponseUsageState.responseUsageCount === 0) {
|
|
5115
5763
|
const aggregateUsage = stream.state.usage;
|
|
5116
|
-
const
|
|
5117
|
-
|
|
5118
|
-
if (typeof aggregateInput === "number" && aggregateInput > 0) {
|
|
5119
|
-
const aggregateContext = providerContextTokens(
|
|
5120
|
-
aggregateUsage as {
|
|
5121
|
-
inputTokens?: number;
|
|
5122
|
-
outputTokens?: number;
|
|
5123
|
-
totalTokens?: number;
|
|
5124
|
-
},
|
|
5125
|
-
);
|
|
5126
|
-
if (aggregateContext !== null) {
|
|
5127
|
-
lastProviderContextTokensObserved = aggregateContext;
|
|
5128
|
-
providerContextRevision += 1;
|
|
5129
|
-
}
|
|
5130
|
-
}
|
|
5764
|
+
const normalizedAggregateUsage = normalizeModelCallUsage(aggregateUsage);
|
|
5765
|
+
const aggregateInput = normalizedAggregateUsage.telemetry.inputTokens;
|
|
5131
5766
|
const aggregateSourceKey = modelUsageSourceKey({
|
|
5132
5767
|
responseId: null,
|
|
5133
5768
|
dispatchId: modelUsageDispatchId,
|
|
5134
5769
|
positionalKey: "aggregate",
|
|
5135
5770
|
});
|
|
5136
|
-
|
|
5137
|
-
|
|
5138
|
-
|
|
5139
|
-
|
|
5140
|
-
|
|
5141
|
-
|
|
5142
|
-
|
|
5143
|
-
|
|
5144
|
-
|
|
5145
|
-
|
|
5146
|
-
|
|
5147
|
-
|
|
5148
|
-
|
|
5149
|
-
|
|
5150
|
-
|
|
5151
|
-
|
|
5152
|
-
|
|
5153
|
-
|
|
5154
|
-
|
|
5155
|
-
|
|
5156
|
-
|
|
5157
|
-
|
|
5158
|
-
|
|
5159
|
-
|
|
5160
|
-
|
|
5161
|
-
|
|
5162
|
-
|
|
5163
|
-
|
|
5164
|
-
|
|
5165
|
-
|
|
5166
|
-
|
|
5167
|
-
|
|
5168
|
-
|
|
5169
|
-
|
|
5170
|
-
|
|
5171
|
-
|
|
5172
|
-
|
|
5173
|
-
|
|
5174
|
-
|
|
5175
|
-
|
|
5176
|
-
|
|
5177
|
-
|
|
5178
|
-
|
|
5179
|
-
|
|
5180
|
-
|
|
5181
|
-
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
5185
|
-
|
|
5186
|
-
|
|
5187
|
-
|
|
5188
|
-
|
|
5189
|
-
|
|
5190
|
-
|
|
5771
|
+
if (!claimedModelUsageSourceKeys.has(aggregateSourceKey)) {
|
|
5772
|
+
claimedModelUsageSourceKeys.add(aggregateSourceKey);
|
|
5773
|
+
// The single aggregate frame is this turn's only model-usage record, so
|
|
5774
|
+
// it is the first (account-switch surfaces here just like a first response).
|
|
5775
|
+
const aggregateAccountCtx = modelCallAccountContext({
|
|
5776
|
+
servingCredentialId: effectiveCodexCredentialId,
|
|
5777
|
+
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
5778
|
+
isFirstCallOfTurn: true,
|
|
5779
|
+
});
|
|
5780
|
+
let aggregateAuthoritative = false;
|
|
5781
|
+
await recordCompletedModelCallBeforeOwnershipFences({
|
|
5782
|
+
renewLease: () => renewCodexLease("model_usage"),
|
|
5783
|
+
leaseLost: () => codexLeaseLost,
|
|
5784
|
+
leaseLostMessage: "Codex credential lease expired during the active turn",
|
|
5785
|
+
recordUsage: async () => {
|
|
5786
|
+
await recordModelUsageAndDebitCredits(settings, db, {
|
|
5787
|
+
accountId: input.accountId,
|
|
5788
|
+
workspaceId: input.workspaceId,
|
|
5789
|
+
sessionId: input.sessionId,
|
|
5790
|
+
turnId: activeTurnId,
|
|
5791
|
+
turnAttemptId: input.attemptId,
|
|
5792
|
+
model: turn.model,
|
|
5793
|
+
externallyBilled: isExternallyBilledTurn,
|
|
5794
|
+
usage: aggregateUsage,
|
|
5795
|
+
normalizedUsage: normalizedAggregateUsage,
|
|
5796
|
+
sourceKey: aggregateSourceKey,
|
|
5797
|
+
observability,
|
|
5798
|
+
});
|
|
5799
|
+
aggregateAuthoritative = await emitModelCallUsage({
|
|
5800
|
+
observability,
|
|
5801
|
+
publish,
|
|
5802
|
+
accountId: input.accountId,
|
|
5803
|
+
workspaceId: input.workspaceId,
|
|
5804
|
+
sessionId: input.sessionId,
|
|
5805
|
+
turnId: activeTurnId,
|
|
5806
|
+
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
5807
|
+
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
5808
|
+
model: turn.model,
|
|
5809
|
+
sourceKey: aggregateSourceKey,
|
|
5810
|
+
usage: { usage: aggregateUsage },
|
|
5811
|
+
normalizedUsage: normalizedAggregateUsage,
|
|
5812
|
+
servingAccountHash: aggregateAccountCtx.servingAccountHash,
|
|
5813
|
+
accountChangedFromPrevCall: aggregateAccountCtx.accountChangedFromPrevCall,
|
|
5814
|
+
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
5815
|
+
});
|
|
5816
|
+
if (aggregateAuthoritative && aggregateInput !== null && aggregateInput > 0) {
|
|
5817
|
+
recordModelInputTokens(observability, streamProvider, aggregateInput);
|
|
5818
|
+
}
|
|
5819
|
+
},
|
|
5820
|
+
recordAttemptSignals: async () => {
|
|
5821
|
+
if (!aggregateAuthoritative) return;
|
|
5822
|
+
if (normalizedAggregateUsage.totalTokens !== null) {
|
|
5823
|
+
modelResponseUsageState.lastProviderContextTokensObserved =
|
|
5824
|
+
normalizedAggregateUsage.totalTokens;
|
|
5825
|
+
modelResponseUsageState.providerContextRevision += 1;
|
|
5826
|
+
}
|
|
5827
|
+
if (aggregateInput !== null && aggregateInput > 0) {
|
|
5828
|
+
await setLastInputTokensFenced(aggregateInput);
|
|
5829
|
+
}
|
|
5830
|
+
},
|
|
5831
|
+
});
|
|
5832
|
+
}
|
|
5191
5833
|
}
|
|
5192
5834
|
if (stream.interruptions.length > 0) {
|
|
5193
5835
|
await reconcileConversationTruth();
|
|
@@ -6251,7 +6893,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6251
6893
|
// truth, recover this SAME accepted turn, then let the workflow re-claim
|
|
6252
6894
|
// it after a pacing delay. This is independent of goal state and never
|
|
6253
6895
|
// relies on a synthetic continuation prompt.
|
|
6254
|
-
const failure = agentRunFailurePayload(error);
|
|
6896
|
+
const failure = agentRunFailurePayload(error, { isCodexTurn });
|
|
6255
6897
|
if (isSessionEventPersistenceError(error)) {
|
|
6256
6898
|
// Never pass the original Drizzle/postgres-js error to telemetry: its
|
|
6257
6899
|
// nested cause may contain raw SQL and bound parameters. The typed DB
|
|
@@ -6355,10 +6997,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6355
6997
|
const finalizerSignal = turnFinalizerCancellationSignal(cancellationSignal, activityStatus);
|
|
6356
6998
|
try {
|
|
6357
6999
|
const toolCancellationFence = toolCancellationFenceRef.current;
|
|
6358
|
-
//
|
|
6359
|
-
//
|
|
6360
|
-
//
|
|
6361
|
-
|
|
7000
|
+
// Every renewal controller is an attempt-owned sandbox writer. Capture
|
|
7001
|
+
// and close all of them before the tool/run-writer drain and before the
|
|
7002
|
+
// quiescence receipt; none may start another admitted write afterward.
|
|
7003
|
+
gitCredentialRenewalClosed = true;
|
|
7004
|
+
const gitRenewalsToStop = gitCredentialRenewals;
|
|
7005
|
+
gitCredentialRenewals = [];
|
|
6362
7006
|
toolspaceTokenRenewalClosed = true;
|
|
6363
7007
|
const toolspaceRenewalToStop =
|
|
6364
7008
|
toolspaceTokenRenewal as ToolspaceTokenRenewalController | null;
|
|
@@ -6366,9 +7010,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6366
7010
|
runCredentialRenewalClosed = true;
|
|
6367
7011
|
const runRenewalToStop = runCredentialRenewal as RunCredentialRenewalController | null;
|
|
6368
7012
|
runCredentialRenewal = null;
|
|
7013
|
+
|
|
7014
|
+
// Attempt-qualified credential deletion is also a real workspace write.
|
|
7015
|
+
// Perform it under the same admission fence before publishing physical
|
|
7016
|
+
// quiescence; a failure deliberately keeps the receipt closed.
|
|
7017
|
+
const credentialSessionToClear = runCredentialSession;
|
|
7018
|
+
runCredentialSession = null;
|
|
7019
|
+
if (credentialSessionToClear) {
|
|
7020
|
+
await runWorkspaceMutationForSandbox(
|
|
7021
|
+
requireResolvedSandboxForMutation(
|
|
7022
|
+
"Run credential cleanup has no exact sandbox lease target",
|
|
7023
|
+
),
|
|
7024
|
+
"runCredentialAttemptClear",
|
|
7025
|
+
async () =>
|
|
7026
|
+
await clearRunCredentialsForAttempt(credentialSessionToClear, {
|
|
7027
|
+
sessionId: input.sessionId,
|
|
7028
|
+
attemptId: input.attemptId,
|
|
7029
|
+
executionGeneration,
|
|
7030
|
+
}),
|
|
7031
|
+
);
|
|
7032
|
+
}
|
|
6369
7033
|
await drainAttemptOwnedSandboxWriters({
|
|
6370
7034
|
toolCancellationFence: acknowledgeQuiescence ? toolCancellationFence : null,
|
|
6371
7035
|
cancellationReason: cancellationSignal?.reason,
|
|
7036
|
+
gitCredentialRenewals: gitRenewalsToStop,
|
|
6372
7037
|
toolspaceTokenRenewal: toolspaceRenewalToStop,
|
|
6373
7038
|
runCredentialRenewal: runRenewalToStop,
|
|
6374
7039
|
});
|
|
@@ -6448,43 +7113,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6448
7113
|
});
|
|
6449
7114
|
}
|
|
6450
7115
|
}
|
|
6451
|
-
// Deleting this attempt's credential generation is post-receipt
|
|
6452
|
-
// housekeeping: pointer mutation is serialized with successor
|
|
6453
|
-
// activation and the attempt-qualified check cannot erase a successor.
|
|
6454
|
-
// It may therefore detach on cancellation without weakening the writer
|
|
6455
|
-
// drain that gates the receipt above.
|
|
6456
|
-
const credentialSessionToClear = runCredentialSession;
|
|
6457
|
-
runCredentialSession = null;
|
|
6458
|
-
if (credentialSessionToClear) {
|
|
6459
|
-
const cleanup = clearRunCredentialsForAttempt(credentialSessionToClear, {
|
|
6460
|
-
sessionId: input.sessionId,
|
|
6461
|
-
attemptId: input.attemptId,
|
|
6462
|
-
executionGeneration,
|
|
6463
|
-
}).catch((error: unknown) => {
|
|
6464
|
-
try {
|
|
6465
|
-
observability.incrementCounter({
|
|
6466
|
-
name: "opengeni_run_credential_cleanup_total",
|
|
6467
|
-
help: "Attempt-owned run credential cleanup outcomes.",
|
|
6468
|
-
labels: { outcome: "error" },
|
|
6469
|
-
});
|
|
6470
|
-
observability.warn("Attempt-owned run credential cleanup failed", {
|
|
6471
|
-
sessionId: input.sessionId,
|
|
6472
|
-
turnId,
|
|
6473
|
-
attemptId: input.attemptId,
|
|
6474
|
-
errorClass: error instanceof Error ? error.name : "UnknownError",
|
|
6475
|
-
});
|
|
6476
|
-
} catch {
|
|
6477
|
-
// Cleanup observability must not own finalizer liveness.
|
|
6478
|
-
}
|
|
6479
|
-
});
|
|
6480
|
-
await waitForTurnFinalizerStep(cleanup, finalizerSignal);
|
|
6481
|
-
}
|
|
6482
|
-
gitCredentialRenewalClosed = true;
|
|
6483
|
-
const renewalsToStop = gitCredentialRenewals;
|
|
6484
|
-
gitCredentialRenewals = [];
|
|
6485
|
-
for (const renewal of renewalsToStop) {
|
|
6486
|
-
await waitForTurnFinalizerStep(renewal.stop(), finalizerSignal);
|
|
6487
|
-
}
|
|
6488
7116
|
// Drain the buffered Connected Machine op events (infra failures + healed
|
|
6489
7117
|
// recoveries) to durable session events — awaited, best-effort, never blocking
|
|
6490
7118
|
// the turn. Sync observer → buffer → single awaited append here (no unawaited
|
|
@@ -6878,11 +7506,17 @@ export function isTransientProviderError(error: unknown): boolean {
|
|
|
6878
7506
|
);
|
|
6879
7507
|
}
|
|
6880
7508
|
|
|
6881
|
-
export function agentRunFailurePayload(
|
|
7509
|
+
export function agentRunFailurePayload(
|
|
7510
|
+
error: unknown,
|
|
7511
|
+
options: { isCodexTurn?: boolean } = {},
|
|
7512
|
+
): {
|
|
6882
7513
|
error: string;
|
|
6883
7514
|
code?: string;
|
|
6884
7515
|
retryable?: boolean;
|
|
6885
7516
|
detail?: string;
|
|
7517
|
+
timeoutClass?: string;
|
|
7518
|
+
responseObserved?: boolean;
|
|
7519
|
+
requestId?: string;
|
|
6886
7520
|
correlationId?: string;
|
|
6887
7521
|
stage?: string;
|
|
6888
7522
|
sqlState?: string | null;
|
|
@@ -6948,6 +7582,22 @@ export function agentRunFailurePayload(error: unknown): {
|
|
|
6948
7582
|
if (usageLimit) {
|
|
6949
7583
|
return codexUsageLimitFailurePayload(usageLimit, message);
|
|
6950
7584
|
}
|
|
7585
|
+
const codexTimeout = classifyCodexResponseTimeoutError(error, {
|
|
7586
|
+
allowLegacyRequestTimeout: options.isCodexTurn === true,
|
|
7587
|
+
});
|
|
7588
|
+
if (codexTimeout) {
|
|
7589
|
+
return {
|
|
7590
|
+
error: codexTimeout.responseObserved
|
|
7591
|
+
? "The Codex response timed out after streaming began. Observed output was checkpointed; automatic replay is disabled because the upstream operation may still be active."
|
|
7592
|
+
: "The Codex response timed out before any response was observed. Upstream acceptance is unknown, so automatic replay is disabled.",
|
|
7593
|
+
code: "codex_response_timeout",
|
|
7594
|
+
retryable: false,
|
|
7595
|
+
timeoutClass: codexTimeout.timeoutClass,
|
|
7596
|
+
responseObserved: codexTimeout.responseObserved,
|
|
7597
|
+
...(codexTimeout.requestId ? { requestId: codexTimeout.requestId } : {}),
|
|
7598
|
+
...(codexTimeout.message ? { detail: codexTimeout.message } : {}),
|
|
7599
|
+
};
|
|
7600
|
+
}
|
|
6951
7601
|
const mcpTimeout = classifyMcpTransportTimeoutError(error);
|
|
6952
7602
|
if (mcpTimeout) {
|
|
6953
7603
|
return {
|
|
@@ -7227,21 +7877,21 @@ class BudgetExhaustedError extends Error {
|
|
|
7227
7877
|
}
|
|
7228
7878
|
}
|
|
7229
7879
|
|
|
7230
|
-
// Exported for unit testing the
|
|
7231
|
-
// of the activity surface. Takes
|
|
7232
|
-
//
|
|
7233
|
-
//
|
|
7880
|
+
// Exported for unit testing the external-billing bypass (codex-billing.test.ts); not
|
|
7881
|
+
// part of the activity surface. Takes the accepted policy's billing attribution and
|
|
7882
|
+
// the optional §7.5 P3 host `entitlements` port (when bound, its `admitRun` REPLACES
|
|
7883
|
+
// the local credit read for an OpenGeni-metered turn; unset → local ledger).
|
|
7234
7884
|
export async function ensureRunAllowed(
|
|
7235
7885
|
settings: Settings,
|
|
7236
7886
|
db: ActivityServices["db"],
|
|
7237
7887
|
accountId: string,
|
|
7238
7888
|
workspaceId: string,
|
|
7239
|
-
|
|
7889
|
+
isExternallyBilledTurn: boolean,
|
|
7240
7890
|
entitlements?: ActivityServices["entitlements"],
|
|
7241
7891
|
): Promise<void> {
|
|
7242
|
-
//
|
|
7243
|
-
//
|
|
7244
|
-
//
|
|
7892
|
+
// Externally billed turns are paid outside OpenGeni: skip the credit-balance
|
|
7893
|
+
// gate and monthly token cap. The agent-run COUNT cap below is a
|
|
7894
|
+
// volume/fairness quota (not a credit/cost gate) and is intentionally kept.
|
|
7245
7895
|
//
|
|
7246
7896
|
// §7.5 P3 — host-entitlements DELEGATION (the worker half of the same seam the
|
|
7247
7897
|
// API edge exposes). For a non-codex turn, when the host binds `entitlements`, its
|
|
@@ -7255,7 +7905,7 @@ export async function ensureRunAllowed(
|
|
|
7255
7905
|
// idempotency-keyed writer at recordModelUsageAndDebitCredits), so a PULL host meter
|
|
7256
7906
|
// is consulted without ever double-charging.
|
|
7257
7907
|
if (
|
|
7258
|
-
!
|
|
7908
|
+
!isExternallyBilledTurn &&
|
|
7259
7909
|
entitlements &&
|
|
7260
7910
|
(settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")
|
|
7261
7911
|
) {
|
|
@@ -7269,7 +7919,7 @@ export async function ensureRunAllowed(
|
|
|
7269
7919
|
throw new Error(decision.reason || "insufficient OpenGeni credits");
|
|
7270
7920
|
}
|
|
7271
7921
|
} else if (
|
|
7272
|
-
!
|
|
7922
|
+
!isExternallyBilledTurn &&
|
|
7273
7923
|
(settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")
|
|
7274
7924
|
) {
|
|
7275
7925
|
const balance = await getBillingBalance(db, accountId);
|
|
@@ -7294,7 +7944,7 @@ export async function ensureRunAllowed(
|
|
|
7294
7944
|
);
|
|
7295
7945
|
}
|
|
7296
7946
|
}
|
|
7297
|
-
if (!
|
|
7947
|
+
if (!isExternallyBilledTurn && limits.maxMonthlyTokensPerWorkspace) {
|
|
7298
7948
|
const used = await sumUsageQuantity(db, {
|
|
7299
7949
|
workspaceId,
|
|
7300
7950
|
eventType: "model.tokens",
|
|
@@ -7307,7 +7957,7 @@ export async function ensureRunAllowed(
|
|
|
7307
7957
|
}
|
|
7308
7958
|
}
|
|
7309
7959
|
|
|
7310
|
-
// Exported for unit testing the
|
|
7960
|
+
// Exported for unit testing the external-billing bypass; not part of the activity surface.
|
|
7311
7961
|
export async function recordModelUsageAndDebitCredits(
|
|
7312
7962
|
settings: Settings,
|
|
7313
7963
|
db: ActivityServices["db"],
|
|
@@ -7318,8 +7968,9 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
7318
7968
|
turnId: string;
|
|
7319
7969
|
turnAttemptId: string;
|
|
7320
7970
|
model: string;
|
|
7321
|
-
|
|
7322
|
-
usage?: ModelUsageInput | null;
|
|
7971
|
+
externallyBilled: boolean;
|
|
7972
|
+
usage?: ModelUsageInput | ModelCallUsageInput | null;
|
|
7973
|
+
normalizedUsage?: ModelCallUsageNormalization;
|
|
7323
7974
|
sourceKey: string;
|
|
7324
7975
|
observability?: ActivityServices["observability"];
|
|
7325
7976
|
},
|
|
@@ -7327,10 +7978,12 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
7327
7978
|
if (!input.usage) {
|
|
7328
7979
|
return;
|
|
7329
7980
|
}
|
|
7330
|
-
const
|
|
7331
|
-
const
|
|
7332
|
-
const
|
|
7333
|
-
|
|
7981
|
+
const normalizedUsage = input.normalizedUsage ?? normalizeModelCallUsage(input.usage);
|
|
7982
|
+
const sanitizedUsage = sanitizedModelUsageInput(normalizedUsage);
|
|
7983
|
+
const inputTokens = sanitizedUsage.inputTokens ?? 0;
|
|
7984
|
+
const outputTokens = sanitizedUsage.outputTokens ?? 0;
|
|
7985
|
+
const totalTokens = sanitizedUsage.totalTokens ?? 0;
|
|
7986
|
+
// An externally billed turn is paid outside OpenGeni, so it
|
|
7334
7987
|
// consumes ZERO OpenGeni credits and must never feed an OpenGeni cap. A
|
|
7335
7988
|
// codex/<slug> model has no entry in configuredModelPricing, so the normal path
|
|
7336
7989
|
// below would throw "Missing model pricing". We:
|
|
@@ -7339,7 +7992,7 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
7339
7992
|
// any row would count against maxMonthlyTokensPerWorkspace);
|
|
7340
7993
|
// - record a `model.cost = 0` audit marker (harmless to the monthly cost cap);
|
|
7341
7994
|
// - never look up pricing and never debit credits.
|
|
7342
|
-
if (input.
|
|
7995
|
+
if (input.externallyBilled) {
|
|
7343
7996
|
await recordUsageEvent(db, {
|
|
7344
7997
|
accountId: input.accountId,
|
|
7345
7998
|
workspaceId: input.workspaceId,
|
|
@@ -7377,7 +8030,7 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
7377
8030
|
if (!configuredModelPricing(settings)[input.model]) {
|
|
7378
8031
|
throw new Error(`Missing model pricing for ${input.model}`);
|
|
7379
8032
|
}
|
|
7380
|
-
const costMicros = calculateModelUsageCostMicros(settings, input.model,
|
|
8033
|
+
const costMicros = calculateModelUsageCostMicros(settings, input.model, sanitizedUsage);
|
|
7381
8034
|
await recordUsageEvent(db, {
|
|
7382
8035
|
accountId: input.accountId,
|
|
7383
8036
|
workspaceId: input.workspaceId,
|
|
@@ -7411,18 +8064,26 @@ export async function recordModelUsageAndDebitCredits(
|
|
|
7411
8064
|
// Additive: the prompt-cache slice of this call's input tokens, so the
|
|
7412
8065
|
// per-call debit record carries cache efficiency alongside the token
|
|
7413
8066
|
// counts. 0 when the provider did not report cached tokens.
|
|
7414
|
-
cachedTokens:
|
|
7415
|
-
modelCallUsageTelemetry(input.usage as Parameters<typeof modelCallUsageTelemetry>[0])
|
|
7416
|
-
.cachedTokens,
|
|
7417
|
-
),
|
|
8067
|
+
cachedTokens: normalizedUsage.telemetry.cachedTokens ?? 0,
|
|
7418
8068
|
},
|
|
7419
8069
|
});
|
|
7420
8070
|
recordCreditMicros(input.observability, "usage", result.debitedMicros);
|
|
7421
8071
|
}
|
|
7422
8072
|
}
|
|
7423
8073
|
|
|
7424
|
-
function
|
|
7425
|
-
return
|
|
8074
|
+
function sanitizedModelUsageInput(normalized: ModelCallUsageNormalization): ModelUsageInput {
|
|
8075
|
+
return {
|
|
8076
|
+
...(normalized.telemetry.inputTokens !== null
|
|
8077
|
+
? { inputTokens: normalized.telemetry.inputTokens }
|
|
8078
|
+
: {}),
|
|
8079
|
+
...(normalized.telemetry.outputTokens !== null
|
|
8080
|
+
? { outputTokens: normalized.telemetry.outputTokens }
|
|
8081
|
+
: {}),
|
|
8082
|
+
...(normalized.totalTokens !== null ? { totalTokens: normalized.totalTokens } : {}),
|
|
8083
|
+
...(normalized.telemetry.cachedTokens !== null
|
|
8084
|
+
? { inputTokensDetails: { cached_tokens: normalized.telemetry.cachedTokens } }
|
|
8085
|
+
: {}),
|
|
8086
|
+
};
|
|
7426
8087
|
}
|
|
7427
8088
|
|
|
7428
8089
|
function startOfUtcMonth(): Date {
|