@opengeni/worker-bundle 0.8.3 → 0.10.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -56,7 +56,7 @@ import {
56
56
  getLatestRunState as getLatestRunState2,
57
57
  getHumanInputResumeForEvent as getHumanInputResumeForEvent2,
58
58
  getSessionHumanInputRequest,
59
- isCodexBilledTurn as isCodexBilledTurn2,
59
+ installOrReadTurnExecutionPolicyForAttempt,
60
60
  workspaceCodexSubscriptionActive as workspaceCodexSubscriptionActive2,
61
61
  acquireCodexCredentialLease,
62
62
  armCodexCapacityWait,
@@ -109,8 +109,8 @@ import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@
109
109
  import {
110
110
  sandboxStateEntryFromRunState,
111
111
  maxTurnsExceededRunState,
112
- modelCallUsageTelemetry,
113
112
  modelResponseUsageFromSdkEvent,
113
+ normalizeModelCallUsage,
114
114
  normalizeSdkEvent,
115
115
  sanitizeHistoryItemsForModel as sanitizeHistoryItemsForModel2,
116
116
  isEphemeralInternalContext,
@@ -165,12 +165,13 @@ function connectionTokenResolverForTurn(input) {
165
165
 
166
166
  // src/activities/agent-turn.ts
167
167
  import {
168
- builtinProviderId,
168
+ assertTurnExecutionPolicyMatchesConfigV1,
169
169
  calculateModelUsageCostMicros,
170
170
  configuredModelPricing,
171
171
  configuredStaticUsageLimits as configuredStaticUsageLimits2,
172
172
  sandboxWarmRateMicrosPerSecond,
173
- settingsWithResolvedModelContext
173
+ settingsWithResolvedModelContext,
174
+ resolveTurnExecutionPolicyV1
174
175
  } from "@opengeni/config";
175
176
  import { CancelledFailure } from "@temporalio/activity";
176
177
 
@@ -204,7 +205,7 @@ async function settingsWithEnabledCapabilityMcpServers(db, workspaceId, settings
204
205
  const enabled = await listEnabledMcpCapabilityServers(db, workspaceId);
205
206
  return settingsWithMcpCapabilityServers(settings, enabled);
206
207
  }
207
- async function settingsWithSessionMcpServersForRun(db, workspaceId, sessionId, settings) {
208
+ async function settingsWithSessionMcpServersForRun(db, workspaceId, sessionId, attemptId, settings) {
208
209
  const encryptionKey = environmentsEncryptionKeyBytes(settings);
209
210
  if (!encryptionKey) {
210
211
  const metadata = await listSessionMcpServerMetadata(db, workspaceId, sessionId);
@@ -219,7 +220,7 @@ async function settingsWithSessionMcpServersForRun(db, workspaceId, sessionId, s
219
220
  }
220
221
  return settingsWithSessionMcpServers(
221
222
  settings,
222
- await listSessionMcpServersForRun(db, workspaceId, sessionId, encryptionKey ?? null)
223
+ await listSessionMcpServersForRun(db, workspaceId, sessionId, attemptId, encryptionKey ?? null)
223
224
  );
224
225
  }
225
226
  function settingsWithSessionMcpServers(settings, servers) {
@@ -287,6 +288,7 @@ function withCodexProvider(settings) {
287
288
  baseUrl: CODEX_PROVIDER_BASE_URL,
288
289
  models: CODEX_FALLBACK_MODEL_SLUGS.map((slug) => ({
289
290
  id: `${CODEX_MODEL_ID_PREFIX}${slug}`,
291
+ upstreamModelId: slug,
290
292
  label: slug,
291
293
  reasoningEffort: true,
292
294
  // These three values are the live Codex CLI catalog policy, kept distinct
@@ -826,6 +828,7 @@ import {
826
828
  CODEX_CLIENT_VERSION,
827
829
  CODEX_FALLBACK_MODEL_SLUGS as CODEX_FALLBACK_MODEL_SLUGS2,
828
830
  CodexReloginRequired,
831
+ classifyCodexResponseTimeoutError,
829
832
  classifyCodexUsageLimitError,
830
833
  codexRequestStorage,
831
834
  isCodexTransportError
@@ -856,6 +859,9 @@ function workflowIdForSession(sessionId) {
856
859
  return `session-${sessionId}`;
857
860
  }
858
861
 
862
+ // src/activities/agent-turn.ts
863
+ import { enabledCapabilityMcpToolRefs, resolveSessionToolPolicy } from "@opengeni/core";
864
+
859
865
  // src/activities/context-compaction.ts
860
866
  import {
861
867
  applyContextCompaction,
@@ -1097,7 +1103,7 @@ import {
1097
1103
  } from "@opengeni/db";
1098
1104
  import { createGitHubAppInstallationTokenWithExpiry, githubAppBotIdentity } from "@opengeni/github";
1099
1105
  var TOOLSPACE_TOKEN_TTL_SECONDS = 60 * 60;
1100
- async function mintSandboxToolspaceToken(settings, scope, sessionId, runId, nowMs = Date.now()) {
1106
+ async function mintSandboxToolspaceToken(settings, scope, authority, nowMs = Date.now()) {
1101
1107
  if (!settings.toolspaceEnabled || !settings.delegationSecret) {
1102
1108
  return void 0;
1103
1109
  }
@@ -1105,10 +1111,10 @@ async function mintSandboxToolspaceToken(settings, scope, sessionId, runId, nowM
1105
1111
  const token = await signDelegatedAccessToken(settings.delegationSecret, {
1106
1112
  accountId: scope.accountId,
1107
1113
  workspaceId: scope.workspaceId,
1108
- subjectId: `sandbox:${runId}`,
1114
+ subjectId: `sandbox:${authority.sessionId}`,
1109
1115
  subjectLabel: "sandbox toolspace",
1110
1116
  permissions: ["toolspace:call"],
1111
- sessionId,
1117
+ sessionId: authority.sessionId,
1112
1118
  exp: expiresAtSeconds
1113
1119
  });
1114
1120
  return { token, expiresAt: new Date(expiresAtSeconds * 1e3) };
@@ -1155,7 +1161,7 @@ async function sandboxEnvironmentForRun(settings, resources, workspaceEnvironmen
1155
1161
  const stableOptions = options.scope ? { workspaceId: options.scope.workspaceId } : {};
1156
1162
  const environment = stableSandboxEnvironmentForRun(settings, workspaceEnvironment, stableOptions);
1157
1163
  const toolspaceScope = options.scope;
1158
- const toolspaceToken = toolspaceScope && options.sessionId && options.runId ? await mintSandboxToolspaceToken(settings, toolspaceScope, options.sessionId, options.runId) : void 0;
1164
+ const toolspaceToken = toolspaceScope && options.toolspaceAuthority ? await mintSandboxToolspaceToken(settings, toolspaceScope, options.toolspaceAuthority) : void 0;
1159
1165
  if (toolspaceToken && toolspaceScope) {
1160
1166
  environment.OPENGENI_TOOLSPACE_URL ??= firstPartyMcpWorkspaceUrl(
1161
1167
  settings,
@@ -1229,6 +1235,7 @@ async function mintRunGitTokensWithIdentity(settings, selections, options) {
1229
1235
  let identity = null;
1230
1236
  for (const selection of selections) {
1231
1237
  let token = null;
1238
+ let transport;
1232
1239
  let tokenExpiresAt;
1233
1240
  if (selection.provider === "github" && selection.installationId > 0 && selection.repositoryIds.length > 0) {
1234
1241
  await options.authorizeGitHubTokenMint?.({
@@ -1252,6 +1259,7 @@ async function mintRunGitTokensWithIdentity(settings, selections, options) {
1252
1259
  );
1253
1260
  }
1254
1261
  token = minted.token;
1262
+ transport = validatedGitCredentialTransport(selection, minted);
1255
1263
  tokenExpiresAt = minted.expiresAt ? validatedGitCredentialExpiry(selection.provider, minted.expiresAt) : void 0;
1256
1264
  if (minted.identity) {
1257
1265
  identity = minted.identity;
@@ -1272,6 +1280,7 @@ async function mintRunGitTokensWithIdentity(settings, selections, options) {
1272
1280
  credentialBindingId: selection.credentialBindingId,
1273
1281
  provider: selection.provider,
1274
1282
  token,
1283
+ ...transport ? { transport } : {},
1275
1284
  ...tokenExpiresAt ? { expiresAt: tokenExpiresAt } : {}
1276
1285
  });
1277
1286
  }
@@ -1284,6 +1293,7 @@ async function mintRunGitTokensWithIdentity(settings, selections, options) {
1284
1293
  }
1285
1294
  for (const binding of bindings) {
1286
1295
  binding.providerBindingCount = counts.get(binding.provider) ?? 1;
1296
+ if (binding.transport?.kind === "http_broker") continue;
1287
1297
  if (counts.get(binding.provider) !== 1) continue;
1288
1298
  gitTokens[binding.provider] = binding.token;
1289
1299
  if (binding.expiresAt) expiresAt[binding.provider] = binding.expiresAt;
@@ -1297,6 +1307,83 @@ function validatedGitCredentialExpiry(provider, value) {
1297
1307
  }
1298
1308
  return new Date(timestamp).toISOString();
1299
1309
  }
1310
+ var MAX_GIT_HTTP_BROKER_ROUTES = 256;
1311
+ var MAX_GIT_HTTP_BROKER_URI_LENGTH = 4096;
1312
+ function validatedGitCredentialTransport(selection, minted) {
1313
+ const transport = minted.transport;
1314
+ if (!transport) return void 0;
1315
+ if (typeof transport !== "object" || transport.kind !== "http_broker" || !Array.isArray(transport.repositories)) {
1316
+ throw new Error(
1317
+ `connection-credential provider (${selection.provider}) returned an invalid Git credential transport`
1318
+ );
1319
+ }
1320
+ const requestedUris = new Set(selection.repositoryRefs.map((repository) => repository.uri));
1321
+ if (requestedUris.size === 0 || requestedUris.size > MAX_GIT_HTTP_BROKER_ROUTES || transport.repositories.length !== requestedUris.size) {
1322
+ throw new Error(
1323
+ `connection-credential provider (${selection.provider}) returned an incomplete Git HTTP broker route set`
1324
+ );
1325
+ }
1326
+ const echoedUris = /* @__PURE__ */ new Set();
1327
+ const brokerUris = /* @__PURE__ */ new Set();
1328
+ const repositories = transport.repositories.map((route) => {
1329
+ if (!route || typeof route !== "object" || typeof route.repositoryUri !== "string" || typeof route.brokerUri !== "string") {
1330
+ throw new Error(
1331
+ `connection-credential provider (${selection.provider}) returned an invalid Git HTTP broker route`
1332
+ );
1333
+ }
1334
+ if (!requestedUris.has(route.repositoryUri) || echoedUris.has(route.repositoryUri)) {
1335
+ throw new Error(
1336
+ `connection-credential provider (${selection.provider}) returned an unexpected Git HTTP broker repository`
1337
+ );
1338
+ }
1339
+ echoedUris.add(route.repositoryUri);
1340
+ if (route.brokerUri.length === 0 || route.brokerUri.length > MAX_GIT_HTTP_BROKER_URI_LENGTH || route.brokerUri.trim() !== route.brokerUri) {
1341
+ throw new Error(
1342
+ `connection-credential provider (${selection.provider}) returned an invalid Git HTTP broker URI`
1343
+ );
1344
+ }
1345
+ let brokerUrl;
1346
+ try {
1347
+ brokerUrl = new URL(route.brokerUri);
1348
+ } catch {
1349
+ throw new Error(
1350
+ `connection-credential provider (${selection.provider}) returned an invalid Git HTTP broker URI`
1351
+ );
1352
+ }
1353
+ if (brokerUrl.protocol !== "https:" || brokerUrl.username || brokerUrl.password || brokerUrl.search || brokerUrl.hash || brokerUris.has(brokerUrl.href) || brokerUrl.href === route.repositoryUri) {
1354
+ throw new Error(
1355
+ `connection-credential provider (${selection.provider}) returned an unsafe Git HTTP broker URI`
1356
+ );
1357
+ }
1358
+ brokerUris.add(brokerUrl.href);
1359
+ return {
1360
+ repositoryUri: route.repositoryUri,
1361
+ brokerUri: brokerUrl.href
1362
+ };
1363
+ });
1364
+ if ([...requestedUris].some((uri) => !echoedUris.has(uri))) {
1365
+ throw new Error(
1366
+ `connection-credential provider (${selection.provider}) returned an incomplete Git HTTP broker route set`
1367
+ );
1368
+ }
1369
+ return { kind: "http_broker", repositories };
1370
+ }
1371
+ function assertGitCredentialRenewalTransportUnchanged(initial, renewed) {
1372
+ if (!initial.transport && !renewed.transport) return;
1373
+ if (!initial.transport || !renewed.transport || initial.transport.kind !== renewed.transport.kind) {
1374
+ throw new Error(
1375
+ `renewed ${initial.provider} credential changed transport for binding ${initial.credentialBindingId}`
1376
+ );
1377
+ }
1378
+ const routeKey = (route) => `${route.repositoryUri}\0${route.brokerUri}`;
1379
+ const initialRoutes = initial.transport.repositories.map(routeKey).sort();
1380
+ const renewedRoutes = renewed.transport.repositories.map(routeKey).sort();
1381
+ if (initialRoutes.length !== renewedRoutes.length || initialRoutes.some((route, index) => route !== renewedRoutes[index])) {
1382
+ throw new Error(
1383
+ `renewed ${initial.provider} credential changed Git HTTP broker routes for binding ${initial.credentialBindingId}`
1384
+ );
1385
+ }
1386
+ }
1300
1387
  async function resolveRunGitIdentityWithSelections(settings, selections, options) {
1301
1388
  let identity = null;
1302
1389
  for (const selection of selections) {
@@ -3621,10 +3708,13 @@ function makeTurnOpJournal(context, details) {
3621
3708
  import { createHash } from "crypto";
3622
3709
  import { errorCodeToJSON } from "@opengeni/agent-proto";
3623
3710
  import {
3624
- SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES
3711
+ SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
3712
+ modelUsageTokenCountOrNull
3625
3713
  } from "@opengeni/runtime";
3626
3714
  var turnTrackers = /* @__PURE__ */ new WeakMap();
3627
3715
  var creditBalanceGaugeAccounts = /* @__PURE__ */ new WeakMap();
3716
+ var modelCacheCounterTotals = /* @__PURE__ */ new WeakMap();
3717
+ var MAX_MODEL_CACHE_COUNTER_TOTAL = 1e12;
3628
3718
  function observabilityEventLogger(observability) {
3629
3719
  return {
3630
3720
  debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),
@@ -4004,7 +4094,8 @@ var MODEL_INPUT_TOKENS_BUCKETS = [
4004
4094
  1e6
4005
4095
  ];
4006
4096
  function recordModelInputTokens(observability, provider, inputTokens) {
4007
- if (!(inputTokens > 0)) {
4097
+ const normalizedInputTokens = modelUsageTokenCountOrNull(inputTokens);
4098
+ if (normalizedInputTokens === null || normalizedInputTokens === 0) {
4008
4099
  return;
4009
4100
  }
4010
4101
  observability.observeHistogram({
@@ -4012,7 +4103,7 @@ function recordModelInputTokens(observability, provider, inputTokens) {
4012
4103
  help: "Observed input (context) tokens per model response, by provider.",
4013
4104
  buckets: MODEL_INPUT_TOKENS_BUCKETS,
4014
4105
  labels: { provider },
4015
- value: inputTokens
4106
+ value: normalizedInputTokens
4016
4107
  });
4017
4108
  }
4018
4109
  function recordContextCompaction(observability, trigger) {
@@ -4038,17 +4129,35 @@ var MODEL_CACHE_HIT_RATIO_BUCKETS = [
4038
4129
  1
4039
4130
  ];
4040
4131
  function recordModelCacheTokens(observability, provider, input) {
4041
- const cached = nonNegativeTokenCount(input.cachedTokens);
4042
- const prompt = nonNegativeTokenCount(input.promptTokens);
4043
- if (cached > 0) {
4044
- observability.incrementCounter({
4132
+ const cached = modelUsageTokenCountOrNull(input.cachedTokens);
4133
+ const cacheWrite = modelUsageTokenCountOrNull(input.cacheWriteTokens);
4134
+ const prompt = modelUsageTokenCountOrNull(input.promptTokens);
4135
+ incrementBoundedModelCacheCounter(observability, {
4136
+ name: "opengeni_model_cache_read_telemetry_total",
4137
+ help: "Authoritative model calls with reported or missing cache-read telemetry, by provider.",
4138
+ provider,
4139
+ labels: { provider, status: cached === null ? "missing" : "reported" },
4140
+ amount: 1
4141
+ });
4142
+ if (cached !== null && cached > 0) {
4143
+ incrementBoundedModelCacheCounter(observability, {
4045
4144
  name: "opengeni_model_cached_tokens_total",
4046
4145
  help: "Total prompt tokens served from the provider's prompt cache, by provider.",
4146
+ provider,
4047
4147
  labels: { provider },
4048
4148
  amount: cached
4049
4149
  });
4050
4150
  }
4051
- if (prompt > 0) {
4151
+ if (cacheWrite !== null && cacheWrite > 0) {
4152
+ incrementBoundedModelCacheCounter(observability, {
4153
+ name: "opengeni_model_cache_write_tokens_total",
4154
+ help: "Total prompt tokens written to the provider's prompt cache, by provider.",
4155
+ provider,
4156
+ labels: { provider },
4157
+ amount: cacheWrite
4158
+ });
4159
+ }
4160
+ if (cached !== null && prompt !== null && prompt > 0) {
4052
4161
  observability.observeHistogram({
4053
4162
  name: "opengeni_model_cache_hit_ratio",
4054
4163
  help: "Per-call prompt-cache hit ratio (cached/prompt tokens) by provider.",
@@ -4060,8 +4169,28 @@ function recordModelCacheTokens(observability, provider, input) {
4060
4169
  });
4061
4170
  }
4062
4171
  }
4063
- function nonNegativeTokenCount(value) {
4064
- return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0;
4172
+ function incrementBoundedModelCacheCounter(observability, input) {
4173
+ const totals = modelCacheCounterTotals.get(observability) ?? /* @__PURE__ */ new Map();
4174
+ if (!modelCacheCounterTotals.has(observability)) {
4175
+ modelCacheCounterTotals.set(observability, totals);
4176
+ }
4177
+ const labelKey = Object.entries(input.labels).sort(([left], [right]) => left.localeCompare(right)).map(([key2, value]) => `${key2}=${value}`).join(",");
4178
+ const key = `${input.name}:${labelKey}`;
4179
+ const current = totals.get(key) ?? 0;
4180
+ if (input.amount > MAX_MODEL_CACHE_COUNTER_TOTAL - current) {
4181
+ observability.warn("model cache metric cumulative limit reached", {
4182
+ provider: input.provider,
4183
+ metric: input.name
4184
+ });
4185
+ return;
4186
+ }
4187
+ observability.incrementCounter({
4188
+ name: input.name,
4189
+ help: input.help,
4190
+ labels: input.labels,
4191
+ amount: input.amount
4192
+ });
4193
+ totals.set(key, current + input.amount);
4065
4194
  }
4066
4195
  function stableAccountHash(credentialId) {
4067
4196
  if (!credentialId) {
@@ -4939,10 +5068,27 @@ import {
4939
5068
  } from "@opengeni/runtime";
4940
5069
  import {
4941
5070
  CAPABILITY_DESCRIPTORS,
4942
- evaluateWorkspaceModelPolicy as evaluateWorkspaceModelPolicy2
5071
+ evaluateWorkspaceModelPolicy as evaluateWorkspaceModelPolicy2,
5072
+ readTurnExecutionPolicyV1
4943
5073
  } from "@opengeni/contracts";
4944
5074
  import { createHash as createHash3, randomUUID } from "crypto";
4945
5075
  var PROVIDER_BACKPRESSURE_DELAY_MS = 6e4;
5076
+ function turnExecutionPolicyBillingIdentity(policy) {
5077
+ return {
5078
+ externallyBilled: policy.billing.metering === "external",
5079
+ codexSubscription: policy.providerId === "codex-subscription" && policy.credentialSource.kind === "connected_subscription" && policy.credentialSource.provider === "codex"
5080
+ };
5081
+ }
5082
+ function legacyTurnExecutionPolicyInput(turn) {
5083
+ const explicit = turn.source === "user" || turn.source === "api";
5084
+ return {
5085
+ modelId: turn.model,
5086
+ requestedModelId: explicit ? turn.model : null,
5087
+ modelSource: explicit ? "explicit" : "continuation",
5088
+ reasoningEffort: turn.reasoningEffort,
5089
+ reasoningSource: explicit ? "explicit" : "continuation"
5090
+ };
5091
+ }
4946
5092
  function providerRecoveryResult() {
4947
5093
  return {
4948
5094
  status: "recovering",
@@ -5147,18 +5293,6 @@ function modelUsageSourceKey(input) {
5147
5293
  }
5148
5294
  return input.dispatchId ? `${input.dispatchId}:${input.positionalKey}` : input.positionalKey;
5149
5295
  }
5150
- function providerContextTokens(usage) {
5151
- const total = usage?.totalTokens;
5152
- if (typeof total === "number" && Number.isFinite(total) && total > 0) {
5153
- return total;
5154
- }
5155
- const input = usage?.inputTokens;
5156
- if (typeof input !== "number" || !Number.isFinite(input) || input <= 0) {
5157
- return null;
5158
- }
5159
- const output = usage?.outputTokens;
5160
- return input + (typeof output === "number" && Number.isFinite(output) && output > 0 ? output : 0);
5161
- }
5162
5296
  async function recordCompletedModelCallBeforeOwnershipFences(input) {
5163
5297
  await input.renewLease();
5164
5298
  await input.recordUsage();
@@ -5167,13 +5301,167 @@ async function recordCompletedModelCallBeforeOwnershipFences(input) {
5167
5301
  }
5168
5302
  await input.recordAttemptSignals?.();
5169
5303
  }
5304
+ function createModelResponseUsageEventState(claimedSourceKeys = /* @__PURE__ */ new Set()) {
5305
+ return {
5306
+ responseUsageCount: 0,
5307
+ providerContextRevision: 0,
5308
+ lastProviderContextTokensObserved: null,
5309
+ claimedSourceKeys
5310
+ };
5311
+ }
5312
+ function createCompactionModelUsageEventState(claimedSourceKeys = /* @__PURE__ */ new Set()) {
5313
+ return { usageCount: 0, claimedSourceKeys };
5314
+ }
5315
+ function modelResponseUsageContextSignal(state) {
5316
+ return state.lastProviderContextTokensObserved === null ? null : {
5317
+ revision: state.providerContextRevision,
5318
+ totalTokens: state.lastProviderContextTokensObserved
5319
+ };
5320
+ }
5321
+ async function processModelResponseUsageEvent(input) {
5322
+ const responseUsage = modelResponseUsageFromSdkEvent(input.event);
5323
+ if (!responseUsage) {
5324
+ return { status: "not_usage" };
5325
+ }
5326
+ const responseOrdinal = input.state.responseUsageCount + 1;
5327
+ const sourceKey = modelUsageSourceKey({
5328
+ responseId: responseUsage.responseId,
5329
+ dispatchId: input.dispatchId,
5330
+ positionalKey: `response-${responseOrdinal}`
5331
+ });
5332
+ if (input.state.claimedSourceKeys.has(sourceKey)) {
5333
+ return { status: "duplicate", sourceKey };
5334
+ }
5335
+ input.state.claimedSourceKeys.add(sourceKey);
5336
+ input.state.responseUsageCount = responseOrdinal;
5337
+ const normalizedUsage = normalizeModelCallUsage(responseUsage.usage);
5338
+ const accountContext = modelCallAccountContext({
5339
+ servingCredentialId: input.servingCredentialId,
5340
+ priorSessionCredentialId: input.priorSessionCredentialId,
5341
+ isFirstCallOfTurn: responseOrdinal === 1
5342
+ });
5343
+ let authoritative = false;
5344
+ await recordCompletedModelCallBeforeOwnershipFences({
5345
+ renewLease: input.renewLease,
5346
+ leaseLost: input.leaseLost,
5347
+ leaseLostMessage: input.leaseLostMessage,
5348
+ recordUsage: async () => {
5349
+ await recordModelUsageAndDebitCredits(input.settings, input.db, {
5350
+ accountId: input.accountId,
5351
+ workspaceId: input.workspaceId,
5352
+ sessionId: input.sessionId,
5353
+ turnId: input.turnId,
5354
+ turnAttemptId: input.turnAttemptId,
5355
+ model: input.model,
5356
+ externallyBilled: input.externallyBilled,
5357
+ usage: responseUsage.usage,
5358
+ normalizedUsage,
5359
+ sourceKey,
5360
+ observability: input.observability
5361
+ });
5362
+ authoritative = await emitModelCallUsage({
5363
+ observability: input.observability,
5364
+ publish: input.publish,
5365
+ accountId: input.accountId,
5366
+ workspaceId: input.workspaceId,
5367
+ sessionId: input.sessionId,
5368
+ turnId: input.turnId,
5369
+ provider: input.provider,
5370
+ providerApi: input.providerApi,
5371
+ model: input.model,
5372
+ sourceKey,
5373
+ usage: responseUsage,
5374
+ normalizedUsage,
5375
+ servingAccountHash: accountContext.servingAccountHash,
5376
+ accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
5377
+ emittedSourceKeys: input.emittedSourceKeys
5378
+ });
5379
+ const observedInput = normalizedUsage.telemetry.inputTokens;
5380
+ if (authoritative && observedInput !== null && observedInput > 0) {
5381
+ recordModelInputTokens(input.observability, input.metricProvider, observedInput);
5382
+ }
5383
+ },
5384
+ recordAttemptSignals: async () => {
5385
+ if (!authoritative) return;
5386
+ const observedTotal = normalizedUsage.totalTokens;
5387
+ if (observedTotal !== null) {
5388
+ input.state.lastProviderContextTokensObserved = observedTotal;
5389
+ input.state.providerContextRevision += 1;
5390
+ }
5391
+ const observedInput = normalizedUsage.telemetry.inputTokens;
5392
+ if (observedInput !== null && observedInput > 0) {
5393
+ await input.setLastInputTokens(observedInput);
5394
+ }
5395
+ }
5396
+ });
5397
+ return { status: "processed", sourceKey, authoritative };
5398
+ }
5399
+ async function processCompactionModelUsageEvent(input) {
5400
+ const usageOrdinal = input.state.usageCount + 1;
5401
+ const sourceKey = modelUsageSourceKey({
5402
+ responseId: input.usage.responseId,
5403
+ dispatchId: input.dispatchId,
5404
+ positionalKey: `compaction-${usageOrdinal}`
5405
+ });
5406
+ if (input.state.claimedSourceKeys.has(sourceKey)) {
5407
+ return { status: "duplicate", sourceKey };
5408
+ }
5409
+ input.state.claimedSourceKeys.add(sourceKey);
5410
+ input.state.usageCount = usageOrdinal;
5411
+ const accountContext = modelCallAccountContext({
5412
+ servingCredentialId: input.servingCredentialId,
5413
+ priorSessionCredentialId: input.priorSessionCredentialId,
5414
+ isFirstCallOfTurn: usageOrdinal === 1
5415
+ });
5416
+ const normalizedUsage = normalizeModelCallUsage(input.usage.usage);
5417
+ let authoritative = false;
5418
+ await recordCompletedModelCallBeforeOwnershipFences({
5419
+ renewLease: input.renewLease,
5420
+ leaseLost: input.leaseLost,
5421
+ leaseLostMessage: input.leaseLostMessage,
5422
+ recordUsage: async () => {
5423
+ await recordModelUsageAndDebitCredits(input.settings, input.db, {
5424
+ accountId: input.accountId,
5425
+ workspaceId: input.workspaceId,
5426
+ sessionId: input.sessionId,
5427
+ turnId: input.turnId,
5428
+ turnAttemptId: input.turnAttemptId,
5429
+ model: input.model,
5430
+ externallyBilled: input.externallyBilled,
5431
+ usage: input.usage.usage,
5432
+ normalizedUsage,
5433
+ sourceKey,
5434
+ observability: input.observability
5435
+ });
5436
+ authoritative = await emitModelCallUsage({
5437
+ observability: input.observability,
5438
+ publish: input.publish,
5439
+ accountId: input.accountId,
5440
+ workspaceId: input.workspaceId,
5441
+ sessionId: input.sessionId,
5442
+ turnId: input.turnId,
5443
+ provider: input.provider,
5444
+ providerApi: input.providerApi,
5445
+ model: input.model,
5446
+ sourceKey,
5447
+ usage: input.usage,
5448
+ normalizedUsage,
5449
+ servingAccountHash: accountContext.servingAccountHash,
5450
+ accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
5451
+ emittedSourceKeys: input.emittedSourceKeys
5452
+ });
5453
+ }
5454
+ });
5455
+ return { status: "processed", sourceKey, authoritative };
5456
+ }
5170
5457
  async function emitModelCallUsage(input) {
5171
5458
  const usage = input.usage && typeof input.usage === "object" && "usage" in input.usage ? input.usage.usage : null;
5172
5459
  if (!usage || typeof usage !== "object") {
5173
- return;
5460
+ return false;
5174
5461
  }
5175
- if (input.emittedSourceKeys?.has(input.sourceKey)) return;
5176
- const telemetry = modelCallUsageTelemetry(usage);
5462
+ if (input.emittedSourceKeys?.has(input.sourceKey)) return false;
5463
+ const normalizedUsage = input.normalizedUsage ?? normalizeModelCallUsage(usage);
5464
+ const telemetry = normalizedUsage.telemetry;
5177
5465
  const appended = await input.publish?.(
5178
5466
  [
5179
5467
  {
@@ -5197,7 +5485,7 @@ async function emitModelCallUsage(input) {
5197
5485
  const authoritative = appended?.events.some(
5198
5486
  (event) => event.type === "agent.model.usage" && event.turnAssociation === "current" && event.payload !== null && typeof event.payload === "object" && event.payload.sourceKey === input.sourceKey
5199
5487
  );
5200
- if (!authoritative) return;
5488
+ if (!authoritative) return false;
5201
5489
  try {
5202
5490
  input.observability.info("model call usage", {
5203
5491
  accountId: input.accountId,
@@ -5211,12 +5499,34 @@ async function emitModelCallUsage(input) {
5211
5499
  inputTokens: telemetry.inputTokens,
5212
5500
  outputTokens: telemetry.outputTokens,
5213
5501
  cachedTokens: telemetry.cachedTokens,
5502
+ cacheWriteTokens: telemetry.cacheWriteTokens,
5214
5503
  reasoningTokens: telemetry.reasoningTokens,
5215
5504
  ...input.servingAccountHash !== void 0 ? { servingAccountHash: input.servingAccountHash } : {},
5216
5505
  ...input.accountChangedFromPrevCall !== void 0 ? { accountChangedFromPrevCall: input.accountChangedFromPrevCall } : {}
5217
5506
  });
5507
+ if (normalizedUsage.rejectedFields.length > 0) {
5508
+ input.observability.warn("model call usage fields rejected", {
5509
+ provider: input.provider,
5510
+ providerApi: input.providerApi,
5511
+ model: input.model,
5512
+ sourceKey: input.sourceKey,
5513
+ rejectedFields: normalizedUsage.rejectedFields.join(",")
5514
+ });
5515
+ }
5516
+ } catch {
5517
+ }
5518
+ try {
5519
+ applyCodexCacheTelemetry(input.observability, input.provider, normalizedUsage);
5218
5520
  } catch {
5219
5521
  }
5522
+ return true;
5523
+ }
5524
+ function applyCodexCacheTelemetry(observability, provider, normalizedUsage) {
5525
+ recordModelCacheTokens(observability, provider, {
5526
+ cachedTokens: normalizedUsage.telemetry.cachedTokens,
5527
+ cacheWriteTokens: normalizedUsage.telemetry.cacheWriteTokens,
5528
+ promptTokens: normalizedUsage.telemetry.inputTokens
5529
+ });
5220
5530
  }
5221
5531
  function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = persistedHistoryCount, toolOutputTruncationTokens) {
5222
5532
  const sanitized = sanitizeHistoryItemsForModel2(rawHistory, toolOutputTruncationTokens).filter(
@@ -5575,6 +5885,7 @@ function createRunAgentTurnActivity(services) {
5575
5885
  runtimeMetricsHooksForObservability(observability)
5576
5886
  );
5577
5887
  let isCodexTurn = false;
5888
+ let isExternallyBilledTurn = false;
5578
5889
  let executionGeneration = 0;
5579
5890
  let redispatchesAtDispatch = 0;
5580
5891
  const setLastInputTokensFenced = async (lastInputTokens) => {
@@ -5998,6 +6309,35 @@ function createRunAgentTurnActivity(services) {
5998
6309
  }
5999
6310
  const turn = claim.turn;
6000
6311
  turnId = turn.id;
6312
+ executionGeneration = turn.executionGeneration;
6313
+ const claimedPolicy = readTurnExecutionPolicyV1(turn.metadata);
6314
+ const policyForAbsent = claimedPolicy.kind === "valid" ? claimedPolicy.policy : resolveTurnExecutionPolicyV1(capabilitySettings, legacyTurnExecutionPolicyInput(turn));
6315
+ const installedPolicy = await installOrReadTurnExecutionPolicyForAttempt(db, {
6316
+ accountId: input.accountId,
6317
+ workspaceId: input.workspaceId,
6318
+ sessionId: input.sessionId,
6319
+ turnId,
6320
+ executionGeneration,
6321
+ attemptId: input.attemptId,
6322
+ policyForAbsent
6323
+ });
6324
+ if (!installedPolicy.accepted) {
6325
+ throw new TurnAttemptFencedError(
6326
+ `turn execution policy was fenced: ${installedPolicy.reason}`
6327
+ );
6328
+ }
6329
+ const verifiedExecutionPolicy = assertTurnExecutionPolicyMatchesConfigV1(
6330
+ capabilitySettings,
6331
+ installedPolicy.policy,
6332
+ {
6333
+ modelId: turn.model,
6334
+ reasoningEffort: turn.reasoningEffort
6335
+ }
6336
+ );
6337
+ const turnExecutionPolicy = verifiedExecutionPolicy.policy;
6338
+ const billingIdentity = turnExecutionPolicyBillingIdentity(turnExecutionPolicy);
6339
+ isExternallyBilledTurn = billingIdentity.externallyBilled;
6340
+ isCodexTurn = billingIdentity.codexSubscription;
6001
6341
  triggerEventId = turn.triggerEventId;
6002
6342
  const trigger = await getSessionEvent2(db, input.workspaceId, triggerEventId);
6003
6343
  if (!trigger) {
@@ -6010,27 +6350,19 @@ function createRunAgentTurnActivity(services) {
6010
6350
  trigger
6011
6351
  );
6012
6352
  triggerType = trigger.type;
6013
- executionGeneration = turn.executionGeneration;
6014
6353
  const latestTurnState = await getLatestRunState2(db, input.workspaceId, input.sessionId);
6015
6354
  const continuationCodexCredentialId = latestTurnState?.turnId === turnId ? latestTurnState.frozenCodexCredentialId : null;
6016
6355
  redispatchesAtDispatch = Number(
6017
6356
  turn.metadata?.workerDeathRedispatches ?? 0
6018
6357
  );
6019
6358
  turnLifecycleMetricsFor(observability).start(turnId);
6020
- isCodexTurn = await isCodexBilledTurn2({
6021
- db,
6022
- settings,
6023
- workspaceId: input.workspaceId,
6024
- model: turn.model,
6025
- active: codexSubscriptionActive
6026
- });
6027
6359
  await waitForTurnOperation(
6028
6360
  ensureRunAllowed(
6029
6361
  settings,
6030
6362
  db,
6031
6363
  input.accountId,
6032
6364
  input.workspaceId,
6033
- isCodexTurn,
6365
+ isExternallyBilledTurn,
6034
6366
  entitlements
6035
6367
  ),
6036
6368
  cancellationSignal,
@@ -6055,6 +6387,7 @@ function createRunAgentTurnActivity(services) {
6055
6387
  const producerId = `${input.workflowId}:${turnId}${activityContext ? `:${activityContext.info.activityId}` : ""}`;
6056
6388
  codexLeaseHolderId = dispatchId;
6057
6389
  const modelUsageDispatchId = activityContext?.info.activityId ?? dispatchId;
6390
+ const claimedModelUsageSourceKeys = /* @__PURE__ */ new Set();
6058
6391
  const emittedModelUsageSourceKeys = /* @__PURE__ */ new Set();
6059
6392
  publish = async (events, immediate = false) => {
6060
6393
  const inputs = events.map((event) => ({
@@ -6685,27 +7018,31 @@ function createRunAgentTurnActivity(services) {
6685
7018
  db,
6686
7019
  input.workspaceId,
6687
7020
  input.sessionId,
7021
+ input.attemptId,
6688
7022
  baseRunSettings
6689
7023
  );
6690
- const resolvedModel = runtime.resolveTurnModel(capabilitySettings, turn.model);
7024
+ const resolvedModel = runtime.resolveTurnModel(
7025
+ capabilitySettings,
7026
+ turnExecutionPolicy.productModelId
7027
+ );
6691
7028
  modelRunSettings = resolvedModel ? settingsWithResolvedModelContext(runSettings, resolvedModel.configured) : runSettings;
6692
7029
  {
6693
7030
  const workspaceModelPolicy = await getWorkspaceModelPolicy2(db, input.workspaceId);
6694
7031
  if (workspaceModelPolicy) {
6695
- const effectiveProviderId = resolvedModel ? resolvedModel.provider.id : builtinProviderId(capabilitySettings);
6696
7032
  const verdict = evaluateWorkspaceModelPolicy2(workspaceModelPolicy, {
6697
- providerId: effectiveProviderId,
6698
- modelId: turn.model
7033
+ providerId: turnExecutionPolicy.providerId,
7034
+ modelId: turnExecutionPolicy.productModelId
6699
7035
  });
6700
7036
  if (!verdict.allowed) {
6701
7037
  throw new WorkspaceModelPolicyBlockedError(
6702
- turn.model,
6703
- effectiveProviderId,
7038
+ turnExecutionPolicy.productModelId,
7039
+ turnExecutionPolicy.providerId,
6704
7040
  verdict.reason
6705
7041
  );
6706
7042
  }
6707
7043
  }
6708
7044
  }
7045
+ let codexModelRequestSequence = 0;
6709
7046
  const codexContext = resolvedModel?.provider.kind === "codex-subscription" ? (() => {
6710
7047
  const resolver = buildCodexTokenResolver(
6711
7048
  db,
@@ -6730,72 +7067,72 @@ function createRunAgentTurnActivity(services) {
6730
7067
  ),
6731
7068
  onUsageHeaders: (snapshot) => {
6732
7069
  latestCodexUsage = snapshot;
6733
- }
7070
+ },
6734
7071
  // latest wins; flushed once in finally
7072
+ nextRequestId: () => `${dispatchId}:${++codexModelRequestSequence}`,
7073
+ onModelRequestEvent: async (event) => {
7074
+ if (!publish || !turnId) {
7075
+ throw new Error("Codex model request started before the turn event producer");
7076
+ }
7077
+ await publish([
7078
+ {
7079
+ type: "agent.model.request",
7080
+ payload: {
7081
+ ...event,
7082
+ provider: "codex-subscription",
7083
+ turnId,
7084
+ attemptId: input.attemptId,
7085
+ dispatchId,
7086
+ executionGeneration
7087
+ }
7088
+ }
7089
+ ]);
7090
+ }
6735
7091
  };
6736
7092
  })() : null;
6737
7093
  const withCodex = (fn) => codexContext ? codexRequestStorage.run(codexContext, fn) : fn();
6738
7094
  const promptCacheKey = acceptsPromptCacheKeyForTurn(resolvedModel) ? input.sessionId : void 0;
6739
- let compactionUsageCount = 0;
7095
+ const compactionUsageState = createCompactionModelUsageEventState(
7096
+ claimedModelUsageSourceKeys
7097
+ );
6740
7098
  const recordCompactionUsage = async (usage) => {
6741
- await recordCompletedModelCallBeforeOwnershipFences({
7099
+ await processCompactionModelUsageEvent({
7100
+ usage,
7101
+ state: compactionUsageState,
7102
+ dispatchId: modelUsageDispatchId,
7103
+ settings,
7104
+ db,
7105
+ observability,
7106
+ publish,
7107
+ accountId: input.accountId,
7108
+ workspaceId: input.workspaceId,
7109
+ sessionId: input.sessionId,
7110
+ turnId: turn.id,
7111
+ provider: resolvedModel?.provider.id ?? settings.openaiProvider,
7112
+ providerApi: resolvedModel?.provider.api ?? "responses",
7113
+ model: resolvedModel?.configured.id ?? turn.model,
7114
+ externallyBilled: isExternallyBilledTurn,
7115
+ turnAttemptId: input.attemptId,
7116
+ servingCredentialId: effectiveCodexCredentialId,
7117
+ priorSessionCredentialId: priorSessionCodexCredentialId,
7118
+ emittedSourceKeys: emittedModelUsageSourceKeys,
6742
7119
  renewLease: () => renewCodexLease("model_usage"),
6743
7120
  leaseLost: () => codexLeaseLost,
6744
- leaseLostMessage: "Codex credential lease expired during context compaction",
6745
- recordUsage: async () => {
6746
- compactionUsageCount += 1;
6747
- const sourceKey = modelUsageSourceKey({
6748
- responseId: usage.responseId,
6749
- dispatchId: modelUsageDispatchId,
6750
- positionalKey: `compaction-${compactionUsageCount}`
6751
- });
6752
- const responseAccountCtx = modelCallAccountContext({
6753
- servingCredentialId: effectiveCodexCredentialId,
6754
- priorSessionCredentialId: priorSessionCodexCredentialId,
6755
- isFirstCallOfTurn: compactionUsageCount === 1
6756
- });
6757
- await recordModelUsageAndDebitCredits(settings, db, {
6758
- accountId: input.accountId,
6759
- workspaceId: input.workspaceId,
6760
- sessionId: input.sessionId,
6761
- turnId: turn.id,
6762
- turnAttemptId: input.attemptId,
6763
- model: resolvedModel?.configured.id ?? turn.model,
6764
- isCodexTurn,
6765
- usage: usage.usage,
6766
- sourceKey,
6767
- observability
6768
- });
6769
- await emitModelCallUsage({
6770
- observability,
6771
- publish,
6772
- accountId: input.accountId,
6773
- workspaceId: input.workspaceId,
6774
- sessionId: input.sessionId,
6775
- turnId: turn.id,
6776
- provider: resolvedModel?.provider.id ?? settings.openaiProvider,
6777
- providerApi: resolvedModel?.provider.api ?? "responses",
6778
- model: resolvedModel?.configured.id ?? turn.model,
6779
- sourceKey,
6780
- usage,
6781
- servingAccountHash: responseAccountCtx.servingAccountHash,
6782
- accountChangedFromPrevCall: responseAccountCtx.accountChangedFromPrevCall,
6783
- emittedSourceKeys: emittedModelUsageSourceKeys
6784
- });
6785
- }
7121
+ leaseLostMessage: "Codex credential lease expired during context compaction"
6786
7122
  });
6787
7123
  };
6788
7124
  const compactionSummarizerFor = (systemInstructions) => resolvedModel ? (s, m) => withCodex(
6789
7125
  () => summarizeForCompaction2(s, m, {
6790
7126
  client: resolvedModel.client,
6791
7127
  api: resolvedModel.provider.api,
6792
- model: resolvedModel.configured.id,
7128
+ model: turnExecutionPolicy.upstreamModelId,
6793
7129
  maxOutputTokens: SUMMARY_BUFFER_TOKENS2,
6794
7130
  onUsage: recordCompactionUsage,
6795
7131
  ...systemInstructions ? { systemInstructions } : {},
6796
7132
  ...promptCacheKey ? { promptCacheKey } : {}
6797
7133
  })
6798
7134
  ) : (s, m) => summarizeForCompaction2(s, m, {
7135
+ model: turnExecutionPolicy.upstreamModelId,
6799
7136
  maxOutputTokens: SUMMARY_BUFFER_TOKENS2,
6800
7137
  onUsage: recordCompactionUsage,
6801
7138
  ...systemInstructions ? { systemInstructions } : {},
@@ -6917,9 +7254,19 @@ function createRunAgentTurnActivity(services) {
6917
7254
  return claimedResult({ status: "idle" });
6918
7255
  }
6919
7256
  const turnResources = mergeResourceRefs(session.resources, turn.resources);
7257
+ const effectivePolicyTools = resolveSessionToolPolicy({
7258
+ ...session.toolPolicy ? { toolPolicy: session.toolPolicy } : {},
7259
+ sessionTools: session.tools,
7260
+ turnTools: turn.tools,
7261
+ ...turn.toolsProvided !== void 0 ? { turnToolsProvided: turn.toolsProvided } : {},
7262
+ availableMcpServerIds: runSettings.mcpServers.map((server) => server.id),
7263
+ defaultMcpServerIds: enabledCapabilityMcpToolRefs(settings, mcpSettings).map(
7264
+ (tool) => tool.id
7265
+ )
7266
+ }).toolRefs;
6920
7267
  const turnTools = withCodexAppsTool(
6921
7268
  runSettings,
6922
- withFirstPartyTools(runSettings, mergeToolRefs(session.tools, turn.tools))
7269
+ withFirstPartyTools(runSettings, effectivePolicyTools)
6923
7270
  );
6924
7271
  const connectionScope = {
6925
7272
  accountId: input.accountId,
@@ -7053,6 +7400,9 @@ function createRunAgentTurnActivity(services) {
7053
7400
  attemptId: input.attemptId,
7054
7401
  turn
7055
7402
  }) : void 0;
7403
+ const toolspaceAuthority = {
7404
+ sessionId: input.sessionId
7405
+ };
7056
7406
  const {
7057
7407
  environment: sandboxEnvironment,
7058
7408
  gitToken: sandboxGitToken,
@@ -7075,8 +7425,7 @@ function createRunAgentTurnActivity(services) {
7075
7425
  ...gitCredentialAuthority ? { authority: gitCredentialAuthority } : {},
7076
7426
  gitCredentials: connectionCredentials?.gitCredentials,
7077
7427
  authorizeGitHubTokenMint,
7078
- sessionId: input.sessionId,
7079
- runId: turnId
7428
+ toolspaceAuthority
7080
7429
  }
7081
7430
  ),
7082
7431
  cancellationSignal,
@@ -7112,6 +7461,9 @@ function createRunAgentTurnActivity(services) {
7112
7461
  authorizeGitHubTokenMint
7113
7462
  }
7114
7463
  );
7464
+ if (binding) {
7465
+ assertGitCredentialRenewalTransportUnchanged(initialBinding, binding);
7466
+ }
7115
7467
  pendingBinding = binding;
7116
7468
  return binding ? {
7117
7469
  bindings: [binding],
@@ -7172,7 +7524,7 @@ function createRunAgentTurnActivity(services) {
7172
7524
  toolspaceTokenRenewal = null;
7173
7525
  await previous?.stop();
7174
7526
  if (toolspaceTokenRenewalClosed) return;
7175
- const mint = async () => await mintSandboxToolspaceToken(runSettings, connectionScope, input.sessionId, turn.id);
7527
+ const mint = async () => await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority);
7176
7528
  const write = async (material) => {
7177
7529
  const runAs = sandboxRunAs(runSettings);
7178
7530
  await refreshToolspaceTokenFile(tokenSession, material.token, {
@@ -7544,9 +7896,10 @@ function createRunAgentTurnActivity(services) {
7544
7896
  // token skip does NOT transfer: that token is inert on a connected
7545
7897
  // machine (it uses its own git creds), but the toolspace token is the
7546
7898
  // machine's ONLY path to programmatic tool calling and grants no more
7547
- // than toolspace:call for its own session (own-session-bound, turn TTL,
7548
- // budgeted, approval-tools excluded). The runtime seeds it to the box's
7549
- // token file over the same exec channel, off-manifest, on every backend.
7899
+ // than toolspace:call for its own session (own-session-bound,
7900
+ // short-lived and renewable, per-active-turn budgeted, approval-tools
7901
+ // excluded). The runtime seeds it to the box's token file over the same
7902
+ // exec channel, off-manifest, on every backend.
7550
7903
  ...sandboxToolspaceToken ? {
7551
7904
  toolspaceTokenSeed: sandboxToolspaceToken,
7552
7905
  toolspaceTokenSessionId: input.sessionId
@@ -7669,12 +8022,7 @@ function createRunAgentTurnActivity(services) {
7669
8022
  authorizeGitHubTokenMint
7670
8023
  });
7671
8024
  const lazyGitTokens = lazyGitCredentials?.gitTokens;
7672
- const lazyToolspaceToken = sandboxToolspaceToken ? await mintSandboxToolspaceToken(
7673
- runSettings,
7674
- connectionScope,
7675
- input.sessionId,
7676
- turn.id
7677
- ) : void 0;
8025
+ const lazyToolspaceToken = sandboxToolspaceToken ? await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority) : void 0;
7678
8026
  const provisioned = await resumeBoxForTurn(
7679
8027
  {
7680
8028
  db,
@@ -8013,17 +8361,17 @@ function createRunAgentTurnActivity(services) {
8013
8361
  }
8014
8362
  return outcome;
8015
8363
  };
8364
+ const modelResponseUsageState = createModelResponseUsageEventState(
8365
+ claimedModelUsageSourceKeys
8366
+ );
8016
8367
  const runStreamAttempt = async () => {
8017
8368
  if (!runInput) {
8018
8369
  throw new Error("Run input was not prepared");
8019
8370
  }
8020
8371
  stream = void 0;
8021
8372
  batcher = null;
8022
- let responseUsageCount = 0;
8023
8373
  let currentToolBatchCallIds = /* @__PURE__ */ new Set();
8024
8374
  let currentToolBatchCompletedCallIds = /* @__PURE__ */ new Set();
8025
- let lastProviderContextTokensObserved = null;
8026
- let providerContextRevision = 0;
8027
8375
  throwIfWorkerShuttingDown();
8028
8376
  throwIfTurnCancelled();
8029
8377
  const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
@@ -8074,10 +8422,7 @@ function createRunAgentTurnActivity(services) {
8074
8422
  }
8075
8423
  } : {}
8076
8424
  } : {},
8077
- contextCompactionSignal: () => lastProviderContextTokensObserved === null ? null : {
8078
- revision: providerContextRevision,
8079
- totalTokens: lastProviderContextTokensObserved
8080
- },
8425
+ contextCompactionSignal: () => modelResponseUsageContextSignal(modelResponseUsageState),
8081
8426
  contextCompactionRequested: () => isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
8082
8427
  ...toolCancellationFenceRef.current ? { turnToolCancellationFence: toolCancellationFenceRef.current } : {}
8083
8428
  });
@@ -8108,73 +8453,33 @@ function createRunAgentTurnActivity(services) {
8108
8453
  }
8109
8454
  let stableToolCallIdsToClear = null;
8110
8455
  let completedCurrentToolBatch = false;
8111
- const responseUsage = modelResponseUsageFromSdkEvent(next.value);
8112
- if (responseUsage) {
8113
- await recordCompletedModelCallBeforeOwnershipFences({
8114
- renewLease: () => renewCodexLease("model_usage"),
8115
- leaseLost: () => codexLeaseLost,
8116
- leaseLostMessage: "Codex credential lease expired during the active turn",
8117
- recordUsage: async () => {
8118
- responseUsageCount += 1;
8119
- const responseSourceKey = modelUsageSourceKey({
8120
- responseId: responseUsage.responseId,
8121
- dispatchId: modelUsageDispatchId,
8122
- positionalKey: `response-${responseUsageCount}`
8123
- });
8124
- const responseAccountCtx = modelCallAccountContext({
8125
- servingCredentialId: effectiveCodexCredentialId,
8126
- priorSessionCredentialId: priorSessionCodexCredentialId,
8127
- isFirstCallOfTurn: responseUsageCount === 1
8128
- });
8129
- await recordModelUsageAndDebitCredits(settings, db, {
8130
- accountId: input.accountId,
8131
- workspaceId: input.workspaceId,
8132
- sessionId: input.sessionId,
8133
- turnId: activeTurnId,
8134
- turnAttemptId: input.attemptId,
8135
- model: turn.model,
8136
- isCodexTurn,
8137
- usage: responseUsage.usage,
8138
- sourceKey: responseSourceKey,
8139
- observability
8140
- });
8141
- await emitModelCallUsage({
8142
- observability,
8143
- publish,
8144
- accountId: input.accountId,
8145
- workspaceId: input.workspaceId,
8146
- sessionId: input.sessionId,
8147
- turnId: activeTurnId,
8148
- provider: resolvedModel?.provider.id ?? settings.openaiProvider,
8149
- providerApi: resolvedModel?.provider.api ?? "responses",
8150
- model: turn.model,
8151
- sourceKey: responseSourceKey,
8152
- usage: responseUsage,
8153
- servingAccountHash: responseAccountCtx.servingAccountHash,
8154
- accountChangedFromPrevCall: responseAccountCtx.accountChangedFromPrevCall,
8155
- emittedSourceKeys: emittedModelUsageSourceKeys
8156
- });
8157
- const observed = responseUsage.usage?.inputTokens;
8158
- if (typeof observed === "number" && observed > 0) {
8159
- recordModelInputTokens(observability, streamProvider, observed);
8160
- }
8161
- const observedTotal = providerContextTokens(responseUsage.usage);
8162
- if (observedTotal !== null) {
8163
- lastProviderContextTokensObserved = observedTotal;
8164
- providerContextRevision += 1;
8165
- }
8166
- recordModelCacheTokens(observability, streamProvider, {
8167
- cachedTokens: modelCallUsageTelemetry(responseUsage.usage).cachedTokens,
8168
- promptTokens: responseUsage.usage?.inputTokens
8169
- });
8170
- },
8171
- recordAttemptSignals: async () => {
8172
- const observed = responseUsage.usage?.inputTokens;
8173
- if (typeof observed === "number" && observed > 0) {
8174
- await setLastInputTokensFenced(observed);
8175
- }
8176
- }
8177
- });
8456
+ const responseUsageResult = await processModelResponseUsageEvent({
8457
+ event: next.value,
8458
+ state: modelResponseUsageState,
8459
+ dispatchId: modelUsageDispatchId,
8460
+ settings,
8461
+ db,
8462
+ observability,
8463
+ publish,
8464
+ accountId: input.accountId,
8465
+ workspaceId: input.workspaceId,
8466
+ sessionId: input.sessionId,
8467
+ turnId: activeTurnId,
8468
+ turnAttemptId: input.attemptId,
8469
+ provider: resolvedModel?.provider.id ?? settings.openaiProvider,
8470
+ providerApi: resolvedModel?.provider.api ?? "responses",
8471
+ model: turn.model,
8472
+ metricProvider: streamProvider,
8473
+ externallyBilled: isExternallyBilledTurn,
8474
+ servingCredentialId: effectiveCodexCredentialId,
8475
+ priorSessionCredentialId: priorSessionCodexCredentialId,
8476
+ emittedSourceKeys: emittedModelUsageSourceKeys,
8477
+ renewLease: () => renewCodexLease("model_usage"),
8478
+ leaseLost: () => codexLeaseLost,
8479
+ leaseLostMessage: "Codex credential lease expired during the active turn",
8480
+ setLastInputTokens: setLastInputTokensFenced
8481
+ });
8482
+ if (responseUsageResult.status === "processed") {
8178
8483
  currentToolBatchCallIds = /* @__PURE__ */ new Set();
8179
8484
  currentToolBatchCompletedCallIds = /* @__PURE__ */ new Set();
8180
8485
  await reconcileConversationTruth();
@@ -8184,7 +8489,7 @@ function createRunAgentTurnActivity(services) {
8184
8489
  db,
8185
8490
  input.accountId,
8186
8491
  input.workspaceId,
8187
- isCodexTurn,
8492
+ isExternallyBilledTurn,
8188
8493
  entitlements
8189
8494
  );
8190
8495
  } catch (limitError) {
@@ -8283,74 +8588,74 @@ function createRunAgentTurnActivity(services) {
8283
8588
  stream.completed.catch(() => void 0),
8284
8589
  cancellationSignal
8285
8590
  );
8286
- if (responseUsageCount === 0) {
8591
+ if (modelResponseUsageState.responseUsageCount === 0) {
8287
8592
  const aggregateUsage = stream.state.usage;
8288
- const aggregateInput = aggregateUsage?.inputTokens;
8289
- if (typeof aggregateInput === "number" && aggregateInput > 0) {
8290
- const aggregateContext = providerContextTokens(
8291
- aggregateUsage
8292
- );
8293
- if (aggregateContext !== null) {
8294
- lastProviderContextTokensObserved = aggregateContext;
8295
- providerContextRevision += 1;
8296
- }
8297
- }
8593
+ const normalizedAggregateUsage = normalizeModelCallUsage(aggregateUsage);
8594
+ const aggregateInput = normalizedAggregateUsage.telemetry.inputTokens;
8298
8595
  const aggregateSourceKey = modelUsageSourceKey({
8299
8596
  responseId: null,
8300
8597
  dispatchId: modelUsageDispatchId,
8301
8598
  positionalKey: "aggregate"
8302
8599
  });
8303
- const aggregateAccountCtx = modelCallAccountContext({
8304
- servingCredentialId: effectiveCodexCredentialId,
8305
- priorSessionCredentialId: priorSessionCodexCredentialId,
8306
- isFirstCallOfTurn: true
8307
- });
8308
- recordModelCacheTokens(observability, streamProvider, {
8309
- cachedTokens: modelCallUsageTelemetry(
8310
- aggregateUsage
8311
- ).cachedTokens,
8312
- promptTokens: aggregateUsage?.inputTokens
8313
- });
8314
- await recordCompletedModelCallBeforeOwnershipFences({
8315
- renewLease: () => renewCodexLease("model_usage"),
8316
- leaseLost: () => codexLeaseLost,
8317
- leaseLostMessage: "Codex credential lease expired during the active turn",
8318
- recordUsage: async () => {
8319
- await recordModelUsageAndDebitCredits(settings, db, {
8320
- accountId: input.accountId,
8321
- workspaceId: input.workspaceId,
8322
- sessionId: input.sessionId,
8323
- turnId: activeTurnId,
8324
- turnAttemptId: input.attemptId,
8325
- model: turn.model,
8326
- isCodexTurn,
8327
- usage: aggregateUsage,
8328
- sourceKey: aggregateSourceKey,
8329
- observability
8330
- });
8331
- await emitModelCallUsage({
8332
- observability,
8333
- publish,
8334
- accountId: input.accountId,
8335
- workspaceId: input.workspaceId,
8336
- sessionId: input.sessionId,
8337
- turnId: activeTurnId,
8338
- provider: resolvedModel?.provider.id ?? settings.openaiProvider,
8339
- providerApi: resolvedModel?.provider.api ?? "responses",
8340
- model: turn.model,
8341
- sourceKey: aggregateSourceKey,
8342
- usage: { usage: aggregateUsage },
8343
- servingAccountHash: aggregateAccountCtx.servingAccountHash,
8344
- accountChangedFromPrevCall: aggregateAccountCtx.accountChangedFromPrevCall,
8345
- emittedSourceKeys: emittedModelUsageSourceKeys
8346
- });
8347
- },
8348
- recordAttemptSignals: async () => {
8349
- if (typeof aggregateInput === "number" && aggregateInput > 0) {
8350
- await setLastInputTokensFenced(aggregateInput);
8600
+ if (!claimedModelUsageSourceKeys.has(aggregateSourceKey)) {
8601
+ claimedModelUsageSourceKeys.add(aggregateSourceKey);
8602
+ const aggregateAccountCtx = modelCallAccountContext({
8603
+ servingCredentialId: effectiveCodexCredentialId,
8604
+ priorSessionCredentialId: priorSessionCodexCredentialId,
8605
+ isFirstCallOfTurn: true
8606
+ });
8607
+ let aggregateAuthoritative = false;
8608
+ await recordCompletedModelCallBeforeOwnershipFences({
8609
+ renewLease: () => renewCodexLease("model_usage"),
8610
+ leaseLost: () => codexLeaseLost,
8611
+ leaseLostMessage: "Codex credential lease expired during the active turn",
8612
+ recordUsage: async () => {
8613
+ await recordModelUsageAndDebitCredits(settings, db, {
8614
+ accountId: input.accountId,
8615
+ workspaceId: input.workspaceId,
8616
+ sessionId: input.sessionId,
8617
+ turnId: activeTurnId,
8618
+ turnAttemptId: input.attemptId,
8619
+ model: turn.model,
8620
+ externallyBilled: isExternallyBilledTurn,
8621
+ usage: aggregateUsage,
8622
+ normalizedUsage: normalizedAggregateUsage,
8623
+ sourceKey: aggregateSourceKey,
8624
+ observability
8625
+ });
8626
+ aggregateAuthoritative = await emitModelCallUsage({
8627
+ observability,
8628
+ publish,
8629
+ accountId: input.accountId,
8630
+ workspaceId: input.workspaceId,
8631
+ sessionId: input.sessionId,
8632
+ turnId: activeTurnId,
8633
+ provider: resolvedModel?.provider.id ?? settings.openaiProvider,
8634
+ providerApi: resolvedModel?.provider.api ?? "responses",
8635
+ model: turn.model,
8636
+ sourceKey: aggregateSourceKey,
8637
+ usage: { usage: aggregateUsage },
8638
+ normalizedUsage: normalizedAggregateUsage,
8639
+ servingAccountHash: aggregateAccountCtx.servingAccountHash,
8640
+ accountChangedFromPrevCall: aggregateAccountCtx.accountChangedFromPrevCall,
8641
+ emittedSourceKeys: emittedModelUsageSourceKeys
8642
+ });
8643
+ if (aggregateAuthoritative && aggregateInput !== null && aggregateInput > 0) {
8644
+ recordModelInputTokens(observability, streamProvider, aggregateInput);
8645
+ }
8646
+ },
8647
+ recordAttemptSignals: async () => {
8648
+ if (!aggregateAuthoritative) return;
8649
+ if (normalizedAggregateUsage.totalTokens !== null) {
8650
+ modelResponseUsageState.lastProviderContextTokensObserved = normalizedAggregateUsage.totalTokens;
8651
+ modelResponseUsageState.providerContextRevision += 1;
8652
+ }
8653
+ if (aggregateInput !== null && aggregateInput > 0) {
8654
+ await setLastInputTokensFenced(aggregateInput);
8655
+ }
8351
8656
  }
8352
- }
8353
- });
8657
+ });
8658
+ }
8354
8659
  }
8355
8660
  if (stream.interruptions.length > 0) {
8356
8661
  await reconcileConversationTruth();
@@ -9167,7 +9472,7 @@ function createRunAgentTurnActivity(services) {
9167
9472
  activityStatus = "idle";
9168
9473
  return claimedResult({ status: "idle" });
9169
9474
  }
9170
- const failure = agentRunFailurePayload(error);
9475
+ const failure = agentRunFailurePayload(error, { isCodexTurn });
9171
9476
  if (isSessionEventPersistenceError(error)) {
9172
9477
  observability.error("session event persistence failed", {
9173
9478
  accountId: input.accountId,
@@ -9640,7 +9945,7 @@ function isTransientProviderError(error) {
9640
9945
  message
9641
9946
  );
9642
9947
  }
9643
- function agentRunFailurePayload(error) {
9948
+ function agentRunFailurePayload(error, options = {}) {
9644
9949
  const message = error instanceof Error ? error.message : String(error);
9645
9950
  const status = typeof error === "object" && error !== null && "status" in error ? Number(error.status) : void 0;
9646
9951
  const code = typeof error === "object" && error !== null && "code" in error ? String(error.code) : void 0;
@@ -9671,6 +9976,20 @@ function agentRunFailurePayload(error) {
9671
9976
  if (usageLimit) {
9672
9977
  return codexUsageLimitFailurePayload(usageLimit, message);
9673
9978
  }
9979
+ const codexTimeout = classifyCodexResponseTimeoutError(error, {
9980
+ allowLegacyRequestTimeout: options.isCodexTurn === true
9981
+ });
9982
+ if (codexTimeout) {
9983
+ return {
9984
+ error: codexTimeout.responseObserved ? "The Codex response timed out after streaming began. Observed output was checkpointed; automatic replay is disabled because the upstream operation may still be active." : "The Codex response timed out before any response was observed. Upstream acceptance is unknown, so automatic replay is disabled.",
9985
+ code: "codex_response_timeout",
9986
+ retryable: false,
9987
+ timeoutClass: codexTimeout.timeoutClass,
9988
+ responseObserved: codexTimeout.responseObserved,
9989
+ ...codexTimeout.requestId ? { requestId: codexTimeout.requestId } : {},
9990
+ ...codexTimeout.message ? { detail: codexTimeout.message } : {}
9991
+ };
9992
+ }
9674
9993
  const mcpTimeout = classifyMcpTransportTimeoutError(error);
9675
9994
  if (mcpTimeout) {
9676
9995
  return {
@@ -9820,8 +10139,8 @@ var BudgetExhaustedError = class extends Error {
9820
10139
  this.name = "BudgetExhaustedError";
9821
10140
  }
9822
10141
  };
9823
- async function ensureRunAllowed(settings, db, accountId, workspaceId, isCodexTurn, entitlements) {
9824
- if (!isCodexTurn && entitlements && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
10142
+ async function ensureRunAllowed(settings, db, accountId, workspaceId, isExternallyBilledTurn, entitlements) {
10143
+ if (!isExternallyBilledTurn && entitlements && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
9825
10144
  const decision = await entitlements.admitRun({
9826
10145
  accountId,
9827
10146
  workspaceId,
@@ -9831,7 +10150,7 @@ async function ensureRunAllowed(settings, db, accountId, workspaceId, isCodexTur
9831
10150
  if (!decision.allowed) {
9832
10151
  throw new Error(decision.reason || "insufficient OpenGeni credits");
9833
10152
  }
9834
- } else if (!isCodexTurn && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
10153
+ } else if (!isExternallyBilledTurn && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
9835
10154
  const balance = await getBillingBalance2(db, accountId);
9836
10155
  if (balance.balanceMicros <= 0) {
9837
10156
  throw new Error("insufficient OpenGeni credits");
@@ -9851,7 +10170,7 @@ async function ensureRunAllowed(settings, db, accountId, workspaceId, isCodexTur
9851
10170
  );
9852
10171
  }
9853
10172
  }
9854
- if (!isCodexTurn && limits.maxMonthlyTokensPerWorkspace) {
10173
+ if (!isExternallyBilledTurn && limits.maxMonthlyTokensPerWorkspace) {
9855
10174
  const used = await sumUsageQuantity2(db, {
9856
10175
  workspaceId,
9857
10176
  eventType: "model.tokens",
@@ -9867,10 +10186,12 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
9867
10186
  if (!input.usage) {
9868
10187
  return;
9869
10188
  }
9870
- const inputTokens = positiveInt(input.usage.inputTokens);
9871
- const outputTokens = positiveInt(input.usage.outputTokens);
9872
- const totalTokens = positiveInt(input.usage.totalTokens) || inputTokens + outputTokens;
9873
- if (input.isCodexTurn) {
10189
+ const normalizedUsage = input.normalizedUsage ?? normalizeModelCallUsage(input.usage);
10190
+ const sanitizedUsage = sanitizedModelUsageInput(normalizedUsage);
10191
+ const inputTokens = sanitizedUsage.inputTokens ?? 0;
10192
+ const outputTokens = sanitizedUsage.outputTokens ?? 0;
10193
+ const totalTokens = sanitizedUsage.totalTokens ?? 0;
10194
+ if (input.externallyBilled) {
9874
10195
  await recordUsageEvent2(db, {
9875
10196
  accountId: input.accountId,
9876
10197
  workspaceId: input.workspaceId,
@@ -9908,7 +10229,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
9908
10229
  if (!configuredModelPricing(settings)[input.model]) {
9909
10230
  throw new Error(`Missing model pricing for ${input.model}`);
9910
10231
  }
9911
- const costMicros = calculateModelUsageCostMicros(settings, input.model, input.usage);
10232
+ const costMicros = calculateModelUsageCostMicros(settings, input.model, sanitizedUsage);
9912
10233
  await recordUsageEvent2(db, {
9913
10234
  accountId: input.accountId,
9914
10235
  workspaceId: input.workspaceId,
@@ -9942,16 +10263,19 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
9942
10263
  // Additive: the prompt-cache slice of this call's input tokens, so the
9943
10264
  // per-call debit record carries cache efficiency alongside the token
9944
10265
  // counts. 0 when the provider did not report cached tokens.
9945
- cachedTokens: positiveInt(
9946
- modelCallUsageTelemetry(input.usage).cachedTokens
9947
- )
10266
+ cachedTokens: normalizedUsage.telemetry.cachedTokens ?? 0
9948
10267
  }
9949
10268
  });
9950
10269
  recordCreditMicros(input.observability, "usage", result.debitedMicros);
9951
10270
  }
9952
10271
  }
9953
- function positiveInt(value) {
9954
- return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
10272
+ function sanitizedModelUsageInput(normalized) {
10273
+ return {
10274
+ ...normalized.telemetry.inputTokens !== null ? { inputTokens: normalized.telemetry.inputTokens } : {},
10275
+ ...normalized.telemetry.outputTokens !== null ? { outputTokens: normalized.telemetry.outputTokens } : {},
10276
+ ...normalized.totalTokens !== null ? { totalTokens: normalized.totalTokens } : {},
10277
+ ...normalized.telemetry.cachedTokens !== null ? { inputTokensDetails: { cached_tokens: normalized.telemetry.cachedTokens } } : {}
10278
+ };
9955
10279
  }
9956
10280
  function startOfUtcMonth2() {
9957
10281
  const now = /* @__PURE__ */ new Date();
@@ -10367,12 +10691,29 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
10367
10691
  }
10368
10692
  return forceDrained;
10369
10693
  }
10370
- async function sweepModalOrphansForConfiguredBackend(settings, db, observability) {
10371
- if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret) {
10372
- return 0;
10694
+ function modalOrphanTerminationStillEligible(latest, candidate) {
10695
+ if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {
10696
+ return false;
10373
10697
  }
10374
- let liveLeases;
10375
- try {
10698
+ const leaseId = candidate.tags.opengeni_lease_id;
10699
+ const workspaceId = candidate.tags.opengeni_workspace_id;
10700
+ const sandboxGroupId = candidate.tags.opengeni_sandbox_group_id;
10701
+ if (!leaseId || !workspaceId || !sandboxGroupId) {
10702
+ return true;
10703
+ }
10704
+ const activeAttribution = latest.find(
10705
+ (lease) => lease.leaseId === leaseId && lease.workspaceId === workspaceId && lease.sandboxGroupId === sandboxGroupId
10706
+ );
10707
+ return Boolean(
10708
+ !activeAttribution || activeAttribution.instanceId !== null && activeAttribution.instanceId !== candidate.sandboxId
10709
+ );
10710
+ }
10711
+ async function sweepModalOrphansForConfiguredBackend(settings, db, observability) {
10712
+ if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret) {
10713
+ return 0;
10714
+ }
10715
+ let liveLeases;
10716
+ try {
10376
10717
  liveLeases = await listLiveModalSandboxLeaseAttributions(db);
10377
10718
  } catch (error) {
10378
10719
  observability.warn("sandbox reaper: live Modal lease attribution read failed", {
@@ -10380,7 +10721,12 @@ async function sweepModalOrphansForConfiguredBackend(settings, db, observability
10380
10721
  });
10381
10722
  return 0;
10382
10723
  }
10383
- const result = await sweepModalOrphanSandboxes(settings, liveLeases);
10724
+ const result = await sweepModalOrphanSandboxes(settings, liveLeases, {
10725
+ revalidateTermination: async (candidate) => {
10726
+ const latest = await listLiveModalSandboxLeaseAttributions(db);
10727
+ return modalOrphanTerminationStillEligible(latest, candidate);
10728
+ }
10729
+ });
10384
10730
  for (const terminated of result.terminated) {
10385
10731
  observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
10386
10732
  sandboxId: terminated.sandboxId,
@@ -10630,7 +10976,7 @@ import {
10630
10976
  getBillingBalance as getBillingBalance5,
10631
10977
  getRig,
10632
10978
  getVariableSet,
10633
- isCodexBilledTurn as isCodexBilledTurn3,
10979
+ isCodexBilledTurn as isCodexBilledTurn2,
10634
10980
  markScheduledTaskRunFailedIfQueued,
10635
10981
  recordUsageEvent as recordUsageEvent4,
10636
10982
  requireScheduledTask,
@@ -10649,7 +10995,7 @@ function createScheduledTaskActivities(services) {
10649
10995
  const { settings, db, bus, wakeSessionWorkflow } = await services();
10650
10996
  const task = await requireScheduledTask(db, input.workspaceId, input.taskId);
10651
10997
  const model = task.agentConfig.model ?? settings.openaiModel;
10652
- const isCodexRun = await isCodexBilledTurn3({
10998
+ const isCodexRun = await isCodexBilledTurn2({
10653
10999
  db,
10654
11000
  settings,
10655
11001
  workspaceId: task.workspaceId,
@@ -11282,22 +11628,166 @@ import {
11282
11628
  promoteSetupAppendChange
11283
11629
  } from "@opengeni/core";
11284
11630
  import {
11631
+ acquireLease as acquireLease2,
11285
11632
  beginRigChangeVerificationAttempt,
11633
+ commitWarmingToWarm as commitWarmingToWarm2,
11634
+ failWarmingToCold as failWarmingToCold2,
11286
11635
  getRig as getRig2,
11287
11636
  getRigChange,
11288
11637
  getRigVersionById,
11638
+ markWarmLeaseInstanceLost as markWarmLeaseInstanceLost2,
11639
+ recordWarmingSandboxCreated as recordWarmingSandboxCreated2,
11640
+ releaseLeaseHolder as releaseLeaseHolder2,
11289
11641
  sanitizeEventPayload,
11290
11642
  sanitizeEventString,
11291
11643
  sanitizeMemoryText,
11644
+ touchLeaseHolder as touchLeaseHolder2,
11292
11645
  updateRigChangeStatus
11293
11646
  } from "@opengeni/db";
11294
11647
  import {
11648
+ createTurnToolCancellationController,
11295
11649
  establishSandboxSessionFromEnvelope as establishSandboxSessionFromEnvelope2,
11296
11650
  runRigSetupHook,
11297
11651
  sandboxCommandExitCode,
11298
- sandboxCommandOutput
11652
+ sandboxCommandOutput,
11653
+ serializeEstablishedSandboxEnvelope as serializeEstablishedSandboxEnvelope2,
11654
+ tagModalSandbox as tagModalSandbox2
11299
11655
  } from "@opengeni/runtime";
11300
11656
  var OUTPUT_TAIL_LIMIT = 64 * 1024;
11657
+ var RIG_VERIFICATION_CLEANUP_RESERVE_MAX_MS = 2 * 6e4;
11658
+ var RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS = 6e4;
11659
+ var RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS = 1e4;
11660
+ var RIG_VERIFICATION_HEARTBEAT_MIN_INTERVAL_MS = 250;
11661
+ var RIG_VERIFICATION_OWNER_TTL_MS = 20 * 6e4;
11662
+ var RIG_VERIFICATION_OWNERS_DISABLED_MESSAGE = "Rig verification lease ownership is disabled; refusing to create an unowned verifier sandbox";
11663
+ var RIG_VERIFICATION_HOLDER_TOUCH_INTERVAL_MS = 1e4;
11664
+ var RIG_VERIFICATION_RELEASE_GRACE_MS = 1;
11665
+ var RigVerificationActivityDeadlineError = class extends Error {
11666
+ constructor(startToCloseTimeoutMs, cleanupReserveMs) {
11667
+ super(
11668
+ `Rig verification activity-local deadline reached with ${cleanupReserveMs}ms reserved for cleanup before the ${startToCloseTimeoutMs}ms Temporal start-to-close timeout`
11669
+ );
11670
+ this.startToCloseTimeoutMs = startToCloseTimeoutMs;
11671
+ this.cleanupReserveMs = cleanupReserveMs;
11672
+ }
11673
+ name = "RigVerificationActivityDeadlineError";
11674
+ };
11675
+ function computeCleanupReserveMs(startToCloseTimeoutMs) {
11676
+ return Math.min(
11677
+ RIG_VERIFICATION_CLEANUP_RESERVE_MAX_MS,
11678
+ Math.max(100, Math.floor(startToCloseTimeoutMs / 4)),
11679
+ Math.max(1, startToCloseTimeoutMs - 1)
11680
+ );
11681
+ }
11682
+ function createRigVerificationActivityLifecycle(context = currentActivityContext()) {
11683
+ const controller = new AbortController();
11684
+ const temporalSignal = context?.cancellationSignal;
11685
+ const forwardTemporalCancellation = () => {
11686
+ if (!controller.signal.aborted) {
11687
+ controller.abort(temporalSignal?.reason ?? new Error("Rig verification cancelled"));
11688
+ }
11689
+ };
11690
+ if (temporalSignal?.aborted) {
11691
+ forwardTemporalCancellation();
11692
+ } else {
11693
+ temporalSignal?.addEventListener("abort", forwardTemporalCancellation, { once: true });
11694
+ }
11695
+ const startedAtMs = Date.now();
11696
+ const startToCloseTimeoutMs = context?.info.startToCloseTimeoutMs;
11697
+ const reserveMs = typeof startToCloseTimeoutMs === "number" && startToCloseTimeoutMs > 0 ? computeCleanupReserveMs(startToCloseTimeoutMs) : null;
11698
+ const serverDeadlineAtMs = reserveMs === null || startToCloseTimeoutMs === void 0 ? null : startedAtMs + startToCloseTimeoutMs;
11699
+ const serverReportingMarginMs = reserveMs === null ? 0 : Math.min(3e4, Math.max(1, Math.floor(reserveMs / 4)));
11700
+ const cleanupDeadlineAtMs = serverDeadlineAtMs === null ? null : serverDeadlineAtMs - serverReportingMarginMs;
11701
+ let deadlineTimer = null;
11702
+ if (reserveMs !== null && startToCloseTimeoutMs !== void 0 && !controller.signal.aborted) {
11703
+ deadlineTimer = setTimeout(
11704
+ () => {
11705
+ if (!controller.signal.aborted) {
11706
+ controller.abort(
11707
+ new RigVerificationActivityDeadlineError(startToCloseTimeoutMs, reserveMs)
11708
+ );
11709
+ }
11710
+ },
11711
+ Math.max(1, startToCloseTimeoutMs - reserveMs)
11712
+ );
11713
+ }
11714
+ const heartbeat = () => {
11715
+ if (!context) return;
11716
+ try {
11717
+ context.heartbeat({
11718
+ activity: "rig_verification",
11719
+ at: (/* @__PURE__ */ new Date()).toISOString(),
11720
+ cleanup: controller.signal.aborted,
11721
+ cleanupDeadlineAt: cleanupDeadlineAtMs ? new Date(cleanupDeadlineAtMs).toISOString() : null
11722
+ });
11723
+ } catch (error) {
11724
+ if (!controller.signal.aborted) controller.abort(error);
11725
+ }
11726
+ };
11727
+ heartbeat();
11728
+ const heartbeatTimeoutMs = context?.info.heartbeatTimeoutMs;
11729
+ const heartbeatIntervalMs = typeof heartbeatTimeoutMs === "number" && heartbeatTimeoutMs > 0 ? Math.min(
11730
+ RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS,
11731
+ Math.max(RIG_VERIFICATION_HEARTBEAT_MIN_INTERVAL_MS, Math.floor(heartbeatTimeoutMs / 3))
11732
+ ) : RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS;
11733
+ const heartbeatTimer = context ? setInterval(heartbeat, heartbeatIntervalMs) : null;
11734
+ if (heartbeatTimer && "unref" in heartbeatTimer && typeof heartbeatTimer.unref === "function") {
11735
+ heartbeatTimer.unref();
11736
+ }
11737
+ return {
11738
+ signal: controller.signal,
11739
+ cleanupDeadlineAtMs,
11740
+ dispose: () => {
11741
+ temporalSignal?.removeEventListener("abort", forwardTemporalCancellation);
11742
+ if (deadlineTimer) clearTimeout(deadlineTimer);
11743
+ if (heartbeatTimer) clearInterval(heartbeatTimer);
11744
+ deadlineTimer = null;
11745
+ }
11746
+ };
11747
+ }
11748
+ function abortReason(signal) {
11749
+ return signal.reason ?? new Error("Rig verification cancelled");
11750
+ }
11751
+ function throwIfAborted(signal) {
11752
+ if (signal.aborted) throw abortReason(signal);
11753
+ }
11754
+ async function waitForAbortable(promise, signal) {
11755
+ throwIfAborted(signal);
11756
+ let removeAbortListener = () => void 0;
11757
+ const cancellation = new Promise((_resolve, reject) => {
11758
+ const onAbort = () => reject(abortReason(signal));
11759
+ removeAbortListener = () => signal.removeEventListener("abort", onAbort);
11760
+ signal.addEventListener("abort", onAbort, { once: true });
11761
+ });
11762
+ try {
11763
+ return await Promise.race([promise, cancellation]);
11764
+ } finally {
11765
+ removeAbortListener();
11766
+ }
11767
+ }
11768
+ async function settleWithin(promise, timeoutMs) {
11769
+ if (timeoutMs <= 0) return { status: "timed_out" };
11770
+ let timer = null;
11771
+ try {
11772
+ return await Promise.race([
11773
+ promise.then((value) => ({ status: "completed", value })),
11774
+ new Promise((resolve) => {
11775
+ timer = setTimeout(() => resolve({ status: "timed_out" }), timeoutMs);
11776
+ })
11777
+ ]);
11778
+ } finally {
11779
+ if (timer) clearTimeout(timer);
11780
+ }
11781
+ }
11782
+ function cleanupWaitMs(lifecycle) {
11783
+ if (lifecycle?.cleanupDeadlineAtMs == null) {
11784
+ return RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS;
11785
+ }
11786
+ return Math.max(
11787
+ 0,
11788
+ Math.min(RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS, lifecycle.cleanupDeadlineAtMs - Date.now())
11789
+ );
11790
+ }
11301
11791
  function tail(value, limit = OUTPUT_TAIL_LIMIT) {
11302
11792
  return value.length > limit ? value.slice(-limit) : value;
11303
11793
  }
@@ -11317,23 +11807,331 @@ function systemGrant(rig) {
11317
11807
  }
11318
11808
  async function terminateThrowaway(established) {
11319
11809
  if (!established) {
11320
- return;
11810
+ return true;
11321
11811
  }
11322
11812
  const client = established.client;
11323
11813
  if (typeof client.delete === "function" && established.sessionState !== void 0) {
11324
- await client.delete(established.sessionState).catch(() => void 0);
11325
- return;
11814
+ try {
11815
+ await client.delete(established.sessionState);
11816
+ return true;
11817
+ } catch {
11818
+ return false;
11819
+ }
11326
11820
  }
11327
11821
  const session = established.session;
11328
- if (session.terminate) {
11329
- await session.terminate().catch(() => void 0);
11330
- } else if (session.kill) {
11331
- await session.kill().catch(() => void 0);
11332
- } else if (session.close && !session.closed) {
11333
- await session.close().catch(() => void 0);
11822
+ try {
11823
+ if (session.terminate) {
11824
+ await session.terminate();
11825
+ } else if (session.kill) {
11826
+ await session.kill();
11827
+ } else if (session.close && !session.closed) {
11828
+ await session.close();
11829
+ } else {
11830
+ return false;
11831
+ }
11832
+ return true;
11833
+ } catch {
11834
+ return false;
11334
11835
  }
11335
11836
  }
11336
- async function runCommand(session, command, timeoutMs) {
11837
+ var defaultOwnershipDependencies = {
11838
+ acquire: acquireLease2,
11839
+ establish: establishSandboxSessionFromEnvelope2,
11840
+ recordCreated: recordWarmingSandboxCreated2,
11841
+ commitWarm: commitWarmingToWarm2,
11842
+ touchHolder: touchLeaseHolder2,
11843
+ releaseHolder: releaseLeaseHolder2,
11844
+ failWarming: failWarmingToCold2,
11845
+ markWarmLost: markWarmLeaseInstanceLost2,
11846
+ serialize: serializeEstablishedSandboxEnvelope2,
11847
+ tag: tagModalSandbox2,
11848
+ terminate: terminateThrowaway,
11849
+ createCancellationController: createTurnToolCancellationController,
11850
+ randomUUID: () => crypto.randomUUID()
11851
+ };
11852
+ var RigVerificationLeaseUnavailableError = class extends Error {
11853
+ constructor(sandboxGroupId, role) {
11854
+ super(`Rig verification requires a new clean sandbox, but lease ${sandboxGroupId} was ${role}`);
11855
+ this.sandboxGroupId = sandboxGroupId;
11856
+ this.role = role;
11857
+ }
11858
+ name = "RigVerificationLeaseUnavailableError";
11859
+ };
11860
+ async function runWithOwnedRigVerificationSandbox(input, run, dependencies = defaultOwnershipDependencies) {
11861
+ if (!input.settings.rigVerificationLeaseOwnershipEnabled) {
11862
+ throw new Error(RIG_VERIFICATION_OWNERS_DISABLED_MESSAGE);
11863
+ }
11864
+ const holderId = `rig-verification:${dependencies.randomUUID()}`;
11865
+ const fallbackController = new AbortController();
11866
+ const signal = input.lifecycle?.signal ?? fallbackController.signal;
11867
+ const commandController = dependencies.createCancellationController(signal);
11868
+ const commandRunner = commandController.runSandboxCommand.bind(commandController);
11869
+ let cleanupTarget = null;
11870
+ let acquired = false;
11871
+ let ownsWarmingEpoch = false;
11872
+ let expectedEpoch = null;
11873
+ let holderLivenessTimer = null;
11874
+ let establishmentSettled = false;
11875
+ let establishmentPromise = null;
11876
+ const warnCleanupTimeout = (operation, instanceId) => {
11877
+ input.observability.warn("rig verifier: ownership cleanup timed out", {
11878
+ holderId,
11879
+ instanceId,
11880
+ operation,
11881
+ error: "cleanup did not settle inside the activity reserve; exact owner expiry and the global provider reaper are the bounded delayed backstop"
11882
+ });
11883
+ };
11884
+ const warnCleanupFailure = (operation, instanceId, error) => {
11885
+ input.observability.warn("rig verifier: ownership cleanup failed", {
11886
+ holderId,
11887
+ instanceId,
11888
+ operation,
11889
+ error: error instanceof Error ? error.message : String(error)
11890
+ });
11891
+ };
11892
+ const clearTerminatedLeasePointer = async (instanceId) => {
11893
+ if (expectedEpoch === null) return;
11894
+ const cleanupEpoch = expectedEpoch;
11895
+ const results = await Promise.allSettled([
11896
+ Promise.resolve().then(
11897
+ () => dependencies.markWarmLost(input.db, {
11898
+ accountId: input.accountId,
11899
+ workspaceId: input.workspaceId,
11900
+ sandboxGroupId: input.sandboxGroupId,
11901
+ expectedEpoch: cleanupEpoch + 1,
11902
+ expectedInstanceId: instanceId
11903
+ })
11904
+ ),
11905
+ Promise.resolve().then(
11906
+ () => dependencies.failWarming(input.db, {
11907
+ accountId: input.accountId,
11908
+ workspaceId: input.workspaceId,
11909
+ sandboxGroupId: input.sandboxGroupId,
11910
+ expectedEpoch: cleanupEpoch
11911
+ })
11912
+ )
11913
+ ]);
11914
+ for (const [index, result] of results.entries()) {
11915
+ if (result.status === "rejected") {
11916
+ warnCleanupFailure(
11917
+ index === 0 ? "mark_warm_lost" : "fail_warming",
11918
+ instanceId,
11919
+ result.reason
11920
+ );
11921
+ }
11922
+ }
11923
+ };
11924
+ const terminateAndClear = async (created) => {
11925
+ let terminated = false;
11926
+ try {
11927
+ terminated = await dependencies.terminate(created);
11928
+ } catch (error) {
11929
+ warnCleanupFailure("terminate", created.instanceId, error);
11930
+ }
11931
+ if (!terminated) {
11932
+ warnCleanupFailure(
11933
+ "terminate",
11934
+ created.instanceId,
11935
+ new Error("provider termination was not confirmed; durable lease cleanup remains pending")
11936
+ );
11937
+ return false;
11938
+ }
11939
+ await clearTerminatedLeasePointer(created.instanceId);
11940
+ return true;
11941
+ };
11942
+ try {
11943
+ throwIfAborted(signal);
11944
+ const lease = await dependencies.acquire(input.db, {
11945
+ accountId: input.accountId,
11946
+ workspaceId: input.workspaceId,
11947
+ sandboxGroupId: input.sandboxGroupId,
11948
+ kind: "turn",
11949
+ holderId,
11950
+ subjectId: null,
11951
+ backend: input.settings.sandboxBackend,
11952
+ image: input.settings.modalImageRef ?? input.settings.dockerImage,
11953
+ rigVersionId: input.rigVersionId,
11954
+ leaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
11955
+ warmingLeaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS
11956
+ });
11957
+ acquired = true;
11958
+ expectedEpoch = lease.lease.leaseEpoch;
11959
+ holderLivenessTimer = setInterval(() => {
11960
+ void dependencies.touchHolder(input.db, {
11961
+ accountId: input.accountId,
11962
+ workspaceId: input.workspaceId,
11963
+ sandboxGroupId: input.sandboxGroupId,
11964
+ kind: "turn",
11965
+ holderId
11966
+ }).catch(() => void 0);
11967
+ }, RIG_VERIFICATION_HOLDER_TOUCH_INTERVAL_MS);
11968
+ if ("unref" in holderLivenessTimer && typeof holderLivenessTimer.unref === "function") {
11969
+ holderLivenessTimer.unref();
11970
+ }
11971
+ if (lease.role !== "spawner") {
11972
+ throw new RigVerificationLeaseUnavailableError(input.sandboxGroupId, lease.role);
11973
+ }
11974
+ ownsWarmingEpoch = true;
11975
+ establishmentPromise = Promise.resolve().then(
11976
+ () => dependencies.establish(input.settings, null, {
11977
+ sessionId: `${input.sessionIdPrefix}-${holderId}`,
11978
+ recovery: "create-or-restore",
11979
+ environment: {},
11980
+ onSandboxCreated: async (created) => {
11981
+ cleanupTarget = created;
11982
+ const resumeState2 = created.backendId === "modal" ? null : await dependencies.serialize(created);
11983
+ const recorded = await dependencies.recordCreated(input.db, {
11984
+ accountId: input.accountId,
11985
+ workspaceId: input.workspaceId,
11986
+ sandboxGroupId: input.sandboxGroupId,
11987
+ expectedEpoch: lease.lease.leaseEpoch,
11988
+ instanceId: created.instanceId,
11989
+ resumeBackendId: created.backendId,
11990
+ resumeState: resumeState2,
11991
+ leaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
11992
+ warmingLeaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS
11993
+ });
11994
+ if (!recorded.recorded) {
11995
+ throw new RigVerificationLeaseUnavailableError(input.sandboxGroupId, "fenced");
11996
+ }
11997
+ if (created.backendId === "modal" && !signal.aborted) {
11998
+ try {
11999
+ await waitForAbortable(
12000
+ dependencies.tag(input.settings, created.instanceId, {
12001
+ leaseId: lease.lease.id,
12002
+ workspaceId: input.workspaceId,
12003
+ sandboxGroupId: input.sandboxGroupId
12004
+ }),
12005
+ signal
12006
+ );
12007
+ } catch (error) {
12008
+ if (!signal.aborted) {
12009
+ input.observability.warn("rig verifier: Modal ownership tag failed", {
12010
+ holderId,
12011
+ instanceId: created.instanceId,
12012
+ error: error instanceof Error ? error.message : String(error)
12013
+ });
12014
+ }
12015
+ }
12016
+ }
12017
+ if (signal.aborted) {
12018
+ await terminateAndClear(created);
12019
+ throw abortReason(signal);
12020
+ }
12021
+ }
12022
+ })
12023
+ );
12024
+ void establishmentPromise.then(
12025
+ () => {
12026
+ establishmentSettled = true;
12027
+ },
12028
+ () => {
12029
+ establishmentSettled = true;
12030
+ }
12031
+ );
12032
+ void establishmentPromise.catch(() => void 0);
12033
+ const established = await waitForAbortable(establishmentPromise, signal);
12034
+ cleanupTarget = established;
12035
+ const resumeState = established.backendId === "modal" ? null : await dependencies.serialize(established);
12036
+ const committed = await dependencies.commitWarm(input.db, {
12037
+ accountId: input.accountId,
12038
+ workspaceId: input.workspaceId,
12039
+ sandboxGroupId: input.sandboxGroupId,
12040
+ expectedEpoch: lease.lease.leaseEpoch,
12041
+ instanceId: established.instanceId,
12042
+ dataPlaneUrl: null,
12043
+ resumeBackendId: established.backendId,
12044
+ resumeState,
12045
+ leaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS
12046
+ });
12047
+ if (!committed.committed || !committed.lease) {
12048
+ throw new RigVerificationLeaseUnavailableError(input.sandboxGroupId, "fenced");
12049
+ }
12050
+ const result = await run(established, { signal, commandRunner });
12051
+ throwIfAborted(signal);
12052
+ return result;
12053
+ } catch (error) {
12054
+ if (signal.aborted) throw abortReason(signal);
12055
+ throw error;
12056
+ } finally {
12057
+ if (signal.aborted && establishmentPromise && !establishmentSettled) {
12058
+ const lateEstablishment = await settleWithin(
12059
+ Promise.allSettled([establishmentPromise]),
12060
+ cleanupWaitMs(input.lifecycle)
12061
+ );
12062
+ if (lateEstablishment.status === "timed_out") {
12063
+ warnCleanupTimeout("establish_after_abort", cleanupTarget?.instanceId ?? null);
12064
+ }
12065
+ }
12066
+ if (holderLivenessTimer) {
12067
+ clearInterval(holderLivenessTimer);
12068
+ holderLivenessTimer = null;
12069
+ }
12070
+ commandController.cancel(signal.aborted ? abortReason(signal) : new Error("Verifier complete"));
12071
+ const quiescence = commandController.waitForQuiescence();
12072
+ void quiescence.catch(() => void 0);
12073
+ const quiescenceResult = await settleWithin(quiescence, cleanupWaitMs(input.lifecycle));
12074
+ if (quiescenceResult.status === "timed_out") {
12075
+ warnCleanupTimeout("command_quiescence", cleanupTarget?.instanceId ?? null);
12076
+ }
12077
+ const providerCleanup = Promise.resolve().then(async () => {
12078
+ if (!cleanupTarget) {
12079
+ if (ownsWarmingEpoch && expectedEpoch !== null) {
12080
+ await dependencies.failWarming(input.db, {
12081
+ accountId: input.accountId,
12082
+ workspaceId: input.workspaceId,
12083
+ sandboxGroupId: input.sandboxGroupId,
12084
+ expectedEpoch
12085
+ });
12086
+ }
12087
+ return true;
12088
+ }
12089
+ return await terminateAndClear(cleanupTarget);
12090
+ });
12091
+ void providerCleanup.catch(() => void 0);
12092
+ const providerSettlement = await settleWithin(
12093
+ Promise.allSettled([providerCleanup]),
12094
+ cleanupWaitMs(input.lifecycle)
12095
+ );
12096
+ if (providerSettlement.status === "timed_out") {
12097
+ warnCleanupTimeout("terminate_and_clear", cleanupTarget?.instanceId ?? null);
12098
+ } else if (providerSettlement.value[0]?.status === "rejected") {
12099
+ warnCleanupFailure(
12100
+ "terminate_and_clear",
12101
+ cleanupTarget?.instanceId ?? null,
12102
+ providerSettlement.value[0].reason
12103
+ );
12104
+ }
12105
+ if (acquired) {
12106
+ const release = Promise.resolve().then(
12107
+ () => dependencies.releaseHolder(input.db, {
12108
+ accountId: input.accountId,
12109
+ workspaceId: input.workspaceId,
12110
+ sandboxGroupId: input.sandboxGroupId,
12111
+ kind: "turn",
12112
+ holderId,
12113
+ idleGraceMs: RIG_VERIFICATION_RELEASE_GRACE_MS
12114
+ })
12115
+ );
12116
+ void release.catch(() => void 0);
12117
+ const released = await settleWithin(
12118
+ Promise.allSettled([release]),
12119
+ cleanupWaitMs(input.lifecycle)
12120
+ );
12121
+ if (released.status === "timed_out") {
12122
+ warnCleanupTimeout("release_holder", cleanupTarget?.instanceId ?? null);
12123
+ } else if (released.value[0]?.status === "rejected") {
12124
+ warnCleanupFailure(
12125
+ "release_holder",
12126
+ cleanupTarget?.instanceId ?? null,
12127
+ released.value[0].reason
12128
+ );
12129
+ }
12130
+ }
12131
+ throwIfAborted(signal);
12132
+ }
12133
+ }
12134
+ async function runCommand(session, command, timeoutMs, commandRunner) {
11337
12135
  const args = {
11338
12136
  cmd: command,
11339
12137
  workdir: "/workspace",
@@ -11341,9 +12139,7 @@ async function runCommand(session, command, timeoutMs) {
11341
12139
  yieldTimeMs: timeoutMs,
11342
12140
  maxOutputTokens: 4e4
11343
12141
  };
11344
- const result = session.exec ? await session.exec(args) : session.execCommand ? await session.execCommand(args) : (() => {
11345
- throw new Error("Sandbox session does not support command execution");
11346
- })();
12142
+ const result = await commandRunner(session, args);
11347
12143
  return {
11348
12144
  exitCode: sandboxCommandExitCode(result),
11349
12145
  output: scrubVerificationOutput(tail(sandboxCommandOutput(result)))
@@ -11400,10 +12196,20 @@ async function loadVersionTarget(db, workspaceId, versionId) {
11400
12196
  }
11401
12197
  return { rig, version };
11402
12198
  }
12199
+ async function withRigVerificationActivityLifecycle(run) {
12200
+ const lifecycle = createRigVerificationActivityLifecycle();
12201
+ try {
12202
+ throwIfAborted(lifecycle.signal);
12203
+ return await run(lifecycle);
12204
+ } finally {
12205
+ lifecycle.dispose();
12206
+ }
12207
+ }
11403
12208
  function createRigVerificationActivities(services) {
11404
12209
  return {
11405
- verifyRigChange: async (input) => {
11406
- const { settings, db } = await services();
12210
+ verifyRigChange: (input) => withRigVerificationActivityLifecycle(async (lifecycle) => {
12211
+ const { settings, db, observability } = await services();
12212
+ throwIfAborted(lifecycle.signal);
11407
12213
  const { rig, baseVersion, change } = await loadChangeTarget(
11408
12214
  db,
11409
12215
  input.workspaceId,
@@ -11421,114 +12227,126 @@ function createRigVerificationActivities(services) {
11421
12227
  rigId: rig.id,
11422
12228
  metadata: { changeId: change.id }
11423
12229
  });
11424
- let established = null;
11425
12230
  const verification = { startedAt, checkResults: [] };
11426
12231
  try {
11427
12232
  const candidateVersion = candidateVersionForChange(baseVersion, change);
11428
12233
  const runSettings = settingsWithRigImage(settings, candidateVersion.image);
11429
- established = await establishSandboxSessionFromEnvelope2(runSettings, null, {
11430
- sessionId: `rig-verification-${change.id}`,
11431
- recovery: "create-or-restore",
11432
- environment: {}
11433
- });
11434
- if ((candidateVersion.setupScript ?? "").trim()) {
11435
- await runRigSetupHook(established.session, {
11436
- environment: {},
11437
- runAs: "root",
11438
- rigSetup: {
11439
- rigId: rig.id,
11440
- rigName: rig.name,
11441
- versionId: candidateVersion.id,
11442
- script: candidateVersion.setupScript ?? "",
11443
- timeoutMs: settings.rigSetupTimeoutMs
12234
+ return await runWithOwnedRigVerificationSandbox(
12235
+ {
12236
+ settings: runSettings,
12237
+ db,
12238
+ observability,
12239
+ accountId: rig.accountId,
12240
+ workspaceId: input.workspaceId,
12241
+ sandboxGroupId: change.id,
12242
+ rigVersionId: candidateVersion.id,
12243
+ sessionIdPrefix: `rig-verification-${change.id}`,
12244
+ lifecycle
12245
+ },
12246
+ async (established, runContext) => {
12247
+ if ((candidateVersion.setupScript ?? "").trim()) {
12248
+ await runRigSetupHook(established.session, {
12249
+ environment: {},
12250
+ runAs: "root",
12251
+ commandRunner: runContext.commandRunner,
12252
+ rigSetup: {
12253
+ rigId: rig.id,
12254
+ rigName: rig.name,
12255
+ versionId: candidateVersion.id,
12256
+ script: candidateVersion.setupScript ?? "",
12257
+ timeoutMs: settings.rigSetupTimeoutMs
12258
+ }
12259
+ });
12260
+ verification.setupResult = { exitCode: 0, output: "" };
11444
12261
  }
11445
- });
11446
- verification.setupResult = { exitCode: 0, output: "" };
11447
- }
11448
- const command = setupAppendCommand(change);
11449
- if (command) {
11450
- const commandResult = await runCommand(
11451
- established.session,
11452
- command,
11453
- settings.rigSetupTimeoutMs
11454
- );
11455
- verification.commandResult = commandResult;
11456
- if (commandResult.exitCode !== 0) {
12262
+ const command = setupAppendCommand(change);
12263
+ if (command) {
12264
+ const commandResult = await runCommand(
12265
+ established.session,
12266
+ command,
12267
+ settings.rigSetupTimeoutMs,
12268
+ runContext.commandRunner
12269
+ );
12270
+ verification.commandResult = commandResult;
12271
+ if (commandResult.exitCode !== 0) {
12272
+ verification.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
12273
+ verification.passed = false;
12274
+ const updated2 = await updateRigChangeStatus(db, input.workspaceId, change.id, {
12275
+ status: "rejected",
12276
+ verification: scrubVerificationPayload(verification)
12277
+ });
12278
+ await recordRigAuditEvent(db, {
12279
+ grant,
12280
+ action: "rig.verification.failed",
12281
+ rigId: rig.id,
12282
+ metadata: { changeId: change.id, status: "rejected" }
12283
+ });
12284
+ await recordRigAuditEvent(db, {
12285
+ grant,
12286
+ action: "rig.change.rejected",
12287
+ rigId: rig.id,
12288
+ metadata: { changeId: change.id }
12289
+ });
12290
+ return updated2;
12291
+ }
12292
+ }
12293
+ const checkResults = [];
12294
+ for (const check of candidateVersion.checks) {
12295
+ const result = await runCommand(
12296
+ established.session,
12297
+ check.command,
12298
+ settings.rigSetupTimeoutMs,
12299
+ runContext.commandRunner
12300
+ );
12301
+ checkResults.push({
12302
+ name: check.name,
12303
+ command: scrubVerificationOutput(check.command),
12304
+ ...result
12305
+ });
12306
+ }
12307
+ verification.checkResults = checkResults;
12308
+ const passed = checkResults.every((result) => result.exitCode === 0);
11457
12309
  verification.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
11458
- verification.passed = false;
11459
- const updated2 = await updateRigChangeStatus(db, input.workspaceId, change.id, {
11460
- status: "rejected",
12310
+ verification.passed = passed;
12311
+ const classified = classifyRigVerificationOutcome({ kind: change.kind, passed });
12312
+ if (classified.action === "auto_promote") {
12313
+ await updateRigChangeStatus(db, input.workspaceId, change.id, {
12314
+ status: "verifying",
12315
+ verification: scrubVerificationPayload(verification)
12316
+ });
12317
+ const { change: merged } = await promoteSetupAppendChange({ db }, grant, rig, {
12318
+ ...change,
12319
+ verification
12320
+ });
12321
+ await recordRigAuditEvent(db, {
12322
+ grant,
12323
+ action: "rig.verification.passed",
12324
+ rigId: rig.id,
12325
+ metadata: { changeId: change.id }
12326
+ });
12327
+ return merged;
12328
+ }
12329
+ const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
12330
+ status: classified.status,
11461
12331
  verification: scrubVerificationPayload(verification)
11462
12332
  });
11463
12333
  await recordRigAuditEvent(db, {
11464
12334
  grant,
11465
- action: "rig.verification.failed",
12335
+ action: passed ? "rig.verification.passed" : "rig.verification.failed",
11466
12336
  rigId: rig.id,
11467
- metadata: { changeId: change.id, status: "rejected" }
12337
+ metadata: { changeId: change.id, status: classified.status }
11468
12338
  });
11469
- await recordRigAuditEvent(db, {
11470
- grant,
11471
- action: "rig.change.rejected",
11472
- rigId: rig.id,
11473
- metadata: { changeId: change.id }
11474
- });
11475
- return updated2;
12339
+ if (!passed) {
12340
+ await recordRigAuditEvent(db, {
12341
+ grant,
12342
+ action: "rig.change.rejected",
12343
+ rigId: rig.id,
12344
+ metadata: { changeId: change.id }
12345
+ });
12346
+ }
12347
+ return updated;
11476
12348
  }
11477
- }
11478
- const checkResults = [];
11479
- for (const check of candidateVersion.checks) {
11480
- const result = await runCommand(
11481
- established.session,
11482
- check.command,
11483
- settings.rigSetupTimeoutMs
11484
- );
11485
- checkResults.push({
11486
- name: check.name,
11487
- command: scrubVerificationOutput(check.command),
11488
- ...result
11489
- });
11490
- }
11491
- verification.checkResults = checkResults;
11492
- const passed = checkResults.every((result) => result.exitCode === 0);
11493
- verification.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
11494
- verification.passed = passed;
11495
- const classified = classifyRigVerificationOutcome({ kind: change.kind, passed });
11496
- if (classified.action === "auto_promote") {
11497
- await updateRigChangeStatus(db, input.workspaceId, change.id, {
11498
- status: "verifying",
11499
- verification: scrubVerificationPayload(verification)
11500
- });
11501
- const { change: merged } = await promoteSetupAppendChange({ db }, grant, rig, {
11502
- ...change,
11503
- verification
11504
- });
11505
- await recordRigAuditEvent(db, {
11506
- grant,
11507
- action: "rig.verification.passed",
11508
- rigId: rig.id,
11509
- metadata: { changeId: change.id }
11510
- });
11511
- return merged;
11512
- }
11513
- const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
11514
- status: classified.status,
11515
- verification: scrubVerificationPayload(verification)
11516
- });
11517
- await recordRigAuditEvent(db, {
11518
- grant,
11519
- action: passed ? "rig.verification.passed" : "rig.verification.failed",
11520
- rigId: rig.id,
11521
- metadata: { changeId: change.id, status: classified.status }
11522
- });
11523
- if (!passed) {
11524
- await recordRigAuditEvent(db, {
11525
- grant,
11526
- action: "rig.change.rejected",
11527
- rigId: rig.id,
11528
- metadata: { changeId: change.id }
11529
- });
11530
- }
11531
- return updated;
12349
+ );
11532
12350
  } catch (error) {
11533
12351
  verification.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
11534
12352
  verification.passed = false;
@@ -11551,13 +12369,13 @@ function createRigVerificationActivities(services) {
11551
12369
  rigId: rig.id,
11552
12370
  metadata: { changeId: change.id }
11553
12371
  });
12372
+ if (lifecycle.signal.aborted) throw abortReason(lifecycle.signal);
11554
12373
  return updated;
11555
- } finally {
11556
- await terminateThrowaway(established);
11557
12374
  }
11558
- },
11559
- verifyRigVersion: async (input) => {
11560
- const { settings, db } = await services();
12375
+ }),
12376
+ verifyRigVersion: (input) => withRigVerificationActivityLifecycle(async (lifecycle) => {
12377
+ const { settings, db, observability } = await services();
12378
+ throwIfAborted(lifecycle.signal);
11561
12379
  const { rig, version } = await loadVersionTarget(db, input.workspaceId, input.versionId);
11562
12380
  const grant = systemGrant(rig);
11563
12381
  const startedAt = (/* @__PURE__ */ new Date()).toISOString();
@@ -11567,53 +12385,64 @@ function createRigVerificationActivities(services) {
11567
12385
  rigId: rig.id,
11568
12386
  metadata: { versionId: version.id }
11569
12387
  });
11570
- let established = null;
11571
12388
  try {
11572
12389
  const runSettings = settingsWithRigImage(settings, version.image);
11573
- established = await establishSandboxSessionFromEnvelope2(runSettings, null, {
11574
- sessionId: `rig-version-verification-${version.id}`,
11575
- recovery: "create-or-restore",
11576
- environment: {}
11577
- });
11578
- if ((version.setupScript ?? "").trim()) {
11579
- await runRigSetupHook(established.session, {
11580
- environment: {},
11581
- runAs: "root",
11582
- rigSetup: {
11583
- rigId: rig.id,
11584
- rigName: rig.name,
11585
- versionId: version.id,
11586
- script: version.setupScript ?? "",
11587
- timeoutMs: settings.rigSetupTimeoutMs
12390
+ return await runWithOwnedRigVerificationSandbox(
12391
+ {
12392
+ settings: runSettings,
12393
+ db,
12394
+ observability,
12395
+ accountId: rig.accountId,
12396
+ workspaceId: input.workspaceId,
12397
+ sandboxGroupId: version.id,
12398
+ rigVersionId: version.id,
12399
+ sessionIdPrefix: `rig-version-verification-${version.id}`,
12400
+ lifecycle
12401
+ },
12402
+ async (established, runContext) => {
12403
+ if ((version.setupScript ?? "").trim()) {
12404
+ await runRigSetupHook(established.session, {
12405
+ environment: {},
12406
+ runAs: "root",
12407
+ commandRunner: runContext.commandRunner,
12408
+ rigSetup: {
12409
+ rigId: rig.id,
12410
+ rigName: rig.name,
12411
+ versionId: version.id,
12412
+ script: version.setupScript ?? "",
12413
+ timeoutMs: settings.rigSetupTimeoutMs
12414
+ }
12415
+ });
11588
12416
  }
11589
- });
11590
- }
11591
- const checkResults = [];
11592
- for (const check of version.checks) {
11593
- checkResults.push({
11594
- name: check.name,
11595
- command: scrubVerificationOutput(check.command),
11596
- ...await runCommand(
11597
- established.session,
11598
- check.command,
11599
- settings.rigSetupTimeoutMs
11600
- )
11601
- });
11602
- }
11603
- const passed = checkResults.every((result) => result.exitCode === 0);
11604
- await recordRigAuditEvent(db, {
11605
- grant,
11606
- action: passed ? "rig.verification.passed" : "rig.verification.failed",
11607
- rigId: rig.id,
11608
- metadata: scrubVerificationPayload({
11609
- versionId: version.id,
11610
- startedAt,
11611
- finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
11612
- passed,
11613
- checkResults
11614
- })
11615
- });
11616
- return { versionId: version.id, passed, checkResults };
12417
+ const checkResults = [];
12418
+ for (const check of version.checks) {
12419
+ checkResults.push({
12420
+ name: check.name,
12421
+ command: scrubVerificationOutput(check.command),
12422
+ ...await runCommand(
12423
+ established.session,
12424
+ check.command,
12425
+ settings.rigSetupTimeoutMs,
12426
+ runContext.commandRunner
12427
+ )
12428
+ });
12429
+ }
12430
+ const passed = checkResults.every((result) => result.exitCode === 0);
12431
+ await recordRigAuditEvent(db, {
12432
+ grant,
12433
+ action: passed ? "rig.verification.passed" : "rig.verification.failed",
12434
+ rigId: rig.id,
12435
+ metadata: scrubVerificationPayload({
12436
+ versionId: version.id,
12437
+ startedAt,
12438
+ finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
12439
+ passed,
12440
+ checkResults
12441
+ })
12442
+ });
12443
+ return { versionId: version.id, passed, checkResults };
12444
+ }
12445
+ );
11617
12446
  } catch (error) {
11618
12447
  const detail = tail(
11619
12448
  scrubVerificationOutput(error instanceof Error ? error.message : String(error)),
@@ -11632,10 +12461,8 @@ function createRigVerificationActivities(services) {
11632
12461
  })
11633
12462
  });
11634
12463
  throw error;
11635
- } finally {
11636
- await terminateThrowaway(established);
11637
12464
  }
11638
- }
12465
+ })
11639
12466
  };
11640
12467
  }
11641
12468