@opengeni/worker-bundle 0.8.3 → 0.10.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1261 -434
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +3 -1
- package/package.json +11 -11
- package/src/activities/agent-turn.ts +639 -293
- package/src/activities/capabilities.ts +3 -1
- package/src/activities/environment.ts +142 -11
- package/src/activities/rig-verification.ts +897 -244
- package/src/activities/sandbox-lease.ts +32 -1
- package/src/observability-metrics.ts +83 -15
- package/src/workflows/rig-verification.ts +3 -1
package/dist/index.js
CHANGED
|
@@ -56,7 +56,7 @@ import {
|
|
|
56
56
|
getLatestRunState as getLatestRunState2,
|
|
57
57
|
getHumanInputResumeForEvent as getHumanInputResumeForEvent2,
|
|
58
58
|
getSessionHumanInputRequest,
|
|
59
|
-
|
|
59
|
+
installOrReadTurnExecutionPolicyForAttempt,
|
|
60
60
|
workspaceCodexSubscriptionActive as workspaceCodexSubscriptionActive2,
|
|
61
61
|
acquireCodexCredentialLease,
|
|
62
62
|
armCodexCapacityWait,
|
|
@@ -109,8 +109,8 @@ import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@
|
|
|
109
109
|
import {
|
|
110
110
|
sandboxStateEntryFromRunState,
|
|
111
111
|
maxTurnsExceededRunState,
|
|
112
|
-
modelCallUsageTelemetry,
|
|
113
112
|
modelResponseUsageFromSdkEvent,
|
|
113
|
+
normalizeModelCallUsage,
|
|
114
114
|
normalizeSdkEvent,
|
|
115
115
|
sanitizeHistoryItemsForModel as sanitizeHistoryItemsForModel2,
|
|
116
116
|
isEphemeralInternalContext,
|
|
@@ -165,12 +165,13 @@ function connectionTokenResolverForTurn(input) {
|
|
|
165
165
|
|
|
166
166
|
// src/activities/agent-turn.ts
|
|
167
167
|
import {
|
|
168
|
-
|
|
168
|
+
assertTurnExecutionPolicyMatchesConfigV1,
|
|
169
169
|
calculateModelUsageCostMicros,
|
|
170
170
|
configuredModelPricing,
|
|
171
171
|
configuredStaticUsageLimits as configuredStaticUsageLimits2,
|
|
172
172
|
sandboxWarmRateMicrosPerSecond,
|
|
173
|
-
settingsWithResolvedModelContext
|
|
173
|
+
settingsWithResolvedModelContext,
|
|
174
|
+
resolveTurnExecutionPolicyV1
|
|
174
175
|
} from "@opengeni/config";
|
|
175
176
|
import { CancelledFailure } from "@temporalio/activity";
|
|
176
177
|
|
|
@@ -204,7 +205,7 @@ async function settingsWithEnabledCapabilityMcpServers(db, workspaceId, settings
|
|
|
204
205
|
const enabled = await listEnabledMcpCapabilityServers(db, workspaceId);
|
|
205
206
|
return settingsWithMcpCapabilityServers(settings, enabled);
|
|
206
207
|
}
|
|
207
|
-
async function settingsWithSessionMcpServersForRun(db, workspaceId, sessionId, settings) {
|
|
208
|
+
async function settingsWithSessionMcpServersForRun(db, workspaceId, sessionId, attemptId, settings) {
|
|
208
209
|
const encryptionKey = environmentsEncryptionKeyBytes(settings);
|
|
209
210
|
if (!encryptionKey) {
|
|
210
211
|
const metadata = await listSessionMcpServerMetadata(db, workspaceId, sessionId);
|
|
@@ -219,7 +220,7 @@ async function settingsWithSessionMcpServersForRun(db, workspaceId, sessionId, s
|
|
|
219
220
|
}
|
|
220
221
|
return settingsWithSessionMcpServers(
|
|
221
222
|
settings,
|
|
222
|
-
await listSessionMcpServersForRun(db, workspaceId, sessionId, encryptionKey ?? null)
|
|
223
|
+
await listSessionMcpServersForRun(db, workspaceId, sessionId, attemptId, encryptionKey ?? null)
|
|
223
224
|
);
|
|
224
225
|
}
|
|
225
226
|
function settingsWithSessionMcpServers(settings, servers) {
|
|
@@ -287,6 +288,7 @@ function withCodexProvider(settings) {
|
|
|
287
288
|
baseUrl: CODEX_PROVIDER_BASE_URL,
|
|
288
289
|
models: CODEX_FALLBACK_MODEL_SLUGS.map((slug) => ({
|
|
289
290
|
id: `${CODEX_MODEL_ID_PREFIX}${slug}`,
|
|
291
|
+
upstreamModelId: slug,
|
|
290
292
|
label: slug,
|
|
291
293
|
reasoningEffort: true,
|
|
292
294
|
// These three values are the live Codex CLI catalog policy, kept distinct
|
|
@@ -826,6 +828,7 @@ import {
|
|
|
826
828
|
CODEX_CLIENT_VERSION,
|
|
827
829
|
CODEX_FALLBACK_MODEL_SLUGS as CODEX_FALLBACK_MODEL_SLUGS2,
|
|
828
830
|
CodexReloginRequired,
|
|
831
|
+
classifyCodexResponseTimeoutError,
|
|
829
832
|
classifyCodexUsageLimitError,
|
|
830
833
|
codexRequestStorage,
|
|
831
834
|
isCodexTransportError
|
|
@@ -856,6 +859,9 @@ function workflowIdForSession(sessionId) {
|
|
|
856
859
|
return `session-${sessionId}`;
|
|
857
860
|
}
|
|
858
861
|
|
|
862
|
+
// src/activities/agent-turn.ts
|
|
863
|
+
import { enabledCapabilityMcpToolRefs, resolveSessionToolPolicy } from "@opengeni/core";
|
|
864
|
+
|
|
859
865
|
// src/activities/context-compaction.ts
|
|
860
866
|
import {
|
|
861
867
|
applyContextCompaction,
|
|
@@ -1097,7 +1103,7 @@ import {
|
|
|
1097
1103
|
} from "@opengeni/db";
|
|
1098
1104
|
import { createGitHubAppInstallationTokenWithExpiry, githubAppBotIdentity } from "@opengeni/github";
|
|
1099
1105
|
var TOOLSPACE_TOKEN_TTL_SECONDS = 60 * 60;
|
|
1100
|
-
async function mintSandboxToolspaceToken(settings, scope,
|
|
1106
|
+
async function mintSandboxToolspaceToken(settings, scope, authority, nowMs = Date.now()) {
|
|
1101
1107
|
if (!settings.toolspaceEnabled || !settings.delegationSecret) {
|
|
1102
1108
|
return void 0;
|
|
1103
1109
|
}
|
|
@@ -1105,10 +1111,10 @@ async function mintSandboxToolspaceToken(settings, scope, sessionId, runId, nowM
|
|
|
1105
1111
|
const token = await signDelegatedAccessToken(settings.delegationSecret, {
|
|
1106
1112
|
accountId: scope.accountId,
|
|
1107
1113
|
workspaceId: scope.workspaceId,
|
|
1108
|
-
subjectId: `sandbox:${
|
|
1114
|
+
subjectId: `sandbox:${authority.sessionId}`,
|
|
1109
1115
|
subjectLabel: "sandbox toolspace",
|
|
1110
1116
|
permissions: ["toolspace:call"],
|
|
1111
|
-
sessionId,
|
|
1117
|
+
sessionId: authority.sessionId,
|
|
1112
1118
|
exp: expiresAtSeconds
|
|
1113
1119
|
});
|
|
1114
1120
|
return { token, expiresAt: new Date(expiresAtSeconds * 1e3) };
|
|
@@ -1155,7 +1161,7 @@ async function sandboxEnvironmentForRun(settings, resources, workspaceEnvironmen
|
|
|
1155
1161
|
const stableOptions = options.scope ? { workspaceId: options.scope.workspaceId } : {};
|
|
1156
1162
|
const environment = stableSandboxEnvironmentForRun(settings, workspaceEnvironment, stableOptions);
|
|
1157
1163
|
const toolspaceScope = options.scope;
|
|
1158
|
-
const toolspaceToken = toolspaceScope && options.
|
|
1164
|
+
const toolspaceToken = toolspaceScope && options.toolspaceAuthority ? await mintSandboxToolspaceToken(settings, toolspaceScope, options.toolspaceAuthority) : void 0;
|
|
1159
1165
|
if (toolspaceToken && toolspaceScope) {
|
|
1160
1166
|
environment.OPENGENI_TOOLSPACE_URL ??= firstPartyMcpWorkspaceUrl(
|
|
1161
1167
|
settings,
|
|
@@ -1229,6 +1235,7 @@ async function mintRunGitTokensWithIdentity(settings, selections, options) {
|
|
|
1229
1235
|
let identity = null;
|
|
1230
1236
|
for (const selection of selections) {
|
|
1231
1237
|
let token = null;
|
|
1238
|
+
let transport;
|
|
1232
1239
|
let tokenExpiresAt;
|
|
1233
1240
|
if (selection.provider === "github" && selection.installationId > 0 && selection.repositoryIds.length > 0) {
|
|
1234
1241
|
await options.authorizeGitHubTokenMint?.({
|
|
@@ -1252,6 +1259,7 @@ async function mintRunGitTokensWithIdentity(settings, selections, options) {
|
|
|
1252
1259
|
);
|
|
1253
1260
|
}
|
|
1254
1261
|
token = minted.token;
|
|
1262
|
+
transport = validatedGitCredentialTransport(selection, minted);
|
|
1255
1263
|
tokenExpiresAt = minted.expiresAt ? validatedGitCredentialExpiry(selection.provider, minted.expiresAt) : void 0;
|
|
1256
1264
|
if (minted.identity) {
|
|
1257
1265
|
identity = minted.identity;
|
|
@@ -1272,6 +1280,7 @@ async function mintRunGitTokensWithIdentity(settings, selections, options) {
|
|
|
1272
1280
|
credentialBindingId: selection.credentialBindingId,
|
|
1273
1281
|
provider: selection.provider,
|
|
1274
1282
|
token,
|
|
1283
|
+
...transport ? { transport } : {},
|
|
1275
1284
|
...tokenExpiresAt ? { expiresAt: tokenExpiresAt } : {}
|
|
1276
1285
|
});
|
|
1277
1286
|
}
|
|
@@ -1284,6 +1293,7 @@ async function mintRunGitTokensWithIdentity(settings, selections, options) {
|
|
|
1284
1293
|
}
|
|
1285
1294
|
for (const binding of bindings) {
|
|
1286
1295
|
binding.providerBindingCount = counts.get(binding.provider) ?? 1;
|
|
1296
|
+
if (binding.transport?.kind === "http_broker") continue;
|
|
1287
1297
|
if (counts.get(binding.provider) !== 1) continue;
|
|
1288
1298
|
gitTokens[binding.provider] = binding.token;
|
|
1289
1299
|
if (binding.expiresAt) expiresAt[binding.provider] = binding.expiresAt;
|
|
@@ -1297,6 +1307,83 @@ function validatedGitCredentialExpiry(provider, value) {
|
|
|
1297
1307
|
}
|
|
1298
1308
|
return new Date(timestamp).toISOString();
|
|
1299
1309
|
}
|
|
1310
|
+
var MAX_GIT_HTTP_BROKER_ROUTES = 256;
|
|
1311
|
+
var MAX_GIT_HTTP_BROKER_URI_LENGTH = 4096;
|
|
1312
|
+
function validatedGitCredentialTransport(selection, minted) {
|
|
1313
|
+
const transport = minted.transport;
|
|
1314
|
+
if (!transport) return void 0;
|
|
1315
|
+
if (typeof transport !== "object" || transport.kind !== "http_broker" || !Array.isArray(transport.repositories)) {
|
|
1316
|
+
throw new Error(
|
|
1317
|
+
`connection-credential provider (${selection.provider}) returned an invalid Git credential transport`
|
|
1318
|
+
);
|
|
1319
|
+
}
|
|
1320
|
+
const requestedUris = new Set(selection.repositoryRefs.map((repository) => repository.uri));
|
|
1321
|
+
if (requestedUris.size === 0 || requestedUris.size > MAX_GIT_HTTP_BROKER_ROUTES || transport.repositories.length !== requestedUris.size) {
|
|
1322
|
+
throw new Error(
|
|
1323
|
+
`connection-credential provider (${selection.provider}) returned an incomplete Git HTTP broker route set`
|
|
1324
|
+
);
|
|
1325
|
+
}
|
|
1326
|
+
const echoedUris = /* @__PURE__ */ new Set();
|
|
1327
|
+
const brokerUris = /* @__PURE__ */ new Set();
|
|
1328
|
+
const repositories = transport.repositories.map((route) => {
|
|
1329
|
+
if (!route || typeof route !== "object" || typeof route.repositoryUri !== "string" || typeof route.brokerUri !== "string") {
|
|
1330
|
+
throw new Error(
|
|
1331
|
+
`connection-credential provider (${selection.provider}) returned an invalid Git HTTP broker route`
|
|
1332
|
+
);
|
|
1333
|
+
}
|
|
1334
|
+
if (!requestedUris.has(route.repositoryUri) || echoedUris.has(route.repositoryUri)) {
|
|
1335
|
+
throw new Error(
|
|
1336
|
+
`connection-credential provider (${selection.provider}) returned an unexpected Git HTTP broker repository`
|
|
1337
|
+
);
|
|
1338
|
+
}
|
|
1339
|
+
echoedUris.add(route.repositoryUri);
|
|
1340
|
+
if (route.brokerUri.length === 0 || route.brokerUri.length > MAX_GIT_HTTP_BROKER_URI_LENGTH || route.brokerUri.trim() !== route.brokerUri) {
|
|
1341
|
+
throw new Error(
|
|
1342
|
+
`connection-credential provider (${selection.provider}) returned an invalid Git HTTP broker URI`
|
|
1343
|
+
);
|
|
1344
|
+
}
|
|
1345
|
+
let brokerUrl;
|
|
1346
|
+
try {
|
|
1347
|
+
brokerUrl = new URL(route.brokerUri);
|
|
1348
|
+
} catch {
|
|
1349
|
+
throw new Error(
|
|
1350
|
+
`connection-credential provider (${selection.provider}) returned an invalid Git HTTP broker URI`
|
|
1351
|
+
);
|
|
1352
|
+
}
|
|
1353
|
+
if (brokerUrl.protocol !== "https:" || brokerUrl.username || brokerUrl.password || brokerUrl.search || brokerUrl.hash || brokerUris.has(brokerUrl.href) || brokerUrl.href === route.repositoryUri) {
|
|
1354
|
+
throw new Error(
|
|
1355
|
+
`connection-credential provider (${selection.provider}) returned an unsafe Git HTTP broker URI`
|
|
1356
|
+
);
|
|
1357
|
+
}
|
|
1358
|
+
brokerUris.add(brokerUrl.href);
|
|
1359
|
+
return {
|
|
1360
|
+
repositoryUri: route.repositoryUri,
|
|
1361
|
+
brokerUri: brokerUrl.href
|
|
1362
|
+
};
|
|
1363
|
+
});
|
|
1364
|
+
if ([...requestedUris].some((uri) => !echoedUris.has(uri))) {
|
|
1365
|
+
throw new Error(
|
|
1366
|
+
`connection-credential provider (${selection.provider}) returned an incomplete Git HTTP broker route set`
|
|
1367
|
+
);
|
|
1368
|
+
}
|
|
1369
|
+
return { kind: "http_broker", repositories };
|
|
1370
|
+
}
|
|
1371
|
+
function assertGitCredentialRenewalTransportUnchanged(initial, renewed) {
|
|
1372
|
+
if (!initial.transport && !renewed.transport) return;
|
|
1373
|
+
if (!initial.transport || !renewed.transport || initial.transport.kind !== renewed.transport.kind) {
|
|
1374
|
+
throw new Error(
|
|
1375
|
+
`renewed ${initial.provider} credential changed transport for binding ${initial.credentialBindingId}`
|
|
1376
|
+
);
|
|
1377
|
+
}
|
|
1378
|
+
const routeKey = (route) => `${route.repositoryUri}\0${route.brokerUri}`;
|
|
1379
|
+
const initialRoutes = initial.transport.repositories.map(routeKey).sort();
|
|
1380
|
+
const renewedRoutes = renewed.transport.repositories.map(routeKey).sort();
|
|
1381
|
+
if (initialRoutes.length !== renewedRoutes.length || initialRoutes.some((route, index) => route !== renewedRoutes[index])) {
|
|
1382
|
+
throw new Error(
|
|
1383
|
+
`renewed ${initial.provider} credential changed Git HTTP broker routes for binding ${initial.credentialBindingId}`
|
|
1384
|
+
);
|
|
1385
|
+
}
|
|
1386
|
+
}
|
|
1300
1387
|
async function resolveRunGitIdentityWithSelections(settings, selections, options) {
|
|
1301
1388
|
let identity = null;
|
|
1302
1389
|
for (const selection of selections) {
|
|
@@ -3621,10 +3708,13 @@ function makeTurnOpJournal(context, details) {
|
|
|
3621
3708
|
import { createHash } from "crypto";
|
|
3622
3709
|
import { errorCodeToJSON } from "@opengeni/agent-proto";
|
|
3623
3710
|
import {
|
|
3624
|
-
SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES
|
|
3711
|
+
SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
|
|
3712
|
+
modelUsageTokenCountOrNull
|
|
3625
3713
|
} from "@opengeni/runtime";
|
|
3626
3714
|
var turnTrackers = /* @__PURE__ */ new WeakMap();
|
|
3627
3715
|
var creditBalanceGaugeAccounts = /* @__PURE__ */ new WeakMap();
|
|
3716
|
+
var modelCacheCounterTotals = /* @__PURE__ */ new WeakMap();
|
|
3717
|
+
var MAX_MODEL_CACHE_COUNTER_TOTAL = 1e12;
|
|
3628
3718
|
function observabilityEventLogger(observability) {
|
|
3629
3719
|
return {
|
|
3630
3720
|
debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),
|
|
@@ -4004,7 +4094,8 @@ var MODEL_INPUT_TOKENS_BUCKETS = [
|
|
|
4004
4094
|
1e6
|
|
4005
4095
|
];
|
|
4006
4096
|
function recordModelInputTokens(observability, provider, inputTokens) {
|
|
4007
|
-
|
|
4097
|
+
const normalizedInputTokens = modelUsageTokenCountOrNull(inputTokens);
|
|
4098
|
+
if (normalizedInputTokens === null || normalizedInputTokens === 0) {
|
|
4008
4099
|
return;
|
|
4009
4100
|
}
|
|
4010
4101
|
observability.observeHistogram({
|
|
@@ -4012,7 +4103,7 @@ function recordModelInputTokens(observability, provider, inputTokens) {
|
|
|
4012
4103
|
help: "Observed input (context) tokens per model response, by provider.",
|
|
4013
4104
|
buckets: MODEL_INPUT_TOKENS_BUCKETS,
|
|
4014
4105
|
labels: { provider },
|
|
4015
|
-
value:
|
|
4106
|
+
value: normalizedInputTokens
|
|
4016
4107
|
});
|
|
4017
4108
|
}
|
|
4018
4109
|
function recordContextCompaction(observability, trigger) {
|
|
@@ -4038,17 +4129,35 @@ var MODEL_CACHE_HIT_RATIO_BUCKETS = [
|
|
|
4038
4129
|
1
|
|
4039
4130
|
];
|
|
4040
4131
|
function recordModelCacheTokens(observability, provider, input) {
|
|
4041
|
-
const cached =
|
|
4042
|
-
const
|
|
4043
|
-
|
|
4044
|
-
|
|
4132
|
+
const cached = modelUsageTokenCountOrNull(input.cachedTokens);
|
|
4133
|
+
const cacheWrite = modelUsageTokenCountOrNull(input.cacheWriteTokens);
|
|
4134
|
+
const prompt = modelUsageTokenCountOrNull(input.promptTokens);
|
|
4135
|
+
incrementBoundedModelCacheCounter(observability, {
|
|
4136
|
+
name: "opengeni_model_cache_read_telemetry_total",
|
|
4137
|
+
help: "Authoritative model calls with reported or missing cache-read telemetry, by provider.",
|
|
4138
|
+
provider,
|
|
4139
|
+
labels: { provider, status: cached === null ? "missing" : "reported" },
|
|
4140
|
+
amount: 1
|
|
4141
|
+
});
|
|
4142
|
+
if (cached !== null && cached > 0) {
|
|
4143
|
+
incrementBoundedModelCacheCounter(observability, {
|
|
4045
4144
|
name: "opengeni_model_cached_tokens_total",
|
|
4046
4145
|
help: "Total prompt tokens served from the provider's prompt cache, by provider.",
|
|
4146
|
+
provider,
|
|
4047
4147
|
labels: { provider },
|
|
4048
4148
|
amount: cached
|
|
4049
4149
|
});
|
|
4050
4150
|
}
|
|
4051
|
-
if (
|
|
4151
|
+
if (cacheWrite !== null && cacheWrite > 0) {
|
|
4152
|
+
incrementBoundedModelCacheCounter(observability, {
|
|
4153
|
+
name: "opengeni_model_cache_write_tokens_total",
|
|
4154
|
+
help: "Total prompt tokens written to the provider's prompt cache, by provider.",
|
|
4155
|
+
provider,
|
|
4156
|
+
labels: { provider },
|
|
4157
|
+
amount: cacheWrite
|
|
4158
|
+
});
|
|
4159
|
+
}
|
|
4160
|
+
if (cached !== null && prompt !== null && prompt > 0) {
|
|
4052
4161
|
observability.observeHistogram({
|
|
4053
4162
|
name: "opengeni_model_cache_hit_ratio",
|
|
4054
4163
|
help: "Per-call prompt-cache hit ratio (cached/prompt tokens) by provider.",
|
|
@@ -4060,8 +4169,28 @@ function recordModelCacheTokens(observability, provider, input) {
|
|
|
4060
4169
|
});
|
|
4061
4170
|
}
|
|
4062
4171
|
}
|
|
4063
|
-
function
|
|
4064
|
-
|
|
4172
|
+
function incrementBoundedModelCacheCounter(observability, input) {
|
|
4173
|
+
const totals = modelCacheCounterTotals.get(observability) ?? /* @__PURE__ */ new Map();
|
|
4174
|
+
if (!modelCacheCounterTotals.has(observability)) {
|
|
4175
|
+
modelCacheCounterTotals.set(observability, totals);
|
|
4176
|
+
}
|
|
4177
|
+
const labelKey = Object.entries(input.labels).sort(([left], [right]) => left.localeCompare(right)).map(([key2, value]) => `${key2}=${value}`).join(",");
|
|
4178
|
+
const key = `${input.name}:${labelKey}`;
|
|
4179
|
+
const current = totals.get(key) ?? 0;
|
|
4180
|
+
if (input.amount > MAX_MODEL_CACHE_COUNTER_TOTAL - current) {
|
|
4181
|
+
observability.warn("model cache metric cumulative limit reached", {
|
|
4182
|
+
provider: input.provider,
|
|
4183
|
+
metric: input.name
|
|
4184
|
+
});
|
|
4185
|
+
return;
|
|
4186
|
+
}
|
|
4187
|
+
observability.incrementCounter({
|
|
4188
|
+
name: input.name,
|
|
4189
|
+
help: input.help,
|
|
4190
|
+
labels: input.labels,
|
|
4191
|
+
amount: input.amount
|
|
4192
|
+
});
|
|
4193
|
+
totals.set(key, current + input.amount);
|
|
4065
4194
|
}
|
|
4066
4195
|
function stableAccountHash(credentialId) {
|
|
4067
4196
|
if (!credentialId) {
|
|
@@ -4939,10 +5068,27 @@ import {
|
|
|
4939
5068
|
} from "@opengeni/runtime";
|
|
4940
5069
|
import {
|
|
4941
5070
|
CAPABILITY_DESCRIPTORS,
|
|
4942
|
-
evaluateWorkspaceModelPolicy as evaluateWorkspaceModelPolicy2
|
|
5071
|
+
evaluateWorkspaceModelPolicy as evaluateWorkspaceModelPolicy2,
|
|
5072
|
+
readTurnExecutionPolicyV1
|
|
4943
5073
|
} from "@opengeni/contracts";
|
|
4944
5074
|
import { createHash as createHash3, randomUUID } from "crypto";
|
|
4945
5075
|
var PROVIDER_BACKPRESSURE_DELAY_MS = 6e4;
|
|
5076
|
+
function turnExecutionPolicyBillingIdentity(policy) {
|
|
5077
|
+
return {
|
|
5078
|
+
externallyBilled: policy.billing.metering === "external",
|
|
5079
|
+
codexSubscription: policy.providerId === "codex-subscription" && policy.credentialSource.kind === "connected_subscription" && policy.credentialSource.provider === "codex"
|
|
5080
|
+
};
|
|
5081
|
+
}
|
|
5082
|
+
function legacyTurnExecutionPolicyInput(turn) {
|
|
5083
|
+
const explicit = turn.source === "user" || turn.source === "api";
|
|
5084
|
+
return {
|
|
5085
|
+
modelId: turn.model,
|
|
5086
|
+
requestedModelId: explicit ? turn.model : null,
|
|
5087
|
+
modelSource: explicit ? "explicit" : "continuation",
|
|
5088
|
+
reasoningEffort: turn.reasoningEffort,
|
|
5089
|
+
reasoningSource: explicit ? "explicit" : "continuation"
|
|
5090
|
+
};
|
|
5091
|
+
}
|
|
4946
5092
|
function providerRecoveryResult() {
|
|
4947
5093
|
return {
|
|
4948
5094
|
status: "recovering",
|
|
@@ -5147,18 +5293,6 @@ function modelUsageSourceKey(input) {
|
|
|
5147
5293
|
}
|
|
5148
5294
|
return input.dispatchId ? `${input.dispatchId}:${input.positionalKey}` : input.positionalKey;
|
|
5149
5295
|
}
|
|
5150
|
-
function providerContextTokens(usage) {
|
|
5151
|
-
const total = usage?.totalTokens;
|
|
5152
|
-
if (typeof total === "number" && Number.isFinite(total) && total > 0) {
|
|
5153
|
-
return total;
|
|
5154
|
-
}
|
|
5155
|
-
const input = usage?.inputTokens;
|
|
5156
|
-
if (typeof input !== "number" || !Number.isFinite(input) || input <= 0) {
|
|
5157
|
-
return null;
|
|
5158
|
-
}
|
|
5159
|
-
const output = usage?.outputTokens;
|
|
5160
|
-
return input + (typeof output === "number" && Number.isFinite(output) && output > 0 ? output : 0);
|
|
5161
|
-
}
|
|
5162
5296
|
async function recordCompletedModelCallBeforeOwnershipFences(input) {
|
|
5163
5297
|
await input.renewLease();
|
|
5164
5298
|
await input.recordUsage();
|
|
@@ -5167,13 +5301,167 @@ async function recordCompletedModelCallBeforeOwnershipFences(input) {
|
|
|
5167
5301
|
}
|
|
5168
5302
|
await input.recordAttemptSignals?.();
|
|
5169
5303
|
}
|
|
5304
|
+
function createModelResponseUsageEventState(claimedSourceKeys = /* @__PURE__ */ new Set()) {
|
|
5305
|
+
return {
|
|
5306
|
+
responseUsageCount: 0,
|
|
5307
|
+
providerContextRevision: 0,
|
|
5308
|
+
lastProviderContextTokensObserved: null,
|
|
5309
|
+
claimedSourceKeys
|
|
5310
|
+
};
|
|
5311
|
+
}
|
|
5312
|
+
function createCompactionModelUsageEventState(claimedSourceKeys = /* @__PURE__ */ new Set()) {
|
|
5313
|
+
return { usageCount: 0, claimedSourceKeys };
|
|
5314
|
+
}
|
|
5315
|
+
function modelResponseUsageContextSignal(state) {
|
|
5316
|
+
return state.lastProviderContextTokensObserved === null ? null : {
|
|
5317
|
+
revision: state.providerContextRevision,
|
|
5318
|
+
totalTokens: state.lastProviderContextTokensObserved
|
|
5319
|
+
};
|
|
5320
|
+
}
|
|
5321
|
+
async function processModelResponseUsageEvent(input) {
|
|
5322
|
+
const responseUsage = modelResponseUsageFromSdkEvent(input.event);
|
|
5323
|
+
if (!responseUsage) {
|
|
5324
|
+
return { status: "not_usage" };
|
|
5325
|
+
}
|
|
5326
|
+
const responseOrdinal = input.state.responseUsageCount + 1;
|
|
5327
|
+
const sourceKey = modelUsageSourceKey({
|
|
5328
|
+
responseId: responseUsage.responseId,
|
|
5329
|
+
dispatchId: input.dispatchId,
|
|
5330
|
+
positionalKey: `response-${responseOrdinal}`
|
|
5331
|
+
});
|
|
5332
|
+
if (input.state.claimedSourceKeys.has(sourceKey)) {
|
|
5333
|
+
return { status: "duplicate", sourceKey };
|
|
5334
|
+
}
|
|
5335
|
+
input.state.claimedSourceKeys.add(sourceKey);
|
|
5336
|
+
input.state.responseUsageCount = responseOrdinal;
|
|
5337
|
+
const normalizedUsage = normalizeModelCallUsage(responseUsage.usage);
|
|
5338
|
+
const accountContext = modelCallAccountContext({
|
|
5339
|
+
servingCredentialId: input.servingCredentialId,
|
|
5340
|
+
priorSessionCredentialId: input.priorSessionCredentialId,
|
|
5341
|
+
isFirstCallOfTurn: responseOrdinal === 1
|
|
5342
|
+
});
|
|
5343
|
+
let authoritative = false;
|
|
5344
|
+
await recordCompletedModelCallBeforeOwnershipFences({
|
|
5345
|
+
renewLease: input.renewLease,
|
|
5346
|
+
leaseLost: input.leaseLost,
|
|
5347
|
+
leaseLostMessage: input.leaseLostMessage,
|
|
5348
|
+
recordUsage: async () => {
|
|
5349
|
+
await recordModelUsageAndDebitCredits(input.settings, input.db, {
|
|
5350
|
+
accountId: input.accountId,
|
|
5351
|
+
workspaceId: input.workspaceId,
|
|
5352
|
+
sessionId: input.sessionId,
|
|
5353
|
+
turnId: input.turnId,
|
|
5354
|
+
turnAttemptId: input.turnAttemptId,
|
|
5355
|
+
model: input.model,
|
|
5356
|
+
externallyBilled: input.externallyBilled,
|
|
5357
|
+
usage: responseUsage.usage,
|
|
5358
|
+
normalizedUsage,
|
|
5359
|
+
sourceKey,
|
|
5360
|
+
observability: input.observability
|
|
5361
|
+
});
|
|
5362
|
+
authoritative = await emitModelCallUsage({
|
|
5363
|
+
observability: input.observability,
|
|
5364
|
+
publish: input.publish,
|
|
5365
|
+
accountId: input.accountId,
|
|
5366
|
+
workspaceId: input.workspaceId,
|
|
5367
|
+
sessionId: input.sessionId,
|
|
5368
|
+
turnId: input.turnId,
|
|
5369
|
+
provider: input.provider,
|
|
5370
|
+
providerApi: input.providerApi,
|
|
5371
|
+
model: input.model,
|
|
5372
|
+
sourceKey,
|
|
5373
|
+
usage: responseUsage,
|
|
5374
|
+
normalizedUsage,
|
|
5375
|
+
servingAccountHash: accountContext.servingAccountHash,
|
|
5376
|
+
accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
|
|
5377
|
+
emittedSourceKeys: input.emittedSourceKeys
|
|
5378
|
+
});
|
|
5379
|
+
const observedInput = normalizedUsage.telemetry.inputTokens;
|
|
5380
|
+
if (authoritative && observedInput !== null && observedInput > 0) {
|
|
5381
|
+
recordModelInputTokens(input.observability, input.metricProvider, observedInput);
|
|
5382
|
+
}
|
|
5383
|
+
},
|
|
5384
|
+
recordAttemptSignals: async () => {
|
|
5385
|
+
if (!authoritative) return;
|
|
5386
|
+
const observedTotal = normalizedUsage.totalTokens;
|
|
5387
|
+
if (observedTotal !== null) {
|
|
5388
|
+
input.state.lastProviderContextTokensObserved = observedTotal;
|
|
5389
|
+
input.state.providerContextRevision += 1;
|
|
5390
|
+
}
|
|
5391
|
+
const observedInput = normalizedUsage.telemetry.inputTokens;
|
|
5392
|
+
if (observedInput !== null && observedInput > 0) {
|
|
5393
|
+
await input.setLastInputTokens(observedInput);
|
|
5394
|
+
}
|
|
5395
|
+
}
|
|
5396
|
+
});
|
|
5397
|
+
return { status: "processed", sourceKey, authoritative };
|
|
5398
|
+
}
|
|
5399
|
+
async function processCompactionModelUsageEvent(input) {
|
|
5400
|
+
const usageOrdinal = input.state.usageCount + 1;
|
|
5401
|
+
const sourceKey = modelUsageSourceKey({
|
|
5402
|
+
responseId: input.usage.responseId,
|
|
5403
|
+
dispatchId: input.dispatchId,
|
|
5404
|
+
positionalKey: `compaction-${usageOrdinal}`
|
|
5405
|
+
});
|
|
5406
|
+
if (input.state.claimedSourceKeys.has(sourceKey)) {
|
|
5407
|
+
return { status: "duplicate", sourceKey };
|
|
5408
|
+
}
|
|
5409
|
+
input.state.claimedSourceKeys.add(sourceKey);
|
|
5410
|
+
input.state.usageCount = usageOrdinal;
|
|
5411
|
+
const accountContext = modelCallAccountContext({
|
|
5412
|
+
servingCredentialId: input.servingCredentialId,
|
|
5413
|
+
priorSessionCredentialId: input.priorSessionCredentialId,
|
|
5414
|
+
isFirstCallOfTurn: usageOrdinal === 1
|
|
5415
|
+
});
|
|
5416
|
+
const normalizedUsage = normalizeModelCallUsage(input.usage.usage);
|
|
5417
|
+
let authoritative = false;
|
|
5418
|
+
await recordCompletedModelCallBeforeOwnershipFences({
|
|
5419
|
+
renewLease: input.renewLease,
|
|
5420
|
+
leaseLost: input.leaseLost,
|
|
5421
|
+
leaseLostMessage: input.leaseLostMessage,
|
|
5422
|
+
recordUsage: async () => {
|
|
5423
|
+
await recordModelUsageAndDebitCredits(input.settings, input.db, {
|
|
5424
|
+
accountId: input.accountId,
|
|
5425
|
+
workspaceId: input.workspaceId,
|
|
5426
|
+
sessionId: input.sessionId,
|
|
5427
|
+
turnId: input.turnId,
|
|
5428
|
+
turnAttemptId: input.turnAttemptId,
|
|
5429
|
+
model: input.model,
|
|
5430
|
+
externallyBilled: input.externallyBilled,
|
|
5431
|
+
usage: input.usage.usage,
|
|
5432
|
+
normalizedUsage,
|
|
5433
|
+
sourceKey,
|
|
5434
|
+
observability: input.observability
|
|
5435
|
+
});
|
|
5436
|
+
authoritative = await emitModelCallUsage({
|
|
5437
|
+
observability: input.observability,
|
|
5438
|
+
publish: input.publish,
|
|
5439
|
+
accountId: input.accountId,
|
|
5440
|
+
workspaceId: input.workspaceId,
|
|
5441
|
+
sessionId: input.sessionId,
|
|
5442
|
+
turnId: input.turnId,
|
|
5443
|
+
provider: input.provider,
|
|
5444
|
+
providerApi: input.providerApi,
|
|
5445
|
+
model: input.model,
|
|
5446
|
+
sourceKey,
|
|
5447
|
+
usage: input.usage,
|
|
5448
|
+
normalizedUsage,
|
|
5449
|
+
servingAccountHash: accountContext.servingAccountHash,
|
|
5450
|
+
accountChangedFromPrevCall: accountContext.accountChangedFromPrevCall,
|
|
5451
|
+
emittedSourceKeys: input.emittedSourceKeys
|
|
5452
|
+
});
|
|
5453
|
+
}
|
|
5454
|
+
});
|
|
5455
|
+
return { status: "processed", sourceKey, authoritative };
|
|
5456
|
+
}
|
|
5170
5457
|
async function emitModelCallUsage(input) {
|
|
5171
5458
|
const usage = input.usage && typeof input.usage === "object" && "usage" in input.usage ? input.usage.usage : null;
|
|
5172
5459
|
if (!usage || typeof usage !== "object") {
|
|
5173
|
-
return;
|
|
5460
|
+
return false;
|
|
5174
5461
|
}
|
|
5175
|
-
if (input.emittedSourceKeys?.has(input.sourceKey)) return;
|
|
5176
|
-
const
|
|
5462
|
+
if (input.emittedSourceKeys?.has(input.sourceKey)) return false;
|
|
5463
|
+
const normalizedUsage = input.normalizedUsage ?? normalizeModelCallUsage(usage);
|
|
5464
|
+
const telemetry = normalizedUsage.telemetry;
|
|
5177
5465
|
const appended = await input.publish?.(
|
|
5178
5466
|
[
|
|
5179
5467
|
{
|
|
@@ -5197,7 +5485,7 @@ async function emitModelCallUsage(input) {
|
|
|
5197
5485
|
const authoritative = appended?.events.some(
|
|
5198
5486
|
(event) => event.type === "agent.model.usage" && event.turnAssociation === "current" && event.payload !== null && typeof event.payload === "object" && event.payload.sourceKey === input.sourceKey
|
|
5199
5487
|
);
|
|
5200
|
-
if (!authoritative) return;
|
|
5488
|
+
if (!authoritative) return false;
|
|
5201
5489
|
try {
|
|
5202
5490
|
input.observability.info("model call usage", {
|
|
5203
5491
|
accountId: input.accountId,
|
|
@@ -5211,12 +5499,34 @@ async function emitModelCallUsage(input) {
|
|
|
5211
5499
|
inputTokens: telemetry.inputTokens,
|
|
5212
5500
|
outputTokens: telemetry.outputTokens,
|
|
5213
5501
|
cachedTokens: telemetry.cachedTokens,
|
|
5502
|
+
cacheWriteTokens: telemetry.cacheWriteTokens,
|
|
5214
5503
|
reasoningTokens: telemetry.reasoningTokens,
|
|
5215
5504
|
...input.servingAccountHash !== void 0 ? { servingAccountHash: input.servingAccountHash } : {},
|
|
5216
5505
|
...input.accountChangedFromPrevCall !== void 0 ? { accountChangedFromPrevCall: input.accountChangedFromPrevCall } : {}
|
|
5217
5506
|
});
|
|
5507
|
+
if (normalizedUsage.rejectedFields.length > 0) {
|
|
5508
|
+
input.observability.warn("model call usage fields rejected", {
|
|
5509
|
+
provider: input.provider,
|
|
5510
|
+
providerApi: input.providerApi,
|
|
5511
|
+
model: input.model,
|
|
5512
|
+
sourceKey: input.sourceKey,
|
|
5513
|
+
rejectedFields: normalizedUsage.rejectedFields.join(",")
|
|
5514
|
+
});
|
|
5515
|
+
}
|
|
5516
|
+
} catch {
|
|
5517
|
+
}
|
|
5518
|
+
try {
|
|
5519
|
+
applyCodexCacheTelemetry(input.observability, input.provider, normalizedUsage);
|
|
5218
5520
|
} catch {
|
|
5219
5521
|
}
|
|
5522
|
+
return true;
|
|
5523
|
+
}
|
|
5524
|
+
function applyCodexCacheTelemetry(observability, provider, normalizedUsage) {
|
|
5525
|
+
recordModelCacheTokens(observability, provider, {
|
|
5526
|
+
cachedTokens: normalizedUsage.telemetry.cachedTokens,
|
|
5527
|
+
cacheWriteTokens: normalizedUsage.telemetry.cacheWriteTokens,
|
|
5528
|
+
promptTokens: normalizedUsage.telemetry.inputTokens
|
|
5529
|
+
});
|
|
5220
5530
|
}
|
|
5221
5531
|
function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = persistedHistoryCount, toolOutputTruncationTokens) {
|
|
5222
5532
|
const sanitized = sanitizeHistoryItemsForModel2(rawHistory, toolOutputTruncationTokens).filter(
|
|
@@ -5575,6 +5885,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
5575
5885
|
runtimeMetricsHooksForObservability(observability)
|
|
5576
5886
|
);
|
|
5577
5887
|
let isCodexTurn = false;
|
|
5888
|
+
let isExternallyBilledTurn = false;
|
|
5578
5889
|
let executionGeneration = 0;
|
|
5579
5890
|
let redispatchesAtDispatch = 0;
|
|
5580
5891
|
const setLastInputTokensFenced = async (lastInputTokens) => {
|
|
@@ -5998,6 +6309,35 @@ function createRunAgentTurnActivity(services) {
|
|
|
5998
6309
|
}
|
|
5999
6310
|
const turn = claim.turn;
|
|
6000
6311
|
turnId = turn.id;
|
|
6312
|
+
executionGeneration = turn.executionGeneration;
|
|
6313
|
+
const claimedPolicy = readTurnExecutionPolicyV1(turn.metadata);
|
|
6314
|
+
const policyForAbsent = claimedPolicy.kind === "valid" ? claimedPolicy.policy : resolveTurnExecutionPolicyV1(capabilitySettings, legacyTurnExecutionPolicyInput(turn));
|
|
6315
|
+
const installedPolicy = await installOrReadTurnExecutionPolicyForAttempt(db, {
|
|
6316
|
+
accountId: input.accountId,
|
|
6317
|
+
workspaceId: input.workspaceId,
|
|
6318
|
+
sessionId: input.sessionId,
|
|
6319
|
+
turnId,
|
|
6320
|
+
executionGeneration,
|
|
6321
|
+
attemptId: input.attemptId,
|
|
6322
|
+
policyForAbsent
|
|
6323
|
+
});
|
|
6324
|
+
if (!installedPolicy.accepted) {
|
|
6325
|
+
throw new TurnAttemptFencedError(
|
|
6326
|
+
`turn execution policy was fenced: ${installedPolicy.reason}`
|
|
6327
|
+
);
|
|
6328
|
+
}
|
|
6329
|
+
const verifiedExecutionPolicy = assertTurnExecutionPolicyMatchesConfigV1(
|
|
6330
|
+
capabilitySettings,
|
|
6331
|
+
installedPolicy.policy,
|
|
6332
|
+
{
|
|
6333
|
+
modelId: turn.model,
|
|
6334
|
+
reasoningEffort: turn.reasoningEffort
|
|
6335
|
+
}
|
|
6336
|
+
);
|
|
6337
|
+
const turnExecutionPolicy = verifiedExecutionPolicy.policy;
|
|
6338
|
+
const billingIdentity = turnExecutionPolicyBillingIdentity(turnExecutionPolicy);
|
|
6339
|
+
isExternallyBilledTurn = billingIdentity.externallyBilled;
|
|
6340
|
+
isCodexTurn = billingIdentity.codexSubscription;
|
|
6001
6341
|
triggerEventId = turn.triggerEventId;
|
|
6002
6342
|
const trigger = await getSessionEvent2(db, input.workspaceId, triggerEventId);
|
|
6003
6343
|
if (!trigger) {
|
|
@@ -6010,27 +6350,19 @@ function createRunAgentTurnActivity(services) {
|
|
|
6010
6350
|
trigger
|
|
6011
6351
|
);
|
|
6012
6352
|
triggerType = trigger.type;
|
|
6013
|
-
executionGeneration = turn.executionGeneration;
|
|
6014
6353
|
const latestTurnState = await getLatestRunState2(db, input.workspaceId, input.sessionId);
|
|
6015
6354
|
const continuationCodexCredentialId = latestTurnState?.turnId === turnId ? latestTurnState.frozenCodexCredentialId : null;
|
|
6016
6355
|
redispatchesAtDispatch = Number(
|
|
6017
6356
|
turn.metadata?.workerDeathRedispatches ?? 0
|
|
6018
6357
|
);
|
|
6019
6358
|
turnLifecycleMetricsFor(observability).start(turnId);
|
|
6020
|
-
isCodexTurn = await isCodexBilledTurn2({
|
|
6021
|
-
db,
|
|
6022
|
-
settings,
|
|
6023
|
-
workspaceId: input.workspaceId,
|
|
6024
|
-
model: turn.model,
|
|
6025
|
-
active: codexSubscriptionActive
|
|
6026
|
-
});
|
|
6027
6359
|
await waitForTurnOperation(
|
|
6028
6360
|
ensureRunAllowed(
|
|
6029
6361
|
settings,
|
|
6030
6362
|
db,
|
|
6031
6363
|
input.accountId,
|
|
6032
6364
|
input.workspaceId,
|
|
6033
|
-
|
|
6365
|
+
isExternallyBilledTurn,
|
|
6034
6366
|
entitlements
|
|
6035
6367
|
),
|
|
6036
6368
|
cancellationSignal,
|
|
@@ -6055,6 +6387,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
6055
6387
|
const producerId = `${input.workflowId}:${turnId}${activityContext ? `:${activityContext.info.activityId}` : ""}`;
|
|
6056
6388
|
codexLeaseHolderId = dispatchId;
|
|
6057
6389
|
const modelUsageDispatchId = activityContext?.info.activityId ?? dispatchId;
|
|
6390
|
+
const claimedModelUsageSourceKeys = /* @__PURE__ */ new Set();
|
|
6058
6391
|
const emittedModelUsageSourceKeys = /* @__PURE__ */ new Set();
|
|
6059
6392
|
publish = async (events, immediate = false) => {
|
|
6060
6393
|
const inputs = events.map((event) => ({
|
|
@@ -6685,27 +7018,31 @@ function createRunAgentTurnActivity(services) {
|
|
|
6685
7018
|
db,
|
|
6686
7019
|
input.workspaceId,
|
|
6687
7020
|
input.sessionId,
|
|
7021
|
+
input.attemptId,
|
|
6688
7022
|
baseRunSettings
|
|
6689
7023
|
);
|
|
6690
|
-
const resolvedModel = runtime.resolveTurnModel(
|
|
7024
|
+
const resolvedModel = runtime.resolveTurnModel(
|
|
7025
|
+
capabilitySettings,
|
|
7026
|
+
turnExecutionPolicy.productModelId
|
|
7027
|
+
);
|
|
6691
7028
|
modelRunSettings = resolvedModel ? settingsWithResolvedModelContext(runSettings, resolvedModel.configured) : runSettings;
|
|
6692
7029
|
{
|
|
6693
7030
|
const workspaceModelPolicy = await getWorkspaceModelPolicy2(db, input.workspaceId);
|
|
6694
7031
|
if (workspaceModelPolicy) {
|
|
6695
|
-
const effectiveProviderId = resolvedModel ? resolvedModel.provider.id : builtinProviderId(capabilitySettings);
|
|
6696
7032
|
const verdict = evaluateWorkspaceModelPolicy2(workspaceModelPolicy, {
|
|
6697
|
-
providerId:
|
|
6698
|
-
modelId:
|
|
7033
|
+
providerId: turnExecutionPolicy.providerId,
|
|
7034
|
+
modelId: turnExecutionPolicy.productModelId
|
|
6699
7035
|
});
|
|
6700
7036
|
if (!verdict.allowed) {
|
|
6701
7037
|
throw new WorkspaceModelPolicyBlockedError(
|
|
6702
|
-
|
|
6703
|
-
|
|
7038
|
+
turnExecutionPolicy.productModelId,
|
|
7039
|
+
turnExecutionPolicy.providerId,
|
|
6704
7040
|
verdict.reason
|
|
6705
7041
|
);
|
|
6706
7042
|
}
|
|
6707
7043
|
}
|
|
6708
7044
|
}
|
|
7045
|
+
let codexModelRequestSequence = 0;
|
|
6709
7046
|
const codexContext = resolvedModel?.provider.kind === "codex-subscription" ? (() => {
|
|
6710
7047
|
const resolver = buildCodexTokenResolver(
|
|
6711
7048
|
db,
|
|
@@ -6730,72 +7067,72 @@ function createRunAgentTurnActivity(services) {
|
|
|
6730
7067
|
),
|
|
6731
7068
|
onUsageHeaders: (snapshot) => {
|
|
6732
7069
|
latestCodexUsage = snapshot;
|
|
6733
|
-
}
|
|
7070
|
+
},
|
|
6734
7071
|
// latest wins; flushed once in finally
|
|
7072
|
+
nextRequestId: () => `${dispatchId}:${++codexModelRequestSequence}`,
|
|
7073
|
+
onModelRequestEvent: async (event) => {
|
|
7074
|
+
if (!publish || !turnId) {
|
|
7075
|
+
throw new Error("Codex model request started before the turn event producer");
|
|
7076
|
+
}
|
|
7077
|
+
await publish([
|
|
7078
|
+
{
|
|
7079
|
+
type: "agent.model.request",
|
|
7080
|
+
payload: {
|
|
7081
|
+
...event,
|
|
7082
|
+
provider: "codex-subscription",
|
|
7083
|
+
turnId,
|
|
7084
|
+
attemptId: input.attemptId,
|
|
7085
|
+
dispatchId,
|
|
7086
|
+
executionGeneration
|
|
7087
|
+
}
|
|
7088
|
+
}
|
|
7089
|
+
]);
|
|
7090
|
+
}
|
|
6735
7091
|
};
|
|
6736
7092
|
})() : null;
|
|
6737
7093
|
const withCodex = (fn) => codexContext ? codexRequestStorage.run(codexContext, fn) : fn();
|
|
6738
7094
|
const promptCacheKey = acceptsPromptCacheKeyForTurn(resolvedModel) ? input.sessionId : void 0;
|
|
6739
|
-
|
|
7095
|
+
const compactionUsageState = createCompactionModelUsageEventState(
|
|
7096
|
+
claimedModelUsageSourceKeys
|
|
7097
|
+
);
|
|
6740
7098
|
const recordCompactionUsage = async (usage) => {
|
|
6741
|
-
await
|
|
7099
|
+
await processCompactionModelUsageEvent({
|
|
7100
|
+
usage,
|
|
7101
|
+
state: compactionUsageState,
|
|
7102
|
+
dispatchId: modelUsageDispatchId,
|
|
7103
|
+
settings,
|
|
7104
|
+
db,
|
|
7105
|
+
observability,
|
|
7106
|
+
publish,
|
|
7107
|
+
accountId: input.accountId,
|
|
7108
|
+
workspaceId: input.workspaceId,
|
|
7109
|
+
sessionId: input.sessionId,
|
|
7110
|
+
turnId: turn.id,
|
|
7111
|
+
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
7112
|
+
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
7113
|
+
model: resolvedModel?.configured.id ?? turn.model,
|
|
7114
|
+
externallyBilled: isExternallyBilledTurn,
|
|
7115
|
+
turnAttemptId: input.attemptId,
|
|
7116
|
+
servingCredentialId: effectiveCodexCredentialId,
|
|
7117
|
+
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
7118
|
+
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
6742
7119
|
renewLease: () => renewCodexLease("model_usage"),
|
|
6743
7120
|
leaseLost: () => codexLeaseLost,
|
|
6744
|
-
leaseLostMessage: "Codex credential lease expired during context compaction"
|
|
6745
|
-
recordUsage: async () => {
|
|
6746
|
-
compactionUsageCount += 1;
|
|
6747
|
-
const sourceKey = modelUsageSourceKey({
|
|
6748
|
-
responseId: usage.responseId,
|
|
6749
|
-
dispatchId: modelUsageDispatchId,
|
|
6750
|
-
positionalKey: `compaction-${compactionUsageCount}`
|
|
6751
|
-
});
|
|
6752
|
-
const responseAccountCtx = modelCallAccountContext({
|
|
6753
|
-
servingCredentialId: effectiveCodexCredentialId,
|
|
6754
|
-
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
6755
|
-
isFirstCallOfTurn: compactionUsageCount === 1
|
|
6756
|
-
});
|
|
6757
|
-
await recordModelUsageAndDebitCredits(settings, db, {
|
|
6758
|
-
accountId: input.accountId,
|
|
6759
|
-
workspaceId: input.workspaceId,
|
|
6760
|
-
sessionId: input.sessionId,
|
|
6761
|
-
turnId: turn.id,
|
|
6762
|
-
turnAttemptId: input.attemptId,
|
|
6763
|
-
model: resolvedModel?.configured.id ?? turn.model,
|
|
6764
|
-
isCodexTurn,
|
|
6765
|
-
usage: usage.usage,
|
|
6766
|
-
sourceKey,
|
|
6767
|
-
observability
|
|
6768
|
-
});
|
|
6769
|
-
await emitModelCallUsage({
|
|
6770
|
-
observability,
|
|
6771
|
-
publish,
|
|
6772
|
-
accountId: input.accountId,
|
|
6773
|
-
workspaceId: input.workspaceId,
|
|
6774
|
-
sessionId: input.sessionId,
|
|
6775
|
-
turnId: turn.id,
|
|
6776
|
-
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
6777
|
-
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
6778
|
-
model: resolvedModel?.configured.id ?? turn.model,
|
|
6779
|
-
sourceKey,
|
|
6780
|
-
usage,
|
|
6781
|
-
servingAccountHash: responseAccountCtx.servingAccountHash,
|
|
6782
|
-
accountChangedFromPrevCall: responseAccountCtx.accountChangedFromPrevCall,
|
|
6783
|
-
emittedSourceKeys: emittedModelUsageSourceKeys
|
|
6784
|
-
});
|
|
6785
|
-
}
|
|
7121
|
+
leaseLostMessage: "Codex credential lease expired during context compaction"
|
|
6786
7122
|
});
|
|
6787
7123
|
};
|
|
6788
7124
|
const compactionSummarizerFor = (systemInstructions) => resolvedModel ? (s, m) => withCodex(
|
|
6789
7125
|
() => summarizeForCompaction2(s, m, {
|
|
6790
7126
|
client: resolvedModel.client,
|
|
6791
7127
|
api: resolvedModel.provider.api,
|
|
6792
|
-
model:
|
|
7128
|
+
model: turnExecutionPolicy.upstreamModelId,
|
|
6793
7129
|
maxOutputTokens: SUMMARY_BUFFER_TOKENS2,
|
|
6794
7130
|
onUsage: recordCompactionUsage,
|
|
6795
7131
|
...systemInstructions ? { systemInstructions } : {},
|
|
6796
7132
|
...promptCacheKey ? { promptCacheKey } : {}
|
|
6797
7133
|
})
|
|
6798
7134
|
) : (s, m) => summarizeForCompaction2(s, m, {
|
|
7135
|
+
model: turnExecutionPolicy.upstreamModelId,
|
|
6799
7136
|
maxOutputTokens: SUMMARY_BUFFER_TOKENS2,
|
|
6800
7137
|
onUsage: recordCompactionUsage,
|
|
6801
7138
|
...systemInstructions ? { systemInstructions } : {},
|
|
@@ -6917,9 +7254,19 @@ function createRunAgentTurnActivity(services) {
|
|
|
6917
7254
|
return claimedResult({ status: "idle" });
|
|
6918
7255
|
}
|
|
6919
7256
|
const turnResources = mergeResourceRefs(session.resources, turn.resources);
|
|
7257
|
+
const effectivePolicyTools = resolveSessionToolPolicy({
|
|
7258
|
+
...session.toolPolicy ? { toolPolicy: session.toolPolicy } : {},
|
|
7259
|
+
sessionTools: session.tools,
|
|
7260
|
+
turnTools: turn.tools,
|
|
7261
|
+
...turn.toolsProvided !== void 0 ? { turnToolsProvided: turn.toolsProvided } : {},
|
|
7262
|
+
availableMcpServerIds: runSettings.mcpServers.map((server) => server.id),
|
|
7263
|
+
defaultMcpServerIds: enabledCapabilityMcpToolRefs(settings, mcpSettings).map(
|
|
7264
|
+
(tool) => tool.id
|
|
7265
|
+
)
|
|
7266
|
+
}).toolRefs;
|
|
6920
7267
|
const turnTools = withCodexAppsTool(
|
|
6921
7268
|
runSettings,
|
|
6922
|
-
withFirstPartyTools(runSettings,
|
|
7269
|
+
withFirstPartyTools(runSettings, effectivePolicyTools)
|
|
6923
7270
|
);
|
|
6924
7271
|
const connectionScope = {
|
|
6925
7272
|
accountId: input.accountId,
|
|
@@ -7053,6 +7400,9 @@ function createRunAgentTurnActivity(services) {
|
|
|
7053
7400
|
attemptId: input.attemptId,
|
|
7054
7401
|
turn
|
|
7055
7402
|
}) : void 0;
|
|
7403
|
+
const toolspaceAuthority = {
|
|
7404
|
+
sessionId: input.sessionId
|
|
7405
|
+
};
|
|
7056
7406
|
const {
|
|
7057
7407
|
environment: sandboxEnvironment,
|
|
7058
7408
|
gitToken: sandboxGitToken,
|
|
@@ -7075,8 +7425,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
7075
7425
|
...gitCredentialAuthority ? { authority: gitCredentialAuthority } : {},
|
|
7076
7426
|
gitCredentials: connectionCredentials?.gitCredentials,
|
|
7077
7427
|
authorizeGitHubTokenMint,
|
|
7078
|
-
|
|
7079
|
-
runId: turnId
|
|
7428
|
+
toolspaceAuthority
|
|
7080
7429
|
}
|
|
7081
7430
|
),
|
|
7082
7431
|
cancellationSignal,
|
|
@@ -7112,6 +7461,9 @@ function createRunAgentTurnActivity(services) {
|
|
|
7112
7461
|
authorizeGitHubTokenMint
|
|
7113
7462
|
}
|
|
7114
7463
|
);
|
|
7464
|
+
if (binding) {
|
|
7465
|
+
assertGitCredentialRenewalTransportUnchanged(initialBinding, binding);
|
|
7466
|
+
}
|
|
7115
7467
|
pendingBinding = binding;
|
|
7116
7468
|
return binding ? {
|
|
7117
7469
|
bindings: [binding],
|
|
@@ -7172,7 +7524,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
7172
7524
|
toolspaceTokenRenewal = null;
|
|
7173
7525
|
await previous?.stop();
|
|
7174
7526
|
if (toolspaceTokenRenewalClosed) return;
|
|
7175
|
-
const mint = async () => await mintSandboxToolspaceToken(runSettings, connectionScope,
|
|
7527
|
+
const mint = async () => await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority);
|
|
7176
7528
|
const write = async (material) => {
|
|
7177
7529
|
const runAs = sandboxRunAs(runSettings);
|
|
7178
7530
|
await refreshToolspaceTokenFile(tokenSession, material.token, {
|
|
@@ -7544,9 +7896,10 @@ function createRunAgentTurnActivity(services) {
|
|
|
7544
7896
|
// token skip does NOT transfer: that token is inert on a connected
|
|
7545
7897
|
// machine (it uses its own git creds), but the toolspace token is the
|
|
7546
7898
|
// machine's ONLY path to programmatic tool calling and grants no more
|
|
7547
|
-
// than toolspace:call for its own session (own-session-bound,
|
|
7548
|
-
// budgeted, approval-tools
|
|
7549
|
-
//
|
|
7899
|
+
// than toolspace:call for its own session (own-session-bound,
|
|
7900
|
+
// short-lived and renewable, per-active-turn budgeted, approval-tools
|
|
7901
|
+
// excluded). The runtime seeds it to the box's token file over the same
|
|
7902
|
+
// exec channel, off-manifest, on every backend.
|
|
7550
7903
|
...sandboxToolspaceToken ? {
|
|
7551
7904
|
toolspaceTokenSeed: sandboxToolspaceToken,
|
|
7552
7905
|
toolspaceTokenSessionId: input.sessionId
|
|
@@ -7669,12 +8022,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
7669
8022
|
authorizeGitHubTokenMint
|
|
7670
8023
|
});
|
|
7671
8024
|
const lazyGitTokens = lazyGitCredentials?.gitTokens;
|
|
7672
|
-
const lazyToolspaceToken = sandboxToolspaceToken ? await mintSandboxToolspaceToken(
|
|
7673
|
-
runSettings,
|
|
7674
|
-
connectionScope,
|
|
7675
|
-
input.sessionId,
|
|
7676
|
-
turn.id
|
|
7677
|
-
) : void 0;
|
|
8025
|
+
const lazyToolspaceToken = sandboxToolspaceToken ? await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority) : void 0;
|
|
7678
8026
|
const provisioned = await resumeBoxForTurn(
|
|
7679
8027
|
{
|
|
7680
8028
|
db,
|
|
@@ -8013,17 +8361,17 @@ function createRunAgentTurnActivity(services) {
|
|
|
8013
8361
|
}
|
|
8014
8362
|
return outcome;
|
|
8015
8363
|
};
|
|
8364
|
+
const modelResponseUsageState = createModelResponseUsageEventState(
|
|
8365
|
+
claimedModelUsageSourceKeys
|
|
8366
|
+
);
|
|
8016
8367
|
const runStreamAttempt = async () => {
|
|
8017
8368
|
if (!runInput) {
|
|
8018
8369
|
throw new Error("Run input was not prepared");
|
|
8019
8370
|
}
|
|
8020
8371
|
stream = void 0;
|
|
8021
8372
|
batcher = null;
|
|
8022
|
-
let responseUsageCount = 0;
|
|
8023
8373
|
let currentToolBatchCallIds = /* @__PURE__ */ new Set();
|
|
8024
8374
|
let currentToolBatchCompletedCallIds = /* @__PURE__ */ new Set();
|
|
8025
|
-
let lastProviderContextTokensObserved = null;
|
|
8026
|
-
let providerContextRevision = 0;
|
|
8027
8375
|
throwIfWorkerShuttingDown();
|
|
8028
8376
|
throwIfTurnCancelled();
|
|
8029
8377
|
const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
|
|
@@ -8074,10 +8422,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
8074
8422
|
}
|
|
8075
8423
|
} : {}
|
|
8076
8424
|
} : {},
|
|
8077
|
-
contextCompactionSignal: () =>
|
|
8078
|
-
revision: providerContextRevision,
|
|
8079
|
-
totalTokens: lastProviderContextTokensObserved
|
|
8080
|
-
},
|
|
8425
|
+
contextCompactionSignal: () => modelResponseUsageContextSignal(modelResponseUsageState),
|
|
8081
8426
|
contextCompactionRequested: () => isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
8082
8427
|
...toolCancellationFenceRef.current ? { turnToolCancellationFence: toolCancellationFenceRef.current } : {}
|
|
8083
8428
|
});
|
|
@@ -8108,73 +8453,33 @@ function createRunAgentTurnActivity(services) {
|
|
|
8108
8453
|
}
|
|
8109
8454
|
let stableToolCallIdsToClear = null;
|
|
8110
8455
|
let completedCurrentToolBatch = false;
|
|
8111
|
-
const
|
|
8112
|
-
|
|
8113
|
-
|
|
8114
|
-
|
|
8115
|
-
|
|
8116
|
-
|
|
8117
|
-
|
|
8118
|
-
|
|
8119
|
-
|
|
8120
|
-
|
|
8121
|
-
|
|
8122
|
-
|
|
8123
|
-
|
|
8124
|
-
|
|
8125
|
-
|
|
8126
|
-
|
|
8127
|
-
|
|
8128
|
-
|
|
8129
|
-
|
|
8130
|
-
|
|
8131
|
-
|
|
8132
|
-
|
|
8133
|
-
|
|
8134
|
-
|
|
8135
|
-
|
|
8136
|
-
|
|
8137
|
-
|
|
8138
|
-
sourceKey: responseSourceKey,
|
|
8139
|
-
observability
|
|
8140
|
-
});
|
|
8141
|
-
await emitModelCallUsage({
|
|
8142
|
-
observability,
|
|
8143
|
-
publish,
|
|
8144
|
-
accountId: input.accountId,
|
|
8145
|
-
workspaceId: input.workspaceId,
|
|
8146
|
-
sessionId: input.sessionId,
|
|
8147
|
-
turnId: activeTurnId,
|
|
8148
|
-
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
8149
|
-
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
8150
|
-
model: turn.model,
|
|
8151
|
-
sourceKey: responseSourceKey,
|
|
8152
|
-
usage: responseUsage,
|
|
8153
|
-
servingAccountHash: responseAccountCtx.servingAccountHash,
|
|
8154
|
-
accountChangedFromPrevCall: responseAccountCtx.accountChangedFromPrevCall,
|
|
8155
|
-
emittedSourceKeys: emittedModelUsageSourceKeys
|
|
8156
|
-
});
|
|
8157
|
-
const observed = responseUsage.usage?.inputTokens;
|
|
8158
|
-
if (typeof observed === "number" && observed > 0) {
|
|
8159
|
-
recordModelInputTokens(observability, streamProvider, observed);
|
|
8160
|
-
}
|
|
8161
|
-
const observedTotal = providerContextTokens(responseUsage.usage);
|
|
8162
|
-
if (observedTotal !== null) {
|
|
8163
|
-
lastProviderContextTokensObserved = observedTotal;
|
|
8164
|
-
providerContextRevision += 1;
|
|
8165
|
-
}
|
|
8166
|
-
recordModelCacheTokens(observability, streamProvider, {
|
|
8167
|
-
cachedTokens: modelCallUsageTelemetry(responseUsage.usage).cachedTokens,
|
|
8168
|
-
promptTokens: responseUsage.usage?.inputTokens
|
|
8169
|
-
});
|
|
8170
|
-
},
|
|
8171
|
-
recordAttemptSignals: async () => {
|
|
8172
|
-
const observed = responseUsage.usage?.inputTokens;
|
|
8173
|
-
if (typeof observed === "number" && observed > 0) {
|
|
8174
|
-
await setLastInputTokensFenced(observed);
|
|
8175
|
-
}
|
|
8176
|
-
}
|
|
8177
|
-
});
|
|
8456
|
+
const responseUsageResult = await processModelResponseUsageEvent({
|
|
8457
|
+
event: next.value,
|
|
8458
|
+
state: modelResponseUsageState,
|
|
8459
|
+
dispatchId: modelUsageDispatchId,
|
|
8460
|
+
settings,
|
|
8461
|
+
db,
|
|
8462
|
+
observability,
|
|
8463
|
+
publish,
|
|
8464
|
+
accountId: input.accountId,
|
|
8465
|
+
workspaceId: input.workspaceId,
|
|
8466
|
+
sessionId: input.sessionId,
|
|
8467
|
+
turnId: activeTurnId,
|
|
8468
|
+
turnAttemptId: input.attemptId,
|
|
8469
|
+
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
8470
|
+
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
8471
|
+
model: turn.model,
|
|
8472
|
+
metricProvider: streamProvider,
|
|
8473
|
+
externallyBilled: isExternallyBilledTurn,
|
|
8474
|
+
servingCredentialId: effectiveCodexCredentialId,
|
|
8475
|
+
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
8476
|
+
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
8477
|
+
renewLease: () => renewCodexLease("model_usage"),
|
|
8478
|
+
leaseLost: () => codexLeaseLost,
|
|
8479
|
+
leaseLostMessage: "Codex credential lease expired during the active turn",
|
|
8480
|
+
setLastInputTokens: setLastInputTokensFenced
|
|
8481
|
+
});
|
|
8482
|
+
if (responseUsageResult.status === "processed") {
|
|
8178
8483
|
currentToolBatchCallIds = /* @__PURE__ */ new Set();
|
|
8179
8484
|
currentToolBatchCompletedCallIds = /* @__PURE__ */ new Set();
|
|
8180
8485
|
await reconcileConversationTruth();
|
|
@@ -8184,7 +8489,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
8184
8489
|
db,
|
|
8185
8490
|
input.accountId,
|
|
8186
8491
|
input.workspaceId,
|
|
8187
|
-
|
|
8492
|
+
isExternallyBilledTurn,
|
|
8188
8493
|
entitlements
|
|
8189
8494
|
);
|
|
8190
8495
|
} catch (limitError) {
|
|
@@ -8283,74 +8588,74 @@ function createRunAgentTurnActivity(services) {
|
|
|
8283
8588
|
stream.completed.catch(() => void 0),
|
|
8284
8589
|
cancellationSignal
|
|
8285
8590
|
);
|
|
8286
|
-
if (responseUsageCount === 0) {
|
|
8591
|
+
if (modelResponseUsageState.responseUsageCount === 0) {
|
|
8287
8592
|
const aggregateUsage = stream.state.usage;
|
|
8288
|
-
const
|
|
8289
|
-
|
|
8290
|
-
const aggregateContext = providerContextTokens(
|
|
8291
|
-
aggregateUsage
|
|
8292
|
-
);
|
|
8293
|
-
if (aggregateContext !== null) {
|
|
8294
|
-
lastProviderContextTokensObserved = aggregateContext;
|
|
8295
|
-
providerContextRevision += 1;
|
|
8296
|
-
}
|
|
8297
|
-
}
|
|
8593
|
+
const normalizedAggregateUsage = normalizeModelCallUsage(aggregateUsage);
|
|
8594
|
+
const aggregateInput = normalizedAggregateUsage.telemetry.inputTokens;
|
|
8298
8595
|
const aggregateSourceKey = modelUsageSourceKey({
|
|
8299
8596
|
responseId: null,
|
|
8300
8597
|
dispatchId: modelUsageDispatchId,
|
|
8301
8598
|
positionalKey: "aggregate"
|
|
8302
8599
|
});
|
|
8303
|
-
|
|
8304
|
-
|
|
8305
|
-
|
|
8306
|
-
|
|
8307
|
-
|
|
8308
|
-
|
|
8309
|
-
|
|
8310
|
-
|
|
8311
|
-
|
|
8312
|
-
|
|
8313
|
-
|
|
8314
|
-
|
|
8315
|
-
|
|
8316
|
-
|
|
8317
|
-
|
|
8318
|
-
|
|
8319
|
-
|
|
8320
|
-
|
|
8321
|
-
|
|
8322
|
-
|
|
8323
|
-
|
|
8324
|
-
|
|
8325
|
-
|
|
8326
|
-
|
|
8327
|
-
|
|
8328
|
-
|
|
8329
|
-
|
|
8330
|
-
|
|
8331
|
-
|
|
8332
|
-
|
|
8333
|
-
|
|
8334
|
-
|
|
8335
|
-
|
|
8336
|
-
|
|
8337
|
-
|
|
8338
|
-
|
|
8339
|
-
|
|
8340
|
-
|
|
8341
|
-
|
|
8342
|
-
|
|
8343
|
-
|
|
8344
|
-
|
|
8345
|
-
|
|
8346
|
-
|
|
8347
|
-
|
|
8348
|
-
|
|
8349
|
-
|
|
8350
|
-
|
|
8600
|
+
if (!claimedModelUsageSourceKeys.has(aggregateSourceKey)) {
|
|
8601
|
+
claimedModelUsageSourceKeys.add(aggregateSourceKey);
|
|
8602
|
+
const aggregateAccountCtx = modelCallAccountContext({
|
|
8603
|
+
servingCredentialId: effectiveCodexCredentialId,
|
|
8604
|
+
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
8605
|
+
isFirstCallOfTurn: true
|
|
8606
|
+
});
|
|
8607
|
+
let aggregateAuthoritative = false;
|
|
8608
|
+
await recordCompletedModelCallBeforeOwnershipFences({
|
|
8609
|
+
renewLease: () => renewCodexLease("model_usage"),
|
|
8610
|
+
leaseLost: () => codexLeaseLost,
|
|
8611
|
+
leaseLostMessage: "Codex credential lease expired during the active turn",
|
|
8612
|
+
recordUsage: async () => {
|
|
8613
|
+
await recordModelUsageAndDebitCredits(settings, db, {
|
|
8614
|
+
accountId: input.accountId,
|
|
8615
|
+
workspaceId: input.workspaceId,
|
|
8616
|
+
sessionId: input.sessionId,
|
|
8617
|
+
turnId: activeTurnId,
|
|
8618
|
+
turnAttemptId: input.attemptId,
|
|
8619
|
+
model: turn.model,
|
|
8620
|
+
externallyBilled: isExternallyBilledTurn,
|
|
8621
|
+
usage: aggregateUsage,
|
|
8622
|
+
normalizedUsage: normalizedAggregateUsage,
|
|
8623
|
+
sourceKey: aggregateSourceKey,
|
|
8624
|
+
observability
|
|
8625
|
+
});
|
|
8626
|
+
aggregateAuthoritative = await emitModelCallUsage({
|
|
8627
|
+
observability,
|
|
8628
|
+
publish,
|
|
8629
|
+
accountId: input.accountId,
|
|
8630
|
+
workspaceId: input.workspaceId,
|
|
8631
|
+
sessionId: input.sessionId,
|
|
8632
|
+
turnId: activeTurnId,
|
|
8633
|
+
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
8634
|
+
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
8635
|
+
model: turn.model,
|
|
8636
|
+
sourceKey: aggregateSourceKey,
|
|
8637
|
+
usage: { usage: aggregateUsage },
|
|
8638
|
+
normalizedUsage: normalizedAggregateUsage,
|
|
8639
|
+
servingAccountHash: aggregateAccountCtx.servingAccountHash,
|
|
8640
|
+
accountChangedFromPrevCall: aggregateAccountCtx.accountChangedFromPrevCall,
|
|
8641
|
+
emittedSourceKeys: emittedModelUsageSourceKeys
|
|
8642
|
+
});
|
|
8643
|
+
if (aggregateAuthoritative && aggregateInput !== null && aggregateInput > 0) {
|
|
8644
|
+
recordModelInputTokens(observability, streamProvider, aggregateInput);
|
|
8645
|
+
}
|
|
8646
|
+
},
|
|
8647
|
+
recordAttemptSignals: async () => {
|
|
8648
|
+
if (!aggregateAuthoritative) return;
|
|
8649
|
+
if (normalizedAggregateUsage.totalTokens !== null) {
|
|
8650
|
+
modelResponseUsageState.lastProviderContextTokensObserved = normalizedAggregateUsage.totalTokens;
|
|
8651
|
+
modelResponseUsageState.providerContextRevision += 1;
|
|
8652
|
+
}
|
|
8653
|
+
if (aggregateInput !== null && aggregateInput > 0) {
|
|
8654
|
+
await setLastInputTokensFenced(aggregateInput);
|
|
8655
|
+
}
|
|
8351
8656
|
}
|
|
8352
|
-
}
|
|
8353
|
-
}
|
|
8657
|
+
});
|
|
8658
|
+
}
|
|
8354
8659
|
}
|
|
8355
8660
|
if (stream.interruptions.length > 0) {
|
|
8356
8661
|
await reconcileConversationTruth();
|
|
@@ -9167,7 +9472,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9167
9472
|
activityStatus = "idle";
|
|
9168
9473
|
return claimedResult({ status: "idle" });
|
|
9169
9474
|
}
|
|
9170
|
-
const failure = agentRunFailurePayload(error);
|
|
9475
|
+
const failure = agentRunFailurePayload(error, { isCodexTurn });
|
|
9171
9476
|
if (isSessionEventPersistenceError(error)) {
|
|
9172
9477
|
observability.error("session event persistence failed", {
|
|
9173
9478
|
accountId: input.accountId,
|
|
@@ -9640,7 +9945,7 @@ function isTransientProviderError(error) {
|
|
|
9640
9945
|
message
|
|
9641
9946
|
);
|
|
9642
9947
|
}
|
|
9643
|
-
function agentRunFailurePayload(error) {
|
|
9948
|
+
function agentRunFailurePayload(error, options = {}) {
|
|
9644
9949
|
const message = error instanceof Error ? error.message : String(error);
|
|
9645
9950
|
const status = typeof error === "object" && error !== null && "status" in error ? Number(error.status) : void 0;
|
|
9646
9951
|
const code = typeof error === "object" && error !== null && "code" in error ? String(error.code) : void 0;
|
|
@@ -9671,6 +9976,20 @@ function agentRunFailurePayload(error) {
|
|
|
9671
9976
|
if (usageLimit) {
|
|
9672
9977
|
return codexUsageLimitFailurePayload(usageLimit, message);
|
|
9673
9978
|
}
|
|
9979
|
+
const codexTimeout = classifyCodexResponseTimeoutError(error, {
|
|
9980
|
+
allowLegacyRequestTimeout: options.isCodexTurn === true
|
|
9981
|
+
});
|
|
9982
|
+
if (codexTimeout) {
|
|
9983
|
+
return {
|
|
9984
|
+
error: codexTimeout.responseObserved ? "The Codex response timed out after streaming began. Observed output was checkpointed; automatic replay is disabled because the upstream operation may still be active." : "The Codex response timed out before any response was observed. Upstream acceptance is unknown, so automatic replay is disabled.",
|
|
9985
|
+
code: "codex_response_timeout",
|
|
9986
|
+
retryable: false,
|
|
9987
|
+
timeoutClass: codexTimeout.timeoutClass,
|
|
9988
|
+
responseObserved: codexTimeout.responseObserved,
|
|
9989
|
+
...codexTimeout.requestId ? { requestId: codexTimeout.requestId } : {},
|
|
9990
|
+
...codexTimeout.message ? { detail: codexTimeout.message } : {}
|
|
9991
|
+
};
|
|
9992
|
+
}
|
|
9674
9993
|
const mcpTimeout = classifyMcpTransportTimeoutError(error);
|
|
9675
9994
|
if (mcpTimeout) {
|
|
9676
9995
|
return {
|
|
@@ -9820,8 +10139,8 @@ var BudgetExhaustedError = class extends Error {
|
|
|
9820
10139
|
this.name = "BudgetExhaustedError";
|
|
9821
10140
|
}
|
|
9822
10141
|
};
|
|
9823
|
-
async function ensureRunAllowed(settings, db, accountId, workspaceId,
|
|
9824
|
-
if (!
|
|
10142
|
+
async function ensureRunAllowed(settings, db, accountId, workspaceId, isExternallyBilledTurn, entitlements) {
|
|
10143
|
+
if (!isExternallyBilledTurn && entitlements && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
|
|
9825
10144
|
const decision = await entitlements.admitRun({
|
|
9826
10145
|
accountId,
|
|
9827
10146
|
workspaceId,
|
|
@@ -9831,7 +10150,7 @@ async function ensureRunAllowed(settings, db, accountId, workspaceId, isCodexTur
|
|
|
9831
10150
|
if (!decision.allowed) {
|
|
9832
10151
|
throw new Error(decision.reason || "insufficient OpenGeni credits");
|
|
9833
10152
|
}
|
|
9834
|
-
} else if (!
|
|
10153
|
+
} else if (!isExternallyBilledTurn && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
|
|
9835
10154
|
const balance = await getBillingBalance2(db, accountId);
|
|
9836
10155
|
if (balance.balanceMicros <= 0) {
|
|
9837
10156
|
throw new Error("insufficient OpenGeni credits");
|
|
@@ -9851,7 +10170,7 @@ async function ensureRunAllowed(settings, db, accountId, workspaceId, isCodexTur
|
|
|
9851
10170
|
);
|
|
9852
10171
|
}
|
|
9853
10172
|
}
|
|
9854
|
-
if (!
|
|
10173
|
+
if (!isExternallyBilledTurn && limits.maxMonthlyTokensPerWorkspace) {
|
|
9855
10174
|
const used = await sumUsageQuantity2(db, {
|
|
9856
10175
|
workspaceId,
|
|
9857
10176
|
eventType: "model.tokens",
|
|
@@ -9867,10 +10186,12 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
|
|
|
9867
10186
|
if (!input.usage) {
|
|
9868
10187
|
return;
|
|
9869
10188
|
}
|
|
9870
|
-
const
|
|
9871
|
-
const
|
|
9872
|
-
const
|
|
9873
|
-
|
|
10189
|
+
const normalizedUsage = input.normalizedUsage ?? normalizeModelCallUsage(input.usage);
|
|
10190
|
+
const sanitizedUsage = sanitizedModelUsageInput(normalizedUsage);
|
|
10191
|
+
const inputTokens = sanitizedUsage.inputTokens ?? 0;
|
|
10192
|
+
const outputTokens = sanitizedUsage.outputTokens ?? 0;
|
|
10193
|
+
const totalTokens = sanitizedUsage.totalTokens ?? 0;
|
|
10194
|
+
if (input.externallyBilled) {
|
|
9874
10195
|
await recordUsageEvent2(db, {
|
|
9875
10196
|
accountId: input.accountId,
|
|
9876
10197
|
workspaceId: input.workspaceId,
|
|
@@ -9908,7 +10229,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
|
|
|
9908
10229
|
if (!configuredModelPricing(settings)[input.model]) {
|
|
9909
10230
|
throw new Error(`Missing model pricing for ${input.model}`);
|
|
9910
10231
|
}
|
|
9911
|
-
const costMicros = calculateModelUsageCostMicros(settings, input.model,
|
|
10232
|
+
const costMicros = calculateModelUsageCostMicros(settings, input.model, sanitizedUsage);
|
|
9912
10233
|
await recordUsageEvent2(db, {
|
|
9913
10234
|
accountId: input.accountId,
|
|
9914
10235
|
workspaceId: input.workspaceId,
|
|
@@ -9942,16 +10263,19 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
|
|
|
9942
10263
|
// Additive: the prompt-cache slice of this call's input tokens, so the
|
|
9943
10264
|
// per-call debit record carries cache efficiency alongside the token
|
|
9944
10265
|
// counts. 0 when the provider did not report cached tokens.
|
|
9945
|
-
cachedTokens:
|
|
9946
|
-
modelCallUsageTelemetry(input.usage).cachedTokens
|
|
9947
|
-
)
|
|
10266
|
+
cachedTokens: normalizedUsage.telemetry.cachedTokens ?? 0
|
|
9948
10267
|
}
|
|
9949
10268
|
});
|
|
9950
10269
|
recordCreditMicros(input.observability, "usage", result.debitedMicros);
|
|
9951
10270
|
}
|
|
9952
10271
|
}
|
|
9953
|
-
function
|
|
9954
|
-
return
|
|
10272
|
+
function sanitizedModelUsageInput(normalized) {
|
|
10273
|
+
return {
|
|
10274
|
+
...normalized.telemetry.inputTokens !== null ? { inputTokens: normalized.telemetry.inputTokens } : {},
|
|
10275
|
+
...normalized.telemetry.outputTokens !== null ? { outputTokens: normalized.telemetry.outputTokens } : {},
|
|
10276
|
+
...normalized.totalTokens !== null ? { totalTokens: normalized.totalTokens } : {},
|
|
10277
|
+
...normalized.telemetry.cachedTokens !== null ? { inputTokensDetails: { cached_tokens: normalized.telemetry.cachedTokens } } : {}
|
|
10278
|
+
};
|
|
9955
10279
|
}
|
|
9956
10280
|
function startOfUtcMonth2() {
|
|
9957
10281
|
const now = /* @__PURE__ */ new Date();
|
|
@@ -10367,12 +10691,29 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
|
|
|
10367
10691
|
}
|
|
10368
10692
|
return forceDrained;
|
|
10369
10693
|
}
|
|
10370
|
-
|
|
10371
|
-
if (
|
|
10372
|
-
return
|
|
10694
|
+
function modalOrphanTerminationStillEligible(latest, candidate) {
|
|
10695
|
+
if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {
|
|
10696
|
+
return false;
|
|
10373
10697
|
}
|
|
10374
|
-
|
|
10375
|
-
|
|
10698
|
+
const leaseId = candidate.tags.opengeni_lease_id;
|
|
10699
|
+
const workspaceId = candidate.tags.opengeni_workspace_id;
|
|
10700
|
+
const sandboxGroupId = candidate.tags.opengeni_sandbox_group_id;
|
|
10701
|
+
if (!leaseId || !workspaceId || !sandboxGroupId) {
|
|
10702
|
+
return true;
|
|
10703
|
+
}
|
|
10704
|
+
const activeAttribution = latest.find(
|
|
10705
|
+
(lease) => lease.leaseId === leaseId && lease.workspaceId === workspaceId && lease.sandboxGroupId === sandboxGroupId
|
|
10706
|
+
);
|
|
10707
|
+
return Boolean(
|
|
10708
|
+
!activeAttribution || activeAttribution.instanceId !== null && activeAttribution.instanceId !== candidate.sandboxId
|
|
10709
|
+
);
|
|
10710
|
+
}
|
|
10711
|
+
async function sweepModalOrphansForConfiguredBackend(settings, db, observability) {
|
|
10712
|
+
if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret) {
|
|
10713
|
+
return 0;
|
|
10714
|
+
}
|
|
10715
|
+
let liveLeases;
|
|
10716
|
+
try {
|
|
10376
10717
|
liveLeases = await listLiveModalSandboxLeaseAttributions(db);
|
|
10377
10718
|
} catch (error) {
|
|
10378
10719
|
observability.warn("sandbox reaper: live Modal lease attribution read failed", {
|
|
@@ -10380,7 +10721,12 @@ async function sweepModalOrphansForConfiguredBackend(settings, db, observability
|
|
|
10380
10721
|
});
|
|
10381
10722
|
return 0;
|
|
10382
10723
|
}
|
|
10383
|
-
const result = await sweepModalOrphanSandboxes(settings, liveLeases
|
|
10724
|
+
const result = await sweepModalOrphanSandboxes(settings, liveLeases, {
|
|
10725
|
+
revalidateTermination: async (candidate) => {
|
|
10726
|
+
const latest = await listLiveModalSandboxLeaseAttributions(db);
|
|
10727
|
+
return modalOrphanTerminationStillEligible(latest, candidate);
|
|
10728
|
+
}
|
|
10729
|
+
});
|
|
10384
10730
|
for (const terminated of result.terminated) {
|
|
10385
10731
|
observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
|
|
10386
10732
|
sandboxId: terminated.sandboxId,
|
|
@@ -10630,7 +10976,7 @@ import {
|
|
|
10630
10976
|
getBillingBalance as getBillingBalance5,
|
|
10631
10977
|
getRig,
|
|
10632
10978
|
getVariableSet,
|
|
10633
|
-
isCodexBilledTurn as
|
|
10979
|
+
isCodexBilledTurn as isCodexBilledTurn2,
|
|
10634
10980
|
markScheduledTaskRunFailedIfQueued,
|
|
10635
10981
|
recordUsageEvent as recordUsageEvent4,
|
|
10636
10982
|
requireScheduledTask,
|
|
@@ -10649,7 +10995,7 @@ function createScheduledTaskActivities(services) {
|
|
|
10649
10995
|
const { settings, db, bus, wakeSessionWorkflow } = await services();
|
|
10650
10996
|
const task = await requireScheduledTask(db, input.workspaceId, input.taskId);
|
|
10651
10997
|
const model = task.agentConfig.model ?? settings.openaiModel;
|
|
10652
|
-
const isCodexRun = await
|
|
10998
|
+
const isCodexRun = await isCodexBilledTurn2({
|
|
10653
10999
|
db,
|
|
10654
11000
|
settings,
|
|
10655
11001
|
workspaceId: task.workspaceId,
|
|
@@ -11282,22 +11628,166 @@ import {
|
|
|
11282
11628
|
promoteSetupAppendChange
|
|
11283
11629
|
} from "@opengeni/core";
|
|
11284
11630
|
import {
|
|
11631
|
+
acquireLease as acquireLease2,
|
|
11285
11632
|
beginRigChangeVerificationAttempt,
|
|
11633
|
+
commitWarmingToWarm as commitWarmingToWarm2,
|
|
11634
|
+
failWarmingToCold as failWarmingToCold2,
|
|
11286
11635
|
getRig as getRig2,
|
|
11287
11636
|
getRigChange,
|
|
11288
11637
|
getRigVersionById,
|
|
11638
|
+
markWarmLeaseInstanceLost as markWarmLeaseInstanceLost2,
|
|
11639
|
+
recordWarmingSandboxCreated as recordWarmingSandboxCreated2,
|
|
11640
|
+
releaseLeaseHolder as releaseLeaseHolder2,
|
|
11289
11641
|
sanitizeEventPayload,
|
|
11290
11642
|
sanitizeEventString,
|
|
11291
11643
|
sanitizeMemoryText,
|
|
11644
|
+
touchLeaseHolder as touchLeaseHolder2,
|
|
11292
11645
|
updateRigChangeStatus
|
|
11293
11646
|
} from "@opengeni/db";
|
|
11294
11647
|
import {
|
|
11648
|
+
createTurnToolCancellationController,
|
|
11295
11649
|
establishSandboxSessionFromEnvelope as establishSandboxSessionFromEnvelope2,
|
|
11296
11650
|
runRigSetupHook,
|
|
11297
11651
|
sandboxCommandExitCode,
|
|
11298
|
-
sandboxCommandOutput
|
|
11652
|
+
sandboxCommandOutput,
|
|
11653
|
+
serializeEstablishedSandboxEnvelope as serializeEstablishedSandboxEnvelope2,
|
|
11654
|
+
tagModalSandbox as tagModalSandbox2
|
|
11299
11655
|
} from "@opengeni/runtime";
|
|
11300
11656
|
var OUTPUT_TAIL_LIMIT = 64 * 1024;
|
|
11657
|
+
var RIG_VERIFICATION_CLEANUP_RESERVE_MAX_MS = 2 * 6e4;
|
|
11658
|
+
var RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS = 6e4;
|
|
11659
|
+
var RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS = 1e4;
|
|
11660
|
+
var RIG_VERIFICATION_HEARTBEAT_MIN_INTERVAL_MS = 250;
|
|
11661
|
+
var RIG_VERIFICATION_OWNER_TTL_MS = 20 * 6e4;
|
|
11662
|
+
var RIG_VERIFICATION_OWNERS_DISABLED_MESSAGE = "Rig verification lease ownership is disabled; refusing to create an unowned verifier sandbox";
|
|
11663
|
+
var RIG_VERIFICATION_HOLDER_TOUCH_INTERVAL_MS = 1e4;
|
|
11664
|
+
var RIG_VERIFICATION_RELEASE_GRACE_MS = 1;
|
|
11665
|
+
var RigVerificationActivityDeadlineError = class extends Error {
|
|
11666
|
+
constructor(startToCloseTimeoutMs, cleanupReserveMs) {
|
|
11667
|
+
super(
|
|
11668
|
+
`Rig verification activity-local deadline reached with ${cleanupReserveMs}ms reserved for cleanup before the ${startToCloseTimeoutMs}ms Temporal start-to-close timeout`
|
|
11669
|
+
);
|
|
11670
|
+
this.startToCloseTimeoutMs = startToCloseTimeoutMs;
|
|
11671
|
+
this.cleanupReserveMs = cleanupReserveMs;
|
|
11672
|
+
}
|
|
11673
|
+
name = "RigVerificationActivityDeadlineError";
|
|
11674
|
+
};
|
|
11675
|
+
function computeCleanupReserveMs(startToCloseTimeoutMs) {
|
|
11676
|
+
return Math.min(
|
|
11677
|
+
RIG_VERIFICATION_CLEANUP_RESERVE_MAX_MS,
|
|
11678
|
+
Math.max(100, Math.floor(startToCloseTimeoutMs / 4)),
|
|
11679
|
+
Math.max(1, startToCloseTimeoutMs - 1)
|
|
11680
|
+
);
|
|
11681
|
+
}
|
|
11682
|
+
function createRigVerificationActivityLifecycle(context = currentActivityContext()) {
|
|
11683
|
+
const controller = new AbortController();
|
|
11684
|
+
const temporalSignal = context?.cancellationSignal;
|
|
11685
|
+
const forwardTemporalCancellation = () => {
|
|
11686
|
+
if (!controller.signal.aborted) {
|
|
11687
|
+
controller.abort(temporalSignal?.reason ?? new Error("Rig verification cancelled"));
|
|
11688
|
+
}
|
|
11689
|
+
};
|
|
11690
|
+
if (temporalSignal?.aborted) {
|
|
11691
|
+
forwardTemporalCancellation();
|
|
11692
|
+
} else {
|
|
11693
|
+
temporalSignal?.addEventListener("abort", forwardTemporalCancellation, { once: true });
|
|
11694
|
+
}
|
|
11695
|
+
const startedAtMs = Date.now();
|
|
11696
|
+
const startToCloseTimeoutMs = context?.info.startToCloseTimeoutMs;
|
|
11697
|
+
const reserveMs = typeof startToCloseTimeoutMs === "number" && startToCloseTimeoutMs > 0 ? computeCleanupReserveMs(startToCloseTimeoutMs) : null;
|
|
11698
|
+
const serverDeadlineAtMs = reserveMs === null || startToCloseTimeoutMs === void 0 ? null : startedAtMs + startToCloseTimeoutMs;
|
|
11699
|
+
const serverReportingMarginMs = reserveMs === null ? 0 : Math.min(3e4, Math.max(1, Math.floor(reserveMs / 4)));
|
|
11700
|
+
const cleanupDeadlineAtMs = serverDeadlineAtMs === null ? null : serverDeadlineAtMs - serverReportingMarginMs;
|
|
11701
|
+
let deadlineTimer = null;
|
|
11702
|
+
if (reserveMs !== null && startToCloseTimeoutMs !== void 0 && !controller.signal.aborted) {
|
|
11703
|
+
deadlineTimer = setTimeout(
|
|
11704
|
+
() => {
|
|
11705
|
+
if (!controller.signal.aborted) {
|
|
11706
|
+
controller.abort(
|
|
11707
|
+
new RigVerificationActivityDeadlineError(startToCloseTimeoutMs, reserveMs)
|
|
11708
|
+
);
|
|
11709
|
+
}
|
|
11710
|
+
},
|
|
11711
|
+
Math.max(1, startToCloseTimeoutMs - reserveMs)
|
|
11712
|
+
);
|
|
11713
|
+
}
|
|
11714
|
+
const heartbeat = () => {
|
|
11715
|
+
if (!context) return;
|
|
11716
|
+
try {
|
|
11717
|
+
context.heartbeat({
|
|
11718
|
+
activity: "rig_verification",
|
|
11719
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
11720
|
+
cleanup: controller.signal.aborted,
|
|
11721
|
+
cleanupDeadlineAt: cleanupDeadlineAtMs ? new Date(cleanupDeadlineAtMs).toISOString() : null
|
|
11722
|
+
});
|
|
11723
|
+
} catch (error) {
|
|
11724
|
+
if (!controller.signal.aborted) controller.abort(error);
|
|
11725
|
+
}
|
|
11726
|
+
};
|
|
11727
|
+
heartbeat();
|
|
11728
|
+
const heartbeatTimeoutMs = context?.info.heartbeatTimeoutMs;
|
|
11729
|
+
const heartbeatIntervalMs = typeof heartbeatTimeoutMs === "number" && heartbeatTimeoutMs > 0 ? Math.min(
|
|
11730
|
+
RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS,
|
|
11731
|
+
Math.max(RIG_VERIFICATION_HEARTBEAT_MIN_INTERVAL_MS, Math.floor(heartbeatTimeoutMs / 3))
|
|
11732
|
+
) : RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS;
|
|
11733
|
+
const heartbeatTimer = context ? setInterval(heartbeat, heartbeatIntervalMs) : null;
|
|
11734
|
+
if (heartbeatTimer && "unref" in heartbeatTimer && typeof heartbeatTimer.unref === "function") {
|
|
11735
|
+
heartbeatTimer.unref();
|
|
11736
|
+
}
|
|
11737
|
+
return {
|
|
11738
|
+
signal: controller.signal,
|
|
11739
|
+
cleanupDeadlineAtMs,
|
|
11740
|
+
dispose: () => {
|
|
11741
|
+
temporalSignal?.removeEventListener("abort", forwardTemporalCancellation);
|
|
11742
|
+
if (deadlineTimer) clearTimeout(deadlineTimer);
|
|
11743
|
+
if (heartbeatTimer) clearInterval(heartbeatTimer);
|
|
11744
|
+
deadlineTimer = null;
|
|
11745
|
+
}
|
|
11746
|
+
};
|
|
11747
|
+
}
|
|
11748
|
+
function abortReason(signal) {
|
|
11749
|
+
return signal.reason ?? new Error("Rig verification cancelled");
|
|
11750
|
+
}
|
|
11751
|
+
function throwIfAborted(signal) {
|
|
11752
|
+
if (signal.aborted) throw abortReason(signal);
|
|
11753
|
+
}
|
|
11754
|
+
async function waitForAbortable(promise, signal) {
|
|
11755
|
+
throwIfAborted(signal);
|
|
11756
|
+
let removeAbortListener = () => void 0;
|
|
11757
|
+
const cancellation = new Promise((_resolve, reject) => {
|
|
11758
|
+
const onAbort = () => reject(abortReason(signal));
|
|
11759
|
+
removeAbortListener = () => signal.removeEventListener("abort", onAbort);
|
|
11760
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
11761
|
+
});
|
|
11762
|
+
try {
|
|
11763
|
+
return await Promise.race([promise, cancellation]);
|
|
11764
|
+
} finally {
|
|
11765
|
+
removeAbortListener();
|
|
11766
|
+
}
|
|
11767
|
+
}
|
|
11768
|
+
async function settleWithin(promise, timeoutMs) {
|
|
11769
|
+
if (timeoutMs <= 0) return { status: "timed_out" };
|
|
11770
|
+
let timer = null;
|
|
11771
|
+
try {
|
|
11772
|
+
return await Promise.race([
|
|
11773
|
+
promise.then((value) => ({ status: "completed", value })),
|
|
11774
|
+
new Promise((resolve) => {
|
|
11775
|
+
timer = setTimeout(() => resolve({ status: "timed_out" }), timeoutMs);
|
|
11776
|
+
})
|
|
11777
|
+
]);
|
|
11778
|
+
} finally {
|
|
11779
|
+
if (timer) clearTimeout(timer);
|
|
11780
|
+
}
|
|
11781
|
+
}
|
|
11782
|
+
function cleanupWaitMs(lifecycle) {
|
|
11783
|
+
if (lifecycle?.cleanupDeadlineAtMs == null) {
|
|
11784
|
+
return RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS;
|
|
11785
|
+
}
|
|
11786
|
+
return Math.max(
|
|
11787
|
+
0,
|
|
11788
|
+
Math.min(RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS, lifecycle.cleanupDeadlineAtMs - Date.now())
|
|
11789
|
+
);
|
|
11790
|
+
}
|
|
11301
11791
|
function tail(value, limit = OUTPUT_TAIL_LIMIT) {
|
|
11302
11792
|
return value.length > limit ? value.slice(-limit) : value;
|
|
11303
11793
|
}
|
|
@@ -11317,23 +11807,331 @@ function systemGrant(rig) {
|
|
|
11317
11807
|
}
|
|
11318
11808
|
async function terminateThrowaway(established) {
|
|
11319
11809
|
if (!established) {
|
|
11320
|
-
return;
|
|
11810
|
+
return true;
|
|
11321
11811
|
}
|
|
11322
11812
|
const client = established.client;
|
|
11323
11813
|
if (typeof client.delete === "function" && established.sessionState !== void 0) {
|
|
11324
|
-
|
|
11325
|
-
|
|
11814
|
+
try {
|
|
11815
|
+
await client.delete(established.sessionState);
|
|
11816
|
+
return true;
|
|
11817
|
+
} catch {
|
|
11818
|
+
return false;
|
|
11819
|
+
}
|
|
11326
11820
|
}
|
|
11327
11821
|
const session = established.session;
|
|
11328
|
-
|
|
11329
|
-
|
|
11330
|
-
|
|
11331
|
-
|
|
11332
|
-
|
|
11333
|
-
|
|
11822
|
+
try {
|
|
11823
|
+
if (session.terminate) {
|
|
11824
|
+
await session.terminate();
|
|
11825
|
+
} else if (session.kill) {
|
|
11826
|
+
await session.kill();
|
|
11827
|
+
} else if (session.close && !session.closed) {
|
|
11828
|
+
await session.close();
|
|
11829
|
+
} else {
|
|
11830
|
+
return false;
|
|
11831
|
+
}
|
|
11832
|
+
return true;
|
|
11833
|
+
} catch {
|
|
11834
|
+
return false;
|
|
11334
11835
|
}
|
|
11335
11836
|
}
|
|
11336
|
-
|
|
11837
|
+
var defaultOwnershipDependencies = {
|
|
11838
|
+
acquire: acquireLease2,
|
|
11839
|
+
establish: establishSandboxSessionFromEnvelope2,
|
|
11840
|
+
recordCreated: recordWarmingSandboxCreated2,
|
|
11841
|
+
commitWarm: commitWarmingToWarm2,
|
|
11842
|
+
touchHolder: touchLeaseHolder2,
|
|
11843
|
+
releaseHolder: releaseLeaseHolder2,
|
|
11844
|
+
failWarming: failWarmingToCold2,
|
|
11845
|
+
markWarmLost: markWarmLeaseInstanceLost2,
|
|
11846
|
+
serialize: serializeEstablishedSandboxEnvelope2,
|
|
11847
|
+
tag: tagModalSandbox2,
|
|
11848
|
+
terminate: terminateThrowaway,
|
|
11849
|
+
createCancellationController: createTurnToolCancellationController,
|
|
11850
|
+
randomUUID: () => crypto.randomUUID()
|
|
11851
|
+
};
|
|
11852
|
+
var RigVerificationLeaseUnavailableError = class extends Error {
|
|
11853
|
+
constructor(sandboxGroupId, role) {
|
|
11854
|
+
super(`Rig verification requires a new clean sandbox, but lease ${sandboxGroupId} was ${role}`);
|
|
11855
|
+
this.sandboxGroupId = sandboxGroupId;
|
|
11856
|
+
this.role = role;
|
|
11857
|
+
}
|
|
11858
|
+
name = "RigVerificationLeaseUnavailableError";
|
|
11859
|
+
};
|
|
11860
|
+
async function runWithOwnedRigVerificationSandbox(input, run, dependencies = defaultOwnershipDependencies) {
|
|
11861
|
+
if (!input.settings.rigVerificationLeaseOwnershipEnabled) {
|
|
11862
|
+
throw new Error(RIG_VERIFICATION_OWNERS_DISABLED_MESSAGE);
|
|
11863
|
+
}
|
|
11864
|
+
const holderId = `rig-verification:${dependencies.randomUUID()}`;
|
|
11865
|
+
const fallbackController = new AbortController();
|
|
11866
|
+
const signal = input.lifecycle?.signal ?? fallbackController.signal;
|
|
11867
|
+
const commandController = dependencies.createCancellationController(signal);
|
|
11868
|
+
const commandRunner = commandController.runSandboxCommand.bind(commandController);
|
|
11869
|
+
let cleanupTarget = null;
|
|
11870
|
+
let acquired = false;
|
|
11871
|
+
let ownsWarmingEpoch = false;
|
|
11872
|
+
let expectedEpoch = null;
|
|
11873
|
+
let holderLivenessTimer = null;
|
|
11874
|
+
let establishmentSettled = false;
|
|
11875
|
+
let establishmentPromise = null;
|
|
11876
|
+
const warnCleanupTimeout = (operation, instanceId) => {
|
|
11877
|
+
input.observability.warn("rig verifier: ownership cleanup timed out", {
|
|
11878
|
+
holderId,
|
|
11879
|
+
instanceId,
|
|
11880
|
+
operation,
|
|
11881
|
+
error: "cleanup did not settle inside the activity reserve; exact owner expiry and the global provider reaper are the bounded delayed backstop"
|
|
11882
|
+
});
|
|
11883
|
+
};
|
|
11884
|
+
const warnCleanupFailure = (operation, instanceId, error) => {
|
|
11885
|
+
input.observability.warn("rig verifier: ownership cleanup failed", {
|
|
11886
|
+
holderId,
|
|
11887
|
+
instanceId,
|
|
11888
|
+
operation,
|
|
11889
|
+
error: error instanceof Error ? error.message : String(error)
|
|
11890
|
+
});
|
|
11891
|
+
};
|
|
11892
|
+
const clearTerminatedLeasePointer = async (instanceId) => {
|
|
11893
|
+
if (expectedEpoch === null) return;
|
|
11894
|
+
const cleanupEpoch = expectedEpoch;
|
|
11895
|
+
const results = await Promise.allSettled([
|
|
11896
|
+
Promise.resolve().then(
|
|
11897
|
+
() => dependencies.markWarmLost(input.db, {
|
|
11898
|
+
accountId: input.accountId,
|
|
11899
|
+
workspaceId: input.workspaceId,
|
|
11900
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
11901
|
+
expectedEpoch: cleanupEpoch + 1,
|
|
11902
|
+
expectedInstanceId: instanceId
|
|
11903
|
+
})
|
|
11904
|
+
),
|
|
11905
|
+
Promise.resolve().then(
|
|
11906
|
+
() => dependencies.failWarming(input.db, {
|
|
11907
|
+
accountId: input.accountId,
|
|
11908
|
+
workspaceId: input.workspaceId,
|
|
11909
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
11910
|
+
expectedEpoch: cleanupEpoch
|
|
11911
|
+
})
|
|
11912
|
+
)
|
|
11913
|
+
]);
|
|
11914
|
+
for (const [index, result] of results.entries()) {
|
|
11915
|
+
if (result.status === "rejected") {
|
|
11916
|
+
warnCleanupFailure(
|
|
11917
|
+
index === 0 ? "mark_warm_lost" : "fail_warming",
|
|
11918
|
+
instanceId,
|
|
11919
|
+
result.reason
|
|
11920
|
+
);
|
|
11921
|
+
}
|
|
11922
|
+
}
|
|
11923
|
+
};
|
|
11924
|
+
const terminateAndClear = async (created) => {
|
|
11925
|
+
let terminated = false;
|
|
11926
|
+
try {
|
|
11927
|
+
terminated = await dependencies.terminate(created);
|
|
11928
|
+
} catch (error) {
|
|
11929
|
+
warnCleanupFailure("terminate", created.instanceId, error);
|
|
11930
|
+
}
|
|
11931
|
+
if (!terminated) {
|
|
11932
|
+
warnCleanupFailure(
|
|
11933
|
+
"terminate",
|
|
11934
|
+
created.instanceId,
|
|
11935
|
+
new Error("provider termination was not confirmed; durable lease cleanup remains pending")
|
|
11936
|
+
);
|
|
11937
|
+
return false;
|
|
11938
|
+
}
|
|
11939
|
+
await clearTerminatedLeasePointer(created.instanceId);
|
|
11940
|
+
return true;
|
|
11941
|
+
};
|
|
11942
|
+
try {
|
|
11943
|
+
throwIfAborted(signal);
|
|
11944
|
+
const lease = await dependencies.acquire(input.db, {
|
|
11945
|
+
accountId: input.accountId,
|
|
11946
|
+
workspaceId: input.workspaceId,
|
|
11947
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
11948
|
+
kind: "turn",
|
|
11949
|
+
holderId,
|
|
11950
|
+
subjectId: null,
|
|
11951
|
+
backend: input.settings.sandboxBackend,
|
|
11952
|
+
image: input.settings.modalImageRef ?? input.settings.dockerImage,
|
|
11953
|
+
rigVersionId: input.rigVersionId,
|
|
11954
|
+
leaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
|
|
11955
|
+
warmingLeaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS
|
|
11956
|
+
});
|
|
11957
|
+
acquired = true;
|
|
11958
|
+
expectedEpoch = lease.lease.leaseEpoch;
|
|
11959
|
+
holderLivenessTimer = setInterval(() => {
|
|
11960
|
+
void dependencies.touchHolder(input.db, {
|
|
11961
|
+
accountId: input.accountId,
|
|
11962
|
+
workspaceId: input.workspaceId,
|
|
11963
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
11964
|
+
kind: "turn",
|
|
11965
|
+
holderId
|
|
11966
|
+
}).catch(() => void 0);
|
|
11967
|
+
}, RIG_VERIFICATION_HOLDER_TOUCH_INTERVAL_MS);
|
|
11968
|
+
if ("unref" in holderLivenessTimer && typeof holderLivenessTimer.unref === "function") {
|
|
11969
|
+
holderLivenessTimer.unref();
|
|
11970
|
+
}
|
|
11971
|
+
if (lease.role !== "spawner") {
|
|
11972
|
+
throw new RigVerificationLeaseUnavailableError(input.sandboxGroupId, lease.role);
|
|
11973
|
+
}
|
|
11974
|
+
ownsWarmingEpoch = true;
|
|
11975
|
+
establishmentPromise = Promise.resolve().then(
|
|
11976
|
+
() => dependencies.establish(input.settings, null, {
|
|
11977
|
+
sessionId: `${input.sessionIdPrefix}-${holderId}`,
|
|
11978
|
+
recovery: "create-or-restore",
|
|
11979
|
+
environment: {},
|
|
11980
|
+
onSandboxCreated: async (created) => {
|
|
11981
|
+
cleanupTarget = created;
|
|
11982
|
+
const resumeState2 = created.backendId === "modal" ? null : await dependencies.serialize(created);
|
|
11983
|
+
const recorded = await dependencies.recordCreated(input.db, {
|
|
11984
|
+
accountId: input.accountId,
|
|
11985
|
+
workspaceId: input.workspaceId,
|
|
11986
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
11987
|
+
expectedEpoch: lease.lease.leaseEpoch,
|
|
11988
|
+
instanceId: created.instanceId,
|
|
11989
|
+
resumeBackendId: created.backendId,
|
|
11990
|
+
resumeState: resumeState2,
|
|
11991
|
+
leaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
|
|
11992
|
+
warmingLeaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS
|
|
11993
|
+
});
|
|
11994
|
+
if (!recorded.recorded) {
|
|
11995
|
+
throw new RigVerificationLeaseUnavailableError(input.sandboxGroupId, "fenced");
|
|
11996
|
+
}
|
|
11997
|
+
if (created.backendId === "modal" && !signal.aborted) {
|
|
11998
|
+
try {
|
|
11999
|
+
await waitForAbortable(
|
|
12000
|
+
dependencies.tag(input.settings, created.instanceId, {
|
|
12001
|
+
leaseId: lease.lease.id,
|
|
12002
|
+
workspaceId: input.workspaceId,
|
|
12003
|
+
sandboxGroupId: input.sandboxGroupId
|
|
12004
|
+
}),
|
|
12005
|
+
signal
|
|
12006
|
+
);
|
|
12007
|
+
} catch (error) {
|
|
12008
|
+
if (!signal.aborted) {
|
|
12009
|
+
input.observability.warn("rig verifier: Modal ownership tag failed", {
|
|
12010
|
+
holderId,
|
|
12011
|
+
instanceId: created.instanceId,
|
|
12012
|
+
error: error instanceof Error ? error.message : String(error)
|
|
12013
|
+
});
|
|
12014
|
+
}
|
|
12015
|
+
}
|
|
12016
|
+
}
|
|
12017
|
+
if (signal.aborted) {
|
|
12018
|
+
await terminateAndClear(created);
|
|
12019
|
+
throw abortReason(signal);
|
|
12020
|
+
}
|
|
12021
|
+
}
|
|
12022
|
+
})
|
|
12023
|
+
);
|
|
12024
|
+
void establishmentPromise.then(
|
|
12025
|
+
() => {
|
|
12026
|
+
establishmentSettled = true;
|
|
12027
|
+
},
|
|
12028
|
+
() => {
|
|
12029
|
+
establishmentSettled = true;
|
|
12030
|
+
}
|
|
12031
|
+
);
|
|
12032
|
+
void establishmentPromise.catch(() => void 0);
|
|
12033
|
+
const established = await waitForAbortable(establishmentPromise, signal);
|
|
12034
|
+
cleanupTarget = established;
|
|
12035
|
+
const resumeState = established.backendId === "modal" ? null : await dependencies.serialize(established);
|
|
12036
|
+
const committed = await dependencies.commitWarm(input.db, {
|
|
12037
|
+
accountId: input.accountId,
|
|
12038
|
+
workspaceId: input.workspaceId,
|
|
12039
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
12040
|
+
expectedEpoch: lease.lease.leaseEpoch,
|
|
12041
|
+
instanceId: established.instanceId,
|
|
12042
|
+
dataPlaneUrl: null,
|
|
12043
|
+
resumeBackendId: established.backendId,
|
|
12044
|
+
resumeState,
|
|
12045
|
+
leaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS
|
|
12046
|
+
});
|
|
12047
|
+
if (!committed.committed || !committed.lease) {
|
|
12048
|
+
throw new RigVerificationLeaseUnavailableError(input.sandboxGroupId, "fenced");
|
|
12049
|
+
}
|
|
12050
|
+
const result = await run(established, { signal, commandRunner });
|
|
12051
|
+
throwIfAborted(signal);
|
|
12052
|
+
return result;
|
|
12053
|
+
} catch (error) {
|
|
12054
|
+
if (signal.aborted) throw abortReason(signal);
|
|
12055
|
+
throw error;
|
|
12056
|
+
} finally {
|
|
12057
|
+
if (signal.aborted && establishmentPromise && !establishmentSettled) {
|
|
12058
|
+
const lateEstablishment = await settleWithin(
|
|
12059
|
+
Promise.allSettled([establishmentPromise]),
|
|
12060
|
+
cleanupWaitMs(input.lifecycle)
|
|
12061
|
+
);
|
|
12062
|
+
if (lateEstablishment.status === "timed_out") {
|
|
12063
|
+
warnCleanupTimeout("establish_after_abort", cleanupTarget?.instanceId ?? null);
|
|
12064
|
+
}
|
|
12065
|
+
}
|
|
12066
|
+
if (holderLivenessTimer) {
|
|
12067
|
+
clearInterval(holderLivenessTimer);
|
|
12068
|
+
holderLivenessTimer = null;
|
|
12069
|
+
}
|
|
12070
|
+
commandController.cancel(signal.aborted ? abortReason(signal) : new Error("Verifier complete"));
|
|
12071
|
+
const quiescence = commandController.waitForQuiescence();
|
|
12072
|
+
void quiescence.catch(() => void 0);
|
|
12073
|
+
const quiescenceResult = await settleWithin(quiescence, cleanupWaitMs(input.lifecycle));
|
|
12074
|
+
if (quiescenceResult.status === "timed_out") {
|
|
12075
|
+
warnCleanupTimeout("command_quiescence", cleanupTarget?.instanceId ?? null);
|
|
12076
|
+
}
|
|
12077
|
+
const providerCleanup = Promise.resolve().then(async () => {
|
|
12078
|
+
if (!cleanupTarget) {
|
|
12079
|
+
if (ownsWarmingEpoch && expectedEpoch !== null) {
|
|
12080
|
+
await dependencies.failWarming(input.db, {
|
|
12081
|
+
accountId: input.accountId,
|
|
12082
|
+
workspaceId: input.workspaceId,
|
|
12083
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
12084
|
+
expectedEpoch
|
|
12085
|
+
});
|
|
12086
|
+
}
|
|
12087
|
+
return true;
|
|
12088
|
+
}
|
|
12089
|
+
return await terminateAndClear(cleanupTarget);
|
|
12090
|
+
});
|
|
12091
|
+
void providerCleanup.catch(() => void 0);
|
|
12092
|
+
const providerSettlement = await settleWithin(
|
|
12093
|
+
Promise.allSettled([providerCleanup]),
|
|
12094
|
+
cleanupWaitMs(input.lifecycle)
|
|
12095
|
+
);
|
|
12096
|
+
if (providerSettlement.status === "timed_out") {
|
|
12097
|
+
warnCleanupTimeout("terminate_and_clear", cleanupTarget?.instanceId ?? null);
|
|
12098
|
+
} else if (providerSettlement.value[0]?.status === "rejected") {
|
|
12099
|
+
warnCleanupFailure(
|
|
12100
|
+
"terminate_and_clear",
|
|
12101
|
+
cleanupTarget?.instanceId ?? null,
|
|
12102
|
+
providerSettlement.value[0].reason
|
|
12103
|
+
);
|
|
12104
|
+
}
|
|
12105
|
+
if (acquired) {
|
|
12106
|
+
const release = Promise.resolve().then(
|
|
12107
|
+
() => dependencies.releaseHolder(input.db, {
|
|
12108
|
+
accountId: input.accountId,
|
|
12109
|
+
workspaceId: input.workspaceId,
|
|
12110
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
12111
|
+
kind: "turn",
|
|
12112
|
+
holderId,
|
|
12113
|
+
idleGraceMs: RIG_VERIFICATION_RELEASE_GRACE_MS
|
|
12114
|
+
})
|
|
12115
|
+
);
|
|
12116
|
+
void release.catch(() => void 0);
|
|
12117
|
+
const released = await settleWithin(
|
|
12118
|
+
Promise.allSettled([release]),
|
|
12119
|
+
cleanupWaitMs(input.lifecycle)
|
|
12120
|
+
);
|
|
12121
|
+
if (released.status === "timed_out") {
|
|
12122
|
+
warnCleanupTimeout("release_holder", cleanupTarget?.instanceId ?? null);
|
|
12123
|
+
} else if (released.value[0]?.status === "rejected") {
|
|
12124
|
+
warnCleanupFailure(
|
|
12125
|
+
"release_holder",
|
|
12126
|
+
cleanupTarget?.instanceId ?? null,
|
|
12127
|
+
released.value[0].reason
|
|
12128
|
+
);
|
|
12129
|
+
}
|
|
12130
|
+
}
|
|
12131
|
+
throwIfAborted(signal);
|
|
12132
|
+
}
|
|
12133
|
+
}
|
|
12134
|
+
async function runCommand(session, command, timeoutMs, commandRunner) {
|
|
11337
12135
|
const args = {
|
|
11338
12136
|
cmd: command,
|
|
11339
12137
|
workdir: "/workspace",
|
|
@@ -11341,9 +12139,7 @@ async function runCommand(session, command, timeoutMs) {
|
|
|
11341
12139
|
yieldTimeMs: timeoutMs,
|
|
11342
12140
|
maxOutputTokens: 4e4
|
|
11343
12141
|
};
|
|
11344
|
-
const result =
|
|
11345
|
-
throw new Error("Sandbox session does not support command execution");
|
|
11346
|
-
})();
|
|
12142
|
+
const result = await commandRunner(session, args);
|
|
11347
12143
|
return {
|
|
11348
12144
|
exitCode: sandboxCommandExitCode(result),
|
|
11349
12145
|
output: scrubVerificationOutput(tail(sandboxCommandOutput(result)))
|
|
@@ -11400,10 +12196,20 @@ async function loadVersionTarget(db, workspaceId, versionId) {
|
|
|
11400
12196
|
}
|
|
11401
12197
|
return { rig, version };
|
|
11402
12198
|
}
|
|
12199
|
+
async function withRigVerificationActivityLifecycle(run) {
|
|
12200
|
+
const lifecycle = createRigVerificationActivityLifecycle();
|
|
12201
|
+
try {
|
|
12202
|
+
throwIfAborted(lifecycle.signal);
|
|
12203
|
+
return await run(lifecycle);
|
|
12204
|
+
} finally {
|
|
12205
|
+
lifecycle.dispose();
|
|
12206
|
+
}
|
|
12207
|
+
}
|
|
11403
12208
|
function createRigVerificationActivities(services) {
|
|
11404
12209
|
return {
|
|
11405
|
-
verifyRigChange: async (
|
|
11406
|
-
const { settings, db } = await services();
|
|
12210
|
+
verifyRigChange: (input) => withRigVerificationActivityLifecycle(async (lifecycle) => {
|
|
12211
|
+
const { settings, db, observability } = await services();
|
|
12212
|
+
throwIfAborted(lifecycle.signal);
|
|
11407
12213
|
const { rig, baseVersion, change } = await loadChangeTarget(
|
|
11408
12214
|
db,
|
|
11409
12215
|
input.workspaceId,
|
|
@@ -11421,114 +12227,126 @@ function createRigVerificationActivities(services) {
|
|
|
11421
12227
|
rigId: rig.id,
|
|
11422
12228
|
metadata: { changeId: change.id }
|
|
11423
12229
|
});
|
|
11424
|
-
let established = null;
|
|
11425
12230
|
const verification = { startedAt, checkResults: [] };
|
|
11426
12231
|
try {
|
|
11427
12232
|
const candidateVersion = candidateVersionForChange(baseVersion, change);
|
|
11428
12233
|
const runSettings = settingsWithRigImage(settings, candidateVersion.image);
|
|
11429
|
-
|
|
11430
|
-
|
|
11431
|
-
|
|
11432
|
-
|
|
11433
|
-
|
|
11434
|
-
|
|
11435
|
-
|
|
11436
|
-
|
|
11437
|
-
|
|
11438
|
-
|
|
11439
|
-
|
|
11440
|
-
|
|
11441
|
-
|
|
11442
|
-
|
|
11443
|
-
|
|
12234
|
+
return await runWithOwnedRigVerificationSandbox(
|
|
12235
|
+
{
|
|
12236
|
+
settings: runSettings,
|
|
12237
|
+
db,
|
|
12238
|
+
observability,
|
|
12239
|
+
accountId: rig.accountId,
|
|
12240
|
+
workspaceId: input.workspaceId,
|
|
12241
|
+
sandboxGroupId: change.id,
|
|
12242
|
+
rigVersionId: candidateVersion.id,
|
|
12243
|
+
sessionIdPrefix: `rig-verification-${change.id}`,
|
|
12244
|
+
lifecycle
|
|
12245
|
+
},
|
|
12246
|
+
async (established, runContext) => {
|
|
12247
|
+
if ((candidateVersion.setupScript ?? "").trim()) {
|
|
12248
|
+
await runRigSetupHook(established.session, {
|
|
12249
|
+
environment: {},
|
|
12250
|
+
runAs: "root",
|
|
12251
|
+
commandRunner: runContext.commandRunner,
|
|
12252
|
+
rigSetup: {
|
|
12253
|
+
rigId: rig.id,
|
|
12254
|
+
rigName: rig.name,
|
|
12255
|
+
versionId: candidateVersion.id,
|
|
12256
|
+
script: candidateVersion.setupScript ?? "",
|
|
12257
|
+
timeoutMs: settings.rigSetupTimeoutMs
|
|
12258
|
+
}
|
|
12259
|
+
});
|
|
12260
|
+
verification.setupResult = { exitCode: 0, output: "" };
|
|
11444
12261
|
}
|
|
11445
|
-
|
|
11446
|
-
|
|
11447
|
-
|
|
11448
|
-
|
|
11449
|
-
|
|
11450
|
-
|
|
11451
|
-
|
|
11452
|
-
|
|
11453
|
-
|
|
11454
|
-
|
|
11455
|
-
|
|
11456
|
-
|
|
12262
|
+
const command = setupAppendCommand(change);
|
|
12263
|
+
if (command) {
|
|
12264
|
+
const commandResult = await runCommand(
|
|
12265
|
+
established.session,
|
|
12266
|
+
command,
|
|
12267
|
+
settings.rigSetupTimeoutMs,
|
|
12268
|
+
runContext.commandRunner
|
|
12269
|
+
);
|
|
12270
|
+
verification.commandResult = commandResult;
|
|
12271
|
+
if (commandResult.exitCode !== 0) {
|
|
12272
|
+
verification.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
12273
|
+
verification.passed = false;
|
|
12274
|
+
const updated2 = await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
12275
|
+
status: "rejected",
|
|
12276
|
+
verification: scrubVerificationPayload(verification)
|
|
12277
|
+
});
|
|
12278
|
+
await recordRigAuditEvent(db, {
|
|
12279
|
+
grant,
|
|
12280
|
+
action: "rig.verification.failed",
|
|
12281
|
+
rigId: rig.id,
|
|
12282
|
+
metadata: { changeId: change.id, status: "rejected" }
|
|
12283
|
+
});
|
|
12284
|
+
await recordRigAuditEvent(db, {
|
|
12285
|
+
grant,
|
|
12286
|
+
action: "rig.change.rejected",
|
|
12287
|
+
rigId: rig.id,
|
|
12288
|
+
metadata: { changeId: change.id }
|
|
12289
|
+
});
|
|
12290
|
+
return updated2;
|
|
12291
|
+
}
|
|
12292
|
+
}
|
|
12293
|
+
const checkResults = [];
|
|
12294
|
+
for (const check of candidateVersion.checks) {
|
|
12295
|
+
const result = await runCommand(
|
|
12296
|
+
established.session,
|
|
12297
|
+
check.command,
|
|
12298
|
+
settings.rigSetupTimeoutMs,
|
|
12299
|
+
runContext.commandRunner
|
|
12300
|
+
);
|
|
12301
|
+
checkResults.push({
|
|
12302
|
+
name: check.name,
|
|
12303
|
+
command: scrubVerificationOutput(check.command),
|
|
12304
|
+
...result
|
|
12305
|
+
});
|
|
12306
|
+
}
|
|
12307
|
+
verification.checkResults = checkResults;
|
|
12308
|
+
const passed = checkResults.every((result) => result.exitCode === 0);
|
|
11457
12309
|
verification.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
11458
|
-
verification.passed =
|
|
11459
|
-
const
|
|
11460
|
-
|
|
12310
|
+
verification.passed = passed;
|
|
12311
|
+
const classified = classifyRigVerificationOutcome({ kind: change.kind, passed });
|
|
12312
|
+
if (classified.action === "auto_promote") {
|
|
12313
|
+
await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
12314
|
+
status: "verifying",
|
|
12315
|
+
verification: scrubVerificationPayload(verification)
|
|
12316
|
+
});
|
|
12317
|
+
const { change: merged } = await promoteSetupAppendChange({ db }, grant, rig, {
|
|
12318
|
+
...change,
|
|
12319
|
+
verification
|
|
12320
|
+
});
|
|
12321
|
+
await recordRigAuditEvent(db, {
|
|
12322
|
+
grant,
|
|
12323
|
+
action: "rig.verification.passed",
|
|
12324
|
+
rigId: rig.id,
|
|
12325
|
+
metadata: { changeId: change.id }
|
|
12326
|
+
});
|
|
12327
|
+
return merged;
|
|
12328
|
+
}
|
|
12329
|
+
const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
12330
|
+
status: classified.status,
|
|
11461
12331
|
verification: scrubVerificationPayload(verification)
|
|
11462
12332
|
});
|
|
11463
12333
|
await recordRigAuditEvent(db, {
|
|
11464
12334
|
grant,
|
|
11465
|
-
action: "rig.verification.failed",
|
|
12335
|
+
action: passed ? "rig.verification.passed" : "rig.verification.failed",
|
|
11466
12336
|
rigId: rig.id,
|
|
11467
|
-
metadata: { changeId: change.id, status:
|
|
12337
|
+
metadata: { changeId: change.id, status: classified.status }
|
|
11468
12338
|
});
|
|
11469
|
-
|
|
11470
|
-
|
|
11471
|
-
|
|
11472
|
-
|
|
11473
|
-
|
|
11474
|
-
|
|
11475
|
-
|
|
12339
|
+
if (!passed) {
|
|
12340
|
+
await recordRigAuditEvent(db, {
|
|
12341
|
+
grant,
|
|
12342
|
+
action: "rig.change.rejected",
|
|
12343
|
+
rigId: rig.id,
|
|
12344
|
+
metadata: { changeId: change.id }
|
|
12345
|
+
});
|
|
12346
|
+
}
|
|
12347
|
+
return updated;
|
|
11476
12348
|
}
|
|
11477
|
-
|
|
11478
|
-
const checkResults = [];
|
|
11479
|
-
for (const check of candidateVersion.checks) {
|
|
11480
|
-
const result = await runCommand(
|
|
11481
|
-
established.session,
|
|
11482
|
-
check.command,
|
|
11483
|
-
settings.rigSetupTimeoutMs
|
|
11484
|
-
);
|
|
11485
|
-
checkResults.push({
|
|
11486
|
-
name: check.name,
|
|
11487
|
-
command: scrubVerificationOutput(check.command),
|
|
11488
|
-
...result
|
|
11489
|
-
});
|
|
11490
|
-
}
|
|
11491
|
-
verification.checkResults = checkResults;
|
|
11492
|
-
const passed = checkResults.every((result) => result.exitCode === 0);
|
|
11493
|
-
verification.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
11494
|
-
verification.passed = passed;
|
|
11495
|
-
const classified = classifyRigVerificationOutcome({ kind: change.kind, passed });
|
|
11496
|
-
if (classified.action === "auto_promote") {
|
|
11497
|
-
await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
11498
|
-
status: "verifying",
|
|
11499
|
-
verification: scrubVerificationPayload(verification)
|
|
11500
|
-
});
|
|
11501
|
-
const { change: merged } = await promoteSetupAppendChange({ db }, grant, rig, {
|
|
11502
|
-
...change,
|
|
11503
|
-
verification
|
|
11504
|
-
});
|
|
11505
|
-
await recordRigAuditEvent(db, {
|
|
11506
|
-
grant,
|
|
11507
|
-
action: "rig.verification.passed",
|
|
11508
|
-
rigId: rig.id,
|
|
11509
|
-
metadata: { changeId: change.id }
|
|
11510
|
-
});
|
|
11511
|
-
return merged;
|
|
11512
|
-
}
|
|
11513
|
-
const updated = await updateRigChangeStatus(db, input.workspaceId, change.id, {
|
|
11514
|
-
status: classified.status,
|
|
11515
|
-
verification: scrubVerificationPayload(verification)
|
|
11516
|
-
});
|
|
11517
|
-
await recordRigAuditEvent(db, {
|
|
11518
|
-
grant,
|
|
11519
|
-
action: passed ? "rig.verification.passed" : "rig.verification.failed",
|
|
11520
|
-
rigId: rig.id,
|
|
11521
|
-
metadata: { changeId: change.id, status: classified.status }
|
|
11522
|
-
});
|
|
11523
|
-
if (!passed) {
|
|
11524
|
-
await recordRigAuditEvent(db, {
|
|
11525
|
-
grant,
|
|
11526
|
-
action: "rig.change.rejected",
|
|
11527
|
-
rigId: rig.id,
|
|
11528
|
-
metadata: { changeId: change.id }
|
|
11529
|
-
});
|
|
11530
|
-
}
|
|
11531
|
-
return updated;
|
|
12349
|
+
);
|
|
11532
12350
|
} catch (error) {
|
|
11533
12351
|
verification.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
11534
12352
|
verification.passed = false;
|
|
@@ -11551,13 +12369,13 @@ function createRigVerificationActivities(services) {
|
|
|
11551
12369
|
rigId: rig.id,
|
|
11552
12370
|
metadata: { changeId: change.id }
|
|
11553
12371
|
});
|
|
12372
|
+
if (lifecycle.signal.aborted) throw abortReason(lifecycle.signal);
|
|
11554
12373
|
return updated;
|
|
11555
|
-
} finally {
|
|
11556
|
-
await terminateThrowaway(established);
|
|
11557
12374
|
}
|
|
11558
|
-
},
|
|
11559
|
-
verifyRigVersion: async (
|
|
11560
|
-
const { settings, db } = await services();
|
|
12375
|
+
}),
|
|
12376
|
+
verifyRigVersion: (input) => withRigVerificationActivityLifecycle(async (lifecycle) => {
|
|
12377
|
+
const { settings, db, observability } = await services();
|
|
12378
|
+
throwIfAborted(lifecycle.signal);
|
|
11561
12379
|
const { rig, version } = await loadVersionTarget(db, input.workspaceId, input.versionId);
|
|
11562
12380
|
const grant = systemGrant(rig);
|
|
11563
12381
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
@@ -11567,53 +12385,64 @@ function createRigVerificationActivities(services) {
|
|
|
11567
12385
|
rigId: rig.id,
|
|
11568
12386
|
metadata: { versionId: version.id }
|
|
11569
12387
|
});
|
|
11570
|
-
let established = null;
|
|
11571
12388
|
try {
|
|
11572
12389
|
const runSettings = settingsWithRigImage(settings, version.image);
|
|
11573
|
-
|
|
11574
|
-
|
|
11575
|
-
|
|
11576
|
-
|
|
11577
|
-
|
|
11578
|
-
|
|
11579
|
-
|
|
11580
|
-
|
|
11581
|
-
|
|
11582
|
-
|
|
11583
|
-
|
|
11584
|
-
|
|
11585
|
-
|
|
11586
|
-
|
|
11587
|
-
|
|
12390
|
+
return await runWithOwnedRigVerificationSandbox(
|
|
12391
|
+
{
|
|
12392
|
+
settings: runSettings,
|
|
12393
|
+
db,
|
|
12394
|
+
observability,
|
|
12395
|
+
accountId: rig.accountId,
|
|
12396
|
+
workspaceId: input.workspaceId,
|
|
12397
|
+
sandboxGroupId: version.id,
|
|
12398
|
+
rigVersionId: version.id,
|
|
12399
|
+
sessionIdPrefix: `rig-version-verification-${version.id}`,
|
|
12400
|
+
lifecycle
|
|
12401
|
+
},
|
|
12402
|
+
async (established, runContext) => {
|
|
12403
|
+
if ((version.setupScript ?? "").trim()) {
|
|
12404
|
+
await runRigSetupHook(established.session, {
|
|
12405
|
+
environment: {},
|
|
12406
|
+
runAs: "root",
|
|
12407
|
+
commandRunner: runContext.commandRunner,
|
|
12408
|
+
rigSetup: {
|
|
12409
|
+
rigId: rig.id,
|
|
12410
|
+
rigName: rig.name,
|
|
12411
|
+
versionId: version.id,
|
|
12412
|
+
script: version.setupScript ?? "",
|
|
12413
|
+
timeoutMs: settings.rigSetupTimeoutMs
|
|
12414
|
+
}
|
|
12415
|
+
});
|
|
11588
12416
|
}
|
|
11589
|
-
|
|
11590
|
-
|
|
11591
|
-
|
|
11592
|
-
|
|
11593
|
-
|
|
11594
|
-
|
|
11595
|
-
|
|
11596
|
-
|
|
11597
|
-
|
|
11598
|
-
|
|
11599
|
-
|
|
11600
|
-
|
|
11601
|
-
|
|
11602
|
-
|
|
11603
|
-
|
|
11604
|
-
|
|
11605
|
-
|
|
11606
|
-
|
|
11607
|
-
|
|
11608
|
-
|
|
11609
|
-
|
|
11610
|
-
|
|
11611
|
-
|
|
11612
|
-
|
|
11613
|
-
|
|
11614
|
-
|
|
11615
|
-
|
|
11616
|
-
|
|
12417
|
+
const checkResults = [];
|
|
12418
|
+
for (const check of version.checks) {
|
|
12419
|
+
checkResults.push({
|
|
12420
|
+
name: check.name,
|
|
12421
|
+
command: scrubVerificationOutput(check.command),
|
|
12422
|
+
...await runCommand(
|
|
12423
|
+
established.session,
|
|
12424
|
+
check.command,
|
|
12425
|
+
settings.rigSetupTimeoutMs,
|
|
12426
|
+
runContext.commandRunner
|
|
12427
|
+
)
|
|
12428
|
+
});
|
|
12429
|
+
}
|
|
12430
|
+
const passed = checkResults.every((result) => result.exitCode === 0);
|
|
12431
|
+
await recordRigAuditEvent(db, {
|
|
12432
|
+
grant,
|
|
12433
|
+
action: passed ? "rig.verification.passed" : "rig.verification.failed",
|
|
12434
|
+
rigId: rig.id,
|
|
12435
|
+
metadata: scrubVerificationPayload({
|
|
12436
|
+
versionId: version.id,
|
|
12437
|
+
startedAt,
|
|
12438
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
12439
|
+
passed,
|
|
12440
|
+
checkResults
|
|
12441
|
+
})
|
|
12442
|
+
});
|
|
12443
|
+
return { versionId: version.id, passed, checkResults };
|
|
12444
|
+
}
|
|
12445
|
+
);
|
|
11617
12446
|
} catch (error) {
|
|
11618
12447
|
const detail = tail(
|
|
11619
12448
|
scrubVerificationOutput(error instanceof Error ? error.message : String(error)),
|
|
@@ -11632,10 +12461,8 @@ function createRigVerificationActivities(services) {
|
|
|
11632
12461
|
})
|
|
11633
12462
|
});
|
|
11634
12463
|
throw error;
|
|
11635
|
-
} finally {
|
|
11636
|
-
await terminateThrowaway(established);
|
|
11637
12464
|
}
|
|
11638
|
-
}
|
|
12465
|
+
})
|
|
11639
12466
|
};
|
|
11640
12467
|
}
|
|
11641
12468
|
|