@tangle-network/agent-runtime 0.116.0 → 0.117.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/dist/{activation-DuqQhee6.js → activation-BZ2FyX0O.js} +3 -3
- package/dist/{activation-DuqQhee6.js.map → activation-BZ2FyX0O.js.map} +1 -1
- package/dist/agent.d.ts +3 -64
- package/dist/agent.js +5 -207
- package/dist/agent.js.map +1 -1
- package/dist/{analyst-loop-BoNIG2hA.js → analyst-loop-DvSciOfB.js} +2 -2
- package/dist/{analyst-loop-BoNIG2hA.js.map → analyst-loop-DvSciOfB.js.map} +1 -1
- package/dist/analyst-loop.js +1 -1
- package/dist/candidate-execution/index.d.ts +3 -3
- package/dist/candidate-execution/index.js +5 -5
- package/dist/{candidate-execution-CfpJrd3o.js → candidate-execution-PX61zXd2.js} +4 -4
- package/dist/{candidate-execution-CfpJrd3o.js.map → candidate-execution-PX61zXd2.js.map} +1 -1
- package/dist/{environment-provider-CWsRh6Uz.d.ts → environment-provider-CTZjdIUg.d.ts} +298 -32
- package/dist/{environment-provider-CCaEhA-l.js → environment-provider-CzcEyRL7.js} +140 -31
- package/dist/environment-provider-CzcEyRL7.js.map +1 -0
- package/dist/environment-provider.d.ts +1 -1
- package/dist/environment-provider.js +1 -1
- package/dist/{improvement-cycle-C1cmjvPD.js → improvement-cycle-Bcqz8rZI.js} +160 -19
- package/dist/improvement-cycle-Bcqz8rZI.js.map +1 -0
- package/dist/{index-CYkDeM5L.d.ts → index-BHSYaWrC.d.ts} +3 -3
- package/dist/{index-DcLMNnG5.d.ts → index-BTUG4Xln.d.ts} +6 -6
- package/dist/{index-COumPQka.d.ts → index-DPWJxSRa.d.ts} +847 -275
- package/dist/{index-C-FYUuFG.d.ts → index-X2YwyWkc.d.ts} +2 -2
- package/dist/index.d.ts +8 -8
- package/dist/index.js +12 -12
- package/dist/intelligence.d.ts +8 -7
- package/dist/intelligence.js +15 -9
- package/dist/intelligence.js.map +1 -1
- package/dist/kernel.d.ts +3 -3
- package/dist/kernel.js +8 -8
- package/dist/{knowledge-DOzbywZT.js → knowledge-CDlrayRc.js} +7 -7
- package/dist/knowledge-CDlrayRc.js.map +1 -0
- package/dist/knowledge.d.ts +1 -1
- package/dist/knowledge.js +1 -1
- package/dist/{loop-runner-bin-BuQjc5DR.js → loop-runner-bin-BxhhDtmG.js} +4 -4
- package/dist/{loop-runner-bin-BuQjc5DR.js.map → loop-runner-bin-BxhhDtmG.js.map} +1 -1
- package/dist/{loop-runner-bin-BFrhPLKt.d.ts → loop-runner-bin-oyL7it5O.d.ts} +3 -3
- package/dist/loop-runner-bin.d.ts +1 -1
- package/dist/loop-runner-bin.js +1 -1
- package/dist/mcp/bin.js +43 -10
- package/dist/mcp/bin.js.map +1 -1
- package/dist/mcp/index.d.ts +3 -43
- package/dist/mcp/index.js +5 -204
- package/dist/mcp/index.js.map +1 -1
- package/dist/{openai-tools-_Wyp4udO.js → openai-tools-CO9X36zy.js} +2 -2
- package/dist/{openai-tools-_Wyp4udO.js.map → openai-tools-CO9X36zy.js.map} +1 -1
- package/dist/{prepare-BHQBb02e.js → prepare-BbI6qGOW.js} +5 -2
- package/dist/prepare-BbI6qGOW.js.map +1 -0
- package/dist/primeintellect/index.d.ts +1 -1
- package/dist/{protected-model-port-BP6Z4eau.d.ts → protected-model-port-B0yXMpLL.d.ts} +10 -2
- package/dist/{protected-model-port-DqAH1Z2M.js → protected-model-port-COnyZfML.js} +2 -2
- package/dist/{protected-model-port-DqAH1Z2M.js.map → protected-model-port-COnyZfML.js.map} +1 -1
- package/dist/{runtime-Ut1pkd2n.js → runtime-Ce39zfqn.js} +145 -53
- package/dist/runtime-Ce39zfqn.js.map +1 -0
- package/dist/{sandbox-events-DeI5xX8P.js → sandbox-events-Yhd1GYWl.js} +4 -2
- package/dist/sandbox-events-Yhd1GYWl.js.map +1 -0
- package/dist/spawn-journal-B_mu_oH2.js +857 -0
- package/dist/spawn-journal-B_mu_oH2.js.map +1 -0
- package/dist/{structural-rollout-CVY_0hJp.js → structural-rollout-DLwW69R4.js} +4 -4
- package/dist/structural-rollout-DLwW69R4.js.map +1 -0
- package/dist/{supervise-BUR9ByF7.js → supervise-U7_K5yuV.js} +2598 -821
- package/dist/supervise-U7_K5yuV.js.map +1 -0
- package/dist/{supervisor-BBbPBXpe.js → supervisor-CAXiCKnX.js} +2808 -973
- package/dist/supervisor-CAXiCKnX.js.map +1 -0
- package/dist/testing.js +98 -76
- package/dist/testing.js.map +1 -1
- package/dist/{workspace-archive-DXzJq7WP.js → workspace-archive-CqPIijFA.js} +2 -2
- package/dist/{workspace-archive-DXzJq7WP.js.map → workspace-archive-CqPIijFA.js.map} +1 -1
- package/package.json +1 -1
- package/dist/environment-provider-CCaEhA-l.js.map +0 -1
- package/dist/improvement-cycle-C1cmjvPD.js.map +0 -1
- package/dist/knowledge-DOzbywZT.js.map +0 -1
- package/dist/prepare-BHQBb02e.js.map +0 -1
- package/dist/runtime-Ut1pkd2n.js.map +0 -1
- package/dist/sandbox-events-DeI5xX8P.js.map +0 -1
- package/dist/spawn-journal-DCPbicXB.js +0 -457
- package/dist/spawn-journal-DCPbicXB.js.map +0 -1
- package/dist/structural-rollout-CVY_0hJp.js.map +0 -1
- package/dist/supervise-BUR9ByF7.js.map +0 -1
- package/dist/supervisor-BBbPBXpe.js.map +0 -1
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { c as RuntimeRunStateError, u as ValidationError } from "./errors-DEAvWQPy.js";
|
|
2
|
+
import { S as detachedSnapshot, _ as captureWorkerTraceEvidence, c as replaySpawnTree, f as attestNestedDriverTreeOwner, h as runtimeOwnedNestedDriverTreeRoot, m as nestedDriverTreeRoot, o as materializeTreeView, p as driverRuntime, s as pendingWaits, x as contentAddress } from "./spawn-journal-B_mu_oH2.js";
|
|
2
3
|
import { n as addTokenUsage, p as zeroTokenUsage } from "./util-Cc9g9Y-o.js";
|
|
3
|
-
import { a as contentAddress, c as replaySpawnTree, o as materializeTreeView, s as pendingWaits } from "./spawn-journal-DCPbicXB.js";
|
|
4
4
|
import { i as notifyRuntimeHookEvent } from "./runtime-hooks-C7iJOWm3.js";
|
|
5
5
|
import { estimateCost, isModelPriced } from "@tangle-network/agent-eval";
|
|
6
|
+
import { AGENT_PROFILE_MATERIALIZATION_AXES, agentProfileSchema, canonicalCandidateDigest, profileMaterializationAxes, sha256DigestSchema } from "@tangle-network/agent-interface";
|
|
6
7
|
import { createHash, randomUUID } from "node:crypto";
|
|
7
|
-
import { applyWorkspacePlan, materializeProfile, resolveAgentProfileResources } from "@tangle-network/agent-profile-materialize";
|
|
8
|
+
import { applyWorkspacePlan, hashWorkspacePlan, materializeProfile, resolveAgentProfileResources } from "@tangle-network/agent-profile-materialize";
|
|
8
9
|
import { access, chmod, copyFile, mkdir, mkdtemp, open, readFile, realpath, rm, symlink, writeFile } from "node:fs/promises";
|
|
9
|
-
import { constants, createReadStream } from "node:fs";
|
|
10
|
+
import { appendFileSync, constants, createReadStream } from "node:fs";
|
|
10
11
|
import { basename, delimiter, dirname, isAbsolute, join, resolve, sep, win32 } from "node:path";
|
|
11
12
|
import { spawn } from "node:child_process";
|
|
12
13
|
import { homedir, tmpdir } from "node:os";
|
|
@@ -1140,6 +1141,255 @@ function naturalNumber(value, field) {
|
|
|
1140
1141
|
return value;
|
|
1141
1142
|
}
|
|
1142
1143
|
//#endregion
|
|
1144
|
+
//#region src/agent/profile-materialization.ts
|
|
1145
|
+
const KNOWN_AXIS_SET = new Set(AGENT_PROFILE_MATERIALIZATION_AXES);
|
|
1146
|
+
/**
|
|
1147
|
+
* Compound AgentProfile properties and the canonical leaves they cover.
|
|
1148
|
+
*
|
|
1149
|
+
* agent-interface publishes two axis vocabularies, and this module has to sit between them:
|
|
1150
|
+
* `profileMaterializationAxes` emits canonical LEAVES, while `changedAgentProfileAxes` emits
|
|
1151
|
+
* DIFF axes, which are compound property names (`model`, `prompt`, `resources`, `identity`) for
|
|
1152
|
+
* everything except the already-scalar properties. A changed-axis input is therefore expanded
|
|
1153
|
+
* through this map, so both producers compose with this validator.
|
|
1154
|
+
*
|
|
1155
|
+
* A CONTRACT may still only name leaves. Expanding an input is safe — it asks about more, never
|
|
1156
|
+
* less — whereas letting a contract claim a parent is what silently swallows a dropped child.
|
|
1157
|
+
*/
|
|
1158
|
+
const compoundAxisLeaves = {
|
|
1159
|
+
identity: [
|
|
1160
|
+
"name",
|
|
1161
|
+
"description",
|
|
1162
|
+
"version",
|
|
1163
|
+
"tags"
|
|
1164
|
+
],
|
|
1165
|
+
prompt: ["systemPrompt", "instructions"],
|
|
1166
|
+
model: [
|
|
1167
|
+
"modelDefault",
|
|
1168
|
+
"modelSmall",
|
|
1169
|
+
"modelProvider",
|
|
1170
|
+
"modelReasoningEffort",
|
|
1171
|
+
"modelMetadata"
|
|
1172
|
+
],
|
|
1173
|
+
resources: [
|
|
1174
|
+
"files",
|
|
1175
|
+
"resourceTools",
|
|
1176
|
+
"skills",
|
|
1177
|
+
"resourceAgents",
|
|
1178
|
+
"commands",
|
|
1179
|
+
"resourceInstructions",
|
|
1180
|
+
"resourceFailOnError"
|
|
1181
|
+
],
|
|
1182
|
+
mcpConnections: ["mcp"]
|
|
1183
|
+
};
|
|
1184
|
+
/** Materialization contract for a run path that executes every canonical AgentProfile leaf. */
|
|
1185
|
+
const fullProfileMaterialization = defineProfileMaterializationContract({
|
|
1186
|
+
name: "full-profile-execution",
|
|
1187
|
+
axes: AGENT_PROFILE_MATERIALIZATION_AXES
|
|
1188
|
+
});
|
|
1189
|
+
/**
|
|
1190
|
+
* Materialization contract for an intentionally limited prompt-and-model execution path.
|
|
1191
|
+
* Identity, harness, and metadata are control fields consumed for naming, placement,
|
|
1192
|
+
* authorization, and durable attribution; they are carried without adding worker behavior.
|
|
1193
|
+
* Every behavioral axis other than prompt and model remains unsupported.
|
|
1194
|
+
*/
|
|
1195
|
+
const promptModelProfileMaterialization = defineProfileMaterializationContract({
|
|
1196
|
+
name: "prompt-model-execution",
|
|
1197
|
+
axes: [
|
|
1198
|
+
"name",
|
|
1199
|
+
"systemPrompt",
|
|
1200
|
+
"instructions",
|
|
1201
|
+
"modelDefault",
|
|
1202
|
+
"harness",
|
|
1203
|
+
"metadata"
|
|
1204
|
+
]
|
|
1205
|
+
});
|
|
1206
|
+
/**
|
|
1207
|
+
* Materialization contract for a local coding CLI in an isolated git worktree.
|
|
1208
|
+
* The shared workspace materializer carries native tools, permissions, MCP, hooks, subagents,
|
|
1209
|
+
* modes, and file-backed resources when the selected CLI supports their exact values.
|
|
1210
|
+
* `resourceFailOnError` is carried: it is the fail-closed policy the pre-worktree resource
|
|
1211
|
+
* RESOLUTION step (`resolveAgentProfileResources`) applies to remote profile resources. Runtime
|
|
1212
|
+
* placement concerns (hub connections and confidential execution), provider-native extensions,
|
|
1213
|
+
* and unused model hints are deliberately absent so they fail before a worktree or executor is
|
|
1214
|
+
* created rather than being mistaken for an effective candidate change.
|
|
1215
|
+
*/
|
|
1216
|
+
const worktreeCliProfileMaterialization = defineProfileMaterializationContract({
|
|
1217
|
+
name: "worktree-cli-execution",
|
|
1218
|
+
axes: [
|
|
1219
|
+
"name",
|
|
1220
|
+
"systemPrompt",
|
|
1221
|
+
"instructions",
|
|
1222
|
+
"modelDefault",
|
|
1223
|
+
"modelReasoningEffort",
|
|
1224
|
+
"harness",
|
|
1225
|
+
"permissions",
|
|
1226
|
+
"tools",
|
|
1227
|
+
"mcp",
|
|
1228
|
+
"subagents",
|
|
1229
|
+
"files",
|
|
1230
|
+
"resourceTools",
|
|
1231
|
+
"skills",
|
|
1232
|
+
"resourceAgents",
|
|
1233
|
+
"commands",
|
|
1234
|
+
"resourceInstructions",
|
|
1235
|
+
"resourceFailOnError",
|
|
1236
|
+
"hooks",
|
|
1237
|
+
"modes",
|
|
1238
|
+
"metadata"
|
|
1239
|
+
]
|
|
1240
|
+
});
|
|
1241
|
+
/** Materialization contract for a raw process path that carries only control/identity fields. */
|
|
1242
|
+
const controlProfileMaterialization = defineProfileMaterializationContract({
|
|
1243
|
+
name: "control-only-execution",
|
|
1244
|
+
axes: [
|
|
1245
|
+
"name",
|
|
1246
|
+
"harness",
|
|
1247
|
+
"metadata"
|
|
1248
|
+
]
|
|
1249
|
+
});
|
|
1250
|
+
/** Materialization contract for an injected inference function whose surrounding driver still
|
|
1251
|
+
* applies the profile prompt, name, placement, and metadata, but not model selection. */
|
|
1252
|
+
const promptControlProfileMaterialization = defineProfileMaterializationContract({
|
|
1253
|
+
name: "prompt-control-execution",
|
|
1254
|
+
axes: [
|
|
1255
|
+
"name",
|
|
1256
|
+
"systemPrompt",
|
|
1257
|
+
"instructions",
|
|
1258
|
+
"harness",
|
|
1259
|
+
"metadata"
|
|
1260
|
+
]
|
|
1261
|
+
});
|
|
1262
|
+
/**
|
|
1263
|
+
* Materialization contract for `createSandboxAct`.
|
|
1264
|
+
*
|
|
1265
|
+
* `createSandboxAct` hands the whole `AgentProfile` to the sandbox as `backend.profile`, so every
|
|
1266
|
+
* profile leaf crosses the boundary. `buildBackendOptions` resolves the runner from an explicit
|
|
1267
|
+
* `sandboxOverrides.backend.type`, then `profile.metadata.backendType`, then `profile.harness`,
|
|
1268
|
+
* so a candidate that changes only `harness` runs on the harness it declares — and one declaring
|
|
1269
|
+
* a harness the sandbox cannot run throws rather than running elsewhere and reporting success.
|
|
1270
|
+
*/
|
|
1271
|
+
const sandboxActProfileMaterialization = defineProfileMaterializationContract({
|
|
1272
|
+
name: "createSandboxAct",
|
|
1273
|
+
axes: fullProfileMaterialization.axes
|
|
1274
|
+
});
|
|
1275
|
+
/** Materialization contract for a run path that only injects prompt text. */
|
|
1276
|
+
const promptOnlyProfileMaterialization = defineProfileMaterializationContract({
|
|
1277
|
+
name: "prompt-only-message",
|
|
1278
|
+
axes: ["systemPrompt", "instructions"]
|
|
1279
|
+
});
|
|
1280
|
+
/**
|
|
1281
|
+
* Materialization contract for a run path that injects prompt text plus inline resources.
|
|
1282
|
+
*
|
|
1283
|
+
* `resourceFailOnError` is absent: it is a resolution POLICY the attaching path would have to
|
|
1284
|
+
* enforce, and inlining resource content does not carry it.
|
|
1285
|
+
*/
|
|
1286
|
+
const promptResourceProfileMaterialization = defineProfileMaterializationContract({
|
|
1287
|
+
name: "prompt-resource-attachment",
|
|
1288
|
+
axes: [
|
|
1289
|
+
"systemPrompt",
|
|
1290
|
+
"instructions",
|
|
1291
|
+
"files",
|
|
1292
|
+
"resourceTools",
|
|
1293
|
+
"skills",
|
|
1294
|
+
"resourceAgents",
|
|
1295
|
+
"commands",
|
|
1296
|
+
"resourceInstructions"
|
|
1297
|
+
]
|
|
1298
|
+
});
|
|
1299
|
+
/** Define the profile axes a concrete run path actually carries into execution. */
|
|
1300
|
+
function defineProfileMaterializationContract(options) {
|
|
1301
|
+
const name = options.name.trim();
|
|
1302
|
+
if (!name) throw new ValidationError("defineProfileMaterializationContract: name is required");
|
|
1303
|
+
return {
|
|
1304
|
+
name,
|
|
1305
|
+
axes: normalizeContractAxes(options.axes, `${name}.axes`)
|
|
1306
|
+
};
|
|
1307
|
+
}
|
|
1308
|
+
/** Return every changed profile axis that the selected run path would drop. */
|
|
1309
|
+
function validateProfileMaterialization(options) {
|
|
1310
|
+
const changedAxes = normalizeChangedAxes(options.changedAxes, "changedAxes");
|
|
1311
|
+
const supported = new Set(normalizeContractAxes(options.contract.axes, `${options.contract.name}.axes`));
|
|
1312
|
+
const issues = [];
|
|
1313
|
+
for (const axis of changedAxes) {
|
|
1314
|
+
if (isAxisSupported(axis, supported)) continue;
|
|
1315
|
+
issues.push({
|
|
1316
|
+
contract: options.contract.name,
|
|
1317
|
+
axis,
|
|
1318
|
+
reason: "unsupported-axis",
|
|
1319
|
+
supportedAxes: [...supported]
|
|
1320
|
+
});
|
|
1321
|
+
}
|
|
1322
|
+
return issues;
|
|
1323
|
+
}
|
|
1324
|
+
/** Throw when a candidate changes axes the selected run path cannot carry. */
|
|
1325
|
+
function assertProfileMaterialization(options) {
|
|
1326
|
+
const issues = validateProfileMaterialization(options);
|
|
1327
|
+
if (issues.length === 0) return;
|
|
1328
|
+
throw new ValidationError(renderProfileMaterializationIssues(issues, options.context));
|
|
1329
|
+
}
|
|
1330
|
+
/** Format profile-axis drop issues into a concise operator-facing error. */
|
|
1331
|
+
function renderProfileMaterializationIssues(issues, context) {
|
|
1332
|
+
if (issues.length === 0) return "";
|
|
1333
|
+
const contract = issues[0]?.contract ?? "<unknown>";
|
|
1334
|
+
const prefix = context ? `${context}: ` : "";
|
|
1335
|
+
const droppedAxes = issues.map((issue) => issue.axis).join(", ");
|
|
1336
|
+
const supportedAxes = issues[0]?.supportedAxes.join(", ") || "<none>";
|
|
1337
|
+
return [
|
|
1338
|
+
`${prefix}profile materialization would drop axis changes on "${contract}": ${droppedAxes}.`,
|
|
1339
|
+
`Supported axes: ${supportedAxes}.`,
|
|
1340
|
+
"Use a run path that carries those AgentProfile axes, or remove them from the candidate."
|
|
1341
|
+
].join("\n");
|
|
1342
|
+
}
|
|
1343
|
+
/** Contract side: leaves (or `custom:`) only. A parent claim would hide a dropped child. */
|
|
1344
|
+
function normalizeContractAxes(axes, label) {
|
|
1345
|
+
return dedupe(axes.map((raw) => assertLeafAxis(raw, label)));
|
|
1346
|
+
}
|
|
1347
|
+
/**
|
|
1348
|
+
* Input side: leaves, `custom:`, and compound diff axes, which expand to their leaves so
|
|
1349
|
+
* `changedAgentProfileAxes` output composes with a leaf-only contract.
|
|
1350
|
+
*/
|
|
1351
|
+
function normalizeChangedAxes(axes, label) {
|
|
1352
|
+
const out = [];
|
|
1353
|
+
for (const raw of axes) {
|
|
1354
|
+
const axis = readAxisName(raw, label);
|
|
1355
|
+
const leaves = compoundAxisLeaves[axis];
|
|
1356
|
+
if (leaves && !KNOWN_AXIS_SET.has(axis)) {
|
|
1357
|
+
out.push(...leaves);
|
|
1358
|
+
continue;
|
|
1359
|
+
}
|
|
1360
|
+
out.push(assertLeafAxis(raw, label));
|
|
1361
|
+
}
|
|
1362
|
+
return dedupe(out);
|
|
1363
|
+
}
|
|
1364
|
+
function dedupe(axes) {
|
|
1365
|
+
const out = [];
|
|
1366
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1367
|
+
for (const axis of axes) {
|
|
1368
|
+
if (seen.has(axis)) continue;
|
|
1369
|
+
seen.add(axis);
|
|
1370
|
+
out.push(axis);
|
|
1371
|
+
}
|
|
1372
|
+
return out;
|
|
1373
|
+
}
|
|
1374
|
+
function readAxisName(raw, label) {
|
|
1375
|
+
if (typeof raw !== "string") throw new ValidationError(`${label}: profile axis must be a string`);
|
|
1376
|
+
const axis = raw.trim();
|
|
1377
|
+
if (!axis) throw new ValidationError(`${label}: profile axis must be non-empty`);
|
|
1378
|
+
return axis;
|
|
1379
|
+
}
|
|
1380
|
+
function assertLeafAxis(raw, label) {
|
|
1381
|
+
const axis = readAxisName(raw, label);
|
|
1382
|
+
if (!KNOWN_AXIS_SET.has(axis) && !axis.startsWith("custom:")) {
|
|
1383
|
+
const leaves = compoundAxisLeaves[axis];
|
|
1384
|
+
if (leaves) throw new ValidationError(`${label}: "${axis}" is a compound AgentProfile property, not a materialization axis. Name the exact leaves this path carries: ${leaves.join(", ")}.`);
|
|
1385
|
+
throw new ValidationError(`${label}: unknown profile axis "${axis}". Use a known axis or custom:<name>.`);
|
|
1386
|
+
}
|
|
1387
|
+
return axis;
|
|
1388
|
+
}
|
|
1389
|
+
function isAxisSupported(axis, supported) {
|
|
1390
|
+
return supported.has(axis);
|
|
1391
|
+
}
|
|
1392
|
+
//#endregion
|
|
1143
1393
|
//#region src/mcp/worktree.ts
|
|
1144
1394
|
/**
|
|
1145
1395
|
*
|
|
@@ -1355,6 +1605,11 @@ async function runWorktreeHarness(opts) {
|
|
|
1355
1605
|
const runCommand = opts.runCommand ?? defaultRunCommand;
|
|
1356
1606
|
const checkTimeoutMs = opts.checkTimeoutMs ?? opts.harnessTimeoutMs ?? 300 * 1e3;
|
|
1357
1607
|
const cap = opts.checkOutputCap ?? defaultCheckOutputCap;
|
|
1608
|
+
assertSupportedWorktreeProfile(opts.profile, opts.harness);
|
|
1609
|
+
assertSafeProfileResourcePaths(opts.profile);
|
|
1610
|
+
const profile = await resolveAgentProfileResources(opts.profile, resourceResolutionOptions(opts.resourceResolution, opts.signal));
|
|
1611
|
+
opts.signal?.throwIfAborted();
|
|
1612
|
+
const { plan, resourceInstructions } = prepareWorktreeProfile(profile, opts.harness);
|
|
1358
1613
|
const worktree = await createWorktree({
|
|
1359
1614
|
repoRoot: opts.repoRoot,
|
|
1360
1615
|
runId: opts.runId,
|
|
@@ -1367,14 +1622,6 @@ async function runWorktreeHarness(opts) {
|
|
|
1367
1622
|
...opts.runGit ? { runGit: opts.runGit } : {}
|
|
1368
1623
|
});
|
|
1369
1624
|
try {
|
|
1370
|
-
assertSupportedWorktreeProfile(opts.profile, opts.harness);
|
|
1371
|
-
assertSafeProfileResourcePaths(opts.profile);
|
|
1372
|
-
const profile = await resolveAgentProfileResources(opts.profile, resourceResolutionOptions(opts.resourceResolution, opts.signal));
|
|
1373
|
-
opts.signal?.throwIfAborted();
|
|
1374
|
-
const resourceInstructions = resolveResourceInstructions(profile);
|
|
1375
|
-
const plan = materializeProfile(materializationOnlyProfile(profile), materializerHarness(opts.harness));
|
|
1376
|
-
if (plan.unsupported.length > 0) throw new Error(`runWorktreeHarness: profile cannot be materialized for ${opts.harness}: ${plan.unsupported.map(({ dimension, reason }) => `${dimension}: ${reason}`).join("; ")}`);
|
|
1377
|
-
assertSafeMaterializedPaths(plan);
|
|
1378
1625
|
const applied = applyWorkspacePlan(plan, worktree.path, { existingFiles: "reject" });
|
|
1379
1626
|
if (applied.unsupported.length > 0) throw new Error("runWorktreeHarness: applied profile unexpectedly retained unsupported rows");
|
|
1380
1627
|
const invocationProfile = profileWithResourceInstructions(profile, resourceInstructions);
|
|
@@ -1459,6 +1706,28 @@ async function runWorktreeHarness(opts) {
|
|
|
1459
1706
|
throw err;
|
|
1460
1707
|
}
|
|
1461
1708
|
}
|
|
1709
|
+
/** Compute the same plan identity `runWorktreeHarness` consumes, with no repository or process IO. */
|
|
1710
|
+
function worktreeProfileExecutionPlan(profile, harness) {
|
|
1711
|
+
const { plan, resourceInstructions } = prepareWorktreeProfile(profile, harness);
|
|
1712
|
+
return Object.freeze({
|
|
1713
|
+
workspacePlanDigest: hashWorkspacePlan(plan),
|
|
1714
|
+
harness: plan.harness,
|
|
1715
|
+
resourceInstructions: resourceInstructionReceipt(resourceInstructions)
|
|
1716
|
+
});
|
|
1717
|
+
}
|
|
1718
|
+
function prepareWorktreeProfile(profile, harness) {
|
|
1719
|
+
const resourceInstructions = resolveResourceInstructions(profile);
|
|
1720
|
+
const workspaceProfile = materializationOnlyProfile(profile);
|
|
1721
|
+
assertSupportedWorktreeProfile(profile, harness);
|
|
1722
|
+
assertSafeProfileResourcePaths(profile);
|
|
1723
|
+
const plan = materializeProfile(workspaceProfile, materializerHarness(harness));
|
|
1724
|
+
if (plan.unsupported.length > 0) throw new Error(`runWorktreeHarness: profile cannot be materialized for ${harness}: ${plan.unsupported.map(({ dimension, reason }) => `${dimension}: ${reason}`).join("; ")}`);
|
|
1725
|
+
assertSafeMaterializedPaths(plan);
|
|
1726
|
+
return {
|
|
1727
|
+
plan,
|
|
1728
|
+
resourceInstructions
|
|
1729
|
+
};
|
|
1730
|
+
}
|
|
1462
1731
|
function resourceResolutionOptions(options, signal) {
|
|
1463
1732
|
if (!signal) return options;
|
|
1464
1733
|
const fetchResource = options?.fetch ?? globalThis.fetch;
|
|
@@ -1509,37 +1778,15 @@ function materializationOnlyProfile(profile) {
|
|
|
1509
1778
|
};
|
|
1510
1779
|
}
|
|
1511
1780
|
function assertSupportedWorktreeProfile(profile, harness) {
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
hasEntries(profile.extensions) ? "extensions" : null
|
|
1519
|
-
].filter((axis) => axis !== null);
|
|
1781
|
+
assertProfileMaterialization({
|
|
1782
|
+
contract: worktreeCliProfileMaterialization,
|
|
1783
|
+
changedAxes: profileMaterializationAxes(profile),
|
|
1784
|
+
context: "runWorktreeHarness"
|
|
1785
|
+
});
|
|
1786
|
+
const unsupportedAxes = [];
|
|
1520
1787
|
if (profile.model?.reasoningEffort !== void 0 && harness !== "codex") unsupportedAxes.push("model.reasoningEffort");
|
|
1521
|
-
for (const [name, server] of Object.entries(profile.mcp ?? {})) {
|
|
1522
|
-
const path = `mcp[${JSON.stringify(name)}]`;
|
|
1523
|
-
if (server.enabled === false && harness !== "opencode") unsupportedAxes.push(`${path}.enabled`);
|
|
1524
|
-
if (hasEntries(server.headers) && harness === "codex") unsupportedAxes.push(`${path}.headers`);
|
|
1525
|
-
if (server.cwd !== void 0 && harness === "opencode") unsupportedAxes.push(`${path}.cwd`);
|
|
1526
|
-
}
|
|
1527
|
-
if (harness === "claude") for (const [event, commands] of Object.entries(profile.hooks ?? {})) for (const [index, command] of commands.entries()) {
|
|
1528
|
-
const path = `hooks[${JSON.stringify(event)}][${index}]`;
|
|
1529
|
-
if (hasEntries(command.env)) unsupportedAxes.push(`${path}.env`);
|
|
1530
|
-
if (command.blocking !== void 0) unsupportedAxes.push(`${path}.blocking`);
|
|
1531
|
-
}
|
|
1532
|
-
for (const [name, subagent] of Object.entries(profile.subagents ?? {})) {
|
|
1533
|
-
const path = `subagents[${JSON.stringify(name)}]`;
|
|
1534
|
-
if (hasEntries(subagent.permissions)) unsupportedAxes.push(`${path}.permissions`);
|
|
1535
|
-
if (subagent.maxSteps !== void 0) unsupportedAxes.push(`${path}.maxSteps`);
|
|
1536
|
-
if (hasEntries(subagent.tools) && harness !== "claude") unsupportedAxes.push(`${path}.tools`);
|
|
1537
|
-
}
|
|
1538
1788
|
if (unsupportedAxes.length > 0) throw new Error(`runWorktreeHarness: profile requests unsupported worktree behavior: ${unsupportedAxes.join(", ")}`);
|
|
1539
1789
|
}
|
|
1540
|
-
function hasEntries(value) {
|
|
1541
|
-
return value !== void 0 && Object.keys(value).length > 0;
|
|
1542
|
-
}
|
|
1543
1790
|
function assertSafeMaterializedPaths(plan) {
|
|
1544
1791
|
for (const file of plan.files) if (file.relPath.split("/").some(isGitMetadataSegment)) throw new Error(`runWorktreeHarness: profile file cannot target reserved Git metadata: ${file.relPath}`);
|
|
1545
1792
|
}
|
|
@@ -1551,22 +1798,26 @@ function isGitMetadataSegment(segment) {
|
|
|
1551
1798
|
return windowsCanonical === ".git" || windowsCanonical.startsWith(".git:");
|
|
1552
1799
|
}
|
|
1553
1800
|
function profileMaterializationReceipt(applied, resourceInstructions) {
|
|
1554
|
-
const instructionBytes = resourceInstructions === void 0 ? null : Buffer.from(resourceInstructions, "utf8");
|
|
1555
1801
|
return {
|
|
1556
1802
|
workspacePlanDigest: applied.workspacePlanDigest,
|
|
1557
1803
|
writtenPaths: [...applied.written],
|
|
1558
1804
|
unsupported: [...applied.unsupported],
|
|
1559
1805
|
environmentNames: Object.keys(applied.env).sort(),
|
|
1560
1806
|
flags: applied.flags.map(publicPlanString),
|
|
1561
|
-
resourceInstructions:
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1807
|
+
resourceInstructions: resourceInstructionReceipt(resourceInstructions)
|
|
1808
|
+
};
|
|
1809
|
+
}
|
|
1810
|
+
function resourceInstructionReceipt(resourceInstructions) {
|
|
1811
|
+
if (resourceInstructions === void 0) return {
|
|
1812
|
+
delivery: "none",
|
|
1813
|
+
sha256: null,
|
|
1814
|
+
byteLength: 0
|
|
1815
|
+
};
|
|
1816
|
+
const instructionBytes = Buffer.from(resourceInstructions, "utf8");
|
|
1817
|
+
return {
|
|
1818
|
+
delivery: "invocation-prompt",
|
|
1819
|
+
sha256: `sha256:${createHash("sha256").update(instructionBytes).digest("hex")}`,
|
|
1820
|
+
byteLength: instructionBytes.byteLength
|
|
1570
1821
|
};
|
|
1571
1822
|
}
|
|
1572
1823
|
/** Run the configured test + typecheck commands in the live worktree, projecting exit codes into
|
|
@@ -1918,7 +2169,7 @@ function splitReasoning(rawContent, fieldReasoning) {
|
|
|
1918
2169
|
*/
|
|
1919
2170
|
async function routerChatWithTools(cfg, messages, tools, opts) {
|
|
1920
2171
|
const body = toolCompletionBody(cfg, messages, tools, opts);
|
|
1921
|
-
const data = cfg.complete ? await cfg.complete(body) : await (async () => {
|
|
2172
|
+
const data = cfg.complete ? await cfg.complete(structuredClone(body)) : await (async () => {
|
|
1922
2173
|
const res = await fetch(`${cfg.routerBaseUrl.replace(/\/$/, "")}/chat/completions`, {
|
|
1923
2174
|
method: "POST",
|
|
1924
2175
|
headers: {
|
|
@@ -2209,261 +2460,1006 @@ function routerBrain(cfg, opts = {}) {
|
|
|
2209
2460
|
});
|
|
2210
2461
|
}
|
|
2211
2462
|
//#endregion
|
|
2212
|
-
//#region src/runtime/supervise/
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2463
|
+
//#region src/runtime/supervise/materialization.ts
|
|
2464
|
+
const runtimeOwnedExecutorMaterializations = /* @__PURE__ */ new WeakMap();
|
|
2465
|
+
const runtimeOwnedScopeOwners = /* @__PURE__ */ new WeakMap();
|
|
2466
|
+
const declarationKeys = /* @__PURE__ */ new Set([
|
|
2467
|
+
"effectiveProfile",
|
|
2468
|
+
"backend",
|
|
2469
|
+
"model",
|
|
2470
|
+
"execution",
|
|
2471
|
+
"materializer",
|
|
2472
|
+
"plan",
|
|
2473
|
+
"platformAttachments"
|
|
2474
|
+
]);
|
|
2475
|
+
/**
|
|
2476
|
+
* Bind a declaration to a runtime-owned executor without putting an attestable method on the
|
|
2477
|
+
* public Executor interface. This module is not a package export: arbitrary caller executors stay
|
|
2478
|
+
* unknown even if they add a lookalike `materialization()` property.
|
|
2479
|
+
*/
|
|
2480
|
+
function attestRuntimeOwnedExecutor(executor, declaration, binding) {
|
|
2481
|
+
runtimeOwnedExecutorMaterializations.set(executor, Object.freeze({
|
|
2482
|
+
kind: "known",
|
|
2483
|
+
declaration: detachedSnapshot(declaration, "runtime-owned executor materialization"),
|
|
2484
|
+
...binding === void 0 ? {} : { binding: detachedSnapshot(binding, "runtime-owned executor execution binding") }
|
|
2485
|
+
}));
|
|
2486
|
+
return executor;
|
|
2487
|
+
}
|
|
2488
|
+
/** Mark a runtime-owned orchestration executor whose exact leaf declaration is published through
|
|
2489
|
+
* its private nested Scope before the first backend inference. Arbitrary executors cannot opt in. */
|
|
2490
|
+
function attestRuntimeOwnedDeferredExecutor(executor, runtime) {
|
|
2491
|
+
runtimeOwnedExecutorMaterializations.set(executor, Object.freeze({
|
|
2492
|
+
kind: "deferred",
|
|
2493
|
+
runtime
|
|
2494
|
+
}));
|
|
2495
|
+
return executor;
|
|
2496
|
+
}
|
|
2497
|
+
/** Preserve the runtime-owned attestation when a trusted wrapper changes result semantics only. */
|
|
2498
|
+
function inheritRuntimeOwnedExecutorAttestation(source, wrapper) {
|
|
2499
|
+
const attestation = runtimeOwnedExecutorMaterializations.get(source);
|
|
2500
|
+
if (attestation !== void 0) runtimeOwnedExecutorMaterializations.set(wrapper, attestation);
|
|
2501
|
+
return wrapper;
|
|
2502
|
+
}
|
|
2503
|
+
/** Read a declaration only when Runtime itself branded this exact executor object. */
|
|
2504
|
+
function runtimeOwnedExecutorMaterialization(executor) {
|
|
2505
|
+
const attestation = runtimeOwnedExecutorMaterializations.get(executor);
|
|
2506
|
+
return attestation?.kind === "known" ? attestation.declaration : void 0;
|
|
2507
|
+
}
|
|
2508
|
+
/** Read one attempt binding only from the private brand on this exact executor object. */
|
|
2509
|
+
function runtimeOwnedExecutorExecutionBinding(executor) {
|
|
2510
|
+
const attestation = runtimeOwnedExecutorMaterializations.get(executor);
|
|
2511
|
+
return attestation?.kind === "known" ? attestation.binding : void 0;
|
|
2512
|
+
}
|
|
2513
|
+
/** Read the expected leaf runtime only for a privately branded deferred orchestration executor. */
|
|
2514
|
+
function runtimeOwnedDeferredExecutorRuntime(executor) {
|
|
2515
|
+
const attestation = runtimeOwnedExecutorMaterializations.get(executor);
|
|
2516
|
+
return attestation?.kind === "deferred" ? attestation.runtime : void 0;
|
|
2517
|
+
}
|
|
2518
|
+
/** Propagate trust from a built-in driver adapter to the Agent and recursive executor it owns. */
|
|
2519
|
+
function attestRuntimeOwnedScopeOwner(owner, runtime) {
|
|
2520
|
+
runtimeOwnedScopeOwners.set(owner, runtime);
|
|
2521
|
+
return owner;
|
|
2522
|
+
}
|
|
2523
|
+
/** Return a built-in scope owner's expected concrete leaf runtime; caller-owned functions/Agents
|
|
2524
|
+
* deliberately have no entry even if they add lookalike fields. */
|
|
2525
|
+
function runtimeOwnedScopeOwnerRuntime(owner) {
|
|
2526
|
+
return runtimeOwnedScopeOwners.get(owner);
|
|
2527
|
+
}
|
|
2528
|
+
/** Build one kernel-owned known receipt from a data-only executor declaration. */
|
|
2529
|
+
function knownMaterializationReceipt(input) {
|
|
2530
|
+
const declaration = detachedSnapshot(input.declaration, "executor materialization");
|
|
2531
|
+
assertExactDeclaration(declaration);
|
|
2532
|
+
const effectiveProfile = agentProfileSchema.safeParse(declaration.effectiveProfile);
|
|
2533
|
+
if (!effectiveProfile.success) throw new ValidationError("executor materialization: effectiveProfile must be an AgentProfile");
|
|
2534
|
+
assertNonEmpty(declaration.backend, "backend");
|
|
2535
|
+
assertNonEmpty(declaration.materializer, "materializer");
|
|
2536
|
+
assertNonEmpty(declaration.execution?.kind, "execution.kind");
|
|
2537
|
+
assertNonEmpty(declaration.execution?.id, "execution.id");
|
|
2538
|
+
assertModel(declaration.model);
|
|
2539
|
+
let effectiveProfileDigest;
|
|
2540
|
+
let materializationPlanDigest;
|
|
2541
|
+
let platformAttachmentsDigest;
|
|
2542
|
+
try {
|
|
2543
|
+
effectiveProfileDigest = canonicalCandidateDigest(jsonWireSnapshot(effectiveProfile.data));
|
|
2544
|
+
materializationPlanDigest = canonicalCandidateDigest(jsonWireSnapshot(declaration.plan));
|
|
2545
|
+
platformAttachmentsDigest = declaration.platformAttachments === void 0 ? void 0 : canonicalCandidateDigest(jsonWireSnapshot(declaration.platformAttachments));
|
|
2546
|
+
} catch (error) {
|
|
2547
|
+
throw new ValidationError("executor materialization: profile, plan, and attachments must be finite RFC 8785 JSON", { cause: error });
|
|
2548
|
+
}
|
|
2549
|
+
return Object.freeze({
|
|
2550
|
+
status: "known",
|
|
2551
|
+
authoredProfileDigest: input.authoredProfileDigest,
|
|
2552
|
+
effectiveProfileDigest,
|
|
2553
|
+
materializationPlanDigest,
|
|
2554
|
+
...platformAttachmentsDigest === void 0 ? {} : { platformAttachmentsDigest },
|
|
2555
|
+
runtime: input.runtime,
|
|
2556
|
+
backend: declaration.backend,
|
|
2557
|
+
model: declaration.model,
|
|
2558
|
+
execution: declaration.execution,
|
|
2559
|
+
materializer: declaration.materializer
|
|
2560
|
+
});
|
|
2230
2561
|
}
|
|
2231
|
-
/**
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2562
|
+
/** Build an explicit unknown receipt without fabricating any executor-owned identity. */
|
|
2563
|
+
function unknownMaterializationReceipt(input) {
|
|
2564
|
+
return Object.freeze({
|
|
2565
|
+
status: "unknown",
|
|
2566
|
+
...input.authoredProfileDigest === void 0 ? {} : { authoredProfileDigest: input.authoredProfileDigest },
|
|
2567
|
+
runtime: input.runtime,
|
|
2568
|
+
reason: input.reason
|
|
2569
|
+
});
|
|
2570
|
+
}
|
|
2571
|
+
/** Mint a process-unique attempt id before executor construction. */
|
|
2572
|
+
function newExecutionAttemptId(nodeId) {
|
|
2573
|
+
return `${nodeId}:attempt:${randomUUID()}`;
|
|
2574
|
+
}
|
|
2575
|
+
/** Build the immutable safe receipt for a full, transient execution binding. */
|
|
2576
|
+
function knownExecutionBindingReceipt(materialization, bindingInput) {
|
|
2577
|
+
const binding = detachedSnapshot(bindingInput, "executor execution binding");
|
|
2578
|
+
assertNonEmpty(binding.attemptId, "binding.attemptId");
|
|
2579
|
+
assertSafeDescriptor(binding.descriptor);
|
|
2580
|
+
let materializationReceiptDigest;
|
|
2581
|
+
let bindingDigest;
|
|
2582
|
+
try {
|
|
2583
|
+
materializationReceiptDigest = canonicalCandidateDigest(materialization);
|
|
2584
|
+
bindingDigest = canonicalCandidateDigest(jsonWireSnapshot(binding.binding));
|
|
2585
|
+
} catch (error) {
|
|
2586
|
+
throw new ValidationError("executor binding must be finite RFC 8785 JSON", { cause: error });
|
|
2587
|
+
}
|
|
2588
|
+
return Object.freeze({
|
|
2589
|
+
status: "known",
|
|
2590
|
+
attemptId: binding.attemptId,
|
|
2591
|
+
materializationReceiptDigest,
|
|
2592
|
+
bindingDigest,
|
|
2593
|
+
descriptor: binding.descriptor
|
|
2594
|
+
});
|
|
2595
|
+
}
|
|
2596
|
+
/** Preserve an attempt boundary even when its caller-owned transport cannot be attested. */
|
|
2597
|
+
function unknownExecutionBindingReceipt(materialization, attemptId, reason) {
|
|
2598
|
+
assertNonEmpty(attemptId, "binding.attemptId");
|
|
2599
|
+
return Object.freeze({
|
|
2600
|
+
status: "unknown",
|
|
2601
|
+
attemptId,
|
|
2602
|
+
materializationReceiptDigest: canonicalCandidateDigest(materialization),
|
|
2603
|
+
reason
|
|
2604
|
+
});
|
|
2605
|
+
}
|
|
2606
|
+
/** Derive an authored-profile digest without turning non-canonical input into a fake identity. */
|
|
2607
|
+
function authoredProfileDigest(profile) {
|
|
2608
|
+
try {
|
|
2609
|
+
return canonicalCandidateDigest(profile);
|
|
2610
|
+
} catch {
|
|
2611
|
+
return;
|
|
2612
|
+
}
|
|
2613
|
+
}
|
|
2614
|
+
function assertExactDeclaration(value) {
|
|
2615
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new ValidationError("executor materialization: declaration must be an object");
|
|
2616
|
+
const unknown = Object.keys(value).filter((key) => !declarationKeys.has(key));
|
|
2617
|
+
if (unknown.length > 0) throw new ValidationError(`executor materialization: unknown declaration fields: ${unknown.sort().join(", ")}`);
|
|
2618
|
+
if (!Object.hasOwn(value, "plan")) throw new ValidationError("executor materialization: plan is required");
|
|
2619
|
+
}
|
|
2620
|
+
function assertModel(model) {
|
|
2621
|
+
if (typeof model !== "object" || model === null || Array.isArray(model)) throw new ValidationError("executor materialization: model must be a known/unknown identity");
|
|
2622
|
+
const keys = Object.keys(model).sort();
|
|
2623
|
+
if (model.status === "known") {
|
|
2624
|
+
if (keys.join(",") !== "id,status") throw new ValidationError("executor materialization: known model accepts only status and id");
|
|
2625
|
+
assertNonEmpty(model.id, "model.id");
|
|
2626
|
+
return;
|
|
2627
|
+
}
|
|
2628
|
+
if (model.status === "unknown") {
|
|
2629
|
+
if (keys.join(",") !== "reason,status") throw new ValidationError("executor materialization: unknown model accepts only status and reason");
|
|
2630
|
+
assertNonEmpty(model.reason, "model.reason");
|
|
2631
|
+
return;
|
|
2632
|
+
}
|
|
2633
|
+
throw new ValidationError("executor materialization: model.status must be known or unknown");
|
|
2634
|
+
}
|
|
2635
|
+
function assertNonEmpty(value, field) {
|
|
2636
|
+
if (typeof value !== "string" || value.trim().length === 0) throw new ValidationError(`executor materialization: ${field} must be a non-empty string`);
|
|
2637
|
+
}
|
|
2638
|
+
function assertSafeDescriptor(descriptor) {
|
|
2639
|
+
if (typeof descriptor !== "object" || descriptor === null || Array.isArray(descriptor)) throw new ValidationError("executor binding: descriptor must be an object");
|
|
2640
|
+
for (const [key, value] of Object.entries(descriptor)) {
|
|
2641
|
+
if (/(url|uri|token|key|secret|bearer|credential|authorization)/i.test(key)) throw new ValidationError(`executor binding: unsafe descriptor key ${JSON.stringify(key)}`);
|
|
2642
|
+
if (value !== null && typeof value !== "string" && typeof value !== "number" && typeof value !== "boolean") throw new ValidationError(`executor binding: descriptor ${JSON.stringify(key)} must be scalar`);
|
|
2643
|
+
if (typeof value === "string" && (value.includes("://") || value.includes("@"))) throw new ValidationError(`executor binding: descriptor ${JSON.stringify(key)} contains transport or credential text`);
|
|
2644
|
+
}
|
|
2645
|
+
}
|
|
2646
|
+
function jsonWireSnapshot(value, path = "$") {
|
|
2647
|
+
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
|
2648
|
+
if (typeof value === "number") {
|
|
2649
|
+
if (!Number.isFinite(value)) throw new ValidationError(`executor binding: ${path} must be finite`);
|
|
2650
|
+
return value;
|
|
2651
|
+
}
|
|
2652
|
+
if (Array.isArray(value)) return value.map((item, index) => {
|
|
2653
|
+
if (item === void 0) throw new ValidationError(`executor binding: ${path}[${index}] cannot be undefined`);
|
|
2654
|
+
return jsonWireSnapshot(item, `${path}[${index}]`);
|
|
2655
|
+
});
|
|
2656
|
+
if (typeof value !== "object" || value === null) throw new ValidationError(`executor binding: ${path} must be JSON data`);
|
|
2657
|
+
const prototype = Object.getPrototypeOf(value);
|
|
2658
|
+
if (prototype !== Object.prototype && prototype !== null) throw new ValidationError(`executor binding: ${path} must be a plain object`);
|
|
2659
|
+
const out = {};
|
|
2660
|
+
for (const [key, item] of Object.entries(value)) {
|
|
2661
|
+
if (item === void 0) continue;
|
|
2662
|
+
out[key] = jsonWireSnapshot(item, `${path}.${key}`);
|
|
2663
|
+
}
|
|
2664
|
+
return out;
|
|
2259
2665
|
}
|
|
2260
2666
|
//#endregion
|
|
2261
|
-
//#region src/runtime/supervise/
|
|
2667
|
+
//#region src/runtime/supervise/budget.ts
|
|
2262
2668
|
/**
|
|
2263
2669
|
*
|
|
2264
|
-
*
|
|
2265
|
-
*
|
|
2266
|
-
*
|
|
2267
|
-
*
|
|
2268
|
-
* for a nightly job to land, for a human to reply, for a rate-limit window to reopen. Before
|
|
2269
|
-
* this, the only way to express that was to keep something alive and re-ask — which spends
|
|
2270
|
-
* driver tokens per re-ask and pins a process to the wait.
|
|
2271
|
-
*
|
|
2272
|
-
* Two node kinds, both first-class:
|
|
2273
|
-
* - `timer` — wake at an ABSOLUTE wall-clock instant (`untilMs`).
|
|
2274
|
-
* - `poll` — re-run a named predicate every `intervalMs` until it returns true, or until an
|
|
2275
|
-
* absolute `timeoutAtMs` passes (CI status, file existence, an HTTP probe, an
|
|
2276
|
-
* inbox message).
|
|
2277
|
-
*
|
|
2278
|
-
* ── How this differs from `await_event`'s 15s poll fence (`DEFAULT_AWAIT_EVENT_TIMEOUT_MS`) ──
|
|
2279
|
-
*
|
|
2280
|
-
* They look similar and are not the same mechanism. `await_event` is an IN-RUN RENDEZVOUS: the
|
|
2281
|
-
* driver blocks on the coordination bus for the next event from a live worker, and the 15s fence
|
|
2282
|
-
* exists only so a remote MCP request does not exceed the client's timeout — the caller re-polls.
|
|
2283
|
-
* Every re-poll is another driver inference turn (real tokens) against a process that must stay
|
|
2284
|
-
* up, and nothing about that wait is recorded: kill the process and the wait is simply gone.
|
|
2670
|
+
* The conserved budget reservation pool — the invariant the whole instrument
|
|
2671
|
+
* rests on (critique M5/B3). One root `Budget` becomes a conserved pool of three
|
|
2672
|
+
* quantities (tokens, usd, iterations) plus an absolute deadline. Children reserve
|
|
2673
|
+
* atomically at spawn and reconcile at settle:
|
|
2285
2674
|
*
|
|
2286
|
-
*
|
|
2287
|
-
* ABSOLUTE deadline, and it settles through the same `Scope.next()` cursor as any worker. Nobody
|
|
2288
|
-
* is blocked on it — the driver can stop reasoning entirely, and the process can die. Cost while
|
|
2289
|
-
* waiting: zero LLM calls, zero executor, zero sandbox, zero conserved budget (a wait reserves
|
|
2290
|
-
* nothing from the pool). The only in-process residue is one timer entry.
|
|
2675
|
+
* total ≡ free + reserved + committed (for every known quantity)
|
|
2291
2676
|
*
|
|
2292
|
-
*
|
|
2677
|
+
* `reserve` moves a child's whole ceiling from `free` → `reserved` and fails closed
|
|
2678
|
+
* when `free` can't cover it (never read-then-spawn overcommit, so `Σk(treatment) ≡
|
|
2679
|
+
* Σk(blind)` by construction). `reconcile` releases the reservation, commits ACTUAL
|
|
2680
|
+
* spend, and refunds the unspent remainder to `free`. Tokens and usd are separate
|
|
2681
|
+
* channels (`LoopTokenUsage` has no `usd`); iterations are conserved alongside them.
|
|
2293
2682
|
*
|
|
2294
|
-
*
|
|
2295
|
-
*
|
|
2296
|
-
*
|
|
2297
|
-
*
|
|
2298
|
-
*
|
|
2683
|
+
* Pure and deterministic: `now()` is injected, there is no I/O, and no wall-clock or
|
|
2684
|
+
* RNG read. A `reserve`/`reconcile` ticket is single-use (fail-loud on double or
|
|
2685
|
+
* unknown reconcile) so a child can never refund twice. Reconciling an OPEN ticket always
|
|
2686
|
+
* closes it: a fail-loud condition settles the reservation first and throws afterwards, so no
|
|
2687
|
+
* error path can strand a reservation past the join barrier's `assertNoOpenTickets`.
|
|
2688
|
+
* A child that declared no `maxUsd` reserved no dollar allocation, so its real dollars are
|
|
2689
|
+
* committed as observed spend and debited from the root's balance rather than treated as an
|
|
2690
|
+
* overspend of a $0 ceiling it never asked for. If dollar cost is unknowable under a
|
|
2691
|
+
* dollar limit, reconciliation closes the known token/iteration work, marks the dollar channel
|
|
2692
|
+
* unusable, and refuses later reservations; inventing a numeric dollar total would be worse.
|
|
2693
|
+
* If a TOKEN count is unknowable (a provider that reported no usage for work that ran), the pool
|
|
2694
|
+
* records the work, debits what it knows, and marks `readout().tokensKnown` false — the balance is
|
|
2695
|
+
* then a ceiling, not a measurement. It does not close admission the way the dollar channel does:
|
|
2696
|
+
* tokens are always capped, so one unreported turn must not end the run.
|
|
2299
2697
|
*
|
|
2300
|
-
*
|
|
2301
|
-
*
|
|
2302
|
-
*
|
|
2698
|
+
* A pool can be RESTORED from a prior process's durable record (same-host restart): measured
|
|
2699
|
+
* committed spend is debited exactly once, and each child journaled as started but never settled
|
|
2700
|
+
* is charged at its full declared ceiling — restart never mints capacity, and the unknown flags
|
|
2701
|
+
* stay visible in every later readout. The original absolute deadline never slides.
|
|
2303
2702
|
*
|
|
2304
2703
|
* @experimental
|
|
2305
2704
|
*/
|
|
2306
|
-
/**
|
|
2307
|
-
|
|
2308
|
-
|
|
2705
|
+
/** Reject malformed ceilings before they can mint capacity through negative reservations. */
|
|
2706
|
+
function assertValidBudget(budget, label = "budget") {
|
|
2707
|
+
const safeInteger = (value, field) => {
|
|
2708
|
+
if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${label}.${field} must be a non-negative safe integer`);
|
|
2709
|
+
};
|
|
2710
|
+
const finiteNonNegative = (value, field) => {
|
|
2711
|
+
if (!Number.isFinite(value) || value < 0) throw new Error(`${label}.${field} must be a non-negative finite number`);
|
|
2712
|
+
};
|
|
2713
|
+
safeInteger(budget.maxIterations, "maxIterations");
|
|
2714
|
+
safeInteger(budget.maxTokens, "maxTokens");
|
|
2715
|
+
if (budget.maxUsd !== void 0) finiteNonNegative(budget.maxUsd, "maxUsd");
|
|
2716
|
+
if (budget.deadlineMs !== void 0) finiteNonNegative(budget.deadlineMs, "deadlineMs");
|
|
2717
|
+
}
|
|
2718
|
+
function assertValidSpend(spend, label) {
|
|
2719
|
+
if (!Number.isSafeInteger(spend.iterations) || spend.iterations < 0) throw new Error(`${label}.iterations must be a non-negative safe integer`);
|
|
2720
|
+
for (const [field, value] of Object.entries(spend.tokens)) if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${label}.tokens.${field} must be a non-negative safe integer`);
|
|
2721
|
+
for (const [field, value] of [["usd", spend.usd], ["ms", spend.ms]]) if (!Number.isFinite(value) || value < 0) throw new Error(`${label}.${field} must be a non-negative finite number`);
|
|
2722
|
+
}
|
|
2723
|
+
/** Fold a normalized `UsageEvent` array into a `Spend`. Tokens and usd are separate
|
|
2724
|
+
* channels; iterations come from `'iteration'` events. Pure; `ms` stays zero (the
|
|
2725
|
+
* pool does not read wall-clock). */
|
|
2726
|
+
function spendFromUsageEvents(events) {
|
|
2727
|
+
const tokens = zeroTokenUsage();
|
|
2728
|
+
let usd = 0;
|
|
2729
|
+
let usdKnown = true;
|
|
2730
|
+
let iterations = 0;
|
|
2731
|
+
for (const ev of events) if (ev.kind === "tokens") addTokenUsage(tokens, {
|
|
2732
|
+
input: ev.input,
|
|
2733
|
+
output: ev.output
|
|
2734
|
+
});
|
|
2735
|
+
else if (ev.kind === "cost") {
|
|
2736
|
+
usd += ev.usd;
|
|
2737
|
+
if (ev.usdKnown === false) usdKnown = false;
|
|
2738
|
+
} else iterations += 1;
|
|
2309
2739
|
return {
|
|
2310
|
-
|
|
2311
|
-
|
|
2740
|
+
iterations,
|
|
2741
|
+
tokens,
|
|
2742
|
+
usd,
|
|
2743
|
+
...usdKnown ? {} : { usdKnown: false },
|
|
2744
|
+
ms: 0
|
|
2312
2745
|
};
|
|
2313
2746
|
}
|
|
2314
|
-
|
|
2315
|
-
|
|
2747
|
+
async function foldUsage(events) {
|
|
2748
|
+
if (Array.isArray(events)) return spendFromUsageEvents(events);
|
|
2749
|
+
const tokens = zeroTokenUsage();
|
|
2750
|
+
let usd = 0;
|
|
2751
|
+
let usdKnown = true;
|
|
2752
|
+
let iterations = 0;
|
|
2753
|
+
for await (const ev of events) if (ev.kind === "tokens") addTokenUsage(tokens, {
|
|
2754
|
+
input: ev.input,
|
|
2755
|
+
output: ev.output
|
|
2756
|
+
});
|
|
2757
|
+
else if (ev.kind === "cost") {
|
|
2758
|
+
usd += ev.usd;
|
|
2759
|
+
if (ev.usdKnown === false) usdKnown = false;
|
|
2760
|
+
} else iterations += 1;
|
|
2316
2761
|
return {
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
...
|
|
2321
|
-
|
|
2762
|
+
iterations,
|
|
2763
|
+
tokens,
|
|
2764
|
+
usd,
|
|
2765
|
+
...usdKnown ? {} : { usdKnown: false },
|
|
2766
|
+
ms: 0
|
|
2322
2767
|
};
|
|
2323
2768
|
}
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
const map = new Map(Object.entries(entries));
|
|
2327
|
-
return { resolve: (name) => map.get(name) };
|
|
2328
|
-
}
|
|
2329
|
-
/** Narrow a settlement's `out` to a wait outcome — a wait settles on the SAME cursor as workers,
|
|
2330
|
-
* so a driver that mixes them tags them apart with this. */
|
|
2331
|
-
function isWaitOutcome(value) {
|
|
2332
|
-
return typeof value === "object" && value !== null && value.waitOutcome === true;
|
|
2333
|
-
}
|
|
2334
|
-
/** The absolute instant a spec is bounded by, or `undefined` for an unbounded poll. */
|
|
2335
|
-
function waitUntil(spec) {
|
|
2336
|
-
return spec.kind === "timer" ? spec.untilMs : spec.timeoutAtMs;
|
|
2337
|
-
}
|
|
2338
|
-
/** Structural validation, independent of the run. Returns null when the spec is usable. */
|
|
2339
|
-
function validateWaitSpec(spec) {
|
|
2340
|
-
if (spec.kind === "timer") return Number.isFinite(spec.untilMs) ? null : "timer.untilMs must be a finite epoch-ms instant";
|
|
2341
|
-
if (typeof spec.probe !== "string" || spec.probe.length === 0) return "poll.probe must be a non-empty probe name";
|
|
2342
|
-
if (!Number.isFinite(spec.intervalMs) || spec.intervalMs <= 0) return "poll.intervalMs must be > 0";
|
|
2343
|
-
if (spec.timeoutAtMs !== void 0 && !Number.isFinite(spec.timeoutAtMs)) return "poll.timeoutAtMs must be a finite epoch-ms instant";
|
|
2344
|
-
return null;
|
|
2345
|
-
}
|
|
2346
|
-
/**
|
|
2347
|
-
* A wait must not silently outlive the run's HARD wall-clock ceiling. When the conserved pool
|
|
2348
|
-
* carries an absolute `deadlineMs`, a wait bounded past it — or an unbounded poll — is REFUSED at
|
|
2349
|
-
* arm time (`deadline-exceeded`) rather than sleeping through a ceiling the caller already set.
|
|
2350
|
-
* The alternative (extending the pool's deadline to cover the wait) would let a wait override a
|
|
2351
|
-
* hard budget guard, which is exactly what mechanic D forbids; a run that intends week-long waits
|
|
2352
|
-
* either sets a deadline that covers them or sets none.
|
|
2353
|
-
*/
|
|
2354
|
-
function assertWaitWithinDeadline(spec, deadlineMs) {
|
|
2355
|
-
if (deadlineMs <= 0) return true;
|
|
2356
|
-
const until = waitUntil(spec);
|
|
2357
|
-
if (until === void 0) return false;
|
|
2358
|
-
return until <= deadlineMs;
|
|
2769
|
+
function totalTokens(usage) {
|
|
2770
|
+
return usage.input + usage.output;
|
|
2359
2771
|
}
|
|
2360
2772
|
/**
|
|
2361
|
-
*
|
|
2362
|
-
*
|
|
2773
|
+
* Create a conserved reservation pool from a root `Budget`. `now()` is injected so the
|
|
2774
|
+
* deadline readout is deterministic; defaults to `Date.now` for non-test callers. The
|
|
2775
|
+
* absolute deadline for a fresh pool is fixed at construction (`now() + budget.deadlineMs`). A
|
|
2776
|
+
* restored pool instead retains `restore.absoluteDeadlineMs`, so restart never slides the original
|
|
2777
|
+
* wall-clock limit. The readout is an absolute instant, not a shrinking remainder.
|
|
2363
2778
|
*/
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2779
|
+
function createBudgetPool(root, now = Date.now, restore = {}) {
|
|
2780
|
+
assertValidBudget(root, "root budget");
|
|
2781
|
+
if (restore.absoluteDeadlineMs !== void 0 && (!Number.isFinite(restore.absoluteDeadlineMs) || restore.absoluteDeadlineMs < 0)) throw new Error("budget restore.absoluteDeadlineMs must be a non-negative finite number");
|
|
2782
|
+
let freeTokens = root.maxTokens;
|
|
2783
|
+
let reservedTokens = 0;
|
|
2784
|
+
let committedTokens = 0;
|
|
2785
|
+
let tokensTainted = false;
|
|
2786
|
+
const usdCapped = root.maxUsd !== void 0;
|
|
2787
|
+
let freeUsd = root.maxUsd ?? 0;
|
|
2788
|
+
let reservedUsd = 0;
|
|
2789
|
+
let committedUsd = 0;
|
|
2790
|
+
let usdTainted = false;
|
|
2791
|
+
let usdMeasured = true;
|
|
2792
|
+
let freeIterations = root.maxIterations;
|
|
2793
|
+
let reservedIterations = 0;
|
|
2794
|
+
let committedIterations = 0;
|
|
2795
|
+
const absoluteDeadlineMs = restore.absoluteDeadlineMs ?? (root.deadlineMs !== void 0 ? now() + root.deadlineMs : 0);
|
|
2796
|
+
let nextTicketId = 0;
|
|
2797
|
+
const open = /* @__PURE__ */ new Set();
|
|
2798
|
+
function reserve(b) {
|
|
2799
|
+
assertValidBudget(b, "reservation budget");
|
|
2800
|
+
const wantTokens = b.maxTokens;
|
|
2801
|
+
const wantUsd = b.maxUsd ?? 0;
|
|
2802
|
+
const wantIterations = b.maxIterations;
|
|
2803
|
+
if (usdCapped && usdTainted) return {
|
|
2804
|
+
ok: false,
|
|
2805
|
+
reason: "budget-exhausted"
|
|
2377
2806
|
};
|
|
2378
|
-
return {
|
|
2379
|
-
|
|
2380
|
-
|
|
2807
|
+
if (wantTokens > freeTokens) return {
|
|
2808
|
+
ok: false,
|
|
2809
|
+
reason: "budget-exhausted"
|
|
2381
2810
|
};
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
let polls = 0;
|
|
2386
|
-
let probeErrors = 0;
|
|
2387
|
-
for (;;) {
|
|
2388
|
-
if (signal.aborted) return {
|
|
2389
|
-
kind: "cancelled",
|
|
2390
|
-
reason: "wait aborted"
|
|
2811
|
+
if (wantIterations > freeIterations) return {
|
|
2812
|
+
ok: false,
|
|
2813
|
+
reason: "budget-exhausted"
|
|
2391
2814
|
};
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2815
|
+
if (wantUsd > 0 && !usdCapped) return {
|
|
2816
|
+
ok: false,
|
|
2817
|
+
reason: "usd-unbudgeted"
|
|
2818
|
+
};
|
|
2819
|
+
if (wantUsd > freeUsd) return {
|
|
2820
|
+
ok: false,
|
|
2821
|
+
reason: "budget-exhausted"
|
|
2822
|
+
};
|
|
2823
|
+
freeTokens -= wantTokens;
|
|
2824
|
+
reservedTokens += wantTokens;
|
|
2825
|
+
freeIterations -= wantIterations;
|
|
2826
|
+
reservedIterations += wantIterations;
|
|
2827
|
+
if (wantUsd > 0) {
|
|
2828
|
+
freeUsd -= wantUsd;
|
|
2829
|
+
reservedUsd += wantUsd;
|
|
2398
2830
|
}
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2831
|
+
const id = nextTicketId++;
|
|
2832
|
+
open.add(id);
|
|
2833
|
+
return {
|
|
2834
|
+
ok: true,
|
|
2835
|
+
ticket: {
|
|
2836
|
+
id,
|
|
2837
|
+
reserved: {
|
|
2838
|
+
tokens: wantTokens,
|
|
2839
|
+
usd: wantUsd,
|
|
2840
|
+
iterations: wantIterations,
|
|
2841
|
+
usdBudgeted: b.maxUsd !== void 0
|
|
2842
|
+
}
|
|
2843
|
+
}
|
|
2402
2844
|
};
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2845
|
+
}
|
|
2846
|
+
function reconcile(ticket, spent) {
|
|
2847
|
+
if (!open.has(ticket.id)) throw new Error(`budget pool: reconcile of unknown or already-settled ticket ${ticket.id}`);
|
|
2848
|
+
assertValidSpend(spent, `budget pool ticket ${ticket.id} spend`);
|
|
2849
|
+
const { tokens: rTokens, usd: rUsd, iterations: rIterations } = ticket.reserved;
|
|
2850
|
+
const unknownUnderCap = usdCapped && spent.usdKnown === false;
|
|
2851
|
+
const spentTokens = totalTokens(spent.tokens);
|
|
2852
|
+
const usdBudgeted = ticket.reserved.usdBudgeted !== false;
|
|
2853
|
+
let violation;
|
|
2854
|
+
if (spentTokens > rTokens) violation = `ticket ${ticket.id} spent ${spentTokens} tokens > reserved ${rTokens}`;
|
|
2855
|
+
else if (spent.iterations > rIterations) violation = `ticket ${ticket.id} spent ${spent.iterations} iterations > reserved ${rIterations}`;
|
|
2856
|
+
else if (usdCapped && usdBudgeted && spent.usd > rUsd) violation = `ticket ${ticket.id} spent $${spent.usd} > reserved $${rUsd}`;
|
|
2857
|
+
else if (unknownUnderCap) violation = `ticket ${ticket.id} reported unknown dollar cost under a dollar-capped budget`;
|
|
2858
|
+
open.delete(ticket.id);
|
|
2859
|
+
if (spent.tokensKnown === false) tokensTainted = true;
|
|
2860
|
+
if (spent.usdKnown === false) usdMeasured = false;
|
|
2861
|
+
reservedTokens -= rTokens;
|
|
2862
|
+
committedTokens += spentTokens;
|
|
2863
|
+
freeTokens += rTokens - spentTokens;
|
|
2864
|
+
reservedIterations -= rIterations;
|
|
2865
|
+
committedIterations += spent.iterations;
|
|
2866
|
+
freeIterations += rIterations - spent.iterations;
|
|
2867
|
+
if (usdCapped) {
|
|
2868
|
+
reservedUsd -= rUsd;
|
|
2869
|
+
committedUsd += spent.usd;
|
|
2870
|
+
if (unknownUnderCap) {
|
|
2871
|
+
usdTainted = true;
|
|
2872
|
+
freeUsd = 0;
|
|
2873
|
+
} else freeUsd += rUsd - spent.usd;
|
|
2874
|
+
} else committedUsd += spent.usd;
|
|
2875
|
+
if (violation !== void 0) throw new Error(`budget pool: ${violation}`);
|
|
2876
|
+
}
|
|
2877
|
+
function observe(spend) {
|
|
2878
|
+
assertValidSpend(spend, "observed spend");
|
|
2879
|
+
if (usdCapped && spend.usdKnown === false) throw new Error("budget pool: cannot observe unknown dollar cost under a dollar-capped budget");
|
|
2880
|
+
if (spend.tokensKnown === false) tokensTainted = true;
|
|
2881
|
+
if (spend.usdKnown === false) usdMeasured = false;
|
|
2882
|
+
const tokens = totalTokens(spend.tokens);
|
|
2883
|
+
freeTokens -= tokens;
|
|
2884
|
+
committedTokens += tokens;
|
|
2885
|
+
freeIterations -= spend.iterations;
|
|
2886
|
+
committedIterations += spend.iterations;
|
|
2887
|
+
committedUsd += spend.usd;
|
|
2888
|
+
if (usdCapped) freeUsd -= spend.usd;
|
|
2889
|
+
}
|
|
2890
|
+
function readout() {
|
|
2891
|
+
return {
|
|
2892
|
+
tokensLeft: freeTokens,
|
|
2893
|
+
tokensKnown: !tokensTainted,
|
|
2894
|
+
usdLeft: usdCapped ? usdTainted ? 0 : freeUsd : 0,
|
|
2895
|
+
usdCapped,
|
|
2896
|
+
usdKnown: usdMeasured && !usdTainted,
|
|
2897
|
+
iterationsLeft: freeIterations,
|
|
2898
|
+
deadlineMs: absoluteDeadlineMs,
|
|
2899
|
+
reservedTokens
|
|
2406
2900
|
};
|
|
2407
|
-
await sleep(spec.timeoutAtMs === void 0 ? spec.intervalMs : Math.min(spec.intervalMs, Math.max(0, spec.timeoutAtMs - now())), signal);
|
|
2408
2901
|
}
|
|
2409
|
-
|
|
2410
|
-
|
|
2902
|
+
function assertNoOpenTickets() {
|
|
2903
|
+
if (open.size > 0) throw new Error(`budget pool: ${open.size} reservation(s) still open at join barrier (leaked ticket ids: ${[...open].join(", ")}) — conserved-pool invariant violated`);
|
|
2904
|
+
}
|
|
2905
|
+
if (restore.committed !== void 0) {
|
|
2906
|
+
assertValidSpend(restore.committed, "budget restore committed");
|
|
2907
|
+
const committed = restore.committed;
|
|
2908
|
+
if (committed.tokensKnown === false) tokensTainted = true;
|
|
2909
|
+
if (committed.usdKnown === false) usdMeasured = false;
|
|
2910
|
+
const tokens = totalTokens(committed.tokens);
|
|
2911
|
+
freeTokens -= tokens;
|
|
2912
|
+
committedTokens += tokens;
|
|
2913
|
+
freeIterations -= committed.iterations;
|
|
2914
|
+
committedIterations += committed.iterations;
|
|
2915
|
+
committedUsd += committed.usd;
|
|
2916
|
+
if (usdCapped) {
|
|
2917
|
+
freeUsd -= committed.usd;
|
|
2918
|
+
if (committed.usdKnown === false) {
|
|
2919
|
+
usdTainted = true;
|
|
2920
|
+
if (freeUsd > 0) committedUsd += freeUsd;
|
|
2921
|
+
freeUsd = 0;
|
|
2922
|
+
}
|
|
2923
|
+
}
|
|
2924
|
+
}
|
|
2925
|
+
for (const [index, uncertain] of (restore.uncertainReservations ?? []).entries()) {
|
|
2926
|
+
assertValidBudget(uncertain, `budget restore uncertainReservations[${index}]`);
|
|
2927
|
+
tokensTainted = true;
|
|
2928
|
+
usdMeasured = false;
|
|
2929
|
+
freeTokens -= uncertain.maxTokens;
|
|
2930
|
+
committedTokens += uncertain.maxTokens;
|
|
2931
|
+
freeIterations -= uncertain.maxIterations;
|
|
2932
|
+
committedIterations += uncertain.maxIterations;
|
|
2933
|
+
if (usdCapped) if (uncertain.maxUsd === void 0) {
|
|
2934
|
+
usdTainted = true;
|
|
2935
|
+
committedUsd += freeUsd > 0 ? freeUsd : 0;
|
|
2936
|
+
freeUsd = 0;
|
|
2937
|
+
} else {
|
|
2938
|
+
freeUsd -= uncertain.maxUsd;
|
|
2939
|
+
committedUsd += uncertain.maxUsd;
|
|
2940
|
+
}
|
|
2941
|
+
}
|
|
2411
2942
|
return {
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
wokenAt,
|
|
2419
|
-
polls,
|
|
2420
|
-
probeErrors,
|
|
2421
|
-
resumed
|
|
2943
|
+
reserve,
|
|
2944
|
+
reconcile,
|
|
2945
|
+
spendFrom: foldUsage,
|
|
2946
|
+
readout,
|
|
2947
|
+
observe,
|
|
2948
|
+
assertNoOpenTickets
|
|
2422
2949
|
};
|
|
2423
2950
|
}
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2951
|
+
//#endregion
|
|
2952
|
+
//#region src/runtime/supervise/budget-floor.ts
|
|
2953
|
+
const WORKER_TOKEN_FLOOR = {
|
|
2954
|
+
pi: 31211,
|
|
2955
|
+
opencode: null,
|
|
2956
|
+
"claude-code": null,
|
|
2957
|
+
"kimi-code": null,
|
|
2958
|
+
codex: null,
|
|
2959
|
+
amp: null,
|
|
2960
|
+
"factory-droids": null,
|
|
2961
|
+
hermes: null,
|
|
2962
|
+
forge: null,
|
|
2963
|
+
openclaw: null,
|
|
2964
|
+
nanoclaw: null,
|
|
2965
|
+
acp: null,
|
|
2966
|
+
cursor: null,
|
|
2967
|
+
"cli-base": null
|
|
2968
|
+
};
|
|
2969
|
+
/**
|
|
2970
|
+
* The floor for a spawn's harness, or `null` when there is none to enforce.
|
|
2971
|
+
*
|
|
2972
|
+
* A `null` harness is the router/inline arm: it holds only coordination verbs and re-sends no
|
|
2973
|
+
* harness scaffolding, so it has no floor of this kind.
|
|
2974
|
+
*/
|
|
2975
|
+
function workerTokenFloor(harness) {
|
|
2976
|
+
if (harness === null) return null;
|
|
2977
|
+
return WORKER_TOKEN_FLOOR[harness] ?? null;
|
|
2439
2978
|
}
|
|
2440
2979
|
//#endregion
|
|
2441
|
-
//#region src/
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2980
|
+
//#region src/runtime/supervise/deadline.ts
|
|
2981
|
+
const MAX_TIMER_DELAY_MS = 2147483647;
|
|
2982
|
+
const DEFAULT_SUCCESSFUL_SHUTDOWN_MS = 5e3;
|
|
2983
|
+
const TEARDOWN_ACKNOWLEDGEMENT_MS = 250;
|
|
2984
|
+
/** Arm one wall-clock deadline without keeping the Node.js process alive. */
|
|
2985
|
+
function armDeadlineTimer(delayMs, onDeadline) {
|
|
2986
|
+
const deadlineAtMs = Date.now() + Math.max(0, delayMs);
|
|
2987
|
+
let cleared = false;
|
|
2988
|
+
let timer;
|
|
2989
|
+
const arm = () => {
|
|
2990
|
+
if (cleared) return;
|
|
2991
|
+
const remainingMs = Math.max(0, deadlineAtMs - Date.now());
|
|
2992
|
+
timer = setTimeout(() => {
|
|
2993
|
+
if (cleared) return;
|
|
2994
|
+
if (Date.now() >= deadlineAtMs) onDeadline();
|
|
2995
|
+
else arm();
|
|
2996
|
+
}, Math.min(remainingMs, MAX_TIMER_DELAY_MS));
|
|
2997
|
+
if (typeof timer.unref === "function") timer.unref();
|
|
2455
2998
|
};
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
type: event.type,
|
|
2461
|
-
task: sanitizeTask(event.task, options)
|
|
2999
|
+
arm();
|
|
3000
|
+
return () => {
|
|
3001
|
+
cleared = true;
|
|
3002
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
2462
3003
|
};
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
3004
|
+
}
|
|
3005
|
+
/**
|
|
3006
|
+
* Resolve a spawned child's duration into one absolute cutoff. An omitted child deadline inherits
|
|
3007
|
+
* the parent cutoff; an explicit child duration may shorten, but never extend, that cutoff.
|
|
3008
|
+
*/
|
|
3009
|
+
function boundedChildDeadlineAt(parentDeadlineAtMs, childDeadlineMs, nowMs) {
|
|
3010
|
+
const parent = parentDeadlineAtMs > 0 ? parentDeadlineAtMs : void 0;
|
|
3011
|
+
const child = childDeadlineMs === void 0 ? void 0 : nowMs + childDeadlineMs;
|
|
3012
|
+
if (parent === void 0) return child;
|
|
3013
|
+
if (child === void 0) return parent;
|
|
3014
|
+
return Math.min(parent, child);
|
|
3015
|
+
}
|
|
3016
|
+
/** Invoke teardown once and bound how long the runtime waits for its acknowledgement. A hard
|
|
3017
|
+
* execution deadline always wins; without one, numeric grace gets a short acknowledgement
|
|
3018
|
+
* allowance and a brutal kill gets only that allowance. Explicit `infinity` remains unbounded
|
|
3019
|
+
* only when no execution deadline exists. The losing promise stays observed. */
|
|
3020
|
+
async function teardownExecutor(executor, grace, deadlineAtMs, now) {
|
|
3021
|
+
const work = Promise.resolve(executor.teardown(grace));
|
|
3022
|
+
const requestedWaitMs = grace === "infinity" ? void 0 : grace === "brutalKill" ? TEARDOWN_ACKNOWLEDGEMENT_MS : grace + TEARDOWN_ACKNOWLEDGEMENT_MS;
|
|
3023
|
+
const deadlineWaitMs = deadlineAtMs === void 0 ? void 0 : Math.max(0, deadlineAtMs - now()) + TEARDOWN_ACKNOWLEDGEMENT_MS;
|
|
3024
|
+
const waitMs = requestedWaitMs === void 0 ? deadlineWaitMs : deadlineWaitMs === void 0 ? requestedWaitMs : Math.min(requestedWaitMs, deadlineWaitMs);
|
|
3025
|
+
let receipt;
|
|
3026
|
+
if (waitMs === void 0) receipt = await work;
|
|
3027
|
+
else if (waitMs <= 0) {
|
|
3028
|
+
work.catch(() => void 0);
|
|
3029
|
+
throw new ValidationError("executor teardown did not acknowledge before its deadline");
|
|
3030
|
+
} else {
|
|
3031
|
+
let timer;
|
|
3032
|
+
const timedOut = new Promise((_resolve, reject) => {
|
|
3033
|
+
timer = setTimeout(() => reject(new ValidationError(`executor teardown did not acknowledge within ${waitMs}ms`)), waitMs);
|
|
3034
|
+
});
|
|
3035
|
+
try {
|
|
3036
|
+
receipt = await Promise.race([work, timedOut]);
|
|
3037
|
+
} finally {
|
|
3038
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
3039
|
+
}
|
|
3040
|
+
}
|
|
3041
|
+
if (!receipt.destroyed) throw new ValidationError("executor teardown reported destroyed=false");
|
|
3042
|
+
}
|
|
3043
|
+
//#endregion
|
|
3044
|
+
//#region src/runtime/supervise/dispatch.ts
|
|
3045
|
+
/**
|
|
3046
|
+
*
|
|
3047
|
+
* `rollingDispatch` — the REFILLING dispatch policy over a `Scope`. It keeps `width` children in
|
|
3048
|
+
* flight and admits the next queued unit of work the instant one settles, instead of waiting for
|
|
3049
|
+
* a whole round to drain (`fanout`) or for a driver to decide again (`spawn → await → spawn`).
|
|
3050
|
+
*
|
|
3051
|
+
* The gap it closes: `fanout` opens every item at once and then drains — one round, no refill —
|
|
3052
|
+
* and a driver brain's manual loop opens one worker per model turn. Between those two there was
|
|
3053
|
+
* nothing that holds N slots full, which is why a 5-worker run can peak at 2 live workers and sit
|
|
3054
|
+
* ~50% idle. This is a policy over the EXISTING `Scope.spawn` / `Scope.next` primitives: it adds
|
|
3055
|
+
* no second admission path, so the conserved budget pool stays the only fence on total work and
|
|
3056
|
+
* `width` is only a fence on simultaneous work.
|
|
3057
|
+
*
|
|
3058
|
+
* Fail-closed by construction: an admission rejection (`budget-exhausted` / `depth-exceeded`) is
|
|
3059
|
+
* recorded and STOPS further admission — the loop then drains what is already live and returns.
|
|
3060
|
+
* It never retries a rejected spawn against the same pool, and it never spawns past `width`.
|
|
3061
|
+
*
|
|
3062
|
+
* ── The concurrency-cap ledger (read this before adding a fourth cap) ──────────────────────────
|
|
3063
|
+
*
|
|
3064
|
+
* Three unrelated caps bound "how much runs at once" in this stack, at three different layers.
|
|
3065
|
+
* They are NOT aware of each other, and the smallest one silently wins:
|
|
3066
|
+
*
|
|
3067
|
+
* 1. `SuperviseOptions.maxLiveWorkers` (`src/runtime/supervise/supervise.ts`) — supervised-tree
|
|
3068
|
+
* level. One shared Scope counter bounds every spawned manager and leaf in the recursive tree;
|
|
3069
|
+
* `spawn_agent` fails closed with `error: 'max-live-workers'` past it. Unset ⇒ NO tree cap.
|
|
3070
|
+
* `CoordinationToolsOptions.maxLiveWorkers` remains the local form for a toolbox mounted on a
|
|
3071
|
+
* caller-owned Scope that has no tree limit.
|
|
3072
|
+
* 2. `SandboxLineage`'s `maxConcurrency` / `DEFAULT_FORK_CONCURRENCY = 4`
|
|
3073
|
+
* (`src/runtime/sandbox-lineage.ts`) — kernel level. How many BOXES one `runAgentRounds` fork wave
|
|
3074
|
+
* provisions at once. It bounds a single leaf's fanout, not the supervisor's worker count.
|
|
3075
|
+
* 3. A host's own live-box governor (e.g. the kernel's `ComputeGovernor`, `maxSandboxes = 4`) — fleet
|
|
3076
|
+
* level. How many sandboxes may exist across the whole host process.
|
|
3077
|
+
*
|
|
3078
|
+
* The honest effective limit on simultaneous WORKERS is the minimum of the caps that apply to the
|
|
3079
|
+
* worker layer — (1) and (3). (2) is a different unit (boxes per fork wave inside one leaf) and
|
|
3080
|
+
* must not be min'd into it, or a 4-way fork inside one worker reads as a 4-worker ceiling.
|
|
3081
|
+
* `effectiveConcurrency` computes that minimum in one place so a host derives ONE number and
|
|
3082
|
+
* passes it to BOTH `maxLiveWorkers` and this dispatcher's `width`, instead of leaving a fleet
|
|
3083
|
+
* governor of 4 and an unset worker fence as unrelated numbers.
|
|
3084
|
+
*
|
|
3085
|
+
* ── Why this is not a copy of the kernel's batch loop ──────────────────────────────────────────
|
|
3086
|
+
*
|
|
3087
|
+
* `runBatch` (`src/runtime/run-loop.ts`) runs the same hold-N-slots-full shape over the KERNEL's
|
|
3088
|
+
* substrate: bare promises in a `Set`, raced with `Promise.race`, bounded by `maxConcurrency`.
|
|
3089
|
+
* This one runs it over the SUPERVISOR's substrate — `Scope.spawn`'s atomic reservation against
|
|
3090
|
+
* the conserved pool, `Scope.next`'s journaled settlement cursor, and the blob store behind each
|
|
3091
|
+
* result. Neither can be expressed in the other's terms without dragging its whole substrate
|
|
3092
|
+
* along: the kernel loop has no budget to reserve and no cursor to order settlements by, and this
|
|
3093
|
+
* one cannot race raw promises because a settlement is only real once it is journaled. The shape
|
|
3094
|
+
* repeating across the two deliberate layers is resonance, not duplication — do not "unify" them
|
|
3095
|
+
* into a shared helper that would have to know about both.
|
|
3096
|
+
*
|
|
3097
|
+
* @experimental
|
|
3098
|
+
*/
|
|
3099
|
+
/**
|
|
3100
|
+
* Run the refilling dispatch loop over `scope` until the queue is dry (or a stop fires) and every
|
|
3101
|
+
* admitted child has settled. Returns the settlements in cursor order plus the admission ledger.
|
|
3102
|
+
*
|
|
3103
|
+
* The loop is: fill free slots from `nextUnit` → `await scope.next()` → deliver the settlement →
|
|
3104
|
+
* refill → repeat. Because the refill happens immediately after each settlement rather than after
|
|
3105
|
+
* a whole round, a slow child never idles the other slots.
|
|
3106
|
+
*/
|
|
3107
|
+
async function rollingDispatch(scope, opts) {
|
|
3108
|
+
if (!Number.isInteger(opts.width) || opts.width < 1) throw new ValidationError(`rollingDispatch: width must be a positive integer, got ${String(opts.width)}`);
|
|
3109
|
+
const settled = [];
|
|
3110
|
+
const rejected = [];
|
|
3111
|
+
const live = /* @__PURE__ */ new Set();
|
|
3112
|
+
let admitted = 0;
|
|
3113
|
+
let peakLive = 0;
|
|
3114
|
+
let queueDry = false;
|
|
3115
|
+
let stopReason = "drained";
|
|
3116
|
+
const fill = async () => {
|
|
3117
|
+
while (live.size < opts.width) {
|
|
3118
|
+
if (queueDry) return;
|
|
3119
|
+
if (scope.signal.aborted) {
|
|
3120
|
+
stopReason = "aborted";
|
|
3121
|
+
queueDry = true;
|
|
3122
|
+
return;
|
|
3123
|
+
}
|
|
3124
|
+
if (opts.shouldStop?.() === true) {
|
|
3125
|
+
stopReason = "stopped";
|
|
3126
|
+
queueDry = true;
|
|
3127
|
+
return;
|
|
3128
|
+
}
|
|
3129
|
+
const unit = await opts.nextUnit();
|
|
3130
|
+
if (unit === void 0) {
|
|
3131
|
+
queueDry = true;
|
|
3132
|
+
return;
|
|
3133
|
+
}
|
|
3134
|
+
const res = scope.spawn(unit.agent, unit.task, unit.opts);
|
|
3135
|
+
if (!res.ok) {
|
|
3136
|
+
rejected.push(`${unit.opts.label}: ${res.reason}`);
|
|
3137
|
+
stopReason = "not-admitted";
|
|
3138
|
+
queueDry = true;
|
|
3139
|
+
return;
|
|
3140
|
+
}
|
|
3141
|
+
live.add(res.handle.id);
|
|
3142
|
+
admitted += 1;
|
|
3143
|
+
if (live.size > peakLive) peakLive = live.size;
|
|
3144
|
+
}
|
|
3145
|
+
};
|
|
3146
|
+
await fill();
|
|
3147
|
+
while (live.size > 0) {
|
|
3148
|
+
const s = await scope.next();
|
|
3149
|
+
if (s === null) break;
|
|
3150
|
+
live.delete(s.handle.id);
|
|
3151
|
+
settled.push(s);
|
|
3152
|
+
await opts.onSettled?.(s);
|
|
3153
|
+
await fill();
|
|
3154
|
+
}
|
|
3155
|
+
return {
|
|
3156
|
+
settled,
|
|
3157
|
+
admitted,
|
|
3158
|
+
rejected,
|
|
3159
|
+
stopReason,
|
|
3160
|
+
peakLive
|
|
3161
|
+
};
|
|
3162
|
+
}
|
|
3163
|
+
/**
|
|
3164
|
+
* Free worker slots under a simultaneity cap: `cap - live`, floored at 0, or `null` when there is
|
|
3165
|
+
* no cap (the conserved pool is then the only fence and "free slots" is not a finite number).
|
|
3166
|
+
* The one place the answer is computed, so the driver-facing tool payload and a dispatcher agree.
|
|
3167
|
+
*/
|
|
3168
|
+
function freeSlots(liveCount, cap) {
|
|
3169
|
+
if (cap === void 0 || cap <= 0) return null;
|
|
3170
|
+
return Math.max(0, cap - liveCount);
|
|
3171
|
+
}
|
|
3172
|
+
/**
|
|
3173
|
+
* The ONE honest effective limit on simultaneous workers: the minimum of the caps that actually
|
|
3174
|
+
* bound the worker layer. Ignores unset/non-positive caps; returns `undefined` when no cap applies
|
|
3175
|
+
* (uncapped — the conserved pool remains the only fence).
|
|
3176
|
+
*
|
|
3177
|
+
* Deliberately does NOT fold in `SandboxLineage`'s fork concurrency: that bounds boxes inside ONE
|
|
3178
|
+
* leaf's fork wave, a different unit. Folding it in would report a 4-worker ceiling for what is
|
|
3179
|
+
* really a 4-box fanout inside a single worker.
|
|
3180
|
+
*
|
|
3181
|
+
* Use it once, at the top of a run, and pass the result to BOTH `maxLiveWorkers` and a
|
|
3182
|
+
* dispatcher's `width` — that is what turns three unrelated numbers into one.
|
|
3183
|
+
*/
|
|
3184
|
+
function effectiveConcurrency(caps) {
|
|
3185
|
+
const applicable = [caps.maxLiveWorkers, caps.maxSandboxes].filter((c) => typeof c === "number" && c > 0);
|
|
3186
|
+
if (applicable.length === 0) return void 0;
|
|
3187
|
+
return Math.min(...applicable);
|
|
3188
|
+
}
|
|
3189
|
+
/** Convenience: a `DispatchUnit` factory over a fixed array of tasks, for the common case where
|
|
3190
|
+
* the queue is known up front and only the refill behavior is wanted. */
|
|
3191
|
+
function queueOf(units, budget) {
|
|
3192
|
+
let i = 0;
|
|
3193
|
+
return () => {
|
|
3194
|
+
const u = units[i];
|
|
3195
|
+
if (u === void 0) return void 0;
|
|
3196
|
+
i += 1;
|
|
3197
|
+
return {
|
|
3198
|
+
agent: u.agent,
|
|
3199
|
+
task: u.task,
|
|
3200
|
+
opts: {
|
|
3201
|
+
budget,
|
|
3202
|
+
label: u.label
|
|
3203
|
+
}
|
|
3204
|
+
};
|
|
3205
|
+
};
|
|
3206
|
+
}
|
|
3207
|
+
//#endregion
|
|
3208
|
+
//#region src/runtime/supervise/progress.ts
|
|
3209
|
+
/** How long a worker may produce no metered activity before a `progress()` read calls it stalled.
|
|
3210
|
+
* Deliberately generous: a coding harness routinely spends minutes inside one tool call, and a
|
|
3211
|
+
* false stall that provokes a steer is worse than a late one. */
|
|
3212
|
+
const DEFAULT_STALL_AFTER_MS = 18e4;
|
|
3213
|
+
/** Create a bounded activity ring. `limit` caps memory for a worker that runs thousands of tools. */
|
|
3214
|
+
function createActivityLog(limit = 12) {
|
|
3215
|
+
const notes = [];
|
|
3216
|
+
const cap = Math.max(1, Math.floor(limit));
|
|
3217
|
+
return {
|
|
3218
|
+
push(note) {
|
|
3219
|
+
notes.push(note);
|
|
3220
|
+
if (notes.length > cap) notes.splice(0, notes.length - cap);
|
|
3221
|
+
},
|
|
3222
|
+
read: () => [...notes],
|
|
3223
|
+
last: () => notes[notes.length - 1],
|
|
3224
|
+
size: () => notes.length
|
|
3225
|
+
};
|
|
3226
|
+
}
|
|
3227
|
+
/** Fold the scope-derived facts and the executor's optional enrichment into one read. Pure: the
|
|
3228
|
+
* caller supplies `now`, so a test can observe a stall without waiting for one. */
|
|
3229
|
+
function readWorkerProgress(scope, executor, now, stallAfterMs = DEFAULT_STALL_AFTER_MS) {
|
|
3230
|
+
const live = scope.status !== "done" && scope.status !== "failed" && scope.status !== "cancelled";
|
|
3231
|
+
const executorActivityAt = executor?.recentActivity?.length ? executor.recentActivity[executor.recentActivity.length - 1].at : 0;
|
|
3232
|
+
const lastActivityAt = Math.max(scope.lastActivityAt, executorActivityAt);
|
|
3233
|
+
const idleMs = Math.max(0, now - lastActivityAt);
|
|
3234
|
+
const note = executor?.note;
|
|
3235
|
+
return {
|
|
3236
|
+
id: scope.id,
|
|
3237
|
+
status: scope.status,
|
|
3238
|
+
live,
|
|
3239
|
+
steerable: scope.steerable,
|
|
3240
|
+
startedAt: scope.startedAt,
|
|
3241
|
+
lastActivityAt,
|
|
3242
|
+
idleMs,
|
|
3243
|
+
stalled: live && idleMs > stallAfterMs,
|
|
3244
|
+
stallAfterMs,
|
|
3245
|
+
turns: executor?.turns ?? scope.turns,
|
|
3246
|
+
tokens: scope.tokens,
|
|
3247
|
+
...scope.tokensKnown === false ? { tokensKnown: false } : {},
|
|
3248
|
+
usd: scope.usd,
|
|
3249
|
+
...scope.usdKnown === false ? { usdKnown: false } : {},
|
|
3250
|
+
pendingMessages: executor?.pendingMessages ?? 0,
|
|
3251
|
+
recentActivity: executor?.recentActivity ?? [],
|
|
3252
|
+
...executor?.derived?.length ? { derived: executor.derived } : {},
|
|
3253
|
+
...note ? { note } : {}
|
|
3254
|
+
};
|
|
3255
|
+
}
|
|
3256
|
+
//#endregion
|
|
3257
|
+
//#region src/runtime/supervise/wait.ts
|
|
3258
|
+
/**
|
|
3259
|
+
*
|
|
3260
|
+
* WAIT-STATES — a supervision-tree node that waits on wall-clock time or an external condition
|
|
3261
|
+
* without holding a worker, an executor, a sandbox, or a single LLM turn.
|
|
3262
|
+
*
|
|
3263
|
+
* A long-horizon run spends most of its wall-clock NOT computing: waiting for CI to finish,
|
|
3264
|
+
* for a nightly job to land, for a human to reply, for a rate-limit window to reopen. Before
|
|
3265
|
+
* this, the only way to express that was to keep something alive and re-ask — which spends
|
|
3266
|
+
* driver tokens per re-ask and pins a process to the wait.
|
|
3267
|
+
*
|
|
3268
|
+
* Two node kinds, both first-class:
|
|
3269
|
+
* - `timer` — wake at an ABSOLUTE wall-clock instant (`untilMs`).
|
|
3270
|
+
* - `poll` — re-run a named predicate every `intervalMs` until it returns true, or until an
|
|
3271
|
+
* absolute `timeoutAtMs` passes (CI status, file existence, an HTTP probe, an
|
|
3272
|
+
* inbox message).
|
|
3273
|
+
*
|
|
3274
|
+
* ── How this differs from `await_event`'s 15s poll fence (`DEFAULT_AWAIT_EVENT_TIMEOUT_MS`) ──
|
|
3275
|
+
*
|
|
3276
|
+
* They look similar and are not the same mechanism. `await_event` is an IN-RUN RENDEZVOUS: the
|
|
3277
|
+
* driver blocks on the coordination bus for the next event from a live worker, and the 15s fence
|
|
3278
|
+
* exists only so a remote MCP request does not exceed the client's timeout — the caller re-polls.
|
|
3279
|
+
* Every re-poll is another driver inference turn (real tokens) against a process that must stay
|
|
3280
|
+
* up, and nothing about that wait is recorded: kill the process and the wait is simply gone.
|
|
3281
|
+
*
|
|
3282
|
+
* A wait-state is a NODE, not a call. It has a node id in the tree, a journal record carrying its
|
|
3283
|
+
* ABSOLUTE deadline, and it settles through the same `Scope.next()` cursor as any worker. Nobody
|
|
3284
|
+
* is blocked on it — the driver can stop reasoning entirely, and the process can die. Cost while
|
|
3285
|
+
* waiting: zero LLM calls, zero executor, zero sandbox, zero conserved budget (a wait reserves
|
|
3286
|
+
* nothing from the pool). The only in-process residue is one timer entry.
|
|
3287
|
+
*
|
|
3288
|
+
* ── Why probes are NAMED, not passed as closures ────────────────────────────────────────────
|
|
3289
|
+
*
|
|
3290
|
+
* A wait must survive a process restart with its original deadline intact, and a closure cannot
|
|
3291
|
+
* be journaled. So a `poll` names its predicate (`probe: 'ci-green'`) and the run resolves it
|
|
3292
|
+
* through a `WaitProbeRegistry`. A brand-new process re-resolves the SAME name against its own
|
|
3293
|
+
* registry and re-arms the wait — which is what makes "kill the box, the wait keeps waiting"
|
|
3294
|
+
* true rather than aspirational.
|
|
3295
|
+
*
|
|
3296
|
+
* Absolute instants for the same reason: `untilMs` / `timeoutAtMs` are epoch ms, not durations,
|
|
3297
|
+
* so a resumed wait counts down from the original arm, not from the restart. `timerAt`/`pollFor`
|
|
3298
|
+
* build them from a duration when that is what the caller has.
|
|
3299
|
+
*
|
|
3300
|
+
* @experimental
|
|
3301
|
+
*/
|
|
3302
|
+
/** Build a `timer` spec from a DURATION. The instant is resolved once, at arm time — a resumed
|
|
3303
|
+
* wait re-uses the journaled instant, never a fresh `now + ms`. */
|
|
3304
|
+
function timerAt(ms, now) {
|
|
3305
|
+
return {
|
|
3306
|
+
kind: "timer",
|
|
3307
|
+
untilMs: now + ms
|
|
3308
|
+
};
|
|
3309
|
+
}
|
|
3310
|
+
/** Build a bounded `poll` spec from a duration. */
|
|
3311
|
+
function pollFor(probe, opts, now) {
|
|
3312
|
+
return {
|
|
3313
|
+
kind: "poll",
|
|
3314
|
+
probe,
|
|
3315
|
+
intervalMs: opts.intervalMs,
|
|
3316
|
+
...opts.timeoutMs !== void 0 ? { timeoutAtMs: now + opts.timeoutMs } : {},
|
|
3317
|
+
...opts.args ? { args: opts.args } : {}
|
|
3318
|
+
};
|
|
3319
|
+
}
|
|
3320
|
+
/** Registry over a plain name→predicate record. */
|
|
3321
|
+
function createWaitProbes(entries) {
|
|
3322
|
+
const map = new Map(Object.entries(entries));
|
|
3323
|
+
return { resolve: (name) => map.get(name) };
|
|
3324
|
+
}
|
|
3325
|
+
/** Narrow a settlement's `out` to a wait outcome — a wait settles on the SAME cursor as workers,
|
|
3326
|
+
* so a driver that mixes them tags them apart with this. */
|
|
3327
|
+
function isWaitOutcome(value) {
|
|
3328
|
+
return typeof value === "object" && value !== null && value.waitOutcome === true;
|
|
3329
|
+
}
|
|
3330
|
+
/** The absolute instant a spec is bounded by, or `undefined` for an unbounded poll. */
|
|
3331
|
+
function waitUntil(spec) {
|
|
3332
|
+
return spec.kind === "timer" ? spec.untilMs : spec.timeoutAtMs;
|
|
3333
|
+
}
|
|
3334
|
+
/** Structural validation, independent of the run. Returns null when the spec is usable. */
|
|
3335
|
+
function validateWaitSpec(spec) {
|
|
3336
|
+
if (spec.kind === "timer") return Number.isFinite(spec.untilMs) ? null : "timer.untilMs must be a finite epoch-ms instant";
|
|
3337
|
+
if (typeof spec.probe !== "string" || spec.probe.length === 0) return "poll.probe must be a non-empty probe name";
|
|
3338
|
+
if (!Number.isFinite(spec.intervalMs) || spec.intervalMs <= 0) return "poll.intervalMs must be > 0";
|
|
3339
|
+
if (spec.timeoutAtMs !== void 0 && !Number.isFinite(spec.timeoutAtMs)) return "poll.timeoutAtMs must be a finite epoch-ms instant";
|
|
3340
|
+
return null;
|
|
3341
|
+
}
|
|
3342
|
+
/**
|
|
3343
|
+
* A wait must not silently outlive the run's HARD wall-clock ceiling. When the conserved pool
|
|
3344
|
+
* carries an absolute `deadlineMs`, a wait bounded past it — or an unbounded poll — is REFUSED at
|
|
3345
|
+
* arm time (`deadline-exceeded`) rather than sleeping through a ceiling the caller already set.
|
|
3346
|
+
* The alternative (extending the pool's deadline to cover the wait) would let a wait override a
|
|
3347
|
+
* hard budget guard, which is exactly what mechanic D forbids; a run that intends week-long waits
|
|
3348
|
+
* either sets a deadline that covers them or sets none.
|
|
3349
|
+
*/
|
|
3350
|
+
function assertWaitWithinDeadline(spec, deadlineMs) {
|
|
3351
|
+
if (deadlineMs <= 0) return true;
|
|
3352
|
+
const until = waitUntil(spec);
|
|
3353
|
+
if (until === void 0) return false;
|
|
3354
|
+
return until <= deadlineMs;
|
|
3355
|
+
}
|
|
3356
|
+
/**
|
|
3357
|
+
* Drive one wait to its resolution. Spends nothing: a `timer` is one sleep, a `poll` is a sleep
|
|
3358
|
+
* between predicate checks. No LLM call, no executor, no box, no budget reservation.
|
|
3359
|
+
*/
|
|
3360
|
+
async function runWait(args) {
|
|
3361
|
+
const { spec, label, armedAt, resumed, signal, now } = args;
|
|
3362
|
+
const sleep = args.sleep ?? defaultSleep;
|
|
3363
|
+
if (signal.aborted) return {
|
|
3364
|
+
kind: "cancelled",
|
|
3365
|
+
reason: "aborted before arming"
|
|
3366
|
+
};
|
|
3367
|
+
if (spec.kind === "timer") {
|
|
3368
|
+
const remaining = spec.untilMs - now();
|
|
3369
|
+
if (remaining > 0) await sleep(remaining, signal);
|
|
3370
|
+
if (signal.aborted) return {
|
|
3371
|
+
kind: "cancelled",
|
|
3372
|
+
reason: "wait aborted"
|
|
3373
|
+
};
|
|
3374
|
+
return {
|
|
3375
|
+
kind: "woke",
|
|
3376
|
+
outcome: outcome("timer", "fired", label, spec.untilMs, armedAt, now(), 0, 0, resumed)
|
|
3377
|
+
};
|
|
3378
|
+
}
|
|
3379
|
+
const probe = args.probes?.resolve(spec.probe);
|
|
3380
|
+
if (!probe) throw new ValidationError(`wait '${label}': poll probe '${spec.probe}' is not registered in this run's WaitProbeRegistry`);
|
|
3381
|
+
let polls = 0;
|
|
3382
|
+
let probeErrors = 0;
|
|
3383
|
+
for (;;) {
|
|
3384
|
+
if (signal.aborted) return {
|
|
3385
|
+
kind: "cancelled",
|
|
3386
|
+
reason: "wait aborted"
|
|
3387
|
+
};
|
|
3388
|
+
polls += 1;
|
|
3389
|
+
let fired = false;
|
|
3390
|
+
try {
|
|
3391
|
+
fired = await probe(spec.args, signal);
|
|
3392
|
+
} catch {
|
|
3393
|
+
probeErrors += 1;
|
|
3394
|
+
}
|
|
3395
|
+
if (fired) return {
|
|
3396
|
+
kind: "woke",
|
|
3397
|
+
outcome: outcome("poll", "fired", label, spec.timeoutAtMs, armedAt, now(), polls, probeErrors, resumed)
|
|
3398
|
+
};
|
|
3399
|
+
if (spec.timeoutAtMs !== void 0 && now() >= spec.timeoutAtMs) return {
|
|
3400
|
+
kind: "woke",
|
|
3401
|
+
outcome: outcome("poll", "timeout", label, spec.timeoutAtMs, armedAt, now(), polls, probeErrors, resumed)
|
|
3402
|
+
};
|
|
3403
|
+
await sleep(spec.timeoutAtMs === void 0 ? spec.intervalMs : Math.min(spec.intervalMs, Math.max(0, spec.timeoutAtMs - now())), signal);
|
|
3404
|
+
}
|
|
3405
|
+
}
|
|
3406
|
+
function outcome(kind, settled, label, untilMs, armedAt, wokenAt, polls, probeErrors, resumed) {
|
|
3407
|
+
return {
|
|
3408
|
+
waitOutcome: true,
|
|
3409
|
+
kind,
|
|
3410
|
+
settled,
|
|
3411
|
+
label,
|
|
3412
|
+
...untilMs !== void 0 ? { untilMs } : {},
|
|
3413
|
+
armedAt,
|
|
3414
|
+
wokenAt,
|
|
3415
|
+
polls,
|
|
3416
|
+
probeErrors,
|
|
3417
|
+
resumed
|
|
3418
|
+
};
|
|
3419
|
+
}
|
|
3420
|
+
/** `setTimeout` that resolves early (and clears) when the scope aborts, so a cancelled wait
|
|
3421
|
+
* releases the event loop immediately instead of holding the process to its deadline. */
|
|
3422
|
+
function defaultSleep(ms, signal) {
|
|
3423
|
+
if (ms <= 0 || signal.aborted) return Promise.resolve();
|
|
3424
|
+
return new Promise((resolve) => {
|
|
3425
|
+
const timer = setTimeout(() => {
|
|
3426
|
+
signal.removeEventListener("abort", onAbort);
|
|
3427
|
+
resolve();
|
|
3428
|
+
}, ms);
|
|
3429
|
+
function onAbort() {
|
|
3430
|
+
clearTimeout(timer);
|
|
3431
|
+
resolve();
|
|
3432
|
+
}
|
|
3433
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
3434
|
+
});
|
|
3435
|
+
}
|
|
3436
|
+
//#endregion
|
|
3437
|
+
//#region src/sanitize.ts
|
|
3438
|
+
/** Strip PII and large blobs from a `KnowledgeReadinessReport` for safe telemetry emission. @stable */
|
|
3439
|
+
function sanitizeKnowledgeReadinessReport(report, options = {}) {
|
|
3440
|
+
return {
|
|
3441
|
+
taskId: report.taskId,
|
|
3442
|
+
readinessScore: report.readinessScore,
|
|
3443
|
+
recommendedAction: report.recommendedAction,
|
|
3444
|
+
severity: report.severity,
|
|
3445
|
+
reason: report.reason,
|
|
3446
|
+
blockingMissingRequirements: report.blockingMissingRequirements.map((requirement) => sanitizeKnowledgeRequirement(requirement, options)),
|
|
3447
|
+
nonBlockingGaps: report.nonBlockingGaps.map((requirement) => sanitizeKnowledgeRequirement(requirement, options)),
|
|
3448
|
+
evidenceCount: report.bundle.evidenceIds.length,
|
|
3449
|
+
evidenceIds: options.includeEvidenceIds ? report.bundle.evidenceIds : void 0,
|
|
3450
|
+
missingRequirementIds: report.bundle.missing.map((requirement) => requirement.id)
|
|
3451
|
+
};
|
|
3452
|
+
}
|
|
3453
|
+
/** Reduce an `AgentRuntimeEvent` to a PII-safe, serializable plain object for telemetry. @stable */
|
|
3454
|
+
function sanitizeAgentRuntimeEvent(event, options = {}) {
|
|
3455
|
+
const base = {
|
|
3456
|
+
type: event.type,
|
|
3457
|
+
task: sanitizeTask(event.task, options)
|
|
3458
|
+
};
|
|
3459
|
+
if (event.type === "readiness_start" || event.type === "task_start" || event.type === "control_start") return event.type === "control_start" ? {
|
|
3460
|
+
...base,
|
|
3461
|
+
knowledge: sanitizeKnowledgeReadinessReport(event.knowledge, options)
|
|
3462
|
+
} : base;
|
|
2467
3463
|
if (event.type === "readiness_end") return {
|
|
2468
3464
|
...base,
|
|
2469
3465
|
knowledge: sanitizeKnowledgeReadinessReport(event.knowledge, options)
|
|
@@ -2830,6 +3826,94 @@ const GEN_AI = {
|
|
|
2830
3826
|
outputTokens: "gen_ai.usage.output_tokens"
|
|
2831
3827
|
};
|
|
2832
3828
|
/**
|
|
3829
|
+
* Create an exporter that APPENDS spans to a local OpenInference-JSONL file, one complete span per
|
|
3830
|
+
* line, instead of posting them to a collector.
|
|
3831
|
+
*
|
|
3832
|
+
* Why this exists beside {@link createOtelExporter}: that one needs an OTLP endpoint, so a run on a
|
|
3833
|
+
* laptop, in CI, or inside a sandbox with no collector emits nothing and its per-turn shape is
|
|
3834
|
+
* simply lost. The journal records the TREE (who spawned whom, what settled, what it spent); it
|
|
3835
|
+
* does not record what happened inside a turn. A run whose tree is readable but whose turns are not
|
|
3836
|
+
* is exactly the state that made an observability gap invisible until someone went looking.
|
|
3837
|
+
*
|
|
3838
|
+
* The line shape is the one `@tangle-network/traces` reads (`spans.otlp.jsonl`) and is a standard
|
|
3839
|
+
* OpenInference representation, so the same file feeds any OpenInference tool with no conversion:
|
|
3840
|
+
* snake_case identity fields, ISO-8601 times, `parent_span_id` empty at the root, and attributes as
|
|
3841
|
+
* a plain object rather than OTLP's key/value array.
|
|
3842
|
+
*
|
|
3843
|
+
* Appends synchronously per span so a killed process keeps every span it had already finished —
|
|
3844
|
+
* matching the spawn journal's durability posture, since a trace that only survives a clean exit is
|
|
3845
|
+
* useless for the runs you most want to look at.
|
|
3846
|
+
*/
|
|
3847
|
+
function createOpenInferenceFileExporter(filePath) {
|
|
3848
|
+
const lines = [];
|
|
3849
|
+
let failed;
|
|
3850
|
+
const append = (line) => {
|
|
3851
|
+
try {
|
|
3852
|
+
appendFileSync(filePath, `${line}\n`, "utf8");
|
|
3853
|
+
} catch (error) {
|
|
3854
|
+
failed ??= error instanceof Error ? error : new Error(String(error));
|
|
3855
|
+
}
|
|
3856
|
+
};
|
|
3857
|
+
return {
|
|
3858
|
+
exportSpan(span) {
|
|
3859
|
+
append(JSON.stringify(toOpenInferenceLine(span)));
|
|
3860
|
+
lines.push("");
|
|
3861
|
+
},
|
|
3862
|
+
async flush() {
|
|
3863
|
+
if (failed) throw new Error(`OpenInference file exporter failed writing ${filePath}: ${failed.message}`);
|
|
3864
|
+
},
|
|
3865
|
+
async shutdown() {
|
|
3866
|
+
lines.length = 0;
|
|
3867
|
+
}
|
|
3868
|
+
};
|
|
3869
|
+
}
|
|
3870
|
+
/** OTLP's numeric status codes in the NAMES OpenInference requires. */
|
|
3871
|
+
const OTEL_STATUS_NAME = {
|
|
3872
|
+
0: "UNSET",
|
|
3873
|
+
1: "OK",
|
|
3874
|
+
2: "ERROR"
|
|
3875
|
+
};
|
|
3876
|
+
/** Nanosecond string -> ISO-8601, the time format an OpenInference reader parses with `Date.parse`. */
|
|
3877
|
+
function nanosToIso(nanos) {
|
|
3878
|
+
const ms = Number(BigInt(nanos) / 1000000n);
|
|
3879
|
+
return new Date(ms).toISOString();
|
|
3880
|
+
}
|
|
3881
|
+
/** OTLP's key/value attribute array -> the plain object OpenInference carries. */
|
|
3882
|
+
function attributesToObject(attributes) {
|
|
3883
|
+
const out = {};
|
|
3884
|
+
for (const a of attributes ?? []) {
|
|
3885
|
+
const v = a.value;
|
|
3886
|
+
out[a.key] = v.stringValue ?? (v.intValue !== void 0 ? Number(v.intValue) : void 0) ?? v.doubleValue ?? v.boolValue;
|
|
3887
|
+
}
|
|
3888
|
+
return out;
|
|
3889
|
+
}
|
|
3890
|
+
/** One `OtelSpan` as the OpenInference JSONL line `traces` and other OpenInference tools read. */
|
|
3891
|
+
function toOpenInferenceLine(span) {
|
|
3892
|
+
const attributes = attributesToObject(span.attributes);
|
|
3893
|
+
const kind = attributes["openinference.span.kind"];
|
|
3894
|
+
const resource = {};
|
|
3895
|
+
for (const key of ["service.name", "agent.name"]) if (attributes[key] !== void 0) resource[key] = attributes[key];
|
|
3896
|
+
return {
|
|
3897
|
+
trace_id: span.traceId,
|
|
3898
|
+
span_id: span.spanId,
|
|
3899
|
+
parent_span_id: span.parentSpanId ?? "",
|
|
3900
|
+
name: span.name,
|
|
3901
|
+
kind: typeof kind === "string" ? kind : "CHAIN",
|
|
3902
|
+
start_time: nanosToIso(span.startTimeUnixNano),
|
|
3903
|
+
end_time: nanosToIso(span.endTimeUnixNano),
|
|
3904
|
+
status: {
|
|
3905
|
+
code: OTEL_STATUS_NAME[span.status?.code ?? 0] ?? "UNSET",
|
|
3906
|
+
...span.status?.message === void 0 ? {} : { message: span.status.message }
|
|
3907
|
+
},
|
|
3908
|
+
resource: { attributes: resource },
|
|
3909
|
+
scope: {
|
|
3910
|
+
name: "agent-runtime",
|
|
3911
|
+
version: ""
|
|
3912
|
+
},
|
|
3913
|
+
attributes
|
|
3914
|
+
};
|
|
3915
|
+
}
|
|
3916
|
+
/**
|
|
2833
3917
|
* Create an OTEL exporter. Returns undefined when no endpoint is configured.
|
|
2834
3918
|
*/
|
|
2835
3919
|
function createOtelExporter(config) {
|
|
@@ -3464,30 +4548,38 @@ function workerTraceEnv(ctx) {
|
|
|
3464
4548
|
/**
|
|
3465
4549
|
* The recursion seam key. A `Scope` seeds a value of this on each child's
|
|
3466
4550
|
* `ExecutorContext.seams` so a child whose executor is a DRIVER can mount a NESTED `Scope`
|
|
3467
|
-
* over the
|
|
4551
|
+
* over the driver's reserved child allocation at `depth+1`. A leaf executor never reads it. Single-sourced
|
|
3468
4552
|
* here so the scope and the driver-executor agree on the seam without a circular import.
|
|
3469
4553
|
*/
|
|
3470
4554
|
const nestedScopeSeamKey = "nested-scope";
|
|
3471
|
-
function makeNestedScopeSeam(args, childNodeId) {
|
|
4555
|
+
function makeNestedScopeSeam(args, liveWorkerCapacity, childNodeId, childBudget, childDeadlineAtMs, deferredOwner) {
|
|
4556
|
+
const now = args.now ?? Date.now;
|
|
3472
4557
|
return {
|
|
4558
|
+
nodeId: childNodeId,
|
|
3473
4559
|
depth: args.depth,
|
|
3474
4560
|
...args.maxDepth !== void 0 ? { maxDepth: args.maxDepth } : {},
|
|
3475
4561
|
journalRoot: args.root,
|
|
3476
4562
|
mount(nestedRoot, signal) {
|
|
4563
|
+
const deadlineMs = childDeadlineAtMs === void 0 ? void 0 : Math.max(0, childDeadlineAtMs - now());
|
|
3477
4564
|
return createScope({
|
|
3478
4565
|
parentId: childNodeId,
|
|
3479
4566
|
root: nestedRoot,
|
|
3480
|
-
pool:
|
|
4567
|
+
pool: createBudgetPool({
|
|
4568
|
+
...childBudget,
|
|
4569
|
+
...deadlineMs !== void 0 ? { deadlineMs } : {}
|
|
4570
|
+
}, now),
|
|
3481
4571
|
journal: args.journal,
|
|
3482
4572
|
blobs: args.blobs,
|
|
3483
4573
|
executors: args.executors,
|
|
3484
4574
|
seams: args.seams,
|
|
3485
4575
|
depth: args.depth + 1,
|
|
3486
4576
|
...args.maxDepth !== void 0 ? { maxDepth: args.maxDepth } : {},
|
|
4577
|
+
liveWorkerCapacity,
|
|
3487
4578
|
signal,
|
|
3488
4579
|
...args.now ? { now: args.now } : {},
|
|
3489
4580
|
...args.hooks ? { hooks: args.hooks } : {},
|
|
3490
|
-
...args.workerTrace ? { workerTrace: args.workerTrace } : {}
|
|
4581
|
+
...args.workerTrace ? { workerTrace: args.workerTrace } : {},
|
|
4582
|
+
...deferredOwner.ownerMaterialization === void 0 ? {} : { ownerMaterialization: deferredOwner.ownerMaterialization }
|
|
3491
4583
|
});
|
|
3492
4584
|
}
|
|
3493
4585
|
};
|
|
@@ -3495,6 +4587,10 @@ function makeNestedScopeSeam(args, childNodeId) {
|
|
|
3495
4587
|
/** Create the reactive `Scope` a driver's `Agent.act` runs inside: spawn children on an atomically reserved conserved budget, settle via the `next()` cursor, journal for replay. */
|
|
3496
4588
|
function createScope(args) {
|
|
3497
4589
|
const children = /* @__PURE__ */ new Map();
|
|
4590
|
+
const liveWorkerCapacity = args.liveWorkerCapacity ?? {
|
|
4591
|
+
max: normalizeLiveWorkerLimit(args.maxLiveWorkers),
|
|
4592
|
+
live: 0
|
|
4593
|
+
};
|
|
3498
4594
|
let spawnOrdinal = args.resumeFrom ? args.resumeFrom.maxSpawnOrdinal + 1 : 0;
|
|
3499
4595
|
let cursorSeq = args.resumeFrom ? args.resumeFrom.maxCursorSeq + 1 : 0;
|
|
3500
4596
|
let waitOrdinal = args.resumeFrom ? args.resumeFrom.maxWaitOrdinal + 1 : 0;
|
|
@@ -3502,38 +4598,78 @@ function createScope(args) {
|
|
|
3502
4598
|
const now = args.now ?? Date.now;
|
|
3503
4599
|
const unclaimedWaits = new Map((args.resumeFrom?.waits ?? []).map((w) => [w.label, w]));
|
|
3504
4600
|
const keyed = /* @__PURE__ */ new Map();
|
|
3505
|
-
for (const [key, prior] of args.resumeFrom?.keys ?? []) if (prior.state === "completed" && prior.settled?.kind === "done")
|
|
3506
|
-
|
|
3507
|
-
|
|
3508
|
-
|
|
3509
|
-
|
|
3510
|
-
|
|
3511
|
-
|
|
3512
|
-
|
|
3513
|
-
|
|
3514
|
-
|
|
3515
|
-
|
|
4601
|
+
for (const [key, prior] of args.resumeFrom?.keys ?? []) if (prior.state === "completed" && prior.settled?.kind === "done") {
|
|
4602
|
+
if (prior.identity === void 0) continue;
|
|
4603
|
+
keyed.set(key, {
|
|
4604
|
+
state: "done",
|
|
4605
|
+
id: prior.id,
|
|
4606
|
+
identity: prior.identity,
|
|
4607
|
+
settled: prior.settled
|
|
4608
|
+
});
|
|
4609
|
+
} else if (prior.state === "down" && prior.settled?.kind === "down") {
|
|
4610
|
+
if (prior.identity === void 0) continue;
|
|
4611
|
+
keyed.set(key, {
|
|
4612
|
+
state: "down",
|
|
4613
|
+
id: prior.id,
|
|
4614
|
+
identity: prior.identity,
|
|
4615
|
+
reason: prior.settled.reason
|
|
4616
|
+
});
|
|
4617
|
+
} else if (prior.identity !== void 0) keyed.set(key, {
|
|
3516
4618
|
state: "in-doubt",
|
|
3517
|
-
id: prior.id
|
|
4619
|
+
id: prior.id,
|
|
4620
|
+
identity: prior.identity
|
|
3518
4621
|
});
|
|
3519
4622
|
/** Fold a keyed child's terminal settlement back into the registry, so a later spawn with the
|
|
3520
4623
|
* same key resolves to it (done → committed result; down → explicit retry). */
|
|
3521
4624
|
const recordKeyedSettlement = (key, settled) => {
|
|
4625
|
+
const identity = settled.handle.identity;
|
|
4626
|
+
if (identity === void 0) throw new ValidationError(`scope: keyed settlement '${key}' lost its execution identity`);
|
|
3522
4627
|
if (settled.kind === "done") keyed.set(key, {
|
|
3523
4628
|
state: "done",
|
|
3524
4629
|
id: settled.handle.id,
|
|
4630
|
+
identity,
|
|
3525
4631
|
settled
|
|
3526
4632
|
});
|
|
3527
4633
|
else keyed.set(key, {
|
|
3528
4634
|
state: "down",
|
|
3529
4635
|
id: settled.handle.id,
|
|
4636
|
+
identity,
|
|
3530
4637
|
reason: settled.reason
|
|
3531
4638
|
});
|
|
3532
4639
|
};
|
|
3533
|
-
function spawn(
|
|
4640
|
+
function spawn(agentOrFactory, rawTask, rawOpts) {
|
|
4641
|
+
if (args.signal.aborted) return {
|
|
4642
|
+
ok: false,
|
|
4643
|
+
reason: "scope-aborted"
|
|
4644
|
+
};
|
|
4645
|
+
const task = detachedSnapshot(rawTask, "scope.spawn task");
|
|
4646
|
+
const opts = detachedSnapshot(rawOpts, "scope.spawn options");
|
|
3534
4647
|
let prior;
|
|
4648
|
+
let prepared;
|
|
4649
|
+
const prepare = () => {
|
|
4650
|
+
const agent = typeof agentOrFactory === "function" ? agentOrFactory() : agentOrFactory;
|
|
4651
|
+
const rawSpec = agent.executorSpec;
|
|
4652
|
+
if (!isAgentSpec(rawSpec)) throw new ValidationError(`scope.spawn: agent "${agent.name}" exposes no \`executorSpec\` (AgentSpec) to resolve a Executor`);
|
|
4653
|
+
const spec = snapshotAgentSpec(rawSpec);
|
|
4654
|
+
return {
|
|
4655
|
+
agent,
|
|
4656
|
+
spec,
|
|
4657
|
+
identity: deriveNodeExecutionIdentity(spec, task)
|
|
4658
|
+
};
|
|
4659
|
+
};
|
|
3535
4660
|
if (opts.key !== void 0) {
|
|
3536
4661
|
const existing = keyed.get(opts.key);
|
|
4662
|
+
if (existing !== void 0) {
|
|
4663
|
+
prepared = prepare();
|
|
4664
|
+
if (!isCompleteIdentity(prepared.identity)) return {
|
|
4665
|
+
ok: false,
|
|
4666
|
+
reason: "invalid-identity"
|
|
4667
|
+
};
|
|
4668
|
+
if (!sameNodeExecutionIdentity(existing.identity, prepared.identity)) return {
|
|
4669
|
+
ok: false,
|
|
4670
|
+
reason: "key-conflict"
|
|
4671
|
+
};
|
|
4672
|
+
}
|
|
3537
4673
|
if (existing?.state === "live") return {
|
|
3538
4674
|
ok: false,
|
|
3539
4675
|
reason: "duplicate-key"
|
|
@@ -3560,74 +4696,203 @@ function createScope(args) {
|
|
|
3560
4696
|
ok: false,
|
|
3561
4697
|
reason: "depth-exceeded"
|
|
3562
4698
|
};
|
|
3563
|
-
const
|
|
3564
|
-
if (!
|
|
3565
|
-
const resolved = args.executors.resolve(spec);
|
|
3566
|
-
if (!resolved.succeeded) throw new ValidationError(`scope.spawn: ${resolved.error}`);
|
|
3567
|
-
const reservation = args.pool.reserve(opts.budget);
|
|
3568
|
-
if (!reservation.ok) return {
|
|
4699
|
+
const permit = acquireLiveWorker(liveWorkerCapacity);
|
|
4700
|
+
if (!permit.ok) return {
|
|
3569
4701
|
ok: false,
|
|
3570
|
-
reason:
|
|
4702
|
+
reason: "max-live-workers"
|
|
3571
4703
|
};
|
|
4704
|
+
let reservation;
|
|
4705
|
+
try {
|
|
4706
|
+
reservation = args.pool.reserve(opts.budget);
|
|
4707
|
+
} catch (error) {
|
|
4708
|
+
permit.release();
|
|
4709
|
+
throw error;
|
|
4710
|
+
}
|
|
4711
|
+
if (!reservation.ok) {
|
|
4712
|
+
permit.release();
|
|
4713
|
+
return {
|
|
4714
|
+
ok: false,
|
|
4715
|
+
reason: reservation.reason
|
|
4716
|
+
};
|
|
4717
|
+
}
|
|
4718
|
+
let spec;
|
|
4719
|
+
let resolved;
|
|
4720
|
+
let identity;
|
|
4721
|
+
try {
|
|
4722
|
+
prepared ??= prepare();
|
|
4723
|
+
spec = prepared.spec;
|
|
4724
|
+
identity = prepared.identity;
|
|
4725
|
+
if (opts.key !== void 0 && !isCompleteIdentity(identity)) {
|
|
4726
|
+
args.pool.reconcile(reservation.ticket, zeroSpend$1());
|
|
4727
|
+
permit.release();
|
|
4728
|
+
return {
|
|
4729
|
+
ok: false,
|
|
4730
|
+
reason: "invalid-identity"
|
|
4731
|
+
};
|
|
4732
|
+
}
|
|
4733
|
+
const floor = workerTokenFloor(spec.harness);
|
|
4734
|
+
if (floor !== null && opts.budget.maxTokens < floor) {
|
|
4735
|
+
args.pool.reconcile(reservation.ticket, zeroSpend$1());
|
|
4736
|
+
permit.release();
|
|
4737
|
+
return {
|
|
4738
|
+
ok: false,
|
|
4739
|
+
reason: "below-runtime-floor"
|
|
4740
|
+
};
|
|
4741
|
+
}
|
|
4742
|
+
const outcome = args.executors.resolve(spec);
|
|
4743
|
+
if (!outcome.succeeded) throw new ValidationError(`scope.spawn: ${outcome.error}`);
|
|
4744
|
+
resolved = outcome;
|
|
4745
|
+
} catch (error) {
|
|
4746
|
+
args.pool.reconcile(reservation.ticket, zeroSpend$1());
|
|
4747
|
+
permit.release();
|
|
4748
|
+
throw error;
|
|
4749
|
+
}
|
|
4750
|
+
let cascadeAbort;
|
|
4751
|
+
let clearChildDeadline;
|
|
3572
4752
|
try {
|
|
3573
4753
|
const ordinal = spawnOrdinal++;
|
|
3574
4754
|
const id = `${args.parentId}:s${ordinal}`;
|
|
3575
|
-
const
|
|
3576
|
-
const
|
|
3577
|
-
|
|
4755
|
+
const attemptId = newExecutionAttemptId(id);
|
|
4756
|
+
const startedAt = now();
|
|
4757
|
+
const childDeadlineAtMs = boundedChildDeadlineAt(args.pool.readout().deadlineMs, opts.budget.deadlineMs, startedAt);
|
|
4758
|
+
const controller = new AbortController();
|
|
4759
|
+
cascadeAbort = () => controller.abort(args.signal.reason);
|
|
4760
|
+
if (args.signal.aborted) controller.abort(args.signal.reason);
|
|
3578
4761
|
else args.signal.addEventListener("abort", cascadeAbort, { once: true });
|
|
4762
|
+
if (childDeadlineAtMs !== void 0) clearChildDeadline = armDeadlineTimer(Math.max(0, childDeadlineAtMs - now()), () => controller.abort("child deadline exceeded"));
|
|
4763
|
+
const deferredOwner = {};
|
|
3579
4764
|
const workerTrace = args.workerTrace?.(args.parentId);
|
|
3580
4765
|
const ctx = {
|
|
3581
|
-
signal:
|
|
4766
|
+
signal: controller.signal,
|
|
4767
|
+
node: {
|
|
4768
|
+
rootId: args.root,
|
|
4769
|
+
parentId: args.parentId,
|
|
4770
|
+
nodeId: id,
|
|
4771
|
+
attemptId,
|
|
4772
|
+
...identity ? { identity } : {}
|
|
4773
|
+
},
|
|
3582
4774
|
seams: {
|
|
3583
4775
|
...args.seams,
|
|
3584
|
-
[nestedScopeSeamKey]: makeNestedScopeSeam(args, id),
|
|
4776
|
+
[nestedScopeSeamKey]: makeNestedScopeSeam(args, liveWorkerCapacity, id, opts.budget, childDeadlineAtMs, deferredOwner),
|
|
3585
4777
|
...workerTrace ? { [workerTraceSeamKey]: workerTrace } : {}
|
|
3586
4778
|
}
|
|
3587
4779
|
};
|
|
3588
4780
|
const executor = resolved.value(spec, ctx);
|
|
4781
|
+
const ownedTreeRoot = runtimeOwnedNestedDriverTreeRoot(executor, args.root, id);
|
|
3589
4782
|
const handle = {
|
|
3590
4783
|
id,
|
|
3591
4784
|
label: opts.label,
|
|
4785
|
+
...opts.assignmentId === void 0 ? {} : { assignmentId: opts.assignmentId },
|
|
4786
|
+
...identity ? { identity } : {},
|
|
3592
4787
|
get status() {
|
|
3593
4788
|
return children.get(id)?.status ?? "cancelled";
|
|
3594
4789
|
},
|
|
4790
|
+
get materialization() {
|
|
4791
|
+
return children.get(id)?.materialization;
|
|
4792
|
+
},
|
|
4793
|
+
get executionBindings() {
|
|
4794
|
+
const bindings = children.get(id)?.executionBindings;
|
|
4795
|
+
return bindings && bindings.length > 0 ? Object.freeze([...bindings]) : void 0;
|
|
4796
|
+
},
|
|
3595
4797
|
abort(reason) {
|
|
3596
|
-
|
|
4798
|
+
controller.abort(reason);
|
|
3597
4799
|
}
|
|
3598
4800
|
};
|
|
3599
4801
|
const live = {
|
|
3600
4802
|
id,
|
|
3601
4803
|
status: "acquiring",
|
|
3602
4804
|
runtime: executor.runtime,
|
|
4805
|
+
...ownedTreeRoot === void 0 ? {} : { ownedTreeRoot },
|
|
4806
|
+
...identity ? { identity } : {},
|
|
3603
4807
|
budget: opts.budget,
|
|
3604
4808
|
label: opts.label,
|
|
4809
|
+
...opts.assignmentId === void 0 ? {} : { assignmentId: opts.assignmentId },
|
|
3605
4810
|
...opts.key !== void 0 ? { key: opts.key } : {},
|
|
3606
|
-
spent: zeroSpend(),
|
|
4811
|
+
spent: zeroSpend$1(),
|
|
3607
4812
|
settled: void 0,
|
|
3608
4813
|
delivered: false,
|
|
3609
4814
|
executorDone: false,
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
|
|
4815
|
+
cleanupConfirmed: false,
|
|
4816
|
+
executionBindings: [],
|
|
4817
|
+
startedAt,
|
|
4818
|
+
lastActivityAt: startedAt,
|
|
4819
|
+
...executor.deliver ? { deliver: (message) => executor.deliver?.(message) !== false } : {},
|
|
3613
4820
|
...executor.progress ? { readProgress: executor.progress.bind(executor) } : {},
|
|
3614
4821
|
...executor.traceSource ? { readTraceSource: executor.traceSource.bind(executor) } : {}
|
|
3615
4822
|
};
|
|
3616
4823
|
children.set(id, live);
|
|
3617
4824
|
if (opts.key !== void 0) keyed.set(opts.key, {
|
|
3618
4825
|
state: "live",
|
|
3619
|
-
id
|
|
4826
|
+
id,
|
|
4827
|
+
identity
|
|
3620
4828
|
});
|
|
3621
|
-
args.journal.appendEvent(args.root, {
|
|
4829
|
+
const materializationCommitted = args.journal.appendEvent(args.root, {
|
|
3622
4830
|
kind: "spawned",
|
|
3623
4831
|
id,
|
|
3624
4832
|
parent: args.parentId,
|
|
3625
4833
|
label: opts.label,
|
|
3626
4834
|
...opts.key !== void 0 ? { key: opts.key } : {},
|
|
4835
|
+
...opts.assignmentId === void 0 ? {} : { assignmentId: opts.assignmentId },
|
|
3627
4836
|
budget: opts.budget,
|
|
3628
4837
|
runtime: executor.runtime,
|
|
4838
|
+
...ownedTreeRoot === void 0 ? {} : { ownedTreeRoot },
|
|
4839
|
+
...identity ? { identity } : {},
|
|
3629
4840
|
seq: ordinal,
|
|
3630
4841
|
at: new Date(now()).toISOString()
|
|
4842
|
+
}).then(async () => {
|
|
4843
|
+
const profileDigest = identity?.profileDigest ?? authoredProfileDigest(spec.profile);
|
|
4844
|
+
let receipt;
|
|
4845
|
+
let binding;
|
|
4846
|
+
const declaration = runtimeOwnedExecutorMaterialization(executor);
|
|
4847
|
+
const deferredRuntime = runtimeOwnedDeferredExecutorRuntime(executor);
|
|
4848
|
+
if (deferredRuntime !== void 0) {
|
|
4849
|
+
deferredOwner.ownerMaterialization = {
|
|
4850
|
+
runtime: deferredRuntime,
|
|
4851
|
+
authoredProfile: spec.profile,
|
|
4852
|
+
attemptId,
|
|
4853
|
+
journalRoot: args.root,
|
|
4854
|
+
nodeId: id,
|
|
4855
|
+
requiredKnown: true,
|
|
4856
|
+
onReceipt(materialization, executionBinding) {
|
|
4857
|
+
live.runtime = materialization.runtime;
|
|
4858
|
+
live.materialization = materialization;
|
|
4859
|
+
live.executionBindings.push(executionBinding);
|
|
4860
|
+
}
|
|
4861
|
+
};
|
|
4862
|
+
return;
|
|
4863
|
+
}
|
|
4864
|
+
if (declaration === void 0) {
|
|
4865
|
+
receipt = unknownMaterializationReceipt({
|
|
4866
|
+
...profileDigest === void 0 ? {} : { authoredProfileDigest: profileDigest },
|
|
4867
|
+
runtime: executor.runtime,
|
|
4868
|
+
reason: "executor-did-not-report"
|
|
4869
|
+
});
|
|
4870
|
+
binding = unknownExecutionBindingReceipt(receipt, attemptId, "executor-did-not-report");
|
|
4871
|
+
} else try {
|
|
4872
|
+
if (profileDigest === void 0) throw new ValidationError("scope.spawn: a known materialization requires a canonical authored profile");
|
|
4873
|
+
receipt = knownMaterializationReceipt({
|
|
4874
|
+
authoredProfileDigest: profileDigest,
|
|
4875
|
+
runtime: executor.runtime,
|
|
4876
|
+
declaration
|
|
4877
|
+
});
|
|
4878
|
+
const reportedBinding = runtimeOwnedExecutorExecutionBinding(executor);
|
|
4879
|
+
if (reportedBinding === void 0 || reportedBinding.attemptId !== attemptId) throw new ValidationError("scope.spawn: trusted executor did not bind the kernel-minted attempt id");
|
|
4880
|
+
binding = knownExecutionBindingReceipt(receipt, reportedBinding);
|
|
4881
|
+
} catch (error) {
|
|
4882
|
+
receipt = unknownMaterializationReceipt({
|
|
4883
|
+
...profileDigest === void 0 ? {} : { authoredProfileDigest: profileDigest },
|
|
4884
|
+
runtime: executor.runtime,
|
|
4885
|
+
reason: "invalid-executor-report"
|
|
4886
|
+
});
|
|
4887
|
+
binding = unknownExecutionBindingReceipt(receipt, attemptId, "invalid-executor-report");
|
|
4888
|
+
await appendNodeMaterialization(args, id, ordinal, receipt, binding, now);
|
|
4889
|
+
live.materialization = receipt;
|
|
4890
|
+
live.executionBindings.push(binding);
|
|
4891
|
+
throw new ValidationError(`scope.spawn: executor ${JSON.stringify(executor.runtime)} returned invalid materialization evidence`, { cause: error });
|
|
4892
|
+
}
|
|
4893
|
+
await appendNodeMaterialization(args, id, ordinal, receipt, binding, now);
|
|
4894
|
+
live.materialization = receipt;
|
|
4895
|
+
live.executionBindings.push(binding);
|
|
3631
4896
|
});
|
|
3632
4897
|
notifyRuntimeHookEvent(args.hooks, {
|
|
3633
4898
|
id: `${id}:spawn`,
|
|
@@ -3640,16 +4905,20 @@ function createScope(args) {
|
|
|
3640
4905
|
payload: {
|
|
3641
4906
|
childId: id,
|
|
3642
4907
|
label: opts.label,
|
|
4908
|
+
...opts.assignmentId === void 0 ? {} : { assignmentId: opts.assignmentId },
|
|
3643
4909
|
runtime: executor.runtime,
|
|
4910
|
+
...identity ? { identity } : {},
|
|
3644
4911
|
budget: opts.budget,
|
|
3645
4912
|
depth: args.depth
|
|
3646
4913
|
}
|
|
3647
4914
|
}, { signal: args.signal });
|
|
3648
|
-
live.settled = runChild(live, executor,
|
|
4915
|
+
live.settled = runChild(live, executor, controller, task, opts, args.pool, reservation.ticket, args.blobs, now, materializationCommitted, childDeadlineAtMs).then((s) => {
|
|
3649
4916
|
live.resolved = s;
|
|
3650
4917
|
return s;
|
|
3651
4918
|
}).finally(() => {
|
|
3652
|
-
|
|
4919
|
+
if (live.cleanupConfirmed) permit.release();
|
|
4920
|
+
clearChildDeadline?.();
|
|
4921
|
+
if (cascadeAbort) args.signal.removeEventListener("abort", cascadeAbort);
|
|
3653
4922
|
});
|
|
3654
4923
|
return {
|
|
3655
4924
|
ok: true,
|
|
@@ -3657,7 +4926,10 @@ function createScope(args) {
|
|
|
3657
4926
|
...prior ? { prior } : {}
|
|
3658
4927
|
};
|
|
3659
4928
|
} catch (err) {
|
|
3660
|
-
|
|
4929
|
+
permit.release();
|
|
4930
|
+
clearChildDeadline?.();
|
|
4931
|
+
if (cascadeAbort) args.signal.removeEventListener("abort", cascadeAbort);
|
|
4932
|
+
args.pool.reconcile(reservation.ticket, zeroSpend$1());
|
|
3661
4933
|
throw err;
|
|
3662
4934
|
}
|
|
3663
4935
|
}
|
|
@@ -3692,9 +4964,10 @@ function createScope(args) {
|
|
|
3692
4964
|
}
|
|
3693
4965
|
}
|
|
3694
4966
|
function send(nodeId, msg) {
|
|
4967
|
+
if (args.signal.aborted) return false;
|
|
3695
4968
|
const child = children.get(nodeId);
|
|
3696
4969
|
if (!child || child.delivered || !child.deliver) return false;
|
|
3697
|
-
child.deliver(msg);
|
|
4970
|
+
if (!(child.deliver(msg) !== false)) return false;
|
|
3698
4971
|
child.lastActivityAt = now();
|
|
3699
4972
|
return true;
|
|
3700
4973
|
}
|
|
@@ -3709,6 +4982,10 @@ function createScope(args) {
|
|
|
3709
4982
|
* extend that ceiling either (that would make a wait override a budget guard).
|
|
3710
4983
|
*/
|
|
3711
4984
|
function wait(spec, opts) {
|
|
4985
|
+
if (args.signal.aborted) return {
|
|
4986
|
+
ok: false,
|
|
4987
|
+
reason: "deadline-exceeded"
|
|
4988
|
+
};
|
|
3712
4989
|
if (validateWaitSpec(spec) !== null) return {
|
|
3713
4990
|
ok: false,
|
|
3714
4991
|
reason: "invalid-spec"
|
|
@@ -3751,20 +5028,23 @@ function createScope(args) {
|
|
|
3751
5028
|
maxTokens: 0
|
|
3752
5029
|
},
|
|
3753
5030
|
label: opts.label,
|
|
3754
|
-
spent: zeroSpend(),
|
|
5031
|
+
spent: zeroSpend$1(),
|
|
3755
5032
|
settled: void 0,
|
|
3756
5033
|
delivered: false,
|
|
3757
5034
|
executorDone: false,
|
|
5035
|
+
cleanupConfirmed: true,
|
|
5036
|
+
executionBindings: [],
|
|
3758
5037
|
startedAt: armedAt,
|
|
3759
5038
|
lastActivityAt: now(),
|
|
3760
5039
|
wait: {
|
|
3761
5040
|
spec: effectiveSpec,
|
|
3762
5041
|
armedAt,
|
|
3763
|
-
label: opts.label
|
|
5042
|
+
label: opts.label,
|
|
5043
|
+
armCommitted: adopted !== void 0
|
|
3764
5044
|
}
|
|
3765
5045
|
};
|
|
3766
5046
|
children.set(id, live);
|
|
3767
|
-
|
|
5047
|
+
live.settled = (adopted ? Promise.resolve() : args.journal.appendEvent(args.root, {
|
|
3768
5048
|
kind: "waiting",
|
|
3769
5049
|
id,
|
|
3770
5050
|
parent: args.parentId,
|
|
@@ -3773,33 +5053,35 @@ function createScope(args) {
|
|
|
3773
5053
|
armedAt,
|
|
3774
5054
|
seq: ordinal,
|
|
3775
5055
|
at: new Date(now()).toISOString()
|
|
3776
|
-
})
|
|
3777
|
-
|
|
3778
|
-
|
|
3779
|
-
|
|
3780
|
-
|
|
3781
|
-
|
|
3782
|
-
|
|
3783
|
-
|
|
3784
|
-
|
|
3785
|
-
|
|
3786
|
-
|
|
5056
|
+
})).then(() => {
|
|
5057
|
+
if (live.wait) live.wait.armCommitted = true;
|
|
5058
|
+
notifyRuntimeHookEvent(args.hooks, {
|
|
5059
|
+
id: `${id}:waiting`,
|
|
5060
|
+
runId: args.root,
|
|
5061
|
+
target: "agent.spawn",
|
|
5062
|
+
phase: "after",
|
|
5063
|
+
timestamp: now(),
|
|
5064
|
+
stepIndex: ordinal,
|
|
5065
|
+
parentId: args.parentId,
|
|
5066
|
+
payload: {
|
|
5067
|
+
childId: id,
|
|
5068
|
+
label: opts.label,
|
|
5069
|
+
runtime: "wait",
|
|
5070
|
+
wait: effectiveSpec,
|
|
5071
|
+
armedAt,
|
|
5072
|
+
resumed: adopted !== void 0
|
|
5073
|
+
}
|
|
5074
|
+
}, { signal: args.signal });
|
|
5075
|
+
return runWait({
|
|
5076
|
+
spec: effectiveSpec,
|
|
3787
5077
|
label: opts.label,
|
|
3788
|
-
runtime: "wait",
|
|
3789
|
-
wait: effectiveSpec,
|
|
3790
5078
|
armedAt,
|
|
3791
|
-
resumed: adopted !== void 0
|
|
3792
|
-
|
|
3793
|
-
|
|
3794
|
-
|
|
3795
|
-
|
|
3796
|
-
|
|
3797
|
-
armedAt,
|
|
3798
|
-
resumed: adopted !== void 0,
|
|
3799
|
-
signal: waitAbort.signal,
|
|
3800
|
-
...args.probes ? { probes: args.probes } : {},
|
|
3801
|
-
now,
|
|
3802
|
-
...args.waitSleep ? { sleep: args.waitSleep } : {}
|
|
5079
|
+
resumed: adopted !== void 0,
|
|
5080
|
+
signal: waitAbort.signal,
|
|
5081
|
+
...args.probes ? { probes: args.probes } : {},
|
|
5082
|
+
now,
|
|
5083
|
+
...args.waitSleep ? { sleep: args.waitSleep } : {}
|
|
5084
|
+
});
|
|
3803
5085
|
}).then(async (resolution) => {
|
|
3804
5086
|
live.executorDone = true;
|
|
3805
5087
|
live.lastActivityAt = now();
|
|
@@ -3807,7 +5089,11 @@ function createScope(args) {
|
|
|
3807
5089
|
kind: "down",
|
|
3808
5090
|
reason: resolution.reason,
|
|
3809
5091
|
infra: false,
|
|
3810
|
-
restartCount: 0
|
|
5092
|
+
restartCount: 0,
|
|
5093
|
+
trace: {
|
|
5094
|
+
status: "unavailable",
|
|
5095
|
+
reason: "not-an-executor"
|
|
5096
|
+
}
|
|
3811
5097
|
};
|
|
3812
5098
|
const outRef = contentAddress(resolution.outcome);
|
|
3813
5099
|
await args.blobs.put(outRef, resolution.outcome);
|
|
@@ -3815,7 +5101,11 @@ function createScope(args) {
|
|
|
3815
5101
|
kind: "done",
|
|
3816
5102
|
out: resolution.outcome,
|
|
3817
5103
|
outRef,
|
|
3818
|
-
spent: zeroSpend()
|
|
5104
|
+
spent: zeroSpend$1(),
|
|
5105
|
+
trace: {
|
|
5106
|
+
status: "unavailable",
|
|
5107
|
+
reason: "not-an-executor"
|
|
5108
|
+
}
|
|
3819
5109
|
};
|
|
3820
5110
|
}).catch((err) => {
|
|
3821
5111
|
live.executorDone = true;
|
|
@@ -3823,7 +5113,11 @@ function createScope(args) {
|
|
|
3823
5113
|
kind: "down",
|
|
3824
5114
|
reason: errMessage(err),
|
|
3825
5115
|
infra: true,
|
|
3826
|
-
restartCount: 0
|
|
5116
|
+
restartCount: 0,
|
|
5117
|
+
trace: {
|
|
5118
|
+
status: "unavailable",
|
|
5119
|
+
reason: "not-an-executor"
|
|
5120
|
+
}
|
|
3827
5121
|
};
|
|
3828
5122
|
}).then((s) => {
|
|
3829
5123
|
live.resolved = s;
|
|
@@ -3868,8 +5162,14 @@ function createScope(args) {
|
|
|
3868
5162
|
}
|
|
3869
5163
|
}
|
|
3870
5164
|
async function meter(spend, detail) {
|
|
5165
|
+
if (args.signal.aborted) throw new ValidationError("scope.meter: cannot record new driver work after scope abort");
|
|
3871
5166
|
const seq = meterSeq++;
|
|
3872
|
-
|
|
5167
|
+
let observeError;
|
|
5168
|
+
try {
|
|
5169
|
+
args.pool.observe(spend);
|
|
5170
|
+
} catch (error) {
|
|
5171
|
+
observeError = error;
|
|
5172
|
+
}
|
|
3873
5173
|
await args.journal.appendEvent(args.root, {
|
|
3874
5174
|
kind: "metered",
|
|
3875
5175
|
id: args.parentId,
|
|
@@ -3889,6 +5189,7 @@ function createScope(args) {
|
|
|
3889
5189
|
...detail ?? {}
|
|
3890
5190
|
}
|
|
3891
5191
|
}, { signal: args.signal });
|
|
5192
|
+
if (observeError !== void 0) throw observeError;
|
|
3892
5193
|
}
|
|
3893
5194
|
const resume = args.resumeFrom ? {
|
|
3894
5195
|
settled: args.resumeFrom.settled,
|
|
@@ -3897,7 +5198,7 @@ function createScope(args) {
|
|
|
3897
5198
|
keys: args.resumeFrom.keys,
|
|
3898
5199
|
priorSpend: args.resumeFrom.priorSpend
|
|
3899
5200
|
} : void 0;
|
|
3900
|
-
|
|
5201
|
+
const scope = {
|
|
3901
5202
|
spawn,
|
|
3902
5203
|
next,
|
|
3903
5204
|
nextResolved,
|
|
@@ -3913,8 +5214,177 @@ function createScope(args) {
|
|
|
3913
5214
|
},
|
|
3914
5215
|
get budget() {
|
|
3915
5216
|
return args.pool.readout();
|
|
5217
|
+
},
|
|
5218
|
+
get workerCapacity() {
|
|
5219
|
+
return {
|
|
5220
|
+
live: liveWorkerCapacity.live,
|
|
5221
|
+
freeSlots: freeSlots(liveWorkerCapacity.live, liveWorkerCapacity.max)
|
|
5222
|
+
};
|
|
3916
5223
|
}
|
|
3917
5224
|
};
|
|
5225
|
+
if (args.ownerMaterialization !== void 0) {
|
|
5226
|
+
const authoredProfile = args.ownerMaterialization.authoredProfile === void 0 ? void 0 : detachedSnapshot(args.ownerMaterialization.authoredProfile, "scope owner authored profile");
|
|
5227
|
+
ownerMaterializationStates.set(scope, {
|
|
5228
|
+
journal: args.journal,
|
|
5229
|
+
root: args.ownerMaterialization.journalRoot ?? args.root,
|
|
5230
|
+
nodeId: args.ownerMaterialization.nodeId ?? args.parentId,
|
|
5231
|
+
runtime: args.ownerMaterialization.runtime,
|
|
5232
|
+
attemptId: args.ownerMaterialization.attemptId,
|
|
5233
|
+
...authoredProfile === void 0 ? {} : { authoredProfile },
|
|
5234
|
+
...authoredProfile === void 0 ? {} : { authoredProfileDigest: authoredProfileDigest(authoredProfile) },
|
|
5235
|
+
...args.ownerMaterialization.prior === void 0 ? {} : { prior: args.ownerMaterialization.prior },
|
|
5236
|
+
requiredKnown: args.ownerMaterialization.requiredKnown === true,
|
|
5237
|
+
...args.ownerMaterialization.onReceipt === void 0 ? {} : { onReceipt: args.ownerMaterialization.onReceipt },
|
|
5238
|
+
now,
|
|
5239
|
+
receipt: args.ownerMaterialization.prior,
|
|
5240
|
+
bindingPublished: false,
|
|
5241
|
+
publishedThisProcess: false
|
|
5242
|
+
});
|
|
5243
|
+
}
|
|
5244
|
+
return scope;
|
|
5245
|
+
}
|
|
5246
|
+
const ownerMaterializationStates = /* @__PURE__ */ new WeakMap();
|
|
5247
|
+
/**
|
|
5248
|
+
* @internal Publish exact root-manager materialization from a runtime-owned adapter after dynamic
|
|
5249
|
+
* attachments exist and before its executor starts. This is deliberately a module function backed
|
|
5250
|
+
* by a private WeakMap, not a Scope method an Agent can call.
|
|
5251
|
+
*/
|
|
5252
|
+
async function recordScopeOwnerMaterialization(scope, runtime, declaration, bindingInput) {
|
|
5253
|
+
const state = ownerMaterializationState(scope);
|
|
5254
|
+
if (runtime !== state.runtime) {
|
|
5255
|
+
await rejectOwnerMaterialization(state);
|
|
5256
|
+
throw new ValidationError(`scope owner materialization runtime ${JSON.stringify(runtime)} does not match ${JSON.stringify(state.runtime)}`);
|
|
5257
|
+
}
|
|
5258
|
+
if (state.authoredProfileDigest === void 0) {
|
|
5259
|
+
await rejectOwnerMaterialization(state);
|
|
5260
|
+
throw new ValidationError("scope owner materialization requires an exact authored profile");
|
|
5261
|
+
}
|
|
5262
|
+
let receipt;
|
|
5263
|
+
let binding;
|
|
5264
|
+
try {
|
|
5265
|
+
if (bindingInput.attemptId !== state.attemptId) throw new ValidationError("scope owner execution binding does not use the kernel-minted attempt id");
|
|
5266
|
+
if (canonicalCandidateDigest(declaration.effectiveProfile) !== state.authoredProfileDigest) throw new ValidationError("scope owner stable effective profile conflicts with its admitted authored profile");
|
|
5267
|
+
receipt = knownMaterializationReceipt({
|
|
5268
|
+
authoredProfileDigest: state.authoredProfileDigest,
|
|
5269
|
+
runtime,
|
|
5270
|
+
declaration
|
|
5271
|
+
});
|
|
5272
|
+
binding = knownExecutionBindingReceipt(receipt, bindingInput);
|
|
5273
|
+
} catch (error) {
|
|
5274
|
+
await rejectOwnerMaterialization(state);
|
|
5275
|
+
throw new ValidationError("scope owner returned invalid materialization evidence", { cause: error });
|
|
5276
|
+
}
|
|
5277
|
+
if (state.prior !== void 0) {
|
|
5278
|
+
if (canonicalCandidateDigest(state.prior) !== canonicalCandidateDigest(receipt)) {
|
|
5279
|
+
await rejectOwnerMaterialization(state);
|
|
5280
|
+
throw new ValidationError("scope owner materialization changed across resume; backend, model, execution identity, and plan must match");
|
|
5281
|
+
}
|
|
5282
|
+
state.receipt = state.prior;
|
|
5283
|
+
await appendOwnerBinding(state, binding);
|
|
5284
|
+
state.onReceipt?.(state.prior, binding);
|
|
5285
|
+
state.publishedThisProcess = true;
|
|
5286
|
+
return;
|
|
5287
|
+
}
|
|
5288
|
+
if (state.receipt !== void 0) throw new ValidationError("scope owner materialization was already recorded");
|
|
5289
|
+
await appendOwnerMaterialization(state, receipt, binding);
|
|
5290
|
+
state.onReceipt?.(receipt, binding);
|
|
5291
|
+
state.publishedThisProcess = true;
|
|
5292
|
+
}
|
|
5293
|
+
/** @internal Kernel identity for constructing the exact deferred owner executor. */
|
|
5294
|
+
function scopeOwnerExecutorNodeContext(scope) {
|
|
5295
|
+
const state = ownerMaterializationState(scope);
|
|
5296
|
+
return Object.freeze({
|
|
5297
|
+
rootId: state.root,
|
|
5298
|
+
parentId: state.nodeId,
|
|
5299
|
+
nodeId: state.nodeId,
|
|
5300
|
+
attemptId: state.attemptId
|
|
5301
|
+
});
|
|
5302
|
+
}
|
|
5303
|
+
/** @internal Ensure a deferred root that never published evidence remains visibly unknown. */
|
|
5304
|
+
async function finalizeScopeOwnerMaterialization(scope) {
|
|
5305
|
+
const state = ownerMaterializationStates.get(scope);
|
|
5306
|
+
if (state === void 0 || state.publishedThisProcess) return;
|
|
5307
|
+
if (state.prior !== void 0) {
|
|
5308
|
+
await appendUnknownOwnerBinding(state, state.prior, "root-agent-did-not-report");
|
|
5309
|
+
throw new ValidationError("resumed scope owner did not re-attest its prior materialization before execution");
|
|
5310
|
+
}
|
|
5311
|
+
if (state.receipt !== void 0) return;
|
|
5312
|
+
const receipt = unknownMaterializationReceipt({
|
|
5313
|
+
...state.authoredProfileDigest === void 0 ? {} : { authoredProfileDigest: state.authoredProfileDigest },
|
|
5314
|
+
runtime: state.runtime,
|
|
5315
|
+
reason: "root-agent-did-not-report"
|
|
5316
|
+
});
|
|
5317
|
+
const binding = unknownExecutionBindingReceipt(receipt, state.attemptId, "root-agent-did-not-report");
|
|
5318
|
+
await appendOwnerMaterialization(state, receipt, binding);
|
|
5319
|
+
state.onReceipt?.(receipt, binding);
|
|
5320
|
+
if (state.requiredKnown) throw new ValidationError("runtime-owned scope owner did not publish materialization before completing");
|
|
5321
|
+
}
|
|
5322
|
+
function ownerMaterializationState(scope) {
|
|
5323
|
+
const state = ownerMaterializationStates.get(scope);
|
|
5324
|
+
if (state === void 0) throw new ValidationError("scope has no deferred runtime-owned root materialization channel");
|
|
5325
|
+
return state;
|
|
5326
|
+
}
|
|
5327
|
+
async function rejectOwnerMaterialization(state) {
|
|
5328
|
+
if (state.bindingPublished) return;
|
|
5329
|
+
if (state.prior !== void 0) {
|
|
5330
|
+
await appendUnknownOwnerBinding(state, state.prior, "invalid-executor-report");
|
|
5331
|
+
return;
|
|
5332
|
+
}
|
|
5333
|
+
if (state.receipt !== void 0) {
|
|
5334
|
+
await appendUnknownOwnerBinding(state, state.receipt, "invalid-executor-report");
|
|
5335
|
+
return;
|
|
5336
|
+
}
|
|
5337
|
+
const receipt = unknownMaterializationReceipt({
|
|
5338
|
+
...state.authoredProfileDigest === void 0 ? {} : { authoredProfileDigest: state.authoredProfileDigest },
|
|
5339
|
+
runtime: state.runtime,
|
|
5340
|
+
reason: "invalid-executor-report"
|
|
5341
|
+
});
|
|
5342
|
+
const binding = unknownExecutionBindingReceipt(receipt, state.attemptId, "invalid-executor-report");
|
|
5343
|
+
await appendOwnerMaterialization(state, receipt, binding);
|
|
5344
|
+
state.onReceipt?.(receipt, binding);
|
|
5345
|
+
}
|
|
5346
|
+
async function appendOwnerMaterialization(state, receipt, binding) {
|
|
5347
|
+
await state.journal.appendEvent(state.root, {
|
|
5348
|
+
kind: "materialized",
|
|
5349
|
+
id: state.nodeId,
|
|
5350
|
+
receipt,
|
|
5351
|
+
seq: 0,
|
|
5352
|
+
at: new Date(state.now()).toISOString()
|
|
5353
|
+
});
|
|
5354
|
+
state.receipt = receipt;
|
|
5355
|
+
await appendOwnerBinding(state, binding);
|
|
5356
|
+
}
|
|
5357
|
+
async function appendOwnerBinding(state, binding) {
|
|
5358
|
+
await state.journal.appendEvent(state.root, {
|
|
5359
|
+
kind: "execution-bound",
|
|
5360
|
+
id: state.nodeId,
|
|
5361
|
+
binding,
|
|
5362
|
+
seq: 0,
|
|
5363
|
+
at: new Date(state.now()).toISOString()
|
|
5364
|
+
});
|
|
5365
|
+
state.bindingPublished = true;
|
|
5366
|
+
}
|
|
5367
|
+
async function appendUnknownOwnerBinding(state, receipt, reason) {
|
|
5368
|
+
const binding = unknownExecutionBindingReceipt(receipt, state.attemptId, reason);
|
|
5369
|
+
await appendOwnerBinding(state, binding);
|
|
5370
|
+
state.onReceipt?.(receipt, binding);
|
|
5371
|
+
}
|
|
5372
|
+
async function appendNodeMaterialization(args, id, seq, receipt, binding, now) {
|
|
5373
|
+
const at = new Date(now()).toISOString();
|
|
5374
|
+
await args.journal.appendEvent(args.root, {
|
|
5375
|
+
kind: "materialized",
|
|
5376
|
+
id,
|
|
5377
|
+
receipt,
|
|
5378
|
+
seq,
|
|
5379
|
+
at
|
|
5380
|
+
});
|
|
5381
|
+
await args.journal.appendEvent(args.root, {
|
|
5382
|
+
kind: "execution-bound",
|
|
5383
|
+
id,
|
|
5384
|
+
binding,
|
|
5385
|
+
seq,
|
|
5386
|
+
at
|
|
5387
|
+
});
|
|
3918
5388
|
}
|
|
3919
5389
|
/** Await whichever pending child settles first, returning the child (its `resolved` is set
|
|
3920
5390
|
* by the time this resolves because `runChild`'s `.then` sets it before the promise
|
|
@@ -3927,30 +5397,36 @@ async function raceFirstSettled(pending) {
|
|
|
3927
5397
|
async function finalizeSettlement(child, settlement, seq, args, now) {
|
|
3928
5398
|
const handle = frozenHandle(child);
|
|
3929
5399
|
if (child.wait) return finalizeWait(child, settlement, seq, args, now, handle);
|
|
5400
|
+
const settledAt = now();
|
|
5401
|
+
child.settledAt = settledAt;
|
|
5402
|
+
const at = new Date(settledAt).toISOString();
|
|
3930
5403
|
if (settlement.kind === "down") {
|
|
3931
5404
|
child.status = "failed";
|
|
5405
|
+
child.trace = settlement.trace;
|
|
3932
5406
|
await args.journal.appendEvent(args.root, {
|
|
3933
5407
|
kind: "settled",
|
|
3934
5408
|
id: child.id,
|
|
3935
5409
|
status: "down",
|
|
3936
5410
|
spent: child.spent,
|
|
3937
5411
|
infra: settlement.infra,
|
|
5412
|
+
reason: settlement.reason,
|
|
5413
|
+
trace: settlement.trace,
|
|
3938
5414
|
seq,
|
|
3939
|
-
at
|
|
5415
|
+
at
|
|
3940
5416
|
});
|
|
3941
5417
|
if (settlement.metered) await args.journal.appendEvent(args.root, {
|
|
3942
5418
|
kind: "metered",
|
|
3943
5419
|
id: child.id,
|
|
3944
5420
|
spend: settlement.metered,
|
|
3945
5421
|
seq,
|
|
3946
|
-
at
|
|
5422
|
+
at
|
|
3947
5423
|
});
|
|
3948
5424
|
notifyRuntimeHookEvent(args.hooks, {
|
|
3949
5425
|
id: `${child.id}:settled`,
|
|
3950
5426
|
runId: args.root,
|
|
3951
5427
|
target: "agent.child",
|
|
3952
5428
|
phase: "after",
|
|
3953
|
-
timestamp:
|
|
5429
|
+
timestamp: settledAt,
|
|
3954
5430
|
stepIndex: seq,
|
|
3955
5431
|
parentId: args.parentId,
|
|
3956
5432
|
payload: {
|
|
@@ -3967,12 +5443,15 @@ async function finalizeSettlement(child, settlement, seq, args, now) {
|
|
|
3967
5443
|
reason: settlement.reason,
|
|
3968
5444
|
infra: settlement.infra,
|
|
3969
5445
|
restartCount: settlement.restartCount,
|
|
5446
|
+
trace: settlement.trace,
|
|
5447
|
+
settledAt,
|
|
3970
5448
|
seq
|
|
3971
5449
|
};
|
|
3972
5450
|
}
|
|
3973
5451
|
child.status = "done";
|
|
3974
5452
|
child.outRef = settlement.outRef;
|
|
3975
5453
|
child.spent = settlement.spent;
|
|
5454
|
+
child.trace = settlement.trace;
|
|
3976
5455
|
await args.journal.appendEvent(args.root, {
|
|
3977
5456
|
kind: "settled",
|
|
3978
5457
|
id: child.id,
|
|
@@ -3980,22 +5459,23 @@ async function finalizeSettlement(child, settlement, seq, args, now) {
|
|
|
3980
5459
|
outRef: settlement.outRef,
|
|
3981
5460
|
...settlement.verdict ? { verdict: settlement.verdict } : {},
|
|
3982
5461
|
spent: settlement.spent,
|
|
5462
|
+
trace: settlement.trace,
|
|
3983
5463
|
seq,
|
|
3984
|
-
at
|
|
5464
|
+
at
|
|
3985
5465
|
});
|
|
3986
5466
|
if (settlement.metered) await args.journal.appendEvent(args.root, {
|
|
3987
5467
|
kind: "metered",
|
|
3988
5468
|
id: child.id,
|
|
3989
5469
|
spend: settlement.metered,
|
|
3990
5470
|
seq,
|
|
3991
|
-
at
|
|
5471
|
+
at
|
|
3992
5472
|
});
|
|
3993
5473
|
notifyRuntimeHookEvent(args.hooks, {
|
|
3994
5474
|
id: `${child.id}:settled`,
|
|
3995
5475
|
runId: args.root,
|
|
3996
5476
|
target: "agent.child",
|
|
3997
5477
|
phase: "after",
|
|
3998
|
-
timestamp:
|
|
5478
|
+
timestamp: settledAt,
|
|
3999
5479
|
stepIndex: seq,
|
|
4000
5480
|
parentId: args.parentId,
|
|
4001
5481
|
payload: {
|
|
@@ -4014,6 +5494,8 @@ async function finalizeSettlement(child, settlement, seq, args, now) {
|
|
|
4014
5494
|
outRef: settlement.outRef,
|
|
4015
5495
|
...settlement.verdict ? { verdict: settlement.verdict } : {},
|
|
4016
5496
|
spent: settlement.spent,
|
|
5497
|
+
trace: settlement.trace,
|
|
5498
|
+
settledAt,
|
|
4017
5499
|
seq
|
|
4018
5500
|
};
|
|
4019
5501
|
}
|
|
@@ -4021,10 +5503,12 @@ async function finalizeSettlement(child, settlement, seq, args, now) {
|
|
|
4021
5503
|
* the driver branches on. `by` names WHY it woke — `fired` / `timeout` / `cancelled` — which is
|
|
4022
5504
|
* the fact a resumed reader needs and cannot recover from a payload it may never fetch. */
|
|
4023
5505
|
async function finalizeWait(child, settlement, seq, args, now, handle) {
|
|
4024
|
-
const
|
|
5506
|
+
const settledAt = now();
|
|
5507
|
+
child.settledAt = settledAt;
|
|
5508
|
+
const at = new Date(settledAt).toISOString();
|
|
4025
5509
|
if (settlement.kind === "down") {
|
|
4026
5510
|
child.status = "cancelled";
|
|
4027
|
-
await args.journal.appendEvent(args.root, {
|
|
5511
|
+
if (child.wait?.armCommitted) await args.journal.appendEvent(args.root, {
|
|
4028
5512
|
kind: "woken",
|
|
4029
5513
|
id: child.id,
|
|
4030
5514
|
by: "cancelled",
|
|
@@ -4037,6 +5521,8 @@ async function finalizeWait(child, settlement, seq, args, now, handle) {
|
|
|
4037
5521
|
reason: settlement.reason,
|
|
4038
5522
|
infra: settlement.infra,
|
|
4039
5523
|
restartCount: settlement.restartCount,
|
|
5524
|
+
trace: settlement.trace,
|
|
5525
|
+
settledAt,
|
|
4040
5526
|
seq
|
|
4041
5527
|
};
|
|
4042
5528
|
}
|
|
@@ -4056,7 +5542,7 @@ async function finalizeWait(child, settlement, seq, args, now, handle) {
|
|
|
4056
5542
|
runId: args.root,
|
|
4057
5543
|
target: "agent.child",
|
|
4058
5544
|
phase: "after",
|
|
4059
|
-
timestamp:
|
|
5545
|
+
timestamp: settledAt,
|
|
4060
5546
|
stepIndex: seq,
|
|
4061
5547
|
parentId: args.parentId,
|
|
4062
5548
|
payload: {
|
|
@@ -4071,6 +5557,8 @@ async function finalizeWait(child, settlement, seq, args, now, handle) {
|
|
|
4071
5557
|
out: settlement.out,
|
|
4072
5558
|
outRef: settlement.outRef,
|
|
4073
5559
|
spent: settlement.spent,
|
|
5560
|
+
trace: settlement.trace,
|
|
5561
|
+
settledAt,
|
|
4074
5562
|
seq
|
|
4075
5563
|
};
|
|
4076
5564
|
}
|
|
@@ -4085,54 +5573,101 @@ async function finalizeWait(child, settlement, seq, args, now, handle) {
|
|
|
4085
5573
|
* applied per child). The reservation is reconciled on EVERY path (success, abort, throw)
|
|
4086
5574
|
* so the conserved pool can never leak a reservation.
|
|
4087
5575
|
*/
|
|
4088
|
-
async function runChild(live, executor, childAbort, task, opts, pool, ticket, blobs, now) {
|
|
5576
|
+
async function runChild(live, executor, childAbort, task, opts, pool, ticket, blobs, now, executionReady, deadlineAtMs) {
|
|
4089
5577
|
let reconciled = false;
|
|
5578
|
+
let started = false;
|
|
5579
|
+
let terminalTelemetryCaptured = false;
|
|
5580
|
+
let teardownStarted = false;
|
|
5581
|
+
let traceEvidence;
|
|
5582
|
+
const captureTraceOnce = async () => {
|
|
5583
|
+
traceEvidence ??= await captureWorkerTraceEvidence(live.readTraceSource, blobs, started);
|
|
5584
|
+
return traceEvidence;
|
|
5585
|
+
};
|
|
5586
|
+
const teardownOnce = async (grace) => {
|
|
5587
|
+
if (teardownStarted) return;
|
|
5588
|
+
teardownStarted = true;
|
|
5589
|
+
await teardownExecutor(executor, grace, deadlineAtMs, now);
|
|
5590
|
+
live.cleanupConfirmed = true;
|
|
5591
|
+
};
|
|
4090
5592
|
const reconcileOnce = (spend) => {
|
|
4091
|
-
if (reconciled) return;
|
|
5593
|
+
if (reconciled) return void 0;
|
|
4092
5594
|
reconciled = true;
|
|
4093
|
-
|
|
5595
|
+
try {
|
|
5596
|
+
pool.reconcile(ticket, spend);
|
|
5597
|
+
return;
|
|
5598
|
+
} catch (error) {
|
|
5599
|
+
return error;
|
|
5600
|
+
}
|
|
4094
5601
|
};
|
|
4095
5602
|
try {
|
|
5603
|
+
await executionReady;
|
|
5604
|
+
if (childAbort.signal.aborted) throw abortError(childAbort.signal);
|
|
4096
5605
|
live.status = "running";
|
|
5606
|
+
started = true;
|
|
4097
5607
|
const ran = executor.execute(task, childAbort.signal);
|
|
4098
5608
|
let artifact;
|
|
4099
5609
|
if (isAsyncIterable(ran)) {
|
|
4100
5610
|
const spend = await foldStream(ran, (running) => {
|
|
4101
5611
|
live.spent = running;
|
|
4102
5612
|
live.lastActivityAt = now();
|
|
4103
|
-
});
|
|
5613
|
+
}, childAbort.signal);
|
|
4104
5614
|
live.spent = spend;
|
|
4105
5615
|
artifact = executor.resultArtifact();
|
|
4106
|
-
|
|
5616
|
+
const accounting = executor.accounting?.();
|
|
5617
|
+
const terminalSpend = preserveUnknownTelemetry(spend, artifact.spent);
|
|
5618
|
+
live.spent = accounting?.reported ?? terminalSpend;
|
|
5619
|
+
terminalTelemetryCaptured = true;
|
|
5620
|
+
live.executorDone = true;
|
|
5621
|
+
const reconcileError = reconcileOnce(accounting?.reservation ?? terminalSpend);
|
|
5622
|
+
if (reconcileError !== void 0) throw reconcileError;
|
|
4107
5623
|
} else {
|
|
4108
|
-
const terminal = await ran;
|
|
4109
|
-
|
|
5624
|
+
const terminal = await awaitAbortable(Promise.resolve(ran), childAbort.signal);
|
|
5625
|
+
const accounting = executor.accounting?.();
|
|
5626
|
+
live.spent = accounting?.reported ?? terminal.spent;
|
|
4110
5627
|
artifact = terminal;
|
|
4111
|
-
|
|
5628
|
+
terminalTelemetryCaptured = true;
|
|
5629
|
+
live.executorDone = true;
|
|
5630
|
+
const reconcileError = reconcileOnce(accounting?.reservation ?? terminal.spent);
|
|
5631
|
+
if (reconcileError !== void 0) throw reconcileError;
|
|
4112
5632
|
}
|
|
4113
5633
|
live.executorDone = true;
|
|
4114
5634
|
const ownMetered = executor.metered?.();
|
|
5635
|
+
const trace = await captureTraceOnce();
|
|
4115
5636
|
if (childAbort.signal.aborted) {
|
|
4116
|
-
await
|
|
4117
|
-
return downRecord("aborted before settle", true, ownMetered);
|
|
5637
|
+
await teardownOnce(opts.shutdown ?? "brutalKill");
|
|
5638
|
+
return downRecord("aborted before settle", true, trace, ownMetered);
|
|
4118
5639
|
}
|
|
4119
5640
|
const outRef = contentAddress(artifact.out);
|
|
4120
5641
|
await blobs.put(outRef, artifact.out);
|
|
4121
|
-
await
|
|
5642
|
+
await teardownOnce(opts.shutdown ?? 5e3);
|
|
4122
5643
|
return {
|
|
4123
5644
|
kind: "done",
|
|
4124
5645
|
out: artifact.out,
|
|
4125
5646
|
outRef,
|
|
4126
5647
|
...artifact.verdict ? { verdict: artifact.verdict } : {},
|
|
4127
5648
|
spent: live.spent,
|
|
5649
|
+
trace,
|
|
4128
5650
|
...ownMetered ? { metered: ownMetered } : {}
|
|
4129
5651
|
};
|
|
4130
5652
|
} catch (err) {
|
|
4131
5653
|
live.executorDone = true;
|
|
4132
|
-
|
|
4133
|
-
|
|
5654
|
+
const trace = await captureTraceOnce();
|
|
5655
|
+
let teardownError;
|
|
5656
|
+
try {
|
|
5657
|
+
await teardownOnce("brutalKill");
|
|
5658
|
+
} catch (error) {
|
|
5659
|
+
teardownError = error;
|
|
5660
|
+
}
|
|
5661
|
+
const accounting = executor.accounting?.();
|
|
5662
|
+
if (accounting) live.spent = accounting.reported;
|
|
4134
5663
|
const aborted = childAbort.signal.aborted || isAbortError(err);
|
|
4135
|
-
|
|
5664
|
+
if (started && !terminalTelemetryCaptured && accounting === void 0) live.spent = {
|
|
5665
|
+
...live.spent,
|
|
5666
|
+
tokensKnown: false,
|
|
5667
|
+
usdKnown: false
|
|
5668
|
+
};
|
|
5669
|
+
const reconcileError = reconcileOnce(accounting?.reservation ?? live.spent);
|
|
5670
|
+
return downRecord(errMessage(err), teardownError !== void 0 || reconcileError !== void 0 || aborted || isInfraError(err), trace, executor.metered?.());
|
|
4136
5671
|
}
|
|
4137
5672
|
}
|
|
4138
5673
|
/**
|
|
@@ -4164,6 +5699,25 @@ function settledToIteration(settled) {
|
|
|
4164
5699
|
}
|
|
4165
5700
|
};
|
|
4166
5701
|
}
|
|
5702
|
+
function normalizeLiveWorkerLimit(value) {
|
|
5703
|
+
if (value === void 0 || value <= 0) return void 0;
|
|
5704
|
+
if (!Number.isSafeInteger(value)) throw new ValidationError(`createScope: maxLiveWorkers must be a positive safe integer or <= 0 for uncapped, got ${String(value)}`);
|
|
5705
|
+
return value;
|
|
5706
|
+
}
|
|
5707
|
+
function acquireLiveWorker(capacity) {
|
|
5708
|
+
if (capacity.max !== void 0 && capacity.live >= capacity.max) return { ok: false };
|
|
5709
|
+
capacity.live += 1;
|
|
5710
|
+
let released = false;
|
|
5711
|
+
return {
|
|
5712
|
+
ok: true,
|
|
5713
|
+
release() {
|
|
5714
|
+
if (released) return;
|
|
5715
|
+
released = true;
|
|
5716
|
+
capacity.live -= 1;
|
|
5717
|
+
if (capacity.live < 0) throw new ValidationError("scope: live-worker capacity released more than once");
|
|
5718
|
+
}
|
|
5719
|
+
};
|
|
5720
|
+
}
|
|
4167
5721
|
function makeTreeView(root, children) {
|
|
4168
5722
|
const nodes = [...children.values()].map((c) => ({
|
|
4169
5723
|
id: c.id,
|
|
@@ -4172,8 +5726,15 @@ function makeTreeView(root, children) {
|
|
|
4172
5726
|
status: c.status,
|
|
4173
5727
|
runtime: c.runtime,
|
|
4174
5728
|
budget: c.budget,
|
|
5729
|
+
...c.ownedTreeRoot === void 0 ? {} : { ownedTreeRoot: c.ownedTreeRoot },
|
|
5730
|
+
...c.assignmentId === void 0 ? {} : { assignmentId: c.assignmentId },
|
|
5731
|
+
...c.identity ? { identity: c.identity } : {},
|
|
5732
|
+
...c.materialization ? { materialization: c.materialization } : {},
|
|
5733
|
+
...c.executionBindings.length > 0 ? { executionBindings: Object.freeze([...c.executionBindings]) } : {},
|
|
4175
5734
|
spent: c.spent,
|
|
4176
|
-
...c.
|
|
5735
|
+
...c.settledAt === void 0 ? {} : { settledAt: c.settledAt },
|
|
5736
|
+
...c.outRef ? { outRef: c.outRef } : {},
|
|
5737
|
+
...c.trace ? { trace: c.trace } : {}
|
|
4177
5738
|
}));
|
|
4178
5739
|
return {
|
|
4179
5740
|
root,
|
|
@@ -4187,9 +5748,49 @@ function frozenHandle(child) {
|
|
|
4187
5748
|
id: child.id,
|
|
4188
5749
|
label: child.label,
|
|
4189
5750
|
status: child.status,
|
|
5751
|
+
...child.assignmentId === void 0 ? {} : { assignmentId: child.assignmentId },
|
|
5752
|
+
...child.identity ? { identity: child.identity } : {},
|
|
5753
|
+
...child.materialization ? { materialization: child.materialization } : {},
|
|
5754
|
+
...child.executionBindings.length > 0 ? { executionBindings: Object.freeze([...child.executionBindings]) } : {},
|
|
4190
5755
|
abort() {}
|
|
4191
5756
|
};
|
|
4192
5757
|
}
|
|
5758
|
+
/** Derive the portable identity of the exact profile and task a node will execute. Caller-owned
|
|
5759
|
+
* candidate/correlation fields are checked at this boundary before they enter the durable log. */
|
|
5760
|
+
function deriveNodeExecutionIdentity(spec, task) {
|
|
5761
|
+
const digest = (value) => {
|
|
5762
|
+
try {
|
|
5763
|
+
return canonicalCandidateDigest(value);
|
|
5764
|
+
} catch {
|
|
5765
|
+
return;
|
|
5766
|
+
}
|
|
5767
|
+
};
|
|
5768
|
+
const profileDigest = digest(spec.profile);
|
|
5769
|
+
const taskDigest = digest(task);
|
|
5770
|
+
const candidateDigest = spec.execution?.candidateDigest;
|
|
5771
|
+
if (candidateDigest !== void 0 && !sha256DigestSchema.safeParse(candidateDigest).success) throw new ValidationError("scope.spawn: execution.candidateDigest must be a sha256 digest");
|
|
5772
|
+
const correlation = freezeCorrelation(spec.execution?.correlation);
|
|
5773
|
+
if (!profileDigest && !taskDigest && !candidateDigest && !correlation) return void 0;
|
|
5774
|
+
return Object.freeze({
|
|
5775
|
+
...profileDigest ? { profileDigest } : {},
|
|
5776
|
+
...taskDigest ? { taskDigest } : {},
|
|
5777
|
+
...candidateDigest ? { candidateDigest } : {},
|
|
5778
|
+
...correlation ? { correlation } : {}
|
|
5779
|
+
});
|
|
5780
|
+
}
|
|
5781
|
+
function isCompleteIdentity(identity) {
|
|
5782
|
+
return identity?.profileDigest !== void 0 && identity.taskDigest !== void 0;
|
|
5783
|
+
}
|
|
5784
|
+
function sameNodeExecutionIdentity(a, b) {
|
|
5785
|
+
return canonicalCandidateDigest(a) === canonicalCandidateDigest(b);
|
|
5786
|
+
}
|
|
5787
|
+
function freezeCorrelation(value) {
|
|
5788
|
+
if (value === void 0) return void 0;
|
|
5789
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new ValidationError("scope.spawn: execution.correlation must be a string record");
|
|
5790
|
+
const entries = Object.entries(value);
|
|
5791
|
+
for (const [key, item] of entries) if (key.length === 0 || typeof item !== "string" || item.length === 0) throw new ValidationError("scope.spawn: execution.correlation keys and values must be non-empty strings");
|
|
5792
|
+
return Object.freeze(Object.fromEntries(entries));
|
|
5793
|
+
}
|
|
4193
5794
|
/**
|
|
4194
5795
|
* Fold a streaming executor's normalized usage into the conserved `Spend`, publishing the
|
|
4195
5796
|
* running total after EVERY event via `onProgress`.
|
|
@@ -4200,7 +5801,7 @@ function frozenHandle(child) {
|
|
|
4200
5801
|
* turns deep from one that had produced nothing. It also means a worker that THROWS mid-stream
|
|
4201
5802
|
* now reconciles its real partial spend instead of refunding work the provider already billed.
|
|
4202
5803
|
*/
|
|
4203
|
-
async function foldStream(stream, onProgress) {
|
|
5804
|
+
async function foldStream(stream, onProgress, signal) {
|
|
4204
5805
|
const tokens = {
|
|
4205
5806
|
input: 0,
|
|
4206
5807
|
output: 0
|
|
@@ -4208,21 +5809,30 @@ async function foldStream(stream, onProgress) {
|
|
|
4208
5809
|
let usd = 0;
|
|
4209
5810
|
let usdKnown = true;
|
|
4210
5811
|
let iterations = 0;
|
|
4211
|
-
|
|
4212
|
-
|
|
4213
|
-
|
|
4214
|
-
|
|
4215
|
-
|
|
4216
|
-
|
|
4217
|
-
if (ev.
|
|
4218
|
-
|
|
4219
|
-
|
|
4220
|
-
|
|
4221
|
-
|
|
4222
|
-
|
|
4223
|
-
|
|
4224
|
-
|
|
4225
|
-
|
|
5812
|
+
const iterator = stream[Symbol.asyncIterator]();
|
|
5813
|
+
try {
|
|
5814
|
+
for (;;) {
|
|
5815
|
+
const next = signal ? await awaitAbortable(Promise.resolve(iterator.next()), signal) : await iterator.next();
|
|
5816
|
+
if (next.done) break;
|
|
5817
|
+
const ev = next.value;
|
|
5818
|
+
if (ev.kind === "tokens") {
|
|
5819
|
+
tokens.input += ev.input;
|
|
5820
|
+
tokens.output += ev.output;
|
|
5821
|
+
} else if (ev.kind === "cost") {
|
|
5822
|
+
usd += ev.usd;
|
|
5823
|
+
if (ev.usdKnown === false) usdKnown = false;
|
|
5824
|
+
} else iterations += 1;
|
|
5825
|
+
onProgress?.({
|
|
5826
|
+
iterations,
|
|
5827
|
+
tokens: { ...tokens },
|
|
5828
|
+
usd,
|
|
5829
|
+
...usdKnown ? {} : { usdKnown: false },
|
|
5830
|
+
ms: 0
|
|
5831
|
+
});
|
|
5832
|
+
}
|
|
5833
|
+
} catch (error) {
|
|
5834
|
+
Promise.resolve(iterator.return?.()).catch(() => void 0);
|
|
5835
|
+
throw error;
|
|
4226
5836
|
}
|
|
4227
5837
|
return {
|
|
4228
5838
|
iterations,
|
|
@@ -4232,43 +5842,63 @@ async function foldStream(stream, onProgress) {
|
|
|
4232
5842
|
ms: 0
|
|
4233
5843
|
};
|
|
4234
5844
|
}
|
|
4235
|
-
/**
|
|
4236
|
-
*
|
|
4237
|
-
|
|
4238
|
-
function clampSpend(spend, budget) {
|
|
4239
|
-
const totalTokens = spend.tokens.input + spend.tokens.output;
|
|
4240
|
-
const tokensOk = totalTokens <= budget.maxTokens;
|
|
4241
|
-
const itersOk = spend.iterations <= budget.maxIterations;
|
|
4242
|
-
const usdOk = budget.maxUsd === void 0 || spend.usd <= budget.maxUsd;
|
|
4243
|
-
if (tokensOk && itersOk && usdOk) return spend;
|
|
4244
|
-
const ratio = !tokensOk && totalTokens > 0 ? budget.maxTokens / totalTokens : 1;
|
|
5845
|
+
/** Usage events carry measured increments; the terminal artifact carries whether a provider omitted
|
|
5846
|
+
* a whole accounting channel. Preserve those unknowns on the common streaming path. */
|
|
5847
|
+
function preserveUnknownTelemetry(streamed, terminal) {
|
|
4245
5848
|
return {
|
|
4246
|
-
|
|
4247
|
-
|
|
4248
|
-
|
|
4249
|
-
|
|
4250
|
-
|
|
4251
|
-
usd: budget.maxUsd === void 0 ? spend.usd : Math.min(spend.usd, budget.maxUsd),
|
|
4252
|
-
...spend.tokensKnown === false ? { tokensKnown: false } : {},
|
|
4253
|
-
...spend.usdKnown === false ? { usdKnown: false } : {},
|
|
4254
|
-
ms: spend.ms
|
|
4255
|
-
};
|
|
4256
|
-
}
|
|
4257
|
-
async function teardownSafe(executor, grace) {
|
|
4258
|
-
try {
|
|
4259
|
-
await executor.teardown(grace);
|
|
4260
|
-
} catch {}
|
|
5849
|
+
...streamed,
|
|
5850
|
+
...terminal.tokensKnown === false ? { tokensKnown: false } : {},
|
|
5851
|
+
...terminal.usdKnown === false ? { usdKnown: false } : {},
|
|
5852
|
+
ms: terminal.ms
|
|
5853
|
+
};
|
|
4261
5854
|
}
|
|
4262
|
-
function
|
|
5855
|
+
async function awaitAbortable(work, signal) {
|
|
5856
|
+
if (signal.aborted) {
|
|
5857
|
+
work.catch(() => void 0);
|
|
5858
|
+
throw abortError(signal);
|
|
5859
|
+
}
|
|
5860
|
+
return await new Promise((resolve, reject) => {
|
|
5861
|
+
let settled = false;
|
|
5862
|
+
const onAbort = () => {
|
|
5863
|
+
queueMicrotask(() => {
|
|
5864
|
+
if (settled) return;
|
|
5865
|
+
settled = true;
|
|
5866
|
+
cleanup();
|
|
5867
|
+
reject(abortError(signal));
|
|
5868
|
+
});
|
|
5869
|
+
};
|
|
5870
|
+
const cleanup = () => signal.removeEventListener("abort", onAbort);
|
|
5871
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
5872
|
+
work.then((value) => {
|
|
5873
|
+
if (settled) return;
|
|
5874
|
+
settled = true;
|
|
5875
|
+
cleanup();
|
|
5876
|
+
resolve(value);
|
|
5877
|
+
}, (error) => {
|
|
5878
|
+
if (settled) return;
|
|
5879
|
+
settled = true;
|
|
5880
|
+
cleanup();
|
|
5881
|
+
reject(error);
|
|
5882
|
+
});
|
|
5883
|
+
});
|
|
5884
|
+
}
|
|
5885
|
+
function abortError(signal) {
|
|
5886
|
+
const reason = signal.reason;
|
|
5887
|
+
const error = new Error(typeof reason === "string" && reason.length > 0 ? reason : "execution aborted");
|
|
5888
|
+
error.name = "AbortError";
|
|
5889
|
+
return error;
|
|
5890
|
+
}
|
|
5891
|
+
function downRecord(reason, infra, trace, metered) {
|
|
4263
5892
|
return {
|
|
4264
5893
|
kind: "down",
|
|
4265
5894
|
reason,
|
|
4266
5895
|
infra,
|
|
4267
5896
|
restartCount: 0,
|
|
5897
|
+
trace,
|
|
4268
5898
|
...metered ? { metered } : {}
|
|
4269
5899
|
};
|
|
4270
5900
|
}
|
|
4271
|
-
function zeroSpend() {
|
|
5901
|
+
function zeroSpend$1() {
|
|
4272
5902
|
return {
|
|
4273
5903
|
iterations: 0,
|
|
4274
5904
|
tokens: {
|
|
@@ -4289,6 +5919,20 @@ function isAgentSpec(value) {
|
|
|
4289
5919
|
const v = value;
|
|
4290
5920
|
return "profile" in v && "harness" in v;
|
|
4291
5921
|
}
|
|
5922
|
+
/** Snapshot every data field whose bytes affect identity, admission, or materialization while
|
|
5923
|
+
* preserving the executable callbacks by reference. The executor implementation is trusted code;
|
|
5924
|
+
* its profile and attribution inputs are not. */
|
|
5925
|
+
function snapshotAgentSpec(raw) {
|
|
5926
|
+
const { profile: rawProfile, harness, execution: rawExecution, ...runtimeExtensions } = raw;
|
|
5927
|
+
const profile = detachedSnapshot(rawProfile, "scope.spawn profile");
|
|
5928
|
+
const execution = rawExecution === void 0 ? void 0 : detachedSnapshot(rawExecution, "scope.spawn execution");
|
|
5929
|
+
return Object.freeze({
|
|
5930
|
+
...runtimeExtensions,
|
|
5931
|
+
profile,
|
|
5932
|
+
harness,
|
|
5933
|
+
...execution === void 0 ? {} : { execution }
|
|
5934
|
+
});
|
|
5935
|
+
}
|
|
4292
5936
|
function isAbortError(err) {
|
|
4293
5937
|
return typeof err === "object" && err !== null && "name" in err && err.name === "AbortError";
|
|
4294
5938
|
}
|
|
@@ -4304,474 +5948,358 @@ function errMessage(err) {
|
|
|
4304
5948
|
}
|
|
4305
5949
|
//#endregion
|
|
4306
5950
|
//#region src/runtime/supervise/driver-executor.ts
|
|
4307
|
-
/**
|
|
4308
|
-
*
|
|
4309
|
-
* The recursive driver-executor — the seam that lets a SPAWNED child be a DRIVER, so
|
|
4310
|
-
* agents drive agents drive agents over the one keystone atom.
|
|
4311
|
-
*
|
|
4312
|
-
* A spawned child resolves through the open registry to an `Executor`; the built-in
|
|
4313
|
-
* executors (router/inline, sandbox, cli) are LEAVES — `execute(task, signal)` runs the
|
|
4314
|
-
* work and settles. This executor is the recursive case: on `execute`, it mounts a NESTED
|
|
4315
|
-
* `Scope` (the scope hands it the mount via the `nested-scope` seam) over the SAME
|
|
4316
|
-
* conserved pool + shared journal/blobs + the same open registry, one `depth` deeper, then
|
|
4317
|
-
* runs the wrapped driver `Agent.act(task, nestedScope)`. The driver spawns its own
|
|
4318
|
-
* children into that nested scope; each resolves to EITHER a leaf executor (a worker child)
|
|
4319
|
-
* OR this same driver-executor (a driver child) — recursively. So a driver spawns a driver
|
|
4320
|
-
* spawns a worker, all on one budget-conserving tree.
|
|
4321
|
-
*
|
|
4322
|
-
* Why this preserves every keystone invariant (the scope owns the sharing; this executor
|
|
4323
|
-
* only runs the driver over what the scope mounts):
|
|
4324
|
-
* - Conserved budget: the nested scope reserves from the SAME `BudgetPool` the root owns
|
|
4325
|
-
* (the scope mounts it over `args.pool`), so `Σk` is conserved ACROSS depth by
|
|
4326
|
-
* construction — a deep tree cannot overspend the root ceiling (reserve-on-spawn fails
|
|
4327
|
-
* closed at any depth).
|
|
4328
|
-
* - Journal: the nested scope writes to its OWN tree key (`${journalRoot}/${nodeId}`) so
|
|
4329
|
-
* its cursor `seq`s never collide with the parent's in the per-tree uniqueness guard,
|
|
4330
|
-
* while every nested tree shares the one `SpawnJournal` — the whole recursion is one
|
|
4331
|
-
* journal, queryable tree by tree.
|
|
4332
|
-
* - Settlement bubbling: the driver child settles into its PARENT scope with the conserved
|
|
4333
|
-
* spend summed off its nested tree's settled events, so the parent's pool reconcile +
|
|
4334
|
-
* the supervisor's `spentTotal` see the whole sub-tree's spend rolled up — settlements
|
|
4335
|
-
* bubble to the root.
|
|
4336
|
-
* - Depth ceiling: the nested scope runs at `depth+1`, so the supervisor's `maxDepth`
|
|
4337
|
-
* (paired with the conserved pool per R3) fails a spawn closed once the recursion is too
|
|
4338
|
-
* deep — exactly as it does for a flat tree.
|
|
4339
|
-
*
|
|
4340
|
-
* Layering: pure keystone composition. It reuses the scope's `NestedScopeSeam` + the shared
|
|
4341
|
-
* `SpawnJournal`; it builds NO new budget, journal, or selection logic. The recursion rides
|
|
4342
|
-
* the existing atom.
|
|
4343
|
-
*
|
|
4344
|
-
* @experimental
|
|
4345
|
-
*/
|
|
4346
|
-
/** The runtime tag the registry maps a driver child to. */
|
|
4347
|
-
const driverRuntime = "driver";
|
|
4348
5951
|
/** The metadata marker on a driver child's spec the recursive registry routes on. */
|
|
4349
5952
|
const driverRole = "driver";
|
|
4350
|
-
/** True when a spec is a driver child (carries the role marker + a driver Agent). */
|
|
4351
|
-
function isDriverSpec(spec) {
|
|
4352
|
-
if (spec.profile.metadata?.role !== driverRole) return false;
|
|
4353
|
-
const driver = spec.driver;
|
|
4354
|
-
if (!isAgent(driver)) throw new ValidationError("driverExecutor: a driver-role spec must carry a `driver` Agent to run inside its nested scope");
|
|
4355
|
-
return true;
|
|
4356
|
-
}
|
|
4357
|
-
/**
|
|
4358
|
-
* The recursive driver-executor factory. `withDriverExecutor` routes a child marked
|
|
4359
|
-
* `role: 'driver'` here; any other child resolves to a leaf built-in. On `execute`, it
|
|
4360
|
-
* reads the `nested-scope` seam the SCOPE seeded, mounts a nested `Scope` one `depth`
|
|
4361
|
-
* deeper over the shared pool/journal/blobs/registry, runs the driver
|
|
4362
|
-
* `Agent.act(task, nestedScope)`, and reports the conserved spend summed off the nested
|
|
4363
|
-
* tree's settled events — so the parent scope's reconcile rolls the whole sub-tree's spend
|
|
4364
|
-
* into the conserved total.
|
|
4365
|
-
*
|
|
4366
|
-
* A `down` from the nested driver (a thrown `act` or an aborted scope) propagates as a
|
|
4367
|
-
* thrown executor, which the parent scope types into a `down` settlement — the same
|
|
4368
|
-
* fail-loud-into-typed-down discipline a leaf gets.
|
|
4369
|
-
*/
|
|
4370
|
-
const driverExecutorFactory = (spec, ctx) => {
|
|
4371
|
-
if (!isDriverSpec(spec)) throw new ValidationError("driverExecutorFactory: spec is not a driver child (no role:\"driver\" marker)");
|
|
4372
|
-
const driver = spec.driver;
|
|
4373
|
-
const journal = spec.journal;
|
|
4374
|
-
const seam = readNestedScopeSeam(ctx);
|
|
4375
|
-
let artifact;
|
|
4376
|
-
let meteredSpend;
|
|
4377
|
-
return {
|
|
4378
|
-
runtime: driverRuntime,
|
|
4379
|
-
async execute(task, signal) {
|
|
4380
|
-
const nestedRoot = nestedTreeKey(seam, journal);
|
|
4381
|
-
await journal.beginTree(nestedRoot, (/* @__PURE__ */ new Date(0)).toISOString());
|
|
4382
|
-
const nestedScope = seam.mount(nestedRoot, signal);
|
|
4383
|
-
try {
|
|
4384
|
-
const out = await driver.act(task, nestedScope);
|
|
4385
|
-
const events = await loadTreeEvents(journal, nestedRoot);
|
|
4386
|
-
const settled = events.filter(isSettled);
|
|
4387
|
-
meteredSpend = nonZeroOrUndef(sumMetered(events));
|
|
4388
|
-
const verdict = deriveDeliveryVerdict(settled);
|
|
4389
|
-
artifact = {
|
|
4390
|
-
outRef: `${driverRuntime}:${nestedRoot}`,
|
|
4391
|
-
out,
|
|
4392
|
-
spent: sumSpend(settled),
|
|
4393
|
-
...verdict ? { verdict } : {}
|
|
4394
|
-
};
|
|
4395
|
-
return artifact;
|
|
4396
|
-
} catch (err) {
|
|
4397
|
-
meteredSpend = await safeSumMetered(journal, nestedRoot);
|
|
4398
|
-
throw err;
|
|
4399
|
-
}
|
|
4400
|
-
},
|
|
4401
|
-
metered() {
|
|
4402
|
-
return meteredSpend;
|
|
4403
|
-
},
|
|
4404
|
-
teardown() {
|
|
4405
|
-
return Promise.resolve({ destroyed: true });
|
|
4406
|
-
},
|
|
4407
|
-
resultArtifact() {
|
|
4408
|
-
if (!artifact) throw new ValidationError("driverExecutor: resultArtifact() read before execute()");
|
|
4409
|
-
return artifact;
|
|
4410
|
-
}
|
|
4411
|
-
};
|
|
4412
|
-
};
|
|
4413
5953
|
/**
|
|
4414
|
-
*
|
|
4415
|
-
*
|
|
4416
|
-
*
|
|
4417
|
-
*
|
|
4418
|
-
*
|
|
5954
|
+
* Mark + carry a driver `Agent` so the recursive registry resolves it to the
|
|
5955
|
+
* driver-executor. The returned agent is SPAWNED (never run directly): its
|
|
5956
|
+
* `executorSpec` is marked `role: 'driver'` and carries the driver agent + the shared
|
|
5957
|
+
* journal so the executor can run its `act` inside a nested scope. `act` fails loud if
|
|
5958
|
+
* called directly — a driver child runs THROUGH its nested-scope executor, never as a root.
|
|
4419
5959
|
*/
|
|
4420
|
-
function
|
|
4421
|
-
|
|
4422
|
-
|
|
4423
|
-
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
|
|
4429
|
-
}
|
|
4430
|
-
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
function nestedTreeKey(seam, journal) {
|
|
4436
|
-
return `${seam.journalRoot}/d${nextNestOrdinal(journal)}`;
|
|
4437
|
-
}
|
|
4438
|
-
/** Per-journal monotonic nest counter — keyed on the journal instance so a single run's
|
|
4439
|
-
* nested-tree keys are unique without a shared module global. */
|
|
4440
|
-
const nestCounters = /* @__PURE__ */ new WeakMap();
|
|
4441
|
-
function nextNestOrdinal(journal) {
|
|
4442
|
-
let c = nestCounters.get(journal);
|
|
4443
|
-
if (!c) {
|
|
4444
|
-
c = { n: 0 };
|
|
4445
|
-
nestCounters.set(journal, c);
|
|
4446
|
-
}
|
|
4447
|
-
return c.n++;
|
|
4448
|
-
}
|
|
4449
|
-
/** The nested tree's full event list — the one evidence the spend, verdict, AND driver-inference
|
|
4450
|
-
* roll-ups read off the same journal the supervisor sums. */
|
|
4451
|
-
async function loadTreeEvents(journal, nestedRoot) {
|
|
4452
|
-
const events = await journal.loadTree(nestedRoot);
|
|
4453
|
-
if (events === void 0) throw new ValidationError(`driverExecutor: nested tree '${nestedRoot}' missing from the journal after run (corrupted log)`);
|
|
4454
|
-
return events;
|
|
4455
|
-
}
|
|
4456
|
-
function isSettled(ev) {
|
|
4457
|
-
return ev.kind === "settled";
|
|
4458
|
-
}
|
|
4459
|
-
/** Sum the conserved spend over the nested tree's settled events — the honest per-channel
|
|
4460
|
-
* roll-up of the whole sub-tree's child WORK. */
|
|
4461
|
-
function sumSpend(settled) {
|
|
4462
|
-
const total = {
|
|
4463
|
-
iterations: 0,
|
|
4464
|
-
tokens: {
|
|
4465
|
-
input: 0,
|
|
4466
|
-
output: 0
|
|
4467
|
-
},
|
|
4468
|
-
usd: 0,
|
|
4469
|
-
ms: 0
|
|
4470
|
-
};
|
|
4471
|
-
for (const ev of settled) {
|
|
4472
|
-
total.iterations += ev.spent.iterations;
|
|
4473
|
-
total.tokens.input += ev.spent.tokens.input;
|
|
4474
|
-
total.tokens.output += ev.spent.tokens.output;
|
|
4475
|
-
total.usd += ev.spent.usd;
|
|
4476
|
-
if (ev.spent.tokensKnown === false) total.tokensKnown = false;
|
|
4477
|
-
if (ev.spent.usdKnown === false) total.usdKnown = false;
|
|
4478
|
-
total.ms += ev.spent.ms;
|
|
4479
|
-
}
|
|
4480
|
-
return total;
|
|
4481
|
-
}
|
|
4482
|
-
/** Sum the nested tree's `metered` events — the sub-tree's whole driver INFERENCE (this driver's
|
|
4483
|
-
* own turns + any sub-driver inference already re-homed into this tree). Re-homed up to the parent
|
|
4484
|
-
* as one `metered` event; never reconciled (already pool-debited live via `observe`). */
|
|
4485
|
-
function sumMetered(events) {
|
|
4486
|
-
const total = {
|
|
4487
|
-
iterations: 0,
|
|
4488
|
-
tokens: {
|
|
4489
|
-
input: 0,
|
|
4490
|
-
output: 0
|
|
4491
|
-
},
|
|
4492
|
-
usd: 0,
|
|
4493
|
-
ms: 0
|
|
4494
|
-
};
|
|
4495
|
-
for (const ev of events) {
|
|
4496
|
-
if (ev.kind !== "metered") continue;
|
|
4497
|
-
total.iterations += ev.spend.iterations;
|
|
4498
|
-
total.tokens.input += ev.spend.tokens.input;
|
|
4499
|
-
total.tokens.output += ev.spend.tokens.output;
|
|
4500
|
-
total.usd += ev.spend.usd;
|
|
4501
|
-
if (ev.spend.tokensKnown === false) total.tokensKnown = false;
|
|
4502
|
-
if (ev.spend.usdKnown === false) total.usdKnown = false;
|
|
4503
|
-
total.ms += ev.spend.ms;
|
|
4504
|
-
}
|
|
4505
|
-
return total;
|
|
4506
|
-
}
|
|
4507
|
-
/** An all-zero spend that carries an UNKNOWN marker counts as non-zero: a sub-driver whose turns
|
|
4508
|
-
* went unmeasured did real work, and dropping its `metered` event here would re-hide upstream the
|
|
4509
|
-
* very turn the driver refused to skip. */
|
|
4510
|
-
function isNonZeroSpend(s) {
|
|
4511
|
-
return s.iterations > 0 || s.tokens.input > 0 || s.tokens.output > 0 || s.usd > 0 || s.ms > 0 || s.tokensKnown === false || s.usdKnown === false;
|
|
4512
|
-
}
|
|
4513
|
-
/** A spend, or `undefined` when it is all-zero — so `metered()` returns undefined for a driver
|
|
4514
|
-
* whose sub-tree did no inference (and the parent journals no empty `metered` event). */
|
|
4515
|
-
function nonZeroOrUndef(s) {
|
|
4516
|
-
return isNonZeroSpend(s) ? s : void 0;
|
|
4517
|
-
}
|
|
4518
|
-
/** Sum the nested tree's metered events, tolerating a missing tree (a crash before `beginTree`
|
|
4519
|
-
* landed) — never throw here, or it would mask the original `act` error on the crash path. */
|
|
4520
|
-
async function safeSumMetered(journal, nestedRoot) {
|
|
4521
|
-
try {
|
|
4522
|
-
return nonZeroOrUndef(sumMetered(await loadTreeEvents(journal, nestedRoot)));
|
|
4523
|
-
} catch {
|
|
4524
|
-
return;
|
|
4525
|
-
}
|
|
4526
|
-
}
|
|
4527
|
-
/** Derive the driver child's delivery verdict from its DIRECT children's settlements:
|
|
4528
|
-
* `valid` iff any direct child settled `done` AND `valid` (the keep-best finalize's pick);
|
|
4529
|
-
* `score` = the best delivered score. Returns `undefined` when no child settled at all (the
|
|
4530
|
-
* driver itself produced nothing to bubble a verdict from). Fail-closed: a child whose verdict
|
|
4531
|
-
* carried no `valid` counts as not-delivered. */
|
|
4532
|
-
function deriveDeliveryVerdict(settled) {
|
|
4533
|
-
let sawChild = false;
|
|
4534
|
-
let anyValid = false;
|
|
4535
|
-
let bestValidScore;
|
|
4536
|
-
let bestDoneScore;
|
|
4537
|
-
for (const ev of settled) {
|
|
4538
|
-
sawChild = true;
|
|
4539
|
-
if (ev.status !== "done") continue;
|
|
4540
|
-
const score = ev.verdict?.score;
|
|
4541
|
-
if (score !== void 0 && (bestDoneScore === void 0 || score > bestDoneScore)) bestDoneScore = score;
|
|
4542
|
-
if (ev.verdict?.valid === true) {
|
|
4543
|
-
anyValid = true;
|
|
4544
|
-
if (score !== void 0 && (bestValidScore === void 0 || score > bestValidScore)) bestValidScore = score;
|
|
4545
|
-
}
|
|
4546
|
-
}
|
|
4547
|
-
if (!sawChild) return void 0;
|
|
5960
|
+
function driverChild(profileOrName, driver, journal, execution) {
|
|
5961
|
+
const profile = typeof profileOrName === "string" ? {
|
|
5962
|
+
name: profileOrName,
|
|
5963
|
+
metadata: { role: driverRole }
|
|
5964
|
+
} : profileOrName;
|
|
5965
|
+
const name = profile.name ?? driver.name;
|
|
5966
|
+
const spec = {
|
|
5967
|
+
profile,
|
|
5968
|
+
harness: null,
|
|
5969
|
+
...execution ? { execution } : {},
|
|
5970
|
+
driverRuntime,
|
|
5971
|
+
driver,
|
|
5972
|
+
journal
|
|
5973
|
+
};
|
|
5974
|
+
const deliver = driver.deliver?.bind(driver);
|
|
4548
5975
|
return {
|
|
4549
|
-
|
|
4550
|
-
|
|
5976
|
+
name,
|
|
5977
|
+
executorSpec: spec,
|
|
5978
|
+
...deliver ? { deliver(message) {
|
|
5979
|
+
return deliver(message) !== false;
|
|
5980
|
+
} } : {},
|
|
5981
|
+
act() {
|
|
5982
|
+
throw new ValidationError(`driverChild: "${name}" was run directly; a driver child runs through its nested-scope executor`);
|
|
5983
|
+
}
|
|
4551
5984
|
};
|
|
4552
5985
|
}
|
|
4553
|
-
|
|
4554
|
-
|
|
4555
|
-
if (
|
|
4556
|
-
|
|
4557
|
-
|
|
4558
|
-
|
|
4559
|
-
return typeof value === "object" && value !== null && typeof value.act === "function" && typeof value.name === "string";
|
|
5986
|
+
/** True when a spec is a driver child (carries the role marker + a driver Agent). */
|
|
5987
|
+
function isDriverSpec(spec) {
|
|
5988
|
+
if (spec.driverRuntime !== "driver") return false;
|
|
5989
|
+
const driver = spec.driver;
|
|
5990
|
+
if (!isAgent(driver)) throw new ValidationError("driverExecutor: a driver-role spec must carry a `driver` Agent to run inside its nested scope");
|
|
5991
|
+
return true;
|
|
4560
5992
|
}
|
|
4561
|
-
//#endregion
|
|
4562
|
-
//#region src/runtime/supervise/budget.ts
|
|
4563
5993
|
/**
|
|
5994
|
+
* The recursive driver-executor factory. `withDriverExecutor` routes a child marked
|
|
5995
|
+
* `role: 'driver'` here; any other child resolves to a leaf built-in. On `execute`, it
|
|
5996
|
+
* reads the `nested-scope` seam the SCOPE seeded, mounts a nested `Scope` one `depth`
|
|
5997
|
+
* deeper over the driver's reserved child pool plus shared journal/blobs/registry, runs the driver
|
|
5998
|
+
* `Agent.act(task, nestedScope)`, and reports the conserved spend summed off the nested
|
|
5999
|
+
* tree's settled events — so the parent scope's reconcile rolls the whole sub-tree's spend
|
|
6000
|
+
* into the conserved total.
|
|
4564
6001
|
*
|
|
4565
|
-
*
|
|
4566
|
-
*
|
|
4567
|
-
*
|
|
4568
|
-
* atomically at spawn and reconcile at settle:
|
|
4569
|
-
*
|
|
4570
|
-
* total ≡ free + reserved + committed (for every known quantity)
|
|
4571
|
-
*
|
|
4572
|
-
* `reserve` moves a child's whole ceiling from `free` → `reserved` and fails closed
|
|
4573
|
-
* when `free` can't cover it (never read-then-spawn overcommit, so `Σk(treatment) ≡
|
|
4574
|
-
* Σk(blind)` by construction). `reconcile` releases the reservation, commits ACTUAL
|
|
4575
|
-
* spend, and refunds the unspent remainder to `free`. Tokens and usd are separate
|
|
4576
|
-
* channels (`LoopTokenUsage` has no `usd`); iterations are conserved alongside them.
|
|
4577
|
-
*
|
|
4578
|
-
* Pure and deterministic: `now()` is injected, there is no I/O, and no wall-clock or
|
|
4579
|
-
* RNG read. A `reserve`/`reconcile` ticket is single-use (fail-loud on double or
|
|
4580
|
-
* unknown reconcile) so a child can never refund twice. Reconciling an OPEN ticket always
|
|
4581
|
-
* closes it: a fail-loud condition settles the reservation first and throws afterwards, so no
|
|
4582
|
-
* error path can strand a reservation past the join barrier's `assertNoOpenTickets`.
|
|
4583
|
-
* A child that declared no `maxUsd` reserved no dollar allocation, so its real dollars are
|
|
4584
|
-
* committed as observed spend and debited from the root's balance rather than treated as an
|
|
4585
|
-
* overspend of a $0 ceiling it never asked for. If dollar cost is unknowable under a
|
|
4586
|
-
* dollar limit, reconciliation closes the known token/iteration work, marks the dollar channel
|
|
4587
|
-
* unusable, and refuses later reservations; inventing a numeric dollar total would be worse.
|
|
4588
|
-
* If a TOKEN count is unknowable (a provider that reported no usage for work that ran), the pool
|
|
4589
|
-
* records the work, debits what it knows, and marks `readout().tokensKnown` false — the balance is
|
|
4590
|
-
* then a ceiling, not a measurement. It does not close admission the way the dollar channel does:
|
|
4591
|
-
* tokens are always capped, so one unreported turn must not end the run.
|
|
4592
|
-
*
|
|
4593
|
-
* @experimental
|
|
6002
|
+
* A `down` from the nested driver (a thrown `act` or an aborted scope) propagates as a
|
|
6003
|
+
* thrown executor, which the parent scope types into a `down` settlement — the same
|
|
6004
|
+
* fail-loud-into-typed-down discipline a leaf gets.
|
|
4594
6005
|
*/
|
|
4595
|
-
|
|
4596
|
-
|
|
4597
|
-
|
|
4598
|
-
|
|
4599
|
-
const
|
|
4600
|
-
|
|
4601
|
-
let
|
|
4602
|
-
let
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
if (ev.usdKnown === false) usdKnown = false;
|
|
4610
|
-
} else iterations += 1;
|
|
4611
|
-
return {
|
|
4612
|
-
iterations,
|
|
4613
|
-
tokens,
|
|
4614
|
-
usd,
|
|
4615
|
-
...usdKnown ? {} : { usdKnown: false },
|
|
4616
|
-
ms: 0
|
|
6006
|
+
const driverExecutorFactory = (spec, ctx) => {
|
|
6007
|
+
if (!isDriverSpec(spec)) throw new ValidationError("driverExecutorFactory: spec is not a driver child (no role:\"driver\" marker)");
|
|
6008
|
+
const driver = spec.driver;
|
|
6009
|
+
const deliver = driver.deliver?.bind(driver);
|
|
6010
|
+
const journal = spec.journal;
|
|
6011
|
+
const seam = readNestedScopeSeam(ctx);
|
|
6012
|
+
let artifact;
|
|
6013
|
+
let meteredSpend;
|
|
6014
|
+
let accounting;
|
|
6015
|
+
let active;
|
|
6016
|
+
const closeActive = (reason) => {
|
|
6017
|
+
if (active === void 0) return Promise.resolve();
|
|
6018
|
+
active.close ??= closeNestedScope(active.scope, active.controller, reason);
|
|
6019
|
+
return active.close;
|
|
4617
6020
|
};
|
|
4618
|
-
|
|
4619
|
-
|
|
4620
|
-
|
|
4621
|
-
|
|
4622
|
-
|
|
4623
|
-
|
|
4624
|
-
|
|
4625
|
-
|
|
4626
|
-
|
|
4627
|
-
|
|
6021
|
+
const treeOwner = attestNestedDriverTreeOwner({
|
|
6022
|
+
runtime: driverRuntime,
|
|
6023
|
+
...deliver ? { deliver(message) {
|
|
6024
|
+
return deliver(message) !== false;
|
|
6025
|
+
} } : {},
|
|
6026
|
+
async execute(task, signal) {
|
|
6027
|
+
const nestedRoot = nestedDriverTreeRoot(seam.journalRoot, seam.nodeId);
|
|
6028
|
+
await journal.beginTree(nestedRoot, (/* @__PURE__ */ new Date(0)).toISOString());
|
|
6029
|
+
const controller = new AbortController();
|
|
6030
|
+
const onParentAbort = () => controller.abort(signal.reason);
|
|
6031
|
+
if (signal.aborted) controller.abort(signal.reason);
|
|
6032
|
+
else signal.addEventListener("abort", onParentAbort, { once: true });
|
|
6033
|
+
const nestedScope = seam.mount(nestedRoot, controller.signal);
|
|
6034
|
+
active = {
|
|
6035
|
+
controller,
|
|
6036
|
+
scope: nestedScope
|
|
6037
|
+
};
|
|
6038
|
+
try {
|
|
6039
|
+
const out = await driverActAbortable(() => driver.act(task, nestedScope), controller.signal);
|
|
6040
|
+
await finalizeScopeOwnerMaterialization(nestedScope);
|
|
6041
|
+
await closeActive("driver completed");
|
|
6042
|
+
const events = await loadTreeEvents(journal, nestedRoot);
|
|
6043
|
+
const settled = events.filter(isSettled);
|
|
6044
|
+
const childWork = sumSpend(settled);
|
|
6045
|
+
meteredSpend = nonZeroOrUndef(sumMetered(events));
|
|
6046
|
+
accounting = {
|
|
6047
|
+
reported: childWork,
|
|
6048
|
+
reservation: addSpend$1(childWork, meteredSpend ?? zeroSpend())
|
|
6049
|
+
};
|
|
6050
|
+
const verdict = deriveDeliveryVerdict(settled, out);
|
|
6051
|
+
artifact = {
|
|
6052
|
+
outRef: `${driverRuntime}:${nestedRoot}`,
|
|
6053
|
+
out,
|
|
6054
|
+
spent: childWork,
|
|
6055
|
+
...verdict ? { verdict } : {}
|
|
6056
|
+
};
|
|
6057
|
+
return artifact;
|
|
6058
|
+
} catch (err) {
|
|
6059
|
+
await finalizeScopeOwnerMaterialization(active?.scope ?? nestedScope).catch(() => void 0);
|
|
6060
|
+
await closeActive("driver stopped");
|
|
6061
|
+
const partial = await safeRollup(journal, nestedRoot);
|
|
6062
|
+
meteredSpend = partial?.metered;
|
|
6063
|
+
accounting = partial?.accounting;
|
|
6064
|
+
throw err;
|
|
6065
|
+
} finally {
|
|
6066
|
+
signal.removeEventListener("abort", onParentAbort);
|
|
6067
|
+
active = void 0;
|
|
6068
|
+
}
|
|
6069
|
+
},
|
|
6070
|
+
accounting() {
|
|
6071
|
+
return accounting;
|
|
6072
|
+
},
|
|
6073
|
+
metered() {
|
|
6074
|
+
return meteredSpend;
|
|
6075
|
+
},
|
|
6076
|
+
async teardown() {
|
|
6077
|
+
await closeActive("driver executor teardown");
|
|
6078
|
+
return { destroyed: true };
|
|
6079
|
+
},
|
|
6080
|
+
resultArtifact() {
|
|
6081
|
+
if (!artifact) throw new ValidationError("driverExecutor: resultArtifact() read before execute()");
|
|
6082
|
+
return artifact;
|
|
6083
|
+
}
|
|
4628
6084
|
});
|
|
4629
|
-
|
|
4630
|
-
|
|
4631
|
-
|
|
4632
|
-
|
|
6085
|
+
const ownerRuntime = runtimeOwnedScopeOwnerRuntime(driver);
|
|
6086
|
+
return ownerRuntime === void 0 ? treeOwner : attestRuntimeOwnedDeferredExecutor(treeOwner, ownerRuntime);
|
|
6087
|
+
};
|
|
6088
|
+
/**
|
|
6089
|
+
* Register the driver-executor so a child marked `role: 'driver'` resolves to it. The base
|
|
6090
|
+
* registry resolves by harness alone (it does not read `role`), so a recursive run needs a
|
|
6091
|
+
* registry that routes the driver tag here FIRST. Returns a registry decorator: a
|
|
6092
|
+
* driver-role spec → the driver-executor; everything else → the base registry's resolution
|
|
6093
|
+
* (leaf built-ins + BYO).
|
|
6094
|
+
*/
|
|
6095
|
+
function withDriverExecutor(base) {
|
|
4633
6096
|
return {
|
|
4634
|
-
|
|
4635
|
-
|
|
4636
|
-
|
|
4637
|
-
|
|
4638
|
-
|
|
6097
|
+
register: base.register.bind(base),
|
|
6098
|
+
resolve(spec) {
|
|
6099
|
+
if (spec.driverRuntime === "driver" && !spec.executor) return {
|
|
6100
|
+
succeeded: true,
|
|
6101
|
+
value: driverExecutorFactory
|
|
6102
|
+
};
|
|
6103
|
+
return base.resolve(spec);
|
|
6104
|
+
}
|
|
4639
6105
|
};
|
|
4640
6106
|
}
|
|
4641
|
-
function
|
|
4642
|
-
|
|
6107
|
+
async function closeNestedScope(scope, controller, reason) {
|
|
6108
|
+
if (!controller.signal.aborted) controller.abort(reason);
|
|
6109
|
+
for (;;) if (await scope.next() === null) break;
|
|
6110
|
+
const view = scope.view;
|
|
6111
|
+
if (view.inFlight > 0 || view.waiting > 0) throw new ValidationError(`driverExecutor: nested cleanup left ${view.inFlight} running and ${view.waiting} waiting nodes`);
|
|
4643
6112
|
}
|
|
4644
|
-
|
|
4645
|
-
|
|
4646
|
-
|
|
4647
|
-
|
|
4648
|
-
|
|
4649
|
-
|
|
4650
|
-
|
|
4651
|
-
|
|
4652
|
-
|
|
4653
|
-
|
|
4654
|
-
|
|
4655
|
-
|
|
4656
|
-
let freeUsd = root.maxUsd ?? 0;
|
|
4657
|
-
let reservedUsd = 0;
|
|
4658
|
-
let committedUsd = 0;
|
|
4659
|
-
let usdTainted = false;
|
|
4660
|
-
let freeIterations = root.maxIterations;
|
|
4661
|
-
let reservedIterations = 0;
|
|
4662
|
-
let committedIterations = 0;
|
|
4663
|
-
const absoluteDeadlineMs = root.deadlineMs !== void 0 ? now() + root.deadlineMs : 0;
|
|
4664
|
-
let nextTicketId = 0;
|
|
4665
|
-
const open = /* @__PURE__ */ new Set();
|
|
4666
|
-
function reserve(b) {
|
|
4667
|
-
const wantTokens = b.maxTokens;
|
|
4668
|
-
const wantUsd = b.maxUsd ?? 0;
|
|
4669
|
-
const wantIterations = b.maxIterations;
|
|
4670
|
-
if (usdCapped && usdTainted) return {
|
|
4671
|
-
ok: false,
|
|
4672
|
-
reason: "budget-exhausted"
|
|
4673
|
-
};
|
|
4674
|
-
if (wantTokens > freeTokens) return {
|
|
4675
|
-
ok: false,
|
|
4676
|
-
reason: "budget-exhausted"
|
|
4677
|
-
};
|
|
4678
|
-
if (wantIterations > freeIterations) return {
|
|
4679
|
-
ok: false,
|
|
4680
|
-
reason: "budget-exhausted"
|
|
4681
|
-
};
|
|
4682
|
-
if (wantUsd > 0 && !usdCapped) return {
|
|
4683
|
-
ok: false,
|
|
4684
|
-
reason: "usd-unbudgeted"
|
|
4685
|
-
};
|
|
4686
|
-
if (wantUsd > freeUsd) return {
|
|
4687
|
-
ok: false,
|
|
4688
|
-
reason: "budget-exhausted"
|
|
6113
|
+
async function driverActAbortable(act, signal) {
|
|
6114
|
+
if (signal.aborted) throw driverAbortError(signal);
|
|
6115
|
+
return await new Promise((resolve, reject) => {
|
|
6116
|
+
let settled = false;
|
|
6117
|
+
const cleanup = () => signal.removeEventListener("abort", onAbort);
|
|
6118
|
+
const onAbort = () => {
|
|
6119
|
+
queueMicrotask(() => {
|
|
6120
|
+
if (settled) return;
|
|
6121
|
+
settled = true;
|
|
6122
|
+
cleanup();
|
|
6123
|
+
reject(driverAbortError(signal));
|
|
6124
|
+
});
|
|
4689
6125
|
};
|
|
4690
|
-
|
|
4691
|
-
|
|
4692
|
-
|
|
4693
|
-
|
|
4694
|
-
|
|
4695
|
-
|
|
4696
|
-
|
|
6126
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
6127
|
+
let work;
|
|
6128
|
+
try {
|
|
6129
|
+
work = Promise.resolve(act());
|
|
6130
|
+
} catch (error) {
|
|
6131
|
+
cleanup();
|
|
6132
|
+
reject(error);
|
|
6133
|
+
return;
|
|
4697
6134
|
}
|
|
4698
|
-
|
|
4699
|
-
|
|
4700
|
-
|
|
4701
|
-
|
|
4702
|
-
|
|
4703
|
-
|
|
4704
|
-
|
|
4705
|
-
|
|
4706
|
-
|
|
4707
|
-
|
|
4708
|
-
|
|
4709
|
-
|
|
4710
|
-
|
|
4711
|
-
|
|
4712
|
-
|
|
4713
|
-
|
|
4714
|
-
|
|
4715
|
-
|
|
4716
|
-
|
|
4717
|
-
|
|
4718
|
-
|
|
4719
|
-
|
|
4720
|
-
|
|
4721
|
-
|
|
4722
|
-
|
|
4723
|
-
|
|
4724
|
-
|
|
4725
|
-
|
|
4726
|
-
|
|
4727
|
-
|
|
4728
|
-
|
|
4729
|
-
|
|
4730
|
-
|
|
4731
|
-
|
|
4732
|
-
|
|
4733
|
-
|
|
4734
|
-
|
|
4735
|
-
|
|
4736
|
-
|
|
4737
|
-
|
|
4738
|
-
|
|
4739
|
-
|
|
4740
|
-
|
|
6135
|
+
work.then((value) => {
|
|
6136
|
+
if (settled) return;
|
|
6137
|
+
settled = true;
|
|
6138
|
+
cleanup();
|
|
6139
|
+
resolve(value);
|
|
6140
|
+
}, (error) => {
|
|
6141
|
+
if (settled) return;
|
|
6142
|
+
settled = true;
|
|
6143
|
+
cleanup();
|
|
6144
|
+
reject(error);
|
|
6145
|
+
});
|
|
6146
|
+
});
|
|
6147
|
+
}
|
|
6148
|
+
function driverAbortError(signal) {
|
|
6149
|
+
const reason = signal.reason;
|
|
6150
|
+
const error = new Error(typeof reason === "string" && reason.length > 0 ? reason : "driver aborted");
|
|
6151
|
+
error.name = "AbortError";
|
|
6152
|
+
return error;
|
|
6153
|
+
}
|
|
6154
|
+
/** The nested tree's full event list — the one evidence the spend, verdict, AND driver-inference
|
|
6155
|
+
* roll-ups read off the same journal the supervisor sums. */
|
|
6156
|
+
async function loadTreeEvents(journal, nestedRoot) {
|
|
6157
|
+
const events = await journal.loadTree(nestedRoot);
|
|
6158
|
+
if (events === void 0) throw new ValidationError(`driverExecutor: nested tree '${nestedRoot}' missing from the journal after run (corrupted log)`);
|
|
6159
|
+
return events;
|
|
6160
|
+
}
|
|
6161
|
+
function isSettled(ev) {
|
|
6162
|
+
return ev.kind === "settled";
|
|
6163
|
+
}
|
|
6164
|
+
/** Sum the conserved spend over the nested tree's settled events — the honest per-channel
|
|
6165
|
+
* roll-up of the whole sub-tree's child WORK. */
|
|
6166
|
+
function sumSpend(settled) {
|
|
6167
|
+
const total = {
|
|
6168
|
+
iterations: 0,
|
|
6169
|
+
tokens: {
|
|
6170
|
+
input: 0,
|
|
6171
|
+
output: 0
|
|
6172
|
+
},
|
|
6173
|
+
usd: 0,
|
|
6174
|
+
ms: 0
|
|
6175
|
+
};
|
|
6176
|
+
for (const ev of settled) {
|
|
6177
|
+
total.iterations += ev.spent.iterations;
|
|
6178
|
+
total.tokens.input += ev.spent.tokens.input;
|
|
6179
|
+
total.tokens.output += ev.spent.tokens.output;
|
|
6180
|
+
if (ev.spent.tokensKnown === false) total.tokensKnown = false;
|
|
6181
|
+
total.usd += ev.spent.usd;
|
|
6182
|
+
if (ev.spent.tokensKnown === false) total.tokensKnown = false;
|
|
6183
|
+
if (ev.spent.usdKnown === false) total.usdKnown = false;
|
|
6184
|
+
total.ms += ev.spent.ms;
|
|
4741
6185
|
}
|
|
4742
|
-
|
|
4743
|
-
|
|
4744
|
-
|
|
4745
|
-
|
|
4746
|
-
|
|
4747
|
-
|
|
4748
|
-
|
|
4749
|
-
|
|
4750
|
-
|
|
4751
|
-
|
|
6186
|
+
return total;
|
|
6187
|
+
}
|
|
6188
|
+
/** Sum the nested tree's `metered` events — the sub-tree's whole driver INFERENCE (this driver's
|
|
6189
|
+
* own turns + any sub-driver inference already re-homed into this tree). Re-homed up to the parent
|
|
6190
|
+
* as one `metered` event; never reconciled (already pool-debited live via `observe`). */
|
|
6191
|
+
function sumMetered(events) {
|
|
6192
|
+
const total = {
|
|
6193
|
+
iterations: 0,
|
|
6194
|
+
tokens: {
|
|
6195
|
+
input: 0,
|
|
6196
|
+
output: 0
|
|
6197
|
+
},
|
|
6198
|
+
usd: 0,
|
|
6199
|
+
ms: 0
|
|
6200
|
+
};
|
|
6201
|
+
for (const ev of events) {
|
|
6202
|
+
if (ev.kind !== "metered") continue;
|
|
6203
|
+
total.iterations += ev.spend.iterations;
|
|
6204
|
+
total.tokens.input += ev.spend.tokens.input;
|
|
6205
|
+
total.tokens.output += ev.spend.tokens.output;
|
|
6206
|
+
if (ev.spend.tokensKnown === false) total.tokensKnown = false;
|
|
6207
|
+
total.usd += ev.spend.usd;
|
|
6208
|
+
if (ev.spend.tokensKnown === false) total.tokensKnown = false;
|
|
6209
|
+
if (ev.spend.usdKnown === false) total.usdKnown = false;
|
|
6210
|
+
total.ms += ev.spend.ms;
|
|
4752
6211
|
}
|
|
4753
|
-
|
|
6212
|
+
return total;
|
|
6213
|
+
}
|
|
6214
|
+
function zeroSpend() {
|
|
6215
|
+
return {
|
|
6216
|
+
iterations: 0,
|
|
6217
|
+
tokens: {
|
|
6218
|
+
input: 0,
|
|
6219
|
+
output: 0
|
|
6220
|
+
},
|
|
6221
|
+
usd: 0,
|
|
6222
|
+
ms: 0
|
|
6223
|
+
};
|
|
6224
|
+
}
|
|
6225
|
+
function addSpend$1(a, b) {
|
|
6226
|
+
return {
|
|
6227
|
+
iterations: a.iterations + b.iterations,
|
|
6228
|
+
tokens: {
|
|
6229
|
+
input: a.tokens.input + b.tokens.input,
|
|
6230
|
+
output: a.tokens.output + b.tokens.output
|
|
6231
|
+
},
|
|
6232
|
+
...a.tokensKnown === false || b.tokensKnown === false ? { tokensKnown: false } : {},
|
|
6233
|
+
usd: a.usd + b.usd,
|
|
6234
|
+
...a.usdKnown === false || b.usdKnown === false ? { usdKnown: false } : {},
|
|
6235
|
+
ms: a.ms + b.ms
|
|
6236
|
+
};
|
|
6237
|
+
}
|
|
6238
|
+
/** An all-zero spend that carries an UNKNOWN marker counts as non-zero: a sub-driver whose turns
|
|
6239
|
+
* went unmeasured did real work, and dropping its `metered` event here would re-hide upstream the
|
|
6240
|
+
* very turn the driver refused to skip. */
|
|
6241
|
+
function isNonZeroSpend(s) {
|
|
6242
|
+
return s.iterations > 0 || s.tokens.input > 0 || s.tokens.output > 0 || s.usd > 0 || s.ms > 0 || s.tokensKnown === false || s.usdKnown === false;
|
|
6243
|
+
}
|
|
6244
|
+
/** A spend, or `undefined` when it is all-zero — so `metered()` returns undefined for a driver
|
|
6245
|
+
* whose sub-tree did no inference (and the parent journals no empty `metered` event). */
|
|
6246
|
+
function nonZeroOrUndef(s) {
|
|
6247
|
+
return isNonZeroSpend(s) ? s : void 0;
|
|
6248
|
+
}
|
|
6249
|
+
/** Sum the nested tree's metered events, tolerating a missing tree (a crash before `beginTree`
|
|
6250
|
+
* landed) — never throw here, or it would mask the original `act` error on the crash path. */
|
|
6251
|
+
async function safeRollup(journal, nestedRoot) {
|
|
6252
|
+
try {
|
|
6253
|
+
const events = await loadTreeEvents(journal, nestedRoot);
|
|
6254
|
+
const childWork = sumSpend(events.filter(isSettled));
|
|
6255
|
+
const metered = nonZeroOrUndef(sumMetered(events));
|
|
4754
6256
|
return {
|
|
4755
|
-
|
|
4756
|
-
|
|
4757
|
-
|
|
4758
|
-
|
|
4759
|
-
|
|
4760
|
-
reservedTokens
|
|
6257
|
+
metered,
|
|
6258
|
+
accounting: {
|
|
6259
|
+
reported: childWork,
|
|
6260
|
+
reservation: addSpend$1(childWork, metered ?? zeroSpend())
|
|
6261
|
+
}
|
|
4761
6262
|
};
|
|
6263
|
+
} catch {
|
|
6264
|
+
return;
|
|
4762
6265
|
}
|
|
4763
|
-
|
|
4764
|
-
|
|
6266
|
+
}
|
|
6267
|
+
/** Derive the driver child's delivery verdict from its DIRECT children's settlements and the
|
|
6268
|
+
* manager's actual finalized output: `valid` iff a direct child delivered AND the finalizer
|
|
6269
|
+
* returned a defined output;
|
|
6270
|
+
* `score` = the best delivered score. Returns `undefined` when no child settled at all (the
|
|
6271
|
+
* driver itself produced nothing to bubble a verdict from). Fail-closed: a child whose verdict
|
|
6272
|
+
* carried no `valid` counts as not-delivered. */
|
|
6273
|
+
function deriveDeliveryVerdict(settled, finalizedOutput) {
|
|
6274
|
+
let sawChild = false;
|
|
6275
|
+
let anyValid = false;
|
|
6276
|
+
let bestValidScore;
|
|
6277
|
+
let bestDoneScore;
|
|
6278
|
+
for (const ev of settled) {
|
|
6279
|
+
sawChild = true;
|
|
6280
|
+
if (ev.status !== "done") continue;
|
|
6281
|
+
const score = ev.verdict?.score;
|
|
6282
|
+
if (score !== void 0 && (bestDoneScore === void 0 || score > bestDoneScore)) bestDoneScore = score;
|
|
6283
|
+
if (ev.verdict?.valid === true) {
|
|
6284
|
+
anyValid = true;
|
|
6285
|
+
if (score !== void 0 && (bestValidScore === void 0 || score > bestValidScore)) bestValidScore = score;
|
|
6286
|
+
}
|
|
4765
6287
|
}
|
|
6288
|
+
if (!sawChild) return void 0;
|
|
6289
|
+
const accepted = anyValid && finalizedOutput !== void 0;
|
|
4766
6290
|
return {
|
|
4767
|
-
|
|
4768
|
-
|
|
4769
|
-
spendFrom: foldUsage,
|
|
4770
|
-
readout,
|
|
4771
|
-
observe,
|
|
4772
|
-
assertNoOpenTickets
|
|
6291
|
+
valid: accepted,
|
|
6292
|
+
score: accepted ? bestValidScore ?? 1 : bestDoneScore ?? 0
|
|
4773
6293
|
};
|
|
4774
6294
|
}
|
|
6295
|
+
function readNestedScopeSeam(ctx) {
|
|
6296
|
+
const seam = ctx.seams[nestedScopeSeamKey];
|
|
6297
|
+
if (!seam || typeof seam.mount !== "function") throw new ValidationError(`driverExecutor: missing required seam "${nestedScopeSeamKey}" — a driver child must be spawned through a Scope that seeds it (the keystone scope does)`);
|
|
6298
|
+
return seam;
|
|
6299
|
+
}
|
|
6300
|
+
function isAgent(value) {
|
|
6301
|
+
return typeof value === "object" && value !== null && typeof value.act === "function" && typeof value.name === "string";
|
|
6302
|
+
}
|
|
4775
6303
|
//#endregion
|
|
4776
6304
|
//#region src/runtime/supervise/finalizer.ts
|
|
4777
6305
|
/**
|
|
@@ -4923,20 +6451,99 @@ function keyedAssignments(events, settled) {
|
|
|
4923
6451
|
const spawns = events.filter((ev) => ev.kind === "spawned").sort((a, b) => a.seq - b.seq);
|
|
4924
6452
|
for (const ev of spawns) {
|
|
4925
6453
|
if (ev.key === void 0) continue;
|
|
6454
|
+
if (ev.identity === void 0) throw new RuntimeRunStateError(`supervisor: keyed node '${ev.id}' has no durable execution identity`);
|
|
4926
6455
|
const s = byId.get(ev.id);
|
|
4927
6456
|
keys.set(ev.key, s === void 0 ? {
|
|
4928
6457
|
id: ev.id,
|
|
4929
6458
|
label: ev.label,
|
|
6459
|
+
identity: ev.identity,
|
|
4930
6460
|
state: "in-doubt"
|
|
4931
6461
|
} : {
|
|
4932
6462
|
id: ev.id,
|
|
4933
6463
|
label: ev.label,
|
|
6464
|
+
identity: ev.identity,
|
|
4934
6465
|
state: s.kind === "done" ? "completed" : "down",
|
|
4935
6466
|
settled: s
|
|
4936
6467
|
});
|
|
4937
6468
|
}
|
|
4938
6469
|
return keys;
|
|
4939
6470
|
}
|
|
6471
|
+
/** A resumed process may continue only the exact root contract first recorded for this run id. */
|
|
6472
|
+
function assertResumeContract(events, opts) {
|
|
6473
|
+
const roots = events.filter((event) => event.kind === "spawned" && event.id === opts.runId && event.parent === void 0);
|
|
6474
|
+
if (roots.length !== 1) throw new RuntimeRunStateError(`supervisor: resumed run '${opts.runId}' must contain exactly one root identity event; found ${roots.length}`);
|
|
6475
|
+
const recorded = roots[0];
|
|
6476
|
+
if (contentAddress(recorded.budget) !== contentAddress(opts.budget)) throw new RuntimeRunStateError(`supervisor: resume budget mismatch for run '${opts.runId}'; use a new runId to change limits`);
|
|
6477
|
+
if (!sameOptionalIdentity(recorded.identity, opts.rootIdentity)) throw new RuntimeRunStateError(`supervisor: resume identity mismatch for run '${opts.runId}'; task, profile, candidate, and correlation must match`);
|
|
6478
|
+
const receipts = events.filter((event) => event.kind === "materialized" && event.id === opts.runId);
|
|
6479
|
+
const expectedReceipt = rootMaterializationReceipt(opts);
|
|
6480
|
+
if (expectedReceipt === void 0) {
|
|
6481
|
+
if (receipts.length > 1) throw new RuntimeRunStateError(`supervisor: resumed run '${opts.runId}' contains duplicate root materialization evidence`);
|
|
6482
|
+
} else if (receipts.length !== 1 || contentAddress(receipts[0]?.receipt) !== contentAddress(expectedReceipt)) throw new RuntimeRunStateError(`supervisor: resume materialization mismatch for run '${opts.runId}'; backend, model, execution identity, and plan must match`);
|
|
6483
|
+
return recorded;
|
|
6484
|
+
}
|
|
6485
|
+
/** Mint root evidence at the trusted composition boundary. A generic in-process Agent root has no
|
|
6486
|
+
* executor declaration and remains explicitly unknown rather than inheriting fabricated details. */
|
|
6487
|
+
function rootMaterializationReceipt(opts) {
|
|
6488
|
+
const declaration = opts.rootMaterialization;
|
|
6489
|
+
if (declaration === void 0) return unknownMaterializationReceipt({
|
|
6490
|
+
...opts.rootIdentity?.profileDigest === void 0 ? {} : { authoredProfileDigest: opts.rootIdentity.profileDigest },
|
|
6491
|
+
runtime: "inline",
|
|
6492
|
+
reason: "root-agent-did-not-report"
|
|
6493
|
+
});
|
|
6494
|
+
if (declaration.declaration === "deferred") return void 0;
|
|
6495
|
+
if (opts.rootIdentity?.profileDigest === void 0) throw new RuntimeRunStateError(`supervisor: run '${opts.runId}' cannot record known root materialization without an exact profileDigest`);
|
|
6496
|
+
try {
|
|
6497
|
+
return knownMaterializationReceipt({
|
|
6498
|
+
authoredProfileDigest: opts.rootIdentity.profileDigest,
|
|
6499
|
+
runtime: declaration.runtime,
|
|
6500
|
+
declaration: declaration.declaration
|
|
6501
|
+
});
|
|
6502
|
+
} catch (error) {
|
|
6503
|
+
throw new RuntimeRunStateError(`supervisor: run '${opts.runId}' has invalid root materialization evidence`, { cause: error });
|
|
6504
|
+
}
|
|
6505
|
+
}
|
|
6506
|
+
function rootExecutionBindingReceipt(opts, materialization, attemptId) {
|
|
6507
|
+
const root = opts.rootMaterialization;
|
|
6508
|
+
if (root?.declaration === "deferred") return void 0;
|
|
6509
|
+
if (root === void 0) return unknownExecutionBindingReceipt(materialization, attemptId, "root-agent-did-not-report");
|
|
6510
|
+
try {
|
|
6511
|
+
return knownExecutionBindingReceipt(materialization, {
|
|
6512
|
+
attemptId,
|
|
6513
|
+
...root.binding
|
|
6514
|
+
});
|
|
6515
|
+
} catch (error) {
|
|
6516
|
+
throw new RuntimeRunStateError(`supervisor: run '${opts.runId}' has invalid root execution binding`, { cause: error });
|
|
6517
|
+
}
|
|
6518
|
+
}
|
|
6519
|
+
function sameOptionalIdentity(recorded, requested) {
|
|
6520
|
+
if (recorded === void 0 || requested === void 0) return recorded === requested;
|
|
6521
|
+
return contentAddress(recorded) === contentAddress(requested);
|
|
6522
|
+
}
|
|
6523
|
+
/** Validate caller-supplied root identity before any journal access. Fresh runs may omit identity,
|
|
6524
|
+
* but once supplied it is trusted durable evidence and therefore must be complete and canonical;
|
|
6525
|
+
* resumed runs always require it so the prior root can be matched exactly. */
|
|
6526
|
+
function assertRootIdentity(opts) {
|
|
6527
|
+
const identity = opts.rootIdentity;
|
|
6528
|
+
if (identity === void 0) {
|
|
6529
|
+
if (opts.resume !== true) return;
|
|
6530
|
+
throw new RuntimeRunStateError(`supervisor: resumed run '${opts.runId}' requires an exact rootIdentity with profileDigest and taskDigest`);
|
|
6531
|
+
}
|
|
6532
|
+
const unknownFields = Object.keys(identity).filter((key) => key !== "profileDigest" && key !== "taskDigest" && key !== "candidateDigest" && key !== "correlation");
|
|
6533
|
+
const correlation = identity.correlation;
|
|
6534
|
+
const validCorrelation = correlation === void 0 || typeof correlation === "object" && correlation !== null && !Array.isArray(correlation) && Object.entries(correlation).every(([key, value]) => key.length > 0 && typeof value === "string" && value.length > 0);
|
|
6535
|
+
if (unknownFields.length > 0 || !sha256DigestSchema.safeParse(identity.profileDigest).success || !sha256DigestSchema.safeParse(identity.taskDigest).success || identity.candidateDigest !== void 0 && !sha256DigestSchema.safeParse(identity.candidateDigest).success || !validCorrelation) throw new RuntimeRunStateError(`supervisor: run '${opts.runId}' requires an exact rootIdentity with valid profileDigest, taskDigest, candidateDigest, and correlation`);
|
|
6536
|
+
}
|
|
6537
|
+
function rootDeadline(root) {
|
|
6538
|
+
const startedAt = Date.parse(root.at);
|
|
6539
|
+
if (!Number.isFinite(startedAt)) throw new RuntimeRunStateError(`supervisor: root event for '${root.id}' has an invalid timestamp '${root.at}'`);
|
|
6540
|
+
return startedAt + (root.budget.deadlineMs ?? 0);
|
|
6541
|
+
}
|
|
6542
|
+
/** Child reservations whose spawn was durable but whose terminal record never landed. */
|
|
6543
|
+
function uncertainSpawnBudgets(events) {
|
|
6544
|
+
const terminal = new Set(events.filter((event) => event.kind === "settled" || event.kind === "cancelled" || event.kind === "woken").map((event) => event.id));
|
|
6545
|
+
return events.filter((event) => event.kind === "spawned" && event.parent !== void 0 && !terminal.has(event.id)).map((event) => event.budget);
|
|
6546
|
+
}
|
|
4940
6547
|
/** Highest `seq` among events matching `pred`, or `-1` when none match (so a resumed scope's
|
|
4941
6548
|
* first new ordinal/seq is 0 — the same start a fresh scope uses). */
|
|
4942
6549
|
function maxSeqOf(events, pred) {
|
|
@@ -4974,129 +6581,239 @@ function describeRejection(error) {
|
|
|
4974
6581
|
function createSupervisor() {
|
|
4975
6582
|
let attached;
|
|
4976
6583
|
async function run(root, task, opts) {
|
|
4977
|
-
const now = opts
|
|
4978
|
-
const
|
|
4979
|
-
|
|
4980
|
-
|
|
4981
|
-
|
|
4982
|
-
|
|
4983
|
-
|
|
4984
|
-
|
|
4985
|
-
|
|
4986
|
-
|
|
4987
|
-
|
|
4988
|
-
|
|
4989
|
-
|
|
4990
|
-
|
|
4991
|
-
|
|
4992
|
-
|
|
4993
|
-
|
|
4994
|
-
|
|
4995
|
-
|
|
4996
|
-
|
|
4997
|
-
|
|
4998
|
-
|
|
4999
|
-
|
|
5000
|
-
|
|
5001
|
-
|
|
5002
|
-
seq: 0,
|
|
5003
|
-
at: new Date(now()).toISOString()
|
|
5004
|
-
});
|
|
5005
|
-
}
|
|
5006
|
-
const controller = new AbortController();
|
|
5007
|
-
const cascadeAbort = (reason) => {
|
|
5008
|
-
if (controller.signal.aborted) return;
|
|
5009
|
-
controller.abort(reason);
|
|
5010
|
-
};
|
|
5011
|
-
const onCallerAbort = () => cascadeAbort("caller signal aborted");
|
|
5012
|
-
if (opts.signal) if (opts.signal.aborted) cascadeAbort("caller signal aborted");
|
|
5013
|
-
else opts.signal.addEventListener("abort", onCallerAbort, { once: true });
|
|
5014
|
-
const breaker = createIntensityBreaker(opts, () => cascadeAbort("intensity breaker tripped"));
|
|
5015
|
-
const journal = wrapJournalForBreaker(opts.journal, breaker);
|
|
5016
|
-
const scope = createScope({
|
|
5017
|
-
parentId: opts.runId,
|
|
5018
|
-
root: opts.runId,
|
|
5019
|
-
pool,
|
|
5020
|
-
journal,
|
|
5021
|
-
blobs: opts.blobs,
|
|
5022
|
-
executors: opts.executors,
|
|
5023
|
-
seams: {},
|
|
5024
|
-
depth: 0,
|
|
5025
|
-
maxDepth: opts.maxDepth ?? defaultMaxDepth,
|
|
5026
|
-
signal: controller.signal,
|
|
5027
|
-
now,
|
|
5028
|
-
hooks: opts.hooks,
|
|
5029
|
-
...opts.probes ? { probes: opts.probes } : {},
|
|
5030
|
-
...opts.workerTrace ? { workerTrace: opts.workerTrace } : {},
|
|
5031
|
-
...resumeFrom ? { resumeFrom } : {}
|
|
5032
|
-
});
|
|
5033
|
-
const openScope = scope;
|
|
5034
|
-
if (attached) attached.bind({
|
|
5035
|
-
scope: openScope,
|
|
5036
|
-
cascadeAbort,
|
|
5037
|
-
signal: pushRootSignal(cascadeAbort)
|
|
6584
|
+
const { budget, rootIdentity, rootMaterialization, runId, journal: journalStore, blobs: blobStore, executors: executorRegistry, probes, maxDepth, maxLiveWorkers, maxRestarts, withinMs, resume, now: suppliedNow, signal, hooks, workerTrace } = opts;
|
|
6585
|
+
const input = detachedSnapshot({
|
|
6586
|
+
task,
|
|
6587
|
+
options: {
|
|
6588
|
+
budget,
|
|
6589
|
+
runId,
|
|
6590
|
+
...rootIdentity === void 0 ? {} : { rootIdentity },
|
|
6591
|
+
...rootMaterialization === void 0 ? {} : { rootMaterialization },
|
|
6592
|
+
...maxDepth === void 0 ? {} : { maxDepth },
|
|
6593
|
+
...maxLiveWorkers === void 0 ? {} : { maxLiveWorkers },
|
|
6594
|
+
...maxRestarts === void 0 ? {} : { maxRestarts },
|
|
6595
|
+
...withinMs === void 0 ? {} : { withinMs },
|
|
6596
|
+
...resume === void 0 ? {} : { resume }
|
|
6597
|
+
}
|
|
6598
|
+
}, "supervisor.run");
|
|
6599
|
+
opts = Object.freeze({
|
|
6600
|
+
...input.options,
|
|
6601
|
+
journal: journalStore,
|
|
6602
|
+
blobs: blobStore,
|
|
6603
|
+
executors: executorRegistry,
|
|
6604
|
+
...probes === void 0 ? {} : { probes },
|
|
6605
|
+
...suppliedNow === void 0 ? {} : { now: suppliedNow },
|
|
6606
|
+
...signal === void 0 ? {} : { signal },
|
|
6607
|
+
...hooks === void 0 ? {} : { hooks },
|
|
6608
|
+
...workerTrace === void 0 ? {} : { workerTrace }
|
|
5038
6609
|
});
|
|
5039
|
-
|
|
6610
|
+
task = input.task;
|
|
6611
|
+
const rootAct = root.act.bind(root);
|
|
6612
|
+
const rootDeliver = root.deliver?.bind(root);
|
|
6613
|
+
const now = opts.now ?? Date.now;
|
|
6614
|
+
assertRootIdentity(opts);
|
|
6615
|
+
const rootLease = attached?.acquire();
|
|
5040
6616
|
try {
|
|
5041
|
-
|
|
5042
|
-
|
|
5043
|
-
|
|
6617
|
+
const rootAttemptId = newExecutionAttemptId(opts.runId);
|
|
6618
|
+
const runStartedAtMs = now();
|
|
6619
|
+
const runStartedAt = new Date(runStartedAtMs).toISOString();
|
|
6620
|
+
const existing = await opts.journal.loadTree(opts.runId);
|
|
6621
|
+
if (opts.resume !== true && existing !== void 0) throw new RuntimeRunStateError(`supervisor: runId '${opts.runId}' already exists; pass resume: true to continue it or use a new runId`);
|
|
6622
|
+
const prior = opts.resume === true ? existing : void 0;
|
|
6623
|
+
const resuming = prior !== void 0 && prior.length > 0;
|
|
6624
|
+
let resumeFrom;
|
|
6625
|
+
let pool;
|
|
6626
|
+
if (resuming) {
|
|
6627
|
+
const rootEvent = assertResumeContract(prior, opts);
|
|
6628
|
+
const measured = sumMeasuredSpendFromEvents(prior);
|
|
6629
|
+
const uncertainReservations = uncertainSpawnBudgets(prior);
|
|
6630
|
+
pool = createBudgetPool(opts.budget, now, {
|
|
6631
|
+
committed: addSpend(measured.childWork, measured.driverInference),
|
|
6632
|
+
uncertainReservations,
|
|
6633
|
+
...rootEvent.budget.deadlineMs !== void 0 ? { absoluteDeadlineMs: rootDeadline(rootEvent) } : {}
|
|
6634
|
+
});
|
|
6635
|
+
const settled = await replaySpawnTree(opts.journal, opts.blobs, opts.runId);
|
|
6636
|
+
resumeFrom = {
|
|
6637
|
+
settled,
|
|
6638
|
+
view: materializeTreeView(prior),
|
|
6639
|
+
maxSpawnOrdinal: maxSeqOf(prior, (ev) => ev.kind === "spawned"),
|
|
6640
|
+
maxCursorSeq: maxSeqOf(prior, (ev) => ev.kind === "settled" || ev.kind === "cancelled" || ev.kind === "woken"),
|
|
6641
|
+
maxWaitOrdinal: maxSeqOf(prior, (ev) => ev.kind === "waiting"),
|
|
6642
|
+
waits: pendingWaits(prior),
|
|
6643
|
+
keys: keyedAssignments(prior, settled),
|
|
6644
|
+
priorSpend: sumSpendFromEvents(prior)
|
|
6645
|
+
};
|
|
6646
|
+
} else {
|
|
6647
|
+
pool = createBudgetPool(opts.budget, now, { ...opts.budget.deadlineMs !== void 0 ? { absoluteDeadlineMs: runStartedAtMs + opts.budget.deadlineMs } : {} });
|
|
6648
|
+
await opts.journal.beginTree(opts.runId, runStartedAt);
|
|
6649
|
+
const rootReceipt = rootMaterializationReceipt(opts);
|
|
6650
|
+
const rootRuntime = opts.rootMaterialization?.runtime ?? "inline";
|
|
6651
|
+
await opts.journal.appendEvent(opts.runId, {
|
|
6652
|
+
kind: "spawned",
|
|
6653
|
+
id: opts.runId,
|
|
6654
|
+
label: "root",
|
|
6655
|
+
budget: opts.budget,
|
|
6656
|
+
runtime: rootRuntime,
|
|
6657
|
+
...opts.rootIdentity ? { identity: opts.rootIdentity } : {},
|
|
6658
|
+
seq: 0,
|
|
6659
|
+
at: runStartedAt
|
|
6660
|
+
});
|
|
6661
|
+
if (rootReceipt !== void 0) await opts.journal.appendEvent(opts.runId, {
|
|
6662
|
+
kind: "materialized",
|
|
6663
|
+
id: opts.runId,
|
|
6664
|
+
receipt: rootReceipt,
|
|
6665
|
+
seq: 0,
|
|
6666
|
+
at: runStartedAt
|
|
6667
|
+
});
|
|
6668
|
+
}
|
|
6669
|
+
const stableRootReceipt = resuming ? prior?.find((event) => event.kind === "materialized" && event.id === opts.runId)?.receipt : rootMaterializationReceipt(opts);
|
|
6670
|
+
if (stableRootReceipt !== void 0) {
|
|
6671
|
+
const rootBinding = rootExecutionBindingReceipt(opts, stableRootReceipt, rootAttemptId);
|
|
6672
|
+
if (rootBinding !== void 0) await opts.journal.appendEvent(opts.runId, {
|
|
6673
|
+
kind: "execution-bound",
|
|
6674
|
+
id: opts.runId,
|
|
6675
|
+
binding: rootBinding,
|
|
6676
|
+
seq: 0,
|
|
6677
|
+
at: runStartedAt
|
|
6678
|
+
});
|
|
6679
|
+
}
|
|
6680
|
+
const controller = new AbortController();
|
|
6681
|
+
const cascadeAbort = (reason) => {
|
|
6682
|
+
if (controller.signal.aborted) return false;
|
|
6683
|
+
controller.abort(reason);
|
|
6684
|
+
return true;
|
|
5044
6685
|
};
|
|
5045
|
-
|
|
5046
|
-
|
|
6686
|
+
const onCallerAbort = () => cascadeAbort("caller signal aborted");
|
|
6687
|
+
if (opts.signal) if (opts.signal.aborted) cascadeAbort("caller signal aborted");
|
|
6688
|
+
else opts.signal.addEventListener("abort", onCallerAbort, { once: true });
|
|
6689
|
+
const breaker = createIntensityBreaker(opts, () => cascadeAbort("intensity breaker tripped"));
|
|
6690
|
+
const journal = wrapJournalForBreaker(opts.journal, breaker);
|
|
6691
|
+
const priorRootMaterialization = prior?.find((event) => event.kind === "materialized" && event.id === opts.runId)?.receipt;
|
|
6692
|
+
const scope = createScope({
|
|
6693
|
+
parentId: opts.runId,
|
|
6694
|
+
root: opts.runId,
|
|
6695
|
+
pool,
|
|
6696
|
+
journal,
|
|
6697
|
+
blobs: opts.blobs,
|
|
6698
|
+
executors: opts.executors,
|
|
6699
|
+
seams: {},
|
|
6700
|
+
depth: 0,
|
|
6701
|
+
maxDepth: opts.maxDepth ?? defaultMaxDepth,
|
|
6702
|
+
...opts.maxLiveWorkers !== void 0 ? { maxLiveWorkers: opts.maxLiveWorkers } : {},
|
|
6703
|
+
signal: controller.signal,
|
|
6704
|
+
now,
|
|
6705
|
+
hooks: opts.hooks,
|
|
6706
|
+
...opts.rootMaterialization?.declaration === "deferred" ? { ownerMaterialization: {
|
|
6707
|
+
runtime: opts.rootMaterialization.runtime,
|
|
6708
|
+
authoredProfile: opts.rootMaterialization.authoredProfile,
|
|
6709
|
+
attemptId: rootAttemptId,
|
|
6710
|
+
requiredKnown: true,
|
|
6711
|
+
...priorRootMaterialization === void 0 ? {} : { prior: priorRootMaterialization }
|
|
6712
|
+
} } : {},
|
|
6713
|
+
...opts.probes ? { probes: opts.probes } : {},
|
|
6714
|
+
...opts.workerTrace ? { workerTrace: opts.workerTrace } : {},
|
|
6715
|
+
...resumeFrom ? { resumeFrom } : {}
|
|
6716
|
+
});
|
|
6717
|
+
const openScope = scope;
|
|
6718
|
+
if (rootLease) rootLease.bind({
|
|
6719
|
+
scope: openScope,
|
|
6720
|
+
cascadeAbort,
|
|
6721
|
+
signal: pushRootSignal(cascadeAbort),
|
|
6722
|
+
deliver: rootDeliver ? (message) => rootDeliver(message) !== false : () => false
|
|
6723
|
+
});
|
|
6724
|
+
let deadlineExceeded = false;
|
|
6725
|
+
const rootDeadlineAtMs = pool.readout().deadlineMs;
|
|
6726
|
+
if (rootDeadlineAtMs > 0 && now() >= rootDeadlineAtMs) deadlineExceeded = cascadeAbort("root budget deadline exceeded");
|
|
6727
|
+
const clearRootDeadline = opts.budget.deadlineMs === void 0 ? void 0 : armDeadlineTimer(Math.max(0, rootDeadlineAtMs - now()), () => {
|
|
6728
|
+
deadlineExceeded = cascadeAbort("root budget deadline exceeded");
|
|
6729
|
+
});
|
|
6730
|
+
let actOutcome = {
|
|
5047
6731
|
ok: false,
|
|
5048
|
-
error
|
|
6732
|
+
error: new RuntimeRunStateError("supervisor: root execution did not start")
|
|
5049
6733
|
};
|
|
5050
|
-
|
|
5051
|
-
|
|
5052
|
-
|
|
5053
|
-
|
|
5054
|
-
|
|
5055
|
-
|
|
5056
|
-
|
|
6734
|
+
let executionAborted = controller.signal.aborted;
|
|
6735
|
+
try {
|
|
6736
|
+
actOutcome = {
|
|
6737
|
+
ok: true,
|
|
6738
|
+
out: await runAbortable(() => rootAct(task, scope), controller.signal)
|
|
6739
|
+
};
|
|
6740
|
+
} catch (error) {
|
|
6741
|
+
actOutcome = {
|
|
6742
|
+
ok: false,
|
|
6743
|
+
error
|
|
6744
|
+
};
|
|
6745
|
+
} finally {
|
|
6746
|
+
executionAborted = controller.signal.aborted;
|
|
6747
|
+
if (!controller.signal.aborted && rootDeadlineAtMs > 0 && now() >= rootDeadlineAtMs) deadlineExceeded = true;
|
|
6748
|
+
clearRootDeadline?.();
|
|
6749
|
+
try {
|
|
6750
|
+
await drainLiveChildren(openScope, controller);
|
|
6751
|
+
} catch (error) {
|
|
6752
|
+
if (actOutcome?.ok !== false) actOutcome = {
|
|
6753
|
+
ok: false,
|
|
6754
|
+
error
|
|
6755
|
+
};
|
|
6756
|
+
}
|
|
6757
|
+
try {
|
|
6758
|
+
await finalizeScopeOwnerMaterialization(openScope);
|
|
6759
|
+
} catch (error) {
|
|
6760
|
+
if (actOutcome?.ok !== false) actOutcome = {
|
|
6761
|
+
ok: false,
|
|
6762
|
+
error
|
|
6763
|
+
};
|
|
6764
|
+
}
|
|
6765
|
+
if (opts.signal) opts.signal.removeEventListener("abort", onCallerAbort);
|
|
6766
|
+
rootLease?.release();
|
|
6767
|
+
}
|
|
6768
|
+
const tree = runTree(scope);
|
|
5057
6769
|
pool.assertNoOpenTickets();
|
|
5058
|
-
|
|
5059
|
-
|
|
5060
|
-
const
|
|
5061
|
-
|
|
6770
|
+
if (actOutcome.ok) {
|
|
6771
|
+
if (executionAborted) return noWinner();
|
|
6772
|
+
const out = actOutcome.out;
|
|
6773
|
+
if (out !== void 0) {
|
|
6774
|
+
const outRef = contentAddress(out);
|
|
6775
|
+
await opts.blobs.put(outRef, out);
|
|
6776
|
+
const { childWork, driverInference } = await spentFromJournal(journal, opts.runId);
|
|
6777
|
+
return {
|
|
6778
|
+
kind: "winner",
|
|
6779
|
+
out,
|
|
6780
|
+
outRef,
|
|
6781
|
+
tree,
|
|
6782
|
+
spentTotal: addSpend(childWork, driverInference),
|
|
6783
|
+
...isNonEmptySpend(driverInference) ? { spentBreakdown: {
|
|
6784
|
+
driverInference,
|
|
6785
|
+
childWork
|
|
6786
|
+
} } : {}
|
|
6787
|
+
};
|
|
6788
|
+
}
|
|
6789
|
+
return noWinner();
|
|
6790
|
+
}
|
|
6791
|
+
return noWinner({ error: actOutcome.error });
|
|
6792
|
+
async function noWinner(rejection) {
|
|
5062
6793
|
const { childWork, driverInference } = await spentFromJournal(journal, opts.runId);
|
|
5063
|
-
|
|
5064
|
-
kind: "winner",
|
|
5065
|
-
out,
|
|
5066
|
-
outRef,
|
|
6794
|
+
const common = {
|
|
6795
|
+
kind: "no-winner",
|
|
5067
6796
|
tree,
|
|
5068
|
-
|
|
5069
|
-
|
|
5070
|
-
|
|
5071
|
-
|
|
5072
|
-
|
|
6797
|
+
downCount: breaker.downCount(),
|
|
6798
|
+
spentTotal: addSpend(childWork, driverInference)
|
|
6799
|
+
};
|
|
6800
|
+
const lifecycle = classifyNoWinner(controller, pool, opts, breaker, deadlineExceeded, tree);
|
|
6801
|
+
if (lifecycle !== void 0) return {
|
|
6802
|
+
...common,
|
|
6803
|
+
reason: lifecycle
|
|
6804
|
+
};
|
|
6805
|
+
if (rejection !== void 0) return {
|
|
6806
|
+
...common,
|
|
6807
|
+
reason: "driver-failed",
|
|
6808
|
+
error: describeRejection(rejection.error)
|
|
6809
|
+
};
|
|
6810
|
+
return {
|
|
6811
|
+
...common,
|
|
6812
|
+
reason: "all-children-down"
|
|
5073
6813
|
};
|
|
5074
6814
|
}
|
|
5075
|
-
|
|
5076
|
-
|
|
5077
|
-
return noWinner({ error: actOutcome.error });
|
|
5078
|
-
async function noWinner(rejection) {
|
|
5079
|
-
const { childWork, driverInference } = await spentFromJournal(journal, opts.runId);
|
|
5080
|
-
const common = {
|
|
5081
|
-
kind: "no-winner",
|
|
5082
|
-
tree,
|
|
5083
|
-
downCount: breaker.downCount(),
|
|
5084
|
-
spentTotal: addSpend(childWork, driverInference)
|
|
5085
|
-
};
|
|
5086
|
-
const lifecycle = classifyNoWinner(controller, pool, opts, breaker);
|
|
5087
|
-
if (lifecycle !== void 0) return {
|
|
5088
|
-
...common,
|
|
5089
|
-
reason: lifecycle
|
|
5090
|
-
};
|
|
5091
|
-
if (rejection !== void 0) return {
|
|
5092
|
-
...common,
|
|
5093
|
-
reason: "driver-failed",
|
|
5094
|
-
error: describeRejection(rejection.error)
|
|
5095
|
-
};
|
|
5096
|
-
return {
|
|
5097
|
-
...common,
|
|
5098
|
-
reason: "all-children-down"
|
|
5099
|
-
};
|
|
6815
|
+
} finally {
|
|
6816
|
+
rootLease?.release();
|
|
5100
6817
|
}
|
|
5101
6818
|
}
|
|
5102
6819
|
function attach(h) {
|
|
@@ -5113,6 +6830,56 @@ function createSupervisor() {
|
|
|
5113
6830
|
* can prove a handle is ours and reach its binding without leaking the control onto the
|
|
5114
6831
|
* frozen `RootHandle` shape. */
|
|
5115
6832
|
const rootControls = /* @__PURE__ */ new WeakMap();
|
|
6833
|
+
/**
|
|
6834
|
+
* Mint a `RootHandle` plus its supervisor-private control. The handle is the substrate a
|
|
6835
|
+
* chat/pi-viz client attaches to (Q2): `view()` reads the live tree, `signal()` delivers
|
|
6836
|
+
* an out-of-band message, `abort()` cascades. Before `run` binds it (and after `run`
|
|
6837
|
+
* unbinds it) the handle is fail-loud: a client that talks to a handle that is not
|
|
6838
|
+
* driving a live run gets a typed error, never a silent no-op.
|
|
6839
|
+
*/
|
|
6840
|
+
function createRootHandle() {
|
|
6841
|
+
let binding;
|
|
6842
|
+
let activeLease;
|
|
6843
|
+
const handle = {
|
|
6844
|
+
view() {
|
|
6845
|
+
if (!binding) throw new RuntimeRunStateError("RootHandle.view: handle is not bound to a live run (attach it before run, read after run starts)");
|
|
6846
|
+
return binding.scope.view;
|
|
6847
|
+
},
|
|
6848
|
+
deliver(msg) {
|
|
6849
|
+
if (!binding) throw new RuntimeRunStateError("RootHandle.deliver: handle is not bound to a live run");
|
|
6850
|
+
return binding.deliver(msg);
|
|
6851
|
+
},
|
|
6852
|
+
signal(msg) {
|
|
6853
|
+
if (!binding) throw new RuntimeRunStateError("RootHandle.signal: handle is not bound to a live run");
|
|
6854
|
+
binding.signal(msg);
|
|
6855
|
+
},
|
|
6856
|
+
abort(reason) {
|
|
6857
|
+
if (!binding) throw new RuntimeRunStateError("RootHandle.abort: handle is not bound to a live run");
|
|
6858
|
+
binding.cascadeAbort(reason ?? "root handle aborted");
|
|
6859
|
+
}
|
|
6860
|
+
};
|
|
6861
|
+
rootControls.set(handle, { acquire() {
|
|
6862
|
+
if (activeLease !== void 0) throw new RuntimeRunStateError("RootHandle: handle already controls a live run; use one handle per concurrent run");
|
|
6863
|
+
const token = Symbol("root-handle-lease");
|
|
6864
|
+
activeLease = token;
|
|
6865
|
+
let released = false;
|
|
6866
|
+
return {
|
|
6867
|
+
bind(next) {
|
|
6868
|
+
if (released || activeLease !== token) throw new RuntimeRunStateError("RootHandle: live-run lease is no longer active");
|
|
6869
|
+
if (binding !== void 0) throw new RuntimeRunStateError("RootHandle: live-run lease is already bound");
|
|
6870
|
+
binding = next;
|
|
6871
|
+
},
|
|
6872
|
+
release() {
|
|
6873
|
+
if (released) return;
|
|
6874
|
+
released = true;
|
|
6875
|
+
if (activeLease !== token) return;
|
|
6876
|
+
binding = void 0;
|
|
6877
|
+
activeLease = void 0;
|
|
6878
|
+
}
|
|
6879
|
+
};
|
|
6880
|
+
} });
|
|
6881
|
+
return handle;
|
|
6882
|
+
}
|
|
5116
6883
|
/** A `RootSignal` sink: `cancel` cascades an abort; pause/resume/ask are observability
|
|
5117
6884
|
* signals the substrate accepts but does not act on here (the chat/pi-viz client owns
|
|
5118
6885
|
* pause semantics — building them now would be mechanism ahead of the gate). */
|
|
@@ -5172,27 +6939,82 @@ function wrapJournalForBreaker(journal, breaker) {
|
|
|
5172
6939
|
*/
|
|
5173
6940
|
async function drainLiveChildren(scope, controller) {
|
|
5174
6941
|
const view = scope.view;
|
|
5175
|
-
if (!(view.inFlight > 0 || view.waiting > 0))
|
|
6942
|
+
if (!(view.inFlight > 0 || view.waiting > 0)) {
|
|
6943
|
+
if (scope.workerCapacity.live > 0) throw new RuntimeRunStateError(`supervisor: cleanup ended with ${scope.workerCapacity.live} executor resource(s) not confirmed destroyed`);
|
|
6944
|
+
return;
|
|
6945
|
+
}
|
|
5176
6946
|
if (!controller.signal.aborted) controller.abort();
|
|
5177
|
-
await
|
|
6947
|
+
await drainCursor(scope);
|
|
6948
|
+
const after = scope.view;
|
|
6949
|
+
if (after.inFlight > 0 || after.waiting > 0 || scope.workerCapacity.live > 0) throw new RuntimeRunStateError(`supervisor: cleanup ended with ${after.inFlight} running, ${after.waiting} waiting, and ${scope.workerCapacity.live} executor resource(s) not confirmed destroyed`);
|
|
5178
6950
|
}
|
|
5179
6951
|
async function drainCursor(scope) {
|
|
5180
6952
|
for (;;) if (await scope.next() === null) return;
|
|
5181
6953
|
}
|
|
6954
|
+
/** Race root policy execution against the same signal that stops every child. The losing promise
|
|
6955
|
+
* remains observed, so a late rejection cannot surface as an unhandled process error. */
|
|
6956
|
+
async function runAbortable(act, signal) {
|
|
6957
|
+
if (signal.aborted) throw supervisorAbortError(signal);
|
|
6958
|
+
return await new Promise((resolve, reject) => {
|
|
6959
|
+
let settled = false;
|
|
6960
|
+
const cleanup = () => signal.removeEventListener("abort", onAbort);
|
|
6961
|
+
const onAbort = () => {
|
|
6962
|
+
queueMicrotask(() => {
|
|
6963
|
+
if (settled) return;
|
|
6964
|
+
settled = true;
|
|
6965
|
+
cleanup();
|
|
6966
|
+
reject(supervisorAbortError(signal));
|
|
6967
|
+
});
|
|
6968
|
+
};
|
|
6969
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
6970
|
+
let work;
|
|
6971
|
+
try {
|
|
6972
|
+
work = Promise.resolve(act());
|
|
6973
|
+
} catch (error) {
|
|
6974
|
+
cleanup();
|
|
6975
|
+
reject(error);
|
|
6976
|
+
return;
|
|
6977
|
+
}
|
|
6978
|
+
work.then((value) => {
|
|
6979
|
+
if (settled) return;
|
|
6980
|
+
settled = true;
|
|
6981
|
+
cleanup();
|
|
6982
|
+
resolve(value);
|
|
6983
|
+
}, (error) => {
|
|
6984
|
+
if (settled) return;
|
|
6985
|
+
settled = true;
|
|
6986
|
+
cleanup();
|
|
6987
|
+
reject(error);
|
|
6988
|
+
});
|
|
6989
|
+
});
|
|
6990
|
+
}
|
|
6991
|
+
function supervisorAbortError(signal) {
|
|
6992
|
+
const reason = signal.reason;
|
|
6993
|
+
const error = new Error(typeof reason === "string" && reason.length > 0 ? reason : "supervisor aborted");
|
|
6994
|
+
error.name = "AbortError";
|
|
6995
|
+
return error;
|
|
6996
|
+
}
|
|
5182
6997
|
/**
|
|
5183
6998
|
* The lifecycle cause of a no-winner, or `undefined` when the supervisor's own state proves
|
|
5184
6999
|
* nothing. Returning `undefined` rather than falling through to `all-children-down` is what lets
|
|
5185
7000
|
* the caller decide between the `driver-failed` arm (a rejection was captured) and the residual
|
|
5186
7001
|
* bucket — and it is why `driver-failed` cannot be produced without an `error` to attach.
|
|
5187
7002
|
*/
|
|
5188
|
-
function classifyNoWinner(controller, pool, opts, breaker) {
|
|
7003
|
+
function classifyNoWinner(controller, pool, opts, breaker, deadlineExceeded, tree) {
|
|
5189
7004
|
if (breaker.tripped()) return "all-children-down";
|
|
7005
|
+
if (deadlineExceeded) return "budget-exhausted";
|
|
5190
7006
|
if (controller.signal.aborted) return "aborted";
|
|
7007
|
+
if (allSpawnedChildrenDown(tree)) return "all-children-down";
|
|
5191
7008
|
if (poolExhausted(pool, opts)) return "budget-exhausted";
|
|
5192
7009
|
if (breaker.downCount() > 0) return "all-children-down";
|
|
5193
7010
|
}
|
|
7011
|
+
function allSpawnedChildrenDown(tree) {
|
|
7012
|
+
const children = tree.nodes.filter((node) => node.id !== tree.root);
|
|
7013
|
+
return children.length > 0 && children.every((node) => node.status === "failed");
|
|
7014
|
+
}
|
|
5194
7015
|
function poolExhausted(pool, opts) {
|
|
5195
7016
|
const r = pool.readout();
|
|
7017
|
+
if (r.iterationsLeft <= 0) return true;
|
|
5196
7018
|
if (r.tokensLeft <= 0) return true;
|
|
5197
7019
|
if (opts.budget.maxUsd !== void 0 && r.usdLeft <= 0) return true;
|
|
5198
7020
|
if (opts.budget.deadlineMs !== void 0 && r.deadlineMs > 0 && (opts.now ?? Date.now)() >= r.deadlineMs) return true;
|
|
@@ -5214,6 +7036,21 @@ async function spentFromJournal(journal, root) {
|
|
|
5214
7036
|
* `metered` = driver inference (re-homed up the tree, so a single root-tree pass already
|
|
5215
7037
|
* includes every nested driver's inference). */
|
|
5216
7038
|
function sumSpendFromEvents(events) {
|
|
7039
|
+
const totals = sumMeasuredSpendFromEvents(events);
|
|
7040
|
+
const rootBudget = events.find((event) => event.kind === "spawned" && event.parent === void 0)?.budget;
|
|
7041
|
+
let remainingRootUsd = Math.max(0, (rootBudget?.maxUsd ?? 0) - totals.childWork.usd - totals.driverInference.usd);
|
|
7042
|
+
for (const budget of uncertainSpawnBudgets(events)) {
|
|
7043
|
+
totals.childWork.iterations += budget.maxIterations;
|
|
7044
|
+
totals.childWork.tokens.input += budget.maxTokens;
|
|
7045
|
+
totals.childWork.tokensKnown = false;
|
|
7046
|
+
const usdCharge = budget.maxUsd ?? (rootBudget?.maxUsd !== void 0 ? remainingRootUsd : 0);
|
|
7047
|
+
totals.childWork.usd += usdCharge;
|
|
7048
|
+
totals.childWork.usdKnown = false;
|
|
7049
|
+
remainingRootUsd = Math.max(0, remainingRootUsd - usdCharge);
|
|
7050
|
+
}
|
|
7051
|
+
return totals;
|
|
7052
|
+
}
|
|
7053
|
+
function sumMeasuredSpendFromEvents(events) {
|
|
5217
7054
|
const childWork = {
|
|
5218
7055
|
iterations: 0,
|
|
5219
7056
|
tokens: {
|
|
@@ -5244,8 +7081,8 @@ function accumulate(a, b) {
|
|
|
5244
7081
|
a.iterations += b.iterations;
|
|
5245
7082
|
a.tokens.input += b.tokens.input;
|
|
5246
7083
|
a.tokens.output += b.tokens.output;
|
|
5247
|
-
a.usd += b.usd;
|
|
5248
7084
|
if (b.tokensKnown === false) a.tokensKnown = false;
|
|
7085
|
+
a.usd += b.usd;
|
|
5249
7086
|
if (b.usdKnown === false) a.usdKnown = false;
|
|
5250
7087
|
a.ms += b.ms;
|
|
5251
7088
|
}
|
|
@@ -5258,21 +7095,19 @@ function addSpend(a, b) {
|
|
|
5258
7095
|
input: a.tokens.input + b.tokens.input,
|
|
5259
7096
|
output: a.tokens.output + b.tokens.output
|
|
5260
7097
|
},
|
|
5261
|
-
usd: a.usd + b.usd,
|
|
5262
7098
|
...a.tokensKnown === false || b.tokensKnown === false ? { tokensKnown: false } : {},
|
|
7099
|
+
usd: a.usd + b.usd,
|
|
5263
7100
|
...a.usdKnown === false || b.usdKnown === false ? { usdKnown: false } : {},
|
|
5264
7101
|
ms: a.ms + b.ms
|
|
5265
7102
|
};
|
|
5266
7103
|
}
|
|
5267
7104
|
/** True when any driver metered inference this run (so the winner carries a `spentBreakdown`).
|
|
5268
7105
|
* Checks every channel `addSpend` sums — including `ms` — so the gate stays consistent with the
|
|
5269
|
-
* total even though the coordination driver currently stamps `ms: 0`.
|
|
5270
|
-
* carries an UNKNOWN marker is non-empty: work happened and went unmeasured, which is exactly the
|
|
5271
|
-
* fact a breakdown must not hide by looking like no work at all. */
|
|
7106
|
+
* total even though the coordination driver currently stamps `ms: 0`. */
|
|
5272
7107
|
function isNonEmptySpend(s) {
|
|
5273
7108
|
return s.iterations > 0 || s.tokens.input > 0 || s.tokens.output > 0 || s.usd > 0 || s.ms > 0 || s.tokensKnown === false || s.usdKnown === false;
|
|
5274
7109
|
}
|
|
5275
7110
|
//#endregion
|
|
5276
|
-
export {
|
|
7111
|
+
export { createBudgetPool as $, loopEventToOtelSpan as A, promptResourceProfileMaterialization as At, timerAt as B, buildLoopSpanNodes as C, controlProfileMaterialization as Ct, exportEvalRuns as D, promptControlProfileMaterialization as Dt, createOtelExporter as E, profileMaterializationAxes as Et, sanitizeKnowledgeReadinessReport as F, harnessInvocation as Ft, readWorkerProgress as G, waitUntil as H, sanitizeRuntimeStreamEvent as I, parseCodexTokenUsage as It, queueOf as J, effectiveConcurrency as K, createWaitProbes as L, runLocalHarness as Lt, createRuntimeEventCollector as M, sandboxActProfileMaterialization as Mt, createRuntimeStreamEventCollector as N, validateProfileMaterialization as Nt, flatOtelSpan as O, promptModelProfileMaterialization as Ot, sanitizeAgentRuntimeEvent as P, worktreeCliProfileMaterialization as Pt, assertValidBudget as Q, isWaitOutcome as R, CodexExecutionDiagnosticError as Rt, buildLoopOtelSpans as S, assertProfileMaterialization as St, createOpenInferenceFileExporter as T, fullProfileMaterialization as Tt, DEFAULT_STALL_AFTER_MS as U, validateWaitSpec as V, createActivityLog as W, DEFAULT_SUCCESSFUL_SHUTDOWN_MS as X, rollingDispatch as Y, teardownExecutor as Z, workerTraceSeamKey as _, worktreeProfileExecutionPlan as _t, pickBestDelivered as a, runtimeOwnedExecutorExecutionBinding as at, traceContextToEnv as b, removeWorktree as bt, driverChild as c, routerBrain as ct, deriveNodeExecutionIdentity as d, routerToolLoop as dt, spendFromUsageEvents as et, recordScopeOwnerMaterialization as f, streamRouterChatWithTools as ft, workerTraceEnv as g, runWorktreeHarness as gt, readWorkerTraceContext as h, runWorktreeChecks as ht, collectDelivered as i, newExecutionAttemptId as it, toOtelAttributes as j, renderProfileMaterializationIssues as jt, generateSpanId as k, promptOnlyProfileMaterialization as kt, withDriverExecutor as l, routerChatWithTools as lt, settledToIteration as m, runSettledCommand as mt, createSupervisor as n, attestRuntimeOwnedScopeOwner as nt, runFinalizer as o, runtimeOwnedExecutorMaterialization as ot, scopeOwnerExecutorNodeContext as p, runBrainLoop as pt, freeSlots as q, bestDelivered as r, inheritRuntimeOwnedExecutorAttestation as rt, runTree as s, runtimeOwnedScopeOwnerRuntime as st, createRootHandle as t, attestRuntimeOwnedExecutor as tt, createScope as u, routerChatWithUsage as ut, createPropagatingTraceEmitter as v, captureWorktreeDiff as vt, buildRuntimeEventOtelSpans as w, defineProfileMaterializationContract as wt, INTELLIGENCE_WIRE_VERSION as x, AGENT_PROFILE_MATERIALIZATION_AXES as xt, readTraceContextFromEnv as y, createWorktree as yt, pollFor as z };
|
|
5277
7112
|
|
|
5278
|
-
//# sourceMappingURL=supervisor-
|
|
7113
|
+
//# sourceMappingURL=supervisor-CAXiCKnX.js.map
|