@tangle-network/agent-runtime 0.271.0 → 0.273.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/dist/{activation-DGbFPGR3.js → activation-Cx2dKA_l.js} +2 -2
  2. package/dist/{activation-DGbFPGR3.js.map → activation-Cx2dKA_l.js.map} +1 -1
  3. package/dist/{activation-BrJmwTLB.d.ts → activation-IfDlhR3J.d.ts} +2 -2
  4. package/dist/agent.d.ts +2 -2
  5. package/dist/agent.js +2 -2
  6. package/dist/candidate-execution/index.js +2 -2
  7. package/dist/{coordination-driver-iBGMKDvB.js → coordination-driver-CkcW4dxU.js} +155 -80
  8. package/dist/coordination-driver-CkcW4dxU.js.map +1 -0
  9. package/dist/{delegate-DOqUvaIz.js → delegate-Dvb1-mE-.js} +2 -2
  10. package/dist/{delegate-DOqUvaIz.js.map → delegate-Dvb1-mE-.js.map} +1 -1
  11. package/dist/durable.d.ts +3 -572
  12. package/dist/durable.js +92 -18
  13. package/dist/durable.js.map +1 -1
  14. package/dist/{graph-DnMcepkN.js → graph-C69Uxtzt.js} +14 -8
  15. package/dist/graph-C69Uxtzt.js.map +1 -0
  16. package/dist/{improvement-cycle-1IyRJo5h.js → improvement-cycle-CO2HRCQq.js} +4 -4
  17. package/dist/{improvement-cycle-1IyRJo5h.js.map → improvement-cycle-CO2HRCQq.js.map} +1 -1
  18. package/dist/{index-CL1dduJd.d.ts → index-D2ipSqkP.d.ts} +4318 -3102
  19. package/dist/index.d.ts +6 -450
  20. package/dist/index.js +14 -11
  21. package/dist/index.js.map +1 -1
  22. package/dist/intelligence.d.ts +3 -3
  23. package/dist/intelligence.js +5 -5
  24. package/dist/kernel.d.ts +4 -4
  25. package/dist/kernel.js +9 -9
  26. package/dist/{loop-runner-bin-BPZ9tt1W.js → loop-runner-bin-B9MIq9O7.js} +3 -3
  27. package/dist/{loop-runner-bin-BPZ9tt1W.js.map → loop-runner-bin-B9MIq9O7.js.map} +1 -1
  28. package/dist/{loop-runner-bin-O_vM1w8Y.d.ts → loop-runner-bin-ThlwVkvo.d.ts} +3 -3
  29. package/dist/loop-runner-bin.d.ts +1 -1
  30. package/dist/loop-runner-bin.js +1 -1
  31. package/dist/mcp/bin.js +3 -3
  32. package/dist/mcp/index.d.ts +5 -5
  33. package/dist/mcp/index.js +4 -4
  34. package/dist/{openai-tools-DWHxpwx2.d.ts → openai-tools-Bb0mAr5z.d.ts} +2 -2
  35. package/dist/{prepare-oHUQdC5-.js → prepare-DVlvGP7V.js} +2 -2
  36. package/dist/{prepare-oHUQdC5-.js.map → prepare-DVlvGP7V.js.map} +1 -1
  37. package/dist/profiles.d.ts +2 -2
  38. package/dist/{protected-model-port-DqF0qDU6.js → protected-model-port-BrWgsCnl.js} +2 -2
  39. package/dist/{protected-model-port-DqF0qDU6.js.map → protected-model-port-BrWgsCnl.js.map} +1 -1
  40. package/dist/{protected-redaction--F3v1oo8.js → protected-redaction-DylCC_ot.js} +14 -20
  41. package/dist/protected-redaction-DylCC_ot.js.map +1 -0
  42. package/dist/{provision-supervisor-Cy8pqE95.js → provision-supervisor-BKAvsDtP.js} +3 -3
  43. package/dist/{provision-supervisor-Cy8pqE95.js.map → provision-supervisor-BKAvsDtP.js.map} +1 -1
  44. package/dist/{runtime-DVjIH1eo.js → runtime-CHxXrUeZ.js} +2079 -1441
  45. package/dist/runtime-CHxXrUeZ.js.map +1 -0
  46. package/dist/{server-CN9vzbSe.js → server-2OeFrEVG.js} +3 -3
  47. package/dist/{server-CN9vzbSe.js.map → server-2OeFrEVG.js.map} +1 -1
  48. package/dist/{stream-agent-turn-DbEsKgb_.d.ts → stream-agent-turn-ewm3BZAp.d.ts} +15 -16
  49. package/dist/{structural-rollout-I92sIfNl.js → structural-rollout-DCUprNpH.js} +2 -2
  50. package/dist/{structural-rollout-I92sIfNl.js.map → structural-rollout-DCUprNpH.js.map} +1 -1
  51. package/dist/{substrate-CbC03Vzo.d.ts → substrate-VKjmrksz.d.ts} +2 -2
  52. package/dist/{supervise-DJ_1i9vj.js → supervise-Dej5aSDa.js} +211 -48
  53. package/dist/supervise-Dej5aSDa.js.map +1 -0
  54. package/dist/{supervisor-Cvd5mf7x.js → supervisor-DdKPGluv.js} +952 -221
  55. package/dist/supervisor-DdKPGluv.js.map +1 -0
  56. package/dist/testing.d.ts +2 -2
  57. package/dist/testing.js +60 -60
  58. package/dist/tui/index.d.ts +1 -1
  59. package/dist/tui/index.js +1 -1
  60. package/dist/{types-Bo7z5aG5.d.ts → types-DMRc3jru.d.ts} +356 -64
  61. package/package.json +5 -5
  62. package/dist/coordination-driver-iBGMKDvB.js.map +0 -1
  63. package/dist/graph-DnMcepkN.js.map +0 -1
  64. package/dist/protected-redaction--F3v1oo8.js.map +0 -1
  65. package/dist/runtime-DVjIH1eo.js.map +0 -1
  66. package/dist/supervise-DJ_1i9vj.js.map +0 -1
  67. package/dist/supervisor-Cvd5mf7x.js.map +0 -1
@@ -1,24 +1,24 @@
1
- import { At as insideCursorNamespace, Dn as runAgentRounds, Dt as SpawnEventIndex, Et as InMemorySpawnJournal, Fn as canonicalObservedModelParts, Gn as unavailableSignalOfFailure, Gr as harnessInvocation, Ii as executableAgentProfileSnapshot, Kn as createPushTraceSource, Li as executableAgentSpecSnapshot, Lt as isTraceAnalysisStore, M as settledToIteration, Mn as readPromptOptions, Nn as routerBrain, On as createSandboxLineage, Ot as assertContentAddress, Pn as runBrainLoop, Ri as detachedFrozen, Rn as LEAF_CONTINUATION_TASK, Sa as armDeadlineTimer, Tn as defaultSelectWinner, Tt as InMemoryResultBlobStore, Un as sandboxClientAsProvider, Wn as unavailablePauseMs, Xi as notifySandboxEventObserver, _i as writeAllBytes, _t as createExecutorRegistry, aa as isAbortError, b as withDriverExecutor, ca as sleep, dt as rollingDispatch, ea as addSpend, gi as readCommittedJsonLines, gt as createExecutor, hi as prepareJsonlAppend, hr as contentAddress, ia as hasCompleteCacheBreakdown, jn as assertBoxlessPromptOptions, kn as probeSandboxCapabilities, kt as encodeResultBlob, la as stringifySafe, ma as zeroSpend, na as cloneSpend, r as createSupervisor, sa as randomSuffix, ta as chargedTokens, yt as createWorktreeCliExecutor, zi as detachedSnapshot } from "./supervisor-Cvd5mf7x.js";
1
+ import { $i as detachedSnapshot, Ai as readCommittedJsonLines, Bn as runAgentRounds, Bt as SpawnEventIndex, Ca as stringifySafe, Ct as rollingDispatch, Gn as readPromptOptions, Hn as probeSandboxCapabilities, Ht as encodeResultBlob, Ia as armDeadlineTimer, Jn as canonicalObservedModelParts, Kn as routerBrain, M as withDriverExecutor, O as verdictFromJudgeScore, Oa as zeroSpend, Ot as createExecutor, Qi as detachedFrozen, Rn as defaultSelectWinner, Rt as InMemoryResultBlobStore, Sa as sleep, Ut as insideCursorNamespace, Vn as createSandboxLineage, Vt as assertContentAddress, W as settledToIteration, Wn as assertBoxlessPromptOptions, Xi as executableAgentProfileSnapshot, Xt as isTraceAnalysisStore, Zi as executableAgentSpecSnapshot, Zn as LEAF_CONTINUATION_TASK, _ as CheckUnavailableError, ai as harnessInvocation, ar as createPushTraceSource, ga as cloneSpend, ha as chargedTokens, ir as unavailableSignalOfFailure, ji as writeAllBytes, jt as createWorktreeCliExecutor, ki as prepareJsonlAppend, kr as contentAddress, kt as createExecutorRegistry, ma as addSpend, nr as sandboxClientAsProvider, qn as runBrainLoop, r as createSupervisor, rr as unavailablePauseMs, ua as notifySandboxEventObserver, va as hasCompleteCacheBreakdown, xa as randomSuffix, ya as isAbortError, zt as InMemorySpawnJournal } from "./supervisor-DdKPGluv.js";
2
2
  import { m as ValidationError, n as AnalystError, s as PlannerError } from "./errors-DodWX-cb.js";
3
- import { E as profileChatClient, S as ObservationError, T as renderReport, b as sample, k as strategyAuthorMethod, v as refine, w as observe, x as sampleThenRefine, y as runAgentic } from "./structural-rollout-I92sIfNl.js";
3
+ import { E as profileChatClient, S as ObservationError, T as renderReport, b as sample, k as strategyAuthorMethod, v as refine, w as observe, x as sampleThenRefine, y as runAgentic } from "./structural-rollout-DCUprNpH.js";
4
4
  import { a as concreteProfileModel, c as profileModelExecutionSettings, i as concreteModelId, l as profileProviderModel, o as enforceTokenLimits, t as assertExecutableAgentProfile } from "./model-policy-BbSCSak0.js";
5
5
  import { C as canonicalCandidateDigest$1, D as immutableCandidateValue, k as sha256Bytes$1, m as captureMaterializedWorkspace } from "./workspace-archive-C9lgf77y.js";
6
6
  import { i as notifyRuntimeHookEvent } from "./runtime-hooks-tXpAarhW.js";
7
- import { i as redactProtectedValue, r as redactProtectedReason } from "./protected-redaction--F3v1oo8.js";
8
- import "./coordination-driver-iBGMKDvB.js";
9
- import { C as gateOnDeliverable, r as supervise, w as mapExecutorResult } from "./supervise-DJ_1i9vj.js";
7
+ import { i as redactProtectedValue, r as redactProtectedReason } from "./protected-redaction-DylCC_ot.js";
8
+ import { S as foldCoordinationRecords, y as createInMemoryRunContext } from "./coordination-driver-CkcW4dxU.js";
9
+ import { C as gateOnDeliverable, r as supervise, w as mapExecutorResult } from "./supervise-Dej5aSDa.js";
10
10
  import "./run-layout-vwd4SwX_.js";
11
- import "./provision-supervisor-Cy8pqE95.js";
12
- import "./delegate-DOqUvaIz.js";
13
- import "./graph-DnMcepkN.js";
14
- import { agentProfileSchema, canonicalAgentProfileDigest, canonicalCandidateDigest, validateAgentProfileSecurity } from "@tangle-network/agent-interface";
11
+ import "./provision-supervisor-BKAvsDtP.js";
12
+ import "./delegate-Dvb1-mE-.js";
13
+ import "./graph-C69Uxtzt.js";
14
+ import { agentProfileSchema, canonicalAgentProfileDigest, canonicalCandidateDigest, sha256Bytes, validateAgentProfileSecurity } from "@tangle-network/agent-interface";
15
15
  import { CODING_HARNESSES, DEFAULT_TRACE_ANALYST_KINDS, InMemoryTraceStore, OUTPUT_VALUE, benjaminiHochberg, buildTrajectory, computeFindingId as computeFindingId$1, confidenceInterval, createTraceAnalyst, expandProfileAxes, makeFinding as makeFinding$1, pairedBootstrap, paretoFrontier, wilcoxonSignedRank, wilson } from "@tangle-network/agent-eval";
16
16
  import { createAgentRunOutcomeTracker } from "@tangle-network/sandbox/runtime";
17
17
  import { createHash, randomUUID } from "node:crypto";
18
18
  import { execFile, execFileSync, spawn, spawnSync } from "node:child_process";
19
19
  import { appendFileSync, chmodSync, constants, copyFileSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readlinkSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
20
20
  import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
21
- import { cp, mkdir, mkdtemp, readFile, realpath, stat, writeFile } from "node:fs/promises";
21
+ import { cp, mkdir, mkdtemp, readFile, readdir, realpath, rm, stat, writeFile } from "node:fs/promises";
22
22
  import { materializeProfile } from "@tangle-network/agent-profile-materialize";
23
23
  import { tmpdir } from "node:os";
24
24
  import { isDeepStrictEqual, promisify } from "node:util";
@@ -1015,6 +1015,266 @@ var SqlResultBlobStore = class {
1015
1015
  }
1016
1016
  };
1017
1017
  //#endregion
1018
+ //#region src/durable/sql-run-store.ts
1019
+ /** Refuses a competing or stale SQL run owner before it can publish new work. */
1020
+ var SqlRunOwnershipError = class extends Error {
1021
+ constructor(message) {
1022
+ super(message);
1023
+ this.name = "SqlRunOwnershipError";
1024
+ }
1025
+ };
1026
+ /**
1027
+ * Append-only immutable records, published by ONE compare-and-set of the run's head.
1028
+ *
1029
+ * SqlAdapter has no pinned-connection transaction API. INSERT ... SELECT owner is NOT a
1030
+ * fence on MVCC databases: its snapshot can outlive a takeover. Here both publication and
1031
+ * takeover UPDATE the SAME row. Unpublished records are harmless, unreachable staging.
1032
+ * Healthy appends write only their new payload and one fixed-size head, never the history.
1033
+ *
1034
+ * Liveness uses a persisted progress counter, not incomparable host clocks. A contender must
1035
+ * observe the SAME generation/counter for leaseMs before replacing it with a CAS. Every
1036
+ * heartbeat and publication advances that counter. The interval is stored per run so a
1037
+ * differently configured contender cannot shorten an existing owner's lease.
1038
+ * @internal
1039
+ */
1040
+ async function openSqlRunStore(db, runId, options = {}) {
1041
+ const prefix = options.tablePrefix ?? "agent_run";
1042
+ const leaseMs = options.leaseMs ?? 3e4;
1043
+ const heartbeatMs = options.heartbeatMs ?? Math.floor(leaseMs / 3);
1044
+ if (!/^[A-Za-z_][A-Za-z0-9_]{0,39}$/.test(prefix)) throw new Error("SQL run context tablePrefix must be a SQL identifier of at most 40 characters");
1045
+ if (typeof runId !== "string" || runId.length === 0) throw new Error("SQL runId must be nonempty");
1046
+ if (!Number.isSafeInteger(leaseMs) || leaseMs < 100 || leaseMs > 2147483647) throw new Error("SQL leaseMs must be an integer between 100 and 2147483647");
1047
+ if (!Number.isSafeInteger(heartbeatMs) || heartbeatMs < 1 || heartbeatMs > leaseMs / 3) throw new Error("SQL heartbeatMs must be a positive integer no greater than leaseMs / 3");
1048
+ const heads = `${prefix}_heads`;
1049
+ const records = `${prefix}_records`;
1050
+ await db.exec(`CREATE TABLE IF NOT EXISTS ${heads} (
1051
+ run_id TEXT PRIMARY KEY,
1052
+ owner TEXT NOT NULL,
1053
+ generation INTEGER NOT NULL,
1054
+ pulse INTEGER NOT NULL,
1055
+ head TEXT NOT NULL,
1056
+ revision INTEGER NOT NULL,
1057
+ lease_ms INTEGER NOT NULL,
1058
+ namespace TEXT NOT NULL
1059
+ )`);
1060
+ await db.exec(`CREATE TABLE IF NOT EXISTS ${records} (
1061
+ run_id TEXT NOT NULL,
1062
+ id TEXT NOT NULL,
1063
+ parent TEXT NOT NULL,
1064
+ revision INTEGER NOT NULL,
1065
+ payload TEXT NOT NULL,
1066
+ PRIMARY KEY (run_id, id)
1067
+ )`);
1068
+ await db.exec(`INSERT INTO ${heads} (run_id, owner, generation, pulse, head, revision, lease_ms, namespace)
1069
+ VALUES (?, '', 0, 0, '', 0, ?, ?) ON CONFLICT (run_id) DO NOTHING`, [
1070
+ runId,
1071
+ leaseMs,
1072
+ randomUUID()
1073
+ ]);
1074
+ async function readHead() {
1075
+ const [row] = await db.query(`SELECT * FROM ${heads} WHERE run_id = ?`, [runId]);
1076
+ if (!row) throw new Error(`SQL run '${runId}' has no head`);
1077
+ for (const key of [
1078
+ "generation",
1079
+ "pulse",
1080
+ "revision",
1081
+ "lease_ms"
1082
+ ]) {
1083
+ row[key] = Number(row[key]);
1084
+ if (!Number.isSafeInteger(row[key]) || row[key] < 0) throw new Error(`SQL run '${runId}' has an invalid ${key}`);
1085
+ }
1086
+ if (typeof row.namespace !== "string" || row.namespace.length === 0 || typeof row.head !== "string" || typeof row.owner !== "string" || row.head === "" !== (row.revision === 0)) throw new Error(`SQL run '${runId}' has an invalid head`);
1087
+ if (row.lease_ms !== leaseMs) throw new Error(`SQL run '${runId}' lease configuration differs: recorded ${row.lease_ms}ms, requested ${leaseMs}ms`);
1088
+ return row;
1089
+ }
1090
+ async function load(head) {
1091
+ if (head.revision === 0) return [];
1092
+ const rows = await db.query(`SELECT id, parent, revision, payload FROM ${records} WHERE run_id = ? AND revision <= ?`, [runId, head.revision]);
1093
+ const byId = new Map(rows.map((row) => [row.id, row]));
1094
+ const history = [];
1095
+ let id = head.head;
1096
+ for (let revision = head.revision; revision > 0; revision -= 1) {
1097
+ const row = byId.get(id);
1098
+ if (!row || Number(row.revision) !== revision || recordId(row.parent, revision, row.payload) !== id) throw new Error(`SQL run '${runId}' has a missing or corrupt committed record at revision ${revision}`);
1099
+ history.push(JSON.parse(row.payload));
1100
+ id = row.parent;
1101
+ }
1102
+ if (id !== "") throw new Error(`SQL run '${runId}' has an invalid history origin`);
1103
+ return history.reverse();
1104
+ }
1105
+ return {
1106
+ namespace: (await readHead()).namespace,
1107
+ async read() {
1108
+ return load(await readHead());
1109
+ },
1110
+ async acquire(signal) {
1111
+ signal?.throwIfAborted();
1112
+ const observed = await readHead();
1113
+ if (observed.owner !== "") await wait(leaseMs, signal);
1114
+ signal?.throwIfAborted();
1115
+ const owner = randomUUID();
1116
+ let claimed = false;
1117
+ try {
1118
+ claimed = (await db.exec(`UPDATE ${heads} SET owner = ?, generation = generation + 1, pulse = pulse + 1
1119
+ WHERE run_id = ? AND owner = ? AND generation = ? AND pulse = ?`, [
1120
+ owner,
1121
+ runId,
1122
+ observed.owner,
1123
+ observed.generation,
1124
+ observed.pulse
1125
+ ])).rowsAffected === 1;
1126
+ } catch (error) {
1127
+ if ((await readHead().catch(() => void 0))?.owner !== owner) throw error;
1128
+ claimed = true;
1129
+ }
1130
+ if (!claimed) throw new SqlRunOwnershipError(`SQL run '${runId}' is owned by another live process`);
1131
+ const generation = observed.generation + 1;
1132
+ const controller = new AbortController();
1133
+ let accepting = true;
1134
+ let released;
1135
+ let tail = Promise.resolve();
1136
+ let heartbeat;
1137
+ let progressAt = performance.now();
1138
+ let head = observed.head;
1139
+ let revision = observed.revision;
1140
+ let history = [];
1141
+ const lose = (error) => {
1142
+ accepting = false;
1143
+ clearInterval(timer);
1144
+ controller.abort(error);
1145
+ };
1146
+ const timer = setInterval(() => {
1147
+ if (controller.signal.aborted) return;
1148
+ if (performance.now() - progressAt >= leaseMs) {
1149
+ lose(new SqlRunOwnershipError(`SQL run '${runId}' lease heartbeat expired`));
1150
+ return;
1151
+ }
1152
+ if (heartbeat) return;
1153
+ heartbeat = db.exec(`UPDATE ${heads} SET pulse = pulse + 1 WHERE run_id = ? AND owner = ? AND generation = ?`, [
1154
+ runId,
1155
+ owner,
1156
+ generation
1157
+ ]).then((result) => {
1158
+ if (result.rowsAffected !== 1) throw new SqlRunOwnershipError(`SQL run '${runId}' lease was replaced`);
1159
+ progressAt = performance.now();
1160
+ }).catch(lose).finally(() => {
1161
+ heartbeat = void 0;
1162
+ });
1163
+ }, heartbeatMs);
1164
+ timer.unref();
1165
+ const lease = {
1166
+ signal: controller.signal,
1167
+ get records() {
1168
+ return JSON.parse(JSON.stringify(history));
1169
+ },
1170
+ append(payload) {
1171
+ if (!accepting) return Promise.reject(controller.signal.reason ?? new SqlRunOwnershipError(`SQL run '${runId}' lease is released`));
1172
+ let encoded;
1173
+ try {
1174
+ encoded = JSON.stringify(payload);
1175
+ if (encoded === void 0) throw new Error("SQL run records must be JSON serializable");
1176
+ } catch (error) {
1177
+ return Promise.reject(error);
1178
+ }
1179
+ const write = tail.then(async () => {
1180
+ controller.signal.throwIfAborted();
1181
+ const nextRevision = revision + 1;
1182
+ const id = recordId(head, nextRevision, encoded);
1183
+ try {
1184
+ await db.exec(`INSERT INTO ${records} (run_id, id, parent, revision, payload) VALUES (?, ?, ?, ?, ?)
1185
+ ON CONFLICT (run_id, id) DO NOTHING`, [
1186
+ runId,
1187
+ id,
1188
+ head,
1189
+ nextRevision,
1190
+ encoded
1191
+ ]);
1192
+ } catch (error) {
1193
+ const [staged] = await db.query(`SELECT id, parent, revision, payload FROM ${records} WHERE run_id = ? AND id = ?`, [runId, id]).catch(() => []);
1194
+ if (!staged || staged.payload !== encoded || staged.parent !== head || Number(staged.revision) !== nextRevision) throw error;
1195
+ }
1196
+ controller.signal.throwIfAborted();
1197
+ let published = false;
1198
+ let failure;
1199
+ try {
1200
+ published = (await db.exec(`UPDATE ${heads} SET head = ?, revision = ?, pulse = pulse + 1
1201
+ WHERE run_id = ? AND owner = ? AND generation = ? AND head = ? AND revision = ?`, [
1202
+ id,
1203
+ nextRevision,
1204
+ runId,
1205
+ owner,
1206
+ generation,
1207
+ head,
1208
+ revision
1209
+ ])).rowsAffected === 1;
1210
+ } catch (error) {
1211
+ failure = error;
1212
+ }
1213
+ if (!published) {
1214
+ const current = await readHead();
1215
+ if (current.owner !== owner || current.generation !== generation) throw new SqlRunOwnershipError(`SQL run '${runId}' lease was replaced before publication`);
1216
+ if (current.head !== id || current.revision !== nextRevision) throw failure ?? new SqlRunOwnershipError(`SQL run '${runId}' publication conflicted`);
1217
+ }
1218
+ controller.signal.throwIfAborted();
1219
+ head = id;
1220
+ revision = nextRevision;
1221
+ history.push(JSON.parse(encoded));
1222
+ progressAt = performance.now();
1223
+ });
1224
+ tail = write.catch(lose);
1225
+ return write;
1226
+ },
1227
+ release() {
1228
+ if (released) return released;
1229
+ accepting = false;
1230
+ released = (async () => {
1231
+ await tail;
1232
+ clearInterval(timer);
1233
+ try {
1234
+ await db.exec(`UPDATE ${heads} SET owner = '', pulse = pulse + 1 WHERE run_id = ? AND owner = ? AND generation = ?`, [
1235
+ runId,
1236
+ owner,
1237
+ generation
1238
+ ]);
1239
+ } finally {
1240
+ controller.abort(new SqlRunOwnershipError(`SQL run '${runId}' lease is released`));
1241
+ }
1242
+ })();
1243
+ return released;
1244
+ }
1245
+ };
1246
+ try {
1247
+ history = await load(observed);
1248
+ signal?.throwIfAborted();
1249
+ controller.signal.throwIfAborted();
1250
+ return lease;
1251
+ } catch (error) {
1252
+ await lease.release().catch(() => void 0);
1253
+ throw error;
1254
+ }
1255
+ }
1256
+ };
1257
+ }
1258
+ function recordId(parent, revision, payload) {
1259
+ return createHash("sha256").update(`${revision}\0${parent}\0${payload}`).digest("hex");
1260
+ }
1261
+ function wait(ms, signal) {
1262
+ return new Promise((resolve, reject) => {
1263
+ const cleanup = () => signal?.removeEventListener("abort", abort);
1264
+ const timer = setTimeout(() => {
1265
+ cleanup();
1266
+ resolve();
1267
+ }, ms);
1268
+ const abort = () => {
1269
+ clearTimeout(timer);
1270
+ cleanup();
1271
+ reject(signal?.reason);
1272
+ };
1273
+ if (signal?.aborted) abort();
1274
+ else signal?.addEventListener("abort", abort, { once: true });
1275
+ });
1276
+ }
1277
+ //#endregion
1018
1278
  //#region src/runtime/audit-intent.ts
1019
1279
  /** Default system instruction for intent-auditor agents: diagnose diverged/drifting trajectories. */
1020
1280
  const defaultAuditorInstruction = "You audit whether an AI agent is on the RIGHT ROUTE — not whether it works hard, but whether its actions serve the stated intents. Infer the REVEALED intent from the action pattern (what the trajectory is actually optimizing). Compare against the declared task intent, the user intent when given, and the meta-intent when given. Flawless execution down the wrong route is DIVERGED. Busy-work that neither advances nor harms is DRIFTING. Judge only from the trajectory — be specific about which actions ground your verdict. Recommend abort only when continuing cannot serve the intent.";
@@ -1528,1529 +1788,1733 @@ function deterministicCompletion(check) {
1528
1788
  } };
1529
1789
  }
1530
1790
  //#endregion
1531
- //#region src/runtime/loop-dispatch.ts
1532
- /** Bridge a campaign `DispatchContext.trace` to a `LoopTraceEmitter` so every
1533
- * `loop.*` event lands as a span under the cell's scoped trace. */
1534
- function campaignTraceToLoopEmitter(trace) {
1535
- return { emit(event) {
1536
- trace.span(event.kind, {
1537
- runId: event.runId,
1538
- timestamp: event.timestamp,
1539
- ...event.payload
1540
- }).end();
1541
- } };
1542
- }
1543
- async function runLoopForCell(opts, scenario, profile, ctx) {
1544
- const loopOptions = opts.toLoopOptions(scenario, profile);
1545
- return runLoopWithCampaignContext(opts, loopOptions, ctx, {
1546
- model: profile.model?.default ?? modelFromLoopOptions(loopOptions),
1547
- maximumCharge: typeof opts.maximumCharge === "function" ? opts.maximumCharge(scenario, profile) : opts.maximumCharge,
1548
- resolveModel: opts.resolveCostModel ? (result) => opts.resolveCostModel?.(result, scenario, profile) : void 0
1549
- });
1550
- }
1551
- async function runLoopWithCampaignContext(opts, loopOptions, ctx, cost) {
1552
- const paid = await ctx.cost.runPaidCall({
1553
- channel: "agent",
1554
- actor: opts.costSource ?? "loop",
1555
- model: cost.model,
1556
- signal: ctx.signal,
1557
- ...cost.maximumCharge ? { maximumCharge: cost.maximumCharge } : {},
1558
- execute: (executionSignal) => runAgentRounds({
1559
- ...loopOptions,
1560
- ctx: {
1561
- sandboxClient: opts.sandboxClient,
1562
- ...opts.promptOptions === void 0 ? {} : { promptOptions: opts.promptOptions },
1563
- signal: executionSignal,
1564
- traceEmitter: opts.forwardTrace === false ? void 0 : campaignTraceToLoopEmitter(ctx.trace)
1791
+ //#region src/runtime/isolated-checker.ts
1792
+ const toolchains = [
1793
+ "/usr",
1794
+ "/bin",
1795
+ "/lib",
1796
+ "/lib64"
1797
+ ];
1798
+ const contains = (parent, child) => {
1799
+ const path = relative(parent, child);
1800
+ return path === "" || path !== ".." && !path.startsWith(`..${sep}`) && !path.startsWith(sep);
1801
+ };
1802
+ /**
1803
+ * Run an untrusted check in Linux Bubblewrap, or in its own Sandbox box when `box` is set.
1804
+ * Never falls back to host execution.
1805
+ *
1806
+ * Bubblewrap requires /usr/bin/bwrap and permission to create Linux namespaces.
1807
+ * Only trusted system toolchains, private proc/dev/tmp, and the writable copy are mounted.
1808
+ * The canonical input path remains the working directory; copy writes are discarded.
1809
+ * Limits bound command time and captured output, not copy size or memory consumption.
1810
+ * Callers must keep the input and trusted toolchains stable while preparing the check.
1811
+ *
1812
+ * A box check creates one fresh box with no owner secrets and blocked egress (or strict egress to
1813
+ * the placement's named domains), delivers the tree's regular files verified by sha256, runs the
1814
+ * command there with only the environment `env` names, and deletes the box.
1815
+ */
1816
+ async function runIsolatedCheck(options) {
1817
+ if (options.box) return runInBox(options, options.box);
1818
+ let scratch;
1819
+ let result;
1820
+ try {
1821
+ if (process.platform !== "linux") throw new Error("Linux Bubblewrap namespaces are required");
1822
+ const { timeoutMs, maxOutputBytes } = checkLimits(options);
1823
+ if (options.signal?.aborted) return {
1824
+ succeeded: false,
1825
+ reason: "cancelled",
1826
+ diagnostic: "Check cancelled"
1827
+ };
1828
+ const { workspace, tree } = await protectedTree(options);
1829
+ const binds = [];
1830
+ for (const path of toolchains) {
1831
+ let actual;
1832
+ try {
1833
+ actual = await realpath(path);
1834
+ } catch (error) {
1835
+ if (error.code === "ENOENT") continue;
1836
+ throw error;
1565
1837
  }
1566
- }),
1567
- receipt: (result) => loopCostReceipt(result, cost.resolveModel?.(result) ?? cost.model)
1568
- });
1569
- if (!paid.succeeded) throw paid.error;
1570
- const result = paid.value;
1571
- return (opts.toArtifact ?? ((r) => r.winner?.output))(result);
1572
- }
1573
- function loopCostReceipt(result, model) {
1574
- return costReceiptFromUsage(result.tokenUsage, model, {
1575
- usageUnknown: result.tokenUsage.tokensKnown === false,
1576
- actualCostUsd: result.costUsdKnown !== false ? result.costUsd : void 0,
1577
- costUnknown: result.costUsdKnown === false,
1578
- estimatedCostUsd: result.estimatedCostUsd
1579
- });
1838
+ for (const mount of [path, actual]) if (contains(mount, workspace) || contains(workspace, mount) || contains(mount, tree) || contains(tree, mount)) throw new Error(`Toolchain overlaps protected workspace: ${path}`);
1839
+ binds.push("--ro-bind", actual, path);
1840
+ }
1841
+ if (["/proc", "/dev"].some((path) => contains(path, tree) || contains(tree, path))) throw new Error("Tree overlaps a reserved namespace mount");
1842
+ scratch = await mkdtemp(join(tmpdir(), "runtime-check-"));
1843
+ if (contains(tree, scratch)) throw new Error("Temporary storage must be outside the input tree");
1844
+ const copy = join(scratch, "tree");
1845
+ await cp(tree, copy, {
1846
+ recursive: true,
1847
+ dereference: false,
1848
+ verbatimSymlinks: true
1849
+ });
1850
+ result = await execute([
1851
+ "--unshare-all",
1852
+ "--die-with-parent",
1853
+ "--new-session",
1854
+ "--cap-drop",
1855
+ "ALL",
1856
+ "--clearenv",
1857
+ "--setenv",
1858
+ "PATH",
1859
+ "/usr/bin:/bin",
1860
+ "--setenv",
1861
+ "HOME",
1862
+ "/tmp",
1863
+ "--setenv",
1864
+ "LANG",
1865
+ "C",
1866
+ ...Object.entries(options.env ?? {}).flatMap(([name, value]) => [
1867
+ "--setenv",
1868
+ name,
1869
+ value
1870
+ ]),
1871
+ ...binds,
1872
+ "--proc",
1873
+ "/proc",
1874
+ "--dev",
1875
+ "/dev",
1876
+ "--tmpfs",
1877
+ "/tmp",
1878
+ "--bind",
1879
+ copy,
1880
+ tree,
1881
+ "--chdir",
1882
+ tree,
1883
+ "--",
1884
+ ...options.command
1885
+ ], timeoutMs, maxOutputBytes, options.signal);
1886
+ } catch (error) {
1887
+ result = {
1888
+ succeeded: false,
1889
+ reason: "refused",
1890
+ diagnostic: String(error)
1891
+ };
1892
+ }
1893
+ if (scratch) try {
1894
+ await removeScratch(scratch);
1895
+ } catch (error) {
1896
+ const cleanupDiagnostic = String(error);
1897
+ result = result.succeeded ? {
1898
+ succeeded: false,
1899
+ reason: "cleanup-failed",
1900
+ diagnostic: cleanupDiagnostic,
1901
+ stdout: result.value.stdout,
1902
+ stderr: result.value.stderr,
1903
+ exitCode: 0
1904
+ } : {
1905
+ ...result,
1906
+ cleanupDiagnostic
1907
+ };
1908
+ }
1909
+ return result;
1580
1910
  }
1581
- /** Map Runtime usage into Eval's canonical paid-call receipt without inventing cache classes. */
1582
- function costReceiptFromUsage(usage, model, cost) {
1583
- const cacheComplete = hasCompleteCacheBreakdown(usage) && (usage.freshInput !== void 0 || usage.cacheRead !== void 0 || usage.cacheWrite !== void 0);
1584
- const cacheRead = cacheComplete ? usage.cacheRead ?? 0 : void 0;
1585
- const cacheWrite = cacheComplete ? usage.cacheWrite ?? 0 : void 0;
1586
- const classified = (cacheRead ?? 0) + (cacheWrite ?? 0);
1911
+ function checkLimits(options) {
1912
+ const timeoutMs = options.timeoutMs ?? 3e4;
1913
+ const maxOutputBytes = options.maxOutputBytes ?? 1048576;
1914
+ if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 2147483647 || !Number.isSafeInteger(maxOutputBytes) || maxOutputBytes < 1) throw new Error("Invalid execution limits");
1915
+ if (!options.command.length || !options.command[0]) throw new Error("A command is required");
1587
1916
  return {
1588
- model,
1589
- inputTokens: cacheComplete ? usage.input - classified : usage.input,
1590
- outputTokens: usage.output,
1591
- ...cacheComplete ? {
1592
- cachedTokens: cacheRead,
1593
- cacheWriteTokens: cacheWrite
1594
- } : {},
1595
- ...usage.tokensKnown === false || cost.usageUnknown ? { usageUnknown: true } : {},
1596
- ...cost.actualCostUsd !== void 0 ? { actualCostUsd: cost.actualCostUsd } : {},
1597
- ...cost.costUnknown ? { costUnknown: true } : {},
1598
- ...cost.estimatedCostUsd !== void 0 ? { estimatedCostUsd: cost.estimatedCostUsd } : {}
1917
+ timeoutMs,
1918
+ maxOutputBytes
1599
1919
  };
1600
1920
  }
1601
- /** Eval has one pre-admitted paid call for this dispatch. A tree can settle that call only when
1602
- * Runtime has provider-served identity for every inference attempt that actually started. Plan
1603
- * materialization is deliberately ignored: it describes what Runtime requested, not what the
1604
- * provider served. Every visible terminal child must carry provider attempt evidence; a missing
1605
- * evidence record stays unknown, including zero-valued or incomplete spend.
1606
- * @internal Testable identity projection used by superviseDispatch. */
1607
- function supervisedTreeModelForDispatch(result, rootProfile) {
1608
- const canonicalModels = /* @__PURE__ */ new Set();
1609
- const observedModels = [];
1610
- const wholeTreeEvidence = result.providerModel;
1611
- if (wholeTreeEvidence === void 0 || wholeTreeEvidence.status !== "known" && wholeTreeEvidence.reason !== "provider-model-conflict" || wholeTreeEvidence.attempts.length === 0) return { kind: "unknown" };
1612
- const wholeTreeIdentity = collectProviderModels(wholeTreeEvidence, () => true);
1613
- if (wholeTreeIdentity.kind !== "known") return wholeTreeIdentity;
1614
- for (const model of wholeTreeIdentity.observed) {
1615
- observedModels.push(model.raw);
1616
- canonicalModels.add(model.canonical);
1617
- }
1618
- if (canonicalModels.size === 0) return { kind: "unknown" };
1619
- if (canonicalModels.size !== 1) return {
1620
- kind: "mixed",
1621
- models: [...canonicalModels].sort()
1622
- };
1921
+ async function protectedTree(options) {
1922
+ const workspace = await realpath(options.workspaceRoot);
1923
+ const tree = await realpath(options.tree);
1924
+ if (!contains(workspace, tree) || !(await stat(tree)).isDirectory()) throw new Error("Tree must be a directory inside the workspace");
1623
1925
  return {
1624
- kind: "known",
1625
- model: observedModels[0]
1926
+ workspace,
1927
+ tree
1626
1928
  };
1627
1929
  }
1628
- function collectProviderModels(evidence, validate) {
1629
- if (evidence.status !== "known" && evidence.reason !== "provider-model-conflict" || !Array.isArray(evidence.attempts) || evidence.attempts.length === 0 || !Array.isArray(evidence.models)) return { kind: "unknown" };
1630
- const observed = [];
1631
- const canonical = /* @__PURE__ */ new Set();
1632
- const observedRaw = /* @__PURE__ */ new Set();
1633
- for (const attempt of evidence.attempts) {
1634
- if (attempt.providerDispatch === "not_started") {
1635
- if (attempt.observations.length > 0 || attempt.identityConflict === true) return { kind: "unknown" };
1636
- continue;
1637
- }
1638
- if (attempt === null || typeof attempt !== "object" || !Array.isArray(attempt.observations) || attempt.observations.length === 0 || attempt.identityConflict === true) return { kind: "unknown" };
1639
- let attemptBase;
1640
- let attemptSnapshot;
1641
- let attemptSnapshotKind;
1642
- let representative;
1643
- for (const model of attempt.observations) {
1644
- if (typeof model !== "string" || !validate(model)) return { kind: "unknown" };
1645
- const identity = canonicalObservedModelParts(model);
1646
- if (identity === void 0) return { kind: "unknown" };
1647
- const base = identity.base;
1648
- const snapshot = identity.snapshot;
1649
- if (attemptBase !== void 0 && attemptBase !== base) return {
1650
- kind: "mixed",
1651
- models: [.../* @__PURE__ */ new Set([...attemptBase ? [attemptBase] : [], identity.canonical])]
1652
- };
1653
- if (attemptSnapshot !== void 0 && snapshot !== void 0 && (attemptSnapshot !== snapshot || attemptSnapshotKind !== identity.snapshotKind)) {
1654
- if (attemptBase === void 0 || attemptSnapshotKind === void 0) return { kind: "unknown" };
1655
- return {
1656
- kind: "mixed",
1657
- models: [attemptIdentityText(attemptBase, attemptSnapshot, attemptSnapshotKind), identity.canonical]
1658
- };
1930
+ /** Box-workspace-relative directory that receives the tree and is the command's cwd. */
1931
+ const BOX_TREE = "tree";
1932
+ /** Bound on provisioning, delivery, and deletion; the box expires even if this process dies. */
1933
+ const BOX_SETUP_SECONDS = 900;
1934
+ /** The box's own process timeout is a backstop behind this module's deadline. */
1935
+ const BOX_PROCESS_BACKSTOP_MS = 3e4;
1936
+ /** Concurrent file deliveries into one check box. */
1937
+ const BOX_UPLOADS = 4;
1938
+ /** The box's process supervisor prefixes this to stderr when it cannot start the executable. */
1939
+ const BOX_EXEC_FAILURE = "process-supervisor: exec failed";
1940
+ async function runInBox(options, placement) {
1941
+ let box;
1942
+ let evidence;
1943
+ let result;
1944
+ try {
1945
+ const { timeoutMs, maxOutputBytes } = checkLimits(options);
1946
+ if (options.signal?.aborted) return {
1947
+ succeeded: false,
1948
+ reason: "cancelled",
1949
+ diagnostic: "Check cancelled"
1950
+ };
1951
+ const { tree } = await protectedTree(options);
1952
+ const captured = await captureMaterializedWorkspace(tree);
1953
+ if (placement.builderAccounts.length === 0 || placement.builderAccounts.some((id) => !id)) throw new Error("builderAccounts must name the account of every key the judged run holds");
1954
+ const account = (await placement.client.getIdentity()).customerId;
1955
+ if (!account) throw new Error("Sandbox returned no account for the check key");
1956
+ if (placement.builderAccounts.includes(account)) throw new Error(`Check account ${account} is an account the judged run holds a key to, so the run can reach the check box`);
1957
+ options.signal?.throwIfAborted();
1958
+ box = await placement.client.createIsolated({
1959
+ environment: placement.environment,
1960
+ agent: false,
1961
+ bare: false,
1962
+ ephemeral: true,
1963
+ egressPolicy: egressFor(placement),
1964
+ ...placement.resources ? { resources: placement.resources } : {},
1965
+ maxLifetimeSeconds: Math.ceil(timeoutMs / 1e3) + BOX_SETUP_SECONDS,
1966
+ idempotencyKey: `agent-runtime-check-${randomUUID()}`
1967
+ }, options.signal ? { signal: options.signal } : void 0);
1968
+ const receipt = box.createReceipt();
1969
+ if (receipt?.outcome !== "created" || receipt.ownerContext !== "isolated" || !Array.isArray(receipt.injectedSecrets) || receipt.injectedSecrets.length !== 0) throw new Error("Sandbox did not confirm a fresh box with no owner secrets");
1970
+ const egress = (await box.egress.get()).policy;
1971
+ const wanted = egressFor(placement);
1972
+ if (egress.mode !== wanted.mode || wanted.mode === "strict" && (egress.includeImplicitDomains === true || [...egress.allowDomains ?? []].sort().join(",") !== [...wanted.allowDomains ?? []].sort().join(","))) throw new Error(`Check box egress is ${egress.mode} ${JSON.stringify(egress.allowDomains ?? [])}, not ${wanted.mode} ${JSON.stringify(wanted.allowDomains ?? [])}`);
1973
+ await box.fs.mkdir(BOX_TREE, { recursive: true });
1974
+ const digests = new Map(captured.manifest.files.map((entry) => [entry.path, entry.sha256]));
1975
+ const target = box;
1976
+ const pending = [...captured.files];
1977
+ const deliver = async () => {
1978
+ try {
1979
+ for (let file = pending.shift(); file; file = pending.shift()) {
1980
+ options.signal?.throwIfAborted();
1981
+ const received = await target.fs.uploadData(`${BOX_TREE}/${file.path}`, file.bytes, { mode: file.mode });
1982
+ if (`sha256:${received.hash.replace(/^sha256:/, "").toLowerCase()}` !== digests.get(file.path) || received.size !== file.bytes.byteLength) throw new Error(`Check box received different bytes for ${file.path}`);
1983
+ }
1984
+ } catch (error) {
1985
+ pending.length = 0;
1986
+ throw error;
1659
1987
  }
1660
- attemptBase = base;
1661
- attemptSnapshot ??= snapshot;
1662
- attemptSnapshotKind ??= identity.snapshotKind;
1663
- if (snapshot !== void 0) representative = model;
1664
- observedRaw.add(model);
1665
- }
1666
- if (attemptBase === void 0 || attemptSnapshot === void 0 || attemptSnapshotKind === void 0 || representative === void 0) return { kind: "unknown" };
1667
- const attemptIdentity = attemptIdentityText(attemptBase, attemptSnapshot, attemptSnapshotKind);
1668
- observed.push({
1669
- raw: representative,
1670
- canonical: attemptIdentity
1671
- });
1672
- canonical.add(attemptIdentity);
1988
+ };
1989
+ await Promise.all(Array.from({ length: BOX_UPLOADS }, deliver));
1990
+ evidence = {
1991
+ sandboxId: box.id,
1992
+ account,
1993
+ input: captured.manifest,
1994
+ inputDigest: canonicalCandidateDigest(captured.manifest)
1995
+ };
1996
+ result = await executeInBox(box, options.command, timeoutMs, maxOutputBytes, options.env ?? {}, options.signal);
1997
+ } catch (error) {
1998
+ result = {
1999
+ succeeded: false,
2000
+ reason: options.signal?.aborted ? "cancelled" : "refused",
2001
+ diagnostic: String(error)
2002
+ };
1673
2003
  }
1674
- if (observed.length === 0) return { kind: "unknown" };
1675
- const declaredRaw = /* @__PURE__ */ new Set();
1676
- for (const model of evidence.models) {
1677
- if (typeof model !== "string") return { kind: "unknown" };
1678
- declaredRaw.add(model);
2004
+ if (box) try {
2005
+ await box.delete();
2006
+ } catch (error) {
2007
+ const cleanupDiagnostic = `Check box ${box.id} was not deleted: ${String(error)}`;
2008
+ result = result.succeeded ? {
2009
+ succeeded: false,
2010
+ reason: "cleanup-failed",
2011
+ diagnostic: cleanupDiagnostic,
2012
+ stdout: result.value.stdout,
2013
+ stderr: result.value.stderr,
2014
+ exitCode: 0
2015
+ } : {
2016
+ ...result,
2017
+ cleanupDiagnostic
2018
+ };
1679
2019
  }
1680
- if (declaredRaw.size !== observedRaw.size || [...declaredRaw].some((model) => !observedRaw.has(model))) return { kind: "unknown" };
1681
- return canonical.size > 1 ? {
1682
- kind: "mixed",
1683
- models: [...canonical].sort()
1684
- } : {
1685
- kind: "known",
1686
- observed
1687
- };
2020
+ return evidence ? {
2021
+ ...result,
2022
+ box: evidence
2023
+ } : result;
1688
2024
  }
1689
- function attemptIdentityText(base, snapshot, kind) {
1690
- return kind === "date" ? `${base}-${snapshot}` : `${base}@${snapshot}`;
2025
+ /** Blocked unless the placement names domains; then strict, with no implicit domain list. */
2026
+ function egressFor(placement) {
2027
+ const domains = [...new Set(placement.egress ?? [])].filter((domain) => domain.trim() !== "");
2028
+ return domains.length === 0 ? { mode: "blocked" } : {
2029
+ mode: "strict",
2030
+ allowDomains: domains,
2031
+ includeImplicitDomains: false
2032
+ };
1691
2033
  }
1692
- /** The Eval receipt surface has no pre-admitted per-model recursive-tree receipt bundle yet. */
1693
- var SupervisedTreeModelIdentityError = class extends Error {
1694
- constructor(identity) {
1695
- const detail = identity.kind === "mixed" ? `multiple models (${identity.models.join(", ")})` : "an unknown materialized model";
1696
- super(`superviseDispatch: cannot settle one Eval paid-call receipt for a tree with ${detail}`);
1697
- this.name = "SupervisedTreeModelIdentityError";
2034
+ async function executeInBox(box, command, timeoutMs, limit, env, signal) {
2035
+ const [executable, ...args] = command;
2036
+ const child = await box.process.spawnExact(executable, args, {
2037
+ cwd: BOX_TREE,
2038
+ env: {
2039
+ PATH: "/usr/bin:/bin",
2040
+ HOME: "/tmp",
2041
+ LANG: "C",
2042
+ ...env
2043
+ },
2044
+ inheritEnv: false,
2045
+ timeoutMs: timeoutMs + BOX_PROCESS_BACKSTOP_MS
2046
+ });
2047
+ const stdout = [];
2048
+ const stderr = [];
2049
+ let bytes = 0;
2050
+ let failure;
2051
+ let stopping;
2052
+ const stop = (reason) => {
2053
+ failure ??= reason;
2054
+ stopping ??= child.kill("SIGKILL", { tree: true }).catch(() => void 0);
2055
+ };
2056
+ const clearTimer = armDeadlineTimer(timeoutMs, () => stop("timeout"), true);
2057
+ const cancel = () => stop("cancelled");
2058
+ signal?.addEventListener("abort", cancel, { once: true });
2059
+ if (signal?.aborted) cancel();
2060
+ const capture = async (stream, target) => {
2061
+ for await (const text of stream) {
2062
+ const chunk = Buffer.from(text);
2063
+ const remaining = Math.max(0, limit - bytes);
2064
+ bytes += chunk.length;
2065
+ if (remaining) target.push(chunk.subarray(0, remaining));
2066
+ if (bytes > limit) stop("output-limit");
2067
+ }
2068
+ };
2069
+ let exitCode;
2070
+ let exitSignal;
2071
+ try {
2072
+ await Promise.all([capture(child.stdout(), stdout), capture(child.stderr(), stderr)]);
2073
+ exitCode = await child.wait();
2074
+ exitSignal = (await child.status()).exitSignal;
2075
+ } finally {
2076
+ clearTimer();
2077
+ signal?.removeEventListener("abort", cancel);
2078
+ await stopping;
1698
2079
  }
1699
- };
1700
- function unknownSupervisedTreeReceipt() {
2080
+ const out = Buffer.concat(stdout).toString();
2081
+ const err = Buffer.concat(stderr).toString();
2082
+ const evidence = {
2083
+ stdout: out,
2084
+ stderr: err,
2085
+ exitCode: exitSignal ? null : exitCode
2086
+ };
2087
+ if (failure) return {
2088
+ succeeded: false,
2089
+ reason: failure,
2090
+ diagnostic: err || failure,
2091
+ ...evidence
2092
+ };
2093
+ if (exitCode === 127 && err.startsWith(BOX_EXEC_FAILURE)) return {
2094
+ succeeded: false,
2095
+ reason: "refused",
2096
+ diagnostic: err,
2097
+ ...evidence
2098
+ };
2099
+ if (exitSignal) return {
2100
+ succeeded: false,
2101
+ reason: "failed",
2102
+ diagnostic: err || `Check killed by ${exitSignal}`,
2103
+ ...evidence
2104
+ };
2105
+ if (exitCode !== 0) return {
2106
+ succeeded: false,
2107
+ reason: "failed",
2108
+ diagnostic: err || out || `Check exited ${exitCode}`,
2109
+ ...evidence
2110
+ };
1701
2111
  return {
1702
- model: "unknown",
1703
- inputTokens: 0,
1704
- outputTokens: 0,
1705
- usageUnknown: true,
1706
- costUnknown: true
2112
+ succeeded: true,
2113
+ value: {
2114
+ stdout: out,
2115
+ stderr: err
2116
+ }
1707
2117
  };
1708
2118
  }
1709
- /** Run one recursive supervised tree inside Eval's pre-execution paid-call lifecycle. */
1710
- function superviseDispatch(opts) {
1711
- return async (profile, scenario, ctx) => {
1712
- const superviseOptions = opts.toSuperviseOptions(scenario, profile);
1713
- const task = opts.toTask(scenario, profile);
1714
- const maximumCharge = typeof opts.maximumCharge === "function" ? opts.maximumCharge(scenario, profile) : opts.maximumCharge;
1715
- const paid = await ctx.cost.runPaidCall({
1716
- channel: "agent",
1717
- actor: opts.costSource ?? "supervise",
1718
- model: "unknown",
1719
- signal: ctx.signal,
1720
- ...maximumCharge ? { maximumCharge } : {},
1721
- execute: async (executionSignal) => {
1722
- const result = await supervise(profile, task, {
1723
- ...superviseOptions,
1724
- signal: executionSignal
1725
- });
1726
- const identity = supervisedTreeModelForDispatch(result, profile);
1727
- if (identity.kind !== "known") throw new SupervisedTreeModelIdentityError(identity);
1728
- return result;
1729
- },
1730
- receipt: (result) => {
1731
- const identity = supervisedTreeModelForDispatch(result, profile);
1732
- if (identity.kind !== "known") throw new SupervisedTreeModelIdentityError(identity);
1733
- return costReceiptFromUsage(result.spentTotal.tokens, identity.model, {
1734
- usageUnknown: result.spentTotal.tokensKnown === false,
1735
- actualCostUsd: result.spentTotal.usdKnown !== false ? result.spentTotal.usd : void 0,
1736
- costUnknown: result.spentTotal.usdKnown === false
1737
- });
1738
- },
1739
- receiptFromError: (error) => error instanceof SupervisedTreeModelIdentityError ? unknownSupervisedTreeReceipt() : void 0
2119
+ function execute(args, timeoutMs, limit, signal) {
2120
+ return new Promise((done) => {
2121
+ const child = spawn("/usr/bin/bwrap", [
2122
+ "--json-status-fd",
2123
+ "3",
2124
+ ...args
2125
+ ], {
2126
+ env: {},
2127
+ stdio: [
2128
+ "ignore",
2129
+ "pipe",
2130
+ "pipe",
2131
+ "pipe"
2132
+ ],
2133
+ detached: true
1740
2134
  });
1741
- if (!paid.succeeded) throw paid.error;
1742
- return (opts.toArtifact ?? ((result) => {
1743
- if (result.kind !== "winner") throw new Error(`superviseDispatch: supervised tree ended without a winner (${result.reason})`);
1744
- return result.out;
1745
- }))(paid.value);
2135
+ const stdout = [];
2136
+ const stderr = [];
2137
+ let bytes = 0;
2138
+ let status = "";
2139
+ let failure;
2140
+ const stop = (reason) => {
2141
+ failure ??= reason;
2142
+ if (child.pid) try {
2143
+ process.kill(-child.pid, "SIGKILL");
2144
+ } catch (error) {
2145
+ if (error.code !== "ESRCH") child.kill("SIGKILL");
2146
+ }
2147
+ };
2148
+ const clearTimer = armDeadlineTimer(timeoutMs, () => stop("timeout"), true);
2149
+ const cancel = () => stop("cancelled");
2150
+ signal?.addEventListener("abort", cancel, { once: true });
2151
+ if (signal?.aborted) cancel();
2152
+ const capture = (target) => (chunk) => {
2153
+ const remaining = Math.max(0, limit - bytes);
2154
+ bytes += chunk.length;
2155
+ if (remaining) target.push(chunk.subarray(0, remaining));
2156
+ if (bytes > limit) stop("output-limit");
2157
+ };
2158
+ child.stdio[3]?.on("data", (chunk) => {
2159
+ if (status.length + chunk.length > 16384) {
2160
+ stop("output-limit");
2161
+ return;
2162
+ }
2163
+ status += chunk.toString();
2164
+ });
2165
+ child.stdout.on("data", capture(stdout));
2166
+ child.stderr.on("data", capture(stderr));
2167
+ let spawnError;
2168
+ child.on("error", (error) => {
2169
+ spawnError = error;
2170
+ });
2171
+ child.on("close", (code) => {
2172
+ clearTimer();
2173
+ signal?.removeEventListener("abort", cancel);
2174
+ const out = Buffer.concat(stdout).toString();
2175
+ const err = Buffer.concat(stderr).toString();
2176
+ const evidence = {
2177
+ stdout: out,
2178
+ stderr: err,
2179
+ exitCode: code
2180
+ };
2181
+ const executed = status.split("\n").some((line) => {
2182
+ try {
2183
+ const record = JSON.parse(line);
2184
+ return record !== null && typeof record === "object" && "exit-code" in record && typeof record["exit-code"] === "number" && record["exit-code"] === code;
2185
+ } catch {
2186
+ return false;
2187
+ }
2188
+ });
2189
+ if (failure) done({
2190
+ succeeded: false,
2191
+ reason: failure,
2192
+ diagnostic: err || failure,
2193
+ ...evidence
2194
+ });
2195
+ else if (spawnError) done({
2196
+ succeeded: false,
2197
+ reason: "refused",
2198
+ diagnostic: String(spawnError),
2199
+ ...evidence
2200
+ });
2201
+ else if (!executed) done({
2202
+ succeeded: false,
2203
+ reason: "refused",
2204
+ diagnostic: err || "Bubblewrap did not confirm command execution",
2205
+ ...evidence
2206
+ });
2207
+ else if (code !== 0) done({
2208
+ succeeded: false,
2209
+ reason: "failed",
2210
+ diagnostic: err || out || `Check exited ${code}`,
2211
+ ...evidence
2212
+ });
2213
+ else done({
2214
+ succeeded: true,
2215
+ value: {
2216
+ stdout: out,
2217
+ stderr: err
2218
+ }
2219
+ });
2220
+ });
2221
+ });
2222
+ }
2223
+ const executeFile = promisify(execFile);
2224
+ /** GNU tools traverse relative to open directories, including trees deeper than PATH_MAX.
2225
+ * Recursive chmod ignores encountered symlinks; rm never follows their targets. */
2226
+ async function removeScratch(path) {
2227
+ const options = {
2228
+ env: {},
2229
+ timeout: 1e4,
2230
+ maxBuffer: 16384
1746
2231
  };
2232
+ let permissionError;
2233
+ try {
2234
+ await executeFile("/bin/chmod", [
2235
+ "--recursive",
2236
+ "--preserve-root",
2237
+ "u+rwX",
2238
+ "--",
2239
+ path
2240
+ ], options);
2241
+ } catch (error) {
2242
+ permissionError = error;
2243
+ }
2244
+ try {
2245
+ await executeFile("/bin/rm", [
2246
+ "--recursive",
2247
+ "--force",
2248
+ "--one-file-system",
2249
+ "--",
2250
+ path
2251
+ ], options);
2252
+ } catch (error) {
2253
+ throw new Error([permissionError, error].filter((value) => value !== void 0).map(String).join("; "));
2254
+ }
1747
2255
  }
1748
- function modelFromLoopOptions(options) {
1749
- const profiles = options.agentRun ? [options.agentRun.profile] : options.agentRuns?.map((run) => run.profile) ?? [];
1750
- const models = new Set(profiles.map((profile) => profile.model?.default).filter((model) => !!model));
1751
- if (models.size === 1) return [...models][0];
1752
- return models.size > 1 ? "mixed" : "unknown";
2256
+ //#endregion
2257
+ //#region src/runtime/declared-check.ts
2258
+ /**
2259
+ * A declared check: the one frozen evaluator program that decides "done" for a run, named by
2260
+ * digest, run by Runtime in a fresh box on the check account.
2261
+ *
2262
+ * The same program is read three times (discovery `docs/38-one-loop-and-continuation.md`, section
2263
+ * 1): inside the run on every `submit_result` and at a turn end with no accepted result, after the
2264
+ * run to score it, and after each version to rank versions. Before this, each play wrote its own
2265
+ * check as Lab code in a method bundle — an exact-check judge, a sealed-cases judge, a product
2266
+ * judge — and the in-run check was a separate, weaker "shape and persistence" test, so 22 directors
2267
+ * passed their own check and then failed the outside one.
2268
+ *
2269
+ * The program prints one agent-eval `JudgeScore` as JSON on its last line of stdout: each checked
2270
+ * item is a dimension, `composite` is the score, and `notes` holds one `FAIL <item> <where>:
2271
+ * <reason>` line per failed item. The record states the pass threshold. A program that does not
2272
+ * run, or prints no score, gives no verdict: it is {@link CheckUnavailableError}, and the loop
2273
+ * pauses instead of blaming the director.
2274
+ *
2275
+ * The box receives the program's files at its working directory, the submitted result at
2276
+ * `_input/result.json` when there is one, and the sealed cases at `_sealed/` only when the read
2277
+ * scores a run or a version. `CHECK_SET` is `development` or `sealed`; `CHECK_RESULT` names the
2278
+ * result file when present. Sealed cases never reach an in-run read, so their lines never reach
2279
+ * the director.
2280
+ */
2281
+ const INPUT_DIR = "_input";
2282
+ const SEALED_DIR = "_sealed";
2283
+ const DEFAULT_TIMEOUT_MS = 15 * 6e4;
2284
+ /** Refuse a malformed declaration before any compute. */
2285
+ function assertDeclaredCheck(check, context) {
2286
+ const fail = (message) => {
2287
+ throw new ValidationError(`${context}: check ${message}`);
2288
+ };
2289
+ if (typeof check !== "object" || check === null) fail("must be an object");
2290
+ if (typeof check.program?.dir !== "string" || check.program.dir.trim() === "") fail("program.dir must name the evaluator directory");
2291
+ if (!/^sha256:[0-9a-f]{64}$/u.test(check.program?.digest ?? "")) fail("program.digest must be a sha256 digest");
2292
+ if (!Array.isArray(check.command) || check.command.length === 0 || !check.command[0]) fail("command must name an executable");
2293
+ if (typeof check.environment !== "string" || check.environment.trim() === "") fail("environment must name the box image");
2294
+ if (typeof check.pass !== "number" || !Number.isFinite(check.pass)) fail("pass must be a finite threshold on the composite");
2295
+ if (check.feedback !== "verbatim" && check.feedback !== "pass-only") fail("feedback must be 'verbatim' or 'pass-only'");
2296
+ if (check.sealed !== void 0 && !/^sha256:[0-9a-f]{64}$/u.test(check.sealed.digest ?? "")) fail("sealed.digest must be a sha256 digest");
2297
+ for (const name of check.secrets ?? []) if (!/^[A-Z_][A-Z0-9_]*$/u.test(name)) fail(`secret name ${JSON.stringify(name)} is not an env name`);
2298
+ }
2299
+ /** The canonical digest of a directory's files: the digest a record names a program by. */
2300
+ async function checkProgramDigest(dir) {
2301
+ return canonicalCandidateDigest((await captureMaterializedWorkspace(dir)).manifest);
2302
+ }
2303
+ /** The digest a version judge records: the program, the sealed cases, and how they are run. */
2304
+ function declaredCheckDigest(check) {
2305
+ return sha256Bytes(Buffer.from(JSON.stringify({
2306
+ program: check.program.digest,
2307
+ sealed: check.sealed?.digest ?? null,
2308
+ command: check.command,
2309
+ environment: check.environment,
2310
+ pass: check.pass
2311
+ })));
1753
2312
  }
1754
2313
  /**
1755
- * Adapter for plain `runCampaign` scenarios. This is the Runtime-side pair for
1756
- * agent-eval fixture scenarios: load fixtures in `agent-eval/campaign`, build
1757
- * the Runtime cell here, and keep paid-call admission, receipts, and traces
1758
- * automatic.
2314
+ * Read the check once. `result` is the submitted result, absent for a read of the run's state.
2315
+ * `set: 'sealed'` adds the sealed cases. Throws {@link CheckUnavailableError} when the program
2316
+ * could not run or printed no score.
1759
2317
  */
1760
- function loopCampaignDispatch(opts) {
1761
- return (scenario, ctx) => {
1762
- const loopOptions = opts.toLoopOptions(scenario);
1763
- return runLoopWithCampaignContext(opts, loopOptions, ctx, {
1764
- model: modelFromLoopOptions(loopOptions),
1765
- maximumCharge: typeof opts.maximumCharge === "function" ? opts.maximumCharge(scenario) : opts.maximumCharge,
1766
- resolveModel: opts.resolveCostModel ? (result) => opts.resolveCostModel?.(result, scenario) : void 0
2318
+ async function readDeclaredCheck(check, placement, read) {
2319
+ if (read.set === "sealed" && check.sealed === void 0) throw new ValidationError("readDeclaredCheck: a sealed read needs check.sealed");
2320
+ const environment = placement.env ?? process.env;
2321
+ const secrets = {};
2322
+ for (const name of check.secrets ?? []) {
2323
+ const value = environment[name];
2324
+ if (value === void 0 || value === "") throw new CheckUnavailableError(`the check's secret ${name} is not set on this host`);
2325
+ secrets[name] = value;
2326
+ }
2327
+ const workspace = await mkdtemp(join(tmpdir(), "declared-check-"));
2328
+ try {
2329
+ const tree = join(workspace, "tree");
2330
+ await copyVerified(check.program.dir, check.program.digest, tree, "program");
2331
+ const entries = await readdir(tree);
2332
+ if (entries.includes(INPUT_DIR) || entries.includes(SEALED_DIR)) throw new ValidationError(`readDeclaredCheck: the program may not hold ${INPUT_DIR}/ or ${SEALED_DIR}/; Runtime mounts them`);
2333
+ const env = {
2334
+ ...secrets,
2335
+ CHECK_SET: read.set
2336
+ };
2337
+ if (read.result !== void 0) {
2338
+ await mkdir(join(tree, INPUT_DIR));
2339
+ await writeFile(join(tree, INPUT_DIR, "result.json"), `${JSON.stringify(read.result)}\n`);
2340
+ env.CHECK_RESULT = `${INPUT_DIR}/result.json`;
2341
+ }
2342
+ if (read.set === "sealed" && check.sealed !== void 0) await copyVerified(check.sealed.dir, check.sealed.digest, join(tree, SEALED_DIR), "sealed cases");
2343
+ return verdictOf(await runIsolatedCheck({
2344
+ workspaceRoot: workspace,
2345
+ tree,
2346
+ command: check.command,
2347
+ timeoutMs: check.timeoutMs ?? DEFAULT_TIMEOUT_MS,
2348
+ env,
2349
+ ...read.signal === void 0 ? {} : { signal: read.signal },
2350
+ box: {
2351
+ client: placement.client,
2352
+ builderAccounts: placement.builderAccounts,
2353
+ environment: check.environment,
2354
+ ...check.egress === void 0 ? {} : { egress: check.egress },
2355
+ ...check.resources === void 0 ? {} : { resources: check.resources }
2356
+ }
2357
+ }), check.pass);
2358
+ } finally {
2359
+ await rm(workspace, {
2360
+ recursive: true,
2361
+ force: true
1767
2362
  });
2363
+ }
2364
+ }
2365
+ /** The declared check as a manager's completion check: every in-run read uses development cases. */
2366
+ function declaredCheckDeliverable(check, placement) {
2367
+ assertDeclaredCheck(check, "declaredCheckDeliverable");
2368
+ return {
2369
+ check: (result) => readDeclaredCheck(check, placement, {
2370
+ result,
2371
+ set: "development"
2372
+ }),
2373
+ checkState: () => readDeclaredCheck(check, placement, { set: "development" }),
2374
+ feedback: check.feedback,
2375
+ ...check.sealed === void 0 ? {} : { sealed: true },
2376
+ ...check.describe === void 0 ? {} : { describe: check.describe }
1768
2377
  };
1769
2378
  }
1770
2379
  /**
1771
- * Adapter for `runProfileMatrix` (profile is an axis). Returns a
1772
- * `ProfileDispatchFn` that runs `runAgentRounds` per (profile, scenario) cell
1773
- * inside Eval's paid-call lifecycle.
2380
+ * The declared check as a version chain's judge: it scores a settled version on its sealed cases
2381
+ * when the record has them, and on its development cases otherwise. The per-item verdict rides in
2382
+ * the ledger, so the next version's review names each item. A read that could not run scores
2383
+ * `null`, which never counts as an improvement.
1774
2384
  */
1775
- function loopDispatch(opts) {
1776
- return (profile, scenario, ctx) => runLoopForCell(opts, scenario, profile, ctx);
2385
+ function declaredCheckJudge(check, placement) {
2386
+ assertDeclaredCheck(check, "declaredCheckJudge");
2387
+ const digest = declaredCheckDigest(check);
2388
+ return {
2389
+ digest,
2390
+ async judge(version, signal) {
2391
+ try {
2392
+ const verdict = await readDeclaredCheck(check, placement, {
2393
+ ...version.result.kind === "winner" ? { result: version.result.out } : {},
2394
+ set: check.sealed === void 0 ? "development" : "sealed",
2395
+ signal
2396
+ });
2397
+ return {
2398
+ score: verdict.composite ?? (verdict.pass ? 1 : 0),
2399
+ judgeDigest: digest,
2400
+ check: verdict
2401
+ };
2402
+ } catch (error) {
2403
+ if (!(error instanceof CheckUnavailableError)) throw error;
2404
+ return {
2405
+ score: null,
2406
+ judgeDigest: digest,
2407
+ detail: { unavailable: error.message }
2408
+ };
2409
+ }
2410
+ }
2411
+ };
2412
+ }
2413
+ function verdictOf(outcome, pass) {
2414
+ if (!outcome.succeeded) throw new CheckUnavailableError(`the check program ${outcome.reason}: ${outcome.diagnostic.slice(0, 2e3)}`);
2415
+ const last = outcome.value.stdout.trimEnd().split("\n").at(-1) ?? "";
2416
+ let score;
2417
+ try {
2418
+ score = JSON.parse(last);
2419
+ } catch {
2420
+ throw new CheckUnavailableError("the check program printed no JudgeScore on its last line");
2421
+ }
2422
+ const raw = score;
2423
+ if (typeof raw !== "object" || raw === null || typeof raw.dimensions !== "object" || raw.dimensions === null || typeof raw.composite !== "number" || typeof raw.notes !== "string") throw new CheckUnavailableError("the check program printed a line that is not a JudgeScore");
2424
+ return verdictFromJudgeScore({
2425
+ dimensions: raw.dimensions,
2426
+ composite: raw.composite,
2427
+ notes: raw.notes,
2428
+ ...raw.failed === true ? { failed: true } : {}
2429
+ }, pass);
2430
+ }
2431
+ /** Copy exactly the bytes whose digest was checked: the files are read once, verified, and those
2432
+ * bytes are written, so a file changed after the check cannot reach the box. */
2433
+ async function copyVerified(from, digest, to, label) {
2434
+ const captured = await captureMaterializedWorkspace(from);
2435
+ const actual = canonicalCandidateDigest(captured.manifest);
2436
+ if (actual !== digest) throw new ValidationError(`readDeclaredCheck: the ${label} at ${from} is ${actual}; the record names ${digest}`);
2437
+ for (const file of captured.files) {
2438
+ const path = join(to, file.path);
2439
+ await mkdir(dirname(path), { recursive: true });
2440
+ await writeFile(path, file.bytes, { mode: file.mode });
2441
+ }
1777
2442
  }
1778
2443
  //#endregion
1779
- //#region src/runtime/inline-sandbox-client.ts
1780
- /**
1781
- * The ONE pseudo-box adapter: present any one-shot `Executor` (router / bridge /
1782
- * BYO) as a `SandboxClient` so the round-synchronous `runAgentRounds` can drive it
1783
- * without each call site re-faking a box. This is the single shell that
1784
- * `bench/src/router-executor.ts`, generate-eval's old `bridgeSandboxClient`, and
1785
- * the search-bench bridge transport were each re-implementing.
1786
- *
1787
- * It is deliberately for NON-box executors only — a real sandbox harness already
1788
- * IS a `SandboxClient` (boxes, sessions, fs, fork are real there). Here each
1789
- * `streamPrompt` runs the executor once and emits its result plus the canonical
1790
- * `done` event. The result carries output and usage; `done` confirms execution settled.
1791
- * There are no sessions, no fs,
1792
- * no fork (those degrade gracefully via the optional `SandboxClient` methods).
1793
- */
1794
- function isAsyncIterable$1(v) {
1795
- return typeof v === "object" && v !== null && Symbol.asyncIterator in v;
2444
+ //#region src/runtime/loop-dispatch.ts
2445
+ /** Bridge a campaign `DispatchContext.trace` to a `LoopTraceEmitter` so every
2446
+ * `loop.*` event lands as a span under the cell's scoped trace. */
2447
+ function campaignTraceToLoopEmitter(trace) {
2448
+ return { emit(event) {
2449
+ trace.span(event.kind, {
2450
+ runId: event.runId,
2451
+ timestamp: event.timestamp,
2452
+ ...event.payload
2453
+ }).end();
2454
+ } };
1796
2455
  }
1797
- /** Drive a (possibly streaming) executor to its terminal artifact. */
1798
- async function settle(exec, task, signal) {
1799
- const r = exec.execute(task, signal);
1800
- if (isAsyncIterable$1(r)) {
1801
- for await (const _ of r);
1802
- return exec.resultArtifact();
1803
- }
1804
- return r;
2456
+ async function runLoopForCell(opts, scenario, profile, ctx) {
2457
+ const loopOptions = opts.toLoopOptions(scenario, profile);
2458
+ return runLoopWithCampaignContext(opts, loopOptions, ctx, {
2459
+ model: profile.model?.default ?? modelFromLoopOptions(loopOptions),
2460
+ maximumCharge: typeof opts.maximumCharge === "function" ? opts.maximumCharge(scenario, profile) : opts.maximumCharge,
2461
+ resolveModel: opts.resolveCostModel ? (result) => opts.resolveCostModel?.(result, scenario, profile) : void 0
2462
+ });
1805
2463
  }
1806
- /**
1807
- * Adapt an `ExecutorFactory` into a `SandboxClient` for `runAgentRounds`. The factory is
1808
- * instantiated fresh per `streamPrompt` (mirrors the per-spawn executor lifecycle):
1809
- * run once on the prompt, emit the terminal result event, tear down.
1810
- *
1811
- * There is no box, so a per-prompt `backend` or `model` override is refused rather than dropped;
1812
- * other per-prompt options (`timeoutMs`, `context`) are accepted and ignored.
1813
- */
1814
- function inlineSandboxClient(factory, defaults = {}) {
1815
- const capturedDefaultProfile = defaults.profile === void 0 ? void 0 : agentProfileSchema.parse(structuredClone(defaults.profile));
1816
- let seq = 0;
1817
- return { async create(options) {
1818
- const id = `inline-${seq++}`;
1819
- const createOptions = options;
1820
- return {
1821
- id,
1822
- async *streamPrompt(message, opts) {
1823
- assertBoxlessPromptOptions(opts, "inlineSandboxClient");
1824
- const controller = new AbortController();
1825
- const callerSignal = opts?.signal;
1826
- const onAbort = () => controller.abort(callerSignal?.reason ?? /* @__PURE__ */ new Error("prompt aborted"));
1827
- if (callerSignal) if (callerSignal.aborted) onAbort();
1828
- else callerSignal.addEventListener("abort", onAbort, { once: true });
1829
- const requestedProfile = (options?.backend && typeof options.backend === "object" ? options.backend.profile : void 0) ?? capturedDefaultProfile;
1830
- const parsedProfile = agentProfileSchema.safeParse(requestedProfile);
1831
- if (!parsedProfile.success) throw new Error("inlineSandboxClient: an exact AgentProfile is required; pass defaults.profile or create({ backend: { profile } })");
1832
- const exec = factory({
1833
- profile: parsedProfile.data,
1834
- harness: null
1835
- }, {
1836
- signal: controller.signal,
1837
- seams: { createOptions }
1838
- });
1839
- try {
1840
- const artifact = await settle(exec, message, controller.signal);
1841
- const out = artifact.out;
1842
- const tokensIn = artifact.spent.tokens.input;
1843
- const tokensOut = artifact.spent.tokens.output;
1844
- const costUsd = artifact.spent.usd;
1845
- const estimatedCostUsd = out?.estimatedCostUsd;
1846
- if (artifact.spent.iterations > 0 || artifact.spent.tokensKnown === false || artifact.spent.usdKnown === false || tokensIn > 0 || tokensOut > 0 || costUsd > 0 || estimatedCostUsd !== void 0) yield {
1847
- type: "llm_call",
1848
- data: {
1849
- ...artifact.spent.tokensKnown === false ? {} : {
1850
- tokensIn,
1851
- tokensOut
1852
- },
1853
- ...artifact.spent.usdKnown !== false ? { costUsd } : {},
1854
- ...artifact.spent.tokensKnown === false ? { tokensKnown: false } : {},
1855
- ...artifact.spent.usdKnown === false ? { costKnown: false } : {},
1856
- ...artifact.spent.usdKnown !== false ? {
1857
- costKnown: true,
1858
- costProvenance: "provider-receipt"
1859
- } : {},
1860
- ...estimatedCostUsd !== void 0 ? { estimatedCostUsd } : {},
1861
- ...out?.promptCache ? { promptCache: out.promptCache } : {}
1862
- }
1863
- };
1864
- yield {
1865
- type: "result",
1866
- data: {
1867
- finalText: out?.content ?? "",
1868
- ...artifact.spent.tokensKnown === false ? { tokensKnown: false } : { tokenUsage: {
1869
- inputTokens: tokensIn,
1870
- outputTokens: tokensOut
1871
- } },
1872
- ...artifact.spent.usdKnown === false ? {
1873
- costKnown: false,
1874
- ...costUsd > 0 ? { costUsd } : {}
1875
- } : {
1876
- costUsd,
1877
- costKnown: true,
1878
- costProvenance: "provider-receipt"
1879
- },
1880
- ...estimatedCostUsd !== void 0 ? { estimatedCostUsd } : {},
1881
- ...out?.promptCache ? { promptCache: out.promptCache } : {}
1882
- }
1883
- };
1884
- yield {
1885
- type: "done",
1886
- data: { outcome: { type: "completed" } }
1887
- };
1888
- } finally {
1889
- callerSignal?.removeEventListener("abort", onAbort);
1890
- await exec.teardown("brutalKill").catch(() => {});
1891
- }
1892
- },
1893
- async delete() {}
1894
- };
1895
- } };
1896
- }
1897
- //#endregion
1898
- //#region src/runtime/local-sandbox-client.ts
1899
- /**
1900
- * `localSandboxClient` — the SAME-HOST pseudo-box: a `SandboxClient` whose
1901
- * `create()` MATERIALIZES the profile's stdio MCP servers as local child
1902
- * processes (`materializeLocalMcp`) and whose `streamPrompt` drives a real
1903
- * tool loop (`runBrainLoop` over the router brain) with those live tools.
1904
- *
1905
- * Despite the interface name, this does NOT isolate processes. It is only for
1906
- * author-controlled profiles whose local MCP commands a caller explicitly
1907
- * trusts; user- or model-authored code belongs in a real sandbox. `delete()`
1908
- * kills the children.
1909
- *
1910
- * A per-create profile may change the prompt surface. Permission to start a
1911
- * local MCP process applies only when its full canonical bytes match the fixed
1912
- * constructor profile; a different generated profile is refused.
1913
- *
1914
- * Event protocol matches `inlineSandboxClient`: known token usage is emitted as one `llm_call`;
1915
- * Router catalog cost remains a separately-labelled estimate, never billed spend.
1916
- */
1917
- /** A same-host `SandboxClient` adapter with no process isolation. Local MCP is
1918
- * refused unless the caller explicitly supplies a policy that allows it.
1919
- *
1920
- * There is no box, so a per-prompt `backend` or `model` override is refused rather than dropped;
1921
- * other per-prompt options (`timeoutMs`, `context`) are accepted and ignored. */
1922
- function localSandboxClient(opts) {
1923
- const defaultProfile = opts.profile === void 0 ? void 0 : executableAgentProfileSnapshot(opts.profile, "localSandboxClient default profile");
1924
- const router = Object.freeze({ ...opts.router });
1925
- if (opts.profileSecurityPolicy?.allowLocalMcp && defaultProfile === void 0) throw new ValidationError("localSandboxClient: allowLocalMcp requires a fixed author-controlled profile; dynamic profiles need a real sandbox");
1926
- const trustedProfileDigest = opts.profileSecurityPolicy?.allowLocalMcp && defaultProfile !== void 0 ? canonicalAgentProfileDigest(defaultProfile) : void 0;
1927
- let seq = 0;
1928
- return { async create(options) {
1929
- const profile = executableAgentProfileSnapshot((options?.backend)?.profile ?? defaultProfile, "localSandboxClient");
1930
- const model = profileProviderModel(profile);
1931
- const settings = profileModelExecutionSettings(profile, "localSandboxClient");
1932
- const policyApplies = opts.profileSecurityPolicy !== void 0 && (!opts.profileSecurityPolicy.allowLocalMcp || trustedProfileDigest !== void 0 && canonicalAgentProfileDigest(profile) === trustedProfileDigest);
1933
- const mcp = await materializeLocalMcp(profile, {
1934
- ...opts.keys ? { keys: opts.keys } : {},
1935
- ...policyApplies ? { profileSecurityPolicy: opts.profileSecurityPolicy } : {}
1936
- });
1937
- const brain = routerBrain({
1938
- routerBaseUrl: router.baseUrl,
1939
- routerKey: router.key,
1940
- model,
1941
- ...settings.retry !== void 0 ? { retry: settings.retry } : {},
1942
- ...enforceTokenLimits(settings.tokenLimits, "router", "localSandboxClient").applied,
1943
- ...settings.stream !== void 0 ? { stream: settings.stream } : {}
1944
- }, {
1945
- ...settings.temperature !== void 0 ? { temperature: settings.temperature } : {},
1946
- ...settings.seed !== void 0 ? { seed: settings.seed } : {},
1947
- ...settings.toolChoice !== void 0 ? { toolChoice: settings.toolChoice } : {},
1948
- ...settings.extraBody !== void 0 ? { extraBody: settings.extraBody } : {},
1949
- ...profile.model?.reasoningEffort ? { reasoningEffort: profile.model.reasoningEffort } : {}
1950
- });
1951
- const system = [profile.prompt?.systemPrompt, ...profile.prompt?.instructions ?? []].filter((s) => typeof s === "string" && s.trim().length > 0).join("\n\n");
1952
- return {
1953
- id: `local-${seq++}`,
1954
- async *streamPrompt(message, popts) {
1955
- assertBoxlessPromptOptions(popts, "localSandboxClient");
1956
- let estimatedCostUsd = 0;
1957
- let sawEstimatedCost = false;
1958
- const chat = async (messages, tools) => {
1959
- const r = await brain(messages, tools);
1960
- if (r.costProvenance === "catalog-estimate" && r.costUsd !== void 0) {
1961
- estimatedCostUsd += r.costUsd;
1962
- sawEstimatedCost = true;
1963
- }
1964
- return r;
1965
- };
1966
- const r = await runBrainLoop({
1967
- chat,
1968
- tools: mcp.tools,
1969
- execute: (name, args) => mcp.call(name, args),
1970
- initialMessages: [...system ? [{
1971
- role: "system",
1972
- content: system
1973
- }] : [], {
1974
- role: "user",
1975
- content: message
1976
- }],
1977
- maxTurns: settings.maxTurns ?? 0,
1978
- hooks: { stopBefore: () => popts?.signal?.aborted === true }
1979
- });
1980
- if (r.turns > 0) yield {
1981
- type: "llm_call",
1982
- data: {
1983
- model,
1984
- ...r.tokensKnown === false ? { tokensKnown: false } : {
1985
- tokensIn: r.usage.input,
1986
- tokensOut: r.usage.output
1987
- },
1988
- costKnown: false,
1989
- ...sawEstimatedCost ? { estimatedCostUsd } : {}
1990
- }
1991
- };
1992
- yield {
1993
- type: "result",
1994
- data: {
1995
- finalText: r.final,
1996
- ...r.tokensKnown === false ? { tokensKnown: false } : { tokenUsage: {
1997
- inputTokens: r.usage.input,
1998
- outputTokens: r.usage.output
1999
- } },
2000
- costKnown: false,
2001
- ...sawEstimatedCost ? { estimatedCostUsd } : {}
2002
- }
2003
- };
2004
- },
2005
- async delete() {
2006
- await mcp.close();
2464
+ async function runLoopWithCampaignContext(opts, loopOptions, ctx, cost) {
2465
+ const paid = await ctx.cost.runPaidCall({
2466
+ channel: "agent",
2467
+ actor: opts.costSource ?? "loop",
2468
+ model: cost.model,
2469
+ signal: ctx.signal,
2470
+ ...cost.maximumCharge ? { maximumCharge: cost.maximumCharge } : {},
2471
+ execute: (executionSignal) => runAgentRounds({
2472
+ ...loopOptions,
2473
+ ctx: {
2474
+ sandboxClient: opts.sandboxClient,
2475
+ ...opts.promptOptions === void 0 ? {} : { promptOptions: opts.promptOptions },
2476
+ signal: executionSignal,
2477
+ traceEmitter: opts.forwardTrace === false ? void 0 : campaignTraceToLoopEmitter(ctx.trace)
2007
2478
  }
2008
- };
2009
- } };
2479
+ }),
2480
+ receipt: (result) => loopCostReceipt(result, cost.resolveModel?.(result) ?? cost.model)
2481
+ });
2482
+ if (!paid.succeeded) throw paid.error;
2483
+ const result = paid.value;
2484
+ return (opts.toArtifact ?? ((r) => r.winner?.output))(result);
2010
2485
  }
2011
- //#endregion
2012
- //#region src/runtime/resolve-sandbox-client.ts
2013
- /**
2014
- * The product-facing backend selector: one call picks the execution transport a
2015
- * `runAgentRounds` (or any consumer that drives a `SandboxClient`) runs on, from the
2016
- * package a product already depends on. It is pure sugar over the existing
2017
- * primitives — `createExecutor({ backend })` + `inlineSandboxClient` — and adds
2018
- * ZERO transport code (the node:http cli-bridge transport lives in `bridgeExecutor`).
2019
- *
2020
- * The heavy `resolveBenchClient` in `@tangle-network/agent-bench` is the bench-side
2021
- * sibling: it layers router/search specifics a product should not have to depend on.
2022
- * This is the generic, dep-light core products call to switch backends:
2023
- *
2024
- * - `backend: 'sandbox'` → IN-BOX: return the caller's real Sandbox-backed
2025
- * `sandboxClient` unchanged. Fail loud if absent — a
2026
- * product on the sandbox backend already holds its box.
2027
- * - `backend: 'bridge'` → OFF-BOX: a local cli-bridge fronting a harness CLI
2028
- * (opencode / kimi-code / …) as the leaf executor, wired
2029
- * through the resumable `bridgeExecutor`.
2030
- * - `backend: 'router'` → OFF-BOX: a router chat-completion as the leaf executor,
2031
- * presented as a `SandboxClient` (no sandbox dependency).
2032
- * - `backend: 'local'` → SAME-HOST: a router-brain tool loop with the
2033
- * profile's stdio MCP servers spawned as LOCAL child
2034
- * processes — the only backend that can reach an MCP
2035
- * server built into a host worktree.
2036
- */
2037
- /**
2038
- * Resolve a `SandboxClient` for the chosen backend. The generic, dep-light core
2039
- * that `resolveBenchClient` builds on — reuse this instead of hand-rolling the
2040
- * `createExecutor`/`inlineSandboxClient` branch in each product.
2041
- */
2042
- function resolveSandboxClient(opts) {
2043
- switch (opts.backend) {
2044
- case "sandbox":
2045
- if (!opts.sandboxClient) throw new Error("resolveSandboxClient: backend 'sandbox' requires opts.sandboxClient");
2046
- return opts.sandboxClient;
2047
- case "bridge": {
2048
- const bridge = opts.bridge;
2049
- if (!bridge?.bearer) throw new Error("resolveSandboxClient: backend 'bridge' requires bridge.bearer");
2050
- return inlineSandboxClient(createExecutor({
2051
- backend: "bridge",
2052
- bridgeUrl: bridge.url ?? "http://127.0.0.1:3355",
2053
- bridgeBearer: bridge.bearer,
2054
- timeoutMs: bridge.timeoutMs
2055
- }));
2056
- }
2057
- case "router": {
2058
- const router = opts.router;
2059
- if (!router?.baseUrl || !router.key) throw new Error("resolveSandboxClient: backend 'router' requires router.baseUrl and router.key");
2060
- return inlineSandboxClient(createExecutor({
2061
- backend: "router",
2062
- routerBaseUrl: router.baseUrl,
2063
- routerKey: router.key
2064
- }));
2065
- }
2066
- case "local": {
2067
- const local = opts.local;
2068
- if (!local?.router?.baseUrl || !local.router.key) throw new Error("resolveSandboxClient: backend 'local' requires local.router.baseUrl and local.router.key");
2069
- return localSandboxClient(local);
2070
- }
2071
- }
2486
+ function loopCostReceipt(result, model) {
2487
+ return costReceiptFromUsage(result.tokenUsage, model, {
2488
+ usageUnknown: result.tokenUsage.tokensKnown === false,
2489
+ actualCostUsd: result.costUsdKnown !== false ? result.costUsd : void 0,
2490
+ costUnknown: result.costUsdKnown === false,
2491
+ estimatedCostUsd: result.estimatedCostUsd
2492
+ });
2072
2493
  }
2073
- //#endregion
2074
- //#region src/runtime/define-leaderboard.ts
2075
- /**
2076
- * `defineLeaderboard` — the declarative eval-leaderboard facade.
2077
- *
2078
- * A product's harness×model leaderboard is always the same assembly: expand a
2079
- * base profile across the harness×model axes (`expandProfileAxes`), run every
2080
- * (profile, case) cell as a driven loop (`loopDispatch` + the naive retry driver), score
2081
- * with the domain's grader, and emit ONE `runProfileMatrix` call. Each product
2082
- * hand-rolled that assembly (~650 lines each) and re-hit the same footguns:
2083
- * stale cell-cache reuse, zero-token stub cells, missing model snapshots.
2084
- *
2085
- * This facade IS that assembly, once, with the domain reduced to a declarative
2086
- * spec: `cases` + `prompt` + `score`. It contains NO execution, judging, or
2087
- * metering logic of its own — every moving part is an existing primitive, and
2088
- * every default is overridable:
2089
- *
2090
- * - LEVEL 0 (declarative): `cases` / `prompt` / `score` / `axis`.
2091
- * - LEVEL 1 (seams): `backends`, `flags`, `parseOutput`, `onCellEvents`,
2092
- * `resolveModel`, `setup`/`teardown`, `export`, `matrix`
2093
- * passthrough.
2094
- * - LEVEL 2 (replacement): `dispatch` and `judges` swap out the whole
2095
- * loop wiring or scoring; `runProfileMatrix` itself stays public as the
2096
- * escape floor — a product overriding everything just writes what it has
2097
- * today, no capability removed.
2098
- *
2099
- * `toBenchmarkAdapter()` exposes the same domain surface in the structural
2100
- * `BenchmarkAdapter` shape (`name`/`preflight`/`loadTasks`/`judge`/
2101
- * `goldArtifact`) so a product leaderboard can register into a benchmark
2102
- * registry without this module depending on one.
2103
- *
2104
- * @experimental
2105
- */
2106
- /**
2107
- * The no-signal retry floor as a bare `Driver` literal: re-run the case VERBATIM as an
2108
- * independent attempt until one shot scores (`verdict.valid`) or the shot cap. It reads ONLY
2109
- * `verdict.valid` — never `notes`/`scores` — so a leaderboard cell's retries carry no grader
2110
- * findings into the next shot (the leak-free firewall). Steering POLICY texts are registry data
2111
- * (`delegates/naive-continuation` and siblings in the kernel prompt registry) attached to graph
2112
- * edges; this loop-kernel control folds no text in — each shot is the same task object, and the
2113
- * per-shot nonce in `taskToPrompt` keeps the attempts independent at the router.
2114
- */
2115
- function naiveRetryDriver(shots) {
2494
+ /** Map Runtime usage into Eval's canonical paid-call receipt without inventing cache classes. */
2495
+ function costReceiptFromUsage(usage, model, cost) {
2496
+ const cacheComplete = hasCompleteCacheBreakdown(usage) && (usage.freshInput !== void 0 || usage.cacheRead !== void 0 || usage.cacheWrite !== void 0);
2497
+ const cacheRead = cacheComplete ? usage.cacheRead ?? 0 : void 0;
2498
+ const cacheWrite = cacheComplete ? usage.cacheWrite ?? 0 : void 0;
2499
+ const classified = (cacheRead ?? 0) + (cacheWrite ?? 0);
2116
2500
  return {
2117
- name: "naive",
2118
- plan(task, history) {
2119
- if (history.length === 0) return Promise.resolve([task]);
2120
- if (history[history.length - 1]?.verdict?.valid === true) return Promise.resolve([]);
2121
- if (history.length >= shots) return Promise.resolve([]);
2122
- return Promise.resolve([task]);
2123
- },
2124
- decide(history) {
2125
- if (history.some((it) => it.verdict?.valid)) return "pick-winner";
2126
- return history.length < shots ? "refine" : "fail";
2127
- },
2128
- describePlan() {
2129
- return {
2130
- kind: "refine",
2131
- rationale: "naive fixed continuation (no grade signal)"
2132
- };
2133
- }
2501
+ model,
2502
+ inputTokens: cacheComplete ? usage.input - classified : usage.input,
2503
+ outputTokens: usage.output,
2504
+ ...cacheComplete ? {
2505
+ cachedTokens: cacheRead,
2506
+ cacheWriteTokens: cacheWrite
2507
+ } : {},
2508
+ ...usage.tokensKnown === false || cost.usageUnknown ? { usageUnknown: true } : {},
2509
+ ...cost.actualCostUsd !== void 0 ? { actualCostUsd: cost.actualCostUsd } : {},
2510
+ ...cost.costUnknown ? { costUnknown: true } : {},
2511
+ ...cost.estimatedCostUsd !== void 0 ? { estimatedCostUsd: cost.estimatedCostUsd } : {}
2134
2512
  };
2135
2513
  }
2136
- /** Read `--name <value>` from an argv array. */
2137
- function argOf(argv, name) {
2138
- const i = argv.indexOf(`--${name}`);
2139
- if (i >= 0 && i + 1 < argv.length) return argv[i + 1];
2140
- }
2141
- function splitList(v) {
2142
- if (v === void 0) return void 0;
2143
- const parts = v.split(",").map((s) => s.trim()).filter(Boolean);
2144
- return parts.length > 0 ? parts : void 0;
2145
- }
2146
- /** RunRecords reject a bare model id — the eval IDENTITY model must carry a
2147
- * snapshot (`name@<snapshot>`). Unchanged when already stamped. */
2148
- function withSnapshot(model, snapshot) {
2149
- return model.includes("@") ? model : `${model}@${snapshot}`;
2150
- }
2151
- function gitSha() {
2152
- try {
2153
- return execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim();
2154
- } catch {
2155
- return "unknown";
2156
- }
2157
- }
2158
- function normalizeScore(s) {
2159
- return typeof s === "number" ? { composite: s } : s;
2160
- }
2161
- /**
2162
- * Assemble a declarative spec (`cases` + `prompt` + `score`) into a runnable
2163
- * harness×model leaderboard — `run()` executes the matrix, `toBenchmarkAdapter()`
2164
- * exposes the same domain as a structural `BenchmarkAdapter`.
2165
- */
2166
- function defineLeaderboard(spec) {
2167
- const caseId = (c) => {
2168
- const id = spec.caseId ? spec.caseId(c) : c.id;
2169
- if (typeof id !== "string" || id.length === 0) throw new Error(`defineLeaderboard(${spec.name}): every case needs a stable string id — give cases an \`id\` property or supply spec.caseId`);
2170
- return id;
2171
- };
2172
- const selectCases = (ids) => {
2173
- if (!ids) return spec.cases;
2174
- const byId = new Map(spec.cases.map((c) => [caseId(c), c]));
2175
- return ids.map((id) => {
2176
- const c = byId.get(id);
2177
- if (c === void 0) throw new Error(`defineLeaderboard(${spec.name}): unknown case "${id}" (have: ${[...byId.keys()].join(", ")})`);
2178
- return c;
2179
- });
2514
+ /** Eval has one pre-admitted paid call for this dispatch. A tree can settle that call only when
2515
+ * Runtime has provider-served identity for every inference attempt that actually started. Plan
2516
+ * materialization is deliberately ignored: it describes what Runtime requested, not what the
2517
+ * provider served. Every visible terminal child must carry provider attempt evidence; a missing
2518
+ * evidence record stays unknown, including zero-valued or incomplete spend.
2519
+ * @internal Testable identity projection used by superviseDispatch. */
2520
+ function supervisedTreeModelForDispatch(result, rootProfile) {
2521
+ const canonicalModels = /* @__PURE__ */ new Set();
2522
+ const observedModels = [];
2523
+ const wholeTreeEvidence = result.providerModel;
2524
+ if (wholeTreeEvidence === void 0 || wholeTreeEvidence.status !== "known" && wholeTreeEvidence.reason !== "provider-model-conflict" || wholeTreeEvidence.attempts.length === 0) return { kind: "unknown" };
2525
+ const wholeTreeIdentity = collectProviderModels(wholeTreeEvidence, () => true);
2526
+ if (wholeTreeIdentity.kind !== "known") return wholeTreeIdentity;
2527
+ for (const model of wholeTreeIdentity.observed) {
2528
+ observedModels.push(model.raw);
2529
+ canonicalModels.add(model.canonical);
2530
+ }
2531
+ if (canonicalModels.size === 0) return { kind: "unknown" };
2532
+ if (canonicalModels.size !== 1) return {
2533
+ kind: "mixed",
2534
+ models: [...canonicalModels].sort()
2180
2535
  };
2181
- const scoreJudge = {
2182
- name: `${spec.name}-score`,
2183
- dimensions: [{
2184
- key: "composite",
2185
- description: `${spec.name} case score`
2186
- }],
2187
- score({ artifact, scenario }) {
2188
- const s = normalizeScore(spec.score(artifact, scenario.case));
2189
- return {
2190
- composite: s.composite,
2191
- dimensions: {
2192
- composite: s.composite,
2193
- ...s.dimensions
2194
- },
2195
- notes: s.notes ?? ""
2196
- };
2197
- }
2536
+ return {
2537
+ kind: "known",
2538
+ model: observedModels[0]
2198
2539
  };
2199
- async function run(argv = process.argv.slice(2)) {
2200
- const args = {};
2201
- for (const name of [
2202
- "backend",
2203
- "harnesses",
2204
- "models",
2205
- "cases",
2206
- "shots",
2207
- "reps",
2208
- "model-snapshot",
2209
- "run-dir",
2210
- "export-dir"
2211
- ]) args[name] = argOf(argv, name);
2212
- for (const [name, flag] of Object.entries(spec.flags ?? {})) args[name] = argOf(argv, name) ?? flag.default;
2213
- const backendName = args.backend ?? "sandbox";
2214
- const shots = Number(args.shots ?? spec.shots ?? 1);
2215
- const reps = Number(args.reps ?? spec.reps ?? 1);
2216
- const snapshot = args["model-snapshot"] ?? "leaderboard";
2217
- const runDir = args["run-dir"] ?? join(tmpdir(), `leaderboard-${spec.name}-${Date.now()}-${process.pid}`);
2218
- const exportDir = args["export-dir"] ?? join(runDir, "export");
2219
- mkdirSync(runDir, { recursive: true });
2220
- const cases = selectCases(splitList(args.cases));
2221
- if (cases.length === 0) throw new Error(`defineLeaderboard(${spec.name}): no cases to run`);
2222
- const scenarios = cases.map((c) => ({
2223
- id: caseId(c),
2224
- kind: spec.name,
2225
- case: c
2226
- }));
2227
- const harnesses = splitList(args.harnesses) ?? spec.axis?.harnesses ?? CODING_HARNESSES;
2228
- const rawModels = splitList(args.models) ?? spec.axis?.models ?? (spec.baseProfile?.model?.default !== void 0 ? [spec.baseProfile.model.default] : void 0);
2229
- if (!rawModels || rawModels.length === 0) throw new Error(`defineLeaderboard(${spec.name}): no models — pass --models, set spec.axis.models, or give spec.baseProfile a model.default`);
2230
- const models = rawModels.map((m) => withSnapshot(m, snapshot));
2231
- const profiles = expandProfileAxes({
2232
- base: spec.baseProfile,
2233
- harnesses,
2234
- models
2235
- });
2236
- for (const profile of profiles) assertExecutableAgentProfile(profile, `defineLeaderboard(${spec.name})`);
2237
- const ctx = {
2238
- name: spec.name,
2239
- backend: backendName,
2240
- runDir,
2241
- exportDir,
2242
- args,
2243
- harnesses,
2244
- models,
2245
- caseIds: scenarios.map((s) => s.id),
2246
- shots,
2247
- reps
2248
- };
2249
- const backends = {
2250
- sandbox: () => {
2251
- throw new Error(`defineLeaderboard(${spec.name}): the 'sandbox' backend needs your product's real Sandbox client — supply spec.backends.sandbox (e.g. () => new Sandbox({ apiKey, baseUrl }))`);
2252
- },
2253
- "cli-bridge": () => {
2254
- const bearer = process.env.BRIDGE_BEARER ?? process.env.CLI_BRIDGE_BEARER;
2255
- if (!bearer) throw new Error(`defineLeaderboard(${spec.name}): backend 'cli-bridge' needs BRIDGE_BEARER or CLI_BRIDGE_BEARER set`);
2256
- return resolveSandboxClient({
2257
- backend: "bridge",
2258
- bridge: {
2259
- url: process.env.CLI_BRIDGE_URL,
2260
- bearer,
2261
- timeoutMs: 9e5
2262
- }
2263
- });
2264
- },
2265
- ...spec.backends
2266
- };
2267
- const makeClient = backends[backendName];
2268
- if (!makeClient) throw new Error(`defineLeaderboard(${spec.name}): unknown backend "${backendName}" (have: ${Object.keys(backends).join(", ")})`);
2269
- const sandboxClient = makeClient();
2270
- const promptById = /* @__PURE__ */ new Map();
2271
- for (const s of scenarios) promptById.set(s.id, await spec.prompt(s.case));
2272
- const promptOf = (s) => {
2273
- const p = promptById.get(s.id);
2274
- if (p === void 0) throw new Error(`defineLeaderboard(${spec.name}): no prompt for case "${s.id}"`);
2275
- return p;
2276
- };
2277
- let shotNonce = 0;
2278
- const maximumCharge = spec.maximumCharge;
2279
- const dispatch = spec.dispatch ?? ((profile, scenario, dispatchCtx) => {
2280
- return loopDispatch({
2281
- sandboxClient,
2282
- maximumCharge: typeof maximumCharge === "function" ? (cellScenario, cellProfile) => maximumCharge(cellProfile, cellScenario) : maximumCharge,
2283
- resolveCostModel: (result, _cellScenario, cellProfile) => {
2284
- if (!spec.resolveModel) return cellProfile.model?.default;
2285
- const served = new Set(result.iterations.map((iteration) => spec.resolveModel?.(iteration.events)).filter((model) => model !== void 0));
2286
- if (served.size > 1) throw new Error(`defineLeaderboard(${spec.name}): one cell reported multiple served models: ${[...served].join(", ")}`);
2287
- return [...served][0] ?? cellProfile.model?.default;
2288
- },
2289
- toLoopOptions: (cellScenario, cellProfile) => {
2290
- return {
2291
- driver: naiveRetryDriver(shots),
2292
- agentRun: {
2293
- profile: cellProfile,
2294
- taskToPrompt: (s) => `${promptOf(s)}\n\n<!-- independent-attempt:${shotNonce++} -->`
2295
- },
2296
- output: { parse: (events) => spec.parseOutput ? spec.parseOutput(events, cellScenario.case) : collectAgentResponseText(events) ?? "" },
2297
- validator: { validate: async (output) => {
2298
- const s = normalizeScore(spec.score(output, cellScenario.case));
2299
- return {
2300
- valid: s.composite > 0,
2301
- score: s.composite
2302
- };
2303
- } },
2304
- task: cellScenario,
2305
- maxIterations: shots
2306
- };
2307
- },
2308
- toArtifact: (result) => {
2309
- for (const iter of result.iterations) spec.onCellEvents?.(iter.events, scenario.case, {
2310
- index: iter.index,
2311
- ...iter.error ? { error: iter.error.message } : {},
2312
- ...iter.verdict ? { verdict: { score: iter.verdict.score } } : {}
2313
- });
2314
- return result.winner?.output;
2315
- }
2316
- })(profile, scenario, dispatchCtx);
2317
- });
2318
- await spec.setup?.(ctx);
2319
- try {
2320
- const result = await runProfileMatrix({
2321
- profiles,
2322
- scenarios,
2323
- dispatch,
2324
- judges: spec.judges ?? [scoreJudge],
2325
- runDir,
2326
- commitSha: gitSha(),
2327
- reps,
2328
- ...spec.matrix
2329
- });
2330
- if (spec.export) await spec.export(result, ctx);
2331
- else {
2332
- mkdirSync(exportDir, { recursive: true });
2333
- writeFileSync(join(runDir, "matrix-result.json"), `${JSON.stringify(result, null, 2)}\n`);
2334
- const table = renderLeaderboardMarkdown(leaderboard(result.records, {
2335
- title: spec.name,
2336
- meta: { backend: backendName }
2337
- }));
2338
- writeFileSync(join(exportDir, "leaderboard.md"), table);
2339
- console.log(table);
2340
- }
2341
- return result;
2342
- } finally {
2343
- await spec.teardown?.(ctx);
2540
+ }
2541
+ function collectProviderModels(evidence, validate) {
2542
+ if (evidence.status !== "known" && evidence.reason !== "provider-model-conflict" || !Array.isArray(evidence.attempts) || evidence.attempts.length === 0 || !Array.isArray(evidence.models)) return { kind: "unknown" };
2543
+ const observed = [];
2544
+ const canonical = /* @__PURE__ */ new Set();
2545
+ const observedRaw = /* @__PURE__ */ new Set();
2546
+ for (const attempt of evidence.attempts) {
2547
+ if (attempt.providerDispatch === "not_started") {
2548
+ if (attempt.observations.length > 0 || attempt.identityConflict === true) return { kind: "unknown" };
2549
+ continue;
2344
2550
  }
2345
- }
2346
- function toBenchmarkAdapter() {
2347
- return {
2348
- name: spec.name,
2349
- async preflight() {
2350
- const seen = /* @__PURE__ */ new Set();
2351
- for (const c of spec.cases) {
2352
- const id = caseId(c);
2353
- if (seen.has(id)) throw new Error(`defineLeaderboard(${spec.name}): duplicate case id "${id}"`);
2354
- seen.add(id);
2355
- }
2356
- },
2357
- async loadTasks(opts) {
2358
- const selected = selectCases(opts?.ids);
2359
- const limited = opts?.limit !== void 0 ? selected.slice(0, opts.limit) : selected;
2360
- return Promise.all(limited.map(async (c) => ({
2361
- id: caseId(c),
2362
- prompt: await spec.prompt(c),
2363
- metadata: { case: c }
2364
- })));
2365
- },
2366
- async judge(task, artifact) {
2367
- const [c] = selectCases([task.id]);
2368
- if (c === void 0) throw new Error(`defineLeaderboard(${spec.name}): no case "${task.id}"`);
2369
- const s = normalizeScore(spec.score(artifact, c));
2551
+ if (attempt === null || typeof attempt !== "object" || !Array.isArray(attempt.observations) || attempt.observations.length === 0 || attempt.identityConflict === true) return { kind: "unknown" };
2552
+ let attemptBase;
2553
+ let attemptSnapshot;
2554
+ let attemptSnapshotKind;
2555
+ let representative;
2556
+ for (const model of attempt.observations) {
2557
+ if (typeof model !== "string" || !validate(model)) return { kind: "unknown" };
2558
+ const identity = canonicalObservedModelParts(model);
2559
+ if (identity === void 0) return { kind: "unknown" };
2560
+ const base = identity.base;
2561
+ const snapshot = identity.snapshot;
2562
+ if (attemptBase !== void 0 && attemptBase !== base) return {
2563
+ kind: "mixed",
2564
+ models: [.../* @__PURE__ */ new Set([...attemptBase ? [attemptBase] : [], identity.canonical])]
2565
+ };
2566
+ if (attemptSnapshot !== void 0 && snapshot !== void 0 && (attemptSnapshot !== snapshot || attemptSnapshotKind !== identity.snapshotKind)) {
2567
+ if (attemptBase === void 0 || attemptSnapshotKind === void 0) return { kind: "unknown" };
2370
2568
  return {
2371
- resolved: s.composite > 0,
2372
- score: s.composite,
2373
- detail: s.notes
2569
+ kind: "mixed",
2570
+ models: [attemptIdentityText(attemptBase, attemptSnapshot, attemptSnapshotKind), identity.canonical]
2374
2571
  };
2375
- },
2376
- async goldArtifact() {}
2377
- };
2572
+ }
2573
+ attemptBase = base;
2574
+ attemptSnapshot ??= snapshot;
2575
+ attemptSnapshotKind ??= identity.snapshotKind;
2576
+ if (snapshot !== void 0) representative = model;
2577
+ observedRaw.add(model);
2578
+ }
2579
+ if (attemptBase === void 0 || attemptSnapshot === void 0 || attemptSnapshotKind === void 0 || representative === void 0) return { kind: "unknown" };
2580
+ const attemptIdentity = attemptIdentityText(attemptBase, attemptSnapshot, attemptSnapshotKind);
2581
+ observed.push({
2582
+ raw: representative,
2583
+ canonical: attemptIdentity
2584
+ });
2585
+ canonical.add(attemptIdentity);
2586
+ }
2587
+ if (observed.length === 0) return { kind: "unknown" };
2588
+ const declaredRaw = /* @__PURE__ */ new Set();
2589
+ for (const model of evidence.models) {
2590
+ if (typeof model !== "string") return { kind: "unknown" };
2591
+ declaredRaw.add(model);
2592
+ }
2593
+ if (declaredRaw.size !== observedRaw.size || [...declaredRaw].some((model) => !observedRaw.has(model))) return { kind: "unknown" };
2594
+ return canonical.size > 1 ? {
2595
+ kind: "mixed",
2596
+ models: [...canonical].sort()
2597
+ } : {
2598
+ kind: "known",
2599
+ observed
2600
+ };
2601
+ }
2602
+ function attemptIdentityText(base, snapshot, kind) {
2603
+ return kind === "date" ? `${base}-${snapshot}` : `${base}@${snapshot}`;
2604
+ }
2605
+ /** The Eval receipt surface has no pre-admitted per-model recursive-tree receipt bundle yet. */
2606
+ var SupervisedTreeModelIdentityError = class extends Error {
2607
+ constructor(identity) {
2608
+ const detail = identity.kind === "mixed" ? `multiple models (${identity.models.join(", ")})` : "an unknown materialized model";
2609
+ super(`superviseDispatch: cannot settle one Eval paid-call receipt for a tree with ${detail}`);
2610
+ this.name = "SupervisedTreeModelIdentityError";
2378
2611
  }
2612
+ };
2613
+ function unknownSupervisedTreeReceipt() {
2379
2614
  return {
2380
- run,
2381
- toBenchmarkAdapter
2615
+ model: "unknown",
2616
+ inputTokens: 0,
2617
+ outputTokens: 0,
2618
+ usageUnknown: true,
2619
+ costUnknown: true
2620
+ };
2621
+ }
2622
+ /** Run one recursive supervised tree inside Eval's pre-execution paid-call lifecycle. */
2623
+ function superviseDispatch(opts) {
2624
+ return async (profile, scenario, ctx) => {
2625
+ const superviseOptions = opts.toSuperviseOptions(scenario, profile);
2626
+ const task = opts.toTask(scenario, profile);
2627
+ const maximumCharge = typeof opts.maximumCharge === "function" ? opts.maximumCharge(scenario, profile) : opts.maximumCharge;
2628
+ const paid = await ctx.cost.runPaidCall({
2629
+ channel: "agent",
2630
+ actor: opts.costSource ?? "supervise",
2631
+ model: "unknown",
2632
+ signal: ctx.signal,
2633
+ ...maximumCharge ? { maximumCharge } : {},
2634
+ execute: async (executionSignal) => {
2635
+ const result = await supervise(profile, task, {
2636
+ ...superviseOptions,
2637
+ signal: executionSignal
2638
+ });
2639
+ const identity = supervisedTreeModelForDispatch(result, profile);
2640
+ if (identity.kind !== "known") throw new SupervisedTreeModelIdentityError(identity);
2641
+ return result;
2642
+ },
2643
+ receipt: (result) => {
2644
+ const identity = supervisedTreeModelForDispatch(result, profile);
2645
+ if (identity.kind !== "known") throw new SupervisedTreeModelIdentityError(identity);
2646
+ return costReceiptFromUsage(result.spentTotal.tokens, identity.model, {
2647
+ usageUnknown: result.spentTotal.tokensKnown === false,
2648
+ actualCostUsd: result.spentTotal.usdKnown !== false ? result.spentTotal.usd : void 0,
2649
+ costUnknown: result.spentTotal.usdKnown === false
2650
+ });
2651
+ },
2652
+ receiptFromError: (error) => error instanceof SupervisedTreeModelIdentityError ? unknownSupervisedTreeReceipt() : void 0
2653
+ });
2654
+ if (!paid.succeeded) throw paid.error;
2655
+ return (opts.toArtifact ?? ((result) => {
2656
+ if (result.kind !== "winner") throw new Error(`superviseDispatch: supervised tree ended without a winner (${result.reason})`);
2657
+ return result.out;
2658
+ }))(paid.value);
2659
+ };
2660
+ }
2661
+ function modelFromLoopOptions(options) {
2662
+ const profiles = options.agentRun ? [options.agentRun.profile] : options.agentRuns?.map((run) => run.profile) ?? [];
2663
+ const models = new Set(profiles.map((profile) => profile.model?.default).filter((model) => !!model));
2664
+ if (models.size === 1) return [...models][0];
2665
+ return models.size > 1 ? "mixed" : "unknown";
2666
+ }
2667
+ /**
2668
+ * Adapter for plain `runCampaign` scenarios. This is the Runtime-side pair for
2669
+ * agent-eval fixture scenarios: load fixtures in `agent-eval/campaign`, build
2670
+ * the Runtime cell here, and keep paid-call admission, receipts, and traces
2671
+ * automatic.
2672
+ */
2673
+ function loopCampaignDispatch(opts) {
2674
+ return (scenario, ctx) => {
2675
+ const loopOptions = opts.toLoopOptions(scenario);
2676
+ return runLoopWithCampaignContext(opts, loopOptions, ctx, {
2677
+ model: modelFromLoopOptions(loopOptions),
2678
+ maximumCharge: typeof opts.maximumCharge === "function" ? opts.maximumCharge(scenario) : opts.maximumCharge,
2679
+ resolveModel: opts.resolveCostModel ? (result) => opts.resolveCostModel?.(result, scenario) : void 0
2680
+ });
2382
2681
  };
2383
2682
  }
2683
+ /**
2684
+ * Adapter for `runProfileMatrix` (profile is an axis). Returns a
2685
+ * `ProfileDispatchFn` that runs `runAgentRounds` per (profile, scenario) cell
2686
+ * inside Eval's paid-call lifecycle.
2687
+ */
2688
+ function loopDispatch(opts) {
2689
+ return (profile, scenario, ctx) => runLoopForCell(opts, scenario, profile, ctx);
2690
+ }
2384
2691
  //#endregion
2385
- //#region src/runtime/harvest-corpus.ts
2692
+ //#region src/runtime/inline-sandbox-client.ts
2386
2693
  /**
2387
- * Analyze completed runs through the selected observer and retain findings in a corpus.
2388
- *
2389
- * Store-agnostic by design: the caller maps its trace store's rows (a
2390
- * `ProductionTraceSink` ndjson, OTLP spans, RunRecords) to `ObserveInput` — task text,
2391
- * final output, the event trace, terminal outcome.
2392
- * The default observer reads execution evidence; an injected analysis owns its admitted sources.
2393
- *
2394
- * The nightly product job is then three lines:
2395
- * const runs = mapSinkRowsToObserveInputs(await readSink(yesterday))
2396
- * const report = await harvestCorpus({ runs, profile, executor, corpus })
2397
- * log(report) // runsObserved / findings / learned / failures
2694
+ * The ONE pseudo-box adapter: present any one-shot `Executor` (router / bridge /
2695
+ * BYO) as a `SandboxClient` so the round-synchronous `runAgentRounds` can drive it
2696
+ * without each call site re-faking a box. This is the single shell that
2697
+ * `bench/src/router-executor.ts`, generate-eval's old `bridgeSandboxClient`, and
2698
+ * the search-bench bridge transport were each re-implementing.
2398
2699
  *
2399
- * Consumers choose how retained findings inform later work and how to assess their value.
2700
+ * It is deliberately for NON-box executors only — a real sandbox harness already
2701
+ * IS a `SandboxClient` (boxes, sessions, fs, fork are real there). Here each
2702
+ * `streamPrompt` runs the executor once and emits its result plus the canonical
2703
+ * `done` event. The result carries output and usage; `done` confirms execution settled.
2704
+ * There are no sessions, no fs,
2705
+ * no fork (those degrade gracefully via the optional `SandboxClient` methods).
2400
2706
  */
2401
- /** The completed batch evidence remains available even when every analysis failed. */
2402
- var HarvestError = class extends Error {
2403
- report;
2404
- constructor(report) {
2405
- super(`harvestCorpus: every run failed analysis (${report.failures.length}) — first: ${report.failures[0]?.error}`);
2406
- this.report = report;
2407
- this.name = "HarvestError";
2707
+ function isAsyncIterable$1(v) {
2708
+ return typeof v === "object" && v !== null && Symbol.asyncIterator in v;
2709
+ }
2710
+ /** Drive a (possibly streaming) executor to its terminal artifact. */
2711
+ async function settle(exec, task, signal) {
2712
+ const r = exec.execute(task, signal);
2713
+ if (isAsyncIterable$1(r)) {
2714
+ for await (const _ of r);
2715
+ return exec.resultArtifact();
2408
2716
  }
2409
- };
2410
- /** Batch the selected observation implementation over completed runs and retain its findings. */
2411
- async function harvestCorpus(opts) {
2412
- const concurrency = opts.concurrency ?? 4;
2413
- if (!Number.isSafeInteger(concurrency) || concurrency < 1 || opts.maxRuns !== void 0 && (!Number.isSafeInteger(opts.maxRuns) || opts.maxRuns < 0)) throw new TypeError("harvest limits require positive integer concurrency and nonnegative integer maxRuns");
2414
- const report = {
2415
- runsObserved: 0,
2416
- findings: 0,
2417
- learned: 0,
2418
- failures: [],
2419
- usage: {
2420
- input: 0,
2421
- output: 0,
2422
- known: true
2423
- }
2424
- };
2425
- const iterator = Symbol.asyncIterator in Object(opts.runs) ? opts.runs[Symbol.asyncIterator]() : (async function* () {
2426
- yield* opts.runs;
2427
- })();
2428
- let consumed = 0;
2429
- let done = false;
2430
- const next = async () => {
2431
- if (done || opts.signal?.aborted || opts.maxRuns !== void 0 && consumed >= opts.maxRuns) return null;
2432
- const sequence = ++consumed;
2433
- let r;
2434
- try {
2435
- r = await iterator.next();
2436
- } catch (error) {
2437
- done = true;
2438
- report.failures.push({
2439
- runId: `source-${sequence}`,
2440
- error: `trace source: ${error instanceof Error ? error.message.slice(0, 300) : String(error)}`
2441
- });
2442
- return null;
2443
- }
2444
- if (r.done) {
2445
- done = true;
2446
- return null;
2447
- }
2717
+ return r;
2718
+ }
2719
+ /**
2720
+ * Adapt an `ExecutorFactory` into a `SandboxClient` for `runAgentRounds`. The factory is
2721
+ * instantiated fresh per `streamPrompt` (mirrors the per-spawn executor lifecycle):
2722
+ * run once on the prompt, emit the terminal result event, tear down.
2723
+ *
2724
+ * There is no box, so a per-prompt `backend` or `model` override is refused rather than dropped;
2725
+ * other per-prompt options (`timeoutMs`, `context`) are accepted and ignored.
2726
+ */
2727
+ function inlineSandboxClient(factory, defaults = {}) {
2728
+ const capturedDefaultProfile = defaults.profile === void 0 ? void 0 : agentProfileSchema.parse(structuredClone(defaults.profile));
2729
+ let seq = 0;
2730
+ return { async create(options) {
2731
+ const id = `inline-${seq++}`;
2732
+ const createOptions = options;
2448
2733
  return {
2449
- input: r.value,
2450
- sequence
2451
- };
2452
- };
2453
- const workers = Array.from({ length: concurrency }, async () => {
2454
- for (let run = await next(); run !== null; run = await next()) {
2455
- const { input, sequence } = run;
2456
- if (opts.signal?.aborted) return;
2457
- try {
2458
- const obs = await observe(input, opts);
2459
- report.runsObserved += 1;
2460
- report.findings += obs.findings.length;
2461
- report.learned += obs.learned.length;
2462
- report.usage.input += obs.usage.input;
2463
- report.usage.output += obs.usage.output;
2464
- report.usage.known &&= obs.usage.known;
2465
- } catch (e) {
2466
- report.usage.known &&= e instanceof ObservationError && e.usage.known;
2467
- if (e instanceof ObservationError) {
2468
- report.usage.input += e.usage.input;
2469
- report.usage.output += e.usage.output;
2734
+ id,
2735
+ async *streamPrompt(message, opts) {
2736
+ assertBoxlessPromptOptions(opts, "inlineSandboxClient");
2737
+ const controller = new AbortController();
2738
+ const callerSignal = opts?.signal;
2739
+ const onAbort = () => controller.abort(callerSignal?.reason ?? /* @__PURE__ */ new Error("prompt aborted"));
2740
+ if (callerSignal) if (callerSignal.aborted) onAbort();
2741
+ else callerSignal.addEventListener("abort", onAbort, { once: true });
2742
+ const requestedProfile = (options?.backend && typeof options.backend === "object" ? options.backend.profile : void 0) ?? capturedDefaultProfile;
2743
+ const parsedProfile = agentProfileSchema.safeParse(requestedProfile);
2744
+ if (!parsedProfile.success) throw new Error("inlineSandboxClient: an exact AgentProfile is required; pass defaults.profile or create({ backend: { profile } })");
2745
+ const exec = factory({
2746
+ profile: parsedProfile.data,
2747
+ harness: null
2748
+ }, {
2749
+ signal: controller.signal,
2750
+ seams: { createOptions }
2751
+ });
2752
+ try {
2753
+ const artifact = await settle(exec, message, controller.signal);
2754
+ const out = artifact.out;
2755
+ const tokensIn = artifact.spent.tokens.input;
2756
+ const tokensOut = artifact.spent.tokens.output;
2757
+ const costUsd = artifact.spent.usd;
2758
+ const estimatedCostUsd = out?.estimatedCostUsd;
2759
+ if (artifact.spent.iterations > 0 || artifact.spent.tokensKnown === false || artifact.spent.usdKnown === false || tokensIn > 0 || tokensOut > 0 || costUsd > 0 || estimatedCostUsd !== void 0) yield {
2760
+ type: "llm_call",
2761
+ data: {
2762
+ ...artifact.spent.tokensKnown === false ? {} : {
2763
+ tokensIn,
2764
+ tokensOut
2765
+ },
2766
+ ...artifact.spent.usdKnown !== false ? { costUsd } : {},
2767
+ ...artifact.spent.tokensKnown === false ? { tokensKnown: false } : {},
2768
+ ...artifact.spent.usdKnown === false ? { costKnown: false } : {},
2769
+ ...artifact.spent.usdKnown !== false ? {
2770
+ costKnown: true,
2771
+ costProvenance: "provider-receipt"
2772
+ } : {},
2773
+ ...estimatedCostUsd !== void 0 ? { estimatedCostUsd } : {},
2774
+ ...out?.promptCache ? { promptCache: out.promptCache } : {}
2775
+ }
2776
+ };
2777
+ yield {
2778
+ type: "result",
2779
+ data: {
2780
+ finalText: out?.content ?? "",
2781
+ ...artifact.spent.tokensKnown === false ? { tokensKnown: false } : { tokenUsage: {
2782
+ inputTokens: tokensIn,
2783
+ outputTokens: tokensOut
2784
+ } },
2785
+ ...artifact.spent.usdKnown === false ? {
2786
+ costKnown: false,
2787
+ ...costUsd > 0 ? { costUsd } : {}
2788
+ } : {
2789
+ costUsd,
2790
+ costKnown: true,
2791
+ costProvenance: "provider-receipt"
2792
+ },
2793
+ ...estimatedCostUsd !== void 0 ? { estimatedCostUsd } : {},
2794
+ ...out?.promptCache ? { promptCache: out.promptCache } : {}
2795
+ }
2796
+ };
2797
+ yield {
2798
+ type: "done",
2799
+ data: { outcome: { type: "completed" } }
2800
+ };
2801
+ } finally {
2802
+ callerSignal?.removeEventListener("abort", onAbort);
2803
+ await exec.teardown("brutalKill").catch(() => {});
2470
2804
  }
2471
- report.failures.push({
2472
- runId: input.runId ?? `run-${sequence}`,
2473
- error: e instanceof Error ? e.message.slice(0, 300) : String(e)
2474
- });
2475
- }
2476
- }
2477
- });
2478
- await Promise.all(workers);
2479
- if (report.runsObserved === 0 && report.failures.length > 0) throw new HarvestError(report);
2480
- return report;
2805
+ },
2806
+ async delete() {}
2807
+ };
2808
+ } };
2481
2809
  }
2482
2810
  //#endregion
2483
- //#region src/runtime/in-process-sandbox-client.ts
2811
+ //#region src/runtime/local-sandbox-client.ts
2484
2812
  /**
2485
- * The ONE in-process pseudo-box: present a plain user callback as a
2486
- * `SandboxClient` so an example or test can drive `runAgentRounds` / `openSandboxRun`
2487
- * with zero credentials and zero network — without re-faking a box at each call
2488
- * site via `as unknown as SandboxInstance`.
2813
+ * `localSandboxClient` — the SAME-HOST pseudo-box: a `SandboxClient` whose
2814
+ * `create()` MATERIALIZES the profile's stdio MCP servers as local child
2815
+ * processes (`materializeLocalMcp`) and whose `streamPrompt` drives a real
2816
+ * tool loop (`runBrainLoop` over the router brain) with those live tools.
2489
2817
  *
2490
- * It is the structural sibling of {@link inlineSandboxClient}: that one adapts an
2491
- * `Executor` (router / bridge / BYO) into a box; this one adapts a single
2492
- * `onPrompt(prompt, round)` callback into a box. Both exist because
2493
- * `SandboxInstance` is a `declare class` with private fields — a plain object
2494
- * literal can NEVER structurally satisfy it, so every offline seam was forced to
2495
- * write `as unknown as SandboxInstance`. The one unavoidable cast now lives HERE,
2496
- * documented and tested, instead of in every example.
2818
+ * Despite the interface name, this does NOT isolate processes. It is only for
2819
+ * author-controlled profiles whose local MCP commands a caller explicitly
2820
+ * trusts; user- or model-authored code belongs in a real sandbox. `delete()`
2821
+ * kills the children.
2497
2822
  *
2498
- * What the box implements is exactly the subset `runAgentRounds` + `openSandboxRun`
2499
- * actually call on a box:
2500
- * - `id` — stable per box, used for trace correlation.
2501
- * - `streamPrompt(prompt, opts?)` — runs `onPrompt` and streams its events.
2502
- * - `fs.read` / `fs.write` — over the optional real `workdir` (the deliverable
2503
- * artifact + any seeded fixtures live there). Present only when a `workdir`
2504
- * is given.
2505
- * - `exec(command)` — runs the command in `workdir`. Present only when a
2506
- * `workdir` is given.
2507
- * - `delete()` — removes the per-box temp dir (when one was minted) and is a
2508
- * no-op otherwise; `deleteBoxSafe` calls it at teardown.
2823
+ * A per-create profile may change the prompt surface. Permission to start a
2824
+ * local MCP process applies only when its full canonical bytes match the fixed
2825
+ * constructor profile; a different generated profile is refused.
2509
2826
  *
2510
- * The `SandboxEvent` shape the callback returns — `{ type, data }` — is the
2511
- * runtime metering protocol the kernel reads: emit a flat `llm_call`
2512
- * (`{ tokensIn, tokensOut, costUsd }`) or a terminal `done`
2513
- * (`{ tokenUsage, totalCostUsd }`) / `result` event and the kernel's cost ledger
2514
- * picks it up. No cast is needed on the events (the union is structural); only
2515
- * the box object is cast, once, inside this file.
2827
+ * Event protocol matches `inlineSandboxClient`: known token usage is emitted as one `llm_call`;
2828
+ * Router catalog cost remains a separately-labelled estimate, never billed spend.
2516
2829
  */
2517
- function isAsyncIterable(v) {
2518
- return typeof v === "object" && v !== null && Symbol.asyncIterator in v;
2519
- }
2520
- /**
2521
- * Adapt a single `onPrompt(prompt, ctx)` callback into a `SandboxClient` for
2522
- * `runAgentRounds` / `openSandboxRun`. Returns a PROPERLY-TYPED `SandboxClient`: the
2523
- * lone `SandboxInstance` cast (object literal → `declare class`) lives inside
2524
- * this function, so call sites stay cast-free.
2525
- *
2526
- * There is no box, so a per-prompt `backend` override is refused rather than dropped. A per-turn
2527
- * `model` IS delivered, because `onPrompt` receives the verbatim options and is free to honor it.
2528
- * Other per-prompt options (`timeoutMs`, `context`) are accepted and ignored.
2830
+ /** A same-host `SandboxClient` adapter with no process isolation. Local MCP is
2831
+ * refused unless the caller explicitly supplies a policy that allows it.
2529
2832
  *
2530
- * @experimental
2531
- */
2532
- function inProcessSandboxClient(options) {
2533
- const { onPrompt, workdir: workdirPrefix, id: idOption } = options;
2833
+ * There is no box, so a per-prompt `backend` or `model` override is refused rather than dropped;
2834
+ * other per-prompt options (`timeoutMs`, `context`) are accepted and ignored. */
2835
+ function localSandboxClient(opts) {
2836
+ const defaultProfile = opts.profile === void 0 ? void 0 : executableAgentProfileSnapshot(opts.profile, "localSandboxClient default profile");
2837
+ const router = Object.freeze({ ...opts.router });
2838
+ if (opts.profileSecurityPolicy?.allowLocalMcp && defaultProfile === void 0) throw new ValidationError("localSandboxClient: allowLocalMcp requires a fixed author-controlled profile; dynamic profiles need a real sandbox");
2839
+ const trustedProfileDigest = opts.profileSecurityPolicy?.allowLocalMcp && defaultProfile !== void 0 ? canonicalAgentProfileDigest(defaultProfile) : void 0;
2534
2840
  let seq = 0;
2535
- return { async create(_options) {
2536
- const current = seq++;
2537
- const id = typeof idOption === "function" ? idOption(current) : idOption ?? `in-process-${current}`;
2538
- const boxWorkdir = workdirPrefix !== void 0 ? mkdtempSync(join(tmpdir(), `${workdirPrefix}-`)) : void 0;
2539
- let round = 0;
2540
- const fsMembers = boxWorkdir !== void 0 ? {
2541
- fs: {
2542
- async read(path) {
2543
- return readFile(join(boxWorkdir, path), "utf8");
2544
- },
2545
- async write(path, content) {
2546
- const abs = join(boxWorkdir, path);
2547
- await mkdir(dirname(abs), { recursive: true });
2548
- await writeFile(abs, content, "utf8");
2549
- }
2550
- },
2551
- async exec(command) {
2552
- const { exec: execCb } = await import("node:child_process");
2553
- const { promisify } = await import("node:util");
2554
- const execAsync = promisify(execCb);
2555
- try {
2556
- const { stdout, stderr } = await execAsync(command, {
2557
- cwd: boxWorkdir,
2558
- timeout: 3e4
2559
- });
2560
- return {
2561
- exitCode: 0,
2562
- stdout,
2563
- stderr
2564
- };
2565
- } catch (err) {
2566
- const e = err;
2567
- return {
2568
- exitCode: e.code ?? 1,
2569
- stdout: e.stdout ?? "",
2570
- stderr: e.stderr ?? e.message ?? ""
2571
- };
2572
- }
2573
- }
2574
- } : {};
2575
- async function* drive(behavior, message, opts) {
2576
- const prompt = typeof message === "string" ? message : JSON.stringify(message);
2577
- const { signal: optSignal, ...rest } = opts ?? {};
2578
- const signal = optSignal ?? new AbortController().signal;
2579
- const ctx = {
2580
- round,
2581
- workdir: boxWorkdir,
2582
- signal,
2583
- ...Object.keys(rest).length > 0 ? { options: rest } : {}
2584
- };
2585
- round += 1;
2586
- const produced = await behavior(prompt, ctx);
2587
- if (isAsyncIterable(produced)) for await (const ev of produced) yield ev;
2588
- else for (const ev of produced) yield ev;
2589
- }
2841
+ return { async create(options) {
2842
+ const profile = executableAgentProfileSnapshot((options?.backend)?.profile ?? defaultProfile, "localSandboxClient");
2843
+ const model = profileProviderModel(profile);
2844
+ const settings = profileModelExecutionSettings(profile, "localSandboxClient");
2845
+ const policyApplies = opts.profileSecurityPolicy !== void 0 && (!opts.profileSecurityPolicy.allowLocalMcp || trustedProfileDigest !== void 0 && canonicalAgentProfileDigest(profile) === trustedProfileDigest);
2846
+ const mcp = await materializeLocalMcp(profile, {
2847
+ ...opts.keys ? { keys: opts.keys } : {},
2848
+ ...policyApplies ? { profileSecurityPolicy: opts.profileSecurityPolicy } : {}
2849
+ });
2850
+ const brain = routerBrain({
2851
+ routerBaseUrl: router.baseUrl,
2852
+ routerKey: router.key,
2853
+ model,
2854
+ ...settings.retry !== void 0 ? { retry: settings.retry } : {},
2855
+ ...enforceTokenLimits(settings.tokenLimits, "router", "localSandboxClient").applied,
2856
+ ...settings.stream !== void 0 ? { stream: settings.stream } : {}
2857
+ }, {
2858
+ ...settings.temperature !== void 0 ? { temperature: settings.temperature } : {},
2859
+ ...settings.seed !== void 0 ? { seed: settings.seed } : {},
2860
+ ...settings.toolChoice !== void 0 ? { toolChoice: settings.toolChoice } : {},
2861
+ ...settings.extraBody !== void 0 ? { extraBody: settings.extraBody } : {},
2862
+ ...profile.model?.reasoningEffort ? { reasoningEffort: profile.model.reasoningEffort } : {}
2863
+ });
2864
+ const system = [profile.prompt?.systemPrompt, ...profile.prompt?.instructions ?? []].filter((s) => typeof s === "string" && s.trim().length > 0).join("\n\n");
2590
2865
  return {
2591
- id,
2592
- streamPrompt(message, opts) {
2593
- assertBoxlessPromptOptions(opts, "inProcessSandboxClient", ["backend"]);
2594
- return drive(onPrompt, message, opts);
2595
- },
2596
- ...fsMembers,
2597
- async delete() {
2598
- if (boxWorkdir !== void 0) rmSync(boxWorkdir, {
2599
- recursive: true,
2600
- force: true
2866
+ id: `local-${seq++}`,
2867
+ async *streamPrompt(message, popts) {
2868
+ assertBoxlessPromptOptions(popts, "localSandboxClient");
2869
+ let estimatedCostUsd = 0;
2870
+ let sawEstimatedCost = false;
2871
+ const chat = async (messages, tools) => {
2872
+ const r = await brain(messages, tools);
2873
+ if (r.costProvenance === "catalog-estimate" && r.costUsd !== void 0) {
2874
+ estimatedCostUsd += r.costUsd;
2875
+ sawEstimatedCost = true;
2876
+ }
2877
+ return r;
2878
+ };
2879
+ const r = await runBrainLoop({
2880
+ chat,
2881
+ tools: mcp.tools,
2882
+ execute: (name, args) => mcp.call(name, args),
2883
+ initialMessages: [...system ? [{
2884
+ role: "system",
2885
+ content: system
2886
+ }] : [], {
2887
+ role: "user",
2888
+ content: message
2889
+ }],
2890
+ maxTurns: settings.maxTurns ?? 0,
2891
+ hooks: { stopBefore: () => popts?.signal?.aborted === true }
2601
2892
  });
2893
+ if (r.turns > 0) yield {
2894
+ type: "llm_call",
2895
+ data: {
2896
+ model,
2897
+ ...r.tokensKnown === false ? { tokensKnown: false } : {
2898
+ tokensIn: r.usage.input,
2899
+ tokensOut: r.usage.output
2900
+ },
2901
+ costKnown: false,
2902
+ ...sawEstimatedCost ? { estimatedCostUsd } : {}
2903
+ }
2904
+ };
2905
+ yield {
2906
+ type: "result",
2907
+ data: {
2908
+ finalText: r.final,
2909
+ ...r.tokensKnown === false ? { tokensKnown: false } : { tokenUsage: {
2910
+ inputTokens: r.usage.input,
2911
+ outputTokens: r.usage.output
2912
+ } },
2913
+ costKnown: false,
2914
+ ...sawEstimatedCost ? { estimatedCostUsd } : {}
2915
+ }
2916
+ };
2917
+ },
2918
+ async delete() {
2919
+ await mcp.close();
2602
2920
  }
2603
2921
  };
2604
2922
  } };
2605
2923
  }
2606
2924
  //#endregion
2607
- //#region src/runtime/isolated-checker.ts
2608
- const toolchains = [
2609
- "/usr",
2610
- "/bin",
2611
- "/lib",
2612
- "/lib64"
2613
- ];
2614
- const contains = (parent, child) => {
2615
- const path = relative(parent, child);
2616
- return path === "" || path !== ".." && !path.startsWith(`..${sep}`) && !path.startsWith(sep);
2617
- };
2925
+ //#region src/runtime/resolve-sandbox-client.ts
2618
2926
  /**
2619
- * Run an untrusted check in Linux Bubblewrap, or in its own Sandbox box when `box` is set.
2620
- * Never falls back to host execution.
2927
+ * The product-facing backend selector: one call picks the execution transport a
2928
+ * `runAgentRounds` (or any consumer that drives a `SandboxClient`) runs on, from the
2929
+ * package a product already depends on. It is pure sugar over the existing
2930
+ * primitives — `createExecutor({ backend })` + `inlineSandboxClient` — and adds
2931
+ * ZERO transport code (the node:http cli-bridge transport lives in `bridgeExecutor`).
2621
2932
  *
2622
- * Bubblewrap requires /usr/bin/bwrap and permission to create Linux namespaces.
2623
- * Only trusted system toolchains, private proc/dev/tmp, and the writable copy are mounted.
2624
- * The canonical input path remains the working directory; copy writes are discarded.
2625
- * Limits bound command time and captured output, not copy size or memory consumption.
2626
- * Callers must keep the input and trusted toolchains stable while preparing the check.
2933
+ * The heavy `resolveBenchClient` in `@tangle-network/agent-bench` is the bench-side
2934
+ * sibling: it layers router/search specifics a product should not have to depend on.
2935
+ * This is the generic, dep-light core products call to switch backends:
2627
2936
  *
2628
- * A box check creates one fresh box with no owner secrets and blocked egress, delivers the
2629
- * tree's regular files verified by sha256, runs the command there, and deletes the box.
2937
+ * - `backend: 'sandbox'` → IN-BOX: return the caller's real Sandbox-backed
2938
+ * `sandboxClient` unchanged. Fail loud if absent — a
2939
+ * product on the sandbox backend already holds its box.
2940
+ * - `backend: 'bridge'` → OFF-BOX: a local cli-bridge fronting a harness CLI
2941
+ * (opencode / kimi-code / …) as the leaf executor, wired
2942
+ * through the resumable `bridgeExecutor`.
2943
+ * - `backend: 'router'` → OFF-BOX: a router chat-completion as the leaf executor,
2944
+ * presented as a `SandboxClient` (no sandbox dependency).
2945
+ * - `backend: 'local'` → SAME-HOST: a router-brain tool loop with the
2946
+ * profile's stdio MCP servers spawned as LOCAL child
2947
+ * processes — the only backend that can reach an MCP
2948
+ * server built into a host worktree.
2630
2949
  */
2631
- async function runIsolatedCheck(options) {
2632
- if (options.box) return runInBox(options, options.box);
2633
- let scratch;
2634
- let result;
2635
- try {
2636
- if (process.platform !== "linux") throw new Error("Linux Bubblewrap namespaces are required");
2637
- const { timeoutMs, maxOutputBytes } = checkLimits(options);
2638
- if (options.signal?.aborted) return {
2639
- succeeded: false,
2640
- reason: "cancelled",
2641
- diagnostic: "Check cancelled"
2642
- };
2643
- const { workspace, tree } = await protectedTree(options);
2644
- const binds = [];
2645
- for (const path of toolchains) {
2646
- let actual;
2647
- try {
2648
- actual = await realpath(path);
2649
- } catch (error) {
2650
- if (error.code === "ENOENT") continue;
2651
- throw error;
2652
- }
2653
- for (const mount of [path, actual]) if (contains(mount, workspace) || contains(workspace, mount) || contains(mount, tree) || contains(tree, mount)) throw new Error(`Toolchain overlaps protected workspace: ${path}`);
2654
- binds.push("--ro-bind", actual, path);
2655
- }
2656
- if (["/proc", "/dev"].some((path) => contains(path, tree) || contains(tree, path))) throw new Error("Tree overlaps a reserved namespace mount");
2657
- scratch = await mkdtemp(join(tmpdir(), "runtime-check-"));
2658
- if (contains(tree, scratch)) throw new Error("Temporary storage must be outside the input tree");
2659
- const copy = join(scratch, "tree");
2660
- await cp(tree, copy, {
2661
- recursive: true,
2662
- dereference: false,
2663
- verbatimSymlinks: true
2664
- });
2665
- result = await execute([
2666
- "--unshare-all",
2667
- "--die-with-parent",
2668
- "--new-session",
2669
- "--cap-drop",
2670
- "ALL",
2671
- "--clearenv",
2672
- "--setenv",
2673
- "PATH",
2674
- "/usr/bin:/bin",
2675
- "--setenv",
2676
- "HOME",
2677
- "/tmp",
2678
- "--setenv",
2679
- "LANG",
2680
- "C",
2681
- ...binds,
2682
- "--proc",
2683
- "/proc",
2684
- "--dev",
2685
- "/dev",
2686
- "--tmpfs",
2687
- "/tmp",
2688
- "--bind",
2689
- copy,
2690
- tree,
2691
- "--chdir",
2692
- tree,
2693
- "--",
2694
- ...options.command
2695
- ], timeoutMs, maxOutputBytes, options.signal);
2696
- } catch (error) {
2697
- result = {
2698
- succeeded: false,
2699
- reason: "refused",
2700
- diagnostic: String(error)
2701
- };
2702
- }
2703
- if (scratch) try {
2704
- await removeScratch(scratch);
2705
- } catch (error) {
2706
- const cleanupDiagnostic = String(error);
2707
- result = result.succeeded ? {
2708
- succeeded: false,
2709
- reason: "cleanup-failed",
2710
- diagnostic: cleanupDiagnostic,
2711
- stdout: result.value.stdout,
2712
- stderr: result.value.stderr,
2713
- exitCode: 0
2714
- } : {
2715
- ...result,
2716
- cleanupDiagnostic
2717
- };
2718
- }
2719
- return result;
2720
- }
2721
- function checkLimits(options) {
2722
- const timeoutMs = options.timeoutMs ?? 3e4;
2723
- const maxOutputBytes = options.maxOutputBytes ?? 1048576;
2724
- if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 2147483647 || !Number.isSafeInteger(maxOutputBytes) || maxOutputBytes < 1) throw new Error("Invalid execution limits");
2725
- if (!options.command.length || !options.command[0]) throw new Error("A command is required");
2726
- return {
2727
- timeoutMs,
2728
- maxOutputBytes
2729
- };
2730
- }
2731
- async function protectedTree(options) {
2732
- const workspace = await realpath(options.workspaceRoot);
2733
- const tree = await realpath(options.tree);
2734
- if (!contains(workspace, tree) || !(await stat(tree)).isDirectory()) throw new Error("Tree must be a directory inside the workspace");
2735
- return {
2736
- workspace,
2737
- tree
2738
- };
2739
- }
2740
- /** Box-workspace-relative directory that receives the tree and is the command's cwd. */
2741
- const BOX_TREE = "tree";
2742
- /** Bound on provisioning, delivery, and deletion; the box expires even if this process dies. */
2743
- const BOX_SETUP_SECONDS = 900;
2744
- /** The box's own process timeout is a backstop behind this module's deadline. */
2745
- const BOX_PROCESS_BACKSTOP_MS = 3e4;
2746
- /** Concurrent file deliveries into one check box. */
2747
- const BOX_UPLOADS = 4;
2748
- /** The box's process supervisor prefixes this to stderr when it cannot start the executable. */
2749
- const BOX_EXEC_FAILURE = "process-supervisor: exec failed";
2750
- async function runInBox(options, placement) {
2751
- let box;
2752
- let evidence;
2753
- let result;
2754
- try {
2755
- const { timeoutMs, maxOutputBytes } = checkLimits(options);
2756
- if (options.signal?.aborted) return {
2757
- succeeded: false,
2758
- reason: "cancelled",
2759
- diagnostic: "Check cancelled"
2760
- };
2761
- const { tree } = await protectedTree(options);
2762
- const captured = await captureMaterializedWorkspace(tree);
2763
- if (placement.builderAccounts.length === 0 || placement.builderAccounts.some((id) => !id)) throw new Error("builderAccounts must name the account of every key the judged run holds");
2764
- const account = (await placement.client.getIdentity()).customerId;
2765
- if (!account) throw new Error("Sandbox returned no account for the check key");
2766
- if (placement.builderAccounts.includes(account)) throw new Error(`Check account ${account} is an account the judged run holds a key to, so the run can reach the check box`);
2767
- options.signal?.throwIfAborted();
2768
- box = await placement.client.createIsolated({
2769
- environment: placement.environment,
2770
- agent: false,
2771
- bare: false,
2772
- ephemeral: true,
2773
- egressPolicy: { mode: "blocked" },
2774
- ...placement.resources ? { resources: placement.resources } : {},
2775
- maxLifetimeSeconds: Math.ceil(timeoutMs / 1e3) + BOX_SETUP_SECONDS,
2776
- idempotencyKey: `agent-runtime-check-${randomUUID()}`
2777
- }, options.signal ? { signal: options.signal } : void 0);
2778
- const receipt = box.createReceipt();
2779
- if (receipt?.outcome !== "created" || receipt.ownerContext !== "isolated" || !Array.isArray(receipt.injectedSecrets) || receipt.injectedSecrets.length !== 0) throw new Error("Sandbox did not confirm a fresh box with no owner secrets");
2780
- const egress = (await box.egress.get()).policy;
2781
- if (egress.mode !== "blocked") throw new Error(`Check box egress is ${egress.mode}, not blocked`);
2782
- await box.fs.mkdir(BOX_TREE, { recursive: true });
2783
- const digests = new Map(captured.manifest.files.map((entry) => [entry.path, entry.sha256]));
2784
- const target = box;
2785
- const pending = [...captured.files];
2786
- const deliver = async () => {
2787
- try {
2788
- for (let file = pending.shift(); file; file = pending.shift()) {
2789
- options.signal?.throwIfAborted();
2790
- const received = await target.fs.uploadData(`${BOX_TREE}/${file.path}`, file.bytes, { mode: file.mode });
2791
- if (`sha256:${received.hash.replace(/^sha256:/, "").toLowerCase()}` !== digests.get(file.path) || received.size !== file.bytes.byteLength) throw new Error(`Check box received different bytes for ${file.path}`);
2792
- }
2793
- } catch (error) {
2794
- pending.length = 0;
2795
- throw error;
2796
- }
2797
- };
2798
- await Promise.all(Array.from({ length: BOX_UPLOADS }, deliver));
2799
- evidence = {
2800
- sandboxId: box.id,
2801
- account,
2802
- input: captured.manifest,
2803
- inputDigest: canonicalCandidateDigest(captured.manifest)
2804
- };
2805
- result = await executeInBox(box, options.command, timeoutMs, maxOutputBytes, options.signal);
2806
- } catch (error) {
2807
- result = {
2808
- succeeded: false,
2809
- reason: options.signal?.aborted ? "cancelled" : "refused",
2810
- diagnostic: String(error)
2811
- };
2812
- }
2813
- if (box) try {
2814
- await box.delete();
2815
- } catch (error) {
2816
- const cleanupDiagnostic = `Check box ${box.id} was not deleted: ${String(error)}`;
2817
- result = result.succeeded ? {
2818
- succeeded: false,
2819
- reason: "cleanup-failed",
2820
- diagnostic: cleanupDiagnostic,
2821
- stdout: result.value.stdout,
2822
- stderr: result.value.stderr,
2823
- exitCode: 0
2824
- } : {
2825
- ...result,
2826
- cleanupDiagnostic
2827
- };
2950
+ /**
2951
+ * Resolve a `SandboxClient` for the chosen backend. The generic, dep-light core
2952
+ * that `resolveBenchClient` builds on — reuse this instead of hand-rolling the
2953
+ * `createExecutor`/`inlineSandboxClient` branch in each product.
2954
+ */
2955
+ function resolveSandboxClient(opts) {
2956
+ switch (opts.backend) {
2957
+ case "sandbox":
2958
+ if (!opts.sandboxClient) throw new Error("resolveSandboxClient: backend 'sandbox' requires opts.sandboxClient");
2959
+ return opts.sandboxClient;
2960
+ case "bridge": {
2961
+ const bridge = opts.bridge;
2962
+ if (!bridge?.bearer) throw new Error("resolveSandboxClient: backend 'bridge' requires bridge.bearer");
2963
+ return inlineSandboxClient(createExecutor({
2964
+ backend: "bridge",
2965
+ bridgeUrl: bridge.url ?? "http://127.0.0.1:3355",
2966
+ bridgeBearer: bridge.bearer,
2967
+ timeoutMs: bridge.timeoutMs
2968
+ }));
2969
+ }
2970
+ case "router": {
2971
+ const router = opts.router;
2972
+ if (!router?.baseUrl || !router.key) throw new Error("resolveSandboxClient: backend 'router' requires router.baseUrl and router.key");
2973
+ return inlineSandboxClient(createExecutor({
2974
+ backend: "router",
2975
+ routerBaseUrl: router.baseUrl,
2976
+ routerKey: router.key
2977
+ }));
2978
+ }
2979
+ case "local": {
2980
+ const local = opts.local;
2981
+ if (!local?.router?.baseUrl || !local.router.key) throw new Error("resolveSandboxClient: backend 'local' requires local.router.baseUrl and local.router.key");
2982
+ return localSandboxClient(local);
2983
+ }
2828
2984
  }
2829
- return evidence ? {
2830
- ...result,
2831
- box: evidence
2832
- } : result;
2833
2985
  }
2834
- async function executeInBox(box, command, timeoutMs, limit, signal) {
2835
- const [executable, ...args] = command;
2836
- const child = await box.process.spawnExact(executable, args, {
2837
- cwd: BOX_TREE,
2838
- env: {
2839
- PATH: "/usr/bin:/bin",
2840
- HOME: "/tmp",
2841
- LANG: "C"
2986
+ //#endregion
2987
+ //#region src/runtime/define-leaderboard.ts
2988
+ /**
2989
+ * `defineLeaderboard` — the declarative eval-leaderboard facade.
2990
+ *
2991
+ * A product's harness×model leaderboard is always the same assembly: expand a
2992
+ * base profile across the harness×model axes (`expandProfileAxes`), run every
2993
+ * (profile, case) cell as a driven loop (`loopDispatch` + the naive retry driver), score
2994
+ * with the domain's grader, and emit ONE `runProfileMatrix` call. Each product
2995
+ * hand-rolled that assembly (~650 lines each) and re-hit the same footguns:
2996
+ * stale cell-cache reuse, zero-token stub cells, missing model snapshots.
2997
+ *
2998
+ * This facade IS that assembly, once, with the domain reduced to a declarative
2999
+ * spec: `cases` + `prompt` + `score`. It contains NO execution, judging, or
3000
+ * metering logic of its own — every moving part is an existing primitive, and
3001
+ * every default is overridable:
3002
+ *
3003
+ * - LEVEL 0 (declarative): `cases` / `prompt` / `score` / `axis`.
3004
+ * - LEVEL 1 (seams): `backends`, `flags`, `parseOutput`, `onCellEvents`,
3005
+ * `resolveModel`, `setup`/`teardown`, `export`, `matrix`
3006
+ * passthrough.
3007
+ * - LEVEL 2 (replacement): `dispatch` and `judges` swap out the whole
3008
+ * loop wiring or scoring; `runProfileMatrix` itself stays public as the
3009
+ * escape floor — a product overriding everything just writes what it has
3010
+ * today, no capability removed.
3011
+ *
3012
+ * `toBenchmarkAdapter()` exposes the same domain surface in the structural
3013
+ * `BenchmarkAdapter` shape (`name`/`preflight`/`loadTasks`/`judge`/
3014
+ * `goldArtifact`) so a product leaderboard can register into a benchmark
3015
+ * registry without this module depending on one.
3016
+ *
3017
+ * @experimental
3018
+ */
3019
+ /**
3020
+ * The no-signal retry floor as a bare `Driver` literal: re-run the case VERBATIM as an
3021
+ * independent attempt until one shot scores (`verdict.valid`) or the shot cap. It reads ONLY
3022
+ * `verdict.valid` — never `notes`/`scores` — so a leaderboard cell's retries carry no grader
3023
+ * findings into the next shot (the leak-free firewall). Steering POLICY texts are registry data
3024
+ * (`delegates/naive-continuation` and siblings in the kernel prompt registry) attached to graph
3025
+ * edges; this loop-kernel control folds no text in — each shot is the same task object, and the
3026
+ * per-shot nonce in `taskToPrompt` keeps the attempts independent at the router.
3027
+ */
3028
+ function naiveRetryDriver(shots) {
3029
+ return {
3030
+ name: "naive",
3031
+ plan(task, history) {
3032
+ if (history.length === 0) return Promise.resolve([task]);
3033
+ if (history[history.length - 1]?.verdict?.valid === true) return Promise.resolve([]);
3034
+ if (history.length >= shots) return Promise.resolve([]);
3035
+ return Promise.resolve([task]);
2842
3036
  },
2843
- inheritEnv: false,
2844
- timeoutMs: timeoutMs + BOX_PROCESS_BACKSTOP_MS
2845
- });
2846
- const stdout = [];
2847
- const stderr = [];
2848
- let bytes = 0;
2849
- let failure;
2850
- let stopping;
2851
- const stop = (reason) => {
2852
- failure ??= reason;
2853
- stopping ??= child.kill("SIGKILL", { tree: true }).catch(() => void 0);
2854
- };
2855
- const clearTimer = armDeadlineTimer(timeoutMs, () => stop("timeout"), true);
2856
- const cancel = () => stop("cancelled");
2857
- signal?.addEventListener("abort", cancel, { once: true });
2858
- if (signal?.aborted) cancel();
2859
- const capture = async (stream, target) => {
2860
- for await (const text of stream) {
2861
- const chunk = Buffer.from(text);
2862
- const remaining = Math.max(0, limit - bytes);
2863
- bytes += chunk.length;
2864
- if (remaining) target.push(chunk.subarray(0, remaining));
2865
- if (bytes > limit) stop("output-limit");
3037
+ decide(history) {
3038
+ if (history.some((it) => it.verdict?.valid)) return "pick-winner";
3039
+ return history.length < shots ? "refine" : "fail";
3040
+ },
3041
+ describePlan() {
3042
+ return {
3043
+ kind: "refine",
3044
+ rationale: "naive fixed continuation (no grade signal)"
3045
+ };
2866
3046
  }
2867
3047
  };
2868
- let exitCode;
2869
- let exitSignal;
3048
+ }
3049
+ /** Read `--name <value>` from an argv array. */
3050
+ function argOf(argv, name) {
3051
+ const i = argv.indexOf(`--${name}`);
3052
+ if (i >= 0 && i + 1 < argv.length) return argv[i + 1];
3053
+ }
3054
+ function splitList(v) {
3055
+ if (v === void 0) return void 0;
3056
+ const parts = v.split(",").map((s) => s.trim()).filter(Boolean);
3057
+ return parts.length > 0 ? parts : void 0;
3058
+ }
3059
+ /** RunRecords reject a bare model id — the eval IDENTITY model must carry a
3060
+ * snapshot (`name@<snapshot>`). Unchanged when already stamped. */
3061
+ function withSnapshot(model, snapshot) {
3062
+ return model.includes("@") ? model : `${model}@${snapshot}`;
3063
+ }
3064
+ function gitSha() {
2870
3065
  try {
2871
- await Promise.all([capture(child.stdout(), stdout), capture(child.stderr(), stderr)]);
2872
- exitCode = await child.wait();
2873
- exitSignal = (await child.status()).exitSignal;
2874
- } finally {
2875
- clearTimer();
2876
- signal?.removeEventListener("abort", cancel);
2877
- await stopping;
3066
+ return execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim();
3067
+ } catch {
3068
+ return "unknown";
2878
3069
  }
2879
- const out = Buffer.concat(stdout).toString();
2880
- const err = Buffer.concat(stderr).toString();
2881
- const evidence = {
2882
- stdout: out,
2883
- stderr: err,
2884
- exitCode: exitSignal ? null : exitCode
2885
- };
2886
- if (failure) return {
2887
- succeeded: false,
2888
- reason: failure,
2889
- diagnostic: err || failure,
2890
- ...evidence
2891
- };
2892
- if (exitCode === 127 && err.startsWith(BOX_EXEC_FAILURE)) return {
2893
- succeeded: false,
2894
- reason: "refused",
2895
- diagnostic: err,
2896
- ...evidence
2897
- };
2898
- if (exitSignal) return {
2899
- succeeded: false,
2900
- reason: "failed",
2901
- diagnostic: err || `Check killed by ${exitSignal}`,
2902
- ...evidence
3070
+ }
3071
+ function normalizeScore(s) {
3072
+ return typeof s === "number" ? { composite: s } : s;
3073
+ }
3074
+ /**
3075
+ * Assemble a declarative spec (`cases` + `prompt` + `score`) into a runnable
3076
+ * harness×model leaderboard — `run()` executes the matrix, `toBenchmarkAdapter()`
3077
+ * exposes the same domain as a structural `BenchmarkAdapter`.
3078
+ */
3079
+ function defineLeaderboard(spec) {
3080
+ const caseId = (c) => {
3081
+ const id = spec.caseId ? spec.caseId(c) : c.id;
3082
+ if (typeof id !== "string" || id.length === 0) throw new Error(`defineLeaderboard(${spec.name}): every case needs a stable string id — give cases an \`id\` property or supply spec.caseId`);
3083
+ return id;
2903
3084
  };
2904
- if (exitCode !== 0) return {
2905
- succeeded: false,
2906
- reason: "failed",
2907
- diagnostic: err || out || `Check exited ${exitCode}`,
2908
- ...evidence
3085
+ const selectCases = (ids) => {
3086
+ if (!ids) return spec.cases;
3087
+ const byId = new Map(spec.cases.map((c) => [caseId(c), c]));
3088
+ return ids.map((id) => {
3089
+ const c = byId.get(id);
3090
+ if (c === void 0) throw new Error(`defineLeaderboard(${spec.name}): unknown case "${id}" (have: ${[...byId.keys()].join(", ")})`);
3091
+ return c;
3092
+ });
2909
3093
  };
2910
- return {
2911
- succeeded: true,
2912
- value: {
2913
- stdout: out,
2914
- stderr: err
3094
+ const scoreJudge = {
3095
+ name: `${spec.name}-score`,
3096
+ dimensions: [{
3097
+ key: "composite",
3098
+ description: `${spec.name} case score`
3099
+ }],
3100
+ score({ artifact, scenario }) {
3101
+ const s = normalizeScore(spec.score(artifact, scenario.case));
3102
+ return {
3103
+ composite: s.composite,
3104
+ dimensions: {
3105
+ composite: s.composite,
3106
+ ...s.dimensions
3107
+ },
3108
+ notes: s.notes ?? ""
3109
+ };
2915
3110
  }
2916
3111
  };
2917
- }
2918
- function execute(args, timeoutMs, limit, signal) {
2919
- return new Promise((done) => {
2920
- const child = spawn("/usr/bin/bwrap", [
2921
- "--json-status-fd",
2922
- "3",
2923
- ...args
2924
- ], {
2925
- env: {},
2926
- stdio: [
2927
- "ignore",
2928
- "pipe",
2929
- "pipe",
2930
- "pipe"
2931
- ],
2932
- detached: true
3112
+ async function run(argv = process.argv.slice(2)) {
3113
+ const args = {};
3114
+ for (const name of [
3115
+ "backend",
3116
+ "harnesses",
3117
+ "models",
3118
+ "cases",
3119
+ "shots",
3120
+ "reps",
3121
+ "model-snapshot",
3122
+ "run-dir",
3123
+ "export-dir"
3124
+ ]) args[name] = argOf(argv, name);
3125
+ for (const [name, flag] of Object.entries(spec.flags ?? {})) args[name] = argOf(argv, name) ?? flag.default;
3126
+ const backendName = args.backend ?? "sandbox";
3127
+ const shots = Number(args.shots ?? spec.shots ?? 1);
3128
+ const reps = Number(args.reps ?? spec.reps ?? 1);
3129
+ const snapshot = args["model-snapshot"] ?? "leaderboard";
3130
+ const runDir = args["run-dir"] ?? join(tmpdir(), `leaderboard-${spec.name}-${Date.now()}-${process.pid}`);
3131
+ const exportDir = args["export-dir"] ?? join(runDir, "export");
3132
+ mkdirSync(runDir, { recursive: true });
3133
+ const cases = selectCases(splitList(args.cases));
3134
+ if (cases.length === 0) throw new Error(`defineLeaderboard(${spec.name}): no cases to run`);
3135
+ const scenarios = cases.map((c) => ({
3136
+ id: caseId(c),
3137
+ kind: spec.name,
3138
+ case: c
3139
+ }));
3140
+ const harnesses = splitList(args.harnesses) ?? spec.axis?.harnesses ?? CODING_HARNESSES;
3141
+ const rawModels = splitList(args.models) ?? spec.axis?.models ?? (spec.baseProfile?.model?.default !== void 0 ? [spec.baseProfile.model.default] : void 0);
3142
+ if (!rawModels || rawModels.length === 0) throw new Error(`defineLeaderboard(${spec.name}): no models — pass --models, set spec.axis.models, or give spec.baseProfile a model.default`);
3143
+ const models = rawModels.map((m) => withSnapshot(m, snapshot));
3144
+ const profiles = expandProfileAxes({
3145
+ base: spec.baseProfile,
3146
+ harnesses,
3147
+ models
2933
3148
  });
2934
- const stdout = [];
2935
- const stderr = [];
2936
- let bytes = 0;
2937
- let status = "";
2938
- let failure;
2939
- const stop = (reason) => {
2940
- failure ??= reason;
2941
- if (child.pid) try {
2942
- process.kill(-child.pid, "SIGKILL");
2943
- } catch (error) {
2944
- if (error.code !== "ESRCH") child.kill("SIGKILL");
2945
- }
3149
+ for (const profile of profiles) assertExecutableAgentProfile(profile, `defineLeaderboard(${spec.name})`);
3150
+ const ctx = {
3151
+ name: spec.name,
3152
+ backend: backendName,
3153
+ runDir,
3154
+ exportDir,
3155
+ args,
3156
+ harnesses,
3157
+ models,
3158
+ caseIds: scenarios.map((s) => s.id),
3159
+ shots,
3160
+ reps
2946
3161
  };
2947
- const clearTimer = armDeadlineTimer(timeoutMs, () => stop("timeout"), true);
2948
- const cancel = () => stop("cancelled");
2949
- signal?.addEventListener("abort", cancel, { once: true });
2950
- if (signal?.aborted) cancel();
2951
- const capture = (target) => (chunk) => {
2952
- const remaining = Math.max(0, limit - bytes);
2953
- bytes += chunk.length;
2954
- if (remaining) target.push(chunk.subarray(0, remaining));
2955
- if (bytes > limit) stop("output-limit");
3162
+ const backends = {
3163
+ sandbox: () => {
3164
+ throw new Error(`defineLeaderboard(${spec.name}): the 'sandbox' backend needs your product's real Sandbox client — supply spec.backends.sandbox (e.g. () => new Sandbox({ apiKey, baseUrl }))`);
3165
+ },
3166
+ "cli-bridge": () => {
3167
+ const bearer = process.env.BRIDGE_BEARER ?? process.env.CLI_BRIDGE_BEARER;
3168
+ if (!bearer) throw new Error(`defineLeaderboard(${spec.name}): backend 'cli-bridge' needs BRIDGE_BEARER or CLI_BRIDGE_BEARER set`);
3169
+ return resolveSandboxClient({
3170
+ backend: "bridge",
3171
+ bridge: {
3172
+ url: process.env.CLI_BRIDGE_URL,
3173
+ bearer,
3174
+ timeoutMs: 9e5
3175
+ }
3176
+ });
3177
+ },
3178
+ ...spec.backends
2956
3179
  };
2957
- child.stdio[3]?.on("data", (chunk) => {
2958
- if (status.length + chunk.length > 16384) {
2959
- stop("output-limit");
2960
- return;
2961
- }
2962
- status += chunk.toString();
2963
- });
2964
- child.stdout.on("data", capture(stdout));
2965
- child.stderr.on("data", capture(stderr));
2966
- let spawnError;
2967
- child.on("error", (error) => {
2968
- spawnError = error;
2969
- });
2970
- child.on("close", (code) => {
2971
- clearTimer();
2972
- signal?.removeEventListener("abort", cancel);
2973
- const out = Buffer.concat(stdout).toString();
2974
- const err = Buffer.concat(stderr).toString();
2975
- const evidence = {
2976
- stdout: out,
2977
- stderr: err,
2978
- exitCode: code
2979
- };
2980
- const executed = status.split("\n").some((line) => {
2981
- try {
2982
- const record = JSON.parse(line);
2983
- return record !== null && typeof record === "object" && "exit-code" in record && typeof record["exit-code"] === "number" && record["exit-code"] === code;
2984
- } catch {
2985
- return false;
3180
+ const makeClient = backends[backendName];
3181
+ if (!makeClient) throw new Error(`defineLeaderboard(${spec.name}): unknown backend "${backendName}" (have: ${Object.keys(backends).join(", ")})`);
3182
+ const sandboxClient = makeClient();
3183
+ const promptById = /* @__PURE__ */ new Map();
3184
+ for (const s of scenarios) promptById.set(s.id, await spec.prompt(s.case));
3185
+ const promptOf = (s) => {
3186
+ const p = promptById.get(s.id);
3187
+ if (p === void 0) throw new Error(`defineLeaderboard(${spec.name}): no prompt for case "${s.id}"`);
3188
+ return p;
3189
+ };
3190
+ let shotNonce = 0;
3191
+ const maximumCharge = spec.maximumCharge;
3192
+ const dispatch = spec.dispatch ?? ((profile, scenario, dispatchCtx) => {
3193
+ return loopDispatch({
3194
+ sandboxClient,
3195
+ maximumCharge: typeof maximumCharge === "function" ? (cellScenario, cellProfile) => maximumCharge(cellProfile, cellScenario) : maximumCharge,
3196
+ resolveCostModel: (result, _cellScenario, cellProfile) => {
3197
+ if (!spec.resolveModel) return cellProfile.model?.default;
3198
+ const served = new Set(result.iterations.map((iteration) => spec.resolveModel?.(iteration.events)).filter((model) => model !== void 0));
3199
+ if (served.size > 1) throw new Error(`defineLeaderboard(${spec.name}): one cell reported multiple served models: ${[...served].join(", ")}`);
3200
+ return [...served][0] ?? cellProfile.model?.default;
3201
+ },
3202
+ toLoopOptions: (cellScenario, cellProfile) => {
3203
+ return {
3204
+ driver: naiveRetryDriver(shots),
3205
+ agentRun: {
3206
+ profile: cellProfile,
3207
+ taskToPrompt: (s) => `${promptOf(s)}\n\n<!-- independent-attempt:${shotNonce++} -->`
3208
+ },
3209
+ output: { parse: (events) => spec.parseOutput ? spec.parseOutput(events, cellScenario.case) : collectAgentResponseText(events) ?? "" },
3210
+ validator: { validate: async (output) => {
3211
+ const s = normalizeScore(spec.score(output, cellScenario.case));
3212
+ return {
3213
+ valid: s.composite > 0,
3214
+ score: s.composite
3215
+ };
3216
+ } },
3217
+ task: cellScenario,
3218
+ maxIterations: shots
3219
+ };
3220
+ },
3221
+ toArtifact: (result) => {
3222
+ for (const iter of result.iterations) spec.onCellEvents?.(iter.events, scenario.case, {
3223
+ index: iter.index,
3224
+ ...iter.error ? { error: iter.error.message } : {},
3225
+ ...iter.verdict ? { verdict: { score: iter.verdict.score } } : {}
3226
+ });
3227
+ return result.winner?.output;
2986
3228
  }
3229
+ })(profile, scenario, dispatchCtx);
3230
+ });
3231
+ await spec.setup?.(ctx);
3232
+ try {
3233
+ const result = await runProfileMatrix({
3234
+ profiles,
3235
+ scenarios,
3236
+ dispatch,
3237
+ judges: spec.judges ?? [scoreJudge],
3238
+ runDir,
3239
+ commitSha: gitSha(),
3240
+ reps,
3241
+ ...spec.matrix
2987
3242
  });
2988
- if (failure) done({
2989
- succeeded: false,
2990
- reason: failure,
2991
- diagnostic: err || failure,
2992
- ...evidence
2993
- });
2994
- else if (spawnError) done({
2995
- succeeded: false,
2996
- reason: "refused",
2997
- diagnostic: String(spawnError),
2998
- ...evidence
2999
- });
3000
- else if (!executed) done({
3001
- succeeded: false,
3002
- reason: "refused",
3003
- diagnostic: err || "Bubblewrap did not confirm command execution",
3004
- ...evidence
3005
- });
3006
- else if (code !== 0) done({
3007
- succeeded: false,
3008
- reason: "failed",
3009
- diagnostic: err || out || `Check exited ${code}`,
3010
- ...evidence
3243
+ if (spec.export) await spec.export(result, ctx);
3244
+ else {
3245
+ mkdirSync(exportDir, { recursive: true });
3246
+ writeFileSync(join(runDir, "matrix-result.json"), `${JSON.stringify(result, null, 2)}\n`);
3247
+ const table = renderLeaderboardMarkdown(leaderboard(result.records, {
3248
+ title: spec.name,
3249
+ meta: { backend: backendName }
3250
+ }));
3251
+ writeFileSync(join(exportDir, "leaderboard.md"), table);
3252
+ console.log(table);
3253
+ }
3254
+ return result;
3255
+ } finally {
3256
+ await spec.teardown?.(ctx);
3257
+ }
3258
+ }
3259
+ function toBenchmarkAdapter() {
3260
+ return {
3261
+ name: spec.name,
3262
+ async preflight() {
3263
+ const seen = /* @__PURE__ */ new Set();
3264
+ for (const c of spec.cases) {
3265
+ const id = caseId(c);
3266
+ if (seen.has(id)) throw new Error(`defineLeaderboard(${spec.name}): duplicate case id "${id}"`);
3267
+ seen.add(id);
3268
+ }
3269
+ },
3270
+ async loadTasks(opts) {
3271
+ const selected = selectCases(opts?.ids);
3272
+ const limited = opts?.limit !== void 0 ? selected.slice(0, opts.limit) : selected;
3273
+ return Promise.all(limited.map(async (c) => ({
3274
+ id: caseId(c),
3275
+ prompt: await spec.prompt(c),
3276
+ metadata: { case: c }
3277
+ })));
3278
+ },
3279
+ async judge(task, artifact) {
3280
+ const [c] = selectCases([task.id]);
3281
+ if (c === void 0) throw new Error(`defineLeaderboard(${spec.name}): no case "${task.id}"`);
3282
+ const s = normalizeScore(spec.score(artifact, c));
3283
+ return {
3284
+ resolved: s.composite > 0,
3285
+ score: s.composite,
3286
+ detail: s.notes
3287
+ };
3288
+ },
3289
+ async goldArtifact() {}
3290
+ };
3291
+ }
3292
+ return {
3293
+ run,
3294
+ toBenchmarkAdapter
3295
+ };
3296
+ }
3297
+ //#endregion
3298
+ //#region src/runtime/harvest-corpus.ts
3299
+ /**
3300
+ * Analyze completed runs through the selected observer and retain findings in a corpus.
3301
+ *
3302
+ * Store-agnostic by design: the caller maps its trace store's rows (a
3303
+ * `ProductionTraceSink` ndjson, OTLP spans, RunRecords) to `ObserveInput` — task text,
3304
+ * final output, the event trace, terminal outcome.
3305
+ * The default observer reads execution evidence; an injected analysis owns its admitted sources.
3306
+ *
3307
+ * The nightly product job is then three lines:
3308
+ * const runs = mapSinkRowsToObserveInputs(await readSink(yesterday))
3309
+ * const report = await harvestCorpus({ runs, profile, executor, corpus })
3310
+ * log(report) // runsObserved / findings / learned / failures
3311
+ *
3312
+ * Consumers choose how retained findings inform later work and how to assess their value.
3313
+ */
3314
+ /** The completed batch evidence remains available even when every analysis failed. */
3315
+ var HarvestError = class extends Error {
3316
+ report;
3317
+ constructor(report) {
3318
+ super(`harvestCorpus: every run failed analysis (${report.failures.length}) — first: ${report.failures[0]?.error}`);
3319
+ this.report = report;
3320
+ this.name = "HarvestError";
3321
+ }
3322
+ };
3323
+ /** Batch the selected observation implementation over completed runs and retain its findings. */
3324
+ async function harvestCorpus(opts) {
3325
+ const concurrency = opts.concurrency ?? 4;
3326
+ if (!Number.isSafeInteger(concurrency) || concurrency < 1 || opts.maxRuns !== void 0 && (!Number.isSafeInteger(opts.maxRuns) || opts.maxRuns < 0)) throw new TypeError("harvest limits require positive integer concurrency and nonnegative integer maxRuns");
3327
+ const report = {
3328
+ runsObserved: 0,
3329
+ findings: 0,
3330
+ learned: 0,
3331
+ failures: [],
3332
+ usage: {
3333
+ input: 0,
3334
+ output: 0,
3335
+ known: true
3336
+ }
3337
+ };
3338
+ const iterator = Symbol.asyncIterator in Object(opts.runs) ? opts.runs[Symbol.asyncIterator]() : (async function* () {
3339
+ yield* opts.runs;
3340
+ })();
3341
+ let consumed = 0;
3342
+ let done = false;
3343
+ const next = async () => {
3344
+ if (done || opts.signal?.aborted || opts.maxRuns !== void 0 && consumed >= opts.maxRuns) return null;
3345
+ const sequence = ++consumed;
3346
+ let r;
3347
+ try {
3348
+ r = await iterator.next();
3349
+ } catch (error) {
3350
+ done = true;
3351
+ report.failures.push({
3352
+ runId: `source-${sequence}`,
3353
+ error: `trace source: ${error instanceof Error ? error.message.slice(0, 300) : String(error)}`
3011
3354
  });
3012
- else done({
3013
- succeeded: true,
3014
- value: {
3015
- stdout: out,
3016
- stderr: err
3355
+ return null;
3356
+ }
3357
+ if (r.done) {
3358
+ done = true;
3359
+ return null;
3360
+ }
3361
+ return {
3362
+ input: r.value,
3363
+ sequence
3364
+ };
3365
+ };
3366
+ const workers = Array.from({ length: concurrency }, async () => {
3367
+ for (let run = await next(); run !== null; run = await next()) {
3368
+ const { input, sequence } = run;
3369
+ if (opts.signal?.aborted) return;
3370
+ try {
3371
+ const obs = await observe(input, opts);
3372
+ report.runsObserved += 1;
3373
+ report.findings += obs.findings.length;
3374
+ report.learned += obs.learned.length;
3375
+ report.usage.input += obs.usage.input;
3376
+ report.usage.output += obs.usage.output;
3377
+ report.usage.known &&= obs.usage.known;
3378
+ } catch (e) {
3379
+ report.usage.known &&= e instanceof ObservationError && e.usage.known;
3380
+ if (e instanceof ObservationError) {
3381
+ report.usage.input += e.usage.input;
3382
+ report.usage.output += e.usage.output;
3017
3383
  }
3018
- });
3019
- });
3384
+ report.failures.push({
3385
+ runId: input.runId ?? `run-${sequence}`,
3386
+ error: e instanceof Error ? e.message.slice(0, 300) : String(e)
3387
+ });
3388
+ }
3389
+ }
3020
3390
  });
3391
+ await Promise.all(workers);
3392
+ if (report.runsObserved === 0 && report.failures.length > 0) throw new HarvestError(report);
3393
+ return report;
3021
3394
  }
3022
- const executeFile = promisify(execFile);
3023
- /** GNU tools traverse relative to open directories, including trees deeper than PATH_MAX.
3024
- * Recursive chmod ignores encountered symlinks; rm never follows their targets. */
3025
- async function removeScratch(path) {
3026
- const options = {
3027
- env: {},
3028
- timeout: 1e4,
3029
- maxBuffer: 16384
3030
- };
3031
- let permissionError;
3032
- try {
3033
- await executeFile("/bin/chmod", [
3034
- "--recursive",
3035
- "--preserve-root",
3036
- "u+rwX",
3037
- "--",
3038
- path
3039
- ], options);
3040
- } catch (error) {
3041
- permissionError = error;
3042
- }
3043
- try {
3044
- await executeFile("/bin/rm", [
3045
- "--recursive",
3046
- "--force",
3047
- "--one-file-system",
3048
- "--",
3049
- path
3050
- ], options);
3051
- } catch (error) {
3052
- throw new Error([permissionError, error].filter((value) => value !== void 0).map(String).join("; "));
3053
- }
3395
+ //#endregion
3396
+ //#region src/runtime/in-process-sandbox-client.ts
3397
+ /**
3398
+ * The ONE in-process pseudo-box: present a plain user callback as a
3399
+ * `SandboxClient` so an example or test can drive `runAgentRounds` / `openSandboxRun`
3400
+ * with zero credentials and zero network — without re-faking a box at each call
3401
+ * site via `as unknown as SandboxInstance`.
3402
+ *
3403
+ * It is the structural sibling of {@link inlineSandboxClient}: that one adapts an
3404
+ * `Executor` (router / bridge / BYO) into a box; this one adapts a single
3405
+ * `onPrompt(prompt, round)` callback into a box. Both exist because
3406
+ * `SandboxInstance` is a `declare class` with private fields — a plain object
3407
+ * literal can NEVER structurally satisfy it, so every offline seam was forced to
3408
+ * write `as unknown as SandboxInstance`. The one unavoidable cast now lives HERE,
3409
+ * documented and tested, instead of in every example.
3410
+ *
3411
+ * What the box implements is exactly the subset `runAgentRounds` + `openSandboxRun`
3412
+ * actually call on a box:
3413
+ * - `id` — stable per box, used for trace correlation.
3414
+ * - `streamPrompt(prompt, opts?)` — runs `onPrompt` and streams its events.
3415
+ * - `fs.read` / `fs.write` — over the optional real `workdir` (the deliverable
3416
+ * artifact + any seeded fixtures live there). Present only when a `workdir`
3417
+ * is given.
3418
+ * - `exec(command)` — runs the command in `workdir`. Present only when a
3419
+ * `workdir` is given.
3420
+ * - `delete()` — removes the per-box temp dir (when one was minted) and is a
3421
+ * no-op otherwise; `deleteBoxSafe` calls it at teardown.
3422
+ *
3423
+ * The `SandboxEvent` shape the callback returns — `{ type, data }` — is the
3424
+ * runtime metering protocol the kernel reads: emit a flat `llm_call`
3425
+ * (`{ tokensIn, tokensOut, costUsd }`) or a terminal `done`
3426
+ * (`{ tokenUsage, totalCostUsd }`) / `result` event and the kernel's cost ledger
3427
+ * picks it up. No cast is needed on the events (the union is structural); only
3428
+ * the box object is cast, once, inside this file.
3429
+ */
3430
+ function isAsyncIterable(v) {
3431
+ return typeof v === "object" && v !== null && Symbol.asyncIterator in v;
3432
+ }
3433
+ /**
3434
+ * Adapt a single `onPrompt(prompt, ctx)` callback into a `SandboxClient` for
3435
+ * `runAgentRounds` / `openSandboxRun`. Returns a PROPERLY-TYPED `SandboxClient`: the
3436
+ * lone `SandboxInstance` cast (object literal → `declare class`) lives inside
3437
+ * this function, so call sites stay cast-free.
3438
+ *
3439
+ * There is no box, so a per-prompt `backend` override is refused rather than dropped. A per-turn
3440
+ * `model` IS delivered, because `onPrompt` receives the verbatim options and is free to honor it.
3441
+ * Other per-prompt options (`timeoutMs`, `context`) are accepted and ignored.
3442
+ *
3443
+ * @experimental
3444
+ */
3445
+ function inProcessSandboxClient(options) {
3446
+ const { onPrompt, workdir: workdirPrefix, id: idOption } = options;
3447
+ let seq = 0;
3448
+ return { async create(_options) {
3449
+ const current = seq++;
3450
+ const id = typeof idOption === "function" ? idOption(current) : idOption ?? `in-process-${current}`;
3451
+ const boxWorkdir = workdirPrefix !== void 0 ? mkdtempSync(join(tmpdir(), `${workdirPrefix}-`)) : void 0;
3452
+ let round = 0;
3453
+ const fsMembers = boxWorkdir !== void 0 ? {
3454
+ fs: {
3455
+ async read(path) {
3456
+ return readFile(join(boxWorkdir, path), "utf8");
3457
+ },
3458
+ async write(path, content) {
3459
+ const abs = join(boxWorkdir, path);
3460
+ await mkdir(dirname(abs), { recursive: true });
3461
+ await writeFile(abs, content, "utf8");
3462
+ }
3463
+ },
3464
+ async exec(command) {
3465
+ const { exec: execCb } = await import("node:child_process");
3466
+ const { promisify } = await import("node:util");
3467
+ const execAsync = promisify(execCb);
3468
+ try {
3469
+ const { stdout, stderr } = await execAsync(command, {
3470
+ cwd: boxWorkdir,
3471
+ timeout: 3e4
3472
+ });
3473
+ return {
3474
+ exitCode: 0,
3475
+ stdout,
3476
+ stderr
3477
+ };
3478
+ } catch (err) {
3479
+ const e = err;
3480
+ return {
3481
+ exitCode: e.code ?? 1,
3482
+ stdout: e.stdout ?? "",
3483
+ stderr: e.stderr ?? e.message ?? ""
3484
+ };
3485
+ }
3486
+ }
3487
+ } : {};
3488
+ async function* drive(behavior, message, opts) {
3489
+ const prompt = typeof message === "string" ? message : JSON.stringify(message);
3490
+ const { signal: optSignal, ...rest } = opts ?? {};
3491
+ const signal = optSignal ?? new AbortController().signal;
3492
+ const ctx = {
3493
+ round,
3494
+ workdir: boxWorkdir,
3495
+ signal,
3496
+ ...Object.keys(rest).length > 0 ? { options: rest } : {}
3497
+ };
3498
+ round += 1;
3499
+ const produced = await behavior(prompt, ctx);
3500
+ if (isAsyncIterable(produced)) for await (const ev of produced) yield ev;
3501
+ else for (const ev of produced) yield ev;
3502
+ }
3503
+ return {
3504
+ id,
3505
+ streamPrompt(message, opts) {
3506
+ assertBoxlessPromptOptions(opts, "inProcessSandboxClient", ["backend"]);
3507
+ return drive(onPrompt, message, opts);
3508
+ },
3509
+ ...fsMembers,
3510
+ async delete() {
3511
+ if (boxWorkdir !== void 0) rmSync(boxWorkdir, {
3512
+ recursive: true,
3513
+ force: true
3514
+ });
3515
+ }
3516
+ };
3517
+ } };
3054
3518
  }
3055
3519
  //#endregion
3056
3520
  //#region src/runtime/observation-registry.ts
@@ -7223,6 +7687,180 @@ function createSqlRunContext(db) {
7223
7687
  };
7224
7688
  }
7225
7689
  //#endregion
7690
+ //#region src/runtime/supervise/sql-run-context.ts
7691
+ /**
7692
+ * A cross-machine run context on the existing autocommit SqlAdapter. Reuse the same database,
7693
+ * tablePrefix and runId on every host; no runDir or shared filesystem is required.
7694
+ *
7695
+ * Each ownership generation gets fresh store capabilities. The public context can inspect SQL
7696
+ * at any time, but cannot write without acquire(). runGraph/supervise acquire and release it.
7697
+ * Retained provider execution supplies external admission/result idempotency; SQL does not turn
7698
+ * an arbitrary unkeyed network effect into an exactly-once operation.
7699
+ * @experimental
7700
+ */
7701
+ async function createFencedSqlRunContext(db, runId, options = {}) {
7702
+ const store = await openSqlRunStore(db, runId, options);
7703
+ const { executors } = createInMemoryRunContext({ withDriver: options.withDriver ?? true });
7704
+ const cold = async () => projection(await store.read());
7705
+ const unowned = async () => {
7706
+ throw new SqlRunOwnershipError(`SQL run '${runId}' requires an acquired lease before writing`);
7707
+ };
7708
+ const context = {
7709
+ durability: "sql",
7710
+ runId,
7711
+ namespace: store.namespace,
7712
+ resume: true,
7713
+ executors,
7714
+ journal: {
7715
+ loadTree: async (root) => (await cold()).journal.loadTree(root),
7716
+ beginTree: unowned,
7717
+ appendEvent: unowned,
7718
+ appendEvents: unowned
7719
+ },
7720
+ blobs: {
7721
+ get: async (ref) => (await cold()).blobs.get(ref),
7722
+ put: unowned
7723
+ },
7724
+ coordinationLog: {
7725
+ load: async (id, owner) => (await cold()).loadCoordination(id, owner),
7726
+ append: unowned
7727
+ },
7728
+ async acquire(signal) {
7729
+ const lease = await store.acquire(signal);
7730
+ try {
7731
+ const failureController = new AbortController();
7732
+ const leaseSignal = AbortSignal.any([lease.signal, failureController.signal]);
7733
+ const view = await projection(lease.records);
7734
+ let tail = Promise.resolve();
7735
+ let accepting = true;
7736
+ let failed;
7737
+ let hasFailed = false;
7738
+ const check = () => {
7739
+ leaseSignal.throwIfAborted();
7740
+ if (hasFailed) throw failed;
7741
+ };
7742
+ const read = async (operation) => {
7743
+ await tail;
7744
+ check();
7745
+ return operation();
7746
+ };
7747
+ const commit = (entry) => {
7748
+ if (!accepting) return unowned();
7749
+ const captured = detachedSnapshot(entry, "SQL run record");
7750
+ const write = tail.then(async () => {
7751
+ check();
7752
+ if (await view.apply(captured)) await lease.append(captured);
7753
+ });
7754
+ tail = write.catch((error) => {
7755
+ hasFailed = true;
7756
+ failed = error;
7757
+ failureController.abort(error);
7758
+ });
7759
+ return write;
7760
+ };
7761
+ const bound = {
7762
+ durability: "sql",
7763
+ runId,
7764
+ namespace: store.namespace,
7765
+ resume: true,
7766
+ executors,
7767
+ journal: {
7768
+ loadTree: (root) => read(() => view.journal.loadTree(root)),
7769
+ beginTree: (root, at) => commit({
7770
+ kind: "begin",
7771
+ root,
7772
+ at
7773
+ }),
7774
+ appendEvent: (root, event) => commit({
7775
+ kind: "events",
7776
+ root,
7777
+ events: [event]
7778
+ }),
7779
+ appendEvents: (root, events) => commit({
7780
+ kind: "events",
7781
+ root,
7782
+ events: [...events]
7783
+ })
7784
+ },
7785
+ blobs: {
7786
+ get: (ref) => read(() => view.blobs.get(ref)),
7787
+ put: (ref, value) => commit({
7788
+ kind: "blobs",
7789
+ ref,
7790
+ value
7791
+ })
7792
+ },
7793
+ coordinationLog: {
7794
+ load: (id, owner) => read(() => view.loadCoordination(id, owner)),
7795
+ append: (id, record, ownerId) => commit({
7796
+ kind: "coordination",
7797
+ runId: id,
7798
+ record,
7799
+ ...ownerId === void 0 ? {} : { ownerId }
7800
+ })
7801
+ }
7802
+ };
7803
+ return {
7804
+ context: Object.freeze(bound),
7805
+ signal: leaseSignal,
7806
+ async release() {
7807
+ accepting = false;
7808
+ await tail;
7809
+ await lease.release();
7810
+ }
7811
+ };
7812
+ } catch (error) {
7813
+ await lease.release().catch(() => void 0);
7814
+ throw error;
7815
+ }
7816
+ }
7817
+ };
7818
+ return Object.freeze(context);
7819
+ }
7820
+ /** Reuse the existing journal's uniqueness/sequence rules and blob hash validation verbatim. */
7821
+ async function projection(entries) {
7822
+ const journal = new InMemorySpawnJournal();
7823
+ const blobs = new InMemoryResultBlobStore();
7824
+ const beginnings = /* @__PURE__ */ new Map();
7825
+ const coordination = [];
7826
+ const apply = async (entry) => {
7827
+ switch (entry.kind) {
7828
+ case "begin": {
7829
+ const prior = beginnings.get(entry.root);
7830
+ await journal.beginTree(entry.root, entry.at);
7831
+ beginnings.set(entry.root, entry.at);
7832
+ return prior === void 0;
7833
+ }
7834
+ case "events":
7835
+ for (const event of entry.events) await journal.appendEvent(entry.root, event);
7836
+ return entry.events.length > 0;
7837
+ case "blobs": {
7838
+ const existing = await blobs.get(entry.ref);
7839
+ await blobs.put(entry.ref, entry.value);
7840
+ return existing === void 0;
7841
+ }
7842
+ case "coordination":
7843
+ if (entry.record.event.type === "settled") return false;
7844
+ coordination.push(entry);
7845
+ return true;
7846
+ default: throw new Error("Unknown SQL run record kind");
7847
+ }
7848
+ };
7849
+ for (const entry of entries) await apply(entry);
7850
+ return {
7851
+ journal,
7852
+ blobs,
7853
+ apply,
7854
+ loadCoordination(id, ownerId) {
7855
+ return foldCoordinationRecords(coordination.map(({ runId, ownerId, record }) => ({
7856
+ runId,
7857
+ ...record,
7858
+ ...ownerId === void 0 ? {} : { ownerId }
7859
+ })), id, ownerId);
7860
+ }
7861
+ };
7862
+ }
7863
+ //#endregion
7226
7864
  //#region src/runtime/supervise/trajectory-recorder.ts
7227
7865
  /**
7228
7866
  *
@@ -8506,6 +9144,6 @@ function tail(s) {
8506
9144
  return s.slice(-400);
8507
9145
  }
8508
9146
  //#endregion
8509
- export { trajectoryReport as $, resolveMcpServerLaunch as $t, chatTransportExecutor as A, loopDispatch as At, strategyAuthorSystemPrompt as B, renderLeaderboardSvg as Bt, withUntrackedArtifacts as C, HarvestError as Ct, runCoderChecks as D, localSandboxClient as Dt, patchDelivered as E, resolveSandboxClient as Et, runStrategyEvolution as F, stopSentinel as Ft, sharedBoxRefusal as G, SqlSpawnJournal as Gt, DEFAULT_SHARED_BOX_WORKERS as H, auditIntent as Ht, selectChampion as I, leaderboard as It, openSandboxRun as J, materializeLocalMcp as Jt, sharedWorkerClientName as K, McpSpawnFault as Kt, assertStrategyContract as L, pairwiseSignificance as Lt, createChatSessionStore as M, completionAuthorizes as Mt, discriminatingMeans as N, deterministicCompletion as Nt, codeModeSupervisorTools as O, inlineSandboxClient as Ot, pickChampion as P, sentinelCompletion as Pt, equalKOnCost as Q, mcpSecretEnvMetadataKey as Qt, authorStrategy as R, renderLeaderboardHtml as Rt, copyUntrackedIntoClone as S, inProcessSandboxClient as St, createSqlRunContext as T, defineLeaderboard as Tt, ROUTER_CLIENT_HEADER as U, defaultAuditorInstruction as Ut, DEFAULT_SHARED_BOX_RESOURCES as V, renderPairwiseMarkdown as Vt, sharedBoxPlacement as W, SqlResultBlobStore as Wt, runBenchmark as X, sanitizeMcpToolSchema as Xt, printBenchmarkReport as Y, createMcpEnvironment as Yt, promotionGate as Z, envKeyProvider as Zt, NOTE_MAX_CHARS as _, buildSteerContext as _t, localShell as a, FileCorpus as at, composeWorkerEvidence as b, observationFromRegistry as bt, createVerifierEnvironment as c, fanout as ct, harvestSurfaceDiffs as d, panel as dt, resolveSecretEnv as en, definePersona as et, analystsFromRegistry as f, pipeline as ft, EVIDENCE_MAX_CHARS as g, assertTraceDerivedFindings as gt, worktreeFanout as h, widen as ht, jjWorkspace as i, registerShape as it, chatWorkerSeam as j, superviseDispatch as jt, unsafeInProcessRunner as k, loopCampaignDispatch as kt, boxSurfaceReader as l, flatWidenGate as lt, superviseSurface as m, verify as mt, makeFinding$1 as n, createTangleSandboxExactProcessProvider as nn, builtinShapes as nt, runInWorkspace as o, InMemoryCorpus as ot, failuresAnalyst as p, selectValidWinner as pt, SandboxRunAbortError as q, connectStdioMcp as qt, gitWorkspace as r, createShapeRegistry as rt, createWaterfallCollector as s, renderCorpusToInstructions as st, computeFindingId$1 as t, secretEnvOfMcpServer as tn, runPersonified as tt, fsSurfaceReader as u, loopUntil as ut, VERIFY_TAIL_CHARS as v, createScopeAnalyst as vt, analyzeTrace as w, harvestCorpus as wt, settledWorkerOut as x, runIsolatedCheck as xt, closingWorkerNote as y, registryScopeAnalyst as yt, strategyAuthorContract as z, renderLeaderboardMarkdown as zt };
9147
+ export { equalKOnCost as $, SqlResultBlobStore as $t, unsafeInProcessRunner as A, loopDispatch as At, strategyAuthorContract as B, deterministicCompletion as Bt, withUntrackedArtifacts as C, HarvestError as Ct, patchDelivered as D, localSandboxClient as Dt, createSqlRunContext as E, resolveSandboxClient as Et, pickChampion as F, declaredCheckDigest as Ft, sharedBoxPlacement as G, renderLeaderboardHtml as Gt, DEFAULT_SHARED_BOX_RESOURCES as H, stopSentinel as Ht, runStrategyEvolution as I, declaredCheckJudge as It, SandboxRunAbortError as J, renderPairwiseMarkdown as Jt, sharedBoxRefusal as K, renderLeaderboardMarkdown as Kt, selectChampion as L, readDeclaredCheck as Lt, chatWorkerSeam as M, assertDeclaredCheck as Mt, createChatSessionStore as N, checkProgramDigest as Nt, runCoderChecks as O, inlineSandboxClient as Ot, discriminatingMeans as P, declaredCheckDeliverable as Pt, promotionGate as Q, openSqlRunStore as Qt, assertStrategyContract as R, runIsolatedCheck as Rt, copyUntrackedIntoClone as S, inProcessSandboxClient as St, createFencedSqlRunContext as T, defineLeaderboard as Tt, DEFAULT_SHARED_BOX_WORKERS as U, leaderboard as Ut, strategyAuthorSystemPrompt as V, sentinelCompletion as Vt, ROUTER_CLIENT_HEADER as W, pairwiseSignificance as Wt, printBenchmarkReport as X, defaultAuditorInstruction as Xt, openSandboxRun as Y, auditIntent as Yt, runBenchmark as Z, SqlRunOwnershipError as Zt, NOTE_MAX_CHARS as _, assertTraceDerivedFindings as _t, localShell as a, sanitizeMcpToolSchema as an, registerShape as at, composeWorkerEvidence as b, registryScopeAnalyst as bt, createVerifierEnvironment as c, resolveMcpServerLaunch as cn, renderCorpusToInstructions as ct, harvestSurfaceDiffs as d, createTangleSandboxExactProcessProvider as dn, loopUntil as dt, SqlSpawnJournal as en, trajectoryReport as et, analystsFromRegistry as f, panel as ft, EVIDENCE_MAX_CHARS as g, widen as gt, worktreeFanout as h, verify as ht, jjWorkspace as i, createMcpEnvironment as in, createShapeRegistry as it, chatTransportExecutor as j, superviseDispatch as jt, codeModeSupervisorTools as k, loopCampaignDispatch as kt, boxSurfaceReader as l, resolveSecretEnv as ln, fanout as lt, superviseSurface as m, selectValidWinner as mt, makeFinding$1 as n, connectStdioMcp as nn, runPersonified as nt, runInWorkspace as o, envKeyProvider as on, FileCorpus as ot, failuresAnalyst as p, pipeline as pt, sharedWorkerClientName as q, renderLeaderboardSvg as qt, gitWorkspace as r, materializeLocalMcp as rn, builtinShapes as rt, createWaterfallCollector as s, mcpSecretEnvMetadataKey as sn, InMemoryCorpus as st, computeFindingId$1 as t, McpSpawnFault as tn, definePersona as tt, fsSurfaceReader as u, secretEnvOfMcpServer as un, flatWidenGate as ut, VERIFY_TAIL_CHARS as v, buildSteerContext as vt, analyzeTrace as w, harvestCorpus as wt, settledWorkerOut as x, observationFromRegistry as xt, closingWorkerNote as y, createScopeAnalyst as yt, authorStrategy as z, completionAuthorizes as zt };
8510
9148
 
8511
- //# sourceMappingURL=runtime-DVjIH1eo.js.map
9149
+ //# sourceMappingURL=runtime-CHxXrUeZ.js.map