@tangle-network/agent-runtime 0.271.0 → 0.273.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{activation-DGbFPGR3.js → activation-Cx2dKA_l.js} +2 -2
- package/dist/{activation-DGbFPGR3.js.map → activation-Cx2dKA_l.js.map} +1 -1
- package/dist/{activation-BrJmwTLB.d.ts → activation-IfDlhR3J.d.ts} +2 -2
- package/dist/agent.d.ts +2 -2
- package/dist/agent.js +2 -2
- package/dist/candidate-execution/index.js +2 -2
- package/dist/{coordination-driver-iBGMKDvB.js → coordination-driver-CkcW4dxU.js} +155 -80
- package/dist/coordination-driver-CkcW4dxU.js.map +1 -0
- package/dist/{delegate-DOqUvaIz.js → delegate-Dvb1-mE-.js} +2 -2
- package/dist/{delegate-DOqUvaIz.js.map → delegate-Dvb1-mE-.js.map} +1 -1
- package/dist/durable.d.ts +3 -572
- package/dist/durable.js +92 -18
- package/dist/durable.js.map +1 -1
- package/dist/{graph-DnMcepkN.js → graph-C69Uxtzt.js} +14 -8
- package/dist/graph-C69Uxtzt.js.map +1 -0
- package/dist/{improvement-cycle-1IyRJo5h.js → improvement-cycle-CO2HRCQq.js} +4 -4
- package/dist/{improvement-cycle-1IyRJo5h.js.map → improvement-cycle-CO2HRCQq.js.map} +1 -1
- package/dist/{index-CL1dduJd.d.ts → index-D2ipSqkP.d.ts} +4318 -3102
- package/dist/index.d.ts +6 -450
- package/dist/index.js +14 -11
- package/dist/index.js.map +1 -1
- package/dist/intelligence.d.ts +3 -3
- package/dist/intelligence.js +5 -5
- package/dist/kernel.d.ts +4 -4
- package/dist/kernel.js +9 -9
- package/dist/{loop-runner-bin-BPZ9tt1W.js → loop-runner-bin-B9MIq9O7.js} +3 -3
- package/dist/{loop-runner-bin-BPZ9tt1W.js.map → loop-runner-bin-B9MIq9O7.js.map} +1 -1
- package/dist/{loop-runner-bin-O_vM1w8Y.d.ts → loop-runner-bin-ThlwVkvo.d.ts} +3 -3
- package/dist/loop-runner-bin.d.ts +1 -1
- package/dist/loop-runner-bin.js +1 -1
- package/dist/mcp/bin.js +3 -3
- package/dist/mcp/index.d.ts +5 -5
- package/dist/mcp/index.js +4 -4
- package/dist/{openai-tools-DWHxpwx2.d.ts → openai-tools-Bb0mAr5z.d.ts} +2 -2
- package/dist/{prepare-oHUQdC5-.js → prepare-DVlvGP7V.js} +2 -2
- package/dist/{prepare-oHUQdC5-.js.map → prepare-DVlvGP7V.js.map} +1 -1
- package/dist/profiles.d.ts +2 -2
- package/dist/{protected-model-port-DqF0qDU6.js → protected-model-port-BrWgsCnl.js} +2 -2
- package/dist/{protected-model-port-DqF0qDU6.js.map → protected-model-port-BrWgsCnl.js.map} +1 -1
- package/dist/{protected-redaction--F3v1oo8.js → protected-redaction-DylCC_ot.js} +14 -20
- package/dist/protected-redaction-DylCC_ot.js.map +1 -0
- package/dist/{provision-supervisor-Cy8pqE95.js → provision-supervisor-BKAvsDtP.js} +3 -3
- package/dist/{provision-supervisor-Cy8pqE95.js.map → provision-supervisor-BKAvsDtP.js.map} +1 -1
- package/dist/{runtime-DVjIH1eo.js → runtime-CHxXrUeZ.js} +2079 -1441
- package/dist/runtime-CHxXrUeZ.js.map +1 -0
- package/dist/{server-CN9vzbSe.js → server-2OeFrEVG.js} +3 -3
- package/dist/{server-CN9vzbSe.js.map → server-2OeFrEVG.js.map} +1 -1
- package/dist/{stream-agent-turn-DbEsKgb_.d.ts → stream-agent-turn-ewm3BZAp.d.ts} +15 -16
- package/dist/{structural-rollout-I92sIfNl.js → structural-rollout-DCUprNpH.js} +2 -2
- package/dist/{structural-rollout-I92sIfNl.js.map → structural-rollout-DCUprNpH.js.map} +1 -1
- package/dist/{substrate-CbC03Vzo.d.ts → substrate-VKjmrksz.d.ts} +2 -2
- package/dist/{supervise-DJ_1i9vj.js → supervise-Dej5aSDa.js} +211 -48
- package/dist/supervise-Dej5aSDa.js.map +1 -0
- package/dist/{supervisor-Cvd5mf7x.js → supervisor-DdKPGluv.js} +952 -221
- package/dist/supervisor-DdKPGluv.js.map +1 -0
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +60 -60
- package/dist/tui/index.d.ts +1 -1
- package/dist/tui/index.js +1 -1
- package/dist/{types-Bo7z5aG5.d.ts → types-DMRc3jru.d.ts} +356 -64
- package/package.json +5 -5
- package/dist/coordination-driver-iBGMKDvB.js.map +0 -1
- package/dist/graph-DnMcepkN.js.map +0 -1
- package/dist/protected-redaction--F3v1oo8.js.map +0 -1
- package/dist/runtime-DVjIH1eo.js.map +0 -1
- package/dist/supervise-DJ_1i9vj.js.map +0 -1
- package/dist/supervisor-Cvd5mf7x.js.map +0 -1
|
@@ -1,24 +1,24 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { $i as detachedSnapshot, Ai as readCommittedJsonLines, Bn as runAgentRounds, Bt as SpawnEventIndex, Ca as stringifySafe, Ct as rollingDispatch, Gn as readPromptOptions, Hn as probeSandboxCapabilities, Ht as encodeResultBlob, Ia as armDeadlineTimer, Jn as canonicalObservedModelParts, Kn as routerBrain, M as withDriverExecutor, O as verdictFromJudgeScore, Oa as zeroSpend, Ot as createExecutor, Qi as detachedFrozen, Rn as defaultSelectWinner, Rt as InMemoryResultBlobStore, Sa as sleep, Ut as insideCursorNamespace, Vn as createSandboxLineage, Vt as assertContentAddress, W as settledToIteration, Wn as assertBoxlessPromptOptions, Xi as executableAgentProfileSnapshot, Xt as isTraceAnalysisStore, Zi as executableAgentSpecSnapshot, Zn as LEAF_CONTINUATION_TASK, _ as CheckUnavailableError, ai as harnessInvocation, ar as createPushTraceSource, ga as cloneSpend, ha as chargedTokens, ir as unavailableSignalOfFailure, ji as writeAllBytes, jt as createWorktreeCliExecutor, ki as prepareJsonlAppend, kr as contentAddress, kt as createExecutorRegistry, ma as addSpend, nr as sandboxClientAsProvider, qn as runBrainLoop, r as createSupervisor, rr as unavailablePauseMs, ua as notifySandboxEventObserver, va as hasCompleteCacheBreakdown, xa as randomSuffix, ya as isAbortError, zt as InMemorySpawnJournal } from "./supervisor-DdKPGluv.js";
|
|
2
2
|
import { m as ValidationError, n as AnalystError, s as PlannerError } from "./errors-DodWX-cb.js";
|
|
3
|
-
import { E as profileChatClient, S as ObservationError, T as renderReport, b as sample, k as strategyAuthorMethod, v as refine, w as observe, x as sampleThenRefine, y as runAgentic } from "./structural-rollout-
|
|
3
|
+
import { E as profileChatClient, S as ObservationError, T as renderReport, b as sample, k as strategyAuthorMethod, v as refine, w as observe, x as sampleThenRefine, y as runAgentic } from "./structural-rollout-DCUprNpH.js";
|
|
4
4
|
import { a as concreteProfileModel, c as profileModelExecutionSettings, i as concreteModelId, l as profileProviderModel, o as enforceTokenLimits, t as assertExecutableAgentProfile } from "./model-policy-BbSCSak0.js";
|
|
5
5
|
import { C as canonicalCandidateDigest$1, D as immutableCandidateValue, k as sha256Bytes$1, m as captureMaterializedWorkspace } from "./workspace-archive-C9lgf77y.js";
|
|
6
6
|
import { i as notifyRuntimeHookEvent } from "./runtime-hooks-tXpAarhW.js";
|
|
7
|
-
import { i as redactProtectedValue, r as redactProtectedReason } from "./protected-redaction
|
|
8
|
-
import "./coordination-driver-
|
|
9
|
-
import { C as gateOnDeliverable, r as supervise, w as mapExecutorResult } from "./supervise-
|
|
7
|
+
import { i as redactProtectedValue, r as redactProtectedReason } from "./protected-redaction-DylCC_ot.js";
|
|
8
|
+
import { S as foldCoordinationRecords, y as createInMemoryRunContext } from "./coordination-driver-CkcW4dxU.js";
|
|
9
|
+
import { C as gateOnDeliverable, r as supervise, w as mapExecutorResult } from "./supervise-Dej5aSDa.js";
|
|
10
10
|
import "./run-layout-vwd4SwX_.js";
|
|
11
|
-
import "./provision-supervisor-
|
|
12
|
-
import "./delegate-
|
|
13
|
-
import "./graph-
|
|
14
|
-
import { agentProfileSchema, canonicalAgentProfileDigest, canonicalCandidateDigest, validateAgentProfileSecurity } from "@tangle-network/agent-interface";
|
|
11
|
+
import "./provision-supervisor-BKAvsDtP.js";
|
|
12
|
+
import "./delegate-Dvb1-mE-.js";
|
|
13
|
+
import "./graph-C69Uxtzt.js";
|
|
14
|
+
import { agentProfileSchema, canonicalAgentProfileDigest, canonicalCandidateDigest, sha256Bytes, validateAgentProfileSecurity } from "@tangle-network/agent-interface";
|
|
15
15
|
import { CODING_HARNESSES, DEFAULT_TRACE_ANALYST_KINDS, InMemoryTraceStore, OUTPUT_VALUE, benjaminiHochberg, buildTrajectory, computeFindingId as computeFindingId$1, confidenceInterval, createTraceAnalyst, expandProfileAxes, makeFinding as makeFinding$1, pairedBootstrap, paretoFrontier, wilcoxonSignedRank, wilson } from "@tangle-network/agent-eval";
|
|
16
16
|
import { createAgentRunOutcomeTracker } from "@tangle-network/sandbox/runtime";
|
|
17
17
|
import { createHash, randomUUID } from "node:crypto";
|
|
18
18
|
import { execFile, execFileSync, spawn, spawnSync } from "node:child_process";
|
|
19
19
|
import { appendFileSync, chmodSync, constants, copyFileSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readlinkSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
|
20
20
|
import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
21
|
-
import { cp, mkdir, mkdtemp, readFile, realpath, stat, writeFile } from "node:fs/promises";
|
|
21
|
+
import { cp, mkdir, mkdtemp, readFile, readdir, realpath, rm, stat, writeFile } from "node:fs/promises";
|
|
22
22
|
import { materializeProfile } from "@tangle-network/agent-profile-materialize";
|
|
23
23
|
import { tmpdir } from "node:os";
|
|
24
24
|
import { isDeepStrictEqual, promisify } from "node:util";
|
|
@@ -1015,6 +1015,266 @@ var SqlResultBlobStore = class {
|
|
|
1015
1015
|
}
|
|
1016
1016
|
};
|
|
1017
1017
|
//#endregion
|
|
1018
|
+
//#region src/durable/sql-run-store.ts
|
|
1019
|
+
/** Refuses a competing or stale SQL run owner before it can publish new work. */
|
|
1020
|
+
var SqlRunOwnershipError = class extends Error {
|
|
1021
|
+
constructor(message) {
|
|
1022
|
+
super(message);
|
|
1023
|
+
this.name = "SqlRunOwnershipError";
|
|
1024
|
+
}
|
|
1025
|
+
};
|
|
1026
|
+
/**
|
|
1027
|
+
* Append-only immutable records, published by ONE compare-and-set of the run's head.
|
|
1028
|
+
*
|
|
1029
|
+
* SqlAdapter has no pinned-connection transaction API. INSERT ... SELECT owner is NOT a
|
|
1030
|
+
* fence on MVCC databases: its snapshot can outlive a takeover. Here both publication and
|
|
1031
|
+
* takeover UPDATE the SAME row. Unpublished records are harmless, unreachable staging.
|
|
1032
|
+
* Healthy appends write only their new payload and one fixed-size head, never the history.
|
|
1033
|
+
*
|
|
1034
|
+
* Liveness uses a persisted progress counter, not incomparable host clocks. A contender must
|
|
1035
|
+
* observe the SAME generation/counter for leaseMs before replacing it with a CAS. Every
|
|
1036
|
+
* heartbeat and publication advances that counter. The interval is stored per run so a
|
|
1037
|
+
* differently configured contender cannot shorten an existing owner's lease.
|
|
1038
|
+
* @internal
|
|
1039
|
+
*/
|
|
1040
|
+
async function openSqlRunStore(db, runId, options = {}) {
|
|
1041
|
+
const prefix = options.tablePrefix ?? "agent_run";
|
|
1042
|
+
const leaseMs = options.leaseMs ?? 3e4;
|
|
1043
|
+
const heartbeatMs = options.heartbeatMs ?? Math.floor(leaseMs / 3);
|
|
1044
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]{0,39}$/.test(prefix)) throw new Error("SQL run context tablePrefix must be a SQL identifier of at most 40 characters");
|
|
1045
|
+
if (typeof runId !== "string" || runId.length === 0) throw new Error("SQL runId must be nonempty");
|
|
1046
|
+
if (!Number.isSafeInteger(leaseMs) || leaseMs < 100 || leaseMs > 2147483647) throw new Error("SQL leaseMs must be an integer between 100 and 2147483647");
|
|
1047
|
+
if (!Number.isSafeInteger(heartbeatMs) || heartbeatMs < 1 || heartbeatMs > leaseMs / 3) throw new Error("SQL heartbeatMs must be a positive integer no greater than leaseMs / 3");
|
|
1048
|
+
const heads = `${prefix}_heads`;
|
|
1049
|
+
const records = `${prefix}_records`;
|
|
1050
|
+
await db.exec(`CREATE TABLE IF NOT EXISTS ${heads} (
|
|
1051
|
+
run_id TEXT PRIMARY KEY,
|
|
1052
|
+
owner TEXT NOT NULL,
|
|
1053
|
+
generation INTEGER NOT NULL,
|
|
1054
|
+
pulse INTEGER NOT NULL,
|
|
1055
|
+
head TEXT NOT NULL,
|
|
1056
|
+
revision INTEGER NOT NULL,
|
|
1057
|
+
lease_ms INTEGER NOT NULL,
|
|
1058
|
+
namespace TEXT NOT NULL
|
|
1059
|
+
)`);
|
|
1060
|
+
await db.exec(`CREATE TABLE IF NOT EXISTS ${records} (
|
|
1061
|
+
run_id TEXT NOT NULL,
|
|
1062
|
+
id TEXT NOT NULL,
|
|
1063
|
+
parent TEXT NOT NULL,
|
|
1064
|
+
revision INTEGER NOT NULL,
|
|
1065
|
+
payload TEXT NOT NULL,
|
|
1066
|
+
PRIMARY KEY (run_id, id)
|
|
1067
|
+
)`);
|
|
1068
|
+
await db.exec(`INSERT INTO ${heads} (run_id, owner, generation, pulse, head, revision, lease_ms, namespace)
|
|
1069
|
+
VALUES (?, '', 0, 0, '', 0, ?, ?) ON CONFLICT (run_id) DO NOTHING`, [
|
|
1070
|
+
runId,
|
|
1071
|
+
leaseMs,
|
|
1072
|
+
randomUUID()
|
|
1073
|
+
]);
|
|
1074
|
+
async function readHead() {
|
|
1075
|
+
const [row] = await db.query(`SELECT * FROM ${heads} WHERE run_id = ?`, [runId]);
|
|
1076
|
+
if (!row) throw new Error(`SQL run '${runId}' has no head`);
|
|
1077
|
+
for (const key of [
|
|
1078
|
+
"generation",
|
|
1079
|
+
"pulse",
|
|
1080
|
+
"revision",
|
|
1081
|
+
"lease_ms"
|
|
1082
|
+
]) {
|
|
1083
|
+
row[key] = Number(row[key]);
|
|
1084
|
+
if (!Number.isSafeInteger(row[key]) || row[key] < 0) throw new Error(`SQL run '${runId}' has an invalid ${key}`);
|
|
1085
|
+
}
|
|
1086
|
+
if (typeof row.namespace !== "string" || row.namespace.length === 0 || typeof row.head !== "string" || typeof row.owner !== "string" || row.head === "" !== (row.revision === 0)) throw new Error(`SQL run '${runId}' has an invalid head`);
|
|
1087
|
+
if (row.lease_ms !== leaseMs) throw new Error(`SQL run '${runId}' lease configuration differs: recorded ${row.lease_ms}ms, requested ${leaseMs}ms`);
|
|
1088
|
+
return row;
|
|
1089
|
+
}
|
|
1090
|
+
async function load(head) {
|
|
1091
|
+
if (head.revision === 0) return [];
|
|
1092
|
+
const rows = await db.query(`SELECT id, parent, revision, payload FROM ${records} WHERE run_id = ? AND revision <= ?`, [runId, head.revision]);
|
|
1093
|
+
const byId = new Map(rows.map((row) => [row.id, row]));
|
|
1094
|
+
const history = [];
|
|
1095
|
+
let id = head.head;
|
|
1096
|
+
for (let revision = head.revision; revision > 0; revision -= 1) {
|
|
1097
|
+
const row = byId.get(id);
|
|
1098
|
+
if (!row || Number(row.revision) !== revision || recordId(row.parent, revision, row.payload) !== id) throw new Error(`SQL run '${runId}' has a missing or corrupt committed record at revision ${revision}`);
|
|
1099
|
+
history.push(JSON.parse(row.payload));
|
|
1100
|
+
id = row.parent;
|
|
1101
|
+
}
|
|
1102
|
+
if (id !== "") throw new Error(`SQL run '${runId}' has an invalid history origin`);
|
|
1103
|
+
return history.reverse();
|
|
1104
|
+
}
|
|
1105
|
+
return {
|
|
1106
|
+
namespace: (await readHead()).namespace,
|
|
1107
|
+
async read() {
|
|
1108
|
+
return load(await readHead());
|
|
1109
|
+
},
|
|
1110
|
+
async acquire(signal) {
|
|
1111
|
+
signal?.throwIfAborted();
|
|
1112
|
+
const observed = await readHead();
|
|
1113
|
+
if (observed.owner !== "") await wait(leaseMs, signal);
|
|
1114
|
+
signal?.throwIfAborted();
|
|
1115
|
+
const owner = randomUUID();
|
|
1116
|
+
let claimed = false;
|
|
1117
|
+
try {
|
|
1118
|
+
claimed = (await db.exec(`UPDATE ${heads} SET owner = ?, generation = generation + 1, pulse = pulse + 1
|
|
1119
|
+
WHERE run_id = ? AND owner = ? AND generation = ? AND pulse = ?`, [
|
|
1120
|
+
owner,
|
|
1121
|
+
runId,
|
|
1122
|
+
observed.owner,
|
|
1123
|
+
observed.generation,
|
|
1124
|
+
observed.pulse
|
|
1125
|
+
])).rowsAffected === 1;
|
|
1126
|
+
} catch (error) {
|
|
1127
|
+
if ((await readHead().catch(() => void 0))?.owner !== owner) throw error;
|
|
1128
|
+
claimed = true;
|
|
1129
|
+
}
|
|
1130
|
+
if (!claimed) throw new SqlRunOwnershipError(`SQL run '${runId}' is owned by another live process`);
|
|
1131
|
+
const generation = observed.generation + 1;
|
|
1132
|
+
const controller = new AbortController();
|
|
1133
|
+
let accepting = true;
|
|
1134
|
+
let released;
|
|
1135
|
+
let tail = Promise.resolve();
|
|
1136
|
+
let heartbeat;
|
|
1137
|
+
let progressAt = performance.now();
|
|
1138
|
+
let head = observed.head;
|
|
1139
|
+
let revision = observed.revision;
|
|
1140
|
+
let history = [];
|
|
1141
|
+
const lose = (error) => {
|
|
1142
|
+
accepting = false;
|
|
1143
|
+
clearInterval(timer);
|
|
1144
|
+
controller.abort(error);
|
|
1145
|
+
};
|
|
1146
|
+
const timer = setInterval(() => {
|
|
1147
|
+
if (controller.signal.aborted) return;
|
|
1148
|
+
if (performance.now() - progressAt >= leaseMs) {
|
|
1149
|
+
lose(new SqlRunOwnershipError(`SQL run '${runId}' lease heartbeat expired`));
|
|
1150
|
+
return;
|
|
1151
|
+
}
|
|
1152
|
+
if (heartbeat) return;
|
|
1153
|
+
heartbeat = db.exec(`UPDATE ${heads} SET pulse = pulse + 1 WHERE run_id = ? AND owner = ? AND generation = ?`, [
|
|
1154
|
+
runId,
|
|
1155
|
+
owner,
|
|
1156
|
+
generation
|
|
1157
|
+
]).then((result) => {
|
|
1158
|
+
if (result.rowsAffected !== 1) throw new SqlRunOwnershipError(`SQL run '${runId}' lease was replaced`);
|
|
1159
|
+
progressAt = performance.now();
|
|
1160
|
+
}).catch(lose).finally(() => {
|
|
1161
|
+
heartbeat = void 0;
|
|
1162
|
+
});
|
|
1163
|
+
}, heartbeatMs);
|
|
1164
|
+
timer.unref();
|
|
1165
|
+
const lease = {
|
|
1166
|
+
signal: controller.signal,
|
|
1167
|
+
get records() {
|
|
1168
|
+
return JSON.parse(JSON.stringify(history));
|
|
1169
|
+
},
|
|
1170
|
+
append(payload) {
|
|
1171
|
+
if (!accepting) return Promise.reject(controller.signal.reason ?? new SqlRunOwnershipError(`SQL run '${runId}' lease is released`));
|
|
1172
|
+
let encoded;
|
|
1173
|
+
try {
|
|
1174
|
+
encoded = JSON.stringify(payload);
|
|
1175
|
+
if (encoded === void 0) throw new Error("SQL run records must be JSON serializable");
|
|
1176
|
+
} catch (error) {
|
|
1177
|
+
return Promise.reject(error);
|
|
1178
|
+
}
|
|
1179
|
+
const write = tail.then(async () => {
|
|
1180
|
+
controller.signal.throwIfAborted();
|
|
1181
|
+
const nextRevision = revision + 1;
|
|
1182
|
+
const id = recordId(head, nextRevision, encoded);
|
|
1183
|
+
try {
|
|
1184
|
+
await db.exec(`INSERT INTO ${records} (run_id, id, parent, revision, payload) VALUES (?, ?, ?, ?, ?)
|
|
1185
|
+
ON CONFLICT (run_id, id) DO NOTHING`, [
|
|
1186
|
+
runId,
|
|
1187
|
+
id,
|
|
1188
|
+
head,
|
|
1189
|
+
nextRevision,
|
|
1190
|
+
encoded
|
|
1191
|
+
]);
|
|
1192
|
+
} catch (error) {
|
|
1193
|
+
const [staged] = await db.query(`SELECT id, parent, revision, payload FROM ${records} WHERE run_id = ? AND id = ?`, [runId, id]).catch(() => []);
|
|
1194
|
+
if (!staged || staged.payload !== encoded || staged.parent !== head || Number(staged.revision) !== nextRevision) throw error;
|
|
1195
|
+
}
|
|
1196
|
+
controller.signal.throwIfAborted();
|
|
1197
|
+
let published = false;
|
|
1198
|
+
let failure;
|
|
1199
|
+
try {
|
|
1200
|
+
published = (await db.exec(`UPDATE ${heads} SET head = ?, revision = ?, pulse = pulse + 1
|
|
1201
|
+
WHERE run_id = ? AND owner = ? AND generation = ? AND head = ? AND revision = ?`, [
|
|
1202
|
+
id,
|
|
1203
|
+
nextRevision,
|
|
1204
|
+
runId,
|
|
1205
|
+
owner,
|
|
1206
|
+
generation,
|
|
1207
|
+
head,
|
|
1208
|
+
revision
|
|
1209
|
+
])).rowsAffected === 1;
|
|
1210
|
+
} catch (error) {
|
|
1211
|
+
failure = error;
|
|
1212
|
+
}
|
|
1213
|
+
if (!published) {
|
|
1214
|
+
const current = await readHead();
|
|
1215
|
+
if (current.owner !== owner || current.generation !== generation) throw new SqlRunOwnershipError(`SQL run '${runId}' lease was replaced before publication`);
|
|
1216
|
+
if (current.head !== id || current.revision !== nextRevision) throw failure ?? new SqlRunOwnershipError(`SQL run '${runId}' publication conflicted`);
|
|
1217
|
+
}
|
|
1218
|
+
controller.signal.throwIfAborted();
|
|
1219
|
+
head = id;
|
|
1220
|
+
revision = nextRevision;
|
|
1221
|
+
history.push(JSON.parse(encoded));
|
|
1222
|
+
progressAt = performance.now();
|
|
1223
|
+
});
|
|
1224
|
+
tail = write.catch(lose);
|
|
1225
|
+
return write;
|
|
1226
|
+
},
|
|
1227
|
+
release() {
|
|
1228
|
+
if (released) return released;
|
|
1229
|
+
accepting = false;
|
|
1230
|
+
released = (async () => {
|
|
1231
|
+
await tail;
|
|
1232
|
+
clearInterval(timer);
|
|
1233
|
+
try {
|
|
1234
|
+
await db.exec(`UPDATE ${heads} SET owner = '', pulse = pulse + 1 WHERE run_id = ? AND owner = ? AND generation = ?`, [
|
|
1235
|
+
runId,
|
|
1236
|
+
owner,
|
|
1237
|
+
generation
|
|
1238
|
+
]);
|
|
1239
|
+
} finally {
|
|
1240
|
+
controller.abort(new SqlRunOwnershipError(`SQL run '${runId}' lease is released`));
|
|
1241
|
+
}
|
|
1242
|
+
})();
|
|
1243
|
+
return released;
|
|
1244
|
+
}
|
|
1245
|
+
};
|
|
1246
|
+
try {
|
|
1247
|
+
history = await load(observed);
|
|
1248
|
+
signal?.throwIfAborted();
|
|
1249
|
+
controller.signal.throwIfAborted();
|
|
1250
|
+
return lease;
|
|
1251
|
+
} catch (error) {
|
|
1252
|
+
await lease.release().catch(() => void 0);
|
|
1253
|
+
throw error;
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
};
|
|
1257
|
+
}
|
|
1258
|
+
function recordId(parent, revision, payload) {
|
|
1259
|
+
return createHash("sha256").update(`${revision}\0${parent}\0${payload}`).digest("hex");
|
|
1260
|
+
}
|
|
1261
|
+
function wait(ms, signal) {
|
|
1262
|
+
return new Promise((resolve, reject) => {
|
|
1263
|
+
const cleanup = () => signal?.removeEventListener("abort", abort);
|
|
1264
|
+
const timer = setTimeout(() => {
|
|
1265
|
+
cleanup();
|
|
1266
|
+
resolve();
|
|
1267
|
+
}, ms);
|
|
1268
|
+
const abort = () => {
|
|
1269
|
+
clearTimeout(timer);
|
|
1270
|
+
cleanup();
|
|
1271
|
+
reject(signal?.reason);
|
|
1272
|
+
};
|
|
1273
|
+
if (signal?.aborted) abort();
|
|
1274
|
+
else signal?.addEventListener("abort", abort, { once: true });
|
|
1275
|
+
});
|
|
1276
|
+
}
|
|
1277
|
+
//#endregion
|
|
1018
1278
|
//#region src/runtime/audit-intent.ts
|
|
1019
1279
|
/** Default system instruction for intent-auditor agents: diagnose diverged/drifting trajectories. */
|
|
1020
1280
|
const defaultAuditorInstruction = "You audit whether an AI agent is on the RIGHT ROUTE — not whether it works hard, but whether its actions serve the stated intents. Infer the REVEALED intent from the action pattern (what the trajectory is actually optimizing). Compare against the declared task intent, the user intent when given, and the meta-intent when given. Flawless execution down the wrong route is DIVERGED. Busy-work that neither advances nor harms is DRIFTING. Judge only from the trajectory — be specific about which actions ground your verdict. Recommend abort only when continuing cannot serve the intent.";
|
|
@@ -1528,1529 +1788,1733 @@ function deterministicCompletion(check) {
|
|
|
1528
1788
|
} };
|
|
1529
1789
|
}
|
|
1530
1790
|
//#endregion
|
|
1531
|
-
//#region src/runtime/
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1791
|
+
//#region src/runtime/isolated-checker.ts
|
|
1792
|
+
const toolchains = [
|
|
1793
|
+
"/usr",
|
|
1794
|
+
"/bin",
|
|
1795
|
+
"/lib",
|
|
1796
|
+
"/lib64"
|
|
1797
|
+
];
|
|
1798
|
+
const contains = (parent, child) => {
|
|
1799
|
+
const path = relative(parent, child);
|
|
1800
|
+
return path === "" || path !== ".." && !path.startsWith(`..${sep}`) && !path.startsWith(sep);
|
|
1801
|
+
};
|
|
1802
|
+
/**
|
|
1803
|
+
* Run an untrusted check in Linux Bubblewrap, or in its own Sandbox box when `box` is set.
|
|
1804
|
+
* Never falls back to host execution.
|
|
1805
|
+
*
|
|
1806
|
+
* Bubblewrap requires /usr/bin/bwrap and permission to create Linux namespaces.
|
|
1807
|
+
* Only trusted system toolchains, private proc/dev/tmp, and the writable copy are mounted.
|
|
1808
|
+
* The canonical input path remains the working directory; copy writes are discarded.
|
|
1809
|
+
* Limits bound command time and captured output, not copy size or memory consumption.
|
|
1810
|
+
* Callers must keep the input and trusted toolchains stable while preparing the check.
|
|
1811
|
+
*
|
|
1812
|
+
* A box check creates one fresh box with no owner secrets and blocked egress (or strict egress to
|
|
1813
|
+
* the placement's named domains), delivers the tree's regular files verified by sha256, runs the
|
|
1814
|
+
* command there with only the environment `env` names, and deletes the box.
|
|
1815
|
+
*/
|
|
1816
|
+
async function runIsolatedCheck(options) {
|
|
1817
|
+
if (options.box) return runInBox(options, options.box);
|
|
1818
|
+
let scratch;
|
|
1819
|
+
let result;
|
|
1820
|
+
try {
|
|
1821
|
+
if (process.platform !== "linux") throw new Error("Linux Bubblewrap namespaces are required");
|
|
1822
|
+
const { timeoutMs, maxOutputBytes } = checkLimits(options);
|
|
1823
|
+
if (options.signal?.aborted) return {
|
|
1824
|
+
succeeded: false,
|
|
1825
|
+
reason: "cancelled",
|
|
1826
|
+
diagnostic: "Check cancelled"
|
|
1827
|
+
};
|
|
1828
|
+
const { workspace, tree } = await protectedTree(options);
|
|
1829
|
+
const binds = [];
|
|
1830
|
+
for (const path of toolchains) {
|
|
1831
|
+
let actual;
|
|
1832
|
+
try {
|
|
1833
|
+
actual = await realpath(path);
|
|
1834
|
+
} catch (error) {
|
|
1835
|
+
if (error.code === "ENOENT") continue;
|
|
1836
|
+
throw error;
|
|
1565
1837
|
}
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1838
|
+
for (const mount of [path, actual]) if (contains(mount, workspace) || contains(workspace, mount) || contains(mount, tree) || contains(tree, mount)) throw new Error(`Toolchain overlaps protected workspace: ${path}`);
|
|
1839
|
+
binds.push("--ro-bind", actual, path);
|
|
1840
|
+
}
|
|
1841
|
+
if (["/proc", "/dev"].some((path) => contains(path, tree) || contains(tree, path))) throw new Error("Tree overlaps a reserved namespace mount");
|
|
1842
|
+
scratch = await mkdtemp(join(tmpdir(), "runtime-check-"));
|
|
1843
|
+
if (contains(tree, scratch)) throw new Error("Temporary storage must be outside the input tree");
|
|
1844
|
+
const copy = join(scratch, "tree");
|
|
1845
|
+
await cp(tree, copy, {
|
|
1846
|
+
recursive: true,
|
|
1847
|
+
dereference: false,
|
|
1848
|
+
verbatimSymlinks: true
|
|
1849
|
+
});
|
|
1850
|
+
result = await execute([
|
|
1851
|
+
"--unshare-all",
|
|
1852
|
+
"--die-with-parent",
|
|
1853
|
+
"--new-session",
|
|
1854
|
+
"--cap-drop",
|
|
1855
|
+
"ALL",
|
|
1856
|
+
"--clearenv",
|
|
1857
|
+
"--setenv",
|
|
1858
|
+
"PATH",
|
|
1859
|
+
"/usr/bin:/bin",
|
|
1860
|
+
"--setenv",
|
|
1861
|
+
"HOME",
|
|
1862
|
+
"/tmp",
|
|
1863
|
+
"--setenv",
|
|
1864
|
+
"LANG",
|
|
1865
|
+
"C",
|
|
1866
|
+
...Object.entries(options.env ?? {}).flatMap(([name, value]) => [
|
|
1867
|
+
"--setenv",
|
|
1868
|
+
name,
|
|
1869
|
+
value
|
|
1870
|
+
]),
|
|
1871
|
+
...binds,
|
|
1872
|
+
"--proc",
|
|
1873
|
+
"/proc",
|
|
1874
|
+
"--dev",
|
|
1875
|
+
"/dev",
|
|
1876
|
+
"--tmpfs",
|
|
1877
|
+
"/tmp",
|
|
1878
|
+
"--bind",
|
|
1879
|
+
copy,
|
|
1880
|
+
tree,
|
|
1881
|
+
"--chdir",
|
|
1882
|
+
tree,
|
|
1883
|
+
"--",
|
|
1884
|
+
...options.command
|
|
1885
|
+
], timeoutMs, maxOutputBytes, options.signal);
|
|
1886
|
+
} catch (error) {
|
|
1887
|
+
result = {
|
|
1888
|
+
succeeded: false,
|
|
1889
|
+
reason: "refused",
|
|
1890
|
+
diagnostic: String(error)
|
|
1891
|
+
};
|
|
1892
|
+
}
|
|
1893
|
+
if (scratch) try {
|
|
1894
|
+
await removeScratch(scratch);
|
|
1895
|
+
} catch (error) {
|
|
1896
|
+
const cleanupDiagnostic = String(error);
|
|
1897
|
+
result = result.succeeded ? {
|
|
1898
|
+
succeeded: false,
|
|
1899
|
+
reason: "cleanup-failed",
|
|
1900
|
+
diagnostic: cleanupDiagnostic,
|
|
1901
|
+
stdout: result.value.stdout,
|
|
1902
|
+
stderr: result.value.stderr,
|
|
1903
|
+
exitCode: 0
|
|
1904
|
+
} : {
|
|
1905
|
+
...result,
|
|
1906
|
+
cleanupDiagnostic
|
|
1907
|
+
};
|
|
1908
|
+
}
|
|
1909
|
+
return result;
|
|
1580
1910
|
}
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
const
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
const classified = (cacheRead ?? 0) + (cacheWrite ?? 0);
|
|
1911
|
+
function checkLimits(options) {
|
|
1912
|
+
const timeoutMs = options.timeoutMs ?? 3e4;
|
|
1913
|
+
const maxOutputBytes = options.maxOutputBytes ?? 1048576;
|
|
1914
|
+
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 2147483647 || !Number.isSafeInteger(maxOutputBytes) || maxOutputBytes < 1) throw new Error("Invalid execution limits");
|
|
1915
|
+
if (!options.command.length || !options.command[0]) throw new Error("A command is required");
|
|
1587
1916
|
return {
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
outputTokens: usage.output,
|
|
1591
|
-
...cacheComplete ? {
|
|
1592
|
-
cachedTokens: cacheRead,
|
|
1593
|
-
cacheWriteTokens: cacheWrite
|
|
1594
|
-
} : {},
|
|
1595
|
-
...usage.tokensKnown === false || cost.usageUnknown ? { usageUnknown: true } : {},
|
|
1596
|
-
...cost.actualCostUsd !== void 0 ? { actualCostUsd: cost.actualCostUsd } : {},
|
|
1597
|
-
...cost.costUnknown ? { costUnknown: true } : {},
|
|
1598
|
-
...cost.estimatedCostUsd !== void 0 ? { estimatedCostUsd: cost.estimatedCostUsd } : {}
|
|
1917
|
+
timeoutMs,
|
|
1918
|
+
maxOutputBytes
|
|
1599
1919
|
};
|
|
1600
1920
|
}
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
* evidence record stays unknown, including zero-valued or incomplete spend.
|
|
1606
|
-
* @internal Testable identity projection used by superviseDispatch. */
|
|
1607
|
-
function supervisedTreeModelForDispatch(result, rootProfile) {
|
|
1608
|
-
const canonicalModels = /* @__PURE__ */ new Set();
|
|
1609
|
-
const observedModels = [];
|
|
1610
|
-
const wholeTreeEvidence = result.providerModel;
|
|
1611
|
-
if (wholeTreeEvidence === void 0 || wholeTreeEvidence.status !== "known" && wholeTreeEvidence.reason !== "provider-model-conflict" || wholeTreeEvidence.attempts.length === 0) return { kind: "unknown" };
|
|
1612
|
-
const wholeTreeIdentity = collectProviderModels(wholeTreeEvidence, () => true);
|
|
1613
|
-
if (wholeTreeIdentity.kind !== "known") return wholeTreeIdentity;
|
|
1614
|
-
for (const model of wholeTreeIdentity.observed) {
|
|
1615
|
-
observedModels.push(model.raw);
|
|
1616
|
-
canonicalModels.add(model.canonical);
|
|
1617
|
-
}
|
|
1618
|
-
if (canonicalModels.size === 0) return { kind: "unknown" };
|
|
1619
|
-
if (canonicalModels.size !== 1) return {
|
|
1620
|
-
kind: "mixed",
|
|
1621
|
-
models: [...canonicalModels].sort()
|
|
1622
|
-
};
|
|
1921
|
+
async function protectedTree(options) {
|
|
1922
|
+
const workspace = await realpath(options.workspaceRoot);
|
|
1923
|
+
const tree = await realpath(options.tree);
|
|
1924
|
+
if (!contains(workspace, tree) || !(await stat(tree)).isDirectory()) throw new Error("Tree must be a directory inside the workspace");
|
|
1623
1925
|
return {
|
|
1624
|
-
|
|
1625
|
-
|
|
1926
|
+
workspace,
|
|
1927
|
+
tree
|
|
1626
1928
|
};
|
|
1627
1929
|
}
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1930
|
+
/** Box-workspace-relative directory that receives the tree and is the command's cwd. */
|
|
1931
|
+
const BOX_TREE = "tree";
|
|
1932
|
+
/** Bound on provisioning, delivery, and deletion; the box expires even if this process dies. */
|
|
1933
|
+
const BOX_SETUP_SECONDS = 900;
|
|
1934
|
+
/** The box's own process timeout is a backstop behind this module's deadline. */
|
|
1935
|
+
const BOX_PROCESS_BACKSTOP_MS = 3e4;
|
|
1936
|
+
/** Concurrent file deliveries into one check box. */
|
|
1937
|
+
const BOX_UPLOADS = 4;
|
|
1938
|
+
/** The box's process supervisor prefixes this to stderr when it cannot start the executable. */
|
|
1939
|
+
const BOX_EXEC_FAILURE = "process-supervisor: exec failed";
|
|
1940
|
+
async function runInBox(options, placement) {
|
|
1941
|
+
let box;
|
|
1942
|
+
let evidence;
|
|
1943
|
+
let result;
|
|
1944
|
+
try {
|
|
1945
|
+
const { timeoutMs, maxOutputBytes } = checkLimits(options);
|
|
1946
|
+
if (options.signal?.aborted) return {
|
|
1947
|
+
succeeded: false,
|
|
1948
|
+
reason: "cancelled",
|
|
1949
|
+
diagnostic: "Check cancelled"
|
|
1950
|
+
};
|
|
1951
|
+
const { tree } = await protectedTree(options);
|
|
1952
|
+
const captured = await captureMaterializedWorkspace(tree);
|
|
1953
|
+
if (placement.builderAccounts.length === 0 || placement.builderAccounts.some((id) => !id)) throw new Error("builderAccounts must name the account of every key the judged run holds");
|
|
1954
|
+
const account = (await placement.client.getIdentity()).customerId;
|
|
1955
|
+
if (!account) throw new Error("Sandbox returned no account for the check key");
|
|
1956
|
+
if (placement.builderAccounts.includes(account)) throw new Error(`Check account ${account} is an account the judged run holds a key to, so the run can reach the check box`);
|
|
1957
|
+
options.signal?.throwIfAborted();
|
|
1958
|
+
box = await placement.client.createIsolated({
|
|
1959
|
+
environment: placement.environment,
|
|
1960
|
+
agent: false,
|
|
1961
|
+
bare: false,
|
|
1962
|
+
ephemeral: true,
|
|
1963
|
+
egressPolicy: egressFor(placement),
|
|
1964
|
+
...placement.resources ? { resources: placement.resources } : {},
|
|
1965
|
+
maxLifetimeSeconds: Math.ceil(timeoutMs / 1e3) + BOX_SETUP_SECONDS,
|
|
1966
|
+
idempotencyKey: `agent-runtime-check-${randomUUID()}`
|
|
1967
|
+
}, options.signal ? { signal: options.signal } : void 0);
|
|
1968
|
+
const receipt = box.createReceipt();
|
|
1969
|
+
if (receipt?.outcome !== "created" || receipt.ownerContext !== "isolated" || !Array.isArray(receipt.injectedSecrets) || receipt.injectedSecrets.length !== 0) throw new Error("Sandbox did not confirm a fresh box with no owner secrets");
|
|
1970
|
+
const egress = (await box.egress.get()).policy;
|
|
1971
|
+
const wanted = egressFor(placement);
|
|
1972
|
+
if (egress.mode !== wanted.mode || wanted.mode === "strict" && (egress.includeImplicitDomains === true || [...egress.allowDomains ?? []].sort().join(",") !== [...wanted.allowDomains ?? []].sort().join(","))) throw new Error(`Check box egress is ${egress.mode} ${JSON.stringify(egress.allowDomains ?? [])}, not ${wanted.mode} ${JSON.stringify(wanted.allowDomains ?? [])}`);
|
|
1973
|
+
await box.fs.mkdir(BOX_TREE, { recursive: true });
|
|
1974
|
+
const digests = new Map(captured.manifest.files.map((entry) => [entry.path, entry.sha256]));
|
|
1975
|
+
const target = box;
|
|
1976
|
+
const pending = [...captured.files];
|
|
1977
|
+
const deliver = async () => {
|
|
1978
|
+
try {
|
|
1979
|
+
for (let file = pending.shift(); file; file = pending.shift()) {
|
|
1980
|
+
options.signal?.throwIfAborted();
|
|
1981
|
+
const received = await target.fs.uploadData(`${BOX_TREE}/${file.path}`, file.bytes, { mode: file.mode });
|
|
1982
|
+
if (`sha256:${received.hash.replace(/^sha256:/, "").toLowerCase()}` !== digests.get(file.path) || received.size !== file.bytes.byteLength) throw new Error(`Check box received different bytes for ${file.path}`);
|
|
1983
|
+
}
|
|
1984
|
+
} catch (error) {
|
|
1985
|
+
pending.length = 0;
|
|
1986
|
+
throw error;
|
|
1659
1987
|
}
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1988
|
+
};
|
|
1989
|
+
await Promise.all(Array.from({ length: BOX_UPLOADS }, deliver));
|
|
1990
|
+
evidence = {
|
|
1991
|
+
sandboxId: box.id,
|
|
1992
|
+
account,
|
|
1993
|
+
input: captured.manifest,
|
|
1994
|
+
inputDigest: canonicalCandidateDigest(captured.manifest)
|
|
1995
|
+
};
|
|
1996
|
+
result = await executeInBox(box, options.command, timeoutMs, maxOutputBytes, options.env ?? {}, options.signal);
|
|
1997
|
+
} catch (error) {
|
|
1998
|
+
result = {
|
|
1999
|
+
succeeded: false,
|
|
2000
|
+
reason: options.signal?.aborted ? "cancelled" : "refused",
|
|
2001
|
+
diagnostic: String(error)
|
|
2002
|
+
};
|
|
1673
2003
|
}
|
|
1674
|
-
if (
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
2004
|
+
if (box) try {
|
|
2005
|
+
await box.delete();
|
|
2006
|
+
} catch (error) {
|
|
2007
|
+
const cleanupDiagnostic = `Check box ${box.id} was not deleted: ${String(error)}`;
|
|
2008
|
+
result = result.succeeded ? {
|
|
2009
|
+
succeeded: false,
|
|
2010
|
+
reason: "cleanup-failed",
|
|
2011
|
+
diagnostic: cleanupDiagnostic,
|
|
2012
|
+
stdout: result.value.stdout,
|
|
2013
|
+
stderr: result.value.stderr,
|
|
2014
|
+
exitCode: 0
|
|
2015
|
+
} : {
|
|
2016
|
+
...result,
|
|
2017
|
+
cleanupDiagnostic
|
|
2018
|
+
};
|
|
1679
2019
|
}
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
} : {
|
|
1685
|
-
kind: "known",
|
|
1686
|
-
observed
|
|
1687
|
-
};
|
|
2020
|
+
return evidence ? {
|
|
2021
|
+
...result,
|
|
2022
|
+
box: evidence
|
|
2023
|
+
} : result;
|
|
1688
2024
|
}
|
|
1689
|
-
|
|
1690
|
-
|
|
2025
|
+
/** Blocked unless the placement names domains; then strict, with no implicit domain list. */
|
|
2026
|
+
function egressFor(placement) {
|
|
2027
|
+
const domains = [...new Set(placement.egress ?? [])].filter((domain) => domain.trim() !== "");
|
|
2028
|
+
return domains.length === 0 ? { mode: "blocked" } : {
|
|
2029
|
+
mode: "strict",
|
|
2030
|
+
allowDomains: domains,
|
|
2031
|
+
includeImplicitDomains: false
|
|
2032
|
+
};
|
|
1691
2033
|
}
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
2034
|
+
async function executeInBox(box, command, timeoutMs, limit, env, signal) {
|
|
2035
|
+
const [executable, ...args] = command;
|
|
2036
|
+
const child = await box.process.spawnExact(executable, args, {
|
|
2037
|
+
cwd: BOX_TREE,
|
|
2038
|
+
env: {
|
|
2039
|
+
PATH: "/usr/bin:/bin",
|
|
2040
|
+
HOME: "/tmp",
|
|
2041
|
+
LANG: "C",
|
|
2042
|
+
...env
|
|
2043
|
+
},
|
|
2044
|
+
inheritEnv: false,
|
|
2045
|
+
timeoutMs: timeoutMs + BOX_PROCESS_BACKSTOP_MS
|
|
2046
|
+
});
|
|
2047
|
+
const stdout = [];
|
|
2048
|
+
const stderr = [];
|
|
2049
|
+
let bytes = 0;
|
|
2050
|
+
let failure;
|
|
2051
|
+
let stopping;
|
|
2052
|
+
const stop = (reason) => {
|
|
2053
|
+
failure ??= reason;
|
|
2054
|
+
stopping ??= child.kill("SIGKILL", { tree: true }).catch(() => void 0);
|
|
2055
|
+
};
|
|
2056
|
+
const clearTimer = armDeadlineTimer(timeoutMs, () => stop("timeout"), true);
|
|
2057
|
+
const cancel = () => stop("cancelled");
|
|
2058
|
+
signal?.addEventListener("abort", cancel, { once: true });
|
|
2059
|
+
if (signal?.aborted) cancel();
|
|
2060
|
+
const capture = async (stream, target) => {
|
|
2061
|
+
for await (const text of stream) {
|
|
2062
|
+
const chunk = Buffer.from(text);
|
|
2063
|
+
const remaining = Math.max(0, limit - bytes);
|
|
2064
|
+
bytes += chunk.length;
|
|
2065
|
+
if (remaining) target.push(chunk.subarray(0, remaining));
|
|
2066
|
+
if (bytes > limit) stop("output-limit");
|
|
2067
|
+
}
|
|
2068
|
+
};
|
|
2069
|
+
let exitCode;
|
|
2070
|
+
let exitSignal;
|
|
2071
|
+
try {
|
|
2072
|
+
await Promise.all([capture(child.stdout(), stdout), capture(child.stderr(), stderr)]);
|
|
2073
|
+
exitCode = await child.wait();
|
|
2074
|
+
exitSignal = (await child.status()).exitSignal;
|
|
2075
|
+
} finally {
|
|
2076
|
+
clearTimer();
|
|
2077
|
+
signal?.removeEventListener("abort", cancel);
|
|
2078
|
+
await stopping;
|
|
1698
2079
|
}
|
|
1699
|
-
|
|
1700
|
-
|
|
2080
|
+
const out = Buffer.concat(stdout).toString();
|
|
2081
|
+
const err = Buffer.concat(stderr).toString();
|
|
2082
|
+
const evidence = {
|
|
2083
|
+
stdout: out,
|
|
2084
|
+
stderr: err,
|
|
2085
|
+
exitCode: exitSignal ? null : exitCode
|
|
2086
|
+
};
|
|
2087
|
+
if (failure) return {
|
|
2088
|
+
succeeded: false,
|
|
2089
|
+
reason: failure,
|
|
2090
|
+
diagnostic: err || failure,
|
|
2091
|
+
...evidence
|
|
2092
|
+
};
|
|
2093
|
+
if (exitCode === 127 && err.startsWith(BOX_EXEC_FAILURE)) return {
|
|
2094
|
+
succeeded: false,
|
|
2095
|
+
reason: "refused",
|
|
2096
|
+
diagnostic: err,
|
|
2097
|
+
...evidence
|
|
2098
|
+
};
|
|
2099
|
+
if (exitSignal) return {
|
|
2100
|
+
succeeded: false,
|
|
2101
|
+
reason: "failed",
|
|
2102
|
+
diagnostic: err || `Check killed by ${exitSignal}`,
|
|
2103
|
+
...evidence
|
|
2104
|
+
};
|
|
2105
|
+
if (exitCode !== 0) return {
|
|
2106
|
+
succeeded: false,
|
|
2107
|
+
reason: "failed",
|
|
2108
|
+
diagnostic: err || out || `Check exited ${exitCode}`,
|
|
2109
|
+
...evidence
|
|
2110
|
+
};
|
|
1701
2111
|
return {
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
2112
|
+
succeeded: true,
|
|
2113
|
+
value: {
|
|
2114
|
+
stdout: out,
|
|
2115
|
+
stderr: err
|
|
2116
|
+
}
|
|
1707
2117
|
};
|
|
1708
2118
|
}
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
signal: executionSignal
|
|
1725
|
-
});
|
|
1726
|
-
const identity = supervisedTreeModelForDispatch(result, profile);
|
|
1727
|
-
if (identity.kind !== "known") throw new SupervisedTreeModelIdentityError(identity);
|
|
1728
|
-
return result;
|
|
1729
|
-
},
|
|
1730
|
-
receipt: (result) => {
|
|
1731
|
-
const identity = supervisedTreeModelForDispatch(result, profile);
|
|
1732
|
-
if (identity.kind !== "known") throw new SupervisedTreeModelIdentityError(identity);
|
|
1733
|
-
return costReceiptFromUsage(result.spentTotal.tokens, identity.model, {
|
|
1734
|
-
usageUnknown: result.spentTotal.tokensKnown === false,
|
|
1735
|
-
actualCostUsd: result.spentTotal.usdKnown !== false ? result.spentTotal.usd : void 0,
|
|
1736
|
-
costUnknown: result.spentTotal.usdKnown === false
|
|
1737
|
-
});
|
|
1738
|
-
},
|
|
1739
|
-
receiptFromError: (error) => error instanceof SupervisedTreeModelIdentityError ? unknownSupervisedTreeReceipt() : void 0
|
|
2119
|
+
function execute(args, timeoutMs, limit, signal) {
|
|
2120
|
+
return new Promise((done) => {
|
|
2121
|
+
const child = spawn("/usr/bin/bwrap", [
|
|
2122
|
+
"--json-status-fd",
|
|
2123
|
+
"3",
|
|
2124
|
+
...args
|
|
2125
|
+
], {
|
|
2126
|
+
env: {},
|
|
2127
|
+
stdio: [
|
|
2128
|
+
"ignore",
|
|
2129
|
+
"pipe",
|
|
2130
|
+
"pipe",
|
|
2131
|
+
"pipe"
|
|
2132
|
+
],
|
|
2133
|
+
detached: true
|
|
1740
2134
|
});
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
2135
|
+
const stdout = [];
|
|
2136
|
+
const stderr = [];
|
|
2137
|
+
let bytes = 0;
|
|
2138
|
+
let status = "";
|
|
2139
|
+
let failure;
|
|
2140
|
+
const stop = (reason) => {
|
|
2141
|
+
failure ??= reason;
|
|
2142
|
+
if (child.pid) try {
|
|
2143
|
+
process.kill(-child.pid, "SIGKILL");
|
|
2144
|
+
} catch (error) {
|
|
2145
|
+
if (error.code !== "ESRCH") child.kill("SIGKILL");
|
|
2146
|
+
}
|
|
2147
|
+
};
|
|
2148
|
+
const clearTimer = armDeadlineTimer(timeoutMs, () => stop("timeout"), true);
|
|
2149
|
+
const cancel = () => stop("cancelled");
|
|
2150
|
+
signal?.addEventListener("abort", cancel, { once: true });
|
|
2151
|
+
if (signal?.aborted) cancel();
|
|
2152
|
+
const capture = (target) => (chunk) => {
|
|
2153
|
+
const remaining = Math.max(0, limit - bytes);
|
|
2154
|
+
bytes += chunk.length;
|
|
2155
|
+
if (remaining) target.push(chunk.subarray(0, remaining));
|
|
2156
|
+
if (bytes > limit) stop("output-limit");
|
|
2157
|
+
};
|
|
2158
|
+
child.stdio[3]?.on("data", (chunk) => {
|
|
2159
|
+
if (status.length + chunk.length > 16384) {
|
|
2160
|
+
stop("output-limit");
|
|
2161
|
+
return;
|
|
2162
|
+
}
|
|
2163
|
+
status += chunk.toString();
|
|
2164
|
+
});
|
|
2165
|
+
child.stdout.on("data", capture(stdout));
|
|
2166
|
+
child.stderr.on("data", capture(stderr));
|
|
2167
|
+
let spawnError;
|
|
2168
|
+
child.on("error", (error) => {
|
|
2169
|
+
spawnError = error;
|
|
2170
|
+
});
|
|
2171
|
+
child.on("close", (code) => {
|
|
2172
|
+
clearTimer();
|
|
2173
|
+
signal?.removeEventListener("abort", cancel);
|
|
2174
|
+
const out = Buffer.concat(stdout).toString();
|
|
2175
|
+
const err = Buffer.concat(stderr).toString();
|
|
2176
|
+
const evidence = {
|
|
2177
|
+
stdout: out,
|
|
2178
|
+
stderr: err,
|
|
2179
|
+
exitCode: code
|
|
2180
|
+
};
|
|
2181
|
+
const executed = status.split("\n").some((line) => {
|
|
2182
|
+
try {
|
|
2183
|
+
const record = JSON.parse(line);
|
|
2184
|
+
return record !== null && typeof record === "object" && "exit-code" in record && typeof record["exit-code"] === "number" && record["exit-code"] === code;
|
|
2185
|
+
} catch {
|
|
2186
|
+
return false;
|
|
2187
|
+
}
|
|
2188
|
+
});
|
|
2189
|
+
if (failure) done({
|
|
2190
|
+
succeeded: false,
|
|
2191
|
+
reason: failure,
|
|
2192
|
+
diagnostic: err || failure,
|
|
2193
|
+
...evidence
|
|
2194
|
+
});
|
|
2195
|
+
else if (spawnError) done({
|
|
2196
|
+
succeeded: false,
|
|
2197
|
+
reason: "refused",
|
|
2198
|
+
diagnostic: String(spawnError),
|
|
2199
|
+
...evidence
|
|
2200
|
+
});
|
|
2201
|
+
else if (!executed) done({
|
|
2202
|
+
succeeded: false,
|
|
2203
|
+
reason: "refused",
|
|
2204
|
+
diagnostic: err || "Bubblewrap did not confirm command execution",
|
|
2205
|
+
...evidence
|
|
2206
|
+
});
|
|
2207
|
+
else if (code !== 0) done({
|
|
2208
|
+
succeeded: false,
|
|
2209
|
+
reason: "failed",
|
|
2210
|
+
diagnostic: err || out || `Check exited ${code}`,
|
|
2211
|
+
...evidence
|
|
2212
|
+
});
|
|
2213
|
+
else done({
|
|
2214
|
+
succeeded: true,
|
|
2215
|
+
value: {
|
|
2216
|
+
stdout: out,
|
|
2217
|
+
stderr: err
|
|
2218
|
+
}
|
|
2219
|
+
});
|
|
2220
|
+
});
|
|
2221
|
+
});
|
|
2222
|
+
}
|
|
2223
|
+
const executeFile = promisify(execFile);
|
|
2224
|
+
/** GNU tools traverse relative to open directories, including trees deeper than PATH_MAX.
|
|
2225
|
+
* Recursive chmod ignores encountered symlinks; rm never follows their targets. */
|
|
2226
|
+
async function removeScratch(path) {
|
|
2227
|
+
const options = {
|
|
2228
|
+
env: {},
|
|
2229
|
+
timeout: 1e4,
|
|
2230
|
+
maxBuffer: 16384
|
|
1746
2231
|
};
|
|
2232
|
+
let permissionError;
|
|
2233
|
+
try {
|
|
2234
|
+
await executeFile("/bin/chmod", [
|
|
2235
|
+
"--recursive",
|
|
2236
|
+
"--preserve-root",
|
|
2237
|
+
"u+rwX",
|
|
2238
|
+
"--",
|
|
2239
|
+
path
|
|
2240
|
+
], options);
|
|
2241
|
+
} catch (error) {
|
|
2242
|
+
permissionError = error;
|
|
2243
|
+
}
|
|
2244
|
+
try {
|
|
2245
|
+
await executeFile("/bin/rm", [
|
|
2246
|
+
"--recursive",
|
|
2247
|
+
"--force",
|
|
2248
|
+
"--one-file-system",
|
|
2249
|
+
"--",
|
|
2250
|
+
path
|
|
2251
|
+
], options);
|
|
2252
|
+
} catch (error) {
|
|
2253
|
+
throw new Error([permissionError, error].filter((value) => value !== void 0).map(String).join("; "));
|
|
2254
|
+
}
|
|
1747
2255
|
}
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
2256
|
+
//#endregion
|
|
2257
|
+
//#region src/runtime/declared-check.ts
|
|
2258
|
+
/**
|
|
2259
|
+
* A declared check: the one frozen evaluator program that decides "done" for a run, named by
|
|
2260
|
+
* digest, run by Runtime in a fresh box on the check account.
|
|
2261
|
+
*
|
|
2262
|
+
* The same program is read three times (discovery `docs/38-one-loop-and-continuation.md`, section
|
|
2263
|
+
* 1): inside the run on every `submit_result` and at a turn end with no accepted result, after the
|
|
2264
|
+
* run to score it, and after each version to rank versions. Before this, each play wrote its own
|
|
2265
|
+
* check as Lab code in a method bundle — an exact-check judge, a sealed-cases judge, a product
|
|
2266
|
+
* judge — and the in-run check was a separate, weaker "shape and persistence" test, so 22 directors
|
|
2267
|
+
* passed their own check and then failed the outside one.
|
|
2268
|
+
*
|
|
2269
|
+
* The program prints one agent-eval `JudgeScore` as JSON on its last line of stdout: each checked
|
|
2270
|
+
* item is a dimension, `composite` is the score, and `notes` holds one `FAIL <item> <where>:
|
|
2271
|
+
* <reason>` line per failed item. The record states the pass threshold. A program that does not
|
|
2272
|
+
* run, or prints no score, gives no verdict: it is {@link CheckUnavailableError}, and the loop
|
|
2273
|
+
* pauses instead of blaming the director.
|
|
2274
|
+
*
|
|
2275
|
+
* The box receives the program's files at its working directory, the submitted result at
|
|
2276
|
+
* `_input/result.json` when there is one, and the sealed cases at `_sealed/` only when the read
|
|
2277
|
+
* scores a run or a version. `CHECK_SET` is `development` or `sealed`; `CHECK_RESULT` names the
|
|
2278
|
+
* result file when present. Sealed cases never reach an in-run read, so their lines never reach
|
|
2279
|
+
* the director.
|
|
2280
|
+
*/
|
|
2281
|
+
const INPUT_DIR = "_input";
|
|
2282
|
+
const SEALED_DIR = "_sealed";
|
|
2283
|
+
const DEFAULT_TIMEOUT_MS = 15 * 6e4;
|
|
2284
|
+
/** Refuse a malformed declaration before any compute. */
|
|
2285
|
+
function assertDeclaredCheck(check, context) {
|
|
2286
|
+
const fail = (message) => {
|
|
2287
|
+
throw new ValidationError(`${context}: check ${message}`);
|
|
2288
|
+
};
|
|
2289
|
+
if (typeof check !== "object" || check === null) fail("must be an object");
|
|
2290
|
+
if (typeof check.program?.dir !== "string" || check.program.dir.trim() === "") fail("program.dir must name the evaluator directory");
|
|
2291
|
+
if (!/^sha256:[0-9a-f]{64}$/u.test(check.program?.digest ?? "")) fail("program.digest must be a sha256 digest");
|
|
2292
|
+
if (!Array.isArray(check.command) || check.command.length === 0 || !check.command[0]) fail("command must name an executable");
|
|
2293
|
+
if (typeof check.environment !== "string" || check.environment.trim() === "") fail("environment must name the box image");
|
|
2294
|
+
if (typeof check.pass !== "number" || !Number.isFinite(check.pass)) fail("pass must be a finite threshold on the composite");
|
|
2295
|
+
if (check.feedback !== "verbatim" && check.feedback !== "pass-only") fail("feedback must be 'verbatim' or 'pass-only'");
|
|
2296
|
+
if (check.sealed !== void 0 && !/^sha256:[0-9a-f]{64}$/u.test(check.sealed.digest ?? "")) fail("sealed.digest must be a sha256 digest");
|
|
2297
|
+
for (const name of check.secrets ?? []) if (!/^[A-Z_][A-Z0-9_]*$/u.test(name)) fail(`secret name ${JSON.stringify(name)} is not an env name`);
|
|
2298
|
+
}
|
|
2299
|
+
/** The canonical digest of a directory's files: the digest a record names a program by. */
|
|
2300
|
+
async function checkProgramDigest(dir) {
|
|
2301
|
+
return canonicalCandidateDigest((await captureMaterializedWorkspace(dir)).manifest);
|
|
2302
|
+
}
|
|
2303
|
+
/** The digest a version judge records: the program, the sealed cases, and how they are run. */
|
|
2304
|
+
function declaredCheckDigest(check) {
|
|
2305
|
+
return sha256Bytes(Buffer.from(JSON.stringify({
|
|
2306
|
+
program: check.program.digest,
|
|
2307
|
+
sealed: check.sealed?.digest ?? null,
|
|
2308
|
+
command: check.command,
|
|
2309
|
+
environment: check.environment,
|
|
2310
|
+
pass: check.pass
|
|
2311
|
+
})));
|
|
1753
2312
|
}
|
|
1754
2313
|
/**
|
|
1755
|
-
*
|
|
1756
|
-
*
|
|
1757
|
-
*
|
|
1758
|
-
* automatic.
|
|
2314
|
+
* Read the check once. `result` is the submitted result, absent for a read of the run's state.
|
|
2315
|
+
* `set: 'sealed'` adds the sealed cases. Throws {@link CheckUnavailableError} when the program
|
|
2316
|
+
* could not run or printed no score.
|
|
1759
2317
|
*/
|
|
1760
|
-
function
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
2318
|
+
async function readDeclaredCheck(check, placement, read) {
|
|
2319
|
+
if (read.set === "sealed" && check.sealed === void 0) throw new ValidationError("readDeclaredCheck: a sealed read needs check.sealed");
|
|
2320
|
+
const environment = placement.env ?? process.env;
|
|
2321
|
+
const secrets = {};
|
|
2322
|
+
for (const name of check.secrets ?? []) {
|
|
2323
|
+
const value = environment[name];
|
|
2324
|
+
if (value === void 0 || value === "") throw new CheckUnavailableError(`the check's secret ${name} is not set on this host`);
|
|
2325
|
+
secrets[name] = value;
|
|
2326
|
+
}
|
|
2327
|
+
const workspace = await mkdtemp(join(tmpdir(), "declared-check-"));
|
|
2328
|
+
try {
|
|
2329
|
+
const tree = join(workspace, "tree");
|
|
2330
|
+
await copyVerified(check.program.dir, check.program.digest, tree, "program");
|
|
2331
|
+
const entries = await readdir(tree);
|
|
2332
|
+
if (entries.includes(INPUT_DIR) || entries.includes(SEALED_DIR)) throw new ValidationError(`readDeclaredCheck: the program may not hold ${INPUT_DIR}/ or ${SEALED_DIR}/; Runtime mounts them`);
|
|
2333
|
+
const env = {
|
|
2334
|
+
...secrets,
|
|
2335
|
+
CHECK_SET: read.set
|
|
2336
|
+
};
|
|
2337
|
+
if (read.result !== void 0) {
|
|
2338
|
+
await mkdir(join(tree, INPUT_DIR));
|
|
2339
|
+
await writeFile(join(tree, INPUT_DIR, "result.json"), `${JSON.stringify(read.result)}\n`);
|
|
2340
|
+
env.CHECK_RESULT = `${INPUT_DIR}/result.json`;
|
|
2341
|
+
}
|
|
2342
|
+
if (read.set === "sealed" && check.sealed !== void 0) await copyVerified(check.sealed.dir, check.sealed.digest, join(tree, SEALED_DIR), "sealed cases");
|
|
2343
|
+
return verdictOf(await runIsolatedCheck({
|
|
2344
|
+
workspaceRoot: workspace,
|
|
2345
|
+
tree,
|
|
2346
|
+
command: check.command,
|
|
2347
|
+
timeoutMs: check.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
2348
|
+
env,
|
|
2349
|
+
...read.signal === void 0 ? {} : { signal: read.signal },
|
|
2350
|
+
box: {
|
|
2351
|
+
client: placement.client,
|
|
2352
|
+
builderAccounts: placement.builderAccounts,
|
|
2353
|
+
environment: check.environment,
|
|
2354
|
+
...check.egress === void 0 ? {} : { egress: check.egress },
|
|
2355
|
+
...check.resources === void 0 ? {} : { resources: check.resources }
|
|
2356
|
+
}
|
|
2357
|
+
}), check.pass);
|
|
2358
|
+
} finally {
|
|
2359
|
+
await rm(workspace, {
|
|
2360
|
+
recursive: true,
|
|
2361
|
+
force: true
|
|
1767
2362
|
});
|
|
2363
|
+
}
|
|
2364
|
+
}
|
|
2365
|
+
/** The declared check as a manager's completion check: every in-run read uses development cases. */
|
|
2366
|
+
function declaredCheckDeliverable(check, placement) {
|
|
2367
|
+
assertDeclaredCheck(check, "declaredCheckDeliverable");
|
|
2368
|
+
return {
|
|
2369
|
+
check: (result) => readDeclaredCheck(check, placement, {
|
|
2370
|
+
result,
|
|
2371
|
+
set: "development"
|
|
2372
|
+
}),
|
|
2373
|
+
checkState: () => readDeclaredCheck(check, placement, { set: "development" }),
|
|
2374
|
+
feedback: check.feedback,
|
|
2375
|
+
...check.sealed === void 0 ? {} : { sealed: true },
|
|
2376
|
+
...check.describe === void 0 ? {} : { describe: check.describe }
|
|
1768
2377
|
};
|
|
1769
2378
|
}
|
|
1770
2379
|
/**
|
|
1771
|
-
*
|
|
1772
|
-
*
|
|
1773
|
-
*
|
|
2380
|
+
* The declared check as a version chain's judge: it scores a settled version on its sealed cases
|
|
2381
|
+
* when the record has them, and on its development cases otherwise. The per-item verdict rides in
|
|
2382
|
+
* the ledger, so the next version's review names each item. A read that could not run scores
|
|
2383
|
+
* `null`, which never counts as an improvement.
|
|
1774
2384
|
*/
|
|
1775
|
-
function
|
|
1776
|
-
|
|
2385
|
+
function declaredCheckJudge(check, placement) {
|
|
2386
|
+
assertDeclaredCheck(check, "declaredCheckJudge");
|
|
2387
|
+
const digest = declaredCheckDigest(check);
|
|
2388
|
+
return {
|
|
2389
|
+
digest,
|
|
2390
|
+
async judge(version, signal) {
|
|
2391
|
+
try {
|
|
2392
|
+
const verdict = await readDeclaredCheck(check, placement, {
|
|
2393
|
+
...version.result.kind === "winner" ? { result: version.result.out } : {},
|
|
2394
|
+
set: check.sealed === void 0 ? "development" : "sealed",
|
|
2395
|
+
signal
|
|
2396
|
+
});
|
|
2397
|
+
return {
|
|
2398
|
+
score: verdict.composite ?? (verdict.pass ? 1 : 0),
|
|
2399
|
+
judgeDigest: digest,
|
|
2400
|
+
check: verdict
|
|
2401
|
+
};
|
|
2402
|
+
} catch (error) {
|
|
2403
|
+
if (!(error instanceof CheckUnavailableError)) throw error;
|
|
2404
|
+
return {
|
|
2405
|
+
score: null,
|
|
2406
|
+
judgeDigest: digest,
|
|
2407
|
+
detail: { unavailable: error.message }
|
|
2408
|
+
};
|
|
2409
|
+
}
|
|
2410
|
+
}
|
|
2411
|
+
};
|
|
2412
|
+
}
|
|
2413
|
+
function verdictOf(outcome, pass) {
|
|
2414
|
+
if (!outcome.succeeded) throw new CheckUnavailableError(`the check program ${outcome.reason}: ${outcome.diagnostic.slice(0, 2e3)}`);
|
|
2415
|
+
const last = outcome.value.stdout.trimEnd().split("\n").at(-1) ?? "";
|
|
2416
|
+
let score;
|
|
2417
|
+
try {
|
|
2418
|
+
score = JSON.parse(last);
|
|
2419
|
+
} catch {
|
|
2420
|
+
throw new CheckUnavailableError("the check program printed no JudgeScore on its last line");
|
|
2421
|
+
}
|
|
2422
|
+
const raw = score;
|
|
2423
|
+
if (typeof raw !== "object" || raw === null || typeof raw.dimensions !== "object" || raw.dimensions === null || typeof raw.composite !== "number" || typeof raw.notes !== "string") throw new CheckUnavailableError("the check program printed a line that is not a JudgeScore");
|
|
2424
|
+
return verdictFromJudgeScore({
|
|
2425
|
+
dimensions: raw.dimensions,
|
|
2426
|
+
composite: raw.composite,
|
|
2427
|
+
notes: raw.notes,
|
|
2428
|
+
...raw.failed === true ? { failed: true } : {}
|
|
2429
|
+
}, pass);
|
|
2430
|
+
}
|
|
2431
|
+
/** Copy exactly the bytes whose digest was checked: the files are read once, verified, and those
|
|
2432
|
+
* bytes are written, so a file changed after the check cannot reach the box. */
|
|
2433
|
+
async function copyVerified(from, digest, to, label) {
|
|
2434
|
+
const captured = await captureMaterializedWorkspace(from);
|
|
2435
|
+
const actual = canonicalCandidateDigest(captured.manifest);
|
|
2436
|
+
if (actual !== digest) throw new ValidationError(`readDeclaredCheck: the ${label} at ${from} is ${actual}; the record names ${digest}`);
|
|
2437
|
+
for (const file of captured.files) {
|
|
2438
|
+
const path = join(to, file.path);
|
|
2439
|
+
await mkdir(dirname(path), { recursive: true });
|
|
2440
|
+
await writeFile(path, file.bytes, { mode: file.mode });
|
|
2441
|
+
}
|
|
1777
2442
|
}
|
|
1778
2443
|
//#endregion
|
|
1779
|
-
//#region src/runtime/
|
|
1780
|
-
/**
|
|
1781
|
-
*
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
* `done` event. The result carries output and usage; `done` confirms execution settled.
|
|
1791
|
-
* There are no sessions, no fs,
|
|
1792
|
-
* no fork (those degrade gracefully via the optional `SandboxClient` methods).
|
|
1793
|
-
*/
|
|
1794
|
-
function isAsyncIterable$1(v) {
|
|
1795
|
-
return typeof v === "object" && v !== null && Symbol.asyncIterator in v;
|
|
2444
|
+
//#region src/runtime/loop-dispatch.ts
|
|
2445
|
+
/** Bridge a campaign `DispatchContext.trace` to a `LoopTraceEmitter` so every
|
|
2446
|
+
* `loop.*` event lands as a span under the cell's scoped trace. */
|
|
2447
|
+
function campaignTraceToLoopEmitter(trace) {
|
|
2448
|
+
return { emit(event) {
|
|
2449
|
+
trace.span(event.kind, {
|
|
2450
|
+
runId: event.runId,
|
|
2451
|
+
timestamp: event.timestamp,
|
|
2452
|
+
...event.payload
|
|
2453
|
+
}).end();
|
|
2454
|
+
} };
|
|
1796
2455
|
}
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
}
|
|
1804
|
-
return r;
|
|
2456
|
+
async function runLoopForCell(opts, scenario, profile, ctx) {
|
|
2457
|
+
const loopOptions = opts.toLoopOptions(scenario, profile);
|
|
2458
|
+
return runLoopWithCampaignContext(opts, loopOptions, ctx, {
|
|
2459
|
+
model: profile.model?.default ?? modelFromLoopOptions(loopOptions),
|
|
2460
|
+
maximumCharge: typeof opts.maximumCharge === "function" ? opts.maximumCharge(scenario, profile) : opts.maximumCharge,
|
|
2461
|
+
resolveModel: opts.resolveCostModel ? (result) => opts.resolveCostModel?.(result, scenario, profile) : void 0
|
|
2462
|
+
});
|
|
1805
2463
|
}
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
return {
|
|
1821
|
-
id,
|
|
1822
|
-
async *streamPrompt(message, opts) {
|
|
1823
|
-
assertBoxlessPromptOptions(opts, "inlineSandboxClient");
|
|
1824
|
-
const controller = new AbortController();
|
|
1825
|
-
const callerSignal = opts?.signal;
|
|
1826
|
-
const onAbort = () => controller.abort(callerSignal?.reason ?? /* @__PURE__ */ new Error("prompt aborted"));
|
|
1827
|
-
if (callerSignal) if (callerSignal.aborted) onAbort();
|
|
1828
|
-
else callerSignal.addEventListener("abort", onAbort, { once: true });
|
|
1829
|
-
const requestedProfile = (options?.backend && typeof options.backend === "object" ? options.backend.profile : void 0) ?? capturedDefaultProfile;
|
|
1830
|
-
const parsedProfile = agentProfileSchema.safeParse(requestedProfile);
|
|
1831
|
-
if (!parsedProfile.success) throw new Error("inlineSandboxClient: an exact AgentProfile is required; pass defaults.profile or create({ backend: { profile } })");
|
|
1832
|
-
const exec = factory({
|
|
1833
|
-
profile: parsedProfile.data,
|
|
1834
|
-
harness: null
|
|
1835
|
-
}, {
|
|
1836
|
-
signal: controller.signal,
|
|
1837
|
-
seams: { createOptions }
|
|
1838
|
-
});
|
|
1839
|
-
try {
|
|
1840
|
-
const artifact = await settle(exec, message, controller.signal);
|
|
1841
|
-
const out = artifact.out;
|
|
1842
|
-
const tokensIn = artifact.spent.tokens.input;
|
|
1843
|
-
const tokensOut = artifact.spent.tokens.output;
|
|
1844
|
-
const costUsd = artifact.spent.usd;
|
|
1845
|
-
const estimatedCostUsd = out?.estimatedCostUsd;
|
|
1846
|
-
if (artifact.spent.iterations > 0 || artifact.spent.tokensKnown === false || artifact.spent.usdKnown === false || tokensIn > 0 || tokensOut > 0 || costUsd > 0 || estimatedCostUsd !== void 0) yield {
|
|
1847
|
-
type: "llm_call",
|
|
1848
|
-
data: {
|
|
1849
|
-
...artifact.spent.tokensKnown === false ? {} : {
|
|
1850
|
-
tokensIn,
|
|
1851
|
-
tokensOut
|
|
1852
|
-
},
|
|
1853
|
-
...artifact.spent.usdKnown !== false ? { costUsd } : {},
|
|
1854
|
-
...artifact.spent.tokensKnown === false ? { tokensKnown: false } : {},
|
|
1855
|
-
...artifact.spent.usdKnown === false ? { costKnown: false } : {},
|
|
1856
|
-
...artifact.spent.usdKnown !== false ? {
|
|
1857
|
-
costKnown: true,
|
|
1858
|
-
costProvenance: "provider-receipt"
|
|
1859
|
-
} : {},
|
|
1860
|
-
...estimatedCostUsd !== void 0 ? { estimatedCostUsd } : {},
|
|
1861
|
-
...out?.promptCache ? { promptCache: out.promptCache } : {}
|
|
1862
|
-
}
|
|
1863
|
-
};
|
|
1864
|
-
yield {
|
|
1865
|
-
type: "result",
|
|
1866
|
-
data: {
|
|
1867
|
-
finalText: out?.content ?? "",
|
|
1868
|
-
...artifact.spent.tokensKnown === false ? { tokensKnown: false } : { tokenUsage: {
|
|
1869
|
-
inputTokens: tokensIn,
|
|
1870
|
-
outputTokens: tokensOut
|
|
1871
|
-
} },
|
|
1872
|
-
...artifact.spent.usdKnown === false ? {
|
|
1873
|
-
costKnown: false,
|
|
1874
|
-
...costUsd > 0 ? { costUsd } : {}
|
|
1875
|
-
} : {
|
|
1876
|
-
costUsd,
|
|
1877
|
-
costKnown: true,
|
|
1878
|
-
costProvenance: "provider-receipt"
|
|
1879
|
-
},
|
|
1880
|
-
...estimatedCostUsd !== void 0 ? { estimatedCostUsd } : {},
|
|
1881
|
-
...out?.promptCache ? { promptCache: out.promptCache } : {}
|
|
1882
|
-
}
|
|
1883
|
-
};
|
|
1884
|
-
yield {
|
|
1885
|
-
type: "done",
|
|
1886
|
-
data: { outcome: { type: "completed" } }
|
|
1887
|
-
};
|
|
1888
|
-
} finally {
|
|
1889
|
-
callerSignal?.removeEventListener("abort", onAbort);
|
|
1890
|
-
await exec.teardown("brutalKill").catch(() => {});
|
|
1891
|
-
}
|
|
1892
|
-
},
|
|
1893
|
-
async delete() {}
|
|
1894
|
-
};
|
|
1895
|
-
} };
|
|
1896
|
-
}
|
|
1897
|
-
//#endregion
|
|
1898
|
-
//#region src/runtime/local-sandbox-client.ts
|
|
1899
|
-
/**
|
|
1900
|
-
* `localSandboxClient` — the SAME-HOST pseudo-box: a `SandboxClient` whose
|
|
1901
|
-
* `create()` MATERIALIZES the profile's stdio MCP servers as local child
|
|
1902
|
-
* processes (`materializeLocalMcp`) and whose `streamPrompt` drives a real
|
|
1903
|
-
* tool loop (`runBrainLoop` over the router brain) with those live tools.
|
|
1904
|
-
*
|
|
1905
|
-
* Despite the interface name, this does NOT isolate processes. It is only for
|
|
1906
|
-
* author-controlled profiles whose local MCP commands a caller explicitly
|
|
1907
|
-
* trusts; user- or model-authored code belongs in a real sandbox. `delete()`
|
|
1908
|
-
* kills the children.
|
|
1909
|
-
*
|
|
1910
|
-
* A per-create profile may change the prompt surface. Permission to start a
|
|
1911
|
-
* local MCP process applies only when its full canonical bytes match the fixed
|
|
1912
|
-
* constructor profile; a different generated profile is refused.
|
|
1913
|
-
*
|
|
1914
|
-
* Event protocol matches `inlineSandboxClient`: known token usage is emitted as one `llm_call`;
|
|
1915
|
-
* Router catalog cost remains a separately-labelled estimate, never billed spend.
|
|
1916
|
-
*/
|
|
1917
|
-
/** A same-host `SandboxClient` adapter with no process isolation. Local MCP is
|
|
1918
|
-
* refused unless the caller explicitly supplies a policy that allows it.
|
|
1919
|
-
*
|
|
1920
|
-
* There is no box, so a per-prompt `backend` or `model` override is refused rather than dropped;
|
|
1921
|
-
* other per-prompt options (`timeoutMs`, `context`) are accepted and ignored. */
|
|
1922
|
-
function localSandboxClient(opts) {
|
|
1923
|
-
const defaultProfile = opts.profile === void 0 ? void 0 : executableAgentProfileSnapshot(opts.profile, "localSandboxClient default profile");
|
|
1924
|
-
const router = Object.freeze({ ...opts.router });
|
|
1925
|
-
if (opts.profileSecurityPolicy?.allowLocalMcp && defaultProfile === void 0) throw new ValidationError("localSandboxClient: allowLocalMcp requires a fixed author-controlled profile; dynamic profiles need a real sandbox");
|
|
1926
|
-
const trustedProfileDigest = opts.profileSecurityPolicy?.allowLocalMcp && defaultProfile !== void 0 ? canonicalAgentProfileDigest(defaultProfile) : void 0;
|
|
1927
|
-
let seq = 0;
|
|
1928
|
-
return { async create(options) {
|
|
1929
|
-
const profile = executableAgentProfileSnapshot((options?.backend)?.profile ?? defaultProfile, "localSandboxClient");
|
|
1930
|
-
const model = profileProviderModel(profile);
|
|
1931
|
-
const settings = profileModelExecutionSettings(profile, "localSandboxClient");
|
|
1932
|
-
const policyApplies = opts.profileSecurityPolicy !== void 0 && (!opts.profileSecurityPolicy.allowLocalMcp || trustedProfileDigest !== void 0 && canonicalAgentProfileDigest(profile) === trustedProfileDigest);
|
|
1933
|
-
const mcp = await materializeLocalMcp(profile, {
|
|
1934
|
-
...opts.keys ? { keys: opts.keys } : {},
|
|
1935
|
-
...policyApplies ? { profileSecurityPolicy: opts.profileSecurityPolicy } : {}
|
|
1936
|
-
});
|
|
1937
|
-
const brain = routerBrain({
|
|
1938
|
-
routerBaseUrl: router.baseUrl,
|
|
1939
|
-
routerKey: router.key,
|
|
1940
|
-
model,
|
|
1941
|
-
...settings.retry !== void 0 ? { retry: settings.retry } : {},
|
|
1942
|
-
...enforceTokenLimits(settings.tokenLimits, "router", "localSandboxClient").applied,
|
|
1943
|
-
...settings.stream !== void 0 ? { stream: settings.stream } : {}
|
|
1944
|
-
}, {
|
|
1945
|
-
...settings.temperature !== void 0 ? { temperature: settings.temperature } : {},
|
|
1946
|
-
...settings.seed !== void 0 ? { seed: settings.seed } : {},
|
|
1947
|
-
...settings.toolChoice !== void 0 ? { toolChoice: settings.toolChoice } : {},
|
|
1948
|
-
...settings.extraBody !== void 0 ? { extraBody: settings.extraBody } : {},
|
|
1949
|
-
...profile.model?.reasoningEffort ? { reasoningEffort: profile.model.reasoningEffort } : {}
|
|
1950
|
-
});
|
|
1951
|
-
const system = [profile.prompt?.systemPrompt, ...profile.prompt?.instructions ?? []].filter((s) => typeof s === "string" && s.trim().length > 0).join("\n\n");
|
|
1952
|
-
return {
|
|
1953
|
-
id: `local-${seq++}`,
|
|
1954
|
-
async *streamPrompt(message, popts) {
|
|
1955
|
-
assertBoxlessPromptOptions(popts, "localSandboxClient");
|
|
1956
|
-
let estimatedCostUsd = 0;
|
|
1957
|
-
let sawEstimatedCost = false;
|
|
1958
|
-
const chat = async (messages, tools) => {
|
|
1959
|
-
const r = await brain(messages, tools);
|
|
1960
|
-
if (r.costProvenance === "catalog-estimate" && r.costUsd !== void 0) {
|
|
1961
|
-
estimatedCostUsd += r.costUsd;
|
|
1962
|
-
sawEstimatedCost = true;
|
|
1963
|
-
}
|
|
1964
|
-
return r;
|
|
1965
|
-
};
|
|
1966
|
-
const r = await runBrainLoop({
|
|
1967
|
-
chat,
|
|
1968
|
-
tools: mcp.tools,
|
|
1969
|
-
execute: (name, args) => mcp.call(name, args),
|
|
1970
|
-
initialMessages: [...system ? [{
|
|
1971
|
-
role: "system",
|
|
1972
|
-
content: system
|
|
1973
|
-
}] : [], {
|
|
1974
|
-
role: "user",
|
|
1975
|
-
content: message
|
|
1976
|
-
}],
|
|
1977
|
-
maxTurns: settings.maxTurns ?? 0,
|
|
1978
|
-
hooks: { stopBefore: () => popts?.signal?.aborted === true }
|
|
1979
|
-
});
|
|
1980
|
-
if (r.turns > 0) yield {
|
|
1981
|
-
type: "llm_call",
|
|
1982
|
-
data: {
|
|
1983
|
-
model,
|
|
1984
|
-
...r.tokensKnown === false ? { tokensKnown: false } : {
|
|
1985
|
-
tokensIn: r.usage.input,
|
|
1986
|
-
tokensOut: r.usage.output
|
|
1987
|
-
},
|
|
1988
|
-
costKnown: false,
|
|
1989
|
-
...sawEstimatedCost ? { estimatedCostUsd } : {}
|
|
1990
|
-
}
|
|
1991
|
-
};
|
|
1992
|
-
yield {
|
|
1993
|
-
type: "result",
|
|
1994
|
-
data: {
|
|
1995
|
-
finalText: r.final,
|
|
1996
|
-
...r.tokensKnown === false ? { tokensKnown: false } : { tokenUsage: {
|
|
1997
|
-
inputTokens: r.usage.input,
|
|
1998
|
-
outputTokens: r.usage.output
|
|
1999
|
-
} },
|
|
2000
|
-
costKnown: false,
|
|
2001
|
-
...sawEstimatedCost ? { estimatedCostUsd } : {}
|
|
2002
|
-
}
|
|
2003
|
-
};
|
|
2004
|
-
},
|
|
2005
|
-
async delete() {
|
|
2006
|
-
await mcp.close();
|
|
2464
|
+
async function runLoopWithCampaignContext(opts, loopOptions, ctx, cost) {
|
|
2465
|
+
const paid = await ctx.cost.runPaidCall({
|
|
2466
|
+
channel: "agent",
|
|
2467
|
+
actor: opts.costSource ?? "loop",
|
|
2468
|
+
model: cost.model,
|
|
2469
|
+
signal: ctx.signal,
|
|
2470
|
+
...cost.maximumCharge ? { maximumCharge: cost.maximumCharge } : {},
|
|
2471
|
+
execute: (executionSignal) => runAgentRounds({
|
|
2472
|
+
...loopOptions,
|
|
2473
|
+
ctx: {
|
|
2474
|
+
sandboxClient: opts.sandboxClient,
|
|
2475
|
+
...opts.promptOptions === void 0 ? {} : { promptOptions: opts.promptOptions },
|
|
2476
|
+
signal: executionSignal,
|
|
2477
|
+
traceEmitter: opts.forwardTrace === false ? void 0 : campaignTraceToLoopEmitter(ctx.trace)
|
|
2007
2478
|
}
|
|
2008
|
-
}
|
|
2009
|
-
|
|
2479
|
+
}),
|
|
2480
|
+
receipt: (result) => loopCostReceipt(result, cost.resolveModel?.(result) ?? cost.model)
|
|
2481
|
+
});
|
|
2482
|
+
if (!paid.succeeded) throw paid.error;
|
|
2483
|
+
const result = paid.value;
|
|
2484
|
+
return (opts.toArtifact ?? ((r) => r.winner?.output))(result);
|
|
2010
2485
|
}
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
* ZERO transport code (the node:http cli-bridge transport lives in `bridgeExecutor`).
|
|
2019
|
-
*
|
|
2020
|
-
* The heavy `resolveBenchClient` in `@tangle-network/agent-bench` is the bench-side
|
|
2021
|
-
* sibling: it layers router/search specifics a product should not have to depend on.
|
|
2022
|
-
* This is the generic, dep-light core products call to switch backends:
|
|
2023
|
-
*
|
|
2024
|
-
* - `backend: 'sandbox'` → IN-BOX: return the caller's real Sandbox-backed
|
|
2025
|
-
* `sandboxClient` unchanged. Fail loud if absent — a
|
|
2026
|
-
* product on the sandbox backend already holds its box.
|
|
2027
|
-
* - `backend: 'bridge'` → OFF-BOX: a local cli-bridge fronting a harness CLI
|
|
2028
|
-
* (opencode / kimi-code / …) as the leaf executor, wired
|
|
2029
|
-
* through the resumable `bridgeExecutor`.
|
|
2030
|
-
* - `backend: 'router'` → OFF-BOX: a router chat-completion as the leaf executor,
|
|
2031
|
-
* presented as a `SandboxClient` (no sandbox dependency).
|
|
2032
|
-
* - `backend: 'local'` → SAME-HOST: a router-brain tool loop with the
|
|
2033
|
-
* profile's stdio MCP servers spawned as LOCAL child
|
|
2034
|
-
* processes — the only backend that can reach an MCP
|
|
2035
|
-
* server built into a host worktree.
|
|
2036
|
-
*/
|
|
2037
|
-
/**
|
|
2038
|
-
* Resolve a `SandboxClient` for the chosen backend. The generic, dep-light core
|
|
2039
|
-
* that `resolveBenchClient` builds on — reuse this instead of hand-rolling the
|
|
2040
|
-
* `createExecutor`/`inlineSandboxClient` branch in each product.
|
|
2041
|
-
*/
|
|
2042
|
-
function resolveSandboxClient(opts) {
|
|
2043
|
-
switch (opts.backend) {
|
|
2044
|
-
case "sandbox":
|
|
2045
|
-
if (!opts.sandboxClient) throw new Error("resolveSandboxClient: backend 'sandbox' requires opts.sandboxClient");
|
|
2046
|
-
return opts.sandboxClient;
|
|
2047
|
-
case "bridge": {
|
|
2048
|
-
const bridge = opts.bridge;
|
|
2049
|
-
if (!bridge?.bearer) throw new Error("resolveSandboxClient: backend 'bridge' requires bridge.bearer");
|
|
2050
|
-
return inlineSandboxClient(createExecutor({
|
|
2051
|
-
backend: "bridge",
|
|
2052
|
-
bridgeUrl: bridge.url ?? "http://127.0.0.1:3355",
|
|
2053
|
-
bridgeBearer: bridge.bearer,
|
|
2054
|
-
timeoutMs: bridge.timeoutMs
|
|
2055
|
-
}));
|
|
2056
|
-
}
|
|
2057
|
-
case "router": {
|
|
2058
|
-
const router = opts.router;
|
|
2059
|
-
if (!router?.baseUrl || !router.key) throw new Error("resolveSandboxClient: backend 'router' requires router.baseUrl and router.key");
|
|
2060
|
-
return inlineSandboxClient(createExecutor({
|
|
2061
|
-
backend: "router",
|
|
2062
|
-
routerBaseUrl: router.baseUrl,
|
|
2063
|
-
routerKey: router.key
|
|
2064
|
-
}));
|
|
2065
|
-
}
|
|
2066
|
-
case "local": {
|
|
2067
|
-
const local = opts.local;
|
|
2068
|
-
if (!local?.router?.baseUrl || !local.router.key) throw new Error("resolveSandboxClient: backend 'local' requires local.router.baseUrl and local.router.key");
|
|
2069
|
-
return localSandboxClient(local);
|
|
2070
|
-
}
|
|
2071
|
-
}
|
|
2486
|
+
function loopCostReceipt(result, model) {
|
|
2487
|
+
return costReceiptFromUsage(result.tokenUsage, model, {
|
|
2488
|
+
usageUnknown: result.tokenUsage.tokensKnown === false,
|
|
2489
|
+
actualCostUsd: result.costUsdKnown !== false ? result.costUsd : void 0,
|
|
2490
|
+
costUnknown: result.costUsdKnown === false,
|
|
2491
|
+
estimatedCostUsd: result.estimatedCostUsd
|
|
2492
|
+
});
|
|
2072
2493
|
}
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
* base profile across the harness×model axes (`expandProfileAxes`), run every
|
|
2080
|
-
* (profile, case) cell as a driven loop (`loopDispatch` + the naive retry driver), score
|
|
2081
|
-
* with the domain's grader, and emit ONE `runProfileMatrix` call. Each product
|
|
2082
|
-
* hand-rolled that assembly (~650 lines each) and re-hit the same footguns:
|
|
2083
|
-
* stale cell-cache reuse, zero-token stub cells, missing model snapshots.
|
|
2084
|
-
*
|
|
2085
|
-
* This facade IS that assembly, once, with the domain reduced to a declarative
|
|
2086
|
-
* spec: `cases` + `prompt` + `score`. It contains NO execution, judging, or
|
|
2087
|
-
* metering logic of its own — every moving part is an existing primitive, and
|
|
2088
|
-
* every default is overridable:
|
|
2089
|
-
*
|
|
2090
|
-
* - LEVEL 0 (declarative): `cases` / `prompt` / `score` / `axis`.
|
|
2091
|
-
* - LEVEL 1 (seams): `backends`, `flags`, `parseOutput`, `onCellEvents`,
|
|
2092
|
-
* `resolveModel`, `setup`/`teardown`, `export`, `matrix`
|
|
2093
|
-
* passthrough.
|
|
2094
|
-
* - LEVEL 2 (replacement): `dispatch` and `judges` swap out the whole
|
|
2095
|
-
* loop wiring or scoring; `runProfileMatrix` itself stays public as the
|
|
2096
|
-
* escape floor — a product overriding everything just writes what it has
|
|
2097
|
-
* today, no capability removed.
|
|
2098
|
-
*
|
|
2099
|
-
* `toBenchmarkAdapter()` exposes the same domain surface in the structural
|
|
2100
|
-
* `BenchmarkAdapter` shape (`name`/`preflight`/`loadTasks`/`judge`/
|
|
2101
|
-
* `goldArtifact`) so a product leaderboard can register into a benchmark
|
|
2102
|
-
* registry without this module depending on one.
|
|
2103
|
-
*
|
|
2104
|
-
* @experimental
|
|
2105
|
-
*/
|
|
2106
|
-
/**
|
|
2107
|
-
* The no-signal retry floor as a bare `Driver` literal: re-run the case VERBATIM as an
|
|
2108
|
-
* independent attempt until one shot scores (`verdict.valid`) or the shot cap. It reads ONLY
|
|
2109
|
-
* `verdict.valid` — never `notes`/`scores` — so a leaderboard cell's retries carry no grader
|
|
2110
|
-
* findings into the next shot (the leak-free firewall). Steering POLICY texts are registry data
|
|
2111
|
-
* (`delegates/naive-continuation` and siblings in the kernel prompt registry) attached to graph
|
|
2112
|
-
* edges; this loop-kernel control folds no text in — each shot is the same task object, and the
|
|
2113
|
-
* per-shot nonce in `taskToPrompt` keeps the attempts independent at the router.
|
|
2114
|
-
*/
|
|
2115
|
-
function naiveRetryDriver(shots) {
|
|
2494
|
+
/** Map Runtime usage into Eval's canonical paid-call receipt without inventing cache classes. */
|
|
2495
|
+
function costReceiptFromUsage(usage, model, cost) {
|
|
2496
|
+
const cacheComplete = hasCompleteCacheBreakdown(usage) && (usage.freshInput !== void 0 || usage.cacheRead !== void 0 || usage.cacheWrite !== void 0);
|
|
2497
|
+
const cacheRead = cacheComplete ? usage.cacheRead ?? 0 : void 0;
|
|
2498
|
+
const cacheWrite = cacheComplete ? usage.cacheWrite ?? 0 : void 0;
|
|
2499
|
+
const classified = (cacheRead ?? 0) + (cacheWrite ?? 0);
|
|
2116
2500
|
return {
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
},
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
}
|
|
2128
|
-
describePlan() {
|
|
2129
|
-
return {
|
|
2130
|
-
kind: "refine",
|
|
2131
|
-
rationale: "naive fixed continuation (no grade signal)"
|
|
2132
|
-
};
|
|
2133
|
-
}
|
|
2501
|
+
model,
|
|
2502
|
+
inputTokens: cacheComplete ? usage.input - classified : usage.input,
|
|
2503
|
+
outputTokens: usage.output,
|
|
2504
|
+
...cacheComplete ? {
|
|
2505
|
+
cachedTokens: cacheRead,
|
|
2506
|
+
cacheWriteTokens: cacheWrite
|
|
2507
|
+
} : {},
|
|
2508
|
+
...usage.tokensKnown === false || cost.usageUnknown ? { usageUnknown: true } : {},
|
|
2509
|
+
...cost.actualCostUsd !== void 0 ? { actualCostUsd: cost.actualCostUsd } : {},
|
|
2510
|
+
...cost.costUnknown ? { costUnknown: true } : {},
|
|
2511
|
+
...cost.estimatedCostUsd !== void 0 ? { estimatedCostUsd: cost.estimatedCostUsd } : {}
|
|
2134
2512
|
};
|
|
2135
2513
|
}
|
|
2136
|
-
/**
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
const
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
}
|
|
2158
|
-
function normalizeScore(s) {
|
|
2159
|
-
return typeof s === "number" ? { composite: s } : s;
|
|
2160
|
-
}
|
|
2161
|
-
/**
|
|
2162
|
-
* Assemble a declarative spec (`cases` + `prompt` + `score`) into a runnable
|
|
2163
|
-
* harness×model leaderboard — `run()` executes the matrix, `toBenchmarkAdapter()`
|
|
2164
|
-
* exposes the same domain as a structural `BenchmarkAdapter`.
|
|
2165
|
-
*/
|
|
2166
|
-
function defineLeaderboard(spec) {
|
|
2167
|
-
const caseId = (c) => {
|
|
2168
|
-
const id = spec.caseId ? spec.caseId(c) : c.id;
|
|
2169
|
-
if (typeof id !== "string" || id.length === 0) throw new Error(`defineLeaderboard(${spec.name}): every case needs a stable string id — give cases an \`id\` property or supply spec.caseId`);
|
|
2170
|
-
return id;
|
|
2171
|
-
};
|
|
2172
|
-
const selectCases = (ids) => {
|
|
2173
|
-
if (!ids) return spec.cases;
|
|
2174
|
-
const byId = new Map(spec.cases.map((c) => [caseId(c), c]));
|
|
2175
|
-
return ids.map((id) => {
|
|
2176
|
-
const c = byId.get(id);
|
|
2177
|
-
if (c === void 0) throw new Error(`defineLeaderboard(${spec.name}): unknown case "${id}" (have: ${[...byId.keys()].join(", ")})`);
|
|
2178
|
-
return c;
|
|
2179
|
-
});
|
|
2514
|
+
/** Eval has one pre-admitted paid call for this dispatch. A tree can settle that call only when
|
|
2515
|
+
* Runtime has provider-served identity for every inference attempt that actually started. Plan
|
|
2516
|
+
* materialization is deliberately ignored: it describes what Runtime requested, not what the
|
|
2517
|
+
* provider served. Every visible terminal child must carry provider attempt evidence; a missing
|
|
2518
|
+
* evidence record stays unknown, including zero-valued or incomplete spend.
|
|
2519
|
+
* @internal Testable identity projection used by superviseDispatch. */
|
|
2520
|
+
function supervisedTreeModelForDispatch(result, rootProfile) {
|
|
2521
|
+
const canonicalModels = /* @__PURE__ */ new Set();
|
|
2522
|
+
const observedModels = [];
|
|
2523
|
+
const wholeTreeEvidence = result.providerModel;
|
|
2524
|
+
if (wholeTreeEvidence === void 0 || wholeTreeEvidence.status !== "known" && wholeTreeEvidence.reason !== "provider-model-conflict" || wholeTreeEvidence.attempts.length === 0) return { kind: "unknown" };
|
|
2525
|
+
const wholeTreeIdentity = collectProviderModels(wholeTreeEvidence, () => true);
|
|
2526
|
+
if (wholeTreeIdentity.kind !== "known") return wholeTreeIdentity;
|
|
2527
|
+
for (const model of wholeTreeIdentity.observed) {
|
|
2528
|
+
observedModels.push(model.raw);
|
|
2529
|
+
canonicalModels.add(model.canonical);
|
|
2530
|
+
}
|
|
2531
|
+
if (canonicalModels.size === 0) return { kind: "unknown" };
|
|
2532
|
+
if (canonicalModels.size !== 1) return {
|
|
2533
|
+
kind: "mixed",
|
|
2534
|
+
models: [...canonicalModels].sort()
|
|
2180
2535
|
};
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
key: "composite",
|
|
2185
|
-
description: `${spec.name} case score`
|
|
2186
|
-
}],
|
|
2187
|
-
score({ artifact, scenario }) {
|
|
2188
|
-
const s = normalizeScore(spec.score(artifact, scenario.case));
|
|
2189
|
-
return {
|
|
2190
|
-
composite: s.composite,
|
|
2191
|
-
dimensions: {
|
|
2192
|
-
composite: s.composite,
|
|
2193
|
-
...s.dimensions
|
|
2194
|
-
},
|
|
2195
|
-
notes: s.notes ?? ""
|
|
2196
|
-
};
|
|
2197
|
-
}
|
|
2536
|
+
return {
|
|
2537
|
+
kind: "known",
|
|
2538
|
+
model: observedModels[0]
|
|
2198
2539
|
};
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
"
|
|
2208
|
-
|
|
2209
|
-
"run-dir",
|
|
2210
|
-
"export-dir"
|
|
2211
|
-
]) args[name] = argOf(argv, name);
|
|
2212
|
-
for (const [name, flag] of Object.entries(spec.flags ?? {})) args[name] = argOf(argv, name) ?? flag.default;
|
|
2213
|
-
const backendName = args.backend ?? "sandbox";
|
|
2214
|
-
const shots = Number(args.shots ?? spec.shots ?? 1);
|
|
2215
|
-
const reps = Number(args.reps ?? spec.reps ?? 1);
|
|
2216
|
-
const snapshot = args["model-snapshot"] ?? "leaderboard";
|
|
2217
|
-
const runDir = args["run-dir"] ?? join(tmpdir(), `leaderboard-${spec.name}-${Date.now()}-${process.pid}`);
|
|
2218
|
-
const exportDir = args["export-dir"] ?? join(runDir, "export");
|
|
2219
|
-
mkdirSync(runDir, { recursive: true });
|
|
2220
|
-
const cases = selectCases(splitList(args.cases));
|
|
2221
|
-
if (cases.length === 0) throw new Error(`defineLeaderboard(${spec.name}): no cases to run`);
|
|
2222
|
-
const scenarios = cases.map((c) => ({
|
|
2223
|
-
id: caseId(c),
|
|
2224
|
-
kind: spec.name,
|
|
2225
|
-
case: c
|
|
2226
|
-
}));
|
|
2227
|
-
const harnesses = splitList(args.harnesses) ?? spec.axis?.harnesses ?? CODING_HARNESSES;
|
|
2228
|
-
const rawModels = splitList(args.models) ?? spec.axis?.models ?? (spec.baseProfile?.model?.default !== void 0 ? [spec.baseProfile.model.default] : void 0);
|
|
2229
|
-
if (!rawModels || rawModels.length === 0) throw new Error(`defineLeaderboard(${spec.name}): no models — pass --models, set spec.axis.models, or give spec.baseProfile a model.default`);
|
|
2230
|
-
const models = rawModels.map((m) => withSnapshot(m, snapshot));
|
|
2231
|
-
const profiles = expandProfileAxes({
|
|
2232
|
-
base: spec.baseProfile,
|
|
2233
|
-
harnesses,
|
|
2234
|
-
models
|
|
2235
|
-
});
|
|
2236
|
-
for (const profile of profiles) assertExecutableAgentProfile(profile, `defineLeaderboard(${spec.name})`);
|
|
2237
|
-
const ctx = {
|
|
2238
|
-
name: spec.name,
|
|
2239
|
-
backend: backendName,
|
|
2240
|
-
runDir,
|
|
2241
|
-
exportDir,
|
|
2242
|
-
args,
|
|
2243
|
-
harnesses,
|
|
2244
|
-
models,
|
|
2245
|
-
caseIds: scenarios.map((s) => s.id),
|
|
2246
|
-
shots,
|
|
2247
|
-
reps
|
|
2248
|
-
};
|
|
2249
|
-
const backends = {
|
|
2250
|
-
sandbox: () => {
|
|
2251
|
-
throw new Error(`defineLeaderboard(${spec.name}): the 'sandbox' backend needs your product's real Sandbox client — supply spec.backends.sandbox (e.g. () => new Sandbox({ apiKey, baseUrl }))`);
|
|
2252
|
-
},
|
|
2253
|
-
"cli-bridge": () => {
|
|
2254
|
-
const bearer = process.env.BRIDGE_BEARER ?? process.env.CLI_BRIDGE_BEARER;
|
|
2255
|
-
if (!bearer) throw new Error(`defineLeaderboard(${spec.name}): backend 'cli-bridge' needs BRIDGE_BEARER or CLI_BRIDGE_BEARER set`);
|
|
2256
|
-
return resolveSandboxClient({
|
|
2257
|
-
backend: "bridge",
|
|
2258
|
-
bridge: {
|
|
2259
|
-
url: process.env.CLI_BRIDGE_URL,
|
|
2260
|
-
bearer,
|
|
2261
|
-
timeoutMs: 9e5
|
|
2262
|
-
}
|
|
2263
|
-
});
|
|
2264
|
-
},
|
|
2265
|
-
...spec.backends
|
|
2266
|
-
};
|
|
2267
|
-
const makeClient = backends[backendName];
|
|
2268
|
-
if (!makeClient) throw new Error(`defineLeaderboard(${spec.name}): unknown backend "${backendName}" (have: ${Object.keys(backends).join(", ")})`);
|
|
2269
|
-
const sandboxClient = makeClient();
|
|
2270
|
-
const promptById = /* @__PURE__ */ new Map();
|
|
2271
|
-
for (const s of scenarios) promptById.set(s.id, await spec.prompt(s.case));
|
|
2272
|
-
const promptOf = (s) => {
|
|
2273
|
-
const p = promptById.get(s.id);
|
|
2274
|
-
if (p === void 0) throw new Error(`defineLeaderboard(${spec.name}): no prompt for case "${s.id}"`);
|
|
2275
|
-
return p;
|
|
2276
|
-
};
|
|
2277
|
-
let shotNonce = 0;
|
|
2278
|
-
const maximumCharge = spec.maximumCharge;
|
|
2279
|
-
const dispatch = spec.dispatch ?? ((profile, scenario, dispatchCtx) => {
|
|
2280
|
-
return loopDispatch({
|
|
2281
|
-
sandboxClient,
|
|
2282
|
-
maximumCharge: typeof maximumCharge === "function" ? (cellScenario, cellProfile) => maximumCharge(cellProfile, cellScenario) : maximumCharge,
|
|
2283
|
-
resolveCostModel: (result, _cellScenario, cellProfile) => {
|
|
2284
|
-
if (!spec.resolveModel) return cellProfile.model?.default;
|
|
2285
|
-
const served = new Set(result.iterations.map((iteration) => spec.resolveModel?.(iteration.events)).filter((model) => model !== void 0));
|
|
2286
|
-
if (served.size > 1) throw new Error(`defineLeaderboard(${spec.name}): one cell reported multiple served models: ${[...served].join(", ")}`);
|
|
2287
|
-
return [...served][0] ?? cellProfile.model?.default;
|
|
2288
|
-
},
|
|
2289
|
-
toLoopOptions: (cellScenario, cellProfile) => {
|
|
2290
|
-
return {
|
|
2291
|
-
driver: naiveRetryDriver(shots),
|
|
2292
|
-
agentRun: {
|
|
2293
|
-
profile: cellProfile,
|
|
2294
|
-
taskToPrompt: (s) => `${promptOf(s)}\n\n<!-- independent-attempt:${shotNonce++} -->`
|
|
2295
|
-
},
|
|
2296
|
-
output: { parse: (events) => spec.parseOutput ? spec.parseOutput(events, cellScenario.case) : collectAgentResponseText(events) ?? "" },
|
|
2297
|
-
validator: { validate: async (output) => {
|
|
2298
|
-
const s = normalizeScore(spec.score(output, cellScenario.case));
|
|
2299
|
-
return {
|
|
2300
|
-
valid: s.composite > 0,
|
|
2301
|
-
score: s.composite
|
|
2302
|
-
};
|
|
2303
|
-
} },
|
|
2304
|
-
task: cellScenario,
|
|
2305
|
-
maxIterations: shots
|
|
2306
|
-
};
|
|
2307
|
-
},
|
|
2308
|
-
toArtifact: (result) => {
|
|
2309
|
-
for (const iter of result.iterations) spec.onCellEvents?.(iter.events, scenario.case, {
|
|
2310
|
-
index: iter.index,
|
|
2311
|
-
...iter.error ? { error: iter.error.message } : {},
|
|
2312
|
-
...iter.verdict ? { verdict: { score: iter.verdict.score } } : {}
|
|
2313
|
-
});
|
|
2314
|
-
return result.winner?.output;
|
|
2315
|
-
}
|
|
2316
|
-
})(profile, scenario, dispatchCtx);
|
|
2317
|
-
});
|
|
2318
|
-
await spec.setup?.(ctx);
|
|
2319
|
-
try {
|
|
2320
|
-
const result = await runProfileMatrix({
|
|
2321
|
-
profiles,
|
|
2322
|
-
scenarios,
|
|
2323
|
-
dispatch,
|
|
2324
|
-
judges: spec.judges ?? [scoreJudge],
|
|
2325
|
-
runDir,
|
|
2326
|
-
commitSha: gitSha(),
|
|
2327
|
-
reps,
|
|
2328
|
-
...spec.matrix
|
|
2329
|
-
});
|
|
2330
|
-
if (spec.export) await spec.export(result, ctx);
|
|
2331
|
-
else {
|
|
2332
|
-
mkdirSync(exportDir, { recursive: true });
|
|
2333
|
-
writeFileSync(join(runDir, "matrix-result.json"), `${JSON.stringify(result, null, 2)}\n`);
|
|
2334
|
-
const table = renderLeaderboardMarkdown(leaderboard(result.records, {
|
|
2335
|
-
title: spec.name,
|
|
2336
|
-
meta: { backend: backendName }
|
|
2337
|
-
}));
|
|
2338
|
-
writeFileSync(join(exportDir, "leaderboard.md"), table);
|
|
2339
|
-
console.log(table);
|
|
2340
|
-
}
|
|
2341
|
-
return result;
|
|
2342
|
-
} finally {
|
|
2343
|
-
await spec.teardown?.(ctx);
|
|
2540
|
+
}
|
|
2541
|
+
function collectProviderModels(evidence, validate) {
|
|
2542
|
+
if (evidence.status !== "known" && evidence.reason !== "provider-model-conflict" || !Array.isArray(evidence.attempts) || evidence.attempts.length === 0 || !Array.isArray(evidence.models)) return { kind: "unknown" };
|
|
2543
|
+
const observed = [];
|
|
2544
|
+
const canonical = /* @__PURE__ */ new Set();
|
|
2545
|
+
const observedRaw = /* @__PURE__ */ new Set();
|
|
2546
|
+
for (const attempt of evidence.attempts) {
|
|
2547
|
+
if (attempt.providerDispatch === "not_started") {
|
|
2548
|
+
if (attempt.observations.length > 0 || attempt.identityConflict === true) return { kind: "unknown" };
|
|
2549
|
+
continue;
|
|
2344
2550
|
}
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
prompt: await spec.prompt(c),
|
|
2363
|
-
metadata: { case: c }
|
|
2364
|
-
})));
|
|
2365
|
-
},
|
|
2366
|
-
async judge(task, artifact) {
|
|
2367
|
-
const [c] = selectCases([task.id]);
|
|
2368
|
-
if (c === void 0) throw new Error(`defineLeaderboard(${spec.name}): no case "${task.id}"`);
|
|
2369
|
-
const s = normalizeScore(spec.score(artifact, c));
|
|
2551
|
+
if (attempt === null || typeof attempt !== "object" || !Array.isArray(attempt.observations) || attempt.observations.length === 0 || attempt.identityConflict === true) return { kind: "unknown" };
|
|
2552
|
+
let attemptBase;
|
|
2553
|
+
let attemptSnapshot;
|
|
2554
|
+
let attemptSnapshotKind;
|
|
2555
|
+
let representative;
|
|
2556
|
+
for (const model of attempt.observations) {
|
|
2557
|
+
if (typeof model !== "string" || !validate(model)) return { kind: "unknown" };
|
|
2558
|
+
const identity = canonicalObservedModelParts(model);
|
|
2559
|
+
if (identity === void 0) return { kind: "unknown" };
|
|
2560
|
+
const base = identity.base;
|
|
2561
|
+
const snapshot = identity.snapshot;
|
|
2562
|
+
if (attemptBase !== void 0 && attemptBase !== base) return {
|
|
2563
|
+
kind: "mixed",
|
|
2564
|
+
models: [.../* @__PURE__ */ new Set([...attemptBase ? [attemptBase] : [], identity.canonical])]
|
|
2565
|
+
};
|
|
2566
|
+
if (attemptSnapshot !== void 0 && snapshot !== void 0 && (attemptSnapshot !== snapshot || attemptSnapshotKind !== identity.snapshotKind)) {
|
|
2567
|
+
if (attemptBase === void 0 || attemptSnapshotKind === void 0) return { kind: "unknown" };
|
|
2370
2568
|
return {
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
detail: s.notes
|
|
2569
|
+
kind: "mixed",
|
|
2570
|
+
models: [attemptIdentityText(attemptBase, attemptSnapshot, attemptSnapshotKind), identity.canonical]
|
|
2374
2571
|
};
|
|
2375
|
-
}
|
|
2376
|
-
|
|
2377
|
-
|
|
2572
|
+
}
|
|
2573
|
+
attemptBase = base;
|
|
2574
|
+
attemptSnapshot ??= snapshot;
|
|
2575
|
+
attemptSnapshotKind ??= identity.snapshotKind;
|
|
2576
|
+
if (snapshot !== void 0) representative = model;
|
|
2577
|
+
observedRaw.add(model);
|
|
2578
|
+
}
|
|
2579
|
+
if (attemptBase === void 0 || attemptSnapshot === void 0 || attemptSnapshotKind === void 0 || representative === void 0) return { kind: "unknown" };
|
|
2580
|
+
const attemptIdentity = attemptIdentityText(attemptBase, attemptSnapshot, attemptSnapshotKind);
|
|
2581
|
+
observed.push({
|
|
2582
|
+
raw: representative,
|
|
2583
|
+
canonical: attemptIdentity
|
|
2584
|
+
});
|
|
2585
|
+
canonical.add(attemptIdentity);
|
|
2586
|
+
}
|
|
2587
|
+
if (observed.length === 0) return { kind: "unknown" };
|
|
2588
|
+
const declaredRaw = /* @__PURE__ */ new Set();
|
|
2589
|
+
for (const model of evidence.models) {
|
|
2590
|
+
if (typeof model !== "string") return { kind: "unknown" };
|
|
2591
|
+
declaredRaw.add(model);
|
|
2592
|
+
}
|
|
2593
|
+
if (declaredRaw.size !== observedRaw.size || [...declaredRaw].some((model) => !observedRaw.has(model))) return { kind: "unknown" };
|
|
2594
|
+
return canonical.size > 1 ? {
|
|
2595
|
+
kind: "mixed",
|
|
2596
|
+
models: [...canonical].sort()
|
|
2597
|
+
} : {
|
|
2598
|
+
kind: "known",
|
|
2599
|
+
observed
|
|
2600
|
+
};
|
|
2601
|
+
}
|
|
2602
|
+
function attemptIdentityText(base, snapshot, kind) {
|
|
2603
|
+
return kind === "date" ? `${base}-${snapshot}` : `${base}@${snapshot}`;
|
|
2604
|
+
}
|
|
2605
|
+
/** The Eval receipt surface has no pre-admitted per-model recursive-tree receipt bundle yet. */
|
|
2606
|
+
var SupervisedTreeModelIdentityError = class extends Error {
|
|
2607
|
+
constructor(identity) {
|
|
2608
|
+
const detail = identity.kind === "mixed" ? `multiple models (${identity.models.join(", ")})` : "an unknown materialized model";
|
|
2609
|
+
super(`superviseDispatch: cannot settle one Eval paid-call receipt for a tree with ${detail}`);
|
|
2610
|
+
this.name = "SupervisedTreeModelIdentityError";
|
|
2378
2611
|
}
|
|
2612
|
+
};
|
|
2613
|
+
function unknownSupervisedTreeReceipt() {
|
|
2379
2614
|
return {
|
|
2380
|
-
|
|
2381
|
-
|
|
2615
|
+
model: "unknown",
|
|
2616
|
+
inputTokens: 0,
|
|
2617
|
+
outputTokens: 0,
|
|
2618
|
+
usageUnknown: true,
|
|
2619
|
+
costUnknown: true
|
|
2620
|
+
};
|
|
2621
|
+
}
|
|
2622
|
+
/** Run one recursive supervised tree inside Eval's pre-execution paid-call lifecycle. */
|
|
2623
|
+
function superviseDispatch(opts) {
|
|
2624
|
+
return async (profile, scenario, ctx) => {
|
|
2625
|
+
const superviseOptions = opts.toSuperviseOptions(scenario, profile);
|
|
2626
|
+
const task = opts.toTask(scenario, profile);
|
|
2627
|
+
const maximumCharge = typeof opts.maximumCharge === "function" ? opts.maximumCharge(scenario, profile) : opts.maximumCharge;
|
|
2628
|
+
const paid = await ctx.cost.runPaidCall({
|
|
2629
|
+
channel: "agent",
|
|
2630
|
+
actor: opts.costSource ?? "supervise",
|
|
2631
|
+
model: "unknown",
|
|
2632
|
+
signal: ctx.signal,
|
|
2633
|
+
...maximumCharge ? { maximumCharge } : {},
|
|
2634
|
+
execute: async (executionSignal) => {
|
|
2635
|
+
const result = await supervise(profile, task, {
|
|
2636
|
+
...superviseOptions,
|
|
2637
|
+
signal: executionSignal
|
|
2638
|
+
});
|
|
2639
|
+
const identity = supervisedTreeModelForDispatch(result, profile);
|
|
2640
|
+
if (identity.kind !== "known") throw new SupervisedTreeModelIdentityError(identity);
|
|
2641
|
+
return result;
|
|
2642
|
+
},
|
|
2643
|
+
receipt: (result) => {
|
|
2644
|
+
const identity = supervisedTreeModelForDispatch(result, profile);
|
|
2645
|
+
if (identity.kind !== "known") throw new SupervisedTreeModelIdentityError(identity);
|
|
2646
|
+
return costReceiptFromUsage(result.spentTotal.tokens, identity.model, {
|
|
2647
|
+
usageUnknown: result.spentTotal.tokensKnown === false,
|
|
2648
|
+
actualCostUsd: result.spentTotal.usdKnown !== false ? result.spentTotal.usd : void 0,
|
|
2649
|
+
costUnknown: result.spentTotal.usdKnown === false
|
|
2650
|
+
});
|
|
2651
|
+
},
|
|
2652
|
+
receiptFromError: (error) => error instanceof SupervisedTreeModelIdentityError ? unknownSupervisedTreeReceipt() : void 0
|
|
2653
|
+
});
|
|
2654
|
+
if (!paid.succeeded) throw paid.error;
|
|
2655
|
+
return (opts.toArtifact ?? ((result) => {
|
|
2656
|
+
if (result.kind !== "winner") throw new Error(`superviseDispatch: supervised tree ended without a winner (${result.reason})`);
|
|
2657
|
+
return result.out;
|
|
2658
|
+
}))(paid.value);
|
|
2659
|
+
};
|
|
2660
|
+
}
|
|
2661
|
+
function modelFromLoopOptions(options) {
|
|
2662
|
+
const profiles = options.agentRun ? [options.agentRun.profile] : options.agentRuns?.map((run) => run.profile) ?? [];
|
|
2663
|
+
const models = new Set(profiles.map((profile) => profile.model?.default).filter((model) => !!model));
|
|
2664
|
+
if (models.size === 1) return [...models][0];
|
|
2665
|
+
return models.size > 1 ? "mixed" : "unknown";
|
|
2666
|
+
}
|
|
2667
|
+
/**
|
|
2668
|
+
* Adapter for plain `runCampaign` scenarios. This is the Runtime-side pair for
|
|
2669
|
+
* agent-eval fixture scenarios: load fixtures in `agent-eval/campaign`, build
|
|
2670
|
+
* the Runtime cell here, and keep paid-call admission, receipts, and traces
|
|
2671
|
+
* automatic.
|
|
2672
|
+
*/
|
|
2673
|
+
function loopCampaignDispatch(opts) {
|
|
2674
|
+
return (scenario, ctx) => {
|
|
2675
|
+
const loopOptions = opts.toLoopOptions(scenario);
|
|
2676
|
+
return runLoopWithCampaignContext(opts, loopOptions, ctx, {
|
|
2677
|
+
model: modelFromLoopOptions(loopOptions),
|
|
2678
|
+
maximumCharge: typeof opts.maximumCharge === "function" ? opts.maximumCharge(scenario) : opts.maximumCharge,
|
|
2679
|
+
resolveModel: opts.resolveCostModel ? (result) => opts.resolveCostModel?.(result, scenario) : void 0
|
|
2680
|
+
});
|
|
2382
2681
|
};
|
|
2383
2682
|
}
|
|
2683
|
+
/**
|
|
2684
|
+
* Adapter for `runProfileMatrix` (profile is an axis). Returns a
|
|
2685
|
+
* `ProfileDispatchFn` that runs `runAgentRounds` per (profile, scenario) cell
|
|
2686
|
+
* inside Eval's paid-call lifecycle.
|
|
2687
|
+
*/
|
|
2688
|
+
function loopDispatch(opts) {
|
|
2689
|
+
return (profile, scenario, ctx) => runLoopForCell(opts, scenario, profile, ctx);
|
|
2690
|
+
}
|
|
2384
2691
|
//#endregion
|
|
2385
|
-
//#region src/runtime/
|
|
2692
|
+
//#region src/runtime/inline-sandbox-client.ts
|
|
2386
2693
|
/**
|
|
2387
|
-
*
|
|
2388
|
-
*
|
|
2389
|
-
*
|
|
2390
|
-
* `
|
|
2391
|
-
*
|
|
2392
|
-
* The default observer reads execution evidence; an injected analysis owns its admitted sources.
|
|
2393
|
-
*
|
|
2394
|
-
* The nightly product job is then three lines:
|
|
2395
|
-
* const runs = mapSinkRowsToObserveInputs(await readSink(yesterday))
|
|
2396
|
-
* const report = await harvestCorpus({ runs, profile, executor, corpus })
|
|
2397
|
-
* log(report) // runsObserved / findings / learned / failures
|
|
2694
|
+
* The ONE pseudo-box adapter: present any one-shot `Executor` (router / bridge /
|
|
2695
|
+
* BYO) as a `SandboxClient` so the round-synchronous `runAgentRounds` can drive it
|
|
2696
|
+
* without each call site re-faking a box. This is the single shell that
|
|
2697
|
+
* `bench/src/router-executor.ts`, generate-eval's old `bridgeSandboxClient`, and
|
|
2698
|
+
* the search-bench bridge transport were each re-implementing.
|
|
2398
2699
|
*
|
|
2399
|
-
*
|
|
2700
|
+
* It is deliberately for NON-box executors only — a real sandbox harness already
|
|
2701
|
+
* IS a `SandboxClient` (boxes, sessions, fs, fork are real there). Here each
|
|
2702
|
+
* `streamPrompt` runs the executor once and emits its result plus the canonical
|
|
2703
|
+
* `done` event. The result carries output and usage; `done` confirms execution settled.
|
|
2704
|
+
* There are no sessions, no fs,
|
|
2705
|
+
* no fork (those degrade gracefully via the optional `SandboxClient` methods).
|
|
2400
2706
|
*/
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2707
|
+
function isAsyncIterable$1(v) {
|
|
2708
|
+
return typeof v === "object" && v !== null && Symbol.asyncIterator in v;
|
|
2709
|
+
}
|
|
2710
|
+
/** Drive a (possibly streaming) executor to its terminal artifact. */
|
|
2711
|
+
async function settle(exec, task, signal) {
|
|
2712
|
+
const r = exec.execute(task, signal);
|
|
2713
|
+
if (isAsyncIterable$1(r)) {
|
|
2714
|
+
for await (const _ of r);
|
|
2715
|
+
return exec.resultArtifact();
|
|
2408
2716
|
}
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
}
|
|
2424
|
-
|
|
2425
|
-
const iterator = Symbol.asyncIterator in Object(opts.runs) ? opts.runs[Symbol.asyncIterator]() : (async function* () {
|
|
2426
|
-
yield* opts.runs;
|
|
2427
|
-
})();
|
|
2428
|
-
let consumed = 0;
|
|
2429
|
-
let done = false;
|
|
2430
|
-
const next = async () => {
|
|
2431
|
-
if (done || opts.signal?.aborted || opts.maxRuns !== void 0 && consumed >= opts.maxRuns) return null;
|
|
2432
|
-
const sequence = ++consumed;
|
|
2433
|
-
let r;
|
|
2434
|
-
try {
|
|
2435
|
-
r = await iterator.next();
|
|
2436
|
-
} catch (error) {
|
|
2437
|
-
done = true;
|
|
2438
|
-
report.failures.push({
|
|
2439
|
-
runId: `source-${sequence}`,
|
|
2440
|
-
error: `trace source: ${error instanceof Error ? error.message.slice(0, 300) : String(error)}`
|
|
2441
|
-
});
|
|
2442
|
-
return null;
|
|
2443
|
-
}
|
|
2444
|
-
if (r.done) {
|
|
2445
|
-
done = true;
|
|
2446
|
-
return null;
|
|
2447
|
-
}
|
|
2717
|
+
return r;
|
|
2718
|
+
}
|
|
2719
|
+
/**
|
|
2720
|
+
* Adapt an `ExecutorFactory` into a `SandboxClient` for `runAgentRounds`. The factory is
|
|
2721
|
+
* instantiated fresh per `streamPrompt` (mirrors the per-spawn executor lifecycle):
|
|
2722
|
+
* run once on the prompt, emit the terminal result event, tear down.
|
|
2723
|
+
*
|
|
2724
|
+
* There is no box, so a per-prompt `backend` or `model` override is refused rather than dropped;
|
|
2725
|
+
* other per-prompt options (`timeoutMs`, `context`) are accepted and ignored.
|
|
2726
|
+
*/
|
|
2727
|
+
function inlineSandboxClient(factory, defaults = {}) {
|
|
2728
|
+
const capturedDefaultProfile = defaults.profile === void 0 ? void 0 : agentProfileSchema.parse(structuredClone(defaults.profile));
|
|
2729
|
+
let seq = 0;
|
|
2730
|
+
return { async create(options) {
|
|
2731
|
+
const id = `inline-${seq++}`;
|
|
2732
|
+
const createOptions = options;
|
|
2448
2733
|
return {
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
const
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2734
|
+
id,
|
|
2735
|
+
async *streamPrompt(message, opts) {
|
|
2736
|
+
assertBoxlessPromptOptions(opts, "inlineSandboxClient");
|
|
2737
|
+
const controller = new AbortController();
|
|
2738
|
+
const callerSignal = opts?.signal;
|
|
2739
|
+
const onAbort = () => controller.abort(callerSignal?.reason ?? /* @__PURE__ */ new Error("prompt aborted"));
|
|
2740
|
+
if (callerSignal) if (callerSignal.aborted) onAbort();
|
|
2741
|
+
else callerSignal.addEventListener("abort", onAbort, { once: true });
|
|
2742
|
+
const requestedProfile = (options?.backend && typeof options.backend === "object" ? options.backend.profile : void 0) ?? capturedDefaultProfile;
|
|
2743
|
+
const parsedProfile = agentProfileSchema.safeParse(requestedProfile);
|
|
2744
|
+
if (!parsedProfile.success) throw new Error("inlineSandboxClient: an exact AgentProfile is required; pass defaults.profile or create({ backend: { profile } })");
|
|
2745
|
+
const exec = factory({
|
|
2746
|
+
profile: parsedProfile.data,
|
|
2747
|
+
harness: null
|
|
2748
|
+
}, {
|
|
2749
|
+
signal: controller.signal,
|
|
2750
|
+
seams: { createOptions }
|
|
2751
|
+
});
|
|
2752
|
+
try {
|
|
2753
|
+
const artifact = await settle(exec, message, controller.signal);
|
|
2754
|
+
const out = artifact.out;
|
|
2755
|
+
const tokensIn = artifact.spent.tokens.input;
|
|
2756
|
+
const tokensOut = artifact.spent.tokens.output;
|
|
2757
|
+
const costUsd = artifact.spent.usd;
|
|
2758
|
+
const estimatedCostUsd = out?.estimatedCostUsd;
|
|
2759
|
+
if (artifact.spent.iterations > 0 || artifact.spent.tokensKnown === false || artifact.spent.usdKnown === false || tokensIn > 0 || tokensOut > 0 || costUsd > 0 || estimatedCostUsd !== void 0) yield {
|
|
2760
|
+
type: "llm_call",
|
|
2761
|
+
data: {
|
|
2762
|
+
...artifact.spent.tokensKnown === false ? {} : {
|
|
2763
|
+
tokensIn,
|
|
2764
|
+
tokensOut
|
|
2765
|
+
},
|
|
2766
|
+
...artifact.spent.usdKnown !== false ? { costUsd } : {},
|
|
2767
|
+
...artifact.spent.tokensKnown === false ? { tokensKnown: false } : {},
|
|
2768
|
+
...artifact.spent.usdKnown === false ? { costKnown: false } : {},
|
|
2769
|
+
...artifact.spent.usdKnown !== false ? {
|
|
2770
|
+
costKnown: true,
|
|
2771
|
+
costProvenance: "provider-receipt"
|
|
2772
|
+
} : {},
|
|
2773
|
+
...estimatedCostUsd !== void 0 ? { estimatedCostUsd } : {},
|
|
2774
|
+
...out?.promptCache ? { promptCache: out.promptCache } : {}
|
|
2775
|
+
}
|
|
2776
|
+
};
|
|
2777
|
+
yield {
|
|
2778
|
+
type: "result",
|
|
2779
|
+
data: {
|
|
2780
|
+
finalText: out?.content ?? "",
|
|
2781
|
+
...artifact.spent.tokensKnown === false ? { tokensKnown: false } : { tokenUsage: {
|
|
2782
|
+
inputTokens: tokensIn,
|
|
2783
|
+
outputTokens: tokensOut
|
|
2784
|
+
} },
|
|
2785
|
+
...artifact.spent.usdKnown === false ? {
|
|
2786
|
+
costKnown: false,
|
|
2787
|
+
...costUsd > 0 ? { costUsd } : {}
|
|
2788
|
+
} : {
|
|
2789
|
+
costUsd,
|
|
2790
|
+
costKnown: true,
|
|
2791
|
+
costProvenance: "provider-receipt"
|
|
2792
|
+
},
|
|
2793
|
+
...estimatedCostUsd !== void 0 ? { estimatedCostUsd } : {},
|
|
2794
|
+
...out?.promptCache ? { promptCache: out.promptCache } : {}
|
|
2795
|
+
}
|
|
2796
|
+
};
|
|
2797
|
+
yield {
|
|
2798
|
+
type: "done",
|
|
2799
|
+
data: { outcome: { type: "completed" } }
|
|
2800
|
+
};
|
|
2801
|
+
} finally {
|
|
2802
|
+
callerSignal?.removeEventListener("abort", onAbort);
|
|
2803
|
+
await exec.teardown("brutalKill").catch(() => {});
|
|
2470
2804
|
}
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
}
|
|
2476
|
-
}
|
|
2477
|
-
});
|
|
2478
|
-
await Promise.all(workers);
|
|
2479
|
-
if (report.runsObserved === 0 && report.failures.length > 0) throw new HarvestError(report);
|
|
2480
|
-
return report;
|
|
2805
|
+
},
|
|
2806
|
+
async delete() {}
|
|
2807
|
+
};
|
|
2808
|
+
} };
|
|
2481
2809
|
}
|
|
2482
2810
|
//#endregion
|
|
2483
|
-
//#region src/runtime/
|
|
2811
|
+
//#region src/runtime/local-sandbox-client.ts
|
|
2484
2812
|
/**
|
|
2485
|
-
*
|
|
2486
|
-
* `
|
|
2487
|
-
*
|
|
2488
|
-
*
|
|
2813
|
+
* `localSandboxClient` — the SAME-HOST pseudo-box: a `SandboxClient` whose
|
|
2814
|
+
* `create()` MATERIALIZES the profile's stdio MCP servers as local child
|
|
2815
|
+
* processes (`materializeLocalMcp`) and whose `streamPrompt` drives a real
|
|
2816
|
+
* tool loop (`runBrainLoop` over the router brain) with those live tools.
|
|
2489
2817
|
*
|
|
2490
|
-
*
|
|
2491
|
-
*
|
|
2492
|
-
*
|
|
2493
|
-
*
|
|
2494
|
-
* literal can NEVER structurally satisfy it, so every offline seam was forced to
|
|
2495
|
-
* write `as unknown as SandboxInstance`. The one unavoidable cast now lives HERE,
|
|
2496
|
-
* documented and tested, instead of in every example.
|
|
2818
|
+
* Despite the interface name, this does NOT isolate processes. It is only for
|
|
2819
|
+
* author-controlled profiles whose local MCP commands a caller explicitly
|
|
2820
|
+
* trusts; user- or model-authored code belongs in a real sandbox. `delete()`
|
|
2821
|
+
* kills the children.
|
|
2497
2822
|
*
|
|
2498
|
-
*
|
|
2499
|
-
*
|
|
2500
|
-
*
|
|
2501
|
-
* - `streamPrompt(prompt, opts?)` — runs `onPrompt` and streams its events.
|
|
2502
|
-
* - `fs.read` / `fs.write` — over the optional real `workdir` (the deliverable
|
|
2503
|
-
* artifact + any seeded fixtures live there). Present only when a `workdir`
|
|
2504
|
-
* is given.
|
|
2505
|
-
* - `exec(command)` — runs the command in `workdir`. Present only when a
|
|
2506
|
-
* `workdir` is given.
|
|
2507
|
-
* - `delete()` — removes the per-box temp dir (when one was minted) and is a
|
|
2508
|
-
* no-op otherwise; `deleteBoxSafe` calls it at teardown.
|
|
2823
|
+
* A per-create profile may change the prompt surface. Permission to start a
|
|
2824
|
+
* local MCP process applies only when its full canonical bytes match the fixed
|
|
2825
|
+
* constructor profile; a different generated profile is refused.
|
|
2509
2826
|
*
|
|
2510
|
-
*
|
|
2511
|
-
*
|
|
2512
|
-
* (`{ tokensIn, tokensOut, costUsd }`) or a terminal `done`
|
|
2513
|
-
* (`{ tokenUsage, totalCostUsd }`) / `result` event and the kernel's cost ledger
|
|
2514
|
-
* picks it up. No cast is needed on the events (the union is structural); only
|
|
2515
|
-
* the box object is cast, once, inside this file.
|
|
2827
|
+
* Event protocol matches `inlineSandboxClient`: known token usage is emitted as one `llm_call`;
|
|
2828
|
+
* Router catalog cost remains a separately-labelled estimate, never billed spend.
|
|
2516
2829
|
*/
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
}
|
|
2520
|
-
/**
|
|
2521
|
-
* Adapt a single `onPrompt(prompt, ctx)` callback into a `SandboxClient` for
|
|
2522
|
-
* `runAgentRounds` / `openSandboxRun`. Returns a PROPERLY-TYPED `SandboxClient`: the
|
|
2523
|
-
* lone `SandboxInstance` cast (object literal → `declare class`) lives inside
|
|
2524
|
-
* this function, so call sites stay cast-free.
|
|
2525
|
-
*
|
|
2526
|
-
* There is no box, so a per-prompt `backend` override is refused rather than dropped. A per-turn
|
|
2527
|
-
* `model` IS delivered, because `onPrompt` receives the verbatim options and is free to honor it.
|
|
2528
|
-
* Other per-prompt options (`timeoutMs`, `context`) are accepted and ignored.
|
|
2830
|
+
/** A same-host `SandboxClient` adapter with no process isolation. Local MCP is
|
|
2831
|
+
* refused unless the caller explicitly supplies a policy that allows it.
|
|
2529
2832
|
*
|
|
2530
|
-
*
|
|
2531
|
-
*/
|
|
2532
|
-
function
|
|
2533
|
-
const
|
|
2833
|
+
* There is no box, so a per-prompt `backend` or `model` override is refused rather than dropped;
|
|
2834
|
+
* other per-prompt options (`timeoutMs`, `context`) are accepted and ignored. */
|
|
2835
|
+
function localSandboxClient(opts) {
|
|
2836
|
+
const defaultProfile = opts.profile === void 0 ? void 0 : executableAgentProfileSnapshot(opts.profile, "localSandboxClient default profile");
|
|
2837
|
+
const router = Object.freeze({ ...opts.router });
|
|
2838
|
+
if (opts.profileSecurityPolicy?.allowLocalMcp && defaultProfile === void 0) throw new ValidationError("localSandboxClient: allowLocalMcp requires a fixed author-controlled profile; dynamic profiles need a real sandbox");
|
|
2839
|
+
const trustedProfileDigest = opts.profileSecurityPolicy?.allowLocalMcp && defaultProfile !== void 0 ? canonicalAgentProfileDigest(defaultProfile) : void 0;
|
|
2534
2840
|
let seq = 0;
|
|
2535
|
-
return { async create(
|
|
2536
|
-
const
|
|
2537
|
-
const
|
|
2538
|
-
const
|
|
2539
|
-
|
|
2540
|
-
const
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
}
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
});
|
|
2560
|
-
return {
|
|
2561
|
-
exitCode: 0,
|
|
2562
|
-
stdout,
|
|
2563
|
-
stderr
|
|
2564
|
-
};
|
|
2565
|
-
} catch (err) {
|
|
2566
|
-
const e = err;
|
|
2567
|
-
return {
|
|
2568
|
-
exitCode: e.code ?? 1,
|
|
2569
|
-
stdout: e.stdout ?? "",
|
|
2570
|
-
stderr: e.stderr ?? e.message ?? ""
|
|
2571
|
-
};
|
|
2572
|
-
}
|
|
2573
|
-
}
|
|
2574
|
-
} : {};
|
|
2575
|
-
async function* drive(behavior, message, opts) {
|
|
2576
|
-
const prompt = typeof message === "string" ? message : JSON.stringify(message);
|
|
2577
|
-
const { signal: optSignal, ...rest } = opts ?? {};
|
|
2578
|
-
const signal = optSignal ?? new AbortController().signal;
|
|
2579
|
-
const ctx = {
|
|
2580
|
-
round,
|
|
2581
|
-
workdir: boxWorkdir,
|
|
2582
|
-
signal,
|
|
2583
|
-
...Object.keys(rest).length > 0 ? { options: rest } : {}
|
|
2584
|
-
};
|
|
2585
|
-
round += 1;
|
|
2586
|
-
const produced = await behavior(prompt, ctx);
|
|
2587
|
-
if (isAsyncIterable(produced)) for await (const ev of produced) yield ev;
|
|
2588
|
-
else for (const ev of produced) yield ev;
|
|
2589
|
-
}
|
|
2841
|
+
return { async create(options) {
|
|
2842
|
+
const profile = executableAgentProfileSnapshot((options?.backend)?.profile ?? defaultProfile, "localSandboxClient");
|
|
2843
|
+
const model = profileProviderModel(profile);
|
|
2844
|
+
const settings = profileModelExecutionSettings(profile, "localSandboxClient");
|
|
2845
|
+
const policyApplies = opts.profileSecurityPolicy !== void 0 && (!opts.profileSecurityPolicy.allowLocalMcp || trustedProfileDigest !== void 0 && canonicalAgentProfileDigest(profile) === trustedProfileDigest);
|
|
2846
|
+
const mcp = await materializeLocalMcp(profile, {
|
|
2847
|
+
...opts.keys ? { keys: opts.keys } : {},
|
|
2848
|
+
...policyApplies ? { profileSecurityPolicy: opts.profileSecurityPolicy } : {}
|
|
2849
|
+
});
|
|
2850
|
+
const brain = routerBrain({
|
|
2851
|
+
routerBaseUrl: router.baseUrl,
|
|
2852
|
+
routerKey: router.key,
|
|
2853
|
+
model,
|
|
2854
|
+
...settings.retry !== void 0 ? { retry: settings.retry } : {},
|
|
2855
|
+
...enforceTokenLimits(settings.tokenLimits, "router", "localSandboxClient").applied,
|
|
2856
|
+
...settings.stream !== void 0 ? { stream: settings.stream } : {}
|
|
2857
|
+
}, {
|
|
2858
|
+
...settings.temperature !== void 0 ? { temperature: settings.temperature } : {},
|
|
2859
|
+
...settings.seed !== void 0 ? { seed: settings.seed } : {},
|
|
2860
|
+
...settings.toolChoice !== void 0 ? { toolChoice: settings.toolChoice } : {},
|
|
2861
|
+
...settings.extraBody !== void 0 ? { extraBody: settings.extraBody } : {},
|
|
2862
|
+
...profile.model?.reasoningEffort ? { reasoningEffort: profile.model.reasoningEffort } : {}
|
|
2863
|
+
});
|
|
2864
|
+
const system = [profile.prompt?.systemPrompt, ...profile.prompt?.instructions ?? []].filter((s) => typeof s === "string" && s.trim().length > 0).join("\n\n");
|
|
2590
2865
|
return {
|
|
2591
|
-
id
|
|
2592
|
-
streamPrompt(message,
|
|
2593
|
-
assertBoxlessPromptOptions(
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2866
|
+
id: `local-${seq++}`,
|
|
2867
|
+
async *streamPrompt(message, popts) {
|
|
2868
|
+
assertBoxlessPromptOptions(popts, "localSandboxClient");
|
|
2869
|
+
let estimatedCostUsd = 0;
|
|
2870
|
+
let sawEstimatedCost = false;
|
|
2871
|
+
const chat = async (messages, tools) => {
|
|
2872
|
+
const r = await brain(messages, tools);
|
|
2873
|
+
if (r.costProvenance === "catalog-estimate" && r.costUsd !== void 0) {
|
|
2874
|
+
estimatedCostUsd += r.costUsd;
|
|
2875
|
+
sawEstimatedCost = true;
|
|
2876
|
+
}
|
|
2877
|
+
return r;
|
|
2878
|
+
};
|
|
2879
|
+
const r = await runBrainLoop({
|
|
2880
|
+
chat,
|
|
2881
|
+
tools: mcp.tools,
|
|
2882
|
+
execute: (name, args) => mcp.call(name, args),
|
|
2883
|
+
initialMessages: [...system ? [{
|
|
2884
|
+
role: "system",
|
|
2885
|
+
content: system
|
|
2886
|
+
}] : [], {
|
|
2887
|
+
role: "user",
|
|
2888
|
+
content: message
|
|
2889
|
+
}],
|
|
2890
|
+
maxTurns: settings.maxTurns ?? 0,
|
|
2891
|
+
hooks: { stopBefore: () => popts?.signal?.aborted === true }
|
|
2601
2892
|
});
|
|
2893
|
+
if (r.turns > 0) yield {
|
|
2894
|
+
type: "llm_call",
|
|
2895
|
+
data: {
|
|
2896
|
+
model,
|
|
2897
|
+
...r.tokensKnown === false ? { tokensKnown: false } : {
|
|
2898
|
+
tokensIn: r.usage.input,
|
|
2899
|
+
tokensOut: r.usage.output
|
|
2900
|
+
},
|
|
2901
|
+
costKnown: false,
|
|
2902
|
+
...sawEstimatedCost ? { estimatedCostUsd } : {}
|
|
2903
|
+
}
|
|
2904
|
+
};
|
|
2905
|
+
yield {
|
|
2906
|
+
type: "result",
|
|
2907
|
+
data: {
|
|
2908
|
+
finalText: r.final,
|
|
2909
|
+
...r.tokensKnown === false ? { tokensKnown: false } : { tokenUsage: {
|
|
2910
|
+
inputTokens: r.usage.input,
|
|
2911
|
+
outputTokens: r.usage.output
|
|
2912
|
+
} },
|
|
2913
|
+
costKnown: false,
|
|
2914
|
+
...sawEstimatedCost ? { estimatedCostUsd } : {}
|
|
2915
|
+
}
|
|
2916
|
+
};
|
|
2917
|
+
},
|
|
2918
|
+
async delete() {
|
|
2919
|
+
await mcp.close();
|
|
2602
2920
|
}
|
|
2603
2921
|
};
|
|
2604
2922
|
} };
|
|
2605
2923
|
}
|
|
2606
2924
|
//#endregion
|
|
2607
|
-
//#region src/runtime/
|
|
2608
|
-
const toolchains = [
|
|
2609
|
-
"/usr",
|
|
2610
|
-
"/bin",
|
|
2611
|
-
"/lib",
|
|
2612
|
-
"/lib64"
|
|
2613
|
-
];
|
|
2614
|
-
const contains = (parent, child) => {
|
|
2615
|
-
const path = relative(parent, child);
|
|
2616
|
-
return path === "" || path !== ".." && !path.startsWith(`..${sep}`) && !path.startsWith(sep);
|
|
2617
|
-
};
|
|
2925
|
+
//#region src/runtime/resolve-sandbox-client.ts
|
|
2618
2926
|
/**
|
|
2619
|
-
*
|
|
2620
|
-
*
|
|
2927
|
+
* The product-facing backend selector: one call picks the execution transport a
|
|
2928
|
+
* `runAgentRounds` (or any consumer that drives a `SandboxClient`) runs on, from the
|
|
2929
|
+
* package a product already depends on. It is pure sugar over the existing
|
|
2930
|
+
* primitives — `createExecutor({ backend })` + `inlineSandboxClient` — and adds
|
|
2931
|
+
* ZERO transport code (the node:http cli-bridge transport lives in `bridgeExecutor`).
|
|
2621
2932
|
*
|
|
2622
|
-
*
|
|
2623
|
-
*
|
|
2624
|
-
*
|
|
2625
|
-
* Limits bound command time and captured output, not copy size or memory consumption.
|
|
2626
|
-
* Callers must keep the input and trusted toolchains stable while preparing the check.
|
|
2933
|
+
* The heavy `resolveBenchClient` in `@tangle-network/agent-bench` is the bench-side
|
|
2934
|
+
* sibling: it layers router/search specifics a product should not have to depend on.
|
|
2935
|
+
* This is the generic, dep-light core products call to switch backends:
|
|
2627
2936
|
*
|
|
2628
|
-
*
|
|
2629
|
-
*
|
|
2937
|
+
* - `backend: 'sandbox'` → IN-BOX: return the caller's real Sandbox-backed
|
|
2938
|
+
* `sandboxClient` unchanged. Fail loud if absent — a
|
|
2939
|
+
* product on the sandbox backend already holds its box.
|
|
2940
|
+
* - `backend: 'bridge'` → OFF-BOX: a local cli-bridge fronting a harness CLI
|
|
2941
|
+
* (opencode / kimi-code / …) as the leaf executor, wired
|
|
2942
|
+
* through the resumable `bridgeExecutor`.
|
|
2943
|
+
* - `backend: 'router'` → OFF-BOX: a router chat-completion as the leaf executor,
|
|
2944
|
+
* presented as a `SandboxClient` (no sandbox dependency).
|
|
2945
|
+
* - `backend: 'local'` → SAME-HOST: a router-brain tool loop with the
|
|
2946
|
+
* profile's stdio MCP servers spawned as LOCAL child
|
|
2947
|
+
* processes — the only backend that can reach an MCP
|
|
2948
|
+
* server built into a host worktree.
|
|
2630
2949
|
*/
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
|
|
2649
|
-
}
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
}
|
|
2665
|
-
result = await execute([
|
|
2666
|
-
"--unshare-all",
|
|
2667
|
-
"--die-with-parent",
|
|
2668
|
-
"--new-session",
|
|
2669
|
-
"--cap-drop",
|
|
2670
|
-
"ALL",
|
|
2671
|
-
"--clearenv",
|
|
2672
|
-
"--setenv",
|
|
2673
|
-
"PATH",
|
|
2674
|
-
"/usr/bin:/bin",
|
|
2675
|
-
"--setenv",
|
|
2676
|
-
"HOME",
|
|
2677
|
-
"/tmp",
|
|
2678
|
-
"--setenv",
|
|
2679
|
-
"LANG",
|
|
2680
|
-
"C",
|
|
2681
|
-
...binds,
|
|
2682
|
-
"--proc",
|
|
2683
|
-
"/proc",
|
|
2684
|
-
"--dev",
|
|
2685
|
-
"/dev",
|
|
2686
|
-
"--tmpfs",
|
|
2687
|
-
"/tmp",
|
|
2688
|
-
"--bind",
|
|
2689
|
-
copy,
|
|
2690
|
-
tree,
|
|
2691
|
-
"--chdir",
|
|
2692
|
-
tree,
|
|
2693
|
-
"--",
|
|
2694
|
-
...options.command
|
|
2695
|
-
], timeoutMs, maxOutputBytes, options.signal);
|
|
2696
|
-
} catch (error) {
|
|
2697
|
-
result = {
|
|
2698
|
-
succeeded: false,
|
|
2699
|
-
reason: "refused",
|
|
2700
|
-
diagnostic: String(error)
|
|
2701
|
-
};
|
|
2702
|
-
}
|
|
2703
|
-
if (scratch) try {
|
|
2704
|
-
await removeScratch(scratch);
|
|
2705
|
-
} catch (error) {
|
|
2706
|
-
const cleanupDiagnostic = String(error);
|
|
2707
|
-
result = result.succeeded ? {
|
|
2708
|
-
succeeded: false,
|
|
2709
|
-
reason: "cleanup-failed",
|
|
2710
|
-
diagnostic: cleanupDiagnostic,
|
|
2711
|
-
stdout: result.value.stdout,
|
|
2712
|
-
stderr: result.value.stderr,
|
|
2713
|
-
exitCode: 0
|
|
2714
|
-
} : {
|
|
2715
|
-
...result,
|
|
2716
|
-
cleanupDiagnostic
|
|
2717
|
-
};
|
|
2718
|
-
}
|
|
2719
|
-
return result;
|
|
2720
|
-
}
|
|
2721
|
-
function checkLimits(options) {
|
|
2722
|
-
const timeoutMs = options.timeoutMs ?? 3e4;
|
|
2723
|
-
const maxOutputBytes = options.maxOutputBytes ?? 1048576;
|
|
2724
|
-
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 2147483647 || !Number.isSafeInteger(maxOutputBytes) || maxOutputBytes < 1) throw new Error("Invalid execution limits");
|
|
2725
|
-
if (!options.command.length || !options.command[0]) throw new Error("A command is required");
|
|
2726
|
-
return {
|
|
2727
|
-
timeoutMs,
|
|
2728
|
-
maxOutputBytes
|
|
2729
|
-
};
|
|
2730
|
-
}
|
|
2731
|
-
async function protectedTree(options) {
|
|
2732
|
-
const workspace = await realpath(options.workspaceRoot);
|
|
2733
|
-
const tree = await realpath(options.tree);
|
|
2734
|
-
if (!contains(workspace, tree) || !(await stat(tree)).isDirectory()) throw new Error("Tree must be a directory inside the workspace");
|
|
2735
|
-
return {
|
|
2736
|
-
workspace,
|
|
2737
|
-
tree
|
|
2738
|
-
};
|
|
2739
|
-
}
|
|
2740
|
-
/** Box-workspace-relative directory that receives the tree and is the command's cwd. */
|
|
2741
|
-
const BOX_TREE = "tree";
|
|
2742
|
-
/** Bound on provisioning, delivery, and deletion; the box expires even if this process dies. */
|
|
2743
|
-
const BOX_SETUP_SECONDS = 900;
|
|
2744
|
-
/** The box's own process timeout is a backstop behind this module's deadline. */
|
|
2745
|
-
const BOX_PROCESS_BACKSTOP_MS = 3e4;
|
|
2746
|
-
/** Concurrent file deliveries into one check box. */
|
|
2747
|
-
const BOX_UPLOADS = 4;
|
|
2748
|
-
/** The box's process supervisor prefixes this to stderr when it cannot start the executable. */
|
|
2749
|
-
const BOX_EXEC_FAILURE = "process-supervisor: exec failed";
|
|
2750
|
-
async function runInBox(options, placement) {
|
|
2751
|
-
let box;
|
|
2752
|
-
let evidence;
|
|
2753
|
-
let result;
|
|
2754
|
-
try {
|
|
2755
|
-
const { timeoutMs, maxOutputBytes } = checkLimits(options);
|
|
2756
|
-
if (options.signal?.aborted) return {
|
|
2757
|
-
succeeded: false,
|
|
2758
|
-
reason: "cancelled",
|
|
2759
|
-
diagnostic: "Check cancelled"
|
|
2760
|
-
};
|
|
2761
|
-
const { tree } = await protectedTree(options);
|
|
2762
|
-
const captured = await captureMaterializedWorkspace(tree);
|
|
2763
|
-
if (placement.builderAccounts.length === 0 || placement.builderAccounts.some((id) => !id)) throw new Error("builderAccounts must name the account of every key the judged run holds");
|
|
2764
|
-
const account = (await placement.client.getIdentity()).customerId;
|
|
2765
|
-
if (!account) throw new Error("Sandbox returned no account for the check key");
|
|
2766
|
-
if (placement.builderAccounts.includes(account)) throw new Error(`Check account ${account} is an account the judged run holds a key to, so the run can reach the check box`);
|
|
2767
|
-
options.signal?.throwIfAborted();
|
|
2768
|
-
box = await placement.client.createIsolated({
|
|
2769
|
-
environment: placement.environment,
|
|
2770
|
-
agent: false,
|
|
2771
|
-
bare: false,
|
|
2772
|
-
ephemeral: true,
|
|
2773
|
-
egressPolicy: { mode: "blocked" },
|
|
2774
|
-
...placement.resources ? { resources: placement.resources } : {},
|
|
2775
|
-
maxLifetimeSeconds: Math.ceil(timeoutMs / 1e3) + BOX_SETUP_SECONDS,
|
|
2776
|
-
idempotencyKey: `agent-runtime-check-${randomUUID()}`
|
|
2777
|
-
}, options.signal ? { signal: options.signal } : void 0);
|
|
2778
|
-
const receipt = box.createReceipt();
|
|
2779
|
-
if (receipt?.outcome !== "created" || receipt.ownerContext !== "isolated" || !Array.isArray(receipt.injectedSecrets) || receipt.injectedSecrets.length !== 0) throw new Error("Sandbox did not confirm a fresh box with no owner secrets");
|
|
2780
|
-
const egress = (await box.egress.get()).policy;
|
|
2781
|
-
if (egress.mode !== "blocked") throw new Error(`Check box egress is ${egress.mode}, not blocked`);
|
|
2782
|
-
await box.fs.mkdir(BOX_TREE, { recursive: true });
|
|
2783
|
-
const digests = new Map(captured.manifest.files.map((entry) => [entry.path, entry.sha256]));
|
|
2784
|
-
const target = box;
|
|
2785
|
-
const pending = [...captured.files];
|
|
2786
|
-
const deliver = async () => {
|
|
2787
|
-
try {
|
|
2788
|
-
for (let file = pending.shift(); file; file = pending.shift()) {
|
|
2789
|
-
options.signal?.throwIfAborted();
|
|
2790
|
-
const received = await target.fs.uploadData(`${BOX_TREE}/${file.path}`, file.bytes, { mode: file.mode });
|
|
2791
|
-
if (`sha256:${received.hash.replace(/^sha256:/, "").toLowerCase()}` !== digests.get(file.path) || received.size !== file.bytes.byteLength) throw new Error(`Check box received different bytes for ${file.path}`);
|
|
2792
|
-
}
|
|
2793
|
-
} catch (error) {
|
|
2794
|
-
pending.length = 0;
|
|
2795
|
-
throw error;
|
|
2796
|
-
}
|
|
2797
|
-
};
|
|
2798
|
-
await Promise.all(Array.from({ length: BOX_UPLOADS }, deliver));
|
|
2799
|
-
evidence = {
|
|
2800
|
-
sandboxId: box.id,
|
|
2801
|
-
account,
|
|
2802
|
-
input: captured.manifest,
|
|
2803
|
-
inputDigest: canonicalCandidateDigest(captured.manifest)
|
|
2804
|
-
};
|
|
2805
|
-
result = await executeInBox(box, options.command, timeoutMs, maxOutputBytes, options.signal);
|
|
2806
|
-
} catch (error) {
|
|
2807
|
-
result = {
|
|
2808
|
-
succeeded: false,
|
|
2809
|
-
reason: options.signal?.aborted ? "cancelled" : "refused",
|
|
2810
|
-
diagnostic: String(error)
|
|
2811
|
-
};
|
|
2812
|
-
}
|
|
2813
|
-
if (box) try {
|
|
2814
|
-
await box.delete();
|
|
2815
|
-
} catch (error) {
|
|
2816
|
-
const cleanupDiagnostic = `Check box ${box.id} was not deleted: ${String(error)}`;
|
|
2817
|
-
result = result.succeeded ? {
|
|
2818
|
-
succeeded: false,
|
|
2819
|
-
reason: "cleanup-failed",
|
|
2820
|
-
diagnostic: cleanupDiagnostic,
|
|
2821
|
-
stdout: result.value.stdout,
|
|
2822
|
-
stderr: result.value.stderr,
|
|
2823
|
-
exitCode: 0
|
|
2824
|
-
} : {
|
|
2825
|
-
...result,
|
|
2826
|
-
cleanupDiagnostic
|
|
2827
|
-
};
|
|
2950
|
+
/**
|
|
2951
|
+
* Resolve a `SandboxClient` for the chosen backend. The generic, dep-light core
|
|
2952
|
+
* that `resolveBenchClient` builds on — reuse this instead of hand-rolling the
|
|
2953
|
+
* `createExecutor`/`inlineSandboxClient` branch in each product.
|
|
2954
|
+
*/
|
|
2955
|
+
function resolveSandboxClient(opts) {
|
|
2956
|
+
switch (opts.backend) {
|
|
2957
|
+
case "sandbox":
|
|
2958
|
+
if (!opts.sandboxClient) throw new Error("resolveSandboxClient: backend 'sandbox' requires opts.sandboxClient");
|
|
2959
|
+
return opts.sandboxClient;
|
|
2960
|
+
case "bridge": {
|
|
2961
|
+
const bridge = opts.bridge;
|
|
2962
|
+
if (!bridge?.bearer) throw new Error("resolveSandboxClient: backend 'bridge' requires bridge.bearer");
|
|
2963
|
+
return inlineSandboxClient(createExecutor({
|
|
2964
|
+
backend: "bridge",
|
|
2965
|
+
bridgeUrl: bridge.url ?? "http://127.0.0.1:3355",
|
|
2966
|
+
bridgeBearer: bridge.bearer,
|
|
2967
|
+
timeoutMs: bridge.timeoutMs
|
|
2968
|
+
}));
|
|
2969
|
+
}
|
|
2970
|
+
case "router": {
|
|
2971
|
+
const router = opts.router;
|
|
2972
|
+
if (!router?.baseUrl || !router.key) throw new Error("resolveSandboxClient: backend 'router' requires router.baseUrl and router.key");
|
|
2973
|
+
return inlineSandboxClient(createExecutor({
|
|
2974
|
+
backend: "router",
|
|
2975
|
+
routerBaseUrl: router.baseUrl,
|
|
2976
|
+
routerKey: router.key
|
|
2977
|
+
}));
|
|
2978
|
+
}
|
|
2979
|
+
case "local": {
|
|
2980
|
+
const local = opts.local;
|
|
2981
|
+
if (!local?.router?.baseUrl || !local.router.key) throw new Error("resolveSandboxClient: backend 'local' requires local.router.baseUrl and local.router.key");
|
|
2982
|
+
return localSandboxClient(local);
|
|
2983
|
+
}
|
|
2828
2984
|
}
|
|
2829
|
-
return evidence ? {
|
|
2830
|
-
...result,
|
|
2831
|
-
box: evidence
|
|
2832
|
-
} : result;
|
|
2833
2985
|
}
|
|
2834
|
-
|
|
2835
|
-
|
|
2836
|
-
|
|
2837
|
-
|
|
2838
|
-
|
|
2839
|
-
|
|
2840
|
-
|
|
2841
|
-
|
|
2986
|
+
//#endregion
|
|
2987
|
+
//#region src/runtime/define-leaderboard.ts
|
|
2988
|
+
/**
|
|
2989
|
+
* `defineLeaderboard` — the declarative eval-leaderboard facade.
|
|
2990
|
+
*
|
|
2991
|
+
* A product's harness×model leaderboard is always the same assembly: expand a
|
|
2992
|
+
* base profile across the harness×model axes (`expandProfileAxes`), run every
|
|
2993
|
+
* (profile, case) cell as a driven loop (`loopDispatch` + the naive retry driver), score
|
|
2994
|
+
* with the domain's grader, and emit ONE `runProfileMatrix` call. Each product
|
|
2995
|
+
* hand-rolled that assembly (~650 lines each) and re-hit the same footguns:
|
|
2996
|
+
* stale cell-cache reuse, zero-token stub cells, missing model snapshots.
|
|
2997
|
+
*
|
|
2998
|
+
* This facade IS that assembly, once, with the domain reduced to a declarative
|
|
2999
|
+
* spec: `cases` + `prompt` + `score`. It contains NO execution, judging, or
|
|
3000
|
+
* metering logic of its own — every moving part is an existing primitive, and
|
|
3001
|
+
* every default is overridable:
|
|
3002
|
+
*
|
|
3003
|
+
* - LEVEL 0 (declarative): `cases` / `prompt` / `score` / `axis`.
|
|
3004
|
+
* - LEVEL 1 (seams): `backends`, `flags`, `parseOutput`, `onCellEvents`,
|
|
3005
|
+
* `resolveModel`, `setup`/`teardown`, `export`, `matrix`
|
|
3006
|
+
* passthrough.
|
|
3007
|
+
* - LEVEL 2 (replacement): `dispatch` and `judges` swap out the whole
|
|
3008
|
+
* loop wiring or scoring; `runProfileMatrix` itself stays public as the
|
|
3009
|
+
* escape floor — a product overriding everything just writes what it has
|
|
3010
|
+
* today, no capability removed.
|
|
3011
|
+
*
|
|
3012
|
+
* `toBenchmarkAdapter()` exposes the same domain surface in the structural
|
|
3013
|
+
* `BenchmarkAdapter` shape (`name`/`preflight`/`loadTasks`/`judge`/
|
|
3014
|
+
* `goldArtifact`) so a product leaderboard can register into a benchmark
|
|
3015
|
+
* registry without this module depending on one.
|
|
3016
|
+
*
|
|
3017
|
+
* @experimental
|
|
3018
|
+
*/
|
|
3019
|
+
/**
|
|
3020
|
+
* The no-signal retry floor as a bare `Driver` literal: re-run the case VERBATIM as an
|
|
3021
|
+
* independent attempt until one shot scores (`verdict.valid`) or the shot cap. It reads ONLY
|
|
3022
|
+
* `verdict.valid` — never `notes`/`scores` — so a leaderboard cell's retries carry no grader
|
|
3023
|
+
* findings into the next shot (the leak-free firewall). Steering POLICY texts are registry data
|
|
3024
|
+
* (`delegates/naive-continuation` and siblings in the kernel prompt registry) attached to graph
|
|
3025
|
+
* edges; this loop-kernel control folds no text in — each shot is the same task object, and the
|
|
3026
|
+
* per-shot nonce in `taskToPrompt` keeps the attempts independent at the router.
|
|
3027
|
+
*/
|
|
3028
|
+
function naiveRetryDriver(shots) {
|
|
3029
|
+
return {
|
|
3030
|
+
name: "naive",
|
|
3031
|
+
plan(task, history) {
|
|
3032
|
+
if (history.length === 0) return Promise.resolve([task]);
|
|
3033
|
+
if (history[history.length - 1]?.verdict?.valid === true) return Promise.resolve([]);
|
|
3034
|
+
if (history.length >= shots) return Promise.resolve([]);
|
|
3035
|
+
return Promise.resolve([task]);
|
|
2842
3036
|
},
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
failure ??= reason;
|
|
2853
|
-
stopping ??= child.kill("SIGKILL", { tree: true }).catch(() => void 0);
|
|
2854
|
-
};
|
|
2855
|
-
const clearTimer = armDeadlineTimer(timeoutMs, () => stop("timeout"), true);
|
|
2856
|
-
const cancel = () => stop("cancelled");
|
|
2857
|
-
signal?.addEventListener("abort", cancel, { once: true });
|
|
2858
|
-
if (signal?.aborted) cancel();
|
|
2859
|
-
const capture = async (stream, target) => {
|
|
2860
|
-
for await (const text of stream) {
|
|
2861
|
-
const chunk = Buffer.from(text);
|
|
2862
|
-
const remaining = Math.max(0, limit - bytes);
|
|
2863
|
-
bytes += chunk.length;
|
|
2864
|
-
if (remaining) target.push(chunk.subarray(0, remaining));
|
|
2865
|
-
if (bytes > limit) stop("output-limit");
|
|
3037
|
+
decide(history) {
|
|
3038
|
+
if (history.some((it) => it.verdict?.valid)) return "pick-winner";
|
|
3039
|
+
return history.length < shots ? "refine" : "fail";
|
|
3040
|
+
},
|
|
3041
|
+
describePlan() {
|
|
3042
|
+
return {
|
|
3043
|
+
kind: "refine",
|
|
3044
|
+
rationale: "naive fixed continuation (no grade signal)"
|
|
3045
|
+
};
|
|
2866
3046
|
}
|
|
2867
3047
|
};
|
|
2868
|
-
|
|
2869
|
-
|
|
3048
|
+
}
|
|
3049
|
+
/** Read `--name <value>` from an argv array. */
|
|
3050
|
+
function argOf(argv, name) {
|
|
3051
|
+
const i = argv.indexOf(`--${name}`);
|
|
3052
|
+
if (i >= 0 && i + 1 < argv.length) return argv[i + 1];
|
|
3053
|
+
}
|
|
3054
|
+
function splitList(v) {
|
|
3055
|
+
if (v === void 0) return void 0;
|
|
3056
|
+
const parts = v.split(",").map((s) => s.trim()).filter(Boolean);
|
|
3057
|
+
return parts.length > 0 ? parts : void 0;
|
|
3058
|
+
}
|
|
3059
|
+
/** RunRecords reject a bare model id — the eval IDENTITY model must carry a
|
|
3060
|
+
* snapshot (`name@<snapshot>`). Unchanged when already stamped. */
|
|
3061
|
+
function withSnapshot(model, snapshot) {
|
|
3062
|
+
return model.includes("@") ? model : `${model}@${snapshot}`;
|
|
3063
|
+
}
|
|
3064
|
+
function gitSha() {
|
|
2870
3065
|
try {
|
|
2871
|
-
|
|
2872
|
-
|
|
2873
|
-
|
|
2874
|
-
} finally {
|
|
2875
|
-
clearTimer();
|
|
2876
|
-
signal?.removeEventListener("abort", cancel);
|
|
2877
|
-
await stopping;
|
|
3066
|
+
return execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim();
|
|
3067
|
+
} catch {
|
|
3068
|
+
return "unknown";
|
|
2878
3069
|
}
|
|
2879
|
-
|
|
2880
|
-
|
|
2881
|
-
|
|
2882
|
-
|
|
2883
|
-
|
|
2884
|
-
|
|
2885
|
-
|
|
2886
|
-
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
2891
|
-
|
|
2892
|
-
|
|
2893
|
-
succeeded: false,
|
|
2894
|
-
reason: "refused",
|
|
2895
|
-
diagnostic: err,
|
|
2896
|
-
...evidence
|
|
2897
|
-
};
|
|
2898
|
-
if (exitSignal) return {
|
|
2899
|
-
succeeded: false,
|
|
2900
|
-
reason: "failed",
|
|
2901
|
-
diagnostic: err || `Check killed by ${exitSignal}`,
|
|
2902
|
-
...evidence
|
|
3070
|
+
}
|
|
3071
|
+
function normalizeScore(s) {
|
|
3072
|
+
return typeof s === "number" ? { composite: s } : s;
|
|
3073
|
+
}
|
|
3074
|
+
/**
|
|
3075
|
+
* Assemble a declarative spec (`cases` + `prompt` + `score`) into a runnable
|
|
3076
|
+
* harness×model leaderboard — `run()` executes the matrix, `toBenchmarkAdapter()`
|
|
3077
|
+
* exposes the same domain as a structural `BenchmarkAdapter`.
|
|
3078
|
+
*/
|
|
3079
|
+
function defineLeaderboard(spec) {
|
|
3080
|
+
const caseId = (c) => {
|
|
3081
|
+
const id = spec.caseId ? spec.caseId(c) : c.id;
|
|
3082
|
+
if (typeof id !== "string" || id.length === 0) throw new Error(`defineLeaderboard(${spec.name}): every case needs a stable string id — give cases an \`id\` property or supply spec.caseId`);
|
|
3083
|
+
return id;
|
|
2903
3084
|
};
|
|
2904
|
-
|
|
2905
|
-
|
|
2906
|
-
|
|
2907
|
-
|
|
2908
|
-
|
|
3085
|
+
const selectCases = (ids) => {
|
|
3086
|
+
if (!ids) return spec.cases;
|
|
3087
|
+
const byId = new Map(spec.cases.map((c) => [caseId(c), c]));
|
|
3088
|
+
return ids.map((id) => {
|
|
3089
|
+
const c = byId.get(id);
|
|
3090
|
+
if (c === void 0) throw new Error(`defineLeaderboard(${spec.name}): unknown case "${id}" (have: ${[...byId.keys()].join(", ")})`);
|
|
3091
|
+
return c;
|
|
3092
|
+
});
|
|
2909
3093
|
};
|
|
2910
|
-
|
|
2911
|
-
|
|
2912
|
-
|
|
2913
|
-
|
|
2914
|
-
|
|
3094
|
+
const scoreJudge = {
|
|
3095
|
+
name: `${spec.name}-score`,
|
|
3096
|
+
dimensions: [{
|
|
3097
|
+
key: "composite",
|
|
3098
|
+
description: `${spec.name} case score`
|
|
3099
|
+
}],
|
|
3100
|
+
score({ artifact, scenario }) {
|
|
3101
|
+
const s = normalizeScore(spec.score(artifact, scenario.case));
|
|
3102
|
+
return {
|
|
3103
|
+
composite: s.composite,
|
|
3104
|
+
dimensions: {
|
|
3105
|
+
composite: s.composite,
|
|
3106
|
+
...s.dimensions
|
|
3107
|
+
},
|
|
3108
|
+
notes: s.notes ?? ""
|
|
3109
|
+
};
|
|
2915
3110
|
}
|
|
2916
3111
|
};
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
"
|
|
2922
|
-
"
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
|
|
3112
|
+
async function run(argv = process.argv.slice(2)) {
|
|
3113
|
+
const args = {};
|
|
3114
|
+
for (const name of [
|
|
3115
|
+
"backend",
|
|
3116
|
+
"harnesses",
|
|
3117
|
+
"models",
|
|
3118
|
+
"cases",
|
|
3119
|
+
"shots",
|
|
3120
|
+
"reps",
|
|
3121
|
+
"model-snapshot",
|
|
3122
|
+
"run-dir",
|
|
3123
|
+
"export-dir"
|
|
3124
|
+
]) args[name] = argOf(argv, name);
|
|
3125
|
+
for (const [name, flag] of Object.entries(spec.flags ?? {})) args[name] = argOf(argv, name) ?? flag.default;
|
|
3126
|
+
const backendName = args.backend ?? "sandbox";
|
|
3127
|
+
const shots = Number(args.shots ?? spec.shots ?? 1);
|
|
3128
|
+
const reps = Number(args.reps ?? spec.reps ?? 1);
|
|
3129
|
+
const snapshot = args["model-snapshot"] ?? "leaderboard";
|
|
3130
|
+
const runDir = args["run-dir"] ?? join(tmpdir(), `leaderboard-${spec.name}-${Date.now()}-${process.pid}`);
|
|
3131
|
+
const exportDir = args["export-dir"] ?? join(runDir, "export");
|
|
3132
|
+
mkdirSync(runDir, { recursive: true });
|
|
3133
|
+
const cases = selectCases(splitList(args.cases));
|
|
3134
|
+
if (cases.length === 0) throw new Error(`defineLeaderboard(${spec.name}): no cases to run`);
|
|
3135
|
+
const scenarios = cases.map((c) => ({
|
|
3136
|
+
id: caseId(c),
|
|
3137
|
+
kind: spec.name,
|
|
3138
|
+
case: c
|
|
3139
|
+
}));
|
|
3140
|
+
const harnesses = splitList(args.harnesses) ?? spec.axis?.harnesses ?? CODING_HARNESSES;
|
|
3141
|
+
const rawModels = splitList(args.models) ?? spec.axis?.models ?? (spec.baseProfile?.model?.default !== void 0 ? [spec.baseProfile.model.default] : void 0);
|
|
3142
|
+
if (!rawModels || rawModels.length === 0) throw new Error(`defineLeaderboard(${spec.name}): no models — pass --models, set spec.axis.models, or give spec.baseProfile a model.default`);
|
|
3143
|
+
const models = rawModels.map((m) => withSnapshot(m, snapshot));
|
|
3144
|
+
const profiles = expandProfileAxes({
|
|
3145
|
+
base: spec.baseProfile,
|
|
3146
|
+
harnesses,
|
|
3147
|
+
models
|
|
2933
3148
|
});
|
|
2934
|
-
const
|
|
2935
|
-
const
|
|
2936
|
-
|
|
2937
|
-
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
|
|
3149
|
+
for (const profile of profiles) assertExecutableAgentProfile(profile, `defineLeaderboard(${spec.name})`);
|
|
3150
|
+
const ctx = {
|
|
3151
|
+
name: spec.name,
|
|
3152
|
+
backend: backendName,
|
|
3153
|
+
runDir,
|
|
3154
|
+
exportDir,
|
|
3155
|
+
args,
|
|
3156
|
+
harnesses,
|
|
3157
|
+
models,
|
|
3158
|
+
caseIds: scenarios.map((s) => s.id),
|
|
3159
|
+
shots,
|
|
3160
|
+
reps
|
|
2946
3161
|
};
|
|
2947
|
-
const
|
|
2948
|
-
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
|
|
2952
|
-
|
|
2953
|
-
|
|
2954
|
-
|
|
2955
|
-
|
|
3162
|
+
const backends = {
|
|
3163
|
+
sandbox: () => {
|
|
3164
|
+
throw new Error(`defineLeaderboard(${spec.name}): the 'sandbox' backend needs your product's real Sandbox client — supply spec.backends.sandbox (e.g. () => new Sandbox({ apiKey, baseUrl }))`);
|
|
3165
|
+
},
|
|
3166
|
+
"cli-bridge": () => {
|
|
3167
|
+
const bearer = process.env.BRIDGE_BEARER ?? process.env.CLI_BRIDGE_BEARER;
|
|
3168
|
+
if (!bearer) throw new Error(`defineLeaderboard(${spec.name}): backend 'cli-bridge' needs BRIDGE_BEARER or CLI_BRIDGE_BEARER set`);
|
|
3169
|
+
return resolveSandboxClient({
|
|
3170
|
+
backend: "bridge",
|
|
3171
|
+
bridge: {
|
|
3172
|
+
url: process.env.CLI_BRIDGE_URL,
|
|
3173
|
+
bearer,
|
|
3174
|
+
timeoutMs: 9e5
|
|
3175
|
+
}
|
|
3176
|
+
});
|
|
3177
|
+
},
|
|
3178
|
+
...spec.backends
|
|
2956
3179
|
};
|
|
2957
|
-
|
|
2958
|
-
|
|
2959
|
-
|
|
2960
|
-
|
|
2961
|
-
|
|
2962
|
-
|
|
2963
|
-
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
|
|
2967
|
-
|
|
2968
|
-
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
2973
|
-
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
3180
|
+
const makeClient = backends[backendName];
|
|
3181
|
+
if (!makeClient) throw new Error(`defineLeaderboard(${spec.name}): unknown backend "${backendName}" (have: ${Object.keys(backends).join(", ")})`);
|
|
3182
|
+
const sandboxClient = makeClient();
|
|
3183
|
+
const promptById = /* @__PURE__ */ new Map();
|
|
3184
|
+
for (const s of scenarios) promptById.set(s.id, await spec.prompt(s.case));
|
|
3185
|
+
const promptOf = (s) => {
|
|
3186
|
+
const p = promptById.get(s.id);
|
|
3187
|
+
if (p === void 0) throw new Error(`defineLeaderboard(${spec.name}): no prompt for case "${s.id}"`);
|
|
3188
|
+
return p;
|
|
3189
|
+
};
|
|
3190
|
+
let shotNonce = 0;
|
|
3191
|
+
const maximumCharge = spec.maximumCharge;
|
|
3192
|
+
const dispatch = spec.dispatch ?? ((profile, scenario, dispatchCtx) => {
|
|
3193
|
+
return loopDispatch({
|
|
3194
|
+
sandboxClient,
|
|
3195
|
+
maximumCharge: typeof maximumCharge === "function" ? (cellScenario, cellProfile) => maximumCharge(cellProfile, cellScenario) : maximumCharge,
|
|
3196
|
+
resolveCostModel: (result, _cellScenario, cellProfile) => {
|
|
3197
|
+
if (!spec.resolveModel) return cellProfile.model?.default;
|
|
3198
|
+
const served = new Set(result.iterations.map((iteration) => spec.resolveModel?.(iteration.events)).filter((model) => model !== void 0));
|
|
3199
|
+
if (served.size > 1) throw new Error(`defineLeaderboard(${spec.name}): one cell reported multiple served models: ${[...served].join(", ")}`);
|
|
3200
|
+
return [...served][0] ?? cellProfile.model?.default;
|
|
3201
|
+
},
|
|
3202
|
+
toLoopOptions: (cellScenario, cellProfile) => {
|
|
3203
|
+
return {
|
|
3204
|
+
driver: naiveRetryDriver(shots),
|
|
3205
|
+
agentRun: {
|
|
3206
|
+
profile: cellProfile,
|
|
3207
|
+
taskToPrompt: (s) => `${promptOf(s)}\n\n<!-- independent-attempt:${shotNonce++} -->`
|
|
3208
|
+
},
|
|
3209
|
+
output: { parse: (events) => spec.parseOutput ? spec.parseOutput(events, cellScenario.case) : collectAgentResponseText(events) ?? "" },
|
|
3210
|
+
validator: { validate: async (output) => {
|
|
3211
|
+
const s = normalizeScore(spec.score(output, cellScenario.case));
|
|
3212
|
+
return {
|
|
3213
|
+
valid: s.composite > 0,
|
|
3214
|
+
score: s.composite
|
|
3215
|
+
};
|
|
3216
|
+
} },
|
|
3217
|
+
task: cellScenario,
|
|
3218
|
+
maxIterations: shots
|
|
3219
|
+
};
|
|
3220
|
+
},
|
|
3221
|
+
toArtifact: (result) => {
|
|
3222
|
+
for (const iter of result.iterations) spec.onCellEvents?.(iter.events, scenario.case, {
|
|
3223
|
+
index: iter.index,
|
|
3224
|
+
...iter.error ? { error: iter.error.message } : {},
|
|
3225
|
+
...iter.verdict ? { verdict: { score: iter.verdict.score } } : {}
|
|
3226
|
+
});
|
|
3227
|
+
return result.winner?.output;
|
|
2986
3228
|
}
|
|
3229
|
+
})(profile, scenario, dispatchCtx);
|
|
3230
|
+
});
|
|
3231
|
+
await spec.setup?.(ctx);
|
|
3232
|
+
try {
|
|
3233
|
+
const result = await runProfileMatrix({
|
|
3234
|
+
profiles,
|
|
3235
|
+
scenarios,
|
|
3236
|
+
dispatch,
|
|
3237
|
+
judges: spec.judges ?? [scoreJudge],
|
|
3238
|
+
runDir,
|
|
3239
|
+
commitSha: gitSha(),
|
|
3240
|
+
reps,
|
|
3241
|
+
...spec.matrix
|
|
2987
3242
|
});
|
|
2988
|
-
if (
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3010
|
-
|
|
3243
|
+
if (spec.export) await spec.export(result, ctx);
|
|
3244
|
+
else {
|
|
3245
|
+
mkdirSync(exportDir, { recursive: true });
|
|
3246
|
+
writeFileSync(join(runDir, "matrix-result.json"), `${JSON.stringify(result, null, 2)}\n`);
|
|
3247
|
+
const table = renderLeaderboardMarkdown(leaderboard(result.records, {
|
|
3248
|
+
title: spec.name,
|
|
3249
|
+
meta: { backend: backendName }
|
|
3250
|
+
}));
|
|
3251
|
+
writeFileSync(join(exportDir, "leaderboard.md"), table);
|
|
3252
|
+
console.log(table);
|
|
3253
|
+
}
|
|
3254
|
+
return result;
|
|
3255
|
+
} finally {
|
|
3256
|
+
await spec.teardown?.(ctx);
|
|
3257
|
+
}
|
|
3258
|
+
}
|
|
3259
|
+
function toBenchmarkAdapter() {
|
|
3260
|
+
return {
|
|
3261
|
+
name: spec.name,
|
|
3262
|
+
async preflight() {
|
|
3263
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3264
|
+
for (const c of spec.cases) {
|
|
3265
|
+
const id = caseId(c);
|
|
3266
|
+
if (seen.has(id)) throw new Error(`defineLeaderboard(${spec.name}): duplicate case id "${id}"`);
|
|
3267
|
+
seen.add(id);
|
|
3268
|
+
}
|
|
3269
|
+
},
|
|
3270
|
+
async loadTasks(opts) {
|
|
3271
|
+
const selected = selectCases(opts?.ids);
|
|
3272
|
+
const limited = opts?.limit !== void 0 ? selected.slice(0, opts.limit) : selected;
|
|
3273
|
+
return Promise.all(limited.map(async (c) => ({
|
|
3274
|
+
id: caseId(c),
|
|
3275
|
+
prompt: await spec.prompt(c),
|
|
3276
|
+
metadata: { case: c }
|
|
3277
|
+
})));
|
|
3278
|
+
},
|
|
3279
|
+
async judge(task, artifact) {
|
|
3280
|
+
const [c] = selectCases([task.id]);
|
|
3281
|
+
if (c === void 0) throw new Error(`defineLeaderboard(${spec.name}): no case "${task.id}"`);
|
|
3282
|
+
const s = normalizeScore(spec.score(artifact, c));
|
|
3283
|
+
return {
|
|
3284
|
+
resolved: s.composite > 0,
|
|
3285
|
+
score: s.composite,
|
|
3286
|
+
detail: s.notes
|
|
3287
|
+
};
|
|
3288
|
+
},
|
|
3289
|
+
async goldArtifact() {}
|
|
3290
|
+
};
|
|
3291
|
+
}
|
|
3292
|
+
return {
|
|
3293
|
+
run,
|
|
3294
|
+
toBenchmarkAdapter
|
|
3295
|
+
};
|
|
3296
|
+
}
|
|
3297
|
+
//#endregion
|
|
3298
|
+
//#region src/runtime/harvest-corpus.ts
|
|
3299
|
+
/**
|
|
3300
|
+
* Analyze completed runs through the selected observer and retain findings in a corpus.
|
|
3301
|
+
*
|
|
3302
|
+
* Store-agnostic by design: the caller maps its trace store's rows (a
|
|
3303
|
+
* `ProductionTraceSink` ndjson, OTLP spans, RunRecords) to `ObserveInput` — task text,
|
|
3304
|
+
* final output, the event trace, terminal outcome.
|
|
3305
|
+
* The default observer reads execution evidence; an injected analysis owns its admitted sources.
|
|
3306
|
+
*
|
|
3307
|
+
* The nightly product job is then three lines:
|
|
3308
|
+
* const runs = mapSinkRowsToObserveInputs(await readSink(yesterday))
|
|
3309
|
+
* const report = await harvestCorpus({ runs, profile, executor, corpus })
|
|
3310
|
+
* log(report) // runsObserved / findings / learned / failures
|
|
3311
|
+
*
|
|
3312
|
+
* Consumers choose how retained findings inform later work and how to assess their value.
|
|
3313
|
+
*/
|
|
3314
|
+
/** The completed batch evidence remains available even when every analysis failed. */
|
|
3315
|
+
var HarvestError = class extends Error {
|
|
3316
|
+
report;
|
|
3317
|
+
constructor(report) {
|
|
3318
|
+
super(`harvestCorpus: every run failed analysis (${report.failures.length}) — first: ${report.failures[0]?.error}`);
|
|
3319
|
+
this.report = report;
|
|
3320
|
+
this.name = "HarvestError";
|
|
3321
|
+
}
|
|
3322
|
+
};
|
|
3323
|
+
/** Batch the selected observation implementation over completed runs and retain its findings. */
|
|
3324
|
+
async function harvestCorpus(opts) {
|
|
3325
|
+
const concurrency = opts.concurrency ?? 4;
|
|
3326
|
+
if (!Number.isSafeInteger(concurrency) || concurrency < 1 || opts.maxRuns !== void 0 && (!Number.isSafeInteger(opts.maxRuns) || opts.maxRuns < 0)) throw new TypeError("harvest limits require positive integer concurrency and nonnegative integer maxRuns");
|
|
3327
|
+
const report = {
|
|
3328
|
+
runsObserved: 0,
|
|
3329
|
+
findings: 0,
|
|
3330
|
+
learned: 0,
|
|
3331
|
+
failures: [],
|
|
3332
|
+
usage: {
|
|
3333
|
+
input: 0,
|
|
3334
|
+
output: 0,
|
|
3335
|
+
known: true
|
|
3336
|
+
}
|
|
3337
|
+
};
|
|
3338
|
+
const iterator = Symbol.asyncIterator in Object(opts.runs) ? opts.runs[Symbol.asyncIterator]() : (async function* () {
|
|
3339
|
+
yield* opts.runs;
|
|
3340
|
+
})();
|
|
3341
|
+
let consumed = 0;
|
|
3342
|
+
let done = false;
|
|
3343
|
+
const next = async () => {
|
|
3344
|
+
if (done || opts.signal?.aborted || opts.maxRuns !== void 0 && consumed >= opts.maxRuns) return null;
|
|
3345
|
+
const sequence = ++consumed;
|
|
3346
|
+
let r;
|
|
3347
|
+
try {
|
|
3348
|
+
r = await iterator.next();
|
|
3349
|
+
} catch (error) {
|
|
3350
|
+
done = true;
|
|
3351
|
+
report.failures.push({
|
|
3352
|
+
runId: `source-${sequence}`,
|
|
3353
|
+
error: `trace source: ${error instanceof Error ? error.message.slice(0, 300) : String(error)}`
|
|
3011
3354
|
});
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3355
|
+
return null;
|
|
3356
|
+
}
|
|
3357
|
+
if (r.done) {
|
|
3358
|
+
done = true;
|
|
3359
|
+
return null;
|
|
3360
|
+
}
|
|
3361
|
+
return {
|
|
3362
|
+
input: r.value,
|
|
3363
|
+
sequence
|
|
3364
|
+
};
|
|
3365
|
+
};
|
|
3366
|
+
const workers = Array.from({ length: concurrency }, async () => {
|
|
3367
|
+
for (let run = await next(); run !== null; run = await next()) {
|
|
3368
|
+
const { input, sequence } = run;
|
|
3369
|
+
if (opts.signal?.aborted) return;
|
|
3370
|
+
try {
|
|
3371
|
+
const obs = await observe(input, opts);
|
|
3372
|
+
report.runsObserved += 1;
|
|
3373
|
+
report.findings += obs.findings.length;
|
|
3374
|
+
report.learned += obs.learned.length;
|
|
3375
|
+
report.usage.input += obs.usage.input;
|
|
3376
|
+
report.usage.output += obs.usage.output;
|
|
3377
|
+
report.usage.known &&= obs.usage.known;
|
|
3378
|
+
} catch (e) {
|
|
3379
|
+
report.usage.known &&= e instanceof ObservationError && e.usage.known;
|
|
3380
|
+
if (e instanceof ObservationError) {
|
|
3381
|
+
report.usage.input += e.usage.input;
|
|
3382
|
+
report.usage.output += e.usage.output;
|
|
3017
3383
|
}
|
|
3018
|
-
|
|
3019
|
-
|
|
3384
|
+
report.failures.push({
|
|
3385
|
+
runId: input.runId ?? `run-${sequence}`,
|
|
3386
|
+
error: e instanceof Error ? e.message.slice(0, 300) : String(e)
|
|
3387
|
+
});
|
|
3388
|
+
}
|
|
3389
|
+
}
|
|
3020
3390
|
});
|
|
3391
|
+
await Promise.all(workers);
|
|
3392
|
+
if (report.runsObserved === 0 && report.failures.length > 0) throw new HarvestError(report);
|
|
3393
|
+
return report;
|
|
3021
3394
|
}
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
|
|
3039
|
-
|
|
3040
|
-
|
|
3041
|
-
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
|
|
3052
|
-
|
|
3053
|
-
|
|
3395
|
+
//#endregion
|
|
3396
|
+
//#region src/runtime/in-process-sandbox-client.ts
|
|
3397
|
+
/**
|
|
3398
|
+
* The ONE in-process pseudo-box: present a plain user callback as a
|
|
3399
|
+
* `SandboxClient` so an example or test can drive `runAgentRounds` / `openSandboxRun`
|
|
3400
|
+
* with zero credentials and zero network — without re-faking a box at each call
|
|
3401
|
+
* site via `as unknown as SandboxInstance`.
|
|
3402
|
+
*
|
|
3403
|
+
* It is the structural sibling of {@link inlineSandboxClient}: that one adapts an
|
|
3404
|
+
* `Executor` (router / bridge / BYO) into a box; this one adapts a single
|
|
3405
|
+
* `onPrompt(prompt, round)` callback into a box. Both exist because
|
|
3406
|
+
* `SandboxInstance` is a `declare class` with private fields — a plain object
|
|
3407
|
+
* literal can NEVER structurally satisfy it, so every offline seam was forced to
|
|
3408
|
+
* write `as unknown as SandboxInstance`. The one unavoidable cast now lives HERE,
|
|
3409
|
+
* documented and tested, instead of in every example.
|
|
3410
|
+
*
|
|
3411
|
+
* What the box implements is exactly the subset `runAgentRounds` + `openSandboxRun`
|
|
3412
|
+
* actually call on a box:
|
|
3413
|
+
* - `id` — stable per box, used for trace correlation.
|
|
3414
|
+
* - `streamPrompt(prompt, opts?)` — runs `onPrompt` and streams its events.
|
|
3415
|
+
* - `fs.read` / `fs.write` — over the optional real `workdir` (the deliverable
|
|
3416
|
+
* artifact + any seeded fixtures live there). Present only when a `workdir`
|
|
3417
|
+
* is given.
|
|
3418
|
+
* - `exec(command)` — runs the command in `workdir`. Present only when a
|
|
3419
|
+
* `workdir` is given.
|
|
3420
|
+
* - `delete()` — removes the per-box temp dir (when one was minted) and is a
|
|
3421
|
+
* no-op otherwise; `deleteBoxSafe` calls it at teardown.
|
|
3422
|
+
*
|
|
3423
|
+
* The `SandboxEvent` shape the callback returns — `{ type, data }` — is the
|
|
3424
|
+
* runtime metering protocol the kernel reads: emit a flat `llm_call`
|
|
3425
|
+
* (`{ tokensIn, tokensOut, costUsd }`) or a terminal `done`
|
|
3426
|
+
* (`{ tokenUsage, totalCostUsd }`) / `result` event and the kernel's cost ledger
|
|
3427
|
+
* picks it up. No cast is needed on the events (the union is structural); only
|
|
3428
|
+
* the box object is cast, once, inside this file.
|
|
3429
|
+
*/
|
|
3430
|
+
function isAsyncIterable(v) {
|
|
3431
|
+
return typeof v === "object" && v !== null && Symbol.asyncIterator in v;
|
|
3432
|
+
}
|
|
3433
|
+
/**
|
|
3434
|
+
* Adapt a single `onPrompt(prompt, ctx)` callback into a `SandboxClient` for
|
|
3435
|
+
* `runAgentRounds` / `openSandboxRun`. Returns a PROPERLY-TYPED `SandboxClient`: the
|
|
3436
|
+
* lone `SandboxInstance` cast (object literal → `declare class`) lives inside
|
|
3437
|
+
* this function, so call sites stay cast-free.
|
|
3438
|
+
*
|
|
3439
|
+
* There is no box, so a per-prompt `backend` override is refused rather than dropped. A per-turn
|
|
3440
|
+
* `model` IS delivered, because `onPrompt` receives the verbatim options and is free to honor it.
|
|
3441
|
+
* Other per-prompt options (`timeoutMs`, `context`) are accepted and ignored.
|
|
3442
|
+
*
|
|
3443
|
+
* @experimental
|
|
3444
|
+
*/
|
|
3445
|
+
function inProcessSandboxClient(options) {
|
|
3446
|
+
const { onPrompt, workdir: workdirPrefix, id: idOption } = options;
|
|
3447
|
+
let seq = 0;
|
|
3448
|
+
return { async create(_options) {
|
|
3449
|
+
const current = seq++;
|
|
3450
|
+
const id = typeof idOption === "function" ? idOption(current) : idOption ?? `in-process-${current}`;
|
|
3451
|
+
const boxWorkdir = workdirPrefix !== void 0 ? mkdtempSync(join(tmpdir(), `${workdirPrefix}-`)) : void 0;
|
|
3452
|
+
let round = 0;
|
|
3453
|
+
const fsMembers = boxWorkdir !== void 0 ? {
|
|
3454
|
+
fs: {
|
|
3455
|
+
async read(path) {
|
|
3456
|
+
return readFile(join(boxWorkdir, path), "utf8");
|
|
3457
|
+
},
|
|
3458
|
+
async write(path, content) {
|
|
3459
|
+
const abs = join(boxWorkdir, path);
|
|
3460
|
+
await mkdir(dirname(abs), { recursive: true });
|
|
3461
|
+
await writeFile(abs, content, "utf8");
|
|
3462
|
+
}
|
|
3463
|
+
},
|
|
3464
|
+
async exec(command) {
|
|
3465
|
+
const { exec: execCb } = await import("node:child_process");
|
|
3466
|
+
const { promisify } = await import("node:util");
|
|
3467
|
+
const execAsync = promisify(execCb);
|
|
3468
|
+
try {
|
|
3469
|
+
const { stdout, stderr } = await execAsync(command, {
|
|
3470
|
+
cwd: boxWorkdir,
|
|
3471
|
+
timeout: 3e4
|
|
3472
|
+
});
|
|
3473
|
+
return {
|
|
3474
|
+
exitCode: 0,
|
|
3475
|
+
stdout,
|
|
3476
|
+
stderr
|
|
3477
|
+
};
|
|
3478
|
+
} catch (err) {
|
|
3479
|
+
const e = err;
|
|
3480
|
+
return {
|
|
3481
|
+
exitCode: e.code ?? 1,
|
|
3482
|
+
stdout: e.stdout ?? "",
|
|
3483
|
+
stderr: e.stderr ?? e.message ?? ""
|
|
3484
|
+
};
|
|
3485
|
+
}
|
|
3486
|
+
}
|
|
3487
|
+
} : {};
|
|
3488
|
+
async function* drive(behavior, message, opts) {
|
|
3489
|
+
const prompt = typeof message === "string" ? message : JSON.stringify(message);
|
|
3490
|
+
const { signal: optSignal, ...rest } = opts ?? {};
|
|
3491
|
+
const signal = optSignal ?? new AbortController().signal;
|
|
3492
|
+
const ctx = {
|
|
3493
|
+
round,
|
|
3494
|
+
workdir: boxWorkdir,
|
|
3495
|
+
signal,
|
|
3496
|
+
...Object.keys(rest).length > 0 ? { options: rest } : {}
|
|
3497
|
+
};
|
|
3498
|
+
round += 1;
|
|
3499
|
+
const produced = await behavior(prompt, ctx);
|
|
3500
|
+
if (isAsyncIterable(produced)) for await (const ev of produced) yield ev;
|
|
3501
|
+
else for (const ev of produced) yield ev;
|
|
3502
|
+
}
|
|
3503
|
+
return {
|
|
3504
|
+
id,
|
|
3505
|
+
streamPrompt(message, opts) {
|
|
3506
|
+
assertBoxlessPromptOptions(opts, "inProcessSandboxClient", ["backend"]);
|
|
3507
|
+
return drive(onPrompt, message, opts);
|
|
3508
|
+
},
|
|
3509
|
+
...fsMembers,
|
|
3510
|
+
async delete() {
|
|
3511
|
+
if (boxWorkdir !== void 0) rmSync(boxWorkdir, {
|
|
3512
|
+
recursive: true,
|
|
3513
|
+
force: true
|
|
3514
|
+
});
|
|
3515
|
+
}
|
|
3516
|
+
};
|
|
3517
|
+
} };
|
|
3054
3518
|
}
|
|
3055
3519
|
//#endregion
|
|
3056
3520
|
//#region src/runtime/observation-registry.ts
|
|
@@ -7223,6 +7687,180 @@ function createSqlRunContext(db) {
|
|
|
7223
7687
|
};
|
|
7224
7688
|
}
|
|
7225
7689
|
//#endregion
|
|
7690
|
+
//#region src/runtime/supervise/sql-run-context.ts
|
|
7691
|
+
/**
|
|
7692
|
+
* A cross-machine run context on the existing autocommit SqlAdapter. Reuse the same database,
|
|
7693
|
+
* tablePrefix and runId on every host; no runDir or shared filesystem is required.
|
|
7694
|
+
*
|
|
7695
|
+
* Each ownership generation gets fresh store capabilities. The public context can inspect SQL
|
|
7696
|
+
* at any time, but cannot write without acquire(). runGraph/supervise acquire and release it.
|
|
7697
|
+
* Retained provider execution supplies external admission/result idempotency; SQL does not turn
|
|
7698
|
+
* an arbitrary unkeyed network effect into an exactly-once operation.
|
|
7699
|
+
* @experimental
|
|
7700
|
+
*/
|
|
7701
|
+
async function createFencedSqlRunContext(db, runId, options = {}) {
|
|
7702
|
+
const store = await openSqlRunStore(db, runId, options);
|
|
7703
|
+
const { executors } = createInMemoryRunContext({ withDriver: options.withDriver ?? true });
|
|
7704
|
+
const cold = async () => projection(await store.read());
|
|
7705
|
+
const unowned = async () => {
|
|
7706
|
+
throw new SqlRunOwnershipError(`SQL run '${runId}' requires an acquired lease before writing`);
|
|
7707
|
+
};
|
|
7708
|
+
const context = {
|
|
7709
|
+
durability: "sql",
|
|
7710
|
+
runId,
|
|
7711
|
+
namespace: store.namespace,
|
|
7712
|
+
resume: true,
|
|
7713
|
+
executors,
|
|
7714
|
+
journal: {
|
|
7715
|
+
loadTree: async (root) => (await cold()).journal.loadTree(root),
|
|
7716
|
+
beginTree: unowned,
|
|
7717
|
+
appendEvent: unowned,
|
|
7718
|
+
appendEvents: unowned
|
|
7719
|
+
},
|
|
7720
|
+
blobs: {
|
|
7721
|
+
get: async (ref) => (await cold()).blobs.get(ref),
|
|
7722
|
+
put: unowned
|
|
7723
|
+
},
|
|
7724
|
+
coordinationLog: {
|
|
7725
|
+
load: async (id, owner) => (await cold()).loadCoordination(id, owner),
|
|
7726
|
+
append: unowned
|
|
7727
|
+
},
|
|
7728
|
+
async acquire(signal) {
|
|
7729
|
+
const lease = await store.acquire(signal);
|
|
7730
|
+
try {
|
|
7731
|
+
const failureController = new AbortController();
|
|
7732
|
+
const leaseSignal = AbortSignal.any([lease.signal, failureController.signal]);
|
|
7733
|
+
const view = await projection(lease.records);
|
|
7734
|
+
let tail = Promise.resolve();
|
|
7735
|
+
let accepting = true;
|
|
7736
|
+
let failed;
|
|
7737
|
+
let hasFailed = false;
|
|
7738
|
+
const check = () => {
|
|
7739
|
+
leaseSignal.throwIfAborted();
|
|
7740
|
+
if (hasFailed) throw failed;
|
|
7741
|
+
};
|
|
7742
|
+
const read = async (operation) => {
|
|
7743
|
+
await tail;
|
|
7744
|
+
check();
|
|
7745
|
+
return operation();
|
|
7746
|
+
};
|
|
7747
|
+
const commit = (entry) => {
|
|
7748
|
+
if (!accepting) return unowned();
|
|
7749
|
+
const captured = detachedSnapshot(entry, "SQL run record");
|
|
7750
|
+
const write = tail.then(async () => {
|
|
7751
|
+
check();
|
|
7752
|
+
if (await view.apply(captured)) await lease.append(captured);
|
|
7753
|
+
});
|
|
7754
|
+
tail = write.catch((error) => {
|
|
7755
|
+
hasFailed = true;
|
|
7756
|
+
failed = error;
|
|
7757
|
+
failureController.abort(error);
|
|
7758
|
+
});
|
|
7759
|
+
return write;
|
|
7760
|
+
};
|
|
7761
|
+
const bound = {
|
|
7762
|
+
durability: "sql",
|
|
7763
|
+
runId,
|
|
7764
|
+
namespace: store.namespace,
|
|
7765
|
+
resume: true,
|
|
7766
|
+
executors,
|
|
7767
|
+
journal: {
|
|
7768
|
+
loadTree: (root) => read(() => view.journal.loadTree(root)),
|
|
7769
|
+
beginTree: (root, at) => commit({
|
|
7770
|
+
kind: "begin",
|
|
7771
|
+
root,
|
|
7772
|
+
at
|
|
7773
|
+
}),
|
|
7774
|
+
appendEvent: (root, event) => commit({
|
|
7775
|
+
kind: "events",
|
|
7776
|
+
root,
|
|
7777
|
+
events: [event]
|
|
7778
|
+
}),
|
|
7779
|
+
appendEvents: (root, events) => commit({
|
|
7780
|
+
kind: "events",
|
|
7781
|
+
root,
|
|
7782
|
+
events: [...events]
|
|
7783
|
+
})
|
|
7784
|
+
},
|
|
7785
|
+
blobs: {
|
|
7786
|
+
get: (ref) => read(() => view.blobs.get(ref)),
|
|
7787
|
+
put: (ref, value) => commit({
|
|
7788
|
+
kind: "blobs",
|
|
7789
|
+
ref,
|
|
7790
|
+
value
|
|
7791
|
+
})
|
|
7792
|
+
},
|
|
7793
|
+
coordinationLog: {
|
|
7794
|
+
load: (id, owner) => read(() => view.loadCoordination(id, owner)),
|
|
7795
|
+
append: (id, record, ownerId) => commit({
|
|
7796
|
+
kind: "coordination",
|
|
7797
|
+
runId: id,
|
|
7798
|
+
record,
|
|
7799
|
+
...ownerId === void 0 ? {} : { ownerId }
|
|
7800
|
+
})
|
|
7801
|
+
}
|
|
7802
|
+
};
|
|
7803
|
+
return {
|
|
7804
|
+
context: Object.freeze(bound),
|
|
7805
|
+
signal: leaseSignal,
|
|
7806
|
+
async release() {
|
|
7807
|
+
accepting = false;
|
|
7808
|
+
await tail;
|
|
7809
|
+
await lease.release();
|
|
7810
|
+
}
|
|
7811
|
+
};
|
|
7812
|
+
} catch (error) {
|
|
7813
|
+
await lease.release().catch(() => void 0);
|
|
7814
|
+
throw error;
|
|
7815
|
+
}
|
|
7816
|
+
}
|
|
7817
|
+
};
|
|
7818
|
+
return Object.freeze(context);
|
|
7819
|
+
}
|
|
7820
|
+
/** Reuse the existing journal's uniqueness/sequence rules and blob hash validation verbatim. */
|
|
7821
|
+
async function projection(entries) {
|
|
7822
|
+
const journal = new InMemorySpawnJournal();
|
|
7823
|
+
const blobs = new InMemoryResultBlobStore();
|
|
7824
|
+
const beginnings = /* @__PURE__ */ new Map();
|
|
7825
|
+
const coordination = [];
|
|
7826
|
+
const apply = async (entry) => {
|
|
7827
|
+
switch (entry.kind) {
|
|
7828
|
+
case "begin": {
|
|
7829
|
+
const prior = beginnings.get(entry.root);
|
|
7830
|
+
await journal.beginTree(entry.root, entry.at);
|
|
7831
|
+
beginnings.set(entry.root, entry.at);
|
|
7832
|
+
return prior === void 0;
|
|
7833
|
+
}
|
|
7834
|
+
case "events":
|
|
7835
|
+
for (const event of entry.events) await journal.appendEvent(entry.root, event);
|
|
7836
|
+
return entry.events.length > 0;
|
|
7837
|
+
case "blobs": {
|
|
7838
|
+
const existing = await blobs.get(entry.ref);
|
|
7839
|
+
await blobs.put(entry.ref, entry.value);
|
|
7840
|
+
return existing === void 0;
|
|
7841
|
+
}
|
|
7842
|
+
case "coordination":
|
|
7843
|
+
if (entry.record.event.type === "settled") return false;
|
|
7844
|
+
coordination.push(entry);
|
|
7845
|
+
return true;
|
|
7846
|
+
default: throw new Error("Unknown SQL run record kind");
|
|
7847
|
+
}
|
|
7848
|
+
};
|
|
7849
|
+
for (const entry of entries) await apply(entry);
|
|
7850
|
+
return {
|
|
7851
|
+
journal,
|
|
7852
|
+
blobs,
|
|
7853
|
+
apply,
|
|
7854
|
+
loadCoordination(id, ownerId) {
|
|
7855
|
+
return foldCoordinationRecords(coordination.map(({ runId, ownerId, record }) => ({
|
|
7856
|
+
runId,
|
|
7857
|
+
...record,
|
|
7858
|
+
...ownerId === void 0 ? {} : { ownerId }
|
|
7859
|
+
})), id, ownerId);
|
|
7860
|
+
}
|
|
7861
|
+
};
|
|
7862
|
+
}
|
|
7863
|
+
//#endregion
|
|
7226
7864
|
//#region src/runtime/supervise/trajectory-recorder.ts
|
|
7227
7865
|
/**
|
|
7228
7866
|
*
|
|
@@ -8506,6 +9144,6 @@ function tail(s) {
|
|
|
8506
9144
|
return s.slice(-400);
|
|
8507
9145
|
}
|
|
8508
9146
|
//#endregion
|
|
8509
|
-
export {
|
|
9147
|
+
export { equalKOnCost as $, SqlResultBlobStore as $t, unsafeInProcessRunner as A, loopDispatch as At, strategyAuthorContract as B, deterministicCompletion as Bt, withUntrackedArtifacts as C, HarvestError as Ct, patchDelivered as D, localSandboxClient as Dt, createSqlRunContext as E, resolveSandboxClient as Et, pickChampion as F, declaredCheckDigest as Ft, sharedBoxPlacement as G, renderLeaderboardHtml as Gt, DEFAULT_SHARED_BOX_RESOURCES as H, stopSentinel as Ht, runStrategyEvolution as I, declaredCheckJudge as It, SandboxRunAbortError as J, renderPairwiseMarkdown as Jt, sharedBoxRefusal as K, renderLeaderboardMarkdown as Kt, selectChampion as L, readDeclaredCheck as Lt, chatWorkerSeam as M, assertDeclaredCheck as Mt, createChatSessionStore as N, checkProgramDigest as Nt, runCoderChecks as O, inlineSandboxClient as Ot, discriminatingMeans as P, declaredCheckDeliverable as Pt, promotionGate as Q, openSqlRunStore as Qt, assertStrategyContract as R, runIsolatedCheck as Rt, copyUntrackedIntoClone as S, inProcessSandboxClient as St, createFencedSqlRunContext as T, defineLeaderboard as Tt, DEFAULT_SHARED_BOX_WORKERS as U, leaderboard as Ut, strategyAuthorSystemPrompt as V, sentinelCompletion as Vt, ROUTER_CLIENT_HEADER as W, pairwiseSignificance as Wt, printBenchmarkReport as X, defaultAuditorInstruction as Xt, openSandboxRun as Y, auditIntent as Yt, runBenchmark as Z, SqlRunOwnershipError as Zt, NOTE_MAX_CHARS as _, assertTraceDerivedFindings as _t, localShell as a, sanitizeMcpToolSchema as an, registerShape as at, composeWorkerEvidence as b, registryScopeAnalyst as bt, createVerifierEnvironment as c, resolveMcpServerLaunch as cn, renderCorpusToInstructions as ct, harvestSurfaceDiffs as d, createTangleSandboxExactProcessProvider as dn, loopUntil as dt, SqlSpawnJournal as en, trajectoryReport as et, analystsFromRegistry as f, panel as ft, EVIDENCE_MAX_CHARS as g, widen as gt, worktreeFanout as h, verify as ht, jjWorkspace as i, createMcpEnvironment as in, createShapeRegistry as it, chatTransportExecutor as j, superviseDispatch as jt, codeModeSupervisorTools as k, loopCampaignDispatch as kt, boxSurfaceReader as l, resolveSecretEnv as ln, fanout as lt, superviseSurface as m, selectValidWinner as mt, makeFinding$1 as n, connectStdioMcp as nn, runPersonified as nt, runInWorkspace as o, envKeyProvider as on, FileCorpus as ot, failuresAnalyst as p, pipeline as pt, sharedWorkerClientName as q, renderLeaderboardSvg as qt, gitWorkspace as r, materializeLocalMcp as rn, builtinShapes as rt, createWaterfallCollector as s, mcpSecretEnvMetadataKey as sn, InMemoryCorpus as st, computeFindingId$1 as t, McpSpawnFault as tn, definePersona as tt, fsSurfaceReader as u, secretEnvOfMcpServer as un, flatWidenGate as ut, VERIFY_TAIL_CHARS as v, buildSteerContext as vt, analyzeTrace as w, harvestCorpus as wt, settledWorkerOut as x, observationFromRegistry as xt, closingWorkerNote as y, createScopeAnalyst as yt, authorStrategy as z, completionAuthorizes as zt };
|
|
8510
9148
|
|
|
8511
|
-
//# sourceMappingURL=runtime-
|
|
9149
|
+
//# sourceMappingURL=runtime-CHxXrUeZ.js.map
|