@tangle-network/agent-runtime 0.71.1 → 0.73.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/dist/agent.d.ts +27 -17
- package/dist/agent.js +21 -9
- package/dist/agent.js.map +1 -1
- package/dist/analyst-loop.d.ts +1 -1
- package/dist/analyst-loop.js +3 -2
- package/dist/{chunk-4KGQHS7U.js → chunk-7ODB76J5.js} +2 -18
- package/dist/chunk-7ODB76J5.js.map +1 -0
- package/dist/chunk-HPYWEFVY.js +4039 -0
- package/dist/chunk-HPYWEFVY.js.map +1 -0
- package/dist/{chunk-P5OKDSLB.js → chunk-IODKUOBA.js} +5 -3
- package/dist/{chunk-P5OKDSLB.js.map → chunk-IODKUOBA.js.map} +1 -1
- package/dist/chunk-NBV35BR6.js +68 -0
- package/dist/chunk-NBV35BR6.js.map +1 -0
- package/dist/{chunk-5ISW5JUF.js → chunk-NCH4XUZ7.js} +2004 -6551
- package/dist/chunk-NCH4XUZ7.js.map +1 -0
- package/dist/chunk-PBE35ULD.js +52 -0
- package/dist/chunk-PBE35ULD.js.map +1 -0
- package/dist/{chunk-VLF5RHEQ.js → chunk-T2HVQVB4.js} +1 -66
- package/dist/chunk-T2HVQVB4.js.map +1 -0
- package/dist/{chunk-K3RM4MPM.js → chunk-U56XGKVY.js} +9 -24
- package/dist/chunk-U56XGKVY.js.map +1 -0
- package/dist/chunk-UZ5SODU7.js +29 -0
- package/dist/chunk-UZ5SODU7.js.map +1 -0
- package/dist/{delegates-CsXJPZDH.d.ts → coordination-DU0saWeg.d.ts} +875 -1099
- package/dist/index.d.ts +7 -8
- package/dist/index.js +10 -12
- package/dist/index.js.map +1 -1
- package/dist/intelligence.d.ts +1 -1
- package/dist/lifecycle.d.ts +281 -0
- package/dist/lifecycle.js +174 -0
- package/dist/lifecycle.js.map +1 -0
- package/dist/{loop-runner-bin-DLM_bVQO.d.ts → loop-runner-bin-eD3m0rHW.d.ts} +10 -27
- package/dist/loop-runner-bin.d.ts +5 -6
- package/dist/loop-runner-bin.js +6 -6
- package/dist/loops.d.ts +47 -10
- package/dist/loops.js +47 -42
- package/dist/mcp/bin.js +17 -329
- package/dist/mcp/bin.js.map +1 -1
- package/dist/mcp/index.d.ts +10 -190
- package/dist/mcp/index.js +949 -47
- package/dist/mcp/index.js.map +1 -1
- package/dist/{openai-tools-kdCS-T12.d.ts → openai-tools-CBurv8Cu.d.ts} +11 -13
- package/dist/otel-export-BKmNwiCb.d.ts +180 -0
- package/dist/profiles.d.ts +3 -3
- package/dist/profiles.js +1 -5
- package/dist/profiles.js.map +1 -1
- package/dist/{substrate-BoRXgvka.d.ts → substrate-rNj6TDc3.d.ts} +7 -23
- package/dist/{types-CdnEAE3U.d.ts → types-JufmXF2a.d.ts} +80 -1
- package/dist/worktree-DaxOvw-C.d.ts +702 -0
- package/dist/{worktree-fanout-CK2ypmEm.d.ts → worktree-fanout-DIffZohV.d.ts} +15 -5
- package/package.json +6 -1
- package/dist/chunk-4KGQHS7U.js.map +0 -1
- package/dist/chunk-5ISW5JUF.js.map +0 -1
- package/dist/chunk-74UAWZXE.js +0 -278
- package/dist/chunk-74UAWZXE.js.map +0 -1
- package/dist/chunk-DVQGYDN5.js +0 -59
- package/dist/chunk-DVQGYDN5.js.map +0 -1
- package/dist/chunk-INXDNX2W.js +0 -475
- package/dist/chunk-INXDNX2W.js.map +0 -1
- package/dist/chunk-K3RM4MPM.js.map +0 -1
- package/dist/chunk-VLF5RHEQ.js.map +0 -1
- package/dist/chunk-XRYEZPR6.js +0 -88
- package/dist/chunk-XRYEZPR6.js.map +0 -1
- package/dist/coordination-BPQmuwv8.d.ts +0 -666
- package/dist/delegation-profile-Bvfro2m1.d.ts +0 -99
|
@@ -0,0 +1,4039 @@
|
|
|
1
|
+
import {
|
|
2
|
+
InMemoryResultBlobStore,
|
|
3
|
+
InMemorySpawnJournal,
|
|
4
|
+
addTokenUsage,
|
|
5
|
+
createExecutorRegistry,
|
|
6
|
+
createSandboxLineage,
|
|
7
|
+
createSupervisor,
|
|
8
|
+
createWorktreeCliExecutor,
|
|
9
|
+
defaultSelectWinner,
|
|
10
|
+
gateOnDeliverable,
|
|
11
|
+
isAbortError,
|
|
12
|
+
notifyRuntimeHookEvent,
|
|
13
|
+
probeSandboxCapabilities,
|
|
14
|
+
randomSuffix,
|
|
15
|
+
routerToolLoop,
|
|
16
|
+
runLoop,
|
|
17
|
+
settledToIteration,
|
|
18
|
+
sleep,
|
|
19
|
+
stringifySafe,
|
|
20
|
+
withDriverExecutor,
|
|
21
|
+
zeroTokenUsage
|
|
22
|
+
} from "./chunk-NCH4XUZ7.js";
|
|
23
|
+
import {
|
|
24
|
+
AnalystError,
|
|
25
|
+
PlannerError,
|
|
26
|
+
ValidationError
|
|
27
|
+
} from "./chunk-NBV35BR6.js";
|
|
28
|
+
|
|
29
|
+
// src/runtime/index.ts
|
|
30
|
+
import { computeFindingId, makeFinding as makeFinding2 } from "@tangle-network/agent-eval";
|
|
31
|
+
|
|
32
|
+
// src/runtime/anytime.ts
|
|
33
|
+
var median = (xs) => {
|
|
34
|
+
if (xs.length === 0) return null;
|
|
35
|
+
const s = [...xs].sort((a, b) => a - b);
|
|
36
|
+
const mid = Math.floor(s.length / 2);
|
|
37
|
+
return s.length % 2 === 1 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
|
|
38
|
+
};
|
|
39
|
+
function anytimeReport(spans, opts) {
|
|
40
|
+
const targets = opts?.targets ?? [1];
|
|
41
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
42
|
+
for (const s of spans) {
|
|
43
|
+
if (!s.label.startsWith("shot:")) continue;
|
|
44
|
+
const list = byRun.get(s.runId) ?? [];
|
|
45
|
+
list.push(s);
|
|
46
|
+
byRun.set(s.runId, list);
|
|
47
|
+
}
|
|
48
|
+
const perTask = [];
|
|
49
|
+
for (const [runId, shots] of byRun) {
|
|
50
|
+
const m = runId.match(/^agentic:(.+):(.+)$/);
|
|
51
|
+
const strategy = m?.[1] ?? runId;
|
|
52
|
+
const taskId = m?.[2] ?? runId;
|
|
53
|
+
const ordered = [...shots].sort((a, b) => (a.endMs ?? a.startMs) - (b.endMs ?? b.startMs));
|
|
54
|
+
const t0 = Math.min(...ordered.map((s) => s.startMs));
|
|
55
|
+
const taskTargets = opts?.targetFor ? [opts.targetFor(taskId)] : targets;
|
|
56
|
+
let best = 0;
|
|
57
|
+
let cumUsd = 0;
|
|
58
|
+
const points = [];
|
|
59
|
+
const hits = {};
|
|
60
|
+
for (const t of taskTargets) hits[String(t)] = null;
|
|
61
|
+
for (const s of ordered) {
|
|
62
|
+
cumUsd += s.usd;
|
|
63
|
+
if (typeof s.score === "number" && s.score > best) best = s.score;
|
|
64
|
+
const elapsedMs = (s.endMs ?? s.startMs) - t0;
|
|
65
|
+
points.push({ elapsedMs, cumUsd, best });
|
|
66
|
+
for (const t of taskTargets) {
|
|
67
|
+
if (hits[String(t)] === null && best >= t) {
|
|
68
|
+
hits[String(t)] = { ms: elapsedMs, shots: points.length, usd: cumUsd };
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
perTask.push({ taskId, strategy, points, hits });
|
|
73
|
+
}
|
|
74
|
+
const byStrategy = /* @__PURE__ */ new Map();
|
|
75
|
+
for (const t of perTask) {
|
|
76
|
+
const list = byStrategy.get(t.strategy) ?? [];
|
|
77
|
+
list.push(t);
|
|
78
|
+
byStrategy.set(t.strategy, list);
|
|
79
|
+
}
|
|
80
|
+
const perStrategy = [];
|
|
81
|
+
for (const [strategy, tasks] of byStrategy) {
|
|
82
|
+
const totalMs = tasks.reduce((s, t) => s + (t.points[t.points.length - 1]?.elapsedMs ?? 0), 0);
|
|
83
|
+
const totalUsd = tasks.reduce((s, t) => s + (t.points[t.points.length - 1]?.cumUsd ?? 0), 0);
|
|
84
|
+
const maxShots = Math.max(0, ...tasks.map((t) => t.points.length));
|
|
85
|
+
const curveByShot = [];
|
|
86
|
+
for (let i = 0; i < maxShots; i += 1) {
|
|
87
|
+
const vals = tasks.map(
|
|
88
|
+
(t) => t.points[Math.min(i, t.points.length - 1)].best
|
|
89
|
+
);
|
|
90
|
+
curveByShot.push(vals.reduce((s, v) => s + v, 0) / vals.length);
|
|
91
|
+
}
|
|
92
|
+
const auc = curveByShot.length > 0 ? curveByShot.reduce((s, v) => s + v, 0) / curveByShot.length : 0;
|
|
93
|
+
const summaryTargets = opts?.targetFor ? [Number.NaN] : targets;
|
|
94
|
+
for (const t of summaryTargets) {
|
|
95
|
+
const key = (taskCurve) => opts?.targetFor ? Object.values(taskCurve.hits)[0] ?? null : taskCurve.hits[String(t)] ?? null;
|
|
96
|
+
const reached = tasks.filter((x) => key(x) !== null);
|
|
97
|
+
perStrategy.push({
|
|
98
|
+
strategy,
|
|
99
|
+
target: t,
|
|
100
|
+
tasks: tasks.length,
|
|
101
|
+
reachedTarget: reached.length,
|
|
102
|
+
medianTttMs: median(reached.map((x) => key(x).ms)),
|
|
103
|
+
medianShotsToTarget: median(reached.map((x) => key(x).shots)),
|
|
104
|
+
ertMs: reached.length > 0 ? totalMs / reached.length : null,
|
|
105
|
+
erUsd: reached.length > 0 ? totalUsd / reached.length : null,
|
|
106
|
+
curveByShot,
|
|
107
|
+
auc
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
perStrategy.sort((a, b) => a.strategy.localeCompare(b.strategy) || a.target - b.target);
|
|
112
|
+
return { targets, perTask, perStrategy };
|
|
113
|
+
}
|
|
114
|
+
function renderAnytimeTable(report) {
|
|
115
|
+
const lines = [
|
|
116
|
+
`anytime metrics \xB7 satisficing targets [${report.targets.join(", ")}] \xB7 ERT = \u03A3 all wall-time / #successes (COCO)`,
|
|
117
|
+
"strategy \u2265tgt reach med-TTT med-shots ERT(all-in) $/success AUC curve"
|
|
118
|
+
];
|
|
119
|
+
for (const s of report.perStrategy) {
|
|
120
|
+
const curve = s.curveByShot.map((v) => "\u2581\u2582\u2583\u2584\u2585\u2586\u2587\u2588"[Math.min(7, Math.floor(v * 8))]).join("");
|
|
121
|
+
const tgt = Number.isNaN(s.target) ? "task" : s.target.toFixed(2);
|
|
122
|
+
lines.push(
|
|
123
|
+
`${s.strategy.padEnd(19)} ${tgt.padStart(4)} ${String(s.reachedTarget).padStart(4)}/${String(s.tasks).padEnd(3)} ${s.medianTttMs === null ? " \u2014" : `${(s.medianTttMs / 1e3).toFixed(1).padStart(6)}s`} ${s.medianShotsToTarget === null ? " \u2014" : String(s.medianShotsToTarget).padStart(5)} ${s.ertMs === null ? " \u2014" : `${(s.ertMs / 1e3).toFixed(1).padStart(9)}s`} ${s.erUsd === null ? " \u2014" : `$${s.erUsd.toFixed(4)}`} ${s.auc.toFixed(2)} ${curve}`
|
|
124
|
+
);
|
|
125
|
+
}
|
|
126
|
+
return lines.join("\n");
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// src/runtime/audit-intent.ts
|
|
130
|
+
var defaultAuditorInstruction = "You audit whether an AI agent is on the RIGHT ROUTE \u2014 not whether it works hard, but whether its actions serve the stated intents. Infer the REVEALED intent from the action pattern (what the trajectory is actually optimizing). Compare against the declared task intent, the user intent when given, and the meta-intent when given. Flawless execution down the wrong route is DIVERGED. Busy-work that neither advances nor harms is DRIFTING. Judge only from the trajectory \u2014 be specific about which actions ground your verdict. Recommend abort only when continuing cannot serve the intent.";
|
|
131
|
+
function summarize(trace, maxLines) {
|
|
132
|
+
const lines = [];
|
|
133
|
+
for (const ev of trace) {
|
|
134
|
+
const e = ev;
|
|
135
|
+
const role = e.role;
|
|
136
|
+
if (role === "tool") lines.push(`RESULT ${String(e.content).slice(0, 200)}`);
|
|
137
|
+
else if (role === "assistant") {
|
|
138
|
+
const calls = e.tool_calls?.map((c) => `${c.function?.name}(${(c.function?.arguments ?? "").slice(0, 120)})`).join(", ");
|
|
139
|
+
lines.push(calls ? `CALL ${calls}` : `SAY ${String(e.content).slice(0, 160)}`);
|
|
140
|
+
} else if (role === "user") lines.push(`USER ${String(e.content).slice(0, 160)}`);
|
|
141
|
+
}
|
|
142
|
+
return lines.slice(-maxLines).join("\n");
|
|
143
|
+
}
|
|
144
|
+
var auditSchema = {
|
|
145
|
+
name: "intent_audit",
|
|
146
|
+
schema: {
|
|
147
|
+
type: "object",
|
|
148
|
+
additionalProperties: false,
|
|
149
|
+
required: ["revealedIntent", "verdict", "evidence", "recommendation", "confidence"],
|
|
150
|
+
properties: {
|
|
151
|
+
revealedIntent: { type: "string" },
|
|
152
|
+
verdict: { type: "string", enum: ["aligned", "drifting", "diverged"] },
|
|
153
|
+
evidence: { type: "string" },
|
|
154
|
+
recommendation: { type: "string", enum: ["continue", "steer", "abort"] },
|
|
155
|
+
steer: { type: "string" },
|
|
156
|
+
confidence: { type: "number" }
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
};
|
|
160
|
+
async function auditIntent(input, opts) {
|
|
161
|
+
const res = await opts.chat.chat(
|
|
162
|
+
{
|
|
163
|
+
...opts.model ? { model: opts.model } : {},
|
|
164
|
+
jsonSchema: auditSchema,
|
|
165
|
+
messages: [
|
|
166
|
+
{ role: "system", content: opts.auditorInstruction ?? defaultAuditorInstruction },
|
|
167
|
+
{
|
|
168
|
+
role: "user",
|
|
169
|
+
content: `DECLARED INTENT (the task):
|
|
170
|
+
${input.declaredIntent}
|
|
171
|
+
|
|
172
|
+
` + (input.userIntent ? `USER INTENT (the principal's actual goal):
|
|
173
|
+
${input.userIntent}
|
|
174
|
+
|
|
175
|
+
` : "") + (input.metaIntent ? `META-INTENT (what the whole run is for):
|
|
176
|
+
${input.metaIntent}
|
|
177
|
+
|
|
178
|
+
` : "") + `TRAJECTORY (in order):
|
|
179
|
+
${summarize(input.trace, opts.maxTraceLines ?? 80)}
|
|
180
|
+
|
|
181
|
+
Audit the route: revealed intent, verdict, evidence, one recommendation.`
|
|
182
|
+
}
|
|
183
|
+
]
|
|
184
|
+
},
|
|
185
|
+
{ ...opts.signal ? { signal: opts.signal } : {} }
|
|
186
|
+
);
|
|
187
|
+
let parsed;
|
|
188
|
+
try {
|
|
189
|
+
parsed = JSON.parse(res.content);
|
|
190
|
+
} catch {
|
|
191
|
+
throw new Error(`auditIntent: auditor returned non-JSON: ${res.content.slice(0, 200)}`);
|
|
192
|
+
}
|
|
193
|
+
if (!parsed.verdict || !parsed.recommendation) {
|
|
194
|
+
throw new Error(`auditIntent: missing verdict/recommendation: ${res.content.slice(0, 200)}`);
|
|
195
|
+
}
|
|
196
|
+
return {
|
|
197
|
+
revealedIntent: parsed.revealedIntent ?? "",
|
|
198
|
+
verdict: parsed.verdict,
|
|
199
|
+
evidence: parsed.evidence ?? "",
|
|
200
|
+
recommendation: parsed.recommendation,
|
|
201
|
+
...parsed.steer ? { steer: parsed.steer } : {},
|
|
202
|
+
confidence: typeof parsed.confidence === "number" ? parsed.confidence : 0.5
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// src/runtime/completion.ts
|
|
207
|
+
function completionAuthorizes(v, policy) {
|
|
208
|
+
if (!v?.done) return false;
|
|
209
|
+
if (v.determinism === "deterministic") return true;
|
|
210
|
+
return (v.confidence ?? 0) >= (policy?.minConfidence ?? 0.8);
|
|
211
|
+
}
|
|
212
|
+
function stopSentinel(seed) {
|
|
213
|
+
let h = 2166136261;
|
|
214
|
+
for (let i = 0; i < seed.length; i += 1) {
|
|
215
|
+
h ^= seed.charCodeAt(i);
|
|
216
|
+
h = Math.imul(h, 16777619);
|
|
217
|
+
}
|
|
218
|
+
return `<<<{{STOP:${(h >>> 0).toString(16).padStart(8, "0")}}}>>>`;
|
|
219
|
+
}
|
|
220
|
+
function sentinelCompletion(sentinel, opts) {
|
|
221
|
+
return {
|
|
222
|
+
assess({ history }) {
|
|
223
|
+
const last = history[history.length - 1];
|
|
224
|
+
const out = typeof last?.output === "string" ? last.output : "";
|
|
225
|
+
const done = out.includes(sentinel);
|
|
226
|
+
return {
|
|
227
|
+
done,
|
|
228
|
+
determinism: "probabilistic",
|
|
229
|
+
confidence: done ? opts?.confidence ?? 0.9 : 0,
|
|
230
|
+
reasons: done ? "agent emitted its assigned stop sentinel" : void 0,
|
|
231
|
+
evidence: done && last ? [{ kind: "artifact", uri: `attempt:${last.index}` }] : []
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
function deterministicCompletion(check) {
|
|
237
|
+
return {
|
|
238
|
+
assess({ history }) {
|
|
239
|
+
const last = history[history.length - 1];
|
|
240
|
+
if (last?.output === void 0) {
|
|
241
|
+
return { done: false, determinism: "deterministic", reasons: "no output yet" };
|
|
242
|
+
}
|
|
243
|
+
const r = check(last.output, history);
|
|
244
|
+
return {
|
|
245
|
+
done: r.passed,
|
|
246
|
+
determinism: "deterministic",
|
|
247
|
+
reasons: r.reasons,
|
|
248
|
+
evidence: [{ kind: "artifact", uri: `attempt:${last.index}` }]
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// src/runtime/observe.ts
|
|
255
|
+
import { makeFinding } from "@tangle-network/agent-eval";
|
|
256
|
+
var observerId = "observe/trace";
|
|
257
|
+
var defaultAnalystInstruction = "You are a third-person OBSERVER watching an AI agent work. You see its TRACE (what it did), not its grader. From the trace, name SPECIFIC, behavior-grounded findings: wasted/duplicated tool calls, thrash/retries, token/cost waste, missing verification, failure patterns. For each, a concrete recommended_action, and whether the AGENT (fix its skills/prompt/tools) or the OPERATOR (fix framing/decomposition/config) should act. Only claim what the trace shows. No findings if the run was clean.";
|
|
258
|
+
function summarizeTrace(trace, maxLines) {
|
|
259
|
+
const lines = [];
|
|
260
|
+
for (const ev of trace) {
|
|
261
|
+
const e = ev;
|
|
262
|
+
const t = (e.type ?? "").toLowerCase();
|
|
263
|
+
const d = e.data ?? {};
|
|
264
|
+
const part = d.part ?? {};
|
|
265
|
+
if (part.type === "tool")
|
|
266
|
+
lines.push(`tool:${part.tool}${part.state?.status ? `(${part.state.status})` : ""}`);
|
|
267
|
+
else if (t.includes("error"))
|
|
268
|
+
lines.push(`ERROR: ${String(d.message ?? d.detail ?? "").slice(0, 200)}`);
|
|
269
|
+
else if (t === "status" && typeof d.status === "string") lines.push(`status:${d.status}`);
|
|
270
|
+
else if (t.includes("tool")) lines.push(`tool-event:${t}`);
|
|
271
|
+
}
|
|
272
|
+
const out = [];
|
|
273
|
+
for (const ln of lines) {
|
|
274
|
+
const prev = out[out.length - 1];
|
|
275
|
+
const m = prev?.match(/^(.*?)(?: x(\d+))?$/);
|
|
276
|
+
if (m && m[1] === ln) out[out.length - 1] = `${ln} x${(Number(m[2]) || 1) + 1}`;
|
|
277
|
+
else out.push(ln);
|
|
278
|
+
}
|
|
279
|
+
return out.slice(0, maxLines).join("\n") || "(no tool/error events in trace)";
|
|
280
|
+
}
|
|
281
|
+
var findingsSchema = {
|
|
282
|
+
name: "observer_findings",
|
|
283
|
+
schema: {
|
|
284
|
+
type: "object",
|
|
285
|
+
additionalProperties: false,
|
|
286
|
+
properties: {
|
|
287
|
+
findings: {
|
|
288
|
+
type: "array",
|
|
289
|
+
items: {
|
|
290
|
+
type: "object",
|
|
291
|
+
additionalProperties: false,
|
|
292
|
+
properties: {
|
|
293
|
+
area: {
|
|
294
|
+
type: "string",
|
|
295
|
+
description: "tool-use | cost | verification | process | failure | latency"
|
|
296
|
+
},
|
|
297
|
+
severity: { type: "string", enum: ["critical", "high", "medium", "low", "info"] },
|
|
298
|
+
claim: {
|
|
299
|
+
type: "string",
|
|
300
|
+
description: "what you OBSERVED in the trace (a fact, with the evidence)"
|
|
301
|
+
},
|
|
302
|
+
recommended_action: {
|
|
303
|
+
type: "string",
|
|
304
|
+
description: "the concrete change for the agent or operator"
|
|
305
|
+
},
|
|
306
|
+
audience: {
|
|
307
|
+
type: "string",
|
|
308
|
+
enum: ["agent", "operator"],
|
|
309
|
+
description: "who should act on this"
|
|
310
|
+
},
|
|
311
|
+
confidence: { type: "number" }
|
|
312
|
+
},
|
|
313
|
+
required: ["area", "severity", "claim", "recommended_action", "audience", "confidence"]
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
},
|
|
317
|
+
required: ["findings"]
|
|
318
|
+
}
|
|
319
|
+
};
|
|
320
|
+
async function observe(input, opts) {
|
|
321
|
+
const traceSummary = summarizeTrace(input.trace, opts.maxTraceLines ?? 80);
|
|
322
|
+
const res = await opts.chat.chat(
|
|
323
|
+
{
|
|
324
|
+
...opts.model ? { model: opts.model } : {},
|
|
325
|
+
jsonSchema: findingsSchema,
|
|
326
|
+
messages: [
|
|
327
|
+
{
|
|
328
|
+
role: "system",
|
|
329
|
+
content: opts.analystInstruction ?? defaultAnalystInstruction
|
|
330
|
+
},
|
|
331
|
+
{
|
|
332
|
+
role: "user",
|
|
333
|
+
content: `TASK: ${input.task}
|
|
334
|
+
|
|
335
|
+
OUTCOME: ${input.outcome ?? "unknown"}
|
|
336
|
+
|
|
337
|
+
FINAL OUTPUT (truncated):
|
|
338
|
+
${input.output.slice(0, 1200)}
|
|
339
|
+
|
|
340
|
+
TRACE (in order; "xN" = repeated):
|
|
341
|
+
${traceSummary}`
|
|
342
|
+
}
|
|
343
|
+
]
|
|
344
|
+
},
|
|
345
|
+
{ ...opts.signal ? { signal: opts.signal } : {} }
|
|
346
|
+
);
|
|
347
|
+
const parsed = parseFindings(res.content);
|
|
348
|
+
const producedAt = input.runId ? `${input.runId}` : observerId;
|
|
349
|
+
const findings = parsed.map(
|
|
350
|
+
(f) => makeFinding({
|
|
351
|
+
analyst_id: observerId,
|
|
352
|
+
area: `${f.area}`,
|
|
353
|
+
severity: f.severity,
|
|
354
|
+
claim: f.claim,
|
|
355
|
+
recommended_action: f.recommended_action,
|
|
356
|
+
confidence: typeof f.confidence === "number" ? f.confidence : 0.5,
|
|
357
|
+
evidence_refs: [],
|
|
358
|
+
// The observer reads BEHAVIOR, never the judge verdict — firewall provenance.
|
|
359
|
+
derived_from_judge: false,
|
|
360
|
+
metadata: { audience: f.audience },
|
|
361
|
+
...input.runId ? { subject: input.runId } : {}
|
|
362
|
+
})
|
|
363
|
+
);
|
|
364
|
+
const learned = [];
|
|
365
|
+
if (opts.corpus) {
|
|
366
|
+
for (const f of findings) {
|
|
367
|
+
const record = {
|
|
368
|
+
schemaVersion: "1.0.0",
|
|
369
|
+
id: f.finding_id,
|
|
370
|
+
runId: input.runId ?? observerId,
|
|
371
|
+
producedAt: f.produced_at ?? producedAt,
|
|
372
|
+
area: f.area,
|
|
373
|
+
claim: f.recommended_action ?? f.claim,
|
|
374
|
+
...f.claim ? { rationale: f.claim } : {},
|
|
375
|
+
tags: [...opts.tags ?? [], `audience:${f.metadata?.audience ?? "agent"}`],
|
|
376
|
+
confidence: f.confidence,
|
|
377
|
+
evidence: [{ kind: "finding", uri: f.finding_id }]
|
|
378
|
+
};
|
|
379
|
+
const r = await opts.corpus.append(record);
|
|
380
|
+
if (r.succeeded) learned.push(record);
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
return { findings, learned, report: renderReport(findings) };
|
|
384
|
+
}
|
|
385
|
+
function parseFindings(content) {
|
|
386
|
+
let obj2;
|
|
387
|
+
try {
|
|
388
|
+
obj2 = JSON.parse(content);
|
|
389
|
+
} catch {
|
|
390
|
+
const m = content.match(/\{[\s\S]*\}/);
|
|
391
|
+
obj2 = m ? JSON.parse(m[0]) : { findings: [] };
|
|
392
|
+
}
|
|
393
|
+
const arr = obj2.findings;
|
|
394
|
+
return Array.isArray(arr) ? arr : [];
|
|
395
|
+
}
|
|
396
|
+
function renderReport(findings) {
|
|
397
|
+
if (findings.length === 0) return "\u2713 clean run \u2014 the observer found nothing to change.";
|
|
398
|
+
const audience = (f) => f.metadata?.audience ?? "agent";
|
|
399
|
+
const forAgent = findings.filter((f) => audience(f) === "agent");
|
|
400
|
+
const forOperator = findings.filter((f) => audience(f) === "operator");
|
|
401
|
+
const block = (title, fs) => fs.length === 0 ? "" : `**${title}**
|
|
402
|
+
${fs.map((f) => `- [${f.severity}] ${f.claim}
|
|
403
|
+
\u2192 ${f.recommended_action ?? ""}`).join("\n")}
|
|
404
|
+
`;
|
|
405
|
+
return [
|
|
406
|
+
block("For the agent (fix skills / prompt / tools)", forAgent),
|
|
407
|
+
block("For you (the operator)", forOperator)
|
|
408
|
+
].filter(Boolean).join("\n");
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
// src/runtime/harvest-corpus.ts
|
|
412
|
+
async function harvestCorpus(opts) {
|
|
413
|
+
const concurrency = Math.max(1, opts.concurrency ?? 4);
|
|
414
|
+
const report = { runsObserved: 0, findings: 0, learned: 0, failures: [] };
|
|
415
|
+
const iterator = Symbol.asyncIterator in Object(opts.runs) ? opts.runs[Symbol.asyncIterator]() : (async function* () {
|
|
416
|
+
yield* opts.runs;
|
|
417
|
+
})();
|
|
418
|
+
let consumed = 0;
|
|
419
|
+
let done = false;
|
|
420
|
+
const next = async () => {
|
|
421
|
+
if (done || opts.maxRuns !== void 0 && consumed >= opts.maxRuns) return null;
|
|
422
|
+
const r = await iterator.next();
|
|
423
|
+
if (r.done) {
|
|
424
|
+
done = true;
|
|
425
|
+
return null;
|
|
426
|
+
}
|
|
427
|
+
consumed += 1;
|
|
428
|
+
return r.value;
|
|
429
|
+
};
|
|
430
|
+
const workers = Array.from({ length: concurrency }, async () => {
|
|
431
|
+
for (let input = await next(); input !== null; input = await next()) {
|
|
432
|
+
if (opts.signal?.aborted) return;
|
|
433
|
+
try {
|
|
434
|
+
const obs = await observe(input, {
|
|
435
|
+
chat: opts.chat,
|
|
436
|
+
...opts.model ? { model: opts.model } : {},
|
|
437
|
+
corpus: opts.corpus,
|
|
438
|
+
tags: opts.tags ?? [],
|
|
439
|
+
...opts.analystInstruction ? { analystInstruction: opts.analystInstruction } : {},
|
|
440
|
+
...opts.signal ? { signal: opts.signal } : {}
|
|
441
|
+
});
|
|
442
|
+
report.runsObserved += 1;
|
|
443
|
+
report.findings += obs.findings.length;
|
|
444
|
+
report.learned += obs.learned.length;
|
|
445
|
+
} catch (e) {
|
|
446
|
+
report.failures.push({
|
|
447
|
+
runId: input.runId ?? `run-${consumed}`,
|
|
448
|
+
error: e instanceof Error ? e.message.slice(0, 300) : String(e)
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
});
|
|
453
|
+
await Promise.all(workers);
|
|
454
|
+
if (report.runsObserved === 0 && report.failures.length > 0) {
|
|
455
|
+
throw new Error(
|
|
456
|
+
`harvestCorpus: every run failed analysis (${report.failures.length}) \u2014 first: ${report.failures[0]?.error}`
|
|
457
|
+
);
|
|
458
|
+
}
|
|
459
|
+
return report;
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
// src/runtime/inline-sandbox-client.ts
|
|
463
|
+
function isAsyncIterable(v) {
|
|
464
|
+
return typeof v === "object" && v !== null && Symbol.asyncIterator in v;
|
|
465
|
+
}
|
|
466
|
+
async function settle(exec, task, signal) {
|
|
467
|
+
const r = exec.execute(task, signal);
|
|
468
|
+
if (isAsyncIterable(r)) {
|
|
469
|
+
for await (const _ of r) {
|
|
470
|
+
}
|
|
471
|
+
return exec.resultArtifact();
|
|
472
|
+
}
|
|
473
|
+
return r;
|
|
474
|
+
}
|
|
475
|
+
function inlineSandboxClient(factory) {
|
|
476
|
+
let seq = 0;
|
|
477
|
+
return {
|
|
478
|
+
async create(_options) {
|
|
479
|
+
const id = `inline-${seq++}`;
|
|
480
|
+
return {
|
|
481
|
+
id,
|
|
482
|
+
async *streamPrompt(message) {
|
|
483
|
+
const controller = new AbortController();
|
|
484
|
+
const spec = { profile: { name: id }, harness: null };
|
|
485
|
+
const exec = factory(spec, { signal: controller.signal, seams: {} });
|
|
486
|
+
try {
|
|
487
|
+
const artifact = await settle(exec, message, controller.signal);
|
|
488
|
+
const out = artifact.out;
|
|
489
|
+
const tokensIn = artifact.spent.tokens.input;
|
|
490
|
+
const tokensOut = artifact.spent.tokens.output;
|
|
491
|
+
const costUsd = artifact.spent.usd;
|
|
492
|
+
if (tokensIn || tokensOut || costUsd) {
|
|
493
|
+
yield {
|
|
494
|
+
type: "llm_call",
|
|
495
|
+
data: { tokensIn, tokensOut, costUsd }
|
|
496
|
+
};
|
|
497
|
+
}
|
|
498
|
+
yield {
|
|
499
|
+
type: "result",
|
|
500
|
+
data: {
|
|
501
|
+
finalText: out?.content ?? "",
|
|
502
|
+
tokenUsage: {
|
|
503
|
+
inputTokens: tokensIn,
|
|
504
|
+
outputTokens: tokensOut
|
|
505
|
+
},
|
|
506
|
+
costUsd
|
|
507
|
+
}
|
|
508
|
+
};
|
|
509
|
+
} finally {
|
|
510
|
+
await exec.teardown("brutalKill").catch(() => {
|
|
511
|
+
});
|
|
512
|
+
}
|
|
513
|
+
},
|
|
514
|
+
async delete() {
|
|
515
|
+
}
|
|
516
|
+
};
|
|
517
|
+
}
|
|
518
|
+
};
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
// src/runtime/report-usage.ts
|
|
522
|
+
function reportLoopUsage(cost, result, source = "loop") {
|
|
523
|
+
cost.observe(result.costUsd, source);
|
|
524
|
+
cost.observeTokens(result.tokenUsage);
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
// src/runtime/loop-dispatch.ts
|
|
528
|
+
function campaignTraceToLoopEmitter(trace) {
|
|
529
|
+
return {
|
|
530
|
+
emit(event) {
|
|
531
|
+
trace.span(event.kind, { runId: event.runId, timestamp: event.timestamp, ...event.payload }).end();
|
|
532
|
+
}
|
|
533
|
+
};
|
|
534
|
+
}
|
|
535
|
+
async function runLoopForCell(opts, scenario, profile, ctx) {
|
|
536
|
+
const loopOptions = opts.toLoopOptions(scenario, profile);
|
|
537
|
+
const result = await runLoop({
|
|
538
|
+
...loopOptions,
|
|
539
|
+
ctx: {
|
|
540
|
+
sandboxClient: opts.sandboxClient,
|
|
541
|
+
signal: ctx.signal,
|
|
542
|
+
traceEmitter: opts.forwardTrace === false ? void 0 : campaignTraceToLoopEmitter(ctx.trace)
|
|
543
|
+
}
|
|
544
|
+
});
|
|
545
|
+
reportLoopUsage(ctx.cost, result, opts.costSource ?? "loop");
|
|
546
|
+
const toArtifact = opts.toArtifact ?? ((r) => r.winner?.output);
|
|
547
|
+
return toArtifact(result);
|
|
548
|
+
}
|
|
549
|
+
function loopDispatch(opts) {
|
|
550
|
+
return (profile, scenario, ctx) => runLoopForCell(opts, scenario, profile, ctx);
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
// src/runtime/mcp-environment.ts
|
|
554
|
+
async function rpc(endpoint, body) {
|
|
555
|
+
let lastErr;
|
|
556
|
+
for (let attempt = 0; attempt < 4; attempt += 1) {
|
|
557
|
+
try {
|
|
558
|
+
const r = await fetch(endpoint.url, {
|
|
559
|
+
method: "POST",
|
|
560
|
+
headers: { "content-type": "application/json", ...endpoint.headers ?? {} },
|
|
561
|
+
body: JSON.stringify(body)
|
|
562
|
+
});
|
|
563
|
+
const text = await r.text();
|
|
564
|
+
const dataLines = text.split("\n").filter((l) => l.startsWith("data:")).map((l) => l.slice(5).trim());
|
|
565
|
+
const payload = dataLines.length ? dataLines[dataLines.length - 1] : text;
|
|
566
|
+
try {
|
|
567
|
+
return { status: r.status, json: JSON.parse(payload ?? "null") };
|
|
568
|
+
} catch {
|
|
569
|
+
return { status: r.status, json: text };
|
|
570
|
+
}
|
|
571
|
+
} catch (err) {
|
|
572
|
+
lastErr = err;
|
|
573
|
+
await new Promise((res) => setTimeout(res, 1e3 * (attempt + 1)));
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
throw new Error(
|
|
577
|
+
`mcp rpc ${endpoint.url} failed after 4 attempts: ${lastErr instanceof Error ? lastErr.message : String(lastErr)}`
|
|
578
|
+
);
|
|
579
|
+
}
|
|
580
|
+
function sanitizeSchema(s) {
|
|
581
|
+
const o = s && typeof s === "object" ? s : {};
|
|
582
|
+
const banned = o.oneOf || o.anyOf || o.allOf || o.not || o.enum;
|
|
583
|
+
if (o.type === "object" && !banned && o.properties && typeof o.properties === "object") {
|
|
584
|
+
return {
|
|
585
|
+
type: "object",
|
|
586
|
+
properties: o.properties,
|
|
587
|
+
...Array.isArray(o.required) ? { required: o.required } : {}
|
|
588
|
+
};
|
|
589
|
+
}
|
|
590
|
+
return { type: "object", properties: {} };
|
|
591
|
+
}
|
|
592
|
+
function createMcpEnvironment(opts) {
|
|
593
|
+
const endpoints = /* @__PURE__ */ new Map();
|
|
594
|
+
const maxChars = opts.maxResultChars ?? 1500;
|
|
595
|
+
return {
|
|
596
|
+
name: opts.name,
|
|
597
|
+
async open(task) {
|
|
598
|
+
const { handle, endpoint } = await opts.open(task);
|
|
599
|
+
endpoints.set(handle.id, endpoint);
|
|
600
|
+
return handle;
|
|
601
|
+
},
|
|
602
|
+
async tools(task, handle) {
|
|
603
|
+
const endpoint = endpoints.get(handle.id);
|
|
604
|
+
if (!endpoint) throw new Error(`${opts.name}: tools() before open() for ${handle.id}`);
|
|
605
|
+
const { json } = await rpc(endpoint, {
|
|
606
|
+
jsonrpc: "2.0",
|
|
607
|
+
id: 1,
|
|
608
|
+
method: "tools/list",
|
|
609
|
+
params: {}
|
|
610
|
+
});
|
|
611
|
+
const all = (json.result?.tools ?? []).map(
|
|
612
|
+
(t) => ({
|
|
613
|
+
type: "function",
|
|
614
|
+
function: {
|
|
615
|
+
name: t.name,
|
|
616
|
+
description: (t.description ?? "").slice(0, 1e3),
|
|
617
|
+
parameters: sanitizeSchema(t.inputSchema)
|
|
618
|
+
}
|
|
619
|
+
})
|
|
620
|
+
);
|
|
621
|
+
return opts.selectTools ? opts.selectTools(task, all) : all;
|
|
622
|
+
},
|
|
623
|
+
async call(handle, name, args) {
|
|
624
|
+
const endpoint = endpoints.get(handle.id);
|
|
625
|
+
if (!endpoint) return "ERROR: workspace closed";
|
|
626
|
+
const { json } = await rpc(endpoint, {
|
|
627
|
+
jsonrpc: "2.0",
|
|
628
|
+
id: 2,
|
|
629
|
+
method: "tools/call",
|
|
630
|
+
params: { name, arguments: args }
|
|
631
|
+
});
|
|
632
|
+
const result = json ?? {};
|
|
633
|
+
if (result.error) return `ERROR: ${JSON.stringify(result.error).slice(0, 300)}`;
|
|
634
|
+
const text = result.result?.content?.map((c) => c.text ?? "").join("\n") ?? JSON.stringify(result.result ?? json);
|
|
635
|
+
return text.slice(0, maxChars);
|
|
636
|
+
},
|
|
637
|
+
score: (task, handle) => opts.score(task, handle),
|
|
638
|
+
async close(handle) {
|
|
639
|
+
endpoints.delete(handle.id);
|
|
640
|
+
await opts.close?.(handle);
|
|
641
|
+
}
|
|
642
|
+
};
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
// src/runtime/personify/analyst.ts
|
|
646
|
+
var judgeEvidenceUri = /^(verdict|judge|score)\b/i;
|
|
647
|
+
var assertTraceDerivedFindings = (findings) => {
|
|
648
|
+
for (const f of findings) {
|
|
649
|
+
for (const ref of f.evidence_refs ?? []) {
|
|
650
|
+
if (ref.kind === "metric" && judgeEvidenceUri.test(ref.uri)) {
|
|
651
|
+
throw new PlannerError(
|
|
652
|
+
`steer-firewall: finding ${stringifySafe(f.finding_id)} cites judge-derived evidence (${stringifySafe(ref.uri)}); findings fed to a combinator's steer decision must be trace-derived, not judge-derived (selector \u2260 judge)`
|
|
653
|
+
);
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
};
|
|
658
|
+
function createScopeAnalyst(scope, options) {
|
|
659
|
+
if (!options.analyst || typeof options.analyst.act !== "function") {
|
|
660
|
+
throw new AnalystError("createScopeAnalyst: analyst must be an Agent with an act() method");
|
|
661
|
+
}
|
|
662
|
+
const label = options.label ?? "analyst";
|
|
663
|
+
return {
|
|
664
|
+
async analyze(input) {
|
|
665
|
+
const task = options.buildTask(input);
|
|
666
|
+
const spawned = scope.spawn(options.analyst, task, {
|
|
667
|
+
budget: options.budget,
|
|
668
|
+
label
|
|
669
|
+
});
|
|
670
|
+
if (!spawned.ok) {
|
|
671
|
+
throw new AnalystError(
|
|
672
|
+
`createScopeAnalyst: analyst spawn refused by the conserved pool (${spawned.reason}); cannot steer node ${stringifySafe(input.nodeId)} on an unrun analyst`
|
|
673
|
+
);
|
|
674
|
+
}
|
|
675
|
+
const settled = await drainAnalystSettlement(scope, spawned.handle.id);
|
|
676
|
+
const findings = readAnalystFindings(settled);
|
|
677
|
+
assertTraceDerivedFindings(findings);
|
|
678
|
+
return findings;
|
|
679
|
+
}
|
|
680
|
+
};
|
|
681
|
+
}
|
|
682
|
+
async function drainAnalystSettlement(scope, analystId) {
|
|
683
|
+
for (; ; ) {
|
|
684
|
+
const settled = await scope.next();
|
|
685
|
+
if (settled === null) {
|
|
686
|
+
throw new AnalystError(
|
|
687
|
+
`createScopeAnalyst: scope drained before analyst ${stringifySafe(analystId)} settled`
|
|
688
|
+
);
|
|
689
|
+
}
|
|
690
|
+
if (settled.handle.id === analystId) return settled;
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
function readAnalystFindings(settled) {
|
|
694
|
+
if (settled.kind === "down") {
|
|
695
|
+
throw new AnalystError(
|
|
696
|
+
`createScopeAnalyst: analyst ${stringifySafe(settled.handle.id)} settled down (${settled.infra ? "infra" : "result"}): ${stringifySafe(settled.reason)}`
|
|
697
|
+
);
|
|
698
|
+
}
|
|
699
|
+
const out = settled.out;
|
|
700
|
+
if (!Array.isArray(out)) {
|
|
701
|
+
throw new PlannerError(
|
|
702
|
+
`createScopeAnalyst: analyst ${stringifySafe(settled.handle.id)} must return AnalystFinding[], got ${stringifySafe(out)}`
|
|
703
|
+
);
|
|
704
|
+
}
|
|
705
|
+
return out;
|
|
706
|
+
}
|
|
707
|
+
function registryScopeAnalyst(registry, buildInputs) {
|
|
708
|
+
return {
|
|
709
|
+
async analyze(input) {
|
|
710
|
+
const projection = buildInputs(input);
|
|
711
|
+
const result = await registry.run(projection.runId, projection.inputs, projection.opts);
|
|
712
|
+
const findings = result.findings;
|
|
713
|
+
assertTraceDerivedFindings(findings);
|
|
714
|
+
return findings;
|
|
715
|
+
}
|
|
716
|
+
};
|
|
717
|
+
}
|
|
718
|
+
function buildSteerContext(findings, settledSoFar) {
|
|
719
|
+
assertTraceDerivedFindings(findings);
|
|
720
|
+
const lastValidScore = observedBestScore(settledSoFar);
|
|
721
|
+
return {
|
|
722
|
+
findings,
|
|
723
|
+
settledSoFar,
|
|
724
|
+
...lastValidScore !== void 0 ? { lastValidScore } : {}
|
|
725
|
+
};
|
|
726
|
+
}
|
|
727
|
+
function observedBestScore(settledSoFar) {
|
|
728
|
+
let best;
|
|
729
|
+
for (const s of settledSoFar) {
|
|
730
|
+
if (s.kind !== "done") continue;
|
|
731
|
+
const v = s.verdict;
|
|
732
|
+
if (!v || v.valid !== true || typeof v.score !== "number") continue;
|
|
733
|
+
if (best === void 0 || v.score > best) best = v.score;
|
|
734
|
+
}
|
|
735
|
+
return best;
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
// src/runtime/personify/combinators.ts
|
|
739
|
+
function selectValidWinner(opts) {
|
|
740
|
+
const strategy = opts?.strategy ?? "highest-score";
|
|
741
|
+
const sizeOfIter = (iter) => {
|
|
742
|
+
const out = iter.output;
|
|
743
|
+
if (out === void 0 || opts?.sizeOf === void 0) return Number.POSITIVE_INFINITY;
|
|
744
|
+
const deliverable = out.kind === "done" ? out.deliverable : out;
|
|
745
|
+
return opts.sizeOf(deliverable);
|
|
746
|
+
};
|
|
747
|
+
return (iterations) => {
|
|
748
|
+
const valid = iterations.filter(
|
|
749
|
+
(iter) => iter.output !== void 0 && !iter.error && iter.verdict?.valid === true
|
|
750
|
+
);
|
|
751
|
+
if (valid.length === 0) return void 0;
|
|
752
|
+
switch (strategy) {
|
|
753
|
+
case "first-valid":
|
|
754
|
+
return [...valid].sort((a, b) => a.index - b.index)[0];
|
|
755
|
+
case "smallest-artifact":
|
|
756
|
+
return [...valid].sort((a, b) => sizeOfIter(a) - sizeOfIter(b) || a.index - b.index)[0];
|
|
757
|
+
default:
|
|
758
|
+
return [...valid].sort(
|
|
759
|
+
(a, b) => (b.verdict?.score ?? 0) - (a.verdict?.score ?? 0) || a.index - b.index
|
|
760
|
+
)[0];
|
|
761
|
+
}
|
|
762
|
+
};
|
|
763
|
+
}
|
|
764
|
+
function pipeline(stages) {
|
|
765
|
+
if (stages.length === 0) {
|
|
766
|
+
throw new ValidationError("pipeline: at least one stage is required");
|
|
767
|
+
}
|
|
768
|
+
return (ctx) => ({
|
|
769
|
+
name: `${ctx.persona.name}/pipeline`,
|
|
770
|
+
async act(task, scope) {
|
|
771
|
+
let carry = task;
|
|
772
|
+
for (const stage of stages) {
|
|
773
|
+
const child = ctx.spawnChild(stage.label, ctx.persona.root);
|
|
774
|
+
const res = scope.spawn(child, stage.feed(carry, ctx, task), {
|
|
775
|
+
budget: ctx.budget.perChild,
|
|
776
|
+
label: stage.label
|
|
777
|
+
});
|
|
778
|
+
if (!res.ok) {
|
|
779
|
+
return blocked([`${stage.label}: not admitted (${res.reason})`]);
|
|
780
|
+
}
|
|
781
|
+
const settled = await drainOne(scope, stage.label);
|
|
782
|
+
const out = stage.collect(settled);
|
|
783
|
+
if (out.kind === "blocked") return out;
|
|
784
|
+
carry = out.deliverable;
|
|
785
|
+
}
|
|
786
|
+
return { kind: "done", deliverable: carry };
|
|
787
|
+
}
|
|
788
|
+
});
|
|
789
|
+
}
|
|
790
|
+
function fanout(items, opts) {
|
|
791
|
+
if (opts.synthesize && opts.selectWinner) {
|
|
792
|
+
throw new ValidationError("fanout: pass at most one of `synthesize` or `selectWinner`");
|
|
793
|
+
}
|
|
794
|
+
return (ctx) => ({
|
|
795
|
+
name: `${ctx.persona.name}/fanout`,
|
|
796
|
+
async act(_task, scope) {
|
|
797
|
+
const rejected = [];
|
|
798
|
+
let opened = 0;
|
|
799
|
+
for (const [i, item] of items.entries()) {
|
|
800
|
+
const label = opts.label ? opts.label(item, i) : `item:${i}`;
|
|
801
|
+
const spec = opts.itemSpec ? opts.itemSpec(item, i, ctx) : ctx.persona.root;
|
|
802
|
+
const child = ctx.spawnChild(label, spec);
|
|
803
|
+
const res = scope.spawn(child, opts.itemTask(item, i, ctx), {
|
|
804
|
+
budget: ctx.budget.perChild,
|
|
805
|
+
label
|
|
806
|
+
});
|
|
807
|
+
if (res.ok) opened += 1;
|
|
808
|
+
else rejected.push(`${label}: not admitted (${res.reason})`);
|
|
809
|
+
}
|
|
810
|
+
if (opened === 0) {
|
|
811
|
+
return blocked(
|
|
812
|
+
rejected.length > 0 ? rejected : ["fanout: budget admitted no item (fanout fully rejected)"]
|
|
813
|
+
);
|
|
814
|
+
}
|
|
815
|
+
const drained = await drain(scope);
|
|
816
|
+
if (drained.done.length === 0) {
|
|
817
|
+
return blocked(
|
|
818
|
+
orderedBlockers(
|
|
819
|
+
drained.blockers,
|
|
820
|
+
rejected,
|
|
821
|
+
"fanout: every item settled without a usable result"
|
|
822
|
+
)
|
|
823
|
+
);
|
|
824
|
+
}
|
|
825
|
+
if (!opts.synthesize) {
|
|
826
|
+
const select = opts.selectWinner ?? defaultSelectWinner;
|
|
827
|
+
const winner = select(drained.iterations);
|
|
828
|
+
if (!winner || winner.output === void 0) {
|
|
829
|
+
return blocked(
|
|
830
|
+
orderedBlockers(drained.blockers, rejected, "fanout: no item survived selection")
|
|
831
|
+
);
|
|
832
|
+
}
|
|
833
|
+
return { kind: "done", deliverable: winner.output };
|
|
834
|
+
}
|
|
835
|
+
const synthLabel = "synthesize";
|
|
836
|
+
const synthChild = ctx.spawnChild(synthLabel, ctx.persona.root);
|
|
837
|
+
const synthRes = scope.spawn(synthChild, opts.synthesize.synthesisTask(drained.done, ctx), {
|
|
838
|
+
budget: ctx.budget.perChild,
|
|
839
|
+
label: synthLabel
|
|
840
|
+
});
|
|
841
|
+
if (!synthRes.ok) {
|
|
842
|
+
return blocked([...drained.blockers, `${synthLabel}: not admitted (${synthRes.reason})`]);
|
|
843
|
+
}
|
|
844
|
+
const synthSettled = await drainOne(scope, synthLabel);
|
|
845
|
+
const out = opts.synthesize.collect(synthSettled);
|
|
846
|
+
if (out.kind === "blocked") return out;
|
|
847
|
+
return out;
|
|
848
|
+
}
|
|
849
|
+
});
|
|
850
|
+
}
|
|
851
|
+
function loopUntil(seed, spec) {
|
|
852
|
+
return (ctx) => ({
|
|
853
|
+
name: `${ctx.persona.name}/loopUntil`,
|
|
854
|
+
async act(task, scope) {
|
|
855
|
+
let state = { round: 0, value: seed };
|
|
856
|
+
const blockers = [];
|
|
857
|
+
const settledSoFar = [];
|
|
858
|
+
for (; ; ) {
|
|
859
|
+
const label = spec.label ? spec.label(state.round) : `step:${state.round}`;
|
|
860
|
+
const child = ctx.spawnChild(label, ctx.persona.root);
|
|
861
|
+
const res = scope.spawn(child, spec.step(task, state, ctx), {
|
|
862
|
+
budget: ctx.budget.perChild,
|
|
863
|
+
label
|
|
864
|
+
});
|
|
865
|
+
if (!res.ok) {
|
|
866
|
+
if (state.round === 0) return blocked([`${label}: not admitted (${res.reason})`]);
|
|
867
|
+
break;
|
|
868
|
+
}
|
|
869
|
+
const settled = await drainOne(scope, label);
|
|
870
|
+
if (settled.kind === "down") blockers.push(blockerFromDown(settled));
|
|
871
|
+
settledSoFar.push(settled);
|
|
872
|
+
state = spec.fold(state, settled);
|
|
873
|
+
const findings = ctx.analyst ? await ctx.analyst.analyze({ task, settledSoFar, nodeId: scope.view.root }) : [];
|
|
874
|
+
const reached = spec.until(state, findings);
|
|
875
|
+
if (reached) return reached;
|
|
876
|
+
state = { round: state.round + 1, value: state.value };
|
|
877
|
+
}
|
|
878
|
+
return blocked(
|
|
879
|
+
blockers.length > 0 ? blockers : ["loopUntil: budget exhausted before the satisfiability gate was reached"]
|
|
880
|
+
);
|
|
881
|
+
}
|
|
882
|
+
});
|
|
883
|
+
}
|
|
884
|
+
function panel(spec) {
|
|
885
|
+
if (spec.judges.length === 0) {
|
|
886
|
+
throw new ValidationError("panel: at least one judge is required");
|
|
887
|
+
}
|
|
888
|
+
return (ctx) => ({
|
|
889
|
+
name: `${ctx.persona.name}/panel`,
|
|
890
|
+
async act(task, scope) {
|
|
891
|
+
const artifact = task;
|
|
892
|
+
const byLabel = /* @__PURE__ */ new Map();
|
|
893
|
+
const rejected = [];
|
|
894
|
+
let opened = 0;
|
|
895
|
+
for (const judge of spec.judges) {
|
|
896
|
+
const child = ctx.spawnChild(judge.label, ctx.persona.root);
|
|
897
|
+
const res = scope.spawn(child, spec.judgeTask(artifact, judge, ctx), {
|
|
898
|
+
budget: ctx.budget.perChild,
|
|
899
|
+
label: judge.label
|
|
900
|
+
});
|
|
901
|
+
if (res.ok) {
|
|
902
|
+
byLabel.set(judge.label, judge);
|
|
903
|
+
opened += 1;
|
|
904
|
+
} else {
|
|
905
|
+
rejected.push(`${judge.label}: not admitted (${res.reason})`);
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
if (opened === 0) {
|
|
909
|
+
return blocked(rejected.length > 0 ? rejected : ["panel: budget admitted no judge"]);
|
|
910
|
+
}
|
|
911
|
+
const verdicts = [];
|
|
912
|
+
for (let s = await scope.next(); s !== null; s = await scope.next()) {
|
|
913
|
+
const judge = byLabel.get(s.handle.label);
|
|
914
|
+
if (!judge) {
|
|
915
|
+
throw new ValidationError(
|
|
916
|
+
`panel: settled child "${s.handle.label}" has no judge descriptor`
|
|
917
|
+
);
|
|
918
|
+
}
|
|
919
|
+
if (s.kind === "down") {
|
|
920
|
+
verdicts.push({ judge, down: true });
|
|
921
|
+
continue;
|
|
922
|
+
}
|
|
923
|
+
verdicts.push({
|
|
924
|
+
judge,
|
|
925
|
+
down: false,
|
|
926
|
+
...s.verdict ? { verdict: s.verdict } : {},
|
|
927
|
+
output: s.out
|
|
928
|
+
});
|
|
929
|
+
}
|
|
930
|
+
return spec.merge(verdicts, artifact);
|
|
931
|
+
}
|
|
932
|
+
});
|
|
933
|
+
}
|
|
934
|
+
function verify(spec) {
|
|
935
|
+
return (ctx) => ({
|
|
936
|
+
name: `${ctx.persona.name}/verify`,
|
|
937
|
+
async act(task, scope) {
|
|
938
|
+
const implementLabel = spec.implementLabel ?? "implement";
|
|
939
|
+
const implChild = ctx.spawnChild(implementLabel, ctx.persona.root);
|
|
940
|
+
const implRes = scope.spawn(implChild, spec.implement(task, ctx), {
|
|
941
|
+
budget: ctx.budget.perChild,
|
|
942
|
+
label: implementLabel
|
|
943
|
+
});
|
|
944
|
+
if (!implRes.ok) return blocked([`${implementLabel}: not admitted (${implRes.reason})`]);
|
|
945
|
+
const candidate = await drainOne(scope, implementLabel);
|
|
946
|
+
if (candidate.kind === "down") return blocked([blockerFromDown(candidate)]);
|
|
947
|
+
const verifierLabel = spec.verifierLabel ?? "verify";
|
|
948
|
+
const verifierChild = ctx.spawnChild(verifierLabel, ctx.persona.root);
|
|
949
|
+
const verifierRes = scope.spawn(
|
|
950
|
+
verifierChild,
|
|
951
|
+
spec.verifier(candidate, ctx),
|
|
952
|
+
{ budget: ctx.budget.perChild, label: verifierLabel }
|
|
953
|
+
);
|
|
954
|
+
if (!verifierRes.ok)
|
|
955
|
+
return blocked([`${verifierLabel}: not admitted (${verifierRes.reason})`]);
|
|
956
|
+
const gate = await drainOne(scope, verifierLabel);
|
|
957
|
+
if (gate.kind === "down") return blocked([blockerFromDown(gate)]);
|
|
958
|
+
if (!gate.verdict || gate.verdict.valid !== true) {
|
|
959
|
+
return blocked([`${verifierLabel}: gate rejected the candidate (${verdictDetail(gate)})`]);
|
|
960
|
+
}
|
|
961
|
+
return spec.collect(candidate, gate.verdict);
|
|
962
|
+
}
|
|
963
|
+
});
|
|
964
|
+
}
|
|
965
|
+
function widen(spec) {
|
|
966
|
+
return (ctx) => ({
|
|
967
|
+
name: `${ctx.persona.name}/widen`,
|
|
968
|
+
async act(_task, scope) {
|
|
969
|
+
let opened = 0;
|
|
970
|
+
for (const [i, seed] of spec.seeds.entries()) {
|
|
971
|
+
const label = `seed:${i}`;
|
|
972
|
+
const child = ctx.spawnChild(label, ctx.persona.root);
|
|
973
|
+
const res = scope.spawn(child, spec.seedTask(seed, i, ctx), {
|
|
974
|
+
budget: ctx.budget.perChild,
|
|
975
|
+
label
|
|
976
|
+
});
|
|
977
|
+
if (res.ok) opened += 1;
|
|
978
|
+
}
|
|
979
|
+
if (opened === 0) {
|
|
980
|
+
return blocked(["widen: budget admitted no seed lineage"]);
|
|
981
|
+
}
|
|
982
|
+
const gathered = [];
|
|
983
|
+
let widenIndex = 0;
|
|
984
|
+
for (let s = await scope.next(); s !== null; s = await scope.next()) {
|
|
985
|
+
gathered.push(s);
|
|
986
|
+
const findings = ctx.analyst ? await ctx.analyst.analyze({
|
|
987
|
+
task: _task,
|
|
988
|
+
settledSoFar: gathered,
|
|
989
|
+
nodeId: scope.view.root
|
|
990
|
+
}) : [];
|
|
991
|
+
const decision = spec.gate.decide(s, findings, scope.budget);
|
|
992
|
+
if (decision.kind !== "widen") continue;
|
|
993
|
+
const label = `widen:${widenIndex}`;
|
|
994
|
+
widenIndex += 1;
|
|
995
|
+
const child = ctx.spawnChild(label, ctx.persona.root);
|
|
996
|
+
scope.spawn(child, spec.widenTask(decision.toward, ctx), {
|
|
997
|
+
budget: ctx.budget.perChild,
|
|
998
|
+
label
|
|
999
|
+
});
|
|
1000
|
+
}
|
|
1001
|
+
const done = gathered.filter(isDone);
|
|
1002
|
+
if (done.length === 0) {
|
|
1003
|
+
return blocked(
|
|
1004
|
+
orderedBlockers(
|
|
1005
|
+
gathered.filter(isDown).map(blockerFromDown),
|
|
1006
|
+
[],
|
|
1007
|
+
"widen: every lineage settled without a usable result"
|
|
1008
|
+
)
|
|
1009
|
+
);
|
|
1010
|
+
}
|
|
1011
|
+
return spec.synthesize(done, ctx);
|
|
1012
|
+
}
|
|
1013
|
+
});
|
|
1014
|
+
}
|
|
1015
|
+
function flatWidenGate() {
|
|
1016
|
+
return {
|
|
1017
|
+
decide() {
|
|
1018
|
+
return { kind: "stop" };
|
|
1019
|
+
}
|
|
1020
|
+
};
|
|
1021
|
+
}
|
|
1022
|
+
async function drain(scope) {
|
|
1023
|
+
const iterations = [];
|
|
1024
|
+
const done = [];
|
|
1025
|
+
const blockers = [];
|
|
1026
|
+
for (let s = await scope.next(); s !== null; s = await scope.next()) {
|
|
1027
|
+
if (s.kind === "down") {
|
|
1028
|
+
blockers.push(blockerFromDown(s));
|
|
1029
|
+
continue;
|
|
1030
|
+
}
|
|
1031
|
+
iterations.push(settledToIteration(s));
|
|
1032
|
+
done.push(s);
|
|
1033
|
+
}
|
|
1034
|
+
return { iterations, done, blockers };
|
|
1035
|
+
}
|
|
1036
|
+
async function drainOne(scope, label) {
|
|
1037
|
+
const s = await scope.next();
|
|
1038
|
+
if (s === null) {
|
|
1039
|
+
throw new ValidationError(`combinator: child "${label}" was spawned but never settled`);
|
|
1040
|
+
}
|
|
1041
|
+
return s;
|
|
1042
|
+
}
|
|
1043
|
+
function isDone(s) {
|
|
1044
|
+
return s.kind === "done";
|
|
1045
|
+
}
|
|
1046
|
+
function isDown(s) {
|
|
1047
|
+
return s.kind === "down";
|
|
1048
|
+
}
|
|
1049
|
+
function blockerFromDown(s) {
|
|
1050
|
+
return `${s.handle.label}: ${s.infra ? "infra" : "failed"} \u2014 ${s.reason}`;
|
|
1051
|
+
}
|
|
1052
|
+
function verdictDetail(s) {
|
|
1053
|
+
if (!s.verdict) return "no verdict";
|
|
1054
|
+
return s.verdict.notes ?? `score ${s.verdict.score}`;
|
|
1055
|
+
}
|
|
1056
|
+
function orderedBlockers(drained, rejected, fallback) {
|
|
1057
|
+
const all = [...drained, ...rejected];
|
|
1058
|
+
return all.length > 0 ? all : [fallback];
|
|
1059
|
+
}
|
|
1060
|
+
function blocked(blockers) {
|
|
1061
|
+
if (blockers.length === 0) {
|
|
1062
|
+
throw new ValidationError("combinator: a blocked outcome must name at least one blocker");
|
|
1063
|
+
}
|
|
1064
|
+
return { kind: "blocked", blockers };
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
// src/runtime/personify/corpus.ts
|
|
1068
|
+
var corpusSchemaVersion = "1.0.0";
|
|
1069
|
+
function assertCorpusRecord(value, where) {
|
|
1070
|
+
if (typeof value !== "object" || value === null) {
|
|
1071
|
+
throw new Error(`${where}: corpus record is not an object`);
|
|
1072
|
+
}
|
|
1073
|
+
const r = value;
|
|
1074
|
+
if (r.schemaVersion !== corpusSchemaVersion) {
|
|
1075
|
+
throw new Error(
|
|
1076
|
+
`${where}: corpus record schemaVersion '${String(r.schemaVersion)}' != '${corpusSchemaVersion}'`
|
|
1077
|
+
);
|
|
1078
|
+
}
|
|
1079
|
+
requireNonEmptyString(r.id, `${where}.id`);
|
|
1080
|
+
requireNonEmptyString(r.runId, `${where}.runId`);
|
|
1081
|
+
requireNonEmptyString(r.producedAt, `${where}.producedAt`);
|
|
1082
|
+
requireNonEmptyString(r.area, `${where}.area`);
|
|
1083
|
+
requireNonEmptyString(r.claim, `${where}.claim`);
|
|
1084
|
+
if (r.rationale !== void 0 && typeof r.rationale !== "string") {
|
|
1085
|
+
throw new Error(`${where}.rationale: expected string, got ${typeof r.rationale}`);
|
|
1086
|
+
}
|
|
1087
|
+
if (!Array.isArray(r.tags) || r.tags.some((t) => typeof t !== "string")) {
|
|
1088
|
+
throw new Error(`${where}.tags: expected string[]`);
|
|
1089
|
+
}
|
|
1090
|
+
if (typeof r.confidence !== "number" || !Number.isFinite(r.confidence)) {
|
|
1091
|
+
throw new Error(`${where}.confidence: expected a finite number, got ${String(r.confidence)}`);
|
|
1092
|
+
}
|
|
1093
|
+
if (r.confidence < 0 || r.confidence > 1) {
|
|
1094
|
+
throw new Error(`${where}.confidence: ${r.confidence} is outside the [0, 1] range`);
|
|
1095
|
+
}
|
|
1096
|
+
if (r.evidence !== void 0) {
|
|
1097
|
+
if (!Array.isArray(r.evidence)) {
|
|
1098
|
+
throw new Error(`${where}.evidence: expected an array`);
|
|
1099
|
+
}
|
|
1100
|
+
for (let i = 0; i < r.evidence.length; i++) {
|
|
1101
|
+
const ev = r.evidence[i];
|
|
1102
|
+
if (typeof ev !== "object" || ev === null) {
|
|
1103
|
+
throw new Error(`${where}.evidence[${i}]: not an object`);
|
|
1104
|
+
}
|
|
1105
|
+
requireNonEmptyString(ev.kind, `${where}.evidence[${i}].kind`);
|
|
1106
|
+
requireNonEmptyString(ev.uri, `${where}.evidence[${i}].uri`);
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
function requireNonEmptyString(value, where) {
|
|
1111
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
1112
|
+
throw new Error(`${where}: expected a non-empty string, got ${describe(value)}`);
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1115
|
+
function describe(value) {
|
|
1116
|
+
if (value === void 0) return "undefined";
|
|
1117
|
+
if (value === null) return "null";
|
|
1118
|
+
if (typeof value === "string") return `'${value}'`;
|
|
1119
|
+
return typeof value;
|
|
1120
|
+
}
|
|
1121
|
+
function recordsEqual(a, b) {
|
|
1122
|
+
return stableStringify(a) === stableStringify(b);
|
|
1123
|
+
}
|
|
1124
|
+
function stableStringify(value) {
|
|
1125
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null";
|
|
1126
|
+
if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
|
|
1127
|
+
const entries = Object.entries(value).filter(([, v]) => v !== void 0).sort(([x], [y]) => x < y ? -1 : x > y ? 1 : 0);
|
|
1128
|
+
return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`).join(",")}}`;
|
|
1129
|
+
}
|
|
1130
|
+
function applyFilter(records, filter) {
|
|
1131
|
+
const matched = records.filter((r) => {
|
|
1132
|
+
if (filter.area !== void 0 && r.area !== filter.area) return false;
|
|
1133
|
+
if (filter.runId !== void 0 && r.runId !== filter.runId) return false;
|
|
1134
|
+
if (filter.minConfidence !== void 0 && r.confidence < filter.minConfidence) return false;
|
|
1135
|
+
if (filter.tags !== void 0 && !filter.tags.every((t) => r.tags.includes(t))) return false;
|
|
1136
|
+
return true;
|
|
1137
|
+
});
|
|
1138
|
+
const ordered = [...matched].sort(
|
|
1139
|
+
(a, b) => b.confidence !== a.confidence ? b.confidence - a.confidence : b.producedAt < a.producedAt ? -1 : b.producedAt > a.producedAt ? 1 : 0
|
|
1140
|
+
);
|
|
1141
|
+
if (filter.limit !== void 0) {
|
|
1142
|
+
if (!Number.isInteger(filter.limit) || filter.limit < 0) {
|
|
1143
|
+
throw new Error(
|
|
1144
|
+
`corpus query: filter.limit must be a non-negative integer, got ${filter.limit}`
|
|
1145
|
+
);
|
|
1146
|
+
}
|
|
1147
|
+
return ordered.slice(0, filter.limit);
|
|
1148
|
+
}
|
|
1149
|
+
return ordered;
|
|
1150
|
+
}
|
|
1151
|
+
var InMemoryCorpus = class {
|
|
1152
|
+
byId = /* @__PURE__ */ new Map();
|
|
1153
|
+
async append(record) {
|
|
1154
|
+
try {
|
|
1155
|
+
assertCorpusRecord(record, "append: record");
|
|
1156
|
+
} catch (err) {
|
|
1157
|
+
return { succeeded: false, error: err instanceof Error ? err.message : String(err) };
|
|
1158
|
+
}
|
|
1159
|
+
const existing = this.byId.get(record.id);
|
|
1160
|
+
if (existing) {
|
|
1161
|
+
if (recordsEqual(existing, record)) return { succeeded: true };
|
|
1162
|
+
return {
|
|
1163
|
+
succeeded: false,
|
|
1164
|
+
error: `corpus conflict: id '${record.id}' is already stored with a different record; a learned fact is append-only \u2014 re-mint the id or reconcile before re-appending`
|
|
1165
|
+
};
|
|
1166
|
+
}
|
|
1167
|
+
this.byId.set(record.id, freeze(record));
|
|
1168
|
+
return { succeeded: true };
|
|
1169
|
+
}
|
|
1170
|
+
async query(filter) {
|
|
1171
|
+
return applyFilter([...this.byId.values()], filter);
|
|
1172
|
+
}
|
|
1173
|
+
};
|
|
1174
|
+
var FileCorpus = class {
|
|
1175
|
+
constructor(path) {
|
|
1176
|
+
this.path = path;
|
|
1177
|
+
}
|
|
1178
|
+
path;
|
|
1179
|
+
async append(record) {
|
|
1180
|
+
try {
|
|
1181
|
+
assertCorpusRecord(record, "append: record");
|
|
1182
|
+
} catch (err) {
|
|
1183
|
+
return { succeeded: false, error: err instanceof Error ? err.message : String(err) };
|
|
1184
|
+
}
|
|
1185
|
+
let stored;
|
|
1186
|
+
try {
|
|
1187
|
+
stored = await this.load();
|
|
1188
|
+
} catch (err) {
|
|
1189
|
+
return { succeeded: false, error: err instanceof Error ? err.message : String(err) };
|
|
1190
|
+
}
|
|
1191
|
+
const existing = stored.get(record.id);
|
|
1192
|
+
if (existing) {
|
|
1193
|
+
if (recordsEqual(existing, record)) return { succeeded: true };
|
|
1194
|
+
return {
|
|
1195
|
+
succeeded: false,
|
|
1196
|
+
error: `corpus conflict: id '${record.id}' is already stored in ${this.path} with a different record; a learned fact is append-only \u2014 re-mint the id or reconcile before re-appending`
|
|
1197
|
+
};
|
|
1198
|
+
}
|
|
1199
|
+
await this.appendLine(record);
|
|
1200
|
+
return { succeeded: true };
|
|
1201
|
+
}
|
|
1202
|
+
async query(filter) {
|
|
1203
|
+
const stored = await this.load();
|
|
1204
|
+
return applyFilter([...stored.values()], filter);
|
|
1205
|
+
}
|
|
1206
|
+
async load() {
|
|
1207
|
+
const fs = await import("fs/promises");
|
|
1208
|
+
let text;
|
|
1209
|
+
try {
|
|
1210
|
+
text = await fs.readFile(this.path, "utf8");
|
|
1211
|
+
} catch (err) {
|
|
1212
|
+
if (isNoEntError(err)) return /* @__PURE__ */ new Map();
|
|
1213
|
+
throw err;
|
|
1214
|
+
}
|
|
1215
|
+
const lines = text.split("\n").filter((line) => line.length > 0);
|
|
1216
|
+
const byId = /* @__PURE__ */ new Map();
|
|
1217
|
+
for (let i = 0; i < lines.length; i++) {
|
|
1218
|
+
const line = lines[i];
|
|
1219
|
+
let parsed;
|
|
1220
|
+
try {
|
|
1221
|
+
parsed = JSON.parse(line);
|
|
1222
|
+
} catch (err) {
|
|
1223
|
+
throw new Error(
|
|
1224
|
+
`corpus corrupted: ${this.path} line ${i + 1} is not valid JSON: ` + (err instanceof Error ? err.message : String(err))
|
|
1225
|
+
);
|
|
1226
|
+
}
|
|
1227
|
+
assertCorpusRecord(parsed, `corpus ${this.path} line ${i + 1}`);
|
|
1228
|
+
const existing = byId.get(parsed.id);
|
|
1229
|
+
if (existing && !recordsEqual(existing, parsed)) {
|
|
1230
|
+
throw new Error(
|
|
1231
|
+
`corpus corrupted: ${this.path} has two different records for id '${parsed.id}'; an append-only corpus must never hold a conflicting re-append under one id`
|
|
1232
|
+
);
|
|
1233
|
+
}
|
|
1234
|
+
byId.set(parsed.id, freeze(parsed));
|
|
1235
|
+
}
|
|
1236
|
+
return byId;
|
|
1237
|
+
}
|
|
1238
|
+
async appendLine(record) {
|
|
1239
|
+
const fs = await import("fs/promises");
|
|
1240
|
+
const path = await import("path");
|
|
1241
|
+
await fs.mkdir(path.dirname(this.path), { recursive: true });
|
|
1242
|
+
const fh = await fs.open(this.path, "a");
|
|
1243
|
+
try {
|
|
1244
|
+
await fh.write(`${JSON.stringify(record)}
|
|
1245
|
+
`);
|
|
1246
|
+
await fh.sync();
|
|
1247
|
+
} finally {
|
|
1248
|
+
await fh.close();
|
|
1249
|
+
}
|
|
1250
|
+
}
|
|
1251
|
+
};
|
|
1252
|
+
async function renderCorpusToInstructions(opts) {
|
|
1253
|
+
const { corpus, filter, profile, target = "prompt", maxLines } = opts;
|
|
1254
|
+
if (maxLines !== void 0 && (!Number.isInteger(maxLines) || maxLines < 0)) {
|
|
1255
|
+
throw new Error(
|
|
1256
|
+
`renderCorpusToInstructions: maxLines must be a non-negative integer, got ${maxLines}`
|
|
1257
|
+
);
|
|
1258
|
+
}
|
|
1259
|
+
const matched = await corpus.query(filter);
|
|
1260
|
+
const capped = maxLines !== void 0 ? matched.slice(0, maxLines) : matched;
|
|
1261
|
+
const lines = capped.map(renderLine);
|
|
1262
|
+
if (lines.length === 0) return structuredClone(profile);
|
|
1263
|
+
const next = structuredClone(profile);
|
|
1264
|
+
if (target === "resources") {
|
|
1265
|
+
const resources = next.resources ?? {};
|
|
1266
|
+
const prior = resources.instructions;
|
|
1267
|
+
if (prior !== void 0 && typeof prior !== "string") {
|
|
1268
|
+
throw new Error(
|
|
1269
|
+
"renderCorpusToInstructions: resources.instructions is an AgentProfileResourceRef, not a string; cannot string-append corpus facts without dropping the ref (pass target: 'prompt' or pre-resolve the ref)"
|
|
1270
|
+
);
|
|
1271
|
+
}
|
|
1272
|
+
const blob = [prior, ...lines].filter((s) => typeof s === "string" && s.length > 0);
|
|
1273
|
+
next.resources = { ...resources, instructions: blob.join("\n") };
|
|
1274
|
+
return next;
|
|
1275
|
+
}
|
|
1276
|
+
const prompt = next.prompt ?? {};
|
|
1277
|
+
next.prompt = { ...prompt, instructions: [...prompt.instructions ?? [], ...lines] };
|
|
1278
|
+
return next;
|
|
1279
|
+
}
|
|
1280
|
+
function renderLine(record) {
|
|
1281
|
+
return record.rationale ? `${record.claim} (${record.rationale})` : record.claim;
|
|
1282
|
+
}
|
|
1283
|
+
function freeze(record) {
|
|
1284
|
+
return Object.freeze({ ...record });
|
|
1285
|
+
}
|
|
1286
|
+
function isNoEntError(err) {
|
|
1287
|
+
return typeof err === "object" && err !== null && "code" in err && err.code === "ENOENT";
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
// src/runtime/personify/registry.ts
|
|
1291
|
+
function createShapeRegistry() {
|
|
1292
|
+
const shapes = /* @__PURE__ */ new Map();
|
|
1293
|
+
return {
|
|
1294
|
+
register(name, factory) {
|
|
1295
|
+
if (shapes.has(name)) {
|
|
1296
|
+
throw new ValidationError(`shape registry: shape "${name}" already registered`);
|
|
1297
|
+
}
|
|
1298
|
+
shapes.set(name, factory);
|
|
1299
|
+
},
|
|
1300
|
+
resolve(name) {
|
|
1301
|
+
const factory = shapes.get(name);
|
|
1302
|
+
if (!factory) {
|
|
1303
|
+
return { succeeded: false, error: `shape registry: unknown shape "${name}"` };
|
|
1304
|
+
}
|
|
1305
|
+
return { succeeded: true, value: factory };
|
|
1306
|
+
},
|
|
1307
|
+
names() {
|
|
1308
|
+
return [...shapes.keys()];
|
|
1309
|
+
}
|
|
1310
|
+
};
|
|
1311
|
+
}
|
|
1312
|
+
var builtinShapes = createShapeRegistry();
|
|
1313
|
+
function registerShape(name, factory) {
|
|
1314
|
+
builtinShapes.register(name, factory);
|
|
1315
|
+
}
|
|
1316
|
+
|
|
1317
|
+
// src/runtime/personify/persona.ts
|
|
1318
|
+
function definePersona(input) {
|
|
1319
|
+
if (!input.executors.registry && !input.executors.seams) {
|
|
1320
|
+
throw new ValidationError(
|
|
1321
|
+
`definePersona("${input.name}"): executors must supply a registry or a seams bag (built-in runtimes read their seams off ExecutorContext; neither was provided)`
|
|
1322
|
+
);
|
|
1323
|
+
}
|
|
1324
|
+
if (!input.root || typeof input.root !== "object" || !("harness" in input.root)) {
|
|
1325
|
+
throw new ValidationError(`definePersona("${input.name}"): root must be an AgentSpec`);
|
|
1326
|
+
}
|
|
1327
|
+
return Object.freeze({
|
|
1328
|
+
name: input.name,
|
|
1329
|
+
root: input.root,
|
|
1330
|
+
directive: input.directive,
|
|
1331
|
+
context: input.context,
|
|
1332
|
+
executors: input.executors,
|
|
1333
|
+
...input.extensions ? { extensions: input.extensions } : {}
|
|
1334
|
+
});
|
|
1335
|
+
}
|
|
1336
|
+
function createShapeContext(persona, budget, analyst) {
|
|
1337
|
+
return {
|
|
1338
|
+
persona,
|
|
1339
|
+
budget,
|
|
1340
|
+
...analyst ? { analyst } : {},
|
|
1341
|
+
spawnChild(name, spec) {
|
|
1342
|
+
const agent = {
|
|
1343
|
+
name,
|
|
1344
|
+
executorSpec: spec,
|
|
1345
|
+
act() {
|
|
1346
|
+
throw new ValidationError(
|
|
1347
|
+
`personify: spawned child "${name}" was run directly; its executorSpec drives it (a leaf, or \u2014 for a driver child \u2014 a nested scope through the recursive driver-executor)`
|
|
1348
|
+
);
|
|
1349
|
+
}
|
|
1350
|
+
};
|
|
1351
|
+
return agent;
|
|
1352
|
+
},
|
|
1353
|
+
childSpec(profile, harness) {
|
|
1354
|
+
return {
|
|
1355
|
+
profile,
|
|
1356
|
+
harness: harness === void 0 ? persona.root.harness : harness,
|
|
1357
|
+
...persona.root.executor ? { executor: persona.root.executor } : {}
|
|
1358
|
+
};
|
|
1359
|
+
}
|
|
1360
|
+
};
|
|
1361
|
+
}
|
|
1362
|
+
async function runPersonified(options) {
|
|
1363
|
+
const { persona } = options;
|
|
1364
|
+
const shape = resolveShape(options.shape);
|
|
1365
|
+
const shapeBudget = resolveShapeBudget(options.budget, options.shapeBudget);
|
|
1366
|
+
const ctx = createShapeContext(persona, shapeBudget, options.analyst);
|
|
1367
|
+
const rootAgent = shape(ctx);
|
|
1368
|
+
const executors = personaRegistry(persona);
|
|
1369
|
+
const supervisor = createSupervisor();
|
|
1370
|
+
if (options.handle) supervisor.attach(options.handle);
|
|
1371
|
+
const supervisorOpts = {
|
|
1372
|
+
budget: options.budget,
|
|
1373
|
+
runId: options.runId ?? `${persona.name}:${shapeName(options.shape, shape)}`,
|
|
1374
|
+
journal: options.journal ?? new InMemorySpawnJournal(),
|
|
1375
|
+
blobs: options.blobs ?? new InMemoryResultBlobStore(),
|
|
1376
|
+
executors,
|
|
1377
|
+
...options.maxDepth !== void 0 ? { maxDepth: options.maxDepth } : {},
|
|
1378
|
+
...options.maxRestarts !== void 0 ? { maxRestarts: options.maxRestarts } : {},
|
|
1379
|
+
...options.withinMs !== void 0 ? { withinMs: options.withinMs } : {},
|
|
1380
|
+
...options.now ? { now: options.now } : {},
|
|
1381
|
+
...options.signal ? { signal: options.signal } : {},
|
|
1382
|
+
...options.hooks ? { hooks: options.hooks } : {}
|
|
1383
|
+
};
|
|
1384
|
+
return supervisor.run(rootAgent, options.task, supervisorOpts);
|
|
1385
|
+
}
|
|
1386
|
+
function resolveShape(shape) {
|
|
1387
|
+
if (typeof shape !== "string") return shape;
|
|
1388
|
+
const resolved = builtinShapes.resolve(shape);
|
|
1389
|
+
if (!resolved.succeeded) {
|
|
1390
|
+
throw new ValidationError(
|
|
1391
|
+
`runPersonified: ${resolved.error} (registered: ${builtinShapes.names().join(", ")})`
|
|
1392
|
+
);
|
|
1393
|
+
}
|
|
1394
|
+
return resolved.value;
|
|
1395
|
+
}
|
|
1396
|
+
function shapeName(shape, _resolved) {
|
|
1397
|
+
return typeof shape === "string" ? shape : shape.name || "shape";
|
|
1398
|
+
}
|
|
1399
|
+
function resolveShapeBudget(root, over) {
|
|
1400
|
+
const fanout2 = over?.fanout ?? defaultFanout;
|
|
1401
|
+
const perChild2 = over?.perChild ?? {
|
|
1402
|
+
maxIterations: Math.max(1, Math.floor(root.maxIterations / fanout2)),
|
|
1403
|
+
maxTokens: Math.max(1, Math.floor(root.maxTokens / fanout2)),
|
|
1404
|
+
...root.maxUsd !== void 0 ? { maxUsd: root.maxUsd / fanout2 } : {},
|
|
1405
|
+
...root.deadlineMs !== void 0 ? { deadlineMs: root.deadlineMs } : {}
|
|
1406
|
+
};
|
|
1407
|
+
return { perChild: perChild2, fanout: fanout2 };
|
|
1408
|
+
}
|
|
1409
|
+
var defaultFanout = 3;
|
|
1410
|
+
function personaRegistry(persona) {
|
|
1411
|
+
const { registry, seams } = persona.executors;
|
|
1412
|
+
if (registry) return withDriverExecutor(registry);
|
|
1413
|
+
if (!seams) {
|
|
1414
|
+
throw new ValidationError(
|
|
1415
|
+
`personify: persona "${persona.name}" supplies neither a registry nor seams`
|
|
1416
|
+
);
|
|
1417
|
+
}
|
|
1418
|
+
return withDriverExecutor(withSeams(createExecutorRegistry(), seams));
|
|
1419
|
+
}
|
|
1420
|
+
function withSeams(base, seams) {
|
|
1421
|
+
return {
|
|
1422
|
+
register(runtime, factory) {
|
|
1423
|
+
base.register(runtime, factory);
|
|
1424
|
+
},
|
|
1425
|
+
resolve(spec) {
|
|
1426
|
+
const resolved = base.resolve(spec);
|
|
1427
|
+
if (!resolved.succeeded) return resolved;
|
|
1428
|
+
const inner = resolved.value;
|
|
1429
|
+
const wrapped = (s, ctx) => inner(s, mergeSeams(ctx, seams));
|
|
1430
|
+
return { succeeded: true, value: wrapped };
|
|
1431
|
+
}
|
|
1432
|
+
};
|
|
1433
|
+
}
|
|
1434
|
+
function mergeSeams(ctx, seams) {
|
|
1435
|
+
return { signal: ctx.signal, seams: { ...seams, ...ctx.seams } };
|
|
1436
|
+
}
|
|
1437
|
+
|
|
1438
|
+
// src/runtime/personify/trajectory.ts
|
|
1439
|
+
var defaultEqualKTolerance = 0.05;
|
|
1440
|
+
async function trajectoryReport(journal, blobs, root, options = {}) {
|
|
1441
|
+
const events = await journal.loadTree(root);
|
|
1442
|
+
if (events === void 0) {
|
|
1443
|
+
throw new Error(`trajectoryReport: no journaled tree for root '${root}'`);
|
|
1444
|
+
}
|
|
1445
|
+
const spawns = events.filter(isSpawned).sort(bySeq);
|
|
1446
|
+
const closes = events.filter((ev) => ev.kind !== "spawned" && ev.kind !== "metered").sort(bySeq);
|
|
1447
|
+
const nodes = /* @__PURE__ */ new Map();
|
|
1448
|
+
for (const ev of spawns) {
|
|
1449
|
+
nodes.set(ev.id, {
|
|
1450
|
+
id: ev.id,
|
|
1451
|
+
parent: ev.parent,
|
|
1452
|
+
label: ev.label,
|
|
1453
|
+
runtime: ev.runtime,
|
|
1454
|
+
status: "pending",
|
|
1455
|
+
ownSpend: zeroSpend(),
|
|
1456
|
+
children: []
|
|
1457
|
+
});
|
|
1458
|
+
}
|
|
1459
|
+
for (const ev of closes) {
|
|
1460
|
+
const node = requireNode(nodes, ev.id, root);
|
|
1461
|
+
if (ev.kind === "cancelled") {
|
|
1462
|
+
node.status = "cancelled";
|
|
1463
|
+
continue;
|
|
1464
|
+
}
|
|
1465
|
+
node.status = ev.status === "done" ? "done" : "failed";
|
|
1466
|
+
node.ownSpend = ev.spent;
|
|
1467
|
+
node.verdict = ev.verdict;
|
|
1468
|
+
node.outRef = ev.outRef;
|
|
1469
|
+
}
|
|
1470
|
+
for (const ev of events) {
|
|
1471
|
+
if (ev.kind !== "metered") continue;
|
|
1472
|
+
const node = requireNode(nodes, ev.id, root);
|
|
1473
|
+
node.ownSpend = addNodeSpend(node.ownSpend, ev.spend);
|
|
1474
|
+
}
|
|
1475
|
+
if (!nodes.has(root)) {
|
|
1476
|
+
throw new Error(
|
|
1477
|
+
`trajectoryReport: root '${root}' has no spawned event in its journaled tree (corrupted log)`
|
|
1478
|
+
);
|
|
1479
|
+
}
|
|
1480
|
+
for (const ev of spawns) {
|
|
1481
|
+
if (ev.parent === void 0) continue;
|
|
1482
|
+
requireNode(nodes, ev.parent, root).children.push(ev.id);
|
|
1483
|
+
}
|
|
1484
|
+
const rolledUp = rollUpSpend(nodes, root);
|
|
1485
|
+
if (options.withOutputs) {
|
|
1486
|
+
await attachOutputs(nodes, blobs);
|
|
1487
|
+
}
|
|
1488
|
+
const ordered = spawns.map((ev) => nodes.get(ev.id)).filter(isNode);
|
|
1489
|
+
const reported = ordered.map(
|
|
1490
|
+
(node) => freezeNode(node, requireSpend(rolledUp, node.id, root))
|
|
1491
|
+
);
|
|
1492
|
+
return {
|
|
1493
|
+
root,
|
|
1494
|
+
nodes: reported,
|
|
1495
|
+
total: requireSpend(rolledUp, root, root),
|
|
1496
|
+
statusCounts: countStatuses(reported)
|
|
1497
|
+
};
|
|
1498
|
+
}
|
|
1499
|
+
function equalKOnCost(arms, options = {}) {
|
|
1500
|
+
if (arms.length === 0) {
|
|
1501
|
+
throw new Error("equalKOnCost: no arms to compare");
|
|
1502
|
+
}
|
|
1503
|
+
const tolerance = options.tolerance ?? defaultEqualKTolerance;
|
|
1504
|
+
const armCosts = arms.map((arm) => ({
|
|
1505
|
+
label: arm.label,
|
|
1506
|
+
tokens: arm.report.total.tokens.input + arm.report.total.tokens.output,
|
|
1507
|
+
usd: arm.report.total.usd,
|
|
1508
|
+
iterations: arm.report.total.iterations
|
|
1509
|
+
}));
|
|
1510
|
+
const tokenValues = armCosts.map((a) => a.tokens);
|
|
1511
|
+
const usdValues = armCosts.map((a) => a.usd);
|
|
1512
|
+
const spread = {
|
|
1513
|
+
tokens: spreadOf(tokenValues),
|
|
1514
|
+
usd: spreadOf(usdValues)
|
|
1515
|
+
};
|
|
1516
|
+
const withinTolerance = fractionalSpread(tokenValues) <= tolerance && fractionalSpread(usdValues) <= tolerance;
|
|
1517
|
+
return {
|
|
1518
|
+
withinTolerance,
|
|
1519
|
+
arms: armCosts,
|
|
1520
|
+
spread,
|
|
1521
|
+
tolerance
|
|
1522
|
+
};
|
|
1523
|
+
}
|
|
1524
|
+
function rollUpSpend(nodes, root) {
|
|
1525
|
+
const rolled = /* @__PURE__ */ new Map();
|
|
1526
|
+
const stack = [{ id: root, expanded: false }];
|
|
1527
|
+
while (stack.length > 0) {
|
|
1528
|
+
const frame = stack.pop();
|
|
1529
|
+
if (frame === void 0) continue;
|
|
1530
|
+
const node = requireNode(nodes, frame.id, root);
|
|
1531
|
+
if (!frame.expanded) {
|
|
1532
|
+
stack.push({ id: frame.id, expanded: true });
|
|
1533
|
+
for (const child of node.children) stack.push({ id: child, expanded: false });
|
|
1534
|
+
continue;
|
|
1535
|
+
}
|
|
1536
|
+
const sum = cloneSpend(node.ownSpend);
|
|
1537
|
+
for (const child of node.children) addSpend(sum, requireSpend(rolled, child, root));
|
|
1538
|
+
rolled.set(frame.id, sum);
|
|
1539
|
+
}
|
|
1540
|
+
return rolled;
|
|
1541
|
+
}
|
|
1542
|
+
async function attachOutputs(nodes, blobs) {
|
|
1543
|
+
for (const node of nodes.values()) {
|
|
1544
|
+
if (node.status !== "done" || node.outRef === void 0) continue;
|
|
1545
|
+
const out = await blobs.get(node.outRef);
|
|
1546
|
+
if (out === void 0) {
|
|
1547
|
+
throw new Error(
|
|
1548
|
+
`trajectoryReport: blob store has no artifact for outRef '${node.outRef}' (node '${node.id}')`
|
|
1549
|
+
);
|
|
1550
|
+
}
|
|
1551
|
+
node.output = out;
|
|
1552
|
+
}
|
|
1553
|
+
}
|
|
1554
|
+
function freezeNode(node, rolledUpSpend) {
|
|
1555
|
+
return {
|
|
1556
|
+
id: node.id,
|
|
1557
|
+
parent: node.parent,
|
|
1558
|
+
children: [...node.children],
|
|
1559
|
+
label: node.label,
|
|
1560
|
+
runtime: node.runtime,
|
|
1561
|
+
status: node.status,
|
|
1562
|
+
ownSpend: node.ownSpend,
|
|
1563
|
+
rolledUpSpend,
|
|
1564
|
+
verdict: node.verdict,
|
|
1565
|
+
output: node.output,
|
|
1566
|
+
outRef: node.outRef
|
|
1567
|
+
};
|
|
1568
|
+
}
|
|
1569
|
+
function countStatuses(reported) {
|
|
1570
|
+
const counts = {
|
|
1571
|
+
done: 0,
|
|
1572
|
+
failed: 0,
|
|
1573
|
+
cancelled: 0,
|
|
1574
|
+
pending: 0
|
|
1575
|
+
};
|
|
1576
|
+
for (const node of reported) counts[node.status] += 1;
|
|
1577
|
+
return counts;
|
|
1578
|
+
}
|
|
1579
|
+
function zeroSpend() {
|
|
1580
|
+
return { iterations: 0, tokens: zeroTokenUsage(), usd: 0, ms: 0 };
|
|
1581
|
+
}
|
|
1582
|
+
function addNodeSpend(a, b) {
|
|
1583
|
+
return {
|
|
1584
|
+
iterations: a.iterations + b.iterations,
|
|
1585
|
+
tokens: { input: a.tokens.input + b.tokens.input, output: a.tokens.output + b.tokens.output },
|
|
1586
|
+
usd: a.usd + b.usd,
|
|
1587
|
+
ms: a.ms + b.ms
|
|
1588
|
+
};
|
|
1589
|
+
}
|
|
1590
|
+
function cloneSpend(spend) {
|
|
1591
|
+
return {
|
|
1592
|
+
iterations: spend.iterations,
|
|
1593
|
+
tokens: { input: spend.tokens.input, output: spend.tokens.output },
|
|
1594
|
+
usd: spend.usd,
|
|
1595
|
+
ms: spend.ms
|
|
1596
|
+
};
|
|
1597
|
+
}
|
|
1598
|
+
function addSpend(acc, delta) {
|
|
1599
|
+
acc.iterations += delta.iterations;
|
|
1600
|
+
addTokenUsage(acc.tokens, delta.tokens);
|
|
1601
|
+
acc.usd += delta.usd;
|
|
1602
|
+
acc.ms += delta.ms;
|
|
1603
|
+
}
|
|
1604
|
+
function spreadOf(values) {
|
|
1605
|
+
if (values.length === 0) return 0;
|
|
1606
|
+
return Math.max(...values) - Math.min(...values);
|
|
1607
|
+
}
|
|
1608
|
+
function fractionalSpread(values) {
|
|
1609
|
+
const spread = spreadOf(values);
|
|
1610
|
+
if (spread === 0) return 0;
|
|
1611
|
+
const median2 = medianOf(values);
|
|
1612
|
+
if (median2 === 0) {
|
|
1613
|
+
throw new Error(
|
|
1614
|
+
"equalKOnCost: arms have a non-zero cost spread on a zero-median channel; cannot express it as a fraction"
|
|
1615
|
+
);
|
|
1616
|
+
}
|
|
1617
|
+
return spread / median2;
|
|
1618
|
+
}
|
|
1619
|
+
function medianOf(values) {
|
|
1620
|
+
if (values.length === 0) {
|
|
1621
|
+
throw new Error("equalKOnCost: cannot take the median of an empty channel");
|
|
1622
|
+
}
|
|
1623
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
1624
|
+
const mid = Math.floor(sorted.length / 2);
|
|
1625
|
+
const hi = sorted[mid];
|
|
1626
|
+
if (sorted.length % 2 !== 0) return hi;
|
|
1627
|
+
const lo = sorted[mid - 1];
|
|
1628
|
+
return (lo + hi) / 2;
|
|
1629
|
+
}
|
|
1630
|
+
function isSpawned(ev) {
|
|
1631
|
+
return ev.kind === "spawned";
|
|
1632
|
+
}
|
|
1633
|
+
function isNode(node) {
|
|
1634
|
+
return node !== void 0;
|
|
1635
|
+
}
|
|
1636
|
+
function bySeq(a, b) {
|
|
1637
|
+
return a.seq - b.seq;
|
|
1638
|
+
}
|
|
1639
|
+
function requireNode(nodes, id, root) {
|
|
1640
|
+
const node = nodes.get(id);
|
|
1641
|
+
if (!node) {
|
|
1642
|
+
throw new Error(
|
|
1643
|
+
`trajectoryReport: tree '${root}' references node '${id}' with no prior spawn (corrupted log)`
|
|
1644
|
+
);
|
|
1645
|
+
}
|
|
1646
|
+
return node;
|
|
1647
|
+
}
|
|
1648
|
+
function requireSpend(rolled, id, root) {
|
|
1649
|
+
const spend = rolled.get(id);
|
|
1650
|
+
if (!spend) {
|
|
1651
|
+
throw new Error(
|
|
1652
|
+
`trajectoryReport: node '${id}' was never rolled up in tree '${root}' (unreachable from root)`
|
|
1653
|
+
);
|
|
1654
|
+
}
|
|
1655
|
+
return spend;
|
|
1656
|
+
}
|
|
1657
|
+
|
|
1658
|
+
// src/runtime/promotion-gate.ts
|
|
1659
|
+
import { heldoutSignificance } from "@tangle-network/agent-eval/campaign";
|
|
1660
|
+
function promotionGate(opts) {
|
|
1661
|
+
const mode = opts.mode ?? "superiority";
|
|
1662
|
+
if (opts.candidate === opts.incumbent) {
|
|
1663
|
+
return {
|
|
1664
|
+
promoted: false,
|
|
1665
|
+
reason: "identical-champion",
|
|
1666
|
+
mode,
|
|
1667
|
+
n: 0,
|
|
1668
|
+
lift: { mean: 0, median: 0, low: 0, high: 0 }
|
|
1669
|
+
};
|
|
1670
|
+
}
|
|
1671
|
+
const before = [];
|
|
1672
|
+
const after = [];
|
|
1673
|
+
const incUsd = [];
|
|
1674
|
+
const candUsd = [];
|
|
1675
|
+
const incMs = [];
|
|
1676
|
+
const candMs = [];
|
|
1677
|
+
const cellIds = [];
|
|
1678
|
+
for (const row of opts.report.perTask) {
|
|
1679
|
+
const inc = row.cells?.[opts.incumbent];
|
|
1680
|
+
const cand = row.cells?.[opts.candidate];
|
|
1681
|
+
if (!inc || !cand) continue;
|
|
1682
|
+
before.push(inc.score);
|
|
1683
|
+
after.push(cand.score);
|
|
1684
|
+
incUsd.push(inc.usd);
|
|
1685
|
+
candUsd.push(cand.usd);
|
|
1686
|
+
incMs.push(inc.ms);
|
|
1687
|
+
candMs.push(cand.ms);
|
|
1688
|
+
cellIds.push(row.taskId);
|
|
1689
|
+
}
|
|
1690
|
+
if (before.length === 0) {
|
|
1691
|
+
throw new Error(
|
|
1692
|
+
`promotionGate: no holdout task carried cells for both "${opts.incumbent}" and "${opts.candidate}" \u2014 the report must come from a run that included both strategies`
|
|
1693
|
+
);
|
|
1694
|
+
}
|
|
1695
|
+
const sig = heldoutSignificance(
|
|
1696
|
+
{ before, after, cellIds },
|
|
1697
|
+
{
|
|
1698
|
+
deltaThreshold: opts.deltaThreshold ?? 0,
|
|
1699
|
+
minProductiveRuns: opts.minPairedTasks ?? 6,
|
|
1700
|
+
statistic: opts.statistic ?? "mean",
|
|
1701
|
+
...opts.seed !== void 0 ? { seed: opts.seed } : {},
|
|
1702
|
+
...opts.resamples !== void 0 ? { resamples: opts.resamples } : {}
|
|
1703
|
+
}
|
|
1704
|
+
);
|
|
1705
|
+
const lift = {
|
|
1706
|
+
mean: sig.bootstrap.mean,
|
|
1707
|
+
median: sig.bootstrap.median,
|
|
1708
|
+
low: sig.bootstrap.low,
|
|
1709
|
+
high: sig.bootstrap.high
|
|
1710
|
+
};
|
|
1711
|
+
const latSig = heldoutSignificance(
|
|
1712
|
+
{ before: incMs, after: candMs, cellIds },
|
|
1713
|
+
{
|
|
1714
|
+
deltaThreshold: 0,
|
|
1715
|
+
minProductiveRuns: 1,
|
|
1716
|
+
statistic: opts.statistic ?? "mean",
|
|
1717
|
+
...opts.seed !== void 0 ? { seed: opts.seed } : {},
|
|
1718
|
+
...opts.resamples !== void 0 ? { resamples: opts.resamples } : {}
|
|
1719
|
+
}
|
|
1720
|
+
);
|
|
1721
|
+
const latency = {
|
|
1722
|
+
mean: latSig.bootstrap.mean,
|
|
1723
|
+
median: latSig.bootstrap.median,
|
|
1724
|
+
low: latSig.bootstrap.low,
|
|
1725
|
+
high: latSig.bootstrap.high
|
|
1726
|
+
};
|
|
1727
|
+
if (mode === "superiority") {
|
|
1728
|
+
if (sig.fewRuns) return { promoted: false, reason: "few-tasks", mode, n: sig.n, lift, latency };
|
|
1729
|
+
return sig.significant ? { promoted: true, reason: "significant", mode, n: sig.n, lift, latency } : { promoted: false, reason: "no-margin", mode, n: sig.n, lift, latency };
|
|
1730
|
+
}
|
|
1731
|
+
const tolerance = opts.scoreTolerance ?? 0.05;
|
|
1732
|
+
const scoreSig = heldoutSignificance(
|
|
1733
|
+
{ before, after, cellIds },
|
|
1734
|
+
{
|
|
1735
|
+
deltaThreshold: -tolerance,
|
|
1736
|
+
minProductiveRuns: opts.minPairedTasks ?? 6,
|
|
1737
|
+
statistic: opts.statistic ?? "mean",
|
|
1738
|
+
...opts.seed !== void 0 ? { seed: opts.seed } : {},
|
|
1739
|
+
...opts.resamples !== void 0 ? { resamples: opts.resamples } : {}
|
|
1740
|
+
}
|
|
1741
|
+
);
|
|
1742
|
+
const costSig = heldoutSignificance(
|
|
1743
|
+
{ before: candUsd, after: incUsd, cellIds },
|
|
1744
|
+
{
|
|
1745
|
+
deltaThreshold: 0,
|
|
1746
|
+
minProductiveRuns: opts.minPairedTasks ?? 6,
|
|
1747
|
+
statistic: opts.statistic ?? "mean",
|
|
1748
|
+
...opts.seed !== void 0 ? { seed: opts.seed } : {},
|
|
1749
|
+
...opts.resamples !== void 0 ? { resamples: opts.resamples } : {}
|
|
1750
|
+
}
|
|
1751
|
+
);
|
|
1752
|
+
const costSavings = {
|
|
1753
|
+
mean: costSig.bootstrap.mean,
|
|
1754
|
+
median: costSig.bootstrap.median,
|
|
1755
|
+
low: costSig.bootstrap.low,
|
|
1756
|
+
high: costSig.bootstrap.high
|
|
1757
|
+
};
|
|
1758
|
+
if (scoreSig.fewRuns)
|
|
1759
|
+
return { promoted: false, reason: "few-tasks", mode, n: scoreSig.n, lift, costSavings, latency };
|
|
1760
|
+
if (!scoreSig.significant)
|
|
1761
|
+
return {
|
|
1762
|
+
promoted: false,
|
|
1763
|
+
reason: "non-inferiority-unproven",
|
|
1764
|
+
mode,
|
|
1765
|
+
n: scoreSig.n,
|
|
1766
|
+
lift,
|
|
1767
|
+
costSavings,
|
|
1768
|
+
latency
|
|
1769
|
+
};
|
|
1770
|
+
if (!costSig.significant)
|
|
1771
|
+
return {
|
|
1772
|
+
promoted: false,
|
|
1773
|
+
reason: "not-cheaper",
|
|
1774
|
+
mode,
|
|
1775
|
+
n: scoreSig.n,
|
|
1776
|
+
lift,
|
|
1777
|
+
costSavings,
|
|
1778
|
+
latency
|
|
1779
|
+
};
|
|
1780
|
+
return {
|
|
1781
|
+
promoted: true,
|
|
1782
|
+
reason: "non-inferior-and-cheaper",
|
|
1783
|
+
mode,
|
|
1784
|
+
n: scoreSig.n,
|
|
1785
|
+
lift,
|
|
1786
|
+
costSavings,
|
|
1787
|
+
latency
|
|
1788
|
+
};
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1791
|
+
// src/runtime/run-benchmark.ts
|
|
1792
|
+
import { pairedBootstrap, paretoFrontier } from "@tangle-network/agent-eval";
|
|
1793
|
+
|
|
1794
|
+
// src/runtime/strategy.ts
|
|
1795
|
+
import { createChatClient, estimateCost, isModelPriced } from "@tangle-network/agent-eval";
|
|
1796
|
+
var taskNudge = "Use the available tools to bring the artifact to the required final state. Address EVERY distinct change the request implies. After each tool result, check what remains and continue. Re-read the values you set to confirm they took. Reply DONE only once every required change is made and verified.";
|
|
1797
|
+
async function runShot(surface, _task, handle, tools, messages, opts, modelOverride) {
|
|
1798
|
+
let toolErrors = 0;
|
|
1799
|
+
const execute = async (name, args) => {
|
|
1800
|
+
try {
|
|
1801
|
+
const out = await surface.call(handle, name, args);
|
|
1802
|
+
if (out.startsWith("ERROR:")) toolErrors += 1;
|
|
1803
|
+
return out;
|
|
1804
|
+
} catch (e) {
|
|
1805
|
+
toolErrors += 1;
|
|
1806
|
+
return `ERROR: ${e instanceof Error ? e.message : String(e)}`;
|
|
1807
|
+
}
|
|
1808
|
+
};
|
|
1809
|
+
const r = await routerToolLoop(
|
|
1810
|
+
{
|
|
1811
|
+
routerBaseUrl: opts.routerBaseUrl,
|
|
1812
|
+
routerKey: opts.routerKey,
|
|
1813
|
+
model: modelOverride ?? opts.model
|
|
1814
|
+
},
|
|
1815
|
+
"",
|
|
1816
|
+
"",
|
|
1817
|
+
tools,
|
|
1818
|
+
execute,
|
|
1819
|
+
{
|
|
1820
|
+
maxTurns: opts.innerTurns ?? 4,
|
|
1821
|
+
temperature: opts.temperature ?? 0.7,
|
|
1822
|
+
initialMessages: messages,
|
|
1823
|
+
...opts.maxTokens ? { maxTokens: opts.maxTokens } : {}
|
|
1824
|
+
}
|
|
1825
|
+
);
|
|
1826
|
+
return {
|
|
1827
|
+
messages: r.messages,
|
|
1828
|
+
completions: r.turns,
|
|
1829
|
+
toolCalls: r.toolCalls,
|
|
1830
|
+
toolErrors,
|
|
1831
|
+
tokens: r.usage
|
|
1832
|
+
};
|
|
1833
|
+
}
|
|
1834
|
+
function compactTrajectory(messages) {
|
|
1835
|
+
return messages.filter((m) => m.role === "assistant" || m.role === "tool").map((m) => {
|
|
1836
|
+
if (m.role === "tool") return `RESULT ${String(m.content).slice(0, 280)}`;
|
|
1837
|
+
const calls = m.tool_calls?.map((c) => `${c.function.name}(${c.function.arguments})`).join(", ");
|
|
1838
|
+
return calls ? `CALL ${calls}` : `SAY ${String(m.content).slice(0, 200)}`;
|
|
1839
|
+
}).join("\n").slice(0, 7e3);
|
|
1840
|
+
}
|
|
1841
|
+
async function consultAnalyst(task, messages, instruction, opts) {
|
|
1842
|
+
const trajectory = compactTrajectory(messages);
|
|
1843
|
+
const analystModel = opts.analystModel ?? opts.model;
|
|
1844
|
+
const chat = createChatClient({
|
|
1845
|
+
transport: "router",
|
|
1846
|
+
apiKey: opts.routerKey,
|
|
1847
|
+
baseUrl: opts.routerBaseUrl,
|
|
1848
|
+
defaultModel: analystModel
|
|
1849
|
+
});
|
|
1850
|
+
const res = await chat.chat({
|
|
1851
|
+
model: analystModel,
|
|
1852
|
+
temperature: 0.2,
|
|
1853
|
+
maxTokens: 1024,
|
|
1854
|
+
messages: [
|
|
1855
|
+
{ role: "system", content: instruction },
|
|
1856
|
+
{
|
|
1857
|
+
role: "user",
|
|
1858
|
+
content: `TASK: ${task.userPrompt.slice(0, 1500)}
|
|
1859
|
+
|
|
1860
|
+
TRAJECTORY:
|
|
1861
|
+
${trajectory}`
|
|
1862
|
+
}
|
|
1863
|
+
]
|
|
1864
|
+
});
|
|
1865
|
+
const usage = res.usage;
|
|
1866
|
+
return {
|
|
1867
|
+
steer: res.content.trim(),
|
|
1868
|
+
tokens: {
|
|
1869
|
+
input: usage?.promptTokens ?? usage?.prompt_tokens ?? 0,
|
|
1870
|
+
output: usage?.completionTokens ?? usage?.completion_tokens ?? 0
|
|
1871
|
+
}
|
|
1872
|
+
};
|
|
1873
|
+
}
|
|
1874
|
+
async function analyze(task, messages, opts) {
|
|
1875
|
+
const trajectory = compactTrajectory(messages);
|
|
1876
|
+
const analystModel = opts.analystModel ?? opts.model;
|
|
1877
|
+
const inner = createChatClient({
|
|
1878
|
+
transport: "router",
|
|
1879
|
+
apiKey: opts.routerKey,
|
|
1880
|
+
baseUrl: opts.routerBaseUrl,
|
|
1881
|
+
defaultModel: analystModel
|
|
1882
|
+
});
|
|
1883
|
+
const tokens = { input: 0, output: 0 };
|
|
1884
|
+
const chat = {
|
|
1885
|
+
...inner,
|
|
1886
|
+
chat: async (req, callOpts) => {
|
|
1887
|
+
const res = await inner.chat(req, callOpts);
|
|
1888
|
+
const u = res.usage;
|
|
1889
|
+
if (u) {
|
|
1890
|
+
tokens.input += u.promptTokens ?? u.prompt_tokens ?? 0;
|
|
1891
|
+
tokens.output += u.completionTokens ?? u.completion_tokens ?? 0;
|
|
1892
|
+
}
|
|
1893
|
+
return res;
|
|
1894
|
+
}
|
|
1895
|
+
};
|
|
1896
|
+
const obs = await observe(
|
|
1897
|
+
{
|
|
1898
|
+
task: task.userPrompt,
|
|
1899
|
+
output: trajectory,
|
|
1900
|
+
trace: messages,
|
|
1901
|
+
outcome: "failed",
|
|
1902
|
+
runId: task.id
|
|
1903
|
+
},
|
|
1904
|
+
{
|
|
1905
|
+
chat,
|
|
1906
|
+
model: analystModel,
|
|
1907
|
+
...opts.analystInstruction ? { analystInstruction: opts.analystInstruction } : {},
|
|
1908
|
+
...opts.corpus ? { corpus: opts.corpus, tags: opts.corpusTags ?? [] } : {}
|
|
1909
|
+
}
|
|
1910
|
+
);
|
|
1911
|
+
const steer = obs.findings.map((f) => f.recommended_action).filter((a) => typeof a === "string" && a.trim().length > 0).join("\n").trim();
|
|
1912
|
+
return { steer: steer || "COMPLETE", tokens };
|
|
1913
|
+
}
|
|
1914
|
+
function shotExecutor(surface, opts) {
|
|
1915
|
+
let artifact;
|
|
1916
|
+
return {
|
|
1917
|
+
runtime: "agentic-shot",
|
|
1918
|
+
async execute(task) {
|
|
1919
|
+
const t = task;
|
|
1920
|
+
const own = !t.handle;
|
|
1921
|
+
const handle = t.handle ?? await surface.open(t.task);
|
|
1922
|
+
try {
|
|
1923
|
+
const allTools = await surface.tools(t.task, handle);
|
|
1924
|
+
let tools = allTools;
|
|
1925
|
+
if (t.tools) {
|
|
1926
|
+
const known = new Set(allTools.map((tool) => tool.function.name));
|
|
1927
|
+
const unknown = t.tools.filter((name) => !known.has(name));
|
|
1928
|
+
if (unknown.length > 0) {
|
|
1929
|
+
throw new Error(
|
|
1930
|
+
`shot tools: unknown tool name(s) ${unknown.join(", ")} \u2014 domain offers: ${[...known].join(", ")}`
|
|
1931
|
+
);
|
|
1932
|
+
}
|
|
1933
|
+
const want = new Set(t.tools);
|
|
1934
|
+
tools = allTools.filter((tool) => want.has(tool.function.name));
|
|
1935
|
+
}
|
|
1936
|
+
const messages = t.messages?.length ? t.messages : [
|
|
1937
|
+
{ role: "system", content: t.persona?.systemPrompt ?? t.task.systemPrompt },
|
|
1938
|
+
{ role: "user", content: `${t.task.userPrompt}
|
|
1939
|
+
|
|
1940
|
+
${taskNudge}` }
|
|
1941
|
+
];
|
|
1942
|
+
if (t.messages?.length && t.persona?.systemPrompt) {
|
|
1943
|
+
messages.push({
|
|
1944
|
+
role: "user",
|
|
1945
|
+
content: `[hand-off] You are now acting as: ${t.persona.systemPrompt}`
|
|
1946
|
+
});
|
|
1947
|
+
}
|
|
1948
|
+
if (t.steer) messages.push({ role: "user", content: t.steer });
|
|
1949
|
+
const shot = await runShot(surface, t.task, handle, tools, messages, opts, t.persona?.model);
|
|
1950
|
+
const s = await surface.score(t.task, handle);
|
|
1951
|
+
const score = s.total > 0 ? s.passes / s.total : 0;
|
|
1952
|
+
const out = {
|
|
1953
|
+
messages: shot.messages,
|
|
1954
|
+
score,
|
|
1955
|
+
passes: s.passes,
|
|
1956
|
+
total: s.total,
|
|
1957
|
+
completions: shot.completions,
|
|
1958
|
+
toolErrors: shot.toolErrors
|
|
1959
|
+
};
|
|
1960
|
+
artifact = {
|
|
1961
|
+
outRef: `shot:${handle.id}:${shot.completions}:${s.passes}/${s.total}`,
|
|
1962
|
+
out,
|
|
1963
|
+
verdict: { valid: s.total > 0 && s.passes === s.total, score },
|
|
1964
|
+
// Real usage to the conserved pool: tokens from the router responses; usd only
|
|
1965
|
+
// when the model is in the price table (never a fabricated number).
|
|
1966
|
+
spent: {
|
|
1967
|
+
iterations: shot.completions,
|
|
1968
|
+
tokens: shot.tokens,
|
|
1969
|
+
usd: isModelPriced(opts.model) ? estimateCost(shot.tokens.input, shot.tokens.output, opts.model) : 0,
|
|
1970
|
+
ms: 0
|
|
1971
|
+
}
|
|
1972
|
+
};
|
|
1973
|
+
return artifact;
|
|
1974
|
+
} finally {
|
|
1975
|
+
if (own) await surface.close(handle);
|
|
1976
|
+
}
|
|
1977
|
+
},
|
|
1978
|
+
teardown: () => Promise.resolve({ destroyed: true }),
|
|
1979
|
+
resultArtifact() {
|
|
1980
|
+
if (!artifact) throw new Error("shotExecutor: resultArtifact before execute");
|
|
1981
|
+
return artifact;
|
|
1982
|
+
}
|
|
1983
|
+
};
|
|
1984
|
+
}
|
|
1985
|
+
function analystExecutor(opts) {
|
|
1986
|
+
let artifact;
|
|
1987
|
+
return {
|
|
1988
|
+
runtime: "agentic-analyst",
|
|
1989
|
+
async execute(task) {
|
|
1990
|
+
const t = task;
|
|
1991
|
+
const { steer, tokens } = t.rawInstruction ? await consultAnalyst(t.task, t.messages, t.rawInstruction, opts) : await analyze(t.task, t.messages, opts);
|
|
1992
|
+
const analystModel = opts.analystModel ?? opts.model;
|
|
1993
|
+
artifact = {
|
|
1994
|
+
outRef: `analyst:${steer.length}`,
|
|
1995
|
+
out: steer,
|
|
1996
|
+
spent: {
|
|
1997
|
+
iterations: 1,
|
|
1998
|
+
tokens,
|
|
1999
|
+
usd: isModelPriced(analystModel) ? estimateCost(tokens.input, tokens.output, analystModel) : 0,
|
|
2000
|
+
ms: 0
|
|
2001
|
+
}
|
|
2002
|
+
};
|
|
2003
|
+
return artifact;
|
|
2004
|
+
},
|
|
2005
|
+
teardown: () => Promise.resolve({ destroyed: true }),
|
|
2006
|
+
resultArtifact() {
|
|
2007
|
+
if (!artifact) throw new Error("analystExecutor: resultArtifact before execute");
|
|
2008
|
+
return artifact;
|
|
2009
|
+
}
|
|
2010
|
+
};
|
|
2011
|
+
}
|
|
2012
|
+
function agenticRegistry(surface, opts) {
|
|
2013
|
+
const leaves = {
|
|
2014
|
+
register() {
|
|
2015
|
+
throw new Error("agenticRegistry: register unsupported");
|
|
2016
|
+
},
|
|
2017
|
+
resolve(spec) {
|
|
2018
|
+
const role = spec.profile.metadata?.role;
|
|
2019
|
+
const factory = (_s, _ctx) => role === "analyst" ? analystExecutor(opts) : shotExecutor(surface, opts);
|
|
2020
|
+
return { succeeded: true, value: factory };
|
|
2021
|
+
}
|
|
2022
|
+
};
|
|
2023
|
+
return withDriverExecutor(leaves);
|
|
2024
|
+
}
|
|
2025
|
+
function leaf(name, role) {
|
|
2026
|
+
const agent = {
|
|
2027
|
+
name,
|
|
2028
|
+
executorSpec: { profile: { name, metadata: { role } }, harness: null },
|
|
2029
|
+
act() {
|
|
2030
|
+
throw new Error(`agentic: spawned child "${name}" was run directly (the executor drives it)`);
|
|
2031
|
+
}
|
|
2032
|
+
};
|
|
2033
|
+
return agent;
|
|
2034
|
+
}
|
|
2035
|
+
async function drainOne2(scope) {
|
|
2036
|
+
const s = await scope.next();
|
|
2037
|
+
if (!s) throw new Error("agentic: spawned child never settled");
|
|
2038
|
+
return s;
|
|
2039
|
+
}
|
|
2040
|
+
var perChild = (innerTurns) => ({
|
|
2041
|
+
maxIterations: innerTurns + 1,
|
|
2042
|
+
maxTokens: 1e6
|
|
2043
|
+
});
|
|
2044
|
+
function depthStrategy(surface, task, opts, cfg) {
|
|
2045
|
+
const innerTurns = opts.innerTurns ?? 4;
|
|
2046
|
+
let pendingSteer;
|
|
2047
|
+
return {
|
|
2048
|
+
name: "depth",
|
|
2049
|
+
async act(_t, scope) {
|
|
2050
|
+
const handle = await surface.open(task);
|
|
2051
|
+
const progression = [];
|
|
2052
|
+
let messages;
|
|
2053
|
+
let completions = 0;
|
|
2054
|
+
let shots = 0;
|
|
2055
|
+
try {
|
|
2056
|
+
for (shots = 0; shots < cfg.maxShots; shots += 1) {
|
|
2057
|
+
const child = leaf(`shot:${shots}`, "shot");
|
|
2058
|
+
const steer = shots === 0 ? void 0 : pendingSteer;
|
|
2059
|
+
const res = scope.spawn(child, { task, handle, messages, steer }, {
|
|
2060
|
+
budget: perChild(innerTurns),
|
|
2061
|
+
label: `shot:${shots}`
|
|
2062
|
+
});
|
|
2063
|
+
if (!res.ok) break;
|
|
2064
|
+
const settled = await drainOne2(scope);
|
|
2065
|
+
if (settled.kind === "down") break;
|
|
2066
|
+
const out = settled.out;
|
|
2067
|
+
messages = out.messages;
|
|
2068
|
+
completions += out.completions;
|
|
2069
|
+
progression.push(out.score);
|
|
2070
|
+
if (out.score >= 1 || shots === cfg.maxShots - 1) break;
|
|
2071
|
+
const aChild = leaf(`analyst:${shots}`, "analyst");
|
|
2072
|
+
const aRes = scope.spawn(
|
|
2073
|
+
aChild,
|
|
2074
|
+
{ task, messages },
|
|
2075
|
+
{ budget: perChild(1), label: `analyst:${shots}` }
|
|
2076
|
+
);
|
|
2077
|
+
if (!aRes.ok) break;
|
|
2078
|
+
const aSettled = await drainOne2(scope);
|
|
2079
|
+
completions += 1;
|
|
2080
|
+
if (aSettled.kind === "down") break;
|
|
2081
|
+
const findings = aSettled.out;
|
|
2082
|
+
if (/^\s*COMPLETE\b/i.test(findings)) break;
|
|
2083
|
+
pendingSteer = `A reviewer flagged unfinished items:
|
|
2084
|
+
${findings}
|
|
2085
|
+
|
|
2086
|
+
Address each with the tools, verify they took, then continue.`;
|
|
2087
|
+
}
|
|
2088
|
+
const final = await surface.score(task, handle);
|
|
2089
|
+
const score = final.total > 0 ? final.passes / final.total : 0;
|
|
2090
|
+
return {
|
|
2091
|
+
kind: "done",
|
|
2092
|
+
deliverable: {
|
|
2093
|
+
mode: "depth",
|
|
2094
|
+
score,
|
|
2095
|
+
resolved: final.total > 0 && final.passes === final.total,
|
|
2096
|
+
completions,
|
|
2097
|
+
progression,
|
|
2098
|
+
shots: shots + 1
|
|
2099
|
+
}
|
|
2100
|
+
};
|
|
2101
|
+
} finally {
|
|
2102
|
+
await surface.close(handle);
|
|
2103
|
+
}
|
|
2104
|
+
}
|
|
2105
|
+
};
|
|
2106
|
+
}
|
|
2107
|
+
function breadthStrategy(_surface, task, opts, cfg) {
|
|
2108
|
+
const innerTurns = opts.innerTurns ?? 4;
|
|
2109
|
+
return {
|
|
2110
|
+
name: "breadth",
|
|
2111
|
+
async act(_t, scope) {
|
|
2112
|
+
let opened = 0;
|
|
2113
|
+
for (let k = 0; k < cfg.width; k += 1) {
|
|
2114
|
+
const res = scope.spawn(leaf(`rollout:${k}`, "shot"), { task }, {
|
|
2115
|
+
budget: perChild(innerTurns),
|
|
2116
|
+
label: `rollout:${k}`
|
|
2117
|
+
});
|
|
2118
|
+
if (res.ok) opened += 1;
|
|
2119
|
+
}
|
|
2120
|
+
if (opened === 0) return { kind: "blocked", blockers: ["breadth: pool admitted no rollout"] };
|
|
2121
|
+
let best = -1;
|
|
2122
|
+
let bestResolved = false;
|
|
2123
|
+
let completions = 0;
|
|
2124
|
+
const progression = [];
|
|
2125
|
+
for (let s = await scope.next(); s !== null; s = await scope.next()) {
|
|
2126
|
+
if (s.kind === "down") continue;
|
|
2127
|
+
const out = s.out;
|
|
2128
|
+
completions += out.completions;
|
|
2129
|
+
if (out.score > best) best = out.score;
|
|
2130
|
+
if (out.total > 0 && out.passes === out.total) bestResolved = true;
|
|
2131
|
+
progression.push(best);
|
|
2132
|
+
}
|
|
2133
|
+
if (best < 0) return { kind: "blocked", blockers: ["breadth: every rollout went down"] };
|
|
2134
|
+
return {
|
|
2135
|
+
kind: "done",
|
|
2136
|
+
deliverable: {
|
|
2137
|
+
mode: "breadth",
|
|
2138
|
+
score: best,
|
|
2139
|
+
resolved: bestResolved,
|
|
2140
|
+
completions,
|
|
2141
|
+
progression,
|
|
2142
|
+
shots: opened
|
|
2143
|
+
}
|
|
2144
|
+
};
|
|
2145
|
+
}
|
|
2146
|
+
};
|
|
2147
|
+
}
|
|
2148
|
+
var sample = {
|
|
2149
|
+
name: "sample",
|
|
2150
|
+
driver: (surface, task, opts, budget) => breadthStrategy(surface, task, opts, { width: budget })
|
|
2151
|
+
};
|
|
2152
|
+
var refine = {
|
|
2153
|
+
name: "refine",
|
|
2154
|
+
driver: (surface, task, opts, budget) => depthStrategy(surface, task, opts, { maxShots: budget })
|
|
2155
|
+
};
|
|
2156
|
+
function defineStrategy(name, run) {
|
|
2157
|
+
return {
|
|
2158
|
+
name,
|
|
2159
|
+
driver: (surface, task, opts, budget) => ({
|
|
2160
|
+
name,
|
|
2161
|
+
async act(_t, scope) {
|
|
2162
|
+
let seq = 0;
|
|
2163
|
+
const innerTurns = opts.innerTurns ?? 4;
|
|
2164
|
+
let verifiedBest = 0;
|
|
2165
|
+
let verifiedResolved = false;
|
|
2166
|
+
const openHandles = /* @__PURE__ */ new Set();
|
|
2167
|
+
const ctx = {
|
|
2168
|
+
// Narrowed to open/close — the body gets no raw call()/score() access.
|
|
2169
|
+
surface: {
|
|
2170
|
+
name: surface.name,
|
|
2171
|
+
open: async (t) => {
|
|
2172
|
+
const h = await surface.open(t);
|
|
2173
|
+
openHandles.add(h.id);
|
|
2174
|
+
return h;
|
|
2175
|
+
},
|
|
2176
|
+
close: async (h) => {
|
|
2177
|
+
if (!h || !openHandles.has(h.id)) return;
|
|
2178
|
+
openHandles.delete(h.id);
|
|
2179
|
+
await surface.close(h);
|
|
2180
|
+
}
|
|
2181
|
+
},
|
|
2182
|
+
task,
|
|
2183
|
+
opts,
|
|
2184
|
+
budget,
|
|
2185
|
+
scope,
|
|
2186
|
+
async shot(spec) {
|
|
2187
|
+
const child = leaf(`shot:${seq}`, "shot");
|
|
2188
|
+
seq += 1;
|
|
2189
|
+
const res = scope.spawn(
|
|
2190
|
+
child,
|
|
2191
|
+
{
|
|
2192
|
+
task,
|
|
2193
|
+
handle: spec?.handle,
|
|
2194
|
+
messages: spec?.messages,
|
|
2195
|
+
steer: spec?.steer,
|
|
2196
|
+
persona: spec?.persona,
|
|
2197
|
+
tools: spec?.tools
|
|
2198
|
+
},
|
|
2199
|
+
{ budget: perChild(innerTurns), label: child.name }
|
|
2200
|
+
);
|
|
2201
|
+
if (!res.ok) return null;
|
|
2202
|
+
const settled = await drainOne2(scope);
|
|
2203
|
+
if (settled.kind === "down") return null;
|
|
2204
|
+
const out = settled.out;
|
|
2205
|
+
if (out.score > verifiedBest) verifiedBest = out.score;
|
|
2206
|
+
if (out.total > 0 && out.passes === out.total) verifiedResolved = true;
|
|
2207
|
+
return out;
|
|
2208
|
+
},
|
|
2209
|
+
async listTools(handle) {
|
|
2210
|
+
const tools = await surface.tools(task, handle);
|
|
2211
|
+
return tools.map((t) => ({
|
|
2212
|
+
name: t.function.name,
|
|
2213
|
+
...t.function.description ? { description: t.function.description } : {}
|
|
2214
|
+
}));
|
|
2215
|
+
},
|
|
2216
|
+
async critique(messages) {
|
|
2217
|
+
const child = leaf(`analyst:${seq}`, "analyst");
|
|
2218
|
+
seq += 1;
|
|
2219
|
+
const res = scope.spawn(
|
|
2220
|
+
child,
|
|
2221
|
+
{ task, messages },
|
|
2222
|
+
{ budget: perChild(1), label: child.name }
|
|
2223
|
+
);
|
|
2224
|
+
if (!res.ok) return null;
|
|
2225
|
+
const settled = await drainOne2(scope);
|
|
2226
|
+
if (settled.kind === "down") return null;
|
|
2227
|
+
const findings = settled.out;
|
|
2228
|
+
return /^\s*COMPLETE\b/i.test(findings) ? null : findings;
|
|
2229
|
+
},
|
|
2230
|
+
async consult(messages, instruction) {
|
|
2231
|
+
const child = leaf(`analyst:${seq}`, "analyst");
|
|
2232
|
+
seq += 1;
|
|
2233
|
+
const res = scope.spawn(
|
|
2234
|
+
child,
|
|
2235
|
+
{ task, messages, rawInstruction: instruction },
|
|
2236
|
+
{ budget: perChild(1), label: child.name }
|
|
2237
|
+
);
|
|
2238
|
+
if (!res.ok) return null;
|
|
2239
|
+
const settled = await drainOne2(scope);
|
|
2240
|
+
if (settled.kind === "down") return null;
|
|
2241
|
+
return settled.out;
|
|
2242
|
+
}
|
|
2243
|
+
};
|
|
2244
|
+
const r = await run(ctx);
|
|
2245
|
+
return {
|
|
2246
|
+
kind: "done",
|
|
2247
|
+
deliverable: {
|
|
2248
|
+
mode: name,
|
|
2249
|
+
...r,
|
|
2250
|
+
progression: Array.isArray(r.progression) ? r.progression : [],
|
|
2251
|
+
completions: typeof r.completions === "number" ? r.completions : 0,
|
|
2252
|
+
shots: typeof r.shots === "number" ? r.shots : 0,
|
|
2253
|
+
score: verifiedBest,
|
|
2254
|
+
resolved: verifiedResolved
|
|
2255
|
+
}
|
|
2256
|
+
};
|
|
2257
|
+
}
|
|
2258
|
+
})
|
|
2259
|
+
};
|
|
2260
|
+
}
|
|
2261
|
+
var adaptiveRefine = defineStrategy(
|
|
2262
|
+
"adaptiveRefine",
|
|
2263
|
+
async ({ surface, task, budget, shot, critique }) => {
|
|
2264
|
+
let handle = await surface.open(task);
|
|
2265
|
+
const progression = [];
|
|
2266
|
+
let messages;
|
|
2267
|
+
let steer;
|
|
2268
|
+
let completions = 0;
|
|
2269
|
+
let best = -1;
|
|
2270
|
+
let shots = 0;
|
|
2271
|
+
try {
|
|
2272
|
+
for (shots = 0; shots < budget; shots += 1) {
|
|
2273
|
+
const out = await shot({ handle, messages, steer });
|
|
2274
|
+
if (!out) break;
|
|
2275
|
+
completions += out.completions;
|
|
2276
|
+
progression.push(out.score);
|
|
2277
|
+
if (out.score >= 1) break;
|
|
2278
|
+
if (out.score <= best) {
|
|
2279
|
+
await surface.close(handle);
|
|
2280
|
+
handle = await surface.open(task);
|
|
2281
|
+
messages = void 0;
|
|
2282
|
+
steer = void 0;
|
|
2283
|
+
continue;
|
|
2284
|
+
}
|
|
2285
|
+
best = out.score;
|
|
2286
|
+
messages = out.messages;
|
|
2287
|
+
const findings = await critique(out.messages);
|
|
2288
|
+
completions += 1;
|
|
2289
|
+
if (!findings) break;
|
|
2290
|
+
steer = `A reviewer flagged unfinished items:
|
|
2291
|
+
${findings}
|
|
2292
|
+
|
|
2293
|
+
Address each with the tools, verify they took, then continue.`;
|
|
2294
|
+
}
|
|
2295
|
+
const score = progression.length ? Math.max(...progression) : 0;
|
|
2296
|
+
return { score, resolved: score >= 1, completions, progression, shots };
|
|
2297
|
+
} finally {
|
|
2298
|
+
await surface.close(handle);
|
|
2299
|
+
}
|
|
2300
|
+
}
|
|
2301
|
+
);
|
|
2302
|
+
var sampleThenRefine = defineStrategy(
|
|
2303
|
+
"sampleThenRefine",
|
|
2304
|
+
async ({ surface, task, budget, shot, critique }) => {
|
|
2305
|
+
const explore = Math.max(1, Math.ceil(budget / 2));
|
|
2306
|
+
const open = /* @__PURE__ */ new Set();
|
|
2307
|
+
const progression = [];
|
|
2308
|
+
let completions = 0;
|
|
2309
|
+
let shots = 0;
|
|
2310
|
+
try {
|
|
2311
|
+
let best;
|
|
2312
|
+
for (let i = 0; i < explore; i += 1) {
|
|
2313
|
+
const handle = await surface.open(task);
|
|
2314
|
+
open.add(handle);
|
|
2315
|
+
const out = await shot({ handle });
|
|
2316
|
+
if (!out) continue;
|
|
2317
|
+
shots += 1;
|
|
2318
|
+
completions += out.completions;
|
|
2319
|
+
progression.push(out.score);
|
|
2320
|
+
if (!best || out.score > best.out.score) best = { handle, out };
|
|
2321
|
+
if (out.score >= 1) break;
|
|
2322
|
+
}
|
|
2323
|
+
if (!best) return { score: 0, resolved: false, completions, progression, shots };
|
|
2324
|
+
for (const h of [...open]) {
|
|
2325
|
+
if (h !== best.handle) {
|
|
2326
|
+
await surface.close(h);
|
|
2327
|
+
open.delete(h);
|
|
2328
|
+
}
|
|
2329
|
+
}
|
|
2330
|
+
let messages = best.out.messages;
|
|
2331
|
+
let topScore = best.out.score;
|
|
2332
|
+
for (let i = explore; i < budget && topScore < 1; i += 1) {
|
|
2333
|
+
const findings = await critique(messages);
|
|
2334
|
+
completions += 1;
|
|
2335
|
+
if (!findings) break;
|
|
2336
|
+
const out = await shot({
|
|
2337
|
+
handle: best.handle,
|
|
2338
|
+
messages,
|
|
2339
|
+
steer: `A reviewer flagged unfinished items:
|
|
2340
|
+
${findings}
|
|
2341
|
+
|
|
2342
|
+
Address each with the tools, verify they took, then continue.`
|
|
2343
|
+
});
|
|
2344
|
+
if (!out) break;
|
|
2345
|
+
shots += 1;
|
|
2346
|
+
completions += out.completions;
|
|
2347
|
+
progression.push(out.score);
|
|
2348
|
+
messages = out.messages;
|
|
2349
|
+
if (out.score > topScore) topScore = out.score;
|
|
2350
|
+
}
|
|
2351
|
+
const score = progression.length ? Math.max(...progression) : 0;
|
|
2352
|
+
return { score, resolved: score >= 1, completions, progression, shots };
|
|
2353
|
+
} finally {
|
|
2354
|
+
for (const h of open) await surface.close(h);
|
|
2355
|
+
}
|
|
2356
|
+
}
|
|
2357
|
+
);
|
|
2358
|
+
async function runAgentic(opts) {
|
|
2359
|
+
const strategy = opts.strategy ?? (opts.mode === "breadth" ? sample : refine);
|
|
2360
|
+
const driver = strategy.driver(opts.surface, opts.task, opts, opts.budget);
|
|
2361
|
+
const supervisor = createSupervisor();
|
|
2362
|
+
const root = opts.rootBudget ?? {
|
|
2363
|
+
maxIterations: opts.budget * ((opts.innerTurns ?? 4) + 2),
|
|
2364
|
+
maxTokens: 1e9
|
|
2365
|
+
};
|
|
2366
|
+
const started = Date.now();
|
|
2367
|
+
const result = await supervisor.run(driver, void 0, {
|
|
2368
|
+
budget: root,
|
|
2369
|
+
runId: `agentic:${strategy.name}:${opts.task.id}`,
|
|
2370
|
+
journal: new InMemorySpawnJournal(),
|
|
2371
|
+
blobs: new InMemoryResultBlobStore(),
|
|
2372
|
+
executors: agenticRegistry(opts.surface, opts),
|
|
2373
|
+
maxDepth: 3,
|
|
2374
|
+
...opts.hooks ? { hooks: opts.hooks } : {}
|
|
2375
|
+
});
|
|
2376
|
+
if (result.kind !== "winner" || result.out.kind !== "done") {
|
|
2377
|
+
const reason = result.kind === "winner" ? `blocked: ${result.out.blockers?.join("; ")}` : `no-winner: ${result.reason}`;
|
|
2378
|
+
throw new Error(`runAgentic(${strategy.name}) produced no result \u2014 ${reason}`);
|
|
2379
|
+
}
|
|
2380
|
+
const core = result.out.deliverable;
|
|
2381
|
+
return {
|
|
2382
|
+
...core,
|
|
2383
|
+
usd: result.spentTotal.usd,
|
|
2384
|
+
tokens: result.spentTotal.tokens,
|
|
2385
|
+
ms: Date.now() - started
|
|
2386
|
+
};
|
|
2387
|
+
}
|
|
2388
|
+
|
|
2389
|
+
// src/runtime/run-benchmark.ts
|
|
2390
|
+
async function pool(items, limit, fn) {
|
|
2391
|
+
const out = new Array(items.length);
|
|
2392
|
+
let next = 0;
|
|
2393
|
+
const workers = Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, async () => {
|
|
2394
|
+
while (next < items.length) {
|
|
2395
|
+
const i = next;
|
|
2396
|
+
next += 1;
|
|
2397
|
+
out[i] = await fn(items[i], i);
|
|
2398
|
+
}
|
|
2399
|
+
});
|
|
2400
|
+
await Promise.all(workers);
|
|
2401
|
+
return out;
|
|
2402
|
+
}
|
|
2403
|
+
async function runBenchmark(cfg) {
|
|
2404
|
+
const strategies = cfg.strategies ?? [sample, refine];
|
|
2405
|
+
const budget = cfg.budget ?? 3;
|
|
2406
|
+
const concurrency = cfg.concurrency ?? 3;
|
|
2407
|
+
let settled = 0;
|
|
2408
|
+
const perTask = await pool(cfg.tasks, concurrency, async (task) => {
|
|
2409
|
+
const cells = {};
|
|
2410
|
+
const errors = {};
|
|
2411
|
+
let row;
|
|
2412
|
+
try {
|
|
2413
|
+
for (const s of strategies) {
|
|
2414
|
+
try {
|
|
2415
|
+
const r = await runAgentic({
|
|
2416
|
+
...cfg.worker,
|
|
2417
|
+
surface: cfg.environment,
|
|
2418
|
+
task,
|
|
2419
|
+
strategy: s,
|
|
2420
|
+
budget,
|
|
2421
|
+
...cfg.hooks ? { hooks: cfg.hooks } : {}
|
|
2422
|
+
});
|
|
2423
|
+
cells[s.name] = {
|
|
2424
|
+
score: r.score,
|
|
2425
|
+
resolved: r.resolved,
|
|
2426
|
+
progression: r.progression,
|
|
2427
|
+
usd: r.usd,
|
|
2428
|
+
ms: r.ms,
|
|
2429
|
+
tokens: r.tokens
|
|
2430
|
+
};
|
|
2431
|
+
} catch (e) {
|
|
2432
|
+
errors[s.name] = e instanceof Error ? e.message.slice(0, 300) : String(e);
|
|
2433
|
+
cells[s.name] = {
|
|
2434
|
+
score: 0,
|
|
2435
|
+
resolved: false,
|
|
2436
|
+
progression: [],
|
|
2437
|
+
usd: 0,
|
|
2438
|
+
ms: 0,
|
|
2439
|
+
tokens: { input: 0, output: 0 }
|
|
2440
|
+
};
|
|
2441
|
+
}
|
|
2442
|
+
}
|
|
2443
|
+
row = {
|
|
2444
|
+
taskId: task.id,
|
|
2445
|
+
cells,
|
|
2446
|
+
...Object.keys(errors).length > 0 ? { errors } : {}
|
|
2447
|
+
};
|
|
2448
|
+
} catch (e) {
|
|
2449
|
+
row = { taskId: task.id, error: e instanceof Error ? e.message.slice(0, 300) : String(e) };
|
|
2450
|
+
}
|
|
2451
|
+
settled += 1;
|
|
2452
|
+
cfg.onTask?.(row, settled, cfg.tasks.length);
|
|
2453
|
+
return row;
|
|
2454
|
+
});
|
|
2455
|
+
const ok = perTask.filter(
|
|
2456
|
+
(r) => !!r.cells
|
|
2457
|
+
);
|
|
2458
|
+
const mean = (xs) => xs.length ? xs.reduce((s, x) => s + x, 0) / xs.length : 0;
|
|
2459
|
+
const perStrategy = {};
|
|
2460
|
+
for (const s of strategies) {
|
|
2461
|
+
const cells = ok.map((r) => r.cells[s.name]).filter((c) => !!c);
|
|
2462
|
+
perStrategy[s.name] = {
|
|
2463
|
+
score: mean(cells.map((c) => c.score)),
|
|
2464
|
+
resolved: mean(cells.map((c) => c.resolved ? 1 : 0)),
|
|
2465
|
+
usd: mean(cells.map((c) => c.usd)),
|
|
2466
|
+
ms: mean(cells.map((c) => c.ms))
|
|
2467
|
+
};
|
|
2468
|
+
}
|
|
2469
|
+
const frontier = paretoFrontier(
|
|
2470
|
+
Object.entries(perStrategy).map(([name, v]) => ({ name, score: v.score, usd: v.usd })),
|
|
2471
|
+
[
|
|
2472
|
+
{ name: "score", direction: "maximize", value: (c) => c.score },
|
|
2473
|
+
{ name: "usd", direction: "minimize", value: (c) => c.usd }
|
|
2474
|
+
]
|
|
2475
|
+
).frontier.map((c) => c.name);
|
|
2476
|
+
const report = {
|
|
2477
|
+
n: ok.length,
|
|
2478
|
+
excluded: perTask.length - ok.length,
|
|
2479
|
+
perStrategy,
|
|
2480
|
+
perTask,
|
|
2481
|
+
pareto: frontier
|
|
2482
|
+
};
|
|
2483
|
+
const names = strategies.map((s) => s.name);
|
|
2484
|
+
if (names.includes("refine") && names.includes("sample") && ok.length >= 2) {
|
|
2485
|
+
const b = pairedBootstrap(
|
|
2486
|
+
ok.map((r) => r.cells.sample?.score ?? 0),
|
|
2487
|
+
ok.map((r) => r.cells.refine?.score ?? 0)
|
|
2488
|
+
);
|
|
2489
|
+
report.refineVsSample = { mean: b.mean, low: b.low, high: b.high, n: b.n };
|
|
2490
|
+
}
|
|
2491
|
+
return report;
|
|
2492
|
+
}
|
|
2493
|
+
function printBenchmarkReport(report) {
|
|
2494
|
+
const pct = (x) => `${(x * 100).toFixed(1)}%`;
|
|
2495
|
+
const pp = (x) => `${x >= 0 ? "+" : ""}${(x * 100).toFixed(1)}pp`;
|
|
2496
|
+
console.log(
|
|
2497
|
+
`
|
|
2498
|
+
=== benchmark \xB7 n=${report.n}${report.excluded ? ` (excluded ${report.excluded})` : ""} ===`
|
|
2499
|
+
);
|
|
2500
|
+
console.log(
|
|
2501
|
+
` ${"strategy".padEnd(16)} ${"score".padStart(7)} ${"resolved".padStart(9)} ${"$/task".padStart(8)} ${"s/task".padStart(7)}`
|
|
2502
|
+
);
|
|
2503
|
+
for (const [s, v] of Object.entries(report.perStrategy))
|
|
2504
|
+
console.log(
|
|
2505
|
+
` ${(report.pareto.includes(s) ? `${s} *` : s).padEnd(16)} ${pct(v.score).padStart(7)} ${pct(v.resolved).padStart(9)} ${`$${v.usd.toFixed(3)}`.padStart(8)} ${(v.ms / 1e3).toFixed(0).padStart(6)}s`
|
|
2506
|
+
);
|
|
2507
|
+
if (report.pareto.length) console.log(` * = on the (score, $) Pareto frontier`);
|
|
2508
|
+
for (const row of report.perTask)
|
|
2509
|
+
if (row.error) console.log(` \u26A0 ${row.taskId}: ${row.error.slice(0, 120)}`);
|
|
2510
|
+
const l = report.refineVsSample;
|
|
2511
|
+
if (l) {
|
|
2512
|
+
const sig = l.low > 0 ? "SIGNIF +" : l.high < 0 ? "SIGNIF -" : "n.s.";
|
|
2513
|
+
console.log(` refine \u2212 sample: ${pp(l.mean)} CI [${pp(l.low)}, ${pp(l.high)}] (${sig})`);
|
|
2514
|
+
}
|
|
2515
|
+
}
|
|
2516
|
+
|
|
2517
|
+
// src/runtime/sandbox-run.ts
|
|
2518
|
+
var SandboxRunAbortError = class extends Error {
|
|
2519
|
+
name = "AbortError";
|
|
2520
|
+
/** Events drained from the stream before the abort interrupted the turn. */
|
|
2521
|
+
events;
|
|
2522
|
+
/** The last artifact read error, if the abort fired during the retry loop. */
|
|
2523
|
+
readError;
|
|
2524
|
+
constructor(events, readError) {
|
|
2525
|
+
super("aborted");
|
|
2526
|
+
this.events = events;
|
|
2527
|
+
if (readError !== void 0) this.readError = readError;
|
|
2528
|
+
}
|
|
2529
|
+
};
|
|
2530
|
+
async function openSandboxRun(client, options, deliverable) {
|
|
2531
|
+
const runId = options.runId ?? `sandbox-run-${randomSuffix()}`;
|
|
2532
|
+
const now = options.now ?? Date.now;
|
|
2533
|
+
const capabilities = await probeSandboxCapabilities(client);
|
|
2534
|
+
const lineage = createSandboxLineage(client, capabilities, {
|
|
2535
|
+
...options.maxConcurrency !== void 0 ? { maxConcurrency: options.maxConcurrency } : {}
|
|
2536
|
+
});
|
|
2537
|
+
let handle;
|
|
2538
|
+
let started = false;
|
|
2539
|
+
let runStartedAt;
|
|
2540
|
+
let failed = false;
|
|
2541
|
+
let turnCount = 0;
|
|
2542
|
+
function emit(event) {
|
|
2543
|
+
notifyRuntimeHookEvent(
|
|
2544
|
+
options.hooks,
|
|
2545
|
+
{
|
|
2546
|
+
id: `${runId}:${event.target}:${event.phase}${event.stepIndex === void 0 ? "" : `:${event.stepIndex}`}`,
|
|
2547
|
+
runId,
|
|
2548
|
+
scenarioId: options.scenarioId,
|
|
2549
|
+
target: event.target,
|
|
2550
|
+
phase: event.phase,
|
|
2551
|
+
timestamp: event.timestamp,
|
|
2552
|
+
stepIndex: event.stepIndex,
|
|
2553
|
+
payload: event.payload,
|
|
2554
|
+
metadata: { producer: "openSandboxRun" }
|
|
2555
|
+
},
|
|
2556
|
+
{ signal: options.signal }
|
|
2557
|
+
);
|
|
2558
|
+
}
|
|
2559
|
+
const runPayload = () => ({
|
|
2560
|
+
agentName: options.agentRun.name ?? options.agentRun.profile.name ?? "agent",
|
|
2561
|
+
profileName: options.agentRun.profile.name,
|
|
2562
|
+
backendType: backendType(options.agentRun),
|
|
2563
|
+
deliverableKind: deliverable.kind,
|
|
2564
|
+
...deliverable.kind === "artifact" ? { deliverablePath: deliverable.path } : {},
|
|
2565
|
+
...handle ? { sessionId: handle.sessionId, sandboxId: handle.box.id } : {}
|
|
2566
|
+
});
|
|
2567
|
+
const turnPayload = (prompt, turnKind, startedAt, result, error) => ({
|
|
2568
|
+
...runPayload(),
|
|
2569
|
+
turnKind,
|
|
2570
|
+
promptChars: prompt.length,
|
|
2571
|
+
promptHash: hashText(prompt),
|
|
2572
|
+
...result !== void 0 || error !== void 0 ? { durationMs: Math.max(0, now() - startedAt) } : {},
|
|
2573
|
+
...result ? {
|
|
2574
|
+
eventCount: result.events.length,
|
|
2575
|
+
eventTypes: eventTypeCounts(result.events),
|
|
2576
|
+
...result.readError !== void 0 ? { readError: result.readError } : {}
|
|
2577
|
+
} : {},
|
|
2578
|
+
...error !== void 0 ? { error: errorMessage(error) } : {}
|
|
2579
|
+
});
|
|
2580
|
+
async function settle2(box, events) {
|
|
2581
|
+
const collected = [];
|
|
2582
|
+
try {
|
|
2583
|
+
for await (const ev of events) collected.push(ev);
|
|
2584
|
+
} catch (err) {
|
|
2585
|
+
if (isAbortError(err)) throw new SandboxRunAbortError(collected);
|
|
2586
|
+
throw err;
|
|
2587
|
+
}
|
|
2588
|
+
if (deliverable.kind === "events") {
|
|
2589
|
+
return { out: deliverable.fromEvents(collected), events: collected };
|
|
2590
|
+
}
|
|
2591
|
+
if (options.signal.aborted) throw new SandboxRunAbortError(collected);
|
|
2592
|
+
let raw = "";
|
|
2593
|
+
let readError;
|
|
2594
|
+
const readAttempts = 4;
|
|
2595
|
+
const readDelayMs = options.readRetryDelayMs ?? 1e3;
|
|
2596
|
+
for (let attempt = 0; attempt < readAttempts; attempt += 1) {
|
|
2597
|
+
if (options.signal.aborted) throw new SandboxRunAbortError(collected, readError);
|
|
2598
|
+
try {
|
|
2599
|
+
raw = await box.fs.read(deliverable.path);
|
|
2600
|
+
readError = void 0;
|
|
2601
|
+
break;
|
|
2602
|
+
} catch (err) {
|
|
2603
|
+
readError = err instanceof Error ? err.message : String(err);
|
|
2604
|
+
if (attempt < readAttempts - 1 && readDelayMs > 0)
|
|
2605
|
+
await sleep(readDelayMs * (attempt + 1), options.signal);
|
|
2606
|
+
}
|
|
2607
|
+
}
|
|
2608
|
+
return {
|
|
2609
|
+
out: deliverable.fromArtifact(raw, collected),
|
|
2610
|
+
events: collected,
|
|
2611
|
+
...readError !== void 0 ? { readError } : {}
|
|
2612
|
+
};
|
|
2613
|
+
}
|
|
2614
|
+
return {
|
|
2615
|
+
get box() {
|
|
2616
|
+
if (!handle) throw new Error("openSandboxRun: box unavailable before start()");
|
|
2617
|
+
return handle.box;
|
|
2618
|
+
},
|
|
2619
|
+
get sessionId() {
|
|
2620
|
+
if (!handle) throw new Error("openSandboxRun: sessionId unavailable before start()");
|
|
2621
|
+
return handle.sessionId;
|
|
2622
|
+
},
|
|
2623
|
+
async start(prompt) {
|
|
2624
|
+
if (started)
|
|
2625
|
+
throw new Error(
|
|
2626
|
+
"openSandboxRun: start() already called \u2014 use resume() to continue the session"
|
|
2627
|
+
);
|
|
2628
|
+
started = true;
|
|
2629
|
+
runStartedAt = now();
|
|
2630
|
+
emit({
|
|
2631
|
+
target: "agent.run",
|
|
2632
|
+
phase: "before",
|
|
2633
|
+
timestamp: runStartedAt,
|
|
2634
|
+
payload: { ...runPayload(), turnCount: 0 }
|
|
2635
|
+
});
|
|
2636
|
+
const stepIndex = turnCount;
|
|
2637
|
+
const turnStartedAt = now();
|
|
2638
|
+
emit({
|
|
2639
|
+
target: "agent.turn",
|
|
2640
|
+
phase: "before",
|
|
2641
|
+
timestamp: turnStartedAt,
|
|
2642
|
+
stepIndex,
|
|
2643
|
+
payload: turnPayload(prompt, "start", turnStartedAt)
|
|
2644
|
+
});
|
|
2645
|
+
try {
|
|
2646
|
+
const r = await lineage.start(
|
|
2647
|
+
options.agentRun,
|
|
2648
|
+
prompt,
|
|
2649
|
+
options.signal
|
|
2650
|
+
);
|
|
2651
|
+
handle = r.handle;
|
|
2652
|
+
const result = await settle2(handle.box, r.events);
|
|
2653
|
+
turnCount += 1;
|
|
2654
|
+
emit({
|
|
2655
|
+
target: "agent.turn",
|
|
2656
|
+
phase: "after",
|
|
2657
|
+
timestamp: now(),
|
|
2658
|
+
stepIndex,
|
|
2659
|
+
payload: turnPayload(prompt, "start", turnStartedAt, result)
|
|
2660
|
+
});
|
|
2661
|
+
return result;
|
|
2662
|
+
} catch (error) {
|
|
2663
|
+
failed = true;
|
|
2664
|
+
emit({
|
|
2665
|
+
target: "agent.turn",
|
|
2666
|
+
phase: "error",
|
|
2667
|
+
timestamp: now(),
|
|
2668
|
+
stepIndex,
|
|
2669
|
+
payload: turnPayload(prompt, "start", turnStartedAt, void 0, error)
|
|
2670
|
+
});
|
|
2671
|
+
emit({
|
|
2672
|
+
target: "agent.run",
|
|
2673
|
+
phase: "error",
|
|
2674
|
+
timestamp: now(),
|
|
2675
|
+
payload: { ...runPayload(), turnCount, error: errorMessage(error) }
|
|
2676
|
+
});
|
|
2677
|
+
throw error;
|
|
2678
|
+
}
|
|
2679
|
+
},
|
|
2680
|
+
async resume(prompt) {
|
|
2681
|
+
if (!handle) throw new Error("openSandboxRun: resume() called before start()");
|
|
2682
|
+
const stepIndex = turnCount;
|
|
2683
|
+
const turnStartedAt = now();
|
|
2684
|
+
emit({
|
|
2685
|
+
target: "agent.turn",
|
|
2686
|
+
phase: "before",
|
|
2687
|
+
timestamp: turnStartedAt,
|
|
2688
|
+
stepIndex,
|
|
2689
|
+
payload: turnPayload(prompt, "resume", turnStartedAt)
|
|
2690
|
+
});
|
|
2691
|
+
try {
|
|
2692
|
+
const result = await settle2(
|
|
2693
|
+
handle.box,
|
|
2694
|
+
await lineage.continue(handle, prompt, options.signal)
|
|
2695
|
+
);
|
|
2696
|
+
turnCount += 1;
|
|
2697
|
+
emit({
|
|
2698
|
+
target: "agent.turn",
|
|
2699
|
+
phase: "after",
|
|
2700
|
+
timestamp: now(),
|
|
2701
|
+
stepIndex,
|
|
2702
|
+
payload: turnPayload(prompt, "resume", turnStartedAt, result)
|
|
2703
|
+
});
|
|
2704
|
+
return result;
|
|
2705
|
+
} catch (error) {
|
|
2706
|
+
failed = true;
|
|
2707
|
+
emit({
|
|
2708
|
+
target: "agent.turn",
|
|
2709
|
+
phase: "error",
|
|
2710
|
+
timestamp: now(),
|
|
2711
|
+
stepIndex,
|
|
2712
|
+
payload: turnPayload(prompt, "resume", turnStartedAt, void 0, error)
|
|
2713
|
+
});
|
|
2714
|
+
emit({
|
|
2715
|
+
target: "agent.run",
|
|
2716
|
+
phase: "error",
|
|
2717
|
+
timestamp: now(),
|
|
2718
|
+
payload: { ...runPayload(), turnCount, error: errorMessage(error) }
|
|
2719
|
+
});
|
|
2720
|
+
throw error;
|
|
2721
|
+
}
|
|
2722
|
+
},
|
|
2723
|
+
async close() {
|
|
2724
|
+
await lineage.teardown();
|
|
2725
|
+
if (runStartedAt !== void 0) {
|
|
2726
|
+
emit({
|
|
2727
|
+
target: "agent.run",
|
|
2728
|
+
phase: "after",
|
|
2729
|
+
timestamp: now(),
|
|
2730
|
+
payload: {
|
|
2731
|
+
...runPayload(),
|
|
2732
|
+
turnCount,
|
|
2733
|
+
status: failed ? "error" : "completed",
|
|
2734
|
+
durationMs: Math.max(0, now() - runStartedAt)
|
|
2735
|
+
}
|
|
2736
|
+
});
|
|
2737
|
+
}
|
|
2738
|
+
}
|
|
2739
|
+
};
|
|
2740
|
+
}
|
|
2741
|
+
function backendType(spec) {
|
|
2742
|
+
const backend = spec.sandboxOverrides?.backend;
|
|
2743
|
+
return backend?.type;
|
|
2744
|
+
}
|
|
2745
|
+
function eventTypeCounts(events) {
|
|
2746
|
+
const counts = {};
|
|
2747
|
+
for (const event of events) counts[event.type] = (counts[event.type] ?? 0) + 1;
|
|
2748
|
+
return counts;
|
|
2749
|
+
}
|
|
2750
|
+
function hashText(value) {
|
|
2751
|
+
let hash = 2166136261;
|
|
2752
|
+
for (let i = 0; i < value.length; i += 1) {
|
|
2753
|
+
hash ^= value.charCodeAt(i);
|
|
2754
|
+
hash = Math.imul(hash, 16777619);
|
|
2755
|
+
}
|
|
2756
|
+
return (hash >>> 0).toString(16).padStart(8, "0");
|
|
2757
|
+
}
|
|
2758
|
+
function errorMessage(error) {
|
|
2759
|
+
return error instanceof Error ? error.message : String(error);
|
|
2760
|
+
}
|
|
2761
|
+
|
|
2762
|
+
// src/runtime/strategy-author.ts
|
|
2763
|
+
import { mkdirSync, writeFileSync } from "fs";
|
|
2764
|
+
import { join } from "path";
|
|
2765
|
+
var strategyAuthorContract = `
|
|
2766
|
+
You author an OPTIMIZATION STRATEGY for an agentic loop system. A strategy decides how to
|
|
2767
|
+
spend a compute budget to beat a task's deployable check. You compose exactly two steps:
|
|
2768
|
+
|
|
2769
|
+
shot(spec?: { handle?, messages?, steer?, persona?, tools? }): Promise<ShotResult | null>
|
|
2770
|
+
Runs ONE worker attempt (a bounded tool loop) over an artifact.
|
|
2771
|
+
- omit handle => the shot opens its OWN fresh artifact and closes it after (a sample).
|
|
2772
|
+
- pass handle => the shot CONTINUES that artifact (state accumulates across shots).
|
|
2773
|
+
- messages => the carried conversation (pass the previous ShotResult.messages to continue).
|
|
2774
|
+
- steer => a corrective instruction injected before the shot.
|
|
2775
|
+
- persona => { systemPrompt?, model? } \u2014 give THIS shot its own role and/or model
|
|
2776
|
+
(multi-agent strategies: a researcher shot then an engineer shot, a panel of k
|
|
2777
|
+
personas over one budget). On a fresh shot the systemPrompt replaces the task's; on
|
|
2778
|
+
a carried conversation it arrives as a hand-off message. Same conserved budget.
|
|
2779
|
+
- tools => string[] \u2014 restrict THIS shot to a subset of the task's tools by
|
|
2780
|
+
name (focus an explore shot on read-only tools, an execute shot on write tools).
|
|
2781
|
+
Restriction-only; unknown names make the shot fail. ALWAYS select from
|
|
2782
|
+
await listTools(handle) \u2014 never hardcode. Omitted => the shot sees every tool.
|
|
2783
|
+
ShotResult = { messages, score (0..1 on the task's check), passes, total, completions, toolErrors }
|
|
2784
|
+
Returns null if the attempt failed infra-wise.
|
|
2785
|
+
|
|
2786
|
+
critique(messages): Promise<string | null>
|
|
2787
|
+
A firewalled trace-analyst reads the attempt's trajectory and returns ONE corrective
|
|
2788
|
+
instruction (or null when it judges the work complete). Costs ~1 completion.
|
|
2789
|
+
|
|
2790
|
+
consult(messages, instruction): Promise<string | null>
|
|
2791
|
+
The RAW analyst channel: the same firewalled critic answers YOUR instruction over the
|
|
2792
|
+
trajectory verbatim (no reformatting) \u2014 use it when you need a specific reply format
|
|
2793
|
+
(a decision, a prediction). Costs ~1 completion.
|
|
2794
|
+
|
|
2795
|
+
surface.open(task) / surface.close(handle)
|
|
2796
|
+
Open a persistent artifact you manage yourself (remember to close in a finally).
|
|
2797
|
+
close is idempotent \u2014 closing an already-closed handle is a safe no-op.
|
|
2798
|
+
|
|
2799
|
+
listTools(handle): Promise<Array<{ name, description? }>>
|
|
2800
|
+
The tools THIS task actually offers. TOOL SETS VARY PER TASK \u2014 if you restrict a
|
|
2801
|
+
shot with \`tools\`, you MUST pick names from await listTools(handle); hardcoding
|
|
2802
|
+
names from an example kills your shots on every task whose tools differ.
|
|
2803
|
+
|
|
2804
|
+
Rules:
|
|
2805
|
+
- ALWAYS await every shot/critique/surface call \u2014 a floating promise that rejects
|
|
2806
|
+
crashes the whole benchmark run.
|
|
2807
|
+
- Stay within ~budget total shots; every shot/critique spends from a conserved pool.
|
|
2808
|
+
- For a FRESH attempt OMIT \`messages\` entirely (never pass \`[]\` \u2014 an empty array is a
|
|
2809
|
+
fresh conversation too, but be explicit). To CONTINUE, pass the previous
|
|
2810
|
+
ShotResult.messages unchanged.
|
|
2811
|
+
- Return { score, resolved, completions, progression, shots } \u2014 score = the BEST checkpoint
|
|
2812
|
+
you reached (keep-best, never final-state), progression = score after each shot.
|
|
2813
|
+
- The module must be EXACTLY this shape (no other imports, no commentary outside code):
|
|
2814
|
+
|
|
2815
|
+
import { defineStrategy } from '@tangle-network/agent-runtime/loops'
|
|
2816
|
+
export default defineStrategy('your-strategy-name', async ({ surface, task, budget, shot, critique, listTools }) => {
|
|
2817
|
+
// your composition (listTools comes from the destructured context \u2014 it is NOT a global)
|
|
2818
|
+
})
|
|
2819
|
+
`;
|
|
2820
|
+
function assertStrategyContract(code) {
|
|
2821
|
+
const allowedImport = /^\s*import\s+\{[^}]*\}\s+from\s+['"]@tangle-network\/agent-runtime\/loops['"]/;
|
|
2822
|
+
for (const line of code.split("\n")) {
|
|
2823
|
+
if (/^\s*import\s/.test(line) && !allowedImport.test(line)) {
|
|
2824
|
+
throw new Error(`authored code rejected: foreign import \u2014 ${line.trim().slice(0, 120)}`);
|
|
2825
|
+
}
|
|
2826
|
+
}
|
|
2827
|
+
const banned = [
|
|
2828
|
+
[/\brequire\s*\(/, "require()"],
|
|
2829
|
+
[/\bimport\s*\(/, "dynamic import()"],
|
|
2830
|
+
[/\beval\s*\(/, "eval()"],
|
|
2831
|
+
[/new\s+Function\s*\(/, "new Function()"],
|
|
2832
|
+
[/\bprocess\s*[.[]/, "process access"],
|
|
2833
|
+
[/\bglobalThis\s*[.[]/, "globalThis access"],
|
|
2834
|
+
[/\bfetch\s*\(/, "network access"],
|
|
2835
|
+
[/child_process|node:fs|node:net|node:http|worker_threads/, "node builtin access"]
|
|
2836
|
+
];
|
|
2837
|
+
for (const [re, what] of banned) {
|
|
2838
|
+
if (re.test(code)) throw new Error(`authored code rejected: ${what}`);
|
|
2839
|
+
}
|
|
2840
|
+
}
|
|
2841
|
+
async function requestAuthoredCode(opts, model) {
|
|
2842
|
+
const res = await opts.chat.chat(
|
|
2843
|
+
{
|
|
2844
|
+
...model ? { model } : {},
|
|
2845
|
+
...opts.temperature !== void 0 ? { temperature: opts.temperature } : {},
|
|
2846
|
+
...opts.maxTokens !== void 0 ? { maxTokens: opts.maxTokens } : {},
|
|
2847
|
+
messages: [
|
|
2848
|
+
{
|
|
2849
|
+
role: "system",
|
|
2850
|
+
content: "You are a senior engineer authoring optimization strategies for agent loops. Output exactly one fenced ```ts code block and nothing else."
|
|
2851
|
+
},
|
|
2852
|
+
{
|
|
2853
|
+
role: "user",
|
|
2854
|
+
content: `${opts.contract ?? strategyAuthorContract}
|
|
2855
|
+
|
|
2856
|
+
BASELINE RESULTS on the "${opts.environmentName}" environment (budget=${opts.budget}):
|
|
2857
|
+
${opts.lossesJson}
|
|
2858
|
+
|
|
2859
|
+
Author ONE new strategy that you expect to beat the baselines on THIS environment at the same budget. Use the losses to target the observed failure mode. Output only the module code block.`
|
|
2860
|
+
}
|
|
2861
|
+
]
|
|
2862
|
+
},
|
|
2863
|
+
{ ...opts.signal ? { signal: opts.signal } : {} }
|
|
2864
|
+
);
|
|
2865
|
+
const match = res.content.match(/```(?:ts|typescript)?\s*\n([\s\S]*?)```/);
|
|
2866
|
+
if (!match?.[1]) {
|
|
2867
|
+
throw new Error(
|
|
2868
|
+
`authorStrategy: no code block in the author's reply (model=${model ?? "default"}): ${res.content.slice(0, 300)}`
|
|
2869
|
+
);
|
|
2870
|
+
}
|
|
2871
|
+
return match[1];
|
|
2872
|
+
}
|
|
2873
|
+
async function authorStrategy(opts) {
|
|
2874
|
+
let code;
|
|
2875
|
+
try {
|
|
2876
|
+
code = await requestAuthoredCode(opts, opts.model);
|
|
2877
|
+
} catch (primaryError) {
|
|
2878
|
+
if (!opts.fallbackModel) throw primaryError;
|
|
2879
|
+
code = await requestAuthoredCode(opts, opts.fallbackModel);
|
|
2880
|
+
}
|
|
2881
|
+
assertStrategyContract(code);
|
|
2882
|
+
mkdirSync(opts.outDir, { recursive: true });
|
|
2883
|
+
const file = join(opts.outDir, `authored-${Date.now()}.mts`);
|
|
2884
|
+
writeFileSync(file, code);
|
|
2885
|
+
const mod = await import(`file://${file}`);
|
|
2886
|
+
if (!mod.default || typeof mod.default.driver !== "function" || !mod.default.name) {
|
|
2887
|
+
throw new Error(`authorStrategy: ${file} does not export a default Strategy`);
|
|
2888
|
+
}
|
|
2889
|
+
return { strategy: mod.default, file, code };
|
|
2890
|
+
}
|
|
2891
|
+
|
|
2892
|
+
// src/runtime/strategy-evolution.ts
|
|
2893
|
+
import { existsSync, readFileSync, writeFileSync as writeFileSync2 } from "fs";
|
|
2894
|
+
import { gzipSync } from "zlib";
|
|
2895
|
+
function discriminatingMeans(report, fieldOrder) {
|
|
2896
|
+
const rows = report.perTask.filter((r) => {
|
|
2897
|
+
if (!r.cells) return false;
|
|
2898
|
+
const scores = fieldOrder.map((n) => r.cells?.[n]?.score).filter((s) => s !== void 0);
|
|
2899
|
+
if (scores.length < fieldOrder.length) return false;
|
|
2900
|
+
return Math.max(...scores) - Math.min(...scores) > 0;
|
|
2901
|
+
});
|
|
2902
|
+
if (rows.length === 0) return null;
|
|
2903
|
+
const out = {};
|
|
2904
|
+
for (const name of fieldOrder) {
|
|
2905
|
+
const cells = rows.map((r) => r.cells?.[name]).filter((c) => !!c);
|
|
2906
|
+
out[name] = {
|
|
2907
|
+
score: cells.reduce((s, c) => s + c.score, 0) / cells.length,
|
|
2908
|
+
usd: cells.reduce((s, c) => s + c.usd, 0) / cells.length
|
|
2909
|
+
};
|
|
2910
|
+
}
|
|
2911
|
+
return out;
|
|
2912
|
+
}
|
|
2913
|
+
function pickChampion(means, fieldOrder, policy, epsilon) {
|
|
2914
|
+
const entries = fieldOrder.map((name) => ({ name, summary: means[name] })).filter((e) => !!e.summary);
|
|
2915
|
+
if (entries.length === 0)
|
|
2916
|
+
throw new Error("pickChampion: the means table carries none of the field strategies");
|
|
2917
|
+
const best = Math.max(...entries.map((e) => e.summary.score));
|
|
2918
|
+
const pick = policy === "score" ? entries.find((e) => e.summary.score === best) : entries.filter((e) => e.summary.score >= best - epsilon).sort((a, b) => a.summary.usd - b.summary.usd || b.summary.score - a.summary.score)[0];
|
|
2919
|
+
if (!pick) throw new Error("pickChampion: empty pick (unreachable)");
|
|
2920
|
+
return { name: pick.name, score: pick.summary.score, usd: pick.summary.usd };
|
|
2921
|
+
}
|
|
2922
|
+
function selectChampion(report, fieldOrder, policy, epsilon) {
|
|
2923
|
+
return pickChampion(report.perStrategy, fieldOrder, policy, epsilon);
|
|
2924
|
+
}
|
|
2925
|
+
var fieldSummary = (archive) => archive.map(
|
|
2926
|
+
(n) => `- ${n.name} (${n.source}, gen ${n.generation}, last score ${(n.score * 100).toFixed(0)}%)`
|
|
2927
|
+
).join("\n");
|
|
2928
|
+
var compactLosses = (report, detail) => {
|
|
2929
|
+
const r2 = (x) => Math.round(x * 100) / 100;
|
|
2930
|
+
const rows = report.perTask.map(
|
|
2931
|
+
(row) => row.cells ? {
|
|
2932
|
+
task: row.taskId,
|
|
2933
|
+
...row.errors ? {
|
|
2934
|
+
errors: Object.fromEntries(
|
|
2935
|
+
Object.entries(row.errors).map(([n, msg]) => [n, msg.slice(0, 100)])
|
|
2936
|
+
)
|
|
2937
|
+
} : {},
|
|
2938
|
+
cells: Object.fromEntries(
|
|
2939
|
+
Object.entries(row.cells).map(([name, c]) => [
|
|
2940
|
+
name,
|
|
2941
|
+
// 'binary' is the leakage-bounded channel: the author learns pass/fail per
|
|
2942
|
+
// task and nothing else — the per-generation leak from the evaluation data
|
|
2943
|
+
// is capped at one bit per cell (arXiv:2606.11045 measured that exploration
|
|
2944
|
+
// survives this; whether AUTHORING does is the E1-coarse A/B).
|
|
2945
|
+
detail === "binary" ? { resolved: c.resolved, usd: Math.round(c.usd * 1e4) / 1e4 } : {
|
|
2946
|
+
score: r2(c.score),
|
|
2947
|
+
resolved: c.resolved,
|
|
2948
|
+
usd: Math.round(c.usd * 1e4) / 1e4,
|
|
2949
|
+
progression: (c.progression ?? []).map(r2)
|
|
2950
|
+
}
|
|
2951
|
+
])
|
|
2952
|
+
)
|
|
2953
|
+
} : { task: row.taskId, error: row.error?.slice(0, 80) }
|
|
2954
|
+
);
|
|
2955
|
+
return JSON.stringify(rows).slice(0, 12e3);
|
|
2956
|
+
};
|
|
2957
|
+
function renameStrategy(orig, unique) {
|
|
2958
|
+
if (orig.name === unique) return orig;
|
|
2959
|
+
return {
|
|
2960
|
+
name: unique,
|
|
2961
|
+
driver: (s, t, o, b) => {
|
|
2962
|
+
const agent = orig.driver(s, t, o, b);
|
|
2963
|
+
return {
|
|
2964
|
+
...agent,
|
|
2965
|
+
name: unique,
|
|
2966
|
+
act: async (task, scope) => {
|
|
2967
|
+
const out = await agent.act(task, scope);
|
|
2968
|
+
if (out.kind !== "done") return out;
|
|
2969
|
+
const deliverable = { ...out.deliverable, mode: unique };
|
|
2970
|
+
return { ...out, deliverable };
|
|
2971
|
+
}
|
|
2972
|
+
};
|
|
2973
|
+
}
|
|
2974
|
+
};
|
|
2975
|
+
}
|
|
2976
|
+
async function runStrategyEvolution(cfg) {
|
|
2977
|
+
const budget = cfg.budget ?? 3;
|
|
2978
|
+
const concurrency = cfg.concurrency ?? 3;
|
|
2979
|
+
const generations = cfg.generations ?? 2;
|
|
2980
|
+
const populationSize = cfg.populationSize ?? 2;
|
|
2981
|
+
const baselines = cfg.baselines ?? [sample, refine, sampleThenRefine];
|
|
2982
|
+
const policy = cfg.champion ?? "costAware";
|
|
2983
|
+
const epsilon = cfg.championEpsilon ?? (cfg.objective === "cost" ? cfg.scoreTolerance ?? 0.05 : 0.01);
|
|
2984
|
+
const byName = new Map(baselines.map((s) => [s.name, s]));
|
|
2985
|
+
const codeByName = /* @__PURE__ */ new Map();
|
|
2986
|
+
const fingerprint = {
|
|
2987
|
+
trainN: cfg.trainN,
|
|
2988
|
+
holdoutN: cfg.holdoutN,
|
|
2989
|
+
budget,
|
|
2990
|
+
generations,
|
|
2991
|
+
populationSize
|
|
2992
|
+
};
|
|
2993
|
+
let ckpt;
|
|
2994
|
+
if (cfg.checkpoint?.resume && existsSync(cfg.checkpoint.path)) {
|
|
2995
|
+
const raw = JSON.parse(readFileSync(cfg.checkpoint.path, "utf8"));
|
|
2996
|
+
if (JSON.stringify(raw.fingerprint) !== JSON.stringify(fingerprint)) {
|
|
2997
|
+
throw new Error(
|
|
2998
|
+
`evolution resume: checkpoint design mismatch \u2014 checkpoint ${JSON.stringify(raw.fingerprint)} vs config ${JSON.stringify(fingerprint)}; delete ${cfg.checkpoint.path} or match the config`
|
|
2999
|
+
);
|
|
3000
|
+
}
|
|
3001
|
+
ckpt = raw;
|
|
3002
|
+
}
|
|
3003
|
+
const save = (state) => {
|
|
3004
|
+
if (cfg.checkpoint)
|
|
3005
|
+
writeFileSync2(cfg.checkpoint.path, JSON.stringify({ ...state, fingerprint }, null, 1));
|
|
3006
|
+
};
|
|
3007
|
+
const bench = async (phase, tasks, strategies) => {
|
|
3008
|
+
await cfg.onPhase?.(phase);
|
|
3009
|
+
return runBenchmark({
|
|
3010
|
+
environment: cfg.environment,
|
|
3011
|
+
tasks,
|
|
3012
|
+
worker: cfg.worker,
|
|
3013
|
+
strategies,
|
|
3014
|
+
budget,
|
|
3015
|
+
concurrency,
|
|
3016
|
+
...cfg.onTask ? { onTask: (row, done, total) => cfg.onTask?.(phase, row, done, total) } : {},
|
|
3017
|
+
...cfg.hooks ? { hooks: cfg.hooks } : {}
|
|
3018
|
+
});
|
|
3019
|
+
};
|
|
3020
|
+
const train = await cfg.tasks(0, cfg.trainN);
|
|
3021
|
+
const probeTask = train[0];
|
|
3022
|
+
if (!probeTask) throw new Error("runStrategyEvolution: empty train slice");
|
|
3023
|
+
const probe = await cfg.environment.open(probeTask);
|
|
3024
|
+
let toolCatalog;
|
|
3025
|
+
try {
|
|
3026
|
+
const tools = await cfg.environment.tools(probeTask, probe);
|
|
3027
|
+
toolCatalog = tools.map(
|
|
3028
|
+
(t) => `- ${t.function.name}${t.function.description ? ` \u2014 ${t.function.description.slice(0, 120)}` : ""}`
|
|
3029
|
+
).join("\n");
|
|
3030
|
+
} finally {
|
|
3031
|
+
await cfg.environment.close(probe);
|
|
3032
|
+
}
|
|
3033
|
+
const gen0 = ckpt?.gen0 ?? await bench("gen0", train, baselines);
|
|
3034
|
+
const archive = ckpt?.archive ? [...ckpt.archive] : baselines.map((s) => ({
|
|
3035
|
+
name: s.name,
|
|
3036
|
+
source: "baseline",
|
|
3037
|
+
generation: 0,
|
|
3038
|
+
score: gen0.perStrategy[s.name]?.score ?? 0,
|
|
3039
|
+
usd: gen0.perStrategy[s.name]?.usd ?? 0
|
|
3040
|
+
}));
|
|
3041
|
+
const gen0Champion = ckpt?.gen0Champion ?? selectChampion(
|
|
3042
|
+
gen0,
|
|
3043
|
+
baselines.map((s) => s.name),
|
|
3044
|
+
policy,
|
|
3045
|
+
epsilon
|
|
3046
|
+
);
|
|
3047
|
+
const generationRows = ckpt?.generations ? [...ckpt.generations] : [];
|
|
3048
|
+
const trajectory = ckpt?.trajectory ? [...ckpt.trajectory] : [
|
|
3049
|
+
{
|
|
3050
|
+
generation: 0,
|
|
3051
|
+
champion: gen0Champion.name,
|
|
3052
|
+
score: gen0Champion.score,
|
|
3053
|
+
usd: gen0Champion.usd
|
|
3054
|
+
}
|
|
3055
|
+
];
|
|
3056
|
+
for (const row of generationRows) {
|
|
3057
|
+
for (const c of row.candidates) {
|
|
3058
|
+
if (!c.file || c.error) continue;
|
|
3059
|
+
const mod = await import(`file://${c.file}`);
|
|
3060
|
+
if (!mod.default || typeof mod.default.driver !== "function") {
|
|
3061
|
+
throw new Error(
|
|
3062
|
+
`evolution resume: ${c.file} no longer exports a Strategy \u2014 cannot restore "${c.name}"`
|
|
3063
|
+
);
|
|
3064
|
+
}
|
|
3065
|
+
byName.set(c.name, renameStrategy(mod.default, c.name));
|
|
3066
|
+
codeByName.set(c.name, readFileSync(c.file, "utf8"));
|
|
3067
|
+
}
|
|
3068
|
+
}
|
|
3069
|
+
let authoredOk = generationRows.reduce(
|
|
3070
|
+
(n, row) => n + row.candidates.filter((c) => !c.error).length,
|
|
3071
|
+
0
|
|
3072
|
+
);
|
|
3073
|
+
const lastRow = generationRows[generationRows.length - 1];
|
|
3074
|
+
let incumbent = lastRow ? lastRow.champion : gen0Champion;
|
|
3075
|
+
let latestReport = lastRow ? lastRow.report : gen0;
|
|
3076
|
+
if (!ckpt) save({ gen0, gen0Champion, generations: generationRows, archive, trajectory });
|
|
3077
|
+
for (let g = generationRows.length + 1; g <= generations; g += 1) {
|
|
3078
|
+
const lossesJson = compactLosses(latestReport, cfg.lossesDetail ?? "exact");
|
|
3079
|
+
const candidates = [];
|
|
3080
|
+
const newStrategies = [];
|
|
3081
|
+
for (let i = 0; i < populationSize; i += 1) {
|
|
3082
|
+
const objectiveNote = cfg.objective === "cost" ? `
|
|
3083
|
+
|
|
3084
|
+
YOUR OBJECTIVE: match or exceed the incumbent's SCORE while spending LESS (the losses include usd per task). Promotion requires proven score non-inferiority PLUS significant cost savings \u2014 a strategy that ties the score at half the cost WINS; a cheaper strategy that loses score by more than ${((cfg.scoreTolerance ?? 0.05) * 100).toFixed(0)}pp LOSES.` : "";
|
|
3085
|
+
const contract = `${strategyAuthorContract}${objectiveNote}
|
|
3086
|
+
|
|
3087
|
+
EXAMPLE TOOLS FROM ONE TASK (tool sets VARY per task on this domain \u2014 a strategy MUST select tool names from await listTools(handle) at runtime; hardcoding these example names will zero your score on most tasks):
|
|
3088
|
+
${toolCatalog}
|
|
3089
|
+
|
|
3090
|
+
STRATEGIES ALREADY IN THE TOURNAMENT (author something MEANINGFULLY different \u2014 a new composition, not a rename):
|
|
3091
|
+
${fieldSummary(archive)}
|
|
3092
|
+
|
|
3093
|
+
You are authoring candidate ${i + 1} of ${populationSize} this generation; explore a distinct region of the strategy space from your siblings.`;
|
|
3094
|
+
try {
|
|
3095
|
+
const authored = await authorStrategy({
|
|
3096
|
+
chat: cfg.author.chat,
|
|
3097
|
+
...cfg.author.model ? { model: cfg.author.model } : {},
|
|
3098
|
+
...cfg.author.fallbackModel ? { fallbackModel: cfg.author.fallbackModel } : {},
|
|
3099
|
+
...cfg.author.temperature !== void 0 ? { temperature: cfg.author.temperature } : {},
|
|
3100
|
+
...cfg.author.maxTokens !== void 0 ? { maxTokens: cfg.author.maxTokens } : {},
|
|
3101
|
+
contract,
|
|
3102
|
+
environmentName: cfg.environment.name,
|
|
3103
|
+
lossesJson,
|
|
3104
|
+
budget,
|
|
3105
|
+
outDir: cfg.outDir
|
|
3106
|
+
});
|
|
3107
|
+
const unique = byName.has(authored.strategy.name) ? `${authored.strategy.name}-g${g}c${i + 1}` : authored.strategy.name;
|
|
3108
|
+
const strategy = renameStrategy(authored.strategy, unique);
|
|
3109
|
+
byName.set(unique, strategy);
|
|
3110
|
+
codeByName.set(unique, authored.code);
|
|
3111
|
+
newStrategies.push(strategy);
|
|
3112
|
+
archive.push({
|
|
3113
|
+
name: unique,
|
|
3114
|
+
source: "authored",
|
|
3115
|
+
generation: g,
|
|
3116
|
+
parent: incumbent.name,
|
|
3117
|
+
gzipBits: gzipSync(Buffer.from(authored.code)).length * 8,
|
|
3118
|
+
file: authored.file,
|
|
3119
|
+
score: 0,
|
|
3120
|
+
usd: 0
|
|
3121
|
+
});
|
|
3122
|
+
candidates.push({
|
|
3123
|
+
name: unique,
|
|
3124
|
+
file: authored.file,
|
|
3125
|
+
gzipBits: gzipSync(Buffer.from(authored.code)).length * 8,
|
|
3126
|
+
codeChars: authored.code.length
|
|
3127
|
+
});
|
|
3128
|
+
authoredOk += 1;
|
|
3129
|
+
} catch (e) {
|
|
3130
|
+
candidates.push({
|
|
3131
|
+
name: `(author-failed g${g}c${i + 1})`,
|
|
3132
|
+
error: e instanceof Error ? e.message.slice(0, 300) : String(e)
|
|
3133
|
+
});
|
|
3134
|
+
}
|
|
3135
|
+
}
|
|
3136
|
+
const incumbentStrategy = byName.get(incumbent.name);
|
|
3137
|
+
if (!incumbentStrategy)
|
|
3138
|
+
throw new Error(`evolution: incumbent "${incumbent.name}" missing from the field`);
|
|
3139
|
+
const field = [incumbentStrategy, ...newStrategies];
|
|
3140
|
+
const report = await bench(`gen${g}`, train, field);
|
|
3141
|
+
for (const node of archive) {
|
|
3142
|
+
const cell = report.perStrategy[node.name];
|
|
3143
|
+
if (cell) {
|
|
3144
|
+
node.score = cell.score;
|
|
3145
|
+
node.usd = cell.usd;
|
|
3146
|
+
}
|
|
3147
|
+
}
|
|
3148
|
+
const fieldNames = field.map((s) => s.name);
|
|
3149
|
+
const means = cfg.band ? discriminatingMeans(report, fieldNames) ?? report.perStrategy : report.perStrategy;
|
|
3150
|
+
const champion = pickChampion(means, fieldNames, policy, epsilon);
|
|
3151
|
+
generationRows.push({ generation: g, candidates, report, champion });
|
|
3152
|
+
trajectory.push({
|
|
3153
|
+
generation: g,
|
|
3154
|
+
champion: champion.name,
|
|
3155
|
+
score: champion.score,
|
|
3156
|
+
usd: champion.usd
|
|
3157
|
+
});
|
|
3158
|
+
incumbent = champion;
|
|
3159
|
+
latestReport = report;
|
|
3160
|
+
save({ gen0, gen0Champion, generations: generationRows, archive, trajectory });
|
|
3161
|
+
}
|
|
3162
|
+
if (authoredOk === 0) {
|
|
3163
|
+
throw new Error(
|
|
3164
|
+
"runStrategyEvolution: every author attempt failed across all generations \u2014 no search happened; see the candidates[].error entries"
|
|
3165
|
+
);
|
|
3166
|
+
}
|
|
3167
|
+
const holdoutOffset = cfg.trainN + (cfg.holdoutOffset ?? 0);
|
|
3168
|
+
let holdoutTasks = [];
|
|
3169
|
+
let bandInfo;
|
|
3170
|
+
if (ckpt?.holdout && ckpt.verdict) {
|
|
3171
|
+
bandInfo = ckpt.band;
|
|
3172
|
+
if (cfg.reproducerCheck && codeByName.has(incumbent.name)) {
|
|
3173
|
+
const pool2 = await cfg.tasks(holdoutOffset, cfg.band?.holdoutPoolN ?? cfg.holdoutN);
|
|
3174
|
+
const gateIds = new Set(ckpt.holdout.perTask.map((r) => r.taskId));
|
|
3175
|
+
holdoutTasks = pool2.filter((t) => gateIds.has(t.id));
|
|
3176
|
+
}
|
|
3177
|
+
} else if (cfg.band) {
|
|
3178
|
+
const maxRef = cfg.band.maxRefScore ?? 0.99;
|
|
3179
|
+
const reference = baselines[0];
|
|
3180
|
+
if (!reference)
|
|
3181
|
+
throw new Error("evolution band: baselines[0] required as the screening reference");
|
|
3182
|
+
const pool2 = await cfg.tasks(holdoutOffset, cfg.band.holdoutPoolN);
|
|
3183
|
+
const screen = await bench("band-screen", pool2, [reference]);
|
|
3184
|
+
const refScores = screen.perTask.filter((r) => r.cells?.[reference.name]).map((r) => ({ taskId: r.taskId, score: r.cells?.[reference.name]?.score ?? 0 }));
|
|
3185
|
+
const inBandIds = new Set(refScores.filter((r) => r.score <= maxRef).map((r) => r.taskId));
|
|
3186
|
+
const kept = pool2.filter((t) => inBandIds.has(t.id));
|
|
3187
|
+
if (kept.length < cfg.holdoutN) {
|
|
3188
|
+
throw new Error(
|
|
3189
|
+
`evolution band: only ${kept.length}/${cfg.holdoutN} holdout tasks have headroom (pool ${cfg.band.holdoutPoolN}, reference "${reference.name}" \u2264 ${maxRef}) \u2014 widen holdoutPoolN or raise maxRefScore`
|
|
3190
|
+
);
|
|
3191
|
+
}
|
|
3192
|
+
holdoutTasks = kept.slice(0, cfg.holdoutN);
|
|
3193
|
+
bandInfo = { screened: refScores.length, inBand: kept.length, refScores };
|
|
3194
|
+
} else {
|
|
3195
|
+
holdoutTasks = await cfg.tasks(holdoutOffset, cfg.holdoutN);
|
|
3196
|
+
}
|
|
3197
|
+
let holdout;
|
|
3198
|
+
let verdict;
|
|
3199
|
+
if (ckpt?.holdout && ckpt.verdict) {
|
|
3200
|
+
holdout = ckpt.holdout;
|
|
3201
|
+
verdict = ckpt.verdict;
|
|
3202
|
+
} else {
|
|
3203
|
+
const finalists = [.../* @__PURE__ */ new Set([gen0Champion.name, incumbent.name])].map((n) => byName.get(n)).filter((s) => !!s);
|
|
3204
|
+
holdout = await bench("holdout", holdoutTasks, finalists);
|
|
3205
|
+
verdict = promotionGate({
|
|
3206
|
+
report: holdout,
|
|
3207
|
+
incumbent: gen0Champion.name,
|
|
3208
|
+
candidate: incumbent.name,
|
|
3209
|
+
...cfg.objective === "cost" ? {
|
|
3210
|
+
mode: "non-inferiority",
|
|
3211
|
+
...cfg.scoreTolerance !== void 0 ? { scoreTolerance: cfg.scoreTolerance } : {}
|
|
3212
|
+
} : {},
|
|
3213
|
+
...cfg.minPairedTasks !== void 0 ? { minPairedTasks: cfg.minPairedTasks } : {}
|
|
3214
|
+
});
|
|
3215
|
+
save({
|
|
3216
|
+
gen0,
|
|
3217
|
+
gen0Champion,
|
|
3218
|
+
generations: generationRows,
|
|
3219
|
+
archive,
|
|
3220
|
+
trajectory,
|
|
3221
|
+
holdout,
|
|
3222
|
+
verdict,
|
|
3223
|
+
...bandInfo ? { band: bandInfo } : {}
|
|
3224
|
+
});
|
|
3225
|
+
}
|
|
3226
|
+
let reproduction;
|
|
3227
|
+
const championCode = codeByName.get(incumbent.name);
|
|
3228
|
+
if (cfg.reproducerCheck && championCode) {
|
|
3229
|
+
const words = cfg.reproducerCheck.summaryMaxWords ?? 64;
|
|
3230
|
+
const tolerance = cfg.reproducerCheck.tolerance ?? 0.05;
|
|
3231
|
+
const championHoldoutScore = holdout.perStrategy[incumbent.name]?.score ?? 0;
|
|
3232
|
+
try {
|
|
3233
|
+
const summaryRes = await cfg.author.chat.chat({
|
|
3234
|
+
...cfg.author.model ? { model: cfg.author.model } : {},
|
|
3235
|
+
temperature: 0.2,
|
|
3236
|
+
maxTokens: 512,
|
|
3237
|
+
messages: [
|
|
3238
|
+
{
|
|
3239
|
+
role: "system",
|
|
3240
|
+
content: `Summarize the optimization strategy implemented by this code in at most ${words} words. Describe the COMPOSITION (shots, critique, artifact handling, restarts, stopping) \u2014 not the code. Output only the summary.`
|
|
3241
|
+
},
|
|
3242
|
+
{ role: "user", content: championCode }
|
|
3243
|
+
]
|
|
3244
|
+
});
|
|
3245
|
+
const summary = summaryRes.content.trim();
|
|
3246
|
+
const reproduced = await authorStrategy({
|
|
3247
|
+
chat: cfg.author.chat,
|
|
3248
|
+
...cfg.author.model ? { model: cfg.author.model } : {},
|
|
3249
|
+
...cfg.author.fallbackModel ? { fallbackModel: cfg.author.fallbackModel } : {},
|
|
3250
|
+
...cfg.author.maxTokens !== void 0 ? { maxTokens: cfg.author.maxTokens } : {},
|
|
3251
|
+
temperature: 0.2,
|
|
3252
|
+
contract: `${strategyAuthorContract}
|
|
3253
|
+
|
|
3254
|
+
IMPLEMENT EXACTLY THIS STRATEGY (a colleague's description \u2014 do not invent a different approach):
|
|
3255
|
+
${summary}`,
|
|
3256
|
+
environmentName: cfg.environment.name,
|
|
3257
|
+
lossesJson: "[]",
|
|
3258
|
+
budget,
|
|
3259
|
+
outDir: cfg.outDir
|
|
3260
|
+
});
|
|
3261
|
+
const reproStrategy = {
|
|
3262
|
+
name: `${incumbent.name}-reproduced`,
|
|
3263
|
+
driver: reproduced.strategy.driver
|
|
3264
|
+
};
|
|
3265
|
+
const reproReport = await bench("reproduce", holdoutTasks, [reproStrategy]);
|
|
3266
|
+
const reproducedHoldoutScore = reproReport.perStrategy[reproStrategy.name]?.score ?? 0;
|
|
3267
|
+
reproduction = {
|
|
3268
|
+
summary,
|
|
3269
|
+
reproducedName: reproStrategy.name,
|
|
3270
|
+
file: reproduced.file,
|
|
3271
|
+
championHoldoutScore,
|
|
3272
|
+
reproducedHoldoutScore,
|
|
3273
|
+
gap: championHoldoutScore - reproducedHoldoutScore,
|
|
3274
|
+
reproducible: reproducedHoldoutScore >= championHoldoutScore - tolerance
|
|
3275
|
+
};
|
|
3276
|
+
} catch (e) {
|
|
3277
|
+
reproduction = {
|
|
3278
|
+
summary: "",
|
|
3279
|
+
reproducedName: "",
|
|
3280
|
+
championHoldoutScore,
|
|
3281
|
+
reproducedHoldoutScore: 0,
|
|
3282
|
+
gap: championHoldoutScore,
|
|
3283
|
+
reproducible: false,
|
|
3284
|
+
error: e instanceof Error ? e.message.slice(0, 300) : String(e)
|
|
3285
|
+
};
|
|
3286
|
+
}
|
|
3287
|
+
}
|
|
3288
|
+
return {
|
|
3289
|
+
gen0,
|
|
3290
|
+
gen0Champion,
|
|
3291
|
+
generations: generationRows,
|
|
3292
|
+
archive,
|
|
3293
|
+
finalChampion: incumbent,
|
|
3294
|
+
holdout,
|
|
3295
|
+
verdict,
|
|
3296
|
+
...bandInfo ? { band: bandInfo } : {},
|
|
3297
|
+
...reproduction ? { reproduction } : {},
|
|
3298
|
+
trajectory
|
|
3299
|
+
};
|
|
3300
|
+
}
|
|
3301
|
+
|
|
3302
|
+
// src/runtime/supervise/detector-monitor.ts
|
|
3303
|
+
import {
|
|
3304
|
+
argHash,
|
|
3305
|
+
errorStreakDetector,
|
|
3306
|
+
observeAll,
|
|
3307
|
+
repeatedActionDetector
|
|
3308
|
+
} from "@tangle-network/agent-eval";
|
|
3309
|
+
function defaultToolDetectors() {
|
|
3310
|
+
return [repeatedActionDetector({ maxRepeated: 3 }), errorStreakDetector({ maxErrors: 3 })];
|
|
3311
|
+
}
|
|
3312
|
+
function watchTrace(source, opts = {}) {
|
|
3313
|
+
const detectors = opts.detectors ?? defaultToolDetectors();
|
|
3314
|
+
return source.onSpan((span) => {
|
|
3315
|
+
let fingerprint;
|
|
3316
|
+
try {
|
|
3317
|
+
fingerprint = `${span.toolName}|${argHash(span.args)}`;
|
|
3318
|
+
} catch {
|
|
3319
|
+
fingerprint = `${span.toolName}|<unhashable>`;
|
|
3320
|
+
}
|
|
3321
|
+
const signals = observeAll(detectors, {
|
|
3322
|
+
actionFingerprint: fingerprint,
|
|
3323
|
+
...span.status ? { status: span.status } : {},
|
|
3324
|
+
label: span.toolName
|
|
3325
|
+
});
|
|
3326
|
+
for (const s of signals) void opts.onSignal?.(s, span);
|
|
3327
|
+
});
|
|
3328
|
+
}
|
|
3329
|
+
|
|
3330
|
+
// src/runtime/supervise/patch-checks.ts
|
|
3331
|
+
var DEFAULT_MAX_DIFF_LINES = 400;
|
|
3332
|
+
var secretPathPattern = /(^|\/)(\.env(\.|$)|.*\.(pem|key|p12|pfx|keystore|wallet)|id_rsa|id_ed25519|secrets?\.json|credentials?\.json)$/i;
|
|
3333
|
+
function touchedPathsFromPatch(patch) {
|
|
3334
|
+
const out = /* @__PURE__ */ new Set();
|
|
3335
|
+
for (const line of patch.split(/\r?\n/)) {
|
|
3336
|
+
if (line.startsWith("+++ ") || line.startsWith("--- ")) {
|
|
3337
|
+
const rest = line.slice(4).trim();
|
|
3338
|
+
if (rest === "/dev/null") continue;
|
|
3339
|
+
const stripped = rest.startsWith("a/") || rest.startsWith("b/") ? rest.slice(2) : rest;
|
|
3340
|
+
out.add(stripped);
|
|
3341
|
+
}
|
|
3342
|
+
}
|
|
3343
|
+
return [...out];
|
|
3344
|
+
}
|
|
3345
|
+
function countDiffLines(patch) {
|
|
3346
|
+
let count = 0;
|
|
3347
|
+
for (const line of patch.split(/\r?\n/)) {
|
|
3348
|
+
if ((line.startsWith("+") || line.startsWith("-")) && !line.startsWith("+++") && !line.startsWith("---")) {
|
|
3349
|
+
count += 1;
|
|
3350
|
+
}
|
|
3351
|
+
}
|
|
3352
|
+
return count;
|
|
3353
|
+
}
|
|
3354
|
+
function runCoderChecks(input, constraints = {}) {
|
|
3355
|
+
const maxDiff = constraints.maxDiffLines ?? DEFAULT_MAX_DIFF_LINES;
|
|
3356
|
+
const forbidden = constraints.forbiddenPaths ?? [];
|
|
3357
|
+
const scores = {};
|
|
3358
|
+
const notes = [];
|
|
3359
|
+
let pass = true;
|
|
3360
|
+
const touched = touchedPathsFromPatch(input.patch);
|
|
3361
|
+
if (touched.length === 0 || input.patch.trim().length === 0) {
|
|
3362
|
+
pass = false;
|
|
3363
|
+
scores.nonEmpty = 0;
|
|
3364
|
+
notes.push("empty patch \u2014 no files changed");
|
|
3365
|
+
} else {
|
|
3366
|
+
scores.nonEmpty = 1;
|
|
3367
|
+
}
|
|
3368
|
+
const touchedSecrets = touched.filter((p) => secretPathPattern.test(p));
|
|
3369
|
+
if (touchedSecrets.length > 0) {
|
|
3370
|
+
pass = false;
|
|
3371
|
+
scores.noSecrets = 0;
|
|
3372
|
+
notes.push(`touched secret-shaped paths: ${touchedSecrets.join(", ")}`);
|
|
3373
|
+
} else {
|
|
3374
|
+
scores.noSecrets = 1;
|
|
3375
|
+
}
|
|
3376
|
+
const touchedForbidden = forbidden.filter((path) => {
|
|
3377
|
+
const prefix = path.endsWith("/") ? path : `${path}/`;
|
|
3378
|
+
const exact = prefix.slice(0, -1);
|
|
3379
|
+
return touched.some((p) => p === exact || p.startsWith(prefix));
|
|
3380
|
+
});
|
|
3381
|
+
if (touchedForbidden.length > 0) {
|
|
3382
|
+
pass = false;
|
|
3383
|
+
scores.forbiddenPath = 0;
|
|
3384
|
+
notes.push(`touched forbidden paths: ${touchedForbidden.join(", ")}`);
|
|
3385
|
+
} else {
|
|
3386
|
+
scores.forbiddenPath = 1;
|
|
3387
|
+
}
|
|
3388
|
+
const diffLines = countDiffLines(input.patch);
|
|
3389
|
+
if (diffLines > maxDiff) {
|
|
3390
|
+
pass = false;
|
|
3391
|
+
scores.diffSize = 0;
|
|
3392
|
+
notes.push(`diff ${diffLines} lines exceeds cap ${maxDiff}`);
|
|
3393
|
+
} else {
|
|
3394
|
+
scores.diffSize = maxDiff === 0 ? 0 : Math.max(0, 1 - diffLines / maxDiff);
|
|
3395
|
+
}
|
|
3396
|
+
scores.tests = input.testsPassed ? 1 : 0;
|
|
3397
|
+
scores.typecheck = input.typecheckPassed ? 1 : 0;
|
|
3398
|
+
if (!input.testsPassed) {
|
|
3399
|
+
pass = false;
|
|
3400
|
+
notes.push("tests failed");
|
|
3401
|
+
}
|
|
3402
|
+
if (!input.typecheckPassed) {
|
|
3403
|
+
pass = false;
|
|
3404
|
+
notes.push("typecheck failed");
|
|
3405
|
+
}
|
|
3406
|
+
const score = 0.5 * scores.tests + 0.3 * scores.typecheck + 0.2 * scores.diffSize;
|
|
3407
|
+
const verdict = {
|
|
3408
|
+
valid: pass,
|
|
3409
|
+
score: Number.isFinite(score) ? score : 0,
|
|
3410
|
+
scores
|
|
3411
|
+
};
|
|
3412
|
+
if (notes.length > 0) verdict.notes = notes.join("; ");
|
|
3413
|
+
return verdict;
|
|
3414
|
+
}
|
|
3415
|
+
|
|
3416
|
+
// src/runtime/supervise/patch-deliverable.ts
|
|
3417
|
+
function patchDelivered(options = {}) {
|
|
3418
|
+
const require2 = new Set(options.require ?? []);
|
|
3419
|
+
const constraints = {
|
|
3420
|
+
...options.maxDiffLines !== void 0 ? { maxDiffLines: options.maxDiffLines } : {},
|
|
3421
|
+
...options.forbiddenPaths !== void 0 ? { forbiddenPaths: options.forbiddenPaths } : {}
|
|
3422
|
+
};
|
|
3423
|
+
return {
|
|
3424
|
+
describe: "patch: no-op/secret/forbidden/diff-size + required test/typecheck pass",
|
|
3425
|
+
check(artifact) {
|
|
3426
|
+
const input = {
|
|
3427
|
+
patch: artifact.patch,
|
|
3428
|
+
testsPassed: signalPass(artifact.checks?.tests?.passed, require2.has("tests")),
|
|
3429
|
+
typecheckPassed: signalPass(artifact.checks?.typecheck?.passed, require2.has("typecheck"))
|
|
3430
|
+
};
|
|
3431
|
+
return runCoderChecks(input, constraints).valid === true;
|
|
3432
|
+
}
|
|
3433
|
+
};
|
|
3434
|
+
}
|
|
3435
|
+
function signalPass(value, required) {
|
|
3436
|
+
if (value !== void 0) return value;
|
|
3437
|
+
return !required;
|
|
3438
|
+
}
|
|
3439
|
+
|
|
3440
|
+
// src/runtime/supervise/trace-source.ts
|
|
3441
|
+
function toToolSpan(input, runId, seq, at) {
|
|
3442
|
+
const startedAt = typeof input.startedAt === "number" ? input.startedAt : at;
|
|
3443
|
+
const endedAt = typeof input.endedAt === "number" ? input.endedAt : startedAt;
|
|
3444
|
+
return {
|
|
3445
|
+
spanId: `${runId}-t${seq}`,
|
|
3446
|
+
runId,
|
|
3447
|
+
kind: "tool",
|
|
3448
|
+
name: input.toolName,
|
|
3449
|
+
toolName: input.toolName,
|
|
3450
|
+
args: input.args,
|
|
3451
|
+
status: input.status ?? "ok",
|
|
3452
|
+
startedAt,
|
|
3453
|
+
endedAt,
|
|
3454
|
+
...input.result !== void 0 ? { result: input.result } : {}
|
|
3455
|
+
};
|
|
3456
|
+
}
|
|
3457
|
+
var obj = (v) => v && typeof v === "object" ? v : void 0;
|
|
3458
|
+
var str = (v) => typeof v === "string" && v ? v : void 0;
|
|
3459
|
+
var decodeOpencodePart = (raw) => {
|
|
3460
|
+
if (str(raw.type)?.toLowerCase() !== "tool" || !str(raw.tool)) return void 0;
|
|
3461
|
+
const part = raw;
|
|
3462
|
+
const state = obj(part.state);
|
|
3463
|
+
const status = state?.status;
|
|
3464
|
+
if (!status || status === "pending" || status === "running") return void 0;
|
|
3465
|
+
return {
|
|
3466
|
+
toolName: part.tool,
|
|
3467
|
+
args: state?.input ?? {},
|
|
3468
|
+
// ToolStateError carries 'error' OR 'failed' — the reverse-engineered decoder missed 'failed'.
|
|
3469
|
+
status: status === "error" || status === "failed" ? "error" : "ok",
|
|
3470
|
+
...str(part.callID) ? { callId: part.callID } : {}
|
|
3471
|
+
};
|
|
3472
|
+
};
|
|
3473
|
+
var decodeAnthropicPart = (p) => {
|
|
3474
|
+
if (str(p.type)?.toLowerCase() !== "tool_use") return void 0;
|
|
3475
|
+
const name = str(p.name) ?? str(p.tool);
|
|
3476
|
+
if (!name) return void 0;
|
|
3477
|
+
const id = str(p.id) ?? str(p.tool_use_id);
|
|
3478
|
+
return {
|
|
3479
|
+
toolName: name,
|
|
3480
|
+
args: p.input ?? {},
|
|
3481
|
+
...id ? { callId: id } : {}
|
|
3482
|
+
};
|
|
3483
|
+
};
|
|
3484
|
+
var decodeOpenAiPart = (p) => {
|
|
3485
|
+
const type = str(p.type)?.toLowerCase();
|
|
3486
|
+
const fn = obj(p.function);
|
|
3487
|
+
if (type !== "function" && type !== "tool_call") return void 0;
|
|
3488
|
+
const name = str(fn?.name) ?? str(p.name);
|
|
3489
|
+
if (!name) return void 0;
|
|
3490
|
+
const rawArgs = fn?.arguments ?? p.arguments;
|
|
3491
|
+
return {
|
|
3492
|
+
toolName: name,
|
|
3493
|
+
args: typeof rawArgs === "string" ? safeParse(rawArgs) : rawArgs ?? {},
|
|
3494
|
+
...str(p.id) ? { callId: p.id } : str(fn?.id) ? { callId: fn?.id } : {}
|
|
3495
|
+
};
|
|
3496
|
+
};
|
|
3497
|
+
var toolPartDecoders = {
|
|
3498
|
+
opencode: decodeOpencodePart,
|
|
3499
|
+
"claude-code": decodeAnthropicPart,
|
|
3500
|
+
anthropic: decodeAnthropicPart,
|
|
3501
|
+
codex: decodeOpenAiPart,
|
|
3502
|
+
openai: decodeOpenAiPart,
|
|
3503
|
+
router: decodeOpenAiPart,
|
|
3504
|
+
kimi: decodeOpenAiPart
|
|
3505
|
+
};
|
|
3506
|
+
function decodeToolPart(part, harness) {
|
|
3507
|
+
const p = obj(part);
|
|
3508
|
+
if (!p) return void 0;
|
|
3509
|
+
const specific = harness ? toolPartDecoders[harness] : void 0;
|
|
3510
|
+
if (specific) return specific(p);
|
|
3511
|
+
for (const decode of new Set(Object.values(toolPartDecoders))) {
|
|
3512
|
+
const step = decode(p);
|
|
3513
|
+
if (step) return step;
|
|
3514
|
+
}
|
|
3515
|
+
return void 0;
|
|
3516
|
+
}
|
|
3517
|
+
function safeParse(s) {
|
|
3518
|
+
try {
|
|
3519
|
+
return JSON.parse(s);
|
|
3520
|
+
} catch {
|
|
3521
|
+
return s;
|
|
3522
|
+
}
|
|
3523
|
+
}
|
|
3524
|
+
var runSeq = 0;
|
|
3525
|
+
function createPushTraceSource(opts = {}) {
|
|
3526
|
+
const runId = opts.runId ?? `push-${runSeq++}`;
|
|
3527
|
+
const now = opts.now ?? Date.now;
|
|
3528
|
+
const spans = [];
|
|
3529
|
+
const subs = /* @__PURE__ */ new Set();
|
|
3530
|
+
return {
|
|
3531
|
+
record(input) {
|
|
3532
|
+
const span = toToolSpan(input, runId, spans.length, now());
|
|
3533
|
+
spans.push(span);
|
|
3534
|
+
for (const fn of subs) {
|
|
3535
|
+
try {
|
|
3536
|
+
fn(span);
|
|
3537
|
+
} catch {
|
|
3538
|
+
}
|
|
3539
|
+
}
|
|
3540
|
+
return span;
|
|
3541
|
+
},
|
|
3542
|
+
source: {
|
|
3543
|
+
onSpan(handler) {
|
|
3544
|
+
subs.add(handler);
|
|
3545
|
+
return () => subs.delete(handler);
|
|
3546
|
+
},
|
|
3547
|
+
collect: () => Promise.resolve([...spans])
|
|
3548
|
+
}
|
|
3549
|
+
};
|
|
3550
|
+
}
|
|
3551
|
+
function createPartsTraceSource(opts) {
|
|
3552
|
+
const runId = opts.runId ?? `parts-${runSeq++}`;
|
|
3553
|
+
const now = opts.now ?? Date.now;
|
|
3554
|
+
const subs = /* @__PURE__ */ new Set();
|
|
3555
|
+
const seenLive = /* @__PURE__ */ new Set();
|
|
3556
|
+
let liveSeq = 0;
|
|
3557
|
+
let unsub;
|
|
3558
|
+
const startLive = () => {
|
|
3559
|
+
if (unsub || !opts.subscribeParts) return;
|
|
3560
|
+
unsub = opts.subscribeParts((part) => {
|
|
3561
|
+
const step = decodeToolPart(part, opts.harness);
|
|
3562
|
+
if (!step) return;
|
|
3563
|
+
if (step.callId) {
|
|
3564
|
+
if (seenLive.has(step.callId)) return;
|
|
3565
|
+
seenLive.add(step.callId);
|
|
3566
|
+
}
|
|
3567
|
+
const span = toToolSpan(step, runId, liveSeq++, now());
|
|
3568
|
+
for (const fn of subs) {
|
|
3569
|
+
try {
|
|
3570
|
+
fn(span);
|
|
3571
|
+
} catch {
|
|
3572
|
+
}
|
|
3573
|
+
}
|
|
3574
|
+
});
|
|
3575
|
+
};
|
|
3576
|
+
return {
|
|
3577
|
+
onSpan(handler) {
|
|
3578
|
+
subs.add(handler);
|
|
3579
|
+
startLive();
|
|
3580
|
+
return () => {
|
|
3581
|
+
subs.delete(handler);
|
|
3582
|
+
if (subs.size === 0 && unsub) {
|
|
3583
|
+
unsub();
|
|
3584
|
+
unsub = void 0;
|
|
3585
|
+
}
|
|
3586
|
+
};
|
|
3587
|
+
},
|
|
3588
|
+
async collect() {
|
|
3589
|
+
const parts = await opts.collectParts();
|
|
3590
|
+
const spans = [];
|
|
3591
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3592
|
+
for (const part of parts) {
|
|
3593
|
+
const step = decodeToolPart(part, opts.harness);
|
|
3594
|
+
if (!step) continue;
|
|
3595
|
+
if (step.callId) {
|
|
3596
|
+
if (seen.has(step.callId)) continue;
|
|
3597
|
+
seen.add(step.callId);
|
|
3598
|
+
}
|
|
3599
|
+
spans.push(toToolSpan(step, runId, spans.length, now()));
|
|
3600
|
+
}
|
|
3601
|
+
return spans;
|
|
3602
|
+
}
|
|
3603
|
+
};
|
|
3604
|
+
}
|
|
3605
|
+
function sandboxSessionTraceSource(box, sessionId, opts = {}) {
|
|
3606
|
+
return createPartsTraceSource({
|
|
3607
|
+
collectParts: async () => {
|
|
3608
|
+
const msgs = await box.messages({ sessionId });
|
|
3609
|
+
return msgs.flatMap((m) => m.parts ? [...m.parts] : []);
|
|
3610
|
+
},
|
|
3611
|
+
...opts.harness ? { harness: opts.harness } : {},
|
|
3612
|
+
...opts.subscribeParts ? { subscribeParts: opts.subscribeParts } : {},
|
|
3613
|
+
runId: opts.runId ?? `box-${sessionId}`,
|
|
3614
|
+
...opts.now ? { now: opts.now } : {}
|
|
3615
|
+
});
|
|
3616
|
+
}
|
|
3617
|
+
|
|
3618
|
+
// src/runtime/supervise/trajectory-recorder.ts
|
|
3619
|
+
import { buildTrajectory, InMemoryTraceStore } from "@tangle-network/agent-eval";
|
|
3620
|
+
import { stuckLoopView, toolWasteView } from "@tangle-network/agent-eval/pipelines";
|
|
3621
|
+
async function analyzeTrace(source, runId = "worker") {
|
|
3622
|
+
const spans = await source.collect();
|
|
3623
|
+
const store = new InMemoryTraceStore();
|
|
3624
|
+
for (let i = 0; i < spans.length; i += 1) {
|
|
3625
|
+
const s = spans[i];
|
|
3626
|
+
if (s) await store.appendSpan({ ...s, runId, spanId: `${runId}-t${i}` });
|
|
3627
|
+
}
|
|
3628
|
+
const [trajectory, stuckLoop, toolWaste] = await Promise.all([
|
|
3629
|
+
buildTrajectory(store, runId),
|
|
3630
|
+
stuckLoopView(store, { runId }),
|
|
3631
|
+
toolWasteView(store, { runId })
|
|
3632
|
+
]);
|
|
3633
|
+
return { trajectory, stuckLoop, toolWaste };
|
|
3634
|
+
}
|
|
3635
|
+
|
|
3636
|
+
// src/runtime/supervise/worktree-fanout.ts
|
|
3637
|
+
function worktreeFanout(options) {
|
|
3638
|
+
const deliverable = options.deliverable ?? patchDelivered({
|
|
3639
|
+
...options.maxDiffLines !== void 0 ? { maxDiffLines: options.maxDiffLines } : {},
|
|
3640
|
+
...options.forbiddenPaths !== void 0 ? { forbiddenPaths: options.forbiddenPaths } : {},
|
|
3641
|
+
...options.require !== void 0 ? { require: options.require } : {}
|
|
3642
|
+
});
|
|
3643
|
+
const itemSpec = (item) => {
|
|
3644
|
+
const executor = gateOnDeliverable(
|
|
3645
|
+
createWorktreeCliExecutor({
|
|
3646
|
+
repoRoot: options.repoRoot,
|
|
3647
|
+
profile: item.profile,
|
|
3648
|
+
harness: item.harness,
|
|
3649
|
+
taskPrompt: options.taskPrompt,
|
|
3650
|
+
...item.runId ? { runId: item.runId } : {},
|
|
3651
|
+
...item.baseRef ? { baseRef: item.baseRef } : {},
|
|
3652
|
+
...options.testCmd !== void 0 ? { testCmd: options.testCmd } : {},
|
|
3653
|
+
...options.typecheckCmd !== void 0 ? { typecheckCmd: options.typecheckCmd } : {},
|
|
3654
|
+
...options.harnessTimeoutMs !== void 0 ? { harnessTimeoutMs: options.harnessTimeoutMs } : {},
|
|
3655
|
+
...options.runGit ? { runGit: options.runGit } : {},
|
|
3656
|
+
...options.runHarness ? { runHarness: options.runHarness } : {},
|
|
3657
|
+
...options.runCommand ? { runCommand: options.runCommand } : {}
|
|
3658
|
+
}),
|
|
3659
|
+
deliverable
|
|
3660
|
+
);
|
|
3661
|
+
return { profile: item.profile, harness: null, executor };
|
|
3662
|
+
};
|
|
3663
|
+
const selectWinner = selectValidWinner({
|
|
3664
|
+
strategy: options.winnerStrategy ?? "highest-score",
|
|
3665
|
+
sizeOf: (a) => a.stats.insertions + a.stats.deletions
|
|
3666
|
+
});
|
|
3667
|
+
return fanout(options.harnesses, {
|
|
3668
|
+
itemTask: () => options.taskPrompt,
|
|
3669
|
+
label: (item, i) => `${item.name}:${i}`,
|
|
3670
|
+
itemSpec: (item) => itemSpec(item),
|
|
3671
|
+
selectWinner
|
|
3672
|
+
});
|
|
3673
|
+
}
|
|
3674
|
+
|
|
3675
|
+
// src/runtime/verifier-environment.ts
|
|
3676
|
+
var submitTool = {
|
|
3677
|
+
type: "function",
|
|
3678
|
+
function: {
|
|
3679
|
+
name: "submit_answer",
|
|
3680
|
+
description: "Submit your answer for evaluation. You may submit more than once \u2014 the best-scoring submission counts. Submit the COMPLETE final answer, not a fragment.",
|
|
3681
|
+
parameters: {
|
|
3682
|
+
type: "object",
|
|
3683
|
+
properties: { answer: { type: "string", description: "The complete final answer." } },
|
|
3684
|
+
required: ["answer"]
|
|
3685
|
+
}
|
|
3686
|
+
}
|
|
3687
|
+
};
|
|
3688
|
+
function createVerifierEnvironment(opts) {
|
|
3689
|
+
if (opts.extraTools?.length && !opts.callExtra) {
|
|
3690
|
+
throw new Error(`${opts.name}: extraTools requires callExtra`);
|
|
3691
|
+
}
|
|
3692
|
+
const states = /* @__PURE__ */ new Map();
|
|
3693
|
+
let seq = 0;
|
|
3694
|
+
return {
|
|
3695
|
+
name: opts.name,
|
|
3696
|
+
async open(task) {
|
|
3697
|
+
seq += 1;
|
|
3698
|
+
const handle = { id: `${opts.name}-${seq}`, surface: opts.name };
|
|
3699
|
+
states.set(handle.id, { task, submissions: [] });
|
|
3700
|
+
return handle;
|
|
3701
|
+
},
|
|
3702
|
+
async tools() {
|
|
3703
|
+
return [submitTool, ...opts.extraTools ?? []];
|
|
3704
|
+
},
|
|
3705
|
+
async call(handle, name, args) {
|
|
3706
|
+
const state = states.get(handle.id);
|
|
3707
|
+
if (!state) return "ERROR: workspace closed";
|
|
3708
|
+
if (name === "submit_answer") {
|
|
3709
|
+
const answer = String(args.answer ?? "").trim();
|
|
3710
|
+
if (!answer) return "ERROR: empty answer";
|
|
3711
|
+
state.submissions.push(answer);
|
|
3712
|
+
return `submission ${state.submissions.length} recorded`;
|
|
3713
|
+
}
|
|
3714
|
+
if (opts.callExtra && opts.extraTools?.some((t) => t.function.name === name)) {
|
|
3715
|
+
try {
|
|
3716
|
+
return await opts.callExtra(state.task, name, args);
|
|
3717
|
+
} catch (e) {
|
|
3718
|
+
return `ERROR: ${e instanceof Error ? e.message : String(e)}`;
|
|
3719
|
+
}
|
|
3720
|
+
}
|
|
3721
|
+
return `ERROR: unknown tool ${name}`;
|
|
3722
|
+
},
|
|
3723
|
+
// Keep-best across submissions — the measured law (workers reach correct answers,
|
|
3724
|
+
// then revise past them; final-state scoring undersells every strategy).
|
|
3725
|
+
async score(task, handle) {
|
|
3726
|
+
const state = states.get(handle.id);
|
|
3727
|
+
if (!state || state.submissions.length === 0) return { passes: 0, total: 1, errored: 0 };
|
|
3728
|
+
let best = { passes: 0, total: 1, errored: 0 };
|
|
3729
|
+
const ratio = (s) => s.total > 0 ? s.passes / s.total : 0;
|
|
3730
|
+
for (const answer of state.submissions) {
|
|
3731
|
+
const s = await opts.check(task, answer);
|
|
3732
|
+
if (ratio(s) > ratio(best)) best = s;
|
|
3733
|
+
}
|
|
3734
|
+
return best;
|
|
3735
|
+
},
|
|
3736
|
+
async close(handle) {
|
|
3737
|
+
states.delete(handle.id);
|
|
3738
|
+
}
|
|
3739
|
+
};
|
|
3740
|
+
}
|
|
3741
|
+
|
|
3742
|
+
// src/runtime/waterfall.ts
|
|
3743
|
+
function createWaterfallCollector() {
|
|
3744
|
+
let spans = /* @__PURE__ */ new Map();
|
|
3745
|
+
const onEvent = (event) => {
|
|
3746
|
+
if (event.target === "agent.spawn") {
|
|
3747
|
+
const p = event.payload ?? {};
|
|
3748
|
+
const id = p.childId ?? event.id;
|
|
3749
|
+
spans.set(id, {
|
|
3750
|
+
id,
|
|
3751
|
+
label: p.label ?? id,
|
|
3752
|
+
runId: event.runId,
|
|
3753
|
+
...event.parentId !== void 0 ? { parentId: event.parentId } : {},
|
|
3754
|
+
startMs: event.timestamp,
|
|
3755
|
+
status: "running",
|
|
3756
|
+
usd: 0,
|
|
3757
|
+
tokens: { input: 0, output: 0 }
|
|
3758
|
+
});
|
|
3759
|
+
return;
|
|
3760
|
+
}
|
|
3761
|
+
if (event.target === "agent.child") {
|
|
3762
|
+
const p = event.payload ?? {};
|
|
3763
|
+
const id = p.childId;
|
|
3764
|
+
if (!id) return;
|
|
3765
|
+
const span = spans.get(id);
|
|
3766
|
+
if (!span) return;
|
|
3767
|
+
span.endMs = event.timestamp;
|
|
3768
|
+
span.status = p.status === "down" ? "down" : "done";
|
|
3769
|
+
span.usd = p.spent?.usd ?? 0;
|
|
3770
|
+
span.tokens = {
|
|
3771
|
+
input: p.spent?.tokens?.input ?? 0,
|
|
3772
|
+
output: p.spent?.tokens?.output ?? 0
|
|
3773
|
+
};
|
|
3774
|
+
if (typeof p.score === "number") span.score = p.score;
|
|
3775
|
+
}
|
|
3776
|
+
};
|
|
3777
|
+
const report = () => {
|
|
3778
|
+
const all = [...spans.values()].sort((a, b) => a.startMs - b.startMs);
|
|
3779
|
+
const start = all[0]?.startMs ?? 0;
|
|
3780
|
+
const end = Math.max(start, ...all.map((s) => s.endMs ?? s.startMs));
|
|
3781
|
+
const byKind = {};
|
|
3782
|
+
let totalUsd = 0;
|
|
3783
|
+
const totalTokens = { input: 0, output: 0 };
|
|
3784
|
+
for (const s of all) {
|
|
3785
|
+
totalUsd += s.usd;
|
|
3786
|
+
totalTokens.input += s.tokens.input;
|
|
3787
|
+
totalTokens.output += s.tokens.output;
|
|
3788
|
+
const kind = s.label.includes(":") ? s.label.split(":")[0] : s.label;
|
|
3789
|
+
let k = byKind[kind];
|
|
3790
|
+
if (!k) {
|
|
3791
|
+
k = { count: 0, ms: 0, usd: 0, tokens: { input: 0, output: 0 } };
|
|
3792
|
+
byKind[kind] = k;
|
|
3793
|
+
}
|
|
3794
|
+
k.count += 1;
|
|
3795
|
+
k.ms += (s.endMs ?? s.startMs) - s.startMs;
|
|
3796
|
+
k.usd += s.usd;
|
|
3797
|
+
k.tokens.input += s.tokens.input;
|
|
3798
|
+
k.tokens.output += s.tokens.output;
|
|
3799
|
+
}
|
|
3800
|
+
return { spans: all, totalMs: end - start, totalUsd, totalTokens, byKind };
|
|
3801
|
+
};
|
|
3802
|
+
const render = (opts) => {
|
|
3803
|
+
const { spans: all, totalMs, totalUsd, byKind } = report();
|
|
3804
|
+
if (all.length === 0) return "(no spans observed)";
|
|
3805
|
+
const width = opts?.width ?? 48;
|
|
3806
|
+
const maxRows = opts?.maxRows ?? 60;
|
|
3807
|
+
const start = all[0]?.startMs ?? 0;
|
|
3808
|
+
const scale = totalMs > 0 ? width / totalMs : 0;
|
|
3809
|
+
const lines = [];
|
|
3810
|
+
const labelWidth = Math.min(24, Math.max(...all.map((s) => s.label.length)) + 1);
|
|
3811
|
+
for (const s of all.slice(0, maxRows)) {
|
|
3812
|
+
const offset = Math.round((s.startMs - start) * scale);
|
|
3813
|
+
const dur = (s.endMs ?? s.startMs) - s.startMs;
|
|
3814
|
+
const len = Math.max(1, Math.round(dur * scale));
|
|
3815
|
+
const bar = `${" ".repeat(Math.min(offset, width))}${(s.status === "down" ? "\u2591" : "\u2588").repeat(Math.max(1, Math.min(len, width - Math.min(offset, width) + 1)))}`;
|
|
3816
|
+
const mark = s.status === "down" ? " DOWN" : s.score !== void 0 ? ` ${(s.score * 100).toFixed(0)}%` : "";
|
|
3817
|
+
lines.push(
|
|
3818
|
+
`${s.label.padEnd(labelWidth)}|${bar.padEnd(width + 1)}| ${(dur / 1e3).toFixed(1)}s $${s.usd.toFixed(4)} ${s.tokens.input}/${s.tokens.output}tok${mark}`
|
|
3819
|
+
);
|
|
3820
|
+
}
|
|
3821
|
+
if (all.length > maxRows) lines.push(`\u2026 ${all.length - maxRows} more spans`);
|
|
3822
|
+
lines.push("\u2014".repeat(labelWidth + width + 2));
|
|
3823
|
+
for (const [kind, k] of Object.entries(byKind)) {
|
|
3824
|
+
lines.push(
|
|
3825
|
+
`${kind.padEnd(labelWidth)} \xD7${k.count} ${(k.ms / 1e3).toFixed(1)}s busy $${k.usd.toFixed(4)} ${k.tokens.input}/${k.tokens.output}tok`
|
|
3826
|
+
);
|
|
3827
|
+
}
|
|
3828
|
+
lines.push(
|
|
3829
|
+
`TOTAL${" ".repeat(labelWidth - 5)} ${(totalMs / 1e3).toFixed(1)}s wall $${totalUsd.toFixed(4)}`
|
|
3830
|
+
);
|
|
3831
|
+
return lines.join("\n");
|
|
3832
|
+
};
|
|
3833
|
+
return {
|
|
3834
|
+
hooks: { onEvent },
|
|
3835
|
+
report,
|
|
3836
|
+
render,
|
|
3837
|
+
reset: () => {
|
|
3838
|
+
spans = /* @__PURE__ */ new Map();
|
|
3839
|
+
}
|
|
3840
|
+
};
|
|
3841
|
+
}
|
|
3842
|
+
|
|
3843
|
+
// src/runtime/workspace.ts
|
|
3844
|
+
function localShell() {
|
|
3845
|
+
return async (args, cwd) => {
|
|
3846
|
+
const { execFile } = await import("child_process");
|
|
3847
|
+
const [bin, ...rest] = args;
|
|
3848
|
+
return new Promise((resolve) => {
|
|
3849
|
+
execFile(
|
|
3850
|
+
bin ?? "",
|
|
3851
|
+
rest,
|
|
3852
|
+
{ cwd, encoding: "utf-8", maxBuffer: 64 * 1024 * 1024 },
|
|
3853
|
+
(err, stdout, stderr) => {
|
|
3854
|
+
resolve({
|
|
3855
|
+
stdout: stdout ?? "",
|
|
3856
|
+
stderr: stderr ?? "",
|
|
3857
|
+
code: err ? err.code ?? 1 : 0
|
|
3858
|
+
});
|
|
3859
|
+
}
|
|
3860
|
+
);
|
|
3861
|
+
});
|
|
3862
|
+
};
|
|
3863
|
+
}
|
|
3864
|
+
function gitWorkspace(opts) {
|
|
3865
|
+
const shell = opts.shell ?? localShell();
|
|
3866
|
+
const branch = opts.branch ?? "main";
|
|
3867
|
+
const cfg = opts.noHooks === false ? [] : ["-c", "core.hooksPath=/dev/null"];
|
|
3868
|
+
const ident = ["-c", "user.email=workspace@tangle.local", "-c", "user.name=workspace"];
|
|
3869
|
+
const run = async (args, cwd) => {
|
|
3870
|
+
const res = await shell(["git", ...cfg, ...ident, ...args], cwd);
|
|
3871
|
+
if (res.code !== 0) {
|
|
3872
|
+
throw new Error(
|
|
3873
|
+
`git ${args.join(" ")} failed (${res.code}): ${tail(res.stderr || res.stdout)}`
|
|
3874
|
+
);
|
|
3875
|
+
}
|
|
3876
|
+
return res.stdout;
|
|
3877
|
+
};
|
|
3878
|
+
return {
|
|
3879
|
+
ref: opts.ref,
|
|
3880
|
+
materialize: (dir) => run(["clone", "--branch", branch, opts.ref, dir]).then(() => {
|
|
3881
|
+
}),
|
|
3882
|
+
async commit(dir, message) {
|
|
3883
|
+
await run(["add", "-A"], dir);
|
|
3884
|
+
const status = await run(["status", "--porcelain"], dir);
|
|
3885
|
+
if (!status.trim()) return { ok: true, rev: (await run(["rev-parse", "HEAD"], dir)).trim() };
|
|
3886
|
+
await run(["commit", "-m", message], dir);
|
|
3887
|
+
const pull = await shell(["git", ...cfg, ...ident, "pull", "--rebase", "origin", branch], dir);
|
|
3888
|
+
if (pull.code !== 0) {
|
|
3889
|
+
await shell(["git", ...cfg, "rebase", "--abort"], dir).catch(() => {
|
|
3890
|
+
});
|
|
3891
|
+
return { ok: false, conflict: tail(pull.stderr || pull.stdout) };
|
|
3892
|
+
}
|
|
3893
|
+
const push = await shell(["git", ...cfg, ...ident, "push", "origin", branch], dir);
|
|
3894
|
+
if (push.code !== 0) return { ok: false, conflict: tail(push.stderr || push.stdout) };
|
|
3895
|
+
return { ok: true, rev: (await run(["rev-parse", "HEAD"], dir)).trim() };
|
|
3896
|
+
},
|
|
3897
|
+
async head() {
|
|
3898
|
+
const out = await run(["ls-remote", opts.ref, `refs/heads/${branch}`]);
|
|
3899
|
+
return out.split(/\s+/)[0] ?? "";
|
|
3900
|
+
}
|
|
3901
|
+
};
|
|
3902
|
+
}
|
|
3903
|
+
function jjWorkspace(opts) {
|
|
3904
|
+
const shell = opts.shell ?? localShell();
|
|
3905
|
+
const branch = opts.branch ?? "main";
|
|
3906
|
+
const ident = [
|
|
3907
|
+
"--config-toml",
|
|
3908
|
+
'user.name="workspace"',
|
|
3909
|
+
"--config-toml",
|
|
3910
|
+
'user.email="workspace@tangle.local"'
|
|
3911
|
+
];
|
|
3912
|
+
const jj = async (args, cwd) => {
|
|
3913
|
+
const res = await shell(["jj", ...ident, ...args], cwd);
|
|
3914
|
+
if (res.code !== 0) {
|
|
3915
|
+
throw new Error(
|
|
3916
|
+
`jj ${args.join(" ")} failed (${res.code}): ${tail(res.stderr || res.stdout)}`
|
|
3917
|
+
);
|
|
3918
|
+
}
|
|
3919
|
+
return res.stdout;
|
|
3920
|
+
};
|
|
3921
|
+
return {
|
|
3922
|
+
ref: opts.ref,
|
|
3923
|
+
// Colocated clone: jj manages history, git holds the durable remote.
|
|
3924
|
+
materialize: (dir) => jj(["git", "clone", "--colocate", opts.ref, dir]).then(() => {
|
|
3925
|
+
}),
|
|
3926
|
+
async commit(dir, message) {
|
|
3927
|
+
await jj(["describe", "-m", message], dir);
|
|
3928
|
+
await jj(["new"], dir);
|
|
3929
|
+
const push = await shell(["jj", ...ident, "git", "push", "--branch", branch], dir);
|
|
3930
|
+
if (push.code !== 0) return { ok: false, conflict: tail(push.stderr || push.stdout) };
|
|
3931
|
+
const rev = (await jj(["log", "--no-graph", "-r", "@-", "-T", "commit_id"], dir)).trim();
|
|
3932
|
+
return { ok: true, rev };
|
|
3933
|
+
},
|
|
3934
|
+
async head() {
|
|
3935
|
+
const out = await shell(["git", "ls-remote", opts.ref, `refs/heads/${branch}`]);
|
|
3936
|
+
return out.stdout.split(/\s+/)[0] ?? "";
|
|
3937
|
+
}
|
|
3938
|
+
};
|
|
3939
|
+
}
|
|
3940
|
+
async function runInWorkspace(ws, body, opts = {}) {
|
|
3941
|
+
const { mkdtempSync, rmSync } = await import("fs");
|
|
3942
|
+
const { tmpdir } = await import("os");
|
|
3943
|
+
const { join: join2 } = await import("path");
|
|
3944
|
+
const dir = mkdtempSync(join2(tmpdir(), opts.tmpPrefix ?? "ws-run-"));
|
|
3945
|
+
try {
|
|
3946
|
+
await ws.materialize(dir);
|
|
3947
|
+
const r = await body(dir);
|
|
3948
|
+
if (r.valid || opts.commitOnInvalid) {
|
|
3949
|
+
const message = r.message ?? (r.valid ? "worker: delivered" : "worker: wip");
|
|
3950
|
+
const commit = await ws.commit(dir, message);
|
|
3951
|
+
return { valid: r.valid, value: r.value, commit };
|
|
3952
|
+
}
|
|
3953
|
+
return { valid: r.valid, value: r.value };
|
|
3954
|
+
} finally {
|
|
3955
|
+
rmSync(dir, { recursive: true, force: true });
|
|
3956
|
+
}
|
|
3957
|
+
}
|
|
3958
|
+
function tail(s) {
|
|
3959
|
+
return s.slice(-400);
|
|
3960
|
+
}
|
|
3961
|
+
|
|
3962
|
+
export {
|
|
3963
|
+
anytimeReport,
|
|
3964
|
+
renderAnytimeTable,
|
|
3965
|
+
defaultAuditorInstruction,
|
|
3966
|
+
auditIntent,
|
|
3967
|
+
completionAuthorizes,
|
|
3968
|
+
stopSentinel,
|
|
3969
|
+
sentinelCompletion,
|
|
3970
|
+
deterministicCompletion,
|
|
3971
|
+
defaultAnalystInstruction,
|
|
3972
|
+
observe,
|
|
3973
|
+
renderReport,
|
|
3974
|
+
harvestCorpus,
|
|
3975
|
+
inlineSandboxClient,
|
|
3976
|
+
reportLoopUsage,
|
|
3977
|
+
loopDispatch,
|
|
3978
|
+
createMcpEnvironment,
|
|
3979
|
+
assertTraceDerivedFindings,
|
|
3980
|
+
createScopeAnalyst,
|
|
3981
|
+
registryScopeAnalyst,
|
|
3982
|
+
buildSteerContext,
|
|
3983
|
+
selectValidWinner,
|
|
3984
|
+
pipeline,
|
|
3985
|
+
fanout,
|
|
3986
|
+
loopUntil,
|
|
3987
|
+
panel,
|
|
3988
|
+
verify,
|
|
3989
|
+
widen,
|
|
3990
|
+
flatWidenGate,
|
|
3991
|
+
InMemoryCorpus,
|
|
3992
|
+
FileCorpus,
|
|
3993
|
+
renderCorpusToInstructions,
|
|
3994
|
+
createShapeRegistry,
|
|
3995
|
+
builtinShapes,
|
|
3996
|
+
registerShape,
|
|
3997
|
+
definePersona,
|
|
3998
|
+
runPersonified,
|
|
3999
|
+
trajectoryReport,
|
|
4000
|
+
equalKOnCost,
|
|
4001
|
+
promotionGate,
|
|
4002
|
+
depthStrategy,
|
|
4003
|
+
breadthStrategy,
|
|
4004
|
+
sample,
|
|
4005
|
+
refine,
|
|
4006
|
+
defineStrategy,
|
|
4007
|
+
adaptiveRefine,
|
|
4008
|
+
sampleThenRefine,
|
|
4009
|
+
runAgentic,
|
|
4010
|
+
runBenchmark,
|
|
4011
|
+
printBenchmarkReport,
|
|
4012
|
+
SandboxRunAbortError,
|
|
4013
|
+
openSandboxRun,
|
|
4014
|
+
strategyAuthorContract,
|
|
4015
|
+
assertStrategyContract,
|
|
4016
|
+
authorStrategy,
|
|
4017
|
+
discriminatingMeans,
|
|
4018
|
+
pickChampion,
|
|
4019
|
+
selectChampion,
|
|
4020
|
+
runStrategyEvolution,
|
|
4021
|
+
defaultToolDetectors,
|
|
4022
|
+
watchTrace,
|
|
4023
|
+
runCoderChecks,
|
|
4024
|
+
patchDelivered,
|
|
4025
|
+
decodeToolPart,
|
|
4026
|
+
createPushTraceSource,
|
|
4027
|
+
sandboxSessionTraceSource,
|
|
4028
|
+
analyzeTrace,
|
|
4029
|
+
worktreeFanout,
|
|
4030
|
+
createVerifierEnvironment,
|
|
4031
|
+
createWaterfallCollector,
|
|
4032
|
+
localShell,
|
|
4033
|
+
gitWorkspace,
|
|
4034
|
+
jjWorkspace,
|
|
4035
|
+
runInWorkspace,
|
|
4036
|
+
computeFindingId,
|
|
4037
|
+
makeFinding2 as makeFinding
|
|
4038
|
+
};
|
|
4039
|
+
//# sourceMappingURL=chunk-HPYWEFVY.js.map
|