@tangle-network/agent-runtime 0.105.0 → 0.106.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (218) hide show
  1. package/README.md +2 -2
  2. package/dist/activation-CRjVfOiy.js +203 -0
  3. package/dist/activation-CRjVfOiy.js.map +1 -0
  4. package/dist/activation-DyWB0K6E.d.ts +98 -0
  5. package/dist/agent.d.ts +216 -234
  6. package/dist/agent.js +615 -547
  7. package/dist/agent.js.map +1 -1
  8. package/dist/analyst-loop-DGdtVENg.js +517 -0
  9. package/dist/analyst-loop-DGdtVENg.js.map +1 -0
  10. package/dist/analyst-loop.d.ts +10 -43
  11. package/dist/analyst-loop.js +2 -11
  12. package/dist/backends-Dy8NujB4.js +713 -0
  13. package/dist/backends-Dy8NujB4.js.map +1 -0
  14. package/dist/candidate-execution/index.d.ts +4 -192
  15. package/dist/candidate-execution/index.js +5 -66
  16. package/dist/candidate-execution-DqMbv1rU.js +488 -0
  17. package/dist/candidate-execution-DqMbv1rU.js.map +1 -0
  18. package/dist/coder-yhVWbdWc.js +40 -0
  19. package/dist/coder-yhVWbdWc.js.map +1 -0
  20. package/dist/conversation-XucG_K3l.js +1265 -0
  21. package/dist/conversation-XucG_K3l.js.map +1 -0
  22. package/dist/conversation.d.ts +2 -804
  23. package/dist/conversation.js +2 -55
  24. package/dist/environment-provider-CfLUmKmf.js +1168 -0
  25. package/dist/environment-provider-CfLUmKmf.js.map +1 -0
  26. package/dist/environment-provider-IUGU3epE.d.ts +964 -0
  27. package/dist/environment-provider.d.ts +2 -78
  28. package/dist/environment-provider.js +2 -18
  29. package/dist/errors-DEAvWQPy.js +111 -0
  30. package/dist/errors-DEAvWQPy.js.map +1 -0
  31. package/dist/improvement-adapter-Cr5TBSji.d.ts +166 -0
  32. package/dist/improvement-cycle-C095kS_g.js +2624 -0
  33. package/dist/improvement-cycle-C095kS_g.js.map +1 -0
  34. package/dist/index-BGdxAwfO.d.ts +838 -0
  35. package/dist/index-C-FYUuFG.d.ts +193 -0
  36. package/dist/index-CbNM40xG.d.ts +140 -0
  37. package/dist/index-DruSKs3Z.d.ts +7224 -0
  38. package/dist/index-Efjb3nrQ.d.ts +671 -0
  39. package/dist/index.d.ts +14 -1058
  40. package/dist/index.js +2106 -2082
  41. package/dist/index.js.map +1 -1
  42. package/dist/intelligence.d.ts +768 -889
  43. package/dist/intelligence.js +1466 -1367
  44. package/dist/intelligence.js.map +1 -1
  45. package/dist/{kb-gate-CwHO0vz6.d.ts → kb-gate-C8z2juK8.d.ts} +33 -31
  46. package/dist/kb-gate-DpaSwXVx.js +80 -0
  47. package/dist/kb-gate-DpaSwXVx.js.map +1 -0
  48. package/dist/knowledge-Ku-mPP4f.js +433 -0
  49. package/dist/knowledge-Ku-mPP4f.js.map +1 -0
  50. package/dist/knowledge.d.ts +2 -150
  51. package/dist/knowledge.js +2 -40
  52. package/dist/local-harness-Dh8PJ0ot.d.ts +455 -0
  53. package/dist/loop-runner-bin-BXsFEUcf.d.ts +162 -0
  54. package/dist/loop-runner-bin-BwjQ1NfH.js +268 -0
  55. package/dist/loop-runner-bin-BwjQ1NfH.js.map +1 -0
  56. package/dist/loop-runner-bin.d.ts +2 -15
  57. package/dist/loop-runner-bin.js +2 -24
  58. package/dist/loops.d.ts +5 -4164
  59. package/dist/loops.js +8 -419
  60. package/dist/mcp/bin.d.ts +1 -1
  61. package/dist/mcp/bin.js +116 -146
  62. package/dist/mcp/bin.js.map +1 -1
  63. package/dist/mcp/index.d.ts +295 -436
  64. package/dist/mcp/index.js +1242 -1177
  65. package/dist/mcp/index.js.map +1 -1
  66. package/dist/mcp/memory-bin.d.ts +1 -1
  67. package/dist/mcp/memory-bin.js +37 -16
  68. package/dist/mcp/memory-bin.js.map +1 -1
  69. package/dist/memory-server-DL6cE2Ag.js +234 -0
  70. package/dist/memory-server-DL6cE2Ag.js.map +1 -0
  71. package/dist/model-resolution-Btd9iIKV.js +98 -0
  72. package/dist/model-resolution-Btd9iIKV.js.map +1 -0
  73. package/dist/openai-tools-B68JaOCx.d.ts +24 -0
  74. package/dist/openai-tools-CKLy1C7M.js +44 -0
  75. package/dist/openai-tools-CKLy1C7M.js.map +1 -0
  76. package/dist/otel-export-D34IQW1C.js +843 -0
  77. package/dist/otel-export-D34IQW1C.js.map +1 -0
  78. package/dist/platform.d.ts +188 -185
  79. package/dist/platform.js +185 -218
  80. package/dist/platform.js.map +1 -1
  81. package/dist/prepare-B53sPV7M.js +4292 -0
  82. package/dist/prepare-B53sPV7M.js.map +1 -0
  83. package/dist/primeintellect/index.d.ts +149 -145
  84. package/dist/primeintellect/index.js +626 -1003
  85. package/dist/primeintellect/index.js.map +1 -1
  86. package/dist/profiles.d.ts +296 -490
  87. package/dist/profiles.js +722 -749
  88. package/dist/profiles.js.map +1 -1
  89. package/dist/protected-model-port-BLPH5VVv.js +722 -0
  90. package/dist/protected-model-port-BLPH5VVv.js.map +1 -0
  91. package/dist/protected-model-port-BP6Z4eau.d.ts +283 -0
  92. package/dist/protected-redaction--F3v1oo8.js +200 -0
  93. package/dist/protected-redaction--F3v1oo8.js.map +1 -0
  94. package/dist/redact-D-u-rrcn.js +82 -0
  95. package/dist/redact-D-u-rrcn.js.map +1 -0
  96. package/dist/redact-kbIOjfjM.d.ts +452 -0
  97. package/dist/researcher-CoVqNhfI.js +433 -0
  98. package/dist/researcher-CoVqNhfI.js.map +1 -0
  99. package/dist/runtime-BG0BziaP.js +5408 -0
  100. package/dist/runtime-BG0BziaP.js.map +1 -0
  101. package/dist/sandbox-events-DeI5xX8P.js +316 -0
  102. package/dist/sandbox-events-DeI5xX8P.js.map +1 -0
  103. package/dist/spawn-journal-CwPvKUTa.js +456 -0
  104. package/dist/spawn-journal-CwPvKUTa.js.map +1 -0
  105. package/dist/structural-rollout-IXUEplky.js +1445 -0
  106. package/dist/structural-rollout-IXUEplky.js.map +1 -0
  107. package/dist/substrate-B0TYNrXn.js +27 -0
  108. package/dist/substrate-B0TYNrXn.js.map +1 -0
  109. package/dist/{substrate-Dyiqmd3o.d.ts → substrate-BcnuSHXm.d.ts} +57 -54
  110. package/dist/supervise-BiRutHS9.js +7321 -0
  111. package/dist/supervise-BiRutHS9.js.map +1 -0
  112. package/dist/supervisor-DTKhF-RV.js +3826 -0
  113. package/dist/supervisor-DTKhF-RV.js.map +1 -0
  114. package/dist/testing.d.ts +4 -3
  115. package/dist/testing.js +2478 -2637
  116. package/dist/testing.js.map +1 -1
  117. package/dist/tool-server-RcWgLIsL.js +141 -0
  118. package/dist/tool-server-RcWgLIsL.js.map +1 -0
  119. package/dist/trace-propagation-CJJC7SVB.js +57 -0
  120. package/dist/trace-propagation-CJJC7SVB.js.map +1 -0
  121. package/dist/types-BaIpb0XT.d.ts +492 -0
  122. package/dist/types-BevOjfTY.d.ts +765 -0
  123. package/dist/types-C9j4qg6l.d.ts +500 -0
  124. package/dist/types-Cja5OTGh.d.ts +190 -0
  125. package/dist/util-MVgdwuIS.js +145 -0
  126. package/dist/util-MVgdwuIS.js.map +1 -0
  127. package/dist/workspace-archive-BosQiotA.js +575 -0
  128. package/dist/workspace-archive-BosQiotA.js.map +1 -0
  129. package/package.json +43 -56
  130. package/skills/build-with-agent-runtime/SKILL.md +1 -0
  131. package/dist/activation-Cm-9MMX1.d.ts +0 -69
  132. package/dist/analyst-loop.js.map +0 -1
  133. package/dist/backends-BO4Jqoe2.d.ts +0 -134
  134. package/dist/candidate-execution/index.js.map +0 -1
  135. package/dist/chunk-2KGAN2HM.js +0 -1256
  136. package/dist/chunk-2KGAN2HM.js.map +0 -1
  137. package/dist/chunk-2VZYRB55.js +0 -834
  138. package/dist/chunk-2VZYRB55.js.map +0 -1
  139. package/dist/chunk-3MDZX7YU.js +0 -125
  140. package/dist/chunk-3MDZX7YU.js.map +0 -1
  141. package/dist/chunk-5IBQAPVB.js +0 -29
  142. package/dist/chunk-5IBQAPVB.js.map +0 -1
  143. package/dist/chunk-5QOB7H74.js +0 -387
  144. package/dist/chunk-5QOB7H74.js.map +0 -1
  145. package/dist/chunk-C5RTIBNZ.js +0 -48
  146. package/dist/chunk-C5RTIBNZ.js.map +0 -1
  147. package/dist/chunk-CTRA64LY.js +0 -680
  148. package/dist/chunk-CTRA64LY.js.map +0 -1
  149. package/dist/chunk-DPEUKJRO.js +0 -27
  150. package/dist/chunk-DPEUKJRO.js.map +0 -1
  151. package/dist/chunk-FD2MBMOH.js +0 -494
  152. package/dist/chunk-FD2MBMOH.js.map +0 -1
  153. package/dist/chunk-H5QPIZNX.js +0 -907
  154. package/dist/chunk-H5QPIZNX.js.map +0 -1
  155. package/dist/chunk-HLKC4UYB.js +0 -3514
  156. package/dist/chunk-HLKC4UYB.js.map +0 -1
  157. package/dist/chunk-HNP72PNU.js +0 -192
  158. package/dist/chunk-HNP72PNU.js.map +0 -1
  159. package/dist/chunk-ISPWRSEC.js +0 -408
  160. package/dist/chunk-ISPWRSEC.js.map +0 -1
  161. package/dist/chunk-KRBFHMV6.js +0 -1228
  162. package/dist/chunk-KRBFHMV6.js.map +0 -1
  163. package/dist/chunk-M22PL4GH.js +0 -171
  164. package/dist/chunk-M22PL4GH.js.map +0 -1
  165. package/dist/chunk-NURIQBJQ.js +0 -72
  166. package/dist/chunk-NURIQBJQ.js.map +0 -1
  167. package/dist/chunk-OPVWXJ2H.js +0 -75
  168. package/dist/chunk-OPVWXJ2H.js.map +0 -1
  169. package/dist/chunk-PZZKQVQV.js +0 -52
  170. package/dist/chunk-PZZKQVQV.js.map +0 -1
  171. package/dist/chunk-Q2JSAVQ3.js +0 -671
  172. package/dist/chunk-Q2JSAVQ3.js.map +0 -1
  173. package/dist/chunk-RDOAVVHY.js +0 -699
  174. package/dist/chunk-RDOAVVHY.js.map +0 -1
  175. package/dist/chunk-SGKPNBXE.js +0 -60
  176. package/dist/chunk-SGKPNBXE.js.map +0 -1
  177. package/dist/chunk-SMQXZGLZ.js +0 -351
  178. package/dist/chunk-SMQXZGLZ.js.map +0 -1
  179. package/dist/chunk-UHEZW5BU.js +0 -2918
  180. package/dist/chunk-UHEZW5BU.js.map +0 -1
  181. package/dist/chunk-UPWGXKXB.js +0 -254
  182. package/dist/chunk-UPWGXKXB.js.map +0 -1
  183. package/dist/chunk-VISA6CI3.js +0 -200
  184. package/dist/chunk-VISA6CI3.js.map +0 -1
  185. package/dist/chunk-WMTCUOQL.js +0 -474
  186. package/dist/chunk-WMTCUOQL.js.map +0 -1
  187. package/dist/chunk-WRTOVNN4.js +0 -4728
  188. package/dist/chunk-WRTOVNN4.js.map +0 -1
  189. package/dist/chunk-WSTRQZYQ.js +0 -5576
  190. package/dist/chunk-WSTRQZYQ.js.map +0 -1
  191. package/dist/chunk-XBG2W2VW.js +0 -6434
  192. package/dist/chunk-XBG2W2VW.js.map +0 -1
  193. package/dist/chunk-YEJR7IXO.js +0 -68
  194. package/dist/chunk-YEJR7IXO.js.map +0 -1
  195. package/dist/chunk-YJZA2BIK.js +0 -1338
  196. package/dist/chunk-YJZA2BIK.js.map +0 -1
  197. package/dist/completion-gate-BMy5LGoP.d.ts +0 -256
  198. package/dist/conversation.js.map +0 -1
  199. package/dist/coordination-BZZSVYpZ.d.ts +0 -2570
  200. package/dist/environment-provider.js.map +0 -1
  201. package/dist/improvement-adapter-HAZz-7vK.d.ts +0 -203
  202. package/dist/knowledge.js.map +0 -1
  203. package/dist/local-harness-BDNcl6jI.d.ts +0 -175
  204. package/dist/loop-runner-bin-68aoM9-7.d.ts +0 -199
  205. package/dist/loop-runner-bin.js.map +0 -1
  206. package/dist/loops.js.map +0 -1
  207. package/dist/openai-tools-fnj6SRVg.d.ts +0 -46
  208. package/dist/protected-model-port-B4t-OQjL.d.ts +0 -271
  209. package/dist/redact-BuFjxBUJ.d.ts +0 -481
  210. package/dist/sanitize-DEbPNtyI.d.ts +0 -328
  211. package/dist/structural-rollout-DEh6CEsa.d.ts +0 -446
  212. package/dist/supervise-4_48wwvy.d.ts +0 -379
  213. package/dist/types-BCoemcXU.d.ts +0 -802
  214. package/dist/types-BwoZWq-i.d.ts +0 -510
  215. package/dist/types-Cbp72qyu.d.ts +0 -491
  216. package/dist/types-CmYCMbFT.d.ts +0 -203
  217. package/dist/types-CvNwMzQt.d.ts +0 -957
  218. package/dist/worktree-fanout-DxhAWr5Z.d.ts +0 -999
@@ -1,4728 +0,0 @@
1
- import {
2
- newRuntimeSession,
3
- normalizeBackendStreamEvent,
4
- nowIso
5
- } from "./chunk-Q2JSAVQ3.js";
6
- import {
7
- createExecutor,
8
- createExecutorRegistry,
9
- createSandboxLineage,
10
- createWorktreeCliExecutor,
11
- defaultSelectWinner,
12
- gateOnDeliverable,
13
- probeSandboxCapabilities,
14
- rollingDispatch,
15
- runAgentRounds,
16
- supervise
17
- } from "./chunk-XBG2W2VW.js";
18
- import {
19
- observe,
20
- refine,
21
- runAgentic,
22
- sample,
23
- sampleThenRefine,
24
- strategyAuthorMethod
25
- } from "./chunk-YJZA2BIK.js";
26
- import {
27
- redactProtectedReason,
28
- redactProtectedValue
29
- } from "./chunk-UPWGXKXB.js";
30
- import {
31
- createSupervisor,
32
- notifyRuntimeHookEvent,
33
- routerBrain,
34
- runBrainLoop,
35
- settledToIteration,
36
- withDriverExecutor
37
- } from "./chunk-HLKC4UYB.js";
38
- import {
39
- InMemoryResultBlobStore,
40
- InMemorySpawnJournal
41
- } from "./chunk-ISPWRSEC.js";
42
- import {
43
- addTokenUsage,
44
- isAbortError,
45
- randomSuffix,
46
- sleep,
47
- stringifySafe,
48
- zeroTokenUsage
49
- } from "./chunk-3MDZX7YU.js";
50
- import {
51
- createSandboxToolPartState,
52
- mapSandboxEvent,
53
- mapSandboxToolEvent
54
- } from "./chunk-M22PL4GH.js";
55
- import {
56
- AnalystError,
57
- BackendTransportError,
58
- PlannerError,
59
- ValidationError
60
- } from "./chunk-YEJR7IXO.js";
61
-
62
- // src/runtime/index.ts
63
- import { computeFindingId, makeFinding } from "@tangle-network/agent-eval";
64
-
65
- // src/runtime/audit-intent.ts
66
- var defaultAuditorInstruction = "You audit whether an AI agent is on the RIGHT ROUTE \u2014 not whether it works hard, but whether its actions serve the stated intents. Infer the REVEALED intent from the action pattern (what the trajectory is actually optimizing). Compare against the declared task intent, the user intent when given, and the meta-intent when given. Flawless execution down the wrong route is DIVERGED. Busy-work that neither advances nor harms is DRIFTING. Judge only from the trajectory \u2014 be specific about which actions ground your verdict. Recommend abort only when continuing cannot serve the intent.";
67
- function summarize(trace, maxLines) {
68
- const lines = [];
69
- for (const ev of trace) {
70
- const e = ev;
71
- const role = e.role;
72
- if (role === "tool") lines.push(`RESULT ${String(e.content).slice(0, 200)}`);
73
- else if (role === "assistant") {
74
- const calls = e.tool_calls?.map((c) => `${c.function?.name}(${(c.function?.arguments ?? "").slice(0, 120)})`).join(", ");
75
- lines.push(calls ? `CALL ${calls}` : `SAY ${String(e.content).slice(0, 160)}`);
76
- } else if (role === "user") lines.push(`USER ${String(e.content).slice(0, 160)}`);
77
- }
78
- return lines.slice(-maxLines).join("\n");
79
- }
80
- var auditSchema = {
81
- name: "intent_audit",
82
- schema: {
83
- type: "object",
84
- additionalProperties: false,
85
- required: ["revealedIntent", "verdict", "evidence", "recommendation", "confidence"],
86
- properties: {
87
- revealedIntent: { type: "string" },
88
- verdict: { type: "string", enum: ["aligned", "drifting", "diverged"] },
89
- evidence: { type: "string" },
90
- recommendation: { type: "string", enum: ["continue", "steer", "abort"] },
91
- steer: { type: "string" },
92
- confidence: { type: "number" }
93
- }
94
- }
95
- };
96
- async function auditIntent(input, opts) {
97
- const res = await opts.chat.chat(
98
- {
99
- ...opts.model ? { model: opts.model } : {},
100
- jsonSchema: auditSchema,
101
- messages: [
102
- { role: "system", content: opts.auditorInstruction ?? defaultAuditorInstruction },
103
- {
104
- role: "user",
105
- content: `DECLARED INTENT (the task):
106
- ${input.declaredIntent}
107
-
108
- ` + (input.userIntent ? `USER INTENT (the principal's actual goal):
109
- ${input.userIntent}
110
-
111
- ` : "") + (input.metaIntent ? `META-INTENT (what the whole run is for):
112
- ${input.metaIntent}
113
-
114
- ` : "") + `TRAJECTORY (in order):
115
- ${summarize(input.trace, opts.maxTraceLines ?? 80)}
116
-
117
- Audit the route: revealed intent, verdict, evidence, one recommendation.`
118
- }
119
- ]
120
- },
121
- { ...opts.signal ? { signal: opts.signal } : {} }
122
- );
123
- let parsed;
124
- try {
125
- parsed = JSON.parse(res.content);
126
- } catch {
127
- throw new Error(`auditIntent: auditor returned non-JSON: ${res.content.slice(0, 200)}`);
128
- }
129
- if (!parsed.verdict || !parsed.recommendation) {
130
- throw new Error(`auditIntent: missing verdict/recommendation: ${res.content.slice(0, 200)}`);
131
- }
132
- return {
133
- revealedIntent: parsed.revealedIntent ?? "",
134
- verdict: parsed.verdict,
135
- evidence: parsed.evidence ?? "",
136
- recommendation: parsed.recommendation,
137
- ...parsed.steer ? { steer: parsed.steer } : {},
138
- confidence: typeof parsed.confidence === "number" ? parsed.confidence : 0.5
139
- };
140
- }
141
-
142
- // src/runtime/benchmark-report.ts
143
- import {
144
- benjaminiHochberg,
145
- confidenceInterval,
146
- pairedBootstrap,
147
- pairedTTest,
148
- wilson
149
- } from "@tangle-network/agent-eval";
150
- var defaultScoreOf = (r) => {
151
- const o = r.outcome;
152
- if (typeof o.holdoutScore === "number") return o.holdoutScore;
153
- if (typeof o.searchScore === "number") return o.searchScore;
154
- const raw = o.raw ?? {};
155
- for (const k of ["composite", "score", "passed", "resolved"]) {
156
- if (typeof raw[k] === "number") return raw[k];
157
- }
158
- return void 0;
159
- };
160
- var defaultProfileKeyOf = (r) => {
161
- const cell = r.agentProfile;
162
- const harness = cell?.harness;
163
- return harness ? `${harness}\xB7${r.model}` : r.model;
164
- };
165
- var defaultGroupOf = (r) => r.scenarioId ?? r.experimentId;
166
- function quantile(sorted, q) {
167
- if (sorted.length === 0) return 0;
168
- const idx = Math.min(sorted.length - 1, Math.floor(q * (sorted.length - 1)));
169
- return sorted[idx] ?? 0;
170
- }
171
- function mean(xs) {
172
- return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
173
- }
174
- function meanByScenario(records, scoreOf) {
175
- const sums = /* @__PURE__ */ new Map();
176
- for (const r of records) {
177
- const s = scoreOf(r);
178
- if (typeof s !== "number") continue;
179
- const id = r.scenarioId;
180
- if (!id) {
181
- throw new Error(
182
- `benchmark-report: RunRecord (candidate ${r.candidateId ?? "unknown"}) is missing scenarioId \u2014 cannot pair or interval it honestly. Pass opts.stats only on a scenario-tagged corpus.`
183
- );
184
- }
185
- const acc = sums.get(id) ?? { total: 0, n: 0 };
186
- acc.total += s;
187
- acc.n += 1;
188
- sums.set(id, acc);
189
- }
190
- const out = /* @__PURE__ */ new Map();
191
- for (const [id, acc] of sums) out.set(id, acc.n ? acc.total / acc.n : 0);
192
- return out;
193
- }
194
- function leaderboard(records, opts = {}) {
195
- const scoreOf = opts.scoreOf ?? defaultScoreOf;
196
- const profileKeyOf = opts.profileKeyOf ?? defaultProfileKeyOf;
197
- const groupOf = opts.groupOf ?? defaultGroupOf;
198
- const labelOf = opts.labelOf ?? ((k) => k);
199
- const axisSet = /* @__PURE__ */ new Set();
200
- if (opts.axisScoresOf) {
201
- for (const r of records) for (const k of Object.keys(opts.axisScoresOf(r))) axisSet.add(k);
202
- } else {
203
- for (const r of records) axisSet.add(groupOf(r));
204
- }
205
- const axes = [...axisSet].sort();
206
- const byProfile = /* @__PURE__ */ new Map();
207
- for (const r of records) {
208
- const key = profileKeyOf(r);
209
- const bucket = byProfile.get(key);
210
- if (bucket) bucket.push(r);
211
- else byProfile.set(key, [r]);
212
- }
213
- const rows = [];
214
- for (const [profileKey, recs] of byProfile) {
215
- const scores = recs.map(scoreOf).filter((s) => typeof s === "number");
216
- const axisBuckets = /* @__PURE__ */ new Map();
217
- for (const r of recs) {
218
- if (opts.axisScoresOf) {
219
- for (const [axis, s] of Object.entries(opts.axisScoresOf(r))) {
220
- const b = axisBuckets.get(axis) ?? [];
221
- b.push(s);
222
- axisBuckets.set(axis, b);
223
- }
224
- } else {
225
- const s = scoreOf(r);
226
- if (typeof s === "number") {
227
- const axis = groupOf(r);
228
- const b = axisBuckets.get(axis) ?? [];
229
- b.push(s);
230
- axisBuckets.set(axis, b);
231
- }
232
- }
233
- }
234
- const perAxis = {};
235
- for (const [axis, b] of axisBuckets) perAxis[axis] = mean(b);
236
- const latencies = recs.map((r) => r.wallMs).sort((a, b) => a - b);
237
- const pass = opts.passThreshold ?? 0.999;
238
- let scoreCi;
239
- let passCi;
240
- if (opts.stats) {
241
- const collapsed = [...meanByScenario(recs, scoreOf).values()];
242
- if (collapsed.length > 0) {
243
- const ci2 = confidenceInterval(collapsed, 0.95, { seed: 7 });
244
- scoreCi = { lower: ci2.lower, upper: ci2.upper };
245
- const w = wilson(collapsed.filter((s) => s >= pass).length, collapsed.length, 0.95);
246
- passCi = { lower: w.lower, upper: w.upper };
247
- }
248
- }
249
- rows.push({
250
- profileKey,
251
- label: labelOf(profileKey),
252
- model: recs[0]?.model ?? profileKey,
253
- n: recs.length,
254
- meanScore: mean(scores),
255
- solveRate: scores.length === 0 ? 0 : scores.filter((s) => s >= pass).length / scores.length,
256
- perAxis,
257
- costUsd: recs.reduce((a, r) => a + r.costUsd, 0),
258
- tokensIn: recs.reduce((a, r) => a + (r.tokenUsage?.input ?? 0), 0),
259
- tokensOut: recs.reduce((a, r) => a + (r.tokenUsage?.output ?? 0), 0),
260
- latencyP50Ms: quantile(latencies, 0.5),
261
- latencyP90Ms: quantile(latencies, 0.9),
262
- ...scoreCi ? { scoreCi } : {},
263
- ...passCi ? { passCi } : {}
264
- });
265
- }
266
- rows.sort(
267
- (a, b) => b.meanScore - a.meanScore || a.costUsd - b.costUsd || a.label.localeCompare(b.label)
268
- );
269
- const models = [...new Set(records.map((r) => r.model))].sort();
270
- return {
271
- title: opts.title ?? "Benchmark report",
272
- axes,
273
- profiles: rows,
274
- meta: opts.meta ?? {},
275
- provenance: {
276
- records: records.length,
277
- profiles: rows.length,
278
- axes: axes.length,
279
- models,
280
- totalCostUsd: records.reduce((a, r) => a + r.costUsd, 0)
281
- }
282
- };
283
- }
284
- function pairwiseSignificance(records, opts = {}) {
285
- const scoreOf = opts.scoreOf ?? defaultScoreOf;
286
- const profileKeyOf = opts.profileKeyOf ?? defaultProfileKeyOf;
287
- const labelOf = opts.labelOf ?? ((k) => k);
288
- const minPairs = opts.minPairs ?? 12;
289
- const byProfile = /* @__PURE__ */ new Map();
290
- for (const r of records) {
291
- const k = profileKeyOf(r);
292
- const b = byProfile.get(k);
293
- if (b) b.push(r);
294
- else byProfile.set(k, [r]);
295
- }
296
- const keys = [...byProfile.keys()].sort();
297
- const collapsed = new Map(keys.map((k) => [k, meanByScenario(byProfile.get(k) ?? [], scoreOf)]));
298
- const raw = [];
299
- for (let i = 0; i < keys.length; i += 1) {
300
- for (let j = i + 1; j < keys.length; j += 1) {
301
- const ka = keys[i];
302
- const kb = keys[j];
303
- const am = collapsed.get(ka);
304
- const bm = collapsed.get(kb);
305
- const aScores = [];
306
- const bScores = [];
307
- for (const sid of [...am.keys()].sort()) {
308
- const bv = bm.get(sid);
309
- if (bv !== void 0) {
310
- aScores.push(am.get(sid));
311
- bScores.push(bv);
312
- }
313
- }
314
- if (aScores.length === 0) continue;
315
- const boot = pairedBootstrap(aScores, bScores, { seed: 7, statistic: "median" });
316
- const p = pairedTTest(aScores, bScores).p;
317
- raw.push({
318
- a: labelOf(ka),
319
- b: labelOf(kb),
320
- pairs: aScores.length,
321
- delta: boot.median,
322
- ciLow: boot.low,
323
- ciHigh: boot.high,
324
- p
325
- });
326
- }
327
- }
328
- const { significant } = benjaminiHochberg(
329
- raw.map((r) => r.p),
330
- opts.fdr ?? 0.05
331
- );
332
- return raw.map((r, i) => ({
333
- ...r,
334
- significant: (significant[i] ?? false) && r.pairs >= minPairs
335
- }));
336
- }
337
- var pct = (x) => `${(100 * x).toFixed(1)}%`;
338
- var ci = (iv) => iv ? ` [${pct(iv.lower)}, ${pct(iv.upper)}]` : "";
339
- function renderLeaderboardMarkdown(report) {
340
- const lines = [];
341
- lines.push(`# ${report.title}`, "");
342
- const p = report.provenance;
343
- lines.push(
344
- `**${p.profiles} profiles \xD7 ${p.axes} axes**, ${p.records} runs, ${p.models.length} models \xB7 total $${p.totalCostUsd.toFixed(2)}`,
345
- ""
346
- );
347
- for (const [k, v] of Object.entries(report.meta)) lines.push(`- **${k}:** ${v}`);
348
- if (Object.keys(report.meta).length) lines.push("");
349
- lines.push("## Leaderboard", "");
350
- lines.push(
351
- "| # | Profile | Score (95% CI) | Solved (95% CI) | Runs | Cost | Tok in/out | p50 | p90 |"
352
- );
353
- lines.push("|---|---|--:|--:|--:|--:|--:|--:|--:|");
354
- report.profiles.forEach((r, i) => {
355
- lines.push(
356
- `| ${i + 1} | ${r.label} | ${pct(r.meanScore)}${ci(r.scoreCi)} | ${pct(r.solveRate)}${ci(r.passCi)} | ${r.n} | $${r.costUsd.toFixed(3)} | ${r.tokensIn}/${r.tokensOut} | ${(r.latencyP50Ms / 1e3).toFixed(1)}s | ${(r.latencyP90Ms / 1e3).toFixed(1)}s |`
357
- );
358
- });
359
- lines.push("");
360
- lines.push("## Score matrix \u2014 profile \xD7 axis", "");
361
- lines.push(`| Profile | ${report.axes.join(" | ")} |`);
362
- lines.push(`|---|${report.axes.map(() => "--:").join("|")}|`);
363
- for (const r of report.profiles) {
364
- const cells = report.axes.map((a) => {
365
- const v = r.perAxis[a];
366
- return v === void 0 ? "\xB7" : pct(v);
367
- });
368
- lines.push(`| ${r.label} | ${cells.join(" | ")} |`);
369
- }
370
- lines.push("");
371
- lines.push("> `\xB7` = the profile never ran that axis (blank, never zero).");
372
- return lines.join("\n");
373
- }
374
- function renderPairwiseMarkdown(verdicts, title = "Pairwise significance (paired, BH-corrected)") {
375
- const lines = [`## ${title}`, ""];
376
- if (verdicts.length === 0) return lines.concat("_no comparable pairs_").join("\n");
377
- lines.push("| A vs B | \u0394(b\u2212a) median | 95% CI | pairs | p | verdict |");
378
- lines.push("|---|--:|--:|--:|--:|---|");
379
- for (const v of verdicts) {
380
- const verdict = v.significant ? `**${v.delta >= 0 ? v.b : v.a} wins**` : "ns";
381
- lines.push(
382
- `| ${v.a} vs ${v.b} | ${v.delta >= 0 ? "+" : ""}${pct(v.delta)} | [${pct(v.ciLow)}, ${pct(v.ciHigh)}] | ${v.pairs} | ${v.p.toFixed(3)} | ${verdict} |`
383
- );
384
- }
385
- lines.push(
386
- "",
387
- "> `ns` = not significant after Benjamini\u2013Hochberg (or below the paired-count floor)."
388
- );
389
- return lines.join("\n");
390
- }
391
- function esc(s) {
392
- return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
393
- }
394
- function ramp(score) {
395
- const x = Math.max(0, Math.min(1, score));
396
- const r = Math.round(x < 0.5 ? 220 : 220 - (x - 0.5) * 2 * 160);
397
- const g = Math.round(x < 0.5 ? x * 2 * 170 : 170);
398
- return `rgb(${r},${g},60)`;
399
- }
400
- function renderLeaderboardSvg(report) {
401
- const rowH = 26;
402
- const labelW = 220;
403
- const cellW = 54;
404
- const pad = 16;
405
- const barAreaW = 360;
406
- const profiles = report.profiles;
407
- const barsH = profiles.length * rowH + pad;
408
- const heatTop = barsH + 48;
409
- const heatW = labelW + report.axes.length * cellW + pad;
410
- const heatH = profiles.length * rowH + pad;
411
- const width = Math.max(labelW + barAreaW + pad * 2, heatW);
412
- const height = heatTop + heatH + pad;
413
- const out = [];
414
- out.push(
415
- `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}" font-family="ui-sans-serif,system-ui,sans-serif" font-size="12">`
416
- );
417
- out.push(`<rect width="${width}" height="${height}" fill="white"/>`);
418
- out.push(
419
- `<text x="${pad}" y="${pad + 4}" font-weight="700" font-size="14">${esc(report.title)} \u2014 score</text>`
420
- );
421
- profiles.forEach((r, i) => {
422
- const y = pad + 12 + i * rowH;
423
- out.push(`<text x="${pad}" y="${y + 14}">${esc(r.label)}</text>`);
424
- const w = Math.round(r.meanScore * barAreaW);
425
- out.push(`<rect x="${labelW}" y="${y + 4}" width="${barAreaW}" height="16" fill="#eef0f2"/>`);
426
- out.push(
427
- `<rect x="${labelW}" y="${y + 4}" width="${w}" height="16" fill="${ramp(r.meanScore)}"/>`
428
- );
429
- out.push(`<text x="${labelW + barAreaW + 6}" y="${y + 16}">${pct(r.meanScore)}</text>`);
430
- });
431
- out.push(
432
- `<text x="${pad}" y="${heatTop - 12}" font-weight="700" font-size="14">profile \xD7 axis</text>`
433
- );
434
- report.axes.forEach((a, c) => {
435
- const x = labelW + c * cellW;
436
- out.push(
437
- `<text x="${x + cellW / 2}" y="${heatTop}" text-anchor="middle" fill="#555">${esc(a.length > 8 ? `${a.slice(0, 7)}\u2026` : a)}</text>`
438
- );
439
- });
440
- profiles.forEach((r, i) => {
441
- const y = heatTop + 8 + i * rowH;
442
- out.push(`<text x="${pad}" y="${y + 16}">${esc(r.label)}</text>`);
443
- report.axes.forEach((a, c) => {
444
- const x = labelW + c * cellW;
445
- const s = r.perAxis[a];
446
- if (s !== void 0) {
447
- out.push(
448
- `<rect x="${x}" y="${y}" width="${cellW - 3}" height="${rowH - 4}" fill="${ramp(s)}"/>`
449
- );
450
- out.push(
451
- `<text x="${x + (cellW - 3) / 2}" y="${y + 15}" text-anchor="middle" fill="white" font-size="10">${Math.round(100 * s)}</text>`
452
- );
453
- } else {
454
- out.push(
455
- `<rect x="${x}" y="${y}" width="${cellW - 3}" height="${rowH - 4}" fill="#f3f4f6"/>`
456
- );
457
- }
458
- });
459
- });
460
- out.push("</svg>");
461
- return out.join("\n");
462
- }
463
- function renderLeaderboardHtml(report) {
464
- const svg = renderLeaderboardSvg(report);
465
- const p = report.provenance;
466
- const rows = report.profiles.map(
467
- (r, i) => `<tr><td>${i + 1}</td><td>${esc(r.label)}</td><td class="n">${pct(r.meanScore)}</td><td class="n">${pct(r.solveRate)}</td><td class="n">${r.n}</td><td class="n">$${r.costUsd.toFixed(3)}</td>${report.axes.map((a) => {
468
- const v = r.perAxis[a];
469
- return `<td class="n">${v === void 0 ? "\xB7" : pct(v)}</td>`;
470
- }).join("")}</tr>`
471
- ).join("\n");
472
- const axisHead = report.axes.map((a) => `<th>${esc(a)}</th>`).join("");
473
- return `<!doctype html><html><head><meta charset="utf-8"><title>${esc(report.title)}</title>
474
- <style>
475
- body{font:14px ui-sans-serif,system-ui,sans-serif;margin:2rem;color:#111}
476
- h1{font-size:1.4rem}.sub{color:#555;margin:.2rem 0 1.2rem}
477
- table{border-collapse:collapse;margin-top:1rem}td,th{border:1px solid #e5e7eb;padding:.35rem .6rem}
478
- th{background:#f9fafb;text-align:left}.n{text-align:right;font-variant-numeric:tabular-nums}
479
- tr:first-child td{font-weight:600}
480
- </style></head><body>
481
- <h1>${esc(report.title)}</h1>
482
- <div class="sub">${p.profiles} profiles \xD7 ${p.axes} axes \xB7 ${p.records} runs \xB7 ${p.models.length} models \xB7 total $${p.totalCostUsd.toFixed(2)}</div>
483
- ${svg}
484
- <table><thead><tr><th>#</th><th>Profile</th><th>Score</th><th>Solved</th><th>Runs</th><th>Cost</th>${axisHead}</tr></thead>
485
- <tbody>
486
- ${rows}
487
- </tbody></table>
488
- </body></html>`;
489
- }
490
-
491
- // src/runtime/completion.ts
492
- function completionAuthorizes(v, policy) {
493
- if (!v?.done) return false;
494
- if (v.determinism === "deterministic") return true;
495
- return (v.confidence ?? 0) >= (policy?.minConfidence ?? 0.8);
496
- }
497
- function stopSentinel(seed) {
498
- let h = 2166136261;
499
- for (let i = 0; i < seed.length; i += 1) {
500
- h ^= seed.charCodeAt(i);
501
- h = Math.imul(h, 16777619);
502
- }
503
- return `<<<{{STOP:${(h >>> 0).toString(16).padStart(8, "0")}}}>>>`;
504
- }
505
- function sentinelCompletion(sentinel, opts) {
506
- return {
507
- assess({ history }) {
508
- const last = history[history.length - 1];
509
- const out = typeof last?.output === "string" ? last.output : "";
510
- const done = out.includes(sentinel);
511
- return {
512
- done,
513
- determinism: "probabilistic",
514
- confidence: done ? opts?.confidence ?? 0.9 : 0,
515
- reasons: done ? "agent emitted its assigned stop sentinel" : void 0,
516
- evidence: done && last ? [{ kind: "artifact", uri: `attempt:${last.index}` }] : []
517
- };
518
- }
519
- };
520
- }
521
- function deterministicCompletion(check) {
522
- return {
523
- assess({ history }) {
524
- const last = history[history.length - 1];
525
- if (last?.output === void 0) {
526
- return { done: false, determinism: "deterministic", reasons: "no output yet" };
527
- }
528
- const r = check(last.output, history);
529
- return {
530
- done: r.passed,
531
- determinism: "deterministic",
532
- reasons: r.reasons,
533
- evidence: [{ kind: "artifact", uri: `attempt:${last.index}` }]
534
- };
535
- }
536
- };
537
- }
538
-
539
- // src/runtime/define-leaderboard.ts
540
- import { execFileSync } from "child_process";
541
- import { mkdirSync, writeFileSync } from "fs";
542
- import { tmpdir } from "os";
543
- import { join } from "path";
544
- import {
545
- CODING_HARNESSES,
546
- expandProfileAxes,
547
- harnessAxisOf
548
- } from "@tangle-network/agent-eval";
549
- import {
550
- runProfileMatrix
551
- } from "@tangle-network/agent-eval/campaign";
552
- import { collectAgentResponseText } from "@tangle-network/sandbox";
553
-
554
- // src/runtime/loop-dispatch.ts
555
- function campaignTraceToLoopEmitter(trace) {
556
- return {
557
- emit(event) {
558
- trace.span(event.kind, { runId: event.runId, timestamp: event.timestamp, ...event.payload }).end();
559
- }
560
- };
561
- }
562
- async function runLoopForCell(opts, scenario, profile, ctx) {
563
- const loopOptions = opts.toLoopOptions(scenario, profile);
564
- return runLoopWithCampaignContext(opts, loopOptions, ctx, {
565
- model: profile.model?.default ?? modelFromLoopOptions(loopOptions),
566
- maximumCharge: typeof opts.maximumCharge === "function" ? opts.maximumCharge(scenario, profile) : opts.maximumCharge,
567
- resolveModel: opts.resolveCostModel ? (result) => opts.resolveCostModel?.(result, scenario, profile) : void 0
568
- });
569
- }
570
- async function runLoopWithCampaignContext(opts, loopOptions, ctx, cost) {
571
- const paid = await ctx.cost.runPaidCall({
572
- channel: "agent",
573
- actor: opts.costSource ?? "loop",
574
- model: cost.model,
575
- signal: ctx.signal,
576
- ...cost.maximumCharge ? { maximumCharge: cost.maximumCharge } : {},
577
- execute: (executionSignal) => runAgentRounds({
578
- ...loopOptions,
579
- ctx: {
580
- sandboxClient: opts.sandboxClient,
581
- signal: executionSignal,
582
- traceEmitter: opts.forwardTrace === false ? void 0 : campaignTraceToLoopEmitter(ctx.trace)
583
- }
584
- }),
585
- receipt: (result2) => loopCostReceipt(result2, cost.resolveModel?.(result2) ?? cost.model)
586
- });
587
- if (!paid.succeeded) {
588
- throw paid.error;
589
- }
590
- const result = paid.value;
591
- const toArtifact = opts.toArtifact ?? ((r) => r.winner?.output);
592
- return toArtifact(result);
593
- }
594
- function loopCostReceipt(result, model) {
595
- return {
596
- model,
597
- inputTokens: result.tokenUsage.input,
598
- outputTokens: result.tokenUsage.output,
599
- ...result.costUsd > 0 ? { actualCostUsd: result.costUsd } : {}
600
- };
601
- }
602
- function modelFromLoopOptions(options) {
603
- const profiles = options.agentRun ? [options.agentRun.profile] : options.agentRuns?.map((run) => run.profile) ?? [];
604
- const models = new Set(
605
- profiles.map((profile) => profile.model?.default).filter((model) => !!model)
606
- );
607
- if (models.size === 1) return [...models][0];
608
- return models.size > 1 ? "mixed" : "unknown";
609
- }
610
- function loopCampaignDispatch(opts) {
611
- return (scenario, ctx) => {
612
- const loopOptions = opts.toLoopOptions(scenario);
613
- return runLoopWithCampaignContext(opts, loopOptions, ctx, {
614
- model: modelFromLoopOptions(loopOptions),
615
- maximumCharge: typeof opts.maximumCharge === "function" ? opts.maximumCharge(scenario) : opts.maximumCharge,
616
- resolveModel: opts.resolveCostModel ? (result) => opts.resolveCostModel?.(result, scenario) : void 0
617
- });
618
- };
619
- }
620
- function loopDispatch(opts) {
621
- return (profile, scenario, ctx) => runLoopForCell(opts, scenario, profile, ctx);
622
- }
623
-
624
- // src/runtime/inline-sandbox-client.ts
625
- function isAsyncIterable(v) {
626
- return typeof v === "object" && v !== null && Symbol.asyncIterator in v;
627
- }
628
- async function settle(exec, task, signal) {
629
- const r = exec.execute(task, signal);
630
- if (isAsyncIterable(r)) {
631
- for await (const _ of r) {
632
- }
633
- return exec.resultArtifact();
634
- }
635
- return r;
636
- }
637
- function inlineSandboxClient(factory) {
638
- let seq = 0;
639
- return {
640
- async create(options) {
641
- const id = `inline-${seq++}`;
642
- const createOptions = options;
643
- return {
644
- id,
645
- async *streamPrompt(message, opts) {
646
- const controller = new AbortController();
647
- const callerSignal = opts?.signal;
648
- const onAbort = () => controller.abort(callerSignal?.reason ?? new Error("prompt aborted"));
649
- if (callerSignal) {
650
- if (callerSignal.aborted) onAbort();
651
- else callerSignal.addEventListener("abort", onAbort, { once: true });
652
- }
653
- const spec = { profile: { name: id }, harness: null };
654
- const exec = factory(spec, { signal: controller.signal, seams: { createOptions } });
655
- try {
656
- const artifact = await settle(exec, message, controller.signal);
657
- const out = artifact.out;
658
- const tokensIn = artifact.spent.tokens.input;
659
- const tokensOut = artifact.spent.tokens.output;
660
- const costUsd = artifact.spent.usd;
661
- if (tokensIn || tokensOut || costUsd) {
662
- yield {
663
- type: "llm_call",
664
- data: { tokensIn, tokensOut, costUsd }
665
- };
666
- }
667
- yield {
668
- type: "result",
669
- data: {
670
- finalText: out?.content ?? "",
671
- tokenUsage: {
672
- inputTokens: tokensIn,
673
- outputTokens: tokensOut
674
- },
675
- costUsd
676
- }
677
- };
678
- } finally {
679
- callerSignal?.removeEventListener("abort", onAbort);
680
- await exec.teardown("brutalKill").catch(() => {
681
- });
682
- }
683
- },
684
- async delete() {
685
- }
686
- };
687
- }
688
- };
689
- }
690
-
691
- // src/runtime/local-sandbox-client.ts
692
- import {
693
- canonicalCandidateDigest
694
- } from "@tangle-network/agent-interface";
695
-
696
- // src/runtime/stdio-mcp-client.ts
697
- import { spawn } from "child_process";
698
- import { createInterface } from "readline";
699
- import {
700
- validateAgentProfileSecurity
701
- } from "@tangle-network/agent-interface";
702
-
703
- // src/runtime/key-provider.ts
704
- function envKeyProvider(env = process.env) {
705
- return {
706
- async get(name) {
707
- const value = env[name];
708
- return value !== void 0 && value.trim().length > 0 ? value : void 0;
709
- }
710
- };
711
- }
712
- var mcpSecretEnvMetadataKey = "secretEnv";
713
- function secretEnvOfMcpServer(server) {
714
- const raw = server.metadata?.[mcpSecretEnvMetadataKey];
715
- if (raw === void 0 || raw === null) return void 0;
716
- if (typeof raw !== "object" || Array.isArray(raw)) {
717
- throw new ValidationError(
718
- `secretEnvOfMcpServer: metadata.${mcpSecretEnvMetadataKey} must be an object mapping env var name -> key name`
719
- );
720
- }
721
- const entries = Object.entries(raw);
722
- if (entries.length === 0) return void 0;
723
- for (const [envName, keyName] of entries) {
724
- if (!envName.trim() || typeof keyName !== "string" || !keyName.trim()) {
725
- throw new ValidationError(
726
- `secretEnvOfMcpServer: metadata.${mcpSecretEnvMetadataKey}['${envName}'] must name a non-empty provider key`
727
- );
728
- }
729
- }
730
- return raw;
731
- }
732
- async function resolveSecretEnv(secretEnv, keys, label) {
733
- const entries = Object.entries(secretEnv);
734
- if (entries.length === 0) return {};
735
- if (!keys) {
736
- throw new ValidationError(
737
- `${label} declares secret env (${entries.map(([e]) => e).join(", ")}) but no KeyProvider was supplied \u2014 refusing to boot an external server keyless`
738
- );
739
- }
740
- const resolved = {};
741
- for (const [envName, keyName] of entries) {
742
- const value = await keys.get(keyName);
743
- if (value === void 0) {
744
- throw new ValidationError(
745
- `${label}: the KeyProvider holds no value for '${keyName}' (wanted for env ${envName}) \u2014 provision the key or drop the grant`
746
- );
747
- }
748
- resolved[envName] = value;
749
- }
750
- return resolved;
751
- }
752
-
753
- // src/runtime/mcp-environment.ts
754
- async function rpc(endpoint, body) {
755
- let lastErr;
756
- for (let attempt = 0; attempt < 4; attempt += 1) {
757
- try {
758
- const r = await fetch(endpoint.url, {
759
- method: "POST",
760
- headers: { "content-type": "application/json", ...endpoint.headers ?? {} },
761
- body: JSON.stringify(body)
762
- });
763
- const text = await r.text();
764
- const dataLines = text.split("\n").filter((l) => l.startsWith("data:")).map((l) => l.slice(5).trim());
765
- const payload = dataLines.length ? dataLines[dataLines.length - 1] : text;
766
- try {
767
- return { status: r.status, json: JSON.parse(payload ?? "null") };
768
- } catch {
769
- return { status: r.status, json: text };
770
- }
771
- } catch (err) {
772
- lastErr = err;
773
- await new Promise((res) => setTimeout(res, 1e3 * (attempt + 1)));
774
- }
775
- }
776
- throw new Error(
777
- `mcp rpc ${endpoint.url} failed after 4 attempts: ${lastErr instanceof Error ? lastErr.message : String(lastErr)}`
778
- );
779
- }
780
- function sanitizeMcpToolSchema(s) {
781
- const o = s && typeof s === "object" ? s : {};
782
- const banned = o.oneOf || o.anyOf || o.allOf || o.not || o.enum;
783
- if (o.type === "object" && !banned && o.properties && typeof o.properties === "object") {
784
- return {
785
- type: "object",
786
- properties: o.properties,
787
- ...Array.isArray(o.required) ? { required: o.required } : {}
788
- };
789
- }
790
- return { type: "object", properties: {} };
791
- }
792
- function createMcpEnvironment(opts) {
793
- const endpoints = /* @__PURE__ */ new Map();
794
- const maxChars = opts.maxResultChars ?? 1500;
795
- return {
796
- name: opts.name,
797
- async open(task) {
798
- const { handle, endpoint } = await opts.open(task);
799
- endpoints.set(handle.id, endpoint);
800
- return handle;
801
- },
802
- async tools(task, handle) {
803
- const endpoint = endpoints.get(handle.id);
804
- if (!endpoint) throw new Error(`${opts.name}: tools() before open() for ${handle.id}`);
805
- const { json } = await rpc(endpoint, {
806
- jsonrpc: "2.0",
807
- id: 1,
808
- method: "tools/list",
809
- params: {}
810
- });
811
- const all = (json.result?.tools ?? []).map(
812
- (t) => ({
813
- type: "function",
814
- function: {
815
- name: t.name,
816
- description: (t.description ?? "").slice(0, 1e3),
817
- parameters: sanitizeMcpToolSchema(t.inputSchema)
818
- }
819
- })
820
- );
821
- return opts.selectTools ? opts.selectTools(task, all) : all;
822
- },
823
- async call(handle, name, args) {
824
- const endpoint = endpoints.get(handle.id);
825
- if (!endpoint) return "ERROR: workspace closed";
826
- const { json } = await rpc(endpoint, {
827
- jsonrpc: "2.0",
828
- id: 2,
829
- method: "tools/call",
830
- params: { name, arguments: args }
831
- });
832
- const result = json ?? {};
833
- if (result.error) return `ERROR: ${JSON.stringify(result.error).slice(0, 300)}`;
834
- const text = result.result?.content?.map((c) => c.text ?? "").join("\n") ?? JSON.stringify(result.result ?? json);
835
- return text.slice(0, maxChars);
836
- },
837
- score: (task, handle) => opts.score(task, handle),
838
- async close(handle) {
839
- endpoints.delete(handle.id);
840
- await opts.close?.(handle);
841
- }
842
- };
843
- }
844
-
845
- // src/runtime/stdio-mcp-client.ts
846
- var PROTOCOL_VERSION = "2024-11-05";
847
- var SAFE_INHERITED_ENV_NAMES = [
848
- "PATH",
849
- "LANG",
850
- "LC_ALL",
851
- "LC_CTYPE",
852
- "TZ",
853
- "SYSTEMROOT",
854
- "WINDIR",
855
- "COMSPEC",
856
- "PATHEXT"
857
- ];
858
- function inheritedStdioEnv(source = process.env) {
859
- const env = {};
860
- for (const name of SAFE_INHERITED_ENV_NAMES) {
861
- const value = source[name];
862
- if (value !== void 0) env[name] = value;
863
- }
864
- return env;
865
- }
866
- var McpSpawnFault = class extends Error {
867
- };
868
- async function connectStdioMcp(spec) {
869
- const timeoutMs = spec.timeoutMs ?? 3e4;
870
- const protectedValues = Object.values(spec.protectedEnv ?? {});
871
- const redactReason = (value) => redactProtectedReason(value, protectedValues);
872
- const child = spawn(spec.command, spec.args ?? [], {
873
- ...spec.cwd ? { cwd: spec.cwd } : {},
874
- stdio: ["pipe", "pipe", "pipe"],
875
- env: { ...inheritedStdioEnv(), ...spec.env, ...spec.protectedEnv }
876
- });
877
- const stderr = [];
878
- child.stderr.on("data", (d) => stderr.push(String(d)));
879
- const stderrTail = () => stderr.length > 0 ? `
880
- stderr:
881
- ${redactReason(stderr.join("")).slice(-2e3)}` : "";
882
- let nextId = 1;
883
- let spawnFault;
884
- let closed = false;
885
- const pending = /* @__PURE__ */ new Map();
886
- const failAllPending = (err) => {
887
- for (const p of pending.values()) {
888
- clearTimeout(p.timer);
889
- p.reject(err);
890
- }
891
- pending.clear();
892
- };
893
- child.on("error", (err) => {
894
- const code = err.code;
895
- spawnFault = code === "ENOENT" ? new McpSpawnFault(
896
- `'${spec.command}' not found in PATH (setup bug, not a failed candidate)`
897
- ) : new McpSpawnFault(`'${spec.command}' failed to spawn: ${err.message}`);
898
- failAllPending(spawnFault);
899
- });
900
- child.stdin.on("error", (err) => {
901
- failAllPending(
902
- new Error(redactReason(`writing to MCP server stdin failed: ${err.message}${stderrTail()}`))
903
- );
904
- });
905
- child.on("close", (code, signal) => {
906
- failAllPending(
907
- spawnFault ?? new Error(
908
- redactReason(
909
- `MCP server exited (code ${code}, signal ${signal}) before serving${stderrTail()}`
910
- )
911
- )
912
- );
913
- });
914
- const rl = createInterface({ input: child.stdout });
915
- rl.on("line", (line) => {
916
- let msg;
917
- try {
918
- msg = JSON.parse(line);
919
- } catch {
920
- return;
921
- }
922
- if (!msg || typeof msg !== "object" || typeof msg.id !== "number") return;
923
- const p = pending.get(msg.id);
924
- if (!p) return;
925
- pending.delete(msg.id);
926
- clearTimeout(p.timer);
927
- p.resolve(msg);
928
- });
929
- const send = (msg) => {
930
- child.stdin.write(`${JSON.stringify(msg)}
931
- `);
932
- };
933
- const request = (method, params, timeoutMessage = `MCP server did not answer '${method}' within ${timeoutMs}ms`) => new Promise((resolve, reject) => {
934
- if (spawnFault) return reject(spawnFault);
935
- if (closed) return reject(new Error("MCP connection closed"));
936
- const id = nextId++;
937
- const timer = setTimeout(() => {
938
- pending.delete(id);
939
- reject(new Error(redactReason(`${timeoutMessage}${stderrTail()}`)));
940
- }, timeoutMs);
941
- pending.set(id, { resolve, reject, timer });
942
- try {
943
- send({ jsonrpc: "2.0", id, method, ...params ? { params } : {} });
944
- } catch (err) {
945
- pending.delete(id);
946
- clearTimeout(timer);
947
- reject(
948
- new Error(
949
- redactReason(
950
- `writing to MCP server stdin failed: ${err instanceof Error ? err.message : String(err)}${stderrTail()}`
951
- )
952
- )
953
- );
954
- }
955
- });
956
- const close = async () => {
957
- if (closed) return;
958
- closed = true;
959
- rl.close();
960
- failAllPending(new Error("MCP connection closed"));
961
- child.kill("SIGKILL");
962
- };
963
- const handshakeTimeout = `MCP server did not complete the handshake within ${timeoutMs}ms`;
964
- try {
965
- const init = await request(
966
- "initialize",
967
- {
968
- protocolVersion: PROTOCOL_VERSION,
969
- capabilities: {},
970
- clientInfo: { name: "agent-runtime-local-mcp", version: "0" }
971
- },
972
- handshakeTimeout
973
- );
974
- if (init.error)
975
- throw new Error(
976
- redactReason(`initialize errored: ${JSON.stringify(init.error)}${stderrTail()}`)
977
- );
978
- send({ jsonrpc: "2.0", method: "notifications/initialized" });
979
- const list = await request("tools/list", void 0, handshakeTimeout);
980
- if (list.error)
981
- throw new Error(
982
- redactReason(`tools/list errored: ${JSON.stringify(list.error)}${stderrTail()}`)
983
- );
984
- const rawTools = list.result?.tools;
985
- if (!Array.isArray(rawTools))
986
- throw new Error(`tools/list result has no tools array${stderrTail()}`);
987
- const tools = redactProtectedValue(rawTools, protectedValues).value;
988
- const callTool = async (name, args) => {
989
- const res = await request("tools/call", { name, arguments: args });
990
- if (res.error) return `ERROR: ${redactReason(JSON.stringify(res.error)).slice(0, 300)}`;
991
- const result = res.result;
992
- const chunks = result?.content?.map((c) => c.text ?? "");
993
- const joined = chunks?.join("\n") ?? JSON.stringify(result ?? null);
994
- const collapsed = chunks?.join("");
995
- let text = joined;
996
- if (collapsed !== void 0) {
997
- const collapsedRedacted = redactReason(collapsed);
998
- if (collapsedRedacted !== collapsed) text = collapsedRedacted;
999
- }
1000
- return redactReason(result?.isError ? `ERROR: ${text}` : text);
1001
- };
1002
- return { tools, callTool, close };
1003
- } catch (err) {
1004
- await close();
1005
- throw err;
1006
- }
1007
- }
1008
- async function materializeLocalMcp(profile, opts = {}) {
1009
- const security = validateAgentProfileSecurity(profile, opts.profileSecurityPolicy);
1010
- if (!security.ok) {
1011
- const reasons = security.issues.filter((issue) => issue.level === "error").map((issue) => issue.message).join("; ");
1012
- throw new ValidationError(`materializeLocalMcp: profile host execution refused: ${reasons}`);
1013
- }
1014
- const maxResultChars = opts.maxResultChars ?? 2e3;
1015
- const connections = [];
1016
- const routes = /* @__PURE__ */ new Map();
1017
- const tools = [];
1018
- const close = async () => {
1019
- await Promise.all(connections.map((c) => c.close()));
1020
- };
1021
- try {
1022
- for (const [key, server] of Object.entries(profile.mcp ?? {})) {
1023
- if (server.enabled === false) continue;
1024
- const transport = server.transport ?? "stdio";
1025
- if (transport !== "stdio") {
1026
- throw new ValidationError(
1027
- `materializeLocalMcp: profile.mcp['${key}'] has transport '${transport}' \u2014 the same-host client only spawns stdio servers; a remote (http/sse) server needs an http MCP client (not yet built here) or the sandbox backend. Failing loud: scoring this profile without its declared server would fake the with/without ablation`
1028
- );
1029
- }
1030
- if (!server.command || server.command.trim().length === 0) {
1031
- throw new ValidationError(
1032
- `materializeLocalMcp: profile.mcp['${key}'] declares a stdio server with no command`
1033
- );
1034
- }
1035
- const secretRefs = secretEnvOfMcpServer(server);
1036
- const provisioned = secretRefs ? await resolveSecretEnv(
1037
- secretRefs,
1038
- opts.keys,
1039
- `materializeLocalMcp: profile.mcp['${key}']`
1040
- ) : void 0;
1041
- const conn = await connectStdioMcp({
1042
- command: server.command,
1043
- ...server.args ? { args: server.args } : {},
1044
- ...server.cwd ? { cwd: server.cwd } : {},
1045
- ...server.env ? { env: server.env } : {},
1046
- ...provisioned ? { protectedEnv: provisioned } : {},
1047
- ...opts.timeoutMs !== void 0 ? { timeoutMs: opts.timeoutMs } : {}
1048
- });
1049
- connections.push(conn);
1050
- const prefix = key.replace(/[^a-zA-Z0-9_-]/g, "_");
1051
- for (const t of conn.tools) {
1052
- const name = `${prefix}__${t.name.replace(/[^a-zA-Z0-9_-]/g, "_")}`;
1053
- if (routes.has(name)) {
1054
- throw new ValidationError(
1055
- `materializeLocalMcp: namespaced tool '${name}' collides across servers \u2014 rename the profile.mcp keys`
1056
- );
1057
- }
1058
- routes.set(name, { conn, tool: t.name });
1059
- tools.push({
1060
- type: "function",
1061
- function: {
1062
- name,
1063
- description: (t.description ?? "").slice(0, 1e3),
1064
- parameters: sanitizeMcpToolSchema(t.inputSchema)
1065
- }
1066
- });
1067
- }
1068
- }
1069
- } catch (err) {
1070
- await close();
1071
- throw err;
1072
- }
1073
- return {
1074
- tools,
1075
- owns: (name) => routes.has(name),
1076
- call: async (name, args) => {
1077
- const route = routes.get(name);
1078
- if (!route) throw new Error(`materializeLocalMcp: unknown tool '${name}'`);
1079
- const out = await route.conn.callTool(route.tool, args);
1080
- return out.length > maxResultChars ? out.slice(0, maxResultChars) : out;
1081
- },
1082
- close
1083
- };
1084
- }
1085
-
1086
- // src/runtime/local-sandbox-client.ts
1087
- function localSandboxClient(opts) {
1088
- if (opts.profileSecurityPolicy?.allowLocalMcp && opts.profile === void 0) {
1089
- throw new ValidationError(
1090
- "localSandboxClient: allowLocalMcp requires a fixed author-controlled profile; dynamic profiles need a real sandbox"
1091
- );
1092
- }
1093
- const trustedProfileDigest = opts.profileSecurityPolicy?.allowLocalMcp && opts.profile !== void 0 ? canonicalCandidateDigest(opts.profile) : void 0;
1094
- const maxTurns = opts.maxTurns ?? 8;
1095
- let seq = 0;
1096
- return {
1097
- async create(options) {
1098
- const profile = options?.backend?.profile ?? opts.profile ?? {};
1099
- const policyApplies = opts.profileSecurityPolicy !== void 0 && (!opts.profileSecurityPolicy.allowLocalMcp || trustedProfileDigest !== void 0 && canonicalCandidateDigest(profile) === trustedProfileDigest);
1100
- const mcp = await materializeLocalMcp(profile, {
1101
- ...opts.keys ? { keys: opts.keys } : {},
1102
- ...policyApplies ? { profileSecurityPolicy: opts.profileSecurityPolicy } : {}
1103
- });
1104
- const brain = routerBrain(
1105
- {
1106
- routerBaseUrl: opts.router.baseUrl,
1107
- routerKey: opts.router.key,
1108
- model: opts.router.model
1109
- },
1110
- opts.temperature !== void 0 ? { temperature: opts.temperature } : {}
1111
- );
1112
- const system = [profile.prompt?.systemPrompt, ...profile.prompt?.instructions ?? []].filter((s) => typeof s === "string" && s.trim().length > 0).join("\n\n");
1113
- const id = `local-${seq++}`;
1114
- return {
1115
- id,
1116
- async *streamPrompt(message, popts) {
1117
- let costUsd = 0;
1118
- const chat = async (messages, tools) => {
1119
- const r2 = await brain(messages, tools);
1120
- if (r2.costUsd) costUsd += r2.costUsd;
1121
- return r2;
1122
- };
1123
- const r = await runBrainLoop({
1124
- chat,
1125
- tools: mcp.tools,
1126
- execute: (name, args) => mcp.call(name, args),
1127
- initialMessages: [
1128
- ...system ? [{ role: "system", content: system }] : [],
1129
- { role: "user", content: message }
1130
- ],
1131
- maxTurns,
1132
- hooks: { stopBefore: () => popts?.signal?.aborted === true }
1133
- });
1134
- if (r.usage.input || r.usage.output || costUsd) {
1135
- yield {
1136
- type: "llm_call",
1137
- data: { tokensIn: r.usage.input, tokensOut: r.usage.output, costUsd }
1138
- };
1139
- }
1140
- yield {
1141
- type: "result",
1142
- data: {
1143
- finalText: r.final,
1144
- tokenUsage: { inputTokens: r.usage.input, outputTokens: r.usage.output },
1145
- costUsd
1146
- }
1147
- };
1148
- },
1149
- async delete() {
1150
- await mcp.close();
1151
- }
1152
- };
1153
- }
1154
- };
1155
- }
1156
-
1157
- // src/runtime/resolve-sandbox-client.ts
1158
- function resolveSandboxClient(opts) {
1159
- switch (opts.backend) {
1160
- case "sandbox": {
1161
- if (!opts.sandboxClient) {
1162
- throw new Error("resolveSandboxClient: backend 'sandbox' requires opts.sandboxClient");
1163
- }
1164
- return opts.sandboxClient;
1165
- }
1166
- case "bridge": {
1167
- const bridge = opts.bridge;
1168
- if (!bridge?.bearer || !bridge.model) {
1169
- throw new Error(
1170
- "resolveSandboxClient: backend 'bridge' requires bridge.bearer and bridge.model"
1171
- );
1172
- }
1173
- return inlineSandboxClient(
1174
- createExecutor({
1175
- backend: "bridge",
1176
- bridgeUrl: bridge.url ?? "http://127.0.0.1:3355",
1177
- bridgeBearer: bridge.bearer,
1178
- model: bridge.model,
1179
- timeoutMs: bridge.timeoutMs
1180
- })
1181
- );
1182
- }
1183
- case "router": {
1184
- const router = opts.router;
1185
- if (!router?.baseUrl || !router.key || !router.model) {
1186
- throw new Error(
1187
- "resolveSandboxClient: backend 'router' requires router.baseUrl, router.key and router.model"
1188
- );
1189
- }
1190
- return inlineSandboxClient(
1191
- createExecutor({
1192
- backend: "router",
1193
- routerBaseUrl: router.baseUrl,
1194
- routerKey: router.key,
1195
- model: router.model
1196
- })
1197
- );
1198
- }
1199
- case "local": {
1200
- const local = opts.local;
1201
- if (!local?.router?.baseUrl || !local.router.key || !local.router.model) {
1202
- throw new Error(
1203
- "resolveSandboxClient: backend 'local' requires local.router.baseUrl, local.router.key and local.router.model"
1204
- );
1205
- }
1206
- return localSandboxClient(local);
1207
- }
1208
- }
1209
- }
1210
-
1211
- // src/runtime/steering-drivers.ts
1212
- function decideUntilValidOrCapped(history, maxIterations) {
1213
- if (history.some((it) => it.verdict?.valid)) return "pick-winner";
1214
- return history.length < maxIterations ? "refine" : "fail";
1215
- }
1216
- function naiveDriver(options) {
1217
- const { continuation, applyContinuation, maxIterations, name = "naive" } = options;
1218
- return {
1219
- name,
1220
- plan(task, history) {
1221
- if (history.length === 0) return Promise.resolve([task]);
1222
- const last = history[history.length - 1];
1223
- if (last?.verdict?.valid) return Promise.resolve([]);
1224
- if (history.length >= maxIterations) return Promise.resolve([]);
1225
- return Promise.resolve([applyContinuation(task, continuation)]);
1226
- },
1227
- decide(history) {
1228
- return decideUntilValidOrCapped(history, maxIterations);
1229
- },
1230
- // Pure refine topology (one task per round) — render the steer move.
1231
- describePlan() {
1232
- return { kind: "refine", rationale: "naive fixed continuation (no grade signal)" };
1233
- }
1234
- };
1235
- }
1236
- function dumbDriver(options) {
1237
- const { onPass, onFail, applyContinuation, maxIterations, name = "dumb" } = options;
1238
- return {
1239
- name,
1240
- plan(task, history) {
1241
- if (history.length === 0) return Promise.resolve([task]);
1242
- const last = history[history.length - 1];
1243
- const passed = last?.verdict?.valid === true;
1244
- if (passed) return Promise.resolve([]);
1245
- if (history.length >= maxIterations) return Promise.resolve([]);
1246
- const continuation = passed ? onPass : onFail;
1247
- return Promise.resolve([applyContinuation(task, continuation)]);
1248
- },
1249
- decide(history) {
1250
- return decideUntilValidOrCapped(history, maxIterations);
1251
- },
1252
- describePlan() {
1253
- return { kind: "refine", rationale: "dumb pass/fail-only continuation (no grader findings)" };
1254
- }
1255
- };
1256
- }
1257
-
1258
- // src/runtime/define-leaderboard.ts
1259
- function argOf(argv, name) {
1260
- const i = argv.indexOf(`--${name}`);
1261
- if (i >= 0 && i + 1 < argv.length) return argv[i + 1];
1262
- return void 0;
1263
- }
1264
- function splitList(v) {
1265
- if (v === void 0) return void 0;
1266
- const parts = v.split(",").map((s) => s.trim()).filter(Boolean);
1267
- return parts.length > 0 ? parts : void 0;
1268
- }
1269
- function withSnapshot(model, snapshot) {
1270
- return model.includes("@") ? model : `${model}@${snapshot}`;
1271
- }
1272
- function bareModel(model) {
1273
- return model.split("@")[0] ?? model;
1274
- }
1275
- function gitSha() {
1276
- try {
1277
- return execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim();
1278
- } catch {
1279
- return "unknown";
1280
- }
1281
- }
1282
- function normalizeScore(s) {
1283
- return typeof s === "number" ? { composite: s } : s;
1284
- }
1285
- function defineLeaderboard(spec) {
1286
- const caseId = (c) => {
1287
- const id = spec.caseId ? spec.caseId(c) : c.id;
1288
- if (typeof id !== "string" || id.length === 0) {
1289
- throw new Error(
1290
- `defineLeaderboard(${spec.name}): every case needs a stable string id \u2014 give cases an \`id\` property or supply spec.caseId`
1291
- );
1292
- }
1293
- return id;
1294
- };
1295
- const selectCases = (ids) => {
1296
- if (!ids) return spec.cases;
1297
- const byId = new Map(spec.cases.map((c) => [caseId(c), c]));
1298
- return ids.map((id) => {
1299
- const c = byId.get(id);
1300
- if (c === void 0) {
1301
- throw new Error(
1302
- `defineLeaderboard(${spec.name}): unknown case "${id}" (have: ${[...byId.keys()].join(", ")})`
1303
- );
1304
- }
1305
- return c;
1306
- });
1307
- };
1308
- const scoreJudge = {
1309
- name: `${spec.name}-score`,
1310
- dimensions: [{ key: "composite", description: `${spec.name} case score` }],
1311
- score({ artifact, scenario }) {
1312
- const s = normalizeScore(spec.score(artifact, scenario.case));
1313
- return {
1314
- composite: s.composite,
1315
- dimensions: { composite: s.composite, ...s.dimensions },
1316
- notes: s.notes ?? ""
1317
- };
1318
- }
1319
- };
1320
- async function run(argv = process.argv.slice(2)) {
1321
- const args = {};
1322
- for (const name of [
1323
- "backend",
1324
- "harnesses",
1325
- "models",
1326
- "cases",
1327
- "shots",
1328
- "reps",
1329
- "model-snapshot",
1330
- "run-dir",
1331
- "export-dir"
1332
- ]) {
1333
- args[name] = argOf(argv, name);
1334
- }
1335
- for (const [name, flag] of Object.entries(spec.flags ?? {})) {
1336
- args[name] = argOf(argv, name) ?? flag.default;
1337
- }
1338
- const backendName = args.backend ?? "sandbox";
1339
- const shots = Number(args.shots ?? spec.shots ?? 1);
1340
- const reps = Number(args.reps ?? spec.reps ?? 1);
1341
- const snapshot = args["model-snapshot"] ?? "leaderboard";
1342
- const runDir = args["run-dir"] ?? join(tmpdir(), `leaderboard-${spec.name}-${Date.now()}-${process.pid}`);
1343
- const exportDir = args["export-dir"] ?? join(runDir, "export");
1344
- mkdirSync(runDir, { recursive: true });
1345
- const cases = selectCases(splitList(args.cases));
1346
- if (cases.length === 0) throw new Error(`defineLeaderboard(${spec.name}): no cases to run`);
1347
- const scenarios = cases.map((c) => ({
1348
- id: caseId(c),
1349
- kind: spec.name,
1350
- case: c
1351
- }));
1352
- const harnesses = splitList(args.harnesses) ?? spec.axis?.harnesses ?? CODING_HARNESSES;
1353
- const rawModels = splitList(args.models) ?? spec.axis?.models ?? (spec.baseProfile?.model?.default !== void 0 ? [spec.baseProfile.model.default] : void 0);
1354
- if (!rawModels || rawModels.length === 0) {
1355
- throw new Error(
1356
- `defineLeaderboard(${spec.name}): no models \u2014 pass --models, set spec.axis.models, or give spec.baseProfile a model.default`
1357
- );
1358
- }
1359
- const models = rawModels.map((m) => withSnapshot(m, snapshot));
1360
- const base = spec.baseProfile ?? { name: spec.name, model: { default: bareModel(models[0] ?? "") } };
1361
- const profiles = expandProfileAxes({ base, harnesses, models });
1362
- const ctx = {
1363
- name: spec.name,
1364
- backend: backendName,
1365
- runDir,
1366
- exportDir,
1367
- args,
1368
- harnesses,
1369
- models,
1370
- caseIds: scenarios.map((s) => s.id),
1371
- shots,
1372
- reps
1373
- };
1374
- const backends = {
1375
- sandbox: () => {
1376
- throw new Error(
1377
- `defineLeaderboard(${spec.name}): the 'sandbox' backend needs your product's real SandboxClient \u2014 supply spec.backends.sandbox (e.g. () => new SandboxClient({ apiKey, baseUrl }))`
1378
- );
1379
- },
1380
- "cli-bridge": () => {
1381
- const bearer = process.env.BRIDGE_BEARER ?? process.env.CLI_BRIDGE_BEARER;
1382
- if (!bearer) {
1383
- throw new Error(
1384
- `defineLeaderboard(${spec.name}): backend 'cli-bridge' needs BRIDGE_BEARER or CLI_BRIDGE_BEARER set`
1385
- );
1386
- }
1387
- return resolveSandboxClient({
1388
- backend: "bridge",
1389
- bridge: {
1390
- url: process.env.CLI_BRIDGE_URL,
1391
- bearer,
1392
- model: bareModel(models[0] ?? ""),
1393
- timeoutMs: 9e5
1394
- }
1395
- });
1396
- },
1397
- ...spec.backends
1398
- };
1399
- const makeClient = backends[backendName];
1400
- if (!makeClient) {
1401
- throw new Error(
1402
- `defineLeaderboard(${spec.name}): unknown backend "${backendName}" (have: ${Object.keys(backends).join(", ")})`
1403
- );
1404
- }
1405
- const sandboxClient = makeClient();
1406
- const promptById = /* @__PURE__ */ new Map();
1407
- for (const s of scenarios) promptById.set(s.id, await spec.prompt(s.case));
1408
- const promptOf = (s) => {
1409
- const p = promptById.get(s.id);
1410
- if (p === void 0)
1411
- throw new Error(`defineLeaderboard(${spec.name}): no prompt for case "${s.id}"`);
1412
- return p;
1413
- };
1414
- let shotNonce = 0;
1415
- const maximumCharge = spec.maximumCharge;
1416
- const dispatch = spec.dispatch ?? ((profile, scenario, dispatchCtx) => {
1417
- const cellDispatch = loopDispatch({
1418
- sandboxClient,
1419
- maximumCharge: typeof maximumCharge === "function" ? (cellScenario, cellProfile) => maximumCharge(cellProfile, cellScenario) : maximumCharge,
1420
- resolveCostModel: (result, _cellScenario, cellProfile) => {
1421
- if (!spec.resolveModel) return cellProfile.model?.default;
1422
- const served = new Set(
1423
- result.iterations.map((iteration) => spec.resolveModel?.(iteration.events)).filter((model) => model !== void 0)
1424
- );
1425
- if (served.size > 1) {
1426
- throw new Error(
1427
- `defineLeaderboard(${spec.name}): one cell reported multiple served models: ${[...served].join(", ")}`
1428
- );
1429
- }
1430
- return [...served][0] ?? cellProfile.model?.default;
1431
- },
1432
- toLoopOptions: (cellScenario, cellProfile) => {
1433
- const axis = harnessAxisOf(cellProfile);
1434
- const modelId = bareModel(axis?.model ?? models[0] ?? "");
1435
- return {
1436
- // naiveDriver = the no-signal retry floor: re-run the same case as
1437
- // an independent attempt until one scores (>0) or the shot cap.
1438
- driver: naiveDriver({
1439
- continuation: "",
1440
- applyContinuation: (task) => task,
1441
- maxIterations: shots
1442
- }),
1443
- agentRun: {
1444
- profile: cellProfile,
1445
- taskToPrompt: (s) => `${promptOf(s)}
1446
-
1447
- <!-- independent-attempt:${shotNonce++} -->`,
1448
- ...axis ? {
1449
- sandboxOverrides: {
1450
- backend: {
1451
- type: axis.harness,
1452
- model: { ...spec.modelBackend, model: modelId }
1453
- }
1454
- }
1455
- } : {}
1456
- },
1457
- output: {
1458
- parse: (events) => spec.parseOutput ? spec.parseOutput(events, cellScenario.case) : (
1459
- // The default decode produces string — the TArtifact
1460
- // default. A structured-TArtifact spec supplies parseOutput
1461
- // (documented on the field), so this cast never lies.
1462
- collectAgentResponseText(events) ?? ""
1463
- )
1464
- },
1465
- validator: {
1466
- validate: async (output) => {
1467
- const s = normalizeScore(spec.score(output, cellScenario.case));
1468
- return { valid: s.composite > 0, score: s.composite };
1469
- }
1470
- },
1471
- task: cellScenario,
1472
- maxIterations: shots
1473
- };
1474
- },
1475
- toArtifact: (result) => {
1476
- for (const iter of result.iterations) {
1477
- spec.onCellEvents?.(iter.events, scenario.case, {
1478
- index: iter.index,
1479
- ...iter.error ? { error: iter.error.message } : {},
1480
- ...iter.verdict ? { verdict: { score: iter.verdict.score } } : {}
1481
- });
1482
- }
1483
- return result.winner?.output;
1484
- }
1485
- });
1486
- return cellDispatch(profile, scenario, dispatchCtx);
1487
- });
1488
- await spec.setup?.(ctx);
1489
- try {
1490
- const result = await runProfileMatrix({
1491
- profiles,
1492
- scenarios,
1493
- dispatch,
1494
- judges: spec.judges ?? [scoreJudge],
1495
- runDir,
1496
- commitSha: gitSha(),
1497
- reps,
1498
- ...spec.matrix
1499
- });
1500
- if (spec.export) {
1501
- await spec.export(result, ctx);
1502
- } else {
1503
- mkdirSync(exportDir, { recursive: true });
1504
- writeFileSync(join(runDir, "matrix-result.json"), `${JSON.stringify(result, null, 2)}
1505
- `);
1506
- const table = renderLeaderboardMarkdown(
1507
- leaderboard(result.records, { title: spec.name, meta: { backend: backendName } })
1508
- );
1509
- writeFileSync(join(exportDir, "leaderboard.md"), table);
1510
- console.log(table);
1511
- }
1512
- return result;
1513
- } finally {
1514
- await spec.teardown?.(ctx);
1515
- }
1516
- }
1517
- function toBenchmarkAdapter() {
1518
- return {
1519
- name: spec.name,
1520
- async preflight() {
1521
- const seen = /* @__PURE__ */ new Set();
1522
- for (const c of spec.cases) {
1523
- const id = caseId(c);
1524
- if (seen.has(id)) {
1525
- throw new Error(`defineLeaderboard(${spec.name}): duplicate case id "${id}"`);
1526
- }
1527
- seen.add(id);
1528
- }
1529
- },
1530
- async loadTasks(opts) {
1531
- const selected = selectCases(opts?.ids);
1532
- const limited = opts?.limit !== void 0 ? selected.slice(0, opts.limit) : selected;
1533
- return Promise.all(
1534
- limited.map(async (c) => ({
1535
- id: caseId(c),
1536
- prompt: await spec.prompt(c),
1537
- metadata: { case: c }
1538
- }))
1539
- );
1540
- },
1541
- async judge(task, artifact) {
1542
- const [c] = selectCases([task.id]);
1543
- if (c === void 0)
1544
- throw new Error(`defineLeaderboard(${spec.name}): no case "${task.id}"`);
1545
- const s = normalizeScore(spec.score(artifact, c));
1546
- return { resolved: s.composite > 0, score: s.composite, detail: s.notes };
1547
- },
1548
- async goldArtifact() {
1549
- return void 0;
1550
- }
1551
- };
1552
- }
1553
- return { run, toBenchmarkAdapter };
1554
- }
1555
-
1556
- // src/runtime/harvest-corpus.ts
1557
- async function harvestCorpus(opts) {
1558
- const concurrency = Math.max(1, opts.concurrency ?? 4);
1559
- const report = { runsObserved: 0, findings: 0, learned: 0, failures: [] };
1560
- const iterator = Symbol.asyncIterator in Object(opts.runs) ? opts.runs[Symbol.asyncIterator]() : (async function* () {
1561
- yield* opts.runs;
1562
- })();
1563
- let consumed = 0;
1564
- let done = false;
1565
- const next = async () => {
1566
- if (done || opts.maxRuns !== void 0 && consumed >= opts.maxRuns) return null;
1567
- const r = await iterator.next();
1568
- if (r.done) {
1569
- done = true;
1570
- return null;
1571
- }
1572
- consumed += 1;
1573
- return r.value;
1574
- };
1575
- const workers = Array.from({ length: concurrency }, async () => {
1576
- for (let input = await next(); input !== null; input = await next()) {
1577
- if (opts.signal?.aborted) return;
1578
- try {
1579
- const obs = await observe(input, {
1580
- chat: opts.chat,
1581
- ...opts.model ? { model: opts.model } : {},
1582
- corpus: opts.corpus,
1583
- tags: opts.tags ?? [],
1584
- ...opts.analystInstruction ? { analystInstruction: opts.analystInstruction } : {},
1585
- ...opts.signal ? { signal: opts.signal } : {}
1586
- });
1587
- report.runsObserved += 1;
1588
- report.findings += obs.findings.length;
1589
- report.learned += obs.learned.length;
1590
- } catch (e) {
1591
- report.failures.push({
1592
- runId: input.runId ?? `run-${consumed}`,
1593
- error: e instanceof Error ? e.message.slice(0, 300) : String(e)
1594
- });
1595
- }
1596
- }
1597
- });
1598
- await Promise.all(workers);
1599
- if (report.runsObserved === 0 && report.failures.length > 0) {
1600
- throw new Error(
1601
- `harvestCorpus: every run failed analysis (${report.failures.length}) \u2014 first: ${report.failures[0]?.error}`
1602
- );
1603
- }
1604
- return report;
1605
- }
1606
-
1607
- // src/runtime/in-process-sandbox-client.ts
1608
- import { mkdtempSync, rmSync } from "fs";
1609
- import { mkdir, readFile, writeFile } from "fs/promises";
1610
- import { tmpdir as tmpdir2 } from "os";
1611
- import { dirname, join as join2 } from "path";
1612
- function isAsyncIterable2(v) {
1613
- return typeof v === "object" && v !== null && Symbol.asyncIterator in v;
1614
- }
1615
- function inProcessSandboxClient(options) {
1616
- const { onPrompt, onTask, workdir: workdirPrefix, id: idOption } = options;
1617
- let seq = 0;
1618
- return {
1619
- async create(_options) {
1620
- const current = seq++;
1621
- const id = typeof idOption === "function" ? idOption(current) : idOption ?? `in-process-${current}`;
1622
- const boxWorkdir = workdirPrefix !== void 0 ? mkdtempSync(join2(tmpdir2(), `${workdirPrefix}-`)) : void 0;
1623
- let round = 0;
1624
- const fsMembers = boxWorkdir !== void 0 ? {
1625
- fs: {
1626
- async read(path) {
1627
- return readFile(join2(boxWorkdir, path), "utf8");
1628
- },
1629
- async write(path, content) {
1630
- const abs = join2(boxWorkdir, path);
1631
- await mkdir(dirname(abs), { recursive: true });
1632
- await writeFile(abs, content, "utf8");
1633
- }
1634
- },
1635
- async exec(command) {
1636
- const { exec: execCb } = await import("child_process");
1637
- const { promisify } = await import("util");
1638
- const execAsync = promisify(execCb);
1639
- try {
1640
- const { stdout, stderr } = await execAsync(command, {
1641
- cwd: boxWorkdir,
1642
- timeout: 3e4
1643
- });
1644
- return { exitCode: 0, stdout, stderr };
1645
- } catch (err) {
1646
- const e = err;
1647
- return {
1648
- exitCode: e.code ?? 1,
1649
- stdout: e.stdout ?? "",
1650
- stderr: e.stderr ?? e.message ?? ""
1651
- };
1652
- }
1653
- }
1654
- } : {};
1655
- async function* drive(behavior, mode, message, opts) {
1656
- const prompt = typeof message === "string" ? message : JSON.stringify(message);
1657
- const { signal: optSignal, ...rest } = opts ?? {};
1658
- const signal = optSignal ?? new AbortController().signal;
1659
- const ctx = {
1660
- round,
1661
- workdir: boxWorkdir,
1662
- signal,
1663
- mode,
1664
- ...Object.keys(rest).length > 0 ? { options: rest } : {}
1665
- };
1666
- round += 1;
1667
- const produced = await behavior(prompt, ctx);
1668
- if (isAsyncIterable2(produced)) {
1669
- for await (const ev of produced) yield ev;
1670
- } else {
1671
- for (const ev of produced) yield ev;
1672
- }
1673
- }
1674
- const box = {
1675
- id,
1676
- streamPrompt(message, opts) {
1677
- return drive(onPrompt, "prompt", message, opts);
1678
- },
1679
- // Task-mode verb (`streamAgentTurn`'s `box-task` backend). Drives
1680
- // `onTask` when configured; otherwise the box's one behavior callback.
1681
- streamTask(message, opts) {
1682
- return drive(onTask ?? onPrompt, "task", message, opts);
1683
- },
1684
- ...fsMembers,
1685
- async delete() {
1686
- if (boxWorkdir !== void 0) rmSync(boxWorkdir, { recursive: true, force: true });
1687
- }
1688
- };
1689
- return box;
1690
- }
1691
- };
1692
- }
1693
-
1694
- // src/runtime/personify/analyst.ts
1695
- var judgeEvidenceUri = /^(verdict|judge|score)\b/i;
1696
- var assertTraceDerivedFindings = (findings) => {
1697
- for (const f of findings) {
1698
- for (const ref of f.evidence_refs ?? []) {
1699
- if (ref.kind === "metric" && judgeEvidenceUri.test(ref.uri)) {
1700
- throw new PlannerError(
1701
- `steer-firewall: finding ${stringifySafe(f.finding_id)} cites judge-derived evidence (${stringifySafe(ref.uri)}); findings fed to a combinator's steer decision must be trace-derived, not judge-derived (selector \u2260 judge)`
1702
- );
1703
- }
1704
- }
1705
- }
1706
- };
1707
- function createScopeAnalyst(scope, options) {
1708
- if (!options.analyst || typeof options.analyst.act !== "function") {
1709
- throw new AnalystError("createScopeAnalyst: analyst must be an Agent with an act() method");
1710
- }
1711
- const label = options.label ?? "analyst";
1712
- return {
1713
- async analyze(input) {
1714
- const task = options.buildTask(input);
1715
- const spawned = scope.spawn(options.analyst, task, {
1716
- budget: options.budget,
1717
- label
1718
- });
1719
- if (!spawned.ok) {
1720
- throw new AnalystError(
1721
- `createScopeAnalyst: analyst spawn refused by the conserved pool (${spawned.reason}); cannot steer node ${stringifySafe(input.nodeId)} on an unrun analyst`
1722
- );
1723
- }
1724
- const settled = await drainAnalystSettlement(scope, spawned.handle.id);
1725
- const findings = readAnalystFindings(settled);
1726
- assertTraceDerivedFindings(findings);
1727
- return findings;
1728
- }
1729
- };
1730
- }
1731
- async function drainAnalystSettlement(scope, analystId) {
1732
- for (; ; ) {
1733
- const settled = await scope.next();
1734
- if (settled === null) {
1735
- throw new AnalystError(
1736
- `createScopeAnalyst: scope drained before analyst ${stringifySafe(analystId)} settled`
1737
- );
1738
- }
1739
- if (settled.handle.id === analystId) return settled;
1740
- }
1741
- }
1742
- function readAnalystFindings(settled) {
1743
- if (settled.kind === "down") {
1744
- throw new AnalystError(
1745
- `createScopeAnalyst: analyst ${stringifySafe(settled.handle.id)} settled down (${settled.infra ? "infra" : "result"}): ${stringifySafe(settled.reason)}`
1746
- );
1747
- }
1748
- const out = settled.out;
1749
- if (!Array.isArray(out)) {
1750
- throw new PlannerError(
1751
- `createScopeAnalyst: analyst ${stringifySafe(settled.handle.id)} must return AnalystFinding[], got ${stringifySafe(out)}`
1752
- );
1753
- }
1754
- return out;
1755
- }
1756
- function registryScopeAnalyst(registry, buildInputs) {
1757
- return {
1758
- async analyze(input) {
1759
- const projection = buildInputs(input);
1760
- const result = await registry.run(projection.runId, projection.inputs, projection.opts);
1761
- const findings = result.findings;
1762
- assertTraceDerivedFindings(findings);
1763
- return findings;
1764
- }
1765
- };
1766
- }
1767
- function buildSteerContext(findings, settledSoFar) {
1768
- assertTraceDerivedFindings(findings);
1769
- const lastValidScore = observedBestScore(settledSoFar);
1770
- return {
1771
- findings,
1772
- settledSoFar,
1773
- ...lastValidScore !== void 0 ? { lastValidScore } : {}
1774
- };
1775
- }
1776
- function observedBestScore(settledSoFar) {
1777
- let best;
1778
- for (const s of settledSoFar) {
1779
- if (s.kind !== "done") continue;
1780
- const v = s.verdict;
1781
- if (!v || v.valid !== true || typeof v.score !== "number") continue;
1782
- if (best === void 0 || v.score > best) best = v.score;
1783
- }
1784
- return best;
1785
- }
1786
-
1787
- // src/runtime/personify/combinators.ts
1788
- function selectValidWinner(opts) {
1789
- const strategy = opts?.strategy ?? "highest-score";
1790
- const sizeOfIter = (iter) => {
1791
- const out = iter.output;
1792
- if (out === void 0 || opts?.sizeOf === void 0) return Number.POSITIVE_INFINITY;
1793
- const deliverable = out.kind === "done" ? out.deliverable : out;
1794
- return opts.sizeOf(deliverable);
1795
- };
1796
- return (iterations) => {
1797
- const valid = iterations.filter(
1798
- (iter) => iter.output !== void 0 && !iter.error && iter.verdict?.valid === true
1799
- );
1800
- if (valid.length === 0) return void 0;
1801
- switch (strategy) {
1802
- case "first-valid":
1803
- return [...valid].sort((a, b) => a.index - b.index)[0];
1804
- case "smallest-artifact":
1805
- return [...valid].sort((a, b) => sizeOfIter(a) - sizeOfIter(b) || a.index - b.index)[0];
1806
- default:
1807
- return [...valid].sort(
1808
- (a, b) => (b.verdict?.score ?? 0) - (a.verdict?.score ?? 0) || a.index - b.index
1809
- )[0];
1810
- }
1811
- };
1812
- }
1813
- function pipeline(stages) {
1814
- if (stages.length === 0) {
1815
- throw new ValidationError("pipeline: at least one stage is required");
1816
- }
1817
- return (ctx) => ({
1818
- name: `${ctx.persona.name}/pipeline`,
1819
- async act(task, scope) {
1820
- let carry = task;
1821
- for (const stage of stages) {
1822
- const child = ctx.spawnChild(stage.label, ctx.persona.root);
1823
- const res = scope.spawn(child, stage.feed(carry, ctx, task), {
1824
- budget: ctx.budget.perChild,
1825
- label: stage.label
1826
- });
1827
- if (!res.ok) {
1828
- return blocked([`${stage.label}: not admitted (${res.reason})`]);
1829
- }
1830
- const settled = await drainOne(scope, stage.label);
1831
- const out = stage.collect(settled);
1832
- if (out.kind === "blocked") return out;
1833
- carry = out.deliverable;
1834
- }
1835
- return { kind: "done", deliverable: carry };
1836
- }
1837
- });
1838
- }
1839
- function fanout(items, opts) {
1840
- if (opts.synthesize && opts.selectWinner) {
1841
- throw new ValidationError("fanout: pass at most one of `synthesize` or `selectWinner`");
1842
- }
1843
- return (ctx) => ({
1844
- name: `${ctx.persona.name}/fanout`,
1845
- async act(_task, scope) {
1846
- const rejected = [];
1847
- const unitAt = (i) => {
1848
- const item = items[i];
1849
- if (item === void 0) return void 0;
1850
- const label = opts.label ? opts.label(item, i) : `item:${i}`;
1851
- const spec = opts.itemSpec ? opts.itemSpec(item, i, ctx) : ctx.persona.root;
1852
- return {
1853
- agent: ctx.spawnChild(label, spec),
1854
- task: opts.itemTask(item, i, ctx),
1855
- opts: { budget: ctx.budget.perChild, label }
1856
- };
1857
- };
1858
- let opened = 0;
1859
- let drained;
1860
- if (opts.width !== void 0) {
1861
- let next = 0;
1862
- const report = await rollingDispatch(scope, {
1863
- width: opts.width,
1864
- nextUnit: () => {
1865
- const unit = unitAt(next);
1866
- next += 1;
1867
- return unit;
1868
- }
1869
- });
1870
- opened = report.admitted;
1871
- for (const r of report.rejected) {
1872
- const [label, reason] = splitOnce(r, ": ");
1873
- rejected.push(`${label}: not admitted (${reason})`);
1874
- }
1875
- drained = projectDrain(report.settled);
1876
- } else {
1877
- for (let i = 0; i < items.length; i += 1) {
1878
- const unit = unitAt(i);
1879
- if (unit === void 0) continue;
1880
- const res = scope.spawn(unit.agent, unit.task, unit.opts);
1881
- if (res.ok) opened += 1;
1882
- else rejected.push(`${unit.opts.label}: not admitted (${res.reason})`);
1883
- }
1884
- drained = opened === 0 ? emptyRound() : await drain(scope);
1885
- }
1886
- if (opened === 0) {
1887
- return blocked(
1888
- rejected.length > 0 ? rejected : ["fanout: budget admitted no item (fanout fully rejected)"]
1889
- );
1890
- }
1891
- if (drained.done.length === 0) {
1892
- return blocked(
1893
- orderedBlockers(
1894
- drained.blockers,
1895
- rejected,
1896
- "fanout: every item settled without a usable result"
1897
- )
1898
- );
1899
- }
1900
- if (!opts.synthesize) {
1901
- const select = opts.selectWinner ?? defaultSelectWinner;
1902
- const winner = select(drained.iterations);
1903
- if (!winner || winner.output === void 0) {
1904
- return blocked(
1905
- orderedBlockers(drained.blockers, rejected, "fanout: no item survived selection")
1906
- );
1907
- }
1908
- return { kind: "done", deliverable: winner.output };
1909
- }
1910
- const synthLabel = "synthesize";
1911
- const synthChild = ctx.spawnChild(synthLabel, ctx.persona.root);
1912
- const synthRes = scope.spawn(synthChild, opts.synthesize.synthesisTask(drained.done, ctx), {
1913
- budget: ctx.budget.perChild,
1914
- label: synthLabel
1915
- });
1916
- if (!synthRes.ok) {
1917
- return blocked([...drained.blockers, `${synthLabel}: not admitted (${synthRes.reason})`]);
1918
- }
1919
- const synthSettled = await drainOne(scope, synthLabel);
1920
- const out = opts.synthesize.collect(synthSettled);
1921
- if (out.kind === "blocked") return out;
1922
- return out;
1923
- }
1924
- });
1925
- }
1926
- function loopUntil(seed, spec) {
1927
- return (ctx) => ({
1928
- name: `${ctx.persona.name}/loopUntil`,
1929
- async act(task, scope) {
1930
- let state = { round: 0, value: seed };
1931
- const blockers = [];
1932
- const settledSoFar = [];
1933
- for (; ; ) {
1934
- const label = spec.label ? spec.label(state.round) : `step:${state.round}`;
1935
- const child = ctx.spawnChild(label, ctx.persona.root);
1936
- const res = scope.spawn(child, spec.step(task, state, ctx), {
1937
- budget: ctx.budget.perChild,
1938
- label
1939
- });
1940
- if (!res.ok) {
1941
- if (state.round === 0) return blocked([`${label}: not admitted (${res.reason})`]);
1942
- break;
1943
- }
1944
- const settled = await drainOne(scope, label);
1945
- if (settled.kind === "down") blockers.push(blockerFromDown(settled));
1946
- settledSoFar.push(settled);
1947
- state = spec.fold(state, settled);
1948
- const findings = ctx.analyst ? await ctx.analyst.analyze({ task, settledSoFar, nodeId: scope.view.root }) : [];
1949
- const reached = spec.until(state, findings);
1950
- if (reached) return reached;
1951
- state = { round: state.round + 1, value: state.value };
1952
- }
1953
- return blocked(
1954
- blockers.length > 0 ? blockers : ["loopUntil: budget exhausted before the satisfiability gate was reached"]
1955
- );
1956
- }
1957
- });
1958
- }
1959
- function panel(spec) {
1960
- if (spec.judges.length === 0) {
1961
- throw new ValidationError("panel: at least one judge is required");
1962
- }
1963
- return (ctx) => ({
1964
- name: `${ctx.persona.name}/panel`,
1965
- async act(task, scope) {
1966
- const artifact = task;
1967
- const byLabel = /* @__PURE__ */ new Map();
1968
- const rejected = [];
1969
- let opened = 0;
1970
- for (const judge of spec.judges) {
1971
- const child = ctx.spawnChild(judge.label, ctx.persona.root);
1972
- const res = scope.spawn(child, spec.judgeTask(artifact, judge, ctx), {
1973
- budget: ctx.budget.perChild,
1974
- label: judge.label
1975
- });
1976
- if (res.ok) {
1977
- byLabel.set(judge.label, judge);
1978
- opened += 1;
1979
- } else {
1980
- rejected.push(`${judge.label}: not admitted (${res.reason})`);
1981
- }
1982
- }
1983
- if (opened === 0) {
1984
- return blocked(rejected.length > 0 ? rejected : ["panel: budget admitted no judge"]);
1985
- }
1986
- const verdicts = [];
1987
- for (let s = await scope.next(); s !== null; s = await scope.next()) {
1988
- const judge = byLabel.get(s.handle.label);
1989
- if (!judge) {
1990
- throw new ValidationError(
1991
- `panel: settled child "${s.handle.label}" has no judge descriptor`
1992
- );
1993
- }
1994
- if (s.kind === "down") {
1995
- verdicts.push({ judge, down: true });
1996
- continue;
1997
- }
1998
- verdicts.push({
1999
- judge,
2000
- down: false,
2001
- ...s.verdict ? { verdict: s.verdict } : {},
2002
- output: s.out
2003
- });
2004
- }
2005
- return spec.merge(verdicts, artifact);
2006
- }
2007
- });
2008
- }
2009
- function verify(spec) {
2010
- return (ctx) => ({
2011
- name: `${ctx.persona.name}/verify`,
2012
- async act(task, scope) {
2013
- const implementLabel = spec.implementLabel ?? "implement";
2014
- const implChild = ctx.spawnChild(implementLabel, ctx.persona.root);
2015
- const implRes = scope.spawn(implChild, spec.implement(task, ctx), {
2016
- budget: ctx.budget.perChild,
2017
- label: implementLabel
2018
- });
2019
- if (!implRes.ok) return blocked([`${implementLabel}: not admitted (${implRes.reason})`]);
2020
- const candidate = await drainOne(scope, implementLabel);
2021
- if (candidate.kind === "down") return blocked([blockerFromDown(candidate)]);
2022
- const verifierLabel = spec.verifierLabel ?? "verify";
2023
- const verifierChild = ctx.spawnChild(verifierLabel, ctx.persona.root);
2024
- const verifierRes = scope.spawn(
2025
- verifierChild,
2026
- spec.verifier(candidate, ctx),
2027
- { budget: ctx.budget.perChild, label: verifierLabel }
2028
- );
2029
- if (!verifierRes.ok)
2030
- return blocked([`${verifierLabel}: not admitted (${verifierRes.reason})`]);
2031
- const gate = await drainOne(scope, verifierLabel);
2032
- if (gate.kind === "down") return blocked([blockerFromDown(gate)]);
2033
- if (!gate.verdict || gate.verdict.valid !== true) {
2034
- return blocked([`${verifierLabel}: gate rejected the candidate (${verdictDetail(gate)})`]);
2035
- }
2036
- return spec.collect(candidate, gate.verdict);
2037
- }
2038
- });
2039
- }
2040
- function widen(spec) {
2041
- return (ctx) => ({
2042
- name: `${ctx.persona.name}/widen`,
2043
- async act(_task, scope) {
2044
- let opened = 0;
2045
- for (const [i, seed] of spec.seeds.entries()) {
2046
- const label = `seed:${i}`;
2047
- const child = ctx.spawnChild(label, ctx.persona.root);
2048
- const res = scope.spawn(child, spec.seedTask(seed, i, ctx), {
2049
- budget: ctx.budget.perChild,
2050
- label
2051
- });
2052
- if (res.ok) opened += 1;
2053
- }
2054
- if (opened === 0) {
2055
- return blocked(["widen: budget admitted no seed lineage"]);
2056
- }
2057
- const gathered = [];
2058
- let widenIndex = 0;
2059
- for (let s = await scope.next(); s !== null; s = await scope.next()) {
2060
- gathered.push(s);
2061
- const findings = ctx.analyst ? await ctx.analyst.analyze({
2062
- task: _task,
2063
- settledSoFar: gathered,
2064
- nodeId: scope.view.root
2065
- }) : [];
2066
- const decision = spec.gate.decide(s, findings, scope.budget);
2067
- if (decision.kind !== "widen") continue;
2068
- const label = `widen:${widenIndex}`;
2069
- widenIndex += 1;
2070
- const child = ctx.spawnChild(label, ctx.persona.root);
2071
- scope.spawn(child, spec.widenTask(decision.toward, ctx), {
2072
- budget: ctx.budget.perChild,
2073
- label
2074
- });
2075
- }
2076
- const done = gathered.filter(isDone);
2077
- if (done.length === 0) {
2078
- return blocked(
2079
- orderedBlockers(
2080
- gathered.filter(isDown).map(blockerFromDown),
2081
- [],
2082
- "widen: every lineage settled without a usable result"
2083
- )
2084
- );
2085
- }
2086
- return spec.synthesize(done, ctx);
2087
- }
2088
- });
2089
- }
2090
- function flatWidenGate() {
2091
- return {
2092
- decide() {
2093
- return { kind: "stop" };
2094
- }
2095
- };
2096
- }
2097
- function emptyRound() {
2098
- return { iterations: [], done: [], blockers: [] };
2099
- }
2100
- function projectDrain(settled) {
2101
- const round = emptyRound();
2102
- for (const s of settled) {
2103
- if (s.kind === "down") {
2104
- round.blockers.push(blockerFromDown(s));
2105
- continue;
2106
- }
2107
- round.iterations.push(settledToIteration(s));
2108
- round.done.push(s);
2109
- }
2110
- return round;
2111
- }
2112
- function splitOnce(text, sep) {
2113
- const at = text.indexOf(sep);
2114
- if (at < 0) return [text, ""];
2115
- return [text.slice(0, at), text.slice(at + sep.length)];
2116
- }
2117
- async function drain(scope) {
2118
- const iterations = [];
2119
- const done = [];
2120
- const blockers = [];
2121
- for (let s = await scope.next(); s !== null; s = await scope.next()) {
2122
- if (s.kind === "down") {
2123
- blockers.push(blockerFromDown(s));
2124
- continue;
2125
- }
2126
- iterations.push(settledToIteration(s));
2127
- done.push(s);
2128
- }
2129
- return { iterations, done, blockers };
2130
- }
2131
- async function drainOne(scope, label) {
2132
- const s = await scope.next();
2133
- if (s === null) {
2134
- throw new ValidationError(`combinator: child "${label}" was spawned but never settled`);
2135
- }
2136
- return s;
2137
- }
2138
- function isDone(s) {
2139
- return s.kind === "done";
2140
- }
2141
- function isDown(s) {
2142
- return s.kind === "down";
2143
- }
2144
- function blockerFromDown(s) {
2145
- return `${s.handle.label}: ${s.infra ? "infra" : "failed"} \u2014 ${s.reason}`;
2146
- }
2147
- function verdictDetail(s) {
2148
- if (!s.verdict) return "no verdict";
2149
- return s.verdict.notes ?? `score ${s.verdict.score}`;
2150
- }
2151
- function orderedBlockers(drained, rejected, fallback) {
2152
- const all = [...drained, ...rejected];
2153
- return all.length > 0 ? all : [fallback];
2154
- }
2155
- function blocked(blockers) {
2156
- if (blockers.length === 0) {
2157
- throw new ValidationError("combinator: a blocked outcome must name at least one blocker");
2158
- }
2159
- return { kind: "blocked", blockers };
2160
- }
2161
-
2162
- // src/runtime/personify/corpus.ts
2163
- var corpusSchemaVersion = "1.0.0";
2164
- function assertCorpusRecord(value, where) {
2165
- if (typeof value !== "object" || value === null) {
2166
- throw new Error(`${where}: corpus record is not an object`);
2167
- }
2168
- const r = value;
2169
- if (r.schemaVersion !== corpusSchemaVersion) {
2170
- throw new Error(
2171
- `${where}: corpus record schemaVersion '${String(r.schemaVersion)}' != '${corpusSchemaVersion}'`
2172
- );
2173
- }
2174
- requireNonEmptyString(r.id, `${where}.id`);
2175
- requireNonEmptyString(r.runId, `${where}.runId`);
2176
- requireNonEmptyString(r.producedAt, `${where}.producedAt`);
2177
- requireNonEmptyString(r.area, `${where}.area`);
2178
- requireNonEmptyString(r.claim, `${where}.claim`);
2179
- if (r.rationale !== void 0 && typeof r.rationale !== "string") {
2180
- throw new Error(`${where}.rationale: expected string, got ${typeof r.rationale}`);
2181
- }
2182
- if (!Array.isArray(r.tags) || r.tags.some((t) => typeof t !== "string")) {
2183
- throw new Error(`${where}.tags: expected string[]`);
2184
- }
2185
- if (typeof r.confidence !== "number" || !Number.isFinite(r.confidence)) {
2186
- throw new Error(`${where}.confidence: expected a finite number, got ${String(r.confidence)}`);
2187
- }
2188
- if (r.confidence < 0 || r.confidence > 1) {
2189
- throw new Error(`${where}.confidence: ${r.confidence} is outside the [0, 1] range`);
2190
- }
2191
- if (r.evidence !== void 0) {
2192
- if (!Array.isArray(r.evidence)) {
2193
- throw new Error(`${where}.evidence: expected an array`);
2194
- }
2195
- for (let i = 0; i < r.evidence.length; i++) {
2196
- const ev = r.evidence[i];
2197
- if (typeof ev !== "object" || ev === null) {
2198
- throw new Error(`${where}.evidence[${i}]: not an object`);
2199
- }
2200
- requireNonEmptyString(ev.kind, `${where}.evidence[${i}].kind`);
2201
- requireNonEmptyString(ev.uri, `${where}.evidence[${i}].uri`);
2202
- }
2203
- }
2204
- }
2205
- function requireNonEmptyString(value, where) {
2206
- if (typeof value !== "string" || value.length === 0) {
2207
- throw new Error(`${where}: expected a non-empty string, got ${describe(value)}`);
2208
- }
2209
- }
2210
- function describe(value) {
2211
- if (value === void 0) return "undefined";
2212
- if (value === null) return "null";
2213
- if (typeof value === "string") return `'${value}'`;
2214
- return typeof value;
2215
- }
2216
- function recordsEqual(a, b) {
2217
- return stableStringify(a) === stableStringify(b);
2218
- }
2219
- function stableStringify(value) {
2220
- if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null";
2221
- if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
2222
- const entries = Object.entries(value).filter(([, v]) => v !== void 0).sort(([x], [y]) => x < y ? -1 : x > y ? 1 : 0);
2223
- return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`).join(",")}}`;
2224
- }
2225
- function applyFilter(records, filter) {
2226
- const matched = records.filter((r) => {
2227
- if (filter.area !== void 0 && r.area !== filter.area) return false;
2228
- if (filter.runId !== void 0 && r.runId !== filter.runId) return false;
2229
- if (filter.minConfidence !== void 0 && r.confidence < filter.minConfidence) return false;
2230
- if (filter.tags !== void 0 && !filter.tags.every((t) => r.tags.includes(t))) return false;
2231
- return true;
2232
- });
2233
- const ordered = [...matched].sort(
2234
- (a, b) => b.confidence !== a.confidence ? b.confidence - a.confidence : b.producedAt < a.producedAt ? -1 : b.producedAt > a.producedAt ? 1 : 0
2235
- );
2236
- if (filter.limit !== void 0) {
2237
- if (!Number.isInteger(filter.limit) || filter.limit < 0) {
2238
- throw new Error(
2239
- `corpus query: filter.limit must be a non-negative integer, got ${filter.limit}`
2240
- );
2241
- }
2242
- return ordered.slice(0, filter.limit);
2243
- }
2244
- return ordered;
2245
- }
2246
- var InMemoryCorpus = class {
2247
- byId = /* @__PURE__ */ new Map();
2248
- async append(record) {
2249
- try {
2250
- assertCorpusRecord(record, "append: record");
2251
- } catch (err) {
2252
- return { succeeded: false, error: err instanceof Error ? err.message : String(err) };
2253
- }
2254
- const existing = this.byId.get(record.id);
2255
- if (existing) {
2256
- if (recordsEqual(existing, record)) return { succeeded: true };
2257
- return {
2258
- succeeded: false,
2259
- error: `corpus conflict: id '${record.id}' is already stored with a different record; a learned fact is append-only \u2014 re-mint the id or reconcile before re-appending`
2260
- };
2261
- }
2262
- this.byId.set(record.id, freeze(record));
2263
- return { succeeded: true };
2264
- }
2265
- async query(filter) {
2266
- return applyFilter([...this.byId.values()], filter);
2267
- }
2268
- };
2269
- var FileCorpus = class {
2270
- constructor(path) {
2271
- this.path = path;
2272
- }
2273
- path;
2274
- async append(record) {
2275
- try {
2276
- assertCorpusRecord(record, "append: record");
2277
- } catch (err) {
2278
- return { succeeded: false, error: err instanceof Error ? err.message : String(err) };
2279
- }
2280
- let stored;
2281
- try {
2282
- stored = await this.load();
2283
- } catch (err) {
2284
- return { succeeded: false, error: err instanceof Error ? err.message : String(err) };
2285
- }
2286
- const existing = stored.get(record.id);
2287
- if (existing) {
2288
- if (recordsEqual(existing, record)) return { succeeded: true };
2289
- return {
2290
- succeeded: false,
2291
- error: `corpus conflict: id '${record.id}' is already stored in ${this.path} with a different record; a learned fact is append-only \u2014 re-mint the id or reconcile before re-appending`
2292
- };
2293
- }
2294
- await this.appendLine(record);
2295
- return { succeeded: true };
2296
- }
2297
- async query(filter) {
2298
- const stored = await this.load();
2299
- return applyFilter([...stored.values()], filter);
2300
- }
2301
- async load() {
2302
- const fs = await import("fs/promises");
2303
- let text;
2304
- try {
2305
- text = await fs.readFile(this.path, "utf8");
2306
- } catch (err) {
2307
- if (isNoEntError(err)) return /* @__PURE__ */ new Map();
2308
- throw err;
2309
- }
2310
- const lines = text.split("\n").filter((line) => line.length > 0);
2311
- const byId = /* @__PURE__ */ new Map();
2312
- for (let i = 0; i < lines.length; i++) {
2313
- const line = lines[i];
2314
- let parsed;
2315
- try {
2316
- parsed = JSON.parse(line);
2317
- } catch (err) {
2318
- throw new Error(
2319
- `corpus corrupted: ${this.path} line ${i + 1} is not valid JSON: ` + (err instanceof Error ? err.message : String(err))
2320
- );
2321
- }
2322
- assertCorpusRecord(parsed, `corpus ${this.path} line ${i + 1}`);
2323
- const existing = byId.get(parsed.id);
2324
- if (existing && !recordsEqual(existing, parsed)) {
2325
- throw new Error(
2326
- `corpus corrupted: ${this.path} has two different records for id '${parsed.id}'; an append-only corpus must never hold a conflicting re-append under one id`
2327
- );
2328
- }
2329
- byId.set(parsed.id, freeze(parsed));
2330
- }
2331
- return byId;
2332
- }
2333
- async appendLine(record) {
2334
- const fs = await import("fs/promises");
2335
- const path = await import("path");
2336
- await fs.mkdir(path.dirname(this.path), { recursive: true });
2337
- const fh = await fs.open(this.path, "a");
2338
- try {
2339
- await fh.write(`${JSON.stringify(record)}
2340
- `);
2341
- await fh.sync();
2342
- } finally {
2343
- await fh.close();
2344
- }
2345
- }
2346
- };
2347
- async function renderCorpusToInstructions(opts) {
2348
- const { corpus, filter, profile, target = "prompt", maxLines } = opts;
2349
- if (maxLines !== void 0 && (!Number.isInteger(maxLines) || maxLines < 0)) {
2350
- throw new Error(
2351
- `renderCorpusToInstructions: maxLines must be a non-negative integer, got ${maxLines}`
2352
- );
2353
- }
2354
- const matched = await corpus.query(filter);
2355
- const capped = maxLines !== void 0 ? matched.slice(0, maxLines) : matched;
2356
- const lines = capped.map(renderLine);
2357
- if (lines.length === 0) return structuredClone(profile);
2358
- const next = structuredClone(profile);
2359
- if (target === "resources") {
2360
- const resources = next.resources ?? {};
2361
- const prior = resources.instructions;
2362
- if (prior !== void 0 && typeof prior !== "string") {
2363
- throw new Error(
2364
- "renderCorpusToInstructions: resources.instructions is an AgentProfileResourceRef, not a string; cannot string-append corpus facts without dropping the ref (pass target: 'prompt' or pre-resolve the ref)"
2365
- );
2366
- }
2367
- const blob = [prior, ...lines].filter((s) => typeof s === "string" && s.length > 0);
2368
- next.resources = { ...resources, instructions: blob.join("\n") };
2369
- return next;
2370
- }
2371
- const prompt = next.prompt ?? {};
2372
- next.prompt = { ...prompt, instructions: [...prompt.instructions ?? [], ...lines] };
2373
- return next;
2374
- }
2375
- function renderLine(record) {
2376
- return record.rationale ? `${record.claim} (${record.rationale})` : record.claim;
2377
- }
2378
- function freeze(record) {
2379
- return Object.freeze({ ...record });
2380
- }
2381
- function isNoEntError(err) {
2382
- return typeof err === "object" && err !== null && "code" in err && err.code === "ENOENT";
2383
- }
2384
-
2385
- // src/runtime/personify/registry.ts
2386
- function createShapeRegistry() {
2387
- const shapes = /* @__PURE__ */ new Map();
2388
- return {
2389
- register(name, factory) {
2390
- if (shapes.has(name)) {
2391
- throw new ValidationError(`shape registry: shape "${name}" already registered`);
2392
- }
2393
- shapes.set(name, factory);
2394
- },
2395
- resolve(name) {
2396
- const factory = shapes.get(name);
2397
- if (!factory) {
2398
- return { succeeded: false, error: `shape registry: unknown shape "${name}"` };
2399
- }
2400
- return { succeeded: true, value: factory };
2401
- },
2402
- names() {
2403
- return [...shapes.keys()];
2404
- }
2405
- };
2406
- }
2407
- var builtinShapes = createShapeRegistry();
2408
- function registerShape(name, factory) {
2409
- builtinShapes.register(name, factory);
2410
- }
2411
-
2412
- // src/runtime/personify/persona.ts
2413
- function definePersona(input) {
2414
- if (!input.executors.registry && !input.executors.seams) {
2415
- throw new ValidationError(
2416
- `definePersona("${input.name}"): executors must supply a registry or a seams bag (built-in runtimes read their seams off ExecutorContext; neither was provided)`
2417
- );
2418
- }
2419
- if (!input.root || typeof input.root !== "object" || !("harness" in input.root)) {
2420
- throw new ValidationError(`definePersona("${input.name}"): root must be an AgentSpec`);
2421
- }
2422
- return Object.freeze({
2423
- name: input.name,
2424
- root: input.root,
2425
- directive: input.directive,
2426
- context: input.context,
2427
- executors: input.executors,
2428
- ...input.extensions ? { extensions: input.extensions } : {}
2429
- });
2430
- }
2431
- function createShapeContext(persona, budget, analyst) {
2432
- return {
2433
- persona,
2434
- budget,
2435
- ...analyst ? { analyst } : {},
2436
- spawnChild(name, spec) {
2437
- const agent = {
2438
- name,
2439
- executorSpec: spec,
2440
- act() {
2441
- throw new ValidationError(
2442
- `personify: spawned child "${name}" was run directly; its executorSpec drives it (a leaf, or \u2014 for a driver child \u2014 a nested scope through the recursive driver-executor)`
2443
- );
2444
- }
2445
- };
2446
- return agent;
2447
- },
2448
- childSpec(profile, harness) {
2449
- return {
2450
- profile,
2451
- harness: harness === void 0 ? persona.root.harness : harness,
2452
- ...persona.root.executor ? { executor: persona.root.executor } : {}
2453
- };
2454
- }
2455
- };
2456
- }
2457
- async function runPersonified(options) {
2458
- const { persona } = options;
2459
- const shape = resolveShape(options.shape);
2460
- const shapeBudget = resolveShapeBudget(options.budget, options.shapeBudget);
2461
- const ctx = createShapeContext(persona, shapeBudget, options.analyst);
2462
- const rootAgent = shape(ctx);
2463
- const executors = personaRegistry(persona);
2464
- const supervisor = createSupervisor();
2465
- if (options.handle) supervisor.attach(options.handle);
2466
- const supervisorOpts = {
2467
- budget: options.budget,
2468
- runId: options.runId ?? `${persona.name}:${shapeName(options.shape, shape)}`,
2469
- journal: options.journal ?? new InMemorySpawnJournal(),
2470
- blobs: options.blobs ?? new InMemoryResultBlobStore(),
2471
- executors,
2472
- ...options.maxDepth !== void 0 ? { maxDepth: options.maxDepth } : {},
2473
- ...options.maxRestarts !== void 0 ? { maxRestarts: options.maxRestarts } : {},
2474
- ...options.withinMs !== void 0 ? { withinMs: options.withinMs } : {},
2475
- ...options.now ? { now: options.now } : {},
2476
- ...options.signal ? { signal: options.signal } : {},
2477
- ...options.hooks ? { hooks: options.hooks } : {}
2478
- };
2479
- return supervisor.run(rootAgent, options.task, supervisorOpts);
2480
- }
2481
- function resolveShape(shape) {
2482
- if (typeof shape !== "string") return shape;
2483
- const resolved = builtinShapes.resolve(shape);
2484
- if (!resolved.succeeded) {
2485
- throw new ValidationError(
2486
- `runPersonified: ${resolved.error} (registered: ${builtinShapes.names().join(", ")})`
2487
- );
2488
- }
2489
- return resolved.value;
2490
- }
2491
- function shapeName(shape, _resolved) {
2492
- return typeof shape === "string" ? shape : shape.name || "shape";
2493
- }
2494
- function resolveShapeBudget(root, over) {
2495
- const fanout2 = over?.fanout ?? defaultFanout;
2496
- const perChild = over?.perChild ?? {
2497
- maxIterations: Math.max(1, Math.floor(root.maxIterations / fanout2)),
2498
- maxTokens: Math.max(1, Math.floor(root.maxTokens / fanout2)),
2499
- ...root.maxUsd !== void 0 ? { maxUsd: root.maxUsd / fanout2 } : {},
2500
- ...root.deadlineMs !== void 0 ? { deadlineMs: root.deadlineMs } : {}
2501
- };
2502
- return { perChild, fanout: fanout2 };
2503
- }
2504
- var defaultFanout = 3;
2505
- function personaRegistry(persona) {
2506
- const { registry, seams } = persona.executors;
2507
- if (registry) return withDriverExecutor(registry);
2508
- if (!seams) {
2509
- throw new ValidationError(
2510
- `personify: persona "${persona.name}" supplies neither a registry nor seams`
2511
- );
2512
- }
2513
- return withDriverExecutor(withSeams(createExecutorRegistry(), seams));
2514
- }
2515
- function withSeams(base, seams) {
2516
- return {
2517
- register(runtime, factory) {
2518
- base.register(runtime, factory);
2519
- },
2520
- resolve(spec) {
2521
- const resolved = base.resolve(spec);
2522
- if (!resolved.succeeded) return resolved;
2523
- const inner = resolved.value;
2524
- const wrapped = (s, ctx) => inner(s, mergeSeams(ctx, seams));
2525
- return { succeeded: true, value: wrapped };
2526
- }
2527
- };
2528
- }
2529
- function mergeSeams(ctx, seams) {
2530
- return { signal: ctx.signal, seams: { ...seams, ...ctx.seams } };
2531
- }
2532
-
2533
- // src/runtime/personify/trajectory.ts
2534
- var defaultEqualKTolerance = 0.05;
2535
- async function trajectoryReport(journal, blobs, root, options = {}) {
2536
- const events = await journal.loadTree(root);
2537
- if (events === void 0) {
2538
- throw new Error(`trajectoryReport: no journaled tree for root '${root}'`);
2539
- }
2540
- const spawns = events.filter(isNodeCreation).sort(bySeq);
2541
- const closes = events.filter((ev) => ev.kind !== "spawned" && ev.kind !== "waiting" && ev.kind !== "metered").sort(bySeq);
2542
- const nodes = /* @__PURE__ */ new Map();
2543
- for (const ev of spawns) {
2544
- nodes.set(ev.id, {
2545
- id: ev.id,
2546
- parent: ev.parent,
2547
- label: ev.label,
2548
- // A wait-state has no executor, so it has no runtime in the executor sense — it is tagged
2549
- // `'wait'` so a reader can separate zero-cost waiting from paid work in the trajectory.
2550
- runtime: ev.kind === "waiting" ? "wait" : ev.runtime,
2551
- status: ev.kind === "waiting" ? "waiting" : "pending",
2552
- ownSpend: zeroSpend(),
2553
- children: []
2554
- });
2555
- }
2556
- for (const ev of closes) {
2557
- const node = requireNode(nodes, ev.id, root);
2558
- if (ev.kind === "cancelled") {
2559
- node.status = "cancelled";
2560
- continue;
2561
- }
2562
- if (ev.kind === "woken") {
2563
- node.status = ev.by === "cancelled" ? "cancelled" : "done";
2564
- node.outRef = ev.outRef;
2565
- continue;
2566
- }
2567
- node.status = ev.status === "done" ? "done" : "failed";
2568
- node.ownSpend = ev.spent;
2569
- node.verdict = ev.verdict;
2570
- node.outRef = ev.outRef;
2571
- }
2572
- for (const ev of events) {
2573
- if (ev.kind !== "metered") continue;
2574
- const node = requireNode(nodes, ev.id, root);
2575
- node.ownSpend = addNodeSpend(node.ownSpend, ev.spend);
2576
- }
2577
- if (!nodes.has(root)) {
2578
- throw new Error(
2579
- `trajectoryReport: root '${root}' has no spawned event in its journaled tree (corrupted log)`
2580
- );
2581
- }
2582
- for (const ev of spawns) {
2583
- if (ev.parent === void 0) continue;
2584
- requireNode(nodes, ev.parent, root).children.push(ev.id);
2585
- }
2586
- const rolledUp = rollUpSpend(nodes, root);
2587
- if (options.withOutputs) {
2588
- await attachOutputs(nodes, blobs);
2589
- }
2590
- const ordered = spawns.map((ev) => nodes.get(ev.id)).filter(isNode);
2591
- const reported = ordered.map(
2592
- (node) => freezeNode(node, requireSpend(rolledUp, node.id, root))
2593
- );
2594
- return {
2595
- root,
2596
- nodes: reported,
2597
- total: requireSpend(rolledUp, root, root),
2598
- statusCounts: countStatuses(reported)
2599
- };
2600
- }
2601
- function equalKOnCost(arms, options = {}) {
2602
- if (arms.length === 0) {
2603
- throw new Error("equalKOnCost: no arms to compare");
2604
- }
2605
- const tolerance = options.tolerance ?? defaultEqualKTolerance;
2606
- const armCosts = arms.map((arm) => ({
2607
- label: arm.label,
2608
- tokens: arm.report.total.tokens.input + arm.report.total.tokens.output,
2609
- usd: arm.report.total.usd,
2610
- iterations: arm.report.total.iterations
2611
- }));
2612
- const tokenValues = armCosts.map((a) => a.tokens);
2613
- const usdValues = armCosts.map((a) => a.usd);
2614
- const spread = {
2615
- tokens: spreadOf(tokenValues),
2616
- usd: spreadOf(usdValues)
2617
- };
2618
- const withinTolerance = fractionalSpread(tokenValues) <= tolerance && fractionalSpread(usdValues) <= tolerance;
2619
- return {
2620
- withinTolerance,
2621
- arms: armCosts,
2622
- spread,
2623
- tolerance
2624
- };
2625
- }
2626
- function rollUpSpend(nodes, root) {
2627
- const rolled = /* @__PURE__ */ new Map();
2628
- const stack = [{ id: root, expanded: false }];
2629
- while (stack.length > 0) {
2630
- const frame = stack.pop();
2631
- if (frame === void 0) continue;
2632
- const node = requireNode(nodes, frame.id, root);
2633
- if (!frame.expanded) {
2634
- stack.push({ id: frame.id, expanded: true });
2635
- for (const child of node.children) stack.push({ id: child, expanded: false });
2636
- continue;
2637
- }
2638
- const sum = cloneSpend(node.ownSpend);
2639
- for (const child of node.children) addSpend(sum, requireSpend(rolled, child, root));
2640
- rolled.set(frame.id, sum);
2641
- }
2642
- return rolled;
2643
- }
2644
- async function attachOutputs(nodes, blobs) {
2645
- for (const node of nodes.values()) {
2646
- if (node.status !== "done" || node.outRef === void 0) continue;
2647
- const out = await blobs.get(node.outRef);
2648
- if (out === void 0) {
2649
- throw new Error(
2650
- `trajectoryReport: blob store has no artifact for outRef '${node.outRef}' (node '${node.id}')`
2651
- );
2652
- }
2653
- node.output = out;
2654
- }
2655
- }
2656
- function freezeNode(node, rolledUpSpend) {
2657
- return {
2658
- id: node.id,
2659
- parent: node.parent,
2660
- children: [...node.children],
2661
- label: node.label,
2662
- runtime: node.runtime,
2663
- status: node.status,
2664
- ownSpend: node.ownSpend,
2665
- rolledUpSpend,
2666
- verdict: node.verdict,
2667
- output: node.output,
2668
- outRef: node.outRef
2669
- };
2670
- }
2671
- function countStatuses(reported) {
2672
- const counts = {
2673
- done: 0,
2674
- failed: 0,
2675
- cancelled: 0,
2676
- pending: 0,
2677
- waiting: 0
2678
- };
2679
- for (const node of reported) counts[node.status] += 1;
2680
- return counts;
2681
- }
2682
- function zeroSpend() {
2683
- return { iterations: 0, tokens: zeroTokenUsage(), usd: 0, ms: 0 };
2684
- }
2685
- function addNodeSpend(a, b) {
2686
- return {
2687
- iterations: a.iterations + b.iterations,
2688
- tokens: { input: a.tokens.input + b.tokens.input, output: a.tokens.output + b.tokens.output },
2689
- usd: a.usd + b.usd,
2690
- ...a.usdKnown === false || b.usdKnown === false ? { usdKnown: false } : {},
2691
- ms: a.ms + b.ms
2692
- };
2693
- }
2694
- function cloneSpend(spend) {
2695
- return {
2696
- iterations: spend.iterations,
2697
- tokens: { input: spend.tokens.input, output: spend.tokens.output },
2698
- usd: spend.usd,
2699
- ...spend.usdKnown === false ? { usdKnown: false } : {},
2700
- ms: spend.ms
2701
- };
2702
- }
2703
- function addSpend(acc, delta) {
2704
- acc.iterations += delta.iterations;
2705
- addTokenUsage(acc.tokens, delta.tokens);
2706
- acc.usd += delta.usd;
2707
- if (delta.usdKnown === false) acc.usdKnown = false;
2708
- acc.ms += delta.ms;
2709
- }
2710
- function spreadOf(values) {
2711
- if (values.length === 0) return 0;
2712
- return Math.max(...values) - Math.min(...values);
2713
- }
2714
- function fractionalSpread(values) {
2715
- const spread = spreadOf(values);
2716
- if (spread === 0) return 0;
2717
- const median = medianOf(values);
2718
- if (median === 0) {
2719
- throw new Error(
2720
- "equalKOnCost: arms have a non-zero cost spread on a zero-median channel; cannot express it as a fraction"
2721
- );
2722
- }
2723
- return spread / median;
2724
- }
2725
- function medianOf(values) {
2726
- if (values.length === 0) {
2727
- throw new Error("equalKOnCost: cannot take the median of an empty channel");
2728
- }
2729
- const sorted = [...values].sort((a, b) => a - b);
2730
- const mid = Math.floor(sorted.length / 2);
2731
- const hi = sorted[mid];
2732
- if (sorted.length % 2 !== 0) return hi;
2733
- const lo = sorted[mid - 1];
2734
- return (lo + hi) / 2;
2735
- }
2736
- function isNodeCreation(ev) {
2737
- return ev.kind === "spawned" || ev.kind === "waiting";
2738
- }
2739
- function isNode(node) {
2740
- return node !== void 0;
2741
- }
2742
- function bySeq(a, b) {
2743
- return a.seq - b.seq;
2744
- }
2745
- function requireNode(nodes, id, root) {
2746
- const node = nodes.get(id);
2747
- if (!node) {
2748
- throw new Error(
2749
- `trajectoryReport: tree '${root}' references node '${id}' with no prior spawn (corrupted log)`
2750
- );
2751
- }
2752
- return node;
2753
- }
2754
- function requireSpend(rolled, id, root) {
2755
- const spend = rolled.get(id);
2756
- if (!spend) {
2757
- throw new Error(
2758
- `trajectoryReport: node '${id}' was never rolled up in tree '${root}' (unreachable from root)`
2759
- );
2760
- }
2761
- return spend;
2762
- }
2763
-
2764
- // src/runtime/promotion-gate.ts
2765
- import { heldoutSignificance } from "@tangle-network/agent-eval/campaign";
2766
- function promotionGate(opts) {
2767
- const mode = opts.mode ?? "superiority";
2768
- if (opts.candidate === opts.incumbent) {
2769
- return {
2770
- promoted: false,
2771
- reason: "identical-champion",
2772
- mode,
2773
- n: 0,
2774
- lift: { mean: 0, median: 0, low: 0, high: 0 }
2775
- };
2776
- }
2777
- const before = [];
2778
- const after = [];
2779
- const incUsd = [];
2780
- const candUsd = [];
2781
- const incMs = [];
2782
- const candMs = [];
2783
- const cellIds = [];
2784
- for (const row of opts.report.perTask) {
2785
- const inc = row.cells?.[opts.incumbent];
2786
- const cand = row.cells?.[opts.candidate];
2787
- if (!inc || !cand) continue;
2788
- before.push(inc.score);
2789
- after.push(cand.score);
2790
- incUsd.push(inc.usd);
2791
- candUsd.push(cand.usd);
2792
- incMs.push(inc.ms);
2793
- candMs.push(cand.ms);
2794
- cellIds.push(row.taskId);
2795
- }
2796
- if (before.length === 0) {
2797
- throw new Error(
2798
- `promotionGate: no holdout task carried cells for both "${opts.incumbent}" and "${opts.candidate}" \u2014 the report must come from a run that included both strategies`
2799
- );
2800
- }
2801
- const sig = heldoutSignificance(
2802
- { before, after, cellIds },
2803
- {
2804
- deltaThreshold: opts.deltaThreshold ?? 0,
2805
- minProductiveRuns: opts.minPairedTasks ?? 6,
2806
- statistic: opts.statistic ?? "mean",
2807
- ...opts.seed !== void 0 ? { seed: opts.seed } : {},
2808
- ...opts.resamples !== void 0 ? { resamples: opts.resamples } : {}
2809
- }
2810
- );
2811
- const lift = {
2812
- mean: sig.bootstrap.mean,
2813
- median: sig.bootstrap.median,
2814
- low: sig.bootstrap.low,
2815
- high: sig.bootstrap.high
2816
- };
2817
- const latSig = heldoutSignificance(
2818
- { before: incMs, after: candMs, cellIds },
2819
- {
2820
- deltaThreshold: 0,
2821
- minProductiveRuns: 1,
2822
- statistic: opts.statistic ?? "mean",
2823
- ...opts.seed !== void 0 ? { seed: opts.seed } : {},
2824
- ...opts.resamples !== void 0 ? { resamples: opts.resamples } : {}
2825
- }
2826
- );
2827
- const latency = {
2828
- mean: latSig.bootstrap.mean,
2829
- median: latSig.bootstrap.median,
2830
- low: latSig.bootstrap.low,
2831
- high: latSig.bootstrap.high
2832
- };
2833
- if (mode === "superiority") {
2834
- if (sig.fewRuns) return { promoted: false, reason: "few-tasks", mode, n: sig.n, lift, latency };
2835
- return sig.significant ? { promoted: true, reason: "significant", mode, n: sig.n, lift, latency } : { promoted: false, reason: "no-margin", mode, n: sig.n, lift, latency };
2836
- }
2837
- const tolerance = opts.scoreTolerance ?? 0.05;
2838
- const scoreSig = heldoutSignificance(
2839
- { before, after, cellIds },
2840
- {
2841
- deltaThreshold: -tolerance,
2842
- minProductiveRuns: opts.minPairedTasks ?? 6,
2843
- statistic: opts.statistic ?? "mean",
2844
- ...opts.seed !== void 0 ? { seed: opts.seed } : {},
2845
- ...opts.resamples !== void 0 ? { resamples: opts.resamples } : {}
2846
- }
2847
- );
2848
- const costSig = heldoutSignificance(
2849
- { before: candUsd, after: incUsd, cellIds },
2850
- {
2851
- deltaThreshold: 0,
2852
- minProductiveRuns: opts.minPairedTasks ?? 6,
2853
- statistic: opts.statistic ?? "mean",
2854
- ...opts.seed !== void 0 ? { seed: opts.seed } : {},
2855
- ...opts.resamples !== void 0 ? { resamples: opts.resamples } : {}
2856
- }
2857
- );
2858
- const costSavings = {
2859
- mean: costSig.bootstrap.mean,
2860
- median: costSig.bootstrap.median,
2861
- low: costSig.bootstrap.low,
2862
- high: costSig.bootstrap.high
2863
- };
2864
- if (scoreSig.fewRuns)
2865
- return { promoted: false, reason: "few-tasks", mode, n: scoreSig.n, lift, costSavings, latency };
2866
- if (!scoreSig.significant)
2867
- return {
2868
- promoted: false,
2869
- reason: "non-inferiority-unproven",
2870
- mode,
2871
- n: scoreSig.n,
2872
- lift,
2873
- costSavings,
2874
- latency
2875
- };
2876
- if (!costSig.significant)
2877
- return {
2878
- promoted: false,
2879
- reason: "not-cheaper",
2880
- mode,
2881
- n: scoreSig.n,
2882
- lift,
2883
- costSavings,
2884
- latency
2885
- };
2886
- return {
2887
- promoted: true,
2888
- reason: "non-inferior-and-cheaper",
2889
- mode,
2890
- n: scoreSig.n,
2891
- lift,
2892
- costSavings,
2893
- latency
2894
- };
2895
- }
2896
-
2897
- // src/runtime/report-usage.ts
2898
- function reportLoopUsage(cost, result, source = "loop") {
2899
- cost.observe(result.costUsd, source);
2900
- cost.observeTokens(result.tokenUsage);
2901
- }
2902
-
2903
- // src/runtime/run-benchmark.ts
2904
- import { pairedBootstrap as pairedBootstrap2, paretoFrontier } from "@tangle-network/agent-eval";
2905
- async function pool(items, limit, fn) {
2906
- const out = new Array(items.length);
2907
- let next = 0;
2908
- const workers = Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, async () => {
2909
- while (next < items.length) {
2910
- const i = next;
2911
- next += 1;
2912
- out[i] = await fn(items[i], i);
2913
- }
2914
- });
2915
- await Promise.all(workers);
2916
- return out;
2917
- }
2918
- async function runBenchmark(cfg) {
2919
- const strategies = cfg.strategies ?? [sample, refine];
2920
- const budget = cfg.budget ?? 3;
2921
- const concurrency = cfg.concurrency ?? 3;
2922
- let settled = 0;
2923
- const perTask = await pool(cfg.tasks, concurrency, async (task) => {
2924
- const cells = {};
2925
- const errors = {};
2926
- let row;
2927
- try {
2928
- for (const s of strategies) {
2929
- try {
2930
- const r = await runAgentic({
2931
- ...cfg.worker,
2932
- surface: cfg.environment,
2933
- task,
2934
- strategy: s,
2935
- budget,
2936
- ...cfg.hooks ? { hooks: cfg.hooks } : {}
2937
- });
2938
- cells[s.name] = {
2939
- score: r.score,
2940
- resolved: r.resolved,
2941
- progression: r.progression,
2942
- usd: r.usd,
2943
- ms: r.ms,
2944
- tokens: r.tokens
2945
- };
2946
- } catch (e) {
2947
- errors[s.name] = e instanceof Error ? e.message.slice(0, 300) : String(e);
2948
- cells[s.name] = {
2949
- score: 0,
2950
- resolved: false,
2951
- progression: [],
2952
- usd: 0,
2953
- ms: 0,
2954
- tokens: { input: 0, output: 0 }
2955
- };
2956
- }
2957
- }
2958
- row = {
2959
- taskId: task.id,
2960
- cells,
2961
- ...Object.keys(errors).length > 0 ? { errors } : {}
2962
- };
2963
- } catch (e) {
2964
- row = { taskId: task.id, error: e instanceof Error ? e.message.slice(0, 300) : String(e) };
2965
- }
2966
- settled += 1;
2967
- cfg.onTask?.(row, settled, cfg.tasks.length);
2968
- return row;
2969
- });
2970
- const ok = perTask.filter(
2971
- (r) => !!r.cells
2972
- );
2973
- const mean2 = (xs) => xs.length ? xs.reduce((s, x) => s + x, 0) / xs.length : 0;
2974
- const perStrategy = {};
2975
- for (const s of strategies) {
2976
- const cells = ok.map((r) => r.cells[s.name]).filter((c) => !!c);
2977
- perStrategy[s.name] = {
2978
- score: mean2(cells.map((c) => c.score)),
2979
- resolved: mean2(cells.map((c) => c.resolved ? 1 : 0)),
2980
- usd: mean2(cells.map((c) => c.usd)),
2981
- ms: mean2(cells.map((c) => c.ms))
2982
- };
2983
- }
2984
- const frontier = paretoFrontier(
2985
- Object.entries(perStrategy).map(([name, v]) => ({ name, score: v.score, usd: v.usd })),
2986
- [
2987
- { name: "score", direction: "maximize", value: (c) => c.score },
2988
- { name: "usd", direction: "minimize", value: (c) => c.usd }
2989
- ]
2990
- ).frontier.map((c) => c.name);
2991
- const report = {
2992
- n: ok.length,
2993
- excluded: perTask.length - ok.length,
2994
- perStrategy,
2995
- perTask,
2996
- pareto: frontier
2997
- };
2998
- const names = strategies.map((s) => s.name);
2999
- if (names.includes("refine") && names.includes("sample") && ok.length >= 2) {
3000
- const b = pairedBootstrap2(
3001
- ok.map((r) => r.cells.sample?.score ?? 0),
3002
- ok.map((r) => r.cells.refine?.score ?? 0)
3003
- );
3004
- report.refineVsSample = { mean: b.mean, low: b.low, high: b.high, n: b.n };
3005
- }
3006
- return report;
3007
- }
3008
- function printBenchmarkReport(report) {
3009
- const pct2 = (x) => `${(x * 100).toFixed(1)}%`;
3010
- const pp = (x) => `${x >= 0 ? "+" : ""}${(x * 100).toFixed(1)}pp`;
3011
- console.log(
3012
- `
3013
- === benchmark \xB7 n=${report.n}${report.excluded ? ` (excluded ${report.excluded})` : ""} ===`
3014
- );
3015
- console.log(
3016
- ` ${"strategy".padEnd(16)} ${"score".padStart(7)} ${"resolved".padStart(9)} ${"$/task".padStart(8)} ${"s/task".padStart(7)}`
3017
- );
3018
- for (const [s, v] of Object.entries(report.perStrategy))
3019
- console.log(
3020
- ` ${(report.pareto.includes(s) ? `${s} *` : s).padEnd(16)} ${pct2(v.score).padStart(7)} ${pct2(v.resolved).padStart(9)} ${`$${v.usd.toFixed(3)}`.padStart(8)} ${(v.ms / 1e3).toFixed(0).padStart(6)}s`
3021
- );
3022
- if (report.pareto.length) console.log(` * = on the (score, $) Pareto frontier`);
3023
- for (const row of report.perTask)
3024
- if (row.error) console.log(` \u26A0 ${row.taskId}: ${row.error.slice(0, 120)}`);
3025
- const l = report.refineVsSample;
3026
- if (l) {
3027
- const sig = l.low > 0 ? "SIGNIF +" : l.high < 0 ? "SIGNIF -" : "n.s.";
3028
- console.log(` refine \u2212 sample: ${pp(l.mean)} CI [${pp(l.low)}, ${pp(l.high)}] (${sig})`);
3029
- }
3030
- }
3031
-
3032
- // src/runtime/sandbox-run.ts
3033
- var SandboxRunAbortError = class extends Error {
3034
- name = "AbortError";
3035
- /** Events drained from the stream before the abort interrupted the turn. */
3036
- events;
3037
- /** The last artifact read error, if the abort fired during the retry loop. */
3038
- readError;
3039
- constructor(events, readError) {
3040
- super("aborted");
3041
- this.events = events;
3042
- if (readError !== void 0) this.readError = readError;
3043
- }
3044
- };
3045
- async function openSandboxRun(client, options, deliverable) {
3046
- const runId = options.runId ?? `sandbox-run-${randomSuffix()}`;
3047
- const now = options.now ?? Date.now;
3048
- const capabilities = await probeSandboxCapabilities(client);
3049
- const lineage = createSandboxLineage(client, capabilities, {
3050
- ...options.maxConcurrency !== void 0 ? { maxConcurrency: options.maxConcurrency } : {}
3051
- });
3052
- let handle;
3053
- let started = false;
3054
- let runStartedAt;
3055
- let failed = false;
3056
- let turnCount = 0;
3057
- function emit(event) {
3058
- notifyRuntimeHookEvent(
3059
- options.hooks,
3060
- {
3061
- id: `${runId}:${event.target}:${event.phase}${event.stepIndex === void 0 ? "" : `:${event.stepIndex}`}`,
3062
- runId,
3063
- scenarioId: options.scenarioId,
3064
- target: event.target,
3065
- phase: event.phase,
3066
- timestamp: event.timestamp,
3067
- stepIndex: event.stepIndex,
3068
- payload: event.payload,
3069
- metadata: { producer: "openSandboxRun" }
3070
- },
3071
- { signal: options.signal }
3072
- );
3073
- }
3074
- const runPayload = () => ({
3075
- agentName: options.agentRun.name ?? options.agentRun.profile.name ?? "agent",
3076
- profileName: options.agentRun.profile.name,
3077
- backendType: backendType(options.agentRun),
3078
- deliverableKind: deliverable.kind,
3079
- ...deliverable.kind === "artifact" ? { deliverablePath: deliverable.path } : {},
3080
- ...handle ? { sessionId: handle.sessionId, sandboxId: handle.box.id } : {}
3081
- });
3082
- const turnPayload = (prompt, turnKind, startedAt, result, error) => ({
3083
- ...runPayload(),
3084
- turnKind,
3085
- promptChars: prompt.length,
3086
- promptHash: hashText(prompt),
3087
- ...result !== void 0 || error !== void 0 ? { durationMs: Math.max(0, now() - startedAt) } : {},
3088
- ...result ? {
3089
- eventCount: result.events.length,
3090
- eventTypes: eventTypeCounts(result.events),
3091
- ...result.readError !== void 0 ? { readError: result.readError } : {}
3092
- } : {},
3093
- ...error !== void 0 ? { error: errorMessage(error) } : {}
3094
- });
3095
- async function settle2(box, events) {
3096
- const collected = [];
3097
- try {
3098
- for await (const ev of events) collected.push(ev);
3099
- } catch (err) {
3100
- if (isAbortError(err)) throw new SandboxRunAbortError(collected);
3101
- throw err;
3102
- }
3103
- if (deliverable.kind === "events") {
3104
- return { out: deliverable.fromEvents(collected), events: collected };
3105
- }
3106
- if (options.signal.aborted) throw new SandboxRunAbortError(collected);
3107
- let raw = "";
3108
- let readError;
3109
- const readAttempts = 4;
3110
- const readDelayMs = options.readRetryDelayMs ?? 1e3;
3111
- for (let attempt = 0; attempt < readAttempts; attempt += 1) {
3112
- if (options.signal.aborted) throw new SandboxRunAbortError(collected, readError);
3113
- try {
3114
- raw = await box.fs.read(deliverable.path);
3115
- readError = void 0;
3116
- break;
3117
- } catch (err) {
3118
- readError = err instanceof Error ? err.message : String(err);
3119
- if (attempt < readAttempts - 1 && readDelayMs > 0)
3120
- await sleep(readDelayMs * (attempt + 1), options.signal);
3121
- }
3122
- }
3123
- return {
3124
- out: deliverable.fromArtifact(raw, collected),
3125
- events: collected,
3126
- ...readError !== void 0 ? { readError } : {}
3127
- };
3128
- }
3129
- return {
3130
- get box() {
3131
- if (!handle) throw new Error("openSandboxRun: box unavailable before start()");
3132
- return handle.box;
3133
- },
3134
- get sessionId() {
3135
- if (!handle) throw new Error("openSandboxRun: sessionId unavailable before start()");
3136
- return handle.sessionId;
3137
- },
3138
- async start(prompt) {
3139
- if (started)
3140
- throw new Error(
3141
- "openSandboxRun: start() already called \u2014 use resume() to continue the session"
3142
- );
3143
- started = true;
3144
- runStartedAt = now();
3145
- emit({
3146
- target: "agent.run",
3147
- phase: "before",
3148
- timestamp: runStartedAt,
3149
- payload: { ...runPayload(), turnCount: 0 }
3150
- });
3151
- const stepIndex = turnCount;
3152
- const turnStartedAt = now();
3153
- emit({
3154
- target: "agent.turn",
3155
- phase: "before",
3156
- timestamp: turnStartedAt,
3157
- stepIndex,
3158
- payload: turnPayload(prompt, "start", turnStartedAt)
3159
- });
3160
- try {
3161
- const r = await lineage.start(
3162
- options.agentRun,
3163
- prompt,
3164
- options.signal,
3165
- options.promptOptions
3166
- );
3167
- handle = r.handle;
3168
- await options.beforeStart?.({
3169
- box: handle.box,
3170
- sessionId: handle.sessionId,
3171
- signal: options.signal
3172
- });
3173
- const result = await settle2(handle.box, r.events);
3174
- turnCount += 1;
3175
- emit({
3176
- target: "agent.turn",
3177
- phase: "after",
3178
- timestamp: now(),
3179
- stepIndex,
3180
- payload: turnPayload(prompt, "start", turnStartedAt, result)
3181
- });
3182
- return result;
3183
- } catch (error) {
3184
- failed = true;
3185
- emit({
3186
- target: "agent.turn",
3187
- phase: "error",
3188
- timestamp: now(),
3189
- stepIndex,
3190
- payload: turnPayload(prompt, "start", turnStartedAt, void 0, error)
3191
- });
3192
- emit({
3193
- target: "agent.run",
3194
- phase: "error",
3195
- timestamp: now(),
3196
- payload: { ...runPayload(), turnCount, error: errorMessage(error) }
3197
- });
3198
- throw error;
3199
- }
3200
- },
3201
- async resume(prompt) {
3202
- if (!handle) throw new Error("openSandboxRun: resume() called before start()");
3203
- const stepIndex = turnCount;
3204
- const turnStartedAt = now();
3205
- emit({
3206
- target: "agent.turn",
3207
- phase: "before",
3208
- timestamp: turnStartedAt,
3209
- stepIndex,
3210
- payload: turnPayload(prompt, "resume", turnStartedAt)
3211
- });
3212
- try {
3213
- const result = await settle2(
3214
- handle.box,
3215
- await lineage.continue(handle, prompt, options.signal, options.promptOptions)
3216
- );
3217
- turnCount += 1;
3218
- emit({
3219
- target: "agent.turn",
3220
- phase: "after",
3221
- timestamp: now(),
3222
- stepIndex,
3223
- payload: turnPayload(prompt, "resume", turnStartedAt, result)
3224
- });
3225
- return result;
3226
- } catch (error) {
3227
- failed = true;
3228
- emit({
3229
- target: "agent.turn",
3230
- phase: "error",
3231
- timestamp: now(),
3232
- stepIndex,
3233
- payload: turnPayload(prompt, "resume", turnStartedAt, void 0, error)
3234
- });
3235
- emit({
3236
- target: "agent.run",
3237
- phase: "error",
3238
- timestamp: now(),
3239
- payload: { ...runPayload(), turnCount, error: errorMessage(error) }
3240
- });
3241
- throw error;
3242
- }
3243
- },
3244
- async close() {
3245
- await lineage.teardown();
3246
- if (runStartedAt !== void 0) {
3247
- emit({
3248
- target: "agent.run",
3249
- phase: "after",
3250
- timestamp: now(),
3251
- payload: {
3252
- ...runPayload(),
3253
- turnCount,
3254
- status: failed ? "error" : "completed",
3255
- durationMs: Math.max(0, now() - runStartedAt)
3256
- }
3257
- });
3258
- }
3259
- }
3260
- };
3261
- }
3262
- function backendType(spec) {
3263
- const backend = spec.sandboxOverrides?.backend;
3264
- return backend?.type;
3265
- }
3266
- function eventTypeCounts(events) {
3267
- const counts = {};
3268
- for (const event of events) counts[event.type] = (counts[event.type] ?? 0) + 1;
3269
- return counts;
3270
- }
3271
- function hashText(value) {
3272
- let hash = 2166136261;
3273
- for (let i = 0; i < value.length; i += 1) {
3274
- hash ^= value.charCodeAt(i);
3275
- hash = Math.imul(hash, 16777619);
3276
- }
3277
- return (hash >>> 0).toString(16).padStart(8, "0");
3278
- }
3279
- function errorMessage(error) {
3280
- return error instanceof Error ? error.message : String(error);
3281
- }
3282
-
3283
- // src/runtime/strategy-author.ts
3284
- import { mkdirSync as mkdirSync2, writeFileSync as writeFileSync2 } from "fs";
3285
- import { join as join3 } from "path";
3286
- var strategyAuthorContract = `
3287
- You author an OPTIMIZATION STRATEGY for an agentic loop system. A strategy decides how to
3288
- spend a compute budget to beat a task's deployable check. You compose exactly two steps:
3289
-
3290
- shot(spec?: { handle?, messages?, steer?, persona?, tools? }): Promise<ShotResult | null>
3291
- Runs ONE worker attempt (a bounded tool loop) over an artifact.
3292
- - omit handle => the shot opens its OWN fresh artifact and closes it after (a sample).
3293
- - pass handle => the shot CONTINUES that artifact (state accumulates across shots).
3294
- - messages => the carried conversation (pass the previous ShotResult.messages to continue).
3295
- - steer => a corrective instruction injected before the shot.
3296
- - persona => { systemPrompt?, model? } \u2014 give THIS shot its own role and/or model
3297
- (multi-agent strategies: a researcher shot then an engineer shot, a panel of k
3298
- personas over one budget). On a fresh shot the systemPrompt replaces the task's; on
3299
- a carried conversation it arrives as a hand-off message. Same conserved budget.
3300
- - tools => string[] \u2014 restrict THIS shot to a subset of the task's tools by
3301
- name (focus an explore shot on read-only tools, an execute shot on write tools).
3302
- Restriction-only; unknown names make the shot fail. ALWAYS select from
3303
- await listTools(handle) \u2014 never hardcode. Omitted => the shot sees every tool.
3304
- ShotResult = { messages, score (0..1 on the task's check), passes, total, completions, toolErrors }
3305
- Returns null if the attempt failed infra-wise.
3306
-
3307
- critique(messages): Promise<string | null>
3308
- A firewalled trace-analyst reads the attempt's trajectory and returns ONE corrective
3309
- instruction (or null when it judges the work complete). Costs ~1 completion.
3310
-
3311
- consult(messages, instruction): Promise<string | null>
3312
- The RAW analyst channel: the same firewalled critic answers YOUR instruction over the
3313
- trajectory verbatim (no reformatting) \u2014 use it when you need a specific reply format
3314
- (a decision, a prediction). Costs ~1 completion.
3315
-
3316
- surface.open(task) / surface.close(handle)
3317
- Open a persistent artifact you manage yourself (remember to close in a finally).
3318
- close is idempotent \u2014 closing an already-closed handle is a safe no-op.
3319
-
3320
- listTools(handle): Promise<Array<{ name, description? }>>
3321
- The tools THIS task actually offers. TOOL SETS VARY PER TASK \u2014 if you restrict a
3322
- shot with \`tools\`, you MUST pick names from await listTools(handle); hardcoding
3323
- names from an example kills your shots on every task whose tools differ.
3324
-
3325
- Rules:
3326
- - ALWAYS await every shot/critique/surface call \u2014 a floating promise that rejects
3327
- crashes the whole benchmark run.
3328
- - Stay within ~budget total shots; every shot/critique spends from a conserved pool.
3329
- - For a FRESH attempt OMIT \`messages\` entirely (never pass \`[]\` \u2014 an empty array is a
3330
- fresh conversation too, but be explicit). To CONTINUE, pass the previous
3331
- ShotResult.messages unchanged.
3332
- - Return { score, resolved, completions, progression, shots } \u2014 score = the BEST checkpoint
3333
- you reached (keep-best, never final-state), progression = score after each shot.
3334
- - The module must be EXACTLY this shape (no other imports, no commentary outside code):
3335
-
3336
- import { defineStrategy } from '@tangle-network/agent-runtime/loops'
3337
- export default defineStrategy('your-strategy-name', async ({ surface, task, budget, shot, critique, listTools }) => {
3338
- // your composition (listTools comes from the destructured context \u2014 it is NOT a global)
3339
- })
3340
- `;
3341
- function assertStrategyContract(code) {
3342
- const allowedImport = /^\s*import\s+\{[^}]*\}\s+from\s+['"]@tangle-network\/agent-runtime\/loops['"]/;
3343
- for (const line of code.split("\n")) {
3344
- if (/^\s*import\s/.test(line) && !allowedImport.test(line)) {
3345
- throw new Error(`authored code rejected: foreign import \u2014 ${line.trim().slice(0, 120)}`);
3346
- }
3347
- }
3348
- const banned = [
3349
- [/\brequire\s*\(/, "require()"],
3350
- [/\bimport\s*\(/, "dynamic import()"],
3351
- [/\beval\s*\(/, "eval()"],
3352
- [/new\s+Function\s*\(/, "new Function()"],
3353
- [/\bprocess\s*[.[]/, "process access"],
3354
- [/\bglobalThis\s*[.[]/, "globalThis access"],
3355
- [/\bfetch\s*\(/, "network access"],
3356
- [/child_process|node:fs|node:net|node:http|worker_threads/, "node builtin access"]
3357
- ];
3358
- for (const [re, what] of banned) {
3359
- if (re.test(code)) throw new Error(`authored code rejected: ${what}`);
3360
- }
3361
- }
3362
- async function requestAuthoredCode(opts, model) {
3363
- const res = await opts.chat.chat(
3364
- {
3365
- ...model ? { model } : {},
3366
- ...opts.temperature !== void 0 ? { temperature: opts.temperature } : {},
3367
- ...opts.maxTokens !== void 0 ? { maxTokens: opts.maxTokens } : {},
3368
- messages: [
3369
- {
3370
- role: "system",
3371
- content: "You are a senior researcher authoring optimization strategies for agent loops: you read per-task losses like experimental data, form a mechanism-level hypothesis, and author the one composition that tests it. Output exactly one fenced ```ts code block and nothing else."
3372
- },
3373
- {
3374
- role: "user",
3375
- content: `${opts.contract ?? strategyAuthorContract}
3376
-
3377
- BASELINE RESULTS on the "${opts.environmentName}" environment (budget=${opts.budget}) \u2014 the per-task losses are your gradient:
3378
- ${opts.lossesJson}
3379
-
3380
- Author ONE new strategy that you expect to beat the baselines on THIS environment at the same budget.
3381
- ${strategyAuthorMethod}
3382
-
3383
- Output only the module code block.`
3384
- }
3385
- ]
3386
- },
3387
- { ...opts.signal ? { signal: opts.signal } : {} }
3388
- );
3389
- const match = res.content.match(/```(?:ts|typescript)?\s*\n([\s\S]*?)```/);
3390
- if (!match?.[1]) {
3391
- throw new Error(
3392
- `authorStrategy: no code block in the author's reply (model=${model ?? "default"}): ${res.content.slice(0, 300)}`
3393
- );
3394
- }
3395
- return match[1];
3396
- }
3397
- async function authorStrategy(opts) {
3398
- let code;
3399
- try {
3400
- code = await requestAuthoredCode(opts, opts.model);
3401
- } catch (primaryError) {
3402
- if (!opts.fallbackModel) throw primaryError;
3403
- code = await requestAuthoredCode(opts, opts.fallbackModel);
3404
- }
3405
- assertStrategyContract(code);
3406
- mkdirSync2(opts.outDir, { recursive: true });
3407
- const file = join3(opts.outDir, `authored-${Date.now()}.mts`);
3408
- writeFileSync2(file, code);
3409
- const mod = await import(`file://${file}`);
3410
- if (!mod.default || typeof mod.default.driver !== "function" || !mod.default.name) {
3411
- throw new Error(`authorStrategy: ${file} does not export a default Strategy`);
3412
- }
3413
- return { strategy: mod.default, file, code };
3414
- }
3415
-
3416
- // src/runtime/strategy-evolution.ts
3417
- import { existsSync, readFileSync, writeFileSync as writeFileSync3 } from "fs";
3418
- import { gzipSync } from "zlib";
3419
- function discriminatingMeans(report, fieldOrder) {
3420
- const rows = report.perTask.filter((r) => {
3421
- if (!r.cells) return false;
3422
- const scores = fieldOrder.map((n) => r.cells?.[n]?.score).filter((s) => s !== void 0);
3423
- if (scores.length < fieldOrder.length) return false;
3424
- return Math.max(...scores) - Math.min(...scores) > 0;
3425
- });
3426
- if (rows.length === 0) return null;
3427
- const out = {};
3428
- for (const name of fieldOrder) {
3429
- const cells = rows.map((r) => r.cells?.[name]).filter((c) => !!c);
3430
- out[name] = {
3431
- score: cells.reduce((s, c) => s + c.score, 0) / cells.length,
3432
- usd: cells.reduce((s, c) => s + c.usd, 0) / cells.length
3433
- };
3434
- }
3435
- return out;
3436
- }
3437
- function pickChampion(means, fieldOrder, policy, epsilon) {
3438
- const entries = fieldOrder.map((name) => ({ name, summary: means[name] })).filter((e) => !!e.summary);
3439
- if (entries.length === 0)
3440
- throw new Error("pickChampion: the means table carries none of the field strategies");
3441
- const best = Math.max(...entries.map((e) => e.summary.score));
3442
- const pick = policy === "score" ? entries.find((e) => e.summary.score === best) : entries.filter((e) => e.summary.score >= best - epsilon).sort((a, b) => a.summary.usd - b.summary.usd || b.summary.score - a.summary.score)[0];
3443
- if (!pick) throw new Error("pickChampion: empty pick (unreachable)");
3444
- return { name: pick.name, score: pick.summary.score, usd: pick.summary.usd };
3445
- }
3446
- function selectChampion(report, fieldOrder, policy, epsilon) {
3447
- return pickChampion(report.perStrategy, fieldOrder, policy, epsilon);
3448
- }
3449
- var fieldSummary = (archive) => archive.map(
3450
- (n) => `- ${n.name} (${n.source}, gen ${n.generation}, last score ${(n.score * 100).toFixed(0)}%)`
3451
- ).join("\n");
3452
- var compactLosses = (report, detail) => {
3453
- const r2 = (x) => Math.round(x * 100) / 100;
3454
- const rows = report.perTask.map(
3455
- (row) => row.cells ? {
3456
- task: row.taskId,
3457
- ...row.errors ? {
3458
- errors: Object.fromEntries(
3459
- Object.entries(row.errors).map(([n, msg]) => [n, msg.slice(0, 100)])
3460
- )
3461
- } : {},
3462
- cells: Object.fromEntries(
3463
- Object.entries(row.cells).map(([name, c]) => [
3464
- name,
3465
- // 'binary' is the leakage-bounded channel: the author learns pass/fail per
3466
- // task and nothing else — the per-generation leak from the evaluation data
3467
- // is capped at one bit per cell (arXiv:2606.11045 measured that exploration
3468
- // survives this; whether AUTHORING does is the E1-coarse A/B).
3469
- detail === "binary" ? { resolved: c.resolved, usd: Math.round(c.usd * 1e4) / 1e4 } : {
3470
- score: r2(c.score),
3471
- resolved: c.resolved,
3472
- usd: Math.round(c.usd * 1e4) / 1e4,
3473
- progression: (c.progression ?? []).map(r2)
3474
- }
3475
- ])
3476
- )
3477
- } : { task: row.taskId, error: row.error?.slice(0, 80) }
3478
- );
3479
- return JSON.stringify(rows).slice(0, 12e3);
3480
- };
3481
- function renameStrategy(orig, unique) {
3482
- if (orig.name === unique) return orig;
3483
- return {
3484
- name: unique,
3485
- driver: (s, t, o, b) => {
3486
- const agent = orig.driver(s, t, o, b);
3487
- return {
3488
- ...agent,
3489
- name: unique,
3490
- act: async (task, scope) => {
3491
- const out = await agent.act(task, scope);
3492
- if (out.kind !== "done") return out;
3493
- const deliverable = { ...out.deliverable, mode: unique };
3494
- return { ...out, deliverable };
3495
- }
3496
- };
3497
- }
3498
- };
3499
- }
3500
- async function runStrategyEvolution(cfg) {
3501
- const budget = cfg.budget ?? 3;
3502
- const concurrency = cfg.concurrency ?? 3;
3503
- const generations = cfg.generations ?? 2;
3504
- const populationSize = cfg.populationSize ?? 2;
3505
- const baselines = cfg.baselines ?? [sample, refine, sampleThenRefine];
3506
- const policy = cfg.champion ?? "costAware";
3507
- const epsilon = cfg.championEpsilon ?? (cfg.objective === "cost" ? cfg.scoreTolerance ?? 0.05 : 0.01);
3508
- const byName = new Map(baselines.map((s) => [s.name, s]));
3509
- const codeByName = /* @__PURE__ */ new Map();
3510
- const fingerprint = {
3511
- trainN: cfg.trainN,
3512
- holdoutN: cfg.holdoutN,
3513
- budget,
3514
- generations,
3515
- populationSize
3516
- };
3517
- let ckpt;
3518
- if (cfg.checkpoint?.resume && existsSync(cfg.checkpoint.path)) {
3519
- const raw = JSON.parse(readFileSync(cfg.checkpoint.path, "utf8"));
3520
- if (JSON.stringify(raw.fingerprint) !== JSON.stringify(fingerprint)) {
3521
- throw new Error(
3522
- `evolution resume: checkpoint design mismatch \u2014 checkpoint ${JSON.stringify(raw.fingerprint)} vs config ${JSON.stringify(fingerprint)}; delete ${cfg.checkpoint.path} or match the config`
3523
- );
3524
- }
3525
- ckpt = raw;
3526
- }
3527
- const save = (state) => {
3528
- if (cfg.checkpoint)
3529
- writeFileSync3(cfg.checkpoint.path, JSON.stringify({ ...state, fingerprint }, null, 1));
3530
- };
3531
- const bench = async (phase, tasks, strategies) => {
3532
- await cfg.onPhase?.(phase);
3533
- return runBenchmark({
3534
- environment: cfg.environment,
3535
- tasks,
3536
- worker: cfg.worker,
3537
- strategies,
3538
- budget,
3539
- concurrency,
3540
- ...cfg.onTask ? { onTask: (row, done, total) => cfg.onTask?.(phase, row, done, total) } : {},
3541
- ...cfg.hooks ? { hooks: cfg.hooks } : {}
3542
- });
3543
- };
3544
- const train = await cfg.tasks(0, cfg.trainN);
3545
- const probeTask = train[0];
3546
- if (!probeTask) throw new Error("runStrategyEvolution: empty train slice");
3547
- const probe = await cfg.environment.open(probeTask);
3548
- let toolCatalog;
3549
- try {
3550
- const tools = await cfg.environment.tools(probeTask, probe);
3551
- toolCatalog = tools.map(
3552
- (t) => `- ${t.function.name}${t.function.description ? ` \u2014 ${t.function.description.slice(0, 120)}` : ""}`
3553
- ).join("\n");
3554
- } finally {
3555
- await cfg.environment.close(probe);
3556
- }
3557
- const gen0 = ckpt?.gen0 ?? await bench("gen0", train, baselines);
3558
- const archive = ckpt?.archive ? [...ckpt.archive] : baselines.map((s) => ({
3559
- name: s.name,
3560
- source: "baseline",
3561
- generation: 0,
3562
- score: gen0.perStrategy[s.name]?.score ?? 0,
3563
- usd: gen0.perStrategy[s.name]?.usd ?? 0
3564
- }));
3565
- const gen0Champion = ckpt?.gen0Champion ?? selectChampion(
3566
- gen0,
3567
- baselines.map((s) => s.name),
3568
- policy,
3569
- epsilon
3570
- );
3571
- const generationRows = ckpt?.generations ? [...ckpt.generations] : [];
3572
- const trajectory = ckpt?.trajectory ? [...ckpt.trajectory] : [
3573
- {
3574
- generation: 0,
3575
- champion: gen0Champion.name,
3576
- score: gen0Champion.score,
3577
- usd: gen0Champion.usd
3578
- }
3579
- ];
3580
- for (const row of generationRows) {
3581
- for (const c of row.candidates) {
3582
- if (!c.file || c.error) continue;
3583
- const mod = await import(`file://${c.file}`);
3584
- if (!mod.default || typeof mod.default.driver !== "function") {
3585
- throw new Error(
3586
- `evolution resume: ${c.file} no longer exports a Strategy \u2014 cannot restore "${c.name}"`
3587
- );
3588
- }
3589
- byName.set(c.name, renameStrategy(mod.default, c.name));
3590
- codeByName.set(c.name, readFileSync(c.file, "utf8"));
3591
- }
3592
- }
3593
- let authoredOk = generationRows.reduce(
3594
- (n, row) => n + row.candidates.filter((c) => !c.error).length,
3595
- 0
3596
- );
3597
- const lastRow = generationRows[generationRows.length - 1];
3598
- let incumbent = lastRow ? lastRow.champion : gen0Champion;
3599
- let latestReport = lastRow ? lastRow.report : gen0;
3600
- if (!ckpt) save({ gen0, gen0Champion, generations: generationRows, archive, trajectory });
3601
- for (let g = generationRows.length + 1; g <= generations; g += 1) {
3602
- const lossesJson = compactLosses(latestReport, cfg.lossesDetail ?? "exact");
3603
- const candidates = [];
3604
- const newStrategies = [];
3605
- for (let i = 0; i < populationSize; i += 1) {
3606
- const objectiveNote = cfg.objective === "cost" ? `
3607
-
3608
- YOUR OBJECTIVE: match or exceed the incumbent's SCORE while spending LESS (the losses include usd per task). Promotion requires proven score non-inferiority PLUS significant cost savings \u2014 a strategy that ties the score at half the cost WINS; a cheaper strategy that loses score by more than ${((cfg.scoreTolerance ?? 0.05) * 100).toFixed(0)}pp LOSES.` : "";
3609
- const contract = `${strategyAuthorContract}${objectiveNote}
3610
-
3611
- EXAMPLE TOOLS FROM ONE TASK (tool sets VARY per task on this domain \u2014 a strategy MUST select tool names from await listTools(handle) at runtime; hardcoding these example names will zero your score on most tasks):
3612
- ${toolCatalog}
3613
-
3614
- STRATEGIES ALREADY IN THE TOURNAMENT (author something MEANINGFULLY different \u2014 a new composition, not a rename):
3615
- ${fieldSummary(archive)}
3616
-
3617
- You are authoring candidate ${i + 1} of ${populationSize} this generation; explore a distinct region of the strategy space from your siblings.`;
3618
- try {
3619
- const authored = await authorStrategy({
3620
- chat: cfg.author.chat,
3621
- ...cfg.author.model ? { model: cfg.author.model } : {},
3622
- ...cfg.author.fallbackModel ? { fallbackModel: cfg.author.fallbackModel } : {},
3623
- ...cfg.author.temperature !== void 0 ? { temperature: cfg.author.temperature } : {},
3624
- ...cfg.author.maxTokens !== void 0 ? { maxTokens: cfg.author.maxTokens } : {},
3625
- contract,
3626
- environmentName: cfg.environment.name,
3627
- lossesJson,
3628
- budget,
3629
- outDir: cfg.outDir
3630
- });
3631
- const unique = byName.has(authored.strategy.name) ? `${authored.strategy.name}-g${g}c${i + 1}` : authored.strategy.name;
3632
- const strategy = renameStrategy(authored.strategy, unique);
3633
- byName.set(unique, strategy);
3634
- codeByName.set(unique, authored.code);
3635
- newStrategies.push(strategy);
3636
- archive.push({
3637
- name: unique,
3638
- source: "authored",
3639
- generation: g,
3640
- parent: incumbent.name,
3641
- gzipBits: gzipSync(Buffer.from(authored.code)).length * 8,
3642
- file: authored.file,
3643
- score: 0,
3644
- usd: 0
3645
- });
3646
- candidates.push({
3647
- name: unique,
3648
- file: authored.file,
3649
- gzipBits: gzipSync(Buffer.from(authored.code)).length * 8,
3650
- codeChars: authored.code.length
3651
- });
3652
- authoredOk += 1;
3653
- } catch (e) {
3654
- candidates.push({
3655
- name: `(author-failed g${g}c${i + 1})`,
3656
- error: e instanceof Error ? e.message.slice(0, 300) : String(e)
3657
- });
3658
- }
3659
- }
3660
- const incumbentStrategy = byName.get(incumbent.name);
3661
- if (!incumbentStrategy)
3662
- throw new Error(`evolution: incumbent "${incumbent.name}" missing from the field`);
3663
- const field = [incumbentStrategy, ...newStrategies];
3664
- const report = await bench(`gen${g}`, train, field);
3665
- for (const node of archive) {
3666
- const cell = report.perStrategy[node.name];
3667
- if (cell) {
3668
- node.score = cell.score;
3669
- node.usd = cell.usd;
3670
- }
3671
- }
3672
- const fieldNames = field.map((s) => s.name);
3673
- const means = cfg.band ? discriminatingMeans(report, fieldNames) ?? report.perStrategy : report.perStrategy;
3674
- const champion = pickChampion(means, fieldNames, policy, epsilon);
3675
- generationRows.push({ generation: g, candidates, report, champion });
3676
- trajectory.push({
3677
- generation: g,
3678
- champion: champion.name,
3679
- score: champion.score,
3680
- usd: champion.usd
3681
- });
3682
- incumbent = champion;
3683
- latestReport = report;
3684
- save({ gen0, gen0Champion, generations: generationRows, archive, trajectory });
3685
- }
3686
- if (authoredOk === 0) {
3687
- throw new Error(
3688
- "runStrategyEvolution: every author attempt failed across all generations \u2014 no search happened; see the candidates[].error entries"
3689
- );
3690
- }
3691
- const holdoutOffset = cfg.trainN + (cfg.holdoutOffset ?? 0);
3692
- let holdoutTasks = [];
3693
- let bandInfo;
3694
- if (ckpt?.holdout && ckpt.verdict) {
3695
- bandInfo = ckpt.band;
3696
- if (cfg.reproducerCheck && codeByName.has(incumbent.name)) {
3697
- const pool2 = await cfg.tasks(holdoutOffset, cfg.band?.holdoutPoolN ?? cfg.holdoutN);
3698
- const gateIds = new Set(ckpt.holdout.perTask.map((r) => r.taskId));
3699
- holdoutTasks = pool2.filter((t) => gateIds.has(t.id));
3700
- }
3701
- } else if (cfg.band) {
3702
- const maxRef = cfg.band.maxRefScore ?? 0.99;
3703
- const reference = baselines[0];
3704
- if (!reference)
3705
- throw new Error("evolution band: baselines[0] required as the screening reference");
3706
- const pool2 = await cfg.tasks(holdoutOffset, cfg.band.holdoutPoolN);
3707
- const screen = await bench("band-screen", pool2, [reference]);
3708
- const refScores = screen.perTask.filter((r) => r.cells?.[reference.name]).map((r) => ({ taskId: r.taskId, score: r.cells?.[reference.name]?.score ?? 0 }));
3709
- const inBandIds = new Set(refScores.filter((r) => r.score <= maxRef).map((r) => r.taskId));
3710
- const kept = pool2.filter((t) => inBandIds.has(t.id));
3711
- if (kept.length < cfg.holdoutN) {
3712
- throw new Error(
3713
- `evolution band: only ${kept.length}/${cfg.holdoutN} holdout tasks have headroom (pool ${cfg.band.holdoutPoolN}, reference "${reference.name}" \u2264 ${maxRef}) \u2014 widen holdoutPoolN or raise maxRefScore`
3714
- );
3715
- }
3716
- holdoutTasks = kept.slice(0, cfg.holdoutN);
3717
- bandInfo = { screened: refScores.length, inBand: kept.length, refScores };
3718
- } else {
3719
- holdoutTasks = await cfg.tasks(holdoutOffset, cfg.holdoutN);
3720
- }
3721
- let holdout;
3722
- let verdict;
3723
- if (ckpt?.holdout && ckpt.verdict) {
3724
- holdout = ckpt.holdout;
3725
- verdict = ckpt.verdict;
3726
- } else {
3727
- const finalists = [.../* @__PURE__ */ new Set([gen0Champion.name, incumbent.name])].map((n) => byName.get(n)).filter((s) => !!s);
3728
- holdout = await bench("holdout", holdoutTasks, finalists);
3729
- verdict = promotionGate({
3730
- report: holdout,
3731
- incumbent: gen0Champion.name,
3732
- candidate: incumbent.name,
3733
- ...cfg.objective === "cost" ? {
3734
- mode: "non-inferiority",
3735
- ...cfg.scoreTolerance !== void 0 ? { scoreTolerance: cfg.scoreTolerance } : {}
3736
- } : {},
3737
- ...cfg.minPairedTasks !== void 0 ? { minPairedTasks: cfg.minPairedTasks } : {}
3738
- });
3739
- save({
3740
- gen0,
3741
- gen0Champion,
3742
- generations: generationRows,
3743
- archive,
3744
- trajectory,
3745
- holdout,
3746
- verdict,
3747
- ...bandInfo ? { band: bandInfo } : {}
3748
- });
3749
- }
3750
- let reproduction;
3751
- const championCode = codeByName.get(incumbent.name);
3752
- if (cfg.reproducerCheck && championCode) {
3753
- const words = cfg.reproducerCheck.summaryMaxWords ?? 64;
3754
- const tolerance = cfg.reproducerCheck.tolerance ?? 0.05;
3755
- const championHoldoutScore = holdout.perStrategy[incumbent.name]?.score ?? 0;
3756
- try {
3757
- const summaryRes = await cfg.author.chat.chat({
3758
- ...cfg.author.model ? { model: cfg.author.model } : {},
3759
- temperature: 0.2,
3760
- maxTokens: 512,
3761
- messages: [
3762
- {
3763
- role: "system",
3764
- content: `Summarize the optimization strategy implemented by this code in at most ${words} words. Describe the COMPOSITION (shots, critique, artifact handling, restarts, stopping) \u2014 not the code. Output only the summary.`
3765
- },
3766
- { role: "user", content: championCode }
3767
- ]
3768
- });
3769
- const summary = summaryRes.content.trim();
3770
- const reproduced = await authorStrategy({
3771
- chat: cfg.author.chat,
3772
- ...cfg.author.model ? { model: cfg.author.model } : {},
3773
- ...cfg.author.fallbackModel ? { fallbackModel: cfg.author.fallbackModel } : {},
3774
- ...cfg.author.maxTokens !== void 0 ? { maxTokens: cfg.author.maxTokens } : {},
3775
- temperature: 0.2,
3776
- contract: `${strategyAuthorContract}
3777
-
3778
- IMPLEMENT EXACTLY THIS STRATEGY (a colleague's description \u2014 do not invent a different approach):
3779
- ${summary}`,
3780
- environmentName: cfg.environment.name,
3781
- lossesJson: "[]",
3782
- budget,
3783
- outDir: cfg.outDir
3784
- });
3785
- const reproStrategy = {
3786
- name: `${incumbent.name}-reproduced`,
3787
- driver: reproduced.strategy.driver
3788
- };
3789
- const reproReport = await bench("reproduce", holdoutTasks, [reproStrategy]);
3790
- const reproducedHoldoutScore = reproReport.perStrategy[reproStrategy.name]?.score ?? 0;
3791
- reproduction = {
3792
- summary,
3793
- reproducedName: reproStrategy.name,
3794
- file: reproduced.file,
3795
- championHoldoutScore,
3796
- reproducedHoldoutScore,
3797
- gap: championHoldoutScore - reproducedHoldoutScore,
3798
- reproducible: reproducedHoldoutScore >= championHoldoutScore - tolerance
3799
- };
3800
- } catch (e) {
3801
- reproduction = {
3802
- summary: "",
3803
- reproducedName: "",
3804
- championHoldoutScore,
3805
- reproducedHoldoutScore: 0,
3806
- gap: championHoldoutScore,
3807
- reproducible: false,
3808
- error: e instanceof Error ? e.message.slice(0, 300) : String(e)
3809
- };
3810
- }
3811
- }
3812
- return {
3813
- gen0,
3814
- gen0Champion,
3815
- generations: generationRows,
3816
- archive,
3817
- finalChampion: incumbent,
3818
- holdout,
3819
- verdict,
3820
- ...bandInfo ? { band: bandInfo } : {},
3821
- ...reproduction ? { reproduction } : {},
3822
- trajectory
3823
- };
3824
- }
3825
-
3826
- // src/runtime/stream-agent-turn.ts
3827
- import { scoreKnowledgeReadiness } from "@tangle-network/agent-eval";
3828
- async function* streamAgentTurn(backend, prompt, opts = {}) {
3829
- const label = backend.kind === "chat" ? backend.backend.kind : backend.kind;
3830
- const task = { id: `turn-${crypto.randomUUID()}`, intent: prompt };
3831
- const acc = { deltaText: "", input: 0, output: 0, costUsd: 0 };
3832
- const deadline = deriveTurnSignal(opts.signal, opts.timeoutMs ?? 0);
3833
- let session;
3834
- try {
3835
- session = await startTurnSession(backend, task, prompt, deadline.signal, label);
3836
- yield { type: "backend_start", task, session, backend: label, timestamp: nowIso() };
3837
- const inner = backend.kind === "chat" ? driveChatTurn(backend.backend, task, session, prompt, deadline.signal, acc) : driveBoxTurn(
3838
- backend.kind === "executor" ? await inlineSandboxClient(backend.factory).create() : backend.box,
3839
- prompt,
3840
- deadline.signal,
3841
- backend.agentRunName ?? "agent",
3842
- acc,
3843
- {
3844
- mode: backend.kind === "box-task" ? "task" : "prompt",
3845
- ...backend.kind !== "executor" && backend.options ? { options: backend.options } : {},
3846
- preserveToolParts: opts.preserveToolParts === true,
3847
- ...opts.onRawEvent ? { onRawEvent: opts.onRawEvent } : {}
3848
- }
3849
- );
3850
- for await (const event of inner) {
3851
- yield event;
3852
- throwIfAborted(deadline.signal);
3853
- }
3854
- yield buildFinalEvent(task, session, acc, { status: "completed", reason: "turn completed" });
3855
- } catch (err) {
3856
- const callerAborted = opts.signal?.aborted === true;
3857
- const status = callerAborted ? "aborted" : "failed";
3858
- const message = err instanceof Error ? err.message : String(err);
3859
- const error = err instanceof BackendTransportError ? { kind: "transport", message, status: err.status, body: err.body } : { kind: "backend", message };
3860
- yield {
3861
- type: "backend_error",
3862
- task,
3863
- ...session ? { session } : {},
3864
- backend: label,
3865
- message,
3866
- recoverable: !callerAborted,
3867
- error,
3868
- timestamp: nowIso()
3869
- };
3870
- yield buildFinalEvent(task, session, acc, { status, reason: message, error });
3871
- } finally {
3872
- deadline.dispose();
3873
- }
3874
- }
3875
- async function collectAgentTurn(stream) {
3876
- const events = [];
3877
- for await (const event of stream) events.push(event);
3878
- const final = events.at(-1);
3879
- if (!final || final.type !== "final") {
3880
- throw new Error(
3881
- `collectAgentTurn: stream ended without a terminal 'final' event (last: ${final ? final.type : "none"})`
3882
- );
3883
- }
3884
- const metadata = final.metadata ?? {};
3885
- const tokenUsage = metadata.tokenUsage && typeof metadata.tokenUsage === "object" ? metadata.tokenUsage : {};
3886
- const usage = {
3887
- input: finiteNumber(tokenUsage.input) ?? 0,
3888
- output: finiteNumber(tokenUsage.output) ?? 0
3889
- };
3890
- const costUsd = finiteNumber(metadata.costUsd);
3891
- if (costUsd !== void 0) usage.costUsd = costUsd;
3892
- if (typeof metadata.model === "string" && metadata.model.length > 0) {
3893
- usage.model = metadata.model;
3894
- }
3895
- return {
3896
- finalText: final.text ?? "",
3897
- usage,
3898
- events,
3899
- status: final.status,
3900
- ...final.error ? { error: final.error } : {}
3901
- };
3902
- }
3903
- async function startTurnSession(backend, task, prompt, signal, label) {
3904
- if (backend.kind === "chat" && backend.backend.start) {
3905
- return backend.backend.start(
3906
- { task, message: prompt },
3907
- { task, knowledge: emptyReadiness(task), signal }
3908
- );
3909
- }
3910
- return newRuntimeSession(label);
3911
- }
3912
- async function* driveBoxTurn(box, prompt, signal, agentRunName, acc, cfg) {
3913
- const callOptions = { ...cfg.options ?? {}, signal };
3914
- const stream = cfg.mode === "task" ? box.streamTask(prompt, callOptions) : box.streamPrompt(prompt, callOptions);
3915
- const toolParts = cfg.preserveToolParts ? createSandboxToolPartState() : void 0;
3916
- for await (const event of stream) {
3917
- if (cfg.onRawEvent) await cfg.onRawEvent(event);
3918
- const terminalText = terminalTextFromSandboxEvent(event);
3919
- if (terminalText !== void 0) acc.terminalText = terminalText;
3920
- if (toolParts) {
3921
- for (const toolEvent of mapSandboxToolEvent(event, toolParts)) yield toolEvent;
3922
- }
3923
- const mapped = mapSandboxEvent(event, { agentRunName });
3924
- if (!mapped) continue;
3925
- foldEvent(mapped, acc, agentRunName);
3926
- yield mapped;
3927
- }
3928
- }
3929
- async function* driveChatTurn(backend, task, session, prompt, signal, acc) {
3930
- const input = { task, message: prompt };
3931
- const context = { task, knowledge: emptyReadiness(task), session, signal };
3932
- for await (const raw of backend.stream(input, context)) {
3933
- const event = normalizeBackendStreamEvent(raw, task, session);
3934
- foldEvent(event, acc);
3935
- yield event;
3936
- }
3937
- }
3938
- function foldEvent(event, acc, fallbackModelLabel) {
3939
- if (event.type === "text_delta") {
3940
- acc.deltaText += event.text;
3941
- return;
3942
- }
3943
- if (event.type === "llm_call") {
3944
- acc.input += event.tokensIn ?? 0;
3945
- acc.output += event.tokensOut ?? 0;
3946
- acc.costUsd += event.costUsd ?? 0;
3947
- if (event.model && event.model !== fallbackModelLabel) acc.model = event.model;
3948
- }
3949
- }
3950
- function terminalTextFromSandboxEvent(event) {
3951
- if (!event || typeof event !== "object") return void 0;
3952
- const type = String(event.type ?? "");
3953
- if (type !== "result" && type !== "done" && type !== "final") return void 0;
3954
- const data = event.data && typeof event.data === "object" ? event.data : {};
3955
- for (const key of ["finalText", "text", "response", "content"]) {
3956
- const value = data[key];
3957
- if (typeof value === "string") return value;
3958
- }
3959
- return void 0;
3960
- }
3961
- function buildFinalEvent(task, session, acc, outcome) {
3962
- const finalText = acc.terminalText ?? acc.deltaText;
3963
- return {
3964
- type: "final",
3965
- task,
3966
- ...session ? { session } : {},
3967
- status: outcome.status,
3968
- reason: outcome.reason,
3969
- ...finalText ? { text: finalText } : {},
3970
- metadata: {
3971
- tokenUsage: { input: acc.input, output: acc.output },
3972
- ...acc.costUsd > 0 ? { costUsd: acc.costUsd } : {},
3973
- ...acc.model ? { model: acc.model } : {}
3974
- },
3975
- ...outcome.error ? { error: outcome.error } : {},
3976
- timestamp: nowIso()
3977
- };
3978
- }
3979
- function emptyReadiness(task) {
3980
- return scoreKnowledgeReadiness({ taskId: task.id, requirements: [] });
3981
- }
3982
- function finiteNumber(value) {
3983
- return typeof value === "number" && Number.isFinite(value) ? value : void 0;
3984
- }
3985
- function throwIfAborted(signal) {
3986
- if (!signal.aborted) return;
3987
- throw signal.reason instanceof Error ? signal.reason : new Error(String(signal.reason));
3988
- }
3989
- function deriveTurnSignal(callerSignal, timeoutMs) {
3990
- const controller = new AbortController();
3991
- const timer = timeoutMs > 0 ? setTimeout(
3992
- () => controller.abort(new Error(`agent turn timed out after ${timeoutMs}ms`)),
3993
- timeoutMs
3994
- ) : void 0;
3995
- if (timer && typeof timer.unref === "function") {
3996
- ;
3997
- timer.unref();
3998
- }
3999
- const onCallerAbort = () => controller.abort(callerSignal?.reason ?? new Error("agent turn aborted"));
4000
- if (callerSignal) {
4001
- if (callerSignal.aborted) onCallerAbort();
4002
- else callerSignal.addEventListener("abort", onCallerAbort, { once: true });
4003
- }
4004
- return {
4005
- signal: controller.signal,
4006
- dispose: () => {
4007
- if (timer) clearTimeout(timer);
4008
- callerSignal?.removeEventListener("abort", onCallerAbort);
4009
- }
4010
- };
4011
- }
4012
-
4013
- // src/runtime/supervise/patch-checks.ts
4014
- var DEFAULT_MAX_DIFF_LINES = 400;
4015
- var secretPathPattern = /(^|\/)(\.env(\.|$)|.*\.(pem|key|p12|pfx|keystore|wallet)|id_rsa|id_ed25519|secrets?\.json|credentials?\.json)$/i;
4016
- function touchedPathsFromPatch(patch) {
4017
- const out = /* @__PURE__ */ new Set();
4018
- for (const line of patch.split(/\r?\n/)) {
4019
- if (line.startsWith("+++ ") || line.startsWith("--- ")) {
4020
- const rest = line.slice(4).trim();
4021
- if (rest === "/dev/null") continue;
4022
- const stripped = rest.startsWith("a/") || rest.startsWith("b/") ? rest.slice(2) : rest;
4023
- out.add(stripped);
4024
- }
4025
- }
4026
- return [...out];
4027
- }
4028
- function countDiffLines(patch) {
4029
- let count = 0;
4030
- for (const line of patch.split(/\r?\n/)) {
4031
- if ((line.startsWith("+") || line.startsWith("-")) && !line.startsWith("+++") && !line.startsWith("---")) {
4032
- count += 1;
4033
- }
4034
- }
4035
- return count;
4036
- }
4037
- function runCoderChecks(input, constraints = {}) {
4038
- const maxDiff = constraints.maxDiffLines ?? DEFAULT_MAX_DIFF_LINES;
4039
- const forbidden = constraints.forbiddenPaths ?? [];
4040
- const scores = {};
4041
- const notes = [];
4042
- let pass = true;
4043
- const touched = touchedPathsFromPatch(input.patch);
4044
- if (touched.length === 0 || input.patch.trim().length === 0) {
4045
- pass = false;
4046
- scores.nonEmpty = 0;
4047
- notes.push("empty patch \u2014 no files changed");
4048
- } else {
4049
- scores.nonEmpty = 1;
4050
- }
4051
- const touchedSecrets = touched.filter((p) => secretPathPattern.test(p));
4052
- if (touchedSecrets.length > 0) {
4053
- pass = false;
4054
- scores.noSecrets = 0;
4055
- notes.push(`touched secret-shaped paths: ${touchedSecrets.join(", ")}`);
4056
- } else {
4057
- scores.noSecrets = 1;
4058
- }
4059
- const touchedForbidden = forbidden.filter((path) => {
4060
- const prefix = path.endsWith("/") ? path : `${path}/`;
4061
- const exact = prefix.slice(0, -1);
4062
- return touched.some((p) => p === exact || p.startsWith(prefix));
4063
- });
4064
- if (touchedForbidden.length > 0) {
4065
- pass = false;
4066
- scores.forbiddenPath = 0;
4067
- notes.push(`touched forbidden paths: ${touchedForbidden.join(", ")}`);
4068
- } else {
4069
- scores.forbiddenPath = 1;
4070
- }
4071
- const diffLines = countDiffLines(input.patch);
4072
- if (diffLines > maxDiff) {
4073
- pass = false;
4074
- scores.diffSize = 0;
4075
- notes.push(`diff ${diffLines} lines exceeds cap ${maxDiff}`);
4076
- } else {
4077
- scores.diffSize = maxDiff === 0 ? 0 : Math.max(0, 1 - diffLines / maxDiff);
4078
- }
4079
- scores.tests = input.testsPassed ? 1 : 0;
4080
- scores.typecheck = input.typecheckPassed ? 1 : 0;
4081
- if (!input.testsPassed) {
4082
- pass = false;
4083
- notes.push("tests failed");
4084
- }
4085
- if (!input.typecheckPassed) {
4086
- pass = false;
4087
- notes.push("typecheck failed");
4088
- }
4089
- const score = 0.5 * scores.tests + 0.3 * scores.typecheck + 0.2 * scores.diffSize;
4090
- const verdict = {
4091
- valid: pass,
4092
- score: Number.isFinite(score) ? score : 0,
4093
- scores
4094
- };
4095
- if (notes.length > 0) verdict.notes = notes.join("; ");
4096
- return verdict;
4097
- }
4098
-
4099
- // src/runtime/supervise/patch-deliverable.ts
4100
- function patchDelivered(options = {}) {
4101
- const require2 = new Set(options.require ?? []);
4102
- const constraints = {
4103
- ...options.maxDiffLines !== void 0 ? { maxDiffLines: options.maxDiffLines } : {},
4104
- ...options.forbiddenPaths !== void 0 ? { forbiddenPaths: options.forbiddenPaths } : {}
4105
- };
4106
- return {
4107
- describe: "patch: no-op/secret/forbidden/diff-size + required test/typecheck pass",
4108
- check(artifact) {
4109
- const input = {
4110
- patch: artifact.patch,
4111
- testsPassed: signalPass(artifact.checks?.tests?.passed, require2.has("tests")),
4112
- typecheckPassed: signalPass(artifact.checks?.typecheck?.passed, require2.has("typecheck"))
4113
- };
4114
- return runCoderChecks(input, constraints).valid === true;
4115
- }
4116
- };
4117
- }
4118
- function signalPass(value, required) {
4119
- if (value !== void 0) return value;
4120
- return !required;
4121
- }
4122
-
4123
- // src/runtime/supervise/trajectory-recorder.ts
4124
- import { buildTrajectory, InMemoryTraceStore } from "@tangle-network/agent-eval";
4125
- import { stuckLoopView, toolWasteView } from "@tangle-network/agent-eval/pipelines";
4126
- async function analyzeTrace(source, runId = "worker") {
4127
- const spans = await source.collect();
4128
- const store = new InMemoryTraceStore();
4129
- const laneSpanId = `${runId}-session-lane`;
4130
- if (spans.length > 0) {
4131
- const startedAt = Math.min(...spans.map((span) => span.startedAt));
4132
- const endedAt = Math.max(...spans.map((span) => span.endedAt ?? span.startedAt));
4133
- const agentSpanId = `${runId}-agent`;
4134
- await store.appendSpan({
4135
- spanId: agentSpanId,
4136
- runId,
4137
- kind: "agent",
4138
- name: "trace-source-agent",
4139
- startedAt,
4140
- endedAt
4141
- });
4142
- await store.appendSpan({
4143
- spanId: laneSpanId,
4144
- parentSpanId: agentSpanId,
4145
- runId,
4146
- kind: "custom",
4147
- name: "trace-source-session",
4148
- startedAt,
4149
- endedAt
4150
- });
4151
- }
4152
- for (let i = 0; i < spans.length; i += 1) {
4153
- const s = spans[i];
4154
- if (s) {
4155
- await store.appendSpan({
4156
- ...s,
4157
- runId,
4158
- spanId: `${runId}-t${i}`,
4159
- parentSpanId: laneSpanId
4160
- });
4161
- }
4162
- }
4163
- const [trajectory, stuckLoop, toolWaste] = await Promise.all([
4164
- buildTrajectory(store, runId),
4165
- stuckLoopView(store, { runId, maxInterveningToolCalls: 1 }),
4166
- toolWasteView(store, { runId })
4167
- ]);
4168
- return { trajectory, stuckLoop, toolWaste };
4169
- }
4170
-
4171
- // src/runtime/supervise/worktree-fanout.ts
4172
- function worktreeFanout(options) {
4173
- const deliverable = options.deliverable ?? patchDelivered({
4174
- ...options.maxDiffLines !== void 0 ? { maxDiffLines: options.maxDiffLines } : {},
4175
- ...options.forbiddenPaths !== void 0 ? { forbiddenPaths: options.forbiddenPaths } : {},
4176
- ...options.require !== void 0 ? { require: options.require } : {}
4177
- });
4178
- const itemSpec = (item) => {
4179
- const executor = gateOnDeliverable(
4180
- createWorktreeCliExecutor({
4181
- repoRoot: options.repoRoot,
4182
- profile: item.profile,
4183
- harness: item.harness,
4184
- taskPrompt: options.taskPrompt,
4185
- ...item.runId ? { runId: item.runId } : {},
4186
- ...item.baseRef ? { baseRef: item.baseRef } : {},
4187
- ...options.testCmd !== void 0 ? { testCmd: options.testCmd } : {},
4188
- ...options.typecheckCmd !== void 0 ? { typecheckCmd: options.typecheckCmd } : {},
4189
- ...options.harnessTimeoutMs !== void 0 ? { harnessTimeoutMs: options.harnessTimeoutMs } : {},
4190
- ...options.runGit ? { runGit: options.runGit } : {},
4191
- ...options.runHarness ? { runHarness: options.runHarness } : {},
4192
- ...options.runCommand ? { runCommand: options.runCommand } : {}
4193
- }),
4194
- deliverable
4195
- );
4196
- return { profile: item.profile, harness: null, executor };
4197
- };
4198
- const selectWinner = selectValidWinner({
4199
- strategy: options.winnerStrategy ?? "highest-score",
4200
- sizeOf: (a) => a.stats.insertions + a.stats.deletions
4201
- });
4202
- return fanout(options.harnesses, {
4203
- itemTask: () => options.taskPrompt,
4204
- label: (item, i) => `${item.name}:${i}`,
4205
- itemSpec: (item) => itemSpec(item),
4206
- selectWinner
4207
- });
4208
- }
4209
-
4210
- // src/runtime/supervise-surface.ts
4211
- function captureFailures(base) {
4212
- let lastReport = "";
4213
- const surface = {
4214
- name: base.name,
4215
- open: (t) => base.open(t),
4216
- tools: (t, h) => base.tools(t, h),
4217
- async call(h, name, args) {
4218
- const out = await base.call(h, name, args);
4219
- if (name === "run_tests") lastReport = out;
4220
- return out;
4221
- },
4222
- score: (t, h) => base.score(t, h),
4223
- close: (h) => base.close(h)
4224
- };
4225
- const failing = () => {
4226
- const body = /FAILING:\s*(.+)/i.exec(lastReport)?.[1];
4227
- return body ? body.split(",").map((s) => s.trim()).filter(Boolean) : [];
4228
- };
4229
- return { surface, failing };
4230
- }
4231
- function failuresAnalyst() {
4232
- return {
4233
- kinds: [
4234
- {
4235
- id: "failures",
4236
- description: "Surface the worker's still-failing tests so the driver targets them next.",
4237
- area: "progress"
4238
- }
4239
- ],
4240
- run: async (_kindId, trace) => {
4241
- const w = trace ?? {};
4242
- if (!(typeof w === "object" && w !== null && "resolved" in w))
4243
- return { summary: `worker produced: ${JSON.stringify(trace).slice(0, 300)}` };
4244
- if (w.resolved) return { summary: "worker RESOLVED \u2014 every check passed; stop." };
4245
- const failing = w.failing ?? [];
4246
- const head = `worker did NOT resolve \u2014 score ${(100 * (w.score ?? 0)).toFixed(0)}%, ${w.shots ?? "?"} shot(s)`;
4247
- return {
4248
- summary: failing.length ? `${head}. STILL FAILING (${failing.length}): ${failing.slice(0, 12).join(", ")}. Spawn the next worker to fix exactly these; if a test keeps failing across workers, give it concrete guidance about that case.` : `${head}. (no failing-test list available this round)`
4249
- };
4250
- }
4251
- };
4252
- }
4253
- function surfaceWorkerExecutor(surface, task, worker, strategy) {
4254
- let artifact;
4255
- return {
4256
- runtime: "surface-worker",
4257
- async execute(brief) {
4258
- const guidance = typeof brief === "string" ? brief.trim() : brief ? JSON.stringify(brief) : "";
4259
- const attemptTask = guidance ? {
4260
- ...task,
4261
- systemPrompt: `${task.systemPrompt ?? ""}
4262
-
4263
- \u2014 Supervisor guidance for THIS attempt (incorporate it; do not just repeat a prior approach) \u2014
4264
- ${guidance}`
4265
- } : task;
4266
- const cap = captureFailures(surface);
4267
- const r = await runAgentic({
4268
- surface: cap.surface,
4269
- task: attemptTask,
4270
- strategy,
4271
- budget: worker.budget ?? 1,
4272
- routerBaseUrl: worker.routerBaseUrl,
4273
- routerKey: worker.routerKey,
4274
- model: worker.model,
4275
- ...worker.maxTokens !== void 0 ? { maxTokens: worker.maxTokens } : {},
4276
- ...worker.innerTurns !== void 0 ? { innerTurns: worker.innerTurns } : {}
4277
- });
4278
- const out = {
4279
- resolved: r.resolved,
4280
- score: r.score,
4281
- shots: r.shots,
4282
- summary: `${strategy.name} ${r.shots} shot(s) \u2192 ${(100 * r.score).toFixed(0)}% (${r.resolved ? "resolved" : "unresolved"})`,
4283
- failing: r.resolved ? [] : cap.failing()
4284
- };
4285
- const spent = { iterations: r.completions, tokens: r.tokens, usd: r.usd, ms: r.ms };
4286
- artifact = {
4287
- outRef: `surface-worker:${task.id}:${r.shots}:${r.resolved ? "ok" : "no"}`,
4288
- out,
4289
- verdict: { valid: r.resolved, score: r.score },
4290
- spent
4291
- };
4292
- return artifact;
4293
- },
4294
- teardown: () => Promise.resolve({ destroyed: true }),
4295
- resultArtifact() {
4296
- if (!artifact) throw new Error("surfaceWorkerExecutor: resultArtifact before execute");
4297
- return artifact;
4298
- }
4299
- };
4300
- }
4301
- async function superviseSurface(profile, task, opts) {
4302
- const strategy = opts.strategy ?? refine;
4303
- const innerTurns = opts.worker.innerTurns ?? 6;
4304
- const router = opts.router ?? {
4305
- routerBaseUrl: opts.worker.routerBaseUrl,
4306
- routerKey: opts.worker.routerKey,
4307
- model: opts.worker.model
4308
- };
4309
- const budget = opts.budget ?? {
4310
- maxIterations: (innerTurns + 2) * 5 + 16,
4311
- maxTokens: (opts.worker.maxTokens ?? 4e3) * 8
4312
- };
4313
- const makeWorkerAgent = (rawProfile) => {
4314
- const p = rawProfile ?? {};
4315
- const name = typeof p.name === "string" && p.name.length > 0 ? p.name : "surface-worker";
4316
- const spec = {
4317
- profile: rawProfile,
4318
- harness: null,
4319
- executor: surfaceWorkerExecutor(
4320
- opts.surface,
4321
- task,
4322
- opts.worker,
4323
- strategy
4324
- )
4325
- };
4326
- return { name, act: async () => "", executorSpec: spec };
4327
- };
4328
- const deliverable = {
4329
- describe: `resolve the surface task ${task.id} (every required check passes)`,
4330
- check: (out2) => out2?.resolved === true
4331
- };
4332
- const analysts = opts.analysts === null ? void 0 : opts.analysts ?? failuresAnalyst();
4333
- const result = await supervise(profile, task, {
4334
- makeWorkerAgent,
4335
- deliverable,
4336
- budget,
4337
- maxLiveWorkers: opts.maxLiveWorkers ?? 1,
4338
- // A SMALL per-worker reservation so MULTIPLE workers fit the pool (the default reserves the whole pool
4339
- // per worker → only one ever spawns, defeating the spawn-a-targeted-worker steering).
4340
- perWorker: { maxIterations: innerTurns + 2, maxTokens: opts.worker.maxTokens ?? 4e3 },
4341
- router,
4342
- ...analysts ? { analysts, analyzeOnSettle: analysts.kinds.map((k) => k.id) } : {}
4343
- });
4344
- const out = result.kind === "winner" ? result.out : void 0;
4345
- const sp = result.spentTotal;
4346
- return {
4347
- resolved: out?.resolved ?? false,
4348
- score: out?.score ?? 0,
4349
- usd: sp.usd,
4350
- tokensIn: sp.tokens.input,
4351
- tokensOut: sp.tokens.output,
4352
- ms: sp.ms,
4353
- completions: sp.iterations
4354
- };
4355
- }
4356
-
4357
- // src/runtime/verifier-environment.ts
4358
- var submitTool = {
4359
- type: "function",
4360
- function: {
4361
- name: "submit_answer",
4362
- description: "Submit your answer for evaluation. You may submit more than once \u2014 the best-scoring submission counts. Submit the COMPLETE final answer, not a fragment.",
4363
- parameters: {
4364
- type: "object",
4365
- properties: { answer: { type: "string", description: "The complete final answer." } },
4366
- required: ["answer"]
4367
- }
4368
- }
4369
- };
4370
- function createVerifierEnvironment(opts) {
4371
- if (opts.extraTools?.length && !opts.callExtra) {
4372
- throw new Error(`${opts.name}: extraTools requires callExtra`);
4373
- }
4374
- const states = /* @__PURE__ */ new Map();
4375
- let seq = 0;
4376
- return {
4377
- name: opts.name,
4378
- async open(task) {
4379
- seq += 1;
4380
- const handle = { id: `${opts.name}-${seq}`, surface: opts.name };
4381
- states.set(handle.id, { task, submissions: [] });
4382
- return handle;
4383
- },
4384
- async tools() {
4385
- return [submitTool, ...opts.extraTools ?? []];
4386
- },
4387
- async call(handle, name, args) {
4388
- const state = states.get(handle.id);
4389
- if (!state) return "ERROR: workspace closed";
4390
- if (name === "submit_answer") {
4391
- const answer = String(args.answer ?? "").trim();
4392
- if (!answer) return "ERROR: empty answer";
4393
- state.submissions.push(answer);
4394
- return `submission ${state.submissions.length} recorded`;
4395
- }
4396
- if (opts.callExtra && opts.extraTools?.some((t) => t.function.name === name)) {
4397
- try {
4398
- return await opts.callExtra(state.task, name, args);
4399
- } catch (e) {
4400
- return `ERROR: ${e instanceof Error ? e.message : String(e)}`;
4401
- }
4402
- }
4403
- return `ERROR: unknown tool ${name}`;
4404
- },
4405
- // Keep-best across submissions — the measured law (workers reach correct answers,
4406
- // then revise past them; final-state scoring undersells every strategy).
4407
- async score(task, handle) {
4408
- const state = states.get(handle.id);
4409
- if (!state || state.submissions.length === 0) return { passes: 0, total: 1, errored: 0 };
4410
- let best = { passes: 0, total: 1, errored: 0 };
4411
- const ratio = (s) => s.total > 0 ? s.passes / s.total : 0;
4412
- for (const answer of state.submissions) {
4413
- const s = await opts.check(task, answer);
4414
- if (ratio(s) > ratio(best)) best = s;
4415
- }
4416
- return best;
4417
- },
4418
- async close(handle) {
4419
- states.delete(handle.id);
4420
- }
4421
- };
4422
- }
4423
-
4424
- // src/runtime/waterfall.ts
4425
- function createWaterfallCollector() {
4426
- let spans = /* @__PURE__ */ new Map();
4427
- const onEvent = (event) => {
4428
- if (event.target === "agent.spawn") {
4429
- const p = event.payload ?? {};
4430
- const id = p.childId ?? event.id;
4431
- spans.set(id, {
4432
- id,
4433
- label: p.label ?? id,
4434
- runId: event.runId,
4435
- ...event.parentId !== void 0 ? { parentId: event.parentId } : {},
4436
- startMs: event.timestamp,
4437
- status: "running",
4438
- usd: 0,
4439
- tokens: { input: 0, output: 0 }
4440
- });
4441
- return;
4442
- }
4443
- if (event.target === "agent.child") {
4444
- const p = event.payload ?? {};
4445
- const id = p.childId;
4446
- if (!id) return;
4447
- const span = spans.get(id);
4448
- if (!span) return;
4449
- span.endMs = event.timestamp;
4450
- span.status = p.status === "down" ? "down" : "done";
4451
- span.usd = p.spent?.usd ?? 0;
4452
- span.tokens = {
4453
- input: p.spent?.tokens?.input ?? 0,
4454
- output: p.spent?.tokens?.output ?? 0
4455
- };
4456
- if (typeof p.score === "number") span.score = p.score;
4457
- }
4458
- };
4459
- const report = () => {
4460
- const all = [...spans.values()].sort((a, b) => a.startMs - b.startMs);
4461
- const start = all[0]?.startMs ?? 0;
4462
- const end = Math.max(start, ...all.map((s) => s.endMs ?? s.startMs));
4463
- const byKind = {};
4464
- let totalUsd = 0;
4465
- const totalTokens = { input: 0, output: 0 };
4466
- for (const s of all) {
4467
- totalUsd += s.usd;
4468
- totalTokens.input += s.tokens.input;
4469
- totalTokens.output += s.tokens.output;
4470
- const kind = s.label.includes(":") ? s.label.split(":")[0] : s.label;
4471
- let k = byKind[kind];
4472
- if (!k) {
4473
- k = { count: 0, ms: 0, usd: 0, tokens: { input: 0, output: 0 } };
4474
- byKind[kind] = k;
4475
- }
4476
- k.count += 1;
4477
- k.ms += (s.endMs ?? s.startMs) - s.startMs;
4478
- k.usd += s.usd;
4479
- k.tokens.input += s.tokens.input;
4480
- k.tokens.output += s.tokens.output;
4481
- }
4482
- return { spans: all, totalMs: end - start, totalUsd, totalTokens, byKind };
4483
- };
4484
- const render = (opts) => {
4485
- const { spans: all, totalMs, totalUsd, byKind } = report();
4486
- if (all.length === 0) return "(no spans observed)";
4487
- const width = opts?.width ?? 48;
4488
- const maxRows = opts?.maxRows ?? 60;
4489
- const start = all[0]?.startMs ?? 0;
4490
- const scale = totalMs > 0 ? width / totalMs : 0;
4491
- const lines = [];
4492
- const labelWidth = Math.min(24, Math.max(...all.map((s) => s.label.length)) + 1);
4493
- for (const s of all.slice(0, maxRows)) {
4494
- const offset = Math.round((s.startMs - start) * scale);
4495
- const dur = (s.endMs ?? s.startMs) - s.startMs;
4496
- const len = Math.max(1, Math.round(dur * scale));
4497
- const bar = `${" ".repeat(Math.min(offset, width))}${(s.status === "down" ? "\u2591" : "\u2588").repeat(Math.max(1, Math.min(len, width - Math.min(offset, width) + 1)))}`;
4498
- const mark = s.status === "down" ? " DOWN" : s.score !== void 0 ? ` ${(s.score * 100).toFixed(0)}%` : "";
4499
- lines.push(
4500
- `${s.label.padEnd(labelWidth)}|${bar.padEnd(width + 1)}| ${(dur / 1e3).toFixed(1)}s $${s.usd.toFixed(4)} ${s.tokens.input}/${s.tokens.output}tok${mark}`
4501
- );
4502
- }
4503
- if (all.length > maxRows) lines.push(`\u2026 ${all.length - maxRows} more spans`);
4504
- lines.push("\u2014".repeat(labelWidth + width + 2));
4505
- for (const [kind, k] of Object.entries(byKind)) {
4506
- lines.push(
4507
- `${kind.padEnd(labelWidth)} \xD7${k.count} ${(k.ms / 1e3).toFixed(1)}s busy $${k.usd.toFixed(4)} ${k.tokens.input}/${k.tokens.output}tok`
4508
- );
4509
- }
4510
- lines.push(
4511
- `TOTAL${" ".repeat(labelWidth - 5)} ${(totalMs / 1e3).toFixed(1)}s wall $${totalUsd.toFixed(4)}`
4512
- );
4513
- return lines.join("\n");
4514
- };
4515
- return {
4516
- hooks: { onEvent },
4517
- report,
4518
- render,
4519
- reset: () => {
4520
- spans = /* @__PURE__ */ new Map();
4521
- }
4522
- };
4523
- }
4524
-
4525
- // src/runtime/workspace.ts
4526
- function localShell() {
4527
- return async (args, cwd) => {
4528
- const { execFile } = await import("child_process");
4529
- const [bin, ...rest] = args;
4530
- return new Promise((resolve) => {
4531
- execFile(
4532
- bin ?? "",
4533
- rest,
4534
- { cwd, encoding: "utf-8", maxBuffer: 64 * 1024 * 1024 },
4535
- (err, stdout, stderr) => {
4536
- resolve({
4537
- stdout: stdout ?? "",
4538
- stderr: stderr ?? "",
4539
- code: err ? err.code ?? 1 : 0
4540
- });
4541
- }
4542
- );
4543
- });
4544
- };
4545
- }
4546
- function gitWorkspace(opts) {
4547
- const shell = opts.shell ?? localShell();
4548
- const branch = opts.branch ?? "main";
4549
- const cfg = opts.noHooks === false ? [] : ["-c", "core.hooksPath=/dev/null"];
4550
- const ident = ["-c", "user.email=workspace@tangle.local", "-c", "user.name=workspace"];
4551
- const run = async (args, cwd) => {
4552
- const res = await shell(["git", ...cfg, ...ident, ...args], cwd);
4553
- if (res.code !== 0) {
4554
- throw new Error(
4555
- `git ${args.join(" ")} failed (${res.code}): ${tail(res.stderr || res.stdout)}`
4556
- );
4557
- }
4558
- return res.stdout;
4559
- };
4560
- return {
4561
- ref: opts.ref,
4562
- materialize: (dir) => run(["clone", "--branch", branch, opts.ref, dir]).then(() => {
4563
- }),
4564
- async commit(dir, message) {
4565
- await run(["add", "-A"], dir);
4566
- const status = await run(["status", "--porcelain"], dir);
4567
- if (!status.trim()) return { ok: true, rev: (await run(["rev-parse", "HEAD"], dir)).trim() };
4568
- await run(["commit", "-m", message], dir);
4569
- const pull = await shell(["git", ...cfg, ...ident, "pull", "--rebase", "origin", branch], dir);
4570
- if (pull.code !== 0) {
4571
- await shell(["git", ...cfg, "rebase", "--abort"], dir).catch(() => {
4572
- });
4573
- return { ok: false, conflict: tail(pull.stderr || pull.stdout) };
4574
- }
4575
- const push = await shell(["git", ...cfg, ...ident, "push", "origin", branch], dir);
4576
- if (push.code !== 0) return { ok: false, conflict: tail(push.stderr || push.stdout) };
4577
- return { ok: true, rev: (await run(["rev-parse", "HEAD"], dir)).trim() };
4578
- },
4579
- async head() {
4580
- const out = await run(["ls-remote", opts.ref, `refs/heads/${branch}`]);
4581
- return out.split(/\s+/)[0] ?? "";
4582
- }
4583
- };
4584
- }
4585
- function jjWorkspace(opts) {
4586
- const shell = opts.shell ?? localShell();
4587
- const branch = opts.branch ?? "main";
4588
- const ident = [
4589
- "--config-toml",
4590
- 'user.name="workspace"',
4591
- "--config-toml",
4592
- 'user.email="workspace@tangle.local"'
4593
- ];
4594
- const jj = async (args, cwd) => {
4595
- const res = await shell(["jj", ...ident, ...args], cwd);
4596
- if (res.code !== 0) {
4597
- throw new Error(
4598
- `jj ${args.join(" ")} failed (${res.code}): ${tail(res.stderr || res.stdout)}`
4599
- );
4600
- }
4601
- return res.stdout;
4602
- };
4603
- return {
4604
- ref: opts.ref,
4605
- // Colocated clone: jj manages history, git holds the durable remote.
4606
- materialize: (dir) => jj(["git", "clone", "--colocate", opts.ref, dir]).then(() => {
4607
- }),
4608
- async commit(dir, message) {
4609
- await jj(["describe", "-m", message], dir);
4610
- await jj(["new"], dir);
4611
- const push = await shell(["jj", ...ident, "git", "push", "--branch", branch], dir);
4612
- if (push.code !== 0) return { ok: false, conflict: tail(push.stderr || push.stdout) };
4613
- const rev = (await jj(["log", "--no-graph", "-r", "@-", "-T", "commit_id"], dir)).trim();
4614
- return { ok: true, rev };
4615
- },
4616
- async head() {
4617
- const out = await shell(["git", "ls-remote", opts.ref, `refs/heads/${branch}`]);
4618
- return out.stdout.split(/\s+/)[0] ?? "";
4619
- }
4620
- };
4621
- }
4622
- async function runInWorkspace(ws, body, opts = {}) {
4623
- const { mkdtempSync: mkdtempSync2, rmSync: rmSync2 } = await import("fs");
4624
- const { tmpdir: tmpdir3 } = await import("os");
4625
- const { join: join4 } = await import("path");
4626
- const dir = mkdtempSync2(join4(tmpdir3(), opts.tmpPrefix ?? "ws-run-"));
4627
- try {
4628
- await ws.materialize(dir);
4629
- const r = await body(dir);
4630
- if (r.valid || opts.commitOnInvalid) {
4631
- const message = r.message ?? (r.valid ? "worker: delivered" : "worker: wip");
4632
- const commit = await ws.commit(dir, message);
4633
- return { valid: r.valid, value: r.value, commit };
4634
- }
4635
- return { valid: r.valid, value: r.value };
4636
- } finally {
4637
- rmSync2(dir, { recursive: true, force: true });
4638
- }
4639
- }
4640
- function tail(s) {
4641
- return s.slice(-400);
4642
- }
4643
-
4644
- export {
4645
- envKeyProvider,
4646
- mcpSecretEnvMetadataKey,
4647
- secretEnvOfMcpServer,
4648
- resolveSecretEnv,
4649
- sanitizeMcpToolSchema,
4650
- createMcpEnvironment,
4651
- McpSpawnFault,
4652
- connectStdioMcp,
4653
- materializeLocalMcp,
4654
- defaultAuditorInstruction,
4655
- auditIntent,
4656
- leaderboard,
4657
- pairwiseSignificance,
4658
- renderLeaderboardMarkdown,
4659
- renderPairwiseMarkdown,
4660
- renderLeaderboardSvg,
4661
- renderLeaderboardHtml,
4662
- completionAuthorizes,
4663
- stopSentinel,
4664
- sentinelCompletion,
4665
- deterministicCompletion,
4666
- loopCampaignDispatch,
4667
- loopDispatch,
4668
- inlineSandboxClient,
4669
- localSandboxClient,
4670
- resolveSandboxClient,
4671
- naiveDriver,
4672
- dumbDriver,
4673
- defineLeaderboard,
4674
- harvestCorpus,
4675
- inProcessSandboxClient,
4676
- assertTraceDerivedFindings,
4677
- createScopeAnalyst,
4678
- registryScopeAnalyst,
4679
- buildSteerContext,
4680
- selectValidWinner,
4681
- pipeline,
4682
- fanout,
4683
- loopUntil,
4684
- panel,
4685
- verify,
4686
- widen,
4687
- flatWidenGate,
4688
- InMemoryCorpus,
4689
- FileCorpus,
4690
- renderCorpusToInstructions,
4691
- createShapeRegistry,
4692
- builtinShapes,
4693
- registerShape,
4694
- definePersona,
4695
- runPersonified,
4696
- trajectoryReport,
4697
- equalKOnCost,
4698
- promotionGate,
4699
- reportLoopUsage,
4700
- runBenchmark,
4701
- printBenchmarkReport,
4702
- SandboxRunAbortError,
4703
- openSandboxRun,
4704
- strategyAuthorContract,
4705
- assertStrategyContract,
4706
- authorStrategy,
4707
- discriminatingMeans,
4708
- pickChampion,
4709
- selectChampion,
4710
- runStrategyEvolution,
4711
- streamAgentTurn,
4712
- collectAgentTurn,
4713
- runCoderChecks,
4714
- patchDelivered,
4715
- analyzeTrace,
4716
- worktreeFanout,
4717
- failuresAnalyst,
4718
- superviseSurface,
4719
- createVerifierEnvironment,
4720
- createWaterfallCollector,
4721
- localShell,
4722
- gitWorkspace,
4723
- jjWorkspace,
4724
- runInWorkspace,
4725
- computeFindingId,
4726
- makeFinding
4727
- };
4728
- //# sourceMappingURL=chunk-WRTOVNN4.js.map