@tangle-network/agent-runtime 0.93.2 → 0.94.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +4 -3
  2. package/dist/agent.d.ts +1 -1
  3. package/dist/agent.js +5 -6
  4. package/dist/agent.js.map +1 -1
  5. package/dist/{agentic-generator-B8oeE2Yv.d.ts → agentic-generator-D9ZmAuiL.d.ts} +1 -1
  6. package/dist/analyst-loop.d.ts +1 -1
  7. package/dist/candidate-execution/index.d.ts +10 -3
  8. package/dist/candidate-execution/index.js +9 -3
  9. package/dist/{chunk-6K5CI33W.js → chunk-D336OZHK.js} +1232 -19
  10. package/dist/chunk-D336OZHK.js.map +1 -0
  11. package/dist/{chunk-TU73X3I6.js → chunk-HPCJBBLS.js} +2 -2
  12. package/dist/chunk-ISTDY47H.js +849 -0
  13. package/dist/chunk-ISTDY47H.js.map +1 -0
  14. package/dist/{chunk-RTVVH2KL.js → chunk-JEP2WM24.js} +2 -2
  15. package/dist/{chunk-Z7K5BPKZ.js → chunk-K7OPNHU4.js} +2 -2
  16. package/dist/{chunk-OX3QRJOA.js → chunk-KHLE3C5G.js} +281 -251
  17. package/dist/chunk-KHLE3C5G.js.map +1 -0
  18. package/dist/{chunk-XDSWWUKE.js → chunk-MZH4HORF.js} +6 -29
  19. package/dist/chunk-MZH4HORF.js.map +1 -0
  20. package/dist/chunk-PJNGPWTA.js +634 -0
  21. package/dist/chunk-PJNGPWTA.js.map +1 -0
  22. package/dist/{chunk-X5KJKS33.js → chunk-RTSZNYXM.js} +3 -3
  23. package/dist/{chunk-WGR6QJ5H.js → chunk-SJHFJOCY.js} +8 -161
  24. package/dist/chunk-SJHFJOCY.js.map +1 -0
  25. package/dist/{chunk-2BQMAZAB.js → chunk-URMJTSXB.js} +1378 -28
  26. package/dist/chunk-URMJTSXB.js.map +1 -0
  27. package/dist/{chunk-3D2RHC4K.js → chunk-WIPGQ4GT.js} +1 -1
  28. package/dist/{chunk-3D2RHC4K.js.map → chunk-WIPGQ4GT.js.map} +1 -1
  29. package/dist/{chunk-R5GWDTM3.js → chunk-ZEYAT33L.js} +2 -2
  30. package/dist/{completion-gate-BLaiN0-X.d.ts → completion-gate-BAj3dXeW.d.ts} +14 -2
  31. package/dist/{coordination-DxJ83oZA.d.ts → coordination-B3ZuApR_.d.ts} +7 -5
  32. package/dist/environment-provider.d.ts +2 -2
  33. package/dist/{improve-DDhQaaJT.d.ts → improve-r8SvIxYv.d.ts} +13 -16
  34. package/dist/index.d.ts +39 -110
  35. package/dist/index.js +92 -36
  36. package/dist/index.js.map +1 -1
  37. package/dist/intelligence.d.ts +24 -7
  38. package/dist/intelligence.js +114 -20
  39. package/dist/intelligence.js.map +1 -1
  40. package/dist/knowledge.d.ts +7 -7
  41. package/dist/knowledge.js +5 -5
  42. package/dist/lifecycle.d.ts +3 -3
  43. package/dist/lifecycle.js +2 -2
  44. package/dist/{local-harness-dcD5WTTr.d.ts → local-harness-B7bNuNJw.d.ts} +49 -1
  45. package/dist/{loop-runner-bin-kKUNGLyV.d.ts → loop-runner-bin-D4bMIxmB.d.ts} +2 -2
  46. package/dist/loop-runner-bin.d.ts +6 -6
  47. package/dist/loop-runner-bin.js +6 -7
  48. package/dist/loops.d.ts +457 -18
  49. package/dist/loops.js +39 -41
  50. package/dist/mcp/bin.js +5 -5
  51. package/dist/mcp/index.d.ts +10 -10
  52. package/dist/mcp/index.js +9 -8
  53. package/dist/{mcp-serve-verifier-Bg4C3p5S.d.ts → mcp-serve-verifier-BuXjaZDE.d.ts} +1 -1
  54. package/dist/{openai-tools-E3woykz9.d.ts → openai-tools-D0ZSRCC6.d.ts} +1 -1
  55. package/dist/{prepare-DiVGKcwS.d.ts → prepare-BKxAiUcH.d.ts} +1 -1
  56. package/dist/profiles.d.ts +1 -1
  57. package/dist/{sanitize-C9go6tXj.d.ts → sanitize-C2jicjNf.d.ts} +1 -1
  58. package/dist/{supervise-T2pazU3G.d.ts → supervise-DTNcLDl2.d.ts} +4 -4
  59. package/dist/{types-DAdIm4AC.d.ts → types-Bb-CqxJH.d.ts} +5 -2
  60. package/dist/{types-B00NtbCs.d.ts → types-DWA64rbJ.d.ts} +1 -1
  61. package/dist/{worktree-fanout-BUb2Ag02.d.ts → worktree-fanout-IMKqdV_E.d.ts} +244 -244
  62. package/package.json +4 -4
  63. package/skills/build-with-agent-runtime/SKILL.md +1 -1
  64. package/dist/chunk-2BQMAZAB.js.map +0 -1
  65. package/dist/chunk-6K5CI33W.js.map +0 -1
  66. package/dist/chunk-FDJ7AHXG.js +0 -1229
  67. package/dist/chunk-FDJ7AHXG.js.map +0 -1
  68. package/dist/chunk-J2K6WIG6.js +0 -2172
  69. package/dist/chunk-J2K6WIG6.js.map +0 -1
  70. package/dist/chunk-OX3QRJOA.js.map +0 -1
  71. package/dist/chunk-PX6SXX3M.js +0 -146
  72. package/dist/chunk-PX6SXX3M.js.map +0 -1
  73. package/dist/chunk-WGR6QJ5H.js.map +0 -1
  74. package/dist/chunk-XDSWWUKE.js.map +0 -1
  75. package/dist/structural-rollout-DHGDbhvR.d.ts +0 -446
  76. /package/dist/{chunk-TU73X3I6.js.map → chunk-HPCJBBLS.js.map} +0 -0
  77. /package/dist/{chunk-RTVVH2KL.js.map → chunk-JEP2WM24.js.map} +0 -0
  78. /package/dist/{chunk-Z7K5BPKZ.js.map → chunk-K7OPNHU4.js.map} +0 -0
  79. /package/dist/{chunk-X5KJKS33.js.map → chunk-RTSZNYXM.js.map} +0 -0
  80. /package/dist/{chunk-R5GWDTM3.js.map → chunk-ZEYAT33L.js.map} +0 -0
@@ -1,31 +1,23 @@
1
- import {
2
- observe,
3
- refine,
4
- runAgentic,
5
- sample,
6
- sampleThenRefine
7
- } from "./chunk-FDJ7AHXG.js";
8
1
  import {
9
2
  createExecutor,
10
3
  createExecutorRegistry,
11
4
  createSandboxLineage,
5
+ createSupervisor,
12
6
  createWorktreeCliExecutor,
13
7
  defaultSelectWinner,
14
8
  gateOnDeliverable,
9
+ notifyRuntimeHookEvent,
15
10
  probeSandboxCapabilities,
11
+ routerToolLoop,
16
12
  runLoop,
17
- supervise
18
- } from "./chunk-2BQMAZAB.js";
19
- import {
20
- createSupervisor,
21
- notifyRuntimeHookEvent,
22
13
  settledToIteration,
14
+ supervise,
23
15
  withDriverExecutor
24
- } from "./chunk-J2K6WIG6.js";
16
+ } from "./chunk-URMJTSXB.js";
25
17
  import {
26
18
  InMemoryResultBlobStore,
27
19
  InMemorySpawnJournal
28
- } from "./chunk-3D2RHC4K.js";
20
+ } from "./chunk-WIPGQ4GT.js";
29
21
  import {
30
22
  addTokenUsage,
31
23
  isAbortError,
@@ -47,7 +39,7 @@ import {
47
39
  } from "./chunk-YEJR7IXO.js";
48
40
 
49
41
  // src/runtime/index.ts
50
- import { computeFindingId, makeFinding } from "@tangle-network/agent-eval";
42
+ import { computeFindingId, makeFinding as makeFinding2 } from "@tangle-network/agent-eval";
51
43
 
52
44
  // src/runtime/anytime.ts
53
45
  var median = (xs) => {
@@ -1120,6 +1112,163 @@ function defineLeaderboard(spec) {
1120
1112
  return { run, toBenchmarkAdapter };
1121
1113
  }
1122
1114
 
1115
+ // src/runtime/observe.ts
1116
+ import { makeFinding } from "@tangle-network/agent-eval";
1117
+ var observerId = "observe/trace";
1118
+ var defaultAnalystInstruction = "You are a third-person OBSERVER watching an AI agent work. You see its TRACE (what it did), not its grader. From the trace, name SPECIFIC, behavior-grounded findings: wasted/duplicated tool calls, thrash/retries, token/cost waste, missing verification, failure patterns. For each, a concrete recommended_action, and whether the AGENT (fix its skills/prompt/tools) or the OPERATOR (fix framing/decomposition/config) should act. Only claim what the trace shows. No findings if the run was clean.";
1119
+ function summarizeTrace(trace, maxLines) {
1120
+ const lines = [];
1121
+ for (const ev of trace) {
1122
+ const e = ev;
1123
+ const t = (e.type ?? "").toLowerCase();
1124
+ const d = e.data ?? {};
1125
+ const part = d.part ?? {};
1126
+ if (part.type === "tool")
1127
+ lines.push(`tool:${part.tool}${part.state?.status ? `(${part.state.status})` : ""}`);
1128
+ else if (t.includes("error"))
1129
+ lines.push(`ERROR: ${String(d.message ?? d.detail ?? "").slice(0, 200)}`);
1130
+ else if (t === "status" && typeof d.status === "string") lines.push(`status:${d.status}`);
1131
+ else if (t.includes("tool")) lines.push(`tool-event:${t}`);
1132
+ }
1133
+ const out = [];
1134
+ for (const ln of lines) {
1135
+ const prev = out[out.length - 1];
1136
+ const m = prev?.match(/^(.*?)(?: x(\d+))?$/);
1137
+ if (m && m[1] === ln) out[out.length - 1] = `${ln} x${(Number(m[2]) || 1) + 1}`;
1138
+ else out.push(ln);
1139
+ }
1140
+ return out.slice(0, maxLines).join("\n") || "(no tool/error events in trace)";
1141
+ }
1142
+ var findingsSchema = {
1143
+ name: "observer_findings",
1144
+ schema: {
1145
+ type: "object",
1146
+ additionalProperties: false,
1147
+ properties: {
1148
+ findings: {
1149
+ type: "array",
1150
+ items: {
1151
+ type: "object",
1152
+ additionalProperties: false,
1153
+ properties: {
1154
+ area: {
1155
+ type: "string",
1156
+ description: "tool-use | cost | verification | process | failure | latency"
1157
+ },
1158
+ severity: { type: "string", enum: ["critical", "high", "medium", "low", "info"] },
1159
+ claim: {
1160
+ type: "string",
1161
+ description: "what you OBSERVED in the trace (a fact, with the evidence)"
1162
+ },
1163
+ recommended_action: {
1164
+ type: "string",
1165
+ description: "the concrete change for the agent or operator"
1166
+ },
1167
+ audience: {
1168
+ type: "string",
1169
+ enum: ["agent", "operator"],
1170
+ description: "who should act on this"
1171
+ },
1172
+ confidence: { type: "number" }
1173
+ },
1174
+ required: ["area", "severity", "claim", "recommended_action", "audience", "confidence"]
1175
+ }
1176
+ }
1177
+ },
1178
+ required: ["findings"]
1179
+ }
1180
+ };
1181
+ async function observe(input, opts) {
1182
+ const traceSummary = summarizeTrace(input.trace, opts.maxTraceLines ?? 80);
1183
+ const res = await opts.chat.chat(
1184
+ {
1185
+ ...opts.model ? { model: opts.model } : {},
1186
+ jsonSchema: findingsSchema,
1187
+ messages: [
1188
+ {
1189
+ role: "system",
1190
+ content: opts.analystInstruction ?? defaultAnalystInstruction
1191
+ },
1192
+ {
1193
+ role: "user",
1194
+ content: `TASK: ${input.task}
1195
+
1196
+ OUTCOME: ${input.outcome ?? "unknown"}
1197
+
1198
+ FINAL OUTPUT (truncated):
1199
+ ${input.output.slice(0, 1200)}
1200
+
1201
+ TRACE (in order; "xN" = repeated):
1202
+ ${traceSummary}`
1203
+ }
1204
+ ]
1205
+ },
1206
+ { ...opts.signal ? { signal: opts.signal } : {} }
1207
+ );
1208
+ const parsed = parseFindings(res.content);
1209
+ const producedAt = input.runId ? `${input.runId}` : observerId;
1210
+ const findings = parsed.map(
1211
+ (f) => makeFinding({
1212
+ analyst_id: observerId,
1213
+ area: `${f.area}`,
1214
+ severity: f.severity,
1215
+ claim: f.claim,
1216
+ recommended_action: f.recommended_action,
1217
+ confidence: typeof f.confidence === "number" ? f.confidence : 0.5,
1218
+ evidence_refs: [],
1219
+ // The observer reads BEHAVIOR, never the judge verdict — firewall provenance.
1220
+ derived_from_judge: false,
1221
+ metadata: { audience: f.audience },
1222
+ ...input.runId ? { subject: input.runId } : {}
1223
+ })
1224
+ );
1225
+ const learned = [];
1226
+ if (opts.corpus) {
1227
+ for (const f of findings) {
1228
+ const record = {
1229
+ schemaVersion: "1.0.0",
1230
+ id: f.finding_id,
1231
+ runId: input.runId ?? observerId,
1232
+ producedAt: f.produced_at ?? producedAt,
1233
+ area: f.area,
1234
+ claim: f.recommended_action ?? f.claim,
1235
+ ...f.claim ? { rationale: f.claim } : {},
1236
+ tags: [...opts.tags ?? [], `audience:${f.metadata?.audience ?? "agent"}`],
1237
+ confidence: f.confidence,
1238
+ evidence: [{ kind: "finding", uri: f.finding_id }]
1239
+ };
1240
+ const r = await opts.corpus.append(record);
1241
+ if (r.succeeded) learned.push(record);
1242
+ }
1243
+ }
1244
+ return { findings, learned, report: renderReport(findings) };
1245
+ }
1246
+ function parseFindings(content) {
1247
+ let obj2;
1248
+ try {
1249
+ obj2 = JSON.parse(content);
1250
+ } catch {
1251
+ const m = content.match(/\{[\s\S]*\}/);
1252
+ obj2 = m ? JSON.parse(m[0]) : { findings: [] };
1253
+ }
1254
+ const arr = obj2.findings;
1255
+ return Array.isArray(arr) ? arr : [];
1256
+ }
1257
+ function renderReport(findings) {
1258
+ if (findings.length === 0) return "\u2713 clean run \u2014 the observer found nothing to change.";
1259
+ const audience = (f) => f.metadata?.audience ?? "agent";
1260
+ const forAgent = findings.filter((f) => audience(f) === "agent");
1261
+ const forOperator = findings.filter((f) => audience(f) === "operator");
1262
+ const block = (title, fs) => fs.length === 0 ? "" : `**${title}**
1263
+ ${fs.map((f) => `- [${f.severity}] ${f.claim}
1264
+ \u2192 ${f.recommended_action ?? ""}`).join("\n")}
1265
+ `;
1266
+ return [
1267
+ block("For the agent (fix skills / prompt / tools)", forAgent),
1268
+ block("For you (the operator)", forOperator)
1269
+ ].filter(Boolean).join("\n");
1270
+ }
1271
+
1123
1272
  // src/runtime/harvest-corpus.ts
1124
1273
  async function harvestCorpus(opts) {
1125
1274
  const concurrency = Math.max(1, opts.concurrency ?? 4);
@@ -2106,13 +2255,13 @@ function shapeName(shape, _resolved) {
2106
2255
  }
2107
2256
  function resolveShapeBudget(root, over) {
2108
2257
  const fanout2 = over?.fanout ?? defaultFanout;
2109
- const perChild = over?.perChild ?? {
2258
+ const perChild2 = over?.perChild ?? {
2110
2259
  maxIterations: Math.max(1, Math.floor(root.maxIterations / fanout2)),
2111
2260
  maxTokens: Math.max(1, Math.floor(root.maxTokens / fanout2)),
2112
2261
  ...root.maxUsd !== void 0 ? { maxUsd: root.maxUsd / fanout2 } : {},
2113
2262
  ...root.deadlineMs !== void 0 ? { deadlineMs: root.deadlineMs } : {}
2114
2263
  };
2115
- return { perChild, fanout: fanout2 };
2264
+ return { perChild: perChild2, fanout: fanout2 };
2116
2265
  }
2117
2266
  var defaultFanout = 3;
2118
2267
  function personaRegistry(persona) {
@@ -2292,6 +2441,7 @@ function addNodeSpend(a, b) {
2292
2441
  iterations: a.iterations + b.iterations,
2293
2442
  tokens: { input: a.tokens.input + b.tokens.input, output: a.tokens.output + b.tokens.output },
2294
2443
  usd: a.usd + b.usd,
2444
+ ...a.usdKnown === false || b.usdKnown === false ? { usdKnown: false } : {},
2295
2445
  ms: a.ms + b.ms
2296
2446
  };
2297
2447
  }
@@ -2300,6 +2450,7 @@ function cloneSpend(spend) {
2300
2450
  iterations: spend.iterations,
2301
2451
  tokens: { input: spend.tokens.input, output: spend.tokens.output },
2302
2452
  usd: spend.usd,
2453
+ ...spend.usdKnown === false ? { usdKnown: false } : {},
2303
2454
  ms: spend.ms
2304
2455
  };
2305
2456
  }
@@ -2307,6 +2458,7 @@ function addSpend(acc, delta) {
2307
2458
  acc.iterations += delta.iterations;
2308
2459
  addTokenUsage(acc.tokens, delta.tokens);
2309
2460
  acc.usd += delta.usd;
2461
+ if (delta.usdKnown === false) acc.usdKnown = false;
2310
2462
  acc.ms += delta.ms;
2311
2463
  }
2312
2464
  function spreadOf(values) {
@@ -2498,6 +2650,668 @@ function promotionGate(opts) {
2498
2650
 
2499
2651
  // src/runtime/run-benchmark.ts
2500
2652
  import { pairedBootstrap as pairedBootstrap2, paretoFrontier } from "@tangle-network/agent-eval";
2653
+
2654
+ // src/runtime/strategy.ts
2655
+ import { createChatClient, estimateCost, isModelPriced } from "@tangle-network/agent-eval";
2656
+ var taskNudge = "Use the available tools to bring the artifact to the required final state. Address EVERY distinct change the request implies. After each tool result, check what remains and continue. Re-read the values you set to confirm they took. Reply DONE only once every required change is made and verified.";
2657
+ async function runShot(surface, _task, handle, tools, messages, opts, modelOverride) {
2658
+ let toolErrors = 0;
2659
+ const execute = async (name, args) => {
2660
+ try {
2661
+ const out = await surface.call(handle, name, args);
2662
+ if (out.startsWith("ERROR:")) toolErrors += 1;
2663
+ return out;
2664
+ } catch (e) {
2665
+ toolErrors += 1;
2666
+ return `ERROR: ${e instanceof Error ? e.message : String(e)}`;
2667
+ }
2668
+ };
2669
+ const r = await routerToolLoop(
2670
+ {
2671
+ routerBaseUrl: opts.routerBaseUrl,
2672
+ routerKey: opts.routerKey,
2673
+ model: modelOverride ?? opts.model,
2674
+ ...opts.complete ? { complete: opts.complete } : {}
2675
+ },
2676
+ "",
2677
+ "",
2678
+ tools,
2679
+ execute,
2680
+ {
2681
+ maxTurns: opts.innerTurns ?? 4,
2682
+ temperature: opts.temperature ?? 0.7,
2683
+ initialMessages: messages,
2684
+ ...opts.maxTokens ? { maxTokens: opts.maxTokens } : {}
2685
+ }
2686
+ );
2687
+ return {
2688
+ messages: r.messages,
2689
+ completions: r.turns,
2690
+ toolCalls: r.toolCalls,
2691
+ toolErrors,
2692
+ tokens: r.usage
2693
+ };
2694
+ }
2695
+ function compactTrajectory(messages) {
2696
+ return messages.filter((m) => m.role === "assistant" || m.role === "tool").map((m) => {
2697
+ if (m.role === "tool") return `RESULT ${String(m.content).slice(0, 280)}`;
2698
+ const calls = m.tool_calls?.map((c) => `${c.function.name}(${c.function.arguments})`).join(", ");
2699
+ return calls ? `CALL ${calls}` : `SAY ${String(m.content).slice(0, 200)}`;
2700
+ }).join("\n").slice(0, 7e3);
2701
+ }
2702
+ function analystChat(opts, defaultModel) {
2703
+ if (!opts.complete) {
2704
+ return createChatClient({
2705
+ transport: "router",
2706
+ apiKey: opts.routerKey,
2707
+ baseUrl: opts.routerBaseUrl,
2708
+ defaultModel
2709
+ });
2710
+ }
2711
+ const complete = opts.complete;
2712
+ return createChatClient({
2713
+ transport: "mock",
2714
+ defaultModel,
2715
+ handler: async (req) => {
2716
+ const raw = await complete({
2717
+ model: req.model ?? defaultModel,
2718
+ messages: req.messages,
2719
+ ...req.temperature !== void 0 ? { temperature: req.temperature } : {},
2720
+ ...req.maxTokens !== void 0 ? { max_tokens: req.maxTokens } : {}
2721
+ });
2722
+ const content = raw.choices?.[0]?.message?.content ?? "";
2723
+ const promptTokens = raw.usage?.prompt_tokens ?? 0;
2724
+ const completionTokens = raw.usage?.completion_tokens ?? 0;
2725
+ return {
2726
+ content,
2727
+ usage: {
2728
+ promptTokens,
2729
+ completionTokens,
2730
+ totalTokens: promptTokens + completionTokens
2731
+ },
2732
+ costUsd: null,
2733
+ model: req.model ?? defaultModel,
2734
+ durationMs: 0,
2735
+ finishReason: raw.choices?.[0]?.finish_reason ?? null,
2736
+ contentEmpty: content.trim().length === 0,
2737
+ raw
2738
+ };
2739
+ }
2740
+ });
2741
+ }
2742
+ async function consultAnalyst(task, messages, instruction, opts) {
2743
+ const trajectory = compactTrajectory(messages);
2744
+ const analystModel = opts.analystModel ?? opts.model;
2745
+ const chat = analystChat(opts, analystModel);
2746
+ const consultMessages = trajectory ? [
2747
+ { role: "system", content: instruction },
2748
+ {
2749
+ role: "user",
2750
+ content: `TASK: ${task.userPrompt.slice(0, 1500)}
2751
+
2752
+ TRAJECTORY:
2753
+ ${trajectory}`
2754
+ }
2755
+ ] : [
2756
+ {
2757
+ role: "user",
2758
+ content: `${instruction}
2759
+
2760
+ TASK:
2761
+ ${task.userPrompt.slice(0, 1500)}`
2762
+ }
2763
+ ];
2764
+ const res = await chat.chat({
2765
+ model: analystModel,
2766
+ temperature: 0.2,
2767
+ maxTokens: 1024,
2768
+ messages: consultMessages
2769
+ });
2770
+ const usage = res.usage;
2771
+ return {
2772
+ steer: res.content.trim(),
2773
+ tokens: {
2774
+ input: usage?.promptTokens ?? usage?.prompt_tokens ?? 0,
2775
+ output: usage?.completionTokens ?? usage?.completion_tokens ?? 0
2776
+ }
2777
+ };
2778
+ }
2779
+ async function analyze(task, messages, opts) {
2780
+ const trajectory = compactTrajectory(messages);
2781
+ const analystModel = opts.analystModel ?? opts.model;
2782
+ const inner = analystChat(opts, analystModel);
2783
+ const tokens = { input: 0, output: 0 };
2784
+ const chat = {
2785
+ ...inner,
2786
+ chat: async (req, callOpts) => {
2787
+ const res = await inner.chat(req, callOpts);
2788
+ const u = res.usage;
2789
+ if (u) {
2790
+ tokens.input += u.promptTokens ?? u.prompt_tokens ?? 0;
2791
+ tokens.output += u.completionTokens ?? u.completion_tokens ?? 0;
2792
+ }
2793
+ return res;
2794
+ }
2795
+ };
2796
+ const obs = await observe(
2797
+ {
2798
+ task: task.userPrompt,
2799
+ output: trajectory,
2800
+ trace: messages,
2801
+ outcome: "failed",
2802
+ runId: task.id
2803
+ },
2804
+ {
2805
+ chat,
2806
+ model: analystModel,
2807
+ ...opts.analystInstruction ? { analystInstruction: opts.analystInstruction } : {},
2808
+ ...opts.corpus ? { corpus: opts.corpus, tags: opts.corpusTags ?? [] } : {}
2809
+ }
2810
+ );
2811
+ const steer = obs.findings.map((f) => f.recommended_action).filter((a) => typeof a === "string" && a.trim().length > 0).join("\n").trim();
2812
+ return { steer: steer || "COMPLETE", tokens };
2813
+ }
2814
+ async function renderCorpusReadback(opts) {
2815
+ if (!opts.corpus || !opts.corpusReadback) return "";
2816
+ const maxFacts = opts.corpusReadback.maxFacts ?? 3;
2817
+ if (!Number.isInteger(maxFacts) || maxFacts < 0) {
2818
+ throw new Error(`corpusReadback.maxFacts must be a non-negative integer, got ${maxFacts}`);
2819
+ }
2820
+ if (maxFacts === 0) return "";
2821
+ const tags = [
2822
+ ...opts.corpusTags ?? [],
2823
+ ...opts.corpusReadback.tags ?? [],
2824
+ ...opts.corpusReadback.includeOperatorFacts ? [] : ["audience:agent"]
2825
+ ];
2826
+ const facts = await opts.corpus.query({
2827
+ ...tags.length > 0 ? { tags } : {},
2828
+ minConfidence: opts.corpusReadback.minConfidence ?? 0.7,
2829
+ limit: maxFacts
2830
+ });
2831
+ if (facts.length === 0) return "";
2832
+ const rendered = facts.map(
2833
+ (fact) => fact.rationale ? `- ${fact.claim} (${fact.rationale})` : `- ${fact.claim}`
2834
+ );
2835
+ return `Relevant learned facts from prior attempts:
2836
+ ${rendered.join("\n")}`;
2837
+ }
2838
+ function shotExecutor(surface, opts) {
2839
+ let artifact;
2840
+ return {
2841
+ runtime: "agentic-shot",
2842
+ async execute(task) {
2843
+ const t = task;
2844
+ const own = !t.handle;
2845
+ const handle = t.handle ?? await surface.open(t.task);
2846
+ try {
2847
+ const allTools = await surface.tools(t.task, handle);
2848
+ let tools = allTools;
2849
+ if (t.tools) {
2850
+ const known = new Set(allTools.map((tool) => tool.function.name));
2851
+ const unknown = t.tools.filter((name) => !known.has(name));
2852
+ if (unknown.length > 0) {
2853
+ throw new Error(
2854
+ `shot tools: unknown tool name(s) ${unknown.join(", ")} \u2014 domain offers: ${[...known].join(", ")}`
2855
+ );
2856
+ }
2857
+ const want = new Set(t.tools);
2858
+ tools = allTools.filter((tool) => want.has(tool.function.name));
2859
+ }
2860
+ const messages = t.messages?.length ? t.messages : [
2861
+ { role: "system", content: t.persona?.systemPrompt ?? t.task.systemPrompt },
2862
+ { role: "user", content: `${t.task.userPrompt}
2863
+
2864
+ ${taskNudge}` }
2865
+ ];
2866
+ if (t.messages?.length && t.persona?.systemPrompt) {
2867
+ messages.push({
2868
+ role: "user",
2869
+ content: `[hand-off] You are now acting as: ${t.persona.systemPrompt}`
2870
+ });
2871
+ }
2872
+ if (t.steer) messages.push({ role: "user", content: t.steer });
2873
+ const shot = await runShot(surface, t.task, handle, tools, messages, opts, t.persona?.model);
2874
+ const s = await surface.score(t.task, handle);
2875
+ const score = s.total > 0 ? s.passes / s.total : 0;
2876
+ const out = {
2877
+ messages: shot.messages,
2878
+ score,
2879
+ passes: s.passes,
2880
+ total: s.total,
2881
+ completions: shot.completions,
2882
+ toolErrors: shot.toolErrors
2883
+ };
2884
+ artifact = {
2885
+ outRef: `shot:${handle.id}:${shot.completions}:${s.passes}/${s.total}`,
2886
+ out,
2887
+ verdict: { valid: s.total > 0 && s.passes === s.total, score },
2888
+ // Real usage to the conserved pool: tokens from the router responses; usd only
2889
+ // when the model is in the price table (never a fabricated number).
2890
+ spent: {
2891
+ iterations: shot.completions,
2892
+ tokens: shot.tokens,
2893
+ usd: isModelPriced(opts.model) ? estimateCost(shot.tokens.input, shot.tokens.output, opts.model) : 0,
2894
+ ms: 0
2895
+ }
2896
+ };
2897
+ return artifact;
2898
+ } finally {
2899
+ if (own) await surface.close(handle);
2900
+ }
2901
+ },
2902
+ teardown: () => Promise.resolve({ destroyed: true }),
2903
+ resultArtifact() {
2904
+ if (!artifact) throw new Error("shotExecutor: resultArtifact before execute");
2905
+ return artifact;
2906
+ }
2907
+ };
2908
+ }
2909
+ function analystExecutor(opts) {
2910
+ let artifact;
2911
+ return {
2912
+ runtime: "agentic-analyst",
2913
+ async execute(task) {
2914
+ const t = task;
2915
+ const { steer, tokens } = t.rawInstruction ? await consultAnalyst(t.task, t.messages, t.rawInstruction, opts) : await analyze(t.task, t.messages, opts);
2916
+ const analystModel = opts.analystModel ?? opts.model;
2917
+ artifact = {
2918
+ outRef: `analyst:${steer.length}`,
2919
+ out: steer,
2920
+ spent: {
2921
+ iterations: 1,
2922
+ tokens,
2923
+ usd: isModelPriced(analystModel) ? estimateCost(tokens.input, tokens.output, analystModel) : 0,
2924
+ ms: 0
2925
+ }
2926
+ };
2927
+ return artifact;
2928
+ },
2929
+ teardown: () => Promise.resolve({ destroyed: true }),
2930
+ resultArtifact() {
2931
+ if (!artifact) throw new Error("analystExecutor: resultArtifact before execute");
2932
+ return artifact;
2933
+ }
2934
+ };
2935
+ }
2936
+ function agenticRegistry(surface, opts) {
2937
+ const leaves = {
2938
+ register() {
2939
+ throw new Error("agenticRegistry: register unsupported");
2940
+ },
2941
+ resolve(spec) {
2942
+ const role = spec.profile.metadata?.role;
2943
+ const factory = (_s, _ctx) => role === "analyst" ? analystExecutor(opts) : shotExecutor(surface, opts);
2944
+ return { succeeded: true, value: factory };
2945
+ }
2946
+ };
2947
+ return withDriverExecutor(leaves);
2948
+ }
2949
+ function leaf(name, role) {
2950
+ const agent = {
2951
+ name,
2952
+ executorSpec: { profile: { name, metadata: { role } }, harness: null },
2953
+ act() {
2954
+ throw new Error(`agentic: spawned child "${name}" was run directly (the executor drives it)`);
2955
+ }
2956
+ };
2957
+ return agent;
2958
+ }
2959
+ async function drainOne2(scope) {
2960
+ const s = await scope.next();
2961
+ if (!s) throw new Error("agentic: spawned child never settled");
2962
+ return s;
2963
+ }
2964
+ var perChild = (innerTurns) => ({
2965
+ maxIterations: innerTurns + 1,
2966
+ maxTokens: 1e6
2967
+ });
2968
+ function depthStrategy(surface, task, opts, cfg) {
2969
+ const innerTurns = opts.innerTurns ?? 4;
2970
+ let pendingSteer;
2971
+ return {
2972
+ name: "depth",
2973
+ async act(_t, scope) {
2974
+ const handle = await surface.open(task);
2975
+ const progression = [];
2976
+ let messages;
2977
+ let completions = 0;
2978
+ let shots = 0;
2979
+ try {
2980
+ for (shots = 0; shots < cfg.maxShots; shots += 1) {
2981
+ const child = leaf(`shot:${shots}`, "shot");
2982
+ const memorySteer = await renderCorpusReadback(opts);
2983
+ const steer = [shots === 0 ? void 0 : pendingSteer, memorySteer].filter((part) => typeof part === "string" && part.trim().length > 0).join("\n\n");
2984
+ const res = scope.spawn(child, { task, handle, messages, steer }, {
2985
+ budget: perChild(innerTurns),
2986
+ label: `shot:${shots}`
2987
+ });
2988
+ if (!res.ok) break;
2989
+ const settled = await drainOne2(scope);
2990
+ if (settled.kind === "down") break;
2991
+ const out = settled.out;
2992
+ messages = out.messages;
2993
+ completions += out.completions;
2994
+ progression.push(out.score);
2995
+ if (out.score >= 1 || shots === cfg.maxShots - 1) break;
2996
+ const aChild = leaf(`analyst:${shots}`, "analyst");
2997
+ const aRes = scope.spawn(
2998
+ aChild,
2999
+ { task, messages },
3000
+ { budget: perChild(1), label: `analyst:${shots}` }
3001
+ );
3002
+ if (!aRes.ok) break;
3003
+ const aSettled = await drainOne2(scope);
3004
+ completions += 1;
3005
+ if (aSettled.kind === "down") break;
3006
+ const findings = aSettled.out;
3007
+ if (/^\s*COMPLETE\b/i.test(findings)) break;
3008
+ pendingSteer = `A reviewer flagged unfinished items:
3009
+ ${findings}
3010
+
3011
+ Address each with the tools, verify they took, then continue.`;
3012
+ }
3013
+ const final = await surface.score(task, handle);
3014
+ const score = final.total > 0 ? final.passes / final.total : 0;
3015
+ return {
3016
+ kind: "done",
3017
+ deliverable: {
3018
+ mode: "depth",
3019
+ score,
3020
+ resolved: final.total > 0 && final.passes === final.total,
3021
+ completions,
3022
+ progression,
3023
+ shots: shots + 1
3024
+ }
3025
+ };
3026
+ } finally {
3027
+ await surface.close(handle);
3028
+ }
3029
+ }
3030
+ };
3031
+ }
3032
+ function breadthStrategy(_surface, task, opts, cfg) {
3033
+ const innerTurns = opts.innerTurns ?? 4;
3034
+ return {
3035
+ name: "breadth",
3036
+ async act(_t, scope) {
3037
+ let opened = 0;
3038
+ for (let k = 0; k < cfg.width; k += 1) {
3039
+ const res = scope.spawn(leaf(`rollout:${k}`, "shot"), { task }, {
3040
+ budget: perChild(innerTurns),
3041
+ label: `rollout:${k}`
3042
+ });
3043
+ if (res.ok) opened += 1;
3044
+ }
3045
+ if (opened === 0) return { kind: "blocked", blockers: ["breadth: pool admitted no rollout"] };
3046
+ let best = -1;
3047
+ let bestResolved = false;
3048
+ let completions = 0;
3049
+ const progression = [];
3050
+ for (let s = await scope.next(); s !== null; s = await scope.next()) {
3051
+ if (s.kind === "down") continue;
3052
+ const out = s.out;
3053
+ completions += out.completions;
3054
+ if (out.score > best) best = out.score;
3055
+ if (out.total > 0 && out.passes === out.total) bestResolved = true;
3056
+ progression.push(best);
3057
+ }
3058
+ if (best < 0) return { kind: "blocked", blockers: ["breadth: every rollout went down"] };
3059
+ return {
3060
+ kind: "done",
3061
+ deliverable: {
3062
+ mode: "breadth",
3063
+ score: best,
3064
+ resolved: bestResolved,
3065
+ completions,
3066
+ progression,
3067
+ shots: opened
3068
+ }
3069
+ };
3070
+ }
3071
+ };
3072
+ }
3073
+ var sample = {
3074
+ name: "sample",
3075
+ driver: (surface, task, opts, budget) => breadthStrategy(surface, task, opts, { width: budget })
3076
+ };
3077
+ var refine = {
3078
+ name: "refine",
3079
+ driver: (surface, task, opts, budget) => depthStrategy(surface, task, opts, { maxShots: budget })
3080
+ };
3081
+ function defineStrategy(name, run) {
3082
+ return {
3083
+ name,
3084
+ driver: (surface, task, opts, budget) => ({
3085
+ name,
3086
+ async act(_t, scope) {
3087
+ let seq = 0;
3088
+ const innerTurns = opts.innerTurns ?? 4;
3089
+ let verifiedBest = 0;
3090
+ let verifiedResolved = false;
3091
+ const openHandles = /* @__PURE__ */ new Set();
3092
+ const ctx = {
3093
+ // Narrowed to open/close — the body gets no raw call()/score() access.
3094
+ surface: {
3095
+ name: surface.name,
3096
+ open: async (t) => {
3097
+ const h = await surface.open(t);
3098
+ openHandles.add(h.id);
3099
+ return h;
3100
+ },
3101
+ close: async (h) => {
3102
+ if (!h || !openHandles.has(h.id)) return;
3103
+ openHandles.delete(h.id);
3104
+ await surface.close(h);
3105
+ }
3106
+ },
3107
+ task,
3108
+ opts,
3109
+ budget,
3110
+ scope,
3111
+ async shot(spec) {
3112
+ const child = leaf(`shot:${seq}`, "shot");
3113
+ seq += 1;
3114
+ const res = scope.spawn(
3115
+ child,
3116
+ {
3117
+ task,
3118
+ handle: spec?.handle,
3119
+ messages: spec?.messages,
3120
+ steer: spec?.steer,
3121
+ persona: spec?.persona,
3122
+ tools: spec?.tools
3123
+ },
3124
+ { budget: perChild(innerTurns), label: child.name }
3125
+ );
3126
+ if (!res.ok) return null;
3127
+ const settled = await drainOne2(scope);
3128
+ if (settled.kind === "down") return null;
3129
+ const out = settled.out;
3130
+ if (out.score > verifiedBest) verifiedBest = out.score;
3131
+ if (out.total > 0 && out.passes === out.total) verifiedResolved = true;
3132
+ return out;
3133
+ },
3134
+ async listTools(handle) {
3135
+ const tools = await surface.tools(task, handle);
3136
+ return tools.map((t) => ({
3137
+ name: t.function.name,
3138
+ ...t.function.description ? { description: t.function.description } : {}
3139
+ }));
3140
+ },
3141
+ async critique(messages) {
3142
+ const child = leaf(`analyst:${seq}`, "analyst");
3143
+ seq += 1;
3144
+ const res = scope.spawn(
3145
+ child,
3146
+ { task, messages },
3147
+ { budget: perChild(1), label: child.name }
3148
+ );
3149
+ if (!res.ok) return null;
3150
+ const settled = await drainOne2(scope);
3151
+ if (settled.kind === "down") return null;
3152
+ const findings = settled.out;
3153
+ return /^\s*COMPLETE\b/i.test(findings) ? null : findings;
3154
+ },
3155
+ async consult(messages, instruction) {
3156
+ const child = leaf(`analyst:${seq}`, "analyst");
3157
+ seq += 1;
3158
+ const res = scope.spawn(
3159
+ child,
3160
+ { task, messages, rawInstruction: instruction },
3161
+ { budget: perChild(1), label: child.name }
3162
+ );
3163
+ if (!res.ok) return null;
3164
+ const settled = await drainOne2(scope);
3165
+ if (settled.kind === "down") return null;
3166
+ return settled.out;
3167
+ }
3168
+ };
3169
+ const r = await run(ctx);
3170
+ return {
3171
+ kind: "done",
3172
+ deliverable: {
3173
+ mode: name,
3174
+ ...r,
3175
+ progression: Array.isArray(r.progression) ? r.progression : [],
3176
+ completions: typeof r.completions === "number" ? r.completions : 0,
3177
+ shots: typeof r.shots === "number" ? r.shots : 0,
3178
+ score: verifiedBest,
3179
+ resolved: verifiedResolved
3180
+ }
3181
+ };
3182
+ }
3183
+ })
3184
+ };
3185
+ }
3186
+ var adaptiveRefine = defineStrategy(
3187
+ "adaptiveRefine",
3188
+ async ({ surface, task, budget, shot, critique }) => {
3189
+ let handle = await surface.open(task);
3190
+ const progression = [];
3191
+ let messages;
3192
+ let steer;
3193
+ let completions = 0;
3194
+ let best = -1;
3195
+ let shots = 0;
3196
+ try {
3197
+ for (shots = 0; shots < budget; shots += 1) {
3198
+ const out = await shot({ handle, messages, steer });
3199
+ if (!out) break;
3200
+ completions += out.completions;
3201
+ progression.push(out.score);
3202
+ if (out.score >= 1) break;
3203
+ if (out.score <= best) {
3204
+ await surface.close(handle);
3205
+ handle = await surface.open(task);
3206
+ messages = void 0;
3207
+ steer = void 0;
3208
+ continue;
3209
+ }
3210
+ best = out.score;
3211
+ messages = out.messages;
3212
+ const findings = await critique(out.messages);
3213
+ completions += 1;
3214
+ if (!findings) break;
3215
+ steer = `A reviewer flagged unfinished items:
3216
+ ${findings}
3217
+
3218
+ Address each with the tools, verify they took, then continue.`;
3219
+ }
3220
+ const score = progression.length ? Math.max(...progression) : 0;
3221
+ return { score, resolved: score >= 1, completions, progression, shots };
3222
+ } finally {
3223
+ await surface.close(handle);
3224
+ }
3225
+ }
3226
+ );
3227
+ var sampleThenRefine = defineStrategy(
3228
+ "sampleThenRefine",
3229
+ async ({ surface, task, budget, shot, critique }) => {
3230
+ const explore = Math.max(1, Math.ceil(budget / 2));
3231
+ const open = /* @__PURE__ */ new Set();
3232
+ const progression = [];
3233
+ let completions = 0;
3234
+ let shots = 0;
3235
+ try {
3236
+ let best;
3237
+ for (let i = 0; i < explore; i += 1) {
3238
+ const handle = await surface.open(task);
3239
+ open.add(handle);
3240
+ const out = await shot({ handle });
3241
+ if (!out) continue;
3242
+ shots += 1;
3243
+ completions += out.completions;
3244
+ progression.push(out.score);
3245
+ if (!best || out.score > best.out.score) best = { handle, out };
3246
+ if (out.score >= 1) break;
3247
+ }
3248
+ if (!best) return { score: 0, resolved: false, completions, progression, shots };
3249
+ for (const h of [...open]) {
3250
+ if (h !== best.handle) {
3251
+ await surface.close(h);
3252
+ open.delete(h);
3253
+ }
3254
+ }
3255
+ let messages = best.out.messages;
3256
+ let topScore = best.out.score;
3257
+ for (let i = explore; i < budget && topScore < 1; i += 1) {
3258
+ const findings = await critique(messages);
3259
+ completions += 1;
3260
+ if (!findings) break;
3261
+ const out = await shot({
3262
+ handle: best.handle,
3263
+ messages,
3264
+ steer: `A reviewer flagged unfinished items:
3265
+ ${findings}
3266
+
3267
+ Address each with the tools, verify they took, then continue.`
3268
+ });
3269
+ if (!out) break;
3270
+ shots += 1;
3271
+ completions += out.completions;
3272
+ progression.push(out.score);
3273
+ messages = out.messages;
3274
+ if (out.score > topScore) topScore = out.score;
3275
+ }
3276
+ const score = progression.length ? Math.max(...progression) : 0;
3277
+ return { score, resolved: score >= 1, completions, progression, shots };
3278
+ } finally {
3279
+ for (const h of open) await surface.close(h);
3280
+ }
3281
+ }
3282
+ );
3283
+ async function runAgentic(opts) {
3284
+ const strategy = opts.strategy ?? (opts.mode === "breadth" ? sample : refine);
3285
+ const driver = strategy.driver(opts.surface, opts.task, opts, opts.budget);
3286
+ const supervisor = createSupervisor();
3287
+ const root = opts.rootBudget ?? {
3288
+ maxIterations: opts.budget * ((opts.innerTurns ?? 4) + 2),
3289
+ maxTokens: 1e9
3290
+ };
3291
+ const started = Date.now();
3292
+ const result = await supervisor.run(driver, void 0, {
3293
+ budget: root,
3294
+ runId: `agentic:${strategy.name}:${opts.task.id}`,
3295
+ journal: new InMemorySpawnJournal(),
3296
+ blobs: new InMemoryResultBlobStore(),
3297
+ executors: agenticRegistry(opts.surface, opts),
3298
+ maxDepth: 3,
3299
+ ...opts.hooks ? { hooks: opts.hooks } : {}
3300
+ });
3301
+ if (result.kind !== "winner" || result.out.kind !== "done") {
3302
+ const reason = result.kind === "winner" ? `blocked: ${result.out.blockers?.join("; ")}` : `no-winner: ${result.reason}`;
3303
+ throw new Error(`runAgentic(${strategy.name}) produced no result \u2014 ${reason}`);
3304
+ }
3305
+ const core = result.out.deliverable;
3306
+ return {
3307
+ ...core,
3308
+ usd: result.spentTotal.usd,
3309
+ tokens: result.spentTotal.tokens,
3310
+ ms: Date.now() - started
3311
+ };
3312
+ }
3313
+
3314
+ // src/runtime/run-benchmark.ts
2501
3315
  async function pool(items, limit, fn) {
2502
3316
  const out = new Array(items.length);
2503
3317
  let next = 0;
@@ -4229,6 +5043,381 @@ function deriveTurnSignal(callerSignal, timeoutMs) {
4229
5043
  };
4230
5044
  }
4231
5045
 
5046
+ // src/runtime/structural-rollout.ts
5047
+ import { randomBytes } from "crypto";
5048
+ var defaultStructuralRolloutPolicy = {
5049
+ k: 5,
5050
+ repairRounds: 2,
5051
+ testgen: 6
5052
+ };
5053
+ function resolvePolicy(overrides) {
5054
+ const policy = { ...defaultStructuralRolloutPolicy, ...overrides };
5055
+ if (!Number.isInteger(policy.k) || policy.k < 1) {
5056
+ throw new Error(`structuralRollout: policy.k must be an integer >= 1, got ${policy.k}`);
5057
+ }
5058
+ if (!Number.isInteger(policy.repairRounds) || policy.repairRounds < 0) {
5059
+ throw new Error(
5060
+ `structuralRollout: policy.repairRounds must be an integer >= 0, got ${policy.repairRounds}`
5061
+ );
5062
+ }
5063
+ if (!Number.isInteger(policy.testgen) || policy.testgen < 0) {
5064
+ throw new Error(
5065
+ `structuralRollout: policy.testgen must be an integer >= 0, got ${policy.testgen}`
5066
+ );
5067
+ }
5068
+ return policy;
5069
+ }
5070
+ var authorInstruction = (count, entry) => `Read the task below. Write exactly ${count} single-line assert statements that test the function \`${entry}\`, based ONLY on the behavior the task itself describes. Each assert must be one physical line of the form \`assert ${entry}(...) == expected\` (or a True/False check). Do NOT implement the function. Do NOT copy shown examples verbatim if you can test other cases too. Output ONLY the assert lines inside a single \`\`\`python code block.`;
5071
+ function filterAuthoredAsserts(reply, entrySymbol, count) {
5072
+ const fences = [...reply.matchAll(/```(?:python|py)?\s*\n([\s\S]*?)```/gi)].map(
5073
+ (m) => (m[1] ?? "").trim()
5074
+ );
5075
+ const block = fences.length > 0 ? fences.join("\n") : reply;
5076
+ const balanced = (s) => {
5077
+ let d = 0;
5078
+ for (const ch of s) {
5079
+ if (ch === "(" || ch === "[" || ch === "{") d += 1;
5080
+ else if (ch === ")" || ch === "]" || ch === "}") d -= 1;
5081
+ if (d < 0) return false;
5082
+ }
5083
+ return d === 0;
5084
+ };
5085
+ return block.split("\n").map((l) => l.trim()).filter((l) => l.startsWith("assert ") && l.includes(entrySymbol) && balanced(l)).slice(0, count);
5086
+ }
5087
+ function modelAuthoredChecks(overrides = {}) {
5088
+ return {
5089
+ async generate(_task, ctx) {
5090
+ const count = overrides.count ?? ctx.count;
5091
+ if (count <= 0 || !ctx.entrySymbol) return [];
5092
+ const entry = ctx.entrySymbol;
5093
+ const reply = await ctx.consult(authorInstruction(count, entry));
5094
+ if (!reply) return [];
5095
+ return filterAuthoredAsserts(reply, entry, count).map((code) => ({
5096
+ code,
5097
+ kind: "authored"
5098
+ }));
5099
+ }
5100
+ };
5101
+ }
5102
+ function officialChecksFromMeta(key = "visibleChecks") {
5103
+ return {
5104
+ async generate(task) {
5105
+ const raw = task.meta?.[key];
5106
+ if (!Array.isArray(raw)) return [];
5107
+ return raw.filter((c) => typeof c === "string" && c.trim().length > 0).map((code) => ({ code, kind: "official" }));
5108
+ }
5109
+ };
5110
+ }
5111
+ function composeCheckSources(...sources) {
5112
+ return {
5113
+ async generate(task, ctx) {
5114
+ const all = [];
5115
+ for (const source of sources) all.push(...await source.generate(task, ctx));
5116
+ return all;
5117
+ }
5118
+ };
5119
+ }
5120
+ function resolveEntrySymbol(task) {
5121
+ const meta = task.meta?.entryPoint;
5122
+ if (typeof meta === "string" && meta.trim().length > 0) return meta.trim();
5123
+ const defs = [...task.userPrompt.matchAll(/(?:^|\n)\s*def\s+([A-Za-z_]\w*)\s*\(/g)];
5124
+ const last = defs[defs.length - 1];
5125
+ return last?.[1];
5126
+ }
5127
+ function buildCheckProgram(candidate, official, authored, nonce) {
5128
+ const officialB64 = Buffer.from(JSON.stringify(official), "utf8").toString("base64");
5129
+ const authoredB64 = Buffer.from(JSON.stringify(authored), "utf8").toString("base64");
5130
+ return `${candidate}
5131
+
5132
+ import base64 as _b64, json as _json, sys as _sys
5133
+ _official = _json.loads(_b64.b64decode("${officialB64}").decode("utf8"))
5134
+ _authored = _json.loads(_b64.b64decode("${authoredB64}").decode("utf8"))
5135
+ _lines = []
5136
+ def _run(_tests):
5137
+ _att, _fail = 0, 0
5138
+ for _t in _tests:
5139
+ _att += 1
5140
+ try:
5141
+ exec(_t, dict(globals()))
5142
+ except Exception as _e:
5143
+ _fail += 1
5144
+ _lines.append("CHECK FAILED: %s -> %s: %s" % (_t.strip()[:200], type(_e).__name__, str(_e)[:200]))
5145
+ return _att, _fail
5146
+ _o_att, _o_fail = _run(_official)
5147
+ _a_att, _a_fail = _run(_authored)
5148
+ print("SRCK-${nonce} official=%d/%d authored=%d/%d" % (_o_att - _o_fail, _o_att, _a_att - _a_fail, _a_att))
5149
+ _sys.stdout.write("\\n".join(_lines)[-1500:])
5150
+ _sys.exit(0 if (_o_fail + _a_fail) == 0 and (_o_att + _a_att) > 0 else 1)
5151
+ `;
5152
+ }
5153
+ function sandboxCheckRunner(options = {}) {
5154
+ const python = options.python ?? "python3";
5155
+ const timeoutMs = options.timeoutMs ?? 2e4;
5156
+ return {
5157
+ async run(candidate, checks, ctx) {
5158
+ if (checks.length === 0) {
5159
+ return {
5160
+ passedOfficial: 0,
5161
+ totalOfficial: 0,
5162
+ passedAuthored: 0,
5163
+ totalAuthored: 0,
5164
+ failureOutput: ""
5165
+ };
5166
+ }
5167
+ const box = ctx.box ?? options.box;
5168
+ if (!box) {
5169
+ throw new Error(
5170
+ "sandboxCheckRunner: no execution channel \u2014 bind one via sandboxCheckRunner({ box }) or CheckRunContext.box (ValidationCtx.box / a sandbox instance). Refusing to score without executing: a silent 0 would poison selection."
5171
+ );
5172
+ }
5173
+ const nonce = randomBytes(8).toString("hex");
5174
+ const official = checks.filter((c) => c.kind === "official").map((c) => c.code);
5175
+ const authored = checks.filter((c) => c.kind === "authored").map((c) => c.code);
5176
+ const program = buildCheckProgram(candidate, official, authored, nonce);
5177
+ const b64 = Buffer.from(program, "utf8").toString("base64");
5178
+ const r = await box.exec(`printf '%s' '${b64}' | base64 -d | ${python} -`, { timeoutMs });
5179
+ const summary = new RegExp(
5180
+ `SRCK-${nonce} official=(\\d+)/(\\d+) authored=(\\d+)/(\\d+)`
5181
+ ).exec(r.stdout);
5182
+ if (!summary) {
5183
+ const detail = (r.stderr || r.stdout).slice(-1500) || "no output (crashed or timed out before the checks could run)";
5184
+ return {
5185
+ passedOfficial: 0,
5186
+ totalOfficial: 0,
5187
+ passedAuthored: 0,
5188
+ totalAuthored: 0,
5189
+ failureOutput: detail,
5190
+ crashed: true
5191
+ };
5192
+ }
5193
+ const failureOutput = r.stdout.replace(summary[0], "").slice(-1500).trim();
5194
+ return {
5195
+ passedOfficial: Number(summary[1]),
5196
+ totalOfficial: Number(summary[2]),
5197
+ passedAuthored: Number(summary[3]),
5198
+ totalAuthored: Number(summary[4]),
5199
+ failureOutput
5200
+ };
5201
+ }
5202
+ };
5203
+ }
5204
+ var frac = (passed, total) => total > 0 ? passed / total : 0;
5205
+ function compareCheckOutcomes(a, b) {
5206
+ const aCrashed = a.crashed === true;
5207
+ const bCrashed = b.crashed === true;
5208
+ if (aCrashed !== bCrashed) return aCrashed ? -1 : 1;
5209
+ if (aCrashed) return 0;
5210
+ const official = frac(a.passedOfficial, a.totalOfficial) - frac(b.passedOfficial, b.totalOfficial);
5211
+ if (official !== 0) return official;
5212
+ return frac(a.passedAuthored, a.totalAuthored) - frac(b.passedAuthored, b.totalAuthored);
5213
+ }
5214
+ function visibleCheckScore(o) {
5215
+ if (o.crashed) return -1;
5216
+ return frac(o.passedOfficial, o.totalOfficial) + 1e-3 * frac(o.passedAuthored, o.totalAuthored);
5217
+ }
5218
+ function selectBestIndex(outcomes) {
5219
+ let best = 0;
5220
+ for (let i = 1; i < outcomes.length; i += 1) {
5221
+ if (compareCheckOutcomes(outcomes[i], outcomes[best]) > 0) {
5222
+ best = i;
5223
+ }
5224
+ }
5225
+ return best;
5226
+ }
5227
+ function canDisplace(challenger, incumbent) {
5228
+ if (challenger.crashed === true) return false;
5229
+ if (challenger.passedOfficial < incumbent.passedOfficial) return false;
5230
+ return compareCheckOutcomes(challenger, incumbent) > 0;
5231
+ }
5232
+ var totalChecks = (o) => o.totalOfficial + o.totalAuthored;
5233
+ var passesAllChecks = (o) => o.crashed !== true && totalChecks(o) > 0 && o.passedOfficial === o.totalOfficial && o.passedAuthored === o.totalAuthored;
5234
+ function defaultExtractCandidate(messages) {
5235
+ for (let i = messages.length - 1; i >= 0; i -= 1) {
5236
+ const calls = messages[i]?.tool_calls;
5237
+ if (!calls) continue;
5238
+ for (let j = calls.length - 1; j >= 0; j -= 1) {
5239
+ const call = calls[j];
5240
+ if (call?.function?.name !== "submit_answer") continue;
5241
+ try {
5242
+ const args = JSON.parse(call.function.arguments ?? "{}");
5243
+ if (typeof args.answer === "string" && args.answer.trim()) return args.answer.trim();
5244
+ } catch {
5245
+ }
5246
+ }
5247
+ }
5248
+ const contents = [];
5249
+ for (const m of messages) {
5250
+ if (m.role === "assistant" && typeof m.content === "string" && m.content.trim()) {
5251
+ contents.push(m.content);
5252
+ }
5253
+ }
5254
+ const fencesOf = (text) => [...text.matchAll(/```(?:python|py)?\s*\n([\s\S]*?)```/gi)].map((m) => (m[1] ?? "").trim());
5255
+ for (let i = contents.length - 1; i >= 0; i -= 1) {
5256
+ const fences = fencesOf(contents[i]);
5257
+ for (let j = fences.length - 1; j >= 0; j -= 1) {
5258
+ if (/(^|\n)\s*def\s+\w+/.test(fences[j])) return fences[j];
5259
+ }
5260
+ }
5261
+ for (let i = contents.length - 1; i >= 0; i -= 1) {
5262
+ const fences = fencesOf(contents[i]);
5263
+ if (fences.length > 0) return fences[fences.length - 1];
5264
+ }
5265
+ return (contents[contents.length - 1] ?? "").trim();
5266
+ }
5267
+ var DIVERSE_LENSES = [
5268
+ "Answer directly and decisively from what you already know. State the single best answer without hedging.",
5269
+ "Decompose the question into the sub-facts it depends on. Establish each sub-fact explicitly, then compose them into the answer.",
5270
+ "Reason from first principles. Ignore the most obvious or popular guess; derive the answer from underlying facts and relationships.",
5271
+ "Name the most plausible WRONG answer and the trap that makes it tempting. Rule it out, then commit to the answer that survives."
5272
+ ];
5273
+ function slotLens(slot) {
5274
+ const lens = DIVERSE_LENSES[slot % DIVERSE_LENSES.length];
5275
+ const tag = slot < DIVERSE_LENSES.length ? "" : ` (variant ${Math.floor(slot / DIVERSE_LENSES.length) + 1})`;
5276
+ return `${lens}${tag}`;
5277
+ }
5278
+ function repairSteer(outcome) {
5279
+ return [
5280
+ "Your latest solution failed some of the task-visible checks.",
5281
+ "Result of running the visible checks against it:",
5282
+ "```",
5283
+ outcome.failureOutput.trim() || "(the code crashed before the checks could run)",
5284
+ "```",
5285
+ "Fix the solution so the visible checks pass. Provide the COMPLETE corrected solution the",
5286
+ "same way you provided the original (same tool or format) \u2014 not a fragment or a diff."
5287
+ ].join("\n");
5288
+ }
5289
+ function describeOutcome(label, o) {
5290
+ if (o.crashed) return `${label}: crashed before the checks could run`;
5291
+ return `${label}: official ${o.passedOfficial}/${o.totalOfficial}, authored ${o.passedAuthored}/${o.totalAuthored}`;
5292
+ }
5293
+ function structuralRollout(config = {}) {
5294
+ const policy = resolvePolicy(config.policy);
5295
+ const checkSource = config.checkSource ?? composeCheckSources(officialChecksFromMeta(), modelAuthoredChecks());
5296
+ const checkRunner = config.checkRunner ?? sandboxCheckRunner();
5297
+ const extract = config.extractCandidate ?? defaultExtractCandidate;
5298
+ const inner = defineStrategy(
5299
+ "structuralRollout",
5300
+ async (ctx) => {
5301
+ const { task, shot } = ctx;
5302
+ const progression = [];
5303
+ const receipts = [];
5304
+ let completions = 0;
5305
+ let shots = 0;
5306
+ const consult = async (instruction) => {
5307
+ const reply = await ctx.consult([], instruction);
5308
+ completions += 1;
5309
+ return reply;
5310
+ };
5311
+ const entrySymbol = resolveEntrySymbol(task);
5312
+ const checks = await checkSource.generate(task, {
5313
+ count: policy.testgen,
5314
+ ...entrySymbol ? { entrySymbol } : {},
5315
+ consult
5316
+ });
5317
+ const officialChecks = checks.filter((c) => c.kind === "official").length;
5318
+ const authoredChecks = checks.length - officialChecks;
5319
+ const runCtx = { task, ...config.box ? { box: config.box } : {} };
5320
+ const candidates = [];
5321
+ for (let i = 0; i < policy.k; i += 1) {
5322
+ const out = await shot(policy.diverse ? { steer: slotLens(i) } : void 0);
5323
+ if (!out) break;
5324
+ shots += 1;
5325
+ completions += out.completions;
5326
+ progression.push(out.score);
5327
+ const outcome = await checkRunner.run(extract(out.messages), checks, runCtx);
5328
+ candidates.push({
5329
+ index: candidates.length,
5330
+ messages: out.messages,
5331
+ outcome,
5332
+ shotScore: out.score,
5333
+ shotResolved: out.total > 0 && out.passes === out.total
5334
+ });
5335
+ }
5336
+ if (candidates.length === 0) {
5337
+ return {
5338
+ score: 0,
5339
+ resolved: false,
5340
+ completions,
5341
+ progression,
5342
+ shots,
5343
+ selection: receipts,
5344
+ repairStop: "no-candidates",
5345
+ officialChecks,
5346
+ authoredChecks
5347
+ };
5348
+ }
5349
+ let best = candidates[selectBestIndex(candidates.map((c) => c.outcome))];
5350
+ for (const c of candidates) {
5351
+ receipts.push({
5352
+ candidateIndex: c.index,
5353
+ selected: false,
5354
+ score: visibleCheckScore(c.outcome),
5355
+ reason: describeOutcome("sample", c.outcome),
5356
+ selector: "driver"
5357
+ });
5358
+ }
5359
+ let seq = candidates.length;
5360
+ let repairStop = "already-passing";
5361
+ if (!passesAllChecks(best.outcome)) {
5362
+ if (best.outcome.crashed !== true && totalChecks(best.outcome) === 0) {
5363
+ repairStop = "no-signal";
5364
+ } else {
5365
+ repairStop = "rounds-exhausted";
5366
+ for (let r = 0; r < policy.repairRounds; r += 1) {
5367
+ const out = await shot({ messages: best.messages, steer: repairSteer(best.outcome) });
5368
+ if (!out) break;
5369
+ shots += 1;
5370
+ completions += out.completions;
5371
+ progression.push(out.score);
5372
+ const outcome = await checkRunner.run(extract(out.messages), checks, runCtx);
5373
+ const displaced = canDisplace(outcome, best.outcome);
5374
+ const label = displaced ? "repair (displaced the incumbent)" : outcome.crashed !== true && outcome.passedOfficial < best.outcome.passedOfficial ? "repair (held out: passes fewer official checks than the incumbent)" : "repair (held out: no improvement)";
5375
+ receipts.push({
5376
+ candidateIndex: seq,
5377
+ selected: false,
5378
+ score: visibleCheckScore(outcome),
5379
+ reason: describeOutcome(label, outcome),
5380
+ selector: "driver"
5381
+ });
5382
+ if (displaced) {
5383
+ best = {
5384
+ index: seq,
5385
+ messages: out.messages,
5386
+ outcome,
5387
+ shotScore: out.score,
5388
+ shotResolved: out.total > 0 && out.passes === out.total
5389
+ };
5390
+ }
5391
+ seq += 1;
5392
+ if (passesAllChecks(best.outcome)) {
5393
+ repairStop = "repaired-pass";
5394
+ break;
5395
+ }
5396
+ }
5397
+ }
5398
+ }
5399
+ const winner = receipts.find((r) => r.candidateIndex === best.index);
5400
+ if (winner) winner.selected = true;
5401
+ return {
5402
+ score: best.shotScore,
5403
+ resolved: best.shotResolved,
5404
+ completions,
5405
+ progression,
5406
+ shots,
5407
+ selection: receipts,
5408
+ repairStop,
5409
+ officialChecks,
5410
+ authoredChecks
5411
+ };
5412
+ }
5413
+ );
5414
+ if (policy.temperature === void 0) return inner;
5415
+ return {
5416
+ name: inner.name,
5417
+ driver: (surface, task, opts, budget) => inner.driver(surface, task, { ...opts, temperature: policy.temperature }, budget)
5418
+ };
5419
+ }
5420
+
4232
5421
  // src/runtime/supervise/detector-monitor.ts
4233
5422
  import {
4234
5423
  argHash,
@@ -5067,6 +6256,9 @@ export {
5067
6256
  naiveDriver,
5068
6257
  dumbDriver,
5069
6258
  defineLeaderboard,
6259
+ defaultAnalystInstruction,
6260
+ observe,
6261
+ renderReport,
5070
6262
  harvestCorpus,
5071
6263
  inProcessSandboxClient,
5072
6264
  createMcpEnvironment,
@@ -5093,6 +6285,14 @@ export {
5093
6285
  trajectoryReport,
5094
6286
  equalKOnCost,
5095
6287
  promotionGate,
6288
+ depthStrategy,
6289
+ breadthStrategy,
6290
+ sample,
6291
+ refine,
6292
+ defineStrategy,
6293
+ adaptiveRefine,
6294
+ sampleThenRefine,
6295
+ runAgentic,
5096
6296
  runBenchmark,
5097
6297
  printBenchmarkReport,
5098
6298
  SandboxRunAbortError,
@@ -5106,6 +6306,19 @@ export {
5106
6306
  runStrategyEvolution,
5107
6307
  streamAgentTurn,
5108
6308
  collectAgentTurn,
6309
+ defaultStructuralRolloutPolicy,
6310
+ filterAuthoredAsserts,
6311
+ modelAuthoredChecks,
6312
+ officialChecksFromMeta,
6313
+ composeCheckSources,
6314
+ resolveEntrySymbol,
6315
+ sandboxCheckRunner,
6316
+ compareCheckOutcomes,
6317
+ visibleCheckScore,
6318
+ selectBestIndex,
6319
+ canDisplace,
6320
+ defaultExtractCandidate,
6321
+ structuralRollout,
5109
6322
  defaultToolDetectors,
5110
6323
  watchTrace,
5111
6324
  runCoderChecks,
@@ -5124,6 +6337,6 @@ export {
5124
6337
  jjWorkspace,
5125
6338
  runInWorkspace,
5126
6339
  computeFindingId,
5127
- makeFinding
6340
+ makeFinding2 as makeFinding
5128
6341
  };
5129
- //# sourceMappingURL=chunk-6K5CI33W.js.map
6342
+ //# sourceMappingURL=chunk-D336OZHK.js.map