@tangle-network/agent-runtime 0.79.3 → 0.80.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +54 -325
  2. package/dist/agent.js +5 -5
  3. package/dist/analyst-loop.js +2 -2
  4. package/dist/{chunk-VMNEQHJR.js → chunk-3X53HYRW.js} +17 -2
  5. package/dist/chunk-3X53HYRW.js.map +1 -0
  6. package/dist/{chunk-T2HVQVB4.js → chunk-4J6RBI3K.js} +15 -1
  7. package/dist/chunk-4J6RBI3K.js.map +1 -0
  8. package/dist/{chunk-IODKUOBA.js → chunk-C2FZ6GR6.js} +2 -2
  9. package/dist/{chunk-63MHOCIE.js → chunk-DH72UOYR.js} +4 -4
  10. package/dist/{chunk-Z3RRRPRB.js → chunk-H7IBHAFT.js} +23 -14
  11. package/dist/chunk-H7IBHAFT.js.map +1 -0
  12. package/dist/{chunk-PVPFDTO3.js → chunk-LLJRUTE7.js} +2 -2
  13. package/dist/{chunk-PBE35ULD.js → chunk-M3Y362RW.js} +2 -2
  14. package/dist/{chunk-AG335EXG.js → chunk-QSO2TVDS.js} +668 -74
  15. package/dist/chunk-QSO2TVDS.js.map +1 -0
  16. package/dist/{chunk-KRULXIWS.js → chunk-TPII5AU7.js} +55 -7
  17. package/dist/chunk-TPII5AU7.js.map +1 -0
  18. package/dist/{coordination-DCmljYDf.d.ts → coordination-BI9tpcmF.d.ts} +12 -3
  19. package/dist/environment-provider.d.ts +1 -1
  20. package/dist/environment-provider.js +1 -1
  21. package/dist/index.d.ts +4 -4
  22. package/dist/index.js +8 -8
  23. package/dist/intelligence.d.ts +30 -2
  24. package/dist/intelligence.js +33 -23
  25. package/dist/intelligence.js.map +1 -1
  26. package/dist/{loop-runner-bin-C4X0FZ2Z.d.ts → loop-runner-bin-CBkLsyFg.d.ts} +2 -2
  27. package/dist/loop-runner-bin.d.ts +4 -4
  28. package/dist/loop-runner-bin.js +7 -7
  29. package/dist/loops.d.ts +265 -14
  30. package/dist/loops.js +28 -6
  31. package/dist/mcp/bin.js +5 -5
  32. package/dist/mcp/index.d.ts +6 -6
  33. package/dist/mcp/index.js +7 -7
  34. package/dist/{router-client-Ak2IGuXq.d.ts → router-client-D6Ocf4jG.d.ts} +19 -0
  35. package/dist/{types-C1sozrte.d.ts → types-By9LXllv.d.ts} +9 -2
  36. package/dist/{worktree-fanout-CXGzHET4.d.ts → worktree-fanout-D9Z2dMS9.d.ts} +2 -2
  37. package/dist/{worktree-harness-Bmho9SH0.d.ts → worktree-harness-Dx8XULW3.d.ts} +1 -1
  38. package/package.json +1 -1
  39. package/skills/build-with-agent-runtime/SKILL.md +53 -23
  40. package/dist/chunk-AG335EXG.js.map +0 -1
  41. package/dist/chunk-KRULXIWS.js.map +0 -1
  42. package/dist/chunk-T2HVQVB4.js.map +0 -1
  43. package/dist/chunk-VMNEQHJR.js.map +0 -1
  44. package/dist/chunk-Z3RRRPRB.js.map +0 -1
  45. /package/dist/{chunk-IODKUOBA.js.map → chunk-C2FZ6GR6.js.map} +0 -0
  46. /package/dist/{chunk-63MHOCIE.js.map → chunk-DH72UOYR.js.map} +0 -0
  47. /package/dist/{chunk-PVPFDTO3.js.map → chunk-LLJRUTE7.js.map} +0 -0
  48. /package/dist/{chunk-PBE35ULD.js.map → chunk-M3Y362RW.js.map} +0 -0
package/dist/loops.d.ts CHANGED
@@ -1,23 +1,23 @@
1
- import { ChatClient, AnalystFinding, AgentProfile, AnalystRunInputs, ToolSpan, StreamingDetector, DetectorSignal, buildTrajectory } from '@tangle-network/agent-eval';
1
+ import { ChatClient, RunRecord, AnalystFinding, AgentProfile, AnalystRunInputs, ToolSpan, StreamingDetector, DetectorSignal, buildTrajectory } from '@tangle-network/agent-eval';
2
2
  export { AnalystFinding, DefaultVerdict, computeFindingId, makeFinding } from '@tangle-network/agent-eval';
3
3
  import { SandboxEvent, SandboxInstance, CreateSandboxOptions } from '@tangle-network/sandbox';
4
4
  export { AgentProfile, CreateSandboxOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox';
5
- import { d as ResultBlobStore, S as SpawnJournal, N as NodeId, j as SpawnEvent, E as ExecutorFactory, c as Agent, B as Budget, i as Scope, g as Settled, f as SupervisedResult, h as Spend, U as UsageEvent, b as ExecutorRegistry, k as Supervisor } from './types-C1sozrte.js';
6
- export { A as AgentSpec, a as Executor, l as ExecutorContext, m as ExecutorResult, n as SupervisorOpts, T as TreeView, W as WidenGate } from './types-C1sozrte.js';
7
- import { ak as MakeWorkerAgent, A as AnalystRegistry, o as CoordinationTools, n as CoordinationEvent, ar as QuestionPolicy, a$ as ExecutorConfig } from './coordination-DCmljYDf.js';
8
- export { b0 as BusEvent, b1 as BusRecord, b2 as BusStats, b3 as EventBus, b4 as ProviderSeam, b5 as PublishOptions, b6 as cliWorktreeExecutor, b7 as createEventBus, b8 as createExecutor, b9 as createExecutorRegistry } from './coordination-DCmljYDf.js';
9
- import { R as RuntimeHooks, I as Iteration, S as SandboxClient, W as Driver, A as AgentRunSpec, b as OutputAdapter, V as Validator, E as ExecCtx, X as LoopWinner, Y as LoopLineageOptions, Z as LoopResult, L as LoopTokenUsage, a as RuntimeStreamEvent, _ as MountRecorder } from './types-BF-MEsQB.js';
5
+ import { d as ResultBlobStore, S as SpawnJournal, N as NodeId, j as SpawnEvent, E as ExecutorFactory, c as Agent, B as Budget, i as Scope, g as Settled, f as SupervisedResult, h as Spend, U as UsageEvent, b as ExecutorRegistry, k as Supervisor } from './types-By9LXllv.js';
6
+ export { A as AgentSpec, a as Executor, l as ExecutorContext, m as ExecutorResult, R as Runtime, n as SupervisorOpts, T as TreeView, W as WidenGate } from './types-By9LXllv.js';
7
+ import { ak as MakeWorkerAgent, A as AnalystRegistry, o as CoordinationTools, n as CoordinationEvent, ar as QuestionPolicy, a$ as ExecutorConfig } from './coordination-BI9tpcmF.js';
8
+ export { b0 as BusEvent, b1 as BusRecord, b2 as BusStats, b3 as EventBus, b4 as ProviderSeam, b5 as PublishOptions, b6 as cliWorktreeExecutor, b7 as createEventBus, b8 as createExecutor, b9 as createExecutorRegistry } from './coordination-BI9tpcmF.js';
9
+ import { R as RuntimeHooks, S as SandboxClient, I as Iteration, W as Driver, A as AgentRunSpec, b as OutputAdapter, V as Validator, E as ExecCtx, X as LoopWinner, Y as LoopLineageOptions, Z as LoopResult, L as LoopTokenUsage, a as RuntimeStreamEvent, _ as MountRecorder } from './types-BF-MEsQB.js';
10
10
  export { $ as LoopDecisionPayload, a0 as LoopEndedPayload, a1 as LoopIterationDispatchPayload, a2 as LoopIterationEndedPayload, a3 as LoopIterationStartedPayload, a4 as LoopPlanDescription, a5 as LoopPlanPayload, d as LoopSandboxPlacement, a6 as LoopStartedPayload, a7 as LoopTeardownFailedPayload, e as LoopTraceEmitter, c as LoopTraceEvent, a8 as MountManifestEntry, a9 as RunProvenance, aa as SelectionReceipt, ab as ValidationCtx } from './types-BF-MEsQB.js';
11
11
  export { AgentEnvironmentProviderRef, AgentEnvironmentProviderRegistry, ProviderAsSandboxClientOptions, ProviderExecutorOptions, SandboxClientProviderOptions, createAgentEnvironmentProviderRegistry, providerAsExecutor, providerAsSandboxClient, resolveAgentEnvironmentProvider, sandboxClientAsProvider } from './environment-provider.js';
12
- import { C as CorpusRecord, c as Corpus, O as Outcome, S as ScopeAnalyzeInput, d as AssertTraceDerivedFindings, e as SteerContext, f as ScopeAnalyst, F as FanoutOptions, g as CombinatorShape, h as ScopeWidenGate, L as LoopUntilSpec, P as PanelSpec, i as PipelineStage, W as WinnerStrategy, j as FanoutWinnerSelector, V as VerifySpec, k as WidenSpec, l as CorpusFilter, R as RenderCorpusToInstructionsOptions, D as DefinePersonaInput, m as Persona, n as RunPersonifiedOptions, o as ShapeRegistry, p as LoopShape, E as EqualKArm, q as EqualKOnCostOptions, r as EqualKVerdict, T as TrajectoryReportOptions, s as TrajectoryReport } from './worktree-fanout-CXGzHET4.js';
13
- export { A as AuthoredHarness, t as DefinePersona, u as EqualKOnCost, v as Fanout, w as FanoutSynthesis, x as FlatWidenGate, y as LoopUntil, z as LoopUntilState, B as Panel, G as PanelJudge, H as PanelVerdict, I as PatchDeliverableOptions, J as PersonaContext, K as PersonaExecutors, M as Pipeline, N as RenderCorpusToInstructions, Q as RunPersonified, U as ShapeBudget, X as ShapeContext, Y as TrajectoryNode, Z as TrajectoryReportFn, _ as Verify, $ as Widen, a0 as WidenDecision, a1 as WidenLineage, a2 as WorktreeCliExecutorOptions, a as WorktreeFanoutOptions, b as WorktreePatchArtifact, a3 as createWorktreeCliExecutor, a4 as patchDelivered, a5 as worktreeFanout } from './worktree-fanout-CXGzHET4.js';
14
- import { Scenario, ProfileDispatchFn } from '@tangle-network/agent-eval/campaign';
12
+ import { C as CorpusRecord, c as Corpus, O as Outcome, S as ScopeAnalyzeInput, d as AssertTraceDerivedFindings, e as SteerContext, f as ScopeAnalyst, F as FanoutOptions, g as CombinatorShape, h as ScopeWidenGate, L as LoopUntilSpec, P as PanelSpec, i as PipelineStage, W as WinnerStrategy, j as FanoutWinnerSelector, V as VerifySpec, k as WidenSpec, l as CorpusFilter, R as RenderCorpusToInstructionsOptions, D as DefinePersonaInput, m as Persona, n as RunPersonifiedOptions, o as ShapeRegistry, p as LoopShape, E as EqualKArm, q as EqualKOnCostOptions, r as EqualKVerdict, T as TrajectoryReportOptions, s as TrajectoryReport } from './worktree-fanout-D9Z2dMS9.js';
13
+ export { A as AuthoredHarness, t as DefinePersona, u as EqualKOnCost, v as Fanout, w as FanoutSynthesis, x as FlatWidenGate, y as LoopUntil, z as LoopUntilState, B as Panel, G as PanelJudge, H as PanelVerdict, I as PatchDeliverableOptions, J as PersonaContext, K as PersonaExecutors, M as Pipeline, N as RenderCorpusToInstructions, Q as RunPersonified, U as ShapeBudget, X as ShapeContext, Y as TrajectoryNode, Z as TrajectoryReportFn, _ as Verify, $ as Widen, a0 as WidenDecision, a1 as WidenLineage, a2 as WorktreeCliExecutorOptions, a as WorktreeFanoutOptions, b as WorktreePatchArtifact, a3 as createWorktreeCliExecutor, a4 as patchDelivered, a5 as worktreeFanout } from './worktree-fanout-D9Z2dMS9.js';
14
+ import { Scenario, DispatchFn, ProfileDispatchFn } from '@tangle-network/agent-eval/campaign';
15
15
  import { b as AnalystRegistryLike } from './types-BC3bZpH0.js';
16
16
  import { AgentProfile as AgentProfile$1 } from '@tangle-network/agent-interface';
17
- import { R as RouterConfig, a as ToolLoopChat, b as ToolLoopCompactionOptions } from './router-client-Ak2IGuXq.js';
18
- export { c as RouterChatResult, d as RouterChatToolsResult, e as RouterToolCall, f as RouterToolLoopResult, g as ToolLoopCompaction, T as ToolSpec, r as routerBrain, h as routerChatWithTools, i as routerChatWithUsage, j as routerToolLoop } from './router-client-Ak2IGuXq.js';
19
- import { D as DeliverableSpec } from './worktree-harness-Bmho9SH0.js';
20
- export { g as WorktreeCommandResult, h as gateOnDeliverable } from './worktree-harness-Bmho9SH0.js';
17
+ import { R as RouterConfig, a as ToolLoopChat, b as ToolLoopCompactionOptions } from './router-client-D6Ocf4jG.js';
18
+ export { c as RouterChatResult, d as RouterChatToolsResult, e as RouterToolCall, f as RouterToolLoopResult, g as ToolLoopCompaction, T as ToolSpec, r as routerBrain, h as routerChatWithTools, i as routerChatWithUsage, j as routerToolLoop } from './router-client-D6Ocf4jG.js';
19
+ import { D as DeliverableSpec } from './worktree-harness-Dx8XULW3.js';
20
+ export { g as WorktreeCommandResult, h as gateOnDeliverable } from './worktree-harness-Dx8XULW3.js';
21
21
  import { stuckLoopView, toolWasteView } from '@tangle-network/agent-eval/pipelines';
22
22
  export { AgentEnvironment, AgentEnvironmentCapabilities, AgentEnvironmentEvent, AgentEnvironmentProvider, AgentEnvironmentQuery, AgentEnvironmentStatus, AgentEnvironmentSummary, AgentProfileRef, AgentSession, AgentSessionRef, AgentSessionStatus, AgentTurnInput, AgentTurnResult, CheckpointRef, CheckpointRequest, CreateAgentEnvironmentInput, ExecRequest, ExecResult, ForkRequest, PlacementInfo, ResourceRequest, WorkspaceRequest } from '@tangle-network/agent-interface/environment-provider';
23
23
  import './substrate-rNj6TDc3.js';
@@ -276,6 +276,156 @@ interface IntentAudit {
276
276
  declare const defaultAuditorInstruction: string;
277
277
  declare function auditIntent(input: AuditIntentInput, opts: AuditIntentOptions): Promise<IntentAudit>;
278
278
 
279
+ /**
280
+ * benchmark-report — turn a fleet of `RunRecord`s into a publishable, multi-axis benchmark report:
281
+ * a ranked leaderboard, the full profile×axis score matrix, the cost/latency/token columns, and
282
+ * embeddable charts (SVG) + a self-contained HTML page. Domain-agnostic by construction — it reads ONLY
283
+ * the universal `RunRecord` currency (model, scenario, score, cost, tokens, latency, the `outcome.raw`
284
+ * metric bag), so the SAME engine reports any benchmark in any domain: coding, search, agents, multimodal.
285
+ *
286
+ * This is the surface a hosted leaderboard (à la vals.ai) renders: for every harness×model profile, its
287
+ * score on every axis, not a curated subset. It pairs with `runProfileMatrix` (whose `result.records`
288
+ * feed straight in) but takes the records directly, so it is independent of how they were produced.
289
+ *
290
+ * An AXIS is any way to slice the score into columns — by default one column per scenario group
291
+ * (`groupOf`, default = the scenario id), so the matrix is profile × scenario. Pass `axisScoresOf` to
292
+ * score along judge dimensions or any custom decomposition instead. The reporter never invents a number:
293
+ * a missing cell renders blank, never zero.
294
+ */
295
+
296
+ /** Pull the headline score in [0,1] from a record. Default: the held-out split, else the search split,
297
+ * else a `composite`/`passed`/`score` entry in the raw bag. Override to score a domain differently. */
298
+ type ScoreOf = (record: RunRecord) => number | undefined;
299
+ /** The profile (matrix row) a record belongs to — default `harness·model` from the record's profile cell,
300
+ * falling back to the model. This is the leaderboard's unit of comparison. */
301
+ type ProfileKeyOf = (record: RunRecord) => string;
302
+ /** The axis (matrix column) a record contributes to — default the scenario group. */
303
+ type GroupOf = (record: RunRecord) => string;
304
+ /** Decompose ONE record into per-axis scores (e.g. judge dimensions). When set, it REPLACES the
305
+ * scenario-group axes: the column set is the union of returned keys. */
306
+ type AxisScoresOf = (record: RunRecord) => Record<string, number>;
307
+ interface LeaderboardOptions {
308
+ readonly title?: string;
309
+ readonly scoreOf?: ScoreOf;
310
+ readonly profileKeyOf?: ProfileKeyOf;
311
+ readonly groupOf?: GroupOf;
312
+ readonly axisScoresOf?: AxisScoresOf;
313
+ /** Display label for a profile key (default: the key itself). */
314
+ readonly labelOf?: (profileKey: string) => string;
315
+ /** Commit SHA / dataset / dates surfaced in the provenance block. */
316
+ readonly meta?: Record<string, string>;
317
+ /** Compute per-row confidence intervals (bootstrap on score, Wilson on pass rate). Needs a
318
+ * `scenarioId` on every record (reps are collapsed per scenario for the honest n). Default off. */
319
+ readonly stats?: boolean;
320
+ /** A score ≥ this counts as a "pass" for the pass-rate proportion + its Wilson CI. Default 0.999
321
+ * (fully solved). Lower it (e.g. 0.6) for a partial-credit domain. */
322
+ readonly passThreshold?: number;
323
+ }
324
+ /** A 95%-by-default confidence interval. */
325
+ interface Interval {
326
+ readonly lower: number;
327
+ readonly upper: number;
328
+ }
329
+ /** One leaderboard row — a harness×model profile, every measured column. */
330
+ interface LeaderboardRow {
331
+ readonly profileKey: string;
332
+ readonly label: string;
333
+ readonly model: string;
334
+ readonly n: number;
335
+ readonly meanScore: number;
336
+ /** Fraction of records scoring ≥ `passThreshold` (default 0.999) — the binary pass rate. */
337
+ readonly solveRate: number;
338
+ /** axis → mean score for this profile (blank in render when the profile never ran that axis). */
339
+ readonly perAxis: Record<string, number>;
340
+ readonly costUsd: number;
341
+ readonly tokensIn: number;
342
+ readonly tokensOut: number;
343
+ readonly latencyP50Ms: number;
344
+ readonly latencyP90Ms: number;
345
+ /** Bootstrap CI on the mean score — present only when `opts.stats` is set. Computed over
346
+ * per-scenario means (reps collapsed first), so identical reps can't fake a narrow interval. */
347
+ readonly scoreCi?: Interval;
348
+ /** Wilson CI on the pass rate — present only when `opts.stats` is set. */
349
+ readonly passCi?: Interval;
350
+ }
351
+ interface Leaderboard {
352
+ readonly title: string;
353
+ /** Column order — scenario groups (default) or dimension keys (`axisScoresOf`). */
354
+ readonly axes: readonly string[];
355
+ /** Rows ranked by `meanScore` desc (ties → lower cost, then label). */
356
+ readonly profiles: readonly LeaderboardRow[];
357
+ readonly meta: Record<string, string>;
358
+ /** Provenance counts — the denominators every honest report leads with. */
359
+ readonly provenance: {
360
+ readonly records: number;
361
+ readonly profiles: number;
362
+ readonly axes: number;
363
+ readonly models: readonly string[];
364
+ readonly totalCostUsd: number;
365
+ };
366
+ }
367
+ /** Aggregate a fleet of records into the ranked, multi-axis report. Pure — no IO, deterministic. */
368
+ declare function leaderboard(records: readonly RunRecord[], opts?: LeaderboardOptions): Leaderboard;
369
+ /** One profile pair compared on the scenarios they BOTH ran — the "who actually beat whom" verdict. */
370
+ interface PairwiseVerdict {
371
+ readonly a: string;
372
+ readonly b: string;
373
+ /** Paired unit count (shared scenarios). The significance is suppressed below `minPairs`. */
374
+ readonly pairs: number;
375
+ /** Median paired delta (b − a) and its bootstrap CI. */
376
+ readonly delta: number;
377
+ readonly ciLow: number;
378
+ readonly ciHigh: number;
379
+ /** Paired-test p-value (before correction). */
380
+ readonly p: number;
381
+ /** BH-significant across ALL pairs AND above the `minPairs` power floor. */
382
+ readonly significant: boolean;
383
+ }
384
+ interface PairwiseOptions {
385
+ readonly scoreOf?: ScoreOf;
386
+ readonly profileKeyOf?: ProfileKeyOf;
387
+ readonly labelOf?: (profileKey: string) => string;
388
+ /** False-discovery rate for the Benjamini–Hochberg correction. Default 0.05. */
389
+ readonly fdr?: number;
390
+ /** Below this many shared scenarios a paired test can't defensibly separate two profiles, so the
391
+ * `significant` tag is suppressed regardless of p (small-n mirage protection). Default 12. */
392
+ readonly minPairs?: number;
393
+ }
394
+ /** Compare EVERY profile pair on the scenarios they both ran — paired-bootstrap effect + CI, a real
395
+ * paired-test p-value, BH-corrected across all pairs. This is the honest "did A beat B" table the
396
+ * leaderboard's point ranking cannot answer. Reuses the agent-eval statistics substrate. */
397
+ declare function pairwiseSignificance(records: readonly RunRecord[], opts?: PairwiseOptions): PairwiseVerdict[];
398
+ /** Render the report as a publishable Markdown document: provenance → leaderboard → the full profile×axis
399
+ * matrix → cost/latency/token columns. Every axis is shown — a curated subset is a reporting failure. */
400
+ declare function renderLeaderboardMarkdown(report: Leaderboard): string;
401
+ /** Render the pairwise-significance table — every profile pair's paired delta, CI, and BH-corrected
402
+ * verdict. Feed it `pairwiseSignificance(records)`. This is the "did A really beat B" evidence the point
403
+ * ranking cannot give. */
404
+ declare function renderPairwiseMarkdown(verdicts: readonly PairwiseVerdict[], title?: string): string;
405
+ /** Render a self-contained SVG: a ranked score bar chart on top, the profile×axis heatmap below. No deps,
406
+ * embeddable anywhere (README, HTML page, hosted leaderboard). */
407
+ declare function renderLeaderboardSvg(report: Leaderboard): string;
408
+ /** Render a self-contained HTML leaderboard page (the hosted surface): the SVG charts + the full Markdown
409
+ * matrix as a table. Single file, no assets, opens in any browser. */
410
+ declare function renderLeaderboardHtml(report: Leaderboard): string;
411
+
412
+ interface CliBridgeConfig {
413
+ /** cli-bridge base URL. Defaults to `http://127.0.0.1:3355`. */
414
+ url?: string;
415
+ /** Bearer token the bridge requires (fail loud when empty). */
416
+ bearer: string;
417
+ /** Default harness when a `create({ backend })` override does not set one. */
418
+ harness?: string;
419
+ /** Default model when a `create({ backend })` override does not set one. */
420
+ model?: string;
421
+ }
422
+ /**
423
+ * A `SandboxClient` whose every prompt is one cli-bridge chat completion, metered
424
+ * and driven by the shared `runLoop` kernel. Pass a fixed `harness`/`model`, or omit
425
+ * them and set the target per cell via `create({ backend: { type, model: { model } } })`.
426
+ */
427
+ declare function cliBridgeSandboxClient(cfg: CliBridgeConfig): SandboxClient;
428
+
279
429
  /**
280
430
  * @experimental
281
431
  *
@@ -679,6 +829,25 @@ interface LoopDispatchOptions<Task, Output, Decision, TScenario extends Scenario
679
829
  /** Cost-meter source label for the loop's spend. Default `'loop'`. */
680
830
  costSource?: string;
681
831
  }
832
+ /** Options for adapting plain agent-eval campaign scenarios into runtime `runLoop` cells. */
833
+ interface LoopCampaignDispatchOptions<Task, Output, Decision, TScenario extends Scenario, TArtifact> {
834
+ /** Sandbox client used for every campaign cell's `runLoop`. */
835
+ sandboxClient: SandboxClient;
836
+ /** Build the per-cell runLoop options from the campaign scenario. */
837
+ toLoopOptions: (scenario: TScenario) => LoopOptionsForDispatch<Task, Output, Decision>;
838
+ /** Map the finished loop to the artifact the campaign judges score. */
839
+ toArtifact?: (result: LoopResult<Task, Output, Decision>) => TArtifact;
840
+ /** Forward `loop.*` trace events into the campaign's scoped trace. Default true. */
841
+ forwardTrace?: boolean;
842
+ /** Cost-meter source label for the loop's spend. Default `'loop'`. */
843
+ costSource?: string;
844
+ }
845
+ /**
846
+ * Adapter for plain `runCampaign` scenarios. This is the runtime-side pair for
847
+ * agent-eval fixture scenarios: load fixtures in `agent-eval/campaign`, build
848
+ * the runtime loop here, and keep cost + token + trace reporting automatic.
849
+ */
850
+ declare function loopCampaignDispatch<Task, Output, Decision, TScenario extends Scenario, TArtifact>(opts: LoopCampaignDispatchOptions<Task, Output, Decision, TScenario, TArtifact>): DispatchFn<TScenario, TArtifact>;
682
851
  /**
683
852
  * Adapter for `runProfileMatrix` (profile is an axis). Returns a
684
853
  * `ProfileDispatchFn` that runs `runLoop` per (profile, scenario) cell and
@@ -1650,6 +1819,24 @@ interface CriuCapableClient {
1650
1819
  declare function extractLlmCallEvent(event: SandboxEvent, agentRunName: string): (RuntimeStreamEvent & {
1651
1820
  type: 'llm_call';
1652
1821
  }) | undefined;
1822
+ /**
1823
+ * Sum the token usage + USD cost of a sandbox turn's events — the one honest way to meter an
1824
+ * `openSandboxRun` cell. Folds `extractLlmCallEvent` over the stream (which reads usage off EVERY backend
1825
+ * event shape), so a `runProfileMatrix` dispatch can report it to `ctx.cost`:
1826
+ *
1827
+ * const turn = await run.start(prompt)
1828
+ * const u = sumSandboxUsage(turn.events)
1829
+ * if (u.input || u.output) ctx.cost.observeTokens({ input: u.input, output: u.output })
1830
+ * if (u.costUsd) ctx.cost.observe(u.costUsd, 'sandbox-cell')
1831
+ *
1832
+ * Without this a cell reads `{tokens:0, cost:0}` and the backend-integrity guard correctly aborts the
1833
+ * matrix as a stub. `agentRunName` is the fallback model label for cost-only events (default `'agent'`).
1834
+ */
1835
+ declare function sumSandboxUsage(events: readonly SandboxEvent[], agentRunName?: string): {
1836
+ input: number;
1837
+ output: number;
1838
+ costUsd: number;
1839
+ };
1653
1840
  /**
1654
1841
  * Project one `SandboxEvent` onto the `RuntimeStreamEvent` chat-UX vocabulary,
1655
1842
  * for runtimes that bridge a sandbox `streamPrompt` into the
@@ -2645,6 +2832,9 @@ interface CoordinationMcpHandle {
2645
2832
  valid?: boolean;
2646
2833
  outRef?: string;
2647
2834
  }>;
2835
+ /** Post-loop drain of already-settled, unpulled children into the ledger — call before reading
2836
+ * `settled()` for a finalize, so a delivered child the harness never awaited is not lost. */
2837
+ drainResolved: CoordinationTools['drainResolved'];
2648
2838
  isStopped(): boolean;
2649
2839
  /** The full ordered bus-event log — observability audit + replay trail. */
2650
2840
  history: CoordinationTools['history'];
@@ -3190,6 +3380,67 @@ interface TrajectoryAnalysis {
3190
3380
  /** Collect the source's spans and run the agent-eval batch analyzers over them under one `runId`. */
3191
3381
  declare function analyzeTrace(source: TraceSource, runId?: string): Promise<TrajectoryAnalysis>;
3192
3382
 
3383
+ /** What a surface worker settles with — the surface verdict the driver + deliverable read. `resolved` is
3384
+ * the surface check's pass/fail (settled ⟺ resolved); `score` is the partial-credit fraction; `failing`
3385
+ * carries the tests this worker left red (so the analyst can target them). */
3386
+ interface SurfaceWorkerOut {
3387
+ readonly resolved: boolean;
3388
+ readonly score: number;
3389
+ readonly shots: number;
3390
+ readonly summary: string;
3391
+ readonly failing?: readonly string[];
3392
+ }
3393
+ /** The default self-improvement LENS — authored content, not a code path. On each settled worker it hands
3394
+ * the driver the still-FAILING tests (not just a score), so the next spawn targets the persistently-hard
3395
+ * cases. Swap `analysts` to change what the driver improves from — that's the one knob. */
3396
+ declare function failuresAnalyst(): AnalystRegistry;
3397
+ /** How a worker runs the surface task (its router substrate + per-attempt bounds). */
3398
+ interface SurfaceWorkerConfig {
3399
+ readonly routerBaseUrl: string;
3400
+ readonly routerKey: string;
3401
+ readonly model: string;
3402
+ readonly maxTokens?: number;
3403
+ readonly innerTurns?: number;
3404
+ /** Refine-shot budget for ONE worker attempt (max steered shots). Default 1. */
3405
+ readonly budget?: number;
3406
+ }
3407
+ interface SuperviseSurfaceOptions {
3408
+ /** The graded surface workers solve (open/tools/call/score/close). */
3409
+ readonly surface: AgenticSurface;
3410
+ /** Where/how each worker runs the surface task. */
3411
+ readonly worker: SurfaceWorkerConfig;
3412
+ /** The conserved compute pool for the whole supervised run. Default: sized off the worker's inner-loop
3413
+ * bounds for a handful of worker spawns — raise it to let the driver try more. */
3414
+ readonly budget?: Budget;
3415
+ /** The driver brain's router substrate (its own inference). Default: the worker's router + model — the
3416
+ * driver and workers share one router unless you separate them (e.g. a stronger driver model). */
3417
+ readonly router?: RouterConfig;
3418
+ /** The self-improvement lens fed to the driver on each settled worker. Default `failuresAnalyst()`
3419
+ * (target the still-failing tests). Pass a custom registry to change it, or `null` to turn the
3420
+ * within-run self-improvement OFF (the driver sees raw settled outputs). */
3421
+ readonly analysts?: AnalystRegistry | null;
3422
+ /** The strategy each worker runs over the surface. Default `refine` (iterate-with-feedback). */
3423
+ readonly strategy?: Strategy;
3424
+ /** Max workers live at once. Default 1 (serial — required when workers share a persistent artifact, so
3425
+ * they continue each other instead of racing the file). */
3426
+ readonly maxLiveWorkers?: number;
3427
+ }
3428
+ /** The deployable outcome of a supervised surface run. */
3429
+ interface SuperviseSurfaceResult {
3430
+ readonly resolved: boolean;
3431
+ readonly score: number;
3432
+ readonly usd: number;
3433
+ readonly tokensIn: number;
3434
+ readonly tokensOut: number;
3435
+ readonly ms: number;
3436
+ /** Total conserved-pool iterations = the driver + worker LLM rounds the run actually spent. */
3437
+ readonly completions: number;
3438
+ }
3439
+ /** Drive a team of agents (spawned + steered by `profile`) to solve a graded `AgenticSurface` task, and
3440
+ * report the deployable outcome + the full conserved spend. This is `supervise()` configured for surfaces
3441
+ * — there is no other entrypoint to learn. */
3442
+ declare function superviseSurface(profile: SupervisorProfile, task: AgenticTask, opts: SuperviseSurfaceOptions): Promise<SuperviseSurfaceResult>;
3443
+
3193
3444
  /**
3194
3445
  * createVerifierEnvironment — ANY checkable task as an `Environment`, no tool surface
3195
3446
  * required. The generalization piece: EOPS/commit0-style domains have tools that mutate
@@ -3285,4 +3536,4 @@ declare function runInWorkspace<T>(ws: Workspace, body: (cwd: string) => Promise
3285
3536
  commitOnInvalid?: boolean;
3286
3537
  }): Promise<WorkspaceRun<T>>;
3287
3538
 
3288
- export { Agent, AgentRunSpec, type AgenticOptions, type AgenticRunResult, type AgenticSurface, type AgenticTask, type AgenticTool, AnalystRegistry, type AnytimeReport, type AnytimeStrategySummary, type AnytimeTaskCurve, type ApplyContinuation, type ArtifactHandle, AssertTraceDerivedFindings, type AuditIntentInput, type AuditIntentOptions, type AuthorStrategyOptions, type AuthoredProfile, type AuthoredStrategy, type BenchmarkCell, type BenchmarkConfig, type BenchmarkLift, type BenchmarkReport, type BenchmarkStrategySummary, type BenchmarkTaskRow, Budget, type BudgetPool, type BudgetReadout, type ChampionPick, type ChampionPolicy, type CheckpointCapableBox, CombinatorShape, type CompletionAnalyst, type CompletionEvidence, type CompletionPolicy, type CompletionVerdict, CoordinationEvent, type CoordinationMcpHandle, Corpus, CorpusFilter, CorpusRecord, type CreateScopeAnalystOptions, type CriuCapableClient, DefinePersonaInput, type DelegateOptions, type Deliverable, DeliverableSpec, type DriveHarness, Driver, type DriverAgentOptions, type DumbDriverOptions, type Environment, EqualKArm, EqualKOnCostOptions, EqualKVerdict, type EvolutionArchiveNode, type EvolutionAuthor, type EvolutionBandInfo, type EvolutionCandidate, type EvolutionGeneration, type EvolutionReport, ExecCtx, ExecutorConfig, FanoutOptions, FanoutWinnerSelector, FileCorpus, type ForkCapableBox, type GitWorkspaceOptions, type HarvestCorpusOptions, type HarvestFailure, type HarvestReport, InMemoryCorpus, InMemoryResultBlobStore, type InMemoryRunContext, type InMemoryRunContextOptions, InMemorySpawnJournal, type InProcessOnPrompt, type InProcessPromptCtx, type InProcessSandboxClientOptions, type Inbox, type InboxMessage, type IntentAudit, Iteration, type LoopDispatchOptions, LoopLineageOptions, type LoopOptionsForDispatch, LoopResult, LoopShape, LoopTokenUsage, LoopUntilSpec, LoopWinner, MakeWorkerAgent, type McpEndpoint, type McpEnvironmentOptions, MountRecorder, type NaiveDriverOptions, type Observation, type ObserveInput, type ObserveOptions, type OpenSandboxRunOptions, Outcome, OutputAdapter, PanelSpec, Persona, PipelineStage, type ProfileRichness, type ProfileRichnessThresholds, type PromotionGateOptions, type PromotionVerdict, type RegistryAnalyzeProjection, RenderCorpusToInstructionsOptions, type ReservationTicket, ResultBlobStore, RouterConfig, type RunAgenticOptions, RunPersonifiedOptions, type SandboxCapabilities, SandboxClient, type SandboxLineage, type SandboxLineageHandle, type SandboxRun, SandboxRunAbortError, Scope, ScopeAnalyst, ScopeAnalyzeInput, ScopeWidenGate, type SessionCapableBox, type SessionTraceBox, Settled, ShapeRegistry, type Shell, type ShotPersona, type ShotSpec, Spend, SteerContext, type SteeringDecision, type Strategy, type StrategyCtx, type StrategyEvolutionConfig, type StrategyResult, type SuperviseOptions, SupervisedResult, Supervisor, type SupervisorAgentDeps, type SupervisorProfile, type SurfaceScore, ToolLoopChat, ToolLoopCompactionOptions, type TraceSource, type TrajectoryAnalysis, TrajectoryReport, TrajectoryReportOptions, type TurnResult, UsageEvent, type UsageSink, Validator, type VerifierEnvironmentOptions, VerifySpec, type WatchTraceOptions, type WaterfallCollector, type WaterfallReport, type WaterfallSpan, WidenSpec, WinnerStrategy, type Workspace, type WorkspaceCommit, type WorkspaceRun, acquireSandbox, adaptiveRefine, analyzeTrace, anytimeReport, asAuthoredProfile, assertModelAllowed, assertStrategyContract, assertTraceDerivedFindings, assessAuthoredProfile, auditIntent, authorStrategy, authoredWorker, breadthStrategy, buildSteerContext, builtinShapes, completionAuthorizes, contentAddress, createBudgetPool, createInMemoryRunContext, createInbox, createMcpEnvironment, createPushTraceSource, createSandboxLineage, createScope, createScopeAnalyst, createShapeRegistry, createSupervisor, createVerifierEnvironment, createWaterfallCollector, decodeToolPart, defaultAnalystInstruction, defaultAuditorInstruction, defaultDelegateBudget, defaultProfileRichnessThresholds, defaultSelectWinner, defaultToolDetectors, definePersona, defineStrategy, delegate, depthStrategy, deterministicCompletion, discriminatingMeans, driverAgent, dumbDriver, equalKOnCost, extractLlmCallEvent, fanout, finalizeBestDelivered, flatWidenGate, gitWorkspace, harvestCorpus, inProcessSandboxClient, inlineSandboxClient, jjWorkspace, localShell, loopDispatch, loopUntil, mapSandboxEvent, naiveDriver, observe, openSandboxRun, panel, pickChampion, pipeline, printBenchmarkReport, probeSandboxCapabilities, profileRichnessFinding, promotionGate, refine, registerShape, registryScopeAnalyst, renderAnytimeTable, renderCorpusToInstructions, renderReport, reportLoopUsage, runAgentic, runBenchmark, runInWorkspace, runLoop, runPersonified, runStrategyEvolution, sample, sampleThenRefine, sandboxSessionTraceSource, selectChampion, selectValidWinner, sentinelCompletion, serveCoordinationMcp, settledToIteration, spendFromUsageEvents, stopSentinel, strategyAuthorContract, supervise, supervisorAgent, supervisorInstructions, trajectoryReport, verify, watchTrace, widen, workerFromBackend };
3539
+ export { Agent, AgentRunSpec, type AgenticOptions, type AgenticRunResult, type AgenticSurface, type AgenticTask, type AgenticTool, AnalystRegistry, type AnytimeReport, type AnytimeStrategySummary, type AnytimeTaskCurve, type ApplyContinuation, type ArtifactHandle, AssertTraceDerivedFindings, type AuditIntentInput, type AuditIntentOptions, type AuthorStrategyOptions, type AuthoredProfile, type AuthoredStrategy, type BenchmarkCell, type BenchmarkConfig, type BenchmarkLift, type BenchmarkReport, type BenchmarkStrategySummary, type BenchmarkTaskRow, Budget, type BudgetPool, type BudgetReadout, type ChampionPick, type ChampionPolicy, type CheckpointCapableBox, type CliBridgeConfig, CombinatorShape, type CompletionAnalyst, type CompletionEvidence, type CompletionPolicy, type CompletionVerdict, CoordinationEvent, type CoordinationMcpHandle, Corpus, CorpusFilter, CorpusRecord, type CreateScopeAnalystOptions, type CriuCapableClient, DefinePersonaInput, type DelegateOptions, type Deliverable, DeliverableSpec, type DriveHarness, Driver, type DriverAgentOptions, type DumbDriverOptions, type Environment, EqualKArm, EqualKOnCostOptions, EqualKVerdict, type EvolutionArchiveNode, type EvolutionAuthor, type EvolutionBandInfo, type EvolutionCandidate, type EvolutionGeneration, type EvolutionReport, ExecCtx, ExecutorConfig, ExecutorFactory, ExecutorRegistry, FanoutOptions, FanoutWinnerSelector, FileCorpus, type ForkCapableBox, type GitWorkspaceOptions, type HarvestCorpusOptions, type HarvestFailure, type HarvestReport, InMemoryCorpus, InMemoryResultBlobStore, type InMemoryRunContext, type InMemoryRunContextOptions, InMemorySpawnJournal, type InProcessOnPrompt, type InProcessPromptCtx, type InProcessSandboxClientOptions, type Inbox, type InboxMessage, type IntentAudit, type Interval, Iteration, type Leaderboard, type LeaderboardOptions, type LeaderboardRow, type LoopCampaignDispatchOptions, type LoopDispatchOptions, LoopLineageOptions, type LoopOptionsForDispatch, LoopResult, LoopShape, LoopTokenUsage, LoopUntilSpec, LoopWinner, MakeWorkerAgent, type McpEndpoint, type McpEnvironmentOptions, MountRecorder, type NaiveDriverOptions, type Observation, type ObserveInput, type ObserveOptions, type OpenSandboxRunOptions, Outcome, OutputAdapter, type PairwiseOptions, type PairwiseVerdict, PanelSpec, Persona, PipelineStage, type ProfileRichness, type ProfileRichnessThresholds, type PromotionGateOptions, type PromotionVerdict, type RegistryAnalyzeProjection, RenderCorpusToInstructionsOptions, type ReservationTicket, ResultBlobStore, RouterConfig, type RunAgenticOptions, RunPersonifiedOptions, type SandboxCapabilities, SandboxClient, type SandboxLineage, type SandboxLineageHandle, type SandboxRun, SandboxRunAbortError, Scope, ScopeAnalyst, ScopeAnalyzeInput, ScopeWidenGate, type SessionCapableBox, type SessionTraceBox, Settled, ShapeRegistry, type Shell, type ShotPersona, type ShotSpec, Spend, SteerContext, type SteeringDecision, type Strategy, type StrategyCtx, type StrategyEvolutionConfig, type StrategyResult, type SuperviseOptions, type SuperviseSurfaceOptions, type SuperviseSurfaceResult, SupervisedResult, Supervisor, type SupervisorAgentDeps, type SupervisorProfile, type SurfaceScore, type SurfaceWorkerConfig, type SurfaceWorkerOut, ToolLoopChat, ToolLoopCompactionOptions, type TraceSource, type TrajectoryAnalysis, TrajectoryReport, TrajectoryReportOptions, type TurnResult, UsageEvent, type UsageSink, Validator, type VerifierEnvironmentOptions, VerifySpec, type WatchTraceOptions, type WaterfallCollector, type WaterfallReport, type WaterfallSpan, WidenSpec, WinnerStrategy, type Workspace, type WorkspaceCommit, type WorkspaceRun, acquireSandbox, adaptiveRefine, analyzeTrace, anytimeReport, asAuthoredProfile, assertModelAllowed, assertStrategyContract, assertTraceDerivedFindings, assessAuthoredProfile, auditIntent, authorStrategy, authoredWorker, breadthStrategy, buildSteerContext, builtinShapes, cliBridgeSandboxClient, completionAuthorizes, contentAddress, createBudgetPool, createInMemoryRunContext, createInbox, createMcpEnvironment, createPushTraceSource, createSandboxLineage, createScope, createScopeAnalyst, createShapeRegistry, createSupervisor, createVerifierEnvironment, createWaterfallCollector, decodeToolPart, defaultAnalystInstruction, defaultAuditorInstruction, defaultDelegateBudget, defaultProfileRichnessThresholds, defaultSelectWinner, defaultToolDetectors, definePersona, defineStrategy, delegate, depthStrategy, deterministicCompletion, discriminatingMeans, driverAgent, dumbDriver, equalKOnCost, extractLlmCallEvent, failuresAnalyst, fanout, finalizeBestDelivered, flatWidenGate, gitWorkspace, harvestCorpus, inProcessSandboxClient, inlineSandboxClient, jjWorkspace, leaderboard, localShell, loopCampaignDispatch, loopDispatch, loopUntil, mapSandboxEvent, naiveDriver, observe, openSandboxRun, pairwiseSignificance, panel, pickChampion, pipeline, printBenchmarkReport, probeSandboxCapabilities, profileRichnessFinding, promotionGate, refine, registerShape, registryScopeAnalyst, renderAnytimeTable, renderCorpusToInstructions, renderLeaderboardHtml, renderLeaderboardMarkdown, renderLeaderboardSvg, renderPairwiseMarkdown, renderReport, reportLoopUsage, runAgentic, runBenchmark, runInWorkspace, runLoop, runPersonified, runStrategyEvolution, sample, sampleThenRefine, sandboxSessionTraceSource, selectChampion, selectValidWinner, sentinelCompletion, serveCoordinationMcp, settledToIteration, spendFromUsageEvents, stopSentinel, strategyAuthorContract, sumSandboxUsage, supervise, superviseSurface, supervisorAgent, supervisorInstructions, trajectoryReport, verify, watchTrace, widen, workerFromBackend };
package/dist/loops.js CHANGED
@@ -12,6 +12,7 @@ import {
12
12
  breadthStrategy,
13
13
  buildSteerContext,
14
14
  builtinShapes,
15
+ cliBridgeSandboxClient,
15
16
  completionAuthorizes,
16
17
  computeFindingId,
17
18
  createMcpEnvironment,
@@ -31,6 +32,7 @@ import {
31
32
  discriminatingMeans,
32
33
  dumbDriver,
33
34
  equalKOnCost,
35
+ failuresAnalyst,
34
36
  fanout,
35
37
  flatWidenGate,
36
38
  gitWorkspace,
@@ -38,13 +40,16 @@ import {
38
40
  inProcessSandboxClient,
39
41
  inlineSandboxClient,
40
42
  jjWorkspace,
43
+ leaderboard,
41
44
  localShell,
45
+ loopCampaignDispatch,
42
46
  loopDispatch,
43
47
  loopUntil,
44
48
  makeFinding,
45
49
  naiveDriver,
46
50
  observe,
47
51
  openSandboxRun,
52
+ pairwiseSignificance,
48
53
  panel,
49
54
  patchDelivered,
50
55
  pickChampion,
@@ -56,6 +61,10 @@ import {
56
61
  registryScopeAnalyst,
57
62
  renderAnytimeTable,
58
63
  renderCorpusToInstructions,
64
+ renderLeaderboardHtml,
65
+ renderLeaderboardMarkdown,
66
+ renderLeaderboardSvg,
67
+ renderPairwiseMarkdown,
59
68
  renderReport,
60
69
  reportLoopUsage,
61
70
  runAgentic,
@@ -71,12 +80,13 @@ import {
71
80
  sentinelCompletion,
72
81
  stopSentinel,
73
82
  strategyAuthorContract,
83
+ superviseSurface,
74
84
  trajectoryReport,
75
85
  verify,
76
86
  watchTrace,
77
87
  widen,
78
88
  worktreeFanout
79
- } from "./chunk-AG335EXG.js";
89
+ } from "./chunk-QSO2TVDS.js";
80
90
  import {
81
91
  InMemoryResultBlobStore,
82
92
  InMemorySpawnJournal,
@@ -118,19 +128,20 @@ import {
118
128
  supervisorAgent,
119
129
  supervisorInstructions,
120
130
  workerFromBackend
121
- } from "./chunk-KRULXIWS.js";
122
- import "./chunk-VMNEQHJR.js";
131
+ } from "./chunk-TPII5AU7.js";
132
+ import "./chunk-3X53HYRW.js";
123
133
  import {
124
134
  createAgentEnvironmentProviderRegistry,
125
135
  providerAsExecutor,
126
136
  providerAsSandboxClient,
127
137
  resolveAgentEnvironmentProvider,
128
138
  sandboxClientAsProvider
129
- } from "./chunk-Z3RRRPRB.js";
139
+ } from "./chunk-H7IBHAFT.js";
130
140
  import {
131
141
  extractLlmCallEvent,
132
- mapSandboxEvent
133
- } from "./chunk-T2HVQVB4.js";
142
+ mapSandboxEvent,
143
+ sumSandboxUsage
144
+ } from "./chunk-4J6RBI3K.js";
134
145
  import "./chunk-5JAUQZQA.js";
135
146
  import "./chunk-NBV35BR6.js";
136
147
  import "./chunk-WIR4HOOJ.js";
@@ -156,6 +167,7 @@ export {
156
167
  breadthStrategy,
157
168
  buildSteerContext,
158
169
  builtinShapes,
170
+ cliBridgeSandboxClient,
159
171
  cliWorktreeExecutor,
160
172
  completionAuthorizes,
161
173
  computeFindingId,
@@ -194,6 +206,7 @@ export {
194
206
  dumbDriver,
195
207
  equalKOnCost,
196
208
  extractLlmCallEvent,
209
+ failuresAnalyst,
197
210
  fanout,
198
211
  finalizeBestDelivered,
199
212
  flatWidenGate,
@@ -203,7 +216,9 @@ export {
203
216
  inProcessSandboxClient,
204
217
  inlineSandboxClient,
205
218
  jjWorkspace,
219
+ leaderboard,
206
220
  localShell,
221
+ loopCampaignDispatch,
207
222
  loopDispatch,
208
223
  loopUntil,
209
224
  makeFinding,
@@ -211,6 +226,7 @@ export {
211
226
  naiveDriver,
212
227
  observe,
213
228
  openSandboxRun,
229
+ pairwiseSignificance,
214
230
  panel,
215
231
  patchDelivered,
216
232
  pickChampion,
@@ -226,6 +242,10 @@ export {
226
242
  registryScopeAnalyst,
227
243
  renderAnytimeTable,
228
244
  renderCorpusToInstructions,
245
+ renderLeaderboardHtml,
246
+ renderLeaderboardMarkdown,
247
+ renderLeaderboardSvg,
248
+ renderPairwiseMarkdown,
229
249
  renderReport,
230
250
  reportLoopUsage,
231
251
  resolveAgentEnvironmentProvider,
@@ -251,7 +271,9 @@ export {
251
271
  spendFromUsageEvents,
252
272
  stopSentinel,
253
273
  strategyAuthorContract,
274
+ sumSandboxUsage,
254
275
  supervise,
276
+ superviseSurface,
255
277
  supervisorAgent,
256
278
  supervisorInstructions,
257
279
  trajectoryReport,
package/dist/mcp/bin.js CHANGED
@@ -1,7 +1,7 @@
1
1
  #!/usr/bin/env node
2
2
  import {
3
3
  readTraceContextFromEnv
4
- } from "../chunk-PBE35ULD.js";
4
+ } from "../chunk-M3Y362RW.js";
5
5
  import {
6
6
  resolveRouterBaseUrl
7
7
  } from "../chunk-GRAGM4MC.js";
@@ -9,10 +9,10 @@ import {
9
9
  DelegationTaskQueue,
10
10
  FileDelegationStore,
11
11
  createMcpServer
12
- } from "../chunk-KRULXIWS.js";
13
- import "../chunk-VMNEQHJR.js";
14
- import "../chunk-Z3RRRPRB.js";
15
- import "../chunk-T2HVQVB4.js";
12
+ } from "../chunk-TPII5AU7.js";
13
+ import "../chunk-3X53HYRW.js";
14
+ import "../chunk-H7IBHAFT.js";
15
+ import "../chunk-4J6RBI3K.js";
16
16
  import "../chunk-5JAUQZQA.js";
17
17
  import "../chunk-NBV35BR6.js";
18
18
  import "../chunk-WIR4HOOJ.js";
@@ -1,19 +1,19 @@
1
1
  import { S as SandboxClient, d as LoopSandboxPlacement } from '../types-BF-MEsQB.js';
2
- import { F as FleetHandle, D as DelegationExecutor, a as DelegationTaskQueue, b as FeedbackStore, c as DelegateFeedbackResult, d as DelegateFeedbackArgs, U as UiAuditorDelegate, e as DelegateUiAuditArgs, f as DelegateUiAuditResult, g as DelegationHistoryResult, h as DelegationHistoryArgs, i as DelegationStatusResult, j as DelegationStatusArgs } from '../coordination-DCmljYDf.js';
3
- export { A as AnalystRegistry, C as CappedDelegationTrace, k as CoderDelegate, l as CoderReview, m as CoderReviewer, n as CoordinationEvent, o as CoordinationTools, p as CoordinationToolsOptions, q as DELEGATE_DESCRIPTION, r as DELEGATE_INPUT_SCHEMA, s as DELEGATE_TOOL_NAME, t as DELEGATION_TRACE_MAX_BYTES, u as DELEGATION_TRACE_MAX_SPANS, v as DelegateArgs, w as DelegateCodeArgs, x as DelegateCodeConfig, y as DelegateCodeResult, z as DelegateHandlerOptions, B as DelegateResearchArgs, E as DelegateResearchConfig, G as DelegateResearchResult, H as DelegateResult, I as DelegateRunCtx, J as DelegateUiAuditConfig, K as DelegateUiAuditRoute, L as DelegationError, M as DelegationFeedbackSnapshot, N as DelegationHistoryEntry, O as DelegationPersistenceError, P as DelegationProfile, Q as DelegationProgress, R as DelegationRecord, S as DelegationResultPayload, T as DelegationResumeContext, V as DelegationResumeDriver, W as DelegationResumeTick, X as DelegationRunContext, Y as DelegationStateCorruptError, Z as DelegationStatus, _ as DelegationStore, $ as DelegationTaskQueueOptions, a0 as DelegationTraceCaps, a1 as DelegationTraceCollector, a2 as DelegationTraceSpan, a3 as DetachedSessionDelegateOptions, a4 as DetachedSessionRefParts, a5 as DetachedTurn, a6 as DetachedTurnResumeDriverOptions, a7 as DetachedWinnerSelection, a8 as DriveTurnCapableBox, a9 as DriveTurnTick, aa as FeedbackEvent, ab as FeedbackRating, ac as FeedbackRefersTo, ad as FileDelegationStore, ae as FileDelegationStoreOptions, af as FleetWorkspaceExecutorOptions, ag as InMemoryDelegationStore, ah as InMemoryFeedbackStore, ai as JsonRpcMessage, aj as JsonRpcResponse, ak as MakeWorkerAgent, al as McpServer, am as McpServerOptions, an as McpToolDescriptor, ao as McpTransport, ap as Question, aq as QuestionDecision, ar as QuestionPolicy, as as QuestionRecord, at as ResearchOutputShape, au as ResearchSource, av as RunDetachedTurnOptions, aw as SettleDetachedCoderTurnOptions, ax as SettledWorker, ay as SiblingSandboxExecutorOptions, az as SubmitInput, aA as SubmitOutput, aB as TraceContext, aC as UiAuditorDelegationOutput, aD as buildDelegationTraceSpans, aE as capDelegationTrace, aF as coderTaskFromArgs, aG as composeLoopTraceEmitters, aH as createCoordinationTools, aI as createDelegateHandler, aJ as createDelegationTraceCollector, aK as createDetachedTurnResumeDriver, aL as createFleetWorkspaceExecutor, aM as createInProcessTransport, aN as createMcpServer, aO as createPropagatingTraceEmitter, aP as createSiblingSandboxExecutor, aQ as detachedSessionDelegate, aR as detachedTurnEvents, aS as eventToSnapshot, aT as formatDetachedSessionRef, aU as hashIdempotencyInput, aV as parseDetachedSessionRef, aW as readTraceContextFromEnv, aX as runDetachedTurn, aY as settleDetachedCoderTurn, aZ as traceContextToEnv, a_ as validateDelegateArgs } from '../coordination-DCmljYDf.js';
2
+ import { F as FleetHandle, D as DelegationExecutor, a as DelegationTaskQueue, b as FeedbackStore, c as DelegateFeedbackResult, d as DelegateFeedbackArgs, U as UiAuditorDelegate, e as DelegateUiAuditArgs, f as DelegateUiAuditResult, g as DelegationHistoryResult, h as DelegationHistoryArgs, i as DelegationStatusResult, j as DelegationStatusArgs } from '../coordination-BI9tpcmF.js';
3
+ export { A as AnalystRegistry, C as CappedDelegationTrace, k as CoderDelegate, l as CoderReview, m as CoderReviewer, n as CoordinationEvent, o as CoordinationTools, p as CoordinationToolsOptions, q as DELEGATE_DESCRIPTION, r as DELEGATE_INPUT_SCHEMA, s as DELEGATE_TOOL_NAME, t as DELEGATION_TRACE_MAX_BYTES, u as DELEGATION_TRACE_MAX_SPANS, v as DelegateArgs, w as DelegateCodeArgs, x as DelegateCodeConfig, y as DelegateCodeResult, z as DelegateHandlerOptions, B as DelegateResearchArgs, E as DelegateResearchConfig, G as DelegateResearchResult, H as DelegateResult, I as DelegateRunCtx, J as DelegateUiAuditConfig, K as DelegateUiAuditRoute, L as DelegationError, M as DelegationFeedbackSnapshot, N as DelegationHistoryEntry, O as DelegationPersistenceError, P as DelegationProfile, Q as DelegationProgress, R as DelegationRecord, S as DelegationResultPayload, T as DelegationResumeContext, V as DelegationResumeDriver, W as DelegationResumeTick, X as DelegationRunContext, Y as DelegationStateCorruptError, Z as DelegationStatus, _ as DelegationStore, $ as DelegationTaskQueueOptions, a0 as DelegationTraceCaps, a1 as DelegationTraceCollector, a2 as DelegationTraceSpan, a3 as DetachedSessionDelegateOptions, a4 as DetachedSessionRefParts, a5 as DetachedTurn, a6 as DetachedTurnResumeDriverOptions, a7 as DetachedWinnerSelection, a8 as DriveTurnCapableBox, a9 as DriveTurnTick, aa as FeedbackEvent, ab as FeedbackRating, ac as FeedbackRefersTo, ad as FileDelegationStore, ae as FileDelegationStoreOptions, af as FleetWorkspaceExecutorOptions, ag as InMemoryDelegationStore, ah as InMemoryFeedbackStore, ai as JsonRpcMessage, aj as JsonRpcResponse, ak as MakeWorkerAgent, al as McpServer, am as McpServerOptions, an as McpToolDescriptor, ao as McpTransport, ap as Question, aq as QuestionDecision, ar as QuestionPolicy, as as QuestionRecord, at as ResearchOutputShape, au as ResearchSource, av as RunDetachedTurnOptions, aw as SettleDetachedCoderTurnOptions, ax as SettledWorker, ay as SiblingSandboxExecutorOptions, az as SubmitInput, aA as SubmitOutput, aB as TraceContext, aC as UiAuditorDelegationOutput, aD as buildDelegationTraceSpans, aE as capDelegationTrace, aF as coderTaskFromArgs, aG as composeLoopTraceEmitters, aH as createCoordinationTools, aI as createDelegateHandler, aJ as createDelegationTraceCollector, aK as createDetachedTurnResumeDriver, aL as createFleetWorkspaceExecutor, aM as createInProcessTransport, aN as createMcpServer, aO as createPropagatingTraceEmitter, aP as createSiblingSandboxExecutor, aQ as detachedSessionDelegate, aR as detachedTurnEvents, aS as eventToSnapshot, aT as formatDetachedSessionRef, aU as hashIdempotencyInput, aV as parseDetachedSessionRef, aW as readTraceContextFromEnv, aX as runDetachedTurn, aY as settleDetachedCoderTurn, aZ as traceContextToEnv, a_ as validateDelegateArgs } from '../coordination-BI9tpcmF.js';
4
4
  import { L as LocalHarness, r as runLocalHarness } from '../local-harness-DU7yV6mG.js';
5
5
  export { a as LocalHarnessResult, R as RunLocalHarnessOptions } from '../local-harness-DU7yV6mG.js';
6
- import { G as GitRunner } from '../worktree-harness-Bmho9SH0.js';
7
- export { C as CreateWorktreeOptions, b as DiffOptions, c as DiffResult, R as RemoveWorktreeOptions, d as WorktreeHandle, e as captureWorktreeDiff, f as createWorktree, r as removeWorktree } from '../worktree-harness-Bmho9SH0.js';
6
+ import { G as GitRunner } from '../worktree-harness-Dx8XULW3.js';
7
+ export { C as CreateWorktreeOptions, b as DiffOptions, c as DiffResult, R as RemoveWorktreeOptions, d as WorktreeHandle, e as captureWorktreeDiff, f as createWorktree, r as removeWorktree } from '../worktree-harness-Dx8XULW3.js';
8
8
  export { C as CreateKbGateOptions, F as FactCandidate, a as FactJudge, b as FactJudgeVerdict, K as KbGateResult, c as createKbGate } from '../kb-gate-CuzMYGYM.js';
9
9
  export { m as mcpToolsForRuntimeMcp, a as mcpToolsForRuntimeMcpSubset } from '../openai-tools-B-3v06BE.js';
10
10
  import { AnalystFinding } from '@tangle-network/agent-eval';
11
11
  import { U as UiLens } from '../substrate-rNj6TDc3.js';
12
12
  import '@tangle-network/sandbox';
13
- import '../types-C1sozrte.js';
13
+ import '../types-By9LXllv.js';
14
14
  import '@tangle-network/agent-interface';
15
15
  import '../otel-export-BKmNwiCb.js';
16
- import '../router-client-Ak2IGuXq.js';
16
+ import '../router-client-D6Ocf4jG.js';
17
17
  import '../environment-provider.js';
18
18
  import '@tangle-network/agent-interface/environment-provider';
19
19
  import 'node:child_process';
package/dist/mcp/index.js CHANGED
@@ -2,11 +2,11 @@ import {
2
2
  createPropagatingTraceEmitter,
3
3
  readTraceContextFromEnv,
4
4
  traceContextToEnv
5
- } from "../chunk-PBE35ULD.js";
5
+ } from "../chunk-M3Y362RW.js";
6
6
  import {
7
7
  mcpToolsForRuntimeMcp,
8
8
  mcpToolsForRuntimeMcpSubset
9
- } from "../chunk-PVPFDTO3.js";
9
+ } from "../chunk-LLJRUTE7.js";
10
10
  import {
11
11
  createKbGate
12
12
  } from "../chunk-FNMGYYSS.js";
@@ -14,7 +14,7 @@ import {
14
14
  assertTraceDerivedFindings,
15
15
  runCoderChecks,
16
16
  selectValidWinner
17
- } from "../chunk-AG335EXG.js";
17
+ } from "../chunk-QSO2TVDS.js";
18
18
  import {
19
19
  DELEGATE_DESCRIPTION,
20
20
  DELEGATE_FEEDBACK_DESCRIPTION,
@@ -64,15 +64,15 @@ import {
64
64
  validateDelegateUiAuditArgs,
65
65
  validateDelegationHistoryArgs,
66
66
  validateDelegationStatusArgs
67
- } from "../chunk-KRULXIWS.js";
68
- import "../chunk-VMNEQHJR.js";
67
+ } from "../chunk-TPII5AU7.js";
68
+ import "../chunk-3X53HYRW.js";
69
69
  import {
70
70
  deleteBoxSafe,
71
71
  sleep,
72
72
  throwAbort,
73
73
  throwIfAborted
74
- } from "../chunk-Z3RRRPRB.js";
75
- import "../chunk-T2HVQVB4.js";
74
+ } from "../chunk-H7IBHAFT.js";
75
+ import "../chunk-4J6RBI3K.js";
76
76
  import {
77
77
  runLocalHarness
78
78
  } from "../chunk-5JAUQZQA.js";
@@ -82,7 +82,17 @@ interface RouterConfig {
82
82
  complete?: (body: Record<string, unknown>) => Promise<unknown>;
83
83
  }
84
84
  interface RouterChatResult {
85
+ /** The final answer, with any inline `<think>...</think>` block stripped into `reasoning`. */
85
86
  content: string;
87
+ /**
88
+ * Thinking-model reasoning, when the provider surfaced it — either as a separate
89
+ * `reasoning`/`reasoning_content` message field (OpenRouter style) or inlined into
90
+ * `content` as a `<think>` block (Groq style). Undefined for non-thinking models.
91
+ * Downstream parsers that match single-token answers must read `content`, which is
92
+ * clean either way; before this split, Groq-style inlining made the same model look
93
+ * broken on one provider and fine on another.
94
+ */
95
+ reasoning?: string;
86
96
  /** REAL usage, or undefined when the provider reported none. */
87
97
  usage?: {
88
98
  input: number;
@@ -98,6 +108,15 @@ declare function routerChatWithUsage(cfg: RouterConfig, messages: Array<{
98
108
  temperature?: number;
99
109
  signal?: AbortSignal;
100
110
  maxTokens?: number;
111
+ /**
112
+ * Reasoning control for thinking models, forwarded as `reasoning_effort`.
113
+ * 'none' is the load-bearing value: binary/single-token decisions (routing,
114
+ * gating) on a thinking model otherwise burn the whole token budget inside
115
+ * the think block — on slow backends (CPU-local) that turns into a client
116
+ * timeout, not just waste. Providers that ignore the field are handled by
117
+ * the reasoning/content split in `parseChatResult`.
118
+ */
119
+ reasoningEffort?: 'none' | 'low' | 'medium' | 'high';
101
120
  }): Promise<RouterChatResult>;
102
121
  /** A tool-call the model emitted (provider-neutral; mirrors the runtime's ToolCallRequest). */
103
122
  interface RouterToolCall {