@tangle-network/agent-runtime 0.79.3 → 0.80.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -325
- package/dist/agent.js +5 -5
- package/dist/analyst-loop.js +2 -2
- package/dist/{chunk-VMNEQHJR.js → chunk-3X53HYRW.js} +17 -2
- package/dist/chunk-3X53HYRW.js.map +1 -0
- package/dist/{chunk-T2HVQVB4.js → chunk-4J6RBI3K.js} +15 -1
- package/dist/chunk-4J6RBI3K.js.map +1 -0
- package/dist/{chunk-IODKUOBA.js → chunk-C2FZ6GR6.js} +2 -2
- package/dist/{chunk-63MHOCIE.js → chunk-DH72UOYR.js} +4 -4
- package/dist/{chunk-Z3RRRPRB.js → chunk-H7IBHAFT.js} +23 -14
- package/dist/chunk-H7IBHAFT.js.map +1 -0
- package/dist/{chunk-PVPFDTO3.js → chunk-LLJRUTE7.js} +2 -2
- package/dist/{chunk-PBE35ULD.js → chunk-M3Y362RW.js} +2 -2
- package/dist/{chunk-AG335EXG.js → chunk-QSO2TVDS.js} +668 -74
- package/dist/chunk-QSO2TVDS.js.map +1 -0
- package/dist/{chunk-KRULXIWS.js → chunk-TPII5AU7.js} +55 -7
- package/dist/chunk-TPII5AU7.js.map +1 -0
- package/dist/{coordination-DCmljYDf.d.ts → coordination-BI9tpcmF.d.ts} +12 -3
- package/dist/environment-provider.d.ts +1 -1
- package/dist/environment-provider.js +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.js +8 -8
- package/dist/intelligence.d.ts +30 -2
- package/dist/intelligence.js +33 -23
- package/dist/intelligence.js.map +1 -1
- package/dist/{loop-runner-bin-C4X0FZ2Z.d.ts → loop-runner-bin-CBkLsyFg.d.ts} +2 -2
- package/dist/loop-runner-bin.d.ts +4 -4
- package/dist/loop-runner-bin.js +7 -7
- package/dist/loops.d.ts +265 -14
- package/dist/loops.js +28 -6
- package/dist/mcp/bin.js +5 -5
- package/dist/mcp/index.d.ts +6 -6
- package/dist/mcp/index.js +7 -7
- package/dist/{router-client-Ak2IGuXq.d.ts → router-client-D6Ocf4jG.d.ts} +19 -0
- package/dist/{types-C1sozrte.d.ts → types-By9LXllv.d.ts} +9 -2
- package/dist/{worktree-fanout-CXGzHET4.d.ts → worktree-fanout-D9Z2dMS9.d.ts} +2 -2
- package/dist/{worktree-harness-Bmho9SH0.d.ts → worktree-harness-Dx8XULW3.d.ts} +1 -1
- package/package.json +1 -1
- package/skills/build-with-agent-runtime/SKILL.md +53 -23
- package/dist/chunk-AG335EXG.js.map +0 -1
- package/dist/chunk-KRULXIWS.js.map +0 -1
- package/dist/chunk-T2HVQVB4.js.map +0 -1
- package/dist/chunk-VMNEQHJR.js.map +0 -1
- package/dist/chunk-Z3RRRPRB.js.map +0 -1
- /package/dist/{chunk-IODKUOBA.js.map → chunk-C2FZ6GR6.js.map} +0 -0
- /package/dist/{chunk-63MHOCIE.js.map → chunk-DH72UOYR.js.map} +0 -0
- /package/dist/{chunk-PVPFDTO3.js.map → chunk-LLJRUTE7.js.map} +0 -0
- /package/dist/{chunk-PBE35ULD.js.map → chunk-M3Y362RW.js.map} +0 -0
package/dist/loops.d.ts
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
|
-
import { ChatClient, AnalystFinding, AgentProfile, AnalystRunInputs, ToolSpan, StreamingDetector, DetectorSignal, buildTrajectory } from '@tangle-network/agent-eval';
|
|
1
|
+
import { ChatClient, RunRecord, AnalystFinding, AgentProfile, AnalystRunInputs, ToolSpan, StreamingDetector, DetectorSignal, buildTrajectory } from '@tangle-network/agent-eval';
|
|
2
2
|
export { AnalystFinding, DefaultVerdict, computeFindingId, makeFinding } from '@tangle-network/agent-eval';
|
|
3
3
|
import { SandboxEvent, SandboxInstance, CreateSandboxOptions } from '@tangle-network/sandbox';
|
|
4
4
|
export { AgentProfile, CreateSandboxOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox';
|
|
5
|
-
import { d as ResultBlobStore, S as SpawnJournal, N as NodeId, j as SpawnEvent, E as ExecutorFactory, c as Agent, B as Budget, i as Scope, g as Settled, f as SupervisedResult, h as Spend, U as UsageEvent, b as ExecutorRegistry, k as Supervisor } from './types-
|
|
6
|
-
export { A as AgentSpec, a as Executor, l as ExecutorContext, m as ExecutorResult, n as SupervisorOpts, T as TreeView, W as WidenGate } from './types-
|
|
7
|
-
import { ak as MakeWorkerAgent, A as AnalystRegistry, o as CoordinationTools, n as CoordinationEvent, ar as QuestionPolicy, a$ as ExecutorConfig } from './coordination-
|
|
8
|
-
export { b0 as BusEvent, b1 as BusRecord, b2 as BusStats, b3 as EventBus, b4 as ProviderSeam, b5 as PublishOptions, b6 as cliWorktreeExecutor, b7 as createEventBus, b8 as createExecutor, b9 as createExecutorRegistry } from './coordination-
|
|
9
|
-
import { R as RuntimeHooks,
|
|
5
|
+
import { d as ResultBlobStore, S as SpawnJournal, N as NodeId, j as SpawnEvent, E as ExecutorFactory, c as Agent, B as Budget, i as Scope, g as Settled, f as SupervisedResult, h as Spend, U as UsageEvent, b as ExecutorRegistry, k as Supervisor } from './types-By9LXllv.js';
|
|
6
|
+
export { A as AgentSpec, a as Executor, l as ExecutorContext, m as ExecutorResult, R as Runtime, n as SupervisorOpts, T as TreeView, W as WidenGate } from './types-By9LXllv.js';
|
|
7
|
+
import { ak as MakeWorkerAgent, A as AnalystRegistry, o as CoordinationTools, n as CoordinationEvent, ar as QuestionPolicy, a$ as ExecutorConfig } from './coordination-BI9tpcmF.js';
|
|
8
|
+
export { b0 as BusEvent, b1 as BusRecord, b2 as BusStats, b3 as EventBus, b4 as ProviderSeam, b5 as PublishOptions, b6 as cliWorktreeExecutor, b7 as createEventBus, b8 as createExecutor, b9 as createExecutorRegistry } from './coordination-BI9tpcmF.js';
|
|
9
|
+
import { R as RuntimeHooks, S as SandboxClient, I as Iteration, W as Driver, A as AgentRunSpec, b as OutputAdapter, V as Validator, E as ExecCtx, X as LoopWinner, Y as LoopLineageOptions, Z as LoopResult, L as LoopTokenUsage, a as RuntimeStreamEvent, _ as MountRecorder } from './types-BF-MEsQB.js';
|
|
10
10
|
export { $ as LoopDecisionPayload, a0 as LoopEndedPayload, a1 as LoopIterationDispatchPayload, a2 as LoopIterationEndedPayload, a3 as LoopIterationStartedPayload, a4 as LoopPlanDescription, a5 as LoopPlanPayload, d as LoopSandboxPlacement, a6 as LoopStartedPayload, a7 as LoopTeardownFailedPayload, e as LoopTraceEmitter, c as LoopTraceEvent, a8 as MountManifestEntry, a9 as RunProvenance, aa as SelectionReceipt, ab as ValidationCtx } from './types-BF-MEsQB.js';
|
|
11
11
|
export { AgentEnvironmentProviderRef, AgentEnvironmentProviderRegistry, ProviderAsSandboxClientOptions, ProviderExecutorOptions, SandboxClientProviderOptions, createAgentEnvironmentProviderRegistry, providerAsExecutor, providerAsSandboxClient, resolveAgentEnvironmentProvider, sandboxClientAsProvider } from './environment-provider.js';
|
|
12
|
-
import { C as CorpusRecord, c as Corpus, O as Outcome, S as ScopeAnalyzeInput, d as AssertTraceDerivedFindings, e as SteerContext, f as ScopeAnalyst, F as FanoutOptions, g as CombinatorShape, h as ScopeWidenGate, L as LoopUntilSpec, P as PanelSpec, i as PipelineStage, W as WinnerStrategy, j as FanoutWinnerSelector, V as VerifySpec, k as WidenSpec, l as CorpusFilter, R as RenderCorpusToInstructionsOptions, D as DefinePersonaInput, m as Persona, n as RunPersonifiedOptions, o as ShapeRegistry, p as LoopShape, E as EqualKArm, q as EqualKOnCostOptions, r as EqualKVerdict, T as TrajectoryReportOptions, s as TrajectoryReport } from './worktree-fanout-
|
|
13
|
-
export { A as AuthoredHarness, t as DefinePersona, u as EqualKOnCost, v as Fanout, w as FanoutSynthesis, x as FlatWidenGate, y as LoopUntil, z as LoopUntilState, B as Panel, G as PanelJudge, H as PanelVerdict, I as PatchDeliverableOptions, J as PersonaContext, K as PersonaExecutors, M as Pipeline, N as RenderCorpusToInstructions, Q as RunPersonified, U as ShapeBudget, X as ShapeContext, Y as TrajectoryNode, Z as TrajectoryReportFn, _ as Verify, $ as Widen, a0 as WidenDecision, a1 as WidenLineage, a2 as WorktreeCliExecutorOptions, a as WorktreeFanoutOptions, b as WorktreePatchArtifact, a3 as createWorktreeCliExecutor, a4 as patchDelivered, a5 as worktreeFanout } from './worktree-fanout-
|
|
14
|
-
import { Scenario, ProfileDispatchFn } from '@tangle-network/agent-eval/campaign';
|
|
12
|
+
import { C as CorpusRecord, c as Corpus, O as Outcome, S as ScopeAnalyzeInput, d as AssertTraceDerivedFindings, e as SteerContext, f as ScopeAnalyst, F as FanoutOptions, g as CombinatorShape, h as ScopeWidenGate, L as LoopUntilSpec, P as PanelSpec, i as PipelineStage, W as WinnerStrategy, j as FanoutWinnerSelector, V as VerifySpec, k as WidenSpec, l as CorpusFilter, R as RenderCorpusToInstructionsOptions, D as DefinePersonaInput, m as Persona, n as RunPersonifiedOptions, o as ShapeRegistry, p as LoopShape, E as EqualKArm, q as EqualKOnCostOptions, r as EqualKVerdict, T as TrajectoryReportOptions, s as TrajectoryReport } from './worktree-fanout-D9Z2dMS9.js';
|
|
13
|
+
export { A as AuthoredHarness, t as DefinePersona, u as EqualKOnCost, v as Fanout, w as FanoutSynthesis, x as FlatWidenGate, y as LoopUntil, z as LoopUntilState, B as Panel, G as PanelJudge, H as PanelVerdict, I as PatchDeliverableOptions, J as PersonaContext, K as PersonaExecutors, M as Pipeline, N as RenderCorpusToInstructions, Q as RunPersonified, U as ShapeBudget, X as ShapeContext, Y as TrajectoryNode, Z as TrajectoryReportFn, _ as Verify, $ as Widen, a0 as WidenDecision, a1 as WidenLineage, a2 as WorktreeCliExecutorOptions, a as WorktreeFanoutOptions, b as WorktreePatchArtifact, a3 as createWorktreeCliExecutor, a4 as patchDelivered, a5 as worktreeFanout } from './worktree-fanout-D9Z2dMS9.js';
|
|
14
|
+
import { Scenario, DispatchFn, ProfileDispatchFn } from '@tangle-network/agent-eval/campaign';
|
|
15
15
|
import { b as AnalystRegistryLike } from './types-BC3bZpH0.js';
|
|
16
16
|
import { AgentProfile as AgentProfile$1 } from '@tangle-network/agent-interface';
|
|
17
|
-
import { R as RouterConfig, a as ToolLoopChat, b as ToolLoopCompactionOptions } from './router-client-
|
|
18
|
-
export { c as RouterChatResult, d as RouterChatToolsResult, e as RouterToolCall, f as RouterToolLoopResult, g as ToolLoopCompaction, T as ToolSpec, r as routerBrain, h as routerChatWithTools, i as routerChatWithUsage, j as routerToolLoop } from './router-client-
|
|
19
|
-
import { D as DeliverableSpec } from './worktree-harness-
|
|
20
|
-
export { g as WorktreeCommandResult, h as gateOnDeliverable } from './worktree-harness-
|
|
17
|
+
import { R as RouterConfig, a as ToolLoopChat, b as ToolLoopCompactionOptions } from './router-client-D6Ocf4jG.js';
|
|
18
|
+
export { c as RouterChatResult, d as RouterChatToolsResult, e as RouterToolCall, f as RouterToolLoopResult, g as ToolLoopCompaction, T as ToolSpec, r as routerBrain, h as routerChatWithTools, i as routerChatWithUsage, j as routerToolLoop } from './router-client-D6Ocf4jG.js';
|
|
19
|
+
import { D as DeliverableSpec } from './worktree-harness-Dx8XULW3.js';
|
|
20
|
+
export { g as WorktreeCommandResult, h as gateOnDeliverable } from './worktree-harness-Dx8XULW3.js';
|
|
21
21
|
import { stuckLoopView, toolWasteView } from '@tangle-network/agent-eval/pipelines';
|
|
22
22
|
export { AgentEnvironment, AgentEnvironmentCapabilities, AgentEnvironmentEvent, AgentEnvironmentProvider, AgentEnvironmentQuery, AgentEnvironmentStatus, AgentEnvironmentSummary, AgentProfileRef, AgentSession, AgentSessionRef, AgentSessionStatus, AgentTurnInput, AgentTurnResult, CheckpointRef, CheckpointRequest, CreateAgentEnvironmentInput, ExecRequest, ExecResult, ForkRequest, PlacementInfo, ResourceRequest, WorkspaceRequest } from '@tangle-network/agent-interface/environment-provider';
|
|
23
23
|
import './substrate-rNj6TDc3.js';
|
|
@@ -276,6 +276,156 @@ interface IntentAudit {
|
|
|
276
276
|
declare const defaultAuditorInstruction: string;
|
|
277
277
|
declare function auditIntent(input: AuditIntentInput, opts: AuditIntentOptions): Promise<IntentAudit>;
|
|
278
278
|
|
|
279
|
+
/**
|
|
280
|
+
* benchmark-report — turn a fleet of `RunRecord`s into a publishable, multi-axis benchmark report:
|
|
281
|
+
* a ranked leaderboard, the full profile×axis score matrix, the cost/latency/token columns, and
|
|
282
|
+
* embeddable charts (SVG) + a self-contained HTML page. Domain-agnostic by construction — it reads ONLY
|
|
283
|
+
* the universal `RunRecord` currency (model, scenario, score, cost, tokens, latency, the `outcome.raw`
|
|
284
|
+
* metric bag), so the SAME engine reports any benchmark in any domain: coding, search, agents, multimodal.
|
|
285
|
+
*
|
|
286
|
+
* This is the surface a hosted leaderboard (à la vals.ai) renders: for every harness×model profile, its
|
|
287
|
+
* score on every axis, not a curated subset. It pairs with `runProfileMatrix` (whose `result.records`
|
|
288
|
+
* feed straight in) but takes the records directly, so it is independent of how they were produced.
|
|
289
|
+
*
|
|
290
|
+
* An AXIS is any way to slice the score into columns — by default one column per scenario group
|
|
291
|
+
* (`groupOf`, default = the scenario id), so the matrix is profile × scenario. Pass `axisScoresOf` to
|
|
292
|
+
* score along judge dimensions or any custom decomposition instead. The reporter never invents a number:
|
|
293
|
+
* a missing cell renders blank, never zero.
|
|
294
|
+
*/
|
|
295
|
+
|
|
296
|
+
/** Pull the headline score in [0,1] from a record. Default: the held-out split, else the search split,
|
|
297
|
+
* else a `composite`/`passed`/`score` entry in the raw bag. Override to score a domain differently. */
|
|
298
|
+
type ScoreOf = (record: RunRecord) => number | undefined;
|
|
299
|
+
/** The profile (matrix row) a record belongs to — default `harness·model` from the record's profile cell,
|
|
300
|
+
* falling back to the model. This is the leaderboard's unit of comparison. */
|
|
301
|
+
type ProfileKeyOf = (record: RunRecord) => string;
|
|
302
|
+
/** The axis (matrix column) a record contributes to — default the scenario group. */
|
|
303
|
+
type GroupOf = (record: RunRecord) => string;
|
|
304
|
+
/** Decompose ONE record into per-axis scores (e.g. judge dimensions). When set, it REPLACES the
|
|
305
|
+
* scenario-group axes: the column set is the union of returned keys. */
|
|
306
|
+
type AxisScoresOf = (record: RunRecord) => Record<string, number>;
|
|
307
|
+
interface LeaderboardOptions {
|
|
308
|
+
readonly title?: string;
|
|
309
|
+
readonly scoreOf?: ScoreOf;
|
|
310
|
+
readonly profileKeyOf?: ProfileKeyOf;
|
|
311
|
+
readonly groupOf?: GroupOf;
|
|
312
|
+
readonly axisScoresOf?: AxisScoresOf;
|
|
313
|
+
/** Display label for a profile key (default: the key itself). */
|
|
314
|
+
readonly labelOf?: (profileKey: string) => string;
|
|
315
|
+
/** Commit SHA / dataset / dates surfaced in the provenance block. */
|
|
316
|
+
readonly meta?: Record<string, string>;
|
|
317
|
+
/** Compute per-row confidence intervals (bootstrap on score, Wilson on pass rate). Needs a
|
|
318
|
+
* `scenarioId` on every record (reps are collapsed per scenario for the honest n). Default off. */
|
|
319
|
+
readonly stats?: boolean;
|
|
320
|
+
/** A score ≥ this counts as a "pass" for the pass-rate proportion + its Wilson CI. Default 0.999
|
|
321
|
+
* (fully solved). Lower it (e.g. 0.6) for a partial-credit domain. */
|
|
322
|
+
readonly passThreshold?: number;
|
|
323
|
+
}
|
|
324
|
+
/** A 95%-by-default confidence interval. */
|
|
325
|
+
interface Interval {
|
|
326
|
+
readonly lower: number;
|
|
327
|
+
readonly upper: number;
|
|
328
|
+
}
|
|
329
|
+
/** One leaderboard row — a harness×model profile, every measured column. */
|
|
330
|
+
interface LeaderboardRow {
|
|
331
|
+
readonly profileKey: string;
|
|
332
|
+
readonly label: string;
|
|
333
|
+
readonly model: string;
|
|
334
|
+
readonly n: number;
|
|
335
|
+
readonly meanScore: number;
|
|
336
|
+
/** Fraction of records scoring ≥ `passThreshold` (default 0.999) — the binary pass rate. */
|
|
337
|
+
readonly solveRate: number;
|
|
338
|
+
/** axis → mean score for this profile (blank in render when the profile never ran that axis). */
|
|
339
|
+
readonly perAxis: Record<string, number>;
|
|
340
|
+
readonly costUsd: number;
|
|
341
|
+
readonly tokensIn: number;
|
|
342
|
+
readonly tokensOut: number;
|
|
343
|
+
readonly latencyP50Ms: number;
|
|
344
|
+
readonly latencyP90Ms: number;
|
|
345
|
+
/** Bootstrap CI on the mean score — present only when `opts.stats` is set. Computed over
|
|
346
|
+
* per-scenario means (reps collapsed first), so identical reps can't fake a narrow interval. */
|
|
347
|
+
readonly scoreCi?: Interval;
|
|
348
|
+
/** Wilson CI on the pass rate — present only when `opts.stats` is set. */
|
|
349
|
+
readonly passCi?: Interval;
|
|
350
|
+
}
|
|
351
|
+
interface Leaderboard {
|
|
352
|
+
readonly title: string;
|
|
353
|
+
/** Column order — scenario groups (default) or dimension keys (`axisScoresOf`). */
|
|
354
|
+
readonly axes: readonly string[];
|
|
355
|
+
/** Rows ranked by `meanScore` desc (ties → lower cost, then label). */
|
|
356
|
+
readonly profiles: readonly LeaderboardRow[];
|
|
357
|
+
readonly meta: Record<string, string>;
|
|
358
|
+
/** Provenance counts — the denominators every honest report leads with. */
|
|
359
|
+
readonly provenance: {
|
|
360
|
+
readonly records: number;
|
|
361
|
+
readonly profiles: number;
|
|
362
|
+
readonly axes: number;
|
|
363
|
+
readonly models: readonly string[];
|
|
364
|
+
readonly totalCostUsd: number;
|
|
365
|
+
};
|
|
366
|
+
}
|
|
367
|
+
/** Aggregate a fleet of records into the ranked, multi-axis report. Pure — no IO, deterministic. */
|
|
368
|
+
declare function leaderboard(records: readonly RunRecord[], opts?: LeaderboardOptions): Leaderboard;
|
|
369
|
+
/** One profile pair compared on the scenarios they BOTH ran — the "who actually beat whom" verdict. */
|
|
370
|
+
interface PairwiseVerdict {
|
|
371
|
+
readonly a: string;
|
|
372
|
+
readonly b: string;
|
|
373
|
+
/** Paired unit count (shared scenarios). The significance is suppressed below `minPairs`. */
|
|
374
|
+
readonly pairs: number;
|
|
375
|
+
/** Median paired delta (b − a) and its bootstrap CI. */
|
|
376
|
+
readonly delta: number;
|
|
377
|
+
readonly ciLow: number;
|
|
378
|
+
readonly ciHigh: number;
|
|
379
|
+
/** Paired-test p-value (before correction). */
|
|
380
|
+
readonly p: number;
|
|
381
|
+
/** BH-significant across ALL pairs AND above the `minPairs` power floor. */
|
|
382
|
+
readonly significant: boolean;
|
|
383
|
+
}
|
|
384
|
+
interface PairwiseOptions {
|
|
385
|
+
readonly scoreOf?: ScoreOf;
|
|
386
|
+
readonly profileKeyOf?: ProfileKeyOf;
|
|
387
|
+
readonly labelOf?: (profileKey: string) => string;
|
|
388
|
+
/** False-discovery rate for the Benjamini–Hochberg correction. Default 0.05. */
|
|
389
|
+
readonly fdr?: number;
|
|
390
|
+
/** Below this many shared scenarios a paired test can't defensibly separate two profiles, so the
|
|
391
|
+
* `significant` tag is suppressed regardless of p (small-n mirage protection). Default 12. */
|
|
392
|
+
readonly minPairs?: number;
|
|
393
|
+
}
|
|
394
|
+
/** Compare EVERY profile pair on the scenarios they both ran — paired-bootstrap effect + CI, a real
|
|
395
|
+
* paired-test p-value, BH-corrected across all pairs. This is the honest "did A beat B" table the
|
|
396
|
+
* leaderboard's point ranking cannot answer. Reuses the agent-eval statistics substrate. */
|
|
397
|
+
declare function pairwiseSignificance(records: readonly RunRecord[], opts?: PairwiseOptions): PairwiseVerdict[];
|
|
398
|
+
/** Render the report as a publishable Markdown document: provenance → leaderboard → the full profile×axis
|
|
399
|
+
* matrix → cost/latency/token columns. Every axis is shown — a curated subset is a reporting failure. */
|
|
400
|
+
declare function renderLeaderboardMarkdown(report: Leaderboard): string;
|
|
401
|
+
/** Render the pairwise-significance table — every profile pair's paired delta, CI, and BH-corrected
|
|
402
|
+
* verdict. Feed it `pairwiseSignificance(records)`. This is the "did A really beat B" evidence the point
|
|
403
|
+
* ranking cannot give. */
|
|
404
|
+
declare function renderPairwiseMarkdown(verdicts: readonly PairwiseVerdict[], title?: string): string;
|
|
405
|
+
/** Render a self-contained SVG: a ranked score bar chart on top, the profile×axis heatmap below. No deps,
|
|
406
|
+
* embeddable anywhere (README, HTML page, hosted leaderboard). */
|
|
407
|
+
declare function renderLeaderboardSvg(report: Leaderboard): string;
|
|
408
|
+
/** Render a self-contained HTML leaderboard page (the hosted surface): the SVG charts + the full Markdown
|
|
409
|
+
* matrix as a table. Single file, no assets, opens in any browser. */
|
|
410
|
+
declare function renderLeaderboardHtml(report: Leaderboard): string;
|
|
411
|
+
|
|
412
|
+
interface CliBridgeConfig {
|
|
413
|
+
/** cli-bridge base URL. Defaults to `http://127.0.0.1:3355`. */
|
|
414
|
+
url?: string;
|
|
415
|
+
/** Bearer token the bridge requires (fail loud when empty). */
|
|
416
|
+
bearer: string;
|
|
417
|
+
/** Default harness when a `create({ backend })` override does not set one. */
|
|
418
|
+
harness?: string;
|
|
419
|
+
/** Default model when a `create({ backend })` override does not set one. */
|
|
420
|
+
model?: string;
|
|
421
|
+
}
|
|
422
|
+
/**
|
|
423
|
+
* A `SandboxClient` whose every prompt is one cli-bridge chat completion, metered
|
|
424
|
+
* and driven by the shared `runLoop` kernel. Pass a fixed `harness`/`model`, or omit
|
|
425
|
+
* them and set the target per cell via `create({ backend: { type, model: { model } } })`.
|
|
426
|
+
*/
|
|
427
|
+
declare function cliBridgeSandboxClient(cfg: CliBridgeConfig): SandboxClient;
|
|
428
|
+
|
|
279
429
|
/**
|
|
280
430
|
* @experimental
|
|
281
431
|
*
|
|
@@ -679,6 +829,25 @@ interface LoopDispatchOptions<Task, Output, Decision, TScenario extends Scenario
|
|
|
679
829
|
/** Cost-meter source label for the loop's spend. Default `'loop'`. */
|
|
680
830
|
costSource?: string;
|
|
681
831
|
}
|
|
832
|
+
/** Options for adapting plain agent-eval campaign scenarios into runtime `runLoop` cells. */
|
|
833
|
+
interface LoopCampaignDispatchOptions<Task, Output, Decision, TScenario extends Scenario, TArtifact> {
|
|
834
|
+
/** Sandbox client used for every campaign cell's `runLoop`. */
|
|
835
|
+
sandboxClient: SandboxClient;
|
|
836
|
+
/** Build the per-cell runLoop options from the campaign scenario. */
|
|
837
|
+
toLoopOptions: (scenario: TScenario) => LoopOptionsForDispatch<Task, Output, Decision>;
|
|
838
|
+
/** Map the finished loop to the artifact the campaign judges score. */
|
|
839
|
+
toArtifact?: (result: LoopResult<Task, Output, Decision>) => TArtifact;
|
|
840
|
+
/** Forward `loop.*` trace events into the campaign's scoped trace. Default true. */
|
|
841
|
+
forwardTrace?: boolean;
|
|
842
|
+
/** Cost-meter source label for the loop's spend. Default `'loop'`. */
|
|
843
|
+
costSource?: string;
|
|
844
|
+
}
|
|
845
|
+
/**
|
|
846
|
+
* Adapter for plain `runCampaign` scenarios. This is the runtime-side pair for
|
|
847
|
+
* agent-eval fixture scenarios: load fixtures in `agent-eval/campaign`, build
|
|
848
|
+
* the runtime loop here, and keep cost + token + trace reporting automatic.
|
|
849
|
+
*/
|
|
850
|
+
declare function loopCampaignDispatch<Task, Output, Decision, TScenario extends Scenario, TArtifact>(opts: LoopCampaignDispatchOptions<Task, Output, Decision, TScenario, TArtifact>): DispatchFn<TScenario, TArtifact>;
|
|
682
851
|
/**
|
|
683
852
|
* Adapter for `runProfileMatrix` (profile is an axis). Returns a
|
|
684
853
|
* `ProfileDispatchFn` that runs `runLoop` per (profile, scenario) cell and
|
|
@@ -1650,6 +1819,24 @@ interface CriuCapableClient {
|
|
|
1650
1819
|
declare function extractLlmCallEvent(event: SandboxEvent, agentRunName: string): (RuntimeStreamEvent & {
|
|
1651
1820
|
type: 'llm_call';
|
|
1652
1821
|
}) | undefined;
|
|
1822
|
+
/**
|
|
1823
|
+
* Sum the token usage + USD cost of a sandbox turn's events — the one honest way to meter an
|
|
1824
|
+
* `openSandboxRun` cell. Folds `extractLlmCallEvent` over the stream (which reads usage off EVERY backend
|
|
1825
|
+
* event shape), so a `runProfileMatrix` dispatch can report it to `ctx.cost`:
|
|
1826
|
+
*
|
|
1827
|
+
* const turn = await run.start(prompt)
|
|
1828
|
+
* const u = sumSandboxUsage(turn.events)
|
|
1829
|
+
* if (u.input || u.output) ctx.cost.observeTokens({ input: u.input, output: u.output })
|
|
1830
|
+
* if (u.costUsd) ctx.cost.observe(u.costUsd, 'sandbox-cell')
|
|
1831
|
+
*
|
|
1832
|
+
* Without this a cell reads `{tokens:0, cost:0}` and the backend-integrity guard correctly aborts the
|
|
1833
|
+
* matrix as a stub. `agentRunName` is the fallback model label for cost-only events (default `'agent'`).
|
|
1834
|
+
*/
|
|
1835
|
+
declare function sumSandboxUsage(events: readonly SandboxEvent[], agentRunName?: string): {
|
|
1836
|
+
input: number;
|
|
1837
|
+
output: number;
|
|
1838
|
+
costUsd: number;
|
|
1839
|
+
};
|
|
1653
1840
|
/**
|
|
1654
1841
|
* Project one `SandboxEvent` onto the `RuntimeStreamEvent` chat-UX vocabulary,
|
|
1655
1842
|
* for runtimes that bridge a sandbox `streamPrompt` into the
|
|
@@ -2645,6 +2832,9 @@ interface CoordinationMcpHandle {
|
|
|
2645
2832
|
valid?: boolean;
|
|
2646
2833
|
outRef?: string;
|
|
2647
2834
|
}>;
|
|
2835
|
+
/** Post-loop drain of already-settled, unpulled children into the ledger — call before reading
|
|
2836
|
+
* `settled()` for a finalize, so a delivered child the harness never awaited is not lost. */
|
|
2837
|
+
drainResolved: CoordinationTools['drainResolved'];
|
|
2648
2838
|
isStopped(): boolean;
|
|
2649
2839
|
/** The full ordered bus-event log — observability audit + replay trail. */
|
|
2650
2840
|
history: CoordinationTools['history'];
|
|
@@ -3190,6 +3380,67 @@ interface TrajectoryAnalysis {
|
|
|
3190
3380
|
/** Collect the source's spans and run the agent-eval batch analyzers over them under one `runId`. */
|
|
3191
3381
|
declare function analyzeTrace(source: TraceSource, runId?: string): Promise<TrajectoryAnalysis>;
|
|
3192
3382
|
|
|
3383
|
+
/** What a surface worker settles with — the surface verdict the driver + deliverable read. `resolved` is
|
|
3384
|
+
* the surface check's pass/fail (settled ⟺ resolved); `score` is the partial-credit fraction; `failing`
|
|
3385
|
+
* carries the tests this worker left red (so the analyst can target them). */
|
|
3386
|
+
interface SurfaceWorkerOut {
|
|
3387
|
+
readonly resolved: boolean;
|
|
3388
|
+
readonly score: number;
|
|
3389
|
+
readonly shots: number;
|
|
3390
|
+
readonly summary: string;
|
|
3391
|
+
readonly failing?: readonly string[];
|
|
3392
|
+
}
|
|
3393
|
+
/** The default self-improvement LENS — authored content, not a code path. On each settled worker it hands
|
|
3394
|
+
* the driver the still-FAILING tests (not just a score), so the next spawn targets the persistently-hard
|
|
3395
|
+
* cases. Swap `analysts` to change what the driver improves from — that's the one knob. */
|
|
3396
|
+
declare function failuresAnalyst(): AnalystRegistry;
|
|
3397
|
+
/** How a worker runs the surface task (its router substrate + per-attempt bounds). */
|
|
3398
|
+
interface SurfaceWorkerConfig {
|
|
3399
|
+
readonly routerBaseUrl: string;
|
|
3400
|
+
readonly routerKey: string;
|
|
3401
|
+
readonly model: string;
|
|
3402
|
+
readonly maxTokens?: number;
|
|
3403
|
+
readonly innerTurns?: number;
|
|
3404
|
+
/** Refine-shot budget for ONE worker attempt (max steered shots). Default 1. */
|
|
3405
|
+
readonly budget?: number;
|
|
3406
|
+
}
|
|
3407
|
+
interface SuperviseSurfaceOptions {
|
|
3408
|
+
/** The graded surface workers solve (open/tools/call/score/close). */
|
|
3409
|
+
readonly surface: AgenticSurface;
|
|
3410
|
+
/** Where/how each worker runs the surface task. */
|
|
3411
|
+
readonly worker: SurfaceWorkerConfig;
|
|
3412
|
+
/** The conserved compute pool for the whole supervised run. Default: sized off the worker's inner-loop
|
|
3413
|
+
* bounds for a handful of worker spawns — raise it to let the driver try more. */
|
|
3414
|
+
readonly budget?: Budget;
|
|
3415
|
+
/** The driver brain's router substrate (its own inference). Default: the worker's router + model — the
|
|
3416
|
+
* driver and workers share one router unless you separate them (e.g. a stronger driver model). */
|
|
3417
|
+
readonly router?: RouterConfig;
|
|
3418
|
+
/** The self-improvement lens fed to the driver on each settled worker. Default `failuresAnalyst()`
|
|
3419
|
+
* (target the still-failing tests). Pass a custom registry to change it, or `null` to turn the
|
|
3420
|
+
* within-run self-improvement OFF (the driver sees raw settled outputs). */
|
|
3421
|
+
readonly analysts?: AnalystRegistry | null;
|
|
3422
|
+
/** The strategy each worker runs over the surface. Default `refine` (iterate-with-feedback). */
|
|
3423
|
+
readonly strategy?: Strategy;
|
|
3424
|
+
/** Max workers live at once. Default 1 (serial — required when workers share a persistent artifact, so
|
|
3425
|
+
* they continue each other instead of racing the file). */
|
|
3426
|
+
readonly maxLiveWorkers?: number;
|
|
3427
|
+
}
|
|
3428
|
+
/** The deployable outcome of a supervised surface run. */
|
|
3429
|
+
interface SuperviseSurfaceResult {
|
|
3430
|
+
readonly resolved: boolean;
|
|
3431
|
+
readonly score: number;
|
|
3432
|
+
readonly usd: number;
|
|
3433
|
+
readonly tokensIn: number;
|
|
3434
|
+
readonly tokensOut: number;
|
|
3435
|
+
readonly ms: number;
|
|
3436
|
+
/** Total conserved-pool iterations = the driver + worker LLM rounds the run actually spent. */
|
|
3437
|
+
readonly completions: number;
|
|
3438
|
+
}
|
|
3439
|
+
/** Drive a team of agents (spawned + steered by `profile`) to solve a graded `AgenticSurface` task, and
|
|
3440
|
+
* report the deployable outcome + the full conserved spend. This is `supervise()` configured for surfaces
|
|
3441
|
+
* — there is no other entrypoint to learn. */
|
|
3442
|
+
declare function superviseSurface(profile: SupervisorProfile, task: AgenticTask, opts: SuperviseSurfaceOptions): Promise<SuperviseSurfaceResult>;
|
|
3443
|
+
|
|
3193
3444
|
/**
|
|
3194
3445
|
* createVerifierEnvironment — ANY checkable task as an `Environment`, no tool surface
|
|
3195
3446
|
* required. The generalization piece: EOPS/commit0-style domains have tools that mutate
|
|
@@ -3285,4 +3536,4 @@ declare function runInWorkspace<T>(ws: Workspace, body: (cwd: string) => Promise
|
|
|
3285
3536
|
commitOnInvalid?: boolean;
|
|
3286
3537
|
}): Promise<WorkspaceRun<T>>;
|
|
3287
3538
|
|
|
3288
|
-
export { Agent, AgentRunSpec, type AgenticOptions, type AgenticRunResult, type AgenticSurface, type AgenticTask, type AgenticTool, AnalystRegistry, type AnytimeReport, type AnytimeStrategySummary, type AnytimeTaskCurve, type ApplyContinuation, type ArtifactHandle, AssertTraceDerivedFindings, type AuditIntentInput, type AuditIntentOptions, type AuthorStrategyOptions, type AuthoredProfile, type AuthoredStrategy, type BenchmarkCell, type BenchmarkConfig, type BenchmarkLift, type BenchmarkReport, type BenchmarkStrategySummary, type BenchmarkTaskRow, Budget, type BudgetPool, type BudgetReadout, type ChampionPick, type ChampionPolicy, type CheckpointCapableBox, CombinatorShape, type CompletionAnalyst, type CompletionEvidence, type CompletionPolicy, type CompletionVerdict, CoordinationEvent, type CoordinationMcpHandle, Corpus, CorpusFilter, CorpusRecord, type CreateScopeAnalystOptions, type CriuCapableClient, DefinePersonaInput, type DelegateOptions, type Deliverable, DeliverableSpec, type DriveHarness, Driver, type DriverAgentOptions, type DumbDriverOptions, type Environment, EqualKArm, EqualKOnCostOptions, EqualKVerdict, type EvolutionArchiveNode, type EvolutionAuthor, type EvolutionBandInfo, type EvolutionCandidate, type EvolutionGeneration, type EvolutionReport, ExecCtx, ExecutorConfig, FanoutOptions, FanoutWinnerSelector, FileCorpus, type ForkCapableBox, type GitWorkspaceOptions, type HarvestCorpusOptions, type HarvestFailure, type HarvestReport, InMemoryCorpus, InMemoryResultBlobStore, type InMemoryRunContext, type InMemoryRunContextOptions, InMemorySpawnJournal, type InProcessOnPrompt, type InProcessPromptCtx, type InProcessSandboxClientOptions, type Inbox, type InboxMessage, type IntentAudit, Iteration, type LoopDispatchOptions, LoopLineageOptions, type LoopOptionsForDispatch, LoopResult, LoopShape, LoopTokenUsage, LoopUntilSpec, LoopWinner, MakeWorkerAgent, type McpEndpoint, type McpEnvironmentOptions, MountRecorder, type NaiveDriverOptions, type Observation, type ObserveInput, type ObserveOptions, type OpenSandboxRunOptions, Outcome, OutputAdapter, PanelSpec, Persona, PipelineStage, type ProfileRichness, type ProfileRichnessThresholds, type PromotionGateOptions, type PromotionVerdict, type RegistryAnalyzeProjection, RenderCorpusToInstructionsOptions, type ReservationTicket, ResultBlobStore, RouterConfig, type RunAgenticOptions, RunPersonifiedOptions, type SandboxCapabilities, SandboxClient, type SandboxLineage, type SandboxLineageHandle, type SandboxRun, SandboxRunAbortError, Scope, ScopeAnalyst, ScopeAnalyzeInput, ScopeWidenGate, type SessionCapableBox, type SessionTraceBox, Settled, ShapeRegistry, type Shell, type ShotPersona, type ShotSpec, Spend, SteerContext, type SteeringDecision, type Strategy, type StrategyCtx, type StrategyEvolutionConfig, type StrategyResult, type SuperviseOptions, SupervisedResult, Supervisor, type SupervisorAgentDeps, type SupervisorProfile, type SurfaceScore, ToolLoopChat, ToolLoopCompactionOptions, type TraceSource, type TrajectoryAnalysis, TrajectoryReport, TrajectoryReportOptions, type TurnResult, UsageEvent, type UsageSink, Validator, type VerifierEnvironmentOptions, VerifySpec, type WatchTraceOptions, type WaterfallCollector, type WaterfallReport, type WaterfallSpan, WidenSpec, WinnerStrategy, type Workspace, type WorkspaceCommit, type WorkspaceRun, acquireSandbox, adaptiveRefine, analyzeTrace, anytimeReport, asAuthoredProfile, assertModelAllowed, assertStrategyContract, assertTraceDerivedFindings, assessAuthoredProfile, auditIntent, authorStrategy, authoredWorker, breadthStrategy, buildSteerContext, builtinShapes, completionAuthorizes, contentAddress, createBudgetPool, createInMemoryRunContext, createInbox, createMcpEnvironment, createPushTraceSource, createSandboxLineage, createScope, createScopeAnalyst, createShapeRegistry, createSupervisor, createVerifierEnvironment, createWaterfallCollector, decodeToolPart, defaultAnalystInstruction, defaultAuditorInstruction, defaultDelegateBudget, defaultProfileRichnessThresholds, defaultSelectWinner, defaultToolDetectors, definePersona, defineStrategy, delegate, depthStrategy, deterministicCompletion, discriminatingMeans, driverAgent, dumbDriver, equalKOnCost, extractLlmCallEvent, fanout, finalizeBestDelivered, flatWidenGate, gitWorkspace, harvestCorpus, inProcessSandboxClient, inlineSandboxClient, jjWorkspace, localShell, loopDispatch, loopUntil, mapSandboxEvent, naiveDriver, observe, openSandboxRun, panel, pickChampion, pipeline, printBenchmarkReport, probeSandboxCapabilities, profileRichnessFinding, promotionGate, refine, registerShape, registryScopeAnalyst, renderAnytimeTable, renderCorpusToInstructions, renderReport, reportLoopUsage, runAgentic, runBenchmark, runInWorkspace, runLoop, runPersonified, runStrategyEvolution, sample, sampleThenRefine, sandboxSessionTraceSource, selectChampion, selectValidWinner, sentinelCompletion, serveCoordinationMcp, settledToIteration, spendFromUsageEvents, stopSentinel, strategyAuthorContract, supervise, supervisorAgent, supervisorInstructions, trajectoryReport, verify, watchTrace, widen, workerFromBackend };
|
|
3539
|
+
export { Agent, AgentRunSpec, type AgenticOptions, type AgenticRunResult, type AgenticSurface, type AgenticTask, type AgenticTool, AnalystRegistry, type AnytimeReport, type AnytimeStrategySummary, type AnytimeTaskCurve, type ApplyContinuation, type ArtifactHandle, AssertTraceDerivedFindings, type AuditIntentInput, type AuditIntentOptions, type AuthorStrategyOptions, type AuthoredProfile, type AuthoredStrategy, type BenchmarkCell, type BenchmarkConfig, type BenchmarkLift, type BenchmarkReport, type BenchmarkStrategySummary, type BenchmarkTaskRow, Budget, type BudgetPool, type BudgetReadout, type ChampionPick, type ChampionPolicy, type CheckpointCapableBox, type CliBridgeConfig, CombinatorShape, type CompletionAnalyst, type CompletionEvidence, type CompletionPolicy, type CompletionVerdict, CoordinationEvent, type CoordinationMcpHandle, Corpus, CorpusFilter, CorpusRecord, type CreateScopeAnalystOptions, type CriuCapableClient, DefinePersonaInput, type DelegateOptions, type Deliverable, DeliverableSpec, type DriveHarness, Driver, type DriverAgentOptions, type DumbDriverOptions, type Environment, EqualKArm, EqualKOnCostOptions, EqualKVerdict, type EvolutionArchiveNode, type EvolutionAuthor, type EvolutionBandInfo, type EvolutionCandidate, type EvolutionGeneration, type EvolutionReport, ExecCtx, ExecutorConfig, ExecutorFactory, ExecutorRegistry, FanoutOptions, FanoutWinnerSelector, FileCorpus, type ForkCapableBox, type GitWorkspaceOptions, type HarvestCorpusOptions, type HarvestFailure, type HarvestReport, InMemoryCorpus, InMemoryResultBlobStore, type InMemoryRunContext, type InMemoryRunContextOptions, InMemorySpawnJournal, type InProcessOnPrompt, type InProcessPromptCtx, type InProcessSandboxClientOptions, type Inbox, type InboxMessage, type IntentAudit, type Interval, Iteration, type Leaderboard, type LeaderboardOptions, type LeaderboardRow, type LoopCampaignDispatchOptions, type LoopDispatchOptions, LoopLineageOptions, type LoopOptionsForDispatch, LoopResult, LoopShape, LoopTokenUsage, LoopUntilSpec, LoopWinner, MakeWorkerAgent, type McpEndpoint, type McpEnvironmentOptions, MountRecorder, type NaiveDriverOptions, type Observation, type ObserveInput, type ObserveOptions, type OpenSandboxRunOptions, Outcome, OutputAdapter, type PairwiseOptions, type PairwiseVerdict, PanelSpec, Persona, PipelineStage, type ProfileRichness, type ProfileRichnessThresholds, type PromotionGateOptions, type PromotionVerdict, type RegistryAnalyzeProjection, RenderCorpusToInstructionsOptions, type ReservationTicket, ResultBlobStore, RouterConfig, type RunAgenticOptions, RunPersonifiedOptions, type SandboxCapabilities, SandboxClient, type SandboxLineage, type SandboxLineageHandle, type SandboxRun, SandboxRunAbortError, Scope, ScopeAnalyst, ScopeAnalyzeInput, ScopeWidenGate, type SessionCapableBox, type SessionTraceBox, Settled, ShapeRegistry, type Shell, type ShotPersona, type ShotSpec, Spend, SteerContext, type SteeringDecision, type Strategy, type StrategyCtx, type StrategyEvolutionConfig, type StrategyResult, type SuperviseOptions, type SuperviseSurfaceOptions, type SuperviseSurfaceResult, SupervisedResult, Supervisor, type SupervisorAgentDeps, type SupervisorProfile, type SurfaceScore, type SurfaceWorkerConfig, type SurfaceWorkerOut, ToolLoopChat, ToolLoopCompactionOptions, type TraceSource, type TrajectoryAnalysis, TrajectoryReport, TrajectoryReportOptions, type TurnResult, UsageEvent, type UsageSink, Validator, type VerifierEnvironmentOptions, VerifySpec, type WatchTraceOptions, type WaterfallCollector, type WaterfallReport, type WaterfallSpan, WidenSpec, WinnerStrategy, type Workspace, type WorkspaceCommit, type WorkspaceRun, acquireSandbox, adaptiveRefine, analyzeTrace, anytimeReport, asAuthoredProfile, assertModelAllowed, assertStrategyContract, assertTraceDerivedFindings, assessAuthoredProfile, auditIntent, authorStrategy, authoredWorker, breadthStrategy, buildSteerContext, builtinShapes, cliBridgeSandboxClient, completionAuthorizes, contentAddress, createBudgetPool, createInMemoryRunContext, createInbox, createMcpEnvironment, createPushTraceSource, createSandboxLineage, createScope, createScopeAnalyst, createShapeRegistry, createSupervisor, createVerifierEnvironment, createWaterfallCollector, decodeToolPart, defaultAnalystInstruction, defaultAuditorInstruction, defaultDelegateBudget, defaultProfileRichnessThresholds, defaultSelectWinner, defaultToolDetectors, definePersona, defineStrategy, delegate, depthStrategy, deterministicCompletion, discriminatingMeans, driverAgent, dumbDriver, equalKOnCost, extractLlmCallEvent, failuresAnalyst, fanout, finalizeBestDelivered, flatWidenGate, gitWorkspace, harvestCorpus, inProcessSandboxClient, inlineSandboxClient, jjWorkspace, leaderboard, localShell, loopCampaignDispatch, loopDispatch, loopUntil, mapSandboxEvent, naiveDriver, observe, openSandboxRun, pairwiseSignificance, panel, pickChampion, pipeline, printBenchmarkReport, probeSandboxCapabilities, profileRichnessFinding, promotionGate, refine, registerShape, registryScopeAnalyst, renderAnytimeTable, renderCorpusToInstructions, renderLeaderboardHtml, renderLeaderboardMarkdown, renderLeaderboardSvg, renderPairwiseMarkdown, renderReport, reportLoopUsage, runAgentic, runBenchmark, runInWorkspace, runLoop, runPersonified, runStrategyEvolution, sample, sampleThenRefine, sandboxSessionTraceSource, selectChampion, selectValidWinner, sentinelCompletion, serveCoordinationMcp, settledToIteration, spendFromUsageEvents, stopSentinel, strategyAuthorContract, sumSandboxUsage, supervise, superviseSurface, supervisorAgent, supervisorInstructions, trajectoryReport, verify, watchTrace, widen, workerFromBackend };
|
package/dist/loops.js
CHANGED
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
breadthStrategy,
|
|
13
13
|
buildSteerContext,
|
|
14
14
|
builtinShapes,
|
|
15
|
+
cliBridgeSandboxClient,
|
|
15
16
|
completionAuthorizes,
|
|
16
17
|
computeFindingId,
|
|
17
18
|
createMcpEnvironment,
|
|
@@ -31,6 +32,7 @@ import {
|
|
|
31
32
|
discriminatingMeans,
|
|
32
33
|
dumbDriver,
|
|
33
34
|
equalKOnCost,
|
|
35
|
+
failuresAnalyst,
|
|
34
36
|
fanout,
|
|
35
37
|
flatWidenGate,
|
|
36
38
|
gitWorkspace,
|
|
@@ -38,13 +40,16 @@ import {
|
|
|
38
40
|
inProcessSandboxClient,
|
|
39
41
|
inlineSandboxClient,
|
|
40
42
|
jjWorkspace,
|
|
43
|
+
leaderboard,
|
|
41
44
|
localShell,
|
|
45
|
+
loopCampaignDispatch,
|
|
42
46
|
loopDispatch,
|
|
43
47
|
loopUntil,
|
|
44
48
|
makeFinding,
|
|
45
49
|
naiveDriver,
|
|
46
50
|
observe,
|
|
47
51
|
openSandboxRun,
|
|
52
|
+
pairwiseSignificance,
|
|
48
53
|
panel,
|
|
49
54
|
patchDelivered,
|
|
50
55
|
pickChampion,
|
|
@@ -56,6 +61,10 @@ import {
|
|
|
56
61
|
registryScopeAnalyst,
|
|
57
62
|
renderAnytimeTable,
|
|
58
63
|
renderCorpusToInstructions,
|
|
64
|
+
renderLeaderboardHtml,
|
|
65
|
+
renderLeaderboardMarkdown,
|
|
66
|
+
renderLeaderboardSvg,
|
|
67
|
+
renderPairwiseMarkdown,
|
|
59
68
|
renderReport,
|
|
60
69
|
reportLoopUsage,
|
|
61
70
|
runAgentic,
|
|
@@ -71,12 +80,13 @@ import {
|
|
|
71
80
|
sentinelCompletion,
|
|
72
81
|
stopSentinel,
|
|
73
82
|
strategyAuthorContract,
|
|
83
|
+
superviseSurface,
|
|
74
84
|
trajectoryReport,
|
|
75
85
|
verify,
|
|
76
86
|
watchTrace,
|
|
77
87
|
widen,
|
|
78
88
|
worktreeFanout
|
|
79
|
-
} from "./chunk-
|
|
89
|
+
} from "./chunk-QSO2TVDS.js";
|
|
80
90
|
import {
|
|
81
91
|
InMemoryResultBlobStore,
|
|
82
92
|
InMemorySpawnJournal,
|
|
@@ -118,19 +128,20 @@ import {
|
|
|
118
128
|
supervisorAgent,
|
|
119
129
|
supervisorInstructions,
|
|
120
130
|
workerFromBackend
|
|
121
|
-
} from "./chunk-
|
|
122
|
-
import "./chunk-
|
|
131
|
+
} from "./chunk-TPII5AU7.js";
|
|
132
|
+
import "./chunk-3X53HYRW.js";
|
|
123
133
|
import {
|
|
124
134
|
createAgentEnvironmentProviderRegistry,
|
|
125
135
|
providerAsExecutor,
|
|
126
136
|
providerAsSandboxClient,
|
|
127
137
|
resolveAgentEnvironmentProvider,
|
|
128
138
|
sandboxClientAsProvider
|
|
129
|
-
} from "./chunk-
|
|
139
|
+
} from "./chunk-H7IBHAFT.js";
|
|
130
140
|
import {
|
|
131
141
|
extractLlmCallEvent,
|
|
132
|
-
mapSandboxEvent
|
|
133
|
-
|
|
142
|
+
mapSandboxEvent,
|
|
143
|
+
sumSandboxUsage
|
|
144
|
+
} from "./chunk-4J6RBI3K.js";
|
|
134
145
|
import "./chunk-5JAUQZQA.js";
|
|
135
146
|
import "./chunk-NBV35BR6.js";
|
|
136
147
|
import "./chunk-WIR4HOOJ.js";
|
|
@@ -156,6 +167,7 @@ export {
|
|
|
156
167
|
breadthStrategy,
|
|
157
168
|
buildSteerContext,
|
|
158
169
|
builtinShapes,
|
|
170
|
+
cliBridgeSandboxClient,
|
|
159
171
|
cliWorktreeExecutor,
|
|
160
172
|
completionAuthorizes,
|
|
161
173
|
computeFindingId,
|
|
@@ -194,6 +206,7 @@ export {
|
|
|
194
206
|
dumbDriver,
|
|
195
207
|
equalKOnCost,
|
|
196
208
|
extractLlmCallEvent,
|
|
209
|
+
failuresAnalyst,
|
|
197
210
|
fanout,
|
|
198
211
|
finalizeBestDelivered,
|
|
199
212
|
flatWidenGate,
|
|
@@ -203,7 +216,9 @@ export {
|
|
|
203
216
|
inProcessSandboxClient,
|
|
204
217
|
inlineSandboxClient,
|
|
205
218
|
jjWorkspace,
|
|
219
|
+
leaderboard,
|
|
206
220
|
localShell,
|
|
221
|
+
loopCampaignDispatch,
|
|
207
222
|
loopDispatch,
|
|
208
223
|
loopUntil,
|
|
209
224
|
makeFinding,
|
|
@@ -211,6 +226,7 @@ export {
|
|
|
211
226
|
naiveDriver,
|
|
212
227
|
observe,
|
|
213
228
|
openSandboxRun,
|
|
229
|
+
pairwiseSignificance,
|
|
214
230
|
panel,
|
|
215
231
|
patchDelivered,
|
|
216
232
|
pickChampion,
|
|
@@ -226,6 +242,10 @@ export {
|
|
|
226
242
|
registryScopeAnalyst,
|
|
227
243
|
renderAnytimeTable,
|
|
228
244
|
renderCorpusToInstructions,
|
|
245
|
+
renderLeaderboardHtml,
|
|
246
|
+
renderLeaderboardMarkdown,
|
|
247
|
+
renderLeaderboardSvg,
|
|
248
|
+
renderPairwiseMarkdown,
|
|
229
249
|
renderReport,
|
|
230
250
|
reportLoopUsage,
|
|
231
251
|
resolveAgentEnvironmentProvider,
|
|
@@ -251,7 +271,9 @@ export {
|
|
|
251
271
|
spendFromUsageEvents,
|
|
252
272
|
stopSentinel,
|
|
253
273
|
strategyAuthorContract,
|
|
274
|
+
sumSandboxUsage,
|
|
254
275
|
supervise,
|
|
276
|
+
superviseSurface,
|
|
255
277
|
supervisorAgent,
|
|
256
278
|
supervisorInstructions,
|
|
257
279
|
trajectoryReport,
|
package/dist/mcp/bin.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
readTraceContextFromEnv
|
|
4
|
-
} from "../chunk-
|
|
4
|
+
} from "../chunk-M3Y362RW.js";
|
|
5
5
|
import {
|
|
6
6
|
resolveRouterBaseUrl
|
|
7
7
|
} from "../chunk-GRAGM4MC.js";
|
|
@@ -9,10 +9,10 @@ import {
|
|
|
9
9
|
DelegationTaskQueue,
|
|
10
10
|
FileDelegationStore,
|
|
11
11
|
createMcpServer
|
|
12
|
-
} from "../chunk-
|
|
13
|
-
import "../chunk-
|
|
14
|
-
import "../chunk-
|
|
15
|
-
import "../chunk-
|
|
12
|
+
} from "../chunk-TPII5AU7.js";
|
|
13
|
+
import "../chunk-3X53HYRW.js";
|
|
14
|
+
import "../chunk-H7IBHAFT.js";
|
|
15
|
+
import "../chunk-4J6RBI3K.js";
|
|
16
16
|
import "../chunk-5JAUQZQA.js";
|
|
17
17
|
import "../chunk-NBV35BR6.js";
|
|
18
18
|
import "../chunk-WIR4HOOJ.js";
|
package/dist/mcp/index.d.ts
CHANGED
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
import { S as SandboxClient, d as LoopSandboxPlacement } from '../types-BF-MEsQB.js';
|
|
2
|
-
import { F as FleetHandle, D as DelegationExecutor, a as DelegationTaskQueue, b as FeedbackStore, c as DelegateFeedbackResult, d as DelegateFeedbackArgs, U as UiAuditorDelegate, e as DelegateUiAuditArgs, f as DelegateUiAuditResult, g as DelegationHistoryResult, h as DelegationHistoryArgs, i as DelegationStatusResult, j as DelegationStatusArgs } from '../coordination-
|
|
3
|
-
export { A as AnalystRegistry, C as CappedDelegationTrace, k as CoderDelegate, l as CoderReview, m as CoderReviewer, n as CoordinationEvent, o as CoordinationTools, p as CoordinationToolsOptions, q as DELEGATE_DESCRIPTION, r as DELEGATE_INPUT_SCHEMA, s as DELEGATE_TOOL_NAME, t as DELEGATION_TRACE_MAX_BYTES, u as DELEGATION_TRACE_MAX_SPANS, v as DelegateArgs, w as DelegateCodeArgs, x as DelegateCodeConfig, y as DelegateCodeResult, z as DelegateHandlerOptions, B as DelegateResearchArgs, E as DelegateResearchConfig, G as DelegateResearchResult, H as DelegateResult, I as DelegateRunCtx, J as DelegateUiAuditConfig, K as DelegateUiAuditRoute, L as DelegationError, M as DelegationFeedbackSnapshot, N as DelegationHistoryEntry, O as DelegationPersistenceError, P as DelegationProfile, Q as DelegationProgress, R as DelegationRecord, S as DelegationResultPayload, T as DelegationResumeContext, V as DelegationResumeDriver, W as DelegationResumeTick, X as DelegationRunContext, Y as DelegationStateCorruptError, Z as DelegationStatus, _ as DelegationStore, $ as DelegationTaskQueueOptions, a0 as DelegationTraceCaps, a1 as DelegationTraceCollector, a2 as DelegationTraceSpan, a3 as DetachedSessionDelegateOptions, a4 as DetachedSessionRefParts, a5 as DetachedTurn, a6 as DetachedTurnResumeDriverOptions, a7 as DetachedWinnerSelection, a8 as DriveTurnCapableBox, a9 as DriveTurnTick, aa as FeedbackEvent, ab as FeedbackRating, ac as FeedbackRefersTo, ad as FileDelegationStore, ae as FileDelegationStoreOptions, af as FleetWorkspaceExecutorOptions, ag as InMemoryDelegationStore, ah as InMemoryFeedbackStore, ai as JsonRpcMessage, aj as JsonRpcResponse, ak as MakeWorkerAgent, al as McpServer, am as McpServerOptions, an as McpToolDescriptor, ao as McpTransport, ap as Question, aq as QuestionDecision, ar as QuestionPolicy, as as QuestionRecord, at as ResearchOutputShape, au as ResearchSource, av as RunDetachedTurnOptions, aw as SettleDetachedCoderTurnOptions, ax as SettledWorker, ay as SiblingSandboxExecutorOptions, az as SubmitInput, aA as SubmitOutput, aB as TraceContext, aC as UiAuditorDelegationOutput, aD as buildDelegationTraceSpans, aE as capDelegationTrace, aF as coderTaskFromArgs, aG as composeLoopTraceEmitters, aH as createCoordinationTools, aI as createDelegateHandler, aJ as createDelegationTraceCollector, aK as createDetachedTurnResumeDriver, aL as createFleetWorkspaceExecutor, aM as createInProcessTransport, aN as createMcpServer, aO as createPropagatingTraceEmitter, aP as createSiblingSandboxExecutor, aQ as detachedSessionDelegate, aR as detachedTurnEvents, aS as eventToSnapshot, aT as formatDetachedSessionRef, aU as hashIdempotencyInput, aV as parseDetachedSessionRef, aW as readTraceContextFromEnv, aX as runDetachedTurn, aY as settleDetachedCoderTurn, aZ as traceContextToEnv, a_ as validateDelegateArgs } from '../coordination-
|
|
2
|
+
import { F as FleetHandle, D as DelegationExecutor, a as DelegationTaskQueue, b as FeedbackStore, c as DelegateFeedbackResult, d as DelegateFeedbackArgs, U as UiAuditorDelegate, e as DelegateUiAuditArgs, f as DelegateUiAuditResult, g as DelegationHistoryResult, h as DelegationHistoryArgs, i as DelegationStatusResult, j as DelegationStatusArgs } from '../coordination-BI9tpcmF.js';
|
|
3
|
+
export { A as AnalystRegistry, C as CappedDelegationTrace, k as CoderDelegate, l as CoderReview, m as CoderReviewer, n as CoordinationEvent, o as CoordinationTools, p as CoordinationToolsOptions, q as DELEGATE_DESCRIPTION, r as DELEGATE_INPUT_SCHEMA, s as DELEGATE_TOOL_NAME, t as DELEGATION_TRACE_MAX_BYTES, u as DELEGATION_TRACE_MAX_SPANS, v as DelegateArgs, w as DelegateCodeArgs, x as DelegateCodeConfig, y as DelegateCodeResult, z as DelegateHandlerOptions, B as DelegateResearchArgs, E as DelegateResearchConfig, G as DelegateResearchResult, H as DelegateResult, I as DelegateRunCtx, J as DelegateUiAuditConfig, K as DelegateUiAuditRoute, L as DelegationError, M as DelegationFeedbackSnapshot, N as DelegationHistoryEntry, O as DelegationPersistenceError, P as DelegationProfile, Q as DelegationProgress, R as DelegationRecord, S as DelegationResultPayload, T as DelegationResumeContext, V as DelegationResumeDriver, W as DelegationResumeTick, X as DelegationRunContext, Y as DelegationStateCorruptError, Z as DelegationStatus, _ as DelegationStore, $ as DelegationTaskQueueOptions, a0 as DelegationTraceCaps, a1 as DelegationTraceCollector, a2 as DelegationTraceSpan, a3 as DetachedSessionDelegateOptions, a4 as DetachedSessionRefParts, a5 as DetachedTurn, a6 as DetachedTurnResumeDriverOptions, a7 as DetachedWinnerSelection, a8 as DriveTurnCapableBox, a9 as DriveTurnTick, aa as FeedbackEvent, ab as FeedbackRating, ac as FeedbackRefersTo, ad as FileDelegationStore, ae as FileDelegationStoreOptions, af as FleetWorkspaceExecutorOptions, ag as InMemoryDelegationStore, ah as InMemoryFeedbackStore, ai as JsonRpcMessage, aj as JsonRpcResponse, ak as MakeWorkerAgent, al as McpServer, am as McpServerOptions, an as McpToolDescriptor, ao as McpTransport, ap as Question, aq as QuestionDecision, ar as QuestionPolicy, as as QuestionRecord, at as ResearchOutputShape, au as ResearchSource, av as RunDetachedTurnOptions, aw as SettleDetachedCoderTurnOptions, ax as SettledWorker, ay as SiblingSandboxExecutorOptions, az as SubmitInput, aA as SubmitOutput, aB as TraceContext, aC as UiAuditorDelegationOutput, aD as buildDelegationTraceSpans, aE as capDelegationTrace, aF as coderTaskFromArgs, aG as composeLoopTraceEmitters, aH as createCoordinationTools, aI as createDelegateHandler, aJ as createDelegationTraceCollector, aK as createDetachedTurnResumeDriver, aL as createFleetWorkspaceExecutor, aM as createInProcessTransport, aN as createMcpServer, aO as createPropagatingTraceEmitter, aP as createSiblingSandboxExecutor, aQ as detachedSessionDelegate, aR as detachedTurnEvents, aS as eventToSnapshot, aT as formatDetachedSessionRef, aU as hashIdempotencyInput, aV as parseDetachedSessionRef, aW as readTraceContextFromEnv, aX as runDetachedTurn, aY as settleDetachedCoderTurn, aZ as traceContextToEnv, a_ as validateDelegateArgs } from '../coordination-BI9tpcmF.js';
|
|
4
4
|
import { L as LocalHarness, r as runLocalHarness } from '../local-harness-DU7yV6mG.js';
|
|
5
5
|
export { a as LocalHarnessResult, R as RunLocalHarnessOptions } from '../local-harness-DU7yV6mG.js';
|
|
6
|
-
import { G as GitRunner } from '../worktree-harness-
|
|
7
|
-
export { C as CreateWorktreeOptions, b as DiffOptions, c as DiffResult, R as RemoveWorktreeOptions, d as WorktreeHandle, e as captureWorktreeDiff, f as createWorktree, r as removeWorktree } from '../worktree-harness-
|
|
6
|
+
import { G as GitRunner } from '../worktree-harness-Dx8XULW3.js';
|
|
7
|
+
export { C as CreateWorktreeOptions, b as DiffOptions, c as DiffResult, R as RemoveWorktreeOptions, d as WorktreeHandle, e as captureWorktreeDiff, f as createWorktree, r as removeWorktree } from '../worktree-harness-Dx8XULW3.js';
|
|
8
8
|
export { C as CreateKbGateOptions, F as FactCandidate, a as FactJudge, b as FactJudgeVerdict, K as KbGateResult, c as createKbGate } from '../kb-gate-CuzMYGYM.js';
|
|
9
9
|
export { m as mcpToolsForRuntimeMcp, a as mcpToolsForRuntimeMcpSubset } from '../openai-tools-B-3v06BE.js';
|
|
10
10
|
import { AnalystFinding } from '@tangle-network/agent-eval';
|
|
11
11
|
import { U as UiLens } from '../substrate-rNj6TDc3.js';
|
|
12
12
|
import '@tangle-network/sandbox';
|
|
13
|
-
import '../types-
|
|
13
|
+
import '../types-By9LXllv.js';
|
|
14
14
|
import '@tangle-network/agent-interface';
|
|
15
15
|
import '../otel-export-BKmNwiCb.js';
|
|
16
|
-
import '../router-client-
|
|
16
|
+
import '../router-client-D6Ocf4jG.js';
|
|
17
17
|
import '../environment-provider.js';
|
|
18
18
|
import '@tangle-network/agent-interface/environment-provider';
|
|
19
19
|
import 'node:child_process';
|
package/dist/mcp/index.js
CHANGED
|
@@ -2,11 +2,11 @@ import {
|
|
|
2
2
|
createPropagatingTraceEmitter,
|
|
3
3
|
readTraceContextFromEnv,
|
|
4
4
|
traceContextToEnv
|
|
5
|
-
} from "../chunk-
|
|
5
|
+
} from "../chunk-M3Y362RW.js";
|
|
6
6
|
import {
|
|
7
7
|
mcpToolsForRuntimeMcp,
|
|
8
8
|
mcpToolsForRuntimeMcpSubset
|
|
9
|
-
} from "../chunk-
|
|
9
|
+
} from "../chunk-LLJRUTE7.js";
|
|
10
10
|
import {
|
|
11
11
|
createKbGate
|
|
12
12
|
} from "../chunk-FNMGYYSS.js";
|
|
@@ -14,7 +14,7 @@ import {
|
|
|
14
14
|
assertTraceDerivedFindings,
|
|
15
15
|
runCoderChecks,
|
|
16
16
|
selectValidWinner
|
|
17
|
-
} from "../chunk-
|
|
17
|
+
} from "../chunk-QSO2TVDS.js";
|
|
18
18
|
import {
|
|
19
19
|
DELEGATE_DESCRIPTION,
|
|
20
20
|
DELEGATE_FEEDBACK_DESCRIPTION,
|
|
@@ -64,15 +64,15 @@ import {
|
|
|
64
64
|
validateDelegateUiAuditArgs,
|
|
65
65
|
validateDelegationHistoryArgs,
|
|
66
66
|
validateDelegationStatusArgs
|
|
67
|
-
} from "../chunk-
|
|
68
|
-
import "../chunk-
|
|
67
|
+
} from "../chunk-TPII5AU7.js";
|
|
68
|
+
import "../chunk-3X53HYRW.js";
|
|
69
69
|
import {
|
|
70
70
|
deleteBoxSafe,
|
|
71
71
|
sleep,
|
|
72
72
|
throwAbort,
|
|
73
73
|
throwIfAborted
|
|
74
|
-
} from "../chunk-
|
|
75
|
-
import "../chunk-
|
|
74
|
+
} from "../chunk-H7IBHAFT.js";
|
|
75
|
+
import "../chunk-4J6RBI3K.js";
|
|
76
76
|
import {
|
|
77
77
|
runLocalHarness
|
|
78
78
|
} from "../chunk-5JAUQZQA.js";
|
|
@@ -82,7 +82,17 @@ interface RouterConfig {
|
|
|
82
82
|
complete?: (body: Record<string, unknown>) => Promise<unknown>;
|
|
83
83
|
}
|
|
84
84
|
interface RouterChatResult {
|
|
85
|
+
/** The final answer, with any inline `<think>...</think>` block stripped into `reasoning`. */
|
|
85
86
|
content: string;
|
|
87
|
+
/**
|
|
88
|
+
* Thinking-model reasoning, when the provider surfaced it — either as a separate
|
|
89
|
+
* `reasoning`/`reasoning_content` message field (OpenRouter style) or inlined into
|
|
90
|
+
* `content` as a `<think>` block (Groq style). Undefined for non-thinking models.
|
|
91
|
+
* Downstream parsers that match single-token answers must read `content`, which is
|
|
92
|
+
* clean either way; before this split, Groq-style inlining made the same model look
|
|
93
|
+
* broken on one provider and fine on another.
|
|
94
|
+
*/
|
|
95
|
+
reasoning?: string;
|
|
86
96
|
/** REAL usage, or undefined when the provider reported none. */
|
|
87
97
|
usage?: {
|
|
88
98
|
input: number;
|
|
@@ -98,6 +108,15 @@ declare function routerChatWithUsage(cfg: RouterConfig, messages: Array<{
|
|
|
98
108
|
temperature?: number;
|
|
99
109
|
signal?: AbortSignal;
|
|
100
110
|
maxTokens?: number;
|
|
111
|
+
/**
|
|
112
|
+
* Reasoning control for thinking models, forwarded as `reasoning_effort`.
|
|
113
|
+
* 'none' is the load-bearing value: binary/single-token decisions (routing,
|
|
114
|
+
* gating) on a thinking model otherwise burn the whole token budget inside
|
|
115
|
+
* the think block — on slow backends (CPU-local) that turns into a client
|
|
116
|
+
* timeout, not just waste. Providers that ignore the field are handled by
|
|
117
|
+
* the reasoning/content split in `parseChatResult`.
|
|
118
|
+
*/
|
|
119
|
+
reasoningEffort?: 'none' | 'low' | 'medium' | 'high';
|
|
101
120
|
}): Promise<RouterChatResult>;
|
|
102
121
|
/** A tool-call the model emitted (provider-neutral; mirrors the runtime's ToolCallRequest). */
|
|
103
122
|
interface RouterToolCall {
|