@tangle-network/agent-runtime 0.79.3 → 0.79.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -327
- package/dist/agent.js +5 -5
- package/dist/analyst-loop.js +2 -2
- package/dist/{chunk-KRULXIWS.js → chunk-44CX5JU6.js} +4 -4
- package/dist/{chunk-T2HVQVB4.js → chunk-4J6RBI3K.js} +15 -1
- package/dist/chunk-4J6RBI3K.js.map +1 -0
- package/dist/{chunk-AG335EXG.js → chunk-AHZ3YBL6.js} +524 -12
- package/dist/chunk-AHZ3YBL6.js.map +1 -0
- package/dist/{chunk-IODKUOBA.js → chunk-C2FZ6GR6.js} +2 -2
- package/dist/{chunk-Z3RRRPRB.js → chunk-H7IBHAFT.js} +23 -14
- package/dist/chunk-H7IBHAFT.js.map +1 -0
- package/dist/{chunk-VMNEQHJR.js → chunk-IPEQ3ERC.js} +17 -2
- package/dist/chunk-IPEQ3ERC.js.map +1 -0
- package/dist/{chunk-PBE35ULD.js → chunk-NXZEVWKP.js} +2 -2
- package/dist/{chunk-PVPFDTO3.js → chunk-RUJZK6VH.js} +2 -2
- package/dist/{chunk-63MHOCIE.js → chunk-U6AP535M.js} +4 -4
- package/dist/{coordination-DCmljYDf.d.ts → coordination-BFVtgRax.d.ts} +2 -2
- package/dist/environment-provider.d.ts +1 -1
- package/dist/environment-provider.js +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.js +8 -8
- package/dist/intelligence.d.ts +29 -1
- package/dist/intelligence.js +33 -23
- package/dist/intelligence.js.map +1 -1
- package/dist/{loop-runner-bin-C4X0FZ2Z.d.ts → loop-runner-bin-CTVja8e0.d.ts} +2 -2
- package/dist/loop-runner-bin.d.ts +4 -4
- package/dist/loop-runner-bin.js +7 -7
- package/dist/loops.d.ts +242 -11
- package/dist/loops.js +26 -6
- package/dist/mcp/bin.js +5 -5
- package/dist/mcp/index.d.ts +5 -5
- package/dist/mcp/index.js +7 -7
- package/dist/{types-C1sozrte.d.ts → types-DYW0tloU.d.ts} +2 -2
- package/dist/{worktree-fanout-CXGzHET4.d.ts → worktree-fanout-DaUDwCA_.d.ts} +2 -2
- package/dist/{worktree-harness-Bmho9SH0.d.ts → worktree-harness-Dt6s_m3z.d.ts} +1 -1
- package/package.json +1 -1
- package/skills/build-with-agent-runtime/SKILL.md +53 -23
- package/dist/chunk-AG335EXG.js.map +0 -1
- package/dist/chunk-T2HVQVB4.js.map +0 -1
- package/dist/chunk-VMNEQHJR.js.map +0 -1
- package/dist/chunk-Z3RRRPRB.js.map +0 -1
- /package/dist/{chunk-KRULXIWS.js.map → chunk-44CX5JU6.js.map} +0 -0
- /package/dist/{chunk-IODKUOBA.js.map → chunk-C2FZ6GR6.js.map} +0 -0
- /package/dist/{chunk-PBE35ULD.js.map → chunk-NXZEVWKP.js.map} +0 -0
- /package/dist/{chunk-PVPFDTO3.js.map → chunk-RUJZK6VH.js.map} +0 -0
- /package/dist/{chunk-63MHOCIE.js.map → chunk-U6AP535M.js.map} +0 -0
package/dist/loops.d.ts
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
|
-
import { ChatClient, AnalystFinding, AgentProfile, AnalystRunInputs, ToolSpan, StreamingDetector, DetectorSignal, buildTrajectory } from '@tangle-network/agent-eval';
|
|
1
|
+
import { ChatClient, RunRecord, AnalystFinding, AgentProfile, AnalystRunInputs, ToolSpan, StreamingDetector, DetectorSignal, buildTrajectory } from '@tangle-network/agent-eval';
|
|
2
2
|
export { AnalystFinding, DefaultVerdict, computeFindingId, makeFinding } from '@tangle-network/agent-eval';
|
|
3
3
|
import { SandboxEvent, SandboxInstance, CreateSandboxOptions } from '@tangle-network/sandbox';
|
|
4
4
|
export { AgentProfile, CreateSandboxOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox';
|
|
5
|
-
import { d as ResultBlobStore, S as SpawnJournal, N as NodeId, j as SpawnEvent, E as ExecutorFactory, c as Agent, B as Budget, i as Scope, g as Settled, f as SupervisedResult, h as Spend, U as UsageEvent, b as ExecutorRegistry, k as Supervisor } from './types-
|
|
6
|
-
export { A as AgentSpec, a as Executor, l as ExecutorContext, m as ExecutorResult, n as SupervisorOpts, T as TreeView, W as WidenGate } from './types-
|
|
7
|
-
import { ak as MakeWorkerAgent, A as AnalystRegistry, o as CoordinationTools, n as CoordinationEvent, ar as QuestionPolicy, a$ as ExecutorConfig } from './coordination-
|
|
8
|
-
export { b0 as BusEvent, b1 as BusRecord, b2 as BusStats, b3 as EventBus, b4 as ProviderSeam, b5 as PublishOptions, b6 as cliWorktreeExecutor, b7 as createEventBus, b8 as createExecutor, b9 as createExecutorRegistry } from './coordination-
|
|
5
|
+
import { d as ResultBlobStore, S as SpawnJournal, N as NodeId, j as SpawnEvent, E as ExecutorFactory, c as Agent, B as Budget, i as Scope, g as Settled, f as SupervisedResult, h as Spend, U as UsageEvent, b as ExecutorRegistry, k as Supervisor } from './types-DYW0tloU.js';
|
|
6
|
+
export { A as AgentSpec, a as Executor, l as ExecutorContext, m as ExecutorResult, R as Runtime, n as SupervisorOpts, T as TreeView, W as WidenGate } from './types-DYW0tloU.js';
|
|
7
|
+
import { ak as MakeWorkerAgent, A as AnalystRegistry, o as CoordinationTools, n as CoordinationEvent, ar as QuestionPolicy, a$ as ExecutorConfig } from './coordination-BFVtgRax.js';
|
|
8
|
+
export { b0 as BusEvent, b1 as BusRecord, b2 as BusStats, b3 as EventBus, b4 as ProviderSeam, b5 as PublishOptions, b6 as cliWorktreeExecutor, b7 as createEventBus, b8 as createExecutor, b9 as createExecutorRegistry } from './coordination-BFVtgRax.js';
|
|
9
9
|
import { R as RuntimeHooks, I as Iteration, S as SandboxClient, W as Driver, A as AgentRunSpec, b as OutputAdapter, V as Validator, E as ExecCtx, X as LoopWinner, Y as LoopLineageOptions, Z as LoopResult, L as LoopTokenUsage, a as RuntimeStreamEvent, _ as MountRecorder } from './types-BF-MEsQB.js';
|
|
10
10
|
export { $ as LoopDecisionPayload, a0 as LoopEndedPayload, a1 as LoopIterationDispatchPayload, a2 as LoopIterationEndedPayload, a3 as LoopIterationStartedPayload, a4 as LoopPlanDescription, a5 as LoopPlanPayload, d as LoopSandboxPlacement, a6 as LoopStartedPayload, a7 as LoopTeardownFailedPayload, e as LoopTraceEmitter, c as LoopTraceEvent, a8 as MountManifestEntry, a9 as RunProvenance, aa as SelectionReceipt, ab as ValidationCtx } from './types-BF-MEsQB.js';
|
|
11
11
|
export { AgentEnvironmentProviderRef, AgentEnvironmentProviderRegistry, ProviderAsSandboxClientOptions, ProviderExecutorOptions, SandboxClientProviderOptions, createAgentEnvironmentProviderRegistry, providerAsExecutor, providerAsSandboxClient, resolveAgentEnvironmentProvider, sandboxClientAsProvider } from './environment-provider.js';
|
|
12
|
-
import { C as CorpusRecord, c as Corpus, O as Outcome, S as ScopeAnalyzeInput, d as AssertTraceDerivedFindings, e as SteerContext, f as ScopeAnalyst, F as FanoutOptions, g as CombinatorShape, h as ScopeWidenGate, L as LoopUntilSpec, P as PanelSpec, i as PipelineStage, W as WinnerStrategy, j as FanoutWinnerSelector, V as VerifySpec, k as WidenSpec, l as CorpusFilter, R as RenderCorpusToInstructionsOptions, D as DefinePersonaInput, m as Persona, n as RunPersonifiedOptions, o as ShapeRegistry, p as LoopShape, E as EqualKArm, q as EqualKOnCostOptions, r as EqualKVerdict, T as TrajectoryReportOptions, s as TrajectoryReport } from './worktree-fanout-
|
|
13
|
-
export { A as AuthoredHarness, t as DefinePersona, u as EqualKOnCost, v as Fanout, w as FanoutSynthesis, x as FlatWidenGate, y as LoopUntil, z as LoopUntilState, B as Panel, G as PanelJudge, H as PanelVerdict, I as PatchDeliverableOptions, J as PersonaContext, K as PersonaExecutors, M as Pipeline, N as RenderCorpusToInstructions, Q as RunPersonified, U as ShapeBudget, X as ShapeContext, Y as TrajectoryNode, Z as TrajectoryReportFn, _ as Verify, $ as Widen, a0 as WidenDecision, a1 as WidenLineage, a2 as WorktreeCliExecutorOptions, a as WorktreeFanoutOptions, b as WorktreePatchArtifact, a3 as createWorktreeCliExecutor, a4 as patchDelivered, a5 as worktreeFanout } from './worktree-fanout-
|
|
14
|
-
import { Scenario, ProfileDispatchFn } from '@tangle-network/agent-eval/campaign';
|
|
12
|
+
import { C as CorpusRecord, c as Corpus, O as Outcome, S as ScopeAnalyzeInput, d as AssertTraceDerivedFindings, e as SteerContext, f as ScopeAnalyst, F as FanoutOptions, g as CombinatorShape, h as ScopeWidenGate, L as LoopUntilSpec, P as PanelSpec, i as PipelineStage, W as WinnerStrategy, j as FanoutWinnerSelector, V as VerifySpec, k as WidenSpec, l as CorpusFilter, R as RenderCorpusToInstructionsOptions, D as DefinePersonaInput, m as Persona, n as RunPersonifiedOptions, o as ShapeRegistry, p as LoopShape, E as EqualKArm, q as EqualKOnCostOptions, r as EqualKVerdict, T as TrajectoryReportOptions, s as TrajectoryReport } from './worktree-fanout-DaUDwCA_.js';
|
|
13
|
+
export { A as AuthoredHarness, t as DefinePersona, u as EqualKOnCost, v as Fanout, w as FanoutSynthesis, x as FlatWidenGate, y as LoopUntil, z as LoopUntilState, B as Panel, G as PanelJudge, H as PanelVerdict, I as PatchDeliverableOptions, J as PersonaContext, K as PersonaExecutors, M as Pipeline, N as RenderCorpusToInstructions, Q as RunPersonified, U as ShapeBudget, X as ShapeContext, Y as TrajectoryNode, Z as TrajectoryReportFn, _ as Verify, $ as Widen, a0 as WidenDecision, a1 as WidenLineage, a2 as WorktreeCliExecutorOptions, a as WorktreeFanoutOptions, b as WorktreePatchArtifact, a3 as createWorktreeCliExecutor, a4 as patchDelivered, a5 as worktreeFanout } from './worktree-fanout-DaUDwCA_.js';
|
|
14
|
+
import { Scenario, DispatchFn, ProfileDispatchFn } from '@tangle-network/agent-eval/campaign';
|
|
15
15
|
import { b as AnalystRegistryLike } from './types-BC3bZpH0.js';
|
|
16
16
|
import { AgentProfile as AgentProfile$1 } from '@tangle-network/agent-interface';
|
|
17
17
|
import { R as RouterConfig, a as ToolLoopChat, b as ToolLoopCompactionOptions } from './router-client-Ak2IGuXq.js';
|
|
18
18
|
export { c as RouterChatResult, d as RouterChatToolsResult, e as RouterToolCall, f as RouterToolLoopResult, g as ToolLoopCompaction, T as ToolSpec, r as routerBrain, h as routerChatWithTools, i as routerChatWithUsage, j as routerToolLoop } from './router-client-Ak2IGuXq.js';
|
|
19
|
-
import { D as DeliverableSpec } from './worktree-harness-
|
|
20
|
-
export { g as WorktreeCommandResult, h as gateOnDeliverable } from './worktree-harness-
|
|
19
|
+
import { D as DeliverableSpec } from './worktree-harness-Dt6s_m3z.js';
|
|
20
|
+
export { g as WorktreeCommandResult, h as gateOnDeliverable } from './worktree-harness-Dt6s_m3z.js';
|
|
21
21
|
import { stuckLoopView, toolWasteView } from '@tangle-network/agent-eval/pipelines';
|
|
22
22
|
export { AgentEnvironment, AgentEnvironmentCapabilities, AgentEnvironmentEvent, AgentEnvironmentProvider, AgentEnvironmentQuery, AgentEnvironmentStatus, AgentEnvironmentSummary, AgentProfileRef, AgentSession, AgentSessionRef, AgentSessionStatus, AgentTurnInput, AgentTurnResult, CheckpointRef, CheckpointRequest, CreateAgentEnvironmentInput, ExecRequest, ExecResult, ForkRequest, PlacementInfo, ResourceRequest, WorkspaceRequest } from '@tangle-network/agent-interface/environment-provider';
|
|
23
23
|
import './substrate-rNj6TDc3.js';
|
|
@@ -276,6 +276,139 @@ interface IntentAudit {
|
|
|
276
276
|
declare const defaultAuditorInstruction: string;
|
|
277
277
|
declare function auditIntent(input: AuditIntentInput, opts: AuditIntentOptions): Promise<IntentAudit>;
|
|
278
278
|
|
|
279
|
+
/**
|
|
280
|
+
* benchmark-report — turn a fleet of `RunRecord`s into a publishable, multi-axis benchmark report:
|
|
281
|
+
* a ranked leaderboard, the full profile×axis score matrix, the cost/latency/token columns, and
|
|
282
|
+
* embeddable charts (SVG) + a self-contained HTML page. Domain-agnostic by construction — it reads ONLY
|
|
283
|
+
* the universal `RunRecord` currency (model, scenario, score, cost, tokens, latency, the `outcome.raw`
|
|
284
|
+
* metric bag), so the SAME engine reports any benchmark in any domain: coding, search, agents, multimodal.
|
|
285
|
+
*
|
|
286
|
+
* This is the surface a hosted leaderboard (à la vals.ai) renders: for every harness×model profile, its
|
|
287
|
+
* score on every axis, not a curated subset. It pairs with `runProfileMatrix` (whose `result.records`
|
|
288
|
+
* feed straight in) but takes the records directly, so it is independent of how they were produced.
|
|
289
|
+
*
|
|
290
|
+
* An AXIS is any way to slice the score into columns — by default one column per scenario group
|
|
291
|
+
* (`groupOf`, default = the scenario id), so the matrix is profile × scenario. Pass `axisScoresOf` to
|
|
292
|
+
* score along judge dimensions or any custom decomposition instead. The reporter never invents a number:
|
|
293
|
+
* a missing cell renders blank, never zero.
|
|
294
|
+
*/
|
|
295
|
+
|
|
296
|
+
/** Pull the headline score in [0,1] from a record. Default: the held-out split, else the search split,
|
|
297
|
+
* else a `composite`/`passed`/`score` entry in the raw bag. Override to score a domain differently. */
|
|
298
|
+
type ScoreOf = (record: RunRecord) => number | undefined;
|
|
299
|
+
/** The profile (matrix row) a record belongs to — default `harness·model` from the record's profile cell,
|
|
300
|
+
* falling back to the model. This is the leaderboard's unit of comparison. */
|
|
301
|
+
type ProfileKeyOf = (record: RunRecord) => string;
|
|
302
|
+
/** The axis (matrix column) a record contributes to — default the scenario group. */
|
|
303
|
+
type GroupOf = (record: RunRecord) => string;
|
|
304
|
+
/** Decompose ONE record into per-axis scores (e.g. judge dimensions). When set, it REPLACES the
|
|
305
|
+
* scenario-group axes: the column set is the union of returned keys. */
|
|
306
|
+
type AxisScoresOf = (record: RunRecord) => Record<string, number>;
|
|
307
|
+
interface LeaderboardOptions {
|
|
308
|
+
readonly title?: string;
|
|
309
|
+
readonly scoreOf?: ScoreOf;
|
|
310
|
+
readonly profileKeyOf?: ProfileKeyOf;
|
|
311
|
+
readonly groupOf?: GroupOf;
|
|
312
|
+
readonly axisScoresOf?: AxisScoresOf;
|
|
313
|
+
/** Display label for a profile key (default: the key itself). */
|
|
314
|
+
readonly labelOf?: (profileKey: string) => string;
|
|
315
|
+
/** Commit SHA / dataset / dates surfaced in the provenance block. */
|
|
316
|
+
readonly meta?: Record<string, string>;
|
|
317
|
+
/** Compute per-row confidence intervals (bootstrap on score, Wilson on pass rate). Needs a
|
|
318
|
+
* `scenarioId` on every record (reps are collapsed per scenario for the honest n). Default off. */
|
|
319
|
+
readonly stats?: boolean;
|
|
320
|
+
/** A score ≥ this counts as a "pass" for the pass-rate proportion + its Wilson CI. Default 0.999
|
|
321
|
+
* (fully solved). Lower it (e.g. 0.6) for a partial-credit domain. */
|
|
322
|
+
readonly passThreshold?: number;
|
|
323
|
+
}
|
|
324
|
+
/** A 95%-by-default confidence interval. */
|
|
325
|
+
interface Interval {
|
|
326
|
+
readonly lower: number;
|
|
327
|
+
readonly upper: number;
|
|
328
|
+
}
|
|
329
|
+
/** One leaderboard row — a harness×model profile, every measured column. */
|
|
330
|
+
interface LeaderboardRow {
|
|
331
|
+
readonly profileKey: string;
|
|
332
|
+
readonly label: string;
|
|
333
|
+
readonly model: string;
|
|
334
|
+
readonly n: number;
|
|
335
|
+
readonly meanScore: number;
|
|
336
|
+
/** Fraction of records scoring ≥ `passThreshold` (default 0.999) — the binary pass rate. */
|
|
337
|
+
readonly solveRate: number;
|
|
338
|
+
/** axis → mean score for this profile (blank in render when the profile never ran that axis). */
|
|
339
|
+
readonly perAxis: Record<string, number>;
|
|
340
|
+
readonly costUsd: number;
|
|
341
|
+
readonly tokensIn: number;
|
|
342
|
+
readonly tokensOut: number;
|
|
343
|
+
readonly latencyP50Ms: number;
|
|
344
|
+
readonly latencyP90Ms: number;
|
|
345
|
+
/** Bootstrap CI on the mean score — present only when `opts.stats` is set. Computed over
|
|
346
|
+
* per-scenario means (reps collapsed first), so identical reps can't fake a narrow interval. */
|
|
347
|
+
readonly scoreCi?: Interval;
|
|
348
|
+
/** Wilson CI on the pass rate — present only when `opts.stats` is set. */
|
|
349
|
+
readonly passCi?: Interval;
|
|
350
|
+
}
|
|
351
|
+
interface Leaderboard {
|
|
352
|
+
readonly title: string;
|
|
353
|
+
/** Column order — scenario groups (default) or dimension keys (`axisScoresOf`). */
|
|
354
|
+
readonly axes: readonly string[];
|
|
355
|
+
/** Rows ranked by `meanScore` desc (ties → lower cost, then label). */
|
|
356
|
+
readonly profiles: readonly LeaderboardRow[];
|
|
357
|
+
readonly meta: Record<string, string>;
|
|
358
|
+
/** Provenance counts — the denominators every honest report leads with. */
|
|
359
|
+
readonly provenance: {
|
|
360
|
+
readonly records: number;
|
|
361
|
+
readonly profiles: number;
|
|
362
|
+
readonly axes: number;
|
|
363
|
+
readonly models: readonly string[];
|
|
364
|
+
readonly totalCostUsd: number;
|
|
365
|
+
};
|
|
366
|
+
}
|
|
367
|
+
/** Aggregate a fleet of records into the ranked, multi-axis report. Pure — no IO, deterministic. */
|
|
368
|
+
declare function leaderboard(records: readonly RunRecord[], opts?: LeaderboardOptions): Leaderboard;
|
|
369
|
+
/** One profile pair compared on the scenarios they BOTH ran — the "who actually beat whom" verdict. */
|
|
370
|
+
interface PairwiseVerdict {
|
|
371
|
+
readonly a: string;
|
|
372
|
+
readonly b: string;
|
|
373
|
+
/** Paired unit count (shared scenarios). The significance is suppressed below `minPairs`. */
|
|
374
|
+
readonly pairs: number;
|
|
375
|
+
/** Median paired delta (b − a) and its bootstrap CI. */
|
|
376
|
+
readonly delta: number;
|
|
377
|
+
readonly ciLow: number;
|
|
378
|
+
readonly ciHigh: number;
|
|
379
|
+
/** Paired-test p-value (before correction). */
|
|
380
|
+
readonly p: number;
|
|
381
|
+
/** BH-significant across ALL pairs AND above the `minPairs` power floor. */
|
|
382
|
+
readonly significant: boolean;
|
|
383
|
+
}
|
|
384
|
+
interface PairwiseOptions {
|
|
385
|
+
readonly scoreOf?: ScoreOf;
|
|
386
|
+
readonly profileKeyOf?: ProfileKeyOf;
|
|
387
|
+
readonly labelOf?: (profileKey: string) => string;
|
|
388
|
+
/** False-discovery rate for the Benjamini–Hochberg correction. Default 0.05. */
|
|
389
|
+
readonly fdr?: number;
|
|
390
|
+
/** Below this many shared scenarios a paired test can't defensibly separate two profiles, so the
|
|
391
|
+
* `significant` tag is suppressed regardless of p (small-n mirage protection). Default 12. */
|
|
392
|
+
readonly minPairs?: number;
|
|
393
|
+
}
|
|
394
|
+
/** Compare EVERY profile pair on the scenarios they both ran — paired-bootstrap effect + CI, a real
|
|
395
|
+
* paired-test p-value, BH-corrected across all pairs. This is the honest "did A beat B" table the
|
|
396
|
+
* leaderboard's point ranking cannot answer. Reuses the agent-eval statistics substrate. */
|
|
397
|
+
declare function pairwiseSignificance(records: readonly RunRecord[], opts?: PairwiseOptions): PairwiseVerdict[];
|
|
398
|
+
/** Render the report as a publishable Markdown document: provenance → leaderboard → the full profile×axis
|
|
399
|
+
* matrix → cost/latency/token columns. Every axis is shown — a curated subset is a reporting failure. */
|
|
400
|
+
declare function renderLeaderboardMarkdown(report: Leaderboard): string;
|
|
401
|
+
/** Render the pairwise-significance table — every profile pair's paired delta, CI, and BH-corrected
|
|
402
|
+
* verdict. Feed it `pairwiseSignificance(records)`. This is the "did A really beat B" evidence the point
|
|
403
|
+
* ranking cannot give. */
|
|
404
|
+
declare function renderPairwiseMarkdown(verdicts: readonly PairwiseVerdict[], title?: string): string;
|
|
405
|
+
/** Render a self-contained SVG: a ranked score bar chart on top, the profile×axis heatmap below. No deps,
|
|
406
|
+
* embeddable anywhere (README, HTML page, hosted leaderboard). */
|
|
407
|
+
declare function renderLeaderboardSvg(report: Leaderboard): string;
|
|
408
|
+
/** Render a self-contained HTML leaderboard page (the hosted surface): the SVG charts + the full Markdown
|
|
409
|
+
* matrix as a table. Single file, no assets, opens in any browser. */
|
|
410
|
+
declare function renderLeaderboardHtml(report: Leaderboard): string;
|
|
411
|
+
|
|
279
412
|
/**
|
|
280
413
|
* @experimental
|
|
281
414
|
*
|
|
@@ -679,6 +812,25 @@ interface LoopDispatchOptions<Task, Output, Decision, TScenario extends Scenario
|
|
|
679
812
|
/** Cost-meter source label for the loop's spend. Default `'loop'`. */
|
|
680
813
|
costSource?: string;
|
|
681
814
|
}
|
|
815
|
+
/** Options for adapting plain agent-eval campaign scenarios into runtime `runLoop` cells. */
|
|
816
|
+
interface LoopCampaignDispatchOptions<Task, Output, Decision, TScenario extends Scenario, TArtifact> {
|
|
817
|
+
/** Sandbox client used for every campaign cell's `runLoop`. */
|
|
818
|
+
sandboxClient: SandboxClient;
|
|
819
|
+
/** Build the per-cell runLoop options from the campaign scenario. */
|
|
820
|
+
toLoopOptions: (scenario: TScenario) => LoopOptionsForDispatch<Task, Output, Decision>;
|
|
821
|
+
/** Map the finished loop to the artifact the campaign judges score. */
|
|
822
|
+
toArtifact?: (result: LoopResult<Task, Output, Decision>) => TArtifact;
|
|
823
|
+
/** Forward `loop.*` trace events into the campaign's scoped trace. Default true. */
|
|
824
|
+
forwardTrace?: boolean;
|
|
825
|
+
/** Cost-meter source label for the loop's spend. Default `'loop'`. */
|
|
826
|
+
costSource?: string;
|
|
827
|
+
}
|
|
828
|
+
/**
|
|
829
|
+
* Adapter for plain `runCampaign` scenarios. This is the runtime-side pair for
|
|
830
|
+
* agent-eval fixture scenarios: load fixtures in `agent-eval/campaign`, build
|
|
831
|
+
* the runtime loop here, and keep cost + token + trace reporting automatic.
|
|
832
|
+
*/
|
|
833
|
+
declare function loopCampaignDispatch<Task, Output, Decision, TScenario extends Scenario, TArtifact>(opts: LoopCampaignDispatchOptions<Task, Output, Decision, TScenario, TArtifact>): DispatchFn<TScenario, TArtifact>;
|
|
682
834
|
/**
|
|
683
835
|
* Adapter for `runProfileMatrix` (profile is an axis). Returns a
|
|
684
836
|
* `ProfileDispatchFn` that runs `runLoop` per (profile, scenario) cell and
|
|
@@ -1650,6 +1802,24 @@ interface CriuCapableClient {
|
|
|
1650
1802
|
declare function extractLlmCallEvent(event: SandboxEvent, agentRunName: string): (RuntimeStreamEvent & {
|
|
1651
1803
|
type: 'llm_call';
|
|
1652
1804
|
}) | undefined;
|
|
1805
|
+
/**
|
|
1806
|
+
* Sum the token usage + USD cost of a sandbox turn's events — the one honest way to meter an
|
|
1807
|
+
* `openSandboxRun` cell. Folds `extractLlmCallEvent` over the stream (which reads usage off EVERY backend
|
|
1808
|
+
* event shape), so a `runProfileMatrix` dispatch can report it to `ctx.cost`:
|
|
1809
|
+
*
|
|
1810
|
+
* const turn = await run.start(prompt)
|
|
1811
|
+
* const u = sumSandboxUsage(turn.events)
|
|
1812
|
+
* if (u.input || u.output) ctx.cost.observeTokens({ input: u.input, output: u.output })
|
|
1813
|
+
* if (u.costUsd) ctx.cost.observe(u.costUsd, 'sandbox-cell')
|
|
1814
|
+
*
|
|
1815
|
+
* Without this a cell reads `{tokens:0, cost:0}` and the backend-integrity guard correctly aborts the
|
|
1816
|
+
* matrix as a stub. `agentRunName` is the fallback model label for cost-only events (default `'agent'`).
|
|
1817
|
+
*/
|
|
1818
|
+
declare function sumSandboxUsage(events: readonly SandboxEvent[], agentRunName?: string): {
|
|
1819
|
+
input: number;
|
|
1820
|
+
output: number;
|
|
1821
|
+
costUsd: number;
|
|
1822
|
+
};
|
|
1653
1823
|
/**
|
|
1654
1824
|
* Project one `SandboxEvent` onto the `RuntimeStreamEvent` chat-UX vocabulary,
|
|
1655
1825
|
* for runtimes that bridge a sandbox `streamPrompt` into the
|
|
@@ -3190,6 +3360,67 @@ interface TrajectoryAnalysis {
|
|
|
3190
3360
|
/** Collect the source's spans and run the agent-eval batch analyzers over them under one `runId`. */
|
|
3191
3361
|
declare function analyzeTrace(source: TraceSource, runId?: string): Promise<TrajectoryAnalysis>;
|
|
3192
3362
|
|
|
3363
|
+
/** What a surface worker settles with — the surface verdict the driver + deliverable read. `resolved` is
|
|
3364
|
+
* the surface check's pass/fail (settled ⟺ resolved); `score` is the partial-credit fraction; `failing`
|
|
3365
|
+
* carries the tests this worker left red (so the analyst can target them). */
|
|
3366
|
+
interface SurfaceWorkerOut {
|
|
3367
|
+
readonly resolved: boolean;
|
|
3368
|
+
readonly score: number;
|
|
3369
|
+
readonly shots: number;
|
|
3370
|
+
readonly summary: string;
|
|
3371
|
+
readonly failing?: readonly string[];
|
|
3372
|
+
}
|
|
3373
|
+
/** The default self-improvement LENS — authored content, not a code path. On each settled worker it hands
|
|
3374
|
+
* the driver the still-FAILING tests (not just a score), so the next spawn targets the persistently-hard
|
|
3375
|
+
* cases. Swap `analysts` to change what the driver improves from — that's the one knob. */
|
|
3376
|
+
declare function failuresAnalyst(): AnalystRegistry;
|
|
3377
|
+
/** How a worker runs the surface task (its router substrate + per-attempt bounds). */
|
|
3378
|
+
interface SurfaceWorkerConfig {
|
|
3379
|
+
readonly routerBaseUrl: string;
|
|
3380
|
+
readonly routerKey: string;
|
|
3381
|
+
readonly model: string;
|
|
3382
|
+
readonly maxTokens?: number;
|
|
3383
|
+
readonly innerTurns?: number;
|
|
3384
|
+
/** Refine-shot budget for ONE worker attempt (max steered shots). Default 1. */
|
|
3385
|
+
readonly budget?: number;
|
|
3386
|
+
}
|
|
3387
|
+
interface SuperviseSurfaceOptions {
|
|
3388
|
+
/** The graded surface workers solve (open/tools/call/score/close). */
|
|
3389
|
+
readonly surface: AgenticSurface;
|
|
3390
|
+
/** Where/how each worker runs the surface task. */
|
|
3391
|
+
readonly worker: SurfaceWorkerConfig;
|
|
3392
|
+
/** The conserved compute pool for the whole supervised run. Default: sized off the worker's inner-loop
|
|
3393
|
+
* bounds for a handful of worker spawns — raise it to let the driver try more. */
|
|
3394
|
+
readonly budget?: Budget;
|
|
3395
|
+
/** The driver brain's router substrate (its own inference). Default: the worker's router + model — the
|
|
3396
|
+
* driver and workers share one router unless you separate them (e.g. a stronger driver model). */
|
|
3397
|
+
readonly router?: RouterConfig;
|
|
3398
|
+
/** The self-improvement lens fed to the driver on each settled worker. Default `failuresAnalyst()`
|
|
3399
|
+
* (target the still-failing tests). Pass a custom registry to change it, or `null` to turn the
|
|
3400
|
+
* within-run self-improvement OFF (the driver sees raw settled outputs). */
|
|
3401
|
+
readonly analysts?: AnalystRegistry | null;
|
|
3402
|
+
/** The strategy each worker runs over the surface. Default `refine` (iterate-with-feedback). */
|
|
3403
|
+
readonly strategy?: Strategy;
|
|
3404
|
+
/** Max workers live at once. Default 1 (serial — required when workers share a persistent artifact, so
|
|
3405
|
+
* they continue each other instead of racing the file). */
|
|
3406
|
+
readonly maxLiveWorkers?: number;
|
|
3407
|
+
}
|
|
3408
|
+
/** The deployable outcome of a supervised surface run. */
|
|
3409
|
+
interface SuperviseSurfaceResult {
|
|
3410
|
+
readonly resolved: boolean;
|
|
3411
|
+
readonly score: number;
|
|
3412
|
+
readonly usd: number;
|
|
3413
|
+
readonly tokensIn: number;
|
|
3414
|
+
readonly tokensOut: number;
|
|
3415
|
+
readonly ms: number;
|
|
3416
|
+
/** Total conserved-pool iterations = the driver + worker LLM rounds the run actually spent. */
|
|
3417
|
+
readonly completions: number;
|
|
3418
|
+
}
|
|
3419
|
+
/** Drive a team of agents (spawned + steered by `profile`) to solve a graded `AgenticSurface` task, and
|
|
3420
|
+
* report the deployable outcome + the full conserved spend. This is `supervise()` configured for surfaces
|
|
3421
|
+
* — there is no other entrypoint to learn. */
|
|
3422
|
+
declare function superviseSurface(profile: SupervisorProfile, task: AgenticTask, opts: SuperviseSurfaceOptions): Promise<SuperviseSurfaceResult>;
|
|
3423
|
+
|
|
3193
3424
|
/**
|
|
3194
3425
|
* createVerifierEnvironment — ANY checkable task as an `Environment`, no tool surface
|
|
3195
3426
|
* required. The generalization piece: EOPS/commit0-style domains have tools that mutate
|
|
@@ -3285,4 +3516,4 @@ declare function runInWorkspace<T>(ws: Workspace, body: (cwd: string) => Promise
|
|
|
3285
3516
|
commitOnInvalid?: boolean;
|
|
3286
3517
|
}): Promise<WorkspaceRun<T>>;
|
|
3287
3518
|
|
|
3288
|
-
export { Agent, AgentRunSpec, type AgenticOptions, type AgenticRunResult, type AgenticSurface, type AgenticTask, type AgenticTool, AnalystRegistry, type AnytimeReport, type AnytimeStrategySummary, type AnytimeTaskCurve, type ApplyContinuation, type ArtifactHandle, AssertTraceDerivedFindings, type AuditIntentInput, type AuditIntentOptions, type AuthorStrategyOptions, type AuthoredProfile, type AuthoredStrategy, type BenchmarkCell, type BenchmarkConfig, type BenchmarkLift, type BenchmarkReport, type BenchmarkStrategySummary, type BenchmarkTaskRow, Budget, type BudgetPool, type BudgetReadout, type ChampionPick, type ChampionPolicy, type CheckpointCapableBox, CombinatorShape, type CompletionAnalyst, type CompletionEvidence, type CompletionPolicy, type CompletionVerdict, CoordinationEvent, type CoordinationMcpHandle, Corpus, CorpusFilter, CorpusRecord, type CreateScopeAnalystOptions, type CriuCapableClient, DefinePersonaInput, type DelegateOptions, type Deliverable, DeliverableSpec, type DriveHarness, Driver, type DriverAgentOptions, type DumbDriverOptions, type Environment, EqualKArm, EqualKOnCostOptions, EqualKVerdict, type EvolutionArchiveNode, type EvolutionAuthor, type EvolutionBandInfo, type EvolutionCandidate, type EvolutionGeneration, type EvolutionReport, ExecCtx, ExecutorConfig, FanoutOptions, FanoutWinnerSelector, FileCorpus, type ForkCapableBox, type GitWorkspaceOptions, type HarvestCorpusOptions, type HarvestFailure, type HarvestReport, InMemoryCorpus, InMemoryResultBlobStore, type InMemoryRunContext, type InMemoryRunContextOptions, InMemorySpawnJournal, type InProcessOnPrompt, type InProcessPromptCtx, type InProcessSandboxClientOptions, type Inbox, type InboxMessage, type IntentAudit, Iteration, type LoopDispatchOptions, LoopLineageOptions, type LoopOptionsForDispatch, LoopResult, LoopShape, LoopTokenUsage, LoopUntilSpec, LoopWinner, MakeWorkerAgent, type McpEndpoint, type McpEnvironmentOptions, MountRecorder, type NaiveDriverOptions, type Observation, type ObserveInput, type ObserveOptions, type OpenSandboxRunOptions, Outcome, OutputAdapter, PanelSpec, Persona, PipelineStage, type ProfileRichness, type ProfileRichnessThresholds, type PromotionGateOptions, type PromotionVerdict, type RegistryAnalyzeProjection, RenderCorpusToInstructionsOptions, type ReservationTicket, ResultBlobStore, RouterConfig, type RunAgenticOptions, RunPersonifiedOptions, type SandboxCapabilities, SandboxClient, type SandboxLineage, type SandboxLineageHandle, type SandboxRun, SandboxRunAbortError, Scope, ScopeAnalyst, ScopeAnalyzeInput, ScopeWidenGate, type SessionCapableBox, type SessionTraceBox, Settled, ShapeRegistry, type Shell, type ShotPersona, type ShotSpec, Spend, SteerContext, type SteeringDecision, type Strategy, type StrategyCtx, type StrategyEvolutionConfig, type StrategyResult, type SuperviseOptions, SupervisedResult, Supervisor, type SupervisorAgentDeps, type SupervisorProfile, type SurfaceScore, ToolLoopChat, ToolLoopCompactionOptions, type TraceSource, type TrajectoryAnalysis, TrajectoryReport, TrajectoryReportOptions, type TurnResult, UsageEvent, type UsageSink, Validator, type VerifierEnvironmentOptions, VerifySpec, type WatchTraceOptions, type WaterfallCollector, type WaterfallReport, type WaterfallSpan, WidenSpec, WinnerStrategy, type Workspace, type WorkspaceCommit, type WorkspaceRun, acquireSandbox, adaptiveRefine, analyzeTrace, anytimeReport, asAuthoredProfile, assertModelAllowed, assertStrategyContract, assertTraceDerivedFindings, assessAuthoredProfile, auditIntent, authorStrategy, authoredWorker, breadthStrategy, buildSteerContext, builtinShapes, completionAuthorizes, contentAddress, createBudgetPool, createInMemoryRunContext, createInbox, createMcpEnvironment, createPushTraceSource, createSandboxLineage, createScope, createScopeAnalyst, createShapeRegistry, createSupervisor, createVerifierEnvironment, createWaterfallCollector, decodeToolPart, defaultAnalystInstruction, defaultAuditorInstruction, defaultDelegateBudget, defaultProfileRichnessThresholds, defaultSelectWinner, defaultToolDetectors, definePersona, defineStrategy, delegate, depthStrategy, deterministicCompletion, discriminatingMeans, driverAgent, dumbDriver, equalKOnCost, extractLlmCallEvent, fanout, finalizeBestDelivered, flatWidenGate, gitWorkspace, harvestCorpus, inProcessSandboxClient, inlineSandboxClient, jjWorkspace, localShell, loopDispatch, loopUntil, mapSandboxEvent, naiveDriver, observe, openSandboxRun, panel, pickChampion, pipeline, printBenchmarkReport, probeSandboxCapabilities, profileRichnessFinding, promotionGate, refine, registerShape, registryScopeAnalyst, renderAnytimeTable, renderCorpusToInstructions, renderReport, reportLoopUsage, runAgentic, runBenchmark, runInWorkspace, runLoop, runPersonified, runStrategyEvolution, sample, sampleThenRefine, sandboxSessionTraceSource, selectChampion, selectValidWinner, sentinelCompletion, serveCoordinationMcp, settledToIteration, spendFromUsageEvents, stopSentinel, strategyAuthorContract, supervise, supervisorAgent, supervisorInstructions, trajectoryReport, verify, watchTrace, widen, workerFromBackend };
|
|
3519
|
+
export { Agent, AgentRunSpec, type AgenticOptions, type AgenticRunResult, type AgenticSurface, type AgenticTask, type AgenticTool, AnalystRegistry, type AnytimeReport, type AnytimeStrategySummary, type AnytimeTaskCurve, type ApplyContinuation, type ArtifactHandle, AssertTraceDerivedFindings, type AuditIntentInput, type AuditIntentOptions, type AuthorStrategyOptions, type AuthoredProfile, type AuthoredStrategy, type BenchmarkCell, type BenchmarkConfig, type BenchmarkLift, type BenchmarkReport, type BenchmarkStrategySummary, type BenchmarkTaskRow, Budget, type BudgetPool, type BudgetReadout, type ChampionPick, type ChampionPolicy, type CheckpointCapableBox, CombinatorShape, type CompletionAnalyst, type CompletionEvidence, type CompletionPolicy, type CompletionVerdict, CoordinationEvent, type CoordinationMcpHandle, Corpus, CorpusFilter, CorpusRecord, type CreateScopeAnalystOptions, type CriuCapableClient, DefinePersonaInput, type DelegateOptions, type Deliverable, DeliverableSpec, type DriveHarness, Driver, type DriverAgentOptions, type DumbDriverOptions, type Environment, EqualKArm, EqualKOnCostOptions, EqualKVerdict, type EvolutionArchiveNode, type EvolutionAuthor, type EvolutionBandInfo, type EvolutionCandidate, type EvolutionGeneration, type EvolutionReport, ExecCtx, ExecutorConfig, ExecutorFactory, ExecutorRegistry, FanoutOptions, FanoutWinnerSelector, FileCorpus, type ForkCapableBox, type GitWorkspaceOptions, type HarvestCorpusOptions, type HarvestFailure, type HarvestReport, InMemoryCorpus, InMemoryResultBlobStore, type InMemoryRunContext, type InMemoryRunContextOptions, InMemorySpawnJournal, type InProcessOnPrompt, type InProcessPromptCtx, type InProcessSandboxClientOptions, type Inbox, type InboxMessage, type IntentAudit, type Interval, Iteration, type Leaderboard, type LeaderboardOptions, type LeaderboardRow, type LoopCampaignDispatchOptions, type LoopDispatchOptions, LoopLineageOptions, type LoopOptionsForDispatch, LoopResult, LoopShape, LoopTokenUsage, LoopUntilSpec, LoopWinner, MakeWorkerAgent, type McpEndpoint, type McpEnvironmentOptions, MountRecorder, type NaiveDriverOptions, type Observation, type ObserveInput, type ObserveOptions, type OpenSandboxRunOptions, Outcome, OutputAdapter, type PairwiseOptions, type PairwiseVerdict, PanelSpec, Persona, PipelineStage, type ProfileRichness, type ProfileRichnessThresholds, type PromotionGateOptions, type PromotionVerdict, type RegistryAnalyzeProjection, RenderCorpusToInstructionsOptions, type ReservationTicket, ResultBlobStore, RouterConfig, type RunAgenticOptions, RunPersonifiedOptions, type SandboxCapabilities, SandboxClient, type SandboxLineage, type SandboxLineageHandle, type SandboxRun, SandboxRunAbortError, Scope, ScopeAnalyst, ScopeAnalyzeInput, ScopeWidenGate, type SessionCapableBox, type SessionTraceBox, Settled, ShapeRegistry, type Shell, type ShotPersona, type ShotSpec, Spend, SteerContext, type SteeringDecision, type Strategy, type StrategyCtx, type StrategyEvolutionConfig, type StrategyResult, type SuperviseOptions, type SuperviseSurfaceOptions, type SuperviseSurfaceResult, SupervisedResult, Supervisor, type SupervisorAgentDeps, type SupervisorProfile, type SurfaceScore, type SurfaceWorkerConfig, type SurfaceWorkerOut, ToolLoopChat, ToolLoopCompactionOptions, type TraceSource, type TrajectoryAnalysis, TrajectoryReport, TrajectoryReportOptions, type TurnResult, UsageEvent, type UsageSink, Validator, type VerifierEnvironmentOptions, VerifySpec, type WatchTraceOptions, type WaterfallCollector, type WaterfallReport, type WaterfallSpan, WidenSpec, WinnerStrategy, type Workspace, type WorkspaceCommit, type WorkspaceRun, acquireSandbox, adaptiveRefine, analyzeTrace, anytimeReport, asAuthoredProfile, assertModelAllowed, assertStrategyContract, assertTraceDerivedFindings, assessAuthoredProfile, auditIntent, authorStrategy, authoredWorker, breadthStrategy, buildSteerContext, builtinShapes, completionAuthorizes, contentAddress, createBudgetPool, createInMemoryRunContext, createInbox, createMcpEnvironment, createPushTraceSource, createSandboxLineage, createScope, createScopeAnalyst, createShapeRegistry, createSupervisor, createVerifierEnvironment, createWaterfallCollector, decodeToolPart, defaultAnalystInstruction, defaultAuditorInstruction, defaultDelegateBudget, defaultProfileRichnessThresholds, defaultSelectWinner, defaultToolDetectors, definePersona, defineStrategy, delegate, depthStrategy, deterministicCompletion, discriminatingMeans, driverAgent, dumbDriver, equalKOnCost, extractLlmCallEvent, failuresAnalyst, fanout, finalizeBestDelivered, flatWidenGate, gitWorkspace, harvestCorpus, inProcessSandboxClient, inlineSandboxClient, jjWorkspace, leaderboard, localShell, loopCampaignDispatch, loopDispatch, loopUntil, mapSandboxEvent, naiveDriver, observe, openSandboxRun, pairwiseSignificance, panel, pickChampion, pipeline, printBenchmarkReport, probeSandboxCapabilities, profileRichnessFinding, promotionGate, refine, registerShape, registryScopeAnalyst, renderAnytimeTable, renderCorpusToInstructions, renderLeaderboardHtml, renderLeaderboardMarkdown, renderLeaderboardSvg, renderPairwiseMarkdown, renderReport, reportLoopUsage, runAgentic, runBenchmark, runInWorkspace, runLoop, runPersonified, runStrategyEvolution, sample, sampleThenRefine, sandboxSessionTraceSource, selectChampion, selectValidWinner, sentinelCompletion, serveCoordinationMcp, settledToIteration, spendFromUsageEvents, stopSentinel, strategyAuthorContract, sumSandboxUsage, supervise, superviseSurface, supervisorAgent, supervisorInstructions, trajectoryReport, verify, watchTrace, widen, workerFromBackend };
|
package/dist/loops.js
CHANGED
|
@@ -31,6 +31,7 @@ import {
|
|
|
31
31
|
discriminatingMeans,
|
|
32
32
|
dumbDriver,
|
|
33
33
|
equalKOnCost,
|
|
34
|
+
failuresAnalyst,
|
|
34
35
|
fanout,
|
|
35
36
|
flatWidenGate,
|
|
36
37
|
gitWorkspace,
|
|
@@ -38,13 +39,16 @@ import {
|
|
|
38
39
|
inProcessSandboxClient,
|
|
39
40
|
inlineSandboxClient,
|
|
40
41
|
jjWorkspace,
|
|
42
|
+
leaderboard,
|
|
41
43
|
localShell,
|
|
44
|
+
loopCampaignDispatch,
|
|
42
45
|
loopDispatch,
|
|
43
46
|
loopUntil,
|
|
44
47
|
makeFinding,
|
|
45
48
|
naiveDriver,
|
|
46
49
|
observe,
|
|
47
50
|
openSandboxRun,
|
|
51
|
+
pairwiseSignificance,
|
|
48
52
|
panel,
|
|
49
53
|
patchDelivered,
|
|
50
54
|
pickChampion,
|
|
@@ -56,6 +60,10 @@ import {
|
|
|
56
60
|
registryScopeAnalyst,
|
|
57
61
|
renderAnytimeTable,
|
|
58
62
|
renderCorpusToInstructions,
|
|
63
|
+
renderLeaderboardHtml,
|
|
64
|
+
renderLeaderboardMarkdown,
|
|
65
|
+
renderLeaderboardSvg,
|
|
66
|
+
renderPairwiseMarkdown,
|
|
59
67
|
renderReport,
|
|
60
68
|
reportLoopUsage,
|
|
61
69
|
runAgentic,
|
|
@@ -71,12 +79,13 @@ import {
|
|
|
71
79
|
sentinelCompletion,
|
|
72
80
|
stopSentinel,
|
|
73
81
|
strategyAuthorContract,
|
|
82
|
+
superviseSurface,
|
|
74
83
|
trajectoryReport,
|
|
75
84
|
verify,
|
|
76
85
|
watchTrace,
|
|
77
86
|
widen,
|
|
78
87
|
worktreeFanout
|
|
79
|
-
} from "./chunk-
|
|
88
|
+
} from "./chunk-AHZ3YBL6.js";
|
|
80
89
|
import {
|
|
81
90
|
InMemoryResultBlobStore,
|
|
82
91
|
InMemorySpawnJournal,
|
|
@@ -118,19 +127,20 @@ import {
|
|
|
118
127
|
supervisorAgent,
|
|
119
128
|
supervisorInstructions,
|
|
120
129
|
workerFromBackend
|
|
121
|
-
} from "./chunk-
|
|
122
|
-
import "./chunk-
|
|
130
|
+
} from "./chunk-44CX5JU6.js";
|
|
131
|
+
import "./chunk-IPEQ3ERC.js";
|
|
123
132
|
import {
|
|
124
133
|
createAgentEnvironmentProviderRegistry,
|
|
125
134
|
providerAsExecutor,
|
|
126
135
|
providerAsSandboxClient,
|
|
127
136
|
resolveAgentEnvironmentProvider,
|
|
128
137
|
sandboxClientAsProvider
|
|
129
|
-
} from "./chunk-
|
|
138
|
+
} from "./chunk-H7IBHAFT.js";
|
|
130
139
|
import {
|
|
131
140
|
extractLlmCallEvent,
|
|
132
|
-
mapSandboxEvent
|
|
133
|
-
|
|
141
|
+
mapSandboxEvent,
|
|
142
|
+
sumSandboxUsage
|
|
143
|
+
} from "./chunk-4J6RBI3K.js";
|
|
134
144
|
import "./chunk-5JAUQZQA.js";
|
|
135
145
|
import "./chunk-NBV35BR6.js";
|
|
136
146
|
import "./chunk-WIR4HOOJ.js";
|
|
@@ -194,6 +204,7 @@ export {
|
|
|
194
204
|
dumbDriver,
|
|
195
205
|
equalKOnCost,
|
|
196
206
|
extractLlmCallEvent,
|
|
207
|
+
failuresAnalyst,
|
|
197
208
|
fanout,
|
|
198
209
|
finalizeBestDelivered,
|
|
199
210
|
flatWidenGate,
|
|
@@ -203,7 +214,9 @@ export {
|
|
|
203
214
|
inProcessSandboxClient,
|
|
204
215
|
inlineSandboxClient,
|
|
205
216
|
jjWorkspace,
|
|
217
|
+
leaderboard,
|
|
206
218
|
localShell,
|
|
219
|
+
loopCampaignDispatch,
|
|
207
220
|
loopDispatch,
|
|
208
221
|
loopUntil,
|
|
209
222
|
makeFinding,
|
|
@@ -211,6 +224,7 @@ export {
|
|
|
211
224
|
naiveDriver,
|
|
212
225
|
observe,
|
|
213
226
|
openSandboxRun,
|
|
227
|
+
pairwiseSignificance,
|
|
214
228
|
panel,
|
|
215
229
|
patchDelivered,
|
|
216
230
|
pickChampion,
|
|
@@ -226,6 +240,10 @@ export {
|
|
|
226
240
|
registryScopeAnalyst,
|
|
227
241
|
renderAnytimeTable,
|
|
228
242
|
renderCorpusToInstructions,
|
|
243
|
+
renderLeaderboardHtml,
|
|
244
|
+
renderLeaderboardMarkdown,
|
|
245
|
+
renderLeaderboardSvg,
|
|
246
|
+
renderPairwiseMarkdown,
|
|
229
247
|
renderReport,
|
|
230
248
|
reportLoopUsage,
|
|
231
249
|
resolveAgentEnvironmentProvider,
|
|
@@ -251,7 +269,9 @@ export {
|
|
|
251
269
|
spendFromUsageEvents,
|
|
252
270
|
stopSentinel,
|
|
253
271
|
strategyAuthorContract,
|
|
272
|
+
sumSandboxUsage,
|
|
254
273
|
supervise,
|
|
274
|
+
superviseSurface,
|
|
255
275
|
supervisorAgent,
|
|
256
276
|
supervisorInstructions,
|
|
257
277
|
trajectoryReport,
|
package/dist/mcp/bin.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
readTraceContextFromEnv
|
|
4
|
-
} from "../chunk-
|
|
4
|
+
} from "../chunk-NXZEVWKP.js";
|
|
5
5
|
import {
|
|
6
6
|
resolveRouterBaseUrl
|
|
7
7
|
} from "../chunk-GRAGM4MC.js";
|
|
@@ -9,10 +9,10 @@ import {
|
|
|
9
9
|
DelegationTaskQueue,
|
|
10
10
|
FileDelegationStore,
|
|
11
11
|
createMcpServer
|
|
12
|
-
} from "../chunk-
|
|
13
|
-
import "../chunk-
|
|
14
|
-
import "../chunk-
|
|
15
|
-
import "../chunk-
|
|
12
|
+
} from "../chunk-44CX5JU6.js";
|
|
13
|
+
import "../chunk-IPEQ3ERC.js";
|
|
14
|
+
import "../chunk-H7IBHAFT.js";
|
|
15
|
+
import "../chunk-4J6RBI3K.js";
|
|
16
16
|
import "../chunk-5JAUQZQA.js";
|
|
17
17
|
import "../chunk-NBV35BR6.js";
|
|
18
18
|
import "../chunk-WIR4HOOJ.js";
|
package/dist/mcp/index.d.ts
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
import { S as SandboxClient, d as LoopSandboxPlacement } from '../types-BF-MEsQB.js';
|
|
2
|
-
import { F as FleetHandle, D as DelegationExecutor, a as DelegationTaskQueue, b as FeedbackStore, c as DelegateFeedbackResult, d as DelegateFeedbackArgs, U as UiAuditorDelegate, e as DelegateUiAuditArgs, f as DelegateUiAuditResult, g as DelegationHistoryResult, h as DelegationHistoryArgs, i as DelegationStatusResult, j as DelegationStatusArgs } from '../coordination-
|
|
3
|
-
export { A as AnalystRegistry, C as CappedDelegationTrace, k as CoderDelegate, l as CoderReview, m as CoderReviewer, n as CoordinationEvent, o as CoordinationTools, p as CoordinationToolsOptions, q as DELEGATE_DESCRIPTION, r as DELEGATE_INPUT_SCHEMA, s as DELEGATE_TOOL_NAME, t as DELEGATION_TRACE_MAX_BYTES, u as DELEGATION_TRACE_MAX_SPANS, v as DelegateArgs, w as DelegateCodeArgs, x as DelegateCodeConfig, y as DelegateCodeResult, z as DelegateHandlerOptions, B as DelegateResearchArgs, E as DelegateResearchConfig, G as DelegateResearchResult, H as DelegateResult, I as DelegateRunCtx, J as DelegateUiAuditConfig, K as DelegateUiAuditRoute, L as DelegationError, M as DelegationFeedbackSnapshot, N as DelegationHistoryEntry, O as DelegationPersistenceError, P as DelegationProfile, Q as DelegationProgress, R as DelegationRecord, S as DelegationResultPayload, T as DelegationResumeContext, V as DelegationResumeDriver, W as DelegationResumeTick, X as DelegationRunContext, Y as DelegationStateCorruptError, Z as DelegationStatus, _ as DelegationStore, $ as DelegationTaskQueueOptions, a0 as DelegationTraceCaps, a1 as DelegationTraceCollector, a2 as DelegationTraceSpan, a3 as DetachedSessionDelegateOptions, a4 as DetachedSessionRefParts, a5 as DetachedTurn, a6 as DetachedTurnResumeDriverOptions, a7 as DetachedWinnerSelection, a8 as DriveTurnCapableBox, a9 as DriveTurnTick, aa as FeedbackEvent, ab as FeedbackRating, ac as FeedbackRefersTo, ad as FileDelegationStore, ae as FileDelegationStoreOptions, af as FleetWorkspaceExecutorOptions, ag as InMemoryDelegationStore, ah as InMemoryFeedbackStore, ai as JsonRpcMessage, aj as JsonRpcResponse, ak as MakeWorkerAgent, al as McpServer, am as McpServerOptions, an as McpToolDescriptor, ao as McpTransport, ap as Question, aq as QuestionDecision, ar as QuestionPolicy, as as QuestionRecord, at as ResearchOutputShape, au as ResearchSource, av as RunDetachedTurnOptions, aw as SettleDetachedCoderTurnOptions, ax as SettledWorker, ay as SiblingSandboxExecutorOptions, az as SubmitInput, aA as SubmitOutput, aB as TraceContext, aC as UiAuditorDelegationOutput, aD as buildDelegationTraceSpans, aE as capDelegationTrace, aF as coderTaskFromArgs, aG as composeLoopTraceEmitters, aH as createCoordinationTools, aI as createDelegateHandler, aJ as createDelegationTraceCollector, aK as createDetachedTurnResumeDriver, aL as createFleetWorkspaceExecutor, aM as createInProcessTransport, aN as createMcpServer, aO as createPropagatingTraceEmitter, aP as createSiblingSandboxExecutor, aQ as detachedSessionDelegate, aR as detachedTurnEvents, aS as eventToSnapshot, aT as formatDetachedSessionRef, aU as hashIdempotencyInput, aV as parseDetachedSessionRef, aW as readTraceContextFromEnv, aX as runDetachedTurn, aY as settleDetachedCoderTurn, aZ as traceContextToEnv, a_ as validateDelegateArgs } from '../coordination-
|
|
2
|
+
import { F as FleetHandle, D as DelegationExecutor, a as DelegationTaskQueue, b as FeedbackStore, c as DelegateFeedbackResult, d as DelegateFeedbackArgs, U as UiAuditorDelegate, e as DelegateUiAuditArgs, f as DelegateUiAuditResult, g as DelegationHistoryResult, h as DelegationHistoryArgs, i as DelegationStatusResult, j as DelegationStatusArgs } from '../coordination-BFVtgRax.js';
|
|
3
|
+
export { A as AnalystRegistry, C as CappedDelegationTrace, k as CoderDelegate, l as CoderReview, m as CoderReviewer, n as CoordinationEvent, o as CoordinationTools, p as CoordinationToolsOptions, q as DELEGATE_DESCRIPTION, r as DELEGATE_INPUT_SCHEMA, s as DELEGATE_TOOL_NAME, t as DELEGATION_TRACE_MAX_BYTES, u as DELEGATION_TRACE_MAX_SPANS, v as DelegateArgs, w as DelegateCodeArgs, x as DelegateCodeConfig, y as DelegateCodeResult, z as DelegateHandlerOptions, B as DelegateResearchArgs, E as DelegateResearchConfig, G as DelegateResearchResult, H as DelegateResult, I as DelegateRunCtx, J as DelegateUiAuditConfig, K as DelegateUiAuditRoute, L as DelegationError, M as DelegationFeedbackSnapshot, N as DelegationHistoryEntry, O as DelegationPersistenceError, P as DelegationProfile, Q as DelegationProgress, R as DelegationRecord, S as DelegationResultPayload, T as DelegationResumeContext, V as DelegationResumeDriver, W as DelegationResumeTick, X as DelegationRunContext, Y as DelegationStateCorruptError, Z as DelegationStatus, _ as DelegationStore, $ as DelegationTaskQueueOptions, a0 as DelegationTraceCaps, a1 as DelegationTraceCollector, a2 as DelegationTraceSpan, a3 as DetachedSessionDelegateOptions, a4 as DetachedSessionRefParts, a5 as DetachedTurn, a6 as DetachedTurnResumeDriverOptions, a7 as DetachedWinnerSelection, a8 as DriveTurnCapableBox, a9 as DriveTurnTick, aa as FeedbackEvent, ab as FeedbackRating, ac as FeedbackRefersTo, ad as FileDelegationStore, ae as FileDelegationStoreOptions, af as FleetWorkspaceExecutorOptions, ag as InMemoryDelegationStore, ah as InMemoryFeedbackStore, ai as JsonRpcMessage, aj as JsonRpcResponse, ak as MakeWorkerAgent, al as McpServer, am as McpServerOptions, an as McpToolDescriptor, ao as McpTransport, ap as Question, aq as QuestionDecision, ar as QuestionPolicy, as as QuestionRecord, at as ResearchOutputShape, au as ResearchSource, av as RunDetachedTurnOptions, aw as SettleDetachedCoderTurnOptions, ax as SettledWorker, ay as SiblingSandboxExecutorOptions, az as SubmitInput, aA as SubmitOutput, aB as TraceContext, aC as UiAuditorDelegationOutput, aD as buildDelegationTraceSpans, aE as capDelegationTrace, aF as coderTaskFromArgs, aG as composeLoopTraceEmitters, aH as createCoordinationTools, aI as createDelegateHandler, aJ as createDelegationTraceCollector, aK as createDetachedTurnResumeDriver, aL as createFleetWorkspaceExecutor, aM as createInProcessTransport, aN as createMcpServer, aO as createPropagatingTraceEmitter, aP as createSiblingSandboxExecutor, aQ as detachedSessionDelegate, aR as detachedTurnEvents, aS as eventToSnapshot, aT as formatDetachedSessionRef, aU as hashIdempotencyInput, aV as parseDetachedSessionRef, aW as readTraceContextFromEnv, aX as runDetachedTurn, aY as settleDetachedCoderTurn, aZ as traceContextToEnv, a_ as validateDelegateArgs } from '../coordination-BFVtgRax.js';
|
|
4
4
|
import { L as LocalHarness, r as runLocalHarness } from '../local-harness-DU7yV6mG.js';
|
|
5
5
|
export { a as LocalHarnessResult, R as RunLocalHarnessOptions } from '../local-harness-DU7yV6mG.js';
|
|
6
|
-
import { G as GitRunner } from '../worktree-harness-
|
|
7
|
-
export { C as CreateWorktreeOptions, b as DiffOptions, c as DiffResult, R as RemoveWorktreeOptions, d as WorktreeHandle, e as captureWorktreeDiff, f as createWorktree, r as removeWorktree } from '../worktree-harness-
|
|
6
|
+
import { G as GitRunner } from '../worktree-harness-Dt6s_m3z.js';
|
|
7
|
+
export { C as CreateWorktreeOptions, b as DiffOptions, c as DiffResult, R as RemoveWorktreeOptions, d as WorktreeHandle, e as captureWorktreeDiff, f as createWorktree, r as removeWorktree } from '../worktree-harness-Dt6s_m3z.js';
|
|
8
8
|
export { C as CreateKbGateOptions, F as FactCandidate, a as FactJudge, b as FactJudgeVerdict, K as KbGateResult, c as createKbGate } from '../kb-gate-CuzMYGYM.js';
|
|
9
9
|
export { m as mcpToolsForRuntimeMcp, a as mcpToolsForRuntimeMcpSubset } from '../openai-tools-B-3v06BE.js';
|
|
10
10
|
import { AnalystFinding } from '@tangle-network/agent-eval';
|
|
11
11
|
import { U as UiLens } from '../substrate-rNj6TDc3.js';
|
|
12
12
|
import '@tangle-network/sandbox';
|
|
13
|
-
import '../types-
|
|
13
|
+
import '../types-DYW0tloU.js';
|
|
14
14
|
import '@tangle-network/agent-interface';
|
|
15
15
|
import '../otel-export-BKmNwiCb.js';
|
|
16
16
|
import '../router-client-Ak2IGuXq.js';
|
package/dist/mcp/index.js
CHANGED
|
@@ -2,11 +2,11 @@ import {
|
|
|
2
2
|
createPropagatingTraceEmitter,
|
|
3
3
|
readTraceContextFromEnv,
|
|
4
4
|
traceContextToEnv
|
|
5
|
-
} from "../chunk-
|
|
5
|
+
} from "../chunk-NXZEVWKP.js";
|
|
6
6
|
import {
|
|
7
7
|
mcpToolsForRuntimeMcp,
|
|
8
8
|
mcpToolsForRuntimeMcpSubset
|
|
9
|
-
} from "../chunk-
|
|
9
|
+
} from "../chunk-RUJZK6VH.js";
|
|
10
10
|
import {
|
|
11
11
|
createKbGate
|
|
12
12
|
} from "../chunk-FNMGYYSS.js";
|
|
@@ -14,7 +14,7 @@ import {
|
|
|
14
14
|
assertTraceDerivedFindings,
|
|
15
15
|
runCoderChecks,
|
|
16
16
|
selectValidWinner
|
|
17
|
-
} from "../chunk-
|
|
17
|
+
} from "../chunk-AHZ3YBL6.js";
|
|
18
18
|
import {
|
|
19
19
|
DELEGATE_DESCRIPTION,
|
|
20
20
|
DELEGATE_FEEDBACK_DESCRIPTION,
|
|
@@ -64,15 +64,15 @@ import {
|
|
|
64
64
|
validateDelegateUiAuditArgs,
|
|
65
65
|
validateDelegationHistoryArgs,
|
|
66
66
|
validateDelegationStatusArgs
|
|
67
|
-
} from "../chunk-
|
|
68
|
-
import "../chunk-
|
|
67
|
+
} from "../chunk-44CX5JU6.js";
|
|
68
|
+
import "../chunk-IPEQ3ERC.js";
|
|
69
69
|
import {
|
|
70
70
|
deleteBoxSafe,
|
|
71
71
|
sleep,
|
|
72
72
|
throwAbort,
|
|
73
73
|
throwIfAborted
|
|
74
|
-
} from "../chunk-
|
|
75
|
-
import "../chunk-
|
|
74
|
+
} from "../chunk-H7IBHAFT.js";
|
|
75
|
+
import "../chunk-4J6RBI3K.js";
|
|
76
76
|
import {
|
|
77
77
|
runLocalHarness
|
|
78
78
|
} from "../chunk-5JAUQZQA.js";
|
|
@@ -129,8 +129,8 @@ type UsageEvent = {
|
|
|
129
129
|
} | {
|
|
130
130
|
kind: 'iteration';
|
|
131
131
|
};
|
|
132
|
-
/** The runtime tag of a `Executor` impl. Open by intent
|
|
133
|
-
*
|
|
132
|
+
/** The runtime tag of a `Executor` impl. Open by intent: custom runtimes use their own string name.
|
|
133
|
+
* External executors can register additional runtime strings without widening this type. */
|
|
134
134
|
type Runtime = 'router' | 'inline' | 'sandbox' | 'cli' | (string & {});
|
|
135
135
|
/**
|
|
136
136
|
* `AgentProfile` does NOT carry a `harness`/backend field — `harness` lives on the
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { AgentProfile } from '@tangle-network/agent-interface';
|
|
2
2
|
import { AnalystFinding, DefaultVerdict } from '@tangle-network/agent-eval';
|
|
3
|
-
import { A as AgentSpec, b as ExecutorRegistry, B as Budget, c as Agent, S as SpawnJournal, d as ResultBlobStore, e as RootHandle, f as SupervisedResult, N as NodeId, g as Settled, h as Spend, i as Scope, a as Executor } from './types-
|
|
3
|
+
import { A as AgentSpec, b as ExecutorRegistry, B as Budget, c as Agent, S as SpawnJournal, d as ResultBlobStore, e as RootHandle, f as SupervisedResult, N as NodeId, g as Settled, h as Spend, i as Scope, a as Executor } from './types-DYW0tloU.js';
|
|
4
4
|
import { R as RuntimeHooks, I as Iteration } from './types-BF-MEsQB.js';
|
|
5
5
|
import { BackendType } from '@tangle-network/sandbox';
|
|
6
|
-
import { W as WorktreeHarnessResult, G as GitRunner, a as WorktreeCheckRunner, D as DeliverableSpec } from './worktree-harness-
|
|
6
|
+
import { W as WorktreeHarnessResult, G as GitRunner, a as WorktreeCheckRunner, D as DeliverableSpec } from './worktree-harness-Dt6s_m3z.js';
|
|
7
7
|
import { L as LocalHarness, r as runLocalHarness } from './local-harness-DU7yV6mG.js';
|
|
8
8
|
|
|
9
9
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/agent-runtime",
|
|
3
|
-
"version": "0.79.
|
|
3
|
+
"version": "0.79.4",
|
|
4
4
|
"description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.",
|
|
5
5
|
"homepage": "https://github.com/tangle-network/agent-runtime#readme",
|
|
6
6
|
"repository": {
|