@tangle-network/agent-runtime 0.105.0 → 0.106.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/activation-CRjVfOiy.js +203 -0
- package/dist/activation-CRjVfOiy.js.map +1 -0
- package/dist/activation-DyWB0K6E.d.ts +98 -0
- package/dist/agent.d.ts +216 -234
- package/dist/agent.js +615 -547
- package/dist/agent.js.map +1 -1
- package/dist/analyst-loop-DGdtVENg.js +517 -0
- package/dist/analyst-loop-DGdtVENg.js.map +1 -0
- package/dist/analyst-loop.d.ts +10 -43
- package/dist/analyst-loop.js +2 -11
- package/dist/backends-Dy8NujB4.js +713 -0
- package/dist/backends-Dy8NujB4.js.map +1 -0
- package/dist/candidate-execution/index.d.ts +4 -192
- package/dist/candidate-execution/index.js +5 -66
- package/dist/candidate-execution-DqMbv1rU.js +488 -0
- package/dist/candidate-execution-DqMbv1rU.js.map +1 -0
- package/dist/coder-yhVWbdWc.js +40 -0
- package/dist/coder-yhVWbdWc.js.map +1 -0
- package/dist/conversation-XucG_K3l.js +1265 -0
- package/dist/conversation-XucG_K3l.js.map +1 -0
- package/dist/conversation.d.ts +2 -804
- package/dist/conversation.js +2 -55
- package/dist/environment-provider-CfLUmKmf.js +1168 -0
- package/dist/environment-provider-CfLUmKmf.js.map +1 -0
- package/dist/environment-provider-IUGU3epE.d.ts +964 -0
- package/dist/environment-provider.d.ts +2 -78
- package/dist/environment-provider.js +2 -18
- package/dist/errors-DEAvWQPy.js +111 -0
- package/dist/errors-DEAvWQPy.js.map +1 -0
- package/dist/improvement-adapter-Cr5TBSji.d.ts +166 -0
- package/dist/improvement-cycle-C095kS_g.js +2624 -0
- package/dist/improvement-cycle-C095kS_g.js.map +1 -0
- package/dist/index-BGdxAwfO.d.ts +838 -0
- package/dist/index-C-FYUuFG.d.ts +193 -0
- package/dist/index-CbNM40xG.d.ts +140 -0
- package/dist/index-DruSKs3Z.d.ts +7224 -0
- package/dist/index-Efjb3nrQ.d.ts +671 -0
- package/dist/index.d.ts +14 -1058
- package/dist/index.js +2106 -2082
- package/dist/index.js.map +1 -1
- package/dist/intelligence.d.ts +768 -889
- package/dist/intelligence.js +1466 -1367
- package/dist/intelligence.js.map +1 -1
- package/dist/{kb-gate-CwHO0vz6.d.ts → kb-gate-C8z2juK8.d.ts} +33 -31
- package/dist/kb-gate-DpaSwXVx.js +80 -0
- package/dist/kb-gate-DpaSwXVx.js.map +1 -0
- package/dist/knowledge-Ku-mPP4f.js +433 -0
- package/dist/knowledge-Ku-mPP4f.js.map +1 -0
- package/dist/knowledge.d.ts +2 -150
- package/dist/knowledge.js +2 -40
- package/dist/local-harness-Dh8PJ0ot.d.ts +455 -0
- package/dist/loop-runner-bin-BXsFEUcf.d.ts +162 -0
- package/dist/loop-runner-bin-BwjQ1NfH.js +268 -0
- package/dist/loop-runner-bin-BwjQ1NfH.js.map +1 -0
- package/dist/loop-runner-bin.d.ts +2 -15
- package/dist/loop-runner-bin.js +2 -24
- package/dist/loops.d.ts +5 -4164
- package/dist/loops.js +8 -419
- package/dist/mcp/bin.d.ts +1 -1
- package/dist/mcp/bin.js +116 -146
- package/dist/mcp/bin.js.map +1 -1
- package/dist/mcp/index.d.ts +295 -436
- package/dist/mcp/index.js +1242 -1177
- package/dist/mcp/index.js.map +1 -1
- package/dist/mcp/memory-bin.d.ts +1 -1
- package/dist/mcp/memory-bin.js +37 -16
- package/dist/mcp/memory-bin.js.map +1 -1
- package/dist/memory-server-DL6cE2Ag.js +234 -0
- package/dist/memory-server-DL6cE2Ag.js.map +1 -0
- package/dist/model-resolution-Btd9iIKV.js +98 -0
- package/dist/model-resolution-Btd9iIKV.js.map +1 -0
- package/dist/openai-tools-B68JaOCx.d.ts +24 -0
- package/dist/openai-tools-CKLy1C7M.js +44 -0
- package/dist/openai-tools-CKLy1C7M.js.map +1 -0
- package/dist/otel-export-D34IQW1C.js +843 -0
- package/dist/otel-export-D34IQW1C.js.map +1 -0
- package/dist/platform.d.ts +188 -185
- package/dist/platform.js +185 -218
- package/dist/platform.js.map +1 -1
- package/dist/prepare-B53sPV7M.js +4292 -0
- package/dist/prepare-B53sPV7M.js.map +1 -0
- package/dist/primeintellect/index.d.ts +149 -145
- package/dist/primeintellect/index.js +626 -1003
- package/dist/primeintellect/index.js.map +1 -1
- package/dist/profiles.d.ts +296 -490
- package/dist/profiles.js +722 -749
- package/dist/profiles.js.map +1 -1
- package/dist/protected-model-port-BLPH5VVv.js +722 -0
- package/dist/protected-model-port-BLPH5VVv.js.map +1 -0
- package/dist/protected-model-port-BP6Z4eau.d.ts +283 -0
- package/dist/protected-redaction--F3v1oo8.js +200 -0
- package/dist/protected-redaction--F3v1oo8.js.map +1 -0
- package/dist/redact-D-u-rrcn.js +82 -0
- package/dist/redact-D-u-rrcn.js.map +1 -0
- package/dist/redact-kbIOjfjM.d.ts +452 -0
- package/dist/researcher-CoVqNhfI.js +433 -0
- package/dist/researcher-CoVqNhfI.js.map +1 -0
- package/dist/runtime-BG0BziaP.js +5408 -0
- package/dist/runtime-BG0BziaP.js.map +1 -0
- package/dist/sandbox-events-DeI5xX8P.js +316 -0
- package/dist/sandbox-events-DeI5xX8P.js.map +1 -0
- package/dist/spawn-journal-CwPvKUTa.js +456 -0
- package/dist/spawn-journal-CwPvKUTa.js.map +1 -0
- package/dist/structural-rollout-IXUEplky.js +1445 -0
- package/dist/structural-rollout-IXUEplky.js.map +1 -0
- package/dist/substrate-B0TYNrXn.js +27 -0
- package/dist/substrate-B0TYNrXn.js.map +1 -0
- package/dist/{substrate-Dyiqmd3o.d.ts → substrate-BcnuSHXm.d.ts} +57 -54
- package/dist/supervise-BiRutHS9.js +7321 -0
- package/dist/supervise-BiRutHS9.js.map +1 -0
- package/dist/supervisor-DTKhF-RV.js +3826 -0
- package/dist/supervisor-DTKhF-RV.js.map +1 -0
- package/dist/testing.d.ts +4 -3
- package/dist/testing.js +2478 -2637
- package/dist/testing.js.map +1 -1
- package/dist/tool-server-RcWgLIsL.js +141 -0
- package/dist/tool-server-RcWgLIsL.js.map +1 -0
- package/dist/trace-propagation-CJJC7SVB.js +57 -0
- package/dist/trace-propagation-CJJC7SVB.js.map +1 -0
- package/dist/types-BaIpb0XT.d.ts +492 -0
- package/dist/types-BevOjfTY.d.ts +765 -0
- package/dist/types-C9j4qg6l.d.ts +500 -0
- package/dist/types-Cja5OTGh.d.ts +190 -0
- package/dist/util-MVgdwuIS.js +145 -0
- package/dist/util-MVgdwuIS.js.map +1 -0
- package/dist/workspace-archive-BosQiotA.js +575 -0
- package/dist/workspace-archive-BosQiotA.js.map +1 -0
- package/package.json +43 -56
- package/skills/build-with-agent-runtime/SKILL.md +1 -0
- package/dist/activation-Cm-9MMX1.d.ts +0 -69
- package/dist/analyst-loop.js.map +0 -1
- package/dist/backends-BO4Jqoe2.d.ts +0 -134
- package/dist/candidate-execution/index.js.map +0 -1
- package/dist/chunk-2KGAN2HM.js +0 -1256
- package/dist/chunk-2KGAN2HM.js.map +0 -1
- package/dist/chunk-2VZYRB55.js +0 -834
- package/dist/chunk-2VZYRB55.js.map +0 -1
- package/dist/chunk-3MDZX7YU.js +0 -125
- package/dist/chunk-3MDZX7YU.js.map +0 -1
- package/dist/chunk-5IBQAPVB.js +0 -29
- package/dist/chunk-5IBQAPVB.js.map +0 -1
- package/dist/chunk-5QOB7H74.js +0 -387
- package/dist/chunk-5QOB7H74.js.map +0 -1
- package/dist/chunk-C5RTIBNZ.js +0 -48
- package/dist/chunk-C5RTIBNZ.js.map +0 -1
- package/dist/chunk-CTRA64LY.js +0 -680
- package/dist/chunk-CTRA64LY.js.map +0 -1
- package/dist/chunk-DPEUKJRO.js +0 -27
- package/dist/chunk-DPEUKJRO.js.map +0 -1
- package/dist/chunk-FD2MBMOH.js +0 -494
- package/dist/chunk-FD2MBMOH.js.map +0 -1
- package/dist/chunk-H5QPIZNX.js +0 -907
- package/dist/chunk-H5QPIZNX.js.map +0 -1
- package/dist/chunk-HLKC4UYB.js +0 -3514
- package/dist/chunk-HLKC4UYB.js.map +0 -1
- package/dist/chunk-HNP72PNU.js +0 -192
- package/dist/chunk-HNP72PNU.js.map +0 -1
- package/dist/chunk-ISPWRSEC.js +0 -408
- package/dist/chunk-ISPWRSEC.js.map +0 -1
- package/dist/chunk-KRBFHMV6.js +0 -1228
- package/dist/chunk-KRBFHMV6.js.map +0 -1
- package/dist/chunk-M22PL4GH.js +0 -171
- package/dist/chunk-M22PL4GH.js.map +0 -1
- package/dist/chunk-NURIQBJQ.js +0 -72
- package/dist/chunk-NURIQBJQ.js.map +0 -1
- package/dist/chunk-OPVWXJ2H.js +0 -75
- package/dist/chunk-OPVWXJ2H.js.map +0 -1
- package/dist/chunk-PZZKQVQV.js +0 -52
- package/dist/chunk-PZZKQVQV.js.map +0 -1
- package/dist/chunk-Q2JSAVQ3.js +0 -671
- package/dist/chunk-Q2JSAVQ3.js.map +0 -1
- package/dist/chunk-RDOAVVHY.js +0 -699
- package/dist/chunk-RDOAVVHY.js.map +0 -1
- package/dist/chunk-SGKPNBXE.js +0 -60
- package/dist/chunk-SGKPNBXE.js.map +0 -1
- package/dist/chunk-SMQXZGLZ.js +0 -351
- package/dist/chunk-SMQXZGLZ.js.map +0 -1
- package/dist/chunk-UHEZW5BU.js +0 -2918
- package/dist/chunk-UHEZW5BU.js.map +0 -1
- package/dist/chunk-UPWGXKXB.js +0 -254
- package/dist/chunk-UPWGXKXB.js.map +0 -1
- package/dist/chunk-VISA6CI3.js +0 -200
- package/dist/chunk-VISA6CI3.js.map +0 -1
- package/dist/chunk-WMTCUOQL.js +0 -474
- package/dist/chunk-WMTCUOQL.js.map +0 -1
- package/dist/chunk-WRTOVNN4.js +0 -4728
- package/dist/chunk-WRTOVNN4.js.map +0 -1
- package/dist/chunk-WSTRQZYQ.js +0 -5576
- package/dist/chunk-WSTRQZYQ.js.map +0 -1
- package/dist/chunk-XBG2W2VW.js +0 -6434
- package/dist/chunk-XBG2W2VW.js.map +0 -1
- package/dist/chunk-YEJR7IXO.js +0 -68
- package/dist/chunk-YEJR7IXO.js.map +0 -1
- package/dist/chunk-YJZA2BIK.js +0 -1338
- package/dist/chunk-YJZA2BIK.js.map +0 -1
- package/dist/completion-gate-BMy5LGoP.d.ts +0 -256
- package/dist/conversation.js.map +0 -1
- package/dist/coordination-BZZSVYpZ.d.ts +0 -2570
- package/dist/environment-provider.js.map +0 -1
- package/dist/improvement-adapter-HAZz-7vK.d.ts +0 -203
- package/dist/knowledge.js.map +0 -1
- package/dist/local-harness-BDNcl6jI.d.ts +0 -175
- package/dist/loop-runner-bin-68aoM9-7.d.ts +0 -199
- package/dist/loop-runner-bin.js.map +0 -1
- package/dist/loops.js.map +0 -1
- package/dist/openai-tools-fnj6SRVg.d.ts +0 -46
- package/dist/protected-model-port-B4t-OQjL.d.ts +0 -271
- package/dist/redact-BuFjxBUJ.d.ts +0 -481
- package/dist/sanitize-DEbPNtyI.d.ts +0 -328
- package/dist/structural-rollout-DEh6CEsa.d.ts +0 -446
- package/dist/supervise-4_48wwvy.d.ts +0 -379
- package/dist/types-BCoemcXU.d.ts +0 -802
- package/dist/types-BwoZWq-i.d.ts +0 -510
- package/dist/types-Cbp72qyu.d.ts +0 -491
- package/dist/types-CmYCMbFT.d.ts +0 -203
- package/dist/types-CvNwMzQt.d.ts +0 -957
- package/dist/worktree-fanout-DxhAWr5Z.d.ts +0 -999
package/dist/loops.d.ts
CHANGED
|
@@ -1,4164 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
import { SandboxEvent, SandboxInstance, CreateSandboxOptions, PromptOptions, TaskOptions } from '@tangle-network/sandbox';
|
|
4
|
-
|
|
5
|
-
import { b as ResultBlobStore, c as SpawnJournal, N as NodeId, l as SpawnEvent, m as TreeView, P as PendingWait, a as Settled, E as ExecutorFactory, A as Agent, B as Budget, S as Scope, e as SupervisedResult, g as Spend, U as UsageEvent, n as SpawnOpts, i as ExecutorRegistry, d as WaitProbeRegistry, o as Supervisor, T as TraceSource } from './types-CvNwMzQt.js';
|
|
6
|
-
export { p as ActivityLog, q as ActivityNote, h as AgentSpec, D as DEFAULT_STALL_AFTER_MS, f as Executor, r as ExecutorContext, k as ExecutorProgress, s as ExecutorResult, t as ResumedWork, R as Runtime, u as ScopeProgressInput, v as SessionTraceBox, w as SupervisorOpts, x as WaitOutcome, y as WaitProbe, z as WaitRejection, C as WaitSpec, F as WidenGate, W as WorkerProgress, G as createActivityLog, H as createPushTraceSource, I as createWaitProbes, J as decodeToolPart, K as isWaitOutcome, L as pollFor, M as readWorkerProgress, O as sandboxSessionTraceSource, Q as timerAt, V as validateWaitSpec, X as waitUntil } from './types-CvNwMzQt.js';
|
|
7
|
-
import { M as MakeWorkerAgent, A as AnalystRegistry, W as WorkerWatchOptions, o as CoordinationTools, n as CoordinationEvent, at as QuestionPolicy, E as ExecutorConfig } from './coordination-BZZSVYpZ.js';
|
|
8
|
-
export { bi as BusEvent, bj as BusRecord, bk as BusStats, bl as DEFAULT_SANDBOX_STEERING_MAX_TURNS, bm as EventBus, bn as Inbox, bo as InboxMessage, bp as PI_RUNTIME, bq as PiSeam, br as ProviderSeam, bs as PublishOptions, bt as SandboxSteeringOptions, bu as SteerableSandboxSession, bv as WatchTraceOptions, bw as cliWorktreeExecutor, bx as createEventBus, by as createExecutor, bz as createExecutorRegistry, bA as createInbox, bB as createSteerableSandboxSession, bC as defaultToolDetectors, bD as piExecutor, bE as piSeamKey, bF as watchTrace } from './coordination-BZZSVYpZ.js';
|
|
9
|
-
import { R as RuntimeHooks, I as Iteration, a as SandboxClient, D as Driver, A as AgentRunSpec, O as OutputAdapter, V as Validator, E as ExecCtx, u as LoopWinner, v as LoopLineageOptions, w as LoopResult, L as LoopTokenUsage, M as MountRecorder } from './types-BCoemcXU.js';
|
|
10
|
-
export { x as LoopDecisionPayload, y as LoopEndedPayload, z as LoopIterationDispatchPayload, B as LoopIterationEndedPayload, C as LoopIterationStartedPayload, F as LoopPlanDescription, G as LoopPlanPayload, b as LoopSandboxPlacement, H as LoopStartedPayload, J as LoopTeardownFailedPayload, t as LoopTraceEmitter, c as LoopTraceEvent, K as MountManifestEntry, N as RunProvenance, S as SelectionReceipt, P as ValidationCtx } from './types-BCoemcXU.js';
|
|
11
|
-
import { RunProfileMatrixResult, Scenario, ProfileDispatchFn, JudgeConfig, RunProfileMatrixOptions, DispatchFn } from '@tangle-network/agent-eval/campaign';
|
|
12
|
-
export { AgentEnvironmentProviderRef, AgentEnvironmentProviderRegistry, CreateTangleSandboxExactProcessProviderOptions, ProviderAsSandboxClientOptions, ProviderExecutorOptions, SandboxClientProviderOptions, createAgentEnvironmentProviderRegistry, createTangleSandboxExactProcessProvider, providerAsExecutor, providerAsSandboxClient, resolveAgentEnvironmentProvider, sandboxClientAsProvider } from './environment-provider.js';
|
|
13
|
-
import { c as CorpusRecord, C as Corpus, S as ScopeAnalyzeInput, d as AssertTraceDerivedFindings, O as Outcome, e as SteerContext, f as ScopeAnalyst, F as FanoutOptions, g as CombinatorShape, h as ScopeWidenGate, L as LoopUntilSpec, P as PanelSpec, i as PipelineStage, W as WinnerStrategy, j as FanoutWinnerSelector, V as VerifySpec, k as WidenSpec, l as CorpusFilter, R as RenderCorpusToInstructionsOptions, D as DefinePersonaInput, m as Persona, n as RunPersonifiedOptions, o as ShapeRegistry, p as LoopShape, E as EqualKArm, q as EqualKOnCostOptions, r as EqualKVerdict, T as TrajectoryReportOptions, s as TrajectoryReport } from './worktree-fanout-DxhAWr5Z.js';
|
|
14
|
-
export { A as AuthoredHarness, t as DefinePersona, u as EqualKOnCost, v as Fanout, w as FanoutSynthesis, x as FlatWidenGate, y as LoopUntil, z as LoopUntilState, B as Panel, G as PanelJudge, H as PanelVerdict, I as PatchDeliverableOptions, J as PersonaContext, K as PersonaExecutors, M as Pipeline, N as RenderCorpusToInstructions, Q as RunPersonified, U as ShapeBudget, X as ShapeContext, Y as TrajectoryNode, Z as TrajectoryReportFn, _ as Verify, $ as Widen, a0 as WidenDecision, a1 as WidenLineage, a2 as WorktreeCliExecutorOptions, a as WorktreeFanoutOptions, b as WorktreePatchArtifact, a3 as createWorktreeCliExecutor, a4 as patchDelivered, a5 as worktreeFanout } from './worktree-fanout-DxhAWr5Z.js';
|
|
15
|
-
import { AgentProfileMcpServer, AgentProfile as AgentProfile$1, AgentProfileSecurityPolicy } from '@tangle-network/agent-interface';
|
|
16
|
-
import { A as AgenticSurface, a as AgenticTask, b as AgenticOptions, c as Strategy, d as ArtifactHandle, e as SurfaceScore, f as AgenticTool } from './structural-rollout-DEh6CEsa.js';
|
|
17
|
-
export { g as AgenticRunResult, C as CheckExecChannel, h as CheckOutcome, i as CheckRunContext, j as CheckRunner, k as CheckSource, l as CheckSourceCtx, m as CorpusReadbackOptions, R as RepairStop, n as RunAgenticOptions, o as ShotPersona, p as ShotSpec, q as StrategyCtx, r as StrategyResult, s as StructuralRolloutConfig, S as StructuralRolloutPolicy, t as StructuralRolloutResult, V as VisibleCheck, u as adaptiveRefine, v as breadthStrategy, w as canDisplace, x as compareCheckOutcomes, y as composeCheckSources, z as defaultExtractCandidate, B as defaultStructuralRolloutPolicy, D as defineStrategy, E as depthStrategy, F as filterAuthoredAsserts, G as modelAuthoredChecks, H as officialChecksFromMeta, I as refine, J as resolveEntrySymbol, K as runAgentic, L as sample, M as sampleThenRefine, N as sandboxCheckRunner, O as selectBestIndex, P as structuralRollout, Q as visibleCheckScore } from './structural-rollout-DEh6CEsa.js';
|
|
18
|
-
import { b as AnalystRegistryLike } from './types-CmYCMbFT.js';
|
|
19
|
-
import { R as RouterConfig, T as ToolLoopChat, a as ToolLoopCompactionOptions } from './sanitize-DEbPNtyI.js';
|
|
20
|
-
export { j as RouterChatResult, k as RouterChatToolsResult, l as RouterToolCall, m as RouterToolLoopResult, n as ToolLoopCompaction, b as ToolSpec, r as routerBrain, o as routerChatWithTools, p as routerChatWithUsage, q as routerToolLoop } from './sanitize-DEbPNtyI.js';
|
|
21
|
-
import { R as RuntimeStreamEvent, c as AgentExecutionBackend, f as AgentTaskStatus, B as BackendErrorDetail } from './types-BwoZWq-i.js';
|
|
22
|
-
import { D as DeliverableSpec } from './completion-gate-BMy5LGoP.js';
|
|
23
|
-
export { g as WorktreeCommandResult, h as WorktreeProfileMaterializationReceipt, i as gateOnDeliverable } from './completion-gate-BMy5LGoP.js';
|
|
24
|
-
import { S as StopRule, a as SupervisorProfile } from './supervise-4_48wwvy.js';
|
|
25
|
-
export { A as AllWorkersStalledOptions, D as DriveHarness, N as NoProgressForOptions, P as PlateauOptions, b as ProgressSample, c as ProgressTracker, d as ProgressTrackerOptions, e as ProgressView, f as StopDecision, g as SuperviseOptions, h as SupervisorAgentDeps, i as allOf, j as allWorkersStalled, k as anyOf, l as createProgressTracker, n as noProgressFor, p as plateau, s as sampleFromSettled, m as supervise, o as supervisorAgent, w as workerFromBackend } from './supervise-4_48wwvy.js';
|
|
26
|
-
import { stuckLoopView, toolWasteView } from '@tangle-network/agent-eval/pipelines';
|
|
27
|
-
export { AgentEnvironment, AgentEnvironmentCapabilities, AgentEnvironmentEvent, AgentEnvironmentProvider, AgentEnvironmentQuery, AgentEnvironmentStatus, AgentEnvironmentSummary, AgentProfileRef, AgentSession, AgentSessionRef, AgentSessionStatus, AgentTurnInput, AgentTurnResult, CheckpointRef, CheckpointRequest, CreateAgentEnvironmentInput, ExecRequest, ExecResult, ForkRequest, PlacementInfo, ResourceRequest, WorkspaceRequest } from '@tangle-network/agent-interface/environment-provider';
|
|
28
|
-
import './substrate-Dyiqmd3o.js';
|
|
29
|
-
import './local-harness-BDNcl6jI.js';
|
|
30
|
-
import 'node:child_process';
|
|
31
|
-
import '@tangle-network/agent-profile-materialize';
|
|
32
|
-
|
|
33
|
-
/**
|
|
34
|
-
*
|
|
35
|
-
* Event-sourced spawn journal for the recursive execution atom (build steps 3 + 7).
|
|
36
|
-
*
|
|
37
|
-
* The supervision tree is journaled as an append-only event log: every `spawned`,
|
|
38
|
-
* `settled`, and `cancelled` is recorded AFTER it is observed-committed (never
|
|
39
|
-
* speculative), mirroring `ConversationJournal`'s begin/append/load shape. The log
|
|
40
|
-
* holds only the THIN decision record — ids, parentage, budget, the spend a decision
|
|
41
|
-
* consumed, and a content-addressed `outRef`. The payloads the driver branched on
|
|
42
|
-
* (the `out` artifacts) live in a separate `ResultBlobStore`, keyed by `outRef`, so
|
|
43
|
-
* the journal stays small (decisions) and replay rehydrates the exact `Settled` from
|
|
44
|
-
* the blob store (evidence). This is the decision/payload split the replay argument
|
|
45
|
-
* rests on (B1/B2).
|
|
46
|
-
*
|
|
47
|
-
* Replay determinism (B2): `seq` is the monotonic cursor order `scope.next()` yielded
|
|
48
|
-
* each settlement — NOT wall-clock. `replaySpawnTree` sorts strictly by `seq` before
|
|
49
|
-
* touching the blob store, so the order in which rehydration `get`s resolve can never
|
|
50
|
-
* reorder the replayed `Settled[]`; the result is identical regardless of blob latency.
|
|
51
|
-
*
|
|
52
|
-
* @experimental
|
|
53
|
-
*/
|
|
54
|
-
|
|
55
|
-
/**
|
|
56
|
-
* Mint the content-addressed `outRef` for a result artifact: `sha256:<hex>` over a
|
|
57
|
-
* stable JSON encoding. Producers call this to derive the `outRef` they journal and
|
|
58
|
-
* `put`; the FS/in-mem stores re-derive it on `put` to verify the supplied ref
|
|
59
|
-
* matches (fail loud on a mismatch — a forged ref breaks the replay invariant).
|
|
60
|
-
*
|
|
61
|
-
* Stable encoding: object keys are sorted recursively so two structurally-equal
|
|
62
|
-
* artifacts hash identically regardless of key insertion order.
|
|
63
|
-
*/
|
|
64
|
-
declare function contentAddress(artifact: unknown): string;
|
|
65
|
-
/**
|
|
66
|
-
* In-memory `ResultBlobStore`. Content-addressed: `put` verifies the supplied
|
|
67
|
-
* `outRef` matches the artifact's hash so a stale/forged ref fails loud rather than
|
|
68
|
-
* silently rehydrating the wrong payload. Idempotent on an identical re-put.
|
|
69
|
-
*/
|
|
70
|
-
declare class InMemoryResultBlobStore implements ResultBlobStore {
|
|
71
|
-
private readonly blobs;
|
|
72
|
-
put(outRef: string, artifact: unknown): Promise<void>;
|
|
73
|
-
get(outRef: string): Promise<unknown | undefined>;
|
|
74
|
-
}
|
|
75
|
-
/**
|
|
76
|
-
* FS `ResultBlobStore`. One JSON file per artifact under `dir`, named by a
|
|
77
|
-
* filesystem-safe encoding of the `outRef` (`sha256:<hex>` → `sha256-<hex>.json`).
|
|
78
|
-
* `put` fsyncs so a crash between writes never loses an acknowledged blob.
|
|
79
|
-
*/
|
|
80
|
-
declare class FileResultBlobStore implements ResultBlobStore {
|
|
81
|
-
private readonly dir;
|
|
82
|
-
constructor(dir: string);
|
|
83
|
-
put(outRef: string, artifact: unknown): Promise<void>;
|
|
84
|
-
get(outRef: string): Promise<unknown | undefined>;
|
|
85
|
-
private blobPath;
|
|
86
|
-
}
|
|
87
|
-
/**
|
|
88
|
-
* In-memory `SpawnJournal`. Appends are observed-committed only; the impl enforces
|
|
89
|
-
* the corruption guards a durable replay rests on:
|
|
90
|
-
* - an event before `beginTree` is a corrupted tree (fail loud),
|
|
91
|
-
* - a duplicate `seq` within a tree is a corrupted cursor (fail loud) — two
|
|
92
|
-
* settlements cannot share the cursor position replay orders by.
|
|
93
|
-
*/
|
|
94
|
-
declare class InMemorySpawnJournal implements SpawnJournal {
|
|
95
|
-
private readonly trees;
|
|
96
|
-
loadTree(root: NodeId): Promise<SpawnEvent[] | undefined>;
|
|
97
|
-
beginTree(root: NodeId, at: string): Promise<void>;
|
|
98
|
-
appendEvent(root: NodeId, ev: SpawnEvent): Promise<void>;
|
|
99
|
-
}
|
|
100
|
-
/**
|
|
101
|
-
* JSONL on disk. One line per record: the first record is `begin`, subsequent records
|
|
102
|
-
* are `event` envelopes wrapping a `SpawnEvent`. `loadTree` replays the whole file,
|
|
103
|
-
* filtering by `root`, and applies the same begin-precedes-events + unique-seq
|
|
104
|
-
* corruption guards as the in-memory impl. Each append fsyncs so a crash between
|
|
105
|
-
* writes never loses an acknowledged event.
|
|
106
|
-
*/
|
|
107
|
-
declare class FileSpawnJournal implements SpawnJournal {
|
|
108
|
-
private readonly path;
|
|
109
|
-
constructor(path: string);
|
|
110
|
-
loadTree(root: NodeId): Promise<SpawnEvent[] | undefined>;
|
|
111
|
-
beginTree(root: NodeId, at: string): Promise<void>;
|
|
112
|
-
appendEvent(root: NodeId, ev: SpawnEvent): Promise<void>;
|
|
113
|
-
private loadTreeBegin;
|
|
114
|
-
private appendRecord;
|
|
115
|
-
}
|
|
116
|
-
/**
|
|
117
|
-
* Re-feed a journaled spawn tree in strict `seq` order, rehydrating each settled
|
|
118
|
-
* child's `out` from the blob store by `outRef`, and return the `Settled[]` exactly
|
|
119
|
-
* as `scope.next()` originally delivered them.
|
|
120
|
-
*
|
|
121
|
-
* Determinism (B2): the events are sorted by `seq` BEFORE any blob `get`, so the
|
|
122
|
-
* replay order is the recorded cursor order regardless of how fast each rehydration
|
|
123
|
-
* resolves. `at` (wall-clock) is never a replay input. Fail loud on a tree that was
|
|
124
|
-
* never begun, a settled-done event missing its `outRef`, or a blob the store can't
|
|
125
|
-
* rehydrate — a silent gap would let `act` branch on the wrong evidence.
|
|
126
|
-
*/
|
|
127
|
-
declare function replaySpawnTree(journal: SpawnJournal, blobs: ResultBlobStore, root: NodeId): Promise<Settled<unknown>[]>;
|
|
128
|
-
/**
|
|
129
|
-
* Materialize a recorded `TreeView` from a journaled event list for inspection. Folds
|
|
130
|
-
* `spawned`/`settled`/`cancelled` into a per-node snapshot in `seq` order, then adds each
|
|
131
|
-
* `metered` event's driver-inference spend onto its node in a separate additive pass so the view
|
|
132
|
-
* matches the recorded cursor. It does not recover live executors or driver state after restart.
|
|
133
|
-
*/
|
|
134
|
-
declare function materializeTreeView(events: SpawnEvent[]): TreeView;
|
|
135
|
-
/**
|
|
136
|
-
* The waits a journaled tree shows as ARMED but never woken — what a resumed run re-arms with the
|
|
137
|
-
* ORIGINAL absolute deadline. Reading it from the journal (rather than from any live state) is
|
|
138
|
-
* what makes "SIGKILL a waiting tree, a new process keeps waiting to the same instant" true.
|
|
139
|
-
*/
|
|
140
|
-
declare function pendingWaits(events: SpawnEvent[]): PendingWait[];
|
|
141
|
-
|
|
142
|
-
/**
|
|
143
|
-
* createWaterfallCollector — 100% trajectory observability from the lifecycle stream:
|
|
144
|
-
* every spawn/settle (shots, analysts, nested agents) becomes one timed, billed span.
|
|
145
|
-
* The sum of spans IS the run's cost story — what each step cost in dollars, tokens,
|
|
146
|
-
* and wall-clock, rendered as a text waterfall or exported as structured rows for any
|
|
147
|
-
* chart. Attach the collector's `hooks` to `runAgentic`/`runBenchmark`; spans accumulate
|
|
148
|
-
* across every task the hooks observe.
|
|
149
|
-
*/
|
|
150
|
-
|
|
151
|
-
interface WaterfallSpan {
|
|
152
|
-
id: string;
|
|
153
|
-
/** The spawn label (`shot:0`, `analyst:1`, a nested agent's label) — the row name. */
|
|
154
|
-
label: string;
|
|
155
|
-
runId: string;
|
|
156
|
-
parentId?: string;
|
|
157
|
-
startMs: number;
|
|
158
|
-
endMs?: number;
|
|
159
|
-
status: 'running' | 'done' | 'down';
|
|
160
|
-
usd: number;
|
|
161
|
-
tokens: {
|
|
162
|
-
input: number;
|
|
163
|
-
output: number;
|
|
164
|
-
};
|
|
165
|
-
score?: number;
|
|
166
|
-
}
|
|
167
|
-
interface WaterfallReport {
|
|
168
|
-
spans: WaterfallSpan[];
|
|
169
|
-
/** Wall-clock of the observed window (first spawn → last settle). */
|
|
170
|
-
totalMs: number;
|
|
171
|
-
totalUsd: number;
|
|
172
|
-
totalTokens: {
|
|
173
|
-
input: number;
|
|
174
|
-
output: number;
|
|
175
|
-
};
|
|
176
|
-
/** Rollup by label prefix (the part before ':') — shots vs analysts vs anything else. */
|
|
177
|
-
byKind: Record<string, {
|
|
178
|
-
count: number;
|
|
179
|
-
ms: number;
|
|
180
|
-
usd: number;
|
|
181
|
-
tokens: {
|
|
182
|
-
input: number;
|
|
183
|
-
output: number;
|
|
184
|
-
};
|
|
185
|
-
}>;
|
|
186
|
-
}
|
|
187
|
-
interface WaterfallCollector {
|
|
188
|
-
/** Attach these to RunAgenticOptions.hooks / BenchmarkConfig.hooks. */
|
|
189
|
-
hooks: RuntimeHooks;
|
|
190
|
-
report(): WaterfallReport;
|
|
191
|
-
/** The text waterfall — one row per span, bars scaled to the observed window. */
|
|
192
|
-
render(opts?: {
|
|
193
|
-
width?: number;
|
|
194
|
-
maxRows?: number;
|
|
195
|
-
}): string;
|
|
196
|
-
reset(): void;
|
|
197
|
-
}
|
|
198
|
-
/** Build a `WaterfallCollector` that records agent spans and renders them as an ASCII timeline. */
|
|
199
|
-
declare function createWaterfallCollector(): WaterfallCollector;
|
|
200
|
-
|
|
201
|
-
/**
|
|
202
|
-
* anytimeReport — time-to-satisfactory-output metrics, derived entirely from the
|
|
203
|
-
* waterfall's spans (no new instrumentation): per task, the best-so-far score after each
|
|
204
|
-
* shot with its elapsed wall-clock and cumulative spend; per strategy, the standard
|
|
205
|
-
* anytime-optimization metrics:
|
|
206
|
-
*
|
|
207
|
-
* TTT time-to-target — elapsed ms until best-so-far ≥ the target (per task; median
|
|
208
|
-
* over tasks that reached it)
|
|
209
|
-
* STT shots-to-target — attempts until best-so-far ≥ target
|
|
210
|
-
* ERT expected running time (the COCO benchmarking convention): TOTAL time spent
|
|
211
|
-
* across all tasks — including failures' full budgets — divided by the number of
|
|
212
|
-
* tasks that reached the target. The honest "how long per success, all-in".
|
|
213
|
-
* AUC the anytime curve's area (mean best-so-far score across the budget, per shot
|
|
214
|
-
* index) — higher = climbs earlier.
|
|
215
|
-
*
|
|
216
|
-
* The "satisfactory" bar follows the COCO/BBOB convention: a SET of satisficing targets
|
|
217
|
-
* (e.g. [0.5, 0.8, 1.0] on the normalized check score), each measured independently —
|
|
218
|
-
* runtime-to-target per (task, target) pair — optionally overridden per task
|
|
219
|
-
* (`targetFor`) when satisfaction is task-specific. Spans come from
|
|
220
|
-
* `createWaterfallCollector().report()`; tasks are grouped by the supervisor runId
|
|
221
|
-
* (`agentic:<strategy>:<taskId>`); shot spans are `shot:N` labels.
|
|
222
|
-
*/
|
|
223
|
-
|
|
224
|
-
interface AnytimeTaskCurve {
|
|
225
|
-
taskId: string;
|
|
226
|
-
strategy: string;
|
|
227
|
-
/** Best-so-far after each settled shot: elapsed ms from the task's first spawn,
|
|
228
|
-
* cumulative usd, and the running max score. */
|
|
229
|
-
points: Array<{
|
|
230
|
-
elapsedMs: number;
|
|
231
|
-
cumUsd: number;
|
|
232
|
-
best: number;
|
|
233
|
-
}>;
|
|
234
|
-
/** Per satisficing target (keyed by the target value as a string): the first point
|
|
235
|
-
* where best ≥ target, or null when never reached within budget. */
|
|
236
|
-
hits: Record<string, {
|
|
237
|
-
ms: number;
|
|
238
|
-
shots: number;
|
|
239
|
-
usd: number;
|
|
240
|
-
} | null>;
|
|
241
|
-
}
|
|
242
|
-
interface AnytimeStrategySummary {
|
|
243
|
-
strategy: string;
|
|
244
|
-
/** The satisficing target this row summarizes. */
|
|
245
|
-
target: number;
|
|
246
|
-
tasks: number;
|
|
247
|
-
reachedTarget: number;
|
|
248
|
-
/** Median time-to-target over the tasks that reached it (null when none did). */
|
|
249
|
-
medianTttMs: number | null;
|
|
250
|
-
medianShotsToTarget: number | null;
|
|
251
|
-
/** COCO ERT: Σ all task wall-time (incl. failures) / #successes. Null when 0 succeed. */
|
|
252
|
-
ertMs: number | null;
|
|
253
|
-
/** Same construction over dollars: Σ all spend / #successes. */
|
|
254
|
-
erUsd: number | null;
|
|
255
|
-
/** Mean best-so-far score by shot index (the anytime curve, averaged over tasks). */
|
|
256
|
-
curveByShot: number[];
|
|
257
|
-
/** Area under the per-shot anytime curve, normalized to [0,1]. */
|
|
258
|
-
auc: number;
|
|
259
|
-
}
|
|
260
|
-
interface AnytimeReport {
|
|
261
|
-
targets: number[];
|
|
262
|
-
perTask: AnytimeTaskCurve[];
|
|
263
|
-
/** One summary per (strategy, target) pair — the COCO-style multi-target view. */
|
|
264
|
-
perStrategy: AnytimeStrategySummary[];
|
|
265
|
-
}
|
|
266
|
-
/**
|
|
267
|
-
* The best-so-far fold — the ONE definition of "how good was the run after k results", shared by
|
|
268
|
-
* the post-run anytime report below and by the LIVE progress-based stop rules
|
|
269
|
-
* (`supervise/stop-rules.ts`). Given the observed objective per settled result in order, it returns
|
|
270
|
-
* the running maximum. A result with no objective (`undefined` — it failed, or it was never
|
|
271
|
-
* scored) carries the previous best forward rather than resetting it.
|
|
272
|
-
*
|
|
273
|
-
* It is extracted rather than duplicated on purpose: a stop rule that decides a run has plateaued
|
|
274
|
-
* must agree, number for number, with the report that later says whether stopping was right.
|
|
275
|
-
*/
|
|
276
|
-
declare function bestSoFar(values: ReadonlyArray<number | undefined>): number[];
|
|
277
|
-
/** Mean of a best-so-far curve — the anytime AUC when the curve is normalized to [0,1]. Higher =
|
|
278
|
-
* the run climbed earlier. Shared with the stop rules so "improving" means one thing. */
|
|
279
|
-
declare function areaUnderCurve(curve: ReadonlyArray<number>): number;
|
|
280
|
-
/**
|
|
281
|
-
* How many trailing entries of a best-so-far curve are within `minDelta` of the curve's value
|
|
282
|
-
* `window` steps back — i.e. the length of the current PLATEAU, in settles. `0` means the most
|
|
283
|
-
* recent settle improved the best by more than `minDelta`.
|
|
284
|
-
*
|
|
285
|
-
* The plateau math the live stop rules read. Defined here, beside the report that measures whether
|
|
286
|
-
* the plateau was real, so there is exactly one notion of "not improving".
|
|
287
|
-
*/
|
|
288
|
-
declare function plateauLength(curve: ReadonlyArray<number>, minDelta: number): number;
|
|
289
|
-
/** Derive anytime metrics from waterfall spans. `targets` are the satisficing score
|
|
290
|
-
* bars (default [1] = fully resolved; COCO-style multi-target: [0.5, 0.8, 1]);
|
|
291
|
-
* `targetFor` overrides the bar per task (task-specific satisfaction) — when set, the
|
|
292
|
-
* per-task bar replaces every entry of `targets` for that task. */
|
|
293
|
-
declare function anytimeReport(spans: WaterfallSpan[], opts?: {
|
|
294
|
-
targets?: number[];
|
|
295
|
-
targetFor?: (taskId: string) => number;
|
|
296
|
-
}): AnytimeReport;
|
|
297
|
-
/** One row per (strategy, satisficing target): the shareable time-to-satisfactory table. */
|
|
298
|
-
declare function renderAnytimeTable(report: AnytimeReport): string;
|
|
299
|
-
|
|
300
|
-
/**
|
|
301
|
-
* auditIntent — the route-rigor analyst: is this trajectory even going the RIGHT WAY?
|
|
302
|
-
*
|
|
303
|
-
* `observe()` critiques execution quality ("what's unfinished"). This audits ALIGNMENT —
|
|
304
|
-
* a different failure class the score can't see until it's too late: an agent can be
|
|
305
|
-
* executing flawlessly down the wrong route. The auditor reads the trajectory and
|
|
306
|
-
* compares three intents:
|
|
307
|
-
*
|
|
308
|
-
* declared — what the task says to do (the prompt / acceptance criteria)
|
|
309
|
-
* revealed — what the agent is ACTUALLY optimizing, inferred from its action pattern
|
|
310
|
-
* (the inverse-inference move: actions reveal objectives)
|
|
311
|
-
* user — what the principal actually wants (the contract, when it differs from
|
|
312
|
-
* the literal task text), plus where the user's own trajectory is heading
|
|
313
|
-
*
|
|
314
|
-
* and returns a verdict (aligned / drifting / diverged) with evidence and ONE
|
|
315
|
-
* recommended intervention. FIREWALLED like every analyst: input is the trajectory and
|
|
316
|
-
* the intents — never the verifier or its data (zero check-leakage, so route auditing
|
|
317
|
-
* is always Goodhart-safe to run online).
|
|
318
|
-
*
|
|
319
|
-
* Where it runs: between shots (steer the next one), as a watchdog over the lifecycle
|
|
320
|
-
* stream (abort-and-refund a diverged rollout — the budget pool makes early abort
|
|
321
|
-
* strictly valuable), or post-hoc over a whole BenchmarkReport (the meta-intent pass:
|
|
322
|
-
* is the LOOP optimizing the right thing — degenerate submissions, check-gaming shapes,
|
|
323
|
-
* objective drift across tasks).
|
|
324
|
-
*/
|
|
325
|
-
|
|
326
|
-
interface AuditIntentInput {
|
|
327
|
-
/** The declared intent: the task text / acceptance criteria the agent was given. */
|
|
328
|
-
declaredIntent: string;
|
|
329
|
-
/** The trajectory so far — tool calls + results + assistant turns (any event shapes). */
|
|
330
|
-
trace: ReadonlyArray<unknown>;
|
|
331
|
-
/** The principal's actual intent when it differs from the literal task (the contract). */
|
|
332
|
-
userIntent?: string;
|
|
333
|
-
/** The loop-level purpose (meta-intent): what the WHOLE run is for — lets the auditor
|
|
334
|
-
* flag locally-sensible work that serves the wrong larger objective. */
|
|
335
|
-
metaIntent?: string;
|
|
336
|
-
runId?: string;
|
|
337
|
-
}
|
|
338
|
-
interface AuditIntentOptions {
|
|
339
|
-
chat: ChatClient;
|
|
340
|
-
model?: string;
|
|
341
|
-
/** Override the auditor instruction (optimizable like any analyst prompt). */
|
|
342
|
-
auditorInstruction?: string;
|
|
343
|
-
/** Cap trace lines fed to the auditor. Default 80. */
|
|
344
|
-
maxTraceLines?: number;
|
|
345
|
-
signal?: AbortSignal;
|
|
346
|
-
}
|
|
347
|
-
interface IntentAudit {
|
|
348
|
-
/** What the agent's actions reveal it is actually optimizing — one sentence. */
|
|
349
|
-
revealedIntent: string;
|
|
350
|
-
verdict: 'aligned' | 'drifting' | 'diverged';
|
|
351
|
-
/** Trajectory-grounded evidence for the verdict (specific calls/patterns). */
|
|
352
|
-
evidence: string;
|
|
353
|
-
/** The single recommended intervention. */
|
|
354
|
-
recommendation: 'continue' | 'steer' | 'abort';
|
|
355
|
-
/** When recommendation is 'steer': the corrective instruction to inject. */
|
|
356
|
-
steer?: string;
|
|
357
|
-
confidence: number;
|
|
358
|
-
}
|
|
359
|
-
/** Default system instruction for intent-auditor agents: diagnose diverged/drifting trajectories. */
|
|
360
|
-
declare const defaultAuditorInstruction: string;
|
|
361
|
-
/** The route-rigor analyst: compare declared vs revealed vs user intent over a trajectory and return aligned / drifting / diverged with evidence and one recommended intervention. */
|
|
362
|
-
declare function auditIntent(input: AuditIntentInput, opts: AuditIntentOptions): Promise<IntentAudit>;
|
|
363
|
-
|
|
364
|
-
/**
|
|
365
|
-
* benchmark-report — turn a fleet of `RunRecord`s into a publishable, multi-axis benchmark report:
|
|
366
|
-
* a ranked leaderboard, the full profile×axis score matrix, the cost/latency/token columns, and
|
|
367
|
-
* embeddable charts (SVG) + a self-contained HTML page. Domain-agnostic by construction — it reads ONLY
|
|
368
|
-
* the universal `RunRecord` currency (model, scenario, score, cost, tokens, latency, the `outcome.raw`
|
|
369
|
-
* metric bag), so the SAME engine reports any benchmark in any domain: coding, search, agents, multimodal.
|
|
370
|
-
*
|
|
371
|
-
* This is the surface a hosted leaderboard (à la vals.ai) renders: for every harness×model profile, its
|
|
372
|
-
* score on every axis, not a curated subset. It pairs with `runProfileMatrix` (whose `result.records`
|
|
373
|
-
* feed straight in) but takes the records directly, so it is independent of how they were produced.
|
|
374
|
-
*
|
|
375
|
-
* An AXIS is any way to slice the score into columns — by default one column per scenario group
|
|
376
|
-
* (`groupOf`, default = the scenario id), so the matrix is profile × scenario. Pass `axisScoresOf` to
|
|
377
|
-
* score along judge dimensions or any custom decomposition instead. The reporter never invents a number:
|
|
378
|
-
* a missing cell renders blank, never zero.
|
|
379
|
-
*/
|
|
380
|
-
|
|
381
|
-
/** Pull the headline score in [0,1] from a record. Default: the held-out split, else the search split,
|
|
382
|
-
* else a `composite`/`passed`/`score` entry in the raw bag. Override to score a domain differently. */
|
|
383
|
-
type ScoreOf = (record: RunRecord) => number | undefined;
|
|
384
|
-
/** The profile (matrix row) a record belongs to — default `harness·model` from the record's profile cell,
|
|
385
|
-
* falling back to the model. This is the leaderboard's unit of comparison. */
|
|
386
|
-
type ProfileKeyOf = (record: RunRecord) => string;
|
|
387
|
-
/** The axis (matrix column) a record contributes to — default the scenario group. */
|
|
388
|
-
type GroupOf = (record: RunRecord) => string;
|
|
389
|
-
/** Decompose ONE record into per-axis scores (e.g. judge dimensions). When set, it REPLACES the
|
|
390
|
-
* scenario-group axes: the column set is the union of returned keys. */
|
|
391
|
-
type AxisScoresOf = (record: RunRecord) => Record<string, number>;
|
|
392
|
-
interface LeaderboardOptions {
|
|
393
|
-
readonly title?: string;
|
|
394
|
-
readonly scoreOf?: ScoreOf;
|
|
395
|
-
readonly profileKeyOf?: ProfileKeyOf;
|
|
396
|
-
readonly groupOf?: GroupOf;
|
|
397
|
-
readonly axisScoresOf?: AxisScoresOf;
|
|
398
|
-
/** Display label for a profile key (default: the key itself). */
|
|
399
|
-
readonly labelOf?: (profileKey: string) => string;
|
|
400
|
-
/** Commit SHA / dataset / dates surfaced in the provenance block. */
|
|
401
|
-
readonly meta?: Record<string, string>;
|
|
402
|
-
/** Compute per-row confidence intervals (bootstrap on score, Wilson on pass rate). Needs a
|
|
403
|
-
* `scenarioId` on every record (reps are collapsed per scenario for the honest n). Default off. */
|
|
404
|
-
readonly stats?: boolean;
|
|
405
|
-
/** A score ≥ this counts as a "pass" for the pass-rate proportion + its Wilson CI. Default 0.999
|
|
406
|
-
* (fully solved). Lower it (e.g. 0.6) for a partial-credit domain. */
|
|
407
|
-
readonly passThreshold?: number;
|
|
408
|
-
}
|
|
409
|
-
/** A 95%-by-default confidence interval. */
|
|
410
|
-
interface Interval {
|
|
411
|
-
readonly lower: number;
|
|
412
|
-
readonly upper: number;
|
|
413
|
-
}
|
|
414
|
-
/** One leaderboard row — a harness×model profile, every measured column. */
|
|
415
|
-
interface LeaderboardRow {
|
|
416
|
-
readonly profileKey: string;
|
|
417
|
-
readonly label: string;
|
|
418
|
-
readonly model: string;
|
|
419
|
-
readonly n: number;
|
|
420
|
-
readonly meanScore: number;
|
|
421
|
-
/** Fraction of records scoring ≥ `passThreshold` (default 0.999) — the binary pass rate. */
|
|
422
|
-
readonly solveRate: number;
|
|
423
|
-
/** axis → mean score for this profile (blank in render when the profile never ran that axis). */
|
|
424
|
-
readonly perAxis: Record<string, number>;
|
|
425
|
-
readonly costUsd: number;
|
|
426
|
-
readonly tokensIn: number;
|
|
427
|
-
readonly tokensOut: number;
|
|
428
|
-
readonly latencyP50Ms: number;
|
|
429
|
-
readonly latencyP90Ms: number;
|
|
430
|
-
/** Bootstrap CI on the mean score — present only when `opts.stats` is set. Computed over
|
|
431
|
-
* per-scenario means (reps collapsed first), so identical reps can't fake a narrow interval. */
|
|
432
|
-
readonly scoreCi?: Interval;
|
|
433
|
-
/** Wilson CI on the pass rate — present only when `opts.stats` is set. */
|
|
434
|
-
readonly passCi?: Interval;
|
|
435
|
-
}
|
|
436
|
-
interface Leaderboard {
|
|
437
|
-
readonly title: string;
|
|
438
|
-
/** Column order — scenario groups (default) or dimension keys (`axisScoresOf`). */
|
|
439
|
-
readonly axes: readonly string[];
|
|
440
|
-
/** Rows ranked by `meanScore` desc (ties → lower cost, then label). */
|
|
441
|
-
readonly profiles: readonly LeaderboardRow[];
|
|
442
|
-
readonly meta: Record<string, string>;
|
|
443
|
-
/** Provenance counts — the denominators every honest report leads with. */
|
|
444
|
-
readonly provenance: {
|
|
445
|
-
readonly records: number;
|
|
446
|
-
readonly profiles: number;
|
|
447
|
-
readonly axes: number;
|
|
448
|
-
readonly models: readonly string[];
|
|
449
|
-
readonly totalCostUsd: number;
|
|
450
|
-
};
|
|
451
|
-
}
|
|
452
|
-
/** Aggregate a fleet of records into the ranked, multi-axis report. Pure — no IO, deterministic. */
|
|
453
|
-
declare function leaderboard(records: readonly RunRecord[], opts?: LeaderboardOptions): Leaderboard;
|
|
454
|
-
/** One profile pair compared on the scenarios they BOTH ran — the "who actually beat whom" verdict. */
|
|
455
|
-
interface PairwiseVerdict {
|
|
456
|
-
readonly a: string;
|
|
457
|
-
readonly b: string;
|
|
458
|
-
/** Paired unit count (shared scenarios). The significance is suppressed below `minPairs`. */
|
|
459
|
-
readonly pairs: number;
|
|
460
|
-
/** Median paired delta (b − a) and its bootstrap CI. */
|
|
461
|
-
readonly delta: number;
|
|
462
|
-
readonly ciLow: number;
|
|
463
|
-
readonly ciHigh: number;
|
|
464
|
-
/** Paired-test p-value (before correction). */
|
|
465
|
-
readonly p: number;
|
|
466
|
-
/** BH-significant across ALL pairs AND above the `minPairs` power floor. */
|
|
467
|
-
readonly significant: boolean;
|
|
468
|
-
}
|
|
469
|
-
interface PairwiseOptions {
|
|
470
|
-
readonly scoreOf?: ScoreOf;
|
|
471
|
-
readonly profileKeyOf?: ProfileKeyOf;
|
|
472
|
-
readonly labelOf?: (profileKey: string) => string;
|
|
473
|
-
/** False-discovery rate for the Benjamini–Hochberg correction. Default 0.05. */
|
|
474
|
-
readonly fdr?: number;
|
|
475
|
-
/** Below this many shared scenarios a paired test can't defensibly separate two profiles, so the
|
|
476
|
-
* `significant` tag is suppressed regardless of p (small-n mirage protection). Default 12. */
|
|
477
|
-
readonly minPairs?: number;
|
|
478
|
-
}
|
|
479
|
-
/** Compare EVERY profile pair on the scenarios they both ran — paired-bootstrap effect + CI, a real
|
|
480
|
-
* paired-test p-value, BH-corrected across all pairs. This is the honest "did A beat B" table the
|
|
481
|
-
* leaderboard's point ranking cannot answer. Reuses the agent-eval statistics substrate. */
|
|
482
|
-
declare function pairwiseSignificance(records: readonly RunRecord[], opts?: PairwiseOptions): PairwiseVerdict[];
|
|
483
|
-
/** Render the report as a publishable Markdown document: provenance → leaderboard → the full profile×axis
|
|
484
|
-
* matrix → cost/latency/token columns. Every axis is shown — a curated subset is a reporting failure. */
|
|
485
|
-
declare function renderLeaderboardMarkdown(report: Leaderboard): string;
|
|
486
|
-
/** Render the pairwise-significance table — every profile pair's paired delta, CI, and BH-corrected
|
|
487
|
-
* verdict. Feed it `pairwiseSignificance(records)`. This is the "did A really beat B" evidence the point
|
|
488
|
-
* ranking cannot give. */
|
|
489
|
-
declare function renderPairwiseMarkdown(verdicts: readonly PairwiseVerdict[], title?: string): string;
|
|
490
|
-
/** Render a self-contained SVG: a ranked score bar chart on top, the profile×axis heatmap below. No deps,
|
|
491
|
-
* embeddable anywhere (README, HTML page, hosted leaderboard). */
|
|
492
|
-
declare function renderLeaderboardSvg(report: Leaderboard): string;
|
|
493
|
-
/** Render a self-contained HTML leaderboard page (the hosted surface): the SVG charts + the full Markdown
|
|
494
|
-
* matrix as a table. Single file, no assets, opens in any browser. */
|
|
495
|
-
declare function renderLeaderboardHtml(report: Leaderboard): string;
|
|
496
|
-
|
|
497
|
-
/**
|
|
498
|
-
*
|
|
499
|
-
* Completion / satisfiability — the OTHER output of the pluggable analyst (the steer output
|
|
500
|
-
* is `AnalystFinding[]` via the `analyze` hook; this is the "is it done?" output via the
|
|
501
|
-
* `complete` hook). A `CompletionAnalyst` reads a node's trace and returns a `CompletionVerdict`
|
|
502
|
-
* the PARENT (driver) acts on: end the node, or keep going. It fits ANY node and composes to
|
|
503
|
-
* any depth — a 1-deep loop has one; an N-deep tree has one per node.
|
|
504
|
-
*
|
|
505
|
-
* The verdict's authority scales with its DETERMINISM (the thing that varies by task):
|
|
506
|
-
* - `deterministic` — build/test/lint pass, a proof checks, every claim's citation resolves:
|
|
507
|
-
* ground truth, the driver TRUSTS it and ends. Not an opinion.
|
|
508
|
-
* - `probabilistic` — a quality/soundness judgment (marketing, "the experiment is sound"):
|
|
509
|
-
* ADVISORY. It passes to the driver with its reasons; the driver validates (here: a
|
|
510
|
-
* confidence threshold; a richer driver may re-examine the reasons) before ending.
|
|
511
|
-
*
|
|
512
|
-
* Two stop-signal mechanisms, by node mode, both → one `CompletionVerdict`:
|
|
513
|
-
* - sandbox-agent (text stream): a unique per-node STOP SENTINEL the agent emits when done
|
|
514
|
-
* (`stopSentinel` / `sentinelCompletion`) — ralph-loop style; the seed makes it
|
|
515
|
-
* unguessable + attributable, so it can't be spuriously emitted or confused with content.
|
|
516
|
-
* - deterministic check (compile/test/citation/proof): `deterministicCompletion(check)` —
|
|
517
|
-
* a verifier over the output, never the judge verdict (selector ≠ judge holds).
|
|
518
|
-
*
|
|
519
|
-
* @experimental
|
|
520
|
-
*/
|
|
521
|
-
|
|
522
|
-
/** Trace-derived evidence for a completion claim — an artifact (output) or a verifier metric,
|
|
523
|
-
* never the judge's own verdict. Mirrors the steer-firewall's provenance discipline. */
|
|
524
|
-
interface CompletionEvidence {
|
|
525
|
-
kind: 'artifact' | 'metric';
|
|
526
|
-
uri: string;
|
|
527
|
-
}
|
|
528
|
-
/** The "is it done?" verdict an analyst returns to the parent. */
|
|
529
|
-
interface CompletionVerdict {
|
|
530
|
-
done: boolean;
|
|
531
|
-
/** How verifiable the claim is — sets whether the driver trusts it or validates it. */
|
|
532
|
-
determinism: 'deterministic' | 'probabilistic';
|
|
533
|
-
/** Why the analyst believes it is (or isn't) done — what the driver validates. */
|
|
534
|
-
reasons?: string;
|
|
535
|
-
/** 0..1, for probabilistic verdicts; the driver's validation threshold reads this. */
|
|
536
|
-
confidence?: number;
|
|
537
|
-
evidence?: ReadonlyArray<CompletionEvidence>;
|
|
538
|
-
}
|
|
539
|
-
/** Reads a node's trace → a completion verdict. Same input shape as the `analyze` hook, so
|
|
540
|
-
* ONE analyst node can back both channels (findings for steer, a verdict for stop). */
|
|
541
|
-
interface CompletionAnalyst<Task, Output> {
|
|
542
|
-
assess(input: {
|
|
543
|
-
task: Task;
|
|
544
|
-
history: ReadonlyArray<Iteration<Task, Output>>;
|
|
545
|
-
}): CompletionVerdict | Promise<CompletionVerdict>;
|
|
546
|
-
}
|
|
547
|
-
/** When a verdict authorizes the driver to END. Deterministic → trust (ground truth);
|
|
548
|
-
* probabilistic → validate by confidence threshold (the driver's check). */
|
|
549
|
-
interface CompletionPolicy {
|
|
550
|
-
/** Minimum confidence a PROBABILISTIC verdict must clear to end. Default 0.8. */
|
|
551
|
-
minConfidence?: number;
|
|
552
|
-
}
|
|
553
|
-
/** Decide whether a `CompletionVerdict` may end the node under the policy: authority scales with the verdict's determinism, and probabilistic verdicts must clear `minConfidence`. */
|
|
554
|
-
declare function completionAuthorizes(v: CompletionVerdict, policy?: CompletionPolicy): boolean;
|
|
555
|
-
/**
|
|
556
|
-
* A unique, attributable stop sentinel for a node (ralph-loop style). Deterministic from the
|
|
557
|
-
* seed (no Math.random — reproducible + attributable to the node); the agent is instructed to
|
|
558
|
-
* emit it VERBATIM when it judges itself done. Unguessable enough that content never trips it.
|
|
559
|
-
*/
|
|
560
|
-
declare function stopSentinel(seed: string): string;
|
|
561
|
-
/**
|
|
562
|
-
* Completion for a sandbox-agent node: done iff the latest output carries the node's stop
|
|
563
|
-
* sentinel. PROBABILISTIC (the agent's own self-judgment) — the driver validates it.
|
|
564
|
-
*/
|
|
565
|
-
declare function sentinelCompletion<Task>(sentinel: string, opts?: {
|
|
566
|
-
confidence?: number;
|
|
567
|
-
}): CompletionAnalyst<Task, string>;
|
|
568
|
-
/**
|
|
569
|
-
* Completion for a DETERMINISTIC check (build/test/lint/citation/proof): done iff the check
|
|
570
|
-
* passes. Ground truth — the driver ends directly, no validation. The check reads the output
|
|
571
|
-
* (a verifier), never the judge verdict — selector ≠ judge stays intact.
|
|
572
|
-
*/
|
|
573
|
-
declare function deterministicCompletion<Task, Output>(check: (output: Output, history: ReadonlyArray<Iteration<Task, Output>>) => {
|
|
574
|
-
passed: boolean;
|
|
575
|
-
reasons?: string;
|
|
576
|
-
}): CompletionAnalyst<Task, Output>;
|
|
577
|
-
|
|
578
|
-
/** Structured per-case verdict a `score` function may return (a bare number is
|
|
579
|
-
* shorthand for `{ composite }`). `composite` is the [0,1] leaderboard score;
|
|
580
|
-
* `dimensions` are recorded as extra judge dimensions. */
|
|
581
|
-
interface LeaderboardScore {
|
|
582
|
-
composite: number;
|
|
583
|
-
dimensions?: Record<string, number>;
|
|
584
|
-
notes?: string;
|
|
585
|
-
}
|
|
586
|
-
/** The campaign scenario a case is wrapped into: the case rides along so
|
|
587
|
-
* judges and hooks can reach the full domain payload, not just its id. */
|
|
588
|
-
interface LeaderboardScenario<TCase> extends Scenario {
|
|
589
|
-
case: TCase;
|
|
590
|
-
}
|
|
591
|
-
/** One extra CLI flag a spec declares. Parsed by `run()` as `--<name> <value>`
|
|
592
|
-
* and surfaced to every hook via `ctx.args`. */
|
|
593
|
-
interface LeaderboardFlagSpec {
|
|
594
|
-
default?: string;
|
|
595
|
-
description: string;
|
|
596
|
-
}
|
|
597
|
-
/** Resolved run configuration handed to `setup` / `teardown` / `export`. */
|
|
598
|
-
interface LeaderboardRunContext {
|
|
599
|
-
name: string;
|
|
600
|
-
/** Execution backend name (`--backend`), a key of `backends`. */
|
|
601
|
-
backend: string;
|
|
602
|
-
runDir: string;
|
|
603
|
-
exportDir: string;
|
|
604
|
-
/** Every parsed flag (standard + `spec.flags`), by name without `--`. */
|
|
605
|
-
args: Record<string, string | undefined>;
|
|
606
|
-
harnesses: readonly HarnessType[];
|
|
607
|
-
/** Snapshot-stamped model ids (`name@snapshot`) — the eval identity models. */
|
|
608
|
-
models: readonly string[];
|
|
609
|
-
caseIds: readonly string[];
|
|
610
|
-
shots: number;
|
|
611
|
-
reps: number;
|
|
612
|
-
}
|
|
613
|
-
/** Structurally `BenchTask` (bench registry shape) — declared locally so this
|
|
614
|
-
* module adds no dependency on a benchmark package. */
|
|
615
|
-
interface LeaderboardBenchTask {
|
|
616
|
-
id: string;
|
|
617
|
-
prompt: string;
|
|
618
|
-
split?: string;
|
|
619
|
-
metadata?: Record<string, unknown>;
|
|
620
|
-
}
|
|
621
|
-
/** Structurally `BenchScore` (bench registry shape). */
|
|
622
|
-
interface LeaderboardBenchScore {
|
|
623
|
-
resolved: boolean;
|
|
624
|
-
score: number;
|
|
625
|
-
detail?: string;
|
|
626
|
-
}
|
|
627
|
-
/** Structurally `BenchmarkAdapter` (bench registry shape): `name`,
|
|
628
|
-
* `preflight()`, `loadTasks()`, deterministic `judge()`, `goldArtifact()`.
|
|
629
|
-
* Generic over the artifact channel; the `string` default IS the registry
|
|
630
|
-
* shape, so a default-artifact adapter registers unchanged. */
|
|
631
|
-
interface LeaderboardBenchmarkAdapter<TArtifact = string> {
|
|
632
|
-
readonly name: string;
|
|
633
|
-
preflight(): Promise<void>;
|
|
634
|
-
loadTasks(opts?: {
|
|
635
|
-
limit?: number;
|
|
636
|
-
split?: string;
|
|
637
|
-
ids?: string[];
|
|
638
|
-
}): Promise<LeaderboardBenchTask[]>;
|
|
639
|
-
judge(task: LeaderboardBenchTask, artifact: TArtifact): Promise<LeaderboardBenchScore>;
|
|
640
|
-
goldArtifact(task: LeaderboardBenchTask): Promise<string | undefined>;
|
|
641
|
-
}
|
|
642
|
-
/** Per-shot outcome context passed as `onCellEvents`'s third argument — how a
|
|
643
|
-
* thrown shot (which never reaches `parseOutput`) stays visible through the
|
|
644
|
-
* facade instead of surfacing only as an empty zero-token cell. */
|
|
645
|
-
interface LeaderboardIterationInfo {
|
|
646
|
-
/** 0-based shot index within the cell. */
|
|
647
|
-
index: number;
|
|
648
|
-
/** The shot's thrown error message, when the shot failed before scoring. */
|
|
649
|
-
error?: string;
|
|
650
|
-
/** The shot's validator verdict, when the shot reached scoring. */
|
|
651
|
-
verdict?: {
|
|
652
|
-
score?: number;
|
|
653
|
-
};
|
|
654
|
-
}
|
|
655
|
-
/**
|
|
656
|
-
* The declarative leaderboard spec. `TArtifact` is the artifact channel the
|
|
657
|
-
* dispatch produces and the judges score — `string` (the default) is the plain
|
|
658
|
-
* agent-response-text path; a structured artifact type flows natively once the
|
|
659
|
-
* spec supplies `parseOutput` (or a LEVEL-2 `dispatch`) producing it.
|
|
660
|
-
*/
|
|
661
|
-
interface LeaderboardSpec<TCase, TArtifact = string> {
|
|
662
|
-
/** Leaderboard name — the scenario `kind`, default profile name, and report title. */
|
|
663
|
-
name: string;
|
|
664
|
-
/** The case corpus. Every case needs a stable string id (see `caseId`). */
|
|
665
|
-
cases: TCase[];
|
|
666
|
-
/** Stable id extractor. Default: the case's own `id` property (fail-loud
|
|
667
|
-
* when absent or not a string). */
|
|
668
|
-
caseId?: (c: TCase) => string;
|
|
669
|
-
/** The per-case task prompt. May be async (e.g. built by shelling out to a
|
|
670
|
-
* reference implementation); resolved ONCE per case before dispatch. */
|
|
671
|
-
prompt: (c: TCase) => string | Promise<string>;
|
|
672
|
-
/** The domain grader: agent output artifact → score. Used BOTH as the
|
|
673
|
-
* per-shot validator (a shot with `composite > 0` stops the naive retry
|
|
674
|
-
* loop) and, wrapped as a campaign judge, as the recorded leaderboard score. */
|
|
675
|
-
score: (output: TArtifact, c: TCase) => number | LeaderboardScore;
|
|
676
|
-
/** Harness × model axes for `expandProfileAxes`. Defaults: the canonical
|
|
677
|
-
* `CODING_HARNESSES` × the base profile's `model.default`. `--harnesses` /
|
|
678
|
-
* `--models` override per run. */
|
|
679
|
-
axis?: {
|
|
680
|
-
harnesses?: readonly HarnessType[];
|
|
681
|
-
models?: readonly string[];
|
|
682
|
-
};
|
|
683
|
-
/** Base profile the axes expand over (prompt/tools/skills held fixed).
|
|
684
|
-
* Default: a minimal `{ name, model: { default: <first model> } }`. */
|
|
685
|
-
baseProfile?: AgentProfile;
|
|
686
|
-
/**
|
|
687
|
-
* Execution-backend registry: `--backend <name>` picks the factory that
|
|
688
|
-
* yields the `SandboxClient` every cell runs on. Merged over the defaults:
|
|
689
|
-
* - `sandbox` — throws with guidance (a product must supply its real
|
|
690
|
-
* Sandbox-backed client; the facade has no credentials).
|
|
691
|
-
* - `cli-bridge` — `resolveSandboxClient({ backend: 'bridge' })` reading
|
|
692
|
-
* `CLI_BRIDGE_URL` + `BRIDGE_BEARER`/`CLI_BRIDGE_BEARER`; the per-cell
|
|
693
|
-
* harness/model ride in via `sandboxOverrides.backend`.
|
|
694
|
-
*/
|
|
695
|
-
backends?: Record<string, (() => SandboxClient) | undefined>;
|
|
696
|
-
/** Extra `--flag value` CLI args `run()` parses and surfaces via `ctx.args`. */
|
|
697
|
-
flags?: Record<string, LeaderboardFlagSpec>;
|
|
698
|
-
/** Extra fields merged into each cell's `backend.model` create override —
|
|
699
|
-
* e.g. `{ provider: 'openai-compat', apiKey, baseUrl }` for a router-backed
|
|
700
|
-
* sandbox. The cell's bare model id is set by the facade from the axis. */
|
|
701
|
-
modelBackend?: Record<string, unknown>;
|
|
702
|
-
/** Runs once before the matrix (fetch fixtures, warm caches). */
|
|
703
|
-
setup?: (ctx: LeaderboardRunContext) => Promise<void> | void;
|
|
704
|
-
/** Runs once after the matrix, even on failure (reap boxes, close handles). */
|
|
705
|
-
teardown?: (ctx: LeaderboardRunContext) => Promise<void> | void;
|
|
706
|
-
/** Per-cell event tap: the raw sandbox events of EVERY shot, with the case —
|
|
707
|
-
* the seam for domain metric capture (search counts, citations) without a
|
|
708
|
-
* substrate change. Fires once per shot after the cell's loop settles, in
|
|
709
|
-
* shot order, including thrown shots (whose events may be partial or empty);
|
|
710
|
-
* the third argument carries the shot's index + error/verdict outcome. */
|
|
711
|
-
onCellEvents?: (events: readonly SandboxEvent[], c: TCase, iteration?: LeaderboardIterationInfo) => void;
|
|
712
|
-
/** Output decode override: raw events → the scored artifact. Default: the
|
|
713
|
-
* sandbox SDK's `collectAgentResponseText` (final answer text; empty string
|
|
714
|
-
* when the stream carried none — which then scores 0). The default only
|
|
715
|
-
* produces `string`, so a spec with a structured `TArtifact` MUST supply
|
|
716
|
-
* this (or a LEVEL-2 `dispatch`). */
|
|
717
|
-
parseOutput?: (events: readonly SandboxEvent[], c: TCase) => TArtifact;
|
|
718
|
-
/**
|
|
719
|
-
* Resolve the model the backend ACTUALLY served off a shot's raw events.
|
|
720
|
-
* Required for HARNESS_NATIVE_MODEL-snapped cells (a vendor-locked harness ×
|
|
721
|
-
* an out-of-family model expands to the `default` sentinel): the RunRecord
|
|
722
|
-
* must pin a real snapshot-bearing model id, which only the dispatch —
|
|
723
|
-
* reading the backend's usage/terminal events — can know. When this returns
|
|
724
|
-
* a value the default dispatch records it on the paid-call receipt;
|
|
725
|
-
* in-family cells (concrete declared model) never need it.
|
|
726
|
-
*/
|
|
727
|
-
resolveModel?: (events: readonly SandboxEvent[]) => string | undefined;
|
|
728
|
-
/** Result export. Default: write `matrix-result.json` under the run dir and
|
|
729
|
-
* print (+ write) the ranked leaderboard markdown under the export dir. */
|
|
730
|
-
export?: (result: RunProfileMatrixResult<TArtifact, LeaderboardScenario<TCase>>, ctx: LeaderboardRunContext) => Promise<void> | void;
|
|
731
|
-
/** LEVEL 2 — full dispatch replacement (in-process products bring their own).
|
|
732
|
-
* The default is `loopDispatch` + `naiveDriver` over the resolved backend. */
|
|
733
|
-
dispatch?: ProfileDispatchFn<LeaderboardScenario<TCase>, TArtifact>;
|
|
734
|
-
/** LEVEL 2 — full judge replacement. Default: `score` wrapped as one judge. */
|
|
735
|
-
judges?: JudgeConfig<TArtifact, LeaderboardScenario<TCase>>[];
|
|
736
|
-
/** Naive-retry shot cap per cell (`--shots`). Default 1. */
|
|
737
|
-
shots?: number;
|
|
738
|
-
/** Replicates per cell (`--reps`). Default 1. */
|
|
739
|
-
reps?: number;
|
|
740
|
-
/** Provider- or executor-enforced maximum for one cell dispatch. Required
|
|
741
|
-
* before execution when `matrix.costCeiling` is configured. */
|
|
742
|
-
maximumCharge?: MaximumCharge | ((profile: AgentProfile, scenario: LeaderboardScenario<TCase>) => MaximumCharge | undefined);
|
|
743
|
-
/** Passthrough overrides spread onto the final `runProfileMatrix` call
|
|
744
|
-
* (e.g. `maxConcurrency`, `costCeiling`, `integrity`, `storage`) — spread
|
|
745
|
-
* LAST, so anything the facade wired can be overridden. */
|
|
746
|
-
matrix?: Partial<RunProfileMatrixOptions<LeaderboardScenario<TCase>, TArtifact>>;
|
|
747
|
-
}
|
|
748
|
-
interface DefinedLeaderboard<TCase, TArtifact = string> {
|
|
749
|
-
/**
|
|
750
|
-
* Parse flags, run the matrix, export, and return the raw result.
|
|
751
|
-
*
|
|
752
|
-
* Standard flags: `--backend <name>` (default `sandbox`), `--harnesses a,b`,
|
|
753
|
-
* `--models m1,m2`, `--cases id1,id2`, `--shots N`, `--reps N`,
|
|
754
|
-
* `--model-snapshot <tag>`, `--run-dir <path>`, `--export-dir <path>`,
|
|
755
|
-
* plus every `spec.flags` entry. `argv` defaults to `process.argv.slice(2)`.
|
|
756
|
-
*
|
|
757
|
-
* The default run dir is FRESH per invocation (timestamp+pid under the OS
|
|
758
|
-
* tmpdir). `runProfileMatrix` caches cells by run dir, and a stable default
|
|
759
|
-
* would silently reuse a prior FAILED zero-token cell and skip dispatch —
|
|
760
|
-
* only an explicit `--run-dir` opts into that resume behavior.
|
|
761
|
-
*/
|
|
762
|
-
run(argv?: string[]): Promise<RunProfileMatrixResult<TArtifact, LeaderboardScenario<TCase>>>;
|
|
763
|
-
/** The same domain surface in the structural `BenchmarkAdapter` shape. */
|
|
764
|
-
toBenchmarkAdapter(): LeaderboardBenchmarkAdapter<TArtifact>;
|
|
765
|
-
}
|
|
766
|
-
/**
|
|
767
|
-
* Assemble a declarative spec (`cases` + `prompt` + `score`) into a runnable
|
|
768
|
-
* harness×model leaderboard — `run()` executes the matrix, `toBenchmarkAdapter()`
|
|
769
|
-
* exposes the same domain as a structural `BenchmarkAdapter`.
|
|
770
|
-
*/
|
|
771
|
-
declare function defineLeaderboard<TCase, TArtifact = string>(spec: LeaderboardSpec<TCase, TArtifact>): DefinedLeaderboard<TCase, TArtifact>;
|
|
772
|
-
|
|
773
|
-
/**
|
|
774
|
-
* The third-person observer — the connective tissue that closes the loop.
|
|
775
|
-
*
|
|
776
|
-
* A driver spawns a worker; the worker can't see itself. `observe` reads the
|
|
777
|
-
* worker's TRACE (what it actually did — every tool call, cost, failure) and
|
|
778
|
-
* produces two streams:
|
|
779
|
-
* - `findings` / `report` — fed back DOWN (a steer for the next attempt) and
|
|
780
|
-
* OUT (the operator-facing "what I noticed + what to change").
|
|
781
|
-
* - `learned` — durable facts written to the cross-run `Corpus` so the NEXT
|
|
782
|
-
* run starts smarter (the continuous half of "continuous self-improvement").
|
|
783
|
-
*
|
|
784
|
-
* Findings are TRACE-derived, never JUDGE-derived (`derived_from_judge:false`):
|
|
785
|
-
* the observer reads behavior, never the acceptance verdict — the selector≠judge
|
|
786
|
-
* firewall (docs/learning-flywheel.md). The observer is harness-agnostic: it
|
|
787
|
-
* reads a trace + an output, so it watches opencode, codex, hermes, or a BYO
|
|
788
|
-
* agent identically.
|
|
789
|
-
*/
|
|
790
|
-
|
|
791
|
-
interface ObserveInput {
|
|
792
|
-
/** What the worker was asked to do. */
|
|
793
|
-
task: string;
|
|
794
|
-
/** What it produced (its final answer / artifact summary). */
|
|
795
|
-
output: string;
|
|
796
|
-
/** The worker's trace — any event array (sandbox events, tool-call records). */
|
|
797
|
-
trace: ReadonlyArray<unknown>;
|
|
798
|
-
/** Terminal status only (passed/failed/unknown) — NOT a judge score; the
|
|
799
|
-
* observer never reads the verdict, it reads behavior. */
|
|
800
|
-
outcome?: 'passed' | 'failed' | 'unknown';
|
|
801
|
-
/** Provenance back to the run. */
|
|
802
|
-
runId?: string;
|
|
803
|
-
}
|
|
804
|
-
interface ObserveOptions {
|
|
805
|
-
/** The model-call seam (agent-eval `createChatClient`: router / cli-bridge / …). */
|
|
806
|
-
chat: ChatClient;
|
|
807
|
-
model?: string;
|
|
808
|
-
/** When set, learned facts are appended (idempotent) for the next run to read. */
|
|
809
|
-
corpus?: Corpus;
|
|
810
|
-
/** Tags written onto learned facts + used by the next run's corpus query. */
|
|
811
|
-
tags?: ReadonlyArray<string>;
|
|
812
|
-
signal?: AbortSignal;
|
|
813
|
-
/** Cap the trace lines fed to the observer (keeps the call cheap). Default 80. */
|
|
814
|
-
maxTraceLines?: number;
|
|
815
|
-
/** Override the analyst's system instruction — the prompt that turns a trace into
|
|
816
|
-
* findings + recommended_actions. The analyst IS the steerer, so this is the knob a
|
|
817
|
-
* prompt optimizer (GEPA) tunes. Omitted ⇒ the default observer instruction. The
|
|
818
|
-
* firewall (trace-only, never the verdict) is structural (input has no score), so a
|
|
819
|
-
* custom instruction cannot break it. */
|
|
820
|
-
analystInstruction?: string;
|
|
821
|
-
}
|
|
822
|
-
/** The default observer instruction — exported so an optimizer can seed its population. */
|
|
823
|
-
declare const defaultAnalystInstruction: string;
|
|
824
|
-
interface Observation {
|
|
825
|
-
findings: AnalystFinding[];
|
|
826
|
-
/** Facts persisted to the corpus (empty when no corpus was supplied). */
|
|
827
|
-
learned: CorpusRecord[];
|
|
828
|
-
/** Operator-facing markdown: what the observer noticed + what to change. */
|
|
829
|
-
report: string;
|
|
830
|
-
}
|
|
831
|
-
/** The third-person trace analyst: read a worker's trace and produce steer findings for the next attempt plus durable `learned` facts for the cross-run corpus. */
|
|
832
|
-
declare function observe(input: ObserveInput, opts: ObserveOptions): Promise<Observation>;
|
|
833
|
-
/** Operator-facing report, split by who should act. The agent block is the
|
|
834
|
-
* steer; the operator block is the advice. */
|
|
835
|
-
declare function renderReport(findings: ReadonlyArray<AnalystFinding>): string;
|
|
836
|
-
|
|
837
|
-
/**
|
|
838
|
-
* harvestCorpus — production traces → corpus, the G2 bridge (the playbook's step 6).
|
|
839
|
-
* The flywheel's write side, batched: run the firewalled `observe()` analyst over a
|
|
840
|
-
* stream of completed runs (yesterday's production traces, a benchmark's rollouts, a
|
|
841
|
-
* fleet's day) and accrete the trace-derived facts into the durable corpus.
|
|
842
|
-
*
|
|
843
|
-
* Store-agnostic by design: the caller maps its trace store's rows (a
|
|
844
|
-
* `ProductionTraceSink` ndjson, OTLP spans, RunRecords) to `ObserveInput` — task text,
|
|
845
|
-
* final output, the event trace, terminal outcome. The analyst reads BEHAVIOR only
|
|
846
|
-
* (the firewall is structural: the input carries no judge verdict), and corpus appends
|
|
847
|
-
* are idempotent on (claim + tags), so re-harvesting the same window is safe.
|
|
848
|
-
*
|
|
849
|
-
* The nightly product job is then three lines:
|
|
850
|
-
* const runs = mapSinkRowsToObserveInputs(await readSink(yesterday))
|
|
851
|
-
* const report = await harvestCorpus({ runs, chat, corpus, tags: ['gtm-agent'] })
|
|
852
|
-
* log(report) // runsObserved / findings / learned / failures
|
|
853
|
-
*
|
|
854
|
-
* NOTE on the read side: harvesting is safe and cheap; *injecting* facts back into runs
|
|
855
|
-
* is the measured danger zone — naive unconditional priming tested NEGATIVE (−11.6pp,
|
|
856
|
-
* context pollution; result now in .evolve/current.json + memory). Gate any priming design on its
|
|
857
|
-
* own A/B; the corpus's first consumers are operators and optimizers, not prompts.
|
|
858
|
-
*/
|
|
859
|
-
|
|
860
|
-
interface HarvestCorpusOptions {
|
|
861
|
-
/** The completed runs to analyze — map your store's rows to `ObserveInput`. */
|
|
862
|
-
runs: AsyncIterable<ObserveInput> | Iterable<ObserveInput>;
|
|
863
|
-
/** The model-call seam (agent-eval `createChatClient`). */
|
|
864
|
-
chat: ChatClient;
|
|
865
|
-
model?: string;
|
|
866
|
-
/** The durable corpus the facts accrete into. */
|
|
867
|
-
corpus: Corpus;
|
|
868
|
-
/** Tags written onto learned facts (the product/domain key the read side queries by). */
|
|
869
|
-
tags?: ReadonlyArray<string>;
|
|
870
|
-
/** Override the analyst instruction (the GEPA-tunable knob). */
|
|
871
|
-
analystInstruction?: string;
|
|
872
|
-
/** Runs analyzed in parallel. Default 4. */
|
|
873
|
-
concurrency?: number;
|
|
874
|
-
/** Hard cap on runs consumed from the stream (a cost guard for unbounded stores). */
|
|
875
|
-
maxRuns?: number;
|
|
876
|
-
signal?: AbortSignal;
|
|
877
|
-
}
|
|
878
|
-
interface HarvestFailure {
|
|
879
|
-
runId: string;
|
|
880
|
-
error: string;
|
|
881
|
-
}
|
|
882
|
-
interface HarvestReport {
|
|
883
|
-
runsObserved: number;
|
|
884
|
-
/** Total findings the analyst produced (including ones already known). */
|
|
885
|
-
findings: number;
|
|
886
|
-
/** NEW facts actually appended (idempotent dedup excludes re-learned ones). */
|
|
887
|
-
learned: number;
|
|
888
|
-
/** Per-run analysis failures — reported, never silently dropped. */
|
|
889
|
-
failures: HarvestFailure[];
|
|
890
|
-
}
|
|
891
|
-
/** Batch the firewalled `observe()` analyst over completed runs and accrete the trace-derived facts into the durable corpus — the production-traces→corpus write side of the flywheel. */
|
|
892
|
-
declare function harvestCorpus(opts: HarvestCorpusOptions): Promise<HarvestReport>;
|
|
893
|
-
|
|
894
|
-
/** Context handed to each `onPrompt` / `onTask` call. */
|
|
895
|
-
interface InProcessPromptCtx {
|
|
896
|
-
/** 0-based round index — increments per `streamPrompt`/`streamTask` on the
|
|
897
|
-
* SAME box (so a refine driver's round N can differ from round N-1). Fresh
|
|
898
|
-
* boxes start at 0. */
|
|
899
|
-
round: number;
|
|
900
|
-
/** Absolute path of this box's workspace, when a `workdir` was configured.
|
|
901
|
-
* Write the deliverable / fixtures here; `fs.read`/`fs.write`/`exec` operate
|
|
902
|
-
* over it. `undefined` for pure event-only boxes. */
|
|
903
|
-
workdir?: string;
|
|
904
|
-
/** Cooperative cancellation channel for this turn. */
|
|
905
|
-
signal: AbortSignal;
|
|
906
|
-
/** Which box verb produced this call: `prompt` = `streamPrompt`,
|
|
907
|
-
* `task` = `streamTask`. */
|
|
908
|
-
mode: 'prompt' | 'task';
|
|
909
|
-
/** The verbatim per-call options the caller passed to the box verb (minus
|
|
910
|
-
* `signal`, surfaced above) — lets an offline test assert an options
|
|
911
|
-
* passthrough (`model`, `sessionId`, `maxTurns`, …) actually arrived. */
|
|
912
|
-
options?: Record<string, unknown>;
|
|
913
|
-
}
|
|
914
|
-
/**
|
|
915
|
-
* The user callback: given a prompt and its round, produce the box's event
|
|
916
|
-
* stream for that turn. Return a plain `SandboxEvent[]` (the common case) or an
|
|
917
|
-
* async iterable for streaming. The callback may also write files into
|
|
918
|
-
* `ctx.workdir` (read back via `fs.read` or graded by `exec`).
|
|
919
|
-
*/
|
|
920
|
-
type InProcessOnPrompt = (prompt: string, ctx: InProcessPromptCtx) => SandboxEvent[] | AsyncIterable<SandboxEvent> | Promise<SandboxEvent[]>;
|
|
921
|
-
/** @experimental */
|
|
922
|
-
interface InProcessSandboxClientOptions {
|
|
923
|
-
/** The per-turn behavior — see {@link InProcessOnPrompt}. */
|
|
924
|
-
onPrompt: InProcessOnPrompt;
|
|
925
|
-
/**
|
|
926
|
-
* Task-mode behavior, driven by `box.streamTask` (the verb `streamAgentTurn`'s
|
|
927
|
-
* `box-task` backend calls). When omitted, `streamTask` drives `onPrompt` —
|
|
928
|
-
* the pseudo-box has ONE behavior callback and both verbs exercise it
|
|
929
|
-
* (`ctx.mode` tells them apart). Provide `onTask` when a test must
|
|
930
|
-
* discriminate the verbs or script different task-mode behavior.
|
|
931
|
-
*/
|
|
932
|
-
onTask?: InProcessOnPrompt;
|
|
933
|
-
/**
|
|
934
|
-
* Opt in to a REAL filesystem-backed box. When set, each `create()` mints a
|
|
935
|
-
* fresh temp directory (prefixed `<workdir>-`) and the box exposes
|
|
936
|
-
* `fs.read`/`fs.write` and `exec` over it; `delete()` removes the dir. Omit
|
|
937
|
-
* for a pure event-only box (no `fs`/`exec` members), which is all a driver
|
|
938
|
-
* or fanout loop needs.
|
|
939
|
-
*/
|
|
940
|
-
workdir?: string;
|
|
941
|
-
/**
|
|
942
|
-
* Override the box `id`. A string is used verbatim; a function receives the
|
|
943
|
-
* 0-based create-sequence and returns the id (e.g. machine-keyed placement
|
|
944
|
-
* demos). Default `in-process-<seq>`. The id is the value `describePlacement`
|
|
945
|
-
* tags, so set it when a demo's output reads on a meaningful sandbox id.
|
|
946
|
-
*/
|
|
947
|
-
id?: string | ((seq: number) => string);
|
|
948
|
-
}
|
|
949
|
-
/**
|
|
950
|
-
* Adapt a single `onPrompt(prompt, ctx)` callback into a `SandboxClient` for
|
|
951
|
-
* `runAgentRounds` / `openSandboxRun`. Returns a PROPERLY-TYPED `SandboxClient`: the
|
|
952
|
-
* lone `SandboxInstance` cast (object literal → `declare class`) lives inside
|
|
953
|
-
* this function, so call sites stay cast-free.
|
|
954
|
-
*
|
|
955
|
-
* @experimental
|
|
956
|
-
*/
|
|
957
|
-
declare function inProcessSandboxClient(options: InProcessSandboxClientOptions): SandboxClient;
|
|
958
|
-
|
|
959
|
-
/**
|
|
960
|
-
* Adapt an `ExecutorFactory` into a `SandboxClient` for `runAgentRounds`. The factory is
|
|
961
|
-
* instantiated fresh per `streamPrompt` (mirrors the per-spawn executor lifecycle):
|
|
962
|
-
* run once on the prompt, emit the terminal result event, tear down.
|
|
963
|
-
*/
|
|
964
|
-
declare function inlineSandboxClient(factory: ExecutorFactory<unknown>): SandboxClient;
|
|
965
|
-
|
|
966
|
-
/**
|
|
967
|
-
* `KeyProvider` — API-key provisioning for adopted external MCP servers.
|
|
968
|
-
*
|
|
969
|
-
* An adopted server (a `connection` artifact, or a `buildableGenerator` remote
|
|
970
|
-
* emit) usually needs a credential. The credential must never ride the profile
|
|
971
|
-
* or the artifact — both are logged, diffed, and stored as audit records — so
|
|
972
|
-
* the profile carries only a DECLARATIVE reference:
|
|
973
|
-
*
|
|
974
|
-
* `profile.mcp[key].metadata[mcpSecretEnvMetadataKey]`
|
|
975
|
-
* = { ENV_VAR_NAME: 'PROVIDER_KEY_NAME', … }
|
|
976
|
-
*
|
|
977
|
-
* and the VALUE is resolved at materialize time (`materializeLocalMcp`'s
|
|
978
|
-
* `keys` option) and injected straight into the spawned server child's env.
|
|
979
|
-
* Values exist only in the child process env — never in the profile, the
|
|
980
|
-
* artifact registry, an error message, or a log line (errors name the KEY
|
|
981
|
-
* NAME only).
|
|
982
|
-
*
|
|
983
|
-
* Fail-closed at every hop: a declared secret with no provider, or a provider
|
|
984
|
-
* that does not hold the named key, THROWS — booting an external server
|
|
985
|
-
* keyless would fail opaquely mid-eval or silently score a broken candidate.
|
|
986
|
-
*
|
|
987
|
-
* `envKeyProvider` is the same-host default: it reads the process env, which
|
|
988
|
-
* the operator loads via dotenvx (the secrets files never touch the repo).
|
|
989
|
-
*
|
|
990
|
-
* >>> REAL SECRET STORES PLUG IN HERE (flagged): the sandbox SDK's
|
|
991
|
-
* `client.secrets` (`SecretsManager.get(name)`) and the platform hub's
|
|
992
|
-
* short-lived capability tokens (`PlatformHubClient.mintToken` →
|
|
993
|
-
* `{ ENV: token }`) both satisfy `KeyProvider` with a one-line adapter; both
|
|
994
|
-
* need a live service, so neither is constructed here.
|
|
995
|
-
*/
|
|
996
|
-
|
|
997
|
-
/** Resolve named secrets. The ONE seam every secret store adapts to. */
|
|
998
|
-
interface KeyProvider {
|
|
999
|
-
/** The value for `name`, or `undefined` when this provider does not hold it. */
|
|
1000
|
-
get(name: string): Promise<string | undefined>;
|
|
1001
|
-
}
|
|
1002
|
-
/** The env-backed provider: reads the (dotenvx-loaded) process env. Empty /
|
|
1003
|
-
* whitespace-only values count as absent — fail loud, not with a blank key. */
|
|
1004
|
-
declare function envKeyProvider(env?: Record<string, string | undefined>): KeyProvider;
|
|
1005
|
-
/** The `AgentProfileMcpServer.metadata` key the declarative secret-env map
|
|
1006
|
-
* rides under: `{ ENV_VAR_NAME: 'PROVIDER_KEY_NAME' }`. Names only — values
|
|
1007
|
-
* are resolved at materialize time and never stored. */
|
|
1008
|
-
declare const mcpSecretEnvMetadataKey = "secretEnv";
|
|
1009
|
-
/** Read (and validate) a server entry's declared secret-env map, if any.
|
|
1010
|
-
* Malformed metadata throws — a half-declared secret must never half-boot. */
|
|
1011
|
-
declare function secretEnvOfMcpServer(server: AgentProfileMcpServer): Record<string, string> | undefined;
|
|
1012
|
-
/**
|
|
1013
|
-
* Resolve a declared secret-env map into the real env entries for a server
|
|
1014
|
-
* spawn. Fail-closed: no provider or a missing key throws, naming the KEY
|
|
1015
|
-
* NAME only (the value never appears in any message). `label` names the
|
|
1016
|
-
* server for the error (e.g. `profile.mcp['exa']`).
|
|
1017
|
-
*/
|
|
1018
|
-
declare function resolveSecretEnv(secretEnv: Record<string, string>, keys: KeyProvider | undefined, label: string): Promise<Record<string, string>>;
|
|
1019
|
-
|
|
1020
|
-
/**
|
|
1021
|
-
* `localSandboxClient` — the SAME-HOST pseudo-box: a `SandboxClient` whose
|
|
1022
|
-
* `create()` MATERIALIZES the profile's stdio MCP servers as local child
|
|
1023
|
-
* processes (`materializeLocalMcp`) and whose `streamPrompt` drives a real
|
|
1024
|
-
* tool loop (`runBrainLoop` over the router brain) with those live tools.
|
|
1025
|
-
*
|
|
1026
|
-
* Despite the interface name, this does NOT isolate processes. It is only for
|
|
1027
|
-
* author-controlled profiles whose local MCP commands a caller explicitly
|
|
1028
|
-
* trusts; user- or model-authored code belongs in a real sandbox. `delete()`
|
|
1029
|
-
* kills the children.
|
|
1030
|
-
*
|
|
1031
|
-
* A per-create profile may change the prompt surface. Permission to start a
|
|
1032
|
-
* local MCP process applies only when its full canonical bytes match the fixed
|
|
1033
|
-
* constructor profile; a different generated profile is refused.
|
|
1034
|
-
*
|
|
1035
|
-
* Event protocol matches `inlineSandboxClient`: one `llm_call` metering event
|
|
1036
|
-
* + one terminal `result` event with finalText/tokenUsage/costUsd.
|
|
1037
|
-
*/
|
|
1038
|
-
|
|
1039
|
-
interface LocalSandboxClientOptions {
|
|
1040
|
-
/** The worker brain: router chat-completions with tool-calling. All three required. */
|
|
1041
|
-
router: {
|
|
1042
|
-
baseUrl: string;
|
|
1043
|
-
key: string;
|
|
1044
|
-
model: string;
|
|
1045
|
-
};
|
|
1046
|
-
/** Tool-loop turns per prompt. Default 8. */
|
|
1047
|
-
maxTurns?: number;
|
|
1048
|
-
/** Brain sampling temperature. Default: `routerBrain`'s (0.4). */
|
|
1049
|
-
temperature?: number;
|
|
1050
|
-
/** Fallback profile when `create(options)` carries none on `backend.profile`. */
|
|
1051
|
-
profile?: AgentProfile$1;
|
|
1052
|
-
/** Resolves profile-declared MCP secret names at child-process spawn time. */
|
|
1053
|
-
keys?: KeyProvider;
|
|
1054
|
-
/** Explicit trust decision for the exact `profile` bytes supplied here.
|
|
1055
|
-
* Omit to refuse local processes. A permissive policy never transfers to a
|
|
1056
|
-
* different per-create profile and provides no host isolation. */
|
|
1057
|
-
profileSecurityPolicy?: AgentProfileSecurityPolicy;
|
|
1058
|
-
}
|
|
1059
|
-
/** A same-host `SandboxClient` adapter with no process isolation. Local MCP is
|
|
1060
|
-
* refused unless the caller explicitly supplies a policy that allows it. */
|
|
1061
|
-
declare function localSandboxClient(opts: LocalSandboxClientOptions): SandboxClient;
|
|
1062
|
-
|
|
1063
|
-
/**
|
|
1064
|
-
*
|
|
1065
|
-
* `runAgentRounds` — the topology-agnostic kernel built atop the sandbox SDK.
|
|
1066
|
-
*
|
|
1067
|
-
* Each iteration:
|
|
1068
|
-
* 1. `driver.plan(task, history)` → N tasks (1 = refine, N = fanout, 0 = stop)
|
|
1069
|
-
* 2. For each task (parallel, bounded by `maxConcurrency`):
|
|
1070
|
-
* a. round-robin an `AgentRunSpec` from `agentRuns`
|
|
1071
|
-
* b. `sandboxClient.create({ backend: { profile }, ...overrides })`
|
|
1072
|
-
* c. emit `loop.iteration.dispatch` with the placement
|
|
1073
|
-
* (`{ sibling, sandboxId }` or `{ fleet, fleetId, machineId, sandboxId }`)
|
|
1074
|
-
* d. iterate `box.streamPrompt(taskToPrompt(task))` and collect events
|
|
1075
|
-
* 3. `output.parse(events)` → typed `Output`
|
|
1076
|
-
* 4. `validator?.validate(output)` → `DefaultVerdict`
|
|
1077
|
-
* 5. Append `Iteration` to history; emit `loop.iteration.ended`
|
|
1078
|
-
* 6. `driver.decide(history)` → if terminal, return result + winner
|
|
1079
|
-
*
|
|
1080
|
-
* The kernel owns: iteration accounting, per-iteration timing, error
|
|
1081
|
-
* capture, abort propagation, concurrency cap, cost aggregation, and trace
|
|
1082
|
-
* emission. The kernel does NOT own: what the agent runs (sandbox SDK +
|
|
1083
|
-
* profile), how outputs are decoded (output adapter), how outputs are
|
|
1084
|
-
* scored (validator), or topology (driver).
|
|
1085
|
-
*
|
|
1086
|
-
* @experimental
|
|
1087
|
-
*/
|
|
1088
|
-
|
|
1089
|
-
/** @experimental */
|
|
1090
|
-
interface RunAgentRoundsOptions<Task, Output, Decision> {
|
|
1091
|
-
driver: Driver<Task, Output, Decision>;
|
|
1092
|
-
/**
|
|
1093
|
-
* Single agent spec — every iteration uses this profile. Mutually
|
|
1094
|
-
* exclusive with `agentRuns`.
|
|
1095
|
-
*/
|
|
1096
|
-
agentRun?: AgentRunSpec<Task>;
|
|
1097
|
-
/**
|
|
1098
|
-
* Multiple specs for heterogeneous fanout. The kernel round-robins
|
|
1099
|
-
* through them when the driver plans N tasks. Mutually exclusive with
|
|
1100
|
-
* `agentRun`.
|
|
1101
|
-
*/
|
|
1102
|
-
agentRuns?: AgentRunSpec<Task>[];
|
|
1103
|
-
output: OutputAdapter<Output>;
|
|
1104
|
-
validator?: Validator<Output>;
|
|
1105
|
-
task: Task;
|
|
1106
|
-
ctx: ExecCtx;
|
|
1107
|
-
/** Default 10. Hard cap on total iterations across all `plan()` rounds. */
|
|
1108
|
-
maxIterations?: number;
|
|
1109
|
-
/** Default 4. In-flight worker cap within a single `plan()` batch. */
|
|
1110
|
-
maxConcurrency?: number;
|
|
1111
|
-
/**
|
|
1112
|
-
* Pre-allocated id for trace correlation. Default = `loop-${random}`.
|
|
1113
|
-
* Surfaces as `runId` on every emitted `LoopTraceEvent`.
|
|
1114
|
-
*/
|
|
1115
|
-
runId?: string;
|
|
1116
|
-
/**
|
|
1117
|
-
* Clock override; default `Date.now`. Deterministic tests pass a
|
|
1118
|
-
* monotonic counter to stabilize iteration timing fields.
|
|
1119
|
-
*/
|
|
1120
|
-
now?: () => number;
|
|
1121
|
-
/**
|
|
1122
|
-
* Override the default winner selector (highest-valid-score, ties broken
|
|
1123
|
-
* by earliest iteration).
|
|
1124
|
-
*/
|
|
1125
|
-
selectWinner?: (iterations: Iteration<Task, Output>[]) => LoopWinner<Task, Output> | undefined;
|
|
1126
|
-
/**
|
|
1127
|
-
* Same-sandbox driver mode — a kernel→caller out-channel, not a value handed
|
|
1128
|
-
* in. When set, the kernel keeps each finished worker box alive across the
|
|
1129
|
-
* `plan()` boundary and hands it here, so a same-sandbox planner
|
|
1130
|
-
* (one that reuses the worker's box) can stream its move INTO the
|
|
1131
|
-
* worker's live box — steering from the worker's real filesystem and state,
|
|
1132
|
-
* not just a history summary. The kernel owns teardown: every box kept alive
|
|
1133
|
-
* this way is destroyed at loop end (and the callback is invoked with
|
|
1134
|
-
* `undefined` then as a teardown sentinel). Without it, worker boxes are torn
|
|
1135
|
-
* down per-iteration (default) and a same-sandbox planner has nothing to
|
|
1136
|
-
* reuse. Intended for single-worker (refine) loops: under fanout every box is
|
|
1137
|
-
* still kept for teardown, but only the last-finishing box is handed here, so
|
|
1138
|
-
* a planner sees an arbitrary branch's filesystem — pair it with refine.
|
|
1139
|
-
*/
|
|
1140
|
-
onWorkerBox?: (box: SandboxInstance | undefined) => void;
|
|
1141
|
-
/**
|
|
1142
|
-
* Opt-in box-lineage controls. Default OFF — unset means every iteration
|
|
1143
|
-
* acquires a fresh box, streams once, and tears it down (today's behavior,
|
|
1144
|
-
* byte-identical). With `sessionContinuity` on, a refine round continues the
|
|
1145
|
-
* parent iteration's session on its live box; with `forkFanout` on (and a
|
|
1146
|
-
* fork-capable platform), a fanout round forks the parent's checkpoint so the
|
|
1147
|
-
* branches share a context prefix. The lineage owns every box it starts or
|
|
1148
|
-
* forks and tears them all down at loop end — so these paths are mutually
|
|
1149
|
-
* exclusive with `onWorkerBox`, which claims the same box-ownership channel.
|
|
1150
|
-
* @experimental
|
|
1151
|
-
*/
|
|
1152
|
-
lineage?: LoopLineageOptions;
|
|
1153
|
-
}
|
|
1154
|
-
/**
|
|
1155
|
-
* The round-synchronous MULTI-AGENT kernel: each round `driver.plan()` fans N tasks
|
|
1156
|
-
* out to N sandboxes (bounded concurrency), parses + validates each output, and folds
|
|
1157
|
-
* the round's results through `driver.decide` — fanout → validate → vote/select →
|
|
1158
|
-
* refine, repeated until the driver says stop. One call spans many agent sessions.
|
|
1159
|
-
*
|
|
1160
|
-
* Not to be confused with `runToolLoop` / `streamToolLoop` (package root entry): those
|
|
1161
|
-
* run ONE chat turn against ONE model, dispatching the tool calls that turn emits and
|
|
1162
|
-
* folding the results back in until the model stops calling tools. No sandboxes, no
|
|
1163
|
-
* rounds, no winner selection.
|
|
1164
|
-
*
|
|
1165
|
-
* @experimental
|
|
1166
|
-
*/
|
|
1167
|
-
declare function runAgentRounds<Task, Output, Decision>(options: RunAgentRoundsOptions<Task, Output, Decision>): Promise<LoopResult<Task, Output, Decision>>;
|
|
1168
|
-
/**
|
|
1169
|
-
* Pre-rename name for {@link runAgentRounds}; identical function, kept so existing
|
|
1170
|
-
* call sites keep working.
|
|
1171
|
-
*
|
|
1172
|
-
* @deprecated Use {@link runAgentRounds}. The clearer name says what it is: the
|
|
1173
|
-
* multi-agent fanout/vote/refine kernel over sandboxes, NOT the one-turn tool loop
|
|
1174
|
-
* (`runToolLoop` / `streamToolLoop`, package root entry). `runLoop` shipped on `/loops`
|
|
1175
|
-
* next to `routerToolLoop`, which made the two read as variants of one thing. The alias
|
|
1176
|
-
* is removed in the next major.
|
|
1177
|
-
*/
|
|
1178
|
-
declare const runLoop: typeof runAgentRounds;
|
|
1179
|
-
/**
|
|
1180
|
-
* Pre-rename name for {@link RunAgentRoundsOptions}.
|
|
1181
|
-
*
|
|
1182
|
-
* @deprecated Use {@link RunAgentRoundsOptions}. Removed in the next major.
|
|
1183
|
-
*/
|
|
1184
|
-
type RunLoopOptions<Task, Output, Decision> = RunAgentRoundsOptions<Task, Output, Decision>;
|
|
1185
|
-
/**
|
|
1186
|
-
* The kernel's winner argmax — best-valid-score, ties broken by earliest index,
|
|
1187
|
-
* falling back to the best-scoring non-errored output when none is valid. Exported
|
|
1188
|
-
* so the `runProgram` tree executor selects across merged sub-loop iterations with
|
|
1189
|
-
* the SAME semantics the kernel uses at a single loop's finalize (one selector, not
|
|
1190
|
-
* a forked copy).
|
|
1191
|
-
*/
|
|
1192
|
-
declare function defaultSelectWinner<Task, Output>(iterations: Iteration<Task, Output>[]): LoopWinner<Task, Output> | undefined;
|
|
1193
|
-
|
|
1194
|
-
/**
|
|
1195
|
-
* `loopDispatch` — turn `runAgentRounds` into an agent-eval campaign dispatch.
|
|
1196
|
-
*
|
|
1197
|
-
* Without this adapter a consumer wiring `runAgentRounds` into `runProfileMatrix` /
|
|
1198
|
-
* `runCampaign` has to, by hand, every time: (a) build an `ExecCtx` with a
|
|
1199
|
-
* sandbox client, (b) adapt the campaign `DispatchContext.trace` into a
|
|
1200
|
-
* `LoopTraceEmitter` (or lose all loop trace correlation), and (c) remember to
|
|
1201
|
-
* forward the loop's cost + tokens via `ctx.cost` (forgetting it yields a
|
|
1202
|
-
* `{0,0}` cell the backend-integrity guard reads as a stub). Three foot-guns,
|
|
1203
|
-
* the third silent. The fleet's products skipped (c) and fell back to a
|
|
1204
|
-
* `workerRecords[]` side-channel — the exact anti-pattern the substrate exists
|
|
1205
|
-
* to kill.
|
|
1206
|
-
*
|
|
1207
|
-
* `loopDispatch` collapses all three into one typed call:
|
|
1208
|
-
*
|
|
1209
|
-
* const dispatch = loopDispatch({
|
|
1210
|
-
* sandboxClient,
|
|
1211
|
-
* toLoopOptions: (scenario, profile) => ({ driver, agentRun, output, validator, task }),
|
|
1212
|
-
* })
|
|
1213
|
-
* await runProfileMatrix({ profiles, scenarios, dispatch, judges, commitSha })
|
|
1214
|
-
*
|
|
1215
|
-
* Usage is reported automatically; trace events are forwarded automatically;
|
|
1216
|
-
* the ctx is built automatically. The seam becomes impossible to mis-wire.
|
|
1217
|
-
*
|
|
1218
|
-
* Typed structurally against the campaign `DispatchContext` (imported type-only
|
|
1219
|
-
* from `@tangle-network/agent-eval/campaign`) — a downward dependency, never an
|
|
1220
|
-
* inversion.
|
|
1221
|
-
*/
|
|
1222
|
-
|
|
1223
|
-
/** runAgentRounds options minus the `ctx` (loopDispatch builds the ctx). */
|
|
1224
|
-
type LoopOptionsForDispatch<Task, Output, Decision> = Omit<RunAgentRoundsOptions<Task, Output, Decision>, 'ctx'>;
|
|
1225
|
-
interface LoopDispatchOptions<Task, Output, Decision, TScenario extends Scenario, TArtifact> {
|
|
1226
|
-
/** Sandbox client used for every cell's `runAgentRounds`. Supplied once. */
|
|
1227
|
-
sandboxClient: SandboxClient;
|
|
1228
|
-
/** Build the per-cell runAgentRounds options from the scenario (+ profile, when
|
|
1229
|
-
* used with `runProfileMatrix`). */
|
|
1230
|
-
toLoopOptions: (scenario: TScenario, profile: AgentProfile) => LoopOptionsForDispatch<Task, Output, Decision>;
|
|
1231
|
-
/** Map the finished loop to the artifact the judges score. Default:
|
|
1232
|
-
* `result.winner?.output`. A loop with no winner yields `undefined` (judges
|
|
1233
|
-
* skip the cell) — but the loop's token usage is STILL reported, so the
|
|
1234
|
-
* integrity guard sees real activity. */
|
|
1235
|
-
toArtifact?: (result: LoopResult<Task, Output, Decision>) => TArtifact;
|
|
1236
|
-
/** Forward `loop.*` trace events into the campaign's scoped trace so loop
|
|
1237
|
-
* spans correlate with the cell. Default true. */
|
|
1238
|
-
forwardTrace?: boolean;
|
|
1239
|
-
/** Cost-meter source label for the loop's spend. Default `'loop'`. */
|
|
1240
|
-
costSource?: string;
|
|
1241
|
-
/** Provider- or executor-enforced maximum for this whole cell dispatch.
|
|
1242
|
-
* Required by agent-eval before execution when the campaign is cost-capped. */
|
|
1243
|
-
maximumCharge?: MaximumCharge | ((scenario: TScenario, profile: AgentProfile) => MaximumCharge | undefined);
|
|
1244
|
-
/** Resolve the model actually served from the completed loop. */
|
|
1245
|
-
resolveCostModel?: (result: LoopResult<Task, Output, Decision>, scenario: TScenario, profile: AgentProfile) => string | undefined;
|
|
1246
|
-
}
|
|
1247
|
-
/** Options for adapting plain agent-eval campaign scenarios into runtime `runAgentRounds` cells. */
|
|
1248
|
-
interface LoopCampaignDispatchOptions<Task, Output, Decision, TScenario extends Scenario, TArtifact> {
|
|
1249
|
-
/** Sandbox client used for every campaign cell's `runAgentRounds`. */
|
|
1250
|
-
sandboxClient: SandboxClient;
|
|
1251
|
-
/** Build the per-cell runAgentRounds options from the campaign scenario. */
|
|
1252
|
-
toLoopOptions: (scenario: TScenario) => LoopOptionsForDispatch<Task, Output, Decision>;
|
|
1253
|
-
/** Map the finished loop to the artifact the campaign judges score. */
|
|
1254
|
-
toArtifact?: (result: LoopResult<Task, Output, Decision>) => TArtifact;
|
|
1255
|
-
/** Forward `loop.*` trace events into the campaign's scoped trace. Default true. */
|
|
1256
|
-
forwardTrace?: boolean;
|
|
1257
|
-
/** Cost-meter source label for the loop's spend. Default `'loop'`. */
|
|
1258
|
-
costSource?: string;
|
|
1259
|
-
/** Provider- or executor-enforced maximum for this whole cell dispatch. */
|
|
1260
|
-
maximumCharge?: MaximumCharge | ((scenario: TScenario) => MaximumCharge | undefined);
|
|
1261
|
-
/** Resolve the model actually served from the completed loop. */
|
|
1262
|
-
resolveCostModel?: (result: LoopResult<Task, Output, Decision>, scenario: TScenario) => string | undefined;
|
|
1263
|
-
}
|
|
1264
|
-
/**
|
|
1265
|
-
* Adapter for plain `runCampaign` scenarios. This is the runtime-side pair for
|
|
1266
|
-
* agent-eval fixture scenarios: load fixtures in `agent-eval/campaign`, build
|
|
1267
|
-
* the runtime loop here, and keep cost + token + trace reporting automatic.
|
|
1268
|
-
*/
|
|
1269
|
-
declare function loopCampaignDispatch<Task, Output, Decision, TScenario extends Scenario, TArtifact>(opts: LoopCampaignDispatchOptions<Task, Output, Decision, TScenario, TArtifact>): DispatchFn<TScenario, TArtifact>;
|
|
1270
|
-
/**
|
|
1271
|
-
* Adapter for `runProfileMatrix` (profile is an axis). Returns a
|
|
1272
|
-
* `ProfileDispatchFn` that runs `runAgentRounds` per (profile, scenario) cell and
|
|
1273
|
-
* reports usage automatically.
|
|
1274
|
-
*/
|
|
1275
|
-
declare function loopDispatch<Task, Output, Decision, TScenario extends Scenario, TArtifact>(opts: LoopDispatchOptions<Task, Output, Decision, TScenario, TArtifact>): ProfileDispatchFn<TScenario, TArtifact>;
|
|
1276
|
-
|
|
1277
|
-
/**
|
|
1278
|
-
* runBenchmark — the packaged optimization suite. Define a domain by implementing an
|
|
1279
|
-
* `Environment` (open / tools / call / score / close); get the optimization strategies
|
|
1280
|
-
* compared, scored by your own deployable check, with a paired-bootstrap report — free.
|
|
1281
|
-
*
|
|
1282
|
-
* The mental model: you have a TASK + a deployable CHECK + a compute BUDGET. A strategy
|
|
1283
|
-
* is how you spend the budget to beat the check. Two built-ins:
|
|
1284
|
-
*
|
|
1285
|
-
* sample — N independent attempts, keep the best-verifying one. (best-of-N / resample)
|
|
1286
|
-
* refine — attempt → a critic reads the trace → steer the next → repeat. (iterate-with-feedback)
|
|
1287
|
-
*
|
|
1288
|
-
* Both run at equal budget through the Supervisor's conserved pool; the headline is the
|
|
1289
|
-
* paired lift of refine over sample. Author your own strategy with `defineStrategy`.
|
|
1290
|
-
*/
|
|
1291
|
-
|
|
1292
|
-
/** A checkable task domain — implement these 5 hooks and the suite does the rest. The
|
|
1293
|
-
* same seam as `AgenticSurface`; `Environment` is the RL/gym-standard name for it. */
|
|
1294
|
-
type Environment = AgenticSurface;
|
|
1295
|
-
interface BenchmarkConfig {
|
|
1296
|
-
/** The task domain (5 hooks). */
|
|
1297
|
-
environment: Environment;
|
|
1298
|
-
/** The tasks to score across. */
|
|
1299
|
-
tasks: AgenticTask[];
|
|
1300
|
-
/** The worker: model + router + (optional) the critic's instruction (the steerer knob). */
|
|
1301
|
-
worker: AgenticOptions;
|
|
1302
|
-
/** Which strategies to compare. Pass the built-ins (`refine`, `sample`) or your own.
|
|
1303
|
-
* Default: [sample, refine]. */
|
|
1304
|
-
strategies?: Strategy[];
|
|
1305
|
-
/** Shots (refine) / width (sample) — the equal compute budget per strategy. Default 3. */
|
|
1306
|
-
budget?: number;
|
|
1307
|
-
/** Tasks scored in parallel. Default 3. */
|
|
1308
|
-
concurrency?: number;
|
|
1309
|
-
/** Progress hook — fires as each task settles (the live-monitoring seam: append to a
|
|
1310
|
-
* progress file, render a tree, stream to a dashboard). `done` counts settled tasks. */
|
|
1311
|
-
onTask?: (row: BenchmarkTaskRow, done: number, total: number) => void;
|
|
1312
|
-
/** Lifecycle observability — every spawn/settle of every cell's shots/analysts streams
|
|
1313
|
-
* here live (the watchdog/route-auditor seam, passed through to `runAgentic`). */
|
|
1314
|
-
hooks?: RuntimeHooks;
|
|
1315
|
-
}
|
|
1316
|
-
interface BenchmarkLift {
|
|
1317
|
-
/** Mean of paired deltas (refine − sample). */
|
|
1318
|
-
mean: number;
|
|
1319
|
-
low: number;
|
|
1320
|
-
high: number;
|
|
1321
|
-
n: number;
|
|
1322
|
-
}
|
|
1323
|
-
/** One strategy's outcome on one task — the per-task cell an optimizer consumes. */
|
|
1324
|
-
interface BenchmarkCell {
|
|
1325
|
-
score: number;
|
|
1326
|
-
resolved: boolean;
|
|
1327
|
-
/** The progress curve (refine: score per shot; sample: best-so-far per rollout). */
|
|
1328
|
-
progression: number[];
|
|
1329
|
-
usd: number;
|
|
1330
|
-
ms: number;
|
|
1331
|
-
tokens: {
|
|
1332
|
-
input: number;
|
|
1333
|
-
output: number;
|
|
1334
|
-
};
|
|
1335
|
-
}
|
|
1336
|
-
interface BenchmarkTaskRow {
|
|
1337
|
-
taskId: string;
|
|
1338
|
-
/** Per-strategy cells; absent when the task errored before completing all strategies. */
|
|
1339
|
-
cells?: Record<string, BenchmarkCell>;
|
|
1340
|
-
/** Per-strategy failures on this task: the strategy competed, threw, and scored an
|
|
1341
|
-
* honest zero — it loses, it does not poison the row. The message is kept so a later
|
|
1342
|
-
* generation's author can see WHY a candidate died. */
|
|
1343
|
-
errors?: Record<string, string>;
|
|
1344
|
-
/** Why the task was excluded (infra/setup failure) — never silently dropped. */
|
|
1345
|
-
error?: string;
|
|
1346
|
-
}
|
|
1347
|
-
interface BenchmarkStrategySummary {
|
|
1348
|
-
/** Mean verifier score (0..1). */
|
|
1349
|
-
score: number;
|
|
1350
|
-
/** Fraction of tasks fully resolved. */
|
|
1351
|
-
resolved: number;
|
|
1352
|
-
/** Mean cost vector per task. */
|
|
1353
|
-
usd: number;
|
|
1354
|
-
ms: number;
|
|
1355
|
-
}
|
|
1356
|
-
/** Benchmark output: per-strategy means plus the full per-task × per-strategy losses table an optimizer mines. */
|
|
1357
|
-
interface BenchmarkReport {
|
|
1358
|
-
n: number;
|
|
1359
|
-
excluded: number;
|
|
1360
|
-
/** Per-strategy means (keyed by strategy.name). */
|
|
1361
|
-
perStrategy: Record<string, BenchmarkStrategySummary>;
|
|
1362
|
-
/** The full per-task × per-strategy table — the LOSSES an optimizer (GEPA, a
|
|
1363
|
-
* strategy-author, an operator) consumes. Includes errored tasks with the reason. */
|
|
1364
|
-
perTask: BenchmarkTaskRow[];
|
|
1365
|
-
/** The non-dominated strategies on (score ↑, $/task ↓) — collapse-last, per the canon:
|
|
1366
|
-
* a strategy that ties on score at half the cost WINS and a scalar would hide it. */
|
|
1367
|
-
pareto: string[];
|
|
1368
|
-
/** The headline when both `refine` and `sample` ran: paired-bootstrap lift of refine over sample. */
|
|
1369
|
-
refineVsSample?: BenchmarkLift;
|
|
1370
|
-
}
|
|
1371
|
-
/** Run the requested strategies over the tasks, scored by the Environment's own check.
|
|
1372
|
-
* Resilient: a task whose rollouts fail (transient infra) is excluded from the stats but
|
|
1373
|
-
* reported in `perTask` with the error — never silently dropped. */
|
|
1374
|
-
declare function runBenchmark(cfg: BenchmarkConfig): Promise<BenchmarkReport>;
|
|
1375
|
-
/** Pretty-print a report — the "free optimization" verdict, with the cost vector. */
|
|
1376
|
-
declare function printBenchmarkReport(report: BenchmarkReport): void;
|
|
1377
|
-
|
|
1378
|
-
/**
|
|
1379
|
-
* createMcpEnvironment — wrap any MCP server as an `Environment` (the product-adoption
|
|
1380
|
-
* primitive: a product's agent tools are usually already an MCP surface, so the domain
|
|
1381
|
-
* only writes the lifecycle hooks — open a scoped artifact, score it with a deployable
|
|
1382
|
-
* check, close it — and the tool plumbing is derived from the server).
|
|
1383
|
-
*
|
|
1384
|
-
* What the helper owns (the generic 80%, hardened on the EnterpriseOps gym):
|
|
1385
|
-
* - JSON-RPC `tools/list` → `AgenticTool[]`, with schemas coerced to the
|
|
1386
|
-
* OpenAI-tool-valid shape (top-level oneOf/anyOf/allOf/enum/not are rejected by
|
|
1387
|
-
* tool-calling providers; nested combinators are fine).
|
|
1388
|
-
* - JSON-RPC `tools/call` → the tool's text content (errors surfaced as `ERROR: …`
|
|
1389
|
-
* strings — a bad call is the agent's outcome, not an infra fault).
|
|
1390
|
-
* - SSE response parsing (streamable-HTTP MCP servers answer with `data:` lines).
|
|
1391
|
-
* - Bounded retry with backoff on thrown fetches (transient network ≠ task failure).
|
|
1392
|
-
*
|
|
1393
|
-
* What the domain supplies: `open` (create/seed the per-task artifact and return its
|
|
1394
|
-
* MCP endpoint — url + headers carry the per-artifact scoping, e.g. a database id
|
|
1395
|
-
* header), `score` (the deployable check), and optional `close`/`selectTools`.
|
|
1396
|
-
*/
|
|
1397
|
-
|
|
1398
|
-
/** Where a handle's MCP server lives; headers carry per-artifact scoping. */
|
|
1399
|
-
interface McpEndpoint {
|
|
1400
|
-
url: string;
|
|
1401
|
-
headers?: Record<string, string>;
|
|
1402
|
-
}
|
|
1403
|
-
interface McpEnvironmentOptions {
|
|
1404
|
-
name: string;
|
|
1405
|
-
/** Create/seed the per-task artifact; return its handle + the MCP endpoint scoped to it. */
|
|
1406
|
-
open(task: AgenticTask): Promise<{
|
|
1407
|
-
handle: ArtifactHandle;
|
|
1408
|
-
endpoint: McpEndpoint;
|
|
1409
|
-
}>;
|
|
1410
|
-
/** The deployable check over the artifact's current state. */
|
|
1411
|
-
score(task: AgenticTask, handle: ArtifactHandle): Promise<SurfaceScore>;
|
|
1412
|
-
/** Teardown (delete the seeded artifact). Optional — omit for stateless servers. */
|
|
1413
|
-
close?(handle: ArtifactHandle): Promise<void>;
|
|
1414
|
-
/** Restrict/order the server's tools per task (e.g. the task's selected_tools). Default: all. */
|
|
1415
|
-
selectTools?(task: AgenticTask, all: AgenticTool[]): AgenticTool[];
|
|
1416
|
-
/** Cap on a tool result's text fed back to the worker. Default 1500 chars. */
|
|
1417
|
-
maxResultChars?: number;
|
|
1418
|
-
}
|
|
1419
|
-
/** Coerce an MCP inputSchema to an OpenAI-tool-valid top-level object schema.
|
|
1420
|
-
* Shared with the same-host stdio client (`materializeLocalMcp`) — one coercion
|
|
1421
|
-
* rule for every MCP tool a worker sees, regardless of transport. */
|
|
1422
|
-
declare function sanitizeMcpToolSchema(s: unknown): Record<string, unknown>;
|
|
1423
|
-
/** Wrap any MCP server as an `Environment`: `tools/list` becomes `AgenticTool[]` with provider-safe schemas; the domain supplies only the artifact lifecycle hooks. */
|
|
1424
|
-
declare function createMcpEnvironment(opts: McpEnvironmentOptions): Environment;
|
|
1425
|
-
|
|
1426
|
-
/**
|
|
1427
|
-
*
|
|
1428
|
-
* Analyst-on-scope (G1) — the analyze→findings→steer wire over the reactive `Scope`.
|
|
1429
|
-
*
|
|
1430
|
-
* The analyst runs over the children drained so far and hands its findings to the steer
|
|
1431
|
-
* decision behind a provenance firewall (`assertTraceDerivedFindings`) that keeps the external
|
|
1432
|
-
* write-only judge out of that decision (selector ≠ judge). The reactive `Scope` has no rounds:
|
|
1433
|
-
* a combinator's `act` asks a `ScopeAnalyst` to turn the children it has drained off
|
|
1434
|
-
* `scope.next()` SO FAR into `AnalystFinding[]`, and steers from THOSE findings through a single
|
|
1435
|
-
* `SteerContext`.
|
|
1436
|
-
*
|
|
1437
|
-
* The analyst itself is not a new type — it is "just an `Agent<unknown, AnalystFinding[]>`" the
|
|
1438
|
-
* combinator spawns over a child's trace (harness `null`/`cli`). `createScopeAnalyst` spawns that
|
|
1439
|
-
* agent through `Scope.spawn` (so its compute is metered by the conserved pool like any child),
|
|
1440
|
-
* drains its settlement, then enforces the firewall on the way out — a judge-derived finding
|
|
1441
|
-
* ABORTS, it is never filtered. Fail loud: a down analyst, a non-array result, or a tainted finding
|
|
1442
|
-
* throws; there is no silent empty-findings path that would let a combinator steer on nothing.
|
|
1443
|
-
*
|
|
1444
|
-
* @experimental
|
|
1445
|
-
*/
|
|
1446
|
-
|
|
1447
|
-
declare const assertTraceDerivedFindings: AssertTraceDerivedFindings;
|
|
1448
|
-
/**
|
|
1449
|
-
* The analyst run an `Agent<unknown, AnalystFinding[]>` performs over the children settled so far.
|
|
1450
|
-
* The combinator supplies the analyst's task projection (how to frame the drained settlements as
|
|
1451
|
-
* the analyst's input) — the analyst's `act` reads the trace and returns its raw findings; the
|
|
1452
|
-
* firewall is enforced afterwards by `createScopeAnalyst`, not by the analyst itself.
|
|
1453
|
-
*/
|
|
1454
|
-
interface CreateScopeAnalystOptions<D> {
|
|
1455
|
-
/** The analyst agent the combinator spawns over the trace. `harness` is the persona's choice
|
|
1456
|
-
* (`null` for an inline router analyst, a `BackendType` for a sandboxed one). Its `act` returns
|
|
1457
|
-
* the RAW findings; this module asserts the firewall on them before returning. */
|
|
1458
|
-
readonly analyst: Agent<unknown, ReadonlyArray<AnalystFinding>>;
|
|
1459
|
-
/** Build the analyst agent's task from the analyze input (the root-task framing + the children
|
|
1460
|
-
* drained so far). Pure projection — the analyst interprets it, this never reads it. */
|
|
1461
|
-
buildTask(input: ScopeAnalyzeInput<D>): unknown;
|
|
1462
|
-
/** The conserved budget reserved for one analyst spawn. The pool reserves against it and fails
|
|
1463
|
-
* closed; an analyst that cannot be admitted is a fail-loud abort, never silent empty findings. */
|
|
1464
|
-
readonly budget: Budget;
|
|
1465
|
-
/** Trace/journal label for the spawned analyst child. Default `'analyst'`. */
|
|
1466
|
-
readonly label?: string;
|
|
1467
|
-
}
|
|
1468
|
-
/**
|
|
1469
|
-
* Build a `ScopeAnalyst` that spawns the analyst agent through `Scope.spawn` (so its compute is
|
|
1470
|
-
* metered by the conserved pool), drains its single settlement, and enforces the trace-derived
|
|
1471
|
-
* firewall before returning. The `scope` is the SAME scope the combinator is draining its children
|
|
1472
|
-
* from — the analyst is spawned as a sibling and its result is read off `scope.next()` in cursor
|
|
1473
|
-
* order, replay-safe like any other child.
|
|
1474
|
-
*
|
|
1475
|
-
* Fail loud (no silent empty findings):
|
|
1476
|
-
* - the pool refuses the analyst spawn → `AnalystError` (the steer would otherwise run on nothing)
|
|
1477
|
-
* - the analyst settles `down` → `AnalystError` (a broken capture path, not a verdict)
|
|
1478
|
-
* - the analyst returns a non-array → `PlannerError`
|
|
1479
|
-
* - any finding cites judge-derived metric evidence → `PlannerError` via the firewall
|
|
1480
|
-
*/
|
|
1481
|
-
declare function createScopeAnalyst<D>(scope: Scope<Outcome<D>>, options: CreateScopeAnalystOptions<D>): ScopeAnalyst<D>;
|
|
1482
|
-
/**
|
|
1483
|
-
* Project a `ScopeAnalyzeInput` into the `AnalystRegistry.run` arguments. The registry runs over a
|
|
1484
|
-
* `runId` + `AnalystRunInputs` (a trace store / run record / artifact dir), NOT in-memory scope
|
|
1485
|
-
* settlements — so the CALLER owns the projection from the combinator's drained children to the
|
|
1486
|
-
* registry's inputs (e.g. the trace store the run already wrote). This adapter never invents that
|
|
1487
|
-
* bridge; it only runs the projected inputs and firewalls the merged findings.
|
|
1488
|
-
*/
|
|
1489
|
-
interface RegistryAnalyzeProjection {
|
|
1490
|
-
readonly runId: string;
|
|
1491
|
-
readonly inputs: AnalystRunInputs;
|
|
1492
|
-
/** Optional `run` opts (e.g. `priorFindings`) forwarded verbatim to the registry. */
|
|
1493
|
-
readonly opts?: Parameters<AnalystRegistryLike['run']>[2];
|
|
1494
|
-
}
|
|
1495
|
-
/**
|
|
1496
|
-
* A `ScopeAnalyst` backed by an `AnalystRegistry` — the panel-of-analysts seam. The registry merges
|
|
1497
|
-
* N analyst KINDS into one `AnalystRunResult.findings`; `analyze` runs it over the caller-projected
|
|
1498
|
-
* `{ runId, inputs }` and pipes the merged findings through the SAME `assertTraceDerivedFindings`
|
|
1499
|
-
* firewall `createScopeAnalyst` uses (single-sourced selector≠judge). Distinct from `panel()`
|
|
1500
|
-
* (judges-vs-one-artifact) — this is analysts-over-a-trace, the diagnosis side of the wire.
|
|
1501
|
-
*
|
|
1502
|
-
* Fail loud: a registry that throws propagates; a judge-derived finding aborts via the firewall.
|
|
1503
|
-
* The projection is the caller's (`buildInputs`) — if the scope settlements do not cleanly map to
|
|
1504
|
-
* the registry's `AnalystRunInputs`, that is a caller-side contract gap, surfaced there, not papered
|
|
1505
|
-
* over with a fabricated input here.
|
|
1506
|
-
*/
|
|
1507
|
-
declare function registryScopeAnalyst<D>(registry: AnalystRegistryLike, buildInputs: (input: ScopeAnalyzeInput<D>) => RegistryAnalyzeProjection): ScopeAnalyst<D>;
|
|
1508
|
-
/**
|
|
1509
|
-
* Build the `SteerContext` a combinator reads to steer (its `loopUntil.until`, `widen` gate, any
|
|
1510
|
-
* future steer). One place enforces the firewall: `findings` is asserted trace-derived before it is
|
|
1511
|
-
* surfaced, and `lastValidScore` is provided for OBSERVABILITY only — a combinator that steers off
|
|
1512
|
-
* it re-introduces selector = judge, the coupling the architecture forbids.
|
|
1513
|
-
*
|
|
1514
|
-
* `findings` is re-asserted here even when it came from `createScopeAnalyst` (which already asserted
|
|
1515
|
-
* it): the assertion is cheap and idempotent, and a `SteerContext` may be built from findings that
|
|
1516
|
-
* arrived by another path (a caller-supplied diagnosis). Belt-and-suspenders on the one coupling
|
|
1517
|
-
* that must never leak.
|
|
1518
|
-
*/
|
|
1519
|
-
declare function buildSteerContext<D>(findings: ReadonlyArray<AnalystFinding>, settledSoFar: ReadonlyArray<Settled<Outcome<D>>>): SteerContext<D>;
|
|
1520
|
-
|
|
1521
|
-
/**
|
|
1522
|
-
*
|
|
1523
|
-
* The generic combinator library — the content-free act-bodies the wave's §1 contract froze.
|
|
1524
|
-
*
|
|
1525
|
-
* Each export is a `CombinatorShape<Task, D>` (an alias of `LoopShape<Task, D>`): a factory
|
|
1526
|
-
* `(ShapeContext) => Agent<Task, Outcome<D>>` whose `act` runs ONE composition shape over the
|
|
1527
|
-
* keystone `Scope` — spawn children through `ctx.spawnChild` + `scope.spawn`, drain settlements
|
|
1528
|
-
* via `scope.next()`, select across `done` children with the SINGLE-SOURCED `settledToIteration`
|
|
1529
|
-
* + `defaultSelectWinner` (selector≠judge — never a re-rank behind the driver), and synthesize a
|
|
1530
|
-
* terminal `Outcome<D>`.
|
|
1531
|
-
*
|
|
1532
|
-
* The shapes carry NO domain: a "research sweep over angles" is `fanout(angles, { synthesize })`
|
|
1533
|
-
* under a research persona; a "code build test" is `pipeline([plan, implement, integrate])` under
|
|
1534
|
-
* a coder persona. The SHAPE is here; the model/prompt/role/goal live on the `Persona` + task,
|
|
1535
|
-
* threaded to each child verbatim by the spec-objects the builders take. No model name, prompt,
|
|
1536
|
-
* role, or domain noun appears below.
|
|
1537
|
-
*
|
|
1538
|
-
* Two fail-loud invariants every combinator honors: a child the conserved pool cannot admit is a
|
|
1539
|
-
* CONCRETE blocker (never an eager over-fan, never a silent drop), and a `blocked` outcome always
|
|
1540
|
-
* names at least one blocker (a shape that cannot finish MUST say why — `blocked([])` throws).
|
|
1541
|
-
*
|
|
1542
|
-
* @experimental
|
|
1543
|
-
*/
|
|
1544
|
-
|
|
1545
|
-
/**
|
|
1546
|
-
* The single content-free valid-only winner selector. Among the gated-VALID children only
|
|
1547
|
-
* (`verdict.valid === true`), pick by `strategy` — best score / smallest delivered artifact /
|
|
1548
|
-
* earliest — ties broken by earliest index; returns `undefined` when NONE is valid (an ungated
|
|
1549
|
-
* output can never win — the deliverable gate is the point). `sizeOf` (for `'smallest-artifact'`)
|
|
1550
|
-
* reads the child's settled deliverable — the raw value a leaf settles, or the unwrapped `Outcome<D>`
|
|
1551
|
-
* a delegate path produces; a domain passes e.g. patch diff-lines. This is the de-duplicated home of
|
|
1552
|
-
* the selection logic previously copied per role.
|
|
1553
|
-
*/
|
|
1554
|
-
declare function selectValidWinner<D>(opts?: {
|
|
1555
|
-
strategy?: WinnerStrategy;
|
|
1556
|
-
sizeOf?: (deliverable: D) => number;
|
|
1557
|
-
}): FanoutWinnerSelector<D>;
|
|
1558
|
-
/**
|
|
1559
|
-
* `pipeline(stages)` — run the stages in order, feeding each stage's `done` deliverable into the
|
|
1560
|
-
* next stage's task. The first stage that ends `blocked` (a child that went down, a child the
|
|
1561
|
-
* pool would not admit, or a stage whose `collect` chose to block) short-circuits — its blockers
|
|
1562
|
-
* ARE the pipeline's blockers, never coerced past a failed stage. The terminal stage's `done`
|
|
1563
|
-
* deliverable is the pipeline's deliverable.
|
|
1564
|
-
*/
|
|
1565
|
-
declare function pipeline<Task, D>(stages: ReadonlyArray<PipelineStage<Task, unknown, unknown>>): CombinatorShape<Task, D>;
|
|
1566
|
-
/**
|
|
1567
|
-
* `fanout(items, opts)` — spawn one child per item in a single round (bounded by the conserved
|
|
1568
|
-
* pool's fail-closed admission), drain via `scope.next()`, then either synthesize over the
|
|
1569
|
-
* gathered settlements (one SEPARATE synthesis child) or return the best-valid child via the
|
|
1570
|
-
* single-sourced selector. A round that admitted zero children, or whose synthesis child could
|
|
1571
|
-
* not be admitted, is a concrete blocker.
|
|
1572
|
-
*
|
|
1573
|
-
* `opts.width` swaps the single round for `rollingDispatch`: at most `width` items live at once,
|
|
1574
|
-
* refilled the instant one settles. Selection, blockers, and the conserved pool are unchanged —
|
|
1575
|
-
* the refill behavior lives in the existing combinator rather than in a rival primitive.
|
|
1576
|
-
*/
|
|
1577
|
-
declare function fanout<Task, Item, D>(items: ReadonlyArray<Item>, opts: FanoutOptions<Item, D>): CombinatorShape<Task, D>;
|
|
1578
|
-
/**
|
|
1579
|
-
* `loopUntil(seed, spec)` — one `step` child per round; `fold` accumulates each settlement into
|
|
1580
|
-
* the running state; `until` (reading the round's trace findings, NOT a fresh raw verdict) is
|
|
1581
|
-
* the deployable stop. The conserved pool IS the loop bound: once `spawn` fails closed the loop
|
|
1582
|
-
* stops. A loop that exhausted the pool without `until` ever satisfying is a concrete blocker.
|
|
1583
|
-
*
|
|
1584
|
-
* When `ctx.analyst` is set, each round runs it over the children settled so far and steers
|
|
1585
|
-
* `until` on the resulting trace-derived findings (the analyst spawns into THIS scope, so its
|
|
1586
|
-
* compute is conserved-pooled — equal-k holds by construction). Absent an analyst the findings
|
|
1587
|
-
* argument is the empty array — never a fabricated finding (fail-loud honesty over a silent default).
|
|
1588
|
-
*/
|
|
1589
|
-
declare function loopUntil<Task, State, D>(seed: State, spec: LoopUntilSpec<Task, State, D>): CombinatorShape<Task, D>;
|
|
1590
|
-
/**
|
|
1591
|
-
* `panel(spec)` — spawn the M judge children over the SAME artifact, drain their settlements,
|
|
1592
|
-
* and fold them into a panel verdict via the pure WRITE-ONLY `merge` (a judge's output never
|
|
1593
|
-
* reaches another judge's task; the merge never spawns or re-ranks). A `down` judge carries no
|
|
1594
|
-
* verdict and is excluded from the merge denominator. A panel that admitted no judge is a
|
|
1595
|
-
* concrete blocker before `merge` is consulted.
|
|
1596
|
-
*/
|
|
1597
|
-
declare function panel<Task, Artifact, D>(spec: PanelSpec<Artifact, D>): CombinatorShape<Task, D>;
|
|
1598
|
-
/**
|
|
1599
|
-
* `verify(spec)` — an IMPLEMENT child produces a candidate, then a SEPARATE VERIFIER child grades
|
|
1600
|
-
* it; only a `valid` verifier verdict ships. Any other outcome (implement down, verifier down,
|
|
1601
|
-
* verifier verdict absent or not `valid`) is a concrete blocker carrying the failure verbatim —
|
|
1602
|
-
* never a coerced "done". The implement child does not grade itself.
|
|
1603
|
-
*/
|
|
1604
|
-
declare function verify<Task, Candidate, D>(spec: VerifySpec<Task, Candidate, D>): CombinatorShape<Task, D>;
|
|
1605
|
-
/**
|
|
1606
|
-
* `widen(spec)` — the streaming spawn-on-completion driver. Spawns the seed lineages, then REACTS
|
|
1607
|
-
* to each `scope.next()`: on every settled child it consults `spec.gate.decide` and, when the gate
|
|
1608
|
-
* returns `widen`, spawns AT MOST ONE more child toward the chosen lineage under the remaining
|
|
1609
|
-
* conserved pool. `promising` is derived from the round's trace findings (the analyst seam),
|
|
1610
|
-
* never a child's raw `verdict` — and the default gate (`flatWidenGate`) never widens, so the R2
|
|
1611
|
-
* firewall stays dormant. Terminal selection is `spec.synthesize` over every settled lineage.
|
|
1612
|
-
*
|
|
1613
|
-
* When `ctx.analyst` is set, `decide` is consulted with that round's trace-derived findings;
|
|
1614
|
-
* absent an analyst the findings argument is the empty array a flat gate ignores. The analyst
|
|
1615
|
-
* spawns into THIS scope (conserved-pooled, so equal-k holds). Streaming caveat: a wired analyst
|
|
1616
|
-
* drains its own child off the SHARED cursor by id-match, so on a NON-flat gate (which spawns
|
|
1617
|
-
* widen children that are live concurrently) the analyst can consume a sibling's settlement before
|
|
1618
|
-
* the widen loop sees it. The shipped default (`flatWidenGate`) never widens, so no widen child is
|
|
1619
|
-
* ever live when the analyst runs and the wire is exact; a non-flat gate must drive the analyst on
|
|
1620
|
-
* a scope whose siblings are quiesced, or read findings without the shared-cursor drain.
|
|
1621
|
-
*/
|
|
1622
|
-
declare function widen<Task, Seed, D>(spec: WidenSpec<Seed, D>): CombinatorShape<Task, D>;
|
|
1623
|
-
/**
|
|
1624
|
-
* The flat default `ScopeWidenGate` — never widens, keeping the R2 selector≠judge collision
|
|
1625
|
-
* dormant. A gate run passes this explicitly; a test asserts the default is flat.
|
|
1626
|
-
*/
|
|
1627
|
-
declare function flatWidenGate<D>(): ScopeWidenGate<D>;
|
|
1628
|
-
|
|
1629
|
-
/**
|
|
1630
|
-
*
|
|
1631
|
-
* The cross-run corpus (G2) — the learning-flywheel's durable accreted-fact store.
|
|
1632
|
-
*
|
|
1633
|
-
* `Corpus` is DISTINCT from the per-run `SpawnJournal` (decisions/replay) and `ResultBlobStore`
|
|
1634
|
-
* (payloads): a `CorpusRecord` is a FACT one run LEARNED that a FUTURE run reads back (the
|
|
1635
|
-
* world-model), not a replay input. This module owns the two impls the wave surface pins —
|
|
1636
|
-
* `InMemoryCorpus` and `FileCorpus` (JSONL, append-only) — plus `renderCorpusToInstructions`,
|
|
1637
|
-
* the READ side that projects accreted facts into a fresh `AgentProfile`'s instruction seams.
|
|
1638
|
-
*
|
|
1639
|
-
* The boundary is fail-loud, typed-outcome: `append` is idempotent on an identical record and
|
|
1640
|
-
* returns a typed error (never throws, never a silent overwrite) on a conflicting re-append under
|
|
1641
|
-
* the same `id`. Malformed records — a structurally-invalid `CorpusRecord` from disk or a caller —
|
|
1642
|
-
* fail loud (the validator throws), since a corpus that silently accepts garbage would poison
|
|
1643
|
-
* every downstream run that reads it back.
|
|
1644
|
-
*
|
|
1645
|
-
* @experimental
|
|
1646
|
-
*/
|
|
1647
|
-
|
|
1648
|
-
/**
|
|
1649
|
-
* In-memory `Corpus`. Keyed by record `id`; `append` validates the record, is idempotent on an
|
|
1650
|
-
* identical re-append, and returns a typed `{ succeeded: false }` on a conflicting re-append under
|
|
1651
|
-
* the same `id` (never overwrites). `query` routes through the single-sourced `applyFilter`.
|
|
1652
|
-
*/
|
|
1653
|
-
declare class InMemoryCorpus implements Corpus {
|
|
1654
|
-
private readonly byId;
|
|
1655
|
-
append(record: CorpusRecord): Promise<{
|
|
1656
|
-
succeeded: true;
|
|
1657
|
-
} | {
|
|
1658
|
-
succeeded: false;
|
|
1659
|
-
error: string;
|
|
1660
|
-
}>;
|
|
1661
|
-
query(filter: CorpusFilter): Promise<ReadonlyArray<CorpusRecord>>;
|
|
1662
|
-
}
|
|
1663
|
-
/**
|
|
1664
|
-
* JSONL on disk — one validated `CorpusRecord` per line, append-only. `query` replays the whole
|
|
1665
|
-
* file, validating every line (a malformed line fails loud — a corrupted corpus must never read
|
|
1666
|
-
* back silently) and folding by `id`: a later identical line dedups, a later conflicting line
|
|
1667
|
-
* under the same `id` is a corruption (fail loud). `append` first replays to enforce the same
|
|
1668
|
-
* idempotence/conflict contract as the in-mem impl, then fsyncs the new line so a crash between
|
|
1669
|
-
* writes never loses an acknowledged fact. Shares the JSONL append-line spine with the spawn
|
|
1670
|
-
* journal, but the interface stays separate (a learned fact is not a replay record).
|
|
1671
|
-
*/
|
|
1672
|
-
declare class FileCorpus implements Corpus {
|
|
1673
|
-
private readonly path;
|
|
1674
|
-
constructor(path: string);
|
|
1675
|
-
append(record: CorpusRecord): Promise<{
|
|
1676
|
-
succeeded: true;
|
|
1677
|
-
} | {
|
|
1678
|
-
succeeded: false;
|
|
1679
|
-
error: string;
|
|
1680
|
-
}>;
|
|
1681
|
-
query(filter: CorpusFilter): Promise<ReadonlyArray<CorpusRecord>>;
|
|
1682
|
-
private load;
|
|
1683
|
-
private appendLine;
|
|
1684
|
-
}
|
|
1685
|
-
/**
|
|
1686
|
-
* The learning-flywheel READ side. Queries the corpus through `filter`, renders the matching facts
|
|
1687
|
-
* (most-confident first, capped by `maxLines`) into instruction lines, and returns a FRESH
|
|
1688
|
-
* `AgentProfile` with them merged in — never mutates the input profile. Default `target: 'prompt'`
|
|
1689
|
-
* appends the lines to `prompt.instructions[]` (the additive append-line seam); `target:
|
|
1690
|
-
* 'resources'` folds them into the single-blob `resources.instructions` string (preserving any
|
|
1691
|
-
* existing blob, but failing loud on a non-string existing blob — a `resources.instructions` that
|
|
1692
|
-
* was already an `AgentProfileResourceRef` cannot be string-appended without dropping it).
|
|
1693
|
-
*
|
|
1694
|
-
* An empty query result returns a fresh COPY of the profile with no instruction change (a valid
|
|
1695
|
-
* "nothing learned yet" read, not an error).
|
|
1696
|
-
*/
|
|
1697
|
-
declare function renderCorpusToInstructions(opts: RenderCorpusToInstructionsOptions): Promise<AgentProfile$1>;
|
|
1698
|
-
|
|
1699
|
-
/**
|
|
1700
|
-
*
|
|
1701
|
-
* The personify layer impl — `definePersona` (the thin builder) + `runPersonified` (composes
|
|
1702
|
-
* the persona + chosen shape onto the keystone `Supervisor`), plus `createShapeContext`, the
|
|
1703
|
-
* seam that hands a shape its spawn helpers without it touching the registry.
|
|
1704
|
-
*
|
|
1705
|
-
* This file adds NO engine: `runPersonified` is `createSupervisor().run(rootAgent, task, …)`
|
|
1706
|
-
* where `rootAgent` is the persona's chosen `LoopShape` applied to a `ShapeContext`. All the
|
|
1707
|
-
* conserved-budget / journal / abort / typed-result machinery is the keystone's; this layer
|
|
1708
|
-
* only wires the persona's CONTENT (root spec + directive + context + seams) into it.
|
|
1709
|
-
*
|
|
1710
|
-
* One non-obvious invariant it must honor: `createSupervisor().run` builds the root `Scope`
|
|
1711
|
-
* with an EMPTY seam bag (`seams: {}`), so the built-in metered runtimes (router/sandbox/cli)
|
|
1712
|
-
* cannot read their seams off `ExecutorContext` through the default supervisor path. A persona
|
|
1713
|
-
* that supplies raw `seams` is therefore wrapped here into a registry whose resolved factories
|
|
1714
|
-
* receive a ctx with the persona seams merged in — so a persona never has to pre-close its
|
|
1715
|
-
* factories by hand. A persona may instead supply a fully-built `registry` and skip the wrap.
|
|
1716
|
-
*
|
|
1717
|
-
* @experimental
|
|
1718
|
-
*/
|
|
1719
|
-
|
|
1720
|
-
/**
|
|
1721
|
-
* Build a frozen `Persona`. Fails loud on the executors-supplied invariant: a persona with
|
|
1722
|
-
* neither a pre-built registry nor a seam bag cannot resolve its built-in runtimes, so it is
|
|
1723
|
-
* unrunnable — refuse it at definition time, not at the first spawn. Pure; no I/O.
|
|
1724
|
-
*/
|
|
1725
|
-
declare function definePersona<D = unknown>(input: DefinePersonaInput<D>): Persona<D>;
|
|
1726
|
-
/**
|
|
1727
|
-
* Compose the persona + chosen shape onto a fresh keystone `Supervisor`. Resolves the shape
|
|
1728
|
-
* (a factory verbatim, or a registered name through `builtinShapes`), applies it to a
|
|
1729
|
-
* `ShapeContext`, and runs the resulting root `Agent` to a typed `SupervisedResult<Outcome>`.
|
|
1730
|
-
* Fail loud on an unknown shape name or an unresolvable persona registry — never a silent
|
|
1731
|
-
* default-shape fallback.
|
|
1732
|
-
*/
|
|
1733
|
-
declare function runPersonified<Task, D>(options: RunPersonifiedOptions<Task, D>): Promise<SupervisedResult<Outcome<D>>>;
|
|
1734
|
-
|
|
1735
|
-
/**
|
|
1736
|
-
*
|
|
1737
|
-
* The loop-shape registry — the OPEN, content-free extension point for the personify layer.
|
|
1738
|
-
*
|
|
1739
|
-
* A `LoopShape` is reusable STRUCTURE (how to decompose / fan out / verify / synthesize),
|
|
1740
|
-
* parameterized by a persona's CONTENT. The registry lets a caller resolve a composed shape by
|
|
1741
|
-
* NAME: register a factory once, then `runPersonified({ shape: '<name>' })` resolves it with zero
|
|
1742
|
-
* edits elsewhere. `register` fails loud on a duplicate; `resolve` returns a typed outcome so an
|
|
1743
|
-
* unknown name is a named error, never a silent default.
|
|
1744
|
-
*
|
|
1745
|
-
* No shape is pre-registered: the generic combinators (`pipeline`/`fanout`/`loopUntil`/`panel`/
|
|
1746
|
-
* `verify`/`widen`) take spec arguments, so they are not bare zero-arg factories — a caller that
|
|
1747
|
-
* wants name-resolution registers its own COMPOSED shape (a combinator already applied to its
|
|
1748
|
-
* spec) on a registry instance. The registry carries SHAPE only; the domain lives on the persona.
|
|
1749
|
-
*
|
|
1750
|
-
* @experimental
|
|
1751
|
-
*/
|
|
1752
|
-
|
|
1753
|
-
/**
|
|
1754
|
-
* Build a fresh open `ShapeRegistry`. A factory is stored type-erased and re-cast on resolve — the
|
|
1755
|
-
* caller asserts the `<Task, D>` it expects, exactly as the executor registry stores its factories.
|
|
1756
|
-
*/
|
|
1757
|
-
declare function createShapeRegistry(): ShapeRegistry;
|
|
1758
|
-
/** The default registry `runPersonified` resolves a shape name against. Empty by construction —
|
|
1759
|
-
* a caller registers its own composed shapes; the engine ships no domain shape. */
|
|
1760
|
-
declare const builtinShapes: ShapeRegistry;
|
|
1761
|
-
/** Register a composed shape on the default `builtinShapes` registry — the one-call extension
|
|
1762
|
-
* point a caller invokes so its shape is resolvable by name with zero edits to the engine. */
|
|
1763
|
-
declare function registerShape<Task, D>(name: string, factory: LoopShape<Task, D>): void;
|
|
1764
|
-
|
|
1765
|
-
/**
|
|
1766
|
-
*
|
|
1767
|
-
* Trajectory trace + cost ledger — the post-hoc tree reconstructor (§4 of `wave-types`).
|
|
1768
|
-
*
|
|
1769
|
-
* `trajectoryReport` rebuilds the WHOLE realized spawn tree from the durable
|
|
1770
|
-
* `SpawnJournal` (+ optionally the `ResultBlobStore` for `done` artifacts): every node
|
|
1771
|
-
* (driver AND leaf), the real parent/child edges, each node's terminal status, its OWN
|
|
1772
|
-
* conserved `Spend`, and the `Spend` ROLLED UP over its subtree. Roll-up is a post-order
|
|
1773
|
-
* fold over the parent edges: a node's `rolledUpSpend` is its own spend plus every
|
|
1774
|
-
* descendant's, so a driver is charged for the fanout it caused — the root's roll-up is
|
|
1775
|
-
* the whole run's conserved total (tokens + usd + iterations + ms).
|
|
1776
|
-
*
|
|
1777
|
-
* `equalKOnCost` compares separate runs (arms) on that conserved COST, not on raw
|
|
1778
|
-
* iteration COUNT. The sandbox executor reports tokens/usd INCLUSIVE of a leaf's internal
|
|
1779
|
-
* sub-agent fanout, so charging an arm by `total.tokens`/`total.usd` (not by how many
|
|
1780
|
-
* `next()` cursors it logged) closes the leaf-fanout confound: a treatment leaf that fanned
|
|
1781
|
-
* out internally pays for it in cost, where a per-iteration count would hide it. The
|
|
1782
|
-
* within-run conserved pool already guarantees `Σk` equal by construction; this check is the
|
|
1783
|
-
* CROSS-run analogue the pool cannot reach — proving equal compute before any win is claimed.
|
|
1784
|
-
*
|
|
1785
|
-
* Pure over the journal/blobs — no live agent calls; safe to run on a finished run's log.
|
|
1786
|
-
*
|
|
1787
|
-
* @experimental
|
|
1788
|
-
*/
|
|
1789
|
-
|
|
1790
|
-
/**
|
|
1791
|
-
* Reconstruct the whole spawn tree for `root` with per-node + rolled-up `Spend`. Reads the
|
|
1792
|
-
* journal for structure + spend and, when `withOutputs`, the blob store for each `done`
|
|
1793
|
-
* node's artifact. Fail loud on a tree that was never journaled, a settle/cancel for an
|
|
1794
|
-
* un-spawned node (a corrupted log), or — under `withOutputs` — a `done` node whose blob the
|
|
1795
|
-
* store cannot rehydrate (a silent gap would mis-cost or mis-evidence the tree).
|
|
1796
|
-
*/
|
|
1797
|
-
declare function trajectoryReport(journal: SpawnJournal, blobs: ResultBlobStore, root: NodeId, options?: TrajectoryReportOptions): Promise<TrajectoryReport>;
|
|
1798
|
-
/**
|
|
1799
|
-
* Assert the arms are comparable at EQUAL conserved COST (tokens + usd), NOT raw iteration
|
|
1800
|
-
* count. Compares each arm's root-rolled-up `total` on the two conserved channels: an arm is
|
|
1801
|
-
* within-tolerance when the per-channel spread (max − min across arms) over the median is
|
|
1802
|
-
* `≤ tolerance`. Pure over the reports — no I/O. Fails loud on an empty arm list (nothing to
|
|
1803
|
-
* compare) so a vacuous "equal" is never returned.
|
|
1804
|
-
*/
|
|
1805
|
-
declare function equalKOnCost(arms: ReadonlyArray<EqualKArm>, options?: EqualKOnCostOptions): EqualKVerdict;
|
|
1806
|
-
|
|
1807
|
-
interface PromotionGateOptions {
|
|
1808
|
-
/** The HOLDOUT report — must carry per-task cells for both strategy names. */
|
|
1809
|
-
report: BenchmarkReport;
|
|
1810
|
-
/** The incumbent champion's strategy name. */
|
|
1811
|
-
incumbent: string;
|
|
1812
|
-
/** The challenger's strategy name. */
|
|
1813
|
-
candidate: string;
|
|
1814
|
-
/** 'superiority' (default): the candidate must score significantly BETTER.
|
|
1815
|
-
* 'non-inferiority': the candidate must prove its score is not worse than the
|
|
1816
|
-
* incumbent by more than `scoreTolerance` AND its cost savings are significant —
|
|
1817
|
-
* the gate for "same quality, cheaper" claims. */
|
|
1818
|
-
mode?: 'superiority' | 'non-inferiority';
|
|
1819
|
-
/** non-inferiority: the score CI lower bound must clear −scoreTolerance. Default 0.05. */
|
|
1820
|
-
scoreTolerance?: number;
|
|
1821
|
-
/** The CI lower bound on the paired lift must EXCEED this (score scale). Default 0. */
|
|
1822
|
-
deltaThreshold?: number;
|
|
1823
|
-
/** Minimum paired tasks before significance can be claimed. Default 6 — below that
|
|
1824
|
-
* the bootstrap CI is too wide to separate a real lift from the per-task noise. */
|
|
1825
|
-
minPairedTasks?: number;
|
|
1826
|
-
/** Bootstrap statistic over the paired deltas. Default 'mean'. */
|
|
1827
|
-
statistic?: 'mean' | 'median';
|
|
1828
|
-
/** Fixed by the substrate by default — the same report always yields the same verdict. */
|
|
1829
|
-
seed?: number;
|
|
1830
|
-
resamples?: number;
|
|
1831
|
-
}
|
|
1832
|
-
interface PromotionVerdict {
|
|
1833
|
-
promoted: boolean;
|
|
1834
|
-
reason: 'identical-champion' | 'few-tasks' | 'no-margin' | 'significant' | 'non-inferior-and-cheaper' | 'non-inferiority-unproven' | 'not-cheaper';
|
|
1835
|
-
mode: 'superiority' | 'non-inferiority';
|
|
1836
|
-
/** Paired tasks that carried both strategies' cells. */
|
|
1837
|
-
n: number;
|
|
1838
|
-
/** Paired (candidate − incumbent) lift across the holdout tasks. */
|
|
1839
|
-
lift: {
|
|
1840
|
-
mean: number;
|
|
1841
|
-
median: number;
|
|
1842
|
-
low: number;
|
|
1843
|
-
high: number;
|
|
1844
|
-
};
|
|
1845
|
-
/** non-inferiority mode: paired (incumbent − candidate) cost SAVINGS per task (usd) —
|
|
1846
|
-
* positive means the candidate is cheaper; significant iff the CI low clears zero. */
|
|
1847
|
-
costSavings?: {
|
|
1848
|
-
mean: number;
|
|
1849
|
-
median: number;
|
|
1850
|
-
low: number;
|
|
1851
|
-
high: number;
|
|
1852
|
-
};
|
|
1853
|
-
/** Paired (candidate − incumbent) wall-clock per task (ms) — negative = the candidate
|
|
1854
|
-
* is FASTER. Informational in every mode (never gates); the latency answer to "what
|
|
1855
|
-
* does this win actually cost the user?". */
|
|
1856
|
-
latency?: {
|
|
1857
|
-
mean: number;
|
|
1858
|
-
median: number;
|
|
1859
|
-
low: number;
|
|
1860
|
-
high: number;
|
|
1861
|
-
};
|
|
1862
|
-
}
|
|
1863
|
-
/** Statistical promotion decision over a holdout benchmark: a seeded paired bootstrap (`heldoutSignificance`) whose CI lower bound must clear `deltaThreshold`. */
|
|
1864
|
-
declare function promotionGate(opts: PromotionGateOptions): PromotionVerdict;
|
|
1865
|
-
|
|
1866
|
-
/**
|
|
1867
|
-
* Bridge a finished `runAgentRounds` into an agent-eval campaign / profile-matrix
|
|
1868
|
-
* dispatch.
|
|
1869
|
-
*
|
|
1870
|
-
* `runProfileMatrix` (and `runCampaign`) run the backend-integrity guard over
|
|
1871
|
-
* the token usage a dispatch reports through `ctx.cost`. A dispatch that wraps
|
|
1872
|
-
* `runAgentRounds` must forward the loop's cost AND token usage, or the guard reads
|
|
1873
|
-
* the run as a stub and throws. `reportLoopUsage` is that one line:
|
|
1874
|
-
*
|
|
1875
|
-
* const dispatch: ProfileDispatchFn<S, A> = async (profile, scenario, ctx) => {
|
|
1876
|
-
* const result = await runAgentRounds({ ...optsFor(profile, scenario), ctx: loopCtx })
|
|
1877
|
-
* reportLoopUsage(ctx, result)
|
|
1878
|
-
* return result.winner?.output as A
|
|
1879
|
-
* }
|
|
1880
|
-
*
|
|
1881
|
-
* Typed structurally against the campaign `DispatchContext.cost` so this module
|
|
1882
|
-
* stays free of an agent-eval import — it works with any cost meter exposing
|
|
1883
|
-
* `observe` + `observeTokens`.
|
|
1884
|
-
*/
|
|
1885
|
-
|
|
1886
|
-
/** The slice of an agent-eval campaign `DispatchContext.cost` this needs. */
|
|
1887
|
-
interface UsageSink {
|
|
1888
|
-
observe(amountUsd: number, source: string): void;
|
|
1889
|
-
observeTokens(usage: LoopTokenUsage): void;
|
|
1890
|
-
}
|
|
1891
|
-
/**
|
|
1892
|
-
* Forward a `LoopResult`'s aggregated cost + token usage into a campaign cost
|
|
1893
|
-
* meter so the backend-integrity guard sees real LLM activity. `source`
|
|
1894
|
-
* defaults to `'loop'`.
|
|
1895
|
-
*/
|
|
1896
|
-
declare function reportLoopUsage<Task, Output, Decision>(cost: UsageSink, result: Pick<LoopResult<Task, Output, Decision>, 'costUsd' | 'tokenUsage'>, source?: string): void;
|
|
1897
|
-
|
|
1898
|
-
/**
|
|
1899
|
-
* The product-facing backend selector: one call picks the execution transport a
|
|
1900
|
-
* `runAgentRounds` (or any consumer that drives a `SandboxClient`) runs on, from the
|
|
1901
|
-
* package a product already depends on. It is pure sugar over the existing
|
|
1902
|
-
* primitives — `createExecutor({ backend })` + `inlineSandboxClient` — and adds
|
|
1903
|
-
* ZERO transport code (the node:http cli-bridge transport lives in `bridgeExecutor`).
|
|
1904
|
-
*
|
|
1905
|
-
* The heavy `resolveBenchClient` in `@tangle-network/agent-bench` is the bench-side
|
|
1906
|
-
* sibling: it layers router/search specifics a product should not have to depend on.
|
|
1907
|
-
* This is the generic, dep-light core products call to switch backends:
|
|
1908
|
-
*
|
|
1909
|
-
* - `backend: 'sandbox'` → IN-BOX: return the caller's real Sandbox-backed
|
|
1910
|
-
* `sandboxClient` unchanged. Fail loud if absent — a
|
|
1911
|
-
* product on the sandbox backend already holds its box.
|
|
1912
|
-
* - `backend: 'bridge'` → OFF-BOX: a local cli-bridge fronting a harness CLI
|
|
1913
|
-
* (opencode / kimi-code / …) as the leaf executor, wired
|
|
1914
|
-
* through the resumable `bridgeExecutor`.
|
|
1915
|
-
* - `backend: 'router'` → OFF-BOX: a router chat-completion as the leaf executor,
|
|
1916
|
-
* presented as a `SandboxClient` (no sandbox dependency).
|
|
1917
|
-
* - `backend: 'local'` → SAME-HOST: a router-brain tool loop with the
|
|
1918
|
-
* profile's stdio MCP servers spawned as LOCAL child
|
|
1919
|
-
* processes — the only backend that can reach an MCP
|
|
1920
|
-
* server built into a host worktree.
|
|
1921
|
-
*/
|
|
1922
|
-
|
|
1923
|
-
interface ResolveSandboxClientOptions {
|
|
1924
|
-
/** The execution transport for the driven loop. */
|
|
1925
|
-
backend: 'sandbox' | 'bridge' | 'router' | 'local';
|
|
1926
|
-
/** `sandbox` backend: the caller's real Sandbox-backed client. Required for that backend. */
|
|
1927
|
-
sandboxClient?: SandboxClient;
|
|
1928
|
-
/** `bridge` backend: local cli-bridge transport. `bearer` + `model` required. */
|
|
1929
|
-
bridge?: {
|
|
1930
|
-
/** cli-bridge base URL. Defaults to `http://127.0.0.1:3355`. */
|
|
1931
|
-
url?: string;
|
|
1932
|
-
bearer: string;
|
|
1933
|
-
/** Bridge model id, doubling as the harness selector (e.g. `claude-code/sonnet`). */
|
|
1934
|
-
model: string;
|
|
1935
|
-
/** Per-turn deadline (ms). */
|
|
1936
|
-
timeoutMs?: number;
|
|
1937
|
-
};
|
|
1938
|
-
/** `router` backend: router chat-completion transport. All three fields required. */
|
|
1939
|
-
router?: {
|
|
1940
|
-
baseUrl: string;
|
|
1941
|
-
key: string;
|
|
1942
|
-
model: string;
|
|
1943
|
-
};
|
|
1944
|
-
/** `local` backend: same-host pseudo-box — the router brain drives a tool loop
|
|
1945
|
-
* with the profile's stdio MCP servers spawned as local children. */
|
|
1946
|
-
local?: LocalSandboxClientOptions;
|
|
1947
|
-
}
|
|
1948
|
-
/**
|
|
1949
|
-
* Resolve a `SandboxClient` for the chosen backend. The generic, dep-light core
|
|
1950
|
-
* that `resolveBenchClient` builds on — reuse this instead of hand-rolling the
|
|
1951
|
-
* `createExecutor`/`inlineSandboxClient` branch in each product.
|
|
1952
|
-
*/
|
|
1953
|
-
declare function resolveSandboxClient(opts: ResolveSandboxClientOptions): SandboxClient;
|
|
1954
|
-
|
|
1955
|
-
/**
|
|
1956
|
-
*
|
|
1957
|
-
* `acquireSandbox` — cold-start-resilient sandbox acquisition. Eliminates the
|
|
1958
|
-
* "create timed out at the proxy" failure mode conceptually by DECOUPLING "the
|
|
1959
|
-
* create HTTP call returned" from "the sandbox is ready":
|
|
1960
|
-
*
|
|
1961
|
-
* - Create is initiated with a known `name`.
|
|
1962
|
-
* - Readiness is observed from the sandbox's own `status` (`refresh()` polls
|
|
1963
|
-
* true state), NOT from whether the create call returned in time.
|
|
1964
|
-
* - If the create call itself times out at a gateway (502/503/504/522/524 or
|
|
1965
|
-
* a transport timeout), provisioning is still running server-side — so we
|
|
1966
|
-
* find the named sandbox via `list()` and wait for it to reach `running`.
|
|
1967
|
-
*
|
|
1968
|
-
* Result: a scale-from-zero cold start (node boot + host-agent registration,
|
|
1969
|
-
* minutes) can no longer surface as a create failure behind a ~100s proxy
|
|
1970
|
-
* limit. The loop becomes indifferent to whether the host pool is warm or cold.
|
|
1971
|
-
*
|
|
1972
|
-
* Invariant: an instance reporting no `status` (the minimal test fakes) is
|
|
1973
|
-
* treated as ready; only an explicit `pending`/`provisioning` status triggers
|
|
1974
|
-
* waiting, and only a retryable THROW triggers the find-by-name path. Real
|
|
1975
|
-
* errors (auth, validation, budget) fail loud. A box that is created (or found)
|
|
1976
|
-
* but never reaches `running` (abort, terminal status, budget) is torn down
|
|
1977
|
-
* before the failure propagates, so an abort storm during cold start does not
|
|
1978
|
-
* leak live sandboxes.
|
|
1979
|
-
*
|
|
1980
|
-
* @experimental
|
|
1981
|
-
*/
|
|
1982
|
-
|
|
1983
|
-
/** @experimental */
|
|
1984
|
-
interface AcquireOptions {
|
|
1985
|
-
/**
|
|
1986
|
-
* Total budget for the sandbox to reach `running`, covering on-demand node
|
|
1987
|
-
* cold-start. Default 600_000ms — matches the orchestrator's pending-host
|
|
1988
|
-
* registration window so we never give up before the platform itself would.
|
|
1989
|
-
*/
|
|
1990
|
-
readyTimeoutMs?: number;
|
|
1991
|
-
/** Poll interval while waiting for `running` / for the named sandbox to appear. */
|
|
1992
|
-
pollIntervalMs?: number;
|
|
1993
|
-
/** Cancellation (user abort). Distinct from create-call timeouts. */
|
|
1994
|
-
signal?: AbortSignal;
|
|
1995
|
-
/** Stamp a name so a timed-out create is recoverable by lookup. Auto-generated if absent. */
|
|
1996
|
-
name?: string;
|
|
1997
|
-
/** Clock override for deterministic tests. */
|
|
1998
|
-
now?: () => number;
|
|
1999
|
-
/** Sleep override for deterministic tests. */
|
|
2000
|
-
sleep?: (ms: number) => Promise<void>;
|
|
2001
|
-
}
|
|
2002
|
-
/**
|
|
2003
|
-
* Cold-start-resilient sandbox acquisition: create by name, observe readiness from the sandbox's own status (not the create call), and re-attach after gateway timeouts.
|
|
2004
|
-
*
|
|
2005
|
-
* @experimental
|
|
2006
|
-
*/
|
|
2007
|
-
declare function acquireSandbox(client: SandboxClient, options: CreateSandboxOptions, acquire?: AcquireOptions): Promise<SandboxInstance>;
|
|
2008
|
-
|
|
2009
|
-
/**
|
|
2010
|
-
*
|
|
2011
|
-
* Capability probe for the loop kernel's backend-blind lineage seams. The
|
|
2012
|
-
* kernel must NEVER ask "is this Docker or Firecracker?"; it asks "can this
|
|
2013
|
-
* platform fork a checkpoint?" via `client.criuStatus()` and degrades to fresh
|
|
2014
|
-
* boxes when the answer is no. CRIU availability is a per-platform fact, so the
|
|
2015
|
-
* probe is memoized per client — one network round-trip, reused across every
|
|
2016
|
-
* fanout in the run.
|
|
2017
|
-
*
|
|
2018
|
-
* Invariant: a client with no `criuStatus` method (the loop's test fakes, the
|
|
2019
|
-
* raw SDK before it grew the probe) reports `canFork = false`. The seam is
|
|
2020
|
-
* fail-CLOSED — never assume forking works, only enable it on a positive probe.
|
|
2021
|
-
*
|
|
2022
|
-
* @experimental
|
|
2023
|
-
*/
|
|
2024
|
-
|
|
2025
|
-
/**
|
|
2026
|
-
* What the loop kernel is allowed to know about a sandbox backend: a single
|
|
2027
|
-
* capability bit, never the backend's identity. `canFork` gates the
|
|
2028
|
-
* checkpoint+fork fanout path; everything else (session continuation) is a
|
|
2029
|
-
* universal SDK feature that needs no probe.
|
|
2030
|
-
*
|
|
2031
|
-
* @experimental
|
|
2032
|
-
*/
|
|
2033
|
-
interface SandboxCapabilities {
|
|
2034
|
-
/**
|
|
2035
|
-
* True only when `client.criuStatus()` returned `{ available: true }`. When
|
|
2036
|
-
* false, a fork-enabled fanout degrades to independent fresh boxes — same
|
|
2037
|
-
* result, no shared context prefix.
|
|
2038
|
-
*/
|
|
2039
|
-
canFork: boolean;
|
|
2040
|
-
}
|
|
2041
|
-
/**
|
|
2042
|
-
* Probe (and memoize per client) what the loop may rely on. A client without a
|
|
2043
|
-
* `criuStatus` method, or whose probe rejects, yields `canFork = false` — a
|
|
2044
|
-
* failed probe must never claim a capability the platform may not have. The
|
|
2045
|
-
* promise is cached so concurrent fanout branches share one round-trip.
|
|
2046
|
-
*
|
|
2047
|
-
* @experimental
|
|
2048
|
-
*/
|
|
2049
|
-
declare function probeSandboxCapabilities(client: SandboxClient): Promise<SandboxCapabilities>;
|
|
2050
|
-
/**
|
|
2051
|
-
* Narrowed view of the optional CRIU probe. The loop-side `SandboxClient`
|
|
2052
|
-
* does not require `criuStatus`; this widens it optionally so the probe can be
|
|
2053
|
-
* read without importing sandbox-backend specifics. @experimental
|
|
2054
|
-
*/
|
|
2055
|
-
interface CriuCapableClient {
|
|
2056
|
-
criuStatus?: () => Promise<{
|
|
2057
|
-
available: boolean;
|
|
2058
|
-
criuVersion?: string;
|
|
2059
|
-
reason?: string;
|
|
2060
|
-
}>;
|
|
2061
|
-
}
|
|
2062
|
-
|
|
2063
|
-
/**
|
|
2064
|
-
* Sandbox-event → runtime-event mapping.
|
|
2065
|
-
*
|
|
2066
|
-
* The sandbox SDK emits a polymorphic `SandboxEvent = { type, data, id? }`
|
|
2067
|
-
* whose `type` vocabulary is backend-determined (opencode, etc.) rather than
|
|
2068
|
-
* enumerated by the SDK. Two consumers project it:
|
|
2069
|
-
* - the loop kernel's cost ledger (`extractLlmCallEvent`) — sums usage off
|
|
2070
|
-
* every cost-bearing event, regardless of stream shape;
|
|
2071
|
-
* - the `AgentRuntime.act` streaming contract (`mapSandboxEvent`) — projects
|
|
2072
|
-
* incremental events to the `RuntimeStreamEvent` chat-UX vocabulary.
|
|
2073
|
-
*
|
|
2074
|
-
* Both live here so the empirically-observed `type` vocabulary has one home.
|
|
2075
|
-
*/
|
|
2076
|
-
|
|
2077
|
-
/**
|
|
2078
|
-
* Extract a `RuntimeStreamEvent`-shaped `llm_call` from a sandbox event when
|
|
2079
|
-
* the event carries usage/cost data. Returns `undefined` for non-cost events
|
|
2080
|
-
* so the kernel can iterate the full stream without branching.
|
|
2081
|
-
*
|
|
2082
|
-
* Canonical cost-carrying types observed in the wild:
|
|
2083
|
-
* - `llm_call` — `data: { model, tokensIn, tokensOut, costUsd, ... }`
|
|
2084
|
-
* - `message.completed` / `result` — `data: { usage: { inputTokens,
|
|
2085
|
-
* outputTokens, totalCostUsd? } }`
|
|
2086
|
-
* - `cost.usage` / `usage` — same shape under a dedicated type
|
|
2087
|
-
*
|
|
2088
|
-
* Numeric coercion is strict: `Number.isFinite` gates every accumulator write
|
|
2089
|
-
* so a sentinel `NaN` from a misbehaving backend cannot poison the ledger.
|
|
2090
|
-
*/
|
|
2091
|
-
declare function extractLlmCallEvent(event: SandboxEvent, agentRunName: string): (RuntimeStreamEvent & {
|
|
2092
|
-
type: 'llm_call';
|
|
2093
|
-
}) | undefined;
|
|
2094
|
-
/**
|
|
2095
|
-
* Sum the token usage + USD cost of a sandbox turn's events — the one honest way to meter an
|
|
2096
|
-
* `openSandboxRun` cell. Folds `extractLlmCallEvent` over the stream (which reads usage off EVERY backend
|
|
2097
|
-
* event shape), so a `runProfileMatrix` dispatch can report it to `ctx.cost`:
|
|
2098
|
-
*
|
|
2099
|
-
* receipt: (turn) => {
|
|
2100
|
-
* const u = sumSandboxUsage(turn.events)
|
|
2101
|
-
* return { model, inputTokens: u.input, outputTokens: u.output,
|
|
2102
|
-
* ...(u.costUsd > 0 ? { actualCostUsd: u.costUsd } : {}) }
|
|
2103
|
-
* }
|
|
2104
|
-
*
|
|
2105
|
-
* Without this a cell reads `{tokens:0, cost:0}` and the backend-integrity guard correctly aborts the
|
|
2106
|
-
* matrix as a stub. `agentRunName` is the fallback model label for cost-only events (default `'agent'`).
|
|
2107
|
-
*/
|
|
2108
|
-
declare function sumSandboxUsage(events: readonly SandboxEvent[], agentRunName?: string): {
|
|
2109
|
-
input: number;
|
|
2110
|
-
output: number;
|
|
2111
|
-
costUsd: number;
|
|
2112
|
-
};
|
|
2113
|
-
/**
|
|
2114
|
-
* Cross-event state for {@link mapSandboxToolEvent}. Sandbox backends emit a
|
|
2115
|
-
* tool invocation as MANY `message.part.updated` frames on the same call id
|
|
2116
|
-
* (pending → running → completed), so faithful projection needs per-call
|
|
2117
|
-
* status memory: one `tool_call` on first sighting, at most one `tool_result`
|
|
2118
|
-
* on the terminal transition, nothing on intermediate re-frames. Create one
|
|
2119
|
-
* state per turn via {@link createSandboxToolPartState}.
|
|
2120
|
-
*
|
|
2121
|
-
* @experimental
|
|
2122
|
-
*/
|
|
2123
|
-
interface SandboxToolPartState {
|
|
2124
|
-
/** Last seen status per tool call id. A terminal status is sticky — later
|
|
2125
|
-
* frames on a settled call project to nothing. */
|
|
2126
|
-
statusByCall: Map<string, string>;
|
|
2127
|
-
/** Sequence for synthesized call ids when an event carries none. */
|
|
2128
|
-
seq: number;
|
|
2129
|
-
}
|
|
2130
|
-
/**
|
|
2131
|
-
* Fresh per-turn {@link SandboxToolPartState} for {@link mapSandboxToolEvent} — an
|
|
2132
|
-
* empty call-status map so each turn projects tool frames independently.
|
|
2133
|
-
*
|
|
2134
|
-
* @experimental
|
|
2135
|
-
*/
|
|
2136
|
-
declare function createSandboxToolPartState(): SandboxToolPartState;
|
|
2137
|
-
/**
|
|
2138
|
-
* Project one `SandboxEvent` onto the `tool_call` / `tool_result` variants of
|
|
2139
|
-
* `RuntimeStreamEvent` — the tool-part projection `mapSandboxEvent`
|
|
2140
|
-
* deliberately does NOT perform. Opt-in and additive: `mapSandboxEvent`'s
|
|
2141
|
-
* default vocabulary (text/reasoning deltas + `llm_call`) is unchanged;
|
|
2142
|
-
* consumers that need the tool surface (chat UIs rendering tool activity)
|
|
2143
|
-
* compose this projector alongside it — `streamAgentTurn` does exactly that
|
|
2144
|
-
* under its `preserveToolParts` option.
|
|
2145
|
-
*
|
|
2146
|
-
* Handled shapes (observed on the opencode / claude-code sandbox backends):
|
|
2147
|
-
* - `message.part.updated` with `part.type === 'tool'` — stateful: a
|
|
2148
|
-
* `tool_call` on the call id's first frame (args from `state.input` or
|
|
2149
|
-
* `state.metadata.input`), a `tool_result` when the status transitions to
|
|
2150
|
-
* `completed` (result from `state.output` / `metadata.output`) or to a
|
|
2151
|
-
* terminal failure (result is `{ error, status, output? }` — the error
|
|
2152
|
-
* surfaced in-band, never dropped).
|
|
2153
|
-
* - bare `tool*` event types (`tool.call`, `tool_result`, …) — stateless:
|
|
2154
|
-
* `*result*` types project to `tool_result`, the rest to `tool_call`.
|
|
2155
|
-
*
|
|
2156
|
-
* Returns `[]` for every non-tool event.
|
|
2157
|
-
*
|
|
2158
|
-
* @experimental
|
|
2159
|
-
*/
|
|
2160
|
-
declare function mapSandboxToolEvent(event: SandboxEvent, state: SandboxToolPartState): (RuntimeStreamEvent & {
|
|
2161
|
-
type: 'tool_call' | 'tool_result';
|
|
2162
|
-
})[];
|
|
2163
|
-
/**
|
|
2164
|
-
* Project one `SandboxEvent` onto the `RuntimeStreamEvent` chat-UX vocabulary,
|
|
2165
|
-
* for runtimes that bridge a sandbox `streamPrompt` into the
|
|
2166
|
-
* `AgentRuntime.act` streaming contract. Returns `undefined` for events that
|
|
2167
|
-
* have no faithful projection — the raw stream is preserved separately for the
|
|
2168
|
-
* `OutputAdapter`, so an unmapped event never loses data.
|
|
2169
|
-
*
|
|
2170
|
-
* Mapped (the task-optional incremental variants — no synthesized task
|
|
2171
|
-
* lifecycle, no guessed tool-part shapes):
|
|
2172
|
-
* - `message.part.updated` text part → `text_delta`
|
|
2173
|
-
* - `message.part.updated` reasoning/thinking part → `reasoning_delta`
|
|
2174
|
-
* - cost-bearing events → `llm_call` (shared with the ledger extractor)
|
|
2175
|
-
*
|
|
2176
|
-
* Tool parts are deliberately NOT mapped here (unchanged default) — compose
|
|
2177
|
-
* {@link mapSandboxToolEvent} alongside when a consumer needs them.
|
|
2178
|
-
*
|
|
2179
|
-
* The opencode backend emits incremental text as
|
|
2180
|
-
* `{ type: 'message.part.updated', data: { part: { type, text }, delta } }`;
|
|
2181
|
-
* `delta` is the increment, `part.text` the running accumulation.
|
|
2182
|
-
*/
|
|
2183
|
-
declare function mapSandboxEvent(event: SandboxEvent, opts?: {
|
|
2184
|
-
agentRunName?: string;
|
|
2185
|
-
}): RuntimeStreamEvent | undefined;
|
|
2186
|
-
|
|
2187
|
-
/**
|
|
2188
|
-
*
|
|
2189
|
-
* `SandboxLineage` — the backend-blind owner of box + session handles for a
|
|
2190
|
-
* single `runAgentRounds` invocation. It exists so `run-loop.ts` never references a
|
|
2191
|
-
* backend (Docker / Firecracker): the lineage turns "continue this session" and
|
|
2192
|
-
* "fork this branch" into capability-gated sandbox-SDK calls and degrades to
|
|
2193
|
-
* fresh boxes when a capability is absent.
|
|
2194
|
-
*
|
|
2195
|
-
* Three operations, mirroring the kernel's per-iteration choices:
|
|
2196
|
-
* - `start(spec, prompt)` → a fresh box; the FIRST `streamPrompt` carries a
|
|
2197
|
-
* minted `sessionId` so later `continue` calls reuse the same server-side
|
|
2198
|
-
* conversation instead of re-injecting prior context as prompt text.
|
|
2199
|
-
* - `continue(handle, prompt)` → the SAME box, `streamPrompt({ sessionId })`.
|
|
2200
|
-
* The context lives in the sandbox; the prompt is only the new turn. Before
|
|
2201
|
-
* streaming it ASSERTS the session is still live server-side (via
|
|
2202
|
-
* `box.session(id).status()`): if the platform never honored the
|
|
2203
|
-
* client-minted id (or reaped it), `status()` is `null` and `continue`
|
|
2204
|
-
* fails loud rather than silently re-running the turn without prior context.
|
|
2205
|
-
* - `fork(handle, n, ...)` → when `canFork`, `checkpoint({ leaveRunning })` on
|
|
2206
|
-
* the parent then `fork(checkpointId)` × n so N branches inherit a shared
|
|
2207
|
-
* context prefix; otherwise N independent fresh boxes (same result, no
|
|
2208
|
-
* prefix). Either way each branch streams its own turn. Child-box creation
|
|
2209
|
-
* is bounded by the lineage's `maxConcurrency` — a 20-way fanout under a
|
|
2210
|
-
* concurrency cap of 2 provisions boxes in bounded waves, not all at once.
|
|
2211
|
-
*
|
|
2212
|
-
* Invariant: the lineage OWNS every box it starts or forks and tears them all
|
|
2213
|
-
* down on `teardown()` (or earlier via `prune`). It never tears down a box
|
|
2214
|
-
* mid-flight — the kernel decides when a handle is done. Streaming itself stays
|
|
2215
|
-
* in `run-loop.ts`; the lineage only hands back the live `streamPrompt` iterable
|
|
2216
|
-
* so the kernel keeps ownership of event collection, cost accounting, and trace
|
|
2217
|
-
* emission.
|
|
2218
|
-
*
|
|
2219
|
-
* @experimental
|
|
2220
|
-
*/
|
|
2221
|
-
|
|
2222
|
-
/**
|
|
2223
|
-
* A live box plus the session that threads its iterations together. Handed back
|
|
2224
|
-
* by `start`/`fork`, passed into `continue`/`fork` to descend from. Opaque to
|
|
2225
|
-
* the kernel beyond `box` (for placement/teardown) and `sessionId` (trace).
|
|
2226
|
-
*
|
|
2227
|
-
* @experimental
|
|
2228
|
-
*/
|
|
2229
|
-
interface SandboxLineageHandle {
|
|
2230
|
-
/** The owned, running sandbox this handle drives. */
|
|
2231
|
-
box: SandboxInstance;
|
|
2232
|
-
/**
|
|
2233
|
-
* Stable session id threaded through this box's `streamPrompt` calls. Minted
|
|
2234
|
-
* by the lineage on `start`; reused on `continue` so the server continues the
|
|
2235
|
-
* same conversation. A forked handle starts a fresh session on its new box —
|
|
2236
|
-
* the shared context comes from the checkpoint, not a shared session id.
|
|
2237
|
-
*/
|
|
2238
|
-
sessionId: string;
|
|
2239
|
-
}
|
|
2240
|
-
/**
|
|
2241
|
-
* Owns box + session handles for one loop run and offers the three
|
|
2242
|
-
* capability-gated lifecycle moves. Construct via `createSandboxLineage`.
|
|
2243
|
-
*
|
|
2244
|
-
* @experimental
|
|
2245
|
-
*/
|
|
2246
|
-
interface SandboxLineage {
|
|
2247
|
-
/**
|
|
2248
|
-
* Acquire a fresh box and begin a new session on it. Returns the handle and
|
|
2249
|
-
* the live `streamPrompt` iterable for the first turn (caller drains it).
|
|
2250
|
-
*/
|
|
2251
|
-
start(spec: AgentRunSpec<unknown>, prompt: string, signal: AbortSignal, promptOptions?: Omit<PromptOptions, 'signal' | 'sessionId'>): Promise<{
|
|
2252
|
-
handle: SandboxLineageHandle;
|
|
2253
|
-
events: AsyncIterable<SandboxEvent>;
|
|
2254
|
-
}>;
|
|
2255
|
-
/**
|
|
2256
|
-
* Continue an existing handle's session with one more turn on the SAME box.
|
|
2257
|
-
* The prior context is server-side; `prompt` is only the new turn. Asserts the
|
|
2258
|
-
* session is still known to the sandbox first (fail-loud) so a platform that
|
|
2259
|
-
* silently dropped the client-minted session id surfaces as an error instead
|
|
2260
|
-
* of a contextless turn the caller mistakes for a real continuation.
|
|
2261
|
-
*/
|
|
2262
|
-
continue(handle: SandboxLineageHandle, prompt: string, signal: AbortSignal, promptOptions?: Omit<PromptOptions, 'signal' | 'sessionId'>): Promise<AsyncIterable<SandboxEvent>>;
|
|
2263
|
-
/**
|
|
2264
|
-
* Branch `count` children from `parent`. When the platform can fork, each
|
|
2265
|
-
* child inherits `parent`'s checkpoint — and therefore the parent's IMAGE and
|
|
2266
|
-
* PROFILE: under a real fork `specs[i]` does NOT re-select a per-branch
|
|
2267
|
-
* profile (the SDK forks the running box, it can't swap the image). `specs[i]`
|
|
2268
|
-
* picks the per-branch profile ONLY on the degraded fresh-box path (no CRIU).
|
|
2269
|
-
* A heterogeneous-profile fanout therefore homogenizes to the parent's profile
|
|
2270
|
-
* when fork is available — pass a single shared spec for forked fanouts, or
|
|
2271
|
-
* use `random@k` (no fork) when branches must differ. Each child's first turn
|
|
2272
|
-
* streams `prompts[i]`. Child-box creation is bounded by `maxConcurrency`.
|
|
2273
|
-
*/
|
|
2274
|
-
fork(parent: SandboxLineageHandle, prompts: string[], specs: AgentRunSpec<unknown>[], signal: AbortSignal): Promise<{
|
|
2275
|
-
handle: SandboxLineageHandle;
|
|
2276
|
-
events: AsyncIterable<SandboxEvent>;
|
|
2277
|
-
}[]>;
|
|
2278
|
-
/**
|
|
2279
|
-
* Destroy every owned box whose handle is NOT in `keep`, freeing it before
|
|
2280
|
-
* loop end. The kernel calls this after a round when it can prove no future
|
|
2281
|
-
* round will descend from the pruned boxes (deterministic, monotonic branch
|
|
2282
|
-
* selection); boxes still reachable as a future branch source are retained.
|
|
2283
|
-
* Best-effort, bounded, parallel — a failed delete never throws.
|
|
2284
|
-
*/
|
|
2285
|
-
prune(keep: Iterable<SandboxLineageHandle>): Promise<void>;
|
|
2286
|
-
/** Destroy every box this lineage owns. Best-effort, bounded, parallel. */
|
|
2287
|
-
teardown(): Promise<void>;
|
|
2288
|
-
}
|
|
2289
|
-
/**
|
|
2290
|
-
* Build a lineage bound to one client + its probed capabilities. The
|
|
2291
|
-
* capabilities are passed in (not re-probed) so the kernel probes once per run
|
|
2292
|
-
* and the lineage stays a pure function of "what this platform can do".
|
|
2293
|
-
*
|
|
2294
|
-
* @experimental
|
|
2295
|
-
*/
|
|
2296
|
-
declare function createSandboxLineage(client: SandboxClient, capabilities: SandboxCapabilities, options?: {
|
|
2297
|
-
maxConcurrency?: number;
|
|
2298
|
-
streaming?: 'sse' | 'poll';
|
|
2299
|
-
/** Run provenance recorder forwarded to every `prepareBox` the lineage runs
|
|
2300
|
-
* (fresh start, continue, and fork branches). Absent ⇒ mounts go unrecorded
|
|
2301
|
-
* (a no-op recorder stands in so the ctx shape is always satisfied). */
|
|
2302
|
-
recordMount?: MountRecorder;
|
|
2303
|
-
}): SandboxLineage;
|
|
2304
|
-
/**
|
|
2305
|
-
* Loop-side widening of the box's optional checkpoint method. The
|
|
2306
|
-
* `SandboxClient`/`SandboxInstance` surface the kernel relies on does not
|
|
2307
|
-
* require checkpointing; this reads it optionally so the lineage can probe-gate
|
|
2308
|
-
* without importing sandbox-backend specifics. @experimental
|
|
2309
|
-
*/
|
|
2310
|
-
interface CheckpointCapableBox {
|
|
2311
|
-
checkpoint?: (options?: {
|
|
2312
|
-
leaveRunning?: boolean;
|
|
2313
|
-
tags?: string[];
|
|
2314
|
-
}) => Promise<{
|
|
2315
|
-
checkpointId: string;
|
|
2316
|
-
}>;
|
|
2317
|
-
}
|
|
2318
|
-
/** Loop-side widening of the box's optional fork method. @experimental */
|
|
2319
|
-
interface ForkCapableBox {
|
|
2320
|
-
fork?: (checkpointId: string, options?: {
|
|
2321
|
-
name?: string;
|
|
2322
|
-
}) => Promise<SandboxInstance>;
|
|
2323
|
-
}
|
|
2324
|
-
/**
|
|
2325
|
-
* Loop-side widening of the box's optional session accessor. The real
|
|
2326
|
-
* `SandboxInstance` exposes `session(id).status()`; the loop reads it optionally
|
|
2327
|
-
* so `continue` can assert session liveness without requiring it of the test
|
|
2328
|
-
* fakes. `status()` resolves `null` when the id is unknown to the sandbox.
|
|
2329
|
-
* @experimental
|
|
2330
|
-
*/
|
|
2331
|
-
interface SessionCapableBox {
|
|
2332
|
-
session?: (id: string) => {
|
|
2333
|
-
status: () => Promise<unknown | null>;
|
|
2334
|
-
};
|
|
2335
|
-
}
|
|
2336
|
-
|
|
2337
|
-
/**
|
|
2338
|
-
* `openSandboxRun` — the ONE harness-agnostic seam for running an agent in a
|
|
2339
|
-
* sandbox over a persistent artifact: run it, stream it, RESUME the same session
|
|
2340
|
-
* across turns. Domain-agnostic: a coding agent, a research agent, a tax/legal
|
|
2341
|
-
* agent — all flow through this; the domain lives only in the `Deliverable<Out>`
|
|
2342
|
-
* the caller supplies, never in a per-domain copy of this function.
|
|
2343
|
-
*
|
|
2344
|
-
* It is a thin facade (NOT a new layer) over code that already exists and is
|
|
2345
|
-
* already hardened:
|
|
2346
|
-
* - `acquireSandbox` — cold-start / 502-503-504 / gateway-timeout recovery,
|
|
2347
|
-
* - `buildBackendOptions` — the harness IS `backend.type` (opencode / codex /
|
|
2348
|
-
* claude-code / kimi-code / hermes / pi); the only "which agent" knob,
|
|
2349
|
-
* - `createSandboxLineage` — `start` mints a session; `resume` continues the
|
|
2350
|
-
* SAME server-side session with a fail-loud `assertSessionLive`.
|
|
2351
|
-
*
|
|
2352
|
-
* The one genuinely-new piece is {@link Deliverable}: it widens the pure
|
|
2353
|
-
* `OutputAdapter.parse(events)` to ALSO admit a post-turn read off the box FS —
|
|
2354
|
-
* the structural gap that made the bench gates hand-roll `box.fs.read`, because a
|
|
2355
|
-
* large produced file (a git diff, a generated document) truncates in the chat
|
|
2356
|
-
* stream and a pure events-parser cannot reach the workspace. Per the SDK, a
|
|
2357
|
-
* RELATIVE `deliverable.path` resolves from the workspace root and an ABSOLUTE one
|
|
2358
|
-
* (e.g. `/tmp/solution.patch`) reads the container filesystem directly — both are
|
|
2359
|
-
* valid; pick the one the agent actually wrote to. Avoid `..` traversal segments.
|
|
2360
|
-
*
|
|
2361
|
-
* What this deliberately does NOT do (so it stays a facade, not slop): no custom
|
|
2362
|
-
* reconnect/replay (the SDK + platform own per-session buffering + `Last-Event-ID`);
|
|
2363
|
-
* no fork verb (platform CRIU is probe-gated and currently absent — fork lives in
|
|
2364
|
-
* `SandboxLineage.fork` behind the capability probe, surfaced only if it returns).
|
|
2365
|
-
* It is also distinct from `runAgentRounds`: `runAgentRounds` is the multi-round, driver-driven
|
|
2366
|
-
* kernel (fresh box per round, events deliverable); this is a SINGLE rollout +
|
|
2367
|
-
* artifact-or-events deliverable + resume over ONE persistent box.
|
|
2368
|
-
*/
|
|
2369
|
-
|
|
2370
|
-
/**
|
|
2371
|
-
* How a typed deliverable `Out` is materialized from a finished turn.
|
|
2372
|
-
* - `events` — pure parse over the event array (identical to `OutputAdapter`).
|
|
2373
|
-
* - `artifact` — read a file off the box AFTER the turn drains, then map it (+ the
|
|
2374
|
-
* events). For diffs/codebases/documents that don't fit the chat
|
|
2375
|
-
* stream. `path` relative ⇒ workspace root; absolute ⇒ container FS.
|
|
2376
|
-
*
|
|
2377
|
-
* @experimental
|
|
2378
|
-
*/
|
|
2379
|
-
type Deliverable<Out> = {
|
|
2380
|
-
kind: 'events';
|
|
2381
|
-
fromEvents: (events: SandboxEvent[]) => Out;
|
|
2382
|
-
} | {
|
|
2383
|
-
kind: 'artifact';
|
|
2384
|
-
path: string;
|
|
2385
|
-
fromArtifact: (raw: string, events: SandboxEvent[]) => Out;
|
|
2386
|
-
};
|
|
2387
|
-
/**
|
|
2388
|
-
* One finished turn over the artifact. A failed FS read is surfaced in `readError`
|
|
2389
|
-
* (never masked as an empty deliverable) so a caller distinguishes "agent produced
|
|
2390
|
-
* nothing" from a transport/FS fault.
|
|
2391
|
-
*
|
|
2392
|
-
* @experimental
|
|
2393
|
-
*/
|
|
2394
|
-
interface TurnResult<Out> {
|
|
2395
|
-
out: Out;
|
|
2396
|
-
events: SandboxEvent[];
|
|
2397
|
-
readError?: string;
|
|
2398
|
-
}
|
|
2399
|
-
/**
|
|
2400
|
-
* Thrown when a turn is aborted/timed-out mid-settle. Carries the events drained
|
|
2401
|
-
* BEFORE the abort fired (and any in-progress `readError`) so an aborted run is
|
|
2402
|
-
* DIAGNOSABLE — the caller can tell never-started (`events: []`) from looped
|
|
2403
|
-
* (many events, no terminal `result`) from produced-nothing-then-cancelled.
|
|
2404
|
-
*
|
|
2405
|
-
* `name === 'AbortError'`, so existing `err.name === 'AbortError'` callers (the
|
|
2406
|
-
* loop kernel, scope, supervise runtime) keep matching it unchanged.
|
|
2407
|
-
*
|
|
2408
|
-
* @experimental
|
|
2409
|
-
*/
|
|
2410
|
-
declare class SandboxRunAbortError extends Error {
|
|
2411
|
-
readonly name = "AbortError";
|
|
2412
|
-
/** Events drained from the stream before the abort interrupted the turn. */
|
|
2413
|
-
readonly events: SandboxEvent[];
|
|
2414
|
-
/** The last artifact read error, if the abort fired during the retry loop. */
|
|
2415
|
-
readonly readError?: string;
|
|
2416
|
-
constructor(events: SandboxEvent[], readError?: string);
|
|
2417
|
-
}
|
|
2418
|
-
/** @experimental A live run over ONE persistent artifact (box + session). Close it
|
|
2419
|
-
* when done — `close()` tears the box down. */
|
|
2420
|
-
interface SandboxRun<Out> {
|
|
2421
|
-
readonly box: SandboxInstance;
|
|
2422
|
-
readonly sessionId: string;
|
|
2423
|
-
/** First turn over the fresh box (mints the session). Throws if already started. */
|
|
2424
|
-
start(prompt: string): Promise<TurnResult<Out>>;
|
|
2425
|
-
/** Continue THE SAME session over THE SAME artifact — a resumed turn/rollout. */
|
|
2426
|
-
resume(prompt: string): Promise<TurnResult<Out>>;
|
|
2427
|
-
close(): Promise<void>;
|
|
2428
|
-
}
|
|
2429
|
-
/** Prompt options forwarded to every sandbox prompt turn in this run. The
|
|
2430
|
-
* runtime owns `sessionId` and `signal` so callers cannot accidentally break
|
|
2431
|
-
* resume or cancellation semantics while still setting backend-level prompt
|
|
2432
|
-
* controls such as `timeoutMs`.
|
|
2433
|
-
*
|
|
2434
|
-
* @experimental
|
|
2435
|
-
*/
|
|
2436
|
-
type OpenSandboxRunPromptOptions = Omit<PromptOptions, 'signal' | 'sessionId'>;
|
|
2437
|
-
/** Context available after the box/session exists and before the first prompt is
|
|
2438
|
-
* drained. Intended for benchmark-owned workspace setup such as cloning a repo
|
|
2439
|
-
* into a fixed path. */
|
|
2440
|
-
interface OpenSandboxRunBeforeStartContext {
|
|
2441
|
-
readonly box: SandboxInstance;
|
|
2442
|
-
readonly sessionId: string;
|
|
2443
|
-
readonly signal: AbortSignal;
|
|
2444
|
-
}
|
|
2445
|
-
/** @experimental */
|
|
2446
|
-
interface OpenSandboxRunOptions {
|
|
2447
|
-
/** Profile + sandbox env/overrides. `sandboxOverrides.backend.type` is the harness. */
|
|
2448
|
-
agentRun: AgentRunSpec<string>;
|
|
2449
|
-
signal: AbortSignal;
|
|
2450
|
-
/** Optional execution-scoped observers. Hook failures never fail the run. */
|
|
2451
|
-
hooks?: RuntimeHooks;
|
|
2452
|
-
/** Stable run id for trace joins. Defaults to a short runtime-minted id. */
|
|
2453
|
-
runId?: string;
|
|
2454
|
-
/** Optional benchmark/scenario id carried into emitted hook events. */
|
|
2455
|
-
scenarioId?: string;
|
|
2456
|
-
/** Per-prompt sandbox SDK options forwarded to both `start()` and `resume()`.
|
|
2457
|
-
* The runtime still owns the session id and abort signal for each turn. */
|
|
2458
|
-
promptOptions?: OpenSandboxRunPromptOptions;
|
|
2459
|
-
/** Optional pre-start workspace setup. Runs after `lineage.start()` creates the
|
|
2460
|
-
* box/session and before the first prompt stream is consumed. A thrown error
|
|
2461
|
-
* fails the turn before the agent spends tokens. */
|
|
2462
|
-
beforeStart?: (ctx: OpenSandboxRunBeforeStartContext) => Promise<void> | void;
|
|
2463
|
-
/** Test seam for deterministic hook timestamps. Defaults to `Date.now`. */
|
|
2464
|
-
now?: () => number;
|
|
2465
|
-
/** Bounds box-creation bursts inside lineage fanout. Default from lineage. */
|
|
2466
|
-
maxConcurrency?: number;
|
|
2467
|
-
/** Base backoff (ms) for retrying a transient artifact `fs.read` failure; the i-th
|
|
2468
|
-
* retry waits `readRetryDelayMs * i`. Default 1000. Set 0 to disable the wait (tests). */
|
|
2469
|
-
readRetryDelayMs?: number;
|
|
2470
|
-
}
|
|
2471
|
-
/**
|
|
2472
|
-
* Open a sandbox run. Harness-agnostic: the harness lives in
|
|
2473
|
-
* `options.agentRun.sandboxOverrides.backend.type`, so opencode/codex/claude-code/
|
|
2474
|
-
* kimi-code all flow through this one entrypoint with identical env/auth wiring.
|
|
2475
|
-
*
|
|
2476
|
-
* @experimental
|
|
2477
|
-
*/
|
|
2478
|
-
declare function openSandboxRun<Out>(client: SandboxClient, options: OpenSandboxRunOptions, deliverable: Deliverable<Out>): Promise<SandboxRun<Out>>;
|
|
2479
|
-
|
|
2480
|
-
/**
|
|
2481
|
-
* Same-host stdio MCP: the ONE persistent newline-delimited JSON-RPC 2.0
|
|
2482
|
-
* connection to a spawned MCP server child process. This is the handshake
|
|
2483
|
-
* `mcpServeVerifier` boots for its probe (`initialize` →
|
|
2484
|
-
* `notifications/initialized` → `tools/list`), extracted so a trusted
|
|
2485
|
-
* same-host consumer can keep the server running and route `tools/call` to it.
|
|
2486
|
-
* This module does not isolate the child: callers must use a real sandbox for
|
|
2487
|
-
* user- or model-authored code.
|
|
2488
|
-
*
|
|
2489
|
-
* Two layers:
|
|
2490
|
-
* - `connectStdioMcp` — spawn ONE server at its cwd, run the real MCP
|
|
2491
|
-
* handshake, return a live connection: the listed tools, `callTool`, `close`.
|
|
2492
|
-
* - `materializeLocalMcp` — spawn EVERY enabled stdio server in
|
|
2493
|
-
* `profile.mcp`, namespace each server's tools as `<server>__<tool>` so
|
|
2494
|
-
* they can share a worker's tool list with a domain surface's tools, and
|
|
2495
|
-
* expose one route/close facade over the set. Fail-CLOSED: a declared
|
|
2496
|
-
* server that cannot boot throws — silently scoring the profile as if it
|
|
2497
|
-
* had no MCP surface would fake the with/without ablation.
|
|
2498
|
-
*
|
|
2499
|
-
* Failure taxonomy (mirrors `commandVerifier`/`mcpServeVerifier`): a missing
|
|
2500
|
-
* start binary or spawn fault is a SETUP bug (`McpSpawnFault` — candidate
|
|
2501
|
-
* graders must rethrow it, never score it); a server that crashes, errors the
|
|
2502
|
-
* handshake, or times out is an ordinary `Error` carrying the stderr tail; a
|
|
2503
|
-
* JSON-RPC error on `tools/call` is the AGENT's outcome — returned as an
|
|
2504
|
-
* `ERROR: …` string, never thrown (the `createMcpEnvironment` convention).
|
|
2505
|
-
*
|
|
2506
|
-
* Protocol matches the runtime's own stdio MCP server (src/mcp/server.ts):
|
|
2507
|
-
* newline-delimited JSON-RPC 2.0, protocol version 2024-11-05.
|
|
2508
|
-
*/
|
|
2509
|
-
|
|
2510
|
-
interface StdioMcpServerSpec {
|
|
2511
|
-
/** Command that starts the MCP server (stdio transport). */
|
|
2512
|
-
command: string;
|
|
2513
|
-
args?: string[];
|
|
2514
|
-
/** Working directory the server starts in (a built candidate's worktree, typically). */
|
|
2515
|
-
cwd?: string;
|
|
2516
|
-
/** Declared public env for the server process. Only a minimal non-sensitive
|
|
2517
|
-
* subset of the parent env is inherited. */
|
|
2518
|
-
env?: Record<string, string>;
|
|
2519
|
-
/** Sensitive env for the server process. These values override `env` and are
|
|
2520
|
-
* redacted from child-supplied errors, tool metadata, and tool results. */
|
|
2521
|
-
protectedEnv?: Record<string, string>;
|
|
2522
|
-
/** Handshake AND per-request timeout (ms). Default 30s. */
|
|
2523
|
-
timeoutMs?: number;
|
|
2524
|
-
}
|
|
2525
|
-
/** A missing start binary / spawn fault: a SETUP bug, never a failed candidate.
|
|
2526
|
-
* Graders (the serve verifier) must rethrow this instead of scoring it. */
|
|
2527
|
-
declare class McpSpawnFault extends Error {
|
|
2528
|
-
}
|
|
2529
|
-
interface McpToolDescriptor {
|
|
2530
|
-
name: string;
|
|
2531
|
-
description?: string;
|
|
2532
|
-
inputSchema?: unknown;
|
|
2533
|
-
}
|
|
2534
|
-
interface StdioMcpConnection {
|
|
2535
|
-
/** The tools the server exposed at connect time (`tools/list`). */
|
|
2536
|
-
readonly tools: readonly McpToolDescriptor[];
|
|
2537
|
-
/** `tools/call` → the result's text content. A JSON-RPC error / `isError`
|
|
2538
|
-
* result becomes an `ERROR: …` string (the agent's outcome); a dead
|
|
2539
|
-
* transport or timeout throws (an infra fault). */
|
|
2540
|
-
callTool(name: string, args: Record<string, unknown>): Promise<string>;
|
|
2541
|
-
/** Kill the server child. Idempotent. */
|
|
2542
|
-
close(): Promise<void>;
|
|
2543
|
-
}
|
|
2544
|
-
/** Spawn a trusted host command, complete the stdio MCP handshake, and return
|
|
2545
|
-
* the live connection. This low-level function provides no process isolation. */
|
|
2546
|
-
declare function connectStdioMcp(spec: StdioMcpServerSpec): Promise<StdioMcpConnection>;
|
|
2547
|
-
interface MaterializeLocalMcpOptions {
|
|
2548
|
-
/** Handshake / per-request timeout per server (ms). Default 30s. */
|
|
2549
|
-
timeoutMs?: number;
|
|
2550
|
-
/** Cap on a tool result's text fed back to the worker. Default 2000 chars. */
|
|
2551
|
-
maxResultChars?: number;
|
|
2552
|
-
/** Resolves a server's DECLARED secrets (`metadata.secretEnv`: env var name →
|
|
2553
|
-
* provider key name) at spawn time. The resolved values reach ONLY the child
|
|
2554
|
-
* process env — never the profile, the logs, or an error message. Fail-closed:
|
|
2555
|
-
* a server declaring secrets without a provider (or with a missing key)
|
|
2556
|
-
* throws instead of booting keyless. */
|
|
2557
|
-
keys?: KeyProvider;
|
|
2558
|
-
/** Required trust decision for profiles that declare local MCP processes.
|
|
2559
|
-
* Omit to refuse all profile-controlled host execution. Passing
|
|
2560
|
-
* `allowLocalMcp: true` is only safe for an author-controlled profile: the
|
|
2561
|
-
* process receives this Runtime's filesystem and network privileges. */
|
|
2562
|
-
profileSecurityPolicy?: AgentProfileSecurityPolicy;
|
|
2563
|
-
}
|
|
2564
|
-
/** The live same-host materialization of a profile's `mcp` surface. */
|
|
2565
|
-
interface LocalMcpMaterialization {
|
|
2566
|
-
/** Worker-facing tool specs: namespaced `<server>__<tool>`, provider-safe schemas. */
|
|
2567
|
-
tools: AgenticTool[];
|
|
2568
|
-
/** Whether `name` is one of this materialization's namespaced tools. */
|
|
2569
|
-
owns(name: string): boolean;
|
|
2570
|
-
/** Route a namespaced call to its server's live stdio child. */
|
|
2571
|
-
call(name: string, args: Record<string, unknown>): Promise<string>;
|
|
2572
|
-
/** Kill every spawned server. Idempotent. */
|
|
2573
|
-
close(): Promise<void>;
|
|
2574
|
-
}
|
|
2575
|
-
/**
|
|
2576
|
-
* Spawn every explicitly trusted stdio server in `profile.mcp` as a same-host
|
|
2577
|
-
* child and expose its tools under `<server>__<tool>` names. The default policy
|
|
2578
|
-
* refuses local processes. A profile with no MCP surface returns zero tools.
|
|
2579
|
-
*/
|
|
2580
|
-
declare function materializeLocalMcp(profile: AgentProfile$1, opts?: MaterializeLocalMcpOptions): Promise<LocalMcpMaterialization>;
|
|
2581
|
-
|
|
2582
|
-
/**
|
|
2583
|
-
*
|
|
2584
|
-
* Leak-free steering drivers — the non-LLM controls for the driven loop.
|
|
2585
|
-
*
|
|
2586
|
-
* These are the two sibling `Driver`s of the `refine` reference driver
|
|
2587
|
-
* (examples/driver-loop/driver-loop.ts), differing ONLY in how much of the
|
|
2588
|
-
* prior `verdict` their `plan()` is allowed to read. The amount of the verdict
|
|
2589
|
-
* a driver reads IS the experimental axis:
|
|
2590
|
-
*
|
|
2591
|
-
* • refine — reads `verdict.notes` (and/or `verdict.scores`): the grader's
|
|
2592
|
-
* findings. A reviewer/judge LLM turns those findings into the
|
|
2593
|
-
* next prompt. This is the "coached" condition.
|
|
2594
|
-
* • dumb — reads ONLY `verdict.valid` (the pass/fail boolean). No findings,
|
|
2595
|
-
* no scores, no LLM. The next prompt is one of two fixed strings
|
|
2596
|
-
* keyed on pass/fail. This is the leak-free pass/fail control.
|
|
2597
|
-
* • naive — reads NOTHING from the verdict. The next prompt is one fixed
|
|
2598
|
-
* continuation string every round. This is the leak-free
|
|
2599
|
-
* no-signal control.
|
|
2600
|
-
*
|
|
2601
|
-
* The `dumb → refine` gap measured on the same task therefore isolates how much
|
|
2602
|
-
* the grader's findings (its notes/scores) inflate the result beyond a bare
|
|
2603
|
-
* pass/fail signal: it is exactly the delta between reading `verdict.valid` and
|
|
2604
|
-
* reading `verdict.notes`. The `naive → dumb` gap isolates the value of the
|
|
2605
|
-
* pass/fail bit alone. Run all three against one task to attribute a loop's
|
|
2606
|
-
* lift to its actual source instead of crediting it to the grader for free.
|
|
2607
|
-
*
|
|
2608
|
-
* GENERALITY CONTRACT
|
|
2609
|
-
* -------------------
|
|
2610
|
-
* These builders carry ZERO domain coupling:
|
|
2611
|
-
* • The continuation strings are PARAMETERS. The caller passes its own text;
|
|
2612
|
-
* the substrate hardcodes none.
|
|
2613
|
-
* • The Task shape is OPAQUE. The builder never assumes a `prompt` field (or
|
|
2614
|
-
* any field). The caller supplies `applyContinuation(task, text) → Task`,
|
|
2615
|
-
* which folds a steering string into the caller's own Task shape — exactly
|
|
2616
|
-
* symmetric with the caller-supplied `AgentRunSpec.taskToPrompt`. For a Task
|
|
2617
|
-
* that is `{ prompt: string; … }`, the fold is the one-liner
|
|
2618
|
-
* `(task, text) => ({ ...task, prompt: text })`.
|
|
2619
|
-
*
|
|
2620
|
-
* They consume nothing the loop kernel does not already give `plan()`/`decide()`
|
|
2621
|
-
* — `runAgentRounds` requires no change. A benchmark picks a driver by name and the
|
|
2622
|
-
* kernel does the rest.
|
|
2623
|
-
*
|
|
2624
|
-
* @experimental
|
|
2625
|
-
*/
|
|
2626
|
-
|
|
2627
|
-
/**
|
|
2628
|
-
* Terminal-or-continue decision shared by all three steering drivers. The
|
|
2629
|
-
* non-terminal `'refine'` keeps the loop running another shot; the terminal
|
|
2630
|
-
* `'pick-winner'`/`'fail'` stop it (`isTerminalDecision` in run-loop.ts treats
|
|
2631
|
-
* `'pick-winner'` and `'fail'` as terminal and any other string as a request
|
|
2632
|
-
* for another round). Identical to the reference refine driver's decision set.
|
|
2633
|
-
*/
|
|
2634
|
-
type SteeringDecision = 'refine' | 'pick-winner' | 'fail';
|
|
2635
|
-
/**
|
|
2636
|
-
* Fold a steering string into the caller's Task shape, producing the Task for
|
|
2637
|
-
* the next shot. The substrate never assumes how a Task carries its prompt, so
|
|
2638
|
-
* the caller supplies this — the same way it supplies `taskToPrompt`. The
|
|
2639
|
-
* original `task` is passed so the fold can preserve task-level fields (ids,
|
|
2640
|
-
* fixtures, feature names) and replace only the instruction.
|
|
2641
|
-
*/
|
|
2642
|
-
type ApplyContinuation<Task> = (task: Task, continuation: string) => Task;
|
|
2643
|
-
/** Options for {@link naiveDriver}. */
|
|
2644
|
-
interface NaiveDriverOptions<Task> {
|
|
2645
|
-
/**
|
|
2646
|
-
* The fixed continuation issued every round after shot 0. The same string is
|
|
2647
|
-
* sent whether the prior shot passed inspection or not — the naive driver
|
|
2648
|
-
* reads no part of the verdict. Domain text is the caller's; the substrate
|
|
2649
|
-
* supplies none.
|
|
2650
|
-
*/
|
|
2651
|
-
continuation: string;
|
|
2652
|
-
/** Folds `continuation` into the caller's Task shape for the next shot. */
|
|
2653
|
-
applyContinuation: ApplyContinuation<Task>;
|
|
2654
|
-
/** Hard shot cap. The loop stops refining once history reaches this length. */
|
|
2655
|
-
maxIterations: number;
|
|
2656
|
-
/** Trace-event identifier. Default `'naive'`. */
|
|
2657
|
-
name?: string;
|
|
2658
|
-
}
|
|
2659
|
-
/**
|
|
2660
|
-
* `naiveDriver` — the no-signal steering control.
|
|
2661
|
-
*
|
|
2662
|
-
* `plan()` runs the initial `task` at shot 0, then issues the SAME fixed
|
|
2663
|
-
* `continuation` every subsequent round until a shot is valid or the cap is
|
|
2664
|
-
* hit. It reads NOTHING from `history[last].verdict` — not `.valid`, not
|
|
2665
|
-
* `.notes`, not `.scores`. It is the floor a coached loop must beat to earn its
|
|
2666
|
-
* coaching: any lift over naive that is not also present in `dumb` is
|
|
2667
|
-
* attributable to the pass/fail bit, and any lift of `refine` over `dumb` is
|
|
2668
|
-
* attributable to the grader's findings.
|
|
2669
|
-
*/
|
|
2670
|
-
declare function naiveDriver<Task, Output>(options: NaiveDriverOptions<Task>): Driver<Task, Output, SteeringDecision>;
|
|
2671
|
-
/** Options for {@link dumbDriver}. */
|
|
2672
|
-
interface DumbDriverOptions<Task> {
|
|
2673
|
-
/**
|
|
2674
|
-
* Continuation issued when the prior shot's verdict is valid. In a
|
|
2675
|
-
* stop-on-pass loop this is rarely reached (a valid shot ends the loop), but
|
|
2676
|
-
* it is required so the driver is total over the pass/fail bit; pass a
|
|
2677
|
-
* confirmation/keep-going string.
|
|
2678
|
-
*/
|
|
2679
|
-
onPass: string;
|
|
2680
|
-
/** Continuation issued when the prior shot's verdict is NOT valid. */
|
|
2681
|
-
onFail: string;
|
|
2682
|
-
/** Folds the chosen continuation into the caller's Task shape. */
|
|
2683
|
-
applyContinuation: ApplyContinuation<Task>;
|
|
2684
|
-
/** Hard shot cap. The loop stops refining once history reaches this length. */
|
|
2685
|
-
maxIterations: number;
|
|
2686
|
-
/** Trace-event identifier. Default `'dumb'`. */
|
|
2687
|
-
name?: string;
|
|
2688
|
-
}
|
|
2689
|
-
/**
|
|
2690
|
-
* `dumbDriver` — the pass/fail-only steering control.
|
|
2691
|
-
*
|
|
2692
|
-
* `plan()` runs the initial `task` at shot 0, then reads ONLY
|
|
2693
|
-
* `history[last].verdict.valid` (the boolean) and issues `onPass` or `onFail`
|
|
2694
|
-
* accordingly. It MUST NOT read `.notes` or `.scores` — that boundary is the
|
|
2695
|
-
* leak-free firewall. A `verdict` with no `valid` set (or no verdict) is
|
|
2696
|
-
* treated as not-valid, so the driver is total and never throws on a
|
|
2697
|
-
* grader/transport gap.
|
|
2698
|
-
*
|
|
2699
|
-
* The `dumb → refine` gap is the headline measurement: refine reads the
|
|
2700
|
-
* grader's `notes`, dumb reads only the pass/fail bit, so the difference is
|
|
2701
|
-
* exactly the value the findings add over a bare boolean.
|
|
2702
|
-
*/
|
|
2703
|
-
declare function dumbDriver<Task, Output>(options: DumbDriverOptions<Task>): Driver<Task, Output, SteeringDecision>;
|
|
2704
|
-
|
|
2705
|
-
/**
|
|
2706
|
-
* authorStrategy — the agent-authored layer as a package primitive (software-3.0): an
|
|
2707
|
-
* LLM reads a benchmark's per-task LOSSES + the defineStrategy contract and writes a NEW
|
|
2708
|
-
* optimization strategy as code; the caller gates it like any human-built candidate
|
|
2709
|
-
* (runBenchmark + a frozen holdout).
|
|
2710
|
-
*
|
|
2711
|
-
* Structurally safe by construction: the authored body composes shot()/critique() and
|
|
2712
|
-
* spends through the Supervisor's conserved pool — it can be wrong, but it cannot
|
|
2713
|
-
* Goodhart the check (it never sees the verifiers) and it cannot win by overspending.
|
|
2714
|
-
*
|
|
2715
|
-
* The authored module is written to `outDir` and dynamically imported — run under a
|
|
2716
|
-
* TS-capable loader (tsx) since models often emit type annotations.
|
|
2717
|
-
*/
|
|
2718
|
-
|
|
2719
|
-
/** The compressed consumable a skill carries: everything an author needs to emit a loop. */
|
|
2720
|
-
declare const strategyAuthorContract = "\nYou author an OPTIMIZATION STRATEGY for an agentic loop system. A strategy decides how to\nspend a compute budget to beat a task's deployable check. You compose exactly two steps:\n\n shot(spec?: { handle?, messages?, steer?, persona?, tools? }): Promise<ShotResult | null>\n Runs ONE worker attempt (a bounded tool loop) over an artifact.\n - omit handle => the shot opens its OWN fresh artifact and closes it after (a sample).\n - pass handle => the shot CONTINUES that artifact (state accumulates across shots).\n - messages => the carried conversation (pass the previous ShotResult.messages to continue).\n - steer => a corrective instruction injected before the shot.\n - persona => { systemPrompt?, model? } \u2014 give THIS shot its own role and/or model\n (multi-agent strategies: a researcher shot then an engineer shot, a panel of k\n personas over one budget). On a fresh shot the systemPrompt replaces the task's; on\n a carried conversation it arrives as a hand-off message. Same conserved budget.\n - tools => string[] \u2014 restrict THIS shot to a subset of the task's tools by\n name (focus an explore shot on read-only tools, an execute shot on write tools).\n Restriction-only; unknown names make the shot fail. ALWAYS select from\n await listTools(handle) \u2014 never hardcode. Omitted => the shot sees every tool.\n ShotResult = { messages, score (0..1 on the task's check), passes, total, completions, toolErrors }\n Returns null if the attempt failed infra-wise.\n\n critique(messages): Promise<string | null>\n A firewalled trace-analyst reads the attempt's trajectory and returns ONE corrective\n instruction (or null when it judges the work complete). Costs ~1 completion.\n\n consult(messages, instruction): Promise<string | null>\n The RAW analyst channel: the same firewalled critic answers YOUR instruction over the\n trajectory verbatim (no reformatting) \u2014 use it when you need a specific reply format\n (a decision, a prediction). Costs ~1 completion.\n\n surface.open(task) / surface.close(handle)\n Open a persistent artifact you manage yourself (remember to close in a finally).\n close is idempotent \u2014 closing an already-closed handle is a safe no-op.\n\n listTools(handle): Promise<Array<{ name, description? }>>\n The tools THIS task actually offers. TOOL SETS VARY PER TASK \u2014 if you restrict a\n shot with `tools`, you MUST pick names from await listTools(handle); hardcoding\n names from an example kills your shots on every task whose tools differ.\n\nRules:\n- ALWAYS await every shot/critique/surface call \u2014 a floating promise that rejects\n crashes the whole benchmark run.\n- Stay within ~budget total shots; every shot/critique spends from a conserved pool.\n- For a FRESH attempt OMIT `messages` entirely (never pass `[]` \u2014 an empty array is a\n fresh conversation too, but be explicit). To CONTINUE, pass the previous\n ShotResult.messages unchanged.\n- Return { score, resolved, completions, progression, shots } \u2014 score = the BEST checkpoint\n you reached (keep-best, never final-state), progression = score after each shot.\n- The module must be EXACTLY this shape (no other imports, no commentary outside code):\n\nimport { defineStrategy } from '@tangle-network/agent-runtime/loops'\nexport default defineStrategy('your-strategy-name', async ({ surface, task, budget, shot, critique, listTools }) => {\n // your composition (listTools comes from the destructured context \u2014 it is NOT a global)\n})\n";
|
|
2721
|
-
interface AuthorStrategyOptions {
|
|
2722
|
-
/** The model-call seam (agent-eval `createChatClient`). */
|
|
2723
|
-
chat: ChatClient;
|
|
2724
|
-
model?: string;
|
|
2725
|
-
/** A NAMED fallback author tried once when the primary call fails or returns no code
|
|
2726
|
-
* block (thinking models time out at the edge on long authoring prompts, or return
|
|
2727
|
-
* empty content without `maxTokens`). Opt-in — absent means the primary's failure
|
|
2728
|
-
* propagates. */
|
|
2729
|
-
fallbackModel?: string;
|
|
2730
|
-
/** The contract text shown to the author. Default `strategyAuthorContract`. The
|
|
2731
|
-
* meta-optimization coordinate: a GEPA/skill loop can evolve this text and gate each
|
|
2732
|
-
* variant on the same frozen holdout as any strategy. */
|
|
2733
|
-
contract?: string;
|
|
2734
|
-
/** The environment the losses came from (orientation only — never the verifiers). */
|
|
2735
|
-
environmentName: string;
|
|
2736
|
-
/** The per-task losses table (e.g. JSON.stringify(report.perTask)) — the gradient. */
|
|
2737
|
-
lossesJson: string;
|
|
2738
|
-
/** The budget the strategy must respect (shots/width). */
|
|
2739
|
-
budget: number;
|
|
2740
|
-
/** Where the authored module file is written (created if missing). */
|
|
2741
|
-
outDir: string;
|
|
2742
|
-
temperature?: number;
|
|
2743
|
-
/** Completion cap — required by thinking-model authors that stream reasoning first. */
|
|
2744
|
-
maxTokens?: number;
|
|
2745
|
-
signal?: AbortSignal;
|
|
2746
|
-
}
|
|
2747
|
-
/** Static CONTRACT lint over an authored strategy module — the module-boundary
|
|
2748
|
-
* enforcement of the harness's two measurement invariants:
|
|
2749
|
-
* - author blindness: the only import allowed is the loops surface. A body that could
|
|
2750
|
-
* reach the filesystem, network, or process could read or mutate verifier/artifact
|
|
2751
|
-
* state outside the brokered shots, and the harness-verified score would stop
|
|
2752
|
-
* meaning "what the shots achieved".
|
|
2753
|
-
* - conserved dose: no out-of-band compute (fetch/require/eval) — every unit a
|
|
2754
|
-
* strategy spends is metered by the Supervisor's pool, which is what makes
|
|
2755
|
-
* equal-budget comparisons between strategies valid.
|
|
2756
|
-
* A lint, not a sandbox: its job is keeping the benchmark numbers interpretable. */
|
|
2757
|
-
declare function assertStrategyContract(code: string): void;
|
|
2758
|
-
interface AuthoredStrategy {
|
|
2759
|
-
strategy: Strategy;
|
|
2760
|
-
file: string;
|
|
2761
|
-
code: string;
|
|
2762
|
-
}
|
|
2763
|
-
/** Author + load a strategy from losses. Throws when the author emits no loadable module;
|
|
2764
|
-
* with `fallbackModel` set, the named fallback gets one attempt first. */
|
|
2765
|
-
declare function authorStrategy(opts: AuthorStrategyOptions): Promise<AuthoredStrategy>;
|
|
2766
|
-
|
|
2767
|
-
/**
|
|
2768
|
-
* runStrategyEvolution — the multi-generation strategy search: per generation the system
|
|
2769
|
-
* authors a POPULATION of candidate strategies from the current tournament's losses,
|
|
2770
|
-
* plays them against the incumbent at equal budget, and advances a champion; one final
|
|
2771
|
-
* promotion decision runs on a NEVER-BEFORE-USED holdout slice through `promotionGate`.
|
|
2772
|
-
*
|
|
2773
|
-
* Measurement invariants (the reasons this design is shaped the way it is):
|
|
2774
|
-
* - The author sees TRAIN losses only. The holdout slice is drawn fresh (disjoint task
|
|
2775
|
-
* offsets) after all authoring is done — one promotion decision, one untouched slice,
|
|
2776
|
-
* so adaptive reuse of evaluation data never enters the verdict.
|
|
2777
|
-
* - Every tournament runs at the same per-strategy budget through the conserved pool;
|
|
2778
|
-
* candidates cannot win by overspending.
|
|
2779
|
-
* - Champion selection within the search is a SEARCH policy (configurable, default
|
|
2780
|
-
* cost-aware: ties on score go to the cheapest strategy — a scalar hides a strategy
|
|
2781
|
-
* that ties at half the cost). The promotion verdict never comes from search
|
|
2782
|
-
* selection; it comes from the gate on the fresh slice.
|
|
2783
|
-
* - Every authored artifact's description length (gzip bits) is recorded, so the
|
|
2784
|
-
* artifact-complexity-vs-holdout-gap relation is analyzable from any run's report.
|
|
2785
|
-
*
|
|
2786
|
-
* Lineage fields (`parent`, `generation`) are recorded on every archive node so a
|
|
2787
|
-
* descendant-productivity parent-selection policy can be added without changing the
|
|
2788
|
-
* report schema; the v1 search authors from the latest tournament's losses.
|
|
2789
|
-
*/
|
|
2790
|
-
|
|
2791
|
-
interface EvolutionAuthor {
|
|
2792
|
-
/** The model-call seam (agent-eval `createChatClient`). */
|
|
2793
|
-
chat: ChatClient;
|
|
2794
|
-
model?: string;
|
|
2795
|
-
fallbackModel?: string;
|
|
2796
|
-
temperature?: number;
|
|
2797
|
-
maxTokens?: number;
|
|
2798
|
-
}
|
|
2799
|
-
type ChampionPolicy = 'score' | 'costAware';
|
|
2800
|
-
interface StrategyEvolutionConfig {
|
|
2801
|
-
environment: Environment;
|
|
2802
|
-
/** Task supply by DISJOINT slice: `(offset, n)` must return n tasks unique to that
|
|
2803
|
-
* offset range. Train draws [0, trainN); the holdout draws [trainN + holdoutOffset,
|
|
2804
|
-
* …) — tasks the search never touched. */
|
|
2805
|
-
tasks: (offset: number, n: number) => Promise<AgenticTask[]>;
|
|
2806
|
-
trainN: number;
|
|
2807
|
-
holdoutN: number;
|
|
2808
|
-
/** Extra offset past the train slice for the holdout draw (rotate across runs). */
|
|
2809
|
-
holdoutOffset?: number;
|
|
2810
|
-
worker: AgenticOptions;
|
|
2811
|
-
author: EvolutionAuthor;
|
|
2812
|
-
/** Rollouts (sample) / shots (refine) per strategy per task. Default 3. */
|
|
2813
|
-
budget?: number;
|
|
2814
|
-
concurrency?: number;
|
|
2815
|
-
/** Author→tournament rounds after gen0. Default 2. */
|
|
2816
|
-
generations?: number;
|
|
2817
|
-
/** Authored candidates per generation. Default 2. */
|
|
2818
|
-
populationSize?: number;
|
|
2819
|
-
/** The gen0 field. Default [sample, refine, sampleThenRefine]. */
|
|
2820
|
-
baselines?: Strategy[];
|
|
2821
|
-
/** What "better" means for PROMOTION. 'score' (default): the candidate must beat the
|
|
2822
|
-
* incumbent's score (superiority gate). 'cost': the candidate must prove score
|
|
2823
|
-
* NON-INFERIORITY (not worse by more than `scoreTolerance`) plus significant cost
|
|
2824
|
-
* savings — the "same quality, cheaper" objective. The author is told the objective
|
|
2825
|
-
* and sees per-task spend either way. */
|
|
2826
|
-
objective?: 'score' | 'cost';
|
|
2827
|
-
/** Cost objective: the score CI lower bound must clear −scoreTolerance. Default 0.05. */
|
|
2828
|
-
scoreTolerance?: number;
|
|
2829
|
-
/** Search-side champion selection. Default 'costAware'. */
|
|
2830
|
-
champion?: ChampionPolicy;
|
|
2831
|
-
/** Score band treated as a tie under 'costAware'. Default 0.01. */
|
|
2832
|
-
championEpsilon?: number;
|
|
2833
|
-
/** Where authored modules are written. */
|
|
2834
|
-
outDir: string;
|
|
2835
|
-
/** Promotion-gate evidence floor (paired holdout tasks). */
|
|
2836
|
-
minPairedTasks?: number;
|
|
2837
|
-
/** BAND-AWARE scoring — concentrate the measurement where lift is possible.
|
|
2838
|
-
* Holdout: draw `holdoutPoolN` candidate tasks and run `baselines[0]` once at the run
|
|
2839
|
-
* budget as an INDEPENDENT reference screen; keep tasks scoring ≤ `maxRefScore`
|
|
2840
|
-
* (headroom exists) and take the first `holdoutN`. Band membership is decided before
|
|
2841
|
-
* either finalist touches a task and both finalists then face the SAME tasks — the
|
|
2842
|
-
* estimand becomes "paired lift on headroom tasks", pre-registered by this config.
|
|
2843
|
-
* Train: champion selection ignores zero-spread tasks (every field strategy scored
|
|
2844
|
-
* identically — zero selection information, pure noise dilution). */
|
|
2845
|
-
band?: {
|
|
2846
|
-
holdoutPoolN: number;
|
|
2847
|
-
/** Keep holdout tasks where the reference scores ≤ this. Default 0.99 — drop only
|
|
2848
|
-
* tasks the reference already solves fully (no headroom, a candidate can only tie). */
|
|
2849
|
-
maxRefScore?: number;
|
|
2850
|
-
};
|
|
2851
|
-
/** What the author learns from a tournament. 'exact' (default) = scores + progressions
|
|
2852
|
-
* per task; 'binary' = pass/fail only — the leakage-bounded channel (one bit per cell
|
|
2853
|
-
* per generation reaches the author from the evaluation data). */
|
|
2854
|
-
lossesDetail?: 'exact' | 'binary';
|
|
2855
|
-
/** Reproducer certification (arXiv:2606.11045): when the final champion is AUTHORED,
|
|
2856
|
-
* compress it to a short natural-language summary, have a fresh author re-implement
|
|
2857
|
-
* from the summary alone (no losses, no code), and score the reproduction on the same
|
|
2858
|
-
* holdout. A reproduction gap is an overfitting signal (their detector: 100%
|
|
2859
|
-
* sensitivity / 91% specificity in the ML-agent setting) — recorded on the report,
|
|
2860
|
-
* never gate-blocking in v1. */
|
|
2861
|
-
reproducerCheck?: {
|
|
2862
|
-
/** Word budget for the strategy summary. Default 64. */
|
|
2863
|
-
summaryMaxWords?: number;
|
|
2864
|
-
/** Reproduction counts as faithful when reproducedScore ≥ championScore − tolerance.
|
|
2865
|
-
* Default 0.05. */
|
|
2866
|
-
tolerance?: number;
|
|
2867
|
-
};
|
|
2868
|
-
/** Endurance: write the run state after every completed phase; with `resume`, a
|
|
2869
|
-
* restart skips completed phases (authored modules re-imported from their files).
|
|
2870
|
-
* Worst case after a mid-run death is re-paying ONE phase, never the run. */
|
|
2871
|
-
checkpoint?: {
|
|
2872
|
-
path: string;
|
|
2873
|
-
resume?: boolean;
|
|
2874
|
-
};
|
|
2875
|
-
/** Called before each benchmark phase (gen0, gen1…, band-screen, holdout, reproduce).
|
|
2876
|
-
* The seam for environment recycling — no artifacts span phases, so a runner may
|
|
2877
|
-
* recreate a wedge-prone environment container here. */
|
|
2878
|
-
onPhase?: (phase: string) => Promise<void>;
|
|
2879
|
-
onTask?: (phase: string, row: BenchmarkTaskRow, done: number, total: number) => void;
|
|
2880
|
-
hooks?: RuntimeHooks;
|
|
2881
|
-
}
|
|
2882
|
-
interface ChampionPick {
|
|
2883
|
-
name: string;
|
|
2884
|
-
score: number;
|
|
2885
|
-
usd: number;
|
|
2886
|
-
}
|
|
2887
|
-
interface EvolutionCandidate {
|
|
2888
|
-
name: string;
|
|
2889
|
-
file?: string;
|
|
2890
|
-
gzipBits?: number;
|
|
2891
|
-
codeChars?: number;
|
|
2892
|
-
/** Present when this author attempt failed (recorded, never silent). */
|
|
2893
|
-
error?: string;
|
|
2894
|
-
}
|
|
2895
|
-
interface EvolutionGeneration {
|
|
2896
|
-
generation: number;
|
|
2897
|
-
candidates: EvolutionCandidate[];
|
|
2898
|
-
report: BenchmarkReport;
|
|
2899
|
-
champion: ChampionPick;
|
|
2900
|
-
}
|
|
2901
|
-
interface EvolutionArchiveNode {
|
|
2902
|
-
name: string;
|
|
2903
|
-
source: 'baseline' | 'authored';
|
|
2904
|
-
generation: number;
|
|
2905
|
-
/** The champion whose tournament losses this candidate was authored from. */
|
|
2906
|
-
parent?: string;
|
|
2907
|
-
gzipBits?: number;
|
|
2908
|
-
file?: string;
|
|
2909
|
-
/** Latest measured tournament result — 0 until the node's first tournament settles
|
|
2910
|
-
* (an authored node is created before its generation's benchmark runs). */
|
|
2911
|
-
score: number;
|
|
2912
|
-
usd: number;
|
|
2913
|
-
}
|
|
2914
|
-
interface ReproductionCheck {
|
|
2915
|
-
/** The compressed strategy description the reproducer implemented from. */
|
|
2916
|
-
summary: string;
|
|
2917
|
-
reproducedName: string;
|
|
2918
|
-
file?: string;
|
|
2919
|
-
championHoldoutScore: number;
|
|
2920
|
-
reproducedHoldoutScore: number;
|
|
2921
|
-
/** champion − reproduced (positive = the reproduction fell short). */
|
|
2922
|
-
gap: number;
|
|
2923
|
-
/** reproducedScore ≥ championScore − tolerance. A failed reproduction is an
|
|
2924
|
-
* overfitting signal: the champion's win did not fit through the summary. */
|
|
2925
|
-
reproducible: boolean;
|
|
2926
|
-
/** Infra failure during reproduction (distinct from a semantic reproduction failure). */
|
|
2927
|
-
error?: string;
|
|
2928
|
-
}
|
|
2929
|
-
interface EvolutionBandInfo {
|
|
2930
|
-
/** Tasks screened by the reference on the holdout pool. */
|
|
2931
|
-
screened: number;
|
|
2932
|
-
/** Tasks kept (reference score ≤ maxRefScore) before truncating to holdoutN. */
|
|
2933
|
-
inBand: number;
|
|
2934
|
-
/** Reference scores per screened task (the screening record). */
|
|
2935
|
-
refScores: Array<{
|
|
2936
|
-
taskId: string;
|
|
2937
|
-
score: number;
|
|
2938
|
-
}>;
|
|
2939
|
-
}
|
|
2940
|
-
interface EvolutionReport {
|
|
2941
|
-
gen0: BenchmarkReport;
|
|
2942
|
-
gen0Champion: ChampionPick;
|
|
2943
|
-
generations: EvolutionGeneration[];
|
|
2944
|
-
archive: EvolutionArchiveNode[];
|
|
2945
|
-
finalChampion: ChampionPick;
|
|
2946
|
-
holdout: BenchmarkReport;
|
|
2947
|
-
verdict: PromotionVerdict;
|
|
2948
|
-
/** Present when band screening ran — the verdict's estimand is then "paired lift on
|
|
2949
|
-
* headroom tasks" (band membership fixed by the reference screen, pre-registered). */
|
|
2950
|
-
band?: EvolutionBandInfo;
|
|
2951
|
-
/** Present when reproducerCheck ran (final champion was authored). */
|
|
2952
|
-
reproduction?: ReproductionCheck;
|
|
2953
|
-
/** SEARCH TELEMETRY, not evidence: each entry is that generation's own train-slice
|
|
2954
|
-
* re-measurement, so cross-generation deltas mix true drift with run-to-run variance
|
|
2955
|
-
* (entries are unpaired across generations). The only evidence-grade comparison in
|
|
2956
|
-
* this report is `verdict` — both finalists measured fresh, paired, on the holdout. */
|
|
2957
|
-
trajectory: Array<{
|
|
2958
|
-
generation: number;
|
|
2959
|
-
champion: string;
|
|
2960
|
-
score: number;
|
|
2961
|
-
usd: number;
|
|
2962
|
-
}>;
|
|
2963
|
-
}
|
|
2964
|
-
/** Strategy means recomputed over the DISCRIMINATING tasks only — tasks where the field
|
|
2965
|
-
* strategies did not all score identically. Zero-spread tasks (everyone 1.0, everyone
|
|
2966
|
-
* 0.0, everyone tied) carry no selection information; averaging over them dilutes real
|
|
2967
|
-
* differences toward zero. Search-side denoising only — the gate never uses this. */
|
|
2968
|
-
declare function discriminatingMeans(report: BenchmarkReport, fieldOrder: string[]): Record<string, {
|
|
2969
|
-
score: number;
|
|
2970
|
-
usd: number;
|
|
2971
|
-
}> | null;
|
|
2972
|
-
/** The champion pick over a means table. 'score' takes the best mean score (ties →
|
|
2973
|
-
* field order). 'costAware' treats scores within `epsilon` of the best as tied and
|
|
2974
|
-
* takes the cheapest — the (score, $) Pareto rule collapsed to one pick. */
|
|
2975
|
-
declare function pickChampion(means: Record<string, {
|
|
2976
|
-
score: number;
|
|
2977
|
-
usd: number;
|
|
2978
|
-
}>, fieldOrder: string[], policy: ChampionPolicy, epsilon: number): ChampionPick;
|
|
2979
|
-
/** Search-side champion selection over a tournament report. */
|
|
2980
|
-
declare function selectChampion(report: BenchmarkReport, fieldOrder: string[], policy: ChampionPolicy, epsilon: number): ChampionPick;
|
|
2981
|
-
/** Multi-generation strategy search: author candidates from tournament losses, play them against the incumbent at equal budget, promote via `promotionGate` on an untouched holdout slice. */
|
|
2982
|
-
declare function runStrategyEvolution(cfg: StrategyEvolutionConfig): Promise<EvolutionReport>;
|
|
2983
|
-
|
|
2984
|
-
/**
|
|
2985
|
-
* `streamAgentTurn` — the ONE run-a-turn event-stream contract over every
|
|
2986
|
-
* execution substrate: a sandbox box (`SandboxInstance.streamPrompt`, or
|
|
2987
|
-
* `streamTask` via the `box-task` kind for autonomous-task semantics), a
|
|
2988
|
-
* one-shot `Executor` (cli-bridge / router / BYO, via `ExecutorFactory`), and
|
|
2989
|
-
* an in-process `AgentExecutionBackend` (the `resolveAgentBackend` output).
|
|
2990
|
-
*
|
|
2991
|
-
* One function, one vocabulary: every backend kind yields the existing
|
|
2992
|
-
* `RuntimeStreamEvent` union incrementally and ALWAYS terminates with a
|
|
2993
|
-
* `final` event whose `text` is the turn's final text and whose
|
|
2994
|
-
* `metadata.tokenUsage` / `metadata.costUsd` / `metadata.model` carry the
|
|
2995
|
-
* turn's metered usage. `collectAgentTurn` drains a stream into that terminal
|
|
2996
|
-
* summary plus the full event list.
|
|
2997
|
-
*
|
|
2998
|
-
* This is a UNIFICATION seam, not a new stream parser — each kind is a thin
|
|
2999
|
-
* adapter over code that already exists and is already hardened:
|
|
3000
|
-
* - `box` — `mapSandboxEvent` + `extractLlmCallEvent` (sandbox-events.ts)
|
|
3001
|
-
* project the sandbox event stream; nothing is re-mapped here.
|
|
3002
|
-
* - `box-task` — the same projection over `box.streamTask` (the sandbox
|
|
3003
|
-
* SDK's autonomous-task verb: the agent works to completion,
|
|
3004
|
-
* multi-turn, session state maintained) with per-task
|
|
3005
|
-
* `TaskOptions` passthrough.
|
|
3006
|
-
* - `executor` — `inlineSandboxClient` (the ONE executor→box adapter) turns
|
|
3007
|
-
* the factory into a box, then the box path drives it. The
|
|
3008
|
-
* executor's settle/teardown lifecycle stays in that adapter.
|
|
3009
|
-
* - `chat` — the backend's own `stream()` surface, normalized by
|
|
3010
|
-
* `normalizeBackendStreamEvent` (the same projection
|
|
3011
|
-
* `runAgentTaskStream` applies).
|
|
3012
|
-
*
|
|
3013
|
-
* Distinct from `openSandboxRun` (box-only, session resume over one persistent
|
|
3014
|
-
* artifact, raw `SandboxEvent` deliverables) and from `runAgentTaskStream`
|
|
3015
|
-
* (full task lifecycle: knowledge preflight, session store, resume). This is
|
|
3016
|
-
* the minimal turn primitive underneath both worlds: prompt in, one normalized
|
|
3017
|
-
* event stream out, terminal result+usage guaranteed on every non-thrown path.
|
|
3018
|
-
*
|
|
3019
|
-
* Stream envelope: `backend_start` → incremental events → (`backend_error` on
|
|
3020
|
-
* failure) → `final`. A caller-initiated abort terminates with
|
|
3021
|
-
* `final.status: 'aborted'`; an expired `timeoutMs` deadline with
|
|
3022
|
-
* `final.status: 'failed'` — so cancellation stays distinguishable from a
|
|
3023
|
-
* blown deadline.
|
|
3024
|
-
*
|
|
3025
|
-
* Mid-stream lifecycle work needs NO extra API: the generator is pull-based,
|
|
3026
|
-
* so the producer is suspended between yields and resumes only when the caller
|
|
3027
|
-
* pulls again. A consumer can therefore run arbitrary async work between
|
|
3028
|
-
* events — sync state on each `tool_result`, decide a no-op retry after
|
|
3029
|
-
* draining, run a pre-`done` flush when it receives `final` and BEFORE it
|
|
3030
|
-
* forwards its own terminal event downstream. The interleaving is guaranteed
|
|
3031
|
-
* (and locked by test): nothing is produced past the event the caller is
|
|
3032
|
-
* holding.
|
|
3033
|
-
*
|
|
3034
|
-
* @experimental
|
|
3035
|
-
*/
|
|
3036
|
-
|
|
3037
|
-
/**
|
|
3038
|
-
* The execution substrate one turn runs on — a closed discriminated union over
|
|
3039
|
-
* the three stream surfaces the runtime already owns.
|
|
3040
|
-
*
|
|
3041
|
-
* @experimental
|
|
3042
|
-
*/
|
|
3043
|
-
type AgentTurnBackend = {
|
|
3044
|
-
/** A live sandbox box: the turn is one `box.streamPrompt(prompt)` call. */
|
|
3045
|
-
kind: 'box';
|
|
3046
|
-
box: SandboxInstance;
|
|
3047
|
-
/**
|
|
3048
|
-
* Per-turn `PromptOptions` forwarded verbatim to `streamPrompt`
|
|
3049
|
-
* (`sessionId`, `turnId`, `model`, `backend` profile, `timeoutMs`, …).
|
|
3050
|
-
* The turn's derived abort signal (caller `signal` + `timeoutMs`
|
|
3051
|
-
* deadline) is always installed as `signal` — pass cancellation through
|
|
3052
|
-
* `StreamAgentTurnOptions`, not here.
|
|
3053
|
-
*/
|
|
3054
|
-
options?: Omit<PromptOptions, 'signal'>;
|
|
3055
|
-
/** Model label stamped on cost-only `llm_call` events. Default `'agent'`. */
|
|
3056
|
-
agentRunName?: string;
|
|
3057
|
-
} | {
|
|
3058
|
-
/**
|
|
3059
|
-
* A live sandbox box in TASK mode: the turn is one
|
|
3060
|
-
* `box.streamTask(prompt)` call — the sandbox SDK's autonomous-task
|
|
3061
|
-
* verb. Unlike `streamPrompt` (one chat turn), the agent works until
|
|
3062
|
-
* the task completes or errors, session state is maintained for
|
|
3063
|
-
* continuity, and `options.maxTurns` bounds the agent's internal turns.
|
|
3064
|
-
* Event projection, usage folding, and the terminal `final` contract
|
|
3065
|
-
* are identical to the `box` kind.
|
|
3066
|
-
*/
|
|
3067
|
-
kind: 'box-task';
|
|
3068
|
-
box: SandboxInstance;
|
|
3069
|
-
/**
|
|
3070
|
-
* Per-task `TaskOptions` forwarded verbatim to `streamTask`
|
|
3071
|
-
* (`maxTurns` plus every `PromptOptions` field). The turn's derived
|
|
3072
|
-
* abort signal is always installed as `signal`.
|
|
3073
|
-
*/
|
|
3074
|
-
options?: Omit<TaskOptions, 'signal'>;
|
|
3075
|
-
/** Model label stamped on cost-only `llm_call` events. Default `'agent'`. */
|
|
3076
|
-
agentRunName?: string;
|
|
3077
|
-
} | {
|
|
3078
|
-
/**
|
|
3079
|
-
* A one-shot `Executor` (cli-bridge / router / BYO): the factory is
|
|
3080
|
-
* instantiated fresh for the turn via `inlineSandboxClient`, run once on
|
|
3081
|
-
* the prompt, and torn down — the same per-spawn lifecycle the supervise
|
|
3082
|
-
* runtime gives it.
|
|
3083
|
-
*/
|
|
3084
|
-
kind: 'executor';
|
|
3085
|
-
factory: ExecutorFactory<unknown>;
|
|
3086
|
-
/** Model label stamped on cost-only `llm_call` events. Default `'agent'`. */
|
|
3087
|
-
agentRunName?: string;
|
|
3088
|
-
} | {
|
|
3089
|
-
/**
|
|
3090
|
-
* An in-process `AgentExecutionBackend` (`resolveAgentBackend` output or
|
|
3091
|
-
* any custom backend): the turn is one `backend.stream()` call.
|
|
3092
|
-
*/
|
|
3093
|
-
kind: 'chat';
|
|
3094
|
-
backend: AgentExecutionBackend;
|
|
3095
|
-
};
|
|
3096
|
-
/** @experimental */
|
|
3097
|
-
interface StreamAgentTurnOptions {
|
|
3098
|
-
/** Caller-initiated cancellation. Terminates the stream with `final.status: 'aborted'`. */
|
|
3099
|
-
signal?: AbortSignal;
|
|
3100
|
-
/**
|
|
3101
|
-
* Wall-clock deadline for the whole turn in ms. An expired deadline aborts
|
|
3102
|
-
* the backend and terminates the stream with `final.status: 'failed'`
|
|
3103
|
-
* (a blown deadline is a turn failure, not a caller cancellation).
|
|
3104
|
-
*/
|
|
3105
|
-
timeoutMs?: number;
|
|
3106
|
-
/**
|
|
3107
|
-
* Opt-in tool-part projection for box-kind backends (`box`, `box-task`,
|
|
3108
|
-
* `executor`): sandbox tool parts additionally surface in-stream as
|
|
3109
|
-
* `tool_call` / `tool_result` events (`mapSandboxToolEvent`), so a consumer
|
|
3110
|
-
* rendering tool activity needs no bespoke sandbox-event parser. Default
|
|
3111
|
-
* off — the stream vocabulary existing consumers see is unchanged. No-op
|
|
3112
|
-
* for the `chat` kind (its backend emits `RuntimeStreamEvent`s directly,
|
|
3113
|
-
* tool events included when the backend produces them).
|
|
3114
|
-
*/
|
|
3115
|
-
preserveToolParts?: boolean;
|
|
3116
|
-
/**
|
|
3117
|
-
* Raw-event tap for box-kind backends: called (and awaited) with every
|
|
3118
|
-
* unmapped `SandboxEvent` BEFORE it is projected, so a consumer can read
|
|
3119
|
-
* parts the chat-UX projection drops (part ids, step markers, custom
|
|
3120
|
-
* backend events) without forking the mapper. Purely observational — it
|
|
3121
|
-
* cannot alter the mapped stream. Never called for the `chat` kind, which
|
|
3122
|
-
* has no sandbox events.
|
|
3123
|
-
*/
|
|
3124
|
-
onRawEvent?: (event: SandboxEvent) => void | Promise<void>;
|
|
3125
|
-
}
|
|
3126
|
-
/**
|
|
3127
|
-
* Metered usage of one turn, summed over every cost-bearing event the backend
|
|
3128
|
-
* emitted. `input`/`output` are token counts (0 when the backend reported
|
|
3129
|
-
* none — the honest sum, never a fabricated estimate). `costUsd`/`model` are
|
|
3130
|
-
* present only when the backend actually reported them.
|
|
3131
|
-
*
|
|
3132
|
-
* @experimental
|
|
3133
|
-
*/
|
|
3134
|
-
interface AgentTurnUsage {
|
|
3135
|
-
input: number;
|
|
3136
|
-
output: number;
|
|
3137
|
-
costUsd?: number;
|
|
3138
|
-
model?: string;
|
|
3139
|
-
}
|
|
3140
|
-
/**
|
|
3141
|
-
* A drained turn: the terminal summary plus every event the stream yielded.
|
|
3142
|
-
* `status`/`error` mirror the terminal `final` event so a failed or aborted
|
|
3143
|
-
* turn stays inspectable without re-scanning `events`.
|
|
3144
|
-
*
|
|
3145
|
-
* @experimental
|
|
3146
|
-
*/
|
|
3147
|
-
interface CollectedAgentTurn {
|
|
3148
|
-
finalText: string;
|
|
3149
|
-
usage: AgentTurnUsage;
|
|
3150
|
-
events: RuntimeStreamEvent[];
|
|
3151
|
-
status: AgentTaskStatus;
|
|
3152
|
-
error?: BackendErrorDetail;
|
|
3153
|
-
}
|
|
3154
|
-
/**
|
|
3155
|
-
* Run ONE agent turn on any backend kind and stream its events. Yields the
|
|
3156
|
-
* `RuntimeStreamEvent` vocabulary incrementally and always ends with a `final`
|
|
3157
|
-
* event carrying the turn's text and usage (`metadata.tokenUsage`,
|
|
3158
|
-
* `metadata.costUsd?`, `metadata.model?`) — on success, failure, abort, and
|
|
3159
|
-
* timeout alike. The generator never throws; failures surface in-band as
|
|
3160
|
-
* `backend_error` + `final` with a typed `error` detail.
|
|
3161
|
-
*
|
|
3162
|
-
* @experimental
|
|
3163
|
-
*/
|
|
3164
|
-
declare function streamAgentTurn(backend: AgentTurnBackend, prompt: string, opts?: StreamAgentTurnOptions): AsyncGenerator<RuntimeStreamEvent>;
|
|
3165
|
-
/**
|
|
3166
|
-
* Drain a `streamAgentTurn` stream (or any `RuntimeStreamEvent` stream that
|
|
3167
|
-
* honors its terminal contract) into the turn summary plus the full event
|
|
3168
|
-
* list. Fail-loud: throws when the stream ends without a terminal `final`
|
|
3169
|
-
* event — a stream that violates the contract must not read as an empty turn.
|
|
3170
|
-
*
|
|
3171
|
-
* @experimental
|
|
3172
|
-
*/
|
|
3173
|
-
declare function collectAgentTurn(stream: AsyncIterable<RuntimeStreamEvent>): Promise<CollectedAgentTurn>;
|
|
3174
|
-
|
|
3175
|
-
/**
|
|
3176
|
-
*
|
|
3177
|
-
* The supervisor's intelligence is AUTHORING the agents it spawns — not pressing buttons.
|
|
3178
|
-
*
|
|
3179
|
-
* Every agent here is three things: instructions (system prompt), tools, and a model — its
|
|
3180
|
-
* `AgentProfile`. The supervisor's job is to WRITE those profiles: read the task, decompose it,
|
|
3181
|
-
* and for each sub-task author a tailored worker recipe. `supervisorInstructions` is the how-to the
|
|
3182
|
-
* supervisor reads (its system prompt); `authoredWorker` builds a worker AGENT from a profile the
|
|
3183
|
-
* supervisor authored — the authored systemPrompt + model shape the worker's call.
|
|
3184
|
-
*
|
|
3185
|
-
* The skill is the single OPTIMIZABLE surface: edit it → the supervisor designs better agents.
|
|
3186
|
-
* That is the self-improvement lever (the prompt/skill lever), not the execution plumbing.
|
|
3187
|
-
*
|
|
3188
|
-
* @experimental
|
|
3189
|
-
*/
|
|
3190
|
-
|
|
3191
|
-
/** What the supervisor AUTHORS per sub-task — a worker recipe (a partial `AgentProfile`). */
|
|
3192
|
-
interface AuthoredProfile {
|
|
3193
|
-
name: string;
|
|
3194
|
-
/** The rich, task-specific instructions the supervisor wrote for THIS worker. */
|
|
3195
|
-
systemPrompt: string;
|
|
3196
|
-
/** The model the supervisor chose for this sub-task (falls back to the run default). */
|
|
3197
|
-
model?: string;
|
|
3198
|
-
}
|
|
3199
|
-
/** Narrow an untyped `spawn_agent` profile argument to an `AuthoredProfile`, or null if the
|
|
3200
|
-
* supervisor failed to author one (empty/placeholder profile — a skill violation worth catching). */
|
|
3201
|
-
declare function asAuthoredProfile(raw: unknown): AuthoredProfile | null;
|
|
3202
|
-
/** The supervisor SKILL — the how-to the supervisor reads (its system prompt). THE optimizable
|
|
3203
|
-
* surface: editing this changes how the supervisor designs every agent it spawns. */
|
|
3204
|
-
declare function supervisorInstructions(opts?: {
|
|
3205
|
-
goal?: string;
|
|
3206
|
-
}): string;
|
|
3207
|
-
/** Build a worker AGENT from a profile the supervisor authored: the authored `systemPrompt` +
|
|
3208
|
-
* `model` shape the worker's one model call; the deliverable gates settlement (valid ⟺ delivered). */
|
|
3209
|
-
declare function authoredWorker(profile: AuthoredProfile, opts: {
|
|
3210
|
-
cfg: RouterConfig;
|
|
3211
|
-
taskPrompt: string;
|
|
3212
|
-
deliverable: DeliverableSpec;
|
|
3213
|
-
temperature?: number;
|
|
3214
|
-
}): Agent<unknown, unknown>;
|
|
3215
|
-
/** Thresholds below which a system prompt is treated as a thin stub. Tunable per call. */
|
|
3216
|
-
interface ProfileRichnessThresholds {
|
|
3217
|
-
/** A prompt shorter than this many characters is thin (default 600). */
|
|
3218
|
-
readonly minSystemPromptChars: number;
|
|
3219
|
-
/** A prompt with fewer than this many non-blank lines is thin (default 6). */
|
|
3220
|
-
readonly minSystemPromptLines: number;
|
|
3221
|
-
}
|
|
3222
|
-
/** Default thresholds for `ProfileRichnessThresholds` — 600 chars / 6 lines minimum system prompt. */
|
|
3223
|
-
declare const defaultProfileRichnessThresholds: ProfileRichnessThresholds;
|
|
3224
|
-
/** Per-field verdict on one authored profile — the raw material the bench renders + scores. */
|
|
3225
|
-
interface ProfileRichness {
|
|
3226
|
-
readonly name: string;
|
|
3227
|
-
/** The resolved system prompt (canonical `prompt.systemPrompt`, the sandbox `prompt.system`
|
|
3228
|
-
* convention, or a bare-string prompt — whichever the author used). */
|
|
3229
|
-
readonly systemPrompt: string;
|
|
3230
|
-
readonly systemPromptChars: number;
|
|
3231
|
-
readonly systemPromptLines: number;
|
|
3232
|
-
readonly sentenceCount: number;
|
|
3233
|
-
readonly hasDescription: boolean;
|
|
3234
|
-
readonly hasTools: boolean;
|
|
3235
|
-
readonly hasSkills: boolean;
|
|
3236
|
-
readonly hasMcp: boolean;
|
|
3237
|
-
readonly hasSubagents: boolean;
|
|
3238
|
-
/** 0..1 — fraction of richness signals present (prompt-depth + the four levers). */
|
|
3239
|
-
readonly richness: number;
|
|
3240
|
-
/** True when the supervisor authored a stub instead of a real profile. */
|
|
3241
|
-
readonly thin: boolean;
|
|
3242
|
-
/** The specific reasons it is thin (empty when rich) — used in the finding's action. */
|
|
3243
|
-
readonly reasons: string[];
|
|
3244
|
-
}
|
|
3245
|
-
/** OBSERVE one authored `AgentProfile` and score its richness (no judge verdict is read). The task
|
|
3246
|
-
* context (`needsMcp`) lets a domain say "this work needs a data/tool MCP" so a missing MCP counts. */
|
|
3247
|
-
declare function assessAuthoredProfile(profile: AgentProfile$1, opts?: {
|
|
3248
|
-
needsMcp?: boolean;
|
|
3249
|
-
thresholds?: Partial<ProfileRichnessThresholds>;
|
|
3250
|
-
}): ProfileRichness;
|
|
3251
|
-
/** Turn a {@link ProfileRichness} verdict into a bus-routable `AnalystFinding` (area `profile-quality`).
|
|
3252
|
-
* Severity scales with thinness; the recommended action names the MISSING lever so the supervisor can
|
|
3253
|
-
* re-author. `subject` = the worker name so per-worker findings diff cleanly across re-authors. */
|
|
3254
|
-
declare function profileRichnessFinding(richness: ProfileRichness, opts?: {
|
|
3255
|
-
analystId?: string;
|
|
3256
|
-
runId?: string;
|
|
3257
|
-
}): AnalystFinding;
|
|
3258
|
-
|
|
3259
|
-
/**
|
|
3260
|
-
*
|
|
3261
|
-
* The conserved budget reservation pool — the invariant the whole instrument
|
|
3262
|
-
* rests on (critique M5/B3). One root `Budget` becomes a conserved pool of three
|
|
3263
|
-
* quantities (tokens, usd, iterations) plus an absolute deadline. Children reserve
|
|
3264
|
-
* atomically at spawn and reconcile at settle:
|
|
3265
|
-
*
|
|
3266
|
-
* total ≡ free + reserved + committed (invariant, always)
|
|
3267
|
-
*
|
|
3268
|
-
* `reserve` moves a child's whole ceiling from `free` → `reserved` and fails closed
|
|
3269
|
-
* when `free` can't cover it (never read-then-spawn overcommit, so `Σk(treatment) ≡
|
|
3270
|
-
* Σk(blind)` by construction). `reconcile` releases the reservation, commits ACTUAL
|
|
3271
|
-
* spend, and refunds the unspent remainder to `free`. Tokens and usd are separate
|
|
3272
|
-
* channels (`LoopTokenUsage` has no `usd`); iterations are conserved alongside them.
|
|
3273
|
-
*
|
|
3274
|
-
* Pure and deterministic: `now()` is injected, there is no I/O, and no wall-clock or
|
|
3275
|
-
* RNG read. A `reserve`/`reconcile` ticket is single-use (fail-loud on double or
|
|
3276
|
-
* unknown reconcile) so a child can never refund twice.
|
|
3277
|
-
*
|
|
3278
|
-
* @experimental
|
|
3279
|
-
*/
|
|
3280
|
-
|
|
3281
|
-
/** Opaque, single-use reservation handle returned by `reserve` and consumed by
|
|
3282
|
-
* `reconcile`. Carries the reserved ceilings so reconciliation needs no lookup. */
|
|
3283
|
-
interface ReservationTicket {
|
|
3284
|
-
readonly id: number;
|
|
3285
|
-
readonly reserved: {
|
|
3286
|
-
readonly tokens: number;
|
|
3287
|
-
readonly usd: number;
|
|
3288
|
-
readonly iterations: number;
|
|
3289
|
-
};
|
|
3290
|
-
}
|
|
3291
|
-
/** Post-reservation pool readout — the shape `Scope.budget` exposes. `tokensLeft`,
|
|
3292
|
-
* `usdLeft`, and `reservedTokens` reflect committed-but-unsettled reservations;
|
|
3293
|
-
* `deadlineMs` is the ABSOLUTE wall-clock deadline (0 when the root set none).
|
|
3294
|
-
* `usdCapped` distinguishes a real `usdLeft <= 0` exhaustion from an uncapped pool (which always
|
|
3295
|
-
* reads `usdLeft: 0`) — the in-loop guard needs it to bound a usd-capped driver. */
|
|
3296
|
-
type BudgetReadout = Readonly<{
|
|
3297
|
-
tokensLeft: number;
|
|
3298
|
-
usdLeft: number;
|
|
3299
|
-
usdCapped: boolean;
|
|
3300
|
-
deadlineMs: number;
|
|
3301
|
-
reservedTokens: number;
|
|
3302
|
-
}>;
|
|
3303
|
-
interface BudgetPool {
|
|
3304
|
-
/**
|
|
3305
|
-
* Atomically reserve a child's full ceiling from the free balance. Fails closed
|
|
3306
|
-
* ({ ok: false }) when the pool can't cover tokens, usd, or iterations — the
|
|
3307
|
-
* caller inspects `ok` before `ticket`.
|
|
3308
|
-
*/
|
|
3309
|
-
reserve(b: Budget): {
|
|
3310
|
-
ok: true;
|
|
3311
|
-
ticket: ReservationTicket;
|
|
3312
|
-
} | {
|
|
3313
|
-
ok: false;
|
|
3314
|
-
reason: 'budget-exhausted';
|
|
3315
|
-
};
|
|
3316
|
-
/**
|
|
3317
|
-
* Release a reservation: commit the actual `spent`, refund the unspent remainder
|
|
3318
|
-
* to the free pool. Throws on an unknown or already-reconciled ticket (fail loud —
|
|
3319
|
-
* a double refund would silently break conservation).
|
|
3320
|
-
*/
|
|
3321
|
-
reconcile(ticket: ReservationTicket, spent: Spend): void;
|
|
3322
|
-
/** Fold a normalized `UsageEvent` stream (or array) into a `Spend`. Tokens via
|
|
3323
|
-
* `addTokenUsage`, usd on its own channel, iterations from `'iteration'` events.
|
|
3324
|
-
* `ms` is left zero — wall-clock duration is the caller's to record, not the pool's. */
|
|
3325
|
-
spendFrom(events: AsyncIterable<UsageEvent> | UsageEvent[]): Promise<Spend>;
|
|
3326
|
-
/** The current readout, reflecting all outstanding reservations. */
|
|
3327
|
-
readout(): BudgetReadout;
|
|
3328
|
-
/**
|
|
3329
|
-
* Record OBSERVED spend that did NOT go through reserve/reconcile — the driver's OWN inference
|
|
3330
|
-
* (its chat turns), which is real compute but not a spawned child. A direct `free → committed`
|
|
3331
|
-
* debit, so `total ≡ free + reserved + committed` is preserved: equal-k counts the driver's
|
|
3332
|
-
* tokens and the in-loop budget guard (`readout().tokensLeft`) sees them. `free` may go negative
|
|
3333
|
-
* when a run overspends — that is honest (the readout then signals exhaustion). It never throws:
|
|
3334
|
-
* the spend already happened, so accounting records reality; the in-loop guard prevents MORE.
|
|
3335
|
-
* The DURABLE record is the journal's `metered` event (written by `Scope.meter`); this debit
|
|
3336
|
-
* only makes the live `readout()` reflect driver inference for the in-loop guard.
|
|
3337
|
-
*/
|
|
3338
|
-
observe(spend: Spend): void;
|
|
3339
|
-
/** Fail loud if any reservation is still open — the conserved-pool leak detector. Called at the
|
|
3340
|
-
* supervisor's join barrier: once every child has settled, no ticket may remain (a leaked
|
|
3341
|
-
* reservation would silently break `total ≡ free + reserved + committed`). */
|
|
3342
|
-
assertNoOpenTickets(): void;
|
|
3343
|
-
}
|
|
3344
|
-
/** Fold a normalized `UsageEvent` array into a `Spend`. Tokens and usd are separate
|
|
3345
|
-
* channels; iterations come from `'iteration'` events. Pure; `ms` stays zero (the
|
|
3346
|
-
* pool does not read wall-clock). */
|
|
3347
|
-
declare function spendFromUsageEvents(events: UsageEvent[]): Spend;
|
|
3348
|
-
/**
|
|
3349
|
-
* Create a conserved reservation pool from a root `Budget`. `now()` is injected so the
|
|
3350
|
-
* deadline readout is deterministic; defaults to `Date.now` for non-test callers. The
|
|
3351
|
-
* absolute deadline is fixed at construction (`now() + budget.deadlineMs`) so the
|
|
3352
|
-
* readout's `deadlineMs` is a stable wall-clock instant, not a shrinking remainder.
|
|
3353
|
-
*/
|
|
3354
|
-
declare function createBudgetPool(root: Budget, now?: () => number): BudgetPool;
|
|
3355
|
-
|
|
3356
|
-
/**
|
|
3357
|
-
*
|
|
3358
|
-
* `driverAgent` — the driver's BRAIN.
|
|
3359
|
-
*
|
|
3360
|
-
* The recursive driver-executor (`driver-executor.ts`) runs a driver `Agent.act` inside a
|
|
3361
|
-
* nested `Scope`; this is the intelligent `act`: it mounts the coordination MCP verbs
|
|
3362
|
-
* (`createCoordinationTools`) over that scope and runs an LLM tool-loop, so the driver
|
|
3363
|
-
* REASONS — spawn / observe / steer / await / stop — about how to drive its children,
|
|
3364
|
-
* instead of running a fixed script. Each turn: ask the driver LLM for tool calls, run them
|
|
3365
|
-
* against the live scope, fold the results back, repeat until the driver stops (no tool
|
|
3366
|
-
* calls) or the turn cap forces a keep-best finalize.
|
|
3367
|
-
*
|
|
3368
|
-
* Recursion composes through `makeWorkerAgent`: `spawn_agent` resolves a `profile` to a
|
|
3369
|
-
* worker LEAF or — when the profile is a driver — a `driverChild` wrapping ANOTHER
|
|
3370
|
-
* `driverAgent` over its own nested scope (see `driver-executor.ts`). So an agent
|
|
3371
|
-
* drives an agent that drives an agent, each an LLM tool-loop, all on one conserved-budget
|
|
3372
|
-
* tree.
|
|
3373
|
-
*
|
|
3374
|
-
* Two seams are INJECTED so the loop runs offline with no creds and stays decoupled:
|
|
3375
|
-
* - `brain` (`ToolLoopChat`) — one driver-LLM turn over the canonical tool-loop seam; a test
|
|
3376
|
-
* drives a scripted mock, production passes the router's tool-calling (`routerBrain`), a
|
|
3377
|
-
* sandboxed harness drives the verbs as MCP tools. The same seam every tool-loop uses.
|
|
3378
|
-
* - `systemPrompt` — the driver's stance (the agent-eval worker-driver prompt / the prompt
|
|
3379
|
-
* generator). Injected, never hardcoded — the prompt is a pluggable role.
|
|
3380
|
-
*
|
|
3381
|
-
* @experimental
|
|
3382
|
-
*/
|
|
3383
|
-
|
|
3384
|
-
interface DriverAgentOptions {
|
|
3385
|
-
readonly name: string;
|
|
3386
|
-
/** The driver-LLM seam — ONE inference turn over the conversation + the coordination tool specs
|
|
3387
|
-
* (the canonical `ToolLoopChat`): a scripted mock offline, the router's tool-calling in
|
|
3388
|
-
* production, or a sandboxed harness. The same seam every tool-loop uses; no bespoke shape. */
|
|
3389
|
-
readonly brain: ToolLoopChat;
|
|
3390
|
-
/** Shared blob store — `observe_agent` reads settled outputs through it. */
|
|
3391
|
-
readonly blobs: ResultBlobStore;
|
|
3392
|
-
/** Resolve a spawned `profile` to a worker LEAF or a driver child (the recursion seam). */
|
|
3393
|
-
readonly makeWorkerAgent: MakeWorkerAgent;
|
|
3394
|
-
/** Per-child budget reserved from the conserved pool on each spawn. */
|
|
3395
|
-
readonly perWorker: Budget;
|
|
3396
|
-
/** Hard cap on simultaneously-LIVE workers — `spawn_agent` fails closed once this many are in
|
|
3397
|
-
* flight (a concurrency fence on top of the conserved-pool fence). Omit/`<= 0` = no cap. */
|
|
3398
|
-
readonly maxLiveWorkers?: number;
|
|
3399
|
-
/** The analyst lenses available to the driver. Required for `analyzeOnSettle` (and `run_analyst`).
|
|
3400
|
-
* Unset → no analyst feed (status quo: the driver gets settled outputs, no findings). */
|
|
3401
|
-
readonly analysts?: AnalystRegistry;
|
|
3402
|
-
/** Analyst kind ids run AUTOMATICALLY when a worker settles `done` — each result re-enters as a
|
|
3403
|
-
* `finding` the driver pulls and composes its next steer from. The UP-leg of the self-improving
|
|
3404
|
-
* loop. Omit/empty = no auto-analysis (status quo). Requires `analysts`. */
|
|
3405
|
-
readonly analyzeOnSettle?: ReadonlyArray<string>;
|
|
3406
|
-
/** Run the ONLINE detector panel over each worker's LIVE tool trace and raise a `finding` the
|
|
3407
|
-
* moment it loops/error-storms — mid-run evidence to steer on, not a settle-time post-mortem.
|
|
3408
|
-
* Omit = no online watching. */
|
|
3409
|
-
readonly watchWorkers?: WorkerWatchOptions;
|
|
3410
|
-
/** Idle time after which `observe_agent` reports a worker as stalled (a derived read; nothing is
|
|
3411
|
-
* killed). Omit = the runtime default. */
|
|
3412
|
-
readonly stallAfterMs?: number;
|
|
3413
|
-
/** The driver's stance — a string, or built from the task (the worker-driver prompt /
|
|
3414
|
-
* the generator). INJECTED so the prompt is a pluggable, optimizable role. */
|
|
3415
|
-
readonly systemPrompt: string | ((task: unknown) => string);
|
|
3416
|
-
/** WORK tools the driver may call DIRECTLY (alongside the coordination verbs) — so the driver is
|
|
3417
|
-
* not a pure manager but a full agent that can ACT (do simple work itself) OR SPAWN (delegate).
|
|
3418
|
-
* Each is a router tool spec; their names must not collide with the coordination verbs. Pair with
|
|
3419
|
-
* `executeExtraTool`. Unset → coordination-only (the prior behavior). */
|
|
3420
|
-
readonly extraTools?: ReadonlyArray<{
|
|
3421
|
-
readonly name: string;
|
|
3422
|
-
readonly description?: string;
|
|
3423
|
-
readonly parameters: Record<string, unknown>;
|
|
3424
|
-
}>;
|
|
3425
|
-
/** Runs an `extraTools` call. Returns a string result, or null/undefined to signal "not handled"
|
|
3426
|
-
* so the call falls through to the coordination dispatch. Required iff `extraTools` is set. */
|
|
3427
|
-
readonly executeExtraTool?: (name: string, args: Record<string, unknown>) => Promise<string | null | undefined>;
|
|
3428
|
-
/** Max driver turns before the loop force-finalizes on the best settled child. Default 16.
|
|
3429
|
-
* `0` lifts the turn-COUNT cap: the loop is bounded instead by the conserved budget pool,
|
|
3430
|
-
* an absolute deadline, the driver's own stop, and abort (checked in-loop). A finite
|
|
3431
|
-
* anti-runaway tripwire still guards a degenerate driver that loops on a no-spawn tool. */
|
|
3432
|
-
readonly maxTurns?: number;
|
|
3433
|
-
/** Injected clock for the in-loop absolute-deadline guard — keeps the deadline check
|
|
3434
|
-
* deterministic in tests. Defaults to `Date.now`. */
|
|
3435
|
-
readonly now?: () => number;
|
|
3436
|
-
/**
|
|
3437
|
-
* PROGRESS-derived stop (mechanic D). Today a run ends on a ceiling — iterations, tokens,
|
|
3438
|
-
* dollars, deadline, turn cap — which answers "may it continue?" and never "is it still getting
|
|
3439
|
-
* anywhere?". A stop rule reads the run's own progress (best-so-far over settled work, time
|
|
3440
|
-
* since the last settle, the live worker feed) and ends a run that has stopped learning BEFORE
|
|
3441
|
-
* it exhausts a budget.
|
|
3442
|
-
*
|
|
3443
|
-
* Composes with, and can never override, the hard guards: `poolStarved` / `deadlinePassed` /
|
|
3444
|
-
* abort / the driver's own stop are evaluated first, so a rule can only ADD a stop.
|
|
3445
|
-
*
|
|
3446
|
-
* THRESHOLDS are the caller's judgment, not this module's — build the rule with
|
|
3447
|
-
* `plateau({window, minDelta})` / `noProgressFor({...})` / `allWorkersStalled({...})` from
|
|
3448
|
-
* `supervise/stop-rules`. Omit ⇒ ceilings only (unchanged behavior).
|
|
3449
|
-
*/
|
|
3450
|
-
readonly stopRule?: StopRule;
|
|
3451
|
-
/** Called once with the rule's reason when a `stopRule` ends the run — so a caller can record
|
|
3452
|
-
* WHY a run stopped early instead of inferring it from an unexhausted budget. */
|
|
3453
|
-
readonly onProgressStop?: (reason: string) => void;
|
|
3454
|
-
/** Give the driver brain a chapter-lifecycle on its OWN context window. The LLM-brain front doors
|
|
3455
|
-
* lose to a dumb-Ralph respawn because the brain re-bills its whole coordination transcript every
|
|
3456
|
-
* turn — the same context overflow a single steered agent suffers, one level up. With this set,
|
|
3457
|
-
* once the brain's running conversation exceeds `thresholdTokens` it distills the accumulated
|
|
3458
|
-
* history to a compact progress note and continues fresh: the supervisor analog of respawning
|
|
3459
|
-
* against external tracking state, except the live `Scope` roster IS the durable state. Default
|
|
3460
|
-
* off (no behavior change). `distill` defaults to a self-summary authored by the brain combined
|
|
3461
|
-
* with the factual settled-worker roster; override to supply your own. */
|
|
3462
|
-
readonly compaction?: ToolLoopCompactionOptions;
|
|
3463
|
-
}
|
|
3464
|
-
/**
|
|
3465
|
-
* Build the intelligent recursive driver. Its `act` is the LLM tool-loop; spawn it as a
|
|
3466
|
-
* `driverChild` (`driver-executor.ts`) to run it inside a nested scope, recursively.
|
|
3467
|
-
*/
|
|
3468
|
-
declare function driverAgent(opts: DriverAgentOptions): Agent<unknown, unknown>;
|
|
3469
|
-
/** Keep-best finalize under the completion-oracle: return the highest-scoring DELIVERED child's
|
|
3470
|
-
* output (settled `done` AND `valid` — its deliverable check passed). Returns undefined when no
|
|
3471
|
-
* child delivered — an honest "the driver produced nothing", never a high-scoring result that
|
|
3472
|
-
* ran without passing its check (Foreman's 0/18 lesson). `valid` is the single delivery signal,
|
|
3473
|
-
* matching `defaultSelectWinner`'s valid-first rule; the oracle just doesn't fall back to an
|
|
3474
|
-
* unchecked best-effort. */
|
|
3475
|
-
declare function finalizeBestDelivered(settled: ReadonlyArray<{
|
|
3476
|
-
status: string;
|
|
3477
|
-
score?: number;
|
|
3478
|
-
valid?: boolean;
|
|
3479
|
-
outRef?: string;
|
|
3480
|
-
}>, blobs: ResultBlobStore): Promise<unknown>;
|
|
3481
|
-
|
|
3482
|
-
/**
|
|
3483
|
-
*
|
|
3484
|
-
* Serve the coordination verbs (spawn_agent / await_event / observe_agent / steer_agent / stop)
|
|
3485
|
-
* as a real HTTP MCP server over a LIVE `Scope`. This is the keystone that lets a coding-harness
|
|
3486
|
-
* agent (opencode via the cli-bridge, claude-code, codex) BE the supervisor: it mounts this MCP
|
|
3487
|
-
* (`mcp.mcpServers.coordination`) and calls `spawn_agent` as a native tool, which lands on
|
|
3488
|
-
* `Scope.spawn` — a real box driving real boxes, not emulated function-tools.
|
|
3489
|
-
*
|
|
3490
|
-
* Coordination vs DELEGATION (`../../mcp/delegates.ts`): coordination SPAWNS workers in a CHOSEN
|
|
3491
|
-
* backend (`createExecutor({ backend })` — sandbox OR cli-bridge) and live-drives them — observe /
|
|
3492
|
-
* steer / resume, recursive sub-drivers, one conserved budget. To instead delegate a coding task
|
|
3493
|
-
* INSIDE the agent's OWN sandbox (a durable fire-and-poll job that survives an MCP restart), use the
|
|
3494
|
-
* delegation MCP. Coordination is the live, cross-backend supervisor; delegation is own-sandbox async.
|
|
3495
|
-
*
|
|
3496
|
-
* Transport: JSON-RPC over HTTP POST (the MCP streamable-HTTP shape — `application/json` for a
|
|
3497
|
-
* single response). The server is created INSIDE an agent's `act(task, scope)` so it fronts that
|
|
3498
|
-
* agent's live scope; tear it down when the act returns.
|
|
3499
|
-
*
|
|
3500
|
-
* @experimental
|
|
3501
|
-
*/
|
|
3502
|
-
|
|
3503
|
-
interface CoordinationMcpHandle {
|
|
3504
|
-
/** The URL an in-box harness mounts as `mcp.mcpServers.coordination.url`. */
|
|
3505
|
-
readonly url: string;
|
|
3506
|
-
readonly port: number;
|
|
3507
|
-
/** The coordination tools' settled-worker ledger (for the driver's finalize). */
|
|
3508
|
-
settled(): ReadonlyArray<{
|
|
3509
|
-
status: string;
|
|
3510
|
-
score?: number;
|
|
3511
|
-
valid?: boolean;
|
|
3512
|
-
outRef?: string;
|
|
3513
|
-
}>;
|
|
3514
|
-
/** Post-loop drain of already-settled, unpulled children into the ledger — call before reading
|
|
3515
|
-
* `settled()` for a finalize, so a delivered child the harness never awaited is not lost. */
|
|
3516
|
-
drainResolved: CoordinationTools['drainResolved'];
|
|
3517
|
-
isStopped(): boolean;
|
|
3518
|
-
/** The full ordered bus-event log — observability audit + replay trail. */
|
|
3519
|
-
history: CoordinationTools['history'];
|
|
3520
|
-
/** Bus throughput counters for live dashboards. */
|
|
3521
|
-
stats: CoordinationTools['stats'];
|
|
3522
|
-
/** Raise a `finding` on the bus from an online detector watching a worker's live pipe. */
|
|
3523
|
-
raiseFinding: CoordinationTools['raiseFinding'];
|
|
3524
|
-
close(): Promise<void>;
|
|
3525
|
-
}
|
|
3526
|
-
/** Stand up the coordination MCP over a live scope. The HOST address is `127.0.0.1` (the bridge runs
|
|
3527
|
-
* opencode locally, same host); pass `host` to bind elsewhere when the harness is remote. */
|
|
3528
|
-
declare function serveCoordinationMcp(opts: {
|
|
3529
|
-
scope: Scope<unknown>;
|
|
3530
|
-
blobs: ResultBlobStore;
|
|
3531
|
-
makeWorkerAgent: MakeWorkerAgent;
|
|
3532
|
-
perWorker: Budget;
|
|
3533
|
-
/** Hard cap on simultaneously-LIVE workers — `spawn_agent` fails closed once this many are in
|
|
3534
|
-
* flight (a concurrency fence on top of the conserved-pool fence). Omit/`<= 0` = no cap. */
|
|
3535
|
-
maxLiveWorkers?: number;
|
|
3536
|
-
/** Max wall-clock ms a single `await_event` may block before returning a re-pollable
|
|
3537
|
-
* `{ pending, live }` snapshot instead of erroring on the client's request timeout. Omit =
|
|
3538
|
-
* {@link DEFAULT_AWAIT_EVENT_TIMEOUT_MS}; `<= 0` = prior unbounded block (in-process only). */
|
|
3539
|
-
awaitTimeoutMs?: number;
|
|
3540
|
-
port?: number;
|
|
3541
|
-
host?: string;
|
|
3542
|
-
/** Trace-analyst lenses the driver can run (`run_analyst`) or auto-fire on settle. */
|
|
3543
|
-
analysts?: AnalystRegistry;
|
|
3544
|
-
/** Analyst kinds to auto-run when a worker settles `done` — findings flow up the bus. */
|
|
3545
|
-
analyzeOnSettle?: ReadonlyArray<string>;
|
|
3546
|
-
/** Run the ONLINE detector panel over each worker's live tool trace (raises `finding` events). */
|
|
3547
|
-
watchWorkers?: WorkerWatchOptions;
|
|
3548
|
-
/** Idle time after which `observe_agent` reports a worker as stalled. */
|
|
3549
|
-
stallAfterMs?: number;
|
|
3550
|
-
/** Pass-through subscriber for every bus event (settled / question / finding). */
|
|
3551
|
-
onEvent?: (event: CoordinationEvent) => void | Promise<void>;
|
|
3552
|
-
questionPolicy?: QuestionPolicy;
|
|
3553
|
-
}): Promise<CoordinationMcpHandle>;
|
|
3554
|
-
|
|
3555
|
-
/**
|
|
3556
|
-
*
|
|
3557
|
-
* `delegate` — the one generic delegation verb. You hand it an INTENT (what you want done) and it
|
|
3558
|
-
* hands that intent to a default AUTHORING supervisor: a router-brained supervisor whose standing
|
|
3559
|
-
* instruction is `supervisorInstructions()` (the authoring-agent-profiles skill). The supervisor
|
|
3560
|
-
* DECOMPOSES the intent and AUTHORS the worker profile it needs per sub-task — there is NO hardcoded
|
|
3561
|
-
* coder/researcher profile here. That is the whole point: `delegate('fix the failing test', …)` and
|
|
3562
|
-
* `delegate('research X and cite sources', …)` route through the SAME front door; the supervisor
|
|
3563
|
-
* writes a code-shaped or research-shaped worker on its own.
|
|
3564
|
-
*
|
|
3565
|
-
* It is a thin wrapper over `supervise()` — the one front door — so the conserved-budget pool, the
|
|
3566
|
-
* completion oracle (`deliverable`), the coordination toolbox, and equal-compute accounting all come
|
|
3567
|
-
* for free; nothing is hand-rolled. The result is `supervise()`'s `SupervisedResult` returned
|
|
3568
|
-
* UNCHANGED, so its `spentTotal` (`{ iterations, tokens, usd, ms }`) rides straight back to the
|
|
3569
|
-
* caller on BOTH paths — a `winner` carries the delivered worker's spend, a `no-winner` carries the
|
|
3570
|
-
* spend incurred before it failed. That cost channel means a `delegate()` caller always learns what
|
|
3571
|
-
* the delegation actually spent.
|
|
3572
|
-
*
|
|
3573
|
-
* @experimental
|
|
3574
|
-
*/
|
|
3575
|
-
|
|
3576
|
-
/** The conserved pool a `delegate()` call applies when the caller does not pass its own `budget`.
|
|
3577
|
-
* A modest token ceiling + a small iteration ceiling — generous enough for a few-worker decompose,
|
|
3578
|
-
* bounded enough that an unsupervised intent cannot run away. Callers override via `opts.budget`. */
|
|
3579
|
-
declare const defaultDelegateBudget: Budget;
|
|
3580
|
-
/** Inputs to {@link delegate}. The intent is the first positional arg; everything here is optional
|
|
3581
|
-
* with sensible defaults, so the common call is `delegate(intent, { backend, router })`. */
|
|
3582
|
-
interface DelegateOptions<Out = unknown> {
|
|
3583
|
-
/** The completion oracle (settled ⟺ delivered) the authored workers settle against. Strongly
|
|
3584
|
-
* recommended — without it the supervisor trusts a worker's self-report. For a code intent,
|
|
3585
|
-
* `patchDelivered()` is the canonical example; for a free-form answer, a content check. */
|
|
3586
|
-
readonly deliverable?: DeliverableSpec<Out>;
|
|
3587
|
-
/** WHERE the authored workers run — the worker-execution backend (`router-tools` / `sandbox` /
|
|
3588
|
-
* `cli-worktree` / …). The supervisor authors the worker PROFILE; this is the substrate it runs
|
|
3589
|
-
* on. Provide this OR `makeWorkerAgent`-style wiring through `supervise()` is unavailable. */
|
|
3590
|
-
readonly backend?: ExecutorConfig;
|
|
3591
|
-
/** The conserved compute pool for the whole delegation. Defaults to {@link defaultDelegateBudget}. */
|
|
3592
|
-
readonly budget?: Budget;
|
|
3593
|
-
/** The model the supervisor BRAIN runs on (the router model). The brain must tool-call
|
|
3594
|
-
* (`spawn_agent` / `await_event`), so a delegator model, not a hidden-reasoning model. */
|
|
3595
|
-
readonly model?: string;
|
|
3596
|
-
/** The supervisor brain's router substrate. REQUIRED for the default router-brained supervisor
|
|
3597
|
-
* (the brain is resolved from this), unless a test injects `brain` directly. `model` overrides
|
|
3598
|
-
* `router.model`. (Design delta vs the bare `supervise()` profile: the brain needs a router.) */
|
|
3599
|
-
readonly router?: RouterConfig;
|
|
3600
|
-
/** Inject the supervisor brain directly (tests / advanced) instead of resolving it from `router`. */
|
|
3601
|
-
readonly brain?: ToolLoopChat;
|
|
3602
|
-
/** Override the default authoring-supervisor profile (name / extra system-prompt stance). The
|
|
3603
|
-
* default already carries the authoring skill; override only to add a goal or rename. */
|
|
3604
|
-
readonly supervisor?: Partial<Pick<SupervisorProfile, 'name' | 'systemPrompt'>>;
|
|
3605
|
-
/** Restrict the run to this subset of models (forwarded to `supervise()`). */
|
|
3606
|
-
readonly allowedModels?: readonly string[];
|
|
3607
|
-
readonly runId?: string;
|
|
3608
|
-
}
|
|
3609
|
-
/**
|
|
3610
|
-
* Delegate an INTENT to a default authoring supervisor and return its `SupervisedResult` unchanged.
|
|
3611
|
-
*
|
|
3612
|
-
* The supervisor authors + spawns whatever worker the intent needs over the conserved-budget pool;
|
|
3613
|
-
* `result.spentTotal` reports what the whole delegation actually cost. A `winner` result carries the
|
|
3614
|
-
* authored worker's delivered output; a `no-winner` result names why (never a fabricated success).
|
|
3615
|
-
*/
|
|
3616
|
-
declare function delegate<Out = unknown>(intent: string, opts?: DelegateOptions<Out>): Promise<SupervisedResult<Out>>;
|
|
3617
|
-
|
|
3618
|
-
/**
|
|
3619
|
-
*
|
|
3620
|
-
* `rollingDispatch` — the REFILLING dispatch policy over a `Scope`. It keeps `width` children in
|
|
3621
|
-
* flight and admits the next queued unit of work the instant one settles, instead of waiting for
|
|
3622
|
-
* a whole round to drain (`fanout`) or for a driver to decide again (`spawn → await → spawn`).
|
|
3623
|
-
*
|
|
3624
|
-
* The gap it closes: `fanout` opens every item at once and then drains — one round, no refill —
|
|
3625
|
-
* and a driver brain's manual loop opens one worker per model turn. Between those two there was
|
|
3626
|
-
* nothing that holds N slots full, which is why a 5-worker run can peak at 2 live workers and sit
|
|
3627
|
-
* ~50% idle. This is a policy over the EXISTING `Scope.spawn` / `Scope.next` primitives: it adds
|
|
3628
|
-
* no second admission path, so the conserved budget pool stays the only fence on total work and
|
|
3629
|
-
* `width` is only a fence on simultaneous work.
|
|
3630
|
-
*
|
|
3631
|
-
* Fail-closed by construction: an admission rejection (`budget-exhausted` / `depth-exceeded`) is
|
|
3632
|
-
* recorded and STOPS further admission — the loop then drains what is already live and returns.
|
|
3633
|
-
* It never retries a rejected spawn against the same pool, and it never spawns past `width`.
|
|
3634
|
-
*
|
|
3635
|
-
* ── The concurrency-cap ledger (read this before adding a fourth cap) ──────────────────────────
|
|
3636
|
-
*
|
|
3637
|
-
* Three unrelated caps bound "how much runs at once" in this stack, at three different layers.
|
|
3638
|
-
* They are NOT aware of each other, and the smallest one silently wins:
|
|
3639
|
-
*
|
|
3640
|
-
* 1. `CoordinationToolsOptions.maxLiveWorkers` (`src/mcp/tools/coordination.ts`) — supervisor
|
|
3641
|
-
* level. How many workers may be spawned-but-not-settled at once; `spawn_agent` fails closed
|
|
3642
|
-
* with `error: 'max-live-workers'` past it. Unset by default ⇒ NO cap at this layer.
|
|
3643
|
-
* 2. `SandboxLineage`'s `maxConcurrency` / `DEFAULT_FORK_CONCURRENCY = 4`
|
|
3644
|
-
* (`src/runtime/sandbox-lineage.ts`) — kernel level. How many BOXES one `runAgentRounds` fork wave
|
|
3645
|
-
* provisions at once. It bounds a single leaf's fanout, not the supervisor's worker count.
|
|
3646
|
-
* 3. A host's own live-box governor (e.g. loops' `ComputeGovernor`, `maxSandboxes = 4`) — fleet
|
|
3647
|
-
* level. How many sandboxes may exist across the whole host process.
|
|
3648
|
-
*
|
|
3649
|
-
* The honest effective limit on simultaneous WORKERS is the minimum of the caps that apply to the
|
|
3650
|
-
* worker layer — (1) and (3). (2) is a different unit (boxes per fork wave inside one leaf) and
|
|
3651
|
-
* must not be min'd into it, or a 4-way fork inside one worker reads as a 4-worker ceiling.
|
|
3652
|
-
* `effectiveConcurrency` computes that minimum in one place so a host derives ONE number and
|
|
3653
|
-
* passes it to BOTH `maxLiveWorkers` and this dispatcher's `width`, instead of leaving a fleet
|
|
3654
|
-
* governor of 4 and an unset worker fence as unrelated numbers.
|
|
3655
|
-
*
|
|
3656
|
-
* ── Why this is not a copy of the kernel's batch loop ──────────────────────────────────────────
|
|
3657
|
-
*
|
|
3658
|
-
* `runBatch` (`src/runtime/run-loop.ts`) runs the same hold-N-slots-full shape over the KERNEL's
|
|
3659
|
-
* substrate: bare promises in a `Set`, raced with `Promise.race`, bounded by `maxConcurrency`.
|
|
3660
|
-
* This one runs it over the SUPERVISOR's substrate — `Scope.spawn`'s atomic reservation against
|
|
3661
|
-
* the conserved pool, `Scope.next`'s journaled settlement cursor, and the blob store behind each
|
|
3662
|
-
* result. Neither can be expressed in the other's terms without dragging its whole substrate
|
|
3663
|
-
* along: the kernel loop has no budget to reserve and no cursor to order settlements by, and this
|
|
3664
|
-
* one cannot race raw promises because a settlement is only real once it is journaled. The shape
|
|
3665
|
-
* repeating across the two deliberate layers is resonance, not duplication — do not "unify" them
|
|
3666
|
-
* into a shared helper that would have to know about both.
|
|
3667
|
-
*
|
|
3668
|
-
* @experimental
|
|
3669
|
-
*/
|
|
3670
|
-
|
|
3671
|
-
/** One unit of queued work: the agent to run, its task, and the spawn options (budget + label).
|
|
3672
|
-
* `nextUnit` mints these lazily so a queue can be generated, re-ordered, or grown while the
|
|
3673
|
-
* dispatcher runs. */
|
|
3674
|
-
interface DispatchUnit<Out> {
|
|
3675
|
-
readonly agent: Agent<unknown, Out>;
|
|
3676
|
-
readonly task: unknown;
|
|
3677
|
-
readonly opts: SpawnOpts;
|
|
3678
|
-
}
|
|
3679
|
-
/** Why the dispatcher stopped admitting work. `drained` = the queue ran dry (the ordinary end);
|
|
3680
|
-
* `not-admitted` = the conserved pool or the depth ceiling refused a spawn; `stopped` = the
|
|
3681
|
-
* caller's `shouldStop` returned true; `aborted` = the scope's signal fired. */
|
|
3682
|
-
type DispatchStopReason = 'drained' | 'not-admitted' | 'stopped' | 'aborted';
|
|
3683
|
-
interface RollingDispatchOptions<Out> {
|
|
3684
|
-
/**
|
|
3685
|
-
* How many children to hold in flight. Must be a positive integer. This is a SIMULTANEITY fence
|
|
3686
|
-
* only — the conserved pool still bounds total work, and a `width` larger than the pool can
|
|
3687
|
-
* afford simply hits `not-admitted` sooner. Derive it with `effectiveConcurrency` when the host
|
|
3688
|
-
* also runs a fleet-level box governor.
|
|
3689
|
-
*/
|
|
3690
|
-
readonly width: number;
|
|
3691
|
-
/**
|
|
3692
|
-
* Produce the next unit of work, or `undefined` when the queue is dry. Called only when a slot
|
|
3693
|
-
* is free, so a caller may compute the next unit from what has already settled (the point of a
|
|
3694
|
-
* refilling dispatcher: the queue is allowed to react). Never called after a stop.
|
|
3695
|
-
*/
|
|
3696
|
-
nextUnit(): DispatchUnit<Out> | undefined | Promise<DispatchUnit<Out> | undefined>;
|
|
3697
|
-
/**
|
|
3698
|
-
* Called once per settlement, in cursor order, BEFORE the freed slot is refilled — so an
|
|
3699
|
-
* `onSettled` that appends to the caller's queue is visible to the very next `nextUnit`.
|
|
3700
|
-
*/
|
|
3701
|
-
onSettled?(settled: Settled<Out>): void | Promise<void>;
|
|
3702
|
-
/**
|
|
3703
|
-
* Consulted before each admission. `true` stops admitting; the already-live children are still
|
|
3704
|
-
* drained to completion (no orphan, no lost settlement). Use it for a progress/plateau rule.
|
|
3705
|
-
*/
|
|
3706
|
-
shouldStop?(): boolean;
|
|
3707
|
-
}
|
|
3708
|
-
interface DispatchReport<Out> {
|
|
3709
|
-
/** Every settlement, in the order `scope.next()` yielded them. */
|
|
3710
|
-
readonly settled: ReadonlyArray<Settled<Out>>;
|
|
3711
|
-
/** How many children this dispatcher admitted. */
|
|
3712
|
-
readonly admitted: number;
|
|
3713
|
-
/** Admission rejections, in order — `label: reason`. Non-empty ⇒ the pool or depth fenced. */
|
|
3714
|
-
readonly rejected: ReadonlyArray<string>;
|
|
3715
|
-
readonly stopReason: DispatchStopReason;
|
|
3716
|
-
/** The highest simultaneous live count actually reached — the number to compare against
|
|
3717
|
-
* `width` when asking "did the slots really stay full?" */
|
|
3718
|
-
readonly peakLive: number;
|
|
3719
|
-
}
|
|
3720
|
-
/**
|
|
3721
|
-
* Run the refilling dispatch loop over `scope` until the queue is dry (or a stop fires) and every
|
|
3722
|
-
* admitted child has settled. Returns the settlements in cursor order plus the admission ledger.
|
|
3723
|
-
*
|
|
3724
|
-
* The loop is: fill free slots from `nextUnit` → `await scope.next()` → deliver the settlement →
|
|
3725
|
-
* refill → repeat. Because the refill happens immediately after each settlement rather than after
|
|
3726
|
-
* a whole round, a slow child never idles the other slots.
|
|
3727
|
-
*/
|
|
3728
|
-
declare function rollingDispatch<Out>(scope: Scope<Out>, opts: RollingDispatchOptions<Out>): Promise<DispatchReport<Out>>;
|
|
3729
|
-
/**
|
|
3730
|
-
* Free worker slots under a simultaneity cap: `cap - live`, floored at 0, or `null` when there is
|
|
3731
|
-
* no cap (the conserved pool is then the only fence and "free slots" is not a finite number).
|
|
3732
|
-
* The one place the answer is computed, so the driver-facing tool payload and a dispatcher agree.
|
|
3733
|
-
*/
|
|
3734
|
-
declare function freeSlots(liveCount: number, cap: number | undefined): number | null;
|
|
3735
|
-
/** The caps a host can set on simultaneous work. See the ledger in this module's header for what
|
|
3736
|
-
* each one actually bounds. */
|
|
3737
|
-
interface ConcurrencyCaps {
|
|
3738
|
-
/** Supervisor level: max spawned-but-unsettled workers. */
|
|
3739
|
-
readonly maxLiveWorkers?: number;
|
|
3740
|
-
/** Fleet level: max live sandboxes/boxes across the host process (a `ComputeGovernor`-style
|
|
3741
|
-
* cap). Applies to the worker layer, so it participates in the minimum. */
|
|
3742
|
-
readonly maxSandboxes?: number;
|
|
3743
|
-
}
|
|
3744
|
-
/**
|
|
3745
|
-
* The ONE honest effective limit on simultaneous workers: the minimum of the caps that actually
|
|
3746
|
-
* bound the worker layer. Ignores unset/non-positive caps; returns `undefined` when no cap applies
|
|
3747
|
-
* (uncapped — the conserved pool remains the only fence).
|
|
3748
|
-
*
|
|
3749
|
-
* Deliberately does NOT fold in `SandboxLineage`'s fork concurrency: that bounds boxes inside ONE
|
|
3750
|
-
* leaf's fork wave, a different unit. Folding it in would report a 4-worker ceiling for what is
|
|
3751
|
-
* really a 4-box fanout inside a single worker.
|
|
3752
|
-
*
|
|
3753
|
-
* Use it once, at the top of a run, and pass the result to BOTH `maxLiveWorkers` and a
|
|
3754
|
-
* dispatcher's `width` — that is what turns three unrelated numbers into one.
|
|
3755
|
-
*/
|
|
3756
|
-
declare function effectiveConcurrency(caps: ConcurrencyCaps): number | undefined;
|
|
3757
|
-
/** Convenience: a `DispatchUnit` factory over a fixed array of tasks, for the common case where
|
|
3758
|
-
* the queue is known up front and only the refill behavior is wanted. */
|
|
3759
|
-
declare function queueOf<Out>(units: ReadonlyArray<{
|
|
3760
|
-
agent: Agent<unknown, Out>;
|
|
3761
|
-
task: unknown;
|
|
3762
|
-
label: string;
|
|
3763
|
-
}>, budget: Budget): () => DispatchUnit<Out> | undefined;
|
|
3764
|
-
|
|
3765
|
-
/**
|
|
3766
|
-
* Throw a `ConfigError` when `allowed` is set, `model` is defined, and `model` is not a
|
|
3767
|
-
* member of `allowed`. No-op when `allowed` is unset (the unrestricted default) or when
|
|
3768
|
-
* `model` is undefined (nothing was configured to check).
|
|
3769
|
-
*/
|
|
3770
|
-
declare function assertModelAllowed(model: string | undefined, allowed: readonly string[] | undefined): void;
|
|
3771
|
-
|
|
3772
|
-
/**
|
|
3773
|
-
*
|
|
3774
|
-
* `createInMemoryRunContext` — the one-call bundle of the in-memory stores a
|
|
3775
|
-
* `createSupervisor().run(root, task, opts)` needs: a fresh `InMemorySpawnJournal`
|
|
3776
|
-
* (the event-sourced spawn log), a fresh `InMemoryResultBlobStore` (the
|
|
3777
|
-
* content-addressed `outRef` payload store the driver's `observe`/`finalize` reads
|
|
3778
|
-
* settled outputs through), and a fresh `createExecutorRegistry()` (the open
|
|
3779
|
-
* `AgentSpec → Executor` resolver).
|
|
3780
|
-
*
|
|
3781
|
-
* It exists to kill the boilerplate every offline/local supervised run repeats by
|
|
3782
|
-
* hand — three constructors threaded into `SupervisorOpts` — and to single-source the
|
|
3783
|
-
* ONE wiring invariant that is easy to get wrong: when the root is the recursive
|
|
3784
|
-
* `driverAgent` LLM-driver brain AND it may spawn DRIVER children (agents
|
|
3785
|
-
* driving agents), the registry MUST be wrapped with `withDriverExecutor` so a
|
|
3786
|
-
* `role: 'driver'` child resolves to the nested-scope executor — and that SAME blob
|
|
3787
|
-
* store MUST be the one passed to `driverAgent({ blobs })`, or the driver
|
|
3788
|
-
* reads from a different store than the scope writes to. Pass `{ withDriver: true }`
|
|
3789
|
-
* and reuse the returned `blobs` for both.
|
|
3790
|
-
*
|
|
3791
|
-
* The spread shape matches `SupervisorOpts` exactly, so the call site reads:
|
|
3792
|
-
* const run = createInMemoryRunContext()
|
|
3793
|
-
* await createSupervisor().run(root, task, { budget, runId, ...run })
|
|
3794
|
-
*
|
|
3795
|
-
* @experimental
|
|
3796
|
-
*/
|
|
3797
|
-
|
|
3798
|
-
/** Options for a supervised run context. */
|
|
3799
|
-
interface InMemoryRunContextOptions {
|
|
3800
|
-
/**
|
|
3801
|
-
* Wrap the executor registry with `withDriverExecutor` so a spawned child marked
|
|
3802
|
-
* `role: 'driver'` resolves to the recursive driver-executor (agents driving agents
|
|
3803
|
-
* over a nested `Scope` on the same conserved pool). Leave `false` for a flat tree of
|
|
3804
|
-
* leaf workers. Default `false`.
|
|
3805
|
-
*/
|
|
3806
|
-
readonly withDriver?: boolean;
|
|
3807
|
-
}
|
|
3808
|
-
/**
|
|
3809
|
-
* The bundle of stores a supervised run needs, shaped to spread into `SupervisorOpts`.
|
|
3810
|
-
* The fields are exactly `SupervisorOpts`' `journal` / `blobs` / `executors`.
|
|
3811
|
-
*/
|
|
3812
|
-
interface InMemoryRunContext {
|
|
3813
|
-
readonly journal: SpawnJournal;
|
|
3814
|
-
readonly blobs: ResultBlobStore;
|
|
3815
|
-
readonly executors: ExecutorRegistry;
|
|
3816
|
-
/**
|
|
3817
|
-
* Present (and `true`) only on a DURABLE context (`createFileRunContext`), so spreading the
|
|
3818
|
-
* context into `SupervisorOpts` also opts the run into resume-first. An in-memory context
|
|
3819
|
-
* leaves it undefined: there is never a prior tree to resume, and the default stays fresh-run.
|
|
3820
|
-
*/
|
|
3821
|
-
readonly resume?: boolean;
|
|
3822
|
-
}
|
|
3823
|
-
/** The stores a supervised run needs, in-memory or file-backed. `InMemoryRunContext` is the
|
|
3824
|
-
* historical name for the same shape. */
|
|
3825
|
-
type RunContext = InMemoryRunContext;
|
|
3826
|
-
/**
|
|
3827
|
-
* Build a fresh in-memory run context. Every call returns NEW stores (no shared global
|
|
3828
|
-
* state between runs), so two runs never cross-contaminate their journals/blobs.
|
|
3829
|
-
*/
|
|
3830
|
-
declare function createInMemoryRunContext(opts?: InMemoryRunContextOptions): InMemoryRunContext;
|
|
3831
|
-
/**
|
|
3832
|
-
* Build a DURABLE run context: the spawn journal and the result blobs are file-backed (fsynced
|
|
3833
|
-
* per append/write) under `dir`, and the context carries `resume: true` so spreading it into
|
|
3834
|
-
* `SupervisorOpts` makes the supervisor `loadTree`-first. A run that dies mid-flight therefore
|
|
3835
|
-
* resumes when it is re-run with the SAME `runId` and the SAME `dir`: the committed children come
|
|
3836
|
-
* back on `Scope.resume` (rehydrated by `replaySpawnTree`) instead of being re-executed.
|
|
3837
|
-
*
|
|
3838
|
-
* Layout: `${dir}/spawn-journal.jsonl` (one JSONL record per event) and `${dir}/blobs/` (one
|
|
3839
|
-
* content-addressed JSON file per settled result). The directory is created on first write.
|
|
3840
|
-
*
|
|
3841
|
-
* Opt-in by construction — `createInMemoryRunContext()` is unchanged and stays the default, so no
|
|
3842
|
-
* existing consumer writes to disk or resumes unless it asks for this.
|
|
3843
|
-
*/
|
|
3844
|
-
declare function createFileRunContext(dir: string, opts?: InMemoryRunContextOptions): RunContext;
|
|
3845
|
-
|
|
3846
|
-
/**
|
|
3847
|
-
*
|
|
3848
|
-
* The reactive `Scope` impl (KEYSTONE, build step 4 + the step-8 adapter).
|
|
3849
|
-
*
|
|
3850
|
-
* An `Agent.act` runs inside a `Scope`. It `spawn`s children dynamically and reacts to
|
|
3851
|
-
* them via `next()`. The scope owns ONE in-memory nursery — the authoritative live set —
|
|
3852
|
-
* and is the single place that drives a child's lifecycle: reserve budget atomically,
|
|
3853
|
-
* resolve a `Executor` through the open registry, run it (one-shot OR streaming),
|
|
3854
|
-
* fold its normalized `UsageEvent`s into a conserved `Spend`, reconcile the reservation
|
|
3855
|
-
* (refunding the unspent remainder), persist the result blob + journal records, and
|
|
3856
|
-
* deliver the `Settled` through the `next()` cursor.
|
|
3857
|
-
*
|
|
3858
|
-
* Three invariants this impl enforces by construction:
|
|
3859
|
-
* - `next()` is a ray.wait n=1 cursor over THIS scope's live set; it assigns the
|
|
3860
|
-
* monotonic `seq` (the recorded cursor order) at the moment it yields a settlement, so
|
|
3861
|
-
* replay re-delivers in the identical order — `seq` is never wall-clock.
|
|
3862
|
-
* - Budget is reserved at spawn and reconciled at settle through the shared `BudgetPool`,
|
|
3863
|
-
* so `spawn` fails CLOSED on an exhausted pool and total ≡ free + reserved + committed.
|
|
3864
|
-
* - `view` reads the in-memory nursery, never the journal — O(live), synchronous.
|
|
3865
|
-
*
|
|
3866
|
-
* The settle path is the only writer of journal `settled` events; the spawn path the only
|
|
3867
|
-
* writer of `spawned` events. The result blob is `put` BEFORE the journal `settled` record
|
|
3868
|
-
* references its `outRef`, so a crash can never leave a journaled ref with no blob.
|
|
3869
|
-
*
|
|
3870
|
-
* @experimental
|
|
3871
|
-
*/
|
|
3872
|
-
|
|
3873
|
-
/** Construction args for `createScope`. The supervisor threads the shared pool, journal,
|
|
3874
|
-
* blob store, and executor registry through; `depth`/`maxDepth` pair the runtime
|
|
3875
|
-
* recursion ceiling with the conserved pool (R3). */
|
|
3876
|
-
interface ScopeArgs {
|
|
3877
|
-
/** This scope's owning node id — children get `${parentId}:s${seq}` ids. */
|
|
3878
|
-
readonly parentId: NodeId;
|
|
3879
|
-
/** Journal/blob root key the supervisor `beginTree`'d. */
|
|
3880
|
-
readonly root: NodeId;
|
|
3881
|
-
/** The shared conserved reservation pool (one per supervised run). */
|
|
3882
|
-
readonly pool: BudgetPool;
|
|
3883
|
-
/** Append-only spawn journal; this scope writes `spawned` + `settled` records. */
|
|
3884
|
-
readonly journal: SpawnJournal;
|
|
3885
|
-
/** Content-addressed result store backing `outRef` rehydration. */
|
|
3886
|
-
readonly blobs: ResultBlobStore;
|
|
3887
|
-
/** The open executor resolver (BYO → router/inline → registered harness factory). */
|
|
3888
|
-
readonly executors: ExecutorRegistry;
|
|
3889
|
-
/** Predicate resolver for `poll` wait-states. Absent ⇒ `wait` refuses a `poll` with
|
|
3890
|
-
* `unknown-probe`; `timer` waits never touch it. */
|
|
3891
|
-
readonly probes?: WaitProbeRegistry;
|
|
3892
|
-
/** Injected sleeper for wait-states — a test drives a week-long timer in microseconds. */
|
|
3893
|
-
readonly waitSleep?: (ms: number, signal: AbortSignal) => Promise<void>;
|
|
3894
|
-
/** Per-spawn executor-construction seams (sandbox client, router config, cli bin). */
|
|
3895
|
-
readonly seams: Readonly<Record<string, unknown>>;
|
|
3896
|
-
/** This scope's recursion depth (root = 0). */
|
|
3897
|
-
readonly depth: number;
|
|
3898
|
-
/** Runtime recursion-depth ceiling — a spawn past it fails closed `depth-exceeded`. */
|
|
3899
|
-
readonly maxDepth?: number;
|
|
3900
|
-
/** Abort signal for this scope; an abort cascades into every live child's executor. */
|
|
3901
|
-
readonly signal: AbortSignal;
|
|
3902
|
-
/** Injected clock — keeps the journal `at` timestamp deterministic in tests. */
|
|
3903
|
-
readonly now?: () => number;
|
|
3904
|
-
/** Lifecycle stream sink. `spawn` emits `agent.spawn`, `next` emits `agent.child` — the
|
|
3905
|
-
* SAME stream `runAgentRounds`/`tool-loop` feed, so the recursive tree is ONE observable stream
|
|
3906
|
-
* (the topology viewer reads it). Undefined ⇒ the journal stays the only record. */
|
|
3907
|
-
readonly hooks?: RuntimeHooks;
|
|
3908
|
-
/**
|
|
3909
|
-
* Resume seam — set ONLY by the supervisor when `SupervisorOpts.resume` is on AND a non-empty
|
|
3910
|
-
* journal tree exists for this root. It carries the replayed committed work (so `scope.resume`
|
|
3911
|
-
* exposes it to a resume-aware `act`) and the recorded ordinal/cursor maxima the new counters
|
|
3912
|
-
* continue past, so a freshly-spawned child never reuses a journaled `seq`. Absent ⇒ fresh run.
|
|
3913
|
-
*/
|
|
3914
|
-
readonly resumeFrom?: {
|
|
3915
|
-
readonly settled: ReadonlyArray<Settled<unknown>>;
|
|
3916
|
-
readonly view: TreeView;
|
|
3917
|
-
/** Highest `spawned` ordinal already journaled; new spawns start at `+1`. */
|
|
3918
|
-
readonly maxSpawnOrdinal: number;
|
|
3919
|
-
/** Highest cursor `seq` already journaled; new settlements start at `+1`. */
|
|
3920
|
-
readonly maxCursorSeq: number;
|
|
3921
|
-
/** Highest `waiting` ordinal already journaled; new waits start at `+1`. */
|
|
3922
|
-
readonly maxWaitOrdinal: number;
|
|
3923
|
-
/** Waits journaled as armed but never woken — re-armed (same node id, same absolute deadline)
|
|
3924
|
-
* when `wait` is called again with the SAME label. */
|
|
3925
|
-
readonly waits: ReadonlyArray<PendingWait>;
|
|
3926
|
-
};
|
|
3927
|
-
}
|
|
3928
|
-
/** Create the reactive `Scope` a driver's `Agent.act` runs inside: spawn children on an atomically reserved conserved budget, settle via the `next()` cursor, journal for replay. */
|
|
3929
|
-
declare function createScope<Out>(args: ScopeArgs): Scope<Out>;
|
|
3930
|
-
/**
|
|
3931
|
-
* The step-8 merge-boundary adapter (M4): rehydrate a `Settled.done` into the kernel's
|
|
3932
|
-
* `Iteration` shape so `defaultSelectWinner` stays single-sourced — the supervisor selects
|
|
3933
|
-
* across settled children with the SAME argmax the loop kernel uses, not a forked copy.
|
|
3934
|
-
*
|
|
3935
|
-
* `index` is the cursor `seq` (the recorded, replay-stable order); `output`/`verdict`/
|
|
3936
|
-
* `tokenUsage`/`costUsd` are read straight off the settlement (already rehydrated from the
|
|
3937
|
-
* `outRef` blob by `next()`). Events are empty — a settled child is an opaque leaf result,
|
|
3938
|
-
* not a sandbox event stream — and the timing/cost fields project its conserved `Spend`.
|
|
3939
|
-
* Fail loud on a `down` settlement: only a `done` child is an iteration.
|
|
3940
|
-
*/
|
|
3941
|
-
declare function settledToIteration<Out>(settled: Settled<Out>): Iteration<unknown, Out>;
|
|
3942
|
-
|
|
3943
|
-
/**
|
|
3944
|
-
*
|
|
3945
|
-
* The `Supervisor` impl (KEYSTONE, build step 5).
|
|
3946
|
-
*
|
|
3947
|
-
* Owns the four things a free-running recursive `act` cannot own itself: the GLOBAL
|
|
3948
|
-
* conserved budget pool, the event-sourced spawn log, the abort cascade over the whole
|
|
3949
|
-
* live tree, and the OTP intensity breaker. `run` builds the root `Scope` over those,
|
|
3950
|
-
* runs the root `Agent.act`, and returns a TYPED `SupervisedResult` — a no-winner is
|
|
3951
|
-
* never coerced into a best-effort `Out`.
|
|
3952
|
-
*
|
|
3953
|
-
* Three lifecycle invariants this impl enforces by construction:
|
|
3954
|
-
* - Join barrier: when `act()` settles (resolve OR reject), every still-live child is
|
|
3955
|
-
* torn down before `run` returns — the generalization of the kernel's
|
|
3956
|
-
* `finally{ Promise.allSettled(destroy) }` barrier (run-loop.ts) from boxes to the
|
|
3957
|
-
* whole sub-tree. A teardown failure is `allSettled`'d and journaled as a
|
|
3958
|
-
* `cancelled` event; it NEVER masks act()'s own outcome. act()'s rejection is the
|
|
3959
|
-
* PRIMARY error (the kernel's firstError precedence), so a teardown throw during the
|
|
3960
|
-
* barrier can never overwrite the real failure.
|
|
3961
|
-
* - Abort cascade: a root abort (caller signal, `RootHandle.abort`, a tripped breaker,
|
|
3962
|
-
* or pool exhaustion) aborts ONE internal controller whose signal is the root scope's
|
|
3963
|
-
* signal. The scope cascades that into every live child's executor abort — which, for
|
|
3964
|
-
* an `acquiring` child, chains into the `acquireSandbox` signal and reaps the
|
|
3965
|
-
* find-by-name orphan box (M1). The supervisor never reaps children directly.
|
|
3966
|
-
* - The supervisor NEVER re-enters a child (m3): the kernel/`acquireSandbox` already
|
|
3967
|
-
* retried at the leaf, and a driver re-spawns through `scope.spawn`. The breaker only
|
|
3968
|
-
* COUNTS `down` settlements within the intensity window and trips to a typed
|
|
3969
|
-
* no-winner; it does not restart anything.
|
|
3970
|
-
*
|
|
3971
|
-
* Selection lives in the driver, not here (selector≠judge): `act` returns the synthesized
|
|
3972
|
-
* winner `Out`. The supervisor content-addresses that `Out` for its replay `outRef`, reads
|
|
3973
|
-
* `spentTotal` off the journal (`settled` child work + `metered` driver inference), and wraps
|
|
3974
|
-
* it as a typed `winner` — it does not re-rank children behind the driver's back.
|
|
3975
|
-
*
|
|
3976
|
-
* @experimental
|
|
3977
|
-
*/
|
|
3978
|
-
|
|
3979
|
-
declare function createSupervisor<Task, Out>(): Supervisor<Task, Out>;
|
|
3980
|
-
|
|
3981
|
-
/**
|
|
3982
|
-
*
|
|
3983
|
-
* The SETTLE-time analyst: when a worker finishes, collect its tool spans from a `TraceSource` and run
|
|
3984
|
-
* agent-eval's PUBLISHED batch analyzers over them — `buildTrajectory` (structured run summary),
|
|
3985
|
-
* `stuckLoopView` (full-run repeated-call view, complementing the online consecutive detector), and
|
|
3986
|
-
* `toolWasteView`. Substrate-agnostic: the spans come from any source (an owned loop's buffer OR a
|
|
3987
|
-
* sandbox box session). No analysis reimplemented — this is the thin bridge into agent-eval's analyzers.
|
|
3988
|
-
*
|
|
3989
|
-
* @experimental
|
|
3990
|
-
*/
|
|
3991
|
-
|
|
3992
|
-
interface TrajectoryAnalysis {
|
|
3993
|
-
/** Structured run summary (tool-call count, step order). Steps carry a single timestamp, so per-span
|
|
3994
|
-
* duration is 0; loop/waste detection keys on call PATTERNS + cross-span windows, not durations. */
|
|
3995
|
-
readonly trajectory: Awaited<ReturnType<typeof buildTrajectory>>;
|
|
3996
|
-
/** Full-run repeated-call view (total occurrences + window) — allows one intervening call so it
|
|
3997
|
-
* catches a loop the online consecutive detector interleaves past. */
|
|
3998
|
-
readonly stuckLoop: Awaited<ReturnType<typeof stuckLoopView>>;
|
|
3999
|
-
/** Wasted-vs-total tool-call ratio for the run. */
|
|
4000
|
-
readonly toolWaste: Awaited<ReturnType<typeof toolWasteView>>;
|
|
4001
|
-
}
|
|
4002
|
-
/** Collect the source's spans and run the agent-eval batch analyzers over them under one `runId`. */
|
|
4003
|
-
declare function analyzeTrace(source: TraceSource, runId?: string): Promise<TrajectoryAnalysis>;
|
|
4004
|
-
|
|
4005
|
-
/** What a surface worker settles with — the surface verdict the driver + deliverable read. `resolved` is
|
|
4006
|
-
* the surface check's pass/fail (settled ⟺ resolved); `score` is the partial-credit fraction; `failing`
|
|
4007
|
-
* carries the tests this worker left red (so the analyst can target them). */
|
|
4008
|
-
interface SurfaceWorkerOut {
|
|
4009
|
-
readonly resolved: boolean;
|
|
4010
|
-
readonly score: number;
|
|
4011
|
-
readonly shots: number;
|
|
4012
|
-
readonly summary: string;
|
|
4013
|
-
readonly failing?: readonly string[];
|
|
4014
|
-
}
|
|
4015
|
-
/** The default self-improvement LENS — authored content, not a code path. On each settled worker it hands
|
|
4016
|
-
* the driver the still-FAILING tests (not just a score), so the next spawn targets the persistently-hard
|
|
4017
|
-
* cases. Swap `analysts` to change what the driver improves from — that's the one knob. */
|
|
4018
|
-
declare function failuresAnalyst(): AnalystRegistry;
|
|
4019
|
-
/** How a worker runs the surface task (its router substrate + per-attempt bounds). */
|
|
4020
|
-
interface SurfaceWorkerConfig {
|
|
4021
|
-
readonly routerBaseUrl: string;
|
|
4022
|
-
readonly routerKey: string;
|
|
4023
|
-
readonly model: string;
|
|
4024
|
-
readonly maxTokens?: number;
|
|
4025
|
-
readonly innerTurns?: number;
|
|
4026
|
-
/** Refine-shot budget for ONE worker attempt (max steered shots). Default 1. */
|
|
4027
|
-
readonly budget?: number;
|
|
4028
|
-
}
|
|
4029
|
-
interface SuperviseSurfaceOptions {
|
|
4030
|
-
/** The graded surface workers solve (open/tools/call/score/close). */
|
|
4031
|
-
readonly surface: AgenticSurface;
|
|
4032
|
-
/** Where/how each worker runs the surface task. */
|
|
4033
|
-
readonly worker: SurfaceWorkerConfig;
|
|
4034
|
-
/** The conserved compute pool for the whole supervised run. Default: sized off the worker's inner-loop
|
|
4035
|
-
* bounds for a handful of worker spawns — raise it to let the driver try more. */
|
|
4036
|
-
readonly budget?: Budget;
|
|
4037
|
-
/** The driver brain's router substrate (its own inference). Default: the worker's router + model — the
|
|
4038
|
-
* driver and workers share one router unless you separate them (e.g. a stronger driver model). */
|
|
4039
|
-
readonly router?: RouterConfig;
|
|
4040
|
-
/** The self-improvement lens fed to the driver on each settled worker. Default `failuresAnalyst()`
|
|
4041
|
-
* (target the still-failing tests). Pass a custom registry to change it, or `null` to turn the
|
|
4042
|
-
* within-run self-improvement OFF (the driver sees raw settled outputs). */
|
|
4043
|
-
readonly analysts?: AnalystRegistry | null;
|
|
4044
|
-
/** The strategy each worker runs over the surface. Default `refine` (iterate-with-feedback). */
|
|
4045
|
-
readonly strategy?: Strategy;
|
|
4046
|
-
/** Max workers live at once. Default 1 (serial — required when workers share a persistent artifact, so
|
|
4047
|
-
* they continue each other instead of racing the file). */
|
|
4048
|
-
readonly maxLiveWorkers?: number;
|
|
4049
|
-
}
|
|
4050
|
-
/** The deployable outcome of a supervised surface run. */
|
|
4051
|
-
interface SuperviseSurfaceResult {
|
|
4052
|
-
readonly resolved: boolean;
|
|
4053
|
-
readonly score: number;
|
|
4054
|
-
readonly usd: number;
|
|
4055
|
-
readonly tokensIn: number;
|
|
4056
|
-
readonly tokensOut: number;
|
|
4057
|
-
readonly ms: number;
|
|
4058
|
-
/** Total conserved-pool iterations = the driver + worker LLM rounds the run actually spent. */
|
|
4059
|
-
readonly completions: number;
|
|
4060
|
-
}
|
|
4061
|
-
/** Drive a team of agents (spawned + steered by `profile`) to solve a graded `AgenticSurface` task, and
|
|
4062
|
-
* report the deployable outcome + the full conserved spend. This is `supervise()` configured for surfaces
|
|
4063
|
-
* — there is no other entrypoint to learn. */
|
|
4064
|
-
declare function superviseSurface(profile: SupervisorProfile, task: AgenticTask, opts: SuperviseSurfaceOptions): Promise<SuperviseSurfaceResult>;
|
|
4065
|
-
|
|
4066
|
-
/**
|
|
4067
|
-
* createVerifierEnvironment — ANY checkable task as an `Environment`, no tool surface
|
|
4068
|
-
* required. The generalization piece: EOPS/commit0-style domains have tools that mutate
|
|
4069
|
-
* an external artifact, but math problems, legal drafts, creative briefs, GTM copy, and
|
|
4070
|
-
* QA tasks have a different shape — the artifact IS the worker's answer, and the domain
|
|
4071
|
-
* is defined by one function: the deployable check over that answer.
|
|
4072
|
-
*
|
|
4073
|
-
* const gsm8k = createVerifierEnvironment({
|
|
4074
|
-
* name: 'gsm8k',
|
|
4075
|
-
* check: (task, answer) => ({
|
|
4076
|
-
* passes: extractFinalNumber(answer) === task.meta?.answer ? 1 : 0,
|
|
4077
|
-
* total: 1,
|
|
4078
|
-
* errored: 0,
|
|
4079
|
-
* }),
|
|
4080
|
-
* })
|
|
4081
|
-
* await runBenchmark({ environment: gsm8k, tasks, worker }) // sample vs refine on math
|
|
4082
|
-
*
|
|
4083
|
-
* The worker gets one built-in tool — `submit_answer` — plus any read-only domain tools
|
|
4084
|
-
* the caller adds (a calculator, a retrieval call, a style guide lookup). Every
|
|
4085
|
-
* submission is kept; `score()` checks the BEST submission (keep-best is the measured
|
|
4086
|
-
* law: workers reach correct answers then revise past them). The refine strategy's
|
|
4087
|
-
* critic reads the submission trajectory like any other trace, so iterate-with-feedback
|
|
4088
|
-
* works unchanged on answer domains.
|
|
4089
|
-
*
|
|
4090
|
-
* The check can be graded (passes/total expresses partial credit — rubric points,
|
|
4091
|
-
* sub-answers, unit-test counts), and MUST be deployable (computable without an oracle
|
|
4092
|
-
* at serve time): exact/numeric match, schema validation, a compiled rubric — not a
|
|
4093
|
-
* peek at held-out labels the production system wouldn't have.
|
|
4094
|
-
*/
|
|
4095
|
-
|
|
4096
|
-
interface VerifierEnvironmentOptions {
|
|
4097
|
-
name: string;
|
|
4098
|
-
/** The deployable check over a submitted answer. Graded via passes/total. */
|
|
4099
|
-
check(task: AgenticTask, answer: string): Promise<SurfaceScore> | SurfaceScore;
|
|
4100
|
-
/** Extra domain tools (read-only helpers: calculator, retrieval, style lookup). */
|
|
4101
|
-
extraTools?: AgenticTool[];
|
|
4102
|
-
/** Executes the extra tools. Required when `extraTools` is set. */
|
|
4103
|
-
callExtra?(task: AgenticTask, name: string, args: Record<string, unknown>): Promise<string> | string;
|
|
4104
|
-
}
|
|
4105
|
-
/** Any checkable task as an `Environment`, no tool surface required: the artifact is the worker's answer and the domain is one deployable `check` over it. */
|
|
4106
|
-
declare function createVerifierEnvironment(opts: VerifierEnvironmentOptions): Environment;
|
|
4107
|
-
|
|
4108
|
-
/** Command runner seam. Host code can use `localShell`; sandbox code can wrap `box.exec`. */
|
|
4109
|
-
type Shell = (args: ReadonlyArray<string>, cwd?: string) => Promise<{
|
|
4110
|
-
stdout: string;
|
|
4111
|
-
stderr: string;
|
|
4112
|
-
code: number;
|
|
4113
|
-
}>;
|
|
4114
|
-
type WorkspaceCommit = {
|
|
4115
|
-
readonly ok: true;
|
|
4116
|
-
readonly rev: string;
|
|
4117
|
-
} | {
|
|
4118
|
-
readonly ok: false;
|
|
4119
|
-
readonly conflict: string;
|
|
4120
|
-
};
|
|
4121
|
-
interface Workspace {
|
|
4122
|
-
readonly ref: string;
|
|
4123
|
-
materialize(dir: string): Promise<void>;
|
|
4124
|
-
commit(dir: string, message: string): Promise<WorkspaceCommit>;
|
|
4125
|
-
head(): Promise<string>;
|
|
4126
|
-
}
|
|
4127
|
-
/** Host-process `Shell`: run a command via `execFile`, resolving `{ stdout, stderr, code }` (never throws on non-zero exit). */
|
|
4128
|
-
declare function localShell(): Shell;
|
|
4129
|
-
interface GitWorkspaceOptions {
|
|
4130
|
-
readonly ref: string;
|
|
4131
|
-
readonly shell?: Shell;
|
|
4132
|
-
readonly branch?: string;
|
|
4133
|
-
readonly noHooks?: boolean;
|
|
4134
|
-
}
|
|
4135
|
-
/** A `Workspace` over a git checkout: materialize an isolated worktree at `ref`, commit produced changes (conflict-aware), and read `head` — hooks disabled, identity pinned. */
|
|
4136
|
-
declare function gitWorkspace(opts: GitWorkspaceOptions): Workspace;
|
|
4137
|
-
/** A jj-backed `Workspace` (Jujutsu, colocated with git for the durable remote).
|
|
4138
|
-
* Same port, same `Shell` — a drop-in for `gitWorkspace`. jj suits agent loops:
|
|
4139
|
-
* no staging area, and a first-class operation log (native resume/undo). Live use
|
|
4140
|
-
* requires `jj` on the `Shell`'s host. */
|
|
4141
|
-
declare function jjWorkspace(opts: GitWorkspaceOptions): Workspace;
|
|
4142
|
-
interface WorkspaceRun<T> {
|
|
4143
|
-
readonly valid: boolean;
|
|
4144
|
-
readonly value: T;
|
|
4145
|
-
/** Present when a commit was attempted (valid, or `commitOnInvalid`). */
|
|
4146
|
-
readonly commit?: WorkspaceCommit;
|
|
4147
|
-
}
|
|
4148
|
-
/**
|
|
4149
|
-
* Run a worker `body` inside a FRESH clone of a shared `Workspace`, then commit its work back
|
|
4150
|
-
* so the next worker (or the supervisor) builds on it. This is the seam that turns isolated
|
|
4151
|
-
* per-worker cwds into one compounding artifact — `body` gets a real materialized dir, its
|
|
4152
|
-
* delivery is committed to the shared ref iff it's valid (a conflict is returned, never thrown).
|
|
4153
|
-
* The clone is removed after; durable state lives only in the ref.
|
|
4154
|
-
*/
|
|
4155
|
-
declare function runInWorkspace<T>(ws: Workspace, body: (cwd: string) => Promise<{
|
|
4156
|
-
valid: boolean;
|
|
4157
|
-
value: T;
|
|
4158
|
-
message?: string;
|
|
4159
|
-
}>, opts?: {
|
|
4160
|
-
tmpPrefix?: string;
|
|
4161
|
-
commitOnInvalid?: boolean;
|
|
4162
|
-
}): Promise<WorkspaceRun<T>>;
|
|
4163
|
-
|
|
4164
|
-
export { Agent, AgentRunSpec, type AgentTurnBackend, type AgentTurnUsage, AgenticOptions, AgenticSurface, AgenticTask, AgenticTool, AnalystRegistry, type AnytimeReport, type AnytimeStrategySummary, type AnytimeTaskCurve, type ApplyContinuation, ArtifactHandle, AssertTraceDerivedFindings, type AuditIntentInput, type AuditIntentOptions, type AuthorStrategyOptions, type AuthoredProfile, type AuthoredStrategy, type BenchmarkCell, type BenchmarkConfig, type BenchmarkLift, type BenchmarkReport, type BenchmarkStrategySummary, type BenchmarkTaskRow, Budget, type BudgetPool, type BudgetReadout, type ChampionPick, type ChampionPolicy, type CheckpointCapableBox, type CollectedAgentTurn, CombinatorShape, type CompletionAnalyst, type CompletionEvidence, type CompletionPolicy, type CompletionVerdict, type ConcurrencyCaps, CoordinationEvent, type CoordinationMcpHandle, Corpus, CorpusFilter, CorpusRecord, type CreateScopeAnalystOptions, type CriuCapableClient, DefinePersonaInput, type DefinedLeaderboard, type DelegateOptions, type Deliverable, DeliverableSpec, type DispatchReport, type DispatchStopReason, type DispatchUnit, Driver, type DriverAgentOptions, type DumbDriverOptions, type Environment, EqualKArm, EqualKOnCostOptions, EqualKVerdict, type EvolutionArchiveNode, type EvolutionAuthor, type EvolutionBandInfo, type EvolutionCandidate, type EvolutionGeneration, type EvolutionReport, ExecCtx, ExecutorConfig, ExecutorFactory, ExecutorRegistry, FanoutOptions, FanoutWinnerSelector, FileCorpus, FileResultBlobStore, FileSpawnJournal, type ForkCapableBox, type GitWorkspaceOptions, type HarvestCorpusOptions, type HarvestFailure, type HarvestReport, InMemoryCorpus, InMemoryResultBlobStore, type InMemoryRunContext, type InMemoryRunContextOptions, InMemorySpawnJournal, type InProcessOnPrompt, type InProcessPromptCtx, type InProcessSandboxClientOptions, type IntentAudit, type Interval, Iteration, type KeyProvider, type Leaderboard, type LeaderboardBenchScore, type LeaderboardBenchTask, type LeaderboardBenchmarkAdapter, type LeaderboardFlagSpec, type LeaderboardIterationInfo, type LeaderboardOptions, type LeaderboardRow, type LeaderboardRunContext, type LeaderboardScenario, type LeaderboardScore, type LeaderboardSpec, type LocalMcpMaterialization, type LocalSandboxClientOptions, type LoopCampaignDispatchOptions, type LoopDispatchOptions, LoopLineageOptions, type LoopOptionsForDispatch, LoopResult, LoopShape, LoopTokenUsage, LoopUntilSpec, LoopWinner, MakeWorkerAgent, type MaterializeLocalMcpOptions, type McpEndpoint, type McpEnvironmentOptions, McpSpawnFault, type McpToolDescriptor, MountRecorder, type NaiveDriverOptions, NodeId, type Observation, type ObserveInput, type ObserveOptions, type OpenSandboxRunBeforeStartContext, type OpenSandboxRunOptions, type OpenSandboxRunPromptOptions, Outcome, OutputAdapter, type PairwiseOptions, type PairwiseVerdict, PanelSpec, PendingWait, Persona, PipelineStage, type ProfileRichness, type ProfileRichnessThresholds, type PromotionGateOptions, type PromotionVerdict, type RegistryAnalyzeProjection, RenderCorpusToInstructionsOptions, type ReservationTicket, type ResolveSandboxClientOptions, ResultBlobStore, type RollingDispatchOptions, RouterConfig, type RunAgentRoundsOptions, type RunContext, type RunLoopOptions, RunPersonifiedOptions, type SandboxCapabilities, SandboxClient, type SandboxLineage, type SandboxLineageHandle, type SandboxRun, SandboxRunAbortError, type SandboxToolPartState, Scope, ScopeAnalyst, ScopeAnalyzeInput, ScopeWidenGate, type SessionCapableBox, Settled, ShapeRegistry, type Shell, SpawnEvent, SpawnJournal, SpawnOpts, Spend, type StdioMcpConnection, type StdioMcpServerSpec, SteerContext, type SteeringDecision, StopRule, Strategy, type StrategyEvolutionConfig, type StreamAgentTurnOptions, type SuperviseSurfaceOptions, type SuperviseSurfaceResult, SupervisedResult, Supervisor, SupervisorProfile, SurfaceScore, type SurfaceWorkerConfig, type SurfaceWorkerOut, ToolLoopChat, ToolLoopCompactionOptions, TraceSource, type TrajectoryAnalysis, TrajectoryReport, TrajectoryReportOptions, TreeView, type TurnResult, UsageEvent, type UsageSink, Validator, type VerifierEnvironmentOptions, VerifySpec, WaitProbeRegistry, type WaterfallCollector, type WaterfallReport, type WaterfallSpan, WidenSpec, WinnerStrategy, type Workspace, type WorkspaceCommit, type WorkspaceRun, acquireSandbox, analyzeTrace, anytimeReport, areaUnderCurve, asAuthoredProfile, assertModelAllowed, assertStrategyContract, assertTraceDerivedFindings, assessAuthoredProfile, auditIntent, authorStrategy, authoredWorker, bestSoFar, buildSteerContext, builtinShapes, collectAgentTurn, completionAuthorizes, connectStdioMcp, contentAddress, createBudgetPool, createFileRunContext, createInMemoryRunContext, createMcpEnvironment, createSandboxLineage, createSandboxToolPartState, createScope, createScopeAnalyst, createShapeRegistry, createSupervisor, createVerifierEnvironment, createWaterfallCollector, defaultAnalystInstruction, defaultAuditorInstruction, defaultDelegateBudget, defaultProfileRichnessThresholds, defaultSelectWinner, defineLeaderboard, definePersona, delegate, deterministicCompletion, discriminatingMeans, driverAgent, dumbDriver, effectiveConcurrency, envKeyProvider, equalKOnCost, extractLlmCallEvent, failuresAnalyst, fanout, finalizeBestDelivered, flatWidenGate, freeSlots, gitWorkspace, harvestCorpus, inProcessSandboxClient, inlineSandboxClient, jjWorkspace, leaderboard, localSandboxClient, localShell, loopCampaignDispatch, loopDispatch, loopUntil, mapSandboxEvent, mapSandboxToolEvent, materializeLocalMcp, materializeTreeView, mcpSecretEnvMetadataKey, naiveDriver, observe, openSandboxRun, pairwiseSignificance, panel, pendingWaits, pickChampion, pipeline, plateauLength, printBenchmarkReport, probeSandboxCapabilities, profileRichnessFinding, promotionGate, queueOf, registerShape, registryScopeAnalyst, renderAnytimeTable, renderCorpusToInstructions, renderLeaderboardHtml, renderLeaderboardMarkdown, renderLeaderboardSvg, renderPairwiseMarkdown, renderReport, replaySpawnTree, reportLoopUsage, resolveSandboxClient, resolveSecretEnv, rollingDispatch, runAgentRounds, runBenchmark, runInWorkspace, runLoop, runPersonified, runStrategyEvolution, sanitizeMcpToolSchema, secretEnvOfMcpServer, selectChampion, selectValidWinner, sentinelCompletion, serveCoordinationMcp, settledToIteration, spendFromUsageEvents, stopSentinel, strategyAuthorContract, streamAgentTurn, sumSandboxUsage, superviseSurface, supervisorInstructions, trajectoryReport, verify, widen };
|
|
1
|
+
import { C as MountRecorder, D as SelectionReceipt, E as SandboxClient, O as ValidationCtx, S as MountManifestEntry, T as RunProvenance, _ as LoopTeardownFailedPayload, a as Iteration, b as LoopTraceEvent, c as LoopIterationDispatchPayload, d as LoopLineageOptions, f as LoopPlanDescription, g as LoopStartedPayload, h as LoopSandboxPlacement, i as ExecCtx, k as Validator, l as LoopIterationEndedPayload, m as LoopResult, n as DefaultVerdict, o as LoopDecisionPayload, p as LoopPlanPayload, r as Driver, s as LoopEndedPayload, t as AgentRunSpec, u as LoopIterationStartedPayload, v as LoopTokenUsage, w as OutputAdapter, x as LoopWinner, y as LoopTraceEmitter } from "./types-BevOjfTY.js";
|
|
2
|
+
import { $ as RootSignal, A as providerAsSandboxClient, At as createPushTraceSource, B as Executor, Bt as readWorkerProgress, C as ProviderAsSandboxClientOptions, Ct as timerAt, D as WorkspaceRequest, Dt as SessionTraceBox, E as SandboxClientProviderOptions, Et as SessionMessageLike, F as createTangleSandboxExactProcessProvider, Ft as DEFAULT_STALL_AFTER_MS, G as Handle, H as ExecutorFactory, I as Agent, It as ExecutorProgress, J as NodeStatus, K as NodeId, L as AgentSpec, Lt as ScopeProgressInput, M as sandboxClientAsProvider, Mt as sandboxSessionTraceSource, N as CreateTangleSandboxExactProcessProviderOptions, Nt as ActivityLog, O as createAgentEnvironmentProviderRegistry, Ot as ToolStepInput, P as SandboxControlClient, Pt as ActivityNote, Q as RootHandle, R as Budget, Rt as WorkerProgress, S as PlacementInfo, St as pollFor, T as ResourceRequest, Tt as waitUntil, U as ExecutorRegistry, V as ExecutorContext, W as ExecutorResult, X as ResultBlobStore, Y as Restart, Z as ResumedWork, _ as CheckpointRequest, _t as WaitProbeRegistry, a as AgentEnvironmentProviderRef, at as SpawnOpts, b as ExecResult, bt as createWaitProbes, c as AgentEnvironmentStatus, ct as Supervisor, d as AgentSession, dt as UsageEvent, et as Runtime, f as AgentSessionRef, ft as WaitOpts, g as CheckpointRef, gt as WaitProbe, h as AgentTurnResult, ht as WaitOutcome, i as AgentEnvironmentProvider, it as SpawnJournal, j as resolveAgentEnvironmentProvider, jt as decodeToolPart, k as providerAsExecutor, kt as TraceSource, l as AgentEnvironmentSummary, lt as SupervisorOpts, m as AgentTurnInput, mt as PendingWait, n as AgentEnvironmentCapabilities, nt as Settled, o as AgentEnvironmentProviderRegistry, ot as Spend, p as AgentSessionStatus, pt as WidenGate, q as NodeSnapshot, r as AgentEnvironmentEvent, rt as SpawnEvent, s as AgentEnvironmentQuery, st as SupervisedResult, t as AgentEnvironment, tt as Scope, u as AgentProfileRef, ut as TreeView, v as CreateAgentEnvironmentInput, vt as WaitRejection, w as ProviderExecutorOptions, wt as validateWaitSpec, x as ForkRequest, xt as isWaitOutcome, y as ExecRequest, yt as WaitSpec, zt as createActivityLog } from "./environment-provider-IUGU3epE.js";
|
|
3
|
+
import { $ as effectiveConcurrency, $a as WidenDecision, $i as mcpSecretEnvMetadataKey, $n as connectStdioMcp, $o as auditIntent, $r as createScopeAnalyst, $s as createExecutor, $t as StructuralRolloutMessage, $u as FileSpawnJournal, A as createSupervisor, Aa as FlatWidenGate, Ai as SurfaceScore, An as ReproductionCheck, Ao as completionAuthorizes, Ar as promotionGate, At as sampleFromSettled, B as createFileRunContext, Ba as RenderCorpusToInstructions, Bi as LoopDispatchOptions, Bn as strategyAuthorContract, Bo as PairwiseOptions, Br as renderCorpusToInstructions, Bt as assessAuthoredProfile, C as failuresAnalyst, Ca as EqualKOnCost, Ci as ShotSpec, Cn as ChampionPolicy, Co as LeaderboardScore, Cr as probeSandboxCapabilities, Ct as StopRule, D as worktreeFanout, Da as FanoutOptions, Dc as gateOnDeliverable, Di as StrategyMessage, Dn as EvolutionCandidate, Do as CompletionEvidence, Dr as resolveSandboxClient, Dt as createProgressTracker, E as WorktreeFanoutOptions, Ea as Fanout, Ec as DeliverableSpec, Ei as StrategyCtx, En as EvolutionBandInfo, Eo as CompletionAnalyst, Er as ResolveSandboxClientOptions, Et as anyOf, F as createScope, Fa as PanelJudge, Fi as refine, Fn as selectChampion, Fo as GroupOf, Fr as registerShape, Ft as spendFromUsageEvents, G as WorktreePatchArtifact, Ga as SteerContext, Gi as RunLoopOptions, Gn as dumbDriver, Go as pairwiseSignificance, Gr as pipeline, Gs as CliWorktreeBridgeSeam, Gt as CheckExecChannel, Gu as BusStats, H as PatchDeliverableOptions, Ha as ScopeAnalyst, Hi as loopCampaignDispatch, Hn as DumbDriverOptions, Ho as ProfileKeyOf, Hr as flatWidenGate, Ht as defaultProfileRichnessThresholds, I as settledToIteration, Ia as PanelSpec, Ii as runAgentic, In as AuthorStrategyOptions, Io as Interval, Ir as definePersona, It as AuthoredProfile, J as ConcurrencyCaps, Ja as TrajectoryReportFn, Ji as runLoop, Jn as MaterializeLocalMcpOptions, Jo as renderLeaderboardSvg, Jr as widen, Js as ProviderSeam, Jt as CheckRunner, Ju as createEventBus, K as createWorktreeCliExecutor, Ka as TrajectoryNode, Ki as defaultSelectWinner, Kn as naiveDriver, Ko as renderLeaderboardHtml, Kr as selectValidWinner, Ks as CliWorktreeSeam, Kt as CheckOutcome, Ku as EventBus, L as InMemoryRunContext, La as PanelVerdict, Li as sample, Ln as AuthoredStrategy, Lo as Leaderboard, Lr as runPersonified, Lt as ProfileRichness, M as supervise, Ma as LoopUntilSpec, Mi as breadthStrategy, Mn as discriminatingMeans, Mo as sentinelCompletion, Mr as trajectoryReport, Mt as BudgetReadout, N as workerFromBackend, Na as LoopUntilState, Ni as defineStrategy, Nn as pickChampion, No as stopSentinel, Nr as builtinShapes, Nt as ReservationTicket, O as TrajectoryAnalysis, Oa as FanoutSynthesis, Oi as StrategyResult, On as EvolutionGeneration, Oo as CompletionPolicy, Or as PromotionGateOptions, Ot as noProgressFor, P as ScopeArgs, Pa as Panel, Pi as depthStrategy, Pn as runStrategyEvolution, Po as AxisScoresOf, Pr as createShapeRegistry, Pt as createBudgetPool, Q as RollingDispatchOptions, Qa as Widen, Qi as envKeyProvider, Qn as StdioMcpServerSpec, Qo as IntentAudit, Qr as buildSteerContext, Qs as cliWorktreeExecutor, Qt as StructuralRolloutConfig, Qu as FileResultBlobStore, R as InMemoryRunContextOptions, Ra as Pipeline, Ri as sampleThenRefine, Rn as assertStrategyContract, Ro as LeaderboardOptions, Rr as FileCorpus, Rt as ProfileRichnessThresholds, S as SurfaceWorkerOut, Sa as EqualKArm, Si as ShotPersona, Sn as ChampionPick, So as LeaderboardScenario, Sr as SandboxCapabilities, St as StopDecision, T as AuthoredHarness, Ta as EqualKVerdict, Ti as StrategyArtifacts, Tn as EvolutionAuthor, To as defineLeaderboard, Tr as acquireSandbox, Tt as allWorkersStalled, U as patchDelivered, Ua as ScopeAnalyzeInput, Ui as loopDispatch, Un as NaiveDriverOptions, Uo as ScoreOf, Ur as loopUntil, Us as BridgeSeam, Ut as profileRichnessFinding, Uu as BusEvent, V as createInMemoryRunContext, Va as RenderCorpusToInstructionsOptions, Vi as LoopOptionsForDispatch, Vn as ApplyContinuation, Vo as PairwiseVerdict, Vr as fanout, Vt as authoredWorker, W as WorktreeCliExecutorOptions, Wa as ScopeWidenGate, Wi as RunAgentRoundsOptions, Wn as SteeringDecision, Wo as leaderboard, Wr as panel, Ws as CliSeam, Wt as supervisorInstructions, Wu as BusRecord, X as DispatchStopReason, Xa as Verify, Xi as localSandboxClient, Xn as McpToolDescriptor, Xo as AuditIntentInput, Xr as RegistryAnalyzeProjection, Xs as RouterToolsSeam, Xt as CheckSourceCtx, Xu as defaultToolDetectors, Y as DispatchReport, Ya as TrajectoryReportOptions, Yi as LocalSandboxClientOptions, Yn as McpSpawnFault, Yo as renderPairwiseMarkdown, Yr as CreateScopeAnalystOptions, Ys as RouterSeam, Yt as CheckSource, Yu as WatchTraceOptions, Z as DispatchUnit, Za as VerifySpec, Zi as KeyProvider, Zn as StdioMcpConnection, Zo as AuditIntentOptions, Zr as assertTraceDerivedFindings, Zs as SandboxSeam, Zt as RepairStop, Zu as watchTrace, _ as VerifierEnvironmentOptions, _a as AssertTraceDerivedFindings, _i as AgenticTask, _n as AgentTurnUsage, _o as LeaderboardBenchTask, _r as extractLlmCallEvent, _t as PlateauOptions, a as SandboxInstance, aa as InProcessSandboxClientOptions, ac as createSteerableSandboxSession, ad as replaySpawnTree, ai as BenchmarkCell, an as composeCheckSources, ao as LoopShape, ar as SandboxRun, as as areaUnderCurve, at as delegate, b as SuperviseSurfaceResult, ba as CorpusFilter, bi as CorpusReadbackOptions, bn as collectAgentTurn, bo as LeaderboardIterationInfo, br as sumSandboxUsage, bs as MakeWorkerAgent, bt as ProgressTrackerOptions, c as GitWorkspaceOptions, ca as HarvestFailure, cc as createInbox, ci as BenchmarkReport, cn as filterAuthoredAsserts, co as PersonaContext, cr as openSandboxRun, cs as renderAnytimeTable, ct as SupervisorProfile, d as WorkspaceCommit, da as Observation, dc as piExecutor, di as Environment, dn as resolveEntrySymbol, do as RunPersonifiedOptions, dr as SandboxLineage, ds as WaterfallSpan, dt as serveCoordinationMcp, ea as resolveSecretEnv, ec as createExecutorRegistry, ed as InMemoryResultBlobStore, ei as registryScopeAnalyst, en as StructuralRolloutPolicy, eo as WidenLineage, er as materializeLocalMcp, es as defaultAuditorInstruction, et as freeSlots, f as WorkspaceRun, fa as ObserveInput, fc as piSeamKey, fi as printBenchmarkReport, fn as sandboxCheckRunner, fo as ShapeBudget, fr as SandboxLineageHandle, fs as createWaterfallCollector, ft as DriverAgentOptions, g as runInWorkspace, ga as renderReport, gc as WorktreeProfileMaterializationReceipt, gi as AgenticSurface, gn as AgentTurnBackend, go as LeaderboardBenchScore, gr as createSandboxToolPartState, gt as NoProgressForOptions, h as localShell, ha as observe, hc as WorktreeHarnessResult, hi as AgenticRunResult, hn as visibleCheckScore, ho as DefinedLeaderboard, hr as SandboxToolPartState, hs as CoordinationEvent, ht as AllWorkersStalledOptions, i as SandboxEvent, ia as InProcessPromptCtx, ic as SteerableSandboxSession, id as pendingWaits, ii as sanitizeMcpToolSchema, in as compareCheckOutcomes, io as DefinePersonaInput, ir as OpenSandboxRunPromptOptions, is as anytimeReport, it as defaultDelegateBudget, j as SuperviseOptions, ja as LoopUntil, ji as adaptiveRefine, jn as StrategyEvolutionConfig, jo as deterministicCompletion, jr as equalKOnCost, jt as BudgetPool, k as analyzeTrace, ka as FanoutWinnerSelector, ki as StrategyShotResult, kn as EvolutionReport, ko as CompletionVerdict, kr as PromotionVerdict, kt as plateau, l as Shell, la as HarvestReport, lc as PI_RUNTIME, li as BenchmarkStrategySummary, ln as modelAuthoredChecks, lo as PersonaExecutors, lr as CheckpointCapableBox, ls as WaterfallCollector, lt as supervisorAgent, m as jjWorkspace, ma as defaultAnalystInstruction, mc as WorktreeCommandResult, mi as AgenticOptions, mn as structuralRollout, mo as ShapeRegistry, mr as createSandboxLineage, ms as AnalystRegistry, mt as finalizeBestDelivered, n as AnalystFinding, na as inlineSandboxClient, nc as SandboxSteeringOptions, nd as contentAddress, ni as McpEnvironmentOptions, nn as VisibleCheck, no as WinnerStrategy, nr as OpenSandboxRunBeforeStartContext, ns as AnytimeStrategySummary, nt as rollingDispatch, o as computeFindingId, oa as inProcessSandboxClient, oc as Inbox, oi as BenchmarkConfig, on as defaultExtractCandidate, oo as Outcome, or as SandboxRunAbortError, os as bestSoFar, ot as DriveHarness, p as gitWorkspace, pa as ObserveOptions, pc as WorktreeCheckRunner, pi as runBenchmark, pn as selectBestIndex, po as ShapeContext, pr as SessionCapableBox, ps as AnalystFindingEvent, pt as driverAgent, q as assertModelAllowed, qa as TrajectoryReport, qi as runAgentRounds, qn as LocalMcpMaterialization, qo as renderLeaderboardMarkdown, qr as verify, qs as ExecutorConfig, qt as CheckRunContext, qu as PublishOptions, r as CreateSandboxOptions, ra as InProcessOnPrompt, rc as SteerableSandboxArgs, rd as materializeTreeView, ri as createMcpEnvironment, rn as canDisplace, ro as DefinePersona, rr as OpenSandboxRunOptions, rs as AnytimeTaskCurve, rt as DelegateOptions, s as makeFinding, sa as HarvestCorpusOptions, sc as InboxMessage, si as BenchmarkLift, sn as defaultStructuralRolloutPolicy, so as Persona, sr as TurnResult, ss as plateauLength, st as SupervisorAgentDeps, t as AgentProfile, ta as secretEnvOfMcpServer, tc as DEFAULT_SANDBOX_STEERING_MAX_TURNS, td as InMemorySpawnJournal, ti as McpEndpoint, tn as StructuralRolloutResult, to as WidenSpec, tr as Deliverable, ts as AnytimeReport, tt as queueOf, u as Workspace, ua as harvestCorpus, uc as PiSeam, ui as BenchmarkTaskRow, un as officialChecksFromMeta, uo as RunPersonified, ur as ForkCapableBox, us as WaterfallReport, ut as CoordinationMcpHandle, v as createVerifierEnvironment, va as CombinatorShape, vi as AgenticTool, vn as CollectedAgentTurn, vo as LeaderboardBenchmarkAdapter, vr as mapSandboxEvent, vs as DEFAULT_AWAIT_EVENT_TIMEOUT_MS, vt as ProgressSample, w as superviseSurface, wa as EqualKOnCostOptions, wi as Strategy, wn as EvolutionArchiveNode, wo as LeaderboardSpec, wr as AcquireOptions, wt as allOf, x as SurfaceWorkerConfig, xa as CorpusRecord, xi as RunAgenticOptions, xn as streamAgentTurn, xo as LeaderboardRunContext, xr as CriuCapableClient, xt as ProgressView, y as SuperviseSurfaceOptions, ya as Corpus, yi as ArtifactHandle, yn as StreamAgentTurnOptions, yo as LeaderboardFlagSpec, yr as mapSandboxToolEvent, ys as DownMessageEvent, yt as ProgressTracker, z as RunContext, za as PipelineStage, zi as LoopCampaignDispatchOptions, zn as authorStrategy, zo as LeaderboardRow, zr as InMemoryCorpus, zt as asAuthoredProfile } from "./index-DruSKs3Z.js";
|
|
4
|
+
import { S as ToolLoopMessageRecord, _ as routerChatWithUsage, b as ToolLoopCompaction, d as RouterConfig, f as RouterToolCall, g as routerChatWithTools, h as routerBrain, l as RouterChatResult, m as ToolSpec, p as RouterToolLoopResult, u as RouterChatToolsResult, v as routerToolLoop, x as ToolLoopCompactionOptions, y as ToolLoopChat } from "./local-harness-Dh8PJ0ot.js";
|
|
5
|
+
export { type AcquireOptions, type ActivityLog, type ActivityNote, type Agent, type AgentEnvironment, type AgentEnvironmentCapabilities, type AgentEnvironmentEvent, type AgentEnvironmentProvider, type AgentEnvironmentProviderRef, type AgentEnvironmentProviderRegistry, type AgentEnvironmentQuery, type AgentEnvironmentStatus, type AgentEnvironmentSummary, type AgentProfile, type AgentProfileRef, type AgentRunSpec, type AgentSession, type AgentSessionRef, type AgentSessionStatus, type AgentSpec, type AgentTurnBackend, type AgentTurnInput, type AgentTurnResult, type AgentTurnUsage, type AgenticOptions, type AgenticRunResult, type AgenticSurface, type AgenticTask, type AgenticTool, type AllWorkersStalledOptions, type AnalystFinding, type AnalystFindingEvent, type AnalystRegistry, type AnytimeReport, type AnytimeStrategySummary, type AnytimeTaskCurve, type ApplyContinuation, type ArtifactHandle, type AssertTraceDerivedFindings, type AuditIntentInput, type AuditIntentOptions, type AuthorStrategyOptions, type AuthoredHarness, type AuthoredProfile, type AuthoredStrategy, type AxisScoresOf, type BenchmarkCell, type BenchmarkConfig, type BenchmarkLift, type BenchmarkReport, type BenchmarkStrategySummary, type BenchmarkTaskRow, type BridgeSeam, type Budget, type BudgetPool, type BudgetReadout, type BusEvent, type BusRecord, type BusStats, type ChampionPick, type ChampionPolicy, type CheckExecChannel, type CheckOutcome, type CheckRunContext, type CheckRunner, type CheckSource, type CheckSourceCtx, type CheckpointCapableBox, type CheckpointRef, type CheckpointRequest, type CliSeam, type CliWorktreeBridgeSeam, type CliWorktreeSeam, type CollectedAgentTurn, type CombinatorShape, type CompletionAnalyst, type CompletionEvidence, type CompletionPolicy, type CompletionVerdict, type ConcurrencyCaps, type CoordinationEvent, type CoordinationMcpHandle, type Corpus, type CorpusFilter, type CorpusReadbackOptions, type CorpusRecord, type CreateAgentEnvironmentInput, type CreateSandboxOptions, type CreateScopeAnalystOptions, type CreateTangleSandboxExactProcessProviderOptions, type CriuCapableClient, DEFAULT_AWAIT_EVENT_TIMEOUT_MS, DEFAULT_SANDBOX_STEERING_MAX_TURNS, DEFAULT_STALL_AFTER_MS, type DefaultVerdict, type DefinePersona, type DefinePersonaInput, type DefinedLeaderboard, type DelegateOptions, type Deliverable, type DeliverableSpec, type DispatchReport, type DispatchStopReason, type DispatchUnit, type DownMessageEvent, type DriveHarness, type Driver, type DriverAgentOptions, type DumbDriverOptions, type Environment, type EqualKArm, type EqualKOnCost, type EqualKOnCostOptions, type EqualKVerdict, type EventBus, type EvolutionArchiveNode, type EvolutionAuthor, type EvolutionBandInfo, type EvolutionCandidate, type EvolutionGeneration, type EvolutionReport, type ExecCtx, type ExecRequest, type ExecResult, type Executor, type ExecutorConfig, type ExecutorContext, type ExecutorFactory, type ExecutorProgress, type ExecutorRegistry, type ExecutorResult, type Fanout, type FanoutOptions, type FanoutSynthesis, type FanoutWinnerSelector, FileCorpus, FileResultBlobStore, FileSpawnJournal, type FlatWidenGate, type ForkCapableBox, type ForkRequest, type GitWorkspaceOptions, type GroupOf, type Handle, type HarvestCorpusOptions, type HarvestFailure, type HarvestReport, InMemoryCorpus, InMemoryResultBlobStore, type InMemoryRunContext, type InMemoryRunContextOptions, InMemorySpawnJournal, type InProcessOnPrompt, type InProcessPromptCtx, type InProcessSandboxClientOptions, type Inbox, type InboxMessage, type IntentAudit, type Interval, type Iteration, type KeyProvider, type Leaderboard, type LeaderboardBenchScore, type LeaderboardBenchTask, type LeaderboardBenchmarkAdapter, type LeaderboardFlagSpec, type LeaderboardIterationInfo, type LeaderboardOptions, type LeaderboardRow, type LeaderboardRunContext, type LeaderboardScenario, type LeaderboardScore, type LeaderboardSpec, type LocalMcpMaterialization, type LocalSandboxClientOptions, type LoopCampaignDispatchOptions, type LoopDecisionPayload, type LoopDispatchOptions, type LoopEndedPayload, type LoopIterationDispatchPayload, type LoopIterationEndedPayload, type LoopIterationStartedPayload, type LoopLineageOptions, type LoopOptionsForDispatch, type LoopPlanDescription, type LoopPlanPayload, type LoopResult, type LoopSandboxPlacement, type LoopShape, type LoopStartedPayload, type LoopTeardownFailedPayload, type LoopTokenUsage, type LoopTraceEmitter, type LoopTraceEvent, type LoopUntil, type LoopUntilSpec, type LoopUntilState, type LoopWinner, type MakeWorkerAgent, type MaterializeLocalMcpOptions, type McpEndpoint, type McpEnvironmentOptions, McpSpawnFault, type McpToolDescriptor, type MountManifestEntry, type MountRecorder, type NaiveDriverOptions, type NoProgressForOptions, type NodeId, type NodeSnapshot, type NodeStatus, type Observation, type ObserveInput, type ObserveOptions, type OpenSandboxRunBeforeStartContext, type OpenSandboxRunOptions, type OpenSandboxRunPromptOptions, type Outcome, type OutputAdapter, PI_RUNTIME, type PairwiseOptions, type PairwiseVerdict, type Panel, type PanelJudge, type PanelSpec, type PanelVerdict, type PatchDeliverableOptions, type PendingWait, type Persona, type PersonaContext, type PersonaExecutors, type PiSeam, type Pipeline, type PipelineStage, type PlacementInfo, type PlateauOptions, type ProfileKeyOf, type ProfileRichness, type ProfileRichnessThresholds, type ProgressSample, type ProgressTracker, type ProgressTrackerOptions, type ProgressView, type PromotionGateOptions, type PromotionVerdict, type ProviderAsSandboxClientOptions, type ProviderExecutorOptions, type ProviderSeam, type PublishOptions, type RegistryAnalyzeProjection, type RenderCorpusToInstructions, type RenderCorpusToInstructionsOptions, type RepairStop, type ReproductionCheck, type ReservationTicket, type ResolveSandboxClientOptions, type ResourceRequest, type Restart, type ResultBlobStore, type ResumedWork, type RollingDispatchOptions, type RootHandle, type RootSignal, type RouterChatResult, type RouterChatToolsResult, type RouterConfig, type RouterSeam, type RouterToolCall, type RouterToolLoopResult, type RouterToolsSeam, type RunAgentRoundsOptions, type RunAgenticOptions, type RunContext, type RunLoopOptions, type RunPersonified, type RunPersonifiedOptions, type RunProvenance, type Runtime, type SandboxCapabilities, type SandboxClient, type SandboxClientProviderOptions, type SandboxControlClient, type SandboxEvent, type SandboxInstance, type SandboxLineage, type SandboxLineageHandle, type SandboxRun, SandboxRunAbortError, type SandboxSeam, type SandboxSteeringOptions, type SandboxToolPartState, type Scope, type ScopeAnalyst, type ScopeAnalyzeInput, type ScopeArgs, type ScopeProgressInput, type ScopeWidenGate, type ScoreOf, type SelectionReceipt, type SessionCapableBox, type SessionMessageLike, type SessionTraceBox, type Settled, type ShapeBudget, type ShapeContext, type ShapeRegistry, type Shell, type ShotPersona, type ShotSpec, type SpawnEvent, type SpawnJournal, type SpawnOpts, type Spend, type StdioMcpConnection, type StdioMcpServerSpec, type SteerContext, type SteerableSandboxArgs, type SteerableSandboxSession, type SteeringDecision, type StopDecision, type StopRule, type Strategy, type StrategyArtifacts, type StrategyCtx, type StrategyEvolutionConfig, type StrategyMessage, type StrategyResult, type StrategyShotResult, type StreamAgentTurnOptions, type StructuralRolloutConfig, type StructuralRolloutMessage, type StructuralRolloutPolicy, type StructuralRolloutResult, type SuperviseOptions, type SuperviseSurfaceOptions, type SuperviseSurfaceResult, type SupervisedResult, type Supervisor, type SupervisorAgentDeps, type SupervisorOpts, type SupervisorProfile, type SurfaceScore, type SurfaceWorkerConfig, type SurfaceWorkerOut, type ToolLoopChat, type ToolLoopCompaction, type ToolLoopCompactionOptions, type ToolLoopMessageRecord, type ToolSpec, type ToolStepInput, type TraceSource, type TrajectoryAnalysis, type TrajectoryNode, type TrajectoryReport, type TrajectoryReportFn, type TrajectoryReportOptions, type TreeView, type TurnResult, type UsageEvent, type ValidationCtx, type Validator, type VerifierEnvironmentOptions, type Verify, type VerifySpec, type VisibleCheck, type WaitOpts, type WaitOutcome, type WaitProbe, type WaitProbeRegistry, type WaitRejection, type WaitSpec, type WatchTraceOptions, type WaterfallCollector, type WaterfallReport, type WaterfallSpan, type Widen, type WidenDecision, type WidenGate, type WidenLineage, type WidenSpec, type WinnerStrategy, type WorkerProgress, type Workspace, type WorkspaceCommit, type WorkspaceRequest, type WorkspaceRun, type WorktreeCheckRunner, type WorktreeCliExecutorOptions, type WorktreeCommandResult, type WorktreeFanoutOptions, type WorktreeHarnessResult, type WorktreePatchArtifact, type WorktreeProfileMaterializationReceipt, acquireSandbox, adaptiveRefine, allOf, allWorkersStalled, analyzeTrace, anyOf, anytimeReport, areaUnderCurve, asAuthoredProfile, assertModelAllowed, assertStrategyContract, assertTraceDerivedFindings, assessAuthoredProfile, auditIntent, authorStrategy, authoredWorker, bestSoFar, breadthStrategy, buildSteerContext, builtinShapes, canDisplace, cliWorktreeExecutor, collectAgentTurn, compareCheckOutcomes, completionAuthorizes, composeCheckSources, computeFindingId, connectStdioMcp, contentAddress, createActivityLog, createAgentEnvironmentProviderRegistry, createBudgetPool, createEventBus, createExecutor, createExecutorRegistry, createFileRunContext, createInMemoryRunContext, createInbox, createMcpEnvironment, createProgressTracker, createPushTraceSource, createSandboxLineage, createSandboxToolPartState, createScope, createScopeAnalyst, createShapeRegistry, createSteerableSandboxSession, createSupervisor, createTangleSandboxExactProcessProvider, createVerifierEnvironment, createWaitProbes, createWaterfallCollector, createWorktreeCliExecutor, decodeToolPart, defaultAnalystInstruction, defaultAuditorInstruction, defaultDelegateBudget, defaultExtractCandidate, defaultProfileRichnessThresholds, defaultSelectWinner, defaultStructuralRolloutPolicy, defaultToolDetectors, defineLeaderboard, definePersona, defineStrategy, delegate, depthStrategy, deterministicCompletion, discriminatingMeans, driverAgent, dumbDriver, effectiveConcurrency, envKeyProvider, equalKOnCost, extractLlmCallEvent, failuresAnalyst, fanout, filterAuthoredAsserts, finalizeBestDelivered, flatWidenGate, freeSlots, gateOnDeliverable, gitWorkspace, harvestCorpus, inProcessSandboxClient, inlineSandboxClient, isWaitOutcome, jjWorkspace, leaderboard, localSandboxClient, localShell, loopCampaignDispatch, loopDispatch, loopUntil, makeFinding, mapSandboxEvent, mapSandboxToolEvent, materializeLocalMcp, materializeTreeView, mcpSecretEnvMetadataKey, modelAuthoredChecks, naiveDriver, noProgressFor, observe, officialChecksFromMeta, openSandboxRun, pairwiseSignificance, panel, patchDelivered, pendingWaits, piExecutor, piSeamKey, pickChampion, pipeline, plateau, plateauLength, pollFor, printBenchmarkReport, probeSandboxCapabilities, profileRichnessFinding, promotionGate, providerAsExecutor, providerAsSandboxClient, queueOf, readWorkerProgress, refine, registerShape, registryScopeAnalyst, renderAnytimeTable, renderCorpusToInstructions, renderLeaderboardHtml, renderLeaderboardMarkdown, renderLeaderboardSvg, renderPairwiseMarkdown, renderReport, replaySpawnTree, resolveAgentEnvironmentProvider, resolveEntrySymbol, resolveSandboxClient, resolveSecretEnv, rollingDispatch, routerBrain, routerChatWithTools, routerChatWithUsage, routerToolLoop, runAgentRounds, runAgentic, runBenchmark, runInWorkspace, runLoop, runPersonified, runStrategyEvolution, sample, sampleFromSettled, sampleThenRefine, sandboxCheckRunner, sandboxClientAsProvider, sandboxSessionTraceSource, sanitizeMcpToolSchema, secretEnvOfMcpServer, selectBestIndex, selectChampion, selectValidWinner, sentinelCompletion, serveCoordinationMcp, settledToIteration, spendFromUsageEvents, stopSentinel, strategyAuthorContract, streamAgentTurn, structuralRollout, sumSandboxUsage, supervise, superviseSurface, supervisorAgent, supervisorInstructions, timerAt, trajectoryReport, validateWaitSpec, verify, visibleCheckScore, waitUntil, watchTrace, widen, workerFromBackend, worktreeFanout };
|