@tangle-network/agent-runtime 0.68.0 → 0.70.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +81 -13
- package/dist/agent.d.ts +4 -5
- package/dist/agent.js +3 -6
- package/dist/agent.js.map +1 -1
- package/dist/chunk-BYZCXQHF.js +474 -0
- package/dist/chunk-BYZCXQHF.js.map +1 -0
- package/dist/{chunk-OYU5LR4T.js → chunk-EDCVUZZC.js} +2 -2
- package/dist/{chunk-QNUEFNU7.js → chunk-L5ZFBVT6.js} +6 -6
- package/dist/chunk-L5ZFBVT6.js.map +1 -0
- package/dist/{chunk-VKUHUFX7.js → chunk-QXWGSDAQ.js} +914 -590
- package/dist/chunk-QXWGSDAQ.js.map +1 -0
- package/dist/{chunk-LTEF2LYD.js → chunk-ZNQVMMR5.js} +3 -3
- package/dist/{coordination-Curpzeyc.d.ts → coordination-C7WxwHXq.d.ts} +8 -1
- package/dist/{delegates-CLFNAKyi.d.ts → delegates-DqAgo32T.d.ts} +105 -10
- package/dist/{improvement-adapter-BC4HhuAR.d.ts → improvement-adapter-BVuMragr.d.ts} +1 -1
- package/dist/index.d.ts +439 -100
- package/dist/index.js +583 -125
- package/dist/index.js.map +1 -1
- package/dist/intelligence.d.ts +2 -3
- package/dist/{loop-runner-bin-B6dzNZC8.d.ts → loop-runner-bin-B0NeLTRd.d.ts} +4 -4
- package/dist/loop-runner-bin.d.ts +7 -10
- package/dist/loop-runner-bin.js +4 -7
- package/dist/loops.d.ts +3149 -16
- package/dist/loops.js +34 -63
- package/dist/mcp/bin.js +14 -16
- package/dist/mcp/bin.js.map +1 -1
- package/dist/mcp/index.d.ts +8 -12
- package/dist/mcp/index.js +16 -19
- package/dist/mcp/index.js.map +1 -1
- package/dist/{openai-tools-CA2N3-Ak.d.ts → openai-tools-DPx9Gzvn.d.ts} +1 -1
- package/dist/profiles.d.ts +94 -6
- package/dist/profiles.js +320 -9
- package/dist/profiles.js.map +1 -1
- package/dist/{router-client-30Y_pca8.d.ts → router-client-C7kp_ECN.d.ts} +37 -1
- package/dist/{substrate-CUgk7F7s.d.ts → substrate-BoRXgvka.d.ts} +52 -1
- package/dist/{types-Crxftafi.d.ts → types-BYa2ZOAx.d.ts} +83 -2
- package/dist/{types-p8dWBIXL.d.ts → types-DJu6TBGp.d.ts} +2 -16
- package/dist/{worktree-fanout-DUiKPApb.d.ts → worktree-fanout-gNfl0Byj.d.ts} +9 -37
- package/package.json +9 -39
- package/dist/analyst-loop.d.ts +0 -52
- package/dist/analyst-loop.js +0 -11
- package/dist/analyst-loop.js.map +0 -1
- package/dist/audit.d.ts +0 -93
- package/dist/audit.js +0 -312
- package/dist/audit.js.map +0 -1
- package/dist/chunk-4B6U4CVQ.js +0 -15
- package/dist/chunk-4B6U4CVQ.js.map +0 -1
- package/dist/chunk-FCIPUALH.js +0 -214
- package/dist/chunk-FCIPUALH.js.map +0 -1
- package/dist/chunk-O2UPHN7X.js +0 -114
- package/dist/chunk-O2UPHN7X.js.map +0 -1
- package/dist/chunk-P5OKDSLB.js +0 -580
- package/dist/chunk-P5OKDSLB.js.map +0 -1
- package/dist/chunk-QNUEFNU7.js.map +0 -1
- package/dist/chunk-VKUHUFX7.js.map +0 -1
- package/dist/chunk-VLF5RHEQ.js +0 -143
- package/dist/chunk-VLF5RHEQ.js.map +0 -1
- package/dist/coder-2leJPOvC.d.ts +0 -52
- package/dist/improvement.d.ts +0 -208
- package/dist/improvement.js +0 -343
- package/dist/improvement.js.map +0 -1
- package/dist/local-harness-BE_h8szs.d.ts +0 -93
- package/dist/platform.d.ts +0 -255
- package/dist/platform.js +0 -229
- package/dist/platform.js.map +0 -1
- package/dist/run-loop-D3PwlG7J.d.ts +0 -112
- package/dist/runtime-hooks-C7JwKb9E.d.ts +0 -70
- package/dist/runtime.d.ts +0 -3108
- package/dist/runtime.js +0 -263
- package/dist/runtime.js.map +0 -1
- package/dist/topology.d.ts +0 -126
- package/dist/topology.js +0 -333
- package/dist/topology.js.map +0 -1
- package/dist/workflow.d.ts +0 -551
- package/dist/workflow.js +0 -1781
- package/dist/workflow.js.map +0 -1
- /package/dist/{chunk-OYU5LR4T.js.map → chunk-EDCVUZZC.js.map} +0 -0
- /package/dist/{chunk-LTEF2LYD.js.map → chunk-ZNQVMMR5.js.map} +0 -0
package/dist/loops.d.ts
CHANGED
|
@@ -1,18 +1,3151 @@
|
|
|
1
|
+
import { ChatClient, AnalystFinding, AgentProfile, AnalystRunInputs, ToolSpan, StreamingDetector, DetectorSignal, buildTrajectory } from '@tangle-network/agent-eval';
|
|
2
|
+
export { AnalystFinding, DefaultVerdict, computeFindingId, makeFinding } from '@tangle-network/agent-eval';
|
|
3
|
+
import { SandboxInstance, CreateSandboxOptions, SandboxEvent, BackendType } from '@tangle-network/sandbox';
|
|
1
4
|
export { AgentProfile, CreateSandboxOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox';
|
|
2
|
-
|
|
3
|
-
export {
|
|
4
|
-
|
|
5
|
-
export {
|
|
6
|
-
|
|
7
|
-
export {
|
|
8
|
-
|
|
9
|
-
export {
|
|
10
|
-
import '
|
|
11
|
-
import '
|
|
12
|
-
import '
|
|
13
|
-
import '
|
|
14
|
-
|
|
5
|
+
import { g as ResultBlobStore, i as SpawnJournal, N as NodeId, bb as SpawnEvent, bc as ExecutorFactory, A as Agent, B as Budget, S as Scope, l as Settled, k as SupervisedResult, m as Spend, bd as UsageEvent, E as ExecutorRegistry, L as LocalHarness, be as Supervisor } from './delegates-DqAgo32T.js';
|
|
6
|
+
export { h as AgentSpec, n as Executor, bf as ExecutorContext, bg as ExecutorResult, bh as SupervisorOpts, bi as TreeView, bj as WidenGate } from './delegates-DqAgo32T.js';
|
|
7
|
+
import { M as MakeWorkerAgent, a as CoordinationTools, A as AnalystRegistry, C as CoordinationEvent, j as QuestionPolicy } from './coordination-C7WxwHXq.js';
|
|
8
|
+
export { B as BusEvent, p as BusRecord, q as BusStats, E as EventBus, P as PublishOptions, r as createEventBus } from './coordination-C7WxwHXq.js';
|
|
9
|
+
import { b as RuntimeHooks, I as Iteration, S as SandboxClient, T as Driver, A as AgentRunSpec, a as OutputAdapter, V as Validator, U as ExecCtx, W as LoopWinner, X as LoopLineageOptions, Y as LoopResult, d as LoopTokenUsage, R as RuntimeStreamEvent } from './types-BYa2ZOAx.js';
|
|
10
|
+
export { Z as LoopDecisionPayload, _ as LoopEndedPayload, $ as LoopIterationDispatchPayload, a0 as LoopIterationEndedPayload, a1 as LoopIterationStartedPayload, a2 as LoopPlanDescription, a3 as LoopPlanPayload, c as LoopSandboxPlacement, a4 as LoopStartedPayload, a5 as LoopTeardownFailedPayload, e as LoopTraceEmitter, L as LoopTraceEvent, a6 as ValidationCtx } from './types-BYa2ZOAx.js';
|
|
11
|
+
import { C as CorpusRecord, c as Corpus, O as Outcome, S as ScopeAnalyzeInput, d as AssertTraceDerivedFindings, e as SteerContext, f as ScopeAnalyst, F as FanoutOptions, g as CombinatorShape, h as ScopeWidenGate, L as LoopUntilSpec, P as PanelSpec, i as PipelineStage, W as WinnerStrategy, j as FanoutWinnerSelector, V as VerifySpec, k as WidenSpec, l as CorpusFilter, R as RenderCorpusToInstructionsOptions, D as DefinePersonaInput, m as Persona, n as RunPersonifiedOptions, o as ShapeRegistry, p as LoopShape, E as EqualKArm, q as EqualKOnCostOptions, r as EqualKVerdict, T as TrajectoryReportOptions, s as TrajectoryReport, t as DeliverableSpec } from './worktree-fanout-gNfl0Byj.js';
|
|
12
|
+
export { A as AuthoredHarness, u as DefinePersona, v as EqualKOnCost, w as Fanout, x as FanoutSynthesis, y as FlatWidenGate, z as LoopUntil, B as LoopUntilState, G as Panel, H as PanelJudge, I as PanelVerdict, J as PatchDeliverableOptions, K as PersonaContext, M as PersonaExecutors, N as Pipeline, Q as RenderCorpusToInstructions, U as RunPersonified, X as ShapeBudget, Y as ShapeContext, Z as TrajectoryNode, _ as TrajectoryReportFn, $ as Verify, a0 as Widen, a1 as WidenDecision, a2 as WidenLineage, a3 as WorktreeCliExecutorOptions, a4 as WorktreeCommandResult, a as WorktreeFanoutOptions, b as WorktreePatchArtifact, a5 as createWorktreeCliExecutor, a6 as gateOnDeliverable, a7 as patchDelivered, a8 as worktreeFanout } from './worktree-fanout-gNfl0Byj.js';
|
|
13
|
+
import { Scenario, ProfileDispatchFn } from '@tangle-network/agent-eval/campaign';
|
|
14
|
+
import { A as AnalystRegistryLike } from './types-DJu6TBGp.js';
|
|
15
|
+
import { AgentProfile as AgentProfile$1 } from '@tangle-network/agent-interface';
|
|
16
|
+
import { R as RouterConfig, a as ToolLoopChat, T as ToolSpec } from './router-client-C7kp_ECN.js';
|
|
17
|
+
export { b as RouterChatResult, c as RouterChatToolsResult, d as RouterToolCall, e as RouterToolLoopResult, r as routerBrain, f as routerChatWithTools, g as routerChatWithUsage, h as routerToolLoop } from './router-client-C7kp_ECN.js';
|
|
18
|
+
import { stuckLoopView, toolWasteView } from '@tangle-network/agent-eval/pipelines';
|
|
15
19
|
import 'node:child_process';
|
|
16
|
-
import '
|
|
17
|
-
|
|
18
|
-
|
|
20
|
+
import './substrate-BoRXgvka.js';
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* @experimental
|
|
24
|
+
*
|
|
25
|
+
* Event-sourced spawn journal for the recursive execution atom (build steps 3 + 7).
|
|
26
|
+
*
|
|
27
|
+
* The supervision tree is journaled as an append-only event log: every `spawned`,
|
|
28
|
+
* `settled`, and `cancelled` is recorded AFTER it is observed-committed (never
|
|
29
|
+
* speculative), mirroring `ConversationJournal`'s begin/append/load shape. The log
|
|
30
|
+
* holds only the THIN decision record — ids, parentage, budget, the spend a decision
|
|
31
|
+
* consumed, and a content-addressed `outRef`. The payloads the driver branched on
|
|
32
|
+
* (the `out` artifacts) live in a separate `ResultBlobStore`, keyed by `outRef`, so
|
|
33
|
+
* the journal stays small (decisions) and replay rehydrates the exact `Settled` from
|
|
34
|
+
* the blob store (evidence). This is the decision/payload split the replay argument
|
|
35
|
+
* rests on (B1/B2).
|
|
36
|
+
*
|
|
37
|
+
* Replay determinism (B2): `seq` is the monotonic cursor order `scope.next()` yielded
|
|
38
|
+
* each settlement — NOT wall-clock. `replaySpawnTree` sorts strictly by `seq` before
|
|
39
|
+
* touching the blob store, so the order in which rehydration `get`s resolve can never
|
|
40
|
+
* reorder the replayed `Settled[]`; the result is identical regardless of blob latency.
|
|
41
|
+
*/
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Mint the content-addressed `outRef` for a result artifact: `sha256:<hex>` over a
|
|
45
|
+
* stable JSON encoding. Producers call this to derive the `outRef` they journal and
|
|
46
|
+
* `put`; the FS/in-mem stores re-derive it on `put` to verify the supplied ref
|
|
47
|
+
* matches (fail loud on a mismatch — a forged ref breaks the replay invariant).
|
|
48
|
+
*
|
|
49
|
+
* Stable encoding: object keys are sorted recursively so two structurally-equal
|
|
50
|
+
* artifacts hash identically regardless of key insertion order.
|
|
51
|
+
*/
|
|
52
|
+
declare function contentAddress(artifact: unknown): string;
|
|
53
|
+
/**
|
|
54
|
+
* In-memory `ResultBlobStore`. Content-addressed: `put` verifies the supplied
|
|
55
|
+
* `outRef` matches the artifact's hash so a stale/forged ref fails loud rather than
|
|
56
|
+
* silently rehydrating the wrong payload. Idempotent on an identical re-put.
|
|
57
|
+
*/
|
|
58
|
+
declare class InMemoryResultBlobStore implements ResultBlobStore {
|
|
59
|
+
private readonly blobs;
|
|
60
|
+
put(outRef: string, artifact: unknown): Promise<void>;
|
|
61
|
+
get(outRef: string): Promise<unknown | undefined>;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* In-memory `SpawnJournal`. Appends are observed-committed only; the impl enforces
|
|
65
|
+
* the corruption guards a durable replay rests on:
|
|
66
|
+
* - an event before `beginTree` is a corrupted tree (fail loud),
|
|
67
|
+
* - a duplicate `seq` within a tree is a corrupted cursor (fail loud) — two
|
|
68
|
+
* settlements cannot share the cursor position replay orders by.
|
|
69
|
+
*/
|
|
70
|
+
declare class InMemorySpawnJournal implements SpawnJournal {
|
|
71
|
+
private readonly trees;
|
|
72
|
+
loadTree(root: NodeId): Promise<SpawnEvent[] | undefined>;
|
|
73
|
+
beginTree(root: NodeId, at: string): Promise<void>;
|
|
74
|
+
appendEvent(root: NodeId, ev: SpawnEvent): Promise<void>;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* createWaterfallCollector — 100% trajectory observability from the lifecycle stream:
|
|
79
|
+
* every spawn/settle (shots, analysts, nested agents) becomes one timed, billed span.
|
|
80
|
+
* The sum of spans IS the run's cost story — what each step cost in dollars, tokens,
|
|
81
|
+
* and wall-clock, rendered as a text waterfall or exported as structured rows for any
|
|
82
|
+
* chart. Attach the collector's `hooks` to `runAgentic`/`runBenchmark`; spans accumulate
|
|
83
|
+
* across every task the hooks observe.
|
|
84
|
+
*/
|
|
85
|
+
|
|
86
|
+
interface WaterfallSpan {
|
|
87
|
+
id: string;
|
|
88
|
+
/** The spawn label (`shot:0`, `analyst:1`, a nested agent's label) — the row name. */
|
|
89
|
+
label: string;
|
|
90
|
+
runId: string;
|
|
91
|
+
parentId?: string;
|
|
92
|
+
startMs: number;
|
|
93
|
+
endMs?: number;
|
|
94
|
+
status: 'running' | 'done' | 'down';
|
|
95
|
+
usd: number;
|
|
96
|
+
tokens: {
|
|
97
|
+
input: number;
|
|
98
|
+
output: number;
|
|
99
|
+
};
|
|
100
|
+
score?: number;
|
|
101
|
+
}
|
|
102
|
+
interface WaterfallReport {
|
|
103
|
+
spans: WaterfallSpan[];
|
|
104
|
+
/** Wall-clock of the observed window (first spawn → last settle). */
|
|
105
|
+
totalMs: number;
|
|
106
|
+
totalUsd: number;
|
|
107
|
+
totalTokens: {
|
|
108
|
+
input: number;
|
|
109
|
+
output: number;
|
|
110
|
+
};
|
|
111
|
+
/** Rollup by label prefix (the part before ':') — shots vs analysts vs anything else. */
|
|
112
|
+
byKind: Record<string, {
|
|
113
|
+
count: number;
|
|
114
|
+
ms: number;
|
|
115
|
+
usd: number;
|
|
116
|
+
tokens: {
|
|
117
|
+
input: number;
|
|
118
|
+
output: number;
|
|
119
|
+
};
|
|
120
|
+
}>;
|
|
121
|
+
}
|
|
122
|
+
interface WaterfallCollector {
|
|
123
|
+
/** Attach these to RunAgenticOptions.hooks / BenchmarkConfig.hooks. */
|
|
124
|
+
hooks: RuntimeHooks;
|
|
125
|
+
report(): WaterfallReport;
|
|
126
|
+
/** The text waterfall — one row per span, bars scaled to the observed window. */
|
|
127
|
+
render(opts?: {
|
|
128
|
+
width?: number;
|
|
129
|
+
maxRows?: number;
|
|
130
|
+
}): string;
|
|
131
|
+
reset(): void;
|
|
132
|
+
}
|
|
133
|
+
declare function createWaterfallCollector(): WaterfallCollector;
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* anytimeReport — time-to-satisfactory-output metrics, derived entirely from the
|
|
137
|
+
* waterfall's spans (no new instrumentation): per task, the best-so-far score after each
|
|
138
|
+
* shot with its elapsed wall-clock and cumulative spend; per strategy, the standard
|
|
139
|
+
* anytime-optimization metrics:
|
|
140
|
+
*
|
|
141
|
+
* TTT time-to-target — elapsed ms until best-so-far ≥ the target (per task; median
|
|
142
|
+
* over tasks that reached it)
|
|
143
|
+
* STT shots-to-target — attempts until best-so-far ≥ target
|
|
144
|
+
* ERT expected running time (the COCO benchmarking convention): TOTAL time spent
|
|
145
|
+
* across all tasks — including failures' full budgets — divided by the number of
|
|
146
|
+
* tasks that reached the target. The honest "how long per success, all-in".
|
|
147
|
+
* AUC the anytime curve's area (mean best-so-far score across the budget, per shot
|
|
148
|
+
* index) — higher = climbs earlier.
|
|
149
|
+
*
|
|
150
|
+
* The "satisfactory" bar follows the COCO/BBOB convention: a SET of satisficing targets
|
|
151
|
+
* (e.g. [0.5, 0.8, 1.0] on the normalized check score), each measured independently —
|
|
152
|
+
* runtime-to-target per (task, target) pair — optionally overridden per task
|
|
153
|
+
* (`targetFor`) when satisfaction is task-specific. Spans come from
|
|
154
|
+
* `createWaterfallCollector().report()`; tasks are grouped by the supervisor runId
|
|
155
|
+
* (`agentic:<strategy>:<taskId>`); shot spans are `shot:N` labels.
|
|
156
|
+
*/
|
|
157
|
+
|
|
158
|
+
interface AnytimeTaskCurve {
|
|
159
|
+
taskId: string;
|
|
160
|
+
strategy: string;
|
|
161
|
+
/** Best-so-far after each settled shot: elapsed ms from the task's first spawn,
|
|
162
|
+
* cumulative usd, and the running max score. */
|
|
163
|
+
points: Array<{
|
|
164
|
+
elapsedMs: number;
|
|
165
|
+
cumUsd: number;
|
|
166
|
+
best: number;
|
|
167
|
+
}>;
|
|
168
|
+
/** Per satisficing target (keyed by the target value as a string): the first point
|
|
169
|
+
* where best ≥ target, or null when never reached within budget. */
|
|
170
|
+
hits: Record<string, {
|
|
171
|
+
ms: number;
|
|
172
|
+
shots: number;
|
|
173
|
+
usd: number;
|
|
174
|
+
} | null>;
|
|
175
|
+
}
|
|
176
|
+
interface AnytimeStrategySummary {
|
|
177
|
+
strategy: string;
|
|
178
|
+
/** The satisficing target this row summarizes. */
|
|
179
|
+
target: number;
|
|
180
|
+
tasks: number;
|
|
181
|
+
reachedTarget: number;
|
|
182
|
+
/** Median time-to-target over the tasks that reached it (null when none did). */
|
|
183
|
+
medianTttMs: number | null;
|
|
184
|
+
medianShotsToTarget: number | null;
|
|
185
|
+
/** COCO ERT: Σ all task wall-time (incl. failures) / #successes. Null when 0 succeed. */
|
|
186
|
+
ertMs: number | null;
|
|
187
|
+
/** Same construction over dollars: Σ all spend / #successes. */
|
|
188
|
+
erUsd: number | null;
|
|
189
|
+
/** Mean best-so-far score by shot index (the anytime curve, averaged over tasks). */
|
|
190
|
+
curveByShot: number[];
|
|
191
|
+
/** Area under the per-shot anytime curve, normalized to [0,1]. */
|
|
192
|
+
auc: number;
|
|
193
|
+
}
|
|
194
|
+
interface AnytimeReport {
|
|
195
|
+
targets: number[];
|
|
196
|
+
perTask: AnytimeTaskCurve[];
|
|
197
|
+
/** One summary per (strategy, target) pair — the COCO-style multi-target view. */
|
|
198
|
+
perStrategy: AnytimeStrategySummary[];
|
|
199
|
+
}
|
|
200
|
+
/** Derive anytime metrics from waterfall spans. `targets` are the satisficing score
|
|
201
|
+
* bars (default [1] = fully resolved; COCO-style multi-target: [0.5, 0.8, 1]);
|
|
202
|
+
* `targetFor` overrides the bar per task (task-specific satisfaction) — when set, the
|
|
203
|
+
* per-task bar replaces every entry of `targets` for that task. */
|
|
204
|
+
declare function anytimeReport(spans: WaterfallSpan[], opts?: {
|
|
205
|
+
targets?: number[];
|
|
206
|
+
targetFor?: (taskId: string) => number;
|
|
207
|
+
}): AnytimeReport;
|
|
208
|
+
/** One row per (strategy, satisficing target): the shareable time-to-satisfactory table. */
|
|
209
|
+
declare function renderAnytimeTable(report: AnytimeReport): string;
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* auditIntent — the route-rigor analyst: is this trajectory even going the RIGHT WAY?
|
|
213
|
+
*
|
|
214
|
+
* `observe()` critiques execution quality ("what's unfinished"). This audits ALIGNMENT —
|
|
215
|
+
* a different failure class the score can't see until it's too late: an agent can be
|
|
216
|
+
* executing flawlessly down the wrong route. The auditor reads the trajectory and
|
|
217
|
+
* compares three intents:
|
|
218
|
+
*
|
|
219
|
+
* declared — what the task says to do (the prompt / acceptance criteria)
|
|
220
|
+
* revealed — what the agent is ACTUALLY optimizing, inferred from its action pattern
|
|
221
|
+
* (the inverse-inference move: actions reveal objectives)
|
|
222
|
+
* user — what the principal actually wants (the contract, when it differs from
|
|
223
|
+
* the literal task text), plus where the user's own trajectory is heading
|
|
224
|
+
*
|
|
225
|
+
* and returns a verdict (aligned / drifting / diverged) with evidence and ONE
|
|
226
|
+
* recommended intervention. FIREWALLED like every analyst: input is the trajectory and
|
|
227
|
+
* the intents — never the verifier or its data (zero check-leakage, so route auditing
|
|
228
|
+
* is always Goodhart-safe to run online).
|
|
229
|
+
*
|
|
230
|
+
* Where it runs: between shots (steer the next one), as a watchdog over the lifecycle
|
|
231
|
+
* stream (abort-and-refund a diverged rollout — the budget pool makes early abort
|
|
232
|
+
* strictly valuable), or post-hoc over a whole BenchmarkReport (the meta-intent pass:
|
|
233
|
+
* is the LOOP optimizing the right thing — degenerate submissions, check-gaming shapes,
|
|
234
|
+
* objective drift across tasks).
|
|
235
|
+
*/
|
|
236
|
+
|
|
237
|
+
interface AuditIntentInput {
|
|
238
|
+
/** The declared intent: the task text / acceptance criteria the agent was given. */
|
|
239
|
+
declaredIntent: string;
|
|
240
|
+
/** The trajectory so far — tool calls + results + assistant turns (any event shapes). */
|
|
241
|
+
trace: ReadonlyArray<unknown>;
|
|
242
|
+
/** The principal's actual intent when it differs from the literal task (the contract). */
|
|
243
|
+
userIntent?: string;
|
|
244
|
+
/** The loop-level purpose (meta-intent): what the WHOLE run is for — lets the auditor
|
|
245
|
+
* flag locally-sensible work that serves the wrong larger objective. */
|
|
246
|
+
metaIntent?: string;
|
|
247
|
+
runId?: string;
|
|
248
|
+
}
|
|
249
|
+
interface AuditIntentOptions {
|
|
250
|
+
chat: ChatClient;
|
|
251
|
+
model?: string;
|
|
252
|
+
/** Override the auditor instruction (optimizable like any analyst prompt). */
|
|
253
|
+
auditorInstruction?: string;
|
|
254
|
+
/** Cap trace lines fed to the auditor. Default 80. */
|
|
255
|
+
maxTraceLines?: number;
|
|
256
|
+
signal?: AbortSignal;
|
|
257
|
+
}
|
|
258
|
+
interface IntentAudit {
|
|
259
|
+
/** What the agent's actions reveal it is actually optimizing — one sentence. */
|
|
260
|
+
revealedIntent: string;
|
|
261
|
+
verdict: 'aligned' | 'drifting' | 'diverged';
|
|
262
|
+
/** Trajectory-grounded evidence for the verdict (specific calls/patterns). */
|
|
263
|
+
evidence: string;
|
|
264
|
+
/** The single recommended intervention. */
|
|
265
|
+
recommendation: 'continue' | 'steer' | 'abort';
|
|
266
|
+
/** When recommendation is 'steer': the corrective instruction to inject. */
|
|
267
|
+
steer?: string;
|
|
268
|
+
confidence: number;
|
|
269
|
+
}
|
|
270
|
+
declare const defaultAuditorInstruction: string;
|
|
271
|
+
declare function auditIntent(input: AuditIntentInput, opts: AuditIntentOptions): Promise<IntentAudit>;
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* @experimental
|
|
275
|
+
*
|
|
276
|
+
* Completion / satisfiability — the OTHER output of the pluggable analyst (the steer output
|
|
277
|
+
* is `AnalystFinding[]` via the `analyze` hook; this is the "is it done?" output via the
|
|
278
|
+
* `complete` hook). A `CompletionAnalyst` reads a node's trace and returns a `CompletionVerdict`
|
|
279
|
+
* the PARENT (driver) acts on: end the node, or keep going. It fits ANY node and composes to
|
|
280
|
+
* any depth — a 1-deep loop has one; an N-deep tree has one per node.
|
|
281
|
+
*
|
|
282
|
+
* The verdict's authority scales with its DETERMINISM (the thing that varies by task):
|
|
283
|
+
* - `deterministic` — build/test/lint pass, a proof checks, every claim's citation resolves:
|
|
284
|
+
* ground truth, the driver TRUSTS it and ends. Not an opinion.
|
|
285
|
+
* - `probabilistic` — a quality/soundness judgment (marketing, "the experiment is sound"):
|
|
286
|
+
* ADVISORY. It passes to the driver with its reasons; the driver validates (here: a
|
|
287
|
+
* confidence threshold; a richer driver may re-examine the reasons) before ending.
|
|
288
|
+
*
|
|
289
|
+
* Two stop-signal mechanisms, by node mode, both → one `CompletionVerdict`:
|
|
290
|
+
* - sandbox-agent (text stream): a unique per-node STOP SENTINEL the agent emits when done
|
|
291
|
+
* (`stopSentinel` / `sentinelCompletion`) — ralph-loop style; the seed makes it
|
|
292
|
+
* unguessable + attributable, so it can't be spuriously emitted or confused with content.
|
|
293
|
+
* - deterministic check (compile/test/citation/proof): `deterministicCompletion(check)` —
|
|
294
|
+
* a verifier over the output, never the judge verdict (selector ≠ judge holds).
|
|
295
|
+
*/
|
|
296
|
+
|
|
297
|
+
/** Trace-derived evidence for a completion claim — an artifact (output) or a verifier metric,
|
|
298
|
+
* never the judge's own verdict. Mirrors the steer-firewall's provenance discipline. */
|
|
299
|
+
interface CompletionEvidence {
|
|
300
|
+
kind: 'artifact' | 'metric';
|
|
301
|
+
uri: string;
|
|
302
|
+
}
|
|
303
|
+
/** The "is it done?" verdict an analyst returns to the parent. */
|
|
304
|
+
interface CompletionVerdict {
|
|
305
|
+
done: boolean;
|
|
306
|
+
/** How verifiable the claim is — sets whether the driver trusts it or validates it. */
|
|
307
|
+
determinism: 'deterministic' | 'probabilistic';
|
|
308
|
+
/** Why the analyst believes it is (or isn't) done — what the driver validates. */
|
|
309
|
+
reasons?: string;
|
|
310
|
+
/** 0..1, for probabilistic verdicts; the driver's validation threshold reads this. */
|
|
311
|
+
confidence?: number;
|
|
312
|
+
evidence?: ReadonlyArray<CompletionEvidence>;
|
|
313
|
+
}
|
|
314
|
+
/** Reads a node's trace → a completion verdict. Same input shape as the `analyze` hook, so
|
|
315
|
+
* ONE analyst node can back both channels (findings for steer, a verdict for stop). */
|
|
316
|
+
interface CompletionAnalyst<Task, Output> {
|
|
317
|
+
assess(input: {
|
|
318
|
+
task: Task;
|
|
319
|
+
history: ReadonlyArray<Iteration<Task, Output>>;
|
|
320
|
+
}): CompletionVerdict | Promise<CompletionVerdict>;
|
|
321
|
+
}
|
|
322
|
+
/** When a verdict authorizes the driver to END. Deterministic → trust (ground truth);
|
|
323
|
+
* probabilistic → validate by confidence threshold (the driver's check). */
|
|
324
|
+
interface CompletionPolicy {
|
|
325
|
+
/** Minimum confidence a PROBABILISTIC verdict must clear to end. Default 0.8. */
|
|
326
|
+
minConfidence?: number;
|
|
327
|
+
}
|
|
328
|
+
declare function completionAuthorizes(v: CompletionVerdict, policy?: CompletionPolicy): boolean;
|
|
329
|
+
/**
|
|
330
|
+
* A unique, attributable stop sentinel for a node (ralph-loop style). Deterministic from the
|
|
331
|
+
* seed (no Math.random — reproducible + attributable to the node); the agent is instructed to
|
|
332
|
+
* emit it VERBATIM when it judges itself done. Unguessable enough that content never trips it.
|
|
333
|
+
*/
|
|
334
|
+
declare function stopSentinel(seed: string): string;
|
|
335
|
+
/**
|
|
336
|
+
* Completion for a sandbox-agent node: done iff the latest output carries the node's stop
|
|
337
|
+
* sentinel. PROBABILISTIC (the agent's own self-judgment) — the driver validates it.
|
|
338
|
+
*/
|
|
339
|
+
declare function sentinelCompletion<Task>(sentinel: string, opts?: {
|
|
340
|
+
confidence?: number;
|
|
341
|
+
}): CompletionAnalyst<Task, string>;
|
|
342
|
+
/**
|
|
343
|
+
* Completion for a DETERMINISTIC check (build/test/lint/citation/proof): done iff the check
|
|
344
|
+
* passes. Ground truth — the driver ends directly, no validation. The check reads the output
|
|
345
|
+
* (a verifier), never the judge verdict — selector ≠ judge stays intact.
|
|
346
|
+
*/
|
|
347
|
+
declare function deterministicCompletion<Task, Output>(check: (output: Output, history: ReadonlyArray<Iteration<Task, Output>>) => {
|
|
348
|
+
passed: boolean;
|
|
349
|
+
reasons?: string;
|
|
350
|
+
}): CompletionAnalyst<Task, Output>;
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* The third-person observer — the connective tissue that closes the loop.
|
|
354
|
+
*
|
|
355
|
+
* A driver spawns a worker; the worker can't see itself. `observe` reads the
|
|
356
|
+
* worker's TRACE (what it actually did — every tool call, cost, failure) and
|
|
357
|
+
* produces two streams:
|
|
358
|
+
* - `findings` / `report` — fed back DOWN (a steer for the next attempt) and
|
|
359
|
+
* OUT (the operator-facing "what I noticed + what to change").
|
|
360
|
+
* - `learned` — durable facts written to the cross-run `Corpus` so the NEXT
|
|
361
|
+
* run starts smarter (the continuous half of "continuous self-improvement").
|
|
362
|
+
*
|
|
363
|
+
* Findings are TRACE-derived, never JUDGE-derived (`derived_from_judge:false`):
|
|
364
|
+
* the observer reads behavior, never the acceptance verdict — the selector≠judge
|
|
365
|
+
* firewall (docs/learning-flywheel.md). The observer is harness-agnostic: it
|
|
366
|
+
* reads a trace + an output, so it watches opencode, codex, hermes, or a BYO
|
|
367
|
+
* agent identically.
|
|
368
|
+
*/
|
|
369
|
+
|
|
370
|
+
interface ObserveInput {
|
|
371
|
+
/** What the worker was asked to do. */
|
|
372
|
+
task: string;
|
|
373
|
+
/** What it produced (its final answer / artifact summary). */
|
|
374
|
+
output: string;
|
|
375
|
+
/** The worker's trace — any event array (sandbox events, tool-call records). */
|
|
376
|
+
trace: ReadonlyArray<unknown>;
|
|
377
|
+
/** Terminal status only (passed/failed/unknown) — NOT a judge score; the
|
|
378
|
+
* observer never reads the verdict, it reads behavior. */
|
|
379
|
+
outcome?: 'passed' | 'failed' | 'unknown';
|
|
380
|
+
/** Provenance back to the run. */
|
|
381
|
+
runId?: string;
|
|
382
|
+
}
|
|
383
|
+
interface ObserveOptions {
|
|
384
|
+
/** The model-call seam (agent-eval `createChatClient`: router / cli-bridge / …). */
|
|
385
|
+
chat: ChatClient;
|
|
386
|
+
model?: string;
|
|
387
|
+
/** When set, learned facts are appended (idempotent) for the next run to read. */
|
|
388
|
+
corpus?: Corpus;
|
|
389
|
+
/** Tags written onto learned facts + used by the next run's corpus query. */
|
|
390
|
+
tags?: ReadonlyArray<string>;
|
|
391
|
+
signal?: AbortSignal;
|
|
392
|
+
/** Cap the trace lines fed to the observer (keeps the call cheap). Default 80. */
|
|
393
|
+
maxTraceLines?: number;
|
|
394
|
+
/** Override the analyst's system instruction — the prompt that turns a trace into
|
|
395
|
+
* findings + recommended_actions. The analyst IS the steerer, so this is the knob a
|
|
396
|
+
* prompt optimizer (GEPA) tunes. Omitted ⇒ the default observer instruction. The
|
|
397
|
+
* firewall (trace-only, never the verdict) is structural (input has no score), so a
|
|
398
|
+
* custom instruction cannot break it. */
|
|
399
|
+
analystInstruction?: string;
|
|
400
|
+
}
|
|
401
|
+
/** The default observer instruction — exported so an optimizer can seed its population. */
|
|
402
|
+
declare const defaultAnalystInstruction: string;
|
|
403
|
+
interface Observation {
|
|
404
|
+
findings: AnalystFinding[];
|
|
405
|
+
/** Facts persisted to the corpus (empty when no corpus was supplied). */
|
|
406
|
+
learned: CorpusRecord[];
|
|
407
|
+
/** Operator-facing markdown: what the observer noticed + what to change. */
|
|
408
|
+
report: string;
|
|
409
|
+
}
|
|
410
|
+
declare function observe(input: ObserveInput, opts: ObserveOptions): Promise<Observation>;
|
|
411
|
+
/** Operator-facing report, split by who should act. The agent block is the
|
|
412
|
+
* steer; the operator block is the advice. */
|
|
413
|
+
declare function renderReport(findings: ReadonlyArray<AnalystFinding>): string;
|
|
414
|
+
|
|
415
|
+
/**
|
|
416
|
+
* harvestCorpus — production traces → corpus, the G2 bridge (the playbook's step 6).
|
|
417
|
+
* The flywheel's write side, batched: run the firewalled `observe()` analyst over a
|
|
418
|
+
* stream of completed runs (yesterday's production traces, a benchmark's rollouts, a
|
|
419
|
+
* fleet's day) and accrete the trace-derived facts into the durable corpus.
|
|
420
|
+
*
|
|
421
|
+
* Store-agnostic by design: the caller maps its trace store's rows (a
|
|
422
|
+
* `ProductionTraceSink` ndjson, OTLP spans, RunRecords) to `ObserveInput` — task text,
|
|
423
|
+
* final output, the event trace, terminal outcome. The analyst reads BEHAVIOR only
|
|
424
|
+
* (the firewall is structural: the input carries no judge verdict), and corpus appends
|
|
425
|
+
* are idempotent on (claim + tags), so re-harvesting the same window is safe.
|
|
426
|
+
*
|
|
427
|
+
* The nightly product job is then three lines:
|
|
428
|
+
* const runs = mapSinkRowsToObserveInputs(await readSink(yesterday))
|
|
429
|
+
* const report = await harvestCorpus({ runs, chat, corpus, tags: ['gtm-agent'] })
|
|
430
|
+
* log(report) // runsObserved / findings / learned / failures
|
|
431
|
+
*
|
|
432
|
+
* NOTE on the read side: harvesting is safe and cheap; *injecting* facts back into runs
|
|
433
|
+
* is the measured danger zone — naive unconditional priming tested NEGATIVE (−11.6pp,
|
|
434
|
+
* context pollution; result now in .evolve/current.json + memory). Gate any priming design on its
|
|
435
|
+
* own A/B; the corpus's first consumers are operators and optimizers, not prompts.
|
|
436
|
+
*/
|
|
437
|
+
|
|
438
|
+
interface HarvestCorpusOptions {
|
|
439
|
+
/** The completed runs to analyze — map your store's rows to `ObserveInput`. */
|
|
440
|
+
runs: AsyncIterable<ObserveInput> | Iterable<ObserveInput>;
|
|
441
|
+
/** The model-call seam (agent-eval `createChatClient`). */
|
|
442
|
+
chat: ChatClient;
|
|
443
|
+
model?: string;
|
|
444
|
+
/** The durable corpus the facts accrete into. */
|
|
445
|
+
corpus: Corpus;
|
|
446
|
+
/** Tags written onto learned facts (the product/domain key the read side queries by). */
|
|
447
|
+
tags?: ReadonlyArray<string>;
|
|
448
|
+
/** Override the analyst instruction (the GEPA-tunable knob). */
|
|
449
|
+
analystInstruction?: string;
|
|
450
|
+
/** Runs analyzed in parallel. Default 4. */
|
|
451
|
+
concurrency?: number;
|
|
452
|
+
/** Hard cap on runs consumed from the stream (a cost guard for unbounded stores). */
|
|
453
|
+
maxRuns?: number;
|
|
454
|
+
signal?: AbortSignal;
|
|
455
|
+
}
|
|
456
|
+
interface HarvestFailure {
|
|
457
|
+
runId: string;
|
|
458
|
+
error: string;
|
|
459
|
+
}
|
|
460
|
+
interface HarvestReport {
|
|
461
|
+
runsObserved: number;
|
|
462
|
+
/** Total findings the analyst produced (including ones already known). */
|
|
463
|
+
findings: number;
|
|
464
|
+
/** NEW facts actually appended (idempotent dedup excludes re-learned ones). */
|
|
465
|
+
learned: number;
|
|
466
|
+
/** Per-run analysis failures — reported, never silently dropped. */
|
|
467
|
+
failures: HarvestFailure[];
|
|
468
|
+
}
|
|
469
|
+
declare function harvestCorpus(opts: HarvestCorpusOptions): Promise<HarvestReport>;
|
|
470
|
+
|
|
471
|
+
/**
|
|
472
|
+
* Adapt an `ExecutorFactory` into a `SandboxClient` for `runLoop`. The factory is
|
|
473
|
+
* instantiated fresh per `streamPrompt` (mirrors the per-spawn executor lifecycle):
|
|
474
|
+
* run once on the prompt, emit the terminal result event, tear down.
|
|
475
|
+
*/
|
|
476
|
+
declare function inlineSandboxClient(factory: ExecutorFactory<unknown>): SandboxClient;
|
|
477
|
+
|
|
478
|
+
/**
|
|
479
|
+
* @experimental
|
|
480
|
+
*
|
|
481
|
+
* `runLoop` — the topology-agnostic kernel built atop the sandbox SDK.
|
|
482
|
+
*
|
|
483
|
+
* Each iteration:
|
|
484
|
+
* 1. `driver.plan(task, history)` → N tasks (1 = refine, N = fanout, 0 = stop)
|
|
485
|
+
* 2. For each task (parallel, bounded by `maxConcurrency`):
|
|
486
|
+
* a. round-robin an `AgentRunSpec` from `agentRuns`
|
|
487
|
+
* b. `sandboxClient.create({ backend: { profile }, ...overrides })`
|
|
488
|
+
* c. emit `loop.iteration.dispatch` with the placement
|
|
489
|
+
* (`{ sibling, sandboxId }` or `{ fleet, fleetId, machineId, sandboxId }`)
|
|
490
|
+
* d. iterate `box.streamPrompt(taskToPrompt(task))` and collect events
|
|
491
|
+
* 3. `output.parse(events)` → typed `Output`
|
|
492
|
+
* 4. `validator?.validate(output)` → `DefaultVerdict`
|
|
493
|
+
* 5. Append `Iteration` to history; emit `loop.iteration.ended`
|
|
494
|
+
* 6. `driver.decide(history)` → if terminal, return result + winner
|
|
495
|
+
*
|
|
496
|
+
* The kernel owns: iteration accounting, per-iteration timing, error
|
|
497
|
+
* capture, abort propagation, concurrency cap, cost aggregation, and trace
|
|
498
|
+
* emission. The kernel does NOT own: what the agent runs (sandbox SDK +
|
|
499
|
+
* profile), how outputs are decoded (output adapter), how outputs are
|
|
500
|
+
* scored (validator), or topology (driver).
|
|
501
|
+
*/
|
|
502
|
+
|
|
503
|
+
/** @experimental */
|
|
504
|
+
interface RunLoopOptions<Task, Output, Decision> {
|
|
505
|
+
driver: Driver<Task, Output, Decision>;
|
|
506
|
+
/**
|
|
507
|
+
* Single agent spec — every iteration uses this profile. Mutually
|
|
508
|
+
* exclusive with `agentRuns`.
|
|
509
|
+
*/
|
|
510
|
+
agentRun?: AgentRunSpec<Task>;
|
|
511
|
+
/**
|
|
512
|
+
* Multiple specs for heterogeneous fanout. The kernel round-robins
|
|
513
|
+
* through them when the driver plans N tasks. Mutually exclusive with
|
|
514
|
+
* `agentRun`.
|
|
515
|
+
*/
|
|
516
|
+
agentRuns?: AgentRunSpec<Task>[];
|
|
517
|
+
output: OutputAdapter<Output>;
|
|
518
|
+
validator?: Validator<Output>;
|
|
519
|
+
task: Task;
|
|
520
|
+
ctx: ExecCtx;
|
|
521
|
+
/** Default 10. Hard cap on total iterations across all `plan()` rounds. */
|
|
522
|
+
maxIterations?: number;
|
|
523
|
+
/** Default 4. In-flight worker cap within a single `plan()` batch. */
|
|
524
|
+
maxConcurrency?: number;
|
|
525
|
+
/**
|
|
526
|
+
* Pre-allocated id for trace correlation. Default = `loop-${random}`.
|
|
527
|
+
* Surfaces as `runId` on every emitted `LoopTraceEvent`.
|
|
528
|
+
*/
|
|
529
|
+
runId?: string;
|
|
530
|
+
/**
|
|
531
|
+
* Clock override; default `Date.now`. Deterministic tests pass a
|
|
532
|
+
* monotonic counter to stabilize iteration timing fields.
|
|
533
|
+
*/
|
|
534
|
+
now?: () => number;
|
|
535
|
+
/**
|
|
536
|
+
* Override the default winner selector (highest-valid-score, ties broken
|
|
537
|
+
* by earliest iteration).
|
|
538
|
+
*/
|
|
539
|
+
selectWinner?: (iterations: Iteration<Task, Output>[]) => LoopWinner<Task, Output> | undefined;
|
|
540
|
+
/**
|
|
541
|
+
* Same-sandbox driver mode — a kernel→caller out-channel, not a value handed
|
|
542
|
+
* in. When set, the kernel keeps each finished worker box alive across the
|
|
543
|
+
* `plan()` boundary and hands it here, so a same-sandbox planner
|
|
544
|
+
* (one that reuses the worker's box) can stream its move INTO the
|
|
545
|
+
* worker's live box — steering from the worker's real filesystem and state,
|
|
546
|
+
* not just a history summary. The kernel owns teardown: every box kept alive
|
|
547
|
+
* this way is destroyed at loop end (and the callback is invoked with
|
|
548
|
+
* `undefined` then as a teardown sentinel). Without it, worker boxes are torn
|
|
549
|
+
* down per-iteration (default) and a same-sandbox planner has nothing to
|
|
550
|
+
* reuse. Intended for single-worker (refine) loops: under fanout every box is
|
|
551
|
+
* still kept for teardown, but only the last-finishing box is handed here, so
|
|
552
|
+
* a planner sees an arbitrary branch's filesystem — pair it with refine.
|
|
553
|
+
*/
|
|
554
|
+
onWorkerBox?: (box: SandboxInstance | undefined) => void;
|
|
555
|
+
/**
|
|
556
|
+
* Opt-in box-lineage controls. Default OFF — unset means every iteration
|
|
557
|
+
* acquires a fresh box, streams once, and tears it down (today's behavior,
|
|
558
|
+
* byte-identical). With `sessionContinuity` on, a refine round continues the
|
|
559
|
+
* parent iteration's session on its live box; with `forkFanout` on (and a
|
|
560
|
+
* fork-capable platform), a fanout round forks the parent's checkpoint so the
|
|
561
|
+
* branches share a context prefix. The lineage owns every box it starts or
|
|
562
|
+
* forks and tears them all down at loop end — so these paths are mutually
|
|
563
|
+
* exclusive with `onWorkerBox`, which claims the same box-ownership channel.
|
|
564
|
+
* @experimental
|
|
565
|
+
*/
|
|
566
|
+
lineage?: LoopLineageOptions;
|
|
567
|
+
}
|
|
568
|
+
/** @experimental */
|
|
569
|
+
declare function runLoop<Task, Output, Decision>(options: RunLoopOptions<Task, Output, Decision>): Promise<LoopResult<Task, Output, Decision>>;
|
|
570
|
+
/**
|
|
571
|
+
* The kernel's winner argmax — best-valid-score, ties broken by earliest index,
|
|
572
|
+
* falling back to the best-scoring non-errored output when none is valid. Exported
|
|
573
|
+
* so the `runProgram` tree executor selects across merged sub-loop iterations with
|
|
574
|
+
* the SAME semantics the kernel uses at a single loop's finalize (one selector, not
|
|
575
|
+
* a forked copy).
|
|
576
|
+
*/
|
|
577
|
+
declare function defaultSelectWinner<Task, Output>(iterations: Iteration<Task, Output>[]): LoopWinner<Task, Output> | undefined;
|
|
578
|
+
|
|
579
|
+
/**
|
|
580
|
+
* `loopDispatch` — turn `runLoop` into an agent-eval campaign dispatch.
|
|
581
|
+
*
|
|
582
|
+
* Without this adapter a consumer wiring `runLoop` into `runProfileMatrix` /
|
|
583
|
+
* `runCampaign` has to, by hand, every time: (a) build an `ExecCtx` with a
|
|
584
|
+
* sandbox client, (b) adapt the campaign `DispatchContext.trace` into a
|
|
585
|
+
* `LoopTraceEmitter` (or lose all loop trace correlation), and (c) remember to
|
|
586
|
+
* forward the loop's cost + tokens via `ctx.cost` (forgetting it yields a
|
|
587
|
+
* `{0,0}` cell the backend-integrity guard reads as a stub). Three foot-guns,
|
|
588
|
+
* the third silent. The fleet's products skipped (c) and fell back to a
|
|
589
|
+
* `workerRecords[]` side-channel — the exact anti-pattern the substrate exists
|
|
590
|
+
* to kill.
|
|
591
|
+
*
|
|
592
|
+
* `loopDispatch` collapses all three into one typed call:
|
|
593
|
+
*
|
|
594
|
+
* const dispatch = loopDispatch({
|
|
595
|
+
* sandboxClient,
|
|
596
|
+
* toLoopOptions: (scenario, profile) => ({ driver, agentRun, output, validator, task }),
|
|
597
|
+
* })
|
|
598
|
+
* await runProfileMatrix({ profiles, scenarios, dispatch, judges, commitSha })
|
|
599
|
+
*
|
|
600
|
+
* Usage is reported automatically; trace events are forwarded automatically;
|
|
601
|
+
* the ctx is built automatically. The seam becomes impossible to mis-wire.
|
|
602
|
+
*
|
|
603
|
+
* Typed structurally against the campaign `DispatchContext` (imported type-only
|
|
604
|
+
* from `@tangle-network/agent-eval/campaign`) — a downward dependency, never an
|
|
605
|
+
* inversion.
|
|
606
|
+
*/
|
|
607
|
+
|
|
608
|
+
/** runLoop options minus the `ctx` (loopDispatch builds the ctx). */
|
|
609
|
+
type LoopOptionsForDispatch<Task, Output, Decision> = Omit<RunLoopOptions<Task, Output, Decision>, 'ctx'>;
|
|
610
|
+
interface LoopDispatchOptions<Task, Output, Decision, TScenario extends Scenario, TArtifact> {
|
|
611
|
+
/** Sandbox client used for every cell's `runLoop`. Supplied once. */
|
|
612
|
+
sandboxClient: SandboxClient;
|
|
613
|
+
/** Build the per-cell runLoop options from the scenario (+ profile, when
|
|
614
|
+
* used with `runProfileMatrix`). */
|
|
615
|
+
toLoopOptions: (scenario: TScenario, profile: AgentProfile) => LoopOptionsForDispatch<Task, Output, Decision>;
|
|
616
|
+
/** Map the finished loop to the artifact the judges score. Default:
|
|
617
|
+
* `result.winner?.output`. A loop with no winner yields `undefined` (judges
|
|
618
|
+
* skip the cell) — but the loop's token usage is STILL reported, so the
|
|
619
|
+
* integrity guard sees real activity. */
|
|
620
|
+
toArtifact?: (result: LoopResult<Task, Output, Decision>) => TArtifact;
|
|
621
|
+
/** Forward `loop.*` trace events into the campaign's scoped trace so loop
|
|
622
|
+
* spans correlate with the cell. Default true. */
|
|
623
|
+
forwardTrace?: boolean;
|
|
624
|
+
/** Cost-meter source label for the loop's spend. Default `'loop'`. */
|
|
625
|
+
costSource?: string;
|
|
626
|
+
}
|
|
627
|
+
/**
|
|
628
|
+
* Adapter for `runProfileMatrix` (profile is an axis). Returns a
|
|
629
|
+
* `ProfileDispatchFn` that runs `runLoop` per (profile, scenario) cell and
|
|
630
|
+
* reports usage automatically.
|
|
631
|
+
*/
|
|
632
|
+
declare function loopDispatch<Task, Output, Decision, TScenario extends Scenario, TArtifact>(opts: LoopDispatchOptions<Task, Output, Decision, TScenario, TArtifact>): ProfileDispatchFn<TScenario, TArtifact>;
|
|
633
|
+
|
|
634
|
+
/**
|
|
635
|
+
* The general agentic primitive — sequential (depth) and parallel (breadth) over a shared,
|
|
636
|
+
* checkable artifact, driven through the keystone Supervisor as one recursive `Agent.act`.
|
|
637
|
+
*
|
|
638
|
+
* The domain lives behind ONE seam — `AgenticSurface` (open an artifact, list tools, call a tool,
|
|
639
|
+
* score the artifact, close it). EnterpriseOps implements it (seed a gym DB, MCP tools, SQL
|
|
640
|
+
* verifier); Commit0/AppWorld/terminal-bench implement it the same way (a repo workspace, shell
|
|
641
|
+
* tools, the test suite). The drivers below are domain-blind: they run over any surface.
|
|
642
|
+
*
|
|
643
|
+
* Two shapes, the agent's POMDP rollout as the unit:
|
|
644
|
+
* - DEPTH one persistent artifact carried across shots. Each shot the agent works the tool loop;
|
|
645
|
+
* between shots a trace-analyst (selector≠judge: reads the trajectory, never the score)
|
|
646
|
+
* steers the resumed session toward what's unfinished. shot n stands on shot n-1's
|
|
647
|
+
* artifact state + history. This is continuation — long-horizon, same artifact.
|
|
648
|
+
* - BREADTH K independent artifacts, each a fresh rollout, the deployable verifier picks the best.
|
|
649
|
+
*
|
|
650
|
+
* Both are an `Agent` whose `act` spawns leaf shots through `scope.spawn` and reacts via
|
|
651
|
+
* `scope.next()` — so the conserved budget pool meters them (equal-k by construction), the journal
|
|
652
|
+
* records the tree, and the same primitive nests. `runAgentic` runs the chosen driver through
|
|
653
|
+
* `createSupervisor().run`. The leaf (one shot over a handle) is resolved per-spawn from a
|
|
654
|
+
* surface-closed registry — the open `Executor` seam, not bespoke per-benchmark glue.
|
|
655
|
+
*/
|
|
656
|
+
|
|
657
|
+
interface AgenticTask {
|
|
658
|
+
readonly id: string;
|
|
659
|
+
readonly systemPrompt: string;
|
|
660
|
+
readonly userPrompt: string;
|
|
661
|
+
/** Opaque domain payload the surface reads (EOPS: servers/verifiers/tools). Drivers never read it. */
|
|
662
|
+
readonly meta?: Record<string, unknown>;
|
|
663
|
+
}
|
|
664
|
+
interface ArtifactHandle {
|
|
665
|
+
readonly id: string;
|
|
666
|
+
readonly surface: string;
|
|
667
|
+
/** Opaque per-artifact context the surface stashes (EOPS: the seeded gym server + db id). */
|
|
668
|
+
readonly ctx?: unknown;
|
|
669
|
+
}
|
|
670
|
+
interface AgenticTool {
|
|
671
|
+
readonly type: 'function';
|
|
672
|
+
readonly function: {
|
|
673
|
+
name: string;
|
|
674
|
+
description?: string;
|
|
675
|
+
parameters: Record<string, unknown>;
|
|
676
|
+
};
|
|
677
|
+
}
|
|
678
|
+
interface SurfaceScore {
|
|
679
|
+
passes: number;
|
|
680
|
+
total: number;
|
|
681
|
+
/** Checks excluded as malformed (data defect, not the agent). `total === 0` ⇒ unscoreable. */
|
|
682
|
+
errored: number;
|
|
683
|
+
}
|
|
684
|
+
/** A stateful, checkable environment an agent operates over with tools. Open behind one interface. */
|
|
685
|
+
interface AgenticSurface {
|
|
686
|
+
readonly name: string;
|
|
687
|
+
open(task: AgenticTask): Promise<ArtifactHandle>;
|
|
688
|
+
tools(task: AgenticTask, handle: ArtifactHandle): Promise<AgenticTool[]>;
|
|
689
|
+
call(handle: ArtifactHandle, name: string, args: Record<string, unknown>): Promise<string>;
|
|
690
|
+
score(task: AgenticTask, handle: ArtifactHandle): Promise<SurfaceScore>;
|
|
691
|
+
close(handle: ArtifactHandle): Promise<void>;
|
|
692
|
+
}
|
|
693
|
+
interface AgenticOptions {
|
|
694
|
+
routerBaseUrl: string;
|
|
695
|
+
routerKey: string;
|
|
696
|
+
model: string;
|
|
697
|
+
temperature?: number;
|
|
698
|
+
/** Completion cap per worker turn — REQUIRED for thinking models (they burn unbounded
|
|
699
|
+
* budgets on reasoning and return empty content without it). Omitted ⇒ provider default. */
|
|
700
|
+
maxTokens?: number;
|
|
701
|
+
/** Turns the agent may take within ONE shot before the driver intervenes. */
|
|
702
|
+
innerTurns?: number;
|
|
703
|
+
/** The depth STEERER's analyst instruction (observe()'s system prompt). The knob a
|
|
704
|
+
* prompt optimizer (GEPA) tunes — the analyst IS the steerer. Omitted ⇒ the default. */
|
|
705
|
+
analystInstruction?: string;
|
|
706
|
+
/** The critic's model — lets the analyst be a stronger (or cheaper) model than the
|
|
707
|
+
* worker. Omitted ⇒ the worker's `model`. */
|
|
708
|
+
analystModel?: string;
|
|
709
|
+
/** Across-run learning: when set, the analyst's observe() pass appends trace-derived
|
|
710
|
+
* facts here (the flywheel write side). Priming (the read side) is the caller's move —
|
|
711
|
+
* query the corpus and fold facts into the task's systemPrompt before runAgentic. */
|
|
712
|
+
corpus?: Corpus;
|
|
713
|
+
/** Tags written onto learned facts (and used by the caller's priming query). */
|
|
714
|
+
corpusTags?: string[];
|
|
715
|
+
}
|
|
716
|
+
type Msg = Record<string, unknown>;
|
|
717
|
+
interface ShotResult {
|
|
718
|
+
messages: Msg[];
|
|
719
|
+
score: number;
|
|
720
|
+
passes: number;
|
|
721
|
+
total: number;
|
|
722
|
+
completions: number;
|
|
723
|
+
toolErrors: number;
|
|
724
|
+
}
|
|
725
|
+
interface AgenticRunResult {
|
|
726
|
+
/** The strategy name (built-in 'depth'/'breadth' or a custom strategy's name). */
|
|
727
|
+
mode: string;
|
|
728
|
+
score: number;
|
|
729
|
+
resolved: boolean;
|
|
730
|
+
completions: number;
|
|
731
|
+
/** DEPTH: score after each shot — the progress-over-rounds curve. BREADTH: best-so-far per rollout. */
|
|
732
|
+
progression: number[];
|
|
733
|
+
shots: number;
|
|
734
|
+
/** The cost vector, stamped by `runAgentic` from the Supervisor's conserved pool: real
|
|
735
|
+
* router tokens, priced usd (0 when the model is unpriced — never fabricated), wall ms. */
|
|
736
|
+
usd: number;
|
|
737
|
+
ms: number;
|
|
738
|
+
tokens: {
|
|
739
|
+
input: number;
|
|
740
|
+
output: number;
|
|
741
|
+
};
|
|
742
|
+
}
|
|
743
|
+
/** DEPTH: one persistent artifact, carried across analyst-steered shots. */
|
|
744
|
+
declare function depthStrategy(surface: AgenticSurface, task: AgenticTask, opts: AgenticOptions, cfg: {
|
|
745
|
+
maxShots: number;
|
|
746
|
+
}): Agent<unknown, Outcome<unknown>>;
|
|
747
|
+
/** BREADTH: K independent rollouts (each own artifact), verifier picks the best. */
|
|
748
|
+
declare function breadthStrategy(_surface: AgenticSurface, task: AgenticTask, opts: AgenticOptions, cfg: {
|
|
749
|
+
width: number;
|
|
750
|
+
}): Agent<unknown, Outcome<unknown>>;
|
|
751
|
+
/**
|
|
752
|
+
* A Strategy is HOW you spend the compute budget to beat the Environment's check — it
|
|
753
|
+
* builds the driver `Agent` the Supervisor runs. This is the OPEN extension point: a dev
|
|
754
|
+
* authors their own by implementing `driver()` to return an Agent whose `act()` spawns
|
|
755
|
+
* shots/analysts via `scope.spawn` / `scope.next` / `scope.send`. The two built-ins are
|
|
756
|
+
* the reference implementations to copy:
|
|
757
|
+
* sample — K INDEPENDENT attempts, keep the best-verifying (best-of-N / resample).
|
|
758
|
+
* refine — attempt → observe() reads the trace → steer the next → repeat (iterate).
|
|
759
|
+
* (A multi-agent "team" is just a Strategy whose driver spawns several different agents.)
|
|
760
|
+
*/
|
|
761
|
+
interface Strategy {
|
|
762
|
+
readonly name: string;
|
|
763
|
+
driver(surface: AgenticSurface, task: AgenticTask, opts: AgenticOptions, budget: number): Agent<unknown, Outcome<unknown>>;
|
|
764
|
+
}
|
|
765
|
+
declare const sample: Strategy;
|
|
766
|
+
declare const refine: Strategy;
|
|
767
|
+
/** A role for one shot — multi-agent loops (researcher + engineer, a panel of k
|
|
768
|
+
* researchers) give each shot its own system prompt and optionally its own model. */
|
|
769
|
+
interface ShotPersona {
|
|
770
|
+
/** Replaces the task's systemPrompt for a FRESH shot; on a carried conversation it is
|
|
771
|
+
* injected as a hand-off message (the transcript's earlier roles stay intact). */
|
|
772
|
+
systemPrompt?: string;
|
|
773
|
+
/** Per-shot model override (e.g. a stronger model for the engineer shot). */
|
|
774
|
+
model?: string;
|
|
775
|
+
}
|
|
776
|
+
interface ShotSpec {
|
|
777
|
+
/** present ⇒ continue this artifact (depth); absent ⇒ the shot opens a fresh one (sample/restart). */
|
|
778
|
+
handle?: ArtifactHandle;
|
|
779
|
+
messages?: Msg[];
|
|
780
|
+
steer?: string;
|
|
781
|
+
persona?: ShotPersona;
|
|
782
|
+
/** Restrict THIS shot to a subset of the domain's tools (by name) — focus a shot on
|
|
783
|
+
* the relevant capabilities. Restriction-only; unknown names throw. Omitted ⇒ all. */
|
|
784
|
+
tools?: string[];
|
|
785
|
+
}
|
|
786
|
+
interface StrategyResult {
|
|
787
|
+
score: number;
|
|
788
|
+
resolved: boolean;
|
|
789
|
+
completions: number;
|
|
790
|
+
progression: number[];
|
|
791
|
+
shots: number;
|
|
792
|
+
}
|
|
793
|
+
/** Artifact lifecycle a strategy may manage itself — open/close ONLY. Raw `call`/`score`
|
|
794
|
+
* are withheld: scores reach the body solely through `shot()`'s ShotResult (the
|
|
795
|
+
* harness-verified channel), so a body cannot peek the check or fabricate around it. */
|
|
796
|
+
interface StrategyArtifacts {
|
|
797
|
+
readonly name: string;
|
|
798
|
+
open(task: AgenticTask): Promise<ArtifactHandle>;
|
|
799
|
+
close(handle: ArtifactHandle): Promise<void>;
|
|
800
|
+
}
|
|
801
|
+
/** What a strategy body composes with: the artifact lifecycle, the budget, and the two steps. */
|
|
802
|
+
interface StrategyCtx {
|
|
803
|
+
/** Open/close artifacts the body manages itself (e.g. one persistent handle for depth). */
|
|
804
|
+
readonly surface: StrategyArtifacts;
|
|
805
|
+
readonly task: AgenticTask;
|
|
806
|
+
readonly opts: AgenticOptions;
|
|
807
|
+
readonly budget: number;
|
|
808
|
+
readonly scope: Scope<Outcome<unknown>>;
|
|
809
|
+
/** Run ONE worker shot; its harness-scored result, or null if it went down. */
|
|
810
|
+
shot(spec?: ShotSpec): Promise<ShotResult | null>;
|
|
811
|
+
/** The firewalled critic reads the trajectory → a steer string, or null on COMPLETE/down. */
|
|
812
|
+
critique(messages: Msg[]): Promise<string | null>;
|
|
813
|
+
/** The RAW analyst channel: the firewalled critic answers `instruction` over the
|
|
814
|
+
* trajectory verbatim — no findings extraction, so verdict-shaped formats
|
|
815
|
+
* (CONTINUE/STOP decisions, calibrated predictions) survive. Same firewall:
|
|
816
|
+
* trajectory in, never scores. Null when the analyst went down. */
|
|
817
|
+
consult(messages: Msg[], instruction: string): Promise<string | null>;
|
|
818
|
+
/** The tools THIS artifact's task actually offers (names + descriptions only — never
|
|
819
|
+
* the implementations). Tool sets vary per task on heterogeneous domains; a strategy
|
|
820
|
+
* that restricts shots MUST select from this list, never from hardcoded names. */
|
|
821
|
+
listTools(handle: ArtifactHandle): Promise<Array<{
|
|
822
|
+
name: string;
|
|
823
|
+
description?: string;
|
|
824
|
+
}>>;
|
|
825
|
+
}
|
|
826
|
+
/** Author a Strategy from the composable steps — the open, compact way. */
|
|
827
|
+
declare function defineStrategy(name: string, run: (ctx: StrategyCtx) => Promise<StrategyResult>): Strategy;
|
|
828
|
+
/** A NEW strategy, authored from the steps (~20 lines): refine, but when a steered shot
|
|
829
|
+
* fails to improve the score it ABANDONS that line and restarts fresh (branch-when-stuck)
|
|
830
|
+
* — the widen/MCTS idea the depth-stuck failure motivated. Scored keep-best (the best
|
|
831
|
+
* checkpoint across all lines), the deployable metric. This is the "experts build BETTER
|
|
832
|
+
* optimizations" path: a new technique, compact, with zero Supervisor ceremony. */
|
|
833
|
+
declare const adaptiveRefine: Strategy;
|
|
834
|
+
/** The explore-then-exploit MIX: spend ⌈budget/2⌉ on independent samples (kept open),
|
|
835
|
+
* then refine the best-verifying line with the remaining budget. Sample's basin escape +
|
|
836
|
+
* refine's accumulation — the third built-in, authored from the public steps. */
|
|
837
|
+
declare const sampleThenRefine: Strategy;
|
|
838
|
+
interface RunAgenticOptions extends AgenticOptions {
|
|
839
|
+
surface: AgenticSurface;
|
|
840
|
+
task: AgenticTask;
|
|
841
|
+
/** Lifecycle observability — every spawn/settle (shots, analysts) streams here live.
|
|
842
|
+
* The seam online watchdogs/route-auditors subscribe to. */
|
|
843
|
+
hooks?: RuntimeHooks;
|
|
844
|
+
/** A Strategy (the open way) — author/pass your own. Overrides `mode` when present. */
|
|
845
|
+
strategy?: Strategy;
|
|
846
|
+
/** Built-in shorthand: 'depth'→refine, 'breadth'→sample. Default 'depth'. */
|
|
847
|
+
mode?: 'depth' | 'breadth';
|
|
848
|
+
/** budget: refine→max shots; sample→rollout width. */
|
|
849
|
+
budget: number;
|
|
850
|
+
rootBudget?: Budget;
|
|
851
|
+
}
|
|
852
|
+
/** Run a Strategy through the keystone Supervisor — `Agent.act` over a conserved-budget Scope. */
|
|
853
|
+
declare function runAgentic(opts: RunAgenticOptions): Promise<AgenticRunResult>;
|
|
854
|
+
|
|
855
|
+
/**
|
|
856
|
+
* runBenchmark — the packaged optimization suite. Define a domain by implementing an
|
|
857
|
+
* `Environment` (open / tools / call / score / close); get the optimization strategies
|
|
858
|
+
* compared, scored by your own deployable check, with a paired-bootstrap report — free.
|
|
859
|
+
*
|
|
860
|
+
* The mental model: you have a TASK + a deployable CHECK + a compute BUDGET. A strategy
|
|
861
|
+
* is how you spend the budget to beat the check. Two built-ins:
|
|
862
|
+
*
|
|
863
|
+
* sample — N independent attempts, keep the best-verifying one. (best-of-N / resample)
|
|
864
|
+
* refine — attempt → a critic reads the trace → steer the next → repeat. (iterate-with-feedback)
|
|
865
|
+
*
|
|
866
|
+
* Both run at equal budget through the Supervisor's conserved pool; the headline is the
|
|
867
|
+
* paired lift of refine over sample. Author your own strategy with `defineStrategy`.
|
|
868
|
+
*/
|
|
869
|
+
|
|
870
|
+
/** A checkable task domain — implement these 5 hooks and the suite does the rest. The
|
|
871
|
+
* same seam as `AgenticSurface`; `Environment` is the RL/gym-standard name for it. */
|
|
872
|
+
type Environment = AgenticSurface;
|
|
873
|
+
interface BenchmarkConfig {
|
|
874
|
+
/** The task domain (5 hooks). */
|
|
875
|
+
environment: Environment;
|
|
876
|
+
/** The tasks to score across. */
|
|
877
|
+
tasks: AgenticTask[];
|
|
878
|
+
/** The worker: model + router + (optional) the critic's instruction (the steerer knob). */
|
|
879
|
+
worker: AgenticOptions;
|
|
880
|
+
/** Which strategies to compare. Pass the built-ins (`refine`, `sample`) or your own.
|
|
881
|
+
* Default: [sample, refine]. */
|
|
882
|
+
strategies?: Strategy[];
|
|
883
|
+
/** Shots (refine) / width (sample) — the equal compute budget per strategy. Default 3. */
|
|
884
|
+
budget?: number;
|
|
885
|
+
/** Tasks scored in parallel. Default 3. */
|
|
886
|
+
concurrency?: number;
|
|
887
|
+
/** Progress hook — fires as each task settles (the live-monitoring seam: append to a
|
|
888
|
+
* progress file, render a tree, stream to a dashboard). `done` counts settled tasks. */
|
|
889
|
+
onTask?: (row: BenchmarkTaskRow, done: number, total: number) => void;
|
|
890
|
+
/** Lifecycle observability — every spawn/settle of every cell's shots/analysts streams
|
|
891
|
+
* here live (the watchdog/route-auditor seam, passed through to `runAgentic`). */
|
|
892
|
+
hooks?: RuntimeHooks;
|
|
893
|
+
}
|
|
894
|
+
interface BenchmarkLift {
|
|
895
|
+
/** Mean of paired deltas (refine − sample). */
|
|
896
|
+
mean: number;
|
|
897
|
+
low: number;
|
|
898
|
+
high: number;
|
|
899
|
+
n: number;
|
|
900
|
+
}
|
|
901
|
+
/** One strategy's outcome on one task — the per-task cell an optimizer consumes. */
|
|
902
|
+
interface BenchmarkCell {
|
|
903
|
+
score: number;
|
|
904
|
+
resolved: boolean;
|
|
905
|
+
/** The progress curve (refine: score per shot; sample: best-so-far per rollout). */
|
|
906
|
+
progression: number[];
|
|
907
|
+
usd: number;
|
|
908
|
+
ms: number;
|
|
909
|
+
tokens: {
|
|
910
|
+
input: number;
|
|
911
|
+
output: number;
|
|
912
|
+
};
|
|
913
|
+
}
|
|
914
|
+
interface BenchmarkTaskRow {
|
|
915
|
+
taskId: string;
|
|
916
|
+
/** Per-strategy cells; absent when the task errored before completing all strategies. */
|
|
917
|
+
cells?: Record<string, BenchmarkCell>;
|
|
918
|
+
/** Per-strategy failures on this task: the strategy competed, threw, and scored an
|
|
919
|
+
* honest zero — it loses, it does not poison the row. The message is kept so a later
|
|
920
|
+
* generation's author can see WHY a candidate died. */
|
|
921
|
+
errors?: Record<string, string>;
|
|
922
|
+
/** Why the task was excluded (infra/setup failure) — never silently dropped. */
|
|
923
|
+
error?: string;
|
|
924
|
+
}
|
|
925
|
+
interface BenchmarkStrategySummary {
|
|
926
|
+
/** Mean verifier score (0..1). */
|
|
927
|
+
score: number;
|
|
928
|
+
/** Fraction of tasks fully resolved. */
|
|
929
|
+
resolved: number;
|
|
930
|
+
/** Mean cost vector per task. */
|
|
931
|
+
usd: number;
|
|
932
|
+
ms: number;
|
|
933
|
+
}
|
|
934
|
+
interface BenchmarkReport {
|
|
935
|
+
n: number;
|
|
936
|
+
excluded: number;
|
|
937
|
+
/** Per-strategy means (keyed by strategy.name). */
|
|
938
|
+
perStrategy: Record<string, BenchmarkStrategySummary>;
|
|
939
|
+
/** The full per-task × per-strategy table — the LOSSES an optimizer (GEPA, a
|
|
940
|
+
* strategy-author, an operator) consumes. Includes errored tasks with the reason. */
|
|
941
|
+
perTask: BenchmarkTaskRow[];
|
|
942
|
+
/** The non-dominated strategies on (score ↑, $/task ↓) — collapse-last, per the canon:
|
|
943
|
+
* a strategy that ties on score at half the cost WINS and a scalar would hide it. */
|
|
944
|
+
pareto: string[];
|
|
945
|
+
/** The headline when both `refine` and `sample` ran: paired-bootstrap lift of refine over sample. */
|
|
946
|
+
refineVsSample?: BenchmarkLift;
|
|
947
|
+
}
|
|
948
|
+
/** Run the requested strategies over the tasks, scored by the Environment's own check.
|
|
949
|
+
* Resilient: a task whose rollouts fail (transient infra) is excluded from the stats but
|
|
950
|
+
* reported in `perTask` with the error — never silently dropped. */
|
|
951
|
+
declare function runBenchmark(cfg: BenchmarkConfig): Promise<BenchmarkReport>;
|
|
952
|
+
/** Pretty-print a report — the "free optimization" verdict, with the cost vector. */
|
|
953
|
+
declare function printBenchmarkReport(report: BenchmarkReport): void;
|
|
954
|
+
|
|
955
|
+
/**
|
|
956
|
+
* createMcpEnvironment — wrap any MCP server as an `Environment` (the product-adoption
|
|
957
|
+
* primitive: a product's agent tools are usually already an MCP surface, so the domain
|
|
958
|
+
* only writes the lifecycle hooks — open a scoped artifact, score it with a deployable
|
|
959
|
+
* check, close it — and the tool plumbing is derived from the server).
|
|
960
|
+
*
|
|
961
|
+
* What the helper owns (the generic 80%, hardened on the EnterpriseOps gym):
|
|
962
|
+
* - JSON-RPC `tools/list` → `AgenticTool[]`, with schemas coerced to the
|
|
963
|
+
* OpenAI-tool-valid shape (top-level oneOf/anyOf/allOf/enum/not are rejected by
|
|
964
|
+
* tool-calling providers; nested combinators are fine).
|
|
965
|
+
* - JSON-RPC `tools/call` → the tool's text content (errors surfaced as `ERROR: …`
|
|
966
|
+
* strings — a bad call is the agent's outcome, not an infra fault).
|
|
967
|
+
* - SSE response parsing (streamable-HTTP MCP servers answer with `data:` lines).
|
|
968
|
+
* - Bounded retry with backoff on thrown fetches (transient network ≠ task failure).
|
|
969
|
+
*
|
|
970
|
+
* What the domain supplies: `open` (create/seed the per-task artifact and return its
|
|
971
|
+
* MCP endpoint — url + headers carry the per-artifact scoping, e.g. a database id
|
|
972
|
+
* header), `score` (the deployable check), and optional `close`/`selectTools`.
|
|
973
|
+
*/
|
|
974
|
+
|
|
975
|
+
/** Where a handle's MCP server lives; headers carry per-artifact scoping. */
|
|
976
|
+
interface McpEndpoint {
|
|
977
|
+
url: string;
|
|
978
|
+
headers?: Record<string, string>;
|
|
979
|
+
}
|
|
980
|
+
interface McpEnvironmentOptions {
|
|
981
|
+
name: string;
|
|
982
|
+
/** Create/seed the per-task artifact; return its handle + the MCP endpoint scoped to it. */
|
|
983
|
+
open(task: AgenticTask): Promise<{
|
|
984
|
+
handle: ArtifactHandle;
|
|
985
|
+
endpoint: McpEndpoint;
|
|
986
|
+
}>;
|
|
987
|
+
/** The deployable check over the artifact's current state. */
|
|
988
|
+
score(task: AgenticTask, handle: ArtifactHandle): Promise<SurfaceScore>;
|
|
989
|
+
/** Teardown (delete the seeded artifact). Optional — omit for stateless servers. */
|
|
990
|
+
close?(handle: ArtifactHandle): Promise<void>;
|
|
991
|
+
/** Restrict/order the server's tools per task (e.g. the task's selected_tools). Default: all. */
|
|
992
|
+
selectTools?(task: AgenticTask, all: AgenticTool[]): AgenticTool[];
|
|
993
|
+
/** Cap on a tool result's text fed back to the worker. Default 1500 chars. */
|
|
994
|
+
maxResultChars?: number;
|
|
995
|
+
}
|
|
996
|
+
declare function createMcpEnvironment(opts: McpEnvironmentOptions): Environment;
|
|
997
|
+
|
|
998
|
+
/**
|
|
999
|
+
* @experimental
|
|
1000
|
+
*
|
|
1001
|
+
* Analyst-on-scope (G1) — the analyze→findings→steer wire over the reactive `Scope`.
|
|
1002
|
+
*
|
|
1003
|
+
* The analyst runs over the children drained so far and hands its findings to the steer
|
|
1004
|
+
* decision behind a provenance firewall (`assertTraceDerivedFindings`) that keeps the external
|
|
1005
|
+
* write-only judge out of that decision (selector ≠ judge). The reactive `Scope` has no rounds:
|
|
1006
|
+
* a combinator's `act` asks a `ScopeAnalyst` to turn the children it has drained off
|
|
1007
|
+
* `scope.next()` SO FAR into `AnalystFinding[]`, and steers from THOSE findings through a single
|
|
1008
|
+
* `SteerContext`.
|
|
1009
|
+
*
|
|
1010
|
+
* The analyst itself is not a new type — it is "just an `Agent<unknown, AnalystFinding[]>`" the
|
|
1011
|
+
* combinator spawns over a child's trace (harness `null`/`cli`). `createScopeAnalyst` spawns that
|
|
1012
|
+
* agent through `Scope.spawn` (so its compute is metered by the conserved pool like any child),
|
|
1013
|
+
* drains its settlement, then enforces the firewall on the way out — a judge-derived finding
|
|
1014
|
+
* ABORTS, it is never filtered. Fail loud: a down analyst, a non-array result, or a tainted finding
|
|
1015
|
+
* throws; there is no silent empty-findings path that would let a combinator steer on nothing.
|
|
1016
|
+
*/
|
|
1017
|
+
|
|
1018
|
+
declare const assertTraceDerivedFindings: AssertTraceDerivedFindings;
|
|
1019
|
+
/**
|
|
1020
|
+
* The analyst run an `Agent<unknown, AnalystFinding[]>` performs over the children settled so far.
|
|
1021
|
+
* The combinator supplies the analyst's task projection (how to frame the drained settlements as
|
|
1022
|
+
* the analyst's input) — the analyst's `act` reads the trace and returns its raw findings; the
|
|
1023
|
+
* firewall is enforced afterwards by `createScopeAnalyst`, not by the analyst itself.
|
|
1024
|
+
*/
|
|
1025
|
+
interface CreateScopeAnalystOptions<D> {
|
|
1026
|
+
/** The analyst agent the combinator spawns over the trace. `harness` is the persona's choice
|
|
1027
|
+
* (`null` for an inline router analyst, a `BackendType` for a sandboxed one). Its `act` returns
|
|
1028
|
+
* the RAW findings; this module asserts the firewall on them before returning. */
|
|
1029
|
+
readonly analyst: Agent<unknown, ReadonlyArray<AnalystFinding>>;
|
|
1030
|
+
/** Build the analyst agent's task from the analyze input (the root-task framing + the children
|
|
1031
|
+
* drained so far). Pure projection — the analyst interprets it, this never reads it. */
|
|
1032
|
+
buildTask(input: ScopeAnalyzeInput<D>): unknown;
|
|
1033
|
+
/** The conserved budget reserved for one analyst spawn. The pool reserves against it and fails
|
|
1034
|
+
* closed; an analyst that cannot be admitted is a fail-loud abort, never silent empty findings. */
|
|
1035
|
+
readonly budget: Budget;
|
|
1036
|
+
/** Trace/journal label for the spawned analyst child. Default `'analyst'`. */
|
|
1037
|
+
readonly label?: string;
|
|
1038
|
+
}
|
|
1039
|
+
/**
|
|
1040
|
+
* Build a `ScopeAnalyst` that spawns the analyst agent through `Scope.spawn` (so its compute is
|
|
1041
|
+
* metered by the conserved pool), drains its single settlement, and enforces the trace-derived
|
|
1042
|
+
* firewall before returning. The `scope` is the SAME scope the combinator is draining its children
|
|
1043
|
+
* from — the analyst is spawned as a sibling and its result is read off `scope.next()` in cursor
|
|
1044
|
+
* order, replay-safe like any other child.
|
|
1045
|
+
*
|
|
1046
|
+
* Fail loud (no silent empty findings):
|
|
1047
|
+
* - the pool refuses the analyst spawn → `AnalystError` (the steer would otherwise run on nothing)
|
|
1048
|
+
* - the analyst settles `down` → `AnalystError` (a broken capture path, not a verdict)
|
|
1049
|
+
* - the analyst returns a non-array → `PlannerError`
|
|
1050
|
+
* - any finding cites judge-derived metric evidence → `PlannerError` via the firewall
|
|
1051
|
+
*/
|
|
1052
|
+
declare function createScopeAnalyst<D>(scope: Scope<Outcome<D>>, options: CreateScopeAnalystOptions<D>): ScopeAnalyst<D>;
|
|
1053
|
+
/**
|
|
1054
|
+
* Project a `ScopeAnalyzeInput` into the `AnalystRegistry.run` arguments. The registry runs over a
|
|
1055
|
+
* `runId` + `AnalystRunInputs` (a trace store / run record / artifact dir), NOT in-memory scope
|
|
1056
|
+
* settlements — so the CALLER owns the projection from the combinator's drained children to the
|
|
1057
|
+
* registry's inputs (e.g. the trace store the run already wrote). This adapter never invents that
|
|
1058
|
+
* bridge; it only runs the projected inputs and firewalls the merged findings.
|
|
1059
|
+
*/
|
|
1060
|
+
interface RegistryAnalyzeProjection {
|
|
1061
|
+
readonly runId: string;
|
|
1062
|
+
readonly inputs: AnalystRunInputs;
|
|
1063
|
+
/** Optional `run` opts (e.g. `priorFindings`) forwarded verbatim to the registry. */
|
|
1064
|
+
readonly opts?: Parameters<AnalystRegistryLike['run']>[2];
|
|
1065
|
+
}
|
|
1066
|
+
/**
|
|
1067
|
+
* A `ScopeAnalyst` backed by an `AnalystRegistry` — the panel-of-analysts seam. The registry merges
|
|
1068
|
+
* N analyst KINDS into one `AnalystRunResult.findings`; `analyze` runs it over the caller-projected
|
|
1069
|
+
* `{ runId, inputs }` and pipes the merged findings through the SAME `assertTraceDerivedFindings`
|
|
1070
|
+
* firewall `createScopeAnalyst` uses (single-sourced selector≠judge). Distinct from `panel()`
|
|
1071
|
+
* (judges-vs-one-artifact) — this is analysts-over-a-trace, the diagnosis side of the wire.
|
|
1072
|
+
*
|
|
1073
|
+
* Fail loud: a registry that throws propagates; a judge-derived finding aborts via the firewall.
|
|
1074
|
+
* The projection is the caller's (`buildInputs`) — if the scope settlements do not cleanly map to
|
|
1075
|
+
* the registry's `AnalystRunInputs`, that is a caller-side contract gap, surfaced there, not papered
|
|
1076
|
+
* over with a fabricated input here.
|
|
1077
|
+
*/
|
|
1078
|
+
declare function registryScopeAnalyst<D>(registry: AnalystRegistryLike, buildInputs: (input: ScopeAnalyzeInput<D>) => RegistryAnalyzeProjection): ScopeAnalyst<D>;
|
|
1079
|
+
/**
|
|
1080
|
+
* Build the `SteerContext` a combinator reads to steer (its `loopUntil.until`, `widen` gate, any
|
|
1081
|
+
* future steer). One place enforces the firewall: `findings` is asserted trace-derived before it is
|
|
1082
|
+
* surfaced, and `lastValidScore` is provided for OBSERVABILITY only — a combinator that steers off
|
|
1083
|
+
* it re-introduces selector = judge, the coupling the architecture forbids.
|
|
1084
|
+
*
|
|
1085
|
+
* `findings` is re-asserted here even when it came from `createScopeAnalyst` (which already asserted
|
|
1086
|
+
* it): the assertion is cheap and idempotent, and a `SteerContext` may be built from findings that
|
|
1087
|
+
* arrived by another path (a caller-supplied diagnosis). Belt-and-suspenders on the one coupling
|
|
1088
|
+
* that must never leak.
|
|
1089
|
+
*/
|
|
1090
|
+
declare function buildSteerContext<D>(findings: ReadonlyArray<AnalystFinding>, settledSoFar: ReadonlyArray<Settled<Outcome<D>>>): SteerContext<D>;
|
|
1091
|
+
|
|
1092
|
+
/**
|
|
1093
|
+
* @experimental
|
|
1094
|
+
*
|
|
1095
|
+
* The generic combinator library — the content-free act-bodies the wave's §1 contract froze.
|
|
1096
|
+
*
|
|
1097
|
+
* Each export is a `CombinatorShape<Task, D>` (an alias of `LoopShape<Task, D>`): a factory
|
|
1098
|
+
* `(ShapeContext) => Agent<Task, Outcome<D>>` whose `act` runs ONE composition shape over the
|
|
1099
|
+
* keystone `Scope` — spawn children through `ctx.spawnChild` + `scope.spawn`, drain settlements
|
|
1100
|
+
* via `scope.next()`, select across `done` children with the SINGLE-SOURCED `settledToIteration`
|
|
1101
|
+
* + `defaultSelectWinner` (selector≠judge — never a re-rank behind the driver), and synthesize a
|
|
1102
|
+
* terminal `Outcome<D>`.
|
|
1103
|
+
*
|
|
1104
|
+
* The shapes carry NO domain: a "research sweep over angles" is `fanout(angles, { synthesize })`
|
|
1105
|
+
* under a research persona; a "code build test" is `pipeline([plan, implement, integrate])` under
|
|
1106
|
+
* a coder persona. The SHAPE is here; the model/prompt/role/goal live on the `Persona` + task,
|
|
1107
|
+
* threaded to each child verbatim by the spec-objects the builders take. No model name, prompt,
|
|
1108
|
+
* role, or domain noun appears below.
|
|
1109
|
+
*
|
|
1110
|
+
* Two fail-loud invariants every combinator honors: a child the conserved pool cannot admit is a
|
|
1111
|
+
* CONCRETE blocker (never an eager over-fan, never a silent drop), and a `blocked` outcome always
|
|
1112
|
+
* names at least one blocker (a shape that cannot finish MUST say why — `blocked([])` throws).
|
|
1113
|
+
*/
|
|
1114
|
+
|
|
1115
|
+
/**
|
|
1116
|
+
* The single content-free valid-only winner selector. Among the gated-VALID children only
|
|
1117
|
+
* (`verdict.valid === true`), pick by `strategy` — best score / smallest delivered artifact /
|
|
1118
|
+
* earliest — ties broken by earliest index; returns `undefined` when NONE is valid (an ungated
|
|
1119
|
+
* output can never win — the deliverable gate is the point). `sizeOf` (for `'smallest-artifact'`)
|
|
1120
|
+
* reads the child's settled deliverable — the raw value a leaf settles, or the unwrapped `Outcome<D>`
|
|
1121
|
+
* a delegate path produces; a domain passes e.g. patch diff-lines. This is the de-duplicated home of
|
|
1122
|
+
* the selection logic previously copied per role.
|
|
1123
|
+
*/
|
|
1124
|
+
declare function selectValidWinner<D>(opts?: {
|
|
1125
|
+
strategy?: WinnerStrategy;
|
|
1126
|
+
sizeOf?: (deliverable: D) => number;
|
|
1127
|
+
}): FanoutWinnerSelector<D>;
|
|
1128
|
+
/**
|
|
1129
|
+
* `pipeline(stages)` — run the stages in order, feeding each stage's `done` deliverable into the
|
|
1130
|
+
* next stage's task. The first stage that ends `blocked` (a child that went down, a child the
|
|
1131
|
+
* pool would not admit, or a stage whose `collect` chose to block) short-circuits — its blockers
|
|
1132
|
+
* ARE the pipeline's blockers, never coerced past a failed stage. The terminal stage's `done`
|
|
1133
|
+
* deliverable is the pipeline's deliverable.
|
|
1134
|
+
*/
|
|
1135
|
+
declare function pipeline<Task, D>(stages: ReadonlyArray<PipelineStage<Task, unknown, unknown>>): CombinatorShape<Task, D>;
|
|
1136
|
+
/**
|
|
1137
|
+
* `fanout(items, opts)` — spawn one child per item in a single round (bounded by the conserved
|
|
1138
|
+
* pool's fail-closed admission), drain via `scope.next()`, then either synthesize over the
|
|
1139
|
+
* gathered settlements (one SEPARATE synthesis child) or return the best-valid child via the
|
|
1140
|
+
* single-sourced selector. A round that admitted zero children, or whose synthesis child could
|
|
1141
|
+
* not be admitted, is a concrete blocker.
|
|
1142
|
+
*/
|
|
1143
|
+
declare function fanout<Task, Item, D>(items: ReadonlyArray<Item>, opts: FanoutOptions<Item, D>): CombinatorShape<Task, D>;
|
|
1144
|
+
/**
|
|
1145
|
+
* `loopUntil(seed, spec)` — one `step` child per round; `fold` accumulates each settlement into
|
|
1146
|
+
* the running state; `until` (reading the round's trace findings, NOT a fresh raw verdict) is
|
|
1147
|
+
* the deployable stop. The conserved pool IS the loop bound: once `spawn` fails closed the loop
|
|
1148
|
+
* stops. A loop that exhausted the pool without `until` ever satisfying is a concrete blocker.
|
|
1149
|
+
*
|
|
1150
|
+
* When `ctx.analyst` is set, each round runs it over the children settled so far and steers
|
|
1151
|
+
* `until` on the resulting trace-derived findings (the analyst spawns into THIS scope, so its
|
|
1152
|
+
* compute is conserved-pooled — equal-k holds by construction). Absent an analyst the findings
|
|
1153
|
+
* argument is the empty array — never a fabricated finding (fail-loud honesty over a silent default).
|
|
1154
|
+
*/
|
|
1155
|
+
declare function loopUntil<Task, State, D>(seed: State, spec: LoopUntilSpec<Task, State, D>): CombinatorShape<Task, D>;
|
|
1156
|
+
/**
|
|
1157
|
+
* `panel(spec)` — spawn the M judge children over the SAME artifact, drain their settlements,
|
|
1158
|
+
* and fold them into a panel verdict via the pure WRITE-ONLY `merge` (a judge's output never
|
|
1159
|
+
* reaches another judge's task; the merge never spawns or re-ranks). A `down` judge carries no
|
|
1160
|
+
* verdict and is excluded from the merge denominator. A panel that admitted no judge is a
|
|
1161
|
+
* concrete blocker before `merge` is consulted.
|
|
1162
|
+
*/
|
|
1163
|
+
declare function panel<Task, Artifact, D>(spec: PanelSpec<Artifact, D>): CombinatorShape<Task, D>;
|
|
1164
|
+
/**
|
|
1165
|
+
* `verify(spec)` — an IMPLEMENT child produces a candidate, then a SEPARATE VERIFIER child grades
|
|
1166
|
+
* it; only a `valid` verifier verdict ships. Any other outcome (implement down, verifier down,
|
|
1167
|
+
* verifier verdict absent or not `valid`) is a concrete blocker carrying the failure verbatim —
|
|
1168
|
+
* never a coerced "done". The implement child does not grade itself.
|
|
1169
|
+
*/
|
|
1170
|
+
declare function verify<Task, Candidate, D>(spec: VerifySpec<Task, Candidate, D>): CombinatorShape<Task, D>;
|
|
1171
|
+
/**
|
|
1172
|
+
* `widen(spec)` — the streaming spawn-on-completion driver. Spawns the seed lineages, then REACTS
|
|
1173
|
+
* to each `scope.next()`: on every settled child it consults `spec.gate.decide` and, when the gate
|
|
1174
|
+
* returns `widen`, spawns AT MOST ONE more child toward the chosen lineage under the remaining
|
|
1175
|
+
* conserved pool. `promising` is derived from the round's trace findings (the analyst seam),
|
|
1176
|
+
* never a child's raw `verdict` — and the default gate (`flatWidenGate`) never widens, so the R2
|
|
1177
|
+
* firewall stays dormant. Terminal selection is `spec.synthesize` over every settled lineage.
|
|
1178
|
+
*
|
|
1179
|
+
* When `ctx.analyst` is set, `decide` is consulted with that round's trace-derived findings;
|
|
1180
|
+
* absent an analyst the findings argument is the empty array a flat gate ignores. The analyst
|
|
1181
|
+
* spawns into THIS scope (conserved-pooled, so equal-k holds). Streaming caveat: a wired analyst
|
|
1182
|
+
* drains its own child off the SHARED cursor by id-match, so on a NON-flat gate (which spawns
|
|
1183
|
+
* widen children that are live concurrently) the analyst can consume a sibling's settlement before
|
|
1184
|
+
* the widen loop sees it. The shipped default (`flatWidenGate`) never widens, so no widen child is
|
|
1185
|
+
* ever live when the analyst runs and the wire is exact; a non-flat gate must drive the analyst on
|
|
1186
|
+
* a scope whose siblings are quiesced, or read findings without the shared-cursor drain.
|
|
1187
|
+
*/
|
|
1188
|
+
declare function widen<Task, Seed, D>(spec: WidenSpec<Seed, D>): CombinatorShape<Task, D>;
|
|
1189
|
+
/**
|
|
1190
|
+
* The flat default `ScopeWidenGate` — never widens, keeping the R2 selector≠judge collision
|
|
1191
|
+
* dormant. A gate run passes this explicitly; a test asserts the default is flat.
|
|
1192
|
+
*/
|
|
1193
|
+
declare function flatWidenGate<D>(): ScopeWidenGate<D>;
|
|
1194
|
+
|
|
1195
|
+
/**
|
|
1196
|
+
* @experimental
|
|
1197
|
+
*
|
|
1198
|
+
* The cross-run corpus (G2) — the learning-flywheel's durable accreted-fact store.
|
|
1199
|
+
*
|
|
1200
|
+
* `Corpus` is DISTINCT from the per-run `SpawnJournal` (decisions/replay) and `ResultBlobStore`
|
|
1201
|
+
* (payloads): a `CorpusRecord` is a FACT one run LEARNED that a FUTURE run reads back (the
|
|
1202
|
+
* world-model), not a replay input. This module owns the two impls the wave surface pins —
|
|
1203
|
+
* `InMemoryCorpus` and `FileCorpus` (JSONL, append-only) — plus `renderCorpusToInstructions`,
|
|
1204
|
+
* the READ side that projects accreted facts into a fresh `AgentProfile`'s instruction seams.
|
|
1205
|
+
*
|
|
1206
|
+
* The boundary is fail-loud, typed-outcome: `append` is idempotent on an identical record and
|
|
1207
|
+
* returns a typed error (never throws, never a silent overwrite) on a conflicting re-append under
|
|
1208
|
+
* the same `id`. Malformed records — a structurally-invalid `CorpusRecord` from disk or a caller —
|
|
1209
|
+
* fail loud (the validator throws), since a corpus that silently accepts garbage would poison
|
|
1210
|
+
* every downstream run that reads it back.
|
|
1211
|
+
*/
|
|
1212
|
+
|
|
1213
|
+
/**
|
|
1214
|
+
* In-memory `Corpus`. Keyed by record `id`; `append` validates the record, is idempotent on an
|
|
1215
|
+
* identical re-append, and returns a typed `{ succeeded: false }` on a conflicting re-append under
|
|
1216
|
+
* the same `id` (never overwrites). `query` routes through the single-sourced `applyFilter`.
|
|
1217
|
+
*/
|
|
1218
|
+
declare class InMemoryCorpus implements Corpus {
|
|
1219
|
+
private readonly byId;
|
|
1220
|
+
append(record: CorpusRecord): Promise<{
|
|
1221
|
+
succeeded: true;
|
|
1222
|
+
} | {
|
|
1223
|
+
succeeded: false;
|
|
1224
|
+
error: string;
|
|
1225
|
+
}>;
|
|
1226
|
+
query(filter: CorpusFilter): Promise<ReadonlyArray<CorpusRecord>>;
|
|
1227
|
+
}
|
|
1228
|
+
/**
|
|
1229
|
+
* JSONL on disk — one validated `CorpusRecord` per line, append-only. `query` replays the whole
|
|
1230
|
+
* file, validating every line (a malformed line fails loud — a corrupted corpus must never read
|
|
1231
|
+
* back silently) and folding by `id`: a later identical line dedups, a later conflicting line
|
|
1232
|
+
* under the same `id` is a corruption (fail loud). `append` first replays to enforce the same
|
|
1233
|
+
* idempotence/conflict contract as the in-mem impl, then fsyncs the new line so a crash between
|
|
1234
|
+
* writes never loses an acknowledged fact. Shares the JSONL append-line spine with the spawn
|
|
1235
|
+
* journal, but the interface stays separate (a learned fact is not a replay record).
|
|
1236
|
+
*/
|
|
1237
|
+
declare class FileCorpus implements Corpus {
|
|
1238
|
+
private readonly path;
|
|
1239
|
+
constructor(path: string);
|
|
1240
|
+
append(record: CorpusRecord): Promise<{
|
|
1241
|
+
succeeded: true;
|
|
1242
|
+
} | {
|
|
1243
|
+
succeeded: false;
|
|
1244
|
+
error: string;
|
|
1245
|
+
}>;
|
|
1246
|
+
query(filter: CorpusFilter): Promise<ReadonlyArray<CorpusRecord>>;
|
|
1247
|
+
private load;
|
|
1248
|
+
private appendLine;
|
|
1249
|
+
}
|
|
1250
|
+
/**
|
|
1251
|
+
* The learning-flywheel READ side. Queries the corpus through `filter`, renders the matching facts
|
|
1252
|
+
* (most-confident first, capped by `maxLines`) into instruction lines, and returns a FRESH
|
|
1253
|
+
* `AgentProfile` with them merged in — never mutates the input profile. Default `target: 'prompt'`
|
|
1254
|
+
* appends the lines to `prompt.instructions[]` (the additive append-line seam); `target:
|
|
1255
|
+
* 'resources'` folds them into the single-blob `resources.instructions` string (preserving any
|
|
1256
|
+
* existing blob, but failing loud on a non-string existing blob — a `resources.instructions` that
|
|
1257
|
+
* was already an `AgentProfileResourceRef` cannot be string-appended without dropping it).
|
|
1258
|
+
*
|
|
1259
|
+
* An empty query result returns a fresh COPY of the profile with no instruction change (a valid
|
|
1260
|
+
* "nothing learned yet" read, not an error).
|
|
1261
|
+
*/
|
|
1262
|
+
declare function renderCorpusToInstructions(opts: RenderCorpusToInstructionsOptions): Promise<AgentProfile$1>;
|
|
1263
|
+
|
|
1264
|
+
/**
|
|
1265
|
+
* @experimental
|
|
1266
|
+
*
|
|
1267
|
+
* The personify layer impl — `definePersona` (the thin builder) + `runPersonified` (composes
|
|
1268
|
+
* the persona + chosen shape onto the keystone `Supervisor`), plus `createShapeContext`, the
|
|
1269
|
+
* seam that hands a shape its spawn helpers without it touching the registry.
|
|
1270
|
+
*
|
|
1271
|
+
* This file adds NO engine: `runPersonified` is `createSupervisor().run(rootAgent, task, …)`
|
|
1272
|
+
* where `rootAgent` is the persona's chosen `LoopShape` applied to a `ShapeContext`. All the
|
|
1273
|
+
* conserved-budget / journal / abort / typed-result machinery is the keystone's; this layer
|
|
1274
|
+
* only wires the persona's CONTENT (root spec + directive + context + seams) into it.
|
|
1275
|
+
*
|
|
1276
|
+
* One non-obvious invariant it must honor: `createSupervisor().run` builds the root `Scope`
|
|
1277
|
+
* with an EMPTY seam bag (`seams: {}`), so the built-in metered runtimes (router/sandbox/cli)
|
|
1278
|
+
* cannot read their seams off `ExecutorContext` through the default supervisor path. A persona
|
|
1279
|
+
* that supplies raw `seams` is therefore wrapped here into a registry whose resolved factories
|
|
1280
|
+
* receive a ctx with the persona seams merged in — so a persona never has to pre-close its
|
|
1281
|
+
* factories by hand. A persona may instead supply a fully-built `registry` and skip the wrap.
|
|
1282
|
+
*/
|
|
1283
|
+
|
|
1284
|
+
/**
|
|
1285
|
+
* Build a frozen `Persona`. Fails loud on the executors-supplied invariant: a persona with
|
|
1286
|
+
* neither a pre-built registry nor a seam bag cannot resolve its built-in runtimes, so it is
|
|
1287
|
+
* unrunnable — refuse it at definition time, not at the first spawn. Pure; no I/O.
|
|
1288
|
+
*/
|
|
1289
|
+
declare function definePersona<D = unknown>(input: DefinePersonaInput<D>): Persona<D>;
|
|
1290
|
+
/**
|
|
1291
|
+
* Compose the persona + chosen shape onto a fresh keystone `Supervisor`. Resolves the shape
|
|
1292
|
+
* (a factory verbatim, or a registered name through `builtinShapes`), applies it to a
|
|
1293
|
+
* `ShapeContext`, and runs the resulting root `Agent` to a typed `SupervisedResult<Outcome>`.
|
|
1294
|
+
* Fail loud on an unknown shape name or an unresolvable persona registry — never a silent
|
|
1295
|
+
* default-shape fallback.
|
|
1296
|
+
*/
|
|
1297
|
+
declare function runPersonified<Task, D>(options: RunPersonifiedOptions<Task, D>): Promise<SupervisedResult<Outcome<D>>>;
|
|
1298
|
+
|
|
1299
|
+
/**
|
|
1300
|
+
* @experimental
|
|
1301
|
+
*
|
|
1302
|
+
* The loop-shape registry — the OPEN, content-free extension point for the personify layer.
|
|
1303
|
+
*
|
|
1304
|
+
* A `LoopShape` is reusable STRUCTURE (how to decompose / fan out / verify / synthesize),
|
|
1305
|
+
* parameterized by a persona's CONTENT. The registry lets a caller resolve a composed shape by
|
|
1306
|
+
* NAME: register a factory once, then `runPersonified({ shape: '<name>' })` resolves it with zero
|
|
1307
|
+
* edits elsewhere. `register` fails loud on a duplicate; `resolve` returns a typed outcome so an
|
|
1308
|
+
* unknown name is a named error, never a silent default.
|
|
1309
|
+
*
|
|
1310
|
+
* No shape is pre-registered: the generic combinators (`pipeline`/`fanout`/`loopUntil`/`panel`/
|
|
1311
|
+
* `verify`/`widen`) take spec arguments, so they are not bare zero-arg factories — a caller that
|
|
1312
|
+
* wants name-resolution registers its own COMPOSED shape (a combinator already applied to its
|
|
1313
|
+
* spec) on a registry instance. The registry carries SHAPE only; the domain lives on the persona.
|
|
1314
|
+
*/
|
|
1315
|
+
|
|
1316
|
+
/**
|
|
1317
|
+
* Build a fresh open `ShapeRegistry`. A factory is stored type-erased and re-cast on resolve — the
|
|
1318
|
+
* caller asserts the `<Task, D>` it expects, exactly as the executor registry stores its factories.
|
|
1319
|
+
*/
|
|
1320
|
+
declare function createShapeRegistry(): ShapeRegistry;
|
|
1321
|
+
/** The default registry `runPersonified` resolves a shape name against. Empty by construction —
|
|
1322
|
+
* a caller registers its own composed shapes; the engine ships no domain shape. */
|
|
1323
|
+
declare const builtinShapes: ShapeRegistry;
|
|
1324
|
+
/** Register a composed shape on the default `builtinShapes` registry — the one-call extension
|
|
1325
|
+
* point a caller invokes so its shape is resolvable by name with zero edits to the engine. */
|
|
1326
|
+
declare function registerShape<Task, D>(name: string, factory: LoopShape<Task, D>): void;
|
|
1327
|
+
|
|
1328
|
+
/**
|
|
1329
|
+
* @experimental
|
|
1330
|
+
*
|
|
1331
|
+
* Trajectory trace + cost ledger — the post-hoc tree reconstructor (§4 of `wave-types`).
|
|
1332
|
+
*
|
|
1333
|
+
* `trajectoryReport` rebuilds the WHOLE realized spawn tree from the durable
|
|
1334
|
+
* `SpawnJournal` (+ optionally the `ResultBlobStore` for `done` artifacts): every node
|
|
1335
|
+
* (driver AND leaf), the real parent/child edges, each node's terminal status, its OWN
|
|
1336
|
+
* conserved `Spend`, and the `Spend` ROLLED UP over its subtree. Roll-up is a post-order
|
|
1337
|
+
* fold over the parent edges: a node's `rolledUpSpend` is its own spend plus every
|
|
1338
|
+
* descendant's, so a driver is charged for the fanout it caused — the root's roll-up is
|
|
1339
|
+
* the whole run's conserved total (tokens + usd + iterations + ms).
|
|
1340
|
+
*
|
|
1341
|
+
* `equalKOnCost` compares separate runs (arms) on that conserved COST, not on raw
|
|
1342
|
+
* iteration COUNT. The sandbox executor reports tokens/usd INCLUSIVE of a leaf's internal
|
|
1343
|
+
* sub-agent fanout, so charging an arm by `total.tokens`/`total.usd` (not by how many
|
|
1344
|
+
* `next()` cursors it logged) closes the leaf-fanout confound: a treatment leaf that fanned
|
|
1345
|
+
* out internally pays for it in cost, where a per-iteration count would hide it. The
|
|
1346
|
+
* within-run conserved pool already guarantees `Σk` equal by construction; this check is the
|
|
1347
|
+
* CROSS-run analogue the pool cannot reach — proving equal compute before any win is claimed.
|
|
1348
|
+
*
|
|
1349
|
+
* Pure over the journal/blobs — no live agent calls; safe to run on a finished run's log.
|
|
1350
|
+
*/
|
|
1351
|
+
|
|
1352
|
+
/**
|
|
1353
|
+
* Reconstruct the whole spawn tree for `root` with per-node + rolled-up `Spend`. Reads the
|
|
1354
|
+
* journal for structure + spend and, when `withOutputs`, the blob store for each `done`
|
|
1355
|
+
* node's artifact. Fail loud on a tree that was never journaled, a settle/cancel for an
|
|
1356
|
+
* un-spawned node (a corrupted log), or — under `withOutputs` — a `done` node whose blob the
|
|
1357
|
+
* store cannot rehydrate (a silent gap would mis-cost or mis-evidence the tree).
|
|
1358
|
+
*/
|
|
1359
|
+
declare function trajectoryReport(journal: SpawnJournal, blobs: ResultBlobStore, root: NodeId, options?: TrajectoryReportOptions): Promise<TrajectoryReport>;
|
|
1360
|
+
/**
|
|
1361
|
+
* Assert the arms are comparable at EQUAL conserved COST (tokens + usd), NOT raw iteration
|
|
1362
|
+
* count. Compares each arm's root-rolled-up `total` on the two conserved channels: an arm is
|
|
1363
|
+
* within-tolerance when the per-channel spread (max − min across arms) over the median is
|
|
1364
|
+
* `≤ tolerance`. Pure over the reports — no I/O. Fails loud on an empty arm list (nothing to
|
|
1365
|
+
* compare) so a vacuous "equal" is never returned.
|
|
1366
|
+
*/
|
|
1367
|
+
declare function equalKOnCost(arms: ReadonlyArray<EqualKArm>, options?: EqualKOnCostOptions): EqualKVerdict;
|
|
1368
|
+
|
|
1369
|
+
interface PromotionGateOptions {
|
|
1370
|
+
/** The HOLDOUT report — must carry per-task cells for both strategy names. */
|
|
1371
|
+
report: BenchmarkReport;
|
|
1372
|
+
/** The incumbent champion's strategy name. */
|
|
1373
|
+
incumbent: string;
|
|
1374
|
+
/** The challenger's strategy name. */
|
|
1375
|
+
candidate: string;
|
|
1376
|
+
/** 'superiority' (default): the candidate must score significantly BETTER.
|
|
1377
|
+
* 'non-inferiority': the candidate must prove its score is not worse than the
|
|
1378
|
+
* incumbent by more than `scoreTolerance` AND its cost savings are significant —
|
|
1379
|
+
* the gate for "same quality, cheaper" claims. */
|
|
1380
|
+
mode?: 'superiority' | 'non-inferiority';
|
|
1381
|
+
/** non-inferiority: the score CI lower bound must clear −scoreTolerance. Default 0.05. */
|
|
1382
|
+
scoreTolerance?: number;
|
|
1383
|
+
/** The CI lower bound on the paired lift must EXCEED this (score scale). Default 0. */
|
|
1384
|
+
deltaThreshold?: number;
|
|
1385
|
+
/** Minimum paired tasks before significance can be claimed. Default 6 — below that
|
|
1386
|
+
* the bootstrap CI is too wide to separate a real lift from the per-task noise. */
|
|
1387
|
+
minPairedTasks?: number;
|
|
1388
|
+
/** Bootstrap statistic over the paired deltas. Default 'mean'. */
|
|
1389
|
+
statistic?: 'mean' | 'median';
|
|
1390
|
+
/** Fixed by the substrate by default — the same report always yields the same verdict. */
|
|
1391
|
+
seed?: number;
|
|
1392
|
+
resamples?: number;
|
|
1393
|
+
}
|
|
1394
|
+
interface PromotionVerdict {
|
|
1395
|
+
promoted: boolean;
|
|
1396
|
+
reason: 'identical-champion' | 'few-tasks' | 'no-margin' | 'significant' | 'non-inferior-and-cheaper' | 'non-inferiority-unproven' | 'not-cheaper';
|
|
1397
|
+
mode: 'superiority' | 'non-inferiority';
|
|
1398
|
+
/** Paired tasks that carried both strategies' cells. */
|
|
1399
|
+
n: number;
|
|
1400
|
+
/** Paired (candidate − incumbent) lift across the holdout tasks. */
|
|
1401
|
+
lift: {
|
|
1402
|
+
mean: number;
|
|
1403
|
+
median: number;
|
|
1404
|
+
low: number;
|
|
1405
|
+
high: number;
|
|
1406
|
+
};
|
|
1407
|
+
/** non-inferiority mode: paired (incumbent − candidate) cost SAVINGS per task (usd) —
|
|
1408
|
+
* positive means the candidate is cheaper; significant iff the CI low clears zero. */
|
|
1409
|
+
costSavings?: {
|
|
1410
|
+
mean: number;
|
|
1411
|
+
median: number;
|
|
1412
|
+
low: number;
|
|
1413
|
+
high: number;
|
|
1414
|
+
};
|
|
1415
|
+
/** Paired (candidate − incumbent) wall-clock per task (ms) — negative = the candidate
|
|
1416
|
+
* is FASTER. Informational in every mode (never gates); the latency answer to "what
|
|
1417
|
+
* does this win actually cost the user?". */
|
|
1418
|
+
latency?: {
|
|
1419
|
+
mean: number;
|
|
1420
|
+
median: number;
|
|
1421
|
+
low: number;
|
|
1422
|
+
high: number;
|
|
1423
|
+
};
|
|
1424
|
+
}
|
|
1425
|
+
declare function promotionGate(opts: PromotionGateOptions): PromotionVerdict;
|
|
1426
|
+
|
|
1427
|
+
/**
|
|
1428
|
+
* Bridge a finished `runLoop` into an agent-eval campaign / profile-matrix
|
|
1429
|
+
* dispatch.
|
|
1430
|
+
*
|
|
1431
|
+
* `runProfileMatrix` (and `runCampaign`) run the backend-integrity guard over
|
|
1432
|
+
* the token usage a dispatch reports through `ctx.cost`. A dispatch that wraps
|
|
1433
|
+
* `runLoop` must forward the loop's cost AND token usage, or the guard reads
|
|
1434
|
+
* the run as a stub and throws. `reportLoopUsage` is that one line:
|
|
1435
|
+
*
|
|
1436
|
+
* const dispatch: ProfileDispatchFn<S, A> = async (profile, scenario, ctx) => {
|
|
1437
|
+
* const result = await runLoop({ ...optsFor(profile, scenario), ctx: loopCtx })
|
|
1438
|
+
* reportLoopUsage(ctx, result)
|
|
1439
|
+
* return result.winner?.output as A
|
|
1440
|
+
* }
|
|
1441
|
+
*
|
|
1442
|
+
* Typed structurally against the campaign `DispatchContext.cost` so this module
|
|
1443
|
+
* stays free of an agent-eval import — it works with any cost meter exposing
|
|
1444
|
+
* `observe` + `observeTokens`.
|
|
1445
|
+
*/
|
|
1446
|
+
|
|
1447
|
+
/** The slice of an agent-eval campaign `DispatchContext.cost` this needs. */
|
|
1448
|
+
interface UsageSink {
|
|
1449
|
+
observe(amountUsd: number, source: string): void;
|
|
1450
|
+
observeTokens(usage: LoopTokenUsage): void;
|
|
1451
|
+
}
|
|
1452
|
+
/**
|
|
1453
|
+
* Forward a `LoopResult`'s aggregated cost + token usage into a campaign cost
|
|
1454
|
+
* meter so the backend-integrity guard sees real LLM activity. `source`
|
|
1455
|
+
* defaults to `'loop'`.
|
|
1456
|
+
*/
|
|
1457
|
+
declare function reportLoopUsage<Task, Output, Decision>(cost: UsageSink, result: Pick<LoopResult<Task, Output, Decision>, 'costUsd' | 'tokenUsage'>, source?: string): void;
|
|
1458
|
+
|
|
1459
|
+
/**
|
|
1460
|
+
* @experimental
|
|
1461
|
+
*
|
|
1462
|
+
* `acquireSandbox` — cold-start-resilient sandbox acquisition. Eliminates the
|
|
1463
|
+
* "create timed out at the proxy" failure mode conceptually by DECOUPLING "the
|
|
1464
|
+
* create HTTP call returned" from "the sandbox is ready":
|
|
1465
|
+
*
|
|
1466
|
+
* - Create is initiated with a known `name`.
|
|
1467
|
+
* - Readiness is observed from the sandbox's own `status` (`refresh()` polls
|
|
1468
|
+
* true state), NOT from whether the create call returned in time.
|
|
1469
|
+
* - If the create call itself times out at a gateway (502/503/504/522/524 or
|
|
1470
|
+
* a transport timeout), provisioning is still running server-side — so we
|
|
1471
|
+
* find the named sandbox via `list()` and wait for it to reach `running`.
|
|
1472
|
+
*
|
|
1473
|
+
* Result: a scale-from-zero cold start (node boot + host-agent registration,
|
|
1474
|
+
* minutes) can no longer surface as a create failure behind a ~100s proxy
|
|
1475
|
+
* limit. The loop becomes indifferent to whether the host pool is warm or cold.
|
|
1476
|
+
*
|
|
1477
|
+
* Invariant: an instance reporting no `status` (the minimal test fakes) is
|
|
1478
|
+
* treated as ready; only an explicit `pending`/`provisioning` status triggers
|
|
1479
|
+
* waiting, and only a retryable THROW triggers the find-by-name path. Real
|
|
1480
|
+
* errors (auth, validation, budget) fail loud. A box that is created (or found)
|
|
1481
|
+
* but never reaches `running` (abort, terminal status, budget) is torn down
|
|
1482
|
+
* before the failure propagates, so an abort storm during cold start does not
|
|
1483
|
+
* leak live sandboxes.
|
|
1484
|
+
*/
|
|
1485
|
+
|
|
1486
|
+
/** @experimental */
|
|
1487
|
+
interface AcquireOptions {
|
|
1488
|
+
/**
|
|
1489
|
+
* Total budget for the sandbox to reach `running`, covering on-demand node
|
|
1490
|
+
* cold-start. Default 600_000ms — matches the orchestrator's pending-host
|
|
1491
|
+
* registration window so we never give up before the platform itself would.
|
|
1492
|
+
*/
|
|
1493
|
+
readyTimeoutMs?: number;
|
|
1494
|
+
/** Poll interval while waiting for `running` / for the named sandbox to appear. */
|
|
1495
|
+
pollIntervalMs?: number;
|
|
1496
|
+
/** Cancellation (user abort). Distinct from create-call timeouts. */
|
|
1497
|
+
signal?: AbortSignal;
|
|
1498
|
+
/** Stamp a name so a timed-out create is recoverable by lookup. Auto-generated if absent. */
|
|
1499
|
+
name?: string;
|
|
1500
|
+
/** Clock override for deterministic tests. */
|
|
1501
|
+
now?: () => number;
|
|
1502
|
+
/** Sleep override for deterministic tests. */
|
|
1503
|
+
sleep?: (ms: number) => Promise<void>;
|
|
1504
|
+
}
|
|
1505
|
+
/** @experimental */
|
|
1506
|
+
declare function acquireSandbox(client: SandboxClient, options: CreateSandboxOptions, acquire?: AcquireOptions): Promise<SandboxInstance>;
|
|
1507
|
+
|
|
1508
|
+
/**
|
|
1509
|
+
* @experimental
|
|
1510
|
+
*
|
|
1511
|
+
* Capability probe for the loop kernel's backend-blind lineage seams. The
|
|
1512
|
+
* kernel must NEVER ask "is this Docker or Firecracker?"; it asks "can this
|
|
1513
|
+
* platform fork a checkpoint?" via `client.criuStatus()` and degrades to fresh
|
|
1514
|
+
* boxes when the answer is no. CRIU availability is a per-platform fact, so the
|
|
1515
|
+
* probe is memoized per client — one network round-trip, reused across every
|
|
1516
|
+
* fanout in the run.
|
|
1517
|
+
*
|
|
1518
|
+
* Invariant: a client with no `criuStatus` method (the loop's test fakes, the
|
|
1519
|
+
* raw SDK before it grew the probe) reports `canFork = false`. The seam is
|
|
1520
|
+
* fail-CLOSED — never assume forking works, only enable it on a positive probe.
|
|
1521
|
+
*/
|
|
1522
|
+
|
|
1523
|
+
/**
|
|
1524
|
+
* What the loop kernel is allowed to know about a sandbox backend: a single
|
|
1525
|
+
* capability bit, never the backend's identity. `canFork` gates the
|
|
1526
|
+
* checkpoint+fork fanout path; everything else (session continuation) is a
|
|
1527
|
+
* universal SDK feature that needs no probe.
|
|
1528
|
+
*
|
|
1529
|
+
* @experimental
|
|
1530
|
+
*/
|
|
1531
|
+
interface SandboxCapabilities {
|
|
1532
|
+
/**
|
|
1533
|
+
* True only when `client.criuStatus()` returned `{ available: true }`. When
|
|
1534
|
+
* false, a fork-enabled fanout degrades to independent fresh boxes — same
|
|
1535
|
+
* result, no shared context prefix.
|
|
1536
|
+
*/
|
|
1537
|
+
canFork: boolean;
|
|
1538
|
+
}
|
|
1539
|
+
/**
|
|
1540
|
+
* Probe (and memoize per client) what the loop may rely on. A client without a
|
|
1541
|
+
* `criuStatus` method, or whose probe rejects, yields `canFork = false` — a
|
|
1542
|
+
* failed probe must never claim a capability the platform may not have. The
|
|
1543
|
+
* promise is cached so concurrent fanout branches share one round-trip.
|
|
1544
|
+
*
|
|
1545
|
+
* @experimental
|
|
1546
|
+
*/
|
|
1547
|
+
declare function probeSandboxCapabilities(client: SandboxClient): Promise<SandboxCapabilities>;
|
|
1548
|
+
/**
|
|
1549
|
+
* Narrowed view of the optional CRIU probe. The loop-side `SandboxClient`
|
|
1550
|
+
* does not require `criuStatus`; this widens it optionally so the probe can be
|
|
1551
|
+
* read without importing sandbox-backend specifics. @experimental
|
|
1552
|
+
*/
|
|
1553
|
+
interface CriuCapableClient {
|
|
1554
|
+
criuStatus?: () => Promise<{
|
|
1555
|
+
available: boolean;
|
|
1556
|
+
criuVersion?: string;
|
|
1557
|
+
reason?: string;
|
|
1558
|
+
}>;
|
|
1559
|
+
}
|
|
1560
|
+
|
|
1561
|
+
/**
|
|
1562
|
+
* Sandbox-event → runtime-event mapping.
|
|
1563
|
+
*
|
|
1564
|
+
* The sandbox SDK emits a polymorphic `SandboxEvent = { type, data, id? }`
|
|
1565
|
+
* whose `type` vocabulary is backend-determined (opencode, etc.) rather than
|
|
1566
|
+
* enumerated by the SDK. Two consumers project it:
|
|
1567
|
+
* - the loop kernel's cost ledger (`extractLlmCallEvent`) — sums usage off
|
|
1568
|
+
* every cost-bearing event, regardless of stream shape;
|
|
1569
|
+
* - the `AgentRuntime.act` streaming contract (`mapSandboxEvent`) — projects
|
|
1570
|
+
* incremental events to the `RuntimeStreamEvent` chat-UX vocabulary.
|
|
1571
|
+
*
|
|
1572
|
+
* Both live here so the empirically-observed `type` vocabulary has one home.
|
|
1573
|
+
*/
|
|
1574
|
+
|
|
1575
|
+
/**
|
|
1576
|
+
* Extract a `RuntimeStreamEvent`-shaped `llm_call` from a sandbox event when
|
|
1577
|
+
* the event carries usage/cost data. Returns `undefined` for non-cost events
|
|
1578
|
+
* so the kernel can iterate the full stream without branching.
|
|
1579
|
+
*
|
|
1580
|
+
* Canonical cost-carrying types observed in the wild:
|
|
1581
|
+
* - `llm_call` — `data: { model, tokensIn, tokensOut, costUsd, ... }`
|
|
1582
|
+
* - `message.completed` / `result` — `data: { usage: { inputTokens,
|
|
1583
|
+
* outputTokens, totalCostUsd? } }`
|
|
1584
|
+
* - `cost.usage` / `usage` — same shape under a dedicated type
|
|
1585
|
+
*
|
|
1586
|
+
* Numeric coercion is strict: `Number.isFinite` gates every accumulator write
|
|
1587
|
+
* so a sentinel `NaN` from a misbehaving backend cannot poison the ledger.
|
|
1588
|
+
*/
|
|
1589
|
+
declare function extractLlmCallEvent(event: SandboxEvent, agentRunName: string): (RuntimeStreamEvent & {
|
|
1590
|
+
type: 'llm_call';
|
|
1591
|
+
}) | undefined;
|
|
1592
|
+
/**
|
|
1593
|
+
* Project one `SandboxEvent` onto the `RuntimeStreamEvent` chat-UX vocabulary,
|
|
1594
|
+
* for runtimes that bridge a sandbox `streamPrompt` into the
|
|
1595
|
+
* `AgentRuntime.act` streaming contract. Returns `undefined` for events that
|
|
1596
|
+
* have no faithful projection — the raw stream is preserved separately for the
|
|
1597
|
+
* `OutputAdapter`, so an unmapped event never loses data.
|
|
1598
|
+
*
|
|
1599
|
+
* Mapped (the task-optional incremental variants — no synthesized task
|
|
1600
|
+
* lifecycle, no guessed tool-part shapes):
|
|
1601
|
+
* - `message.part.updated` text part → `text_delta`
|
|
1602
|
+
* - `message.part.updated` reasoning/thinking part → `reasoning_delta`
|
|
1603
|
+
* - cost-bearing events → `llm_call` (shared with the ledger extractor)
|
|
1604
|
+
*
|
|
1605
|
+
* The opencode backend emits incremental text as
|
|
1606
|
+
* `{ type: 'message.part.updated', data: { part: { type, text }, delta } }`;
|
|
1607
|
+
* `delta` is the increment, `part.text` the running accumulation.
|
|
1608
|
+
*/
|
|
1609
|
+
declare function mapSandboxEvent(event: SandboxEvent, opts?: {
|
|
1610
|
+
agentRunName?: string;
|
|
1611
|
+
}): RuntimeStreamEvent | undefined;
|
|
1612
|
+
|
|
1613
|
+
/**
|
|
1614
|
+
* @experimental
|
|
1615
|
+
*
|
|
1616
|
+
* `SandboxLineage` — the backend-blind owner of box + session handles for a
|
|
1617
|
+
* single `runLoop` invocation. It exists so `run-loop.ts` never references a
|
|
1618
|
+
* backend (Docker / Firecracker): the lineage turns "continue this session" and
|
|
1619
|
+
* "fork this branch" into capability-gated sandbox-SDK calls and degrades to
|
|
1620
|
+
* fresh boxes when a capability is absent.
|
|
1621
|
+
*
|
|
1622
|
+
* Three operations, mirroring the kernel's per-iteration choices:
|
|
1623
|
+
* - `start(spec, prompt)` → a fresh box; the FIRST `streamPrompt` carries a
|
|
1624
|
+
* minted `sessionId` so later `continue` calls reuse the same server-side
|
|
1625
|
+
* conversation instead of re-injecting prior context as prompt text.
|
|
1626
|
+
* - `continue(handle, prompt)` → the SAME box, `streamPrompt({ sessionId })`.
|
|
1627
|
+
* The context lives in the sandbox; the prompt is only the new turn. Before
|
|
1628
|
+
* streaming it ASSERTS the session is still live server-side (via
|
|
1629
|
+
* `box.session(id).status()`): if the platform never honored the
|
|
1630
|
+
* client-minted id (or reaped it), `status()` is `null` and `continue`
|
|
1631
|
+
* fails loud rather than silently re-running the turn without prior context.
|
|
1632
|
+
* - `fork(handle, n, ...)` → when `canFork`, `checkpoint({ leaveRunning })` on
|
|
1633
|
+
* the parent then `fork(checkpointId)` × n so N branches inherit a shared
|
|
1634
|
+
* context prefix; otherwise N independent fresh boxes (same result, no
|
|
1635
|
+
* prefix). Either way each branch streams its own turn. Child-box creation
|
|
1636
|
+
* is bounded by the lineage's `maxConcurrency` — a 20-way fanout under a
|
|
1637
|
+
* concurrency cap of 2 provisions boxes in bounded waves, not all at once.
|
|
1638
|
+
*
|
|
1639
|
+
* Invariant: the lineage OWNS every box it starts or forks and tears them all
|
|
1640
|
+
* down on `teardown()` (or earlier via `prune`). It never tears down a box
|
|
1641
|
+
* mid-flight — the kernel decides when a handle is done. Streaming itself stays
|
|
1642
|
+
* in `run-loop.ts`; the lineage only hands back the live `streamPrompt` iterable
|
|
1643
|
+
* so the kernel keeps ownership of event collection, cost accounting, and trace
|
|
1644
|
+
* emission.
|
|
1645
|
+
*/
|
|
1646
|
+
|
|
1647
|
+
/**
|
|
1648
|
+
* A live box plus the session that threads its iterations together. Handed back
|
|
1649
|
+
* by `start`/`fork`, passed into `continue`/`fork` to descend from. Opaque to
|
|
1650
|
+
* the kernel beyond `box` (for placement/teardown) and `sessionId` (trace).
|
|
1651
|
+
*
|
|
1652
|
+
* @experimental
|
|
1653
|
+
*/
|
|
1654
|
+
interface SandboxLineageHandle {
|
|
1655
|
+
/** The owned, running sandbox this handle drives. */
|
|
1656
|
+
box: SandboxInstance;
|
|
1657
|
+
/**
|
|
1658
|
+
* Stable session id threaded through this box's `streamPrompt` calls. Minted
|
|
1659
|
+
* by the lineage on `start`; reused on `continue` so the server continues the
|
|
1660
|
+
* same conversation. A forked handle starts a fresh session on its new box —
|
|
1661
|
+
* the shared context comes from the checkpoint, not a shared session id.
|
|
1662
|
+
*/
|
|
1663
|
+
sessionId: string;
|
|
1664
|
+
}
|
|
1665
|
+
/**
|
|
1666
|
+
* Owns box + session handles for one loop run and offers the three
|
|
1667
|
+
* capability-gated lifecycle moves. Construct via `createSandboxLineage`.
|
|
1668
|
+
*
|
|
1669
|
+
* @experimental
|
|
1670
|
+
*/
|
|
1671
|
+
interface SandboxLineage {
|
|
1672
|
+
/**
|
|
1673
|
+
* Acquire a fresh box and begin a new session on it. Returns the handle and
|
|
1674
|
+
* the live `streamPrompt` iterable for the first turn (caller drains it).
|
|
1675
|
+
*/
|
|
1676
|
+
start(spec: AgentRunSpec<unknown>, prompt: string, signal: AbortSignal): Promise<{
|
|
1677
|
+
handle: SandboxLineageHandle;
|
|
1678
|
+
events: AsyncIterable<SandboxEvent>;
|
|
1679
|
+
}>;
|
|
1680
|
+
/**
|
|
1681
|
+
* Continue an existing handle's session with one more turn on the SAME box.
|
|
1682
|
+
* The prior context is server-side; `prompt` is only the new turn. Asserts the
|
|
1683
|
+
* session is still known to the sandbox first (fail-loud) so a platform that
|
|
1684
|
+
* silently dropped the client-minted session id surfaces as an error instead
|
|
1685
|
+
* of a contextless turn the caller mistakes for a real continuation.
|
|
1686
|
+
*/
|
|
1687
|
+
continue(handle: SandboxLineageHandle, prompt: string, signal: AbortSignal): Promise<AsyncIterable<SandboxEvent>>;
|
|
1688
|
+
/**
|
|
1689
|
+
* Branch `count` children from `parent`. When the platform can fork, each
|
|
1690
|
+
* child inherits `parent`'s checkpoint — and therefore the parent's IMAGE and
|
|
1691
|
+
* PROFILE: under a real fork `specs[i]` does NOT re-select a per-branch
|
|
1692
|
+
* profile (the SDK forks the running box, it can't swap the image). `specs[i]`
|
|
1693
|
+
* picks the per-branch profile ONLY on the degraded fresh-box path (no CRIU).
|
|
1694
|
+
* A heterogeneous-profile fanout therefore homogenizes to the parent's profile
|
|
1695
|
+
* when fork is available — pass a single shared spec for forked fanouts, or
|
|
1696
|
+
* use `random@k` (no fork) when branches must differ. Each child's first turn
|
|
1697
|
+
* streams `prompts[i]`. Child-box creation is bounded by `maxConcurrency`.
|
|
1698
|
+
*/
|
|
1699
|
+
fork(parent: SandboxLineageHandle, prompts: string[], specs: AgentRunSpec<unknown>[], signal: AbortSignal): Promise<{
|
|
1700
|
+
handle: SandboxLineageHandle;
|
|
1701
|
+
events: AsyncIterable<SandboxEvent>;
|
|
1702
|
+
}[]>;
|
|
1703
|
+
/**
|
|
1704
|
+
* Destroy every owned box whose handle is NOT in `keep`, freeing it before
|
|
1705
|
+
* loop end. The kernel calls this after a round when it can prove no future
|
|
1706
|
+
* round will descend from the pruned boxes (deterministic, monotonic branch
|
|
1707
|
+
* selection); boxes still reachable as a future branch source are retained.
|
|
1708
|
+
* Best-effort, bounded, parallel — a failed delete never throws.
|
|
1709
|
+
*/
|
|
1710
|
+
prune(keep: Iterable<SandboxLineageHandle>): Promise<void>;
|
|
1711
|
+
/** Destroy every box this lineage owns. Best-effort, bounded, parallel. */
|
|
1712
|
+
teardown(): Promise<void>;
|
|
1713
|
+
}
|
|
1714
|
+
/**
|
|
1715
|
+
* Build a lineage bound to one client + its probed capabilities. The
|
|
1716
|
+
* capabilities are passed in (not re-probed) so the kernel probes once per run
|
|
1717
|
+
* and the lineage stays a pure function of "what this platform can do".
|
|
1718
|
+
*
|
|
1719
|
+
* @experimental
|
|
1720
|
+
*/
|
|
1721
|
+
declare function createSandboxLineage(client: SandboxClient, capabilities: SandboxCapabilities, options?: {
|
|
1722
|
+
maxConcurrency?: number;
|
|
1723
|
+
streaming?: 'sse' | 'poll';
|
|
1724
|
+
}): SandboxLineage;
|
|
1725
|
+
/**
|
|
1726
|
+
* Loop-side widening of the box's optional checkpoint method. The
|
|
1727
|
+
* `SandboxClient`/`SandboxInstance` surface the kernel relies on does not
|
|
1728
|
+
* require checkpointing; this reads it optionally so the lineage can probe-gate
|
|
1729
|
+
* without importing sandbox-backend specifics. @experimental
|
|
1730
|
+
*/
|
|
1731
|
+
interface CheckpointCapableBox {
|
|
1732
|
+
checkpoint?: (options?: {
|
|
1733
|
+
leaveRunning?: boolean;
|
|
1734
|
+
tags?: string[];
|
|
1735
|
+
}) => Promise<{
|
|
1736
|
+
checkpointId: string;
|
|
1737
|
+
}>;
|
|
1738
|
+
}
|
|
1739
|
+
/** Loop-side widening of the box's optional fork method. @experimental */
|
|
1740
|
+
interface ForkCapableBox {
|
|
1741
|
+
fork?: (checkpointId: string, options?: {
|
|
1742
|
+
name?: string;
|
|
1743
|
+
}) => Promise<SandboxInstance>;
|
|
1744
|
+
}
|
|
1745
|
+
/**
|
|
1746
|
+
* Loop-side widening of the box's optional session accessor. The real
|
|
1747
|
+
* `SandboxInstance` exposes `session(id).status()`; the loop reads it optionally
|
|
1748
|
+
* so `continue` can assert session liveness without requiring it of the test
|
|
1749
|
+
* fakes. `status()` resolves `null` when the id is unknown to the sandbox.
|
|
1750
|
+
* @experimental
|
|
1751
|
+
*/
|
|
1752
|
+
interface SessionCapableBox {
|
|
1753
|
+
session?: (id: string) => {
|
|
1754
|
+
status: () => Promise<unknown | null>;
|
|
1755
|
+
};
|
|
1756
|
+
}
|
|
1757
|
+
|
|
1758
|
+
/**
|
|
1759
|
+
* `openSandboxRun` — the ONE harness-agnostic seam for running an agent in a
|
|
1760
|
+
* sandbox over a persistent artifact: run it, stream it, RESUME the same session
|
|
1761
|
+
* across turns. Domain-agnostic: a coding agent, a research agent, a tax/legal
|
|
1762
|
+
* agent — all flow through this; the domain lives only in the `Deliverable<Out>`
|
|
1763
|
+
* the caller supplies, never in a per-domain copy of this function.
|
|
1764
|
+
*
|
|
1765
|
+
* It is a thin facade (NOT a new layer) over code that already exists and is
|
|
1766
|
+
* already hardened:
|
|
1767
|
+
* - `acquireSandbox` — cold-start / 502-503-504 / gateway-timeout recovery,
|
|
1768
|
+
* - `buildBackendOptions` — the harness IS `backend.type` (opencode / codex /
|
|
1769
|
+
* claude-code / kimi-code / hermes / pi); the only "which agent" knob,
|
|
1770
|
+
* - `createSandboxLineage` — `start` mints a session; `resume` continues the
|
|
1771
|
+
* SAME server-side session with a fail-loud `assertSessionLive`.
|
|
1772
|
+
*
|
|
1773
|
+
* The one genuinely-new piece is {@link Deliverable}: it widens the pure
|
|
1774
|
+
* `OutputAdapter.parse(events)` to ALSO admit a post-turn read off the box FS —
|
|
1775
|
+
* the structural gap that made the bench gates hand-roll `box.fs.read`, because a
|
|
1776
|
+
* large produced file (a git diff, a generated document) truncates in the chat
|
|
1777
|
+
* stream and a pure events-parser cannot reach the workspace. Per the SDK, a
|
|
1778
|
+
* RELATIVE `deliverable.path` resolves from the workspace root and an ABSOLUTE one
|
|
1779
|
+
* (e.g. `/tmp/solution.patch`) reads the container filesystem directly — both are
|
|
1780
|
+
* valid; pick the one the agent actually wrote to. Avoid `..` traversal segments.
|
|
1781
|
+
*
|
|
1782
|
+
* What this deliberately does NOT do (so it stays a facade, not slop): no custom
|
|
1783
|
+
* reconnect/replay (the SDK + platform own per-session buffering + `Last-Event-ID`);
|
|
1784
|
+
* no fork verb (platform CRIU is probe-gated and currently absent — fork lives in
|
|
1785
|
+
* `SandboxLineage.fork` behind the capability probe, surfaced only if it returns).
|
|
1786
|
+
* It is also distinct from `runLoop`: `runLoop` is the multi-round, driver-driven
|
|
1787
|
+
* kernel (fresh box per round, events deliverable); this is a SINGLE rollout +
|
|
1788
|
+
* artifact-or-events deliverable + resume over ONE persistent box.
|
|
1789
|
+
*/
|
|
1790
|
+
|
|
1791
|
+
/**
|
|
1792
|
+
* @experimental
|
|
1793
|
+
* How a typed deliverable `Out` is materialized from a finished turn.
|
|
1794
|
+
* - `events` — pure parse over the event array (identical to `OutputAdapter`).
|
|
1795
|
+
* - `artifact` — read a file off the box AFTER the turn drains, then map it (+ the
|
|
1796
|
+
* events). For diffs/codebases/documents that don't fit the chat
|
|
1797
|
+
* stream. `path` relative ⇒ workspace root; absolute ⇒ container FS.
|
|
1798
|
+
*/
|
|
1799
|
+
type Deliverable<Out> = {
|
|
1800
|
+
kind: 'events';
|
|
1801
|
+
fromEvents: (events: SandboxEvent[]) => Out;
|
|
1802
|
+
} | {
|
|
1803
|
+
kind: 'artifact';
|
|
1804
|
+
path: string;
|
|
1805
|
+
fromArtifact: (raw: string, events: SandboxEvent[]) => Out;
|
|
1806
|
+
};
|
|
1807
|
+
/**
|
|
1808
|
+
* @experimental
|
|
1809
|
+
* One finished turn over the artifact. A failed FS read is surfaced in `readError`
|
|
1810
|
+
* (never masked as an empty deliverable) so a caller distinguishes "agent produced
|
|
1811
|
+
* nothing" from a transport/FS fault.
|
|
1812
|
+
*/
|
|
1813
|
+
interface TurnResult<Out> {
|
|
1814
|
+
out: Out;
|
|
1815
|
+
events: SandboxEvent[];
|
|
1816
|
+
readError?: string;
|
|
1817
|
+
}
|
|
1818
|
+
/** @experimental A live run over ONE persistent artifact (box + session). Close it
|
|
1819
|
+
* when done — `close()` tears the box down. */
|
|
1820
|
+
interface SandboxRun<Out> {
|
|
1821
|
+
readonly box: SandboxInstance;
|
|
1822
|
+
readonly sessionId: string;
|
|
1823
|
+
/** First turn over the fresh box (mints the session). Throws if already started. */
|
|
1824
|
+
start(prompt: string): Promise<TurnResult<Out>>;
|
|
1825
|
+
/** Continue THE SAME session over THE SAME artifact — a resumed turn/rollout. */
|
|
1826
|
+
resume(prompt: string): Promise<TurnResult<Out>>;
|
|
1827
|
+
close(): Promise<void>;
|
|
1828
|
+
}
|
|
1829
|
+
/** @experimental */
|
|
1830
|
+
interface OpenSandboxRunOptions {
|
|
1831
|
+
/** Profile + sandbox env/overrides. `sandboxOverrides.backend.type` is the harness. */
|
|
1832
|
+
agentRun: AgentRunSpec<string>;
|
|
1833
|
+
signal: AbortSignal;
|
|
1834
|
+
/** Optional execution-scoped observers. Hook failures never fail the run. */
|
|
1835
|
+
hooks?: RuntimeHooks;
|
|
1836
|
+
/** Stable run id for trace joins. Defaults to a short runtime-minted id. */
|
|
1837
|
+
runId?: string;
|
|
1838
|
+
/** Optional benchmark/scenario id carried into emitted hook events. */
|
|
1839
|
+
scenarioId?: string;
|
|
1840
|
+
/** Test seam for deterministic hook timestamps. Defaults to `Date.now`. */
|
|
1841
|
+
now?: () => number;
|
|
1842
|
+
/** Bounds box-creation bursts inside lineage fanout. Default from lineage. */
|
|
1843
|
+
maxConcurrency?: number;
|
|
1844
|
+
/** Base backoff (ms) for retrying a transient artifact `fs.read` failure; the i-th
|
|
1845
|
+
* retry waits `readRetryDelayMs * i`. Default 1000. Set 0 to disable the wait (tests). */
|
|
1846
|
+
readRetryDelayMs?: number;
|
|
1847
|
+
}
|
|
1848
|
+
/**
|
|
1849
|
+
* @experimental
|
|
1850
|
+
* Open a sandbox run. Harness-agnostic: the harness lives in
|
|
1851
|
+
* `options.agentRun.sandboxOverrides.backend.type`, so opencode/codex/claude-code/
|
|
1852
|
+
* kimi-code all flow through this one entrypoint with identical env/auth wiring.
|
|
1853
|
+
*/
|
|
1854
|
+
declare function openSandboxRun<Out>(client: SandboxClient, options: OpenSandboxRunOptions, deliverable: Deliverable<Out>): Promise<SandboxRun<Out>>;
|
|
1855
|
+
|
|
1856
|
+
/**
|
|
1857
|
+
* authorStrategy — the agent-authored layer as a package primitive (software-3.0): an
|
|
1858
|
+
* LLM reads a benchmark's per-task LOSSES + the defineStrategy contract and writes a NEW
|
|
1859
|
+
* optimization strategy as code; the caller gates it like any human-built candidate
|
|
1860
|
+
* (runBenchmark + a frozen holdout).
|
|
1861
|
+
*
|
|
1862
|
+
* Structurally safe by construction: the authored body composes shot()/critique() and
|
|
1863
|
+
* spends through the Supervisor's conserved pool — it can be wrong, but it cannot
|
|
1864
|
+
* Goodhart the check (it never sees the verifiers) and it cannot win by overspending.
|
|
1865
|
+
*
|
|
1866
|
+
* The authored module is written to `outDir` and dynamically imported — run under a
|
|
1867
|
+
* TS-capable loader (tsx) since models often emit type annotations.
|
|
1868
|
+
*/
|
|
1869
|
+
|
|
1870
|
+
/** The compressed consumable a skill carries: everything an author needs to emit a loop. */
|
|
1871
|
+
declare const strategyAuthorContract = "\nYou author an OPTIMIZATION STRATEGY for an agentic loop system. A strategy decides how to\nspend a compute budget to beat a task's deployable check. You compose exactly two steps:\n\n shot(spec?: { handle?, messages?, steer?, persona?, tools? }): Promise<ShotResult | null>\n Runs ONE worker attempt (a bounded tool loop) over an artifact.\n - omit handle => the shot opens its OWN fresh artifact and closes it after (a sample).\n - pass handle => the shot CONTINUES that artifact (state accumulates across shots).\n - messages => the carried conversation (pass the previous ShotResult.messages to continue).\n - steer => a corrective instruction injected before the shot.\n - persona => { systemPrompt?, model? } \u2014 give THIS shot its own role and/or model\n (multi-agent strategies: a researcher shot then an engineer shot, a panel of k\n personas over one budget). On a fresh shot the systemPrompt replaces the task's; on\n a carried conversation it arrives as a hand-off message. Same conserved budget.\n - tools => string[] \u2014 restrict THIS shot to a subset of the task's tools by\n name (focus an explore shot on read-only tools, an execute shot on write tools).\n Restriction-only; unknown names make the shot fail. ALWAYS select from\n await listTools(handle) \u2014 never hardcode. Omitted => the shot sees every tool.\n ShotResult = { messages, score (0..1 on the task's check), passes, total, completions, toolErrors }\n Returns null if the attempt failed infra-wise.\n\n critique(messages): Promise<string | null>\n A firewalled trace-analyst reads the attempt's trajectory and returns ONE corrective\n instruction (or null when it judges the work complete). Costs ~1 completion.\n\n consult(messages, instruction): Promise<string | null>\n The RAW analyst channel: the same firewalled critic answers YOUR instruction over the\n trajectory verbatim (no reformatting) \u2014 use it when you need a specific reply format\n (a decision, a prediction). Costs ~1 completion.\n\n surface.open(task) / surface.close(handle)\n Open a persistent artifact you manage yourself (remember to close in a finally).\n close is idempotent \u2014 closing an already-closed handle is a safe no-op.\n\n listTools(handle): Promise<Array<{ name, description? }>>\n The tools THIS task actually offers. TOOL SETS VARY PER TASK \u2014 if you restrict a\n shot with `tools`, you MUST pick names from await listTools(handle); hardcoding\n names from an example kills your shots on every task whose tools differ.\n\nRules:\n- ALWAYS await every shot/critique/surface call \u2014 a floating promise that rejects\n crashes the whole benchmark run.\n- Stay within ~budget total shots; every shot/critique spends from a conserved pool.\n- For a FRESH attempt OMIT `messages` entirely (never pass `[]` \u2014 an empty array is a\n fresh conversation too, but be explicit). To CONTINUE, pass the previous\n ShotResult.messages unchanged.\n- Return { score, resolved, completions, progression, shots } \u2014 score = the BEST checkpoint\n you reached (keep-best, never final-state), progression = score after each shot.\n- The module must be EXACTLY this shape (no other imports, no commentary outside code):\n\nimport { defineStrategy } from '@tangle-network/agent-runtime/loops'\nexport default defineStrategy('your-strategy-name', async ({ surface, task, budget, shot, critique, listTools }) => {\n // your composition (listTools comes from the destructured context \u2014 it is NOT a global)\n})\n";
|
|
1872
|
+
interface AuthorStrategyOptions {
|
|
1873
|
+
/** The model-call seam (agent-eval `createChatClient`). */
|
|
1874
|
+
chat: ChatClient;
|
|
1875
|
+
model?: string;
|
|
1876
|
+
/** A NAMED fallback author tried once when the primary call fails or returns no code
|
|
1877
|
+
* block (thinking models time out at the edge on long authoring prompts, or return
|
|
1878
|
+
* empty content without `maxTokens`). Opt-in — absent means the primary's failure
|
|
1879
|
+
* propagates. */
|
|
1880
|
+
fallbackModel?: string;
|
|
1881
|
+
/** The contract text shown to the author. Default `strategyAuthorContract`. The
|
|
1882
|
+
* meta-optimization coordinate: a GEPA/skill loop can evolve this text and gate each
|
|
1883
|
+
* variant on the same frozen holdout as any strategy. */
|
|
1884
|
+
contract?: string;
|
|
1885
|
+
/** The environment the losses came from (orientation only — never the verifiers). */
|
|
1886
|
+
environmentName: string;
|
|
1887
|
+
/** The per-task losses table (e.g. JSON.stringify(report.perTask)) — the gradient. */
|
|
1888
|
+
lossesJson: string;
|
|
1889
|
+
/** The budget the strategy must respect (shots/width). */
|
|
1890
|
+
budget: number;
|
|
1891
|
+
/** Where the authored module file is written (created if missing). */
|
|
1892
|
+
outDir: string;
|
|
1893
|
+
temperature?: number;
|
|
1894
|
+
/** Completion cap — required by thinking-model authors that stream reasoning first. */
|
|
1895
|
+
maxTokens?: number;
|
|
1896
|
+
signal?: AbortSignal;
|
|
1897
|
+
}
|
|
1898
|
+
/** Static CONTRACT lint over an authored strategy module — the module-boundary
|
|
1899
|
+
* enforcement of the harness's two measurement invariants:
|
|
1900
|
+
* - author blindness: the only import allowed is the loops surface. A body that could
|
|
1901
|
+
* reach the filesystem, network, or process could read or mutate verifier/artifact
|
|
1902
|
+
* state outside the brokered shots, and the harness-verified score would stop
|
|
1903
|
+
* meaning "what the shots achieved".
|
|
1904
|
+
* - conserved dose: no out-of-band compute (fetch/require/eval) — every unit a
|
|
1905
|
+
* strategy spends is metered by the Supervisor's pool, which is what makes
|
|
1906
|
+
* equal-budget comparisons between strategies valid.
|
|
1907
|
+
* A lint, not a sandbox: its job is keeping the benchmark numbers interpretable. */
|
|
1908
|
+
declare function assertStrategyContract(code: string): void;
|
|
1909
|
+
interface AuthoredStrategy {
|
|
1910
|
+
strategy: Strategy;
|
|
1911
|
+
file: string;
|
|
1912
|
+
code: string;
|
|
1913
|
+
}
|
|
1914
|
+
/** Author + load a strategy from losses. Throws when the author emits no loadable module;
|
|
1915
|
+
* with `fallbackModel` set, the named fallback gets one attempt first. */
|
|
1916
|
+
declare function authorStrategy(opts: AuthorStrategyOptions): Promise<AuthoredStrategy>;
|
|
1917
|
+
|
|
1918
|
+
/**
|
|
1919
|
+
* runStrategyEvolution — the multi-generation strategy search: per generation the system
|
|
1920
|
+
* authors a POPULATION of candidate strategies from the current tournament's losses,
|
|
1921
|
+
* plays them against the incumbent at equal budget, and advances a champion; one final
|
|
1922
|
+
* promotion decision runs on a NEVER-BEFORE-USED holdout slice through `promotionGate`.
|
|
1923
|
+
*
|
|
1924
|
+
* Measurement invariants (the reasons this design is shaped the way it is):
|
|
1925
|
+
* - The author sees TRAIN losses only. The holdout slice is drawn fresh (disjoint task
|
|
1926
|
+
* offsets) after all authoring is done — one promotion decision, one untouched slice,
|
|
1927
|
+
* so adaptive reuse of evaluation data never enters the verdict.
|
|
1928
|
+
* - Every tournament runs at the same per-strategy budget through the conserved pool;
|
|
1929
|
+
* candidates cannot win by overspending.
|
|
1930
|
+
* - Champion selection within the search is a SEARCH policy (configurable, default
|
|
1931
|
+
* cost-aware: ties on score go to the cheapest strategy — a scalar hides a strategy
|
|
1932
|
+
* that ties at half the cost). The promotion verdict never comes from search
|
|
1933
|
+
* selection; it comes from the gate on the fresh slice.
|
|
1934
|
+
* - Every authored artifact's description length (gzip bits) is recorded, so the
|
|
1935
|
+
* artifact-complexity-vs-holdout-gap relation is analyzable from any run's report.
|
|
1936
|
+
*
|
|
1937
|
+
* Lineage fields (`parent`, `generation`) are recorded on every archive node so a
|
|
1938
|
+
* descendant-productivity parent-selection policy can be added without changing the
|
|
1939
|
+
* report schema; the v1 search authors from the latest tournament's losses.
|
|
1940
|
+
*/
|
|
1941
|
+
|
|
1942
|
+
interface EvolutionAuthor {
|
|
1943
|
+
/** The model-call seam (agent-eval `createChatClient`). */
|
|
1944
|
+
chat: ChatClient;
|
|
1945
|
+
model?: string;
|
|
1946
|
+
fallbackModel?: string;
|
|
1947
|
+
temperature?: number;
|
|
1948
|
+
maxTokens?: number;
|
|
1949
|
+
}
|
|
1950
|
+
type ChampionPolicy = 'score' | 'costAware';
|
|
1951
|
+
interface StrategyEvolutionConfig {
|
|
1952
|
+
environment: Environment;
|
|
1953
|
+
/** Task supply by DISJOINT slice: `(offset, n)` must return n tasks unique to that
|
|
1954
|
+
* offset range. Train draws [0, trainN); the holdout draws [trainN + holdoutOffset,
|
|
1955
|
+
* …) — tasks the search never touched. */
|
|
1956
|
+
tasks: (offset: number, n: number) => Promise<AgenticTask[]>;
|
|
1957
|
+
trainN: number;
|
|
1958
|
+
holdoutN: number;
|
|
1959
|
+
/** Extra offset past the train slice for the holdout draw (rotate across runs). */
|
|
1960
|
+
holdoutOffset?: number;
|
|
1961
|
+
worker: AgenticOptions;
|
|
1962
|
+
author: EvolutionAuthor;
|
|
1963
|
+
/** Rollouts (sample) / shots (refine) per strategy per task. Default 3. */
|
|
1964
|
+
budget?: number;
|
|
1965
|
+
concurrency?: number;
|
|
1966
|
+
/** Author→tournament rounds after gen0. Default 2. */
|
|
1967
|
+
generations?: number;
|
|
1968
|
+
/** Authored candidates per generation. Default 2. */
|
|
1969
|
+
populationSize?: number;
|
|
1970
|
+
/** The gen0 field. Default [sample, refine, sampleThenRefine]. */
|
|
1971
|
+
baselines?: Strategy[];
|
|
1972
|
+
/** What "better" means for PROMOTION. 'score' (default): the candidate must beat the
|
|
1973
|
+
* incumbent's score (superiority gate). 'cost': the candidate must prove score
|
|
1974
|
+
* NON-INFERIORITY (not worse by more than `scoreTolerance`) plus significant cost
|
|
1975
|
+
* savings — the "same quality, cheaper" objective. The author is told the objective
|
|
1976
|
+
* and sees per-task spend either way. */
|
|
1977
|
+
objective?: 'score' | 'cost';
|
|
1978
|
+
/** Cost objective: the score CI lower bound must clear −scoreTolerance. Default 0.05. */
|
|
1979
|
+
scoreTolerance?: number;
|
|
1980
|
+
/** Search-side champion selection. Default 'costAware'. */
|
|
1981
|
+
champion?: ChampionPolicy;
|
|
1982
|
+
/** Score band treated as a tie under 'costAware'. Default 0.01. */
|
|
1983
|
+
championEpsilon?: number;
|
|
1984
|
+
/** Where authored modules are written. */
|
|
1985
|
+
outDir: string;
|
|
1986
|
+
/** Promotion-gate evidence floor (paired holdout tasks). */
|
|
1987
|
+
minPairedTasks?: number;
|
|
1988
|
+
/** BAND-AWARE scoring — concentrate the measurement where lift is possible.
|
|
1989
|
+
* Holdout: draw `holdoutPoolN` candidate tasks and run `baselines[0]` once at the run
|
|
1990
|
+
* budget as an INDEPENDENT reference screen; keep tasks scoring ≤ `maxRefScore`
|
|
1991
|
+
* (headroom exists) and take the first `holdoutN`. Band membership is decided before
|
|
1992
|
+
* either finalist touches a task and both finalists then face the SAME tasks — the
|
|
1993
|
+
* estimand becomes "paired lift on headroom tasks", pre-registered by this config.
|
|
1994
|
+
* Train: champion selection ignores zero-spread tasks (every field strategy scored
|
|
1995
|
+
* identically — zero selection information, pure noise dilution). */
|
|
1996
|
+
band?: {
|
|
1997
|
+
holdoutPoolN: number;
|
|
1998
|
+
/** Keep holdout tasks where the reference scores ≤ this. Default 0.99 — drop only
|
|
1999
|
+
* tasks the reference already solves fully (no headroom, a candidate can only tie). */
|
|
2000
|
+
maxRefScore?: number;
|
|
2001
|
+
};
|
|
2002
|
+
/** What the author learns from a tournament. 'exact' (default) = scores + progressions
|
|
2003
|
+
* per task; 'binary' = pass/fail only — the leakage-bounded channel (one bit per cell
|
|
2004
|
+
* per generation reaches the author from the evaluation data). */
|
|
2005
|
+
lossesDetail?: 'exact' | 'binary';
|
|
2006
|
+
/** Reproducer certification (arXiv:2606.11045): when the final champion is AUTHORED,
|
|
2007
|
+
* compress it to a short natural-language summary, have a fresh author re-implement
|
|
2008
|
+
* from the summary alone (no losses, no code), and score the reproduction on the same
|
|
2009
|
+
* holdout. A reproduction gap is an overfitting signal (their detector: 100%
|
|
2010
|
+
* sensitivity / 91% specificity in the ML-agent setting) — recorded on the report,
|
|
2011
|
+
* never gate-blocking in v1. */
|
|
2012
|
+
reproducerCheck?: {
|
|
2013
|
+
/** Word budget for the strategy summary. Default 64. */
|
|
2014
|
+
summaryMaxWords?: number;
|
|
2015
|
+
/** Reproduction counts as faithful when reproducedScore ≥ championScore − tolerance.
|
|
2016
|
+
* Default 0.05. */
|
|
2017
|
+
tolerance?: number;
|
|
2018
|
+
};
|
|
2019
|
+
/** Endurance: write the run state after every completed phase; with `resume`, a
|
|
2020
|
+
* restart skips completed phases (authored modules re-imported from their files).
|
|
2021
|
+
* Worst case after a mid-run death is re-paying ONE phase, never the run. */
|
|
2022
|
+
checkpoint?: {
|
|
2023
|
+
path: string;
|
|
2024
|
+
resume?: boolean;
|
|
2025
|
+
};
|
|
2026
|
+
/** Called before each benchmark phase (gen0, gen1…, band-screen, holdout, reproduce).
|
|
2027
|
+
* The seam for environment recycling — no artifacts span phases, so a runner may
|
|
2028
|
+
* recreate a wedge-prone environment container here. */
|
|
2029
|
+
onPhase?: (phase: string) => Promise<void>;
|
|
2030
|
+
onTask?: (phase: string, row: BenchmarkTaskRow, done: number, total: number) => void;
|
|
2031
|
+
hooks?: RuntimeHooks;
|
|
2032
|
+
}
|
|
2033
|
+
interface ChampionPick {
|
|
2034
|
+
name: string;
|
|
2035
|
+
score: number;
|
|
2036
|
+
usd: number;
|
|
2037
|
+
}
|
|
2038
|
+
interface EvolutionCandidate {
|
|
2039
|
+
name: string;
|
|
2040
|
+
file?: string;
|
|
2041
|
+
gzipBits?: number;
|
|
2042
|
+
codeChars?: number;
|
|
2043
|
+
/** Present when this author attempt failed (recorded, never silent). */
|
|
2044
|
+
error?: string;
|
|
2045
|
+
}
|
|
2046
|
+
interface EvolutionGeneration {
|
|
2047
|
+
generation: number;
|
|
2048
|
+
candidates: EvolutionCandidate[];
|
|
2049
|
+
report: BenchmarkReport;
|
|
2050
|
+
champion: ChampionPick;
|
|
2051
|
+
}
|
|
2052
|
+
interface EvolutionArchiveNode {
|
|
2053
|
+
name: string;
|
|
2054
|
+
source: 'baseline' | 'authored';
|
|
2055
|
+
generation: number;
|
|
2056
|
+
/** The champion whose tournament losses this candidate was authored from. */
|
|
2057
|
+
parent?: string;
|
|
2058
|
+
gzipBits?: number;
|
|
2059
|
+
file?: string;
|
|
2060
|
+
/** Latest measured tournament result — 0 until the node's first tournament settles
|
|
2061
|
+
* (an authored node is created before its generation's benchmark runs). */
|
|
2062
|
+
score: number;
|
|
2063
|
+
usd: number;
|
|
2064
|
+
}
|
|
2065
|
+
interface ReproductionCheck {
|
|
2066
|
+
/** The compressed strategy description the reproducer implemented from. */
|
|
2067
|
+
summary: string;
|
|
2068
|
+
reproducedName: string;
|
|
2069
|
+
file?: string;
|
|
2070
|
+
championHoldoutScore: number;
|
|
2071
|
+
reproducedHoldoutScore: number;
|
|
2072
|
+
/** champion − reproduced (positive = the reproduction fell short). */
|
|
2073
|
+
gap: number;
|
|
2074
|
+
/** reproducedScore ≥ championScore − tolerance. A failed reproduction is an
|
|
2075
|
+
* overfitting signal: the champion's win did not fit through the summary. */
|
|
2076
|
+
reproducible: boolean;
|
|
2077
|
+
/** Infra failure during reproduction (distinct from a semantic reproduction failure). */
|
|
2078
|
+
error?: string;
|
|
2079
|
+
}
|
|
2080
|
+
interface EvolutionBandInfo {
|
|
2081
|
+
/** Tasks screened by the reference on the holdout pool. */
|
|
2082
|
+
screened: number;
|
|
2083
|
+
/** Tasks kept (reference score ≤ maxRefScore) before truncating to holdoutN. */
|
|
2084
|
+
inBand: number;
|
|
2085
|
+
/** Reference scores per screened task (the screening record). */
|
|
2086
|
+
refScores: Array<{
|
|
2087
|
+
taskId: string;
|
|
2088
|
+
score: number;
|
|
2089
|
+
}>;
|
|
2090
|
+
}
|
|
2091
|
+
interface EvolutionReport {
|
|
2092
|
+
gen0: BenchmarkReport;
|
|
2093
|
+
gen0Champion: ChampionPick;
|
|
2094
|
+
generations: EvolutionGeneration[];
|
|
2095
|
+
archive: EvolutionArchiveNode[];
|
|
2096
|
+
finalChampion: ChampionPick;
|
|
2097
|
+
holdout: BenchmarkReport;
|
|
2098
|
+
verdict: PromotionVerdict;
|
|
2099
|
+
/** Present when band screening ran — the verdict's estimand is then "paired lift on
|
|
2100
|
+
* headroom tasks" (band membership fixed by the reference screen, pre-registered). */
|
|
2101
|
+
band?: EvolutionBandInfo;
|
|
2102
|
+
/** Present when reproducerCheck ran (final champion was authored). */
|
|
2103
|
+
reproduction?: ReproductionCheck;
|
|
2104
|
+
/** SEARCH TELEMETRY, not evidence: each entry is that generation's own train-slice
|
|
2105
|
+
* re-measurement, so cross-generation deltas mix true drift with run-to-run variance
|
|
2106
|
+
* (entries are unpaired across generations). The only evidence-grade comparison in
|
|
2107
|
+
* this report is `verdict` — both finalists measured fresh, paired, on the holdout. */
|
|
2108
|
+
trajectory: Array<{
|
|
2109
|
+
generation: number;
|
|
2110
|
+
champion: string;
|
|
2111
|
+
score: number;
|
|
2112
|
+
usd: number;
|
|
2113
|
+
}>;
|
|
2114
|
+
}
|
|
2115
|
+
/** Strategy means recomputed over the DISCRIMINATING tasks only — tasks where the field
|
|
2116
|
+
* strategies did not all score identically. Zero-spread tasks (everyone 1.0, everyone
|
|
2117
|
+
* 0.0, everyone tied) carry no selection information; averaging over them dilutes real
|
|
2118
|
+
* differences toward zero. Search-side denoising only — the gate never uses this. */
|
|
2119
|
+
declare function discriminatingMeans(report: BenchmarkReport, fieldOrder: string[]): Record<string, {
|
|
2120
|
+
score: number;
|
|
2121
|
+
usd: number;
|
|
2122
|
+
}> | null;
|
|
2123
|
+
/** The champion pick over a means table. 'score' takes the best mean score (ties →
|
|
2124
|
+
* field order). 'costAware' treats scores within `epsilon` of the best as tied and
|
|
2125
|
+
* takes the cheapest — the (score, $) Pareto rule collapsed to one pick. */
|
|
2126
|
+
declare function pickChampion(means: Record<string, {
|
|
2127
|
+
score: number;
|
|
2128
|
+
usd: number;
|
|
2129
|
+
}>, fieldOrder: string[], policy: ChampionPolicy, epsilon: number): ChampionPick;
|
|
2130
|
+
/** Search-side champion selection over a tournament report. */
|
|
2131
|
+
declare function selectChampion(report: BenchmarkReport, fieldOrder: string[], policy: ChampionPolicy, epsilon: number): ChampionPick;
|
|
2132
|
+
declare function runStrategyEvolution(cfg: StrategyEvolutionConfig): Promise<EvolutionReport>;
|
|
2133
|
+
|
|
2134
|
+
/**
|
|
2135
|
+
* @experimental
|
|
2136
|
+
*
|
|
2137
|
+
* The supervisor's intelligence is AUTHORING the agents it spawns — not pressing buttons.
|
|
2138
|
+
*
|
|
2139
|
+
* Every agent here is three things: instructions (system prompt), tools, and a model — its
|
|
2140
|
+
* `AgentProfile`. The supervisor's job is to WRITE those profiles: read the task, decompose it,
|
|
2141
|
+
* and for each sub-task author a tailored worker recipe. `supervisorInstructions` is the how-to the
|
|
2142
|
+
* supervisor reads (its system prompt); `authoredWorker` builds a worker AGENT from a profile the
|
|
2143
|
+
* supervisor authored — the authored systemPrompt + model shape the worker's call.
|
|
2144
|
+
*
|
|
2145
|
+
* The skill is the single OPTIMIZABLE surface: edit it → the supervisor designs better agents.
|
|
2146
|
+
* That is the self-improvement lever (the prompt/skill lever), not the execution plumbing.
|
|
2147
|
+
*/
|
|
2148
|
+
|
|
2149
|
+
/** What the supervisor AUTHORS per sub-task — a worker recipe (a partial `AgentProfile`). */
|
|
2150
|
+
interface AuthoredProfile {
|
|
2151
|
+
name: string;
|
|
2152
|
+
/** The rich, task-specific instructions the supervisor wrote for THIS worker. */
|
|
2153
|
+
systemPrompt: string;
|
|
2154
|
+
/** The model the supervisor chose for this sub-task (falls back to the run default). */
|
|
2155
|
+
model?: string;
|
|
2156
|
+
}
|
|
2157
|
+
/** Narrow an untyped `spawn_agent` profile argument to an `AuthoredProfile`, or null if the
|
|
2158
|
+
* supervisor failed to author one (empty/placeholder profile — a skill violation worth catching). */
|
|
2159
|
+
declare function asAuthoredProfile(raw: unknown): AuthoredProfile | null;
|
|
2160
|
+
/** The supervisor SKILL — the how-to the supervisor reads (its system prompt). THE optimizable
|
|
2161
|
+
* surface: editing this changes how the supervisor designs every agent it spawns. */
|
|
2162
|
+
declare function supervisorInstructions(opts?: {
|
|
2163
|
+
goal?: string;
|
|
2164
|
+
}): string;
|
|
2165
|
+
/** Build a worker AGENT from a profile the supervisor authored: the authored `systemPrompt` +
|
|
2166
|
+
* `model` shape the worker's one model call; the deliverable gates settlement (valid ⟺ delivered). */
|
|
2167
|
+
declare function authoredWorker(profile: AuthoredProfile, opts: {
|
|
2168
|
+
cfg: RouterConfig;
|
|
2169
|
+
taskPrompt: string;
|
|
2170
|
+
deliverable: DeliverableSpec;
|
|
2171
|
+
temperature?: number;
|
|
2172
|
+
}): Agent<unknown, unknown>;
|
|
2173
|
+
/** Thresholds below which a system prompt is treated as a thin stub. Tunable per call. */
|
|
2174
|
+
interface ProfileRichnessThresholds {
|
|
2175
|
+
/** A prompt shorter than this many characters is thin (default 600). */
|
|
2176
|
+
readonly minSystemPromptChars: number;
|
|
2177
|
+
/** A prompt with fewer than this many non-blank lines is thin (default 6). */
|
|
2178
|
+
readonly minSystemPromptLines: number;
|
|
2179
|
+
}
|
|
2180
|
+
declare const defaultProfileRichnessThresholds: ProfileRichnessThresholds;
|
|
2181
|
+
/** Per-field verdict on one authored profile — the raw material the bench renders + scores. */
|
|
2182
|
+
interface ProfileRichness {
|
|
2183
|
+
readonly name: string;
|
|
2184
|
+
/** The resolved system prompt (canonical `prompt.systemPrompt`, the sandbox `prompt.system`
|
|
2185
|
+
* convention, or a bare-string prompt — whichever the author used). */
|
|
2186
|
+
readonly systemPrompt: string;
|
|
2187
|
+
readonly systemPromptChars: number;
|
|
2188
|
+
readonly systemPromptLines: number;
|
|
2189
|
+
readonly sentenceCount: number;
|
|
2190
|
+
readonly hasDescription: boolean;
|
|
2191
|
+
readonly hasTools: boolean;
|
|
2192
|
+
readonly hasSkills: boolean;
|
|
2193
|
+
readonly hasMcp: boolean;
|
|
2194
|
+
readonly hasSubagents: boolean;
|
|
2195
|
+
/** 0..1 — fraction of richness signals present (prompt-depth + the four levers). */
|
|
2196
|
+
readonly richness: number;
|
|
2197
|
+
/** True when the supervisor authored a stub instead of a real profile. */
|
|
2198
|
+
readonly thin: boolean;
|
|
2199
|
+
/** The specific reasons it is thin (empty when rich) — used in the finding's action. */
|
|
2200
|
+
readonly reasons: string[];
|
|
2201
|
+
}
|
|
2202
|
+
/** OBSERVE one authored `AgentProfile` and score its richness (no judge verdict is read). The task
|
|
2203
|
+
* context (`needsMcp`) lets a domain say "this work needs a data/tool MCP" so a missing MCP counts. */
|
|
2204
|
+
declare function assessAuthoredProfile(profile: AgentProfile$1, opts?: {
|
|
2205
|
+
needsMcp?: boolean;
|
|
2206
|
+
thresholds?: Partial<ProfileRichnessThresholds>;
|
|
2207
|
+
}): ProfileRichness;
|
|
2208
|
+
/** Turn a {@link ProfileRichness} verdict into a bus-routable `AnalystFinding` (area `profile-quality`).
|
|
2209
|
+
* Severity scales with thinness; the recommended action names the MISSING lever so the supervisor can
|
|
2210
|
+
* re-author. `subject` = the worker name so per-worker findings diff cleanly across re-authors. */
|
|
2211
|
+
declare function profileRichnessFinding(richness: ProfileRichness, opts?: {
|
|
2212
|
+
analystId?: string;
|
|
2213
|
+
runId?: string;
|
|
2214
|
+
}): AnalystFinding;
|
|
2215
|
+
|
|
2216
|
+
/**
|
|
2217
|
+
* @experimental
|
|
2218
|
+
*
|
|
2219
|
+
* The conserved budget reservation pool — the invariant the whole instrument
|
|
2220
|
+
* rests on (critique M5/B3). One root `Budget` becomes a conserved pool of three
|
|
2221
|
+
* quantities (tokens, usd, iterations) plus an absolute deadline. Children reserve
|
|
2222
|
+
* atomically at spawn and reconcile at settle:
|
|
2223
|
+
*
|
|
2224
|
+
* total ≡ free + reserved + committed (invariant, always)
|
|
2225
|
+
*
|
|
2226
|
+
* `reserve` moves a child's whole ceiling from `free` → `reserved` and fails closed
|
|
2227
|
+
* when `free` can't cover it (never read-then-spawn overcommit, so `Σk(treatment) ≡
|
|
2228
|
+
* Σk(blind)` by construction). `reconcile` releases the reservation, commits ACTUAL
|
|
2229
|
+
* spend, and refunds the unspent remainder to `free`. Tokens and usd are separate
|
|
2230
|
+
* channels (`LoopTokenUsage` has no `usd`); iterations are conserved alongside them.
|
|
2231
|
+
*
|
|
2232
|
+
* Pure and deterministic: `now()` is injected, there is no I/O, and no wall-clock or
|
|
2233
|
+
* RNG read. A `reserve`/`reconcile` ticket is single-use (fail-loud on double or
|
|
2234
|
+
* unknown reconcile) so a child can never refund twice.
|
|
2235
|
+
*/
|
|
2236
|
+
|
|
2237
|
+
/** Opaque, single-use reservation handle returned by `reserve` and consumed by
|
|
2238
|
+
* `reconcile`. Carries the reserved ceilings so reconciliation needs no lookup. */
|
|
2239
|
+
interface ReservationTicket {
|
|
2240
|
+
readonly id: number;
|
|
2241
|
+
readonly reserved: {
|
|
2242
|
+
readonly tokens: number;
|
|
2243
|
+
readonly usd: number;
|
|
2244
|
+
readonly iterations: number;
|
|
2245
|
+
};
|
|
2246
|
+
}
|
|
2247
|
+
/** Post-reservation pool readout — the shape `Scope.budget` exposes. `tokensLeft`,
|
|
2248
|
+
* `usdLeft`, and `reservedTokens` reflect committed-but-unsettled reservations;
|
|
2249
|
+
* `deadlineMs` is the ABSOLUTE wall-clock deadline (0 when the root set none).
|
|
2250
|
+
* `usdCapped` distinguishes a real `usdLeft <= 0` exhaustion from an uncapped pool (which always
|
|
2251
|
+
* reads `usdLeft: 0`) — the in-loop guard needs it to bound a usd-capped driver. */
|
|
2252
|
+
type BudgetReadout = Readonly<{
|
|
2253
|
+
tokensLeft: number;
|
|
2254
|
+
usdLeft: number;
|
|
2255
|
+
usdCapped: boolean;
|
|
2256
|
+
deadlineMs: number;
|
|
2257
|
+
reservedTokens: number;
|
|
2258
|
+
}>;
|
|
2259
|
+
interface BudgetPool {
|
|
2260
|
+
/**
|
|
2261
|
+
* Atomically reserve a child's full ceiling from the free balance. Fails closed
|
|
2262
|
+
* ({ ok: false }) when the pool can't cover tokens, usd, or iterations — the
|
|
2263
|
+
* caller inspects `ok` before `ticket`.
|
|
2264
|
+
*/
|
|
2265
|
+
reserve(b: Budget): {
|
|
2266
|
+
ok: true;
|
|
2267
|
+
ticket: ReservationTicket;
|
|
2268
|
+
} | {
|
|
2269
|
+
ok: false;
|
|
2270
|
+
reason: 'budget-exhausted';
|
|
2271
|
+
};
|
|
2272
|
+
/**
|
|
2273
|
+
* Release a reservation: commit the actual `spent`, refund the unspent remainder
|
|
2274
|
+
* to the free pool. Throws on an unknown or already-reconciled ticket (fail loud —
|
|
2275
|
+
* a double refund would silently break conservation).
|
|
2276
|
+
*/
|
|
2277
|
+
reconcile(ticket: ReservationTicket, spent: Spend): void;
|
|
2278
|
+
/** Fold a normalized `UsageEvent` stream (or array) into a `Spend`. Tokens via
|
|
2279
|
+
* `addTokenUsage`, usd on its own channel, iterations from `'iteration'` events.
|
|
2280
|
+
* `ms` is left zero — wall-clock duration is the caller's to record, not the pool's. */
|
|
2281
|
+
spendFrom(events: AsyncIterable<UsageEvent> | UsageEvent[]): Promise<Spend>;
|
|
2282
|
+
/** The current readout, reflecting all outstanding reservations. */
|
|
2283
|
+
readout(): BudgetReadout;
|
|
2284
|
+
/**
|
|
2285
|
+
* Record OBSERVED spend that did NOT go through reserve/reconcile — the driver's OWN inference
|
|
2286
|
+
* (its chat turns), which is real compute but not a spawned child. A direct `free → committed`
|
|
2287
|
+
* debit, so `total ≡ free + reserved + committed` is preserved: equal-k counts the driver's
|
|
2288
|
+
* tokens and the in-loop budget guard (`readout().tokensLeft`) sees them. `free` may go negative
|
|
2289
|
+
* when a run overspends — that is honest (the readout then signals exhaustion). It never throws:
|
|
2290
|
+
* the spend already happened, so accounting records reality; the in-loop guard prevents MORE.
|
|
2291
|
+
* The DURABLE record is the journal's `metered` event (written by `Scope.meter`); this debit
|
|
2292
|
+
* only makes the live `readout()` reflect driver inference for the in-loop guard.
|
|
2293
|
+
*/
|
|
2294
|
+
observe(spend: Spend): void;
|
|
2295
|
+
/** Fail loud if any reservation is still open — the conserved-pool leak detector. Called at the
|
|
2296
|
+
* supervisor's join barrier: once every child has settled, no ticket may remain (a leaked
|
|
2297
|
+
* reservation would silently break `total ≡ free + reserved + committed`). */
|
|
2298
|
+
assertNoOpenTickets(): void;
|
|
2299
|
+
}
|
|
2300
|
+
/** Fold a normalized `UsageEvent` array into a `Spend`. Tokens and usd are separate
|
|
2301
|
+
* channels; iterations come from `'iteration'` events. Pure; `ms` stays zero (the
|
|
2302
|
+
* pool does not read wall-clock). */
|
|
2303
|
+
declare function spendFromUsageEvents(events: UsageEvent[]): Spend;
|
|
2304
|
+
/**
|
|
2305
|
+
* Create a conserved reservation pool from a root `Budget`. `now()` is injected so the
|
|
2306
|
+
* deadline readout is deterministic; defaults to `Date.now` for non-test callers. The
|
|
2307
|
+
* absolute deadline is fixed at construction (`now() + budget.deadlineMs`) so the
|
|
2308
|
+
* readout's `deadlineMs` is a stable wall-clock instant, not a shrinking remainder.
|
|
2309
|
+
*/
|
|
2310
|
+
declare function createBudgetPool(root: Budget, now?: () => number): BudgetPool;
|
|
2311
|
+
|
|
2312
|
+
/**
|
|
2313
|
+
* @experimental
|
|
2314
|
+
*
|
|
2315
|
+
* `driverAgent` — the driver's BRAIN.
|
|
2316
|
+
*
|
|
2317
|
+
* The recursive driver-executor (`driver-executor.ts`) runs a driver `Agent.act` inside a
|
|
2318
|
+
* nested `Scope`; this is the intelligent `act`: it mounts the coordination MCP verbs
|
|
2319
|
+
* (`createCoordinationTools`) over that scope and runs an LLM tool-loop, so the driver
|
|
2320
|
+
* REASONS — spawn / observe / steer / await / stop — about how to drive its children,
|
|
2321
|
+
* instead of running a fixed script. Each turn: ask the driver LLM for tool calls, run them
|
|
2322
|
+
* against the live scope, fold the results back, repeat until the driver stops (no tool
|
|
2323
|
+
* calls) or the turn cap forces a keep-best finalize.
|
|
2324
|
+
*
|
|
2325
|
+
* Recursion composes through `makeWorkerAgent`: `spawn_agent` resolves a `profile` to a
|
|
2326
|
+
* worker LEAF or — when the profile is a driver — a `driverChild` wrapping ANOTHER
|
|
2327
|
+
* `driverAgent` over its own nested scope (see `driver-executor.ts`). So an agent
|
|
2328
|
+
* drives an agent that drives an agent, each an LLM tool-loop, all on one conserved-budget
|
|
2329
|
+
* tree.
|
|
2330
|
+
*
|
|
2331
|
+
* Two seams are INJECTED so the loop runs offline with no creds and stays decoupled:
|
|
2332
|
+
* - `brain` (`ToolLoopChat`) — one driver-LLM turn over the canonical tool-loop seam; a test
|
|
2333
|
+
* drives a scripted mock, production passes the router's tool-calling (`routerBrain`), a
|
|
2334
|
+
* sandboxed harness drives the verbs as MCP tools. The same seam every tool-loop uses.
|
|
2335
|
+
* - `systemPrompt` — the driver's stance (the agent-eval worker-driver prompt / the prompt
|
|
2336
|
+
* generator). Injected, never hardcoded — the prompt is a pluggable role.
|
|
2337
|
+
*/
|
|
2338
|
+
|
|
2339
|
+
interface DriverAgentOptions {
|
|
2340
|
+
readonly name: string;
|
|
2341
|
+
/** The driver-LLM seam — ONE inference turn over the conversation + the coordination tool specs
|
|
2342
|
+
* (the canonical `ToolLoopChat`): a scripted mock offline, the router's tool-calling in
|
|
2343
|
+
* production, or a sandboxed harness. The same seam every tool-loop uses; no bespoke shape. */
|
|
2344
|
+
readonly brain: ToolLoopChat;
|
|
2345
|
+
/** Shared blob store — `observe_agent` reads settled outputs through it. */
|
|
2346
|
+
readonly blobs: ResultBlobStore;
|
|
2347
|
+
/** Resolve a spawned `profile` to a worker LEAF or a driver child (the recursion seam). */
|
|
2348
|
+
readonly makeWorkerAgent: MakeWorkerAgent;
|
|
2349
|
+
/** Per-child budget reserved from the conserved pool on each spawn. */
|
|
2350
|
+
readonly perWorker: Budget;
|
|
2351
|
+
/** The driver's stance — a string, or built from the task (the worker-driver prompt /
|
|
2352
|
+
* the generator). INJECTED so the prompt is a pluggable, optimizable role. */
|
|
2353
|
+
readonly systemPrompt: string | ((task: unknown) => string);
|
|
2354
|
+
/** WORK tools the driver may call DIRECTLY (alongside the coordination verbs) — so the driver is
|
|
2355
|
+
* not a pure manager but a full agent that can ACT (do simple work itself) OR SPAWN (delegate).
|
|
2356
|
+
* Each is a router tool spec; their names must not collide with the coordination verbs. Pair with
|
|
2357
|
+
* `executeExtraTool`. Unset → coordination-only (the prior behavior). */
|
|
2358
|
+
readonly extraTools?: ReadonlyArray<{
|
|
2359
|
+
readonly name: string;
|
|
2360
|
+
readonly description?: string;
|
|
2361
|
+
readonly parameters: Record<string, unknown>;
|
|
2362
|
+
}>;
|
|
2363
|
+
/** Runs an `extraTools` call. Returns a string result, or null/undefined to signal "not handled"
|
|
2364
|
+
* so the call falls through to the coordination dispatch. Required iff `extraTools` is set. */
|
|
2365
|
+
readonly executeExtraTool?: (name: string, args: Record<string, unknown>) => Promise<string | null | undefined>;
|
|
2366
|
+
/** Max driver turns before the loop force-finalizes on the best settled child. Default 16.
|
|
2367
|
+
* `0` lifts the turn-COUNT cap: the loop is bounded instead by the conserved budget pool,
|
|
2368
|
+
* an absolute deadline, the driver's own stop, and abort (checked in-loop). A finite
|
|
2369
|
+
* anti-runaway tripwire still guards a degenerate driver that loops on a no-spawn tool. */
|
|
2370
|
+
readonly maxTurns?: number;
|
|
2371
|
+
/** Injected clock for the in-loop absolute-deadline guard — keeps the deadline check
|
|
2372
|
+
* deterministic in tests. Defaults to `Date.now`. */
|
|
2373
|
+
readonly now?: () => number;
|
|
2374
|
+
}
|
|
2375
|
+
/**
|
|
2376
|
+
* Build the intelligent recursive driver. Its `act` is the LLM tool-loop; spawn it as a
|
|
2377
|
+
* `driverChild` (`driver-executor.ts`) to run it inside a nested scope, recursively.
|
|
2378
|
+
*/
|
|
2379
|
+
declare function driverAgent(opts: DriverAgentOptions): Agent<unknown, unknown>;
|
|
2380
|
+
/** Keep-best finalize under the completion-oracle: return the highest-scoring DELIVERED child's
|
|
2381
|
+
* output (settled `done` AND `valid` — its deliverable check passed). Returns undefined when no
|
|
2382
|
+
* child delivered — an honest "the driver produced nothing", never a high-scoring result that
|
|
2383
|
+
* ran without passing its check (Foreman's 0/18 lesson). `valid` is the single delivery signal,
|
|
2384
|
+
* matching `defaultSelectWinner`'s valid-first rule; the oracle just doesn't fall back to an
|
|
2385
|
+
* unchecked best-effort. */
|
|
2386
|
+
declare function finalizeBestDelivered(settled: ReadonlyArray<{
|
|
2387
|
+
status: string;
|
|
2388
|
+
score?: number;
|
|
2389
|
+
valid?: boolean;
|
|
2390
|
+
outRef?: string;
|
|
2391
|
+
}>, blobs: ResultBlobStore): Promise<unknown>;
|
|
2392
|
+
|
|
2393
|
+
/**
|
|
2394
|
+
* @experimental
|
|
2395
|
+
*
|
|
2396
|
+
* Serve the coordination verbs (spawn_agent / await_event / observe_agent / steer_agent / stop)
|
|
2397
|
+
* as a real HTTP MCP server over a LIVE `Scope`. This is the keystone that lets a coding-harness
|
|
2398
|
+
* agent (opencode via the cli-bridge, claude-code, codex) BE the supervisor: it mounts this MCP
|
|
2399
|
+
* (`mcp.mcpServers.coordination`) and calls `spawn_agent` as a native tool, which lands on
|
|
2400
|
+
* `Scope.spawn` — a real box driving real boxes, not emulated function-tools.
|
|
2401
|
+
*
|
|
2402
|
+
* Coordination vs DELEGATION (`../../mcp/delegates.ts`): coordination SPAWNS workers in a CHOSEN
|
|
2403
|
+
* backend (`createExecutor({ backend })` — sandbox OR cli-bridge) and live-drives them — observe /
|
|
2404
|
+
* steer / resume, recursive sub-drivers, one conserved budget. To instead delegate a coding task
|
|
2405
|
+
* INSIDE the agent's OWN sandbox (a durable fire-and-poll job that survives an MCP restart), use the
|
|
2406
|
+
* delegation MCP. Coordination is the live, cross-backend supervisor; delegation is own-sandbox async.
|
|
2407
|
+
*
|
|
2408
|
+
* Transport: JSON-RPC over HTTP POST (the MCP streamable-HTTP shape — `application/json` for a
|
|
2409
|
+
* single response). The server is created INSIDE an agent's `act(task, scope)` so it fronts that
|
|
2410
|
+
* agent's live scope; tear it down when the act returns.
|
|
2411
|
+
*/
|
|
2412
|
+
|
|
2413
|
+
interface CoordinationMcpHandle {
|
|
2414
|
+
/** The URL an in-box harness mounts as `mcp.mcpServers.coordination.url`. */
|
|
2415
|
+
readonly url: string;
|
|
2416
|
+
readonly port: number;
|
|
2417
|
+
/** The coordination tools' settled-worker ledger (for the driver's finalize). */
|
|
2418
|
+
settled(): ReadonlyArray<{
|
|
2419
|
+
status: string;
|
|
2420
|
+
score?: number;
|
|
2421
|
+
valid?: boolean;
|
|
2422
|
+
outRef?: string;
|
|
2423
|
+
}>;
|
|
2424
|
+
isStopped(): boolean;
|
|
2425
|
+
/** The full ordered bus-event log — observability audit + replay trail. */
|
|
2426
|
+
history: CoordinationTools['history'];
|
|
2427
|
+
/** Bus throughput counters for live dashboards. */
|
|
2428
|
+
stats: CoordinationTools['stats'];
|
|
2429
|
+
/** Raise a `finding` on the bus from an online detector watching a worker's live pipe. */
|
|
2430
|
+
raiseFinding: CoordinationTools['raiseFinding'];
|
|
2431
|
+
close(): Promise<void>;
|
|
2432
|
+
}
|
|
2433
|
+
/** Stand up the coordination MCP over a live scope. The HOST address is `127.0.0.1` (the bridge runs
|
|
2434
|
+
* opencode locally, same host); pass `host` to bind elsewhere when the harness is remote. */
|
|
2435
|
+
declare function serveCoordinationMcp(opts: {
|
|
2436
|
+
scope: Scope<unknown>;
|
|
2437
|
+
blobs: ResultBlobStore;
|
|
2438
|
+
makeWorkerAgent: MakeWorkerAgent;
|
|
2439
|
+
perWorker: Budget;
|
|
2440
|
+
port?: number;
|
|
2441
|
+
host?: string;
|
|
2442
|
+
/** Trace-analyst lenses the driver can run (`run_analyst`) or auto-fire on settle. */
|
|
2443
|
+
analysts?: AnalystRegistry;
|
|
2444
|
+
/** Analyst kinds to auto-run when a worker settles `done` — findings flow up the bus. */
|
|
2445
|
+
analyzeOnSettle?: ReadonlyArray<string>;
|
|
2446
|
+
/** Pass-through subscriber for every bus event (settled / question / finding). */
|
|
2447
|
+
onEvent?: (event: CoordinationEvent) => void | Promise<void>;
|
|
2448
|
+
questionPolicy?: QuestionPolicy;
|
|
2449
|
+
}): Promise<CoordinationMcpHandle>;
|
|
2450
|
+
|
|
2451
|
+
/**
|
|
2452
|
+
* @experimental
|
|
2453
|
+
*
|
|
2454
|
+
* `TraceSource` — the ONE substrate-agnostic source of a worker's tool-call trace. The online
|
|
2455
|
+
* detectors and the settle-time analyzers consume agent-eval `ToolSpan`s from here, regardless of
|
|
2456
|
+
* whether the worker is:
|
|
2457
|
+
* - an OWNED tool loop (router-tools, cli-bridge tool dispatch) → push spans as we dispatch them;
|
|
2458
|
+
* - a SANDBOX / fleet box → read the harness's tool calls off the session (`streamPrompt` parts
|
|
2459
|
+
* live, `session.messages()` / `findCompletedTurn` at settle).
|
|
2460
|
+
*
|
|
2461
|
+
* The common currency is agent-eval's `ToolSpan` (so the same detectors + `buildTrajectory`/
|
|
2462
|
+
* `stuckLoopView`/`toolWasteView` run over any source). A source exposes two lanes:
|
|
2463
|
+
* - `onSpan` — live spans for ONLINE detection (best-effort; a black-box box may only collect).
|
|
2464
|
+
* - `collect` — the full span set at settle for the BATCH analyzers (always available).
|
|
2465
|
+
*
|
|
2466
|
+
* This module imports NO substrate SDK — it decodes generic message parts / OpenAI tool-call shapes.
|
|
2467
|
+
* The sandbox wiring (`sandboxSessionTraceSource`) is the thin adapter that feeds box session parts in.
|
|
2468
|
+
*/
|
|
2469
|
+
|
|
2470
|
+
interface ToolStepInput {
|
|
2471
|
+
readonly toolName: string;
|
|
2472
|
+
readonly args: unknown;
|
|
2473
|
+
readonly status?: 'ok' | 'error';
|
|
2474
|
+
readonly result?: unknown;
|
|
2475
|
+
/** Stable id of the tool call — used to de-duplicate the repeated state transitions a harness
|
|
2476
|
+
* streams for one call (opencode emits pending→running→completed, plus a `raw`-wrapped copy). */
|
|
2477
|
+
readonly callId?: string;
|
|
2478
|
+
/** Real per-call wall-clock when the source has it (owned tool-loop; opencode parts with `time`).
|
|
2479
|
+
* When omitted the span collapses to a single instant (`at`) — order + counts only, no duration. */
|
|
2480
|
+
readonly startedAt?: number;
|
|
2481
|
+
readonly endedAt?: number;
|
|
2482
|
+
}
|
|
2483
|
+
interface TraceSource {
|
|
2484
|
+
/** Subscribe to tool spans as they are produced (ONLINE). Returns an unsubscribe. A source that
|
|
2485
|
+
* only exposes its trace at the end registers nothing and returns a no-op. */
|
|
2486
|
+
onSpan(handler: (span: ToolSpan) => void): () => void;
|
|
2487
|
+
/** The full set of tool spans for the run (SETTLE / batch). Always available. */
|
|
2488
|
+
collect(): Promise<ToolSpan[]>;
|
|
2489
|
+
}
|
|
2490
|
+
/** Decode a part with a specific harness's adapter when known, else try every registered adapter
|
|
2491
|
+
* (the composite — robust to mixed/unknown streams). Never throws. */
|
|
2492
|
+
declare function decodeToolPart(part: unknown, harness?: string): ToolStepInput | undefined;
|
|
2493
|
+
/** A push source for OWNED tool loops (router-tools / cli-bridge tool dispatch): the loop calls
|
|
2494
|
+
* `record(step)` for each tool call; it becomes a span, fan-out to live subscribers + buffered for
|
|
2495
|
+
* `collect`. */
|
|
2496
|
+
declare function createPushTraceSource(opts?: {
|
|
2497
|
+
runId?: string;
|
|
2498
|
+
now?: () => number;
|
|
2499
|
+
}): {
|
|
2500
|
+
source: TraceSource;
|
|
2501
|
+
record: (input: ToolStepInput) => ToolSpan;
|
|
2502
|
+
};
|
|
2503
|
+
/** A harness session message carrying parts (the shape `box.messages()` returns). Structurally typed
|
|
2504
|
+
* so this works with the real `@tangle-network/sandbox` box AND a test double, no SDK import. */
|
|
2505
|
+
interface SessionMessageLike {
|
|
2506
|
+
readonly parts?: ReadonlyArray<unknown>;
|
|
2507
|
+
}
|
|
2508
|
+
/** The minimal box surface this needs: list a session's messages (incl. mid-turn partials). */
|
|
2509
|
+
interface SessionTraceBox {
|
|
2510
|
+
messages(opts: {
|
|
2511
|
+
sessionId: string;
|
|
2512
|
+
}): Promise<ReadonlyArray<SessionMessageLike>>;
|
|
2513
|
+
}
|
|
2514
|
+
/** The SANDBOX / fleet trace source: read a box session's message parts and decode the harness's tool
|
|
2515
|
+
* calls into spans. `collect` (settle) is the solid path — `box.messages({sessionId})` → parts → spans;
|
|
2516
|
+
* black-box harnesses aren't mid-step interruptible, so online steering is the owned-loop's job and a
|
|
2517
|
+
* live `subscribe` is opt-in (pass `subscribeParts` from `streamPrompt` when the harness streams parts). */
|
|
2518
|
+
declare function sandboxSessionTraceSource(box: SessionTraceBox, sessionId: string, opts?: {
|
|
2519
|
+
/** The box's harness (e.g. 'opencode', 'claude-code') → selects its decoder adapter. */
|
|
2520
|
+
harness?: string;
|
|
2521
|
+
subscribeParts?: (onPart: (part: unknown) => void) => () => void;
|
|
2522
|
+
runId?: string;
|
|
2523
|
+
now?: () => number;
|
|
2524
|
+
}): TraceSource;
|
|
2525
|
+
|
|
2526
|
+
/**
|
|
2527
|
+
* @experimental
|
|
2528
|
+
*
|
|
2529
|
+
* The ONLINE analyst: watch a `TraceSource` and fold each tool span through agent-eval's published
|
|
2530
|
+
* streaming detector kernel (`repeatedActionDetector`/`errorStreakDetector` — the SAME kernel the
|
|
2531
|
+
* control loop folds), firing `onSignal` the moment a worker loops or error-storms. Substrate-
|
|
2532
|
+
* agnostic: it consumes spans from any source (owned router/bridge loop OR a sandbox box session),
|
|
2533
|
+
* never the raw tool seam. Detection logic + the failure taxonomy live in agent-eval; not reimplemented.
|
|
2534
|
+
*/
|
|
2535
|
+
|
|
2536
|
+
interface WatchTraceOptions {
|
|
2537
|
+
/** The detectors to run online. Defaults to a stuck-loop + error-streak panel. */
|
|
2538
|
+
readonly detectors?: ReadonlyArray<StreamingDetector>;
|
|
2539
|
+
/** Fired for each signal a detector raises — the seam that raises a `finding` on the bus. */
|
|
2540
|
+
readonly onSignal?: (signal: DetectorSignal, span: ToolSpan) => void | Promise<void>;
|
|
2541
|
+
}
|
|
2542
|
+
/** The default online panel for a tool-call pipe: a worker repeating the same call, or hammering
|
|
2543
|
+
* consecutive errors. (No-progress needs a domain progress-probe, so it is opt-in, not default.)
|
|
2544
|
+
*
|
|
2545
|
+
* Coverage note: `repeated-action` works for EVERY harness (it needs only tool name + args, which
|
|
2546
|
+
* every adapter provides). `error-streak` needs per-call status — opencode carries it inline
|
|
2547
|
+
* (`state.status`, VALIDATED live), but claude-code/codex tool-call parts do NOT (their errors live
|
|
2548
|
+
* in separate result blocks not yet decoded), so error-streak is silent for those until result-block
|
|
2549
|
+
* decoding is added + live-validated. It is in the panel because it is correct where status exists. */
|
|
2550
|
+
declare function defaultToolDetectors(): StreamingDetector[];
|
|
2551
|
+
/** Subscribe to a `TraceSource` and run the streaming detectors over its live spans. Returns an
|
|
2552
|
+
* unsubscribe. A defensive `argHash` failure (circular args) never throws out of the side-channel. */
|
|
2553
|
+
declare function watchTrace(source: TraceSource, opts?: WatchTraceOptions): () => void;
|
|
2554
|
+
|
|
2555
|
+
/**
|
|
2556
|
+
* @experimental
|
|
2557
|
+
*
|
|
2558
|
+
* The worker-side receive end of the down-leg: a per-worker inbox an executor exposes as
|
|
2559
|
+
* `Executor.deliver`. The driver's `steer_agent` / `answer_question` land here,
|
|
2560
|
+
* and the worker's agent loop drains them at two points (Drew's two delivery modes):
|
|
2561
|
+
*
|
|
2562
|
+
* - QUEUED (default): the message accumulates and is FLUSHED at the next step boundary — folded
|
|
2563
|
+
* into the conversation before the next think. A worker is also forced to flush BEFORE it may
|
|
2564
|
+
* settle, so it can never finish while a steer/answer it never read is still pending.
|
|
2565
|
+
* - FORCEFUL (`interrupt: true`): trips `freshInterrupt()`'s signal so the loop can abort its
|
|
2566
|
+
* in-flight turn immediately, then re-plan with the message folded in — breaking the worker out
|
|
2567
|
+
* of a wrong path mid-task instead of waiting for it to finish the step.
|
|
2568
|
+
*
|
|
2569
|
+
* `deliver` never throws — a malformed message is ignored, per the `Executor.deliver` contract.
|
|
2570
|
+
*/
|
|
2571
|
+
interface InboxMessage {
|
|
2572
|
+
readonly kind: 'steer' | 'answer';
|
|
2573
|
+
readonly text: string;
|
|
2574
|
+
/** Forceful messages abort the in-flight turn; queued ones wait for the boundary flush. */
|
|
2575
|
+
readonly interrupt: boolean;
|
|
2576
|
+
/** Present for an `answer` — the question id it resolves. */
|
|
2577
|
+
readonly questionId?: string;
|
|
2578
|
+
}
|
|
2579
|
+
interface Inbox {
|
|
2580
|
+
/** The `Executor.deliver` implementation — accept a raw down-message from `Scope.send`. */
|
|
2581
|
+
deliver(msg: unknown): void;
|
|
2582
|
+
/** Remove and return all pending messages (the flush). */
|
|
2583
|
+
drain(): InboxMessage[];
|
|
2584
|
+
pending(): number;
|
|
2585
|
+
/** Open a fresh per-turn interrupt signal; a later forceful `deliver` aborts it. The loop links
|
|
2586
|
+
* this into the signal it passes to its inference call, then re-plans when it fires. */
|
|
2587
|
+
freshInterrupt(): AbortSignal;
|
|
2588
|
+
/** Render drained messages as ONE operator turn to fold into the worker's conversation. */
|
|
2589
|
+
fold(messages: ReadonlyArray<InboxMessage>): string;
|
|
2590
|
+
}
|
|
2591
|
+
declare function createInbox(): Inbox;
|
|
2592
|
+
|
|
2593
|
+
/**
|
|
2594
|
+
* Throw a `ConfigError` when `allowed` is set, `model` is defined, and `model` is not a
|
|
2595
|
+
* member of `allowed`. No-op when `allowed` is unset (the unrestricted default) or when
|
|
2596
|
+
* `model` is undefined (nothing was configured to check).
|
|
2597
|
+
*/
|
|
2598
|
+
declare function assertModelAllowed(model: string | undefined, allowed: readonly string[] | undefined): void;
|
|
2599
|
+
|
|
2600
|
+
/**
|
|
2601
|
+
* @experimental
|
|
2602
|
+
*
|
|
2603
|
+
* `createInMemoryRunContext` — the one-call bundle of the in-memory stores a
|
|
2604
|
+
* `createSupervisor().run(root, task, opts)` needs: a fresh `InMemorySpawnJournal`
|
|
2605
|
+
* (the event-sourced spawn log), a fresh `InMemoryResultBlobStore` (the
|
|
2606
|
+
* content-addressed `outRef` payload store the driver's `observe`/`finalize` reads
|
|
2607
|
+
* settled outputs through), and a fresh `createExecutorRegistry()` (the open
|
|
2608
|
+
* `AgentSpec → Executor` resolver).
|
|
2609
|
+
*
|
|
2610
|
+
* It exists to kill the boilerplate every offline/local supervised run repeats by
|
|
2611
|
+
* hand — three constructors threaded into `SupervisorOpts` — and to single-source the
|
|
2612
|
+
* ONE wiring invariant that is easy to get wrong: when the root is the recursive
|
|
2613
|
+
* `driverAgent` LLM-driver brain AND it may spawn DRIVER children (agents
|
|
2614
|
+
* driving agents), the registry MUST be wrapped with `withDriverExecutor` so a
|
|
2615
|
+
* `role: 'driver'` child resolves to the nested-scope executor — and that SAME blob
|
|
2616
|
+
* store MUST be the one passed to `driverAgent({ blobs })`, or the driver
|
|
2617
|
+
* reads from a different store than the scope writes to. Pass `{ withDriver: true }`
|
|
2618
|
+
* and reuse the returned `blobs` for both.
|
|
2619
|
+
*
|
|
2620
|
+
* The spread shape matches `SupervisorOpts` exactly, so the call site reads:
|
|
2621
|
+
* const run = createInMemoryRunContext()
|
|
2622
|
+
* await createSupervisor().run(root, task, { budget, runId, ...run })
|
|
2623
|
+
*/
|
|
2624
|
+
|
|
2625
|
+
/** Options for the in-memory run context. */
|
|
2626
|
+
interface InMemoryRunContextOptions {
|
|
2627
|
+
/**
|
|
2628
|
+
* Wrap the executor registry with `withDriverExecutor` so a spawned child marked
|
|
2629
|
+
* `role: 'driver'` resolves to the recursive driver-executor (agents driving agents
|
|
2630
|
+
* over a nested `Scope` on the same conserved pool). Leave `false` for a flat tree of
|
|
2631
|
+
* leaf workers. Default `false`.
|
|
2632
|
+
*/
|
|
2633
|
+
readonly withDriver?: boolean;
|
|
2634
|
+
}
|
|
2635
|
+
/**
|
|
2636
|
+
* The bundle of stores a supervised run needs, shaped to spread into `SupervisorOpts`.
|
|
2637
|
+
* The fields are exactly `SupervisorOpts`' `journal` / `blobs` / `executors`.
|
|
2638
|
+
*/
|
|
2639
|
+
interface InMemoryRunContext {
|
|
2640
|
+
readonly journal: SpawnJournal;
|
|
2641
|
+
readonly blobs: ResultBlobStore;
|
|
2642
|
+
readonly executors: ExecutorRegistry;
|
|
2643
|
+
}
|
|
2644
|
+
/**
|
|
2645
|
+
* Build a fresh in-memory run context. Every call returns NEW stores (no shared global
|
|
2646
|
+
* state between runs), so two runs never cross-contaminate their journals/blobs.
|
|
2647
|
+
*/
|
|
2648
|
+
declare function createInMemoryRunContext(opts?: InMemoryRunContextOptions): InMemoryRunContext;
|
|
2649
|
+
|
|
2650
|
+
/**
|
|
2651
|
+
* @experimental
|
|
2652
|
+
*
|
|
2653
|
+
* The leaf runtime — the built-in `Executor` IMPLEMENTATIONS behind the ONE
|
|
2654
|
+
* open interface frozen in `./types`, plus the open resolver/registry that maps
|
|
2655
|
+
* an `AgentSpec` to one of them OR accepts a bring-your-own executor verbatim.
|
|
2656
|
+
*
|
|
2657
|
+
* The interface is the extension point, not a closed `inline|sandbox|cli` union:
|
|
2658
|
+
* - router/inline : a direct OpenAI-compatible Router call, no box (one-shot).
|
|
2659
|
+
* - sandbox : COMPOSES the existing `runLoop` kernel as a single-task
|
|
2660
|
+
* leaf and surfaces its token/cost usage as `UsageEvent`s;
|
|
2661
|
+
* forwards PR #150's optional `lineage` passthrough WITHOUT
|
|
2662
|
+
* reinventing checkpoint/fork (streaming).
|
|
2663
|
+
* - cli : a Halo/RLM subprocess; `budgetExempt` (no token accounting),
|
|
2664
|
+
* excluded from the equal-k arms by construction (streaming).
|
|
2665
|
+
* Every metered runtime reports through the SAME normalized `UsageEvent` channel
|
|
2666
|
+
* so the conserved budget pool meters them identically. A user's own agent is
|
|
2667
|
+
* first-class the moment it implements `Executor` — register it by name or
|
|
2668
|
+
* pass it as `AgentSpec.executor`.
|
|
2669
|
+
*
|
|
2670
|
+
* Layering: `estimateCost`/`isModelPriced` are substrate primitives from
|
|
2671
|
+
* `@tangle-network/agent-eval`; `runLoop`/`acquireSandbox` are runtime kernels
|
|
2672
|
+
* from this package. No per-vendor adapters live here.
|
|
2673
|
+
*/
|
|
2674
|
+
|
|
2675
|
+
/**
|
|
2676
|
+
* Router/inline connection seam. A direct OpenAI-compatible Router endpoint —
|
|
2677
|
+
* the cheapest leaf, no box, no tools. `model` overrides the profile's model
|
|
2678
|
+
* hint when present; otherwise the profile's `model.default` is required.
|
|
2679
|
+
*/
|
|
2680
|
+
interface RouterSeam {
|
|
2681
|
+
routerBaseUrl: string;
|
|
2682
|
+
routerKey: string;
|
|
2683
|
+
model?: string;
|
|
2684
|
+
}
|
|
2685
|
+
/**
|
|
2686
|
+
* Sandbox executor seam. The `sandboxClient` the composed `runLoop` creates
|
|
2687
|
+
* boxes through, plus the optional trace/run/lineage wiring forwarded into the
|
|
2688
|
+
* loop. `lineage` is opaque here (PR #150's `RunLoopOptions.lineage`): forwarded
|
|
2689
|
+
* forward-compatibly, never inspected — this executor does NOT reinvent
|
|
2690
|
+
* checkpoint/fork.
|
|
2691
|
+
*/
|
|
2692
|
+
interface SandboxSeam {
|
|
2693
|
+
sandboxClient: SandboxClient;
|
|
2694
|
+
/** Forwarded into the composed `runLoop`'s `ctx` (trace emitter, run handle, etc.). */
|
|
2695
|
+
loopCtx?: Partial<Omit<ExecCtx, 'sandboxClient' | 'signal'>>;
|
|
2696
|
+
/** PR #150 `RunLoopOptions.lineage` passthrough — opaque; forwarded, not parsed. */
|
|
2697
|
+
lineage?: unknown;
|
|
2698
|
+
/** Hard cap on the composed loop's iterations. The budget pool reserves against
|
|
2699
|
+
* the spawn `Budget.maxIterations`; this is the leaf's own ceiling. Default 1. */
|
|
2700
|
+
maxIterations?: number;
|
|
2701
|
+
}
|
|
2702
|
+
/** CLI subprocess seam. `bin` + `args` describe the Halo/RLM process to spawn. */
|
|
2703
|
+
interface CliSeam {
|
|
2704
|
+
bin: string;
|
|
2705
|
+
args?: string[];
|
|
2706
|
+
/** Extra environment for the subprocess (merged over `process.env`). */
|
|
2707
|
+
env?: Record<string, string>;
|
|
2708
|
+
/** Working directory for the subprocess. */
|
|
2709
|
+
cwd?: string;
|
|
2710
|
+
}
|
|
2711
|
+
/**
|
|
2712
|
+
* cli-worktree seam. A supervisor-authored `AgentProfile` driving a local coding-harness CLI
|
|
2713
|
+
* (claude / codex / opencode) on its own git worktree — the leaf `createWorktreeCliExecutor`
|
|
2714
|
+
* named as data. `harness` + `repoRoot` + `taskPrompt` are required; the authored
|
|
2715
|
+
* `profile.prompt.systemPrompt` + `profile.model.default` reach the harness via the §1.5
|
|
2716
|
+
* `harnessInvocation` mapper. Everything else mirrors `WorktreeCliExecutorOptions`.
|
|
2717
|
+
*/
|
|
2718
|
+
interface CliWorktreeSeam {
|
|
2719
|
+
repoRoot: string;
|
|
2720
|
+
harness: LocalHarness;
|
|
2721
|
+
taskPrompt: string;
|
|
2722
|
+
runId?: string;
|
|
2723
|
+
baseRef?: string;
|
|
2724
|
+
harnessTimeoutMs?: number;
|
|
2725
|
+
}
|
|
2726
|
+
/**
|
|
2727
|
+
* cli-bridge seam. A local OpenAI-compatible bridge that fronts harness CLIs
|
|
2728
|
+
* (claude-code / opencode / kimi / pi) behind one HTTP surface; `model` doubles
|
|
2729
|
+
* as the harness selector (e.g. `claude-code/sonnet`, `opencode/<provider>/<model>`).
|
|
2730
|
+
* `agentProfile` is the bridge-dialect profile (metadata.disallowedTools, mcp)
|
|
2731
|
+
* forwarded verbatim per request — how an arm disables native tools or injects
|
|
2732
|
+
* a provider search MCP.
|
|
2733
|
+
*
|
|
2734
|
+
* The executor opens a RESUMABLE cli-bridge session — structurally identical to the
|
|
2735
|
+
* sandbox executor's persistent box, just local. `sessionId` is the stable
|
|
2736
|
+
* caller-owned id cli-bridge maps to the harness's internal conversation id; a
|
|
2737
|
+
* follow-up steer/resume on the SAME id continues the SAME harness session (opencode
|
|
2738
|
+
* `-s`, claude `--resume`, …). Omit it and the executor mints a stable one per spawn.
|
|
2739
|
+
*/
|
|
2740
|
+
interface BridgeSeam {
|
|
2741
|
+
bridgeUrl: string;
|
|
2742
|
+
bridgeBearer: string;
|
|
2743
|
+
model: string;
|
|
2744
|
+
agentProfile?: Record<string, unknown>;
|
|
2745
|
+
timeoutMs?: number;
|
|
2746
|
+
/** Stable, caller-owned cli-bridge session id for harness-side resume. Defaults
|
|
2747
|
+
* to a freshly minted per-spawn id so each worker is its own resumable session. */
|
|
2748
|
+
sessionId?: string;
|
|
2749
|
+
/** Per-resume-turn inference cap before the worker settles on its last output.
|
|
2750
|
+
* Mirrors `routerToolsInlineExecutor.maxTurns`; default 200 (runaway backstop). */
|
|
2751
|
+
maxTurns?: number;
|
|
2752
|
+
}
|
|
2753
|
+
/**
|
|
2754
|
+
* Router seam WITH tool use — the tool-using router backend. Same direct
|
|
2755
|
+
* OpenAI-compatible endpoint as `RouterSeam`, but each turn passes `tools`; when
|
|
2756
|
+
* the model emits tool_calls they run via `executeToolCall` ON THIS HOST and the
|
|
2757
|
+
* results fold back as `tool` messages, repeating until the model answers without
|
|
2758
|
+
* a tool or `maxTurns` is hit. A real agentic loop, OFF-BOX — no sandbox, so it
|
|
2759
|
+
* is unaffected by a box's egress allowlist. One turn = one completion = the
|
|
2760
|
+
* equal-compute unit. `executeToolCall` receives the task so per-task tool
|
|
2761
|
+
* surfaces (e.g. a gym keyed by task) can dispatch correctly.
|
|
2762
|
+
*/
|
|
2763
|
+
interface RouterToolsSeam {
|
|
2764
|
+
routerBaseUrl: string;
|
|
2765
|
+
routerKey: string;
|
|
2766
|
+
model?: string;
|
|
2767
|
+
tools: ReadonlyArray<ToolSpec>;
|
|
2768
|
+
executeToolCall: (name: string, args: Record<string, unknown>, task: unknown) => Promise<string>;
|
|
2769
|
+
/** Online observer of each tool step — the seam a `DetectorMonitor` taps to watch the live pipe
|
|
2770
|
+
* (raise a `finding` when the worker loops/errors). Called after every tool call resolves, with
|
|
2771
|
+
* real per-call wall-clock (`startedAt`/`endedAt`/`durationMs`) so a push `TraceSource` can carry
|
|
2772
|
+
* non-zero span durations onto the unified timeline. */
|
|
2773
|
+
onToolStep?: (step: {
|
|
2774
|
+
toolName: string;
|
|
2775
|
+
args: Record<string, unknown>;
|
|
2776
|
+
status: 'ok' | 'error';
|
|
2777
|
+
startedAt?: number;
|
|
2778
|
+
endedAt?: number;
|
|
2779
|
+
durationMs?: number;
|
|
2780
|
+
}) => void;
|
|
2781
|
+
/** Max inference turns. Default 200 (runaway backstop — set far above any
|
|
2782
|
+
* legitimate workflow). For tighter per-workflow limits use a cost budget
|
|
2783
|
+
* or wall-clock deadline at the call site. */
|
|
2784
|
+
maxTurns?: number;
|
|
2785
|
+
}
|
|
2786
|
+
/**
|
|
2787
|
+
* The leaf `createWorktreeCliExecutor` as a backend-as-data factory: a supervisor-authored
|
|
2788
|
+
* `AgentProfile` driving claude / codex / opencode on its own worktree. `budgetExempt` like
|
|
2789
|
+
* the other CLI leaves; the authored systemPrompt + model reach the harness via §1.5.
|
|
2790
|
+
*/
|
|
2791
|
+
declare const cliWorktreeExecutor: ExecutorFactory<unknown>;
|
|
2792
|
+
/**
|
|
2793
|
+
* Config for {@link createExecutor}: the backend is DATA — the cost dial a profile,
|
|
2794
|
+
* an experiment config, or a replay journal can name — not an import choice. Each
|
|
2795
|
+
* variant carries its backend's seam (router/router-tools/bridge/cli/cli-worktree/sandbox).
|
|
2796
|
+
*/
|
|
2797
|
+
type ExecutorConfig = ({
|
|
2798
|
+
backend: 'router';
|
|
2799
|
+
} & RouterSeam) | ({
|
|
2800
|
+
backend: 'router-tools';
|
|
2801
|
+
} & RouterToolsSeam) | ({
|
|
2802
|
+
backend: 'bridge';
|
|
2803
|
+
} & BridgeSeam) | ({
|
|
2804
|
+
backend: 'cli';
|
|
2805
|
+
} & CliSeam) | ({
|
|
2806
|
+
backend: 'cli-worktree';
|
|
2807
|
+
} & CliWorktreeSeam) | ({
|
|
2808
|
+
backend: 'sandbox';
|
|
2809
|
+
harness?: BackendType;
|
|
2810
|
+
} & SandboxSeam);
|
|
2811
|
+
/**
|
|
2812
|
+
* The single built-in executor factory. Picks a leaf backend by data (`config.backend`),
|
|
2813
|
+
* injects the matching seam, and delegates to that backend's built-in implementation.
|
|
2814
|
+
* The `Executor` port stays OPEN: bring-your-own agents implement `Executor` directly
|
|
2815
|
+
* and never pass through here. Use this (or `createExecutorRegistry`) instead of a
|
|
2816
|
+
* per-vendor adapter or a closed `inline|sandbox|cli` switch — those bypass the
|
|
2817
|
+
* `UsageEvent` reporting channel.
|
|
2818
|
+
*/
|
|
2819
|
+
declare function createExecutor(config: ExecutorConfig): ExecutorFactory<unknown>;
|
|
2820
|
+
/**
|
|
2821
|
+
* The open resolver/registry. Pre-registers the three built-ins under their
|
|
2822
|
+
* runtime tags (`'router'`, `'sandbox'`, `'cli'`) and accepts `register(name,
|
|
2823
|
+
* factory)` for any additional runtime — and a BYO `AgentSpec.executor` resolves
|
|
2824
|
+
* without touching the registry at all. NOT a closed switch; registration + BYO
|
|
2825
|
+
* ARE the extension points.
|
|
2826
|
+
*
|
|
2827
|
+
* `resolve` precedence (frozen in `ExecutorRegistry`): a BYO `spec.executor` →
|
|
2828
|
+
* `harness === null` → the `'router'` factory; else a registered factory for the
|
|
2829
|
+
* harness-derived runtime (`'sandbox'` for any `BackendType`); else fail loud.
|
|
2830
|
+
*/
|
|
2831
|
+
declare function createExecutorRegistry(): ExecutorRegistry;
|
|
2832
|
+
|
|
2833
|
+
/**
|
|
2834
|
+
* @experimental
|
|
2835
|
+
*
|
|
2836
|
+
* The reactive `Scope` impl (KEYSTONE, build step 4 + the step-8 adapter).
|
|
2837
|
+
*
|
|
2838
|
+
* An `Agent.act` runs inside a `Scope`. It `spawn`s children dynamically and reacts to
|
|
2839
|
+
* them via `next()`. The scope owns ONE in-memory nursery — the authoritative live set —
|
|
2840
|
+
* and is the single place that drives a child's lifecycle: reserve budget atomically,
|
|
2841
|
+
* resolve a `Executor` through the open registry, run it (one-shot OR streaming),
|
|
2842
|
+
* fold its normalized `UsageEvent`s into a conserved `Spend`, reconcile the reservation
|
|
2843
|
+
* (refunding the unspent remainder), persist the result blob + journal records, and
|
|
2844
|
+
* deliver the `Settled` through the `next()` cursor.
|
|
2845
|
+
*
|
|
2846
|
+
* Three invariants this impl enforces by construction:
|
|
2847
|
+
* - `next()` is a ray.wait n=1 cursor over THIS scope's live set; it assigns the
|
|
2848
|
+
* monotonic `seq` (the recorded cursor order) at the moment it yields a settlement, so
|
|
2849
|
+
* replay re-delivers in the identical order — `seq` is never wall-clock.
|
|
2850
|
+
* - Budget is reserved at spawn and reconciled at settle through the shared `BudgetPool`,
|
|
2851
|
+
* so `spawn` fails CLOSED on an exhausted pool and total ≡ free + reserved + committed.
|
|
2852
|
+
* - `view` reads the in-memory nursery, never the journal — O(live), synchronous.
|
|
2853
|
+
*
|
|
2854
|
+
* The settle path is the only writer of journal `settled` events; the spawn path the only
|
|
2855
|
+
* writer of `spawned` events. The result blob is `put` BEFORE the journal `settled` record
|
|
2856
|
+
* references its `outRef`, so a crash can never leave a journaled ref with no blob.
|
|
2857
|
+
*/
|
|
2858
|
+
|
|
2859
|
+
/** Construction args for `createScope`. The supervisor threads the shared pool, journal,
|
|
2860
|
+
* blob store, and executor registry through; `depth`/`maxDepth` pair the runtime
|
|
2861
|
+
* recursion ceiling with the conserved pool (R3). */
|
|
2862
|
+
interface ScopeArgs {
|
|
2863
|
+
/** This scope's owning node id — children get `${parentId}:s${seq}` ids. */
|
|
2864
|
+
readonly parentId: NodeId;
|
|
2865
|
+
/** Journal/blob root key the supervisor `beginTree`'d. */
|
|
2866
|
+
readonly root: NodeId;
|
|
2867
|
+
/** The shared conserved reservation pool (one per supervised run). */
|
|
2868
|
+
readonly pool: BudgetPool;
|
|
2869
|
+
/** Append-only spawn journal; this scope writes `spawned` + `settled` records. */
|
|
2870
|
+
readonly journal: SpawnJournal;
|
|
2871
|
+
/** Content-addressed result store backing `outRef` rehydration. */
|
|
2872
|
+
readonly blobs: ResultBlobStore;
|
|
2873
|
+
/** The open executor resolver (BYO → router/inline → registered harness factory). */
|
|
2874
|
+
readonly executors: ExecutorRegistry;
|
|
2875
|
+
/** Per-spawn executor-construction seams (sandbox client, router config, cli bin). */
|
|
2876
|
+
readonly seams: Readonly<Record<string, unknown>>;
|
|
2877
|
+
/** This scope's recursion depth (root = 0). */
|
|
2878
|
+
readonly depth: number;
|
|
2879
|
+
/** Runtime recursion-depth ceiling — a spawn past it fails closed `depth-exceeded`. */
|
|
2880
|
+
readonly maxDepth?: number;
|
|
2881
|
+
/** Abort signal for this scope; an abort cascades into every live child's executor. */
|
|
2882
|
+
readonly signal: AbortSignal;
|
|
2883
|
+
/** Injected clock — keeps the journal `at` timestamp deterministic in tests. */
|
|
2884
|
+
readonly now?: () => number;
|
|
2885
|
+
/** Lifecycle stream sink. `spawn` emits `agent.spawn`, `next` emits `agent.child` — the
|
|
2886
|
+
* SAME stream `runLoop`/`tool-loop` feed, so the recursive tree is ONE observable stream
|
|
2887
|
+
* (the topology viewer reads it). Undefined ⇒ the journal stays the only record. */
|
|
2888
|
+
readonly hooks?: RuntimeHooks;
|
|
2889
|
+
}
|
|
2890
|
+
declare function createScope<Out>(args: ScopeArgs): Scope<Out>;
|
|
2891
|
+
/**
|
|
2892
|
+
* The step-8 merge-boundary adapter (M4): rehydrate a `Settled.done` into the kernel's
|
|
2893
|
+
* `Iteration` shape so `defaultSelectWinner` stays single-sourced — the supervisor selects
|
|
2894
|
+
* across settled children with the SAME argmax the loop kernel uses, not a forked copy.
|
|
2895
|
+
*
|
|
2896
|
+
* `index` is the cursor `seq` (the recorded, replay-stable order); `output`/`verdict`/
|
|
2897
|
+
* `tokenUsage`/`costUsd` are read straight off the settlement (already rehydrated from the
|
|
2898
|
+
* `outRef` blob by `next()`). Events are empty — a settled child is an opaque leaf result,
|
|
2899
|
+
* not a sandbox event stream — and the timing/cost fields project its conserved `Spend`.
|
|
2900
|
+
* Fail loud on a `down` settlement: only a `done` child is an iteration.
|
|
2901
|
+
*/
|
|
2902
|
+
declare function settledToIteration<Out>(settled: Settled<Out>): Iteration<unknown, Out>;
|
|
2903
|
+
|
|
2904
|
+
/** The supervisor's profile — the subset of an `AgentProfile` that selects + shapes its brain.
|
|
2905
|
+
* `harness` is the backend-as-data discriminant; `systemPrompt` is the standing instruction. */
|
|
2906
|
+
interface SupervisorProfile {
|
|
2907
|
+
readonly name?: string;
|
|
2908
|
+
/** null/undefined → router brain (in-process tool-loop); a coding-CLI harness → sandboxed brain. */
|
|
2909
|
+
readonly harness?: string | null;
|
|
2910
|
+
/** The router model when the brain is router-driven (falls back to the deps router config). */
|
|
2911
|
+
readonly model?: string;
|
|
2912
|
+
/** The standing instructions ("you delegate, you do not solve"). */
|
|
2913
|
+
readonly systemPrompt?: string;
|
|
2914
|
+
}
|
|
2915
|
+
/** How to run a sandboxed harness as the DRIVER, with the coordination verbs mounted — the substrate
|
|
2916
|
+
* seam the caller supplies (mirrors `makeWorkerAgent` for spawned children). It runs `profile` on
|
|
2917
|
+
* `task` in its backend (sandbox / cli-bridge) with `coordinationMcpUrl` mounted as an MCP server,
|
|
2918
|
+
* so the harness calls spawn_agent / await_event / stop as native tools over the live scope. */
|
|
2919
|
+
type DriveHarness = (args: {
|
|
2920
|
+
readonly profile: SupervisorProfile;
|
|
2921
|
+
readonly task: unknown;
|
|
2922
|
+
readonly scope: Scope<unknown>;
|
|
2923
|
+
readonly coordinationMcpUrl: string;
|
|
2924
|
+
}) => Promise<void>;
|
|
2925
|
+
interface SupervisorAgentDeps {
|
|
2926
|
+
readonly blobs: ResultBlobStore;
|
|
2927
|
+
/** Resolve a spawned worker `profile` to a leaf agent — the recursion seam (same for both arms). */
|
|
2928
|
+
readonly makeWorkerAgent: MakeWorkerAgent;
|
|
2929
|
+
/** Per-child budget reserved from the conserved pool on each spawn. */
|
|
2930
|
+
readonly perWorker: Budget;
|
|
2931
|
+
/** Router substrate for a router-brained supervisor (`harness` null). The profile's model wins. */
|
|
2932
|
+
readonly router?: RouterConfig;
|
|
2933
|
+
/** Inject the brain directly (tests / advanced) instead of resolving `routerBrain` from the profile. */
|
|
2934
|
+
readonly brain?: ToolLoopChat;
|
|
2935
|
+
/** Required for a sandboxed-harness supervisor (`harness` set): runs the harness as the driver. */
|
|
2936
|
+
readonly driveHarness?: DriveHarness;
|
|
2937
|
+
/** WORK tools the supervisor may call DIRECTLY (router arm) — so it can do simple work ITSELF and
|
|
2938
|
+
* only delegate when it needs parallelism. Pair with `executeExtraTool`. */
|
|
2939
|
+
readonly extraTools?: ReadonlyArray<{
|
|
2940
|
+
readonly name: string;
|
|
2941
|
+
readonly description?: string;
|
|
2942
|
+
readonly parameters: Record<string, unknown>;
|
|
2943
|
+
}>;
|
|
2944
|
+
/** Runs an `extraTools` call; null/undefined falls through to the coordination dispatch. */
|
|
2945
|
+
readonly executeExtraTool?: (name: string, args: Record<string, unknown>) => Promise<string | null | undefined>;
|
|
2946
|
+
readonly maxTurns?: number;
|
|
2947
|
+
}
|
|
2948
|
+
declare function supervisorAgent(profile: SupervisorProfile, deps: SupervisorAgentDeps): Agent<unknown, unknown>;
|
|
2949
|
+
|
|
2950
|
+
/** Build the worker seam from a backend (WHERE workers run) + an optional completion oracle (the
|
|
2951
|
+
* deliverable check that makes "settled ⟺ delivered" true — the guard against "ran but didn't
|
|
2952
|
+
* deliver"). The ONE place a backend becomes a spawnable worker. */
|
|
2953
|
+
declare function workerFromBackend(backend: ExecutorConfig, deliverable?: DeliverableSpec<unknown>): MakeWorkerAgent;
|
|
2954
|
+
interface SuperviseOptions {
|
|
2955
|
+
/** The conserved compute pool for the whole run. */
|
|
2956
|
+
readonly budget: Budget;
|
|
2957
|
+
/** WHERE workers run — derives the worker seam. Provide this OR an explicit `makeWorkerAgent`. */
|
|
2958
|
+
readonly backend?: ExecutorConfig;
|
|
2959
|
+
/** The completion oracle for backend-derived workers (settled ⟺ delivered). Strongly recommended:
|
|
2960
|
+
* without it the supervisor trusts a worker's self-report — exactly the "ran but didn't deliver"
|
|
2961
|
+
* failure mode of a static orchestrator. */
|
|
2962
|
+
readonly deliverable?: DeliverableSpec<unknown>;
|
|
2963
|
+
/** Override the worker seam directly (tests / advanced) instead of deriving it from `backend`. */
|
|
2964
|
+
readonly makeWorkerAgent?: MakeWorkerAgent;
|
|
2965
|
+
/** The supervisor's router substrate (`harness` null). The profile's model wins. */
|
|
2966
|
+
readonly router?: RouterConfig;
|
|
2967
|
+
/** Inject the supervisor brain directly (tests / advanced). */
|
|
2968
|
+
readonly brain?: ToolLoopChat;
|
|
2969
|
+
/** Run a sandboxed-harness supervisor (`harness` set). */
|
|
2970
|
+
readonly driveHarness?: DriveHarness;
|
|
2971
|
+
/** WORK tools the supervisor may call DIRECTLY — so a recursive atom can ACT (do simple work
|
|
2972
|
+
* itself) OR SPAWN (delegate when it needs parallelism), not be a pure manager. Pair with
|
|
2973
|
+
* `executeExtraTool`. Router arm only (`harness` null). */
|
|
2974
|
+
readonly extraTools?: ReadonlyArray<{
|
|
2975
|
+
readonly name: string;
|
|
2976
|
+
readonly description?: string;
|
|
2977
|
+
readonly parameters: Record<string, unknown>;
|
|
2978
|
+
}>;
|
|
2979
|
+
/** Runs an `extraTools` call; null/undefined falls through to the coordination dispatch. */
|
|
2980
|
+
readonly executeExtraTool?: (name: string, args: Record<string, unknown>) => Promise<string | null | undefined>;
|
|
2981
|
+
/** Per-child budget reserved on each spawn. Defaults to a quarter of the pool's tokens. */
|
|
2982
|
+
readonly perWorker?: Budget;
|
|
2983
|
+
/** Worker output store. Defaults to in-memory. */
|
|
2984
|
+
readonly blobs?: ResultBlobStore;
|
|
2985
|
+
readonly maxDepth?: number;
|
|
2986
|
+
readonly maxTurns?: number;
|
|
2987
|
+
readonly runId?: string;
|
|
2988
|
+
readonly now?: () => number;
|
|
2989
|
+
/** Restrict the run to this subset of models. When set, every configured model — the
|
|
2990
|
+
* supervisor router model, the profile's model, and the backend's model — must be a member,
|
|
2991
|
+
* or `supervise()` throws a `ConfigError` before any compute is spent. Unset = unrestricted. */
|
|
2992
|
+
readonly allowedModels?: readonly string[];
|
|
2993
|
+
}
|
|
2994
|
+
declare function supervise(profile: SupervisorProfile, task: unknown, opts: SuperviseOptions): Promise<SupervisedResult<unknown>>;
|
|
2995
|
+
|
|
2996
|
+
/**
|
|
2997
|
+
* @experimental
|
|
2998
|
+
*
|
|
2999
|
+
* The `Supervisor` impl (KEYSTONE, build step 5).
|
|
3000
|
+
*
|
|
3001
|
+
* Owns the four things a free-running recursive `act` cannot own itself: the GLOBAL
|
|
3002
|
+
* conserved budget pool, the event-sourced spawn log, the abort cascade over the whole
|
|
3003
|
+
* live tree, and the OTP intensity breaker. `run` builds the root `Scope` over those,
|
|
3004
|
+
* runs the root `Agent.act`, and returns a TYPED `SupervisedResult` — a no-winner is
|
|
3005
|
+
* never coerced into a best-effort `Out`.
|
|
3006
|
+
*
|
|
3007
|
+
* Three lifecycle invariants this impl enforces by construction:
|
|
3008
|
+
* - Join barrier: when `act()` settles (resolve OR reject), every still-live child is
|
|
3009
|
+
* torn down before `run` returns — the generalization of the kernel's
|
|
3010
|
+
* `finally{ Promise.allSettled(destroy) }` barrier (run-loop.ts) from boxes to the
|
|
3011
|
+
* whole sub-tree. A teardown failure is `allSettled`'d and journaled as a
|
|
3012
|
+
* `cancelled` event; it NEVER masks act()'s own outcome. act()'s rejection is the
|
|
3013
|
+
* PRIMARY error (the kernel's firstError precedence), so a teardown throw during the
|
|
3014
|
+
* barrier can never overwrite the real failure.
|
|
3015
|
+
* - Abort cascade: a root abort (caller signal, `RootHandle.abort`, a tripped breaker,
|
|
3016
|
+
* or pool exhaustion) aborts ONE internal controller whose signal is the root scope's
|
|
3017
|
+
* signal. The scope cascades that into every live child's executor abort — which, for
|
|
3018
|
+
* an `acquiring` child, chains into the `acquireSandbox` signal and reaps the
|
|
3019
|
+
* find-by-name orphan box (M1). The supervisor never reaps children directly.
|
|
3020
|
+
* - The supervisor NEVER re-enters a child (m3): the kernel/`acquireSandbox` already
|
|
3021
|
+
* retried at the leaf, and a driver re-spawns through `scope.spawn`. The breaker only
|
|
3022
|
+
* COUNTS `down` settlements within the intensity window and trips to a typed
|
|
3023
|
+
* no-winner; it does not restart anything.
|
|
3024
|
+
*
|
|
3025
|
+
* Selection lives in the driver, not here (selector≠judge): `act` returns the synthesized
|
|
3026
|
+
* winner `Out`. The supervisor content-addresses that `Out` for its replay `outRef`, reads
|
|
3027
|
+
* `spentTotal` off the journal (`settled` child work + `metered` driver inference), and wraps
|
|
3028
|
+
* it as a typed `winner` — it does not re-rank children behind the driver's back.
|
|
3029
|
+
*/
|
|
3030
|
+
|
|
3031
|
+
declare function createSupervisor<Task, Out>(): Supervisor<Task, Out>;
|
|
3032
|
+
|
|
3033
|
+
/**
|
|
3034
|
+
* @experimental
|
|
3035
|
+
*
|
|
3036
|
+
* The SETTLE-time analyst: when a worker finishes, collect its tool spans from a `TraceSource` and run
|
|
3037
|
+
* agent-eval's PUBLISHED batch analyzers over them — `buildTrajectory` (structured run summary),
|
|
3038
|
+
* `stuckLoopView` (full-run repeated-call view, complementing the online consecutive detector), and
|
|
3039
|
+
* `toolWasteView`. Substrate-agnostic: the spans come from any source (an owned loop's buffer OR a
|
|
3040
|
+
* sandbox box session). No analysis reimplemented — this is the thin bridge into agent-eval's analyzers.
|
|
3041
|
+
*/
|
|
3042
|
+
|
|
3043
|
+
interface TrajectoryAnalysis {
|
|
3044
|
+
/** Structured run summary (tool-call count, step order). Steps carry a single timestamp, so per-span
|
|
3045
|
+
* duration is 0; loop/waste detection keys on call PATTERNS + cross-span windows, not durations. */
|
|
3046
|
+
readonly trajectory: Awaited<ReturnType<typeof buildTrajectory>>;
|
|
3047
|
+
/** Full-run repeated-call view (total occurrences + window) — catches a loop the online consecutive
|
|
3048
|
+
* detector interleaves past. */
|
|
3049
|
+
readonly stuckLoop: Awaited<ReturnType<typeof stuckLoopView>>;
|
|
3050
|
+
/** Wasted-vs-total tool-call ratio for the run. */
|
|
3051
|
+
readonly toolWaste: Awaited<ReturnType<typeof toolWasteView>>;
|
|
3052
|
+
}
|
|
3053
|
+
/** Collect the source's spans and run the agent-eval batch analyzers over them under one `runId`. */
|
|
3054
|
+
declare function analyzeTrace(source: TraceSource, runId?: string): Promise<TrajectoryAnalysis>;
|
|
3055
|
+
|
|
3056
|
+
/**
|
|
3057
|
+
* createVerifierEnvironment — ANY checkable task as an `Environment`, no tool surface
|
|
3058
|
+
* required. The generalization piece: EOPS/commit0-style domains have tools that mutate
|
|
3059
|
+
* an external artifact, but math problems, legal drafts, creative briefs, GTM copy, and
|
|
3060
|
+
* QA tasks have a different shape — the artifact IS the worker's answer, and the domain
|
|
3061
|
+
* is defined by one function: the deployable check over that answer.
|
|
3062
|
+
*
|
|
3063
|
+
* const gsm8k = createVerifierEnvironment({
|
|
3064
|
+
* name: 'gsm8k',
|
|
3065
|
+
* check: (task, answer) => ({
|
|
3066
|
+
* passes: extractFinalNumber(answer) === task.meta?.answer ? 1 : 0,
|
|
3067
|
+
* total: 1,
|
|
3068
|
+
* errored: 0,
|
|
3069
|
+
* }),
|
|
3070
|
+
* })
|
|
3071
|
+
* await runBenchmark({ environment: gsm8k, tasks, worker }) // sample vs refine on math
|
|
3072
|
+
*
|
|
3073
|
+
* The worker gets one built-in tool — `submit_answer` — plus any read-only domain tools
|
|
3074
|
+
* the caller adds (a calculator, a retrieval call, a style guide lookup). Every
|
|
3075
|
+
* submission is kept; `score()` checks the BEST submission (keep-best is the measured
|
|
3076
|
+
* law: workers reach correct answers then revise past them). The refine strategy's
|
|
3077
|
+
* critic reads the submission trajectory like any other trace, so iterate-with-feedback
|
|
3078
|
+
* works unchanged on answer domains.
|
|
3079
|
+
*
|
|
3080
|
+
* The check can be graded (passes/total expresses partial credit — rubric points,
|
|
3081
|
+
* sub-answers, unit-test counts), and MUST be deployable (computable without an oracle
|
|
3082
|
+
* at serve time): exact/numeric match, schema validation, a compiled rubric — not a
|
|
3083
|
+
* peek at held-out labels the production system wouldn't have.
|
|
3084
|
+
*/
|
|
3085
|
+
|
|
3086
|
+
interface VerifierEnvironmentOptions {
|
|
3087
|
+
name: string;
|
|
3088
|
+
/** The deployable check over a submitted answer. Graded via passes/total. */
|
|
3089
|
+
check(task: AgenticTask, answer: string): Promise<SurfaceScore> | SurfaceScore;
|
|
3090
|
+
/** Extra domain tools (read-only helpers: calculator, retrieval, style lookup). */
|
|
3091
|
+
extraTools?: AgenticTool[];
|
|
3092
|
+
/** Executes the extra tools. Required when `extraTools` is set. */
|
|
3093
|
+
callExtra?(task: AgenticTask, name: string, args: Record<string, unknown>): Promise<string> | string;
|
|
3094
|
+
}
|
|
3095
|
+
declare function createVerifierEnvironment(opts: VerifierEnvironmentOptions): Environment;
|
|
3096
|
+
|
|
3097
|
+
/** Command runner seam. Host code can use `localShell`; sandbox code can wrap `box.exec`. */
|
|
3098
|
+
type Shell = (args: ReadonlyArray<string>, cwd?: string) => Promise<{
|
|
3099
|
+
stdout: string;
|
|
3100
|
+
stderr: string;
|
|
3101
|
+
code: number;
|
|
3102
|
+
}>;
|
|
3103
|
+
type WorkspaceCommit = {
|
|
3104
|
+
readonly ok: true;
|
|
3105
|
+
readonly rev: string;
|
|
3106
|
+
} | {
|
|
3107
|
+
readonly ok: false;
|
|
3108
|
+
readonly conflict: string;
|
|
3109
|
+
};
|
|
3110
|
+
interface Workspace {
|
|
3111
|
+
readonly ref: string;
|
|
3112
|
+
materialize(dir: string): Promise<void>;
|
|
3113
|
+
commit(dir: string, message: string): Promise<WorkspaceCommit>;
|
|
3114
|
+
head(): Promise<string>;
|
|
3115
|
+
}
|
|
3116
|
+
declare function localShell(): Shell;
|
|
3117
|
+
interface GitWorkspaceOptions {
|
|
3118
|
+
readonly ref: string;
|
|
3119
|
+
readonly shell?: Shell;
|
|
3120
|
+
readonly branch?: string;
|
|
3121
|
+
readonly noHooks?: boolean;
|
|
3122
|
+
}
|
|
3123
|
+
declare function gitWorkspace(opts: GitWorkspaceOptions): Workspace;
|
|
3124
|
+
/** A jj-backed `Workspace` (Jujutsu, colocated with git for the durable remote).
|
|
3125
|
+
* Same port, same `Shell` — a drop-in for `gitWorkspace`. jj suits agent loops:
|
|
3126
|
+
* no staging area, and a first-class operation log (native resume/undo). Live use
|
|
3127
|
+
* requires `jj` on the `Shell`'s host. */
|
|
3128
|
+
declare function jjWorkspace(opts: GitWorkspaceOptions): Workspace;
|
|
3129
|
+
interface WorkspaceRun<T> {
|
|
3130
|
+
readonly valid: boolean;
|
|
3131
|
+
readonly value: T;
|
|
3132
|
+
/** Present when a commit was attempted (valid, or `commitOnInvalid`). */
|
|
3133
|
+
readonly commit?: WorkspaceCommit;
|
|
3134
|
+
}
|
|
3135
|
+
/**
|
|
3136
|
+
* Run a worker `body` inside a FRESH clone of a shared `Workspace`, then commit its work back
|
|
3137
|
+
* so the next worker (or the supervisor) builds on it. This is the seam that turns isolated
|
|
3138
|
+
* per-worker cwds into one compounding artifact — `body` gets a real materialized dir, its
|
|
3139
|
+
* delivery is committed to the shared ref iff it's valid (a conflict is returned, never thrown).
|
|
3140
|
+
* The clone is removed after; durable state lives only in the ref.
|
|
3141
|
+
*/
|
|
3142
|
+
declare function runInWorkspace<T>(ws: Workspace, body: (cwd: string) => Promise<{
|
|
3143
|
+
valid: boolean;
|
|
3144
|
+
value: T;
|
|
3145
|
+
message?: string;
|
|
3146
|
+
}>, opts?: {
|
|
3147
|
+
tmpPrefix?: string;
|
|
3148
|
+
commitOnInvalid?: boolean;
|
|
3149
|
+
}): Promise<WorkspaceRun<T>>;
|
|
3150
|
+
|
|
3151
|
+
export { Agent, AgentRunSpec, type AgenticOptions, type AgenticRunResult, type AgenticSurface, type AgenticTask, type AgenticTool, type AnytimeReport, type AnytimeStrategySummary, type AnytimeTaskCurve, type ArtifactHandle, AssertTraceDerivedFindings, type AuditIntentInput, type AuditIntentOptions, type AuthorStrategyOptions, type AuthoredProfile, type AuthoredStrategy, type BenchmarkCell, type BenchmarkConfig, type BenchmarkLift, type BenchmarkReport, type BenchmarkStrategySummary, type BenchmarkTaskRow, Budget, type BudgetPool, type BudgetReadout, type ChampionPick, type ChampionPolicy, type CheckpointCapableBox, CombinatorShape, type CompletionAnalyst, type CompletionEvidence, type CompletionPolicy, type CompletionVerdict, CoordinationEvent, type CoordinationMcpHandle, Corpus, CorpusFilter, CorpusRecord, type CreateScopeAnalystOptions, type CriuCapableClient, DefinePersonaInput, type Deliverable, DeliverableSpec, type DriveHarness, Driver, type DriverAgentOptions, type Environment, EqualKArm, EqualKOnCostOptions, EqualKVerdict, type EvolutionArchiveNode, type EvolutionAuthor, type EvolutionBandInfo, type EvolutionCandidate, type EvolutionGeneration, type EvolutionReport, ExecCtx, type ExecutorConfig, FanoutOptions, FanoutWinnerSelector, FileCorpus, type ForkCapableBox, type GitWorkspaceOptions, type HarvestCorpusOptions, type HarvestFailure, type HarvestReport, InMemoryCorpus, InMemoryResultBlobStore, type InMemoryRunContext, type InMemoryRunContextOptions, InMemorySpawnJournal, type Inbox, type InboxMessage, type IntentAudit, Iteration, type LoopDispatchOptions, LoopLineageOptions, type LoopOptionsForDispatch, LoopResult, LoopShape, LoopTokenUsage, LoopUntilSpec, LoopWinner, type McpEndpoint, type McpEnvironmentOptions, type Observation, type ObserveInput, type ObserveOptions, type OpenSandboxRunOptions, Outcome, OutputAdapter, PanelSpec, Persona, PipelineStage, type ProfileRichness, type ProfileRichnessThresholds, type PromotionGateOptions, type PromotionVerdict, type RegistryAnalyzeProjection, RenderCorpusToInstructionsOptions, type ReservationTicket, ResultBlobStore, RouterConfig, type RunAgenticOptions, RunPersonifiedOptions, type SandboxCapabilities, SandboxClient, type SandboxLineage, type SandboxLineageHandle, type SandboxRun, Scope, ScopeAnalyst, ScopeAnalyzeInput, ScopeWidenGate, type SessionCapableBox, type SessionTraceBox, Settled, ShapeRegistry, type Shell, type ShotPersona, type ShotSpec, Spend, SteerContext, type Strategy, type StrategyCtx, type StrategyEvolutionConfig, type StrategyResult, type SuperviseOptions, SupervisedResult, Supervisor, type SupervisorAgentDeps, type SupervisorProfile, type SurfaceScore, ToolLoopChat, ToolSpec, type TraceSource, type TrajectoryAnalysis, TrajectoryReport, TrajectoryReportOptions, type TurnResult, UsageEvent, type UsageSink, Validator, type VerifierEnvironmentOptions, VerifySpec, type WatchTraceOptions, type WaterfallCollector, type WaterfallReport, type WaterfallSpan, WidenSpec, WinnerStrategy, type Workspace, type WorkspaceCommit, type WorkspaceRun, acquireSandbox, adaptiveRefine, analyzeTrace, anytimeReport, asAuthoredProfile, assertModelAllowed, assertStrategyContract, assertTraceDerivedFindings, assessAuthoredProfile, auditIntent, authorStrategy, authoredWorker, breadthStrategy, buildSteerContext, builtinShapes, cliWorktreeExecutor, completionAuthorizes, contentAddress, createBudgetPool, createExecutor, createExecutorRegistry, createInMemoryRunContext, createInbox, createMcpEnvironment, createPushTraceSource, createSandboxLineage, createScope, createScopeAnalyst, createShapeRegistry, createSupervisor, createVerifierEnvironment, createWaterfallCollector, decodeToolPart, defaultAnalystInstruction, defaultAuditorInstruction, defaultProfileRichnessThresholds, defaultSelectWinner, defaultToolDetectors, definePersona, defineStrategy, depthStrategy, deterministicCompletion, discriminatingMeans, driverAgent, equalKOnCost, extractLlmCallEvent, fanout, finalizeBestDelivered, flatWidenGate, gitWorkspace, harvestCorpus, inlineSandboxClient, jjWorkspace, localShell, loopDispatch, loopUntil, mapSandboxEvent, observe, openSandboxRun, panel, pickChampion, pipeline, printBenchmarkReport, probeSandboxCapabilities, profileRichnessFinding, promotionGate, refine, registerShape, registryScopeAnalyst, renderAnytimeTable, renderCorpusToInstructions, renderReport, reportLoopUsage, runAgentic, runBenchmark, runInWorkspace, runLoop, runPersonified, runStrategyEvolution, sample, sampleThenRefine, sandboxSessionTraceSource, selectChampion, selectValidWinner, sentinelCompletion, serveCoordinationMcp, settledToIteration, spendFromUsageEvents, stopSentinel, strategyAuthorContract, supervise, supervisorAgent, supervisorInstructions, trajectoryReport, verify, watchTrace, widen, workerFromBackend };
|