@tangle-network/agent-runtime 0.128.0 → 0.131.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +70 -20
- package/dist/{activation-DhWJ3p8N.js → activation-BCzMOTaV.js} +3 -3
- package/dist/{activation-DhWJ3p8N.js.map → activation-BCzMOTaV.js.map} +1 -1
- package/dist/agent.d.ts +2 -3
- package/dist/agent.js +4 -5
- package/dist/agent.js.map +1 -1
- package/dist/{analyst-loop-DvSciOfB.js → analyst-loop-BE8cDs5Q.js} +2 -2
- package/dist/{analyst-loop-DvSciOfB.js.map → analyst-loop-BE8cDs5Q.js.map} +1 -1
- package/dist/analyst-loop.d.ts +1 -1
- package/dist/analyst-loop.js +1 -1
- package/dist/authoring-Dv3t6SXe.js +163 -0
- package/dist/authoring-Dv3t6SXe.js.map +1 -0
- package/dist/candidate-execution/index.js +4 -4
- package/dist/{candidate-execution-BFpq-Xi6.js → candidate-execution-BNxKr-Bu.js} +5 -5
- package/dist/{candidate-execution-BFpq-Xi6.js.map → candidate-execution-BNxKr-Bu.js.map} +1 -1
- package/dist/{conversation-BpLQZGPH.js → conversation-DNtxaJ1Z.js} +113 -31
- package/dist/conversation-DNtxaJ1Z.js.map +1 -0
- package/dist/conversation.d.ts +2 -2
- package/dist/conversation.js +2 -2
- package/dist/environment-provider-CxvSd1W6.d.ts +86 -0
- package/dist/{environment-provider-DqFS6FSZ.js → environment-provider-Dyg8DtLK.js} +8 -4
- package/dist/environment-provider-Dyg8DtLK.js.map +1 -0
- package/dist/environment-provider.d.ts +1 -1
- package/dist/environment-provider.js +1 -1
- package/dist/graph-xWdv53Le.js +471 -0
- package/dist/graph-xWdv53Le.js.map +1 -0
- package/dist/{improvement-cycle-IJgCbKWQ.js → improvement-cycle-Csp38cWg.js} +133 -224
- package/dist/improvement-cycle-Csp38cWg.js.map +1 -0
- package/dist/{index-BhZhQw77.d.ts → index-CoO7atyo.d.ts} +556 -1278
- package/dist/{index-BhuzfG2r.d.ts → index-DwGtu9nc.d.ts} +7 -9
- package/dist/{index-Efjb3nrQ.d.ts → index-qYHpsmG2.d.ts} +36 -18
- package/dist/index.d.ts +353 -11
- package/dist/index.js +111 -354
- package/dist/index.js.map +1 -1
- package/dist/intelligence.d.ts +6 -6
- package/dist/intelligence.js +9 -8
- package/dist/intelligence.js.map +1 -1
- package/dist/kernel.d.ts +7 -5
- package/dist/kernel.js +13 -9
- package/dist/{knowledge-DF63xPr4.js → knowledge-DPEu4f-0.js} +19 -17
- package/dist/knowledge-DPEu4f-0.js.map +1 -0
- package/dist/knowledge.d.ts +1 -1
- package/dist/knowledge.js +1 -1
- package/dist/{loop-runner-bin-Ckp_9tmD.d.ts → loop-runner-bin-BwgZ8m_7.d.ts} +6 -3
- package/dist/{loop-runner-bin-CWqOpCEw.js → loop-runner-bin-dg6li2-b.js} +5 -27
- package/dist/loop-runner-bin-dg6li2-b.js.map +1 -0
- package/dist/loop-runner-bin.d.ts +1 -1
- package/dist/loop-runner-bin.js +1 -1
- package/dist/materialization-COJ1UYQ-.js +272 -0
- package/dist/materialization-COJ1UYQ-.js.map +1 -0
- package/dist/mcp/bin.js +39 -47
- package/dist/mcp/bin.js.map +1 -1
- package/dist/mcp/index.d.ts +24 -26
- package/dist/mcp/index.js +67 -84
- package/dist/mcp/index.js.map +1 -1
- package/dist/mcp/memory-bin.js +1 -1
- package/dist/{memory-server-DL6cE2Ag.js → memory-server-eD2baiRO.js} +3 -3
- package/dist/{memory-server-DL6cE2Ag.js.map → memory-server-eD2baiRO.js.map} +1 -1
- package/dist/model-policy-CqziaqS1.js +232 -0
- package/dist/model-policy-CqziaqS1.js.map +1 -0
- package/dist/{openai-tools-B68JaOCx.d.ts → openai-tools-D3oMyVGl.d.ts} +2 -2
- package/dist/{openai-tools-D3XfrrQ6.js → openai-tools-zRphjXS4.js} +2 -2
- package/dist/openai-tools-zRphjXS4.js.map +1 -0
- package/dist/{prepare--8EvLqCr.js → prepare-DYWjVcPx.js} +169 -153
- package/dist/prepare-DYWjVcPx.js.map +1 -0
- package/dist/primeintellect/index.d.ts +7 -6
- package/dist/primeintellect/index.js +9 -11
- package/dist/primeintellect/index.js.map +1 -1
- package/dist/profiles.d.ts +21 -174
- package/dist/profiles.js +67 -276
- package/dist/profiles.js.map +1 -1
- package/dist/{protected-model-port-CXVfOUu_.js → protected-model-port-48ALLGxT.js} +2 -2
- package/dist/{protected-model-port-CXVfOUu_.js.map → protected-model-port-48ALLGxT.js.map} +1 -1
- package/dist/{redact-PQzmE1Jn.d.ts → redact-9_Gf-8m3.d.ts} +58 -69
- package/dist/{researcher-CoVqNhfI.js → researcher-Skz5-Uc8.js} +50 -26
- package/dist/researcher-Skz5-Uc8.js.map +1 -0
- package/dist/{run-layout-C2FGmZ3v.js → run-layout-CeJsEAom.js} +2 -2
- package/dist/{run-layout-C2FGmZ3v.js.map → run-layout-CeJsEAom.js.map} +1 -1
- package/dist/runtime-D-QfLbSd.d.ts +893 -0
- package/dist/{runtime-5uDVVfER.js → runtime-cOzDOOHr.js} +315 -1191
- package/dist/runtime-cOzDOOHr.js.map +1 -0
- package/dist/{sandbox-events-Yhd1GYWl.js → sandbox-events-CRDwc5WN.js} +42 -5
- package/dist/sandbox-events-CRDwc5WN.js.map +1 -0
- package/dist/snapshot-CXiiuHhL.js +21 -0
- package/dist/snapshot-CXiiuHhL.js.map +1 -0
- package/dist/{spawn-journal-DsZKDqeh.js → spawn-journal-saHQzqYi.js} +15 -21
- package/dist/spawn-journal-saHQzqYi.js.map +1 -0
- package/dist/stream-agent-turn-C852AgMT.d.ts +128 -0
- package/dist/stream-agent-turn-rYgaOLO0.js +910 -0
- package/dist/stream-agent-turn-rYgaOLO0.js.map +1 -0
- package/dist/{structural-rollout-zY0oqQzO.js → structural-rollout-3uxVGcg2.js} +459 -235
- package/dist/structural-rollout-3uxVGcg2.js.map +1 -0
- package/dist/{supervise-CsTKbH9R.js → supervise-DHYX8gO2.js} +867 -4788
- package/dist/supervise-DHYX8gO2.js.map +1 -0
- package/dist/{supervisor-DpjO0Gmy.js → supervisor-CV6Jh28D.js} +7439 -2897
- package/dist/supervisor-CV6Jh28D.js.map +1 -0
- package/dist/testing.d.ts +3 -1
- package/dist/testing.js +271 -221
- package/dist/testing.js.map +1 -1
- package/dist/{tool-server-RcWgLIsL.js → tool-server-Gs3VvfSK.js} +22 -9
- package/dist/tool-server-Gs3VvfSK.js.map +1 -0
- package/dist/{top-app-5unxqovu.js → top-app-G4M18b6i.js} +3 -3
- package/dist/{top-app-5unxqovu.js.map → top-app-G4M18b6i.js.map} +1 -1
- package/dist/tui/bin.js +1 -1
- package/dist/tui/index.js +1 -1
- package/dist/{environment-provider-CUFsyymu.d.ts → types-C6Q-J0Dt.d.ts} +51 -114
- package/dist/{types-DnNGJ5Gz.d.ts → types-ebIY0dMG.d.ts} +556 -30
- package/dist/{util-MVgdwuIS.js → util-Bw6srryQ.js} +3 -2
- package/dist/{util-MVgdwuIS.js.map → util-Bw6srryQ.js.map} +1 -1
- package/dist/{workspace-archive-BQxvkypI.js → workspace-archive-aOfJ47ms.js} +3 -3
- package/dist/{workspace-archive-BQxvkypI.js.map → workspace-archive-aOfJ47ms.js.map} +1 -1
- package/package.json +13 -15
- package/skills/agent-graphs/IMPROVE.md +3 -3
- package/skills/agent-graphs/SKILL.md +4 -5
- package/skills/agent-graphs/cases/review-pipeline.json +1 -2
- package/skills/agent-graphs/cases/unmeasured-harness.json +2 -4
- package/dist/backends-CiOCyRHb.js +0 -743
- package/dist/backends-CiOCyRHb.js.map +0 -1
- package/dist/conversation-BpLQZGPH.js.map +0 -1
- package/dist/environment-provider-DqFS6FSZ.js.map +0 -1
- package/dist/improvement-cycle-IJgCbKWQ.js.map +0 -1
- package/dist/index-DLM0W1h1.d.ts +0 -545
- package/dist/knowledge-DF63xPr4.js.map +0 -1
- package/dist/local-harness-BIajef4A.d.ts +0 -465
- package/dist/loop-runner-bin-CWqOpCEw.js.map +0 -1
- package/dist/model-resolution-Btd9iIKV.js +0 -98
- package/dist/model-resolution-Btd9iIKV.js.map +0 -1
- package/dist/openai-tools-D3XfrrQ6.js.map +0 -1
- package/dist/prepare--8EvLqCr.js.map +0 -1
- package/dist/researcher-CoVqNhfI.js.map +0 -1
- package/dist/runtime-5uDVVfER.js.map +0 -1
- package/dist/sandbox-events-Yhd1GYWl.js.map +0 -1
- package/dist/spawn-journal-DsZKDqeh.js.map +0 -1
- package/dist/structural-rollout-zY0oqQzO.js.map +0 -1
- package/dist/supervise-CsTKbH9R.js.map +0 -1
- package/dist/supervisor-DpjO0Gmy.js.map +0 -1
- package/dist/tool-server-RcWgLIsL.js.map +0 -1
- package/dist/types-C9j4qg6l.d.ts +0 -500
- package/skills/agent-graphs/cases/floor-trap-pi.json +0 -11
|
@@ -1,14 +1,17 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { n as streamAgentTurn, t as collectAgentTurn } from "./stream-agent-turn-rYgaOLO0.js";
|
|
2
|
+
import { i as InMemorySpawnJournal, r as InMemoryResultBlobStore } from "./spawn-journal-saHQzqYi.js";
|
|
3
|
+
import { i as concreteModelId, s as profileModelExecutionSettings, t as assertExecutableAgentProfile } from "./model-policy-CqziaqS1.js";
|
|
4
|
+
import { v as executableAgentProfileSnapshot } from "./materialization-COJ1UYQ-.js";
|
|
5
|
+
import { N as createExecutor, j as captureReusableExecutorConfig, l as withDriverExecutor, n as createSupervisor } from "./supervisor-CV6Jh28D.js";
|
|
6
|
+
import { costForTokenPricing, makeProposalFinding } from "@tangle-network/agent-eval";
|
|
7
|
+
import { agentProfileSchema, canonicalAgentProfileDigest, canonicalCandidateDigest } from "@tangle-network/agent-interface";
|
|
4
8
|
import { randomBytes } from "node:crypto";
|
|
5
9
|
import { assertProposalFindings } from "@tangle-network/agent-eval/analyst";
|
|
6
10
|
//#region src/improvement/optimizer-prompt.ts
|
|
7
11
|
/**
|
|
8
12
|
* The senior scientific-method optimizer doctrine — the ONE substantial prompt
|
|
9
13
|
* core shared by every builder/author surface (tool build, MCP build, codebase
|
|
10
|
-
* improvement, strategy authoring)
|
|
11
|
-
* (`driverLoopGenerator`).
|
|
14
|
+
* improvement, and strategy authoring).
|
|
12
15
|
*
|
|
13
16
|
* Seeded from the proven senior prompts rather than invented: GEPA's
|
|
14
17
|
* `REFLECTION_SYSTEM` (localize → diagnose → minimal generalizable fix →
|
|
@@ -58,69 +61,6 @@ const optimizerMethod = [
|
|
|
58
61
|
" what you would try next if the measured lift comes back null."
|
|
59
62
|
].join("\n");
|
|
60
63
|
/**
|
|
61
|
-
* The driver's stance for `driverLoopGenerator` — the build-domain instance of
|
|
62
|
-
* the supervisor doctrine (observe → rate → decide; refine / re-scope /
|
|
63
|
-
* decompose; the check decides delivery, never the driver's prose).
|
|
64
|
-
*/
|
|
65
|
-
const buildDriverSystem = [
|
|
66
|
-
"You are the DRIVER of a build loop: a senior engineering lead steering a coding WORKER inside",
|
|
67
|
-
"an isolated git worktree toward a verified artifact. You never edit files yourself — your only",
|
|
68
|
-
"levers are the tools below. Your intelligence goes into three places: how you AUTHOR each",
|
|
69
|
-
"worker instruction, how you OBSERVE and RATE what a session actually produced, and what you",
|
|
70
|
-
"DECIDE next.",
|
|
71
|
-
"",
|
|
72
|
-
"TOOLS",
|
|
73
|
-
"- run_worker{instruction}: one full coding-harness session in the worktree, with your",
|
|
74
|
-
" instruction as its entire goal. Sessions are expensive and capped — author each one well.",
|
|
75
|
-
" The worktree PERSISTS between sessions: a later worker resumes on top of earlier edits.",
|
|
76
|
-
"- inspect_worktree{}: current git status + diff — what has actually changed so far.",
|
|
77
|
-
"- read_file{path,maxBytes?}: read one file from the worktree (new untracked files do not show",
|
|
78
|
-
" in the diff — read them to rate the work).",
|
|
79
|
-
"- run_verifier{}: the intrinsic check of the surface (compile+tests for a tool, boot-and-probe",
|
|
80
|
-
" for an MCP server). Its exit decides what counts as delivered — your opinion does not.",
|
|
81
|
-
"",
|
|
82
|
-
"AUTHOR RICHLY. A worker handed a one-line label will flail. Each instruction must carry: the",
|
|
83
|
-
"outcome in concrete terms; the hypothesis and sub-goal it serves; what already exists in the",
|
|
84
|
-
"tree that it must build on, not duplicate; where the deliverable must land; and the exact check",
|
|
85
|
-
"it will be held to. The worker sees NOTHING you were given (no findings, no method, no prior",
|
|
86
|
-
"session context) unless you fold it into the instruction.",
|
|
87
|
-
"",
|
|
88
|
-
"THE LOOP — every turn: observe, rate, decide.",
|
|
89
|
-
"- OBSERVE: after each session, inspect the worktree and run the verifier. Read WHAT failed and",
|
|
90
|
-
" WHY — \"it failed\" alone tells you nothing.",
|
|
91
|
-
"- RATE: judge with a reason. Verified and complete → stop. Close, one correctable fault →",
|
|
92
|
-
" REFINE: author \"continue from the current tree; the check fails because X; fix X\" (depth,",
|
|
93
|
-
" not a fresh start — never let a worker revert its own near-miss). Empty tree, or it solved a",
|
|
94
|
-
" different problem → RE-SCOPE: re-author narrower and more concrete; a second identical",
|
|
95
|
-
" instruction fails identically. Too big for one session → DECOMPOSE: author the first",
|
|
96
|
-
" self-contained slice, verify it, then author the next on top.",
|
|
97
|
-
"- DECIDE: exactly one move per turn, with the reason stated in one line.",
|
|
98
|
-
"",
|
|
99
|
-
"STOP when the verifier passes — reply with no tool call and a short reflection: what you",
|
|
100
|
-
"predicted, what the verifier showed, what you would try next if the measured lift comes back",
|
|
101
|
-
"null. If the session budget runs out first, say plainly what remains and why. An honest",
|
|
102
|
-
"no-winner is a real result; a claimed success is not — the final keep/discard decision is made",
|
|
103
|
-
"by code from the verifier exit and the tree state, never from your words."
|
|
104
|
-
].join("\n");
|
|
105
|
-
/**
|
|
106
|
-
* The driver's ADOPT-not-build doctrine, appended to `buildDriverSystem` when
|
|
107
|
-
* a `research` tool is wired into the loop (`DriverLoopGeneratorOptions.
|
|
108
|
-
* research`). Kept separate so a driver WITHOUT the tool is never told to
|
|
109
|
-
* call a tool it does not have.
|
|
110
|
-
*/
|
|
111
|
-
const researchDriverNote = [
|
|
112
|
-
"RESEARCH — ADOPT BEFORE BUILD. A research{query} tool is provisioned for this run. Before",
|
|
113
|
-
"authoring a from-scratch build, spend one turn researching whether an EXISTING external MCP",
|
|
114
|
-
"server already provides the missing capability — registries and vendor docs list maintained",
|
|
115
|
-
"servers for most common gaps (web search, fetch, GitHub, filesystems, databases).",
|
|
116
|
-
"- If a maintained server fits, ADOPT it: report in your final reflection its endpoint or",
|
|
117
|
-
" launch command and the API key it needs BY NAME (e.g. EXA_API_KEY) — never a key value —",
|
|
118
|
-
" so the dispatch can emit a connection candidate with provisioned secrets. Adopting a fit",
|
|
119
|
-
" server beats rebuilding it: less code to verify, maintained upstream, same measured gate.",
|
|
120
|
-
"- If nothing fits (unmaintained, wrong tool surface, heavier than the gap), build — and state",
|
|
121
|
-
" in one line why adoption lost."
|
|
122
|
-
].join("\n");
|
|
123
|
-
/**
|
|
124
64
|
* The senior authoring process for `authorStrategy` — the same method, shaped
|
|
125
65
|
* to the strategy contract (author-blind, conserved budget, one module out).
|
|
126
66
|
*/
|
|
@@ -143,6 +83,285 @@ const strategyAuthorMethod = [
|
|
|
143
83
|
" comment at the top of the module — the holdout verdict will be read against it."
|
|
144
84
|
].join("\n");
|
|
145
85
|
//#endregion
|
|
86
|
+
//#region src/runtime/profile-chat-client.ts
|
|
87
|
+
/** Profile-exact adapter for packages that consume agent-eval's ChatClient contract.
|
|
88
|
+
* Every call still enters Runtime through createExecutor -> streamAgentTurn, and every
|
|
89
|
+
* behavioral field is checked against the exact AgentProfile before any transport runs. */
|
|
90
|
+
function profileChatClient(args) {
|
|
91
|
+
const binding = bindProfileChat(args);
|
|
92
|
+
return {
|
|
93
|
+
transport: "custom",
|
|
94
|
+
defaultModel: binding.model,
|
|
95
|
+
...binding.settings.retry?.maxAttempts !== void 0 ? { maximumAttempts: binding.settings.retry.maxAttempts } : {},
|
|
96
|
+
async chat(req, callOpts) {
|
|
97
|
+
const run = await runBoundProfileChat(binding, req, callOpts);
|
|
98
|
+
if (!run.succeeded) throw new Error(run.error);
|
|
99
|
+
return run.response;
|
|
100
|
+
}
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
/** Profile-exact adapter for agent-eval's external optimizer callback.
|
|
104
|
+
* Eval validates and freezes the provider-neutral request; Runtime owns the exact
|
|
105
|
+
* AgentProfile, execution route, retries, usage, and finite execution evidence. */
|
|
106
|
+
function profileOptimizerModelCall(args) {
|
|
107
|
+
const binding = bindProfileChat(args);
|
|
108
|
+
const profileDigest = canonicalAgentProfileDigest(binding.profile);
|
|
109
|
+
return async (request) => {
|
|
110
|
+
const requestDigest = canonicalCandidateDigest({
|
|
111
|
+
callId: request.callId,
|
|
112
|
+
request: request.request,
|
|
113
|
+
endpointFormat: request.endpointFormat ?? null
|
|
114
|
+
});
|
|
115
|
+
let run;
|
|
116
|
+
try {
|
|
117
|
+
run = await runBoundProfileChat(binding, structuredClone(request.request), {
|
|
118
|
+
signal: request.signal,
|
|
119
|
+
idempotencyKey: request.callId,
|
|
120
|
+
correlationId: request.callId
|
|
121
|
+
});
|
|
122
|
+
} catch (error) {
|
|
123
|
+
return {
|
|
124
|
+
succeeded: false,
|
|
125
|
+
error: errorMessage(error),
|
|
126
|
+
receipt: unknownOptimizerReceipt(binding.model),
|
|
127
|
+
execution: {
|
|
128
|
+
kind: "agent-runtime-profile-model-call",
|
|
129
|
+
profileDigest,
|
|
130
|
+
requestDigest,
|
|
131
|
+
callId: request.callId,
|
|
132
|
+
endpointFormat: request.endpointFormat ?? null,
|
|
133
|
+
executed: false,
|
|
134
|
+
error: errorMessage(error)
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
const execution = optimizerExecution(profileDigest, requestDigest, request, run);
|
|
139
|
+
try {
|
|
140
|
+
const receipt = optimizerReceipt(binding.model, run, args.pricing);
|
|
141
|
+
return run.succeeded ? {
|
|
142
|
+
succeeded: true,
|
|
143
|
+
response: {
|
|
144
|
+
...run.response,
|
|
145
|
+
costUsd: optimizerResponseCostUsd(receipt)
|
|
146
|
+
},
|
|
147
|
+
receipt,
|
|
148
|
+
execution
|
|
149
|
+
} : {
|
|
150
|
+
succeeded: false,
|
|
151
|
+
error: run.error,
|
|
152
|
+
receipt,
|
|
153
|
+
execution
|
|
154
|
+
};
|
|
155
|
+
} catch (error) {
|
|
156
|
+
const message = `profile optimizer receipt normalization failed after execution: ${errorMessage(error)}`;
|
|
157
|
+
return {
|
|
158
|
+
succeeded: false,
|
|
159
|
+
error: message,
|
|
160
|
+
receipt: rawOptimizerReceipt(binding.model, run, args.pricing),
|
|
161
|
+
execution: {
|
|
162
|
+
...execution,
|
|
163
|
+
succeeded: false,
|
|
164
|
+
postCallError: message
|
|
165
|
+
}
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
function bindProfileChat(args) {
|
|
171
|
+
const profile = executableAgentProfileSnapshot(args.profile, args.context);
|
|
172
|
+
const model = concreteModelId(profile.model?.default);
|
|
173
|
+
if (!model) throw new Error(`${args.context}: AgentProfile.model.default must be concrete`);
|
|
174
|
+
return {
|
|
175
|
+
profile,
|
|
176
|
+
executor: captureReusableExecutorConfig(args.executor, args.context),
|
|
177
|
+
context: args.context,
|
|
178
|
+
model,
|
|
179
|
+
settings: profileModelExecutionSettings(profile, args.context)
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
async function runBoundProfileChat(binding, req, callOpts) {
|
|
183
|
+
assertProfileChatRequest(req, binding.model, binding.profile.model?.reasoningEffort, binding.settings, binding.context);
|
|
184
|
+
assertSupportedChatCallOptions(callOpts, binding.context);
|
|
185
|
+
const turnProfile = responseProfile(binding.profile, req, binding.context);
|
|
186
|
+
const startedAt = performance.now();
|
|
187
|
+
const turn = await collectAgentTurn(streamAgentTurn({
|
|
188
|
+
kind: "executor",
|
|
189
|
+
profile: turnProfile,
|
|
190
|
+
factory: createExecutor(binding.executor)
|
|
191
|
+
}, { messages: req.messages }, {
|
|
192
|
+
...req.timeoutMs !== void 0 ? { timeoutMs: req.timeoutMs } : {},
|
|
193
|
+
...callOpts?.signal ? { signal: callOpts.signal } : {},
|
|
194
|
+
...callOpts?.idempotencyKey ? { callId: callOpts.idempotencyKey } : {},
|
|
195
|
+
...callOpts?.correlationId ? { correlationId: callOpts.correlationId } : {}
|
|
196
|
+
}));
|
|
197
|
+
if (turn.status !== "completed") return {
|
|
198
|
+
succeeded: false,
|
|
199
|
+
error: `${binding.context} failed: ${turn.error?.message ?? turn.status}`,
|
|
200
|
+
turn
|
|
201
|
+
};
|
|
202
|
+
const observedModel = turn.usage.model;
|
|
203
|
+
if (observedModel === void 0) return {
|
|
204
|
+
succeeded: false,
|
|
205
|
+
error: `${binding.context}: Runtime turn did not report the model actually used; refusing to label the response with the requested model`,
|
|
206
|
+
turn
|
|
207
|
+
};
|
|
208
|
+
if (observedModel !== binding.model) return {
|
|
209
|
+
succeeded: false,
|
|
210
|
+
error: `${binding.context}: Runtime reported model ${JSON.stringify(observedModel)} but AgentProfile requires ${JSON.stringify(binding.model)}`,
|
|
211
|
+
turn
|
|
212
|
+
};
|
|
213
|
+
const resultOut = turn.output;
|
|
214
|
+
const promptTokens = turn.usage.input;
|
|
215
|
+
const completionTokens = turn.usage.output;
|
|
216
|
+
return {
|
|
217
|
+
succeeded: true,
|
|
218
|
+
turn,
|
|
219
|
+
response: {
|
|
220
|
+
content: turn.finalText,
|
|
221
|
+
usage: {
|
|
222
|
+
promptTokens,
|
|
223
|
+
completionTokens,
|
|
224
|
+
totalTokens: promptTokens + completionTokens,
|
|
225
|
+
...turn.usage.tokensKnown === false ? { captured: false } : {},
|
|
226
|
+
...typeof turn.usage.promptCache?.readTokens === "number" ? { cachedPromptTokens: turn.usage.promptCache.readTokens } : {},
|
|
227
|
+
...turn.usage.reasoningTokens !== void 0 ? { reasoningTokens: turn.usage.reasoningTokens } : {}
|
|
228
|
+
},
|
|
229
|
+
costUsd: turn.usage.usdKnown === false || turn.usage.costUsd === void 0 ? null : turn.usage.costUsd,
|
|
230
|
+
model: observedModel,
|
|
231
|
+
durationMs: terminalDurationMs(turn.events, performance.now() - startedAt),
|
|
232
|
+
finishReason: resultOut?.finishReason ?? null,
|
|
233
|
+
contentEmpty: turn.finalText.trim().length === 0,
|
|
234
|
+
raw: {
|
|
235
|
+
...turn.usage.estimatedCostUsd !== void 0 ? { estimatedCostUsd: turn.usage.estimatedCostUsd } : {},
|
|
236
|
+
...turn.usage.promptCache ? { promptCache: turn.usage.promptCache } : {},
|
|
237
|
+
...turn.transportAttempts !== void 0 ? { transportAttempts: turn.transportAttempts } : {}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
/** @internal Exported for the adapter's fail-honest duration contract test. */
|
|
243
|
+
function terminalDurationMs(events, measuredWallMs) {
|
|
244
|
+
const final = events.at(-1)?.metadata?.timing;
|
|
245
|
+
if (final && typeof final === "object") {
|
|
246
|
+
const duration = final.durationMs;
|
|
247
|
+
if (typeof duration === "number" && Number.isFinite(duration) && duration >= 0) return duration;
|
|
248
|
+
}
|
|
249
|
+
if (Number.isFinite(measuredWallMs) && measuredWallMs >= 0) return measuredWallMs;
|
|
250
|
+
throw new Error("profileChatClient: measured wall duration must be a finite non-negative number");
|
|
251
|
+
}
|
|
252
|
+
function assertSupportedChatCallOptions(opts, context) {
|
|
253
|
+
if (opts?.maxCostUsd !== void 0) throw new Error(`${context}: maxCostUsd is not enforced by Runtime's exact turn path; refusing to treat it as a limit`);
|
|
254
|
+
}
|
|
255
|
+
function responseProfile(profile, req, context) {
|
|
256
|
+
const responseFormat = req.jsonSchema ? {
|
|
257
|
+
type: "json_schema",
|
|
258
|
+
json_schema: req.jsonSchema
|
|
259
|
+
} : req.jsonMode ? { type: "json_object" } : void 0;
|
|
260
|
+
if (!responseFormat) return profile;
|
|
261
|
+
const existing = profile.model?.metadata?.extraBody;
|
|
262
|
+
if (existing !== void 0 && (typeof existing !== "object" || existing === null || Array.isArray(existing))) throw new Error(`${context}: AgentProfile.model.metadata.extraBody must be an object`);
|
|
263
|
+
const existingFormat = existing?.response_format;
|
|
264
|
+
if (existingFormat !== void 0 && JSON.stringify(existingFormat) !== JSON.stringify(responseFormat)) throw new Error(`${context}: requested response format conflicts with AgentProfile`);
|
|
265
|
+
return agentProfileSchema.parse({
|
|
266
|
+
...profile,
|
|
267
|
+
model: {
|
|
268
|
+
...profile.model,
|
|
269
|
+
metadata: {
|
|
270
|
+
...profile.model?.metadata ?? {},
|
|
271
|
+
extraBody: {
|
|
272
|
+
...existing,
|
|
273
|
+
response_format: responseFormat
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
function assertProfileChatRequest(req, model, reasoningEffort, settings, context) {
|
|
280
|
+
if (req.model !== void 0 && req.model !== model) throw new Error(`${context}: request model ${JSON.stringify(req.model)} conflicts with AgentProfile model ${JSON.stringify(model)}`);
|
|
281
|
+
if (req.temperature !== void 0 && req.temperature !== settings.temperature) throw new Error(`${context}: request temperature conflicts with AgentProfile model metadata`);
|
|
282
|
+
if (req.maxTokens !== void 0 && req.maxTokens !== settings.maxTokens) throw new Error(`${context}: request maxTokens conflicts with AgentProfile model metadata`);
|
|
283
|
+
if (req.thinking !== void 0) {
|
|
284
|
+
const expected = reasoningEffort === void 0 ? void 0 : reasoningEffort === "none" ? "disabled" : "enabled";
|
|
285
|
+
if (req.thinking !== expected) throw new Error(`${context}: request thinking conflicts with AgentProfile.model.reasoningEffort`);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
function optimizerReceipt(model, run, pricing) {
|
|
289
|
+
const usage = run.turn.usage;
|
|
290
|
+
const actualCostUsd = usage.usdKnown === false ? void 0 : usage.costUsd;
|
|
291
|
+
if (usage.tokensKnown === false) return {
|
|
292
|
+
model,
|
|
293
|
+
inputTokens: 0,
|
|
294
|
+
outputTokens: 0,
|
|
295
|
+
usageUnknown: true,
|
|
296
|
+
...actualCostUsd !== void 0 ? { actualCostUsd } : usage.estimatedCostUsd !== void 0 ? { estimatedCostUsd: usage.estimatedCostUsd } : { costUnknown: true }
|
|
297
|
+
};
|
|
298
|
+
const cachedTokens = optimizerTokenCount(usage.promptCache?.readTokens, "cache read tokens");
|
|
299
|
+
const cacheWriteTokens = optimizerTokenCount(usage.promptCache?.writeTokens, "cache write tokens");
|
|
300
|
+
const classified = (cachedTokens ?? 0) + (cacheWriteTokens ?? 0);
|
|
301
|
+
if (classified > usage.input) throw new Error("profile optimizer cache classes exceed total input tokens");
|
|
302
|
+
return {
|
|
303
|
+
model,
|
|
304
|
+
inputTokens: usage.input - classified,
|
|
305
|
+
outputTokens: usage.output,
|
|
306
|
+
...cachedTokens !== void 0 ? { cachedTokens } : {},
|
|
307
|
+
...cacheWriteTokens !== void 0 ? { cacheWriteTokens } : {},
|
|
308
|
+
...usage.reasoningTokens !== void 0 ? { reasoningTokens: usage.reasoningTokens } : {},
|
|
309
|
+
...actualCostUsd !== void 0 ? { actualCostUsd } : usage.estimatedCostUsd !== void 0 ? { estimatedCostUsd: usage.estimatedCostUsd } : pricing ? { customTokenPricing: pricing } : { costUnknown: true }
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
/** Preserve the observed call totals when finer receipt classification is inconsistent. */
|
|
313
|
+
function rawOptimizerReceipt(model, run, pricing) {
|
|
314
|
+
const usage = run.turn.usage;
|
|
315
|
+
const tokensKnown = usage.tokensKnown !== false;
|
|
316
|
+
const actualCostUsd = usage.usdKnown === false ? void 0 : usage.costUsd;
|
|
317
|
+
return {
|
|
318
|
+
model,
|
|
319
|
+
inputTokens: tokensKnown ? usage.input : 0,
|
|
320
|
+
outputTokens: tokensKnown ? usage.output : 0,
|
|
321
|
+
...tokensKnown ? {} : { usageUnknown: true },
|
|
322
|
+
...usage.reasoningTokens !== void 0 ? { reasoningTokens: usage.reasoningTokens } : {},
|
|
323
|
+
...actualCostUsd !== void 0 ? { actualCostUsd } : usage.estimatedCostUsd !== void 0 ? { estimatedCostUsd: usage.estimatedCostUsd } : pricing ? { customTokenPricing: pricing } : { costUnknown: true }
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
function optimizerExecution(profileDigest, requestDigest, request, run) {
|
|
327
|
+
return {
|
|
328
|
+
kind: "agent-runtime-profile-model-call",
|
|
329
|
+
profileDigest,
|
|
330
|
+
requestDigest,
|
|
331
|
+
callId: request.callId,
|
|
332
|
+
endpointFormat: request.endpointFormat ?? null,
|
|
333
|
+
executed: true,
|
|
334
|
+
succeeded: run.succeeded,
|
|
335
|
+
status: run.turn.status,
|
|
336
|
+
model: run.turn.usage.model ?? null,
|
|
337
|
+
transportAttempts: run.turn.transportAttempts ?? null,
|
|
338
|
+
eventTypes: run.turn.events.map((event) => event.type)
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
function unknownOptimizerReceipt(model) {
|
|
342
|
+
return {
|
|
343
|
+
model,
|
|
344
|
+
inputTokens: 0,
|
|
345
|
+
outputTokens: 0,
|
|
346
|
+
usageUnknown: true,
|
|
347
|
+
costUnknown: true
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
function optimizerResponseCostUsd(receipt) {
|
|
351
|
+
if (receipt.actualCostUsd !== void 0) return receipt.actualCostUsd;
|
|
352
|
+
if (receipt.estimatedCostUsd !== void 0) return receipt.estimatedCostUsd;
|
|
353
|
+
if (receipt.customTokenPricing !== void 0 && receipt.usageUnknown !== true) return costForTokenPricing(receipt.customTokenPricing, receipt);
|
|
354
|
+
return null;
|
|
355
|
+
}
|
|
356
|
+
function optimizerTokenCount(value, label) {
|
|
357
|
+
if (value === void 0) return void 0;
|
|
358
|
+
if (!Number.isSafeInteger(value) || value < 0) throw new Error(`profile optimizer ${label} must be a non-negative integer`);
|
|
359
|
+
return value;
|
|
360
|
+
}
|
|
361
|
+
function errorMessage(error) {
|
|
362
|
+
return error instanceof Error ? error.message : String(error);
|
|
363
|
+
}
|
|
364
|
+
//#endregion
|
|
146
365
|
//#region src/runtime/observe.ts
|
|
147
366
|
/**
|
|
148
367
|
* The third-person observer — the connective tissue that closes the loop.
|
|
@@ -241,17 +460,18 @@ const findingsSchema = {
|
|
|
241
460
|
/** The third-person trace analyst: read a worker's trace and produce steer findings for the next attempt plus durable `learned` facts for the cross-run corpus. */
|
|
242
461
|
async function observe(input, opts) {
|
|
243
462
|
const traceSummary = summarizeTrace(input.trace, opts.maxTraceLines ?? 80);
|
|
244
|
-
const
|
|
245
|
-
|
|
463
|
+
const res = await profileChatClient({
|
|
464
|
+
profile: opts.profile,
|
|
465
|
+
executor: opts.executor,
|
|
466
|
+
context: "observe analyst"
|
|
467
|
+
}).chat({
|
|
246
468
|
jsonSchema: findingsSchema,
|
|
247
469
|
messages: [{
|
|
248
|
-
role: "system",
|
|
249
|
-
content: opts.analystInstruction ?? "You are a third-person OBSERVER watching an AI agent work. You see its TRACE (what it did), not its grader. From the trace, name SPECIFIC, behavior-grounded findings: wasted/duplicated tool calls, thrash/retries, token/cost waste, missing verification, failure patterns. For each, a concrete recommended_action, and whether the AGENT (fix its skills/prompt/tools) or the OPERATOR (fix framing/decomposition/config) should act. Only claim what the trace shows. No findings if the run was clean."
|
|
250
|
-
}, {
|
|
251
470
|
role: "user",
|
|
252
471
|
content: `TASK: ${input.task}\n\nOUTCOME: ${input.outcome ?? "unknown"}\n\nFINAL OUTPUT (truncated):\n${input.output.slice(0, 1200)}\n\nTRACE (in order; "xN" = repeated):\n${traceSummary}`
|
|
253
472
|
}]
|
|
254
|
-
}, { ...opts.signal ? { signal: opts.signal } : {} })
|
|
473
|
+
}, { ...opts.signal ? { signal: opts.signal } : {} });
|
|
474
|
+
const parsed = parseFindings(res.content);
|
|
255
475
|
const producedAt = input.runId ? `${input.runId}` : observerId;
|
|
256
476
|
const findings = assertProposalFindings(parsed.map((f) => makeProposalFinding({
|
|
257
477
|
analyst_id: observerId,
|
|
@@ -285,10 +505,18 @@ async function observe(input, opts) {
|
|
|
285
505
|
};
|
|
286
506
|
if ((await opts.corpus.append(record)).succeeded) learned.push(record);
|
|
287
507
|
}
|
|
508
|
+
const usage = res.usage;
|
|
509
|
+
const inputTokens = usage?.promptTokens;
|
|
510
|
+
const outputTokens = usage?.completionTokens;
|
|
288
511
|
return {
|
|
289
512
|
findings: [...findings],
|
|
290
513
|
learned,
|
|
291
|
-
report: renderReport(findings)
|
|
514
|
+
report: renderReport(findings),
|
|
515
|
+
usage: {
|
|
516
|
+
input: inputTokens ?? 0,
|
|
517
|
+
output: outputTokens ?? 0,
|
|
518
|
+
known: usage?.captured !== false && typeof inputTokens === "number" && typeof outputTokens === "number"
|
|
519
|
+
}
|
|
292
520
|
};
|
|
293
521
|
}
|
|
294
522
|
function parseFindings(content) {
|
|
@@ -314,32 +542,35 @@ function renderReport(findings) {
|
|
|
314
542
|
}
|
|
315
543
|
//#endregion
|
|
316
544
|
//#region src/runtime/strategy.ts
|
|
317
|
-
/**
|
|
318
|
-
* The general agentic primitive — sequential (depth) and parallel (breadth) over a shared,
|
|
319
|
-
* checkable artifact, driven through the keystone Supervisor as one recursive `Agent.act`.
|
|
320
|
-
*
|
|
321
|
-
* The domain lives behind ONE seam — `AgenticSurface` (open an artifact, list tools, call a tool,
|
|
322
|
-
* score the artifact, close it). EnterpriseOps implements it (seed a gym DB, MCP tools, SQL
|
|
323
|
-
* verifier); Commit0/AppWorld/terminal-bench implement it the same way (a repo workspace, shell
|
|
324
|
-
* tools, the test suite). The drivers below are domain-blind: they run over any surface.
|
|
325
|
-
*
|
|
326
|
-
* Two shapes, the agent's POMDP rollout as the unit:
|
|
327
|
-
* - DEPTH one persistent artifact carried across shots. Each shot the agent works the tool loop;
|
|
328
|
-
* between shots a trace-analyst (selector≠judge: reads the trajectory, never the score)
|
|
329
|
-
* steers the resumed session toward what's unfinished. shot n stands on shot n-1's
|
|
330
|
-
* artifact state + history. This is continuation — long-horizon, same artifact.
|
|
331
|
-
* - BREADTH K independent artifacts, each a fresh rollout, the deployable verifier picks the best.
|
|
332
|
-
*
|
|
333
|
-
* Both are an `Agent` whose `act` spawns leaf shots through `scope.spawn` and reacts via
|
|
334
|
-
* `scope.next()` — so the conserved budget pool meters them (equal-k by construction), the journal
|
|
335
|
-
* records the tree, and the same primitive nests. `runAgentic` runs the chosen driver through
|
|
336
|
-
* `createSupervisor().run`. The leaf (one shot over a handle) is resolved per-spawn from a
|
|
337
|
-
* surface-closed registry — the open `Executor` seam, not bespoke per-benchmark glue.
|
|
338
|
-
*/
|
|
339
545
|
const taskNudge = "Use the available tools to bring the artifact to the required final state. Address EVERY distinct change the request implies. After each tool result, check what remains and continue. Re-read the values you set to confirm they took. Reply DONE only once every required change is made and verified.";
|
|
546
|
+
function exactAgenticProfile(profile, context) {
|
|
547
|
+
const parsed = agentProfileSchema.safeParse(profile);
|
|
548
|
+
if (!parsed.success) throw new Error(`${context}: invalid AgentProfile: ${parsed.error.message}`);
|
|
549
|
+
return parsed.data;
|
|
550
|
+
}
|
|
551
|
+
function requiredProfileModel(profile, context) {
|
|
552
|
+
assertExecutableAgentProfile(profile, context);
|
|
553
|
+
const model = concreteModelId(profile.model?.default);
|
|
554
|
+
if (!model) throw new Error(`${context}: AgentProfile.model.default must name the exact provider model; runtime-selected and missing models are not executable`);
|
|
555
|
+
return model;
|
|
556
|
+
}
|
|
557
|
+
function profileSystemPrompt(profile) {
|
|
558
|
+
const sections = [profile.prompt?.systemPrompt, ...profile.prompt?.instructions ?? []].filter((value) => typeof value === "string" && value.trim().length > 0);
|
|
559
|
+
const instructions = profile.resources?.instructions;
|
|
560
|
+
if (typeof instructions === "string" && instructions.trim()) sections.push(instructions);
|
|
561
|
+
else if (instructions && typeof instructions === "object" && instructions.kind === "inline" && instructions.content.trim()) sections.push(instructions.content);
|
|
562
|
+
else if (instructions && typeof instructions === "object" && instructions.kind === "github") throw new Error("agentic profile: github resource instructions require a workspace materializer; use inline instructions for the direct Router worker");
|
|
563
|
+
return sections.join("\n\n");
|
|
564
|
+
}
|
|
565
|
+
function assertProfileTools(profile, tools, context) {
|
|
566
|
+
const supplied = new Set(tools.map((tool) => tool.function.name));
|
|
567
|
+
const declared = profile.tools ?? {};
|
|
568
|
+
for (const name of supplied) if (declared[name] !== true) throw new Error(`${context}: tool ${JSON.stringify(name)} is not enabled by AgentProfile.tools`);
|
|
569
|
+
for (const [name, enabled] of Object.entries(declared)) if (enabled && !supplied.has(name)) throw new Error(`${context}: AgentProfile enables tool ${JSON.stringify(name)} but the surface did not supply it`);
|
|
570
|
+
}
|
|
340
571
|
/** One shot: run the agent's tool loop (≤ innerTurns) over the handle, mutating the artifact via
|
|
341
572
|
* `surface.call`, carrying `messages`. Returns the updated conversation + counts. */
|
|
342
|
-
async function runShot(surface, _task, handle, tools, messages, opts,
|
|
573
|
+
async function runShot(surface, _task, handle, tools, messages, opts, profileOverride) {
|
|
343
574
|
let toolErrors = 0;
|
|
344
575
|
const execute = async (name, args) => {
|
|
345
576
|
try {
|
|
@@ -351,23 +582,34 @@ async function runShot(surface, _task, handle, tools, messages, opts, modelOverr
|
|
|
351
582
|
return `ERROR: ${e instanceof Error ? e.message : String(e)}`;
|
|
352
583
|
}
|
|
353
584
|
};
|
|
354
|
-
const
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
585
|
+
const profile = exactAgenticProfile(profileOverride ?? opts.workerProfile, "agentic shot");
|
|
586
|
+
requiredProfileModel(profile, "agentic shot");
|
|
587
|
+
profileModelExecutionSettings(profile, "agentic shot");
|
|
588
|
+
assertProfileTools(profile, tools, "agentic shot");
|
|
589
|
+
const turn = await collectAgentTurn(streamAgentTurn({
|
|
590
|
+
kind: "executor",
|
|
591
|
+
profile,
|
|
592
|
+
factory: createExecutor({
|
|
593
|
+
backend: "router-tools",
|
|
594
|
+
routerBaseUrl: opts.routerBaseUrl,
|
|
595
|
+
routerKey: opts.routerKey,
|
|
596
|
+
tools,
|
|
597
|
+
executeToolCall: execute,
|
|
598
|
+
...opts.complete ? { complete: opts.complete } : {}
|
|
599
|
+
})
|
|
600
|
+
}, { messages }));
|
|
601
|
+
if (turn.status !== "completed") throw new Error(`agentic shot failed: ${turn.error?.message ?? turn.status}`);
|
|
602
|
+
const out = turn.output;
|
|
365
603
|
return {
|
|
366
|
-
messages:
|
|
367
|
-
completions:
|
|
368
|
-
toolCalls:
|
|
604
|
+
messages: out?.messages ?? messages,
|
|
605
|
+
completions: out?.turns ?? 0,
|
|
606
|
+
toolCalls: out?.toolCalls ?? 0,
|
|
369
607
|
toolErrors,
|
|
370
|
-
tokens:
|
|
608
|
+
tokens: {
|
|
609
|
+
input: turn.usage.input,
|
|
610
|
+
output: turn.usage.output
|
|
611
|
+
},
|
|
612
|
+
...turn.usage.tokensKnown === false ? { tokensKnown: false } : {}
|
|
371
613
|
};
|
|
372
614
|
}
|
|
373
615
|
/** The firewall's input shape: the trajectory as compacted text — calls, results,
|
|
@@ -384,41 +626,15 @@ function compactTrajectory(messages) {
|
|
|
384
626
|
* worker and the analyst share the one injected responder, exactly as a localhost mock would
|
|
385
627
|
* serve both). The critic speaks the OpenAI request shape; we forward it to `complete` and lift
|
|
386
628
|
* the parsed `/chat/completions` JSON back into a `ChatResponse`. */
|
|
387
|
-
function analystChat(opts,
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
transport: "mock",
|
|
397
|
-
defaultModel,
|
|
398
|
-
handler: async (req) => {
|
|
399
|
-
const raw = await complete({
|
|
400
|
-
model: req.model ?? defaultModel,
|
|
401
|
-
messages: req.messages,
|
|
402
|
-
...req.temperature !== void 0 ? { temperature: req.temperature } : {},
|
|
403
|
-
...req.maxTokens !== void 0 ? { max_tokens: req.maxTokens } : {}
|
|
404
|
-
});
|
|
405
|
-
const content = raw.choices?.[0]?.message?.content ?? "";
|
|
406
|
-
const promptTokens = raw.usage?.prompt_tokens ?? 0;
|
|
407
|
-
const completionTokens = raw.usage?.completion_tokens ?? 0;
|
|
408
|
-
return {
|
|
409
|
-
content,
|
|
410
|
-
usage: {
|
|
411
|
-
promptTokens,
|
|
412
|
-
completionTokens,
|
|
413
|
-
totalTokens: promptTokens + completionTokens
|
|
414
|
-
},
|
|
415
|
-
costUsd: null,
|
|
416
|
-
model: req.model ?? defaultModel,
|
|
417
|
-
durationMs: 0,
|
|
418
|
-
finishReason: raw.choices?.[0]?.finish_reason ?? null,
|
|
419
|
-
contentEmpty: content.trim().length === 0,
|
|
420
|
-
raw
|
|
421
|
-
};
|
|
629
|
+
function analystChat(opts, profile) {
|
|
630
|
+
return profileChatClient({
|
|
631
|
+
profile,
|
|
632
|
+
context: "agentic analyst",
|
|
633
|
+
executor: {
|
|
634
|
+
backend: "router",
|
|
635
|
+
routerBaseUrl: opts.routerBaseUrl,
|
|
636
|
+
routerKey: opts.routerKey,
|
|
637
|
+
...opts.complete ? { complete: opts.complete } : {}
|
|
422
638
|
}
|
|
423
639
|
});
|
|
424
640
|
}
|
|
@@ -429,70 +645,59 @@ function analystChat(opts, defaultModel) {
|
|
|
429
645
|
* trajectory in, never scores. */
|
|
430
646
|
async function consultAnalyst(task, messages, instruction, opts) {
|
|
431
647
|
const trajectory = compactTrajectory(messages);
|
|
432
|
-
const
|
|
433
|
-
const
|
|
434
|
-
const
|
|
435
|
-
|
|
436
|
-
content: instruction
|
|
437
|
-
}, {
|
|
438
|
-
role: "user",
|
|
439
|
-
content: `TASK: ${task.userPrompt.slice(0, 1500)}\n\nTRAJECTORY:\n${trajectory}`
|
|
440
|
-
}] : [{
|
|
648
|
+
const analystProfile = exactAgenticProfile(opts.analystProfile ?? opts.workerProfile, "agentic analyst");
|
|
649
|
+
const analystModel = requiredProfileModel(analystProfile, "agentic analyst");
|
|
650
|
+
const chat = analystChat(opts, analystProfile);
|
|
651
|
+
const consultMessages = [{
|
|
441
652
|
role: "user",
|
|
442
|
-
content: `${instruction}\n\nTASK:\n${task.userPrompt.slice(0, 1500)}`
|
|
653
|
+
content: trajectory ? `${instruction}\n\nTASK: ${task.userPrompt.slice(0, 1500)}\n\nTRAJECTORY:\n${trajectory}` : `${instruction}\n\nTASK:\n${task.userPrompt.slice(0, 1500)}`
|
|
443
654
|
}];
|
|
444
655
|
const res = await chat.chat({
|
|
445
656
|
model: analystModel,
|
|
446
|
-
temperature: .2,
|
|
447
|
-
maxTokens: 1024,
|
|
448
657
|
messages: consultMessages
|
|
449
658
|
});
|
|
450
659
|
const usage = res.usage;
|
|
660
|
+
const input = usage?.promptTokens ?? usage?.prompt_tokens;
|
|
661
|
+
const output = usage?.completionTokens ?? usage?.completion_tokens;
|
|
662
|
+
const tokensKnown = usage?.captured !== false && typeof input === "number" && typeof output === "number";
|
|
451
663
|
return {
|
|
452
664
|
steer: res.content.trim(),
|
|
453
665
|
tokens: {
|
|
454
|
-
input:
|
|
455
|
-
output:
|
|
456
|
-
}
|
|
666
|
+
input: input ?? 0,
|
|
667
|
+
output: output ?? 0
|
|
668
|
+
},
|
|
669
|
+
...tokensKnown ? {} : { tokensKnown: false }
|
|
457
670
|
};
|
|
458
671
|
}
|
|
459
672
|
async function analyze(task, messages, opts) {
|
|
460
673
|
const trajectory = compactTrajectory(messages);
|
|
461
|
-
const
|
|
462
|
-
const
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
674
|
+
const analystProfile = exactAgenticProfile(opts.analystProfile ?? opts.workerProfile, "agentic analyst");
|
|
675
|
+
const obs = await observe({
|
|
676
|
+
task: task.userPrompt,
|
|
677
|
+
output: trajectory,
|
|
678
|
+
trace: messages,
|
|
679
|
+
outcome: "failed",
|
|
680
|
+
runId: task.id
|
|
681
|
+
}, {
|
|
682
|
+
profile: analystProfile,
|
|
683
|
+
executor: {
|
|
684
|
+
backend: "router",
|
|
685
|
+
routerBaseUrl: opts.routerBaseUrl,
|
|
686
|
+
routerKey: opts.routerKey,
|
|
687
|
+
...opts.complete ? { complete: opts.complete } : {}
|
|
688
|
+
},
|
|
689
|
+
...opts.corpus ? {
|
|
690
|
+
corpus: opts.corpus,
|
|
691
|
+
tags: opts.corpusTags ?? []
|
|
692
|
+
} : {}
|
|
693
|
+
});
|
|
479
694
|
return {
|
|
480
|
-
steer: (
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
}, {
|
|
487
|
-
chat,
|
|
488
|
-
model: analystModel,
|
|
489
|
-
...opts.analystInstruction ? { analystInstruction: opts.analystInstruction } : {},
|
|
490
|
-
...opts.corpus ? {
|
|
491
|
-
corpus: opts.corpus,
|
|
492
|
-
tags: opts.corpusTags ?? []
|
|
493
|
-
} : {}
|
|
494
|
-
})).findings.map((f) => f.recommended_action).filter((a) => typeof a === "string" && a.trim().length > 0).join("\n").trim() || "COMPLETE",
|
|
495
|
-
tokens
|
|
695
|
+
steer: obs.findings.map((f) => f.recommended_action).filter((a) => typeof a === "string" && a.trim().length > 0).join("\n").trim() || "COMPLETE",
|
|
696
|
+
tokens: {
|
|
697
|
+
input: obs.usage.input,
|
|
698
|
+
output: obs.usage.output
|
|
699
|
+
},
|
|
700
|
+
...obs.usage.known ? {} : { tokensKnown: false }
|
|
496
701
|
};
|
|
497
702
|
}
|
|
498
703
|
async function renderCorpusReadback(opts) {
|
|
@@ -533,22 +738,24 @@ function shotExecutor(surface, opts) {
|
|
|
533
738
|
const want = new Set(t.tools);
|
|
534
739
|
tools = allTools.filter((tool) => want.has(tool.function.name));
|
|
535
740
|
}
|
|
536
|
-
const
|
|
741
|
+
const profile = exactAgenticProfile(t.profile ?? opts.workerProfile, "agentic shot");
|
|
742
|
+
const systemPrompt = profileSystemPrompt(profile);
|
|
743
|
+
const messages = t.messages?.length ? [...t.messages] : [...systemPrompt ? [{
|
|
537
744
|
role: "system",
|
|
538
|
-
content:
|
|
539
|
-
}, {
|
|
745
|
+
content: systemPrompt
|
|
746
|
+
}] : [], {
|
|
540
747
|
role: "user",
|
|
541
748
|
content: `${t.task.userPrompt}\n\n${taskNudge}`
|
|
542
749
|
}];
|
|
543
|
-
if (t.messages?.length && t.
|
|
750
|
+
if (t.messages?.length && t.profile && systemPrompt) messages.push({
|
|
544
751
|
role: "user",
|
|
545
|
-
content: `[hand-off] You are now acting as: ${
|
|
752
|
+
content: `[hand-off] You are now acting as: ${systemPrompt}`
|
|
546
753
|
});
|
|
547
754
|
if (t.steer) messages.push({
|
|
548
755
|
role: "user",
|
|
549
756
|
content: t.steer
|
|
550
757
|
});
|
|
551
|
-
const shot = await runShot(surface, t.task, handle, tools, messages, opts,
|
|
758
|
+
const shot = await runShot(surface, t.task, handle, tools, messages, opts, profile);
|
|
552
759
|
const s = await surface.score(t.task, handle);
|
|
553
760
|
const score = s.total > 0 ? s.passes / s.total : 0;
|
|
554
761
|
const out = {
|
|
@@ -569,7 +776,9 @@ function shotExecutor(surface, opts) {
|
|
|
569
776
|
spent: {
|
|
570
777
|
iterations: shot.completions,
|
|
571
778
|
tokens: shot.tokens,
|
|
572
|
-
|
|
779
|
+
...shot.tokensKnown === false ? { tokensKnown: false } : {},
|
|
780
|
+
usd: 0,
|
|
781
|
+
usdKnown: false,
|
|
573
782
|
ms: 0
|
|
574
783
|
}
|
|
575
784
|
};
|
|
@@ -591,15 +800,16 @@ function analystExecutor(opts) {
|
|
|
591
800
|
runtime: "agentic-analyst",
|
|
592
801
|
async execute(task) {
|
|
593
802
|
const t = task;
|
|
594
|
-
const { steer, tokens } = t.rawInstruction ? await consultAnalyst(t.task, t.messages, t.rawInstruction, opts) : await analyze(t.task, t.messages, opts);
|
|
595
|
-
const analystModel = opts.analystModel ?? opts.model;
|
|
803
|
+
const { steer, tokens, tokensKnown } = t.rawInstruction ? await consultAnalyst(t.task, t.messages, t.rawInstruction, opts) : await analyze(t.task, t.messages, opts);
|
|
596
804
|
artifact = {
|
|
597
805
|
outRef: `analyst:${steer.length}`,
|
|
598
806
|
out: steer,
|
|
599
807
|
spent: {
|
|
600
808
|
iterations: 1,
|
|
601
809
|
tokens,
|
|
602
|
-
|
|
810
|
+
...tokensKnown === false ? { tokensKnown: false } : {},
|
|
811
|
+
usd: 0,
|
|
812
|
+
usdKnown: false,
|
|
603
813
|
ms: 0
|
|
604
814
|
}
|
|
605
815
|
};
|
|
@@ -634,13 +844,18 @@ function agenticRegistry(surface, opts) {
|
|
|
634
844
|
}
|
|
635
845
|
});
|
|
636
846
|
}
|
|
637
|
-
function leaf(name, role) {
|
|
847
|
+
function leaf(name, role, profile) {
|
|
848
|
+
const exactProfile = exactAgenticProfile(profile, `agentic ${role}`);
|
|
638
849
|
return {
|
|
639
850
|
name,
|
|
640
851
|
executorSpec: {
|
|
641
852
|
profile: {
|
|
853
|
+
...exactProfile,
|
|
642
854
|
name,
|
|
643
|
-
metadata: {
|
|
855
|
+
metadata: {
|
|
856
|
+
...exactProfile.metadata,
|
|
857
|
+
role
|
|
858
|
+
}
|
|
644
859
|
},
|
|
645
860
|
harness: null
|
|
646
861
|
},
|
|
@@ -655,13 +870,17 @@ async function drainOne(scope) {
|
|
|
655
870
|
if (!s) throw new Error("agentic: spawned child never settled");
|
|
656
871
|
return s;
|
|
657
872
|
}
|
|
658
|
-
const
|
|
659
|
-
|
|
873
|
+
const UNBOUNDED_TURN_RESERVATION = 1e9;
|
|
874
|
+
function profileTurnLimit(profile, context) {
|
|
875
|
+
return profileModelExecutionSettings(profile, context).maxTurns ?? 0;
|
|
876
|
+
}
|
|
877
|
+
const perChild = (maxTurns) => ({
|
|
878
|
+
maxIterations: maxTurns === 0 ? UNBOUNDED_TURN_RESERVATION : maxTurns + 1,
|
|
660
879
|
maxTokens: 1e6
|
|
661
880
|
});
|
|
662
881
|
/** DEPTH: one persistent artifact, carried across analyst-steered shots. */
|
|
663
882
|
function depthStrategy(surface, task, opts, cfg) {
|
|
664
|
-
const innerTurns = opts.
|
|
883
|
+
const innerTurns = profileTurnLimit(opts.workerProfile, "depth worker");
|
|
665
884
|
let pendingSteer;
|
|
666
885
|
return {
|
|
667
886
|
name: "depth",
|
|
@@ -673,7 +892,7 @@ function depthStrategy(surface, task, opts, cfg) {
|
|
|
673
892
|
let shots = 0;
|
|
674
893
|
try {
|
|
675
894
|
for (shots = 0; shots < cfg.maxShots; shots += 1) {
|
|
676
|
-
const child = leaf(`shot:${shots}`, "shot");
|
|
895
|
+
const child = leaf(`shot:${shots}`, "shot", opts.workerProfile);
|
|
677
896
|
const memorySteer = await renderCorpusReadback(opts);
|
|
678
897
|
const steer = [shots === 0 ? void 0 : pendingSteer, memorySteer].filter((part) => typeof part === "string" && part.trim().length > 0).join("\n\n");
|
|
679
898
|
if (!scope.spawn(child, {
|
|
@@ -692,7 +911,7 @@ function depthStrategy(surface, task, opts, cfg) {
|
|
|
692
911
|
completions += out.completions;
|
|
693
912
|
progression.push(out.score);
|
|
694
913
|
if (out.score >= 1 || shots === cfg.maxShots - 1) break;
|
|
695
|
-
const aChild = leaf(`analyst:${shots}`, "analyst");
|
|
914
|
+
const aChild = leaf(`analyst:${shots}`, "analyst", opts.analystProfile ?? opts.workerProfile);
|
|
696
915
|
if (!scope.spawn(aChild, {
|
|
697
916
|
task,
|
|
698
917
|
messages
|
|
@@ -727,12 +946,12 @@ function depthStrategy(surface, task, opts, cfg) {
|
|
|
727
946
|
}
|
|
728
947
|
/** BREADTH: K independent rollouts (each own artifact), verifier picks the best. */
|
|
729
948
|
function breadthStrategy(_surface, task, opts, cfg) {
|
|
730
|
-
const innerTurns = opts.
|
|
949
|
+
const innerTurns = profileTurnLimit(opts.workerProfile, "breadth worker");
|
|
731
950
|
return {
|
|
732
951
|
name: "breadth",
|
|
733
952
|
async act(_t, scope) {
|
|
734
953
|
let opened = 0;
|
|
735
|
-
for (let k = 0; k < cfg.width; k += 1) if (scope.spawn(leaf(`rollout:${k}`, "shot"), { task }, {
|
|
954
|
+
for (let k = 0; k < cfg.width; k += 1) if (scope.spawn(leaf(`rollout:${k}`, "shot", opts.workerProfile), { task }, {
|
|
736
955
|
budget: perChild(innerTurns),
|
|
737
956
|
label: `rollout:${k}`
|
|
738
957
|
}).ok) opened += 1;
|
|
@@ -788,7 +1007,6 @@ function defineStrategy(name, run) {
|
|
|
788
1007
|
name,
|
|
789
1008
|
async act(_t, scope) {
|
|
790
1009
|
let seq = 0;
|
|
791
|
-
const innerTurns = opts.innerTurns ?? 4;
|
|
792
1010
|
let verifiedBest = 0;
|
|
793
1011
|
let verifiedResolved = false;
|
|
794
1012
|
const openHandles = /* @__PURE__ */ new Set();
|
|
@@ -811,14 +1029,16 @@ function defineStrategy(name, run) {
|
|
|
811
1029
|
budget,
|
|
812
1030
|
scope,
|
|
813
1031
|
async shot(spec) {
|
|
814
|
-
const
|
|
1032
|
+
const profile = spec?.profile ?? opts.workerProfile;
|
|
1033
|
+
const innerTurns = profileTurnLimit(profile, "authored strategy shot");
|
|
1034
|
+
const child = leaf(`shot:${seq}`, "shot", profile);
|
|
815
1035
|
seq += 1;
|
|
816
1036
|
if (!scope.spawn(child, {
|
|
817
1037
|
task,
|
|
818
1038
|
handle: spec?.handle,
|
|
819
1039
|
messages: spec?.messages,
|
|
820
1040
|
steer: spec?.steer,
|
|
821
|
-
|
|
1041
|
+
profile,
|
|
822
1042
|
tools: spec?.tools
|
|
823
1043
|
}, {
|
|
824
1044
|
budget: perChild(innerTurns),
|
|
@@ -838,7 +1058,7 @@ function defineStrategy(name, run) {
|
|
|
838
1058
|
}));
|
|
839
1059
|
},
|
|
840
1060
|
async critique(messages) {
|
|
841
|
-
const child = leaf(`analyst:${seq}`, "analyst");
|
|
1061
|
+
const child = leaf(`analyst:${seq}`, "analyst", opts.analystProfile ?? opts.workerProfile);
|
|
842
1062
|
seq += 1;
|
|
843
1063
|
if (!scope.spawn(child, {
|
|
844
1064
|
task,
|
|
@@ -853,7 +1073,7 @@ function defineStrategy(name, run) {
|
|
|
853
1073
|
return /^\s*COMPLETE\b/i.test(findings) ? null : findings;
|
|
854
1074
|
},
|
|
855
1075
|
async consult(messages, instruction) {
|
|
856
|
-
const child = leaf(`analyst:${seq}`, "analyst");
|
|
1076
|
+
const child = leaf(`analyst:${seq}`, "analyst", opts.analystProfile ?? opts.workerProfile);
|
|
857
1077
|
seq += 1;
|
|
858
1078
|
if (!scope.spawn(child, {
|
|
859
1079
|
task,
|
|
@@ -1002,11 +1222,21 @@ const sampleThenRefine = defineStrategy("sampleThenRefine", async ({ surface, ta
|
|
|
1002
1222
|
});
|
|
1003
1223
|
/** Run a Strategy through the keystone Supervisor — `Agent.act` over a conserved-budget Scope. */
|
|
1004
1224
|
async function runAgentic(opts) {
|
|
1225
|
+
const workerProfile = exactAgenticProfile(opts.workerProfile, "runAgentic worker");
|
|
1226
|
+
requiredProfileModel(workerProfile, "runAgentic worker");
|
|
1227
|
+
const analystProfile = exactAgenticProfile(opts.analystProfile ?? workerProfile, "runAgentic analyst");
|
|
1228
|
+
requiredProfileModel(analystProfile, "runAgentic analyst");
|
|
1229
|
+
const exactOpts = {
|
|
1230
|
+
...opts,
|
|
1231
|
+
workerProfile,
|
|
1232
|
+
analystProfile
|
|
1233
|
+
};
|
|
1005
1234
|
const strategy = opts.strategy ?? (opts.mode === "breadth" ? sample : refine);
|
|
1006
|
-
const driver = strategy.driver(opts.surface, opts.task,
|
|
1235
|
+
const driver = strategy.driver(opts.surface, opts.task, exactOpts, opts.budget);
|
|
1007
1236
|
const supervisor = createSupervisor();
|
|
1237
|
+
const rootTurnLimit = profileTurnLimit(workerProfile, "runAgentic worker");
|
|
1008
1238
|
const root = opts.rootBudget ?? {
|
|
1009
|
-
maxIterations: opts.budget * ((
|
|
1239
|
+
maxIterations: opts.budget * ((rootTurnLimit === 0 ? UNBOUNDED_TURN_RESERVATION : rootTurnLimit) + 2),
|
|
1010
1240
|
maxTokens: 1e9
|
|
1011
1241
|
};
|
|
1012
1242
|
const started = Date.now();
|
|
@@ -1015,7 +1245,7 @@ async function runAgentic(opts) {
|
|
|
1015
1245
|
runId: `agentic:${strategy.name}:${opts.task.id}`,
|
|
1016
1246
|
journal: new InMemorySpawnJournal(),
|
|
1017
1247
|
blobs: new InMemoryResultBlobStore(),
|
|
1018
|
-
executors: agenticRegistry(opts.surface,
|
|
1248
|
+
executors: agenticRegistry(opts.surface, exactOpts),
|
|
1019
1249
|
maxDepth: 3,
|
|
1020
1250
|
...opts.hooks ? { hooks: opts.hooks } : {}
|
|
1021
1251
|
});
|
|
@@ -1026,7 +1256,9 @@ async function runAgentic(opts) {
|
|
|
1026
1256
|
return {
|
|
1027
1257
|
...result.out.deliverable,
|
|
1028
1258
|
usd: result.spentTotal.usd,
|
|
1259
|
+
usdKnown: result.spentTotal.usdKnown !== false,
|
|
1029
1260
|
tokens: result.spentTotal.tokens,
|
|
1261
|
+
tokensKnown: result.spentTotal.tokensKnown !== false,
|
|
1030
1262
|
ms: Date.now() - started
|
|
1031
1263
|
};
|
|
1032
1264
|
}
|
|
@@ -1315,7 +1547,7 @@ function structuralRollout(config = {}) {
|
|
|
1315
1547
|
const checkSource = config.checkSource ?? composeCheckSources(officialChecksFromMeta(), modelAuthoredChecks());
|
|
1316
1548
|
const checkRunner = config.checkRunner ?? sandboxCheckRunner();
|
|
1317
1549
|
const extract = config.extractCandidate ?? defaultExtractCandidate;
|
|
1318
|
-
|
|
1550
|
+
return defineStrategy("structuralRollout", async (ctx) => {
|
|
1319
1551
|
const { task, shot } = ctx;
|
|
1320
1552
|
const progression = [];
|
|
1321
1553
|
const receipts = [];
|
|
@@ -1431,16 +1663,8 @@ function structuralRollout(config = {}) {
|
|
|
1431
1663
|
authoredChecks
|
|
1432
1664
|
};
|
|
1433
1665
|
});
|
|
1434
|
-
if (policy.temperature === void 0) return inner;
|
|
1435
|
-
return {
|
|
1436
|
-
name: inner.name,
|
|
1437
|
-
driver: (surface, task, opts, budget) => inner.driver(surface, task, {
|
|
1438
|
-
...opts,
|
|
1439
|
-
temperature: policy.temperature
|
|
1440
|
-
}, budget)
|
|
1441
|
-
};
|
|
1442
1666
|
}
|
|
1443
1667
|
//#endregion
|
|
1444
|
-
export { observe as C,
|
|
1668
|
+
export { observe as C, optimizerMethod as D, profileOptimizerModelCall as E, strategyAuthorMethod as O, defaultAnalystInstruction as S, profileChatClient as T, depthStrategy as _, defaultStructuralRolloutPolicy as a, sample as b, officialChecksFromMeta as c, selectBestIndex as d, structuralRollout as f, defineStrategy as g, breadthStrategy as h, defaultExtractCandidate as i, resolveEntrySymbol as l, adaptiveRefine as m, compareCheckOutcomes as n, filterAuthoredAsserts as o, visibleCheckScore as p, composeCheckSources as r, modelAuthoredChecks as s, canDisplace as t, sandboxCheckRunner as u, refine as v, renderReport as w, sampleThenRefine as x, runAgentic as y };
|
|
1445
1669
|
|
|
1446
|
-
//# sourceMappingURL=structural-rollout-
|
|
1670
|
+
//# sourceMappingURL=structural-rollout-3uxVGcg2.js.map
|