@providerkit/agents 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. package/LICENSE-APACHE-2.0 +202 -0
  2. package/README.md +2108 -52
  3. package/THIRD_PARTY_NOTICES.md +521 -0
  4. package/dist/a2a/auth.d.ts +10 -0
  5. package/dist/a2a/auth.d.ts.map +1 -0
  6. package/dist/a2a/auth.js +52 -0
  7. package/dist/a2a/auth.js.map +1 -0
  8. package/dist/a2a/client.d.ts +74 -0
  9. package/dist/a2a/client.d.ts.map +1 -0
  10. package/dist/a2a/client.js +601 -0
  11. package/dist/a2a/client.js.map +1 -0
  12. package/dist/a2a/index.d.ts +22 -0
  13. package/dist/a2a/index.d.ts.map +1 -0
  14. package/dist/a2a/index.js +11 -0
  15. package/dist/a2a/index.js.map +1 -0
  16. package/dist/a2a/memory-runs.d.ts +62 -0
  17. package/dist/a2a/memory-runs.d.ts.map +1 -0
  18. package/dist/a2a/memory-runs.js +349 -0
  19. package/dist/a2a/memory-runs.js.map +1 -0
  20. package/dist/a2a/policy.d.ts +15 -0
  21. package/dist/a2a/policy.d.ts.map +1 -0
  22. package/dist/a2a/policy.js +114 -0
  23. package/dist/a2a/policy.js.map +1 -0
  24. package/dist/a2a/push.d.ts +34 -0
  25. package/dist/a2a/push.d.ts.map +1 -0
  26. package/dist/a2a/push.js +180 -0
  27. package/dist/a2a/push.js.map +1 -0
  28. package/dist/a2a/remote.d.ts +155 -0
  29. package/dist/a2a/remote.d.ts.map +1 -0
  30. package/dist/a2a/remote.js +1196 -0
  31. package/dist/a2a/remote.js.map +1 -0
  32. package/dist/a2a/server.d.ts +110 -0
  33. package/dist/a2a/server.d.ts.map +1 -0
  34. package/dist/a2a/server.js +1139 -0
  35. package/dist/a2a/server.js.map +1 -0
  36. package/dist/a2a/signing.d.ts +21 -0
  37. package/dist/a2a/signing.d.ts.map +1 -0
  38. package/dist/a2a/signing.js +305 -0
  39. package/dist/a2a/signing.js.map +1 -0
  40. package/dist/a2a/tasks.d.ts +88 -0
  41. package/dist/a2a/tasks.d.ts.map +1 -0
  42. package/dist/a2a/tasks.js +170 -0
  43. package/dist/a2a/tasks.js.map +1 -0
  44. package/dist/a2a/wire.d.ts +267 -0
  45. package/dist/a2a/wire.d.ts.map +1 -0
  46. package/dist/a2a/wire.js +1202 -0
  47. package/dist/a2a/wire.js.map +1 -0
  48. package/dist/background-types.d.ts +76 -0
  49. package/dist/background-types.d.ts.map +1 -0
  50. package/dist/background-types.js +2 -0
  51. package/dist/background-types.js.map +1 -0
  52. package/dist/child.d.ts +25 -0
  53. package/dist/child.d.ts.map +1 -0
  54. package/dist/child.js +129 -0
  55. package/dist/child.js.map +1 -0
  56. package/dist/code-driver.d.ts +40 -0
  57. package/dist/code-driver.d.ts.map +1 -0
  58. package/dist/code-driver.js +2 -0
  59. package/dist/code-driver.js.map +1 -0
  60. package/dist/code-mode.d.ts +3 -0
  61. package/dist/code-mode.d.ts.map +1 -0
  62. package/dist/code-mode.js +2 -0
  63. package/dist/code-mode.js.map +1 -0
  64. package/dist/code-script.d.ts +3 -0
  65. package/dist/code-script.d.ts.map +1 -0
  66. package/dist/code-script.js +149 -0
  67. package/dist/code-script.js.map +1 -0
  68. package/dist/code.d.ts +75 -0
  69. package/dist/code.d.ts.map +1 -0
  70. package/dist/code.js +326 -0
  71. package/dist/code.js.map +1 -0
  72. package/dist/compaction.d.ts +15 -0
  73. package/dist/compaction.d.ts.map +1 -0
  74. package/dist/compaction.js +168 -0
  75. package/dist/compaction.js.map +1 -0
  76. package/dist/delegation.d.ts +85 -16
  77. package/dist/delegation.d.ts.map +1 -1
  78. package/dist/delegation.js +240 -106
  79. package/dist/delegation.js.map +1 -1
  80. package/dist/evals/driver.d.ts +10 -0
  81. package/dist/evals/driver.d.ts.map +1 -0
  82. package/dist/evals/driver.js +192 -0
  83. package/dist/evals/driver.js.map +1 -0
  84. package/dist/evals/journal.d.ts +26 -0
  85. package/dist/evals/journal.d.ts.map +1 -0
  86. package/dist/evals/journal.js +483 -0
  87. package/dist/evals/journal.js.map +1 -0
  88. package/dist/evals/judge.d.ts +31 -0
  89. package/dist/evals/judge.d.ts.map +1 -0
  90. package/dist/evals/judge.js +142 -0
  91. package/dist/evals/judge.js.map +1 -0
  92. package/dist/evals/report.d.ts +38 -0
  93. package/dist/evals/report.d.ts.map +1 -0
  94. package/dist/evals/report.js +241 -0
  95. package/dist/evals/report.js.map +1 -0
  96. package/dist/evals/runner.d.ts +24 -0
  97. package/dist/evals/runner.d.ts.map +1 -0
  98. package/dist/evals/runner.js +655 -0
  99. package/dist/evals/runner.js.map +1 -0
  100. package/dist/evals/scorers.d.ts +36 -0
  101. package/dist/evals/scorers.d.ts.map +1 -0
  102. package/dist/evals/scorers.js +259 -0
  103. package/dist/evals/scorers.js.map +1 -0
  104. package/dist/evals/simulation.d.ts +7 -0
  105. package/dist/evals/simulation.d.ts.map +1 -0
  106. package/dist/evals/simulation.js +38 -0
  107. package/dist/evals/simulation.js.map +1 -0
  108. package/dist/evals/testing.d.ts +36 -0
  109. package/dist/evals/testing.d.ts.map +1 -0
  110. package/dist/evals/testing.js +87 -0
  111. package/dist/evals/testing.js.map +1 -0
  112. package/dist/evals/trajectory.d.ts +11 -0
  113. package/dist/evals/trajectory.d.ts.map +1 -0
  114. package/dist/evals/trajectory.js +180 -0
  115. package/dist/evals/trajectory.js.map +1 -0
  116. package/dist/evals/types.d.ts +431 -0
  117. package/dist/evals/types.d.ts.map +1 -0
  118. package/dist/evals/types.js +4 -0
  119. package/dist/evals/types.js.map +1 -0
  120. package/dist/evals.d.ts +14 -0
  121. package/dist/evals.d.ts.map +1 -0
  122. package/dist/evals.js +9 -0
  123. package/dist/evals.js.map +1 -0
  124. package/dist/extensions.d.ts +33 -0
  125. package/dist/extensions.d.ts.map +1 -0
  126. package/dist/extensions.js +188 -0
  127. package/dist/extensions.js.map +1 -0
  128. package/dist/handoff.d.ts +58 -0
  129. package/dist/handoff.d.ts.map +1 -0
  130. package/dist/handoff.js +160 -0
  131. package/dist/handoff.js.map +1 -0
  132. package/dist/index.d.ts +14 -0
  133. package/dist/index.d.ts.map +1 -1
  134. package/dist/index.js +8 -0
  135. package/dist/index.js.map +1 -1
  136. package/dist/input.d.ts +42 -0
  137. package/dist/input.d.ts.map +1 -0
  138. package/dist/input.js +266 -0
  139. package/dist/input.js.map +1 -0
  140. package/dist/mcp.d.ts +77 -0
  141. package/dist/mcp.d.ts.map +1 -0
  142. package/dist/mcp.js +505 -0
  143. package/dist/mcp.js.map +1 -0
  144. package/dist/memory-store.d.ts +11 -0
  145. package/dist/memory-store.d.ts.map +1 -0
  146. package/dist/memory-store.js +272 -0
  147. package/dist/memory-store.js.map +1 -0
  148. package/dist/memory.d.ts +212 -0
  149. package/dist/memory.d.ts.map +1 -0
  150. package/dist/memory.js +613 -0
  151. package/dist/memory.js.map +1 -0
  152. package/dist/otel.d.ts +74 -0
  153. package/dist/otel.d.ts.map +1 -0
  154. package/dist/otel.js +427 -0
  155. package/dist/otel.js.map +1 -0
  156. package/dist/progress.d.ts +17 -0
  157. package/dist/progress.d.ts.map +1 -0
  158. package/dist/progress.js +193 -0
  159. package/dist/progress.js.map +1 -0
  160. package/dist/quickjs.d.ts +19 -0
  161. package/dist/quickjs.d.ts.map +1 -0
  162. package/dist/quickjs.js +519 -0
  163. package/dist/quickjs.js.map +1 -0
  164. package/dist/recovery.d.ts +10 -0
  165. package/dist/recovery.d.ts.map +1 -0
  166. package/dist/recovery.js +62 -0
  167. package/dist/recovery.js.map +1 -0
  168. package/dist/request.d.ts +88 -0
  169. package/dist/request.d.ts.map +1 -0
  170. package/dist/request.js +422 -0
  171. package/dist/request.js.map +1 -0
  172. package/dist/run.d.ts +16 -4
  173. package/dist/run.d.ts.map +1 -1
  174. package/dist/run.js +2016 -245
  175. package/dist/run.js.map +1 -1
  176. package/dist/runs.d.ts +77 -34
  177. package/dist/runs.d.ts.map +1 -1
  178. package/dist/runs.js +267 -85
  179. package/dist/runs.js.map +1 -1
  180. package/dist/schema-to-ts.d.ts +4 -0
  181. package/dist/schema-to-ts.d.ts.map +1 -0
  182. package/dist/schema-to-ts.js +191 -0
  183. package/dist/schema-to-ts.js.map +1 -0
  184. package/dist/session.d.ts +23 -0
  185. package/dist/session.d.ts.map +1 -0
  186. package/dist/session.js +137 -0
  187. package/dist/session.js.map +1 -0
  188. package/dist/snapshot.d.ts +57 -8
  189. package/dist/snapshot.d.ts.map +1 -1
  190. package/dist/snapshot.js +1206 -109
  191. package/dist/snapshot.js.map +1 -1
  192. package/dist/steering.d.ts +75 -0
  193. package/dist/steering.d.ts.map +1 -0
  194. package/dist/steering.js +214 -0
  195. package/dist/steering.js.map +1 -0
  196. package/dist/subagents.d.ts +141 -0
  197. package/dist/subagents.d.ts.map +1 -0
  198. package/dist/subagents.js +746 -0
  199. package/dist/subagents.js.map +1 -0
  200. package/dist/tool-retry.d.ts +12 -0
  201. package/dist/tool-retry.d.ts.map +1 -0
  202. package/dist/tool-retry.js +53 -0
  203. package/dist/tool-retry.js.map +1 -0
  204. package/dist/tree.d.ts +62 -6
  205. package/dist/tree.d.ts.map +1 -1
  206. package/dist/tree.js +192 -23
  207. package/dist/tree.js.map +1 -1
  208. package/dist/types.d.ts +767 -52
  209. package/dist/types.d.ts.map +1 -1
  210. package/dist/workflow-snapshot.d.ts +8 -0
  211. package/dist/workflow-snapshot.d.ts.map +1 -0
  212. package/dist/workflow-snapshot.js +276 -0
  213. package/dist/workflow-snapshot.js.map +1 -0
  214. package/dist/workflow-types.d.ts +171 -0
  215. package/dist/workflow-types.d.ts.map +1 -0
  216. package/dist/workflow-types.js +2 -0
  217. package/dist/workflow-types.js.map +1 -0
  218. package/dist/workflow.d.ts +19 -0
  219. package/dist/workflow.d.ts.map +1 -0
  220. package/dist/workflow.js +1053 -0
  221. package/dist/workflow.js.map +1 -0
  222. package/examples/a2a-remote.ts +85 -0
  223. package/examples/a2a-server.ts +100 -0
  224. package/examples/background.ts +18 -4
  225. package/examples/code-mode.ts +76 -0
  226. package/examples/conversation.ts +57 -0
  227. package/examples/evals.ts +79 -0
  228. package/examples/guardrails.ts +56 -0
  229. package/examples/handoff.ts +79 -0
  230. package/examples/memory.ts +44 -0
  231. package/examples/otel.ts +99 -0
  232. package/examples/steering.ts +69 -0
  233. package/examples/structured-handoff.ts +92 -0
  234. package/examples/subagents.ts +127 -0
  235. package/examples/workflow.ts +74 -0
  236. package/package.json +58 -6
  237. package/src/a2a/auth.ts +62 -0
  238. package/src/a2a/client.ts +733 -0
  239. package/src/a2a/index.ts +74 -0
  240. package/src/a2a/memory-runs.ts +448 -0
  241. package/src/a2a/policy.ts +148 -0
  242. package/src/a2a/push.ts +251 -0
  243. package/src/a2a/remote.ts +1652 -0
  244. package/src/a2a/server.ts +1390 -0
  245. package/src/a2a/signing.ts +348 -0
  246. package/src/a2a/tasks.ts +260 -0
  247. package/src/a2a/wire.ts +1443 -0
  248. package/src/background-types.ts +86 -0
  249. package/src/child.ts +181 -0
  250. package/src/code-driver.ts +48 -0
  251. package/src/code-mode.ts +12 -0
  252. package/src/code-script.ts +149 -0
  253. package/src/code.ts +424 -0
  254. package/src/compaction.ts +212 -0
  255. package/src/delegation.ts +444 -155
  256. package/src/evals/driver.ts +222 -0
  257. package/src/evals/journal.ts +605 -0
  258. package/src/evals/judge.ts +188 -0
  259. package/src/evals/report.ts +295 -0
  260. package/src/evals/runner.ts +826 -0
  261. package/src/evals/scorers.ts +339 -0
  262. package/src/evals/simulation.ts +46 -0
  263. package/src/evals/testing.ts +126 -0
  264. package/src/evals/trajectory.ts +223 -0
  265. package/src/evals/types.ts +463 -0
  266. package/src/evals.ts +72 -0
  267. package/src/extensions.ts +247 -0
  268. package/src/handoff.ts +214 -0
  269. package/src/index.ts +51 -0
  270. package/src/input.ts +319 -0
  271. package/src/mcp.ts +690 -0
  272. package/src/memory-store.ts +366 -0
  273. package/src/memory.ts +893 -0
  274. package/src/otel.ts +568 -0
  275. package/src/progress.ts +214 -0
  276. package/src/quickjs.ts +524 -0
  277. package/src/recovery.ts +90 -0
  278. package/src/request.ts +513 -0
  279. package/src/run.ts +2678 -313
  280. package/src/runs.ts +464 -133
  281. package/src/schema-to-ts.ts +194 -0
  282. package/src/session.ts +178 -0
  283. package/src/snapshot.ts +1595 -124
  284. package/src/steering.ts +329 -0
  285. package/src/subagents.ts +1084 -0
  286. package/src/tool-retry.ts +86 -0
  287. package/src/tree.ts +223 -27
  288. package/src/types.ts +711 -52
  289. package/src/workflow-snapshot.ts +366 -0
  290. package/src/workflow-types.ts +213 -0
  291. package/src/workflow.ts +1316 -0
@@ -0,0 +1,605 @@
1
+ import {
2
+ addUsage,
3
+ costUsd,
4
+ EMPTY_USAGE,
5
+ HOSTED_TOOL_TYPES,
6
+ parseToolArgs,
7
+ type TokenUsage,
8
+ } from "@providerkit/core";
9
+ import {
10
+ RESULT_TOOL_NAME,
11
+ type AskRecord,
12
+ type RunSnapshot,
13
+ type ToolExecution,
14
+ type TreeNode,
15
+ } from "../index.ts";
16
+ import { allModels } from "../compaction.ts";
17
+ import type { RemoteSnapshot } from "../a2a/remote.ts";
18
+ import { inputOf, readInput } from "../input.ts";
19
+ import {
20
+ agentSnapshot,
21
+ copyData,
22
+ isId,
23
+ isRecord,
24
+ isUsage,
25
+ jsonArguments,
26
+ readLimits,
27
+ readSnapshot,
28
+ } from "../snapshot.ts";
29
+ import { snapshots } from "../tree.ts";
30
+ import { checkExpectation } from "./trajectory.ts";
31
+ import type {
32
+ EvalCase,
33
+ EvalPrice,
34
+ EvalRun,
35
+ EvalRunResult,
36
+ EvalTurn,
37
+ EvalUsage,
38
+ ModelRef,
39
+ Trajectory,
40
+ TrajectoryStep,
41
+ } from "./types.ts";
42
+
43
+ /** Uses the SDK's structural and journal checks, never resume or an output validator. A stored
44
+ * journal is data, not permission to run a tool. Unknown output contracts stay unknown JSON. */
45
+ export function readJournal(raw: unknown): RunSnapshot {
46
+ const value = isRecord(raw) && "snapshot" in raw ? raw.snapshot : raw;
47
+ if (!isRecord(value) || value.kind !== "agent" || !isId(value.agentId) || !isId(value.version))
48
+ throw new TypeError("Provide a format-3 agent run journal or an SDK agent outcome.");
49
+ const limits = readLimits(value.limits);
50
+ const definition = value.definition;
51
+ const checkedOutput =
52
+ isRecord(definition) &&
53
+ Array.isArray(definition.enabledTools) &&
54
+ definition.enabledTools.includes(RESULT_TOOL_NAME);
55
+ const root = readSnapshot(
56
+ value,
57
+ value.agentId,
58
+ value.version,
59
+ limits,
60
+ checkedOutput ? RESULT_TOOL_NAME : undefined,
61
+ );
62
+ // readSnapshot checks the mixed tree's shape and every workflow's operations. Agent
63
+ // descendants need their own operation checks, including those below workflow steps.
64
+ for (const node of snapshots(root)) {
65
+ if (node.kind !== "agent") continue;
66
+ if (node !== root)
67
+ readSnapshot(
68
+ node,
69
+ node.agentId,
70
+ node.version,
71
+ node.limits,
72
+ node.definition.enabledTools.includes(RESULT_TOOL_NAME) ? RESULT_TOOL_NAME : undefined,
73
+ );
74
+ for (const model of allModels(node)) {
75
+ checkUsage(model.usage);
76
+ for (const attempt of model.interrupted) checkUsage(attempt.usage);
77
+ }
78
+ }
79
+ return root;
80
+ }
81
+
82
+ function checkUsage(value: TokenUsage): void {
83
+ if (
84
+ !isUsage(value) ||
85
+ (value.hostedCalls !== undefined &&
86
+ (!isRecord(value.hostedCalls) ||
87
+ Object.entries(value.hostedCalls).some(
88
+ ([key, count]) =>
89
+ !HOSTED_TOOL_TYPES.some((type) => type === key) ||
90
+ typeof count !== "number" ||
91
+ !Number.isFinite(count) ||
92
+ count < 0,
93
+ )))
94
+ )
95
+ throw new TypeError("The journal has invalid token or hosted-call usage.");
96
+ }
97
+
98
+ export function emptyUsage(): EvalUsage {
99
+ return {
100
+ usage: { ...EMPTY_USAGE },
101
+ knownCostUsd: 0,
102
+ costUsd: 0,
103
+ unknownCharges: 0,
104
+ unpricedCalls: 0,
105
+ };
106
+ }
107
+
108
+ export function combineUsage(values: Iterable<EvalUsage>): EvalUsage {
109
+ const result = emptyUsage();
110
+ for (const value of values) {
111
+ result.usage = addUsage(result.usage, value.usage);
112
+ result.knownCostUsd += value.knownCostUsd;
113
+ result.unknownCharges += value.unknownCharges;
114
+ result.unpricedCalls += value.unpricedCalls;
115
+ }
116
+ result.costUsd = result.unknownCharges || result.unpricedCalls ? null : result.knownCostUsd;
117
+ return result;
118
+ }
119
+
120
+ export function usageOf(root: TreeNode, price?: EvalPrice): EvalUsage {
121
+ const totals = emptyUsage();
122
+ const seen = new Set<string>();
123
+ for (const node of snapshots(root)) {
124
+ if (node.kind === "remote") {
125
+ // The peer supplied no local model bill. Each admitted send/cancel remains unknown.
126
+ totals.unknownCharges += node.remote.sends.length + (node.remote.cancel ? 1 : 0);
127
+ continue;
128
+ }
129
+ if (node.kind !== "agent") continue;
130
+ for (const model of allModels(node)) {
131
+ if (seen.has(model.id)) continue;
132
+ seen.add(model.id);
133
+ const ref: ModelRef = {
134
+ agentId: node.agentId,
135
+ ...(model.status === "completed" ? model.turn.source : undefined),
136
+ };
137
+ for (const attempt of [
138
+ ...model.interrupted.map((attempt) => ({ ...attempt, billing: "unknown" })),
139
+ model,
140
+ ]) {
141
+ checkUsage(attempt.usage);
142
+ totals.usage = addUsage(totals.usage, attempt.usage);
143
+ if (attempt.billing === "unknown") totals.unknownCharges++;
144
+ const measured = attempt.usage;
145
+ if (measured.reportedCostUsd !== undefined) {
146
+ totals.knownCostUsd += measured.reportedCostUsd;
147
+ continue;
148
+ }
149
+ // A reserved operation with no usage has not given us a priceable measurement yet.
150
+ const hasMeasurement =
151
+ measured.inputTokens + measured.outputTokens > 0 ||
152
+ Object.values(measured.hostedCalls ?? {}).some((n) => n > 0);
153
+ if (!hasMeasurement && model.status === "running" && root.state.status === "running")
154
+ continue;
155
+ // An interrupted attempt can have used a different fallback route. The replacement
156
+ // turn's source is not evidence of which model served the lost attempt.
157
+ const rate = price?.(attempt === model ? ref : { agentId: node.agentId });
158
+ if (
159
+ rate &&
160
+ [
161
+ rate.input,
162
+ rate.output,
163
+ rate.cacheRead,
164
+ rate.cacheWrite ?? rate.input,
165
+ ...Object.values(rate.hosted ?? {}),
166
+ ].some((n) => !Number.isFinite(n) || n < 0)
167
+ )
168
+ throw new TypeError("Prices must be finite, non-negative USD rates.");
169
+ const missingHostedRate = HOSTED_TOOL_TYPES.some(
170
+ (type) => (measured.hostedCalls?.[type] ?? 0) > 0 && rate?.hosted?.[type] === undefined,
171
+ );
172
+ const settled =
173
+ attempt !== model || model.status === "completed" || root.state.status !== "running";
174
+ // A running receipt has no source yet. Do not reject a model-specific caller price
175
+ // before the completed turn identifies the model. The unknown charge still counts.
176
+ if ((!rate || missingHostedRate) && settled) totals.unpricedCalls++;
177
+ if (rate) totals.knownCostUsd += costUsd(measured, rate);
178
+ }
179
+ }
180
+ }
181
+ totals.costUsd = totals.unknownCharges || totals.unpricedCalls ? null : totals.knownCostUsd;
182
+ return totals;
183
+ }
184
+
185
+ function childTrajectory({
186
+ key,
187
+ snapshot,
188
+ }: {
189
+ key: string;
190
+ snapshot: TreeNode;
191
+ }): TrajectoryStep["children"][number] {
192
+ return {
193
+ key,
194
+ ...(snapshot.kind === "workflow"
195
+ ? { kind: "workflow" as const, workflowId: snapshot.workflowId }
196
+ : snapshot.kind === "remote"
197
+ ? { kind: "remote" as const, agentId: snapshot.agentId, version: snapshot.version }
198
+ : { kind: "agent" as const, agentId: snapshot.agentId }),
199
+ runId: snapshot.runId,
200
+ trajectory: journalTrajectory(snapshot),
201
+ };
202
+ }
203
+
204
+ function remoteAttempts(snapshot: RemoteSnapshot): EvalRunResult["remoteAttempts"] {
205
+ return [
206
+ ...snapshot.remote.sends.map((send) => ({ ...send, action: "send" as const })),
207
+ ...(snapshot.remote.cancel ? [{ ...snapshot.remote.cancel, action: "cancel" as const }] : []),
208
+ ].map((receipt) => ({
209
+ kind: "remote",
210
+ action: receipt.action,
211
+ agentId: snapshot.agentId,
212
+ version: snapshot.version,
213
+ runId: snapshot.runId,
214
+ operationId: receipt.id,
215
+ attemptId: receipt.attemptId,
216
+ status: receipt.status,
217
+ billing: "unknown",
218
+ }));
219
+ }
220
+
221
+ function approvedInput<T>(input: T, asks: readonly AskRecord[] = []): T | Record<string, unknown> {
222
+ let effective: T | Record<string, unknown> = input;
223
+ for (const ask of asks) {
224
+ if (ask.decision?.type === "approve") effective = ask.decision.arguments ?? input;
225
+ }
226
+ return effective;
227
+ }
228
+
229
+ // Admission reserves budget for a pending ask; evals count only journaled attempt receipts.
230
+ function toolAttempts(tool: ToolExecution): number {
231
+ return tool.interrupted.length + tool.retries.length + (tool.status === "pending" ? 0 : 1);
232
+ }
233
+
234
+ /** Admission order comes from the journal, never completion time. Workflow steps retain their
235
+ * own kind and input; only agent nodes contribute model/tool calls. Hosted charges enter usageOf. */
236
+ export function journalTrajectory(root: TreeNode): Trajectory {
237
+ const result: Trajectory = {
238
+ steps: [],
239
+ flat: [],
240
+ modelCalls:
241
+ root.kind === "agent"
242
+ ? allModels(root).reduce((count, model) => count + 1 + model.interrupted.length, 0)
243
+ : 0,
244
+ toolCalls: 0,
245
+ };
246
+ if (root.kind === "workflow") {
247
+ result.steps = root.steps.map((step) => ({
248
+ kind: step.kind,
249
+ name: step.key,
250
+ workflowId: root.workflowId,
251
+ runId: root.runId,
252
+ operationId: step.id,
253
+ input: copyData(approvedInput(step.definition.input, step.asks)),
254
+ ...(step.status === "completed" ? { output: copyData(step.result) } : {}),
255
+ status: step.status,
256
+ attempts:
257
+ step.kind === "step"
258
+ ? step.attempts.filter((attempt) => attempt.status !== "not_run").length
259
+ : step.kind === "agent" || step.kind === "remote" || step.kind === "workflow"
260
+ ? step.children.length
261
+ : 1,
262
+ asks: copyData(step.asks ?? []),
263
+ children: step.children.map(childTrajectory),
264
+ }));
265
+ } else if (root.kind === "remote") {
266
+ result.steps = remoteAttempts(root).map((receipt, index) => ({
267
+ ...receipt,
268
+ kind: "remote-operation",
269
+ name: `remote_${receipt.action}`,
270
+ attempts: 1,
271
+ ...(receipt.action === "send" && {
272
+ input: copyData(root.remote.sends[index]?.message.parts),
273
+ }),
274
+ asks: copyData(root.remote.sends[index]?.asks ?? []),
275
+ children: [],
276
+ }));
277
+ } else {
278
+ for (const [index, model] of root.models.entries()) {
279
+ if (model.status !== "completed") continue;
280
+ for (const [position, call] of (model.turn.message.toolCalls ?? []).entries()) {
281
+ const execution = model.tools[position];
282
+ if (!execution || call.name === RESULT_TOOL_NAME) continue;
283
+ const attempts = toolAttempts(execution);
284
+ if (!attempts && !execution.calls?.length) continue;
285
+ const journal = root.delegations.find((entry) => entry.operationId === execution.id);
286
+ const code = Object.hasOwn(root.definition.codeTools ?? {}, call.name);
287
+ const args = approvedInput(
288
+ code ? (jsonArguments(call.arguments) ?? {}) : parseToolArgs(call.arguments),
289
+ execution.asks,
290
+ );
291
+ result.steps.push({
292
+ kind: code ? "code" : (journal?.kind ?? "tool"),
293
+ name: call.name,
294
+ arguments: copyData(args),
295
+ status:
296
+ execution.status === "completed"
297
+ ? "completed"
298
+ : execution.status === "pending"
299
+ ? "interrupted"
300
+ : "running",
301
+ attempts,
302
+ ...(execution.status === "completed" ? { result: copyData(execution.result) } : {}),
303
+ asks: copyData(execution.asks ?? []),
304
+ agentId: root.agentId,
305
+ runId: root.runId,
306
+ operationId: execution.id,
307
+ turn: index + 1,
308
+ children: (journal?.children ?? []).map(childTrajectory),
309
+ });
310
+ result.toolCalls += attempts;
311
+ for (const inner of execution.calls ?? []) {
312
+ const attempts = toolAttempts(inner);
313
+ result.steps.push({
314
+ kind: "tool",
315
+ name: inner.name,
316
+ arguments: copyData(approvedInput(inner.arguments, inner.asks)),
317
+ status: inner.status,
318
+ attempts,
319
+ ...(inner.status === "completed" && { result: copyData(inner.result) }),
320
+ asks: copyData(inner.asks ?? []),
321
+ agentId: root.agentId,
322
+ runId: root.runId,
323
+ operationId: inner.id,
324
+ parentOperationId: execution.id,
325
+ turn: index + 1,
326
+ children: [],
327
+ });
328
+ result.toolCalls += attempts;
329
+ }
330
+ }
331
+ }
332
+ }
333
+ for (const step of result.steps) {
334
+ result.flat.push(step);
335
+ for (const child of step.children) {
336
+ result.flat.push(...child.trajectory.flat);
337
+ result.modelCalls += child.trajectory.modelCalls;
338
+ result.toolCalls += child.trajectory.toolCalls;
339
+ }
340
+ }
341
+ return result;
342
+ }
343
+
344
+ export function trajectoryOf(raw: unknown): Trajectory {
345
+ return journalTrajectory(readJournal(raw));
346
+ }
347
+
348
+ export function evalRun(root: RunSnapshot, price?: EvalPrice, durationMs?: number): EvalRun {
349
+ // The last admitted handoff owns the answer; delegated specialists do not.
350
+ let active = root;
351
+ while (true) {
352
+ const child = active.delegations.find(
353
+ (entry) => entry.kind === "handoff" && entry.children.length > 0,
354
+ )?.children[0];
355
+ if (!child) break;
356
+ active = agentSnapshot(child.snapshot);
357
+ }
358
+ const last = active.models.at(-1);
359
+ return {
360
+ runId: root.runId,
361
+ agentId: root.agentId,
362
+ version: root.version,
363
+ input: inputOf(root),
364
+ status: root.state.status,
365
+ ...(root.state.status === "completed" ? { value: copyData(root.state.candidate) } : {}),
366
+ text: last?.status === "completed" ? last.turn.message.content : "",
367
+ snapshot: root,
368
+ activeAgent: { id: active.agentId, version: active.version },
369
+ trajectory: journalTrajectory(root),
370
+ durationMs,
371
+ ...usageOf(root, price),
372
+ };
373
+ }
374
+
375
+ export function runResult(
376
+ snapshot: RunSnapshot,
377
+ usage: EvalUsage,
378
+ role: EvalRunResult["role"],
379
+ includePrivateData = false,
380
+ turn?: number,
381
+ durationMs?: number,
382
+ ): EvalRunResult {
383
+ const run = evalRun(snapshot, undefined, durationMs);
384
+ const agents = [...snapshots(snapshot)].filter((node) => node.kind === "agent");
385
+ return {
386
+ runId: snapshot.runId,
387
+ attemptId: snapshot.attemptId,
388
+ agent: { id: snapshot.agentId, version: snapshot.version },
389
+ activeAgent: run.activeAgent,
390
+ role,
391
+ turn,
392
+ status: snapshot.state.status,
393
+ modelCalls: run.trajectory.modelCalls,
394
+ toolCalls: run.trajectory.toolCalls,
395
+ durationMs,
396
+ ...usage,
397
+ modelAttempts: agents.flatMap((node) =>
398
+ allModels(node).flatMap((model) => [
399
+ ...model.interrupted.map((attempt) => ({
400
+ operationId: model.id,
401
+ attemptId: attempt.attemptId,
402
+ status: "interrupted" as const,
403
+ billing: "unknown" as const,
404
+ usage: copyData(attempt.usage),
405
+ })),
406
+ {
407
+ operationId: model.id,
408
+ attemptId: model.attemptId,
409
+ status: model.status,
410
+ billing: model.billing,
411
+ usage: copyData(model.usage),
412
+ },
413
+ ]),
414
+ ),
415
+ toolAttempts: agents.flatMap((node) =>
416
+ node.models.flatMap((model) =>
417
+ model.status !== "completed"
418
+ ? []
419
+ : model.tools.flatMap((outer) =>
420
+ [
421
+ { tool: outer, parentOperationId: undefined },
422
+ ...(outer.calls ?? []).map((tool) => ({ tool, parentOperationId: outer.id })),
423
+ ].flatMap(({ tool, parentOperationId }) => [
424
+ ...tool.interrupted.map((attempt) => ({
425
+ operationId: tool.id,
426
+ ...(parentOperationId && { parentOperationId }),
427
+ attemptId: attempt.attemptId,
428
+ status: "interrupted" as const,
429
+ })),
430
+ ...tool.retries.map((attempt) => ({
431
+ operationId: tool.id,
432
+ ...(parentOperationId && { parentOperationId }),
433
+ attemptId: attempt.attemptId,
434
+ status: "completed" as const,
435
+ ok: false,
436
+ })),
437
+ ...(tool.status === "pending"
438
+ ? []
439
+ : [
440
+ {
441
+ operationId: tool.id,
442
+ ...(parentOperationId && { parentOperationId }),
443
+ attemptId: tool.attemptId,
444
+ status:
445
+ tool.status === "completed"
446
+ ? ("completed" as const)
447
+ : ("running" as const),
448
+ ...(tool.status === "completed" ? { ok: tool.result.ok } : {}),
449
+ },
450
+ ]),
451
+ ]),
452
+ ),
453
+ ),
454
+ ),
455
+ remoteAttempts: [...snapshots(snapshot)].flatMap((node) =>
456
+ node.kind === "remote" ? remoteAttempts(node) : [],
457
+ ),
458
+ ...(includePrivateData ? { snapshot: copyData(snapshot) } : {}),
459
+ };
460
+ }
461
+
462
+ /** Strip executable scorers before freezing context or validating stored case data. */
463
+ export function caseData<E>(item: EvalCase<E>): EvalCase<E> {
464
+ const { scorers: _scorers, turns, ...data } = item;
465
+ return {
466
+ ...data,
467
+ ...(turns ? { turns: turns.map(({ scorers: _turnScorers, ...turn }) => turn) } : {}),
468
+ };
469
+ }
470
+
471
+ /** Validates data only; a dataset file cannot supply functions, drivers or scorer names.
472
+ * Supply validateExpected to read typed expected values, including each turn's expectation. */
473
+ export function readCases(raw: unknown): EvalCase[];
474
+ export function readCases<E>(raw: unknown, validateExpected: (value: unknown) => E): EvalCase<E>[];
475
+ export function readCases(
476
+ raw: unknown,
477
+ validateExpected?: (value: unknown) => unknown,
478
+ ): EvalCase[] {
479
+ if (!Array.isArray(raw)) throw new TypeError("Cases must be an array.");
480
+ const ids = new Set<string>();
481
+ function expectations(
482
+ value: Record<string, unknown>,
483
+ ): Pick<EvalCase, "expected" | "expectStatus" | "trajectory"> {
484
+ const status = value.expectStatus;
485
+ if (
486
+ status !== undefined &&
487
+ status !== "completed" &&
488
+ status !== "failed" &&
489
+ status !== "limited" &&
490
+ status !== "suspended" &&
491
+ status !== "cancelled"
492
+ )
493
+ throw new TypeError("Invalid expected run status.");
494
+ if (value.trajectory !== undefined) checkExpectation(value.trajectory);
495
+ return {
496
+ ...(Object.hasOwn(value, "expected")
497
+ ? {
498
+ expected: validateExpected
499
+ ? validateExpected(value.expected)
500
+ : copyData(value.expected),
501
+ }
502
+ : {}),
503
+ ...(status ? { expectStatus: status } : {}),
504
+ ...(value.trajectory === undefined ? {} : { trajectory: value.trajectory }),
505
+ };
506
+ }
507
+ return raw.map((rawValue): EvalCase => {
508
+ const value = copyData(rawValue);
509
+ if (
510
+ !isRecord(value) ||
511
+ !isId(value.id) ||
512
+ ids.has(value.id) ||
513
+ Object.keys(value).some(
514
+ (key) =>
515
+ !["id", "input", "turns", "user", "expected", "expectStatus", "trajectory"].includes(key),
516
+ ) ||
517
+ [value.input, value.turns, value.user].filter((entry) => entry !== undefined).length !== 1
518
+ )
519
+ throw new TypeError(
520
+ "Each case needs a unique id and exactly one of input, turns and user. Drivers and code are not case data.",
521
+ );
522
+ ids.add(value.id);
523
+ const base = { id: value.id, ...expectations(value) };
524
+ if (value.turns !== undefined) {
525
+ if (!Array.isArray(value.turns) || !value.turns.length)
526
+ throw new TypeError("A conversation case needs at least one turn.");
527
+ const turns = value.turns.map((turn): EvalTurn => {
528
+ if (
529
+ !isRecord(turn) ||
530
+ Object.keys(turn).some(
531
+ (key) => !["input", "expected", "expectStatus", "trajectory"].includes(key),
532
+ ) ||
533
+ isRecord(turn.input)
534
+ )
535
+ throw new TypeError(
536
+ "Each session turn needs a new user input and only supported data fields.",
537
+ );
538
+ return { input: readInput(turn.input, "caller").input, ...expectations(turn) };
539
+ });
540
+ return { ...base, turns };
541
+ }
542
+ if (value.user !== undefined) {
543
+ const user = value.user;
544
+ if (
545
+ !isRecord(user) ||
546
+ Object.keys(user).some(
547
+ (key) => !["opening", "plan", "persona", "maxTurns"].includes(key),
548
+ ) ||
549
+ !isId(user.plan) ||
550
+ (user.persona !== undefined && !isId(user.persona)) ||
551
+ isRecord(user.opening) ||
552
+ (user.maxTurns !== undefined &&
553
+ (typeof user.maxTurns !== "number" ||
554
+ !Number.isSafeInteger(user.maxTurns) ||
555
+ user.maxTurns < 1 ||
556
+ user.maxTurns > 100))
557
+ )
558
+ throw new TypeError(
559
+ "A simulated user needs a plan, an opening and a turn limit from 1 to 100.",
560
+ );
561
+ return {
562
+ ...base,
563
+ user: {
564
+ opening: readInput(user.opening, "caller").input,
565
+ plan: user.plan,
566
+ ...(user.persona === undefined ? {} : { persona: user.persona }),
567
+ ...(user.maxTurns === undefined ? {} : { maxTurns: user.maxTurns }),
568
+ },
569
+ };
570
+ }
571
+ const input = readInput(value.input, "caller");
572
+ return {
573
+ ...base,
574
+ input: input.history.length
575
+ ? { messages: [...input.history, { role: "user", content: input.input }] }
576
+ : input.input,
577
+ };
578
+ });
579
+ }
580
+
581
+ /** Explicitly copies private input, output and tool arguments. Nothing is executed or approved.
582
+ * Supply a sandboxed agent when you later run this case; stored results are not tool mocks. */
583
+ export function caseFromRun(raw: unknown, options: { id: string }): EvalCase {
584
+ if (!isId(options.id)) throw new TypeError("Give the golden case a non-empty id.");
585
+ const snapshot = readJournal(raw);
586
+ if (snapshot.state.status !== "completed")
587
+ throw new TypeError(
588
+ "Only a completed run can become a golden case. No interrupted work is resumed.",
589
+ );
590
+ if ([...snapshots(snapshot)].some((node) => node.kind === "agent" && node.inputs.length > 0))
591
+ throw new TypeError(
592
+ "This run received messages while working. Record its input timing in the host instead of replaying it as one case.",
593
+ );
594
+ return {
595
+ id: options.id,
596
+ input: inputOf(snapshot),
597
+ expected: copyData(snapshot.state.candidate),
598
+ trajectory: {
599
+ match: "exact",
600
+ calls: journalTrajectory(snapshot)
601
+ .steps.filter((step) => step.result?.ok)
602
+ .map((step) => ({ name: step.name, arguments: step.arguments, argumentsMatch: "exact" })),
603
+ },
604
+ };
605
+ }