@providerkit/agents 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (296) hide show
  1. package/LICENSE +21 -0
  2. package/LICENSE-APACHE-2.0 +202 -0
  3. package/README.md +2328 -2
  4. package/THIRD_PARTY_NOTICES.md +521 -0
  5. package/dist/a2a/auth.d.ts +10 -0
  6. package/dist/a2a/auth.d.ts.map +1 -0
  7. package/dist/a2a/auth.js +52 -0
  8. package/dist/a2a/auth.js.map +1 -0
  9. package/dist/a2a/client.d.ts +74 -0
  10. package/dist/a2a/client.d.ts.map +1 -0
  11. package/dist/a2a/client.js +601 -0
  12. package/dist/a2a/client.js.map +1 -0
  13. package/dist/a2a/index.d.ts +22 -0
  14. package/dist/a2a/index.d.ts.map +1 -0
  15. package/dist/a2a/index.js +11 -0
  16. package/dist/a2a/index.js.map +1 -0
  17. package/dist/a2a/memory-runs.d.ts +62 -0
  18. package/dist/a2a/memory-runs.d.ts.map +1 -0
  19. package/dist/a2a/memory-runs.js +349 -0
  20. package/dist/a2a/memory-runs.js.map +1 -0
  21. package/dist/a2a/policy.d.ts +15 -0
  22. package/dist/a2a/policy.d.ts.map +1 -0
  23. package/dist/a2a/policy.js +114 -0
  24. package/dist/a2a/policy.js.map +1 -0
  25. package/dist/a2a/push.d.ts +34 -0
  26. package/dist/a2a/push.d.ts.map +1 -0
  27. package/dist/a2a/push.js +180 -0
  28. package/dist/a2a/push.js.map +1 -0
  29. package/dist/a2a/remote.d.ts +155 -0
  30. package/dist/a2a/remote.d.ts.map +1 -0
  31. package/dist/a2a/remote.js +1196 -0
  32. package/dist/a2a/remote.js.map +1 -0
  33. package/dist/a2a/server.d.ts +110 -0
  34. package/dist/a2a/server.d.ts.map +1 -0
  35. package/dist/a2a/server.js +1139 -0
  36. package/dist/a2a/server.js.map +1 -0
  37. package/dist/a2a/signing.d.ts +21 -0
  38. package/dist/a2a/signing.d.ts.map +1 -0
  39. package/dist/a2a/signing.js +305 -0
  40. package/dist/a2a/signing.js.map +1 -0
  41. package/dist/a2a/tasks.d.ts +88 -0
  42. package/dist/a2a/tasks.d.ts.map +1 -0
  43. package/dist/a2a/tasks.js +170 -0
  44. package/dist/a2a/tasks.js.map +1 -0
  45. package/dist/a2a/wire.d.ts +267 -0
  46. package/dist/a2a/wire.d.ts.map +1 -0
  47. package/dist/a2a/wire.js +1202 -0
  48. package/dist/a2a/wire.js.map +1 -0
  49. package/dist/background-types.d.ts +76 -0
  50. package/dist/background-types.d.ts.map +1 -0
  51. package/dist/background-types.js +2 -0
  52. package/dist/background-types.js.map +1 -0
  53. package/dist/child.d.ts +25 -0
  54. package/dist/child.d.ts.map +1 -0
  55. package/dist/child.js +129 -0
  56. package/dist/child.js.map +1 -0
  57. package/dist/code-driver.d.ts +40 -0
  58. package/dist/code-driver.d.ts.map +1 -0
  59. package/dist/code-driver.js +2 -0
  60. package/dist/code-driver.js.map +1 -0
  61. package/dist/code-mode.d.ts +3 -0
  62. package/dist/code-mode.d.ts.map +1 -0
  63. package/dist/code-mode.js +2 -0
  64. package/dist/code-mode.js.map +1 -0
  65. package/dist/code-script.d.ts +3 -0
  66. package/dist/code-script.d.ts.map +1 -0
  67. package/dist/code-script.js +149 -0
  68. package/dist/code-script.js.map +1 -0
  69. package/dist/code.d.ts +75 -0
  70. package/dist/code.d.ts.map +1 -0
  71. package/dist/code.js +326 -0
  72. package/dist/code.js.map +1 -0
  73. package/dist/compaction.d.ts +15 -0
  74. package/dist/compaction.d.ts.map +1 -0
  75. package/dist/compaction.js +168 -0
  76. package/dist/compaction.js.map +1 -0
  77. package/dist/delegation.d.ts +135 -0
  78. package/dist/delegation.d.ts.map +1 -0
  79. package/dist/delegation.js +401 -0
  80. package/dist/delegation.js.map +1 -0
  81. package/dist/evals/driver.d.ts +10 -0
  82. package/dist/evals/driver.d.ts.map +1 -0
  83. package/dist/evals/driver.js +192 -0
  84. package/dist/evals/driver.js.map +1 -0
  85. package/dist/evals/journal.d.ts +26 -0
  86. package/dist/evals/journal.d.ts.map +1 -0
  87. package/dist/evals/journal.js +483 -0
  88. package/dist/evals/journal.js.map +1 -0
  89. package/dist/evals/judge.d.ts +31 -0
  90. package/dist/evals/judge.d.ts.map +1 -0
  91. package/dist/evals/judge.js +142 -0
  92. package/dist/evals/judge.js.map +1 -0
  93. package/dist/evals/report.d.ts +38 -0
  94. package/dist/evals/report.d.ts.map +1 -0
  95. package/dist/evals/report.js +241 -0
  96. package/dist/evals/report.js.map +1 -0
  97. package/dist/evals/runner.d.ts +24 -0
  98. package/dist/evals/runner.d.ts.map +1 -0
  99. package/dist/evals/runner.js +655 -0
  100. package/dist/evals/runner.js.map +1 -0
  101. package/dist/evals/scorers.d.ts +36 -0
  102. package/dist/evals/scorers.d.ts.map +1 -0
  103. package/dist/evals/scorers.js +259 -0
  104. package/dist/evals/scorers.js.map +1 -0
  105. package/dist/evals/simulation.d.ts +7 -0
  106. package/dist/evals/simulation.d.ts.map +1 -0
  107. package/dist/evals/simulation.js +38 -0
  108. package/dist/evals/simulation.js.map +1 -0
  109. package/dist/evals/testing.d.ts +36 -0
  110. package/dist/evals/testing.d.ts.map +1 -0
  111. package/dist/evals/testing.js +87 -0
  112. package/dist/evals/testing.js.map +1 -0
  113. package/dist/evals/trajectory.d.ts +11 -0
  114. package/dist/evals/trajectory.d.ts.map +1 -0
  115. package/dist/evals/trajectory.js +180 -0
  116. package/dist/evals/trajectory.js.map +1 -0
  117. package/dist/evals/types.d.ts +431 -0
  118. package/dist/evals/types.d.ts.map +1 -0
  119. package/dist/evals/types.js +4 -0
  120. package/dist/evals/types.js.map +1 -0
  121. package/dist/evals.d.ts +14 -0
  122. package/dist/evals.d.ts.map +1 -0
  123. package/dist/evals.js +9 -0
  124. package/dist/evals.js.map +1 -0
  125. package/dist/extensions.d.ts +33 -0
  126. package/dist/extensions.d.ts.map +1 -0
  127. package/dist/extensions.js +188 -0
  128. package/dist/extensions.js.map +1 -0
  129. package/dist/handoff.d.ts +58 -0
  130. package/dist/handoff.d.ts.map +1 -0
  131. package/dist/handoff.js +160 -0
  132. package/dist/handoff.js.map +1 -0
  133. package/dist/index.d.ts +19 -0
  134. package/dist/index.d.ts.map +1 -0
  135. package/dist/index.js +11 -0
  136. package/dist/index.js.map +1 -0
  137. package/dist/input.d.ts +42 -0
  138. package/dist/input.d.ts.map +1 -0
  139. package/dist/input.js +266 -0
  140. package/dist/input.js.map +1 -0
  141. package/dist/mcp.d.ts +77 -0
  142. package/dist/mcp.d.ts.map +1 -0
  143. package/dist/mcp.js +505 -0
  144. package/dist/mcp.js.map +1 -0
  145. package/dist/memory-store.d.ts +11 -0
  146. package/dist/memory-store.d.ts.map +1 -0
  147. package/dist/memory-store.js +272 -0
  148. package/dist/memory-store.js.map +1 -0
  149. package/dist/memory.d.ts +212 -0
  150. package/dist/memory.d.ts.map +1 -0
  151. package/dist/memory.js +613 -0
  152. package/dist/memory.js.map +1 -0
  153. package/dist/otel.d.ts +74 -0
  154. package/dist/otel.d.ts.map +1 -0
  155. package/dist/otel.js +427 -0
  156. package/dist/otel.js.map +1 -0
  157. package/dist/progress.d.ts +17 -0
  158. package/dist/progress.d.ts.map +1 -0
  159. package/dist/progress.js +193 -0
  160. package/dist/progress.js.map +1 -0
  161. package/dist/quickjs.d.ts +19 -0
  162. package/dist/quickjs.d.ts.map +1 -0
  163. package/dist/quickjs.js +519 -0
  164. package/dist/quickjs.js.map +1 -0
  165. package/dist/recovery.d.ts +10 -0
  166. package/dist/recovery.d.ts.map +1 -0
  167. package/dist/recovery.js +62 -0
  168. package/dist/recovery.js.map +1 -0
  169. package/dist/request.d.ts +88 -0
  170. package/dist/request.d.ts.map +1 -0
  171. package/dist/request.js +422 -0
  172. package/dist/request.js.map +1 -0
  173. package/dist/run.d.ts +20 -0
  174. package/dist/run.d.ts.map +1 -0
  175. package/dist/run.js +2407 -0
  176. package/dist/run.js.map +1 -0
  177. package/dist/runs.d.ts +110 -0
  178. package/dist/runs.d.ts.map +1 -0
  179. package/dist/runs.js +438 -0
  180. package/dist/runs.js.map +1 -0
  181. package/dist/schema-to-ts.d.ts +4 -0
  182. package/dist/schema-to-ts.d.ts.map +1 -0
  183. package/dist/schema-to-ts.js +191 -0
  184. package/dist/schema-to-ts.js.map +1 -0
  185. package/dist/session.d.ts +23 -0
  186. package/dist/session.d.ts.map +1 -0
  187. package/dist/session.js +137 -0
  188. package/dist/session.js.map +1 -0
  189. package/dist/snapshot.d.ts +75 -0
  190. package/dist/snapshot.d.ts.map +1 -0
  191. package/dist/snapshot.js +1607 -0
  192. package/dist/snapshot.js.map +1 -0
  193. package/dist/steering.d.ts +75 -0
  194. package/dist/steering.d.ts.map +1 -0
  195. package/dist/steering.js +214 -0
  196. package/dist/steering.js.map +1 -0
  197. package/dist/subagents.d.ts +141 -0
  198. package/dist/subagents.d.ts.map +1 -0
  199. package/dist/subagents.js +746 -0
  200. package/dist/subagents.js.map +1 -0
  201. package/dist/tool-retry.d.ts +12 -0
  202. package/dist/tool-retry.d.ts.map +1 -0
  203. package/dist/tool-retry.js +53 -0
  204. package/dist/tool-retry.js.map +1 -0
  205. package/dist/tree.d.ts +98 -0
  206. package/dist/tree.d.ts.map +1 -0
  207. package/dist/tree.js +320 -0
  208. package/dist/tree.js.map +1 -0
  209. package/dist/types.d.ts +1007 -0
  210. package/dist/types.d.ts.map +1 -0
  211. package/dist/types.js +2 -0
  212. package/dist/types.js.map +1 -0
  213. package/dist/workflow-snapshot.d.ts +8 -0
  214. package/dist/workflow-snapshot.d.ts.map +1 -0
  215. package/dist/workflow-snapshot.js +276 -0
  216. package/dist/workflow-snapshot.js.map +1 -0
  217. package/dist/workflow-types.d.ts +171 -0
  218. package/dist/workflow-types.d.ts.map +1 -0
  219. package/dist/workflow-types.js +2 -0
  220. package/dist/workflow-types.js.map +1 -0
  221. package/dist/workflow.d.ts +19 -0
  222. package/dist/workflow.d.ts.map +1 -0
  223. package/dist/workflow.js +1053 -0
  224. package/dist/workflow.js.map +1 -0
  225. package/examples/a2a-remote.ts +85 -0
  226. package/examples/a2a-server.ts +100 -0
  227. package/examples/background.ts +59 -0
  228. package/examples/basic.ts +44 -0
  229. package/examples/code-mode.ts +76 -0
  230. package/examples/conversation.ts +57 -0
  231. package/examples/evals.ts +79 -0
  232. package/examples/guardrails.ts +56 -0
  233. package/examples/handoff.ts +79 -0
  234. package/examples/memory.ts +44 -0
  235. package/examples/otel.ts +99 -0
  236. package/examples/specialists.ts +99 -0
  237. package/examples/steering.ts +69 -0
  238. package/examples/structured-handoff.ts +92 -0
  239. package/examples/subagents.ts +127 -0
  240. package/examples/workflow.ts +74 -0
  241. package/package.json +114 -4
  242. package/src/a2a/auth.ts +62 -0
  243. package/src/a2a/client.ts +733 -0
  244. package/src/a2a/index.ts +74 -0
  245. package/src/a2a/memory-runs.ts +448 -0
  246. package/src/a2a/policy.ts +148 -0
  247. package/src/a2a/push.ts +251 -0
  248. package/src/a2a/remote.ts +1652 -0
  249. package/src/a2a/server.ts +1390 -0
  250. package/src/a2a/signing.ts +348 -0
  251. package/src/a2a/tasks.ts +260 -0
  252. package/src/a2a/wire.ts +1443 -0
  253. package/src/background-types.ts +86 -0
  254. package/src/child.ts +181 -0
  255. package/src/code-driver.ts +48 -0
  256. package/src/code-mode.ts +12 -0
  257. package/src/code-script.ts +149 -0
  258. package/src/code.ts +424 -0
  259. package/src/compaction.ts +212 -0
  260. package/src/delegation.ts +694 -0
  261. package/src/evals/driver.ts +222 -0
  262. package/src/evals/journal.ts +605 -0
  263. package/src/evals/judge.ts +188 -0
  264. package/src/evals/report.ts +295 -0
  265. package/src/evals/runner.ts +826 -0
  266. package/src/evals/scorers.ts +339 -0
  267. package/src/evals/simulation.ts +46 -0
  268. package/src/evals/testing.ts +126 -0
  269. package/src/evals/trajectory.ts +223 -0
  270. package/src/evals/types.ts +463 -0
  271. package/src/evals.ts +72 -0
  272. package/src/extensions.ts +247 -0
  273. package/src/handoff.ts +214 -0
  274. package/src/index.ts +62 -0
  275. package/src/input.ts +319 -0
  276. package/src/mcp.ts +690 -0
  277. package/src/memory-store.ts +366 -0
  278. package/src/memory.ts +893 -0
  279. package/src/otel.ts +568 -0
  280. package/src/progress.ts +214 -0
  281. package/src/quickjs.ts +524 -0
  282. package/src/recovery.ts +90 -0
  283. package/src/request.ts +513 -0
  284. package/src/run.ts +3199 -0
  285. package/src/runs.ts +696 -0
  286. package/src/schema-to-ts.ts +194 -0
  287. package/src/session.ts +178 -0
  288. package/src/snapshot.ts +2099 -0
  289. package/src/steering.ts +329 -0
  290. package/src/subagents.ts +1084 -0
  291. package/src/tool-retry.ts +86 -0
  292. package/src/tree.ts +366 -0
  293. package/src/types.ts +944 -0
  294. package/src/workflow-snapshot.ts +366 -0
  295. package/src/workflow-types.ts +213 -0
  296. package/src/workflow.ts +1316 -0
@@ -0,0 +1,223 @@
1
+ // Adapted from PydanticAI, pydantic_evals/pydantic_evals/evaluators/agentic.py (MIT).
2
+ // Copyright (c) Pydantic Services Inc. 2024 to present.
3
+ // Changed: journals replace spans, matching includes arguments/children, and reasons omit data.
4
+ import { copyData, equalData, isId, isRecord } from "../snapshot.ts";
5
+ import type { ExpectedCall, Trajectory, TrajectoryExpectation, TrajectoryStep } from "./types.ts";
6
+
7
+ export function nonNegative(value: unknown): value is number {
8
+ return typeof value === "number" && Number.isFinite(value) && value >= 0;
9
+ }
10
+
11
+ export function checkExpectation(raw: unknown): asserts raw is TrajectoryExpectation {
12
+ copyData(raw);
13
+ if (
14
+ !isRecord(raw) ||
15
+ Object.keys(raw).some(
16
+ (key) =>
17
+ ![
18
+ "calls",
19
+ "match",
20
+ "scope",
21
+ "forbidden",
22
+ "forbiddenSequences",
23
+ "maxToolCalls",
24
+ "maxModelCalls",
25
+ "noRepeats",
26
+ "includeFailed",
27
+ ].includes(key),
28
+ ) ||
29
+ (raw.match !== undefined &&
30
+ !["exact", "inOrder", "anyOrder", "subset"].some((v) => v === raw.match)) ||
31
+ (raw.scope !== undefined && raw.scope !== "root" && raw.scope !== "tree") ||
32
+ [raw.maxToolCalls, raw.maxModelCalls].some(
33
+ (v) => v !== undefined && (!nonNegative(v) || !Number.isSafeInteger(v)),
34
+ ) ||
35
+ [raw.noRepeats, raw.includeFailed].some((v) => v !== undefined && typeof v !== "boolean") ||
36
+ (raw.forbidden !== undefined &&
37
+ (!Array.isArray(raw.forbidden) || !raw.forbidden.every(isId))) ||
38
+ (raw.forbiddenSequences !== undefined &&
39
+ (!Array.isArray(raw.forbiddenSequences) ||
40
+ !raw.forbiddenSequences.every(
41
+ (seq) => Array.isArray(seq) && seq.length > 0 && seq.every(isId),
42
+ )))
43
+ )
44
+ throw new TypeError("Invalid trajectory expectation.");
45
+ if (raw.calls === undefined) return;
46
+ if (!Array.isArray(raw.calls)) throw new TypeError("Trajectory calls must be an array.");
47
+ for (const call of raw.calls) {
48
+ if (
49
+ !isRecord(call) ||
50
+ !isId(call.name) ||
51
+ Object.keys(call).some(
52
+ (key) => !["name", "arguments", "argumentsMatch", "input", "inside"].includes(key),
53
+ ) ||
54
+ (call.arguments !== undefined && !isRecord(call.arguments)) ||
55
+ (call.argumentsMatch !== undefined &&
56
+ !["exact", "subset", "ignore"].some((v) => v === call.argumentsMatch))
57
+ )
58
+ throw new TypeError("Each expected call needs a name and valid argument matching options.");
59
+ if (call.inside !== undefined) checkExpectation(call.inside);
60
+ }
61
+ }
62
+
63
+ /** Subset applies to top-level keys. Nested values still compare in full. */
64
+ export function argumentsMatch(
65
+ actual: Record<string, unknown>,
66
+ expected: Record<string, unknown>,
67
+ match: "subset" | "exact" = "subset",
68
+ ): boolean {
69
+ return match === "exact"
70
+ ? equalData(actual, expected)
71
+ : Object.entries(expected).every(
72
+ ([key, value]) => Object.hasOwn(actual, key) && equalData(actual[key], value),
73
+ );
74
+ }
75
+
76
+ function matches(actual: TrajectoryStep, expected: ExpectedCall): boolean {
77
+ const inside = expected.inside;
78
+ return (
79
+ actual.name === expected.name &&
80
+ (expected.arguments === undefined ||
81
+ expected.argumentsMatch === "ignore" ||
82
+ (actual.arguments !== undefined &&
83
+ argumentsMatch(actual.arguments, expected.arguments, expected.argumentsMatch))) &&
84
+ (expected.input === undefined ||
85
+ (actual.workflowId !== undefined && equalData(actual.input, expected.input))) &&
86
+ (!inside || actual.children.some((child) => matchTrajectory(child.trajectory, inside).passed))
87
+ );
88
+ }
89
+
90
+ function lcs(actual: readonly TrajectoryStep[], expected: readonly ExpectedCall[]): number {
91
+ let previous: number[] = Array.from({ length: expected.length + 1 }, () => 0);
92
+ // ponytail: O(actual * expected), with one rolling row. Run limits bound these lists.
93
+ // Large offline traces should preselect the operations to score before using this matcher.
94
+ for (const step of actual) {
95
+ const current = [0];
96
+ for (const [index, call] of expected.entries())
97
+ current.push(
98
+ matches(step, call)
99
+ ? (previous[index] ?? 0) + 1
100
+ : Math.max(previous[index + 1] ?? 0, current[index] ?? 0),
101
+ );
102
+ previous = current;
103
+ }
104
+ return previous.at(-1) ?? 0;
105
+ }
106
+
107
+ /** A greedy match fails when a broad argument subset takes the only match for a narrow one.
108
+ * Reassign matches along an augmenting path so duplicates and overlapping subsets are correct. */
109
+ function unorderedOverlap(
110
+ actual: readonly TrajectoryStep[],
111
+ expected: readonly ExpectedCall[],
112
+ ): number {
113
+ const owners = new Map<number, number>();
114
+ function assign(index: number, seen: Set<number>): boolean {
115
+ const wanted = expected[index];
116
+ if (!wanted) return false;
117
+ for (const [position, step] of actual.entries()) {
118
+ if (seen.has(position) || !matches(step, wanted)) continue;
119
+ seen.add(position);
120
+ const previous = owners.get(position);
121
+ if (previous === undefined || assign(previous, seen)) {
122
+ owners.set(position, index);
123
+ return true;
124
+ }
125
+ }
126
+ return false;
127
+ }
128
+ for (const index of expected.keys()) assign(index, new Set());
129
+ return owners.size;
130
+ }
131
+
132
+ export function matchTrajectory(
133
+ actual: Trajectory,
134
+ expected: TrajectoryExpectation,
135
+ ): { passed: boolean; score: number; reason?: string } {
136
+ const all = expected.scope === "tree" ? actual.flat : actual.steps;
137
+ const steps = all.filter(
138
+ (step) =>
139
+ expected.includeFailed ||
140
+ (step.kind === "remote-operation"
141
+ ? step.status === "accepted"
142
+ : step.workflowId !== undefined
143
+ ? step.status === "completed"
144
+ : step.result?.ok === true),
145
+ );
146
+ const calls = expected.calls ?? [];
147
+ const mode = expected.match ?? "inOrder";
148
+ const problems: string[] = [];
149
+ let overlap = 0;
150
+ let passed = true;
151
+ if (expected.calls !== undefined) {
152
+ if (mode === "exact") {
153
+ passed =
154
+ steps.length === calls.length &&
155
+ steps.every((step, index) => {
156
+ const call = calls[index];
157
+ return call !== undefined && matches(step, call);
158
+ });
159
+ overlap = passed ? calls.length : 0;
160
+ } else {
161
+ overlap = mode === "inOrder" ? lcs(steps, calls) : unorderedOverlap(steps, calls);
162
+ passed = mode === "subset" ? overlap === steps.length : overlap === calls.length;
163
+ }
164
+ if (!passed) problems.push("The recorded calls do not match the expected calls and arguments.");
165
+ }
166
+ // Prohibitions include failed and interrupted work at every depth, even when positive
167
+ // expectations deliberately inspect only successful root calls.
168
+ if (actual.flat.some((step) => expected.forbidden?.includes(step.name)))
169
+ problems.push("A forbidden tool or workflow step was recorded.");
170
+ const runs = new Map<string, TrajectoryStep[]>();
171
+ for (const step of actual.flat) {
172
+ const run = runs.get(step.runId) ?? [];
173
+ run.push(step);
174
+ runs.set(step.runId, run);
175
+ }
176
+ for (const run of runs.values()) {
177
+ const previousByParent = new Map<string | undefined, TrajectoryStep>();
178
+ if (
179
+ expected.noRepeats &&
180
+ run.some((step) => {
181
+ const previous = previousByParent.get(step.parentOperationId);
182
+ previousByParent.set(step.parentOperationId, step);
183
+ return (
184
+ step.attempts > 1 ||
185
+ (previous?.kind === step.kind &&
186
+ previous.name === step.name &&
187
+ equalData(previous.arguments, step.arguments) &&
188
+ equalData(previous.input, step.input))
189
+ );
190
+ })
191
+ )
192
+ problems.push("A tool or workflow step repeated the same input.");
193
+ if (
194
+ expected.forbiddenSequences?.some((sequence) =>
195
+ run.some((_step, index) =>
196
+ sequence.every((name, offset) => run[index + offset]?.name === name),
197
+ ),
198
+ )
199
+ )
200
+ problems.push("A forbidden sequence of calls was recorded.");
201
+ }
202
+ if (expected.maxToolCalls !== undefined && actual.toolCalls > expected.maxToolCalls)
203
+ problems.push("The tool-call budget was exceeded.");
204
+ if (expected.maxModelCalls !== undefined && actual.modelCalls > expected.maxModelCalls)
205
+ problems.push("The model-call budget was exceeded.");
206
+ const constraintsFailed = problems.some(
207
+ (problem) => problem !== "The recorded calls do not match the expected calls and arguments.",
208
+ );
209
+ const score = constraintsFailed
210
+ ? 0
211
+ : expected.calls === undefined
212
+ ? 1
213
+ : mode === "exact" || mode === "subset"
214
+ ? Number(passed)
215
+ : steps.length + calls.length === 0
216
+ ? 1
217
+ : (2 * overlap) / (steps.length + calls.length);
218
+ return {
219
+ passed: problems.length === 0,
220
+ score,
221
+ ...(problems.length ? { reason: [...new Set(problems)].join(" ") } : {}),
222
+ };
223
+ }
@@ -0,0 +1,463 @@
1
+ import type { ModelRate, Provider, TokenUsage } from "@providerkit/core";
2
+ import type { RemoteReceipt } from "../a2a/remote.ts";
3
+ import type {
4
+ Agent,
5
+ AgentHost,
6
+ AgentConfig,
7
+ AgentInput,
8
+ Decision,
9
+ ModelSettings,
10
+ UserContent,
11
+ AgentRef,
12
+ AskRecord,
13
+ RunOptions,
14
+ RunOutcome,
15
+ RunSnapshot,
16
+ StepExecution,
17
+ StepKind,
18
+ ToolResult,
19
+ } from "../index.ts";
20
+
21
+ export interface EvalTurn<E = unknown> {
22
+ input: UserContent;
23
+ expected?: E;
24
+ expectStatus?: RunOutcome["status"];
25
+ trajectory?: TrajectoryExpectation;
26
+ scorers?: readonly Scorer<E>[];
27
+ }
28
+
29
+ export interface SimulatedUser {
30
+ opening: UserContent;
31
+ plan: string;
32
+ persona?: string;
33
+ /** Target-agent turns, including opening. Default 10, maximum 100. */
34
+ maxTurns?: number;
35
+ }
36
+
37
+ export interface EvalCase<E = unknown> {
38
+ /** A public label, not a prompt or a person's identifier. Unique within the dataset. */
39
+ id: string;
40
+ /** Exactly one of input, turns and user. Turns share a fresh SDK memory session. */
41
+ input?: AgentInput;
42
+ turns?: readonly EvalTurn<E>[];
43
+ user?: SimulatedUser;
44
+ expected?: E;
45
+ expectStatus?: RunOutcome["status"];
46
+ trajectory?: TrajectoryExpectation;
47
+ scorers?: readonly Scorer<E>[];
48
+ }
49
+
50
+ type ScriptedDecision =
51
+ | {
52
+ ask: "approval";
53
+ decision: Extract<Decision, { type: "approve" | "deny" | "cancel" | "stop" }>;
54
+ }
55
+ | {
56
+ ask: "question";
57
+ key: string;
58
+ decision: Extract<Decision, { type: "answer" | "cancel" | "stop" }>;
59
+ };
60
+
61
+ /** Trusted test-driver data, never loaded by readCases. Each answer is consumed once.
62
+ * Match exact agent/tool or workflow/step identities and question keys, not request prose.
63
+ * More than one unused match leaves the request pending; script order grants no authority. */
64
+ export type ScriptedAnswer = {
65
+ /** Child keys from the root. Omitted means root-only, never any delegated child. */
66
+ path?: readonly string[];
67
+ } & (
68
+ | ({
69
+ agentId: string;
70
+ tool: string;
71
+ arguments?: Record<string, unknown>;
72
+ workflowId?: never;
73
+ step?: never;
74
+ input?: never;
75
+ } & (
76
+ | ScriptedDecision
77
+ | { ask: "client"; decision: Extract<Decision, { type: "result" | "cancel" | "stop" }> }
78
+ ))
79
+ | ({
80
+ workflowId: string;
81
+ step: string;
82
+ /** Exact JSON input recorded for this step, before any approval edits. */
83
+ input?: unknown;
84
+ agentId?: never;
85
+ tool?: never;
86
+ arguments?: never;
87
+ } & ScriptedDecision)
88
+ );
89
+
90
+ export interface ToolMock {
91
+ name: string;
92
+ arguments?: Record<string, unknown>;
93
+ result?: unknown;
94
+ error?: string;
95
+ }
96
+
97
+ export interface EvalDriver {
98
+ answers?: readonly ScriptedAnswer[];
99
+ /** Applied only by factory.stub(), including code-only bindings. Unmatched calls fail closed.
100
+ * Direct and code paths share the same replacement object and current host checks.
101
+ * Replaces validation and execution: this does not test the real backend's checks. */
102
+ toolMocks?: readonly ToolMock[];
103
+ }
104
+
105
+ export type EvalTools = NonNullable<AgentConfig["tools"]>;
106
+ export interface AgentFactoryContext<E = unknown> {
107
+ case: EvalCase<E>;
108
+ repeat: number;
109
+ signal: AbortSignal;
110
+ stub(tools: EvalTools): EvalTools;
111
+ }
112
+
113
+ export interface EvalDataset<E = unknown> {
114
+ name: string;
115
+ cases: readonly EvalCase<E>[];
116
+ scorers?: readonly Scorer<E>[];
117
+ }
118
+
119
+ export interface ExpectedCall {
120
+ /** Tool name or workflow step key. */
121
+ name: string;
122
+ arguments?: Record<string, unknown>;
123
+ argumentsMatch?: "subset" | "exact" | "ignore";
124
+ /** Exact workflow step input. Does not match agent tool arguments. */
125
+ input?: unknown;
126
+ /** At least one child must match. Other children still obey tree-wide prohibitions. */
127
+ inside?: TrajectoryExpectation;
128
+ }
129
+
130
+ export interface TrajectoryExpectation {
131
+ calls?: readonly ExpectedCall[];
132
+ match?: "exact" | "inOrder" | "anyOrder" | "subset";
133
+ scope?: "root" | "tree";
134
+ forbidden?: readonly string[];
135
+ forbiddenSequences?: readonly (readonly string[])[];
136
+ maxToolCalls?: number;
137
+ maxModelCalls?: number;
138
+ noRepeats?: boolean;
139
+ includeFailed?: boolean;
140
+ }
141
+
142
+ export type TrajectoryStep = {
143
+ /** Tool name or workflow step key. */
144
+ name: string;
145
+ /** Admitted effects/children, excluding reconciled not-run attempts. Questions and sleeps count once. */
146
+ attempts: number;
147
+ asks: AskRecord[];
148
+ runId: string;
149
+ operationId: string;
150
+ /** The real program operation for an inner tool, never a child run. */
151
+ parentOperationId?: string;
152
+ children: ({ key: string; runId: string; trajectory: Trajectory } & (
153
+ | { kind: "agent"; agentId: string }
154
+ | { kind: "remote"; agentId: string; version: string }
155
+ | { kind: "workflow"; workflowId: string }
156
+ ))[];
157
+ } & (
158
+ | {
159
+ kind: "tool" | "code" | "delegation" | "handoff";
160
+ arguments: Record<string, unknown>;
161
+ status: "pending" | "running" | "completed" | "interrupted";
162
+ result?: ToolResult;
163
+ agentId: string;
164
+ turn: number;
165
+ workflowId?: never;
166
+ input?: never;
167
+ output?: never;
168
+ }
169
+ | {
170
+ /** A remote receipt, not a workflow's remote-child step. Acceptance is not task completion. */
171
+ kind: "remote-operation";
172
+ action: RemoteReceipt["action"];
173
+ status: RemoteReceipt["status"];
174
+ billing: "unknown";
175
+ agentId: string;
176
+ version: string;
177
+ attemptId: string;
178
+ workflowId?: never;
179
+ /** Message parts only, without transport IDs. Kept out of built-in judge prompts. */
180
+ input?: unknown;
181
+ output?: never;
182
+ arguments?: never;
183
+ result?: never;
184
+ turn?: never;
185
+ }
186
+ | {
187
+ kind: StepKind;
188
+ workflowId: string;
189
+ input?: unknown;
190
+ output?: unknown;
191
+ status: StepExecution["status"];
192
+ arguments?: never;
193
+ result?: never;
194
+ agentId?: never;
195
+ turn?: never;
196
+ }
197
+ );
198
+
199
+ export interface Trajectory {
200
+ /** This run's operations in admission order, including inner calls after their code operation. */
201
+ steps: TrajectoryStep[];
202
+ flat: TrajectoryStep[];
203
+ /** SDK admissions, including interrupted attempts, not provider-internal retries. */
204
+ modelCalls: number;
205
+ /** Agent tool calls and coordination attempts, including inner code calls but not workflow
206
+ * steps or submit_result. Failed/interrupted attempts count; pending calls have no current receipt. */
207
+ toolCalls: number;
208
+ }
209
+
210
+ export interface ModelRef {
211
+ agentId: string;
212
+ provider?: string;
213
+ model?: string;
214
+ }
215
+
216
+ export type EvalPrice = (model: ModelRef) => ModelRate | undefined;
217
+
218
+ export interface EvalUsage {
219
+ usage: TokenUsage;
220
+ /** Sum of available bills/estimates, including partial usage. Not a full bill. */
221
+ knownCostUsd: number;
222
+ /** Null if any model attempt or remote operation has unknown billing, or a model lacks a price. */
223
+ costUsd: number | null;
224
+ unknownCharges: number;
225
+ unpricedCalls: number;
226
+ }
227
+
228
+ export interface EvalRun extends EvalUsage {
229
+ runId: string;
230
+ agentId: string;
231
+ version: string;
232
+ input: AgentInput;
233
+ status: RunSnapshot["state"]["status"];
234
+ /** The journal's JSON candidate, not a second call to the agent's output validator. */
235
+ value?: unknown;
236
+ text: string;
237
+ snapshot: RunSnapshot;
238
+ /** The admitted handoff chain's owner, never an ordinary delegated specialist. */
239
+ activeAgent: AgentRef;
240
+ trajectory: Trajectory;
241
+ /** Stored journals have no completion timestamp. Never inferred from the current time. */
242
+ durationMs?: number;
243
+ }
244
+
245
+ export type ScoreValue = boolean | number | string;
246
+ export type NamedScore = ScoreValue | { value: ScoreValue; reason?: string };
247
+ // Adapted from Mastra, packages/core/src/evals/not-scorable.ts (Apache-2.0).
248
+ // Copyright (c) 2025 Kepler Software, Inc. Changed: required reason and our own brand.
249
+ export const SKIP: unique symbol = Symbol.for("providerkit.evals.skip");
250
+ export interface Skip {
251
+ readonly [SKIP]: true;
252
+ readonly reason: string;
253
+ }
254
+ export type ScorerOutput = NamedScore | Skip | Record<string, NamedScore>;
255
+ export type ScoreThreshold = number | { min?: number; max?: number };
256
+
257
+ export interface ScoreContext<E = unknown> {
258
+ case: EvalCase<E>;
259
+ /** One-based case turn. Absent for scoreRun on a stored journal. */
260
+ turn?: number;
261
+ runs: readonly EvalRun[];
262
+ expected?: E;
263
+ run: EvalRun;
264
+ signal: AbortSignal;
265
+ /** Only available with allowJudge. Shares this case's cancellation and spend accounting.
266
+ * Helpers are serial and maxJudgeRuns bounds their number. No automatic resume or retry. */
267
+ runAgent<R>(agent: Agent<R>, input: AgentInput): Promise<RunOutcome<R>>;
268
+ }
269
+
270
+ export interface Scorer<E = unknown> {
271
+ readonly name: string;
272
+ readonly threshold?: ScoreThreshold;
273
+ readonly gate?: boolean;
274
+ score(context: ScoreContext<E>): ScorerOutput | Promise<ScorerOutput>;
275
+ }
276
+
277
+ export interface ScoreResult {
278
+ turn?: number;
279
+ scorer: string;
280
+ name: string;
281
+ gate: boolean;
282
+ kind: "assertion" | "score" | "label" | "skipped" | "error";
283
+ /** Label text, reasons and exception messages require includePrivateData. */
284
+ value?: ScoreValue;
285
+ passed?: boolean;
286
+ reason?: string;
287
+ error?: "scorer_failed" | "invalid_score" | "cancelled";
288
+ }
289
+
290
+ /** New executions only. Only the separate trusted driver may answer; stored effects never resume. */
291
+ export type EvalHost = Pick<
292
+ AgentHost,
293
+ | "checkpoint"
294
+ | "loadMedia"
295
+ | "prepareRequest"
296
+ | "needsApproval"
297
+ | "beforeTool"
298
+ | "beforeStep"
299
+ | "canRunConcurrently"
300
+ >;
301
+
302
+ export interface ScoringOptions<E = unknown> {
303
+ case?: EvalCase<E>;
304
+ price?: EvalPrice;
305
+ signal?: AbortSignal;
306
+ /** Explicit opt-in, also required by scoreRun and createLiveScorer. */
307
+ allowJudge?: boolean;
308
+ /** Helper agent runs per case, default 3. Each built-in judge also has SDK model-call limits. */
309
+ maxJudgeRuns?: number;
310
+ /** Default 60 seconds, including custom scorers. Cancellation is cooperative. */
311
+ timeoutMs?: number;
312
+ /** Includes raw data and reasons. Keep such reports in trusted storage. Default false. */
313
+ includePrivateData?: boolean;
314
+ }
315
+
316
+ export interface EvalOptions<T = string, E = unknown>
317
+ extends EvalDataset<E>, Omit<ScoringOptions<E>, "case"> {
318
+ /** Cases start from an agent. Its delegated tree may contain workflows and remote agents. */
319
+ agent: Agent<T> | ((context: AgentFactoryContext<E>) => Agent<T> | Promise<Agent<T>>);
320
+ /** Explicit routing after a handoff, by both id and version. Missing targets stop the case. */
321
+ resolveAgent?: (ref: AgentRef) => Agent<T> | Promise<Agent<T>>;
322
+ /** Trusted scripts keyed by case id. Dataset files cannot set these. */
323
+ drivers?: Readonly<Record<string, EvalDriver>>;
324
+ /** Separate paid-work consent from allowJudge. No default simulator provider. */
325
+ allowSimulation?: boolean;
326
+ user?: {
327
+ provider: Provider;
328
+ modelSettings?: Omit<ModelSettings, "toolChoice" | "hostedTools" | "maxAttempts">;
329
+ };
330
+ repeat?: number;
331
+ k?: readonly number[];
332
+ /** Active case slots, default 1. External work that ignores abort may continue. */
333
+ concurrency?: number;
334
+ /** Admission/spend thresholds, NOT hard monetary or token caps. On reaching one, no new
335
+ * case/helper starts and in-flight work is signalled to stop. SDK limits still bound each
336
+ * tree. Provider retries, hosted charges and ignored cancellation can exceed thresholds. */
337
+ budget?: { usd?: number; tokens?: number };
338
+ run?: Pick<RunOptions, "context" | "modelSettings" | "enabledTools">;
339
+ host?: EvalHost;
340
+ }
341
+
342
+ export type EvalStatus = "passed" | "failed" | "error" | "unscored" | "cancelled" | "skipped";
343
+
344
+ export interface EvalRunResult extends EvalUsage {
345
+ runId: string;
346
+ attemptId: string;
347
+ agent: AgentRef;
348
+ activeAgent: AgentRef;
349
+ role: "agent" | "judge" | "user";
350
+ turn?: number;
351
+ status: EvalRun["status"];
352
+ modelCalls: number;
353
+ toolCalls: number;
354
+ durationMs?: number;
355
+ /** Model attempts, including interrupted ones. No prompts or tool results. */
356
+ modelAttempts: {
357
+ operationId: string;
358
+ attemptId: string;
359
+ status: "running" | "completed" | "interrupted";
360
+ billing: "reported" | "unknown";
361
+ usage: TokenUsage;
362
+ }[];
363
+ toolAttempts: {
364
+ operationId: string;
365
+ parentOperationId?: string;
366
+ attemptId: string;
367
+ status: "running" | "completed" | "interrupted";
368
+ ok?: boolean;
369
+ }[];
370
+ /** Remote sends and cancellation requests, not local model or tool attempts. No peer endpoints,
371
+ * callbacks, message bodies or task/context handles. Accepted does not mean cancelled. */
372
+ remoteAttempts: (Pick<RemoteReceipt, "kind" | "action" | "attemptId" | "status" | "billing"> & {
373
+ agentId: string;
374
+ version: string;
375
+ runId: string;
376
+ operationId: string;
377
+ })[];
378
+ /** Present only after explicit consent to include private data. */
379
+ snapshot?: RunSnapshot;
380
+ }
381
+
382
+ export interface EvalTurnResult {
383
+ index: number;
384
+ status: EvalStatus;
385
+ runId?: string;
386
+ scores: ScoreResult[];
387
+ problems: string[];
388
+ }
389
+
390
+ export interface CaseResult {
391
+ id: string;
392
+ repeat: number;
393
+ status: EvalStatus;
394
+ problems: string[];
395
+ /** Last target run, for the single-line report. runs retains all target and helper work. */
396
+ run?: EvalRunResult;
397
+ runs: EvalRunResult[];
398
+ turns: EvalTurnResult[];
399
+ simulation?: "finished" | "turn_limit" | "incomplete";
400
+ scores: ScoreResult[];
401
+ agentUsage: EvalUsage;
402
+ judgeUsage: EvalUsage;
403
+ userUsage: EvalUsage;
404
+ durationMs: number;
405
+ }
406
+
407
+ export interface EvalCounts {
408
+ total: number;
409
+ passed: number;
410
+ failed: number;
411
+ errors: number;
412
+ unscored: number;
413
+ cancelled: number;
414
+ skipped: number;
415
+ decided: number;
416
+ /** Passed / decided. Always read alongside decided and total. */
417
+ passRate: number | null;
418
+ /** Passed / total, including errors and unscored work in the denominator. */
419
+ successRate: number | null;
420
+ }
421
+
422
+ export interface ScoreAggregate {
423
+ name: string;
424
+ scorer: string;
425
+ measured: number;
426
+ skipped: number;
427
+ errors: number;
428
+ mean: number | null;
429
+ passRate: number | null;
430
+ }
431
+
432
+ export interface EvalGroup extends EvalCounts {
433
+ id: string;
434
+ passAt: Record<string, number | null>;
435
+ passHat: Record<string, number | null>;
436
+ }
437
+
438
+ export interface EvalReport {
439
+ format: 1;
440
+ name: string;
441
+ startedAt: number;
442
+ durationMs: number;
443
+ options: { repeat: number; k: number[]; concurrency: number; budget?: EvalOptions["budget"] };
444
+ cases: CaseResult[];
445
+ groups: EvalGroup[];
446
+ summary: EvalCounts & {
447
+ scores: ScoreAggregate[];
448
+ runStatuses: Record<string, number>;
449
+ agentUsage: EvalUsage;
450
+ judgeUsage: EvalUsage;
451
+ userUsage: EvalUsage;
452
+ turns: (EvalCounts & { index: number; scores: ScoreAggregate[] })[];
453
+ };
454
+ stopped?: { reason: "budget" | "unknown_cost" | "cancelled"; message: string };
455
+ }
456
+
457
+ export interface ScoredRun {
458
+ runId: string;
459
+ status: EvalRun["status"];
460
+ scores: ScoreResult[];
461
+ agentUsage: EvalUsage;
462
+ judgeUsage: EvalUsage;
463
+ }