@providerkit/agents 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. package/LICENSE-APACHE-2.0 +202 -0
  2. package/README.md +2108 -52
  3. package/THIRD_PARTY_NOTICES.md +521 -0
  4. package/dist/a2a/auth.d.ts +10 -0
  5. package/dist/a2a/auth.d.ts.map +1 -0
  6. package/dist/a2a/auth.js +52 -0
  7. package/dist/a2a/auth.js.map +1 -0
  8. package/dist/a2a/client.d.ts +74 -0
  9. package/dist/a2a/client.d.ts.map +1 -0
  10. package/dist/a2a/client.js +601 -0
  11. package/dist/a2a/client.js.map +1 -0
  12. package/dist/a2a/index.d.ts +22 -0
  13. package/dist/a2a/index.d.ts.map +1 -0
  14. package/dist/a2a/index.js +11 -0
  15. package/dist/a2a/index.js.map +1 -0
  16. package/dist/a2a/memory-runs.d.ts +62 -0
  17. package/dist/a2a/memory-runs.d.ts.map +1 -0
  18. package/dist/a2a/memory-runs.js +349 -0
  19. package/dist/a2a/memory-runs.js.map +1 -0
  20. package/dist/a2a/policy.d.ts +15 -0
  21. package/dist/a2a/policy.d.ts.map +1 -0
  22. package/dist/a2a/policy.js +114 -0
  23. package/dist/a2a/policy.js.map +1 -0
  24. package/dist/a2a/push.d.ts +34 -0
  25. package/dist/a2a/push.d.ts.map +1 -0
  26. package/dist/a2a/push.js +180 -0
  27. package/dist/a2a/push.js.map +1 -0
  28. package/dist/a2a/remote.d.ts +155 -0
  29. package/dist/a2a/remote.d.ts.map +1 -0
  30. package/dist/a2a/remote.js +1196 -0
  31. package/dist/a2a/remote.js.map +1 -0
  32. package/dist/a2a/server.d.ts +110 -0
  33. package/dist/a2a/server.d.ts.map +1 -0
  34. package/dist/a2a/server.js +1139 -0
  35. package/dist/a2a/server.js.map +1 -0
  36. package/dist/a2a/signing.d.ts +21 -0
  37. package/dist/a2a/signing.d.ts.map +1 -0
  38. package/dist/a2a/signing.js +305 -0
  39. package/dist/a2a/signing.js.map +1 -0
  40. package/dist/a2a/tasks.d.ts +88 -0
  41. package/dist/a2a/tasks.d.ts.map +1 -0
  42. package/dist/a2a/tasks.js +170 -0
  43. package/dist/a2a/tasks.js.map +1 -0
  44. package/dist/a2a/wire.d.ts +267 -0
  45. package/dist/a2a/wire.d.ts.map +1 -0
  46. package/dist/a2a/wire.js +1202 -0
  47. package/dist/a2a/wire.js.map +1 -0
  48. package/dist/background-types.d.ts +76 -0
  49. package/dist/background-types.d.ts.map +1 -0
  50. package/dist/background-types.js +2 -0
  51. package/dist/background-types.js.map +1 -0
  52. package/dist/child.d.ts +25 -0
  53. package/dist/child.d.ts.map +1 -0
  54. package/dist/child.js +129 -0
  55. package/dist/child.js.map +1 -0
  56. package/dist/code-driver.d.ts +40 -0
  57. package/dist/code-driver.d.ts.map +1 -0
  58. package/dist/code-driver.js +2 -0
  59. package/dist/code-driver.js.map +1 -0
  60. package/dist/code-mode.d.ts +3 -0
  61. package/dist/code-mode.d.ts.map +1 -0
  62. package/dist/code-mode.js +2 -0
  63. package/dist/code-mode.js.map +1 -0
  64. package/dist/code-script.d.ts +3 -0
  65. package/dist/code-script.d.ts.map +1 -0
  66. package/dist/code-script.js +149 -0
  67. package/dist/code-script.js.map +1 -0
  68. package/dist/code.d.ts +75 -0
  69. package/dist/code.d.ts.map +1 -0
  70. package/dist/code.js +326 -0
  71. package/dist/code.js.map +1 -0
  72. package/dist/compaction.d.ts +15 -0
  73. package/dist/compaction.d.ts.map +1 -0
  74. package/dist/compaction.js +168 -0
  75. package/dist/compaction.js.map +1 -0
  76. package/dist/delegation.d.ts +85 -16
  77. package/dist/delegation.d.ts.map +1 -1
  78. package/dist/delegation.js +240 -106
  79. package/dist/delegation.js.map +1 -1
  80. package/dist/evals/driver.d.ts +10 -0
  81. package/dist/evals/driver.d.ts.map +1 -0
  82. package/dist/evals/driver.js +192 -0
  83. package/dist/evals/driver.js.map +1 -0
  84. package/dist/evals/journal.d.ts +26 -0
  85. package/dist/evals/journal.d.ts.map +1 -0
  86. package/dist/evals/journal.js +483 -0
  87. package/dist/evals/journal.js.map +1 -0
  88. package/dist/evals/judge.d.ts +31 -0
  89. package/dist/evals/judge.d.ts.map +1 -0
  90. package/dist/evals/judge.js +142 -0
  91. package/dist/evals/judge.js.map +1 -0
  92. package/dist/evals/report.d.ts +38 -0
  93. package/dist/evals/report.d.ts.map +1 -0
  94. package/dist/evals/report.js +241 -0
  95. package/dist/evals/report.js.map +1 -0
  96. package/dist/evals/runner.d.ts +24 -0
  97. package/dist/evals/runner.d.ts.map +1 -0
  98. package/dist/evals/runner.js +655 -0
  99. package/dist/evals/runner.js.map +1 -0
  100. package/dist/evals/scorers.d.ts +36 -0
  101. package/dist/evals/scorers.d.ts.map +1 -0
  102. package/dist/evals/scorers.js +259 -0
  103. package/dist/evals/scorers.js.map +1 -0
  104. package/dist/evals/simulation.d.ts +7 -0
  105. package/dist/evals/simulation.d.ts.map +1 -0
  106. package/dist/evals/simulation.js +38 -0
  107. package/dist/evals/simulation.js.map +1 -0
  108. package/dist/evals/testing.d.ts +36 -0
  109. package/dist/evals/testing.d.ts.map +1 -0
  110. package/dist/evals/testing.js +87 -0
  111. package/dist/evals/testing.js.map +1 -0
  112. package/dist/evals/trajectory.d.ts +11 -0
  113. package/dist/evals/trajectory.d.ts.map +1 -0
  114. package/dist/evals/trajectory.js +180 -0
  115. package/dist/evals/trajectory.js.map +1 -0
  116. package/dist/evals/types.d.ts +431 -0
  117. package/dist/evals/types.d.ts.map +1 -0
  118. package/dist/evals/types.js +4 -0
  119. package/dist/evals/types.js.map +1 -0
  120. package/dist/evals.d.ts +14 -0
  121. package/dist/evals.d.ts.map +1 -0
  122. package/dist/evals.js +9 -0
  123. package/dist/evals.js.map +1 -0
  124. package/dist/extensions.d.ts +33 -0
  125. package/dist/extensions.d.ts.map +1 -0
  126. package/dist/extensions.js +188 -0
  127. package/dist/extensions.js.map +1 -0
  128. package/dist/handoff.d.ts +58 -0
  129. package/dist/handoff.d.ts.map +1 -0
  130. package/dist/handoff.js +160 -0
  131. package/dist/handoff.js.map +1 -0
  132. package/dist/index.d.ts +14 -0
  133. package/dist/index.d.ts.map +1 -1
  134. package/dist/index.js +8 -0
  135. package/dist/index.js.map +1 -1
  136. package/dist/input.d.ts +42 -0
  137. package/dist/input.d.ts.map +1 -0
  138. package/dist/input.js +266 -0
  139. package/dist/input.js.map +1 -0
  140. package/dist/mcp.d.ts +77 -0
  141. package/dist/mcp.d.ts.map +1 -0
  142. package/dist/mcp.js +505 -0
  143. package/dist/mcp.js.map +1 -0
  144. package/dist/memory-store.d.ts +11 -0
  145. package/dist/memory-store.d.ts.map +1 -0
  146. package/dist/memory-store.js +272 -0
  147. package/dist/memory-store.js.map +1 -0
  148. package/dist/memory.d.ts +212 -0
  149. package/dist/memory.d.ts.map +1 -0
  150. package/dist/memory.js +613 -0
  151. package/dist/memory.js.map +1 -0
  152. package/dist/otel.d.ts +74 -0
  153. package/dist/otel.d.ts.map +1 -0
  154. package/dist/otel.js +427 -0
  155. package/dist/otel.js.map +1 -0
  156. package/dist/progress.d.ts +17 -0
  157. package/dist/progress.d.ts.map +1 -0
  158. package/dist/progress.js +193 -0
  159. package/dist/progress.js.map +1 -0
  160. package/dist/quickjs.d.ts +19 -0
  161. package/dist/quickjs.d.ts.map +1 -0
  162. package/dist/quickjs.js +519 -0
  163. package/dist/quickjs.js.map +1 -0
  164. package/dist/recovery.d.ts +10 -0
  165. package/dist/recovery.d.ts.map +1 -0
  166. package/dist/recovery.js +62 -0
  167. package/dist/recovery.js.map +1 -0
  168. package/dist/request.d.ts +88 -0
  169. package/dist/request.d.ts.map +1 -0
  170. package/dist/request.js +422 -0
  171. package/dist/request.js.map +1 -0
  172. package/dist/run.d.ts +16 -4
  173. package/dist/run.d.ts.map +1 -1
  174. package/dist/run.js +2016 -245
  175. package/dist/run.js.map +1 -1
  176. package/dist/runs.d.ts +77 -34
  177. package/dist/runs.d.ts.map +1 -1
  178. package/dist/runs.js +267 -85
  179. package/dist/runs.js.map +1 -1
  180. package/dist/schema-to-ts.d.ts +4 -0
  181. package/dist/schema-to-ts.d.ts.map +1 -0
  182. package/dist/schema-to-ts.js +191 -0
  183. package/dist/schema-to-ts.js.map +1 -0
  184. package/dist/session.d.ts +23 -0
  185. package/dist/session.d.ts.map +1 -0
  186. package/dist/session.js +137 -0
  187. package/dist/session.js.map +1 -0
  188. package/dist/snapshot.d.ts +57 -8
  189. package/dist/snapshot.d.ts.map +1 -1
  190. package/dist/snapshot.js +1206 -109
  191. package/dist/snapshot.js.map +1 -1
  192. package/dist/steering.d.ts +75 -0
  193. package/dist/steering.d.ts.map +1 -0
  194. package/dist/steering.js +214 -0
  195. package/dist/steering.js.map +1 -0
  196. package/dist/subagents.d.ts +141 -0
  197. package/dist/subagents.d.ts.map +1 -0
  198. package/dist/subagents.js +746 -0
  199. package/dist/subagents.js.map +1 -0
  200. package/dist/tool-retry.d.ts +12 -0
  201. package/dist/tool-retry.d.ts.map +1 -0
  202. package/dist/tool-retry.js +53 -0
  203. package/dist/tool-retry.js.map +1 -0
  204. package/dist/tree.d.ts +62 -6
  205. package/dist/tree.d.ts.map +1 -1
  206. package/dist/tree.js +192 -23
  207. package/dist/tree.js.map +1 -1
  208. package/dist/types.d.ts +767 -52
  209. package/dist/types.d.ts.map +1 -1
  210. package/dist/workflow-snapshot.d.ts +8 -0
  211. package/dist/workflow-snapshot.d.ts.map +1 -0
  212. package/dist/workflow-snapshot.js +276 -0
  213. package/dist/workflow-snapshot.js.map +1 -0
  214. package/dist/workflow-types.d.ts +171 -0
  215. package/dist/workflow-types.d.ts.map +1 -0
  216. package/dist/workflow-types.js +2 -0
  217. package/dist/workflow-types.js.map +1 -0
  218. package/dist/workflow.d.ts +19 -0
  219. package/dist/workflow.d.ts.map +1 -0
  220. package/dist/workflow.js +1053 -0
  221. package/dist/workflow.js.map +1 -0
  222. package/examples/a2a-remote.ts +85 -0
  223. package/examples/a2a-server.ts +100 -0
  224. package/examples/background.ts +18 -4
  225. package/examples/code-mode.ts +76 -0
  226. package/examples/conversation.ts +57 -0
  227. package/examples/evals.ts +79 -0
  228. package/examples/guardrails.ts +56 -0
  229. package/examples/handoff.ts +79 -0
  230. package/examples/memory.ts +44 -0
  231. package/examples/otel.ts +99 -0
  232. package/examples/steering.ts +69 -0
  233. package/examples/structured-handoff.ts +92 -0
  234. package/examples/subagents.ts +127 -0
  235. package/examples/workflow.ts +74 -0
  236. package/package.json +58 -6
  237. package/src/a2a/auth.ts +62 -0
  238. package/src/a2a/client.ts +733 -0
  239. package/src/a2a/index.ts +74 -0
  240. package/src/a2a/memory-runs.ts +448 -0
  241. package/src/a2a/policy.ts +148 -0
  242. package/src/a2a/push.ts +251 -0
  243. package/src/a2a/remote.ts +1652 -0
  244. package/src/a2a/server.ts +1390 -0
  245. package/src/a2a/signing.ts +348 -0
  246. package/src/a2a/tasks.ts +260 -0
  247. package/src/a2a/wire.ts +1443 -0
  248. package/src/background-types.ts +86 -0
  249. package/src/child.ts +181 -0
  250. package/src/code-driver.ts +48 -0
  251. package/src/code-mode.ts +12 -0
  252. package/src/code-script.ts +149 -0
  253. package/src/code.ts +424 -0
  254. package/src/compaction.ts +212 -0
  255. package/src/delegation.ts +444 -155
  256. package/src/evals/driver.ts +222 -0
  257. package/src/evals/journal.ts +605 -0
  258. package/src/evals/judge.ts +188 -0
  259. package/src/evals/report.ts +295 -0
  260. package/src/evals/runner.ts +826 -0
  261. package/src/evals/scorers.ts +339 -0
  262. package/src/evals/simulation.ts +46 -0
  263. package/src/evals/testing.ts +126 -0
  264. package/src/evals/trajectory.ts +223 -0
  265. package/src/evals/types.ts +463 -0
  266. package/src/evals.ts +72 -0
  267. package/src/extensions.ts +247 -0
  268. package/src/handoff.ts +214 -0
  269. package/src/index.ts +51 -0
  270. package/src/input.ts +319 -0
  271. package/src/mcp.ts +690 -0
  272. package/src/memory-store.ts +366 -0
  273. package/src/memory.ts +893 -0
  274. package/src/otel.ts +568 -0
  275. package/src/progress.ts +214 -0
  276. package/src/quickjs.ts +524 -0
  277. package/src/recovery.ts +90 -0
  278. package/src/request.ts +513 -0
  279. package/src/run.ts +2678 -313
  280. package/src/runs.ts +464 -133
  281. package/src/schema-to-ts.ts +194 -0
  282. package/src/session.ts +178 -0
  283. package/src/snapshot.ts +1595 -124
  284. package/src/steering.ts +329 -0
  285. package/src/subagents.ts +1084 -0
  286. package/src/tool-retry.ts +86 -0
  287. package/src/tree.ts +223 -27
  288. package/src/types.ts +711 -52
  289. package/src/workflow-snapshot.ts +366 -0
  290. package/src/workflow-types.ts +213 -0
  291. package/src/workflow.ts +1316 -0
@@ -0,0 +1,483 @@
1
+ import { addUsage, costUsd, EMPTY_USAGE, HOSTED_TOOL_TYPES, parseToolArgs, } from "@providerkit/core";
2
+ import { RESULT_TOOL_NAME, } from "../index.js";
3
+ import { allModels } from "../compaction.js";
4
+ import { inputOf, readInput } from "../input.js";
5
+ import { agentSnapshot, copyData, isId, isRecord, isUsage, jsonArguments, readLimits, readSnapshot, } from "../snapshot.js";
6
+ import { snapshots } from "../tree.js";
7
+ import { checkExpectation } from "./trajectory.js";
8
+ /** Uses the SDK's structural and journal checks, never resume or an output validator. A stored
9
+ * journal is data, not permission to run a tool. Unknown output contracts stay unknown JSON. */
10
+ export function readJournal(raw) {
11
+ const value = isRecord(raw) && "snapshot" in raw ? raw.snapshot : raw;
12
+ if (!isRecord(value) || value.kind !== "agent" || !isId(value.agentId) || !isId(value.version))
13
+ throw new TypeError("Provide a format-3 agent run journal or an SDK agent outcome.");
14
+ const limits = readLimits(value.limits);
15
+ const definition = value.definition;
16
+ const checkedOutput = isRecord(definition) &&
17
+ Array.isArray(definition.enabledTools) &&
18
+ definition.enabledTools.includes(RESULT_TOOL_NAME);
19
+ const root = readSnapshot(value, value.agentId, value.version, limits, checkedOutput ? RESULT_TOOL_NAME : undefined);
20
+ // readSnapshot checks the mixed tree's shape and every workflow's operations. Agent
21
+ // descendants need their own operation checks, including those below workflow steps.
22
+ for (const node of snapshots(root)) {
23
+ if (node.kind !== "agent")
24
+ continue;
25
+ if (node !== root)
26
+ readSnapshot(node, node.agentId, node.version, node.limits, node.definition.enabledTools.includes(RESULT_TOOL_NAME) ? RESULT_TOOL_NAME : undefined);
27
+ for (const model of allModels(node)) {
28
+ checkUsage(model.usage);
29
+ for (const attempt of model.interrupted)
30
+ checkUsage(attempt.usage);
31
+ }
32
+ }
33
+ return root;
34
+ }
35
+ function checkUsage(value) {
36
+ if (!isUsage(value) ||
37
+ (value.hostedCalls !== undefined &&
38
+ (!isRecord(value.hostedCalls) ||
39
+ Object.entries(value.hostedCalls).some(([key, count]) => !HOSTED_TOOL_TYPES.some((type) => type === key) ||
40
+ typeof count !== "number" ||
41
+ !Number.isFinite(count) ||
42
+ count < 0))))
43
+ throw new TypeError("The journal has invalid token or hosted-call usage.");
44
+ }
45
+ export function emptyUsage() {
46
+ return {
47
+ usage: { ...EMPTY_USAGE },
48
+ knownCostUsd: 0,
49
+ costUsd: 0,
50
+ unknownCharges: 0,
51
+ unpricedCalls: 0,
52
+ };
53
+ }
54
+ export function combineUsage(values) {
55
+ const result = emptyUsage();
56
+ for (const value of values) {
57
+ result.usage = addUsage(result.usage, value.usage);
58
+ result.knownCostUsd += value.knownCostUsd;
59
+ result.unknownCharges += value.unknownCharges;
60
+ result.unpricedCalls += value.unpricedCalls;
61
+ }
62
+ result.costUsd = result.unknownCharges || result.unpricedCalls ? null : result.knownCostUsd;
63
+ return result;
64
+ }
65
+ export function usageOf(root, price) {
66
+ const totals = emptyUsage();
67
+ const seen = new Set();
68
+ for (const node of snapshots(root)) {
69
+ if (node.kind === "remote") {
70
+ // The peer supplied no local model bill. Each admitted send/cancel remains unknown.
71
+ totals.unknownCharges += node.remote.sends.length + (node.remote.cancel ? 1 : 0);
72
+ continue;
73
+ }
74
+ if (node.kind !== "agent")
75
+ continue;
76
+ for (const model of allModels(node)) {
77
+ if (seen.has(model.id))
78
+ continue;
79
+ seen.add(model.id);
80
+ const ref = {
81
+ agentId: node.agentId,
82
+ ...(model.status === "completed" ? model.turn.source : undefined),
83
+ };
84
+ for (const attempt of [
85
+ ...model.interrupted.map((attempt) => ({ ...attempt, billing: "unknown" })),
86
+ model,
87
+ ]) {
88
+ checkUsage(attempt.usage);
89
+ totals.usage = addUsage(totals.usage, attempt.usage);
90
+ if (attempt.billing === "unknown")
91
+ totals.unknownCharges++;
92
+ const measured = attempt.usage;
93
+ if (measured.reportedCostUsd !== undefined) {
94
+ totals.knownCostUsd += measured.reportedCostUsd;
95
+ continue;
96
+ }
97
+ // A reserved operation with no usage has not given us a priceable measurement yet.
98
+ const hasMeasurement = measured.inputTokens + measured.outputTokens > 0 ||
99
+ Object.values(measured.hostedCalls ?? {}).some((n) => n > 0);
100
+ if (!hasMeasurement && model.status === "running" && root.state.status === "running")
101
+ continue;
102
+ // An interrupted attempt can have used a different fallback route. The replacement
103
+ // turn's source is not evidence of which model served the lost attempt.
104
+ const rate = price?.(attempt === model ? ref : { agentId: node.agentId });
105
+ if (rate &&
106
+ [
107
+ rate.input,
108
+ rate.output,
109
+ rate.cacheRead,
110
+ rate.cacheWrite ?? rate.input,
111
+ ...Object.values(rate.hosted ?? {}),
112
+ ].some((n) => !Number.isFinite(n) || n < 0))
113
+ throw new TypeError("Prices must be finite, non-negative USD rates.");
114
+ const missingHostedRate = HOSTED_TOOL_TYPES.some((type) => (measured.hostedCalls?.[type] ?? 0) > 0 && rate?.hosted?.[type] === undefined);
115
+ const settled = attempt !== model || model.status === "completed" || root.state.status !== "running";
116
+ // A running receipt has no source yet. Do not reject a model-specific caller price
117
+ // before the completed turn identifies the model. The unknown charge still counts.
118
+ if ((!rate || missingHostedRate) && settled)
119
+ totals.unpricedCalls++;
120
+ if (rate)
121
+ totals.knownCostUsd += costUsd(measured, rate);
122
+ }
123
+ }
124
+ }
125
+ totals.costUsd = totals.unknownCharges || totals.unpricedCalls ? null : totals.knownCostUsd;
126
+ return totals;
127
+ }
128
+ function childTrajectory({ key, snapshot, }) {
129
+ return {
130
+ key,
131
+ ...(snapshot.kind === "workflow"
132
+ ? { kind: "workflow", workflowId: snapshot.workflowId }
133
+ : snapshot.kind === "remote"
134
+ ? { kind: "remote", agentId: snapshot.agentId, version: snapshot.version }
135
+ : { kind: "agent", agentId: snapshot.agentId }),
136
+ runId: snapshot.runId,
137
+ trajectory: journalTrajectory(snapshot),
138
+ };
139
+ }
140
+ function remoteAttempts(snapshot) {
141
+ return [
142
+ ...snapshot.remote.sends.map((send) => ({ ...send, action: "send" })),
143
+ ...(snapshot.remote.cancel ? [{ ...snapshot.remote.cancel, action: "cancel" }] : []),
144
+ ].map((receipt) => ({
145
+ kind: "remote",
146
+ action: receipt.action,
147
+ agentId: snapshot.agentId,
148
+ version: snapshot.version,
149
+ runId: snapshot.runId,
150
+ operationId: receipt.id,
151
+ attemptId: receipt.attemptId,
152
+ status: receipt.status,
153
+ billing: "unknown",
154
+ }));
155
+ }
156
+ function approvedInput(input, asks = []) {
157
+ let effective = input;
158
+ for (const ask of asks) {
159
+ if (ask.decision?.type === "approve")
160
+ effective = ask.decision.arguments ?? input;
161
+ }
162
+ return effective;
163
+ }
164
+ // Admission reserves budget for a pending ask; evals count only journaled attempt receipts.
165
+ function toolAttempts(tool) {
166
+ return tool.interrupted.length + tool.retries.length + (tool.status === "pending" ? 0 : 1);
167
+ }
168
+ /** Admission order comes from the journal, never completion time. Workflow steps retain their
169
+ * own kind and input; only agent nodes contribute model/tool calls. Hosted charges enter usageOf. */
170
+ export function journalTrajectory(root) {
171
+ const result = {
172
+ steps: [],
173
+ flat: [],
174
+ modelCalls: root.kind === "agent"
175
+ ? allModels(root).reduce((count, model) => count + 1 + model.interrupted.length, 0)
176
+ : 0,
177
+ toolCalls: 0,
178
+ };
179
+ if (root.kind === "workflow") {
180
+ result.steps = root.steps.map((step) => ({
181
+ kind: step.kind,
182
+ name: step.key,
183
+ workflowId: root.workflowId,
184
+ runId: root.runId,
185
+ operationId: step.id,
186
+ input: copyData(approvedInput(step.definition.input, step.asks)),
187
+ ...(step.status === "completed" ? { output: copyData(step.result) } : {}),
188
+ status: step.status,
189
+ attempts: step.kind === "step"
190
+ ? step.attempts.filter((attempt) => attempt.status !== "not_run").length
191
+ : step.kind === "agent" || step.kind === "remote" || step.kind === "workflow"
192
+ ? step.children.length
193
+ : 1,
194
+ asks: copyData(step.asks ?? []),
195
+ children: step.children.map(childTrajectory),
196
+ }));
197
+ }
198
+ else if (root.kind === "remote") {
199
+ result.steps = remoteAttempts(root).map((receipt, index) => ({
200
+ ...receipt,
201
+ kind: "remote-operation",
202
+ name: `remote_${receipt.action}`,
203
+ attempts: 1,
204
+ ...(receipt.action === "send" && {
205
+ input: copyData(root.remote.sends[index]?.message.parts),
206
+ }),
207
+ asks: copyData(root.remote.sends[index]?.asks ?? []),
208
+ children: [],
209
+ }));
210
+ }
211
+ else {
212
+ for (const [index, model] of root.models.entries()) {
213
+ if (model.status !== "completed")
214
+ continue;
215
+ for (const [position, call] of (model.turn.message.toolCalls ?? []).entries()) {
216
+ const execution = model.tools[position];
217
+ if (!execution || call.name === RESULT_TOOL_NAME)
218
+ continue;
219
+ const attempts = toolAttempts(execution);
220
+ if (!attempts && !execution.calls?.length)
221
+ continue;
222
+ const journal = root.delegations.find((entry) => entry.operationId === execution.id);
223
+ const code = Object.hasOwn(root.definition.codeTools ?? {}, call.name);
224
+ const args = approvedInput(code ? (jsonArguments(call.arguments) ?? {}) : parseToolArgs(call.arguments), execution.asks);
225
+ result.steps.push({
226
+ kind: code ? "code" : (journal?.kind ?? "tool"),
227
+ name: call.name,
228
+ arguments: copyData(args),
229
+ status: execution.status === "completed"
230
+ ? "completed"
231
+ : execution.status === "pending"
232
+ ? "interrupted"
233
+ : "running",
234
+ attempts,
235
+ ...(execution.status === "completed" ? { result: copyData(execution.result) } : {}),
236
+ asks: copyData(execution.asks ?? []),
237
+ agentId: root.agentId,
238
+ runId: root.runId,
239
+ operationId: execution.id,
240
+ turn: index + 1,
241
+ children: (journal?.children ?? []).map(childTrajectory),
242
+ });
243
+ result.toolCalls += attempts;
244
+ for (const inner of execution.calls ?? []) {
245
+ const attempts = toolAttempts(inner);
246
+ result.steps.push({
247
+ kind: "tool",
248
+ name: inner.name,
249
+ arguments: copyData(approvedInput(inner.arguments, inner.asks)),
250
+ status: inner.status,
251
+ attempts,
252
+ ...(inner.status === "completed" && { result: copyData(inner.result) }),
253
+ asks: copyData(inner.asks ?? []),
254
+ agentId: root.agentId,
255
+ runId: root.runId,
256
+ operationId: inner.id,
257
+ parentOperationId: execution.id,
258
+ turn: index + 1,
259
+ children: [],
260
+ });
261
+ result.toolCalls += attempts;
262
+ }
263
+ }
264
+ }
265
+ }
266
+ for (const step of result.steps) {
267
+ result.flat.push(step);
268
+ for (const child of step.children) {
269
+ result.flat.push(...child.trajectory.flat);
270
+ result.modelCalls += child.trajectory.modelCalls;
271
+ result.toolCalls += child.trajectory.toolCalls;
272
+ }
273
+ }
274
+ return result;
275
+ }
276
+ export function trajectoryOf(raw) {
277
+ return journalTrajectory(readJournal(raw));
278
+ }
279
+ export function evalRun(root, price, durationMs) {
280
+ // The last admitted handoff owns the answer; delegated specialists do not.
281
+ let active = root;
282
+ while (true) {
283
+ const child = active.delegations.find((entry) => entry.kind === "handoff" && entry.children.length > 0)?.children[0];
284
+ if (!child)
285
+ break;
286
+ active = agentSnapshot(child.snapshot);
287
+ }
288
+ const last = active.models.at(-1);
289
+ return {
290
+ runId: root.runId,
291
+ agentId: root.agentId,
292
+ version: root.version,
293
+ input: inputOf(root),
294
+ status: root.state.status,
295
+ ...(root.state.status === "completed" ? { value: copyData(root.state.candidate) } : {}),
296
+ text: last?.status === "completed" ? last.turn.message.content : "",
297
+ snapshot: root,
298
+ activeAgent: { id: active.agentId, version: active.version },
299
+ trajectory: journalTrajectory(root),
300
+ durationMs,
301
+ ...usageOf(root, price),
302
+ };
303
+ }
304
+ export function runResult(snapshot, usage, role, includePrivateData = false, turn, durationMs) {
305
+ const run = evalRun(snapshot, undefined, durationMs);
306
+ const agents = [...snapshots(snapshot)].filter((node) => node.kind === "agent");
307
+ return {
308
+ runId: snapshot.runId,
309
+ attemptId: snapshot.attemptId,
310
+ agent: { id: snapshot.agentId, version: snapshot.version },
311
+ activeAgent: run.activeAgent,
312
+ role,
313
+ turn,
314
+ status: snapshot.state.status,
315
+ modelCalls: run.trajectory.modelCalls,
316
+ toolCalls: run.trajectory.toolCalls,
317
+ durationMs,
318
+ ...usage,
319
+ modelAttempts: agents.flatMap((node) => allModels(node).flatMap((model) => [
320
+ ...model.interrupted.map((attempt) => ({
321
+ operationId: model.id,
322
+ attemptId: attempt.attemptId,
323
+ status: "interrupted",
324
+ billing: "unknown",
325
+ usage: copyData(attempt.usage),
326
+ })),
327
+ {
328
+ operationId: model.id,
329
+ attemptId: model.attemptId,
330
+ status: model.status,
331
+ billing: model.billing,
332
+ usage: copyData(model.usage),
333
+ },
334
+ ])),
335
+ toolAttempts: agents.flatMap((node) => node.models.flatMap((model) => model.status !== "completed"
336
+ ? []
337
+ : model.tools.flatMap((outer) => [
338
+ { tool: outer, parentOperationId: undefined },
339
+ ...(outer.calls ?? []).map((tool) => ({ tool, parentOperationId: outer.id })),
340
+ ].flatMap(({ tool, parentOperationId }) => [
341
+ ...tool.interrupted.map((attempt) => ({
342
+ operationId: tool.id,
343
+ ...(parentOperationId && { parentOperationId }),
344
+ attemptId: attempt.attemptId,
345
+ status: "interrupted",
346
+ })),
347
+ ...tool.retries.map((attempt) => ({
348
+ operationId: tool.id,
349
+ ...(parentOperationId && { parentOperationId }),
350
+ attemptId: attempt.attemptId,
351
+ status: "completed",
352
+ ok: false,
353
+ })),
354
+ ...(tool.status === "pending"
355
+ ? []
356
+ : [
357
+ {
358
+ operationId: tool.id,
359
+ ...(parentOperationId && { parentOperationId }),
360
+ attemptId: tool.attemptId,
361
+ status: tool.status === "completed"
362
+ ? "completed"
363
+ : "running",
364
+ ...(tool.status === "completed" ? { ok: tool.result.ok } : {}),
365
+ },
366
+ ]),
367
+ ])))),
368
+ remoteAttempts: [...snapshots(snapshot)].flatMap((node) => node.kind === "remote" ? remoteAttempts(node) : []),
369
+ ...(includePrivateData ? { snapshot: copyData(snapshot) } : {}),
370
+ };
371
+ }
372
+ /** Strip executable scorers before freezing context or validating stored case data. */
373
+ export function caseData(item) {
374
+ const { scorers: _scorers, turns, ...data } = item;
375
+ return {
376
+ ...data,
377
+ ...(turns ? { turns: turns.map(({ scorers: _turnScorers, ...turn }) => turn) } : {}),
378
+ };
379
+ }
380
+ export function readCases(raw, validateExpected) {
381
+ if (!Array.isArray(raw))
382
+ throw new TypeError("Cases must be an array.");
383
+ const ids = new Set();
384
+ function expectations(value) {
385
+ const status = value.expectStatus;
386
+ if (status !== undefined &&
387
+ status !== "completed" &&
388
+ status !== "failed" &&
389
+ status !== "limited" &&
390
+ status !== "suspended" &&
391
+ status !== "cancelled")
392
+ throw new TypeError("Invalid expected run status.");
393
+ if (value.trajectory !== undefined)
394
+ checkExpectation(value.trajectory);
395
+ return {
396
+ ...(Object.hasOwn(value, "expected")
397
+ ? {
398
+ expected: validateExpected
399
+ ? validateExpected(value.expected)
400
+ : copyData(value.expected),
401
+ }
402
+ : {}),
403
+ ...(status ? { expectStatus: status } : {}),
404
+ ...(value.trajectory === undefined ? {} : { trajectory: value.trajectory }),
405
+ };
406
+ }
407
+ return raw.map((rawValue) => {
408
+ const value = copyData(rawValue);
409
+ if (!isRecord(value) ||
410
+ !isId(value.id) ||
411
+ ids.has(value.id) ||
412
+ Object.keys(value).some((key) => !["id", "input", "turns", "user", "expected", "expectStatus", "trajectory"].includes(key)) ||
413
+ [value.input, value.turns, value.user].filter((entry) => entry !== undefined).length !== 1)
414
+ throw new TypeError("Each case needs a unique id and exactly one of input, turns and user. Drivers and code are not case data.");
415
+ ids.add(value.id);
416
+ const base = { id: value.id, ...expectations(value) };
417
+ if (value.turns !== undefined) {
418
+ if (!Array.isArray(value.turns) || !value.turns.length)
419
+ throw new TypeError("A conversation case needs at least one turn.");
420
+ const turns = value.turns.map((turn) => {
421
+ if (!isRecord(turn) ||
422
+ Object.keys(turn).some((key) => !["input", "expected", "expectStatus", "trajectory"].includes(key)) ||
423
+ isRecord(turn.input))
424
+ throw new TypeError("Each session turn needs a new user input and only supported data fields.");
425
+ return { input: readInput(turn.input, "caller").input, ...expectations(turn) };
426
+ });
427
+ return { ...base, turns };
428
+ }
429
+ if (value.user !== undefined) {
430
+ const user = value.user;
431
+ if (!isRecord(user) ||
432
+ Object.keys(user).some((key) => !["opening", "plan", "persona", "maxTurns"].includes(key)) ||
433
+ !isId(user.plan) ||
434
+ (user.persona !== undefined && !isId(user.persona)) ||
435
+ isRecord(user.opening) ||
436
+ (user.maxTurns !== undefined &&
437
+ (typeof user.maxTurns !== "number" ||
438
+ !Number.isSafeInteger(user.maxTurns) ||
439
+ user.maxTurns < 1 ||
440
+ user.maxTurns > 100)))
441
+ throw new TypeError("A simulated user needs a plan, an opening and a turn limit from 1 to 100.");
442
+ return {
443
+ ...base,
444
+ user: {
445
+ opening: readInput(user.opening, "caller").input,
446
+ plan: user.plan,
447
+ ...(user.persona === undefined ? {} : { persona: user.persona }),
448
+ ...(user.maxTurns === undefined ? {} : { maxTurns: user.maxTurns }),
449
+ },
450
+ };
451
+ }
452
+ const input = readInput(value.input, "caller");
453
+ return {
454
+ ...base,
455
+ input: input.history.length
456
+ ? { messages: [...input.history, { role: "user", content: input.input }] }
457
+ : input.input,
458
+ };
459
+ });
460
+ }
461
+ /** Explicitly copies private input, output and tool arguments. Nothing is executed or approved.
462
+ * Supply a sandboxed agent when you later run this case; stored results are not tool mocks. */
463
+ export function caseFromRun(raw, options) {
464
+ if (!isId(options.id))
465
+ throw new TypeError("Give the golden case a non-empty id.");
466
+ const snapshot = readJournal(raw);
467
+ if (snapshot.state.status !== "completed")
468
+ throw new TypeError("Only a completed run can become a golden case. No interrupted work is resumed.");
469
+ if ([...snapshots(snapshot)].some((node) => node.kind === "agent" && node.inputs.length > 0))
470
+ throw new TypeError("This run received messages while working. Record its input timing in the host instead of replaying it as one case.");
471
+ return {
472
+ id: options.id,
473
+ input: inputOf(snapshot),
474
+ expected: copyData(snapshot.state.candidate),
475
+ trajectory: {
476
+ match: "exact",
477
+ calls: journalTrajectory(snapshot)
478
+ .steps.filter((step) => step.result?.ok)
479
+ .map((step) => ({ name: step.name, arguments: step.arguments, argumentsMatch: "exact" })),
480
+ },
481
+ };
482
+ }
483
+ //# sourceMappingURL=journal.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"journal.js","sourceRoot":"","sources":["../../src/evals/journal.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,OAAO,EACP,WAAW,EACX,iBAAiB,EACjB,aAAa,GAEd,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,gBAAgB,GAKjB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EACL,aAAa,EACb,QAAQ,EACR,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,aAAa,EACb,UAAU,EACV,YAAY,GACb,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAanD;gGACgG;AAChG,MAAM,UAAU,WAAW,CAAC,GAAY;IACtC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,UAAU,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC;IACtE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;QAC5F,MAAM,IAAI,SAAS,CAAC,+DAA+D,CAAC,CAAC;IACvF,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;IACpC,MAAM,aAAa,GACjB,QAAQ,CAAC,UAAU,CAAC;QACpB,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC;QACtC,UAAU,CAAC,YAAY,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IACrD,MAAM,IAAI,GAAG,YAAY,CACvB,KAAK,EACL,KAAK,CAAC,OAAO,EACb,KAAK,CAAC,OAAO,EACb,MAAM,EACN,aAAa,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS,CAC7C,CAAC;IACF,oFAAoF;IACpF,qFAAqF;IACrF,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO;YAAE,SAAS;QACpC,IAAI,IAAI,KAAK,IAAI;YACf,YAAY,CACV,IAAI,EACJ,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS,CACvF,CAAC;QACJ,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YACpC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACxB,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,WAAW;gBAAE,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,UAAU,CAAC,KAAiB;IACnC,IACE,CAAC,OAAO,CAAC,KAAK,CAAC;QACf,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS;YAC9B,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC;gBAC3B,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CACpC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CACf,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC;oBAC/C,OAAO,KAAK,KAAK,QAAQ;oBACzB,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;oBACvB,KAAK,GAAG,CAAC,CACZ,CAAC,CAAC;QAEP,MAAM,IAAI,SAAS,CAAC,qDAAqD,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,UAAU,UAAU;IACxB,OAAO;QACL,KAAK,EAAE,EAAE,GAAG,WAAW,EAAE;QACzB,YAAY,EAAE,CAAC;QACf,OAAO,EAAE,CAAC;QACV,cAAc,EAAE,CAAC;QACjB,aAAa,EAAE,CAAC;KACjB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAA2B;IACtD,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAC5B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QACnD,MAAM,CAAC,YAAY,IAAI,KAAK,CAAC,YAAY,CAAC;QAC1C,MAAM,CAAC,cAAc,IAAI,KAAK,CAAC,cAAc,CAAC;QAC9C,MAAM,CAAC,aAAa,IAAI,KAAK,CAAC,aAAa,CAAC;IAC9C,CAAC;IACD,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;IAC5F,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,IAAc,EAAE,KAAiB;IACvD,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,oFAAoF;YACpF,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACjF,SAAS;QACX,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO;YAAE,SAAS;QACpC,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBAAE,SAAS;YACjC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACnB,MAAM,GAAG,GAAa;gBACpB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;aAClE,CAAC;YACF,KAAK,MAAM,OAAO,IAAI;gBACpB,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;gBAC3E,KAAK;aACN,EAAE,CAAC;gBACF,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAC1B,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;gBACrD,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS;oBAAE,MAAM,CAAC,cAAc,EAAE,CAAC;gBAC3D,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC;gBAC/B,IAAI,QAAQ,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;oBAC3C,MAAM,CAAC,YAAY,IAAI,QAAQ,CAAC,eAAe,CAAC;oBAChD,SAAS;gBACX,CAAC;gBACD,mFAAmF;gBACnF,MAAM,cAAc,GAClB,QAAQ,CAAC,WAAW,GAAG,QAAQ,CAAC,YAAY,GAAG,CAAC;oBAChD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC/D,IAAI,CAAC,cAAc,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS;oBAClF,SAAS;gBACX,mFAAmF;gBACnF,wEAAwE;gBACxE,MAAM,IAAI,GAAG,KAAK,EAAE,CAAC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC1E,IACE,IAAI;oBACJ;wBACE,IAAI,CAAC,KAAK;wBACV,IAAI,CAAC,MAAM;wBACX,IAAI,CAAC,SAAS;wBACd,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,KAAK;wBAC7B,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;qBACpC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAE3C,MAAM,IAAI,SAAS,CAAC,gDAAgD,CAAC,CAAC;gBACxE,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,IAAI,CAC9C,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,KAAK,SAAS,CACxF,CAAC;gBACF,MAAM,OAAO,GACX,OAAO,KAAK,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC;gBACvF,mFAAmF;gBACnF,mFAAmF;gBACnF,IAAI,CAAC,CAAC,IAAI,IAAI,iBAAiB,CAAC,IAAI,OAAO;oBAAE,MAAM,CAAC,aAAa,EAAE,CAAC;gBACpE,IAAI,IAAI;oBAAE,MAAM,CAAC,YAAY,IAAI,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;IAC5F,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,eAAe,CAAC,EACvB,GAAG,EACH,QAAQ,GAIT;IACC,OAAO;QACL,GAAG;QACH,GAAG,CAAC,QAAQ,CAAC,IAAI,KAAK,UAAU;YAC9B,CAAC,CAAC,EAAE,IAAI,EAAE,UAAmB,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE;YAChE,CAAC,CAAC,QAAQ,CAAC,IAAI,KAAK,QAAQ;gBAC1B,CAAC,CAAC,EAAE,IAAI,EAAE,QAAiB,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE;gBACnF,CAAC,CAAC,EAAE,IAAI,EAAE,OAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC5D,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,UAAU,EAAE,iBAAiB,CAAC,QAAQ,CAAC;KACxC,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,QAAwB;IAC9C,OAAO;QACL,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,MAAe,EAAE,CAAC,CAAC;QAC9E,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,QAAiB,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9F,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClB,IAAI,EAAE,QAAQ;QACd,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,WAAW,EAAE,OAAO,CAAC,EAAE;QACvB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,OAAO,EAAE,SAAS;KACnB,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,aAAa,CAAI,KAAQ,EAAE,IAAI,GAAyB,EAAE;IACjE,IAAI,SAAS,GAAgC,KAAK,CAAC;IACnD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,QAAQ,EAAE,IAAI,KAAK,SAAS;YAAE,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,SAAS,IAAI,KAAK,CAAC;IACpF,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,4FAA4F;AAC5F,SAAS,YAAY,CAAC,IAAmB;IACvC,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7F,CAAC;AAED;qGACqG;AACrG,MAAM,UAAU,iBAAiB,CAAC,IAAc;IAC9C,MAAM,MAAM,GAAe;QACzB,KAAK,EAAE,EAAE;QACT,IAAI,EAAE,EAAE;QACR,UAAU,EACR,IAAI,CAAC,IAAI,KAAK,OAAO;YACnB,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;YACnF,CAAC,CAAC,CAAC;QACP,SAAS,EAAE,CAAC;KACb,CAAC;IACF,IAAI,IAAI,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QAC7B,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACvC,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,GAAG;YACd,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,IAAI,CAAC,EAAE;YACpB,KAAK,EAAE,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YAChE,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzE,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,QAAQ,EACN,IAAI,CAAC,IAAI,KAAK,MAAM;gBAClB,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM;gBACxE,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,KAAK,UAAU;oBAC3E,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM;oBACtB,CAAC,CAAC,CAAC;YACT,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YAC/B,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC;SAC7C,CAAC,CAAC,CAAC;IACN,CAAC;SAAM,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAClC,MAAM,CAAC,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YAC3D,GAAG,OAAO;YACV,IAAI,EAAE,kBAAkB;YACxB,IAAI,EAAE,UAAU,OAAO,CAAC,MAAM,EAAE;YAChC,QAAQ,EAAE,CAAC;YACX,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,MAAM,IAAI;gBAC/B,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC;aACzD,CAAC;YACF,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC;YACpD,QAAQ,EAAE,EAAE;SACb,CAAC,CAAC,CAAC;IACN,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YACnD,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW;gBAAE,SAAS;YAC3C,KAAK,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;gBAC9E,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACxC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,gBAAgB;oBAAE,SAAS;gBAC3D,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;gBACzC,IAAI,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM;oBAAE,SAAS;gBACpD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,EAAE,CAAC,CAAC;gBACrF,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,IAAI,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBACvE,MAAM,IAAI,GAAG,aAAa,CACxB,IAAI,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,EAC5E,SAAS,CAAC,IAAI,CACf,CAAC;gBACF,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;oBAChB,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,IAAI,IAAI,MAAM,CAAC;oBAC/C,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC;oBACzB,MAAM,EACJ,SAAS,CAAC,MAAM,KAAK,WAAW;wBAC9B,CAAC,CAAC,WAAW;wBACb,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,SAAS;4BAC9B,CAAC,CAAC,aAAa;4BACf,CAAC,CAAC,SAAS;oBACjB,QAAQ;oBACR,GAAG,CAAC,SAAS,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnF,IAAI,EAAE,QAAQ,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,CAAC;oBACpC,OAAO,EAAE,IAAI,CAAC,OAAO;oBACrB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,WAAW,EAAE,SAAS,CAAC,EAAE;oBACzB,IAAI,EAAE,KAAK,GAAG,CAAC;oBACf,QAAQ,EAAE,CAAC,OAAO,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC;iBACzD,CAAC,CAAC;gBACH,MAAM,CAAC,SAAS,IAAI,QAAQ,CAAC;gBAC7B,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;oBAC1C,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;oBACrC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;wBAChB,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,SAAS,EAAE,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;wBAC/D,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,QAAQ;wBACR,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;wBACvE,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC;wBAChC,OAAO,EAAE,IAAI,CAAC,OAAO;wBACrB,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,WAAW,EAAE,KAAK,CAAC,EAAE;wBACrB,iBAAiB,EAAE,SAAS,CAAC,EAAE;wBAC/B,IAAI,EAAE,KAAK,GAAG,CAAC;wBACf,QAAQ,EAAE,EAAE;qBACb,CAAC,CAAC;oBACH,MAAM,CAAC,SAAS,IAAI,QAAQ,CAAC;gBAC/B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YAC3C,MAAM,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC;YACjD,MAAM,CAAC,SAAS,IAAI,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC;QACjD,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAY;IACvC,OAAO,iBAAiB,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,IAAiB,EAAE,KAAiB,EAAE,UAAmB;IAC/E,2EAA2E;IAC3E,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CACnC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CACjE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;QACf,IAAI,CAAC,KAAK;YAAE,MAAM;QAClB,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACzC,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC;QACpB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;QACzB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvF,IAAI,EAAE,IAAI,EAAE,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;QACnE,QAAQ,EAAE,IAAI;QACd,WAAW,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE;QAC5D,UAAU,EAAE,iBAAiB,CAAC,IAAI,CAAC;QACnC,UAAU;QACV,GAAG,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;KACxB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,QAAqB,EACrB,KAAgB,EAChB,IAA2B,EAC3B,kBAAkB,GAAG,KAAK,EAC1B,IAAa,EACb,UAAmB;IAEnB,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;IAChF,OAAO;QACL,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,KAAK,EAAE,EAAE,EAAE,EAAE,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE;QAC1D,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,IAAI;QACJ,IAAI;QACJ,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM;QAC7B,UAAU,EAAE,GAAG,CAAC,UAAU,CAAC,UAAU;QACrC,SAAS,EAAE,GAAG,CAAC,UAAU,CAAC,SAAS;QACnC,UAAU;QACV,GAAG,KAAK;QACR,aAAa,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CACrC,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;YACjC,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBACrC,WAAW,EAAE,KAAK,CAAC,EAAE;gBACrB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,MAAM,EAAE,aAAsB;gBAC9B,OAAO,EAAE,SAAkB;gBAC3B,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC;aAC/B,CAAC,CAAC;YACH;gBACE,WAAW,EAAE,KAAK,CAAC,EAAE;gBACrB,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC;aAC7B;SACF,CAAC,CACH;QACD,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CACpC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAC5B,KAAK,CAAC,MAAM,KAAK,WAAW;YAC1B,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAC5B;gBACE,EAAE,IAAI,EAAE,KAAK,EAAE,iBAAiB,EAAE,SAAS,EAAE;gBAC7C,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;aAC9E,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC;gBACzC,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;oBACpC,WAAW,EAAE,IAAI,CAAC,EAAE;oBACpB,GAAG,CAAC,iBAAiB,IAAI,EAAE,iBAAiB,EAAE,CAAC;oBAC/C,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,MAAM,EAAE,aAAsB;iBAC/B,CAAC,CAAC;gBACH,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;oBAChC,WAAW,EAAE,IAAI,CAAC,EAAE;oBACpB,GAAG,CAAC,iBAAiB,IAAI,EAAE,iBAAiB,EAAE,CAAC;oBAC/C,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,MAAM,EAAE,WAAoB;oBAC5B,EAAE,EAAE,KAAK;iBACV,CAAC,CAAC;gBACH,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS;oBAC3B,CAAC,CAAC,EAAE;oBACJ,CAAC,CAAC;wBACE;4BACE,WAAW,EAAE,IAAI,CAAC,EAAE;4BACpB,GAAG,CAAC,iBAAiB,IAAI,EAAE,iBAAiB,EAAE,CAAC;4BAC/C,SAAS,EAAE,IAAI,CAAC,SAAS;4BACzB,MAAM,EACJ,IAAI,CAAC,MAAM,KAAK,WAAW;gCACzB,CAAC,CAAE,WAAqB;gCACxB,CAAC,CAAE,SAAmB;4BAC1B,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;yBAC/D;qBACF,CAAC;aACP,CAAC,CACH,CACN,CACF;QACD,cAAc,EAAE,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CACxD,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CACnD;QACD,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChE,CAAC;AACJ,CAAC;AAED,uFAAuF;AACvF,MAAM,UAAU,QAAQ,CAAI,IAAiB;IAC3C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC;IACnD,OAAO;QACL,GAAG,IAAI;QACP,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACrF,CAAC;AACJ,CAAC;AAMD,MAAM,UAAU,SAAS,CACvB,GAAY,EACZ,gBAA8C;IAE9C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,yBAAyB,CAAC,CAAC;IACxE,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,SAAS,YAAY,CACnB,KAA8B;QAE9B,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC;QAClC,IACE,MAAM,KAAK,SAAS;YACpB,MAAM,KAAK,WAAW;YACtB,MAAM,KAAK,QAAQ;YACnB,MAAM,KAAK,SAAS;YACpB,MAAM,KAAK,WAAW;YACtB,MAAM,KAAK,WAAW;YAEtB,MAAM,IAAI,SAAS,CAAC,8BAA8B,CAAC,CAAC;QACtD,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS;YAAE,gBAAgB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACvE,OAAO;YACL,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC;gBAClC,CAAC,CAAC;oBACE,QAAQ,EAAE,gBAAgB;wBACxB,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC;wBAClC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC;iBAC7B;gBACH,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3C,GAAG,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;SAC5E,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAY,EAAE;QACpC,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACjC,IACE,CAAC,QAAQ,CAAC,KAAK,CAAC;YAChB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACf,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACjB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CACrB,CAAC,GAAG,EAAE,EAAE,CACN,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAC5F;YACD,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,KAAK,CAAC;YAE1F,MAAM,IAAI,SAAS,CACjB,2GAA2G,CAC5G,CAAC;QACJ,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAClB,MAAM,IAAI,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;QACtD,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC9B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM;gBACpD,MAAM,IAAI,SAAS,CAAC,8CAA8C,CAAC,CAAC;YACtE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAY,EAAE;gBAC/C,IACE,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACf,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CACpB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAC5E;oBACD,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC;oBAEpB,MAAM,IAAI,SAAS,CACjB,0EAA0E,CAC3E,CAAC;gBACJ,OAAO,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,KAAK,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;YACjF,CAAC,CAAC,CAAC;YACH,OAAO,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;YACxB,IACE,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CACpB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CACnE;gBACD,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;gBAChB,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACnD,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC;gBACtB,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS;oBAC1B,CAAC,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ;wBAChC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC;wBACpC,IAAI,CAAC,QAAQ,GAAG,CAAC;wBACjB,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC;gBAEzB,MAAM,IAAI,SAAS,CACjB,2EAA2E,CAC5E,CAAC;YACJ,OAAO;gBACL,GAAG,IAAI;gBACP,IAAI,EAAE;oBACJ,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,KAAK;oBAChD,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,GAAG,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;oBAChE,GAAG,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;iBACpE;aACF,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC/C,OAAO;YACL,GAAG,IAAI;YACP,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;gBACzB,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE;gBAC1E,CAAC,CAAC,KAAK,CAAC,KAAK;SAChB,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;+FAC+F;AAC/F,MAAM,UAAU,WAAW,CAAC,GAAY,EAAE,OAAuB;IAC/D,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,sCAAsC,CAAC,CAAC;IACnF,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW;QACvC,MAAM,IAAI,SAAS,CACjB,gFAAgF,CACjF,CAAC;IACJ,IAAI,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QAC1F,MAAM,IAAI,SAAS,CACjB,oHAAoH,CACrH,CAAC;IACJ,OAAO;QACL,EAAE,EAAE,OAAO,CAAC,EAAE;QACd,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC;QACxB,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC;QAC5C,UAAU,EAAE;YACV,KAAK,EAAE,OAAO;YACd,KAAK,EAAE,iBAAiB,CAAC,QAAQ,CAAC;iBAC/B,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;iBACvC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;SAC5F;KACF,CAAC;AACJ,CAAC"}
@@ -0,0 +1,31 @@
1
+ import type { Provider } from "@providerkit/core";
2
+ import { type ModelSettings } from "../index.ts";
3
+ import type { Scorer } from "./types.ts";
4
+ export interface JudgeOptions {
5
+ provider: Provider;
6
+ rubric: string;
7
+ /** Text/JSON only. Output is always sent. Input and operation data are opt-in; expected defaults on. */
8
+ include?: {
9
+ input?: boolean;
10
+ expected?: boolean;
11
+ trajectory?: boolean;
12
+ };
13
+ scale?: {
14
+ min: number;
15
+ max: number;
16
+ steps: readonly string[];
17
+ };
18
+ threshold?: number;
19
+ samples?: number;
20
+ modelSettings?: Omit<ModelSettings, "toolChoice" | "hostedTools" | "maxAttempts">;
21
+ name?: string;
22
+ gate?: boolean;
23
+ }
24
+ export declare function section(tag: string, value: unknown): string;
25
+ /** Costs money only when explicitly evaluated with allowJudge and the supplied provider.
26
+ * A judge is a fallible measurement, not proof that an output is safe or correct. */
27
+ export declare function llmJudge(options: JudgeOptions): Scorer;
28
+ /** A measured model opinion about recorded evidence, not proof of truth. Sends tool results
29
+ * to the explicit judge provider only with allowJudge. No usable results means no score. */
30
+ export declare function grounded(options: Omit<JudgeOptions, "rubric" | "include">): Scorer;
31
+ //# sourceMappingURL=judge.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../src/evals/judge.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAiC,KAAK,aAAa,EAAE,MAAM,aAAa,CAAC;AAGhF,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAEzC,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,wGAAwG;IACxG,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IACxE,KAAK,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;KAAE,CAAC;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,IAAI,CAAC,aAAa,EAAE,YAAY,GAAG,aAAa,GAAG,aAAa,CAAC,CAAC;IAClF,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAQD,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,MAAM,CAG3D;AAED;qFACqF;AACrF,wBAAgB,QAAQ,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM,CAmItD;AAED;4FAC4F;AAC5F,wBAAgB,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE,QAAQ,GAAG,SAAS,CAAC,GAAG,MAAM,CAgBlF"}