@rudderjs/ai 1.17.3 → 1.18.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (377) hide show
  1. package/README.md +19 -1274
  2. package/dist/budget-orm/index.d.ts +1 -95
  3. package/dist/budget-orm/index.d.ts.map +1 -1
  4. package/dist/budget-orm/index.js +4 -176
  5. package/dist/budget-orm/index.js.map +1 -1
  6. package/dist/chat-mentions.d.ts +1 -58
  7. package/dist/chat-mentions.d.ts.map +1 -1
  8. package/dist/chat-mentions.js +4 -80
  9. package/dist/chat-mentions.js.map +1 -1
  10. package/dist/commands/ai-eval.d.ts +1 -92
  11. package/dist/commands/ai-eval.d.ts.map +1 -1
  12. package/dist/commands/ai-eval.js +4 -377
  13. package/dist/commands/ai-eval.js.map +1 -1
  14. package/dist/commands/make-agent.d.ts +1 -2
  15. package/dist/commands/make-agent.d.ts.map +1 -1
  16. package/dist/commands/make-agent.js +4 -22
  17. package/dist/commands/make-agent.js.map +1 -1
  18. package/dist/computer-use/index.d.ts +1 -52
  19. package/dist/computer-use/index.d.ts.map +1 -1
  20. package/dist/computer-use/index.js +4 -50
  21. package/dist/computer-use/index.js.map +1 -1
  22. package/dist/conversation-orm/index.d.ts +1 -108
  23. package/dist/conversation-orm/index.d.ts.map +1 -1
  24. package/dist/conversation-orm/index.js +4 -214
  25. package/dist/conversation-orm/index.js.map +1 -1
  26. package/dist/doctor.d.ts +1 -1
  27. package/dist/doctor.d.ts.map +1 -1
  28. package/dist/doctor.js +4 -65
  29. package/dist/doctor.js.map +1 -1
  30. package/dist/eval/index.d.ts +1 -270
  31. package/dist/eval/index.d.ts.map +1 -1
  32. package/dist/eval/index.js +4 -509
  33. package/dist/eval/index.js.map +1 -1
  34. package/dist/gateway/index.d.ts +1 -10
  35. package/dist/gateway/index.d.ts.map +1 -1
  36. package/dist/gateway/index.js +4 -10
  37. package/dist/gateway/index.js.map +1 -1
  38. package/dist/index.d.ts +1 -66
  39. package/dist/index.d.ts.map +1 -1
  40. package/dist/index.js +4 -78
  41. package/dist/index.js.map +1 -1
  42. package/dist/mcp/index.d.ts +1 -15
  43. package/dist/mcp/index.d.ts.map +1 -1
  44. package/dist/mcp/index.js +4 -14
  45. package/dist/mcp/index.js.map +1 -1
  46. package/dist/memory-embedding/index.d.ts +1 -120
  47. package/dist/memory-embedding/index.d.ts.map +1 -1
  48. package/dist/memory-embedding/index.js +4 -228
  49. package/dist/memory-embedding/index.js.map +1 -1
  50. package/dist/memory-orm/index.d.ts +1 -117
  51. package/dist/memory-orm/index.d.ts.map +1 -1
  52. package/dist/memory-orm/index.js +4 -186
  53. package/dist/memory-orm/index.js.map +1 -1
  54. package/dist/node/index.d.ts +1 -2
  55. package/dist/node/index.d.ts.map +1 -1
  56. package/dist/node/index.js +4 -2
  57. package/dist/node/index.js.map +1 -1
  58. package/dist/observers.d.ts +1 -129
  59. package/dist/observers.d.ts.map +1 -1
  60. package/dist/observers.js +4 -39
  61. package/dist/observers.js.map +1 -1
  62. package/dist/react/index.d.ts +1 -15
  63. package/dist/react/index.d.ts.map +1 -1
  64. package/dist/react/index.js +4 -15
  65. package/dist/react/index.js.map +1 -1
  66. package/dist/server/index.d.ts +1 -1
  67. package/dist/server/index.d.ts.map +1 -1
  68. package/dist/server/index.js +4 -1
  69. package/dist/server/index.js.map +1 -1
  70. package/package.json +9 -13
  71. package/boost/guidelines.md +0 -260
  72. package/boost/skills/ai-agents/SKILL.md +0 -240
  73. package/boost/skills/ai-tools/SKILL.md +0 -260
  74. package/dist/agent-run-store.d.ts +0 -161
  75. package/dist/agent-run-store.d.ts.map +0 -1
  76. package/dist/agent-run-store.js +0 -98
  77. package/dist/agent-run-store.js.map +0 -1
  78. package/dist/agent-sse.d.ts +0 -153
  79. package/dist/agent-sse.d.ts.map +0 -1
  80. package/dist/agent-sse.js +0 -282
  81. package/dist/agent-sse.js.map +0 -1
  82. package/dist/agent.d.ts +0 -508
  83. package/dist/agent.d.ts.map +0 -1
  84. package/dist/agent.js +0 -1538
  85. package/dist/agent.js.map +0 -1
  86. package/dist/attachment.d.ts +0 -31
  87. package/dist/attachment.d.ts.map +0 -1
  88. package/dist/attachment.js +0 -89
  89. package/dist/attachment.js.map +0 -1
  90. package/dist/audio.d.ts +0 -45
  91. package/dist/audio.d.ts.map +0 -1
  92. package/dist/audio.js +0 -93
  93. package/dist/audio.js.map +0 -1
  94. package/dist/base64.d.ts +0 -7
  95. package/dist/base64.d.ts.map +0 -1
  96. package/dist/base64.js +0 -39
  97. package/dist/base64.js.map +0 -1
  98. package/dist/budget/pricing.d.ts +0 -124
  99. package/dist/budget/pricing.d.ts.map +0 -1
  100. package/dist/budget/pricing.js +0 -175
  101. package/dist/budget/pricing.js.map +0 -1
  102. package/dist/budget/storage.d.ts +0 -104
  103. package/dist/budget/storage.d.ts.map +0 -1
  104. package/dist/budget/storage.js +0 -0
  105. package/dist/budget/storage.js.map +0 -1
  106. package/dist/budget/with-budget.d.ts +0 -119
  107. package/dist/budget/with-budget.d.ts.map +0 -1
  108. package/dist/budget/with-budget.js +0 -175
  109. package/dist/budget/with-budget.js.map +0 -1
  110. package/dist/cached-embedding.d.ts +0 -14
  111. package/dist/cached-embedding.d.ts.map +0 -1
  112. package/dist/cached-embedding.js +0 -44
  113. package/dist/cached-embedding.js.map +0 -1
  114. package/dist/computer-use/actions.d.ts +0 -214
  115. package/dist/computer-use/actions.d.ts.map +0 -1
  116. package/dist/computer-use/actions.js +0 -48
  117. package/dist/computer-use/actions.js.map +0 -1
  118. package/dist/computer-use/errors.d.ts +0 -57
  119. package/dist/computer-use/errors.d.ts.map +0 -1
  120. package/dist/computer-use/errors.js +0 -76
  121. package/dist/computer-use/errors.js.map +0 -1
  122. package/dist/computer-use/playwright.d.ts +0 -76
  123. package/dist/computer-use/playwright.d.ts.map +0 -1
  124. package/dist/computer-use/playwright.js +0 -270
  125. package/dist/computer-use/playwright.js.map +0 -1
  126. package/dist/computer-use/tool.d.ts +0 -154
  127. package/dist/computer-use/tool.d.ts.map +0 -1
  128. package/dist/computer-use/tool.js +0 -210
  129. package/dist/computer-use/tool.js.map +0 -1
  130. package/dist/continuation-validation.d.ts +0 -85
  131. package/dist/continuation-validation.d.ts.map +0 -1
  132. package/dist/continuation-validation.js +0 -166
  133. package/dist/continuation-validation.js.map +0 -1
  134. package/dist/conversation-persistence.d.ts +0 -46
  135. package/dist/conversation-persistence.d.ts.map +0 -1
  136. package/dist/conversation-persistence.js +0 -176
  137. package/dist/conversation-persistence.js.map +0 -1
  138. package/dist/conversation.d.ts +0 -11
  139. package/dist/conversation.d.ts.map +0 -1
  140. package/dist/conversation.js +0 -55
  141. package/dist/conversation.js.map +0 -1
  142. package/dist/eval/fixtures.d.ts +0 -65
  143. package/dist/eval/fixtures.d.ts.map +0 -1
  144. package/dist/eval/fixtures.js +0 -110
  145. package/dist/eval/fixtures.js.map +0 -1
  146. package/dist/eval/html-reporter.d.ts +0 -25
  147. package/dist/eval/html-reporter.d.ts.map +0 -1
  148. package/dist/eval/html-reporter.js +0 -209
  149. package/dist/eval/html-reporter.js.map +0 -1
  150. package/dist/eval/json-reporter.d.ts +0 -43
  151. package/dist/eval/json-reporter.d.ts.map +0 -1
  152. package/dist/eval/json-reporter.js +0 -40
  153. package/dist/eval/json-reporter.js.map +0 -1
  154. package/dist/facade.d.ts +0 -96
  155. package/dist/facade.d.ts.map +0 -1
  156. package/dist/facade.js +0 -146
  157. package/dist/facade.js.map +0 -1
  158. package/dist/fake.d.ts +0 -201
  159. package/dist/fake.d.ts.map +0 -1
  160. package/dist/fake.js +0 -428
  161. package/dist/fake.js.map +0 -1
  162. package/dist/file-search.d.ts +0 -168
  163. package/dist/file-search.d.ts.map +0 -1
  164. package/dist/file-search.js +0 -158
  165. package/dist/file-search.js.map +0 -1
  166. package/dist/files.d.ts +0 -27
  167. package/dist/files.d.ts.map +0 -1
  168. package/dist/files.js +0 -44
  169. package/dist/files.js.map +0 -1
  170. package/dist/gateway/http-gateway-adapter.d.ts +0 -94
  171. package/dist/gateway/http-gateway-adapter.d.ts.map +0 -1
  172. package/dist/gateway/http-gateway-adapter.js +0 -106
  173. package/dist/gateway/http-gateway-adapter.js.map +0 -1
  174. package/dist/gateway/sse.d.ts +0 -28
  175. package/dist/gateway/sse.d.ts.map +0 -1
  176. package/dist/gateway/sse.js +0 -78
  177. package/dist/gateway/sse.js.map +0 -1
  178. package/dist/handoff.d.ts +0 -95
  179. package/dist/handoff.d.ts.map +0 -1
  180. package/dist/handoff.js +0 -78
  181. package/dist/handoff.js.map +0 -1
  182. package/dist/handoffs-driver.d.ts +0 -58
  183. package/dist/handoffs-driver.d.ts.map +0 -1
  184. package/dist/handoffs-driver.js +0 -103
  185. package/dist/handoffs-driver.js.map +0 -1
  186. package/dist/image.d.ts +0 -40
  187. package/dist/image.d.ts.map +0 -1
  188. package/dist/image.js +0 -109
  189. package/dist/image.js.map +0 -1
  190. package/dist/mcp/client-tools.d.ts +0 -39
  191. package/dist/mcp/client-tools.d.ts.map +0 -1
  192. package/dist/mcp/client-tools.js +0 -147
  193. package/dist/mcp/client-tools.js.map +0 -1
  194. package/dist/mcp/server-from-agent.d.ts +0 -24
  195. package/dist/mcp/server-from-agent.d.ts.map +0 -1
  196. package/dist/mcp/server-from-agent.js +0 -113
  197. package/dist/mcp/server-from-agent.js.map +0 -1
  198. package/dist/mcp/types.d.ts +0 -64
  199. package/dist/mcp/types.d.ts.map +0 -1
  200. package/dist/mcp/types.js +0 -6
  201. package/dist/mcp/types.js.map +0 -1
  202. package/dist/memory-extract.d.ts +0 -60
  203. package/dist/memory-extract.d.ts.map +0 -1
  204. package/dist/memory-extract.js +0 -163
  205. package/dist/memory-extract.js.map +0 -1
  206. package/dist/memory-inject.d.ts +0 -39
  207. package/dist/memory-inject.d.ts.map +0 -1
  208. package/dist/memory-inject.js +0 -135
  209. package/dist/memory-inject.js.map +0 -1
  210. package/dist/memory.d.ts +0 -55
  211. package/dist/memory.d.ts.map +0 -1
  212. package/dist/memory.js +0 -132
  213. package/dist/memory.js.map +0 -1
  214. package/dist/middleware.d.ts +0 -18
  215. package/dist/middleware.d.ts.map +0 -1
  216. package/dist/middleware.js +0 -72
  217. package/dist/middleware.js.map +0 -1
  218. package/dist/node/attachment.d.ts +0 -6
  219. package/dist/node/attachment.d.ts.map +0 -1
  220. package/dist/node/attachment.js +0 -35
  221. package/dist/node/attachment.js.map +0 -1
  222. package/dist/node/transcription.d.ts +0 -4
  223. package/dist/node/transcription.d.ts.map +0 -1
  224. package/dist/node/transcription.js +0 -8
  225. package/dist/node/transcription.js.map +0 -1
  226. package/dist/output.d.ts +0 -22
  227. package/dist/output.d.ts.map +0 -1
  228. package/dist/output.js +0 -60
  229. package/dist/output.js.map +0 -1
  230. package/dist/provider-tools.d.ts +0 -87
  231. package/dist/provider-tools.d.ts.map +0 -1
  232. package/dist/provider-tools.js +0 -189
  233. package/dist/provider-tools.js.map +0 -1
  234. package/dist/providers/anthropic.d.ts +0 -24
  235. package/dist/providers/anthropic.d.ts.map +0 -1
  236. package/dist/providers/anthropic.js +0 -405
  237. package/dist/providers/anthropic.js.map +0 -1
  238. package/dist/providers/azure.d.ts +0 -13
  239. package/dist/providers/azure.d.ts.map +0 -1
  240. package/dist/providers/azure.js +0 -15
  241. package/dist/providers/azure.js.map +0 -1
  242. package/dist/providers/bedrock.d.ts +0 -75
  243. package/dist/providers/bedrock.d.ts.map +0 -1
  244. package/dist/providers/bedrock.js +0 -181
  245. package/dist/providers/bedrock.js.map +0 -1
  246. package/dist/providers/cohere.d.ts +0 -13
  247. package/dist/providers/cohere.d.ts.map +0 -1
  248. package/dist/providers/cohere.js +0 -87
  249. package/dist/providers/cohere.js.map +0 -1
  250. package/dist/providers/deepseek.d.ts +0 -12
  251. package/dist/providers/deepseek.d.ts.map +0 -1
  252. package/dist/providers/deepseek.js +0 -15
  253. package/dist/providers/deepseek.js.map +0 -1
  254. package/dist/providers/elevenlabs.d.ts +0 -98
  255. package/dist/providers/elevenlabs.d.ts.map +0 -1
  256. package/dist/providers/elevenlabs.js +0 -229
  257. package/dist/providers/elevenlabs.js.map +0 -1
  258. package/dist/providers/google-cache-registry.d.ts +0 -132
  259. package/dist/providers/google-cache-registry.d.ts.map +0 -1
  260. package/dist/providers/google-cache-registry.js +0 -209
  261. package/dist/providers/google-cache-registry.js.map +0 -1
  262. package/dist/providers/google.d.ts +0 -38
  263. package/dist/providers/google.d.ts.map +0 -1
  264. package/dist/providers/google.js +0 -903
  265. package/dist/providers/google.js.map +0 -1
  266. package/dist/providers/groq.d.ts +0 -12
  267. package/dist/providers/groq.d.ts.map +0 -1
  268. package/dist/providers/groq.js +0 -15
  269. package/dist/providers/groq.js.map +0 -1
  270. package/dist/providers/jina.d.ts +0 -13
  271. package/dist/providers/jina.d.ts.map +0 -1
  272. package/dist/providers/jina.js +0 -90
  273. package/dist/providers/jina.js.map +0 -1
  274. package/dist/providers/mistral.d.ts +0 -13
  275. package/dist/providers/mistral.d.ts.map +0 -1
  276. package/dist/providers/mistral.js +0 -46
  277. package/dist/providers/mistral.js.map +0 -1
  278. package/dist/providers/ollama.d.ts +0 -11
  279. package/dist/providers/ollama.d.ts.map +0 -1
  280. package/dist/providers/ollama.js +0 -15
  281. package/dist/providers/ollama.js.map +0 -1
  282. package/dist/providers/openai.d.ts +0 -79
  283. package/dist/providers/openai.d.ts.map +0 -1
  284. package/dist/providers/openai.js +0 -792
  285. package/dist/providers/openai.js.map +0 -1
  286. package/dist/providers/openrouter.d.ts +0 -43
  287. package/dist/providers/openrouter.d.ts.map +0 -1
  288. package/dist/providers/openrouter.js +0 -21
  289. package/dist/providers/openrouter.js.map +0 -1
  290. package/dist/providers/voyage.d.ts +0 -91
  291. package/dist/providers/voyage.d.ts.map +0 -1
  292. package/dist/providers/voyage.js +0 -166
  293. package/dist/providers/voyage.js.map +0 -1
  294. package/dist/providers/xai.d.ts +0 -12
  295. package/dist/providers/xai.d.ts.map +0 -1
  296. package/dist/providers/xai.js +0 -15
  297. package/dist/providers/xai.js.map +0 -1
  298. package/dist/queue-job.d.ts +0 -100
  299. package/dist/queue-job.d.ts.map +0 -1
  300. package/dist/queue-job.js +0 -185
  301. package/dist/queue-job.js.map +0 -1
  302. package/dist/react/agent-run.d.ts +0 -111
  303. package/dist/react/agent-run.d.ts.map +0 -1
  304. package/dist/react/agent-run.js +0 -107
  305. package/dist/react/agent-run.js.map +0 -1
  306. package/dist/react/useAgentRun.d.ts +0 -68
  307. package/dist/react/useAgentRun.d.ts.map +0 -1
  308. package/dist/react/useAgentRun.js +0 -125
  309. package/dist/react/useAgentRun.js.map +0 -1
  310. package/dist/registry.d.ts +0 -45
  311. package/dist/registry.d.ts.map +0 -1
  312. package/dist/registry.js +0 -131
  313. package/dist/registry.js.map +0 -1
  314. package/dist/rerank.d.ts +0 -20
  315. package/dist/rerank.d.ts.map +0 -1
  316. package/dist/rerank.js +0 -40
  317. package/dist/rerank.js.map +0 -1
  318. package/dist/resume-approval.d.ts +0 -30
  319. package/dist/resume-approval.d.ts.map +0 -1
  320. package/dist/resume-approval.js +0 -147
  321. package/dist/resume-approval.js.map +0 -1
  322. package/dist/sanitize-conversation.d.ts +0 -43
  323. package/dist/sanitize-conversation.d.ts.map +0 -1
  324. package/dist/sanitize-conversation.js +0 -85
  325. package/dist/sanitize-conversation.js.map +0 -1
  326. package/dist/scoped-tool.d.ts +0 -98
  327. package/dist/scoped-tool.d.ts.map +0 -1
  328. package/dist/scoped-tool.js +0 -174
  329. package/dist/scoped-tool.js.map +0 -1
  330. package/dist/server/provider.d.ts +0 -22
  331. package/dist/server/provider.d.ts.map +0 -1
  332. package/dist/server/provider.js +0 -194
  333. package/dist/server/provider.js.map +0 -1
  334. package/dist/similarity-search.d.ts +0 -163
  335. package/dist/similarity-search.d.ts.map +0 -1
  336. package/dist/similarity-search.js +0 -147
  337. package/dist/similarity-search.js.map +0 -1
  338. package/dist/sub-agent-run-store.d.ts +0 -157
  339. package/dist/sub-agent-run-store.d.ts.map +0 -1
  340. package/dist/sub-agent-run-store.js +0 -87
  341. package/dist/sub-agent-run-store.js.map +0 -1
  342. package/dist/tool-execution.d.ts +0 -16
  343. package/dist/tool-execution.d.ts.map +0 -1
  344. package/dist/tool-execution.js +0 -498
  345. package/dist/tool-execution.js.map +0 -1
  346. package/dist/tool-helpers.d.ts +0 -77
  347. package/dist/tool-helpers.d.ts.map +0 -1
  348. package/dist/tool-helpers.js +0 -117
  349. package/dist/tool-helpers.js.map +0 -1
  350. package/dist/tool.d.ts +0 -216
  351. package/dist/tool.d.ts.map +0 -1
  352. package/dist/tool.js +0 -175
  353. package/dist/tool.js.map +0 -1
  354. package/dist/transcription.d.ts +0 -42
  355. package/dist/transcription.d.ts.map +0 -1
  356. package/dist/transcription.js +0 -77
  357. package/dist/transcription.js.map +0 -1
  358. package/dist/types.d.ts +0 -1020
  359. package/dist/types.d.ts.map +0 -1
  360. package/dist/types.js +0 -2
  361. package/dist/types.js.map +0 -1
  362. package/dist/util/hash.d.ts +0 -11
  363. package/dist/util/hash.d.ts.map +0 -1
  364. package/dist/util/hash.js +0 -23
  365. package/dist/util/hash.js.map +0 -1
  366. package/dist/vector-stores/index.d.ts +0 -96
  367. package/dist/vector-stores/index.d.ts.map +0 -1
  368. package/dist/vector-stores/index.js +0 -153
  369. package/dist/vector-stores/index.js.map +0 -1
  370. package/dist/vercel-protocol.d.ts +0 -18
  371. package/dist/vercel-protocol.d.ts.map +0 -1
  372. package/dist/vercel-protocol.js +0 -75
  373. package/dist/vercel-protocol.js.map +0 -1
  374. package/dist/zod-to-json-schema.d.ts +0 -16
  375. package/dist/zod-to-json-schema.d.ts.map +0 -1
  376. package/dist/zod-to-json-schema.js +0 -17
  377. package/dist/zod-to-json-schema.js.map +0 -1
@@ -1,110 +0,0 @@
1
- /**
2
- * Fixture I/O for `pnpm rudder ai:eval --record` / `--replay` (#A5
3
- * Phase 4). Each case writes one JSON file under
4
- * `evals/__fixtures__/<suite>/<case>.json` carrying the assistant
5
- * turns from a real provider run, normalized into the
6
- * {@link AiFakeStep} shape so `--replay` can re-feed them via
7
- * `AiFake.respondWithSequence` for zero-API regression tests.
8
- *
9
- * The fixture format is versioned. Bumping `version` forces a
10
- * re-record on stale fixtures rather than silently mis-replaying.
11
- */
12
- import { mkdir, readFile, writeFile } from 'node:fs/promises';
13
- import path from 'node:path';
14
- /**
15
- * Convert an `AgentResponse` into the assistant-turn `AiFakeStep[]`
16
- * sequence that `AiFake.respondWithSequence` expects.
17
- *
18
- * - Drops user/tool turns — those are framework-generated during a
19
- * replayed run, not provider output.
20
- * - Multi-modal assistant content collapses to its concatenated text
21
- * parts (the fake's transport is text-only; image/document parts
22
- * wouldn't replay meaningfully).
23
- * - `toolCalls` carry through verbatim so multi-step tool loops
24
- * replay deterministically.
25
- */
26
- export function stepsFromResponse(response) {
27
- return response.steps
28
- .filter(step => step.message.role === 'assistant')
29
- .map(step => {
30
- const out = {
31
- text: contentToText(step.message.content),
32
- finishReason: step.finishReason,
33
- };
34
- if (step.toolCalls.length > 0)
35
- out.toolCalls = step.toolCalls;
36
- return out;
37
- });
38
- }
39
- function contentToText(content) {
40
- if (typeof content === 'string')
41
- return content;
42
- return content.filter(p => p.type === 'text').map(p => p.text).join('');
43
- }
44
- // ─── Fixture path conventions ─────────────────────────────
45
- /**
46
- * Default fixtures directory: `<cwd>/evals/__fixtures__`. Override
47
- * via the CLI handler's options for tests / non-standard layouts.
48
- */
49
- export function defaultFixturesDir(cwd) {
50
- return path.join(cwd, 'evals', '__fixtures__');
51
- }
52
- /**
53
- * Filesystem-safe slug for `<suite>/<case>` segments. Letters,
54
- * digits, dot, dash, underscore pass through; everything else
55
- * collapses to `-`. Multiple consecutive `-` collapse to one.
56
- *
57
- * Pure function; tested directly so suite/case rename diffs stay
58
- * predictable across editors.
59
- */
60
- export function slugify(s) {
61
- return s.replace(/[^A-Za-z0-9._-]+/g, '-').replace(/^-+|-+$/g, '') || '_';
62
- }
63
- export function fixturePath(dir, suite, caseName) {
64
- return path.join(dir, slugify(suite), `${slugify(caseName)}.json`);
65
- }
66
- // ─── Read / write ─────────────────────────────────────────
67
- /**
68
- * Read a fixture file. Returns `null` when the fixture is missing
69
- * (replay falls back to running normally with a clear stderr line).
70
- *
71
- * Throws on parse / version errors — corruption is not a passing
72
- * case and silently ignoring it would mask real regressions.
73
- */
74
- export async function readFixture(dir, suite, caseName) {
75
- const file = fixturePath(dir, suite, caseName);
76
- let raw;
77
- try {
78
- raw = await readFile(file, 'utf8');
79
- }
80
- catch (err) {
81
- if (err.code === 'ENOENT')
82
- return null;
83
- throw err;
84
- }
85
- const parsed = JSON.parse(raw);
86
- if (parsed.version !== 1) {
87
- throw new Error(`[Rudder AI] Fixture ${file} is version ${String(parsed.version)}; expected 1. ` +
88
- `Re-record with \`pnpm rudder ai:eval --record\`.`);
89
- }
90
- return parsed;
91
- }
92
- /**
93
- * Write a fixture, creating intermediate directories as needed.
94
- * Pretty-printed (2-space) so PR diffs remain readable when the
95
- * model output evolves.
96
- */
97
- export async function writeFixture(dir, suite, caseName, payload) {
98
- const file = fixturePath(dir, suite, caseName);
99
- await mkdir(path.dirname(file), { recursive: true });
100
- const fixture = {
101
- version: 1,
102
- suite,
103
- case: caseName,
104
- recordedAt: new Date().toISOString(),
105
- ...payload,
106
- };
107
- await writeFile(file, `${JSON.stringify(fixture, null, 2)}\n`);
108
- return file;
109
- }
110
- //# sourceMappingURL=fixtures.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"fixtures.js","sourceRoot":"","sources":["../../src/eval/fixtures.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC7D,OAAO,IAAI,MAAM,WAAW,CAAA;AAc5B;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAuB;IACvD,OAAO,QAAQ,CAAC,KAAK;SAClB,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,WAAW,CAAC;SACjD,GAAG,CAAC,IAAI,CAAC,EAAE;QACV,MAAM,GAAG,GAAe;YACtB,IAAI,EAAU,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;YACjD,YAAY,EAAE,IAAI,CAAC,YAAY;SAChC,CAAA;QACD,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;YAAE,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAA;QAC7D,OAAO,GAAG,CAAA;IACZ,CAAC,CAAC,CAAA;AACN,CAAC;AAED,SAAS,aAAa,CAAC,OAA+B;IACpD,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAA;IAC/C,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;AACzE,CAAC;AAED,6DAA6D;AAE7D;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC5C,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,cAAc,CAAC,CAAA;AAChD,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,OAAO,CAAC,CAAS;IAC/B,OAAO,CAAC,CAAC,OAAO,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,GAAG,CAAA;AAC3E,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,GAAW,EAAE,KAAa,EAAE,QAAgB;IACtE,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAA;AACpE,CAAC;AAED,6DAA6D;AAE7D;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,GAAgB,EAChB,KAAgB,EAChB,QAAgB;IAEhB,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAA;IAC9C,IAAI,GAAW,CAAA;IACf,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACpC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAA;QACjE,MAAM,GAAG,CAAA;IACX,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAgB,CAAA;IAC7C,IAAI,MAAM,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CACb,uBAAuB,IAAI,eAAe,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,gBAAgB;YAChF,kDAAkD,CACnD,CAAA;IACH,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAgB,EAChB,KAAgB,EAChB,QAAgB,EAChB,OAAwE;IAExE,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAA;IAC9C,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACpD,MAAM,OAAO,GAAgB;QAC3B,OAAO,EAAK,CAAC;QACb,KAAK;QACL,IAAI,EAAQ,QAAQ;QACpB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,GAAG,OAAO;KACX,CAAA;IACD,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;IAC9D,OAAO,IAAI,CAAA;AACb,CAAC"}
@@ -1,25 +0,0 @@
1
- /**
2
- * Self-contained HTML reporter for `pnpm rudder ai:eval --html`
3
- * (#A5 Phase 5). Renders one or more {@link SuiteReport}s into a
4
- * single HTML string with inline styles and minimal vanilla JS for
5
- * row expand/collapse — no framework deps, no external assets, safe
6
- * to paste into a PR comment or open offline.
7
- *
8
- * Defensive HTML-escape on every piece of user content (suite name,
9
- * case name, input, response, metadata). Long responses get a
10
- * `<pre>` block with `white-space: pre-wrap` so output stays
11
- * scannable without a horizontal scroll.
12
- */
13
- import type { SuiteReport } from './index.js';
14
- export interface HtmlReportOptions {
15
- /** Document `<title>`. Defaults to `"Eval Report"`. */
16
- title?: string;
17
- /** ISO timestamp shown in the header. Defaults to `new Date().toISOString()`. */
18
- generatedAt?: string;
19
- }
20
- /**
21
- * Render an array of {@link SuiteReport}s as a single self-contained
22
- * HTML document.
23
- */
24
- export declare function reportHtml(reports: SuiteReport[], opts?: HtmlReportOptions): string;
25
- //# sourceMappingURL=html-reporter.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"html-reporter.d.ts","sourceRoot":"","sources":["../../src/eval/html-reporter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAgB,MAAM,YAAY,CAAA;AAE3D,MAAM,WAAW,iBAAiB;IAChC,uDAAuD;IACvD,KAAK,CAAC,EAAQ,MAAM,CAAA;IACpB,iFAAiF;IACjF,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;;GAGG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,WAAW,EAAE,EAAE,IAAI,GAAE,iBAAsB,GAAG,MAAM,CA6CvF"}
@@ -1,209 +0,0 @@
1
- /**
2
- * Self-contained HTML reporter for `pnpm rudder ai:eval --html`
3
- * (#A5 Phase 5). Renders one or more {@link SuiteReport}s into a
4
- * single HTML string with inline styles and minimal vanilla JS for
5
- * row expand/collapse — no framework deps, no external assets, safe
6
- * to paste into a PR comment or open offline.
7
- *
8
- * Defensive HTML-escape on every piece of user content (suite name,
9
- * case name, input, response, metadata). Long responses get a
10
- * `<pre>` block with `white-space: pre-wrap` so output stays
11
- * scannable without a horizontal scroll.
12
- */
13
- /**
14
- * Render an array of {@link SuiteReport}s as a single self-contained
15
- * HTML document.
16
- */
17
- export function reportHtml(reports, opts = {}) {
18
- const title = opts.title ?? 'Eval Report';
19
- const generatedAt = opts.generatedAt ?? new Date().toISOString();
20
- const totals = reports.reduce((a, r) => ({
21
- cases: a.cases + r.cases.length,
22
- passed: a.passed + r.passed,
23
- failed: a.failed + r.failed,
24
- skipped: a.skipped + r.skipped,
25
- cost: a.cost + r.cost,
26
- tokens: a.tokens + r.tokens,
27
- duration: a.duration + r.duration,
28
- }), { cases: 0, passed: 0, failed: 0, skipped: 0, cost: 0, tokens: 0, duration: 0 });
29
- const passRate = totals.cases > 0 ? Math.round((totals.passed / totals.cases) * 100) : 0;
30
- return `<!DOCTYPE html>
31
- <html lang="en">
32
- <head>
33
- <meta charset="UTF-8">
34
- <meta name="viewport" content="width=device-width,initial-scale=1">
35
- <title>${escapeHtml(title)}</title>
36
- <style>${STYLE}</style>
37
- </head>
38
- <body>
39
- <header class="page-header">
40
- <h1>${escapeHtml(title)}</h1>
41
- <div class="meta">
42
- Generated ${escapeHtml(generatedAt)} ·
43
- ${reports.length} suite${plural(reports.length)} ·
44
- ${totals.cases} case${plural(totals.cases)} ·
45
- <strong class="${passRate === 100 ? 'ok' : passRate >= 80 ? 'warn' : 'bad'}">${passRate}% pass</strong> ·
46
- ${formatCost(totals.cost)} ·
47
- ${totals.tokens.toLocaleString()} tokens ·
48
- ${formatMs(totals.duration)}
49
- </div>
50
- </header>
51
- ${reports.map(renderSuite).join('\n')}
52
- <script>${SCRIPT}</script>
53
- </body>
54
- </html>
55
- `;
56
- }
57
- function renderSuite(r) {
58
- const passRate = r.cases.length > 0 ? Math.round((r.passed / r.cases.length) * 100) : 0;
59
- return `<section class="suite">
60
- <header class="suite-header">
61
- <h2>${escapeHtml(r.suite)}</h2>
62
- <div class="suite-stats">
63
- <span class="${passRate === 100 ? 'ok' : passRate >= 80 ? 'warn' : 'bad'}">${r.passed}/${r.cases.length} passed</span>
64
- ${r.skipped > 0 ? `· <span class="muted">${r.skipped} skipped</span>` : ''}
65
- · ${formatCost(r.cost)}
66
- · ${r.tokens.toLocaleString()} tokens
67
- · ${formatMs(r.duration)}
68
- </div>
69
- ${renderMetadata(r.metadata)}
70
- </header>
71
- <table class="cases">
72
- <thead>
73
- <tr>
74
- <th>Case</th>
75
- <th>Status</th>
76
- <th class="num">Tokens</th>
77
- <th class="num">Cost</th>
78
- <th class="num">Duration</th>
79
- </tr>
80
- </thead>
81
- <tbody>
82
- ${r.cases.map(renderCase).join('\n')}
83
- </tbody>
84
- </table>
85
- </section>`;
86
- }
87
- function renderCase(c) {
88
- const glyph = c.status === 'passed' ? '✓' : c.status === 'failed' ? '✗' : '○';
89
- const responseBlock = c.responseText !== undefined
90
- ? `<h4>Response</h4><pre>${escapeHtml(c.responseText)}</pre>`
91
- : '<h4>Response</h4><pre class="muted">&lt;no response — agent threw or skipped&gt;</pre>';
92
- const reasonBlock = (c.metric?.reason ?? c.reason)
93
- ? `<h4>Reason</h4><pre>${escapeHtml(c.metric?.reason ?? c.reason)}</pre>`
94
- : '';
95
- const scoreBlock = c.metric?.score !== undefined
96
- ? `<h4>Score</h4><pre>${c.metric.score.toFixed(3)}</pre>`
97
- : '';
98
- return ` <tr class="case ${c.status}" tabindex="0" aria-expanded="false">
99
- <td><span class="glyph">${glyph}</span> ${escapeHtml(c.name)}</td>
100
- <td><span class="badge ${c.status}">${c.status}</span></td>
101
- <td class="num">${c.tokens.toLocaleString()}</td>
102
- <td class="num">${formatCost(c.cost)}</td>
103
- <td class="num">${formatMs(c.duration)}</td>
104
- </tr>
105
- <tr class="case-detail" hidden>
106
- <td colspan="5">
107
- <h4>Input</h4>
108
- <pre>${escapeHtml(c.input)}</pre>
109
- ${responseBlock}
110
- ${scoreBlock}
111
- ${reasonBlock}
112
- </td>
113
- </tr>`;
114
- }
115
- function renderMetadata(meta) {
116
- if (!meta)
117
- return '';
118
- const rows = Object.entries(meta).filter(([, v]) => v !== undefined && v !== '');
119
- if (rows.length === 0)
120
- return '';
121
- return `<dl class="metadata">${rows.map(([k, v]) => `<dt>${escapeHtml(formatLabel(k))}</dt><dd>${escapeHtml(v)}</dd>`).join('')}</dl>`;
122
- }
123
- function formatLabel(key) {
124
- // camelCase → Title Case for the well-known keys; pass others through.
125
- if (key === 'lastReviewed')
126
- return 'Last reviewed';
127
- return key.charAt(0).toUpperCase() + key.slice(1);
128
- }
129
- // ─── HTML escape (no external dep) ───────────────────────
130
- const ESCAPE_MAP = {
131
- '&': '&amp;',
132
- '<': '&lt;',
133
- '>': '&gt;',
134
- '"': '&quot;',
135
- "'": '&#39;',
136
- };
137
- function escapeHtml(s) {
138
- return s.replace(/[&<>"']/g, ch => ESCAPE_MAP[ch]);
139
- }
140
- function plural(n) {
141
- return n === 1 ? '' : 's';
142
- }
143
- function formatMs(ms) {
144
- if (ms < 1000)
145
- return `${Math.round(ms)}ms`;
146
- return `${(ms / 1000).toFixed(1)}s`;
147
- }
148
- function formatCost(usd) {
149
- if (usd === 0)
150
- return '$0.000';
151
- if (usd < 0.001)
152
- return '<$0.001';
153
- return `$${usd.toFixed(3)}`;
154
- }
155
- // ─── Inline assets ────────────────────────────────────────
156
- const STYLE = `
157
- :root { color-scheme: light dark; --fg: #1a1a1a; --bg: #fff; --muted: #6a6a6a; --border: #e2e2e2; --row-hover: #f7f7f7; --ok: #1a7f37; --warn: #b08800; --bad: #b91c1c; --pre-bg: #f6f8fa; }
158
- @media (prefers-color-scheme: dark) {
159
- :root { --fg: #e6e6e6; --bg: #0d1117; --muted: #8a8a8a; --border: #30363d; --row-hover: #161b22; --ok: #3fb950; --warn: #d29922; --bad: #f85149; --pre-bg: #161b22; }
160
- }
161
- * { box-sizing: border-box }
162
- body { font: 14px/1.5 -apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif; color: var(--fg); background: var(--bg); margin: 0; padding: 24px; max-width: 1100px; margin-inline: auto }
163
- h1 { margin: 0 0 4px; font-size: 24px }
164
- h2 { margin: 0 0 4px; font-size: 18px }
165
- h4 { margin: 12px 0 4px; font-size: 12px; text-transform: uppercase; letter-spacing: .04em; color: var(--muted) }
166
- pre { background: var(--pre-bg); border: 1px solid var(--border); border-radius: 6px; padding: 8px 12px; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word; font: 12px/1.45 ui-monospace,SFMono-Regular,Menlo,monospace; margin: 0 }
167
- .page-header { border-bottom: 1px solid var(--border); padding-bottom: 12px; margin-bottom: 20px }
168
- .page-header .meta { color: var(--muted); font-size: 13px }
169
- .suite { margin-bottom: 28px }
170
- .suite-header h2 { display: inline }
171
- .suite-stats { color: var(--muted); font-size: 13px; margin-top: 4px }
172
- .metadata { display: grid; grid-template-columns: max-content 1fr; gap: 4px 12px; margin: 8px 0 0; font-size: 13px }
173
- .metadata dt { color: var(--muted); font-weight: normal }
174
- .metadata dd { margin: 0 }
175
- .cases { width: 100%; border-collapse: collapse; margin-top: 12px; border: 1px solid var(--border); border-radius: 6px; overflow: hidden }
176
- .cases th, .cases td { padding: 8px 12px; text-align: left; border-bottom: 1px solid var(--border) }
177
- .cases th { background: var(--pre-bg); font-weight: 600; font-size: 12px; text-transform: uppercase; letter-spacing: .04em; color: var(--muted) }
178
- .cases tr:last-child td { border-bottom: none }
179
- .cases tr.case { cursor: pointer; user-select: none }
180
- .cases tr.case:hover { background: var(--row-hover) }
181
- .cases tr.case:focus { outline: 2px solid var(--warn); outline-offset: -2px }
182
- .cases tr.case-detail td { background: var(--pre-bg) }
183
- .cases td.num { text-align: right; font-variant-numeric: tabular-nums }
184
- .glyph { display: inline-block; width: 14px; font-weight: bold }
185
- .cases tr.case.passed .glyph { color: var(--ok) }
186
- .cases tr.case.failed .glyph { color: var(--bad) }
187
- .cases tr.case.skipped .glyph { color: var(--muted) }
188
- .badge { display: inline-block; padding: 1px 8px; border-radius: 999px; font-size: 11px; font-weight: 600; text-transform: uppercase; letter-spacing: .04em }
189
- .badge.passed { background: rgba(63,185,80,.15); color: var(--ok) }
190
- .badge.failed { background: rgba(248,81,73,.15); color: var(--bad) }
191
- .badge.skipped { background: rgba(138,138,138,.15); color: var(--muted) }
192
- .ok { color: var(--ok) } .warn { color: var(--warn) } .bad { color: var(--bad) } .muted { color: var(--muted) }
193
- `.trim();
194
- const SCRIPT = `
195
- document.querySelectorAll('tr.case').forEach(function(row) {
196
- function toggle() {
197
- var detail = row.nextElementSibling;
198
- if (!detail || !detail.classList.contains('case-detail')) return;
199
- var open = !detail.hidden;
200
- detail.hidden = open;
201
- row.setAttribute('aria-expanded', String(!open));
202
- }
203
- row.addEventListener('click', toggle);
204
- row.addEventListener('keydown', function(e) {
205
- if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); toggle(); }
206
- });
207
- });
208
- `.trim();
209
- //# sourceMappingURL=html-reporter.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"html-reporter.js","sourceRoot":"","sources":["../../src/eval/html-reporter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAWH;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,OAAsB,EAAE,OAA0B,EAAE;IAC7E,MAAM,KAAK,GAAS,IAAI,CAAC,KAAK,IAAI,aAAa,CAAA;IAC/C,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAEhE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAC3B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACT,KAAK,EAAK,CAAC,CAAC,KAAK,GAAM,CAAC,CAAC,KAAK,CAAC,MAAM;QACrC,MAAM,EAAI,CAAC,CAAC,MAAM,GAAK,CAAC,CAAC,MAAM;QAC/B,MAAM,EAAI,CAAC,CAAC,MAAM,GAAK,CAAC,CAAC,MAAM;QAC/B,OAAO,EAAG,CAAC,CAAC,OAAO,GAAI,CAAC,CAAC,OAAO;QAChC,IAAI,EAAM,CAAC,CAAC,IAAI,GAAO,CAAC,CAAC,IAAI;QAC7B,MAAM,EAAI,CAAC,CAAC,MAAM,GAAK,CAAC,CAAC,MAAM;QAC/B,QAAQ,EAAE,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ;KAClC,CAAC,EACF,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAChF,CAAA;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAExF,OAAO;;;;;SAKA,UAAU,CAAC,KAAK,CAAC;SACjB,KAAK;;;;QAIN,UAAU,CAAC,KAAK,CAAC;;gBAET,UAAU,CAAC,WAAW,CAAC;MACjC,OAAO,CAAC,MAAM,SAAS,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;MAC7C,MAAM,CAAC,KAAK,QAAQ,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;qBACzB,QAAQ,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,KAAK,QAAQ;MACrF,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC;MACvB,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE;MAC9B,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;;;EAG7B,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;UAC3B,MAAM;;;CAGf,CAAA;AACD,CAAC;AAED,SAAS,WAAW,CAAC,CAAc;IACjC,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACvF,OAAO;;UAEC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;;qBAER,QAAQ,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM;QACrG,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,OAAO,iBAAiB,CAAC,CAAC,CAAC,EAAE;UACtE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;UAClB,CAAC,CAAC,MAAM,CAAC,cAAc,EAAE;UACzB,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;;MAExB,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC;;;;;;;;;;;;;EAa9B,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;;;WAGzB,CAAA;AACX,CAAC;AAED,SAAS,UAAU,CAAC,CAA+B;IACjD,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAA;IAC7E,MAAM,aAAa,GAAG,CAAC,CAAC,YAAY,KAAK,SAAS;QAChD,CAAC,CAAC,yBAAyB,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC,QAAQ;QAC7D,CAAC,CAAC,wFAAwF,CAAA;IAC5F,MAAM,WAAW,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC;QAChD,CAAC,CAAC,uBAAuB,UAAU,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC,CAAC,MAAO,CAAC,QAAQ;QAC1E,CAAC,CAAC,EAAE,CAAA;IACN,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,KAAK,SAAS;QAC9C,CAAC,CAAC,sBAAsB,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ;QACzD,CAAC,CAAC,EAAE,CAAA;IACN,OAAO,yBAAyB,CAAC,CAAC,MAAM;kCACR,KAAK,WAAW,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;iCACnC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;0BAC5B,CAAC,CAAC,MAAM,CAAC,cAAc,EAAE;0BACzB,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;0BAClB,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;;;;;iBAK7B,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;YACxB,aAAa;YACb,UAAU;YACV,WAAW;;YAEX,CAAA;AACZ,CAAC;AAED,SAAS,cAAc,CAAC,IAA8B;IACpD,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAA;IACpB,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,EAAE,CAAC,CAAA;IAChF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAA;IAChC,OAAO,wBACL,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,YAAY,UAAU,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAClG,OAAO,CAAA;AACT,CAAC;AAED,SAAS,WAAW,CAAC,GAAW;IAC9B,uEAAuE;IACvE,IAAI,GAAG,KAAK,cAAc;QAAE,OAAO,eAAe,CAAA;IAClD,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;AACnD,CAAC;AAED,4DAA4D;AAE5D,MAAM,UAAU,GAA2B;IACzC,GAAG,EAAG,OAAO;IACb,GAAG,EAAG,MAAM;IACZ,GAAG,EAAG,MAAM;IACZ,GAAG,EAAG,QAAQ;IACd,GAAG,EAAG,OAAO;CACd,CAAA;AACD,SAAS,UAAU,CAAC,CAAS;IAC3B,OAAO,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,CAAE,CAAC,CAAA;AACrD,CAAC;AAED,SAAS,MAAM,CAAC,CAAS;IACvB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAA;AAC3B,CAAC;AAED,SAAS,QAAQ,CAAC,EAAU;IAC1B,IAAI,EAAE,GAAG,IAAI;QAAE,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAA;IAC3C,OAAO,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;AACrC,CAAC;AAED,SAAS,UAAU,CAAC,GAAW;IAC7B,IAAI,GAAG,KAAK,CAAC;QAAM,OAAO,QAAQ,CAAA;IAClC,IAAI,GAAG,GAAG,KAAK;QAAI,OAAO,SAAS,CAAA;IACnC,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAA;AAC7B,CAAC;AAED,6DAA6D;AAE7D,MAAM,KAAK,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqCb,CAAC,IAAI,EAAE,CAAA;AAER,MAAM,MAAM,GAAG;;;;;;;;;;;;;;CAcd,CAAC,IAAI,EAAE,CAAA"}
@@ -1,43 +0,0 @@
1
- import type { SuiteReport } from './index.js';
2
- /**
3
- * Single case row in the JSON output. `metric.pass`/`metric.score`/
4
- * `metric.reason` are flattened up so consumers don't have to reach
5
- * through nested objects in CI scripts.
6
- */
7
- export interface SuiteJsonCase {
8
- name: string;
9
- status: 'passed' | 'failed' | 'skipped';
10
- pass: boolean;
11
- score?: number;
12
- reason?: string;
13
- duration: number;
14
- tokens: number;
15
- cost: number;
16
- }
17
- /**
18
- * Machine-readable suite output emitted by `pnpm rudder ai:eval --json`.
19
- * Stable shape — bumping fields here is a minor (additive) bump for
20
- * `@rudderjs/ai`. Removing or renaming fields is a major.
21
- */
22
- export interface SuiteJson {
23
- suite: string;
24
- passed: number;
25
- failed: number;
26
- skipped: number;
27
- duration: number;
28
- cost: number;
29
- tokens: number;
30
- cases: SuiteJsonCase[];
31
- }
32
- /**
33
- * JSON reporter — flattens a `SuiteReport` for CI consumption.
34
- *
35
- * Mirrors the `command_run` MCP tool envelope shape so the boost
36
- * agent surface and the eval CLI feel like one family.
37
- *
38
- * @example
39
- * const report = await runSuite(suite)
40
- * process.stdout.write(JSON.stringify(reportJson(report)))
41
- */
42
- export declare function reportJson(report: SuiteReport): SuiteJson;
43
- //# sourceMappingURL=json-reporter.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"json-reporter.d.ts","sourceRoot":"","sources":["../../src/eval/json-reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAE7C;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAO,MAAM,CAAA;IACjB,MAAM,EAAK,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAA;IAC1C,IAAI,EAAO,OAAO,CAAA;IAClB,KAAK,CAAC,EAAK,MAAM,CAAA;IACjB,MAAM,CAAC,EAAI,MAAM,CAAA;IACjB,QAAQ,EAAG,MAAM,CAAA;IACjB,MAAM,EAAK,MAAM,CAAA;IACjB,IAAI,EAAO,MAAM,CAAA;CAClB;AAED;;;;GAIG;AACH,MAAM,WAAW,SAAS;IACxB,KAAK,EAAK,MAAM,CAAA;IAChB,MAAM,EAAI,MAAM,CAAA;IAChB,MAAM,EAAI,MAAM,CAAA;IAChB,OAAO,EAAG,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,IAAI,EAAM,MAAM,CAAA;IAChB,MAAM,EAAI,MAAM,CAAA;IAChB,KAAK,EAAK,aAAa,EAAE,CAAA;CAC1B;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS,CAWzD"}
@@ -1,40 +0,0 @@
1
- /**
2
- * JSON reporter — flattens a `SuiteReport` for CI consumption.
3
- *
4
- * Mirrors the `command_run` MCP tool envelope shape so the boost
5
- * agent surface and the eval CLI feel like one family.
6
- *
7
- * @example
8
- * const report = await runSuite(suite)
9
- * process.stdout.write(JSON.stringify(reportJson(report)))
10
- */
11
- export function reportJson(report) {
12
- return {
13
- suite: report.suite,
14
- passed: report.passed,
15
- failed: report.failed,
16
- skipped: report.skipped,
17
- duration: report.duration,
18
- cost: report.cost,
19
- tokens: report.tokens,
20
- cases: report.cases.map(toJsonCase),
21
- };
22
- }
23
- function toJsonCase(c) {
24
- const out = {
25
- name: c.name,
26
- status: c.status,
27
- pass: c.status === 'passed',
28
- duration: c.duration,
29
- tokens: c.tokens,
30
- cost: c.cost,
31
- };
32
- if (c.metric?.score !== undefined)
33
- out.score = c.metric.score;
34
- if (c.status === 'skipped' && c.reason)
35
- out.reason = c.reason;
36
- else if (c.metric?.reason)
37
- out.reason = c.metric.reason;
38
- return out;
39
- }
40
- //# sourceMappingURL=json-reporter.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"json-reporter.js","sourceRoot":"","sources":["../../src/eval/json-reporter.ts"],"names":[],"mappings":"AAkCA;;;;;;;;;GASG;AACH,MAAM,UAAU,UAAU,CAAC,MAAmB;IAC5C,OAAO;QACL,KAAK,EAAK,MAAM,CAAC,KAAK;QACtB,MAAM,EAAI,MAAM,CAAC,MAAM;QACvB,MAAM,EAAI,MAAM,CAAC,MAAM;QACvB,OAAO,EAAG,MAAM,CAAC,OAAO;QACxB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,IAAI,EAAM,MAAM,CAAC,IAAI;QACrB,MAAM,EAAI,MAAM,CAAC,MAAM;QACvB,KAAK,EAAK,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC;KACvC,CAAA;AACH,CAAC;AAED,SAAS,UAAU,CAAC,CAA+B;IACjD,MAAM,GAAG,GAAkB;QACzB,IAAI,EAAM,CAAC,CAAC,IAAI;QAChB,MAAM,EAAI,CAAC,CAAC,MAAM;QAClB,IAAI,EAAM,CAAC,CAAC,MAAM,KAAK,QAAQ;QAC/B,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,MAAM,EAAI,CAAC,CAAC,MAAM;QAClB,IAAI,EAAM,CAAC,CAAC,IAAI;KACjB,CAAA;IACD,IAAI,CAAC,CAAC,MAAM,EAAE,KAAK,KAAK,SAAS;QAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAA;IAC7D,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM;QAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAA;SACxD,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM;QAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAA;IACvD,OAAO,GAAG,CAAA;AACZ,CAAC"}
package/dist/facade.d.ts DELETED
@@ -1,96 +0,0 @@
1
- import { ImageGenerator } from './image.js';
2
- import { AudioGenerator } from './audio.js';
3
- import { Transcription } from './transcription.js';
4
- import { Reranker } from './rerank.js';
5
- import { FileManager } from './files.js';
6
- import type { Agent } from './agent.js';
7
- import type { AgentResponse, AnyTool, AiMiddleware, EmbeddingResult, RerankingResult } from './types.js';
8
- /**
9
- * AI facade — static entry point for quick prompts, embeddings, and image generation.
10
- *
11
- * @example
12
- * const response = await AI.prompt('Hello')
13
- * const a = AI.agent('You are helpful.')
14
- * const result = await AI.embed('Some text')
15
- * const image = await AI.image('A sunset').generate()
16
- */
17
- export declare class AI {
18
- /** Quick prompt with default model */
19
- static prompt(input: string, options?: {
20
- model?: string | undefined;
21
- }): Promise<AgentResponse>;
22
- /** Create an anonymous agent */
23
- static agent(instructionsOrOptions: string | {
24
- instructions: string;
25
- tools?: AnyTool[] | undefined;
26
- model?: string | undefined;
27
- middleware?: AiMiddleware[] | undefined;
28
- }): Agent;
29
- /** Create an image generator with a fluent API */
30
- static image(prompt: string): ImageGenerator;
31
- /** Create a text-to-speech audio generator */
32
- static audio(text: string): AudioGenerator;
33
- /**
34
- * Create a speech-to-text transcription from raw audio bytes.
35
- *
36
- * For Node-only path-based loading, use `transcribeFromPath` from `@rudderjs/ai/node`.
37
- */
38
- static transcribe(bytes: Uint8Array): Transcription;
39
- /**
40
- * Rerank documents by relevance to a query.
41
- *
42
- * Returns a fluent builder when called with just query + documents.
43
- * Pass `options` for a one-shot call.
44
- *
45
- * @example
46
- * const result = await AI.rerank('search query', documents)
47
- * const result = await AI.rerank('query', docs, { model: 'cohere/rerank-v3.5', topK: 5 })
48
- */
49
- static rerank(query: string, documents: string[]): Reranker;
50
- static rerank(query: string, documents: string[], options: {
51
- model?: string | undefined;
52
- topK?: number | undefined;
53
- }): Promise<RerankingResult>;
54
- /**
55
- * Get a file manager for a provider.
56
- *
57
- * @example
58
- * const files = AI.files('openai')
59
- * const uploaded = await files.upload('./report.pdf', { purpose: 'assistants' })
60
- * const list = await files.list()
61
- * await files.delete(uploaded.id)
62
- */
63
- static files(provider?: string): FileManager;
64
- /**
65
- * Generate embeddings for text.
66
- *
67
- * Large arrays (100+ items) are automatically chunked into batches.
68
- * Pass `cache: true` to enable in-memory caching of embeddings.
69
- *
70
- * @example
71
- * const result = await AI.embed('Hello world')
72
- * const result = await AI.embed(['text1', 'text2'])
73
- * const result = await AI.embed('text', { model: 'openai/text-embedding-3-small' })
74
- * const result = await AI.embed('text', { cache: true })
75
- */
76
- static embed(input: string | string[], options?: {
77
- model?: string | undefined;
78
- cache?: boolean | undefined;
79
- }): Promise<EmbeddingResult>;
80
- /**
81
- * Cache `CachedEmbeddingAdapter` instances keyed by `<provider>::<model>` so
82
- * the in-memory cache persists across `AI.embed(..., { cache: true })` calls.
83
- *
84
- * The earlier `WeakMap<EmbeddingAdapter, CachedEmbeddingAdapter>` keyed on
85
- * adapter identity, but every `embed()` call constructed a fresh inner
86
- * adapter via `factory.createEmbedding(modelId)` — so the WeakMap lookup
87
- * always missed and `{ cache: true }` was a silent no-op. Keying by
88
- * `(provider, model)` is the only stable identity we have at this layer.
89
- *
90
- * Tied to `AiRegistry.reset()` so tests that swap fakes between resets
91
- * don't bind to a stale inner adapter.
92
- */
93
- private static cachedAdapters;
94
- private static getCachedAdapter;
95
- }
96
- //# sourceMappingURL=facade.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"facade.d.ts","sourceRoot":"","sources":["../src/facade.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAExC,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AACvC,OAAO,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,YAAY,EAAoB,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAE1H;;;;;;;;GAQG;AACH,qBAAa,EAAE;IACb,sCAAsC;WACzB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,GAAG,OAAO,CAAC,aAAa,CAAC;IAQpG,gCAAgC;IAChC,MAAM,CAAC,KAAK,CACV,qBAAqB,EAAE,MAAM,GAAG;QAC9B,YAAY,EAAE,MAAM,CAAA;QACpB,KAAK,CAAC,EAAE,OAAO,EAAE,GAAG,SAAS,CAAA;QAC7B,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;QAC1B,UAAU,CAAC,EAAE,YAAY,EAAE,GAAG,SAAS,CAAA;KACxC,GACA,KAAK;IAIR,kDAAkD;IAClD,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc;IAI5C,8CAA8C;IAC9C,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc;IAI1C;;;;OAIG;IACH,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,UAAU,GAAG,aAAa;IAInD;;;;;;;;;OASG;IACH,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,QAAQ;IAC3D,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,GAAG,OAAO,CAAC,eAAe,CAAC;IAa/I;;;;;;;;OAQG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,WAAW;IAK5C;;;;;;;;;;;OAWG;WACU,KAAK,CAChB,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,EACxB,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;KAAE,GACpE,OAAO,CAAC,eAAe,CAAC;IA0C3B;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,MAAM,CAAC,cAAc,CAA4C;IAEzE,OAAO,CAAC,MAAM,CAAC,gBAAgB;CAgBhC"}