@terreno/ai 57.8.0 → 57.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (425) hide show
  1. package/.cursorrules +2 -2
  2. package/README.md +2 -1
  3. package/bunfig.toml +1 -1
  4. package/dist/aiApp.d.ts +16 -5
  5. package/dist/aiApp.js +17 -7
  6. package/dist/harness/adminScreens.d.ts +6 -0
  7. package/dist/harness/adminScreens.js +11 -0
  8. package/dist/harness/agentLoop.d.ts +61 -0
  9. package/dist/harness/agentLoop.js +798 -0
  10. package/dist/harness/agentTaskNames.d.ts +4 -0
  11. package/dist/harness/agentTaskNames.js +7 -0
  12. package/dist/harness/approvalPolicy.d.ts +3 -0
  13. package/dist/harness/approvalPolicy.js +23 -0
  14. package/dist/harness/approvals.d.ts +101 -0
  15. package/dist/harness/approvals.js +468 -0
  16. package/dist/harness/commit.d.ts +238 -0
  17. package/dist/harness/commit.js +678 -0
  18. package/dist/harness/conversation.d.ts +75 -0
  19. package/dist/harness/conversation.js +397 -0
  20. package/dist/harness/defineAgent.d.ts +9 -0
  21. package/dist/harness/defineAgent.js +65 -0
  22. package/dist/harness/defineTask.d.ts +11 -0
  23. package/dist/harness/defineTask.js +59 -0
  24. package/dist/harness/defineTool.d.ts +7 -0
  25. package/dist/harness/defineTool.js +36 -0
  26. package/dist/harness/definitionError.d.ts +8 -0
  27. package/dist/harness/definitionError.js +15 -0
  28. package/dist/harness/errors.d.ts +127 -0
  29. package/dist/harness/errors.js +101 -0
  30. package/dist/harness/eventHub.d.ts +45 -0
  31. package/dist/harness/eventHub.js +135 -0
  32. package/dist/harness/events.d.ts +95 -0
  33. package/dist/harness/events.js +267 -0
  34. package/dist/harness/executionEnv.d.ts +31 -0
  35. package/dist/harness/executionEnv.js +2 -0
  36. package/dist/harness/extensions.d.ts +82 -0
  37. package/dist/harness/extensions.js +274 -0
  38. package/dist/harness/harness.d.ts +163 -0
  39. package/dist/harness/harness.js +520 -0
  40. package/dist/harness/harnessApp.d.ts +44 -0
  41. package/dist/harness/harnessApp.js +62 -0
  42. package/dist/harness/internalRuntime.d.ts +16 -0
  43. package/dist/harness/internalRuntime.js +18 -0
  44. package/dist/harness/leases.d.ts +35 -0
  45. package/dist/harness/leases.js +86 -0
  46. package/dist/harness/memo.d.ts +24 -0
  47. package/dist/harness/memo.js +78 -0
  48. package/dist/harness/modelCall.d.ts +45 -0
  49. package/dist/harness/modelCall.js +147 -0
  50. package/dist/harness/models/harnessApproval.d.ts +2 -0
  51. package/dist/harness/models/harnessApproval.js +100 -0
  52. package/dist/harness/models/harnessConversation.d.ts +2 -0
  53. package/dist/harness/models/harnessConversation.js +117 -0
  54. package/dist/harness/models/harnessEvent.d.ts +3 -0
  55. package/dist/harness/models/harnessEvent.js +73 -0
  56. package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
  57. package/dist/harness/models/harnessInboxEvent.js +58 -0
  58. package/dist/harness/models/harnessMemo.d.ts +2 -0
  59. package/dist/harness/models/harnessMemo.js +42 -0
  60. package/dist/harness/models/harnessMessage.d.ts +2 -0
  61. package/dist/harness/models/harnessMessage.js +73 -0
  62. package/dist/harness/models/harnessOwner.d.ts +2 -0
  63. package/dist/harness/models/harnessOwner.js +36 -0
  64. package/dist/harness/models/harnessTask.d.ts +2 -0
  65. package/dist/harness/models/harnessTask.js +186 -0
  66. package/dist/harness/ownership.d.ts +89 -0
  67. package/dist/harness/ownership.js +438 -0
  68. package/dist/harness/registry.d.ts +29 -0
  69. package/dist/harness/registry.js +93 -0
  70. package/dist/harness/retryBackoff.d.ts +31 -0
  71. package/dist/harness/retryBackoff.js +50 -0
  72. package/dist/harness/routes/approvals.d.ts +13 -0
  73. package/dist/harness/routes/approvals.js +93 -0
  74. package/dist/harness/routes/conversations.d.ts +11 -0
  75. package/dist/harness/routes/conversations.js +66 -0
  76. package/dist/harness/routes/events.d.ts +19 -0
  77. package/dist/harness/routes/events.js +215 -0
  78. package/dist/harness/routes/tasks.d.ts +11 -0
  79. package/dist/harness/routes/tasks.js +108 -0
  80. package/dist/harness/runners/inProcessRunner.d.ts +57 -0
  81. package/dist/harness/runners/inProcessRunner.js +196 -0
  82. package/dist/harness/runtime.d.ts +34 -0
  83. package/dist/harness/runtime.js +551 -0
  84. package/dist/harness/subagent.d.ts +53 -0
  85. package/dist/harness/subagent.js +134 -0
  86. package/dist/harness/suspend.d.ts +8 -0
  87. package/dist/harness/suspend.js +16 -0
  88. package/dist/harness/transaction.d.ts +3 -0
  89. package/dist/harness/transaction.js +22 -0
  90. package/dist/harness/waits.d.ts +100 -0
  91. package/dist/harness/waits.js +214 -0
  92. package/dist/index.d.ts +19 -1
  93. package/dist/index.js +45 -2
  94. package/dist/models/aiDataset.d.ts +2 -0
  95. package/dist/models/aiDataset.js +67 -0
  96. package/dist/models/aiRequest.js +1 -1
  97. package/dist/models/gptHistory.js +88 -2
  98. package/dist/observability/adminScreens.d.ts +5 -0
  99. package/dist/observability/adminScreens.js +32 -0
  100. package/dist/observability/compileTemplate.d.ts +1 -0
  101. package/dist/observability/compileTemplate.js +13 -0
  102. package/dist/observability/datasetImport.d.ts +11 -0
  103. package/dist/observability/datasetImport.js +185 -0
  104. package/dist/observability/evaluate.d.ts +31 -0
  105. package/dist/observability/evaluate.js +200 -0
  106. package/dist/observability/evaluatorTemplates.d.ts +17 -0
  107. package/dist/observability/evaluatorTemplates.js +111 -0
  108. package/dist/observability/jsonPath.d.ts +2 -0
  109. package/dist/observability/jsonPath.js +57 -0
  110. package/dist/observability/local/datasetStore.d.ts +87 -0
  111. package/dist/observability/local/datasetStore.js +326 -0
  112. package/dist/observability/local/evaluatorStore.d.ts +44 -0
  113. package/dist/observability/local/evaluatorStore.js +182 -0
  114. package/dist/observability/local/experimentRunner.d.ts +96 -0
  115. package/dist/observability/local/experimentRunner.js +419 -0
  116. package/dist/observability/local/localPlugin.d.ts +22 -0
  117. package/dist/observability/local/localPlugin.js +76 -0
  118. package/dist/observability/local/models/obsDataset.d.ts +2 -0
  119. package/dist/observability/local/models/obsDataset.js +37 -0
  120. package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
  121. package/dist/observability/local/models/obsDatasetItem.js +76 -0
  122. package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
  123. package/dist/observability/local/models/obsEvaluator.js +98 -0
  124. package/dist/observability/local/models/obsExperiment.d.ts +2 -0
  125. package/dist/observability/local/models/obsExperiment.js +87 -0
  126. package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
  127. package/dist/observability/local/models/obsExperimentItem.js +45 -0
  128. package/dist/observability/local/models/obsPrompt.d.ts +2 -0
  129. package/dist/observability/local/models/obsPrompt.js +42 -0
  130. package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
  131. package/dist/observability/local/models/obsPromptLabel.js +39 -0
  132. package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
  133. package/dist/observability/local/models/obsPromptVersion.js +96 -0
  134. package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
  135. package/dist/observability/local/models/obsReviewItem.js +75 -0
  136. package/dist/observability/local/models/obsScore.d.ts +2 -0
  137. package/dist/observability/local/models/obsScore.js +58 -0
  138. package/dist/observability/local/models/obsSpan.d.ts +2 -0
  139. package/dist/observability/local/models/obsSpan.js +64 -0
  140. package/dist/observability/local/models/obsTrace.d.ts +2 -0
  141. package/dist/observability/local/models/obsTrace.js +93 -0
  142. package/dist/observability/local/promptStore.d.ts +180 -0
  143. package/dist/observability/local/promptStore.js +444 -0
  144. package/dist/observability/local/reviewStore.d.ts +50 -0
  145. package/dist/observability/local/reviewStore.js +203 -0
  146. package/dist/observability/local/traceStore.d.ts +103 -0
  147. package/dist/observability/local/traceStore.js +325 -0
  148. package/dist/observability/observabilityApp.d.ts +24 -0
  149. package/dist/observability/observabilityApp.js +157 -0
  150. package/dist/observability/observabilityAppRegistry.d.ts +14 -0
  151. package/dist/observability/observabilityAppRegistry.js +16 -0
  152. package/dist/observability/permissions.d.ts +7 -0
  153. package/dist/observability/permissions.js +60 -0
  154. package/dist/observability/reviewPanels.d.ts +18 -0
  155. package/dist/observability/reviewPanels.js +46 -0
  156. package/dist/observability/routes/datasets.d.ts +7 -0
  157. package/dist/observability/routes/datasets.js +149 -0
  158. package/dist/observability/routes/evaluators.d.ts +7 -0
  159. package/dist/observability/routes/evaluators.js +123 -0
  160. package/dist/observability/routes/experiments.d.ts +7 -0
  161. package/dist/observability/routes/experiments.js +95 -0
  162. package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
  163. package/dist/observability/routes/observabilityRouteAccess.js +67 -0
  164. package/dist/observability/routes/prompts.d.ts +12 -0
  165. package/dist/observability/routes/prompts.js +172 -0
  166. package/dist/observability/routes/requestAiService.d.ts +13 -0
  167. package/dist/observability/routes/requestAiService.js +21 -0
  168. package/dist/observability/routes/review.d.ts +7 -0
  169. package/dist/observability/routes/review.js +79 -0
  170. package/dist/observability/routes/status.d.ts +5 -0
  171. package/dist/observability/routes/status.js +31 -0
  172. package/dist/observability/routes/testMultiStage.d.ts +9 -0
  173. package/dist/observability/routes/testMultiStage.js +67 -0
  174. package/dist/observability/routes/traces.d.ts +7 -0
  175. package/dist/observability/routes/traces.js +129 -0
  176. package/dist/observability/schemaValidation.d.ts +5 -0
  177. package/dist/observability/schemaValidation.js +26 -0
  178. package/dist/observability/sopThresholds.d.ts +2 -0
  179. package/dist/observability/sopThresholds.js +21 -0
  180. package/dist/observability/status.d.ts +23 -0
  181. package/dist/observability/status.js +34 -0
  182. package/dist/observability/testMultiStageSchemas.d.ts +10 -0
  183. package/dist/observability/testMultiStageSchemas.js +79 -0
  184. package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
  185. package/dist/observability/testMultiStageWorkflow.js +218 -0
  186. package/dist/observability/types.d.ts +159 -0
  187. package/dist/observability/types.js +67 -0
  188. package/dist/routes/files.js +20 -7
  189. package/dist/routes/gpt.js +179 -488
  190. package/dist/routes/gptActions.d.ts +9 -0
  191. package/dist/routes/gptActions.js +212 -0
  192. package/dist/routes/gptDatasets.d.ts +5 -0
  193. package/dist/routes/gptDatasets.js +94 -0
  194. package/dist/routes/gptHistories.d.ts +3 -1
  195. package/dist/routes/gptHistories.js +188 -2
  196. package/dist/service/aiDatasets.d.ts +57 -0
  197. package/dist/service/aiDatasets.js +253 -0
  198. package/dist/service/aiService.d.ts +50 -1
  199. package/dist/service/aiService.js +431 -77
  200. package/dist/service/askFiles.d.ts +49 -0
  201. package/dist/service/askFiles.js +162 -0
  202. package/dist/service/asks.d.ts +76 -0
  203. package/dist/service/asks.js +284 -0
  204. package/dist/service/chatTurn.d.ts +128 -0
  205. package/dist/service/chatTurn.js +1674 -0
  206. package/dist/service/fileStorage.d.ts +4 -0
  207. package/dist/service/fileStorage.js +7 -2
  208. package/dist/service/fileUploadsGate.d.ts +8 -0
  209. package/dist/service/fileUploadsGate.js +18 -0
  210. package/dist/service/fillActionIds.d.ts +6 -0
  211. package/dist/service/fillActionIds.js +79 -0
  212. package/dist/service/gemini.d.ts +3 -3
  213. package/dist/service/gemini.js +3 -3
  214. package/dist/service/jsonFenceModel.d.ts +7 -0
  215. package/dist/service/jsonFenceModel.js +45 -0
  216. package/dist/service/prompts.d.ts +52 -0
  217. package/dist/service/prompts.js +96 -1
  218. package/dist/service/sanitizeHtml.d.ts +11 -0
  219. package/dist/service/sanitizeHtml.js +108 -0
  220. package/dist/service/vertex.d.ts +6 -6
  221. package/dist/service/vertex.js +5 -5
  222. package/dist/types/harness.d.ts +934 -0
  223. package/dist/types/harness.js +133 -0
  224. package/dist/types/index.d.ts +219 -13
  225. package/dist/types/index.js +2 -0
  226. package/dist/types/observability.d.ts +288 -0
  227. package/dist/types/observability.js +2 -0
  228. package/package.json +17 -5
  229. package/src/aiApp.test.ts +207 -1
  230. package/src/aiApp.ts +31 -8
  231. package/src/harness/adminScreens.test.ts +34 -0
  232. package/src/harness/adminScreens.ts +11 -0
  233. package/src/harness/agentLoop.test.ts +1034 -0
  234. package/src/harness/agentLoop.ts +1082 -0
  235. package/src/harness/agentRecovery.test.ts +380 -0
  236. package/src/harness/agentTaskNames.ts +4 -0
  237. package/src/harness/approvalPolicy.ts +28 -0
  238. package/src/harness/approvals.test.ts +1094 -0
  239. package/src/harness/approvals.ts +644 -0
  240. package/src/harness/commit.ts +1049 -0
  241. package/src/harness/conversation.ts +503 -0
  242. package/src/harness/defineAgent.test.ts +92 -0
  243. package/src/harness/defineAgent.ts +82 -0
  244. package/src/harness/defineTask.test.ts +59 -0
  245. package/src/harness/defineTask.ts +76 -0
  246. package/src/harness/defineTool.ts +45 -0
  247. package/src/harness/definitionError.ts +13 -0
  248. package/src/harness/errors.test.ts +90 -0
  249. package/src/harness/errors.ts +119 -0
  250. package/src/harness/eventHub.test.ts +124 -0
  251. package/src/harness/eventHub.ts +159 -0
  252. package/src/harness/events.test.ts +757 -0
  253. package/src/harness/events.ts +384 -0
  254. package/src/harness/executionEnv.ts +34 -0
  255. package/src/harness/extensions.test.ts +1218 -0
  256. package/src/harness/extensions.ts +382 -0
  257. package/src/harness/harness.test.ts +693 -0
  258. package/src/harness/harness.ts +767 -0
  259. package/src/harness/harnessApp.ts +76 -0
  260. package/src/harness/internalRuntime.ts +44 -0
  261. package/src/harness/leases.ts +132 -0
  262. package/src/harness/memo.ts +107 -0
  263. package/src/harness/modelCall.test.ts +134 -0
  264. package/src/harness/modelCall.ts +200 -0
  265. package/src/harness/models/harnessApproval.ts +109 -0
  266. package/src/harness/models/harnessConversation.ts +142 -0
  267. package/src/harness/models/harnessEvent.ts +92 -0
  268. package/src/harness/models/harnessInboxEvent.ts +73 -0
  269. package/src/harness/models/harnessMemo.ts +43 -0
  270. package/src/harness/models/harnessMessage.ts +85 -0
  271. package/src/harness/models/harnessModels.test.ts +60 -0
  272. package/src/harness/models/harnessOwner.ts +40 -0
  273. package/src/harness/models/harnessTask.ts +201 -0
  274. package/src/harness/ownership.test.ts +1087 -0
  275. package/src/harness/ownership.ts +618 -0
  276. package/src/harness/recovery.test.ts +912 -0
  277. package/src/harness/registry.ts +124 -0
  278. package/src/harness/retries.test.ts +292 -0
  279. package/src/harness/retryBackoff.test.ts +100 -0
  280. package/src/harness/retryBackoff.ts +77 -0
  281. package/src/harness/routes/approvals.ts +123 -0
  282. package/src/harness/routes/conversations.test.ts +518 -0
  283. package/src/harness/routes/conversations.ts +102 -0
  284. package/src/harness/routes/events.ts +280 -0
  285. package/src/harness/routes/tasks.test.ts +198 -0
  286. package/src/harness/routes/tasks.ts +146 -0
  287. package/src/harness/runners/inProcessRunner.test.ts +402 -0
  288. package/src/harness/runners/inProcessRunner.ts +220 -0
  289. package/src/harness/runtime.ts +714 -0
  290. package/src/harness/subagent.test.ts +696 -0
  291. package/src/harness/subagent.ts +192 -0
  292. package/src/harness/suspend.ts +13 -0
  293. package/src/harness/transaction.ts +17 -0
  294. package/src/harness/versioning.test.ts +288 -0
  295. package/src/harness/waits.test.ts +693 -0
  296. package/src/harness/waits.ts +356 -0
  297. package/src/index.test.ts +13 -0
  298. package/src/index.ts +70 -0
  299. package/src/isolated/adminScreensConfig.isolated.ts +143 -0
  300. package/src/models/aiDataset.test.ts +34 -0
  301. package/src/models/aiDataset.ts +78 -0
  302. package/src/models/aiRequest.test.ts +2 -2
  303. package/src/models/aiRequest.ts +2 -1
  304. package/src/models/gptHistory.test.ts +156 -0
  305. package/src/models/gptHistory.ts +106 -4
  306. package/src/observability/adminScreens.test.ts +71 -0
  307. package/src/observability/adminScreens.ts +42 -0
  308. package/src/observability/compileTemplate.test.ts +14 -0
  309. package/src/observability/compileTemplate.ts +10 -0
  310. package/src/observability/datasetImport.test.ts +95 -0
  311. package/src/observability/datasetImport.ts +220 -0
  312. package/src/observability/evaluate.test.ts +549 -0
  313. package/src/observability/evaluate.ts +254 -0
  314. package/src/observability/evaluatorTemplates.ts +124 -0
  315. package/src/observability/jsonPath.test.ts +52 -0
  316. package/src/observability/jsonPath.ts +60 -0
  317. package/src/observability/local/datasetStore.test.ts +230 -0
  318. package/src/observability/local/datasetStore.ts +452 -0
  319. package/src/observability/local/evaluatorStore.test.ts +138 -0
  320. package/src/observability/local/evaluatorStore.ts +224 -0
  321. package/src/observability/local/experimentRunner.test.ts +489 -0
  322. package/src/observability/local/experimentRunner.ts +548 -0
  323. package/src/observability/local/localPlugin.ts +88 -0
  324. package/src/observability/local/models/obsDataset.ts +38 -0
  325. package/src/observability/local/models/obsDatasetItem.ts +80 -0
  326. package/src/observability/local/models/obsEvaluator.ts +102 -0
  327. package/src/observability/local/models/obsExperiment.ts +91 -0
  328. package/src/observability/local/models/obsExperimentItem.ts +52 -0
  329. package/src/observability/local/models/obsPrompt.ts +43 -0
  330. package/src/observability/local/models/obsPromptLabel.ts +43 -0
  331. package/src/observability/local/models/obsPromptVersion.ts +107 -0
  332. package/src/observability/local/models/obsReviewItem.ts +79 -0
  333. package/src/observability/local/models/obsScore.ts +59 -0
  334. package/src/observability/local/models/obsSpan.ts +65 -0
  335. package/src/observability/local/models/obsTrace.ts +94 -0
  336. package/src/observability/local/obsModels.test.ts +119 -0
  337. package/src/observability/local/promptStore.test.ts +351 -0
  338. package/src/observability/local/promptStore.ts +665 -0
  339. package/src/observability/local/reviewStore.test.ts +164 -0
  340. package/src/observability/local/reviewStore.ts +264 -0
  341. package/src/observability/local/traceStore.test.ts +264 -0
  342. package/src/observability/local/traceStore.ts +440 -0
  343. package/src/observability/observabilityApp.test.ts +92 -0
  344. package/src/observability/observabilityApp.ts +185 -0
  345. package/src/observability/observabilityAppRegistry.test.ts +30 -0
  346. package/src/observability/observabilityAppRegistry.ts +35 -0
  347. package/src/observability/permissions.test.ts +96 -0
  348. package/src/observability/permissions.ts +78 -0
  349. package/src/observability/reviewPanels.test.ts +35 -0
  350. package/src/observability/reviewPanels.ts +64 -0
  351. package/src/observability/routes/datasets.test.ts +176 -0
  352. package/src/observability/routes/datasets.ts +252 -0
  353. package/src/observability/routes/evaluators.test.ts +119 -0
  354. package/src/observability/routes/evaluators.ts +207 -0
  355. package/src/observability/routes/experiments.test.ts +282 -0
  356. package/src/observability/routes/experiments.ts +152 -0
  357. package/src/observability/routes/observabilityRbac.test.ts +767 -0
  358. package/src/observability/routes/observabilityRouteAccess.ts +106 -0
  359. package/src/observability/routes/prompts.test.ts +320 -0
  360. package/src/observability/routes/prompts.ts +263 -0
  361. package/src/observability/routes/requestAiService.ts +36 -0
  362. package/src/observability/routes/review.test.ts +120 -0
  363. package/src/observability/routes/review.ts +125 -0
  364. package/src/observability/routes/status.test.ts +116 -0
  365. package/src/observability/routes/status.ts +47 -0
  366. package/src/observability/routes/testMultiStage.test.ts +214 -0
  367. package/src/observability/routes/testMultiStage.ts +96 -0
  368. package/src/observability/routes/traces.test.ts +167 -0
  369. package/src/observability/routes/traces.ts +172 -0
  370. package/src/observability/schemaValidation.ts +29 -0
  371. package/src/observability/sopThresholds.ts +20 -0
  372. package/src/observability/status.test.ts +38 -0
  373. package/src/observability/status.ts +56 -0
  374. package/src/observability/testMultiStageSchemas.ts +95 -0
  375. package/src/observability/testMultiStageWorkflow.test.ts +114 -0
  376. package/src/observability/testMultiStageWorkflow.ts +307 -0
  377. package/src/observability/types.ts +262 -0
  378. package/src/routes/files.test.ts +84 -9
  379. package/src/routes/files.ts +23 -7
  380. package/src/routes/gpt.test.ts +3279 -0
  381. package/src/routes/gpt.ts +214 -552
  382. package/src/routes/gptActions.test.ts +241 -0
  383. package/src/routes/gptActions.ts +255 -0
  384. package/src/routes/gptApprovals.test.ts +762 -0
  385. package/src/routes/gptDatasets.test.ts +86 -0
  386. package/src/routes/gptDatasets.ts +104 -0
  387. package/src/routes/gptFiles.test.ts +589 -0
  388. package/src/routes/gptHistories.test.ts +933 -0
  389. package/src/routes/gptHistories.ts +266 -5
  390. package/src/routes/routes.test.ts +676 -5
  391. package/src/service/aiDatasets.test.ts +123 -0
  392. package/src/service/aiDatasets.ts +316 -0
  393. package/src/service/aiService.test.ts +614 -0
  394. package/src/service/aiService.ts +575 -111
  395. package/src/service/askFiles.test.ts +96 -0
  396. package/src/service/askFiles.ts +222 -0
  397. package/src/service/asks.test.ts +946 -0
  398. package/src/service/asks.ts +423 -0
  399. package/src/service/chatTurn.ts +2287 -0
  400. package/src/service/fileStorage.test.ts +11 -0
  401. package/src/service/fileStorage.ts +10 -2
  402. package/src/service/fileUploadsGate.ts +23 -0
  403. package/src/service/fillActionIds.test.ts +65 -0
  404. package/src/service/fillActionIds.ts +98 -0
  405. package/src/service/gemini.test.ts +16 -10
  406. package/src/service/gemini.ts +3 -3
  407. package/src/service/jsonFenceModel.test.ts +47 -0
  408. package/src/service/jsonFenceModel.ts +53 -0
  409. package/src/service/prompts.test.ts +21 -0
  410. package/src/service/prompts.ts +147 -0
  411. package/src/service/sanitizeHtml.test.ts +61 -0
  412. package/src/service/sanitizeHtml.ts +109 -0
  413. package/src/service/vertex.test.ts +49 -49
  414. package/src/service/vertex.ts +7 -8
  415. package/src/tests/bunSetup.test.ts +7 -4
  416. package/src/tests/bunSetup.ts +7 -7
  417. package/src/tests/chatHarness.ts +478 -0
  418. package/src/tests/generateStream.ts +49 -0
  419. package/src/tests/harnessErrors.ts +22 -0
  420. package/src/tests/harnessStreaming.ts +227 -0
  421. package/src/tests/helpers.ts +9 -2
  422. package/src/types/harness.ts +1095 -0
  423. package/src/types/index.ts +237 -12
  424. package/src/types/observability.ts +344 -0
  425. package/src/types/sanitize-html.d.ts +25 -0
@@ -0,0 +1,164 @@
1
+ import {afterEach, beforeEach, describe, expect, it} from "bun:test";
2
+ import {assert} from "chai";
3
+ import {DateTime} from "luxon";
4
+ import mongoose from "mongoose";
5
+ import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
6
+ import {LocalEvaluatorStore} from "./evaluatorStore";
7
+ import {createLocalObservabilityPlugin} from "./localPlugin";
8
+ import {registerObsEvaluator} from "./models/obsEvaluator";
9
+ import {registerObsReviewItem} from "./models/obsReviewItem";
10
+ import {registerObsTrace} from "./models/obsTrace";
11
+ import {LocalReviewStore} from "./reviewStore";
12
+ import {LocalTraceStore, MemoryScoreSink} from "./traceStore";
13
+
14
+ describe("LocalReviewStore", () => {
15
+ let reviewStore: LocalReviewStore;
16
+ let evaluatorId: string;
17
+ let traceId: string;
18
+ let scoreSink: MemoryScoreSink;
19
+
20
+ afterEach(() => {
21
+ resetObservabilityApp();
22
+ });
23
+
24
+ beforeEach(async () => {
25
+ scoreSink = new MemoryScoreSink();
26
+ const plugin = createLocalObservabilityPlugin();
27
+ plugin.scoreSink = scoreSink;
28
+ new ObservabilityApp({plugins: [plugin]});
29
+ reviewStore = new LocalReviewStore();
30
+ await registerObsReviewItem().deleteMany({});
31
+ await registerObsEvaluator().deleteMany({});
32
+ await registerObsTrace().deleteMany({});
33
+
34
+ const evaluator = await new LocalEvaluatorStore().create({
35
+ dimensions: [{dataType: "boolean", key: "correct", required: true}],
36
+ instructions: "Pass if correct.",
37
+ name: "correctness-review",
38
+ target: "full trace",
39
+ type: "human",
40
+ });
41
+ evaluatorId = evaluator.id;
42
+ const exported = await new LocalTraceStore().exportTrace({
43
+ id: "t",
44
+ input: {raw: "prompt text"},
45
+ name: "gen",
46
+ output: {raw: "model text"},
47
+ prompts: [],
48
+ sensitive: false,
49
+ spans: [
50
+ {
51
+ id: "s",
52
+ kind: "LLM",
53
+ name: "llm",
54
+ startedAt: DateTime.utc().toISO() ?? "",
55
+ status: "ok",
56
+ },
57
+ ],
58
+ startedAt: DateTime.utc().toISO() ?? "",
59
+ status: "ok",
60
+ });
61
+ traceId = exported.id;
62
+ });
63
+
64
+ it("submits scores, hides the item from pending, and falls back panel keys", async () => {
65
+ const [item] = await reviewStore.enqueue({
66
+ evaluatorId,
67
+ reason: "manual",
68
+ traceIds: [traceId],
69
+ });
70
+ const detail = await reviewStore.getDetail(item.id);
71
+ expect(item.traceName).toBe("gen");
72
+ expect(detail.rawInput).toEqual({raw: "prompt text"});
73
+ expect(detail.rawOutput).toEqual({raw: "model text"});
74
+ expect(detail.panels.given[0]?.key).toBe("raw");
75
+ expect(detail.panels.wrote[0]?.key).toBe("raw");
76
+
77
+ await expect(
78
+ reviewStore.submit({
79
+ id: item.id,
80
+ scores: {},
81
+ sinks: [scoreSink],
82
+ })
83
+ ).rejects.toMatchObject({status: 400, title: 'Score "correct" is required'});
84
+
85
+ await reviewStore.submit({
86
+ comment: "looks good",
87
+ id: item.id,
88
+ scores: {correct: true},
89
+ sinks: [scoreSink],
90
+ });
91
+ expect(scoreSink.scores[0]?.name).toBe("correct");
92
+ expect(scoreSink.scores[0]?.value).toBe(true);
93
+
94
+ const pending = await reviewStore.list("pending");
95
+ expect(pending.data.map((row) => row.id)).not.toContain(item.id);
96
+ expect(pending.counts.done).toBe(1);
97
+
98
+ const other = (
99
+ await reviewStore.enqueue({evaluatorId, reason: "manual", traceIds: [traceId]})
100
+ )[0];
101
+ await reviewStore.skip(other.id);
102
+ const skipped = await reviewStore.list("skipped");
103
+ expect(skipped.data.map((row) => row.id)).toContain(other.id);
104
+
105
+ const assigned = (
106
+ await reviewStore.enqueue({evaluatorId, reason: "manual", traceIds: [traceId]})
107
+ )[0];
108
+ await reviewStore.assign(assigned.id, new mongoose.Types.ObjectId().toString());
109
+ const inProgress = await reviewStore.list("in_progress");
110
+ expect(inProgress.data.map((row) => row.id)).toContain(assigned.id);
111
+ });
112
+
113
+ it("rejects automatic evaluators from the human review queue", async () => {
114
+ const automatic = await new LocalEvaluatorStore().create({
115
+ assertion: {constraint: "exists", path: "output"},
116
+ dimensions: [{dataType: "boolean", key: "valid", required: true}],
117
+ name: "automatic-schema-check",
118
+ target: "full trace",
119
+ type: "json-assert",
120
+ });
121
+
122
+ try {
123
+ await reviewStore.enqueue({
124
+ evaluatorId: automatic.id,
125
+ reason: "manual",
126
+ traceIds: [traceId],
127
+ });
128
+ assert.fail("Expected human-review enqueue to reject an automatic evaluator");
129
+ } catch (error) {
130
+ assert.deepInclude(error as object, {
131
+ status: 400,
132
+ title: "Human review requires a human evaluator",
133
+ });
134
+ }
135
+ });
136
+
137
+ it("skips blank optional dimensions and stores scores without configured sinks", async () => {
138
+ const optionalEvaluator = await new LocalEvaluatorStore().create({
139
+ dimensions: [
140
+ {dataType: "boolean", key: "correct", required: true},
141
+ {dataType: "categorical", key: "failureMode", required: false},
142
+ ],
143
+ name: "optional-human-score",
144
+ target: "full trace",
145
+ type: "human",
146
+ });
147
+ const [item] = await reviewStore.enqueue({
148
+ evaluatorId: optionalEvaluator.id,
149
+ reason: "manual",
150
+ traceIds: [traceId],
151
+ });
152
+
153
+ await reviewStore.submit({
154
+ id: item.id,
155
+ scores: {correct: true},
156
+ sinks: [],
157
+ });
158
+
159
+ const trace = await new LocalTraceStore().getDetail(traceId);
160
+ assert.deepInclude(trace.scores[0], {name: "correct", value: true});
161
+ assert.lengthOf(trace.scores, 1);
162
+ assert.equal((await reviewStore.getDetail(item.id)).status, "done");
163
+ });
164
+ });
@@ -0,0 +1,264 @@
1
+ import {APIError} from "@terreno/api";
2
+ import {DateTime} from "luxon";
3
+ import mongoose from "mongoose";
4
+
5
+ import type {EvaluatorDimension} from "../../types/observability";
6
+ import {buildReviewPanels, type ReviewPanels} from "../reviewPanels";
7
+ import type {ScoreRecord, ScoreSink} from "../types";
8
+ import {registerObsEvaluator} from "./models/obsEvaluator";
9
+ import {registerObsPrompt} from "./models/obsPrompt";
10
+ import {registerObsPromptVersion} from "./models/obsPromptVersion";
11
+ import {registerObsReviewItem} from "./models/obsReviewItem";
12
+ import {registerObsTrace} from "./models/obsTrace";
13
+ import {LocalTraceStore} from "./traceStore";
14
+
15
+ export type ReviewStatus = "done" | "in_progress" | "pending" | "skipped";
16
+
17
+ export interface ReviewListItem {
18
+ assigneeId?: string;
19
+ enqueuedAt: string;
20
+ evaluatorId: string;
21
+ id: string;
22
+ promptName?: string;
23
+ reason: string;
24
+ status: ReviewStatus;
25
+ traceId: string;
26
+ traceName: string;
27
+ }
28
+
29
+ const objectId = (value: string, label: string): mongoose.Types.ObjectId => {
30
+ if (!mongoose.Types.ObjectId.isValid(value)) {
31
+ throw new APIError({status: 400, title: `${label} must be an object id`});
32
+ }
33
+ return new mongoose.Types.ObjectId(value);
34
+ };
35
+
36
+ const toIso = (value: Date): string => {
37
+ return DateTime.fromJSDate(value).toUTC().toISO() ?? "";
38
+ };
39
+
40
+ export class LocalReviewStore {
41
+ async enqueue(params: {
42
+ evaluatorId: string;
43
+ reason?: "dataset_candidate" | "eval" | "feedback" | "manual";
44
+ spanId?: string;
45
+ traceIds: string[];
46
+ }): Promise<ReviewListItem[]> {
47
+ if (params.traceIds.length === 0) {
48
+ throw new APIError({status: 400, title: "traceIds are required"});
49
+ }
50
+ const evaluator = await registerObsEvaluator().findOneOrNone({
51
+ _id: objectId(params.evaluatorId, "evaluatorId"),
52
+ });
53
+ if (!evaluator) {
54
+ throw new APIError({status: 404, title: "Unknown evaluator"});
55
+ }
56
+ if (evaluator.type !== "human") {
57
+ throw new APIError({
58
+ status: 400,
59
+ title: "Human review requires a human evaluator",
60
+ });
61
+ }
62
+ const created: ReviewListItem[] = [];
63
+ for (const traceId of params.traceIds) {
64
+ const trace = await registerObsTrace().findOneOrNone({_id: objectId(traceId, "traceId")});
65
+ if (!trace) {
66
+ throw new APIError({status: 404, title: `Unknown trace "${traceId}"`});
67
+ }
68
+ const item = await registerObsReviewItem().create({
69
+ enqueuedAt: DateTime.utc().toJSDate(),
70
+ evaluatorId: evaluator._id,
71
+ reason: params.reason ?? "manual",
72
+ status: "pending",
73
+ traceId: trace._id,
74
+ ...(params.spanId ? {spanId: objectId(params.spanId, "spanId")} : {}),
75
+ });
76
+ created.push(this.toListItem(item, trace.name, trace.prompts[0]?.name));
77
+ }
78
+ return created;
79
+ }
80
+
81
+ async list(status?: ReviewStatus): Promise<{
82
+ counts: Record<ReviewStatus, number>;
83
+ data: ReviewListItem[];
84
+ }> {
85
+ const ObsReviewItem = registerObsReviewItem();
86
+ const counts = {
87
+ done: await ObsReviewItem.countDocuments({status: "done"}),
88
+ in_progress: await ObsReviewItem.countDocuments({status: "in_progress"}),
89
+ pending: await ObsReviewItem.countDocuments({status: "pending"}),
90
+ skipped: await ObsReviewItem.countDocuments({status: "skipped"}),
91
+ };
92
+ const filter = status ? {status} : {};
93
+ const rows = await ObsReviewItem.find(filter).sort({enqueuedAt: 1});
94
+ const data: ReviewListItem[] = [];
95
+ for (const row of rows) {
96
+ const trace = await registerObsTrace().findOneOrNone({_id: row.traceId});
97
+ data.push(this.toListItem(row, trace?.name ?? "Unknown trace", trace?.prompts[0]?.name));
98
+ }
99
+ return {counts, data};
100
+ }
101
+
102
+ async getDetail(id: string): Promise<{
103
+ comment?: string;
104
+ dimensions: EvaluatorDimension[];
105
+ evaluatorId: string;
106
+ id: string;
107
+ instructions?: string;
108
+ panels: ReviewPanels;
109
+ rawInput?: unknown;
110
+ rawOutput?: unknown;
111
+ scores?: Record<string, boolean | number | string>;
112
+ status: ReviewStatus;
113
+ traceId: string;
114
+ }> {
115
+ const item = await registerObsReviewItem().findOneOrNone({_id: objectId(id, "id")});
116
+ if (!item) {
117
+ throw new APIError({status: 404, title: `Unknown review item "${id}"`});
118
+ }
119
+ const evaluator = await registerObsEvaluator().findOneOrNone({_id: item.evaluatorId});
120
+ if (!evaluator) {
121
+ throw new APIError({status: 404, title: "Unknown evaluator"});
122
+ }
123
+ const trace = await registerObsTrace().findOneOrNone({_id: item.traceId});
124
+ const promptRef = trace?.prompts[0];
125
+ let variables:
126
+ | {key: string; label?: string; required: boolean; reviewerNote?: string}[]
127
+ | undefined;
128
+ let outputFieldNotes: Record<string, string> | undefined;
129
+ let outputSchema: Record<string, unknown> | undefined;
130
+ if (promptRef) {
131
+ const prompt = await registerObsPrompt().findOneOrNone({name: promptRef.name});
132
+ if (prompt) {
133
+ const version = await registerObsPromptVersion().findOneOrNone({
134
+ promptId: prompt._id,
135
+ version: promptRef.version,
136
+ });
137
+ variables = version?.variables;
138
+ outputFieldNotes = version?.outputFieldNotes;
139
+ outputSchema = version?.outputSchema;
140
+ }
141
+ }
142
+ return {
143
+ comment: item.comment,
144
+ dimensions: evaluator.dimensions,
145
+ evaluatorId: String(evaluator._id),
146
+ id: String(item._id),
147
+ instructions: evaluator.instructions,
148
+ panels: buildReviewPanels({
149
+ input: trace?.input,
150
+ output: trace?.output,
151
+ outputFieldNotes,
152
+ outputSchema,
153
+ variables,
154
+ }),
155
+ rawInput: trace?.input,
156
+ rawOutput: trace?.output,
157
+ scores: item.scores,
158
+ status: item.status,
159
+ traceId: String(item.traceId),
160
+ };
161
+ }
162
+
163
+ async submit(params: {
164
+ comment?: string;
165
+ id: string;
166
+ scores: Record<string, boolean | number | string>;
167
+ sinks: ScoreSink[];
168
+ }): Promise<void> {
169
+ const item = await this.requireItem(params.id);
170
+ const evaluator = await registerObsEvaluator().findOneOrNone({_id: item.evaluatorId});
171
+ if (!evaluator) {
172
+ throw new APIError({status: 404, title: "Unknown evaluator"});
173
+ }
174
+ const missing = evaluator.dimensions.find((dimension) => {
175
+ return dimension.required && params.scores[dimension.key] === undefined;
176
+ });
177
+ if (missing) {
178
+ throw new APIError({status: 400, title: `Score "${missing.key}" is required`});
179
+ }
180
+ const records: ScoreRecord[] = [];
181
+ for (const dimension of evaluator.dimensions) {
182
+ const value = params.scores[dimension.key];
183
+ if (value === undefined) {
184
+ continue;
185
+ }
186
+ records.push({
187
+ comment: params.comment,
188
+ dataType: dimension.dataType,
189
+ evaluatorId: String(evaluator._id),
190
+ name: dimension.key,
191
+ source: "human" as const,
192
+ traceId: String(item.traceId),
193
+ value,
194
+ });
195
+ }
196
+ for (const record of records) {
197
+ if (params.sinks.length === 0) {
198
+ await new LocalTraceStore().exportScore(record);
199
+ continue;
200
+ }
201
+ const results = await Promise.allSettled(
202
+ params.sinks.map((sink) => {
203
+ return sink.export(record);
204
+ })
205
+ );
206
+ for (const result of results) {
207
+ if (result.status === "rejected") {
208
+ throw result.reason;
209
+ }
210
+ }
211
+ }
212
+ item.comment = params.comment;
213
+ item.scores = params.scores;
214
+ item.status = "done";
215
+ await item.save();
216
+ }
217
+
218
+ async skip(id: string): Promise<void> {
219
+ const item = await this.requireItem(id);
220
+ item.status = "skipped";
221
+ await item.save();
222
+ }
223
+
224
+ async assign(id: string, assigneeId: string): Promise<void> {
225
+ const item = await this.requireItem(id);
226
+ item.assigneeId = objectId(assigneeId, "assigneeId");
227
+ item.status = "in_progress";
228
+ await item.save();
229
+ }
230
+
231
+ private async requireItem(id: string) {
232
+ const item = await registerObsReviewItem().findOneOrNone({_id: objectId(id, "id")});
233
+ if (!item) {
234
+ throw new APIError({status: 404, title: `Unknown review item "${id}"`});
235
+ }
236
+ return item;
237
+ }
238
+
239
+ private toListItem(
240
+ row: {
241
+ _id: mongoose.Types.ObjectId;
242
+ assigneeId?: mongoose.Types.ObjectId;
243
+ enqueuedAt: Date;
244
+ evaluatorId: mongoose.Types.ObjectId;
245
+ reason: string;
246
+ status: ReviewStatus;
247
+ traceId: mongoose.Types.ObjectId;
248
+ },
249
+ traceName: string,
250
+ promptName?: string
251
+ ): ReviewListItem {
252
+ return {
253
+ assigneeId: row.assigneeId ? String(row.assigneeId) : undefined,
254
+ enqueuedAt: toIso(row.enqueuedAt),
255
+ evaluatorId: String(row.evaluatorId),
256
+ id: String(row._id),
257
+ promptName,
258
+ reason: row.reason,
259
+ status: row.status,
260
+ traceId: String(row.traceId),
261
+ traceName,
262
+ };
263
+ }
264
+ }
@@ -0,0 +1,264 @@
1
+ import {afterEach, beforeEach, describe, expect, it} from "bun:test";
2
+ import {DateTime} from "luxon";
3
+ import mongoose from "mongoose";
4
+
5
+ import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
6
+ import type {ScoreRecord, TraceRecord} from "../types";
7
+ import {createLocalObservabilityPlugin} from "./localPlugin";
8
+ import {registerObsScore} from "./models/obsScore";
9
+ import {registerObsSpan} from "./models/obsSpan";
10
+ import {registerObsTrace} from "./models/obsTrace";
11
+ import {LocalTraceSink, LocalTraceStore, MemoryScoreSink, MemoryTraceSink} from "./traceStore";
12
+
13
+ const iso = (msAgo: number): string => {
14
+ return DateTime.utc().minus({milliseconds: msAgo}).toISO() ?? "";
15
+ };
16
+
17
+ const baseTrace = (overrides: Partial<TraceRecord> = {}): TraceRecord => {
18
+ return {
19
+ endedAt: iso(0),
20
+ id: "trace-root",
21
+ name: "generate",
22
+ prompts: [{name: "summarize", version: 1}],
23
+ sensitive: false,
24
+ spans: [
25
+ {
26
+ durationMs: 12,
27
+ endedAt: iso(0),
28
+ id: "span-root",
29
+ kind: "LLM",
30
+ name: "generate",
31
+ startedAt: iso(12),
32
+ status: "ok",
33
+ usage: {inputTokens: 5, model: "mock-model", outputTokens: 10},
34
+ },
35
+ ],
36
+ startedAt: iso(12),
37
+ status: "ok",
38
+ ...overrides,
39
+ };
40
+ };
41
+
42
+ describe("LocalTraceStore", () => {
43
+ let store: LocalTraceStore;
44
+
45
+ afterEach(() => {
46
+ resetObservabilityApp();
47
+ });
48
+
49
+ beforeEach(async () => {
50
+ new ObservabilityApp({plugins: [createLocalObservabilityPlugin()]});
51
+ store = new LocalTraceStore();
52
+ await registerObsTrace().deleteMany({});
53
+ await registerObsSpan().deleteMany({});
54
+ await registerObsScore().deleteMany({});
55
+ });
56
+
57
+ it("round-trips nested spans and derives errorSummary from the first error span", async () => {
58
+ const exported = await store.exportTrace(
59
+ baseTrace({
60
+ prompts: [
61
+ {label: "production", name: "summarize", version: 1},
62
+ {name: "title", version: 2},
63
+ ],
64
+ spans: [
65
+ {
66
+ durationMs: 40,
67
+ id: "parent",
68
+ kind: "CHAIN",
69
+ name: "agent",
70
+ startedAt: iso(40),
71
+ status: "ok",
72
+ },
73
+ {
74
+ durationMs: 10,
75
+ error: "tool timed out",
76
+ id: "child",
77
+ kind: "TOOL",
78
+ name: "search",
79
+ parentSpanId: "parent",
80
+ startedAt: iso(20),
81
+ startOffsetMs: 8,
82
+ status: "error",
83
+ },
84
+ ],
85
+ status: "error",
86
+ })
87
+ );
88
+
89
+ const detail = await store.getDetail(exported.id);
90
+ expect(detail.prompts).toHaveLength(2);
91
+ expect(detail.errorSummary).toBe("tool timed out");
92
+ expect(detail.spans).toEqual([
93
+ expect.objectContaining({
94
+ children: [
95
+ expect.objectContaining({
96
+ error: "tool timed out",
97
+ kind: "TOOL",
98
+ name: "search",
99
+ startOffsetMs: 8,
100
+ status: "error",
101
+ }),
102
+ ],
103
+ kind: "CHAIN",
104
+ name: "agent",
105
+ }),
106
+ ]);
107
+ });
108
+
109
+ it("omits costUsd when the exported usage has no price", async () => {
110
+ const exported = await store.exportTrace(
111
+ baseTrace({
112
+ usage: {inputTokens: 5, model: "unpriced", outputTokens: 10},
113
+ })
114
+ );
115
+ const detail = await store.getDetail(exported.id);
116
+ expect(detail.usage).toEqual({inputTokens: 5, model: "unpriced", outputTokens: 10});
117
+ expect("costUsd" in (detail.usage ?? {})).toBe(false);
118
+ });
119
+
120
+ it("applies list filters, pagination, and hasScore", async () => {
121
+ const userId = new mongoose.Types.ObjectId();
122
+ const older = await store.exportTrace(
123
+ baseTrace({
124
+ flaggedForDataset: true,
125
+ name: "old",
126
+ sessionId: "sess-a",
127
+ startedAt: iso(86_400_000),
128
+ userId: userId.toString(),
129
+ })
130
+ );
131
+ const newer = await store.exportTrace(
132
+ baseTrace({
133
+ name: "new",
134
+ prompts: [{name: "other", version: 1}],
135
+ sensitive: true,
136
+ sessionId: "sess-b",
137
+ startedAt: iso(1_000),
138
+ status: "error",
139
+ userId: new mongoose.Types.ObjectId().toString(),
140
+ })
141
+ );
142
+ await store.exportScore({
143
+ dataType: "boolean",
144
+ name: "correct",
145
+ source: "human",
146
+ traceId: older.id,
147
+ value: true,
148
+ });
149
+
150
+ const byPrompt = await store.list({prompt: "summarize"});
151
+ expect(byPrompt.data.map((row) => row.id)).toEqual([older.id]);
152
+
153
+ const byStatus = await store.list({status: "error"});
154
+ expect(byStatus.data.map((row) => row.id)).toEqual([newer.id]);
155
+
156
+ const byUser = await store.list({userId: userId.toString()});
157
+ expect(byUser.data.map((row) => row.id)).toEqual([older.id]);
158
+
159
+ const bySession = await store.list({sessionId: "sess-b"});
160
+ expect(bySession.data.map((row) => row.id)).toEqual([newer.id]);
161
+
162
+ const bySensitive = await store.list({sensitive: true});
163
+ expect(bySensitive.data.map((row) => row.id)).toEqual([newer.id]);
164
+
165
+ const byFlag = await store.list({flaggedForDataset: true});
166
+ expect(byFlag.data.map((row) => row.id)).toEqual([older.id]);
167
+
168
+ const scored = await store.list({hasScore: true});
169
+ expect(scored.data.map((row) => row.id)).toEqual([older.id]);
170
+ expect(scored.data[0]?.spanCount).toBe(1);
171
+ expect(scored.data[0]?.scoreCount).toBe(1);
172
+
173
+ const unscored = await store.list({hasScore: false});
174
+ expect(unscored.data.map((row) => row.id)).toEqual([newer.id]);
175
+
176
+ const since = DateTime.utc().minus({hours: 1}).toISO() ?? "";
177
+ const recent = await store.list({from: since});
178
+ expect(recent.data.map((row) => row.id)).toEqual([newer.id]);
179
+
180
+ const until = DateTime.utc().minus({hours: 12}).toISO() ?? "";
181
+ const olderWindow = await store.list({to: until});
182
+ expect(olderWindow.data.map((row) => row.id)).toEqual([older.id]);
183
+
184
+ const page = await store.list({limit: 1, page: 2});
185
+ expect(page.meta).toEqual({limit: 1, page: 2, total: 2});
186
+ expect(page.data).toHaveLength(1);
187
+
188
+ const byPromptVersion = await store.list({prompt: "summarize", promptVersion: 2});
189
+ expect(byPromptVersion.data).toHaveLength(0);
190
+ await store.exportTrace(
191
+ baseTrace({
192
+ name: "summarize-v2",
193
+ prompts: [{name: "summarize", version: 2}],
194
+ startedAt: iso(500),
195
+ })
196
+ );
197
+ const v2Only = await store.list({prompt: "summarize", promptVersion: 2});
198
+ expect(v2Only.data.map((row) => row.name)).toEqual(["summarize-v2"]);
199
+ const v1Only = await store.list({prompt: "summarize", promptVersion: 1});
200
+ expect(v1Only.data.map((row) => row.id)).toEqual([older.id]);
201
+ });
202
+
203
+ it("returns score metadata on trace detail and exports through the sink wrapper", async () => {
204
+ const exported = await store.exportTrace(baseTrace());
205
+ await store.exportScore({
206
+ comment: "looks good",
207
+ confidence: 0.9,
208
+ dataType: "numeric",
209
+ evaluatorId: new mongoose.Types.ObjectId().toString(),
210
+ name: "quality",
211
+ source: "human",
212
+ spanId: "span-root",
213
+ traceId: exported.id,
214
+ value: 5,
215
+ });
216
+
217
+ const detail = await store.getDetail(exported.id);
218
+ expect(detail.scores).toEqual([
219
+ expect.objectContaining({
220
+ comment: "looks good",
221
+ confidence: 0.9,
222
+ dataType: "numeric",
223
+ name: "quality",
224
+ source: "human",
225
+ traceId: exported.id,
226
+ value: 5,
227
+ }),
228
+ ]);
229
+
230
+ const sink = new LocalTraceSink(store);
231
+ await sink.export(baseTrace({name: "sink"}));
232
+ expect(sink.store).toBe(store);
233
+ const listed = await store.list({limit: 10});
234
+ const sinkRow = listed.data.find((row) => row.name === "sink");
235
+ expect(sinkRow).toBeDefined();
236
+ const sinkDetail = await store.getDetail(sinkRow!.id);
237
+ expect(sinkDetail.name).toBe("sink");
238
+ });
239
+
240
+ it("throws when loading an unknown trace", async () => {
241
+ await expect(store.getDetail(new mongoose.Types.ObjectId().toString())).rejects.toThrow(
242
+ /Unknown trace/
243
+ );
244
+ });
245
+
246
+ it("buffers exported traces and scores in memory sinks", async () => {
247
+ const trace = baseTrace({id: "memory-trace"});
248
+ const score: ScoreRecord = {
249
+ dataType: "boolean",
250
+ name: "correct",
251
+ source: "human",
252
+ traceId: "memory-trace",
253
+ value: true,
254
+ };
255
+ const traceSink = new MemoryTraceSink();
256
+ const scoreSink = new MemoryScoreSink();
257
+
258
+ await traceSink.export(trace);
259
+ await scoreSink.export(score);
260
+
261
+ expect(traceSink.traces).toEqual([trace]);
262
+ expect(scoreSink.scores).toEqual([score]);
263
+ });
264
+ });