@terreno/ai 57.9.0 → 57.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (412) hide show
  1. package/README.md +1 -0
  2. package/bunfig.toml +1 -1
  3. package/dist/aiApp.d.ts +8 -2
  4. package/dist/aiApp.js +12 -6
  5. package/dist/harness/adminScreens.d.ts +6 -0
  6. package/dist/harness/adminScreens.js +11 -0
  7. package/dist/harness/agentLoop.d.ts +61 -0
  8. package/dist/harness/agentLoop.js +798 -0
  9. package/dist/harness/agentTaskNames.d.ts +4 -0
  10. package/dist/harness/agentTaskNames.js +7 -0
  11. package/dist/harness/approvalPolicy.d.ts +3 -0
  12. package/dist/harness/approvalPolicy.js +23 -0
  13. package/dist/harness/approvals.d.ts +101 -0
  14. package/dist/harness/approvals.js +468 -0
  15. package/dist/harness/commit.d.ts +238 -0
  16. package/dist/harness/commit.js +678 -0
  17. package/dist/harness/conversation.d.ts +75 -0
  18. package/dist/harness/conversation.js +397 -0
  19. package/dist/harness/defineAgent.d.ts +9 -0
  20. package/dist/harness/defineAgent.js +65 -0
  21. package/dist/harness/defineTask.d.ts +11 -0
  22. package/dist/harness/defineTask.js +59 -0
  23. package/dist/harness/defineTool.d.ts +7 -0
  24. package/dist/harness/defineTool.js +36 -0
  25. package/dist/harness/definitionError.d.ts +8 -0
  26. package/dist/harness/definitionError.js +15 -0
  27. package/dist/harness/errors.d.ts +127 -0
  28. package/dist/harness/errors.js +101 -0
  29. package/dist/harness/eventHub.d.ts +45 -0
  30. package/dist/harness/eventHub.js +135 -0
  31. package/dist/harness/events.d.ts +95 -0
  32. package/dist/harness/events.js +267 -0
  33. package/dist/harness/executionEnv.d.ts +31 -0
  34. package/dist/harness/executionEnv.js +2 -0
  35. package/dist/harness/extensions.d.ts +82 -0
  36. package/dist/harness/extensions.js +274 -0
  37. package/dist/harness/harness.d.ts +163 -0
  38. package/dist/harness/harness.js +520 -0
  39. package/dist/harness/harnessApp.d.ts +44 -0
  40. package/dist/harness/harnessApp.js +62 -0
  41. package/dist/harness/internalRuntime.d.ts +16 -0
  42. package/dist/harness/internalRuntime.js +18 -0
  43. package/dist/harness/leases.d.ts +35 -0
  44. package/dist/harness/leases.js +86 -0
  45. package/dist/harness/memo.d.ts +24 -0
  46. package/dist/harness/memo.js +78 -0
  47. package/dist/harness/modelCall.d.ts +45 -0
  48. package/dist/harness/modelCall.js +147 -0
  49. package/dist/harness/models/harnessApproval.d.ts +2 -0
  50. package/dist/harness/models/harnessApproval.js +100 -0
  51. package/dist/harness/models/harnessConversation.d.ts +2 -0
  52. package/dist/harness/models/harnessConversation.js +117 -0
  53. package/dist/harness/models/harnessEvent.d.ts +3 -0
  54. package/dist/harness/models/harnessEvent.js +73 -0
  55. package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
  56. package/dist/harness/models/harnessInboxEvent.js +58 -0
  57. package/dist/harness/models/harnessMemo.d.ts +2 -0
  58. package/dist/harness/models/harnessMemo.js +42 -0
  59. package/dist/harness/models/harnessMessage.d.ts +2 -0
  60. package/dist/harness/models/harnessMessage.js +73 -0
  61. package/dist/harness/models/harnessOwner.d.ts +2 -0
  62. package/dist/harness/models/harnessOwner.js +36 -0
  63. package/dist/harness/models/harnessTask.d.ts +2 -0
  64. package/dist/harness/models/harnessTask.js +186 -0
  65. package/dist/harness/ownership.d.ts +89 -0
  66. package/dist/harness/ownership.js +438 -0
  67. package/dist/harness/registry.d.ts +29 -0
  68. package/dist/harness/registry.js +93 -0
  69. package/dist/harness/retryBackoff.d.ts +31 -0
  70. package/dist/harness/retryBackoff.js +50 -0
  71. package/dist/harness/routes/approvals.d.ts +13 -0
  72. package/dist/harness/routes/approvals.js +93 -0
  73. package/dist/harness/routes/conversations.d.ts +11 -0
  74. package/dist/harness/routes/conversations.js +66 -0
  75. package/dist/harness/routes/events.d.ts +19 -0
  76. package/dist/harness/routes/events.js +215 -0
  77. package/dist/harness/routes/tasks.d.ts +11 -0
  78. package/dist/harness/routes/tasks.js +108 -0
  79. package/dist/harness/runners/inProcessRunner.d.ts +57 -0
  80. package/dist/harness/runners/inProcessRunner.js +196 -0
  81. package/dist/harness/runtime.d.ts +34 -0
  82. package/dist/harness/runtime.js +551 -0
  83. package/dist/harness/subagent.d.ts +53 -0
  84. package/dist/harness/subagent.js +134 -0
  85. package/dist/harness/suspend.d.ts +8 -0
  86. package/dist/harness/suspend.js +16 -0
  87. package/dist/harness/transaction.d.ts +3 -0
  88. package/dist/harness/transaction.js +22 -0
  89. package/dist/harness/waits.d.ts +100 -0
  90. package/dist/harness/waits.js +214 -0
  91. package/dist/index.d.ts +19 -1
  92. package/dist/index.js +45 -2
  93. package/dist/models/aiDataset.d.ts +2 -0
  94. package/dist/models/aiDataset.js +67 -0
  95. package/dist/models/aiRequest.js +1 -1
  96. package/dist/models/gptHistory.js +73 -0
  97. package/dist/observability/adminScreens.d.ts +5 -0
  98. package/dist/observability/adminScreens.js +32 -0
  99. package/dist/observability/compileTemplate.d.ts +1 -0
  100. package/dist/observability/compileTemplate.js +13 -0
  101. package/dist/observability/datasetImport.d.ts +11 -0
  102. package/dist/observability/datasetImport.js +185 -0
  103. package/dist/observability/evaluate.d.ts +31 -0
  104. package/dist/observability/evaluate.js +200 -0
  105. package/dist/observability/evaluatorTemplates.d.ts +17 -0
  106. package/dist/observability/evaluatorTemplates.js +111 -0
  107. package/dist/observability/jsonPath.d.ts +2 -0
  108. package/dist/observability/jsonPath.js +57 -0
  109. package/dist/observability/local/datasetStore.d.ts +87 -0
  110. package/dist/observability/local/datasetStore.js +326 -0
  111. package/dist/observability/local/evaluatorStore.d.ts +44 -0
  112. package/dist/observability/local/evaluatorStore.js +182 -0
  113. package/dist/observability/local/experimentRunner.d.ts +96 -0
  114. package/dist/observability/local/experimentRunner.js +419 -0
  115. package/dist/observability/local/localPlugin.d.ts +22 -0
  116. package/dist/observability/local/localPlugin.js +76 -0
  117. package/dist/observability/local/models/obsDataset.d.ts +2 -0
  118. package/dist/observability/local/models/obsDataset.js +37 -0
  119. package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
  120. package/dist/observability/local/models/obsDatasetItem.js +76 -0
  121. package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
  122. package/dist/observability/local/models/obsEvaluator.js +98 -0
  123. package/dist/observability/local/models/obsExperiment.d.ts +2 -0
  124. package/dist/observability/local/models/obsExperiment.js +87 -0
  125. package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
  126. package/dist/observability/local/models/obsExperimentItem.js +45 -0
  127. package/dist/observability/local/models/obsPrompt.d.ts +2 -0
  128. package/dist/observability/local/models/obsPrompt.js +42 -0
  129. package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
  130. package/dist/observability/local/models/obsPromptLabel.js +39 -0
  131. package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
  132. package/dist/observability/local/models/obsPromptVersion.js +96 -0
  133. package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
  134. package/dist/observability/local/models/obsReviewItem.js +75 -0
  135. package/dist/observability/local/models/obsScore.d.ts +2 -0
  136. package/dist/observability/local/models/obsScore.js +58 -0
  137. package/dist/observability/local/models/obsSpan.d.ts +2 -0
  138. package/dist/observability/local/models/obsSpan.js +64 -0
  139. package/dist/observability/local/models/obsTrace.d.ts +2 -0
  140. package/dist/observability/local/models/obsTrace.js +93 -0
  141. package/dist/observability/local/promptStore.d.ts +180 -0
  142. package/dist/observability/local/promptStore.js +444 -0
  143. package/dist/observability/local/reviewStore.d.ts +50 -0
  144. package/dist/observability/local/reviewStore.js +203 -0
  145. package/dist/observability/local/traceStore.d.ts +103 -0
  146. package/dist/observability/local/traceStore.js +325 -0
  147. package/dist/observability/observabilityApp.d.ts +24 -0
  148. package/dist/observability/observabilityApp.js +157 -0
  149. package/dist/observability/observabilityAppRegistry.d.ts +14 -0
  150. package/dist/observability/observabilityAppRegistry.js +16 -0
  151. package/dist/observability/permissions.d.ts +7 -0
  152. package/dist/observability/permissions.js +60 -0
  153. package/dist/observability/reviewPanels.d.ts +18 -0
  154. package/dist/observability/reviewPanels.js +46 -0
  155. package/dist/observability/routes/datasets.d.ts +7 -0
  156. package/dist/observability/routes/datasets.js +149 -0
  157. package/dist/observability/routes/evaluators.d.ts +7 -0
  158. package/dist/observability/routes/evaluators.js +123 -0
  159. package/dist/observability/routes/experiments.d.ts +7 -0
  160. package/dist/observability/routes/experiments.js +95 -0
  161. package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
  162. package/dist/observability/routes/observabilityRouteAccess.js +67 -0
  163. package/dist/observability/routes/prompts.d.ts +12 -0
  164. package/dist/observability/routes/prompts.js +172 -0
  165. package/dist/observability/routes/requestAiService.d.ts +13 -0
  166. package/dist/observability/routes/requestAiService.js +21 -0
  167. package/dist/observability/routes/review.d.ts +7 -0
  168. package/dist/observability/routes/review.js +79 -0
  169. package/dist/observability/routes/status.d.ts +5 -0
  170. package/dist/observability/routes/status.js +31 -0
  171. package/dist/observability/routes/testMultiStage.d.ts +9 -0
  172. package/dist/observability/routes/testMultiStage.js +67 -0
  173. package/dist/observability/routes/traces.d.ts +7 -0
  174. package/dist/observability/routes/traces.js +129 -0
  175. package/dist/observability/schemaValidation.d.ts +5 -0
  176. package/dist/observability/schemaValidation.js +26 -0
  177. package/dist/observability/sopThresholds.d.ts +2 -0
  178. package/dist/observability/sopThresholds.js +21 -0
  179. package/dist/observability/status.d.ts +23 -0
  180. package/dist/observability/status.js +34 -0
  181. package/dist/observability/testMultiStageSchemas.d.ts +10 -0
  182. package/dist/observability/testMultiStageSchemas.js +79 -0
  183. package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
  184. package/dist/observability/testMultiStageWorkflow.js +218 -0
  185. package/dist/observability/types.d.ts +159 -0
  186. package/dist/observability/types.js +67 -0
  187. package/dist/routes/files.js +18 -7
  188. package/dist/routes/gpt.js +79 -689
  189. package/dist/routes/gptActions.d.ts +9 -0
  190. package/dist/routes/gptActions.js +212 -0
  191. package/dist/routes/gptDatasets.d.ts +5 -0
  192. package/dist/routes/gptDatasets.js +94 -0
  193. package/dist/routes/gptHistories.d.ts +3 -1
  194. package/dist/routes/gptHistories.js +188 -2
  195. package/dist/service/aiDatasets.d.ts +57 -0
  196. package/dist/service/aiDatasets.js +253 -0
  197. package/dist/service/aiService.d.ts +50 -1
  198. package/dist/service/aiService.js +426 -77
  199. package/dist/service/askFiles.d.ts +49 -0
  200. package/dist/service/askFiles.js +162 -0
  201. package/dist/service/asks.d.ts +76 -0
  202. package/dist/service/asks.js +284 -0
  203. package/dist/service/chatTurn.d.ts +128 -0
  204. package/dist/service/chatTurn.js +1674 -0
  205. package/dist/service/fileStorage.d.ts +4 -0
  206. package/dist/service/fileStorage.js +7 -2
  207. package/dist/service/fillActionIds.d.ts +6 -0
  208. package/dist/service/fillActionIds.js +79 -0
  209. package/dist/service/jsonFenceModel.d.ts +7 -0
  210. package/dist/service/jsonFenceModel.js +45 -0
  211. package/dist/service/prompts.d.ts +52 -0
  212. package/dist/service/prompts.js +96 -1
  213. package/dist/service/sanitizeHtml.d.ts +11 -0
  214. package/dist/service/sanitizeHtml.js +108 -0
  215. package/dist/types/harness.d.ts +934 -0
  216. package/dist/types/harness.js +133 -0
  217. package/dist/types/index.d.ts +192 -14
  218. package/dist/types/index.js +2 -0
  219. package/dist/types/observability.d.ts +288 -0
  220. package/dist/types/observability.js +2 -0
  221. package/package.json +17 -5
  222. package/src/aiApp.test.ts +207 -1
  223. package/src/aiApp.ts +19 -6
  224. package/src/harness/adminScreens.test.ts +34 -0
  225. package/src/harness/adminScreens.ts +11 -0
  226. package/src/harness/agentLoop.test.ts +1034 -0
  227. package/src/harness/agentLoop.ts +1082 -0
  228. package/src/harness/agentRecovery.test.ts +380 -0
  229. package/src/harness/agentTaskNames.ts +4 -0
  230. package/src/harness/approvalPolicy.ts +28 -0
  231. package/src/harness/approvals.test.ts +1094 -0
  232. package/src/harness/approvals.ts +644 -0
  233. package/src/harness/commit.ts +1049 -0
  234. package/src/harness/conversation.ts +503 -0
  235. package/src/harness/defineAgent.test.ts +92 -0
  236. package/src/harness/defineAgent.ts +82 -0
  237. package/src/harness/defineTask.test.ts +59 -0
  238. package/src/harness/defineTask.ts +76 -0
  239. package/src/harness/defineTool.ts +45 -0
  240. package/src/harness/definitionError.ts +13 -0
  241. package/src/harness/errors.test.ts +90 -0
  242. package/src/harness/errors.ts +119 -0
  243. package/src/harness/eventHub.test.ts +124 -0
  244. package/src/harness/eventHub.ts +159 -0
  245. package/src/harness/events.test.ts +757 -0
  246. package/src/harness/events.ts +384 -0
  247. package/src/harness/executionEnv.ts +34 -0
  248. package/src/harness/extensions.test.ts +1218 -0
  249. package/src/harness/extensions.ts +382 -0
  250. package/src/harness/harness.test.ts +693 -0
  251. package/src/harness/harness.ts +767 -0
  252. package/src/harness/harnessApp.ts +76 -0
  253. package/src/harness/internalRuntime.ts +44 -0
  254. package/src/harness/leases.ts +132 -0
  255. package/src/harness/memo.ts +107 -0
  256. package/src/harness/modelCall.test.ts +134 -0
  257. package/src/harness/modelCall.ts +200 -0
  258. package/src/harness/models/harnessApproval.ts +109 -0
  259. package/src/harness/models/harnessConversation.ts +142 -0
  260. package/src/harness/models/harnessEvent.ts +92 -0
  261. package/src/harness/models/harnessInboxEvent.ts +73 -0
  262. package/src/harness/models/harnessMemo.ts +43 -0
  263. package/src/harness/models/harnessMessage.ts +85 -0
  264. package/src/harness/models/harnessModels.test.ts +60 -0
  265. package/src/harness/models/harnessOwner.ts +40 -0
  266. package/src/harness/models/harnessTask.ts +201 -0
  267. package/src/harness/ownership.test.ts +1087 -0
  268. package/src/harness/ownership.ts +618 -0
  269. package/src/harness/recovery.test.ts +912 -0
  270. package/src/harness/registry.ts +124 -0
  271. package/src/harness/retries.test.ts +292 -0
  272. package/src/harness/retryBackoff.test.ts +100 -0
  273. package/src/harness/retryBackoff.ts +77 -0
  274. package/src/harness/routes/approvals.ts +123 -0
  275. package/src/harness/routes/conversations.test.ts +518 -0
  276. package/src/harness/routes/conversations.ts +102 -0
  277. package/src/harness/routes/events.ts +280 -0
  278. package/src/harness/routes/tasks.test.ts +198 -0
  279. package/src/harness/routes/tasks.ts +146 -0
  280. package/src/harness/runners/inProcessRunner.test.ts +402 -0
  281. package/src/harness/runners/inProcessRunner.ts +220 -0
  282. package/src/harness/runtime.ts +714 -0
  283. package/src/harness/subagent.test.ts +696 -0
  284. package/src/harness/subagent.ts +192 -0
  285. package/src/harness/suspend.ts +13 -0
  286. package/src/harness/transaction.ts +17 -0
  287. package/src/harness/versioning.test.ts +288 -0
  288. package/src/harness/waits.test.ts +693 -0
  289. package/src/harness/waits.ts +356 -0
  290. package/src/index.test.ts +13 -0
  291. package/src/index.ts +70 -0
  292. package/src/isolated/adminScreensConfig.isolated.ts +143 -0
  293. package/src/models/aiDataset.test.ts +34 -0
  294. package/src/models/aiDataset.ts +78 -0
  295. package/src/models/aiRequest.ts +2 -1
  296. package/src/models/gptHistory.test.ts +156 -0
  297. package/src/models/gptHistory.ts +89 -1
  298. package/src/observability/adminScreens.test.ts +71 -0
  299. package/src/observability/adminScreens.ts +42 -0
  300. package/src/observability/compileTemplate.test.ts +14 -0
  301. package/src/observability/compileTemplate.ts +10 -0
  302. package/src/observability/datasetImport.test.ts +95 -0
  303. package/src/observability/datasetImport.ts +220 -0
  304. package/src/observability/evaluate.test.ts +549 -0
  305. package/src/observability/evaluate.ts +254 -0
  306. package/src/observability/evaluatorTemplates.ts +124 -0
  307. package/src/observability/jsonPath.test.ts +52 -0
  308. package/src/observability/jsonPath.ts +60 -0
  309. package/src/observability/local/datasetStore.test.ts +230 -0
  310. package/src/observability/local/datasetStore.ts +452 -0
  311. package/src/observability/local/evaluatorStore.test.ts +138 -0
  312. package/src/observability/local/evaluatorStore.ts +224 -0
  313. package/src/observability/local/experimentRunner.test.ts +489 -0
  314. package/src/observability/local/experimentRunner.ts +548 -0
  315. package/src/observability/local/localPlugin.ts +88 -0
  316. package/src/observability/local/models/obsDataset.ts +38 -0
  317. package/src/observability/local/models/obsDatasetItem.ts +80 -0
  318. package/src/observability/local/models/obsEvaluator.ts +102 -0
  319. package/src/observability/local/models/obsExperiment.ts +91 -0
  320. package/src/observability/local/models/obsExperimentItem.ts +52 -0
  321. package/src/observability/local/models/obsPrompt.ts +43 -0
  322. package/src/observability/local/models/obsPromptLabel.ts +43 -0
  323. package/src/observability/local/models/obsPromptVersion.ts +107 -0
  324. package/src/observability/local/models/obsReviewItem.ts +79 -0
  325. package/src/observability/local/models/obsScore.ts +59 -0
  326. package/src/observability/local/models/obsSpan.ts +65 -0
  327. package/src/observability/local/models/obsTrace.ts +94 -0
  328. package/src/observability/local/obsModels.test.ts +119 -0
  329. package/src/observability/local/promptStore.test.ts +351 -0
  330. package/src/observability/local/promptStore.ts +665 -0
  331. package/src/observability/local/reviewStore.test.ts +164 -0
  332. package/src/observability/local/reviewStore.ts +264 -0
  333. package/src/observability/local/traceStore.test.ts +264 -0
  334. package/src/observability/local/traceStore.ts +440 -0
  335. package/src/observability/observabilityApp.test.ts +92 -0
  336. package/src/observability/observabilityApp.ts +185 -0
  337. package/src/observability/observabilityAppRegistry.test.ts +30 -0
  338. package/src/observability/observabilityAppRegistry.ts +35 -0
  339. package/src/observability/permissions.test.ts +96 -0
  340. package/src/observability/permissions.ts +78 -0
  341. package/src/observability/reviewPanels.test.ts +35 -0
  342. package/src/observability/reviewPanels.ts +64 -0
  343. package/src/observability/routes/datasets.test.ts +176 -0
  344. package/src/observability/routes/datasets.ts +252 -0
  345. package/src/observability/routes/evaluators.test.ts +119 -0
  346. package/src/observability/routes/evaluators.ts +207 -0
  347. package/src/observability/routes/experiments.test.ts +282 -0
  348. package/src/observability/routes/experiments.ts +152 -0
  349. package/src/observability/routes/observabilityRbac.test.ts +767 -0
  350. package/src/observability/routes/observabilityRouteAccess.ts +106 -0
  351. package/src/observability/routes/prompts.test.ts +320 -0
  352. package/src/observability/routes/prompts.ts +263 -0
  353. package/src/observability/routes/requestAiService.ts +36 -0
  354. package/src/observability/routes/review.test.ts +120 -0
  355. package/src/observability/routes/review.ts +125 -0
  356. package/src/observability/routes/status.test.ts +116 -0
  357. package/src/observability/routes/status.ts +47 -0
  358. package/src/observability/routes/testMultiStage.test.ts +214 -0
  359. package/src/observability/routes/testMultiStage.ts +96 -0
  360. package/src/observability/routes/traces.test.ts +167 -0
  361. package/src/observability/routes/traces.ts +172 -0
  362. package/src/observability/schemaValidation.ts +29 -0
  363. package/src/observability/sopThresholds.ts +20 -0
  364. package/src/observability/status.test.ts +38 -0
  365. package/src/observability/status.ts +56 -0
  366. package/src/observability/testMultiStageSchemas.ts +95 -0
  367. package/src/observability/testMultiStageWorkflow.test.ts +114 -0
  368. package/src/observability/testMultiStageWorkflow.ts +307 -0
  369. package/src/observability/types.ts +262 -0
  370. package/src/routes/files.test.ts +63 -9
  371. package/src/routes/files.ts +21 -7
  372. package/src/routes/gpt.test.ts +3279 -0
  373. package/src/routes/gpt.ts +91 -816
  374. package/src/routes/gptActions.test.ts +241 -0
  375. package/src/routes/gptActions.ts +255 -0
  376. package/src/routes/gptApprovals.test.ts +762 -0
  377. package/src/routes/gptDatasets.test.ts +86 -0
  378. package/src/routes/gptDatasets.ts +104 -0
  379. package/src/routes/gptFiles.test.ts +589 -0
  380. package/src/routes/gptHistories.test.ts +933 -0
  381. package/src/routes/gptHistories.ts +266 -5
  382. package/src/routes/routes.test.ts +222 -2
  383. package/src/service/aiDatasets.test.ts +123 -0
  384. package/src/service/aiDatasets.ts +316 -0
  385. package/src/service/aiService.test.ts +593 -0
  386. package/src/service/aiService.ts +568 -111
  387. package/src/service/askFiles.test.ts +96 -0
  388. package/src/service/askFiles.ts +222 -0
  389. package/src/service/asks.test.ts +946 -0
  390. package/src/service/asks.ts +423 -0
  391. package/src/service/chatTurn.ts +2287 -0
  392. package/src/service/fileStorage.test.ts +11 -0
  393. package/src/service/fileStorage.ts +10 -2
  394. package/src/service/fillActionIds.test.ts +65 -0
  395. package/src/service/fillActionIds.ts +98 -0
  396. package/src/service/jsonFenceModel.test.ts +47 -0
  397. package/src/service/jsonFenceModel.ts +53 -0
  398. package/src/service/prompts.test.ts +21 -0
  399. package/src/service/prompts.ts +147 -0
  400. package/src/service/sanitizeHtml.test.ts +61 -0
  401. package/src/service/sanitizeHtml.ts +109 -0
  402. package/src/tests/bunSetup.test.ts +7 -4
  403. package/src/tests/bunSetup.ts +7 -7
  404. package/src/tests/chatHarness.ts +478 -0
  405. package/src/tests/generateStream.ts +49 -0
  406. package/src/tests/harnessErrors.ts +22 -0
  407. package/src/tests/harnessStreaming.ts +227 -0
  408. package/src/tests/helpers.ts +9 -2
  409. package/src/types/harness.ts +1095 -0
  410. package/src/types/index.ts +209 -13
  411. package/src/types/observability.ts +344 -0
  412. package/src/types/sanitize-html.d.ts +25 -0
@@ -0,0 +1,230 @@
1
+ import {afterEach, beforeEach, describe, it} from "bun:test";
2
+ import {assert} from "chai";
3
+ import {DateTime} from "luxon";
4
+
5
+ import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
6
+ import {LocalDatasetStore, throwOnDatasetImportErrors} from "./datasetStore";
7
+ import {createLocalObservabilityBundle} from "./localPlugin";
8
+ import {registerObsDataset} from "./models/obsDataset";
9
+ import {registerObsDatasetItem} from "./models/obsDatasetItem";
10
+ import {registerObsTrace} from "./models/obsTrace";
11
+ import {LocalPromptStore} from "./promptStore";
12
+
13
+ describe("LocalDatasetStore", () => {
14
+ let store: LocalDatasetStore;
15
+ let promptStore: LocalPromptStore;
16
+
17
+ beforeEach(async () => {
18
+ createLocalObservabilityBundle();
19
+ promptStore = new LocalPromptStore();
20
+ store = new LocalDatasetStore(promptStore);
21
+ await registerObsDataset().deleteMany({});
22
+ await registerObsDatasetItem().deleteMany({});
23
+ await registerObsTrace().deleteMany({});
24
+ new ObservabilityApp({plugins: [createLocalObservabilityBundle().plugin]});
25
+ });
26
+
27
+ afterEach(() => {
28
+ resetObservabilityApp();
29
+ });
30
+
31
+ it("imports JSON rows and validates bound input schemas", async () => {
32
+ await promptStore.create({
33
+ folder: "examples",
34
+ inputSchema: {
35
+ properties: {text: {type: "string"}},
36
+ required: ["text"],
37
+ type: "object",
38
+ },
39
+ name: "bound-prompt",
40
+ type: "text",
41
+ });
42
+ await promptStore.moveLabel("bound-prompt", {label: "production", version: 1});
43
+ const dataset = await store.create({
44
+ inputSchemaPromptName: "bound-prompt",
45
+ name: "gold",
46
+ });
47
+ const ok = await store.importJson(dataset.id, [{text: "hello"}]);
48
+ assert.equal(ok.created, 1);
49
+ try {
50
+ await store.createItem(dataset.id, {input: {text: 123}});
51
+ assert.fail("expected schema failure");
52
+ } catch (error) {
53
+ assert.match(String(error), /\/text/);
54
+ }
55
+ });
56
+
57
+ it("keeps reserved metadata fields out of a flat JSON row input", async () => {
58
+ const dataset = await store.create({name: "flat-json"});
59
+ const result = await store.importJson(dataset.id, [
60
+ {proofread: true, tags: ["gold"], text: "hello"},
61
+ ]);
62
+
63
+ assert.equal(result.created, 1);
64
+ const items = await store.listItems(dataset.id);
65
+ assert.deepEqual(items[0]?.input, {text: "hello"});
66
+ assert.equal(items[0]?.proofread, true);
67
+ assert.deepEqual(items[0]?.tags, ["gold"]);
68
+ });
69
+
70
+ it("imports quoted CSV rows with nested input columns", async () => {
71
+ const dataset = await store.create({name: "csv-set"});
72
+ const csv = [
73
+ "input.text,expectedOutput.answer,proofread,tags",
74
+ '"hello, world","4",false,"gold|edge"',
75
+ ].join("\n");
76
+ const result = await store.importCsv(dataset.id, csv);
77
+ assert.equal(result.created, 1);
78
+ const items = await store.listItems(dataset.id);
79
+ assert.deepEqual(items[0]?.input, {text: "hello, world"});
80
+ assert.deepEqual(items[0]?.expectedOutput, {answer: 4});
81
+ assert.equal(items[0]?.proofread, false);
82
+ assert.deepEqual(items[0]?.tags, ["gold", "edge"]);
83
+ });
84
+
85
+ it("forces sensitive traces to proofread false and keeps sourceTraceId", async () => {
86
+ const dataset = await store.create({name: "trace-set"});
87
+ const trace = await registerObsTrace().create({
88
+ input: {text: "secret"},
89
+ name: "sensitive-trace",
90
+ output: {answer: "hidden"},
91
+ prompts: [],
92
+ sensitive: true,
93
+ startedAt: DateTime.utc().toJSDate(),
94
+ status: "ok",
95
+ });
96
+ const [item] = await store.addTracesToDataset({
97
+ datasetId: dataset.id,
98
+ traceIds: [String(trace._id)],
99
+ });
100
+ assert.equal(item.proofread, false);
101
+ assert.equal(item.sourceTraceId, String(trace._id));
102
+ await store.removeItem(dataset.id, item.id);
103
+ const reloaded = await registerObsTrace().findOneOrNone({_id: trace._id});
104
+ assert.isOk(reloaded);
105
+ assert.equal(reloaded?.sensitive, true);
106
+ });
107
+
108
+ it("includes human/auto/needsReview counts on list and detail", async () => {
109
+ const dataset = await store.create({name: "counts"});
110
+ await store.createItem(dataset.id, {input: {a: 1}, proofread: true});
111
+ await store.createItem(dataset.id, {input: {a: 2}, proofread: false});
112
+ await store.createItem(dataset.id, {
113
+ input: {a: 3},
114
+ origin: "synthetic",
115
+ proofread: false,
116
+ });
117
+ await store.createItem(dataset.id, {
118
+ input: {a: 4},
119
+ origin: "trace",
120
+ proofread: true,
121
+ });
122
+ const detail = await store.get(dataset.id);
123
+ assert.equal(detail.counts.total, 4);
124
+ assert.equal(detail.counts.human, 2);
125
+ assert.equal(detail.counts.auto, 1);
126
+ assert.equal(detail.counts.needsReview, 2);
127
+ const listed = await store.list();
128
+ assert.equal(
129
+ listed.some((row) => row.id === dataset.id),
130
+ true
131
+ );
132
+ });
133
+
134
+ it("supports dataset CRUD, duplicate protection, and item updates", async () => {
135
+ const created = await store.create({
136
+ description: "gold",
137
+ expectedOutputSchema: {type: "string"},
138
+ inputSchemaPromptName: "prompt",
139
+ name: "crud-set",
140
+ tags: ["a"],
141
+ });
142
+ const updated = await store.update(created.id, {description: "updated", name: "crud-set"});
143
+ assert.equal(updated.description, "updated");
144
+
145
+ try {
146
+ await store.create({name: "crud-set"});
147
+ assert.fail("expected duplicate dataset rejection");
148
+ } catch (error) {
149
+ assert.match(String(error), /already exists/);
150
+ }
151
+
152
+ const item = await store.createItem(created.id, {
153
+ input: {text: "one"},
154
+ metadata: {source: "seed"},
155
+ proofread: false,
156
+ tags: ["gold"],
157
+ });
158
+ const saved = await store.updateItem(created.id, item.id, {
159
+ input: {text: "two"},
160
+ proofread: true,
161
+ tags: ["reviewed"],
162
+ });
163
+ assert.deepEqual(saved.input, {text: "two"});
164
+ assert.equal(saved.proofread, true);
165
+ const clearedDataset = await store.update(created.id, {
166
+ description: null,
167
+ expectedOutputSchema: null,
168
+ inputSchemaPromptName: null,
169
+ });
170
+ assert.isUndefined(clearedDataset.description);
171
+ assert.isUndefined(clearedDataset.expectedOutputSchema);
172
+ assert.isUndefined(clearedDataset.inputSchemaPromptName);
173
+ const clearedItem = await store.updateItem(created.id, item.id, {
174
+ expectedOutput: null,
175
+ metadata: null,
176
+ outcomeClass: null,
177
+ });
178
+ assert.isUndefined(clearedItem.expectedOutput);
179
+ assert.isUndefined(clearedItem.metadata);
180
+ assert.isUndefined(clearedItem.outcomeClass);
181
+
182
+ await store.removeItem(created.id, item.id);
183
+ await store.remove(created.id);
184
+ try {
185
+ await store.get(created.id);
186
+ assert.fail("expected deleted dataset to 404");
187
+ } catch (error) {
188
+ assert.match(String(error), /Unknown dataset/);
189
+ }
190
+ });
191
+
192
+ it("wraps import validation errors and supports single-trace adds", async () => {
193
+ await promptStore.create({
194
+ folder: "examples",
195
+ inputSchema: {
196
+ properties: {text: {type: "string"}},
197
+ required: ["text"],
198
+ type: "object",
199
+ },
200
+ name: "import-bound",
201
+ type: "text",
202
+ });
203
+ await promptStore.moveLabel("import-bound", {label: "production", version: 1});
204
+ const dataset = await store.create({
205
+ inputSchemaPromptName: "import-bound",
206
+ name: "import-errors",
207
+ });
208
+ const failed = await store.importJson(dataset.id, [{text: 123}]);
209
+ try {
210
+ throwOnDatasetImportErrors(failed);
211
+ assert.fail("expected import validation throw");
212
+ } catch (error) {
213
+ assert.match(String(error), /Import failed on row/);
214
+ }
215
+
216
+ const trace = await registerObsTrace().create({
217
+ input: {text: "trace"},
218
+ name: "trace-add",
219
+ output: {answer: "ok"},
220
+ prompts: [],
221
+ startedAt: DateTime.utc().toJSDate(),
222
+ status: "ok",
223
+ });
224
+ const single = await store.addTraceToDataset({
225
+ datasetId: dataset.id,
226
+ traceId: String(trace._id),
227
+ });
228
+ assert.equal(single.sourceTraceId, String(trace._id));
229
+ });
230
+ });
@@ -0,0 +1,452 @@
1
+ import {APIError} from "@terreno/api";
2
+ import {DateTime} from "luxon";
3
+ import mongoose from "mongoose";
4
+
5
+ import type {ObsDatasetAnnotatedBy, ObsDatasetItemDocument} from "../../types/observability";
6
+ import {
7
+ flattenValidationPath,
8
+ type ParsedDatasetImportRow,
9
+ parseDatasetCsvImport,
10
+ parseDatasetJsonImport,
11
+ } from "../datasetImport";
12
+ import {validateAgainstSchema} from "../schemaValidation";
13
+ import {registerObsDataset} from "./models/obsDataset";
14
+ import {registerObsDatasetItem} from "./models/obsDatasetItem";
15
+ import {registerObsTrace} from "./models/obsTrace";
16
+ import {LocalPromptStore} from "./promptStore";
17
+
18
+ export interface DatasetWriteInput {
19
+ description?: string | null;
20
+ expectedOutputSchema?: Record<string, unknown> | null;
21
+ inputSchemaPromptName?: string | null;
22
+ name: string;
23
+ tags?: string[];
24
+ }
25
+
26
+ export interface DatasetItemWriteInput {
27
+ expectedOutput?: unknown | null;
28
+ input: unknown;
29
+ metadata?: Record<string, unknown> | null;
30
+ origin?: "manual" | "synthetic" | "trace";
31
+ outcomeClass?: "fn" | "fp" | "tn" | "tp" | null;
32
+ proofread?: boolean;
33
+ sourceTraceId?: string;
34
+ tags?: string[];
35
+ }
36
+
37
+ export interface DatasetCounts {
38
+ auto: number;
39
+ human: number;
40
+ needsReview: number;
41
+ total: number;
42
+ }
43
+
44
+ export interface DatasetView {
45
+ counts: DatasetCounts;
46
+ created: string;
47
+ description?: string;
48
+ expectedOutputSchema?: Record<string, unknown>;
49
+ id: string;
50
+ inputSchemaPromptName?: string;
51
+ name: string;
52
+ tags: string[];
53
+ updated: string;
54
+ }
55
+
56
+ export interface DatasetItemView {
57
+ annotatedBy?: ObsDatasetAnnotatedBy;
58
+ created: string;
59
+ datasetId: string;
60
+ expectedOutput?: unknown;
61
+ id: string;
62
+ input: unknown;
63
+ metadata?: Record<string, unknown>;
64
+ origin: "manual" | "synthetic" | "trace";
65
+ outcomeClass?: "fn" | "fp" | "tn" | "tp";
66
+ proofread: boolean;
67
+ sourceTraceId?: string;
68
+ tags: string[];
69
+ updated: string;
70
+ }
71
+
72
+ export interface DatasetImportResult {
73
+ created: number;
74
+ errors: Array<{message: string; path?: string; row: number}>;
75
+ }
76
+
77
+ export const throwOnDatasetImportErrors = (result: DatasetImportResult): void => {
78
+ const first = result.errors[0];
79
+ if (!first) {
80
+ return;
81
+ }
82
+ throw new APIError({
83
+ status: 400,
84
+ title: `Import failed on row ${first.row}: ${first.path ?? ""} ${first.message}`.trim(),
85
+ });
86
+ };
87
+
88
+ const toDatasetView = async (doc: {
89
+ _id: mongoose.Types.ObjectId;
90
+ created: Date;
91
+ description?: string;
92
+ expectedOutputSchema?: Record<string, unknown>;
93
+ inputSchemaPromptName?: string;
94
+ name: string;
95
+ tags: string[];
96
+ updated: Date;
97
+ }): Promise<DatasetView> => {
98
+ const counts = await computeDatasetCounts(String(doc._id));
99
+ return {
100
+ counts,
101
+ created: DateTime.fromJSDate(doc.created).toUTC().toISO() ?? "",
102
+ description: doc.description,
103
+ expectedOutputSchema: doc.expectedOutputSchema,
104
+ id: String(doc._id),
105
+ inputSchemaPromptName: doc.inputSchemaPromptName,
106
+ name: doc.name,
107
+ tags: doc.tags,
108
+ updated: DateTime.fromJSDate(doc.updated).toUTC().toISO() ?? "",
109
+ };
110
+ };
111
+
112
+ const toItemView = (doc: {
113
+ _id: mongoose.Types.ObjectId;
114
+ annotatedBy?: ObsDatasetAnnotatedBy;
115
+ created: Date;
116
+ datasetId: mongoose.Types.ObjectId;
117
+ expectedOutput?: unknown;
118
+ input: unknown;
119
+ metadata?: Record<string, unknown>;
120
+ origin: "manual" | "synthetic" | "trace";
121
+ outcomeClass?: "fn" | "fp" | "tn" | "tp";
122
+ proofread: boolean;
123
+ sourceTraceId?: mongoose.Types.ObjectId;
124
+ tags: string[];
125
+ updated: Date;
126
+ }): DatasetItemView => {
127
+ return {
128
+ annotatedBy: doc.annotatedBy,
129
+ created: DateTime.fromJSDate(doc.created).toUTC().toISO() ?? "",
130
+ datasetId: String(doc.datasetId),
131
+ expectedOutput: doc.expectedOutput,
132
+ id: String(doc._id),
133
+ input: doc.input,
134
+ metadata: doc.metadata,
135
+ origin: doc.origin,
136
+ outcomeClass: doc.outcomeClass,
137
+ proofread: doc.proofread,
138
+ sourceTraceId: doc.sourceTraceId ? String(doc.sourceTraceId) : undefined,
139
+ tags: doc.tags,
140
+ updated: DateTime.fromJSDate(doc.updated).toUTC().toISO() ?? "",
141
+ };
142
+ };
143
+
144
+ const computeDatasetCounts = async (datasetId: string): Promise<DatasetCounts> => {
145
+ const ObsDatasetItem = registerObsDatasetItem();
146
+ const objectId = new mongoose.Types.ObjectId(datasetId);
147
+ const [total, human, auto, needsReview] = await Promise.all([
148
+ ObsDatasetItem.countDocuments({datasetId: objectId}),
149
+ ObsDatasetItem.countDocuments({datasetId: objectId, proofread: true}),
150
+ ObsDatasetItem.countDocuments({
151
+ datasetId: objectId,
152
+ origin: {$in: ["synthetic", "trace"]},
153
+ proofread: false,
154
+ }),
155
+ ObsDatasetItem.countDocuments({datasetId: objectId, proofread: false}),
156
+ ]);
157
+ return {
158
+ auto,
159
+ human,
160
+ needsReview,
161
+ total,
162
+ };
163
+ };
164
+
165
+ export class LocalDatasetStore {
166
+ constructor(private readonly promptStore = new LocalPromptStore()) {}
167
+
168
+ async create(input: DatasetWriteInput): Promise<DatasetView> {
169
+ if (!input.name) {
170
+ throw new APIError({status: 400, title: "dataset name is required"});
171
+ }
172
+ try {
173
+ const created = await registerObsDataset().create({
174
+ description: input.description ?? undefined,
175
+ expectedOutputSchema: input.expectedOutputSchema ?? undefined,
176
+ inputSchemaPromptName: input.inputSchemaPromptName ?? undefined,
177
+ name: input.name,
178
+ tags: input.tags ?? [],
179
+ });
180
+ return toDatasetView(created);
181
+ } catch (error) {
182
+ if (
183
+ error &&
184
+ typeof error === "object" &&
185
+ "code" in error &&
186
+ (error as {code?: number}).code === 11000
187
+ ) {
188
+ throw new APIError({status: 409, title: `Dataset "${input.name}" already exists`});
189
+ }
190
+ throw error;
191
+ }
192
+ }
193
+
194
+ async list(): Promise<DatasetView[]> {
195
+ const rows = await registerObsDataset().find({}).sort({name: 1});
196
+ return Promise.all(
197
+ rows.map((row) => {
198
+ return toDatasetView(row);
199
+ })
200
+ );
201
+ }
202
+
203
+ async get(id: string): Promise<DatasetView> {
204
+ const doc = await registerObsDataset().findOneOrNone({_id: id});
205
+ if (!doc) {
206
+ throw new APIError({status: 404, title: `Unknown dataset "${id}"`});
207
+ }
208
+ return toDatasetView(doc);
209
+ }
210
+
211
+ async update(id: string, input: Partial<DatasetWriteInput>): Promise<DatasetView> {
212
+ const doc = await registerObsDataset().findOneOrNone({_id: id});
213
+ if (!doc) {
214
+ throw new APIError({status: 404, title: `Unknown dataset "${id}"`});
215
+ }
216
+ if (input.description !== undefined) {
217
+ doc.description = input.description ?? undefined;
218
+ }
219
+ if (input.expectedOutputSchema !== undefined) {
220
+ doc.expectedOutputSchema = input.expectedOutputSchema ?? undefined;
221
+ }
222
+ if (input.inputSchemaPromptName !== undefined) {
223
+ doc.inputSchemaPromptName = input.inputSchemaPromptName ?? undefined;
224
+ }
225
+ doc.name = input.name ?? doc.name;
226
+ doc.tags = input.tags ?? doc.tags;
227
+ await doc.save();
228
+ return toDatasetView(doc);
229
+ }
230
+
231
+ async remove(id: string): Promise<void> {
232
+ const doc = await registerObsDataset().findOneOrNone({_id: id});
233
+ if (!doc) {
234
+ throw new APIError({status: 404, title: `Unknown dataset "${id}"`});
235
+ }
236
+ doc.deleted = true;
237
+ await doc.save();
238
+ await registerObsDatasetItem().updateMany({datasetId: doc._id}, {deleted: true});
239
+ }
240
+
241
+ async listItems(datasetId: string): Promise<DatasetItemView[]> {
242
+ await this.get(datasetId);
243
+ const rows = await registerObsDatasetItem().find({datasetId}).sort({created: -1});
244
+ return rows.map((row) => {
245
+ return toItemView(row);
246
+ });
247
+ }
248
+
249
+ async createItem(datasetId: string, input: DatasetItemWriteInput): Promise<DatasetItemView> {
250
+ const dataset = await registerObsDataset().findOneOrNone({_id: datasetId});
251
+ if (!dataset) {
252
+ throw new APIError({status: 404, title: `Unknown dataset "${datasetId}"`});
253
+ }
254
+ await this.validateItemInput(dataset, input.input, 1);
255
+ const created = await registerObsDatasetItem().create({
256
+ datasetId: dataset._id,
257
+ expectedOutput: input.expectedOutput ?? undefined,
258
+ input: input.input,
259
+ metadata: input.metadata ?? undefined,
260
+ origin: input.origin ?? "manual",
261
+ outcomeClass: input.outcomeClass ?? undefined,
262
+ proofread: input.proofread ?? false,
263
+ sourceTraceId: input.sourceTraceId,
264
+ tags: input.tags ?? [],
265
+ });
266
+ return toItemView(created);
267
+ }
268
+
269
+ async updateItem(
270
+ datasetId: string,
271
+ itemId: string,
272
+ input: Partial<DatasetItemWriteInput>
273
+ ): Promise<DatasetItemView> {
274
+ const doc = await registerObsDatasetItem().findOneOrNone({_id: itemId, datasetId});
275
+ if (!doc) {
276
+ throw new APIError({status: 404, title: `Unknown dataset item "${itemId}"`});
277
+ }
278
+ if (input.input !== undefined) {
279
+ const dataset = await registerObsDataset().findOneOrNone({_id: datasetId});
280
+ if (!dataset) {
281
+ throw new APIError({status: 404, title: `Unknown dataset "${datasetId}"`});
282
+ }
283
+ await this.validateItemInput(dataset, input.input, 1);
284
+ doc.input = input.input;
285
+ }
286
+ if (input.expectedOutput !== undefined) {
287
+ doc.expectedOutput = input.expectedOutput ?? undefined;
288
+ }
289
+ if (input.metadata !== undefined) {
290
+ doc.metadata = input.metadata ?? undefined;
291
+ }
292
+ if (input.outcomeClass !== undefined) {
293
+ doc.outcomeClass = input.outcomeClass ?? undefined;
294
+ }
295
+ if (input.proofread !== undefined) {
296
+ doc.proofread = input.proofread;
297
+ }
298
+ doc.tags = input.tags ?? doc.tags;
299
+ await doc.save();
300
+ return toItemView(doc);
301
+ }
302
+
303
+ async removeItem(datasetId: string, itemId: string): Promise<void> {
304
+ const doc = await registerObsDatasetItem().findOneOrNone({_id: itemId, datasetId});
305
+ if (!doc) {
306
+ throw new APIError({status: 404, title: `Unknown dataset item "${itemId}"`});
307
+ }
308
+ doc.deleted = true;
309
+ await doc.save();
310
+ }
311
+
312
+ async importRows(
313
+ datasetId: string,
314
+ rows: ParsedDatasetImportRow[],
315
+ origin: "manual" | "synthetic" | "trace" = "manual"
316
+ ): Promise<DatasetImportResult> {
317
+ const dataset = await registerObsDataset().findOneOrNone({_id: datasetId});
318
+ if (!dataset) {
319
+ throw new APIError({status: 404, title: `Unknown dataset "${datasetId}"`});
320
+ }
321
+ const errors: DatasetImportResult["errors"] = [];
322
+ const validRows: ParsedDatasetImportRow[] = [];
323
+ for (let index = 0; index < rows.length; index += 1) {
324
+ const row = rows[index];
325
+ const rowNumber = index + 1;
326
+ const validationErrors = await this.validateItemInput(dataset, row.input, rowNumber, false);
327
+ if (validationErrors.length > 0) {
328
+ errors.push(...validationErrors);
329
+ continue;
330
+ }
331
+ validRows.push(row);
332
+ }
333
+ if (errors.length > 0) {
334
+ return {created: 0, errors};
335
+ }
336
+ const createdItems: ObsDatasetItemDocument[] = [];
337
+ for (const row of validRows) {
338
+ createdItems.push(
339
+ await registerObsDatasetItem().create({
340
+ datasetId: dataset._id,
341
+ expectedOutput: row.expectedOutput,
342
+ input: row.input,
343
+ metadata: row.metadata,
344
+ origin,
345
+ outcomeClass: row.outcomeClass,
346
+ proofread: row.proofread ?? false,
347
+ tags: row.tags,
348
+ })
349
+ );
350
+ }
351
+ return {created: createdItems.length, errors: []};
352
+ }
353
+
354
+ async importJson(datasetId: string, payload: unknown): Promise<DatasetImportResult> {
355
+ const rows = parseDatasetJsonImport(payload);
356
+ return this.importRows(datasetId, rows);
357
+ }
358
+
359
+ async importCsv(datasetId: string, content: string): Promise<DatasetImportResult> {
360
+ const rows = parseDatasetCsvImport(content);
361
+ return this.importRows(datasetId, rows);
362
+ }
363
+
364
+ async addTraceToDataset(params: {datasetId: string; traceId: string}): Promise<DatasetItemView> {
365
+ const result = await this.addTracesToDataset({
366
+ datasetId: params.datasetId,
367
+ traceIds: [params.traceId],
368
+ });
369
+ const first = result[0];
370
+ if (!first) {
371
+ throw new APIError({status: 404, title: `Unknown trace "${params.traceId}"`});
372
+ }
373
+ return first;
374
+ }
375
+
376
+ async addTracesToDataset(params: {
377
+ datasetId: string;
378
+ traceIds: string[];
379
+ }): Promise<DatasetItemView[]> {
380
+ const dataset = await registerObsDataset().findOneOrNone({_id: params.datasetId});
381
+ if (!dataset) {
382
+ throw new APIError({status: 404, title: `Unknown dataset "${params.datasetId}"`});
383
+ }
384
+ const created: DatasetItemView[] = [];
385
+ for (const traceId of params.traceIds) {
386
+ const trace = await registerObsTrace().findOneOrNone({_id: traceId});
387
+ if (!trace) {
388
+ throw new APIError({status: 404, title: `Unknown trace "${traceId}"`});
389
+ }
390
+ const input = trace.input ?? {};
391
+ await this.validateItemInput(dataset, input, 1);
392
+ const item = await registerObsDatasetItem().create({
393
+ datasetId: dataset._id,
394
+ expectedOutput: trace.output,
395
+ input,
396
+ origin: "trace",
397
+ proofread: !trace.sensitive,
398
+ sourceTraceId: trace._id,
399
+ tags: [],
400
+ });
401
+ created.push(toItemView(item));
402
+ }
403
+ return created;
404
+ }
405
+
406
+ async listExperimentItems(
407
+ datasetId: string,
408
+ includeUnproofread: boolean
409
+ ): Promise<DatasetItemView[]> {
410
+ const filter: Record<string, unknown> = {datasetId};
411
+ if (!includeUnproofread) {
412
+ filter.proofread = true;
413
+ }
414
+ const rows = await registerObsDatasetItem().find(filter).sort({created: 1});
415
+ return rows.map((row) => {
416
+ return toItemView(row);
417
+ });
418
+ }
419
+
420
+ private async validateItemInput(
421
+ dataset: {inputSchemaPromptName?: string},
422
+ input: unknown,
423
+ row: number,
424
+ throwOnError = true
425
+ ): Promise<DatasetImportResult["errors"]> {
426
+ if (!dataset.inputSchemaPromptName) {
427
+ return [];
428
+ }
429
+ const promptVersion = await this.promptStore.getVersionByLabel(
430
+ dataset.inputSchemaPromptName,
431
+ "production"
432
+ );
433
+ if (!promptVersion?.inputSchema) {
434
+ return [];
435
+ }
436
+ const errors = validateAgainstSchema(promptVersion.inputSchema, input).map((error) => {
437
+ return {
438
+ message: error.message,
439
+ path: flattenValidationPath(error.path),
440
+ row,
441
+ };
442
+ });
443
+ if (errors.length > 0 && throwOnError) {
444
+ const first = errors[0];
445
+ throw new APIError({
446
+ status: 400,
447
+ title: `Row ${first.row} failed input schema at ${first.path}: ${first.message}`,
448
+ });
449
+ }
450
+ return errors;
451
+ }
452
+ }