@terreno/ai 57.8.0 → 57.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (425) hide show
  1. package/.cursorrules +2 -2
  2. package/README.md +2 -1
  3. package/bunfig.toml +1 -1
  4. package/dist/aiApp.d.ts +16 -5
  5. package/dist/aiApp.js +17 -7
  6. package/dist/harness/adminScreens.d.ts +6 -0
  7. package/dist/harness/adminScreens.js +11 -0
  8. package/dist/harness/agentLoop.d.ts +61 -0
  9. package/dist/harness/agentLoop.js +798 -0
  10. package/dist/harness/agentTaskNames.d.ts +4 -0
  11. package/dist/harness/agentTaskNames.js +7 -0
  12. package/dist/harness/approvalPolicy.d.ts +3 -0
  13. package/dist/harness/approvalPolicy.js +23 -0
  14. package/dist/harness/approvals.d.ts +101 -0
  15. package/dist/harness/approvals.js +468 -0
  16. package/dist/harness/commit.d.ts +238 -0
  17. package/dist/harness/commit.js +678 -0
  18. package/dist/harness/conversation.d.ts +75 -0
  19. package/dist/harness/conversation.js +397 -0
  20. package/dist/harness/defineAgent.d.ts +9 -0
  21. package/dist/harness/defineAgent.js +65 -0
  22. package/dist/harness/defineTask.d.ts +11 -0
  23. package/dist/harness/defineTask.js +59 -0
  24. package/dist/harness/defineTool.d.ts +7 -0
  25. package/dist/harness/defineTool.js +36 -0
  26. package/dist/harness/definitionError.d.ts +8 -0
  27. package/dist/harness/definitionError.js +15 -0
  28. package/dist/harness/errors.d.ts +127 -0
  29. package/dist/harness/errors.js +101 -0
  30. package/dist/harness/eventHub.d.ts +45 -0
  31. package/dist/harness/eventHub.js +135 -0
  32. package/dist/harness/events.d.ts +95 -0
  33. package/dist/harness/events.js +267 -0
  34. package/dist/harness/executionEnv.d.ts +31 -0
  35. package/dist/harness/executionEnv.js +2 -0
  36. package/dist/harness/extensions.d.ts +82 -0
  37. package/dist/harness/extensions.js +274 -0
  38. package/dist/harness/harness.d.ts +163 -0
  39. package/dist/harness/harness.js +520 -0
  40. package/dist/harness/harnessApp.d.ts +44 -0
  41. package/dist/harness/harnessApp.js +62 -0
  42. package/dist/harness/internalRuntime.d.ts +16 -0
  43. package/dist/harness/internalRuntime.js +18 -0
  44. package/dist/harness/leases.d.ts +35 -0
  45. package/dist/harness/leases.js +86 -0
  46. package/dist/harness/memo.d.ts +24 -0
  47. package/dist/harness/memo.js +78 -0
  48. package/dist/harness/modelCall.d.ts +45 -0
  49. package/dist/harness/modelCall.js +147 -0
  50. package/dist/harness/models/harnessApproval.d.ts +2 -0
  51. package/dist/harness/models/harnessApproval.js +100 -0
  52. package/dist/harness/models/harnessConversation.d.ts +2 -0
  53. package/dist/harness/models/harnessConversation.js +117 -0
  54. package/dist/harness/models/harnessEvent.d.ts +3 -0
  55. package/dist/harness/models/harnessEvent.js +73 -0
  56. package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
  57. package/dist/harness/models/harnessInboxEvent.js +58 -0
  58. package/dist/harness/models/harnessMemo.d.ts +2 -0
  59. package/dist/harness/models/harnessMemo.js +42 -0
  60. package/dist/harness/models/harnessMessage.d.ts +2 -0
  61. package/dist/harness/models/harnessMessage.js +73 -0
  62. package/dist/harness/models/harnessOwner.d.ts +2 -0
  63. package/dist/harness/models/harnessOwner.js +36 -0
  64. package/dist/harness/models/harnessTask.d.ts +2 -0
  65. package/dist/harness/models/harnessTask.js +186 -0
  66. package/dist/harness/ownership.d.ts +89 -0
  67. package/dist/harness/ownership.js +438 -0
  68. package/dist/harness/registry.d.ts +29 -0
  69. package/dist/harness/registry.js +93 -0
  70. package/dist/harness/retryBackoff.d.ts +31 -0
  71. package/dist/harness/retryBackoff.js +50 -0
  72. package/dist/harness/routes/approvals.d.ts +13 -0
  73. package/dist/harness/routes/approvals.js +93 -0
  74. package/dist/harness/routes/conversations.d.ts +11 -0
  75. package/dist/harness/routes/conversations.js +66 -0
  76. package/dist/harness/routes/events.d.ts +19 -0
  77. package/dist/harness/routes/events.js +215 -0
  78. package/dist/harness/routes/tasks.d.ts +11 -0
  79. package/dist/harness/routes/tasks.js +108 -0
  80. package/dist/harness/runners/inProcessRunner.d.ts +57 -0
  81. package/dist/harness/runners/inProcessRunner.js +196 -0
  82. package/dist/harness/runtime.d.ts +34 -0
  83. package/dist/harness/runtime.js +551 -0
  84. package/dist/harness/subagent.d.ts +53 -0
  85. package/dist/harness/subagent.js +134 -0
  86. package/dist/harness/suspend.d.ts +8 -0
  87. package/dist/harness/suspend.js +16 -0
  88. package/dist/harness/transaction.d.ts +3 -0
  89. package/dist/harness/transaction.js +22 -0
  90. package/dist/harness/waits.d.ts +100 -0
  91. package/dist/harness/waits.js +214 -0
  92. package/dist/index.d.ts +19 -1
  93. package/dist/index.js +45 -2
  94. package/dist/models/aiDataset.d.ts +2 -0
  95. package/dist/models/aiDataset.js +67 -0
  96. package/dist/models/aiRequest.js +1 -1
  97. package/dist/models/gptHistory.js +88 -2
  98. package/dist/observability/adminScreens.d.ts +5 -0
  99. package/dist/observability/adminScreens.js +32 -0
  100. package/dist/observability/compileTemplate.d.ts +1 -0
  101. package/dist/observability/compileTemplate.js +13 -0
  102. package/dist/observability/datasetImport.d.ts +11 -0
  103. package/dist/observability/datasetImport.js +185 -0
  104. package/dist/observability/evaluate.d.ts +31 -0
  105. package/dist/observability/evaluate.js +200 -0
  106. package/dist/observability/evaluatorTemplates.d.ts +17 -0
  107. package/dist/observability/evaluatorTemplates.js +111 -0
  108. package/dist/observability/jsonPath.d.ts +2 -0
  109. package/dist/observability/jsonPath.js +57 -0
  110. package/dist/observability/local/datasetStore.d.ts +87 -0
  111. package/dist/observability/local/datasetStore.js +326 -0
  112. package/dist/observability/local/evaluatorStore.d.ts +44 -0
  113. package/dist/observability/local/evaluatorStore.js +182 -0
  114. package/dist/observability/local/experimentRunner.d.ts +96 -0
  115. package/dist/observability/local/experimentRunner.js +419 -0
  116. package/dist/observability/local/localPlugin.d.ts +22 -0
  117. package/dist/observability/local/localPlugin.js +76 -0
  118. package/dist/observability/local/models/obsDataset.d.ts +2 -0
  119. package/dist/observability/local/models/obsDataset.js +37 -0
  120. package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
  121. package/dist/observability/local/models/obsDatasetItem.js +76 -0
  122. package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
  123. package/dist/observability/local/models/obsEvaluator.js +98 -0
  124. package/dist/observability/local/models/obsExperiment.d.ts +2 -0
  125. package/dist/observability/local/models/obsExperiment.js +87 -0
  126. package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
  127. package/dist/observability/local/models/obsExperimentItem.js +45 -0
  128. package/dist/observability/local/models/obsPrompt.d.ts +2 -0
  129. package/dist/observability/local/models/obsPrompt.js +42 -0
  130. package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
  131. package/dist/observability/local/models/obsPromptLabel.js +39 -0
  132. package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
  133. package/dist/observability/local/models/obsPromptVersion.js +96 -0
  134. package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
  135. package/dist/observability/local/models/obsReviewItem.js +75 -0
  136. package/dist/observability/local/models/obsScore.d.ts +2 -0
  137. package/dist/observability/local/models/obsScore.js +58 -0
  138. package/dist/observability/local/models/obsSpan.d.ts +2 -0
  139. package/dist/observability/local/models/obsSpan.js +64 -0
  140. package/dist/observability/local/models/obsTrace.d.ts +2 -0
  141. package/dist/observability/local/models/obsTrace.js +93 -0
  142. package/dist/observability/local/promptStore.d.ts +180 -0
  143. package/dist/observability/local/promptStore.js +444 -0
  144. package/dist/observability/local/reviewStore.d.ts +50 -0
  145. package/dist/observability/local/reviewStore.js +203 -0
  146. package/dist/observability/local/traceStore.d.ts +103 -0
  147. package/dist/observability/local/traceStore.js +325 -0
  148. package/dist/observability/observabilityApp.d.ts +24 -0
  149. package/dist/observability/observabilityApp.js +157 -0
  150. package/dist/observability/observabilityAppRegistry.d.ts +14 -0
  151. package/dist/observability/observabilityAppRegistry.js +16 -0
  152. package/dist/observability/permissions.d.ts +7 -0
  153. package/dist/observability/permissions.js +60 -0
  154. package/dist/observability/reviewPanels.d.ts +18 -0
  155. package/dist/observability/reviewPanels.js +46 -0
  156. package/dist/observability/routes/datasets.d.ts +7 -0
  157. package/dist/observability/routes/datasets.js +149 -0
  158. package/dist/observability/routes/evaluators.d.ts +7 -0
  159. package/dist/observability/routes/evaluators.js +123 -0
  160. package/dist/observability/routes/experiments.d.ts +7 -0
  161. package/dist/observability/routes/experiments.js +95 -0
  162. package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
  163. package/dist/observability/routes/observabilityRouteAccess.js +67 -0
  164. package/dist/observability/routes/prompts.d.ts +12 -0
  165. package/dist/observability/routes/prompts.js +172 -0
  166. package/dist/observability/routes/requestAiService.d.ts +13 -0
  167. package/dist/observability/routes/requestAiService.js +21 -0
  168. package/dist/observability/routes/review.d.ts +7 -0
  169. package/dist/observability/routes/review.js +79 -0
  170. package/dist/observability/routes/status.d.ts +5 -0
  171. package/dist/observability/routes/status.js +31 -0
  172. package/dist/observability/routes/testMultiStage.d.ts +9 -0
  173. package/dist/observability/routes/testMultiStage.js +67 -0
  174. package/dist/observability/routes/traces.d.ts +7 -0
  175. package/dist/observability/routes/traces.js +129 -0
  176. package/dist/observability/schemaValidation.d.ts +5 -0
  177. package/dist/observability/schemaValidation.js +26 -0
  178. package/dist/observability/sopThresholds.d.ts +2 -0
  179. package/dist/observability/sopThresholds.js +21 -0
  180. package/dist/observability/status.d.ts +23 -0
  181. package/dist/observability/status.js +34 -0
  182. package/dist/observability/testMultiStageSchemas.d.ts +10 -0
  183. package/dist/observability/testMultiStageSchemas.js +79 -0
  184. package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
  185. package/dist/observability/testMultiStageWorkflow.js +218 -0
  186. package/dist/observability/types.d.ts +159 -0
  187. package/dist/observability/types.js +67 -0
  188. package/dist/routes/files.js +20 -7
  189. package/dist/routes/gpt.js +179 -488
  190. package/dist/routes/gptActions.d.ts +9 -0
  191. package/dist/routes/gptActions.js +212 -0
  192. package/dist/routes/gptDatasets.d.ts +5 -0
  193. package/dist/routes/gptDatasets.js +94 -0
  194. package/dist/routes/gptHistories.d.ts +3 -1
  195. package/dist/routes/gptHistories.js +188 -2
  196. package/dist/service/aiDatasets.d.ts +57 -0
  197. package/dist/service/aiDatasets.js +253 -0
  198. package/dist/service/aiService.d.ts +50 -1
  199. package/dist/service/aiService.js +431 -77
  200. package/dist/service/askFiles.d.ts +49 -0
  201. package/dist/service/askFiles.js +162 -0
  202. package/dist/service/asks.d.ts +76 -0
  203. package/dist/service/asks.js +284 -0
  204. package/dist/service/chatTurn.d.ts +128 -0
  205. package/dist/service/chatTurn.js +1674 -0
  206. package/dist/service/fileStorage.d.ts +4 -0
  207. package/dist/service/fileStorage.js +7 -2
  208. package/dist/service/fileUploadsGate.d.ts +8 -0
  209. package/dist/service/fileUploadsGate.js +18 -0
  210. package/dist/service/fillActionIds.d.ts +6 -0
  211. package/dist/service/fillActionIds.js +79 -0
  212. package/dist/service/gemini.d.ts +3 -3
  213. package/dist/service/gemini.js +3 -3
  214. package/dist/service/jsonFenceModel.d.ts +7 -0
  215. package/dist/service/jsonFenceModel.js +45 -0
  216. package/dist/service/prompts.d.ts +52 -0
  217. package/dist/service/prompts.js +96 -1
  218. package/dist/service/sanitizeHtml.d.ts +11 -0
  219. package/dist/service/sanitizeHtml.js +108 -0
  220. package/dist/service/vertex.d.ts +6 -6
  221. package/dist/service/vertex.js +5 -5
  222. package/dist/types/harness.d.ts +934 -0
  223. package/dist/types/harness.js +133 -0
  224. package/dist/types/index.d.ts +219 -13
  225. package/dist/types/index.js +2 -0
  226. package/dist/types/observability.d.ts +288 -0
  227. package/dist/types/observability.js +2 -0
  228. package/package.json +17 -5
  229. package/src/aiApp.test.ts +207 -1
  230. package/src/aiApp.ts +31 -8
  231. package/src/harness/adminScreens.test.ts +34 -0
  232. package/src/harness/adminScreens.ts +11 -0
  233. package/src/harness/agentLoop.test.ts +1034 -0
  234. package/src/harness/agentLoop.ts +1082 -0
  235. package/src/harness/agentRecovery.test.ts +380 -0
  236. package/src/harness/agentTaskNames.ts +4 -0
  237. package/src/harness/approvalPolicy.ts +28 -0
  238. package/src/harness/approvals.test.ts +1094 -0
  239. package/src/harness/approvals.ts +644 -0
  240. package/src/harness/commit.ts +1049 -0
  241. package/src/harness/conversation.ts +503 -0
  242. package/src/harness/defineAgent.test.ts +92 -0
  243. package/src/harness/defineAgent.ts +82 -0
  244. package/src/harness/defineTask.test.ts +59 -0
  245. package/src/harness/defineTask.ts +76 -0
  246. package/src/harness/defineTool.ts +45 -0
  247. package/src/harness/definitionError.ts +13 -0
  248. package/src/harness/errors.test.ts +90 -0
  249. package/src/harness/errors.ts +119 -0
  250. package/src/harness/eventHub.test.ts +124 -0
  251. package/src/harness/eventHub.ts +159 -0
  252. package/src/harness/events.test.ts +757 -0
  253. package/src/harness/events.ts +384 -0
  254. package/src/harness/executionEnv.ts +34 -0
  255. package/src/harness/extensions.test.ts +1218 -0
  256. package/src/harness/extensions.ts +382 -0
  257. package/src/harness/harness.test.ts +693 -0
  258. package/src/harness/harness.ts +767 -0
  259. package/src/harness/harnessApp.ts +76 -0
  260. package/src/harness/internalRuntime.ts +44 -0
  261. package/src/harness/leases.ts +132 -0
  262. package/src/harness/memo.ts +107 -0
  263. package/src/harness/modelCall.test.ts +134 -0
  264. package/src/harness/modelCall.ts +200 -0
  265. package/src/harness/models/harnessApproval.ts +109 -0
  266. package/src/harness/models/harnessConversation.ts +142 -0
  267. package/src/harness/models/harnessEvent.ts +92 -0
  268. package/src/harness/models/harnessInboxEvent.ts +73 -0
  269. package/src/harness/models/harnessMemo.ts +43 -0
  270. package/src/harness/models/harnessMessage.ts +85 -0
  271. package/src/harness/models/harnessModels.test.ts +60 -0
  272. package/src/harness/models/harnessOwner.ts +40 -0
  273. package/src/harness/models/harnessTask.ts +201 -0
  274. package/src/harness/ownership.test.ts +1087 -0
  275. package/src/harness/ownership.ts +618 -0
  276. package/src/harness/recovery.test.ts +912 -0
  277. package/src/harness/registry.ts +124 -0
  278. package/src/harness/retries.test.ts +292 -0
  279. package/src/harness/retryBackoff.test.ts +100 -0
  280. package/src/harness/retryBackoff.ts +77 -0
  281. package/src/harness/routes/approvals.ts +123 -0
  282. package/src/harness/routes/conversations.test.ts +518 -0
  283. package/src/harness/routes/conversations.ts +102 -0
  284. package/src/harness/routes/events.ts +280 -0
  285. package/src/harness/routes/tasks.test.ts +198 -0
  286. package/src/harness/routes/tasks.ts +146 -0
  287. package/src/harness/runners/inProcessRunner.test.ts +402 -0
  288. package/src/harness/runners/inProcessRunner.ts +220 -0
  289. package/src/harness/runtime.ts +714 -0
  290. package/src/harness/subagent.test.ts +696 -0
  291. package/src/harness/subagent.ts +192 -0
  292. package/src/harness/suspend.ts +13 -0
  293. package/src/harness/transaction.ts +17 -0
  294. package/src/harness/versioning.test.ts +288 -0
  295. package/src/harness/waits.test.ts +693 -0
  296. package/src/harness/waits.ts +356 -0
  297. package/src/index.test.ts +13 -0
  298. package/src/index.ts +70 -0
  299. package/src/isolated/adminScreensConfig.isolated.ts +143 -0
  300. package/src/models/aiDataset.test.ts +34 -0
  301. package/src/models/aiDataset.ts +78 -0
  302. package/src/models/aiRequest.test.ts +2 -2
  303. package/src/models/aiRequest.ts +2 -1
  304. package/src/models/gptHistory.test.ts +156 -0
  305. package/src/models/gptHistory.ts +106 -4
  306. package/src/observability/adminScreens.test.ts +71 -0
  307. package/src/observability/adminScreens.ts +42 -0
  308. package/src/observability/compileTemplate.test.ts +14 -0
  309. package/src/observability/compileTemplate.ts +10 -0
  310. package/src/observability/datasetImport.test.ts +95 -0
  311. package/src/observability/datasetImport.ts +220 -0
  312. package/src/observability/evaluate.test.ts +549 -0
  313. package/src/observability/evaluate.ts +254 -0
  314. package/src/observability/evaluatorTemplates.ts +124 -0
  315. package/src/observability/jsonPath.test.ts +52 -0
  316. package/src/observability/jsonPath.ts +60 -0
  317. package/src/observability/local/datasetStore.test.ts +230 -0
  318. package/src/observability/local/datasetStore.ts +452 -0
  319. package/src/observability/local/evaluatorStore.test.ts +138 -0
  320. package/src/observability/local/evaluatorStore.ts +224 -0
  321. package/src/observability/local/experimentRunner.test.ts +489 -0
  322. package/src/observability/local/experimentRunner.ts +548 -0
  323. package/src/observability/local/localPlugin.ts +88 -0
  324. package/src/observability/local/models/obsDataset.ts +38 -0
  325. package/src/observability/local/models/obsDatasetItem.ts +80 -0
  326. package/src/observability/local/models/obsEvaluator.ts +102 -0
  327. package/src/observability/local/models/obsExperiment.ts +91 -0
  328. package/src/observability/local/models/obsExperimentItem.ts +52 -0
  329. package/src/observability/local/models/obsPrompt.ts +43 -0
  330. package/src/observability/local/models/obsPromptLabel.ts +43 -0
  331. package/src/observability/local/models/obsPromptVersion.ts +107 -0
  332. package/src/observability/local/models/obsReviewItem.ts +79 -0
  333. package/src/observability/local/models/obsScore.ts +59 -0
  334. package/src/observability/local/models/obsSpan.ts +65 -0
  335. package/src/observability/local/models/obsTrace.ts +94 -0
  336. package/src/observability/local/obsModels.test.ts +119 -0
  337. package/src/observability/local/promptStore.test.ts +351 -0
  338. package/src/observability/local/promptStore.ts +665 -0
  339. package/src/observability/local/reviewStore.test.ts +164 -0
  340. package/src/observability/local/reviewStore.ts +264 -0
  341. package/src/observability/local/traceStore.test.ts +264 -0
  342. package/src/observability/local/traceStore.ts +440 -0
  343. package/src/observability/observabilityApp.test.ts +92 -0
  344. package/src/observability/observabilityApp.ts +185 -0
  345. package/src/observability/observabilityAppRegistry.test.ts +30 -0
  346. package/src/observability/observabilityAppRegistry.ts +35 -0
  347. package/src/observability/permissions.test.ts +96 -0
  348. package/src/observability/permissions.ts +78 -0
  349. package/src/observability/reviewPanels.test.ts +35 -0
  350. package/src/observability/reviewPanels.ts +64 -0
  351. package/src/observability/routes/datasets.test.ts +176 -0
  352. package/src/observability/routes/datasets.ts +252 -0
  353. package/src/observability/routes/evaluators.test.ts +119 -0
  354. package/src/observability/routes/evaluators.ts +207 -0
  355. package/src/observability/routes/experiments.test.ts +282 -0
  356. package/src/observability/routes/experiments.ts +152 -0
  357. package/src/observability/routes/observabilityRbac.test.ts +767 -0
  358. package/src/observability/routes/observabilityRouteAccess.ts +106 -0
  359. package/src/observability/routes/prompts.test.ts +320 -0
  360. package/src/observability/routes/prompts.ts +263 -0
  361. package/src/observability/routes/requestAiService.ts +36 -0
  362. package/src/observability/routes/review.test.ts +120 -0
  363. package/src/observability/routes/review.ts +125 -0
  364. package/src/observability/routes/status.test.ts +116 -0
  365. package/src/observability/routes/status.ts +47 -0
  366. package/src/observability/routes/testMultiStage.test.ts +214 -0
  367. package/src/observability/routes/testMultiStage.ts +96 -0
  368. package/src/observability/routes/traces.test.ts +167 -0
  369. package/src/observability/routes/traces.ts +172 -0
  370. package/src/observability/schemaValidation.ts +29 -0
  371. package/src/observability/sopThresholds.ts +20 -0
  372. package/src/observability/status.test.ts +38 -0
  373. package/src/observability/status.ts +56 -0
  374. package/src/observability/testMultiStageSchemas.ts +95 -0
  375. package/src/observability/testMultiStageWorkflow.test.ts +114 -0
  376. package/src/observability/testMultiStageWorkflow.ts +307 -0
  377. package/src/observability/types.ts +262 -0
  378. package/src/routes/files.test.ts +84 -9
  379. package/src/routes/files.ts +23 -7
  380. package/src/routes/gpt.test.ts +3279 -0
  381. package/src/routes/gpt.ts +214 -552
  382. package/src/routes/gptActions.test.ts +241 -0
  383. package/src/routes/gptActions.ts +255 -0
  384. package/src/routes/gptApprovals.test.ts +762 -0
  385. package/src/routes/gptDatasets.test.ts +86 -0
  386. package/src/routes/gptDatasets.ts +104 -0
  387. package/src/routes/gptFiles.test.ts +589 -0
  388. package/src/routes/gptHistories.test.ts +933 -0
  389. package/src/routes/gptHistories.ts +266 -5
  390. package/src/routes/routes.test.ts +676 -5
  391. package/src/service/aiDatasets.test.ts +123 -0
  392. package/src/service/aiDatasets.ts +316 -0
  393. package/src/service/aiService.test.ts +614 -0
  394. package/src/service/aiService.ts +575 -111
  395. package/src/service/askFiles.test.ts +96 -0
  396. package/src/service/askFiles.ts +222 -0
  397. package/src/service/asks.test.ts +946 -0
  398. package/src/service/asks.ts +423 -0
  399. package/src/service/chatTurn.ts +2287 -0
  400. package/src/service/fileStorage.test.ts +11 -0
  401. package/src/service/fileStorage.ts +10 -2
  402. package/src/service/fileUploadsGate.ts +23 -0
  403. package/src/service/fillActionIds.test.ts +65 -0
  404. package/src/service/fillActionIds.ts +98 -0
  405. package/src/service/gemini.test.ts +16 -10
  406. package/src/service/gemini.ts +3 -3
  407. package/src/service/jsonFenceModel.test.ts +47 -0
  408. package/src/service/jsonFenceModel.ts +53 -0
  409. package/src/service/prompts.test.ts +21 -0
  410. package/src/service/prompts.ts +147 -0
  411. package/src/service/sanitizeHtml.test.ts +61 -0
  412. package/src/service/sanitizeHtml.ts +109 -0
  413. package/src/service/vertex.test.ts +49 -49
  414. package/src/service/vertex.ts +7 -8
  415. package/src/tests/bunSetup.test.ts +7 -4
  416. package/src/tests/bunSetup.ts +7 -7
  417. package/src/tests/chatHarness.ts +478 -0
  418. package/src/tests/generateStream.ts +49 -0
  419. package/src/tests/harnessErrors.ts +22 -0
  420. package/src/tests/harnessStreaming.ts +227 -0
  421. package/src/tests/helpers.ts +9 -2
  422. package/src/types/harness.ts +1095 -0
  423. package/src/types/index.ts +237 -12
  424. package/src/types/observability.ts +344 -0
  425. package/src/types/sanitize-html.d.ts +25 -0
@@ -0,0 +1,444 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.LocalPromptStore = void 0;
4
+ const api_1 = require("@terreno/api");
5
+ const luxon_1 = require("luxon");
6
+ const compileTemplate_1 = require("../compileTemplate");
7
+ const obsExperiment_1 = require("./models/obsExperiment");
8
+ const obsPrompt_1 = require("./models/obsPrompt");
9
+ const obsPromptLabel_1 = require("./models/obsPromptLabel");
10
+ const obsPromptVersion_1 = require("./models/obsPromptVersion");
11
+ const obsTrace_1 = require("./models/obsTrace");
12
+ const PROMPT_RELATIONSHIP_LIMIT = 20;
13
+ const versionBody = (system, template) => {
14
+ if (system && system.length > 0) {
15
+ return system;
16
+ }
17
+ return template ?? "";
18
+ };
19
+ const escapeRegex = (value) => {
20
+ return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
21
+ };
22
+ const computeCostUsd = (params) => {
23
+ if (!params.priceMap) {
24
+ return undefined;
25
+ }
26
+ const price = params.priceMap[params.modelId];
27
+ if (!price || params.inputTokens === undefined || params.outputTokens === undefined) {
28
+ return undefined;
29
+ }
30
+ return ((price.inputPerMTok * params.inputTokens + price.outputPerMTok * params.outputTokens) /
31
+ 1_000_000);
32
+ };
33
+ class LocalPromptStore {
34
+ async create(input) {
35
+ if (!input.folder || !input.name) {
36
+ throw new api_1.APIError({ status: 400, title: "folder and name are required" });
37
+ }
38
+ const ObsPrompt = (0, obsPrompt_1.registerObsPrompt)();
39
+ const existing = await ObsPrompt.findOneOrNone({ name: input.name });
40
+ if (existing) {
41
+ throw new api_1.APIError({ status: 409, title: `Prompt "${input.name}" already exists` });
42
+ }
43
+ try {
44
+ const prompt = await ObsPrompt.create({
45
+ ...(input.description !== undefined ? { description: input.description } : {}),
46
+ folder: input.folder,
47
+ name: input.name,
48
+ tags: input.tags ?? [],
49
+ });
50
+ const version = await this.insertVersion(prompt._id, 1, input);
51
+ await this.upsertLabel(prompt._id, "latest", version._id);
52
+ return { name: prompt.name, version: 1 };
53
+ }
54
+ catch (error) {
55
+ if (this.isDuplicateKey(error)) {
56
+ throw new api_1.APIError({ status: 409, title: `Prompt "${input.name}" already exists` });
57
+ }
58
+ throw error;
59
+ }
60
+ }
61
+ async createVersion(name, fields) {
62
+ const prompt = await this.requirePrompt(name);
63
+ const ObsPromptVersion = (0, obsPromptVersion_1.registerObsPromptVersion)();
64
+ const latestDocs = await ObsPromptVersion.find({ promptId: prompt._id })
65
+ .sort({ version: -1 })
66
+ .limit(1);
67
+ const nextVersion = (latestDocs[0]?.version ?? 0) + 1;
68
+ const created = await this.insertVersion(prompt._id, nextVersion, fields);
69
+ await this.upsertLabel(prompt._id, "latest", created._id);
70
+ return { name: prompt.name, version: nextVersion };
71
+ }
72
+ async moveLabel(name, params) {
73
+ if (params.label !== "production" && params.label !== "staging") {
74
+ throw new api_1.APIError({ status: 400, title: "label must be production or staging" });
75
+ }
76
+ const prompt = await this.requirePrompt(name);
77
+ const ObsPromptVersion = (0, obsPromptVersion_1.registerObsPromptVersion)();
78
+ const versionDoc = await ObsPromptVersion.findOneOrNone({
79
+ promptId: prompt._id,
80
+ version: params.version,
81
+ });
82
+ if (!versionDoc) {
83
+ throw new api_1.APIError({
84
+ status: 404,
85
+ title: `Unknown version ${params.version} for prompt "${name}"`,
86
+ });
87
+ }
88
+ const outgoingVersion = await this.upsertLabel(prompt._id, params.label, versionDoc._id);
89
+ return { label: params.label, outgoingVersion, version: params.version };
90
+ }
91
+ async get(args) {
92
+ const prompt = await (0, obsPrompt_1.registerObsPrompt)().findOneOrNone({ name: args.name });
93
+ if (!prompt) {
94
+ return undefined;
95
+ }
96
+ const label = args.label ?? "production";
97
+ const labelDoc = await (0, obsPromptLabel_1.registerObsPromptLabel)().findOneOrNone({
98
+ label,
99
+ promptId: prompt._id,
100
+ });
101
+ if (!labelDoc) {
102
+ return undefined;
103
+ }
104
+ const version = await (0, obsPromptVersion_1.registerObsPromptVersion)().findOneOrNone({ _id: labelDoc.versionId });
105
+ if (!version) {
106
+ return undefined;
107
+ }
108
+ return {
109
+ body: versionBody(version.system, version.template),
110
+ label,
111
+ name: prompt.name,
112
+ sensitive: version.sensitive,
113
+ version: version.version,
114
+ };
115
+ }
116
+ async getDetail(name, options) {
117
+ const prompt = await this.requirePrompt(name);
118
+ const versions = await (0, obsPromptVersion_1.registerObsPromptVersion)().find({ promptId: prompt._id }).sort({
119
+ version: 1,
120
+ });
121
+ const labels = await (0, obsPromptLabel_1.registerObsPromptLabel)().find({ promptId: prompt._id });
122
+ const versionById = new Map(versions.map((row) => [String(row._id), row.version]));
123
+ return {
124
+ ...(prompt.description !== undefined ? { description: prompt.description } : {}),
125
+ folder: prompt.folder,
126
+ labels: labels.map((row) => {
127
+ return { label: row.label, version: versionById.get(String(row.versionId)) ?? 0 };
128
+ }),
129
+ name: prompt.name,
130
+ relationships: await this.loadRelationships(prompt.name, options?.promptVersion),
131
+ tags: prompt.tags,
132
+ versions: versions.map((row) => {
133
+ return {
134
+ config: row.config,
135
+ created: luxon_1.DateTime.fromJSDate(row.created).toUTC().toISO() ?? "",
136
+ sensitive: row.sensitive,
137
+ system: row.system,
138
+ template: row.template,
139
+ type: row.type,
140
+ variables: row.variables,
141
+ version: row.version,
142
+ };
143
+ }),
144
+ };
145
+ }
146
+ async list(params) {
147
+ const filter = {};
148
+ if (params.folder) {
149
+ filter.folder = params.folder;
150
+ }
151
+ if (params.search) {
152
+ filter.name = { $options: "i", $regex: escapeRegex(params.search) };
153
+ }
154
+ const prompts = await (0, obsPrompt_1.registerObsPrompt)().find(filter).sort({ folder: 1, name: 1 });
155
+ const usageByName = params.includeUsage7d
156
+ ? await this.usageLast7d(prompts.map((row) => row.name))
157
+ : undefined;
158
+ const items = [];
159
+ for (const prompt of prompts) {
160
+ const versions = await (0, obsPromptVersion_1.registerObsPromptVersion)()
161
+ .find({ promptId: prompt._id })
162
+ .sort({ version: -1 })
163
+ .limit(1);
164
+ const productionLabel = await (0, obsPromptLabel_1.registerObsPromptLabel)().findOneOrNone({
165
+ label: "production",
166
+ promptId: prompt._id,
167
+ });
168
+ let production = "—";
169
+ if (productionLabel) {
170
+ const productionVersion = await (0, obsPromptVersion_1.registerObsPromptVersion)().findOneOrNone({
171
+ _id: productionLabel.versionId,
172
+ });
173
+ if (productionVersion) {
174
+ production = productionVersion.version;
175
+ }
176
+ }
177
+ const latest = versions[0];
178
+ items.push({
179
+ ...(prompt.description !== undefined ? { description: prompt.description } : {}),
180
+ folder: prompt.folder,
181
+ latestVersion: latest?.version ?? 0,
182
+ name: prompt.name,
183
+ production,
184
+ type: latest?.type ?? "text",
185
+ ...(usageByName ? { usage7d: usageByName.get(prompt.name) ?? { calls: 0 } } : {}),
186
+ });
187
+ }
188
+ return items;
189
+ }
190
+ compile(params) {
191
+ const variables = params.variables ?? {};
192
+ const messages = [];
193
+ const system = (0, compileTemplate_1.compileTemplate)(params.system ?? "", variables);
194
+ if (system.length > 0) {
195
+ messages.push({ content: system, role: "system" });
196
+ }
197
+ const template = (0, compileTemplate_1.compileTemplate)(params.template ?? "", variables);
198
+ const userContent = template.length > 0 ? template : (params.userPrompt ?? "");
199
+ if (userContent.length > 0) {
200
+ messages.push({ content: userContent, role: "user" });
201
+ }
202
+ return messages;
203
+ }
204
+ async runPlayground(params) {
205
+ const prompt = await this.requirePrompt(params.name);
206
+ const versionDoc = await this.resolvePlaygroundVersion(prompt._id, params.version);
207
+ const compiledMessages = this.compile({
208
+ system: versionDoc.system,
209
+ template: versionDoc.template,
210
+ userPrompt: params.userPrompt,
211
+ variables: params.variables,
212
+ });
213
+ const systemMessage = compiledMessages.find((message) => {
214
+ return message.role === "system";
215
+ });
216
+ const userMessage = compiledMessages.find((message) => {
217
+ return message.role === "user";
218
+ });
219
+ const generated = await params.generator.generate({
220
+ prompt: userMessage?.content ?? "",
221
+ systemPrompt: systemMessage?.content,
222
+ userId: params.userId,
223
+ });
224
+ const tokens = {
225
+ inputTokens: generated.inputTokens,
226
+ outputTokens: generated.outputTokens,
227
+ totalTokens: generated.inputTokens === undefined && generated.outputTokens === undefined
228
+ ? undefined
229
+ : (generated.inputTokens ?? 0) + (generated.outputTokens ?? 0),
230
+ };
231
+ const costUsd = computeCostUsd({
232
+ inputTokens: generated.inputTokens,
233
+ modelId: params.modelId,
234
+ outputTokens: generated.outputTokens,
235
+ priceMap: params.priceMap,
236
+ });
237
+ return {
238
+ compiledMessages,
239
+ ...(costUsd === undefined ? {} : { costUsd }),
240
+ latencyMs: generated.latencyMs,
241
+ output: generated.output,
242
+ tokens,
243
+ };
244
+ }
245
+ async resolvePlaygroundVersion(promptId, version) {
246
+ const ObsPromptVersion = (0, obsPromptVersion_1.registerObsPromptVersion)();
247
+ if (version !== undefined) {
248
+ const match = await ObsPromptVersion.findOneOrNone({ promptId, version });
249
+ if (!match) {
250
+ throw new api_1.APIError({ status: 404, title: `Unknown version ${version}` });
251
+ }
252
+ return match;
253
+ }
254
+ const latestLabel = await (0, obsPromptLabel_1.registerObsPromptLabel)().findOneOrNone({
255
+ label: "latest",
256
+ promptId,
257
+ });
258
+ if (latestLabel) {
259
+ const labelled = await ObsPromptVersion.findOneOrNone({ _id: latestLabel.versionId });
260
+ if (labelled) {
261
+ return labelled;
262
+ }
263
+ }
264
+ const latestDocs = await ObsPromptVersion.find({ promptId }).sort({ version: -1 }).limit(1);
265
+ if (!latestDocs[0]) {
266
+ throw new api_1.APIError({ status: 404, title: "Prompt has no versions" });
267
+ }
268
+ return latestDocs[0];
269
+ }
270
+ async usageLast7d(names) {
271
+ const result = new Map();
272
+ for (const name of names) {
273
+ result.set(name, { calls: 0 });
274
+ }
275
+ if (names.length === 0) {
276
+ return result;
277
+ }
278
+ const since = luxon_1.DateTime.utc().minus({ days: 7 }).toJSDate();
279
+ const traces = await (0, obsTrace_1.registerObsTrace)().find({
280
+ created: { $gte: since },
281
+ "prompts.name": { $in: names },
282
+ });
283
+ for (const trace of traces) {
284
+ const seen = new Set();
285
+ for (const ref of trace.prompts) {
286
+ if (!names.includes(ref.name) || seen.has(ref.name)) {
287
+ continue;
288
+ }
289
+ seen.add(ref.name);
290
+ const current = result.get(ref.name) ?? { calls: 0 };
291
+ current.calls += 1;
292
+ if (trace.usage?.costUsd !== undefined) {
293
+ current.costUsd = (current.costUsd ?? 0) + trace.usage.costUsd;
294
+ }
295
+ result.set(ref.name, current);
296
+ }
297
+ }
298
+ return result;
299
+ }
300
+ async insertVersion(promptId, version, fields) {
301
+ return (0, obsPromptVersion_1.registerObsPromptVersion)().create({
302
+ config: fields.config,
303
+ inputSchema: fields.inputSchema,
304
+ outputFieldNotes: fields.outputFieldNotes,
305
+ outputSchema: fields.outputSchema,
306
+ promptId,
307
+ sensitive: fields.sensitive ?? false,
308
+ system: fields.system,
309
+ template: fields.template,
310
+ type: fields.type,
311
+ variables: fields.variables ?? [],
312
+ version,
313
+ });
314
+ }
315
+ async upsertLabel(promptId, label, versionId) {
316
+ const ObsPromptLabel = (0, obsPromptLabel_1.registerObsPromptLabel)();
317
+ const existing = await ObsPromptLabel.findOneOrNone({ label, promptId });
318
+ if (!existing) {
319
+ await ObsPromptLabel.create({ label, promptId, versionId });
320
+ return undefined;
321
+ }
322
+ const previous = await (0, obsPromptVersion_1.registerObsPromptVersion)().findOneOrNone({ _id: existing.versionId });
323
+ existing.versionId = versionId;
324
+ await existing.save();
325
+ return previous?.version;
326
+ }
327
+ async getVersionByLabel(name, label = "production") {
328
+ const prompt = await (0, obsPrompt_1.registerObsPrompt)().findOneOrNone({ name });
329
+ if (!prompt) {
330
+ return undefined;
331
+ }
332
+ const labelDoc = await (0, obsPromptLabel_1.registerObsPromptLabel)().findOneOrNone({
333
+ label,
334
+ promptId: prompt._id,
335
+ });
336
+ if (!labelDoc) {
337
+ return undefined;
338
+ }
339
+ const version = await (0, obsPromptVersion_1.registerObsPromptVersion)().findOneOrNone({ _id: labelDoc.versionId });
340
+ if (!version) {
341
+ return undefined;
342
+ }
343
+ return {
344
+ inputSchema: version.inputSchema,
345
+ outputSchema: version.outputSchema,
346
+ system: version.system,
347
+ template: version.template,
348
+ type: version.type,
349
+ variables: version.variables,
350
+ version: version.version,
351
+ };
352
+ }
353
+ async getVersionByNumber(name, versionNumber) {
354
+ const prompt = await (0, obsPrompt_1.registerObsPrompt)().findOneOrNone({ name });
355
+ if (!prompt) {
356
+ return undefined;
357
+ }
358
+ const version = await (0, obsPromptVersion_1.registerObsPromptVersion)().findOneOrNone({
359
+ promptId: prompt._id,
360
+ version: versionNumber,
361
+ });
362
+ if (!version) {
363
+ return undefined;
364
+ }
365
+ return {
366
+ inputSchema: version.inputSchema,
367
+ outputSchema: version.outputSchema,
368
+ system: version.system,
369
+ template: version.template,
370
+ type: version.type,
371
+ variables: version.variables,
372
+ version: version.version,
373
+ };
374
+ }
375
+ async loadRelationships(promptName, promptVersion) {
376
+ const traceFilter = promptVersion === undefined
377
+ ? { "prompts.name": promptName }
378
+ : { prompts: { $elemMatch: { name: promptName, version: promptVersion } } };
379
+ const ObsTrace = (0, obsTrace_1.registerObsTrace)();
380
+ const totalTraces = await ObsTrace.countDocuments(traceFilter);
381
+ const traceRows = await ObsTrace.find(traceFilter)
382
+ .sort({ created: -1 })
383
+ .limit(PROMPT_RELATIONSHIP_LIMIT);
384
+ const traceItems = [];
385
+ for (const row of traceRows) {
386
+ const ref = row.prompts.find((promptRef) => {
387
+ return promptRef.name === promptName;
388
+ });
389
+ if (!ref) {
390
+ continue;
391
+ }
392
+ traceItems.push({
393
+ id: String(row._id),
394
+ name: row.name,
395
+ promptName: ref.name,
396
+ promptVersion: ref.version,
397
+ sensitive: row.sensitive,
398
+ startedAt: luxon_1.DateTime.fromJSDate(row.startedAt).toUTC().toISO() ?? "",
399
+ status: row.status,
400
+ });
401
+ }
402
+ const experimentFilter = { promptName };
403
+ const ObsExperiment = (0, obsExperiment_1.registerObsExperiment)();
404
+ const totalExperiments = await ObsExperiment.countDocuments(experimentFilter);
405
+ const experimentRows = await ObsExperiment.find(experimentFilter)
406
+ .sort({ created: -1 })
407
+ .limit(PROMPT_RELATIONSHIP_LIMIT);
408
+ return {
409
+ experiments: {
410
+ items: experimentRows.map((row) => {
411
+ return {
412
+ created: luxon_1.DateTime.fromJSDate(row.created).toUTC().toISO() ?? "",
413
+ id: String(row._id),
414
+ name: row.name,
415
+ promptName: row.promptName,
416
+ status: row.status,
417
+ versions: row.versions,
418
+ };
419
+ }),
420
+ limit: PROMPT_RELATIONSHIP_LIMIT,
421
+ total: totalExperiments,
422
+ },
423
+ traces: {
424
+ items: traceItems,
425
+ limit: PROMPT_RELATIONSHIP_LIMIT,
426
+ total: totalTraces,
427
+ },
428
+ };
429
+ }
430
+ async requirePrompt(name) {
431
+ const prompt = await (0, obsPrompt_1.registerObsPrompt)().findOneOrNone({ name });
432
+ if (!prompt) {
433
+ throw new api_1.APIError({ status: 404, title: `Unknown prompt "${name}"` });
434
+ }
435
+ return prompt;
436
+ }
437
+ isDuplicateKey(error) {
438
+ return Boolean(error &&
439
+ typeof error === "object" &&
440
+ "code" in error &&
441
+ error.code === 11000);
442
+ }
443
+ }
444
+ exports.LocalPromptStore = LocalPromptStore;
@@ -0,0 +1,50 @@
1
+ import type { EvaluatorDimension } from "../../types/observability";
2
+ import { type ReviewPanels } from "../reviewPanels";
3
+ import type { ScoreSink } from "../types";
4
+ export type ReviewStatus = "done" | "in_progress" | "pending" | "skipped";
5
+ export interface ReviewListItem {
6
+ assigneeId?: string;
7
+ enqueuedAt: string;
8
+ evaluatorId: string;
9
+ id: string;
10
+ promptName?: string;
11
+ reason: string;
12
+ status: ReviewStatus;
13
+ traceId: string;
14
+ traceName: string;
15
+ }
16
+ export declare class LocalReviewStore {
17
+ enqueue(params: {
18
+ evaluatorId: string;
19
+ reason?: "dataset_candidate" | "eval" | "feedback" | "manual";
20
+ spanId?: string;
21
+ traceIds: string[];
22
+ }): Promise<ReviewListItem[]>;
23
+ list(status?: ReviewStatus): Promise<{
24
+ counts: Record<ReviewStatus, number>;
25
+ data: ReviewListItem[];
26
+ }>;
27
+ getDetail(id: string): Promise<{
28
+ comment?: string;
29
+ dimensions: EvaluatorDimension[];
30
+ evaluatorId: string;
31
+ id: string;
32
+ instructions?: string;
33
+ panels: ReviewPanels;
34
+ rawInput?: unknown;
35
+ rawOutput?: unknown;
36
+ scores?: Record<string, boolean | number | string>;
37
+ status: ReviewStatus;
38
+ traceId: string;
39
+ }>;
40
+ submit(params: {
41
+ comment?: string;
42
+ id: string;
43
+ scores: Record<string, boolean | number | string>;
44
+ sinks: ScoreSink[];
45
+ }): Promise<void>;
46
+ skip(id: string): Promise<void>;
47
+ assign(id: string, assigneeId: string): Promise<void>;
48
+ private requireItem;
49
+ private toListItem;
50
+ }
@@ -0,0 +1,203 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.LocalReviewStore = void 0;
7
+ const api_1 = require("@terreno/api");
8
+ const luxon_1 = require("luxon");
9
+ const mongoose_1 = __importDefault(require("mongoose"));
10
+ const reviewPanels_1 = require("../reviewPanels");
11
+ const obsEvaluator_1 = require("./models/obsEvaluator");
12
+ const obsPrompt_1 = require("./models/obsPrompt");
13
+ const obsPromptVersion_1 = require("./models/obsPromptVersion");
14
+ const obsReviewItem_1 = require("./models/obsReviewItem");
15
+ const obsTrace_1 = require("./models/obsTrace");
16
+ const traceStore_1 = require("./traceStore");
17
+ const objectId = (value, label) => {
18
+ if (!mongoose_1.default.Types.ObjectId.isValid(value)) {
19
+ throw new api_1.APIError({ status: 400, title: `${label} must be an object id` });
20
+ }
21
+ return new mongoose_1.default.Types.ObjectId(value);
22
+ };
23
+ const toIso = (value) => {
24
+ return luxon_1.DateTime.fromJSDate(value).toUTC().toISO() ?? "";
25
+ };
26
+ class LocalReviewStore {
27
+ async enqueue(params) {
28
+ if (params.traceIds.length === 0) {
29
+ throw new api_1.APIError({ status: 400, title: "traceIds are required" });
30
+ }
31
+ const evaluator = await (0, obsEvaluator_1.registerObsEvaluator)().findOneOrNone({
32
+ _id: objectId(params.evaluatorId, "evaluatorId"),
33
+ });
34
+ if (!evaluator) {
35
+ throw new api_1.APIError({ status: 404, title: "Unknown evaluator" });
36
+ }
37
+ if (evaluator.type !== "human") {
38
+ throw new api_1.APIError({
39
+ status: 400,
40
+ title: "Human review requires a human evaluator",
41
+ });
42
+ }
43
+ const created = [];
44
+ for (const traceId of params.traceIds) {
45
+ const trace = await (0, obsTrace_1.registerObsTrace)().findOneOrNone({ _id: objectId(traceId, "traceId") });
46
+ if (!trace) {
47
+ throw new api_1.APIError({ status: 404, title: `Unknown trace "${traceId}"` });
48
+ }
49
+ const item = await (0, obsReviewItem_1.registerObsReviewItem)().create({
50
+ enqueuedAt: luxon_1.DateTime.utc().toJSDate(),
51
+ evaluatorId: evaluator._id,
52
+ reason: params.reason ?? "manual",
53
+ status: "pending",
54
+ traceId: trace._id,
55
+ ...(params.spanId ? { spanId: objectId(params.spanId, "spanId") } : {}),
56
+ });
57
+ created.push(this.toListItem(item, trace.name, trace.prompts[0]?.name));
58
+ }
59
+ return created;
60
+ }
61
+ async list(status) {
62
+ const ObsReviewItem = (0, obsReviewItem_1.registerObsReviewItem)();
63
+ const counts = {
64
+ done: await ObsReviewItem.countDocuments({ status: "done" }),
65
+ in_progress: await ObsReviewItem.countDocuments({ status: "in_progress" }),
66
+ pending: await ObsReviewItem.countDocuments({ status: "pending" }),
67
+ skipped: await ObsReviewItem.countDocuments({ status: "skipped" }),
68
+ };
69
+ const filter = status ? { status } : {};
70
+ const rows = await ObsReviewItem.find(filter).sort({ enqueuedAt: 1 });
71
+ const data = [];
72
+ for (const row of rows) {
73
+ const trace = await (0, obsTrace_1.registerObsTrace)().findOneOrNone({ _id: row.traceId });
74
+ data.push(this.toListItem(row, trace?.name ?? "Unknown trace", trace?.prompts[0]?.name));
75
+ }
76
+ return { counts, data };
77
+ }
78
+ async getDetail(id) {
79
+ const item = await (0, obsReviewItem_1.registerObsReviewItem)().findOneOrNone({ _id: objectId(id, "id") });
80
+ if (!item) {
81
+ throw new api_1.APIError({ status: 404, title: `Unknown review item "${id}"` });
82
+ }
83
+ const evaluator = await (0, obsEvaluator_1.registerObsEvaluator)().findOneOrNone({ _id: item.evaluatorId });
84
+ if (!evaluator) {
85
+ throw new api_1.APIError({ status: 404, title: "Unknown evaluator" });
86
+ }
87
+ const trace = await (0, obsTrace_1.registerObsTrace)().findOneOrNone({ _id: item.traceId });
88
+ const promptRef = trace?.prompts[0];
89
+ let variables;
90
+ let outputFieldNotes;
91
+ let outputSchema;
92
+ if (promptRef) {
93
+ const prompt = await (0, obsPrompt_1.registerObsPrompt)().findOneOrNone({ name: promptRef.name });
94
+ if (prompt) {
95
+ const version = await (0, obsPromptVersion_1.registerObsPromptVersion)().findOneOrNone({
96
+ promptId: prompt._id,
97
+ version: promptRef.version,
98
+ });
99
+ variables = version?.variables;
100
+ outputFieldNotes = version?.outputFieldNotes;
101
+ outputSchema = version?.outputSchema;
102
+ }
103
+ }
104
+ return {
105
+ comment: item.comment,
106
+ dimensions: evaluator.dimensions,
107
+ evaluatorId: String(evaluator._id),
108
+ id: String(item._id),
109
+ instructions: evaluator.instructions,
110
+ panels: (0, reviewPanels_1.buildReviewPanels)({
111
+ input: trace?.input,
112
+ output: trace?.output,
113
+ outputFieldNotes,
114
+ outputSchema,
115
+ variables,
116
+ }),
117
+ rawInput: trace?.input,
118
+ rawOutput: trace?.output,
119
+ scores: item.scores,
120
+ status: item.status,
121
+ traceId: String(item.traceId),
122
+ };
123
+ }
124
+ async submit(params) {
125
+ const item = await this.requireItem(params.id);
126
+ const evaluator = await (0, obsEvaluator_1.registerObsEvaluator)().findOneOrNone({ _id: item.evaluatorId });
127
+ if (!evaluator) {
128
+ throw new api_1.APIError({ status: 404, title: "Unknown evaluator" });
129
+ }
130
+ const missing = evaluator.dimensions.find((dimension) => {
131
+ return dimension.required && params.scores[dimension.key] === undefined;
132
+ });
133
+ if (missing) {
134
+ throw new api_1.APIError({ status: 400, title: `Score "${missing.key}" is required` });
135
+ }
136
+ const records = [];
137
+ for (const dimension of evaluator.dimensions) {
138
+ const value = params.scores[dimension.key];
139
+ if (value === undefined) {
140
+ continue;
141
+ }
142
+ records.push({
143
+ comment: params.comment,
144
+ dataType: dimension.dataType,
145
+ evaluatorId: String(evaluator._id),
146
+ name: dimension.key,
147
+ source: "human",
148
+ traceId: String(item.traceId),
149
+ value,
150
+ });
151
+ }
152
+ for (const record of records) {
153
+ if (params.sinks.length === 0) {
154
+ await new traceStore_1.LocalTraceStore().exportScore(record);
155
+ continue;
156
+ }
157
+ const results = await Promise.allSettled(params.sinks.map((sink) => {
158
+ return sink.export(record);
159
+ }));
160
+ for (const result of results) {
161
+ if (result.status === "rejected") {
162
+ throw result.reason;
163
+ }
164
+ }
165
+ }
166
+ item.comment = params.comment;
167
+ item.scores = params.scores;
168
+ item.status = "done";
169
+ await item.save();
170
+ }
171
+ async skip(id) {
172
+ const item = await this.requireItem(id);
173
+ item.status = "skipped";
174
+ await item.save();
175
+ }
176
+ async assign(id, assigneeId) {
177
+ const item = await this.requireItem(id);
178
+ item.assigneeId = objectId(assigneeId, "assigneeId");
179
+ item.status = "in_progress";
180
+ await item.save();
181
+ }
182
+ async requireItem(id) {
183
+ const item = await (0, obsReviewItem_1.registerObsReviewItem)().findOneOrNone({ _id: objectId(id, "id") });
184
+ if (!item) {
185
+ throw new api_1.APIError({ status: 404, title: `Unknown review item "${id}"` });
186
+ }
187
+ return item;
188
+ }
189
+ toListItem(row, traceName, promptName) {
190
+ return {
191
+ assigneeId: row.assigneeId ? String(row.assigneeId) : undefined,
192
+ enqueuedAt: toIso(row.enqueuedAt),
193
+ evaluatorId: String(row.evaluatorId),
194
+ id: String(row._id),
195
+ promptName,
196
+ reason: row.reason,
197
+ status: row.status,
198
+ traceId: String(row.traceId),
199
+ traceName,
200
+ };
201
+ }
202
+ }
203
+ exports.LocalReviewStore = LocalReviewStore;