@terreno/ai 57.9.0 → 57.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (417) hide show
  1. package/README.md +1 -0
  2. package/bunfig.toml +1 -1
  3. package/dist/aiApp.d.ts +8 -2
  4. package/dist/aiApp.js +12 -6
  5. package/dist/harness/adminScreens.d.ts +6 -0
  6. package/dist/harness/adminScreens.js +11 -0
  7. package/dist/harness/agentLoop.d.ts +61 -0
  8. package/dist/harness/agentLoop.js +798 -0
  9. package/dist/harness/agentTaskNames.d.ts +4 -0
  10. package/dist/harness/agentTaskNames.js +7 -0
  11. package/dist/harness/approvalPolicy.d.ts +3 -0
  12. package/dist/harness/approvalPolicy.js +23 -0
  13. package/dist/harness/approvals.d.ts +101 -0
  14. package/dist/harness/approvals.js +468 -0
  15. package/dist/harness/commit.d.ts +238 -0
  16. package/dist/harness/commit.js +678 -0
  17. package/dist/harness/conversation.d.ts +75 -0
  18. package/dist/harness/conversation.js +397 -0
  19. package/dist/harness/defineAgent.d.ts +9 -0
  20. package/dist/harness/defineAgent.js +65 -0
  21. package/dist/harness/defineTask.d.ts +11 -0
  22. package/dist/harness/defineTask.js +59 -0
  23. package/dist/harness/defineTool.d.ts +7 -0
  24. package/dist/harness/defineTool.js +36 -0
  25. package/dist/harness/definitionError.d.ts +8 -0
  26. package/dist/harness/definitionError.js +15 -0
  27. package/dist/harness/errors.d.ts +133 -0
  28. package/dist/harness/errors.js +107 -0
  29. package/dist/harness/eventHub.d.ts +45 -0
  30. package/dist/harness/eventHub.js +135 -0
  31. package/dist/harness/events.d.ts +95 -0
  32. package/dist/harness/events.js +267 -0
  33. package/dist/harness/executionEnv.d.ts +31 -0
  34. package/dist/harness/executionEnv.js +2 -0
  35. package/dist/harness/extensions.d.ts +82 -0
  36. package/dist/harness/extensions.js +274 -0
  37. package/dist/harness/harness.d.ts +163 -0
  38. package/dist/harness/harness.js +522 -0
  39. package/dist/harness/harnessApp.d.ts +44 -0
  40. package/dist/harness/harnessApp.js +62 -0
  41. package/dist/harness/internalRuntime.d.ts +16 -0
  42. package/dist/harness/internalRuntime.js +18 -0
  43. package/dist/harness/leases.d.ts +35 -0
  44. package/dist/harness/leases.js +86 -0
  45. package/dist/harness/memo.d.ts +24 -0
  46. package/dist/harness/memo.js +78 -0
  47. package/dist/harness/modelCall.d.ts +45 -0
  48. package/dist/harness/modelCall.js +147 -0
  49. package/dist/harness/models/harnessApproval.d.ts +2 -0
  50. package/dist/harness/models/harnessApproval.js +100 -0
  51. package/dist/harness/models/harnessConversation.d.ts +2 -0
  52. package/dist/harness/models/harnessConversation.js +117 -0
  53. package/dist/harness/models/harnessEvent.d.ts +3 -0
  54. package/dist/harness/models/harnessEvent.js +73 -0
  55. package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
  56. package/dist/harness/models/harnessInboxEvent.js +58 -0
  57. package/dist/harness/models/harnessMemo.d.ts +2 -0
  58. package/dist/harness/models/harnessMemo.js +42 -0
  59. package/dist/harness/models/harnessMessage.d.ts +2 -0
  60. package/dist/harness/models/harnessMessage.js +73 -0
  61. package/dist/harness/models/harnessOwner.d.ts +2 -0
  62. package/dist/harness/models/harnessOwner.js +36 -0
  63. package/dist/harness/models/harnessTask.d.ts +2 -0
  64. package/dist/harness/models/harnessTask.js +191 -0
  65. package/dist/harness/ownership.d.ts +89 -0
  66. package/dist/harness/ownership.js +438 -0
  67. package/dist/harness/registry.d.ts +29 -0
  68. package/dist/harness/registry.js +93 -0
  69. package/dist/harness/retryBackoff.d.ts +31 -0
  70. package/dist/harness/retryBackoff.js +50 -0
  71. package/dist/harness/routes/approvals.d.ts +13 -0
  72. package/dist/harness/routes/approvals.js +93 -0
  73. package/dist/harness/routes/conversations.d.ts +11 -0
  74. package/dist/harness/routes/conversations.js +66 -0
  75. package/dist/harness/routes/events.d.ts +19 -0
  76. package/dist/harness/routes/events.js +215 -0
  77. package/dist/harness/routes/tasks.d.ts +11 -0
  78. package/dist/harness/routes/tasks.js +108 -0
  79. package/dist/harness/runners/inProcessRunner.d.ts +64 -0
  80. package/dist/harness/runners/inProcessRunner.js +227 -0
  81. package/dist/harness/runners/jobsRunner.d.ts +62 -0
  82. package/dist/harness/runners/jobsRunner.js +212 -0
  83. package/dist/harness/runtime.d.ts +52 -0
  84. package/dist/harness/runtime.js +604 -0
  85. package/dist/harness/subagent.d.ts +53 -0
  86. package/dist/harness/subagent.js +134 -0
  87. package/dist/harness/suspend.d.ts +8 -0
  88. package/dist/harness/suspend.js +16 -0
  89. package/dist/harness/transaction.d.ts +3 -0
  90. package/dist/harness/transaction.js +22 -0
  91. package/dist/harness/waits.d.ts +100 -0
  92. package/dist/harness/waits.js +214 -0
  93. package/dist/index.d.ts +19 -1
  94. package/dist/index.js +45 -2
  95. package/dist/models/aiDataset.d.ts +2 -0
  96. package/dist/models/aiDataset.js +67 -0
  97. package/dist/models/aiRequest.js +1 -1
  98. package/dist/models/gptHistory.js +73 -0
  99. package/dist/observability/adminScreens.d.ts +5 -0
  100. package/dist/observability/adminScreens.js +32 -0
  101. package/dist/observability/compileTemplate.d.ts +1 -0
  102. package/dist/observability/compileTemplate.js +13 -0
  103. package/dist/observability/datasetImport.d.ts +11 -0
  104. package/dist/observability/datasetImport.js +185 -0
  105. package/dist/observability/evaluate.d.ts +31 -0
  106. package/dist/observability/evaluate.js +200 -0
  107. package/dist/observability/evaluatorTemplates.d.ts +17 -0
  108. package/dist/observability/evaluatorTemplates.js +111 -0
  109. package/dist/observability/jsonPath.d.ts +2 -0
  110. package/dist/observability/jsonPath.js +57 -0
  111. package/dist/observability/local/datasetStore.d.ts +87 -0
  112. package/dist/observability/local/datasetStore.js +326 -0
  113. package/dist/observability/local/evaluatorStore.d.ts +44 -0
  114. package/dist/observability/local/evaluatorStore.js +182 -0
  115. package/dist/observability/local/experimentRunner.d.ts +96 -0
  116. package/dist/observability/local/experimentRunner.js +419 -0
  117. package/dist/observability/local/localPlugin.d.ts +22 -0
  118. package/dist/observability/local/localPlugin.js +76 -0
  119. package/dist/observability/local/models/obsDataset.d.ts +2 -0
  120. package/dist/observability/local/models/obsDataset.js +37 -0
  121. package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
  122. package/dist/observability/local/models/obsDatasetItem.js +76 -0
  123. package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
  124. package/dist/observability/local/models/obsEvaluator.js +98 -0
  125. package/dist/observability/local/models/obsExperiment.d.ts +2 -0
  126. package/dist/observability/local/models/obsExperiment.js +87 -0
  127. package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
  128. package/dist/observability/local/models/obsExperimentItem.js +45 -0
  129. package/dist/observability/local/models/obsPrompt.d.ts +2 -0
  130. package/dist/observability/local/models/obsPrompt.js +42 -0
  131. package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
  132. package/dist/observability/local/models/obsPromptLabel.js +39 -0
  133. package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
  134. package/dist/observability/local/models/obsPromptVersion.js +96 -0
  135. package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
  136. package/dist/observability/local/models/obsReviewItem.js +75 -0
  137. package/dist/observability/local/models/obsScore.d.ts +2 -0
  138. package/dist/observability/local/models/obsScore.js +58 -0
  139. package/dist/observability/local/models/obsSpan.d.ts +2 -0
  140. package/dist/observability/local/models/obsSpan.js +64 -0
  141. package/dist/observability/local/models/obsTrace.d.ts +2 -0
  142. package/dist/observability/local/models/obsTrace.js +93 -0
  143. package/dist/observability/local/promptStore.d.ts +180 -0
  144. package/dist/observability/local/promptStore.js +444 -0
  145. package/dist/observability/local/reviewStore.d.ts +50 -0
  146. package/dist/observability/local/reviewStore.js +203 -0
  147. package/dist/observability/local/traceStore.d.ts +103 -0
  148. package/dist/observability/local/traceStore.js +325 -0
  149. package/dist/observability/observabilityApp.d.ts +24 -0
  150. package/dist/observability/observabilityApp.js +157 -0
  151. package/dist/observability/observabilityAppRegistry.d.ts +14 -0
  152. package/dist/observability/observabilityAppRegistry.js +16 -0
  153. package/dist/observability/permissions.d.ts +7 -0
  154. package/dist/observability/permissions.js +60 -0
  155. package/dist/observability/reviewPanels.d.ts +18 -0
  156. package/dist/observability/reviewPanels.js +46 -0
  157. package/dist/observability/routes/datasets.d.ts +7 -0
  158. package/dist/observability/routes/datasets.js +149 -0
  159. package/dist/observability/routes/evaluators.d.ts +7 -0
  160. package/dist/observability/routes/evaluators.js +123 -0
  161. package/dist/observability/routes/experiments.d.ts +7 -0
  162. package/dist/observability/routes/experiments.js +95 -0
  163. package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
  164. package/dist/observability/routes/observabilityRouteAccess.js +67 -0
  165. package/dist/observability/routes/prompts.d.ts +12 -0
  166. package/dist/observability/routes/prompts.js +172 -0
  167. package/dist/observability/routes/requestAiService.d.ts +13 -0
  168. package/dist/observability/routes/requestAiService.js +21 -0
  169. package/dist/observability/routes/review.d.ts +7 -0
  170. package/dist/observability/routes/review.js +79 -0
  171. package/dist/observability/routes/status.d.ts +5 -0
  172. package/dist/observability/routes/status.js +31 -0
  173. package/dist/observability/routes/testMultiStage.d.ts +9 -0
  174. package/dist/observability/routes/testMultiStage.js +67 -0
  175. package/dist/observability/routes/traces.d.ts +7 -0
  176. package/dist/observability/routes/traces.js +129 -0
  177. package/dist/observability/schemaValidation.d.ts +5 -0
  178. package/dist/observability/schemaValidation.js +26 -0
  179. package/dist/observability/sopThresholds.d.ts +2 -0
  180. package/dist/observability/sopThresholds.js +21 -0
  181. package/dist/observability/status.d.ts +23 -0
  182. package/dist/observability/status.js +34 -0
  183. package/dist/observability/testMultiStageSchemas.d.ts +10 -0
  184. package/dist/observability/testMultiStageSchemas.js +79 -0
  185. package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
  186. package/dist/observability/testMultiStageWorkflow.js +218 -0
  187. package/dist/observability/types.d.ts +159 -0
  188. package/dist/observability/types.js +67 -0
  189. package/dist/routes/files.js +18 -7
  190. package/dist/routes/gpt.js +79 -689
  191. package/dist/routes/gptActions.d.ts +9 -0
  192. package/dist/routes/gptActions.js +212 -0
  193. package/dist/routes/gptDatasets.d.ts +5 -0
  194. package/dist/routes/gptDatasets.js +94 -0
  195. package/dist/routes/gptHistories.d.ts +3 -1
  196. package/dist/routes/gptHistories.js +188 -2
  197. package/dist/service/aiDatasets.d.ts +57 -0
  198. package/dist/service/aiDatasets.js +253 -0
  199. package/dist/service/aiService.d.ts +50 -1
  200. package/dist/service/aiService.js +426 -77
  201. package/dist/service/askFiles.d.ts +49 -0
  202. package/dist/service/askFiles.js +162 -0
  203. package/dist/service/asks.d.ts +76 -0
  204. package/dist/service/asks.js +284 -0
  205. package/dist/service/chatTurn.d.ts +128 -0
  206. package/dist/service/chatTurn.js +1674 -0
  207. package/dist/service/fileStorage.d.ts +4 -0
  208. package/dist/service/fileStorage.js +7 -2
  209. package/dist/service/fillActionIds.d.ts +6 -0
  210. package/dist/service/fillActionIds.js +79 -0
  211. package/dist/service/jsonFenceModel.d.ts +7 -0
  212. package/dist/service/jsonFenceModel.js +45 -0
  213. package/dist/service/prompts.d.ts +52 -0
  214. package/dist/service/prompts.js +96 -1
  215. package/dist/service/sanitizeHtml.d.ts +11 -0
  216. package/dist/service/sanitizeHtml.js +108 -0
  217. package/dist/types/harness.d.ts +952 -0
  218. package/dist/types/harness.js +133 -0
  219. package/dist/types/index.d.ts +192 -14
  220. package/dist/types/index.js +2 -0
  221. package/dist/types/observability.d.ts +288 -0
  222. package/dist/types/observability.js +2 -0
  223. package/package.json +28 -5
  224. package/src/aiApp.test.ts +207 -1
  225. package/src/aiApp.ts +19 -6
  226. package/src/harness/adminScreens.test.ts +34 -0
  227. package/src/harness/adminScreens.ts +11 -0
  228. package/src/harness/agentLoop.test.ts +1034 -0
  229. package/src/harness/agentLoop.ts +1082 -0
  230. package/src/harness/agentRecovery.test.ts +380 -0
  231. package/src/harness/agentTaskNames.ts +4 -0
  232. package/src/harness/approvalPolicy.ts +28 -0
  233. package/src/harness/approvals.test.ts +1094 -0
  234. package/src/harness/approvals.ts +644 -0
  235. package/src/harness/commit.ts +1049 -0
  236. package/src/harness/concurrency.test.ts +155 -0
  237. package/src/harness/conversation.ts +503 -0
  238. package/src/harness/defineAgent.test.ts +92 -0
  239. package/src/harness/defineAgent.ts +82 -0
  240. package/src/harness/defineTask.test.ts +59 -0
  241. package/src/harness/defineTask.ts +76 -0
  242. package/src/harness/defineTool.ts +45 -0
  243. package/src/harness/definitionError.ts +13 -0
  244. package/src/harness/errors.test.ts +90 -0
  245. package/src/harness/errors.ts +125 -0
  246. package/src/harness/eventHub.test.ts +124 -0
  247. package/src/harness/eventHub.ts +159 -0
  248. package/src/harness/events.test.ts +769 -0
  249. package/src/harness/events.ts +384 -0
  250. package/src/harness/executionEnv.ts +34 -0
  251. package/src/harness/extensions.test.ts +1218 -0
  252. package/src/harness/extensions.ts +382 -0
  253. package/src/harness/harness.test.ts +693 -0
  254. package/src/harness/harness.ts +778 -0
  255. package/src/harness/harnessApp.ts +76 -0
  256. package/src/harness/internalRuntime.ts +44 -0
  257. package/src/harness/leases.ts +132 -0
  258. package/src/harness/memo.ts +107 -0
  259. package/src/harness/modelCall.test.ts +134 -0
  260. package/src/harness/modelCall.ts +200 -0
  261. package/src/harness/models/harnessApproval.ts +109 -0
  262. package/src/harness/models/harnessConversation.ts +142 -0
  263. package/src/harness/models/harnessEvent.ts +92 -0
  264. package/src/harness/models/harnessInboxEvent.ts +73 -0
  265. package/src/harness/models/harnessMemo.ts +43 -0
  266. package/src/harness/models/harnessMessage.ts +85 -0
  267. package/src/harness/models/harnessModels.test.ts +60 -0
  268. package/src/harness/models/harnessOwner.ts +40 -0
  269. package/src/harness/models/harnessTask.ts +206 -0
  270. package/src/harness/ownership.test.ts +1087 -0
  271. package/src/harness/ownership.ts +618 -0
  272. package/src/harness/recovery.test.ts +907 -0
  273. package/src/harness/registry.ts +124 -0
  274. package/src/harness/retries.test.ts +292 -0
  275. package/src/harness/retryBackoff.test.ts +100 -0
  276. package/src/harness/retryBackoff.ts +77 -0
  277. package/src/harness/routes/approvals.ts +123 -0
  278. package/src/harness/routes/conversations.test.ts +518 -0
  279. package/src/harness/routes/conversations.ts +102 -0
  280. package/src/harness/routes/events.ts +280 -0
  281. package/src/harness/routes/tasks.test.ts +198 -0
  282. package/src/harness/routes/tasks.ts +146 -0
  283. package/src/harness/runners/inProcessRunner.test.ts +530 -0
  284. package/src/harness/runners/inProcessRunner.ts +260 -0
  285. package/src/harness/runners/jobsRunner.test.ts +709 -0
  286. package/src/harness/runners/jobsRunner.ts +252 -0
  287. package/src/harness/runtime.ts +802 -0
  288. package/src/harness/subagent.test.ts +696 -0
  289. package/src/harness/subagent.ts +192 -0
  290. package/src/harness/suspend.ts +13 -0
  291. package/src/harness/transaction.ts +17 -0
  292. package/src/harness/versioning.test.ts +288 -0
  293. package/src/harness/waits.test.ts +693 -0
  294. package/src/harness/waits.ts +356 -0
  295. package/src/index.test.ts +13 -0
  296. package/src/index.ts +70 -0
  297. package/src/isolated/adminScreensConfig.isolated.ts +143 -0
  298. package/src/models/aiDataset.test.ts +34 -0
  299. package/src/models/aiDataset.ts +78 -0
  300. package/src/models/aiRequest.ts +2 -1
  301. package/src/models/gptHistory.test.ts +156 -0
  302. package/src/models/gptHistory.ts +89 -1
  303. package/src/observability/adminScreens.test.ts +71 -0
  304. package/src/observability/adminScreens.ts +42 -0
  305. package/src/observability/compileTemplate.test.ts +14 -0
  306. package/src/observability/compileTemplate.ts +10 -0
  307. package/src/observability/datasetImport.test.ts +95 -0
  308. package/src/observability/datasetImport.ts +220 -0
  309. package/src/observability/evaluate.test.ts +549 -0
  310. package/src/observability/evaluate.ts +254 -0
  311. package/src/observability/evaluatorTemplates.ts +124 -0
  312. package/src/observability/jsonPath.test.ts +52 -0
  313. package/src/observability/jsonPath.ts +60 -0
  314. package/src/observability/local/datasetStore.test.ts +230 -0
  315. package/src/observability/local/datasetStore.ts +452 -0
  316. package/src/observability/local/evaluatorStore.test.ts +138 -0
  317. package/src/observability/local/evaluatorStore.ts +224 -0
  318. package/src/observability/local/experimentRunner.test.ts +489 -0
  319. package/src/observability/local/experimentRunner.ts +548 -0
  320. package/src/observability/local/localPlugin.ts +88 -0
  321. package/src/observability/local/models/obsDataset.ts +38 -0
  322. package/src/observability/local/models/obsDatasetItem.ts +80 -0
  323. package/src/observability/local/models/obsEvaluator.ts +102 -0
  324. package/src/observability/local/models/obsExperiment.ts +91 -0
  325. package/src/observability/local/models/obsExperimentItem.ts +52 -0
  326. package/src/observability/local/models/obsPrompt.ts +43 -0
  327. package/src/observability/local/models/obsPromptLabel.ts +43 -0
  328. package/src/observability/local/models/obsPromptVersion.ts +107 -0
  329. package/src/observability/local/models/obsReviewItem.ts +79 -0
  330. package/src/observability/local/models/obsScore.ts +59 -0
  331. package/src/observability/local/models/obsSpan.ts +65 -0
  332. package/src/observability/local/models/obsTrace.ts +94 -0
  333. package/src/observability/local/obsModels.test.ts +119 -0
  334. package/src/observability/local/promptStore.test.ts +351 -0
  335. package/src/observability/local/promptStore.ts +665 -0
  336. package/src/observability/local/reviewStore.test.ts +164 -0
  337. package/src/observability/local/reviewStore.ts +264 -0
  338. package/src/observability/local/traceStore.test.ts +264 -0
  339. package/src/observability/local/traceStore.ts +440 -0
  340. package/src/observability/observabilityApp.test.ts +92 -0
  341. package/src/observability/observabilityApp.ts +185 -0
  342. package/src/observability/observabilityAppRegistry.test.ts +30 -0
  343. package/src/observability/observabilityAppRegistry.ts +35 -0
  344. package/src/observability/permissions.test.ts +96 -0
  345. package/src/observability/permissions.ts +78 -0
  346. package/src/observability/reviewPanels.test.ts +35 -0
  347. package/src/observability/reviewPanels.ts +64 -0
  348. package/src/observability/routes/datasets.test.ts +176 -0
  349. package/src/observability/routes/datasets.ts +252 -0
  350. package/src/observability/routes/evaluators.test.ts +119 -0
  351. package/src/observability/routes/evaluators.ts +207 -0
  352. package/src/observability/routes/experiments.test.ts +282 -0
  353. package/src/observability/routes/experiments.ts +152 -0
  354. package/src/observability/routes/observabilityRbac.test.ts +767 -0
  355. package/src/observability/routes/observabilityRouteAccess.ts +106 -0
  356. package/src/observability/routes/prompts.test.ts +320 -0
  357. package/src/observability/routes/prompts.ts +263 -0
  358. package/src/observability/routes/requestAiService.ts +36 -0
  359. package/src/observability/routes/review.test.ts +120 -0
  360. package/src/observability/routes/review.ts +125 -0
  361. package/src/observability/routes/status.test.ts +116 -0
  362. package/src/observability/routes/status.ts +47 -0
  363. package/src/observability/routes/testMultiStage.test.ts +214 -0
  364. package/src/observability/routes/testMultiStage.ts +96 -0
  365. package/src/observability/routes/traces.test.ts +167 -0
  366. package/src/observability/routes/traces.ts +172 -0
  367. package/src/observability/schemaValidation.ts +29 -0
  368. package/src/observability/sopThresholds.ts +20 -0
  369. package/src/observability/status.test.ts +38 -0
  370. package/src/observability/status.ts +56 -0
  371. package/src/observability/testMultiStageSchemas.ts +95 -0
  372. package/src/observability/testMultiStageWorkflow.test.ts +114 -0
  373. package/src/observability/testMultiStageWorkflow.ts +307 -0
  374. package/src/observability/types.ts +262 -0
  375. package/src/routes/files.test.ts +63 -9
  376. package/src/routes/files.ts +21 -7
  377. package/src/routes/gpt.test.ts +3279 -0
  378. package/src/routes/gpt.ts +91 -816
  379. package/src/routes/gptActions.test.ts +241 -0
  380. package/src/routes/gptActions.ts +255 -0
  381. package/src/routes/gptApprovals.test.ts +762 -0
  382. package/src/routes/gptDatasets.test.ts +86 -0
  383. package/src/routes/gptDatasets.ts +104 -0
  384. package/src/routes/gptFiles.test.ts +589 -0
  385. package/src/routes/gptHistories.test.ts +933 -0
  386. package/src/routes/gptHistories.ts +266 -5
  387. package/src/routes/routes.test.ts +222 -2
  388. package/src/service/aiDatasets.test.ts +123 -0
  389. package/src/service/aiDatasets.ts +316 -0
  390. package/src/service/aiService.test.ts +593 -0
  391. package/src/service/aiService.ts +568 -111
  392. package/src/service/askFiles.test.ts +96 -0
  393. package/src/service/askFiles.ts +222 -0
  394. package/src/service/asks.test.ts +946 -0
  395. package/src/service/asks.ts +423 -0
  396. package/src/service/chatTurn.ts +2287 -0
  397. package/src/service/fileStorage.test.ts +11 -0
  398. package/src/service/fileStorage.ts +10 -2
  399. package/src/service/fillActionIds.test.ts +65 -0
  400. package/src/service/fillActionIds.ts +98 -0
  401. package/src/service/jsonFenceModel.test.ts +47 -0
  402. package/src/service/jsonFenceModel.ts +53 -0
  403. package/src/service/prompts.test.ts +21 -0
  404. package/src/service/prompts.ts +147 -0
  405. package/src/service/sanitizeHtml.test.ts +61 -0
  406. package/src/service/sanitizeHtml.ts +109 -0
  407. package/src/tests/bunSetup.test.ts +7 -4
  408. package/src/tests/bunSetup.ts +7 -7
  409. package/src/tests/chatHarness.ts +478 -0
  410. package/src/tests/generateStream.ts +49 -0
  411. package/src/tests/harnessErrors.ts +22 -0
  412. package/src/tests/harnessStreaming.ts +227 -0
  413. package/src/tests/helpers.ts +9 -2
  414. package/src/types/harness.ts +1119 -0
  415. package/src/types/index.ts +209 -13
  416. package/src/types/observability.ts +344 -0
  417. package/src/types/sanitize-html.d.ts +25 -0
@@ -1,5 +1,6 @@
1
1
  import type {FindExactlyOnePlugin, FindOneOrNonePlugin} from "@terreno/api";
2
- import type {LanguageModel, StopCondition, ToolSet} from "ai";
2
+ import type {Ask, AskKind, ConfirmAskInput, DatasetColumn, SimpleCard} from "@terreno/blocks";
3
+ import type {LanguageModel, ModelMessage, StopCondition, ToolSet} from "ai";
3
4
  import type mongoose from "mongoose";
4
5
 
5
6
  // ============================================================
@@ -14,6 +15,8 @@ export const DEFAULT_AI_REQUEST_TYPES = [
14
15
  "remix",
15
16
  "summarization",
16
17
  "translation",
18
+ "ui_action",
19
+ "ui_blocks",
17
20
  ] as const;
18
21
  export type DefaultAIRequestType = (typeof DEFAULT_AI_REQUEST_TYPES)[number];
19
22
  export type AIRequestType = DefaultAIRequestType | (string & {});
@@ -100,6 +103,19 @@ export type MessageContentPart = TextContentPart | ImageContentPart | FileConten
100
103
  // GptHistory Types
101
104
  // ============================================================
102
105
 
106
+ export type GptHistoryAskStatus = "pending" | "answered" | "cancelled";
107
+
108
+ /** Set on asks the server makes itself: `approval` asks before a host tool with `needsApproval` runs. */
109
+ export type AskOrigin = "approval";
110
+
111
+ /** Marks a `tool-call` row as an ask and records whether the user has answered it. */
112
+ export interface GptHistoryPromptAsk {
113
+ kind: AskKind;
114
+ /** `approval` when the server asked before a host tool runs; the row is then display-only. */
115
+ origin?: AskOrigin;
116
+ status: GptHistoryAskStatus;
117
+ }
118
+
103
119
  /** Lifecycle of an assistant reply that is persisted while it streams. */
104
120
  export type GptHistoryPromptStatus = "streaming" | "complete" | "error";
105
121
 
@@ -117,11 +133,31 @@ export interface GptHistoryPrompt {
117
133
  toolName?: string;
118
134
  args?: Record<string, unknown>;
119
135
  result?: unknown;
136
+ ask?: GptHistoryPromptAsk;
137
+ }
138
+
139
+ interface GptHistoryPendingAskState {
140
+ /** The AI SDK approval request an `approval` ask answers. It is also the ask's `toolCallId`. */
141
+ approvalId?: string;
142
+ created: Date;
143
+ origin?: AskOrigin;
144
+ /** Number of leading `prompts` rows that form the paused turn's history. */
145
+ promptIndex: number;
146
+ /** AI SDK messages the paused turn produced, replayed verbatim when the user answers. */
147
+ responseMessages: ModelMessage[];
148
+ simple: SimpleCard;
149
+ toolCallId: string;
150
+ /** The host tool an `approval` ask asks to run. */
151
+ toolName?: string;
120
152
  }
121
153
 
154
+ /** The ask a history is waiting on. At most one per history. */
155
+ export type GptHistoryPendingAsk = Ask & GptHistoryPendingAskState;
156
+
122
157
  export interface GptHistoryDocument extends mongoose.Document<mongoose.Types.ObjectId> {
123
158
  created: Date;
124
159
  deleted: boolean;
160
+ pendingAsk?: GptHistoryPendingAsk;
125
161
  projectId?: mongoose.Types.ObjectId;
126
162
  prompts: GptHistoryPrompt[];
127
163
  title?: string;
@@ -172,7 +208,16 @@ export interface AIServiceOptions {
172
208
  model: LanguageModel;
173
209
  }
174
210
 
175
- export interface GenerateTextOptions {
211
+ export interface GenerateObservabilityOptions {
212
+ priceMap?: Record<string, {inputPerMTok: number; outputPerMTok: number}>;
213
+ promptLabel?: string;
214
+ promptName?: string;
215
+ sensitive?: boolean;
216
+ sessionId?: string;
217
+ skipTrace?: boolean;
218
+ }
219
+
220
+ export interface GenerateTextOptions extends GenerateObservabilityOptions {
176
221
  maxOutputTokens?: number;
177
222
  prompt: string;
178
223
  stopWhen?: StopCondition<ToolSet>;
@@ -183,7 +228,7 @@ export interface GenerateTextOptions {
183
228
  userId?: mongoose.Types.ObjectId;
184
229
  }
185
230
 
186
- export interface GenerateStreamOptions {
231
+ export interface GenerateStreamOptions extends GenerateObservabilityOptions {
187
232
  maxOutputTokens?: number;
188
233
  prompt: string;
189
234
  systemPrompt?: string;
@@ -191,7 +236,7 @@ export interface GenerateStreamOptions {
191
236
  userId?: mongoose.Types.ObjectId;
192
237
  }
193
238
 
194
- export interface GenerateChatStreamOptions {
239
+ export interface GenerateChatStreamOptions extends GenerateObservabilityOptions {
195
240
  messages: Array<{content: string; role: "user" | "assistant" | "system"}>;
196
241
  stopWhen?: StopCondition<ToolSet>;
197
242
  systemPrompt?: string;
@@ -200,24 +245,24 @@ export interface GenerateChatStreamOptions {
200
245
  userId?: mongoose.Types.ObjectId;
201
246
  }
202
247
 
203
- export interface RemixOptions {
248
+ export interface RemixOptions extends GenerateObservabilityOptions {
204
249
  text: string;
205
250
  userId?: mongoose.Types.ObjectId;
206
251
  }
207
252
 
208
- export interface SummaryOptions {
253
+ export interface SummaryOptions extends GenerateObservabilityOptions {
209
254
  text: string;
210
255
  userId?: mongoose.Types.ObjectId;
211
256
  }
212
257
 
213
- export interface TranslateOptions {
258
+ export interface TranslateOptions extends GenerateObservabilityOptions {
214
259
  sourceLanguage?: string;
215
260
  targetLanguage: string;
216
261
  text: string;
217
262
  userId?: mongoose.Types.ObjectId;
218
263
  }
219
264
 
220
- export interface GenerateJsonValueOptions {
265
+ export interface GenerateJsonValueOptions extends GenerateObservabilityOptions {
221
266
  maxOutputTokens?: number;
222
267
  /** Optional name passed to the provider for structured-output guidance. */
223
268
  outputName?: string;
@@ -229,7 +274,7 @@ export interface GenerateJsonValueOptions {
229
274
  userId?: mongoose.Types.ObjectId;
230
275
  }
231
276
 
232
- export interface GenerateJsonObjectOptions<OBJECT> {
277
+ export interface GenerateJsonObjectOptions<OBJECT> extends GenerateObservabilityOptions {
233
278
  maxOutputTokens?: number;
234
279
  prompt: string;
235
280
  /** Zod schema, `jsonSchema(...)`, or other `FlexibleSchema` accepted by the AI SDK. */
@@ -241,7 +286,28 @@ export interface GenerateJsonObjectOptions<OBJECT> {
241
286
  userId?: mongoose.Types.ObjectId;
242
287
  }
243
288
 
244
- export interface GenerateJsonArrayOptions<ELEMENT> {
289
+ export interface ObsTestMultiStageCall1Output {
290
+ phrase: string;
291
+ }
292
+
293
+ export interface ObsTestMultiStageCall2Output {
294
+ keywords: string[];
295
+ }
296
+
297
+ export interface ObsTestMultiStageMetrics {
298
+ call1: {charCount: number; wordCount: number};
299
+ call2: {charCount: number; wordCount: number};
300
+ combinedCharCount: number;
301
+ }
302
+
303
+ export interface ObsTestMultiStageFinalOutput {
304
+ keywords: string[];
305
+ metrics: ObsTestMultiStageMetrics;
306
+ phrase: string;
307
+ sentence: string;
308
+ }
309
+
310
+ export interface GenerateJsonArrayOptions<ELEMENT> extends GenerateObservabilityOptions {
245
311
  /** Schema for each array element. */
246
312
  element: import("ai").FlexibleSchema<ELEMENT>;
247
313
  /** Optional description for the array output (provider guidance). */
@@ -259,17 +325,102 @@ export interface GenerateJsonArrayOptions<ELEMENT> {
259
325
  // Route Option Types
260
326
  // ============================================================
261
327
 
328
+ /** Makes the `confirm` input of the approval ask for one call of a host tool, from the call's input. */
329
+ export type ApprovalAskInput = (input: unknown) => ConfirmAskInput;
330
+
331
+ export interface AsksOptions {
332
+ /**
333
+ * The approval ask for host tools with `needsApproval`, by tool name. Without an entry, or when
334
+ * it throws or returns an input that is not a valid `confirm` input, the ask is "Allow
335
+ * <toolName>?" with the tool's description and Allow / Deny buttons.
336
+ */
337
+ approvals?: Record<string, ApprovalAskInput>;
338
+ /** Ask kinds offered to the model, each as the tool `ask_<kind>`. Defaults to every kind. */
339
+ kinds?: AskKind[];
340
+ /**
341
+ * The per-file cap for answers to a `files` ask, in bytes. Defaults to 10 MB, the
342
+ * `/files/upload` default; `AiApp` applies it to `/files/upload` too.
343
+ */
344
+ maxFileSizeBytes?: number;
345
+ }
346
+
347
+ /** Loads an upload's bytes by its GCS key. `FileStorageService` implements it. */
348
+ export interface AskFileDownloader {
349
+ download: (gcsKey: string) => Promise<Buffer>;
350
+ }
351
+
352
+ /** What a host callback may return. `blocks` is a whole-reply document. */
353
+ export interface HostActionResult {
354
+ blocks?: unknown;
355
+ replace?: "block" | "message";
356
+ text?: string;
357
+ }
358
+
359
+ /** A Zod schema's `safeParse`, so hosts can pass `z.object(...)` without this package depending on a Zod version. */
360
+ export interface HostPayloadSchema {
361
+ safeParse: (
362
+ value: unknown
363
+ ) =>
364
+ | {data: unknown; success: true}
365
+ | {error: {issues: {message: string; path: PropertyKey[]}[]}; success: false};
366
+ }
367
+
368
+ export interface HostActionContext {
369
+ blockId: string;
370
+ elementId: string;
371
+ history: GptHistoryDocument;
372
+ messageId: string;
373
+ payload: unknown;
374
+ user: {_id?: mongoose.Types.ObjectId};
375
+ }
376
+
377
+ /** `addGptRoutes` `uiBlocks`. `true` checks every assistant reply. `hostActions` names the callbacks the model may emit. */
378
+ export interface UiBlocksOptions {
379
+ /** Milliseconds before a host callback returns 504. Default 10 seconds. */
380
+ actionTimeoutMs?: number;
381
+ /** Rows stored by `registerAiDataset`. Default 50,000. A larger write returns 413. */
382
+ datasetMaxRows?: number;
383
+ /** Days before a stored dataset expires. `0` (the default) keeps it. */
384
+ datasetTtlDays?: number;
385
+ /** When true, `html` blocks are allowed and sanitized before they are stored. */
386
+ html?: boolean;
387
+ /** Hostnames allowed on `https` image sources. Empty rejects every https image. */
388
+ imageHosts?: readonly string[];
389
+ hostActions?: Record<
390
+ string,
391
+ {
392
+ handler?: (
393
+ context: HostActionContext
394
+ ) => Promise<HostActionResult | undefined> | HostActionResult | undefined;
395
+ payload?: HostPayloadSchema;
396
+ }
397
+ >;
398
+ repair?: boolean;
399
+ }
400
+
262
401
  export interface GptRouteOptions {
263
402
  /** Pre-configured AIService. Optional when using per-request keys or demo mode. */
264
403
  aiService?: import("../service/aiService").AIService;
404
+ /**
405
+ * Let the model ask the user typed questions with client-side ask tools. `true` offers every
406
+ * ask kind. Off by default; when off, tools, system prompt, SSE events, and the `/gpt/histories`
407
+ * routes are unchanged.
408
+ */
409
+ asks?: boolean | AsksOptions;
265
410
  /** Factory to create a LanguageModel from a per-request API key (x-ai-api-key header). */
266
411
  createModelFn?: (apiKey: string, modelId?: string) => import("ai").LanguageModel;
267
412
  /** Factory to create a LanguageModel on the server side without a per-request key (e.g. Vertex AI with ADC). Used for model switching when no x-ai-api-key header is present. Returns undefined if no provider is configured (falls through to demo mode). */
268
413
  createServerModelFn?: (modelId?: string) => import("ai").LanguageModel | undefined;
269
414
  /** Factory to create per-request tools (e.g. tools that need the request's API key). Merged with static tools. */
270
415
  createRequestTools?: (req: import("express").Request) => Record<string, import("ai").Tool>;
271
- /** Return canned responses when no AI service is available. */
416
+ /** Not read: the routes send a canned demo reply whenever no AI service resolves. */
272
417
  demoMode?: boolean;
418
+ /**
419
+ * Where uploads are stored. `download` loads a `files` ask by `fileId`. `upload` and
420
+ * `getSignedUrl` store `data:` attachments and sign them on later turns. A service with only
421
+ * `download` still answers `files` asks; attachments are then saved as sent.
422
+ */
423
+ fileStorageService?: import("../service/fileStorage").FileStorageService | AskFileDownloader;
273
424
  mcpService?: import("../service/mcpService").MCPService;
274
425
  openApiOptions?: Record<string, unknown>;
275
426
  tools?: Record<string, import("ai").Tool>;
@@ -277,11 +428,16 @@ export interface GptRouteOptions {
277
428
  maxSteps?: number;
278
429
  /** Cheap model ID used for generating conversation titles (e.g. "gemini-3.5-flash-lite"). Falls back to the main model if not set. */
279
430
  titleModelId?: string;
431
+ /**
432
+ * Assistant replies are whole-reply block documents. Off by default; when off, the system
433
+ * prompt, SSE events, `/gpt/datasets`, and `/gpt/actions` are unchanged. `true` validates the
434
+ * final text, emits `{blocks}` before `{done}`, and mounts `GET /gpt/datasets/:id` and
435
+ * `POST /gpt/actions`. `{repair: true}` runs one repair call when that check fails.
436
+ */
437
+ uiBlocks?: boolean | UiBlocksOptions;
280
438
  /** Langfuse prompt name to load and use as the system prompt. Compiled with no variables.
281
439
  * Falls back gracefully if Langfuse is not configured or the prompt is not found. */
282
440
  langfuseSystemPromptName?: string;
283
- /** When set, `data:` attachments are uploaded to durable storage and history stores the reference. */
284
- fileStorageService?: import("../service/fileStorage").FileStorageService;
285
441
  /**
286
442
  * When `false` or the function returns `false`, prompts that include attachments are rejected.
287
443
  * Omit or pass `true` to leave uploads enabled.
@@ -296,6 +452,13 @@ export interface GptRouteOptions {
296
452
  }
297
453
 
298
454
  export interface GptHistoryRouteOptions {
455
+ /**
456
+ * The chat options headless turns run with, usually the ones passed to `addGptRoutes`. When they
457
+ * turn `asks` on, `/gpt/histories` adds `GET pendingAsks` and `POST /:id/turn` for clients that do
458
+ * not read server-sent events, such as a watch app. With `asks` off it adds neither, so a host
459
+ * that never turned asks on gets no new endpoints.
460
+ */
461
+ chat?: GptRouteOptions;
299
462
  openApiOptions?: Record<string, unknown>;
300
463
  }
301
464
 
@@ -323,6 +486,39 @@ export interface McpRouteOptions {
323
486
  // File Attachment Types
324
487
  // ============================================================
325
488
 
489
+ // ============================================================
490
+ // AIDataset Types
491
+ // ============================================================
492
+
493
+ export type AIDatasetColumn = DatasetColumn;
494
+
495
+ export type AIDatasetCell = string | number | null;
496
+
497
+ /** No instance methods. `ownerId` is a virtual of `userId`. */
498
+ export type AIDatasetMethods = Record<string, never>;
499
+
500
+ export interface AIDatasetDocument extends mongoose.Document<mongoose.Types.ObjectId> {
501
+ columns: AIDatasetColumn[];
502
+ created: Date;
503
+ deleted: boolean;
504
+ expiresAt?: Date;
505
+ historyId: mongoose.Types.ObjectId;
506
+ rowCount: number;
507
+ rows: AIDatasetCell[][];
508
+ updated: Date;
509
+ userId: mongoose.Types.ObjectId;
510
+ }
511
+
512
+ export interface AIDatasetStatics
513
+ extends FindExactlyOnePlugin<AIDatasetDocument>,
514
+ FindOneOrNonePlugin<AIDatasetDocument> {}
515
+
516
+ export interface AIDatasetModel
517
+ extends mongoose.Model<AIDatasetDocument, object, AIDatasetMethods>,
518
+ AIDatasetStatics {}
519
+
520
+ export type AIDatasetSchema = mongoose.Schema<AIDatasetDocument, AIDatasetModel, AIDatasetMethods>;
521
+
326
522
  export interface FileAttachmentDocument extends mongoose.Document<mongoose.Types.ObjectId> {
327
523
  created: Date;
328
524
  deleted: boolean;
@@ -0,0 +1,344 @@
1
+ import type {FindExactlyOnePlugin, FindOneOrNonePlugin} from "@terreno/api";
2
+ import type mongoose from "mongoose";
3
+
4
+ export interface ObsPromptVariable {
5
+ key: string;
6
+ label?: string;
7
+ required: boolean;
8
+ reviewerNote?: string;
9
+ }
10
+
11
+ export interface ObsPromptUsage {
12
+ costUsd?: number;
13
+ inputTokens?: number;
14
+ model?: string;
15
+ outputTokens?: number;
16
+ }
17
+
18
+ export interface ObsPromptRef {
19
+ label?: string;
20
+ name: string;
21
+ version: number;
22
+ }
23
+
24
+ export interface ObsPromptDocument extends mongoose.Document<mongoose.Types.ObjectId> {
25
+ created: Date;
26
+ deleted: boolean;
27
+ description?: string;
28
+ folder: string;
29
+ name: string;
30
+ tags: string[];
31
+ updated: Date;
32
+ }
33
+
34
+ export interface ObsPromptStatics
35
+ extends FindExactlyOnePlugin<ObsPromptDocument>,
36
+ FindOneOrNonePlugin<ObsPromptDocument> {}
37
+
38
+ export interface ObsPromptModel extends mongoose.Model<ObsPromptDocument>, ObsPromptStatics {}
39
+
40
+ export interface ObsPromptVersionDocument extends mongoose.Document<mongoose.Types.ObjectId> {
41
+ config?: Record<string, unknown>;
42
+ created: Date;
43
+ deleted: boolean;
44
+ inputSchema?: Record<string, unknown>;
45
+ outputFieldNotes?: Record<string, string>;
46
+ outputSchema?: Record<string, unknown>;
47
+ promptId: mongoose.Types.ObjectId;
48
+ sensitive: boolean;
49
+ system?: string;
50
+ template?: string;
51
+ type: "chat" | "text";
52
+ updated: Date;
53
+ variables: ObsPromptVariable[];
54
+ version: number;
55
+ }
56
+
57
+ export interface ObsPromptVersionStatics
58
+ extends FindExactlyOnePlugin<ObsPromptVersionDocument>,
59
+ FindOneOrNonePlugin<ObsPromptVersionDocument> {}
60
+
61
+ export interface ObsPromptVersionModel
62
+ extends mongoose.Model<ObsPromptVersionDocument>,
63
+ ObsPromptVersionStatics {}
64
+
65
+ export interface ObsPromptLabelDocument extends mongoose.Document<mongoose.Types.ObjectId> {
66
+ created: Date;
67
+ deleted: boolean;
68
+ label: string;
69
+ promptId: mongoose.Types.ObjectId;
70
+ updated: Date;
71
+ versionId: mongoose.Types.ObjectId;
72
+ }
73
+
74
+ export interface ObsPromptLabelStatics
75
+ extends FindExactlyOnePlugin<ObsPromptLabelDocument>,
76
+ FindOneOrNonePlugin<ObsPromptLabelDocument> {}
77
+
78
+ export interface ObsPromptLabelModel
79
+ extends mongoose.Model<ObsPromptLabelDocument>,
80
+ ObsPromptLabelStatics {}
81
+
82
+ export interface ObsTraceDocument extends mongoose.Document<mongoose.Types.ObjectId> {
83
+ created: Date;
84
+ deleted: boolean;
85
+ endedAt?: Date;
86
+ errorSummary?: string;
87
+ flaggedForDataset: boolean;
88
+ input?: unknown;
89
+ name: string;
90
+ output?: unknown;
91
+ prompts: ObsPromptRef[];
92
+ sensitive: boolean;
93
+ sessionId?: string;
94
+ startedAt: Date;
95
+ status: "error" | "ok";
96
+ updated: Date;
97
+ usage?: ObsPromptUsage;
98
+ userId?: mongoose.Types.ObjectId;
99
+ }
100
+
101
+ export interface ObsTraceStatics
102
+ extends FindExactlyOnePlugin<ObsTraceDocument>,
103
+ FindOneOrNonePlugin<ObsTraceDocument> {}
104
+
105
+ export interface ObsTraceModel extends mongoose.Model<ObsTraceDocument>, ObsTraceStatics {}
106
+
107
+ export interface ObsSpanDocument extends mongoose.Document<mongoose.Types.ObjectId> {
108
+ created: Date;
109
+ deleted: boolean;
110
+ durationMs?: number;
111
+ endedAt?: Date;
112
+ error?: string;
113
+ input?: unknown;
114
+ kind: "AGENT" | "CHAIN" | "EVALUATOR" | "LLM" | "RETRIEVER" | "TOOL";
115
+ name: string;
116
+ output?: unknown;
117
+ parentSpanId?: mongoose.Types.ObjectId;
118
+ sensitive?: boolean;
119
+ startOffsetMs?: number;
120
+ startedAt: Date;
121
+ status: "error" | "ok";
122
+ traceId: mongoose.Types.ObjectId;
123
+ updated: Date;
124
+ usage?: ObsPromptUsage;
125
+ }
126
+
127
+ export interface ObsSpanStatics
128
+ extends FindExactlyOnePlugin<ObsSpanDocument>,
129
+ FindOneOrNonePlugin<ObsSpanDocument> {}
130
+
131
+ export interface ObsSpanModel extends mongoose.Model<ObsSpanDocument>, ObsSpanStatics {}
132
+
133
+ export interface ObsScoreDocument extends mongoose.Document<mongoose.Types.ObjectId> {
134
+ comment?: string;
135
+ confidence?: number;
136
+ created: Date;
137
+ dataType: "boolean" | "categorical" | "numeric";
138
+ deleted: boolean;
139
+ evaluatorId?: mongoose.Types.ObjectId;
140
+ name: string;
141
+ source: "code" | "human" | "llm-judge" | "user-feedback";
142
+ spanId?: mongoose.Types.ObjectId;
143
+ traceId: mongoose.Types.ObjectId;
144
+ updated: Date;
145
+ value: boolean | number | string;
146
+ }
147
+
148
+ export interface ObsScoreStatics
149
+ extends FindExactlyOnePlugin<ObsScoreDocument>,
150
+ FindOneOrNonePlugin<ObsScoreDocument> {}
151
+
152
+ export interface ObsScoreModel extends mongoose.Model<ObsScoreDocument>, ObsScoreStatics {}
153
+
154
+ export interface EvaluatorDimension {
155
+ dataType: "boolean" | "categorical" | "numeric";
156
+ key: string;
157
+ range?: string;
158
+ required: boolean;
159
+ }
160
+
161
+ export interface EvaluatorRunModes {
162
+ allowManualRun: boolean;
163
+ availableInExperiments: boolean;
164
+ liveSampleRate: number;
165
+ }
166
+
167
+ export interface ObsEvaluatorDocument extends mongoose.Document<mongoose.Types.ObjectId> {
168
+ assertion?: {constraint: string; path: string};
169
+ confidenceAlertBelow: number;
170
+ created: Date;
171
+ deleted: boolean;
172
+ description?: string;
173
+ dimensions: EvaluatorDimension[];
174
+ instructions?: string;
175
+ judgePromptName?: string;
176
+ name: string;
177
+ runModes: EvaluatorRunModes;
178
+ target: "dataset item" | "full trace" | "generation span";
179
+ type: "human" | "json-assert" | "llm-judge";
180
+ updated: Date;
181
+ }
182
+
183
+ export interface ObsEvaluatorStatics
184
+ extends FindExactlyOnePlugin<ObsEvaluatorDocument>,
185
+ FindOneOrNonePlugin<ObsEvaluatorDocument> {}
186
+
187
+ export interface ObsEvaluatorModel
188
+ extends mongoose.Model<ObsEvaluatorDocument>,
189
+ ObsEvaluatorStatics {}
190
+
191
+ export interface ObsReviewItemDocument extends mongoose.Document<mongoose.Types.ObjectId> {
192
+ assigneeId?: mongoose.Types.ObjectId;
193
+ comment?: string;
194
+ created: Date;
195
+ datasetItemId?: mongoose.Types.ObjectId;
196
+ deleted: boolean;
197
+ enqueuedAt: Date;
198
+ evaluatorId: mongoose.Types.ObjectId;
199
+ reason: "dataset_candidate" | "eval" | "feedback" | "manual";
200
+ scores?: Record<string, boolean | number | string>;
201
+ spanId?: mongoose.Types.ObjectId;
202
+ status: "done" | "in_progress" | "pending" | "skipped";
203
+ traceId: mongoose.Types.ObjectId;
204
+ updated: Date;
205
+ }
206
+
207
+ export interface ObsReviewItemStatics
208
+ extends FindExactlyOnePlugin<ObsReviewItemDocument>,
209
+ FindOneOrNonePlugin<ObsReviewItemDocument> {}
210
+
211
+ export interface ObsReviewItemModel
212
+ extends mongoose.Model<ObsReviewItemDocument>,
213
+ ObsReviewItemStatics {}
214
+
215
+ export interface ObsDatasetAnnotatedBy {
216
+ label: string;
217
+ reviewItemId?: string;
218
+ userId?: string;
219
+ }
220
+
221
+ export interface ObsDatasetDocument extends mongoose.Document<mongoose.Types.ObjectId> {
222
+ created: Date;
223
+ deleted: boolean;
224
+ description?: string;
225
+ expectedOutputSchema?: Record<string, unknown>;
226
+ inputSchemaPromptName?: string;
227
+ name: string;
228
+ tags: string[];
229
+ updated: Date;
230
+ }
231
+
232
+ export interface ObsDatasetStatics
233
+ extends FindExactlyOnePlugin<ObsDatasetDocument>,
234
+ FindOneOrNonePlugin<ObsDatasetDocument> {}
235
+
236
+ export interface ObsDatasetModel extends mongoose.Model<ObsDatasetDocument>, ObsDatasetStatics {}
237
+
238
+ export interface ObsDatasetItemDocument extends mongoose.Document<mongoose.Types.ObjectId> {
239
+ annotatedBy?: ObsDatasetAnnotatedBy;
240
+ created: Date;
241
+ datasetId: mongoose.Types.ObjectId;
242
+ deleted: boolean;
243
+ expectedOutput?: unknown;
244
+ input: unknown;
245
+ metadata?: Record<string, unknown>;
246
+ origin: "manual" | "synthetic" | "trace";
247
+ outcomeClass?: "fn" | "fp" | "tn" | "tp";
248
+ proofread: boolean;
249
+ sourceTraceId?: mongoose.Types.ObjectId;
250
+ tags: string[];
251
+ updated: Date;
252
+ }
253
+
254
+ export interface ObsDatasetItemStatics
255
+ extends FindExactlyOnePlugin<ObsDatasetItemDocument>,
256
+ FindOneOrNonePlugin<ObsDatasetItemDocument> {}
257
+
258
+ export interface ObsDatasetItemModel
259
+ extends mongoose.Model<ObsDatasetItemDocument>,
260
+ ObsDatasetItemStatics {}
261
+
262
+ export interface ScoreThreshold {
263
+ aggregate: "mean" | "trueRate";
264
+ dimension: string;
265
+ evaluatorName: string;
266
+ op: "eq" | "gte" | "lte";
267
+ value: number;
268
+ }
269
+
270
+ export interface ExperimentGateResult extends ScoreThreshold {
271
+ actual?: number;
272
+ passed: boolean;
273
+ version: number;
274
+ }
275
+
276
+ export interface ExperimentEstimate {
277
+ costUsd?: number;
278
+ generations: number;
279
+ wallClockSeconds: number;
280
+ }
281
+
282
+ export interface ExperimentAggregates {
283
+ gates: ExperimentGateResult[];
284
+ lowConfidenceItemIds: string[];
285
+ outlierItemIds: string[];
286
+ progress: {completed: number; total: number};
287
+ totalCostUsd?: number;
288
+ }
289
+
290
+ export interface ObsExperimentDocument extends mongoose.Document<mongoose.Types.ObjectId> {
291
+ backgroundTaskId?: mongoose.Types.ObjectId;
292
+ created: Date;
293
+ datasetId: mongoose.Types.ObjectId;
294
+ deleted: boolean;
295
+ estimate?: ExperimentEstimate;
296
+ evaluatorIds: mongoose.Types.ObjectId[];
297
+ includeUnproofread: boolean;
298
+ modelOverride?: string;
299
+ name: string;
300
+ promptName: string;
301
+ results?: ExperimentAggregates;
302
+ status: "completed" | "failed" | "pending" | "running";
303
+ thresholds: ScoreThreshold[];
304
+ updated: Date;
305
+ versions: number[];
306
+ }
307
+
308
+ export interface ObsExperimentStatics
309
+ extends FindExactlyOnePlugin<ObsExperimentDocument>,
310
+ FindOneOrNonePlugin<ObsExperimentDocument> {}
311
+
312
+ export interface ObsExperimentModel
313
+ extends mongoose.Model<ObsExperimentDocument>,
314
+ ObsExperimentStatics {}
315
+
316
+ export interface ExperimentVersionEvaluatorScore {
317
+ confidence?: number;
318
+ error?: string;
319
+ scores?: Record<string, boolean | number | string>;
320
+ }
321
+
322
+ export interface ExperimentVersionResult {
323
+ error?: string;
324
+ evaluatorScores: Record<string, ExperimentVersionEvaluatorScore>;
325
+ output?: unknown;
326
+ }
327
+
328
+ export interface ObsExperimentItemDocument extends mongoose.Document<mongoose.Types.ObjectId> {
329
+ created: Date;
330
+ datasetItemId: mongoose.Types.ObjectId;
331
+ deleted: boolean;
332
+ experimentId: mongoose.Types.ObjectId;
333
+ failed: boolean;
334
+ updated: Date;
335
+ versionResults: Record<string, ExperimentVersionResult>;
336
+ }
337
+
338
+ export interface ObsExperimentItemStatics
339
+ extends FindExactlyOnePlugin<ObsExperimentItemDocument>,
340
+ FindOneOrNonePlugin<ObsExperimentItemDocument> {}
341
+
342
+ export interface ObsExperimentItemModel
343
+ extends mongoose.Model<ObsExperimentItemDocument>,
344
+ ObsExperimentItemStatics {}