@terreno/ai 57.8.0 → 57.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (425) hide show
  1. package/.cursorrules +2 -2
  2. package/README.md +2 -1
  3. package/bunfig.toml +1 -1
  4. package/dist/aiApp.d.ts +16 -5
  5. package/dist/aiApp.js +17 -7
  6. package/dist/harness/adminScreens.d.ts +6 -0
  7. package/dist/harness/adminScreens.js +11 -0
  8. package/dist/harness/agentLoop.d.ts +61 -0
  9. package/dist/harness/agentLoop.js +798 -0
  10. package/dist/harness/agentTaskNames.d.ts +4 -0
  11. package/dist/harness/agentTaskNames.js +7 -0
  12. package/dist/harness/approvalPolicy.d.ts +3 -0
  13. package/dist/harness/approvalPolicy.js +23 -0
  14. package/dist/harness/approvals.d.ts +101 -0
  15. package/dist/harness/approvals.js +468 -0
  16. package/dist/harness/commit.d.ts +238 -0
  17. package/dist/harness/commit.js +678 -0
  18. package/dist/harness/conversation.d.ts +75 -0
  19. package/dist/harness/conversation.js +397 -0
  20. package/dist/harness/defineAgent.d.ts +9 -0
  21. package/dist/harness/defineAgent.js +65 -0
  22. package/dist/harness/defineTask.d.ts +11 -0
  23. package/dist/harness/defineTask.js +59 -0
  24. package/dist/harness/defineTool.d.ts +7 -0
  25. package/dist/harness/defineTool.js +36 -0
  26. package/dist/harness/definitionError.d.ts +8 -0
  27. package/dist/harness/definitionError.js +15 -0
  28. package/dist/harness/errors.d.ts +127 -0
  29. package/dist/harness/errors.js +101 -0
  30. package/dist/harness/eventHub.d.ts +45 -0
  31. package/dist/harness/eventHub.js +135 -0
  32. package/dist/harness/events.d.ts +95 -0
  33. package/dist/harness/events.js +267 -0
  34. package/dist/harness/executionEnv.d.ts +31 -0
  35. package/dist/harness/executionEnv.js +2 -0
  36. package/dist/harness/extensions.d.ts +82 -0
  37. package/dist/harness/extensions.js +274 -0
  38. package/dist/harness/harness.d.ts +163 -0
  39. package/dist/harness/harness.js +520 -0
  40. package/dist/harness/harnessApp.d.ts +44 -0
  41. package/dist/harness/harnessApp.js +62 -0
  42. package/dist/harness/internalRuntime.d.ts +16 -0
  43. package/dist/harness/internalRuntime.js +18 -0
  44. package/dist/harness/leases.d.ts +35 -0
  45. package/dist/harness/leases.js +86 -0
  46. package/dist/harness/memo.d.ts +24 -0
  47. package/dist/harness/memo.js +78 -0
  48. package/dist/harness/modelCall.d.ts +45 -0
  49. package/dist/harness/modelCall.js +147 -0
  50. package/dist/harness/models/harnessApproval.d.ts +2 -0
  51. package/dist/harness/models/harnessApproval.js +100 -0
  52. package/dist/harness/models/harnessConversation.d.ts +2 -0
  53. package/dist/harness/models/harnessConversation.js +117 -0
  54. package/dist/harness/models/harnessEvent.d.ts +3 -0
  55. package/dist/harness/models/harnessEvent.js +73 -0
  56. package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
  57. package/dist/harness/models/harnessInboxEvent.js +58 -0
  58. package/dist/harness/models/harnessMemo.d.ts +2 -0
  59. package/dist/harness/models/harnessMemo.js +42 -0
  60. package/dist/harness/models/harnessMessage.d.ts +2 -0
  61. package/dist/harness/models/harnessMessage.js +73 -0
  62. package/dist/harness/models/harnessOwner.d.ts +2 -0
  63. package/dist/harness/models/harnessOwner.js +36 -0
  64. package/dist/harness/models/harnessTask.d.ts +2 -0
  65. package/dist/harness/models/harnessTask.js +186 -0
  66. package/dist/harness/ownership.d.ts +89 -0
  67. package/dist/harness/ownership.js +438 -0
  68. package/dist/harness/registry.d.ts +29 -0
  69. package/dist/harness/registry.js +93 -0
  70. package/dist/harness/retryBackoff.d.ts +31 -0
  71. package/dist/harness/retryBackoff.js +50 -0
  72. package/dist/harness/routes/approvals.d.ts +13 -0
  73. package/dist/harness/routes/approvals.js +93 -0
  74. package/dist/harness/routes/conversations.d.ts +11 -0
  75. package/dist/harness/routes/conversations.js +66 -0
  76. package/dist/harness/routes/events.d.ts +19 -0
  77. package/dist/harness/routes/events.js +215 -0
  78. package/dist/harness/routes/tasks.d.ts +11 -0
  79. package/dist/harness/routes/tasks.js +108 -0
  80. package/dist/harness/runners/inProcessRunner.d.ts +57 -0
  81. package/dist/harness/runners/inProcessRunner.js +196 -0
  82. package/dist/harness/runtime.d.ts +34 -0
  83. package/dist/harness/runtime.js +551 -0
  84. package/dist/harness/subagent.d.ts +53 -0
  85. package/dist/harness/subagent.js +134 -0
  86. package/dist/harness/suspend.d.ts +8 -0
  87. package/dist/harness/suspend.js +16 -0
  88. package/dist/harness/transaction.d.ts +3 -0
  89. package/dist/harness/transaction.js +22 -0
  90. package/dist/harness/waits.d.ts +100 -0
  91. package/dist/harness/waits.js +214 -0
  92. package/dist/index.d.ts +19 -1
  93. package/dist/index.js +45 -2
  94. package/dist/models/aiDataset.d.ts +2 -0
  95. package/dist/models/aiDataset.js +67 -0
  96. package/dist/models/aiRequest.js +1 -1
  97. package/dist/models/gptHistory.js +88 -2
  98. package/dist/observability/adminScreens.d.ts +5 -0
  99. package/dist/observability/adminScreens.js +32 -0
  100. package/dist/observability/compileTemplate.d.ts +1 -0
  101. package/dist/observability/compileTemplate.js +13 -0
  102. package/dist/observability/datasetImport.d.ts +11 -0
  103. package/dist/observability/datasetImport.js +185 -0
  104. package/dist/observability/evaluate.d.ts +31 -0
  105. package/dist/observability/evaluate.js +200 -0
  106. package/dist/observability/evaluatorTemplates.d.ts +17 -0
  107. package/dist/observability/evaluatorTemplates.js +111 -0
  108. package/dist/observability/jsonPath.d.ts +2 -0
  109. package/dist/observability/jsonPath.js +57 -0
  110. package/dist/observability/local/datasetStore.d.ts +87 -0
  111. package/dist/observability/local/datasetStore.js +326 -0
  112. package/dist/observability/local/evaluatorStore.d.ts +44 -0
  113. package/dist/observability/local/evaluatorStore.js +182 -0
  114. package/dist/observability/local/experimentRunner.d.ts +96 -0
  115. package/dist/observability/local/experimentRunner.js +419 -0
  116. package/dist/observability/local/localPlugin.d.ts +22 -0
  117. package/dist/observability/local/localPlugin.js +76 -0
  118. package/dist/observability/local/models/obsDataset.d.ts +2 -0
  119. package/dist/observability/local/models/obsDataset.js +37 -0
  120. package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
  121. package/dist/observability/local/models/obsDatasetItem.js +76 -0
  122. package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
  123. package/dist/observability/local/models/obsEvaluator.js +98 -0
  124. package/dist/observability/local/models/obsExperiment.d.ts +2 -0
  125. package/dist/observability/local/models/obsExperiment.js +87 -0
  126. package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
  127. package/dist/observability/local/models/obsExperimentItem.js +45 -0
  128. package/dist/observability/local/models/obsPrompt.d.ts +2 -0
  129. package/dist/observability/local/models/obsPrompt.js +42 -0
  130. package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
  131. package/dist/observability/local/models/obsPromptLabel.js +39 -0
  132. package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
  133. package/dist/observability/local/models/obsPromptVersion.js +96 -0
  134. package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
  135. package/dist/observability/local/models/obsReviewItem.js +75 -0
  136. package/dist/observability/local/models/obsScore.d.ts +2 -0
  137. package/dist/observability/local/models/obsScore.js +58 -0
  138. package/dist/observability/local/models/obsSpan.d.ts +2 -0
  139. package/dist/observability/local/models/obsSpan.js +64 -0
  140. package/dist/observability/local/models/obsTrace.d.ts +2 -0
  141. package/dist/observability/local/models/obsTrace.js +93 -0
  142. package/dist/observability/local/promptStore.d.ts +180 -0
  143. package/dist/observability/local/promptStore.js +444 -0
  144. package/dist/observability/local/reviewStore.d.ts +50 -0
  145. package/dist/observability/local/reviewStore.js +203 -0
  146. package/dist/observability/local/traceStore.d.ts +103 -0
  147. package/dist/observability/local/traceStore.js +325 -0
  148. package/dist/observability/observabilityApp.d.ts +24 -0
  149. package/dist/observability/observabilityApp.js +157 -0
  150. package/dist/observability/observabilityAppRegistry.d.ts +14 -0
  151. package/dist/observability/observabilityAppRegistry.js +16 -0
  152. package/dist/observability/permissions.d.ts +7 -0
  153. package/dist/observability/permissions.js +60 -0
  154. package/dist/observability/reviewPanels.d.ts +18 -0
  155. package/dist/observability/reviewPanels.js +46 -0
  156. package/dist/observability/routes/datasets.d.ts +7 -0
  157. package/dist/observability/routes/datasets.js +149 -0
  158. package/dist/observability/routes/evaluators.d.ts +7 -0
  159. package/dist/observability/routes/evaluators.js +123 -0
  160. package/dist/observability/routes/experiments.d.ts +7 -0
  161. package/dist/observability/routes/experiments.js +95 -0
  162. package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
  163. package/dist/observability/routes/observabilityRouteAccess.js +67 -0
  164. package/dist/observability/routes/prompts.d.ts +12 -0
  165. package/dist/observability/routes/prompts.js +172 -0
  166. package/dist/observability/routes/requestAiService.d.ts +13 -0
  167. package/dist/observability/routes/requestAiService.js +21 -0
  168. package/dist/observability/routes/review.d.ts +7 -0
  169. package/dist/observability/routes/review.js +79 -0
  170. package/dist/observability/routes/status.d.ts +5 -0
  171. package/dist/observability/routes/status.js +31 -0
  172. package/dist/observability/routes/testMultiStage.d.ts +9 -0
  173. package/dist/observability/routes/testMultiStage.js +67 -0
  174. package/dist/observability/routes/traces.d.ts +7 -0
  175. package/dist/observability/routes/traces.js +129 -0
  176. package/dist/observability/schemaValidation.d.ts +5 -0
  177. package/dist/observability/schemaValidation.js +26 -0
  178. package/dist/observability/sopThresholds.d.ts +2 -0
  179. package/dist/observability/sopThresholds.js +21 -0
  180. package/dist/observability/status.d.ts +23 -0
  181. package/dist/observability/status.js +34 -0
  182. package/dist/observability/testMultiStageSchemas.d.ts +10 -0
  183. package/dist/observability/testMultiStageSchemas.js +79 -0
  184. package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
  185. package/dist/observability/testMultiStageWorkflow.js +218 -0
  186. package/dist/observability/types.d.ts +159 -0
  187. package/dist/observability/types.js +67 -0
  188. package/dist/routes/files.js +20 -7
  189. package/dist/routes/gpt.js +179 -488
  190. package/dist/routes/gptActions.d.ts +9 -0
  191. package/dist/routes/gptActions.js +212 -0
  192. package/dist/routes/gptDatasets.d.ts +5 -0
  193. package/dist/routes/gptDatasets.js +94 -0
  194. package/dist/routes/gptHistories.d.ts +3 -1
  195. package/dist/routes/gptHistories.js +188 -2
  196. package/dist/service/aiDatasets.d.ts +57 -0
  197. package/dist/service/aiDatasets.js +253 -0
  198. package/dist/service/aiService.d.ts +50 -1
  199. package/dist/service/aiService.js +431 -77
  200. package/dist/service/askFiles.d.ts +49 -0
  201. package/dist/service/askFiles.js +162 -0
  202. package/dist/service/asks.d.ts +76 -0
  203. package/dist/service/asks.js +284 -0
  204. package/dist/service/chatTurn.d.ts +128 -0
  205. package/dist/service/chatTurn.js +1674 -0
  206. package/dist/service/fileStorage.d.ts +4 -0
  207. package/dist/service/fileStorage.js +7 -2
  208. package/dist/service/fileUploadsGate.d.ts +8 -0
  209. package/dist/service/fileUploadsGate.js +18 -0
  210. package/dist/service/fillActionIds.d.ts +6 -0
  211. package/dist/service/fillActionIds.js +79 -0
  212. package/dist/service/gemini.d.ts +3 -3
  213. package/dist/service/gemini.js +3 -3
  214. package/dist/service/jsonFenceModel.d.ts +7 -0
  215. package/dist/service/jsonFenceModel.js +45 -0
  216. package/dist/service/prompts.d.ts +52 -0
  217. package/dist/service/prompts.js +96 -1
  218. package/dist/service/sanitizeHtml.d.ts +11 -0
  219. package/dist/service/sanitizeHtml.js +108 -0
  220. package/dist/service/vertex.d.ts +6 -6
  221. package/dist/service/vertex.js +5 -5
  222. package/dist/types/harness.d.ts +934 -0
  223. package/dist/types/harness.js +133 -0
  224. package/dist/types/index.d.ts +219 -13
  225. package/dist/types/index.js +2 -0
  226. package/dist/types/observability.d.ts +288 -0
  227. package/dist/types/observability.js +2 -0
  228. package/package.json +17 -5
  229. package/src/aiApp.test.ts +207 -1
  230. package/src/aiApp.ts +31 -8
  231. package/src/harness/adminScreens.test.ts +34 -0
  232. package/src/harness/adminScreens.ts +11 -0
  233. package/src/harness/agentLoop.test.ts +1034 -0
  234. package/src/harness/agentLoop.ts +1082 -0
  235. package/src/harness/agentRecovery.test.ts +380 -0
  236. package/src/harness/agentTaskNames.ts +4 -0
  237. package/src/harness/approvalPolicy.ts +28 -0
  238. package/src/harness/approvals.test.ts +1094 -0
  239. package/src/harness/approvals.ts +644 -0
  240. package/src/harness/commit.ts +1049 -0
  241. package/src/harness/conversation.ts +503 -0
  242. package/src/harness/defineAgent.test.ts +92 -0
  243. package/src/harness/defineAgent.ts +82 -0
  244. package/src/harness/defineTask.test.ts +59 -0
  245. package/src/harness/defineTask.ts +76 -0
  246. package/src/harness/defineTool.ts +45 -0
  247. package/src/harness/definitionError.ts +13 -0
  248. package/src/harness/errors.test.ts +90 -0
  249. package/src/harness/errors.ts +119 -0
  250. package/src/harness/eventHub.test.ts +124 -0
  251. package/src/harness/eventHub.ts +159 -0
  252. package/src/harness/events.test.ts +757 -0
  253. package/src/harness/events.ts +384 -0
  254. package/src/harness/executionEnv.ts +34 -0
  255. package/src/harness/extensions.test.ts +1218 -0
  256. package/src/harness/extensions.ts +382 -0
  257. package/src/harness/harness.test.ts +693 -0
  258. package/src/harness/harness.ts +767 -0
  259. package/src/harness/harnessApp.ts +76 -0
  260. package/src/harness/internalRuntime.ts +44 -0
  261. package/src/harness/leases.ts +132 -0
  262. package/src/harness/memo.ts +107 -0
  263. package/src/harness/modelCall.test.ts +134 -0
  264. package/src/harness/modelCall.ts +200 -0
  265. package/src/harness/models/harnessApproval.ts +109 -0
  266. package/src/harness/models/harnessConversation.ts +142 -0
  267. package/src/harness/models/harnessEvent.ts +92 -0
  268. package/src/harness/models/harnessInboxEvent.ts +73 -0
  269. package/src/harness/models/harnessMemo.ts +43 -0
  270. package/src/harness/models/harnessMessage.ts +85 -0
  271. package/src/harness/models/harnessModels.test.ts +60 -0
  272. package/src/harness/models/harnessOwner.ts +40 -0
  273. package/src/harness/models/harnessTask.ts +201 -0
  274. package/src/harness/ownership.test.ts +1087 -0
  275. package/src/harness/ownership.ts +618 -0
  276. package/src/harness/recovery.test.ts +912 -0
  277. package/src/harness/registry.ts +124 -0
  278. package/src/harness/retries.test.ts +292 -0
  279. package/src/harness/retryBackoff.test.ts +100 -0
  280. package/src/harness/retryBackoff.ts +77 -0
  281. package/src/harness/routes/approvals.ts +123 -0
  282. package/src/harness/routes/conversations.test.ts +518 -0
  283. package/src/harness/routes/conversations.ts +102 -0
  284. package/src/harness/routes/events.ts +280 -0
  285. package/src/harness/routes/tasks.test.ts +198 -0
  286. package/src/harness/routes/tasks.ts +146 -0
  287. package/src/harness/runners/inProcessRunner.test.ts +402 -0
  288. package/src/harness/runners/inProcessRunner.ts +220 -0
  289. package/src/harness/runtime.ts +714 -0
  290. package/src/harness/subagent.test.ts +696 -0
  291. package/src/harness/subagent.ts +192 -0
  292. package/src/harness/suspend.ts +13 -0
  293. package/src/harness/transaction.ts +17 -0
  294. package/src/harness/versioning.test.ts +288 -0
  295. package/src/harness/waits.test.ts +693 -0
  296. package/src/harness/waits.ts +356 -0
  297. package/src/index.test.ts +13 -0
  298. package/src/index.ts +70 -0
  299. package/src/isolated/adminScreensConfig.isolated.ts +143 -0
  300. package/src/models/aiDataset.test.ts +34 -0
  301. package/src/models/aiDataset.ts +78 -0
  302. package/src/models/aiRequest.test.ts +2 -2
  303. package/src/models/aiRequest.ts +2 -1
  304. package/src/models/gptHistory.test.ts +156 -0
  305. package/src/models/gptHistory.ts +106 -4
  306. package/src/observability/adminScreens.test.ts +71 -0
  307. package/src/observability/adminScreens.ts +42 -0
  308. package/src/observability/compileTemplate.test.ts +14 -0
  309. package/src/observability/compileTemplate.ts +10 -0
  310. package/src/observability/datasetImport.test.ts +95 -0
  311. package/src/observability/datasetImport.ts +220 -0
  312. package/src/observability/evaluate.test.ts +549 -0
  313. package/src/observability/evaluate.ts +254 -0
  314. package/src/observability/evaluatorTemplates.ts +124 -0
  315. package/src/observability/jsonPath.test.ts +52 -0
  316. package/src/observability/jsonPath.ts +60 -0
  317. package/src/observability/local/datasetStore.test.ts +230 -0
  318. package/src/observability/local/datasetStore.ts +452 -0
  319. package/src/observability/local/evaluatorStore.test.ts +138 -0
  320. package/src/observability/local/evaluatorStore.ts +224 -0
  321. package/src/observability/local/experimentRunner.test.ts +489 -0
  322. package/src/observability/local/experimentRunner.ts +548 -0
  323. package/src/observability/local/localPlugin.ts +88 -0
  324. package/src/observability/local/models/obsDataset.ts +38 -0
  325. package/src/observability/local/models/obsDatasetItem.ts +80 -0
  326. package/src/observability/local/models/obsEvaluator.ts +102 -0
  327. package/src/observability/local/models/obsExperiment.ts +91 -0
  328. package/src/observability/local/models/obsExperimentItem.ts +52 -0
  329. package/src/observability/local/models/obsPrompt.ts +43 -0
  330. package/src/observability/local/models/obsPromptLabel.ts +43 -0
  331. package/src/observability/local/models/obsPromptVersion.ts +107 -0
  332. package/src/observability/local/models/obsReviewItem.ts +79 -0
  333. package/src/observability/local/models/obsScore.ts +59 -0
  334. package/src/observability/local/models/obsSpan.ts +65 -0
  335. package/src/observability/local/models/obsTrace.ts +94 -0
  336. package/src/observability/local/obsModels.test.ts +119 -0
  337. package/src/observability/local/promptStore.test.ts +351 -0
  338. package/src/observability/local/promptStore.ts +665 -0
  339. package/src/observability/local/reviewStore.test.ts +164 -0
  340. package/src/observability/local/reviewStore.ts +264 -0
  341. package/src/observability/local/traceStore.test.ts +264 -0
  342. package/src/observability/local/traceStore.ts +440 -0
  343. package/src/observability/observabilityApp.test.ts +92 -0
  344. package/src/observability/observabilityApp.ts +185 -0
  345. package/src/observability/observabilityAppRegistry.test.ts +30 -0
  346. package/src/observability/observabilityAppRegistry.ts +35 -0
  347. package/src/observability/permissions.test.ts +96 -0
  348. package/src/observability/permissions.ts +78 -0
  349. package/src/observability/reviewPanels.test.ts +35 -0
  350. package/src/observability/reviewPanels.ts +64 -0
  351. package/src/observability/routes/datasets.test.ts +176 -0
  352. package/src/observability/routes/datasets.ts +252 -0
  353. package/src/observability/routes/evaluators.test.ts +119 -0
  354. package/src/observability/routes/evaluators.ts +207 -0
  355. package/src/observability/routes/experiments.test.ts +282 -0
  356. package/src/observability/routes/experiments.ts +152 -0
  357. package/src/observability/routes/observabilityRbac.test.ts +767 -0
  358. package/src/observability/routes/observabilityRouteAccess.ts +106 -0
  359. package/src/observability/routes/prompts.test.ts +320 -0
  360. package/src/observability/routes/prompts.ts +263 -0
  361. package/src/observability/routes/requestAiService.ts +36 -0
  362. package/src/observability/routes/review.test.ts +120 -0
  363. package/src/observability/routes/review.ts +125 -0
  364. package/src/observability/routes/status.test.ts +116 -0
  365. package/src/observability/routes/status.ts +47 -0
  366. package/src/observability/routes/testMultiStage.test.ts +214 -0
  367. package/src/observability/routes/testMultiStage.ts +96 -0
  368. package/src/observability/routes/traces.test.ts +167 -0
  369. package/src/observability/routes/traces.ts +172 -0
  370. package/src/observability/schemaValidation.ts +29 -0
  371. package/src/observability/sopThresholds.ts +20 -0
  372. package/src/observability/status.test.ts +38 -0
  373. package/src/observability/status.ts +56 -0
  374. package/src/observability/testMultiStageSchemas.ts +95 -0
  375. package/src/observability/testMultiStageWorkflow.test.ts +114 -0
  376. package/src/observability/testMultiStageWorkflow.ts +307 -0
  377. package/src/observability/types.ts +262 -0
  378. package/src/routes/files.test.ts +84 -9
  379. package/src/routes/files.ts +23 -7
  380. package/src/routes/gpt.test.ts +3279 -0
  381. package/src/routes/gpt.ts +214 -552
  382. package/src/routes/gptActions.test.ts +241 -0
  383. package/src/routes/gptActions.ts +255 -0
  384. package/src/routes/gptApprovals.test.ts +762 -0
  385. package/src/routes/gptDatasets.test.ts +86 -0
  386. package/src/routes/gptDatasets.ts +104 -0
  387. package/src/routes/gptFiles.test.ts +589 -0
  388. package/src/routes/gptHistories.test.ts +933 -0
  389. package/src/routes/gptHistories.ts +266 -5
  390. package/src/routes/routes.test.ts +676 -5
  391. package/src/service/aiDatasets.test.ts +123 -0
  392. package/src/service/aiDatasets.ts +316 -0
  393. package/src/service/aiService.test.ts +614 -0
  394. package/src/service/aiService.ts +575 -111
  395. package/src/service/askFiles.test.ts +96 -0
  396. package/src/service/askFiles.ts +222 -0
  397. package/src/service/asks.test.ts +946 -0
  398. package/src/service/asks.ts +423 -0
  399. package/src/service/chatTurn.ts +2287 -0
  400. package/src/service/fileStorage.test.ts +11 -0
  401. package/src/service/fileStorage.ts +10 -2
  402. package/src/service/fileUploadsGate.ts +23 -0
  403. package/src/service/fillActionIds.test.ts +65 -0
  404. package/src/service/fillActionIds.ts +98 -0
  405. package/src/service/gemini.test.ts +16 -10
  406. package/src/service/gemini.ts +3 -3
  407. package/src/service/jsonFenceModel.test.ts +47 -0
  408. package/src/service/jsonFenceModel.ts +53 -0
  409. package/src/service/prompts.test.ts +21 -0
  410. package/src/service/prompts.ts +147 -0
  411. package/src/service/sanitizeHtml.test.ts +61 -0
  412. package/src/service/sanitizeHtml.ts +109 -0
  413. package/src/service/vertex.test.ts +49 -49
  414. package/src/service/vertex.ts +7 -8
  415. package/src/tests/bunSetup.test.ts +7 -4
  416. package/src/tests/bunSetup.ts +7 -7
  417. package/src/tests/chatHarness.ts +478 -0
  418. package/src/tests/generateStream.ts +49 -0
  419. package/src/tests/harnessErrors.ts +22 -0
  420. package/src/tests/harnessStreaming.ts +227 -0
  421. package/src/tests/helpers.ts +9 -2
  422. package/src/types/harness.ts +1095 -0
  423. package/src/types/index.ts +237 -12
  424. package/src/types/observability.ts +344 -0
  425. package/src/types/sanitize-html.d.ts +25 -0
@@ -0,0 +1,548 @@
1
+ import {APIError, BackgroundTask} from "@terreno/api";
2
+ import {DateTime} from "luxon";
3
+ import mongoose from "mongoose";
4
+
5
+ import type {
6
+ ExperimentAggregates,
7
+ ExperimentEstimate,
8
+ ExperimentGateResult,
9
+ ExperimentVersionResult,
10
+ ObsExperimentDocument,
11
+ ScoreThreshold,
12
+ } from "../../types/observability";
13
+ import {compileTemplate} from "../compileTemplate";
14
+ import {runEvaluator} from "../evaluate";
15
+ import {SOP_DEFAULT_THRESHOLDS} from "../sopThresholds";
16
+ import type {
17
+ ModelPrice,
18
+ ObservabilityAiServiceFactory,
19
+ ObservabilityGenerateClient,
20
+ } from "../types";
21
+ import type {LocalDatasetStore} from "./datasetStore";
22
+ import type {LocalEvaluatorStore} from "./evaluatorStore";
23
+ import {registerObsExperiment} from "./models/obsExperiment";
24
+ import {registerObsExperimentItem} from "./models/obsExperimentItem";
25
+ import type {LocalPromptStore} from "./promptStore";
26
+
27
+ export interface ExperimentCreateInput {
28
+ datasetId: string;
29
+ evaluatorIds: string[];
30
+ includeUnproofread?: boolean;
31
+ modelOverride?: string;
32
+ name: string;
33
+ promptName: string;
34
+ thresholds?: ScoreThreshold[];
35
+ versions: number[];
36
+ }
37
+
38
+ export interface ExperimentView {
39
+ backgroundTaskId?: string;
40
+ created: string;
41
+ datasetId: string;
42
+ estimate?: ExperimentEstimate;
43
+ evaluatorIds: string[];
44
+ id: string;
45
+ includeUnproofread: boolean;
46
+ items: ExperimentItemView[];
47
+ modelOverride?: string;
48
+ name: string;
49
+ promptName: string;
50
+ results?: ExperimentAggregates;
51
+ status: "completed" | "failed" | "pending" | "running";
52
+ thresholds: ScoreThreshold[];
53
+ updated: string;
54
+ versions: number[];
55
+ }
56
+
57
+ export interface ExperimentItemView {
58
+ datasetItemId: string;
59
+ failed: boolean;
60
+ id: string;
61
+ versionResults: Record<string, ExperimentVersionResult>;
62
+ }
63
+
64
+ export const EXPERIMENT_LIST_DEFAULT_LIMIT = 50;
65
+ const EXPERIMENT_LIST_MAX_LIMIT = 100;
66
+
67
+ export interface ExperimentListResult {
68
+ data: ExperimentView[];
69
+ meta: {limit: number; page: number; total: number};
70
+ }
71
+
72
+ export interface BackgroundTaskRunner {
73
+ enqueue: (params: {execute: () => Promise<void>; taskType: string}) => Promise<{taskId: string}>;
74
+ }
75
+
76
+ export interface ExperimentAiDependencies {
77
+ aiService: ObservabilityGenerateClient;
78
+ aiServiceFactory?: ObservabilityAiServiceFactory;
79
+ priceMap?: Record<string, ModelPrice>;
80
+ }
81
+
82
+ const defaultBackgroundTaskRunner: BackgroundTaskRunner = {
83
+ enqueue: async ({execute, taskType}) => {
84
+ const task = await BackgroundTask.create({
85
+ status: "pending",
86
+ taskType,
87
+ });
88
+ const taskId = String(task._id);
89
+ setImmediate(async () => {
90
+ task.status = "running";
91
+ task.startedAt = DateTime.now().toJSDate();
92
+ await task.save();
93
+ try {
94
+ await execute();
95
+ task.status = "completed";
96
+ task.completedAt = DateTime.now().toJSDate();
97
+ } catch (error) {
98
+ task.status = "failed";
99
+ task.error = error instanceof Error ? error.message : "experiment failed";
100
+ task.completedAt = DateTime.now().toJSDate();
101
+ }
102
+ await task.save();
103
+ });
104
+ return {taskId};
105
+ },
106
+ };
107
+
108
+ const compareGate = (actual: number, op: ScoreThreshold["op"], expected: number): boolean => {
109
+ if (op === "eq") {
110
+ return actual === expected;
111
+ }
112
+ if (op === "gte") {
113
+ return actual >= expected;
114
+ }
115
+ return actual <= expected;
116
+ };
117
+
118
+ const computeGateValue = (
119
+ values: Array<boolean | number | undefined>,
120
+ aggregate: ScoreThreshold["aggregate"]
121
+ ): number | undefined => {
122
+ const filtered = values.filter((value) => {
123
+ return value !== undefined;
124
+ }) as Array<boolean | number>;
125
+ if (filtered.length === 0) {
126
+ return undefined;
127
+ }
128
+ if (aggregate === "trueRate") {
129
+ const trueCount = filtered.filter((value) => {
130
+ return value === true || value === 1;
131
+ }).length;
132
+ return trueCount / filtered.length;
133
+ }
134
+ const numeric = filtered.map((value) => {
135
+ return Number(value);
136
+ });
137
+ return numeric.reduce((sum, value) => sum + value, 0) / numeric.length;
138
+ };
139
+
140
+ const parseGenerationOutput = (output: string): unknown => {
141
+ try {
142
+ return JSON.parse(output);
143
+ } catch {
144
+ return output;
145
+ }
146
+ };
147
+
148
+ const gateBucketKey = (version: number, threshold: ScoreThreshold): string => {
149
+ return `${version}:${threshold.evaluatorName}:${threshold.dimension}:${threshold.aggregate}`;
150
+ };
151
+
152
+ export class LocalExperimentRunner {
153
+ private aiDeps?: ExperimentAiDependencies;
154
+
155
+ constructor(
156
+ private readonly deps: {
157
+ backgroundTaskRunner?: BackgroundTaskRunner;
158
+ datasetStore: LocalDatasetStore;
159
+ evaluatorStore: LocalEvaluatorStore;
160
+ promptStore: LocalPromptStore;
161
+ }
162
+ ) {}
163
+
164
+ configureAi(aiDeps: ExperimentAiDependencies): void {
165
+ this.aiDeps = aiDeps;
166
+ }
167
+
168
+ private requireAiDeps(): ExperimentAiDependencies {
169
+ if (!this.aiDeps?.aiService) {
170
+ throw new APIError({status: 400, title: "experiments require an AI service"});
171
+ }
172
+ return this.aiDeps;
173
+ }
174
+
175
+ private resolveAiClient(modelOverride?: string): ObservabilityGenerateClient {
176
+ const aiDeps = this.requireAiDeps();
177
+ if (!modelOverride) {
178
+ return aiDeps.aiService;
179
+ }
180
+ if (!aiDeps.aiServiceFactory) {
181
+ throw new APIError({
182
+ status: 400,
183
+ title: `modelOverride "${modelOverride}" requires aiServiceFactory on ObservabilityApp`,
184
+ });
185
+ }
186
+ const client = aiDeps.aiServiceFactory(modelOverride);
187
+ if (!client) {
188
+ throw new APIError({
189
+ status: 400,
190
+ title: `Unsupported modelOverride "${modelOverride}"`,
191
+ });
192
+ }
193
+ return client;
194
+ }
195
+
196
+ async estimate(input: {
197
+ datasetId: string;
198
+ evaluatorIds: string[];
199
+ includeUnproofread?: boolean;
200
+ modelOverride?: string;
201
+ versions: number[];
202
+ }): Promise<ExperimentEstimate> {
203
+ const items = await this.deps.datasetStore.listExperimentItems(
204
+ input.datasetId,
205
+ input.includeUnproofread ?? false
206
+ );
207
+ const generations = items.length * input.versions.length;
208
+ const evaluatorCount = input.evaluatorIds.length;
209
+ const secondsPerGeneration = 2;
210
+ const judgeSeconds = generations * evaluatorCount * 0.5;
211
+ const modelId =
212
+ (input.modelOverride
213
+ ? this.resolveAiClient(input.modelOverride).modelId
214
+ : this.aiDeps?.aiService.modelId) ?? "unknown";
215
+ const price = this.aiDeps?.priceMap?.[modelId];
216
+ const tokensPerGeneration = 1000;
217
+ const costUsd =
218
+ price && generations > 0
219
+ ? (price.inputPerMTok * tokensPerGeneration * generations) / 1_000_000
220
+ : undefined;
221
+ return {
222
+ costUsd,
223
+ generations,
224
+ wallClockSeconds: generations * secondsPerGeneration + judgeSeconds,
225
+ };
226
+ }
227
+
228
+ private async assertAutomaticEvaluators(evaluatorIds: string[]): Promise<void> {
229
+ for (const id of evaluatorIds) {
230
+ const evaluator = await this.deps.evaluatorStore.get(id);
231
+ if (evaluator.type !== "llm-judge" && evaluator.type !== "json-assert") {
232
+ throw new APIError({
233
+ status: 400,
234
+ title: "Experiments only accept llm-judge and json-assert evaluators",
235
+ });
236
+ }
237
+ }
238
+ }
239
+
240
+ async create(input: ExperimentCreateInput): Promise<ExperimentView> {
241
+ if (input.versions.length < 2 || input.versions.length > 3) {
242
+ throw new APIError({status: 400, title: "experiments must compare 2–3 prompt versions"});
243
+ }
244
+ await this.assertAutomaticEvaluators(input.evaluatorIds);
245
+ this.resolveAiClient(input.modelOverride);
246
+ await this.deps.datasetStore.get(input.datasetId);
247
+ const estimate = await this.estimate({
248
+ datasetId: input.datasetId,
249
+ evaluatorIds: input.evaluatorIds,
250
+ includeUnproofread: input.includeUnproofread,
251
+ modelOverride: input.modelOverride,
252
+ versions: input.versions,
253
+ });
254
+ const experiment = await registerObsExperiment().create({
255
+ datasetId: new mongoose.Types.ObjectId(input.datasetId),
256
+ estimate,
257
+ evaluatorIds: input.evaluatorIds.map((id) => {
258
+ return new mongoose.Types.ObjectId(id);
259
+ }),
260
+ includeUnproofread: input.includeUnproofread ?? false,
261
+ modelOverride: input.modelOverride,
262
+ name: input.name,
263
+ promptName: input.promptName,
264
+ status: "pending",
265
+ thresholds: input.thresholds ?? SOP_DEFAULT_THRESHOLDS,
266
+ versions: input.versions,
267
+ });
268
+ const runner = this.deps.backgroundTaskRunner ?? defaultBackgroundTaskRunner;
269
+ const {taskId} = await runner.enqueue({
270
+ execute: async () => {
271
+ await this.executeExperiment(String(experiment._id));
272
+ },
273
+ taskType: `obs-experiment-${String(experiment._id)}`,
274
+ });
275
+ experiment.backgroundTaskId = new mongoose.Types.ObjectId(taskId);
276
+ experiment.status = "running";
277
+ await experiment.save();
278
+ return this.get(String(experiment._id));
279
+ }
280
+
281
+ async list(params?: {
282
+ limit?: number;
283
+ page?: number;
284
+ promptName?: string;
285
+ }): Promise<ExperimentListResult> {
286
+ const page = params?.page && params.page > 0 ? params.page : 1;
287
+ const limit =
288
+ params?.limit && params.limit > 0
289
+ ? Math.min(params.limit, EXPERIMENT_LIST_MAX_LIMIT)
290
+ : EXPERIMENT_LIST_DEFAULT_LIMIT;
291
+ const filter: Record<string, unknown> = {};
292
+ if (params?.promptName) {
293
+ filter.promptName = params.promptName;
294
+ }
295
+ const ObsExperiment = registerObsExperiment();
296
+ const total = await ObsExperiment.countDocuments(filter);
297
+ const rows = await ObsExperiment.find(filter)
298
+ .sort({created: -1})
299
+ .skip((page - 1) * limit)
300
+ .limit(limit);
301
+ return {
302
+ data: rows.map((row) => {
303
+ return this.toSummaryView(row);
304
+ }),
305
+ meta: {limit, page, total},
306
+ };
307
+ }
308
+
309
+ async get(id: string): Promise<ExperimentView> {
310
+ const experiment = await registerObsExperiment().findOneOrNone({_id: id});
311
+ if (!experiment) {
312
+ throw new APIError({status: 404, title: `Unknown experiment "${id}"`});
313
+ }
314
+ const items = await registerObsExperimentItem().find({experimentId: experiment._id});
315
+ const sortedItems = items.sort((left, right) => {
316
+ if (left.failed !== right.failed) {
317
+ return left.failed ? -1 : 1;
318
+ }
319
+ return left.created.getTime() - right.created.getTime();
320
+ });
321
+ return {
322
+ ...this.toSummaryView(experiment),
323
+ items: sortedItems.map((row) => {
324
+ return {
325
+ datasetItemId: String(row.datasetItemId),
326
+ failed: row.failed,
327
+ id: String(row._id),
328
+ versionResults: row.versionResults,
329
+ };
330
+ }),
331
+ };
332
+ }
333
+
334
+ private toSummaryView(experiment: ObsExperimentDocument): ExperimentView {
335
+ return {
336
+ backgroundTaskId: experiment.backgroundTaskId
337
+ ? String(experiment.backgroundTaskId)
338
+ : undefined,
339
+ created: DateTime.fromJSDate(experiment.created).toUTC().toISO() ?? "",
340
+ datasetId: String(experiment.datasetId),
341
+ estimate: experiment.estimate,
342
+ evaluatorIds: experiment.evaluatorIds.map((value) => String(value)),
343
+ id: String(experiment._id),
344
+ includeUnproofread: experiment.includeUnproofread,
345
+ items: [],
346
+ modelOverride: experiment.modelOverride,
347
+ name: experiment.name,
348
+ promptName: experiment.promptName,
349
+ results: experiment.results,
350
+ status: experiment.status,
351
+ thresholds: experiment.thresholds,
352
+ updated: DateTime.fromJSDate(experiment.updated).toUTC().toISO() ?? "",
353
+ versions: experiment.versions,
354
+ };
355
+ }
356
+
357
+ async promote(id: string, version: number): Promise<{label: string; outgoingVersion?: number}> {
358
+ const experiment = await this.get(id);
359
+ if (!experiment.results) {
360
+ throw new APIError({status: 409, title: "Experiment has not finished running"});
361
+ }
362
+ if (!experiment.versions.includes(version)) {
363
+ throw new APIError({
364
+ status: 400,
365
+ title: `Version ${version} was not part of experiment ${id}`,
366
+ });
367
+ }
368
+ const failingGate = experiment.results.gates.find((gate) => {
369
+ return gate.version === version && !gate.passed;
370
+ });
371
+ if (failingGate) {
372
+ throw new APIError({
373
+ status: 409,
374
+ title: `Promote blocked: gate failed for v${version} ${failingGate.evaluatorName}.${failingGate.dimension}`,
375
+ });
376
+ }
377
+ return this.deps.promptStore.moveLabel(experiment.promptName, {
378
+ label: "production",
379
+ version,
380
+ });
381
+ }
382
+
383
+ async executeExperiment(id: string): Promise<void> {
384
+ const experiment = await registerObsExperiment().findOneOrNone({_id: id});
385
+ if (!experiment) {
386
+ throw new APIError({status: 404, title: `Unknown experiment "${id}"`});
387
+ }
388
+ const aiService = this.resolveAiClient(experiment.modelOverride);
389
+ const datasetItems = await this.deps.datasetStore.listExperimentItems(
390
+ String(experiment.datasetId),
391
+ experiment.includeUnproofread
392
+ );
393
+ const evaluators = await Promise.all(
394
+ experiment.evaluatorIds.map((evaluatorId) => {
395
+ return this.deps.evaluatorStore.get(String(evaluatorId));
396
+ })
397
+ );
398
+ const evaluatorById = new Map(
399
+ evaluators.map((evaluator) => {
400
+ return [evaluator.id, evaluator];
401
+ })
402
+ );
403
+ const judgeSchemas = new Map<string, Record<string, unknown> | undefined>();
404
+ for (const evaluator of evaluators) {
405
+ if (evaluator.type === "llm-judge" && evaluator.judgePromptName) {
406
+ const version = await this.deps.promptStore.getVersionByLabel(evaluator.judgePromptName);
407
+ judgeSchemas.set(evaluator.id, version?.outputSchema);
408
+ }
409
+ }
410
+
411
+ let completed = 0;
412
+ const gateBuckets = new Map<string, Array<boolean | number | undefined>>();
413
+ const lowConfidenceItemIds: string[] = [];
414
+ const outlierItemIds: string[] = [];
415
+
416
+ for (const datasetItem of datasetItems) {
417
+ const versionResults: Record<string, ExperimentVersionResult> = {};
418
+ let rowFailed = false;
419
+
420
+ for (const versionNumber of experiment.versions) {
421
+ const promptVersion = await this.deps.promptStore.getVersionByNumber(
422
+ experiment.promptName,
423
+ versionNumber
424
+ );
425
+ if (!promptVersion) {
426
+ throw new APIError({
427
+ status: 404,
428
+ title: `Unknown version ${versionNumber} for prompt "${experiment.promptName}"`,
429
+ });
430
+ }
431
+ const variables =
432
+ promptVersion.variables.length > 0 &&
433
+ datasetItem.input &&
434
+ typeof datasetItem.input === "object" &&
435
+ !Array.isArray(datasetItem.input)
436
+ ? (datasetItem.input as Record<string, string>)
437
+ : {input: JSON.stringify(datasetItem.input)};
438
+ const compiled = compileTemplate(promptVersion.template ?? "", variables);
439
+ const outputText = await aiService.generateText({
440
+ prompt: compiled,
441
+ skipTrace: true,
442
+ systemPrompt: promptVersion.system,
443
+ });
444
+ const parsedOutput = parseGenerationOutput(outputText);
445
+ const evaluatorScores: ExperimentVersionResult["evaluatorScores"] = {};
446
+ for (const evaluatorId of experiment.evaluatorIds) {
447
+ const evaluator = evaluatorById.get(String(evaluatorId));
448
+ if (!evaluator) {
449
+ continue;
450
+ }
451
+ const result = await runEvaluator(
452
+ {
453
+ evaluator,
454
+ expectedOutput: datasetItem.expectedOutput,
455
+ input: datasetItem.input,
456
+ output: parsedOutput,
457
+ outputSchema: promptVersion.outputSchema,
458
+ },
459
+ {
460
+ ai: aiService,
461
+ judgeOutputSchema: judgeSchemas.get(evaluator.id),
462
+ }
463
+ );
464
+ evaluatorScores[evaluator.id] = {
465
+ confidence: result.confidence,
466
+ error: result.error,
467
+ scores: result.scores,
468
+ };
469
+ if (result.error) {
470
+ rowFailed = true;
471
+ outlierItemIds.push(datasetItem.id);
472
+ }
473
+ if (
474
+ result.confidence !== undefined &&
475
+ result.confidence < evaluator.confidenceAlertBelow
476
+ ) {
477
+ lowConfidenceItemIds.push(datasetItem.id);
478
+ outlierItemIds.push(datasetItem.id);
479
+ }
480
+ for (const threshold of experiment.thresholds) {
481
+ if (threshold.evaluatorName !== evaluator.name || !result.scores) {
482
+ continue;
483
+ }
484
+ const bucketKey = gateBucketKey(versionNumber, threshold);
485
+ const raw = result.scores?.[threshold.dimension];
486
+ const bucket = gateBuckets.get(bucketKey) ?? [];
487
+ if (typeof raw === "boolean" || typeof raw === "number") {
488
+ bucket.push(raw);
489
+ }
490
+ gateBuckets.set(bucketKey, bucket);
491
+ }
492
+ }
493
+ versionResults[String(versionNumber)] = {
494
+ evaluatorScores,
495
+ output: parsedOutput,
496
+ };
497
+ }
498
+
499
+ await registerObsExperimentItem().create({
500
+ datasetItemId: new mongoose.Types.ObjectId(datasetItem.id),
501
+ experimentId: experiment._id,
502
+ failed: rowFailed,
503
+ versionResults,
504
+ });
505
+ completed += 1;
506
+ experiment.results = {
507
+ gates: this.computeGates(experiment.versions, experiment.thresholds, gateBuckets),
508
+ lowConfidenceItemIds: [...new Set(lowConfidenceItemIds)],
509
+ outlierItemIds: [...new Set(outlierItemIds)],
510
+ progress: {completed, total: datasetItems.length},
511
+ totalCostUsd: experiment.estimate?.costUsd,
512
+ };
513
+ await experiment.save();
514
+ }
515
+
516
+ experiment.status = "completed";
517
+ experiment.results = {
518
+ ...experiment.results,
519
+ gates: this.computeGates(experiment.versions, experiment.thresholds, gateBuckets),
520
+ lowConfidenceItemIds: [...new Set(lowConfidenceItemIds)],
521
+ outlierItemIds: [...new Set(outlierItemIds)],
522
+ progress: {completed: datasetItems.length, total: datasetItems.length},
523
+ totalCostUsd: experiment.estimate?.costUsd,
524
+ };
525
+ await experiment.save();
526
+ }
527
+
528
+ private computeGates(
529
+ versions: number[],
530
+ thresholds: ScoreThreshold[],
531
+ gateBuckets: Map<string, Array<boolean | number | undefined>>
532
+ ): ExperimentGateResult[] {
533
+ return versions.flatMap((version) => {
534
+ return thresholds.map((threshold) => {
535
+ const bucketKey = gateBucketKey(version, threshold);
536
+ const actual = computeGateValue(gateBuckets.get(bucketKey) ?? [], threshold.aggregate);
537
+ const passed =
538
+ actual === undefined ? false : compareGate(actual, threshold.op, threshold.value);
539
+ return {
540
+ ...threshold,
541
+ actual,
542
+ passed,
543
+ version,
544
+ };
545
+ });
546
+ });
547
+ }
548
+ }
@@ -0,0 +1,88 @@
1
+ import type {ModelPrice, ObservabilityPlugin} from "../types";
2
+ import {LocalDatasetStore} from "./datasetStore";
3
+ import {LocalEvaluatorStore} from "./evaluatorStore";
4
+ import {type BackgroundTaskRunner, LocalExperimentRunner} from "./experimentRunner";
5
+ import {registerObsDataset} from "./models/obsDataset";
6
+ import {registerObsDatasetItem} from "./models/obsDatasetItem";
7
+ import {registerObsEvaluator} from "./models/obsEvaluator";
8
+ import {registerObsExperiment} from "./models/obsExperiment";
9
+ import {registerObsExperimentItem} from "./models/obsExperimentItem";
10
+ import {registerObsPrompt} from "./models/obsPrompt";
11
+ import {registerObsPromptLabel} from "./models/obsPromptLabel";
12
+ import {registerObsPromptVersion} from "./models/obsPromptVersion";
13
+ import {registerObsReviewItem} from "./models/obsReviewItem";
14
+ import {registerObsScore} from "./models/obsScore";
15
+ import {registerObsSpan} from "./models/obsSpan";
16
+ import {registerObsTrace} from "./models/obsTrace";
17
+ import {LocalPromptStore} from "./promptStore";
18
+ import {LocalScoreSink, LocalTraceSink, LocalTraceStore} from "./traceStore";
19
+
20
+ export interface LocalObservabilityPluginBundle {
21
+ datasetStore: LocalDatasetStore;
22
+ evaluatorStore: LocalEvaluatorStore;
23
+ experimentRunner: LocalExperimentRunner;
24
+ plugin: ObservabilityPlugin;
25
+ promptStore: LocalPromptStore;
26
+ traceStore: LocalTraceStore;
27
+ }
28
+
29
+ const registerLocalObservabilityModels = (): void => {
30
+ registerObsPrompt();
31
+ registerObsPromptVersion();
32
+ registerObsPromptLabel();
33
+ registerObsTrace();
34
+ registerObsSpan();
35
+ registerObsScore();
36
+ registerObsEvaluator();
37
+ registerObsReviewItem();
38
+ registerObsDataset();
39
+ registerObsDatasetItem();
40
+ registerObsExperiment();
41
+ registerObsExperimentItem();
42
+ };
43
+
44
+ export const createLocalObservabilityBundle = (
45
+ options: {backgroundTaskRunner?: BackgroundTaskRunner; priceMap?: Record<string, ModelPrice>} = {}
46
+ ): LocalObservabilityPluginBundle => {
47
+ registerLocalObservabilityModels();
48
+ const promptStore = new LocalPromptStore();
49
+ const traces = new LocalTraceStore();
50
+ const datasetStore = new LocalDatasetStore(promptStore);
51
+ const evaluatorStore = new LocalEvaluatorStore(promptStore);
52
+ const experimentRunner = new LocalExperimentRunner({
53
+ backgroundTaskRunner: options.backgroundTaskRunner,
54
+ datasetStore,
55
+ evaluatorStore,
56
+ promptStore,
57
+ });
58
+ return {
59
+ datasetStore,
60
+ evaluatorStore,
61
+ experimentRunner,
62
+ plugin: {
63
+ capabilities: new Set([
64
+ "datasets",
65
+ "experiments",
66
+ "prompts",
67
+ "reviewQueue",
68
+ "scores",
69
+ "traces",
70
+ ]),
71
+ datasetStore,
72
+ experimentRunner,
73
+ id: "local",
74
+ promptRegistry: promptStore,
75
+ reviewQueue: {},
76
+ scoreSink: new LocalScoreSink(traces),
77
+ traceSink: new LocalTraceSink(traces),
78
+ },
79
+ promptStore,
80
+ traceStore: traces,
81
+ };
82
+ };
83
+
84
+ export const createLocalObservabilityPlugin = (
85
+ options: {backgroundTaskRunner?: BackgroundTaskRunner; priceMap?: Record<string, ModelPrice>} = {}
86
+ ): ObservabilityPlugin => {
87
+ return createLocalObservabilityBundle(options).plugin;
88
+ };
@@ -0,0 +1,38 @@
1
+ import {createdUpdatedPlugin, findExactlyOne, findOneOrNone, isDeletedPlugin} from "@terreno/api";
2
+ import mongoose from "mongoose";
3
+
4
+ import type {ObsDatasetDocument, ObsDatasetModel} from "../../../types/observability";
5
+
6
+ const obsDatasetSchema = new mongoose.Schema<ObsDatasetDocument, ObsDatasetModel>(
7
+ {
8
+ description: {description: "Operator-facing dataset description", type: String},
9
+ expectedOutputSchema: {
10
+ description: "Optional JSON Schema for expected output labels",
11
+ type: mongoose.Schema.Types.Mixed,
12
+ },
13
+ inputSchemaPromptName: {
14
+ description: "Optional prompt whose production inputSchema validates imported items",
15
+ type: String,
16
+ },
17
+ name: {description: "Unique dataset name", required: true, type: String},
18
+ tags: {
19
+ default: [],
20
+ description: "Free-form dataset tags for filtering",
21
+ type: [String],
22
+ },
23
+ },
24
+ {strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
25
+ );
26
+
27
+ obsDatasetSchema.plugin(createdUpdatedPlugin);
28
+ obsDatasetSchema.plugin(isDeletedPlugin);
29
+ obsDatasetSchema.plugin(findOneOrNone);
30
+ obsDatasetSchema.plugin(findExactlyOne);
31
+ obsDatasetSchema.index({name: 1}, {unique: true});
32
+
33
+ export const registerObsDataset = (): ObsDatasetModel => {
34
+ if (mongoose.models.ObsDataset) {
35
+ return mongoose.models.ObsDataset as ObsDatasetModel;
36
+ }
37
+ return mongoose.model<ObsDatasetDocument, ObsDatasetModel>("ObsDataset", obsDatasetSchema);
38
+ };