@terreno/ai 57.9.0 → 57.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (412) hide show
  1. package/README.md +1 -0
  2. package/bunfig.toml +1 -1
  3. package/dist/aiApp.d.ts +8 -2
  4. package/dist/aiApp.js +12 -6
  5. package/dist/harness/adminScreens.d.ts +6 -0
  6. package/dist/harness/adminScreens.js +11 -0
  7. package/dist/harness/agentLoop.d.ts +61 -0
  8. package/dist/harness/agentLoop.js +798 -0
  9. package/dist/harness/agentTaskNames.d.ts +4 -0
  10. package/dist/harness/agentTaskNames.js +7 -0
  11. package/dist/harness/approvalPolicy.d.ts +3 -0
  12. package/dist/harness/approvalPolicy.js +23 -0
  13. package/dist/harness/approvals.d.ts +101 -0
  14. package/dist/harness/approvals.js +468 -0
  15. package/dist/harness/commit.d.ts +238 -0
  16. package/dist/harness/commit.js +678 -0
  17. package/dist/harness/conversation.d.ts +75 -0
  18. package/dist/harness/conversation.js +397 -0
  19. package/dist/harness/defineAgent.d.ts +9 -0
  20. package/dist/harness/defineAgent.js +65 -0
  21. package/dist/harness/defineTask.d.ts +11 -0
  22. package/dist/harness/defineTask.js +59 -0
  23. package/dist/harness/defineTool.d.ts +7 -0
  24. package/dist/harness/defineTool.js +36 -0
  25. package/dist/harness/definitionError.d.ts +8 -0
  26. package/dist/harness/definitionError.js +15 -0
  27. package/dist/harness/errors.d.ts +127 -0
  28. package/dist/harness/errors.js +101 -0
  29. package/dist/harness/eventHub.d.ts +45 -0
  30. package/dist/harness/eventHub.js +135 -0
  31. package/dist/harness/events.d.ts +95 -0
  32. package/dist/harness/events.js +267 -0
  33. package/dist/harness/executionEnv.d.ts +31 -0
  34. package/dist/harness/executionEnv.js +2 -0
  35. package/dist/harness/extensions.d.ts +82 -0
  36. package/dist/harness/extensions.js +274 -0
  37. package/dist/harness/harness.d.ts +163 -0
  38. package/dist/harness/harness.js +520 -0
  39. package/dist/harness/harnessApp.d.ts +44 -0
  40. package/dist/harness/harnessApp.js +62 -0
  41. package/dist/harness/internalRuntime.d.ts +16 -0
  42. package/dist/harness/internalRuntime.js +18 -0
  43. package/dist/harness/leases.d.ts +35 -0
  44. package/dist/harness/leases.js +86 -0
  45. package/dist/harness/memo.d.ts +24 -0
  46. package/dist/harness/memo.js +78 -0
  47. package/dist/harness/modelCall.d.ts +45 -0
  48. package/dist/harness/modelCall.js +147 -0
  49. package/dist/harness/models/harnessApproval.d.ts +2 -0
  50. package/dist/harness/models/harnessApproval.js +100 -0
  51. package/dist/harness/models/harnessConversation.d.ts +2 -0
  52. package/dist/harness/models/harnessConversation.js +117 -0
  53. package/dist/harness/models/harnessEvent.d.ts +3 -0
  54. package/dist/harness/models/harnessEvent.js +73 -0
  55. package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
  56. package/dist/harness/models/harnessInboxEvent.js +58 -0
  57. package/dist/harness/models/harnessMemo.d.ts +2 -0
  58. package/dist/harness/models/harnessMemo.js +42 -0
  59. package/dist/harness/models/harnessMessage.d.ts +2 -0
  60. package/dist/harness/models/harnessMessage.js +73 -0
  61. package/dist/harness/models/harnessOwner.d.ts +2 -0
  62. package/dist/harness/models/harnessOwner.js +36 -0
  63. package/dist/harness/models/harnessTask.d.ts +2 -0
  64. package/dist/harness/models/harnessTask.js +186 -0
  65. package/dist/harness/ownership.d.ts +89 -0
  66. package/dist/harness/ownership.js +438 -0
  67. package/dist/harness/registry.d.ts +29 -0
  68. package/dist/harness/registry.js +93 -0
  69. package/dist/harness/retryBackoff.d.ts +31 -0
  70. package/dist/harness/retryBackoff.js +50 -0
  71. package/dist/harness/routes/approvals.d.ts +13 -0
  72. package/dist/harness/routes/approvals.js +93 -0
  73. package/dist/harness/routes/conversations.d.ts +11 -0
  74. package/dist/harness/routes/conversations.js +66 -0
  75. package/dist/harness/routes/events.d.ts +19 -0
  76. package/dist/harness/routes/events.js +215 -0
  77. package/dist/harness/routes/tasks.d.ts +11 -0
  78. package/dist/harness/routes/tasks.js +108 -0
  79. package/dist/harness/runners/inProcessRunner.d.ts +57 -0
  80. package/dist/harness/runners/inProcessRunner.js +196 -0
  81. package/dist/harness/runtime.d.ts +34 -0
  82. package/dist/harness/runtime.js +551 -0
  83. package/dist/harness/subagent.d.ts +53 -0
  84. package/dist/harness/subagent.js +134 -0
  85. package/dist/harness/suspend.d.ts +8 -0
  86. package/dist/harness/suspend.js +16 -0
  87. package/dist/harness/transaction.d.ts +3 -0
  88. package/dist/harness/transaction.js +22 -0
  89. package/dist/harness/waits.d.ts +100 -0
  90. package/dist/harness/waits.js +214 -0
  91. package/dist/index.d.ts +19 -1
  92. package/dist/index.js +45 -2
  93. package/dist/models/aiDataset.d.ts +2 -0
  94. package/dist/models/aiDataset.js +67 -0
  95. package/dist/models/aiRequest.js +1 -1
  96. package/dist/models/gptHistory.js +73 -0
  97. package/dist/observability/adminScreens.d.ts +5 -0
  98. package/dist/observability/adminScreens.js +32 -0
  99. package/dist/observability/compileTemplate.d.ts +1 -0
  100. package/dist/observability/compileTemplate.js +13 -0
  101. package/dist/observability/datasetImport.d.ts +11 -0
  102. package/dist/observability/datasetImport.js +185 -0
  103. package/dist/observability/evaluate.d.ts +31 -0
  104. package/dist/observability/evaluate.js +200 -0
  105. package/dist/observability/evaluatorTemplates.d.ts +17 -0
  106. package/dist/observability/evaluatorTemplates.js +111 -0
  107. package/dist/observability/jsonPath.d.ts +2 -0
  108. package/dist/observability/jsonPath.js +57 -0
  109. package/dist/observability/local/datasetStore.d.ts +87 -0
  110. package/dist/observability/local/datasetStore.js +326 -0
  111. package/dist/observability/local/evaluatorStore.d.ts +44 -0
  112. package/dist/observability/local/evaluatorStore.js +182 -0
  113. package/dist/observability/local/experimentRunner.d.ts +96 -0
  114. package/dist/observability/local/experimentRunner.js +419 -0
  115. package/dist/observability/local/localPlugin.d.ts +22 -0
  116. package/dist/observability/local/localPlugin.js +76 -0
  117. package/dist/observability/local/models/obsDataset.d.ts +2 -0
  118. package/dist/observability/local/models/obsDataset.js +37 -0
  119. package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
  120. package/dist/observability/local/models/obsDatasetItem.js +76 -0
  121. package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
  122. package/dist/observability/local/models/obsEvaluator.js +98 -0
  123. package/dist/observability/local/models/obsExperiment.d.ts +2 -0
  124. package/dist/observability/local/models/obsExperiment.js +87 -0
  125. package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
  126. package/dist/observability/local/models/obsExperimentItem.js +45 -0
  127. package/dist/observability/local/models/obsPrompt.d.ts +2 -0
  128. package/dist/observability/local/models/obsPrompt.js +42 -0
  129. package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
  130. package/dist/observability/local/models/obsPromptLabel.js +39 -0
  131. package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
  132. package/dist/observability/local/models/obsPromptVersion.js +96 -0
  133. package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
  134. package/dist/observability/local/models/obsReviewItem.js +75 -0
  135. package/dist/observability/local/models/obsScore.d.ts +2 -0
  136. package/dist/observability/local/models/obsScore.js +58 -0
  137. package/dist/observability/local/models/obsSpan.d.ts +2 -0
  138. package/dist/observability/local/models/obsSpan.js +64 -0
  139. package/dist/observability/local/models/obsTrace.d.ts +2 -0
  140. package/dist/observability/local/models/obsTrace.js +93 -0
  141. package/dist/observability/local/promptStore.d.ts +180 -0
  142. package/dist/observability/local/promptStore.js +444 -0
  143. package/dist/observability/local/reviewStore.d.ts +50 -0
  144. package/dist/observability/local/reviewStore.js +203 -0
  145. package/dist/observability/local/traceStore.d.ts +103 -0
  146. package/dist/observability/local/traceStore.js +325 -0
  147. package/dist/observability/observabilityApp.d.ts +24 -0
  148. package/dist/observability/observabilityApp.js +157 -0
  149. package/dist/observability/observabilityAppRegistry.d.ts +14 -0
  150. package/dist/observability/observabilityAppRegistry.js +16 -0
  151. package/dist/observability/permissions.d.ts +7 -0
  152. package/dist/observability/permissions.js +60 -0
  153. package/dist/observability/reviewPanels.d.ts +18 -0
  154. package/dist/observability/reviewPanels.js +46 -0
  155. package/dist/observability/routes/datasets.d.ts +7 -0
  156. package/dist/observability/routes/datasets.js +149 -0
  157. package/dist/observability/routes/evaluators.d.ts +7 -0
  158. package/dist/observability/routes/evaluators.js +123 -0
  159. package/dist/observability/routes/experiments.d.ts +7 -0
  160. package/dist/observability/routes/experiments.js +95 -0
  161. package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
  162. package/dist/observability/routes/observabilityRouteAccess.js +67 -0
  163. package/dist/observability/routes/prompts.d.ts +12 -0
  164. package/dist/observability/routes/prompts.js +172 -0
  165. package/dist/observability/routes/requestAiService.d.ts +13 -0
  166. package/dist/observability/routes/requestAiService.js +21 -0
  167. package/dist/observability/routes/review.d.ts +7 -0
  168. package/dist/observability/routes/review.js +79 -0
  169. package/dist/observability/routes/status.d.ts +5 -0
  170. package/dist/observability/routes/status.js +31 -0
  171. package/dist/observability/routes/testMultiStage.d.ts +9 -0
  172. package/dist/observability/routes/testMultiStage.js +67 -0
  173. package/dist/observability/routes/traces.d.ts +7 -0
  174. package/dist/observability/routes/traces.js +129 -0
  175. package/dist/observability/schemaValidation.d.ts +5 -0
  176. package/dist/observability/schemaValidation.js +26 -0
  177. package/dist/observability/sopThresholds.d.ts +2 -0
  178. package/dist/observability/sopThresholds.js +21 -0
  179. package/dist/observability/status.d.ts +23 -0
  180. package/dist/observability/status.js +34 -0
  181. package/dist/observability/testMultiStageSchemas.d.ts +10 -0
  182. package/dist/observability/testMultiStageSchemas.js +79 -0
  183. package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
  184. package/dist/observability/testMultiStageWorkflow.js +218 -0
  185. package/dist/observability/types.d.ts +159 -0
  186. package/dist/observability/types.js +67 -0
  187. package/dist/routes/files.js +18 -7
  188. package/dist/routes/gpt.js +79 -689
  189. package/dist/routes/gptActions.d.ts +9 -0
  190. package/dist/routes/gptActions.js +212 -0
  191. package/dist/routes/gptDatasets.d.ts +5 -0
  192. package/dist/routes/gptDatasets.js +94 -0
  193. package/dist/routes/gptHistories.d.ts +3 -1
  194. package/dist/routes/gptHistories.js +188 -2
  195. package/dist/service/aiDatasets.d.ts +57 -0
  196. package/dist/service/aiDatasets.js +253 -0
  197. package/dist/service/aiService.d.ts +50 -1
  198. package/dist/service/aiService.js +426 -77
  199. package/dist/service/askFiles.d.ts +49 -0
  200. package/dist/service/askFiles.js +162 -0
  201. package/dist/service/asks.d.ts +76 -0
  202. package/dist/service/asks.js +284 -0
  203. package/dist/service/chatTurn.d.ts +128 -0
  204. package/dist/service/chatTurn.js +1674 -0
  205. package/dist/service/fileStorage.d.ts +4 -0
  206. package/dist/service/fileStorage.js +7 -2
  207. package/dist/service/fillActionIds.d.ts +6 -0
  208. package/dist/service/fillActionIds.js +79 -0
  209. package/dist/service/jsonFenceModel.d.ts +7 -0
  210. package/dist/service/jsonFenceModel.js +45 -0
  211. package/dist/service/prompts.d.ts +52 -0
  212. package/dist/service/prompts.js +96 -1
  213. package/dist/service/sanitizeHtml.d.ts +11 -0
  214. package/dist/service/sanitizeHtml.js +108 -0
  215. package/dist/types/harness.d.ts +934 -0
  216. package/dist/types/harness.js +133 -0
  217. package/dist/types/index.d.ts +192 -14
  218. package/dist/types/index.js +2 -0
  219. package/dist/types/observability.d.ts +288 -0
  220. package/dist/types/observability.js +2 -0
  221. package/package.json +17 -5
  222. package/src/aiApp.test.ts +207 -1
  223. package/src/aiApp.ts +19 -6
  224. package/src/harness/adminScreens.test.ts +34 -0
  225. package/src/harness/adminScreens.ts +11 -0
  226. package/src/harness/agentLoop.test.ts +1034 -0
  227. package/src/harness/agentLoop.ts +1082 -0
  228. package/src/harness/agentRecovery.test.ts +380 -0
  229. package/src/harness/agentTaskNames.ts +4 -0
  230. package/src/harness/approvalPolicy.ts +28 -0
  231. package/src/harness/approvals.test.ts +1094 -0
  232. package/src/harness/approvals.ts +644 -0
  233. package/src/harness/commit.ts +1049 -0
  234. package/src/harness/conversation.ts +503 -0
  235. package/src/harness/defineAgent.test.ts +92 -0
  236. package/src/harness/defineAgent.ts +82 -0
  237. package/src/harness/defineTask.test.ts +59 -0
  238. package/src/harness/defineTask.ts +76 -0
  239. package/src/harness/defineTool.ts +45 -0
  240. package/src/harness/definitionError.ts +13 -0
  241. package/src/harness/errors.test.ts +90 -0
  242. package/src/harness/errors.ts +119 -0
  243. package/src/harness/eventHub.test.ts +124 -0
  244. package/src/harness/eventHub.ts +159 -0
  245. package/src/harness/events.test.ts +757 -0
  246. package/src/harness/events.ts +384 -0
  247. package/src/harness/executionEnv.ts +34 -0
  248. package/src/harness/extensions.test.ts +1218 -0
  249. package/src/harness/extensions.ts +382 -0
  250. package/src/harness/harness.test.ts +693 -0
  251. package/src/harness/harness.ts +767 -0
  252. package/src/harness/harnessApp.ts +76 -0
  253. package/src/harness/internalRuntime.ts +44 -0
  254. package/src/harness/leases.ts +132 -0
  255. package/src/harness/memo.ts +107 -0
  256. package/src/harness/modelCall.test.ts +134 -0
  257. package/src/harness/modelCall.ts +200 -0
  258. package/src/harness/models/harnessApproval.ts +109 -0
  259. package/src/harness/models/harnessConversation.ts +142 -0
  260. package/src/harness/models/harnessEvent.ts +92 -0
  261. package/src/harness/models/harnessInboxEvent.ts +73 -0
  262. package/src/harness/models/harnessMemo.ts +43 -0
  263. package/src/harness/models/harnessMessage.ts +85 -0
  264. package/src/harness/models/harnessModels.test.ts +60 -0
  265. package/src/harness/models/harnessOwner.ts +40 -0
  266. package/src/harness/models/harnessTask.ts +201 -0
  267. package/src/harness/ownership.test.ts +1087 -0
  268. package/src/harness/ownership.ts +618 -0
  269. package/src/harness/recovery.test.ts +912 -0
  270. package/src/harness/registry.ts +124 -0
  271. package/src/harness/retries.test.ts +292 -0
  272. package/src/harness/retryBackoff.test.ts +100 -0
  273. package/src/harness/retryBackoff.ts +77 -0
  274. package/src/harness/routes/approvals.ts +123 -0
  275. package/src/harness/routes/conversations.test.ts +518 -0
  276. package/src/harness/routes/conversations.ts +102 -0
  277. package/src/harness/routes/events.ts +280 -0
  278. package/src/harness/routes/tasks.test.ts +198 -0
  279. package/src/harness/routes/tasks.ts +146 -0
  280. package/src/harness/runners/inProcessRunner.test.ts +402 -0
  281. package/src/harness/runners/inProcessRunner.ts +220 -0
  282. package/src/harness/runtime.ts +714 -0
  283. package/src/harness/subagent.test.ts +696 -0
  284. package/src/harness/subagent.ts +192 -0
  285. package/src/harness/suspend.ts +13 -0
  286. package/src/harness/transaction.ts +17 -0
  287. package/src/harness/versioning.test.ts +288 -0
  288. package/src/harness/waits.test.ts +693 -0
  289. package/src/harness/waits.ts +356 -0
  290. package/src/index.test.ts +13 -0
  291. package/src/index.ts +70 -0
  292. package/src/isolated/adminScreensConfig.isolated.ts +143 -0
  293. package/src/models/aiDataset.test.ts +34 -0
  294. package/src/models/aiDataset.ts +78 -0
  295. package/src/models/aiRequest.ts +2 -1
  296. package/src/models/gptHistory.test.ts +156 -0
  297. package/src/models/gptHistory.ts +89 -1
  298. package/src/observability/adminScreens.test.ts +71 -0
  299. package/src/observability/adminScreens.ts +42 -0
  300. package/src/observability/compileTemplate.test.ts +14 -0
  301. package/src/observability/compileTemplate.ts +10 -0
  302. package/src/observability/datasetImport.test.ts +95 -0
  303. package/src/observability/datasetImport.ts +220 -0
  304. package/src/observability/evaluate.test.ts +549 -0
  305. package/src/observability/evaluate.ts +254 -0
  306. package/src/observability/evaluatorTemplates.ts +124 -0
  307. package/src/observability/jsonPath.test.ts +52 -0
  308. package/src/observability/jsonPath.ts +60 -0
  309. package/src/observability/local/datasetStore.test.ts +230 -0
  310. package/src/observability/local/datasetStore.ts +452 -0
  311. package/src/observability/local/evaluatorStore.test.ts +138 -0
  312. package/src/observability/local/evaluatorStore.ts +224 -0
  313. package/src/observability/local/experimentRunner.test.ts +489 -0
  314. package/src/observability/local/experimentRunner.ts +548 -0
  315. package/src/observability/local/localPlugin.ts +88 -0
  316. package/src/observability/local/models/obsDataset.ts +38 -0
  317. package/src/observability/local/models/obsDatasetItem.ts +80 -0
  318. package/src/observability/local/models/obsEvaluator.ts +102 -0
  319. package/src/observability/local/models/obsExperiment.ts +91 -0
  320. package/src/observability/local/models/obsExperimentItem.ts +52 -0
  321. package/src/observability/local/models/obsPrompt.ts +43 -0
  322. package/src/observability/local/models/obsPromptLabel.ts +43 -0
  323. package/src/observability/local/models/obsPromptVersion.ts +107 -0
  324. package/src/observability/local/models/obsReviewItem.ts +79 -0
  325. package/src/observability/local/models/obsScore.ts +59 -0
  326. package/src/observability/local/models/obsSpan.ts +65 -0
  327. package/src/observability/local/models/obsTrace.ts +94 -0
  328. package/src/observability/local/obsModels.test.ts +119 -0
  329. package/src/observability/local/promptStore.test.ts +351 -0
  330. package/src/observability/local/promptStore.ts +665 -0
  331. package/src/observability/local/reviewStore.test.ts +164 -0
  332. package/src/observability/local/reviewStore.ts +264 -0
  333. package/src/observability/local/traceStore.test.ts +264 -0
  334. package/src/observability/local/traceStore.ts +440 -0
  335. package/src/observability/observabilityApp.test.ts +92 -0
  336. package/src/observability/observabilityApp.ts +185 -0
  337. package/src/observability/observabilityAppRegistry.test.ts +30 -0
  338. package/src/observability/observabilityAppRegistry.ts +35 -0
  339. package/src/observability/permissions.test.ts +96 -0
  340. package/src/observability/permissions.ts +78 -0
  341. package/src/observability/reviewPanels.test.ts +35 -0
  342. package/src/observability/reviewPanels.ts +64 -0
  343. package/src/observability/routes/datasets.test.ts +176 -0
  344. package/src/observability/routes/datasets.ts +252 -0
  345. package/src/observability/routes/evaluators.test.ts +119 -0
  346. package/src/observability/routes/evaluators.ts +207 -0
  347. package/src/observability/routes/experiments.test.ts +282 -0
  348. package/src/observability/routes/experiments.ts +152 -0
  349. package/src/observability/routes/observabilityRbac.test.ts +767 -0
  350. package/src/observability/routes/observabilityRouteAccess.ts +106 -0
  351. package/src/observability/routes/prompts.test.ts +320 -0
  352. package/src/observability/routes/prompts.ts +263 -0
  353. package/src/observability/routes/requestAiService.ts +36 -0
  354. package/src/observability/routes/review.test.ts +120 -0
  355. package/src/observability/routes/review.ts +125 -0
  356. package/src/observability/routes/status.test.ts +116 -0
  357. package/src/observability/routes/status.ts +47 -0
  358. package/src/observability/routes/testMultiStage.test.ts +214 -0
  359. package/src/observability/routes/testMultiStage.ts +96 -0
  360. package/src/observability/routes/traces.test.ts +167 -0
  361. package/src/observability/routes/traces.ts +172 -0
  362. package/src/observability/schemaValidation.ts +29 -0
  363. package/src/observability/sopThresholds.ts +20 -0
  364. package/src/observability/status.test.ts +38 -0
  365. package/src/observability/status.ts +56 -0
  366. package/src/observability/testMultiStageSchemas.ts +95 -0
  367. package/src/observability/testMultiStageWorkflow.test.ts +114 -0
  368. package/src/observability/testMultiStageWorkflow.ts +307 -0
  369. package/src/observability/types.ts +262 -0
  370. package/src/routes/files.test.ts +63 -9
  371. package/src/routes/files.ts +21 -7
  372. package/src/routes/gpt.test.ts +3279 -0
  373. package/src/routes/gpt.ts +91 -816
  374. package/src/routes/gptActions.test.ts +241 -0
  375. package/src/routes/gptActions.ts +255 -0
  376. package/src/routes/gptApprovals.test.ts +762 -0
  377. package/src/routes/gptDatasets.test.ts +86 -0
  378. package/src/routes/gptDatasets.ts +104 -0
  379. package/src/routes/gptFiles.test.ts +589 -0
  380. package/src/routes/gptHistories.test.ts +933 -0
  381. package/src/routes/gptHistories.ts +266 -5
  382. package/src/routes/routes.test.ts +222 -2
  383. package/src/service/aiDatasets.test.ts +123 -0
  384. package/src/service/aiDatasets.ts +316 -0
  385. package/src/service/aiService.test.ts +593 -0
  386. package/src/service/aiService.ts +568 -111
  387. package/src/service/askFiles.test.ts +96 -0
  388. package/src/service/askFiles.ts +222 -0
  389. package/src/service/asks.test.ts +946 -0
  390. package/src/service/asks.ts +423 -0
  391. package/src/service/chatTurn.ts +2287 -0
  392. package/src/service/fileStorage.test.ts +11 -0
  393. package/src/service/fileStorage.ts +10 -2
  394. package/src/service/fillActionIds.test.ts +65 -0
  395. package/src/service/fillActionIds.ts +98 -0
  396. package/src/service/jsonFenceModel.test.ts +47 -0
  397. package/src/service/jsonFenceModel.ts +53 -0
  398. package/src/service/prompts.test.ts +21 -0
  399. package/src/service/prompts.ts +147 -0
  400. package/src/service/sanitizeHtml.test.ts +61 -0
  401. package/src/service/sanitizeHtml.ts +109 -0
  402. package/src/tests/bunSetup.test.ts +7 -4
  403. package/src/tests/bunSetup.ts +7 -7
  404. package/src/tests/chatHarness.ts +478 -0
  405. package/src/tests/generateStream.ts +49 -0
  406. package/src/tests/harnessErrors.ts +22 -0
  407. package/src/tests/harnessStreaming.ts +227 -0
  408. package/src/tests/helpers.ts +9 -2
  409. package/src/types/harness.ts +1095 -0
  410. package/src/types/index.ts +209 -13
  411. package/src/types/observability.ts +344 -0
  412. package/src/types/sanitize-html.d.ts +25 -0
@@ -0,0 +1,65 @@
1
+ import {createdUpdatedPlugin, findExactlyOne, findOneOrNone, isDeletedPlugin} from "@terreno/api";
2
+ import mongoose from "mongoose";
3
+
4
+ import type {ObsSpanDocument, ObsSpanModel} from "../../../types/observability";
5
+
6
+ const obsSpanSchema = new mongoose.Schema<ObsSpanDocument, ObsSpanModel>(
7
+ {
8
+ durationMs: {description: "Span duration in milliseconds", type: Number},
9
+ endedAt: {description: "When this span finished", type: Date},
10
+ error: {description: "Short error cause when status is error", type: String},
11
+ input: {description: "Span input payload", type: mongoose.Schema.Types.Mixed},
12
+ kind: {
13
+ description: "OpenInference span kind",
14
+ enum: ["AGENT", "CHAIN", "EVALUATOR", "LLM", "RETRIEVER", "TOOL"],
15
+ required: true,
16
+ type: String,
17
+ },
18
+ name: {description: "Span display name", required: true, type: String},
19
+ output: {description: "Span output payload", type: mongoose.Schema.Types.Mixed},
20
+ parentSpanId: {
21
+ description: "Parent span for nested waterfalls",
22
+ ref: "ObsSpan",
23
+ type: mongoose.Schema.Types.ObjectId,
24
+ },
25
+ sensitive: {
26
+ description: "When true, this span's I/O is treated as sensitive",
27
+ type: Boolean,
28
+ },
29
+ startedAt: {description: "When this span started", required: true, type: Date},
30
+ startOffsetMs: {
31
+ description: "Milliseconds after the root trace start",
32
+ type: Number,
33
+ },
34
+ status: {
35
+ description: "ok or error for this span",
36
+ enum: ["error", "ok"],
37
+ required: true,
38
+ type: String,
39
+ },
40
+ traceId: {
41
+ description: "Root trace this span belongs to",
42
+ ref: "ObsTrace",
43
+ required: true,
44
+ type: mongoose.Schema.Types.ObjectId,
45
+ },
46
+ usage: {
47
+ description: "Per-span token and cost usage",
48
+ type: mongoose.Schema.Types.Mixed,
49
+ },
50
+ },
51
+ {strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
52
+ );
53
+
54
+ obsSpanSchema.plugin(createdUpdatedPlugin);
55
+ obsSpanSchema.plugin(isDeletedPlugin);
56
+ obsSpanSchema.plugin(findOneOrNone);
57
+ obsSpanSchema.plugin(findExactlyOne);
58
+ obsSpanSchema.index({traceId: 1});
59
+
60
+ export const registerObsSpan = (): ObsSpanModel => {
61
+ if (mongoose.models.ObsSpan) {
62
+ return mongoose.models.ObsSpan as ObsSpanModel;
63
+ }
64
+ return mongoose.model<ObsSpanDocument, ObsSpanModel>("ObsSpan", obsSpanSchema);
65
+ };
@@ -0,0 +1,94 @@
1
+ import {createdUpdatedPlugin, findExactlyOne, findOneOrNone, isDeletedPlugin} from "@terreno/api";
2
+ import mongoose from "mongoose";
3
+
4
+ import type {ObsTraceDocument, ObsTraceModel} from "../../../types/observability";
5
+
6
+ const obsTraceSchema = new mongoose.Schema<ObsTraceDocument, ObsTraceModel>(
7
+ {
8
+ endedAt: {description: "When the root trace finished", type: Date},
9
+ errorSummary: {
10
+ description: "Short error copied from the first failed span",
11
+ type: String,
12
+ },
13
+ flaggedForDataset: {
14
+ default: false,
15
+ description: "When true, operators marked this trace as a dataset candidate",
16
+ type: Boolean,
17
+ },
18
+ input: {description: "Root trace input payload", type: mongoose.Schema.Types.Mixed},
19
+ name: {description: "Trace display name", required: true, type: String},
20
+ output: {description: "Root trace output payload", type: mongoose.Schema.Types.Mixed},
21
+ prompts: {
22
+ default: [],
23
+ description: "Prompt versions used during this trace",
24
+ type: [
25
+ {
26
+ label: {description: "Label resolved at call time", type: String},
27
+ name: {description: "Prompt name", required: true, type: String},
28
+ version: {description: "Prompt version number", required: true, type: Number},
29
+ },
30
+ ],
31
+ },
32
+ sensitive: {
33
+ default: false,
34
+ description: "When true, admin I/O disclosures start collapsed",
35
+ type: Boolean,
36
+ },
37
+ sessionId: {description: "Client session identifier", type: String},
38
+ startedAt: {description: "When the root trace started", required: true, type: Date},
39
+ status: {
40
+ description: "ok when every span succeeded, otherwise error",
41
+ enum: ["error", "ok"],
42
+ required: true,
43
+ type: String,
44
+ },
45
+ usage: {
46
+ description: "Aggregated token and cost usage for the trace",
47
+ type: mongoose.Schema.Types.Mixed,
48
+ },
49
+ userId: {
50
+ description: "User who initiated the generate call",
51
+ ref: "User",
52
+ type: mongoose.Schema.Types.ObjectId,
53
+ },
54
+ },
55
+ {strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
56
+ );
57
+
58
+ obsTraceSchema.plugin(createdUpdatedPlugin);
59
+ obsTraceSchema.plugin(isDeletedPlugin);
60
+ obsTraceSchema.plugin(findOneOrNone);
61
+ obsTraceSchema.plugin(findExactlyOne);
62
+ // Compound index field order is query-significant; do not alphabetize keys.
63
+ const traceIndexSpecs: Array<Array<[string, -1 | 1]>> = [
64
+ [
65
+ ["created", -1],
66
+ ["userId", 1],
67
+ ],
68
+ [
69
+ ["sessionId", 1],
70
+ ["created", -1],
71
+ ],
72
+ [
73
+ ["status", 1],
74
+ ["created", -1],
75
+ ],
76
+ [
77
+ ["prompts.name", 1],
78
+ ["prompts.version", 1],
79
+ ],
80
+ [
81
+ ["flaggedForDataset", 1],
82
+ ["created", -1],
83
+ ],
84
+ ];
85
+ for (const spec of traceIndexSpecs) {
86
+ obsTraceSchema.index(Object.fromEntries(spec));
87
+ }
88
+
89
+ export const registerObsTrace = (): ObsTraceModel => {
90
+ if (mongoose.models.ObsTrace) {
91
+ return mongoose.models.ObsTrace as ObsTraceModel;
92
+ }
93
+ return mongoose.model<ObsTraceDocument, ObsTraceModel>("ObsTrace", obsTraceSchema);
94
+ };
@@ -0,0 +1,119 @@
1
+ import {afterEach, beforeEach, describe, expect, it} from "bun:test";
2
+ import mongoose from "mongoose";
3
+ import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
4
+ import {createLocalObservabilityPlugin} from "./localPlugin";
5
+ import {registerObsExperiment} from "./models/obsExperiment";
6
+ import {registerObsPrompt} from "./models/obsPrompt";
7
+ import {registerObsPromptLabel} from "./models/obsPromptLabel";
8
+ import {registerObsPromptVersion} from "./models/obsPromptVersion";
9
+ import {registerObsScore} from "./models/obsScore";
10
+ import {registerObsTrace} from "./models/obsTrace";
11
+
12
+ describe("local observability models", () => {
13
+ afterEach(() => {
14
+ resetObservabilityApp();
15
+ });
16
+
17
+ beforeEach(async () => {
18
+ const ObsPrompt = registerObsPrompt();
19
+ const ObsPromptVersion = registerObsPromptVersion();
20
+ const ObsPromptLabel = registerObsPromptLabel();
21
+ const ObsScore = registerObsScore();
22
+ await ObsPrompt.deleteMany({});
23
+ await ObsPromptVersion.deleteMany({});
24
+ await ObsPromptLabel.deleteMany({});
25
+ await ObsScore.deleteMany({});
26
+ });
27
+
28
+ it("registers models when the local plugin constructs", () => {
29
+ const plugin = createLocalObservabilityPlugin();
30
+ const app = new ObservabilityApp({plugins: [plugin]});
31
+ expect(app.control.prompts).toBe("local");
32
+ expect(mongoose.models.ObsPrompt).toBeDefined();
33
+ expect(mongoose.models.ObsTrace).toBeDefined();
34
+ });
35
+
36
+ it("leaves a prompt version unchanged after a later save attempt", async () => {
37
+ const ObsPrompt = registerObsPrompt();
38
+ const ObsPromptVersion = registerObsPromptVersion();
39
+ const prompt = await ObsPrompt.create({folder: "examples", name: "greeter"});
40
+ const version = await ObsPromptVersion.create({
41
+ promptId: prompt._id,
42
+ system: "You are v1",
43
+ type: "text",
44
+ version: 1,
45
+ });
46
+
47
+ version.system = "mutated";
48
+ await expect(version.save()).rejects.toThrow(/immutable/);
49
+
50
+ const reloaded = await ObsPromptVersion.findExactlyOne({_id: version._id});
51
+ expect(reloaded.system).toBe("You are v1");
52
+ });
53
+
54
+ it("enforces unique (promptId, label)", async () => {
55
+ const ObsPrompt = registerObsPrompt();
56
+ const ObsPromptVersion = registerObsPromptVersion();
57
+ const ObsPromptLabel = registerObsPromptLabel();
58
+ const prompt = await ObsPrompt.create({folder: "examples", name: "unique-label"});
59
+ const version = await ObsPromptVersion.create({
60
+ promptId: prompt._id,
61
+ type: "text",
62
+ version: 1,
63
+ });
64
+ await ObsPromptLabel.syncIndexes();
65
+ await ObsPromptLabel.create({
66
+ label: "production",
67
+ promptId: prompt._id,
68
+ versionId: version._id,
69
+ });
70
+
71
+ await expect(
72
+ ObsPromptLabel.create({
73
+ label: "production",
74
+ promptId: prompt._id,
75
+ versionId: version._id,
76
+ })
77
+ ).rejects.toThrow();
78
+ });
79
+
80
+ it("loads a version with findExactlyOne and findOneOrNone", async () => {
81
+ const ObsPrompt = registerObsPrompt();
82
+ const ObsPromptVersion = registerObsPromptVersion();
83
+ const prompt = await ObsPrompt.create({folder: "examples", name: "lookup"});
84
+ const created = await ObsPromptVersion.create({
85
+ promptId: prompt._id,
86
+ type: "text",
87
+ version: 1,
88
+ });
89
+
90
+ const found = await ObsPromptVersion.findExactlyOne({_id: created._id});
91
+ expect(found.version).toBe(1);
92
+ const missing = await ObsPromptVersion.findOneOrNone({version: 99});
93
+ expect(missing).toBeNull();
94
+ });
95
+
96
+ it("does not declare a unique index on scores", () => {
97
+ const ObsScore = registerObsScore();
98
+ const uniqueIndexes = ObsScore.schema
99
+ .indexes()
100
+ .filter((entry) => Boolean((entry[1] as {unique?: boolean} | undefined)?.unique));
101
+ expect(uniqueIndexes.length).toBe(0);
102
+ });
103
+
104
+ it("declares ObsTrace compound indexes in IP key order", () => {
105
+ const ObsTrace = registerObsTrace();
106
+ const serialized = ObsTrace.schema.indexes().map(([fields]) => JSON.stringify(fields));
107
+ expect(serialized).toContain('{"created":-1,"userId":1}');
108
+ expect(serialized).toContain('{"sessionId":1,"created":-1}');
109
+ expect(serialized).toContain('{"status":1,"created":-1}');
110
+ expect(serialized).toContain('{"prompts.name":1,"prompts.version":1}');
111
+ expect(serialized).toContain('{"flaggedForDataset":1,"created":-1}');
112
+ });
113
+
114
+ it("declares ObsExperiment promptName list index", () => {
115
+ const ObsExperiment = registerObsExperiment();
116
+ const serialized = ObsExperiment.schema.indexes().map(([fields]) => JSON.stringify(fields));
117
+ expect(serialized).toContain('{"promptName":1,"created":-1}');
118
+ });
119
+ });
@@ -0,0 +1,351 @@
1
+ import {afterEach, beforeEach, describe, expect, it} from "bun:test";
2
+ import {assert} from "chai";
3
+ import {DateTime} from "luxon";
4
+ import mongoose from "mongoose";
5
+
6
+ import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
7
+ import {createLocalObservabilityPlugin} from "./localPlugin";
8
+ import {registerObsExperiment} from "./models/obsExperiment";
9
+ import {registerObsPrompt} from "./models/obsPrompt";
10
+ import {registerObsPromptLabel} from "./models/obsPromptLabel";
11
+ import {registerObsPromptVersion} from "./models/obsPromptVersion";
12
+ import {registerObsTrace} from "./models/obsTrace";
13
+ import {LocalPromptStore} from "./promptStore";
14
+
15
+ describe("LocalPromptStore", () => {
16
+ let store: LocalPromptStore;
17
+
18
+ afterEach(() => {
19
+ resetObservabilityApp();
20
+ });
21
+
22
+ beforeEach(async () => {
23
+ store = new LocalPromptStore();
24
+ new ObservabilityApp({plugins: [createLocalObservabilityPlugin()]});
25
+ await registerObsPrompt().deleteMany({});
26
+ await registerObsPromptVersion().deleteMany({});
27
+ await registerObsPromptLabel().deleteMany({});
28
+ await registerObsTrace().deleteMany({});
29
+ await registerObsExperiment().deleteMany({});
30
+ });
31
+
32
+ it("round-trips an optional description on create, list, and detail", async () => {
33
+ await store.create({
34
+ description: "Summarizes user notes for clinicians",
35
+ folder: "examples",
36
+ name: "summarize",
37
+ system: "Summarize",
38
+ type: "text",
39
+ });
40
+ await store.create({
41
+ folder: "examples",
42
+ name: "legacy-no-description",
43
+ system: "Legacy",
44
+ type: "text",
45
+ });
46
+
47
+ const listed = await store.list({folder: "examples"});
48
+ expect(listed.find((row) => row.name === "summarize")?.description).toBe(
49
+ "Summarizes user notes for clinicians"
50
+ );
51
+ expect(listed.find((row) => row.name === "legacy-no-description")?.description).toBeUndefined();
52
+
53
+ const detail = await store.getDetail("summarize");
54
+ expect(detail.description).toBe("Summarizes user notes for clinicians");
55
+ const legacy = await store.getDetail("legacy-no-description");
56
+ expect(legacy.description).toBeUndefined();
57
+ });
58
+
59
+ it("composes bounded relationships with only this prompt's traces and experiments", async () => {
60
+ await store.create({
61
+ folder: "examples",
62
+ name: "hub-prompt",
63
+ system: "v1",
64
+ type: "text",
65
+ });
66
+ await store.createVersion("hub-prompt", {system: "v2", type: "text"});
67
+ const ObsTrace = registerObsTrace();
68
+ await ObsTrace.create({
69
+ name: "hub-v1",
70
+ prompts: [{name: "hub-prompt", version: 1}],
71
+ startedAt: DateTime.utc().minus({minutes: 2}).toJSDate(),
72
+ status: "ok",
73
+ });
74
+ await ObsTrace.create({
75
+ name: "hub-v2",
76
+ prompts: [{name: "hub-prompt", version: 2}],
77
+ startedAt: DateTime.utc().minus({minutes: 1}).toJSDate(),
78
+ status: "error",
79
+ });
80
+ await ObsTrace.create({
81
+ name: "other-prompt",
82
+ prompts: [{name: "other", version: 1}],
83
+ startedAt: DateTime.utc().toJSDate(),
84
+ status: "ok",
85
+ });
86
+ const ObsExperiment = registerObsExperiment();
87
+ await ObsExperiment.create({
88
+ datasetId: new mongoose.Types.ObjectId(),
89
+ evaluatorIds: [],
90
+ name: "hub-run",
91
+ promptName: "hub-prompt",
92
+ status: "completed",
93
+ thresholds: [],
94
+ versions: [1, 2],
95
+ });
96
+ await ObsExperiment.create({
97
+ datasetId: new mongoose.Types.ObjectId(),
98
+ evaluatorIds: [],
99
+ name: "foreign-run",
100
+ promptName: "other",
101
+ status: "pending",
102
+ thresholds: [],
103
+ versions: [1, 2],
104
+ });
105
+
106
+ const detail = await store.getDetail("hub-prompt");
107
+ expect(detail.relationships.experiments.total).toBe(1);
108
+ expect(detail.relationships.experiments.items).toEqual([
109
+ expect.objectContaining({name: "hub-run", promptName: "hub-prompt"}),
110
+ ]);
111
+ expect(detail.relationships.traces.total).toBe(2);
112
+ expect(detail.relationships.traces.items.map((row) => row.name).sort()).toEqual([
113
+ "hub-v1",
114
+ "hub-v2",
115
+ ]);
116
+ expect(
117
+ detail.relationships.traces.items.every((row) => {
118
+ return row.promptName === "hub-prompt";
119
+ })
120
+ ).toBe(true);
121
+ expect(
122
+ detail.relationships.traces.items.find((row) => row.name === "hub-v2")?.promptVersion
123
+ ).toBe(2);
124
+ });
125
+
126
+ it("filters hub traces by prompt version and keeps unfiltered detail", async () => {
127
+ await store.create({
128
+ folder: "examples",
129
+ name: "version-filter",
130
+ system: "v1",
131
+ type: "text",
132
+ });
133
+ await store.createVersion("version-filter", {system: "v2", type: "text"});
134
+ const ObsTrace = registerObsTrace();
135
+ await ObsTrace.create({
136
+ name: "only-v1",
137
+ prompts: [{name: "version-filter", version: 1}],
138
+ startedAt: DateTime.utc().minus({minutes: 2}).toJSDate(),
139
+ status: "ok",
140
+ });
141
+ await ObsTrace.create({
142
+ name: "only-v2",
143
+ prompts: [{name: "version-filter", version: 2}],
144
+ startedAt: DateTime.utc().minus({minutes: 1}).toJSDate(),
145
+ status: "ok",
146
+ });
147
+
148
+ const v2 = await store.getDetail("version-filter", {promptVersion: 2});
149
+ expect(v2.relationships.traces.total).toBe(1);
150
+ expect(v2.relationships.traces.items.map((row) => row.name)).toEqual(["only-v2"]);
151
+ expect(v2.relationships.traces.items[0]?.promptVersion).toBe(2);
152
+
153
+ const all = await store.getDetail("version-filter");
154
+ expect(all.relationships.traces.total).toBe(2);
155
+ expect(all.relationships.traces.items.map((row) => row.name).sort()).toEqual([
156
+ "only-v1",
157
+ "only-v2",
158
+ ]);
159
+ });
160
+
161
+ it("caps relationship traces at the published limit while reporting total", async () => {
162
+ await store.create({
163
+ folder: "examples",
164
+ name: "bounded-traces",
165
+ system: "v1",
166
+ type: "text",
167
+ });
168
+ const ObsTrace = registerObsTrace();
169
+ const seedCount = 25;
170
+ for (let index = 0; index < seedCount; index += 1) {
171
+ await ObsTrace.create({
172
+ name: `trace-${index}`,
173
+ prompts: [{name: "bounded-traces", version: 1}],
174
+ startedAt: DateTime.utc().minus({minutes: index}).toJSDate(),
175
+ status: "ok",
176
+ });
177
+ }
178
+
179
+ const detail = await store.getDetail("bounded-traces");
180
+ expect(detail.relationships.traces.total).toBe(seedCount);
181
+ expect(detail.relationships.traces.limit).toBe(20);
182
+ expect(detail.relationships.traces.items).toHaveLength(20);
183
+ });
184
+
185
+ it("leaves v1 unchanged after creating v2 and resolves production by label", async () => {
186
+ await store.create({
187
+ folder: "examples",
188
+ name: "greeter",
189
+ system: "You are v1",
190
+ template: "Hello {{name}}",
191
+ type: "text",
192
+ });
193
+ await store.moveLabel("greeter", {label: "production", version: 1});
194
+ await store.createVersion("greeter", {
195
+ system: "You are v2",
196
+ template: "Hi {{name}}",
197
+ type: "text",
198
+ });
199
+ const moved = await store.moveLabel("greeter", {label: "production", version: 2});
200
+
201
+ const v1 = await registerObsPromptVersion().findExactlyOne({version: 1});
202
+ expect(v1.system).toBe("You are v1");
203
+ expect(moved.outgoingVersion).toBe(1);
204
+
205
+ const production = await store.get({label: "production", name: "greeter"});
206
+ expect(production?.version).toBe(2);
207
+ expect(production?.body).toBe("You are v2");
208
+ });
209
+
210
+ it("returns — for production and 7-day usage rollup when include=usage7d", async () => {
211
+ await store.create({
212
+ folder: "examples",
213
+ name: "summarize",
214
+ system: "Summarize",
215
+ type: "text",
216
+ });
217
+ await store.create({
218
+ folder: "other",
219
+ name: "unrelated",
220
+ system: "Nope",
221
+ type: "text",
222
+ });
223
+ const ObsTrace = registerObsTrace();
224
+ await ObsTrace.create({
225
+ created: DateTime.utc().minus({days: 1}).toJSDate(),
226
+ name: "summarize-call",
227
+ prompts: [{label: "production", name: "summarize", version: 1}],
228
+ startedAt: DateTime.utc().minus({days: 1}).toJSDate(),
229
+ status: "ok",
230
+ usage: {costUsd: 0.4, inputTokens: 10, outputTokens: 20},
231
+ });
232
+ await ObsTrace.create({
233
+ created: DateTime.utc().minus({days: 8}).toJSDate(),
234
+ name: "old-call",
235
+ prompts: [{name: "summarize", version: 1}],
236
+ startedAt: DateTime.utc().minus({days: 8}).toJSDate(),
237
+ status: "ok",
238
+ usage: {costUsd: 9},
239
+ });
240
+
241
+ const listed = await store.list({folder: "examples", includeUsage7d: true, search: "sum"});
242
+ expect(listed).toEqual([
243
+ {
244
+ folder: "examples",
245
+ latestVersion: 1,
246
+ name: "summarize",
247
+ production: "—",
248
+ type: "text",
249
+ usage7d: {calls: 1, costUsd: 0.4},
250
+ },
251
+ ]);
252
+ });
253
+
254
+ it("validates create/moveLabel inputs and resolves versions by label or number", async () => {
255
+ try {
256
+ await store.create({folder: "", name: "bad"});
257
+ expect.unreachable();
258
+ } catch (error) {
259
+ expect(String(error)).toMatch(/folder and name/);
260
+ }
261
+
262
+ await store.create({
263
+ folder: "examples",
264
+ name: "versioned",
265
+ system: "v1",
266
+ type: "text",
267
+ });
268
+ await store.createVersion("versioned", {system: "v2", type: "text"});
269
+
270
+ try {
271
+ await store.moveLabel("versioned", {label: "qa", version: 2});
272
+ expect.unreachable();
273
+ } catch (error) {
274
+ expect(String(error)).toMatch(/production or staging/);
275
+ }
276
+
277
+ try {
278
+ await store.moveLabel("versioned", {label: "production", version: 99});
279
+ expect.unreachable();
280
+ } catch (error) {
281
+ expect(String(error)).toMatch(/Unknown version 99/);
282
+ }
283
+
284
+ await store.create({
285
+ folder: "examples",
286
+ name: "duplicate",
287
+ system: "one",
288
+ type: "text",
289
+ });
290
+ try {
291
+ await store.create({folder: "examples", name: "duplicate", system: "two", type: "text"});
292
+ expect.unreachable();
293
+ } catch (error) {
294
+ expect(String(error)).toMatch(/already exists/);
295
+ }
296
+
297
+ expect(await store.getVersionByLabel("missing")).toBeUndefined();
298
+ expect((await store.getVersionByNumber("versioned", 2))?.system).toBe("v2");
299
+ expect(await store.getVersionByNumber("versioned", 99)).toBeUndefined();
300
+ });
301
+
302
+ it("compiles templates and runs playground generation with optional pricing", async () => {
303
+ await store.create({
304
+ folder: "examples",
305
+ name: "playground",
306
+ system: "System {{name}}",
307
+ template: "Hello {{name}}",
308
+ type: "text",
309
+ variables: [{key: "name", required: true}],
310
+ });
311
+ expect(store.compile({system: "", template: "Hi", variables: {}})).toEqual([
312
+ {content: "Hi", role: "user"},
313
+ ]);
314
+
315
+ const result = await store.runPlayground({
316
+ generator: {
317
+ generate: async () => {
318
+ return {inputTokens: 10, latencyMs: 12, output: "done", outputTokens: 5};
319
+ },
320
+ },
321
+ modelId: "mock-model",
322
+ name: "playground",
323
+ priceMap: {["mock-model"]: {inputPerMTok: 1, outputPerMTok: 2}},
324
+ variables: {name: "Ada"},
325
+ version: 1,
326
+ });
327
+ expect(result.output).toBe("done");
328
+ expect(result.costUsd).toBeCloseTo(0.00002);
329
+ expect(result.compiledMessages).toEqual([
330
+ {content: "System Ada", role: "system"},
331
+ {content: "Hello Ada", role: "user"},
332
+ ]);
333
+
334
+ const latest = await store.runPlayground({
335
+ generator: {
336
+ generate: async () => {
337
+ return {output: "latest"};
338
+ },
339
+ },
340
+ name: "playground",
341
+ variables: {name: "Grace"},
342
+ });
343
+ assert.equal(latest.output, "latest");
344
+ assert.deepEqual(latest.compiledMessages, [
345
+ {content: "System Grace", role: "system"},
346
+ {content: "Hello Grace", role: "user"},
347
+ ]);
348
+ assert.isUndefined(await store.get({name: "missing"}));
349
+ assert.isUndefined(await store.getVersionByNumber("missing", 1));
350
+ });
351
+ });