@terreno/ai 57.9.0 → 57.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (412) hide show
  1. package/README.md +1 -0
  2. package/bunfig.toml +1 -1
  3. package/dist/aiApp.d.ts +8 -2
  4. package/dist/aiApp.js +12 -6
  5. package/dist/harness/adminScreens.d.ts +6 -0
  6. package/dist/harness/adminScreens.js +11 -0
  7. package/dist/harness/agentLoop.d.ts +61 -0
  8. package/dist/harness/agentLoop.js +798 -0
  9. package/dist/harness/agentTaskNames.d.ts +4 -0
  10. package/dist/harness/agentTaskNames.js +7 -0
  11. package/dist/harness/approvalPolicy.d.ts +3 -0
  12. package/dist/harness/approvalPolicy.js +23 -0
  13. package/dist/harness/approvals.d.ts +101 -0
  14. package/dist/harness/approvals.js +468 -0
  15. package/dist/harness/commit.d.ts +238 -0
  16. package/dist/harness/commit.js +678 -0
  17. package/dist/harness/conversation.d.ts +75 -0
  18. package/dist/harness/conversation.js +397 -0
  19. package/dist/harness/defineAgent.d.ts +9 -0
  20. package/dist/harness/defineAgent.js +65 -0
  21. package/dist/harness/defineTask.d.ts +11 -0
  22. package/dist/harness/defineTask.js +59 -0
  23. package/dist/harness/defineTool.d.ts +7 -0
  24. package/dist/harness/defineTool.js +36 -0
  25. package/dist/harness/definitionError.d.ts +8 -0
  26. package/dist/harness/definitionError.js +15 -0
  27. package/dist/harness/errors.d.ts +127 -0
  28. package/dist/harness/errors.js +101 -0
  29. package/dist/harness/eventHub.d.ts +45 -0
  30. package/dist/harness/eventHub.js +135 -0
  31. package/dist/harness/events.d.ts +95 -0
  32. package/dist/harness/events.js +267 -0
  33. package/dist/harness/executionEnv.d.ts +31 -0
  34. package/dist/harness/executionEnv.js +2 -0
  35. package/dist/harness/extensions.d.ts +82 -0
  36. package/dist/harness/extensions.js +274 -0
  37. package/dist/harness/harness.d.ts +163 -0
  38. package/dist/harness/harness.js +520 -0
  39. package/dist/harness/harnessApp.d.ts +44 -0
  40. package/dist/harness/harnessApp.js +62 -0
  41. package/dist/harness/internalRuntime.d.ts +16 -0
  42. package/dist/harness/internalRuntime.js +18 -0
  43. package/dist/harness/leases.d.ts +35 -0
  44. package/dist/harness/leases.js +86 -0
  45. package/dist/harness/memo.d.ts +24 -0
  46. package/dist/harness/memo.js +78 -0
  47. package/dist/harness/modelCall.d.ts +45 -0
  48. package/dist/harness/modelCall.js +147 -0
  49. package/dist/harness/models/harnessApproval.d.ts +2 -0
  50. package/dist/harness/models/harnessApproval.js +100 -0
  51. package/dist/harness/models/harnessConversation.d.ts +2 -0
  52. package/dist/harness/models/harnessConversation.js +117 -0
  53. package/dist/harness/models/harnessEvent.d.ts +3 -0
  54. package/dist/harness/models/harnessEvent.js +73 -0
  55. package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
  56. package/dist/harness/models/harnessInboxEvent.js +58 -0
  57. package/dist/harness/models/harnessMemo.d.ts +2 -0
  58. package/dist/harness/models/harnessMemo.js +42 -0
  59. package/dist/harness/models/harnessMessage.d.ts +2 -0
  60. package/dist/harness/models/harnessMessage.js +73 -0
  61. package/dist/harness/models/harnessOwner.d.ts +2 -0
  62. package/dist/harness/models/harnessOwner.js +36 -0
  63. package/dist/harness/models/harnessTask.d.ts +2 -0
  64. package/dist/harness/models/harnessTask.js +186 -0
  65. package/dist/harness/ownership.d.ts +89 -0
  66. package/dist/harness/ownership.js +438 -0
  67. package/dist/harness/registry.d.ts +29 -0
  68. package/dist/harness/registry.js +93 -0
  69. package/dist/harness/retryBackoff.d.ts +31 -0
  70. package/dist/harness/retryBackoff.js +50 -0
  71. package/dist/harness/routes/approvals.d.ts +13 -0
  72. package/dist/harness/routes/approvals.js +93 -0
  73. package/dist/harness/routes/conversations.d.ts +11 -0
  74. package/dist/harness/routes/conversations.js +66 -0
  75. package/dist/harness/routes/events.d.ts +19 -0
  76. package/dist/harness/routes/events.js +215 -0
  77. package/dist/harness/routes/tasks.d.ts +11 -0
  78. package/dist/harness/routes/tasks.js +108 -0
  79. package/dist/harness/runners/inProcessRunner.d.ts +57 -0
  80. package/dist/harness/runners/inProcessRunner.js +196 -0
  81. package/dist/harness/runtime.d.ts +34 -0
  82. package/dist/harness/runtime.js +551 -0
  83. package/dist/harness/subagent.d.ts +53 -0
  84. package/dist/harness/subagent.js +134 -0
  85. package/dist/harness/suspend.d.ts +8 -0
  86. package/dist/harness/suspend.js +16 -0
  87. package/dist/harness/transaction.d.ts +3 -0
  88. package/dist/harness/transaction.js +22 -0
  89. package/dist/harness/waits.d.ts +100 -0
  90. package/dist/harness/waits.js +214 -0
  91. package/dist/index.d.ts +19 -1
  92. package/dist/index.js +45 -2
  93. package/dist/models/aiDataset.d.ts +2 -0
  94. package/dist/models/aiDataset.js +67 -0
  95. package/dist/models/aiRequest.js +1 -1
  96. package/dist/models/gptHistory.js +73 -0
  97. package/dist/observability/adminScreens.d.ts +5 -0
  98. package/dist/observability/adminScreens.js +32 -0
  99. package/dist/observability/compileTemplate.d.ts +1 -0
  100. package/dist/observability/compileTemplate.js +13 -0
  101. package/dist/observability/datasetImport.d.ts +11 -0
  102. package/dist/observability/datasetImport.js +185 -0
  103. package/dist/observability/evaluate.d.ts +31 -0
  104. package/dist/observability/evaluate.js +200 -0
  105. package/dist/observability/evaluatorTemplates.d.ts +17 -0
  106. package/dist/observability/evaluatorTemplates.js +111 -0
  107. package/dist/observability/jsonPath.d.ts +2 -0
  108. package/dist/observability/jsonPath.js +57 -0
  109. package/dist/observability/local/datasetStore.d.ts +87 -0
  110. package/dist/observability/local/datasetStore.js +326 -0
  111. package/dist/observability/local/evaluatorStore.d.ts +44 -0
  112. package/dist/observability/local/evaluatorStore.js +182 -0
  113. package/dist/observability/local/experimentRunner.d.ts +96 -0
  114. package/dist/observability/local/experimentRunner.js +419 -0
  115. package/dist/observability/local/localPlugin.d.ts +22 -0
  116. package/dist/observability/local/localPlugin.js +76 -0
  117. package/dist/observability/local/models/obsDataset.d.ts +2 -0
  118. package/dist/observability/local/models/obsDataset.js +37 -0
  119. package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
  120. package/dist/observability/local/models/obsDatasetItem.js +76 -0
  121. package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
  122. package/dist/observability/local/models/obsEvaluator.js +98 -0
  123. package/dist/observability/local/models/obsExperiment.d.ts +2 -0
  124. package/dist/observability/local/models/obsExperiment.js +87 -0
  125. package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
  126. package/dist/observability/local/models/obsExperimentItem.js +45 -0
  127. package/dist/observability/local/models/obsPrompt.d.ts +2 -0
  128. package/dist/observability/local/models/obsPrompt.js +42 -0
  129. package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
  130. package/dist/observability/local/models/obsPromptLabel.js +39 -0
  131. package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
  132. package/dist/observability/local/models/obsPromptVersion.js +96 -0
  133. package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
  134. package/dist/observability/local/models/obsReviewItem.js +75 -0
  135. package/dist/observability/local/models/obsScore.d.ts +2 -0
  136. package/dist/observability/local/models/obsScore.js +58 -0
  137. package/dist/observability/local/models/obsSpan.d.ts +2 -0
  138. package/dist/observability/local/models/obsSpan.js +64 -0
  139. package/dist/observability/local/models/obsTrace.d.ts +2 -0
  140. package/dist/observability/local/models/obsTrace.js +93 -0
  141. package/dist/observability/local/promptStore.d.ts +180 -0
  142. package/dist/observability/local/promptStore.js +444 -0
  143. package/dist/observability/local/reviewStore.d.ts +50 -0
  144. package/dist/observability/local/reviewStore.js +203 -0
  145. package/dist/observability/local/traceStore.d.ts +103 -0
  146. package/dist/observability/local/traceStore.js +325 -0
  147. package/dist/observability/observabilityApp.d.ts +24 -0
  148. package/dist/observability/observabilityApp.js +157 -0
  149. package/dist/observability/observabilityAppRegistry.d.ts +14 -0
  150. package/dist/observability/observabilityAppRegistry.js +16 -0
  151. package/dist/observability/permissions.d.ts +7 -0
  152. package/dist/observability/permissions.js +60 -0
  153. package/dist/observability/reviewPanels.d.ts +18 -0
  154. package/dist/observability/reviewPanels.js +46 -0
  155. package/dist/observability/routes/datasets.d.ts +7 -0
  156. package/dist/observability/routes/datasets.js +149 -0
  157. package/dist/observability/routes/evaluators.d.ts +7 -0
  158. package/dist/observability/routes/evaluators.js +123 -0
  159. package/dist/observability/routes/experiments.d.ts +7 -0
  160. package/dist/observability/routes/experiments.js +95 -0
  161. package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
  162. package/dist/observability/routes/observabilityRouteAccess.js +67 -0
  163. package/dist/observability/routes/prompts.d.ts +12 -0
  164. package/dist/observability/routes/prompts.js +172 -0
  165. package/dist/observability/routes/requestAiService.d.ts +13 -0
  166. package/dist/observability/routes/requestAiService.js +21 -0
  167. package/dist/observability/routes/review.d.ts +7 -0
  168. package/dist/observability/routes/review.js +79 -0
  169. package/dist/observability/routes/status.d.ts +5 -0
  170. package/dist/observability/routes/status.js +31 -0
  171. package/dist/observability/routes/testMultiStage.d.ts +9 -0
  172. package/dist/observability/routes/testMultiStage.js +67 -0
  173. package/dist/observability/routes/traces.d.ts +7 -0
  174. package/dist/observability/routes/traces.js +129 -0
  175. package/dist/observability/schemaValidation.d.ts +5 -0
  176. package/dist/observability/schemaValidation.js +26 -0
  177. package/dist/observability/sopThresholds.d.ts +2 -0
  178. package/dist/observability/sopThresholds.js +21 -0
  179. package/dist/observability/status.d.ts +23 -0
  180. package/dist/observability/status.js +34 -0
  181. package/dist/observability/testMultiStageSchemas.d.ts +10 -0
  182. package/dist/observability/testMultiStageSchemas.js +79 -0
  183. package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
  184. package/dist/observability/testMultiStageWorkflow.js +218 -0
  185. package/dist/observability/types.d.ts +159 -0
  186. package/dist/observability/types.js +67 -0
  187. package/dist/routes/files.js +18 -7
  188. package/dist/routes/gpt.js +79 -689
  189. package/dist/routes/gptActions.d.ts +9 -0
  190. package/dist/routes/gptActions.js +212 -0
  191. package/dist/routes/gptDatasets.d.ts +5 -0
  192. package/dist/routes/gptDatasets.js +94 -0
  193. package/dist/routes/gptHistories.d.ts +3 -1
  194. package/dist/routes/gptHistories.js +188 -2
  195. package/dist/service/aiDatasets.d.ts +57 -0
  196. package/dist/service/aiDatasets.js +253 -0
  197. package/dist/service/aiService.d.ts +50 -1
  198. package/dist/service/aiService.js +426 -77
  199. package/dist/service/askFiles.d.ts +49 -0
  200. package/dist/service/askFiles.js +162 -0
  201. package/dist/service/asks.d.ts +76 -0
  202. package/dist/service/asks.js +284 -0
  203. package/dist/service/chatTurn.d.ts +128 -0
  204. package/dist/service/chatTurn.js +1674 -0
  205. package/dist/service/fileStorage.d.ts +4 -0
  206. package/dist/service/fileStorage.js +7 -2
  207. package/dist/service/fillActionIds.d.ts +6 -0
  208. package/dist/service/fillActionIds.js +79 -0
  209. package/dist/service/jsonFenceModel.d.ts +7 -0
  210. package/dist/service/jsonFenceModel.js +45 -0
  211. package/dist/service/prompts.d.ts +52 -0
  212. package/dist/service/prompts.js +96 -1
  213. package/dist/service/sanitizeHtml.d.ts +11 -0
  214. package/dist/service/sanitizeHtml.js +108 -0
  215. package/dist/types/harness.d.ts +934 -0
  216. package/dist/types/harness.js +133 -0
  217. package/dist/types/index.d.ts +192 -14
  218. package/dist/types/index.js +2 -0
  219. package/dist/types/observability.d.ts +288 -0
  220. package/dist/types/observability.js +2 -0
  221. package/package.json +17 -5
  222. package/src/aiApp.test.ts +207 -1
  223. package/src/aiApp.ts +19 -6
  224. package/src/harness/adminScreens.test.ts +34 -0
  225. package/src/harness/adminScreens.ts +11 -0
  226. package/src/harness/agentLoop.test.ts +1034 -0
  227. package/src/harness/agentLoop.ts +1082 -0
  228. package/src/harness/agentRecovery.test.ts +380 -0
  229. package/src/harness/agentTaskNames.ts +4 -0
  230. package/src/harness/approvalPolicy.ts +28 -0
  231. package/src/harness/approvals.test.ts +1094 -0
  232. package/src/harness/approvals.ts +644 -0
  233. package/src/harness/commit.ts +1049 -0
  234. package/src/harness/conversation.ts +503 -0
  235. package/src/harness/defineAgent.test.ts +92 -0
  236. package/src/harness/defineAgent.ts +82 -0
  237. package/src/harness/defineTask.test.ts +59 -0
  238. package/src/harness/defineTask.ts +76 -0
  239. package/src/harness/defineTool.ts +45 -0
  240. package/src/harness/definitionError.ts +13 -0
  241. package/src/harness/errors.test.ts +90 -0
  242. package/src/harness/errors.ts +119 -0
  243. package/src/harness/eventHub.test.ts +124 -0
  244. package/src/harness/eventHub.ts +159 -0
  245. package/src/harness/events.test.ts +757 -0
  246. package/src/harness/events.ts +384 -0
  247. package/src/harness/executionEnv.ts +34 -0
  248. package/src/harness/extensions.test.ts +1218 -0
  249. package/src/harness/extensions.ts +382 -0
  250. package/src/harness/harness.test.ts +693 -0
  251. package/src/harness/harness.ts +767 -0
  252. package/src/harness/harnessApp.ts +76 -0
  253. package/src/harness/internalRuntime.ts +44 -0
  254. package/src/harness/leases.ts +132 -0
  255. package/src/harness/memo.ts +107 -0
  256. package/src/harness/modelCall.test.ts +134 -0
  257. package/src/harness/modelCall.ts +200 -0
  258. package/src/harness/models/harnessApproval.ts +109 -0
  259. package/src/harness/models/harnessConversation.ts +142 -0
  260. package/src/harness/models/harnessEvent.ts +92 -0
  261. package/src/harness/models/harnessInboxEvent.ts +73 -0
  262. package/src/harness/models/harnessMemo.ts +43 -0
  263. package/src/harness/models/harnessMessage.ts +85 -0
  264. package/src/harness/models/harnessModels.test.ts +60 -0
  265. package/src/harness/models/harnessOwner.ts +40 -0
  266. package/src/harness/models/harnessTask.ts +201 -0
  267. package/src/harness/ownership.test.ts +1087 -0
  268. package/src/harness/ownership.ts +618 -0
  269. package/src/harness/recovery.test.ts +912 -0
  270. package/src/harness/registry.ts +124 -0
  271. package/src/harness/retries.test.ts +292 -0
  272. package/src/harness/retryBackoff.test.ts +100 -0
  273. package/src/harness/retryBackoff.ts +77 -0
  274. package/src/harness/routes/approvals.ts +123 -0
  275. package/src/harness/routes/conversations.test.ts +518 -0
  276. package/src/harness/routes/conversations.ts +102 -0
  277. package/src/harness/routes/events.ts +280 -0
  278. package/src/harness/routes/tasks.test.ts +198 -0
  279. package/src/harness/routes/tasks.ts +146 -0
  280. package/src/harness/runners/inProcessRunner.test.ts +402 -0
  281. package/src/harness/runners/inProcessRunner.ts +220 -0
  282. package/src/harness/runtime.ts +714 -0
  283. package/src/harness/subagent.test.ts +696 -0
  284. package/src/harness/subagent.ts +192 -0
  285. package/src/harness/suspend.ts +13 -0
  286. package/src/harness/transaction.ts +17 -0
  287. package/src/harness/versioning.test.ts +288 -0
  288. package/src/harness/waits.test.ts +693 -0
  289. package/src/harness/waits.ts +356 -0
  290. package/src/index.test.ts +13 -0
  291. package/src/index.ts +70 -0
  292. package/src/isolated/adminScreensConfig.isolated.ts +143 -0
  293. package/src/models/aiDataset.test.ts +34 -0
  294. package/src/models/aiDataset.ts +78 -0
  295. package/src/models/aiRequest.ts +2 -1
  296. package/src/models/gptHistory.test.ts +156 -0
  297. package/src/models/gptHistory.ts +89 -1
  298. package/src/observability/adminScreens.test.ts +71 -0
  299. package/src/observability/adminScreens.ts +42 -0
  300. package/src/observability/compileTemplate.test.ts +14 -0
  301. package/src/observability/compileTemplate.ts +10 -0
  302. package/src/observability/datasetImport.test.ts +95 -0
  303. package/src/observability/datasetImport.ts +220 -0
  304. package/src/observability/evaluate.test.ts +549 -0
  305. package/src/observability/evaluate.ts +254 -0
  306. package/src/observability/evaluatorTemplates.ts +124 -0
  307. package/src/observability/jsonPath.test.ts +52 -0
  308. package/src/observability/jsonPath.ts +60 -0
  309. package/src/observability/local/datasetStore.test.ts +230 -0
  310. package/src/observability/local/datasetStore.ts +452 -0
  311. package/src/observability/local/evaluatorStore.test.ts +138 -0
  312. package/src/observability/local/evaluatorStore.ts +224 -0
  313. package/src/observability/local/experimentRunner.test.ts +489 -0
  314. package/src/observability/local/experimentRunner.ts +548 -0
  315. package/src/observability/local/localPlugin.ts +88 -0
  316. package/src/observability/local/models/obsDataset.ts +38 -0
  317. package/src/observability/local/models/obsDatasetItem.ts +80 -0
  318. package/src/observability/local/models/obsEvaluator.ts +102 -0
  319. package/src/observability/local/models/obsExperiment.ts +91 -0
  320. package/src/observability/local/models/obsExperimentItem.ts +52 -0
  321. package/src/observability/local/models/obsPrompt.ts +43 -0
  322. package/src/observability/local/models/obsPromptLabel.ts +43 -0
  323. package/src/observability/local/models/obsPromptVersion.ts +107 -0
  324. package/src/observability/local/models/obsReviewItem.ts +79 -0
  325. package/src/observability/local/models/obsScore.ts +59 -0
  326. package/src/observability/local/models/obsSpan.ts +65 -0
  327. package/src/observability/local/models/obsTrace.ts +94 -0
  328. package/src/observability/local/obsModels.test.ts +119 -0
  329. package/src/observability/local/promptStore.test.ts +351 -0
  330. package/src/observability/local/promptStore.ts +665 -0
  331. package/src/observability/local/reviewStore.test.ts +164 -0
  332. package/src/observability/local/reviewStore.ts +264 -0
  333. package/src/observability/local/traceStore.test.ts +264 -0
  334. package/src/observability/local/traceStore.ts +440 -0
  335. package/src/observability/observabilityApp.test.ts +92 -0
  336. package/src/observability/observabilityApp.ts +185 -0
  337. package/src/observability/observabilityAppRegistry.test.ts +30 -0
  338. package/src/observability/observabilityAppRegistry.ts +35 -0
  339. package/src/observability/permissions.test.ts +96 -0
  340. package/src/observability/permissions.ts +78 -0
  341. package/src/observability/reviewPanels.test.ts +35 -0
  342. package/src/observability/reviewPanels.ts +64 -0
  343. package/src/observability/routes/datasets.test.ts +176 -0
  344. package/src/observability/routes/datasets.ts +252 -0
  345. package/src/observability/routes/evaluators.test.ts +119 -0
  346. package/src/observability/routes/evaluators.ts +207 -0
  347. package/src/observability/routes/experiments.test.ts +282 -0
  348. package/src/observability/routes/experiments.ts +152 -0
  349. package/src/observability/routes/observabilityRbac.test.ts +767 -0
  350. package/src/observability/routes/observabilityRouteAccess.ts +106 -0
  351. package/src/observability/routes/prompts.test.ts +320 -0
  352. package/src/observability/routes/prompts.ts +263 -0
  353. package/src/observability/routes/requestAiService.ts +36 -0
  354. package/src/observability/routes/review.test.ts +120 -0
  355. package/src/observability/routes/review.ts +125 -0
  356. package/src/observability/routes/status.test.ts +116 -0
  357. package/src/observability/routes/status.ts +47 -0
  358. package/src/observability/routes/testMultiStage.test.ts +214 -0
  359. package/src/observability/routes/testMultiStage.ts +96 -0
  360. package/src/observability/routes/traces.test.ts +167 -0
  361. package/src/observability/routes/traces.ts +172 -0
  362. package/src/observability/schemaValidation.ts +29 -0
  363. package/src/observability/sopThresholds.ts +20 -0
  364. package/src/observability/status.test.ts +38 -0
  365. package/src/observability/status.ts +56 -0
  366. package/src/observability/testMultiStageSchemas.ts +95 -0
  367. package/src/observability/testMultiStageWorkflow.test.ts +114 -0
  368. package/src/observability/testMultiStageWorkflow.ts +307 -0
  369. package/src/observability/types.ts +262 -0
  370. package/src/routes/files.test.ts +63 -9
  371. package/src/routes/files.ts +21 -7
  372. package/src/routes/gpt.test.ts +3279 -0
  373. package/src/routes/gpt.ts +91 -816
  374. package/src/routes/gptActions.test.ts +241 -0
  375. package/src/routes/gptActions.ts +255 -0
  376. package/src/routes/gptApprovals.test.ts +762 -0
  377. package/src/routes/gptDatasets.test.ts +86 -0
  378. package/src/routes/gptDatasets.ts +104 -0
  379. package/src/routes/gptFiles.test.ts +589 -0
  380. package/src/routes/gptHistories.test.ts +933 -0
  381. package/src/routes/gptHistories.ts +266 -5
  382. package/src/routes/routes.test.ts +222 -2
  383. package/src/service/aiDatasets.test.ts +123 -0
  384. package/src/service/aiDatasets.ts +316 -0
  385. package/src/service/aiService.test.ts +593 -0
  386. package/src/service/aiService.ts +568 -111
  387. package/src/service/askFiles.test.ts +96 -0
  388. package/src/service/askFiles.ts +222 -0
  389. package/src/service/asks.test.ts +946 -0
  390. package/src/service/asks.ts +423 -0
  391. package/src/service/chatTurn.ts +2287 -0
  392. package/src/service/fileStorage.test.ts +11 -0
  393. package/src/service/fileStorage.ts +10 -2
  394. package/src/service/fillActionIds.test.ts +65 -0
  395. package/src/service/fillActionIds.ts +98 -0
  396. package/src/service/jsonFenceModel.test.ts +47 -0
  397. package/src/service/jsonFenceModel.ts +53 -0
  398. package/src/service/prompts.test.ts +21 -0
  399. package/src/service/prompts.ts +147 -0
  400. package/src/service/sanitizeHtml.test.ts +61 -0
  401. package/src/service/sanitizeHtml.ts +109 -0
  402. package/src/tests/bunSetup.test.ts +7 -4
  403. package/src/tests/bunSetup.ts +7 -7
  404. package/src/tests/chatHarness.ts +478 -0
  405. package/src/tests/generateStream.ts +49 -0
  406. package/src/tests/harnessErrors.ts +22 -0
  407. package/src/tests/harnessStreaming.ts +227 -0
  408. package/src/tests/helpers.ts +9 -2
  409. package/src/types/harness.ts +1095 -0
  410. package/src/types/index.ts +209 -13
  411. package/src/types/observability.ts +344 -0
  412. package/src/types/sanitize-html.d.ts +25 -0
@@ -0,0 +1,36 @@
1
+ import {APIError} from "@terreno/api";
2
+ import type express from "express";
3
+
4
+ import type {ObservabilityGenerateClient, ObservabilityRequestAiServiceFactory} from "../types";
5
+
6
+ const MISSING_AI_SERVICE_TITLE =
7
+ "No AI service is available. Configure ObservabilityApp.aiService or provide an AI API key.";
8
+
9
+ export interface ResolveRequestAiServiceOptions {
10
+ aiService?: ObservabilityGenerateClient;
11
+ modelId?: string;
12
+ req: express.Request;
13
+ requestAiServiceFactory?: ObservabilityRequestAiServiceFactory;
14
+ }
15
+
16
+ /**
17
+ * Prefer the server-wide client, then let the host build one from the caller's
18
+ * `x-ai-api-key` header so admins can run AI routes with their own provider key.
19
+ */
20
+ export const resolveRequestAiService = ({
21
+ aiService,
22
+ modelId,
23
+ req,
24
+ requestAiServiceFactory,
25
+ }: ResolveRequestAiServiceOptions): ObservabilityGenerateClient => {
26
+ const resolved =
27
+ aiService ??
28
+ requestAiServiceFactory?.({
29
+ apiKey: req.header("x-ai-api-key"),
30
+ modelId,
31
+ });
32
+ if (!resolved) {
33
+ throw new APIError({status: 503, title: MISSING_AI_SERVICE_TITLE});
34
+ }
35
+ return resolved;
36
+ };
@@ -0,0 +1,120 @@
1
+ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
2
+ import {TerrenoApp} from "@terreno/api";
3
+ import {assert} from "chai";
4
+ import type express from "express";
5
+ import {DateTime} from "luxon";
6
+
7
+ import {authAsUser, ensureTestUsers, UserModel} from "../../tests/helpers";
8
+ import {LocalEvaluatorStore} from "../local/evaluatorStore";
9
+ import {createLocalObservabilityPlugin} from "../local/localPlugin";
10
+ import {registerObsEvaluator} from "../local/models/obsEvaluator";
11
+ import {registerObsReviewItem} from "../local/models/obsReviewItem";
12
+ import {registerObsTrace} from "../local/models/obsTrace";
13
+ import {LocalTraceStore} from "../local/traceStore";
14
+ import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
15
+
16
+ describe("observability review routes", () => {
17
+ let app: express.Application;
18
+
19
+ beforeAll(async () => {
20
+ await ensureTestUsers();
21
+ });
22
+
23
+ afterEach(() => {
24
+ resetObservabilityApp();
25
+ });
26
+
27
+ beforeEach(async () => {
28
+ await registerObsReviewItem().deleteMany({});
29
+ await registerObsEvaluator().deleteMany({});
30
+ await registerObsTrace().deleteMany({});
31
+ app = new TerrenoApp({skipListen: true, userModel: UserModel})
32
+ .register(new ObservabilityApp({plugins: [createLocalObservabilityPlugin()]}))
33
+ .build();
34
+ });
35
+
36
+ it("enqueues a trace and lists it pending oldest-first", async () => {
37
+ const evaluator = await new LocalEvaluatorStore().create({
38
+ dimensions: [{dataType: "boolean", key: "correct", required: true}],
39
+ name: "queue-human",
40
+ target: "full trace",
41
+ type: "human",
42
+ });
43
+ const trace = await new LocalTraceStore().exportTrace({
44
+ id: "t",
45
+ input: {text: "hello"},
46
+ name: "gen",
47
+ output: {summary: "hi"},
48
+ prompts: [{name: "summarize", version: 1}],
49
+ sensitive: false,
50
+ spans: [
51
+ {
52
+ id: "s",
53
+ kind: "LLM",
54
+ name: "llm",
55
+ startedAt: DateTime.utc().toISO() ?? "",
56
+ status: "ok",
57
+ },
58
+ ],
59
+ startedAt: DateTime.utc().toISO() ?? "",
60
+ status: "ok",
61
+ });
62
+ const agent = await authAsUser(app, "admin");
63
+ const enqueued = await agent.post("/ai/observability/traces/review").send({
64
+ evaluatorId: evaluator.id,
65
+ reason: "manual",
66
+ traceIds: [trace.id],
67
+ });
68
+ expect(enqueued.status).toBe(201);
69
+ const listed = await agent.get("/ai/observability/review?status=pending");
70
+ expect(listed.status).toBe(200);
71
+ expect(listed.body.counts.pending).toBe(1);
72
+ expect(listed.body.more).toBe(false);
73
+ expect(listed.body.data[0].traceId).toBe(trace.id);
74
+ expect(listed.body.data[0].traceName).toBe("gen");
75
+ const detail = await agent.get(`/ai/observability/review/${listed.body.data[0].id}`);
76
+ expect(detail.status).toBe(200);
77
+ expect(detail.body.data.rawInput).toEqual({text: "hello"});
78
+ expect(detail.body.data.rawOutput).toEqual({summary: "hi"});
79
+
80
+ const submitted = await agent
81
+ .post(`/ai/observability/review/${listed.body.data[0].id}`)
82
+ .send({action: "submit", comment: "grounded", scores: {correct: true}});
83
+ expect(submitted.status).toBe(200);
84
+ expect(submitted.body.data.status).toBe("done");
85
+ const pending = await agent.get("/ai/observability/review?status=pending");
86
+ expect(pending.body.counts.pending).toBe(0);
87
+ expect(pending.body.counts.done).toBe(1);
88
+ });
89
+
90
+ it("rejects a non-human evaluator from human review intake", async () => {
91
+ const evaluator = await new LocalEvaluatorStore().create({
92
+ assertion: {constraint: "exists", path: "output"},
93
+ dimensions: [{dataType: "boolean", key: "valid", required: true}],
94
+ name: "queue-automatic",
95
+ target: "full trace",
96
+ type: "json-assert",
97
+ });
98
+ const trace = await new LocalTraceStore().exportTrace({
99
+ id: "automatic-review",
100
+ input: {text: "hello"},
101
+ name: "gen",
102
+ output: {summary: "hi"},
103
+ prompts: [],
104
+ sensitive: false,
105
+ spans: [],
106
+ startedAt: DateTime.utc().toISO() ?? "",
107
+ status: "ok",
108
+ });
109
+ const response = await (await authAsUser(app, "admin"))
110
+ .post("/ai/observability/traces/review")
111
+ .send({
112
+ evaluatorId: evaluator.id,
113
+ reason: "manual",
114
+ traceIds: [trace.id],
115
+ });
116
+
117
+ assert.equal(response.status, 400);
118
+ assert.equal(response.body.title, "Human review requires a human evaluator");
119
+ });
120
+ });
@@ -0,0 +1,125 @@
1
+ import {APIError, asyncHandler, authenticateMiddleware, createOpenApiBuilder} from "@terreno/api";
2
+ import type express from "express";
3
+
4
+ import type {LocalReviewStore, ReviewStatus} from "../local/reviewStore";
5
+ import {getObservabilityApp} from "../observabilityAppRegistry";
6
+ import {
7
+ type ObservabilityRouteAccessOptions,
8
+ observabilityReviewMutationMiddleware,
9
+ observabilityRouteMiddleware,
10
+ } from "./observabilityRouteAccess";
11
+
12
+ const BASE_PATH = "/ai/observability";
13
+
14
+ export interface ObservabilityReviewRouteOptions extends ObservabilityRouteAccessOptions {
15
+ store: LocalReviewStore;
16
+ }
17
+
18
+ export const addObservabilityReviewRoutes = (
19
+ router: express.Application,
20
+ options: ObservabilityReviewRouteOptions
21
+ ): void => {
22
+ const openApiOptions = options.openApi ? {openApi: options.openApi} : {};
23
+ const builder = (): ReturnType<typeof createOpenApiBuilder> => {
24
+ return createOpenApiBuilder(openApiOptions as Parameters<typeof createOpenApiBuilder>[0]);
25
+ };
26
+
27
+ router.post(
28
+ `${BASE_PATH}/traces/review`,
29
+ observabilityRouteMiddleware(
30
+ options.accessControl,
31
+ {action: "assign", resource: "aiReview"},
32
+ builder()
33
+ .withTags(["observability"])
34
+ .withSummary("Enqueue traces for human review")
35
+ .withResponse(201, {data: {type: "array"}})
36
+ .build()
37
+ ),
38
+ asyncHandler(async (req, res) => {
39
+ const body = req.body as {evaluatorId?: string; reason?: "manual"; traceIds?: string[]};
40
+ if (!body.evaluatorId || !body.traceIds) {
41
+ throw new APIError({status: 400, title: "evaluatorId and traceIds are required"});
42
+ }
43
+ const data = await options.store.enqueue({
44
+ evaluatorId: body.evaluatorId,
45
+ reason: body.reason ?? "manual",
46
+ traceIds: body.traceIds,
47
+ });
48
+ return res.status(201).json({data});
49
+ })
50
+ );
51
+
52
+ router.get(
53
+ `${BASE_PATH}/review`,
54
+ observabilityRouteMiddleware(
55
+ options.accessControl,
56
+ {action: "list", resource: "aiReview"},
57
+ builder()
58
+ .withTags(["observability"])
59
+ .withSummary("List the review queue")
60
+ .withResponse(200, {data: {type: "array"}})
61
+ .build()
62
+ ),
63
+ asyncHandler(async (req, res) => {
64
+ const status = req.query.status as ReviewStatus | undefined;
65
+ return res.json({...(await options.store.list(status)), more: false});
66
+ })
67
+ );
68
+
69
+ router.get(
70
+ `${BASE_PATH}/review/:id`,
71
+ observabilityRouteMiddleware(
72
+ options.accessControl,
73
+ {action: "read", resource: "aiReview"},
74
+ builder()
75
+ .withTags(["observability"])
76
+ .withSummary("Get a review item")
77
+ .withPathParameter("id", {type: "string"})
78
+ .withResponse(200, {data: {type: "object"}})
79
+ .build()
80
+ ),
81
+ asyncHandler(async (req, res) => {
82
+ return res.json({data: await options.store.getDetail(req.params.id)});
83
+ })
84
+ );
85
+
86
+ router.post(
87
+ `${BASE_PATH}/review/:id`,
88
+ [
89
+ authenticateMiddleware(),
90
+ observabilityReviewMutationMiddleware(options.accessControl),
91
+ builder()
92
+ .withTags(["observability"])
93
+ .withSummary("Submit, skip, or assign a review item")
94
+ .withPathParameter("id", {type: "string"})
95
+ .withResponse(200, {data: {type: "object"}})
96
+ .build(),
97
+ ],
98
+ asyncHandler(async (req, res) => {
99
+ const body = req.body as {
100
+ action?: "assign" | "skip" | "submit";
101
+ assigneeId?: string;
102
+ comment?: string;
103
+ scores?: Record<string, boolean | number | string>;
104
+ };
105
+ if (body.action === "submit") {
106
+ await options.store.submit({
107
+ comment: body.comment,
108
+ id: req.params.id,
109
+ scores: body.scores ?? {},
110
+ sinks: getObservabilityApp()?.scoreSinks ?? [],
111
+ });
112
+ } else if (body.action === "skip") {
113
+ await options.store.skip(req.params.id);
114
+ } else if (body.action === "assign") {
115
+ if (!body.assigneeId) {
116
+ throw new APIError({status: 400, title: "assigneeId is required"});
117
+ }
118
+ await options.store.assign(req.params.id, body.assigneeId);
119
+ } else {
120
+ throw new APIError({status: 400, title: "action must be submit, skip, or assign"});
121
+ }
122
+ return res.json({data: await options.store.getDetail(req.params.id)});
123
+ })
124
+ );
125
+ };
@@ -0,0 +1,116 @@
1
+ import {afterEach, beforeAll, beforeEach, describe, expect, it, mock} from "bun:test";
2
+ import {TerrenoApp} from "@terreno/api";
3
+ import type {LanguageModel} from "ai";
4
+ import type express from "express";
5
+
6
+ import {AIService} from "../../service/aiService";
7
+ import {authAsUser, ensureTestUsers, UserModel} from "../../tests/helpers";
8
+ import {createLocalObservabilityPlugin} from "../local/localPlugin";
9
+ import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
10
+ import type {ObservabilityPlugin} from "../types";
11
+
12
+ const createLangfusePlugin = (): ObservabilityPlugin => {
13
+ return {
14
+ capabilities: new Set(["datasets", "experiments", "prompts", "scores", "traces"]),
15
+ datasetStore: {},
16
+ experimentRunner: {},
17
+ id: "langfuse",
18
+ promptRegistry: {get: async () => undefined},
19
+ scoreSink: {export: async () => {}},
20
+ traceSink: {export: async () => {}},
21
+ };
22
+ };
23
+
24
+ describe("observability status routes", () => {
25
+ let app: express.Application;
26
+
27
+ beforeAll(async () => {
28
+ await ensureTestUsers();
29
+ });
30
+
31
+ afterEach(() => {
32
+ resetObservabilityApp();
33
+ });
34
+
35
+ beforeEach(() => {
36
+ app = new TerrenoApp({skipListen: true, userModel: UserModel})
37
+ .register(
38
+ new ObservabilityApp({
39
+ plugins: [createLocalObservabilityPlugin(), createLangfusePlugin()],
40
+ })
41
+ )
42
+ .build();
43
+ });
44
+
45
+ it("returns plugin ids, capabilities, primaries, and local-on for admins", async () => {
46
+ const agent = await authAsUser(app, "admin");
47
+ const response = await agent.get("/ai/observability/status");
48
+ expect(response.status).toBe(200);
49
+ expect(response.body.data.localOn).toBe(true);
50
+ expect(response.body.data.primaries).toEqual({
51
+ datasets: "local",
52
+ experiments: "local",
53
+ prompts: "local",
54
+ reviewQueue: "local",
55
+ });
56
+ expect(response.body.data.plugins.map((plugin: {id: string}) => plugin.id)).toEqual([
57
+ "local",
58
+ "langfuse",
59
+ ]);
60
+ expect(response.body.data.plugins[0].capabilities).toContain("reviewQueue");
61
+ });
62
+
63
+ it("rejects non-admin callers", async () => {
64
+ const agent = await authAsUser(app, "notAdmin");
65
+ const response = await agent.get("/ai/observability/status");
66
+ expect(response.status).toBe(403);
67
+ });
68
+
69
+ it("reports playgroundAi source for server, request-key, and unavailable setups", async () => {
70
+ const unavailableAgent = await authAsUser(app, "admin");
71
+ const unavailable = await unavailableAgent.get("/ai/observability/status");
72
+ expect(unavailable.body.data.playgroundAi).toEqual({source: "unavailable"});
73
+
74
+ const model = {
75
+ doGenerate: mock(async () => ({
76
+ content: [{text: "ok", type: "text" as const}],
77
+ finishReason: "stop" as const,
78
+ usage: {inputTokens: 1, outputTokens: 1},
79
+ })),
80
+ doStream: mock(async () => ({
81
+ stream: new ReadableStream({
82
+ start(controller) {
83
+ controller.close();
84
+ },
85
+ }),
86
+ })),
87
+ modelId: "mock-model",
88
+ provider: "mock-provider",
89
+ specificationVersion: "v2" as const,
90
+ supportedUrls: {},
91
+ };
92
+ const serverApp = new TerrenoApp({skipListen: true, userModel: UserModel})
93
+ .register(
94
+ new ObservabilityApp({
95
+ aiService: new AIService({model: model as unknown as LanguageModel}),
96
+ plugins: [createLocalObservabilityPlugin()],
97
+ })
98
+ )
99
+ .build();
100
+ const serverAgent = await authAsUser(serverApp, "admin");
101
+ const serverStatus = await serverAgent.get("/ai/observability/status");
102
+ expect(serverStatus.body.data.playgroundAi).toEqual({source: "server"});
103
+
104
+ const requestKeyApp = new TerrenoApp({skipListen: true, userModel: UserModel})
105
+ .register(
106
+ new ObservabilityApp({
107
+ plugins: [createLocalObservabilityPlugin()],
108
+ requestAiServiceFactory: () => undefined,
109
+ })
110
+ )
111
+ .build();
112
+ const requestKeyAgent = await authAsUser(requestKeyApp, "admin");
113
+ const requestKeyStatus = await requestKeyAgent.get("/ai/observability/status");
114
+ expect(requestKeyStatus.body.data.playgroundAi).toEqual({source: "request-key"});
115
+ });
116
+ });
@@ -0,0 +1,47 @@
1
+ import {APIError, asyncHandler, createOpenApiBuilder} from "@terreno/api";
2
+ import type express from "express";
3
+ import {getObservabilityApp} from "../observabilityAppRegistry";
4
+ import {resolveObservabilityPermissions} from "../permissions";
5
+ import {buildObservabilityStatus} from "../status";
6
+ import {
7
+ type ObservabilityRouteAccessOptions,
8
+ observabilityRouteMiddleware,
9
+ } from "./observabilityRouteAccess";
10
+
11
+ const BASE_PATH = "/ai/observability";
12
+
13
+ export interface ObservabilityStatusRouteOptions extends ObservabilityRouteAccessOptions {}
14
+
15
+ export const addObservabilityStatusRoutes = (
16
+ router: express.Application,
17
+ options: ObservabilityStatusRouteOptions = {}
18
+ ): void => {
19
+ const openApiOptions = options.openApi ? {openApi: options.openApi} : {};
20
+ const builder = (): ReturnType<typeof createOpenApiBuilder> => {
21
+ return createOpenApiBuilder(openApiOptions as Parameters<typeof createOpenApiBuilder>[0]);
22
+ };
23
+
24
+ router.get(
25
+ `${BASE_PATH}/status`,
26
+ observabilityRouteMiddleware(
27
+ options.accessControl,
28
+ undefined,
29
+ builder()
30
+ .withTags(["observability"])
31
+ .withSummary("Observability plugin status for admin chrome")
32
+ .withResponse(200, {data: {type: "object"}})
33
+ .build()
34
+ ),
35
+ asyncHandler(async (req, res) => {
36
+ const app = getObservabilityApp();
37
+ if (!app) {
38
+ throw new APIError({status: 503, title: "ObservabilityApp is not registered"});
39
+ }
40
+ const permissions = await resolveObservabilityPermissions({
41
+ accessControl: options.accessControl,
42
+ user: req.user,
43
+ });
44
+ return res.json({data: buildObservabilityStatus(app, permissions)});
45
+ })
46
+ );
47
+ };
@@ -0,0 +1,214 @@
1
+ import {afterEach, beforeAll, beforeEach, describe, expect, it, mock} from "bun:test";
2
+ import {TerrenoApp} from "@terreno/api";
3
+ import {assert} from "chai";
4
+ import type express from "express";
5
+
6
+ import {authAsUser, ensureTestUsers, UserModel} from "../../tests/helpers";
7
+ import {createLocalObservabilityPlugin} from "../local/localPlugin";
8
+ import {registerObsSpan} from "../local/models/obsSpan";
9
+ import {registerObsTrace} from "../local/models/obsTrace";
10
+ import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
11
+ import {
12
+ OBS_TEST_MULTI_STAGE_CALL_1_SCHEMA_NAME,
13
+ OBS_TEST_MULTI_STAGE_CALL_2_SCHEMA_NAME,
14
+ OBS_TEST_MULTI_STAGE_FINAL_SCHEMA_NAME,
15
+ } from "../testMultiStageSchemas";
16
+ import type {ObservabilityGenerateClient, SpanKind} from "../types";
17
+
18
+ const MISSING_AI_SERVICE_TITLE =
19
+ "No AI service is available. Configure ObservabilityApp.aiService or provide an AI API key.";
20
+
21
+ const call1Output = {phrase: "alpha phrase"};
22
+ const call2Output = {keywords: ["beta", "gamma"]};
23
+ const finalOutput = {
24
+ keywords: ["beta", "gamma"],
25
+ metrics: {
26
+ call1: {charCount: 12, wordCount: 2},
27
+ call2: {charCount: 11, wordCount: 2},
28
+ combinedCharCount: 23,
29
+ },
30
+ phrase: "alpha phrase",
31
+ sentence: "combined output",
32
+ };
33
+
34
+ const createFakeAiService = (
35
+ behavior: Partial<ObservabilityGenerateClient> & {
36
+ failOn?: string;
37
+ } = {}
38
+ ): ObservabilityGenerateClient => {
39
+ return {
40
+ generateJsonObject: mock(async ({schemaName}) => {
41
+ if (behavior.failOn && schemaName === behavior.failOn) {
42
+ throw new Error(`failed on ${behavior.failOn}`);
43
+ }
44
+ if (schemaName === OBS_TEST_MULTI_STAGE_CALL_1_SCHEMA_NAME) {
45
+ return call1Output as never;
46
+ }
47
+ if (schemaName === OBS_TEST_MULTI_STAGE_CALL_2_SCHEMA_NAME) {
48
+ return call2Output as never;
49
+ }
50
+ if (schemaName === OBS_TEST_MULTI_STAGE_FINAL_SCHEMA_NAME) {
51
+ return finalOutput as never;
52
+ }
53
+ throw new Error(`unexpected schema ${schemaName}`);
54
+ }),
55
+ generateText: mock(async () => {
56
+ throw new Error("multi-stage workflow must use generateJsonObject");
57
+ }),
58
+ modelId: "fake-observability-model",
59
+ ...behavior,
60
+ };
61
+ };
62
+
63
+ const flattenKinds = (nodes: Array<{children: unknown[]; kind: SpanKind}>): SpanKind[] => {
64
+ const kinds: SpanKind[] = [];
65
+ for (const node of nodes) {
66
+ kinds.push(node.kind);
67
+ kinds.push(...flattenKinds(node.children as Array<{children: unknown[]; kind: SpanKind}>));
68
+ }
69
+ return kinds;
70
+ };
71
+
72
+ describe("observability test-multi-stage route", () => {
73
+ let app: express.Application;
74
+ let fakeAiService: ObservabilityGenerateClient;
75
+
76
+ beforeAll(async () => {
77
+ await ensureTestUsers();
78
+ });
79
+
80
+ afterEach(() => {
81
+ resetObservabilityApp();
82
+ });
83
+
84
+ beforeEach(async () => {
85
+ await registerObsTrace().deleteMany({});
86
+ await registerObsSpan().deleteMany({});
87
+ const plugin = createLocalObservabilityPlugin();
88
+ fakeAiService = createFakeAiService();
89
+ app = new TerrenoApp({skipListen: true, userModel: UserModel})
90
+ .register(new ObservabilityApp({aiService: fakeAiService, plugins: [plugin]}))
91
+ .build();
92
+ });
93
+
94
+ it("runs the workflow, exports one nested trace, and returns trace detail", async () => {
95
+ const agent = await authAsUser(app, "admin");
96
+ const res = await agent.post("/ai/observability/traces/test-multi-stage").send({
97
+ input: "Terreno observability smoke input",
98
+ });
99
+ expect(res.status).toBe(200);
100
+ assert.isString(res.body.data.traceId);
101
+ assert.equal(res.body.data.output.sentence, "combined output");
102
+ assert.deepEqual(res.body.data.output.keywords, ["beta", "gamma"]);
103
+ assert.equal(res.body.data.stages.length, 4);
104
+
105
+ const detail = await agent.get(`/ai/observability/traces/${res.body.data.traceId}`);
106
+ expect(detail.status).toBe(200);
107
+ assert.equal(detail.body.data.name, "test-multi-stage");
108
+ assert.equal(flattenKinds(detail.body.data.spans).join(","), "CHAIN,LLM,LLM,TOOL,LLM");
109
+ assert.equal(detail.body.data.spans[0].children[0].parentSpanId, detail.body.data.spans[0].id);
110
+ assert.equal(detail.body.data.output.sentence, "combined output");
111
+ assert.equal(
112
+ detail.body.data.spans[0].children[0].input.schemaName,
113
+ OBS_TEST_MULTI_STAGE_CALL_1_SCHEMA_NAME
114
+ );
115
+ assert.isOk(detail.body.data.spans[0].children[0].input.outputSchema);
116
+ });
117
+
118
+ it("returns 403 for non-admin callers", async () => {
119
+ const agent = await authAsUser(app, "notAdmin");
120
+ const res = await agent.post("/ai/observability/traces/test-multi-stage").send({
121
+ input: "blocked",
122
+ });
123
+ expect(res.status).toBe(403);
124
+ });
125
+
126
+ it("returns 503 when no server or request AI service is available", async () => {
127
+ const plugin = createLocalObservabilityPlugin();
128
+ const noAiApp = new TerrenoApp({skipListen: true, userModel: UserModel})
129
+ .register(new ObservabilityApp({plugins: [plugin]}))
130
+ .build();
131
+ const agent = await authAsUser(noAiApp, "admin");
132
+ const res = await agent.post("/ai/observability/traces/test-multi-stage").send({
133
+ input: "missing service",
134
+ });
135
+ expect(res.status).toBe(503);
136
+ assert.equal(res.body.title, MISSING_AI_SERVICE_TITLE);
137
+ });
138
+
139
+ it("builds an AI service from the request api key when the server has none", async () => {
140
+ const requestAiServiceFactory = mock(({apiKey}: {apiKey?: string}) => {
141
+ return apiKey === "request-key" ? createFakeAiService() : undefined;
142
+ });
143
+ const plugin = createLocalObservabilityPlugin();
144
+ const requestKeyApp = new TerrenoApp({skipListen: true, userModel: UserModel})
145
+ .register(new ObservabilityApp({plugins: [plugin], requestAiServiceFactory}))
146
+ .build();
147
+ const agent = await authAsUser(requestKeyApp, "admin");
148
+
149
+ const res = await agent
150
+ .post("/ai/observability/traces/test-multi-stage")
151
+ .set("x-ai-api-key", "request-key")
152
+ .send({input: "request key smoke test"});
153
+
154
+ expect(res.status).toBe(200);
155
+ assert.isString(res.body.data.traceId);
156
+ assert.equal(requestAiServiceFactory.mock.calls[0]?.[0].apiKey, "request-key");
157
+ });
158
+
159
+ it("exports an error trace and rethrows when a child LLM stage fails", async () => {
160
+ const failingService = createFakeAiService({failOn: OBS_TEST_MULTI_STAGE_CALL_2_SCHEMA_NAME});
161
+ const plugin = createLocalObservabilityPlugin();
162
+ const failingApp = new TerrenoApp({skipListen: true, userModel: UserModel})
163
+ .register(new ObservabilityApp({aiService: failingService, plugins: [plugin]}))
164
+ .build();
165
+ const agent = await authAsUser(failingApp, "admin");
166
+ const res = await agent.post("/ai/observability/traces/test-multi-stage").send({
167
+ input: "fail on second stage",
168
+ });
169
+ expect(res.status).toBe(500);
170
+
171
+ const listed = await agent.get("/ai/observability/traces?status=error&limit=5");
172
+ expect(listed.status).toBe(200);
173
+ assert.isAtLeast(listed.body.data.length, 1);
174
+ const traceId = listed.body.data[0].id as string;
175
+ const detail = await agent.get(`/ai/observability/traces/${traceId}`);
176
+ assert.equal(detail.body.data.status, "error");
177
+ const childKinds = flattenKinds(detail.body.data.spans);
178
+ assert.include(childKinds, "LLM");
179
+ assert.equal(childKinds.filter((kind) => kind === "LLM").length, 2);
180
+ const failedSpan = detail.body.data.spans[0].children.find(
181
+ (span: {name: string; status: string}) => span.name === "call-2"
182
+ );
183
+ assert.equal(failedSpan?.status, "error");
184
+ });
185
+
186
+ it("exports the completed child stages when final synthesis fails", async () => {
187
+ const failingService = createFakeAiService({failOn: OBS_TEST_MULTI_STAGE_FINAL_SCHEMA_NAME});
188
+ const plugin = createLocalObservabilityPlugin();
189
+ const failingApp = new TerrenoApp({skipListen: true, userModel: UserModel})
190
+ .register(new ObservabilityApp({aiService: failingService, plugins: [plugin]}))
191
+ .build();
192
+ const agent = await authAsUser(failingApp, "admin");
193
+
194
+ const res = await agent.post("/ai/observability/traces/test-multi-stage").send({
195
+ input: "fail on final stage",
196
+ });
197
+
198
+ expect(res.status).toBe(500);
199
+ const listed = await agent.get("/ai/observability/traces?status=error&limit=5");
200
+ const traceId = listed.body.data[0].id as string;
201
+ const detail = await agent.get(`/ai/observability/traces/${traceId}`);
202
+ const children = detail.body.data.spans[0].children as Array<{
203
+ kind: SpanKind;
204
+ name: string;
205
+ status: string;
206
+ }>;
207
+ assert.deepEqual(
208
+ children.map((span) => span.kind),
209
+ ["LLM", "LLM", "TOOL", "LLM"]
210
+ );
211
+ assert.equal(children.at(-1)?.name, "final");
212
+ assert.equal(children.at(-1)?.status, "error");
213
+ });
214
+ });