@terreno/ai 57.9.0 → 57.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (417) hide show
  1. package/README.md +1 -0
  2. package/bunfig.toml +1 -1
  3. package/dist/aiApp.d.ts +8 -2
  4. package/dist/aiApp.js +12 -6
  5. package/dist/harness/adminScreens.d.ts +6 -0
  6. package/dist/harness/adminScreens.js +11 -0
  7. package/dist/harness/agentLoop.d.ts +61 -0
  8. package/dist/harness/agentLoop.js +798 -0
  9. package/dist/harness/agentTaskNames.d.ts +4 -0
  10. package/dist/harness/agentTaskNames.js +7 -0
  11. package/dist/harness/approvalPolicy.d.ts +3 -0
  12. package/dist/harness/approvalPolicy.js +23 -0
  13. package/dist/harness/approvals.d.ts +101 -0
  14. package/dist/harness/approvals.js +468 -0
  15. package/dist/harness/commit.d.ts +238 -0
  16. package/dist/harness/commit.js +678 -0
  17. package/dist/harness/conversation.d.ts +75 -0
  18. package/dist/harness/conversation.js +397 -0
  19. package/dist/harness/defineAgent.d.ts +9 -0
  20. package/dist/harness/defineAgent.js +65 -0
  21. package/dist/harness/defineTask.d.ts +11 -0
  22. package/dist/harness/defineTask.js +59 -0
  23. package/dist/harness/defineTool.d.ts +7 -0
  24. package/dist/harness/defineTool.js +36 -0
  25. package/dist/harness/definitionError.d.ts +8 -0
  26. package/dist/harness/definitionError.js +15 -0
  27. package/dist/harness/errors.d.ts +133 -0
  28. package/dist/harness/errors.js +107 -0
  29. package/dist/harness/eventHub.d.ts +45 -0
  30. package/dist/harness/eventHub.js +135 -0
  31. package/dist/harness/events.d.ts +95 -0
  32. package/dist/harness/events.js +267 -0
  33. package/dist/harness/executionEnv.d.ts +31 -0
  34. package/dist/harness/executionEnv.js +2 -0
  35. package/dist/harness/extensions.d.ts +82 -0
  36. package/dist/harness/extensions.js +274 -0
  37. package/dist/harness/harness.d.ts +163 -0
  38. package/dist/harness/harness.js +522 -0
  39. package/dist/harness/harnessApp.d.ts +44 -0
  40. package/dist/harness/harnessApp.js +62 -0
  41. package/dist/harness/internalRuntime.d.ts +16 -0
  42. package/dist/harness/internalRuntime.js +18 -0
  43. package/dist/harness/leases.d.ts +35 -0
  44. package/dist/harness/leases.js +86 -0
  45. package/dist/harness/memo.d.ts +24 -0
  46. package/dist/harness/memo.js +78 -0
  47. package/dist/harness/modelCall.d.ts +45 -0
  48. package/dist/harness/modelCall.js +147 -0
  49. package/dist/harness/models/harnessApproval.d.ts +2 -0
  50. package/dist/harness/models/harnessApproval.js +100 -0
  51. package/dist/harness/models/harnessConversation.d.ts +2 -0
  52. package/dist/harness/models/harnessConversation.js +117 -0
  53. package/dist/harness/models/harnessEvent.d.ts +3 -0
  54. package/dist/harness/models/harnessEvent.js +73 -0
  55. package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
  56. package/dist/harness/models/harnessInboxEvent.js +58 -0
  57. package/dist/harness/models/harnessMemo.d.ts +2 -0
  58. package/dist/harness/models/harnessMemo.js +42 -0
  59. package/dist/harness/models/harnessMessage.d.ts +2 -0
  60. package/dist/harness/models/harnessMessage.js +73 -0
  61. package/dist/harness/models/harnessOwner.d.ts +2 -0
  62. package/dist/harness/models/harnessOwner.js +36 -0
  63. package/dist/harness/models/harnessTask.d.ts +2 -0
  64. package/dist/harness/models/harnessTask.js +191 -0
  65. package/dist/harness/ownership.d.ts +89 -0
  66. package/dist/harness/ownership.js +438 -0
  67. package/dist/harness/registry.d.ts +29 -0
  68. package/dist/harness/registry.js +93 -0
  69. package/dist/harness/retryBackoff.d.ts +31 -0
  70. package/dist/harness/retryBackoff.js +50 -0
  71. package/dist/harness/routes/approvals.d.ts +13 -0
  72. package/dist/harness/routes/approvals.js +93 -0
  73. package/dist/harness/routes/conversations.d.ts +11 -0
  74. package/dist/harness/routes/conversations.js +66 -0
  75. package/dist/harness/routes/events.d.ts +19 -0
  76. package/dist/harness/routes/events.js +215 -0
  77. package/dist/harness/routes/tasks.d.ts +11 -0
  78. package/dist/harness/routes/tasks.js +108 -0
  79. package/dist/harness/runners/inProcessRunner.d.ts +64 -0
  80. package/dist/harness/runners/inProcessRunner.js +227 -0
  81. package/dist/harness/runners/jobsRunner.d.ts +62 -0
  82. package/dist/harness/runners/jobsRunner.js +212 -0
  83. package/dist/harness/runtime.d.ts +52 -0
  84. package/dist/harness/runtime.js +604 -0
  85. package/dist/harness/subagent.d.ts +53 -0
  86. package/dist/harness/subagent.js +134 -0
  87. package/dist/harness/suspend.d.ts +8 -0
  88. package/dist/harness/suspend.js +16 -0
  89. package/dist/harness/transaction.d.ts +3 -0
  90. package/dist/harness/transaction.js +22 -0
  91. package/dist/harness/waits.d.ts +100 -0
  92. package/dist/harness/waits.js +214 -0
  93. package/dist/index.d.ts +19 -1
  94. package/dist/index.js +45 -2
  95. package/dist/models/aiDataset.d.ts +2 -0
  96. package/dist/models/aiDataset.js +67 -0
  97. package/dist/models/aiRequest.js +1 -1
  98. package/dist/models/gptHistory.js +73 -0
  99. package/dist/observability/adminScreens.d.ts +5 -0
  100. package/dist/observability/adminScreens.js +32 -0
  101. package/dist/observability/compileTemplate.d.ts +1 -0
  102. package/dist/observability/compileTemplate.js +13 -0
  103. package/dist/observability/datasetImport.d.ts +11 -0
  104. package/dist/observability/datasetImport.js +185 -0
  105. package/dist/observability/evaluate.d.ts +31 -0
  106. package/dist/observability/evaluate.js +200 -0
  107. package/dist/observability/evaluatorTemplates.d.ts +17 -0
  108. package/dist/observability/evaluatorTemplates.js +111 -0
  109. package/dist/observability/jsonPath.d.ts +2 -0
  110. package/dist/observability/jsonPath.js +57 -0
  111. package/dist/observability/local/datasetStore.d.ts +87 -0
  112. package/dist/observability/local/datasetStore.js +326 -0
  113. package/dist/observability/local/evaluatorStore.d.ts +44 -0
  114. package/dist/observability/local/evaluatorStore.js +182 -0
  115. package/dist/observability/local/experimentRunner.d.ts +96 -0
  116. package/dist/observability/local/experimentRunner.js +419 -0
  117. package/dist/observability/local/localPlugin.d.ts +22 -0
  118. package/dist/observability/local/localPlugin.js +76 -0
  119. package/dist/observability/local/models/obsDataset.d.ts +2 -0
  120. package/dist/observability/local/models/obsDataset.js +37 -0
  121. package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
  122. package/dist/observability/local/models/obsDatasetItem.js +76 -0
  123. package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
  124. package/dist/observability/local/models/obsEvaluator.js +98 -0
  125. package/dist/observability/local/models/obsExperiment.d.ts +2 -0
  126. package/dist/observability/local/models/obsExperiment.js +87 -0
  127. package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
  128. package/dist/observability/local/models/obsExperimentItem.js +45 -0
  129. package/dist/observability/local/models/obsPrompt.d.ts +2 -0
  130. package/dist/observability/local/models/obsPrompt.js +42 -0
  131. package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
  132. package/dist/observability/local/models/obsPromptLabel.js +39 -0
  133. package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
  134. package/dist/observability/local/models/obsPromptVersion.js +96 -0
  135. package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
  136. package/dist/observability/local/models/obsReviewItem.js +75 -0
  137. package/dist/observability/local/models/obsScore.d.ts +2 -0
  138. package/dist/observability/local/models/obsScore.js +58 -0
  139. package/dist/observability/local/models/obsSpan.d.ts +2 -0
  140. package/dist/observability/local/models/obsSpan.js +64 -0
  141. package/dist/observability/local/models/obsTrace.d.ts +2 -0
  142. package/dist/observability/local/models/obsTrace.js +93 -0
  143. package/dist/observability/local/promptStore.d.ts +180 -0
  144. package/dist/observability/local/promptStore.js +444 -0
  145. package/dist/observability/local/reviewStore.d.ts +50 -0
  146. package/dist/observability/local/reviewStore.js +203 -0
  147. package/dist/observability/local/traceStore.d.ts +103 -0
  148. package/dist/observability/local/traceStore.js +325 -0
  149. package/dist/observability/observabilityApp.d.ts +24 -0
  150. package/dist/observability/observabilityApp.js +157 -0
  151. package/dist/observability/observabilityAppRegistry.d.ts +14 -0
  152. package/dist/observability/observabilityAppRegistry.js +16 -0
  153. package/dist/observability/permissions.d.ts +7 -0
  154. package/dist/observability/permissions.js +60 -0
  155. package/dist/observability/reviewPanels.d.ts +18 -0
  156. package/dist/observability/reviewPanels.js +46 -0
  157. package/dist/observability/routes/datasets.d.ts +7 -0
  158. package/dist/observability/routes/datasets.js +149 -0
  159. package/dist/observability/routes/evaluators.d.ts +7 -0
  160. package/dist/observability/routes/evaluators.js +123 -0
  161. package/dist/observability/routes/experiments.d.ts +7 -0
  162. package/dist/observability/routes/experiments.js +95 -0
  163. package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
  164. package/dist/observability/routes/observabilityRouteAccess.js +67 -0
  165. package/dist/observability/routes/prompts.d.ts +12 -0
  166. package/dist/observability/routes/prompts.js +172 -0
  167. package/dist/observability/routes/requestAiService.d.ts +13 -0
  168. package/dist/observability/routes/requestAiService.js +21 -0
  169. package/dist/observability/routes/review.d.ts +7 -0
  170. package/dist/observability/routes/review.js +79 -0
  171. package/dist/observability/routes/status.d.ts +5 -0
  172. package/dist/observability/routes/status.js +31 -0
  173. package/dist/observability/routes/testMultiStage.d.ts +9 -0
  174. package/dist/observability/routes/testMultiStage.js +67 -0
  175. package/dist/observability/routes/traces.d.ts +7 -0
  176. package/dist/observability/routes/traces.js +129 -0
  177. package/dist/observability/schemaValidation.d.ts +5 -0
  178. package/dist/observability/schemaValidation.js +26 -0
  179. package/dist/observability/sopThresholds.d.ts +2 -0
  180. package/dist/observability/sopThresholds.js +21 -0
  181. package/dist/observability/status.d.ts +23 -0
  182. package/dist/observability/status.js +34 -0
  183. package/dist/observability/testMultiStageSchemas.d.ts +10 -0
  184. package/dist/observability/testMultiStageSchemas.js +79 -0
  185. package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
  186. package/dist/observability/testMultiStageWorkflow.js +218 -0
  187. package/dist/observability/types.d.ts +159 -0
  188. package/dist/observability/types.js +67 -0
  189. package/dist/routes/files.js +18 -7
  190. package/dist/routes/gpt.js +79 -689
  191. package/dist/routes/gptActions.d.ts +9 -0
  192. package/dist/routes/gptActions.js +212 -0
  193. package/dist/routes/gptDatasets.d.ts +5 -0
  194. package/dist/routes/gptDatasets.js +94 -0
  195. package/dist/routes/gptHistories.d.ts +3 -1
  196. package/dist/routes/gptHistories.js +188 -2
  197. package/dist/service/aiDatasets.d.ts +57 -0
  198. package/dist/service/aiDatasets.js +253 -0
  199. package/dist/service/aiService.d.ts +50 -1
  200. package/dist/service/aiService.js +426 -77
  201. package/dist/service/askFiles.d.ts +49 -0
  202. package/dist/service/askFiles.js +162 -0
  203. package/dist/service/asks.d.ts +76 -0
  204. package/dist/service/asks.js +284 -0
  205. package/dist/service/chatTurn.d.ts +128 -0
  206. package/dist/service/chatTurn.js +1674 -0
  207. package/dist/service/fileStorage.d.ts +4 -0
  208. package/dist/service/fileStorage.js +7 -2
  209. package/dist/service/fillActionIds.d.ts +6 -0
  210. package/dist/service/fillActionIds.js +79 -0
  211. package/dist/service/jsonFenceModel.d.ts +7 -0
  212. package/dist/service/jsonFenceModel.js +45 -0
  213. package/dist/service/prompts.d.ts +52 -0
  214. package/dist/service/prompts.js +96 -1
  215. package/dist/service/sanitizeHtml.d.ts +11 -0
  216. package/dist/service/sanitizeHtml.js +108 -0
  217. package/dist/types/harness.d.ts +952 -0
  218. package/dist/types/harness.js +133 -0
  219. package/dist/types/index.d.ts +192 -14
  220. package/dist/types/index.js +2 -0
  221. package/dist/types/observability.d.ts +288 -0
  222. package/dist/types/observability.js +2 -0
  223. package/package.json +28 -5
  224. package/src/aiApp.test.ts +207 -1
  225. package/src/aiApp.ts +19 -6
  226. package/src/harness/adminScreens.test.ts +34 -0
  227. package/src/harness/adminScreens.ts +11 -0
  228. package/src/harness/agentLoop.test.ts +1034 -0
  229. package/src/harness/agentLoop.ts +1082 -0
  230. package/src/harness/agentRecovery.test.ts +380 -0
  231. package/src/harness/agentTaskNames.ts +4 -0
  232. package/src/harness/approvalPolicy.ts +28 -0
  233. package/src/harness/approvals.test.ts +1094 -0
  234. package/src/harness/approvals.ts +644 -0
  235. package/src/harness/commit.ts +1049 -0
  236. package/src/harness/concurrency.test.ts +155 -0
  237. package/src/harness/conversation.ts +503 -0
  238. package/src/harness/defineAgent.test.ts +92 -0
  239. package/src/harness/defineAgent.ts +82 -0
  240. package/src/harness/defineTask.test.ts +59 -0
  241. package/src/harness/defineTask.ts +76 -0
  242. package/src/harness/defineTool.ts +45 -0
  243. package/src/harness/definitionError.ts +13 -0
  244. package/src/harness/errors.test.ts +90 -0
  245. package/src/harness/errors.ts +125 -0
  246. package/src/harness/eventHub.test.ts +124 -0
  247. package/src/harness/eventHub.ts +159 -0
  248. package/src/harness/events.test.ts +769 -0
  249. package/src/harness/events.ts +384 -0
  250. package/src/harness/executionEnv.ts +34 -0
  251. package/src/harness/extensions.test.ts +1218 -0
  252. package/src/harness/extensions.ts +382 -0
  253. package/src/harness/harness.test.ts +693 -0
  254. package/src/harness/harness.ts +778 -0
  255. package/src/harness/harnessApp.ts +76 -0
  256. package/src/harness/internalRuntime.ts +44 -0
  257. package/src/harness/leases.ts +132 -0
  258. package/src/harness/memo.ts +107 -0
  259. package/src/harness/modelCall.test.ts +134 -0
  260. package/src/harness/modelCall.ts +200 -0
  261. package/src/harness/models/harnessApproval.ts +109 -0
  262. package/src/harness/models/harnessConversation.ts +142 -0
  263. package/src/harness/models/harnessEvent.ts +92 -0
  264. package/src/harness/models/harnessInboxEvent.ts +73 -0
  265. package/src/harness/models/harnessMemo.ts +43 -0
  266. package/src/harness/models/harnessMessage.ts +85 -0
  267. package/src/harness/models/harnessModels.test.ts +60 -0
  268. package/src/harness/models/harnessOwner.ts +40 -0
  269. package/src/harness/models/harnessTask.ts +206 -0
  270. package/src/harness/ownership.test.ts +1087 -0
  271. package/src/harness/ownership.ts +618 -0
  272. package/src/harness/recovery.test.ts +907 -0
  273. package/src/harness/registry.ts +124 -0
  274. package/src/harness/retries.test.ts +292 -0
  275. package/src/harness/retryBackoff.test.ts +100 -0
  276. package/src/harness/retryBackoff.ts +77 -0
  277. package/src/harness/routes/approvals.ts +123 -0
  278. package/src/harness/routes/conversations.test.ts +518 -0
  279. package/src/harness/routes/conversations.ts +102 -0
  280. package/src/harness/routes/events.ts +280 -0
  281. package/src/harness/routes/tasks.test.ts +198 -0
  282. package/src/harness/routes/tasks.ts +146 -0
  283. package/src/harness/runners/inProcessRunner.test.ts +530 -0
  284. package/src/harness/runners/inProcessRunner.ts +260 -0
  285. package/src/harness/runners/jobsRunner.test.ts +709 -0
  286. package/src/harness/runners/jobsRunner.ts +252 -0
  287. package/src/harness/runtime.ts +802 -0
  288. package/src/harness/subagent.test.ts +696 -0
  289. package/src/harness/subagent.ts +192 -0
  290. package/src/harness/suspend.ts +13 -0
  291. package/src/harness/transaction.ts +17 -0
  292. package/src/harness/versioning.test.ts +288 -0
  293. package/src/harness/waits.test.ts +693 -0
  294. package/src/harness/waits.ts +356 -0
  295. package/src/index.test.ts +13 -0
  296. package/src/index.ts +70 -0
  297. package/src/isolated/adminScreensConfig.isolated.ts +143 -0
  298. package/src/models/aiDataset.test.ts +34 -0
  299. package/src/models/aiDataset.ts +78 -0
  300. package/src/models/aiRequest.ts +2 -1
  301. package/src/models/gptHistory.test.ts +156 -0
  302. package/src/models/gptHistory.ts +89 -1
  303. package/src/observability/adminScreens.test.ts +71 -0
  304. package/src/observability/adminScreens.ts +42 -0
  305. package/src/observability/compileTemplate.test.ts +14 -0
  306. package/src/observability/compileTemplate.ts +10 -0
  307. package/src/observability/datasetImport.test.ts +95 -0
  308. package/src/observability/datasetImport.ts +220 -0
  309. package/src/observability/evaluate.test.ts +549 -0
  310. package/src/observability/evaluate.ts +254 -0
  311. package/src/observability/evaluatorTemplates.ts +124 -0
  312. package/src/observability/jsonPath.test.ts +52 -0
  313. package/src/observability/jsonPath.ts +60 -0
  314. package/src/observability/local/datasetStore.test.ts +230 -0
  315. package/src/observability/local/datasetStore.ts +452 -0
  316. package/src/observability/local/evaluatorStore.test.ts +138 -0
  317. package/src/observability/local/evaluatorStore.ts +224 -0
  318. package/src/observability/local/experimentRunner.test.ts +489 -0
  319. package/src/observability/local/experimentRunner.ts +548 -0
  320. package/src/observability/local/localPlugin.ts +88 -0
  321. package/src/observability/local/models/obsDataset.ts +38 -0
  322. package/src/observability/local/models/obsDatasetItem.ts +80 -0
  323. package/src/observability/local/models/obsEvaluator.ts +102 -0
  324. package/src/observability/local/models/obsExperiment.ts +91 -0
  325. package/src/observability/local/models/obsExperimentItem.ts +52 -0
  326. package/src/observability/local/models/obsPrompt.ts +43 -0
  327. package/src/observability/local/models/obsPromptLabel.ts +43 -0
  328. package/src/observability/local/models/obsPromptVersion.ts +107 -0
  329. package/src/observability/local/models/obsReviewItem.ts +79 -0
  330. package/src/observability/local/models/obsScore.ts +59 -0
  331. package/src/observability/local/models/obsSpan.ts +65 -0
  332. package/src/observability/local/models/obsTrace.ts +94 -0
  333. package/src/observability/local/obsModels.test.ts +119 -0
  334. package/src/observability/local/promptStore.test.ts +351 -0
  335. package/src/observability/local/promptStore.ts +665 -0
  336. package/src/observability/local/reviewStore.test.ts +164 -0
  337. package/src/observability/local/reviewStore.ts +264 -0
  338. package/src/observability/local/traceStore.test.ts +264 -0
  339. package/src/observability/local/traceStore.ts +440 -0
  340. package/src/observability/observabilityApp.test.ts +92 -0
  341. package/src/observability/observabilityApp.ts +185 -0
  342. package/src/observability/observabilityAppRegistry.test.ts +30 -0
  343. package/src/observability/observabilityAppRegistry.ts +35 -0
  344. package/src/observability/permissions.test.ts +96 -0
  345. package/src/observability/permissions.ts +78 -0
  346. package/src/observability/reviewPanels.test.ts +35 -0
  347. package/src/observability/reviewPanels.ts +64 -0
  348. package/src/observability/routes/datasets.test.ts +176 -0
  349. package/src/observability/routes/datasets.ts +252 -0
  350. package/src/observability/routes/evaluators.test.ts +119 -0
  351. package/src/observability/routes/evaluators.ts +207 -0
  352. package/src/observability/routes/experiments.test.ts +282 -0
  353. package/src/observability/routes/experiments.ts +152 -0
  354. package/src/observability/routes/observabilityRbac.test.ts +767 -0
  355. package/src/observability/routes/observabilityRouteAccess.ts +106 -0
  356. package/src/observability/routes/prompts.test.ts +320 -0
  357. package/src/observability/routes/prompts.ts +263 -0
  358. package/src/observability/routes/requestAiService.ts +36 -0
  359. package/src/observability/routes/review.test.ts +120 -0
  360. package/src/observability/routes/review.ts +125 -0
  361. package/src/observability/routes/status.test.ts +116 -0
  362. package/src/observability/routes/status.ts +47 -0
  363. package/src/observability/routes/testMultiStage.test.ts +214 -0
  364. package/src/observability/routes/testMultiStage.ts +96 -0
  365. package/src/observability/routes/traces.test.ts +167 -0
  366. package/src/observability/routes/traces.ts +172 -0
  367. package/src/observability/schemaValidation.ts +29 -0
  368. package/src/observability/sopThresholds.ts +20 -0
  369. package/src/observability/status.test.ts +38 -0
  370. package/src/observability/status.ts +56 -0
  371. package/src/observability/testMultiStageSchemas.ts +95 -0
  372. package/src/observability/testMultiStageWorkflow.test.ts +114 -0
  373. package/src/observability/testMultiStageWorkflow.ts +307 -0
  374. package/src/observability/types.ts +262 -0
  375. package/src/routes/files.test.ts +63 -9
  376. package/src/routes/files.ts +21 -7
  377. package/src/routes/gpt.test.ts +3279 -0
  378. package/src/routes/gpt.ts +91 -816
  379. package/src/routes/gptActions.test.ts +241 -0
  380. package/src/routes/gptActions.ts +255 -0
  381. package/src/routes/gptApprovals.test.ts +762 -0
  382. package/src/routes/gptDatasets.test.ts +86 -0
  383. package/src/routes/gptDatasets.ts +104 -0
  384. package/src/routes/gptFiles.test.ts +589 -0
  385. package/src/routes/gptHistories.test.ts +933 -0
  386. package/src/routes/gptHistories.ts +266 -5
  387. package/src/routes/routes.test.ts +222 -2
  388. package/src/service/aiDatasets.test.ts +123 -0
  389. package/src/service/aiDatasets.ts +316 -0
  390. package/src/service/aiService.test.ts +593 -0
  391. package/src/service/aiService.ts +568 -111
  392. package/src/service/askFiles.test.ts +96 -0
  393. package/src/service/askFiles.ts +222 -0
  394. package/src/service/asks.test.ts +946 -0
  395. package/src/service/asks.ts +423 -0
  396. package/src/service/chatTurn.ts +2287 -0
  397. package/src/service/fileStorage.test.ts +11 -0
  398. package/src/service/fileStorage.ts +10 -2
  399. package/src/service/fillActionIds.test.ts +65 -0
  400. package/src/service/fillActionIds.ts +98 -0
  401. package/src/service/jsonFenceModel.test.ts +47 -0
  402. package/src/service/jsonFenceModel.ts +53 -0
  403. package/src/service/prompts.test.ts +21 -0
  404. package/src/service/prompts.ts +147 -0
  405. package/src/service/sanitizeHtml.test.ts +61 -0
  406. package/src/service/sanitizeHtml.ts +109 -0
  407. package/src/tests/bunSetup.test.ts +7 -4
  408. package/src/tests/bunSetup.ts +7 -7
  409. package/src/tests/chatHarness.ts +478 -0
  410. package/src/tests/generateStream.ts +49 -0
  411. package/src/tests/harnessErrors.ts +22 -0
  412. package/src/tests/harnessStreaming.ts +227 -0
  413. package/src/tests/helpers.ts +9 -2
  414. package/src/types/harness.ts +1119 -0
  415. package/src/types/index.ts +209 -13
  416. package/src/types/observability.ts +344 -0
  417. package/src/types/sanitize-html.d.ts +25 -0
@@ -0,0 +1,767 @@
1
+ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
2
+ import {
3
+ addAuthRoutes,
4
+ apiErrorMiddleware,
5
+ apiUnauthorizedMiddleware,
6
+ createAccess,
7
+ setupAuth,
8
+ TerrenoApp,
9
+ terrenoStatements,
10
+ type UserModel as UserModelType,
11
+ } from "@terreno/api";
12
+ import {getBaseServer} from "@terreno/test";
13
+ import type express from "express";
14
+ import {DateTime} from "luxon";
15
+ import mongoose from "mongoose";
16
+ import type TestAgent from "supertest/lib/agent";
17
+
18
+ import {
19
+ authAsUser,
20
+ authAsUserWithCredentials,
21
+ ensureTestUsers,
22
+ UserModel,
23
+ } from "../../tests/helpers";
24
+ import {LocalDatasetStore} from "../local/datasetStore";
25
+ import {LocalEvaluatorStore} from "../local/evaluatorStore";
26
+ import {createLocalObservabilityPlugin} from "../local/localPlugin";
27
+ import {registerObsDataset} from "../local/models/obsDataset";
28
+ import {registerObsDatasetItem} from "../local/models/obsDatasetItem";
29
+ import {registerObsEvaluator} from "../local/models/obsEvaluator";
30
+ import {registerObsPrompt} from "../local/models/obsPrompt";
31
+ import {registerObsPromptLabel} from "../local/models/obsPromptLabel";
32
+ import {registerObsPromptVersion} from "../local/models/obsPromptVersion";
33
+ import {registerObsReviewItem} from "../local/models/obsReviewItem";
34
+ import {registerObsTrace} from "../local/models/obsTrace";
35
+ import {LocalPromptStore} from "../local/promptStore";
36
+ import {LocalTraceStore} from "../local/traceStore";
37
+ import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
38
+
39
+ const typedUserModel = UserModel as unknown as UserModelType;
40
+ const ADMIN_EMAIL = "admin@example.com";
41
+ const RBAC_OPERATOR_EMAIL = "obs-rbac-operator@example.com";
42
+ const RBAC_OPERATOR_PASSWORD = "obs-rbac-password";
43
+ const FIXTURE_PROMPT = "rbac-fixture";
44
+ const ADMIN_SHELL: Record<string, string[]> = {admin: ["access"]};
45
+
46
+ type PasswordedUser = {setPassword: (password: string) => Promise<void>};
47
+
48
+ interface RouteMappingCase {
49
+ allowAction: string;
50
+ body?: Record<string, unknown>;
51
+ denyAction: string;
52
+ label: string;
53
+ method: "delete" | "get" | "patch" | "post";
54
+ path: string;
55
+ resource: string;
56
+ }
57
+
58
+ interface FixtureIds {
59
+ datasetId: string;
60
+ datasetItemId: string;
61
+ evaluatorId: string;
62
+ reviewItemId: string;
63
+ traceId: string;
64
+ }
65
+
66
+ const ensureRbacOperatorUser = async (): Promise<void> => {
67
+ await UserModel.deleteMany({email: RBAC_OPERATOR_EMAIL});
68
+ const doc = await UserModel.create({
69
+ admin: false,
70
+ email: RBAC_OPERATOR_EMAIL,
71
+ name: "Obs RBAC Operator",
72
+ });
73
+ await (doc as unknown as PasswordedUser).setPassword(RBAC_OPERATOR_PASSWORD);
74
+ await doc.save();
75
+ };
76
+
77
+ const authAsRbacOperator = async (app: express.Application): Promise<TestAgent> => {
78
+ return authAsUserWithCredentials(app, {
79
+ email: RBAC_OPERATOR_EMAIL,
80
+ password: RBAC_OPERATOR_PASSWORD,
81
+ });
82
+ };
83
+
84
+ const buildObservabilityApp = (
85
+ accessControl?: ReturnType<typeof createAccess>
86
+ ): express.Application => {
87
+ const obsApp = new ObservabilityApp({
88
+ accessControl,
89
+ plugins: [createLocalObservabilityPlugin()],
90
+ });
91
+
92
+ if (accessControl) {
93
+ const app = getBaseServer();
94
+ setupAuth(app, typedUserModel);
95
+ addAuthRoutes(app, typedUserModel);
96
+ obsApp.register(app);
97
+ app.use(apiUnauthorizedMiddleware);
98
+ app.use(apiErrorMiddleware);
99
+ return app;
100
+ }
101
+
102
+ return new TerrenoApp({logRequests: false, skipListen: true, userModel: typedUserModel})
103
+ .register(obsApp)
104
+ .build();
105
+ };
106
+
107
+ const buildRbacObservabilityApp = (
108
+ grantsByEmail: Record<string, Record<string, string[]>>
109
+ ): express.Application => {
110
+ const accessControl = createAccess({
111
+ connection: mongoose.connection,
112
+ resolvePermissions: async ({user}) => {
113
+ const email = "email" in user && typeof user.email === "string" ? user.email : "";
114
+ return grantsByEmail[email] ?? {};
115
+ },
116
+ statements: terrenoStatements,
117
+ });
118
+ return buildObservabilityApp(accessControl);
119
+ };
120
+
121
+ const grantsFor = (resource: string, actions: string[]): Record<string, string[]> => ({
122
+ ...ADMIN_SHELL,
123
+ [resource]: actions,
124
+ });
125
+
126
+ const sendRequest = async (
127
+ agent: TestAgent,
128
+ case_: RouteMappingCase
129
+ ): Promise<{status: number}> => {
130
+ const path = case_.path;
131
+ if (case_.method === "get") {
132
+ return agent.get(path);
133
+ }
134
+ if (case_.method === "delete") {
135
+ return agent.delete(path);
136
+ }
137
+ if (case_.method === "patch") {
138
+ return agent.patch(path).send(case_.body ?? {});
139
+ }
140
+ return agent.post(path).send(case_.body ?? {});
141
+ };
142
+
143
+ const clearObservabilityFixtures = async (): Promise<void> => {
144
+ await registerObsPrompt().deleteMany({});
145
+ await registerObsPromptVersion().deleteMany({});
146
+ await registerObsPromptLabel().deleteMany({});
147
+ await registerObsDataset().deleteMany({});
148
+ await registerObsDatasetItem().deleteMany({});
149
+ await registerObsEvaluator().deleteMany({});
150
+ await registerObsTrace().deleteMany({});
151
+ await registerObsReviewItem().deleteMany({});
152
+ };
153
+
154
+ const seedFixtures = async (): Promise<FixtureIds> => {
155
+ await clearObservabilityFixtures();
156
+ const promptStore = new LocalPromptStore();
157
+ await promptStore.create({
158
+ folder: "rbac",
159
+ name: FIXTURE_PROMPT,
160
+ system: "system",
161
+ template: "hello",
162
+ type: "text",
163
+ });
164
+
165
+ const datasetStore = new LocalDatasetStore();
166
+ const dataset = await datasetStore.create({name: "rbac-dataset"});
167
+ const datasetItem = await datasetStore.createItem(dataset.id, {input: {text: "fixture"}});
168
+ const evaluator = await new LocalEvaluatorStore().create({
169
+ dimensions: [{dataType: "boolean", key: "ok", required: true}],
170
+ name: "rbac-human",
171
+ target: "full trace",
172
+ type: "human",
173
+ });
174
+ const trace = await new LocalTraceStore().exportTrace({
175
+ id: "rbac-trace",
176
+ name: "rbac",
177
+ prompts: [],
178
+ sensitive: false,
179
+ spans: [
180
+ {
181
+ id: "span",
182
+ kind: "LLM",
183
+ name: "llm",
184
+ startedAt: DateTime.utc().toISO() ?? "",
185
+ status: "ok",
186
+ },
187
+ ],
188
+ startedAt: DateTime.utc().toISO() ?? "",
189
+ status: "ok",
190
+ });
191
+ const agentApp = buildObservabilityApp();
192
+ const adminAgent = await authAsUser(agentApp, "admin");
193
+ const enqueued = await adminAgent.post("/ai/observability/traces/review").send({
194
+ evaluatorId: evaluator.id,
195
+ reason: "manual",
196
+ traceIds: [trace.id],
197
+ });
198
+ const reviewItemId = enqueued.body.data[0].id as string;
199
+
200
+ return {
201
+ datasetId: dataset.id,
202
+ datasetItemId: datasetItem.id,
203
+ evaluatorId: evaluator.id,
204
+ reviewItemId,
205
+ traceId: trace.id,
206
+ };
207
+ };
208
+
209
+ describe("observability route RBAC", () => {
210
+ beforeAll(async () => {
211
+ await ensureTestUsers();
212
+ await ensureRbacOperatorUser();
213
+ });
214
+
215
+ afterEach(() => {
216
+ resetObservabilityApp();
217
+ });
218
+
219
+ beforeEach(async () => {
220
+ await clearObservabilityFixtures();
221
+ });
222
+
223
+ it("allows legacy admin callers when accessControl is omitted", async () => {
224
+ const app = buildObservabilityApp();
225
+ const agent = await authAsUser(app, "admin");
226
+ const response = await agent.post("/ai/observability/prompts").send({
227
+ folder: "examples",
228
+ name: "legacy-admin",
229
+ system: "Hi",
230
+ template: "Hello",
231
+ type: "text",
232
+ });
233
+ expect(response.status).toBe(201);
234
+ });
235
+
236
+ it("denies observability routes when RBAC grants omit admin:access", async () => {
237
+ const app = buildRbacObservabilityApp({
238
+ [RBAC_OPERATOR_EMAIL]: {
239
+ aiPrompt: ["list", "read"],
240
+ },
241
+ });
242
+ const agent = await authAsRbacOperator(app);
243
+ await agent.get("/ai/observability/prompts").expect(403);
244
+ });
245
+
246
+ it("allows composed read-only shell with admin:access and aiPrompt list/read only", async () => {
247
+ await seedFixtures();
248
+ const app = buildRbacObservabilityApp({
249
+ [RBAC_OPERATOR_EMAIL]: {
250
+ admin: ["access"],
251
+ aiPrompt: ["list", "read"],
252
+ },
253
+ });
254
+ const agent = await authAsRbacOperator(app);
255
+ await agent.get("/ai/observability/prompts").expect(200);
256
+ const createResponse = await agent.post("/ai/observability/prompts").send({
257
+ folder: "rbac",
258
+ name: "read-only-shell-deny",
259
+ system: "s",
260
+ template: "t",
261
+ type: "text",
262
+ });
263
+ expect(createResponse.status).toBe(403);
264
+ });
265
+
266
+ it("keeps legacy user.admin as full access when accessControl is configured", async () => {
267
+ const app = buildRbacObservabilityApp({
268
+ [ADMIN_EMAIL]: {
269
+ admin: ["access"],
270
+ },
271
+ });
272
+ const agent = await authAsUser(app, "admin");
273
+ await agent
274
+ .post("/ai/observability/prompts")
275
+ .send({
276
+ folder: "examples",
277
+ name: "legacy-bypass",
278
+ system: "Hi",
279
+ template: "Hello",
280
+ type: "text",
281
+ })
282
+ .expect(201);
283
+ });
284
+
285
+ const buildMappingCases = (fixtures: FixtureIds): RouteMappingCase[] => {
286
+ const missingId = new mongoose.Types.ObjectId().toString();
287
+ return [
288
+ {
289
+ allowAction: "list",
290
+ denyAction: "read",
291
+ label: "aiPrompt:list",
292
+ method: "get",
293
+ path: "/ai/observability/prompts",
294
+ resource: "aiPrompt",
295
+ },
296
+ {
297
+ allowAction: "read",
298
+ denyAction: "list",
299
+ label: "aiPrompt:read",
300
+ method: "get",
301
+ path: `/ai/observability/prompts/${FIXTURE_PROMPT}`,
302
+ resource: "aiPrompt",
303
+ },
304
+ {
305
+ allowAction: "create",
306
+ body: {folder: "rbac", name: "rbac-new", system: "s", template: "t", type: "text"},
307
+ denyAction: "list",
308
+ label: "aiPrompt:create",
309
+ method: "post",
310
+ path: "/ai/observability/prompts",
311
+ resource: "aiPrompt",
312
+ },
313
+ {
314
+ allowAction: "update",
315
+ body: {system: "v2", template: "v2", type: "text"},
316
+ denyAction: "create",
317
+ label: "aiPrompt:update",
318
+ method: "post",
319
+ path: `/ai/observability/prompts/${FIXTURE_PROMPT}/versions`,
320
+ resource: "aiPrompt",
321
+ },
322
+ {
323
+ allowAction: "promote",
324
+ body: {label: "production", version: 1},
325
+ denyAction: "update",
326
+ label: "aiPrompt:promote",
327
+ method: "post",
328
+ path: `/ai/observability/prompts/${FIXTURE_PROMPT}/labels`,
329
+ resource: "aiPrompt",
330
+ },
331
+ {
332
+ allowAction: "playground",
333
+ body: {variables: {}, version: 1},
334
+ denyAction: "promote",
335
+ label: "aiPrompt:playground",
336
+ method: "post",
337
+ path: `/ai/observability/prompts/${FIXTURE_PROMPT}/playground`,
338
+ resource: "aiPrompt",
339
+ },
340
+ {
341
+ allowAction: "list",
342
+ denyAction: "read",
343
+ label: "aiTrace:list (traces)",
344
+ method: "get",
345
+ path: "/ai/observability/traces",
346
+ resource: "aiTrace",
347
+ },
348
+ {
349
+ allowAction: "read",
350
+ denyAction: "list",
351
+ label: "aiTrace:read (detail)",
352
+ method: "get",
353
+ path: `/ai/observability/traces/${fixtures.traceId}`,
354
+ resource: "aiTrace",
355
+ },
356
+ {
357
+ allowAction: "read",
358
+ body: {},
359
+ denyAction: "list",
360
+ label: "aiTrace:read (test-multi-stage)",
361
+ method: "post",
362
+ path: "/ai/observability/traces/test-multi-stage",
363
+ resource: "aiTrace",
364
+ },
365
+ {
366
+ allowAction: "list",
367
+ denyAction: "read",
368
+ label: "aiReview:list",
369
+ method: "get",
370
+ path: "/ai/observability/review",
371
+ resource: "aiReview",
372
+ },
373
+ {
374
+ allowAction: "read",
375
+ denyAction: "list",
376
+ label: "aiReview:read",
377
+ method: "get",
378
+ path: `/ai/observability/review/${fixtures.reviewItemId}`,
379
+ resource: "aiReview",
380
+ },
381
+ {
382
+ allowAction: "score",
383
+ body: {name: "manual", value: 1},
384
+ denyAction: "assign",
385
+ label: "aiReview:score (trace scores)",
386
+ method: "post",
387
+ path: `/ai/observability/traces/${fixtures.traceId}/scores`,
388
+ resource: "aiReview",
389
+ },
390
+ {
391
+ allowAction: "score",
392
+ body: {action: "submit", scores: {ok: true}},
393
+ denyAction: "assign",
394
+ label: "aiReview:score (review submit)",
395
+ method: "post",
396
+ path: `/ai/observability/review/${fixtures.reviewItemId}`,
397
+ resource: "aiReview",
398
+ },
399
+ {
400
+ allowAction: "assign",
401
+ body: {
402
+ evaluatorId: fixtures.evaluatorId,
403
+ reason: "manual",
404
+ traceIds: [fixtures.traceId],
405
+ },
406
+ denyAction: "score",
407
+ label: "aiReview:assign (enqueue)",
408
+ method: "post",
409
+ path: "/ai/observability/traces/review",
410
+ resource: "aiReview",
411
+ },
412
+ {
413
+ allowAction: "assign",
414
+ body: {action: "assign", assigneeId: new mongoose.Types.ObjectId().toString()},
415
+ denyAction: "score",
416
+ label: "aiReview:assign (review item)",
417
+ method: "post",
418
+ path: `/ai/observability/review/${fixtures.reviewItemId}`,
419
+ resource: "aiReview",
420
+ },
421
+ {
422
+ allowAction: "list",
423
+ denyAction: "read",
424
+ label: "aiDataset:list",
425
+ method: "get",
426
+ path: "/ai/observability/datasets",
427
+ resource: "aiDataset",
428
+ },
429
+ {
430
+ allowAction: "read",
431
+ denyAction: "list",
432
+ label: "aiDataset:read",
433
+ method: "get",
434
+ path: `/ai/observability/datasets/${fixtures.datasetId}`,
435
+ resource: "aiDataset",
436
+ },
437
+ {
438
+ allowAction: "create",
439
+ body: {name: "rbac-dataset-2"},
440
+ denyAction: "list",
441
+ label: "aiDataset:create",
442
+ method: "post",
443
+ path: "/ai/observability/datasets",
444
+ resource: "aiDataset",
445
+ },
446
+ {
447
+ allowAction: "update",
448
+ body: {name: "rbac-dataset-renamed"},
449
+ denyAction: "create",
450
+ label: "aiDataset:update (patch)",
451
+ method: "patch",
452
+ path: `/ai/observability/datasets/${fixtures.datasetId}`,
453
+ resource: "aiDataset",
454
+ },
455
+ {
456
+ allowAction: "update",
457
+ body: {rows: [{input: {text: "x"}}]},
458
+ denyAction: "read",
459
+ label: "aiDataset:update (import)",
460
+ method: "post",
461
+ path: `/ai/observability/datasets/${fixtures.datasetId}/import`,
462
+ resource: "aiDataset",
463
+ },
464
+ {
465
+ allowAction: "update",
466
+ body: {datasetId: fixtures.datasetId, traceId: fixtures.traceId},
467
+ denyAction: "read",
468
+ label: "aiDataset:update (add-to-dataset)",
469
+ method: "post",
470
+ path: "/ai/observability/traces/add-to-dataset",
471
+ resource: "aiDataset",
472
+ },
473
+ {
474
+ allowAction: "read",
475
+ denyAction: "list",
476
+ label: "aiDataset:read (items list)",
477
+ method: "get",
478
+ path: `/ai/observability/datasets/${fixtures.datasetId}/items`,
479
+ resource: "aiDataset",
480
+ },
481
+ {
482
+ allowAction: "update",
483
+ body: {input: {text: "rbac-new-item"}},
484
+ denyAction: "read",
485
+ label: "aiDataset:update (create item)",
486
+ method: "post",
487
+ path: `/ai/observability/datasets/${fixtures.datasetId}/items`,
488
+ resource: "aiDataset",
489
+ },
490
+ {
491
+ allowAction: "update",
492
+ body: {input: {text: "rbac-patched-item"}},
493
+ denyAction: "create",
494
+ label: "aiDataset:update (patch item)",
495
+ method: "patch",
496
+ path: `/ai/observability/datasets/${fixtures.datasetId}/items/${fixtures.datasetItemId}`,
497
+ resource: "aiDataset",
498
+ },
499
+ {
500
+ allowAction: "delete",
501
+ denyAction: "update",
502
+ label: "aiDataset:delete (item)",
503
+ method: "delete",
504
+ path: `/ai/observability/datasets/${fixtures.datasetId}/items/${fixtures.datasetItemId}`,
505
+ resource: "aiDataset",
506
+ },
507
+ {
508
+ allowAction: "delete",
509
+ denyAction: "update",
510
+ label: "aiDataset:delete",
511
+ method: "delete",
512
+ path: `/ai/observability/datasets/${missingId}`,
513
+ resource: "aiDataset",
514
+ },
515
+ {
516
+ allowAction: "list",
517
+ denyAction: "read",
518
+ label: "aiExperiment:list",
519
+ method: "get",
520
+ path: "/ai/observability/experiments",
521
+ resource: "aiExperiment",
522
+ },
523
+ {
524
+ allowAction: "read",
525
+ denyAction: "list",
526
+ label: "aiExperiment:read",
527
+ method: "get",
528
+ path: `/ai/observability/experiments/${missingId}`,
529
+ resource: "aiExperiment",
530
+ },
531
+ {
532
+ allowAction: "create",
533
+ body: {
534
+ datasetId: fixtures.datasetId,
535
+ evaluatorIds: [],
536
+ promptName: FIXTURE_PROMPT,
537
+ versions: [1, 2],
538
+ },
539
+ denyAction: "list",
540
+ label: "aiExperiment:create (estimate)",
541
+ method: "post",
542
+ path: "/ai/observability/experiments/estimate",
543
+ resource: "aiExperiment",
544
+ },
545
+ {
546
+ allowAction: "create",
547
+ body: {
548
+ datasetId: fixtures.datasetId,
549
+ evaluatorIds: [],
550
+ promptName: FIXTURE_PROMPT,
551
+ versions: [1, 2],
552
+ },
553
+ denyAction: "list",
554
+ label: "aiExperiment:create",
555
+ method: "post",
556
+ path: "/ai/observability/experiments",
557
+ resource: "aiExperiment",
558
+ },
559
+ {
560
+ allowAction: "promote",
561
+ body: {version: 1},
562
+ denyAction: "read",
563
+ label: "aiExperiment:promote",
564
+ method: "post",
565
+ path: `/ai/observability/experiments/${missingId}/promote`,
566
+ resource: "aiExperiment",
567
+ },
568
+ {
569
+ allowAction: "list",
570
+ denyAction: "read",
571
+ label: "aiEvaluator:list (templates)",
572
+ method: "get",
573
+ path: "/ai/observability/evaluators/templates",
574
+ resource: "aiEvaluator",
575
+ },
576
+ {
577
+ allowAction: "list",
578
+ denyAction: "read",
579
+ label: "aiEvaluator:list",
580
+ method: "get",
581
+ path: "/ai/observability/evaluators",
582
+ resource: "aiEvaluator",
583
+ },
584
+ {
585
+ allowAction: "create",
586
+ denyAction: "list",
587
+ label: "aiEvaluator:create (template)",
588
+ method: "post",
589
+ path: "/ai/observability/evaluators/templates/correctness-human",
590
+ resource: "aiEvaluator",
591
+ },
592
+ {
593
+ allowAction: "create",
594
+ body: {
595
+ dimensions: [{dataType: "boolean", key: "pass", required: true}],
596
+ name: "rbac-eval",
597
+ target: "full trace",
598
+ type: "human",
599
+ },
600
+ denyAction: "list",
601
+ label: "aiEvaluator:create",
602
+ method: "post",
603
+ path: "/ai/observability/evaluators",
604
+ resource: "aiEvaluator",
605
+ },
606
+ {
607
+ allowAction: "read",
608
+ denyAction: "list",
609
+ label: "aiEvaluator:read",
610
+ method: "get",
611
+ path: `/ai/observability/evaluators/${fixtures.evaluatorId}`,
612
+ resource: "aiEvaluator",
613
+ },
614
+ {
615
+ allowAction: "update",
616
+ body: {description: "updated"},
617
+ denyAction: "read",
618
+ label: "aiEvaluator:update",
619
+ method: "patch",
620
+ path: `/ai/observability/evaluators/${fixtures.evaluatorId}`,
621
+ resource: "aiEvaluator",
622
+ },
623
+ {
624
+ allowAction: "delete",
625
+ denyAction: "update",
626
+ label: "aiEvaluator:delete",
627
+ method: "delete",
628
+ path: `/ai/observability/evaluators/${fixtures.evaluatorId}`,
629
+ resource: "aiEvaluator",
630
+ },
631
+ ];
632
+ };
633
+
634
+ describe("GET /status permissions", () => {
635
+ const NO_SHELL_EMAIL = "obs-no-shell@example.com";
636
+ const NO_SHELL_PASSWORD = "obs-no-shell-password";
637
+
638
+ const ensureNoShellUser = async (): Promise<void> => {
639
+ await UserModel.deleteMany({email: NO_SHELL_EMAIL});
640
+ const doc = await UserModel.create({
641
+ admin: false,
642
+ email: NO_SHELL_EMAIL,
643
+ name: "Obs No Shell",
644
+ });
645
+ await (doc as unknown as PasswordedUser).setPassword(NO_SHELL_PASSWORD);
646
+ await doc.save();
647
+ };
648
+
649
+ const authAsNoShell = async (app: express.Application): Promise<TestAgent> => {
650
+ return authAsUserWithCredentials(app, {
651
+ email: NO_SHELL_EMAIL,
652
+ password: NO_SHELL_PASSWORD,
653
+ });
654
+ };
655
+
656
+ it("returns 403 without admin:access even when observability list is granted", async () => {
657
+ await ensureNoShellUser();
658
+ const app = buildRbacObservabilityApp({
659
+ [NO_SHELL_EMAIL]: {
660
+ aiPrompt: ["list", "read"],
661
+ },
662
+ });
663
+ const agent = await authAsNoShell(app);
664
+ await agent.get("/ai/observability/status").expect(403);
665
+ });
666
+
667
+ it("returns all observability actions false for admin shell without resource grants", async () => {
668
+ const app = buildRbacObservabilityApp({
669
+ [RBAC_OPERATOR_EMAIL]: ADMIN_SHELL,
670
+ });
671
+ const agent = await authAsRbacOperator(app);
672
+ const response = await agent.get("/ai/observability/status").expect(200);
673
+ const observabilityResources = [
674
+ "aiDataset",
675
+ "aiEvaluator",
676
+ "aiExperiment",
677
+ "aiPrompt",
678
+ "aiReview",
679
+ "aiTrace",
680
+ ] as const;
681
+ for (const resource of observabilityResources) {
682
+ for (const action of terrenoStatements[resource]) {
683
+ expect(response.body.data.permissions[resource]?.[action]).toBe(false);
684
+ }
685
+ }
686
+ });
687
+
688
+ it("is available with admin shell access and returns effective action flags", async () => {
689
+ const app = buildRbacObservabilityApp({
690
+ [RBAC_OPERATOR_EMAIL]: {
691
+ ...ADMIN_SHELL,
692
+ aiPrompt: ["list", "read"],
693
+ },
694
+ });
695
+ const agent = await authAsRbacOperator(app);
696
+ const response = await agent.get("/ai/observability/status").expect(200);
697
+ expect(response.body.data.permissions.aiPrompt).toEqual({
698
+ create: false,
699
+ list: true,
700
+ playground: false,
701
+ promote: false,
702
+ read: true,
703
+ update: false,
704
+ });
705
+ expect(response.body.data.permissions.aiTrace?.list).toBe(false);
706
+ });
707
+
708
+ it("does not grant writes: status create false and POST prompts returns 403", async () => {
709
+ const app = buildRbacObservabilityApp({
710
+ [RBAC_OPERATOR_EMAIL]: {
711
+ ...ADMIN_SHELL,
712
+ aiPrompt: ["list", "read"],
713
+ },
714
+ });
715
+ const agent = await authAsRbacOperator(app);
716
+ const status = await agent.get("/ai/observability/status").expect(200);
717
+ expect(status.body.data.permissions.aiPrompt?.create).toBe(false);
718
+
719
+ await agent
720
+ .post("/ai/observability/prompts")
721
+ .send({
722
+ folder: "examples",
723
+ name: `rbac-deny-${Date.now()}`,
724
+ template: "Hello",
725
+ type: "chat",
726
+ })
727
+ .expect(403);
728
+ });
729
+
730
+ it("grants every action for legacy admin callers", async () => {
731
+ const app = buildObservabilityApp();
732
+ const agent = await authAsUser(app, "admin");
733
+ const response = await agent.get("/ai/observability/status").expect(200);
734
+ expect(response.body.data.permissions.aiPrompt?.playground).toBe(true);
735
+ expect(response.body.data.permissions.aiExperiment?.promote).toBe(true);
736
+ });
737
+ });
738
+
739
+ describe("route action mapping", () => {
740
+ it("returns 403 when only the neighboring action is granted", async () => {
741
+ const fixtures = await seedFixtures();
742
+ for (const case_ of buildMappingCases(fixtures)) {
743
+ const app = buildRbacObservabilityApp({
744
+ [RBAC_OPERATOR_EMAIL]: grantsFor(case_.resource, [case_.denyAction]),
745
+ });
746
+ const agent = await authAsRbacOperator(app);
747
+ const response = await sendRequest(agent, case_);
748
+ expect(response.status).toBe(403);
749
+ }
750
+ }, 30_000);
751
+
752
+ it("passes middleware when the mapped action is granted", async () => {
753
+ const fixtures = await seedFixtures();
754
+ const cases = buildMappingCases(fixtures);
755
+ for (let index = 0; index < cases.length; index += 1) {
756
+ const freshFixtures = await seedFixtures();
757
+ const case_ = buildMappingCases(freshFixtures)[index];
758
+ const app = buildRbacObservabilityApp({
759
+ [RBAC_OPERATOR_EMAIL]: grantsFor(case_.resource, [case_.allowAction]),
760
+ });
761
+ const agent = await authAsRbacOperator(app);
762
+ const response = await sendRequest(agent, case_);
763
+ expect(response.status).not.toBe(403);
764
+ }
765
+ }, 30_000);
766
+ });
767
+ });