@terreno/ai 57.9.0 → 57.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (412) hide show
  1. package/README.md +1 -0
  2. package/bunfig.toml +1 -1
  3. package/dist/aiApp.d.ts +8 -2
  4. package/dist/aiApp.js +12 -6
  5. package/dist/harness/adminScreens.d.ts +6 -0
  6. package/dist/harness/adminScreens.js +11 -0
  7. package/dist/harness/agentLoop.d.ts +61 -0
  8. package/dist/harness/agentLoop.js +798 -0
  9. package/dist/harness/agentTaskNames.d.ts +4 -0
  10. package/dist/harness/agentTaskNames.js +7 -0
  11. package/dist/harness/approvalPolicy.d.ts +3 -0
  12. package/dist/harness/approvalPolicy.js +23 -0
  13. package/dist/harness/approvals.d.ts +101 -0
  14. package/dist/harness/approvals.js +468 -0
  15. package/dist/harness/commit.d.ts +238 -0
  16. package/dist/harness/commit.js +678 -0
  17. package/dist/harness/conversation.d.ts +75 -0
  18. package/dist/harness/conversation.js +397 -0
  19. package/dist/harness/defineAgent.d.ts +9 -0
  20. package/dist/harness/defineAgent.js +65 -0
  21. package/dist/harness/defineTask.d.ts +11 -0
  22. package/dist/harness/defineTask.js +59 -0
  23. package/dist/harness/defineTool.d.ts +7 -0
  24. package/dist/harness/defineTool.js +36 -0
  25. package/dist/harness/definitionError.d.ts +8 -0
  26. package/dist/harness/definitionError.js +15 -0
  27. package/dist/harness/errors.d.ts +127 -0
  28. package/dist/harness/errors.js +101 -0
  29. package/dist/harness/eventHub.d.ts +45 -0
  30. package/dist/harness/eventHub.js +135 -0
  31. package/dist/harness/events.d.ts +95 -0
  32. package/dist/harness/events.js +267 -0
  33. package/dist/harness/executionEnv.d.ts +31 -0
  34. package/dist/harness/executionEnv.js +2 -0
  35. package/dist/harness/extensions.d.ts +82 -0
  36. package/dist/harness/extensions.js +274 -0
  37. package/dist/harness/harness.d.ts +163 -0
  38. package/dist/harness/harness.js +520 -0
  39. package/dist/harness/harnessApp.d.ts +44 -0
  40. package/dist/harness/harnessApp.js +62 -0
  41. package/dist/harness/internalRuntime.d.ts +16 -0
  42. package/dist/harness/internalRuntime.js +18 -0
  43. package/dist/harness/leases.d.ts +35 -0
  44. package/dist/harness/leases.js +86 -0
  45. package/dist/harness/memo.d.ts +24 -0
  46. package/dist/harness/memo.js +78 -0
  47. package/dist/harness/modelCall.d.ts +45 -0
  48. package/dist/harness/modelCall.js +147 -0
  49. package/dist/harness/models/harnessApproval.d.ts +2 -0
  50. package/dist/harness/models/harnessApproval.js +100 -0
  51. package/dist/harness/models/harnessConversation.d.ts +2 -0
  52. package/dist/harness/models/harnessConversation.js +117 -0
  53. package/dist/harness/models/harnessEvent.d.ts +3 -0
  54. package/dist/harness/models/harnessEvent.js +73 -0
  55. package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
  56. package/dist/harness/models/harnessInboxEvent.js +58 -0
  57. package/dist/harness/models/harnessMemo.d.ts +2 -0
  58. package/dist/harness/models/harnessMemo.js +42 -0
  59. package/dist/harness/models/harnessMessage.d.ts +2 -0
  60. package/dist/harness/models/harnessMessage.js +73 -0
  61. package/dist/harness/models/harnessOwner.d.ts +2 -0
  62. package/dist/harness/models/harnessOwner.js +36 -0
  63. package/dist/harness/models/harnessTask.d.ts +2 -0
  64. package/dist/harness/models/harnessTask.js +186 -0
  65. package/dist/harness/ownership.d.ts +89 -0
  66. package/dist/harness/ownership.js +438 -0
  67. package/dist/harness/registry.d.ts +29 -0
  68. package/dist/harness/registry.js +93 -0
  69. package/dist/harness/retryBackoff.d.ts +31 -0
  70. package/dist/harness/retryBackoff.js +50 -0
  71. package/dist/harness/routes/approvals.d.ts +13 -0
  72. package/dist/harness/routes/approvals.js +93 -0
  73. package/dist/harness/routes/conversations.d.ts +11 -0
  74. package/dist/harness/routes/conversations.js +66 -0
  75. package/dist/harness/routes/events.d.ts +19 -0
  76. package/dist/harness/routes/events.js +215 -0
  77. package/dist/harness/routes/tasks.d.ts +11 -0
  78. package/dist/harness/routes/tasks.js +108 -0
  79. package/dist/harness/runners/inProcessRunner.d.ts +57 -0
  80. package/dist/harness/runners/inProcessRunner.js +196 -0
  81. package/dist/harness/runtime.d.ts +34 -0
  82. package/dist/harness/runtime.js +551 -0
  83. package/dist/harness/subagent.d.ts +53 -0
  84. package/dist/harness/subagent.js +134 -0
  85. package/dist/harness/suspend.d.ts +8 -0
  86. package/dist/harness/suspend.js +16 -0
  87. package/dist/harness/transaction.d.ts +3 -0
  88. package/dist/harness/transaction.js +22 -0
  89. package/dist/harness/waits.d.ts +100 -0
  90. package/dist/harness/waits.js +214 -0
  91. package/dist/index.d.ts +19 -1
  92. package/dist/index.js +45 -2
  93. package/dist/models/aiDataset.d.ts +2 -0
  94. package/dist/models/aiDataset.js +67 -0
  95. package/dist/models/aiRequest.js +1 -1
  96. package/dist/models/gptHistory.js +73 -0
  97. package/dist/observability/adminScreens.d.ts +5 -0
  98. package/dist/observability/adminScreens.js +32 -0
  99. package/dist/observability/compileTemplate.d.ts +1 -0
  100. package/dist/observability/compileTemplate.js +13 -0
  101. package/dist/observability/datasetImport.d.ts +11 -0
  102. package/dist/observability/datasetImport.js +185 -0
  103. package/dist/observability/evaluate.d.ts +31 -0
  104. package/dist/observability/evaluate.js +200 -0
  105. package/dist/observability/evaluatorTemplates.d.ts +17 -0
  106. package/dist/observability/evaluatorTemplates.js +111 -0
  107. package/dist/observability/jsonPath.d.ts +2 -0
  108. package/dist/observability/jsonPath.js +57 -0
  109. package/dist/observability/local/datasetStore.d.ts +87 -0
  110. package/dist/observability/local/datasetStore.js +326 -0
  111. package/dist/observability/local/evaluatorStore.d.ts +44 -0
  112. package/dist/observability/local/evaluatorStore.js +182 -0
  113. package/dist/observability/local/experimentRunner.d.ts +96 -0
  114. package/dist/observability/local/experimentRunner.js +419 -0
  115. package/dist/observability/local/localPlugin.d.ts +22 -0
  116. package/dist/observability/local/localPlugin.js +76 -0
  117. package/dist/observability/local/models/obsDataset.d.ts +2 -0
  118. package/dist/observability/local/models/obsDataset.js +37 -0
  119. package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
  120. package/dist/observability/local/models/obsDatasetItem.js +76 -0
  121. package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
  122. package/dist/observability/local/models/obsEvaluator.js +98 -0
  123. package/dist/observability/local/models/obsExperiment.d.ts +2 -0
  124. package/dist/observability/local/models/obsExperiment.js +87 -0
  125. package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
  126. package/dist/observability/local/models/obsExperimentItem.js +45 -0
  127. package/dist/observability/local/models/obsPrompt.d.ts +2 -0
  128. package/dist/observability/local/models/obsPrompt.js +42 -0
  129. package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
  130. package/dist/observability/local/models/obsPromptLabel.js +39 -0
  131. package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
  132. package/dist/observability/local/models/obsPromptVersion.js +96 -0
  133. package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
  134. package/dist/observability/local/models/obsReviewItem.js +75 -0
  135. package/dist/observability/local/models/obsScore.d.ts +2 -0
  136. package/dist/observability/local/models/obsScore.js +58 -0
  137. package/dist/observability/local/models/obsSpan.d.ts +2 -0
  138. package/dist/observability/local/models/obsSpan.js +64 -0
  139. package/dist/observability/local/models/obsTrace.d.ts +2 -0
  140. package/dist/observability/local/models/obsTrace.js +93 -0
  141. package/dist/observability/local/promptStore.d.ts +180 -0
  142. package/dist/observability/local/promptStore.js +444 -0
  143. package/dist/observability/local/reviewStore.d.ts +50 -0
  144. package/dist/observability/local/reviewStore.js +203 -0
  145. package/dist/observability/local/traceStore.d.ts +103 -0
  146. package/dist/observability/local/traceStore.js +325 -0
  147. package/dist/observability/observabilityApp.d.ts +24 -0
  148. package/dist/observability/observabilityApp.js +157 -0
  149. package/dist/observability/observabilityAppRegistry.d.ts +14 -0
  150. package/dist/observability/observabilityAppRegistry.js +16 -0
  151. package/dist/observability/permissions.d.ts +7 -0
  152. package/dist/observability/permissions.js +60 -0
  153. package/dist/observability/reviewPanels.d.ts +18 -0
  154. package/dist/observability/reviewPanels.js +46 -0
  155. package/dist/observability/routes/datasets.d.ts +7 -0
  156. package/dist/observability/routes/datasets.js +149 -0
  157. package/dist/observability/routes/evaluators.d.ts +7 -0
  158. package/dist/observability/routes/evaluators.js +123 -0
  159. package/dist/observability/routes/experiments.d.ts +7 -0
  160. package/dist/observability/routes/experiments.js +95 -0
  161. package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
  162. package/dist/observability/routes/observabilityRouteAccess.js +67 -0
  163. package/dist/observability/routes/prompts.d.ts +12 -0
  164. package/dist/observability/routes/prompts.js +172 -0
  165. package/dist/observability/routes/requestAiService.d.ts +13 -0
  166. package/dist/observability/routes/requestAiService.js +21 -0
  167. package/dist/observability/routes/review.d.ts +7 -0
  168. package/dist/observability/routes/review.js +79 -0
  169. package/dist/observability/routes/status.d.ts +5 -0
  170. package/dist/observability/routes/status.js +31 -0
  171. package/dist/observability/routes/testMultiStage.d.ts +9 -0
  172. package/dist/observability/routes/testMultiStage.js +67 -0
  173. package/dist/observability/routes/traces.d.ts +7 -0
  174. package/dist/observability/routes/traces.js +129 -0
  175. package/dist/observability/schemaValidation.d.ts +5 -0
  176. package/dist/observability/schemaValidation.js +26 -0
  177. package/dist/observability/sopThresholds.d.ts +2 -0
  178. package/dist/observability/sopThresholds.js +21 -0
  179. package/dist/observability/status.d.ts +23 -0
  180. package/dist/observability/status.js +34 -0
  181. package/dist/observability/testMultiStageSchemas.d.ts +10 -0
  182. package/dist/observability/testMultiStageSchemas.js +79 -0
  183. package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
  184. package/dist/observability/testMultiStageWorkflow.js +218 -0
  185. package/dist/observability/types.d.ts +159 -0
  186. package/dist/observability/types.js +67 -0
  187. package/dist/routes/files.js +18 -7
  188. package/dist/routes/gpt.js +79 -689
  189. package/dist/routes/gptActions.d.ts +9 -0
  190. package/dist/routes/gptActions.js +212 -0
  191. package/dist/routes/gptDatasets.d.ts +5 -0
  192. package/dist/routes/gptDatasets.js +94 -0
  193. package/dist/routes/gptHistories.d.ts +3 -1
  194. package/dist/routes/gptHistories.js +188 -2
  195. package/dist/service/aiDatasets.d.ts +57 -0
  196. package/dist/service/aiDatasets.js +253 -0
  197. package/dist/service/aiService.d.ts +50 -1
  198. package/dist/service/aiService.js +426 -77
  199. package/dist/service/askFiles.d.ts +49 -0
  200. package/dist/service/askFiles.js +162 -0
  201. package/dist/service/asks.d.ts +76 -0
  202. package/dist/service/asks.js +284 -0
  203. package/dist/service/chatTurn.d.ts +128 -0
  204. package/dist/service/chatTurn.js +1674 -0
  205. package/dist/service/fileStorage.d.ts +4 -0
  206. package/dist/service/fileStorage.js +7 -2
  207. package/dist/service/fillActionIds.d.ts +6 -0
  208. package/dist/service/fillActionIds.js +79 -0
  209. package/dist/service/jsonFenceModel.d.ts +7 -0
  210. package/dist/service/jsonFenceModel.js +45 -0
  211. package/dist/service/prompts.d.ts +52 -0
  212. package/dist/service/prompts.js +96 -1
  213. package/dist/service/sanitizeHtml.d.ts +11 -0
  214. package/dist/service/sanitizeHtml.js +108 -0
  215. package/dist/types/harness.d.ts +934 -0
  216. package/dist/types/harness.js +133 -0
  217. package/dist/types/index.d.ts +192 -14
  218. package/dist/types/index.js +2 -0
  219. package/dist/types/observability.d.ts +288 -0
  220. package/dist/types/observability.js +2 -0
  221. package/package.json +17 -5
  222. package/src/aiApp.test.ts +207 -1
  223. package/src/aiApp.ts +19 -6
  224. package/src/harness/adminScreens.test.ts +34 -0
  225. package/src/harness/adminScreens.ts +11 -0
  226. package/src/harness/agentLoop.test.ts +1034 -0
  227. package/src/harness/agentLoop.ts +1082 -0
  228. package/src/harness/agentRecovery.test.ts +380 -0
  229. package/src/harness/agentTaskNames.ts +4 -0
  230. package/src/harness/approvalPolicy.ts +28 -0
  231. package/src/harness/approvals.test.ts +1094 -0
  232. package/src/harness/approvals.ts +644 -0
  233. package/src/harness/commit.ts +1049 -0
  234. package/src/harness/conversation.ts +503 -0
  235. package/src/harness/defineAgent.test.ts +92 -0
  236. package/src/harness/defineAgent.ts +82 -0
  237. package/src/harness/defineTask.test.ts +59 -0
  238. package/src/harness/defineTask.ts +76 -0
  239. package/src/harness/defineTool.ts +45 -0
  240. package/src/harness/definitionError.ts +13 -0
  241. package/src/harness/errors.test.ts +90 -0
  242. package/src/harness/errors.ts +119 -0
  243. package/src/harness/eventHub.test.ts +124 -0
  244. package/src/harness/eventHub.ts +159 -0
  245. package/src/harness/events.test.ts +757 -0
  246. package/src/harness/events.ts +384 -0
  247. package/src/harness/executionEnv.ts +34 -0
  248. package/src/harness/extensions.test.ts +1218 -0
  249. package/src/harness/extensions.ts +382 -0
  250. package/src/harness/harness.test.ts +693 -0
  251. package/src/harness/harness.ts +767 -0
  252. package/src/harness/harnessApp.ts +76 -0
  253. package/src/harness/internalRuntime.ts +44 -0
  254. package/src/harness/leases.ts +132 -0
  255. package/src/harness/memo.ts +107 -0
  256. package/src/harness/modelCall.test.ts +134 -0
  257. package/src/harness/modelCall.ts +200 -0
  258. package/src/harness/models/harnessApproval.ts +109 -0
  259. package/src/harness/models/harnessConversation.ts +142 -0
  260. package/src/harness/models/harnessEvent.ts +92 -0
  261. package/src/harness/models/harnessInboxEvent.ts +73 -0
  262. package/src/harness/models/harnessMemo.ts +43 -0
  263. package/src/harness/models/harnessMessage.ts +85 -0
  264. package/src/harness/models/harnessModels.test.ts +60 -0
  265. package/src/harness/models/harnessOwner.ts +40 -0
  266. package/src/harness/models/harnessTask.ts +201 -0
  267. package/src/harness/ownership.test.ts +1087 -0
  268. package/src/harness/ownership.ts +618 -0
  269. package/src/harness/recovery.test.ts +912 -0
  270. package/src/harness/registry.ts +124 -0
  271. package/src/harness/retries.test.ts +292 -0
  272. package/src/harness/retryBackoff.test.ts +100 -0
  273. package/src/harness/retryBackoff.ts +77 -0
  274. package/src/harness/routes/approvals.ts +123 -0
  275. package/src/harness/routes/conversations.test.ts +518 -0
  276. package/src/harness/routes/conversations.ts +102 -0
  277. package/src/harness/routes/events.ts +280 -0
  278. package/src/harness/routes/tasks.test.ts +198 -0
  279. package/src/harness/routes/tasks.ts +146 -0
  280. package/src/harness/runners/inProcessRunner.test.ts +402 -0
  281. package/src/harness/runners/inProcessRunner.ts +220 -0
  282. package/src/harness/runtime.ts +714 -0
  283. package/src/harness/subagent.test.ts +696 -0
  284. package/src/harness/subagent.ts +192 -0
  285. package/src/harness/suspend.ts +13 -0
  286. package/src/harness/transaction.ts +17 -0
  287. package/src/harness/versioning.test.ts +288 -0
  288. package/src/harness/waits.test.ts +693 -0
  289. package/src/harness/waits.ts +356 -0
  290. package/src/index.test.ts +13 -0
  291. package/src/index.ts +70 -0
  292. package/src/isolated/adminScreensConfig.isolated.ts +143 -0
  293. package/src/models/aiDataset.test.ts +34 -0
  294. package/src/models/aiDataset.ts +78 -0
  295. package/src/models/aiRequest.ts +2 -1
  296. package/src/models/gptHistory.test.ts +156 -0
  297. package/src/models/gptHistory.ts +89 -1
  298. package/src/observability/adminScreens.test.ts +71 -0
  299. package/src/observability/adminScreens.ts +42 -0
  300. package/src/observability/compileTemplate.test.ts +14 -0
  301. package/src/observability/compileTemplate.ts +10 -0
  302. package/src/observability/datasetImport.test.ts +95 -0
  303. package/src/observability/datasetImport.ts +220 -0
  304. package/src/observability/evaluate.test.ts +549 -0
  305. package/src/observability/evaluate.ts +254 -0
  306. package/src/observability/evaluatorTemplates.ts +124 -0
  307. package/src/observability/jsonPath.test.ts +52 -0
  308. package/src/observability/jsonPath.ts +60 -0
  309. package/src/observability/local/datasetStore.test.ts +230 -0
  310. package/src/observability/local/datasetStore.ts +452 -0
  311. package/src/observability/local/evaluatorStore.test.ts +138 -0
  312. package/src/observability/local/evaluatorStore.ts +224 -0
  313. package/src/observability/local/experimentRunner.test.ts +489 -0
  314. package/src/observability/local/experimentRunner.ts +548 -0
  315. package/src/observability/local/localPlugin.ts +88 -0
  316. package/src/observability/local/models/obsDataset.ts +38 -0
  317. package/src/observability/local/models/obsDatasetItem.ts +80 -0
  318. package/src/observability/local/models/obsEvaluator.ts +102 -0
  319. package/src/observability/local/models/obsExperiment.ts +91 -0
  320. package/src/observability/local/models/obsExperimentItem.ts +52 -0
  321. package/src/observability/local/models/obsPrompt.ts +43 -0
  322. package/src/observability/local/models/obsPromptLabel.ts +43 -0
  323. package/src/observability/local/models/obsPromptVersion.ts +107 -0
  324. package/src/observability/local/models/obsReviewItem.ts +79 -0
  325. package/src/observability/local/models/obsScore.ts +59 -0
  326. package/src/observability/local/models/obsSpan.ts +65 -0
  327. package/src/observability/local/models/obsTrace.ts +94 -0
  328. package/src/observability/local/obsModels.test.ts +119 -0
  329. package/src/observability/local/promptStore.test.ts +351 -0
  330. package/src/observability/local/promptStore.ts +665 -0
  331. package/src/observability/local/reviewStore.test.ts +164 -0
  332. package/src/observability/local/reviewStore.ts +264 -0
  333. package/src/observability/local/traceStore.test.ts +264 -0
  334. package/src/observability/local/traceStore.ts +440 -0
  335. package/src/observability/observabilityApp.test.ts +92 -0
  336. package/src/observability/observabilityApp.ts +185 -0
  337. package/src/observability/observabilityAppRegistry.test.ts +30 -0
  338. package/src/observability/observabilityAppRegistry.ts +35 -0
  339. package/src/observability/permissions.test.ts +96 -0
  340. package/src/observability/permissions.ts +78 -0
  341. package/src/observability/reviewPanels.test.ts +35 -0
  342. package/src/observability/reviewPanels.ts +64 -0
  343. package/src/observability/routes/datasets.test.ts +176 -0
  344. package/src/observability/routes/datasets.ts +252 -0
  345. package/src/observability/routes/evaluators.test.ts +119 -0
  346. package/src/observability/routes/evaluators.ts +207 -0
  347. package/src/observability/routes/experiments.test.ts +282 -0
  348. package/src/observability/routes/experiments.ts +152 -0
  349. package/src/observability/routes/observabilityRbac.test.ts +767 -0
  350. package/src/observability/routes/observabilityRouteAccess.ts +106 -0
  351. package/src/observability/routes/prompts.test.ts +320 -0
  352. package/src/observability/routes/prompts.ts +263 -0
  353. package/src/observability/routes/requestAiService.ts +36 -0
  354. package/src/observability/routes/review.test.ts +120 -0
  355. package/src/observability/routes/review.ts +125 -0
  356. package/src/observability/routes/status.test.ts +116 -0
  357. package/src/observability/routes/status.ts +47 -0
  358. package/src/observability/routes/testMultiStage.test.ts +214 -0
  359. package/src/observability/routes/testMultiStage.ts +96 -0
  360. package/src/observability/routes/traces.test.ts +167 -0
  361. package/src/observability/routes/traces.ts +172 -0
  362. package/src/observability/schemaValidation.ts +29 -0
  363. package/src/observability/sopThresholds.ts +20 -0
  364. package/src/observability/status.test.ts +38 -0
  365. package/src/observability/status.ts +56 -0
  366. package/src/observability/testMultiStageSchemas.ts +95 -0
  367. package/src/observability/testMultiStageWorkflow.test.ts +114 -0
  368. package/src/observability/testMultiStageWorkflow.ts +307 -0
  369. package/src/observability/types.ts +262 -0
  370. package/src/routes/files.test.ts +63 -9
  371. package/src/routes/files.ts +21 -7
  372. package/src/routes/gpt.test.ts +3279 -0
  373. package/src/routes/gpt.ts +91 -816
  374. package/src/routes/gptActions.test.ts +241 -0
  375. package/src/routes/gptActions.ts +255 -0
  376. package/src/routes/gptApprovals.test.ts +762 -0
  377. package/src/routes/gptDatasets.test.ts +86 -0
  378. package/src/routes/gptDatasets.ts +104 -0
  379. package/src/routes/gptFiles.test.ts +589 -0
  380. package/src/routes/gptHistories.test.ts +933 -0
  381. package/src/routes/gptHistories.ts +266 -5
  382. package/src/routes/routes.test.ts +222 -2
  383. package/src/service/aiDatasets.test.ts +123 -0
  384. package/src/service/aiDatasets.ts +316 -0
  385. package/src/service/aiService.test.ts +593 -0
  386. package/src/service/aiService.ts +568 -111
  387. package/src/service/askFiles.test.ts +96 -0
  388. package/src/service/askFiles.ts +222 -0
  389. package/src/service/asks.test.ts +946 -0
  390. package/src/service/asks.ts +423 -0
  391. package/src/service/chatTurn.ts +2287 -0
  392. package/src/service/fileStorage.test.ts +11 -0
  393. package/src/service/fileStorage.ts +10 -2
  394. package/src/service/fillActionIds.test.ts +65 -0
  395. package/src/service/fillActionIds.ts +98 -0
  396. package/src/service/jsonFenceModel.test.ts +47 -0
  397. package/src/service/jsonFenceModel.ts +53 -0
  398. package/src/service/prompts.test.ts +21 -0
  399. package/src/service/prompts.ts +147 -0
  400. package/src/service/sanitizeHtml.test.ts +61 -0
  401. package/src/service/sanitizeHtml.ts +109 -0
  402. package/src/tests/bunSetup.test.ts +7 -4
  403. package/src/tests/bunSetup.ts +7 -7
  404. package/src/tests/chatHarness.ts +478 -0
  405. package/src/tests/generateStream.ts +49 -0
  406. package/src/tests/harnessErrors.ts +22 -0
  407. package/src/tests/harnessStreaming.ts +227 -0
  408. package/src/tests/helpers.ts +9 -2
  409. package/src/types/harness.ts +1095 -0
  410. package/src/types/index.ts +209 -13
  411. package/src/types/observability.ts +344 -0
  412. package/src/types/sanitize-html.d.ts +25 -0
@@ -0,0 +1,762 @@
1
+ import {afterEach, beforeAll, describe, expect, it, mock} from "bun:test";
2
+ import {jsonSchema, tool} from "ai";
3
+
4
+ import {AIRequest} from "../models/aiRequest";
5
+ import {GptHistory} from "../models/gptHistory";
6
+ import {
7
+ type Agent,
8
+ buildApp,
9
+ conversationOf,
10
+ createScriptedModel,
11
+ failNextAskClaim,
12
+ loadHistory,
13
+ type ModelPromptMessage,
14
+ modelCall,
15
+ ONE_ASK_AT_A_TIME,
16
+ onlyHistoryId,
17
+ PLAN_ASK_CALL,
18
+ PLAN_ASK_ROW,
19
+ pauseOnPlanAsk,
20
+ pendingAskOf,
21
+ rowsOf,
22
+ type ScriptedModel,
23
+ streamPrompt,
24
+ TEAM_ANSWER,
25
+ textStep,
26
+ toolCallStep,
27
+ } from "../tests/chatHarness";
28
+ import {authAsUser, ensureTestUsers} from "../tests/helpers";
29
+ import type {GptRouteOptions} from "../types";
30
+
31
+ const DELETE_DESCRIPTION = "Delete the user's completed todos.";
32
+ const DELETE_INPUT = {scope: "completed"};
33
+ const DELETE_CALL = {
34
+ input: DELETE_INPUT,
35
+ toolCallId: "call_delete",
36
+ toolName: "deleteCompletedTodos",
37
+ };
38
+ const ARCHIVE_CALL = {input: {}, toolCallId: "call_archive", toolName: "archiveTodos"};
39
+ const DELETED = {deleted: 2};
40
+ const USER_PROMPT = "Delete my completed todos";
41
+ const DELETED_REPLY = "Deleted 2 completed todos.";
42
+ const TITLE = "Workspace setup";
43
+
44
+ const DEFAULT_APPROVAL_INPUT = {
45
+ confirmLabel: "Allow",
46
+ denyLabel: "Deny",
47
+ prompt: `Allow deleteCompletedTodos? ${DELETE_DESCRIPTION}`,
48
+ };
49
+
50
+ const DESTRUCTIVE_APPROVAL_INPUT = {
51
+ confirmLabel: "Delete",
52
+ denyLabel: "Keep them",
53
+ destructive: true,
54
+ prompt: "Delete all completed todos?",
55
+ title: "Clean up todos",
56
+ };
57
+
58
+ const APPROVE = {action: "accept", content: {confirmed: true}};
59
+ const DENY = {action: "accept", content: {confirmed: false}};
60
+
61
+ interface HostTools {
62
+ archive: ReturnType<typeof mock>;
63
+ execute: ReturnType<typeof mock>;
64
+ tools: GptRouteOptions["tools"];
65
+ }
66
+
67
+ const hostTools = (): HostTools => {
68
+ const execute = mock(async () => DELETED);
69
+ const archive = mock(async () => ({archived: 1}));
70
+ return {
71
+ archive,
72
+ execute,
73
+ tools: {
74
+ archiveTodos: tool({
75
+ description: "Archive the user's old todos.",
76
+ execute: archive,
77
+ inputSchema: jsonSchema<Record<string, never>>({properties: {}, type: "object"}),
78
+ needsApproval: true,
79
+ }),
80
+ deleteCompletedTodos: tool({
81
+ description: DELETE_DESCRIPTION,
82
+ execute,
83
+ inputSchema: jsonSchema<{scope: string}>({
84
+ properties: {scope: {type: "string"}},
85
+ type: "object",
86
+ }),
87
+ needsApproval: true,
88
+ }),
89
+ },
90
+ };
91
+ };
92
+
93
+ interface PausedApproval {
94
+ agent: Agent;
95
+ approvalId: string;
96
+ events: Record<string, unknown>[];
97
+ historyId: string;
98
+ host: HostTools;
99
+ model: ScriptedModel;
100
+ }
101
+
102
+ const pauseOnApproval = async ({
103
+ prompt = USER_PROMPT,
104
+ steps = [toolCallStep(DELETE_CALL), textStep(DELETED_REPLY)],
105
+ surface,
106
+ ...routeOptions
107
+ }: {
108
+ prompt?: string;
109
+ steps?: ReturnType<typeof textStep>[];
110
+ surface?: string;
111
+ } & Partial<GptRouteOptions>): Promise<PausedApproval> => {
112
+ const host = hostTools();
113
+ const model = createScriptedModel({steps});
114
+ const agent = await authAsUser(
115
+ buildApp({asks: true, model, tools: host.tools, ...routeOptions}),
116
+ "notAdmin"
117
+ );
118
+ const {events} = await streamPrompt(agent, {prompt, ...(surface ? {surface} : {})});
119
+ const askEvent = events.find((event) => "ask" in event) as
120
+ | {ask: {toolCallId: string}}
121
+ | undefined;
122
+ if (!askEvent) {
123
+ expect.unreachable(`The turn did not pause: ${JSON.stringify(events)}`);
124
+ }
125
+ return {
126
+ agent,
127
+ approvalId: askEvent.ask.toolCallId,
128
+ events,
129
+ historyId: await onlyHistoryId(),
130
+ host,
131
+ model,
132
+ };
133
+ };
134
+
135
+ const answer = (
136
+ {agent, approvalId, historyId}: Pick<PausedApproval, "agent" | "approvalId" | "historyId">,
137
+ response: Record<string, unknown>
138
+ ) => streamPrompt(agent, {askResponse: {...response, toolCallId: approvalId}, historyId});
139
+
140
+ const toolMessagesOf = (conversation: ModelPromptMessage[]): unknown[] =>
141
+ conversation.filter((message) => message.role === "tool").flatMap((message) => message.content);
142
+
143
+ const toolResultFor = (model: ScriptedModel, index: number, toolCallId: string): unknown => {
144
+ const part = toolMessagesOf(conversationOf(modelCall(model, index))).find(
145
+ (candidate) => (candidate as {toolCallId?: string}).toolCallId === toolCallId
146
+ ) as {output?: unknown} | undefined;
147
+ if (!part) {
148
+ expect.unreachable(`Model call ${index} has no result for ${toolCallId}`);
149
+ }
150
+ return part.output;
151
+ };
152
+
153
+ const eventKeys = (events: Record<string, unknown>[]): string[] =>
154
+ events.map((event) => Object.keys(event)[0]);
155
+
156
+ describe("host tools that need approval", () => {
157
+ beforeAll(async () => {
158
+ await ensureTestUsers();
159
+ });
160
+
161
+ afterEach(async () => {
162
+ await AIRequest.deleteMany({});
163
+ await GptHistory.deleteMany({});
164
+ });
165
+
166
+ describe("pause", () => {
167
+ it("pauses on a server-made confirm and does not run the tool", async () => {
168
+ const {approvalId, events, historyId, host, model} = await pauseOnApproval({});
169
+
170
+ expect(approvalId).not.toBe("call_delete");
171
+ expect(host.execute).not.toHaveBeenCalled();
172
+ expect(model.doStream).toHaveBeenCalledTimes(1);
173
+ expect(events).toEqual([
174
+ {
175
+ toolCall: {
176
+ args: DELETE_INPUT,
177
+ toolCallId: "call_delete",
178
+ toolName: "deleteCompletedTodos",
179
+ },
180
+ },
181
+ {
182
+ ask: {
183
+ input: DEFAULT_APPROVAL_INPUT,
184
+ kind: "confirm",
185
+ origin: "approval",
186
+ simple: {
187
+ buttons: [
188
+ {
189
+ id: "approve",
190
+ label: "Allow",
191
+ response: APPROVE,
192
+ style: "primary",
193
+ },
194
+ {id: "deny", label: "Deny", response: DENY, style: "cancel"},
195
+ ],
196
+ handoff: false,
197
+ kind: "confirm",
198
+ text: DEFAULT_APPROVAL_INPUT.prompt,
199
+ toolCallId: approvalId,
200
+ },
201
+ toolCallId: approvalId,
202
+ toolName: "deleteCompletedTodos",
203
+ },
204
+ historyId,
205
+ },
206
+ {done: true, historyId, pendingAsk: {toolCallId: approvalId}},
207
+ ]);
208
+
209
+ const history = await loadHistory(historyId);
210
+ expect(rowsOf(history)).toEqual([
211
+ {text: USER_PROMPT, type: "user"},
212
+ {
213
+ args: DELETE_INPUT,
214
+ text: "Tool call: deleteCompletedTodos",
215
+ toolCallId: "call_delete",
216
+ toolName: "deleteCompletedTodos",
217
+ type: "tool-call",
218
+ },
219
+ {
220
+ args: DEFAULT_APPROVAL_INPUT,
221
+ ask: {kind: "confirm", origin: "approval", status: "pending"},
222
+ text: "Tool call: deleteCompletedTodos",
223
+ toolCallId: approvalId,
224
+ toolName: "deleteCompletedTodos",
225
+ type: "tool-call",
226
+ },
227
+ ]);
228
+ expect(pendingAskOf(history)).toMatchObject({
229
+ approvalId,
230
+ input: DEFAULT_APPROVAL_INPUT,
231
+ kind: "confirm",
232
+ origin: "approval",
233
+ promptIndex: 1,
234
+ toolCallId: approvalId,
235
+ toolName: "deleteCompletedTodos",
236
+ });
237
+
238
+ const [request] = await AIRequest.find({});
239
+ expect(request?.metadata).toEqual({
240
+ ask: {
241
+ kind: "confirm",
242
+ origin: "approval",
243
+ phase: "asked",
244
+ toolCallId: approvalId,
245
+ toolName: "deleteCompletedTodos",
246
+ },
247
+ });
248
+ });
249
+
250
+ it("asks with the host's approval input, with a destructive approve first", async () => {
251
+ const approval = mock(() => DESTRUCTIVE_APPROVAL_INPUT);
252
+ const {approvalId, events} = await pauseOnApproval({
253
+ asks: {approvals: {deleteCompletedTodos: approval}},
254
+ });
255
+
256
+ expect(approval).toHaveBeenCalledWith(DELETE_INPUT);
257
+ const {ask} = events.find((event) => "ask" in event) as {ask: Record<string, unknown>};
258
+ expect(ask.input).toEqual(DESTRUCTIVE_APPROVAL_INPUT);
259
+ expect(ask.simple).toMatchObject({
260
+ buttons: [
261
+ {id: "approve", label: "Delete", style: "destructive"},
262
+ {id: "deny", label: "Keep them", style: "cancel"},
263
+ ],
264
+ title: "Clean up todos",
265
+ toolCallId: approvalId,
266
+ });
267
+ });
268
+
269
+ it.each([
270
+ [
271
+ "returns an invalid input",
272
+ () => ({confirmLabel: "Same", denyLabel: "Same", prompt: "Go?"}),
273
+ ],
274
+ [
275
+ "throws",
276
+ () => {
277
+ throw new Error("approval prompt failed");
278
+ },
279
+ ],
280
+ ])("falls back to the default approval when asks.approvals %s", async (_case, approval) => {
281
+ const {events, host} = await pauseOnApproval({
282
+ asks: {approvals: {deleteCompletedTodos: approval}},
283
+ });
284
+
285
+ const {ask} = events.find((event) => "ask" in event) as {ask: Record<string, unknown>};
286
+ expect(ask.input).toEqual(DEFAULT_APPROVAL_INPUT);
287
+ expect(host.execute).not.toHaveBeenCalled();
288
+ });
289
+
290
+ it("pauses on the compact surface with the same confirm card", async () => {
291
+ const {events, historyId} = await pauseOnApproval({surface: "compact"});
292
+
293
+ const {ask} = events.find((event) => "ask" in event) as {ask: Record<string, unknown>};
294
+ expect(ask).toMatchObject({
295
+ input: DEFAULT_APPROVAL_INPUT,
296
+ kind: "confirm",
297
+ origin: "approval",
298
+ });
299
+ expect(pendingAskOf(await loadHistory(historyId))).toMatchObject({origin: "approval"});
300
+ });
301
+
302
+ it("never runs the tool when asks are off, because nothing can approve it", async () => {
303
+ const host = hostTools();
304
+ const model = createScriptedModel({steps: [toolCallStep(DELETE_CALL)]});
305
+ const agent = await authAsUser(buildApp({model, tools: host.tools}), "notAdmin");
306
+
307
+ const {events} = await streamPrompt(agent, {prompt: USER_PROMPT});
308
+
309
+ expect(eventKeys(events)).toEqual(["toolCall", "done"]);
310
+ expect(host.execute).not.toHaveBeenCalled();
311
+ expect((await loadHistory(await onlyHistoryId())).pendingAsk).toBeUndefined();
312
+ });
313
+ });
314
+
315
+ describe("answer", () => {
316
+ it("approving runs the tool once and the model sees its result", async () => {
317
+ const paused = await pauseOnApproval({});
318
+ const {approvalId, historyId, host, model} = paused;
319
+
320
+ const {events} = await answer(paused, APPROVE);
321
+
322
+ expect(host.execute).toHaveBeenCalledTimes(1);
323
+ expect(host.execute.mock.calls[0]?.[0]).toEqual(DELETE_INPUT);
324
+ expect(events).toEqual([
325
+ {askResolved: {action: "accept", toolCallId: approvalId}},
326
+ {
327
+ toolResult: {
328
+ result: DELETED,
329
+ toolCallId: "call_delete",
330
+ toolName: "deleteCompletedTodos",
331
+ },
332
+ },
333
+ {text: DELETED_REPLY},
334
+ {done: true, historyId, title: TITLE},
335
+ ]);
336
+ expect(toolResultFor(model, 1, "call_delete")).toEqual({type: "json", value: DELETED});
337
+
338
+ const history = await loadHistory(historyId);
339
+ expect(history.pendingAsk).toBeUndefined();
340
+ expect(rowsOf(history).slice(2)).toEqual([
341
+ {
342
+ args: DEFAULT_APPROVAL_INPUT,
343
+ ask: {kind: "confirm", origin: "approval", status: "answered"},
344
+ text: "Tool call: deleteCompletedTodos",
345
+ toolCallId: approvalId,
346
+ toolName: "deleteCompletedTodos",
347
+ type: "tool-call",
348
+ },
349
+ {
350
+ result: APPROVE,
351
+ text: "Tool result: deleteCompletedTodos",
352
+ toolCallId: approvalId,
353
+ toolName: "deleteCompletedTodos",
354
+ type: "tool-result",
355
+ },
356
+ {
357
+ result: DELETED,
358
+ text: "Tool result: deleteCompletedTodos",
359
+ toolCallId: "call_delete",
360
+ toolName: "deleteCompletedTodos",
361
+ type: "tool-result",
362
+ },
363
+ {model: "scripted-model", text: DELETED_REPLY, type: "assistant"},
364
+ ]);
365
+
366
+ const answered = await AIRequest.findOne({prompt: JSON.stringify(APPROVE)});
367
+ expect(answered?.metadata).toEqual({
368
+ ask: {
369
+ action: "accept",
370
+ kind: "confirm",
371
+ origin: "approval",
372
+ phase: "answered",
373
+ toolCallId: approvalId,
374
+ toolName: "deleteCompletedTodos",
375
+ },
376
+ });
377
+ });
378
+
379
+ it.each([
380
+ ["denying", DENY, {}, "user_denied"],
381
+ ["declining", {action: "decline"}, {allowDecline: true}, "user_declined"],
382
+ ["cancelling", {action: "cancel", reason: "closed_the_card"}, {}, "closed_the_card"],
383
+ ])(
384
+ "%s never runs the tool and the model sees the denial",
385
+ async (_case, response, extraInput, reason) => {
386
+ const paused = await pauseOnApproval({
387
+ asks: {
388
+ approvals: {deleteCompletedTodos: () => ({...DEFAULT_APPROVAL_INPUT, ...extraInput})},
389
+ },
390
+ steps: [toolCallStep(DELETE_CALL), textStep("Okay, I left them.")],
391
+ });
392
+
393
+ const {events} = await answer(paused, response);
394
+
395
+ expect(paused.host.execute).not.toHaveBeenCalled();
396
+ expect(eventKeys(events)).toEqual(["askResolved", "toolResult", "text", "done"]);
397
+ expect(events[1]).toEqual({
398
+ toolResult: {
399
+ result: {approved: false, reason},
400
+ toolCallId: "call_delete",
401
+ toolName: "deleteCompletedTodos",
402
+ },
403
+ });
404
+ expect(toolResultFor(paused.model, 1, "call_delete")).toMatchObject({reason});
405
+ expect(JSON.stringify(toolResultFor(paused.model, 1, "call_delete"))).not.toContain(
406
+ "deleted"
407
+ );
408
+ }
409
+ );
410
+
411
+ it("keeps the approval pending and never runs the tool when the tools fail to load on resume", async () => {
412
+ let isToolLoadBroken = false;
413
+ const paused = await pauseOnApproval({
414
+ createRequestTools: () => {
415
+ if (isToolLoadBroken) {
416
+ throw new Error("Tool registry is down");
417
+ }
418
+ return {};
419
+ },
420
+ });
421
+ const pausedAsk = pendingAskOf(await loadHistory(paused.historyId));
422
+ isToolLoadBroken = true;
423
+
424
+ const failed = await paused.agent.post("/gpt/prompt").send({
425
+ askResponse: {...APPROVE, toolCallId: paused.approvalId},
426
+ historyId: paused.historyId,
427
+ });
428
+
429
+ expect(failed.status).toBe(500);
430
+ expect(paused.host.execute).not.toHaveBeenCalled();
431
+ expect(pendingAskOf(await loadHistory(paused.historyId))).toEqual(pausedAsk);
432
+
433
+ isToolLoadBroken = false;
434
+ const retry = await answer(paused, APPROVE);
435
+ expect(eventKeys(retry.events)).toEqual(["askResolved", "toolResult", "text", "done"]);
436
+ expect(paused.host.execute).toHaveBeenCalledTimes(1);
437
+ });
438
+
439
+ it("keeps the approval answered when the model fails after the approved tool ran", async () => {
440
+ const paused = await pauseOnApproval({});
441
+ paused.model.doStream.mockImplementationOnce(async () => {
442
+ throw new Error("The model is overloaded");
443
+ });
444
+
445
+ const {events} = await answer(paused, APPROVE);
446
+
447
+ expect(eventKeys(events)).toEqual(["askResolved", "toolResult", "error", "done"]);
448
+ expect(events.at(-1)).toEqual({done: true, historyId: paused.historyId});
449
+ expect(paused.host.execute).toHaveBeenCalledTimes(1);
450
+ expect((await loadHistory(paused.historyId)).pendingAsk).toBeUndefined();
451
+ const retry = await paused.agent.post("/gpt/prompt").send({
452
+ askResponse: {...APPROVE, toolCallId: paused.approvalId},
453
+ historyId: paused.historyId,
454
+ });
455
+ expect(retry.status).toBe(409);
456
+ expect(paused.host.execute).toHaveBeenCalledTimes(1);
457
+ });
458
+
459
+ it("keeps an ask's answer and cancels the approval when claiming it fails after its rows were saved", async () => {
460
+ const host = hostTools();
461
+ const model = createScriptedModel({
462
+ steps: [toolCallStep(PLAN_ASK_CALL), toolCallStep(DELETE_CALL), textStep(DELETED_REPLY)],
463
+ });
464
+ const agent = await authAsUser(buildApp({asks: true, model, tools: host.tools}), "notAdmin");
465
+ const historyId = await pauseOnPlanAsk(agent);
466
+ const claim = failNextAskClaim();
467
+
468
+ const {events} = await streamPrompt(agent, {
469
+ askResponse: {toolCallId: "call_plan", ...TEAM_ANSWER},
470
+ historyId,
471
+ });
472
+ claim.mockRestore();
473
+
474
+ expect(eventKeys(events)).toEqual(["askResolved", "toolCall", "error", "done"]);
475
+ expect(events.at(-1)).toEqual({done: true, historyId});
476
+ const history = await loadHistory(historyId);
477
+ expect(history.pendingAsk).toBeUndefined();
478
+ expect(history.prompts.filter((row) => row.ask?.status === "pending")).toHaveLength(0);
479
+ expect(history.prompts.find((row) => row.toolCallId === "call_plan")?.ask?.status).toBe(
480
+ "answered"
481
+ );
482
+ const retry = await streamPrompt(agent, {
483
+ askResponse: {toolCallId: "call_plan", ...TEAM_ANSWER},
484
+ historyId,
485
+ });
486
+ expect(retry.status).toBe(409);
487
+ const after = await loadHistory(historyId);
488
+ expect(
489
+ after.prompts.filter((row) => row.type === "tool-result" && row.toolCallId === "call_plan")
490
+ ).toHaveLength(1);
491
+ expect(host.execute).not.toHaveBeenCalled();
492
+ });
493
+
494
+ it("a new message cancels the pending approval and the tool never runs", async () => {
495
+ const paused = await pauseOnApproval({
496
+ steps: [toolCallStep(DELETE_CALL), textStep("Sure, what next?")],
497
+ });
498
+
499
+ const {events} = await streamPrompt(paused.agent, {
500
+ historyId: paused.historyId,
501
+ prompt: "Never mind",
502
+ });
503
+
504
+ expect(paused.host.execute).not.toHaveBeenCalled();
505
+ expect(events[0]).toEqual({askResolved: {action: "cancel", toolCallId: paused.approvalId}});
506
+ const history = await loadHistory(paused.historyId);
507
+ expect(history.pendingAsk).toBeUndefined();
508
+ expect(rowsOf(history).slice(2, 4)).toEqual([
509
+ {
510
+ args: DEFAULT_APPROVAL_INPUT,
511
+ ask: {kind: "confirm", origin: "approval", status: "cancelled"},
512
+ text: "Tool call: deleteCompletedTodos",
513
+ toolCallId: paused.approvalId,
514
+ toolName: "deleteCompletedTodos",
515
+ type: "tool-call",
516
+ },
517
+ {
518
+ result: {action: "cancel", reason: "user_sent_message"},
519
+ text: "Tool result: deleteCompletedTodos",
520
+ toolCallId: paused.approvalId,
521
+ toolName: "deleteCompletedTodos",
522
+ type: "tool-result",
523
+ },
524
+ ]);
525
+ // Approval rows are display-only, so the next turn replays only the user's messages.
526
+ expect(conversationOf(modelCall(paused.model, 1)).map((message) => message.role)).toEqual([
527
+ "user",
528
+ "user",
529
+ ]);
530
+ });
531
+
532
+ it("pauses again when the resumed turn asks for another approval", async () => {
533
+ const paused = await pauseOnApproval({
534
+ steps: [toolCallStep(DELETE_CALL), toolCallStep(ARCHIVE_CALL), textStep("All tidy.")],
535
+ });
536
+
537
+ const first = await answer(paused, APPROVE);
538
+ const next = first.events.find((event) => "ask" in event) as {
539
+ ask: {toolCallId: string; toolName: string};
540
+ };
541
+ expect(next.ask).toMatchObject({origin: "approval", toolName: "archiveTodos"});
542
+ const resumed = await AIRequest.findOne({prompt: JSON.stringify(APPROVE)});
543
+ expect(resumed?.metadata?.nextAsk).toEqual({
544
+ kind: "confirm",
545
+ origin: "approval",
546
+ phase: "asked",
547
+ toolCallId: next.ask.toolCallId,
548
+ toolName: "archiveTodos",
549
+ });
550
+
551
+ const second = await answer({...paused, approvalId: next.ask.toolCallId}, APPROVE);
552
+
553
+ expect(eventKeys(second.events)).toEqual(["askResolved", "toolResult", "text", "done"]);
554
+ expect(paused.host.execute).toHaveBeenCalledTimes(1);
555
+ expect(paused.host.archive).toHaveBeenCalledTimes(1);
556
+ expect(toolResultFor(paused.model, 2, "call_archive")).toEqual({
557
+ type: "json",
558
+ value: {archived: 1},
559
+ });
560
+ });
561
+
562
+ it("later turns do not replay the approval as a tool call", async () => {
563
+ const paused = await pauseOnApproval({
564
+ steps: [toolCallStep(DELETE_CALL), textStep(DELETED_REPLY), textStep("You're welcome.")],
565
+ });
566
+ await answer(paused, APPROVE);
567
+
568
+ await streamPrompt(paused.agent, {historyId: paused.historyId, prompt: "Thanks"});
569
+
570
+ expect(conversationOf(modelCall(paused.model, 2)).map((message) => message.role)).toEqual([
571
+ "user",
572
+ "assistant",
573
+ "user",
574
+ ]);
575
+ expect(paused.host.execute).toHaveBeenCalledTimes(1);
576
+ });
577
+ });
578
+
579
+ describe("one pending ask at a time", () => {
580
+ it("denies every extra approval in the step with one_ask_at_a_time", async () => {
581
+ const paused = await pauseOnApproval({
582
+ steps: [toolCallStep(DELETE_CALL, ARCHIVE_CALL), textStep(DELETED_REPLY)],
583
+ });
584
+ const {historyId, host, model} = paused;
585
+ expect(pendingAskOf(await loadHistory(historyId))).toMatchObject({
586
+ toolName: "deleteCompletedTodos",
587
+ });
588
+
589
+ const {events} = await answer(paused, APPROVE);
590
+
591
+ expect(host.execute).toHaveBeenCalledTimes(1);
592
+ expect(host.archive).not.toHaveBeenCalled();
593
+ expect(toolResultFor(model, 1, "call_delete")).toEqual({type: "json", value: DELETED});
594
+ expect(toolResultFor(model, 1, "call_archive")).toMatchObject({
595
+ reason: "one_ask_at_a_time",
596
+ });
597
+ expect(events).toContainEqual({
598
+ toolResult: {
599
+ result: {approved: false, reason: "one_ask_at_a_time"},
600
+ toolCallId: "call_archive",
601
+ toolName: "archiveTodos",
602
+ },
603
+ });
604
+ });
605
+
606
+ it("denies the approval when an ask comes first in the step", async () => {
607
+ const paused = await (async () => {
608
+ const host = hostTools();
609
+ const model = createScriptedModel({
610
+ steps: [toolCallStep(PLAN_ASK_CALL, DELETE_CALL), textStep("Team it is.")],
611
+ });
612
+ const agent = await authAsUser(
613
+ buildApp({asks: true, model, tools: host.tools}),
614
+ "notAdmin"
615
+ );
616
+ const {events} = await streamPrompt(agent, {prompt: USER_PROMPT});
617
+ return {agent, events, historyId: await onlyHistoryId(), host, model};
618
+ })();
619
+ expect(paused.events.find((event) => "ask" in event)).toMatchObject({
620
+ ask: {kind: "choice", toolCallId: "call_plan"},
621
+ });
622
+ expect(pendingAskOf(await loadHistory(paused.historyId))).not.toHaveProperty("origin");
623
+
624
+ await streamPrompt(paused.agent, {
625
+ askResponse: {...TEAM_ANSWER, toolCallId: "call_plan"},
626
+ historyId: paused.historyId,
627
+ });
628
+
629
+ expect(paused.host.execute).not.toHaveBeenCalled();
630
+ expect(toolResultFor(paused.model, 1, "call_plan")).toEqual({
631
+ type: "json",
632
+ value: TEAM_ANSWER,
633
+ });
634
+ expect(toolResultFor(paused.model, 1, "call_delete")).toMatchObject({
635
+ reason: "one_ask_at_a_time",
636
+ });
637
+ });
638
+
639
+ it("cancels the ask when an approval comes first in the step", async () => {
640
+ const paused = await pauseOnApproval({
641
+ steps: [toolCallStep(DELETE_CALL, PLAN_ASK_CALL), textStep(DELETED_REPLY)],
642
+ });
643
+ const rows = rowsOf(await loadHistory(paused.historyId));
644
+ expect(rows).toContainEqual({...PLAN_ASK_ROW, ask: {kind: "choice", status: "cancelled"}});
645
+ expect(rows).toContainEqual({
646
+ result: ONE_ASK_AT_A_TIME,
647
+ text: "Tool result: ask_choice",
648
+ toolCallId: "call_plan",
649
+ toolName: "ask_choice",
650
+ type: "tool-result",
651
+ });
652
+
653
+ await answer(paused, APPROVE);
654
+
655
+ expect(paused.host.execute).toHaveBeenCalledTimes(1);
656
+ expect(toolResultFor(paused.model, 1, "call_plan")).toEqual({
657
+ type: "json",
658
+ value: ONE_ASK_AT_A_TIME,
659
+ });
660
+ expect(toolResultFor(paused.model, 1, "call_delete")).toEqual({type: "json", value: DELETED});
661
+ });
662
+ });
663
+
664
+ describe("headless turn", () => {
665
+ it("the approve button runs the tool once", async () => {
666
+ const host = hostTools();
667
+ const model = createScriptedModel({
668
+ steps: [toolCallStep(DELETE_CALL), textStep(DELETED_REPLY)],
669
+ });
670
+ const agent = await authAsUser(buildApp({asks: true, model, tools: host.tools}), "notAdmin");
671
+ const created = await agent.post("/gpt/histories").send({});
672
+ const historyId = created.body.data._id as string;
673
+
674
+ const asked = await agent
675
+ .post(`/gpt/histories/${historyId}/turn`)
676
+ .send({prompt: USER_PROMPT});
677
+ expect(asked.status).toBe(200);
678
+ const {pendingAsk} = asked.body.data;
679
+ expect(pendingAsk).toMatchObject({kind: "confirm", simple: {kind: "confirm"}});
680
+ expect(host.execute).not.toHaveBeenCalled();
681
+
682
+ const listed = await agent.get("/gpt/histories/pendingAsks");
683
+ expect(listed.body.data).toMatchObject([
684
+ {historyId, kind: "confirm", toolCallId: pendingAsk.toolCallId},
685
+ ]);
686
+
687
+ const approved = await agent
688
+ .post(`/gpt/histories/${historyId}/turn`)
689
+ .send({buttonId: "approve", toolCallId: pendingAsk.toolCallId});
690
+
691
+ expect(approved.status).toBe(200);
692
+ expect(approved.body.data).toEqual({historyId, text: DELETED_REPLY, title: TITLE});
693
+ expect(host.execute).toHaveBeenCalledTimes(1);
694
+ expect(toolResultFor(model, 1, "call_delete")).toEqual({type: "json", value: DELETED});
695
+ });
696
+
697
+ it("the deny button never runs the tool", async () => {
698
+ const paused = await pauseOnApproval({
699
+ steps: [toolCallStep(DELETE_CALL), textStep("Kept them.")],
700
+ });
701
+
702
+ const denied = await paused.agent
703
+ .post(`/gpt/histories/${paused.historyId}/turn`)
704
+ .send({buttonId: "deny", toolCallId: paused.approvalId});
705
+
706
+ expect(denied.body.data).toEqual({
707
+ historyId: paused.historyId,
708
+ text: "Kept them.",
709
+ title: TITLE,
710
+ });
711
+ expect(paused.host.execute).not.toHaveBeenCalled();
712
+ });
713
+ });
714
+
715
+ describe("crafted answers", () => {
716
+ it("rejects an answer that names the host tool call instead of the approval", async () => {
717
+ const paused = await pauseOnApproval({});
718
+
719
+ const res = await paused.agent.post("/gpt/prompt").send({
720
+ askResponse: {...APPROVE, toolCallId: "call_delete"},
721
+ historyId: paused.historyId,
722
+ });
723
+
724
+ expect(res.status).toBe(409);
725
+ expect(paused.host.execute).not.toHaveBeenCalled();
726
+ expect((await loadHistory(paused.historyId)).pendingAsk?.toolCallId).toBe(paused.approvalId);
727
+ });
728
+
729
+ it.each([
730
+ ["a non-boolean confirmed", {action: "accept", content: {confirmed: "yes"}}],
731
+ ["extra approval fields", {action: "accept", approved: true, content: {confirmed: true}}],
732
+ ["an approval envelope", {action: "accept", content: {approvalId: "x", approved: true}}],
733
+ ["a decline the approval does not allow", {action: "decline"}],
734
+ ])("rejects %s with a 400 and does not run the tool", async (_case, response) => {
735
+ const paused = await pauseOnApproval({});
736
+
737
+ const res = await paused.agent.post("/gpt/prompt").send({
738
+ askResponse: {...response, toolCallId: paused.approvalId},
739
+ historyId: paused.historyId,
740
+ });
741
+
742
+ expect(res.status).toBe(400);
743
+ expect(paused.host.execute).not.toHaveBeenCalled();
744
+ expect((await loadHistory(paused.historyId)).pendingAsk?.toolCallId).toBe(paused.approvalId);
745
+ });
746
+
747
+ it("ignores a pendingAsk a client writes, so it cannot plant an approval", async () => {
748
+ const paused = await pauseOnApproval({});
749
+ const stored = await loadHistory(paused.historyId);
750
+
751
+ await paused.agent.patch(`/gpt/histories/${paused.historyId}`).send({
752
+ pendingAsk: {...JSON.parse(JSON.stringify(stored.pendingAsk)), approvalId: "forged"},
753
+ "pendingAsk.responseMessages": [],
754
+ });
755
+
756
+ expect(JSON.parse(JSON.stringify((await loadHistory(paused.historyId)).pendingAsk))).toEqual(
757
+ JSON.parse(JSON.stringify(stored.pendingAsk))
758
+ );
759
+ expect(paused.host.execute).not.toHaveBeenCalled();
760
+ });
761
+ });
762
+ });