@terreno/ai 57.8.0 → 57.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (425) hide show
  1. package/.cursorrules +2 -2
  2. package/README.md +2 -1
  3. package/bunfig.toml +1 -1
  4. package/dist/aiApp.d.ts +16 -5
  5. package/dist/aiApp.js +17 -7
  6. package/dist/harness/adminScreens.d.ts +6 -0
  7. package/dist/harness/adminScreens.js +11 -0
  8. package/dist/harness/agentLoop.d.ts +61 -0
  9. package/dist/harness/agentLoop.js +798 -0
  10. package/dist/harness/agentTaskNames.d.ts +4 -0
  11. package/dist/harness/agentTaskNames.js +7 -0
  12. package/dist/harness/approvalPolicy.d.ts +3 -0
  13. package/dist/harness/approvalPolicy.js +23 -0
  14. package/dist/harness/approvals.d.ts +101 -0
  15. package/dist/harness/approvals.js +468 -0
  16. package/dist/harness/commit.d.ts +238 -0
  17. package/dist/harness/commit.js +678 -0
  18. package/dist/harness/conversation.d.ts +75 -0
  19. package/dist/harness/conversation.js +397 -0
  20. package/dist/harness/defineAgent.d.ts +9 -0
  21. package/dist/harness/defineAgent.js +65 -0
  22. package/dist/harness/defineTask.d.ts +11 -0
  23. package/dist/harness/defineTask.js +59 -0
  24. package/dist/harness/defineTool.d.ts +7 -0
  25. package/dist/harness/defineTool.js +36 -0
  26. package/dist/harness/definitionError.d.ts +8 -0
  27. package/dist/harness/definitionError.js +15 -0
  28. package/dist/harness/errors.d.ts +127 -0
  29. package/dist/harness/errors.js +101 -0
  30. package/dist/harness/eventHub.d.ts +45 -0
  31. package/dist/harness/eventHub.js +135 -0
  32. package/dist/harness/events.d.ts +95 -0
  33. package/dist/harness/events.js +267 -0
  34. package/dist/harness/executionEnv.d.ts +31 -0
  35. package/dist/harness/executionEnv.js +2 -0
  36. package/dist/harness/extensions.d.ts +82 -0
  37. package/dist/harness/extensions.js +274 -0
  38. package/dist/harness/harness.d.ts +163 -0
  39. package/dist/harness/harness.js +520 -0
  40. package/dist/harness/harnessApp.d.ts +44 -0
  41. package/dist/harness/harnessApp.js +62 -0
  42. package/dist/harness/internalRuntime.d.ts +16 -0
  43. package/dist/harness/internalRuntime.js +18 -0
  44. package/dist/harness/leases.d.ts +35 -0
  45. package/dist/harness/leases.js +86 -0
  46. package/dist/harness/memo.d.ts +24 -0
  47. package/dist/harness/memo.js +78 -0
  48. package/dist/harness/modelCall.d.ts +45 -0
  49. package/dist/harness/modelCall.js +147 -0
  50. package/dist/harness/models/harnessApproval.d.ts +2 -0
  51. package/dist/harness/models/harnessApproval.js +100 -0
  52. package/dist/harness/models/harnessConversation.d.ts +2 -0
  53. package/dist/harness/models/harnessConversation.js +117 -0
  54. package/dist/harness/models/harnessEvent.d.ts +3 -0
  55. package/dist/harness/models/harnessEvent.js +73 -0
  56. package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
  57. package/dist/harness/models/harnessInboxEvent.js +58 -0
  58. package/dist/harness/models/harnessMemo.d.ts +2 -0
  59. package/dist/harness/models/harnessMemo.js +42 -0
  60. package/dist/harness/models/harnessMessage.d.ts +2 -0
  61. package/dist/harness/models/harnessMessage.js +73 -0
  62. package/dist/harness/models/harnessOwner.d.ts +2 -0
  63. package/dist/harness/models/harnessOwner.js +36 -0
  64. package/dist/harness/models/harnessTask.d.ts +2 -0
  65. package/dist/harness/models/harnessTask.js +186 -0
  66. package/dist/harness/ownership.d.ts +89 -0
  67. package/dist/harness/ownership.js +438 -0
  68. package/dist/harness/registry.d.ts +29 -0
  69. package/dist/harness/registry.js +93 -0
  70. package/dist/harness/retryBackoff.d.ts +31 -0
  71. package/dist/harness/retryBackoff.js +50 -0
  72. package/dist/harness/routes/approvals.d.ts +13 -0
  73. package/dist/harness/routes/approvals.js +93 -0
  74. package/dist/harness/routes/conversations.d.ts +11 -0
  75. package/dist/harness/routes/conversations.js +66 -0
  76. package/dist/harness/routes/events.d.ts +19 -0
  77. package/dist/harness/routes/events.js +215 -0
  78. package/dist/harness/routes/tasks.d.ts +11 -0
  79. package/dist/harness/routes/tasks.js +108 -0
  80. package/dist/harness/runners/inProcessRunner.d.ts +57 -0
  81. package/dist/harness/runners/inProcessRunner.js +196 -0
  82. package/dist/harness/runtime.d.ts +34 -0
  83. package/dist/harness/runtime.js +551 -0
  84. package/dist/harness/subagent.d.ts +53 -0
  85. package/dist/harness/subagent.js +134 -0
  86. package/dist/harness/suspend.d.ts +8 -0
  87. package/dist/harness/suspend.js +16 -0
  88. package/dist/harness/transaction.d.ts +3 -0
  89. package/dist/harness/transaction.js +22 -0
  90. package/dist/harness/waits.d.ts +100 -0
  91. package/dist/harness/waits.js +214 -0
  92. package/dist/index.d.ts +19 -1
  93. package/dist/index.js +45 -2
  94. package/dist/models/aiDataset.d.ts +2 -0
  95. package/dist/models/aiDataset.js +67 -0
  96. package/dist/models/aiRequest.js +1 -1
  97. package/dist/models/gptHistory.js +88 -2
  98. package/dist/observability/adminScreens.d.ts +5 -0
  99. package/dist/observability/adminScreens.js +32 -0
  100. package/dist/observability/compileTemplate.d.ts +1 -0
  101. package/dist/observability/compileTemplate.js +13 -0
  102. package/dist/observability/datasetImport.d.ts +11 -0
  103. package/dist/observability/datasetImport.js +185 -0
  104. package/dist/observability/evaluate.d.ts +31 -0
  105. package/dist/observability/evaluate.js +200 -0
  106. package/dist/observability/evaluatorTemplates.d.ts +17 -0
  107. package/dist/observability/evaluatorTemplates.js +111 -0
  108. package/dist/observability/jsonPath.d.ts +2 -0
  109. package/dist/observability/jsonPath.js +57 -0
  110. package/dist/observability/local/datasetStore.d.ts +87 -0
  111. package/dist/observability/local/datasetStore.js +326 -0
  112. package/dist/observability/local/evaluatorStore.d.ts +44 -0
  113. package/dist/observability/local/evaluatorStore.js +182 -0
  114. package/dist/observability/local/experimentRunner.d.ts +96 -0
  115. package/dist/observability/local/experimentRunner.js +419 -0
  116. package/dist/observability/local/localPlugin.d.ts +22 -0
  117. package/dist/observability/local/localPlugin.js +76 -0
  118. package/dist/observability/local/models/obsDataset.d.ts +2 -0
  119. package/dist/observability/local/models/obsDataset.js +37 -0
  120. package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
  121. package/dist/observability/local/models/obsDatasetItem.js +76 -0
  122. package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
  123. package/dist/observability/local/models/obsEvaluator.js +98 -0
  124. package/dist/observability/local/models/obsExperiment.d.ts +2 -0
  125. package/dist/observability/local/models/obsExperiment.js +87 -0
  126. package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
  127. package/dist/observability/local/models/obsExperimentItem.js +45 -0
  128. package/dist/observability/local/models/obsPrompt.d.ts +2 -0
  129. package/dist/observability/local/models/obsPrompt.js +42 -0
  130. package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
  131. package/dist/observability/local/models/obsPromptLabel.js +39 -0
  132. package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
  133. package/dist/observability/local/models/obsPromptVersion.js +96 -0
  134. package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
  135. package/dist/observability/local/models/obsReviewItem.js +75 -0
  136. package/dist/observability/local/models/obsScore.d.ts +2 -0
  137. package/dist/observability/local/models/obsScore.js +58 -0
  138. package/dist/observability/local/models/obsSpan.d.ts +2 -0
  139. package/dist/observability/local/models/obsSpan.js +64 -0
  140. package/dist/observability/local/models/obsTrace.d.ts +2 -0
  141. package/dist/observability/local/models/obsTrace.js +93 -0
  142. package/dist/observability/local/promptStore.d.ts +180 -0
  143. package/dist/observability/local/promptStore.js +444 -0
  144. package/dist/observability/local/reviewStore.d.ts +50 -0
  145. package/dist/observability/local/reviewStore.js +203 -0
  146. package/dist/observability/local/traceStore.d.ts +103 -0
  147. package/dist/observability/local/traceStore.js +325 -0
  148. package/dist/observability/observabilityApp.d.ts +24 -0
  149. package/dist/observability/observabilityApp.js +157 -0
  150. package/dist/observability/observabilityAppRegistry.d.ts +14 -0
  151. package/dist/observability/observabilityAppRegistry.js +16 -0
  152. package/dist/observability/permissions.d.ts +7 -0
  153. package/dist/observability/permissions.js +60 -0
  154. package/dist/observability/reviewPanels.d.ts +18 -0
  155. package/dist/observability/reviewPanels.js +46 -0
  156. package/dist/observability/routes/datasets.d.ts +7 -0
  157. package/dist/observability/routes/datasets.js +149 -0
  158. package/dist/observability/routes/evaluators.d.ts +7 -0
  159. package/dist/observability/routes/evaluators.js +123 -0
  160. package/dist/observability/routes/experiments.d.ts +7 -0
  161. package/dist/observability/routes/experiments.js +95 -0
  162. package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
  163. package/dist/observability/routes/observabilityRouteAccess.js +67 -0
  164. package/dist/observability/routes/prompts.d.ts +12 -0
  165. package/dist/observability/routes/prompts.js +172 -0
  166. package/dist/observability/routes/requestAiService.d.ts +13 -0
  167. package/dist/observability/routes/requestAiService.js +21 -0
  168. package/dist/observability/routes/review.d.ts +7 -0
  169. package/dist/observability/routes/review.js +79 -0
  170. package/dist/observability/routes/status.d.ts +5 -0
  171. package/dist/observability/routes/status.js +31 -0
  172. package/dist/observability/routes/testMultiStage.d.ts +9 -0
  173. package/dist/observability/routes/testMultiStage.js +67 -0
  174. package/dist/observability/routes/traces.d.ts +7 -0
  175. package/dist/observability/routes/traces.js +129 -0
  176. package/dist/observability/schemaValidation.d.ts +5 -0
  177. package/dist/observability/schemaValidation.js +26 -0
  178. package/dist/observability/sopThresholds.d.ts +2 -0
  179. package/dist/observability/sopThresholds.js +21 -0
  180. package/dist/observability/status.d.ts +23 -0
  181. package/dist/observability/status.js +34 -0
  182. package/dist/observability/testMultiStageSchemas.d.ts +10 -0
  183. package/dist/observability/testMultiStageSchemas.js +79 -0
  184. package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
  185. package/dist/observability/testMultiStageWorkflow.js +218 -0
  186. package/dist/observability/types.d.ts +159 -0
  187. package/dist/observability/types.js +67 -0
  188. package/dist/routes/files.js +20 -7
  189. package/dist/routes/gpt.js +179 -488
  190. package/dist/routes/gptActions.d.ts +9 -0
  191. package/dist/routes/gptActions.js +212 -0
  192. package/dist/routes/gptDatasets.d.ts +5 -0
  193. package/dist/routes/gptDatasets.js +94 -0
  194. package/dist/routes/gptHistories.d.ts +3 -1
  195. package/dist/routes/gptHistories.js +188 -2
  196. package/dist/service/aiDatasets.d.ts +57 -0
  197. package/dist/service/aiDatasets.js +253 -0
  198. package/dist/service/aiService.d.ts +50 -1
  199. package/dist/service/aiService.js +431 -77
  200. package/dist/service/askFiles.d.ts +49 -0
  201. package/dist/service/askFiles.js +162 -0
  202. package/dist/service/asks.d.ts +76 -0
  203. package/dist/service/asks.js +284 -0
  204. package/dist/service/chatTurn.d.ts +128 -0
  205. package/dist/service/chatTurn.js +1674 -0
  206. package/dist/service/fileStorage.d.ts +4 -0
  207. package/dist/service/fileStorage.js +7 -2
  208. package/dist/service/fileUploadsGate.d.ts +8 -0
  209. package/dist/service/fileUploadsGate.js +18 -0
  210. package/dist/service/fillActionIds.d.ts +6 -0
  211. package/dist/service/fillActionIds.js +79 -0
  212. package/dist/service/gemini.d.ts +3 -3
  213. package/dist/service/gemini.js +3 -3
  214. package/dist/service/jsonFenceModel.d.ts +7 -0
  215. package/dist/service/jsonFenceModel.js +45 -0
  216. package/dist/service/prompts.d.ts +52 -0
  217. package/dist/service/prompts.js +96 -1
  218. package/dist/service/sanitizeHtml.d.ts +11 -0
  219. package/dist/service/sanitizeHtml.js +108 -0
  220. package/dist/service/vertex.d.ts +6 -6
  221. package/dist/service/vertex.js +5 -5
  222. package/dist/types/harness.d.ts +934 -0
  223. package/dist/types/harness.js +133 -0
  224. package/dist/types/index.d.ts +219 -13
  225. package/dist/types/index.js +2 -0
  226. package/dist/types/observability.d.ts +288 -0
  227. package/dist/types/observability.js +2 -0
  228. package/package.json +17 -5
  229. package/src/aiApp.test.ts +207 -1
  230. package/src/aiApp.ts +31 -8
  231. package/src/harness/adminScreens.test.ts +34 -0
  232. package/src/harness/adminScreens.ts +11 -0
  233. package/src/harness/agentLoop.test.ts +1034 -0
  234. package/src/harness/agentLoop.ts +1082 -0
  235. package/src/harness/agentRecovery.test.ts +380 -0
  236. package/src/harness/agentTaskNames.ts +4 -0
  237. package/src/harness/approvalPolicy.ts +28 -0
  238. package/src/harness/approvals.test.ts +1094 -0
  239. package/src/harness/approvals.ts +644 -0
  240. package/src/harness/commit.ts +1049 -0
  241. package/src/harness/conversation.ts +503 -0
  242. package/src/harness/defineAgent.test.ts +92 -0
  243. package/src/harness/defineAgent.ts +82 -0
  244. package/src/harness/defineTask.test.ts +59 -0
  245. package/src/harness/defineTask.ts +76 -0
  246. package/src/harness/defineTool.ts +45 -0
  247. package/src/harness/definitionError.ts +13 -0
  248. package/src/harness/errors.test.ts +90 -0
  249. package/src/harness/errors.ts +119 -0
  250. package/src/harness/eventHub.test.ts +124 -0
  251. package/src/harness/eventHub.ts +159 -0
  252. package/src/harness/events.test.ts +757 -0
  253. package/src/harness/events.ts +384 -0
  254. package/src/harness/executionEnv.ts +34 -0
  255. package/src/harness/extensions.test.ts +1218 -0
  256. package/src/harness/extensions.ts +382 -0
  257. package/src/harness/harness.test.ts +693 -0
  258. package/src/harness/harness.ts +767 -0
  259. package/src/harness/harnessApp.ts +76 -0
  260. package/src/harness/internalRuntime.ts +44 -0
  261. package/src/harness/leases.ts +132 -0
  262. package/src/harness/memo.ts +107 -0
  263. package/src/harness/modelCall.test.ts +134 -0
  264. package/src/harness/modelCall.ts +200 -0
  265. package/src/harness/models/harnessApproval.ts +109 -0
  266. package/src/harness/models/harnessConversation.ts +142 -0
  267. package/src/harness/models/harnessEvent.ts +92 -0
  268. package/src/harness/models/harnessInboxEvent.ts +73 -0
  269. package/src/harness/models/harnessMemo.ts +43 -0
  270. package/src/harness/models/harnessMessage.ts +85 -0
  271. package/src/harness/models/harnessModels.test.ts +60 -0
  272. package/src/harness/models/harnessOwner.ts +40 -0
  273. package/src/harness/models/harnessTask.ts +201 -0
  274. package/src/harness/ownership.test.ts +1087 -0
  275. package/src/harness/ownership.ts +618 -0
  276. package/src/harness/recovery.test.ts +912 -0
  277. package/src/harness/registry.ts +124 -0
  278. package/src/harness/retries.test.ts +292 -0
  279. package/src/harness/retryBackoff.test.ts +100 -0
  280. package/src/harness/retryBackoff.ts +77 -0
  281. package/src/harness/routes/approvals.ts +123 -0
  282. package/src/harness/routes/conversations.test.ts +518 -0
  283. package/src/harness/routes/conversations.ts +102 -0
  284. package/src/harness/routes/events.ts +280 -0
  285. package/src/harness/routes/tasks.test.ts +198 -0
  286. package/src/harness/routes/tasks.ts +146 -0
  287. package/src/harness/runners/inProcessRunner.test.ts +402 -0
  288. package/src/harness/runners/inProcessRunner.ts +220 -0
  289. package/src/harness/runtime.ts +714 -0
  290. package/src/harness/subagent.test.ts +696 -0
  291. package/src/harness/subagent.ts +192 -0
  292. package/src/harness/suspend.ts +13 -0
  293. package/src/harness/transaction.ts +17 -0
  294. package/src/harness/versioning.test.ts +288 -0
  295. package/src/harness/waits.test.ts +693 -0
  296. package/src/harness/waits.ts +356 -0
  297. package/src/index.test.ts +13 -0
  298. package/src/index.ts +70 -0
  299. package/src/isolated/adminScreensConfig.isolated.ts +143 -0
  300. package/src/models/aiDataset.test.ts +34 -0
  301. package/src/models/aiDataset.ts +78 -0
  302. package/src/models/aiRequest.test.ts +2 -2
  303. package/src/models/aiRequest.ts +2 -1
  304. package/src/models/gptHistory.test.ts +156 -0
  305. package/src/models/gptHistory.ts +106 -4
  306. package/src/observability/adminScreens.test.ts +71 -0
  307. package/src/observability/adminScreens.ts +42 -0
  308. package/src/observability/compileTemplate.test.ts +14 -0
  309. package/src/observability/compileTemplate.ts +10 -0
  310. package/src/observability/datasetImport.test.ts +95 -0
  311. package/src/observability/datasetImport.ts +220 -0
  312. package/src/observability/evaluate.test.ts +549 -0
  313. package/src/observability/evaluate.ts +254 -0
  314. package/src/observability/evaluatorTemplates.ts +124 -0
  315. package/src/observability/jsonPath.test.ts +52 -0
  316. package/src/observability/jsonPath.ts +60 -0
  317. package/src/observability/local/datasetStore.test.ts +230 -0
  318. package/src/observability/local/datasetStore.ts +452 -0
  319. package/src/observability/local/evaluatorStore.test.ts +138 -0
  320. package/src/observability/local/evaluatorStore.ts +224 -0
  321. package/src/observability/local/experimentRunner.test.ts +489 -0
  322. package/src/observability/local/experimentRunner.ts +548 -0
  323. package/src/observability/local/localPlugin.ts +88 -0
  324. package/src/observability/local/models/obsDataset.ts +38 -0
  325. package/src/observability/local/models/obsDatasetItem.ts +80 -0
  326. package/src/observability/local/models/obsEvaluator.ts +102 -0
  327. package/src/observability/local/models/obsExperiment.ts +91 -0
  328. package/src/observability/local/models/obsExperimentItem.ts +52 -0
  329. package/src/observability/local/models/obsPrompt.ts +43 -0
  330. package/src/observability/local/models/obsPromptLabel.ts +43 -0
  331. package/src/observability/local/models/obsPromptVersion.ts +107 -0
  332. package/src/observability/local/models/obsReviewItem.ts +79 -0
  333. package/src/observability/local/models/obsScore.ts +59 -0
  334. package/src/observability/local/models/obsSpan.ts +65 -0
  335. package/src/observability/local/models/obsTrace.ts +94 -0
  336. package/src/observability/local/obsModels.test.ts +119 -0
  337. package/src/observability/local/promptStore.test.ts +351 -0
  338. package/src/observability/local/promptStore.ts +665 -0
  339. package/src/observability/local/reviewStore.test.ts +164 -0
  340. package/src/observability/local/reviewStore.ts +264 -0
  341. package/src/observability/local/traceStore.test.ts +264 -0
  342. package/src/observability/local/traceStore.ts +440 -0
  343. package/src/observability/observabilityApp.test.ts +92 -0
  344. package/src/observability/observabilityApp.ts +185 -0
  345. package/src/observability/observabilityAppRegistry.test.ts +30 -0
  346. package/src/observability/observabilityAppRegistry.ts +35 -0
  347. package/src/observability/permissions.test.ts +96 -0
  348. package/src/observability/permissions.ts +78 -0
  349. package/src/observability/reviewPanels.test.ts +35 -0
  350. package/src/observability/reviewPanels.ts +64 -0
  351. package/src/observability/routes/datasets.test.ts +176 -0
  352. package/src/observability/routes/datasets.ts +252 -0
  353. package/src/observability/routes/evaluators.test.ts +119 -0
  354. package/src/observability/routes/evaluators.ts +207 -0
  355. package/src/observability/routes/experiments.test.ts +282 -0
  356. package/src/observability/routes/experiments.ts +152 -0
  357. package/src/observability/routes/observabilityRbac.test.ts +767 -0
  358. package/src/observability/routes/observabilityRouteAccess.ts +106 -0
  359. package/src/observability/routes/prompts.test.ts +320 -0
  360. package/src/observability/routes/prompts.ts +263 -0
  361. package/src/observability/routes/requestAiService.ts +36 -0
  362. package/src/observability/routes/review.test.ts +120 -0
  363. package/src/observability/routes/review.ts +125 -0
  364. package/src/observability/routes/status.test.ts +116 -0
  365. package/src/observability/routes/status.ts +47 -0
  366. package/src/observability/routes/testMultiStage.test.ts +214 -0
  367. package/src/observability/routes/testMultiStage.ts +96 -0
  368. package/src/observability/routes/traces.test.ts +167 -0
  369. package/src/observability/routes/traces.ts +172 -0
  370. package/src/observability/schemaValidation.ts +29 -0
  371. package/src/observability/sopThresholds.ts +20 -0
  372. package/src/observability/status.test.ts +38 -0
  373. package/src/observability/status.ts +56 -0
  374. package/src/observability/testMultiStageSchemas.ts +95 -0
  375. package/src/observability/testMultiStageWorkflow.test.ts +114 -0
  376. package/src/observability/testMultiStageWorkflow.ts +307 -0
  377. package/src/observability/types.ts +262 -0
  378. package/src/routes/files.test.ts +84 -9
  379. package/src/routes/files.ts +23 -7
  380. package/src/routes/gpt.test.ts +3279 -0
  381. package/src/routes/gpt.ts +214 -552
  382. package/src/routes/gptActions.test.ts +241 -0
  383. package/src/routes/gptActions.ts +255 -0
  384. package/src/routes/gptApprovals.test.ts +762 -0
  385. package/src/routes/gptDatasets.test.ts +86 -0
  386. package/src/routes/gptDatasets.ts +104 -0
  387. package/src/routes/gptFiles.test.ts +589 -0
  388. package/src/routes/gptHistories.test.ts +933 -0
  389. package/src/routes/gptHistories.ts +266 -5
  390. package/src/routes/routes.test.ts +676 -5
  391. package/src/service/aiDatasets.test.ts +123 -0
  392. package/src/service/aiDatasets.ts +316 -0
  393. package/src/service/aiService.test.ts +614 -0
  394. package/src/service/aiService.ts +575 -111
  395. package/src/service/askFiles.test.ts +96 -0
  396. package/src/service/askFiles.ts +222 -0
  397. package/src/service/asks.test.ts +946 -0
  398. package/src/service/asks.ts +423 -0
  399. package/src/service/chatTurn.ts +2287 -0
  400. package/src/service/fileStorage.test.ts +11 -0
  401. package/src/service/fileStorage.ts +10 -2
  402. package/src/service/fileUploadsGate.ts +23 -0
  403. package/src/service/fillActionIds.test.ts +65 -0
  404. package/src/service/fillActionIds.ts +98 -0
  405. package/src/service/gemini.test.ts +16 -10
  406. package/src/service/gemini.ts +3 -3
  407. package/src/service/jsonFenceModel.test.ts +47 -0
  408. package/src/service/jsonFenceModel.ts +53 -0
  409. package/src/service/prompts.test.ts +21 -0
  410. package/src/service/prompts.ts +147 -0
  411. package/src/service/sanitizeHtml.test.ts +61 -0
  412. package/src/service/sanitizeHtml.ts +109 -0
  413. package/src/service/vertex.test.ts +49 -49
  414. package/src/service/vertex.ts +7 -8
  415. package/src/tests/bunSetup.test.ts +7 -4
  416. package/src/tests/bunSetup.ts +7 -7
  417. package/src/tests/chatHarness.ts +478 -0
  418. package/src/tests/generateStream.ts +49 -0
  419. package/src/tests/harnessErrors.ts +22 -0
  420. package/src/tests/harnessStreaming.ts +227 -0
  421. package/src/tests/helpers.ts +9 -2
  422. package/src/types/harness.ts +1095 -0
  423. package/src/types/index.ts +237 -12
  424. package/src/types/observability.ts +344 -0
  425. package/src/types/sanitize-html.d.ts +25 -0
@@ -0,0 +1,1087 @@
1
+ import {afterEach, beforeAll, beforeEach, describe, expect, it, setDefaultTimeout} from "bun:test";
2
+ import mongoose from "mongoose";
3
+
4
+ import {createLocalObservabilityPlugin} from "../observability/local/localPlugin";
5
+ import {registerObsSpan} from "../observability/local/models/obsSpan";
6
+ import {registerObsTrace} from "../observability/local/models/obsTrace";
7
+ import {harnessErrorMatching} from "../tests/harnessErrors";
8
+ import type {HarnessChildOutcome, HarnessTaskDocument, HarnessTestHooks} from "../types/harness";
9
+ import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
10
+ import type {HarnessModels} from "./commit";
11
+ import {
12
+ type AnyHarnessTaskDefinition,
13
+ defineTask,
14
+ Harness,
15
+ HarnessCommitConflictError,
16
+ InProcessRunner,
17
+ } from "./harness";
18
+ import {registerHarnessEvent, registerHarnessEventStream} from "./models/harnessEvent";
19
+ import {registerHarnessOwner} from "./models/harnessOwner";
20
+ import {registerHarnessTask} from "./models/harnessTask";
21
+ import {type HarnessEngine, sweepWaitingTasks} from "./ownership";
22
+
23
+ // Multi-task trees under a loaded full run need more than Bun's 5s default.
24
+ setDefaultTimeout(30_000);
25
+
26
+ const TaskModel = registerHarnessTask();
27
+ const OwnerModel = registerHarnessOwner();
28
+ let SpanModel: ObsSpanModel;
29
+ let TraceModel: ObsTraceModel;
30
+ const openHarnesses: Harness[] = [];
31
+
32
+ const pause = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
33
+
34
+ const waitUntil = async (predicate: () => Promise<boolean>, label: string): Promise<void> => {
35
+ for (let i = 0; i < 300; i++) {
36
+ if (await predicate()) {
37
+ return;
38
+ }
39
+ await pause(10);
40
+ }
41
+ throw new Error(`waitUntil timed out: ${label}`);
42
+ };
43
+
44
+ const openHarness = async (
45
+ registry: ReadonlyArray<AnyHarnessTaskDefinition>,
46
+ testHooks?: HarnessTestHooks
47
+ ): Promise<Harness> => {
48
+ const harness = await Harness.open({
49
+ registry,
50
+ runner: new InProcessRunner({pollInterval: {milliseconds: 20}}),
51
+ testHooks,
52
+ });
53
+ openHarnesses.push(harness);
54
+ return harness;
55
+ };
56
+
57
+ const findTask = (id: mongoose.Types.ObjectId | string): Promise<HarnessTaskDocument> =>
58
+ TaskModel.findExactlyOne({_id: id});
59
+
60
+ const childrenOf = (id: mongoose.Types.ObjectId): Promise<HarnessTaskDocument[]> =>
61
+ TaskModel.find({"ownership.id": id, "ownership.kind": "task"}).sort({created: 1});
62
+
63
+ const asAny = (definition: unknown): AnyHarnessTaskDefinition =>
64
+ definition as AnyHarnessTaskDefinition;
65
+
66
+ const completesWith = (name: string, result: unknown): AnyHarnessTaskDefinition =>
67
+ asAny(
68
+ defineTask<unknown, unknown, unknown>({
69
+ initial: () => ({phase: "work"}),
70
+ name,
71
+ phases: {
72
+ work: {run: async (_task, rt) => rt.commit({terminal: {result, status: "completed"}})},
73
+ },
74
+ version: 1,
75
+ })
76
+ );
77
+
78
+ const failsWith = (name: string, error: string): AnyHarnessTaskDefinition =>
79
+ asAny(
80
+ defineTask<unknown, unknown, unknown>({
81
+ initial: () => ({phase: "work"}),
82
+ name,
83
+ phases: {
84
+ work: {
85
+ run: async () => {
86
+ throw new Error(error);
87
+ },
88
+ },
89
+ },
90
+ retry: {maxAttempts: 1},
91
+ version: 1,
92
+ })
93
+ );
94
+
95
+ /**
96
+ * A parent that creates `children` (by definition) in one phase and waits on them with
97
+ * `policy`, completing with the outcomes.
98
+ */
99
+ const fanOut = ({
100
+ children,
101
+ name,
102
+ policy,
103
+ runs,
104
+ }: {
105
+ children: AnyHarnessTaskDefinition[];
106
+ name: string;
107
+ policy?: "all" | "failFast";
108
+ runs: {count: number};
109
+ }): AnyHarnessTaskDefinition =>
110
+ asAny(
111
+ defineTask<unknown, unknown, HarnessChildOutcome[]>({
112
+ initial: () => ({phase: "fanOut"}),
113
+ name,
114
+ phases: {
115
+ fanOut: {
116
+ run: async (_task, rt) => {
117
+ runs.count += 1;
118
+ const ids: string[] = [];
119
+ for (const child of children) {
120
+ ids.push(await rt.createTask(child, {from: name}));
121
+ }
122
+ const outcomes = await rt.waitForTasks(ids, {policy});
123
+ await rt.commit({terminal: {result: outcomes, status: "completed"}});
124
+ },
125
+ },
126
+ },
127
+ version: 1,
128
+ })
129
+ );
130
+
131
+ describe("Harness ownership tree", () => {
132
+ beforeAll(() => {
133
+ createLocalObservabilityPlugin();
134
+ SpanModel = registerObsSpan();
135
+ TraceModel = registerObsTrace();
136
+ });
137
+
138
+ beforeEach(async () => {
139
+ await Promise.all([
140
+ TaskModel.deleteMany({}),
141
+ SpanModel.deleteMany({}),
142
+ TraceModel.deleteMany({}),
143
+ OwnerModel.deleteMany({}),
144
+ ]);
145
+ });
146
+
147
+ afterEach(async () => {
148
+ await Promise.all(openHarnesses.splice(0).map((harness) => harness.stop()));
149
+ });
150
+
151
+ describe("rt.createTask + rt.waitForTasks", () => {
152
+ it("runs a child in the parent's trace, nests its spans, and hands back its outcome", async () => {
153
+ const child = completesWith("test.child", {summary: "ok"});
154
+ const runs = {count: 0};
155
+ const parent = fanOut({children: [child], name: "test.parent", runs});
156
+ const harness = await openHarness([parent, child]);
157
+ await harness.start();
158
+
159
+ const created = await harness.createTask(parent, {}, {userId: new mongoose.Types.ObjectId()});
160
+ const done = await harness.waitForTask(created._id);
161
+ expect(done.status).toBe("completed");
162
+
163
+ const [childTask] = await childrenOf(done._id);
164
+ expect(done.outcome?.result).toEqual([
165
+ {
166
+ id: String(childTask._id),
167
+ name: "test.child",
168
+ result: {summary: "ok"},
169
+ status: "completed",
170
+ },
171
+ ]);
172
+ // The phase ran, waited, and re-ran once woken; the second run reused the child.
173
+ expect(runs.count).toBe(2);
174
+ expect(await childrenOf(done._id)).toHaveLength(1);
175
+
176
+ expect(childTask.toObject().ownership).toEqual({id: done._id, kind: "task"});
177
+ expect(String(childTask.rootTaskId)).toBe(String(done._id));
178
+ expect(String(childTask.traceId)).toBe(String(done.traceId));
179
+ expect(String(childTask.userId)).toBe(String(done.userId));
180
+ expect(childTask.background).toBe(false);
181
+
182
+ const childSpan = await SpanModel.findExactlyOne({_id: childTask.rootSpanId});
183
+ expect(childSpan.name).toBe("test.child@1");
184
+ expect(String(childSpan.parentSpanId)).toBe(String(done.rootSpanId));
185
+ expect(childSpan.status).toBe("ok");
186
+ expect(childSpan.endedAt).toBeInstanceOf(Date);
187
+ const childPhase = await SpanModel.findExactlyOne({name: "work", traceId: done.traceId});
188
+ expect(String(childPhase.parentSpanId)).toBe(String(childTask.rootSpanId));
189
+
190
+ // One wait span plus the completing span for the parent's phase.
191
+ const parentPhases = await SpanModel.find({
192
+ name: "fanOut",
193
+ parentSpanId: done.rootSpanId,
194
+ }).sort({_id: 1});
195
+ expect(parentPhases.map((span) => span.output)).toEqual([
196
+ {waiting: {kind: "tasks", policy: "all", taskIds: [String(childTask._id)]}},
197
+ {terminal: {result: done.outcome?.result, status: "completed"}},
198
+ ]);
199
+
200
+ // Only the root task closed the shared trace.
201
+ expect(await TraceModel.countDocuments({})).toBe(1);
202
+ const trace = await TraceModel.findExactlyOne({_id: done.traceId});
203
+ expect(trace.output).toEqual(done.outcome?.result);
204
+ });
205
+
206
+ it("waits for every child under the default policy, in the order passed", async () => {
207
+ const ok = completesWith("test.okChild", 1);
208
+ const bad = failsWith("test.badChild", "lab system down");
209
+ const runs = {count: 0};
210
+ const parent = fanOut({children: [bad, ok], name: "test.waitAll", runs});
211
+ const harness = await openHarness([parent, ok, bad]);
212
+ await harness.start();
213
+
214
+ const done = await harness.waitForTask((await harness.createTask(parent, {}))._id);
215
+ expect(done.status).toBe("completed");
216
+ const outcomes = done.outcome?.result as HarnessChildOutcome[];
217
+ expect(
218
+ outcomes.map(({error, name, result, status}) => ({error, name, result, status}))
219
+ ).toEqual([
220
+ {error: "lab system down", name: "test.badChild", result: undefined, status: "failed"},
221
+ {error: undefined, name: "test.okChild", result: 1, status: "completed"},
222
+ ]);
223
+ });
224
+
225
+ it("stores background on the child", async () => {
226
+ const child = completesWith("test.bgChild", null);
227
+ const parent = defineTask<unknown, unknown, unknown>({
228
+ initial: () => ({phase: "spawn"}),
229
+ name: "test.bgParent",
230
+ phases: {
231
+ spawn: {
232
+ run: async (_task, rt) => {
233
+ await rt.createTask(child, {}, {background: true});
234
+ await rt.commit({terminal: {status: "completed"}});
235
+ },
236
+ },
237
+ },
238
+ version: 1,
239
+ });
240
+ const harness = await openHarness([asAny(parent), child]);
241
+ await harness.start();
242
+
243
+ const done = await harness.waitForTask((await harness.createTask(parent, {}))._id);
244
+ const [childTask] = await childrenOf(done._id);
245
+ expect(childTask.background).toBe(true);
246
+ expect((await harness.waitForTask(childTask._id)).status).toBe("completed");
247
+ });
248
+
249
+ it("rejects children of unregistered definitions and waits on tasks it does not own", async () => {
250
+ const stranger = completesWith("test.stranger", null);
251
+ const unregistered = completesWith("test.unregistered", null);
252
+ const spawnsUnregistered = defineTask<unknown, unknown, unknown>({
253
+ initial: () => ({phase: "spawn"}),
254
+ name: "test.spawnsUnregistered",
255
+ phases: {
256
+ spawn: {
257
+ run: async (_task, rt) => {
258
+ await rt.createTask(unregistered, {});
259
+ },
260
+ },
261
+ },
262
+ retry: {backoffMs: 0, maxAttempts: 3},
263
+ version: 1,
264
+ });
265
+ let strangerId = "";
266
+ const waitsOnStranger = defineTask<unknown, unknown, unknown>({
267
+ initial: () => ({phase: "wait"}),
268
+ name: "test.waitsOnStranger",
269
+ phases: {
270
+ wait: {
271
+ run: async (_task, rt) => {
272
+ await rt.waitForTasks([strangerId]);
273
+ },
274
+ },
275
+ },
276
+ retry: {backoffMs: 0, maxAttempts: 3},
277
+ version: 1,
278
+ });
279
+ const harness = await openHarness([
280
+ asAny(spawnsUnregistered),
281
+ asAny(waitsOnStranger),
282
+ stranger,
283
+ ]);
284
+ strangerId = String((await harness.createTask(stranger, {}))._id);
285
+ await harness.start();
286
+
287
+ const spawned = await harness.waitForTask(
288
+ (await harness.createTask(spawnsUnregistered, {}))._id
289
+ );
290
+ expect(spawned.status).toBe("failed");
291
+ expect(spawned.attempt).toBe(0);
292
+ expect(spawned.outcome?.error).toBe("test.unregistered@1 is not in this harness registry");
293
+
294
+ const waited = await harness.waitForTask((await harness.createTask(waitsOnStranger, {}))._id);
295
+ expect(waited.status).toBe("failed");
296
+ expect(waited.outcome?.error).toBe(
297
+ "test.waitsOnStranger@1: rt.waitForTasks only waits on tasks this task created with rt.createTask"
298
+ );
299
+ });
300
+
301
+ it("refuses to create a child once the parent's lease is gone", async () => {
302
+ const child = completesWith("test.fencedChild", null);
303
+ let createError: unknown;
304
+ const settled = Promise.withResolvers<void>();
305
+ const parent = defineTask<unknown, unknown, unknown>({
306
+ initial: () => ({phase: "spawn"}),
307
+ name: "test.fencedParent",
308
+ phases: {
309
+ spawn: {
310
+ run: async (task, rt) => {
311
+ await TaskModel.updateOne({_id: task.id}, {$set: {"lease.token": "other-runner"}});
312
+ try {
313
+ await rt.createTask(child, {});
314
+ } catch (error: unknown) {
315
+ createError = error;
316
+ }
317
+ settled.resolve();
318
+ throw createError;
319
+ },
320
+ },
321
+ },
322
+ version: 1,
323
+ });
324
+ const harness = await openHarness([asAny(parent), child]);
325
+ await harness.start();
326
+ const created = await harness.createTask(parent, {});
327
+ await settled.promise;
328
+ await harness.stop();
329
+
330
+ expect(createError).toBeInstanceOf(HarnessCommitConflictError);
331
+ expect(await childrenOf(created._id)).toHaveLength(0);
332
+ expect(await SpanModel.countDocuments({name: "test.fencedChild@1"})).toBe(0);
333
+ });
334
+
335
+ it("writes no waiting state when the parent's lease is gone", async () => {
336
+ const child = completesWith("test.waitFencedChild", null);
337
+ let waitError: unknown;
338
+ const settled = Promise.withResolvers<void>();
339
+ const parent = defineTask<unknown, unknown, unknown>({
340
+ initial: () => ({phase: "spawn"}),
341
+ name: "test.waitFenced",
342
+ phases: {
343
+ spawn: {
344
+ run: async (task, rt) => {
345
+ const id = await rt.createTask(child, {});
346
+ await TaskModel.updateOne({_id: task.id}, {$set: {"lease.token": "other-runner"}});
347
+ try {
348
+ await rt.waitForTasks([id]);
349
+ } catch (error: unknown) {
350
+ waitError = error;
351
+ }
352
+ settled.resolve();
353
+ throw waitError;
354
+ },
355
+ },
356
+ },
357
+ version: 1,
358
+ });
359
+ const harness = await openHarness([asAny(parent), child]);
360
+ await harness.start();
361
+ const created = await harness.createTask(parent, {});
362
+ await settled.promise;
363
+ await harness.stop();
364
+
365
+ expect(waitError).toBeInstanceOf(HarnessCommitConflictError);
366
+ const task = await findTask(created._id);
367
+ expect(task.status).toBe("running");
368
+ expect(task.waiting?.kind).toBeUndefined();
369
+ expect(await SpanModel.countDocuments({name: "spawn"})).toBe(0);
370
+ });
371
+
372
+ it("wakes itself when a child settles between the check and the waiting commit", async () => {
373
+ const child = completesWith("test.racingChild", null);
374
+ const runs = {count: 0};
375
+ const parent = fanOut({children: [child], name: "test.racingParent", runs});
376
+ let hasRaced = false;
377
+ const harness = await openHarness([parent, child], {
378
+ beforeCommitEnd: async ({phase, taskId}) => {
379
+ if (phase !== "fanOut" || hasRaced) {
380
+ return;
381
+ }
382
+ hasRaced = true;
383
+ // Another process finishes the child while this parent's waiting commit is open.
384
+ await TaskModel.updateOne(
385
+ {"ownership.id": new mongoose.Types.ObjectId(taskId)},
386
+ {$set: {outcome: {result: "raced", status: "completed"}, status: "completed"}}
387
+ );
388
+ },
389
+ });
390
+ await harness.start();
391
+
392
+ // Shorter than the owner heartbeat, so only the post-commit re-check can wake it.
393
+ const done = await harness.waitForTask((await harness.createTask(parent, {}))._id, {
394
+ timeout: {seconds: 3},
395
+ });
396
+ expect(hasRaced).toBe(true);
397
+ expect(done.status).toBe("completed");
398
+ expect(runs.count).toBe(2);
399
+ });
400
+
401
+ // Retry backoff plus two child runs on a replica set can pass the 5 s default.
402
+ it("creates fresh children on a retry, and reuses them on a wake", async () => {
403
+ let childRuns = 0;
404
+ const flakyChild = defineTask<unknown, unknown, unknown>({
405
+ initial: () => ({phase: "work"}),
406
+ name: "test.flakyChild",
407
+ phases: {
408
+ work: {
409
+ run: async (_task, rt) => {
410
+ childRuns += 1;
411
+ if (childRuns === 1) {
412
+ throw new Error("first child failed");
413
+ }
414
+ await rt.commit({terminal: {result: "second child ok", status: "completed"}});
415
+ },
416
+ },
417
+ },
418
+ retry: {maxAttempts: 1},
419
+ version: 1,
420
+ });
421
+ const parent = defineTask<unknown, unknown, unknown>({
422
+ initial: () => ({phase: "fanOut"}),
423
+ name: "test.retryingParent",
424
+ phases: {
425
+ fanOut: {
426
+ run: async (_task, rt) => {
427
+ const [outcome] = await rt.waitForTasks([await rt.createTask(flakyChild, {})]);
428
+ if (outcome.status !== "completed") {
429
+ throw new Error(`child ${outcome.status}`);
430
+ }
431
+ await rt.commit({terminal: {result: outcome.result, status: "completed"}});
432
+ },
433
+ },
434
+ },
435
+ retry: {backoffMs: 0, maxAttempts: 2},
436
+ version: 1,
437
+ });
438
+ const harness = await openHarness([asAny(parent), asAny(flakyChild)]);
439
+ await harness.start();
440
+
441
+ const done = await harness.waitForTask((await harness.createTask(parent, {}))._id);
442
+ expect(done.status).toBe("completed");
443
+ expect(done.outcome?.result).toBe("second child ok");
444
+ const children = await childrenOf(done._id);
445
+ expect(children.map((task) => task.status)).toEqual(["failed", "completed"]);
446
+ }, 20_000);
447
+
448
+ it("failFast: the first failed child aborts its in-flight siblings (with their subtrees)", async () => {
449
+ const handlerOrder: string[] = [];
450
+ const grandchildRuns = {count: 0};
451
+ const grandchild = defineTask<unknown, unknown, unknown>({
452
+ abort: async () => {
453
+ handlerOrder.push("grandchild");
454
+ },
455
+ initial: () => ({phase: "work"}),
456
+ name: "test.ffGrandchild",
457
+ phases: {
458
+ work: {
459
+ run: async (_task, rt) => {
460
+ grandchildRuns.count += 1;
461
+ await rt.commit({terminal: {status: "completed"}});
462
+ },
463
+ },
464
+ },
465
+ version: 1,
466
+ });
467
+ // Waits on its own child, so it is in flight (waiting) when its sibling fails.
468
+ const waitingSibling = defineTask<unknown, unknown, unknown>({
469
+ abort: async (task) => {
470
+ const [owned] = await childrenOf(new mongoose.Types.ObjectId(task.id));
471
+ handlerOrder.push(`waitingSibling(grandchild ${owned.status})`);
472
+ },
473
+ initial: () => ({phase: "delegate"}),
474
+ name: "test.ffWaitingSibling",
475
+ phases: {
476
+ delegate: {
477
+ run: async (_task, rt) => {
478
+ const id = await rt.createTask(grandchild, {});
479
+ await rt.waitForTasks([id]);
480
+ await rt.commit({terminal: {status: "completed"}});
481
+ },
482
+ },
483
+ },
484
+ version: 1,
485
+ });
486
+ const pendingSibling = defineTask<unknown, unknown, unknown>({
487
+ abort: async () => {
488
+ handlerOrder.push("pendingSibling");
489
+ },
490
+ initial: () => ({phase: "work"}),
491
+ name: "test.ffPendingSibling",
492
+ phases: {work: {run: async (_task, rt) => rt.commit({terminal: {status: "completed"}})}},
493
+ version: 1,
494
+ });
495
+ const failing = failsWith("test.ffFailing", "allergy check failed");
496
+ const runs = {count: 0};
497
+ const parent = fanOut({
498
+ children: [asAny(waitingSibling), failing, asAny(pendingSibling)],
499
+ name: "test.failFast",
500
+ policy: "failFast",
501
+ runs,
502
+ });
503
+ const harness = await openHarness([
504
+ parent,
505
+ failing,
506
+ asAny(waitingSibling),
507
+ asAny(pendingSibling),
508
+ asAny(grandchild),
509
+ ]);
510
+ await harness.start();
511
+
512
+ const done = await harness.waitForTask((await harness.createTask(parent, {}))._id);
513
+ expect(done.status).toBe("completed");
514
+ const outcomes = done.outcome?.result as HarnessChildOutcome[];
515
+ expect(outcomes.map(({name, status}) => ({name, status}))).toEqual([
516
+ {name: "test.ffWaitingSibling", status: "aborted"},
517
+ {name: "test.ffFailing", status: "failed"},
518
+ {name: "test.ffPendingSibling", status: "aborted"},
519
+ ]);
520
+ expect(outcomes[0].error).toMatch(/^Aborted: Sibling task \S+ failed \(failFast wait of /);
521
+ expect(handlerOrder).toEqual([
522
+ "grandchild",
523
+ "waitingSibling(grandchild aborted)",
524
+ "pendingSibling",
525
+ ]);
526
+ expect(grandchildRuns.count).toBe(0);
527
+ const [waitingTask] = await childrenOf(done._id);
528
+ const [grandchildTask] = await childrenOf(waitingTask._id);
529
+ expect(grandchildTask.status).toBe("aborted");
530
+ });
531
+
532
+ it("wakes a waiting parent when an operator aborts its child directly", async () => {
533
+ const released = Promise.withResolvers<void>();
534
+ const parked = defineTask<unknown, unknown, unknown>({
535
+ initial: () => ({phase: "work"}),
536
+ name: "test.parkedChild",
537
+ phases: {
538
+ work: {
539
+ run: async (_task, rt) => {
540
+ await new Promise<void>((resolve) => {
541
+ rt.signal.addEventListener("abort", () => resolve());
542
+ released.promise.then(resolve);
543
+ });
544
+ throw new Error("stopped");
545
+ },
546
+ },
547
+ },
548
+ version: 1,
549
+ });
550
+ const runs = {count: 0};
551
+ const parent = fanOut({children: [asAny(parked)], name: "test.waitsOnParked", runs});
552
+ const harness = await openHarness([parent, asAny(parked)]);
553
+ await harness.start();
554
+
555
+ const created = await harness.createTask(parent, {});
556
+ await waitUntil(
557
+ async () => (await findTask(created._id)).status === "waiting",
558
+ "parent waiting"
559
+ );
560
+ const [child] = await childrenOf(created._id);
561
+ await waitUntil(
562
+ async () => (await findTask(child._id)).status === "running",
563
+ "child running"
564
+ );
565
+
566
+ const aborted = await harness.abort(child._id, {reason: "Patient left"});
567
+ expect(aborted.status).toBe("aborted");
568
+ released.resolve();
569
+
570
+ const done = await harness.waitForTask(created._id);
571
+ expect(done.status).toBe("completed");
572
+ const [outcome] = (done.outcome?.result ?? []) as HarnessChildOutcome[];
573
+ expect(outcome).toMatchObject({error: "Aborted: Patient left", status: "aborted"});
574
+ // The aborted run's late throw recorded nothing.
575
+ expect((await findTask(child._id)).status).toBe("aborted");
576
+ expect(await SpanModel.countDocuments({name: "work", traceId: done.traceId})).toBe(0);
577
+ });
578
+ });
579
+
580
+ describe("harness.abort", () => {
581
+ it("aborts three levels bottom-up, running each abort handler after its children are aborted", async () => {
582
+ const events: string[] = [];
583
+ const signals: AbortSignal[] = [];
584
+ const userId = new mongoose.Types.ObjectId();
585
+ const statusOf = async (id: mongoose.Types.ObjectId): Promise<string> => {
586
+ const owned = await childrenOf(id);
587
+ return owned.map((task) => task.status).join(",") || "none";
588
+ };
589
+
590
+ const grandchild = defineTask<unknown, unknown, unknown>({
591
+ abort: async (task, rt) => {
592
+ events.push(
593
+ `abort grandchild (phase signal ${signals[0]?.aborted}, by ${rt.userId}, ${rt.reason}, children ${await statusOf(new mongoose.Types.ObjectId(task.id))})`
594
+ );
595
+ },
596
+ initial: () => ({phase: "work"}),
597
+ name: "test.grandchild",
598
+ phases: {
599
+ work: {
600
+ run: async (_task, rt) => {
601
+ signals.push(rt.signal);
602
+ events.push("grandchild running");
603
+ await new Promise<void>((resolve) =>
604
+ rt.signal.addEventListener("abort", () => resolve())
605
+ );
606
+ events.push("grandchild saw abort signal");
607
+ // A late commit from an aborted run is fenced out.
608
+ await rt.commit({terminal: {result: "too late", status: "completed"}});
609
+ },
610
+ },
611
+ },
612
+ version: 1,
613
+ });
614
+ const child = defineTask<unknown, unknown, unknown>({
615
+ abort: async (task) => {
616
+ events.push(
617
+ `abort child (children ${await statusOf(new mongoose.Types.ObjectId(task.id))})`
618
+ );
619
+ },
620
+ initial: () => ({phase: "delegate"}),
621
+ name: "test.middle",
622
+ phases: {
623
+ delegate: {
624
+ run: async (_task, rt) => {
625
+ await rt.waitForTasks([await rt.createTask(grandchild, {})]);
626
+ await rt.commit({terminal: {status: "completed"}});
627
+ },
628
+ },
629
+ },
630
+ version: 1,
631
+ });
632
+ const root = defineTask<unknown, unknown, unknown>({
633
+ abort: async (task) => {
634
+ events.push(
635
+ `abort root (children ${await statusOf(new mongoose.Types.ObjectId(task.id))})`
636
+ );
637
+ },
638
+ initial: () => ({phase: "delegate"}),
639
+ name: "test.root",
640
+ phases: {
641
+ delegate: {
642
+ run: async (_task, rt) => {
643
+ await rt.waitForTasks([await rt.createTask(child, {})]);
644
+ await rt.commit({terminal: {status: "completed"}});
645
+ },
646
+ },
647
+ },
648
+ version: 1,
649
+ });
650
+ const harness = await openHarness([asAny(root), asAny(child), asAny(grandchild)]);
651
+ await harness.start();
652
+
653
+ const created = await harness.createTask(root, {});
654
+ await waitUntil(async () => events.includes("grandchild running"), "grandchild running");
655
+
656
+ const aborted = await harness.abort(created._id, {reason: "Duplicate intake", userId});
657
+ expect(aborted.status).toBe("aborted");
658
+ expect(aborted.toObject().outcome).toEqual({
659
+ error: "Aborted: Duplicate intake",
660
+ status: "aborted",
661
+ });
662
+
663
+ expect(events.filter((event) => event.startsWith("abort"))).toEqual([
664
+ `abort grandchild (phase signal true, by ${userId}, Duplicate intake, children none)`,
665
+ "abort child (children aborted)",
666
+ "abort root (children aborted)",
667
+ ]);
668
+ expect(signals[0]?.aborted).toBe(true);
669
+
670
+ const [middle] = await childrenOf(created._id);
671
+ const [leaf] = await childrenOf(middle._id);
672
+ for (const task of [middle, leaf]) {
673
+ const fresh = await findTask(task._id);
674
+ expect(fresh.status).toBe("aborted");
675
+ expect(fresh.lease?.token).toBeUndefined();
676
+ expect(fresh.waiting?.kind).toBeUndefined();
677
+ }
678
+
679
+ const abortSpans = await SpanModel.find({name: "abort", traceId: created.traceId}).sort({
680
+ _id: 1,
681
+ });
682
+ expect(abortSpans.map((span) => String(span.parentSpanId))).toEqual([
683
+ String(leaf.rootSpanId),
684
+ String(middle.rootSpanId),
685
+ String(created.rootSpanId),
686
+ ]);
687
+ for (const span of abortSpans) {
688
+ expect(span.status).toBe("ok");
689
+ expect(span.output).toEqual({
690
+ abortedBy: String(userId),
691
+ abortHandler: {status: "ok"},
692
+ reason: "Duplicate intake",
693
+ requestedOn: String(created._id),
694
+ });
695
+ }
696
+ const trace = await TraceModel.findExactlyOne({_id: created.traceId});
697
+ expect(trace.status).toBe("error");
698
+ expect(trace.errorSummary).toBe("Aborted: Duplicate intake");
699
+
700
+ // The runner is free again and the aborted grandchild's late commit wrote nothing.
701
+ await waitUntil(async () => events.includes("grandchild saw abort signal"), "signal seen");
702
+ const next = completesWith("test.afterAbort", "fine");
703
+ const nextHarness = await openHarness([next]);
704
+ await harness.stop();
705
+ await nextHarness.start();
706
+ expect(
707
+ (await nextHarness.waitForTask((await nextHarness.createTask(next, {}))._id)).status
708
+ ).toBe("completed");
709
+ expect((await findTask(leaf._id)).status).toBe("aborted");
710
+ expect(await SpanModel.countDocuments({name: "work", traceId: created.traceId})).toBe(0);
711
+ });
712
+
713
+ it("aborts a waiting parent without waking it", async () => {
714
+ const blocker = defineTask<unknown, unknown, unknown>({
715
+ initial: () => ({phase: "work"}),
716
+ name: "test.blocker",
717
+ phases: {
718
+ work: {
719
+ run: async (_task, rt) => {
720
+ await new Promise<void>((resolve) =>
721
+ rt.signal.addEventListener("abort", () => resolve())
722
+ );
723
+ throw new Error("stopped");
724
+ },
725
+ },
726
+ },
727
+ version: 1,
728
+ });
729
+ const runs = {count: 0};
730
+ const parent = fanOut({children: [asAny(blocker)], name: "test.waitingParent", runs});
731
+ const harness = await openHarness([parent, asAny(blocker)]);
732
+ await harness.start();
733
+
734
+ const created = await harness.createTask(parent, {});
735
+ await waitUntil(async () => (await findTask(created._id)).status === "waiting", "waiting");
736
+ const aborted = await harness.abort(created._id, {reason: "Cancelled by clinician"});
737
+ await pause(100);
738
+
739
+ expect(aborted.status).toBe("aborted");
740
+ expect(runs.count).toBe(1);
741
+ const [child] = await childrenOf(created._id);
742
+ expect((await findTask(child._id)).status).toBe("aborted");
743
+ expect((await findTask(created._id)).status).toBe("aborted");
744
+ });
745
+
746
+ it("records a failing abort handler on the span and aborts anyway", async () => {
747
+ const brittle = defineTask<unknown, unknown, unknown>({
748
+ abort: async () => {
749
+ throw new Error("EHR delete failed");
750
+ },
751
+ initial: () => ({phase: "work"}),
752
+ name: "test.brittleAbort",
753
+ phases: {work: {run: async (_task, rt) => rt.commit({terminal: {status: "completed"}})}},
754
+ version: 1,
755
+ });
756
+ const harness = await openHarness([asAny(brittle)]);
757
+ const created = await harness.createTask(brittle, {});
758
+
759
+ const aborted = await harness.abort(created._id, {reason: "Wrong patient"});
760
+ expect(aborted.status).toBe("aborted");
761
+ const span = await SpanModel.findExactlyOne({name: "abort", traceId: created.traceId});
762
+ expect(span.status).toBe("error");
763
+ expect(span.error).toBe("Abort handler failed: EHR delete failed");
764
+ expect(span.output).toMatchObject({
765
+ abortHandler: {error: "EHR delete failed", status: "error"},
766
+ });
767
+ });
768
+
769
+ it("runs each handler once when two aborts of the same tree overlap", async () => {
770
+ let handlerRuns = 0;
771
+ const slowHandler = defineTask<unknown, unknown, unknown>({
772
+ abort: async () => {
773
+ handlerRuns += 1;
774
+ await pause(100);
775
+ },
776
+ initial: () => ({phase: "work"}),
777
+ name: "test.slowHandler",
778
+ phases: {work: {run: async (_task, rt) => rt.commit({terminal: {status: "completed"}})}},
779
+ version: 1,
780
+ });
781
+ const harness = await openHarness([asAny(slowHandler)]);
782
+ const created = await harness.createTask(slowHandler, {});
783
+
784
+ const [first, second] = await Promise.all([
785
+ harness.abort(created._id, {reason: "first"}),
786
+ harness.abort(created._id, {reason: "second"}),
787
+ ]);
788
+ expect(handlerRuns).toBe(1);
789
+ expect(first.status).toBe("aborted");
790
+ expect(second.status).toBe("aborted");
791
+ expect(await SpanModel.countDocuments({name: "abort", traceId: created.traceId})).toBe(1);
792
+ // The first request recorded wins; the other waited for its handler to finish.
793
+ expect(["first", "second"]).toContain((await findTask(created._id)).abortRequested?.reason);
794
+ });
795
+
796
+ it("keeps aborting the tree when a child's handler throws", async () => {
797
+ const handled: string[] = [];
798
+ const brittleChild = defineTask<unknown, unknown, unknown>({
799
+ abort: async () => {
800
+ handled.push("child");
801
+ throw new Error("could not cancel lab order");
802
+ },
803
+ initial: () => ({phase: "work"}),
804
+ name: "test.brittleChild",
805
+ phases: {
806
+ work: {
807
+ run: async (_task, rt) => {
808
+ await new Promise<void>((resolve) =>
809
+ rt.signal.addEventListener("abort", () => resolve())
810
+ );
811
+ throw new Error("stopped");
812
+ },
813
+ },
814
+ },
815
+ version: 1,
816
+ });
817
+ const parent = defineTask<unknown, unknown, unknown>({
818
+ abort: async () => {
819
+ handled.push("parent");
820
+ },
821
+ initial: () => ({phase: "fanOut"}),
822
+ name: "test.brittleParent",
823
+ phases: {
824
+ fanOut: {
825
+ run: async (_task, rt) => {
826
+ await rt.waitForTasks([await rt.createTask(brittleChild, {})]);
827
+ },
828
+ },
829
+ },
830
+ version: 1,
831
+ });
832
+ const harness = await openHarness([asAny(parent), asAny(brittleChild)]);
833
+ await harness.start();
834
+ const created = await harness.createTask(parent, {});
835
+ await waitUntil(async () => (await findTask(created._id)).status === "waiting", "waiting");
836
+
837
+ const aborted = await harness.abort(created._id, {reason: "Visit cancelled"});
838
+ expect(aborted.status).toBe("aborted");
839
+ expect(handled).toEqual(["child", "parent"]);
840
+ const [child] = await childrenOf(created._id);
841
+ expect((await findTask(child._id)).status).toBe("aborted");
842
+ const spans = await SpanModel.find({name: "abort", traceId: created.traceId}).sort({_id: 1});
843
+ expect(spans.map((span) => span.status)).toEqual(["error", "ok"]);
844
+ });
845
+
846
+ it("rejects a blank reason and an already-terminal task", async () => {
847
+ const quick = completesWith("test.quick", 1);
848
+ const harness = await openHarness([quick]);
849
+ await harness.start();
850
+ const done = await harness.waitForTask((await harness.createTask(quick, {}))._id);
851
+
852
+ await expect(harness.abort(done._id, {reason: " "})).rejects.toThrow(
853
+ harnessErrorMatching("invalidRequest", "abort requires a reason")
854
+ );
855
+ await expect(harness.abort(done._id, {reason: "late"})).rejects.toThrow(
856
+ harnessErrorMatching("taskTerminal", `Task ${done._id} is already completed`)
857
+ );
858
+ expect(await SpanModel.countDocuments({name: "abort"})).toBe(0);
859
+ });
860
+
861
+ it("records an unregistered version's abort as handler-less", async () => {
862
+ const v1 = completesWith("test.versioned", 1);
863
+ const writer = await openHarness([v1]);
864
+ const created = await writer.createTask(v1, {});
865
+ const other = await openHarness([completesWith("test.other", 1)]);
866
+
867
+ const aborted = await other.abort(created._id, {reason: "Retired workflow"});
868
+ expect(aborted.status).toBe("aborted");
869
+ const span = await SpanModel.findExactlyOne({name: "abort", traceId: created.traceId});
870
+ expect(span.output).toMatchObject({
871
+ abortHandler: {
872
+ error: "test.versioned@1 is not in this harness registry; no abort handler ran",
873
+ status: "unregistered",
874
+ },
875
+ });
876
+ expect(span.status).toBe("ok");
877
+ });
878
+
879
+ it("runs the abort handler when resolveInterrupted chooses abort", async () => {
880
+ const handled: string[] = [];
881
+ const write = defineTask<unknown, unknown, unknown>({
882
+ abort: async (_task, rt) => {
883
+ handled.push(rt.reason);
884
+ },
885
+ initial: () => ({phase: "write"}),
886
+ name: "test.interruptedWrite",
887
+ phases: {write: {run: async (_task, rt) => rt.commit({terminal: {status: "completed"}})}},
888
+ version: 1,
889
+ });
890
+ const harness = await openHarness([asAny(write)]);
891
+ const created = await harness.createTask(write, {});
892
+ await TaskModel.updateOne({_id: created._id}, {$set: {status: "interrupted"}});
893
+
894
+ const resolved = await harness.resolveInterrupted(created._id, {
895
+ action: "abort",
896
+ reason: "Note already filed by hand",
897
+ });
898
+ expect(resolved.status).toBe("aborted");
899
+ expect(handled).toEqual(["Note already filed by hand"]);
900
+ const span = await SpanModel.findExactlyOne({
901
+ name: "resolveInterrupted",
902
+ traceId: created.traceId,
903
+ });
904
+ expect(span.output).toEqual({
905
+ abortHandler: {status: "ok"},
906
+ action: "abort",
907
+ phase: "write",
908
+ reason: "Note already filed by hand",
909
+ });
910
+ });
911
+ });
912
+
913
+ describe("resolveInterrupted on an aborting task", () => {
914
+ it("refuses retry once an abort has started", async () => {
915
+ const write = completesWith("test.abortingWrite", null);
916
+ const harness = await openHarness([write]);
917
+ const created = await harness.createTask(write, {});
918
+ await TaskModel.updateOne(
919
+ {_id: created._id},
920
+ {
921
+ $set: {
922
+ abortRequested: {at: new Date(), reason: "crashed mid-abort"},
923
+ status: "interrupted",
924
+ },
925
+ }
926
+ );
927
+
928
+ await expect(
929
+ harness.resolveInterrupted(created._id, {action: "retry", reason: "try again"})
930
+ ).rejects.toThrow(
931
+ harnessErrorMatching("taskAborting", "is being aborted; resolve it with abort, not retry")
932
+ );
933
+ const resolved = await harness.resolveInterrupted(created._id, {
934
+ action: "abort",
935
+ reason: "finish the abort",
936
+ });
937
+ expect(resolved.status).toBe("aborted");
938
+ });
939
+ });
940
+
941
+ describe("waiting sweep", () => {
942
+ const buildEngine = (
943
+ definitions: ReadonlyArray<AnyHarnessTaskDefinition> = []
944
+ ): {engine: HarnessEngine; wakes: {count: number}} => {
945
+ const wakes = {count: 0};
946
+ const models = {
947
+ event: registerHarnessEvent(),
948
+ eventStream: registerHarnessEventStream(),
949
+ owner: OwnerModel,
950
+ span: SpanModel,
951
+ task: TaskModel,
952
+ trace: TraceModel,
953
+ } as unknown as HarnessModels;
954
+ return {
955
+ engine: {
956
+ controllers: new Map(),
957
+ definitions: new Map(definitions.map((definition) => [definition.key, definition])),
958
+ models,
959
+ wake: () => {
960
+ wakes.count += 1;
961
+ },
962
+ },
963
+ wakes,
964
+ };
965
+ };
966
+
967
+ type SeedStatus = "completed" | "failed" | "pending" | "running";
968
+
969
+ /**
970
+ * A parent `waiting` on children in `childStatuses`, written straight to Mongo with
971
+ * the trace and spans the commit paths expect.
972
+ */
973
+ const seedWait = async ({
974
+ childStatuses,
975
+ name = "test.seeded",
976
+ policy,
977
+ }: {
978
+ childStatuses: SeedStatus[];
979
+ name?: string;
980
+ policy: "all" | "failFast";
981
+ }): Promise<{childIds: mongoose.Types.ObjectId[]; parentId: mongoose.Types.ObjectId}> => {
982
+ const parentId = new mongoose.Types.ObjectId();
983
+ const traceId = new mongoose.Types.ObjectId();
984
+ const startedAt = new Date();
985
+ await TraceModel.create({_id: traceId, name, startedAt, status: "ok"});
986
+ const seedSpan = async (): Promise<mongoose.Types.ObjectId> => {
987
+ const [span] = await SpanModel.create([
988
+ {kind: "CHAIN", name, startedAt, startOffsetMs: 0, status: "ok", traceId},
989
+ ]);
990
+ return span._id;
991
+ };
992
+ const childIds = childStatuses.map(() => new mongoose.Types.ObjectId());
993
+ const common = {name, phase: "fanOut", rootTaskId: parentId, traceId, version: 1};
994
+ await TaskModel.create({
995
+ ...common,
996
+ _id: parentId,
997
+ rootSpanId: await seedSpan(),
998
+ status: "waiting",
999
+ waiting: {kind: "tasks", policy, taskIds: childIds},
1000
+ });
1001
+ for (const [index, status] of childStatuses.entries()) {
1002
+ await TaskModel.create({
1003
+ ...common,
1004
+ _id: childIds[index],
1005
+ name: "test.seededChild",
1006
+ outcome:
1007
+ status === "completed" || status === "failed"
1008
+ ? {...(status === "failed" ? {error: "seeded failure"} : {result: index}), status}
1009
+ : undefined,
1010
+ ownership: {id: parentId, kind: "task"},
1011
+ phase: "work",
1012
+ // Matches what rt.createTask would name child `index` of the parent's first visit.
1013
+ requestId: `harness-child:${parentId}:0:0:${index}`,
1014
+ rootSpanId: await seedSpan(),
1015
+ status,
1016
+ });
1017
+ }
1018
+ return {childIds, parentId};
1019
+ };
1020
+
1021
+ it("wakes a parent whose children settled without waking it (a lost wake)", async () => {
1022
+ const {engine, wakes} = buildEngine();
1023
+ const {parentId} = await seedWait({childStatuses: ["completed"], policy: "all"});
1024
+ const stillWaiting = await seedWait({childStatuses: ["completed", "pending"], policy: "all"});
1025
+
1026
+ expect(await sweepWaitingTasks(engine)).toBe(1);
1027
+ expect(wakes.count).toBe(1);
1028
+ const woken = await findTask(parentId);
1029
+ expect(woken.status).toBe("pending");
1030
+ expect(woken.waiting?.kind).toBeUndefined();
1031
+ expect((await findTask(stillWaiting.parentId)).status).toBe("waiting");
1032
+ expect(await sweepWaitingTasks(engine)).toBe(0);
1033
+ });
1034
+
1035
+ it("failFast: a failed child wakes the parent and aborts the siblings still in flight", async () => {
1036
+ const {engine} = buildEngine();
1037
+ const failFast = await seedWait({
1038
+ childStatuses: ["failed", "pending", "running"],
1039
+ policy: "failFast",
1040
+ });
1041
+ const all = await seedWait({childStatuses: ["failed", "pending"], policy: "all"});
1042
+
1043
+ expect(await sweepWaitingTasks(engine)).toBe(1);
1044
+ expect((await findTask(failFast.parentId)).status).toBe("pending");
1045
+ const [, pending, running] = await Promise.all(failFast.childIds.map(findTask));
1046
+ expect(pending.status).toBe("aborted");
1047
+ expect(running.status).toBe("aborted");
1048
+ expect(pending.outcome?.error).toMatch(/^Aborted: Sibling task \S+ failed \(failFast/);
1049
+ expect((await findTask(all.parentId)).status).toBe("waiting");
1050
+ expect((await findTask(all.childIds[1])).status).toBe("pending");
1051
+ });
1052
+
1053
+ it("tolerates two failed failFast siblings and repeated sweeps", async () => {
1054
+ const {engine} = buildEngine();
1055
+ const {childIds, parentId} = await seedWait({
1056
+ childStatuses: ["failed", "failed", "running"],
1057
+ policy: "failFast",
1058
+ });
1059
+
1060
+ const results = await Promise.all([sweepWaitingTasks(engine), sweepWaitingTasks(engine)]);
1061
+ expect(results.reduce((sum, woken) => sum + woken, 0)).toBe(1);
1062
+ expect((await findTask(parentId)).status).toBe("pending");
1063
+ expect((await findTask(childIds[2])).status).toBe("aborted");
1064
+ expect(await SpanModel.countDocuments({name: "abort"})).toBe(1);
1065
+ });
1066
+
1067
+ it("the running harness sweeps lost wakes and finishes the parent", async () => {
1068
+ const child = completesWith("test.seededChild", null);
1069
+ const runs = {count: 0};
1070
+ const parent = fanOut({children: [child], name: "test.seededParent", runs});
1071
+ const {parentId} = await seedWait({
1072
+ childStatuses: ["completed"],
1073
+ name: "test.seededParent",
1074
+ policy: "all",
1075
+ });
1076
+ const harness = await openHarness([parent, child]);
1077
+ await harness.start();
1078
+
1079
+ const done = await harness.waitForTask(parentId, {timeout: {seconds: 5}});
1080
+ expect(done.status).toBe("completed");
1081
+ expect(runs.count).toBe(1);
1082
+ expect(done.outcome?.result).toEqual([
1083
+ {id: expect.any(String), name: "test.seededChild", result: 0, status: "completed"},
1084
+ ]);
1085
+ });
1086
+ });
1087
+ });