@terreno/ai 57.9.0 → 57.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (412) hide show
  1. package/README.md +1 -0
  2. package/bunfig.toml +1 -1
  3. package/dist/aiApp.d.ts +8 -2
  4. package/dist/aiApp.js +12 -6
  5. package/dist/harness/adminScreens.d.ts +6 -0
  6. package/dist/harness/adminScreens.js +11 -0
  7. package/dist/harness/agentLoop.d.ts +61 -0
  8. package/dist/harness/agentLoop.js +798 -0
  9. package/dist/harness/agentTaskNames.d.ts +4 -0
  10. package/dist/harness/agentTaskNames.js +7 -0
  11. package/dist/harness/approvalPolicy.d.ts +3 -0
  12. package/dist/harness/approvalPolicy.js +23 -0
  13. package/dist/harness/approvals.d.ts +101 -0
  14. package/dist/harness/approvals.js +468 -0
  15. package/dist/harness/commit.d.ts +238 -0
  16. package/dist/harness/commit.js +678 -0
  17. package/dist/harness/conversation.d.ts +75 -0
  18. package/dist/harness/conversation.js +397 -0
  19. package/dist/harness/defineAgent.d.ts +9 -0
  20. package/dist/harness/defineAgent.js +65 -0
  21. package/dist/harness/defineTask.d.ts +11 -0
  22. package/dist/harness/defineTask.js +59 -0
  23. package/dist/harness/defineTool.d.ts +7 -0
  24. package/dist/harness/defineTool.js +36 -0
  25. package/dist/harness/definitionError.d.ts +8 -0
  26. package/dist/harness/definitionError.js +15 -0
  27. package/dist/harness/errors.d.ts +127 -0
  28. package/dist/harness/errors.js +101 -0
  29. package/dist/harness/eventHub.d.ts +45 -0
  30. package/dist/harness/eventHub.js +135 -0
  31. package/dist/harness/events.d.ts +95 -0
  32. package/dist/harness/events.js +267 -0
  33. package/dist/harness/executionEnv.d.ts +31 -0
  34. package/dist/harness/executionEnv.js +2 -0
  35. package/dist/harness/extensions.d.ts +82 -0
  36. package/dist/harness/extensions.js +274 -0
  37. package/dist/harness/harness.d.ts +163 -0
  38. package/dist/harness/harness.js +520 -0
  39. package/dist/harness/harnessApp.d.ts +44 -0
  40. package/dist/harness/harnessApp.js +62 -0
  41. package/dist/harness/internalRuntime.d.ts +16 -0
  42. package/dist/harness/internalRuntime.js +18 -0
  43. package/dist/harness/leases.d.ts +35 -0
  44. package/dist/harness/leases.js +86 -0
  45. package/dist/harness/memo.d.ts +24 -0
  46. package/dist/harness/memo.js +78 -0
  47. package/dist/harness/modelCall.d.ts +45 -0
  48. package/dist/harness/modelCall.js +147 -0
  49. package/dist/harness/models/harnessApproval.d.ts +2 -0
  50. package/dist/harness/models/harnessApproval.js +100 -0
  51. package/dist/harness/models/harnessConversation.d.ts +2 -0
  52. package/dist/harness/models/harnessConversation.js +117 -0
  53. package/dist/harness/models/harnessEvent.d.ts +3 -0
  54. package/dist/harness/models/harnessEvent.js +73 -0
  55. package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
  56. package/dist/harness/models/harnessInboxEvent.js +58 -0
  57. package/dist/harness/models/harnessMemo.d.ts +2 -0
  58. package/dist/harness/models/harnessMemo.js +42 -0
  59. package/dist/harness/models/harnessMessage.d.ts +2 -0
  60. package/dist/harness/models/harnessMessage.js +73 -0
  61. package/dist/harness/models/harnessOwner.d.ts +2 -0
  62. package/dist/harness/models/harnessOwner.js +36 -0
  63. package/dist/harness/models/harnessTask.d.ts +2 -0
  64. package/dist/harness/models/harnessTask.js +186 -0
  65. package/dist/harness/ownership.d.ts +89 -0
  66. package/dist/harness/ownership.js +438 -0
  67. package/dist/harness/registry.d.ts +29 -0
  68. package/dist/harness/registry.js +93 -0
  69. package/dist/harness/retryBackoff.d.ts +31 -0
  70. package/dist/harness/retryBackoff.js +50 -0
  71. package/dist/harness/routes/approvals.d.ts +13 -0
  72. package/dist/harness/routes/approvals.js +93 -0
  73. package/dist/harness/routes/conversations.d.ts +11 -0
  74. package/dist/harness/routes/conversations.js +66 -0
  75. package/dist/harness/routes/events.d.ts +19 -0
  76. package/dist/harness/routes/events.js +215 -0
  77. package/dist/harness/routes/tasks.d.ts +11 -0
  78. package/dist/harness/routes/tasks.js +108 -0
  79. package/dist/harness/runners/inProcessRunner.d.ts +57 -0
  80. package/dist/harness/runners/inProcessRunner.js +196 -0
  81. package/dist/harness/runtime.d.ts +34 -0
  82. package/dist/harness/runtime.js +551 -0
  83. package/dist/harness/subagent.d.ts +53 -0
  84. package/dist/harness/subagent.js +134 -0
  85. package/dist/harness/suspend.d.ts +8 -0
  86. package/dist/harness/suspend.js +16 -0
  87. package/dist/harness/transaction.d.ts +3 -0
  88. package/dist/harness/transaction.js +22 -0
  89. package/dist/harness/waits.d.ts +100 -0
  90. package/dist/harness/waits.js +214 -0
  91. package/dist/index.d.ts +19 -1
  92. package/dist/index.js +45 -2
  93. package/dist/models/aiDataset.d.ts +2 -0
  94. package/dist/models/aiDataset.js +67 -0
  95. package/dist/models/aiRequest.js +1 -1
  96. package/dist/models/gptHistory.js +73 -0
  97. package/dist/observability/adminScreens.d.ts +5 -0
  98. package/dist/observability/adminScreens.js +32 -0
  99. package/dist/observability/compileTemplate.d.ts +1 -0
  100. package/dist/observability/compileTemplate.js +13 -0
  101. package/dist/observability/datasetImport.d.ts +11 -0
  102. package/dist/observability/datasetImport.js +185 -0
  103. package/dist/observability/evaluate.d.ts +31 -0
  104. package/dist/observability/evaluate.js +200 -0
  105. package/dist/observability/evaluatorTemplates.d.ts +17 -0
  106. package/dist/observability/evaluatorTemplates.js +111 -0
  107. package/dist/observability/jsonPath.d.ts +2 -0
  108. package/dist/observability/jsonPath.js +57 -0
  109. package/dist/observability/local/datasetStore.d.ts +87 -0
  110. package/dist/observability/local/datasetStore.js +326 -0
  111. package/dist/observability/local/evaluatorStore.d.ts +44 -0
  112. package/dist/observability/local/evaluatorStore.js +182 -0
  113. package/dist/observability/local/experimentRunner.d.ts +96 -0
  114. package/dist/observability/local/experimentRunner.js +419 -0
  115. package/dist/observability/local/localPlugin.d.ts +22 -0
  116. package/dist/observability/local/localPlugin.js +76 -0
  117. package/dist/observability/local/models/obsDataset.d.ts +2 -0
  118. package/dist/observability/local/models/obsDataset.js +37 -0
  119. package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
  120. package/dist/observability/local/models/obsDatasetItem.js +76 -0
  121. package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
  122. package/dist/observability/local/models/obsEvaluator.js +98 -0
  123. package/dist/observability/local/models/obsExperiment.d.ts +2 -0
  124. package/dist/observability/local/models/obsExperiment.js +87 -0
  125. package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
  126. package/dist/observability/local/models/obsExperimentItem.js +45 -0
  127. package/dist/observability/local/models/obsPrompt.d.ts +2 -0
  128. package/dist/observability/local/models/obsPrompt.js +42 -0
  129. package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
  130. package/dist/observability/local/models/obsPromptLabel.js +39 -0
  131. package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
  132. package/dist/observability/local/models/obsPromptVersion.js +96 -0
  133. package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
  134. package/dist/observability/local/models/obsReviewItem.js +75 -0
  135. package/dist/observability/local/models/obsScore.d.ts +2 -0
  136. package/dist/observability/local/models/obsScore.js +58 -0
  137. package/dist/observability/local/models/obsSpan.d.ts +2 -0
  138. package/dist/observability/local/models/obsSpan.js +64 -0
  139. package/dist/observability/local/models/obsTrace.d.ts +2 -0
  140. package/dist/observability/local/models/obsTrace.js +93 -0
  141. package/dist/observability/local/promptStore.d.ts +180 -0
  142. package/dist/observability/local/promptStore.js +444 -0
  143. package/dist/observability/local/reviewStore.d.ts +50 -0
  144. package/dist/observability/local/reviewStore.js +203 -0
  145. package/dist/observability/local/traceStore.d.ts +103 -0
  146. package/dist/observability/local/traceStore.js +325 -0
  147. package/dist/observability/observabilityApp.d.ts +24 -0
  148. package/dist/observability/observabilityApp.js +157 -0
  149. package/dist/observability/observabilityAppRegistry.d.ts +14 -0
  150. package/dist/observability/observabilityAppRegistry.js +16 -0
  151. package/dist/observability/permissions.d.ts +7 -0
  152. package/dist/observability/permissions.js +60 -0
  153. package/dist/observability/reviewPanels.d.ts +18 -0
  154. package/dist/observability/reviewPanels.js +46 -0
  155. package/dist/observability/routes/datasets.d.ts +7 -0
  156. package/dist/observability/routes/datasets.js +149 -0
  157. package/dist/observability/routes/evaluators.d.ts +7 -0
  158. package/dist/observability/routes/evaluators.js +123 -0
  159. package/dist/observability/routes/experiments.d.ts +7 -0
  160. package/dist/observability/routes/experiments.js +95 -0
  161. package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
  162. package/dist/observability/routes/observabilityRouteAccess.js +67 -0
  163. package/dist/observability/routes/prompts.d.ts +12 -0
  164. package/dist/observability/routes/prompts.js +172 -0
  165. package/dist/observability/routes/requestAiService.d.ts +13 -0
  166. package/dist/observability/routes/requestAiService.js +21 -0
  167. package/dist/observability/routes/review.d.ts +7 -0
  168. package/dist/observability/routes/review.js +79 -0
  169. package/dist/observability/routes/status.d.ts +5 -0
  170. package/dist/observability/routes/status.js +31 -0
  171. package/dist/observability/routes/testMultiStage.d.ts +9 -0
  172. package/dist/observability/routes/testMultiStage.js +67 -0
  173. package/dist/observability/routes/traces.d.ts +7 -0
  174. package/dist/observability/routes/traces.js +129 -0
  175. package/dist/observability/schemaValidation.d.ts +5 -0
  176. package/dist/observability/schemaValidation.js +26 -0
  177. package/dist/observability/sopThresholds.d.ts +2 -0
  178. package/dist/observability/sopThresholds.js +21 -0
  179. package/dist/observability/status.d.ts +23 -0
  180. package/dist/observability/status.js +34 -0
  181. package/dist/observability/testMultiStageSchemas.d.ts +10 -0
  182. package/dist/observability/testMultiStageSchemas.js +79 -0
  183. package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
  184. package/dist/observability/testMultiStageWorkflow.js +218 -0
  185. package/dist/observability/types.d.ts +159 -0
  186. package/dist/observability/types.js +67 -0
  187. package/dist/routes/files.js +18 -7
  188. package/dist/routes/gpt.js +79 -689
  189. package/dist/routes/gptActions.d.ts +9 -0
  190. package/dist/routes/gptActions.js +212 -0
  191. package/dist/routes/gptDatasets.d.ts +5 -0
  192. package/dist/routes/gptDatasets.js +94 -0
  193. package/dist/routes/gptHistories.d.ts +3 -1
  194. package/dist/routes/gptHistories.js +188 -2
  195. package/dist/service/aiDatasets.d.ts +57 -0
  196. package/dist/service/aiDatasets.js +253 -0
  197. package/dist/service/aiService.d.ts +50 -1
  198. package/dist/service/aiService.js +426 -77
  199. package/dist/service/askFiles.d.ts +49 -0
  200. package/dist/service/askFiles.js +162 -0
  201. package/dist/service/asks.d.ts +76 -0
  202. package/dist/service/asks.js +284 -0
  203. package/dist/service/chatTurn.d.ts +128 -0
  204. package/dist/service/chatTurn.js +1674 -0
  205. package/dist/service/fileStorage.d.ts +4 -0
  206. package/dist/service/fileStorage.js +7 -2
  207. package/dist/service/fillActionIds.d.ts +6 -0
  208. package/dist/service/fillActionIds.js +79 -0
  209. package/dist/service/jsonFenceModel.d.ts +7 -0
  210. package/dist/service/jsonFenceModel.js +45 -0
  211. package/dist/service/prompts.d.ts +52 -0
  212. package/dist/service/prompts.js +96 -1
  213. package/dist/service/sanitizeHtml.d.ts +11 -0
  214. package/dist/service/sanitizeHtml.js +108 -0
  215. package/dist/types/harness.d.ts +934 -0
  216. package/dist/types/harness.js +133 -0
  217. package/dist/types/index.d.ts +192 -14
  218. package/dist/types/index.js +2 -0
  219. package/dist/types/observability.d.ts +288 -0
  220. package/dist/types/observability.js +2 -0
  221. package/package.json +17 -5
  222. package/src/aiApp.test.ts +207 -1
  223. package/src/aiApp.ts +19 -6
  224. package/src/harness/adminScreens.test.ts +34 -0
  225. package/src/harness/adminScreens.ts +11 -0
  226. package/src/harness/agentLoop.test.ts +1034 -0
  227. package/src/harness/agentLoop.ts +1082 -0
  228. package/src/harness/agentRecovery.test.ts +380 -0
  229. package/src/harness/agentTaskNames.ts +4 -0
  230. package/src/harness/approvalPolicy.ts +28 -0
  231. package/src/harness/approvals.test.ts +1094 -0
  232. package/src/harness/approvals.ts +644 -0
  233. package/src/harness/commit.ts +1049 -0
  234. package/src/harness/conversation.ts +503 -0
  235. package/src/harness/defineAgent.test.ts +92 -0
  236. package/src/harness/defineAgent.ts +82 -0
  237. package/src/harness/defineTask.test.ts +59 -0
  238. package/src/harness/defineTask.ts +76 -0
  239. package/src/harness/defineTool.ts +45 -0
  240. package/src/harness/definitionError.ts +13 -0
  241. package/src/harness/errors.test.ts +90 -0
  242. package/src/harness/errors.ts +119 -0
  243. package/src/harness/eventHub.test.ts +124 -0
  244. package/src/harness/eventHub.ts +159 -0
  245. package/src/harness/events.test.ts +757 -0
  246. package/src/harness/events.ts +384 -0
  247. package/src/harness/executionEnv.ts +34 -0
  248. package/src/harness/extensions.test.ts +1218 -0
  249. package/src/harness/extensions.ts +382 -0
  250. package/src/harness/harness.test.ts +693 -0
  251. package/src/harness/harness.ts +767 -0
  252. package/src/harness/harnessApp.ts +76 -0
  253. package/src/harness/internalRuntime.ts +44 -0
  254. package/src/harness/leases.ts +132 -0
  255. package/src/harness/memo.ts +107 -0
  256. package/src/harness/modelCall.test.ts +134 -0
  257. package/src/harness/modelCall.ts +200 -0
  258. package/src/harness/models/harnessApproval.ts +109 -0
  259. package/src/harness/models/harnessConversation.ts +142 -0
  260. package/src/harness/models/harnessEvent.ts +92 -0
  261. package/src/harness/models/harnessInboxEvent.ts +73 -0
  262. package/src/harness/models/harnessMemo.ts +43 -0
  263. package/src/harness/models/harnessMessage.ts +85 -0
  264. package/src/harness/models/harnessModels.test.ts +60 -0
  265. package/src/harness/models/harnessOwner.ts +40 -0
  266. package/src/harness/models/harnessTask.ts +201 -0
  267. package/src/harness/ownership.test.ts +1087 -0
  268. package/src/harness/ownership.ts +618 -0
  269. package/src/harness/recovery.test.ts +912 -0
  270. package/src/harness/registry.ts +124 -0
  271. package/src/harness/retries.test.ts +292 -0
  272. package/src/harness/retryBackoff.test.ts +100 -0
  273. package/src/harness/retryBackoff.ts +77 -0
  274. package/src/harness/routes/approvals.ts +123 -0
  275. package/src/harness/routes/conversations.test.ts +518 -0
  276. package/src/harness/routes/conversations.ts +102 -0
  277. package/src/harness/routes/events.ts +280 -0
  278. package/src/harness/routes/tasks.test.ts +198 -0
  279. package/src/harness/routes/tasks.ts +146 -0
  280. package/src/harness/runners/inProcessRunner.test.ts +402 -0
  281. package/src/harness/runners/inProcessRunner.ts +220 -0
  282. package/src/harness/runtime.ts +714 -0
  283. package/src/harness/subagent.test.ts +696 -0
  284. package/src/harness/subagent.ts +192 -0
  285. package/src/harness/suspend.ts +13 -0
  286. package/src/harness/transaction.ts +17 -0
  287. package/src/harness/versioning.test.ts +288 -0
  288. package/src/harness/waits.test.ts +693 -0
  289. package/src/harness/waits.ts +356 -0
  290. package/src/index.test.ts +13 -0
  291. package/src/index.ts +70 -0
  292. package/src/isolated/adminScreensConfig.isolated.ts +143 -0
  293. package/src/models/aiDataset.test.ts +34 -0
  294. package/src/models/aiDataset.ts +78 -0
  295. package/src/models/aiRequest.ts +2 -1
  296. package/src/models/gptHistory.test.ts +156 -0
  297. package/src/models/gptHistory.ts +89 -1
  298. package/src/observability/adminScreens.test.ts +71 -0
  299. package/src/observability/adminScreens.ts +42 -0
  300. package/src/observability/compileTemplate.test.ts +14 -0
  301. package/src/observability/compileTemplate.ts +10 -0
  302. package/src/observability/datasetImport.test.ts +95 -0
  303. package/src/observability/datasetImport.ts +220 -0
  304. package/src/observability/evaluate.test.ts +549 -0
  305. package/src/observability/evaluate.ts +254 -0
  306. package/src/observability/evaluatorTemplates.ts +124 -0
  307. package/src/observability/jsonPath.test.ts +52 -0
  308. package/src/observability/jsonPath.ts +60 -0
  309. package/src/observability/local/datasetStore.test.ts +230 -0
  310. package/src/observability/local/datasetStore.ts +452 -0
  311. package/src/observability/local/evaluatorStore.test.ts +138 -0
  312. package/src/observability/local/evaluatorStore.ts +224 -0
  313. package/src/observability/local/experimentRunner.test.ts +489 -0
  314. package/src/observability/local/experimentRunner.ts +548 -0
  315. package/src/observability/local/localPlugin.ts +88 -0
  316. package/src/observability/local/models/obsDataset.ts +38 -0
  317. package/src/observability/local/models/obsDatasetItem.ts +80 -0
  318. package/src/observability/local/models/obsEvaluator.ts +102 -0
  319. package/src/observability/local/models/obsExperiment.ts +91 -0
  320. package/src/observability/local/models/obsExperimentItem.ts +52 -0
  321. package/src/observability/local/models/obsPrompt.ts +43 -0
  322. package/src/observability/local/models/obsPromptLabel.ts +43 -0
  323. package/src/observability/local/models/obsPromptVersion.ts +107 -0
  324. package/src/observability/local/models/obsReviewItem.ts +79 -0
  325. package/src/observability/local/models/obsScore.ts +59 -0
  326. package/src/observability/local/models/obsSpan.ts +65 -0
  327. package/src/observability/local/models/obsTrace.ts +94 -0
  328. package/src/observability/local/obsModels.test.ts +119 -0
  329. package/src/observability/local/promptStore.test.ts +351 -0
  330. package/src/observability/local/promptStore.ts +665 -0
  331. package/src/observability/local/reviewStore.test.ts +164 -0
  332. package/src/observability/local/reviewStore.ts +264 -0
  333. package/src/observability/local/traceStore.test.ts +264 -0
  334. package/src/observability/local/traceStore.ts +440 -0
  335. package/src/observability/observabilityApp.test.ts +92 -0
  336. package/src/observability/observabilityApp.ts +185 -0
  337. package/src/observability/observabilityAppRegistry.test.ts +30 -0
  338. package/src/observability/observabilityAppRegistry.ts +35 -0
  339. package/src/observability/permissions.test.ts +96 -0
  340. package/src/observability/permissions.ts +78 -0
  341. package/src/observability/reviewPanels.test.ts +35 -0
  342. package/src/observability/reviewPanels.ts +64 -0
  343. package/src/observability/routes/datasets.test.ts +176 -0
  344. package/src/observability/routes/datasets.ts +252 -0
  345. package/src/observability/routes/evaluators.test.ts +119 -0
  346. package/src/observability/routes/evaluators.ts +207 -0
  347. package/src/observability/routes/experiments.test.ts +282 -0
  348. package/src/observability/routes/experiments.ts +152 -0
  349. package/src/observability/routes/observabilityRbac.test.ts +767 -0
  350. package/src/observability/routes/observabilityRouteAccess.ts +106 -0
  351. package/src/observability/routes/prompts.test.ts +320 -0
  352. package/src/observability/routes/prompts.ts +263 -0
  353. package/src/observability/routes/requestAiService.ts +36 -0
  354. package/src/observability/routes/review.test.ts +120 -0
  355. package/src/observability/routes/review.ts +125 -0
  356. package/src/observability/routes/status.test.ts +116 -0
  357. package/src/observability/routes/status.ts +47 -0
  358. package/src/observability/routes/testMultiStage.test.ts +214 -0
  359. package/src/observability/routes/testMultiStage.ts +96 -0
  360. package/src/observability/routes/traces.test.ts +167 -0
  361. package/src/observability/routes/traces.ts +172 -0
  362. package/src/observability/schemaValidation.ts +29 -0
  363. package/src/observability/sopThresholds.ts +20 -0
  364. package/src/observability/status.test.ts +38 -0
  365. package/src/observability/status.ts +56 -0
  366. package/src/observability/testMultiStageSchemas.ts +95 -0
  367. package/src/observability/testMultiStageWorkflow.test.ts +114 -0
  368. package/src/observability/testMultiStageWorkflow.ts +307 -0
  369. package/src/observability/types.ts +262 -0
  370. package/src/routes/files.test.ts +63 -9
  371. package/src/routes/files.ts +21 -7
  372. package/src/routes/gpt.test.ts +3279 -0
  373. package/src/routes/gpt.ts +91 -816
  374. package/src/routes/gptActions.test.ts +241 -0
  375. package/src/routes/gptActions.ts +255 -0
  376. package/src/routes/gptApprovals.test.ts +762 -0
  377. package/src/routes/gptDatasets.test.ts +86 -0
  378. package/src/routes/gptDatasets.ts +104 -0
  379. package/src/routes/gptFiles.test.ts +589 -0
  380. package/src/routes/gptHistories.test.ts +933 -0
  381. package/src/routes/gptHistories.ts +266 -5
  382. package/src/routes/routes.test.ts +222 -2
  383. package/src/service/aiDatasets.test.ts +123 -0
  384. package/src/service/aiDatasets.ts +316 -0
  385. package/src/service/aiService.test.ts +593 -0
  386. package/src/service/aiService.ts +568 -111
  387. package/src/service/askFiles.test.ts +96 -0
  388. package/src/service/askFiles.ts +222 -0
  389. package/src/service/asks.test.ts +946 -0
  390. package/src/service/asks.ts +423 -0
  391. package/src/service/chatTurn.ts +2287 -0
  392. package/src/service/fileStorage.test.ts +11 -0
  393. package/src/service/fileStorage.ts +10 -2
  394. package/src/service/fillActionIds.test.ts +65 -0
  395. package/src/service/fillActionIds.ts +98 -0
  396. package/src/service/jsonFenceModel.test.ts +47 -0
  397. package/src/service/jsonFenceModel.ts +53 -0
  398. package/src/service/prompts.test.ts +21 -0
  399. package/src/service/prompts.ts +147 -0
  400. package/src/service/sanitizeHtml.test.ts +61 -0
  401. package/src/service/sanitizeHtml.ts +109 -0
  402. package/src/tests/bunSetup.test.ts +7 -4
  403. package/src/tests/bunSetup.ts +7 -7
  404. package/src/tests/chatHarness.ts +478 -0
  405. package/src/tests/generateStream.ts +49 -0
  406. package/src/tests/harnessErrors.ts +22 -0
  407. package/src/tests/harnessStreaming.ts +227 -0
  408. package/src/tests/helpers.ts +9 -2
  409. package/src/types/harness.ts +1095 -0
  410. package/src/types/index.ts +209 -13
  411. package/src/types/observability.ts +344 -0
  412. package/src/types/sanitize-html.d.ts +25 -0
@@ -0,0 +1,912 @@
1
+ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
2
+ import {DateTime} from "luxon";
3
+ import mongoose from "mongoose";
4
+
5
+ import {createLocalObservabilityPlugin} from "../observability/local/localPlugin";
6
+ import {registerObsSpan} from "../observability/local/models/obsSpan";
7
+ import {registerObsTrace} from "../observability/local/models/obsTrace";
8
+ import {harnessErrorMatching} from "../tests/harnessErrors";
9
+ import type {HarnessTaskDocument} from "../types/harness";
10
+ import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
11
+ import {
12
+ type AnyHarnessTaskDefinition,
13
+ defineTask,
14
+ Harness,
15
+ HarnessCommitConflictError,
16
+ InProcessRunner,
17
+ } from "./harness";
18
+ import {registerHarnessOwner} from "./models/harnessOwner";
19
+ import {registerHarnessTask} from "./models/harnessTask";
20
+
21
+ const LEASE_MS = 300;
22
+ const HEARTBEAT_MS = 50;
23
+ /** Lease for the test that holds a phase past it; roomy so a stalled machine cannot flake it. */
24
+ const SLOW_LEASE_MS = 1000;
25
+
26
+ const TaskModel = registerHarnessTask();
27
+ const OwnerModel = registerHarnessOwner();
28
+ let SpanModel: ObsSpanModel;
29
+ let TraceModel: ObsTraceModel;
30
+
31
+ interface Gate {
32
+ promise: Promise<void>;
33
+ release: () => void;
34
+ }
35
+
36
+ const gate = (): Gate => {
37
+ let release: () => void = () => {};
38
+ const promise = new Promise<void>((resolve) => {
39
+ release = resolve;
40
+ });
41
+ return {promise, release};
42
+ };
43
+
44
+ const pause = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
45
+
46
+ const waitUntil = async (
47
+ predicate: () => Promise<boolean> | boolean,
48
+ label: string
49
+ ): Promise<void> => {
50
+ for (let i = 0; i < 300; i++) {
51
+ if (await predicate()) {
52
+ return;
53
+ }
54
+ await pause(10);
55
+ }
56
+ throw new Error(`waitUntil timed out: ${label}`);
57
+ };
58
+
59
+ interface ProcessHandle {
60
+ /** Freeze every lease renewal, as if the process stopped running. */
61
+ die: () => void;
62
+ harness: Harness;
63
+ runner: InProcessRunner;
64
+ }
65
+
66
+ const liveHandles: ProcessHandle[] = [];
67
+ /** Phases left hanging by a simulated crash; released after each test so stop() settles. */
68
+ const hungGates: Gate[] = [];
69
+
70
+ /** One simulated process: a Harness plus its runner, sharing the test database. */
71
+ const openProcess = async ({
72
+ leaseMs = LEASE_MS,
73
+ ownerId,
74
+ registry,
75
+ }: {
76
+ leaseMs?: number;
77
+ ownerId: string;
78
+ registry: ReadonlyArray<AnyHarnessTaskDefinition>;
79
+ }): Promise<ProcessHandle> => {
80
+ let isDead = false;
81
+ const runner = new InProcessRunner({
82
+ heartbeatInterval: {milliseconds: HEARTBEAT_MS},
83
+ leaseDuration: {milliseconds: leaseMs},
84
+ ownerId,
85
+ pollInterval: {milliseconds: 20},
86
+ });
87
+ const harness = await Harness.open({
88
+ registry,
89
+ runner,
90
+ testHooks: {isHeartbeatSuspended: () => isDead},
91
+ });
92
+ const handle = {
93
+ die: () => {
94
+ isDead = true;
95
+ },
96
+ harness,
97
+ runner,
98
+ };
99
+ liveHandles.push(handle);
100
+ return handle;
101
+ };
102
+
103
+ const phaseSpans = async (
104
+ task: HarnessTaskDocument
105
+ ): Promise<Array<{error?: string; name: string; output?: unknown; status: string}>> => {
106
+ const spans = await SpanModel.find({
107
+ _id: {$ne: task.rootSpanId},
108
+ traceId: task.traceId,
109
+ }).sort({_id: 1, created: 1});
110
+ return spans.map((span) => ({
111
+ error: span.error,
112
+ name: span.name,
113
+ output: span.output,
114
+ status: span.status,
115
+ }));
116
+ };
117
+
118
+ const findTask = (id: mongoose.Types.ObjectId): Promise<HarnessTaskDocument> =>
119
+ TaskModel.findExactlyOne({_id: id});
120
+
121
+ /** A `fetch` phase whose first run hangs on `hung` until the test lets it go. */
122
+ const buildCrashingTask = ({
123
+ hung,
124
+ name,
125
+ onRun,
126
+ replay,
127
+ }: {
128
+ hung: Gate;
129
+ name: string;
130
+ onRun: (run: number) => Promise<void> | void;
131
+ replay?: "never" | "safe";
132
+ }): {definition: AnyHarnessTaskDefinition; started: Gate; zombieError: () => unknown} => {
133
+ const started = gate();
134
+ let runs = 0;
135
+ let zombieError: unknown;
136
+ const definition = defineTask<{patientId: string}, {chart?: string}, {status: string}>({
137
+ initial: () => ({phase: "fetch", state: {}}),
138
+ name,
139
+ phases: {
140
+ fetch: {
141
+ replay,
142
+ run: async (task, rt) => {
143
+ runs += 1;
144
+ const run = runs;
145
+ await onRun(run);
146
+ if (run === 1) {
147
+ started.release();
148
+ await hung.promise;
149
+ try {
150
+ await rt.commit({phase: "finish", state: {chart: "from-the-dead-runner"}});
151
+ } catch (error: unknown) {
152
+ zombieError = error;
153
+ throw error;
154
+ }
155
+ return;
156
+ }
157
+ await rt.commit({
158
+ phase: "finish",
159
+ state: {chart: `chart-${task.input.patientId}-${run}`},
160
+ });
161
+ },
162
+ },
163
+ finish: {
164
+ run: async (task, rt) =>
165
+ rt.commit({
166
+ terminal: {result: {status: `filed ${task.state.chart}`}, status: "completed"},
167
+ }),
168
+ },
169
+ },
170
+ version: 1,
171
+ });
172
+ return {
173
+ definition: definition as unknown as AnyHarnessTaskDefinition,
174
+ started,
175
+ zombieError: () => zombieError,
176
+ };
177
+ };
178
+
179
+ /** When the crashed owner started the phase it never finished. */
180
+ const ABANDONED_ACQUIRED_AT = DateTime.now().minus({minutes: 2}).startOf("second");
181
+
182
+ /** Put a task in `running` with a lease that already lapsed, as a crashed owner leaves it. */
183
+ const abandonRunning = async (id: mongoose.Types.ObjectId, phase?: string): Promise<void> => {
184
+ await TaskModel.updateOne(
185
+ {_id: id},
186
+ {
187
+ $set: {
188
+ lease: {
189
+ acquiredAt: ABANDONED_ACQUIRED_AT.toJSDate(),
190
+ expiresAt: DateTime.now().minus({minutes: 1}).toJSDate(),
191
+ owner: "crashed-process",
192
+ token: "stale-token",
193
+ },
194
+ status: "running",
195
+ ...(phase ? {phase} : {}),
196
+ },
197
+ }
198
+ );
199
+ };
200
+
201
+ const writeOnce = (onWrite: () => void): AnyHarnessTaskDefinition =>
202
+ defineTask<unknown, {draft: string}, {status: string}>({
203
+ initial: () => ({phase: "write", state: {draft: "note"}}),
204
+ name: "test.ehrWrite",
205
+ phases: {
206
+ write: {
207
+ run: async (_task, rt) => {
208
+ onWrite();
209
+ await rt.commit({terminal: {result: {status: "filed"}, status: "completed"}});
210
+ },
211
+ },
212
+ },
213
+ version: 1,
214
+ }) as unknown as AnyHarnessTaskDefinition;
215
+
216
+ describe("Harness leases and crash recovery", () => {
217
+ beforeAll(() => {
218
+ createLocalObservabilityPlugin();
219
+ SpanModel = registerObsSpan();
220
+ TraceModel = registerObsTrace();
221
+ });
222
+
223
+ beforeEach(async () => {
224
+ await Promise.all([
225
+ TaskModel.deleteMany({}),
226
+ OwnerModel.deleteMany({}),
227
+ SpanModel.deleteMany({}),
228
+ TraceModel.deleteMany({}),
229
+ ]);
230
+ });
231
+
232
+ afterEach(async () => {
233
+ for (const hung of hungGates.splice(0)) {
234
+ hung.release();
235
+ }
236
+ await Promise.all(liveHandles.splice(0).map(({harness}) => harness.stop()));
237
+ });
238
+
239
+ describe("process death mid-phase", () => {
240
+ it("re-runs a replay-safe phase from its checkpoint on a fresh harness after the lease expires", async () => {
241
+ const hung = gate();
242
+ hungGates.push(hung);
243
+ const leaseOwners: string[] = [];
244
+ const crashing = buildCrashingTask({
245
+ hung,
246
+ name: "test.safeCrash",
247
+ onRun: async () => {
248
+ const row = await TaskModel.findExactlyOne({name: "test.safeCrash"});
249
+ leaseOwners.push(String(row.lease?.owner));
250
+ },
251
+ replay: "safe",
252
+ });
253
+ // A lease long enough that the standby check below cannot race its expiry.
254
+ const first = await openProcess({
255
+ leaseMs: 1500,
256
+ ownerId: "process-1",
257
+ registry: [crashing.definition],
258
+ });
259
+ await first.harness.start();
260
+ const created = await first.harness.createTask(crashing.definition as never, {
261
+ patientId: "p1",
262
+ });
263
+ await crashing.started.promise;
264
+ first.die();
265
+
266
+ const second = await openProcess({ownerId: "process-2", registry: [crashing.definition]});
267
+ await second.harness.start();
268
+ // The dead owner's lease has not lapsed yet, so the fresh process waits on standby.
269
+ await pause(HEARTBEAT_MS * 2);
270
+ expect(second.runner.role).toBe("standby");
271
+ expect((await findTask(created._id)).lease?.owner).toBe("process-1");
272
+
273
+ const done = await second.harness.waitForTask(created._id, {timeout: {seconds: 5}});
274
+ expect(second.runner.role).toBe("owner");
275
+ expect(first.runner.role).toBe("standby");
276
+ expect(done.status).toBe("completed");
277
+ expect(done.phase).toBe("finish");
278
+ expect(done.state).toEqual({chart: "chart-p1-2"});
279
+ expect(done.outcome?.result).toEqual({status: "filed chart-p1-2"});
280
+ expect(done.lease?.token).toBeUndefined();
281
+ expect(leaseOwners).toEqual(["process-1", "process-2"]);
282
+
283
+ const spans = await phaseSpans(done);
284
+ expect(spans.map(({name, status}) => `${name}:${status}`)).toEqual([
285
+ "fetch:error",
286
+ "fetch:ok",
287
+ "finish:ok",
288
+ ]);
289
+ expect(spans[0].error).toBe(
290
+ "Interrupted: lease of process-1 expired mid-phase; re-running (replay: safe)"
291
+ );
292
+ expect(spans[0].output).toEqual({
293
+ interrupted: true,
294
+ leaseOwner: "process-1",
295
+ replay: "safe",
296
+ status: "pending",
297
+ });
298
+ const trace = await TraceModel.findExactlyOne({_id: done.traceId});
299
+ expect(trace.status).toBe("ok");
300
+ });
301
+
302
+ it("parks a default-replay phase as interrupted and never re-runs it", async () => {
303
+ const hung = gate();
304
+ hungGates.push(hung);
305
+ let runs = 0;
306
+ const crashing = buildCrashingTask({
307
+ hung,
308
+ name: "test.neverCrash",
309
+ onRun: () => {
310
+ runs += 1;
311
+ },
312
+ });
313
+ const first = await openProcess({ownerId: "process-1", registry: [crashing.definition]});
314
+ await first.harness.start();
315
+ const created = await first.harness.createTask(crashing.definition as never, {
316
+ patientId: "p2",
317
+ });
318
+ await crashing.started.promise;
319
+ first.die();
320
+
321
+ const second = await openProcess({ownerId: "process-2", registry: [crashing.definition]});
322
+ await second.harness.start();
323
+ await waitUntil(
324
+ async () => (await findTask(created._id)).status === "interrupted",
325
+ "task parked as interrupted"
326
+ );
327
+ // Several more heartbeats and polls must not pick the parked task back up.
328
+ await pause(LEASE_MS * 2);
329
+
330
+ const parked = await findTask(created._id);
331
+ expect(parked.status).toBe("interrupted");
332
+ expect(parked.phase).toBe("fetch");
333
+ // The untouched checkpoint: initial state `{}` (the task schema keeps empty objects).
334
+ expect(parked.state).toEqual({});
335
+ expect(parked.lease?.token).toBeUndefined();
336
+ expect(runs).toBe(1);
337
+
338
+ const spans = await phaseSpans(parked);
339
+ expect(spans).toHaveLength(1);
340
+ expect(spans[0]).toEqual({
341
+ error:
342
+ "Interrupted: lease of process-1 expired mid-phase; awaiting resolveInterrupted (replay: never)",
343
+ name: "fetch",
344
+ output: {
345
+ interrupted: true,
346
+ leaseOwner: "process-1",
347
+ replay: "never",
348
+ status: "interrupted",
349
+ },
350
+ status: "error",
351
+ });
352
+ // Interruption is not terminal: the root span and trace stay open.
353
+ const root = await SpanModel.findExactlyOne({_id: parked.rootSpanId});
354
+ expect(root.endedAt).toBeUndefined();
355
+ const trace = await TraceModel.findExactlyOne({_id: parked.traceId});
356
+ expect(trace.endedAt).toBeUndefined();
357
+ });
358
+
359
+ it("rejects the dead runner's late commit by its stale lease token and writes nothing", async () => {
360
+ const hung = gate();
361
+ hungGates.push(hung);
362
+ const rerunHeld = gate();
363
+ hungGates.push(rerunHeld);
364
+ const crashing = buildCrashingTask({
365
+ hung,
366
+ name: "test.staleToken",
367
+ onRun: async (run) => {
368
+ // Hold the replay so the zombie commits while the task is running at the
369
+ // same phase: only the lease token differs.
370
+ if (run === 2) {
371
+ await rerunHeld.promise;
372
+ }
373
+ },
374
+ replay: "safe",
375
+ });
376
+ const first = await openProcess({ownerId: "process-1", registry: [crashing.definition]});
377
+ await first.harness.start();
378
+ const created = await first.harness.createTask(crashing.definition as never, {
379
+ patientId: "p3",
380
+ });
381
+ await crashing.started.promise;
382
+ const deadToken = (await findTask(created._id)).lease?.token;
383
+ first.die();
384
+
385
+ const second = await openProcess({ownerId: "process-2", registry: [crashing.definition]});
386
+ await second.harness.start();
387
+ await waitUntil(
388
+ async () => (await findTask(created._id)).lease?.owner === "process-2",
389
+ "replay claimed by process-2"
390
+ );
391
+ const beforeZombie = await findTask(created._id);
392
+ const spansBeforeZombie = await SpanModel.countDocuments({});
393
+ expect(beforeZombie.status).toBe("running");
394
+ expect(beforeZombie.phase).toBe("fetch");
395
+ expect(beforeZombie.lease?.token).not.toBe(deadToken);
396
+
397
+ hung.release();
398
+ await waitUntil(() => crashing.zombieError() !== undefined, "zombie commit settled");
399
+ expect(crashing.zombieError()).toBeInstanceOf(HarnessCommitConflictError);
400
+
401
+ const afterZombie = await findTask(created._id);
402
+ expect(afterZombie.status).toBe("running");
403
+ expect(afterZombie.phase).toBe("fetch");
404
+ expect(afterZombie.state).toEqual({});
405
+ expect(afterZombie.lease?.token).toBe(beforeZombie.lease?.token);
406
+ expect(await SpanModel.countDocuments({})).toBe(spansBeforeZombie);
407
+
408
+ rerunHeld.release();
409
+ const done = await second.harness.waitForTask(created._id, {timeout: {seconds: 5}});
410
+ expect(done.state).toEqual({chart: "chart-p3-2"});
411
+ expect(await SpanModel.countDocuments({error: {$exists: false}, name: "fetch"})).toBe(1);
412
+ });
413
+
414
+ it("stops renewing and cannot commit once another runner holds the task lease", async () => {
415
+ const finished = gate();
416
+ let commitError: unknown;
417
+ let expiresAfterTakeover: number | undefined;
418
+ const takenOver = defineTask<unknown, unknown, unknown>({
419
+ initial: () => ({phase: "work"}),
420
+ name: "test.takenOver",
421
+ phases: {
422
+ work: {
423
+ replay: "safe",
424
+ run: async (task, rt) => {
425
+ // Another runner re-claimed the task under its own token while this one ran.
426
+ const foreignExpiry = DateTime.now().plus({minutes: 5}).toJSDate();
427
+ await TaskModel.updateOne(
428
+ {_id: task.id},
429
+ {
430
+ $set: {
431
+ "lease.expiresAt": foreignExpiry,
432
+ "lease.owner": "other",
433
+ "lease.token": "foreign",
434
+ },
435
+ }
436
+ );
437
+ expiresAfterTakeover = foreignExpiry.getTime();
438
+ await pause(HEARTBEAT_MS * 3);
439
+ try {
440
+ await rt.commit({terminal: {status: "completed"}});
441
+ } catch (error: unknown) {
442
+ commitError = error;
443
+ throw error;
444
+ } finally {
445
+ finished.release();
446
+ }
447
+ },
448
+ },
449
+ },
450
+ version: 1,
451
+ });
452
+ const handle = await openProcess({ownerId: "process-1", registry: [takenOver as never]});
453
+ await handle.harness.start();
454
+ const created = await handle.harness.createTask(takenOver, {});
455
+ await finished.promise;
456
+
457
+ expect(commitError).toBeInstanceOf(HarnessCommitConflictError);
458
+ const task = await findTask(created._id);
459
+ expect(task.status).toBe("running");
460
+ expect(task.lease?.token).toBe("foreign");
461
+ // The original runner's heartbeat never overwrote the new holder's expiry.
462
+ expect(task.lease?.expiresAt?.getTime()).toBe(expiresAfterTakeover);
463
+ expect(await SpanModel.countDocuments({name: "work"})).toBe(0);
464
+ });
465
+
466
+ it("keeps a phase that outlives the lease alive through task heartbeats", async () => {
467
+ const tokens: Array<string | undefined> = [];
468
+ let fetchRuns = 0;
469
+ const slow = defineTask<unknown, unknown, unknown>({
470
+ initial: () => ({phase: "fetch"}),
471
+ name: "test.slow",
472
+ phases: {
473
+ fetch: {
474
+ replay: "safe",
475
+ run: async (task, rt) => {
476
+ fetchRuns += 1;
477
+ tokens.push((await TaskModel.findExactlyOne({_id: task.id})).lease?.token);
478
+ await pause(SLOW_LEASE_MS * 3);
479
+ await rt.commit({phase: "finish"});
480
+ },
481
+ },
482
+ finish: {
483
+ run: async (task, rt) => {
484
+ tokens.push((await TaskModel.findExactlyOne({_id: task.id})).lease?.token);
485
+ await rt.commit({terminal: {status: "completed"}});
486
+ },
487
+ },
488
+ },
489
+ version: 1,
490
+ });
491
+ const only = await openProcess({
492
+ leaseMs: SLOW_LEASE_MS,
493
+ ownerId: "process-1",
494
+ registry: [slow as never],
495
+ });
496
+ // The owner's heartbeat sweep would interrupt the task if its lease ever lapsed, and
497
+ // the standby would take over if the owner lease lapsed while the phase ran.
498
+ const standby = await openProcess({
499
+ leaseMs: SLOW_LEASE_MS,
500
+ ownerId: "process-2",
501
+ registry: [slow as never],
502
+ });
503
+ await only.harness.start();
504
+ await waitUntil(() => only.runner.role === "owner", "process-1 owns");
505
+ await standby.harness.start();
506
+
507
+ const created = await only.harness.createTask(slow, {});
508
+ const done = await only.harness.waitForTask(created._id, {timeout: {seconds: 10}});
509
+
510
+ expect(done.status).toBe("completed");
511
+ expect(fetchRuns).toBe(1);
512
+ expect(await SpanModel.countDocuments({status: "error"})).toBe(0);
513
+ // Each phase starts under its own fencing token.
514
+ expect(tokens).toHaveLength(2);
515
+ expect(tokens[0]).toBeString();
516
+ expect(tokens[1]).toBeString();
517
+ expect(tokens[0]).not.toBe(tokens[1]);
518
+ expect(only.runner.role).toBe("owner");
519
+ expect(standby.runner.role).toBe("standby");
520
+ }, 15000);
521
+ });
522
+
523
+ describe("recovery on start()", () => {
524
+ it("parks a never phase and re-runs a safe phase found with an expired lease", async () => {
525
+ let writes = 0;
526
+ const ehrWrite = writeOnce(() => {
527
+ writes += 1;
528
+ });
529
+ const safeTask = defineTask<unknown, unknown, unknown>({
530
+ initial: () => ({phase: "fetch"}),
531
+ name: "test.safeResume",
532
+ phases: {
533
+ fetch: {
534
+ replay: "safe",
535
+ run: async (_task, rt) => rt.commit({terminal: {status: "completed"}}),
536
+ },
537
+ },
538
+ version: 1,
539
+ });
540
+ const handle = await openProcess({ownerId: "fresh", registry: [ehrWrite, safeTask as never]});
541
+ const parkedTask = await handle.harness.createTask(ehrWrite as never, {});
542
+ const resumedTask = await handle.harness.createTask(safeTask, {});
543
+ await abandonRunning(parkedTask._id);
544
+ await abandonRunning(resumedTask._id);
545
+
546
+ await handle.harness.start();
547
+ const resumed = await handle.harness.waitForTask(resumedTask._id, {timeout: {seconds: 5}});
548
+ await waitUntil(
549
+ async () => (await findTask(parkedTask._id)).status === "interrupted",
550
+ "write parked"
551
+ );
552
+
553
+ expect(resumed.status).toBe("completed");
554
+ expect(writes).toBe(0);
555
+ const spans = await phaseSpans(resumed);
556
+ expect(spans.map(({name, status}) => `${name}:${status}`)).toEqual([
557
+ "fetch:error",
558
+ "fetch:ok",
559
+ ]);
560
+ expect(spans[0].error).toContain("lease of crashed-process expired");
561
+ // The interruption span covers the cut-off phase from when its lease was taken.
562
+ const interruption = await SpanModel.findExactlyOne({
563
+ status: "error",
564
+ traceId: resumed.traceId,
565
+ });
566
+ expect(interruption.startedAt.getTime()).toBe(ABANDONED_ACQUIRED_AT.toMillis());
567
+ });
568
+
569
+ it("leaves the task running under its old lease when the interruption commit aborts", async () => {
570
+ const attempted = gate();
571
+ const ehrWrite = writeOnce(() => {});
572
+ const handle = await openProcess({ownerId: "fresh", registry: [ehrWrite]});
573
+ const ownTestHooks = {
574
+ // Every recovery attempt fails inside its transaction.
575
+ beforeCommitEnd: () => {
576
+ attempted.release();
577
+ throw new Error("audit store down");
578
+ },
579
+ };
580
+ // Rebuild the harness with a failing commit hook; recovery must roll back atomically.
581
+ const failing = await Harness.open({
582
+ registry: [ehrWrite],
583
+ runner: new InProcessRunner({
584
+ heartbeatInterval: {milliseconds: HEARTBEAT_MS},
585
+ leaseDuration: {milliseconds: LEASE_MS},
586
+ ownerId: "failing",
587
+ pollInterval: {milliseconds: 20},
588
+ }),
589
+ testHooks: ownTestHooks,
590
+ });
591
+ const created = await handle.harness.createTask(ehrWrite as never, {});
592
+ await abandonRunning(created._id);
593
+
594
+ await failing.start();
595
+ await attempted.promise;
596
+ await failing.stop();
597
+
598
+ const task = await findTask(created._id);
599
+ expect(task.status).toBe("running");
600
+ expect(task.lease?.token).toBe("stale-token");
601
+ expect(task.lease?.owner).toBe("crashed-process");
602
+ expect(await SpanModel.countDocuments({name: "write"})).toBe(0);
603
+ });
604
+
605
+ it("leaves expired tasks of versions created after start that it does not register", async () => {
606
+ const other = defineTask<unknown, unknown, unknown>({
607
+ initial: () => ({phase: "only"}),
608
+ name: "test.unrelated",
609
+ phases: {only: {run: async (_task, rt) => rt.commit({terminal: {status: "completed"}})}},
610
+ version: 1,
611
+ });
612
+ const handle = await openProcess({ownerId: "fresh", registry: [other as never]});
613
+ await handle.harness.start();
614
+ await waitUntil(() => handle.runner.role === "owner", "fresh owns");
615
+ // Another deploy creates a task of a version this process does not run.
616
+ const writer = await openProcess({ownerId: "writer", registry: [writeOnce(() => {})]});
617
+ const foreign = await writer.harness.createTask(writeOnce(() => {}) as never, {});
618
+ await abandonRunning(foreign._id);
619
+ // Several heartbeat sweeps run while owner.
620
+ await pause(HEARTBEAT_MS * 4);
621
+
622
+ const untouched = await findTask(foreign._id);
623
+ expect(untouched.status).toBe("running");
624
+ expect(untouched.lease?.owner).toBe("crashed-process");
625
+ });
626
+
627
+ it("parks a task whose stored phase no longer exists", async () => {
628
+ const ehrWrite = writeOnce(() => {});
629
+ const handle = await openProcess({ownerId: "fresh", registry: [ehrWrite]});
630
+ const created = await handle.harness.createTask(ehrWrite as never, {});
631
+ await abandonRunning(created._id, "removedPhase");
632
+
633
+ await handle.harness.start();
634
+ await waitUntil(async () => (await findTask(created._id)).status === "interrupted", "parked");
635
+ const [span] = await phaseSpans(await findTask(created._id));
636
+ expect(span).toEqual({
637
+ error:
638
+ "Interrupted: lease of crashed-process expired mid-phase; awaiting resolveInterrupted (replay: never)",
639
+ name: "removedPhase",
640
+ output: {
641
+ interrupted: true,
642
+ leaseOwner: "crashed-process",
643
+ replay: "never",
644
+ status: "interrupted",
645
+ },
646
+ status: "error",
647
+ });
648
+ });
649
+ });
650
+
651
+ describe("resolveInterrupted", () => {
652
+ const parkWrite = async (
653
+ onWrite: () => void = () => {}
654
+ ): Promise<{created: HarnessTaskDocument; handle: ProcessHandle}> => {
655
+ const ehrWrite = writeOnce(onWrite);
656
+ const handle = await openProcess({ownerId: "operator-process", registry: [ehrWrite]});
657
+ const created = await handle.harness.createTask(ehrWrite as never, {});
658
+ await abandonRunning(created._id);
659
+ await handle.harness.start();
660
+ await waitUntil(async () => (await findTask(created._id)).status === "interrupted", "parked");
661
+ return {created, handle};
662
+ };
663
+
664
+ const resolutionSpan = async (
665
+ task: HarnessTaskDocument
666
+ ): Promise<{output?: unknown; status: string}> => {
667
+ const span = await SpanModel.findExactlyOne({
668
+ name: "resolveInterrupted",
669
+ traceId: task.traceId,
670
+ });
671
+ return {output: span.output, status: span.status};
672
+ };
673
+
674
+ it("retry re-runs the same phase to completion and audits the decision", async () => {
675
+ let writes = 0;
676
+ const {created, handle} = await parkWrite(() => {
677
+ writes += 1;
678
+ });
679
+ const operator = new mongoose.Types.ObjectId();
680
+
681
+ const resolved = await handle.harness.resolveInterrupted(created._id, {
682
+ action: "retry",
683
+ reason: "Confirmed in the EHR that no note was filed",
684
+ userId: operator,
685
+ });
686
+ expect(resolved.status).toBe("pending");
687
+ expect(resolved.phase).toBe("write");
688
+
689
+ const done = await handle.harness.waitForTask(created._id, {timeout: {seconds: 5}});
690
+ expect(done.status).toBe("completed");
691
+ expect(writes).toBe(1);
692
+ expect(await resolutionSpan(done)).toEqual({
693
+ output: {
694
+ action: "retry",
695
+ decidedBy: String(operator),
696
+ phase: "write",
697
+ reason: "Confirmed in the EHR that no note was filed",
698
+ },
699
+ status: "ok",
700
+ });
701
+ });
702
+
703
+ it("abort ends the task aborted and closes the trace as an error", async () => {
704
+ let writes = 0;
705
+ const {created, handle} = await parkWrite(() => {
706
+ writes += 1;
707
+ });
708
+
709
+ const done = await handle.harness.resolveInterrupted(String(created._id), {
710
+ action: "abort",
711
+ reason: "Duplicate intake",
712
+ });
713
+ await pause(LEASE_MS);
714
+
715
+ expect(done.status).toBe("aborted");
716
+ expect(done.toObject().outcome).toEqual({
717
+ error: "Aborted after interruption: Duplicate intake",
718
+ status: "aborted",
719
+ });
720
+ expect((await findTask(created._id)).status).toBe("aborted");
721
+ expect(writes).toBe(0);
722
+ expect(await resolutionSpan(done)).toEqual({
723
+ output: {
724
+ abortHandler: {status: "none"},
725
+ action: "abort",
726
+ phase: "write",
727
+ reason: "Duplicate intake",
728
+ },
729
+ status: "ok",
730
+ });
731
+ const trace = await TraceModel.findExactlyOne({_id: done.traceId});
732
+ expect(trace.status).toBe("error");
733
+ expect(trace.errorSummary).toBe("Aborted after interruption: Duplicate intake");
734
+ expect(trace.endedAt).toBeInstanceOf(Date);
735
+ const root = await SpanModel.findExactlyOne({_id: done.rootSpanId});
736
+ expect(root.status).toBe("error");
737
+ });
738
+
739
+ it("complete ends the task completed with the operator's result", async () => {
740
+ const {created, handle} = await parkWrite();
741
+ const operator = new mongoose.Types.ObjectId();
742
+
743
+ const done = await handle.harness.resolveInterrupted(created._id, {
744
+ action: "complete",
745
+ reason: "Note was filed before the crash",
746
+ result: {status: "filed"},
747
+ userId: String(operator),
748
+ });
749
+
750
+ expect(done.status).toBe("completed");
751
+ expect(done.outcome?.result).toEqual({status: "filed"});
752
+ expect(await resolutionSpan(done)).toEqual({
753
+ output: {
754
+ action: "complete",
755
+ decidedBy: String(operator),
756
+ phase: "write",
757
+ reason: "Note was filed before the crash",
758
+ result: {status: "filed"},
759
+ },
760
+ status: "ok",
761
+ });
762
+ const trace = await TraceModel.findExactlyOne({_id: done.traceId});
763
+ expect(trace.status).toBe("ok");
764
+ expect(trace.output).toEqual({status: "filed"});
765
+ });
766
+
767
+ it("complete without a result closes the trace ok with no output", async () => {
768
+ const {created, handle} = await parkWrite();
769
+
770
+ const done = await handle.harness.resolveInterrupted(created._id, {
771
+ action: "complete",
772
+ reason: "Handled outside the harness",
773
+ });
774
+
775
+ expect(done.status).toBe("completed");
776
+ expect(done.outcome?.result).toBeUndefined();
777
+ const trace = await TraceModel.findExactlyOne({_id: done.traceId});
778
+ expect(trace.status).toBe("ok");
779
+ expect(trace.endedAt).toBeInstanceOf(Date);
780
+ });
781
+
782
+ it("lets only one of two concurrent resolutions win", async () => {
783
+ const {created, handle} = await parkWrite();
784
+
785
+ const results = await Promise.allSettled([
786
+ handle.harness.resolveInterrupted(created._id, {action: "abort", reason: "first"}),
787
+ handle.harness.resolveInterrupted(created._id, {action: "complete", reason: "second"}),
788
+ ]);
789
+
790
+ const winners = results.filter((result) => result.status === "fulfilled");
791
+ const losers = results.filter((result) => result.status === "rejected");
792
+ expect(winners).toHaveLength(1);
793
+ expect(losers).toHaveLength(1);
794
+ // The loser either saw the winner's status or lost the fenced commit.
795
+ const loserError = (losers[0] as PromiseRejectedResult).reason as Error;
796
+ expect(
797
+ loserError instanceof HarnessCommitConflictError ||
798
+ loserError.message.endsWith("not interrupted")
799
+ ).toBe(true);
800
+ const winner = (winners[0] as PromiseFulfilledResult<HarnessTaskDocument>).value;
801
+ expect((await findTask(created._id)).status).toBe(winner.status);
802
+ expect(["aborted", "completed"]).toContain(winner.status);
803
+ expect(await SpanModel.countDocuments({name: "resolveInterrupted"})).toBe(1);
804
+ });
805
+
806
+ it("rejects a task that is not interrupted, a blank reason, and an unknown action", async () => {
807
+ const {created, handle} = await parkWrite();
808
+ const running = await handle.harness.createTask(writeOnce(() => {}) as never, {});
809
+ await handle.harness.waitForTask(running._id, {timeout: {seconds: 5}});
810
+
811
+ await expect(
812
+ handle.harness.resolveInterrupted(running._id, {action: "retry", reason: "nope"})
813
+ ).rejects.toThrow(
814
+ harnessErrorMatching(
815
+ "taskNotInterrupted",
816
+ `Task ${running._id} is completed, not interrupted`
817
+ )
818
+ );
819
+ await expect(
820
+ handle.harness.resolveInterrupted(created._id, {action: "retry", reason: " "})
821
+ ).rejects.toThrow(
822
+ harnessErrorMatching("invalidRequest", "resolveInterrupted requires a reason")
823
+ );
824
+ await expect(
825
+ handle.harness.resolveInterrupted(created._id, {
826
+ action: "replay" as never,
827
+ reason: "typo",
828
+ })
829
+ ).rejects.toThrow(
830
+ harnessErrorMatching(
831
+ "invalidRequest",
832
+ "resolveInterrupted action must be one of abort, complete, retry"
833
+ )
834
+ );
835
+ expect((await findTask(created._id)).status).toBe("interrupted");
836
+ expect(await SpanModel.countDocuments({name: "resolveInterrupted"})).toBe(0);
837
+ });
838
+ });
839
+
840
+ describe("owner lease", () => {
841
+ it("never lets two live harnesses run tasks at the same time", async () => {
842
+ const activeByTask = new Map<string, number>();
843
+ let maxConcurrent = 0;
844
+ let active = 0;
845
+ const ranBy: string[] = [];
846
+ const counted = defineTask<unknown, unknown, unknown>({
847
+ initial: () => ({phase: "work"}),
848
+ name: "test.counted",
849
+ phases: {
850
+ work: {
851
+ replay: "safe",
852
+ run: async (task, rt) => {
853
+ active += 1;
854
+ activeByTask.set(task.id, (activeByTask.get(task.id) ?? 0) + 1);
855
+ maxConcurrent = Math.max(maxConcurrent, active);
856
+ ranBy.push(String((await TaskModel.findExactlyOne({_id: task.id})).lease?.owner));
857
+ await pause(15);
858
+ active -= 1;
859
+ await rt.commit({terminal: {status: "completed"}});
860
+ },
861
+ },
862
+ },
863
+ version: 1,
864
+ });
865
+ // A long lease keeps ownership stable even when the test machine stalls.
866
+ const a = await openProcess({
867
+ leaseMs: 5000,
868
+ ownerId: "process-a",
869
+ registry: [counted as never],
870
+ });
871
+ const b = await openProcess({
872
+ leaseMs: 5000,
873
+ ownerId: "process-b",
874
+ registry: [counted as never],
875
+ });
876
+ await Promise.all([a.harness.start(), b.harness.start()]);
877
+
878
+ const created = await Promise.all(
879
+ [1, 2, 3, 4, 5, 6].map((n) => (n % 2 === 0 ? a : b).harness.createTask(counted, {n}))
880
+ );
881
+ await Promise.all(
882
+ created.map((task) => a.harness.waitForTask(task._id, {timeout: {seconds: 5}}))
883
+ );
884
+
885
+ expect(maxConcurrent).toBe(1);
886
+ expect([...activeByTask.values()]).toEqual([1, 1, 1, 1, 1, 1]);
887
+ expect(new Set(ranBy).size).toBe(1);
888
+ expect([a.runner.role, b.runner.role].sort()).toEqual(["owner", "standby"]);
889
+ });
890
+
891
+ it("hands ownership to the standby immediately when the owner stops", async () => {
892
+ const ehrWrite = writeOnce(() => {});
893
+ // A lease far longer than the test: only the release on stop() lets b take over.
894
+ const a = await openProcess({leaseMs: 60_000, ownerId: "process-a", registry: [ehrWrite]});
895
+ const b = await openProcess({leaseMs: 60_000, ownerId: "process-b", registry: [ehrWrite]});
896
+ await a.harness.start();
897
+ await waitUntil(() => a.runner.role === "owner", "a owns");
898
+ await b.harness.start();
899
+ await pause(HEARTBEAT_MS * 2);
900
+ expect(b.runner.role).toBe("standby");
901
+
902
+ await a.harness.stop();
903
+ const created = await b.harness.createTask(ehrWrite as never, {});
904
+ const done = await b.harness.waitForTask(created._id, {timeout: {seconds: 5}});
905
+
906
+ expect(done.status).toBe("completed");
907
+ expect(b.runner.role).toBe("owner");
908
+ const owner = await OwnerModel.findExactlyOne({key: "default"});
909
+ expect(owner.owner).toBe("process-b");
910
+ });
911
+ });
912
+ });