@terreno/ai 57.9.0 → 57.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (417) hide show
  1. package/README.md +1 -0
  2. package/bunfig.toml +1 -1
  3. package/dist/aiApp.d.ts +8 -2
  4. package/dist/aiApp.js +12 -6
  5. package/dist/harness/adminScreens.d.ts +6 -0
  6. package/dist/harness/adminScreens.js +11 -0
  7. package/dist/harness/agentLoop.d.ts +61 -0
  8. package/dist/harness/agentLoop.js +798 -0
  9. package/dist/harness/agentTaskNames.d.ts +4 -0
  10. package/dist/harness/agentTaskNames.js +7 -0
  11. package/dist/harness/approvalPolicy.d.ts +3 -0
  12. package/dist/harness/approvalPolicy.js +23 -0
  13. package/dist/harness/approvals.d.ts +101 -0
  14. package/dist/harness/approvals.js +468 -0
  15. package/dist/harness/commit.d.ts +238 -0
  16. package/dist/harness/commit.js +678 -0
  17. package/dist/harness/conversation.d.ts +75 -0
  18. package/dist/harness/conversation.js +397 -0
  19. package/dist/harness/defineAgent.d.ts +9 -0
  20. package/dist/harness/defineAgent.js +65 -0
  21. package/dist/harness/defineTask.d.ts +11 -0
  22. package/dist/harness/defineTask.js +59 -0
  23. package/dist/harness/defineTool.d.ts +7 -0
  24. package/dist/harness/defineTool.js +36 -0
  25. package/dist/harness/definitionError.d.ts +8 -0
  26. package/dist/harness/definitionError.js +15 -0
  27. package/dist/harness/errors.d.ts +133 -0
  28. package/dist/harness/errors.js +107 -0
  29. package/dist/harness/eventHub.d.ts +45 -0
  30. package/dist/harness/eventHub.js +135 -0
  31. package/dist/harness/events.d.ts +95 -0
  32. package/dist/harness/events.js +267 -0
  33. package/dist/harness/executionEnv.d.ts +31 -0
  34. package/dist/harness/executionEnv.js +2 -0
  35. package/dist/harness/extensions.d.ts +82 -0
  36. package/dist/harness/extensions.js +274 -0
  37. package/dist/harness/harness.d.ts +163 -0
  38. package/dist/harness/harness.js +522 -0
  39. package/dist/harness/harnessApp.d.ts +44 -0
  40. package/dist/harness/harnessApp.js +62 -0
  41. package/dist/harness/internalRuntime.d.ts +16 -0
  42. package/dist/harness/internalRuntime.js +18 -0
  43. package/dist/harness/leases.d.ts +35 -0
  44. package/dist/harness/leases.js +86 -0
  45. package/dist/harness/memo.d.ts +24 -0
  46. package/dist/harness/memo.js +78 -0
  47. package/dist/harness/modelCall.d.ts +45 -0
  48. package/dist/harness/modelCall.js +147 -0
  49. package/dist/harness/models/harnessApproval.d.ts +2 -0
  50. package/dist/harness/models/harnessApproval.js +100 -0
  51. package/dist/harness/models/harnessConversation.d.ts +2 -0
  52. package/dist/harness/models/harnessConversation.js +117 -0
  53. package/dist/harness/models/harnessEvent.d.ts +3 -0
  54. package/dist/harness/models/harnessEvent.js +73 -0
  55. package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
  56. package/dist/harness/models/harnessInboxEvent.js +58 -0
  57. package/dist/harness/models/harnessMemo.d.ts +2 -0
  58. package/dist/harness/models/harnessMemo.js +42 -0
  59. package/dist/harness/models/harnessMessage.d.ts +2 -0
  60. package/dist/harness/models/harnessMessage.js +73 -0
  61. package/dist/harness/models/harnessOwner.d.ts +2 -0
  62. package/dist/harness/models/harnessOwner.js +36 -0
  63. package/dist/harness/models/harnessTask.d.ts +2 -0
  64. package/dist/harness/models/harnessTask.js +191 -0
  65. package/dist/harness/ownership.d.ts +89 -0
  66. package/dist/harness/ownership.js +438 -0
  67. package/dist/harness/registry.d.ts +29 -0
  68. package/dist/harness/registry.js +93 -0
  69. package/dist/harness/retryBackoff.d.ts +31 -0
  70. package/dist/harness/retryBackoff.js +50 -0
  71. package/dist/harness/routes/approvals.d.ts +13 -0
  72. package/dist/harness/routes/approvals.js +93 -0
  73. package/dist/harness/routes/conversations.d.ts +11 -0
  74. package/dist/harness/routes/conversations.js +66 -0
  75. package/dist/harness/routes/events.d.ts +19 -0
  76. package/dist/harness/routes/events.js +215 -0
  77. package/dist/harness/routes/tasks.d.ts +11 -0
  78. package/dist/harness/routes/tasks.js +108 -0
  79. package/dist/harness/runners/inProcessRunner.d.ts +64 -0
  80. package/dist/harness/runners/inProcessRunner.js +227 -0
  81. package/dist/harness/runners/jobsRunner.d.ts +62 -0
  82. package/dist/harness/runners/jobsRunner.js +212 -0
  83. package/dist/harness/runtime.d.ts +52 -0
  84. package/dist/harness/runtime.js +604 -0
  85. package/dist/harness/subagent.d.ts +53 -0
  86. package/dist/harness/subagent.js +134 -0
  87. package/dist/harness/suspend.d.ts +8 -0
  88. package/dist/harness/suspend.js +16 -0
  89. package/dist/harness/transaction.d.ts +3 -0
  90. package/dist/harness/transaction.js +22 -0
  91. package/dist/harness/waits.d.ts +100 -0
  92. package/dist/harness/waits.js +214 -0
  93. package/dist/index.d.ts +19 -1
  94. package/dist/index.js +45 -2
  95. package/dist/models/aiDataset.d.ts +2 -0
  96. package/dist/models/aiDataset.js +67 -0
  97. package/dist/models/aiRequest.js +1 -1
  98. package/dist/models/gptHistory.js +73 -0
  99. package/dist/observability/adminScreens.d.ts +5 -0
  100. package/dist/observability/adminScreens.js +32 -0
  101. package/dist/observability/compileTemplate.d.ts +1 -0
  102. package/dist/observability/compileTemplate.js +13 -0
  103. package/dist/observability/datasetImport.d.ts +11 -0
  104. package/dist/observability/datasetImport.js +185 -0
  105. package/dist/observability/evaluate.d.ts +31 -0
  106. package/dist/observability/evaluate.js +200 -0
  107. package/dist/observability/evaluatorTemplates.d.ts +17 -0
  108. package/dist/observability/evaluatorTemplates.js +111 -0
  109. package/dist/observability/jsonPath.d.ts +2 -0
  110. package/dist/observability/jsonPath.js +57 -0
  111. package/dist/observability/local/datasetStore.d.ts +87 -0
  112. package/dist/observability/local/datasetStore.js +326 -0
  113. package/dist/observability/local/evaluatorStore.d.ts +44 -0
  114. package/dist/observability/local/evaluatorStore.js +182 -0
  115. package/dist/observability/local/experimentRunner.d.ts +96 -0
  116. package/dist/observability/local/experimentRunner.js +419 -0
  117. package/dist/observability/local/localPlugin.d.ts +22 -0
  118. package/dist/observability/local/localPlugin.js +76 -0
  119. package/dist/observability/local/models/obsDataset.d.ts +2 -0
  120. package/dist/observability/local/models/obsDataset.js +37 -0
  121. package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
  122. package/dist/observability/local/models/obsDatasetItem.js +76 -0
  123. package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
  124. package/dist/observability/local/models/obsEvaluator.js +98 -0
  125. package/dist/observability/local/models/obsExperiment.d.ts +2 -0
  126. package/dist/observability/local/models/obsExperiment.js +87 -0
  127. package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
  128. package/dist/observability/local/models/obsExperimentItem.js +45 -0
  129. package/dist/observability/local/models/obsPrompt.d.ts +2 -0
  130. package/dist/observability/local/models/obsPrompt.js +42 -0
  131. package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
  132. package/dist/observability/local/models/obsPromptLabel.js +39 -0
  133. package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
  134. package/dist/observability/local/models/obsPromptVersion.js +96 -0
  135. package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
  136. package/dist/observability/local/models/obsReviewItem.js +75 -0
  137. package/dist/observability/local/models/obsScore.d.ts +2 -0
  138. package/dist/observability/local/models/obsScore.js +58 -0
  139. package/dist/observability/local/models/obsSpan.d.ts +2 -0
  140. package/dist/observability/local/models/obsSpan.js +64 -0
  141. package/dist/observability/local/models/obsTrace.d.ts +2 -0
  142. package/dist/observability/local/models/obsTrace.js +93 -0
  143. package/dist/observability/local/promptStore.d.ts +180 -0
  144. package/dist/observability/local/promptStore.js +444 -0
  145. package/dist/observability/local/reviewStore.d.ts +50 -0
  146. package/dist/observability/local/reviewStore.js +203 -0
  147. package/dist/observability/local/traceStore.d.ts +103 -0
  148. package/dist/observability/local/traceStore.js +325 -0
  149. package/dist/observability/observabilityApp.d.ts +24 -0
  150. package/dist/observability/observabilityApp.js +157 -0
  151. package/dist/observability/observabilityAppRegistry.d.ts +14 -0
  152. package/dist/observability/observabilityAppRegistry.js +16 -0
  153. package/dist/observability/permissions.d.ts +7 -0
  154. package/dist/observability/permissions.js +60 -0
  155. package/dist/observability/reviewPanels.d.ts +18 -0
  156. package/dist/observability/reviewPanels.js +46 -0
  157. package/dist/observability/routes/datasets.d.ts +7 -0
  158. package/dist/observability/routes/datasets.js +149 -0
  159. package/dist/observability/routes/evaluators.d.ts +7 -0
  160. package/dist/observability/routes/evaluators.js +123 -0
  161. package/dist/observability/routes/experiments.d.ts +7 -0
  162. package/dist/observability/routes/experiments.js +95 -0
  163. package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
  164. package/dist/observability/routes/observabilityRouteAccess.js +67 -0
  165. package/dist/observability/routes/prompts.d.ts +12 -0
  166. package/dist/observability/routes/prompts.js +172 -0
  167. package/dist/observability/routes/requestAiService.d.ts +13 -0
  168. package/dist/observability/routes/requestAiService.js +21 -0
  169. package/dist/observability/routes/review.d.ts +7 -0
  170. package/dist/observability/routes/review.js +79 -0
  171. package/dist/observability/routes/status.d.ts +5 -0
  172. package/dist/observability/routes/status.js +31 -0
  173. package/dist/observability/routes/testMultiStage.d.ts +9 -0
  174. package/dist/observability/routes/testMultiStage.js +67 -0
  175. package/dist/observability/routes/traces.d.ts +7 -0
  176. package/dist/observability/routes/traces.js +129 -0
  177. package/dist/observability/schemaValidation.d.ts +5 -0
  178. package/dist/observability/schemaValidation.js +26 -0
  179. package/dist/observability/sopThresholds.d.ts +2 -0
  180. package/dist/observability/sopThresholds.js +21 -0
  181. package/dist/observability/status.d.ts +23 -0
  182. package/dist/observability/status.js +34 -0
  183. package/dist/observability/testMultiStageSchemas.d.ts +10 -0
  184. package/dist/observability/testMultiStageSchemas.js +79 -0
  185. package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
  186. package/dist/observability/testMultiStageWorkflow.js +218 -0
  187. package/dist/observability/types.d.ts +159 -0
  188. package/dist/observability/types.js +67 -0
  189. package/dist/routes/files.js +18 -7
  190. package/dist/routes/gpt.js +79 -689
  191. package/dist/routes/gptActions.d.ts +9 -0
  192. package/dist/routes/gptActions.js +212 -0
  193. package/dist/routes/gptDatasets.d.ts +5 -0
  194. package/dist/routes/gptDatasets.js +94 -0
  195. package/dist/routes/gptHistories.d.ts +3 -1
  196. package/dist/routes/gptHistories.js +188 -2
  197. package/dist/service/aiDatasets.d.ts +57 -0
  198. package/dist/service/aiDatasets.js +253 -0
  199. package/dist/service/aiService.d.ts +50 -1
  200. package/dist/service/aiService.js +426 -77
  201. package/dist/service/askFiles.d.ts +49 -0
  202. package/dist/service/askFiles.js +162 -0
  203. package/dist/service/asks.d.ts +76 -0
  204. package/dist/service/asks.js +284 -0
  205. package/dist/service/chatTurn.d.ts +128 -0
  206. package/dist/service/chatTurn.js +1674 -0
  207. package/dist/service/fileStorage.d.ts +4 -0
  208. package/dist/service/fileStorage.js +7 -2
  209. package/dist/service/fillActionIds.d.ts +6 -0
  210. package/dist/service/fillActionIds.js +79 -0
  211. package/dist/service/jsonFenceModel.d.ts +7 -0
  212. package/dist/service/jsonFenceModel.js +45 -0
  213. package/dist/service/prompts.d.ts +52 -0
  214. package/dist/service/prompts.js +96 -1
  215. package/dist/service/sanitizeHtml.d.ts +11 -0
  216. package/dist/service/sanitizeHtml.js +108 -0
  217. package/dist/types/harness.d.ts +952 -0
  218. package/dist/types/harness.js +133 -0
  219. package/dist/types/index.d.ts +192 -14
  220. package/dist/types/index.js +2 -0
  221. package/dist/types/observability.d.ts +288 -0
  222. package/dist/types/observability.js +2 -0
  223. package/package.json +28 -5
  224. package/src/aiApp.test.ts +207 -1
  225. package/src/aiApp.ts +19 -6
  226. package/src/harness/adminScreens.test.ts +34 -0
  227. package/src/harness/adminScreens.ts +11 -0
  228. package/src/harness/agentLoop.test.ts +1034 -0
  229. package/src/harness/agentLoop.ts +1082 -0
  230. package/src/harness/agentRecovery.test.ts +380 -0
  231. package/src/harness/agentTaskNames.ts +4 -0
  232. package/src/harness/approvalPolicy.ts +28 -0
  233. package/src/harness/approvals.test.ts +1094 -0
  234. package/src/harness/approvals.ts +644 -0
  235. package/src/harness/commit.ts +1049 -0
  236. package/src/harness/concurrency.test.ts +155 -0
  237. package/src/harness/conversation.ts +503 -0
  238. package/src/harness/defineAgent.test.ts +92 -0
  239. package/src/harness/defineAgent.ts +82 -0
  240. package/src/harness/defineTask.test.ts +59 -0
  241. package/src/harness/defineTask.ts +76 -0
  242. package/src/harness/defineTool.ts +45 -0
  243. package/src/harness/definitionError.ts +13 -0
  244. package/src/harness/errors.test.ts +90 -0
  245. package/src/harness/errors.ts +125 -0
  246. package/src/harness/eventHub.test.ts +124 -0
  247. package/src/harness/eventHub.ts +159 -0
  248. package/src/harness/events.test.ts +769 -0
  249. package/src/harness/events.ts +384 -0
  250. package/src/harness/executionEnv.ts +34 -0
  251. package/src/harness/extensions.test.ts +1218 -0
  252. package/src/harness/extensions.ts +382 -0
  253. package/src/harness/harness.test.ts +693 -0
  254. package/src/harness/harness.ts +778 -0
  255. package/src/harness/harnessApp.ts +76 -0
  256. package/src/harness/internalRuntime.ts +44 -0
  257. package/src/harness/leases.ts +132 -0
  258. package/src/harness/memo.ts +107 -0
  259. package/src/harness/modelCall.test.ts +134 -0
  260. package/src/harness/modelCall.ts +200 -0
  261. package/src/harness/models/harnessApproval.ts +109 -0
  262. package/src/harness/models/harnessConversation.ts +142 -0
  263. package/src/harness/models/harnessEvent.ts +92 -0
  264. package/src/harness/models/harnessInboxEvent.ts +73 -0
  265. package/src/harness/models/harnessMemo.ts +43 -0
  266. package/src/harness/models/harnessMessage.ts +85 -0
  267. package/src/harness/models/harnessModels.test.ts +60 -0
  268. package/src/harness/models/harnessOwner.ts +40 -0
  269. package/src/harness/models/harnessTask.ts +206 -0
  270. package/src/harness/ownership.test.ts +1087 -0
  271. package/src/harness/ownership.ts +618 -0
  272. package/src/harness/recovery.test.ts +907 -0
  273. package/src/harness/registry.ts +124 -0
  274. package/src/harness/retries.test.ts +292 -0
  275. package/src/harness/retryBackoff.test.ts +100 -0
  276. package/src/harness/retryBackoff.ts +77 -0
  277. package/src/harness/routes/approvals.ts +123 -0
  278. package/src/harness/routes/conversations.test.ts +518 -0
  279. package/src/harness/routes/conversations.ts +102 -0
  280. package/src/harness/routes/events.ts +280 -0
  281. package/src/harness/routes/tasks.test.ts +198 -0
  282. package/src/harness/routes/tasks.ts +146 -0
  283. package/src/harness/runners/inProcessRunner.test.ts +530 -0
  284. package/src/harness/runners/inProcessRunner.ts +260 -0
  285. package/src/harness/runners/jobsRunner.test.ts +709 -0
  286. package/src/harness/runners/jobsRunner.ts +252 -0
  287. package/src/harness/runtime.ts +802 -0
  288. package/src/harness/subagent.test.ts +696 -0
  289. package/src/harness/subagent.ts +192 -0
  290. package/src/harness/suspend.ts +13 -0
  291. package/src/harness/transaction.ts +17 -0
  292. package/src/harness/versioning.test.ts +288 -0
  293. package/src/harness/waits.test.ts +693 -0
  294. package/src/harness/waits.ts +356 -0
  295. package/src/index.test.ts +13 -0
  296. package/src/index.ts +70 -0
  297. package/src/isolated/adminScreensConfig.isolated.ts +143 -0
  298. package/src/models/aiDataset.test.ts +34 -0
  299. package/src/models/aiDataset.ts +78 -0
  300. package/src/models/aiRequest.ts +2 -1
  301. package/src/models/gptHistory.test.ts +156 -0
  302. package/src/models/gptHistory.ts +89 -1
  303. package/src/observability/adminScreens.test.ts +71 -0
  304. package/src/observability/adminScreens.ts +42 -0
  305. package/src/observability/compileTemplate.test.ts +14 -0
  306. package/src/observability/compileTemplate.ts +10 -0
  307. package/src/observability/datasetImport.test.ts +95 -0
  308. package/src/observability/datasetImport.ts +220 -0
  309. package/src/observability/evaluate.test.ts +549 -0
  310. package/src/observability/evaluate.ts +254 -0
  311. package/src/observability/evaluatorTemplates.ts +124 -0
  312. package/src/observability/jsonPath.test.ts +52 -0
  313. package/src/observability/jsonPath.ts +60 -0
  314. package/src/observability/local/datasetStore.test.ts +230 -0
  315. package/src/observability/local/datasetStore.ts +452 -0
  316. package/src/observability/local/evaluatorStore.test.ts +138 -0
  317. package/src/observability/local/evaluatorStore.ts +224 -0
  318. package/src/observability/local/experimentRunner.test.ts +489 -0
  319. package/src/observability/local/experimentRunner.ts +548 -0
  320. package/src/observability/local/localPlugin.ts +88 -0
  321. package/src/observability/local/models/obsDataset.ts +38 -0
  322. package/src/observability/local/models/obsDatasetItem.ts +80 -0
  323. package/src/observability/local/models/obsEvaluator.ts +102 -0
  324. package/src/observability/local/models/obsExperiment.ts +91 -0
  325. package/src/observability/local/models/obsExperimentItem.ts +52 -0
  326. package/src/observability/local/models/obsPrompt.ts +43 -0
  327. package/src/observability/local/models/obsPromptLabel.ts +43 -0
  328. package/src/observability/local/models/obsPromptVersion.ts +107 -0
  329. package/src/observability/local/models/obsReviewItem.ts +79 -0
  330. package/src/observability/local/models/obsScore.ts +59 -0
  331. package/src/observability/local/models/obsSpan.ts +65 -0
  332. package/src/observability/local/models/obsTrace.ts +94 -0
  333. package/src/observability/local/obsModels.test.ts +119 -0
  334. package/src/observability/local/promptStore.test.ts +351 -0
  335. package/src/observability/local/promptStore.ts +665 -0
  336. package/src/observability/local/reviewStore.test.ts +164 -0
  337. package/src/observability/local/reviewStore.ts +264 -0
  338. package/src/observability/local/traceStore.test.ts +264 -0
  339. package/src/observability/local/traceStore.ts +440 -0
  340. package/src/observability/observabilityApp.test.ts +92 -0
  341. package/src/observability/observabilityApp.ts +185 -0
  342. package/src/observability/observabilityAppRegistry.test.ts +30 -0
  343. package/src/observability/observabilityAppRegistry.ts +35 -0
  344. package/src/observability/permissions.test.ts +96 -0
  345. package/src/observability/permissions.ts +78 -0
  346. package/src/observability/reviewPanels.test.ts +35 -0
  347. package/src/observability/reviewPanels.ts +64 -0
  348. package/src/observability/routes/datasets.test.ts +176 -0
  349. package/src/observability/routes/datasets.ts +252 -0
  350. package/src/observability/routes/evaluators.test.ts +119 -0
  351. package/src/observability/routes/evaluators.ts +207 -0
  352. package/src/observability/routes/experiments.test.ts +282 -0
  353. package/src/observability/routes/experiments.ts +152 -0
  354. package/src/observability/routes/observabilityRbac.test.ts +767 -0
  355. package/src/observability/routes/observabilityRouteAccess.ts +106 -0
  356. package/src/observability/routes/prompts.test.ts +320 -0
  357. package/src/observability/routes/prompts.ts +263 -0
  358. package/src/observability/routes/requestAiService.ts +36 -0
  359. package/src/observability/routes/review.test.ts +120 -0
  360. package/src/observability/routes/review.ts +125 -0
  361. package/src/observability/routes/status.test.ts +116 -0
  362. package/src/observability/routes/status.ts +47 -0
  363. package/src/observability/routes/testMultiStage.test.ts +214 -0
  364. package/src/observability/routes/testMultiStage.ts +96 -0
  365. package/src/observability/routes/traces.test.ts +167 -0
  366. package/src/observability/routes/traces.ts +172 -0
  367. package/src/observability/schemaValidation.ts +29 -0
  368. package/src/observability/sopThresholds.ts +20 -0
  369. package/src/observability/status.test.ts +38 -0
  370. package/src/observability/status.ts +56 -0
  371. package/src/observability/testMultiStageSchemas.ts +95 -0
  372. package/src/observability/testMultiStageWorkflow.test.ts +114 -0
  373. package/src/observability/testMultiStageWorkflow.ts +307 -0
  374. package/src/observability/types.ts +262 -0
  375. package/src/routes/files.test.ts +63 -9
  376. package/src/routes/files.ts +21 -7
  377. package/src/routes/gpt.test.ts +3279 -0
  378. package/src/routes/gpt.ts +91 -816
  379. package/src/routes/gptActions.test.ts +241 -0
  380. package/src/routes/gptActions.ts +255 -0
  381. package/src/routes/gptApprovals.test.ts +762 -0
  382. package/src/routes/gptDatasets.test.ts +86 -0
  383. package/src/routes/gptDatasets.ts +104 -0
  384. package/src/routes/gptFiles.test.ts +589 -0
  385. package/src/routes/gptHistories.test.ts +933 -0
  386. package/src/routes/gptHistories.ts +266 -5
  387. package/src/routes/routes.test.ts +222 -2
  388. package/src/service/aiDatasets.test.ts +123 -0
  389. package/src/service/aiDatasets.ts +316 -0
  390. package/src/service/aiService.test.ts +593 -0
  391. package/src/service/aiService.ts +568 -111
  392. package/src/service/askFiles.test.ts +96 -0
  393. package/src/service/askFiles.ts +222 -0
  394. package/src/service/asks.test.ts +946 -0
  395. package/src/service/asks.ts +423 -0
  396. package/src/service/chatTurn.ts +2287 -0
  397. package/src/service/fileStorage.test.ts +11 -0
  398. package/src/service/fileStorage.ts +10 -2
  399. package/src/service/fillActionIds.test.ts +65 -0
  400. package/src/service/fillActionIds.ts +98 -0
  401. package/src/service/jsonFenceModel.test.ts +47 -0
  402. package/src/service/jsonFenceModel.ts +53 -0
  403. package/src/service/prompts.test.ts +21 -0
  404. package/src/service/prompts.ts +147 -0
  405. package/src/service/sanitizeHtml.test.ts +61 -0
  406. package/src/service/sanitizeHtml.ts +109 -0
  407. package/src/tests/bunSetup.test.ts +7 -4
  408. package/src/tests/bunSetup.ts +7 -7
  409. package/src/tests/chatHarness.ts +478 -0
  410. package/src/tests/generateStream.ts +49 -0
  411. package/src/tests/harnessErrors.ts +22 -0
  412. package/src/tests/harnessStreaming.ts +227 -0
  413. package/src/tests/helpers.ts +9 -2
  414. package/src/types/harness.ts +1119 -0
  415. package/src/types/index.ts +209 -13
  416. package/src/types/observability.ts +344 -0
  417. package/src/types/sanitize-html.d.ts +25 -0
@@ -0,0 +1,709 @@
1
+ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
2
+ import {existsSync, readFileSync} from "node:fs";
3
+ import {dirname, resolve} from "node:path";
4
+ import {getJobsService, Job, JobsApp, unregisterJobsService} from "@terreno/jobs";
5
+ import express from "express";
6
+ import {Duration} from "luxon";
7
+ import type mongoose from "mongoose";
8
+
9
+ import {createLocalObservabilityPlugin} from "../../observability/local/localPlugin";
10
+ import {registerObsSpan} from "../../observability/local/models/obsSpan";
11
+ import {registerObsTrace} from "../../observability/local/models/obsTrace";
12
+ import {harnessErrorMatching} from "../../tests/harnessErrors";
13
+ import type {HarnessRunnerContext, HarnessTaskDocument} from "../../types/harness";
14
+ import type {ObsSpanModel, ObsTraceModel} from "../../types/observability";
15
+ import {
16
+ type AnyHarnessTaskDefinition,
17
+ defineTask,
18
+ Harness,
19
+ HarnessCommitConflictError,
20
+ } from "../harness";
21
+ import {registerHarnessTask} from "../models/harnessTask";
22
+ import {claimTaskById, listRunnableTasks} from "../runtime";
23
+ import {HARNESS_PHASE_JOB_NAME, JobsRunner} from "./jobsRunner";
24
+
25
+ /** Roomy default so a stalled machine cannot trigger recovery in tests that count claims. */
26
+ const LEASE_MS = 5000;
27
+ /** Short lease for the tests that let a lease lapse on purpose. */
28
+ const SHORT_LEASE_MS = 400;
29
+ const HEARTBEAT_MS = 50;
30
+
31
+ const TaskModel = registerHarnessTask();
32
+ let SpanModel: ObsSpanModel;
33
+ let TraceModel: ObsTraceModel;
34
+
35
+ interface Gate {
36
+ promise: Promise<void>;
37
+ release: () => void;
38
+ }
39
+
40
+ const gate = (): Gate => {
41
+ let release: () => void = () => {};
42
+ const promise = new Promise<void>((done) => {
43
+ release = done;
44
+ });
45
+ return {promise, release};
46
+ };
47
+
48
+ const pause = (ms: number): Promise<void> => new Promise((done) => setTimeout(done, ms));
49
+
50
+ const waitUntil = async (
51
+ predicate: () => Promise<boolean> | boolean,
52
+ label: string
53
+ ): Promise<void> => {
54
+ for (let i = 0; i < 500; i++) {
55
+ if (await predicate()) {
56
+ return;
57
+ }
58
+ await pause(10);
59
+ }
60
+ throw new Error(`waitUntil timed out: ${label}`);
61
+ };
62
+
63
+ interface Worker {
64
+ /** Freeze every task lease renewal, as if the process stopped running. */
65
+ die: () => void;
66
+ harness: Harness;
67
+ jobsApp: JobsApp;
68
+ runner: JobsRunner;
69
+ }
70
+
71
+ const liveWorkers: Worker[] = [];
72
+ const hungGates: Gate[] = [];
73
+
74
+ /** One simulated instance: its own JobsApp worker, JobsRunner, and Harness on the shared DB. */
75
+ const openWorker = async ({
76
+ leaseMs = LEASE_MS,
77
+ ownerId,
78
+ registry,
79
+ }: {
80
+ leaseMs?: number;
81
+ ownerId: string;
82
+ registry: ReadonlyArray<AnyHarnessTaskDefinition>;
83
+ }): Promise<Worker> => {
84
+ let isDead = false;
85
+ const jobsApp = new JobsApp({pollIntervalMs: 10});
86
+ const runner = new JobsRunner({
87
+ heartbeatInterval: {milliseconds: HEARTBEAT_MS},
88
+ jobs: jobsApp,
89
+ leaseDuration: {milliseconds: leaseMs},
90
+ ownerId,
91
+ pollInterval: {milliseconds: 20},
92
+ });
93
+ jobsApp.register(express());
94
+ const harness = await Harness.open({
95
+ registry,
96
+ runner,
97
+ testHooks: {isHeartbeatSuspended: () => isDead},
98
+ });
99
+ const worker = {
100
+ die: () => {
101
+ isDead = true;
102
+ },
103
+ harness,
104
+ jobsApp,
105
+ runner,
106
+ };
107
+ liveWorkers.push(worker);
108
+ await harness.start();
109
+ await jobsApp.startWorker();
110
+ return worker;
111
+ };
112
+
113
+ const phaseJobKeys = async (taskId: mongoose.Types.ObjectId): Promise<string[]> => {
114
+ const jobs = await Job.find({
115
+ name: HARNESS_PHASE_JOB_NAME,
116
+ "payload.taskId": String(taskId),
117
+ }).sort({created: 1});
118
+ return jobs.map((job) => String(job.idempotencyKey));
119
+ };
120
+
121
+ const okPhaseSpans = async (task: HarnessTaskDocument): Promise<string[]> => {
122
+ const spans = await SpanModel.find({
123
+ _id: {$ne: task.rootSpanId},
124
+ status: "ok",
125
+ traceId: task.traceId,
126
+ }).sort({_id: 1});
127
+ return spans.map((span) => span.name);
128
+ };
129
+
130
+ const twoPhase = ({
131
+ name,
132
+ onFetch,
133
+ }: {
134
+ name: string;
135
+ onFetch: (rt: {owner: () => Promise<string>}) => Promise<void> | void;
136
+ }): AnyHarnessTaskDefinition =>
137
+ defineTask<{patientId: string}, {chart?: string}, {status: string}>({
138
+ initial: () => ({phase: "fetch", state: {}}),
139
+ name,
140
+ phases: {
141
+ fetch: {
142
+ replay: "safe",
143
+ run: async (task, rt) => {
144
+ await onFetch({
145
+ owner: async () =>
146
+ String((await TaskModel.findExactlyOne({_id: task.id})).lease?.owner),
147
+ });
148
+ await rt.commit({phase: "finish", state: {chart: `chart-${task.input.patientId}`}});
149
+ },
150
+ },
151
+ finish: {
152
+ run: async (task, rt) =>
153
+ rt.commit({
154
+ terminal: {result: {status: `filed ${task.state.chart}`}, status: "completed"},
155
+ }),
156
+ },
157
+ },
158
+ version: 1,
159
+ }) as unknown as AnyHarnessTaskDefinition;
160
+
161
+ describe("JobsRunner", () => {
162
+ beforeAll(() => {
163
+ createLocalObservabilityPlugin();
164
+ SpanModel = registerObsSpan();
165
+ TraceModel = registerObsTrace();
166
+ });
167
+
168
+ beforeEach(async () => {
169
+ await Promise.all([
170
+ Job.deleteMany({}),
171
+ TaskModel.deleteMany({}),
172
+ SpanModel.deleteMany({}),
173
+ TraceModel.deleteMany({}),
174
+ ]);
175
+ });
176
+
177
+ afterEach(async () => {
178
+ for (const hung of hungGates.splice(0)) {
179
+ hung.release();
180
+ }
181
+ for (const worker of liveWorkers.splice(0)) {
182
+ await worker.jobsApp.stopWorker();
183
+ await worker.harness.stop();
184
+ }
185
+ unregisterJobsService();
186
+ });
187
+
188
+ it("runs each phase as its own terreno.harness.phase job to completion", async () => {
189
+ const definition = twoPhase({name: "test.jobsTwoPhase", onFetch: () => {}});
190
+ const worker = await openWorker({ownerId: "worker-1", registry: [definition]});
191
+
192
+ const created = await worker.harness.createTask(definition as never, {patientId: "p1"});
193
+ const done = await worker.harness.waitForTask(created._id, {timeout: {seconds: 5}});
194
+
195
+ expect(done.status).toBe("completed");
196
+ expect(done.outcome?.result).toEqual({status: "filed chart-p1"});
197
+ expect(done.claims).toBe(2);
198
+ // One job per runnable visit: `taskId:phase:attempt:claims`.
199
+ expect(await phaseJobKeys(created._id)).toEqual([
200
+ `${created._id}:fetch:0:0`,
201
+ `${created._id}:finish:0:1`,
202
+ ]);
203
+ expect(await okPhaseSpans(done)).toEqual(["fetch", "finish"]);
204
+ });
205
+
206
+ it("lets a second job for a phase already running claim nothing, so the phase commits once", async () => {
207
+ const hung = gate();
208
+ hungGates.push(hung);
209
+ const started = gate();
210
+ let fetchRuns = 0;
211
+ const definition = twoPhase({
212
+ name: "test.jobsRace",
213
+ onFetch: async () => {
214
+ fetchRuns += 1;
215
+ started.release();
216
+ await hung.promise;
217
+ },
218
+ });
219
+ const first = await openWorker({ownerId: "worker-1", registry: [definition]});
220
+ const second = await openWorker({ownerId: "worker-2", registry: [definition]});
221
+
222
+ const created = await first.harness.createTask(definition as never, {patientId: "p2"});
223
+ await started.promise;
224
+ // Both workers race the same phase: a second job for the visit, as a re-dispatch makes.
225
+ await getJobsService().enqueue({
226
+ idempotencyKey: `${created._id}:fetch:0:0:duplicate`,
227
+ name: HARNESS_PHASE_JOB_NAME,
228
+ payload: {taskId: String(created._id)},
229
+ });
230
+ await waitUntil(
231
+ async () =>
232
+ (await Job.countDocuments({
233
+ idempotencyKey: `${created._id}:fetch:0:0:duplicate`,
234
+ status: "completed",
235
+ })) === 1,
236
+ "duplicate job finished without claiming"
237
+ );
238
+ hung.release();
239
+
240
+ const done = await second.harness.waitForTask(created._id, {timeout: {seconds: 5}});
241
+ expect(done.status).toBe("completed");
242
+ expect(fetchRuns).toBe(1);
243
+ expect(await okPhaseSpans(done)).toEqual(["fetch", "finish"]);
244
+ });
245
+
246
+ it("rejects a stale lease holder's commit after another worker recovers the phase", async () => {
247
+ const hung = gate();
248
+ hungGates.push(hung);
249
+ const owners: string[] = [];
250
+ let zombieError: unknown;
251
+ let fetchRuns = 0;
252
+ const definition = defineTask<{patientId: string}, {chart?: string}, {status: string}>({
253
+ initial: () => ({phase: "fetch", state: {}}),
254
+ name: "test.jobsStale",
255
+ phases: {
256
+ fetch: {
257
+ replay: "safe",
258
+ run: async (task, rt) => {
259
+ fetchRuns += 1;
260
+ const run = fetchRuns;
261
+ owners.push(String((await TaskModel.findExactlyOne({_id: task.id})).lease?.owner));
262
+ if (run === 1) {
263
+ await hung.promise;
264
+ try {
265
+ await rt.commit({phase: "finish", state: {chart: "from-the-dead-worker"}});
266
+ } catch (error: unknown) {
267
+ zombieError = error;
268
+ throw error;
269
+ }
270
+ return;
271
+ }
272
+ await rt.commit({phase: "finish", state: {chart: `chart-${task.input.patientId}`}});
273
+ },
274
+ },
275
+ finish: {
276
+ run: async (task, rt) =>
277
+ rt.commit({
278
+ terminal: {result: {status: `filed ${task.state.chart}`}, status: "completed"},
279
+ }),
280
+ },
281
+ },
282
+ version: 1,
283
+ }) as unknown as AnyHarnessTaskDefinition;
284
+ const first = await openWorker({
285
+ leaseMs: SHORT_LEASE_MS,
286
+ ownerId: "worker-1",
287
+ registry: [definition],
288
+ });
289
+ const created = await first.harness.createTask(definition as never, {patientId: "p3"});
290
+ await waitUntil(() => fetchRuns === 1, "worker-1 started fetch");
291
+ first.die();
292
+
293
+ const second = await openWorker({
294
+ leaseMs: SHORT_LEASE_MS,
295
+ ownerId: "worker-2",
296
+ registry: [definition],
297
+ });
298
+ const done = await second.harness.waitForTask(created._id, {timeout: {seconds: 5}});
299
+ expect(done.status).toBe("completed");
300
+ expect(done.state).toEqual({chart: "chart-p3"});
301
+
302
+ hung.release();
303
+ await waitUntil(() => zombieError !== undefined, "stale commit settled");
304
+ expect(zombieError).toBeInstanceOf(HarnessCommitConflictError);
305
+ expect(owners.slice(0, 2)).toEqual(["worker-1", expect.stringMatching(/^worker-[12]$/)]);
306
+ const after = await TaskModel.findExactlyOne({_id: created._id});
307
+ expect(after.state).toEqual({chart: "chart-p3"});
308
+ expect(await okPhaseSpans(after)).toEqual(["fetch", "finish"]);
309
+ // The recovered visit got its own job key, so it was not deduplicated away.
310
+ expect(await phaseJobKeys(created._id)).toContain(`${created._id}:fetch:0:1`);
311
+ });
312
+
313
+ it("re-dispatches past a chain of jobs that all ended without claiming the task", async () => {
314
+ const definition = twoPhase({name: "test.jobsRedispatch", onFetch: () => {}});
315
+ const jobsApp = new JobsApp();
316
+ jobsApp.define(HARNESS_PHASE_JOB_NAME, {handler: async () => {}});
317
+ jobsApp.register(express());
318
+ const created = await (
319
+ await Harness.open({registry: [definition], runner: new JobsRunner({jobs: new JobsApp()})})
320
+ ).createTask(definition as never, {patientId: "p4"});
321
+ // Six jobs for the first visit already ended unclaimed, e.g. on old instances mid-deploy.
322
+ const baseKey = `${created._id}:fetch:0:0`;
323
+ const chain: string[] = [];
324
+ let key = baseKey;
325
+ for (let i = 0; i < 6; i++) {
326
+ const ended = await getJobsService().enqueue({
327
+ idempotencyKey: key,
328
+ name: HARNESS_PHASE_JOB_NAME,
329
+ payload: {taskId: String(created._id)},
330
+ });
331
+ await Job.updateOne({_id: ended._id}, {$set: {status: i % 2 === 0 ? "dead" : "completed"}});
332
+ chain.push(key);
333
+ key = `${baseKey}:${ended._id}`;
334
+ }
335
+ unregisterJobsService();
336
+
337
+ const worker = await openWorker({ownerId: "worker-1", registry: [definition]});
338
+ const done = await worker.harness.waitForTask(created._id, {timeout: {seconds: 5}});
339
+
340
+ expect(done.status).toBe("completed");
341
+ expect((await phaseJobKeys(created._id)).slice(0, 7)).toEqual([...chain, key]);
342
+ });
343
+
344
+ it("dispatches a fresh job when an event wakes a task back into the same phase", async () => {
345
+ const waits: unknown[] = [];
346
+ const definition = defineTask<unknown, Record<string, never>, {status: string}>({
347
+ initial: () => ({phase: "review", state: {}}),
348
+ name: "test.jobsEventWake",
349
+ phases: {
350
+ review: {
351
+ run: async (_task, rt) => {
352
+ waits.push(await rt.waitFor("signed"));
353
+ await rt.commit({terminal: {result: {status: "signed"}, status: "completed"}});
354
+ },
355
+ },
356
+ },
357
+ version: 1,
358
+ }) as unknown as AnyHarnessTaskDefinition;
359
+ const worker = await openWorker({ownerId: "worker-1", registry: [definition]});
360
+ const created = await worker.harness.createTask(definition as never, {});
361
+ await waitUntil(
362
+ async () => (await TaskModel.findExactlyOne({_id: created._id})).status === "waiting",
363
+ "task waits for the event"
364
+ );
365
+
366
+ await worker.harness.sendEvent(created._id, "signed", {by: "clinician"});
367
+ const done = await worker.harness.waitForTask(created._id, {timeout: {seconds: 5}});
368
+
369
+ expect(done.status).toBe("completed");
370
+ expect(waits).toEqual([{by: "clinician"}]);
371
+ // Same phase and attempt both times; only `claims` tells the two visits apart.
372
+ expect(await phaseJobKeys(created._id)).toEqual([
373
+ `${created._id}:review:0:0`,
374
+ `${created._id}:review:0:1`,
375
+ ]);
376
+ });
377
+
378
+ it("ends a task aborted mid-phase as aborted instead of handing it back", async () => {
379
+ const hung = gate();
380
+ hungGates.push(hung);
381
+ const started = gate();
382
+ const definition = twoPhase({
383
+ name: "test.jobsAbort",
384
+ onFetch: async () => {
385
+ started.release();
386
+ await hung.promise;
387
+ },
388
+ });
389
+ const worker = await openWorker({ownerId: "worker-1", registry: [definition]});
390
+ const created = await worker.harness.createTask(definition as never, {patientId: "p6"});
391
+ await started.promise;
392
+
393
+ const abortion = worker.harness.abort(created._id, {reason: "patient withdrew"});
394
+ hung.release();
395
+ await abortion;
396
+ const done = await worker.harness.waitForTask(created._id, {timeout: {seconds: 5}});
397
+
398
+ expect(done.status).toBe("aborted");
399
+ expect(done.lease?.token).toBeUndefined();
400
+ // Whether fetch committed before the abort landed or not, the task never ran on.
401
+ const spans = await okPhaseSpans(done);
402
+ expect(spans).not.toContain("finish");
403
+ expect(spans.at(-1)).toBe("abort");
404
+ });
405
+
406
+ it("leaves a task whose lease moved on during the phase instead of handing it back", async () => {
407
+ const definition = twoPhase({name: "test.jobsLostHandoff", onFetch: () => {}});
408
+ const stolen = defineTask<{patientId: string}, Record<string, never>, {status: string}>({
409
+ initial: () => ({phase: "fetch", state: {}}),
410
+ name: "test.jobsStolen",
411
+ phases: {
412
+ fetch: {
413
+ run: async (task, rt) => {
414
+ await rt.commit({phase: "finish"});
415
+ // Another runner takes the lease after the commit, before the hand-off.
416
+ await TaskModel.updateOne({_id: task.id}, {$set: {"lease.token": "stolen"}});
417
+ },
418
+ },
419
+ finish: {run: async (_task, rt) => rt.commit({terminal: {status: "completed"}})},
420
+ },
421
+ version: 1,
422
+ }) as unknown as AnyHarnessTaskDefinition;
423
+ const worker = await openWorker({ownerId: "worker-1", registry: [definition, stolen]});
424
+
425
+ const created = await worker.harness.createTask(stolen as never, {patientId: "p5"});
426
+ await waitUntil(
427
+ async () =>
428
+ (await Job.countDocuments({
429
+ idempotencyKey: `${created._id}:fetch:0:0`,
430
+ status: "completed",
431
+ })) === 1,
432
+ "fetch job finished"
433
+ );
434
+ // Stop before asserting so recovery cannot park the stolen task meanwhile.
435
+ await worker.jobsApp.stopWorker();
436
+ await worker.harness.stop();
437
+
438
+ const task = await TaskModel.findExactlyOne({_id: created._id});
439
+ expect(task.status).toBe("running");
440
+ expect(task.phase).toBe("finish");
441
+ expect(task.lease?.token).toBe("stolen");
442
+ });
443
+
444
+ it("claims nothing for an unknown id and lists nothing without definitions", async () => {
445
+ const definition = twoPhase({name: "test.jobsNoClaim", onFetch: () => {}});
446
+ const definitions = new Map([["test.jobsNoClaim@1", definition as never]]);
447
+ const lease = {
448
+ duration: Duration.fromObject({seconds: 30}),
449
+ heartbeat: Duration.fromObject({seconds: 10}),
450
+ owner: "worker-1",
451
+ };
452
+ const models = {task: TaskModel} as never;
453
+
454
+ expect(await claimTaskById({definitions, lease, models, taskId: "not-an-id"})).toBeNull();
455
+ expect(await claimTaskById({definitions: new Map(), lease, models, taskId: "x"})).toBeNull();
456
+ expect(await listRunnableTasks({definitions: new Map(), limit: 10, models})).toEqual([]);
457
+ });
458
+
459
+ describe("phase job handler", () => {
460
+ const fakeContext = (overrides: Partial<HarnessRunnerContext>): HarnessRunnerContext => ({
461
+ acquireOwnerLease: async () => true,
462
+ claimNext: async () => null,
463
+ claimTask: async () => null,
464
+ listRunnable: async () => [],
465
+ recoverExpired: async () => 0,
466
+ releaseOwnerLease: async () => {},
467
+ runTask: async () => {},
468
+ ...overrides,
469
+ });
470
+
471
+ const phaseHandler = (jobsApp: JobsApp): ((payload: unknown) => Promise<void>) => {
472
+ const definition = jobsApp.getDefinition(HARNESS_PHASE_JOB_NAME);
473
+ if (!definition) {
474
+ throw new Error("phase job not defined");
475
+ }
476
+ return (payload) =>
477
+ definition.handler(payload, {
478
+ jobId: "job",
479
+ log: undefined as never,
480
+ signal: new AbortController().signal,
481
+ });
482
+ };
483
+
484
+ it("rejects a payload without a taskId", async () => {
485
+ const jobsApp = new JobsApp();
486
+ new JobsRunner({jobs: jobsApp});
487
+
488
+ await expect(phaseHandler(jobsApp)({})).rejects.toThrow(
489
+ harnessErrorMatching(
490
+ "invalidRequest",
491
+ "terreno.harness.phase payload needs a taskId string"
492
+ )
493
+ );
494
+ });
495
+
496
+ it("fails the job so it retries while the runner is stopped", async () => {
497
+ const jobsApp = new JobsApp();
498
+ new JobsRunner({jobs: jobsApp});
499
+
500
+ await expect(phaseHandler(jobsApp)({taskId: "abc"})).rejects.toThrow(
501
+ harnessErrorMatching("runnerStopped", "JobsRunner is not running; task abc stays runnable")
502
+ );
503
+ });
504
+
505
+ it("runs one phase of a claimed task and skips a task another worker claimed", async () => {
506
+ const jobsApp = new JobsApp();
507
+ jobsApp.register(express());
508
+ const runner = new JobsRunner({jobs: jobsApp, pollInterval: {minutes: 10}});
509
+ const runs: Array<{maxPhases?: number; taskId: string}> = [];
510
+ await runner.start(
511
+ fakeContext({
512
+ claimTask: async (taskId) =>
513
+ taskId === "free" ? ({_id: taskId} as unknown as HarnessTaskDocument) : null,
514
+ runTask: async (task, _lease, options) => {
515
+ runs.push({maxPhases: options?.maxPhases, taskId: String(task._id)});
516
+ },
517
+ })
518
+ );
519
+
520
+ await phaseHandler(jobsApp)({taskId: "taken"});
521
+ await phaseHandler(jobsApp)({taskId: "free"});
522
+ await runner.stop();
523
+
524
+ expect(runs).toEqual([{maxPhases: 1, taskId: "free"}]);
525
+ });
526
+
527
+ it("waits for a claim in flight before stop resolves", async () => {
528
+ const jobsApp = new JobsApp();
529
+ jobsApp.register(express());
530
+ const runner = new JobsRunner({jobs: jobsApp, pollInterval: {minutes: 10}});
531
+ const claimHeld = gate();
532
+ let isClaimDone = false;
533
+ await runner.start(
534
+ fakeContext({
535
+ claimTask: async () => {
536
+ await claimHeld.promise;
537
+ isClaimDone = true;
538
+ return null;
539
+ },
540
+ })
541
+ );
542
+ const job = phaseHandler(jobsApp)({taskId: "claiming"});
543
+ await pause(10);
544
+
545
+ const stopped = runner.stop();
546
+ await pause(20);
547
+ expect(isClaimDone).toBe(false);
548
+ claimHeld.release();
549
+ await stopped;
550
+ await job;
551
+
552
+ expect(isClaimDone).toBe(true);
553
+ });
554
+
555
+ it("waits for a phase in flight before stop resolves", async () => {
556
+ const jobsApp = new JobsApp();
557
+ jobsApp.register(express());
558
+ const runner = new JobsRunner({jobs: jobsApp, pollInterval: {minutes: 10}});
559
+ const held = gate();
560
+ let isPhaseDone = false;
561
+ await runner.start(
562
+ fakeContext({
563
+ claimTask: async (taskId) => ({_id: taskId}) as unknown as HarnessTaskDocument,
564
+ runTask: async () => {
565
+ await held.promise;
566
+ isPhaseDone = true;
567
+ },
568
+ })
569
+ );
570
+ const job = phaseHandler(jobsApp)({taskId: "slow"});
571
+ await pause(10);
572
+
573
+ const stopped = runner.stop();
574
+ await pause(20);
575
+ expect(isPhaseDone).toBe(false);
576
+ held.release();
577
+ await stopped;
578
+ await job;
579
+
580
+ expect(isPhaseDone).toBe(true);
581
+ });
582
+ });
583
+
584
+ it("keeps dispatching after a scan error", async () => {
585
+ const jobsApp = new JobsApp();
586
+ jobsApp.register(express());
587
+ const runner = new JobsRunner({jobs: jobsApp, pollInterval: {milliseconds: 5}});
588
+ let scans = 0;
589
+ await runner.start({
590
+ acquireOwnerLease: async () => true,
591
+ claimNext: async () => null,
592
+ claimTask: async () => null,
593
+ listRunnable: async () => {
594
+ scans += 1;
595
+ if (scans === 1) {
596
+ throw new Error("primary stepped down");
597
+ }
598
+ return [];
599
+ },
600
+ recoverExpired: async () => 0,
601
+ releaseOwnerLease: async () => {},
602
+ runTask: async () => {},
603
+ });
604
+ await waitUntil(() => scans >= 3, "dispatcher scanned again");
605
+ await runner.stop();
606
+ expect(scans).toBeGreaterThanOrEqual(3);
607
+ });
608
+
609
+ it("rejects a heartbeat that is not shorter than the lease and a second start", async () => {
610
+ expect(
611
+ () =>
612
+ new JobsRunner({
613
+ heartbeatInterval: {seconds: 30},
614
+ jobs: new JobsApp(),
615
+ leaseDuration: {seconds: 30},
616
+ })
617
+ ).toThrow(
618
+ harnessErrorMatching(
619
+ "configInvalid",
620
+ "JobsRunner heartbeatInterval must be positive and shorter than leaseDuration"
621
+ )
622
+ );
623
+
624
+ const runner = new JobsRunner({jobs: new JobsApp(), ownerId: "worker-x"});
625
+ expect(runner.ownerId).toBe("worker-x");
626
+ const registered = new JobsApp();
627
+ registered.register(express());
628
+ const context = {
629
+ acquireOwnerLease: async () => true,
630
+ claimNext: async () => null,
631
+ claimTask: async () => null,
632
+ listRunnable: async () => [],
633
+ recoverExpired: async () => 0,
634
+ releaseOwnerLease: async () => {},
635
+ runTask: async () => {},
636
+ };
637
+ await runner.start(context);
638
+ await expect(runner.start(context)).rejects.toThrow(
639
+ harnessErrorMatching("alreadyStarted", "JobsRunner is already started")
640
+ );
641
+ await runner.stop();
642
+ await runner.stop();
643
+ });
644
+
645
+ it("rejects a dispatchBatchSize that is not a positive integer", () => {
646
+ // A Mongo limit of 0 means "no limit", so 0 must not slip through.
647
+ for (const dispatchBatchSize of [0, -1, 2.5]) {
648
+ expect(() => new JobsRunner({dispatchBatchSize, jobs: new JobsApp()})).toThrow(
649
+ harnessErrorMatching(
650
+ "configInvalid",
651
+ "JobsRunner dispatchBatchSize must be a positive integer"
652
+ )
653
+ );
654
+ }
655
+ });
656
+
657
+ it("refuses to start before a JobsApp is registered", async () => {
658
+ const runner = new JobsRunner({jobs: new JobsApp()});
659
+
660
+ await expect(
661
+ runner.start({
662
+ acquireOwnerLease: async () => true,
663
+ claimNext: async () => null,
664
+ claimTask: async () => null,
665
+ listRunnable: async () => [],
666
+ recoverExpired: async () => 0,
667
+ releaseOwnerLease: async () => {},
668
+ runTask: async () => {},
669
+ })
670
+ ).rejects.toThrow(
671
+ harnessErrorMatching(
672
+ "configInvalid",
673
+ "JobsRunner needs a registered JobsApp: register it and build the app before harness.start()"
674
+ )
675
+ );
676
+ });
677
+
678
+ it("is not reachable from the root @terreno/ai or @terreno/ai/harness imports", () => {
679
+ const srcRoot = resolve(import.meta.dir, "../..");
680
+ const seen = new Set<string>();
681
+ const importsJobs: string[] = [];
682
+ const visit = (file: string): void => {
683
+ if (seen.has(file)) {
684
+ return;
685
+ }
686
+ seen.add(file);
687
+ const source = readFileSync(file, "utf8");
688
+ for (const match of source.matchAll(/(?:from|import)\s*\(?\s*"([^"]+)"/g)) {
689
+ const specifier = match[1];
690
+ if (specifier.startsWith("@terreno/jobs")) {
691
+ importsJobs.push(file);
692
+ }
693
+ if (specifier.startsWith(".")) {
694
+ const base = resolve(dirname(file), specifier.replace(/\.js$/, ""));
695
+ const candidates = [`${base}.ts`, `${base}.tsx`, resolve(base, "index.ts")];
696
+ const target = candidates.find((candidate) => existsSync(candidate));
697
+ if (target) {
698
+ visit(target);
699
+ }
700
+ }
701
+ }
702
+ };
703
+ visit(resolve(srcRoot, "index.ts"));
704
+ visit(resolve(srcRoot, "harness/harness.ts"));
705
+
706
+ expect(seen.size).toBeGreaterThan(20);
707
+ expect(importsJobs).toEqual([]);
708
+ });
709
+ });