@terreno/ai 57.9.0 → 57.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (417) hide show
  1. package/README.md +1 -0
  2. package/bunfig.toml +1 -1
  3. package/dist/aiApp.d.ts +8 -2
  4. package/dist/aiApp.js +12 -6
  5. package/dist/harness/adminScreens.d.ts +6 -0
  6. package/dist/harness/adminScreens.js +11 -0
  7. package/dist/harness/agentLoop.d.ts +61 -0
  8. package/dist/harness/agentLoop.js +798 -0
  9. package/dist/harness/agentTaskNames.d.ts +4 -0
  10. package/dist/harness/agentTaskNames.js +7 -0
  11. package/dist/harness/approvalPolicy.d.ts +3 -0
  12. package/dist/harness/approvalPolicy.js +23 -0
  13. package/dist/harness/approvals.d.ts +101 -0
  14. package/dist/harness/approvals.js +468 -0
  15. package/dist/harness/commit.d.ts +238 -0
  16. package/dist/harness/commit.js +678 -0
  17. package/dist/harness/conversation.d.ts +75 -0
  18. package/dist/harness/conversation.js +397 -0
  19. package/dist/harness/defineAgent.d.ts +9 -0
  20. package/dist/harness/defineAgent.js +65 -0
  21. package/dist/harness/defineTask.d.ts +11 -0
  22. package/dist/harness/defineTask.js +59 -0
  23. package/dist/harness/defineTool.d.ts +7 -0
  24. package/dist/harness/defineTool.js +36 -0
  25. package/dist/harness/definitionError.d.ts +8 -0
  26. package/dist/harness/definitionError.js +15 -0
  27. package/dist/harness/errors.d.ts +133 -0
  28. package/dist/harness/errors.js +107 -0
  29. package/dist/harness/eventHub.d.ts +45 -0
  30. package/dist/harness/eventHub.js +135 -0
  31. package/dist/harness/events.d.ts +95 -0
  32. package/dist/harness/events.js +267 -0
  33. package/dist/harness/executionEnv.d.ts +31 -0
  34. package/dist/harness/executionEnv.js +2 -0
  35. package/dist/harness/extensions.d.ts +82 -0
  36. package/dist/harness/extensions.js +274 -0
  37. package/dist/harness/harness.d.ts +163 -0
  38. package/dist/harness/harness.js +522 -0
  39. package/dist/harness/harnessApp.d.ts +44 -0
  40. package/dist/harness/harnessApp.js +62 -0
  41. package/dist/harness/internalRuntime.d.ts +16 -0
  42. package/dist/harness/internalRuntime.js +18 -0
  43. package/dist/harness/leases.d.ts +35 -0
  44. package/dist/harness/leases.js +86 -0
  45. package/dist/harness/memo.d.ts +24 -0
  46. package/dist/harness/memo.js +78 -0
  47. package/dist/harness/modelCall.d.ts +45 -0
  48. package/dist/harness/modelCall.js +147 -0
  49. package/dist/harness/models/harnessApproval.d.ts +2 -0
  50. package/dist/harness/models/harnessApproval.js +100 -0
  51. package/dist/harness/models/harnessConversation.d.ts +2 -0
  52. package/dist/harness/models/harnessConversation.js +117 -0
  53. package/dist/harness/models/harnessEvent.d.ts +3 -0
  54. package/dist/harness/models/harnessEvent.js +73 -0
  55. package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
  56. package/dist/harness/models/harnessInboxEvent.js +58 -0
  57. package/dist/harness/models/harnessMemo.d.ts +2 -0
  58. package/dist/harness/models/harnessMemo.js +42 -0
  59. package/dist/harness/models/harnessMessage.d.ts +2 -0
  60. package/dist/harness/models/harnessMessage.js +73 -0
  61. package/dist/harness/models/harnessOwner.d.ts +2 -0
  62. package/dist/harness/models/harnessOwner.js +36 -0
  63. package/dist/harness/models/harnessTask.d.ts +2 -0
  64. package/dist/harness/models/harnessTask.js +191 -0
  65. package/dist/harness/ownership.d.ts +89 -0
  66. package/dist/harness/ownership.js +438 -0
  67. package/dist/harness/registry.d.ts +29 -0
  68. package/dist/harness/registry.js +93 -0
  69. package/dist/harness/retryBackoff.d.ts +31 -0
  70. package/dist/harness/retryBackoff.js +50 -0
  71. package/dist/harness/routes/approvals.d.ts +13 -0
  72. package/dist/harness/routes/approvals.js +93 -0
  73. package/dist/harness/routes/conversations.d.ts +11 -0
  74. package/dist/harness/routes/conversations.js +66 -0
  75. package/dist/harness/routes/events.d.ts +19 -0
  76. package/dist/harness/routes/events.js +215 -0
  77. package/dist/harness/routes/tasks.d.ts +11 -0
  78. package/dist/harness/routes/tasks.js +108 -0
  79. package/dist/harness/runners/inProcessRunner.d.ts +64 -0
  80. package/dist/harness/runners/inProcessRunner.js +227 -0
  81. package/dist/harness/runners/jobsRunner.d.ts +62 -0
  82. package/dist/harness/runners/jobsRunner.js +212 -0
  83. package/dist/harness/runtime.d.ts +52 -0
  84. package/dist/harness/runtime.js +604 -0
  85. package/dist/harness/subagent.d.ts +53 -0
  86. package/dist/harness/subagent.js +134 -0
  87. package/dist/harness/suspend.d.ts +8 -0
  88. package/dist/harness/suspend.js +16 -0
  89. package/dist/harness/transaction.d.ts +3 -0
  90. package/dist/harness/transaction.js +22 -0
  91. package/dist/harness/waits.d.ts +100 -0
  92. package/dist/harness/waits.js +214 -0
  93. package/dist/index.d.ts +19 -1
  94. package/dist/index.js +45 -2
  95. package/dist/models/aiDataset.d.ts +2 -0
  96. package/dist/models/aiDataset.js +67 -0
  97. package/dist/models/aiRequest.js +1 -1
  98. package/dist/models/gptHistory.js +73 -0
  99. package/dist/observability/adminScreens.d.ts +5 -0
  100. package/dist/observability/adminScreens.js +32 -0
  101. package/dist/observability/compileTemplate.d.ts +1 -0
  102. package/dist/observability/compileTemplate.js +13 -0
  103. package/dist/observability/datasetImport.d.ts +11 -0
  104. package/dist/observability/datasetImport.js +185 -0
  105. package/dist/observability/evaluate.d.ts +31 -0
  106. package/dist/observability/evaluate.js +200 -0
  107. package/dist/observability/evaluatorTemplates.d.ts +17 -0
  108. package/dist/observability/evaluatorTemplates.js +111 -0
  109. package/dist/observability/jsonPath.d.ts +2 -0
  110. package/dist/observability/jsonPath.js +57 -0
  111. package/dist/observability/local/datasetStore.d.ts +87 -0
  112. package/dist/observability/local/datasetStore.js +326 -0
  113. package/dist/observability/local/evaluatorStore.d.ts +44 -0
  114. package/dist/observability/local/evaluatorStore.js +182 -0
  115. package/dist/observability/local/experimentRunner.d.ts +96 -0
  116. package/dist/observability/local/experimentRunner.js +419 -0
  117. package/dist/observability/local/localPlugin.d.ts +22 -0
  118. package/dist/observability/local/localPlugin.js +76 -0
  119. package/dist/observability/local/models/obsDataset.d.ts +2 -0
  120. package/dist/observability/local/models/obsDataset.js +37 -0
  121. package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
  122. package/dist/observability/local/models/obsDatasetItem.js +76 -0
  123. package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
  124. package/dist/observability/local/models/obsEvaluator.js +98 -0
  125. package/dist/observability/local/models/obsExperiment.d.ts +2 -0
  126. package/dist/observability/local/models/obsExperiment.js +87 -0
  127. package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
  128. package/dist/observability/local/models/obsExperimentItem.js +45 -0
  129. package/dist/observability/local/models/obsPrompt.d.ts +2 -0
  130. package/dist/observability/local/models/obsPrompt.js +42 -0
  131. package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
  132. package/dist/observability/local/models/obsPromptLabel.js +39 -0
  133. package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
  134. package/dist/observability/local/models/obsPromptVersion.js +96 -0
  135. package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
  136. package/dist/observability/local/models/obsReviewItem.js +75 -0
  137. package/dist/observability/local/models/obsScore.d.ts +2 -0
  138. package/dist/observability/local/models/obsScore.js +58 -0
  139. package/dist/observability/local/models/obsSpan.d.ts +2 -0
  140. package/dist/observability/local/models/obsSpan.js +64 -0
  141. package/dist/observability/local/models/obsTrace.d.ts +2 -0
  142. package/dist/observability/local/models/obsTrace.js +93 -0
  143. package/dist/observability/local/promptStore.d.ts +180 -0
  144. package/dist/observability/local/promptStore.js +444 -0
  145. package/dist/observability/local/reviewStore.d.ts +50 -0
  146. package/dist/observability/local/reviewStore.js +203 -0
  147. package/dist/observability/local/traceStore.d.ts +103 -0
  148. package/dist/observability/local/traceStore.js +325 -0
  149. package/dist/observability/observabilityApp.d.ts +24 -0
  150. package/dist/observability/observabilityApp.js +157 -0
  151. package/dist/observability/observabilityAppRegistry.d.ts +14 -0
  152. package/dist/observability/observabilityAppRegistry.js +16 -0
  153. package/dist/observability/permissions.d.ts +7 -0
  154. package/dist/observability/permissions.js +60 -0
  155. package/dist/observability/reviewPanels.d.ts +18 -0
  156. package/dist/observability/reviewPanels.js +46 -0
  157. package/dist/observability/routes/datasets.d.ts +7 -0
  158. package/dist/observability/routes/datasets.js +149 -0
  159. package/dist/observability/routes/evaluators.d.ts +7 -0
  160. package/dist/observability/routes/evaluators.js +123 -0
  161. package/dist/observability/routes/experiments.d.ts +7 -0
  162. package/dist/observability/routes/experiments.js +95 -0
  163. package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
  164. package/dist/observability/routes/observabilityRouteAccess.js +67 -0
  165. package/dist/observability/routes/prompts.d.ts +12 -0
  166. package/dist/observability/routes/prompts.js +172 -0
  167. package/dist/observability/routes/requestAiService.d.ts +13 -0
  168. package/dist/observability/routes/requestAiService.js +21 -0
  169. package/dist/observability/routes/review.d.ts +7 -0
  170. package/dist/observability/routes/review.js +79 -0
  171. package/dist/observability/routes/status.d.ts +5 -0
  172. package/dist/observability/routes/status.js +31 -0
  173. package/dist/observability/routes/testMultiStage.d.ts +9 -0
  174. package/dist/observability/routes/testMultiStage.js +67 -0
  175. package/dist/observability/routes/traces.d.ts +7 -0
  176. package/dist/observability/routes/traces.js +129 -0
  177. package/dist/observability/schemaValidation.d.ts +5 -0
  178. package/dist/observability/schemaValidation.js +26 -0
  179. package/dist/observability/sopThresholds.d.ts +2 -0
  180. package/dist/observability/sopThresholds.js +21 -0
  181. package/dist/observability/status.d.ts +23 -0
  182. package/dist/observability/status.js +34 -0
  183. package/dist/observability/testMultiStageSchemas.d.ts +10 -0
  184. package/dist/observability/testMultiStageSchemas.js +79 -0
  185. package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
  186. package/dist/observability/testMultiStageWorkflow.js +218 -0
  187. package/dist/observability/types.d.ts +159 -0
  188. package/dist/observability/types.js +67 -0
  189. package/dist/routes/files.js +18 -7
  190. package/dist/routes/gpt.js +79 -689
  191. package/dist/routes/gptActions.d.ts +9 -0
  192. package/dist/routes/gptActions.js +212 -0
  193. package/dist/routes/gptDatasets.d.ts +5 -0
  194. package/dist/routes/gptDatasets.js +94 -0
  195. package/dist/routes/gptHistories.d.ts +3 -1
  196. package/dist/routes/gptHistories.js +188 -2
  197. package/dist/service/aiDatasets.d.ts +57 -0
  198. package/dist/service/aiDatasets.js +253 -0
  199. package/dist/service/aiService.d.ts +50 -1
  200. package/dist/service/aiService.js +426 -77
  201. package/dist/service/askFiles.d.ts +49 -0
  202. package/dist/service/askFiles.js +162 -0
  203. package/dist/service/asks.d.ts +76 -0
  204. package/dist/service/asks.js +284 -0
  205. package/dist/service/chatTurn.d.ts +128 -0
  206. package/dist/service/chatTurn.js +1674 -0
  207. package/dist/service/fileStorage.d.ts +4 -0
  208. package/dist/service/fileStorage.js +7 -2
  209. package/dist/service/fillActionIds.d.ts +6 -0
  210. package/dist/service/fillActionIds.js +79 -0
  211. package/dist/service/jsonFenceModel.d.ts +7 -0
  212. package/dist/service/jsonFenceModel.js +45 -0
  213. package/dist/service/prompts.d.ts +52 -0
  214. package/dist/service/prompts.js +96 -1
  215. package/dist/service/sanitizeHtml.d.ts +11 -0
  216. package/dist/service/sanitizeHtml.js +108 -0
  217. package/dist/types/harness.d.ts +952 -0
  218. package/dist/types/harness.js +133 -0
  219. package/dist/types/index.d.ts +192 -14
  220. package/dist/types/index.js +2 -0
  221. package/dist/types/observability.d.ts +288 -0
  222. package/dist/types/observability.js +2 -0
  223. package/package.json +28 -5
  224. package/src/aiApp.test.ts +207 -1
  225. package/src/aiApp.ts +19 -6
  226. package/src/harness/adminScreens.test.ts +34 -0
  227. package/src/harness/adminScreens.ts +11 -0
  228. package/src/harness/agentLoop.test.ts +1034 -0
  229. package/src/harness/agentLoop.ts +1082 -0
  230. package/src/harness/agentRecovery.test.ts +380 -0
  231. package/src/harness/agentTaskNames.ts +4 -0
  232. package/src/harness/approvalPolicy.ts +28 -0
  233. package/src/harness/approvals.test.ts +1094 -0
  234. package/src/harness/approvals.ts +644 -0
  235. package/src/harness/commit.ts +1049 -0
  236. package/src/harness/concurrency.test.ts +155 -0
  237. package/src/harness/conversation.ts +503 -0
  238. package/src/harness/defineAgent.test.ts +92 -0
  239. package/src/harness/defineAgent.ts +82 -0
  240. package/src/harness/defineTask.test.ts +59 -0
  241. package/src/harness/defineTask.ts +76 -0
  242. package/src/harness/defineTool.ts +45 -0
  243. package/src/harness/definitionError.ts +13 -0
  244. package/src/harness/errors.test.ts +90 -0
  245. package/src/harness/errors.ts +125 -0
  246. package/src/harness/eventHub.test.ts +124 -0
  247. package/src/harness/eventHub.ts +159 -0
  248. package/src/harness/events.test.ts +769 -0
  249. package/src/harness/events.ts +384 -0
  250. package/src/harness/executionEnv.ts +34 -0
  251. package/src/harness/extensions.test.ts +1218 -0
  252. package/src/harness/extensions.ts +382 -0
  253. package/src/harness/harness.test.ts +693 -0
  254. package/src/harness/harness.ts +778 -0
  255. package/src/harness/harnessApp.ts +76 -0
  256. package/src/harness/internalRuntime.ts +44 -0
  257. package/src/harness/leases.ts +132 -0
  258. package/src/harness/memo.ts +107 -0
  259. package/src/harness/modelCall.test.ts +134 -0
  260. package/src/harness/modelCall.ts +200 -0
  261. package/src/harness/models/harnessApproval.ts +109 -0
  262. package/src/harness/models/harnessConversation.ts +142 -0
  263. package/src/harness/models/harnessEvent.ts +92 -0
  264. package/src/harness/models/harnessInboxEvent.ts +73 -0
  265. package/src/harness/models/harnessMemo.ts +43 -0
  266. package/src/harness/models/harnessMessage.ts +85 -0
  267. package/src/harness/models/harnessModels.test.ts +60 -0
  268. package/src/harness/models/harnessOwner.ts +40 -0
  269. package/src/harness/models/harnessTask.ts +206 -0
  270. package/src/harness/ownership.test.ts +1087 -0
  271. package/src/harness/ownership.ts +618 -0
  272. package/src/harness/recovery.test.ts +907 -0
  273. package/src/harness/registry.ts +124 -0
  274. package/src/harness/retries.test.ts +292 -0
  275. package/src/harness/retryBackoff.test.ts +100 -0
  276. package/src/harness/retryBackoff.ts +77 -0
  277. package/src/harness/routes/approvals.ts +123 -0
  278. package/src/harness/routes/conversations.test.ts +518 -0
  279. package/src/harness/routes/conversations.ts +102 -0
  280. package/src/harness/routes/events.ts +280 -0
  281. package/src/harness/routes/tasks.test.ts +198 -0
  282. package/src/harness/routes/tasks.ts +146 -0
  283. package/src/harness/runners/inProcessRunner.test.ts +530 -0
  284. package/src/harness/runners/inProcessRunner.ts +260 -0
  285. package/src/harness/runners/jobsRunner.test.ts +709 -0
  286. package/src/harness/runners/jobsRunner.ts +252 -0
  287. package/src/harness/runtime.ts +802 -0
  288. package/src/harness/subagent.test.ts +696 -0
  289. package/src/harness/subagent.ts +192 -0
  290. package/src/harness/suspend.ts +13 -0
  291. package/src/harness/transaction.ts +17 -0
  292. package/src/harness/versioning.test.ts +288 -0
  293. package/src/harness/waits.test.ts +693 -0
  294. package/src/harness/waits.ts +356 -0
  295. package/src/index.test.ts +13 -0
  296. package/src/index.ts +70 -0
  297. package/src/isolated/adminScreensConfig.isolated.ts +143 -0
  298. package/src/models/aiDataset.test.ts +34 -0
  299. package/src/models/aiDataset.ts +78 -0
  300. package/src/models/aiRequest.ts +2 -1
  301. package/src/models/gptHistory.test.ts +156 -0
  302. package/src/models/gptHistory.ts +89 -1
  303. package/src/observability/adminScreens.test.ts +71 -0
  304. package/src/observability/adminScreens.ts +42 -0
  305. package/src/observability/compileTemplate.test.ts +14 -0
  306. package/src/observability/compileTemplate.ts +10 -0
  307. package/src/observability/datasetImport.test.ts +95 -0
  308. package/src/observability/datasetImport.ts +220 -0
  309. package/src/observability/evaluate.test.ts +549 -0
  310. package/src/observability/evaluate.ts +254 -0
  311. package/src/observability/evaluatorTemplates.ts +124 -0
  312. package/src/observability/jsonPath.test.ts +52 -0
  313. package/src/observability/jsonPath.ts +60 -0
  314. package/src/observability/local/datasetStore.test.ts +230 -0
  315. package/src/observability/local/datasetStore.ts +452 -0
  316. package/src/observability/local/evaluatorStore.test.ts +138 -0
  317. package/src/observability/local/evaluatorStore.ts +224 -0
  318. package/src/observability/local/experimentRunner.test.ts +489 -0
  319. package/src/observability/local/experimentRunner.ts +548 -0
  320. package/src/observability/local/localPlugin.ts +88 -0
  321. package/src/observability/local/models/obsDataset.ts +38 -0
  322. package/src/observability/local/models/obsDatasetItem.ts +80 -0
  323. package/src/observability/local/models/obsEvaluator.ts +102 -0
  324. package/src/observability/local/models/obsExperiment.ts +91 -0
  325. package/src/observability/local/models/obsExperimentItem.ts +52 -0
  326. package/src/observability/local/models/obsPrompt.ts +43 -0
  327. package/src/observability/local/models/obsPromptLabel.ts +43 -0
  328. package/src/observability/local/models/obsPromptVersion.ts +107 -0
  329. package/src/observability/local/models/obsReviewItem.ts +79 -0
  330. package/src/observability/local/models/obsScore.ts +59 -0
  331. package/src/observability/local/models/obsSpan.ts +65 -0
  332. package/src/observability/local/models/obsTrace.ts +94 -0
  333. package/src/observability/local/obsModels.test.ts +119 -0
  334. package/src/observability/local/promptStore.test.ts +351 -0
  335. package/src/observability/local/promptStore.ts +665 -0
  336. package/src/observability/local/reviewStore.test.ts +164 -0
  337. package/src/observability/local/reviewStore.ts +264 -0
  338. package/src/observability/local/traceStore.test.ts +264 -0
  339. package/src/observability/local/traceStore.ts +440 -0
  340. package/src/observability/observabilityApp.test.ts +92 -0
  341. package/src/observability/observabilityApp.ts +185 -0
  342. package/src/observability/observabilityAppRegistry.test.ts +30 -0
  343. package/src/observability/observabilityAppRegistry.ts +35 -0
  344. package/src/observability/permissions.test.ts +96 -0
  345. package/src/observability/permissions.ts +78 -0
  346. package/src/observability/reviewPanels.test.ts +35 -0
  347. package/src/observability/reviewPanels.ts +64 -0
  348. package/src/observability/routes/datasets.test.ts +176 -0
  349. package/src/observability/routes/datasets.ts +252 -0
  350. package/src/observability/routes/evaluators.test.ts +119 -0
  351. package/src/observability/routes/evaluators.ts +207 -0
  352. package/src/observability/routes/experiments.test.ts +282 -0
  353. package/src/observability/routes/experiments.ts +152 -0
  354. package/src/observability/routes/observabilityRbac.test.ts +767 -0
  355. package/src/observability/routes/observabilityRouteAccess.ts +106 -0
  356. package/src/observability/routes/prompts.test.ts +320 -0
  357. package/src/observability/routes/prompts.ts +263 -0
  358. package/src/observability/routes/requestAiService.ts +36 -0
  359. package/src/observability/routes/review.test.ts +120 -0
  360. package/src/observability/routes/review.ts +125 -0
  361. package/src/observability/routes/status.test.ts +116 -0
  362. package/src/observability/routes/status.ts +47 -0
  363. package/src/observability/routes/testMultiStage.test.ts +214 -0
  364. package/src/observability/routes/testMultiStage.ts +96 -0
  365. package/src/observability/routes/traces.test.ts +167 -0
  366. package/src/observability/routes/traces.ts +172 -0
  367. package/src/observability/schemaValidation.ts +29 -0
  368. package/src/observability/sopThresholds.ts +20 -0
  369. package/src/observability/status.test.ts +38 -0
  370. package/src/observability/status.ts +56 -0
  371. package/src/observability/testMultiStageSchemas.ts +95 -0
  372. package/src/observability/testMultiStageWorkflow.test.ts +114 -0
  373. package/src/observability/testMultiStageWorkflow.ts +307 -0
  374. package/src/observability/types.ts +262 -0
  375. package/src/routes/files.test.ts +63 -9
  376. package/src/routes/files.ts +21 -7
  377. package/src/routes/gpt.test.ts +3279 -0
  378. package/src/routes/gpt.ts +91 -816
  379. package/src/routes/gptActions.test.ts +241 -0
  380. package/src/routes/gptActions.ts +255 -0
  381. package/src/routes/gptApprovals.test.ts +762 -0
  382. package/src/routes/gptDatasets.test.ts +86 -0
  383. package/src/routes/gptDatasets.ts +104 -0
  384. package/src/routes/gptFiles.test.ts +589 -0
  385. package/src/routes/gptHistories.test.ts +933 -0
  386. package/src/routes/gptHistories.ts +266 -5
  387. package/src/routes/routes.test.ts +222 -2
  388. package/src/service/aiDatasets.test.ts +123 -0
  389. package/src/service/aiDatasets.ts +316 -0
  390. package/src/service/aiService.test.ts +593 -0
  391. package/src/service/aiService.ts +568 -111
  392. package/src/service/askFiles.test.ts +96 -0
  393. package/src/service/askFiles.ts +222 -0
  394. package/src/service/asks.test.ts +946 -0
  395. package/src/service/asks.ts +423 -0
  396. package/src/service/chatTurn.ts +2287 -0
  397. package/src/service/fileStorage.test.ts +11 -0
  398. package/src/service/fileStorage.ts +10 -2
  399. package/src/service/fillActionIds.test.ts +65 -0
  400. package/src/service/fillActionIds.ts +98 -0
  401. package/src/service/jsonFenceModel.test.ts +47 -0
  402. package/src/service/jsonFenceModel.ts +53 -0
  403. package/src/service/prompts.test.ts +21 -0
  404. package/src/service/prompts.ts +147 -0
  405. package/src/service/sanitizeHtml.test.ts +61 -0
  406. package/src/service/sanitizeHtml.ts +109 -0
  407. package/src/tests/bunSetup.test.ts +7 -4
  408. package/src/tests/bunSetup.ts +7 -7
  409. package/src/tests/chatHarness.ts +478 -0
  410. package/src/tests/generateStream.ts +49 -0
  411. package/src/tests/harnessErrors.ts +22 -0
  412. package/src/tests/harnessStreaming.ts +227 -0
  413. package/src/tests/helpers.ts +9 -2
  414. package/src/types/harness.ts +1119 -0
  415. package/src/types/index.ts +209 -13
  416. package/src/types/observability.ts +344 -0
  417. package/src/types/sanitize-html.d.ts +25 -0
@@ -0,0 +1,1049 @@
1
+ import {randomUUID} from "node:crypto";
2
+ import {APIError} from "@terreno/api";
3
+ import {DateTime, Duration} from "luxon";
4
+ import mongoose, {type ClientSession} from "mongoose";
5
+
6
+ import type {
7
+ HarnessApprovalModel,
8
+ HarnessChildTaskOptions,
9
+ HarnessCommit,
10
+ HarnessConversationModel,
11
+ HarnessCreateTaskOptions,
12
+ HarnessEventModel,
13
+ HarnessEventStreamModel,
14
+ HarnessInboxEventModel,
15
+ HarnessLeaseSettings,
16
+ HarnessMemoModel,
17
+ HarnessMessageModel,
18
+ HarnessOwnerModel,
19
+ HarnessOwnership,
20
+ HarnessReplayPolicy,
21
+ HarnessResolveInterruptedOptions,
22
+ HarnessTaskDefinition,
23
+ HarnessTaskDocument,
24
+ HarnessTaskModel,
25
+ HarnessTaskSpanKind,
26
+ HarnessTestHooks,
27
+ HarnessWaitCall,
28
+ HarnessWaiting,
29
+ HarnessWaitKind,
30
+ } from "../types/harness";
31
+ import {
32
+ HARNESS_RESOLVE_ACTIONS,
33
+ HARNESS_TASK_STATUSES,
34
+ HARNESS_TERMINAL_STATUSES,
35
+ HARNESS_WAIT_KINDS,
36
+ } from "../types/harness";
37
+ import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
38
+ import {AGENT_TURN_TASK_NAME} from "./agentTaskNames";
39
+ import {HarnessDefinitionError} from "./definitionError";
40
+ import {HARNESS_ERRORS, harnessError} from "./errors";
41
+ import {appendTaskTransitionEvents} from "./events";
42
+ import {inTransaction} from "./transaction";
43
+
44
+ export interface HarnessModels {
45
+ approval: HarnessApprovalModel;
46
+ conversation: HarnessConversationModel;
47
+ event: HarnessEventModel;
48
+ eventStream: HarnessEventStreamModel;
49
+ inbox: HarnessInboxEventModel;
50
+ memo: HarnessMemoModel;
51
+ message: HarnessMessageModel;
52
+ owner: HarnessOwnerModel;
53
+ span: ObsSpanModel;
54
+ task: HarnessTaskModel;
55
+ trace: ObsTraceModel;
56
+ }
57
+
58
+ /**
59
+ * The task changed underneath this commit (checkpoint moved, status changed, or another
60
+ * runner holds the lease); nothing was written and the result is discarded.
61
+ */
62
+ export class HarnessCommitConflictError extends APIError {
63
+ constructor(taskId: string, phase: string) {
64
+ super({
65
+ ...HARNESS_ERRORS.commitConflict,
66
+ detail: `Harness commit for task ${taskId} phase "${phase}" lost its checkpoint fence`,
67
+ });
68
+ }
69
+ }
70
+
71
+ const DUPLICATE_KEY_CODE = 11000;
72
+
73
+ export const isDuplicateKeyError = (error: unknown): boolean => {
74
+ return (
75
+ typeof error === "object" &&
76
+ error !== null &&
77
+ "code" in error &&
78
+ (error as {code?: unknown}).code === DUPLICATE_KEY_CODE
79
+ );
80
+ };
81
+
82
+ /**
83
+ * Extra rows (messages, LLM spans, ...) written inside a commit's transaction, after the
84
+ * task update. Every write must pass `session`. `task` is the task as updated (or
85
+ * created); `traceStartedAt` anchors `startOffsetMs` for spans.
86
+ */
87
+ export type HarnessCommitWrites = (context: {
88
+ session: ClientSession;
89
+ task: HarnessTaskDocument;
90
+ traceStartedAt: DateTime;
91
+ }) => Promise<void>;
92
+
93
+ /** How the task's own audit span is named and typed. */
94
+ const taskSpanIdentity = (
95
+ definition: HarnessTaskDefinition,
96
+ input: unknown
97
+ ): {kind: HarnessTaskSpanKind; name: string} => ({
98
+ kind: definition.spanKind ?? "CHAIN",
99
+ name: definition.spanName ? definition.spanName(input) : definition.key,
100
+ });
101
+
102
+ /** Fresh lease fields for a phase that starts now under `lease.owner`. */
103
+ export const newTaskLease = (
104
+ lease: HarnessLeaseSettings
105
+ ): {acquiredAt: Date; expiresAt: Date; owner: string; token: string} => {
106
+ const now = DateTime.now();
107
+ return {
108
+ acquiredAt: now.toJSDate(),
109
+ expiresAt: now.plus(lease.duration).toJSDate(),
110
+ owner: lease.owner,
111
+ token: randomUUID(),
112
+ };
113
+ };
114
+
115
+ const toObjectId = (
116
+ value: mongoose.Types.ObjectId | string | undefined
117
+ ): mongoose.Types.ObjectId | undefined => {
118
+ if (value === undefined) {
119
+ return undefined;
120
+ }
121
+ return typeof value === "string" ? new mongoose.Types.ObjectId(value) : value;
122
+ };
123
+
124
+ const findByRequestId = async ({
125
+ definition,
126
+ models,
127
+ requestId,
128
+ userId,
129
+ }: {
130
+ definition: HarnessTaskDefinition;
131
+ models: HarnessModels;
132
+ requestId: string;
133
+ userId?: mongoose.Types.ObjectId;
134
+ }): Promise<HarnessTaskDocument> => {
135
+ // Soft-deleted owners still hold the unique index entry, so look past the deleted filter.
136
+ const existing = await models.task.findExactlyOne({deleted: {$in: [false, true]}, requestId});
137
+ // Never hand one user's task (input, state) to another caller reusing the key.
138
+ if (String(existing.userId ?? "") !== String(userId ?? "")) {
139
+ throw harnessError({
140
+ detail: `requestId "${requestId}" already belongs to a task for a different user`,
141
+ kind: "requestIdConflict",
142
+ });
143
+ }
144
+ if (existing.name !== definition.name) {
145
+ throw harnessError({
146
+ detail: `requestId "${requestId}" already belongs to task ${existing._id} (${existing.name}), not ${definition.name}`,
147
+ kind: "requestIdConflict",
148
+ });
149
+ }
150
+ return existing;
151
+ };
152
+
153
+ /**
154
+ * Insert a root task with its `ObsTrace` and root `CHAIN` span in one transaction.
155
+ * A repeated `requestId` hits the unique index, rolls the whole transaction back (so no
156
+ * orphan trace survives), and returns the task that already owns it.
157
+ */
158
+ export const createTaskRecords = async ({
159
+ definition,
160
+ input,
161
+ models,
162
+ options,
163
+ ownership = {kind: "root"},
164
+ writes,
165
+ }: {
166
+ definition: HarnessTaskDefinition;
167
+ input: unknown;
168
+ models: HarnessModels;
169
+ options: HarnessCreateTaskOptions;
170
+ /** Root tasks default to `{kind: "root"}`; conversation turns are owned by the conversation. */
171
+ ownership?: HarnessOwnership;
172
+ writes?: HarnessCommitWrites;
173
+ }): Promise<HarnessTaskDocument> => {
174
+ const initial = definition.initial(input);
175
+ if (!definition.phases[initial.phase]) {
176
+ throw new HarnessDefinitionError(
177
+ `${definition.key}: initial phase "${initial.phase}" is not one of ${Object.keys(definition.phases).join(", ")}`
178
+ );
179
+ }
180
+
181
+ const taskId = new mongoose.Types.ObjectId();
182
+ const traceId = new mongoose.Types.ObjectId();
183
+ const rootSpanId = new mongoose.Types.ObjectId();
184
+ const traceStartedAt = DateTime.now();
185
+ const startedAt = traceStartedAt.toJSDate();
186
+ const userId = toObjectId(options.userId);
187
+ const spanIdentity = taskSpanIdentity(definition, input);
188
+
189
+ try {
190
+ return await inTransaction(async (session) => {
191
+ await models.trace.create(
192
+ [{_id: traceId, input, name: definition.key, startedAt, status: "ok", userId}],
193
+ {session}
194
+ );
195
+ await models.span.create(
196
+ [
197
+ {
198
+ _id: rootSpanId,
199
+ input,
200
+ kind: spanIdentity.kind,
201
+ name: spanIdentity.name,
202
+ startedAt,
203
+ startOffsetMs: 0,
204
+ status: "ok",
205
+ traceId,
206
+ },
207
+ ],
208
+ {session}
209
+ );
210
+ const [task] = await models.task.create(
211
+ [
212
+ {
213
+ _id: taskId,
214
+ input,
215
+ name: definition.name,
216
+ ownership,
217
+ phase: initial.phase,
218
+ requestId: options.requestId,
219
+ retry: definition.retry,
220
+ rootSpanId,
221
+ rootTaskId: taskId,
222
+ state: initial.state,
223
+ status: HARNESS_TASK_STATUSES.pending,
224
+ traceId,
225
+ userId,
226
+ version: definition.version,
227
+ },
228
+ ],
229
+ {session}
230
+ );
231
+ await writes?.({session, task, traceStartedAt});
232
+ await appendTaskTransitionEvents({after: task, models, session});
233
+ return task;
234
+ });
235
+ } catch (error: unknown) {
236
+ // An earlier or concurrent create with the same requestId owns the unique index entry.
237
+ if (options.requestId && isDuplicateKeyError(error)) {
238
+ return findByRequestId({definition, models, requestId: options.requestId, userId});
239
+ }
240
+ throw error;
241
+ }
242
+ };
243
+
244
+ /** Whether `task` is the top of its ownership tree, and so owns the `ObsTrace`. */
245
+ const isRootTask = (task: HarnessTaskDocument): boolean =>
246
+ String(task._id) === String(task.rootTaskId);
247
+
248
+ /**
249
+ * Idempotency key that names one child of one attempt of one phase visit of `parent`. A
250
+ * re-run of the same attempt (wake, crash replay) finds the child again; a retry (next
251
+ * attempt) or a later visit of the phase creates a fresh one.
252
+ */
253
+ const childRequestId = ({key, parent}: {key: string; parent: HarnessTaskDocument}): string =>
254
+ `harness-child:${parent._id}:${parent.step ?? 0}:${parent.attempt ?? 0}:${key}`;
255
+
256
+ /**
257
+ * Insert a child task owned by `parent`, in `parent`'s trace, with its own `CHAIN` span
258
+ * nested under `parent`'s span. Fenced on `parent`'s lease token (renewing the lease in
259
+ * the same transaction), so a runner that lost the parent cannot create children. A
260
+ * repeated `key` within the same attempt of a phase visit returns the child created first.
261
+ */
262
+ export const createChildTaskRecords = async ({
263
+ definition,
264
+ input,
265
+ key,
266
+ lease,
267
+ models,
268
+ options,
269
+ parent,
270
+ span,
271
+ writes,
272
+ }: {
273
+ definition: HarnessTaskDefinition;
274
+ input: unknown;
275
+ key: string;
276
+ lease: HarnessLeaseSettings;
277
+ models: HarnessModels;
278
+ options: HarnessChildTaskOptions;
279
+ parent: HarnessTaskDocument;
280
+ /** Replaces the definition's span identity (and the span's input) for this child. */
281
+ span?: {input?: unknown; kind: HarnessTaskSpanKind; name: string};
282
+ /** Rows created with the child, in its transaction (a subagent's conversation). */
283
+ writes?: HarnessCommitWrites;
284
+ }): Promise<HarnessTaskDocument> => {
285
+ const initial = definition.initial(input);
286
+ if (!definition.phases[initial.phase]) {
287
+ throw new HarnessDefinitionError(
288
+ `${definition.key}: initial phase "${initial.phase}" is not one of ${Object.keys(definition.phases).join(", ")}`
289
+ );
290
+ }
291
+ const requestId = childRequestId({key, parent});
292
+ const existing = await models.task.findOneOrNone({deleted: {$in: [false, true]}, requestId});
293
+ if (existing) {
294
+ if (existing.name !== definition.name) {
295
+ throw harnessError({
296
+ detail: `Child key "${key}" already belongs to task ${existing._id} (${existing.name}), not ${definition.name}`,
297
+ kind: "requestIdConflict",
298
+ });
299
+ }
300
+ return existing;
301
+ }
302
+
303
+ const trace = await models.trace.findExactlyOne({_id: parent.traceId});
304
+ const taskId = new mongoose.Types.ObjectId();
305
+ const spanId = new mongoose.Types.ObjectId();
306
+ const startedAt = DateTime.now();
307
+ const spanIdentity = span ?? taskSpanIdentity(definition, input);
308
+ const spanInput = span && "input" in span ? span.input : input;
309
+
310
+ try {
311
+ return await inTransaction(async (session) => {
312
+ const fenced = await models.task.updateOne(
313
+ {
314
+ _id: parent._id,
315
+ "lease.token": parent.lease?.token ?? null,
316
+ phase: parent.phase,
317
+ status: HARNESS_TASK_STATUSES.running,
318
+ },
319
+ {$set: {"lease.expiresAt": startedAt.plus(lease.duration).toJSDate()}},
320
+ {session}
321
+ );
322
+ if (fenced.matchedCount === 0) {
323
+ throw new HarnessCommitConflictError(String(parent._id), parent.phase);
324
+ }
325
+ await models.span.create(
326
+ [
327
+ {
328
+ _id: spanId,
329
+ input: spanInput,
330
+ kind: spanIdentity.kind,
331
+ name: spanIdentity.name,
332
+ parentSpanId: parent.rootSpanId,
333
+ startedAt: startedAt.toJSDate(),
334
+ startOffsetMs: startedAt.diff(DateTime.fromJSDate(trace.startedAt)).toMillis(),
335
+ status: "ok",
336
+ traceId: parent.traceId,
337
+ },
338
+ ],
339
+ {session}
340
+ );
341
+ const [task] = await models.task.create(
342
+ [
343
+ {
344
+ _id: taskId,
345
+ ancestorIds: [...(parent.ancestorIds ?? []), parent._id],
346
+ background: options.background ?? false,
347
+ input,
348
+ name: definition.name,
349
+ ownership: {id: parent._id, kind: "task"},
350
+ phase: initial.phase,
351
+ requestId,
352
+ retry: definition.retry,
353
+ rootSpanId: spanId,
354
+ rootTaskId: parent.rootTaskId,
355
+ state: initial.state,
356
+ status: HARNESS_TASK_STATUSES.pending,
357
+ traceId: parent.traceId,
358
+ userId: parent.userId,
359
+ version: definition.version,
360
+ },
361
+ ],
362
+ {session}
363
+ );
364
+ await writes?.({session, task, traceStartedAt: DateTime.fromJSDate(trace.startedAt)});
365
+ await appendTaskTransitionEvents({after: task, models, session});
366
+ return task;
367
+ });
368
+ } catch (error: unknown) {
369
+ // A concurrent run of the same phase visit created this child first.
370
+ if (isDuplicateKeyError(error)) {
371
+ return models.task.findExactlyOne({deleted: {$in: [false, true]}, requestId});
372
+ }
373
+ throw error;
374
+ }
375
+ };
376
+
377
+ const describeNext = (next: HarnessCommit<unknown, unknown>): Record<string, unknown> => {
378
+ if ("terminal" in next) {
379
+ return {terminal: next.terminal};
380
+ }
381
+ return {phase: next.phase, state: next.state};
382
+ };
383
+
384
+ interface TransitionSpan {
385
+ error?: string;
386
+ input: unknown;
387
+ name: string;
388
+ output: unknown;
389
+ startedAt: DateTime;
390
+ status: "error" | "ok";
391
+ }
392
+
393
+ interface TransitionClose {
394
+ error?: string;
395
+ output: unknown;
396
+ status: "error" | "ok";
397
+ }
398
+
399
+ /**
400
+ * Apply one fenced task update and insert its `CHAIN` span (closing the root span and
401
+ * `ObsTrace` when `close` is set) in a single transaction. Throws
402
+ * `HarnessCommitConflictError`, writing nothing, when `filter` no longer matches.
403
+ */
404
+ const commitTransition = async ({
405
+ close,
406
+ filter,
407
+ models,
408
+ span,
409
+ task,
410
+ testHooks,
411
+ update,
412
+ writes,
413
+ }: {
414
+ close?: TransitionClose;
415
+ filter: Record<string, unknown>;
416
+ models: HarnessModels;
417
+ span: TransitionSpan;
418
+ task: HarnessTaskDocument;
419
+ testHooks?: HarnessTestHooks;
420
+ update: Record<string, unknown>;
421
+ writes?: HarnessCommitWrites;
422
+ }): Promise<HarnessTaskDocument> => {
423
+ const taskId = String(task._id);
424
+ const trace = await models.trace.findExactlyOne({_id: task.traceId});
425
+ const traceStartedAt = DateTime.fromJSDate(trace.startedAt);
426
+ // A root task's span starts with the trace; a child's span starts when it was created.
427
+ const taskSpanStartedAt =
428
+ close && !isRootTask(task)
429
+ ? DateTime.fromJSDate((await models.span.findExactlyOne({_id: task.rootSpanId})).startedAt)
430
+ : traceStartedAt;
431
+ const endedAt = DateTime.now();
432
+
433
+ return inTransaction(async (session) => {
434
+ const updated = await models.task.findOneAndUpdate({_id: task._id, ...filter}, update, {
435
+ returnDocument: "after",
436
+ session,
437
+ });
438
+ if (!updated) {
439
+ throw new HarnessCommitConflictError(taskId, task.phase);
440
+ }
441
+
442
+ await models.span.create(
443
+ [
444
+ {
445
+ durationMs: endedAt.diff(span.startedAt).toMillis(),
446
+ endedAt: endedAt.toJSDate(),
447
+ error: span.error,
448
+ input: span.input,
449
+ kind: "CHAIN",
450
+ name: span.name,
451
+ output: span.output,
452
+ parentSpanId: task.rootSpanId,
453
+ startedAt: span.startedAt.toJSDate(),
454
+ startOffsetMs: span.startedAt.diff(traceStartedAt).toMillis(),
455
+ status: span.status,
456
+ traceId: task.traceId,
457
+ },
458
+ ],
459
+ {session}
460
+ );
461
+
462
+ if (close) {
463
+ // A child closes its own span; only the root task closes the shared trace.
464
+ await models.span.updateOne(
465
+ {_id: task.rootSpanId},
466
+ {
467
+ $set: {
468
+ durationMs: endedAt.diff(taskSpanStartedAt).toMillis(),
469
+ endedAt: endedAt.toJSDate(),
470
+ error: close.error,
471
+ output: close.output,
472
+ status: close.status,
473
+ },
474
+ },
475
+ {session}
476
+ );
477
+ }
478
+ if (close && isRootTask(task)) {
479
+ await models.trace.updateOne(
480
+ {_id: task.traceId},
481
+ {
482
+ $set: {
483
+ endedAt: endedAt.toJSDate(),
484
+ errorSummary: close.error,
485
+ output: close.output,
486
+ status: close.status,
487
+ },
488
+ },
489
+ {session}
490
+ );
491
+ }
492
+
493
+ await writes?.({session, task: updated, traceStartedAt});
494
+ await appendTaskTransitionEvents({after: updated, before: task, models, session});
495
+ await releaseTurnConversation({models, session, task: updated});
496
+ await testHooks?.beforeCommitEnd?.({phase: task.phase, session, taskId});
497
+ return updated;
498
+ });
499
+ };
500
+
501
+ /**
502
+ * Free the conversation of a turn that just became terminal, in the same transaction, on
503
+ * every terminal path (answer, failure, abort, interruption, resolution). Fenced on the
504
+ * turn, so a conversation that already moved on is untouched.
505
+ */
506
+ const releaseTurnConversation = async ({
507
+ models,
508
+ session,
509
+ task,
510
+ }: {
511
+ models: HarnessModels;
512
+ session: ClientSession;
513
+ task: HarnessTaskDocument;
514
+ }): Promise<void> => {
515
+ if (task.name !== AGENT_TURN_TASK_NAME || !HARNESS_TERMINAL_STATUSES.has(task.status)) {
516
+ return;
517
+ }
518
+ const conversationId = (task.input as {conversationId?: unknown} | undefined)?.conversationId;
519
+ if (typeof conversationId !== "string") {
520
+ return;
521
+ }
522
+ await models.conversation.updateOne(
523
+ {_id: conversationId, activeTurnTaskId: task._id, status: "busy"},
524
+ {$set: {status: "idle"}, $unset: {activeTurnTaskId: 1}},
525
+ {session}
526
+ );
527
+ };
528
+
529
+ /** Fence for every commit made by the run that holds `task`'s current lease. */
530
+ const runFence = (task: HarnessTaskDocument): Record<string, unknown> => ({
531
+ "lease.token": task.lease?.token ?? null,
532
+ phase: task.phase,
533
+ status: HARNESS_TASK_STATUSES.running,
534
+ });
535
+
536
+ /**
537
+ * Commit one phase: update the task checkpoint (or terminal outcome) and insert the
538
+ * phase's `CHAIN` span in a single transaction. The commit is fenced by the task's
539
+ * status, phase, and lease token, so a runner that lost its lease writes nothing. A
540
+ * phase commit hands the next phase a fresh lease token; a terminal commit clears the
541
+ * lease and closes the root span and `ObsTrace`.
542
+ */
543
+ export const commitPhase = async ({
544
+ failedAttempts,
545
+ lease,
546
+ models,
547
+ next,
548
+ phaseStartedAt,
549
+ task,
550
+ testHooks,
551
+ writes,
552
+ }: {
553
+ /** Set when the phase failed for good; recorded as the task's `attempt`. */
554
+ failedAttempts?: number;
555
+ lease: HarnessLeaseSettings;
556
+ models: HarnessModels;
557
+ next: HarnessCommit<unknown, unknown>;
558
+ phaseStartedAt: DateTime;
559
+ task: HarnessTaskDocument;
560
+ testHooks?: HarnessTestHooks;
561
+ /** Extra rows committed in the same transaction (agent messages, LLM spans). */
562
+ writes?: HarnessCommitWrites;
563
+ }): Promise<HarnessTaskDocument> => {
564
+ const isTerminal = "terminal" in next;
565
+ const error = isTerminal && next.terminal.status === "failed" ? next.terminal.error : undefined;
566
+ const isFailure = error !== undefined;
567
+
568
+ const update = isTerminal
569
+ ? {
570
+ $set: {
571
+ outcome: next.terminal,
572
+ status: next.terminal.status,
573
+ ...(failedAttempts === undefined ? {} : {attempt: failedAttempts}),
574
+ },
575
+ $unset: {lease: 1, runAt: 1, waits: 1},
576
+ }
577
+ : {
578
+ $inc: {step: 1},
579
+ $set: {
580
+ attempt: 0,
581
+ lease: newTaskLease(lease),
582
+ phase: next.phase,
583
+ state: next.state === undefined ? task.state : next.state,
584
+ },
585
+ // Wait records belong to one phase visit; the next visit starts with none.
586
+ $unset: {runAt: 1, waits: 1},
587
+ };
588
+
589
+ return commitTransition({
590
+ close: isTerminal
591
+ ? {
592
+ error,
593
+ output: next.terminal.status === "failed" ? {error} : next.terminal.result,
594
+ status: isFailure ? "error" : "ok",
595
+ }
596
+ : undefined,
597
+ // A task without a lease (written before leases existed) is fenced on "no token".
598
+ filter: runFence(task),
599
+ models,
600
+ span: {
601
+ error,
602
+ input: {attempt: task.attempt, state: task.state},
603
+ name: task.phase,
604
+ output: describeNext(next),
605
+ startedAt: phaseStartedAt,
606
+ status: isFailure ? "error" : "ok",
607
+ },
608
+ task,
609
+ testHooks,
610
+ update,
611
+ writes,
612
+ });
613
+ };
614
+
615
+ /**
616
+ * Record a thrown phase that has attempts left: back to `pending` at the same checkpoint
617
+ * with `attempt` raised and `runAt` set to the backoff time, plus an error `CHAIN` span,
618
+ * in one transaction fenced on the run's lease token.
619
+ */
620
+ export const commitRetry = async ({
621
+ error,
622
+ failedAttempts,
623
+ maxAttempts,
624
+ models,
625
+ phaseStartedAt,
626
+ runAt,
627
+ task,
628
+ testHooks,
629
+ }: {
630
+ error: string;
631
+ failedAttempts: number;
632
+ maxAttempts: number;
633
+ models: HarnessModels;
634
+ phaseStartedAt: DateTime;
635
+ runAt: DateTime;
636
+ task: HarnessTaskDocument;
637
+ testHooks?: HarnessTestHooks;
638
+ }): Promise<HarnessTaskDocument> => {
639
+ return commitTransition({
640
+ filter: runFence(task),
641
+ models,
642
+ span: {
643
+ error,
644
+ input: {attempt: task.attempt, state: task.state},
645
+ name: task.phase,
646
+ output: {retry: {attempt: failedAttempts, maxAttempts, runAt: runAt.toISO()}},
647
+ startedAt: phaseStartedAt,
648
+ status: "error",
649
+ },
650
+ task,
651
+ testHooks,
652
+ update: {
653
+ $set: {
654
+ attempt: failedAttempts,
655
+ runAt: runAt.toJSDate(),
656
+ status: HARNESS_TASK_STATUSES.pending,
657
+ },
658
+ $unset: {lease: 1},
659
+ },
660
+ });
661
+ };
662
+
663
+ /**
664
+ * Park a running task as `waiting` on `waiting` (child tasks, an event, or a sleep). Gives
665
+ * up the lease so no runner holds the task while it waits; the phase re-runs from its
666
+ * checkpoint once the wait is satisfied. `waitCall` records the `rt.waitFor` / `rt.sleep`
667
+ * call that started the wait. An event wait checks the inbox inside the transaction and,
668
+ * when a matching event is already buffered, returns the task straight to `pending`:
669
+ * `sendEvent` writes the task in its own transaction, so the two can never miss each other.
670
+ */
671
+ export const commitWaiting = async ({
672
+ models,
673
+ phaseStartedAt,
674
+ task,
675
+ testHooks,
676
+ waitCall,
677
+ waiting,
678
+ writes,
679
+ }: {
680
+ models: HarnessModels;
681
+ phaseStartedAt: DateTime;
682
+ task: HarnessTaskDocument;
683
+ testHooks?: HarnessTestHooks;
684
+ waitCall?: {entry: HarnessWaitCall; key: string};
685
+ waiting: HarnessWaiting;
686
+ /** Extra rows committed with the wait (an approval request). */
687
+ writes?: HarnessCommitWrites;
688
+ }): Promise<{isWoken: boolean; task: HarnessTaskDocument}> => {
689
+ let isWoken = false;
690
+ const committed = await commitTransition({
691
+ filter: runFence(task),
692
+ models,
693
+ span: {
694
+ input: {attempt: task.attempt, state: task.state},
695
+ name: task.phase,
696
+ output: {
697
+ waiting: {
698
+ ...waiting,
699
+ taskIds: waiting.taskIds?.map(String),
700
+ timeoutAt: waiting.timeoutAt?.toISOString(),
701
+ },
702
+ },
703
+ startedAt: phaseStartedAt,
704
+ status: "ok",
705
+ },
706
+ task,
707
+ testHooks,
708
+ update: {
709
+ $set: {
710
+ status: HARNESS_TASK_STATUSES.waiting,
711
+ waiting,
712
+ ...(waitCall ? {[`waits.${waitCall.key}`]: waitCall.entry} : {}),
713
+ },
714
+ $unset: {lease: 1},
715
+ },
716
+ writes: async (context) => {
717
+ const {session} = context;
718
+ isWoken = false;
719
+ await writes?.(context);
720
+ if (waiting.kind !== HARNESS_WAIT_KINDS.event || !waiting.key) {
721
+ return;
722
+ }
723
+ const buffered = await models.inbox.countDocuments(
724
+ {consumedKey: {$exists: false}, name: waiting.key, taskId: task._id},
725
+ {session}
726
+ );
727
+ if (buffered === 0) {
728
+ return;
729
+ }
730
+ const woken = await models.task.findOneAndUpdate(
731
+ {_id: task._id},
732
+ {$set: {status: HARNESS_TASK_STATUSES.pending}, $unset: {waiting: 1}},
733
+ {returnDocument: "after", session}
734
+ );
735
+ isWoken = true;
736
+ if (woken) {
737
+ await appendTaskTransitionEvents({after: woken, before: context.task, models, session});
738
+ }
739
+ },
740
+ });
741
+ return {isWoken, task: committed};
742
+ };
743
+
744
+ /**
745
+ * Record how one `rt.waitFor` / `rt.sleep` call resolved, plus its resume span, in one
746
+ * transaction fenced on the run's lease. `writes` claims the delivered event in the same
747
+ * transaction. The phase keeps running under its lease.
748
+ */
749
+ export const commitWaitResolution = async ({
750
+ callKey,
751
+ entry,
752
+ lease,
753
+ models,
754
+ span,
755
+ task,
756
+ testHooks,
757
+ writes,
758
+ }: {
759
+ callKey: string;
760
+ entry: HarnessWaitCall;
761
+ lease: HarnessLeaseSettings;
762
+ models: HarnessModels;
763
+ span: {input: unknown; name: string; output: unknown};
764
+ task: HarnessTaskDocument;
765
+ testHooks?: HarnessTestHooks;
766
+ writes?: HarnessCommitWrites;
767
+ }): Promise<void> => {
768
+ await commitTransition({
769
+ filter: runFence(task),
770
+ models,
771
+ span: {...span, startedAt: DateTime.fromJSDate(entry.startedAt), status: "ok"},
772
+ task,
773
+ testHooks,
774
+ update: {
775
+ $set: {
776
+ "lease.expiresAt": DateTime.now().plus(lease.duration).toJSDate(),
777
+ [`waits.${callKey}`]: entry,
778
+ },
779
+ },
780
+ writes,
781
+ });
782
+ };
783
+
784
+ /**
785
+ * Return a `waiting` task to `pending` at the same checkpoint. Fenced on the wait it was
786
+ * parked on, so only one waker wins and a task that left `waiting` is untouched. Returns
787
+ * whether this call woke it.
788
+ */
789
+ export const wakeWaitingTask = async ({
790
+ kind,
791
+ models,
792
+ taskId,
793
+ }: {
794
+ kind: HarnessWaitKind;
795
+ models: HarnessModels;
796
+ taskId: mongoose.Types.ObjectId;
797
+ }): Promise<boolean> => {
798
+ const result = await models.task.updateOne(
799
+ {_id: taskId, status: HARNESS_TASK_STATUSES.waiting, "waiting.kind": kind},
800
+ {$set: {status: HARNESS_TASK_STATUSES.pending}, $unset: {waiting: 1}}
801
+ );
802
+ return result.modifiedCount > 0;
803
+ };
804
+
805
+ /**
806
+ * First step of an abort: record the request (which stops runners from claiming the
807
+ * task) and, when a phase is running, rotate its lease token so the run's commits,
808
+ * child creation, and lease renewals all fail from here on. Returns the task as it now
809
+ * stands, or null when it is already terminal.
810
+ */
811
+ export const fenceForAbort = async ({
812
+ models,
813
+ reason,
814
+ taskId,
815
+ userId,
816
+ }: {
817
+ models: HarnessModels;
818
+ reason: string;
819
+ taskId: mongoose.Types.ObjectId;
820
+ userId?: string;
821
+ }): Promise<HarnessTaskDocument | null> => {
822
+ const nonTerminal = {$nin: [...HARNESS_TERMINAL_STATUSES]};
823
+ // The first request wins; a repeated abort keeps its record (and any handler claim).
824
+ await models.task.updateOne(
825
+ {_id: taskId, "abortRequested.at": {$exists: false}, status: nonTerminal},
826
+ {$set: {abortRequested: {at: DateTime.now().toJSDate(), reason, userId: toObjectId(userId)}}}
827
+ );
828
+ const requested = await models.task.findOneOrNone({_id: taskId, status: nonTerminal});
829
+ if (!requested || requested.status !== HARNESS_TASK_STATUSES.running) {
830
+ return requested;
831
+ }
832
+ // Not fenced on the old token: a phase that just committed holds a newer one.
833
+ const revoked = await models.task.findOneAndUpdate(
834
+ {_id: taskId, status: HARNESS_TASK_STATUSES.running},
835
+ {$set: {"lease.token": `abort:${randomUUID()}`}},
836
+ {returnDocument: "after"}
837
+ );
838
+ return revoked ?? models.task.findOneOrNone({_id: taskId, status: nonTerminal});
839
+ };
840
+
841
+ /**
842
+ * How long one aborter holds a task's abort-handler claim. Another abort of the same task
843
+ * waits for the claim holder to finish, and takes over only after this lapses (the holder
844
+ * died mid-handler).
845
+ */
846
+ const ABORT_HANDLER_CLAIM = Duration.fromObject({minutes: 1});
847
+
848
+ /**
849
+ * Claim the right to run `taskId`'s abort handler and commit its abort, so concurrent
850
+ * aborts (an operator and a failFast sibling, say) never run one handler twice. Returns
851
+ * false while another live aborter holds the claim, or once the task is terminal.
852
+ */
853
+ export const claimAbortHandler = async ({
854
+ models,
855
+ taskId,
856
+ }: {
857
+ models: HarnessModels;
858
+ taskId: mongoose.Types.ObjectId;
859
+ }): Promise<boolean> => {
860
+ const now = DateTime.now();
861
+ const claimed = await models.task.updateOne(
862
+ {
863
+ _id: taskId,
864
+ $or: [
865
+ {"abortRequested.handlerClaimExpiresAt": {$exists: false}},
866
+ {"abortRequested.handlerClaimExpiresAt": {$lte: now.toJSDate()}},
867
+ ],
868
+ status: {$nin: [...HARNESS_TERMINAL_STATUSES]},
869
+ },
870
+ {$set: {"abortRequested.handlerClaimExpiresAt": now.plus(ABORT_HANDLER_CLAIM).toJSDate()}}
871
+ );
872
+ return claimed.modifiedCount > 0;
873
+ };
874
+
875
+ /**
876
+ * Mark a task `aborted` with `outcome.error`, clear its lease and wait, close its span
877
+ * (and the trace for a root task), and write one audit span, in one transaction. Fenced
878
+ * on `filter` (status only, not the lease token: an abort deliberately overrides the
879
+ * run in flight, whose later commit is then rejected).
880
+ */
881
+ export const commitAbort = async ({
882
+ error,
883
+ filter,
884
+ models,
885
+ span,
886
+ task,
887
+ testHooks,
888
+ }: {
889
+ error: string;
890
+ filter: Record<string, unknown>;
891
+ models: HarnessModels;
892
+ span: {error?: string; name: string; output: unknown; status: "error" | "ok"};
893
+ task: HarnessTaskDocument;
894
+ testHooks?: HarnessTestHooks;
895
+ }): Promise<HarnessTaskDocument> => {
896
+ return commitTransition({
897
+ close: {error, output: {error}, status: "error"},
898
+ filter,
899
+ models,
900
+ span: {
901
+ ...span,
902
+ input: {phase: task.phase, state: task.state, status: task.status},
903
+ startedAt: DateTime.now(),
904
+ },
905
+ task,
906
+ testHooks,
907
+ update: {
908
+ $set: {
909
+ outcome: {error, status: HARNESS_TASK_STATUSES.aborted},
910
+ status: HARNESS_TASK_STATUSES.aborted,
911
+ },
912
+ $unset: {lease: 1, runAt: 1, waiting: 1, waits: 1},
913
+ },
914
+ });
915
+ };
916
+
917
+ /** Mongo filter for a `running` task whose lease lapsed (or that never had one). */
918
+ export const expiredLeaseFilter = (now: DateTime): Record<string, unknown> => ({
919
+ $or: [{"lease.expiresAt": {$lte: now.toJSDate()}}, {"lease.expiresAt": {$exists: false}}],
920
+ status: HARNESS_TASK_STATUSES.running,
921
+ });
922
+
923
+ /** What recovery does with a task whose lease expired mid-phase. */
924
+ export type HarnessInterruptionAction = "fail" | "park" | "replay";
925
+
926
+ const interruptionOutcomes = (
927
+ leaseOwner: string
928
+ ): Record<
929
+ HarnessInterruptionAction,
930
+ {error: string; replay: HarnessReplayPolicy; status: HarnessTaskDocument["status"]}
931
+ > => ({
932
+ fail: {
933
+ error: `Interrupted, not retried: lease of ${leaseOwner} expired mid-phase (replay: never)`,
934
+ replay: "never",
935
+ status: HARNESS_TASK_STATUSES.failed,
936
+ },
937
+ park: {
938
+ error: `Interrupted: lease of ${leaseOwner} expired mid-phase; awaiting resolveInterrupted (replay: never)`,
939
+ replay: "never",
940
+ status: HARNESS_TASK_STATUSES.interrupted,
941
+ },
942
+ replay: {
943
+ error: `Interrupted: lease of ${leaseOwner} expired mid-phase; re-running (replay: safe)`,
944
+ replay: "safe",
945
+ status: HARNESS_TASK_STATUSES.pending,
946
+ },
947
+ });
948
+
949
+ /**
950
+ * Record that a task's lease expired mid-phase. `replay` returns it to `pending` at the
951
+ * same checkpoint so a runner re-runs the phase; `park` sets `interrupted` for
952
+ * `resolveInterrupted`; `fail` (a `replay: "never"` phase of an `onInterrupt: "fail"`
953
+ * task) ends it `failed` with an "Interrupted, not retried" outcome and closes its span.
954
+ * Each writes an error `CHAIN` span in the same transaction. Fenced on the expired lease,
955
+ * so a renewed lease is left alone.
956
+ */
957
+ export const commitInterruption = async ({
958
+ action,
959
+ models,
960
+ task,
961
+ testHooks,
962
+ }: {
963
+ action: HarnessInterruptionAction;
964
+ models: HarnessModels;
965
+ task: HarnessTaskDocument;
966
+ testHooks?: HarnessTestHooks;
967
+ }): Promise<HarnessTaskDocument> => {
968
+ const now = DateTime.now();
969
+ const leaseOwner = task.lease?.owner ?? "unknown";
970
+ const {error, replay, status} = interruptionOutcomes(leaseOwner)[action];
971
+ const isFailure = action === "fail";
972
+
973
+ return commitTransition({
974
+ close: isFailure ? {error, output: {error}, status: "error"} : undefined,
975
+ filter: {
976
+ ...expiredLeaseFilter(now),
977
+ phase: task.phase,
978
+ // Fence on the token from the snapshot so the span names the lease that actually lapsed
979
+ ...(task.lease?.token ? {"lease.token": task.lease.token} : {}),
980
+ },
981
+ models,
982
+ span: {
983
+ error,
984
+ input: {attempt: task.attempt, state: task.state},
985
+ name: task.phase,
986
+ output: {interrupted: true, leaseOwner, replay, status},
987
+ startedAt: task.lease?.acquiredAt ? DateTime.fromJSDate(task.lease.acquiredAt) : now,
988
+ status: "error",
989
+ },
990
+ task,
991
+ testHooks,
992
+ update: isFailure
993
+ ? {$set: {outcome: {error, status}, status}, $unset: {lease: 1, runAt: 1, waits: 1}}
994
+ : {$set: {status}, $unset: {lease: 1}},
995
+ });
996
+ };
997
+
998
+ /**
999
+ * Apply an operator's `retry` or `complete` decision to an `interrupted` task and audit
1000
+ * it with a `resolveInterrupted` span recording action, reason, and `decidedBy`. An
1001
+ * `abort` decision goes through the abort path so the task's abort handler runs.
1002
+ */
1003
+ export const commitResolution = async ({
1004
+ models,
1005
+ options,
1006
+ task,
1007
+ testHooks,
1008
+ }: {
1009
+ models: HarnessModels;
1010
+ options: HarnessResolveInterruptedOptions & {action: "complete" | "retry"};
1011
+ task: HarnessTaskDocument;
1012
+ testHooks?: HarnessTestHooks;
1013
+ }): Promise<HarnessTaskDocument> => {
1014
+ const {action, reason, result} = options;
1015
+ const decidedBy = options.userId === undefined ? undefined : String(options.userId);
1016
+
1017
+ const resolutions = {
1018
+ [HARNESS_RESOLVE_ACTIONS.complete]: {
1019
+ close: {output: result, status: "ok" as const},
1020
+ update: {
1021
+ $set: {
1022
+ outcome: {result, status: HARNESS_TASK_STATUSES.completed},
1023
+ status: HARNESS_TASK_STATUSES.completed,
1024
+ },
1025
+ },
1026
+ },
1027
+ [HARNESS_RESOLVE_ACTIONS.retry]: {
1028
+ close: undefined,
1029
+ update: {$set: {status: HARNESS_TASK_STATUSES.pending}},
1030
+ },
1031
+ };
1032
+ const resolution = resolutions[action];
1033
+
1034
+ return commitTransition({
1035
+ close: resolution.close,
1036
+ filter: {phase: task.phase, status: HARNESS_TASK_STATUSES.interrupted},
1037
+ models,
1038
+ span: {
1039
+ input: {phase: task.phase, state: task.state},
1040
+ name: "resolveInterrupted",
1041
+ output: {action, decidedBy, phase: task.phase, reason, result},
1042
+ startedAt: DateTime.now(),
1043
+ status: "ok",
1044
+ },
1045
+ task,
1046
+ testHooks,
1047
+ update: resolution.update,
1048
+ });
1049
+ };