@penclipai/server 2026.426.0 → 2026.505.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (253) hide show
  1. package/dist/adapters/builtin-adapter-types.d.ts.map +1 -1
  2. package/dist/adapters/builtin-adapter-types.js +3 -0
  3. package/dist/adapters/builtin-adapter-types.js.map +1 -1
  4. package/dist/adapters/index.d.ts +2 -2
  5. package/dist/adapters/index.d.ts.map +1 -1
  6. package/dist/adapters/index.js +1 -1
  7. package/dist/adapters/index.js.map +1 -1
  8. package/dist/adapters/registry.d.ts +2 -1
  9. package/dist/adapters/registry.d.ts.map +1 -1
  10. package/dist/adapters/registry.js +76 -6
  11. package/dist/adapters/registry.js.map +1 -1
  12. package/dist/adapters/types.d.ts +1 -1
  13. package/dist/adapters/types.d.ts.map +1 -1
  14. package/dist/adapters/utils.d.ts.map +1 -1
  15. package/dist/adapters/utils.js +2 -1
  16. package/dist/adapters/utils.js.map +1 -1
  17. package/dist/attachment-types.d.ts +1 -16
  18. package/dist/attachment-types.d.ts.map +1 -1
  19. package/dist/attachment-types.js +7 -0
  20. package/dist/attachment-types.js.map +1 -1
  21. package/dist/auth/better-auth.d.ts +3 -1
  22. package/dist/auth/better-auth.d.ts.map +1 -1
  23. package/dist/auth/better-auth.js +8 -2
  24. package/dist/auth/better-auth.js.map +1 -1
  25. package/dist/index.d.ts.map +1 -1
  26. package/dist/index.js +27 -13
  27. package/dist/index.js.map +1 -1
  28. package/dist/middleware/auth.d.ts.map +1 -1
  29. package/dist/middleware/auth.js +143 -2
  30. package/dist/middleware/auth.js.map +1 -1
  31. package/dist/onboarding-assets/ceo/AGENTS.md +1 -1
  32. package/dist/onboarding-assets/ceo/HEARTBEAT.md +5 -5
  33. package/dist/redaction.d.ts.map +1 -1
  34. package/dist/redaction.js +30 -12
  35. package/dist/redaction.js.map +1 -1
  36. package/dist/routes/access.d.ts.map +1 -1
  37. package/dist/routes/access.js +10 -0
  38. package/dist/routes/access.js.map +1 -1
  39. package/dist/routes/activity.d.ts.map +1 -1
  40. package/dist/routes/activity.js +4 -2
  41. package/dist/routes/activity.js.map +1 -1
  42. package/dist/routes/adapters.d.ts.map +1 -1
  43. package/dist/routes/adapters.js +1 -0
  44. package/dist/routes/adapters.js.map +1 -1
  45. package/dist/routes/agents.d.ts.map +1 -1
  46. package/dist/routes/agents.js +317 -56
  47. package/dist/routes/agents.js.map +1 -1
  48. package/dist/routes/costs.d.ts.map +1 -1
  49. package/dist/routes/costs.js +21 -2
  50. package/dist/routes/costs.js.map +1 -1
  51. package/dist/routes/instance-settings.d.ts.map +1 -1
  52. package/dist/routes/instance-settings.js +37 -2
  53. package/dist/routes/instance-settings.js.map +1 -1
  54. package/dist/routes/issue-tree-control.d.ts.map +1 -1
  55. package/dist/routes/issue-tree-control.js +3 -1
  56. package/dist/routes/issue-tree-control.js.map +1 -1
  57. package/dist/routes/issues.d.ts.map +1 -1
  58. package/dist/routes/issues.js +257 -32
  59. package/dist/routes/issues.js.map +1 -1
  60. package/dist/routes/projects.d.ts.map +1 -1
  61. package/dist/routes/projects.js +10 -3
  62. package/dist/routes/projects.js.map +1 -1
  63. package/dist/routes/routines.d.ts.map +1 -1
  64. package/dist/routes/routines.js +6 -1
  65. package/dist/routes/routines.js.map +1 -1
  66. package/dist/routes/workspace-command-authz.d.ts +1 -1
  67. package/dist/routes/workspace-command-authz.d.ts.map +1 -1
  68. package/dist/routes/workspace-command-authz.js +2 -2
  69. package/dist/routes/workspace-command-authz.js.map +1 -1
  70. package/dist/runtime-api.d.ts +4 -0
  71. package/dist/runtime-api.d.ts.map +1 -1
  72. package/dist/runtime-api.js +38 -10
  73. package/dist/runtime-api.js.map +1 -1
  74. package/dist/services/companies.d.ts +6 -0
  75. package/dist/services/companies.d.ts.map +1 -1
  76. package/dist/services/companies.js +1 -0
  77. package/dist/services/companies.js.map +1 -1
  78. package/dist/services/company-portability.d.ts.map +1 -1
  79. package/dist/services/company-portability.js +16 -15
  80. package/dist/services/company-portability.js.map +1 -1
  81. package/dist/services/costs.d.ts +9 -0
  82. package/dist/services/costs.d.ts.map +1 -1
  83. package/dist/services/costs.js +45 -1
  84. package/dist/services/costs.js.map +1 -1
  85. package/dist/services/environment-execution-target.d.ts.map +1 -1
  86. package/dist/services/environment-execution-target.js +7 -13
  87. package/dist/services/environment-execution-target.js.map +1 -1
  88. package/dist/services/environment-run-orchestrator.d.ts.map +1 -1
  89. package/dist/services/environment-run-orchestrator.js +56 -0
  90. package/dist/services/environment-run-orchestrator.js.map +1 -1
  91. package/dist/services/environment-runtime.d.ts +2 -0
  92. package/dist/services/environment-runtime.d.ts.map +1 -1
  93. package/dist/services/environment-runtime.js +80 -39
  94. package/dist/services/environment-runtime.js.map +1 -1
  95. package/dist/services/heartbeat-stop-metadata.d.ts +2 -1
  96. package/dist/services/heartbeat-stop-metadata.d.ts.map +1 -1
  97. package/dist/services/heartbeat-stop-metadata.js +10 -1
  98. package/dist/services/heartbeat-stop-metadata.js.map +1 -1
  99. package/dist/services/heartbeat-stop-metadata.test.js +24 -0
  100. package/dist/services/heartbeat-stop-metadata.test.js.map +1 -1
  101. package/dist/services/heartbeat.d.ts +156 -5
  102. package/dist/services/heartbeat.d.ts.map +1 -1
  103. package/dist/services/heartbeat.js +1384 -112
  104. package/dist/services/heartbeat.js.map +1 -1
  105. package/dist/services/index.d.ts +1 -0
  106. package/dist/services/index.d.ts.map +1 -1
  107. package/dist/services/index.js +1 -0
  108. package/dist/services/index.js.map +1 -1
  109. package/dist/services/instance-settings.d.ts.map +1 -1
  110. package/dist/services/instance-settings.js +4 -1
  111. package/dist/services/instance-settings.js.map +1 -1
  112. package/dist/services/issue-execution-policy.d.ts +56 -1
  113. package/dist/services/issue-execution-policy.d.ts.map +1 -1
  114. package/dist/services/issue-execution-policy.js +400 -2
  115. package/dist/services/issue-execution-policy.js.map +1 -1
  116. package/dist/services/issue-thread-interactions.d.ts +5 -1
  117. package/dist/services/issue-thread-interactions.d.ts.map +1 -1
  118. package/dist/services/issue-thread-interactions.js +44 -1
  119. package/dist/services/issue-thread-interactions.js.map +1 -1
  120. package/dist/services/issue-tree-control.d.ts +1 -0
  121. package/dist/services/issue-tree-control.d.ts.map +1 -1
  122. package/dist/services/issue-tree-control.js +84 -4
  123. package/dist/services/issue-tree-control.js.map +1 -1
  124. package/dist/services/issues.d.ts +10 -1
  125. package/dist/services/issues.d.ts.map +1 -1
  126. package/dist/services/issues.js +452 -48
  127. package/dist/services/issues.js.map +1 -1
  128. package/dist/services/plugin-environment-driver.d.ts +4 -0
  129. package/dist/services/plugin-environment-driver.d.ts.map +1 -1
  130. package/dist/services/plugin-environment-driver.js +18 -1
  131. package/dist/services/plugin-environment-driver.js.map +1 -1
  132. package/dist/services/productivity-review.d.ts +83 -0
  133. package/dist/services/productivity-review.d.ts.map +1 -0
  134. package/dist/services/productivity-review.js +650 -0
  135. package/dist/services/productivity-review.js.map +1 -0
  136. package/dist/services/recovery/index.d.ts +1 -1
  137. package/dist/services/recovery/index.d.ts.map +1 -1
  138. package/dist/services/recovery/index.js +1 -1
  139. package/dist/services/recovery/index.js.map +1 -1
  140. package/dist/services/recovery/issue-graph-liveness.d.ts +13 -1
  141. package/dist/services/recovery/issue-graph-liveness.d.ts.map +1 -1
  142. package/dist/services/recovery/issue-graph-liveness.js +212 -92
  143. package/dist/services/recovery/issue-graph-liveness.js.map +1 -1
  144. package/dist/services/recovery/origins.d.ts +2 -0
  145. package/dist/services/recovery/origins.d.ts.map +1 -1
  146. package/dist/services/recovery/origins.js +4 -0
  147. package/dist/services/recovery/origins.js.map +1 -1
  148. package/dist/services/recovery/run-liveness-continuations.d.ts.map +1 -1
  149. package/dist/services/recovery/run-liveness-continuations.js.map +1 -1
  150. package/dist/services/recovery/service.d.ts +20 -2
  151. package/dist/services/recovery/service.d.ts.map +1 -1
  152. package/dist/services/recovery/service.js +405 -63
  153. package/dist/services/recovery/service.js.map +1 -1
  154. package/dist/services/routines.d.ts +5 -2
  155. package/dist/services/routines.d.ts.map +1 -1
  156. package/dist/services/routines.js +47 -3
  157. package/dist/services/routines.js.map +1 -1
  158. package/dist/worktree-config.d.ts.map +1 -1
  159. package/dist/worktree-config.js +2 -5
  160. package/dist/worktree-config.js.map +1 -1
  161. package/package.json +16 -15
  162. package/skills/diagnose-why-work-stopped/SKILL.md +161 -0
  163. package/skills/paperclip/SKILL.md +37 -26
  164. package/skills/paperclip/references/api-reference.md +6 -2
  165. package/skills/paperclip-converting-plans-to-tasks/SKILL.md +42 -0
  166. package/skills/paperclip-create-agent/SKILL.md +3 -2
  167. package/skills/paperclip-create-agent/references/agent-instruction-templates.md +1 -1
  168. package/skills/paperclip-create-agent/references/api-reference.md +7 -2
  169. package/skills/paperclip-create-agent/references/baseline-role-guide.md +1 -1
  170. package/skills/paperclip-create-agent/references/draft-review-checklist.md +2 -2
  171. package/skills/paperclip-dev/SKILL.md +267 -0
  172. package/skills/terminal-bench-loop/SKILL.md +236 -0
  173. package/ui-dist/assets/{_basePickBy-BRqa7PJ5.js → _basePickBy-BS0Fg_DB.js} +1 -1
  174. package/ui-dist/assets/{_baseUniq-DhE2yrXC.js → _baseUniq-Dtnt_4SE.js} +1 -1
  175. package/ui-dist/assets/{arc-7qnikTQ3.js → arc-BCoOPxh5.js} +1 -1
  176. package/ui-dist/assets/{architectureDiagram-VXUJARFQ-CH0wVUOM.js → architectureDiagram-VXUJARFQ-C6eX2QUo.js} +1 -1
  177. package/ui-dist/assets/{blockDiagram-VD42YOAC-CeeRyJQX.js → blockDiagram-VD42YOAC-aUueUD4B.js} +1 -1
  178. package/ui-dist/assets/browser-ponyfill-BlAfsWm_.js +2 -0
  179. package/ui-dist/assets/{c4Diagram-YG6GDRKO-C_cV0CGo.js → c4Diagram-YG6GDRKO-CfPWRlOF.js} +1 -1
  180. package/ui-dist/assets/channel-ChNSCFJf.js +1 -0
  181. package/ui-dist/assets/{chunk-4BX2VUAB-DQ6pxPVT.js → chunk-4BX2VUAB-BTD1apA4.js} +1 -1
  182. package/ui-dist/assets/{chunk-55IACEB6-L8pS0IoX.js → chunk-55IACEB6-BXXF_ClN.js} +1 -1
  183. package/ui-dist/assets/{chunk-B4BG7PRW-BZKGE88E.js → chunk-B4BG7PRW-hAZeWGP8.js} +1 -1
  184. package/ui-dist/assets/{chunk-DI55MBZ5-CefSoZ_K.js → chunk-DI55MBZ5-cOH3UoEl.js} +1 -1
  185. package/ui-dist/assets/{chunk-FMBD7UC4-Bc3qTTHB.js → chunk-FMBD7UC4-Cu2yZOcl.js} +1 -1
  186. package/ui-dist/assets/{chunk-QN33PNHL-CjWBr5bI.js → chunk-QN33PNHL-0DNN5aRU.js} +1 -1
  187. package/ui-dist/assets/{chunk-QZHKN3VN-C0JUdmmz.js → chunk-QZHKN3VN-B9_bhK2n.js} +1 -1
  188. package/ui-dist/assets/{chunk-TZMSLE5B-D4d4I82z.js → chunk-TZMSLE5B-Cr5xwxio.js} +1 -1
  189. package/ui-dist/assets/classDiagram-2ON5EDUG-4aK1QZU3.js +1 -0
  190. package/ui-dist/assets/classDiagram-v2-WZHVMYZB-4aK1QZU3.js +1 -0
  191. package/ui-dist/assets/clone-C8lk5Qbc.js +1 -0
  192. package/ui-dist/assets/{cose-bilkent-S5V4N54A-B09h9XGZ.js → cose-bilkent-S5V4N54A-6_Dw6gpQ.js} +1 -1
  193. package/ui-dist/assets/{dagre-6UL2VRFP-CA02PXuX.js → dagre-6UL2VRFP-CFBhlh5H.js} +1 -1
  194. package/ui-dist/assets/{diagram-PSM6KHXK-DaT9cnrY.js → diagram-PSM6KHXK-C88ftcah.js} +1 -1
  195. package/ui-dist/assets/{diagram-QEK2KX5R-Drwc3gBw.js → diagram-QEK2KX5R-9EUupcuH.js} +1 -1
  196. package/ui-dist/assets/{diagram-S2PKOQOG-CpsGCaT6.js → diagram-S2PKOQOG-Dsml0wWh.js} +1 -1
  197. package/ui-dist/assets/{erDiagram-Q2GNP2WA-CVkBh9TY.js → erDiagram-Q2GNP2WA-sM-XdfHS.js} +1 -1
  198. package/ui-dist/assets/{flowDiagram-NV44I4VS-De9sXvPR.js → flowDiagram-NV44I4VS-qll7oaoW.js} +1 -1
  199. package/ui-dist/assets/{ganttDiagram-JELNMOA3-CSFa0gXS.js → ganttDiagram-JELNMOA3-VWnJMcjC.js} +1 -1
  200. package/ui-dist/assets/{gitGraphDiagram-V2S2FVAM-DEJaChxa.js → gitGraphDiagram-V2S2FVAM-DFnocrfl.js} +1 -1
  201. package/ui-dist/assets/{graph-D2R4DCtu.js → graph-nq3Qye4Z.js} +1 -1
  202. package/ui-dist/assets/{index-DEG-9CFs.js → index-3Owzaheh.js} +1 -1
  203. package/ui-dist/assets/{index-DHnKx9xX.js → index-B2A-a635.js} +1 -1
  204. package/ui-dist/assets/{index-C1I0SGDm.js → index-BGFrRiqa.js} +1 -1
  205. package/ui-dist/assets/{index-B44EtLRv.js → index-BVC5UhRK.js} +1 -1
  206. package/ui-dist/assets/{index-C_dAXwxT.js → index-BrP1U_Hy.js} +1 -1
  207. package/ui-dist/assets/{index-flZjKn_n.js → index-CXXHGqM8.js} +1 -1
  208. package/ui-dist/assets/{index-ssM_UKPW.js → index-CgyPAauR.js} +1 -1
  209. package/ui-dist/assets/{index-Ct1AraKR.js → index-CksQ4Ytv.js} +1 -1
  210. package/ui-dist/assets/{index-DQ6I_vpd.js → index-CrNzj2vZ.js} +1 -1
  211. package/ui-dist/assets/{index-DzZID5RY.js → index-CxbZBH3M.js} +1 -1
  212. package/ui-dist/assets/{index-Cn6_RRY5.js → index-D-dSSrf-.js} +1 -1
  213. package/ui-dist/assets/{index-CVa2OHgx.js → index-D6uZ_7Vh.js} +1 -1
  214. package/ui-dist/assets/{index-BzjWQd50.js → index-D7JGmxas.js} +1 -1
  215. package/ui-dist/assets/{index-CnT1_9UF.js → index-DDqO9GAq.js} +1 -1
  216. package/ui-dist/assets/index-DEUtmlPm.js +513 -0
  217. package/ui-dist/assets/{index-D2fEhyQg.js → index-DF5RDSoK.js} +1 -1
  218. package/ui-dist/assets/{index-CZGNe8K3.js → index-DfI92epU.js} +1 -1
  219. package/ui-dist/assets/{index-ByamXtyB.js → index-Dukb9MDQ.js} +1 -1
  220. package/ui-dist/assets/index-HP73_6Vr.css +1 -0
  221. package/ui-dist/assets/{index-BJS4rvUh.js → index-NXDTW2n4.js} +1 -1
  222. package/ui-dist/assets/{index-Bad5Hy7e.js → index-SxPPG9ig.js} +1 -1
  223. package/ui-dist/assets/{index-CC51mhhA.js → index-lC4Yz3Gw.js} +1 -1
  224. package/ui-dist/assets/{index-BFzkl36p.js → index-q2RXGI2V.js} +1 -1
  225. package/ui-dist/assets/{index-40icqWwg.js → index-qjfdrS96.js} +1 -1
  226. package/ui-dist/assets/{infoDiagram-HS3SLOUP-CJcjzWkM.js → infoDiagram-HS3SLOUP-CTrK5xoS.js} +1 -1
  227. package/ui-dist/assets/{journeyDiagram-XKPGCS4Q-ByITI00s.js → journeyDiagram-XKPGCS4Q-YFC7FykG.js} +1 -1
  228. package/ui-dist/assets/{kanban-definition-3W4ZIXB7-DvEjKke-.js → kanban-definition-3W4ZIXB7-B3dlyva0.js} +1 -1
  229. package/ui-dist/assets/{layout-CZcd66hi.js → layout-DefunPTK.js} +1 -1
  230. package/ui-dist/assets/{linear-jTUy3iHu.js → linear-CIPvzeMv.js} +1 -1
  231. package/ui-dist/assets/{mermaid.core-DECSZPbJ.js → mermaid.core-zKYhmnnR.js} +4 -4
  232. package/ui-dist/assets/{mindmap-definition-VGOIOE7T-Twtu17_c.js → mindmap-definition-VGOIOE7T-BlU-ebRa.js} +1 -1
  233. package/ui-dist/assets/{pieDiagram-ADFJNKIX-DlbgZ010.js → pieDiagram-ADFJNKIX-Ceto4LXH.js} +1 -1
  234. package/ui-dist/assets/{quadrantDiagram-AYHSOK5B-CMAa3qAT.js → quadrantDiagram-AYHSOK5B-C6M6hkuE.js} +1 -1
  235. package/ui-dist/assets/{requirementDiagram-UZGBJVZJ-CXRTfJOe.js → requirementDiagram-UZGBJVZJ-B-bcG938.js} +1 -1
  236. package/ui-dist/assets/{sankeyDiagram-TZEHDZUN-DeyO4fer.js → sankeyDiagram-TZEHDZUN-CIqty6Qi.js} +1 -1
  237. package/ui-dist/assets/{sequenceDiagram-WL72ISMW-Ch8wlJIL.js → sequenceDiagram-WL72ISMW-CIt2R5tk.js} +1 -1
  238. package/ui-dist/assets/{stateDiagram-FKZM4ZOC-BgL_AAl9.js → stateDiagram-FKZM4ZOC-BC1RFlfg.js} +1 -1
  239. package/ui-dist/assets/stateDiagram-v2-4FDKWEC3-Iy6tYSSw.js +1 -0
  240. package/ui-dist/assets/{timeline-definition-IT6M3QCI-D1QWd7TQ.js → timeline-definition-IT6M3QCI-DZqvoU94.js} +1 -1
  241. package/ui-dist/assets/{treemap-GDKQZRPO-B5RkmUv8.js → treemap-GDKQZRPO-CSeKauwA.js} +1 -1
  242. package/ui-dist/assets/{xychartDiagram-PRI3JC2R-WtDhjZfk.js → xychartDiagram-PRI3JC2R-Ut3mCiEd.js} +1 -1
  243. package/ui-dist/index.html +2 -2
  244. package/ui-dist/locales/en/common.json +137 -1
  245. package/ui-dist/locales/zh-CN/common.json +111 -1
  246. package/ui-dist/assets/browser-ponyfill-Ct3hGqsr.js +0 -2
  247. package/ui-dist/assets/channel-pHFjGZL-.js +0 -1
  248. package/ui-dist/assets/classDiagram-2ON5EDUG-X4ZksqXl.js +0 -1
  249. package/ui-dist/assets/classDiagram-v2-WZHVMYZB-X4ZksqXl.js +0 -1
  250. package/ui-dist/assets/clone-DZzimpfG.js +0 -1
  251. package/ui-dist/assets/index-C1oE3J7o.css +0 -1
  252. package/ui-dist/assets/index-fSIlEIHr.js +0 -510
  253. package/ui-dist/assets/stateDiagram-v2-4FDKWEC3-gnLzrhSv.js +0 -1
@@ -3,14 +3,14 @@ import path from "node:path";
3
3
  import { execFile as execFileCallback } from "node:child_process";
4
4
  import { promisify } from "node:util";
5
5
  import { randomUUID } from "node:crypto";
6
- import { and, asc, desc, eq, getTableColumns, gt, inArray, isNull, lte, notInArray, or, sql } from "drizzle-orm";
7
- import { AGENT_DEFAULT_MAX_CONCURRENT_RUNS, ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY, } from "@penclipai/shared";
6
+ import { and, asc, desc, eq, getTableColumns, gt, inArray, isNull, lt, lte, notInArray, or, sql } from "drizzle-orm";
7
+ import { AGENT_DEFAULT_MAX_CONCURRENT_RUNS, ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY, MODEL_PROFILE_KEYS, } from "@penclipai/shared";
8
8
  import { agents, agentRuntimeState, agentTaskSessions, agentWakeupRequests, activityLog, companySkills as companySkillsTable, documentRevisions, issueDocuments, heartbeatRunEvents, heartbeatRuns, issueComments, issueRelations, issues, issueWorkProducts, projects, projectWorkspaces, workspaceOperations, } from "@penclipai/db";
9
9
  import { conflict, HttpError, notFound } from "../errors.js";
10
10
  import { logger } from "../middleware/logger.js";
11
11
  import { publishLiveEvent } from "./live-events.js";
12
12
  import { getRunLogStore } from "./run-log-store.js";
13
- import { getServerAdapter, runningProcesses } from "../adapters/index.js";
13
+ import { getServerAdapter, listAdapterModelProfiles, runningProcesses } from "../adapters/index.js";
14
14
  import { createLocalAgentJwt } from "../agent-auth-jwt.js";
15
15
  import { parseObject, asBoolean, asNumber, appendWithByteCap, MAX_EXCERPT_BYTES } from "../adapters/utils.js";
16
16
  import { costService } from "./costs.js";
@@ -21,11 +21,12 @@ import { budgetService } from "./budgets.js";
21
21
  import { secretService } from "./secrets.js";
22
22
  import { resolveDefaultAgentWorkspaceDir, resolveManagedProjectWorkspaceDir } from "../home-paths.js";
23
23
  import { buildHeartbeatRunIssueComment, HEARTBEAT_RUN_RESULT_OUTPUT_MAX_CHARS, HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS, HEARTBEAT_RUN_SAFE_RESULT_JSON_MAX_BYTES, mergeHeartbeatRunResultJson, } from "./heartbeat-run-summary.js";
24
- import { buildHeartbeatRunStopMetadata, mergeHeartbeatRunStopMetadata, } from "./heartbeat-stop-metadata.js";
24
+ import { buildHeartbeatRunStopMetadata, mergeHeartbeatRunStopMetadata, normalizeMaxTurnStopReason, } from "./heartbeat-stop-metadata.js";
25
25
  import { classifyRunLiveness, } from "./run-liveness.js";
26
26
  import { logActivity, publishPluginDomainEvent } from "./activity-log.js";
27
27
  import { buildWorkspaceReadyComment, cleanupExecutionWorkspaceArtifacts, ensureRuntimeServicesForRun, persistAdapterManagedRuntimeServices, realizeExecutionWorkspace, releaseRuntimeServicesForRun, sanitizeRuntimeServiceBaseEnv, } from "./workspace-runtime.js";
28
28
  import { issueService } from "./issues.js";
29
+ import { buildIssueMonitorClearedPatch, buildIssueMonitorTriggeredPatch, normalizeIssueExecutionPolicy, parseIssueExecutionState, } from "./issue-execution-policy.js";
29
30
  import { ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS, isVerifiedIssueTreeControlInteractionWake, issueTreeControlService, } from "./issue-tree-control.js";
30
31
  import { getIssueContinuationSummaryDocument, refreshIssueContinuationSummary, } from "./issue-continuation-summary.js";
31
32
  import { executionWorkspaceService, mergeExecutionWorkspaceConfig } from "./execution-workspaces.js";
@@ -35,11 +36,13 @@ import { resolveRuntimeLocalizationPrompt, } from "./agent-runtime-localization.
35
36
  import { canCoalesceWithRunLocale, materializeRuntimeUiLocaleContextSnapshot, resolveContextRuntimeUiLocale, } from "./heartbeat-runtime-locale.js";
36
37
  import { buildExecutionWorkspaceAdapterConfig, gateProjectExecutionWorkspacePolicy, issueExecutionWorkspaceModeForPersistedWorkspace, parseIssueExecutionWorkspaceSettings, parseProjectExecutionWorkspacePolicy, resolveExecutionWorkspaceEnvironmentId, resolveExecutionWorkspaceMode, } from "./execution-workspace-policy.js";
37
38
  import { instanceSettingsService } from "./instance-settings.js";
38
- import { RUN_LIVENESS_CONTINUATION_REASON, buildRunLivenessContinuationIdempotencyKey, decideRunLivenessContinuation, findExistingRunLivenessContinuationWake, readContinuationAttempt, } from "./recovery/index.js";
39
+ import { RECOVERY_ORIGIN_KINDS, RUN_LIVENESS_CONTINUATION_REASON, buildRunLivenessContinuationIdempotencyKey, decideRunLivenessContinuation, findExistingRunLivenessContinuationWake, readContinuationAttempt, } from "./recovery/index.js";
39
40
  import { isAutomaticRecoverySuppressedByPauseHold } from "./recovery/pause-hold-guard.js";
40
41
  import { recoveryService } from "./recovery/service.js";
42
+ import { productivityReviewService } from "./productivity-review.js";
41
43
  import { withAgentStartLock } from "./agent-start-lock.js";
42
44
  import { redactCurrentUserText, redactCurrentUserValue } from "../log-redaction.js";
45
+ import { redactEventPayload } from "../redaction.js";
43
46
  import { hasSessionCompactionThresholds, resolveSessionCompactionPolicy, } from "@penclipai/adapter-utils";
44
47
  import { readPaperclipSkillSyncPreference, writePaperclipSkillSyncPreference, } from "@penclipai/adapter-utils/server-utils";
45
48
  import { extractSkillMentionIds } from "@penclipai/shared";
@@ -53,7 +56,8 @@ const MAX_RUN_EVENT_PAYLOAD_ARRAY_ITEMS = 50;
53
56
  const MAX_RUN_EVENT_PAYLOAD_OBJECT_KEYS = 100;
54
57
  const MAX_RUN_EVENT_PAYLOAD_DEPTH = 6;
55
58
  const HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT = AGENT_DEFAULT_MAX_CONCURRENT_RUNS;
56
- const HEARTBEAT_MAX_CONCURRENT_RUNS_MAX = 10;
59
+ const HEARTBEAT_MAX_CONCURRENT_RUNS_MIN = 1;
60
+ const HEARTBEAT_MAX_CONCURRENT_RUNS_MAX = 50;
57
61
  const LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS = [
58
62
  "environment.lease_acquired",
59
63
  "environment.lease_released",
@@ -85,6 +89,13 @@ const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_JITTER_RATIO = 0.25;
85
89
  const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON = "transient_failure";
86
90
  const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry";
87
91
  const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length;
92
+ export const MAX_TURN_CONTINUATION_RETRY_REASON = "max_turns_continuation";
93
+ export const MAX_TURN_CONTINUATION_WAKE_REASON = "max_turns_continuation_retry";
94
+ const MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS = 2;
95
+ const MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP = 10;
96
+ const MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS = 1_000;
97
+ const MAX_TURN_CONTINUATION_MAX_DELAY_MS = 5 * 60 * 1000;
98
+ const MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES = ["scheduled_retry", "queued", "running"];
88
99
  function resolveCodexTransientFallbackMode(attempt) {
89
100
  if (attempt <= 1)
90
101
  return "same_session";
@@ -104,6 +115,11 @@ function readHeartbeatRunErrorFamily(run) {
104
115
  }
105
116
  return null;
106
117
  }
118
+ function isMaxTurnExhaustionRun(run) {
119
+ const resultJson = parseObject(run.resultJson);
120
+ return Boolean(normalizeMaxTurnStopReason(resultJson.stopReason) ??
121
+ normalizeMaxTurnStopReason(run.errorCode));
122
+ }
107
123
  function readTransientRetryNotBeforeFromRun(run) {
108
124
  const resultJson = parseObject(run.resultJson);
109
125
  const value = resultJson.retryNotBefore ?? resultJson.transientRetryNotBefore;
@@ -547,6 +563,8 @@ const heartbeatRunIssueSummaryColumns = {
547
563
  status: heartbeatRuns.status,
548
564
  invocationSource: heartbeatRuns.invocationSource,
549
565
  triggerDetail: heartbeatRuns.triggerDetail,
566
+ contextCommentId: sql `${heartbeatRuns.contextSnapshot} ->> 'commentId'`.as("contextCommentId"),
567
+ contextWakeCommentId: sql `${heartbeatRuns.contextSnapshot} ->> 'wakeCommentId'`.as("contextWakeCommentId"),
550
568
  startedAt: heartbeatRuns.startedAt,
551
569
  finishedAt: heartbeatRuns.finishedAt,
552
570
  createdAt: heartbeatRuns.createdAt,
@@ -653,7 +671,7 @@ function normalizeMaxConcurrentRuns(value) {
653
671
  const parsed = Math.floor(asNumber(value, HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT));
654
672
  if (!Number.isFinite(parsed))
655
673
  return HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT;
656
- return Math.max(HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT, Math.min(HEARTBEAT_MAX_CONCURRENT_RUNS_MAX, parsed));
674
+ return Math.max(HEARTBEAT_MAX_CONCURRENT_RUNS_MIN, Math.min(HEARTBEAT_MAX_CONCURRENT_RUNS_MAX, parsed));
657
675
  }
658
676
  export function prioritizeProjectWorkspaceCandidatesForRun(rows, preferredWorkspaceId) {
659
677
  if (!preferredWorkspaceId)
@@ -666,6 +684,113 @@ export function prioritizeProjectWorkspaceCandidatesForRun(rows, preferredWorksp
666
684
  function readNonEmptyString(value) {
667
685
  return typeof value === "string" && value.trim().length > 0 ? value : null;
668
686
  }
687
+ function readModelProfileKey(value) {
688
+ return MODEL_PROFILE_KEYS.includes(value)
689
+ ? value
690
+ : null;
691
+ }
692
+ function readContextModelProfile(contextSnapshot) {
693
+ return readModelProfileKey(contextSnapshot?.modelProfile);
694
+ }
695
+ export function normalizeModelProfileWakeContext(input) {
696
+ const modelProfileFromPayload = readModelProfileKey(input.payload?.modelProfile);
697
+ if (!readContextModelProfile(input.contextSnapshot) && modelProfileFromPayload) {
698
+ input.contextSnapshot.modelProfile = modelProfileFromPayload;
699
+ }
700
+ return input.contextSnapshot;
701
+ }
702
+ function readAgentRuntimeModelProfile(runtimeConfig, key) {
703
+ const modelProfiles = parseObject(parseObject(runtimeConfig).modelProfiles);
704
+ const profile = parseObject(modelProfiles[key]);
705
+ if (Object.keys(profile).length === 0) {
706
+ return { enabled: true, adapterConfig: {}, configured: false };
707
+ }
708
+ return {
709
+ enabled: profile.enabled !== false,
710
+ adapterConfig: parseObject(profile.adapterConfig),
711
+ configured: true,
712
+ };
713
+ }
714
+ export function resolveModelProfileApplication(input) {
715
+ const issueModelProfile = input.issueModelProfile ?? null;
716
+ const contextModelProfile = readContextModelProfile(input.contextSnapshot);
717
+ const requested = issueModelProfile ?? contextModelProfile;
718
+ const requestedBy = issueModelProfile
719
+ ? "issue_override"
720
+ : contextModelProfile
721
+ ? "wake_context"
722
+ : null;
723
+ if (!requested) {
724
+ return {
725
+ requested: null,
726
+ requestedBy: null,
727
+ applied: null,
728
+ configSource: null,
729
+ fallbackReason: null,
730
+ adapterConfig: null,
731
+ };
732
+ }
733
+ const adapterProfile = input.adapterModelProfiles.find((profile) => profile.key === requested) ?? null;
734
+ if (!adapterProfile) {
735
+ return {
736
+ requested,
737
+ requestedBy,
738
+ applied: null,
739
+ configSource: null,
740
+ fallbackReason: input.profileResolutionFallbackReason ?? "adapter_profile_not_supported",
741
+ adapterConfig: null,
742
+ };
743
+ }
744
+ const runtimeProfile = readAgentRuntimeModelProfile(input.agentRuntimeConfig, requested);
745
+ if (!runtimeProfile.enabled) {
746
+ return {
747
+ requested,
748
+ requestedBy,
749
+ applied: null,
750
+ configSource: null,
751
+ fallbackReason: "agent_runtime_profile_disabled",
752
+ adapterConfig: null,
753
+ };
754
+ }
755
+ return {
756
+ requested,
757
+ requestedBy,
758
+ applied: requested,
759
+ configSource: runtimeProfile.configured ? "agent_runtime" : "adapter_default",
760
+ fallbackReason: null,
761
+ adapterConfig: {
762
+ ...parseObject(adapterProfile.adapterConfig),
763
+ ...runtimeProfile.adapterConfig,
764
+ },
765
+ };
766
+ }
767
+ export function mergeModelProfileAdapterConfig(input) {
768
+ return {
769
+ ...input.baseConfig,
770
+ ...(input.modelProfile.adapterConfig ?? {}),
771
+ ...(input.issueAdapterConfig ?? {}),
772
+ };
773
+ }
774
+ function modelProfileRunMetadata(modelProfile) {
775
+ if (!modelProfile.requested)
776
+ return null;
777
+ return {
778
+ requested: modelProfile.requested,
779
+ requestedBy: modelProfile.requestedBy,
780
+ applied: modelProfile.applied,
781
+ configSource: modelProfile.configSource,
782
+ fallbackReason: modelProfile.fallbackReason,
783
+ };
784
+ }
785
+ function mergeModelProfileRunMetadata(resultJson, modelProfile) {
786
+ const metadata = modelProfileRunMetadata(modelProfile);
787
+ if (!metadata)
788
+ return resultJson;
789
+ return {
790
+ ...(resultJson ?? {}),
791
+ modelProfile: metadata,
792
+ };
793
+ }
669
794
  export function summarizeHeartbeatRunContextSnapshot(contextSnapshot) {
670
795
  const summary = {};
671
796
  const allowedKeys = [
@@ -677,6 +802,7 @@ export function summarizeHeartbeatRunContextSnapshot(contextSnapshot) {
677
802
  "wakeReason",
678
803
  "wakeSource",
679
804
  "wakeTriggerDetail",
805
+ "modelProfile",
680
806
  ];
681
807
  for (const key of allowedKeys) {
682
808
  const value = readNonEmptyString(contextSnapshot?.[key]);
@@ -931,14 +1057,18 @@ export function resolveRuntimeSessionParamsForWorkspace(input) {
931
1057
  }
932
1058
  function parseIssueAssigneeAdapterOverrides(raw) {
933
1059
  const parsed = parseObject(raw);
1060
+ const modelProfile = MODEL_PROFILE_KEYS.includes(parsed.modelProfile)
1061
+ ? parsed.modelProfile
1062
+ : null;
934
1063
  const parsedAdapterConfig = parseObject(parsed.adapterConfig);
935
1064
  const adapterConfig = Object.keys(parsedAdapterConfig).length > 0 ? parsedAdapterConfig : null;
936
1065
  const useProjectWorkspace = typeof parsed.useProjectWorkspace === "boolean"
937
1066
  ? parsed.useProjectWorkspace
938
1067
  : null;
939
- if (!adapterConfig && useProjectWorkspace === null)
1068
+ if (!modelProfile && !adapterConfig && useProjectWorkspace === null)
940
1069
  return null;
941
1070
  return {
1071
+ modelProfile,
942
1072
  adapterConfig,
943
1073
  useProjectWorkspace,
944
1074
  };
@@ -1162,6 +1292,7 @@ function enrichWakeContextSnapshot(input) {
1162
1292
  if (!readNonEmptyString(contextSnapshot["wakeTriggerDetail"]) && triggerDetail) {
1163
1293
  contextSnapshot.wakeTriggerDetail = triggerDetail;
1164
1294
  }
1295
+ normalizeModelProfileWakeContext({ contextSnapshot, payload });
1165
1296
  return {
1166
1297
  contextSnapshot,
1167
1298
  issueIdFromPayload,
@@ -1514,7 +1645,23 @@ export function heartbeatService(db, options = {}) {
1514
1645
  };
1515
1646
  const budgets = budgetService(db, budgetHooks);
1516
1647
  const recovery = recoveryService(db, { enqueueWakeup });
1648
+ const productivityReviews = productivityReviewService(db, { enqueueWakeup });
1517
1649
  let unsafeTextProjectionPromise = null;
1650
+ async function releaseEnvironmentLeasesForRun(input) {
1651
+ const releaseResult = await envOrchestrator.releaseForRun({
1652
+ heartbeatRunId: input.runId,
1653
+ companyId: input.companyId,
1654
+ agentId: input.agentId,
1655
+ status: leaseReleaseStatusForRunStatus(input.status),
1656
+ failureReason: input.failureReason ?? undefined,
1657
+ }).catch((err) => {
1658
+ logger.warn({ err, runId: input.runId }, "failed to release environment leases for heartbeat run");
1659
+ return null;
1660
+ });
1661
+ for (const releaseError of releaseResult?.errors ?? []) {
1662
+ logger.warn({ err: releaseError.error, leaseId: releaseError.leaseId, runId: input.runId }, "failed to release environment lease for heartbeat run");
1663
+ }
1664
+ }
1518
1665
  async function hasUnsafeTextProjectionDatabase() {
1519
1666
  if (!unsafeTextProjectionPromise) {
1520
1667
  unsafeTextProjectionPromise = db
@@ -1613,6 +1760,522 @@ export function heartbeatService(db, options = {}) {
1613
1760
  .limit(1)
1614
1761
  .then((rows) => rows[0] ?? null);
1615
1762
  }
1763
+ const issueMonitorDispatchColumns = {
1764
+ id: issues.id,
1765
+ companyId: issues.companyId,
1766
+ projectId: issues.projectId,
1767
+ goalId: issues.goalId,
1768
+ identifier: issues.identifier,
1769
+ title: issues.title,
1770
+ status: issues.status,
1771
+ priority: issues.priority,
1772
+ assigneeAgentId: issues.assigneeAgentId,
1773
+ assigneeUserId: issues.assigneeUserId,
1774
+ billingCode: issues.billingCode,
1775
+ executionPolicy: issues.executionPolicy,
1776
+ executionState: issues.executionState,
1777
+ monitorNextCheckAt: issues.monitorNextCheckAt,
1778
+ monitorWakeRequestedAt: issues.monitorWakeRequestedAt,
1779
+ monitorLastTriggeredAt: issues.monitorLastTriggeredAt,
1780
+ monitorAttemptCount: issues.monitorAttemptCount,
1781
+ monitorNotes: issues.monitorNotes,
1782
+ monitorScheduledBy: issues.monitorScheduledBy,
1783
+ };
1784
+ function parseMonitorDate(value) {
1785
+ if (!value)
1786
+ return null;
1787
+ const date = new Date(value);
1788
+ return Number.isNaN(date.getTime()) ? null : date;
1789
+ }
1790
+ function issueMonitorLimitClearReason(input) {
1791
+ const timeoutAt = parseMonitorDate(input.monitor?.timeoutAt ?? null);
1792
+ if (timeoutAt && input.now.getTime() >= timeoutAt.getTime()) {
1793
+ return "timeout_exceeded";
1794
+ }
1795
+ const maxAttempts = input.monitor?.maxAttempts ?? null;
1796
+ if (maxAttempts !== null && input.nextAttemptCount > maxAttempts) {
1797
+ return "max_attempts_exhausted";
1798
+ }
1799
+ return null;
1800
+ }
1801
+ function monitorRecoveryPolicy(monitor) {
1802
+ return monitor?.recoveryPolicy ?? "wake_owner";
1803
+ }
1804
+ function monitorRecoveryDetails(input) {
1805
+ return {
1806
+ identifier: input.claimed.identifier,
1807
+ nextCheckAt: input.scheduledAtIso,
1808
+ attemptedAttemptCount: input.nextAttemptCount,
1809
+ notes: input.claimed.monitorNotes ?? null,
1810
+ serviceName: input.monitor?.serviceName ?? null,
1811
+ timeoutAt: input.monitor?.timeoutAt ?? null,
1812
+ maxAttempts: input.monitor?.maxAttempts ?? null,
1813
+ clearReason: input.clearReason,
1814
+ recoveryPolicy: input.recoveryPolicy,
1815
+ source: input.source,
1816
+ };
1817
+ }
1818
+ function formatIssueIdentifierLink(identifier, fallback) {
1819
+ if (!identifier)
1820
+ return fallback;
1821
+ const prefix = identifier.split("-")[0];
1822
+ if (!prefix || !/^[A-Z][A-Z0-9]*-\d+$/.test(identifier))
1823
+ return identifier;
1824
+ return `[${identifier}](/${prefix}/issues/${identifier})`;
1825
+ }
1826
+ function monitorRecoveryComment(input) {
1827
+ const label = formatIssueIdentifierLink(input.issue.identifier, input.issue.id);
1828
+ const reason = input.clearReason === "timeout_exceeded"
1829
+ ? "its timeout was reached"
1830
+ : "its maximum attempt count was reached";
1831
+ return [
1832
+ `Paperclip cleared the scheduled external-service monitor for ${label} because ${reason}.`,
1833
+ "",
1834
+ `- Attempt count: ${input.nextAttemptCount}`,
1835
+ `- Recovery policy: ${input.recoveryPolicy}`,
1836
+ "",
1837
+ "Next action: inspect the external service state, record the result on this issue, and restore an explicit execution or waiting path if more work remains.",
1838
+ ].join("\n");
1839
+ }
1840
+ async function findOpenIssueMonitorRecoveryIssue(claimed) {
1841
+ return db
1842
+ .select()
1843
+ .from(issues)
1844
+ .where(and(eq(issues.companyId, claimed.companyId), eq(issues.originKind, RECOVERY_ORIGIN_KINDS.strandedIssueRecovery), eq(issues.originId, claimed.id), isNull(issues.hiddenAt), notInArray(issues.status, ["done", "cancelled"])))
1845
+ .orderBy(desc(issues.createdAt))
1846
+ .limit(1)
1847
+ .then((rows) => rows[0] ?? null);
1848
+ }
1849
+ async function performIssueMonitorRecovery(input) {
1850
+ const details = monitorRecoveryDetails({
1851
+ claimed: input.claimed,
1852
+ scheduledAtIso: input.scheduledAtIso,
1853
+ nextAttemptCount: input.nextAttemptCount,
1854
+ clearReason: input.clearReason,
1855
+ recoveryPolicy: input.recoveryPolicy,
1856
+ monitor: input.monitor,
1857
+ source: input.activitySource,
1858
+ });
1859
+ if (input.recoveryPolicy === "create_recovery_issue") {
1860
+ let recoveryIssue = await findOpenIssueMonitorRecoveryIssue(input.claimed);
1861
+ if (!recoveryIssue) {
1862
+ recoveryIssue = await issuesSvc.create(input.claimed.companyId, {
1863
+ title: `Recover external-service monitor for ${input.claimed.identifier ?? input.claimed.title}`,
1864
+ description: monitorRecoveryComment({
1865
+ issue: input.claimed,
1866
+ clearReason: input.clearReason,
1867
+ recoveryPolicy: input.recoveryPolicy,
1868
+ nextAttemptCount: input.nextAttemptCount,
1869
+ }),
1870
+ status: "todo",
1871
+ priority: "high",
1872
+ parentId: input.claimed.id,
1873
+ projectId: input.claimed.projectId,
1874
+ goalId: input.claimed.goalId,
1875
+ assigneeAgentId: input.claimed.assigneeAgentId,
1876
+ originKind: RECOVERY_ORIGIN_KINDS.strandedIssueRecovery,
1877
+ originId: input.claimed.id,
1878
+ originFingerprint: `issue_monitor:${input.clearReason}`,
1879
+ billingCode: input.claimed.billingCode,
1880
+ });
1881
+ }
1882
+ if (recoveryIssue.assigneeAgentId) {
1883
+ await enqueueWakeup(recoveryIssue.assigneeAgentId, {
1884
+ source: "automation",
1885
+ triggerDetail: "system",
1886
+ reason: "issue_monitor_recovery_issue",
1887
+ idempotencyKey: `issue-monitor-recovery-issue:${input.claimed.id}:${input.clearReason}:${input.scheduledAtIso}`,
1888
+ payload: { issueId: recoveryIssue.id, sourceIssueId: input.claimed.id },
1889
+ requestedByActorType: input.actorType,
1890
+ requestedByActorId: input.actorId,
1891
+ contextSnapshot: {
1892
+ issueId: recoveryIssue.id,
1893
+ sourceIssueId: input.claimed.id,
1894
+ source: "issue.monitor.recovery_issue",
1895
+ wakeReason: "issue_monitor_recovery_issue",
1896
+ },
1897
+ });
1898
+ }
1899
+ await logActivity(db, {
1900
+ companyId: input.claimed.companyId,
1901
+ actorType: input.actorType,
1902
+ actorId: input.actorId,
1903
+ agentId: input.agentId,
1904
+ runId: input.runId,
1905
+ action: "issue.monitor_recovery_issue_created",
1906
+ entityType: "issue",
1907
+ entityId: input.claimed.id,
1908
+ details: {
1909
+ ...details,
1910
+ recoveryIssueId: recoveryIssue.id,
1911
+ recoveryIdentifier: recoveryIssue.identifier,
1912
+ },
1913
+ });
1914
+ return;
1915
+ }
1916
+ if (input.recoveryPolicy === "escalate_to_board") {
1917
+ await db.insert(issueComments).values({
1918
+ companyId: input.claimed.companyId,
1919
+ issueId: input.claimed.id,
1920
+ body: monitorRecoveryComment({
1921
+ issue: input.claimed,
1922
+ clearReason: input.clearReason,
1923
+ recoveryPolicy: input.recoveryPolicy,
1924
+ nextAttemptCount: input.nextAttemptCount,
1925
+ }),
1926
+ });
1927
+ await logActivity(db, {
1928
+ companyId: input.claimed.companyId,
1929
+ actorType: input.actorType,
1930
+ actorId: input.actorId,
1931
+ agentId: input.agentId,
1932
+ runId: input.runId,
1933
+ action: "issue.monitor_escalated_to_board",
1934
+ entityType: "issue",
1935
+ entityId: input.claimed.id,
1936
+ details,
1937
+ });
1938
+ return;
1939
+ }
1940
+ await enqueueWakeup(input.claimed.assigneeAgentId, {
1941
+ source: "automation",
1942
+ triggerDetail: "system",
1943
+ reason: "issue_monitor_recovery",
1944
+ idempotencyKey: `issue-monitor-recovery:${input.claimed.id}:${input.clearReason}:${input.scheduledAtIso}`,
1945
+ payload: {
1946
+ issueId: input.claimed.id,
1947
+ monitorAttemptCount: input.nextAttemptCount,
1948
+ monitorNotes: input.claimed.monitorNotes ?? null,
1949
+ clearReason: input.clearReason,
1950
+ serviceName: input.monitor?.serviceName ?? null,
1951
+ timeoutAt: input.monitor?.timeoutAt ?? null,
1952
+ maxAttempts: input.monitor?.maxAttempts ?? null,
1953
+ },
1954
+ requestedByActorType: input.actorType,
1955
+ requestedByActorId: input.actorId,
1956
+ contextSnapshot: {
1957
+ issueId: input.claimed.id,
1958
+ source: "issue.monitor.recovery",
1959
+ wakeReason: "issue_monitor_recovery",
1960
+ monitorAttemptCount: input.nextAttemptCount,
1961
+ monitorNotes: input.claimed.monitorNotes ?? null,
1962
+ clearReason: input.clearReason,
1963
+ serviceName: input.monitor?.serviceName ?? null,
1964
+ timeoutAt: input.monitor?.timeoutAt ?? null,
1965
+ maxAttempts: input.monitor?.maxAttempts ?? null,
1966
+ },
1967
+ });
1968
+ await logActivity(db, {
1969
+ companyId: input.claimed.companyId,
1970
+ actorType: input.actorType,
1971
+ actorId: input.actorId,
1972
+ agentId: input.agentId,
1973
+ runId: input.runId,
1974
+ action: "issue.monitor_recovery_wake_queued",
1975
+ entityType: "issue",
1976
+ entityId: input.claimed.id,
1977
+ details,
1978
+ });
1979
+ }
1980
+ async function clearIssueMonitorAndRecover(input) {
1981
+ await db
1982
+ .update(issues)
1983
+ .set({
1984
+ ...buildIssueMonitorClearedPatch({
1985
+ issue: input.claimed,
1986
+ policy: input.policy,
1987
+ clearReason: input.clearReason,
1988
+ clearedAt: input.now,
1989
+ }),
1990
+ updatedAt: input.now,
1991
+ })
1992
+ .where(eq(issues.id, input.claimed.id));
1993
+ await logActivity(db, {
1994
+ companyId: input.claimed.companyId,
1995
+ actorType: input.actorType,
1996
+ actorId: input.actorId,
1997
+ agentId: input.agentId,
1998
+ runId: input.runId,
1999
+ action: "issue.monitor_exhausted",
2000
+ entityType: "issue",
2001
+ entityId: input.claimed.id,
2002
+ details: monitorRecoveryDetails({
2003
+ claimed: input.claimed,
2004
+ scheduledAtIso: input.scheduledAtIso,
2005
+ nextAttemptCount: input.nextAttemptCount,
2006
+ clearReason: input.clearReason,
2007
+ recoveryPolicy: input.recoveryPolicy,
2008
+ monitor: input.monitor,
2009
+ source: input.activitySource,
2010
+ }),
2011
+ });
2012
+ await performIssueMonitorRecovery({
2013
+ claimed: input.claimed,
2014
+ scheduledAtIso: input.scheduledAtIso,
2015
+ nextAttemptCount: input.nextAttemptCount,
2016
+ clearReason: input.clearReason,
2017
+ recoveryPolicy: input.recoveryPolicy,
2018
+ monitor: input.monitor,
2019
+ actorType: input.actorType,
2020
+ actorId: input.actorId,
2021
+ agentId: input.agentId,
2022
+ runId: input.runId,
2023
+ activitySource: input.activitySource,
2024
+ });
2025
+ return { outcome: "skipped", reason: input.clearReason };
2026
+ }
2027
+ async function dispatchClaimedIssueMonitor(claimed, input) {
2028
+ if (!claimed.assigneeAgentId || !claimed.monitorNextCheckAt) {
2029
+ throw conflict("Issue monitor is not ready to dispatch");
2030
+ }
2031
+ const scheduledAtIso = claimed.monitorNextCheckAt.toISOString();
2032
+ const nextAttemptCount = (claimed.monitorAttemptCount ?? 0) + 1;
2033
+ const policy = normalizeIssueExecutionPolicy(claimed.executionPolicy ?? null);
2034
+ const monitor = policy?.monitor ?? null;
2035
+ const clearReason = issueMonitorLimitClearReason({ monitor, nextAttemptCount, now: input.now });
2036
+ const recoveryPolicy = monitorRecoveryPolicy(monitor);
2037
+ const monitorMetadata = {
2038
+ serviceName: monitor?.serviceName ?? null,
2039
+ timeoutAt: monitor?.timeoutAt ?? null,
2040
+ maxAttempts: monitor?.maxAttempts ?? null,
2041
+ recoveryPolicy: monitor?.recoveryPolicy ?? null,
2042
+ };
2043
+ if (clearReason) {
2044
+ return clearIssueMonitorAndRecover({
2045
+ claimed,
2046
+ policy,
2047
+ scheduledAtIso,
2048
+ nextAttemptCount,
2049
+ clearReason,
2050
+ recoveryPolicy,
2051
+ monitor,
2052
+ now: input.now,
2053
+ actorType: input.actorType,
2054
+ actorId: input.actorId,
2055
+ agentId: input.agentId,
2056
+ runId: input.runId,
2057
+ activitySource: input.activitySource,
2058
+ });
2059
+ }
2060
+ try {
2061
+ await enqueueWakeup(claimed.assigneeAgentId, {
2062
+ source: input.source,
2063
+ triggerDetail: input.triggerDetail,
2064
+ reason: input.wakeReason,
2065
+ idempotencyKey: `issue-monitor:${claimed.id}:${scheduledAtIso}`,
2066
+ payload: {
2067
+ issueId: claimed.id,
2068
+ nextCheckAt: scheduledAtIso,
2069
+ monitorAttemptCount: nextAttemptCount,
2070
+ monitorNotes: claimed.monitorNotes ?? null,
2071
+ ...monitorMetadata,
2072
+ source: input.activitySource,
2073
+ },
2074
+ requestedByActorType: input.actorType,
2075
+ requestedByActorId: input.actorId,
2076
+ contextSnapshot: {
2077
+ issueId: claimed.id,
2078
+ source: "issue.monitor",
2079
+ wakeReason: input.wakeReason,
2080
+ nextCheckAt: scheduledAtIso,
2081
+ monitorAttemptCount: nextAttemptCount,
2082
+ monitorNotes: claimed.monitorNotes ?? null,
2083
+ ...monitorMetadata,
2084
+ manualTrigger: input.activitySource === "manual",
2085
+ },
2086
+ });
2087
+ await db
2088
+ .update(issues)
2089
+ .set({
2090
+ ...buildIssueMonitorTriggeredPatch({
2091
+ issue: claimed,
2092
+ policy,
2093
+ triggeredAt: input.now,
2094
+ }),
2095
+ updatedAt: new Date(),
2096
+ })
2097
+ .where(eq(issues.id, claimed.id));
2098
+ await logActivity(db, {
2099
+ companyId: claimed.companyId,
2100
+ actorType: input.actorType,
2101
+ actorId: input.actorId,
2102
+ agentId: input.agentId,
2103
+ runId: input.runId,
2104
+ action: "issue.monitor_triggered",
2105
+ entityType: "issue",
2106
+ entityId: claimed.id,
2107
+ details: {
2108
+ identifier: claimed.identifier,
2109
+ nextCheckAt: scheduledAtIso,
2110
+ lastTriggeredAt: input.now.toISOString(),
2111
+ attemptCount: nextAttemptCount,
2112
+ notes: claimed.monitorNotes ?? null,
2113
+ ...monitorMetadata,
2114
+ source: input.activitySource,
2115
+ },
2116
+ });
2117
+ return { outcome: "triggered" };
2118
+ }
2119
+ catch (err) {
2120
+ if (err instanceof HttpError && err.status >= 400 && err.status < 500) {
2121
+ if (input.clearOnClientError) {
2122
+ await db
2123
+ .update(issues)
2124
+ .set({
2125
+ ...buildIssueMonitorClearedPatch({
2126
+ issue: claimed,
2127
+ policy,
2128
+ clearReason: "dispatch_skipped",
2129
+ clearedAt: input.now,
2130
+ }),
2131
+ updatedAt: new Date(),
2132
+ })
2133
+ .where(eq(issues.id, claimed.id));
2134
+ await logActivity(db, {
2135
+ companyId: claimed.companyId,
2136
+ actorType: input.actorType,
2137
+ actorId: input.actorId,
2138
+ agentId: input.agentId,
2139
+ runId: input.runId,
2140
+ action: "issue.monitor_skipped",
2141
+ entityType: "issue",
2142
+ entityId: claimed.id,
2143
+ details: {
2144
+ identifier: claimed.identifier,
2145
+ nextCheckAt: scheduledAtIso,
2146
+ attemptCount: nextAttemptCount,
2147
+ notes: claimed.monitorNotes ?? null,
2148
+ reason: err.message,
2149
+ source: input.activitySource,
2150
+ },
2151
+ });
2152
+ return { outcome: "skipped", reason: err.message };
2153
+ }
2154
+ await db
2155
+ .update(issues)
2156
+ .set({
2157
+ monitorWakeRequestedAt: null,
2158
+ updatedAt: new Date(),
2159
+ })
2160
+ .where(eq(issues.id, claimed.id));
2161
+ }
2162
+ else {
2163
+ await db
2164
+ .update(issues)
2165
+ .set({
2166
+ monitorWakeRequestedAt: null,
2167
+ updatedAt: new Date(),
2168
+ })
2169
+ .where(eq(issues.id, claimed.id));
2170
+ }
2171
+ throw err;
2172
+ }
2173
+ }
2174
+ async function triggerIssueMonitor(issueId, input) {
2175
+ const now = input?.now ?? new Date();
2176
+ const actorType = input?.actorType ?? "system";
2177
+ const actorId = input?.actorId ?? (actorType === "system" ? "heartbeat_scheduler" : null);
2178
+ if (!actorId) {
2179
+ throw conflict("Issue monitor trigger requires an actor");
2180
+ }
2181
+ const issue = await db
2182
+ .select(issueMonitorDispatchColumns)
2183
+ .from(issues)
2184
+ .where(eq(issues.id, issueId))
2185
+ .limit(1)
2186
+ .then((rows) => rows[0] ?? null);
2187
+ if (!issue) {
2188
+ throw notFound("Issue not found");
2189
+ }
2190
+ if (!issue.monitorNextCheckAt) {
2191
+ throw conflict("Issue has no scheduled monitor");
2192
+ }
2193
+ if (!issue.assigneeAgentId || issue.assigneeUserId) {
2194
+ throw conflict("Issue monitor requires an agent assignee");
2195
+ }
2196
+ if (!["in_progress", "in_review"].includes(issue.status)) {
2197
+ throw conflict("Issue monitor can only run while the issue is in progress or in review");
2198
+ }
2199
+ const staleClaimThreshold = new Date(now.getTime() - 5 * 60 * 1000);
2200
+ const claimed = await db.transaction(async (tx) => {
2201
+ const [updated] = await tx
2202
+ .update(issues)
2203
+ .set({
2204
+ monitorWakeRequestedAt: now,
2205
+ updatedAt: now,
2206
+ })
2207
+ .where(and(eq(issues.id, issueId), sql `${issues.monitorNextCheckAt} is not null`, isNull(issues.assigneeUserId), sql `${issues.assigneeAgentId} is not null`, inArray(issues.status, ["in_progress", "in_review"]), or(isNull(issues.monitorWakeRequestedAt), lt(issues.monitorWakeRequestedAt, staleClaimThreshold))))
2208
+ .returning();
2209
+ return (updated ?? null);
2210
+ });
2211
+ if (!claimed) {
2212
+ throw conflict("Issue monitor check is already in progress");
2213
+ }
2214
+ return dispatchClaimedIssueMonitor(claimed, {
2215
+ now,
2216
+ source: "on_demand",
2217
+ triggerDetail: "manual",
2218
+ wakeReason: input?.wakeReason ?? "issue_monitor_due",
2219
+ actorType,
2220
+ actorId,
2221
+ agentId: input?.agentId ?? null,
2222
+ runId: input?.runId ?? null,
2223
+ clearOnClientError: false,
2224
+ activitySource: "manual",
2225
+ });
2226
+ }
2227
+ async function tickDueIssueMonitors(now = new Date()) {
2228
+ const staleClaimThreshold = new Date(now.getTime() - 5 * 60 * 1000);
2229
+ const dueMonitors = await db
2230
+ .select(issueMonitorDispatchColumns)
2231
+ .from(issues)
2232
+ .where(and(sql `${issues.monitorNextCheckAt} is not null`, lte(issues.monitorNextCheckAt, now), isNull(issues.assigneeUserId), sql `${issues.assigneeAgentId} is not null`, inArray(issues.status, ["in_progress", "in_review"]), or(isNull(issues.monitorWakeRequestedAt), lt(issues.monitorWakeRequestedAt, staleClaimThreshold))))
2233
+ .orderBy(asc(issues.monitorNextCheckAt), asc(issues.updatedAt))
2234
+ .limit(50);
2235
+ let triggered = 0;
2236
+ let skipped = 0;
2237
+ for (const due of dueMonitors) {
2238
+ const claimed = await db.transaction(async (tx) => {
2239
+ const [updated] = await tx
2240
+ .update(issues)
2241
+ .set({
2242
+ monitorWakeRequestedAt: now,
2243
+ updatedAt: now,
2244
+ })
2245
+ .where(and(eq(issues.id, due.id), sql `${issues.monitorNextCheckAt} is not null`, lte(issues.monitorNextCheckAt, now), isNull(issues.assigneeUserId), sql `${issues.assigneeAgentId} is not null`, inArray(issues.status, ["in_progress", "in_review"]), or(isNull(issues.monitorWakeRequestedAt), lt(issues.monitorWakeRequestedAt, staleClaimThreshold))))
2246
+ .returning();
2247
+ return (updated ?? null);
2248
+ });
2249
+ if (!claimed)
2250
+ continue;
2251
+ try {
2252
+ const result = await dispatchClaimedIssueMonitor(claimed, {
2253
+ now,
2254
+ source: "automation",
2255
+ triggerDetail: "system",
2256
+ wakeReason: "issue_monitor_due",
2257
+ actorType: "system",
2258
+ actorId: "heartbeat_scheduler",
2259
+ agentId: null,
2260
+ runId: null,
2261
+ clearOnClientError: true,
2262
+ activitySource: "scheduled",
2263
+ });
2264
+ if (result.outcome === "triggered")
2265
+ triggered += 1;
2266
+ if (result.outcome === "skipped")
2267
+ skipped += 1;
2268
+ }
2269
+ catch (err) {
2270
+ logger.error({ err, issueId: claimed.id }, "issue monitor tick failed");
2271
+ }
2272
+ }
2273
+ return {
2274
+ checked: dueMonitors.length,
2275
+ triggered,
2276
+ skipped,
2277
+ };
2278
+ }
1616
2279
  async function getOldestRunForSession(agentId, sessionId) {
1617
2280
  return db
1618
2281
  .select({
@@ -2022,16 +2685,26 @@ export function heartbeatService(db, options = {}) {
2022
2685
  const existing = await getRuntimeState(agent.id);
2023
2686
  if (existing)
2024
2687
  return existing;
2025
- return db
2688
+ const inserted = await db
2026
2689
  .insert(agentRuntimeState)
2027
2690
  .values({
2028
2691
  agentId: agent.id,
2029
2692
  companyId: agent.companyId,
2030
2693
  adapterType: agent.adapterType,
2031
2694
  stateJson: {},
2695
+ })
2696
+ .onConflictDoNothing({
2697
+ target: agentRuntimeState.agentId,
2032
2698
  })
2033
2699
  .returning()
2034
- .then((rows) => rows[0]);
2700
+ .then((rows) => rows[0] ?? null);
2701
+ if (inserted)
2702
+ return inserted;
2703
+ const ensured = await getRuntimeState(agent.id);
2704
+ if (!ensured) {
2705
+ throw new Error(`Failed to ensure runtime state for agent ${agent.id}`);
2706
+ }
2707
+ return ensured;
2035
2708
  }
2036
2709
  async function setRunStatus(runId, status, patch) {
2037
2710
  const updated = await db
@@ -2158,6 +2831,28 @@ export function heartbeatService(db, options = {}) {
2158
2831
  projectId: issue.projectId,
2159
2832
  })
2160
2833
  : null;
2834
+ if (issue) {
2835
+ const productivityHold = await productivityReviews.isProductivityReviewContinuationHoldActive({
2836
+ companyId: issue.companyId,
2837
+ issueId: issue.id,
2838
+ agentId: run.agentId,
2839
+ });
2840
+ if (productivityHold.held) {
2841
+ await setRunStatus(run.id, run.status, {
2842
+ livenessReason: `${run.livenessReason ?? "Run ended without concrete progress"}; continuation held by productivity review ${productivityHold.reviewIdentifier ?? productivityHold.reviewIssueId}`,
2843
+ });
2844
+ await productivityReviews.recordContinuationHold({
2845
+ companyId: issue.companyId,
2846
+ issueId: issue.id,
2847
+ runId: run.id,
2848
+ agentId: run.agentId,
2849
+ reviewIssueId: productivityHold.reviewIssueId,
2850
+ trigger: productivityHold.trigger,
2851
+ reason: productivityHold.reason,
2852
+ });
2853
+ return;
2854
+ }
2855
+ }
2161
2856
  const nextAttempt = readContinuationAttempt(run.continuationAttempt) + 1;
2162
2857
  const idempotencyKey = issue
2163
2858
  ? buildRunLivenessContinuationIdempotencyKey({
@@ -2224,9 +2919,10 @@ export function heartbeatService(db, options = {}) {
2224
2919
  const boundedPayload = event.payload
2225
2920
  ? boundHeartbeatRunEventPayloadForStorage(event.payload)
2226
2921
  : event.payload;
2227
- const sanitizedPayload = boundedPayload
2228
- ? redactCurrentUserValue(boundedPayload, currentUserRedactionOptions)
2229
- : boundedPayload;
2922
+ const secretSanitizedPayload = boundedPayload ? redactEventPayload(boundedPayload) : boundedPayload;
2923
+ const sanitizedPayload = secretSanitizedPayload
2924
+ ? redactCurrentUserValue(secretSanitizedPayload, currentUserRedactionOptions)
2925
+ : secretSanitizedPayload;
2230
2926
  await db.insert(heartbeatRunEvents).values({
2231
2927
  companyId: run.companyId,
2232
2928
  runId: run.id,
@@ -2621,19 +3317,235 @@ export function heartbeatService(db, options = {}) {
2621
3317
  eventType: "lifecycle",
2622
3318
  stream: "system",
2623
3319
  level: "warn",
2624
- message: "Queued automatic retry after orphaned child process was confirmed dead",
3320
+ message: "Queued automatic retry after orphaned child process was confirmed dead",
3321
+ payload: {
3322
+ retryOfRunId: run.id,
3323
+ },
3324
+ });
3325
+ return queued;
3326
+ }
3327
+ async function evaluateScheduledRetryGate(input) {
3328
+ const { run, agent, contextSnapshot } = input;
3329
+ const retryReason = input.retryReason ?? readNonEmptyString(contextSnapshot.retryReason) ?? run.scheduledRetryReason ?? null;
3330
+ const issueId = readNonEmptyString(contextSnapshot.issueId);
3331
+ const projectId = readNonEmptyString(contextSnapshot.projectId);
3332
+ const budgetBlock = await budgets.getInvocationBlock(run.companyId, run.agentId, {
3333
+ issueId,
3334
+ projectId,
3335
+ });
3336
+ if (budgetBlock) {
3337
+ return {
3338
+ allowed: false,
3339
+ reason: budgetBlock.reason,
3340
+ errorCode: "budget_blocked",
3341
+ issueId,
3342
+ details: {
3343
+ scopeType: budgetBlock.scopeType,
3344
+ scopeId: budgetBlock.scopeId,
3345
+ },
3346
+ };
3347
+ }
3348
+ if (agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval") {
3349
+ return {
3350
+ allowed: false,
3351
+ reason: "Scheduled retry suppressed because the agent is not invokable",
3352
+ errorCode: "agent_not_invokable",
3353
+ issueId,
3354
+ details: {
3355
+ agentId: agent.id,
3356
+ agentStatus: agent.status,
3357
+ },
3358
+ };
3359
+ }
3360
+ if (!issueId)
3361
+ return { allowed: true };
3362
+ const issue = await db
3363
+ .select({
3364
+ id: issues.id,
3365
+ status: issues.status,
3366
+ assigneeAgentId: issues.assigneeAgentId,
3367
+ executionRunId: issues.executionRunId,
3368
+ executionState: issues.executionState,
3369
+ })
3370
+ .from(issues)
3371
+ .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))
3372
+ .then((rows) => rows[0] ?? null);
3373
+ if (!issue) {
3374
+ return {
3375
+ allowed: false,
3376
+ reason: "Scheduled retry suppressed because the target issue no longer exists",
3377
+ errorCode: "issue_not_found",
3378
+ issueId,
3379
+ details: { issueId },
3380
+ };
3381
+ }
3382
+ if (issue.assigneeAgentId !== run.agentId) {
3383
+ return {
3384
+ allowed: false,
3385
+ reason: "Scheduled retry suppressed because issue ownership changed",
3386
+ errorCode: "issue_reassigned",
3387
+ issueId,
3388
+ details: {
3389
+ issueId,
3390
+ previousAssigneeAgentId: run.agentId,
3391
+ currentAssigneeAgentId: issue.assigneeAgentId,
3392
+ },
3393
+ };
3394
+ }
3395
+ if (issue.status === "cancelled" || issue.status === "done") {
3396
+ return {
3397
+ allowed: false,
3398
+ reason: `Scheduled retry suppressed because issue reached terminal status (${issue.status})`,
3399
+ errorCode: issue.status === "cancelled" ? "issue_cancelled" : "issue_terminal_status",
3400
+ issueId,
3401
+ details: { issueId, currentStatus: issue.status },
3402
+ };
3403
+ }
3404
+ if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON && issue.status !== "in_progress") {
3405
+ return {
3406
+ allowed: false,
3407
+ reason: `Scheduled max-turn continuation suppressed because issue is no longer in_progress (current status: ${issue.status})`,
3408
+ errorCode: "issue_not_in_progress",
3409
+ issueId,
3410
+ details: { issueId, currentStatus: issue.status, requiredStatus: "in_progress" },
3411
+ };
3412
+ }
3413
+ if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON &&
3414
+ input.enforceIssueExecutionLock &&
3415
+ issue.executionRunId !== run.id) {
3416
+ return {
3417
+ allowed: false,
3418
+ reason: "Scheduled max-turn continuation suppressed because the issue execution lock belongs to a different run",
3419
+ errorCode: "issue_execution_lock_changed",
3420
+ issueId,
3421
+ details: {
3422
+ issueId,
3423
+ expectedExecutionRunId: run.id,
3424
+ currentExecutionRunId: issue.executionRunId,
3425
+ },
3426
+ };
3427
+ }
3428
+ if (issue.status === "in_review") {
3429
+ const executionState = parseIssueExecutionState(issue.executionState);
3430
+ const currentParticipant = executionState?.currentParticipant ?? null;
3431
+ if (currentParticipant) {
3432
+ const participantMatches = currentParticipant.type === "agent" && currentParticipant.agentId === run.agentId;
3433
+ if (!participantMatches) {
3434
+ return {
3435
+ allowed: false,
3436
+ reason: "Scheduled retry suppressed because the issue is waiting on another review participant",
3437
+ errorCode: "issue_review_participant_changed",
3438
+ issueId,
3439
+ details: {
3440
+ issueId,
3441
+ currentStageType: executionState?.currentStageType ?? null,
3442
+ currentParticipant,
3443
+ },
3444
+ };
3445
+ }
3446
+ }
3447
+ }
3448
+ const activePauseHold = await treeControlSvc.getActivePauseHoldGate(run.companyId, issueId);
3449
+ if (activePauseHold) {
3450
+ return {
3451
+ allowed: false,
3452
+ reason: "Scheduled retry suppressed because the issue is held by an active subtree pause hold",
3453
+ errorCode: "issue_paused",
3454
+ issueId,
3455
+ details: {
3456
+ issueId,
3457
+ holdId: activePauseHold.holdId,
3458
+ rootIssueId: activePauseHold.rootIssueId,
3459
+ },
3460
+ };
3461
+ }
3462
+ const dependencyReadiness = await issuesSvc.listDependencyReadiness(run.companyId, [issueId]);
3463
+ const readiness = dependencyReadiness.get(issueId);
3464
+ if (readiness && !readiness.isDependencyReady) {
3465
+ return {
3466
+ allowed: false,
3467
+ reason: "Scheduled retry suppressed because issue dependencies are still blocked",
3468
+ errorCode: "issue_dependencies_blocked",
3469
+ issueId,
3470
+ details: {
3471
+ issueId,
3472
+ unresolvedBlockerIssueIds: readiness.unresolvedBlockerIssueIds,
3473
+ unresolvedBlockerCount: readiness.unresolvedBlockerCount,
3474
+ },
3475
+ };
3476
+ }
3477
+ return { allowed: true };
3478
+ }
3479
+ async function cancelScheduledRetryForGate(run, gate, now) {
3480
+ const cancelled = await db
3481
+ .update(heartbeatRuns)
3482
+ .set({
3483
+ status: "cancelled",
3484
+ finishedAt: now,
3485
+ error: gate.reason,
3486
+ errorCode: gate.errorCode,
3487
+ updatedAt: now,
3488
+ })
3489
+ .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "scheduled_retry"), lte(heartbeatRuns.scheduledRetryAt, now)))
3490
+ .returning()
3491
+ .then((rows) => rows[0] ?? null);
3492
+ if (!cancelled)
3493
+ return null;
3494
+ if (cancelled.wakeupRequestId) {
3495
+ await db
3496
+ .update(agentWakeupRequests)
3497
+ .set({
3498
+ status: "cancelled",
3499
+ finishedAt: now,
3500
+ error: gate.reason,
3501
+ updatedAt: now,
3502
+ })
3503
+ .where(eq(agentWakeupRequests.id, cancelled.wakeupRequestId));
3504
+ }
3505
+ if (gate.issueId) {
3506
+ await db
3507
+ .update(issues)
3508
+ .set({
3509
+ executionRunId: null,
3510
+ executionAgentNameKey: null,
3511
+ executionLockedAt: null,
3512
+ updatedAt: now,
3513
+ })
3514
+ .where(and(eq(issues.companyId, cancelled.companyId), eq(issues.id, gate.issueId), eq(issues.executionRunId, cancelled.id)));
3515
+ }
3516
+ await appendRunEvent(cancelled, await nextRunEventSeq(cancelled.id), {
3517
+ eventType: "lifecycle",
3518
+ stream: "system",
3519
+ level: "warn",
3520
+ message: gate.reason,
2625
3521
  payload: {
2626
- retryOfRunId: run.id,
3522
+ ...gate.details,
3523
+ scheduledRetryAttempt: cancelled.scheduledRetryAttempt,
3524
+ scheduledRetryAt: cancelled.scheduledRetryAt ? new Date(cancelled.scheduledRetryAt).toISOString() : null,
3525
+ scheduledRetryReason: cancelled.scheduledRetryReason,
2627
3526
  },
2628
3527
  });
2629
- return queued;
3528
+ return cancelled;
2630
3529
  }
2631
3530
  async function scheduleBoundedRetryForRun(run, agent, opts) {
2632
3531
  const now = opts?.now ?? new Date();
2633
3532
  const retryReason = opts?.retryReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON;
2634
3533
  const wakeReason = opts?.wakeReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON;
3534
+ const maxAttempts = Math.max(0, Math.floor(opts?.maxAttempts ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS));
2635
3535
  const nextAttempt = (run.scheduledRetryAttempt ?? 0) + 1;
2636
- const baseSchedule = computeBoundedTransientHeartbeatRetrySchedule(nextAttempt, now, opts?.random);
3536
+ const baseSchedule = opts?.delayMs != null
3537
+ ? nextAttempt <= maxAttempts
3538
+ ? {
3539
+ attempt: nextAttempt,
3540
+ baseDelayMs: Math.max(0, Math.floor(opts.delayMs)),
3541
+ delayMs: Math.max(0, Math.floor(opts.delayMs)),
3542
+ dueAt: new Date(now.getTime() + Math.max(0, Math.floor(opts.delayMs))),
3543
+ maxAttempts,
3544
+ }
3545
+ : null
3546
+ : nextAttempt <= maxAttempts
3547
+ ? computeBoundedTransientHeartbeatRetrySchedule(nextAttempt, now, opts?.random)
3548
+ : null;
2637
3549
  const transientRecovery = retryReason === BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON
2638
3550
  ? readTransientRecoveryContractFromRun(run)
2639
3551
  : null;
@@ -2650,13 +3562,13 @@ export function heartbeatService(db, options = {}) {
2650
3562
  payload: {
2651
3563
  retryReason,
2652
3564
  scheduledRetryAttempt: run.scheduledRetryAttempt ?? 0,
2653
- maxAttempts: BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS,
3565
+ maxAttempts,
2654
3566
  },
2655
3567
  });
2656
3568
  return {
2657
3569
  outcome: "retry_exhausted",
2658
3570
  attempt: nextAttempt,
2659
- maxAttempts: BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS,
3571
+ maxAttempts,
2660
3572
  };
2661
3573
  }
2662
3574
  const schedule = transientRetryNotBefore && transientRetryNotBefore.getTime() > baseSchedule.dueAt.getTime()
@@ -2668,6 +3580,29 @@ export function heartbeatService(db, options = {}) {
2668
3580
  : baseSchedule;
2669
3581
  const contextSnapshot = parseObject(run.contextSnapshot);
2670
3582
  const issueId = readNonEmptyString(contextSnapshot.issueId);
3583
+ if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON) {
3584
+ const gate = await evaluateScheduledRetryGate({ run, agent, contextSnapshot, retryReason });
3585
+ if (!gate.allowed) {
3586
+ await appendRunEvent(run, await nextRunEventSeq(run.id), {
3587
+ eventType: "lifecycle",
3588
+ stream: "system",
3589
+ level: "warn",
3590
+ message: gate.reason,
3591
+ payload: {
3592
+ retryReason,
3593
+ scheduledRetryAttempt: nextAttempt,
3594
+ maxAttempts,
3595
+ ...gate.details,
3596
+ },
3597
+ });
3598
+ return {
3599
+ outcome: "not_scheduled",
3600
+ reason: gate.reason,
3601
+ errorCode: gate.errorCode,
3602
+ issueId: gate.issueId,
3603
+ };
3604
+ }
3605
+ }
2671
3606
  const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null);
2672
3607
  const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey);
2673
3608
  const retryContextSnapshot = {
@@ -2681,7 +3616,113 @@ export function heartbeatService(db, options = {}) {
2681
3616
  ...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),
2682
3617
  ...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}),
2683
3618
  };
2684
- const retryRun = await db.transaction(async (tx) => {
3619
+ const maxTurnContinuationIdempotencyKey = retryReason === MAX_TURN_CONTINUATION_RETRY_REASON
3620
+ ? `max-turn-continuation:${run.companyId}:${issueId ?? "no-issue"}:${run.id}:${schedule.attempt}`
3621
+ : null;
3622
+ const scheduleResult = await db.transaction(async (tx) => {
3623
+ if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON) {
3624
+ if (issueId) {
3625
+ await tx.execute(sql `select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`);
3626
+ }
3627
+ else {
3628
+ await tx.execute(sql `select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`);
3629
+ }
3630
+ const existingContinuation = await tx
3631
+ .select()
3632
+ .from(heartbeatRuns)
3633
+ .where(and(eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.retryOfRunId, run.id), eq(heartbeatRuns.scheduledRetryReason, retryReason), eq(heartbeatRuns.scheduledRetryAttempt, schedule.attempt), inArray(heartbeatRuns.status, [...MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES]), issueId
3634
+ ? sql `${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`
3635
+ : sql `${heartbeatRuns.contextSnapshot} ->> 'issueId' is null`))
3636
+ .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id))
3637
+ .limit(1)
3638
+ .then((rows) => rows[0] ?? null);
3639
+ if (existingContinuation) {
3640
+ if (existingContinuation.wakeupRequestId) {
3641
+ const existingWakeup = await tx
3642
+ .select({ coalescedCount: agentWakeupRequests.coalescedCount })
3643
+ .from(agentWakeupRequests)
3644
+ .where(eq(agentWakeupRequests.id, existingContinuation.wakeupRequestId))
3645
+ .then((rows) => rows[0] ?? null);
3646
+ await tx
3647
+ .update(agentWakeupRequests)
3648
+ .set({
3649
+ coalescedCount: (existingWakeup?.coalescedCount ?? 0) + 1,
3650
+ updatedAt: now,
3651
+ })
3652
+ .where(eq(agentWakeupRequests.id, existingContinuation.wakeupRequestId));
3653
+ }
3654
+ return {
3655
+ outcome: "scheduled",
3656
+ run: existingContinuation,
3657
+ reusedExisting: true,
3658
+ };
3659
+ }
3660
+ if (issueId) {
3661
+ const lockedIssue = await tx
3662
+ .select({
3663
+ id: issues.id,
3664
+ status: issues.status,
3665
+ assigneeAgentId: issues.assigneeAgentId,
3666
+ executionRunId: issues.executionRunId,
3667
+ })
3668
+ .from(issues)
3669
+ .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))
3670
+ .then((rows) => rows[0] ?? null);
3671
+ if (!lockedIssue) {
3672
+ return {
3673
+ outcome: "not_scheduled",
3674
+ reason: "Scheduled max-turn continuation suppressed because the target issue no longer exists",
3675
+ errorCode: "issue_not_found",
3676
+ issueId,
3677
+ details: { issueId },
3678
+ };
3679
+ }
3680
+ if (lockedIssue.assigneeAgentId !== run.agentId) {
3681
+ return {
3682
+ outcome: "not_scheduled",
3683
+ reason: "Scheduled max-turn continuation suppressed because issue ownership changed",
3684
+ errorCode: "issue_reassigned",
3685
+ issueId,
3686
+ details: {
3687
+ issueId,
3688
+ previousAssigneeAgentId: run.agentId,
3689
+ currentAssigneeAgentId: lockedIssue.assigneeAgentId,
3690
+ },
3691
+ };
3692
+ }
3693
+ if (lockedIssue.status === "cancelled" || lockedIssue.status === "done") {
3694
+ return {
3695
+ outcome: "not_scheduled",
3696
+ reason: `Scheduled max-turn continuation suppressed because issue reached terminal status (${lockedIssue.status})`,
3697
+ errorCode: lockedIssue.status === "cancelled" ? "issue_cancelled" : "issue_terminal_status",
3698
+ issueId,
3699
+ details: { issueId, currentStatus: lockedIssue.status },
3700
+ };
3701
+ }
3702
+ if (lockedIssue.status !== "in_progress") {
3703
+ return {
3704
+ outcome: "not_scheduled",
3705
+ reason: `Scheduled max-turn continuation suppressed because issue is no longer in_progress (current status: ${lockedIssue.status})`,
3706
+ errorCode: "issue_not_in_progress",
3707
+ issueId,
3708
+ details: { issueId, currentStatus: lockedIssue.status, requiredStatus: "in_progress" },
3709
+ };
3710
+ }
3711
+ if (lockedIssue.executionRunId !== run.id) {
3712
+ return {
3713
+ outcome: "not_scheduled",
3714
+ reason: "Scheduled max-turn continuation suppressed because the issue execution lock belongs to a different run",
3715
+ errorCode: "issue_execution_lock_changed",
3716
+ issueId,
3717
+ details: {
3718
+ issueId,
3719
+ expectedExecutionRunId: run.id,
3720
+ currentExecutionRunId: lockedIssue.executionRunId,
3721
+ },
3722
+ };
3723
+ }
3724
+ }
3725
+ }
2685
3726
  const wakeupRequest = await tx
2686
3727
  .insert(agentWakeupRequests)
2687
3728
  .values({
@@ -2703,6 +3744,7 @@ export function heartbeatService(db, options = {}) {
2703
3744
  status: "queued",
2704
3745
  requestedByActorType: "system",
2705
3746
  requestedByActorId: null,
3747
+ idempotencyKey: maxTurnContinuationIdempotencyKey,
2706
3748
  updatedAt: now,
2707
3749
  })
2708
3750
  .returning()
@@ -2745,8 +3787,57 @@ export function heartbeatService(db, options = {}) {
2745
3787
  })
2746
3788
  .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId), eq(issues.executionRunId, run.id)));
2747
3789
  }
2748
- return scheduledRun;
3790
+ return {
3791
+ outcome: "scheduled",
3792
+ run: scheduledRun,
3793
+ reusedExisting: false,
3794
+ };
2749
3795
  });
3796
+ if (scheduleResult.outcome === "not_scheduled") {
3797
+ await appendRunEvent(run, await nextRunEventSeq(run.id), {
3798
+ eventType: "lifecycle",
3799
+ stream: "system",
3800
+ level: "warn",
3801
+ message: scheduleResult.reason,
3802
+ payload: {
3803
+ retryReason,
3804
+ scheduledRetryAttempt: nextAttempt,
3805
+ maxAttempts,
3806
+ ...scheduleResult.details,
3807
+ },
3808
+ });
3809
+ return {
3810
+ outcome: "not_scheduled",
3811
+ reason: scheduleResult.reason,
3812
+ errorCode: scheduleResult.errorCode,
3813
+ issueId: scheduleResult.issueId,
3814
+ };
3815
+ }
3816
+ const retryRun = scheduleResult.run;
3817
+ const dueAt = retryRun.scheduledRetryAt ? new Date(retryRun.scheduledRetryAt) : schedule.dueAt;
3818
+ if (scheduleResult.reusedExisting) {
3819
+ await appendRunEvent(run, await nextRunEventSeq(run.id), {
3820
+ eventType: "lifecycle",
3821
+ stream: "system",
3822
+ level: "info",
3823
+ message: `Reused existing max-turn continuation ${retryRun.scheduledRetryAttempt}/${schedule.maxAttempts}`,
3824
+ payload: {
3825
+ retryRunId: retryRun.id,
3826
+ retryReason,
3827
+ idempotencyKey: maxTurnContinuationIdempotencyKey,
3828
+ scheduledRetryAttempt: retryRun.scheduledRetryAttempt,
3829
+ scheduledRetryAt: dueAt.toISOString(),
3830
+ },
3831
+ });
3832
+ return {
3833
+ outcome: "scheduled",
3834
+ run: retryRun,
3835
+ dueAt,
3836
+ attempt: retryRun.scheduledRetryAttempt,
3837
+ maxAttempts: schedule.maxAttempts,
3838
+ reusedExisting: true,
3839
+ };
3840
+ }
2750
3841
  await appendRunEvent(run, await nextRunEventSeq(run.id), {
2751
3842
  eventType: "lifecycle",
2752
3843
  stream: "system",
@@ -2767,7 +3858,7 @@ export function heartbeatService(db, options = {}) {
2767
3858
  return {
2768
3859
  outcome: "scheduled",
2769
3860
  run: retryRun,
2770
- dueAt: schedule.dueAt,
3861
+ dueAt,
2771
3862
  attempt: schedule.attempt,
2772
3863
  maxAttempts: schedule.maxAttempts,
2773
3864
  };
@@ -2781,76 +3872,33 @@ export function heartbeatService(db, options = {}) {
2781
3872
  .limit(50);
2782
3873
  const promotedRunIds = [];
2783
3874
  for (const dueRun of dueRuns) {
2784
- const dueRunIssueId = readNonEmptyString(parseObject(dueRun.contextSnapshot).issueId);
2785
- if (dueRunIssueId) {
2786
- const issue = await db
2787
- .select({
2788
- id: issues.id,
2789
- status: issues.status,
2790
- assigneeAgentId: issues.assigneeAgentId,
2791
- executionRunId: issues.executionRunId,
2792
- })
2793
- .from(issues)
2794
- .where(and(eq(issues.id, dueRunIssueId), eq(issues.companyId, dueRun.companyId)))
2795
- .then((rows) => rows[0] ?? null);
2796
- if (issue && (issue.assigneeAgentId !== dueRun.agentId || issue.status === "cancelled")) {
2797
- const issueCancelled = issue.status === "cancelled";
2798
- const reason = issueCancelled
2799
- ? "Cancelled because the issue was cancelled before the scheduled retry became due"
2800
- : "Cancelled because the issue was reassigned before the scheduled retry became due";
2801
- const cancelled = await db
2802
- .update(heartbeatRuns)
2803
- .set({
2804
- status: "cancelled",
2805
- finishedAt: now,
2806
- error: reason,
2807
- errorCode: issueCancelled ? "issue_cancelled" : "issue_reassigned",
2808
- updatedAt: now,
2809
- })
2810
- .where(and(eq(heartbeatRuns.id, dueRun.id), eq(heartbeatRuns.status, "scheduled_retry"), lte(heartbeatRuns.scheduledRetryAt, now)))
2811
- .returning()
2812
- .then((rows) => rows[0] ?? null);
2813
- if (!cancelled)
2814
- continue;
2815
- if (cancelled.wakeupRequestId) {
2816
- await db
2817
- .update(agentWakeupRequests)
2818
- .set({
2819
- status: "cancelled",
2820
- finishedAt: now,
2821
- error: reason,
2822
- updatedAt: now,
2823
- })
2824
- .where(eq(agentWakeupRequests.id, cancelled.wakeupRequestId));
2825
- }
2826
- if (issue.executionRunId === cancelled.id) {
2827
- await db
2828
- .update(issues)
2829
- .set({
2830
- executionRunId: null,
2831
- executionAgentNameKey: null,
2832
- executionLockedAt: null,
2833
- updatedAt: now,
2834
- })
2835
- .where(and(eq(issues.id, issue.id), eq(issues.executionRunId, cancelled.id)));
2836
- }
2837
- await appendRunEvent(cancelled, await nextRunEventSeq(cancelled.id), {
2838
- eventType: "lifecycle",
2839
- stream: "system",
2840
- level: "warn",
2841
- message: issueCancelled
2842
- ? "Scheduled retry cancelled because issue was cancelled before it became due"
2843
- : "Scheduled retry cancelled because issue ownership changed before it became due",
2844
- payload: {
2845
- issueId: issue.id,
2846
- issueStatus: issue.status,
2847
- scheduledRetryAttempt: cancelled.scheduledRetryAttempt,
2848
- scheduledRetryAt: cancelled.scheduledRetryAt ? new Date(cancelled.scheduledRetryAt).toISOString() : null,
2849
- scheduledRetryReason: cancelled.scheduledRetryReason,
2850
- previousRetryAgentId: cancelled.agentId,
2851
- currentAssigneeAgentId: issue.assigneeAgentId,
2852
- },
2853
- });
3875
+ const agent = await getAgent(dueRun.agentId);
3876
+ if (!agent) {
3877
+ await cancelScheduledRetryForGate(dueRun, {
3878
+ allowed: false,
3879
+ reason: "Scheduled retry suppressed because the agent no longer exists",
3880
+ errorCode: "agent_not_invokable",
3881
+ issueId: readNonEmptyString(parseObject(dueRun.contextSnapshot).issueId),
3882
+ details: { agentId: dueRun.agentId },
3883
+ }, now);
3884
+ continue;
3885
+ }
3886
+ const contextSnapshot = parseObject(dueRun.contextSnapshot);
3887
+ const gate = await evaluateScheduledRetryGate({
3888
+ run: dueRun,
3889
+ agent,
3890
+ contextSnapshot,
3891
+ retryReason: dueRun.scheduledRetryReason,
3892
+ enforceIssueExecutionLock: dueRun.scheduledRetryReason === MAX_TURN_CONTINUATION_RETRY_REASON,
3893
+ });
3894
+ if (!gate.allowed) {
3895
+ if (gate.errorCode === "issue_not_found" &&
3896
+ dueRun.scheduledRetryReason !== MAX_TURN_CONTINUATION_RETRY_REASON) {
3897
+ // Preserve legacy transient retry behavior for runs that only carry a
3898
+ // loose task context rather than a persisted issue row.
3899
+ }
3900
+ else {
3901
+ await cancelScheduledRetryForGate(dueRun, gate, now);
2854
3902
  continue;
2855
3903
  }
2856
3904
  }
@@ -2904,6 +3952,18 @@ export function heartbeatService(db, options = {}) {
2904
3952
  maxConcurrentRuns: normalizeMaxConcurrentRuns(heartbeat.maxConcurrentRuns),
2905
3953
  };
2906
3954
  }
3955
+ function parseMaxTurnContinuationPolicy(agent) {
3956
+ const runtimeConfig = parseObject(agent.runtimeConfig);
3957
+ const heartbeat = parseObject(runtimeConfig.heartbeat);
3958
+ const configured = parseObject(heartbeat.maxTurnContinuation);
3959
+ const rawMaxAttempts = Math.floor(asNumber(configured.maxAttempts, MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS));
3960
+ const rawDelayMs = Math.floor(asNumber(configured.delayMs, MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS));
3961
+ return {
3962
+ enabled: asBoolean(configured.enabled, true),
3963
+ maxAttempts: Math.max(0, Math.min(MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP, rawMaxAttempts)),
3964
+ delayMs: Math.max(0, Math.min(MAX_TURN_CONTINUATION_MAX_DELAY_MS, rawDelayMs)),
3965
+ };
3966
+ }
2907
3967
  function issueRunPriorityRank(priority) {
2908
3968
  switch (priority) {
2909
3969
  case "critical":
@@ -2995,6 +4055,12 @@ export function heartbeatService(db, options = {}) {
2995
4055
  logger.info({ runId: run.id, issueId, unresolvedBlockerCount }, "claimQueuedRun: cancelled blocked queued run");
2996
4056
  return null;
2997
4057
  }
4058
+ const staleness = await evaluateQueuedRunStaleness(run, issueId, context);
4059
+ if (staleness.stale) {
4060
+ await cancelQueuedRunForStaleIssue(run, issueId, staleness);
4061
+ logger.info({ runId: run.id, issueId, errorCode: staleness.errorCode }, "claimQueuedRun: cancelled stale queued run");
4062
+ return null;
4063
+ }
2998
4064
  }
2999
4065
  const claimedAt = new Date();
3000
4066
  const claimed = await db
@@ -3089,6 +4155,132 @@ export function heartbeatService(db, options = {}) {
3089
4155
  });
3090
4156
  return cancelled;
3091
4157
  }
4158
+ async function evaluateQueuedRunStaleness(run, issueId, context) {
4159
+ const issue = await db
4160
+ .select({
4161
+ id: issues.id,
4162
+ status: issues.status,
4163
+ assigneeAgentId: issues.assigneeAgentId,
4164
+ executionRunId: issues.executionRunId,
4165
+ executionState: issues.executionState,
4166
+ })
4167
+ .from(issues)
4168
+ .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))
4169
+ .then((rows) => rows[0] ?? null);
4170
+ if (!issue) {
4171
+ return {
4172
+ stale: true,
4173
+ errorCode: "issue_not_found",
4174
+ reason: "Cancelled because the target issue no longer exists",
4175
+ details: { issueId },
4176
+ };
4177
+ }
4178
+ const wakeCommentId = deriveCommentId(context, null);
4179
+ const isInteractionWake = allowsIssueInteractionWake(context);
4180
+ const resumeIntent = context.resumeIntent === true || context.followUpRequested === true;
4181
+ const retryReason = readNonEmptyString(context.retryReason) ?? run.scheduledRetryReason ?? null;
4182
+ if (issue.assigneeAgentId !== run.agentId && !isInteractionWake) {
4183
+ return {
4184
+ stale: true,
4185
+ errorCode: "issue_assignee_changed",
4186
+ reason: "Cancelled because issue assignee changed before the queued run could start; the new owner will be woken instead",
4187
+ details: {
4188
+ issueId,
4189
+ previousAssigneeAgentId: run.agentId,
4190
+ currentAssigneeAgentId: issue.assigneeAgentId,
4191
+ },
4192
+ };
4193
+ }
4194
+ if (issue.status === "done" || issue.status === "cancelled") {
4195
+ if (!resumeIntent && !wakeCommentId) {
4196
+ return {
4197
+ stale: true,
4198
+ errorCode: "issue_terminal_status",
4199
+ reason: `Cancelled because issue reached terminal status (${issue.status}) before the queued run could start`,
4200
+ details: { issueId, currentStatus: issue.status },
4201
+ };
4202
+ }
4203
+ }
4204
+ if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON && issue.status !== "in_progress") {
4205
+ return {
4206
+ stale: true,
4207
+ errorCode: "issue_not_in_progress",
4208
+ reason: `Cancelled because max-turn continuation issue is no longer in_progress (current status: ${issue.status}) before the queued run could start`,
4209
+ details: { issueId, currentStatus: issue.status, requiredStatus: "in_progress" },
4210
+ };
4211
+ }
4212
+ if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON && issue.executionRunId !== run.id) {
4213
+ return {
4214
+ stale: true,
4215
+ errorCode: "issue_execution_lock_changed",
4216
+ reason: "Cancelled because max-turn continuation no longer owns the issue execution lock before the queued run could start",
4217
+ details: {
4218
+ issueId,
4219
+ expectedExecutionRunId: run.id,
4220
+ currentExecutionRunId: issue.executionRunId,
4221
+ },
4222
+ };
4223
+ }
4224
+ if (issue.status === "in_review") {
4225
+ const executionState = parseIssueExecutionState(issue.executionState);
4226
+ const currentParticipant = executionState?.currentParticipant ?? null;
4227
+ if (currentParticipant) {
4228
+ const participantMatches = currentParticipant.type === "agent" && currentParticipant.agentId === run.agentId;
4229
+ if (!participantMatches && !wakeCommentId) {
4230
+ return {
4231
+ stale: true,
4232
+ errorCode: "issue_review_participant_changed",
4233
+ reason: "Cancelled because the in-review participant changed before the queued run could start; the current participant will be woken instead",
4234
+ details: {
4235
+ issueId,
4236
+ currentStageType: executionState?.currentStageType ?? null,
4237
+ currentParticipant,
4238
+ },
4239
+ };
4240
+ }
4241
+ }
4242
+ }
4243
+ return { stale: false };
4244
+ }
4245
+ async function cancelQueuedRunForStaleIssue(run, issueId, staleness) {
4246
+ const now = new Date();
4247
+ const cancelled = await setRunStatus(run.id, "cancelled", {
4248
+ finishedAt: now,
4249
+ error: staleness.reason,
4250
+ errorCode: staleness.errorCode,
4251
+ resultJson: {
4252
+ ...parseObject(run.resultJson),
4253
+ stopReason: staleness.errorCode,
4254
+ effectiveTimeoutSec: 0,
4255
+ timeoutConfigured: false,
4256
+ timeoutSource: "stale_queued_run_gate",
4257
+ timeoutFired: false,
4258
+ },
4259
+ });
4260
+ if (!cancelled)
4261
+ return null;
4262
+ await setWakeupStatus(run.wakeupRequestId, "skipped", {
4263
+ finishedAt: now,
4264
+ error: staleness.reason,
4265
+ });
4266
+ await db
4267
+ .update(issues)
4268
+ .set({
4269
+ executionRunId: null,
4270
+ executionAgentNameKey: null,
4271
+ executionLockedAt: null,
4272
+ updatedAt: now,
4273
+ })
4274
+ .where(and(eq(issues.companyId, run.companyId), eq(issues.id, issueId), eq(issues.executionRunId, run.id)));
4275
+ await appendRunEvent(cancelled, await nextRunEventSeq(cancelled.id), {
4276
+ eventType: "lifecycle",
4277
+ stream: "system",
4278
+ level: "warn",
4279
+ message: staleness.reason,
4280
+ payload: staleness.details,
4281
+ });
4282
+ return cancelled;
4283
+ }
3092
4284
  async function finalizeAgentStatus(agentId, outcome) {
3093
4285
  const existing = await getAgent(agentId);
3094
4286
  if (!existing)
@@ -3359,6 +4551,13 @@ export function heartbeatService(db, options = {}) {
3359
4551
  if (!finalizedRun)
3360
4552
  continue;
3361
4553
  finalizedRun = await classifyAndPersistRunLiveness(finalizedRun, parseObject(finalizedRun.resultJson)) ?? finalizedRun;
4554
+ await releaseEnvironmentLeasesForRun({
4555
+ runId: finalizedRun.id,
4556
+ companyId: finalizedRun.companyId,
4557
+ agentId: finalizedRun.agentId,
4558
+ status: finalizedRun.status,
4559
+ failureReason: finalizedRun.error ?? undefined,
4560
+ });
3362
4561
  let retriedRun = null;
3363
4562
  if (shouldRetry) {
3364
4563
  const agent = await getAgent(run.agentId);
@@ -3420,9 +4619,15 @@ export function heartbeatService(db, options = {}) {
3420
4619
  async function scanSilentActiveRuns(opts) {
3421
4620
  return recovery.scanSilentActiveRuns(opts);
3422
4621
  }
4622
+ async function reconcileProductivityReviews(opts) {
4623
+ return productivityReviews.reconcileProductivityReviews(opts);
4624
+ }
3423
4625
  async function buildRunOutputSilence(run, now = new Date()) {
3424
4626
  return recovery.buildRunOutputSilence(run, now);
3425
4627
  }
4628
+ async function buildIssueGraphLivenessAutoRecoveryPreview(opts) {
4629
+ return recovery.buildIssueGraphLivenessAutoRecoveryPreview(opts);
4630
+ }
3426
4631
  async function reconcileIssueGraphLiveness(opts) {
3427
4632
  return recovery.reconcileIssueGraphLiveness(opts);
3428
4633
  }
@@ -3743,6 +4948,9 @@ export function heartbeatService(db, options = {}) {
3743
4948
  const shouldReuseExisting = issueRef?.executionWorkspacePreference === "reuse_existing" &&
3744
4949
  existingExecutionWorkspace !== null &&
3745
4950
  existingExecutionWorkspace.status !== "archived";
4951
+ const reusableExecutionWorkspaceConfig = shouldReuseExisting
4952
+ ? existingExecutionWorkspace?.config ?? null
4953
+ : null;
3746
4954
  const persistedExecutionWorkspaceMode = shouldReuseExisting && existingExecutionWorkspace
3747
4955
  ? issueExecutionWorkspaceModeForPersistedWorkspace(existingExecutionWorkspace.mode)
3748
4956
  : null;
@@ -3755,7 +4963,7 @@ export function heartbeatService(db, options = {}) {
3755
4963
  const selectedEnvironmentId = resolveExecutionWorkspaceEnvironmentId({
3756
4964
  projectPolicy: projectExecutionWorkspacePolicy,
3757
4965
  issueSettings: issueExecutionWorkspaceSettings,
3758
- workspaceConfig: existingExecutionWorkspace?.config ?? null,
4966
+ workspaceConfig: reusableExecutionWorkspaceConfig,
3759
4967
  agentDefaultEnvironmentId: agent.defaultEnvironmentId,
3760
4968
  defaultEnvironmentId: defaultEnvironment.id,
3761
4969
  });
@@ -3770,12 +4978,45 @@ export function heartbeatService(db, options = {}) {
3770
4978
  });
3771
4979
  const persistedWorkspaceManagedConfig = applyPersistedExecutionWorkspaceConfig({
3772
4980
  config: workspaceManagedConfig,
3773
- workspaceConfig: existingExecutionWorkspace?.config ?? null,
4981
+ workspaceConfig: reusableExecutionWorkspaceConfig,
3774
4982
  mode: effectiveExecutionWorkspaceMode,
3775
4983
  });
3776
- const mergedConfig = issueAssigneeOverrides?.adapterConfig
3777
- ? { ...persistedWorkspaceManagedConfig, ...issueAssigneeOverrides.adapterConfig }
3778
- : persistedWorkspaceManagedConfig;
4984
+ let adapterModelProfiles = [];
4985
+ let profileResolutionFallbackReason = null;
4986
+ try {
4987
+ adapterModelProfiles = await listAdapterModelProfiles(agent.adapterType);
4988
+ }
4989
+ catch (error) {
4990
+ profileResolutionFallbackReason = "adapter_profile_resolution_failed";
4991
+ logger.warn({
4992
+ err: error,
4993
+ companyId: agent.companyId,
4994
+ agentId: agent.id,
4995
+ adapterType: agent.adapterType,
4996
+ runId: run.id,
4997
+ }, "Failed to resolve adapter model profiles; falling back to primary adapter config");
4998
+ }
4999
+ const modelProfileApplication = resolveModelProfileApplication({
5000
+ adapterModelProfiles,
5001
+ agentRuntimeConfig: agent.runtimeConfig,
5002
+ issueModelProfile: issueAssigneeOverrides?.modelProfile ?? null,
5003
+ contextSnapshot: context,
5004
+ profileResolutionFallbackReason,
5005
+ });
5006
+ const modelProfileMetadata = modelProfileRunMetadata(modelProfileApplication);
5007
+ if (modelProfileMetadata) {
5008
+ context.paperclipModelProfile = modelProfileMetadata;
5009
+ if (modelProfileApplication.requested)
5010
+ context.modelProfile = modelProfileApplication.requested;
5011
+ }
5012
+ else {
5013
+ delete context.paperclipModelProfile;
5014
+ }
5015
+ const mergedConfig = mergeModelProfileAdapterConfig({
5016
+ baseConfig: persistedWorkspaceManagedConfig,
5017
+ modelProfile: modelProfileApplication,
5018
+ issueAdapterConfig: issueAssigneeOverrides?.adapterConfig ?? null,
5019
+ });
3779
5020
  const configSnapshot = buildExecutionWorkspaceConfigSnapshot(mergedConfig, selectedEnvironmentId);
3780
5021
  const executionRunConfig = stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig);
3781
5022
  const { resolvedConfig, secretKeys } = await resolveExecutionRunAdapterConfig({
@@ -4298,12 +5539,16 @@ export function heartbeatService(db, options = {}) {
4298
5539
  meta.env[key] = "***REDACTED***";
4299
5540
  }
4300
5541
  }
5542
+ const modelProfileMetadata = modelProfileRunMetadata(modelProfileApplication);
4301
5543
  await appendRunEvent(currentRun, seq++, {
4302
5544
  eventType: "adapter.invoke",
4303
5545
  stream: "system",
4304
5546
  level: "info",
4305
5547
  message: "adapter invocation",
4306
- payload: meta,
5548
+ payload: {
5549
+ ...meta,
5550
+ ...(modelProfileMetadata ? { modelProfile: modelProfileMetadata } : {}),
5551
+ },
4307
5552
  });
4308
5553
  };
4309
5554
  const adapter = getServerAdapter(agent.adapterType);
@@ -4324,6 +5569,7 @@ export function heartbeatService(db, options = {}) {
4324
5569
  runtime: runtimeForAdapter,
4325
5570
  config: runtimeConfig,
4326
5571
  context,
5572
+ runtimeCommandSpec: adapter.getRuntimeCommandSpec?.(runtimeConfig) ?? null,
4327
5573
  executionTarget,
4328
5574
  executionTransport: remoteExecution
4329
5575
  ? { remoteExecution: remoteExecution }
@@ -4465,11 +5711,11 @@ export function heartbeatService(db, options = {}) {
4465
5711
  }
4466
5712
  : null;
4467
5713
  const persistedResultJson = mergeHeartbeatRunResultJson(mergeRunStopMetadataForAgent(agent, outcome, {
4468
- resultJson: mergeAdapterRecoveryMetadata({
5714
+ resultJson: mergeModelProfileRunMetadata(mergeAdapterRecoveryMetadata({
4469
5715
  resultJson: adapterResult.resultJson ?? null,
4470
5716
  errorFamily: adapterResult.errorFamily ?? null,
4471
5717
  retryNotBefore: adapterResult.retryNotBefore ?? null,
4472
- }),
5718
+ }), modelProfileApplication),
4473
5719
  errorCode: runErrorCode,
4474
5720
  errorMessage: runErrorMessage,
4475
5721
  }), adapterResult.summary ?? null);
@@ -4523,7 +5769,30 @@ export function heartbeatService(db, options = {}) {
4523
5769
  await onLog("stderr", `[paperclip] Failed to post run summary comment: ${err instanceof Error ? err.message : String(err)}\n`);
4524
5770
  }
4525
5771
  }
4526
- if (outcome === "failed" && readTransientRecoveryContractFromRun(livenessRun)) {
5772
+ if (outcome === "failed" && isMaxTurnExhaustionRun(livenessRun)) {
5773
+ const policy = parseMaxTurnContinuationPolicy(agent);
5774
+ if (policy.enabled && policy.maxAttempts > 0) {
5775
+ await scheduleBoundedRetryForRun(livenessRun, agent, {
5776
+ retryReason: MAX_TURN_CONTINUATION_RETRY_REASON,
5777
+ wakeReason: MAX_TURN_CONTINUATION_WAKE_REASON,
5778
+ maxAttempts: policy.maxAttempts,
5779
+ delayMs: policy.delayMs,
5780
+ });
5781
+ }
5782
+ else {
5783
+ await appendRunEvent(livenessRun, await nextRunEventSeq(livenessRun.id), {
5784
+ eventType: "lifecycle",
5785
+ stream: "system",
5786
+ level: "warn",
5787
+ message: "Max-turn continuation suppressed because the policy is disabled",
5788
+ payload: {
5789
+ retryReason: MAX_TURN_CONTINUATION_RETRY_REASON,
5790
+ policy,
5791
+ },
5792
+ });
5793
+ }
5794
+ }
5795
+ else if (outcome === "failed" && readTransientRecoveryContractFromRun(livenessRun)) {
4527
5796
  await scheduleBoundedRetryForRun(livenessRun, agent);
4528
5797
  }
4529
5798
  await finalizeIssueCommentPolicy(livenessRun, agent);
@@ -4674,19 +5943,13 @@ export function heartbeatService(db, options = {}) {
4674
5943
  }
4675
5944
  finally {
4676
5945
  const latestRun = await getRun(run.id).catch(() => null);
4677
- const releaseResult = await envOrchestrator.releaseForRun({
4678
- heartbeatRunId: run.id,
5946
+ await releaseEnvironmentLeasesForRun({
5947
+ runId: run.id,
4679
5948
  companyId: run.companyId,
4680
5949
  agentId: run.agentId,
4681
- status: leaseReleaseStatusForRunStatus(latestRun?.status),
5950
+ status: latestRun?.status,
4682
5951
  failureReason: latestRun?.error ?? undefined,
4683
- }).catch((err) => {
4684
- logger.warn({ err, runId: run.id }, "failed to release environment leases for heartbeat run");
4685
- return null;
4686
5952
  });
4687
- for (const releaseError of releaseResult?.errors ?? []) {
4688
- logger.warn({ err: releaseError.error, leaseId: releaseError.leaseId, runId: run.id }, "failed to release environment lease for heartbeat run");
4689
- }
4690
5953
  await releaseRuntimeServicesForRun(run.id).catch(() => undefined);
4691
5954
  activeRunExecutions.delete(run.id);
4692
5955
  await startNextQueuedRunForAgent(run.agentId);
@@ -4933,7 +6196,8 @@ export function heartbeatService(db, options = {}) {
4933
6196
  if (await isAutomaticRecoverySuppressedByPauseHold(db, issue.companyId, issue.id, treeControlSvc)) {
4934
6197
  return { kind: "released" };
4935
6198
  }
4936
- const shouldBlockImmediately = !recoveryAgentInvokable ||
6199
+ const shouldBlockImmediately = issue.originKind === RECOVERY_ORIGIN_KINDS.strandedIssueRecovery ||
6200
+ !recoveryAgentInvokable ||
4937
6201
  !recoveryAgent ||
4938
6202
  didAutomaticRecoveryFail(run, issue.status === "todo" ? "assignment_recovery" : "issue_continuation_needed");
4939
6203
  if (shouldBlockImmediately) {
@@ -5951,6 +7215,7 @@ export function heartbeatService(db, options = {}) {
5951
7215
  requestedByActorId: actor?.actorId ?? null,
5952
7216
  }),
5953
7217
  wakeup: enqueueWakeup,
7218
+ triggerIssueMonitor,
5954
7219
  reportRunActivity: clearDetachedRunWarning,
5955
7220
  reapOrphanedRuns,
5956
7221
  promoteDueScheduledRetries,
@@ -5965,8 +7230,10 @@ export function heartbeatService(db, options = {}) {
5965
7230
  return scheduleBoundedRetryForRun(run, agent, opts);
5966
7231
  },
5967
7232
  reconcileStrandedAssignedIssues,
7233
+ buildIssueGraphLivenessAutoRecoveryPreview,
5968
7234
  reconcileIssueGraphLiveness,
5969
7235
  scanSilentActiveRuns,
7236
+ reconcileProductivityReviews,
5970
7237
  buildRunOutputSilence,
5971
7238
  tickTimers: async (now = new Date()) => {
5972
7239
  const allAgents = await db.select().from(agents);
@@ -6001,7 +7268,12 @@ export function heartbeatService(db, options = {}) {
6001
7268
  else
6002
7269
  skipped += 1;
6003
7270
  }
6004
- return { checked, enqueued, skipped };
7271
+ const issueMonitors = await tickDueIssueMonitors(now);
7272
+ return {
7273
+ checked: checked + issueMonitors.checked,
7274
+ enqueued: enqueued + issueMonitors.triggered,
7275
+ skipped: skipped + issueMonitors.skipped,
7276
+ };
6005
7277
  },
6006
7278
  cancelRun: (runId) => cancelRunInternal(runId),
6007
7279
  cancelActiveForAgent: (agentId) => cancelActiveForAgentInternal(agentId),