@rudderhq/server 0.1.0-canary.2 → 0.1.0-canary.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (187) hide show
  1. package/dist/bootstrap/register-api-routes.js +1 -1
  2. package/dist/bootstrap/register-api-routes.js.map +1 -1
  3. package/dist/dev-server-status.d.ts +1 -7
  4. package/dist/dev-server-status.d.ts.map +1 -1
  5. package/dist/dev-server-status.js +1 -4
  6. package/dist/dev-server-status.js.map +1 -1
  7. package/dist/routes/agents.d.ts +2 -1
  8. package/dist/routes/agents.d.ts.map +1 -1
  9. package/dist/routes/agents.js +158 -1
  10. package/dist/routes/agents.js.map +1 -1
  11. package/dist/routes/chats.d.ts.map +1 -1
  12. package/dist/routes/chats.js +164 -21
  13. package/dist/routes/chats.js.map +1 -1
  14. package/dist/routes/dashboard.d.ts.map +1 -1
  15. package/dist/routes/dashboard.js +24 -0
  16. package/dist/routes/dashboard.js.map +1 -1
  17. package/dist/routes/health.d.ts.map +1 -1
  18. package/dist/routes/health.js +3 -12
  19. package/dist/routes/health.js.map +1 -1
  20. package/dist/routes/instance-settings.d.ts.map +1 -1
  21. package/dist/routes/instance-settings.js +1 -26
  22. package/dist/routes/instance-settings.js.map +1 -1
  23. package/dist/routes/issues.d.ts.map +1 -1
  24. package/dist/routes/issues.js +19 -17
  25. package/dist/routes/issues.js.map +1 -1
  26. package/dist/services/agent-run-context.d.ts +1 -1
  27. package/dist/services/agents.d.ts +13 -13
  28. package/dist/services/chat-assistant.d.ts +1 -1
  29. package/dist/services/chat-assistant.d.ts.map +1 -1
  30. package/dist/services/chat-assistant.js +244 -41
  31. package/dist/services/chat-assistant.js.map +1 -1
  32. package/dist/services/chat-generation-locks.d.ts +2 -1
  33. package/dist/services/chat-generation-locks.d.ts.map +1 -1
  34. package/dist/services/chat-generation-locks.js +12 -3
  35. package/dist/services/chat-generation-locks.js.map +1 -1
  36. package/dist/services/chats.d.ts +3 -3
  37. package/dist/services/chats.d.ts.map +1 -1
  38. package/dist/services/chats.js +21 -2
  39. package/dist/services/chats.js.map +1 -1
  40. package/dist/services/costs.d.ts +1 -1
  41. package/dist/services/documents.d.ts +23 -0
  42. package/dist/services/documents.d.ts.map +1 -1
  43. package/dist/services/documents.js +17 -1
  44. package/dist/services/documents.js.map +1 -1
  45. package/dist/services/finance.d.ts +2 -2
  46. package/dist/services/heartbeat.d.ts +1 -1
  47. package/dist/services/heartbeat.d.ts.map +1 -1
  48. package/dist/services/heartbeat.js +1 -1
  49. package/dist/services/heartbeat.js.map +1 -1
  50. package/dist/services/instance-settings.d.ts +1 -3
  51. package/dist/services/instance-settings.d.ts.map +1 -1
  52. package/dist/services/instance-settings.js +1 -38
  53. package/dist/services/instance-settings.js.map +1 -1
  54. package/dist/services/messenger.d.ts +1 -1
  55. package/dist/services/messenger.d.ts.map +1 -1
  56. package/dist/services/messenger.js +59 -12
  57. package/dist/services/messenger.js.map +1 -1
  58. package/dist/services/runtime-kernel/analytics.d.ts +7 -0
  59. package/dist/services/runtime-kernel/analytics.d.ts.map +1 -0
  60. package/dist/services/runtime-kernel/analytics.js +46 -0
  61. package/dist/services/runtime-kernel/analytics.js.map +1 -0
  62. package/dist/services/runtime-kernel/common.d.ts +4 -0
  63. package/dist/services/runtime-kernel/common.d.ts.map +1 -0
  64. package/dist/services/runtime-kernel/common.js +15 -0
  65. package/dist/services/runtime-kernel/common.js.map +1 -0
  66. package/dist/services/runtime-kernel/execution.d.ts +11 -0
  67. package/dist/services/runtime-kernel/execution.d.ts.map +1 -0
  68. package/dist/services/runtime-kernel/execution.js +28 -0
  69. package/dist/services/runtime-kernel/execution.js.map +1 -0
  70. package/dist/services/runtime-kernel/heartbeat.d.ts +1 -881
  71. package/dist/services/runtime-kernel/heartbeat.d.ts.map +1 -1
  72. package/dist/services/runtime-kernel/heartbeat.js +1 -4344
  73. package/dist/services/runtime-kernel/heartbeat.js.map +1 -1
  74. package/dist/services/runtime-kernel/model-fallback.d.ts +10 -0
  75. package/dist/services/runtime-kernel/model-fallback.d.ts.map +1 -0
  76. package/dist/services/runtime-kernel/model-fallback.js +147 -0
  77. package/dist/services/runtime-kernel/model-fallback.js.map +1 -0
  78. package/dist/services/runtime-kernel/orchestrator.d.ts +826 -0
  79. package/dist/services/runtime-kernel/orchestrator.d.ts.map +1 -0
  80. package/dist/services/runtime-kernel/orchestrator.js +4044 -0
  81. package/dist/services/runtime-kernel/orchestrator.js.map +1 -0
  82. package/dist/services/runtime-kernel/recovery.d.ts +8 -0
  83. package/dist/services/runtime-kernel/recovery.d.ts.map +1 -0
  84. package/dist/services/runtime-kernel/recovery.js +101 -0
  85. package/dist/services/runtime-kernel/recovery.js.map +1 -0
  86. package/dist/services/runtime-kernel/run-state.d.ts +45 -0
  87. package/dist/services/runtime-kernel/run-state.d.ts.map +1 -0
  88. package/dist/services/runtime-kernel/run-state.js +45 -0
  89. package/dist/services/runtime-kernel/run-state.js.map +1 -0
  90. package/dist/services/runtime-kernel/session-policy.d.ts +42 -0
  91. package/dist/services/runtime-kernel/session-policy.d.ts.map +1 -0
  92. package/dist/services/runtime-kernel/session-policy.js +116 -0
  93. package/dist/services/runtime-kernel/session-policy.js.map +1 -0
  94. package/dist/services/runtime-kernel/types.d.ts +13 -0
  95. package/dist/services/runtime-kernel/types.d.ts.map +1 -0
  96. package/dist/services/runtime-kernel/types.js +2 -0
  97. package/dist/services/runtime-kernel/types.js.map +1 -0
  98. package/dist/services/runtime-kernel/wakeup-queue.d.ts +9 -0
  99. package/dist/services/runtime-kernel/wakeup-queue.d.ts.map +1 -0
  100. package/dist/services/runtime-kernel/wakeup-queue.js +19 -0
  101. package/dist/services/runtime-kernel/wakeup-queue.js.map +1 -0
  102. package/package.json +14 -14
  103. package/resources/bundled-skills/rudder-create-agent/SKILL.md +1 -1
  104. package/resources/bundled-skills/rudder-create-agent/references/api-reference.md +4 -2
  105. package/resources/bundled-skills/rudder-create-agent/references/cli-reference.md +1 -1
  106. package/skills/rudder-create-agent/SKILL.md +1 -1
  107. package/skills/rudder-create-agent/references/api-reference.md +4 -2
  108. package/skills/rudder-create-agent/references/cli-reference.md +1 -1
  109. package/ui-dist/assets/{_basePickBy-C8ZhuJlI.js → _basePickBy-DAU2OOE3.js} +1 -1
  110. package/ui-dist/assets/{_baseUniq-BRCVYeJb.js → _baseUniq-BQLJfIXV.js} +1 -1
  111. package/ui-dist/assets/{arc-B1R0dxYm.js → arc-B2UTEkTB.js} +1 -1
  112. package/ui-dist/assets/{architectureDiagram-2XIMDMQ5-D7sRWBaf.js → architectureDiagram-2XIMDMQ5-DicHsSOp.js} +1 -1
  113. package/ui-dist/assets/{blockDiagram-WCTKOSBZ-Byw6t-Eq.js → blockDiagram-WCTKOSBZ-tbkC8ZCs.js} +1 -1
  114. package/ui-dist/assets/{c4Diagram-IC4MRINW-RsrU80Hc.js → c4Diagram-IC4MRINW-pPT_Xw2l.js} +1 -1
  115. package/ui-dist/assets/channel-D8qCsVGC.js +1 -0
  116. package/ui-dist/assets/{chunk-4BX2VUAB-D9oP4FJ_.js → chunk-4BX2VUAB-vX_EgGXp.js} +1 -1
  117. package/ui-dist/assets/{chunk-55IACEB6-DT8xQAKU.js → chunk-55IACEB6-DUyGlyCg.js} +1 -1
  118. package/ui-dist/assets/{chunk-FMBD7UC4-FwqsP6AV.js → chunk-FMBD7UC4-DPCYqV54.js} +1 -1
  119. package/ui-dist/assets/{chunk-JSJVCQXG-CTOvJxU3.js → chunk-JSJVCQXG-CRqpXPJG.js} +1 -1
  120. package/ui-dist/assets/{chunk-KX2RTZJC-8oze7Khf.js → chunk-KX2RTZJC-BWaSxc23.js} +1 -1
  121. package/ui-dist/assets/{chunk-NQ4KR5QH-D38LrSCR.js → chunk-NQ4KR5QH-BCIl9FTX.js} +1 -1
  122. package/ui-dist/assets/{chunk-QZHKN3VN-B3ZhJA7D.js → chunk-QZHKN3VN-BgTqyV62.js} +1 -1
  123. package/ui-dist/assets/{chunk-WL4C6EOR-CSTjCHX6.js → chunk-WL4C6EOR-COlN9kxA.js} +1 -1
  124. package/ui-dist/assets/classDiagram-VBA2DB6C-Cm8BRWIa.js +1 -0
  125. package/ui-dist/assets/classDiagram-v2-RAHNMMFH-Cm8BRWIa.js +1 -0
  126. package/ui-dist/assets/clone-B0pIfwDy.js +1 -0
  127. package/ui-dist/assets/{cose-bilkent-S5V4N54A-CI_3Qz62.js → cose-bilkent-S5V4N54A-Bbgtq4rD.js} +1 -1
  128. package/ui-dist/assets/{dagre-KLK3FWXG-BNakeJ5Q.js → dagre-KLK3FWXG-DgUnjcLS.js} +1 -1
  129. package/ui-dist/assets/{diagram-E7M64L7V-UubVu-IN.js → diagram-E7M64L7V-DC3DIXbu.js} +1 -1
  130. package/ui-dist/assets/{diagram-IFDJBPK2-DRu8FSPT.js → diagram-IFDJBPK2-qv7Ij_rH.js} +1 -1
  131. package/ui-dist/assets/{diagram-P4PSJMXO-t2_nA6WC.js → diagram-P4PSJMXO-Dw2WzhiN.js} +1 -1
  132. package/ui-dist/assets/{erDiagram-INFDFZHY-ET8QDHtO.js → erDiagram-INFDFZHY-BV5s3J3D.js} +1 -1
  133. package/ui-dist/assets/{flowDiagram-PKNHOUZH-QXQeM_4d.js → flowDiagram-PKNHOUZH-kmPgde9o.js} +1 -1
  134. package/ui-dist/assets/{ganttDiagram-A5KZAMGK-B6yFn5Hd.js → ganttDiagram-A5KZAMGK-C2hkJ2Th.js} +1 -1
  135. package/ui-dist/assets/{gitGraphDiagram-K3NZZRJ6-DZAeG6MG.js → gitGraphDiagram-K3NZZRJ6-K5e1NgIt.js} +1 -1
  136. package/ui-dist/assets/{graph-BaTTyfU1.js → graph-Cna3FnWY.js} +1 -1
  137. package/ui-dist/assets/{index-C7kEQtA_.js → index-6luhu7W5.js} +1 -1
  138. package/ui-dist/assets/index-B1iX9NWL.js +1364 -0
  139. package/ui-dist/assets/{index-D-mb6cn2.js → index-BdLts2oZ.js} +1 -1
  140. package/ui-dist/assets/{index-ym7ZDmXE.js → index-CBKnHGYE.js} +1 -1
  141. package/ui-dist/assets/{index-D_ZvBEXt.js → index-CDtBE3Ga.js} +1 -1
  142. package/ui-dist/assets/{index-Cftoq4bF.js → index-CFGJHREd.js} +1 -1
  143. package/ui-dist/assets/{index-Bo1sSJ7x.js → index-CTpC-gpc.js} +1 -1
  144. package/ui-dist/assets/{index-h_UqLbty.js → index-CVAmlnRr.js} +1 -1
  145. package/ui-dist/assets/{index-BWEA-ibM.js → index-CWbJ-pcg.js} +1 -1
  146. package/ui-dist/assets/{index-BSfIb9qw.js → index-ChU0nSCq.js} +1 -1
  147. package/ui-dist/assets/{index-DQeEMbWr.js → index-CypGJ__o.js} +1 -1
  148. package/ui-dist/assets/{index-BXxi8m3U.js → index-D-Fe0jjS.js} +1 -1
  149. package/ui-dist/assets/{index-BGOxzMvq.js → index-D8aBm25c.js} +1 -1
  150. package/ui-dist/assets/index-D9uKiQrM.css +1 -0
  151. package/ui-dist/assets/{index-DBsb_N5Q.js → index-DD0ggVK_.js} +1 -1
  152. package/ui-dist/assets/{index-MYVA1f40.js → index-DJecNMv2.js} +1 -1
  153. package/ui-dist/assets/{index-CAcJz5d9.js → index-DKSfpiAJ.js} +1 -1
  154. package/ui-dist/assets/{index-ZHLqhZdz.js → index-Dgj2bRIV.js} +1 -1
  155. package/ui-dist/assets/{index-DvQZ5FJk.js → index-DsydT3bX.js} +1 -1
  156. package/ui-dist/assets/{index-qVKE_HM2.js → index-U7gGuiCe.js} +1 -1
  157. package/ui-dist/assets/{index-CRtxHtSg.js → index-aL5hFrtS.js} +1 -1
  158. package/ui-dist/assets/{index-DPbDGs74.js → index-o_iEnzyw.js} +1 -1
  159. package/ui-dist/assets/{index-eessTUZm.js → index-qUFX4kXb.js} +1 -1
  160. package/ui-dist/assets/{index-C2T1FM7J.js → index-sV3mB3fg.js} +1 -1
  161. package/ui-dist/assets/{infoDiagram-LFFYTUFH-BOnKcq7S.js → infoDiagram-LFFYTUFH-BiQ6_Z3F.js} +1 -1
  162. package/ui-dist/assets/{ishikawaDiagram-PHBUUO56-etV-MroN.js → ishikawaDiagram-PHBUUO56-C_WqRu1B.js} +1 -1
  163. package/ui-dist/assets/{journeyDiagram-4ABVD52K-mvjGNrBx.js → journeyDiagram-4ABVD52K-BIpPercT.js} +1 -1
  164. package/ui-dist/assets/{kanban-definition-K7BYSVSG-CW-Dw_bU.js → kanban-definition-K7BYSVSG-DaFD_JyB.js} +1 -1
  165. package/ui-dist/assets/{layout-DUcWZ3H3.js → layout-Dj_oike0.js} +1 -1
  166. package/ui-dist/assets/{linear-B0r1V0oG.js → linear-tnVt1ugU.js} +1 -1
  167. package/ui-dist/assets/{mermaid.core-Bo_YuNee.js → mermaid.core-DflBh7Hv.js} +4 -4
  168. package/ui-dist/assets/{mindmap-definition-YRQLILUH-BmEjkBnz.js → mindmap-definition-YRQLILUH-C3H-gFSh.js} +1 -1
  169. package/ui-dist/assets/{pieDiagram-SKSYHLDU-BRER3VVx.js → pieDiagram-SKSYHLDU-DH53odzN.js} +1 -1
  170. package/ui-dist/assets/{quadrantDiagram-337W2JSQ-oYnSbQKi.js → quadrantDiagram-337W2JSQ-DCn4Dric.js} +1 -1
  171. package/ui-dist/assets/{requirementDiagram-Z7DCOOCP-BRLFZsts.js → requirementDiagram-Z7DCOOCP-KypOfBcY.js} +1 -1
  172. package/ui-dist/assets/{sankeyDiagram-WA2Y5GQK-YOKeMxIU.js → sankeyDiagram-WA2Y5GQK-DEmi0ryr.js} +1 -1
  173. package/ui-dist/assets/{sequenceDiagram-2WXFIKYE-Bj-YKQMs.js → sequenceDiagram-2WXFIKYE-sW6ojsVG.js} +1 -1
  174. package/ui-dist/assets/{stateDiagram-RAJIS63D-BYf-CeWe.js → stateDiagram-RAJIS63D-CmrX9BEC.js} +1 -1
  175. package/ui-dist/assets/stateDiagram-v2-FVOUBMTO-DJuCnZbk.js +1 -0
  176. package/ui-dist/assets/{timeline-definition-YZTLITO2-CG6r3GP4.js → timeline-definition-YZTLITO2-CzJ5Y8YU.js} +1 -1
  177. package/ui-dist/assets/{treemap-KZPCXAKY-DCX4FLGH.js → treemap-KZPCXAKY-BwUNliTA.js} +1 -1
  178. package/ui-dist/assets/{vennDiagram-LZ73GAT5-CpbbhHdY.js → vennDiagram-LZ73GAT5-5A50aIn1.js} +1 -1
  179. package/ui-dist/assets/{xychartDiagram-JWTSCODW-Biq5EV_L.js → xychartDiagram-JWTSCODW-D9qTFskE.js} +1 -1
  180. package/ui-dist/index.html +2 -2
  181. package/ui-dist/assets/channel-BExT12GI.js +0 -1
  182. package/ui-dist/assets/classDiagram-VBA2DB6C-B3FSZmLP.js +0 -1
  183. package/ui-dist/assets/classDiagram-v2-RAHNMMFH-B3FSZmLP.js +0 -1
  184. package/ui-dist/assets/clone-DxXNeJhJ.js +0 -1
  185. package/ui-dist/assets/index-2Hx2TlnA.css +0 -1
  186. package/ui-dist/assets/index-wzjO7KZw.js +0 -1343
  187. package/ui-dist/assets/stateDiagram-v2-FVOUBMTO-g0byx18e.js +0 -1
@@ -0,0 +1,4044 @@
1
+ import { and, asc, desc, eq, gt, gte, inArray, lte, sql } from "drizzle-orm";
2
+ import { agents, agentRuntimeState, agentTaskSessions, agentWakeupRequests, activityLog, heartbeatRunEvents, heartbeatRuns, issueComments, issues, organizations, projects, } from "@rudderhq/db";
3
+ import { conflict, notFound } from "../../errors.js";
4
+ import { createExecutionScores, observeExecutionEvent, updateExecutionObservation, updateExecutionTraceIO, updateExecutionTraceName, updateExecutionTraceSession, withExecutionObservation, } from "../../langfuse.js";
5
+ import { emitExecutionTranscriptTree } from "../../langfuse-transcript.js";
6
+ import { logger } from "../../middleware/logger.js";
7
+ import { publishLiveEvent } from "../live-events.js";
8
+ import { getRunLogStore } from "../run-log-store.js";
9
+ import { findServerAdapter, getServerAdapter, runningProcesses } from "../../agent-runtimes/index.js";
10
+ import { createLocalAgentJwt } from "../../agent-auth-jwt.js";
11
+ import { parseObject, asNumber } from "../../agent-runtimes/utils.js";
12
+ import { costService } from "../costs.js";
13
+ import { budgetService } from "../budgets.js";
14
+ import { agentRunContextService, } from "../agent-run-context.js";
15
+ import { resolveDefaultAgentWorkspaceDir, } from "../../home-paths.js";
16
+ import { summarizeHeartbeatRunResultJson } from "../heartbeat-run-summary.js";
17
+ import { buildWorkspaceReadyComment, cleanupExecutionWorkspaceArtifacts, ensureRuntimeServicesForRun, persistAdapterManagedRuntimeServices, realizeExecutionWorkspace, releaseRuntimeServicesForRun, } from "../workspace-runtime.js";
18
+ import { issueService } from "../issues.js";
19
+ import { executionWorkspaceService } from "../execution-workspaces.js";
20
+ import { buildObservedRunLangfuseScores } from "../run-intelligence.js";
21
+ import { workspaceOperationService } from "../workspace-operations.js";
22
+ import { buildExecutionWorkspaceAdapterConfig, issueExecutionWorkspaceModeForPersistedWorkspace, parseIssueExecutionWorkspaceSettings, parseProjectExecutionWorkspacePolicy, resolveExecutionWorkspaceMode, } from "../execution-workspace-policy.js";
23
+ import { instanceSettingsService } from "../instance-settings.js";
24
+ import { logActivity } from "../activity-log.js";
25
+ import { redactCurrentUserText, redactCurrentUserValue } from "../../log-redaction.js";
26
+ import { hasSessionCompactionThresholds, } from "@rudderhq/agent-runtime-utils";
27
+ import { buildCreateAgentBenchmarkTags, coerceCreateAgentBenchmarkMetadata, extractCreateAgentBenchmarkMetadata, } from "@rudderhq/run-intelligence-core";
28
+ import { executeAdapterWithModelFallbacks } from "./model-fallback.js";
29
+ import { normalizeAgentNameKey, readNonEmptyString, truncateDisplayId, } from "./common.js";
30
+ import { buildDateKeysBetween, buildRecentDateKeys, normalizeLoadedSkill, } from "./analytics.js";
31
+ import { buildHeartbeatAdapterInvokePayload, buildHeartbeatObservationName, buildHeartbeatRuntimeTraceMetadata, buildIssueRunTraceName, resolveHeartbeatObservabilitySurface, } from "./run-state.js";
32
+ import { buildExplicitResumeSessionOverride, defaultSessionCodec, formatRuntimeWorkspaceWarningLog, normalizeSessionParams, parseSessionCompactionPolicy, resolveRuntimeSessionParamsForWorkspace, shouldResetTaskSessionForWake, } from "./session-policy.js";
33
+ import { appendExcerpt, appendTranscriptEntriesFromChunk, } from "./execution.js";
34
+ import { isProcessAlive, isTrackedLocalChildProcessAdapter, terminateOrphanedProcess, } from "./recovery.js";
35
+ import { parseHeartbeatPolicy } from "./wakeup-queue.js";
36
+ export { prioritizeProjectWorkspaceCandidatesForRun } from "../agent-run-context.js";
37
+ export { buildDateKeysBetween, buildRecentDateKeys, normalizeLoadedSkill, } from "./analytics.js";
38
+ export { normalizeAgentNameKey, readNonEmptyString, truncateDisplayId, } from "./common.js";
39
+ export { buildHeartbeatAdapterInvokePayload, buildHeartbeatRuntimeTraceMetadata, buildIssueRunTraceName, resolveHeartbeatObservabilitySurface, } from "./run-state.js";
40
+ export { buildExplicitResumeSessionOverride, formatRuntimeWorkspaceWarningLog, parseSessionCompactionPolicy, resolveRuntimeSessionParamsForWorkspace, shouldResetTaskSessionForWake, } from "./session-policy.js";
41
+ export { appendExcerpt, appendTranscriptEntriesFromChunk, } from "./execution.js";
42
+ export { isProcessAlive, isTrackedLocalChildProcessAdapter, terminateOrphanedProcess, } from "./recovery.js";
43
+ export { normalizeMaxConcurrentRuns, parseHeartbeatPolicy } from "./wakeup-queue.js";
44
+ const MAX_LIVE_LOG_CHUNK_BYTES = 8 * 1024;
45
+ const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext";
46
+ const DETACHED_PROCESS_ERROR_CODE = "process_detached";
47
+ const startLocksByAgent = new Map();
48
+ const MAX_RECOVERY_CHAIN_DEPTH = 8;
49
+ const ISSUE_PASSIVE_FOLLOWUP_REASON = "issue_passive_followup";
50
+ const ISSUE_PASSIVE_FOLLOWUP_WAKE_SOURCE = "passive_issue_followup";
51
+ const ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON = "missing_closure";
52
+ const ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS = 2;
53
+ const ISSUE_PASSIVE_FOLLOWUP_COOLDOWN_MS_BY_ATTEMPT = new Map([
54
+ [1, 2 * 60 * 1000],
55
+ [2, 5 * 60 * 1000],
56
+ ]);
57
+ const ISSUE_PASSIVE_FOLLOWUP_TIMER_CONTINUITY_MAX_WINDOW_MS = 15 * 60 * 1000;
58
+ const heartbeatRunListColumns = {
59
+ id: heartbeatRuns.id,
60
+ orgId: heartbeatRuns.orgId,
61
+ agentId: heartbeatRuns.agentId,
62
+ invocationSource: heartbeatRuns.invocationSource,
63
+ triggerDetail: heartbeatRuns.triggerDetail,
64
+ status: heartbeatRuns.status,
65
+ startedAt: heartbeatRuns.startedAt,
66
+ finishedAt: heartbeatRuns.finishedAt,
67
+ error: heartbeatRuns.error,
68
+ wakeupRequestId: heartbeatRuns.wakeupRequestId,
69
+ exitCode: heartbeatRuns.exitCode,
70
+ signal: heartbeatRuns.signal,
71
+ usageJson: heartbeatRuns.usageJson,
72
+ resultJson: heartbeatRuns.resultJson,
73
+ sessionIdBefore: heartbeatRuns.sessionIdBefore,
74
+ sessionIdAfter: heartbeatRuns.sessionIdAfter,
75
+ logStore: heartbeatRuns.logStore,
76
+ logRef: heartbeatRuns.logRef,
77
+ logBytes: heartbeatRuns.logBytes,
78
+ logSha256: heartbeatRuns.logSha256,
79
+ logCompressed: heartbeatRuns.logCompressed,
80
+ stdoutExcerpt: sql `NULL`.as("stdoutExcerpt"),
81
+ stderrExcerpt: sql `NULL`.as("stderrExcerpt"),
82
+ errorCode: heartbeatRuns.errorCode,
83
+ externalRunId: heartbeatRuns.externalRunId,
84
+ processPid: heartbeatRuns.processPid,
85
+ processStartedAt: heartbeatRuns.processStartedAt,
86
+ retryOfRunId: heartbeatRuns.retryOfRunId,
87
+ processLossRetryCount: heartbeatRuns.processLossRetryCount,
88
+ contextSnapshot: heartbeatRuns.contextSnapshot,
89
+ createdAt: heartbeatRuns.createdAt,
90
+ updatedAt: heartbeatRuns.updatedAt,
91
+ };
92
+ async function withAgentStartLock(agentId, fn) {
93
+ const previous = startLocksByAgent.get(agentId) ?? Promise.resolve();
94
+ const run = previous.then(fn);
95
+ const marker = run.then(() => undefined, () => undefined);
96
+ startLocksByAgent.set(agentId, marker);
97
+ try {
98
+ return await run;
99
+ }
100
+ finally {
101
+ if (startLocksByAgent.get(agentId) === marker) {
102
+ startLocksByAgent.delete(agentId);
103
+ }
104
+ }
105
+ }
106
+ function normalizeLedgerBillingType(value) {
107
+ const raw = readNonEmptyString(value);
108
+ switch (raw) {
109
+ case "api":
110
+ case "metered_api":
111
+ return "metered_api";
112
+ case "subscription":
113
+ case "subscription_included":
114
+ return "subscription_included";
115
+ case "subscription_overage":
116
+ return "subscription_overage";
117
+ case "credits":
118
+ return "credits";
119
+ case "fixed":
120
+ return "fixed";
121
+ default:
122
+ return "unknown";
123
+ }
124
+ }
125
+ function resolveLedgerBiller(result) {
126
+ return readNonEmptyString(result.biller) ?? readNonEmptyString(result.provider) ?? "unknown";
127
+ }
128
+ function normalizeBilledCostCents(costUsd, billingType) {
129
+ if (billingType === "subscription_included")
130
+ return 0;
131
+ if (typeof costUsd !== "number" || !Number.isFinite(costUsd))
132
+ return 0;
133
+ return Math.max(0, Math.round(costUsd * 100));
134
+ }
135
+ async function resolveLedgerScopeForRun(db, orgId, run) {
136
+ const context = parseObject(run.contextSnapshot);
137
+ const contextIssueId = readNonEmptyString(context.issueId);
138
+ const contextProjectId = readNonEmptyString(context.projectId);
139
+ if (!contextIssueId) {
140
+ return {
141
+ issueId: null,
142
+ projectId: contextProjectId,
143
+ };
144
+ }
145
+ const issue = await db
146
+ .select({
147
+ id: issues.id,
148
+ projectId: issues.projectId,
149
+ })
150
+ .from(issues)
151
+ .where(and(eq(issues.id, contextIssueId), eq(issues.orgId, orgId)))
152
+ .then((rows) => rows[0] ?? null);
153
+ return {
154
+ issueId: issue?.id ?? null,
155
+ projectId: issue?.projectId ?? contextProjectId,
156
+ };
157
+ }
158
+ function normalizeUsageTotals(usage) {
159
+ if (!usage)
160
+ return null;
161
+ return {
162
+ inputTokens: Math.max(0, Math.floor(asNumber(usage.inputTokens, 0))),
163
+ cachedInputTokens: Math.max(0, Math.floor(asNumber(usage.cachedInputTokens, 0))),
164
+ outputTokens: Math.max(0, Math.floor(asNumber(usage.outputTokens, 0))),
165
+ };
166
+ }
167
+ function readRawUsageTotals(usageJson) {
168
+ const parsed = parseObject(usageJson);
169
+ if (Object.keys(parsed).length === 0)
170
+ return null;
171
+ const inputTokens = Math.max(0, Math.floor(asNumber(parsed.rawInputTokens, asNumber(parsed.inputTokens, 0))));
172
+ const cachedInputTokens = Math.max(0, Math.floor(asNumber(parsed.rawCachedInputTokens, asNumber(parsed.cachedInputTokens, 0))));
173
+ const outputTokens = Math.max(0, Math.floor(asNumber(parsed.rawOutputTokens, asNumber(parsed.outputTokens, 0))));
174
+ if (inputTokens <= 0 && cachedInputTokens <= 0 && outputTokens <= 0) {
175
+ return null;
176
+ }
177
+ return {
178
+ inputTokens,
179
+ cachedInputTokens,
180
+ outputTokens,
181
+ };
182
+ }
183
+ function deriveNormalizedUsageDelta(current, previous) {
184
+ if (!current)
185
+ return null;
186
+ if (!previous)
187
+ return { ...current };
188
+ const inputTokens = current.inputTokens >= previous.inputTokens
189
+ ? current.inputTokens - previous.inputTokens
190
+ : current.inputTokens;
191
+ const cachedInputTokens = current.cachedInputTokens >= previous.cachedInputTokens
192
+ ? current.cachedInputTokens - previous.cachedInputTokens
193
+ : current.cachedInputTokens;
194
+ const outputTokens = current.outputTokens >= previous.outputTokens
195
+ ? current.outputTokens - previous.outputTokens
196
+ : current.outputTokens;
197
+ return {
198
+ inputTokens: Math.max(0, inputTokens),
199
+ cachedInputTokens: Math.max(0, cachedInputTokens),
200
+ outputTokens: Math.max(0, outputTokens),
201
+ };
202
+ }
203
+ function formatCount(value) {
204
+ if (typeof value !== "number" || !Number.isFinite(value))
205
+ return "0";
206
+ return value.toLocaleString("en-US");
207
+ }
208
+ function parseIssueAssigneeAgentRuntimeOverrides(raw) {
209
+ const parsed = parseObject(raw);
210
+ const parsedAdapterConfig = parseObject(parsed.agentRuntimeConfig);
211
+ const agentRuntimeConfig = Object.keys(parsedAdapterConfig).length > 0 ? parsedAdapterConfig : null;
212
+ const useProjectWorkspace = typeof parsed.useProjectWorkspace === "boolean"
213
+ ? parsed.useProjectWorkspace
214
+ : null;
215
+ if (!agentRuntimeConfig && useProjectWorkspace === null)
216
+ return null;
217
+ return {
218
+ agentRuntimeConfig,
219
+ useProjectWorkspace,
220
+ };
221
+ }
222
+ function deriveTaskKey(contextSnapshot, payload) {
223
+ return (readNonEmptyString(contextSnapshot?.taskKey) ??
224
+ readNonEmptyString(contextSnapshot?.taskId) ??
225
+ readNonEmptyString(contextSnapshot?.issueId) ??
226
+ readNonEmptyString(payload?.taskKey) ??
227
+ readNonEmptyString(payload?.taskId) ??
228
+ readNonEmptyString(payload?.issueId) ??
229
+ null);
230
+ }
231
+ function describeSessionResetReason(contextSnapshot) {
232
+ if (contextSnapshot?.forceFreshSession === true)
233
+ return "forceFreshSession was requested";
234
+ const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason);
235
+ if (wakeReason === "issue_assigned")
236
+ return "wake reason is issue_assigned";
237
+ return null;
238
+ }
239
+ function deriveCommentId(contextSnapshot, payload) {
240
+ return (readNonEmptyString(contextSnapshot?.wakeCommentId) ??
241
+ readNonEmptyString(contextSnapshot?.commentId) ??
242
+ readNonEmptyString(payload?.commentId) ??
243
+ null);
244
+ }
245
+ function enrichWakeContextSnapshot(input) {
246
+ const { contextSnapshot, reason, source, triggerDetail, payload } = input;
247
+ const issueIdFromPayload = readNonEmptyString(payload?.["issueId"]);
248
+ const commentIdFromPayload = readNonEmptyString(payload?.["commentId"]);
249
+ const taskKey = deriveTaskKey(contextSnapshot, payload);
250
+ const wakeCommentId = deriveCommentId(contextSnapshot, payload);
251
+ if (!readNonEmptyString(contextSnapshot["wakeReason"]) && reason) {
252
+ contextSnapshot.wakeReason = reason;
253
+ }
254
+ if (!readNonEmptyString(contextSnapshot["issueId"]) && issueIdFromPayload) {
255
+ contextSnapshot.issueId = issueIdFromPayload;
256
+ }
257
+ if (!readNonEmptyString(contextSnapshot["taskId"]) && issueIdFromPayload) {
258
+ contextSnapshot.taskId = issueIdFromPayload;
259
+ }
260
+ if (!readNonEmptyString(contextSnapshot["taskKey"]) && taskKey) {
261
+ contextSnapshot.taskKey = taskKey;
262
+ }
263
+ if (!readNonEmptyString(contextSnapshot["commentId"]) && commentIdFromPayload) {
264
+ contextSnapshot.commentId = commentIdFromPayload;
265
+ }
266
+ if (!readNonEmptyString(contextSnapshot["wakeCommentId"]) && wakeCommentId) {
267
+ contextSnapshot.wakeCommentId = wakeCommentId;
268
+ }
269
+ if (!readNonEmptyString(contextSnapshot["wakeSource"]) && source) {
270
+ contextSnapshot.wakeSource = source;
271
+ }
272
+ if (!readNonEmptyString(contextSnapshot["wakeTriggerDetail"]) && triggerDetail) {
273
+ contextSnapshot.wakeTriggerDetail = triggerDetail;
274
+ }
275
+ return {
276
+ contextSnapshot,
277
+ issueIdFromPayload,
278
+ commentIdFromPayload,
279
+ taskKey,
280
+ wakeCommentId,
281
+ };
282
+ }
283
+ function mergeCoalescedContextSnapshot(existingRaw, incoming) {
284
+ const existing = parseObject(existingRaw);
285
+ const merged = {
286
+ ...existing,
287
+ ...incoming,
288
+ };
289
+ const commentId = deriveCommentId(incoming, null);
290
+ if (commentId) {
291
+ merged.commentId = commentId;
292
+ merged.wakeCommentId = commentId;
293
+ }
294
+ return merged;
295
+ }
296
+ function buildDeferredWakePayload(payload, contextSnapshot, issueId) {
297
+ const deferredPayload = { ...(payload ?? {}) };
298
+ if (issueId && !readNonEmptyString(deferredPayload.issueId)) {
299
+ deferredPayload.issueId = issueId;
300
+ }
301
+ deferredPayload[DEFERRED_WAKE_CONTEXT_KEY] = contextSnapshot;
302
+ return deferredPayload;
303
+ }
304
+ function readDeferredWakeContext(payloadRaw) {
305
+ const payload = parseObject(payloadRaw);
306
+ return parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]);
307
+ }
308
+ function readDeferredWakePayload(payloadRaw) {
309
+ const payload = parseObject(payloadRaw);
310
+ delete payload[DEFERRED_WAKE_CONTEXT_KEY];
311
+ return payload;
312
+ }
313
+ function deriveDeferredWakeTaskKey(payloadRaw) {
314
+ const payload = readDeferredWakePayload(payloadRaw);
315
+ const contextSnapshot = readDeferredWakeContext(payloadRaw);
316
+ return deriveTaskKey(contextSnapshot, payload);
317
+ }
318
+ async function hydrateWakeContextSnapshot(db, orgId, contextSnapshot) {
319
+ const issueId = readNonEmptyString(contextSnapshot.issueId);
320
+ const commentId = deriveCommentId(contextSnapshot, null);
321
+ const issueContext = parseObject(contextSnapshot.issue);
322
+ const commentContext = parseObject(contextSnapshot.comment);
323
+ const needsIssueContext = !!issueId &&
324
+ (!readNonEmptyString(issueContext.id) ||
325
+ !readNonEmptyString(issueContext.title) ||
326
+ !readNonEmptyString(issueContext.status) ||
327
+ !("priority" in issueContext) ||
328
+ !("description" in issueContext));
329
+ const needsProjectId = !!issueId && !readNonEmptyString(contextSnapshot.projectId);
330
+ const needsCommentContext = !!commentId &&
331
+ (!readNonEmptyString(commentContext.id) ||
332
+ !readNonEmptyString(commentContext.body));
333
+ if (!needsIssueContext && !needsProjectId && !needsCommentContext)
334
+ return;
335
+ if (issueId && (needsIssueContext || needsProjectId)) {
336
+ const issueRow = await db
337
+ .select({
338
+ id: issues.id,
339
+ title: issues.title,
340
+ description: issues.description,
341
+ status: issues.status,
342
+ priority: issues.priority,
343
+ projectId: issues.projectId,
344
+ })
345
+ .from(issues)
346
+ .where(and(eq(issues.id, issueId), eq(issues.orgId, orgId)))
347
+ .then((rows) => rows[0] ?? null);
348
+ if (issueRow) {
349
+ contextSnapshot.issue = {
350
+ ...issueContext,
351
+ id: readNonEmptyString(issueContext.id) ?? issueRow.id,
352
+ title: readNonEmptyString(issueContext.title) ?? issueRow.title,
353
+ description: "description" in issueContext ? issueContext.description : issueRow.description,
354
+ status: readNonEmptyString(issueContext.status) ?? issueRow.status,
355
+ priority: "priority" in issueContext ? issueContext.priority : issueRow.priority,
356
+ };
357
+ if (!readNonEmptyString(contextSnapshot.projectId) && issueRow.projectId) {
358
+ contextSnapshot.projectId = issueRow.projectId;
359
+ }
360
+ }
361
+ }
362
+ if (commentId && needsCommentContext) {
363
+ const commentConditions = [eq(issueComments.id, commentId), eq(issueComments.orgId, orgId)];
364
+ if (issueId) {
365
+ commentConditions.push(eq(issueComments.issueId, issueId));
366
+ }
367
+ const commentRow = await db
368
+ .select({
369
+ id: issueComments.id,
370
+ body: issueComments.body,
371
+ authorAgentId: issueComments.authorAgentId,
372
+ authorUserId: issueComments.authorUserId,
373
+ })
374
+ .from(issueComments)
375
+ .where(and(...commentConditions))
376
+ .then((rows) => rows[0] ?? null);
377
+ if (commentRow) {
378
+ contextSnapshot.comment = {
379
+ ...commentContext,
380
+ id: readNonEmptyString(commentContext.id) ?? commentRow.id,
381
+ body: readNonEmptyString(commentContext.body) ?? commentRow.body,
382
+ authorAgentId: "authorAgentId" in commentContext ? commentContext.authorAgentId : commentRow.authorAgentId,
383
+ authorUserId: "authorUserId" in commentContext ? commentContext.authorUserId : commentRow.authorUserId,
384
+ };
385
+ }
386
+ }
387
+ }
388
+ function firstNonEmptyLine(value) {
389
+ if (typeof value !== "string")
390
+ return null;
391
+ const line = value
392
+ .split("\n")
393
+ .map((chunk) => chunk.trim())
394
+ .find(Boolean);
395
+ return line ?? null;
396
+ }
397
+ function deriveRecoveryFailureKind(run) {
398
+ return (readNonEmptyString(run.errorCode) ??
399
+ (run.status === "timed_out" ? "timed_out" : null) ??
400
+ run.status);
401
+ }
402
+ function deriveRecoveryFailureSummary(run) {
403
+ return (firstNonEmptyLine(run.error) ??
404
+ firstNonEmptyLine(run.stderrExcerpt) ??
405
+ firstNonEmptyLine(run.stdoutExcerpt) ??
406
+ (run.status === "timed_out" ? "The run timed out before it completed." : null) ??
407
+ "The previous run failed before it completed.");
408
+ }
409
+ function mergeMissingRecoveryContextFields(target, source) {
410
+ const keysToBackfill = [
411
+ "issueId",
412
+ "taskId",
413
+ "taskKey",
414
+ "projectId",
415
+ "projectWorkspaceId",
416
+ "commentId",
417
+ "wakeCommentId",
418
+ "issue",
419
+ "comment",
420
+ "source",
421
+ "wakeSource",
422
+ "wakeTriggerDetail",
423
+ ];
424
+ for (const key of keysToBackfill) {
425
+ if (!(key in target) || target[key] === null || target[key] === undefined || target[key] === "") {
426
+ const value = source[key];
427
+ if (value !== null && value !== undefined && value !== "") {
428
+ target[key] = value;
429
+ }
430
+ }
431
+ }
432
+ }
433
+ async function hydrateRecoveryBaseContextSnapshot(run, getRunById) {
434
+ const mergedContext = { ...parseObject(run.contextSnapshot) };
435
+ let ancestorRunId = readNonEmptyString(run.retryOfRunId);
436
+ let depth = 0;
437
+ while (ancestorRunId && depth < MAX_RECOVERY_CHAIN_DEPTH) {
438
+ const ancestorRun = await getRunById(ancestorRunId);
439
+ if (!ancestorRun)
440
+ break;
441
+ mergeMissingRecoveryContextFields(mergedContext, parseObject(ancestorRun.contextSnapshot));
442
+ ancestorRunId = readNonEmptyString(ancestorRun.retryOfRunId);
443
+ depth += 1;
444
+ }
445
+ return mergedContext;
446
+ }
447
+ function buildRecoveryContextSnapshot(input) {
448
+ const { baseContextSnapshot, run, recoveryTrigger, wakeReason, wakeSource, triggerDetail } = input;
449
+ const failureKind = deriveRecoveryFailureKind(run);
450
+ const failureSummary = deriveRecoveryFailureSummary(run);
451
+ const recovery = {
452
+ originalRunId: run.id,
453
+ failureKind,
454
+ failureSummary,
455
+ recoveryTrigger,
456
+ recoveryMode: "continue_preferred",
457
+ };
458
+ return {
459
+ ...baseContextSnapshot,
460
+ wakeReason,
461
+ wakeSource,
462
+ wakeTriggerDetail: triggerDetail,
463
+ retryOfRunId: run.id,
464
+ retryReason: failureKind,
465
+ recovery,
466
+ };
467
+ }
468
+ function normalizePassiveFollowupContext(raw) {
469
+ const parsed = parseObject(raw);
470
+ const originRunId = readNonEmptyString(parsed.originRunId);
471
+ if (!originRunId)
472
+ return null;
473
+ const attempt = Math.max(0, Math.floor(asNumber(parsed.attempt, 0)));
474
+ return {
475
+ originRunId,
476
+ previousRunId: readNonEmptyString(parsed.previousRunId),
477
+ attempt,
478
+ maxAttempts: Math.max(1, Math.floor(asNumber(parsed.maxAttempts, ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS))),
479
+ reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
480
+ queuedAt: readNonEmptyString(parsed.queuedAt),
481
+ };
482
+ }
483
+ function passiveFollowupCooldownMs(attempt) {
484
+ return ISSUE_PASSIVE_FOLLOWUP_COOLDOWN_MS_BY_ATTEMPT.get(attempt) ?? 5 * 60 * 1000;
485
+ }
486
+ function isAgentEligibleForTimerContinuation(agent) {
487
+ return (agent.status !== "paused" &&
488
+ agent.status !== "terminated" &&
489
+ agent.status !== "pending_approval");
490
+ }
491
+ function hasCredibleTimerContinuation(input) {
492
+ if (!input.policy.enabled || input.policy.intervalSec <= 0)
493
+ return false;
494
+ if (!isAgentEligibleForTimerContinuation(input.agent))
495
+ return false;
496
+ const intervalMs = input.policy.intervalSec * 1000;
497
+ const nearTermWindowMs = Math.min(intervalMs * 2, ISSUE_PASSIVE_FOLLOWUP_TIMER_CONTINUITY_MAX_WINDOW_MS);
498
+ const lastHeartbeatMs = input.agent.lastHeartbeatAt
499
+ ? new Date(input.agent.lastHeartbeatAt).getTime()
500
+ : new Date(input.agent.createdAt).getTime();
501
+ const runFinishedMs = input.run.finishedAt
502
+ ? new Date(input.run.finishedAt).getTime()
503
+ : input.now.getTime();
504
+ const baselineMs = Math.max(lastHeartbeatMs, runFinishedMs);
505
+ const nextTimerMs = baselineMs + intervalMs;
506
+ return Math.max(0, nextTimerMs - input.now.getTime()) <= nearTermWindowMs;
507
+ }
508
+ function buildPassiveFollowupContextSnapshot(input) {
509
+ const baseContext = { ...parseObject(input.run.contextSnapshot) };
510
+ delete baseContext.recovery;
511
+ delete baseContext.retryOfRunId;
512
+ delete baseContext.retryReason;
513
+ const taskKey = deriveTaskKey(baseContext, { issueId: input.issue.id }) ?? input.issue.id;
514
+ return {
515
+ ...baseContext,
516
+ issueId: input.issue.id,
517
+ taskId: input.issue.id,
518
+ taskKey,
519
+ projectId: readNonEmptyString(baseContext.projectId) ?? input.issue.projectId ?? undefined,
520
+ wakeReason: ISSUE_PASSIVE_FOLLOWUP_REASON,
521
+ wakeSource: ISSUE_PASSIVE_FOLLOWUP_WAKE_SOURCE,
522
+ wakeTriggerDetail: "system",
523
+ issue: {
524
+ id: input.issue.id,
525
+ title: input.issue.title,
526
+ description: input.issue.description,
527
+ status: input.issue.status,
528
+ priority: input.issue.priority,
529
+ },
530
+ passiveFollowup: {
531
+ originRunId: input.originRunId,
532
+ previousRunId: input.run.id,
533
+ attempt: input.attempt,
534
+ maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
535
+ reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
536
+ queuedAt: input.now.toISOString(),
537
+ },
538
+ };
539
+ }
540
+ function runTaskKey(run) {
541
+ return deriveTaskKey(run.contextSnapshot, null);
542
+ }
543
+ function isSameTaskScope(left, right) {
544
+ return (left ?? null) === (right ?? null);
545
+ }
546
+ function getAgentRuntimeSessionCodec(agentRuntimeType) {
547
+ const adapter = getServerAdapter(agentRuntimeType);
548
+ return adapter.sessionCodec ?? defaultSessionCodec;
549
+ }
550
+ function resolveNextSessionState(input) {
551
+ const { codec, adapterResult, previousParams, previousDisplayId, previousLegacySessionId } = input;
552
+ if (adapterResult.clearSession) {
553
+ return {
554
+ params: null,
555
+ displayId: null,
556
+ legacySessionId: null,
557
+ };
558
+ }
559
+ const explicitParams = adapterResult.sessionParams;
560
+ const hasExplicitParams = adapterResult.sessionParams !== undefined;
561
+ const hasExplicitSessionId = adapterResult.sessionId !== undefined;
562
+ const explicitSessionId = readNonEmptyString(adapterResult.sessionId);
563
+ const hasExplicitDisplay = adapterResult.sessionDisplayId !== undefined;
564
+ const explicitDisplayId = readNonEmptyString(adapterResult.sessionDisplayId);
565
+ const shouldUsePrevious = !hasExplicitParams && !hasExplicitSessionId && !hasExplicitDisplay;
566
+ const candidateParams = hasExplicitParams
567
+ ? explicitParams
568
+ : hasExplicitSessionId
569
+ ? (explicitSessionId ? { sessionId: explicitSessionId } : null)
570
+ : previousParams;
571
+ const serialized = normalizeSessionParams(codec.serialize(normalizeSessionParams(candidateParams) ?? null));
572
+ const deserialized = normalizeSessionParams(codec.deserialize(serialized));
573
+ const displayId = truncateDisplayId(explicitDisplayId ??
574
+ (codec.getDisplayId ? codec.getDisplayId(deserialized) : null) ??
575
+ readNonEmptyString(deserialized?.sessionId) ??
576
+ (shouldUsePrevious ? previousDisplayId : null) ??
577
+ explicitSessionId ??
578
+ (shouldUsePrevious ? previousLegacySessionId : null));
579
+ const legacySessionId = explicitSessionId ??
580
+ readNonEmptyString(deserialized?.sessionId) ??
581
+ displayId ??
582
+ (shouldUsePrevious ? previousLegacySessionId : null);
583
+ return {
584
+ params: serialized,
585
+ displayId,
586
+ legacySessionId,
587
+ };
588
+ }
589
+ export function heartbeatService(db) {
590
+ const instanceSettings = instanceSettingsService(db);
591
+ const getCurrentUserRedactionOptions = async () => ({
592
+ enabled: (await instanceSettings.getGeneral()).censorUsernameInLogs,
593
+ });
594
+ const runLogStore = getRunLogStore();
595
+ const runContextSvc = agentRunContextService(db);
596
+ const issuesSvc = issueService(db);
597
+ const executionWorkspacesSvc = executionWorkspaceService(db);
598
+ const workspaceOperationsSvc = workspaceOperationService(db);
599
+ const activeRunExecutions = new Set();
600
+ const budgetHooks = {
601
+ cancelWorkForScope: cancelBudgetScopeWork,
602
+ };
603
+ const budgets = budgetService(db, budgetHooks);
604
+ async function getAgent(agentId) {
605
+ return db
606
+ .select()
607
+ .from(agents)
608
+ .where(eq(agents.id, agentId))
609
+ .then((rows) => rows[0] ?? null);
610
+ }
611
+ async function getRun(runId) {
612
+ return db
613
+ .select()
614
+ .from(heartbeatRuns)
615
+ .where(eq(heartbeatRuns.id, runId))
616
+ .then((rows) => rows[0] ?? null);
617
+ }
618
+ async function getRuntimeState(agentId) {
619
+ return db
620
+ .select()
621
+ .from(agentRuntimeState)
622
+ .where(eq(agentRuntimeState.agentId, agentId))
623
+ .then((rows) => rows[0] ?? null);
624
+ }
625
+ async function getTaskSession(orgId, agentId, agentRuntimeType, taskKey) {
626
+ return db
627
+ .select()
628
+ .from(agentTaskSessions)
629
+ .where(and(eq(agentTaskSessions.orgId, orgId), eq(agentTaskSessions.agentId, agentId), eq(agentTaskSessions.agentRuntimeType, agentRuntimeType), eq(agentTaskSessions.taskKey, taskKey)))
630
+ .then((rows) => rows[0] ?? null);
631
+ }
632
+ async function getLatestRunForSession(agentId, sessionId, opts) {
633
+ const conditions = [
634
+ eq(heartbeatRuns.agentId, agentId),
635
+ eq(heartbeatRuns.sessionIdAfter, sessionId),
636
+ ];
637
+ if (opts?.excludeRunId) {
638
+ conditions.push(sql `${heartbeatRuns.id} <> ${opts.excludeRunId}`);
639
+ }
640
+ return db
641
+ .select()
642
+ .from(heartbeatRuns)
643
+ .where(and(...conditions))
644
+ .orderBy(desc(heartbeatRuns.createdAt))
645
+ .limit(1)
646
+ .then((rows) => rows[0] ?? null);
647
+ }
648
+ async function getOldestRunForSession(agentId, sessionId) {
649
+ return db
650
+ .select({
651
+ id: heartbeatRuns.id,
652
+ createdAt: heartbeatRuns.createdAt,
653
+ })
654
+ .from(heartbeatRuns)
655
+ .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.sessionIdAfter, sessionId)))
656
+ .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id))
657
+ .limit(1)
658
+ .then((rows) => rows[0] ?? null);
659
+ }
660
+ async function resolveNormalizedUsageForSession(input) {
661
+ const { agentId, runId, sessionId, rawUsage } = input;
662
+ if (!sessionId || !rawUsage) {
663
+ return {
664
+ normalizedUsage: rawUsage,
665
+ previousRawUsage: null,
666
+ derivedFromSessionTotals: false,
667
+ };
668
+ }
669
+ const previousRun = await getLatestRunForSession(agentId, sessionId, { excludeRunId: runId });
670
+ const previousRawUsage = readRawUsageTotals(previousRun?.usageJson);
671
+ return {
672
+ normalizedUsage: deriveNormalizedUsageDelta(rawUsage, previousRawUsage),
673
+ previousRawUsage,
674
+ derivedFromSessionTotals: previousRawUsage !== null,
675
+ };
676
+ }
677
+ async function evaluateSessionCompaction(input) {
678
+ const { agent, sessionId, issueId } = input;
679
+ if (!sessionId) {
680
+ return {
681
+ rotate: false,
682
+ reason: null,
683
+ handoffMarkdown: null,
684
+ previousRunId: null,
685
+ };
686
+ }
687
+ const policy = parseSessionCompactionPolicy(agent);
688
+ if (!policy.enabled || !hasSessionCompactionThresholds(policy)) {
689
+ return {
690
+ rotate: false,
691
+ reason: null,
692
+ handoffMarkdown: null,
693
+ previousRunId: null,
694
+ };
695
+ }
696
+ const fetchLimit = Math.max(policy.maxSessionRuns > 0 ? policy.maxSessionRuns + 1 : 0, 4);
697
+ const runs = await db
698
+ .select({
699
+ id: heartbeatRuns.id,
700
+ createdAt: heartbeatRuns.createdAt,
701
+ usageJson: heartbeatRuns.usageJson,
702
+ resultJson: heartbeatRuns.resultJson,
703
+ error: heartbeatRuns.error,
704
+ })
705
+ .from(heartbeatRuns)
706
+ .where(and(eq(heartbeatRuns.agentId, agent.id), eq(heartbeatRuns.sessionIdAfter, sessionId)))
707
+ .orderBy(desc(heartbeatRuns.createdAt))
708
+ .limit(fetchLimit);
709
+ if (runs.length === 0) {
710
+ return {
711
+ rotate: false,
712
+ reason: null,
713
+ handoffMarkdown: null,
714
+ previousRunId: null,
715
+ };
716
+ }
717
+ const latestRun = runs[0] ?? null;
718
+ const oldestRun = policy.maxSessionAgeHours > 0
719
+ ? await getOldestRunForSession(agent.id, sessionId)
720
+ : runs[runs.length - 1] ?? latestRun;
721
+ const latestRawUsage = readRawUsageTotals(latestRun?.usageJson);
722
+ const sessionAgeHours = latestRun && oldestRun
723
+ ? Math.max(0, (new Date(latestRun.createdAt).getTime() - new Date(oldestRun.createdAt).getTime()) / (1000 * 60 * 60))
724
+ : 0;
725
+ let reason = null;
726
+ if (policy.maxSessionRuns > 0 && runs.length > policy.maxSessionRuns) {
727
+ reason = `session exceeded ${policy.maxSessionRuns} runs`;
728
+ }
729
+ else if (policy.maxRawInputTokens > 0 &&
730
+ latestRawUsage &&
731
+ latestRawUsage.inputTokens >= policy.maxRawInputTokens) {
732
+ reason =
733
+ `session raw input reached ${formatCount(latestRawUsage.inputTokens)} tokens ` +
734
+ `(threshold ${formatCount(policy.maxRawInputTokens)})`;
735
+ }
736
+ else if (policy.maxSessionAgeHours > 0 && sessionAgeHours >= policy.maxSessionAgeHours) {
737
+ reason = `session age reached ${Math.floor(sessionAgeHours)} hours`;
738
+ }
739
+ if (!reason || !latestRun) {
740
+ return {
741
+ rotate: false,
742
+ reason: null,
743
+ handoffMarkdown: null,
744
+ previousRunId: latestRun?.id ?? null,
745
+ };
746
+ }
747
+ const latestSummary = summarizeHeartbeatRunResultJson(latestRun.resultJson);
748
+ const latestTextSummary = readNonEmptyString(latestSummary?.summary) ??
749
+ readNonEmptyString(latestSummary?.result) ??
750
+ readNonEmptyString(latestSummary?.message) ??
751
+ readNonEmptyString(latestRun.error);
752
+ const handoffMarkdown = [
753
+ "Rudder session handoff:",
754
+ `- Previous session: ${sessionId}`,
755
+ issueId ? `- Issue: ${issueId}` : "",
756
+ `- Rotation reason: ${reason}`,
757
+ latestTextSummary ? `- Last run summary: ${latestTextSummary}` : "",
758
+ "Continue from the current task state. Rebuild only the minimum context you need.",
759
+ ]
760
+ .filter(Boolean)
761
+ .join("\n");
762
+ return {
763
+ rotate: true,
764
+ reason,
765
+ handoffMarkdown,
766
+ previousRunId: latestRun.id,
767
+ };
768
+ }
769
+ async function resolveSessionBeforeForWakeup(agent, taskKey) {
770
+ if (taskKey) {
771
+ const codec = getAgentRuntimeSessionCodec(agent.agentRuntimeType);
772
+ const existingTaskSession = await getTaskSession(agent.orgId, agent.id, agent.agentRuntimeType, taskKey);
773
+ const parsedParams = normalizeSessionParams(codec.deserialize(existingTaskSession?.sessionParamsJson ?? null));
774
+ return truncateDisplayId(existingTaskSession?.sessionDisplayId ??
775
+ (codec.getDisplayId ? codec.getDisplayId(parsedParams) : null) ??
776
+ readNonEmptyString(parsedParams?.sessionId));
777
+ }
778
+ const runtimeForRun = await getRuntimeState(agent.id);
779
+ return runtimeForRun?.sessionId ?? null;
780
+ }
781
+ async function resolveExplicitResumeSessionOverride(agent, payload, taskKey) {
782
+ const resumeFromRunId = readNonEmptyString(payload?.resumeFromRunId);
783
+ if (!resumeFromRunId)
784
+ return null;
785
+ const resumeRun = await db
786
+ .select({
787
+ id: heartbeatRuns.id,
788
+ contextSnapshot: heartbeatRuns.contextSnapshot,
789
+ sessionIdBefore: heartbeatRuns.sessionIdBefore,
790
+ sessionIdAfter: heartbeatRuns.sessionIdAfter,
791
+ })
792
+ .from(heartbeatRuns)
793
+ .where(and(eq(heartbeatRuns.id, resumeFromRunId), eq(heartbeatRuns.orgId, agent.orgId), eq(heartbeatRuns.agentId, agent.id)))
794
+ .then((rows) => rows[0] ?? null);
795
+ if (!resumeRun)
796
+ return null;
797
+ const resumeContext = parseObject(resumeRun.contextSnapshot);
798
+ const resumeTaskKey = deriveTaskKey(resumeContext, null) ?? taskKey;
799
+ const resumeTaskSession = resumeTaskKey
800
+ ? await getTaskSession(agent.orgId, agent.id, agent.agentRuntimeType, resumeTaskKey)
801
+ : null;
802
+ const sessionCodec = getAgentRuntimeSessionCodec(agent.agentRuntimeType);
803
+ const sessionOverride = buildExplicitResumeSessionOverride({
804
+ resumeFromRunId,
805
+ resumeRunSessionIdBefore: resumeRun.sessionIdBefore,
806
+ resumeRunSessionIdAfter: resumeRun.sessionIdAfter,
807
+ taskSession: resumeTaskSession,
808
+ sessionCodec,
809
+ });
810
+ if (!sessionOverride)
811
+ return null;
812
+ return {
813
+ resumeFromRunId,
814
+ taskKey: resumeTaskKey,
815
+ issueId: readNonEmptyString(resumeContext.issueId),
816
+ taskId: readNonEmptyString(resumeContext.taskId) ?? readNonEmptyString(resumeContext.issueId),
817
+ sessionDisplayId: sessionOverride.sessionDisplayId,
818
+ sessionParams: sessionOverride.sessionParams,
819
+ };
820
+ }
821
+ async function upsertTaskSession(input) {
822
+ const existing = await getTaskSession(input.orgId, input.agentId, input.agentRuntimeType, input.taskKey);
823
+ if (existing) {
824
+ return db
825
+ .update(agentTaskSessions)
826
+ .set({
827
+ sessionParamsJson: input.sessionParamsJson,
828
+ sessionDisplayId: input.sessionDisplayId,
829
+ lastRunId: input.lastRunId,
830
+ lastError: input.lastError,
831
+ updatedAt: new Date(),
832
+ })
833
+ .where(eq(agentTaskSessions.id, existing.id))
834
+ .returning()
835
+ .then((rows) => rows[0] ?? null);
836
+ }
837
+ return db
838
+ .insert(agentTaskSessions)
839
+ .values({
840
+ orgId: input.orgId,
841
+ agentId: input.agentId,
842
+ agentRuntimeType: input.agentRuntimeType,
843
+ taskKey: input.taskKey,
844
+ sessionParamsJson: input.sessionParamsJson,
845
+ sessionDisplayId: input.sessionDisplayId,
846
+ lastRunId: input.lastRunId,
847
+ lastError: input.lastError,
848
+ })
849
+ .returning()
850
+ .then((rows) => rows[0] ?? null);
851
+ }
852
+ async function clearTaskSessions(orgId, agentId, opts) {
853
+ const conditions = [
854
+ eq(agentTaskSessions.orgId, orgId),
855
+ eq(agentTaskSessions.agentId, agentId),
856
+ ];
857
+ if (opts?.taskKey) {
858
+ conditions.push(eq(agentTaskSessions.taskKey, opts.taskKey));
859
+ }
860
+ if (opts?.agentRuntimeType) {
861
+ conditions.push(eq(agentTaskSessions.agentRuntimeType, opts.agentRuntimeType));
862
+ }
863
+ return db
864
+ .delete(agentTaskSessions)
865
+ .where(and(...conditions))
866
+ .returning()
867
+ .then((rows) => rows.length);
868
+ }
869
+ async function ensureRuntimeState(agent) {
870
+ const existing = await getRuntimeState(agent.id);
871
+ if (existing)
872
+ return existing;
873
+ const now = new Date();
874
+ return db
875
+ .insert(agentRuntimeState)
876
+ .values({
877
+ agentId: agent.id,
878
+ orgId: agent.orgId,
879
+ agentRuntimeType: agent.agentRuntimeType,
880
+ stateJson: {},
881
+ createdAt: now,
882
+ updatedAt: now,
883
+ })
884
+ .onConflictDoUpdate({
885
+ target: agentRuntimeState.agentId,
886
+ set: {
887
+ orgId: agent.orgId,
888
+ agentRuntimeType: agent.agentRuntimeType,
889
+ updatedAt: now,
890
+ },
891
+ })
892
+ .returning()
893
+ .then((rows) => rows[0]);
894
+ }
895
+ function buildHeartbeatObservabilityContext(run, overrides = {}) {
896
+ const contextSnapshot = parseObject(run.contextSnapshot);
897
+ const issueSnapshot = parseObject(contextSnapshot.issue);
898
+ const benchmarkMetadata = extractCreateAgentBenchmarkMetadata(readNonEmptyString(issueSnapshot.description))
899
+ ?? coerceCreateAgentBenchmarkMetadata(parseObject(contextSnapshot.benchmark));
900
+ const baseMetadata = {
901
+ wakeupRequestId: run.wakeupRequestId,
902
+ errorCode: run.errorCode,
903
+ retryOfRunId: run.retryOfRunId,
904
+ processLossRetryCount: run.processLossRetryCount,
905
+ externalRunId: run.externalRunId,
906
+ executionWorkspaceId: readNonEmptyString(contextSnapshot.executionWorkspaceId),
907
+ ...(benchmarkMetadata ?? {}),
908
+ };
909
+ const benchmarkTags = benchmarkMetadata ? buildCreateAgentBenchmarkTags(benchmarkMetadata) : [];
910
+ return {
911
+ surface: resolveHeartbeatObservabilitySurface(contextSnapshot),
912
+ rootExecutionId: run.id,
913
+ orgId: run.orgId,
914
+ agentId: run.agentId,
915
+ issueId: readNonEmptyString(contextSnapshot.issueId),
916
+ sessionKey: run.sessionIdAfter ??
917
+ run.sessionIdBefore ??
918
+ readNonEmptyString(contextSnapshot.sessionKey) ??
919
+ readNonEmptyString(contextSnapshot.taskKey),
920
+ runtime: readNonEmptyString(contextSnapshot.agentRuntimeType),
921
+ trigger: run.triggerDetail ?? run.invocationSource,
922
+ status: run.status,
923
+ metadata: {
924
+ ...baseMetadata,
925
+ ...(overrides.metadata ?? {}),
926
+ },
927
+ tags: [...benchmarkTags, ...(overrides.tags ?? [])],
928
+ ...overrides,
929
+ };
930
+ }
931
+ async function emitHeartbeatObservationEvent(run, input, overrides = {}) {
932
+ try {
933
+ await observeExecutionEvent(buildHeartbeatObservabilityContext(run, overrides), input);
934
+ }
935
+ catch (error) {
936
+ logger.warn({
937
+ runId: run.id,
938
+ eventName: input.name,
939
+ err: error instanceof Error ? error.message : String(error),
940
+ }, "Failed to emit Langfuse heartbeat event");
941
+ }
942
+ }
943
+ async function emitHeartbeatLiveEval(runId) {
944
+ try {
945
+ const { detail, scores } = await buildObservedRunLangfuseScores(db, runId);
946
+ await createExecutionScores(buildHeartbeatObservabilityContext(detail.run, {
947
+ runtime: detail.bundle.agentRuntimeType,
948
+ metadata: {
949
+ agentName: detail.agentName,
950
+ orgName: detail.orgName,
951
+ },
952
+ }), scores.map((score) => ({
953
+ rootExecutionId: detail.run.id,
954
+ name: score.name,
955
+ value: score.value,
956
+ comment: score.comment,
957
+ metadata: score.metadata,
958
+ })));
959
+ }
960
+ catch (error) {
961
+ logger.warn({
962
+ runId,
963
+ err: error instanceof Error ? error.message : String(error),
964
+ }, "Failed to emit Langfuse heartbeat scores");
965
+ }
966
+ }
967
+ async function setRunStatus(runId, status, patch) {
968
+ const updated = await db
969
+ .update(heartbeatRuns)
970
+ .set({ status, ...patch, updatedAt: new Date() })
971
+ .where(eq(heartbeatRuns.id, runId))
972
+ .returning()
973
+ .then((rows) => rows[0] ?? null);
974
+ if (updated) {
975
+ publishLiveEvent({
976
+ orgId: updated.orgId,
977
+ type: "heartbeat.run.status",
978
+ payload: {
979
+ runId: updated.id,
980
+ agentId: updated.agentId,
981
+ status: updated.status,
982
+ invocationSource: updated.invocationSource,
983
+ triggerDetail: updated.triggerDetail,
984
+ error: updated.error ?? null,
985
+ errorCode: updated.errorCode ?? null,
986
+ startedAt: updated.startedAt ? new Date(updated.startedAt).toISOString() : null,
987
+ finishedAt: updated.finishedAt ? new Date(updated.finishedAt).toISOString() : null,
988
+ },
989
+ });
990
+ await emitHeartbeatObservationEvent(updated, {
991
+ name: `heartbeat.status.${status}`,
992
+ asType: "event",
993
+ output: {
994
+ status: updated.status,
995
+ error: updated.error,
996
+ errorCode: updated.errorCode,
997
+ startedAt: updated.startedAt ? new Date(updated.startedAt).toISOString() : null,
998
+ finishedAt: updated.finishedAt ? new Date(updated.finishedAt).toISOString() : null,
999
+ },
1000
+ }, {
1001
+ status: updated.status,
1002
+ });
1003
+ }
1004
+ return updated;
1005
+ }
1006
+ async function setWakeupStatus(wakeupRequestId, status, patch) {
1007
+ if (!wakeupRequestId)
1008
+ return;
1009
+ await db
1010
+ .update(agentWakeupRequests)
1011
+ .set({ status, ...patch, updatedAt: new Date() })
1012
+ .where(eq(agentWakeupRequests.id, wakeupRequestId));
1013
+ }
1014
+ async function updateWakeupRequestRecord(tx, wakeupRequestId, patch) {
1015
+ return tx
1016
+ .update(agentWakeupRequests)
1017
+ .set({ ...patch, updatedAt: new Date() })
1018
+ .where(eq(agentWakeupRequests.id, wakeupRequestId))
1019
+ .returning()
1020
+ .then((rows) => rows[0] ?? null);
1021
+ }
1022
+ async function insertWakeupRequestRecord(tx, values) {
1023
+ return tx
1024
+ .insert(agentWakeupRequests)
1025
+ .values(values)
1026
+ .returning()
1027
+ .then((rows) => rows[0] ?? null);
1028
+ }
1029
+ async function appendRunEvent(run, seq, event) {
1030
+ const currentUserRedactionOptions = await getCurrentUserRedactionOptions();
1031
+ const sanitizedMessage = event.message
1032
+ ? redactCurrentUserText(event.message, currentUserRedactionOptions)
1033
+ : event.message;
1034
+ const sanitizedPayload = event.payload
1035
+ ? redactCurrentUserValue(event.payload, currentUserRedactionOptions)
1036
+ : event.payload;
1037
+ await db.insert(heartbeatRunEvents).values({
1038
+ orgId: run.orgId,
1039
+ runId: run.id,
1040
+ agentId: run.agentId,
1041
+ seq,
1042
+ eventType: event.eventType,
1043
+ stream: event.stream,
1044
+ level: event.level,
1045
+ color: event.color,
1046
+ message: sanitizedMessage,
1047
+ payload: sanitizedPayload,
1048
+ });
1049
+ publishLiveEvent({
1050
+ orgId: run.orgId,
1051
+ type: "heartbeat.run.event",
1052
+ payload: {
1053
+ runId: run.id,
1054
+ agentId: run.agentId,
1055
+ seq,
1056
+ eventType: event.eventType,
1057
+ stream: event.stream ?? null,
1058
+ level: event.level ?? null,
1059
+ color: event.color ?? null,
1060
+ message: sanitizedMessage ?? null,
1061
+ payload: sanitizedPayload ?? null,
1062
+ },
1063
+ });
1064
+ await emitHeartbeatObservationEvent(run, {
1065
+ name: `heartbeat.event.${event.eventType}`,
1066
+ asType: "event",
1067
+ level: event.level === "error" ? "ERROR" : event.level === "warn" ? "WARNING" : "DEFAULT",
1068
+ output: {
1069
+ seq,
1070
+ eventType: event.eventType,
1071
+ stream: event.stream ?? null,
1072
+ level: event.level ?? null,
1073
+ color: event.color ?? null,
1074
+ message: sanitizedMessage ?? null,
1075
+ },
1076
+ metadata: sanitizedPayload ?? undefined,
1077
+ }, {
1078
+ status: run.status,
1079
+ });
1080
+ }
1081
+ async function nextRunEventSeq(runId) {
1082
+ const [row] = await db
1083
+ .select({ maxSeq: sql `max(${heartbeatRunEvents.seq})` })
1084
+ .from(heartbeatRunEvents)
1085
+ .where(eq(heartbeatRunEvents.runId, runId));
1086
+ return Number(row?.maxSeq ?? 0) + 1;
1087
+ }
1088
+ async function persistRunProcessMetadata(runId, meta) {
1089
+ const startedAt = new Date(meta.startedAt);
1090
+ const updated = await db
1091
+ .update(heartbeatRuns)
1092
+ .set({
1093
+ processPid: meta.pid,
1094
+ processStartedAt: Number.isNaN(startedAt.getTime()) ? new Date() : startedAt,
1095
+ updatedAt: new Date(),
1096
+ })
1097
+ .where(eq(heartbeatRuns.id, runId))
1098
+ .returning()
1099
+ .then((rows) => rows[0] ?? null);
1100
+ if (updated) {
1101
+ await emitHeartbeatObservationEvent(updated, {
1102
+ name: "heartbeat.process.spawn",
1103
+ asType: "event",
1104
+ output: {
1105
+ pid: meta.pid,
1106
+ startedAt: meta.startedAt,
1107
+ },
1108
+ });
1109
+ }
1110
+ return updated;
1111
+ }
1112
+ async function clearDetachedRunWarning(runId) {
1113
+ const updated = await db
1114
+ .update(heartbeatRuns)
1115
+ .set({
1116
+ error: null,
1117
+ errorCode: null,
1118
+ updatedAt: new Date(),
1119
+ })
1120
+ .where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.errorCode, DETACHED_PROCESS_ERROR_CODE)))
1121
+ .returning()
1122
+ .then((rows) => rows[0] ?? null);
1123
+ if (!updated)
1124
+ return null;
1125
+ await appendRunEvent(updated, await nextRunEventSeq(updated.id), {
1126
+ eventType: "lifecycle",
1127
+ stream: "system",
1128
+ level: "info",
1129
+ message: "Detached child process reported activity; cleared detached warning",
1130
+ });
1131
+ return updated;
1132
+ }
1133
+ async function enqueueRecoveryRun(run, agent, opts) {
1134
+ /**
1135
+ * Recovery runs intentionally clone the prior run's task context and then
1136
+ * layer explicit recovery metadata on top. This keeps retries visible and
1137
+ * auditable while preserving "continue preferred" semantics for issue work.
1138
+ *
1139
+ * Reasoning:
1140
+ * - Manual retry and automatic process-loss retry must assemble the same
1141
+ * recovery contract so prompts/runtime behavior stay aligned.
1142
+ * - We backfill missing context from the retry chain to recover from older
1143
+ * lossy retry runs without mutating the historical source run rows.
1144
+ *
1145
+ * Traceability:
1146
+ * - doc/developing/RUN-RECOVERY.md
1147
+ * - doc/DEVELOPING.md
1148
+ */
1149
+ const baseContextSnapshot = await hydrateRecoveryBaseContextSnapshot(run, getRun);
1150
+ const recoveryContextSnapshot = buildRecoveryContextSnapshot({
1151
+ baseContextSnapshot,
1152
+ run,
1153
+ recoveryTrigger: opts.recoveryTrigger,
1154
+ wakeReason: opts.wakeReason,
1155
+ wakeSource: `recovery.${opts.recoveryTrigger}`,
1156
+ triggerDetail: opts.triggerDetail,
1157
+ });
1158
+ const issueId = readNonEmptyString(recoveryContextSnapshot.issueId);
1159
+ const taskKey = deriveTaskKey(recoveryContextSnapshot, null);
1160
+ const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey);
1161
+ const recovery = recoveryContextSnapshot.recovery;
1162
+ const requestPayload = {
1163
+ originalRunId: run.id,
1164
+ failureKind: recovery.failureKind,
1165
+ recoveryTrigger: recovery.recoveryTrigger,
1166
+ ...(issueId ? { issueId } : {}),
1167
+ };
1168
+ const outcome = await db.transaction(async (tx) => {
1169
+ let issueRow = null;
1170
+ if (issueId) {
1171
+ await tx.execute(sql `select id from issues where id = ${issueId} and org_id = ${run.orgId} for update`);
1172
+ issueRow = await tx
1173
+ .select({
1174
+ id: issues.id,
1175
+ orgId: issues.orgId,
1176
+ executionRunId: issues.executionRunId,
1177
+ executionAgentNameKey: issues.executionAgentNameKey,
1178
+ })
1179
+ .from(issues)
1180
+ .where(and(eq(issues.id, issueId), eq(issues.orgId, run.orgId)))
1181
+ .then((rows) => rows[0] ?? null);
1182
+ }
1183
+ if (issueRow?.executionRunId) {
1184
+ const activeExecutionRun = await tx
1185
+ .select()
1186
+ .from(heartbeatRuns)
1187
+ .where(eq(heartbeatRuns.id, issueRow.executionRunId))
1188
+ .then((rows) => rows[0] ?? null);
1189
+ const isActiveExecutionRun = activeExecutionRun &&
1190
+ (activeExecutionRun.status === "queued" || activeExecutionRun.status === "running");
1191
+ if (!isActiveExecutionRun) {
1192
+ await tx
1193
+ .update(issues)
1194
+ .set({
1195
+ executionRunId: null,
1196
+ executionAgentNameKey: null,
1197
+ executionLockedAt: null,
1198
+ updatedAt: opts.now,
1199
+ })
1200
+ .where(eq(issues.id, issueRow.id));
1201
+ issueRow = {
1202
+ ...issueRow,
1203
+ executionRunId: null,
1204
+ executionAgentNameKey: null,
1205
+ };
1206
+ }
1207
+ else if (activeExecutionRun) {
1208
+ const activeContext = parseObject(activeExecutionRun.contextSnapshot);
1209
+ const activeRecovery = parseObject(activeContext.recovery);
1210
+ if (activeExecutionRun.agentId === run.agentId &&
1211
+ (activeExecutionRun.retryOfRunId === run.id ||
1212
+ readNonEmptyString(activeRecovery.originalRunId) === run.id)) {
1213
+ return { kind: "existing", run: activeExecutionRun };
1214
+ }
1215
+ throw conflict("Issue already has an active execution run", {
1216
+ issueId: issueRow.id,
1217
+ executionRunId: activeExecutionRun.id,
1218
+ });
1219
+ }
1220
+ }
1221
+ const wakeupRequest = await tx
1222
+ .insert(agentWakeupRequests)
1223
+ .values({
1224
+ orgId: run.orgId,
1225
+ agentId: run.agentId,
1226
+ source: opts.source,
1227
+ triggerDetail: opts.triggerDetail,
1228
+ reason: opts.wakeReason,
1229
+ payload: requestPayload,
1230
+ status: "queued",
1231
+ requestedByActorType: opts.requestedByActorType ?? null,
1232
+ requestedByActorId: opts.requestedByActorId ?? null,
1233
+ updatedAt: opts.now,
1234
+ })
1235
+ .returning()
1236
+ .then((rows) => rows[0]);
1237
+ const recoveryRun = await tx
1238
+ .insert(heartbeatRuns)
1239
+ .values({
1240
+ orgId: run.orgId,
1241
+ agentId: run.agentId,
1242
+ invocationSource: opts.source,
1243
+ triggerDetail: opts.triggerDetail,
1244
+ status: "queued",
1245
+ wakeupRequestId: wakeupRequest.id,
1246
+ contextSnapshot: recoveryContextSnapshot,
1247
+ sessionIdBefore: sessionBefore,
1248
+ retryOfRunId: run.id,
1249
+ processLossRetryCount: opts.recoveryTrigger === "automatic"
1250
+ ? (run.processLossRetryCount ?? 0) + 1
1251
+ : (run.processLossRetryCount ?? 0),
1252
+ updatedAt: opts.now,
1253
+ })
1254
+ .returning()
1255
+ .then((rows) => rows[0]);
1256
+ await tx
1257
+ .update(agentWakeupRequests)
1258
+ .set({
1259
+ runId: recoveryRun.id,
1260
+ updatedAt: opts.now,
1261
+ })
1262
+ .where(eq(agentWakeupRequests.id, wakeupRequest.id));
1263
+ if (issueRow) {
1264
+ await tx
1265
+ .update(issues)
1266
+ .set({
1267
+ executionRunId: recoveryRun.id,
1268
+ executionAgentNameKey: normalizeAgentNameKey(agent.name),
1269
+ executionLockedAt: opts.now,
1270
+ updatedAt: opts.now,
1271
+ })
1272
+ .where(eq(issues.id, issueRow.id));
1273
+ }
1274
+ return { kind: "queued", run: recoveryRun };
1275
+ });
1276
+ if (outcome.kind === "existing")
1277
+ return outcome.run;
1278
+ const recoveryRun = outcome.run;
1279
+ await appendRunEvent(recoveryRun, await nextRunEventSeq(recoveryRun.id), {
1280
+ eventType: "lifecycle",
1281
+ stream: "system",
1282
+ level: opts.recoveryTrigger === "automatic" ? "warn" : "info",
1283
+ message: `Recovery queued from run ${run.id}`,
1284
+ payload: {
1285
+ originalRunId: run.id,
1286
+ failureKind: recovery.failureKind,
1287
+ failureSummary: recovery.failureSummary,
1288
+ recoveryTrigger: recovery.recoveryTrigger,
1289
+ recoveryMode: recovery.recoveryMode,
1290
+ },
1291
+ });
1292
+ publishLiveEvent({
1293
+ orgId: recoveryRun.orgId,
1294
+ type: "heartbeat.run.queued",
1295
+ payload: {
1296
+ runId: recoveryRun.id,
1297
+ agentId: recoveryRun.agentId,
1298
+ invocationSource: recoveryRun.invocationSource,
1299
+ triggerDetail: recoveryRun.triggerDetail,
1300
+ wakeupRequestId: recoveryRun.wakeupRequestId,
1301
+ },
1302
+ });
1303
+ if (opts.startImmediately !== false) {
1304
+ await startNextQueuedRunForAgent(agent.id);
1305
+ }
1306
+ return recoveryRun;
1307
+ }
1308
+ async function enqueueProcessLossRetry(run, agent, now) {
1309
+ return enqueueRecoveryRun(run, agent, {
1310
+ recoveryTrigger: "automatic",
1311
+ source: "automation",
1312
+ triggerDetail: "system",
1313
+ wakeReason: "process_lost_retry",
1314
+ requestedByActorType: "system",
1315
+ requestedByActorId: null,
1316
+ startImmediately: false,
1317
+ now,
1318
+ });
1319
+ }
1320
+ async function runHasIssueClosureComment(tx, run, issueId) {
1321
+ const commentActivity = await tx
1322
+ .select({ id: activityLog.id })
1323
+ .from(activityLog)
1324
+ .where(and(eq(activityLog.orgId, run.orgId), eq(activityLog.action, "issue.comment_added"), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issueId), eq(activityLog.runId, run.id)))
1325
+ .limit(1)
1326
+ .then((rows) => rows[0] ?? null);
1327
+ return Boolean(commentActivity);
1328
+ }
1329
+ async function issueHasDeferredWake(tx, orgId, issueId) {
1330
+ const deferred = await tx
1331
+ .select({ id: agentWakeupRequests.id })
1332
+ .from(agentWakeupRequests)
1333
+ .where(and(eq(agentWakeupRequests.orgId, orgId), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql `${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`))
1334
+ .limit(1)
1335
+ .then((rows) => rows[0] ?? null);
1336
+ return Boolean(deferred);
1337
+ }
1338
+ async function passiveFollowupAlreadyRecorded(tx, runId) {
1339
+ const idempotencyKey = `${ISSUE_PASSIVE_FOLLOWUP_REASON}:${runId}`;
1340
+ const existingWake = await tx
1341
+ .select({ id: agentWakeupRequests.id })
1342
+ .from(agentWakeupRequests)
1343
+ .where(eq(agentWakeupRequests.idempotencyKey, idempotencyKey))
1344
+ .limit(1)
1345
+ .then((rows) => rows[0] ?? null);
1346
+ if (existingWake)
1347
+ return true;
1348
+ const existingReview = await tx
1349
+ .select({ id: activityLog.id })
1350
+ .from(activityLog)
1351
+ .where(and(eq(activityLog.runId, runId), eq(activityLog.action, "issue.closure_needs_operator_review")))
1352
+ .limit(1)
1353
+ .then((rows) => rows[0] ?? null);
1354
+ return Boolean(existingReview);
1355
+ }
1356
+ async function evaluatePassiveIssueClosureForLockedIssue(input) {
1357
+ const { tx, run, issue, now } = input;
1358
+ const context = parseObject(run.contextSnapshot);
1359
+ const runIssueId = readNonEmptyString(context.issueId);
1360
+ if (!runIssueId || runIssueId !== issue.id)
1361
+ return { kind: "none", reason: "run_not_issue_backed" };
1362
+ if (run.status !== "succeeded")
1363
+ return { kind: "none", reason: "run_not_successful" };
1364
+ if (issue.status !== "todo" && issue.status !== "in_progress") {
1365
+ return { kind: "none", reason: "issue_has_closure_status" };
1366
+ }
1367
+ if (issue.assigneeAgentId !== run.agentId) {
1368
+ return { kind: "none", reason: "issue_no_longer_assigned_to_run_agent" };
1369
+ }
1370
+ if (await runHasIssueClosureComment(tx, run, issue.id)) {
1371
+ return { kind: "none", reason: "run_authored_issue_comment" };
1372
+ }
1373
+ if (await issueHasDeferredWake(tx, issue.orgId, issue.id)) {
1374
+ return { kind: "none", reason: "deferred_issue_wake_exists" };
1375
+ }
1376
+ if (await passiveFollowupAlreadyRecorded(tx, run.id)) {
1377
+ return { kind: "none", reason: "passive_followup_already_recorded" };
1378
+ }
1379
+ const agent = await tx
1380
+ .select()
1381
+ .from(agents)
1382
+ .where(eq(agents.id, run.agentId))
1383
+ .then((rows) => rows[0] ?? null);
1384
+ if (!agent || agent.orgId !== run.orgId) {
1385
+ return { kind: "none", reason: "agent_not_found" };
1386
+ }
1387
+ const policy = parseHeartbeatPolicy(agent);
1388
+ if (hasCredibleTimerContinuation({ agent, policy, run, now })) {
1389
+ return { kind: "none", reason: "timer_continuity_expected" };
1390
+ }
1391
+ const passiveContext = normalizePassiveFollowupContext(context.passiveFollowup);
1392
+ const currentAttempt = passiveContext?.attempt ?? 0;
1393
+ const originRunId = passiveContext?.originRunId ?? run.id;
1394
+ if (currentAttempt >= ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS) {
1395
+ return {
1396
+ kind: "operator_review",
1397
+ issue,
1398
+ originRunId,
1399
+ previousRunId: run.id,
1400
+ attempts: currentAttempt,
1401
+ reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
1402
+ };
1403
+ }
1404
+ const nextAttempt = currentAttempt + 1;
1405
+ const requestedAt = new Date(now.getTime() + passiveFollowupCooldownMs(nextAttempt));
1406
+ const contextSnapshot = buildPassiveFollowupContextSnapshot({
1407
+ run,
1408
+ issue,
1409
+ originRunId,
1410
+ attempt: nextAttempt,
1411
+ now,
1412
+ });
1413
+ const taskKey = deriveTaskKey(contextSnapshot, { issueId: issue.id });
1414
+ const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey);
1415
+ const requestPayload = {
1416
+ issueId: issue.id,
1417
+ originRunId,
1418
+ previousRunId: run.id,
1419
+ attempt: nextAttempt,
1420
+ reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
1421
+ };
1422
+ const wakeupRequest = await tx
1423
+ .insert(agentWakeupRequests)
1424
+ .values({
1425
+ orgId: run.orgId,
1426
+ agentId: run.agentId,
1427
+ source: "automation",
1428
+ triggerDetail: "system",
1429
+ reason: ISSUE_PASSIVE_FOLLOWUP_REASON,
1430
+ payload: requestPayload,
1431
+ status: "queued",
1432
+ requestedByActorType: "system",
1433
+ requestedByActorId: "issue_closure_governance",
1434
+ idempotencyKey: `${ISSUE_PASSIVE_FOLLOWUP_REASON}:${run.id}`,
1435
+ requestedAt,
1436
+ updatedAt: now,
1437
+ })
1438
+ .returning()
1439
+ .then((rows) => rows[0]);
1440
+ const followupRun = await tx
1441
+ .insert(heartbeatRuns)
1442
+ .values({
1443
+ orgId: run.orgId,
1444
+ agentId: run.agentId,
1445
+ invocationSource: "automation",
1446
+ triggerDetail: "system",
1447
+ status: "queued",
1448
+ wakeupRequestId: wakeupRequest.id,
1449
+ contextSnapshot,
1450
+ sessionIdBefore: sessionBefore,
1451
+ updatedAt: now,
1452
+ })
1453
+ .returning()
1454
+ .then((rows) => rows[0]);
1455
+ await tx
1456
+ .update(agentWakeupRequests)
1457
+ .set({
1458
+ runId: followupRun.id,
1459
+ updatedAt: now,
1460
+ })
1461
+ .where(eq(agentWakeupRequests.id, wakeupRequest.id));
1462
+ await tx
1463
+ .update(issues)
1464
+ .set({
1465
+ executionRunId: followupRun.id,
1466
+ executionAgentNameKey: normalizeAgentNameKey(agent.name),
1467
+ executionLockedAt: now,
1468
+ updatedAt: now,
1469
+ })
1470
+ .where(eq(issues.id, issue.id));
1471
+ return {
1472
+ kind: "queued",
1473
+ run: followupRun,
1474
+ issue,
1475
+ originRunId,
1476
+ previousRunId: run.id,
1477
+ attempt: nextAttempt,
1478
+ requestedAt,
1479
+ };
1480
+ }
1481
+ async function countRunningRunsForAgent(agentId) {
1482
+ const [{ count }] = await db
1483
+ .select({ count: sql `count(*)` })
1484
+ .from(heartbeatRuns)
1485
+ .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "running")));
1486
+ return Number(count ?? 0);
1487
+ }
1488
+ async function claimQueuedRun(run) {
1489
+ if (run.status !== "queued")
1490
+ return run;
1491
+ if (run.wakeupRequestId) {
1492
+ const wakeup = await db
1493
+ .select({ requestedAt: agentWakeupRequests.requestedAt })
1494
+ .from(agentWakeupRequests)
1495
+ .where(eq(agentWakeupRequests.id, run.wakeupRequestId))
1496
+ .then((rows) => rows[0] ?? null);
1497
+ if (wakeup && new Date(wakeup.requestedAt).getTime() > Date.now()) {
1498
+ return null;
1499
+ }
1500
+ }
1501
+ const agent = await getAgent(run.agentId);
1502
+ if (!agent) {
1503
+ await cancelRunInternal(run.id, "Cancelled because the agent no longer exists");
1504
+ return null;
1505
+ }
1506
+ if (agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval") {
1507
+ await cancelRunInternal(run.id, "Cancelled because the agent is not invokable");
1508
+ return null;
1509
+ }
1510
+ const context = parseObject(run.contextSnapshot);
1511
+ const budgetBlock = await budgets.getInvocationBlock(run.orgId, run.agentId, {
1512
+ issueId: readNonEmptyString(context.issueId),
1513
+ projectId: readNonEmptyString(context.projectId),
1514
+ });
1515
+ if (budgetBlock) {
1516
+ await cancelRunInternal(run.id, budgetBlock.reason);
1517
+ return null;
1518
+ }
1519
+ const claimedAt = new Date();
1520
+ const claimed = await db
1521
+ .update(heartbeatRuns)
1522
+ .set({
1523
+ status: "running",
1524
+ startedAt: run.startedAt ?? claimedAt,
1525
+ updatedAt: claimedAt,
1526
+ })
1527
+ .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "queued")))
1528
+ .returning()
1529
+ .then((rows) => rows[0] ?? null);
1530
+ if (!claimed)
1531
+ return null;
1532
+ publishLiveEvent({
1533
+ orgId: claimed.orgId,
1534
+ type: "heartbeat.run.status",
1535
+ payload: {
1536
+ runId: claimed.id,
1537
+ agentId: claimed.agentId,
1538
+ status: claimed.status,
1539
+ invocationSource: claimed.invocationSource,
1540
+ triggerDetail: claimed.triggerDetail,
1541
+ error: claimed.error ?? null,
1542
+ errorCode: claimed.errorCode ?? null,
1543
+ startedAt: claimed.startedAt ? new Date(claimed.startedAt).toISOString() : null,
1544
+ finishedAt: claimed.finishedAt ? new Date(claimed.finishedAt).toISOString() : null,
1545
+ },
1546
+ });
1547
+ await setWakeupStatus(claimed.wakeupRequestId, "claimed", { claimedAt });
1548
+ return claimed;
1549
+ }
1550
+ async function finalizeAgentStatus(agentId, outcome) {
1551
+ const existing = await getAgent(agentId);
1552
+ if (!existing)
1553
+ return;
1554
+ if (existing.status === "paused" || existing.status === "terminated") {
1555
+ return;
1556
+ }
1557
+ const runningCount = await countRunningRunsForAgent(agentId);
1558
+ const nextStatus = runningCount > 0
1559
+ ? "running"
1560
+ : outcome === "succeeded" || outcome === "cancelled"
1561
+ ? "idle"
1562
+ : "error";
1563
+ const updated = await db
1564
+ .update(agents)
1565
+ .set({
1566
+ status: nextStatus,
1567
+ lastHeartbeatAt: new Date(),
1568
+ updatedAt: new Date(),
1569
+ })
1570
+ .where(eq(agents.id, agentId))
1571
+ .returning()
1572
+ .then((rows) => rows[0] ?? null);
1573
+ if (updated) {
1574
+ publishLiveEvent({
1575
+ orgId: updated.orgId,
1576
+ type: "agent.status",
1577
+ payload: {
1578
+ agentId: updated.id,
1579
+ status: updated.status,
1580
+ lastHeartbeatAt: updated.lastHeartbeatAt
1581
+ ? new Date(updated.lastHeartbeatAt).toISOString()
1582
+ : null,
1583
+ outcome,
1584
+ },
1585
+ });
1586
+ }
1587
+ }
1588
+ async function reapOrphanedRuns(opts) {
1589
+ const staleThresholdMs = opts?.staleThresholdMs ?? 0;
1590
+ const now = new Date();
1591
+ // Find all runs stuck in "running" state (queued runs are legitimately waiting; resumeQueuedRuns handles them)
1592
+ const activeRuns = await db
1593
+ .select({
1594
+ run: heartbeatRuns,
1595
+ agentRuntimeType: agents.agentRuntimeType,
1596
+ })
1597
+ .from(heartbeatRuns)
1598
+ .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
1599
+ .where(eq(heartbeatRuns.status, "running"));
1600
+ const reaped = [];
1601
+ for (const { run, agentRuntimeType } of activeRuns) {
1602
+ if (runningProcesses.has(run.id) || activeRunExecutions.has(run.id))
1603
+ continue;
1604
+ // Apply staleness threshold to avoid false positives
1605
+ if (staleThresholdMs > 0) {
1606
+ const refTime = run.updatedAt ? new Date(run.updatedAt).getTime() : 0;
1607
+ if (now.getTime() - refTime < staleThresholdMs)
1608
+ continue;
1609
+ }
1610
+ const tracksLocalChild = isTrackedLocalChildProcessAdapter(agentRuntimeType);
1611
+ let detachedTerminationMessage = null;
1612
+ if (tracksLocalChild && run.processPid && isProcessAlive(run.processPid)) {
1613
+ const termination = await terminateOrphanedProcess(run.processPid);
1614
+ if (termination.stillAlive) {
1615
+ const detachedMessage = termination.error
1616
+ ? `Lost in-memory process handle, child pid ${run.processPid} is still alive, and Rudder could not terminate it: ${termination.error}`
1617
+ : `Lost in-memory process handle, but child pid ${run.processPid} is still alive`;
1618
+ const detachedRun = await setRunStatus(run.id, "running", {
1619
+ error: detachedMessage,
1620
+ errorCode: DETACHED_PROCESS_ERROR_CODE,
1621
+ });
1622
+ if (detachedRun) {
1623
+ await appendRunEvent(detachedRun, await nextRunEventSeq(detachedRun.id), {
1624
+ eventType: "lifecycle",
1625
+ stream: "system",
1626
+ level: "warn",
1627
+ message: detachedMessage,
1628
+ payload: {
1629
+ processPid: run.processPid,
1630
+ },
1631
+ });
1632
+ }
1633
+ continue;
1634
+ }
1635
+ detachedTerminationMessage = termination.terminationSignal
1636
+ ? `Terminated detached child pid ${run.processPid} with ${termination.terminationSignal} after Rudder lost its process handle`
1637
+ : `Detached child pid ${run.processPid} exited before Rudder could terminate it`;
1638
+ }
1639
+ const shouldRetry = tracksLocalChild && !!run.processPid && (run.processLossRetryCount ?? 0) < 1;
1640
+ const baseMessage = run.processPid
1641
+ ? `Process lost -- child pid ${run.processPid} is no longer running`
1642
+ : "Process lost -- server may have restarted";
1643
+ let finalizedRun = await setRunStatus(run.id, "failed", {
1644
+ error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
1645
+ errorCode: "process_lost",
1646
+ finishedAt: now,
1647
+ });
1648
+ await setWakeupStatus(run.wakeupRequestId, "failed", {
1649
+ finishedAt: now,
1650
+ error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
1651
+ });
1652
+ if (!finalizedRun)
1653
+ finalizedRun = await getRun(run.id);
1654
+ if (!finalizedRun)
1655
+ continue;
1656
+ if (detachedTerminationMessage) {
1657
+ await appendRunEvent(finalizedRun, await nextRunEventSeq(finalizedRun.id), {
1658
+ eventType: "lifecycle",
1659
+ stream: "system",
1660
+ level: "warn",
1661
+ message: detachedTerminationMessage,
1662
+ payload: {
1663
+ ...(run.processPid ? { processPid: run.processPid } : {}),
1664
+ },
1665
+ });
1666
+ }
1667
+ let retriedRun = null;
1668
+ if (shouldRetry) {
1669
+ const agent = await getAgent(run.agentId);
1670
+ if (agent) {
1671
+ retriedRun = await enqueueProcessLossRetry(finalizedRun, agent, now);
1672
+ }
1673
+ }
1674
+ else {
1675
+ await releaseIssueExecutionAndPromote(finalizedRun);
1676
+ }
1677
+ await appendRunEvent(finalizedRun, await nextRunEventSeq(finalizedRun.id), {
1678
+ eventType: "lifecycle",
1679
+ stream: "system",
1680
+ level: "error",
1681
+ message: shouldRetry
1682
+ ? `${baseMessage}; queued retry ${retriedRun?.id ?? ""}`.trim()
1683
+ : baseMessage,
1684
+ payload: {
1685
+ ...(run.processPid ? { processPid: run.processPid } : {}),
1686
+ ...(retriedRun ? { retryRunId: retriedRun.id } : {}),
1687
+ },
1688
+ });
1689
+ await finalizeAgentStatus(run.agentId, "failed");
1690
+ await startNextQueuedRunForAgent(run.agentId);
1691
+ runningProcesses.delete(run.id);
1692
+ reaped.push(run.id);
1693
+ }
1694
+ if (reaped.length > 0) {
1695
+ logger.warn({ reapedCount: reaped.length, runIds: reaped }, "reaped orphaned heartbeat runs");
1696
+ }
1697
+ return { reaped: reaped.length, runIds: reaped };
1698
+ }
1699
+ async function resumeQueuedRuns() {
1700
+ const queuedRuns = await db
1701
+ .select({ agentId: heartbeatRuns.agentId })
1702
+ .from(heartbeatRuns)
1703
+ .where(eq(heartbeatRuns.status, "queued"));
1704
+ const agentIds = [...new Set(queuedRuns.map((r) => r.agentId))];
1705
+ for (const agentId of agentIds) {
1706
+ await startNextQueuedRunForAgent(agentId);
1707
+ }
1708
+ }
1709
+ async function updateRuntimeState(agent, run, result, session, normalizedUsage) {
1710
+ await ensureRuntimeState(agent);
1711
+ const usage = normalizedUsage ?? normalizeUsageTotals(result.usage);
1712
+ const inputTokens = usage?.inputTokens ?? 0;
1713
+ const outputTokens = usage?.outputTokens ?? 0;
1714
+ const cachedInputTokens = usage?.cachedInputTokens ?? 0;
1715
+ const billingType = normalizeLedgerBillingType(result.billingType);
1716
+ const additionalCostCents = normalizeBilledCostCents(result.costUsd, billingType);
1717
+ const hasTokenUsage = inputTokens > 0 || outputTokens > 0 || cachedInputTokens > 0;
1718
+ const provider = result.provider ?? "unknown";
1719
+ const biller = resolveLedgerBiller(result);
1720
+ const ledgerScope = await resolveLedgerScopeForRun(db, agent.orgId, run);
1721
+ await db
1722
+ .update(agentRuntimeState)
1723
+ .set({
1724
+ agentRuntimeType: agent.agentRuntimeType,
1725
+ sessionId: session.legacySessionId,
1726
+ lastRunId: run.id,
1727
+ lastRunStatus: run.status,
1728
+ lastError: result.errorMessage ?? null,
1729
+ totalInputTokens: sql `${agentRuntimeState.totalInputTokens} + ${inputTokens}`,
1730
+ totalOutputTokens: sql `${agentRuntimeState.totalOutputTokens} + ${outputTokens}`,
1731
+ totalCachedInputTokens: sql `${agentRuntimeState.totalCachedInputTokens} + ${cachedInputTokens}`,
1732
+ totalCostCents: sql `${agentRuntimeState.totalCostCents} + ${additionalCostCents}`,
1733
+ updatedAt: new Date(),
1734
+ })
1735
+ .where(eq(agentRuntimeState.agentId, agent.id));
1736
+ if (additionalCostCents > 0 || hasTokenUsage) {
1737
+ const costs = costService(db, budgetHooks);
1738
+ await costs.createEvent(agent.orgId, {
1739
+ heartbeatRunId: run.id,
1740
+ agentId: agent.id,
1741
+ issueId: ledgerScope.issueId,
1742
+ projectId: ledgerScope.projectId,
1743
+ provider,
1744
+ biller,
1745
+ billingType,
1746
+ model: result.model ?? "unknown",
1747
+ inputTokens,
1748
+ cachedInputTokens,
1749
+ outputTokens,
1750
+ costCents: additionalCostCents,
1751
+ occurredAt: new Date(),
1752
+ });
1753
+ }
1754
+ }
1755
+ async function startNextQueuedRunForAgent(agentId) {
1756
+ return withAgentStartLock(agentId, async () => {
1757
+ const agent = await getAgent(agentId);
1758
+ if (!agent)
1759
+ return [];
1760
+ if (agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval") {
1761
+ return [];
1762
+ }
1763
+ const policy = parseHeartbeatPolicy(agent);
1764
+ const runningCount = await countRunningRunsForAgent(agentId);
1765
+ const availableSlots = Math.max(0, policy.maxConcurrentRuns - runningCount);
1766
+ if (availableSlots <= 0)
1767
+ return [];
1768
+ const queuedRuns = await db
1769
+ .select()
1770
+ .from(heartbeatRuns)
1771
+ .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued"), sql `(
1772
+ ${heartbeatRuns.wakeupRequestId} is null
1773
+ or exists (
1774
+ select 1
1775
+ from ${agentWakeupRequests}
1776
+ where ${agentWakeupRequests.id} = ${heartbeatRuns.wakeupRequestId}
1777
+ and ${agentWakeupRequests.requestedAt} <= now()
1778
+ )
1779
+ )`))
1780
+ .orderBy(asc(heartbeatRuns.createdAt))
1781
+ .limit(availableSlots);
1782
+ if (queuedRuns.length === 0)
1783
+ return [];
1784
+ const claimedRuns = [];
1785
+ for (const queuedRun of queuedRuns) {
1786
+ const claimed = await claimQueuedRun(queuedRun);
1787
+ if (claimed)
1788
+ claimedRuns.push(claimed);
1789
+ }
1790
+ if (claimedRuns.length === 0)
1791
+ return [];
1792
+ for (const claimedRun of claimedRuns) {
1793
+ void executeRun(claimedRun.id).catch((err) => {
1794
+ logger.error({ err, runId: claimedRun.id }, "queued heartbeat execution failed");
1795
+ });
1796
+ }
1797
+ return claimedRuns;
1798
+ });
1799
+ }
1800
+ async function executeRun(runId) {
1801
+ let run = await getRun(runId);
1802
+ if (!run)
1803
+ return;
1804
+ if (run.status !== "queued" && run.status !== "running")
1805
+ return;
1806
+ if (run.status === "queued") {
1807
+ const claimed = await claimQueuedRun(run);
1808
+ if (!claimed) {
1809
+ // Another worker has already claimed or finalized this run.
1810
+ return;
1811
+ }
1812
+ run = claimed;
1813
+ }
1814
+ activeRunExecutions.add(run.id);
1815
+ try {
1816
+ const agent = await getAgent(run.agentId);
1817
+ if (!agent) {
1818
+ await setRunStatus(runId, "failed", {
1819
+ error: "Agent not found",
1820
+ errorCode: "agent_not_found",
1821
+ finishedAt: new Date(),
1822
+ });
1823
+ await setWakeupStatus(run.wakeupRequestId, "failed", {
1824
+ finishedAt: new Date(),
1825
+ error: "Agent not found",
1826
+ });
1827
+ const failedRun = await getRun(runId);
1828
+ if (failedRun)
1829
+ await releaseIssueExecutionAndPromote(failedRun);
1830
+ return;
1831
+ }
1832
+ const heartbeatObservationContext = buildHeartbeatObservabilityContext(run, {
1833
+ runtime: agent.agentRuntimeType,
1834
+ metadata: {
1835
+ agentName: agent.name,
1836
+ invocationSource: run.invocationSource,
1837
+ triggerDetail: run.triggerDetail,
1838
+ },
1839
+ });
1840
+ await withExecutionObservation(heartbeatObservationContext, {
1841
+ name: buildHeartbeatObservationName(run, agent.name),
1842
+ asType: "agent",
1843
+ input: {
1844
+ agentId: agent.id,
1845
+ agentName: agent.name,
1846
+ invocationSource: run.invocationSource,
1847
+ triggerDetail: run.triggerDetail,
1848
+ issueId: readNonEmptyString(parseObject(run.contextSnapshot).issueId),
1849
+ },
1850
+ }, async (observation) => {
1851
+ const executionTranscript = [];
1852
+ let stdoutTranscriptBuffer = "";
1853
+ let stderrTranscriptBuffer = "";
1854
+ let stdoutTranscriptParser = null;
1855
+ let transcriptFallbackResult = null;
1856
+ let finalObservationOutput = null;
1857
+ let finalObservationStatus = run.status;
1858
+ let finalObservationSessionId = heartbeatObservationContext.sessionKey ?? null;
1859
+ const runtime = await ensureRuntimeState(agent);
1860
+ const context = parseObject(run.contextSnapshot);
1861
+ const taskKey = deriveTaskKey(context, null);
1862
+ const sessionCodec = getAgentRuntimeSessionCodec(agent.agentRuntimeType);
1863
+ const issueId = readNonEmptyString(context.issueId);
1864
+ const issueContext = issueId
1865
+ ? await db
1866
+ .select({
1867
+ id: issues.id,
1868
+ identifier: issues.identifier,
1869
+ title: issues.title,
1870
+ projectId: issues.projectId,
1871
+ projectWorkspaceId: issues.projectWorkspaceId,
1872
+ executionWorkspaceId: issues.executionWorkspaceId,
1873
+ executionWorkspacePreference: issues.executionWorkspacePreference,
1874
+ assigneeAgentId: issues.assigneeAgentId,
1875
+ assigneeAgentRuntimeOverrides: issues.assigneeAgentRuntimeOverrides,
1876
+ executionWorkspaceSettings: issues.executionWorkspaceSettings,
1877
+ })
1878
+ .from(issues)
1879
+ .where(and(eq(issues.id, issueId), eq(issues.orgId, agent.orgId)))
1880
+ .then((rows) => rows[0] ?? null)
1881
+ : null;
1882
+ const issueAssigneeOverrides = issueContext && issueContext.assigneeAgentId === agent.id
1883
+ ? parseIssueAssigneeAgentRuntimeOverrides(issueContext.assigneeAgentRuntimeOverrides)
1884
+ : null;
1885
+ const issueExecutionWorkspaceSettings = parseIssueExecutionWorkspaceSettings(issueContext?.executionWorkspaceSettings);
1886
+ const contextProjectId = readNonEmptyString(context.projectId);
1887
+ const executionProjectId = issueContext?.projectId ?? contextProjectId;
1888
+ const projectExecutionWorkspacePolicy = executionProjectId
1889
+ ? await db
1890
+ .select({ executionWorkspacePolicy: projects.executionWorkspacePolicy })
1891
+ .from(projects)
1892
+ .where(and(eq(projects.id, executionProjectId), eq(projects.orgId, agent.orgId)))
1893
+ .then((rows) => parseProjectExecutionWorkspacePolicy(rows[0]?.executionWorkspacePolicy))
1894
+ : null;
1895
+ const taskSession = taskKey
1896
+ ? await getTaskSession(agent.orgId, agent.id, agent.agentRuntimeType, taskKey)
1897
+ : null;
1898
+ const resetTaskSession = shouldResetTaskSessionForWake(context);
1899
+ const sessionResetReason = describeSessionResetReason(context);
1900
+ const taskSessionForRun = resetTaskSession ? null : taskSession;
1901
+ const explicitResumeSessionParams = normalizeSessionParams(sessionCodec.deserialize(parseObject(context.resumeSessionParams)));
1902
+ const explicitResumeSessionDisplayId = truncateDisplayId(readNonEmptyString(context.resumeSessionDisplayId) ??
1903
+ (sessionCodec.getDisplayId ? sessionCodec.getDisplayId(explicitResumeSessionParams) : null) ??
1904
+ readNonEmptyString(explicitResumeSessionParams?.sessionId));
1905
+ const previousSessionParams = explicitResumeSessionParams ??
1906
+ (explicitResumeSessionDisplayId ? { sessionId: explicitResumeSessionDisplayId } : null) ??
1907
+ normalizeSessionParams(sessionCodec.deserialize(taskSessionForRun?.sessionParamsJson ?? null));
1908
+ const config = parseObject(agent.agentRuntimeConfig);
1909
+ const executionWorkspaceMode = resolveExecutionWorkspaceMode({
1910
+ projectPolicy: projectExecutionWorkspacePolicy,
1911
+ issueSettings: issueExecutionWorkspaceSettings,
1912
+ legacyUseProjectWorkspace: issueAssigneeOverrides?.useProjectWorkspace ?? null,
1913
+ });
1914
+ const resolvedWorkspace = await runContextSvc.resolveWorkspaceForRun(agent, context, previousSessionParams, { useProjectWorkspace: executionWorkspaceMode !== "agent_default" });
1915
+ const workspaceManagedConfig = buildExecutionWorkspaceAdapterConfig({
1916
+ agentConfig: config,
1917
+ projectPolicy: projectExecutionWorkspacePolicy,
1918
+ issueSettings: issueExecutionWorkspaceSettings,
1919
+ mode: executionWorkspaceMode,
1920
+ legacyUseProjectWorkspace: issueAssigneeOverrides?.useProjectWorkspace ?? null,
1921
+ });
1922
+ const mergedConfig = issueAssigneeOverrides?.agentRuntimeConfig
1923
+ ? { ...workspaceManagedConfig, ...issueAssigneeOverrides.agentRuntimeConfig }
1924
+ : workspaceManagedConfig;
1925
+ const { resolvedConfig, runtimeConfig, runtimeSkillEntries, secretKeys } = await runContextSvc.prepareRuntimeConfig({
1926
+ scene: "heartbeat",
1927
+ agent,
1928
+ baseConfig: mergedConfig,
1929
+ });
1930
+ heartbeatObservationContext.metadata = {
1931
+ ...(heartbeatObservationContext.metadata ?? {}),
1932
+ ...buildHeartbeatRuntimeTraceMetadata({
1933
+ runtimeConfig,
1934
+ runtimeSkills: runtimeSkillEntries,
1935
+ }),
1936
+ };
1937
+ const issueRef = issueContext
1938
+ ? {
1939
+ id: issueContext.id,
1940
+ identifier: issueContext.identifier,
1941
+ title: issueContext.title,
1942
+ projectId: issueContext.projectId,
1943
+ projectWorkspaceId: issueContext.projectWorkspaceId,
1944
+ executionWorkspaceId: issueContext.executionWorkspaceId,
1945
+ executionWorkspacePreference: issueContext.executionWorkspacePreference,
1946
+ }
1947
+ : null;
1948
+ const rootObservationInput = {
1949
+ agentId: agent.id,
1950
+ agentName: agent.name,
1951
+ invocationSource: run.invocationSource,
1952
+ triggerDetail: run.triggerDetail,
1953
+ issue: issueRef
1954
+ ? {
1955
+ id: issueRef.id,
1956
+ identifier: issueRef.identifier ?? null,
1957
+ title: issueRef.title ?? null,
1958
+ }
1959
+ : null,
1960
+ };
1961
+ updateExecutionObservation(observation, heartbeatObservationContext, {
1962
+ input: rootObservationInput,
1963
+ });
1964
+ updateExecutionTraceIO(observation, { input: rootObservationInput });
1965
+ if (issueRef) {
1966
+ updateExecutionTraceName(observation, buildIssueRunTraceName({
1967
+ issueTitle: issueRef.title,
1968
+ issueId: issueRef.id,
1969
+ }));
1970
+ }
1971
+ const existingExecutionWorkspace = issueRef?.executionWorkspaceId ? await executionWorkspacesSvc.getById(issueRef.executionWorkspaceId) : null;
1972
+ const workspaceOperationRecorder = workspaceOperationsSvc.createRecorder({
1973
+ orgId: agent.orgId,
1974
+ heartbeatRunId: run.id,
1975
+ executionWorkspaceId: existingExecutionWorkspace?.id ?? null,
1976
+ });
1977
+ const executionWorkspace = await realizeExecutionWorkspace({
1978
+ base: {
1979
+ baseCwd: resolvedWorkspace.cwd,
1980
+ source: resolvedWorkspace.source,
1981
+ projectId: resolvedWorkspace.projectId,
1982
+ workspaceId: resolvedWorkspace.workspaceId,
1983
+ repoUrl: resolvedWorkspace.repoUrl,
1984
+ repoRef: resolvedWorkspace.repoRef,
1985
+ },
1986
+ config: runtimeConfig,
1987
+ issue: issueRef,
1988
+ agent: {
1989
+ id: agent.id,
1990
+ name: agent.name,
1991
+ orgId: agent.orgId,
1992
+ },
1993
+ recorder: workspaceOperationRecorder,
1994
+ });
1995
+ const resolvedProjectId = executionWorkspace.projectId ?? issueRef?.projectId ?? executionProjectId ?? null;
1996
+ const resolvedProjectWorkspaceId = issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null;
1997
+ const shouldReuseExisting = issueRef?.executionWorkspacePreference === "reuse_existing" &&
1998
+ existingExecutionWorkspace &&
1999
+ existingExecutionWorkspace.status !== "archived";
2000
+ let persistedExecutionWorkspace = null;
2001
+ try {
2002
+ persistedExecutionWorkspace = shouldReuseExisting && existingExecutionWorkspace
2003
+ ? await executionWorkspacesSvc.update(existingExecutionWorkspace.id, {
2004
+ cwd: executionWorkspace.cwd,
2005
+ repoUrl: executionWorkspace.repoUrl,
2006
+ baseRef: executionWorkspace.repoRef,
2007
+ branchName: executionWorkspace.branchName,
2008
+ providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs",
2009
+ providerRef: executionWorkspace.worktreePath,
2010
+ status: "active",
2011
+ lastUsedAt: new Date(),
2012
+ metadata: {
2013
+ ...(existingExecutionWorkspace.metadata ?? {}),
2014
+ source: executionWorkspace.source,
2015
+ createdByRuntime: executionWorkspace.created,
2016
+ },
2017
+ })
2018
+ : resolvedProjectId
2019
+ ? await executionWorkspacesSvc.create({
2020
+ orgId: agent.orgId,
2021
+ projectId: resolvedProjectId,
2022
+ projectWorkspaceId: resolvedProjectWorkspaceId,
2023
+ sourceIssueId: issueRef?.id ?? null,
2024
+ mode: executionWorkspaceMode === "isolated_workspace"
2025
+ ? "isolated_workspace"
2026
+ : executionWorkspaceMode === "operator_branch"
2027
+ ? "operator_branch"
2028
+ : executionWorkspaceMode === "agent_default"
2029
+ ? "adapter_managed"
2030
+ : "shared_workspace",
2031
+ strategyType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "project_primary",
2032
+ name: executionWorkspace.branchName ?? issueRef?.identifier ?? `workspace-${agent.id.slice(0, 8)}`,
2033
+ status: "active",
2034
+ cwd: executionWorkspace.cwd,
2035
+ repoUrl: executionWorkspace.repoUrl,
2036
+ baseRef: executionWorkspace.repoRef,
2037
+ branchName: executionWorkspace.branchName,
2038
+ providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs",
2039
+ providerRef: executionWorkspace.worktreePath,
2040
+ lastUsedAt: new Date(),
2041
+ openedAt: new Date(),
2042
+ metadata: {
2043
+ source: executionWorkspace.source,
2044
+ createdByRuntime: executionWorkspace.created,
2045
+ },
2046
+ })
2047
+ : null;
2048
+ }
2049
+ catch (error) {
2050
+ if (executionWorkspace.created) {
2051
+ try {
2052
+ await cleanupExecutionWorkspaceArtifacts({
2053
+ workspace: {
2054
+ id: existingExecutionWorkspace?.id ?? `transient-${run.id}`,
2055
+ cwd: executionWorkspace.cwd,
2056
+ providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs",
2057
+ providerRef: executionWorkspace.worktreePath,
2058
+ branchName: executionWorkspace.branchName,
2059
+ repoUrl: executionWorkspace.repoUrl,
2060
+ baseRef: executionWorkspace.repoRef,
2061
+ projectId: resolvedProjectId,
2062
+ projectWorkspaceId: resolvedProjectWorkspaceId,
2063
+ sourceIssueId: issueRef?.id ?? null,
2064
+ metadata: {
2065
+ createdByRuntime: true,
2066
+ source: executionWorkspace.source,
2067
+ },
2068
+ },
2069
+ projectWorkspace: {
2070
+ cwd: resolvedWorkspace.cwd,
2071
+ cleanupCommand: null,
2072
+ },
2073
+ teardownCommand: projectExecutionWorkspacePolicy?.workspaceStrategy?.teardownCommand ?? null,
2074
+ recorder: workspaceOperationRecorder,
2075
+ });
2076
+ }
2077
+ catch (cleanupError) {
2078
+ logger.warn({
2079
+ runId: run.id,
2080
+ issueId,
2081
+ executionWorkspaceCwd: executionWorkspace.cwd,
2082
+ cleanupError: cleanupError instanceof Error ? cleanupError.message : String(cleanupError),
2083
+ }, "Failed to cleanup realized execution workspace after persistence failure");
2084
+ }
2085
+ }
2086
+ throw error;
2087
+ }
2088
+ await workspaceOperationRecorder.attachExecutionWorkspaceId(persistedExecutionWorkspace?.id ?? null);
2089
+ if (existingExecutionWorkspace &&
2090
+ persistedExecutionWorkspace &&
2091
+ existingExecutionWorkspace.id !== persistedExecutionWorkspace.id &&
2092
+ existingExecutionWorkspace.status === "active") {
2093
+ await executionWorkspacesSvc.update(existingExecutionWorkspace.id, {
2094
+ status: "idle",
2095
+ cleanupReason: null,
2096
+ });
2097
+ }
2098
+ if (issueId && persistedExecutionWorkspace) {
2099
+ const nextIssueWorkspaceMode = issueExecutionWorkspaceModeForPersistedWorkspace(persistedExecutionWorkspace.mode);
2100
+ const shouldSwitchIssueToExistingWorkspace = issueRef?.executionWorkspacePreference === "reuse_existing" ||
2101
+ executionWorkspaceMode === "isolated_workspace" ||
2102
+ executionWorkspaceMode === "operator_branch";
2103
+ const nextIssuePatch = {};
2104
+ if (issueRef?.executionWorkspaceId !== persistedExecutionWorkspace.id) {
2105
+ nextIssuePatch.executionWorkspaceId = persistedExecutionWorkspace.id;
2106
+ }
2107
+ if (resolvedProjectWorkspaceId && issueRef?.projectWorkspaceId !== resolvedProjectWorkspaceId) {
2108
+ nextIssuePatch.projectWorkspaceId = resolvedProjectWorkspaceId;
2109
+ }
2110
+ if (shouldSwitchIssueToExistingWorkspace) {
2111
+ nextIssuePatch.executionWorkspacePreference = "reuse_existing";
2112
+ nextIssuePatch.executionWorkspaceSettings = {
2113
+ ...(issueExecutionWorkspaceSettings ?? {}),
2114
+ mode: nextIssueWorkspaceMode,
2115
+ };
2116
+ }
2117
+ if (Object.keys(nextIssuePatch).length > 0) {
2118
+ await issuesSvc.update(issueId, nextIssuePatch);
2119
+ }
2120
+ }
2121
+ if (persistedExecutionWorkspace) {
2122
+ context.executionWorkspaceId = persistedExecutionWorkspace.id;
2123
+ await db
2124
+ .update(heartbeatRuns)
2125
+ .set({
2126
+ contextSnapshot: context,
2127
+ updatedAt: new Date(),
2128
+ })
2129
+ .where(eq(heartbeatRuns.id, run.id));
2130
+ }
2131
+ const runtimeSessionResolution = resolveRuntimeSessionParamsForWorkspace({
2132
+ orgId: agent.orgId,
2133
+ agent,
2134
+ previousSessionParams,
2135
+ resolvedWorkspace: {
2136
+ ...resolvedWorkspace,
2137
+ cwd: resolveDefaultAgentWorkspaceDir(agent.orgId, agent),
2138
+ source: "agent_home",
2139
+ },
2140
+ });
2141
+ const runtimeSessionParams = runtimeSessionResolution.sessionParams;
2142
+ const runtimeWorkspaceWarnings = [
2143
+ ...resolvedWorkspace.warnings,
2144
+ ...executionWorkspace.warnings,
2145
+ ...(runtimeSessionResolution.warning ? [runtimeSessionResolution.warning] : []),
2146
+ ...(resetTaskSession && sessionResetReason
2147
+ ? [
2148
+ taskKey
2149
+ ? `Skipping saved session resume for task "${taskKey}" because ${sessionResetReason}.`
2150
+ : `Skipping saved session resume because ${sessionResetReason}.`,
2151
+ ]
2152
+ : []),
2153
+ ];
2154
+ const runtimeSceneContext = await runContextSvc.buildSceneContext({
2155
+ scene: "heartbeat",
2156
+ agent,
2157
+ resolvedWorkspace,
2158
+ runtimeConfig,
2159
+ executionWorkspaceMode,
2160
+ executionWorkspace: {
2161
+ cwd: executionWorkspace.cwd,
2162
+ source: executionWorkspace.source,
2163
+ strategy: executionWorkspace.strategy,
2164
+ projectId: executionWorkspace.projectId,
2165
+ workspaceId: executionWorkspace.workspaceId,
2166
+ repoUrl: executionWorkspace.repoUrl,
2167
+ repoRef: executionWorkspace.repoRef,
2168
+ branchName: executionWorkspace.branchName,
2169
+ worktreePath: executionWorkspace.worktreePath,
2170
+ },
2171
+ });
2172
+ context.rudderScene = runtimeSceneContext.rudderScene;
2173
+ context.rudderWorkspace = runtimeSceneContext.rudderWorkspace;
2174
+ context.rudderWorkspaces = runtimeSceneContext.rudderWorkspaces;
2175
+ if (runtimeSceneContext.rudderRuntimeServiceIntents) {
2176
+ context.rudderRuntimeServiceIntents = runtimeSceneContext.rudderRuntimeServiceIntents;
2177
+ }
2178
+ else {
2179
+ delete context.rudderRuntimeServiceIntents;
2180
+ }
2181
+ if (executionWorkspace.projectId && !readNonEmptyString(context.projectId)) {
2182
+ context.projectId = executionWorkspace.projectId;
2183
+ }
2184
+ const runtimeSessionFallback = taskKey || resetTaskSession ? null : runtime.sessionId;
2185
+ let previousSessionDisplayId = truncateDisplayId(explicitResumeSessionDisplayId ??
2186
+ taskSessionForRun?.sessionDisplayId ??
2187
+ (sessionCodec.getDisplayId ? sessionCodec.getDisplayId(runtimeSessionParams) : null) ??
2188
+ readNonEmptyString(runtimeSessionParams?.sessionId) ??
2189
+ runtimeSessionFallback);
2190
+ let runtimeSessionIdForAdapter = readNonEmptyString(runtimeSessionParams?.sessionId) ?? runtimeSessionFallback;
2191
+ let runtimeSessionParamsForAdapter = runtimeSessionParams;
2192
+ const sessionCompaction = await evaluateSessionCompaction({
2193
+ agent,
2194
+ sessionId: previousSessionDisplayId ?? runtimeSessionIdForAdapter,
2195
+ issueId,
2196
+ });
2197
+ if (sessionCompaction.rotate) {
2198
+ context.rudderSessionHandoffMarkdown = sessionCompaction.handoffMarkdown;
2199
+ context.rudderSessionRotationReason = sessionCompaction.reason;
2200
+ context.rudderPreviousSessionId = previousSessionDisplayId ?? runtimeSessionIdForAdapter;
2201
+ runtimeSessionIdForAdapter = null;
2202
+ runtimeSessionParamsForAdapter = null;
2203
+ previousSessionDisplayId = null;
2204
+ if (sessionCompaction.reason) {
2205
+ runtimeWorkspaceWarnings.push(`Starting a fresh session because ${sessionCompaction.reason}.`);
2206
+ }
2207
+ }
2208
+ else {
2209
+ delete context.rudderSessionHandoffMarkdown;
2210
+ delete context.rudderSessionRotationReason;
2211
+ delete context.rudderPreviousSessionId;
2212
+ }
2213
+ const runtimeForAdapter = {
2214
+ sessionId: runtimeSessionIdForAdapter,
2215
+ sessionParams: runtimeSessionParamsForAdapter,
2216
+ sessionDisplayId: previousSessionDisplayId,
2217
+ taskKey,
2218
+ };
2219
+ let seq = 1;
2220
+ let handle = null;
2221
+ let stdoutExcerpt = "";
2222
+ let stderrExcerpt = "";
2223
+ try {
2224
+ const startedAt = run.startedAt ?? new Date();
2225
+ const runningWithSession = await db
2226
+ .update(heartbeatRuns)
2227
+ .set({
2228
+ startedAt,
2229
+ sessionIdBefore: runtimeForAdapter.sessionDisplayId ?? runtimeForAdapter.sessionId,
2230
+ contextSnapshot: context,
2231
+ updatedAt: new Date(),
2232
+ })
2233
+ .where(eq(heartbeatRuns.id, run.id))
2234
+ .returning()
2235
+ .then((rows) => rows[0] ?? null);
2236
+ if (runningWithSession)
2237
+ run = runningWithSession;
2238
+ const runningAgent = await db
2239
+ .update(agents)
2240
+ .set({ status: "running", updatedAt: new Date() })
2241
+ .where(eq(agents.id, agent.id))
2242
+ .returning()
2243
+ .then((rows) => rows[0] ?? null);
2244
+ if (runningAgent) {
2245
+ publishLiveEvent({
2246
+ orgId: runningAgent.orgId,
2247
+ type: "agent.status",
2248
+ payload: {
2249
+ agentId: runningAgent.id,
2250
+ status: runningAgent.status,
2251
+ outcome: "running",
2252
+ },
2253
+ });
2254
+ }
2255
+ const currentRun = run;
2256
+ await appendRunEvent(currentRun, seq++, {
2257
+ eventType: "lifecycle",
2258
+ stream: "system",
2259
+ level: "info",
2260
+ message: "run started",
2261
+ });
2262
+ handle = await runLogStore.begin({
2263
+ orgId: run.orgId,
2264
+ agentId: run.agentId,
2265
+ runId,
2266
+ });
2267
+ await db
2268
+ .update(heartbeatRuns)
2269
+ .set({
2270
+ logStore: handle.store,
2271
+ logRef: handle.logRef,
2272
+ updatedAt: new Date(),
2273
+ })
2274
+ .where(eq(heartbeatRuns.id, runId));
2275
+ const adapter = getServerAdapter(agent.agentRuntimeType);
2276
+ stdoutTranscriptParser = adapter.parseStdoutLine ?? null;
2277
+ const currentUserRedactionOptions = await getCurrentUserRedactionOptions();
2278
+ const onLog = async (stream, chunk) => {
2279
+ const sanitizedChunk = redactCurrentUserText(chunk, currentUserRedactionOptions);
2280
+ if (stream === "stdout")
2281
+ stdoutExcerpt = appendExcerpt(stdoutExcerpt, sanitizedChunk);
2282
+ if (stream === "stderr")
2283
+ stderrExcerpt = appendExcerpt(stderrExcerpt, sanitizedChunk);
2284
+ const ts = new Date().toISOString();
2285
+ if (handle) {
2286
+ await runLogStore.append(handle, {
2287
+ stream,
2288
+ chunk: sanitizedChunk,
2289
+ ts,
2290
+ });
2291
+ }
2292
+ const payloadChunk = sanitizedChunk.length > MAX_LIVE_LOG_CHUNK_BYTES
2293
+ ? sanitizedChunk.slice(sanitizedChunk.length - MAX_LIVE_LOG_CHUNK_BYTES)
2294
+ : sanitizedChunk;
2295
+ publishLiveEvent({
2296
+ orgId: run.orgId,
2297
+ type: "heartbeat.run.log",
2298
+ payload: {
2299
+ runId: run.id,
2300
+ agentId: run.agentId,
2301
+ ts,
2302
+ stream,
2303
+ chunk: payloadChunk,
2304
+ truncated: payloadChunk.length !== sanitizedChunk.length,
2305
+ },
2306
+ });
2307
+ if (stream === "stdout") {
2308
+ stdoutTranscriptBuffer = appendTranscriptEntriesFromChunk({
2309
+ buffer: stdoutTranscriptBuffer,
2310
+ chunk: sanitizedChunk,
2311
+ transcript: executionTranscript,
2312
+ parser: stdoutTranscriptParser,
2313
+ kind: "stdout",
2314
+ });
2315
+ return;
2316
+ }
2317
+ stderrTranscriptBuffer = appendTranscriptEntriesFromChunk({
2318
+ buffer: stderrTranscriptBuffer,
2319
+ chunk: sanitizedChunk,
2320
+ transcript: executionTranscript,
2321
+ kind: "stderr",
2322
+ });
2323
+ };
2324
+ for (const warning of runtimeWorkspaceWarnings) {
2325
+ const logEntry = formatRuntimeWorkspaceWarningLog(warning);
2326
+ await onLog(logEntry.stream, logEntry.chunk);
2327
+ }
2328
+ const adapterEnv = Object.fromEntries(Object.entries(parseObject(resolvedConfig.env)).filter((entry) => typeof entry[0] === "string" && typeof entry[1] === "string"));
2329
+ const runtimeServices = await ensureRuntimeServicesForRun({
2330
+ db,
2331
+ runId: run.id,
2332
+ agent: {
2333
+ id: agent.id,
2334
+ name: agent.name,
2335
+ orgId: agent.orgId,
2336
+ },
2337
+ issue: issueRef,
2338
+ workspace: executionWorkspace,
2339
+ executionWorkspaceId: persistedExecutionWorkspace?.id ?? issueRef?.executionWorkspaceId ?? null,
2340
+ config: resolvedConfig,
2341
+ adapterEnv,
2342
+ onLog,
2343
+ });
2344
+ if (runtimeServices.length > 0) {
2345
+ context.rudderRuntimeServices = runtimeServices;
2346
+ context.rudderRuntimePrimaryUrl =
2347
+ runtimeServices.find((service) => readNonEmptyString(service.url))?.url ?? null;
2348
+ await db
2349
+ .update(heartbeatRuns)
2350
+ .set({
2351
+ contextSnapshot: context,
2352
+ updatedAt: new Date(),
2353
+ })
2354
+ .where(eq(heartbeatRuns.id, run.id));
2355
+ }
2356
+ if (issueId && (executionWorkspace.created || runtimeServices.some((service) => !service.reused))) {
2357
+ try {
2358
+ await issuesSvc.addComment(issueId, buildWorkspaceReadyComment({
2359
+ workspace: executionWorkspace,
2360
+ runtimeServices,
2361
+ }), { agentId: agent.id });
2362
+ }
2363
+ catch (err) {
2364
+ await onLog("stderr", `[rudder] Failed to post workspace-ready comment: ${err instanceof Error ? err.message : String(err)}\n`);
2365
+ }
2366
+ }
2367
+ const onAdapterMeta = async (meta) => {
2368
+ if (meta.env && secretKeys.size > 0) {
2369
+ for (const key of secretKeys) {
2370
+ if (key in meta.env)
2371
+ meta.env[key] = "***REDACTED***";
2372
+ }
2373
+ }
2374
+ heartbeatObservationContext.metadata = {
2375
+ ...(heartbeatObservationContext.metadata ?? {}),
2376
+ ...buildHeartbeatRuntimeTraceMetadata({
2377
+ runtimeConfig,
2378
+ runtimeSkills: runtimeSkillEntries,
2379
+ adapterMeta: meta,
2380
+ }),
2381
+ };
2382
+ updateExecutionObservation(observation, heartbeatObservationContext, {
2383
+ input: rootObservationInput,
2384
+ });
2385
+ await appendRunEvent(currentRun, seq++, {
2386
+ eventType: "adapter.invoke",
2387
+ stream: "system",
2388
+ level: "info",
2389
+ message: "adapter invocation",
2390
+ payload: buildHeartbeatAdapterInvokePayload({
2391
+ meta,
2392
+ runtimeSkills: runtimeSkillEntries,
2393
+ }),
2394
+ });
2395
+ };
2396
+ const authToken = adapter.supportsLocalAgentJwt
2397
+ ? createLocalAgentJwt(agent.id, agent.orgId, agent.agentRuntimeType, run.id)
2398
+ : null;
2399
+ if (adapter.supportsLocalAgentJwt && !authToken) {
2400
+ logger.warn({
2401
+ orgId: agent.orgId,
2402
+ agentId: agent.id,
2403
+ runId: run.id,
2404
+ agentRuntimeType: agent.agentRuntimeType,
2405
+ }, "local agent jwt secret missing or invalid; running without injected RUDDER_API_KEY");
2406
+ }
2407
+ const adapterResult = await executeAdapterWithModelFallbacks(adapter, {
2408
+ runId: run.id,
2409
+ agent,
2410
+ runtime: runtimeForAdapter,
2411
+ config: runtimeConfig,
2412
+ context,
2413
+ onLog,
2414
+ onMeta: onAdapterMeta,
2415
+ onSpawn: async (meta) => {
2416
+ await persistRunProcessMetadata(run.id, meta);
2417
+ },
2418
+ authToken: authToken ?? undefined,
2419
+ }, {
2420
+ resolveAdapter: findServerAdapter,
2421
+ createAuthToken: (agentRuntimeType) => createLocalAgentJwt(agent.id, agent.orgId, agentRuntimeType, run.id) ?? undefined,
2422
+ onAttemptStart: (_attempt, attemptAdapter) => {
2423
+ stdoutTranscriptParser = attemptAdapter.parseStdoutLine ?? null;
2424
+ },
2425
+ });
2426
+ const adapterManagedRuntimeServices = adapterResult.runtimeServices
2427
+ ? await persistAdapterManagedRuntimeServices({
2428
+ db,
2429
+ agentRuntimeType: agent.agentRuntimeType,
2430
+ runId: run.id,
2431
+ agent: {
2432
+ id: agent.id,
2433
+ name: agent.name,
2434
+ orgId: agent.orgId,
2435
+ },
2436
+ issue: issueRef,
2437
+ workspace: executionWorkspace,
2438
+ reports: adapterResult.runtimeServices,
2439
+ })
2440
+ : [];
2441
+ if (adapterManagedRuntimeServices.length > 0) {
2442
+ const combinedRuntimeServices = [
2443
+ ...runtimeServices,
2444
+ ...adapterManagedRuntimeServices,
2445
+ ];
2446
+ context.rudderRuntimeServices = combinedRuntimeServices;
2447
+ context.rudderRuntimePrimaryUrl =
2448
+ combinedRuntimeServices.find((service) => readNonEmptyString(service.url))?.url ?? null;
2449
+ await db
2450
+ .update(heartbeatRuns)
2451
+ .set({
2452
+ contextSnapshot: context,
2453
+ updatedAt: new Date(),
2454
+ })
2455
+ .where(eq(heartbeatRuns.id, run.id));
2456
+ if (issueId) {
2457
+ try {
2458
+ await issuesSvc.addComment(issueId, buildWorkspaceReadyComment({
2459
+ workspace: executionWorkspace,
2460
+ runtimeServices: adapterManagedRuntimeServices,
2461
+ }), { agentId: agent.id });
2462
+ }
2463
+ catch (err) {
2464
+ await onLog("stderr", `[rudder] Failed to post adapter-managed runtime comment: ${err instanceof Error ? err.message : String(err)}\n`);
2465
+ }
2466
+ }
2467
+ }
2468
+ const nextSessionState = resolveNextSessionState({
2469
+ codec: sessionCodec,
2470
+ adapterResult,
2471
+ previousParams: previousSessionParams,
2472
+ previousDisplayId: runtimeForAdapter.sessionDisplayId,
2473
+ previousLegacySessionId: runtimeForAdapter.sessionId,
2474
+ });
2475
+ const rawUsage = normalizeUsageTotals(adapterResult.usage);
2476
+ const sessionUsageResolution = await resolveNormalizedUsageForSession({
2477
+ agentId: agent.id,
2478
+ runId: run.id,
2479
+ sessionId: nextSessionState.displayId ?? nextSessionState.legacySessionId,
2480
+ rawUsage,
2481
+ });
2482
+ const normalizedUsage = sessionUsageResolution.normalizedUsage;
2483
+ let outcome;
2484
+ const latestRun = await getRun(run.id);
2485
+ if (latestRun?.status === "cancelled") {
2486
+ outcome = "cancelled";
2487
+ }
2488
+ else if (adapterResult.timedOut) {
2489
+ outcome = "timed_out";
2490
+ }
2491
+ else if ((adapterResult.exitCode ?? 0) === 0 && !adapterResult.errorMessage) {
2492
+ outcome = "succeeded";
2493
+ }
2494
+ else {
2495
+ outcome = "failed";
2496
+ }
2497
+ let logSummary = null;
2498
+ if (handle) {
2499
+ logSummary = await runLogStore.finalize(handle);
2500
+ }
2501
+ const status = outcome === "succeeded"
2502
+ ? "succeeded"
2503
+ : outcome === "cancelled"
2504
+ ? "cancelled"
2505
+ : outcome === "timed_out"
2506
+ ? "timed_out"
2507
+ : "failed";
2508
+ heartbeatObservationContext.status = status;
2509
+ finalObservationStatus = status;
2510
+ finalObservationSessionId = nextSessionState.displayId ?? nextSessionState.legacySessionId ?? finalObservationSessionId;
2511
+ const adapterResultSummary = summarizeHeartbeatRunResultJson(adapterResult.resultJson);
2512
+ transcriptFallbackResult = {
2513
+ ts: new Date().toISOString(),
2514
+ model: readNonEmptyString(adapterResult.model),
2515
+ output: readNonEmptyString(adapterResult.summary)
2516
+ ?? readNonEmptyString(adapterResultSummary?.result)
2517
+ ?? readNonEmptyString(adapterResultSummary?.summary)
2518
+ ?? readNonEmptyString(adapterResultSummary?.message)
2519
+ ?? null,
2520
+ usage: adapterResult.usage ?? null,
2521
+ costUsd: typeof adapterResult.costUsd === "number" ? adapterResult.costUsd : null,
2522
+ subtype: status,
2523
+ isError: outcome !== "succeeded",
2524
+ errors: adapterResult.errorMessage ? [adapterResult.errorMessage] : [],
2525
+ };
2526
+ const usageJson = normalizedUsage || adapterResult.costUsd != null
2527
+ ? {
2528
+ ...(normalizedUsage ?? {}),
2529
+ ...(rawUsage ? {
2530
+ rawInputTokens: rawUsage.inputTokens,
2531
+ rawCachedInputTokens: rawUsage.cachedInputTokens,
2532
+ rawOutputTokens: rawUsage.outputTokens,
2533
+ } : {}),
2534
+ ...(sessionUsageResolution.derivedFromSessionTotals ? { usageSource: "session_delta" } : {}),
2535
+ ...((nextSessionState.displayId ?? nextSessionState.legacySessionId)
2536
+ ? { persistedSessionId: nextSessionState.displayId ?? nextSessionState.legacySessionId }
2537
+ : {}),
2538
+ sessionReused: runtimeForAdapter.sessionId != null || runtimeForAdapter.sessionDisplayId != null,
2539
+ taskSessionReused: taskSessionForRun != null,
2540
+ freshSession: runtimeForAdapter.sessionId == null && runtimeForAdapter.sessionDisplayId == null,
2541
+ sessionRotated: sessionCompaction.rotate,
2542
+ sessionRotationReason: sessionCompaction.reason,
2543
+ provider: readNonEmptyString(adapterResult.provider) ?? "unknown",
2544
+ biller: resolveLedgerBiller(adapterResult),
2545
+ model: readNonEmptyString(adapterResult.model) ?? "unknown",
2546
+ ...(adapterResult.costUsd != null ? { costUsd: adapterResult.costUsd } : {}),
2547
+ billingType: normalizeLedgerBillingType(adapterResult.billingType),
2548
+ }
2549
+ : null;
2550
+ await setRunStatus(run.id, status, {
2551
+ finishedAt: new Date(),
2552
+ error: outcome === "succeeded"
2553
+ ? null
2554
+ : redactCurrentUserText(adapterResult.errorMessage ?? (outcome === "timed_out" ? "Timed out" : "Adapter failed"), currentUserRedactionOptions),
2555
+ errorCode: outcome === "timed_out"
2556
+ ? "timeout"
2557
+ : outcome === "cancelled"
2558
+ ? "cancelled"
2559
+ : outcome === "failed"
2560
+ ? (adapterResult.errorCode ?? "adapter_failed")
2561
+ : null,
2562
+ exitCode: adapterResult.exitCode,
2563
+ signal: adapterResult.signal,
2564
+ usageJson,
2565
+ resultJson: adapterResult.resultJson ?? null,
2566
+ sessionIdAfter: nextSessionState.displayId ?? nextSessionState.legacySessionId,
2567
+ stdoutExcerpt,
2568
+ stderrExcerpt,
2569
+ logBytes: logSummary?.bytes,
2570
+ logSha256: logSummary?.sha256,
2571
+ logCompressed: logSummary?.compressed ?? false,
2572
+ });
2573
+ await setWakeupStatus(run.wakeupRequestId, outcome === "succeeded" ? "completed" : status, {
2574
+ finishedAt: new Date(),
2575
+ error: adapterResult.errorMessage ?? null,
2576
+ });
2577
+ const finalizedRun = await getRun(run.id);
2578
+ if (finalizedRun) {
2579
+ await appendRunEvent(finalizedRun, seq++, {
2580
+ eventType: "lifecycle",
2581
+ stream: "system",
2582
+ level: outcome === "succeeded" ? "info" : "error",
2583
+ message: `run ${outcome}`,
2584
+ payload: {
2585
+ status,
2586
+ exitCode: adapterResult.exitCode,
2587
+ },
2588
+ });
2589
+ await releaseIssueExecutionAndPromote(finalizedRun);
2590
+ }
2591
+ if (finalizedRun) {
2592
+ await updateRuntimeState(agent, finalizedRun, adapterResult, {
2593
+ legacySessionId: nextSessionState.legacySessionId,
2594
+ }, normalizedUsage);
2595
+ if (taskKey) {
2596
+ if (adapterResult.clearSession || (!nextSessionState.params && !nextSessionState.displayId)) {
2597
+ await clearTaskSessions(agent.orgId, agent.id, {
2598
+ taskKey,
2599
+ agentRuntimeType: agent.agentRuntimeType,
2600
+ });
2601
+ }
2602
+ else {
2603
+ await upsertTaskSession({
2604
+ orgId: agent.orgId,
2605
+ agentId: agent.id,
2606
+ agentRuntimeType: agent.agentRuntimeType,
2607
+ taskKey,
2608
+ sessionParamsJson: nextSessionState.params,
2609
+ sessionDisplayId: nextSessionState.displayId,
2610
+ lastRunId: finalizedRun.id,
2611
+ lastError: outcome === "succeeded" ? null : (adapterResult.errorMessage ?? "run_failed"),
2612
+ });
2613
+ }
2614
+ }
2615
+ await emitHeartbeatLiveEval(finalizedRun.id);
2616
+ }
2617
+ await finalizeAgentStatus(agent.id, outcome);
2618
+ }
2619
+ catch (err) {
2620
+ const message = redactCurrentUserText(err instanceof Error ? err.message : "Unknown adapter failure", await getCurrentUserRedactionOptions());
2621
+ heartbeatObservationContext.status = "failed";
2622
+ finalObservationStatus = "failed";
2623
+ transcriptFallbackResult = {
2624
+ ts: new Date().toISOString(),
2625
+ output: message,
2626
+ subtype: "failed",
2627
+ isError: true,
2628
+ errors: [message],
2629
+ };
2630
+ logger.error({ err, runId }, "heartbeat execution failed");
2631
+ let logSummary = null;
2632
+ if (handle) {
2633
+ try {
2634
+ logSummary = await runLogStore.finalize(handle);
2635
+ }
2636
+ catch (finalizeErr) {
2637
+ logger.warn({ err: finalizeErr, runId }, "failed to finalize run log after error");
2638
+ }
2639
+ }
2640
+ const failedRun = await setRunStatus(run.id, "failed", {
2641
+ error: message,
2642
+ errorCode: "adapter_failed",
2643
+ finishedAt: new Date(),
2644
+ stdoutExcerpt,
2645
+ stderrExcerpt,
2646
+ logBytes: logSummary?.bytes,
2647
+ logSha256: logSummary?.sha256,
2648
+ logCompressed: logSummary?.compressed ?? false,
2649
+ });
2650
+ await setWakeupStatus(run.wakeupRequestId, "failed", {
2651
+ finishedAt: new Date(),
2652
+ error: message,
2653
+ });
2654
+ if (failedRun) {
2655
+ await appendRunEvent(failedRun, seq++, {
2656
+ eventType: "error",
2657
+ stream: "system",
2658
+ level: "error",
2659
+ message,
2660
+ });
2661
+ await releaseIssueExecutionAndPromote(failedRun);
2662
+ await updateRuntimeState(agent, failedRun, {
2663
+ exitCode: null,
2664
+ signal: null,
2665
+ timedOut: false,
2666
+ errorMessage: message,
2667
+ }, {
2668
+ legacySessionId: runtimeForAdapter.sessionId,
2669
+ });
2670
+ if (taskKey && (previousSessionParams || previousSessionDisplayId || taskSession)) {
2671
+ await upsertTaskSession({
2672
+ orgId: agent.orgId,
2673
+ agentId: agent.id,
2674
+ agentRuntimeType: agent.agentRuntimeType,
2675
+ taskKey,
2676
+ sessionParamsJson: previousSessionParams,
2677
+ sessionDisplayId: previousSessionDisplayId,
2678
+ lastRunId: failedRun.id,
2679
+ lastError: message,
2680
+ });
2681
+ }
2682
+ await emitHeartbeatLiveEval(failedRun.id);
2683
+ }
2684
+ await finalizeAgentStatus(agent.id, "failed");
2685
+ }
2686
+ finally {
2687
+ stdoutTranscriptBuffer = appendTranscriptEntriesFromChunk({
2688
+ buffer: stdoutTranscriptBuffer,
2689
+ chunk: "",
2690
+ transcript: executionTranscript,
2691
+ parser: stdoutTranscriptParser,
2692
+ finalize: true,
2693
+ kind: "stdout",
2694
+ });
2695
+ stderrTranscriptBuffer = appendTranscriptEntriesFromChunk({
2696
+ buffer: stderrTranscriptBuffer,
2697
+ chunk: "",
2698
+ transcript: executionTranscript,
2699
+ finalize: true,
2700
+ kind: "stderr",
2701
+ });
2702
+ try {
2703
+ const transcriptStats = emitExecutionTranscriptTree({
2704
+ context: heartbeatObservationContext,
2705
+ parentObservation: observation,
2706
+ transcript: executionTranscript,
2707
+ fallbackResult: transcriptFallbackResult,
2708
+ });
2709
+ finalObservationOutput = transcriptStats.finalOutput ?? transcriptFallbackResult?.output ?? null;
2710
+ finalObservationSessionId = transcriptStats.finalSessionId ?? finalObservationSessionId;
2711
+ }
2712
+ catch (error) {
2713
+ logger.warn({
2714
+ runId: run.id,
2715
+ err: error instanceof Error ? error.message : String(error),
2716
+ }, "Failed to export heartbeat transcript tree to Langfuse");
2717
+ }
2718
+ updateExecutionObservation(observation, heartbeatObservationContext, {
2719
+ input: rootObservationInput,
2720
+ output: finalObservationOutput,
2721
+ level: finalObservationStatus === "failed" || finalObservationStatus === "timed_out" ? "ERROR" : "DEFAULT",
2722
+ statusMessage: finalObservationStatus ?? undefined,
2723
+ });
2724
+ updateExecutionTraceIO(observation, {
2725
+ input: rootObservationInput,
2726
+ output: finalObservationOutput,
2727
+ });
2728
+ updateExecutionTraceSession(observation, finalObservationSessionId);
2729
+ }
2730
+ });
2731
+ }
2732
+ catch (outerErr) {
2733
+ // Setup code before adapter.execute threw (e.g. ensureRuntimeState, resolveWorkspaceForRun).
2734
+ // The inner catch did not fire, so we must record the failure here.
2735
+ const message = outerErr instanceof Error ? outerErr.message : "Unknown setup failure";
2736
+ logger.error({ err: outerErr, runId }, "heartbeat execution setup failed");
2737
+ await setRunStatus(runId, "failed", {
2738
+ error: message,
2739
+ errorCode: "adapter_failed",
2740
+ finishedAt: new Date(),
2741
+ }).catch(() => undefined);
2742
+ await setWakeupStatus(run.wakeupRequestId, "failed", {
2743
+ finishedAt: new Date(),
2744
+ error: message,
2745
+ }).catch(() => undefined);
2746
+ const failedRun = await getRun(runId).catch(() => null);
2747
+ if (failedRun) {
2748
+ // Emit a run-log event so the failure is visible in the run timeline,
2749
+ // consistent with what the inner catch block does for adapter failures.
2750
+ await appendRunEvent(failedRun, 1, {
2751
+ eventType: "error",
2752
+ stream: "system",
2753
+ level: "error",
2754
+ message,
2755
+ }).catch(() => undefined);
2756
+ await emitHeartbeatLiveEval(failedRun.id).catch(() => undefined);
2757
+ await releaseIssueExecutionAndPromote(failedRun).catch(() => undefined);
2758
+ }
2759
+ // Ensure the agent is not left stuck in "running" if the inner catch handler's
2760
+ // DB calls threw (e.g. a transient DB error in finalizeAgentStatus).
2761
+ await finalizeAgentStatus(run.agentId, "failed").catch(() => undefined);
2762
+ }
2763
+ finally {
2764
+ await releaseRuntimeServicesForRun(run.id).catch(() => undefined);
2765
+ activeRunExecutions.delete(run.id);
2766
+ await startNextQueuedRunForAgent(run.agentId);
2767
+ }
2768
+ }
2769
+ async function releaseIssueExecutionAndPromote(run) {
2770
+ const outcome = await db.transaction(async (tx) => {
2771
+ await tx.execute(sql `select id from issues where org_id = ${run.orgId} and execution_run_id = ${run.id} for update`);
2772
+ const issue = await tx
2773
+ .select({
2774
+ id: issues.id,
2775
+ orgId: issues.orgId,
2776
+ title: issues.title,
2777
+ description: issues.description,
2778
+ status: issues.status,
2779
+ priority: issues.priority,
2780
+ projectId: issues.projectId,
2781
+ assigneeAgentId: issues.assigneeAgentId,
2782
+ })
2783
+ .from(issues)
2784
+ .where(and(eq(issues.orgId, run.orgId), eq(issues.executionRunId, run.id)))
2785
+ .then((rows) => rows[0] ?? null);
2786
+ if (!issue)
2787
+ return { promotedRun: null, passiveClosure: null };
2788
+ const passiveClosure = await evaluatePassiveIssueClosureForLockedIssue({
2789
+ tx,
2790
+ run,
2791
+ issue,
2792
+ now: new Date(),
2793
+ });
2794
+ if (passiveClosure.kind === "queued") {
2795
+ return { promotedRun: passiveClosure.run, passiveClosure };
2796
+ }
2797
+ await tx
2798
+ .update(issues)
2799
+ .set({
2800
+ executionRunId: null,
2801
+ executionAgentNameKey: null,
2802
+ executionLockedAt: null,
2803
+ updatedAt: new Date(),
2804
+ })
2805
+ .where(eq(issues.id, issue.id));
2806
+ while (true) {
2807
+ const deferred = await tx
2808
+ .select()
2809
+ .from(agentWakeupRequests)
2810
+ .where(and(eq(agentWakeupRequests.orgId, issue.orgId), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql `${agentWakeupRequests.payload} ->> 'issueId' = ${issue.id}`))
2811
+ .orderBy(asc(agentWakeupRequests.requestedAt))
2812
+ .limit(1)
2813
+ .then((rows) => rows[0] ?? null);
2814
+ if (!deferred)
2815
+ return { promotedRun: null, passiveClosure };
2816
+ const deferredAgent = await tx
2817
+ .select()
2818
+ .from(agents)
2819
+ .where(eq(agents.id, deferred.agentId))
2820
+ .then((rows) => rows[0] ?? null);
2821
+ if (!deferredAgent ||
2822
+ deferredAgent.orgId !== issue.orgId ||
2823
+ deferredAgent.status === "paused" ||
2824
+ deferredAgent.status === "terminated" ||
2825
+ deferredAgent.status === "pending_approval") {
2826
+ await tx
2827
+ .update(agentWakeupRequests)
2828
+ .set({
2829
+ status: "failed",
2830
+ finishedAt: new Date(),
2831
+ error: "Deferred wake could not be promoted: agent is not invokable",
2832
+ updatedAt: new Date(),
2833
+ })
2834
+ .where(eq(agentWakeupRequests.id, deferred.id));
2835
+ continue;
2836
+ }
2837
+ const deferredPayload = parseObject(deferred.payload);
2838
+ const deferredContextSeed = parseObject(deferredPayload[DEFERRED_WAKE_CONTEXT_KEY]);
2839
+ const promotedContextSeed = { ...deferredContextSeed };
2840
+ const promotedReason = readNonEmptyString(deferred.reason) ?? "issue_execution_promoted";
2841
+ const promotedSource = readNonEmptyString(deferred.source) ?? "automation";
2842
+ const promotedTriggerDetail = readNonEmptyString(deferred.triggerDetail) ?? null;
2843
+ const promotedPayload = deferredPayload;
2844
+ delete promotedPayload[DEFERRED_WAKE_CONTEXT_KEY];
2845
+ const { contextSnapshot: promotedContextSnapshot, taskKey: promotedTaskKey, } = enrichWakeContextSnapshot({
2846
+ contextSnapshot: promotedContextSeed,
2847
+ reason: promotedReason,
2848
+ source: promotedSource,
2849
+ triggerDetail: promotedTriggerDetail,
2850
+ payload: promotedPayload,
2851
+ });
2852
+ const sessionBefore = readNonEmptyString(promotedContextSnapshot.resumeSessionDisplayId) ??
2853
+ await resolveSessionBeforeForWakeup(deferredAgent, promotedTaskKey);
2854
+ const now = new Date();
2855
+ const newRun = await tx
2856
+ .insert(heartbeatRuns)
2857
+ .values({
2858
+ orgId: deferredAgent.orgId,
2859
+ agentId: deferredAgent.id,
2860
+ invocationSource: promotedSource,
2861
+ triggerDetail: promotedTriggerDetail,
2862
+ status: "queued",
2863
+ wakeupRequestId: deferred.id,
2864
+ contextSnapshot: promotedContextSnapshot,
2865
+ sessionIdBefore: sessionBefore,
2866
+ })
2867
+ .returning()
2868
+ .then((rows) => rows[0]);
2869
+ await tx
2870
+ .update(agentWakeupRequests)
2871
+ .set({
2872
+ status: "queued",
2873
+ reason: "issue_execution_promoted",
2874
+ runId: newRun.id,
2875
+ claimedAt: null,
2876
+ finishedAt: null,
2877
+ error: null,
2878
+ updatedAt: now,
2879
+ })
2880
+ .where(eq(agentWakeupRequests.id, deferred.id));
2881
+ await tx
2882
+ .update(issues)
2883
+ .set({
2884
+ executionRunId: newRun.id,
2885
+ executionAgentNameKey: normalizeAgentNameKey(deferredAgent.name),
2886
+ executionLockedAt: now,
2887
+ updatedAt: now,
2888
+ })
2889
+ .where(eq(issues.id, issue.id));
2890
+ return { promotedRun: newRun, passiveClosure };
2891
+ }
2892
+ });
2893
+ const passiveClosure = outcome.passiveClosure;
2894
+ if (passiveClosure?.kind === "queued") {
2895
+ await appendRunEvent(run, await nextRunEventSeq(run.id), {
2896
+ eventType: "issue.passive_followup_queued",
2897
+ stream: "system",
2898
+ level: "warn",
2899
+ message: `Queued passive issue follow-up ${passiveClosure.run.id}`,
2900
+ payload: {
2901
+ issueId: passiveClosure.issue.id,
2902
+ followupRunId: passiveClosure.run.id,
2903
+ originRunId: passiveClosure.originRunId,
2904
+ previousRunId: passiveClosure.previousRunId,
2905
+ attempt: passiveClosure.attempt,
2906
+ maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
2907
+ reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
2908
+ requestedAt: passiveClosure.requestedAt.toISOString(),
2909
+ },
2910
+ });
2911
+ await appendRunEvent(passiveClosure.run, await nextRunEventSeq(passiveClosure.run.id), {
2912
+ eventType: "issue.passive_followup_queued",
2913
+ stream: "system",
2914
+ level: "warn",
2915
+ message: `Passive follow-up queued because run ${run.id} ended without issue close-out`,
2916
+ payload: {
2917
+ issueId: passiveClosure.issue.id,
2918
+ originRunId: passiveClosure.originRunId,
2919
+ previousRunId: passiveClosure.previousRunId,
2920
+ attempt: passiveClosure.attempt,
2921
+ maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
2922
+ reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
2923
+ requestedAt: passiveClosure.requestedAt.toISOString(),
2924
+ },
2925
+ });
2926
+ await logActivity(db, {
2927
+ orgId: passiveClosure.issue.orgId,
2928
+ actorType: "system",
2929
+ actorId: "issue_closure_governance",
2930
+ action: "issue.passive_followup_queued",
2931
+ entityType: "issue",
2932
+ entityId: passiveClosure.issue.id,
2933
+ agentId: run.agentId,
2934
+ runId: run.id,
2935
+ details: {
2936
+ issueId: passiveClosure.issue.id,
2937
+ issueTitle: passiveClosure.issue.title,
2938
+ followupRunId: passiveClosure.run.id,
2939
+ originRunId: passiveClosure.originRunId,
2940
+ previousRunId: passiveClosure.previousRunId,
2941
+ attempt: passiveClosure.attempt,
2942
+ maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
2943
+ reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
2944
+ requestedAt: passiveClosure.requestedAt.toISOString(),
2945
+ },
2946
+ });
2947
+ }
2948
+ else if (passiveClosure?.kind === "operator_review") {
2949
+ await appendRunEvent(run, await nextRunEventSeq(run.id), {
2950
+ eventType: "issue.closure_needs_operator_review",
2951
+ stream: "system",
2952
+ level: "warn",
2953
+ message: "Passive issue follow-up stopped and needs operator review",
2954
+ payload: {
2955
+ issueId: passiveClosure.issue.id,
2956
+ originRunId: passiveClosure.originRunId,
2957
+ previousRunId: passiveClosure.previousRunId,
2958
+ attempts: passiveClosure.attempts,
2959
+ maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
2960
+ reason: passiveClosure.reason,
2961
+ },
2962
+ });
2963
+ await logActivity(db, {
2964
+ orgId: passiveClosure.issue.orgId,
2965
+ actorType: "system",
2966
+ actorId: "issue_closure_governance",
2967
+ action: "issue.closure_needs_operator_review",
2968
+ entityType: "issue",
2969
+ entityId: passiveClosure.issue.id,
2970
+ agentId: run.agentId,
2971
+ runId: run.id,
2972
+ details: {
2973
+ issueId: passiveClosure.issue.id,
2974
+ issueTitle: passiveClosure.issue.title,
2975
+ originRunId: passiveClosure.originRunId,
2976
+ previousRunId: passiveClosure.previousRunId,
2977
+ attempts: passiveClosure.attempts,
2978
+ maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
2979
+ reason: passiveClosure.reason,
2980
+ },
2981
+ });
2982
+ }
2983
+ const promotedRun = outcome.promotedRun;
2984
+ if (!promotedRun)
2985
+ return;
2986
+ publishLiveEvent({
2987
+ orgId: promotedRun.orgId,
2988
+ type: "heartbeat.run.queued",
2989
+ payload: {
2990
+ runId: promotedRun.id,
2991
+ agentId: promotedRun.agentId,
2992
+ invocationSource: promotedRun.invocationSource,
2993
+ triggerDetail: promotedRun.triggerDetail,
2994
+ wakeupRequestId: promotedRun.wakeupRequestId,
2995
+ },
2996
+ });
2997
+ await startNextQueuedRunForAgent(promotedRun.agentId);
2998
+ }
2999
+ async function enqueueWakeup(agentId, opts = {}) {
3000
+ const source = opts.source ?? "on_demand";
3001
+ const triggerDetail = opts.triggerDetail ?? null;
3002
+ const contextSnapshot = { ...(opts.contextSnapshot ?? {}) };
3003
+ const reason = opts.reason ?? null;
3004
+ const payload = opts.payload ?? null;
3005
+ const existingWakeupRequestId = readNonEmptyString(opts.existingWakeupRequestId);
3006
+ const { contextSnapshot: enrichedContextSnapshot, issueIdFromPayload, taskKey, wakeCommentId, } = enrichWakeContextSnapshot({
3007
+ contextSnapshot,
3008
+ reason,
3009
+ source,
3010
+ triggerDetail,
3011
+ payload,
3012
+ });
3013
+ let issueId = readNonEmptyString(enrichedContextSnapshot.issueId) ?? issueIdFromPayload;
3014
+ const agent = await getAgent(agentId);
3015
+ if (!agent)
3016
+ throw notFound("Agent not found");
3017
+ const explicitResumeSession = await resolveExplicitResumeSessionOverride(agent, payload, taskKey);
3018
+ if (explicitResumeSession) {
3019
+ enrichedContextSnapshot.resumeFromRunId = explicitResumeSession.resumeFromRunId;
3020
+ enrichedContextSnapshot.resumeSessionDisplayId = explicitResumeSession.sessionDisplayId;
3021
+ enrichedContextSnapshot.resumeSessionParams = explicitResumeSession.sessionParams;
3022
+ if (!readNonEmptyString(enrichedContextSnapshot.issueId) && explicitResumeSession.issueId) {
3023
+ enrichedContextSnapshot.issueId = explicitResumeSession.issueId;
3024
+ }
3025
+ if (!readNonEmptyString(enrichedContextSnapshot.taskId) && explicitResumeSession.taskId) {
3026
+ enrichedContextSnapshot.taskId = explicitResumeSession.taskId;
3027
+ }
3028
+ if (!readNonEmptyString(enrichedContextSnapshot.taskKey) && explicitResumeSession.taskKey) {
3029
+ enrichedContextSnapshot.taskKey = explicitResumeSession.taskKey;
3030
+ }
3031
+ issueId = readNonEmptyString(enrichedContextSnapshot.issueId) ?? issueId;
3032
+ }
3033
+ await hydrateWakeContextSnapshot(db, agent.orgId, enrichedContextSnapshot);
3034
+ const effectiveTaskKey = readNonEmptyString(enrichedContextSnapshot.taskKey) ?? taskKey;
3035
+ const sessionBefore = explicitResumeSession?.sessionDisplayId ??
3036
+ await resolveSessionBeforeForWakeup(agent, effectiveTaskKey);
3037
+ const writeSkippedRequest = async (skipReason) => {
3038
+ if (existingWakeupRequestId) {
3039
+ await setWakeupStatus(existingWakeupRequestId, "skipped", {
3040
+ reason: skipReason,
3041
+ finishedAt: new Date(),
3042
+ runId: null,
3043
+ claimedAt: null,
3044
+ error: null,
3045
+ });
3046
+ return;
3047
+ }
3048
+ await db.insert(agentWakeupRequests).values({
3049
+ orgId: agent.orgId,
3050
+ agentId,
3051
+ source,
3052
+ triggerDetail,
3053
+ reason: skipReason,
3054
+ payload,
3055
+ status: "skipped",
3056
+ requestedByActorType: opts.requestedByActorType ?? null,
3057
+ requestedByActorId: opts.requestedByActorId ?? null,
3058
+ idempotencyKey: opts.idempotencyKey ?? null,
3059
+ finishedAt: new Date(),
3060
+ });
3061
+ };
3062
+ let projectId = readNonEmptyString(enrichedContextSnapshot.projectId);
3063
+ if (!projectId && issueId) {
3064
+ projectId = await db
3065
+ .select({ projectId: issues.projectId })
3066
+ .from(issues)
3067
+ .where(and(eq(issues.id, issueId), eq(issues.orgId, agent.orgId)))
3068
+ .then((rows) => rows[0]?.projectId ?? null);
3069
+ }
3070
+ const budgetBlock = await budgets.getInvocationBlock(agent.orgId, agentId, {
3071
+ issueId,
3072
+ projectId,
3073
+ });
3074
+ if (budgetBlock) {
3075
+ await writeSkippedRequest("budget.blocked");
3076
+ throw conflict(budgetBlock.reason, {
3077
+ scopeType: budgetBlock.scopeType,
3078
+ scopeId: budgetBlock.scopeId,
3079
+ });
3080
+ }
3081
+ if (agent.status === "paused") {
3082
+ const deferredPayload = buildDeferredWakePayload(payload, enrichedContextSnapshot, issueId);
3083
+ if (existingWakeupRequestId) {
3084
+ await setWakeupStatus(existingWakeupRequestId, "deferred_agent_paused", {
3085
+ reason,
3086
+ payload: deferredPayload,
3087
+ runId: null,
3088
+ claimedAt: null,
3089
+ finishedAt: null,
3090
+ error: null,
3091
+ });
3092
+ return null;
3093
+ }
3094
+ await db.transaction(async (tx) => {
3095
+ const deferredRows = await tx
3096
+ .select()
3097
+ .from(agentWakeupRequests)
3098
+ .where(and(eq(agentWakeupRequests.orgId, agent.orgId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_agent_paused"), sql `${agentWakeupRequests.runId} is null`))
3099
+ .orderBy(asc(agentWakeupRequests.requestedAt));
3100
+ const existingDeferred = deferredRows.find((candidate) => isSameTaskScope(deriveDeferredWakeTaskKey(candidate.payload), effectiveTaskKey));
3101
+ if (existingDeferred) {
3102
+ const mergedDeferredContext = mergeCoalescedContextSnapshot(readDeferredWakeContext(existingDeferred.payload), enrichedContextSnapshot);
3103
+ await updateWakeupRequestRecord(tx, existingDeferred.id, {
3104
+ payload: buildDeferredWakePayload({
3105
+ ...readDeferredWakePayload(existingDeferred.payload),
3106
+ ...(payload ?? {}),
3107
+ }, mergedDeferredContext, issueId),
3108
+ coalescedCount: (existingDeferred.coalescedCount ?? 0) + 1,
3109
+ error: null,
3110
+ finishedAt: null,
3111
+ claimedAt: null,
3112
+ runId: null,
3113
+ });
3114
+ return;
3115
+ }
3116
+ await insertWakeupRequestRecord(tx, {
3117
+ orgId: agent.orgId,
3118
+ agentId,
3119
+ source,
3120
+ triggerDetail,
3121
+ reason,
3122
+ payload: deferredPayload,
3123
+ status: "deferred_agent_paused",
3124
+ requestedByActorType: opts.requestedByActorType ?? null,
3125
+ requestedByActorId: opts.requestedByActorId ?? null,
3126
+ idempotencyKey: opts.idempotencyKey ?? null,
3127
+ });
3128
+ });
3129
+ return null;
3130
+ }
3131
+ if (agent.status === "terminated" || agent.status === "pending_approval") {
3132
+ throw conflict("Agent is not invokable in its current state", { status: agent.status });
3133
+ }
3134
+ const policy = parseHeartbeatPolicy(agent);
3135
+ if (source === "timer" && !policy.enabled) {
3136
+ await writeSkippedRequest("heartbeat.disabled");
3137
+ return null;
3138
+ }
3139
+ if (source !== "timer" && !policy.wakeOnDemand) {
3140
+ await writeSkippedRequest("heartbeat.wakeOnDemand.disabled");
3141
+ return null;
3142
+ }
3143
+ const bypassIssueExecutionLock = reason === "issue_comment_mentioned" ||
3144
+ readNonEmptyString(enrichedContextSnapshot.wakeReason) === "issue_comment_mentioned";
3145
+ if (issueId && !bypassIssueExecutionLock) {
3146
+ const agentNameKey = normalizeAgentNameKey(agent.name);
3147
+ const outcome = await db.transaction(async (tx) => {
3148
+ await tx.execute(sql `select id from issues where id = ${issueId} and org_id = ${agent.orgId} for update`);
3149
+ const issue = await tx
3150
+ .select({
3151
+ id: issues.id,
3152
+ orgId: issues.orgId,
3153
+ executionRunId: issues.executionRunId,
3154
+ executionAgentNameKey: issues.executionAgentNameKey,
3155
+ })
3156
+ .from(issues)
3157
+ .where(and(eq(issues.id, issueId), eq(issues.orgId, agent.orgId)))
3158
+ .then((rows) => rows[0] ?? null);
3159
+ if (!issue) {
3160
+ if (existingWakeupRequestId) {
3161
+ await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
3162
+ status: "skipped",
3163
+ reason: "issue_execution_issue_not_found",
3164
+ runId: null,
3165
+ claimedAt: null,
3166
+ finishedAt: new Date(),
3167
+ error: null,
3168
+ });
3169
+ }
3170
+ else {
3171
+ await insertWakeupRequestRecord(tx, {
3172
+ orgId: agent.orgId,
3173
+ agentId,
3174
+ source,
3175
+ triggerDetail,
3176
+ reason: "issue_execution_issue_not_found",
3177
+ payload,
3178
+ status: "skipped",
3179
+ requestedByActorType: opts.requestedByActorType ?? null,
3180
+ requestedByActorId: opts.requestedByActorId ?? null,
3181
+ idempotencyKey: opts.idempotencyKey ?? null,
3182
+ finishedAt: new Date(),
3183
+ });
3184
+ }
3185
+ return { kind: "skipped" };
3186
+ }
3187
+ let activeExecutionRun = issue.executionRunId
3188
+ ? await tx
3189
+ .select()
3190
+ .from(heartbeatRuns)
3191
+ .where(eq(heartbeatRuns.id, issue.executionRunId))
3192
+ .then((rows) => rows[0] ?? null)
3193
+ : null;
3194
+ if (activeExecutionRun && activeExecutionRun.status !== "queued" && activeExecutionRun.status !== "running") {
3195
+ activeExecutionRun = null;
3196
+ }
3197
+ if (!activeExecutionRun && issue.executionRunId) {
3198
+ await tx
3199
+ .update(issues)
3200
+ .set({
3201
+ executionRunId: null,
3202
+ executionAgentNameKey: null,
3203
+ executionLockedAt: null,
3204
+ updatedAt: new Date(),
3205
+ })
3206
+ .where(eq(issues.id, issue.id));
3207
+ }
3208
+ if (!activeExecutionRun) {
3209
+ const legacyRun = await tx
3210
+ .select()
3211
+ .from(heartbeatRuns)
3212
+ .where(and(eq(heartbeatRuns.orgId, issue.orgId), inArray(heartbeatRuns.status, ["queued", "running"]), sql `${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`))
3213
+ .orderBy(sql `case when ${heartbeatRuns.status} = 'running' then 0 else 1 end`, asc(heartbeatRuns.createdAt))
3214
+ .limit(1)
3215
+ .then((rows) => rows[0] ?? null);
3216
+ if (legacyRun) {
3217
+ activeExecutionRun = legacyRun;
3218
+ const legacyAgent = await tx
3219
+ .select({ name: agents.name })
3220
+ .from(agents)
3221
+ .where(eq(agents.id, legacyRun.agentId))
3222
+ .then((rows) => rows[0] ?? null);
3223
+ await tx
3224
+ .update(issues)
3225
+ .set({
3226
+ executionRunId: legacyRun.id,
3227
+ executionAgentNameKey: normalizeAgentNameKey(legacyAgent?.name),
3228
+ executionLockedAt: new Date(),
3229
+ updatedAt: new Date(),
3230
+ })
3231
+ .where(eq(issues.id, issue.id));
3232
+ }
3233
+ }
3234
+ if (activeExecutionRun) {
3235
+ const executionAgent = await tx
3236
+ .select({ name: agents.name })
3237
+ .from(agents)
3238
+ .where(eq(agents.id, activeExecutionRun.agentId))
3239
+ .then((rows) => rows[0] ?? null);
3240
+ const executionAgentNameKey = normalizeAgentNameKey(issue.executionAgentNameKey) ??
3241
+ normalizeAgentNameKey(executionAgent?.name);
3242
+ const isSameExecutionAgent = Boolean(executionAgentNameKey) && executionAgentNameKey === agentNameKey;
3243
+ const shouldQueueFollowupForCommentWake = Boolean(wakeCommentId) &&
3244
+ activeExecutionRun.status === "running" &&
3245
+ isSameExecutionAgent;
3246
+ if (isSameExecutionAgent && !shouldQueueFollowupForCommentWake) {
3247
+ const mergedContextSnapshot = mergeCoalescedContextSnapshot(activeExecutionRun.contextSnapshot, enrichedContextSnapshot);
3248
+ const mergedRun = await tx
3249
+ .update(heartbeatRuns)
3250
+ .set({
3251
+ contextSnapshot: mergedContextSnapshot,
3252
+ updatedAt: new Date(),
3253
+ })
3254
+ .where(eq(heartbeatRuns.id, activeExecutionRun.id))
3255
+ .returning()
3256
+ .then((rows) => rows[0] ?? activeExecutionRun);
3257
+ if (existingWakeupRequestId) {
3258
+ await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
3259
+ status: "coalesced",
3260
+ reason: "issue_execution_same_name",
3261
+ runId: mergedRun.id,
3262
+ claimedAt: null,
3263
+ finishedAt: new Date(),
3264
+ error: null,
3265
+ });
3266
+ }
3267
+ else {
3268
+ await insertWakeupRequestRecord(tx, {
3269
+ orgId: agent.orgId,
3270
+ agentId,
3271
+ source,
3272
+ triggerDetail,
3273
+ reason: "issue_execution_same_name",
3274
+ payload,
3275
+ status: "coalesced",
3276
+ coalescedCount: 1,
3277
+ requestedByActorType: opts.requestedByActorType ?? null,
3278
+ requestedByActorId: opts.requestedByActorId ?? null,
3279
+ idempotencyKey: opts.idempotencyKey ?? null,
3280
+ runId: mergedRun.id,
3281
+ finishedAt: new Date(),
3282
+ });
3283
+ }
3284
+ return { kind: "coalesced", run: mergedRun };
3285
+ }
3286
+ const deferredPayload = buildDeferredWakePayload(payload, enrichedContextSnapshot, issueId);
3287
+ const existingDeferred = await tx
3288
+ .select()
3289
+ .from(agentWakeupRequests)
3290
+ .where(and(eq(agentWakeupRequests.orgId, agent.orgId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql `${agentWakeupRequests.payload} ->> 'issueId' = ${issue.id}`))
3291
+ .orderBy(asc(agentWakeupRequests.requestedAt))
3292
+ .limit(1)
3293
+ .then((rows) => rows[0] ?? null);
3294
+ if (existingDeferred) {
3295
+ const mergedDeferredContext = mergeCoalescedContextSnapshot(readDeferredWakeContext(existingDeferred.payload), enrichedContextSnapshot);
3296
+ const mergedDeferredPayload = buildDeferredWakePayload({
3297
+ ...readDeferredWakePayload(existingDeferred.payload),
3298
+ ...(payload ?? {}),
3299
+ }, mergedDeferredContext, issueId);
3300
+ if (existingWakeupRequestId && existingDeferred.id !== existingWakeupRequestId) {
3301
+ await updateWakeupRequestRecord(tx, existingDeferred.id, {
3302
+ payload: mergedDeferredPayload,
3303
+ coalescedCount: (existingDeferred.coalescedCount ?? 0) + 1,
3304
+ });
3305
+ await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
3306
+ status: "coalesced",
3307
+ reason: "issue_execution_deferred",
3308
+ runId: null,
3309
+ claimedAt: null,
3310
+ finishedAt: new Date(),
3311
+ error: null,
3312
+ });
3313
+ }
3314
+ else {
3315
+ await updateWakeupRequestRecord(tx, existingDeferred.id, {
3316
+ payload: mergedDeferredPayload,
3317
+ coalescedCount: (existingDeferred.coalescedCount ?? 0) + 1,
3318
+ status: "deferred_issue_execution",
3319
+ reason: "issue_execution_deferred",
3320
+ runId: null,
3321
+ claimedAt: null,
3322
+ finishedAt: null,
3323
+ error: null,
3324
+ });
3325
+ }
3326
+ return { kind: "deferred" };
3327
+ }
3328
+ if (existingWakeupRequestId) {
3329
+ await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
3330
+ status: "deferred_issue_execution",
3331
+ reason: "issue_execution_deferred",
3332
+ payload: deferredPayload,
3333
+ runId: null,
3334
+ claimedAt: null,
3335
+ finishedAt: null,
3336
+ error: null,
3337
+ });
3338
+ }
3339
+ else {
3340
+ await insertWakeupRequestRecord(tx, {
3341
+ orgId: agent.orgId,
3342
+ agentId,
3343
+ source,
3344
+ triggerDetail,
3345
+ reason: "issue_execution_deferred",
3346
+ payload: deferredPayload,
3347
+ status: "deferred_issue_execution",
3348
+ requestedByActorType: opts.requestedByActorType ?? null,
3349
+ requestedByActorId: opts.requestedByActorId ?? null,
3350
+ idempotencyKey: opts.idempotencyKey ?? null,
3351
+ });
3352
+ }
3353
+ return { kind: "deferred" };
3354
+ }
3355
+ const wakeupRequest = existingWakeupRequestId
3356
+ ? await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
3357
+ status: "queued",
3358
+ runId: null,
3359
+ claimedAt: null,
3360
+ finishedAt: null,
3361
+ error: null,
3362
+ })
3363
+ : await insertWakeupRequestRecord(tx, {
3364
+ orgId: agent.orgId,
3365
+ agentId,
3366
+ source,
3367
+ triggerDetail,
3368
+ reason,
3369
+ payload,
3370
+ status: "queued",
3371
+ requestedByActorType: opts.requestedByActorType ?? null,
3372
+ requestedByActorId: opts.requestedByActorId ?? null,
3373
+ idempotencyKey: opts.idempotencyKey ?? null,
3374
+ });
3375
+ const newRun = await tx
3376
+ .insert(heartbeatRuns)
3377
+ .values({
3378
+ orgId: agent.orgId,
3379
+ agentId,
3380
+ invocationSource: source,
3381
+ triggerDetail,
3382
+ status: "queued",
3383
+ wakeupRequestId: wakeupRequest.id,
3384
+ contextSnapshot: enrichedContextSnapshot,
3385
+ sessionIdBefore: sessionBefore,
3386
+ })
3387
+ .returning()
3388
+ .then((rows) => rows[0]);
3389
+ await updateWakeupRequestRecord(tx, wakeupRequest.id, {
3390
+ runId: newRun.id,
3391
+ status: "queued",
3392
+ claimedAt: null,
3393
+ finishedAt: null,
3394
+ error: null,
3395
+ });
3396
+ await tx
3397
+ .update(issues)
3398
+ .set({
3399
+ executionRunId: newRun.id,
3400
+ executionAgentNameKey: agentNameKey,
3401
+ executionLockedAt: new Date(),
3402
+ updatedAt: new Date(),
3403
+ })
3404
+ .where(eq(issues.id, issue.id));
3405
+ return { kind: "queued", run: newRun };
3406
+ });
3407
+ if (outcome.kind === "deferred" || outcome.kind === "skipped")
3408
+ return null;
3409
+ if (outcome.kind === "coalesced")
3410
+ return outcome.run;
3411
+ const newRun = outcome.run;
3412
+ publishLiveEvent({
3413
+ orgId: newRun.orgId,
3414
+ type: "heartbeat.run.queued",
3415
+ payload: {
3416
+ runId: newRun.id,
3417
+ agentId: newRun.agentId,
3418
+ invocationSource: newRun.invocationSource,
3419
+ triggerDetail: newRun.triggerDetail,
3420
+ wakeupRequestId: newRun.wakeupRequestId,
3421
+ },
3422
+ });
3423
+ await startNextQueuedRunForAgent(agent.id);
3424
+ return newRun;
3425
+ }
3426
+ const activeRuns = await db
3427
+ .select()
3428
+ .from(heartbeatRuns)
3429
+ .where(and(eq(heartbeatRuns.agentId, agentId), inArray(heartbeatRuns.status, ["queued", "running"])))
3430
+ .orderBy(desc(heartbeatRuns.createdAt));
3431
+ const sameScopeQueuedRun = activeRuns.find((candidate) => candidate.status === "queued" && isSameTaskScope(runTaskKey(candidate), taskKey));
3432
+ const sameScopeRunningRun = activeRuns.find((candidate) => candidate.status === "running" && isSameTaskScope(runTaskKey(candidate), taskKey));
3433
+ const shouldQueueFollowupForCommentWake = Boolean(wakeCommentId) && Boolean(sameScopeRunningRun) && !sameScopeQueuedRun;
3434
+ const coalescedTargetRun = sameScopeQueuedRun ??
3435
+ (shouldQueueFollowupForCommentWake ? null : sameScopeRunningRun ?? null);
3436
+ if (coalescedTargetRun) {
3437
+ const mergedContextSnapshot = mergeCoalescedContextSnapshot(coalescedTargetRun.contextSnapshot, contextSnapshot);
3438
+ const mergedRun = await db
3439
+ .update(heartbeatRuns)
3440
+ .set({
3441
+ contextSnapshot: mergedContextSnapshot,
3442
+ updatedAt: new Date(),
3443
+ })
3444
+ .where(eq(heartbeatRuns.id, coalescedTargetRun.id))
3445
+ .returning()
3446
+ .then((rows) => rows[0] ?? coalescedTargetRun);
3447
+ if (existingWakeupRequestId) {
3448
+ await setWakeupStatus(existingWakeupRequestId, "coalesced", {
3449
+ runId: mergedRun.id,
3450
+ claimedAt: null,
3451
+ finishedAt: new Date(),
3452
+ error: null,
3453
+ });
3454
+ }
3455
+ else {
3456
+ await db.insert(agentWakeupRequests).values({
3457
+ orgId: agent.orgId,
3458
+ agentId,
3459
+ source,
3460
+ triggerDetail,
3461
+ reason,
3462
+ payload,
3463
+ status: "coalesced",
3464
+ coalescedCount: 1,
3465
+ requestedByActorType: opts.requestedByActorType ?? null,
3466
+ requestedByActorId: opts.requestedByActorId ?? null,
3467
+ idempotencyKey: opts.idempotencyKey ?? null,
3468
+ runId: mergedRun.id,
3469
+ finishedAt: new Date(),
3470
+ });
3471
+ }
3472
+ return mergedRun;
3473
+ }
3474
+ const wakeupRequest = existingWakeupRequestId
3475
+ ? await updateWakeupRequestRecord(db, existingWakeupRequestId, {
3476
+ status: "queued",
3477
+ runId: null,
3478
+ claimedAt: null,
3479
+ finishedAt: null,
3480
+ error: null,
3481
+ })
3482
+ : await insertWakeupRequestRecord(db, {
3483
+ orgId: agent.orgId,
3484
+ agentId,
3485
+ source,
3486
+ triggerDetail,
3487
+ reason,
3488
+ payload,
3489
+ status: "queued",
3490
+ requestedByActorType: opts.requestedByActorType ?? null,
3491
+ requestedByActorId: opts.requestedByActorId ?? null,
3492
+ idempotencyKey: opts.idempotencyKey ?? null,
3493
+ });
3494
+ const newRun = await db
3495
+ .insert(heartbeatRuns)
3496
+ .values({
3497
+ orgId: agent.orgId,
3498
+ agentId,
3499
+ invocationSource: source,
3500
+ triggerDetail,
3501
+ status: "queued",
3502
+ wakeupRequestId: wakeupRequest.id,
3503
+ contextSnapshot: enrichedContextSnapshot,
3504
+ sessionIdBefore: sessionBefore,
3505
+ })
3506
+ .returning()
3507
+ .then((rows) => rows[0]);
3508
+ await updateWakeupRequestRecord(db, wakeupRequest.id, {
3509
+ status: "queued",
3510
+ runId: newRun.id,
3511
+ claimedAt: null,
3512
+ finishedAt: null,
3513
+ error: null,
3514
+ });
3515
+ publishLiveEvent({
3516
+ orgId: newRun.orgId,
3517
+ type: "heartbeat.run.queued",
3518
+ payload: {
3519
+ runId: newRun.id,
3520
+ agentId: newRun.agentId,
3521
+ invocationSource: newRun.invocationSource,
3522
+ triggerDetail: newRun.triggerDetail,
3523
+ wakeupRequestId: newRun.wakeupRequestId,
3524
+ },
3525
+ });
3526
+ await startNextQueuedRunForAgent(agent.id);
3527
+ return newRun;
3528
+ }
3529
+ async function resumeDeferredWakeupsForAgent(agentId) {
3530
+ const agent = await getAgent(agentId);
3531
+ if (!agent)
3532
+ throw notFound("Agent not found");
3533
+ const replayedRequestIds = [];
3534
+ while (true) {
3535
+ const deferred = await db
3536
+ .select()
3537
+ .from(agentWakeupRequests)
3538
+ .where(and(eq(agentWakeupRequests.orgId, agent.orgId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_agent_paused"), sql `${agentWakeupRequests.runId} is null`))
3539
+ .orderBy(asc(agentWakeupRequests.requestedAt))
3540
+ .limit(1)
3541
+ .then((rows) => rows[0] ?? null);
3542
+ if (!deferred)
3543
+ break;
3544
+ const replayPayload = readDeferredWakePayload(deferred.payload);
3545
+ const replayContextSnapshot = readDeferredWakeContext(deferred.payload);
3546
+ try {
3547
+ await enqueueWakeup(agentId, {
3548
+ source: readNonEmptyString(deferred.source) ?? "on_demand",
3549
+ triggerDetail: readNonEmptyString(deferred.triggerDetail) ?? undefined,
3550
+ reason: readNonEmptyString(deferred.reason) ?? null,
3551
+ payload: replayPayload,
3552
+ idempotencyKey: deferred.idempotencyKey,
3553
+ requestedByActorType: deferred.requestedByActorType ?? undefined,
3554
+ requestedByActorId: deferred.requestedByActorId,
3555
+ contextSnapshot: replayContextSnapshot,
3556
+ existingWakeupRequestId: deferred.id,
3557
+ });
3558
+ }
3559
+ catch (error) {
3560
+ const current = await db
3561
+ .select({ status: agentWakeupRequests.status })
3562
+ .from(agentWakeupRequests)
3563
+ .where(eq(agentWakeupRequests.id, deferred.id))
3564
+ .then((rows) => rows[0] ?? null);
3565
+ if (current?.status === "deferred_agent_paused") {
3566
+ await setWakeupStatus(deferred.id, "failed", {
3567
+ finishedAt: new Date(),
3568
+ error: error instanceof Error ? error.message : String(error),
3569
+ });
3570
+ }
3571
+ }
3572
+ replayedRequestIds.push(deferred.id);
3573
+ const current = await db
3574
+ .select({ status: agentWakeupRequests.status })
3575
+ .from(agentWakeupRequests)
3576
+ .where(eq(agentWakeupRequests.id, deferred.id))
3577
+ .then((rows) => rows[0] ?? null);
3578
+ if (current?.status === "deferred_agent_paused")
3579
+ break;
3580
+ }
3581
+ return {
3582
+ replayed: replayedRequestIds.length,
3583
+ wakeupRequestIds: replayedRequestIds,
3584
+ };
3585
+ }
3586
+ async function listProjectScopedRunIds(orgId, projectId) {
3587
+ const runIssueId = sql `${heartbeatRuns.contextSnapshot} ->> 'issueId'`;
3588
+ const effectiveProjectId = sql `coalesce(${heartbeatRuns.contextSnapshot} ->> 'projectId', ${issues.projectId}::text)`;
3589
+ const rows = await db
3590
+ .selectDistinctOn([heartbeatRuns.id], { id: heartbeatRuns.id })
3591
+ .from(heartbeatRuns)
3592
+ .leftJoin(issues, and(eq(issues.orgId, orgId), sql `${issues.id}::text = ${runIssueId}`))
3593
+ .where(and(eq(heartbeatRuns.orgId, orgId), inArray(heartbeatRuns.status, ["queued", "running"]), sql `${effectiveProjectId} = ${projectId}`));
3594
+ return rows.map((row) => row.id);
3595
+ }
3596
+ async function listProjectScopedWakeupIds(orgId, projectId) {
3597
+ const wakeIssueId = sql `${agentWakeupRequests.payload} ->> 'issueId'`;
3598
+ const effectiveProjectId = sql `coalesce(${agentWakeupRequests.payload} ->> 'projectId', ${issues.projectId}::text)`;
3599
+ const rows = await db
3600
+ .selectDistinctOn([agentWakeupRequests.id], { id: agentWakeupRequests.id })
3601
+ .from(agentWakeupRequests)
3602
+ .leftJoin(issues, and(eq(issues.orgId, orgId), sql `${issues.id}::text = ${wakeIssueId}`))
3603
+ .where(and(eq(agentWakeupRequests.orgId, orgId), inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), sql `${agentWakeupRequests.runId} is null`, sql `${effectiveProjectId} = ${projectId}`));
3604
+ return rows.map((row) => row.id);
3605
+ }
3606
+ async function cancelPendingWakeupsForBudgetScope(scope) {
3607
+ const now = new Date();
3608
+ let wakeupIds = [];
3609
+ if (scope.scopeType === "organization") {
3610
+ wakeupIds = await db
3611
+ .select({ id: agentWakeupRequests.id })
3612
+ .from(agentWakeupRequests)
3613
+ .where(and(eq(agentWakeupRequests.orgId, scope.orgId), inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), sql `${agentWakeupRequests.runId} is null`))
3614
+ .then((rows) => rows.map((row) => row.id));
3615
+ }
3616
+ else if (scope.scopeType === "agent") {
3617
+ wakeupIds = await db
3618
+ .select({ id: agentWakeupRequests.id })
3619
+ .from(agentWakeupRequests)
3620
+ .where(and(eq(agentWakeupRequests.orgId, scope.orgId), eq(agentWakeupRequests.agentId, scope.scopeId), inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), sql `${agentWakeupRequests.runId} is null`))
3621
+ .then((rows) => rows.map((row) => row.id));
3622
+ }
3623
+ else {
3624
+ wakeupIds = await listProjectScopedWakeupIds(scope.orgId, scope.scopeId);
3625
+ }
3626
+ if (wakeupIds.length === 0)
3627
+ return 0;
3628
+ await db
3629
+ .update(agentWakeupRequests)
3630
+ .set({
3631
+ status: "cancelled",
3632
+ finishedAt: now,
3633
+ error: "Cancelled due to budget pause",
3634
+ updatedAt: now,
3635
+ })
3636
+ .where(inArray(agentWakeupRequests.id, wakeupIds));
3637
+ return wakeupIds.length;
3638
+ }
3639
+ async function cancelRunInternal(runId, reason = "Cancelled by control plane") {
3640
+ const run = await getRun(runId);
3641
+ if (!run)
3642
+ throw notFound("Heartbeat run not found");
3643
+ if (run.status !== "running" && run.status !== "queued")
3644
+ return run;
3645
+ const running = runningProcesses.get(run.id);
3646
+ if (running) {
3647
+ running.child.kill("SIGTERM");
3648
+ const graceMs = Math.max(1, running.graceSec) * 1000;
3649
+ setTimeout(() => {
3650
+ if (!running.child.killed) {
3651
+ running.child.kill("SIGKILL");
3652
+ }
3653
+ }, graceMs);
3654
+ }
3655
+ const cancelled = await setRunStatus(run.id, "cancelled", {
3656
+ finishedAt: new Date(),
3657
+ error: reason,
3658
+ errorCode: "cancelled",
3659
+ });
3660
+ await setWakeupStatus(run.wakeupRequestId, "cancelled", {
3661
+ finishedAt: new Date(),
3662
+ error: reason,
3663
+ });
3664
+ if (cancelled) {
3665
+ await appendRunEvent(cancelled, 1, {
3666
+ eventType: "lifecycle",
3667
+ stream: "system",
3668
+ level: "warn",
3669
+ message: "run cancelled",
3670
+ });
3671
+ await releaseIssueExecutionAndPromote(cancelled);
3672
+ }
3673
+ runningProcesses.delete(run.id);
3674
+ await finalizeAgentStatus(run.agentId, "cancelled");
3675
+ await startNextQueuedRunForAgent(run.agentId);
3676
+ return cancelled;
3677
+ }
3678
+ async function cancelActiveForAgentInternal(agentId, reason = "Cancelled due to agent pause") {
3679
+ const runs = await db
3680
+ .select()
3681
+ .from(heartbeatRuns)
3682
+ .where(and(eq(heartbeatRuns.agentId, agentId), inArray(heartbeatRuns.status, ["queued", "running"])));
3683
+ for (const run of runs) {
3684
+ await setRunStatus(run.id, "cancelled", {
3685
+ finishedAt: new Date(),
3686
+ error: reason,
3687
+ errorCode: "cancelled",
3688
+ });
3689
+ await setWakeupStatus(run.wakeupRequestId, "cancelled", {
3690
+ finishedAt: new Date(),
3691
+ error: reason,
3692
+ });
3693
+ const running = runningProcesses.get(run.id);
3694
+ if (running) {
3695
+ running.child.kill("SIGTERM");
3696
+ runningProcesses.delete(run.id);
3697
+ }
3698
+ await releaseIssueExecutionAndPromote(run);
3699
+ }
3700
+ return runs.length;
3701
+ }
3702
+ async function cancelBudgetScopeWork(scope) {
3703
+ if (scope.scopeType === "agent") {
3704
+ await cancelActiveForAgentInternal(scope.scopeId, "Cancelled due to budget pause");
3705
+ await cancelPendingWakeupsForBudgetScope(scope);
3706
+ return;
3707
+ }
3708
+ const runIds = scope.scopeType === "organization"
3709
+ ? await db
3710
+ .select({ id: heartbeatRuns.id })
3711
+ .from(heartbeatRuns)
3712
+ .where(and(eq(heartbeatRuns.orgId, scope.orgId), inArray(heartbeatRuns.status, ["queued", "running"])))
3713
+ .then((rows) => rows.map((row) => row.id))
3714
+ : await listProjectScopedRunIds(scope.orgId, scope.scopeId);
3715
+ for (const runId of runIds) {
3716
+ await cancelRunInternal(runId, "Cancelled due to budget pause");
3717
+ }
3718
+ await cancelPendingWakeupsForBudgetScope(scope);
3719
+ }
3720
+ async function retryRunInternal(runId, opts) {
3721
+ const run = await getRun(runId);
3722
+ if (!run)
3723
+ throw notFound("Heartbeat run not found");
3724
+ if (run.status !== "failed" && run.status !== "timed_out") {
3725
+ throw conflict("Only failed or timed out runs can be retried", {
3726
+ status: run.status,
3727
+ });
3728
+ }
3729
+ const agent = await getAgent(run.agentId);
3730
+ if (!agent)
3731
+ throw notFound("Agent not found");
3732
+ if (agent.status === "paused" ||
3733
+ agent.status === "terminated" ||
3734
+ agent.status === "pending_approval") {
3735
+ throw conflict("Agent is not invokable in its current state", { status: agent.status });
3736
+ }
3737
+ const policy = parseHeartbeatPolicy(agent);
3738
+ if (!policy.wakeOnDemand) {
3739
+ throw conflict("Agent is not configured for on-demand wakeups");
3740
+ }
3741
+ const context = parseObject(run.contextSnapshot);
3742
+ const issueId = readNonEmptyString(context.issueId);
3743
+ let projectId = readNonEmptyString(context.projectId);
3744
+ if (!projectId && issueId) {
3745
+ projectId = await db
3746
+ .select({ projectId: issues.projectId })
3747
+ .from(issues)
3748
+ .where(and(eq(issues.id, issueId), eq(issues.orgId, agent.orgId)))
3749
+ .then((rows) => rows[0]?.projectId ?? null);
3750
+ }
3751
+ const budgetBlock = await budgets.getInvocationBlock(agent.orgId, agent.id, {
3752
+ issueId,
3753
+ projectId,
3754
+ });
3755
+ if (budgetBlock) {
3756
+ throw conflict(budgetBlock.reason, {
3757
+ scopeType: budgetBlock.scopeType,
3758
+ scopeId: budgetBlock.scopeId,
3759
+ });
3760
+ }
3761
+ return enqueueRecoveryRun(run, agent, {
3762
+ recoveryTrigger: "manual",
3763
+ source: "on_demand",
3764
+ triggerDetail: "manual",
3765
+ wakeReason: "retry_failed_run",
3766
+ requestedByActorType: opts?.requestedByActorType ?? "user",
3767
+ requestedByActorId: opts?.requestedByActorId ?? null,
3768
+ now: opts?.now ?? new Date(),
3769
+ });
3770
+ }
3771
+ async function buildSkillAnalytics(scope, opts) {
3772
+ const now = opts?.now ?? new Date();
3773
+ const customDateKeys = opts?.startDate && opts?.endDate
3774
+ ? buildDateKeysBetween(opts.startDate, opts.endDate).slice(0, 120)
3775
+ : [];
3776
+ const windowDays = customDateKeys.length > 0
3777
+ ? customDateKeys.length
3778
+ : Math.max(1, Math.min(opts?.windowDays ?? 30, 90));
3779
+ const dateKeys = customDateKeys.length > 0
3780
+ ? customDateKeys
3781
+ : buildRecentDateKeys(windowDays, now);
3782
+ const startDate = dateKeys[0];
3783
+ const endDate = dateKeys.at(-1);
3784
+ const windowStart = new Date(`${startDate}T00:00:00.000Z`);
3785
+ const windowEnd = new Date(`${endDate}T23:59:59.999Z`);
3786
+ const rows = await db
3787
+ .select({
3788
+ createdAt: heartbeatRunEvents.createdAt,
3789
+ payload: heartbeatRunEvents.payload,
3790
+ })
3791
+ .from(heartbeatRunEvents)
3792
+ .where(and(eq(heartbeatRunEvents.orgId, scope.orgId), ...(scope.agentId ? [eq(heartbeatRunEvents.agentId, scope.agentId)] : []), eq(heartbeatRunEvents.eventType, "adapter.invoke"), gte(heartbeatRunEvents.createdAt, windowStart), lte(heartbeatRunEvents.createdAt, windowEnd)))
3793
+ .orderBy(asc(heartbeatRunEvents.createdAt), asc(heartbeatRunEvents.id));
3794
+ const days = new Map();
3795
+ for (const date of dateKeys) {
3796
+ days.set(date, { totalCount: 0, runCount: 0, skills: new Map() });
3797
+ }
3798
+ const overallSkills = new Map();
3799
+ let totalCount = 0;
3800
+ let totalRunsWithSkills = 0;
3801
+ for (const row of rows) {
3802
+ const date = new Date(row.createdAt).toISOString().slice(0, 10);
3803
+ const bucket = days.get(date);
3804
+ if (!bucket)
3805
+ continue;
3806
+ const payload = parseObject(row.payload);
3807
+ const loadedSkills = Array.isArray(payload.loadedSkills) ? payload.loadedSkills : [];
3808
+ if (loadedSkills.length === 0)
3809
+ continue;
3810
+ const eventSkills = new Map();
3811
+ for (const entry of loadedSkills) {
3812
+ const normalized = normalizeLoadedSkill(entry);
3813
+ if (!normalized)
3814
+ continue;
3815
+ if (!eventSkills.has(normalized.key)) {
3816
+ eventSkills.set(normalized.key, normalized.label);
3817
+ }
3818
+ }
3819
+ if (eventSkills.size === 0)
3820
+ continue;
3821
+ bucket.runCount += 1;
3822
+ totalRunsWithSkills += 1;
3823
+ for (const [key, label] of eventSkills) {
3824
+ bucket.totalCount += 1;
3825
+ totalCount += 1;
3826
+ const existingDaySkill = bucket.skills.get(key);
3827
+ if (existingDaySkill) {
3828
+ existingDaySkill.count += 1;
3829
+ }
3830
+ else {
3831
+ bucket.skills.set(key, { key, label, count: 1 });
3832
+ }
3833
+ const existingOverallSkill = overallSkills.get(key);
3834
+ if (existingOverallSkill) {
3835
+ existingOverallSkill.count += 1;
3836
+ }
3837
+ else {
3838
+ overallSkills.set(key, { key, label, count: 1 });
3839
+ }
3840
+ }
3841
+ }
3842
+ return {
3843
+ agentId: scope.agentId ?? "__all__",
3844
+ orgId: scope.orgId,
3845
+ windowDays,
3846
+ startDate,
3847
+ endDate,
3848
+ totalCount,
3849
+ totalRunsWithSkills,
3850
+ skills: Array.from(overallSkills.values()).sort((left, right) => (right.count - left.count
3851
+ || left.label.localeCompare(right.label)
3852
+ || left.key.localeCompare(right.key))),
3853
+ days: dateKeys.map((date) => {
3854
+ const bucket = days.get(date);
3855
+ return {
3856
+ date,
3857
+ totalCount: bucket.totalCount,
3858
+ runCount: bucket.runCount,
3859
+ skills: Array.from(bucket.skills.values()).sort((left, right) => (right.count - left.count
3860
+ || left.label.localeCompare(right.label)
3861
+ || left.key.localeCompare(right.key))),
3862
+ };
3863
+ }),
3864
+ };
3865
+ }
3866
+ return {
3867
+ list: async (orgId, agentId, limit) => {
3868
+ const query = db
3869
+ .select(heartbeatRunListColumns)
3870
+ .from(heartbeatRuns)
3871
+ .where(agentId
3872
+ ? and(eq(heartbeatRuns.orgId, orgId), eq(heartbeatRuns.agentId, agentId))
3873
+ : eq(heartbeatRuns.orgId, orgId))
3874
+ .orderBy(desc(heartbeatRuns.createdAt));
3875
+ const rows = limit ? await query.limit(limit) : await query;
3876
+ return rows.map((row) => ({
3877
+ ...row,
3878
+ resultJson: summarizeHeartbeatRunResultJson(row.resultJson),
3879
+ }));
3880
+ },
3881
+ getAgentSkillAnalytics: async (agentId, opts) => {
3882
+ const agent = await getAgent(agentId);
3883
+ if (!agent)
3884
+ throw notFound("Agent not found");
3885
+ return buildSkillAnalytics({ orgId: agent.orgId, agentId: agent.id }, opts);
3886
+ },
3887
+ getOrganizationSkillAnalytics: async (orgId, opts) => {
3888
+ const org = await db
3889
+ .select({ id: organizations.id })
3890
+ .from(organizations)
3891
+ .where(eq(organizations.id, orgId))
3892
+ .limit(1)
3893
+ .then((rows) => rows[0] ?? null);
3894
+ if (!org)
3895
+ throw notFound("Organization not found");
3896
+ return buildSkillAnalytics({ orgId }, opts);
3897
+ },
3898
+ getRun,
3899
+ getRuntimeState: async (agentId) => {
3900
+ const state = await getRuntimeState(agentId);
3901
+ const agent = await getAgent(agentId);
3902
+ if (!agent)
3903
+ return null;
3904
+ const ensured = state ?? (await ensureRuntimeState(agent));
3905
+ const latestTaskSession = await db
3906
+ .select()
3907
+ .from(agentTaskSessions)
3908
+ .where(and(eq(agentTaskSessions.orgId, agent.orgId), eq(agentTaskSessions.agentId, agent.id)))
3909
+ .orderBy(desc(agentTaskSessions.updatedAt))
3910
+ .limit(1)
3911
+ .then((rows) => rows[0] ?? null);
3912
+ return {
3913
+ ...ensured,
3914
+ sessionDisplayId: latestTaskSession?.sessionDisplayId ?? ensured.sessionId,
3915
+ sessionParamsJson: latestTaskSession?.sessionParamsJson ?? null,
3916
+ };
3917
+ },
3918
+ listTaskSessions: async (agentId) => {
3919
+ const agent = await getAgent(agentId);
3920
+ if (!agent)
3921
+ throw notFound("Agent not found");
3922
+ return db
3923
+ .select()
3924
+ .from(agentTaskSessions)
3925
+ .where(and(eq(agentTaskSessions.orgId, agent.orgId), eq(agentTaskSessions.agentId, agentId)))
3926
+ .orderBy(desc(agentTaskSessions.updatedAt), desc(agentTaskSessions.createdAt));
3927
+ },
3928
+ resetRuntimeSession: async (agentId, opts) => {
3929
+ const agent = await getAgent(agentId);
3930
+ if (!agent)
3931
+ throw notFound("Agent not found");
3932
+ await ensureRuntimeState(agent);
3933
+ const taskKey = readNonEmptyString(opts?.taskKey);
3934
+ const clearedTaskSessions = await clearTaskSessions(agent.orgId, agent.id, taskKey ? { taskKey, agentRuntimeType: agent.agentRuntimeType } : undefined);
3935
+ const runtimePatch = {
3936
+ sessionId: null,
3937
+ lastError: null,
3938
+ updatedAt: new Date(),
3939
+ };
3940
+ if (!taskKey) {
3941
+ runtimePatch.stateJson = {};
3942
+ }
3943
+ const updated = await db
3944
+ .update(agentRuntimeState)
3945
+ .set(runtimePatch)
3946
+ .where(eq(agentRuntimeState.agentId, agentId))
3947
+ .returning()
3948
+ .then((rows) => rows[0] ?? null);
3949
+ if (!updated)
3950
+ return null;
3951
+ return {
3952
+ ...updated,
3953
+ sessionDisplayId: null,
3954
+ sessionParamsJson: null,
3955
+ clearedTaskSessions,
3956
+ };
3957
+ },
3958
+ listEvents: (runId, afterSeq = 0, limit = 200) => db
3959
+ .select()
3960
+ .from(heartbeatRunEvents)
3961
+ .where(and(eq(heartbeatRunEvents.runId, runId), gt(heartbeatRunEvents.seq, afterSeq)))
3962
+ .orderBy(asc(heartbeatRunEvents.seq))
3963
+ .limit(Math.max(1, Math.min(limit, 1000))),
3964
+ readLog: async (runId, opts) => {
3965
+ const run = await getRun(runId);
3966
+ if (!run)
3967
+ throw notFound("Heartbeat run not found");
3968
+ if (!run.logStore || !run.logRef)
3969
+ throw notFound("Run log not found");
3970
+ const result = await runLogStore.read({
3971
+ store: run.logStore,
3972
+ logRef: run.logRef,
3973
+ }, opts);
3974
+ return {
3975
+ runId,
3976
+ store: run.logStore,
3977
+ logRef: run.logRef,
3978
+ ...result,
3979
+ content: redactCurrentUserText(result.content, await getCurrentUserRedactionOptions()),
3980
+ };
3981
+ },
3982
+ invoke: async (agentId, source = "on_demand", contextSnapshot = {}, triggerDetail = "manual", actor) => enqueueWakeup(agentId, {
3983
+ source,
3984
+ triggerDetail,
3985
+ contextSnapshot,
3986
+ requestedByActorType: actor?.actorType,
3987
+ requestedByActorId: actor?.actorId ?? null,
3988
+ }),
3989
+ wakeup: enqueueWakeup,
3990
+ resumeDeferredWakeupsForAgent,
3991
+ retryRun: retryRunInternal,
3992
+ reportRunActivity: clearDetachedRunWarning,
3993
+ reapOrphanedRuns,
3994
+ resumeQueuedRuns,
3995
+ tickTimers: async (now = new Date()) => {
3996
+ const allAgents = await db.select().from(agents);
3997
+ let checked = 0;
3998
+ let enqueued = 0;
3999
+ let skipped = 0;
4000
+ for (const agent of allAgents) {
4001
+ if (agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval")
4002
+ continue;
4003
+ const policy = parseHeartbeatPolicy(agent);
4004
+ if (!policy.enabled || policy.intervalSec <= 0)
4005
+ continue;
4006
+ checked += 1;
4007
+ const baseline = new Date(agent.lastHeartbeatAt ?? agent.createdAt).getTime();
4008
+ const elapsedMs = now.getTime() - baseline;
4009
+ if (elapsedMs < policy.intervalSec * 1000)
4010
+ continue;
4011
+ const run = await enqueueWakeup(agent.id, {
4012
+ source: "timer",
4013
+ triggerDetail: "system",
4014
+ reason: "heartbeat_timer",
4015
+ requestedByActorType: "system",
4016
+ requestedByActorId: "heartbeat_scheduler",
4017
+ contextSnapshot: {
4018
+ source: "scheduler",
4019
+ reason: "interval_elapsed",
4020
+ now: now.toISOString(),
4021
+ },
4022
+ });
4023
+ if (run)
4024
+ enqueued += 1;
4025
+ else
4026
+ skipped += 1;
4027
+ }
4028
+ return { checked, enqueued, skipped };
4029
+ },
4030
+ cancelRun: (runId) => cancelRunInternal(runId),
4031
+ cancelActiveForAgent: (agentId) => cancelActiveForAgentInternal(agentId),
4032
+ cancelBudgetScopeWork,
4033
+ getActiveRunForAgent: async (agentId) => {
4034
+ const [run] = await db
4035
+ .select()
4036
+ .from(heartbeatRuns)
4037
+ .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "running")))
4038
+ .orderBy(desc(heartbeatRuns.startedAt))
4039
+ .limit(1);
4040
+ return run ?? null;
4041
+ },
4042
+ };
4043
+ }
4044
+ //# sourceMappingURL=orchestrator.js.map