@rudderhq/server 0.1.0-canary.2 → 0.1.0-canary.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (187) hide show
  1. package/dist/bootstrap/register-api-routes.js +1 -1
  2. package/dist/bootstrap/register-api-routes.js.map +1 -1
  3. package/dist/dev-server-status.d.ts +1 -7
  4. package/dist/dev-server-status.d.ts.map +1 -1
  5. package/dist/dev-server-status.js +1 -4
  6. package/dist/dev-server-status.js.map +1 -1
  7. package/dist/routes/agents.d.ts +2 -1
  8. package/dist/routes/agents.d.ts.map +1 -1
  9. package/dist/routes/agents.js +158 -1
  10. package/dist/routes/agents.js.map +1 -1
  11. package/dist/routes/chats.d.ts.map +1 -1
  12. package/dist/routes/chats.js +164 -21
  13. package/dist/routes/chats.js.map +1 -1
  14. package/dist/routes/dashboard.d.ts.map +1 -1
  15. package/dist/routes/dashboard.js +24 -0
  16. package/dist/routes/dashboard.js.map +1 -1
  17. package/dist/routes/health.d.ts.map +1 -1
  18. package/dist/routes/health.js +3 -12
  19. package/dist/routes/health.js.map +1 -1
  20. package/dist/routes/instance-settings.d.ts.map +1 -1
  21. package/dist/routes/instance-settings.js +1 -26
  22. package/dist/routes/instance-settings.js.map +1 -1
  23. package/dist/routes/issues.d.ts.map +1 -1
  24. package/dist/routes/issues.js +19 -17
  25. package/dist/routes/issues.js.map +1 -1
  26. package/dist/services/agent-run-context.d.ts +1 -1
  27. package/dist/services/agents.d.ts +13 -13
  28. package/dist/services/chat-assistant.d.ts +1 -1
  29. package/dist/services/chat-assistant.d.ts.map +1 -1
  30. package/dist/services/chat-assistant.js +244 -41
  31. package/dist/services/chat-assistant.js.map +1 -1
  32. package/dist/services/chat-generation-locks.d.ts +2 -1
  33. package/dist/services/chat-generation-locks.d.ts.map +1 -1
  34. package/dist/services/chat-generation-locks.js +12 -3
  35. package/dist/services/chat-generation-locks.js.map +1 -1
  36. package/dist/services/chats.d.ts +3 -3
  37. package/dist/services/chats.d.ts.map +1 -1
  38. package/dist/services/chats.js +21 -2
  39. package/dist/services/chats.js.map +1 -1
  40. package/dist/services/costs.d.ts +1 -1
  41. package/dist/services/documents.d.ts +23 -0
  42. package/dist/services/documents.d.ts.map +1 -1
  43. package/dist/services/documents.js +17 -1
  44. package/dist/services/documents.js.map +1 -1
  45. package/dist/services/finance.d.ts +2 -2
  46. package/dist/services/heartbeat.d.ts +1 -1
  47. package/dist/services/heartbeat.d.ts.map +1 -1
  48. package/dist/services/heartbeat.js +1 -1
  49. package/dist/services/heartbeat.js.map +1 -1
  50. package/dist/services/instance-settings.d.ts +1 -3
  51. package/dist/services/instance-settings.d.ts.map +1 -1
  52. package/dist/services/instance-settings.js +1 -38
  53. package/dist/services/instance-settings.js.map +1 -1
  54. package/dist/services/messenger.d.ts +1 -1
  55. package/dist/services/messenger.d.ts.map +1 -1
  56. package/dist/services/messenger.js +59 -12
  57. package/dist/services/messenger.js.map +1 -1
  58. package/dist/services/runtime-kernel/analytics.d.ts +7 -0
  59. package/dist/services/runtime-kernel/analytics.d.ts.map +1 -0
  60. package/dist/services/runtime-kernel/analytics.js +46 -0
  61. package/dist/services/runtime-kernel/analytics.js.map +1 -0
  62. package/dist/services/runtime-kernel/common.d.ts +4 -0
  63. package/dist/services/runtime-kernel/common.d.ts.map +1 -0
  64. package/dist/services/runtime-kernel/common.js +15 -0
  65. package/dist/services/runtime-kernel/common.js.map +1 -0
  66. package/dist/services/runtime-kernel/execution.d.ts +11 -0
  67. package/dist/services/runtime-kernel/execution.d.ts.map +1 -0
  68. package/dist/services/runtime-kernel/execution.js +28 -0
  69. package/dist/services/runtime-kernel/execution.js.map +1 -0
  70. package/dist/services/runtime-kernel/heartbeat.d.ts +1 -881
  71. package/dist/services/runtime-kernel/heartbeat.d.ts.map +1 -1
  72. package/dist/services/runtime-kernel/heartbeat.js +1 -4344
  73. package/dist/services/runtime-kernel/heartbeat.js.map +1 -1
  74. package/dist/services/runtime-kernel/model-fallback.d.ts +10 -0
  75. package/dist/services/runtime-kernel/model-fallback.d.ts.map +1 -0
  76. package/dist/services/runtime-kernel/model-fallback.js +147 -0
  77. package/dist/services/runtime-kernel/model-fallback.js.map +1 -0
  78. package/dist/services/runtime-kernel/orchestrator.d.ts +826 -0
  79. package/dist/services/runtime-kernel/orchestrator.d.ts.map +1 -0
  80. package/dist/services/runtime-kernel/orchestrator.js +4044 -0
  81. package/dist/services/runtime-kernel/orchestrator.js.map +1 -0
  82. package/dist/services/runtime-kernel/recovery.d.ts +8 -0
  83. package/dist/services/runtime-kernel/recovery.d.ts.map +1 -0
  84. package/dist/services/runtime-kernel/recovery.js +101 -0
  85. package/dist/services/runtime-kernel/recovery.js.map +1 -0
  86. package/dist/services/runtime-kernel/run-state.d.ts +45 -0
  87. package/dist/services/runtime-kernel/run-state.d.ts.map +1 -0
  88. package/dist/services/runtime-kernel/run-state.js +45 -0
  89. package/dist/services/runtime-kernel/run-state.js.map +1 -0
  90. package/dist/services/runtime-kernel/session-policy.d.ts +42 -0
  91. package/dist/services/runtime-kernel/session-policy.d.ts.map +1 -0
  92. package/dist/services/runtime-kernel/session-policy.js +116 -0
  93. package/dist/services/runtime-kernel/session-policy.js.map +1 -0
  94. package/dist/services/runtime-kernel/types.d.ts +13 -0
  95. package/dist/services/runtime-kernel/types.d.ts.map +1 -0
  96. package/dist/services/runtime-kernel/types.js +2 -0
  97. package/dist/services/runtime-kernel/types.js.map +1 -0
  98. package/dist/services/runtime-kernel/wakeup-queue.d.ts +9 -0
  99. package/dist/services/runtime-kernel/wakeup-queue.d.ts.map +1 -0
  100. package/dist/services/runtime-kernel/wakeup-queue.js +19 -0
  101. package/dist/services/runtime-kernel/wakeup-queue.js.map +1 -0
  102. package/package.json +14 -14
  103. package/resources/bundled-skills/rudder-create-agent/SKILL.md +1 -1
  104. package/resources/bundled-skills/rudder-create-agent/references/api-reference.md +4 -2
  105. package/resources/bundled-skills/rudder-create-agent/references/cli-reference.md +1 -1
  106. package/skills/rudder-create-agent/SKILL.md +1 -1
  107. package/skills/rudder-create-agent/references/api-reference.md +4 -2
  108. package/skills/rudder-create-agent/references/cli-reference.md +1 -1
  109. package/ui-dist/assets/{_basePickBy-C8ZhuJlI.js → _basePickBy-DAU2OOE3.js} +1 -1
  110. package/ui-dist/assets/{_baseUniq-BRCVYeJb.js → _baseUniq-BQLJfIXV.js} +1 -1
  111. package/ui-dist/assets/{arc-B1R0dxYm.js → arc-B2UTEkTB.js} +1 -1
  112. package/ui-dist/assets/{architectureDiagram-2XIMDMQ5-D7sRWBaf.js → architectureDiagram-2XIMDMQ5-DicHsSOp.js} +1 -1
  113. package/ui-dist/assets/{blockDiagram-WCTKOSBZ-Byw6t-Eq.js → blockDiagram-WCTKOSBZ-tbkC8ZCs.js} +1 -1
  114. package/ui-dist/assets/{c4Diagram-IC4MRINW-RsrU80Hc.js → c4Diagram-IC4MRINW-pPT_Xw2l.js} +1 -1
  115. package/ui-dist/assets/channel-D8qCsVGC.js +1 -0
  116. package/ui-dist/assets/{chunk-4BX2VUAB-D9oP4FJ_.js → chunk-4BX2VUAB-vX_EgGXp.js} +1 -1
  117. package/ui-dist/assets/{chunk-55IACEB6-DT8xQAKU.js → chunk-55IACEB6-DUyGlyCg.js} +1 -1
  118. package/ui-dist/assets/{chunk-FMBD7UC4-FwqsP6AV.js → chunk-FMBD7UC4-DPCYqV54.js} +1 -1
  119. package/ui-dist/assets/{chunk-JSJVCQXG-CTOvJxU3.js → chunk-JSJVCQXG-CRqpXPJG.js} +1 -1
  120. package/ui-dist/assets/{chunk-KX2RTZJC-8oze7Khf.js → chunk-KX2RTZJC-BWaSxc23.js} +1 -1
  121. package/ui-dist/assets/{chunk-NQ4KR5QH-D38LrSCR.js → chunk-NQ4KR5QH-BCIl9FTX.js} +1 -1
  122. package/ui-dist/assets/{chunk-QZHKN3VN-B3ZhJA7D.js → chunk-QZHKN3VN-BgTqyV62.js} +1 -1
  123. package/ui-dist/assets/{chunk-WL4C6EOR-CSTjCHX6.js → chunk-WL4C6EOR-COlN9kxA.js} +1 -1
  124. package/ui-dist/assets/classDiagram-VBA2DB6C-Cm8BRWIa.js +1 -0
  125. package/ui-dist/assets/classDiagram-v2-RAHNMMFH-Cm8BRWIa.js +1 -0
  126. package/ui-dist/assets/clone-B0pIfwDy.js +1 -0
  127. package/ui-dist/assets/{cose-bilkent-S5V4N54A-CI_3Qz62.js → cose-bilkent-S5V4N54A-Bbgtq4rD.js} +1 -1
  128. package/ui-dist/assets/{dagre-KLK3FWXG-BNakeJ5Q.js → dagre-KLK3FWXG-DgUnjcLS.js} +1 -1
  129. package/ui-dist/assets/{diagram-E7M64L7V-UubVu-IN.js → diagram-E7M64L7V-DC3DIXbu.js} +1 -1
  130. package/ui-dist/assets/{diagram-IFDJBPK2-DRu8FSPT.js → diagram-IFDJBPK2-qv7Ij_rH.js} +1 -1
  131. package/ui-dist/assets/{diagram-P4PSJMXO-t2_nA6WC.js → diagram-P4PSJMXO-Dw2WzhiN.js} +1 -1
  132. package/ui-dist/assets/{erDiagram-INFDFZHY-ET8QDHtO.js → erDiagram-INFDFZHY-BV5s3J3D.js} +1 -1
  133. package/ui-dist/assets/{flowDiagram-PKNHOUZH-QXQeM_4d.js → flowDiagram-PKNHOUZH-kmPgde9o.js} +1 -1
  134. package/ui-dist/assets/{ganttDiagram-A5KZAMGK-B6yFn5Hd.js → ganttDiagram-A5KZAMGK-C2hkJ2Th.js} +1 -1
  135. package/ui-dist/assets/{gitGraphDiagram-K3NZZRJ6-DZAeG6MG.js → gitGraphDiagram-K3NZZRJ6-K5e1NgIt.js} +1 -1
  136. package/ui-dist/assets/{graph-BaTTyfU1.js → graph-Cna3FnWY.js} +1 -1
  137. package/ui-dist/assets/{index-C7kEQtA_.js → index-6luhu7W5.js} +1 -1
  138. package/ui-dist/assets/index-B1iX9NWL.js +1364 -0
  139. package/ui-dist/assets/{index-D-mb6cn2.js → index-BdLts2oZ.js} +1 -1
  140. package/ui-dist/assets/{index-ym7ZDmXE.js → index-CBKnHGYE.js} +1 -1
  141. package/ui-dist/assets/{index-D_ZvBEXt.js → index-CDtBE3Ga.js} +1 -1
  142. package/ui-dist/assets/{index-Cftoq4bF.js → index-CFGJHREd.js} +1 -1
  143. package/ui-dist/assets/{index-Bo1sSJ7x.js → index-CTpC-gpc.js} +1 -1
  144. package/ui-dist/assets/{index-h_UqLbty.js → index-CVAmlnRr.js} +1 -1
  145. package/ui-dist/assets/{index-BWEA-ibM.js → index-CWbJ-pcg.js} +1 -1
  146. package/ui-dist/assets/{index-BSfIb9qw.js → index-ChU0nSCq.js} +1 -1
  147. package/ui-dist/assets/{index-DQeEMbWr.js → index-CypGJ__o.js} +1 -1
  148. package/ui-dist/assets/{index-BXxi8m3U.js → index-D-Fe0jjS.js} +1 -1
  149. package/ui-dist/assets/{index-BGOxzMvq.js → index-D8aBm25c.js} +1 -1
  150. package/ui-dist/assets/index-D9uKiQrM.css +1 -0
  151. package/ui-dist/assets/{index-DBsb_N5Q.js → index-DD0ggVK_.js} +1 -1
  152. package/ui-dist/assets/{index-MYVA1f40.js → index-DJecNMv2.js} +1 -1
  153. package/ui-dist/assets/{index-CAcJz5d9.js → index-DKSfpiAJ.js} +1 -1
  154. package/ui-dist/assets/{index-ZHLqhZdz.js → index-Dgj2bRIV.js} +1 -1
  155. package/ui-dist/assets/{index-DvQZ5FJk.js → index-DsydT3bX.js} +1 -1
  156. package/ui-dist/assets/{index-qVKE_HM2.js → index-U7gGuiCe.js} +1 -1
  157. package/ui-dist/assets/{index-CRtxHtSg.js → index-aL5hFrtS.js} +1 -1
  158. package/ui-dist/assets/{index-DPbDGs74.js → index-o_iEnzyw.js} +1 -1
  159. package/ui-dist/assets/{index-eessTUZm.js → index-qUFX4kXb.js} +1 -1
  160. package/ui-dist/assets/{index-C2T1FM7J.js → index-sV3mB3fg.js} +1 -1
  161. package/ui-dist/assets/{infoDiagram-LFFYTUFH-BOnKcq7S.js → infoDiagram-LFFYTUFH-BiQ6_Z3F.js} +1 -1
  162. package/ui-dist/assets/{ishikawaDiagram-PHBUUO56-etV-MroN.js → ishikawaDiagram-PHBUUO56-C_WqRu1B.js} +1 -1
  163. package/ui-dist/assets/{journeyDiagram-4ABVD52K-mvjGNrBx.js → journeyDiagram-4ABVD52K-BIpPercT.js} +1 -1
  164. package/ui-dist/assets/{kanban-definition-K7BYSVSG-CW-Dw_bU.js → kanban-definition-K7BYSVSG-DaFD_JyB.js} +1 -1
  165. package/ui-dist/assets/{layout-DUcWZ3H3.js → layout-Dj_oike0.js} +1 -1
  166. package/ui-dist/assets/{linear-B0r1V0oG.js → linear-tnVt1ugU.js} +1 -1
  167. package/ui-dist/assets/{mermaid.core-Bo_YuNee.js → mermaid.core-DflBh7Hv.js} +4 -4
  168. package/ui-dist/assets/{mindmap-definition-YRQLILUH-BmEjkBnz.js → mindmap-definition-YRQLILUH-C3H-gFSh.js} +1 -1
  169. package/ui-dist/assets/{pieDiagram-SKSYHLDU-BRER3VVx.js → pieDiagram-SKSYHLDU-DH53odzN.js} +1 -1
  170. package/ui-dist/assets/{quadrantDiagram-337W2JSQ-oYnSbQKi.js → quadrantDiagram-337W2JSQ-DCn4Dric.js} +1 -1
  171. package/ui-dist/assets/{requirementDiagram-Z7DCOOCP-BRLFZsts.js → requirementDiagram-Z7DCOOCP-KypOfBcY.js} +1 -1
  172. package/ui-dist/assets/{sankeyDiagram-WA2Y5GQK-YOKeMxIU.js → sankeyDiagram-WA2Y5GQK-DEmi0ryr.js} +1 -1
  173. package/ui-dist/assets/{sequenceDiagram-2WXFIKYE-Bj-YKQMs.js → sequenceDiagram-2WXFIKYE-sW6ojsVG.js} +1 -1
  174. package/ui-dist/assets/{stateDiagram-RAJIS63D-BYf-CeWe.js → stateDiagram-RAJIS63D-CmrX9BEC.js} +1 -1
  175. package/ui-dist/assets/stateDiagram-v2-FVOUBMTO-DJuCnZbk.js +1 -0
  176. package/ui-dist/assets/{timeline-definition-YZTLITO2-CG6r3GP4.js → timeline-definition-YZTLITO2-CzJ5Y8YU.js} +1 -1
  177. package/ui-dist/assets/{treemap-KZPCXAKY-DCX4FLGH.js → treemap-KZPCXAKY-BwUNliTA.js} +1 -1
  178. package/ui-dist/assets/{vennDiagram-LZ73GAT5-CpbbhHdY.js → vennDiagram-LZ73GAT5-5A50aIn1.js} +1 -1
  179. package/ui-dist/assets/{xychartDiagram-JWTSCODW-Biq5EV_L.js → xychartDiagram-JWTSCODW-D9qTFskE.js} +1 -1
  180. package/ui-dist/index.html +2 -2
  181. package/ui-dist/assets/channel-BExT12GI.js +0 -1
  182. package/ui-dist/assets/classDiagram-VBA2DB6C-B3FSZmLP.js +0 -1
  183. package/ui-dist/assets/classDiagram-v2-RAHNMMFH-B3FSZmLP.js +0 -1
  184. package/ui-dist/assets/clone-DxXNeJhJ.js +0 -1
  185. package/ui-dist/assets/index-2Hx2TlnA.css +0 -1
  186. package/ui-dist/assets/index-wzjO7KZw.js +0 -1343
  187. package/ui-dist/assets/stateDiagram-v2-FVOUBMTO-g0byx18e.js +0 -1
@@ -1,4345 +1,2 @@
1
- import path from "node:path";
2
- import { and, asc, desc, eq, gt, gte, inArray, lte, sql } from "drizzle-orm";
3
- import { agents, agentRuntimeState, agentTaskSessions, agentWakeupRequests, activityLog, heartbeatRunEvents, heartbeatRuns, issueComments, issues, projects, } from "@rudderhq/db";
4
- import { conflict, notFound } from "../../errors.js";
5
- import { createExecutionScores, observeExecutionEvent, updateExecutionObservation, updateExecutionTraceIO, updateExecutionTraceName, updateExecutionTraceSession, withExecutionObservation, } from "../../langfuse.js";
6
- import { emitExecutionTranscriptTree } from "../../langfuse-transcript.js";
7
- import { logger } from "../../middleware/logger.js";
8
- import { publishLiveEvent } from "../live-events.js";
9
- import { getRunLogStore } from "../run-log-store.js";
10
- import { getServerAdapter, runningProcesses } from "../../agent-runtimes/index.js";
11
- import { createLocalAgentJwt } from "../../agent-auth-jwt.js";
12
- import { parseObject, asBoolean, asNumber, appendWithCap, MAX_EXCERPT_BYTES } from "../../agent-runtimes/utils.js";
13
- import { costService } from "../costs.js";
14
- import { budgetService } from "../budgets.js";
15
- import { agentRunContextService, } from "../agent-run-context.js";
16
- import { resolveDefaultAgentWorkspaceDir, } from "../../home-paths.js";
17
- import { summarizeHeartbeatRunResultJson } from "../heartbeat-run-summary.js";
18
- import { summarizeRuntimeSkillsForTrace } from "../runtime-trace-metadata.js";
19
- import { buildWorkspaceReadyComment, cleanupExecutionWorkspaceArtifacts, ensureRuntimeServicesForRun, persistAdapterManagedRuntimeServices, realizeExecutionWorkspace, releaseRuntimeServicesForRun, } from "../workspace-runtime.js";
20
- import { issueService } from "../issues.js";
21
- import { executionWorkspaceService } from "../execution-workspaces.js";
22
- import { buildObservedRunLangfuseScores } from "../run-intelligence.js";
23
- import { workspaceOperationService } from "../workspace-operations.js";
24
- import { buildExecutionWorkspaceAdapterConfig, issueExecutionWorkspaceModeForPersistedWorkspace, parseIssueExecutionWorkspaceSettings, parseProjectExecutionWorkspacePolicy, resolveExecutionWorkspaceMode, } from "../execution-workspace-policy.js";
25
- import { instanceSettingsService } from "../instance-settings.js";
26
- import { logActivity } from "../activity-log.js";
27
- import { redactCurrentUserText, redactCurrentUserValue } from "../../log-redaction.js";
28
- import { hasSessionCompactionThresholds, resolveSessionCompactionPolicy, } from "@rudderhq/agent-runtime-utils";
29
- import { buildCreateAgentBenchmarkTags, coerceCreateAgentBenchmarkMetadata, extractCreateAgentBenchmarkMetadata, } from "@rudderhq/run-intelligence-core";
30
- export { prioritizeProjectWorkspaceCandidatesForRun } from "../agent-run-context.js";
31
- const MAX_LIVE_LOG_CHUNK_BYTES = 8 * 1024;
32
- const HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT = 1;
33
- const HEARTBEAT_MAX_CONCURRENT_RUNS_MAX = 10;
34
- const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext";
35
- const DETACHED_PROCESS_ERROR_CODE = "process_detached";
36
- const ORPHANED_PROCESS_TERMINATION_GRACE_MS = 2_000;
37
- const ORPHANED_PROCESS_KILL_WAIT_MS = 500;
38
- const ORPHANED_PROCESS_POLL_INTERVAL_MS = 100;
39
- const startLocksByAgent = new Map();
40
- const MAX_RECOVERY_CHAIN_DEPTH = 8;
41
- const ISSUE_PASSIVE_FOLLOWUP_REASON = "issue_passive_followup";
42
- const ISSUE_PASSIVE_FOLLOWUP_WAKE_SOURCE = "passive_issue_followup";
43
- const ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON = "missing_closure";
44
- const ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS = 2;
45
- const ISSUE_PASSIVE_FOLLOWUP_COOLDOWN_MS_BY_ATTEMPT = new Map([
46
- [1, 2 * 60 * 1000],
47
- [2, 5 * 60 * 1000],
48
- ]);
49
- const ISSUE_PASSIVE_FOLLOWUP_TIMER_CONTINUITY_MAX_WINDOW_MS = 15 * 60 * 1000;
50
- const SESSIONED_LOCAL_ADAPTERS = new Set([
51
- "claude_local",
52
- "codex_local",
53
- "cursor",
54
- "gemini_local",
55
- "opencode_local",
56
- "pi_local",
57
- ]);
58
- const heartbeatRunListColumns = {
59
- id: heartbeatRuns.id,
60
- orgId: heartbeatRuns.orgId,
61
- agentId: heartbeatRuns.agentId,
62
- invocationSource: heartbeatRuns.invocationSource,
63
- triggerDetail: heartbeatRuns.triggerDetail,
64
- status: heartbeatRuns.status,
65
- startedAt: heartbeatRuns.startedAt,
66
- finishedAt: heartbeatRuns.finishedAt,
67
- error: heartbeatRuns.error,
68
- wakeupRequestId: heartbeatRuns.wakeupRequestId,
69
- exitCode: heartbeatRuns.exitCode,
70
- signal: heartbeatRuns.signal,
71
- usageJson: heartbeatRuns.usageJson,
72
- resultJson: heartbeatRuns.resultJson,
73
- sessionIdBefore: heartbeatRuns.sessionIdBefore,
74
- sessionIdAfter: heartbeatRuns.sessionIdAfter,
75
- logStore: heartbeatRuns.logStore,
76
- logRef: heartbeatRuns.logRef,
77
- logBytes: heartbeatRuns.logBytes,
78
- logSha256: heartbeatRuns.logSha256,
79
- logCompressed: heartbeatRuns.logCompressed,
80
- stdoutExcerpt: sql `NULL`.as("stdoutExcerpt"),
81
- stderrExcerpt: sql `NULL`.as("stderrExcerpt"),
82
- errorCode: heartbeatRuns.errorCode,
83
- externalRunId: heartbeatRuns.externalRunId,
84
- processPid: heartbeatRuns.processPid,
85
- processStartedAt: heartbeatRuns.processStartedAt,
86
- retryOfRunId: heartbeatRuns.retryOfRunId,
87
- processLossRetryCount: heartbeatRuns.processLossRetryCount,
88
- contextSnapshot: heartbeatRuns.contextSnapshot,
89
- createdAt: heartbeatRuns.createdAt,
90
- updatedAt: heartbeatRuns.updatedAt,
91
- };
92
- function appendExcerpt(prev, chunk) {
93
- return appendWithCap(prev, chunk, MAX_EXCERPT_BYTES);
94
- }
95
- function appendTranscriptEntriesFromChunk(input) {
96
- const combined = `${input.buffer}${input.chunk}`;
97
- const lines = combined.split(/\r?\n/);
98
- const trailing = lines.pop() ?? "";
99
- const completeLines = input.finalize && trailing ? [...lines, trailing] : lines;
100
- for (const line of completeLines) {
101
- if (!line.trim())
102
- continue;
103
- const ts = new Date().toISOString();
104
- const parsed = input.parser ? input.parser(line, ts) : [];
105
- if (parsed.length > 0) {
106
- input.transcript.push(...parsed);
107
- continue;
108
- }
109
- input.transcript.push({
110
- kind: input.kind,
111
- ts,
112
- text: line,
113
- });
114
- }
115
- return input.finalize ? "" : trailing;
116
- }
117
- function normalizeMaxConcurrentRuns(value) {
118
- const parsed = Math.floor(asNumber(value, HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT));
119
- if (!Number.isFinite(parsed))
120
- return HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT;
121
- return Math.max(HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT, Math.min(HEARTBEAT_MAX_CONCURRENT_RUNS_MAX, parsed));
122
- }
123
- async function withAgentStartLock(agentId, fn) {
124
- const previous = startLocksByAgent.get(agentId) ?? Promise.resolve();
125
- const run = previous.then(fn);
126
- const marker = run.then(() => undefined, () => undefined);
127
- startLocksByAgent.set(agentId, marker);
128
- try {
129
- return await run;
130
- }
131
- finally {
132
- if (startLocksByAgent.get(agentId) === marker) {
133
- startLocksByAgent.delete(agentId);
134
- }
135
- }
136
- }
137
- function readNonEmptyString(value) {
138
- return typeof value === "string" && value.trim().length > 0 ? value : null;
139
- }
140
- export function resolveHeartbeatObservabilitySurface(contextSnapshot) {
141
- return readNonEmptyString(contextSnapshot?.issueId) ? "issue_run" : "heartbeat_run";
142
- }
143
- function buildHeartbeatObservationName(run, agentName) {
144
- const contextSnapshot = parseObject(run.contextSnapshot);
145
- const issueId = readNonEmptyString(contextSnapshot.issueId);
146
- return issueId ? `issue_run:${issueId}` : `heartbeat:${agentName}`;
147
- }
148
- function compactTraceText(value, maxLength = 120) {
149
- const next = value?.replace(/\s+/g, " ").trim();
150
- if (!next)
151
- return null;
152
- return next.length > maxLength ? `${next.slice(0, maxLength - 1)}…` : next;
153
- }
154
- export function buildIssueRunTraceName(input) {
155
- const issueTitle = compactTraceText(input.issueTitle);
156
- return issueTitle ? `issue_run:${issueTitle} [${input.issueId}]` : `issue_run:[${input.issueId}]`;
157
- }
158
- export function buildHeartbeatRuntimeTraceMetadata(input) {
159
- const instructionsFilePath = readNonEmptyString(input.runtimeConfig.instructionsFilePath);
160
- return {
161
- instructionsConfigured: Boolean(instructionsFilePath),
162
- instructionsFilePath,
163
- ...summarizeRuntimeSkillsForTrace(input.runtimeSkills),
164
- ...(input.adapterMeta
165
- ? {
166
- runtimeAgentType: input.adapterMeta.agentRuntimeType,
167
- runtimeCommand: input.adapterMeta.command,
168
- runtimeCwd: input.adapterMeta.cwd ?? null,
169
- runtimeCommandNotes: input.adapterMeta.commandNotes ?? [],
170
- runtimePromptMetrics: input.adapterMeta.promptMetrics ?? null,
171
- }
172
- : {}),
173
- };
174
- }
175
- function buildRecentDateKeys(windowDays, now) {
176
- return Array.from({ length: windowDays }, (_, index) => {
177
- const next = new Date(now);
178
- next.setUTCDate(next.getUTCDate() - (windowDays - 1 - index));
179
- return next.toISOString().slice(0, 10);
180
- });
181
- }
182
- function buildDateKeysBetween(startDate, endDate) {
183
- const start = new Date(`${startDate}T00:00:00.000Z`);
184
- const end = new Date(`${endDate}T00:00:00.000Z`);
185
- if (Number.isNaN(start.getTime()) || Number.isNaN(end.getTime()) || start.getTime() > end.getTime()) {
186
- return [];
187
- }
188
- const days = [];
189
- const cursor = new Date(start);
190
- while (cursor.getTime() <= end.getTime()) {
191
- days.push(cursor.toISOString().slice(0, 10));
192
- cursor.setUTCDate(cursor.getUTCDate() + 1);
193
- }
194
- return days;
195
- }
196
- function fallbackSkillLabel(key) {
197
- const trimmed = key.trim();
198
- if (!trimmed)
199
- return "unknown";
200
- const slashSegments = trimmed.split("/").filter(Boolean);
201
- const lastSlashSegment = slashSegments.at(-1);
202
- if (lastSlashSegment)
203
- return lastSlashSegment;
204
- const colonSegments = trimmed.split(":").filter(Boolean);
205
- return colonSegments.at(-1) ?? trimmed;
206
- }
207
- function normalizeLoadedSkill(value) {
208
- const skill = parseObject(value);
209
- const rawKey = readNonEmptyString(skill.key);
210
- const rawRuntimeName = readNonEmptyString(skill.runtimeName);
211
- const rawName = readNonEmptyString(skill.name);
212
- const key = rawKey ?? rawRuntimeName ?? rawName;
213
- if (!key)
214
- return null;
215
- const label = rawRuntimeName ?? rawName ?? fallbackSkillLabel(key);
216
- return { key, label };
217
- }
218
- function normalizeLedgerBillingType(value) {
219
- const raw = readNonEmptyString(value);
220
- switch (raw) {
221
- case "api":
222
- case "metered_api":
223
- return "metered_api";
224
- case "subscription":
225
- case "subscription_included":
226
- return "subscription_included";
227
- case "subscription_overage":
228
- return "subscription_overage";
229
- case "credits":
230
- return "credits";
231
- case "fixed":
232
- return "fixed";
233
- default:
234
- return "unknown";
235
- }
236
- }
237
- function resolveLedgerBiller(result) {
238
- return readNonEmptyString(result.biller) ?? readNonEmptyString(result.provider) ?? "unknown";
239
- }
240
- function normalizeBilledCostCents(costUsd, billingType) {
241
- if (billingType === "subscription_included")
242
- return 0;
243
- if (typeof costUsd !== "number" || !Number.isFinite(costUsd))
244
- return 0;
245
- return Math.max(0, Math.round(costUsd * 100));
246
- }
247
- async function resolveLedgerScopeForRun(db, orgId, run) {
248
- const context = parseObject(run.contextSnapshot);
249
- const contextIssueId = readNonEmptyString(context.issueId);
250
- const contextProjectId = readNonEmptyString(context.projectId);
251
- if (!contextIssueId) {
252
- return {
253
- issueId: null,
254
- projectId: contextProjectId,
255
- };
256
- }
257
- const issue = await db
258
- .select({
259
- id: issues.id,
260
- projectId: issues.projectId,
261
- })
262
- .from(issues)
263
- .where(and(eq(issues.id, contextIssueId), eq(issues.orgId, orgId)))
264
- .then((rows) => rows[0] ?? null);
265
- return {
266
- issueId: issue?.id ?? null,
267
- projectId: issue?.projectId ?? contextProjectId,
268
- };
269
- }
270
- export function buildExplicitResumeSessionOverride(input) {
271
- const desiredDisplayId = truncateDisplayId(input.resumeRunSessionIdAfter ?? input.resumeRunSessionIdBefore);
272
- const taskSessionParams = normalizeSessionParams(input.sessionCodec.deserialize(input.taskSession?.sessionParamsJson ?? null));
273
- const taskSessionDisplayId = truncateDisplayId(input.taskSession?.sessionDisplayId ??
274
- (input.sessionCodec.getDisplayId ? input.sessionCodec.getDisplayId(taskSessionParams) : null) ??
275
- readNonEmptyString(taskSessionParams?.sessionId));
276
- const canReuseTaskSessionParams = input.taskSession != null &&
277
- (input.taskSession.lastRunId === input.resumeFromRunId ||
278
- (!!desiredDisplayId && taskSessionDisplayId === desiredDisplayId));
279
- const sessionParams = canReuseTaskSessionParams
280
- ? taskSessionParams
281
- : desiredDisplayId
282
- ? { sessionId: desiredDisplayId }
283
- : null;
284
- const sessionDisplayId = desiredDisplayId ?? (canReuseTaskSessionParams ? taskSessionDisplayId : null);
285
- if (!sessionDisplayId && !sessionParams)
286
- return null;
287
- return {
288
- sessionDisplayId,
289
- sessionParams,
290
- };
291
- }
292
- function normalizeUsageTotals(usage) {
293
- if (!usage)
294
- return null;
295
- return {
296
- inputTokens: Math.max(0, Math.floor(asNumber(usage.inputTokens, 0))),
297
- cachedInputTokens: Math.max(0, Math.floor(asNumber(usage.cachedInputTokens, 0))),
298
- outputTokens: Math.max(0, Math.floor(asNumber(usage.outputTokens, 0))),
299
- };
300
- }
301
- function readRawUsageTotals(usageJson) {
302
- const parsed = parseObject(usageJson);
303
- if (Object.keys(parsed).length === 0)
304
- return null;
305
- const inputTokens = Math.max(0, Math.floor(asNumber(parsed.rawInputTokens, asNumber(parsed.inputTokens, 0))));
306
- const cachedInputTokens = Math.max(0, Math.floor(asNumber(parsed.rawCachedInputTokens, asNumber(parsed.cachedInputTokens, 0))));
307
- const outputTokens = Math.max(0, Math.floor(asNumber(parsed.rawOutputTokens, asNumber(parsed.outputTokens, 0))));
308
- if (inputTokens <= 0 && cachedInputTokens <= 0 && outputTokens <= 0) {
309
- return null;
310
- }
311
- return {
312
- inputTokens,
313
- cachedInputTokens,
314
- outputTokens,
315
- };
316
- }
317
- function deriveNormalizedUsageDelta(current, previous) {
318
- if (!current)
319
- return null;
320
- if (!previous)
321
- return { ...current };
322
- const inputTokens = current.inputTokens >= previous.inputTokens
323
- ? current.inputTokens - previous.inputTokens
324
- : current.inputTokens;
325
- const cachedInputTokens = current.cachedInputTokens >= previous.cachedInputTokens
326
- ? current.cachedInputTokens - previous.cachedInputTokens
327
- : current.cachedInputTokens;
328
- const outputTokens = current.outputTokens >= previous.outputTokens
329
- ? current.outputTokens - previous.outputTokens
330
- : current.outputTokens;
331
- return {
332
- inputTokens: Math.max(0, inputTokens),
333
- cachedInputTokens: Math.max(0, cachedInputTokens),
334
- outputTokens: Math.max(0, outputTokens),
335
- };
336
- }
337
- function formatCount(value) {
338
- if (typeof value !== "number" || !Number.isFinite(value))
339
- return "0";
340
- return value.toLocaleString("en-US");
341
- }
342
- export function parseSessionCompactionPolicy(agent) {
343
- return resolveSessionCompactionPolicy(agent.agentRuntimeType, agent.runtimeConfig).policy;
344
- }
345
- export function resolveRuntimeSessionParamsForWorkspace(input) {
346
- const { orgId, agent, previousSessionParams, resolvedWorkspace } = input;
347
- const previousSessionId = readNonEmptyString(previousSessionParams?.sessionId);
348
- if (!previousSessionId) {
349
- return {
350
- sessionParams: previousSessionParams,
351
- warning: null,
352
- };
353
- }
354
- const canonicalAgentCwd = readNonEmptyString(resolvedWorkspace.cwd) ?? resolveDefaultAgentWorkspaceDir(orgId, agent);
355
- if (!canonicalAgentCwd) {
356
- return {
357
- sessionParams: previousSessionParams,
358
- warning: null,
359
- };
360
- }
361
- const previousCwd = readNonEmptyString(previousSessionParams?.cwd);
362
- if (previousCwd && path.resolve(previousCwd) === path.resolve(canonicalAgentCwd)) {
363
- return {
364
- sessionParams: previousSessionParams,
365
- warning: null,
366
- };
367
- }
368
- const previousWorkspaceId = readNonEmptyString(previousSessionParams?.workspaceId);
369
- const migratedSessionParams = {
370
- ...(previousSessionParams ?? {}),
371
- cwd: canonicalAgentCwd,
372
- };
373
- if (!previousWorkspaceId ||
374
- !resolvedWorkspace.workspaceId ||
375
- previousWorkspaceId === resolvedWorkspace.workspaceId) {
376
- if (resolvedWorkspace.workspaceId)
377
- migratedSessionParams.workspaceId = resolvedWorkspace.workspaceId;
378
- if (resolvedWorkspace.repoUrl)
379
- migratedSessionParams.repoUrl = resolvedWorkspace.repoUrl;
380
- if (resolvedWorkspace.repoRef)
381
- migratedSessionParams.repoRef = resolvedWorkspace.repoRef;
382
- }
383
- return {
384
- sessionParams: migratedSessionParams,
385
- warning: previousCwd
386
- ? `Agent workspace "${canonicalAgentCwd}" is now the canonical run workspace. ` +
387
- `Attempting to resume session "${previousSessionId}" that was previously saved in "${previousCwd}".`
388
- : `Agent workspace "${canonicalAgentCwd}" is now the canonical run workspace. ` +
389
- `Attempting to resume session "${previousSessionId}" with the canonical agent workspace attached.`,
390
- };
391
- }
392
- function parseIssueAssigneeAgentRuntimeOverrides(raw) {
393
- const parsed = parseObject(raw);
394
- const parsedAdapterConfig = parseObject(parsed.agentRuntimeConfig);
395
- const agentRuntimeConfig = Object.keys(parsedAdapterConfig).length > 0 ? parsedAdapterConfig : null;
396
- const useProjectWorkspace = typeof parsed.useProjectWorkspace === "boolean"
397
- ? parsed.useProjectWorkspace
398
- : null;
399
- if (!agentRuntimeConfig && useProjectWorkspace === null)
400
- return null;
401
- return {
402
- agentRuntimeConfig,
403
- useProjectWorkspace,
404
- };
405
- }
406
- function deriveTaskKey(contextSnapshot, payload) {
407
- return (readNonEmptyString(contextSnapshot?.taskKey) ??
408
- readNonEmptyString(contextSnapshot?.taskId) ??
409
- readNonEmptyString(contextSnapshot?.issueId) ??
410
- readNonEmptyString(payload?.taskKey) ??
411
- readNonEmptyString(payload?.taskId) ??
412
- readNonEmptyString(payload?.issueId) ??
413
- null);
414
- }
415
- export function shouldResetTaskSessionForWake(contextSnapshot) {
416
- if (contextSnapshot?.forceFreshSession === true)
417
- return true;
418
- const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason);
419
- if (wakeReason === "issue_assigned")
420
- return true;
421
- return false;
422
- }
423
- export function formatRuntimeWorkspaceWarningLog(warning) {
424
- return {
425
- stream: "stdout",
426
- chunk: `[rudder] ${warning}\n`,
427
- };
428
- }
429
- function describeSessionResetReason(contextSnapshot) {
430
- if (contextSnapshot?.forceFreshSession === true)
431
- return "forceFreshSession was requested";
432
- const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason);
433
- if (wakeReason === "issue_assigned")
434
- return "wake reason is issue_assigned";
435
- return null;
436
- }
437
- function deriveCommentId(contextSnapshot, payload) {
438
- return (readNonEmptyString(contextSnapshot?.wakeCommentId) ??
439
- readNonEmptyString(contextSnapshot?.commentId) ??
440
- readNonEmptyString(payload?.commentId) ??
441
- null);
442
- }
443
- function enrichWakeContextSnapshot(input) {
444
- const { contextSnapshot, reason, source, triggerDetail, payload } = input;
445
- const issueIdFromPayload = readNonEmptyString(payload?.["issueId"]);
446
- const commentIdFromPayload = readNonEmptyString(payload?.["commentId"]);
447
- const taskKey = deriveTaskKey(contextSnapshot, payload);
448
- const wakeCommentId = deriveCommentId(contextSnapshot, payload);
449
- if (!readNonEmptyString(contextSnapshot["wakeReason"]) && reason) {
450
- contextSnapshot.wakeReason = reason;
451
- }
452
- if (!readNonEmptyString(contextSnapshot["issueId"]) && issueIdFromPayload) {
453
- contextSnapshot.issueId = issueIdFromPayload;
454
- }
455
- if (!readNonEmptyString(contextSnapshot["taskId"]) && issueIdFromPayload) {
456
- contextSnapshot.taskId = issueIdFromPayload;
457
- }
458
- if (!readNonEmptyString(contextSnapshot["taskKey"]) && taskKey) {
459
- contextSnapshot.taskKey = taskKey;
460
- }
461
- if (!readNonEmptyString(contextSnapshot["commentId"]) && commentIdFromPayload) {
462
- contextSnapshot.commentId = commentIdFromPayload;
463
- }
464
- if (!readNonEmptyString(contextSnapshot["wakeCommentId"]) && wakeCommentId) {
465
- contextSnapshot.wakeCommentId = wakeCommentId;
466
- }
467
- if (!readNonEmptyString(contextSnapshot["wakeSource"]) && source) {
468
- contextSnapshot.wakeSource = source;
469
- }
470
- if (!readNonEmptyString(contextSnapshot["wakeTriggerDetail"]) && triggerDetail) {
471
- contextSnapshot.wakeTriggerDetail = triggerDetail;
472
- }
473
- return {
474
- contextSnapshot,
475
- issueIdFromPayload,
476
- commentIdFromPayload,
477
- taskKey,
478
- wakeCommentId,
479
- };
480
- }
481
- function mergeCoalescedContextSnapshot(existingRaw, incoming) {
482
- const existing = parseObject(existingRaw);
483
- const merged = {
484
- ...existing,
485
- ...incoming,
486
- };
487
- const commentId = deriveCommentId(incoming, null);
488
- if (commentId) {
489
- merged.commentId = commentId;
490
- merged.wakeCommentId = commentId;
491
- }
492
- return merged;
493
- }
494
- function buildDeferredWakePayload(payload, contextSnapshot, issueId) {
495
- const deferredPayload = { ...(payload ?? {}) };
496
- if (issueId && !readNonEmptyString(deferredPayload.issueId)) {
497
- deferredPayload.issueId = issueId;
498
- }
499
- deferredPayload[DEFERRED_WAKE_CONTEXT_KEY] = contextSnapshot;
500
- return deferredPayload;
501
- }
502
- function readDeferredWakeContext(payloadRaw) {
503
- const payload = parseObject(payloadRaw);
504
- return parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]);
505
- }
506
- function readDeferredWakePayload(payloadRaw) {
507
- const payload = parseObject(payloadRaw);
508
- delete payload[DEFERRED_WAKE_CONTEXT_KEY];
509
- return payload;
510
- }
511
- function deriveDeferredWakeTaskKey(payloadRaw) {
512
- const payload = readDeferredWakePayload(payloadRaw);
513
- const contextSnapshot = readDeferredWakeContext(payloadRaw);
514
- return deriveTaskKey(contextSnapshot, payload);
515
- }
516
- async function hydrateWakeContextSnapshot(db, orgId, contextSnapshot) {
517
- const issueId = readNonEmptyString(contextSnapshot.issueId);
518
- const commentId = deriveCommentId(contextSnapshot, null);
519
- const issueContext = parseObject(contextSnapshot.issue);
520
- const commentContext = parseObject(contextSnapshot.comment);
521
- const needsIssueContext = !!issueId &&
522
- (!readNonEmptyString(issueContext.id) ||
523
- !readNonEmptyString(issueContext.title) ||
524
- !readNonEmptyString(issueContext.status) ||
525
- !("priority" in issueContext) ||
526
- !("description" in issueContext));
527
- const needsProjectId = !!issueId && !readNonEmptyString(contextSnapshot.projectId);
528
- const needsCommentContext = !!commentId &&
529
- (!readNonEmptyString(commentContext.id) ||
530
- !readNonEmptyString(commentContext.body));
531
- if (!needsIssueContext && !needsProjectId && !needsCommentContext)
532
- return;
533
- if (issueId && (needsIssueContext || needsProjectId)) {
534
- const issueRow = await db
535
- .select({
536
- id: issues.id,
537
- title: issues.title,
538
- description: issues.description,
539
- status: issues.status,
540
- priority: issues.priority,
541
- projectId: issues.projectId,
542
- })
543
- .from(issues)
544
- .where(and(eq(issues.id, issueId), eq(issues.orgId, orgId)))
545
- .then((rows) => rows[0] ?? null);
546
- if (issueRow) {
547
- contextSnapshot.issue = {
548
- ...issueContext,
549
- id: readNonEmptyString(issueContext.id) ?? issueRow.id,
550
- title: readNonEmptyString(issueContext.title) ?? issueRow.title,
551
- description: "description" in issueContext ? issueContext.description : issueRow.description,
552
- status: readNonEmptyString(issueContext.status) ?? issueRow.status,
553
- priority: "priority" in issueContext ? issueContext.priority : issueRow.priority,
554
- };
555
- if (!readNonEmptyString(contextSnapshot.projectId) && issueRow.projectId) {
556
- contextSnapshot.projectId = issueRow.projectId;
557
- }
558
- }
559
- }
560
- if (commentId && needsCommentContext) {
561
- const commentConditions = [eq(issueComments.id, commentId), eq(issueComments.orgId, orgId)];
562
- if (issueId) {
563
- commentConditions.push(eq(issueComments.issueId, issueId));
564
- }
565
- const commentRow = await db
566
- .select({
567
- id: issueComments.id,
568
- body: issueComments.body,
569
- authorAgentId: issueComments.authorAgentId,
570
- authorUserId: issueComments.authorUserId,
571
- })
572
- .from(issueComments)
573
- .where(and(...commentConditions))
574
- .then((rows) => rows[0] ?? null);
575
- if (commentRow) {
576
- contextSnapshot.comment = {
577
- ...commentContext,
578
- id: readNonEmptyString(commentContext.id) ?? commentRow.id,
579
- body: readNonEmptyString(commentContext.body) ?? commentRow.body,
580
- authorAgentId: "authorAgentId" in commentContext ? commentContext.authorAgentId : commentRow.authorAgentId,
581
- authorUserId: "authorUserId" in commentContext ? commentContext.authorUserId : commentRow.authorUserId,
582
- };
583
- }
584
- }
585
- }
586
- function firstNonEmptyLine(value) {
587
- if (typeof value !== "string")
588
- return null;
589
- const line = value
590
- .split("\n")
591
- .map((chunk) => chunk.trim())
592
- .find(Boolean);
593
- return line ?? null;
594
- }
595
- function deriveRecoveryFailureKind(run) {
596
- return (readNonEmptyString(run.errorCode) ??
597
- (run.status === "timed_out" ? "timed_out" : null) ??
598
- run.status);
599
- }
600
- function deriveRecoveryFailureSummary(run) {
601
- return (firstNonEmptyLine(run.error) ??
602
- firstNonEmptyLine(run.stderrExcerpt) ??
603
- firstNonEmptyLine(run.stdoutExcerpt) ??
604
- (run.status === "timed_out" ? "The run timed out before it completed." : null) ??
605
- "The previous run failed before it completed.");
606
- }
607
- function mergeMissingRecoveryContextFields(target, source) {
608
- const keysToBackfill = [
609
- "issueId",
610
- "taskId",
611
- "taskKey",
612
- "projectId",
613
- "projectWorkspaceId",
614
- "commentId",
615
- "wakeCommentId",
616
- "issue",
617
- "comment",
618
- "source",
619
- "wakeSource",
620
- "wakeTriggerDetail",
621
- ];
622
- for (const key of keysToBackfill) {
623
- if (!(key in target) || target[key] === null || target[key] === undefined || target[key] === "") {
624
- const value = source[key];
625
- if (value !== null && value !== undefined && value !== "") {
626
- target[key] = value;
627
- }
628
- }
629
- }
630
- }
631
- async function hydrateRecoveryBaseContextSnapshot(run, getRunById) {
632
- const mergedContext = { ...parseObject(run.contextSnapshot) };
633
- let ancestorRunId = readNonEmptyString(run.retryOfRunId);
634
- let depth = 0;
635
- while (ancestorRunId && depth < MAX_RECOVERY_CHAIN_DEPTH) {
636
- const ancestorRun = await getRunById(ancestorRunId);
637
- if (!ancestorRun)
638
- break;
639
- mergeMissingRecoveryContextFields(mergedContext, parseObject(ancestorRun.contextSnapshot));
640
- ancestorRunId = readNonEmptyString(ancestorRun.retryOfRunId);
641
- depth += 1;
642
- }
643
- return mergedContext;
644
- }
645
- function buildRecoveryContextSnapshot(input) {
646
- const { baseContextSnapshot, run, recoveryTrigger, wakeReason, wakeSource, triggerDetail } = input;
647
- const failureKind = deriveRecoveryFailureKind(run);
648
- const failureSummary = deriveRecoveryFailureSummary(run);
649
- const recovery = {
650
- originalRunId: run.id,
651
- failureKind,
652
- failureSummary,
653
- recoveryTrigger,
654
- recoveryMode: "continue_preferred",
655
- };
656
- return {
657
- ...baseContextSnapshot,
658
- wakeReason,
659
- wakeSource,
660
- wakeTriggerDetail: triggerDetail,
661
- retryOfRunId: run.id,
662
- retryReason: failureKind,
663
- recovery,
664
- };
665
- }
666
- function normalizePassiveFollowupContext(raw) {
667
- const parsed = parseObject(raw);
668
- const originRunId = readNonEmptyString(parsed.originRunId);
669
- if (!originRunId)
670
- return null;
671
- const attempt = Math.max(0, Math.floor(asNumber(parsed.attempt, 0)));
672
- return {
673
- originRunId,
674
- previousRunId: readNonEmptyString(parsed.previousRunId),
675
- attempt,
676
- maxAttempts: Math.max(1, Math.floor(asNumber(parsed.maxAttempts, ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS))),
677
- reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
678
- queuedAt: readNonEmptyString(parsed.queuedAt),
679
- };
680
- }
681
- function passiveFollowupCooldownMs(attempt) {
682
- return ISSUE_PASSIVE_FOLLOWUP_COOLDOWN_MS_BY_ATTEMPT.get(attempt) ?? 5 * 60 * 1000;
683
- }
684
- function isAgentEligibleForTimerContinuation(agent) {
685
- return (agent.status !== "paused" &&
686
- agent.status !== "terminated" &&
687
- agent.status !== "pending_approval");
688
- }
689
- function hasCredibleTimerContinuation(input) {
690
- if (!input.policy.enabled || input.policy.intervalSec <= 0)
691
- return false;
692
- if (!isAgentEligibleForTimerContinuation(input.agent))
693
- return false;
694
- const intervalMs = input.policy.intervalSec * 1000;
695
- const nearTermWindowMs = Math.min(intervalMs * 2, ISSUE_PASSIVE_FOLLOWUP_TIMER_CONTINUITY_MAX_WINDOW_MS);
696
- const lastHeartbeatMs = input.agent.lastHeartbeatAt
697
- ? new Date(input.agent.lastHeartbeatAt).getTime()
698
- : new Date(input.agent.createdAt).getTime();
699
- const runFinishedMs = input.run.finishedAt
700
- ? new Date(input.run.finishedAt).getTime()
701
- : input.now.getTime();
702
- const baselineMs = Math.max(lastHeartbeatMs, runFinishedMs);
703
- const nextTimerMs = baselineMs + intervalMs;
704
- return Math.max(0, nextTimerMs - input.now.getTime()) <= nearTermWindowMs;
705
- }
706
- function buildPassiveFollowupContextSnapshot(input) {
707
- const baseContext = { ...parseObject(input.run.contextSnapshot) };
708
- delete baseContext.recovery;
709
- delete baseContext.retryOfRunId;
710
- delete baseContext.retryReason;
711
- const taskKey = deriveTaskKey(baseContext, { issueId: input.issue.id }) ?? input.issue.id;
712
- return {
713
- ...baseContext,
714
- issueId: input.issue.id,
715
- taskId: input.issue.id,
716
- taskKey,
717
- projectId: readNonEmptyString(baseContext.projectId) ?? input.issue.projectId ?? undefined,
718
- wakeReason: ISSUE_PASSIVE_FOLLOWUP_REASON,
719
- wakeSource: ISSUE_PASSIVE_FOLLOWUP_WAKE_SOURCE,
720
- wakeTriggerDetail: "system",
721
- issue: {
722
- id: input.issue.id,
723
- title: input.issue.title,
724
- description: input.issue.description,
725
- status: input.issue.status,
726
- priority: input.issue.priority,
727
- },
728
- passiveFollowup: {
729
- originRunId: input.originRunId,
730
- previousRunId: input.run.id,
731
- attempt: input.attempt,
732
- maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
733
- reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
734
- queuedAt: input.now.toISOString(),
735
- },
736
- };
737
- }
738
- function runTaskKey(run) {
739
- return deriveTaskKey(run.contextSnapshot, null);
740
- }
741
- function isSameTaskScope(left, right) {
742
- return (left ?? null) === (right ?? null);
743
- }
744
- function isTrackedLocalChildProcessAdapter(agentRuntimeType) {
745
- return SESSIONED_LOCAL_ADAPTERS.has(agentRuntimeType);
746
- }
747
- // A positive liveness check means some process currently owns the PID.
748
- // On Linux, PIDs can be recycled, so this is a best-effort signal rather
749
- // than proof that the original child is still alive.
750
- function isProcessAlive(pid) {
751
- if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0)
752
- return false;
753
- try {
754
- process.kill(pid, 0);
755
- return true;
756
- }
757
- catch (error) {
758
- const code = error?.code;
759
- if (code === "EPERM")
760
- return true;
761
- if (code === "ESRCH")
762
- return false;
763
- return false;
764
- }
765
- }
766
- async function waitForProcessExit(pid, timeoutMs) {
767
- const deadline = Date.now() + Math.max(0, timeoutMs);
768
- while (Date.now() < deadline) {
769
- if (!isProcessAlive(pid))
770
- return true;
771
- await new Promise((resolve) => setTimeout(resolve, ORPHANED_PROCESS_POLL_INTERVAL_MS));
772
- }
773
- return !isProcessAlive(pid);
774
- }
775
- async function terminateOrphanedProcess(pid) {
776
- if (!isProcessAlive(pid)) {
777
- return {
778
- stillAlive: false,
779
- terminationSignal: null,
780
- error: null,
781
- };
782
- }
783
- let terminationSignal = null;
784
- try {
785
- process.kill(pid, "SIGTERM");
786
- terminationSignal = "SIGTERM";
787
- }
788
- catch (error) {
789
- const code = error?.code;
790
- if (code === "ESRCH") {
791
- return {
792
- stillAlive: false,
793
- terminationSignal: null,
794
- error: null,
795
- };
796
- }
797
- return {
798
- stillAlive: isProcessAlive(pid),
799
- terminationSignal,
800
- error: `SIGTERM failed: ${error instanceof Error ? error.message : String(error)}`,
801
- };
802
- }
803
- if (await waitForProcessExit(pid, ORPHANED_PROCESS_TERMINATION_GRACE_MS)) {
804
- return {
805
- stillAlive: false,
806
- terminationSignal,
807
- error: null,
808
- };
809
- }
810
- try {
811
- process.kill(pid, "SIGKILL");
812
- terminationSignal = "SIGKILL";
813
- }
814
- catch (error) {
815
- const code = error?.code;
816
- if (code === "ESRCH") {
817
- return {
818
- stillAlive: false,
819
- terminationSignal,
820
- error: null,
821
- };
822
- }
823
- return {
824
- stillAlive: isProcessAlive(pid),
825
- terminationSignal,
826
- error: `SIGKILL failed: ${error instanceof Error ? error.message : String(error)}`,
827
- };
828
- }
829
- const exitedAfterKill = await waitForProcessExit(pid, ORPHANED_PROCESS_KILL_WAIT_MS);
830
- return {
831
- stillAlive: !exitedAfterKill,
832
- terminationSignal,
833
- error: exitedAfterKill ? null : `Timed out waiting for child pid ${pid} to exit after ${terminationSignal}`,
834
- };
835
- }
836
- function truncateDisplayId(value, max = 128) {
837
- if (!value)
838
- return null;
839
- return value.length > max ? value.slice(0, max) : value;
840
- }
841
- function normalizeAgentNameKey(value) {
842
- if (typeof value !== "string")
843
- return null;
844
- const normalized = value.trim().toLowerCase();
845
- return normalized.length > 0 ? normalized : null;
846
- }
847
- const defaultSessionCodec = {
848
- deserialize(raw) {
849
- const asObj = parseObject(raw);
850
- if (Object.keys(asObj).length > 0)
851
- return asObj;
852
- const sessionId = readNonEmptyString(raw?.sessionId);
853
- if (sessionId)
854
- return { sessionId };
855
- return null;
856
- },
857
- serialize(params) {
858
- if (!params || Object.keys(params).length === 0)
859
- return null;
860
- return params;
861
- },
862
- getDisplayId(params) {
863
- return readNonEmptyString(params?.sessionId);
864
- },
865
- };
866
- function getAgentRuntimeSessionCodec(agentRuntimeType) {
867
- const adapter = getServerAdapter(agentRuntimeType);
868
- return adapter.sessionCodec ?? defaultSessionCodec;
869
- }
870
- function normalizeSessionParams(params) {
871
- if (!params)
872
- return null;
873
- return Object.keys(params).length > 0 ? params : null;
874
- }
875
- function resolveNextSessionState(input) {
876
- const { codec, adapterResult, previousParams, previousDisplayId, previousLegacySessionId } = input;
877
- if (adapterResult.clearSession) {
878
- return {
879
- params: null,
880
- displayId: null,
881
- legacySessionId: null,
882
- };
883
- }
884
- const explicitParams = adapterResult.sessionParams;
885
- const hasExplicitParams = adapterResult.sessionParams !== undefined;
886
- const hasExplicitSessionId = adapterResult.sessionId !== undefined;
887
- const explicitSessionId = readNonEmptyString(adapterResult.sessionId);
888
- const hasExplicitDisplay = adapterResult.sessionDisplayId !== undefined;
889
- const explicitDisplayId = readNonEmptyString(adapterResult.sessionDisplayId);
890
- const shouldUsePrevious = !hasExplicitParams && !hasExplicitSessionId && !hasExplicitDisplay;
891
- const candidateParams = hasExplicitParams
892
- ? explicitParams
893
- : hasExplicitSessionId
894
- ? (explicitSessionId ? { sessionId: explicitSessionId } : null)
895
- : previousParams;
896
- const serialized = normalizeSessionParams(codec.serialize(normalizeSessionParams(candidateParams) ?? null));
897
- const deserialized = normalizeSessionParams(codec.deserialize(serialized));
898
- const displayId = truncateDisplayId(explicitDisplayId ??
899
- (codec.getDisplayId ? codec.getDisplayId(deserialized) : null) ??
900
- readNonEmptyString(deserialized?.sessionId) ??
901
- (shouldUsePrevious ? previousDisplayId : null) ??
902
- explicitSessionId ??
903
- (shouldUsePrevious ? previousLegacySessionId : null));
904
- const legacySessionId = explicitSessionId ??
905
- readNonEmptyString(deserialized?.sessionId) ??
906
- displayId ??
907
- (shouldUsePrevious ? previousLegacySessionId : null);
908
- return {
909
- params: serialized,
910
- displayId,
911
- legacySessionId,
912
- };
913
- }
914
- export function heartbeatService(db) {
915
- const instanceSettings = instanceSettingsService(db);
916
- const getCurrentUserRedactionOptions = async () => ({
917
- enabled: (await instanceSettings.getGeneral()).censorUsernameInLogs,
918
- });
919
- const runLogStore = getRunLogStore();
920
- const runContextSvc = agentRunContextService(db);
921
- const issuesSvc = issueService(db);
922
- const executionWorkspacesSvc = executionWorkspaceService(db);
923
- const workspaceOperationsSvc = workspaceOperationService(db);
924
- const activeRunExecutions = new Set();
925
- const budgetHooks = {
926
- cancelWorkForScope: cancelBudgetScopeWork,
927
- };
928
- const budgets = budgetService(db, budgetHooks);
929
- async function getAgent(agentId) {
930
- return db
931
- .select()
932
- .from(agents)
933
- .where(eq(agents.id, agentId))
934
- .then((rows) => rows[0] ?? null);
935
- }
936
- async function getRun(runId) {
937
- return db
938
- .select()
939
- .from(heartbeatRuns)
940
- .where(eq(heartbeatRuns.id, runId))
941
- .then((rows) => rows[0] ?? null);
942
- }
943
- async function getRuntimeState(agentId) {
944
- return db
945
- .select()
946
- .from(agentRuntimeState)
947
- .where(eq(agentRuntimeState.agentId, agentId))
948
- .then((rows) => rows[0] ?? null);
949
- }
950
- async function getTaskSession(orgId, agentId, agentRuntimeType, taskKey) {
951
- return db
952
- .select()
953
- .from(agentTaskSessions)
954
- .where(and(eq(agentTaskSessions.orgId, orgId), eq(agentTaskSessions.agentId, agentId), eq(agentTaskSessions.agentRuntimeType, agentRuntimeType), eq(agentTaskSessions.taskKey, taskKey)))
955
- .then((rows) => rows[0] ?? null);
956
- }
957
- async function getLatestRunForSession(agentId, sessionId, opts) {
958
- const conditions = [
959
- eq(heartbeatRuns.agentId, agentId),
960
- eq(heartbeatRuns.sessionIdAfter, sessionId),
961
- ];
962
- if (opts?.excludeRunId) {
963
- conditions.push(sql `${heartbeatRuns.id} <> ${opts.excludeRunId}`);
964
- }
965
- return db
966
- .select()
967
- .from(heartbeatRuns)
968
- .where(and(...conditions))
969
- .orderBy(desc(heartbeatRuns.createdAt))
970
- .limit(1)
971
- .then((rows) => rows[0] ?? null);
972
- }
973
- async function getOldestRunForSession(agentId, sessionId) {
974
- return db
975
- .select({
976
- id: heartbeatRuns.id,
977
- createdAt: heartbeatRuns.createdAt,
978
- })
979
- .from(heartbeatRuns)
980
- .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.sessionIdAfter, sessionId)))
981
- .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id))
982
- .limit(1)
983
- .then((rows) => rows[0] ?? null);
984
- }
985
- async function resolveNormalizedUsageForSession(input) {
986
- const { agentId, runId, sessionId, rawUsage } = input;
987
- if (!sessionId || !rawUsage) {
988
- return {
989
- normalizedUsage: rawUsage,
990
- previousRawUsage: null,
991
- derivedFromSessionTotals: false,
992
- };
993
- }
994
- const previousRun = await getLatestRunForSession(agentId, sessionId, { excludeRunId: runId });
995
- const previousRawUsage = readRawUsageTotals(previousRun?.usageJson);
996
- return {
997
- normalizedUsage: deriveNormalizedUsageDelta(rawUsage, previousRawUsage),
998
- previousRawUsage,
999
- derivedFromSessionTotals: previousRawUsage !== null,
1000
- };
1001
- }
1002
- async function evaluateSessionCompaction(input) {
1003
- const { agent, sessionId, issueId } = input;
1004
- if (!sessionId) {
1005
- return {
1006
- rotate: false,
1007
- reason: null,
1008
- handoffMarkdown: null,
1009
- previousRunId: null,
1010
- };
1011
- }
1012
- const policy = parseSessionCompactionPolicy(agent);
1013
- if (!policy.enabled || !hasSessionCompactionThresholds(policy)) {
1014
- return {
1015
- rotate: false,
1016
- reason: null,
1017
- handoffMarkdown: null,
1018
- previousRunId: null,
1019
- };
1020
- }
1021
- const fetchLimit = Math.max(policy.maxSessionRuns > 0 ? policy.maxSessionRuns + 1 : 0, 4);
1022
- const runs = await db
1023
- .select({
1024
- id: heartbeatRuns.id,
1025
- createdAt: heartbeatRuns.createdAt,
1026
- usageJson: heartbeatRuns.usageJson,
1027
- resultJson: heartbeatRuns.resultJson,
1028
- error: heartbeatRuns.error,
1029
- })
1030
- .from(heartbeatRuns)
1031
- .where(and(eq(heartbeatRuns.agentId, agent.id), eq(heartbeatRuns.sessionIdAfter, sessionId)))
1032
- .orderBy(desc(heartbeatRuns.createdAt))
1033
- .limit(fetchLimit);
1034
- if (runs.length === 0) {
1035
- return {
1036
- rotate: false,
1037
- reason: null,
1038
- handoffMarkdown: null,
1039
- previousRunId: null,
1040
- };
1041
- }
1042
- const latestRun = runs[0] ?? null;
1043
- const oldestRun = policy.maxSessionAgeHours > 0
1044
- ? await getOldestRunForSession(agent.id, sessionId)
1045
- : runs[runs.length - 1] ?? latestRun;
1046
- const latestRawUsage = readRawUsageTotals(latestRun?.usageJson);
1047
- const sessionAgeHours = latestRun && oldestRun
1048
- ? Math.max(0, (new Date(latestRun.createdAt).getTime() - new Date(oldestRun.createdAt).getTime()) / (1000 * 60 * 60))
1049
- : 0;
1050
- let reason = null;
1051
- if (policy.maxSessionRuns > 0 && runs.length > policy.maxSessionRuns) {
1052
- reason = `session exceeded ${policy.maxSessionRuns} runs`;
1053
- }
1054
- else if (policy.maxRawInputTokens > 0 &&
1055
- latestRawUsage &&
1056
- latestRawUsage.inputTokens >= policy.maxRawInputTokens) {
1057
- reason =
1058
- `session raw input reached ${formatCount(latestRawUsage.inputTokens)} tokens ` +
1059
- `(threshold ${formatCount(policy.maxRawInputTokens)})`;
1060
- }
1061
- else if (policy.maxSessionAgeHours > 0 && sessionAgeHours >= policy.maxSessionAgeHours) {
1062
- reason = `session age reached ${Math.floor(sessionAgeHours)} hours`;
1063
- }
1064
- if (!reason || !latestRun) {
1065
- return {
1066
- rotate: false,
1067
- reason: null,
1068
- handoffMarkdown: null,
1069
- previousRunId: latestRun?.id ?? null,
1070
- };
1071
- }
1072
- const latestSummary = summarizeHeartbeatRunResultJson(latestRun.resultJson);
1073
- const latestTextSummary = readNonEmptyString(latestSummary?.summary) ??
1074
- readNonEmptyString(latestSummary?.result) ??
1075
- readNonEmptyString(latestSummary?.message) ??
1076
- readNonEmptyString(latestRun.error);
1077
- const handoffMarkdown = [
1078
- "Rudder session handoff:",
1079
- `- Previous session: ${sessionId}`,
1080
- issueId ? `- Issue: ${issueId}` : "",
1081
- `- Rotation reason: ${reason}`,
1082
- latestTextSummary ? `- Last run summary: ${latestTextSummary}` : "",
1083
- "Continue from the current task state. Rebuild only the minimum context you need.",
1084
- ]
1085
- .filter(Boolean)
1086
- .join("\n");
1087
- return {
1088
- rotate: true,
1089
- reason,
1090
- handoffMarkdown,
1091
- previousRunId: latestRun.id,
1092
- };
1093
- }
1094
- async function resolveSessionBeforeForWakeup(agent, taskKey) {
1095
- if (taskKey) {
1096
- const codec = getAgentRuntimeSessionCodec(agent.agentRuntimeType);
1097
- const existingTaskSession = await getTaskSession(agent.orgId, agent.id, agent.agentRuntimeType, taskKey);
1098
- const parsedParams = normalizeSessionParams(codec.deserialize(existingTaskSession?.sessionParamsJson ?? null));
1099
- return truncateDisplayId(existingTaskSession?.sessionDisplayId ??
1100
- (codec.getDisplayId ? codec.getDisplayId(parsedParams) : null) ??
1101
- readNonEmptyString(parsedParams?.sessionId));
1102
- }
1103
- const runtimeForRun = await getRuntimeState(agent.id);
1104
- return runtimeForRun?.sessionId ?? null;
1105
- }
1106
- async function resolveExplicitResumeSessionOverride(agent, payload, taskKey) {
1107
- const resumeFromRunId = readNonEmptyString(payload?.resumeFromRunId);
1108
- if (!resumeFromRunId)
1109
- return null;
1110
- const resumeRun = await db
1111
- .select({
1112
- id: heartbeatRuns.id,
1113
- contextSnapshot: heartbeatRuns.contextSnapshot,
1114
- sessionIdBefore: heartbeatRuns.sessionIdBefore,
1115
- sessionIdAfter: heartbeatRuns.sessionIdAfter,
1116
- })
1117
- .from(heartbeatRuns)
1118
- .where(and(eq(heartbeatRuns.id, resumeFromRunId), eq(heartbeatRuns.orgId, agent.orgId), eq(heartbeatRuns.agentId, agent.id)))
1119
- .then((rows) => rows[0] ?? null);
1120
- if (!resumeRun)
1121
- return null;
1122
- const resumeContext = parseObject(resumeRun.contextSnapshot);
1123
- const resumeTaskKey = deriveTaskKey(resumeContext, null) ?? taskKey;
1124
- const resumeTaskSession = resumeTaskKey
1125
- ? await getTaskSession(agent.orgId, agent.id, agent.agentRuntimeType, resumeTaskKey)
1126
- : null;
1127
- const sessionCodec = getAgentRuntimeSessionCodec(agent.agentRuntimeType);
1128
- const sessionOverride = buildExplicitResumeSessionOverride({
1129
- resumeFromRunId,
1130
- resumeRunSessionIdBefore: resumeRun.sessionIdBefore,
1131
- resumeRunSessionIdAfter: resumeRun.sessionIdAfter,
1132
- taskSession: resumeTaskSession,
1133
- sessionCodec,
1134
- });
1135
- if (!sessionOverride)
1136
- return null;
1137
- return {
1138
- resumeFromRunId,
1139
- taskKey: resumeTaskKey,
1140
- issueId: readNonEmptyString(resumeContext.issueId),
1141
- taskId: readNonEmptyString(resumeContext.taskId) ?? readNonEmptyString(resumeContext.issueId),
1142
- sessionDisplayId: sessionOverride.sessionDisplayId,
1143
- sessionParams: sessionOverride.sessionParams,
1144
- };
1145
- }
1146
- async function upsertTaskSession(input) {
1147
- const existing = await getTaskSession(input.orgId, input.agentId, input.agentRuntimeType, input.taskKey);
1148
- if (existing) {
1149
- return db
1150
- .update(agentTaskSessions)
1151
- .set({
1152
- sessionParamsJson: input.sessionParamsJson,
1153
- sessionDisplayId: input.sessionDisplayId,
1154
- lastRunId: input.lastRunId,
1155
- lastError: input.lastError,
1156
- updatedAt: new Date(),
1157
- })
1158
- .where(eq(agentTaskSessions.id, existing.id))
1159
- .returning()
1160
- .then((rows) => rows[0] ?? null);
1161
- }
1162
- return db
1163
- .insert(agentTaskSessions)
1164
- .values({
1165
- orgId: input.orgId,
1166
- agentId: input.agentId,
1167
- agentRuntimeType: input.agentRuntimeType,
1168
- taskKey: input.taskKey,
1169
- sessionParamsJson: input.sessionParamsJson,
1170
- sessionDisplayId: input.sessionDisplayId,
1171
- lastRunId: input.lastRunId,
1172
- lastError: input.lastError,
1173
- })
1174
- .returning()
1175
- .then((rows) => rows[0] ?? null);
1176
- }
1177
- async function clearTaskSessions(orgId, agentId, opts) {
1178
- const conditions = [
1179
- eq(agentTaskSessions.orgId, orgId),
1180
- eq(agentTaskSessions.agentId, agentId),
1181
- ];
1182
- if (opts?.taskKey) {
1183
- conditions.push(eq(agentTaskSessions.taskKey, opts.taskKey));
1184
- }
1185
- if (opts?.agentRuntimeType) {
1186
- conditions.push(eq(agentTaskSessions.agentRuntimeType, opts.agentRuntimeType));
1187
- }
1188
- return db
1189
- .delete(agentTaskSessions)
1190
- .where(and(...conditions))
1191
- .returning()
1192
- .then((rows) => rows.length);
1193
- }
1194
- async function ensureRuntimeState(agent) {
1195
- const existing = await getRuntimeState(agent.id);
1196
- if (existing)
1197
- return existing;
1198
- return db
1199
- .insert(agentRuntimeState)
1200
- .values({
1201
- agentId: agent.id,
1202
- orgId: agent.orgId,
1203
- agentRuntimeType: agent.agentRuntimeType,
1204
- stateJson: {},
1205
- })
1206
- .returning()
1207
- .then((rows) => rows[0]);
1208
- }
1209
- function buildHeartbeatObservabilityContext(run, overrides = {}) {
1210
- const contextSnapshot = parseObject(run.contextSnapshot);
1211
- const issueSnapshot = parseObject(contextSnapshot.issue);
1212
- const benchmarkMetadata = extractCreateAgentBenchmarkMetadata(readNonEmptyString(issueSnapshot.description))
1213
- ?? coerceCreateAgentBenchmarkMetadata(parseObject(contextSnapshot.benchmark));
1214
- const baseMetadata = {
1215
- wakeupRequestId: run.wakeupRequestId,
1216
- errorCode: run.errorCode,
1217
- retryOfRunId: run.retryOfRunId,
1218
- processLossRetryCount: run.processLossRetryCount,
1219
- externalRunId: run.externalRunId,
1220
- executionWorkspaceId: readNonEmptyString(contextSnapshot.executionWorkspaceId),
1221
- ...(benchmarkMetadata ?? {}),
1222
- };
1223
- const benchmarkTags = benchmarkMetadata ? buildCreateAgentBenchmarkTags(benchmarkMetadata) : [];
1224
- return {
1225
- surface: resolveHeartbeatObservabilitySurface(contextSnapshot),
1226
- rootExecutionId: run.id,
1227
- orgId: run.orgId,
1228
- agentId: run.agentId,
1229
- issueId: readNonEmptyString(contextSnapshot.issueId),
1230
- sessionKey: run.sessionIdAfter ??
1231
- run.sessionIdBefore ??
1232
- readNonEmptyString(contextSnapshot.sessionKey) ??
1233
- readNonEmptyString(contextSnapshot.taskKey),
1234
- runtime: readNonEmptyString(contextSnapshot.agentRuntimeType),
1235
- trigger: run.triggerDetail ?? run.invocationSource,
1236
- status: run.status,
1237
- metadata: {
1238
- ...baseMetadata,
1239
- ...(overrides.metadata ?? {}),
1240
- },
1241
- tags: [...benchmarkTags, ...(overrides.tags ?? [])],
1242
- ...overrides,
1243
- };
1244
- }
1245
- async function emitHeartbeatObservationEvent(run, input, overrides = {}) {
1246
- try {
1247
- await observeExecutionEvent(buildHeartbeatObservabilityContext(run, overrides), input);
1248
- }
1249
- catch (error) {
1250
- logger.warn({
1251
- runId: run.id,
1252
- eventName: input.name,
1253
- err: error instanceof Error ? error.message : String(error),
1254
- }, "Failed to emit Langfuse heartbeat event");
1255
- }
1256
- }
1257
- async function emitHeartbeatLiveEval(runId) {
1258
- try {
1259
- const { detail, scores } = await buildObservedRunLangfuseScores(db, runId);
1260
- await createExecutionScores(buildHeartbeatObservabilityContext(detail.run, {
1261
- runtime: detail.bundle.agentRuntimeType,
1262
- metadata: {
1263
- agentName: detail.agentName,
1264
- orgName: detail.orgName,
1265
- },
1266
- }), scores.map((score) => ({
1267
- rootExecutionId: detail.run.id,
1268
- name: score.name,
1269
- value: score.value,
1270
- comment: score.comment,
1271
- metadata: score.metadata,
1272
- })));
1273
- }
1274
- catch (error) {
1275
- logger.warn({
1276
- runId,
1277
- err: error instanceof Error ? error.message : String(error),
1278
- }, "Failed to emit Langfuse heartbeat scores");
1279
- }
1280
- }
1281
- async function setRunStatus(runId, status, patch) {
1282
- const updated = await db
1283
- .update(heartbeatRuns)
1284
- .set({ status, ...patch, updatedAt: new Date() })
1285
- .where(eq(heartbeatRuns.id, runId))
1286
- .returning()
1287
- .then((rows) => rows[0] ?? null);
1288
- if (updated) {
1289
- publishLiveEvent({
1290
- orgId: updated.orgId,
1291
- type: "heartbeat.run.status",
1292
- payload: {
1293
- runId: updated.id,
1294
- agentId: updated.agentId,
1295
- status: updated.status,
1296
- invocationSource: updated.invocationSource,
1297
- triggerDetail: updated.triggerDetail,
1298
- error: updated.error ?? null,
1299
- errorCode: updated.errorCode ?? null,
1300
- startedAt: updated.startedAt ? new Date(updated.startedAt).toISOString() : null,
1301
- finishedAt: updated.finishedAt ? new Date(updated.finishedAt).toISOString() : null,
1302
- },
1303
- });
1304
- await emitHeartbeatObservationEvent(updated, {
1305
- name: `heartbeat.status.${status}`,
1306
- asType: "event",
1307
- output: {
1308
- status: updated.status,
1309
- error: updated.error,
1310
- errorCode: updated.errorCode,
1311
- startedAt: updated.startedAt ? new Date(updated.startedAt).toISOString() : null,
1312
- finishedAt: updated.finishedAt ? new Date(updated.finishedAt).toISOString() : null,
1313
- },
1314
- }, {
1315
- status: updated.status,
1316
- });
1317
- }
1318
- return updated;
1319
- }
1320
- async function setWakeupStatus(wakeupRequestId, status, patch) {
1321
- if (!wakeupRequestId)
1322
- return;
1323
- await db
1324
- .update(agentWakeupRequests)
1325
- .set({ status, ...patch, updatedAt: new Date() })
1326
- .where(eq(agentWakeupRequests.id, wakeupRequestId));
1327
- }
1328
- async function updateWakeupRequestRecord(tx, wakeupRequestId, patch) {
1329
- return tx
1330
- .update(agentWakeupRequests)
1331
- .set({ ...patch, updatedAt: new Date() })
1332
- .where(eq(agentWakeupRequests.id, wakeupRequestId))
1333
- .returning()
1334
- .then((rows) => rows[0] ?? null);
1335
- }
1336
- async function insertWakeupRequestRecord(tx, values) {
1337
- return tx
1338
- .insert(agentWakeupRequests)
1339
- .values(values)
1340
- .returning()
1341
- .then((rows) => rows[0] ?? null);
1342
- }
1343
- async function appendRunEvent(run, seq, event) {
1344
- const currentUserRedactionOptions = await getCurrentUserRedactionOptions();
1345
- const sanitizedMessage = event.message
1346
- ? redactCurrentUserText(event.message, currentUserRedactionOptions)
1347
- : event.message;
1348
- const sanitizedPayload = event.payload
1349
- ? redactCurrentUserValue(event.payload, currentUserRedactionOptions)
1350
- : event.payload;
1351
- await db.insert(heartbeatRunEvents).values({
1352
- orgId: run.orgId,
1353
- runId: run.id,
1354
- agentId: run.agentId,
1355
- seq,
1356
- eventType: event.eventType,
1357
- stream: event.stream,
1358
- level: event.level,
1359
- color: event.color,
1360
- message: sanitizedMessage,
1361
- payload: sanitizedPayload,
1362
- });
1363
- publishLiveEvent({
1364
- orgId: run.orgId,
1365
- type: "heartbeat.run.event",
1366
- payload: {
1367
- runId: run.id,
1368
- agentId: run.agentId,
1369
- seq,
1370
- eventType: event.eventType,
1371
- stream: event.stream ?? null,
1372
- level: event.level ?? null,
1373
- color: event.color ?? null,
1374
- message: sanitizedMessage ?? null,
1375
- payload: sanitizedPayload ?? null,
1376
- },
1377
- });
1378
- await emitHeartbeatObservationEvent(run, {
1379
- name: `heartbeat.event.${event.eventType}`,
1380
- asType: "event",
1381
- level: event.level === "error" ? "ERROR" : event.level === "warn" ? "WARNING" : "DEFAULT",
1382
- output: {
1383
- seq,
1384
- eventType: event.eventType,
1385
- stream: event.stream ?? null,
1386
- level: event.level ?? null,
1387
- color: event.color ?? null,
1388
- message: sanitizedMessage ?? null,
1389
- },
1390
- metadata: sanitizedPayload ?? undefined,
1391
- }, {
1392
- status: run.status,
1393
- });
1394
- }
1395
- async function nextRunEventSeq(runId) {
1396
- const [row] = await db
1397
- .select({ maxSeq: sql `max(${heartbeatRunEvents.seq})` })
1398
- .from(heartbeatRunEvents)
1399
- .where(eq(heartbeatRunEvents.runId, runId));
1400
- return Number(row?.maxSeq ?? 0) + 1;
1401
- }
1402
- async function persistRunProcessMetadata(runId, meta) {
1403
- const startedAt = new Date(meta.startedAt);
1404
- const updated = await db
1405
- .update(heartbeatRuns)
1406
- .set({
1407
- processPid: meta.pid,
1408
- processStartedAt: Number.isNaN(startedAt.getTime()) ? new Date() : startedAt,
1409
- updatedAt: new Date(),
1410
- })
1411
- .where(eq(heartbeatRuns.id, runId))
1412
- .returning()
1413
- .then((rows) => rows[0] ?? null);
1414
- if (updated) {
1415
- await emitHeartbeatObservationEvent(updated, {
1416
- name: "heartbeat.process.spawn",
1417
- asType: "event",
1418
- output: {
1419
- pid: meta.pid,
1420
- startedAt: meta.startedAt,
1421
- },
1422
- });
1423
- }
1424
- return updated;
1425
- }
1426
- async function clearDetachedRunWarning(runId) {
1427
- const updated = await db
1428
- .update(heartbeatRuns)
1429
- .set({
1430
- error: null,
1431
- errorCode: null,
1432
- updatedAt: new Date(),
1433
- })
1434
- .where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.errorCode, DETACHED_PROCESS_ERROR_CODE)))
1435
- .returning()
1436
- .then((rows) => rows[0] ?? null);
1437
- if (!updated)
1438
- return null;
1439
- await appendRunEvent(updated, await nextRunEventSeq(updated.id), {
1440
- eventType: "lifecycle",
1441
- stream: "system",
1442
- level: "info",
1443
- message: "Detached child process reported activity; cleared detached warning",
1444
- });
1445
- return updated;
1446
- }
1447
- async function enqueueRecoveryRun(run, agent, opts) {
1448
- /**
1449
- * Recovery runs intentionally clone the prior run's task context and then
1450
- * layer explicit recovery metadata on top. This keeps retries visible and
1451
- * auditable while preserving "continue preferred" semantics for issue work.
1452
- *
1453
- * Reasoning:
1454
- * - Manual retry and automatic process-loss retry must assemble the same
1455
- * recovery contract so prompts/runtime behavior stay aligned.
1456
- * - We backfill missing context from the retry chain to recover from older
1457
- * lossy retry runs without mutating the historical source run rows.
1458
- *
1459
- * Traceability:
1460
- * - doc/developing/RUN-RECOVERY.md
1461
- * - doc/DEVELOPING.md
1462
- */
1463
- const baseContextSnapshot = await hydrateRecoveryBaseContextSnapshot(run, getRun);
1464
- const recoveryContextSnapshot = buildRecoveryContextSnapshot({
1465
- baseContextSnapshot,
1466
- run,
1467
- recoveryTrigger: opts.recoveryTrigger,
1468
- wakeReason: opts.wakeReason,
1469
- wakeSource: `recovery.${opts.recoveryTrigger}`,
1470
- triggerDetail: opts.triggerDetail,
1471
- });
1472
- const issueId = readNonEmptyString(recoveryContextSnapshot.issueId);
1473
- const taskKey = deriveTaskKey(recoveryContextSnapshot, null);
1474
- const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey);
1475
- const recovery = recoveryContextSnapshot.recovery;
1476
- const requestPayload = {
1477
- originalRunId: run.id,
1478
- failureKind: recovery.failureKind,
1479
- recoveryTrigger: recovery.recoveryTrigger,
1480
- ...(issueId ? { issueId } : {}),
1481
- };
1482
- const outcome = await db.transaction(async (tx) => {
1483
- let issueRow = null;
1484
- if (issueId) {
1485
- await tx.execute(sql `select id from issues where id = ${issueId} and org_id = ${run.orgId} for update`);
1486
- issueRow = await tx
1487
- .select({
1488
- id: issues.id,
1489
- orgId: issues.orgId,
1490
- executionRunId: issues.executionRunId,
1491
- executionAgentNameKey: issues.executionAgentNameKey,
1492
- })
1493
- .from(issues)
1494
- .where(and(eq(issues.id, issueId), eq(issues.orgId, run.orgId)))
1495
- .then((rows) => rows[0] ?? null);
1496
- }
1497
- if (issueRow?.executionRunId) {
1498
- const activeExecutionRun = await tx
1499
- .select()
1500
- .from(heartbeatRuns)
1501
- .where(eq(heartbeatRuns.id, issueRow.executionRunId))
1502
- .then((rows) => rows[0] ?? null);
1503
- const isActiveExecutionRun = activeExecutionRun &&
1504
- (activeExecutionRun.status === "queued" || activeExecutionRun.status === "running");
1505
- if (!isActiveExecutionRun) {
1506
- await tx
1507
- .update(issues)
1508
- .set({
1509
- executionRunId: null,
1510
- executionAgentNameKey: null,
1511
- executionLockedAt: null,
1512
- updatedAt: opts.now,
1513
- })
1514
- .where(eq(issues.id, issueRow.id));
1515
- issueRow = {
1516
- ...issueRow,
1517
- executionRunId: null,
1518
- executionAgentNameKey: null,
1519
- };
1520
- }
1521
- else if (activeExecutionRun) {
1522
- const activeContext = parseObject(activeExecutionRun.contextSnapshot);
1523
- const activeRecovery = parseObject(activeContext.recovery);
1524
- if (activeExecutionRun.agentId === run.agentId &&
1525
- (activeExecutionRun.retryOfRunId === run.id ||
1526
- readNonEmptyString(activeRecovery.originalRunId) === run.id)) {
1527
- return { kind: "existing", run: activeExecutionRun };
1528
- }
1529
- throw conflict("Issue already has an active execution run", {
1530
- issueId: issueRow.id,
1531
- executionRunId: activeExecutionRun.id,
1532
- });
1533
- }
1534
- }
1535
- const wakeupRequest = await tx
1536
- .insert(agentWakeupRequests)
1537
- .values({
1538
- orgId: run.orgId,
1539
- agentId: run.agentId,
1540
- source: opts.source,
1541
- triggerDetail: opts.triggerDetail,
1542
- reason: opts.wakeReason,
1543
- payload: requestPayload,
1544
- status: "queued",
1545
- requestedByActorType: opts.requestedByActorType ?? null,
1546
- requestedByActorId: opts.requestedByActorId ?? null,
1547
- updatedAt: opts.now,
1548
- })
1549
- .returning()
1550
- .then((rows) => rows[0]);
1551
- const recoveryRun = await tx
1552
- .insert(heartbeatRuns)
1553
- .values({
1554
- orgId: run.orgId,
1555
- agentId: run.agentId,
1556
- invocationSource: opts.source,
1557
- triggerDetail: opts.triggerDetail,
1558
- status: "queued",
1559
- wakeupRequestId: wakeupRequest.id,
1560
- contextSnapshot: recoveryContextSnapshot,
1561
- sessionIdBefore: sessionBefore,
1562
- retryOfRunId: run.id,
1563
- processLossRetryCount: opts.recoveryTrigger === "automatic"
1564
- ? (run.processLossRetryCount ?? 0) + 1
1565
- : (run.processLossRetryCount ?? 0),
1566
- updatedAt: opts.now,
1567
- })
1568
- .returning()
1569
- .then((rows) => rows[0]);
1570
- await tx
1571
- .update(agentWakeupRequests)
1572
- .set({
1573
- runId: recoveryRun.id,
1574
- updatedAt: opts.now,
1575
- })
1576
- .where(eq(agentWakeupRequests.id, wakeupRequest.id));
1577
- if (issueRow) {
1578
- await tx
1579
- .update(issues)
1580
- .set({
1581
- executionRunId: recoveryRun.id,
1582
- executionAgentNameKey: normalizeAgentNameKey(agent.name),
1583
- executionLockedAt: opts.now,
1584
- updatedAt: opts.now,
1585
- })
1586
- .where(eq(issues.id, issueRow.id));
1587
- }
1588
- return { kind: "queued", run: recoveryRun };
1589
- });
1590
- if (outcome.kind === "existing")
1591
- return outcome.run;
1592
- const recoveryRun = outcome.run;
1593
- await appendRunEvent(recoveryRun, await nextRunEventSeq(recoveryRun.id), {
1594
- eventType: "lifecycle",
1595
- stream: "system",
1596
- level: opts.recoveryTrigger === "automatic" ? "warn" : "info",
1597
- message: `Recovery queued from run ${run.id}`,
1598
- payload: {
1599
- originalRunId: run.id,
1600
- failureKind: recovery.failureKind,
1601
- failureSummary: recovery.failureSummary,
1602
- recoveryTrigger: recovery.recoveryTrigger,
1603
- recoveryMode: recovery.recoveryMode,
1604
- },
1605
- });
1606
- publishLiveEvent({
1607
- orgId: recoveryRun.orgId,
1608
- type: "heartbeat.run.queued",
1609
- payload: {
1610
- runId: recoveryRun.id,
1611
- agentId: recoveryRun.agentId,
1612
- invocationSource: recoveryRun.invocationSource,
1613
- triggerDetail: recoveryRun.triggerDetail,
1614
- wakeupRequestId: recoveryRun.wakeupRequestId,
1615
- },
1616
- });
1617
- if (opts.startImmediately !== false) {
1618
- await startNextQueuedRunForAgent(agent.id);
1619
- }
1620
- return recoveryRun;
1621
- }
1622
- async function enqueueProcessLossRetry(run, agent, now) {
1623
- return enqueueRecoveryRun(run, agent, {
1624
- recoveryTrigger: "automatic",
1625
- source: "automation",
1626
- triggerDetail: "system",
1627
- wakeReason: "process_lost_retry",
1628
- requestedByActorType: "system",
1629
- requestedByActorId: null,
1630
- startImmediately: false,
1631
- now,
1632
- });
1633
- }
1634
- function parseHeartbeatPolicy(agent) {
1635
- const runtimeConfig = parseObject(agent.runtimeConfig);
1636
- const heartbeat = parseObject(runtimeConfig.heartbeat);
1637
- return {
1638
- enabled: asBoolean(heartbeat.enabled, true),
1639
- intervalSec: Math.max(0, asNumber(heartbeat.intervalSec, 0)),
1640
- wakeOnDemand: asBoolean(heartbeat.wakeOnDemand ?? heartbeat.wakeOnAssignment ?? heartbeat.wakeOnOnDemand ?? heartbeat.wakeOnAutomation, true),
1641
- maxConcurrentRuns: normalizeMaxConcurrentRuns(heartbeat.maxConcurrentRuns),
1642
- };
1643
- }
1644
- async function runHasIssueClosureComment(tx, run, issueId) {
1645
- const commentActivity = await tx
1646
- .select({ id: activityLog.id })
1647
- .from(activityLog)
1648
- .where(and(eq(activityLog.orgId, run.orgId), eq(activityLog.action, "issue.comment_added"), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issueId), eq(activityLog.runId, run.id)))
1649
- .limit(1)
1650
- .then((rows) => rows[0] ?? null);
1651
- return Boolean(commentActivity);
1652
- }
1653
- async function issueHasDeferredWake(tx, orgId, issueId) {
1654
- const deferred = await tx
1655
- .select({ id: agentWakeupRequests.id })
1656
- .from(agentWakeupRequests)
1657
- .where(and(eq(agentWakeupRequests.orgId, orgId), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql `${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`))
1658
- .limit(1)
1659
- .then((rows) => rows[0] ?? null);
1660
- return Boolean(deferred);
1661
- }
1662
- async function passiveFollowupAlreadyRecorded(tx, runId) {
1663
- const idempotencyKey = `${ISSUE_PASSIVE_FOLLOWUP_REASON}:${runId}`;
1664
- const existingWake = await tx
1665
- .select({ id: agentWakeupRequests.id })
1666
- .from(agentWakeupRequests)
1667
- .where(eq(agentWakeupRequests.idempotencyKey, idempotencyKey))
1668
- .limit(1)
1669
- .then((rows) => rows[0] ?? null);
1670
- if (existingWake)
1671
- return true;
1672
- const existingReview = await tx
1673
- .select({ id: activityLog.id })
1674
- .from(activityLog)
1675
- .where(and(eq(activityLog.runId, runId), eq(activityLog.action, "issue.closure_needs_operator_review")))
1676
- .limit(1)
1677
- .then((rows) => rows[0] ?? null);
1678
- return Boolean(existingReview);
1679
- }
1680
- async function evaluatePassiveIssueClosureForLockedIssue(input) {
1681
- const { tx, run, issue, now } = input;
1682
- const context = parseObject(run.contextSnapshot);
1683
- const runIssueId = readNonEmptyString(context.issueId);
1684
- if (!runIssueId || runIssueId !== issue.id)
1685
- return { kind: "none", reason: "run_not_issue_backed" };
1686
- if (run.status !== "succeeded")
1687
- return { kind: "none", reason: "run_not_successful" };
1688
- if (issue.status !== "todo" && issue.status !== "in_progress") {
1689
- return { kind: "none", reason: "issue_has_closure_status" };
1690
- }
1691
- if (issue.assigneeAgentId !== run.agentId) {
1692
- return { kind: "none", reason: "issue_no_longer_assigned_to_run_agent" };
1693
- }
1694
- if (await runHasIssueClosureComment(tx, run, issue.id)) {
1695
- return { kind: "none", reason: "run_authored_issue_comment" };
1696
- }
1697
- if (await issueHasDeferredWake(tx, issue.orgId, issue.id)) {
1698
- return { kind: "none", reason: "deferred_issue_wake_exists" };
1699
- }
1700
- if (await passiveFollowupAlreadyRecorded(tx, run.id)) {
1701
- return { kind: "none", reason: "passive_followup_already_recorded" };
1702
- }
1703
- const agent = await tx
1704
- .select()
1705
- .from(agents)
1706
- .where(eq(agents.id, run.agentId))
1707
- .then((rows) => rows[0] ?? null);
1708
- if (!agent || agent.orgId !== run.orgId) {
1709
- return { kind: "none", reason: "agent_not_found" };
1710
- }
1711
- const policy = parseHeartbeatPolicy(agent);
1712
- if (hasCredibleTimerContinuation({ agent, policy, run, now })) {
1713
- return { kind: "none", reason: "timer_continuity_expected" };
1714
- }
1715
- const passiveContext = normalizePassiveFollowupContext(context.passiveFollowup);
1716
- const currentAttempt = passiveContext?.attempt ?? 0;
1717
- const originRunId = passiveContext?.originRunId ?? run.id;
1718
- if (currentAttempt >= ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS) {
1719
- return {
1720
- kind: "operator_review",
1721
- issue,
1722
- originRunId,
1723
- previousRunId: run.id,
1724
- attempts: currentAttempt,
1725
- reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
1726
- };
1727
- }
1728
- const nextAttempt = currentAttempt + 1;
1729
- const requestedAt = new Date(now.getTime() + passiveFollowupCooldownMs(nextAttempt));
1730
- const contextSnapshot = buildPassiveFollowupContextSnapshot({
1731
- run,
1732
- issue,
1733
- originRunId,
1734
- attempt: nextAttempt,
1735
- now,
1736
- });
1737
- const taskKey = deriveTaskKey(contextSnapshot, { issueId: issue.id });
1738
- const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey);
1739
- const requestPayload = {
1740
- issueId: issue.id,
1741
- originRunId,
1742
- previousRunId: run.id,
1743
- attempt: nextAttempt,
1744
- reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
1745
- };
1746
- const wakeupRequest = await tx
1747
- .insert(agentWakeupRequests)
1748
- .values({
1749
- orgId: run.orgId,
1750
- agentId: run.agentId,
1751
- source: "automation",
1752
- triggerDetail: "system",
1753
- reason: ISSUE_PASSIVE_FOLLOWUP_REASON,
1754
- payload: requestPayload,
1755
- status: "queued",
1756
- requestedByActorType: "system",
1757
- requestedByActorId: "issue_closure_governance",
1758
- idempotencyKey: `${ISSUE_PASSIVE_FOLLOWUP_REASON}:${run.id}`,
1759
- requestedAt,
1760
- updatedAt: now,
1761
- })
1762
- .returning()
1763
- .then((rows) => rows[0]);
1764
- const followupRun = await tx
1765
- .insert(heartbeatRuns)
1766
- .values({
1767
- orgId: run.orgId,
1768
- agentId: run.agentId,
1769
- invocationSource: "automation",
1770
- triggerDetail: "system",
1771
- status: "queued",
1772
- wakeupRequestId: wakeupRequest.id,
1773
- contextSnapshot,
1774
- sessionIdBefore: sessionBefore,
1775
- updatedAt: now,
1776
- })
1777
- .returning()
1778
- .then((rows) => rows[0]);
1779
- await tx
1780
- .update(agentWakeupRequests)
1781
- .set({
1782
- runId: followupRun.id,
1783
- updatedAt: now,
1784
- })
1785
- .where(eq(agentWakeupRequests.id, wakeupRequest.id));
1786
- await tx
1787
- .update(issues)
1788
- .set({
1789
- executionRunId: followupRun.id,
1790
- executionAgentNameKey: normalizeAgentNameKey(agent.name),
1791
- executionLockedAt: now,
1792
- updatedAt: now,
1793
- })
1794
- .where(eq(issues.id, issue.id));
1795
- return {
1796
- kind: "queued",
1797
- run: followupRun,
1798
- issue,
1799
- originRunId,
1800
- previousRunId: run.id,
1801
- attempt: nextAttempt,
1802
- requestedAt,
1803
- };
1804
- }
1805
- async function countRunningRunsForAgent(agentId) {
1806
- const [{ count }] = await db
1807
- .select({ count: sql `count(*)` })
1808
- .from(heartbeatRuns)
1809
- .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "running")));
1810
- return Number(count ?? 0);
1811
- }
1812
- async function claimQueuedRun(run) {
1813
- if (run.status !== "queued")
1814
- return run;
1815
- if (run.wakeupRequestId) {
1816
- const wakeup = await db
1817
- .select({ requestedAt: agentWakeupRequests.requestedAt })
1818
- .from(agentWakeupRequests)
1819
- .where(eq(agentWakeupRequests.id, run.wakeupRequestId))
1820
- .then((rows) => rows[0] ?? null);
1821
- if (wakeup && new Date(wakeup.requestedAt).getTime() > Date.now()) {
1822
- return null;
1823
- }
1824
- }
1825
- const agent = await getAgent(run.agentId);
1826
- if (!agent) {
1827
- await cancelRunInternal(run.id, "Cancelled because the agent no longer exists");
1828
- return null;
1829
- }
1830
- if (agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval") {
1831
- await cancelRunInternal(run.id, "Cancelled because the agent is not invokable");
1832
- return null;
1833
- }
1834
- const context = parseObject(run.contextSnapshot);
1835
- const budgetBlock = await budgets.getInvocationBlock(run.orgId, run.agentId, {
1836
- issueId: readNonEmptyString(context.issueId),
1837
- projectId: readNonEmptyString(context.projectId),
1838
- });
1839
- if (budgetBlock) {
1840
- await cancelRunInternal(run.id, budgetBlock.reason);
1841
- return null;
1842
- }
1843
- const claimedAt = new Date();
1844
- const claimed = await db
1845
- .update(heartbeatRuns)
1846
- .set({
1847
- status: "running",
1848
- startedAt: run.startedAt ?? claimedAt,
1849
- updatedAt: claimedAt,
1850
- })
1851
- .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "queued")))
1852
- .returning()
1853
- .then((rows) => rows[0] ?? null);
1854
- if (!claimed)
1855
- return null;
1856
- publishLiveEvent({
1857
- orgId: claimed.orgId,
1858
- type: "heartbeat.run.status",
1859
- payload: {
1860
- runId: claimed.id,
1861
- agentId: claimed.agentId,
1862
- status: claimed.status,
1863
- invocationSource: claimed.invocationSource,
1864
- triggerDetail: claimed.triggerDetail,
1865
- error: claimed.error ?? null,
1866
- errorCode: claimed.errorCode ?? null,
1867
- startedAt: claimed.startedAt ? new Date(claimed.startedAt).toISOString() : null,
1868
- finishedAt: claimed.finishedAt ? new Date(claimed.finishedAt).toISOString() : null,
1869
- },
1870
- });
1871
- await setWakeupStatus(claimed.wakeupRequestId, "claimed", { claimedAt });
1872
- return claimed;
1873
- }
1874
- async function finalizeAgentStatus(agentId, outcome) {
1875
- const existing = await getAgent(agentId);
1876
- if (!existing)
1877
- return;
1878
- if (existing.status === "paused" || existing.status === "terminated") {
1879
- return;
1880
- }
1881
- const runningCount = await countRunningRunsForAgent(agentId);
1882
- const nextStatus = runningCount > 0
1883
- ? "running"
1884
- : outcome === "succeeded" || outcome === "cancelled"
1885
- ? "idle"
1886
- : "error";
1887
- const updated = await db
1888
- .update(agents)
1889
- .set({
1890
- status: nextStatus,
1891
- lastHeartbeatAt: new Date(),
1892
- updatedAt: new Date(),
1893
- })
1894
- .where(eq(agents.id, agentId))
1895
- .returning()
1896
- .then((rows) => rows[0] ?? null);
1897
- if (updated) {
1898
- publishLiveEvent({
1899
- orgId: updated.orgId,
1900
- type: "agent.status",
1901
- payload: {
1902
- agentId: updated.id,
1903
- status: updated.status,
1904
- lastHeartbeatAt: updated.lastHeartbeatAt
1905
- ? new Date(updated.lastHeartbeatAt).toISOString()
1906
- : null,
1907
- outcome,
1908
- },
1909
- });
1910
- }
1911
- }
1912
- async function reapOrphanedRuns(opts) {
1913
- const staleThresholdMs = opts?.staleThresholdMs ?? 0;
1914
- const now = new Date();
1915
- // Find all runs stuck in "running" state (queued runs are legitimately waiting; resumeQueuedRuns handles them)
1916
- const activeRuns = await db
1917
- .select({
1918
- run: heartbeatRuns,
1919
- agentRuntimeType: agents.agentRuntimeType,
1920
- })
1921
- .from(heartbeatRuns)
1922
- .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
1923
- .where(eq(heartbeatRuns.status, "running"));
1924
- const reaped = [];
1925
- for (const { run, agentRuntimeType } of activeRuns) {
1926
- if (runningProcesses.has(run.id) || activeRunExecutions.has(run.id))
1927
- continue;
1928
- // Apply staleness threshold to avoid false positives
1929
- if (staleThresholdMs > 0) {
1930
- const refTime = run.updatedAt ? new Date(run.updatedAt).getTime() : 0;
1931
- if (now.getTime() - refTime < staleThresholdMs)
1932
- continue;
1933
- }
1934
- const tracksLocalChild = isTrackedLocalChildProcessAdapter(agentRuntimeType);
1935
- let detachedTerminationMessage = null;
1936
- if (tracksLocalChild && run.processPid && isProcessAlive(run.processPid)) {
1937
- const termination = await terminateOrphanedProcess(run.processPid);
1938
- if (termination.stillAlive) {
1939
- const detachedMessage = termination.error
1940
- ? `Lost in-memory process handle, child pid ${run.processPid} is still alive, and Rudder could not terminate it: ${termination.error}`
1941
- : `Lost in-memory process handle, but child pid ${run.processPid} is still alive`;
1942
- const detachedRun = await setRunStatus(run.id, "running", {
1943
- error: detachedMessage,
1944
- errorCode: DETACHED_PROCESS_ERROR_CODE,
1945
- });
1946
- if (detachedRun) {
1947
- await appendRunEvent(detachedRun, await nextRunEventSeq(detachedRun.id), {
1948
- eventType: "lifecycle",
1949
- stream: "system",
1950
- level: "warn",
1951
- message: detachedMessage,
1952
- payload: {
1953
- processPid: run.processPid,
1954
- },
1955
- });
1956
- }
1957
- continue;
1958
- }
1959
- detachedTerminationMessage = termination.terminationSignal
1960
- ? `Terminated detached child pid ${run.processPid} with ${termination.terminationSignal} after Rudder lost its process handle`
1961
- : `Detached child pid ${run.processPid} exited before Rudder could terminate it`;
1962
- }
1963
- const shouldRetry = tracksLocalChild && !!run.processPid && (run.processLossRetryCount ?? 0) < 1;
1964
- const baseMessage = run.processPid
1965
- ? `Process lost -- child pid ${run.processPid} is no longer running`
1966
- : "Process lost -- server may have restarted";
1967
- let finalizedRun = await setRunStatus(run.id, "failed", {
1968
- error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
1969
- errorCode: "process_lost",
1970
- finishedAt: now,
1971
- });
1972
- await setWakeupStatus(run.wakeupRequestId, "failed", {
1973
- finishedAt: now,
1974
- error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
1975
- });
1976
- if (!finalizedRun)
1977
- finalizedRun = await getRun(run.id);
1978
- if (!finalizedRun)
1979
- continue;
1980
- if (detachedTerminationMessage) {
1981
- await appendRunEvent(finalizedRun, await nextRunEventSeq(finalizedRun.id), {
1982
- eventType: "lifecycle",
1983
- stream: "system",
1984
- level: "warn",
1985
- message: detachedTerminationMessage,
1986
- payload: {
1987
- ...(run.processPid ? { processPid: run.processPid } : {}),
1988
- },
1989
- });
1990
- }
1991
- let retriedRun = null;
1992
- if (shouldRetry) {
1993
- const agent = await getAgent(run.agentId);
1994
- if (agent) {
1995
- retriedRun = await enqueueProcessLossRetry(finalizedRun, agent, now);
1996
- }
1997
- }
1998
- else {
1999
- await releaseIssueExecutionAndPromote(finalizedRun);
2000
- }
2001
- await appendRunEvent(finalizedRun, await nextRunEventSeq(finalizedRun.id), {
2002
- eventType: "lifecycle",
2003
- stream: "system",
2004
- level: "error",
2005
- message: shouldRetry
2006
- ? `${baseMessage}; queued retry ${retriedRun?.id ?? ""}`.trim()
2007
- : baseMessage,
2008
- payload: {
2009
- ...(run.processPid ? { processPid: run.processPid } : {}),
2010
- ...(retriedRun ? { retryRunId: retriedRun.id } : {}),
2011
- },
2012
- });
2013
- await finalizeAgentStatus(run.agentId, "failed");
2014
- await startNextQueuedRunForAgent(run.agentId);
2015
- runningProcesses.delete(run.id);
2016
- reaped.push(run.id);
2017
- }
2018
- if (reaped.length > 0) {
2019
- logger.warn({ reapedCount: reaped.length, runIds: reaped }, "reaped orphaned heartbeat runs");
2020
- }
2021
- return { reaped: reaped.length, runIds: reaped };
2022
- }
2023
- async function resumeQueuedRuns() {
2024
- const queuedRuns = await db
2025
- .select({ agentId: heartbeatRuns.agentId })
2026
- .from(heartbeatRuns)
2027
- .where(eq(heartbeatRuns.status, "queued"));
2028
- const agentIds = [...new Set(queuedRuns.map((r) => r.agentId))];
2029
- for (const agentId of agentIds) {
2030
- await startNextQueuedRunForAgent(agentId);
2031
- }
2032
- }
2033
- async function updateRuntimeState(agent, run, result, session, normalizedUsage) {
2034
- await ensureRuntimeState(agent);
2035
- const usage = normalizedUsage ?? normalizeUsageTotals(result.usage);
2036
- const inputTokens = usage?.inputTokens ?? 0;
2037
- const outputTokens = usage?.outputTokens ?? 0;
2038
- const cachedInputTokens = usage?.cachedInputTokens ?? 0;
2039
- const billingType = normalizeLedgerBillingType(result.billingType);
2040
- const additionalCostCents = normalizeBilledCostCents(result.costUsd, billingType);
2041
- const hasTokenUsage = inputTokens > 0 || outputTokens > 0 || cachedInputTokens > 0;
2042
- const provider = result.provider ?? "unknown";
2043
- const biller = resolveLedgerBiller(result);
2044
- const ledgerScope = await resolveLedgerScopeForRun(db, agent.orgId, run);
2045
- await db
2046
- .update(agentRuntimeState)
2047
- .set({
2048
- agentRuntimeType: agent.agentRuntimeType,
2049
- sessionId: session.legacySessionId,
2050
- lastRunId: run.id,
2051
- lastRunStatus: run.status,
2052
- lastError: result.errorMessage ?? null,
2053
- totalInputTokens: sql `${agentRuntimeState.totalInputTokens} + ${inputTokens}`,
2054
- totalOutputTokens: sql `${agentRuntimeState.totalOutputTokens} + ${outputTokens}`,
2055
- totalCachedInputTokens: sql `${agentRuntimeState.totalCachedInputTokens} + ${cachedInputTokens}`,
2056
- totalCostCents: sql `${agentRuntimeState.totalCostCents} + ${additionalCostCents}`,
2057
- updatedAt: new Date(),
2058
- })
2059
- .where(eq(agentRuntimeState.agentId, agent.id));
2060
- if (additionalCostCents > 0 || hasTokenUsage) {
2061
- const costs = costService(db, budgetHooks);
2062
- await costs.createEvent(agent.orgId, {
2063
- heartbeatRunId: run.id,
2064
- agentId: agent.id,
2065
- issueId: ledgerScope.issueId,
2066
- projectId: ledgerScope.projectId,
2067
- provider,
2068
- biller,
2069
- billingType,
2070
- model: result.model ?? "unknown",
2071
- inputTokens,
2072
- cachedInputTokens,
2073
- outputTokens,
2074
- costCents: additionalCostCents,
2075
- occurredAt: new Date(),
2076
- });
2077
- }
2078
- }
2079
- async function startNextQueuedRunForAgent(agentId) {
2080
- return withAgentStartLock(agentId, async () => {
2081
- const agent = await getAgent(agentId);
2082
- if (!agent)
2083
- return [];
2084
- if (agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval") {
2085
- return [];
2086
- }
2087
- const policy = parseHeartbeatPolicy(agent);
2088
- const runningCount = await countRunningRunsForAgent(agentId);
2089
- const availableSlots = Math.max(0, policy.maxConcurrentRuns - runningCount);
2090
- if (availableSlots <= 0)
2091
- return [];
2092
- const queuedRuns = await db
2093
- .select()
2094
- .from(heartbeatRuns)
2095
- .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued"), sql `(
2096
- ${heartbeatRuns.wakeupRequestId} is null
2097
- or exists (
2098
- select 1
2099
- from ${agentWakeupRequests}
2100
- where ${agentWakeupRequests.id} = ${heartbeatRuns.wakeupRequestId}
2101
- and ${agentWakeupRequests.requestedAt} <= now()
2102
- )
2103
- )`))
2104
- .orderBy(asc(heartbeatRuns.createdAt))
2105
- .limit(availableSlots);
2106
- if (queuedRuns.length === 0)
2107
- return [];
2108
- const claimedRuns = [];
2109
- for (const queuedRun of queuedRuns) {
2110
- const claimed = await claimQueuedRun(queuedRun);
2111
- if (claimed)
2112
- claimedRuns.push(claimed);
2113
- }
2114
- if (claimedRuns.length === 0)
2115
- return [];
2116
- for (const claimedRun of claimedRuns) {
2117
- void executeRun(claimedRun.id).catch((err) => {
2118
- logger.error({ err, runId: claimedRun.id }, "queued heartbeat execution failed");
2119
- });
2120
- }
2121
- return claimedRuns;
2122
- });
2123
- }
2124
- async function executeRun(runId) {
2125
- let run = await getRun(runId);
2126
- if (!run)
2127
- return;
2128
- if (run.status !== "queued" && run.status !== "running")
2129
- return;
2130
- if (run.status === "queued") {
2131
- const claimed = await claimQueuedRun(run);
2132
- if (!claimed) {
2133
- // Another worker has already claimed or finalized this run.
2134
- return;
2135
- }
2136
- run = claimed;
2137
- }
2138
- activeRunExecutions.add(run.id);
2139
- try {
2140
- const agent = await getAgent(run.agentId);
2141
- if (!agent) {
2142
- await setRunStatus(runId, "failed", {
2143
- error: "Agent not found",
2144
- errorCode: "agent_not_found",
2145
- finishedAt: new Date(),
2146
- });
2147
- await setWakeupStatus(run.wakeupRequestId, "failed", {
2148
- finishedAt: new Date(),
2149
- error: "Agent not found",
2150
- });
2151
- const failedRun = await getRun(runId);
2152
- if (failedRun)
2153
- await releaseIssueExecutionAndPromote(failedRun);
2154
- return;
2155
- }
2156
- const heartbeatObservationContext = buildHeartbeatObservabilityContext(run, {
2157
- runtime: agent.agentRuntimeType,
2158
- metadata: {
2159
- agentName: agent.name,
2160
- invocationSource: run.invocationSource,
2161
- triggerDetail: run.triggerDetail,
2162
- },
2163
- });
2164
- await withExecutionObservation(heartbeatObservationContext, {
2165
- name: buildHeartbeatObservationName(run, agent.name),
2166
- asType: "agent",
2167
- input: {
2168
- agentId: agent.id,
2169
- agentName: agent.name,
2170
- invocationSource: run.invocationSource,
2171
- triggerDetail: run.triggerDetail,
2172
- issueId: readNonEmptyString(parseObject(run.contextSnapshot).issueId),
2173
- },
2174
- }, async (observation) => {
2175
- const executionTranscript = [];
2176
- let stdoutTranscriptBuffer = "";
2177
- let stderrTranscriptBuffer = "";
2178
- let stdoutTranscriptParser = null;
2179
- let transcriptFallbackResult = null;
2180
- let finalObservationOutput = null;
2181
- let finalObservationStatus = run.status;
2182
- let finalObservationSessionId = heartbeatObservationContext.sessionKey ?? null;
2183
- const runtime = await ensureRuntimeState(agent);
2184
- const context = parseObject(run.contextSnapshot);
2185
- const taskKey = deriveTaskKey(context, null);
2186
- const sessionCodec = getAgentRuntimeSessionCodec(agent.agentRuntimeType);
2187
- const issueId = readNonEmptyString(context.issueId);
2188
- const issueContext = issueId
2189
- ? await db
2190
- .select({
2191
- id: issues.id,
2192
- identifier: issues.identifier,
2193
- title: issues.title,
2194
- projectId: issues.projectId,
2195
- projectWorkspaceId: issues.projectWorkspaceId,
2196
- executionWorkspaceId: issues.executionWorkspaceId,
2197
- executionWorkspacePreference: issues.executionWorkspacePreference,
2198
- assigneeAgentId: issues.assigneeAgentId,
2199
- assigneeAgentRuntimeOverrides: issues.assigneeAgentRuntimeOverrides,
2200
- executionWorkspaceSettings: issues.executionWorkspaceSettings,
2201
- })
2202
- .from(issues)
2203
- .where(and(eq(issues.id, issueId), eq(issues.orgId, agent.orgId)))
2204
- .then((rows) => rows[0] ?? null)
2205
- : null;
2206
- const issueAssigneeOverrides = issueContext && issueContext.assigneeAgentId === agent.id
2207
- ? parseIssueAssigneeAgentRuntimeOverrides(issueContext.assigneeAgentRuntimeOverrides)
2208
- : null;
2209
- const issueExecutionWorkspaceSettings = parseIssueExecutionWorkspaceSettings(issueContext?.executionWorkspaceSettings);
2210
- const contextProjectId = readNonEmptyString(context.projectId);
2211
- const executionProjectId = issueContext?.projectId ?? contextProjectId;
2212
- const projectExecutionWorkspacePolicy = executionProjectId
2213
- ? await db
2214
- .select({ executionWorkspacePolicy: projects.executionWorkspacePolicy })
2215
- .from(projects)
2216
- .where(and(eq(projects.id, executionProjectId), eq(projects.orgId, agent.orgId)))
2217
- .then((rows) => parseProjectExecutionWorkspacePolicy(rows[0]?.executionWorkspacePolicy))
2218
- : null;
2219
- const taskSession = taskKey
2220
- ? await getTaskSession(agent.orgId, agent.id, agent.agentRuntimeType, taskKey)
2221
- : null;
2222
- const resetTaskSession = shouldResetTaskSessionForWake(context);
2223
- const sessionResetReason = describeSessionResetReason(context);
2224
- const taskSessionForRun = resetTaskSession ? null : taskSession;
2225
- const explicitResumeSessionParams = normalizeSessionParams(sessionCodec.deserialize(parseObject(context.resumeSessionParams)));
2226
- const explicitResumeSessionDisplayId = truncateDisplayId(readNonEmptyString(context.resumeSessionDisplayId) ??
2227
- (sessionCodec.getDisplayId ? sessionCodec.getDisplayId(explicitResumeSessionParams) : null) ??
2228
- readNonEmptyString(explicitResumeSessionParams?.sessionId));
2229
- const previousSessionParams = explicitResumeSessionParams ??
2230
- (explicitResumeSessionDisplayId ? { sessionId: explicitResumeSessionDisplayId } : null) ??
2231
- normalizeSessionParams(sessionCodec.deserialize(taskSessionForRun?.sessionParamsJson ?? null));
2232
- const config = parseObject(agent.agentRuntimeConfig);
2233
- const executionWorkspaceMode = resolveExecutionWorkspaceMode({
2234
- projectPolicy: projectExecutionWorkspacePolicy,
2235
- issueSettings: issueExecutionWorkspaceSettings,
2236
- legacyUseProjectWorkspace: issueAssigneeOverrides?.useProjectWorkspace ?? null,
2237
- });
2238
- const resolvedWorkspace = await runContextSvc.resolveWorkspaceForRun(agent, context, previousSessionParams, { useProjectWorkspace: executionWorkspaceMode !== "agent_default" });
2239
- const workspaceManagedConfig = buildExecutionWorkspaceAdapterConfig({
2240
- agentConfig: config,
2241
- projectPolicy: projectExecutionWorkspacePolicy,
2242
- issueSettings: issueExecutionWorkspaceSettings,
2243
- mode: executionWorkspaceMode,
2244
- legacyUseProjectWorkspace: issueAssigneeOverrides?.useProjectWorkspace ?? null,
2245
- });
2246
- const mergedConfig = issueAssigneeOverrides?.agentRuntimeConfig
2247
- ? { ...workspaceManagedConfig, ...issueAssigneeOverrides.agentRuntimeConfig }
2248
- : workspaceManagedConfig;
2249
- const { resolvedConfig, runtimeConfig, runtimeSkillEntries, secretKeys } = await runContextSvc.prepareRuntimeConfig({
2250
- scene: "heartbeat",
2251
- agent,
2252
- baseConfig: mergedConfig,
2253
- });
2254
- heartbeatObservationContext.metadata = {
2255
- ...(heartbeatObservationContext.metadata ?? {}),
2256
- ...buildHeartbeatRuntimeTraceMetadata({
2257
- runtimeConfig,
2258
- runtimeSkills: runtimeSkillEntries,
2259
- }),
2260
- };
2261
- const issueRef = issueContext
2262
- ? {
2263
- id: issueContext.id,
2264
- identifier: issueContext.identifier,
2265
- title: issueContext.title,
2266
- projectId: issueContext.projectId,
2267
- projectWorkspaceId: issueContext.projectWorkspaceId,
2268
- executionWorkspaceId: issueContext.executionWorkspaceId,
2269
- executionWorkspacePreference: issueContext.executionWorkspacePreference,
2270
- }
2271
- : null;
2272
- const rootObservationInput = {
2273
- agentId: agent.id,
2274
- agentName: agent.name,
2275
- invocationSource: run.invocationSource,
2276
- triggerDetail: run.triggerDetail,
2277
- issue: issueRef
2278
- ? {
2279
- id: issueRef.id,
2280
- identifier: issueRef.identifier ?? null,
2281
- title: issueRef.title ?? null,
2282
- }
2283
- : null,
2284
- };
2285
- updateExecutionObservation(observation, heartbeatObservationContext, {
2286
- input: rootObservationInput,
2287
- });
2288
- updateExecutionTraceIO(observation, { input: rootObservationInput });
2289
- if (issueRef) {
2290
- updateExecutionTraceName(observation, buildIssueRunTraceName({
2291
- issueTitle: issueRef.title,
2292
- issueId: issueRef.id,
2293
- }));
2294
- }
2295
- const existingExecutionWorkspace = issueRef?.executionWorkspaceId ? await executionWorkspacesSvc.getById(issueRef.executionWorkspaceId) : null;
2296
- const workspaceOperationRecorder = workspaceOperationsSvc.createRecorder({
2297
- orgId: agent.orgId,
2298
- heartbeatRunId: run.id,
2299
- executionWorkspaceId: existingExecutionWorkspace?.id ?? null,
2300
- });
2301
- const executionWorkspace = await realizeExecutionWorkspace({
2302
- base: {
2303
- baseCwd: resolvedWorkspace.cwd,
2304
- source: resolvedWorkspace.source,
2305
- projectId: resolvedWorkspace.projectId,
2306
- workspaceId: resolvedWorkspace.workspaceId,
2307
- repoUrl: resolvedWorkspace.repoUrl,
2308
- repoRef: resolvedWorkspace.repoRef,
2309
- },
2310
- config: runtimeConfig,
2311
- issue: issueRef,
2312
- agent: {
2313
- id: agent.id,
2314
- name: agent.name,
2315
- orgId: agent.orgId,
2316
- },
2317
- recorder: workspaceOperationRecorder,
2318
- });
2319
- const resolvedProjectId = executionWorkspace.projectId ?? issueRef?.projectId ?? executionProjectId ?? null;
2320
- const resolvedProjectWorkspaceId = issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null;
2321
- const shouldReuseExisting = issueRef?.executionWorkspacePreference === "reuse_existing" &&
2322
- existingExecutionWorkspace &&
2323
- existingExecutionWorkspace.status !== "archived";
2324
- let persistedExecutionWorkspace = null;
2325
- try {
2326
- persistedExecutionWorkspace = shouldReuseExisting && existingExecutionWorkspace
2327
- ? await executionWorkspacesSvc.update(existingExecutionWorkspace.id, {
2328
- cwd: executionWorkspace.cwd,
2329
- repoUrl: executionWorkspace.repoUrl,
2330
- baseRef: executionWorkspace.repoRef,
2331
- branchName: executionWorkspace.branchName,
2332
- providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs",
2333
- providerRef: executionWorkspace.worktreePath,
2334
- status: "active",
2335
- lastUsedAt: new Date(),
2336
- metadata: {
2337
- ...(existingExecutionWorkspace.metadata ?? {}),
2338
- source: executionWorkspace.source,
2339
- createdByRuntime: executionWorkspace.created,
2340
- },
2341
- })
2342
- : resolvedProjectId
2343
- ? await executionWorkspacesSvc.create({
2344
- orgId: agent.orgId,
2345
- projectId: resolvedProjectId,
2346
- projectWorkspaceId: resolvedProjectWorkspaceId,
2347
- sourceIssueId: issueRef?.id ?? null,
2348
- mode: executionWorkspaceMode === "isolated_workspace"
2349
- ? "isolated_workspace"
2350
- : executionWorkspaceMode === "operator_branch"
2351
- ? "operator_branch"
2352
- : executionWorkspaceMode === "agent_default"
2353
- ? "adapter_managed"
2354
- : "shared_workspace",
2355
- strategyType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "project_primary",
2356
- name: executionWorkspace.branchName ?? issueRef?.identifier ?? `workspace-${agent.id.slice(0, 8)}`,
2357
- status: "active",
2358
- cwd: executionWorkspace.cwd,
2359
- repoUrl: executionWorkspace.repoUrl,
2360
- baseRef: executionWorkspace.repoRef,
2361
- branchName: executionWorkspace.branchName,
2362
- providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs",
2363
- providerRef: executionWorkspace.worktreePath,
2364
- lastUsedAt: new Date(),
2365
- openedAt: new Date(),
2366
- metadata: {
2367
- source: executionWorkspace.source,
2368
- createdByRuntime: executionWorkspace.created,
2369
- },
2370
- })
2371
- : null;
2372
- }
2373
- catch (error) {
2374
- if (executionWorkspace.created) {
2375
- try {
2376
- await cleanupExecutionWorkspaceArtifacts({
2377
- workspace: {
2378
- id: existingExecutionWorkspace?.id ?? `transient-${run.id}`,
2379
- cwd: executionWorkspace.cwd,
2380
- providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs",
2381
- providerRef: executionWorkspace.worktreePath,
2382
- branchName: executionWorkspace.branchName,
2383
- repoUrl: executionWorkspace.repoUrl,
2384
- baseRef: executionWorkspace.repoRef,
2385
- projectId: resolvedProjectId,
2386
- projectWorkspaceId: resolvedProjectWorkspaceId,
2387
- sourceIssueId: issueRef?.id ?? null,
2388
- metadata: {
2389
- createdByRuntime: true,
2390
- source: executionWorkspace.source,
2391
- },
2392
- },
2393
- projectWorkspace: {
2394
- cwd: resolvedWorkspace.cwd,
2395
- cleanupCommand: null,
2396
- },
2397
- teardownCommand: projectExecutionWorkspacePolicy?.workspaceStrategy?.teardownCommand ?? null,
2398
- recorder: workspaceOperationRecorder,
2399
- });
2400
- }
2401
- catch (cleanupError) {
2402
- logger.warn({
2403
- runId: run.id,
2404
- issueId,
2405
- executionWorkspaceCwd: executionWorkspace.cwd,
2406
- cleanupError: cleanupError instanceof Error ? cleanupError.message : String(cleanupError),
2407
- }, "Failed to cleanup realized execution workspace after persistence failure");
2408
- }
2409
- }
2410
- throw error;
2411
- }
2412
- await workspaceOperationRecorder.attachExecutionWorkspaceId(persistedExecutionWorkspace?.id ?? null);
2413
- if (existingExecutionWorkspace &&
2414
- persistedExecutionWorkspace &&
2415
- existingExecutionWorkspace.id !== persistedExecutionWorkspace.id &&
2416
- existingExecutionWorkspace.status === "active") {
2417
- await executionWorkspacesSvc.update(existingExecutionWorkspace.id, {
2418
- status: "idle",
2419
- cleanupReason: null,
2420
- });
2421
- }
2422
- if (issueId && persistedExecutionWorkspace) {
2423
- const nextIssueWorkspaceMode = issueExecutionWorkspaceModeForPersistedWorkspace(persistedExecutionWorkspace.mode);
2424
- const shouldSwitchIssueToExistingWorkspace = issueRef?.executionWorkspacePreference === "reuse_existing" ||
2425
- executionWorkspaceMode === "isolated_workspace" ||
2426
- executionWorkspaceMode === "operator_branch";
2427
- const nextIssuePatch = {};
2428
- if (issueRef?.executionWorkspaceId !== persistedExecutionWorkspace.id) {
2429
- nextIssuePatch.executionWorkspaceId = persistedExecutionWorkspace.id;
2430
- }
2431
- if (resolvedProjectWorkspaceId && issueRef?.projectWorkspaceId !== resolvedProjectWorkspaceId) {
2432
- nextIssuePatch.projectWorkspaceId = resolvedProjectWorkspaceId;
2433
- }
2434
- if (shouldSwitchIssueToExistingWorkspace) {
2435
- nextIssuePatch.executionWorkspacePreference = "reuse_existing";
2436
- nextIssuePatch.executionWorkspaceSettings = {
2437
- ...(issueExecutionWorkspaceSettings ?? {}),
2438
- mode: nextIssueWorkspaceMode,
2439
- };
2440
- }
2441
- if (Object.keys(nextIssuePatch).length > 0) {
2442
- await issuesSvc.update(issueId, nextIssuePatch);
2443
- }
2444
- }
2445
- if (persistedExecutionWorkspace) {
2446
- context.executionWorkspaceId = persistedExecutionWorkspace.id;
2447
- await db
2448
- .update(heartbeatRuns)
2449
- .set({
2450
- contextSnapshot: context,
2451
- updatedAt: new Date(),
2452
- })
2453
- .where(eq(heartbeatRuns.id, run.id));
2454
- }
2455
- const runtimeSessionResolution = resolveRuntimeSessionParamsForWorkspace({
2456
- orgId: agent.orgId,
2457
- agent,
2458
- previousSessionParams,
2459
- resolvedWorkspace: {
2460
- ...resolvedWorkspace,
2461
- cwd: resolveDefaultAgentWorkspaceDir(agent.orgId, agent),
2462
- source: "agent_home",
2463
- },
2464
- });
2465
- const runtimeSessionParams = runtimeSessionResolution.sessionParams;
2466
- const runtimeWorkspaceWarnings = [
2467
- ...resolvedWorkspace.warnings,
2468
- ...executionWorkspace.warnings,
2469
- ...(runtimeSessionResolution.warning ? [runtimeSessionResolution.warning] : []),
2470
- ...(resetTaskSession && sessionResetReason
2471
- ? [
2472
- taskKey
2473
- ? `Skipping saved session resume for task "${taskKey}" because ${sessionResetReason}.`
2474
- : `Skipping saved session resume because ${sessionResetReason}.`,
2475
- ]
2476
- : []),
2477
- ];
2478
- const runtimeSceneContext = await runContextSvc.buildSceneContext({
2479
- scene: "heartbeat",
2480
- agent,
2481
- resolvedWorkspace,
2482
- runtimeConfig,
2483
- executionWorkspaceMode,
2484
- executionWorkspace: {
2485
- cwd: executionWorkspace.cwd,
2486
- source: executionWorkspace.source,
2487
- strategy: executionWorkspace.strategy,
2488
- projectId: executionWorkspace.projectId,
2489
- workspaceId: executionWorkspace.workspaceId,
2490
- repoUrl: executionWorkspace.repoUrl,
2491
- repoRef: executionWorkspace.repoRef,
2492
- branchName: executionWorkspace.branchName,
2493
- worktreePath: executionWorkspace.worktreePath,
2494
- },
2495
- });
2496
- context.rudderScene = runtimeSceneContext.rudderScene;
2497
- context.rudderWorkspace = runtimeSceneContext.rudderWorkspace;
2498
- context.rudderWorkspaces = runtimeSceneContext.rudderWorkspaces;
2499
- if (runtimeSceneContext.rudderRuntimeServiceIntents) {
2500
- context.rudderRuntimeServiceIntents = runtimeSceneContext.rudderRuntimeServiceIntents;
2501
- }
2502
- else {
2503
- delete context.rudderRuntimeServiceIntents;
2504
- }
2505
- if (executionWorkspace.projectId && !readNonEmptyString(context.projectId)) {
2506
- context.projectId = executionWorkspace.projectId;
2507
- }
2508
- const runtimeSessionFallback = taskKey || resetTaskSession ? null : runtime.sessionId;
2509
- let previousSessionDisplayId = truncateDisplayId(explicitResumeSessionDisplayId ??
2510
- taskSessionForRun?.sessionDisplayId ??
2511
- (sessionCodec.getDisplayId ? sessionCodec.getDisplayId(runtimeSessionParams) : null) ??
2512
- readNonEmptyString(runtimeSessionParams?.sessionId) ??
2513
- runtimeSessionFallback);
2514
- let runtimeSessionIdForAdapter = readNonEmptyString(runtimeSessionParams?.sessionId) ?? runtimeSessionFallback;
2515
- let runtimeSessionParamsForAdapter = runtimeSessionParams;
2516
- const sessionCompaction = await evaluateSessionCompaction({
2517
- agent,
2518
- sessionId: previousSessionDisplayId ?? runtimeSessionIdForAdapter,
2519
- issueId,
2520
- });
2521
- if (sessionCompaction.rotate) {
2522
- context.rudderSessionHandoffMarkdown = sessionCompaction.handoffMarkdown;
2523
- context.rudderSessionRotationReason = sessionCompaction.reason;
2524
- context.rudderPreviousSessionId = previousSessionDisplayId ?? runtimeSessionIdForAdapter;
2525
- runtimeSessionIdForAdapter = null;
2526
- runtimeSessionParamsForAdapter = null;
2527
- previousSessionDisplayId = null;
2528
- if (sessionCompaction.reason) {
2529
- runtimeWorkspaceWarnings.push(`Starting a fresh session because ${sessionCompaction.reason}.`);
2530
- }
2531
- }
2532
- else {
2533
- delete context.rudderSessionHandoffMarkdown;
2534
- delete context.rudderSessionRotationReason;
2535
- delete context.rudderPreviousSessionId;
2536
- }
2537
- const runtimeForAdapter = {
2538
- sessionId: runtimeSessionIdForAdapter,
2539
- sessionParams: runtimeSessionParamsForAdapter,
2540
- sessionDisplayId: previousSessionDisplayId,
2541
- taskKey,
2542
- };
2543
- let seq = 1;
2544
- let handle = null;
2545
- let stdoutExcerpt = "";
2546
- let stderrExcerpt = "";
2547
- try {
2548
- const startedAt = run.startedAt ?? new Date();
2549
- const runningWithSession = await db
2550
- .update(heartbeatRuns)
2551
- .set({
2552
- startedAt,
2553
- sessionIdBefore: runtimeForAdapter.sessionDisplayId ?? runtimeForAdapter.sessionId,
2554
- contextSnapshot: context,
2555
- updatedAt: new Date(),
2556
- })
2557
- .where(eq(heartbeatRuns.id, run.id))
2558
- .returning()
2559
- .then((rows) => rows[0] ?? null);
2560
- if (runningWithSession)
2561
- run = runningWithSession;
2562
- const runningAgent = await db
2563
- .update(agents)
2564
- .set({ status: "running", updatedAt: new Date() })
2565
- .where(eq(agents.id, agent.id))
2566
- .returning()
2567
- .then((rows) => rows[0] ?? null);
2568
- if (runningAgent) {
2569
- publishLiveEvent({
2570
- orgId: runningAgent.orgId,
2571
- type: "agent.status",
2572
- payload: {
2573
- agentId: runningAgent.id,
2574
- status: runningAgent.status,
2575
- outcome: "running",
2576
- },
2577
- });
2578
- }
2579
- const currentRun = run;
2580
- await appendRunEvent(currentRun, seq++, {
2581
- eventType: "lifecycle",
2582
- stream: "system",
2583
- level: "info",
2584
- message: "run started",
2585
- });
2586
- handle = await runLogStore.begin({
2587
- orgId: run.orgId,
2588
- agentId: run.agentId,
2589
- runId,
2590
- });
2591
- await db
2592
- .update(heartbeatRuns)
2593
- .set({
2594
- logStore: handle.store,
2595
- logRef: handle.logRef,
2596
- updatedAt: new Date(),
2597
- })
2598
- .where(eq(heartbeatRuns.id, runId));
2599
- const adapter = getServerAdapter(agent.agentRuntimeType);
2600
- stdoutTranscriptParser = adapter.parseStdoutLine ?? null;
2601
- const currentUserRedactionOptions = await getCurrentUserRedactionOptions();
2602
- const onLog = async (stream, chunk) => {
2603
- const sanitizedChunk = redactCurrentUserText(chunk, currentUserRedactionOptions);
2604
- if (stream === "stdout")
2605
- stdoutExcerpt = appendExcerpt(stdoutExcerpt, sanitizedChunk);
2606
- if (stream === "stderr")
2607
- stderrExcerpt = appendExcerpt(stderrExcerpt, sanitizedChunk);
2608
- const ts = new Date().toISOString();
2609
- if (handle) {
2610
- await runLogStore.append(handle, {
2611
- stream,
2612
- chunk: sanitizedChunk,
2613
- ts,
2614
- });
2615
- }
2616
- const payloadChunk = sanitizedChunk.length > MAX_LIVE_LOG_CHUNK_BYTES
2617
- ? sanitizedChunk.slice(sanitizedChunk.length - MAX_LIVE_LOG_CHUNK_BYTES)
2618
- : sanitizedChunk;
2619
- publishLiveEvent({
2620
- orgId: run.orgId,
2621
- type: "heartbeat.run.log",
2622
- payload: {
2623
- runId: run.id,
2624
- agentId: run.agentId,
2625
- ts,
2626
- stream,
2627
- chunk: payloadChunk,
2628
- truncated: payloadChunk.length !== sanitizedChunk.length,
2629
- },
2630
- });
2631
- if (stream === "stdout") {
2632
- stdoutTranscriptBuffer = appendTranscriptEntriesFromChunk({
2633
- buffer: stdoutTranscriptBuffer,
2634
- chunk: sanitizedChunk,
2635
- transcript: executionTranscript,
2636
- parser: stdoutTranscriptParser,
2637
- kind: "stdout",
2638
- });
2639
- return;
2640
- }
2641
- stderrTranscriptBuffer = appendTranscriptEntriesFromChunk({
2642
- buffer: stderrTranscriptBuffer,
2643
- chunk: sanitizedChunk,
2644
- transcript: executionTranscript,
2645
- kind: "stderr",
2646
- });
2647
- };
2648
- for (const warning of runtimeWorkspaceWarnings) {
2649
- const logEntry = formatRuntimeWorkspaceWarningLog(warning);
2650
- await onLog(logEntry.stream, logEntry.chunk);
2651
- }
2652
- const adapterEnv = Object.fromEntries(Object.entries(parseObject(resolvedConfig.env)).filter((entry) => typeof entry[0] === "string" && typeof entry[1] === "string"));
2653
- const runtimeServices = await ensureRuntimeServicesForRun({
2654
- db,
2655
- runId: run.id,
2656
- agent: {
2657
- id: agent.id,
2658
- name: agent.name,
2659
- orgId: agent.orgId,
2660
- },
2661
- issue: issueRef,
2662
- workspace: executionWorkspace,
2663
- executionWorkspaceId: persistedExecutionWorkspace?.id ?? issueRef?.executionWorkspaceId ?? null,
2664
- config: resolvedConfig,
2665
- adapterEnv,
2666
- onLog,
2667
- });
2668
- if (runtimeServices.length > 0) {
2669
- context.rudderRuntimeServices = runtimeServices;
2670
- context.rudderRuntimePrimaryUrl =
2671
- runtimeServices.find((service) => readNonEmptyString(service.url))?.url ?? null;
2672
- await db
2673
- .update(heartbeatRuns)
2674
- .set({
2675
- contextSnapshot: context,
2676
- updatedAt: new Date(),
2677
- })
2678
- .where(eq(heartbeatRuns.id, run.id));
2679
- }
2680
- if (issueId && (executionWorkspace.created || runtimeServices.some((service) => !service.reused))) {
2681
- try {
2682
- await issuesSvc.addComment(issueId, buildWorkspaceReadyComment({
2683
- workspace: executionWorkspace,
2684
- runtimeServices,
2685
- }), { agentId: agent.id });
2686
- }
2687
- catch (err) {
2688
- await onLog("stderr", `[rudder] Failed to post workspace-ready comment: ${err instanceof Error ? err.message : String(err)}\n`);
2689
- }
2690
- }
2691
- const onAdapterMeta = async (meta) => {
2692
- if (meta.env && secretKeys.size > 0) {
2693
- for (const key of secretKeys) {
2694
- if (key in meta.env)
2695
- meta.env[key] = "***REDACTED***";
2696
- }
2697
- }
2698
- heartbeatObservationContext.metadata = {
2699
- ...(heartbeatObservationContext.metadata ?? {}),
2700
- ...buildHeartbeatRuntimeTraceMetadata({
2701
- runtimeConfig,
2702
- runtimeSkills: runtimeSkillEntries,
2703
- adapterMeta: meta,
2704
- }),
2705
- };
2706
- updateExecutionObservation(observation, heartbeatObservationContext, {
2707
- input: rootObservationInput,
2708
- });
2709
- await appendRunEvent(currentRun, seq++, {
2710
- eventType: "adapter.invoke",
2711
- stream: "system",
2712
- level: "info",
2713
- message: "adapter invocation",
2714
- payload: meta,
2715
- });
2716
- };
2717
- const authToken = adapter.supportsLocalAgentJwt
2718
- ? createLocalAgentJwt(agent.id, agent.orgId, agent.agentRuntimeType, run.id)
2719
- : null;
2720
- if (adapter.supportsLocalAgentJwt && !authToken) {
2721
- logger.warn({
2722
- orgId: agent.orgId,
2723
- agentId: agent.id,
2724
- runId: run.id,
2725
- agentRuntimeType: agent.agentRuntimeType,
2726
- }, "local agent jwt secret missing or invalid; running without injected RUDDER_API_KEY");
2727
- }
2728
- const adapterResult = await adapter.execute({
2729
- runId: run.id,
2730
- agent,
2731
- runtime: runtimeForAdapter,
2732
- config: runtimeConfig,
2733
- context,
2734
- onLog,
2735
- onMeta: onAdapterMeta,
2736
- onSpawn: async (meta) => {
2737
- await persistRunProcessMetadata(run.id, meta);
2738
- },
2739
- authToken: authToken ?? undefined,
2740
- });
2741
- const adapterManagedRuntimeServices = adapterResult.runtimeServices
2742
- ? await persistAdapterManagedRuntimeServices({
2743
- db,
2744
- agentRuntimeType: agent.agentRuntimeType,
2745
- runId: run.id,
2746
- agent: {
2747
- id: agent.id,
2748
- name: agent.name,
2749
- orgId: agent.orgId,
2750
- },
2751
- issue: issueRef,
2752
- workspace: executionWorkspace,
2753
- reports: adapterResult.runtimeServices,
2754
- })
2755
- : [];
2756
- if (adapterManagedRuntimeServices.length > 0) {
2757
- const combinedRuntimeServices = [
2758
- ...runtimeServices,
2759
- ...adapterManagedRuntimeServices,
2760
- ];
2761
- context.rudderRuntimeServices = combinedRuntimeServices;
2762
- context.rudderRuntimePrimaryUrl =
2763
- combinedRuntimeServices.find((service) => readNonEmptyString(service.url))?.url ?? null;
2764
- await db
2765
- .update(heartbeatRuns)
2766
- .set({
2767
- contextSnapshot: context,
2768
- updatedAt: new Date(),
2769
- })
2770
- .where(eq(heartbeatRuns.id, run.id));
2771
- if (issueId) {
2772
- try {
2773
- await issuesSvc.addComment(issueId, buildWorkspaceReadyComment({
2774
- workspace: executionWorkspace,
2775
- runtimeServices: adapterManagedRuntimeServices,
2776
- }), { agentId: agent.id });
2777
- }
2778
- catch (err) {
2779
- await onLog("stderr", `[rudder] Failed to post adapter-managed runtime comment: ${err instanceof Error ? err.message : String(err)}\n`);
2780
- }
2781
- }
2782
- }
2783
- const nextSessionState = resolveNextSessionState({
2784
- codec: sessionCodec,
2785
- adapterResult,
2786
- previousParams: previousSessionParams,
2787
- previousDisplayId: runtimeForAdapter.sessionDisplayId,
2788
- previousLegacySessionId: runtimeForAdapter.sessionId,
2789
- });
2790
- const rawUsage = normalizeUsageTotals(adapterResult.usage);
2791
- const sessionUsageResolution = await resolveNormalizedUsageForSession({
2792
- agentId: agent.id,
2793
- runId: run.id,
2794
- sessionId: nextSessionState.displayId ?? nextSessionState.legacySessionId,
2795
- rawUsage,
2796
- });
2797
- const normalizedUsage = sessionUsageResolution.normalizedUsage;
2798
- let outcome;
2799
- const latestRun = await getRun(run.id);
2800
- if (latestRun?.status === "cancelled") {
2801
- outcome = "cancelled";
2802
- }
2803
- else if (adapterResult.timedOut) {
2804
- outcome = "timed_out";
2805
- }
2806
- else if ((adapterResult.exitCode ?? 0) === 0 && !adapterResult.errorMessage) {
2807
- outcome = "succeeded";
2808
- }
2809
- else {
2810
- outcome = "failed";
2811
- }
2812
- let logSummary = null;
2813
- if (handle) {
2814
- logSummary = await runLogStore.finalize(handle);
2815
- }
2816
- const status = outcome === "succeeded"
2817
- ? "succeeded"
2818
- : outcome === "cancelled"
2819
- ? "cancelled"
2820
- : outcome === "timed_out"
2821
- ? "timed_out"
2822
- : "failed";
2823
- heartbeatObservationContext.status = status;
2824
- finalObservationStatus = status;
2825
- finalObservationSessionId = nextSessionState.displayId ?? nextSessionState.legacySessionId ?? finalObservationSessionId;
2826
- const adapterResultSummary = summarizeHeartbeatRunResultJson(adapterResult.resultJson);
2827
- transcriptFallbackResult = {
2828
- ts: new Date().toISOString(),
2829
- model: readNonEmptyString(adapterResult.model),
2830
- output: readNonEmptyString(adapterResult.summary)
2831
- ?? readNonEmptyString(adapterResultSummary?.result)
2832
- ?? readNonEmptyString(adapterResultSummary?.summary)
2833
- ?? readNonEmptyString(adapterResultSummary?.message)
2834
- ?? null,
2835
- usage: adapterResult.usage ?? null,
2836
- costUsd: typeof adapterResult.costUsd === "number" ? adapterResult.costUsd : null,
2837
- subtype: status,
2838
- isError: outcome !== "succeeded",
2839
- errors: adapterResult.errorMessage ? [adapterResult.errorMessage] : [],
2840
- };
2841
- const usageJson = normalizedUsage || adapterResult.costUsd != null
2842
- ? {
2843
- ...(normalizedUsage ?? {}),
2844
- ...(rawUsage ? {
2845
- rawInputTokens: rawUsage.inputTokens,
2846
- rawCachedInputTokens: rawUsage.cachedInputTokens,
2847
- rawOutputTokens: rawUsage.outputTokens,
2848
- } : {}),
2849
- ...(sessionUsageResolution.derivedFromSessionTotals ? { usageSource: "session_delta" } : {}),
2850
- ...((nextSessionState.displayId ?? nextSessionState.legacySessionId)
2851
- ? { persistedSessionId: nextSessionState.displayId ?? nextSessionState.legacySessionId }
2852
- : {}),
2853
- sessionReused: runtimeForAdapter.sessionId != null || runtimeForAdapter.sessionDisplayId != null,
2854
- taskSessionReused: taskSessionForRun != null,
2855
- freshSession: runtimeForAdapter.sessionId == null && runtimeForAdapter.sessionDisplayId == null,
2856
- sessionRotated: sessionCompaction.rotate,
2857
- sessionRotationReason: sessionCompaction.reason,
2858
- provider: readNonEmptyString(adapterResult.provider) ?? "unknown",
2859
- biller: resolveLedgerBiller(adapterResult),
2860
- model: readNonEmptyString(adapterResult.model) ?? "unknown",
2861
- ...(adapterResult.costUsd != null ? { costUsd: adapterResult.costUsd } : {}),
2862
- billingType: normalizeLedgerBillingType(adapterResult.billingType),
2863
- }
2864
- : null;
2865
- await setRunStatus(run.id, status, {
2866
- finishedAt: new Date(),
2867
- error: outcome === "succeeded"
2868
- ? null
2869
- : redactCurrentUserText(adapterResult.errorMessage ?? (outcome === "timed_out" ? "Timed out" : "Adapter failed"), currentUserRedactionOptions),
2870
- errorCode: outcome === "timed_out"
2871
- ? "timeout"
2872
- : outcome === "cancelled"
2873
- ? "cancelled"
2874
- : outcome === "failed"
2875
- ? (adapterResult.errorCode ?? "adapter_failed")
2876
- : null,
2877
- exitCode: adapterResult.exitCode,
2878
- signal: adapterResult.signal,
2879
- usageJson,
2880
- resultJson: adapterResult.resultJson ?? null,
2881
- sessionIdAfter: nextSessionState.displayId ?? nextSessionState.legacySessionId,
2882
- stdoutExcerpt,
2883
- stderrExcerpt,
2884
- logBytes: logSummary?.bytes,
2885
- logSha256: logSummary?.sha256,
2886
- logCompressed: logSummary?.compressed ?? false,
2887
- });
2888
- await setWakeupStatus(run.wakeupRequestId, outcome === "succeeded" ? "completed" : status, {
2889
- finishedAt: new Date(),
2890
- error: adapterResult.errorMessage ?? null,
2891
- });
2892
- const finalizedRun = await getRun(run.id);
2893
- if (finalizedRun) {
2894
- await appendRunEvent(finalizedRun, seq++, {
2895
- eventType: "lifecycle",
2896
- stream: "system",
2897
- level: outcome === "succeeded" ? "info" : "error",
2898
- message: `run ${outcome}`,
2899
- payload: {
2900
- status,
2901
- exitCode: adapterResult.exitCode,
2902
- },
2903
- });
2904
- await releaseIssueExecutionAndPromote(finalizedRun);
2905
- }
2906
- if (finalizedRun) {
2907
- await updateRuntimeState(agent, finalizedRun, adapterResult, {
2908
- legacySessionId: nextSessionState.legacySessionId,
2909
- }, normalizedUsage);
2910
- if (taskKey) {
2911
- if (adapterResult.clearSession || (!nextSessionState.params && !nextSessionState.displayId)) {
2912
- await clearTaskSessions(agent.orgId, agent.id, {
2913
- taskKey,
2914
- agentRuntimeType: agent.agentRuntimeType,
2915
- });
2916
- }
2917
- else {
2918
- await upsertTaskSession({
2919
- orgId: agent.orgId,
2920
- agentId: agent.id,
2921
- agentRuntimeType: agent.agentRuntimeType,
2922
- taskKey,
2923
- sessionParamsJson: nextSessionState.params,
2924
- sessionDisplayId: nextSessionState.displayId,
2925
- lastRunId: finalizedRun.id,
2926
- lastError: outcome === "succeeded" ? null : (adapterResult.errorMessage ?? "run_failed"),
2927
- });
2928
- }
2929
- }
2930
- await emitHeartbeatLiveEval(finalizedRun.id);
2931
- }
2932
- await finalizeAgentStatus(agent.id, outcome);
2933
- }
2934
- catch (err) {
2935
- const message = redactCurrentUserText(err instanceof Error ? err.message : "Unknown adapter failure", await getCurrentUserRedactionOptions());
2936
- heartbeatObservationContext.status = "failed";
2937
- finalObservationStatus = "failed";
2938
- transcriptFallbackResult = {
2939
- ts: new Date().toISOString(),
2940
- output: message,
2941
- subtype: "failed",
2942
- isError: true,
2943
- errors: [message],
2944
- };
2945
- logger.error({ err, runId }, "heartbeat execution failed");
2946
- let logSummary = null;
2947
- if (handle) {
2948
- try {
2949
- logSummary = await runLogStore.finalize(handle);
2950
- }
2951
- catch (finalizeErr) {
2952
- logger.warn({ err: finalizeErr, runId }, "failed to finalize run log after error");
2953
- }
2954
- }
2955
- const failedRun = await setRunStatus(run.id, "failed", {
2956
- error: message,
2957
- errorCode: "adapter_failed",
2958
- finishedAt: new Date(),
2959
- stdoutExcerpt,
2960
- stderrExcerpt,
2961
- logBytes: logSummary?.bytes,
2962
- logSha256: logSummary?.sha256,
2963
- logCompressed: logSummary?.compressed ?? false,
2964
- });
2965
- await setWakeupStatus(run.wakeupRequestId, "failed", {
2966
- finishedAt: new Date(),
2967
- error: message,
2968
- });
2969
- if (failedRun) {
2970
- await appendRunEvent(failedRun, seq++, {
2971
- eventType: "error",
2972
- stream: "system",
2973
- level: "error",
2974
- message,
2975
- });
2976
- await releaseIssueExecutionAndPromote(failedRun);
2977
- await updateRuntimeState(agent, failedRun, {
2978
- exitCode: null,
2979
- signal: null,
2980
- timedOut: false,
2981
- errorMessage: message,
2982
- }, {
2983
- legacySessionId: runtimeForAdapter.sessionId,
2984
- });
2985
- if (taskKey && (previousSessionParams || previousSessionDisplayId || taskSession)) {
2986
- await upsertTaskSession({
2987
- orgId: agent.orgId,
2988
- agentId: agent.id,
2989
- agentRuntimeType: agent.agentRuntimeType,
2990
- taskKey,
2991
- sessionParamsJson: previousSessionParams,
2992
- sessionDisplayId: previousSessionDisplayId,
2993
- lastRunId: failedRun.id,
2994
- lastError: message,
2995
- });
2996
- }
2997
- await emitHeartbeatLiveEval(failedRun.id);
2998
- }
2999
- await finalizeAgentStatus(agent.id, "failed");
3000
- }
3001
- finally {
3002
- stdoutTranscriptBuffer = appendTranscriptEntriesFromChunk({
3003
- buffer: stdoutTranscriptBuffer,
3004
- chunk: "",
3005
- transcript: executionTranscript,
3006
- parser: stdoutTranscriptParser,
3007
- finalize: true,
3008
- kind: "stdout",
3009
- });
3010
- stderrTranscriptBuffer = appendTranscriptEntriesFromChunk({
3011
- buffer: stderrTranscriptBuffer,
3012
- chunk: "",
3013
- transcript: executionTranscript,
3014
- finalize: true,
3015
- kind: "stderr",
3016
- });
3017
- try {
3018
- const transcriptStats = emitExecutionTranscriptTree({
3019
- context: heartbeatObservationContext,
3020
- parentObservation: observation,
3021
- transcript: executionTranscript,
3022
- fallbackResult: transcriptFallbackResult,
3023
- });
3024
- finalObservationOutput = transcriptStats.finalOutput ?? transcriptFallbackResult?.output ?? null;
3025
- finalObservationSessionId = transcriptStats.finalSessionId ?? finalObservationSessionId;
3026
- }
3027
- catch (error) {
3028
- logger.warn({
3029
- runId: run.id,
3030
- err: error instanceof Error ? error.message : String(error),
3031
- }, "Failed to export heartbeat transcript tree to Langfuse");
3032
- }
3033
- updateExecutionObservation(observation, heartbeatObservationContext, {
3034
- input: rootObservationInput,
3035
- output: finalObservationOutput,
3036
- level: finalObservationStatus === "failed" || finalObservationStatus === "timed_out" ? "ERROR" : "DEFAULT",
3037
- statusMessage: finalObservationStatus ?? undefined,
3038
- });
3039
- updateExecutionTraceIO(observation, {
3040
- input: rootObservationInput,
3041
- output: finalObservationOutput,
3042
- });
3043
- updateExecutionTraceSession(observation, finalObservationSessionId);
3044
- }
3045
- });
3046
- }
3047
- catch (outerErr) {
3048
- // Setup code before adapter.execute threw (e.g. ensureRuntimeState, resolveWorkspaceForRun).
3049
- // The inner catch did not fire, so we must record the failure here.
3050
- const message = outerErr instanceof Error ? outerErr.message : "Unknown setup failure";
3051
- logger.error({ err: outerErr, runId }, "heartbeat execution setup failed");
3052
- await setRunStatus(runId, "failed", {
3053
- error: message,
3054
- errorCode: "adapter_failed",
3055
- finishedAt: new Date(),
3056
- }).catch(() => undefined);
3057
- await setWakeupStatus(run.wakeupRequestId, "failed", {
3058
- finishedAt: new Date(),
3059
- error: message,
3060
- }).catch(() => undefined);
3061
- const failedRun = await getRun(runId).catch(() => null);
3062
- if (failedRun) {
3063
- // Emit a run-log event so the failure is visible in the run timeline,
3064
- // consistent with what the inner catch block does for adapter failures.
3065
- await appendRunEvent(failedRun, 1, {
3066
- eventType: "error",
3067
- stream: "system",
3068
- level: "error",
3069
- message,
3070
- }).catch(() => undefined);
3071
- await emitHeartbeatLiveEval(failedRun.id).catch(() => undefined);
3072
- await releaseIssueExecutionAndPromote(failedRun).catch(() => undefined);
3073
- }
3074
- // Ensure the agent is not left stuck in "running" if the inner catch handler's
3075
- // DB calls threw (e.g. a transient DB error in finalizeAgentStatus).
3076
- await finalizeAgentStatus(run.agentId, "failed").catch(() => undefined);
3077
- }
3078
- finally {
3079
- await releaseRuntimeServicesForRun(run.id).catch(() => undefined);
3080
- activeRunExecutions.delete(run.id);
3081
- await startNextQueuedRunForAgent(run.agentId);
3082
- }
3083
- }
3084
- async function releaseIssueExecutionAndPromote(run) {
3085
- const outcome = await db.transaction(async (tx) => {
3086
- await tx.execute(sql `select id from issues where org_id = ${run.orgId} and execution_run_id = ${run.id} for update`);
3087
- const issue = await tx
3088
- .select({
3089
- id: issues.id,
3090
- orgId: issues.orgId,
3091
- title: issues.title,
3092
- description: issues.description,
3093
- status: issues.status,
3094
- priority: issues.priority,
3095
- projectId: issues.projectId,
3096
- assigneeAgentId: issues.assigneeAgentId,
3097
- })
3098
- .from(issues)
3099
- .where(and(eq(issues.orgId, run.orgId), eq(issues.executionRunId, run.id)))
3100
- .then((rows) => rows[0] ?? null);
3101
- if (!issue)
3102
- return { promotedRun: null, passiveClosure: null };
3103
- const passiveClosure = await evaluatePassiveIssueClosureForLockedIssue({
3104
- tx,
3105
- run,
3106
- issue,
3107
- now: new Date(),
3108
- });
3109
- if (passiveClosure.kind === "queued") {
3110
- return { promotedRun: passiveClosure.run, passiveClosure };
3111
- }
3112
- await tx
3113
- .update(issues)
3114
- .set({
3115
- executionRunId: null,
3116
- executionAgentNameKey: null,
3117
- executionLockedAt: null,
3118
- updatedAt: new Date(),
3119
- })
3120
- .where(eq(issues.id, issue.id));
3121
- while (true) {
3122
- const deferred = await tx
3123
- .select()
3124
- .from(agentWakeupRequests)
3125
- .where(and(eq(agentWakeupRequests.orgId, issue.orgId), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql `${agentWakeupRequests.payload} ->> 'issueId' = ${issue.id}`))
3126
- .orderBy(asc(agentWakeupRequests.requestedAt))
3127
- .limit(1)
3128
- .then((rows) => rows[0] ?? null);
3129
- if (!deferred)
3130
- return { promotedRun: null, passiveClosure };
3131
- const deferredAgent = await tx
3132
- .select()
3133
- .from(agents)
3134
- .where(eq(agents.id, deferred.agentId))
3135
- .then((rows) => rows[0] ?? null);
3136
- if (!deferredAgent ||
3137
- deferredAgent.orgId !== issue.orgId ||
3138
- deferredAgent.status === "paused" ||
3139
- deferredAgent.status === "terminated" ||
3140
- deferredAgent.status === "pending_approval") {
3141
- await tx
3142
- .update(agentWakeupRequests)
3143
- .set({
3144
- status: "failed",
3145
- finishedAt: new Date(),
3146
- error: "Deferred wake could not be promoted: agent is not invokable",
3147
- updatedAt: new Date(),
3148
- })
3149
- .where(eq(agentWakeupRequests.id, deferred.id));
3150
- continue;
3151
- }
3152
- const deferredPayload = parseObject(deferred.payload);
3153
- const deferredContextSeed = parseObject(deferredPayload[DEFERRED_WAKE_CONTEXT_KEY]);
3154
- const promotedContextSeed = { ...deferredContextSeed };
3155
- const promotedReason = readNonEmptyString(deferred.reason) ?? "issue_execution_promoted";
3156
- const promotedSource = readNonEmptyString(deferred.source) ?? "automation";
3157
- const promotedTriggerDetail = readNonEmptyString(deferred.triggerDetail) ?? null;
3158
- const promotedPayload = deferredPayload;
3159
- delete promotedPayload[DEFERRED_WAKE_CONTEXT_KEY];
3160
- const { contextSnapshot: promotedContextSnapshot, taskKey: promotedTaskKey, } = enrichWakeContextSnapshot({
3161
- contextSnapshot: promotedContextSeed,
3162
- reason: promotedReason,
3163
- source: promotedSource,
3164
- triggerDetail: promotedTriggerDetail,
3165
- payload: promotedPayload,
3166
- });
3167
- const sessionBefore = readNonEmptyString(promotedContextSnapshot.resumeSessionDisplayId) ??
3168
- await resolveSessionBeforeForWakeup(deferredAgent, promotedTaskKey);
3169
- const now = new Date();
3170
- const newRun = await tx
3171
- .insert(heartbeatRuns)
3172
- .values({
3173
- orgId: deferredAgent.orgId,
3174
- agentId: deferredAgent.id,
3175
- invocationSource: promotedSource,
3176
- triggerDetail: promotedTriggerDetail,
3177
- status: "queued",
3178
- wakeupRequestId: deferred.id,
3179
- contextSnapshot: promotedContextSnapshot,
3180
- sessionIdBefore: sessionBefore,
3181
- })
3182
- .returning()
3183
- .then((rows) => rows[0]);
3184
- await tx
3185
- .update(agentWakeupRequests)
3186
- .set({
3187
- status: "queued",
3188
- reason: "issue_execution_promoted",
3189
- runId: newRun.id,
3190
- claimedAt: null,
3191
- finishedAt: null,
3192
- error: null,
3193
- updatedAt: now,
3194
- })
3195
- .where(eq(agentWakeupRequests.id, deferred.id));
3196
- await tx
3197
- .update(issues)
3198
- .set({
3199
- executionRunId: newRun.id,
3200
- executionAgentNameKey: normalizeAgentNameKey(deferredAgent.name),
3201
- executionLockedAt: now,
3202
- updatedAt: now,
3203
- })
3204
- .where(eq(issues.id, issue.id));
3205
- return { promotedRun: newRun, passiveClosure };
3206
- }
3207
- });
3208
- const passiveClosure = outcome.passiveClosure;
3209
- if (passiveClosure?.kind === "queued") {
3210
- await appendRunEvent(run, await nextRunEventSeq(run.id), {
3211
- eventType: "issue.passive_followup_queued",
3212
- stream: "system",
3213
- level: "warn",
3214
- message: `Queued passive issue follow-up ${passiveClosure.run.id}`,
3215
- payload: {
3216
- issueId: passiveClosure.issue.id,
3217
- followupRunId: passiveClosure.run.id,
3218
- originRunId: passiveClosure.originRunId,
3219
- previousRunId: passiveClosure.previousRunId,
3220
- attempt: passiveClosure.attempt,
3221
- maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
3222
- reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
3223
- requestedAt: passiveClosure.requestedAt.toISOString(),
3224
- },
3225
- });
3226
- await appendRunEvent(passiveClosure.run, await nextRunEventSeq(passiveClosure.run.id), {
3227
- eventType: "issue.passive_followup_queued",
3228
- stream: "system",
3229
- level: "warn",
3230
- message: `Passive follow-up queued because run ${run.id} ended without issue close-out`,
3231
- payload: {
3232
- issueId: passiveClosure.issue.id,
3233
- originRunId: passiveClosure.originRunId,
3234
- previousRunId: passiveClosure.previousRunId,
3235
- attempt: passiveClosure.attempt,
3236
- maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
3237
- reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
3238
- requestedAt: passiveClosure.requestedAt.toISOString(),
3239
- },
3240
- });
3241
- await logActivity(db, {
3242
- orgId: passiveClosure.issue.orgId,
3243
- actorType: "system",
3244
- actorId: "issue_closure_governance",
3245
- action: "issue.passive_followup_queued",
3246
- entityType: "issue",
3247
- entityId: passiveClosure.issue.id,
3248
- agentId: run.agentId,
3249
- runId: run.id,
3250
- details: {
3251
- issueId: passiveClosure.issue.id,
3252
- issueTitle: passiveClosure.issue.title,
3253
- followupRunId: passiveClosure.run.id,
3254
- originRunId: passiveClosure.originRunId,
3255
- previousRunId: passiveClosure.previousRunId,
3256
- attempt: passiveClosure.attempt,
3257
- maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
3258
- reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
3259
- requestedAt: passiveClosure.requestedAt.toISOString(),
3260
- },
3261
- });
3262
- }
3263
- else if (passiveClosure?.kind === "operator_review") {
3264
- await appendRunEvent(run, await nextRunEventSeq(run.id), {
3265
- eventType: "issue.closure_needs_operator_review",
3266
- stream: "system",
3267
- level: "warn",
3268
- message: "Passive issue follow-up stopped and needs operator review",
3269
- payload: {
3270
- issueId: passiveClosure.issue.id,
3271
- originRunId: passiveClosure.originRunId,
3272
- previousRunId: passiveClosure.previousRunId,
3273
- attempts: passiveClosure.attempts,
3274
- maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
3275
- reason: passiveClosure.reason,
3276
- },
3277
- });
3278
- await logActivity(db, {
3279
- orgId: passiveClosure.issue.orgId,
3280
- actorType: "system",
3281
- actorId: "issue_closure_governance",
3282
- action: "issue.closure_needs_operator_review",
3283
- entityType: "issue",
3284
- entityId: passiveClosure.issue.id,
3285
- agentId: run.agentId,
3286
- runId: run.id,
3287
- details: {
3288
- issueId: passiveClosure.issue.id,
3289
- issueTitle: passiveClosure.issue.title,
3290
- originRunId: passiveClosure.originRunId,
3291
- previousRunId: passiveClosure.previousRunId,
3292
- attempts: passiveClosure.attempts,
3293
- maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
3294
- reason: passiveClosure.reason,
3295
- },
3296
- });
3297
- }
3298
- const promotedRun = outcome.promotedRun;
3299
- if (!promotedRun)
3300
- return;
3301
- publishLiveEvent({
3302
- orgId: promotedRun.orgId,
3303
- type: "heartbeat.run.queued",
3304
- payload: {
3305
- runId: promotedRun.id,
3306
- agentId: promotedRun.agentId,
3307
- invocationSource: promotedRun.invocationSource,
3308
- triggerDetail: promotedRun.triggerDetail,
3309
- wakeupRequestId: promotedRun.wakeupRequestId,
3310
- },
3311
- });
3312
- await startNextQueuedRunForAgent(promotedRun.agentId);
3313
- }
3314
- async function enqueueWakeup(agentId, opts = {}) {
3315
- const source = opts.source ?? "on_demand";
3316
- const triggerDetail = opts.triggerDetail ?? null;
3317
- const contextSnapshot = { ...(opts.contextSnapshot ?? {}) };
3318
- const reason = opts.reason ?? null;
3319
- const payload = opts.payload ?? null;
3320
- const existingWakeupRequestId = readNonEmptyString(opts.existingWakeupRequestId);
3321
- const { contextSnapshot: enrichedContextSnapshot, issueIdFromPayload, taskKey, wakeCommentId, } = enrichWakeContextSnapshot({
3322
- contextSnapshot,
3323
- reason,
3324
- source,
3325
- triggerDetail,
3326
- payload,
3327
- });
3328
- let issueId = readNonEmptyString(enrichedContextSnapshot.issueId) ?? issueIdFromPayload;
3329
- const agent = await getAgent(agentId);
3330
- if (!agent)
3331
- throw notFound("Agent not found");
3332
- const explicitResumeSession = await resolveExplicitResumeSessionOverride(agent, payload, taskKey);
3333
- if (explicitResumeSession) {
3334
- enrichedContextSnapshot.resumeFromRunId = explicitResumeSession.resumeFromRunId;
3335
- enrichedContextSnapshot.resumeSessionDisplayId = explicitResumeSession.sessionDisplayId;
3336
- enrichedContextSnapshot.resumeSessionParams = explicitResumeSession.sessionParams;
3337
- if (!readNonEmptyString(enrichedContextSnapshot.issueId) && explicitResumeSession.issueId) {
3338
- enrichedContextSnapshot.issueId = explicitResumeSession.issueId;
3339
- }
3340
- if (!readNonEmptyString(enrichedContextSnapshot.taskId) && explicitResumeSession.taskId) {
3341
- enrichedContextSnapshot.taskId = explicitResumeSession.taskId;
3342
- }
3343
- if (!readNonEmptyString(enrichedContextSnapshot.taskKey) && explicitResumeSession.taskKey) {
3344
- enrichedContextSnapshot.taskKey = explicitResumeSession.taskKey;
3345
- }
3346
- issueId = readNonEmptyString(enrichedContextSnapshot.issueId) ?? issueId;
3347
- }
3348
- await hydrateWakeContextSnapshot(db, agent.orgId, enrichedContextSnapshot);
3349
- const effectiveTaskKey = readNonEmptyString(enrichedContextSnapshot.taskKey) ?? taskKey;
3350
- const sessionBefore = explicitResumeSession?.sessionDisplayId ??
3351
- await resolveSessionBeforeForWakeup(agent, effectiveTaskKey);
3352
- const writeSkippedRequest = async (skipReason) => {
3353
- if (existingWakeupRequestId) {
3354
- await setWakeupStatus(existingWakeupRequestId, "skipped", {
3355
- reason: skipReason,
3356
- finishedAt: new Date(),
3357
- runId: null,
3358
- claimedAt: null,
3359
- error: null,
3360
- });
3361
- return;
3362
- }
3363
- await db.insert(agentWakeupRequests).values({
3364
- orgId: agent.orgId,
3365
- agentId,
3366
- source,
3367
- triggerDetail,
3368
- reason: skipReason,
3369
- payload,
3370
- status: "skipped",
3371
- requestedByActorType: opts.requestedByActorType ?? null,
3372
- requestedByActorId: opts.requestedByActorId ?? null,
3373
- idempotencyKey: opts.idempotencyKey ?? null,
3374
- finishedAt: new Date(),
3375
- });
3376
- };
3377
- let projectId = readNonEmptyString(enrichedContextSnapshot.projectId);
3378
- if (!projectId && issueId) {
3379
- projectId = await db
3380
- .select({ projectId: issues.projectId })
3381
- .from(issues)
3382
- .where(and(eq(issues.id, issueId), eq(issues.orgId, agent.orgId)))
3383
- .then((rows) => rows[0]?.projectId ?? null);
3384
- }
3385
- const budgetBlock = await budgets.getInvocationBlock(agent.orgId, agentId, {
3386
- issueId,
3387
- projectId,
3388
- });
3389
- if (budgetBlock) {
3390
- await writeSkippedRequest("budget.blocked");
3391
- throw conflict(budgetBlock.reason, {
3392
- scopeType: budgetBlock.scopeType,
3393
- scopeId: budgetBlock.scopeId,
3394
- });
3395
- }
3396
- if (agent.status === "paused") {
3397
- const deferredPayload = buildDeferredWakePayload(payload, enrichedContextSnapshot, issueId);
3398
- if (existingWakeupRequestId) {
3399
- await setWakeupStatus(existingWakeupRequestId, "deferred_agent_paused", {
3400
- reason,
3401
- payload: deferredPayload,
3402
- runId: null,
3403
- claimedAt: null,
3404
- finishedAt: null,
3405
- error: null,
3406
- });
3407
- return null;
3408
- }
3409
- await db.transaction(async (tx) => {
3410
- const deferredRows = await tx
3411
- .select()
3412
- .from(agentWakeupRequests)
3413
- .where(and(eq(agentWakeupRequests.orgId, agent.orgId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_agent_paused"), sql `${agentWakeupRequests.runId} is null`))
3414
- .orderBy(asc(agentWakeupRequests.requestedAt));
3415
- const existingDeferred = deferredRows.find((candidate) => isSameTaskScope(deriveDeferredWakeTaskKey(candidate.payload), effectiveTaskKey));
3416
- if (existingDeferred) {
3417
- const mergedDeferredContext = mergeCoalescedContextSnapshot(readDeferredWakeContext(existingDeferred.payload), enrichedContextSnapshot);
3418
- await updateWakeupRequestRecord(tx, existingDeferred.id, {
3419
- payload: buildDeferredWakePayload({
3420
- ...readDeferredWakePayload(existingDeferred.payload),
3421
- ...(payload ?? {}),
3422
- }, mergedDeferredContext, issueId),
3423
- coalescedCount: (existingDeferred.coalescedCount ?? 0) + 1,
3424
- error: null,
3425
- finishedAt: null,
3426
- claimedAt: null,
3427
- runId: null,
3428
- });
3429
- return;
3430
- }
3431
- await insertWakeupRequestRecord(tx, {
3432
- orgId: agent.orgId,
3433
- agentId,
3434
- source,
3435
- triggerDetail,
3436
- reason,
3437
- payload: deferredPayload,
3438
- status: "deferred_agent_paused",
3439
- requestedByActorType: opts.requestedByActorType ?? null,
3440
- requestedByActorId: opts.requestedByActorId ?? null,
3441
- idempotencyKey: opts.idempotencyKey ?? null,
3442
- });
3443
- });
3444
- return null;
3445
- }
3446
- if (agent.status === "terminated" || agent.status === "pending_approval") {
3447
- throw conflict("Agent is not invokable in its current state", { status: agent.status });
3448
- }
3449
- const policy = parseHeartbeatPolicy(agent);
3450
- if (source === "timer" && !policy.enabled) {
3451
- await writeSkippedRequest("heartbeat.disabled");
3452
- return null;
3453
- }
3454
- if (source !== "timer" && !policy.wakeOnDemand) {
3455
- await writeSkippedRequest("heartbeat.wakeOnDemand.disabled");
3456
- return null;
3457
- }
3458
- const bypassIssueExecutionLock = reason === "issue_comment_mentioned" ||
3459
- readNonEmptyString(enrichedContextSnapshot.wakeReason) === "issue_comment_mentioned";
3460
- if (issueId && !bypassIssueExecutionLock) {
3461
- const agentNameKey = normalizeAgentNameKey(agent.name);
3462
- const outcome = await db.transaction(async (tx) => {
3463
- await tx.execute(sql `select id from issues where id = ${issueId} and org_id = ${agent.orgId} for update`);
3464
- const issue = await tx
3465
- .select({
3466
- id: issues.id,
3467
- orgId: issues.orgId,
3468
- executionRunId: issues.executionRunId,
3469
- executionAgentNameKey: issues.executionAgentNameKey,
3470
- })
3471
- .from(issues)
3472
- .where(and(eq(issues.id, issueId), eq(issues.orgId, agent.orgId)))
3473
- .then((rows) => rows[0] ?? null);
3474
- if (!issue) {
3475
- if (existingWakeupRequestId) {
3476
- await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
3477
- status: "skipped",
3478
- reason: "issue_execution_issue_not_found",
3479
- runId: null,
3480
- claimedAt: null,
3481
- finishedAt: new Date(),
3482
- error: null,
3483
- });
3484
- }
3485
- else {
3486
- await insertWakeupRequestRecord(tx, {
3487
- orgId: agent.orgId,
3488
- agentId,
3489
- source,
3490
- triggerDetail,
3491
- reason: "issue_execution_issue_not_found",
3492
- payload,
3493
- status: "skipped",
3494
- requestedByActorType: opts.requestedByActorType ?? null,
3495
- requestedByActorId: opts.requestedByActorId ?? null,
3496
- idempotencyKey: opts.idempotencyKey ?? null,
3497
- finishedAt: new Date(),
3498
- });
3499
- }
3500
- return { kind: "skipped" };
3501
- }
3502
- let activeExecutionRun = issue.executionRunId
3503
- ? await tx
3504
- .select()
3505
- .from(heartbeatRuns)
3506
- .where(eq(heartbeatRuns.id, issue.executionRunId))
3507
- .then((rows) => rows[0] ?? null)
3508
- : null;
3509
- if (activeExecutionRun && activeExecutionRun.status !== "queued" && activeExecutionRun.status !== "running") {
3510
- activeExecutionRun = null;
3511
- }
3512
- if (!activeExecutionRun && issue.executionRunId) {
3513
- await tx
3514
- .update(issues)
3515
- .set({
3516
- executionRunId: null,
3517
- executionAgentNameKey: null,
3518
- executionLockedAt: null,
3519
- updatedAt: new Date(),
3520
- })
3521
- .where(eq(issues.id, issue.id));
3522
- }
3523
- if (!activeExecutionRun) {
3524
- const legacyRun = await tx
3525
- .select()
3526
- .from(heartbeatRuns)
3527
- .where(and(eq(heartbeatRuns.orgId, issue.orgId), inArray(heartbeatRuns.status, ["queued", "running"]), sql `${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`))
3528
- .orderBy(sql `case when ${heartbeatRuns.status} = 'running' then 0 else 1 end`, asc(heartbeatRuns.createdAt))
3529
- .limit(1)
3530
- .then((rows) => rows[0] ?? null);
3531
- if (legacyRun) {
3532
- activeExecutionRun = legacyRun;
3533
- const legacyAgent = await tx
3534
- .select({ name: agents.name })
3535
- .from(agents)
3536
- .where(eq(agents.id, legacyRun.agentId))
3537
- .then((rows) => rows[0] ?? null);
3538
- await tx
3539
- .update(issues)
3540
- .set({
3541
- executionRunId: legacyRun.id,
3542
- executionAgentNameKey: normalizeAgentNameKey(legacyAgent?.name),
3543
- executionLockedAt: new Date(),
3544
- updatedAt: new Date(),
3545
- })
3546
- .where(eq(issues.id, issue.id));
3547
- }
3548
- }
3549
- if (activeExecutionRun) {
3550
- const executionAgent = await tx
3551
- .select({ name: agents.name })
3552
- .from(agents)
3553
- .where(eq(agents.id, activeExecutionRun.agentId))
3554
- .then((rows) => rows[0] ?? null);
3555
- const executionAgentNameKey = normalizeAgentNameKey(issue.executionAgentNameKey) ??
3556
- normalizeAgentNameKey(executionAgent?.name);
3557
- const isSameExecutionAgent = Boolean(executionAgentNameKey) && executionAgentNameKey === agentNameKey;
3558
- const shouldQueueFollowupForCommentWake = Boolean(wakeCommentId) &&
3559
- activeExecutionRun.status === "running" &&
3560
- isSameExecutionAgent;
3561
- if (isSameExecutionAgent && !shouldQueueFollowupForCommentWake) {
3562
- const mergedContextSnapshot = mergeCoalescedContextSnapshot(activeExecutionRun.contextSnapshot, enrichedContextSnapshot);
3563
- const mergedRun = await tx
3564
- .update(heartbeatRuns)
3565
- .set({
3566
- contextSnapshot: mergedContextSnapshot,
3567
- updatedAt: new Date(),
3568
- })
3569
- .where(eq(heartbeatRuns.id, activeExecutionRun.id))
3570
- .returning()
3571
- .then((rows) => rows[0] ?? activeExecutionRun);
3572
- if (existingWakeupRequestId) {
3573
- await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
3574
- status: "coalesced",
3575
- reason: "issue_execution_same_name",
3576
- runId: mergedRun.id,
3577
- claimedAt: null,
3578
- finishedAt: new Date(),
3579
- error: null,
3580
- });
3581
- }
3582
- else {
3583
- await insertWakeupRequestRecord(tx, {
3584
- orgId: agent.orgId,
3585
- agentId,
3586
- source,
3587
- triggerDetail,
3588
- reason: "issue_execution_same_name",
3589
- payload,
3590
- status: "coalesced",
3591
- coalescedCount: 1,
3592
- requestedByActorType: opts.requestedByActorType ?? null,
3593
- requestedByActorId: opts.requestedByActorId ?? null,
3594
- idempotencyKey: opts.idempotencyKey ?? null,
3595
- runId: mergedRun.id,
3596
- finishedAt: new Date(),
3597
- });
3598
- }
3599
- return { kind: "coalesced", run: mergedRun };
3600
- }
3601
- const deferredPayload = buildDeferredWakePayload(payload, enrichedContextSnapshot, issueId);
3602
- const existingDeferred = await tx
3603
- .select()
3604
- .from(agentWakeupRequests)
3605
- .where(and(eq(agentWakeupRequests.orgId, agent.orgId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql `${agentWakeupRequests.payload} ->> 'issueId' = ${issue.id}`))
3606
- .orderBy(asc(agentWakeupRequests.requestedAt))
3607
- .limit(1)
3608
- .then((rows) => rows[0] ?? null);
3609
- if (existingDeferred) {
3610
- const mergedDeferredContext = mergeCoalescedContextSnapshot(readDeferredWakeContext(existingDeferred.payload), enrichedContextSnapshot);
3611
- const mergedDeferredPayload = buildDeferredWakePayload({
3612
- ...readDeferredWakePayload(existingDeferred.payload),
3613
- ...(payload ?? {}),
3614
- }, mergedDeferredContext, issueId);
3615
- if (existingWakeupRequestId && existingDeferred.id !== existingWakeupRequestId) {
3616
- await updateWakeupRequestRecord(tx, existingDeferred.id, {
3617
- payload: mergedDeferredPayload,
3618
- coalescedCount: (existingDeferred.coalescedCount ?? 0) + 1,
3619
- });
3620
- await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
3621
- status: "coalesced",
3622
- reason: "issue_execution_deferred",
3623
- runId: null,
3624
- claimedAt: null,
3625
- finishedAt: new Date(),
3626
- error: null,
3627
- });
3628
- }
3629
- else {
3630
- await updateWakeupRequestRecord(tx, existingDeferred.id, {
3631
- payload: mergedDeferredPayload,
3632
- coalescedCount: (existingDeferred.coalescedCount ?? 0) + 1,
3633
- status: "deferred_issue_execution",
3634
- reason: "issue_execution_deferred",
3635
- runId: null,
3636
- claimedAt: null,
3637
- finishedAt: null,
3638
- error: null,
3639
- });
3640
- }
3641
- return { kind: "deferred" };
3642
- }
3643
- if (existingWakeupRequestId) {
3644
- await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
3645
- status: "deferred_issue_execution",
3646
- reason: "issue_execution_deferred",
3647
- payload: deferredPayload,
3648
- runId: null,
3649
- claimedAt: null,
3650
- finishedAt: null,
3651
- error: null,
3652
- });
3653
- }
3654
- else {
3655
- await insertWakeupRequestRecord(tx, {
3656
- orgId: agent.orgId,
3657
- agentId,
3658
- source,
3659
- triggerDetail,
3660
- reason: "issue_execution_deferred",
3661
- payload: deferredPayload,
3662
- status: "deferred_issue_execution",
3663
- requestedByActorType: opts.requestedByActorType ?? null,
3664
- requestedByActorId: opts.requestedByActorId ?? null,
3665
- idempotencyKey: opts.idempotencyKey ?? null,
3666
- });
3667
- }
3668
- return { kind: "deferred" };
3669
- }
3670
- const wakeupRequest = existingWakeupRequestId
3671
- ? await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
3672
- status: "queued",
3673
- runId: null,
3674
- claimedAt: null,
3675
- finishedAt: null,
3676
- error: null,
3677
- })
3678
- : await insertWakeupRequestRecord(tx, {
3679
- orgId: agent.orgId,
3680
- agentId,
3681
- source,
3682
- triggerDetail,
3683
- reason,
3684
- payload,
3685
- status: "queued",
3686
- requestedByActorType: opts.requestedByActorType ?? null,
3687
- requestedByActorId: opts.requestedByActorId ?? null,
3688
- idempotencyKey: opts.idempotencyKey ?? null,
3689
- });
3690
- const newRun = await tx
3691
- .insert(heartbeatRuns)
3692
- .values({
3693
- orgId: agent.orgId,
3694
- agentId,
3695
- invocationSource: source,
3696
- triggerDetail,
3697
- status: "queued",
3698
- wakeupRequestId: wakeupRequest.id,
3699
- contextSnapshot: enrichedContextSnapshot,
3700
- sessionIdBefore: sessionBefore,
3701
- })
3702
- .returning()
3703
- .then((rows) => rows[0]);
3704
- await updateWakeupRequestRecord(tx, wakeupRequest.id, {
3705
- runId: newRun.id,
3706
- status: "queued",
3707
- claimedAt: null,
3708
- finishedAt: null,
3709
- error: null,
3710
- });
3711
- await tx
3712
- .update(issues)
3713
- .set({
3714
- executionRunId: newRun.id,
3715
- executionAgentNameKey: agentNameKey,
3716
- executionLockedAt: new Date(),
3717
- updatedAt: new Date(),
3718
- })
3719
- .where(eq(issues.id, issue.id));
3720
- return { kind: "queued", run: newRun };
3721
- });
3722
- if (outcome.kind === "deferred" || outcome.kind === "skipped")
3723
- return null;
3724
- if (outcome.kind === "coalesced")
3725
- return outcome.run;
3726
- const newRun = outcome.run;
3727
- publishLiveEvent({
3728
- orgId: newRun.orgId,
3729
- type: "heartbeat.run.queued",
3730
- payload: {
3731
- runId: newRun.id,
3732
- agentId: newRun.agentId,
3733
- invocationSource: newRun.invocationSource,
3734
- triggerDetail: newRun.triggerDetail,
3735
- wakeupRequestId: newRun.wakeupRequestId,
3736
- },
3737
- });
3738
- await startNextQueuedRunForAgent(agent.id);
3739
- return newRun;
3740
- }
3741
- const activeRuns = await db
3742
- .select()
3743
- .from(heartbeatRuns)
3744
- .where(and(eq(heartbeatRuns.agentId, agentId), inArray(heartbeatRuns.status, ["queued", "running"])))
3745
- .orderBy(desc(heartbeatRuns.createdAt));
3746
- const sameScopeQueuedRun = activeRuns.find((candidate) => candidate.status === "queued" && isSameTaskScope(runTaskKey(candidate), taskKey));
3747
- const sameScopeRunningRun = activeRuns.find((candidate) => candidate.status === "running" && isSameTaskScope(runTaskKey(candidate), taskKey));
3748
- const shouldQueueFollowupForCommentWake = Boolean(wakeCommentId) && Boolean(sameScopeRunningRun) && !sameScopeQueuedRun;
3749
- const coalescedTargetRun = sameScopeQueuedRun ??
3750
- (shouldQueueFollowupForCommentWake ? null : sameScopeRunningRun ?? null);
3751
- if (coalescedTargetRun) {
3752
- const mergedContextSnapshot = mergeCoalescedContextSnapshot(coalescedTargetRun.contextSnapshot, contextSnapshot);
3753
- const mergedRun = await db
3754
- .update(heartbeatRuns)
3755
- .set({
3756
- contextSnapshot: mergedContextSnapshot,
3757
- updatedAt: new Date(),
3758
- })
3759
- .where(eq(heartbeatRuns.id, coalescedTargetRun.id))
3760
- .returning()
3761
- .then((rows) => rows[0] ?? coalescedTargetRun);
3762
- if (existingWakeupRequestId) {
3763
- await setWakeupStatus(existingWakeupRequestId, "coalesced", {
3764
- runId: mergedRun.id,
3765
- claimedAt: null,
3766
- finishedAt: new Date(),
3767
- error: null,
3768
- });
3769
- }
3770
- else {
3771
- await db.insert(agentWakeupRequests).values({
3772
- orgId: agent.orgId,
3773
- agentId,
3774
- source,
3775
- triggerDetail,
3776
- reason,
3777
- payload,
3778
- status: "coalesced",
3779
- coalescedCount: 1,
3780
- requestedByActorType: opts.requestedByActorType ?? null,
3781
- requestedByActorId: opts.requestedByActorId ?? null,
3782
- idempotencyKey: opts.idempotencyKey ?? null,
3783
- runId: mergedRun.id,
3784
- finishedAt: new Date(),
3785
- });
3786
- }
3787
- return mergedRun;
3788
- }
3789
- const wakeupRequest = existingWakeupRequestId
3790
- ? await updateWakeupRequestRecord(db, existingWakeupRequestId, {
3791
- status: "queued",
3792
- runId: null,
3793
- claimedAt: null,
3794
- finishedAt: null,
3795
- error: null,
3796
- })
3797
- : await insertWakeupRequestRecord(db, {
3798
- orgId: agent.orgId,
3799
- agentId,
3800
- source,
3801
- triggerDetail,
3802
- reason,
3803
- payload,
3804
- status: "queued",
3805
- requestedByActorType: opts.requestedByActorType ?? null,
3806
- requestedByActorId: opts.requestedByActorId ?? null,
3807
- idempotencyKey: opts.idempotencyKey ?? null,
3808
- });
3809
- const newRun = await db
3810
- .insert(heartbeatRuns)
3811
- .values({
3812
- orgId: agent.orgId,
3813
- agentId,
3814
- invocationSource: source,
3815
- triggerDetail,
3816
- status: "queued",
3817
- wakeupRequestId: wakeupRequest.id,
3818
- contextSnapshot: enrichedContextSnapshot,
3819
- sessionIdBefore: sessionBefore,
3820
- })
3821
- .returning()
3822
- .then((rows) => rows[0]);
3823
- await updateWakeupRequestRecord(db, wakeupRequest.id, {
3824
- status: "queued",
3825
- runId: newRun.id,
3826
- claimedAt: null,
3827
- finishedAt: null,
3828
- error: null,
3829
- });
3830
- publishLiveEvent({
3831
- orgId: newRun.orgId,
3832
- type: "heartbeat.run.queued",
3833
- payload: {
3834
- runId: newRun.id,
3835
- agentId: newRun.agentId,
3836
- invocationSource: newRun.invocationSource,
3837
- triggerDetail: newRun.triggerDetail,
3838
- wakeupRequestId: newRun.wakeupRequestId,
3839
- },
3840
- });
3841
- await startNextQueuedRunForAgent(agent.id);
3842
- return newRun;
3843
- }
3844
- async function resumeDeferredWakeupsForAgent(agentId) {
3845
- const agent = await getAgent(agentId);
3846
- if (!agent)
3847
- throw notFound("Agent not found");
3848
- const replayedRequestIds = [];
3849
- while (true) {
3850
- const deferred = await db
3851
- .select()
3852
- .from(agentWakeupRequests)
3853
- .where(and(eq(agentWakeupRequests.orgId, agent.orgId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_agent_paused"), sql `${agentWakeupRequests.runId} is null`))
3854
- .orderBy(asc(agentWakeupRequests.requestedAt))
3855
- .limit(1)
3856
- .then((rows) => rows[0] ?? null);
3857
- if (!deferred)
3858
- break;
3859
- const replayPayload = readDeferredWakePayload(deferred.payload);
3860
- const replayContextSnapshot = readDeferredWakeContext(deferred.payload);
3861
- try {
3862
- await enqueueWakeup(agentId, {
3863
- source: readNonEmptyString(deferred.source) ?? "on_demand",
3864
- triggerDetail: readNonEmptyString(deferred.triggerDetail) ?? undefined,
3865
- reason: readNonEmptyString(deferred.reason) ?? null,
3866
- payload: replayPayload,
3867
- idempotencyKey: deferred.idempotencyKey,
3868
- requestedByActorType: deferred.requestedByActorType ?? undefined,
3869
- requestedByActorId: deferred.requestedByActorId,
3870
- contextSnapshot: replayContextSnapshot,
3871
- existingWakeupRequestId: deferred.id,
3872
- });
3873
- }
3874
- catch (error) {
3875
- const current = await db
3876
- .select({ status: agentWakeupRequests.status })
3877
- .from(agentWakeupRequests)
3878
- .where(eq(agentWakeupRequests.id, deferred.id))
3879
- .then((rows) => rows[0] ?? null);
3880
- if (current?.status === "deferred_agent_paused") {
3881
- await setWakeupStatus(deferred.id, "failed", {
3882
- finishedAt: new Date(),
3883
- error: error instanceof Error ? error.message : String(error),
3884
- });
3885
- }
3886
- }
3887
- replayedRequestIds.push(deferred.id);
3888
- const current = await db
3889
- .select({ status: agentWakeupRequests.status })
3890
- .from(agentWakeupRequests)
3891
- .where(eq(agentWakeupRequests.id, deferred.id))
3892
- .then((rows) => rows[0] ?? null);
3893
- if (current?.status === "deferred_agent_paused")
3894
- break;
3895
- }
3896
- return {
3897
- replayed: replayedRequestIds.length,
3898
- wakeupRequestIds: replayedRequestIds,
3899
- };
3900
- }
3901
- async function listProjectScopedRunIds(orgId, projectId) {
3902
- const runIssueId = sql `${heartbeatRuns.contextSnapshot} ->> 'issueId'`;
3903
- const effectiveProjectId = sql `coalesce(${heartbeatRuns.contextSnapshot} ->> 'projectId', ${issues.projectId}::text)`;
3904
- const rows = await db
3905
- .selectDistinctOn([heartbeatRuns.id], { id: heartbeatRuns.id })
3906
- .from(heartbeatRuns)
3907
- .leftJoin(issues, and(eq(issues.orgId, orgId), sql `${issues.id}::text = ${runIssueId}`))
3908
- .where(and(eq(heartbeatRuns.orgId, orgId), inArray(heartbeatRuns.status, ["queued", "running"]), sql `${effectiveProjectId} = ${projectId}`));
3909
- return rows.map((row) => row.id);
3910
- }
3911
- async function listProjectScopedWakeupIds(orgId, projectId) {
3912
- const wakeIssueId = sql `${agentWakeupRequests.payload} ->> 'issueId'`;
3913
- const effectiveProjectId = sql `coalesce(${agentWakeupRequests.payload} ->> 'projectId', ${issues.projectId}::text)`;
3914
- const rows = await db
3915
- .selectDistinctOn([agentWakeupRequests.id], { id: agentWakeupRequests.id })
3916
- .from(agentWakeupRequests)
3917
- .leftJoin(issues, and(eq(issues.orgId, orgId), sql `${issues.id}::text = ${wakeIssueId}`))
3918
- .where(and(eq(agentWakeupRequests.orgId, orgId), inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), sql `${agentWakeupRequests.runId} is null`, sql `${effectiveProjectId} = ${projectId}`));
3919
- return rows.map((row) => row.id);
3920
- }
3921
- async function cancelPendingWakeupsForBudgetScope(scope) {
3922
- const now = new Date();
3923
- let wakeupIds = [];
3924
- if (scope.scopeType === "organization") {
3925
- wakeupIds = await db
3926
- .select({ id: agentWakeupRequests.id })
3927
- .from(agentWakeupRequests)
3928
- .where(and(eq(agentWakeupRequests.orgId, scope.orgId), inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), sql `${agentWakeupRequests.runId} is null`))
3929
- .then((rows) => rows.map((row) => row.id));
3930
- }
3931
- else if (scope.scopeType === "agent") {
3932
- wakeupIds = await db
3933
- .select({ id: agentWakeupRequests.id })
3934
- .from(agentWakeupRequests)
3935
- .where(and(eq(agentWakeupRequests.orgId, scope.orgId), eq(agentWakeupRequests.agentId, scope.scopeId), inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), sql `${agentWakeupRequests.runId} is null`))
3936
- .then((rows) => rows.map((row) => row.id));
3937
- }
3938
- else {
3939
- wakeupIds = await listProjectScopedWakeupIds(scope.orgId, scope.scopeId);
3940
- }
3941
- if (wakeupIds.length === 0)
3942
- return 0;
3943
- await db
3944
- .update(agentWakeupRequests)
3945
- .set({
3946
- status: "cancelled",
3947
- finishedAt: now,
3948
- error: "Cancelled due to budget pause",
3949
- updatedAt: now,
3950
- })
3951
- .where(inArray(agentWakeupRequests.id, wakeupIds));
3952
- return wakeupIds.length;
3953
- }
3954
- async function cancelRunInternal(runId, reason = "Cancelled by control plane") {
3955
- const run = await getRun(runId);
3956
- if (!run)
3957
- throw notFound("Heartbeat run not found");
3958
- if (run.status !== "running" && run.status !== "queued")
3959
- return run;
3960
- const running = runningProcesses.get(run.id);
3961
- if (running) {
3962
- running.child.kill("SIGTERM");
3963
- const graceMs = Math.max(1, running.graceSec) * 1000;
3964
- setTimeout(() => {
3965
- if (!running.child.killed) {
3966
- running.child.kill("SIGKILL");
3967
- }
3968
- }, graceMs);
3969
- }
3970
- const cancelled = await setRunStatus(run.id, "cancelled", {
3971
- finishedAt: new Date(),
3972
- error: reason,
3973
- errorCode: "cancelled",
3974
- });
3975
- await setWakeupStatus(run.wakeupRequestId, "cancelled", {
3976
- finishedAt: new Date(),
3977
- error: reason,
3978
- });
3979
- if (cancelled) {
3980
- await appendRunEvent(cancelled, 1, {
3981
- eventType: "lifecycle",
3982
- stream: "system",
3983
- level: "warn",
3984
- message: "run cancelled",
3985
- });
3986
- await releaseIssueExecutionAndPromote(cancelled);
3987
- }
3988
- runningProcesses.delete(run.id);
3989
- await finalizeAgentStatus(run.agentId, "cancelled");
3990
- await startNextQueuedRunForAgent(run.agentId);
3991
- return cancelled;
3992
- }
3993
- async function cancelActiveForAgentInternal(agentId, reason = "Cancelled due to agent pause") {
3994
- const runs = await db
3995
- .select()
3996
- .from(heartbeatRuns)
3997
- .where(and(eq(heartbeatRuns.agentId, agentId), inArray(heartbeatRuns.status, ["queued", "running"])));
3998
- for (const run of runs) {
3999
- await setRunStatus(run.id, "cancelled", {
4000
- finishedAt: new Date(),
4001
- error: reason,
4002
- errorCode: "cancelled",
4003
- });
4004
- await setWakeupStatus(run.wakeupRequestId, "cancelled", {
4005
- finishedAt: new Date(),
4006
- error: reason,
4007
- });
4008
- const running = runningProcesses.get(run.id);
4009
- if (running) {
4010
- running.child.kill("SIGTERM");
4011
- runningProcesses.delete(run.id);
4012
- }
4013
- await releaseIssueExecutionAndPromote(run);
4014
- }
4015
- return runs.length;
4016
- }
4017
- async function cancelBudgetScopeWork(scope) {
4018
- if (scope.scopeType === "agent") {
4019
- await cancelActiveForAgentInternal(scope.scopeId, "Cancelled due to budget pause");
4020
- await cancelPendingWakeupsForBudgetScope(scope);
4021
- return;
4022
- }
4023
- const runIds = scope.scopeType === "organization"
4024
- ? await db
4025
- .select({ id: heartbeatRuns.id })
4026
- .from(heartbeatRuns)
4027
- .where(and(eq(heartbeatRuns.orgId, scope.orgId), inArray(heartbeatRuns.status, ["queued", "running"])))
4028
- .then((rows) => rows.map((row) => row.id))
4029
- : await listProjectScopedRunIds(scope.orgId, scope.scopeId);
4030
- for (const runId of runIds) {
4031
- await cancelRunInternal(runId, "Cancelled due to budget pause");
4032
- }
4033
- await cancelPendingWakeupsForBudgetScope(scope);
4034
- }
4035
- async function retryRunInternal(runId, opts) {
4036
- const run = await getRun(runId);
4037
- if (!run)
4038
- throw notFound("Heartbeat run not found");
4039
- if (run.status !== "failed" && run.status !== "timed_out") {
4040
- throw conflict("Only failed or timed out runs can be retried", {
4041
- status: run.status,
4042
- });
4043
- }
4044
- const agent = await getAgent(run.agentId);
4045
- if (!agent)
4046
- throw notFound("Agent not found");
4047
- if (agent.status === "paused" ||
4048
- agent.status === "terminated" ||
4049
- agent.status === "pending_approval") {
4050
- throw conflict("Agent is not invokable in its current state", { status: agent.status });
4051
- }
4052
- const policy = parseHeartbeatPolicy(agent);
4053
- if (!policy.wakeOnDemand) {
4054
- throw conflict("Agent is not configured for on-demand wakeups");
4055
- }
4056
- const context = parseObject(run.contextSnapshot);
4057
- const issueId = readNonEmptyString(context.issueId);
4058
- let projectId = readNonEmptyString(context.projectId);
4059
- if (!projectId && issueId) {
4060
- projectId = await db
4061
- .select({ projectId: issues.projectId })
4062
- .from(issues)
4063
- .where(and(eq(issues.id, issueId), eq(issues.orgId, agent.orgId)))
4064
- .then((rows) => rows[0]?.projectId ?? null);
4065
- }
4066
- const budgetBlock = await budgets.getInvocationBlock(agent.orgId, agent.id, {
4067
- issueId,
4068
- projectId,
4069
- });
4070
- if (budgetBlock) {
4071
- throw conflict(budgetBlock.reason, {
4072
- scopeType: budgetBlock.scopeType,
4073
- scopeId: budgetBlock.scopeId,
4074
- });
4075
- }
4076
- return enqueueRecoveryRun(run, agent, {
4077
- recoveryTrigger: "manual",
4078
- source: "on_demand",
4079
- triggerDetail: "manual",
4080
- wakeReason: "retry_failed_run",
4081
- requestedByActorType: opts?.requestedByActorType ?? "user",
4082
- requestedByActorId: opts?.requestedByActorId ?? null,
4083
- now: opts?.now ?? new Date(),
4084
- });
4085
- }
4086
- return {
4087
- list: async (orgId, agentId, limit) => {
4088
- const query = db
4089
- .select(heartbeatRunListColumns)
4090
- .from(heartbeatRuns)
4091
- .where(agentId
4092
- ? and(eq(heartbeatRuns.orgId, orgId), eq(heartbeatRuns.agentId, agentId))
4093
- : eq(heartbeatRuns.orgId, orgId))
4094
- .orderBy(desc(heartbeatRuns.createdAt));
4095
- const rows = limit ? await query.limit(limit) : await query;
4096
- return rows.map((row) => ({
4097
- ...row,
4098
- resultJson: summarizeHeartbeatRunResultJson(row.resultJson),
4099
- }));
4100
- },
4101
- getAgentSkillAnalytics: async (agentId, opts) => {
4102
- const agent = await getAgent(agentId);
4103
- if (!agent)
4104
- throw notFound("Agent not found");
4105
- const now = opts?.now ?? new Date();
4106
- const customDateKeys = opts?.startDate && opts?.endDate
4107
- ? buildDateKeysBetween(opts.startDate, opts.endDate).slice(0, 120)
4108
- : [];
4109
- const windowDays = customDateKeys.length > 0
4110
- ? customDateKeys.length
4111
- : Math.max(1, Math.min(opts?.windowDays ?? 30, 90));
4112
- const dateKeys = customDateKeys.length > 0
4113
- ? customDateKeys
4114
- : buildRecentDateKeys(windowDays, now);
4115
- const startDate = dateKeys[0];
4116
- const endDate = dateKeys.at(-1);
4117
- const windowStart = new Date(`${startDate}T00:00:00.000Z`);
4118
- const windowEnd = new Date(`${endDate}T23:59:59.999Z`);
4119
- const rows = await db
4120
- .select({
4121
- createdAt: heartbeatRunEvents.createdAt,
4122
- payload: heartbeatRunEvents.payload,
4123
- })
4124
- .from(heartbeatRunEvents)
4125
- .where(and(eq(heartbeatRunEvents.orgId, agent.orgId), eq(heartbeatRunEvents.agentId, agent.id), eq(heartbeatRunEvents.eventType, "adapter.invoke"), gte(heartbeatRunEvents.createdAt, windowStart), lte(heartbeatRunEvents.createdAt, windowEnd)))
4126
- .orderBy(asc(heartbeatRunEvents.createdAt), asc(heartbeatRunEvents.id));
4127
- const days = new Map();
4128
- for (const date of dateKeys) {
4129
- days.set(date, { totalCount: 0, runCount: 0, skills: new Map() });
4130
- }
4131
- const overallSkills = new Map();
4132
- let totalCount = 0;
4133
- let totalRunsWithSkills = 0;
4134
- for (const row of rows) {
4135
- const date = new Date(row.createdAt).toISOString().slice(0, 10);
4136
- const bucket = days.get(date);
4137
- if (!bucket)
4138
- continue;
4139
- const payload = parseObject(row.payload);
4140
- const loadedSkills = Array.isArray(payload.loadedSkills) ? payload.loadedSkills : [];
4141
- if (loadedSkills.length === 0)
4142
- continue;
4143
- const eventSkills = new Map();
4144
- for (const entry of loadedSkills) {
4145
- const normalized = normalizeLoadedSkill(entry);
4146
- if (!normalized)
4147
- continue;
4148
- if (!eventSkills.has(normalized.key)) {
4149
- eventSkills.set(normalized.key, normalized.label);
4150
- }
4151
- }
4152
- if (eventSkills.size === 0)
4153
- continue;
4154
- bucket.runCount += 1;
4155
- totalRunsWithSkills += 1;
4156
- for (const [key, label] of eventSkills) {
4157
- bucket.totalCount += 1;
4158
- totalCount += 1;
4159
- const existingDaySkill = bucket.skills.get(key);
4160
- if (existingDaySkill) {
4161
- existingDaySkill.count += 1;
4162
- }
4163
- else {
4164
- bucket.skills.set(key, { key, label, count: 1 });
4165
- }
4166
- const existingOverallSkill = overallSkills.get(key);
4167
- if (existingOverallSkill) {
4168
- existingOverallSkill.count += 1;
4169
- }
4170
- else {
4171
- overallSkills.set(key, { key, label, count: 1 });
4172
- }
4173
- }
4174
- }
4175
- return {
4176
- agentId: agent.id,
4177
- orgId: agent.orgId,
4178
- windowDays,
4179
- startDate,
4180
- endDate,
4181
- totalCount,
4182
- totalRunsWithSkills,
4183
- skills: Array.from(overallSkills.values()).sort((left, right) => (right.count - left.count
4184
- || left.label.localeCompare(right.label)
4185
- || left.key.localeCompare(right.key))),
4186
- days: dateKeys.map((date) => {
4187
- const bucket = days.get(date);
4188
- return {
4189
- date,
4190
- totalCount: bucket.totalCount,
4191
- runCount: bucket.runCount,
4192
- skills: Array.from(bucket.skills.values()).sort((left, right) => (right.count - left.count
4193
- || left.label.localeCompare(right.label)
4194
- || left.key.localeCompare(right.key))),
4195
- };
4196
- }),
4197
- };
4198
- },
4199
- getRun,
4200
- getRuntimeState: async (agentId) => {
4201
- const state = await getRuntimeState(agentId);
4202
- const agent = await getAgent(agentId);
4203
- if (!agent)
4204
- return null;
4205
- const ensured = state ?? (await ensureRuntimeState(agent));
4206
- const latestTaskSession = await db
4207
- .select()
4208
- .from(agentTaskSessions)
4209
- .where(and(eq(agentTaskSessions.orgId, agent.orgId), eq(agentTaskSessions.agentId, agent.id)))
4210
- .orderBy(desc(agentTaskSessions.updatedAt))
4211
- .limit(1)
4212
- .then((rows) => rows[0] ?? null);
4213
- return {
4214
- ...ensured,
4215
- sessionDisplayId: latestTaskSession?.sessionDisplayId ?? ensured.sessionId,
4216
- sessionParamsJson: latestTaskSession?.sessionParamsJson ?? null,
4217
- };
4218
- },
4219
- listTaskSessions: async (agentId) => {
4220
- const agent = await getAgent(agentId);
4221
- if (!agent)
4222
- throw notFound("Agent not found");
4223
- return db
4224
- .select()
4225
- .from(agentTaskSessions)
4226
- .where(and(eq(agentTaskSessions.orgId, agent.orgId), eq(agentTaskSessions.agentId, agentId)))
4227
- .orderBy(desc(agentTaskSessions.updatedAt), desc(agentTaskSessions.createdAt));
4228
- },
4229
- resetRuntimeSession: async (agentId, opts) => {
4230
- const agent = await getAgent(agentId);
4231
- if (!agent)
4232
- throw notFound("Agent not found");
4233
- await ensureRuntimeState(agent);
4234
- const taskKey = readNonEmptyString(opts?.taskKey);
4235
- const clearedTaskSessions = await clearTaskSessions(agent.orgId, agent.id, taskKey ? { taskKey, agentRuntimeType: agent.agentRuntimeType } : undefined);
4236
- const runtimePatch = {
4237
- sessionId: null,
4238
- lastError: null,
4239
- updatedAt: new Date(),
4240
- };
4241
- if (!taskKey) {
4242
- runtimePatch.stateJson = {};
4243
- }
4244
- const updated = await db
4245
- .update(agentRuntimeState)
4246
- .set(runtimePatch)
4247
- .where(eq(agentRuntimeState.agentId, agentId))
4248
- .returning()
4249
- .then((rows) => rows[0] ?? null);
4250
- if (!updated)
4251
- return null;
4252
- return {
4253
- ...updated,
4254
- sessionDisplayId: null,
4255
- sessionParamsJson: null,
4256
- clearedTaskSessions,
4257
- };
4258
- },
4259
- listEvents: (runId, afterSeq = 0, limit = 200) => db
4260
- .select()
4261
- .from(heartbeatRunEvents)
4262
- .where(and(eq(heartbeatRunEvents.runId, runId), gt(heartbeatRunEvents.seq, afterSeq)))
4263
- .orderBy(asc(heartbeatRunEvents.seq))
4264
- .limit(Math.max(1, Math.min(limit, 1000))),
4265
- readLog: async (runId, opts) => {
4266
- const run = await getRun(runId);
4267
- if (!run)
4268
- throw notFound("Heartbeat run not found");
4269
- if (!run.logStore || !run.logRef)
4270
- throw notFound("Run log not found");
4271
- const result = await runLogStore.read({
4272
- store: run.logStore,
4273
- logRef: run.logRef,
4274
- }, opts);
4275
- return {
4276
- runId,
4277
- store: run.logStore,
4278
- logRef: run.logRef,
4279
- ...result,
4280
- content: redactCurrentUserText(result.content, await getCurrentUserRedactionOptions()),
4281
- };
4282
- },
4283
- invoke: async (agentId, source = "on_demand", contextSnapshot = {}, triggerDetail = "manual", actor) => enqueueWakeup(agentId, {
4284
- source,
4285
- triggerDetail,
4286
- contextSnapshot,
4287
- requestedByActorType: actor?.actorType,
4288
- requestedByActorId: actor?.actorId ?? null,
4289
- }),
4290
- wakeup: enqueueWakeup,
4291
- resumeDeferredWakeupsForAgent,
4292
- retryRun: retryRunInternal,
4293
- reportRunActivity: clearDetachedRunWarning,
4294
- reapOrphanedRuns,
4295
- resumeQueuedRuns,
4296
- tickTimers: async (now = new Date()) => {
4297
- const allAgents = await db.select().from(agents);
4298
- let checked = 0;
4299
- let enqueued = 0;
4300
- let skipped = 0;
4301
- for (const agent of allAgents) {
4302
- if (agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval")
4303
- continue;
4304
- const policy = parseHeartbeatPolicy(agent);
4305
- if (!policy.enabled || policy.intervalSec <= 0)
4306
- continue;
4307
- checked += 1;
4308
- const baseline = new Date(agent.lastHeartbeatAt ?? agent.createdAt).getTime();
4309
- const elapsedMs = now.getTime() - baseline;
4310
- if (elapsedMs < policy.intervalSec * 1000)
4311
- continue;
4312
- const run = await enqueueWakeup(agent.id, {
4313
- source: "timer",
4314
- triggerDetail: "system",
4315
- reason: "heartbeat_timer",
4316
- requestedByActorType: "system",
4317
- requestedByActorId: "heartbeat_scheduler",
4318
- contextSnapshot: {
4319
- source: "scheduler",
4320
- reason: "interval_elapsed",
4321
- now: now.toISOString(),
4322
- },
4323
- });
4324
- if (run)
4325
- enqueued += 1;
4326
- else
4327
- skipped += 1;
4328
- }
4329
- return { checked, enqueued, skipped };
4330
- },
4331
- cancelRun: (runId) => cancelRunInternal(runId),
4332
- cancelActiveForAgent: (agentId) => cancelActiveForAgentInternal(agentId),
4333
- cancelBudgetScopeWork,
4334
- getActiveRunForAgent: async (agentId) => {
4335
- const [run] = await db
4336
- .select()
4337
- .from(heartbeatRuns)
4338
- .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "running")))
4339
- .orderBy(desc(heartbeatRuns.startedAt))
4340
- .limit(1);
4341
- return run ?? null;
4342
- },
4343
- };
4344
- }
1
+ export { buildExplicitResumeSessionOverride, buildHeartbeatAdapterInvokePayload, buildHeartbeatRuntimeTraceMetadata, buildIssueRunTraceName, formatRuntimeWorkspaceWarningLog, heartbeatService, parseSessionCompactionPolicy, prioritizeProjectWorkspaceCandidatesForRun, resolveHeartbeatObservabilitySurface, resolveRuntimeSessionParamsForWorkspace, shouldResetTaskSessionForWake, } from "./orchestrator.js";
4345
2
  //# sourceMappingURL=heartbeat.js.map