@rudderhq/server 0.1.0-canary.20 → 0.1.0-canary.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/dist/routes/chats.d.ts.map +1 -1
  2. package/dist/routes/chats.js +15 -0
  3. package/dist/routes/chats.js.map +1 -1
  4. package/dist/routes/dashboard.js +1 -1
  5. package/dist/routes/dashboard.js.map +1 -1
  6. package/dist/services/chat-assistant.d.ts +1 -1
  7. package/dist/services/chat-assistant.d.ts.map +1 -1
  8. package/dist/services/chat-assistant.js +169 -26
  9. package/dist/services/chat-assistant.js.map +1 -1
  10. package/dist/services/chats.d.ts +2 -2
  11. package/dist/services/chats.d.ts.map +1 -1
  12. package/dist/services/chats.js +21 -2
  13. package/dist/services/chats.js.map +1 -1
  14. package/dist/services/runtime-kernel/analytics.d.ts +7 -0
  15. package/dist/services/runtime-kernel/analytics.d.ts.map +1 -0
  16. package/dist/services/runtime-kernel/analytics.js +46 -0
  17. package/dist/services/runtime-kernel/analytics.js.map +1 -0
  18. package/dist/services/runtime-kernel/common.d.ts +4 -0
  19. package/dist/services/runtime-kernel/common.d.ts.map +1 -0
  20. package/dist/services/runtime-kernel/common.js +15 -0
  21. package/dist/services/runtime-kernel/common.js.map +1 -0
  22. package/dist/services/runtime-kernel/execution.d.ts +11 -0
  23. package/dist/services/runtime-kernel/execution.d.ts.map +1 -0
  24. package/dist/services/runtime-kernel/execution.js +28 -0
  25. package/dist/services/runtime-kernel/execution.js.map +1 -0
  26. package/dist/services/runtime-kernel/heartbeat.d.ts +1 -896
  27. package/dist/services/runtime-kernel/heartbeat.d.ts.map +1 -1
  28. package/dist/services/runtime-kernel/heartbeat.js +1 -4387
  29. package/dist/services/runtime-kernel/heartbeat.js.map +1 -1
  30. package/dist/services/runtime-kernel/orchestrator.d.ts +826 -0
  31. package/dist/services/runtime-kernel/orchestrator.d.ts.map +1 -0
  32. package/dist/services/runtime-kernel/orchestrator.js +4044 -0
  33. package/dist/services/runtime-kernel/orchestrator.js.map +1 -0
  34. package/dist/services/runtime-kernel/recovery.d.ts +8 -0
  35. package/dist/services/runtime-kernel/recovery.d.ts.map +1 -0
  36. package/dist/services/runtime-kernel/recovery.js +101 -0
  37. package/dist/services/runtime-kernel/recovery.js.map +1 -0
  38. package/dist/services/runtime-kernel/run-state.d.ts +45 -0
  39. package/dist/services/runtime-kernel/run-state.d.ts.map +1 -0
  40. package/dist/services/runtime-kernel/run-state.js +45 -0
  41. package/dist/services/runtime-kernel/run-state.js.map +1 -0
  42. package/dist/services/runtime-kernel/session-policy.d.ts +42 -0
  43. package/dist/services/runtime-kernel/session-policy.d.ts.map +1 -0
  44. package/dist/services/runtime-kernel/session-policy.js +116 -0
  45. package/dist/services/runtime-kernel/session-policy.js.map +1 -0
  46. package/dist/services/runtime-kernel/types.d.ts +13 -0
  47. package/dist/services/runtime-kernel/types.d.ts.map +1 -0
  48. package/dist/services/runtime-kernel/types.js +2 -0
  49. package/dist/services/runtime-kernel/types.js.map +1 -0
  50. package/dist/services/runtime-kernel/wakeup-queue.d.ts +9 -0
  51. package/dist/services/runtime-kernel/wakeup-queue.d.ts.map +1 -0
  52. package/dist/services/runtime-kernel/wakeup-queue.js +19 -0
  53. package/dist/services/runtime-kernel/wakeup-queue.js.map +1 -0
  54. package/package.json +13 -13
  55. package/ui-dist/assets/{_basePickBy-8ndlG73I.js → _basePickBy-DAU2OOE3.js} +1 -1
  56. package/ui-dist/assets/{_baseUniq-CdTmd3cs.js → _baseUniq-BQLJfIXV.js} +1 -1
  57. package/ui-dist/assets/{arc-Dg3IWr1w.js → arc-B2UTEkTB.js} +1 -1
  58. package/ui-dist/assets/{architectureDiagram-2XIMDMQ5-CL4FzMl4.js → architectureDiagram-2XIMDMQ5-DicHsSOp.js} +1 -1
  59. package/ui-dist/assets/{blockDiagram-WCTKOSBZ-CPAHP8wf.js → blockDiagram-WCTKOSBZ-tbkC8ZCs.js} +1 -1
  60. package/ui-dist/assets/{c4Diagram-IC4MRINW-D9LRotnb.js → c4Diagram-IC4MRINW-pPT_Xw2l.js} +1 -1
  61. package/ui-dist/assets/channel-D8qCsVGC.js +1 -0
  62. package/ui-dist/assets/{chunk-4BX2VUAB-CXYF-alC.js → chunk-4BX2VUAB-vX_EgGXp.js} +1 -1
  63. package/ui-dist/assets/{chunk-55IACEB6-_6gEPxwb.js → chunk-55IACEB6-DUyGlyCg.js} +1 -1
  64. package/ui-dist/assets/{chunk-FMBD7UC4-DBztD7i6.js → chunk-FMBD7UC4-DPCYqV54.js} +1 -1
  65. package/ui-dist/assets/{chunk-JSJVCQXG-D_JITQMs.js → chunk-JSJVCQXG-CRqpXPJG.js} +1 -1
  66. package/ui-dist/assets/{chunk-KX2RTZJC-CDYC8R4t.js → chunk-KX2RTZJC-BWaSxc23.js} +1 -1
  67. package/ui-dist/assets/{chunk-NQ4KR5QH-CuyjnaxX.js → chunk-NQ4KR5QH-BCIl9FTX.js} +1 -1
  68. package/ui-dist/assets/{chunk-QZHKN3VN-BcmldUA2.js → chunk-QZHKN3VN-BgTqyV62.js} +1 -1
  69. package/ui-dist/assets/{chunk-WL4C6EOR-Csc-gUp3.js → chunk-WL4C6EOR-COlN9kxA.js} +1 -1
  70. package/ui-dist/assets/classDiagram-VBA2DB6C-Cm8BRWIa.js +1 -0
  71. package/ui-dist/assets/classDiagram-v2-RAHNMMFH-Cm8BRWIa.js +1 -0
  72. package/ui-dist/assets/clone-B0pIfwDy.js +1 -0
  73. package/ui-dist/assets/{cose-bilkent-S5V4N54A-AQtv05lK.js → cose-bilkent-S5V4N54A-Bbgtq4rD.js} +1 -1
  74. package/ui-dist/assets/{dagre-KLK3FWXG-qsyPc9zE.js → dagre-KLK3FWXG-DgUnjcLS.js} +1 -1
  75. package/ui-dist/assets/{diagram-E7M64L7V-D1y3CfX-.js → diagram-E7M64L7V-DC3DIXbu.js} +1 -1
  76. package/ui-dist/assets/{diagram-IFDJBPK2-CXddZqNx.js → diagram-IFDJBPK2-qv7Ij_rH.js} +1 -1
  77. package/ui-dist/assets/{diagram-P4PSJMXO-bH_AOwhx.js → diagram-P4PSJMXO-Dw2WzhiN.js} +1 -1
  78. package/ui-dist/assets/{erDiagram-INFDFZHY-Dm0NQ5o9.js → erDiagram-INFDFZHY-BV5s3J3D.js} +1 -1
  79. package/ui-dist/assets/{flowDiagram-PKNHOUZH-Bw0gdk4S.js → flowDiagram-PKNHOUZH-kmPgde9o.js} +1 -1
  80. package/ui-dist/assets/{ganttDiagram-A5KZAMGK-DDg3IOSh.js → ganttDiagram-A5KZAMGK-C2hkJ2Th.js} +1 -1
  81. package/ui-dist/assets/{gitGraphDiagram-K3NZZRJ6-COVQ396E.js → gitGraphDiagram-K3NZZRJ6-K5e1NgIt.js} +1 -1
  82. package/ui-dist/assets/{graph-bwl15Qpv.js → graph-Cna3FnWY.js} +1 -1
  83. package/ui-dist/assets/{index-CGKSC15M.js → index-6luhu7W5.js} +1 -1
  84. package/ui-dist/assets/{index-D7r_XRYD.js → index-B1iX9NWL.js} +310 -309
  85. package/ui-dist/assets/{index-B4zUm3gQ.js → index-BdLts2oZ.js} +1 -1
  86. package/ui-dist/assets/{index-9o_SvY62.js → index-CBKnHGYE.js} +1 -1
  87. package/ui-dist/assets/{index-BFOm-l2T.js → index-CDtBE3Ga.js} +1 -1
  88. package/ui-dist/assets/{index-Cej9C4tf.js → index-CFGJHREd.js} +1 -1
  89. package/ui-dist/assets/{index-DxMAOmox.js → index-CTpC-gpc.js} +1 -1
  90. package/ui-dist/assets/{index-BQMKQUTE.js → index-CVAmlnRr.js} +1 -1
  91. package/ui-dist/assets/{index-DTsttLIs.js → index-CWbJ-pcg.js} +1 -1
  92. package/ui-dist/assets/{index-DBLGXUKf.js → index-ChU0nSCq.js} +1 -1
  93. package/ui-dist/assets/{index-C_bGxC5L.js → index-CypGJ__o.js} +1 -1
  94. package/ui-dist/assets/{index-CUPVi6ek.js → index-D-Fe0jjS.js} +1 -1
  95. package/ui-dist/assets/{index-J1ga5P92.js → index-D8aBm25c.js} +1 -1
  96. package/ui-dist/assets/index-D9uKiQrM.css +1 -0
  97. package/ui-dist/assets/{index-WRD2om5d.js → index-DD0ggVK_.js} +1 -1
  98. package/ui-dist/assets/{index-CC_A8kzq.js → index-DJecNMv2.js} +1 -1
  99. package/ui-dist/assets/{index-RhFrB2n9.js → index-DKSfpiAJ.js} +1 -1
  100. package/ui-dist/assets/{index-stSzYVUi.js → index-Dgj2bRIV.js} +1 -1
  101. package/ui-dist/assets/{index-DozeI_ea.js → index-DsydT3bX.js} +1 -1
  102. package/ui-dist/assets/{index-B0lMF-6n.js → index-U7gGuiCe.js} +1 -1
  103. package/ui-dist/assets/{index--J4W4di0.js → index-aL5hFrtS.js} +1 -1
  104. package/ui-dist/assets/{index-dewwxxca.js → index-o_iEnzyw.js} +1 -1
  105. package/ui-dist/assets/{index-bWQpv8XB.js → index-qUFX4kXb.js} +1 -1
  106. package/ui-dist/assets/{index-DSuhUSKb.js → index-sV3mB3fg.js} +1 -1
  107. package/ui-dist/assets/{infoDiagram-LFFYTUFH-DjdShR-K.js → infoDiagram-LFFYTUFH-BiQ6_Z3F.js} +1 -1
  108. package/ui-dist/assets/{ishikawaDiagram-PHBUUO56-DXZf7GwF.js → ishikawaDiagram-PHBUUO56-C_WqRu1B.js} +1 -1
  109. package/ui-dist/assets/{journeyDiagram-4ABVD52K-70bcudq0.js → journeyDiagram-4ABVD52K-BIpPercT.js} +1 -1
  110. package/ui-dist/assets/{kanban-definition-K7BYSVSG-BCdouCZ4.js → kanban-definition-K7BYSVSG-DaFD_JyB.js} +1 -1
  111. package/ui-dist/assets/{layout-Mhq2HNeN.js → layout-Dj_oike0.js} +1 -1
  112. package/ui-dist/assets/{linear-6jA82SeB.js → linear-tnVt1ugU.js} +1 -1
  113. package/ui-dist/assets/{mermaid.core-Bytv0x0U.js → mermaid.core-DflBh7Hv.js} +4 -4
  114. package/ui-dist/assets/{mindmap-definition-YRQLILUH-DK1H9m3i.js → mindmap-definition-YRQLILUH-C3H-gFSh.js} +1 -1
  115. package/ui-dist/assets/{pieDiagram-SKSYHLDU-DOIMV3g5.js → pieDiagram-SKSYHLDU-DH53odzN.js} +1 -1
  116. package/ui-dist/assets/{quadrantDiagram-337W2JSQ-DTfxtKqh.js → quadrantDiagram-337W2JSQ-DCn4Dric.js} +1 -1
  117. package/ui-dist/assets/{requirementDiagram-Z7DCOOCP-DjXfzNQ_.js → requirementDiagram-Z7DCOOCP-KypOfBcY.js} +1 -1
  118. package/ui-dist/assets/{sankeyDiagram-WA2Y5GQK-DCIL8YBw.js → sankeyDiagram-WA2Y5GQK-DEmi0ryr.js} +1 -1
  119. package/ui-dist/assets/{sequenceDiagram-2WXFIKYE-DB7gJ2NE.js → sequenceDiagram-2WXFIKYE-sW6ojsVG.js} +1 -1
  120. package/ui-dist/assets/{stateDiagram-RAJIS63D-ByB0ZTgH.js → stateDiagram-RAJIS63D-CmrX9BEC.js} +1 -1
  121. package/ui-dist/assets/stateDiagram-v2-FVOUBMTO-DJuCnZbk.js +1 -0
  122. package/ui-dist/assets/{timeline-definition-YZTLITO2-BgCHIIAF.js → timeline-definition-YZTLITO2-CzJ5Y8YU.js} +1 -1
  123. package/ui-dist/assets/{treemap-KZPCXAKY-CN3Dkmq8.js → treemap-KZPCXAKY-BwUNliTA.js} +1 -1
  124. package/ui-dist/assets/{vennDiagram-LZ73GAT5-DoESURJl.js → vennDiagram-LZ73GAT5-5A50aIn1.js} +1 -1
  125. package/ui-dist/assets/{xychartDiagram-JWTSCODW-CRDlcdUc.js → xychartDiagram-JWTSCODW-D9qTFskE.js} +1 -1
  126. package/ui-dist/index.html +2 -2
  127. package/ui-dist/assets/channel-W_reUgPl.js +0 -1
  128. package/ui-dist/assets/classDiagram-VBA2DB6C-Z8KHPV3I.js +0 -1
  129. package/ui-dist/assets/classDiagram-v2-RAHNMMFH-Z8KHPV3I.js +0 -1
  130. package/ui-dist/assets/clone-C8TVMdfS.js +0 -1
  131. package/ui-dist/assets/index-C8zmIhnp.css +0 -1
  132. package/ui-dist/assets/stateDiagram-v2-FVOUBMTO-DHKGN5HJ.js +0 -1
@@ -1,4388 +1,2 @@
1
- import path from "node:path";
2
- import { and, asc, desc, eq, gt, gte, inArray, lte, sql } from "drizzle-orm";
3
- import { AGENT_RUN_CONCURRENCY_DEFAULT, AGENT_RUN_CONCURRENCY_MAX, AGENT_RUN_CONCURRENCY_MIN, } from "@rudderhq/shared";
4
- import { agents, agentRuntimeState, agentTaskSessions, agentWakeupRequests, activityLog, heartbeatRunEvents, heartbeatRuns, issueComments, issues, organizations, projects, } from "@rudderhq/db";
5
- import { conflict, notFound } from "../../errors.js";
6
- import { createExecutionScores, observeExecutionEvent, updateExecutionObservation, updateExecutionTraceIO, updateExecutionTraceName, updateExecutionTraceSession, withExecutionObservation, } from "../../langfuse.js";
7
- import { emitExecutionTranscriptTree } from "../../langfuse-transcript.js";
8
- import { logger } from "../../middleware/logger.js";
9
- import { publishLiveEvent } from "../live-events.js";
10
- import { getRunLogStore } from "../run-log-store.js";
11
- import { findServerAdapter, getServerAdapter, runningProcesses } from "../../agent-runtimes/index.js";
12
- import { createLocalAgentJwt } from "../../agent-auth-jwt.js";
13
- import { parseObject, asBoolean, asNumber, appendWithCap, MAX_EXCERPT_BYTES } from "../../agent-runtimes/utils.js";
14
- import { costService } from "../costs.js";
15
- import { budgetService } from "../budgets.js";
16
- import { agentRunContextService, } from "../agent-run-context.js";
17
- import { resolveDefaultAgentWorkspaceDir, } from "../../home-paths.js";
18
- import { summarizeHeartbeatRunResultJson } from "../heartbeat-run-summary.js";
19
- import { summarizeRuntimeSkillsForTrace } from "../runtime-trace-metadata.js";
20
- import { buildWorkspaceReadyComment, cleanupExecutionWorkspaceArtifacts, ensureRuntimeServicesForRun, persistAdapterManagedRuntimeServices, realizeExecutionWorkspace, releaseRuntimeServicesForRun, } from "../workspace-runtime.js";
21
- import { issueService } from "../issues.js";
22
- import { executionWorkspaceService } from "../execution-workspaces.js";
23
- import { buildObservedRunLangfuseScores } from "../run-intelligence.js";
24
- import { workspaceOperationService } from "../workspace-operations.js";
25
- import { buildExecutionWorkspaceAdapterConfig, issueExecutionWorkspaceModeForPersistedWorkspace, parseIssueExecutionWorkspaceSettings, parseProjectExecutionWorkspacePolicy, resolveExecutionWorkspaceMode, } from "../execution-workspace-policy.js";
26
- import { instanceSettingsService } from "../instance-settings.js";
27
- import { logActivity } from "../activity-log.js";
28
- import { redactCurrentUserText, redactCurrentUserValue } from "../../log-redaction.js";
29
- import { hasSessionCompactionThresholds, resolveSessionCompactionPolicy, } from "@rudderhq/agent-runtime-utils";
30
- import { buildCreateAgentBenchmarkTags, coerceCreateAgentBenchmarkMetadata, extractCreateAgentBenchmarkMetadata, } from "@rudderhq/run-intelligence-core";
31
- import { executeAdapterWithModelFallbacks } from "./model-fallback.js";
32
- export { prioritizeProjectWorkspaceCandidatesForRun } from "../agent-run-context.js";
33
- const MAX_LIVE_LOG_CHUNK_BYTES = 8 * 1024;
34
- const HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT = AGENT_RUN_CONCURRENCY_DEFAULT;
35
- const HEARTBEAT_MAX_CONCURRENT_RUNS_MIN = AGENT_RUN_CONCURRENCY_MIN;
36
- const HEARTBEAT_MAX_CONCURRENT_RUNS_MAX = AGENT_RUN_CONCURRENCY_MAX;
37
- const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext";
38
- const DETACHED_PROCESS_ERROR_CODE = "process_detached";
39
- const ORPHANED_PROCESS_TERMINATION_GRACE_MS = 2_000;
40
- const ORPHANED_PROCESS_KILL_WAIT_MS = 500;
41
- const ORPHANED_PROCESS_POLL_INTERVAL_MS = 100;
42
- const startLocksByAgent = new Map();
43
- const MAX_RECOVERY_CHAIN_DEPTH = 8;
44
- const ISSUE_PASSIVE_FOLLOWUP_REASON = "issue_passive_followup";
45
- const ISSUE_PASSIVE_FOLLOWUP_WAKE_SOURCE = "passive_issue_followup";
46
- const ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON = "missing_closure";
47
- const ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS = 2;
48
- const ISSUE_PASSIVE_FOLLOWUP_COOLDOWN_MS_BY_ATTEMPT = new Map([
49
- [1, 2 * 60 * 1000],
50
- [2, 5 * 60 * 1000],
51
- ]);
52
- const ISSUE_PASSIVE_FOLLOWUP_TIMER_CONTINUITY_MAX_WINDOW_MS = 15 * 60 * 1000;
53
- const SESSIONED_LOCAL_ADAPTERS = new Set([
54
- "claude_local",
55
- "codex_local",
56
- "cursor",
57
- "gemini_local",
58
- "opencode_local",
59
- "pi_local",
60
- ]);
61
- const heartbeatRunListColumns = {
62
- id: heartbeatRuns.id,
63
- orgId: heartbeatRuns.orgId,
64
- agentId: heartbeatRuns.agentId,
65
- invocationSource: heartbeatRuns.invocationSource,
66
- triggerDetail: heartbeatRuns.triggerDetail,
67
- status: heartbeatRuns.status,
68
- startedAt: heartbeatRuns.startedAt,
69
- finishedAt: heartbeatRuns.finishedAt,
70
- error: heartbeatRuns.error,
71
- wakeupRequestId: heartbeatRuns.wakeupRequestId,
72
- exitCode: heartbeatRuns.exitCode,
73
- signal: heartbeatRuns.signal,
74
- usageJson: heartbeatRuns.usageJson,
75
- resultJson: heartbeatRuns.resultJson,
76
- sessionIdBefore: heartbeatRuns.sessionIdBefore,
77
- sessionIdAfter: heartbeatRuns.sessionIdAfter,
78
- logStore: heartbeatRuns.logStore,
79
- logRef: heartbeatRuns.logRef,
80
- logBytes: heartbeatRuns.logBytes,
81
- logSha256: heartbeatRuns.logSha256,
82
- logCompressed: heartbeatRuns.logCompressed,
83
- stdoutExcerpt: sql `NULL`.as("stdoutExcerpt"),
84
- stderrExcerpt: sql `NULL`.as("stderrExcerpt"),
85
- errorCode: heartbeatRuns.errorCode,
86
- externalRunId: heartbeatRuns.externalRunId,
87
- processPid: heartbeatRuns.processPid,
88
- processStartedAt: heartbeatRuns.processStartedAt,
89
- retryOfRunId: heartbeatRuns.retryOfRunId,
90
- processLossRetryCount: heartbeatRuns.processLossRetryCount,
91
- contextSnapshot: heartbeatRuns.contextSnapshot,
92
- createdAt: heartbeatRuns.createdAt,
93
- updatedAt: heartbeatRuns.updatedAt,
94
- };
95
- function appendExcerpt(prev, chunk) {
96
- return appendWithCap(prev, chunk, MAX_EXCERPT_BYTES);
97
- }
98
- function appendTranscriptEntriesFromChunk(input) {
99
- const combined = `${input.buffer}${input.chunk}`;
100
- const lines = combined.split(/\r?\n/);
101
- const trailing = lines.pop() ?? "";
102
- const completeLines = input.finalize && trailing ? [...lines, trailing] : lines;
103
- for (const line of completeLines) {
104
- if (!line.trim())
105
- continue;
106
- const ts = new Date().toISOString();
107
- const parsed = input.parser ? input.parser(line, ts) : [];
108
- if (parsed.length > 0) {
109
- input.transcript.push(...parsed);
110
- continue;
111
- }
112
- input.transcript.push({
113
- kind: input.kind,
114
- ts,
115
- text: line,
116
- });
117
- }
118
- return input.finalize ? "" : trailing;
119
- }
120
- function normalizeMaxConcurrentRuns(value) {
121
- const parsed = Math.floor(asNumber(value, HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT));
122
- if (!Number.isFinite(parsed))
123
- return HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT;
124
- return Math.max(HEARTBEAT_MAX_CONCURRENT_RUNS_MIN, Math.min(HEARTBEAT_MAX_CONCURRENT_RUNS_MAX, parsed));
125
- }
126
- async function withAgentStartLock(agentId, fn) {
127
- const previous = startLocksByAgent.get(agentId) ?? Promise.resolve();
128
- const run = previous.then(fn);
129
- const marker = run.then(() => undefined, () => undefined);
130
- startLocksByAgent.set(agentId, marker);
131
- try {
132
- return await run;
133
- }
134
- finally {
135
- if (startLocksByAgent.get(agentId) === marker) {
136
- startLocksByAgent.delete(agentId);
137
- }
138
- }
139
- }
140
- function readNonEmptyString(value) {
141
- return typeof value === "string" && value.trim().length > 0 ? value : null;
142
- }
143
- export function resolveHeartbeatObservabilitySurface(contextSnapshot) {
144
- return readNonEmptyString(contextSnapshot?.issueId) ? "issue_run" : "heartbeat_run";
145
- }
146
- function buildHeartbeatObservationName(run, agentName) {
147
- const contextSnapshot = parseObject(run.contextSnapshot);
148
- const issueId = readNonEmptyString(contextSnapshot.issueId);
149
- return issueId ? `issue_run:${issueId}` : `heartbeat:${agentName}`;
150
- }
151
- function compactTraceText(value, maxLength = 120) {
152
- const next = value?.replace(/\s+/g, " ").trim();
153
- if (!next)
154
- return null;
155
- return next.length > maxLength ? `${next.slice(0, maxLength - 1)}…` : next;
156
- }
157
- export function buildIssueRunTraceName(input) {
158
- const issueTitle = compactTraceText(input.issueTitle);
159
- return issueTitle ? `issue_run:${issueTitle} [${input.issueId}]` : `issue_run:[${input.issueId}]`;
160
- }
161
- export function buildHeartbeatRuntimeTraceMetadata(input) {
162
- const instructionsFilePath = readNonEmptyString(input.runtimeConfig.instructionsFilePath);
163
- return {
164
- instructionsConfigured: Boolean(instructionsFilePath),
165
- instructionsFilePath,
166
- ...summarizeRuntimeSkillsForTrace(input.runtimeSkills),
167
- ...(input.adapterMeta
168
- ? {
169
- runtimeAgentType: input.adapterMeta.agentRuntimeType,
170
- runtimeCommand: input.adapterMeta.command,
171
- runtimeCwd: input.adapterMeta.cwd ?? null,
172
- runtimeCommandNotes: input.adapterMeta.commandNotes ?? [],
173
- runtimePromptMetrics: input.adapterMeta.promptMetrics ?? null,
174
- }
175
- : {}),
176
- };
177
- }
178
- export function buildHeartbeatAdapterInvokePayload(input) {
179
- return {
180
- ...input.meta,
181
- ...summarizeRuntimeSkillsForTrace(input.runtimeSkills),
182
- };
183
- }
184
- function buildRecentDateKeys(windowDays, now) {
185
- return Array.from({ length: windowDays }, (_, index) => {
186
- const next = new Date(now);
187
- next.setUTCDate(next.getUTCDate() - (windowDays - 1 - index));
188
- return next.toISOString().slice(0, 10);
189
- });
190
- }
191
- function buildDateKeysBetween(startDate, endDate) {
192
- const start = new Date(`${startDate}T00:00:00.000Z`);
193
- const end = new Date(`${endDate}T00:00:00.000Z`);
194
- if (Number.isNaN(start.getTime()) || Number.isNaN(end.getTime()) || start.getTime() > end.getTime()) {
195
- return [];
196
- }
197
- const days = [];
198
- const cursor = new Date(start);
199
- while (cursor.getTime() <= end.getTime()) {
200
- days.push(cursor.toISOString().slice(0, 10));
201
- cursor.setUTCDate(cursor.getUTCDate() + 1);
202
- }
203
- return days;
204
- }
205
- function fallbackSkillLabel(key) {
206
- const trimmed = key.trim();
207
- if (!trimmed)
208
- return "unknown";
209
- const slashSegments = trimmed.split("/").filter(Boolean);
210
- const lastSlashSegment = slashSegments.at(-1);
211
- if (lastSlashSegment)
212
- return lastSlashSegment;
213
- const colonSegments = trimmed.split(":").filter(Boolean);
214
- return colonSegments.at(-1) ?? trimmed;
215
- }
216
- function normalizeLoadedSkill(value) {
217
- const skill = parseObject(value);
218
- const rawKey = readNonEmptyString(skill.key);
219
- const rawRuntimeName = readNonEmptyString(skill.runtimeName);
220
- const rawName = readNonEmptyString(skill.name);
221
- const key = rawKey ?? rawRuntimeName ?? rawName;
222
- if (!key)
223
- return null;
224
- const label = rawRuntimeName ?? rawName ?? fallbackSkillLabel(key);
225
- return { key, label };
226
- }
227
- function normalizeLedgerBillingType(value) {
228
- const raw = readNonEmptyString(value);
229
- switch (raw) {
230
- case "api":
231
- case "metered_api":
232
- return "metered_api";
233
- case "subscription":
234
- case "subscription_included":
235
- return "subscription_included";
236
- case "subscription_overage":
237
- return "subscription_overage";
238
- case "credits":
239
- return "credits";
240
- case "fixed":
241
- return "fixed";
242
- default:
243
- return "unknown";
244
- }
245
- }
246
- function resolveLedgerBiller(result) {
247
- return readNonEmptyString(result.biller) ?? readNonEmptyString(result.provider) ?? "unknown";
248
- }
249
- function normalizeBilledCostCents(costUsd, billingType) {
250
- if (billingType === "subscription_included")
251
- return 0;
252
- if (typeof costUsd !== "number" || !Number.isFinite(costUsd))
253
- return 0;
254
- return Math.max(0, Math.round(costUsd * 100));
255
- }
256
- async function resolveLedgerScopeForRun(db, orgId, run) {
257
- const context = parseObject(run.contextSnapshot);
258
- const contextIssueId = readNonEmptyString(context.issueId);
259
- const contextProjectId = readNonEmptyString(context.projectId);
260
- if (!contextIssueId) {
261
- return {
262
- issueId: null,
263
- projectId: contextProjectId,
264
- };
265
- }
266
- const issue = await db
267
- .select({
268
- id: issues.id,
269
- projectId: issues.projectId,
270
- })
271
- .from(issues)
272
- .where(and(eq(issues.id, contextIssueId), eq(issues.orgId, orgId)))
273
- .then((rows) => rows[0] ?? null);
274
- return {
275
- issueId: issue?.id ?? null,
276
- projectId: issue?.projectId ?? contextProjectId,
277
- };
278
- }
279
- export function buildExplicitResumeSessionOverride(input) {
280
- const desiredDisplayId = truncateDisplayId(input.resumeRunSessionIdAfter ?? input.resumeRunSessionIdBefore);
281
- const taskSessionParams = normalizeSessionParams(input.sessionCodec.deserialize(input.taskSession?.sessionParamsJson ?? null));
282
- const taskSessionDisplayId = truncateDisplayId(input.taskSession?.sessionDisplayId ??
283
- (input.sessionCodec.getDisplayId ? input.sessionCodec.getDisplayId(taskSessionParams) : null) ??
284
- readNonEmptyString(taskSessionParams?.sessionId));
285
- const canReuseTaskSessionParams = input.taskSession != null &&
286
- (input.taskSession.lastRunId === input.resumeFromRunId ||
287
- (!!desiredDisplayId && taskSessionDisplayId === desiredDisplayId));
288
- const sessionParams = canReuseTaskSessionParams
289
- ? taskSessionParams
290
- : desiredDisplayId
291
- ? { sessionId: desiredDisplayId }
292
- : null;
293
- const sessionDisplayId = desiredDisplayId ?? (canReuseTaskSessionParams ? taskSessionDisplayId : null);
294
- if (!sessionDisplayId && !sessionParams)
295
- return null;
296
- return {
297
- sessionDisplayId,
298
- sessionParams,
299
- };
300
- }
301
- function normalizeUsageTotals(usage) {
302
- if (!usage)
303
- return null;
304
- return {
305
- inputTokens: Math.max(0, Math.floor(asNumber(usage.inputTokens, 0))),
306
- cachedInputTokens: Math.max(0, Math.floor(asNumber(usage.cachedInputTokens, 0))),
307
- outputTokens: Math.max(0, Math.floor(asNumber(usage.outputTokens, 0))),
308
- };
309
- }
310
- function readRawUsageTotals(usageJson) {
311
- const parsed = parseObject(usageJson);
312
- if (Object.keys(parsed).length === 0)
313
- return null;
314
- const inputTokens = Math.max(0, Math.floor(asNumber(parsed.rawInputTokens, asNumber(parsed.inputTokens, 0))));
315
- const cachedInputTokens = Math.max(0, Math.floor(asNumber(parsed.rawCachedInputTokens, asNumber(parsed.cachedInputTokens, 0))));
316
- const outputTokens = Math.max(0, Math.floor(asNumber(parsed.rawOutputTokens, asNumber(parsed.outputTokens, 0))));
317
- if (inputTokens <= 0 && cachedInputTokens <= 0 && outputTokens <= 0) {
318
- return null;
319
- }
320
- return {
321
- inputTokens,
322
- cachedInputTokens,
323
- outputTokens,
324
- };
325
- }
326
- function deriveNormalizedUsageDelta(current, previous) {
327
- if (!current)
328
- return null;
329
- if (!previous)
330
- return { ...current };
331
- const inputTokens = current.inputTokens >= previous.inputTokens
332
- ? current.inputTokens - previous.inputTokens
333
- : current.inputTokens;
334
- const cachedInputTokens = current.cachedInputTokens >= previous.cachedInputTokens
335
- ? current.cachedInputTokens - previous.cachedInputTokens
336
- : current.cachedInputTokens;
337
- const outputTokens = current.outputTokens >= previous.outputTokens
338
- ? current.outputTokens - previous.outputTokens
339
- : current.outputTokens;
340
- return {
341
- inputTokens: Math.max(0, inputTokens),
342
- cachedInputTokens: Math.max(0, cachedInputTokens),
343
- outputTokens: Math.max(0, outputTokens),
344
- };
345
- }
346
- function formatCount(value) {
347
- if (typeof value !== "number" || !Number.isFinite(value))
348
- return "0";
349
- return value.toLocaleString("en-US");
350
- }
351
- export function parseSessionCompactionPolicy(agent) {
352
- return resolveSessionCompactionPolicy(agent.agentRuntimeType, agent.runtimeConfig).policy;
353
- }
354
- export function resolveRuntimeSessionParamsForWorkspace(input) {
355
- const { orgId, agent, previousSessionParams, resolvedWorkspace } = input;
356
- const previousSessionId = readNonEmptyString(previousSessionParams?.sessionId);
357
- if (!previousSessionId) {
358
- return {
359
- sessionParams: previousSessionParams,
360
- warning: null,
361
- };
362
- }
363
- const canonicalAgentCwd = readNonEmptyString(resolvedWorkspace.cwd) ?? resolveDefaultAgentWorkspaceDir(orgId, agent);
364
- if (!canonicalAgentCwd) {
365
- return {
366
- sessionParams: previousSessionParams,
367
- warning: null,
368
- };
369
- }
370
- const previousCwd = readNonEmptyString(previousSessionParams?.cwd);
371
- if (previousCwd && path.resolve(previousCwd) === path.resolve(canonicalAgentCwd)) {
372
- return {
373
- sessionParams: previousSessionParams,
374
- warning: null,
375
- };
376
- }
377
- const previousWorkspaceId = readNonEmptyString(previousSessionParams?.workspaceId);
378
- const migratedSessionParams = {
379
- ...(previousSessionParams ?? {}),
380
- cwd: canonicalAgentCwd,
381
- };
382
- if (!previousWorkspaceId ||
383
- !resolvedWorkspace.workspaceId ||
384
- previousWorkspaceId === resolvedWorkspace.workspaceId) {
385
- if (resolvedWorkspace.workspaceId)
386
- migratedSessionParams.workspaceId = resolvedWorkspace.workspaceId;
387
- if (resolvedWorkspace.repoUrl)
388
- migratedSessionParams.repoUrl = resolvedWorkspace.repoUrl;
389
- if (resolvedWorkspace.repoRef)
390
- migratedSessionParams.repoRef = resolvedWorkspace.repoRef;
391
- }
392
- return {
393
- sessionParams: migratedSessionParams,
394
- warning: previousCwd
395
- ? `Agent workspace "${canonicalAgentCwd}" is now the canonical run workspace. ` +
396
- `Attempting to resume session "${previousSessionId}" that was previously saved in "${previousCwd}".`
397
- : `Agent workspace "${canonicalAgentCwd}" is now the canonical run workspace. ` +
398
- `Attempting to resume session "${previousSessionId}" with the canonical agent workspace attached.`,
399
- };
400
- }
401
- function parseIssueAssigneeAgentRuntimeOverrides(raw) {
402
- const parsed = parseObject(raw);
403
- const parsedAdapterConfig = parseObject(parsed.agentRuntimeConfig);
404
- const agentRuntimeConfig = Object.keys(parsedAdapterConfig).length > 0 ? parsedAdapterConfig : null;
405
- const useProjectWorkspace = typeof parsed.useProjectWorkspace === "boolean"
406
- ? parsed.useProjectWorkspace
407
- : null;
408
- if (!agentRuntimeConfig && useProjectWorkspace === null)
409
- return null;
410
- return {
411
- agentRuntimeConfig,
412
- useProjectWorkspace,
413
- };
414
- }
415
- function deriveTaskKey(contextSnapshot, payload) {
416
- return (readNonEmptyString(contextSnapshot?.taskKey) ??
417
- readNonEmptyString(contextSnapshot?.taskId) ??
418
- readNonEmptyString(contextSnapshot?.issueId) ??
419
- readNonEmptyString(payload?.taskKey) ??
420
- readNonEmptyString(payload?.taskId) ??
421
- readNonEmptyString(payload?.issueId) ??
422
- null);
423
- }
424
- export function shouldResetTaskSessionForWake(contextSnapshot) {
425
- if (contextSnapshot?.forceFreshSession === true)
426
- return true;
427
- const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason);
428
- if (wakeReason === "issue_assigned")
429
- return true;
430
- return false;
431
- }
432
- export function formatRuntimeWorkspaceWarningLog(warning) {
433
- return {
434
- stream: "stdout",
435
- chunk: `[rudder] ${warning}\n`,
436
- };
437
- }
438
- function describeSessionResetReason(contextSnapshot) {
439
- if (contextSnapshot?.forceFreshSession === true)
440
- return "forceFreshSession was requested";
441
- const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason);
442
- if (wakeReason === "issue_assigned")
443
- return "wake reason is issue_assigned";
444
- return null;
445
- }
446
- function deriveCommentId(contextSnapshot, payload) {
447
- return (readNonEmptyString(contextSnapshot?.wakeCommentId) ??
448
- readNonEmptyString(contextSnapshot?.commentId) ??
449
- readNonEmptyString(payload?.commentId) ??
450
- null);
451
- }
452
- function enrichWakeContextSnapshot(input) {
453
- const { contextSnapshot, reason, source, triggerDetail, payload } = input;
454
- const issueIdFromPayload = readNonEmptyString(payload?.["issueId"]);
455
- const commentIdFromPayload = readNonEmptyString(payload?.["commentId"]);
456
- const taskKey = deriveTaskKey(contextSnapshot, payload);
457
- const wakeCommentId = deriveCommentId(contextSnapshot, payload);
458
- if (!readNonEmptyString(contextSnapshot["wakeReason"]) && reason) {
459
- contextSnapshot.wakeReason = reason;
460
- }
461
- if (!readNonEmptyString(contextSnapshot["issueId"]) && issueIdFromPayload) {
462
- contextSnapshot.issueId = issueIdFromPayload;
463
- }
464
- if (!readNonEmptyString(contextSnapshot["taskId"]) && issueIdFromPayload) {
465
- contextSnapshot.taskId = issueIdFromPayload;
466
- }
467
- if (!readNonEmptyString(contextSnapshot["taskKey"]) && taskKey) {
468
- contextSnapshot.taskKey = taskKey;
469
- }
470
- if (!readNonEmptyString(contextSnapshot["commentId"]) && commentIdFromPayload) {
471
- contextSnapshot.commentId = commentIdFromPayload;
472
- }
473
- if (!readNonEmptyString(contextSnapshot["wakeCommentId"]) && wakeCommentId) {
474
- contextSnapshot.wakeCommentId = wakeCommentId;
475
- }
476
- if (!readNonEmptyString(contextSnapshot["wakeSource"]) && source) {
477
- contextSnapshot.wakeSource = source;
478
- }
479
- if (!readNonEmptyString(contextSnapshot["wakeTriggerDetail"]) && triggerDetail) {
480
- contextSnapshot.wakeTriggerDetail = triggerDetail;
481
- }
482
- return {
483
- contextSnapshot,
484
- issueIdFromPayload,
485
- commentIdFromPayload,
486
- taskKey,
487
- wakeCommentId,
488
- };
489
- }
490
- function mergeCoalescedContextSnapshot(existingRaw, incoming) {
491
- const existing = parseObject(existingRaw);
492
- const merged = {
493
- ...existing,
494
- ...incoming,
495
- };
496
- const commentId = deriveCommentId(incoming, null);
497
- if (commentId) {
498
- merged.commentId = commentId;
499
- merged.wakeCommentId = commentId;
500
- }
501
- return merged;
502
- }
503
- function buildDeferredWakePayload(payload, contextSnapshot, issueId) {
504
- const deferredPayload = { ...(payload ?? {}) };
505
- if (issueId && !readNonEmptyString(deferredPayload.issueId)) {
506
- deferredPayload.issueId = issueId;
507
- }
508
- deferredPayload[DEFERRED_WAKE_CONTEXT_KEY] = contextSnapshot;
509
- return deferredPayload;
510
- }
511
- function readDeferredWakeContext(payloadRaw) {
512
- const payload = parseObject(payloadRaw);
513
- return parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]);
514
- }
515
- function readDeferredWakePayload(payloadRaw) {
516
- const payload = parseObject(payloadRaw);
517
- delete payload[DEFERRED_WAKE_CONTEXT_KEY];
518
- return payload;
519
- }
520
- function deriveDeferredWakeTaskKey(payloadRaw) {
521
- const payload = readDeferredWakePayload(payloadRaw);
522
- const contextSnapshot = readDeferredWakeContext(payloadRaw);
523
- return deriveTaskKey(contextSnapshot, payload);
524
- }
525
- async function hydrateWakeContextSnapshot(db, orgId, contextSnapshot) {
526
- const issueId = readNonEmptyString(contextSnapshot.issueId);
527
- const commentId = deriveCommentId(contextSnapshot, null);
528
- const issueContext = parseObject(contextSnapshot.issue);
529
- const commentContext = parseObject(contextSnapshot.comment);
530
- const needsIssueContext = !!issueId &&
531
- (!readNonEmptyString(issueContext.id) ||
532
- !readNonEmptyString(issueContext.title) ||
533
- !readNonEmptyString(issueContext.status) ||
534
- !("priority" in issueContext) ||
535
- !("description" in issueContext));
536
- const needsProjectId = !!issueId && !readNonEmptyString(contextSnapshot.projectId);
537
- const needsCommentContext = !!commentId &&
538
- (!readNonEmptyString(commentContext.id) ||
539
- !readNonEmptyString(commentContext.body));
540
- if (!needsIssueContext && !needsProjectId && !needsCommentContext)
541
- return;
542
- if (issueId && (needsIssueContext || needsProjectId)) {
543
- const issueRow = await db
544
- .select({
545
- id: issues.id,
546
- title: issues.title,
547
- description: issues.description,
548
- status: issues.status,
549
- priority: issues.priority,
550
- projectId: issues.projectId,
551
- })
552
- .from(issues)
553
- .where(and(eq(issues.id, issueId), eq(issues.orgId, orgId)))
554
- .then((rows) => rows[0] ?? null);
555
- if (issueRow) {
556
- contextSnapshot.issue = {
557
- ...issueContext,
558
- id: readNonEmptyString(issueContext.id) ?? issueRow.id,
559
- title: readNonEmptyString(issueContext.title) ?? issueRow.title,
560
- description: "description" in issueContext ? issueContext.description : issueRow.description,
561
- status: readNonEmptyString(issueContext.status) ?? issueRow.status,
562
- priority: "priority" in issueContext ? issueContext.priority : issueRow.priority,
563
- };
564
- if (!readNonEmptyString(contextSnapshot.projectId) && issueRow.projectId) {
565
- contextSnapshot.projectId = issueRow.projectId;
566
- }
567
- }
568
- }
569
- if (commentId && needsCommentContext) {
570
- const commentConditions = [eq(issueComments.id, commentId), eq(issueComments.orgId, orgId)];
571
- if (issueId) {
572
- commentConditions.push(eq(issueComments.issueId, issueId));
573
- }
574
- const commentRow = await db
575
- .select({
576
- id: issueComments.id,
577
- body: issueComments.body,
578
- authorAgentId: issueComments.authorAgentId,
579
- authorUserId: issueComments.authorUserId,
580
- })
581
- .from(issueComments)
582
- .where(and(...commentConditions))
583
- .then((rows) => rows[0] ?? null);
584
- if (commentRow) {
585
- contextSnapshot.comment = {
586
- ...commentContext,
587
- id: readNonEmptyString(commentContext.id) ?? commentRow.id,
588
- body: readNonEmptyString(commentContext.body) ?? commentRow.body,
589
- authorAgentId: "authorAgentId" in commentContext ? commentContext.authorAgentId : commentRow.authorAgentId,
590
- authorUserId: "authorUserId" in commentContext ? commentContext.authorUserId : commentRow.authorUserId,
591
- };
592
- }
593
- }
594
- }
595
- function firstNonEmptyLine(value) {
596
- if (typeof value !== "string")
597
- return null;
598
- const line = value
599
- .split("\n")
600
- .map((chunk) => chunk.trim())
601
- .find(Boolean);
602
- return line ?? null;
603
- }
604
- function deriveRecoveryFailureKind(run) {
605
- return (readNonEmptyString(run.errorCode) ??
606
- (run.status === "timed_out" ? "timed_out" : null) ??
607
- run.status);
608
- }
609
- function deriveRecoveryFailureSummary(run) {
610
- return (firstNonEmptyLine(run.error) ??
611
- firstNonEmptyLine(run.stderrExcerpt) ??
612
- firstNonEmptyLine(run.stdoutExcerpt) ??
613
- (run.status === "timed_out" ? "The run timed out before it completed." : null) ??
614
- "The previous run failed before it completed.");
615
- }
616
- function mergeMissingRecoveryContextFields(target, source) {
617
- const keysToBackfill = [
618
- "issueId",
619
- "taskId",
620
- "taskKey",
621
- "projectId",
622
- "projectWorkspaceId",
623
- "commentId",
624
- "wakeCommentId",
625
- "issue",
626
- "comment",
627
- "source",
628
- "wakeSource",
629
- "wakeTriggerDetail",
630
- ];
631
- for (const key of keysToBackfill) {
632
- if (!(key in target) || target[key] === null || target[key] === undefined || target[key] === "") {
633
- const value = source[key];
634
- if (value !== null && value !== undefined && value !== "") {
635
- target[key] = value;
636
- }
637
- }
638
- }
639
- }
640
- async function hydrateRecoveryBaseContextSnapshot(run, getRunById) {
641
- const mergedContext = { ...parseObject(run.contextSnapshot) };
642
- let ancestorRunId = readNonEmptyString(run.retryOfRunId);
643
- let depth = 0;
644
- while (ancestorRunId && depth < MAX_RECOVERY_CHAIN_DEPTH) {
645
- const ancestorRun = await getRunById(ancestorRunId);
646
- if (!ancestorRun)
647
- break;
648
- mergeMissingRecoveryContextFields(mergedContext, parseObject(ancestorRun.contextSnapshot));
649
- ancestorRunId = readNonEmptyString(ancestorRun.retryOfRunId);
650
- depth += 1;
651
- }
652
- return mergedContext;
653
- }
654
- function buildRecoveryContextSnapshot(input) {
655
- const { baseContextSnapshot, run, recoveryTrigger, wakeReason, wakeSource, triggerDetail } = input;
656
- const failureKind = deriveRecoveryFailureKind(run);
657
- const failureSummary = deriveRecoveryFailureSummary(run);
658
- const recovery = {
659
- originalRunId: run.id,
660
- failureKind,
661
- failureSummary,
662
- recoveryTrigger,
663
- recoveryMode: "continue_preferred",
664
- };
665
- return {
666
- ...baseContextSnapshot,
667
- wakeReason,
668
- wakeSource,
669
- wakeTriggerDetail: triggerDetail,
670
- retryOfRunId: run.id,
671
- retryReason: failureKind,
672
- recovery,
673
- };
674
- }
675
- function normalizePassiveFollowupContext(raw) {
676
- const parsed = parseObject(raw);
677
- const originRunId = readNonEmptyString(parsed.originRunId);
678
- if (!originRunId)
679
- return null;
680
- const attempt = Math.max(0, Math.floor(asNumber(parsed.attempt, 0)));
681
- return {
682
- originRunId,
683
- previousRunId: readNonEmptyString(parsed.previousRunId),
684
- attempt,
685
- maxAttempts: Math.max(1, Math.floor(asNumber(parsed.maxAttempts, ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS))),
686
- reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
687
- queuedAt: readNonEmptyString(parsed.queuedAt),
688
- };
689
- }
690
- function passiveFollowupCooldownMs(attempt) {
691
- return ISSUE_PASSIVE_FOLLOWUP_COOLDOWN_MS_BY_ATTEMPT.get(attempt) ?? 5 * 60 * 1000;
692
- }
693
- function isAgentEligibleForTimerContinuation(agent) {
694
- return (agent.status !== "paused" &&
695
- agent.status !== "terminated" &&
696
- agent.status !== "pending_approval");
697
- }
698
- function hasCredibleTimerContinuation(input) {
699
- if (!input.policy.enabled || input.policy.intervalSec <= 0)
700
- return false;
701
- if (!isAgentEligibleForTimerContinuation(input.agent))
702
- return false;
703
- const intervalMs = input.policy.intervalSec * 1000;
704
- const nearTermWindowMs = Math.min(intervalMs * 2, ISSUE_PASSIVE_FOLLOWUP_TIMER_CONTINUITY_MAX_WINDOW_MS);
705
- const lastHeartbeatMs = input.agent.lastHeartbeatAt
706
- ? new Date(input.agent.lastHeartbeatAt).getTime()
707
- : new Date(input.agent.createdAt).getTime();
708
- const runFinishedMs = input.run.finishedAt
709
- ? new Date(input.run.finishedAt).getTime()
710
- : input.now.getTime();
711
- const baselineMs = Math.max(lastHeartbeatMs, runFinishedMs);
712
- const nextTimerMs = baselineMs + intervalMs;
713
- return Math.max(0, nextTimerMs - input.now.getTime()) <= nearTermWindowMs;
714
- }
715
- function buildPassiveFollowupContextSnapshot(input) {
716
- const baseContext = { ...parseObject(input.run.contextSnapshot) };
717
- delete baseContext.recovery;
718
- delete baseContext.retryOfRunId;
719
- delete baseContext.retryReason;
720
- const taskKey = deriveTaskKey(baseContext, { issueId: input.issue.id }) ?? input.issue.id;
721
- return {
722
- ...baseContext,
723
- issueId: input.issue.id,
724
- taskId: input.issue.id,
725
- taskKey,
726
- projectId: readNonEmptyString(baseContext.projectId) ?? input.issue.projectId ?? undefined,
727
- wakeReason: ISSUE_PASSIVE_FOLLOWUP_REASON,
728
- wakeSource: ISSUE_PASSIVE_FOLLOWUP_WAKE_SOURCE,
729
- wakeTriggerDetail: "system",
730
- issue: {
731
- id: input.issue.id,
732
- title: input.issue.title,
733
- description: input.issue.description,
734
- status: input.issue.status,
735
- priority: input.issue.priority,
736
- },
737
- passiveFollowup: {
738
- originRunId: input.originRunId,
739
- previousRunId: input.run.id,
740
- attempt: input.attempt,
741
- maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
742
- reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
743
- queuedAt: input.now.toISOString(),
744
- },
745
- };
746
- }
747
- function runTaskKey(run) {
748
- return deriveTaskKey(run.contextSnapshot, null);
749
- }
750
- function isSameTaskScope(left, right) {
751
- return (left ?? null) === (right ?? null);
752
- }
753
- function isTrackedLocalChildProcessAdapter(agentRuntimeType) {
754
- return SESSIONED_LOCAL_ADAPTERS.has(agentRuntimeType);
755
- }
756
- // A positive liveness check means some process currently owns the PID.
757
- // On Linux, PIDs can be recycled, so this is a best-effort signal rather
758
- // than proof that the original child is still alive.
759
- function isProcessAlive(pid) {
760
- if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0)
761
- return false;
762
- try {
763
- process.kill(pid, 0);
764
- return true;
765
- }
766
- catch (error) {
767
- const code = error?.code;
768
- if (code === "EPERM")
769
- return true;
770
- if (code === "ESRCH")
771
- return false;
772
- return false;
773
- }
774
- }
775
- async function waitForProcessExit(pid, timeoutMs) {
776
- const deadline = Date.now() + Math.max(0, timeoutMs);
777
- while (Date.now() < deadline) {
778
- if (!isProcessAlive(pid))
779
- return true;
780
- await new Promise((resolve) => setTimeout(resolve, ORPHANED_PROCESS_POLL_INTERVAL_MS));
781
- }
782
- return !isProcessAlive(pid);
783
- }
784
- async function terminateOrphanedProcess(pid) {
785
- if (!isProcessAlive(pid)) {
786
- return {
787
- stillAlive: false,
788
- terminationSignal: null,
789
- error: null,
790
- };
791
- }
792
- let terminationSignal = null;
793
- try {
794
- process.kill(pid, "SIGTERM");
795
- terminationSignal = "SIGTERM";
796
- }
797
- catch (error) {
798
- const code = error?.code;
799
- if (code === "ESRCH") {
800
- return {
801
- stillAlive: false,
802
- terminationSignal: null,
803
- error: null,
804
- };
805
- }
806
- return {
807
- stillAlive: isProcessAlive(pid),
808
- terminationSignal,
809
- error: `SIGTERM failed: ${error instanceof Error ? error.message : String(error)}`,
810
- };
811
- }
812
- if (await waitForProcessExit(pid, ORPHANED_PROCESS_TERMINATION_GRACE_MS)) {
813
- return {
814
- stillAlive: false,
815
- terminationSignal,
816
- error: null,
817
- };
818
- }
819
- try {
820
- process.kill(pid, "SIGKILL");
821
- terminationSignal = "SIGKILL";
822
- }
823
- catch (error) {
824
- const code = error?.code;
825
- if (code === "ESRCH") {
826
- return {
827
- stillAlive: false,
828
- terminationSignal,
829
- error: null,
830
- };
831
- }
832
- return {
833
- stillAlive: isProcessAlive(pid),
834
- terminationSignal,
835
- error: `SIGKILL failed: ${error instanceof Error ? error.message : String(error)}`,
836
- };
837
- }
838
- const exitedAfterKill = await waitForProcessExit(pid, ORPHANED_PROCESS_KILL_WAIT_MS);
839
- return {
840
- stillAlive: !exitedAfterKill,
841
- terminationSignal,
842
- error: exitedAfterKill ? null : `Timed out waiting for child pid ${pid} to exit after ${terminationSignal}`,
843
- };
844
- }
845
- function truncateDisplayId(value, max = 128) {
846
- if (!value)
847
- return null;
848
- return value.length > max ? value.slice(0, max) : value;
849
- }
850
- function normalizeAgentNameKey(value) {
851
- if (typeof value !== "string")
852
- return null;
853
- const normalized = value.trim().toLowerCase();
854
- return normalized.length > 0 ? normalized : null;
855
- }
856
- const defaultSessionCodec = {
857
- deserialize(raw) {
858
- const asObj = parseObject(raw);
859
- if (Object.keys(asObj).length > 0)
860
- return asObj;
861
- const sessionId = readNonEmptyString(raw?.sessionId);
862
- if (sessionId)
863
- return { sessionId };
864
- return null;
865
- },
866
- serialize(params) {
867
- if (!params || Object.keys(params).length === 0)
868
- return null;
869
- return params;
870
- },
871
- getDisplayId(params) {
872
- return readNonEmptyString(params?.sessionId);
873
- },
874
- };
875
- function getAgentRuntimeSessionCodec(agentRuntimeType) {
876
- const adapter = getServerAdapter(agentRuntimeType);
877
- return adapter.sessionCodec ?? defaultSessionCodec;
878
- }
879
- function normalizeSessionParams(params) {
880
- if (!params)
881
- return null;
882
- return Object.keys(params).length > 0 ? params : null;
883
- }
884
- function resolveNextSessionState(input) {
885
- const { codec, adapterResult, previousParams, previousDisplayId, previousLegacySessionId } = input;
886
- if (adapterResult.clearSession) {
887
- return {
888
- params: null,
889
- displayId: null,
890
- legacySessionId: null,
891
- };
892
- }
893
- const explicitParams = adapterResult.sessionParams;
894
- const hasExplicitParams = adapterResult.sessionParams !== undefined;
895
- const hasExplicitSessionId = adapterResult.sessionId !== undefined;
896
- const explicitSessionId = readNonEmptyString(adapterResult.sessionId);
897
- const hasExplicitDisplay = adapterResult.sessionDisplayId !== undefined;
898
- const explicitDisplayId = readNonEmptyString(adapterResult.sessionDisplayId);
899
- const shouldUsePrevious = !hasExplicitParams && !hasExplicitSessionId && !hasExplicitDisplay;
900
- const candidateParams = hasExplicitParams
901
- ? explicitParams
902
- : hasExplicitSessionId
903
- ? (explicitSessionId ? { sessionId: explicitSessionId } : null)
904
- : previousParams;
905
- const serialized = normalizeSessionParams(codec.serialize(normalizeSessionParams(candidateParams) ?? null));
906
- const deserialized = normalizeSessionParams(codec.deserialize(serialized));
907
- const displayId = truncateDisplayId(explicitDisplayId ??
908
- (codec.getDisplayId ? codec.getDisplayId(deserialized) : null) ??
909
- readNonEmptyString(deserialized?.sessionId) ??
910
- (shouldUsePrevious ? previousDisplayId : null) ??
911
- explicitSessionId ??
912
- (shouldUsePrevious ? previousLegacySessionId : null));
913
- const legacySessionId = explicitSessionId ??
914
- readNonEmptyString(deserialized?.sessionId) ??
915
- displayId ??
916
- (shouldUsePrevious ? previousLegacySessionId : null);
917
- return {
918
- params: serialized,
919
- displayId,
920
- legacySessionId,
921
- };
922
- }
923
- export function heartbeatService(db) {
924
- const instanceSettings = instanceSettingsService(db);
925
- const getCurrentUserRedactionOptions = async () => ({
926
- enabled: (await instanceSettings.getGeneral()).censorUsernameInLogs,
927
- });
928
- const runLogStore = getRunLogStore();
929
- const runContextSvc = agentRunContextService(db);
930
- const issuesSvc = issueService(db);
931
- const executionWorkspacesSvc = executionWorkspaceService(db);
932
- const workspaceOperationsSvc = workspaceOperationService(db);
933
- const activeRunExecutions = new Set();
934
- const budgetHooks = {
935
- cancelWorkForScope: cancelBudgetScopeWork,
936
- };
937
- const budgets = budgetService(db, budgetHooks);
938
- async function getAgent(agentId) {
939
- return db
940
- .select()
941
- .from(agents)
942
- .where(eq(agents.id, agentId))
943
- .then((rows) => rows[0] ?? null);
944
- }
945
- async function getRun(runId) {
946
- return db
947
- .select()
948
- .from(heartbeatRuns)
949
- .where(eq(heartbeatRuns.id, runId))
950
- .then((rows) => rows[0] ?? null);
951
- }
952
- async function getRuntimeState(agentId) {
953
- return db
954
- .select()
955
- .from(agentRuntimeState)
956
- .where(eq(agentRuntimeState.agentId, agentId))
957
- .then((rows) => rows[0] ?? null);
958
- }
959
- async function getTaskSession(orgId, agentId, agentRuntimeType, taskKey) {
960
- return db
961
- .select()
962
- .from(agentTaskSessions)
963
- .where(and(eq(agentTaskSessions.orgId, orgId), eq(agentTaskSessions.agentId, agentId), eq(agentTaskSessions.agentRuntimeType, agentRuntimeType), eq(agentTaskSessions.taskKey, taskKey)))
964
- .then((rows) => rows[0] ?? null);
965
- }
966
- async function getLatestRunForSession(agentId, sessionId, opts) {
967
- const conditions = [
968
- eq(heartbeatRuns.agentId, agentId),
969
- eq(heartbeatRuns.sessionIdAfter, sessionId),
970
- ];
971
- if (opts?.excludeRunId) {
972
- conditions.push(sql `${heartbeatRuns.id} <> ${opts.excludeRunId}`);
973
- }
974
- return db
975
- .select()
976
- .from(heartbeatRuns)
977
- .where(and(...conditions))
978
- .orderBy(desc(heartbeatRuns.createdAt))
979
- .limit(1)
980
- .then((rows) => rows[0] ?? null);
981
- }
982
- async function getOldestRunForSession(agentId, sessionId) {
983
- return db
984
- .select({
985
- id: heartbeatRuns.id,
986
- createdAt: heartbeatRuns.createdAt,
987
- })
988
- .from(heartbeatRuns)
989
- .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.sessionIdAfter, sessionId)))
990
- .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id))
991
- .limit(1)
992
- .then((rows) => rows[0] ?? null);
993
- }
994
- async function resolveNormalizedUsageForSession(input) {
995
- const { agentId, runId, sessionId, rawUsage } = input;
996
- if (!sessionId || !rawUsage) {
997
- return {
998
- normalizedUsage: rawUsage,
999
- previousRawUsage: null,
1000
- derivedFromSessionTotals: false,
1001
- };
1002
- }
1003
- const previousRun = await getLatestRunForSession(agentId, sessionId, { excludeRunId: runId });
1004
- const previousRawUsage = readRawUsageTotals(previousRun?.usageJson);
1005
- return {
1006
- normalizedUsage: deriveNormalizedUsageDelta(rawUsage, previousRawUsage),
1007
- previousRawUsage,
1008
- derivedFromSessionTotals: previousRawUsage !== null,
1009
- };
1010
- }
1011
- async function evaluateSessionCompaction(input) {
1012
- const { agent, sessionId, issueId } = input;
1013
- if (!sessionId) {
1014
- return {
1015
- rotate: false,
1016
- reason: null,
1017
- handoffMarkdown: null,
1018
- previousRunId: null,
1019
- };
1020
- }
1021
- const policy = parseSessionCompactionPolicy(agent);
1022
- if (!policy.enabled || !hasSessionCompactionThresholds(policy)) {
1023
- return {
1024
- rotate: false,
1025
- reason: null,
1026
- handoffMarkdown: null,
1027
- previousRunId: null,
1028
- };
1029
- }
1030
- const fetchLimit = Math.max(policy.maxSessionRuns > 0 ? policy.maxSessionRuns + 1 : 0, 4);
1031
- const runs = await db
1032
- .select({
1033
- id: heartbeatRuns.id,
1034
- createdAt: heartbeatRuns.createdAt,
1035
- usageJson: heartbeatRuns.usageJson,
1036
- resultJson: heartbeatRuns.resultJson,
1037
- error: heartbeatRuns.error,
1038
- })
1039
- .from(heartbeatRuns)
1040
- .where(and(eq(heartbeatRuns.agentId, agent.id), eq(heartbeatRuns.sessionIdAfter, sessionId)))
1041
- .orderBy(desc(heartbeatRuns.createdAt))
1042
- .limit(fetchLimit);
1043
- if (runs.length === 0) {
1044
- return {
1045
- rotate: false,
1046
- reason: null,
1047
- handoffMarkdown: null,
1048
- previousRunId: null,
1049
- };
1050
- }
1051
- const latestRun = runs[0] ?? null;
1052
- const oldestRun = policy.maxSessionAgeHours > 0
1053
- ? await getOldestRunForSession(agent.id, sessionId)
1054
- : runs[runs.length - 1] ?? latestRun;
1055
- const latestRawUsage = readRawUsageTotals(latestRun?.usageJson);
1056
- const sessionAgeHours = latestRun && oldestRun
1057
- ? Math.max(0, (new Date(latestRun.createdAt).getTime() - new Date(oldestRun.createdAt).getTime()) / (1000 * 60 * 60))
1058
- : 0;
1059
- let reason = null;
1060
- if (policy.maxSessionRuns > 0 && runs.length > policy.maxSessionRuns) {
1061
- reason = `session exceeded ${policy.maxSessionRuns} runs`;
1062
- }
1063
- else if (policy.maxRawInputTokens > 0 &&
1064
- latestRawUsage &&
1065
- latestRawUsage.inputTokens >= policy.maxRawInputTokens) {
1066
- reason =
1067
- `session raw input reached ${formatCount(latestRawUsage.inputTokens)} tokens ` +
1068
- `(threshold ${formatCount(policy.maxRawInputTokens)})`;
1069
- }
1070
- else if (policy.maxSessionAgeHours > 0 && sessionAgeHours >= policy.maxSessionAgeHours) {
1071
- reason = `session age reached ${Math.floor(sessionAgeHours)} hours`;
1072
- }
1073
- if (!reason || !latestRun) {
1074
- return {
1075
- rotate: false,
1076
- reason: null,
1077
- handoffMarkdown: null,
1078
- previousRunId: latestRun?.id ?? null,
1079
- };
1080
- }
1081
- const latestSummary = summarizeHeartbeatRunResultJson(latestRun.resultJson);
1082
- const latestTextSummary = readNonEmptyString(latestSummary?.summary) ??
1083
- readNonEmptyString(latestSummary?.result) ??
1084
- readNonEmptyString(latestSummary?.message) ??
1085
- readNonEmptyString(latestRun.error);
1086
- const handoffMarkdown = [
1087
- "Rudder session handoff:",
1088
- `- Previous session: ${sessionId}`,
1089
- issueId ? `- Issue: ${issueId}` : "",
1090
- `- Rotation reason: ${reason}`,
1091
- latestTextSummary ? `- Last run summary: ${latestTextSummary}` : "",
1092
- "Continue from the current task state. Rebuild only the minimum context you need.",
1093
- ]
1094
- .filter(Boolean)
1095
- .join("\n");
1096
- return {
1097
- rotate: true,
1098
- reason,
1099
- handoffMarkdown,
1100
- previousRunId: latestRun.id,
1101
- };
1102
- }
1103
- async function resolveSessionBeforeForWakeup(agent, taskKey) {
1104
- if (taskKey) {
1105
- const codec = getAgentRuntimeSessionCodec(agent.agentRuntimeType);
1106
- const existingTaskSession = await getTaskSession(agent.orgId, agent.id, agent.agentRuntimeType, taskKey);
1107
- const parsedParams = normalizeSessionParams(codec.deserialize(existingTaskSession?.sessionParamsJson ?? null));
1108
- return truncateDisplayId(existingTaskSession?.sessionDisplayId ??
1109
- (codec.getDisplayId ? codec.getDisplayId(parsedParams) : null) ??
1110
- readNonEmptyString(parsedParams?.sessionId));
1111
- }
1112
- const runtimeForRun = await getRuntimeState(agent.id);
1113
- return runtimeForRun?.sessionId ?? null;
1114
- }
1115
- async function resolveExplicitResumeSessionOverride(agent, payload, taskKey) {
1116
- const resumeFromRunId = readNonEmptyString(payload?.resumeFromRunId);
1117
- if (!resumeFromRunId)
1118
- return null;
1119
- const resumeRun = await db
1120
- .select({
1121
- id: heartbeatRuns.id,
1122
- contextSnapshot: heartbeatRuns.contextSnapshot,
1123
- sessionIdBefore: heartbeatRuns.sessionIdBefore,
1124
- sessionIdAfter: heartbeatRuns.sessionIdAfter,
1125
- })
1126
- .from(heartbeatRuns)
1127
- .where(and(eq(heartbeatRuns.id, resumeFromRunId), eq(heartbeatRuns.orgId, agent.orgId), eq(heartbeatRuns.agentId, agent.id)))
1128
- .then((rows) => rows[0] ?? null);
1129
- if (!resumeRun)
1130
- return null;
1131
- const resumeContext = parseObject(resumeRun.contextSnapshot);
1132
- const resumeTaskKey = deriveTaskKey(resumeContext, null) ?? taskKey;
1133
- const resumeTaskSession = resumeTaskKey
1134
- ? await getTaskSession(agent.orgId, agent.id, agent.agentRuntimeType, resumeTaskKey)
1135
- : null;
1136
- const sessionCodec = getAgentRuntimeSessionCodec(agent.agentRuntimeType);
1137
- const sessionOverride = buildExplicitResumeSessionOverride({
1138
- resumeFromRunId,
1139
- resumeRunSessionIdBefore: resumeRun.sessionIdBefore,
1140
- resumeRunSessionIdAfter: resumeRun.sessionIdAfter,
1141
- taskSession: resumeTaskSession,
1142
- sessionCodec,
1143
- });
1144
- if (!sessionOverride)
1145
- return null;
1146
- return {
1147
- resumeFromRunId,
1148
- taskKey: resumeTaskKey,
1149
- issueId: readNonEmptyString(resumeContext.issueId),
1150
- taskId: readNonEmptyString(resumeContext.taskId) ?? readNonEmptyString(resumeContext.issueId),
1151
- sessionDisplayId: sessionOverride.sessionDisplayId,
1152
- sessionParams: sessionOverride.sessionParams,
1153
- };
1154
- }
1155
- async function upsertTaskSession(input) {
1156
- const existing = await getTaskSession(input.orgId, input.agentId, input.agentRuntimeType, input.taskKey);
1157
- if (existing) {
1158
- return db
1159
- .update(agentTaskSessions)
1160
- .set({
1161
- sessionParamsJson: input.sessionParamsJson,
1162
- sessionDisplayId: input.sessionDisplayId,
1163
- lastRunId: input.lastRunId,
1164
- lastError: input.lastError,
1165
- updatedAt: new Date(),
1166
- })
1167
- .where(eq(agentTaskSessions.id, existing.id))
1168
- .returning()
1169
- .then((rows) => rows[0] ?? null);
1170
- }
1171
- return db
1172
- .insert(agentTaskSessions)
1173
- .values({
1174
- orgId: input.orgId,
1175
- agentId: input.agentId,
1176
- agentRuntimeType: input.agentRuntimeType,
1177
- taskKey: input.taskKey,
1178
- sessionParamsJson: input.sessionParamsJson,
1179
- sessionDisplayId: input.sessionDisplayId,
1180
- lastRunId: input.lastRunId,
1181
- lastError: input.lastError,
1182
- })
1183
- .returning()
1184
- .then((rows) => rows[0] ?? null);
1185
- }
1186
- async function clearTaskSessions(orgId, agentId, opts) {
1187
- const conditions = [
1188
- eq(agentTaskSessions.orgId, orgId),
1189
- eq(agentTaskSessions.agentId, agentId),
1190
- ];
1191
- if (opts?.taskKey) {
1192
- conditions.push(eq(agentTaskSessions.taskKey, opts.taskKey));
1193
- }
1194
- if (opts?.agentRuntimeType) {
1195
- conditions.push(eq(agentTaskSessions.agentRuntimeType, opts.agentRuntimeType));
1196
- }
1197
- return db
1198
- .delete(agentTaskSessions)
1199
- .where(and(...conditions))
1200
- .returning()
1201
- .then((rows) => rows.length);
1202
- }
1203
- async function ensureRuntimeState(agent) {
1204
- const existing = await getRuntimeState(agent.id);
1205
- if (existing)
1206
- return existing;
1207
- const now = new Date();
1208
- return db
1209
- .insert(agentRuntimeState)
1210
- .values({
1211
- agentId: agent.id,
1212
- orgId: agent.orgId,
1213
- agentRuntimeType: agent.agentRuntimeType,
1214
- stateJson: {},
1215
- createdAt: now,
1216
- updatedAt: now,
1217
- })
1218
- .onConflictDoUpdate({
1219
- target: agentRuntimeState.agentId,
1220
- set: {
1221
- orgId: agent.orgId,
1222
- agentRuntimeType: agent.agentRuntimeType,
1223
- updatedAt: now,
1224
- },
1225
- })
1226
- .returning()
1227
- .then((rows) => rows[0]);
1228
- }
1229
- function buildHeartbeatObservabilityContext(run, overrides = {}) {
1230
- const contextSnapshot = parseObject(run.contextSnapshot);
1231
- const issueSnapshot = parseObject(contextSnapshot.issue);
1232
- const benchmarkMetadata = extractCreateAgentBenchmarkMetadata(readNonEmptyString(issueSnapshot.description))
1233
- ?? coerceCreateAgentBenchmarkMetadata(parseObject(contextSnapshot.benchmark));
1234
- const baseMetadata = {
1235
- wakeupRequestId: run.wakeupRequestId,
1236
- errorCode: run.errorCode,
1237
- retryOfRunId: run.retryOfRunId,
1238
- processLossRetryCount: run.processLossRetryCount,
1239
- externalRunId: run.externalRunId,
1240
- executionWorkspaceId: readNonEmptyString(contextSnapshot.executionWorkspaceId),
1241
- ...(benchmarkMetadata ?? {}),
1242
- };
1243
- const benchmarkTags = benchmarkMetadata ? buildCreateAgentBenchmarkTags(benchmarkMetadata) : [];
1244
- return {
1245
- surface: resolveHeartbeatObservabilitySurface(contextSnapshot),
1246
- rootExecutionId: run.id,
1247
- orgId: run.orgId,
1248
- agentId: run.agentId,
1249
- issueId: readNonEmptyString(contextSnapshot.issueId),
1250
- sessionKey: run.sessionIdAfter ??
1251
- run.sessionIdBefore ??
1252
- readNonEmptyString(contextSnapshot.sessionKey) ??
1253
- readNonEmptyString(contextSnapshot.taskKey),
1254
- runtime: readNonEmptyString(contextSnapshot.agentRuntimeType),
1255
- trigger: run.triggerDetail ?? run.invocationSource,
1256
- status: run.status,
1257
- metadata: {
1258
- ...baseMetadata,
1259
- ...(overrides.metadata ?? {}),
1260
- },
1261
- tags: [...benchmarkTags, ...(overrides.tags ?? [])],
1262
- ...overrides,
1263
- };
1264
- }
1265
- async function emitHeartbeatObservationEvent(run, input, overrides = {}) {
1266
- try {
1267
- await observeExecutionEvent(buildHeartbeatObservabilityContext(run, overrides), input);
1268
- }
1269
- catch (error) {
1270
- logger.warn({
1271
- runId: run.id,
1272
- eventName: input.name,
1273
- err: error instanceof Error ? error.message : String(error),
1274
- }, "Failed to emit Langfuse heartbeat event");
1275
- }
1276
- }
1277
- async function emitHeartbeatLiveEval(runId) {
1278
- try {
1279
- const { detail, scores } = await buildObservedRunLangfuseScores(db, runId);
1280
- await createExecutionScores(buildHeartbeatObservabilityContext(detail.run, {
1281
- runtime: detail.bundle.agentRuntimeType,
1282
- metadata: {
1283
- agentName: detail.agentName,
1284
- orgName: detail.orgName,
1285
- },
1286
- }), scores.map((score) => ({
1287
- rootExecutionId: detail.run.id,
1288
- name: score.name,
1289
- value: score.value,
1290
- comment: score.comment,
1291
- metadata: score.metadata,
1292
- })));
1293
- }
1294
- catch (error) {
1295
- logger.warn({
1296
- runId,
1297
- err: error instanceof Error ? error.message : String(error),
1298
- }, "Failed to emit Langfuse heartbeat scores");
1299
- }
1300
- }
1301
- async function setRunStatus(runId, status, patch) {
1302
- const updated = await db
1303
- .update(heartbeatRuns)
1304
- .set({ status, ...patch, updatedAt: new Date() })
1305
- .where(eq(heartbeatRuns.id, runId))
1306
- .returning()
1307
- .then((rows) => rows[0] ?? null);
1308
- if (updated) {
1309
- publishLiveEvent({
1310
- orgId: updated.orgId,
1311
- type: "heartbeat.run.status",
1312
- payload: {
1313
- runId: updated.id,
1314
- agentId: updated.agentId,
1315
- status: updated.status,
1316
- invocationSource: updated.invocationSource,
1317
- triggerDetail: updated.triggerDetail,
1318
- error: updated.error ?? null,
1319
- errorCode: updated.errorCode ?? null,
1320
- startedAt: updated.startedAt ? new Date(updated.startedAt).toISOString() : null,
1321
- finishedAt: updated.finishedAt ? new Date(updated.finishedAt).toISOString() : null,
1322
- },
1323
- });
1324
- await emitHeartbeatObservationEvent(updated, {
1325
- name: `heartbeat.status.${status}`,
1326
- asType: "event",
1327
- output: {
1328
- status: updated.status,
1329
- error: updated.error,
1330
- errorCode: updated.errorCode,
1331
- startedAt: updated.startedAt ? new Date(updated.startedAt).toISOString() : null,
1332
- finishedAt: updated.finishedAt ? new Date(updated.finishedAt).toISOString() : null,
1333
- },
1334
- }, {
1335
- status: updated.status,
1336
- });
1337
- }
1338
- return updated;
1339
- }
1340
- async function setWakeupStatus(wakeupRequestId, status, patch) {
1341
- if (!wakeupRequestId)
1342
- return;
1343
- await db
1344
- .update(agentWakeupRequests)
1345
- .set({ status, ...patch, updatedAt: new Date() })
1346
- .where(eq(agentWakeupRequests.id, wakeupRequestId));
1347
- }
1348
- async function updateWakeupRequestRecord(tx, wakeupRequestId, patch) {
1349
- return tx
1350
- .update(agentWakeupRequests)
1351
- .set({ ...patch, updatedAt: new Date() })
1352
- .where(eq(agentWakeupRequests.id, wakeupRequestId))
1353
- .returning()
1354
- .then((rows) => rows[0] ?? null);
1355
- }
1356
- async function insertWakeupRequestRecord(tx, values) {
1357
- return tx
1358
- .insert(agentWakeupRequests)
1359
- .values(values)
1360
- .returning()
1361
- .then((rows) => rows[0] ?? null);
1362
- }
1363
- async function appendRunEvent(run, seq, event) {
1364
- const currentUserRedactionOptions = await getCurrentUserRedactionOptions();
1365
- const sanitizedMessage = event.message
1366
- ? redactCurrentUserText(event.message, currentUserRedactionOptions)
1367
- : event.message;
1368
- const sanitizedPayload = event.payload
1369
- ? redactCurrentUserValue(event.payload, currentUserRedactionOptions)
1370
- : event.payload;
1371
- await db.insert(heartbeatRunEvents).values({
1372
- orgId: run.orgId,
1373
- runId: run.id,
1374
- agentId: run.agentId,
1375
- seq,
1376
- eventType: event.eventType,
1377
- stream: event.stream,
1378
- level: event.level,
1379
- color: event.color,
1380
- message: sanitizedMessage,
1381
- payload: sanitizedPayload,
1382
- });
1383
- publishLiveEvent({
1384
- orgId: run.orgId,
1385
- type: "heartbeat.run.event",
1386
- payload: {
1387
- runId: run.id,
1388
- agentId: run.agentId,
1389
- seq,
1390
- eventType: event.eventType,
1391
- stream: event.stream ?? null,
1392
- level: event.level ?? null,
1393
- color: event.color ?? null,
1394
- message: sanitizedMessage ?? null,
1395
- payload: sanitizedPayload ?? null,
1396
- },
1397
- });
1398
- await emitHeartbeatObservationEvent(run, {
1399
- name: `heartbeat.event.${event.eventType}`,
1400
- asType: "event",
1401
- level: event.level === "error" ? "ERROR" : event.level === "warn" ? "WARNING" : "DEFAULT",
1402
- output: {
1403
- seq,
1404
- eventType: event.eventType,
1405
- stream: event.stream ?? null,
1406
- level: event.level ?? null,
1407
- color: event.color ?? null,
1408
- message: sanitizedMessage ?? null,
1409
- },
1410
- metadata: sanitizedPayload ?? undefined,
1411
- }, {
1412
- status: run.status,
1413
- });
1414
- }
1415
- async function nextRunEventSeq(runId) {
1416
- const [row] = await db
1417
- .select({ maxSeq: sql `max(${heartbeatRunEvents.seq})` })
1418
- .from(heartbeatRunEvents)
1419
- .where(eq(heartbeatRunEvents.runId, runId));
1420
- return Number(row?.maxSeq ?? 0) + 1;
1421
- }
1422
- async function persistRunProcessMetadata(runId, meta) {
1423
- const startedAt = new Date(meta.startedAt);
1424
- const updated = await db
1425
- .update(heartbeatRuns)
1426
- .set({
1427
- processPid: meta.pid,
1428
- processStartedAt: Number.isNaN(startedAt.getTime()) ? new Date() : startedAt,
1429
- updatedAt: new Date(),
1430
- })
1431
- .where(eq(heartbeatRuns.id, runId))
1432
- .returning()
1433
- .then((rows) => rows[0] ?? null);
1434
- if (updated) {
1435
- await emitHeartbeatObservationEvent(updated, {
1436
- name: "heartbeat.process.spawn",
1437
- asType: "event",
1438
- output: {
1439
- pid: meta.pid,
1440
- startedAt: meta.startedAt,
1441
- },
1442
- });
1443
- }
1444
- return updated;
1445
- }
1446
- async function clearDetachedRunWarning(runId) {
1447
- const updated = await db
1448
- .update(heartbeatRuns)
1449
- .set({
1450
- error: null,
1451
- errorCode: null,
1452
- updatedAt: new Date(),
1453
- })
1454
- .where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.errorCode, DETACHED_PROCESS_ERROR_CODE)))
1455
- .returning()
1456
- .then((rows) => rows[0] ?? null);
1457
- if (!updated)
1458
- return null;
1459
- await appendRunEvent(updated, await nextRunEventSeq(updated.id), {
1460
- eventType: "lifecycle",
1461
- stream: "system",
1462
- level: "info",
1463
- message: "Detached child process reported activity; cleared detached warning",
1464
- });
1465
- return updated;
1466
- }
1467
- async function enqueueRecoveryRun(run, agent, opts) {
1468
- /**
1469
- * Recovery runs intentionally clone the prior run's task context and then
1470
- * layer explicit recovery metadata on top. This keeps retries visible and
1471
- * auditable while preserving "continue preferred" semantics for issue work.
1472
- *
1473
- * Reasoning:
1474
- * - Manual retry and automatic process-loss retry must assemble the same
1475
- * recovery contract so prompts/runtime behavior stay aligned.
1476
- * - We backfill missing context from the retry chain to recover from older
1477
- * lossy retry runs without mutating the historical source run rows.
1478
- *
1479
- * Traceability:
1480
- * - doc/developing/RUN-RECOVERY.md
1481
- * - doc/DEVELOPING.md
1482
- */
1483
- const baseContextSnapshot = await hydrateRecoveryBaseContextSnapshot(run, getRun);
1484
- const recoveryContextSnapshot = buildRecoveryContextSnapshot({
1485
- baseContextSnapshot,
1486
- run,
1487
- recoveryTrigger: opts.recoveryTrigger,
1488
- wakeReason: opts.wakeReason,
1489
- wakeSource: `recovery.${opts.recoveryTrigger}`,
1490
- triggerDetail: opts.triggerDetail,
1491
- });
1492
- const issueId = readNonEmptyString(recoveryContextSnapshot.issueId);
1493
- const taskKey = deriveTaskKey(recoveryContextSnapshot, null);
1494
- const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey);
1495
- const recovery = recoveryContextSnapshot.recovery;
1496
- const requestPayload = {
1497
- originalRunId: run.id,
1498
- failureKind: recovery.failureKind,
1499
- recoveryTrigger: recovery.recoveryTrigger,
1500
- ...(issueId ? { issueId } : {}),
1501
- };
1502
- const outcome = await db.transaction(async (tx) => {
1503
- let issueRow = null;
1504
- if (issueId) {
1505
- await tx.execute(sql `select id from issues where id = ${issueId} and org_id = ${run.orgId} for update`);
1506
- issueRow = await tx
1507
- .select({
1508
- id: issues.id,
1509
- orgId: issues.orgId,
1510
- executionRunId: issues.executionRunId,
1511
- executionAgentNameKey: issues.executionAgentNameKey,
1512
- })
1513
- .from(issues)
1514
- .where(and(eq(issues.id, issueId), eq(issues.orgId, run.orgId)))
1515
- .then((rows) => rows[0] ?? null);
1516
- }
1517
- if (issueRow?.executionRunId) {
1518
- const activeExecutionRun = await tx
1519
- .select()
1520
- .from(heartbeatRuns)
1521
- .where(eq(heartbeatRuns.id, issueRow.executionRunId))
1522
- .then((rows) => rows[0] ?? null);
1523
- const isActiveExecutionRun = activeExecutionRun &&
1524
- (activeExecutionRun.status === "queued" || activeExecutionRun.status === "running");
1525
- if (!isActiveExecutionRun) {
1526
- await tx
1527
- .update(issues)
1528
- .set({
1529
- executionRunId: null,
1530
- executionAgentNameKey: null,
1531
- executionLockedAt: null,
1532
- updatedAt: opts.now,
1533
- })
1534
- .where(eq(issues.id, issueRow.id));
1535
- issueRow = {
1536
- ...issueRow,
1537
- executionRunId: null,
1538
- executionAgentNameKey: null,
1539
- };
1540
- }
1541
- else if (activeExecutionRun) {
1542
- const activeContext = parseObject(activeExecutionRun.contextSnapshot);
1543
- const activeRecovery = parseObject(activeContext.recovery);
1544
- if (activeExecutionRun.agentId === run.agentId &&
1545
- (activeExecutionRun.retryOfRunId === run.id ||
1546
- readNonEmptyString(activeRecovery.originalRunId) === run.id)) {
1547
- return { kind: "existing", run: activeExecutionRun };
1548
- }
1549
- throw conflict("Issue already has an active execution run", {
1550
- issueId: issueRow.id,
1551
- executionRunId: activeExecutionRun.id,
1552
- });
1553
- }
1554
- }
1555
- const wakeupRequest = await tx
1556
- .insert(agentWakeupRequests)
1557
- .values({
1558
- orgId: run.orgId,
1559
- agentId: run.agentId,
1560
- source: opts.source,
1561
- triggerDetail: opts.triggerDetail,
1562
- reason: opts.wakeReason,
1563
- payload: requestPayload,
1564
- status: "queued",
1565
- requestedByActorType: opts.requestedByActorType ?? null,
1566
- requestedByActorId: opts.requestedByActorId ?? null,
1567
- updatedAt: opts.now,
1568
- })
1569
- .returning()
1570
- .then((rows) => rows[0]);
1571
- const recoveryRun = await tx
1572
- .insert(heartbeatRuns)
1573
- .values({
1574
- orgId: run.orgId,
1575
- agentId: run.agentId,
1576
- invocationSource: opts.source,
1577
- triggerDetail: opts.triggerDetail,
1578
- status: "queued",
1579
- wakeupRequestId: wakeupRequest.id,
1580
- contextSnapshot: recoveryContextSnapshot,
1581
- sessionIdBefore: sessionBefore,
1582
- retryOfRunId: run.id,
1583
- processLossRetryCount: opts.recoveryTrigger === "automatic"
1584
- ? (run.processLossRetryCount ?? 0) + 1
1585
- : (run.processLossRetryCount ?? 0),
1586
- updatedAt: opts.now,
1587
- })
1588
- .returning()
1589
- .then((rows) => rows[0]);
1590
- await tx
1591
- .update(agentWakeupRequests)
1592
- .set({
1593
- runId: recoveryRun.id,
1594
- updatedAt: opts.now,
1595
- })
1596
- .where(eq(agentWakeupRequests.id, wakeupRequest.id));
1597
- if (issueRow) {
1598
- await tx
1599
- .update(issues)
1600
- .set({
1601
- executionRunId: recoveryRun.id,
1602
- executionAgentNameKey: normalizeAgentNameKey(agent.name),
1603
- executionLockedAt: opts.now,
1604
- updatedAt: opts.now,
1605
- })
1606
- .where(eq(issues.id, issueRow.id));
1607
- }
1608
- return { kind: "queued", run: recoveryRun };
1609
- });
1610
- if (outcome.kind === "existing")
1611
- return outcome.run;
1612
- const recoveryRun = outcome.run;
1613
- await appendRunEvent(recoveryRun, await nextRunEventSeq(recoveryRun.id), {
1614
- eventType: "lifecycle",
1615
- stream: "system",
1616
- level: opts.recoveryTrigger === "automatic" ? "warn" : "info",
1617
- message: `Recovery queued from run ${run.id}`,
1618
- payload: {
1619
- originalRunId: run.id,
1620
- failureKind: recovery.failureKind,
1621
- failureSummary: recovery.failureSummary,
1622
- recoveryTrigger: recovery.recoveryTrigger,
1623
- recoveryMode: recovery.recoveryMode,
1624
- },
1625
- });
1626
- publishLiveEvent({
1627
- orgId: recoveryRun.orgId,
1628
- type: "heartbeat.run.queued",
1629
- payload: {
1630
- runId: recoveryRun.id,
1631
- agentId: recoveryRun.agentId,
1632
- invocationSource: recoveryRun.invocationSource,
1633
- triggerDetail: recoveryRun.triggerDetail,
1634
- wakeupRequestId: recoveryRun.wakeupRequestId,
1635
- },
1636
- });
1637
- if (opts.startImmediately !== false) {
1638
- await startNextQueuedRunForAgent(agent.id);
1639
- }
1640
- return recoveryRun;
1641
- }
1642
- async function enqueueProcessLossRetry(run, agent, now) {
1643
- return enqueueRecoveryRun(run, agent, {
1644
- recoveryTrigger: "automatic",
1645
- source: "automation",
1646
- triggerDetail: "system",
1647
- wakeReason: "process_lost_retry",
1648
- requestedByActorType: "system",
1649
- requestedByActorId: null,
1650
- startImmediately: false,
1651
- now,
1652
- });
1653
- }
1654
- function parseHeartbeatPolicy(agent) {
1655
- const runtimeConfig = parseObject(agent.runtimeConfig);
1656
- const heartbeat = parseObject(runtimeConfig.heartbeat);
1657
- return {
1658
- enabled: asBoolean(heartbeat.enabled, true),
1659
- intervalSec: Math.max(0, asNumber(heartbeat.intervalSec, 0)),
1660
- wakeOnDemand: asBoolean(heartbeat.wakeOnDemand ?? heartbeat.wakeOnAssignment ?? heartbeat.wakeOnOnDemand ?? heartbeat.wakeOnAutomation, true),
1661
- maxConcurrentRuns: normalizeMaxConcurrentRuns(heartbeat.maxConcurrentRuns),
1662
- };
1663
- }
1664
- async function runHasIssueClosureComment(tx, run, issueId) {
1665
- const commentActivity = await tx
1666
- .select({ id: activityLog.id })
1667
- .from(activityLog)
1668
- .where(and(eq(activityLog.orgId, run.orgId), eq(activityLog.action, "issue.comment_added"), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issueId), eq(activityLog.runId, run.id)))
1669
- .limit(1)
1670
- .then((rows) => rows[0] ?? null);
1671
- return Boolean(commentActivity);
1672
- }
1673
- async function issueHasDeferredWake(tx, orgId, issueId) {
1674
- const deferred = await tx
1675
- .select({ id: agentWakeupRequests.id })
1676
- .from(agentWakeupRequests)
1677
- .where(and(eq(agentWakeupRequests.orgId, orgId), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql `${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`))
1678
- .limit(1)
1679
- .then((rows) => rows[0] ?? null);
1680
- return Boolean(deferred);
1681
- }
1682
- async function passiveFollowupAlreadyRecorded(tx, runId) {
1683
- const idempotencyKey = `${ISSUE_PASSIVE_FOLLOWUP_REASON}:${runId}`;
1684
- const existingWake = await tx
1685
- .select({ id: agentWakeupRequests.id })
1686
- .from(agentWakeupRequests)
1687
- .where(eq(agentWakeupRequests.idempotencyKey, idempotencyKey))
1688
- .limit(1)
1689
- .then((rows) => rows[0] ?? null);
1690
- if (existingWake)
1691
- return true;
1692
- const existingReview = await tx
1693
- .select({ id: activityLog.id })
1694
- .from(activityLog)
1695
- .where(and(eq(activityLog.runId, runId), eq(activityLog.action, "issue.closure_needs_operator_review")))
1696
- .limit(1)
1697
- .then((rows) => rows[0] ?? null);
1698
- return Boolean(existingReview);
1699
- }
1700
- async function evaluatePassiveIssueClosureForLockedIssue(input) {
1701
- const { tx, run, issue, now } = input;
1702
- const context = parseObject(run.contextSnapshot);
1703
- const runIssueId = readNonEmptyString(context.issueId);
1704
- if (!runIssueId || runIssueId !== issue.id)
1705
- return { kind: "none", reason: "run_not_issue_backed" };
1706
- if (run.status !== "succeeded")
1707
- return { kind: "none", reason: "run_not_successful" };
1708
- if (issue.status !== "todo" && issue.status !== "in_progress") {
1709
- return { kind: "none", reason: "issue_has_closure_status" };
1710
- }
1711
- if (issue.assigneeAgentId !== run.agentId) {
1712
- return { kind: "none", reason: "issue_no_longer_assigned_to_run_agent" };
1713
- }
1714
- if (await runHasIssueClosureComment(tx, run, issue.id)) {
1715
- return { kind: "none", reason: "run_authored_issue_comment" };
1716
- }
1717
- if (await issueHasDeferredWake(tx, issue.orgId, issue.id)) {
1718
- return { kind: "none", reason: "deferred_issue_wake_exists" };
1719
- }
1720
- if (await passiveFollowupAlreadyRecorded(tx, run.id)) {
1721
- return { kind: "none", reason: "passive_followup_already_recorded" };
1722
- }
1723
- const agent = await tx
1724
- .select()
1725
- .from(agents)
1726
- .where(eq(agents.id, run.agentId))
1727
- .then((rows) => rows[0] ?? null);
1728
- if (!agent || agent.orgId !== run.orgId) {
1729
- return { kind: "none", reason: "agent_not_found" };
1730
- }
1731
- const policy = parseHeartbeatPolicy(agent);
1732
- if (hasCredibleTimerContinuation({ agent, policy, run, now })) {
1733
- return { kind: "none", reason: "timer_continuity_expected" };
1734
- }
1735
- const passiveContext = normalizePassiveFollowupContext(context.passiveFollowup);
1736
- const currentAttempt = passiveContext?.attempt ?? 0;
1737
- const originRunId = passiveContext?.originRunId ?? run.id;
1738
- if (currentAttempt >= ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS) {
1739
- return {
1740
- kind: "operator_review",
1741
- issue,
1742
- originRunId,
1743
- previousRunId: run.id,
1744
- attempts: currentAttempt,
1745
- reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
1746
- };
1747
- }
1748
- const nextAttempt = currentAttempt + 1;
1749
- const requestedAt = new Date(now.getTime() + passiveFollowupCooldownMs(nextAttempt));
1750
- const contextSnapshot = buildPassiveFollowupContextSnapshot({
1751
- run,
1752
- issue,
1753
- originRunId,
1754
- attempt: nextAttempt,
1755
- now,
1756
- });
1757
- const taskKey = deriveTaskKey(contextSnapshot, { issueId: issue.id });
1758
- const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey);
1759
- const requestPayload = {
1760
- issueId: issue.id,
1761
- originRunId,
1762
- previousRunId: run.id,
1763
- attempt: nextAttempt,
1764
- reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
1765
- };
1766
- const wakeupRequest = await tx
1767
- .insert(agentWakeupRequests)
1768
- .values({
1769
- orgId: run.orgId,
1770
- agentId: run.agentId,
1771
- source: "automation",
1772
- triggerDetail: "system",
1773
- reason: ISSUE_PASSIVE_FOLLOWUP_REASON,
1774
- payload: requestPayload,
1775
- status: "queued",
1776
- requestedByActorType: "system",
1777
- requestedByActorId: "issue_closure_governance",
1778
- idempotencyKey: `${ISSUE_PASSIVE_FOLLOWUP_REASON}:${run.id}`,
1779
- requestedAt,
1780
- updatedAt: now,
1781
- })
1782
- .returning()
1783
- .then((rows) => rows[0]);
1784
- const followupRun = await tx
1785
- .insert(heartbeatRuns)
1786
- .values({
1787
- orgId: run.orgId,
1788
- agentId: run.agentId,
1789
- invocationSource: "automation",
1790
- triggerDetail: "system",
1791
- status: "queued",
1792
- wakeupRequestId: wakeupRequest.id,
1793
- contextSnapshot,
1794
- sessionIdBefore: sessionBefore,
1795
- updatedAt: now,
1796
- })
1797
- .returning()
1798
- .then((rows) => rows[0]);
1799
- await tx
1800
- .update(agentWakeupRequests)
1801
- .set({
1802
- runId: followupRun.id,
1803
- updatedAt: now,
1804
- })
1805
- .where(eq(agentWakeupRequests.id, wakeupRequest.id));
1806
- await tx
1807
- .update(issues)
1808
- .set({
1809
- executionRunId: followupRun.id,
1810
- executionAgentNameKey: normalizeAgentNameKey(agent.name),
1811
- executionLockedAt: now,
1812
- updatedAt: now,
1813
- })
1814
- .where(eq(issues.id, issue.id));
1815
- return {
1816
- kind: "queued",
1817
- run: followupRun,
1818
- issue,
1819
- originRunId,
1820
- previousRunId: run.id,
1821
- attempt: nextAttempt,
1822
- requestedAt,
1823
- };
1824
- }
1825
- async function countRunningRunsForAgent(agentId) {
1826
- const [{ count }] = await db
1827
- .select({ count: sql `count(*)` })
1828
- .from(heartbeatRuns)
1829
- .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "running")));
1830
- return Number(count ?? 0);
1831
- }
1832
- async function claimQueuedRun(run) {
1833
- if (run.status !== "queued")
1834
- return run;
1835
- if (run.wakeupRequestId) {
1836
- const wakeup = await db
1837
- .select({ requestedAt: agentWakeupRequests.requestedAt })
1838
- .from(agentWakeupRequests)
1839
- .where(eq(agentWakeupRequests.id, run.wakeupRequestId))
1840
- .then((rows) => rows[0] ?? null);
1841
- if (wakeup && new Date(wakeup.requestedAt).getTime() > Date.now()) {
1842
- return null;
1843
- }
1844
- }
1845
- const agent = await getAgent(run.agentId);
1846
- if (!agent) {
1847
- await cancelRunInternal(run.id, "Cancelled because the agent no longer exists");
1848
- return null;
1849
- }
1850
- if (agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval") {
1851
- await cancelRunInternal(run.id, "Cancelled because the agent is not invokable");
1852
- return null;
1853
- }
1854
- const context = parseObject(run.contextSnapshot);
1855
- const budgetBlock = await budgets.getInvocationBlock(run.orgId, run.agentId, {
1856
- issueId: readNonEmptyString(context.issueId),
1857
- projectId: readNonEmptyString(context.projectId),
1858
- });
1859
- if (budgetBlock) {
1860
- await cancelRunInternal(run.id, budgetBlock.reason);
1861
- return null;
1862
- }
1863
- const claimedAt = new Date();
1864
- const claimed = await db
1865
- .update(heartbeatRuns)
1866
- .set({
1867
- status: "running",
1868
- startedAt: run.startedAt ?? claimedAt,
1869
- updatedAt: claimedAt,
1870
- })
1871
- .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "queued")))
1872
- .returning()
1873
- .then((rows) => rows[0] ?? null);
1874
- if (!claimed)
1875
- return null;
1876
- publishLiveEvent({
1877
- orgId: claimed.orgId,
1878
- type: "heartbeat.run.status",
1879
- payload: {
1880
- runId: claimed.id,
1881
- agentId: claimed.agentId,
1882
- status: claimed.status,
1883
- invocationSource: claimed.invocationSource,
1884
- triggerDetail: claimed.triggerDetail,
1885
- error: claimed.error ?? null,
1886
- errorCode: claimed.errorCode ?? null,
1887
- startedAt: claimed.startedAt ? new Date(claimed.startedAt).toISOString() : null,
1888
- finishedAt: claimed.finishedAt ? new Date(claimed.finishedAt).toISOString() : null,
1889
- },
1890
- });
1891
- await setWakeupStatus(claimed.wakeupRequestId, "claimed", { claimedAt });
1892
- return claimed;
1893
- }
1894
- async function finalizeAgentStatus(agentId, outcome) {
1895
- const existing = await getAgent(agentId);
1896
- if (!existing)
1897
- return;
1898
- if (existing.status === "paused" || existing.status === "terminated") {
1899
- return;
1900
- }
1901
- const runningCount = await countRunningRunsForAgent(agentId);
1902
- const nextStatus = runningCount > 0
1903
- ? "running"
1904
- : outcome === "succeeded" || outcome === "cancelled"
1905
- ? "idle"
1906
- : "error";
1907
- const updated = await db
1908
- .update(agents)
1909
- .set({
1910
- status: nextStatus,
1911
- lastHeartbeatAt: new Date(),
1912
- updatedAt: new Date(),
1913
- })
1914
- .where(eq(agents.id, agentId))
1915
- .returning()
1916
- .then((rows) => rows[0] ?? null);
1917
- if (updated) {
1918
- publishLiveEvent({
1919
- orgId: updated.orgId,
1920
- type: "agent.status",
1921
- payload: {
1922
- agentId: updated.id,
1923
- status: updated.status,
1924
- lastHeartbeatAt: updated.lastHeartbeatAt
1925
- ? new Date(updated.lastHeartbeatAt).toISOString()
1926
- : null,
1927
- outcome,
1928
- },
1929
- });
1930
- }
1931
- }
1932
- async function reapOrphanedRuns(opts) {
1933
- const staleThresholdMs = opts?.staleThresholdMs ?? 0;
1934
- const now = new Date();
1935
- // Find all runs stuck in "running" state (queued runs are legitimately waiting; resumeQueuedRuns handles them)
1936
- const activeRuns = await db
1937
- .select({
1938
- run: heartbeatRuns,
1939
- agentRuntimeType: agents.agentRuntimeType,
1940
- })
1941
- .from(heartbeatRuns)
1942
- .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
1943
- .where(eq(heartbeatRuns.status, "running"));
1944
- const reaped = [];
1945
- for (const { run, agentRuntimeType } of activeRuns) {
1946
- if (runningProcesses.has(run.id) || activeRunExecutions.has(run.id))
1947
- continue;
1948
- // Apply staleness threshold to avoid false positives
1949
- if (staleThresholdMs > 0) {
1950
- const refTime = run.updatedAt ? new Date(run.updatedAt).getTime() : 0;
1951
- if (now.getTime() - refTime < staleThresholdMs)
1952
- continue;
1953
- }
1954
- const tracksLocalChild = isTrackedLocalChildProcessAdapter(agentRuntimeType);
1955
- let detachedTerminationMessage = null;
1956
- if (tracksLocalChild && run.processPid && isProcessAlive(run.processPid)) {
1957
- const termination = await terminateOrphanedProcess(run.processPid);
1958
- if (termination.stillAlive) {
1959
- const detachedMessage = termination.error
1960
- ? `Lost in-memory process handle, child pid ${run.processPid} is still alive, and Rudder could not terminate it: ${termination.error}`
1961
- : `Lost in-memory process handle, but child pid ${run.processPid} is still alive`;
1962
- const detachedRun = await setRunStatus(run.id, "running", {
1963
- error: detachedMessage,
1964
- errorCode: DETACHED_PROCESS_ERROR_CODE,
1965
- });
1966
- if (detachedRun) {
1967
- await appendRunEvent(detachedRun, await nextRunEventSeq(detachedRun.id), {
1968
- eventType: "lifecycle",
1969
- stream: "system",
1970
- level: "warn",
1971
- message: detachedMessage,
1972
- payload: {
1973
- processPid: run.processPid,
1974
- },
1975
- });
1976
- }
1977
- continue;
1978
- }
1979
- detachedTerminationMessage = termination.terminationSignal
1980
- ? `Terminated detached child pid ${run.processPid} with ${termination.terminationSignal} after Rudder lost its process handle`
1981
- : `Detached child pid ${run.processPid} exited before Rudder could terminate it`;
1982
- }
1983
- const shouldRetry = tracksLocalChild && !!run.processPid && (run.processLossRetryCount ?? 0) < 1;
1984
- const baseMessage = run.processPid
1985
- ? `Process lost -- child pid ${run.processPid} is no longer running`
1986
- : "Process lost -- server may have restarted";
1987
- let finalizedRun = await setRunStatus(run.id, "failed", {
1988
- error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
1989
- errorCode: "process_lost",
1990
- finishedAt: now,
1991
- });
1992
- await setWakeupStatus(run.wakeupRequestId, "failed", {
1993
- finishedAt: now,
1994
- error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
1995
- });
1996
- if (!finalizedRun)
1997
- finalizedRun = await getRun(run.id);
1998
- if (!finalizedRun)
1999
- continue;
2000
- if (detachedTerminationMessage) {
2001
- await appendRunEvent(finalizedRun, await nextRunEventSeq(finalizedRun.id), {
2002
- eventType: "lifecycle",
2003
- stream: "system",
2004
- level: "warn",
2005
- message: detachedTerminationMessage,
2006
- payload: {
2007
- ...(run.processPid ? { processPid: run.processPid } : {}),
2008
- },
2009
- });
2010
- }
2011
- let retriedRun = null;
2012
- if (shouldRetry) {
2013
- const agent = await getAgent(run.agentId);
2014
- if (agent) {
2015
- retriedRun = await enqueueProcessLossRetry(finalizedRun, agent, now);
2016
- }
2017
- }
2018
- else {
2019
- await releaseIssueExecutionAndPromote(finalizedRun);
2020
- }
2021
- await appendRunEvent(finalizedRun, await nextRunEventSeq(finalizedRun.id), {
2022
- eventType: "lifecycle",
2023
- stream: "system",
2024
- level: "error",
2025
- message: shouldRetry
2026
- ? `${baseMessage}; queued retry ${retriedRun?.id ?? ""}`.trim()
2027
- : baseMessage,
2028
- payload: {
2029
- ...(run.processPid ? { processPid: run.processPid } : {}),
2030
- ...(retriedRun ? { retryRunId: retriedRun.id } : {}),
2031
- },
2032
- });
2033
- await finalizeAgentStatus(run.agentId, "failed");
2034
- await startNextQueuedRunForAgent(run.agentId);
2035
- runningProcesses.delete(run.id);
2036
- reaped.push(run.id);
2037
- }
2038
- if (reaped.length > 0) {
2039
- logger.warn({ reapedCount: reaped.length, runIds: reaped }, "reaped orphaned heartbeat runs");
2040
- }
2041
- return { reaped: reaped.length, runIds: reaped };
2042
- }
2043
- async function resumeQueuedRuns() {
2044
- const queuedRuns = await db
2045
- .select({ agentId: heartbeatRuns.agentId })
2046
- .from(heartbeatRuns)
2047
- .where(eq(heartbeatRuns.status, "queued"));
2048
- const agentIds = [...new Set(queuedRuns.map((r) => r.agentId))];
2049
- for (const agentId of agentIds) {
2050
- await startNextQueuedRunForAgent(agentId);
2051
- }
2052
- }
2053
- async function updateRuntimeState(agent, run, result, session, normalizedUsage) {
2054
- await ensureRuntimeState(agent);
2055
- const usage = normalizedUsage ?? normalizeUsageTotals(result.usage);
2056
- const inputTokens = usage?.inputTokens ?? 0;
2057
- const outputTokens = usage?.outputTokens ?? 0;
2058
- const cachedInputTokens = usage?.cachedInputTokens ?? 0;
2059
- const billingType = normalizeLedgerBillingType(result.billingType);
2060
- const additionalCostCents = normalizeBilledCostCents(result.costUsd, billingType);
2061
- const hasTokenUsage = inputTokens > 0 || outputTokens > 0 || cachedInputTokens > 0;
2062
- const provider = result.provider ?? "unknown";
2063
- const biller = resolveLedgerBiller(result);
2064
- const ledgerScope = await resolveLedgerScopeForRun(db, agent.orgId, run);
2065
- await db
2066
- .update(agentRuntimeState)
2067
- .set({
2068
- agentRuntimeType: agent.agentRuntimeType,
2069
- sessionId: session.legacySessionId,
2070
- lastRunId: run.id,
2071
- lastRunStatus: run.status,
2072
- lastError: result.errorMessage ?? null,
2073
- totalInputTokens: sql `${agentRuntimeState.totalInputTokens} + ${inputTokens}`,
2074
- totalOutputTokens: sql `${agentRuntimeState.totalOutputTokens} + ${outputTokens}`,
2075
- totalCachedInputTokens: sql `${agentRuntimeState.totalCachedInputTokens} + ${cachedInputTokens}`,
2076
- totalCostCents: sql `${agentRuntimeState.totalCostCents} + ${additionalCostCents}`,
2077
- updatedAt: new Date(),
2078
- })
2079
- .where(eq(agentRuntimeState.agentId, agent.id));
2080
- if (additionalCostCents > 0 || hasTokenUsage) {
2081
- const costs = costService(db, budgetHooks);
2082
- await costs.createEvent(agent.orgId, {
2083
- heartbeatRunId: run.id,
2084
- agentId: agent.id,
2085
- issueId: ledgerScope.issueId,
2086
- projectId: ledgerScope.projectId,
2087
- provider,
2088
- biller,
2089
- billingType,
2090
- model: result.model ?? "unknown",
2091
- inputTokens,
2092
- cachedInputTokens,
2093
- outputTokens,
2094
- costCents: additionalCostCents,
2095
- occurredAt: new Date(),
2096
- });
2097
- }
2098
- }
2099
- async function startNextQueuedRunForAgent(agentId) {
2100
- return withAgentStartLock(agentId, async () => {
2101
- const agent = await getAgent(agentId);
2102
- if (!agent)
2103
- return [];
2104
- if (agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval") {
2105
- return [];
2106
- }
2107
- const policy = parseHeartbeatPolicy(agent);
2108
- const runningCount = await countRunningRunsForAgent(agentId);
2109
- const availableSlots = Math.max(0, policy.maxConcurrentRuns - runningCount);
2110
- if (availableSlots <= 0)
2111
- return [];
2112
- const queuedRuns = await db
2113
- .select()
2114
- .from(heartbeatRuns)
2115
- .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued"), sql `(
2116
- ${heartbeatRuns.wakeupRequestId} is null
2117
- or exists (
2118
- select 1
2119
- from ${agentWakeupRequests}
2120
- where ${agentWakeupRequests.id} = ${heartbeatRuns.wakeupRequestId}
2121
- and ${agentWakeupRequests.requestedAt} <= now()
2122
- )
2123
- )`))
2124
- .orderBy(asc(heartbeatRuns.createdAt))
2125
- .limit(availableSlots);
2126
- if (queuedRuns.length === 0)
2127
- return [];
2128
- const claimedRuns = [];
2129
- for (const queuedRun of queuedRuns) {
2130
- const claimed = await claimQueuedRun(queuedRun);
2131
- if (claimed)
2132
- claimedRuns.push(claimed);
2133
- }
2134
- if (claimedRuns.length === 0)
2135
- return [];
2136
- for (const claimedRun of claimedRuns) {
2137
- void executeRun(claimedRun.id).catch((err) => {
2138
- logger.error({ err, runId: claimedRun.id }, "queued heartbeat execution failed");
2139
- });
2140
- }
2141
- return claimedRuns;
2142
- });
2143
- }
2144
- async function executeRun(runId) {
2145
- let run = await getRun(runId);
2146
- if (!run)
2147
- return;
2148
- if (run.status !== "queued" && run.status !== "running")
2149
- return;
2150
- if (run.status === "queued") {
2151
- const claimed = await claimQueuedRun(run);
2152
- if (!claimed) {
2153
- // Another worker has already claimed or finalized this run.
2154
- return;
2155
- }
2156
- run = claimed;
2157
- }
2158
- activeRunExecutions.add(run.id);
2159
- try {
2160
- const agent = await getAgent(run.agentId);
2161
- if (!agent) {
2162
- await setRunStatus(runId, "failed", {
2163
- error: "Agent not found",
2164
- errorCode: "agent_not_found",
2165
- finishedAt: new Date(),
2166
- });
2167
- await setWakeupStatus(run.wakeupRequestId, "failed", {
2168
- finishedAt: new Date(),
2169
- error: "Agent not found",
2170
- });
2171
- const failedRun = await getRun(runId);
2172
- if (failedRun)
2173
- await releaseIssueExecutionAndPromote(failedRun);
2174
- return;
2175
- }
2176
- const heartbeatObservationContext = buildHeartbeatObservabilityContext(run, {
2177
- runtime: agent.agentRuntimeType,
2178
- metadata: {
2179
- agentName: agent.name,
2180
- invocationSource: run.invocationSource,
2181
- triggerDetail: run.triggerDetail,
2182
- },
2183
- });
2184
- await withExecutionObservation(heartbeatObservationContext, {
2185
- name: buildHeartbeatObservationName(run, agent.name),
2186
- asType: "agent",
2187
- input: {
2188
- agentId: agent.id,
2189
- agentName: agent.name,
2190
- invocationSource: run.invocationSource,
2191
- triggerDetail: run.triggerDetail,
2192
- issueId: readNonEmptyString(parseObject(run.contextSnapshot).issueId),
2193
- },
2194
- }, async (observation) => {
2195
- const executionTranscript = [];
2196
- let stdoutTranscriptBuffer = "";
2197
- let stderrTranscriptBuffer = "";
2198
- let stdoutTranscriptParser = null;
2199
- let transcriptFallbackResult = null;
2200
- let finalObservationOutput = null;
2201
- let finalObservationStatus = run.status;
2202
- let finalObservationSessionId = heartbeatObservationContext.sessionKey ?? null;
2203
- const runtime = await ensureRuntimeState(agent);
2204
- const context = parseObject(run.contextSnapshot);
2205
- const taskKey = deriveTaskKey(context, null);
2206
- const sessionCodec = getAgentRuntimeSessionCodec(agent.agentRuntimeType);
2207
- const issueId = readNonEmptyString(context.issueId);
2208
- const issueContext = issueId
2209
- ? await db
2210
- .select({
2211
- id: issues.id,
2212
- identifier: issues.identifier,
2213
- title: issues.title,
2214
- projectId: issues.projectId,
2215
- projectWorkspaceId: issues.projectWorkspaceId,
2216
- executionWorkspaceId: issues.executionWorkspaceId,
2217
- executionWorkspacePreference: issues.executionWorkspacePreference,
2218
- assigneeAgentId: issues.assigneeAgentId,
2219
- assigneeAgentRuntimeOverrides: issues.assigneeAgentRuntimeOverrides,
2220
- executionWorkspaceSettings: issues.executionWorkspaceSettings,
2221
- })
2222
- .from(issues)
2223
- .where(and(eq(issues.id, issueId), eq(issues.orgId, agent.orgId)))
2224
- .then((rows) => rows[0] ?? null)
2225
- : null;
2226
- const issueAssigneeOverrides = issueContext && issueContext.assigneeAgentId === agent.id
2227
- ? parseIssueAssigneeAgentRuntimeOverrides(issueContext.assigneeAgentRuntimeOverrides)
2228
- : null;
2229
- const issueExecutionWorkspaceSettings = parseIssueExecutionWorkspaceSettings(issueContext?.executionWorkspaceSettings);
2230
- const contextProjectId = readNonEmptyString(context.projectId);
2231
- const executionProjectId = issueContext?.projectId ?? contextProjectId;
2232
- const projectExecutionWorkspacePolicy = executionProjectId
2233
- ? await db
2234
- .select({ executionWorkspacePolicy: projects.executionWorkspacePolicy })
2235
- .from(projects)
2236
- .where(and(eq(projects.id, executionProjectId), eq(projects.orgId, agent.orgId)))
2237
- .then((rows) => parseProjectExecutionWorkspacePolicy(rows[0]?.executionWorkspacePolicy))
2238
- : null;
2239
- const taskSession = taskKey
2240
- ? await getTaskSession(agent.orgId, agent.id, agent.agentRuntimeType, taskKey)
2241
- : null;
2242
- const resetTaskSession = shouldResetTaskSessionForWake(context);
2243
- const sessionResetReason = describeSessionResetReason(context);
2244
- const taskSessionForRun = resetTaskSession ? null : taskSession;
2245
- const explicitResumeSessionParams = normalizeSessionParams(sessionCodec.deserialize(parseObject(context.resumeSessionParams)));
2246
- const explicitResumeSessionDisplayId = truncateDisplayId(readNonEmptyString(context.resumeSessionDisplayId) ??
2247
- (sessionCodec.getDisplayId ? sessionCodec.getDisplayId(explicitResumeSessionParams) : null) ??
2248
- readNonEmptyString(explicitResumeSessionParams?.sessionId));
2249
- const previousSessionParams = explicitResumeSessionParams ??
2250
- (explicitResumeSessionDisplayId ? { sessionId: explicitResumeSessionDisplayId } : null) ??
2251
- normalizeSessionParams(sessionCodec.deserialize(taskSessionForRun?.sessionParamsJson ?? null));
2252
- const config = parseObject(agent.agentRuntimeConfig);
2253
- const executionWorkspaceMode = resolveExecutionWorkspaceMode({
2254
- projectPolicy: projectExecutionWorkspacePolicy,
2255
- issueSettings: issueExecutionWorkspaceSettings,
2256
- legacyUseProjectWorkspace: issueAssigneeOverrides?.useProjectWorkspace ?? null,
2257
- });
2258
- const resolvedWorkspace = await runContextSvc.resolveWorkspaceForRun(agent, context, previousSessionParams, { useProjectWorkspace: executionWorkspaceMode !== "agent_default" });
2259
- const workspaceManagedConfig = buildExecutionWorkspaceAdapterConfig({
2260
- agentConfig: config,
2261
- projectPolicy: projectExecutionWorkspacePolicy,
2262
- issueSettings: issueExecutionWorkspaceSettings,
2263
- mode: executionWorkspaceMode,
2264
- legacyUseProjectWorkspace: issueAssigneeOverrides?.useProjectWorkspace ?? null,
2265
- });
2266
- const mergedConfig = issueAssigneeOverrides?.agentRuntimeConfig
2267
- ? { ...workspaceManagedConfig, ...issueAssigneeOverrides.agentRuntimeConfig }
2268
- : workspaceManagedConfig;
2269
- const { resolvedConfig, runtimeConfig, runtimeSkillEntries, secretKeys } = await runContextSvc.prepareRuntimeConfig({
2270
- scene: "heartbeat",
2271
- agent,
2272
- baseConfig: mergedConfig,
2273
- });
2274
- heartbeatObservationContext.metadata = {
2275
- ...(heartbeatObservationContext.metadata ?? {}),
2276
- ...buildHeartbeatRuntimeTraceMetadata({
2277
- runtimeConfig,
2278
- runtimeSkills: runtimeSkillEntries,
2279
- }),
2280
- };
2281
- const issueRef = issueContext
2282
- ? {
2283
- id: issueContext.id,
2284
- identifier: issueContext.identifier,
2285
- title: issueContext.title,
2286
- projectId: issueContext.projectId,
2287
- projectWorkspaceId: issueContext.projectWorkspaceId,
2288
- executionWorkspaceId: issueContext.executionWorkspaceId,
2289
- executionWorkspacePreference: issueContext.executionWorkspacePreference,
2290
- }
2291
- : null;
2292
- const rootObservationInput = {
2293
- agentId: agent.id,
2294
- agentName: agent.name,
2295
- invocationSource: run.invocationSource,
2296
- triggerDetail: run.triggerDetail,
2297
- issue: issueRef
2298
- ? {
2299
- id: issueRef.id,
2300
- identifier: issueRef.identifier ?? null,
2301
- title: issueRef.title ?? null,
2302
- }
2303
- : null,
2304
- };
2305
- updateExecutionObservation(observation, heartbeatObservationContext, {
2306
- input: rootObservationInput,
2307
- });
2308
- updateExecutionTraceIO(observation, { input: rootObservationInput });
2309
- if (issueRef) {
2310
- updateExecutionTraceName(observation, buildIssueRunTraceName({
2311
- issueTitle: issueRef.title,
2312
- issueId: issueRef.id,
2313
- }));
2314
- }
2315
- const existingExecutionWorkspace = issueRef?.executionWorkspaceId ? await executionWorkspacesSvc.getById(issueRef.executionWorkspaceId) : null;
2316
- const workspaceOperationRecorder = workspaceOperationsSvc.createRecorder({
2317
- orgId: agent.orgId,
2318
- heartbeatRunId: run.id,
2319
- executionWorkspaceId: existingExecutionWorkspace?.id ?? null,
2320
- });
2321
- const executionWorkspace = await realizeExecutionWorkspace({
2322
- base: {
2323
- baseCwd: resolvedWorkspace.cwd,
2324
- source: resolvedWorkspace.source,
2325
- projectId: resolvedWorkspace.projectId,
2326
- workspaceId: resolvedWorkspace.workspaceId,
2327
- repoUrl: resolvedWorkspace.repoUrl,
2328
- repoRef: resolvedWorkspace.repoRef,
2329
- },
2330
- config: runtimeConfig,
2331
- issue: issueRef,
2332
- agent: {
2333
- id: agent.id,
2334
- name: agent.name,
2335
- orgId: agent.orgId,
2336
- },
2337
- recorder: workspaceOperationRecorder,
2338
- });
2339
- const resolvedProjectId = executionWorkspace.projectId ?? issueRef?.projectId ?? executionProjectId ?? null;
2340
- const resolvedProjectWorkspaceId = issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null;
2341
- const shouldReuseExisting = issueRef?.executionWorkspacePreference === "reuse_existing" &&
2342
- existingExecutionWorkspace &&
2343
- existingExecutionWorkspace.status !== "archived";
2344
- let persistedExecutionWorkspace = null;
2345
- try {
2346
- persistedExecutionWorkspace = shouldReuseExisting && existingExecutionWorkspace
2347
- ? await executionWorkspacesSvc.update(existingExecutionWorkspace.id, {
2348
- cwd: executionWorkspace.cwd,
2349
- repoUrl: executionWorkspace.repoUrl,
2350
- baseRef: executionWorkspace.repoRef,
2351
- branchName: executionWorkspace.branchName,
2352
- providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs",
2353
- providerRef: executionWorkspace.worktreePath,
2354
- status: "active",
2355
- lastUsedAt: new Date(),
2356
- metadata: {
2357
- ...(existingExecutionWorkspace.metadata ?? {}),
2358
- source: executionWorkspace.source,
2359
- createdByRuntime: executionWorkspace.created,
2360
- },
2361
- })
2362
- : resolvedProjectId
2363
- ? await executionWorkspacesSvc.create({
2364
- orgId: agent.orgId,
2365
- projectId: resolvedProjectId,
2366
- projectWorkspaceId: resolvedProjectWorkspaceId,
2367
- sourceIssueId: issueRef?.id ?? null,
2368
- mode: executionWorkspaceMode === "isolated_workspace"
2369
- ? "isolated_workspace"
2370
- : executionWorkspaceMode === "operator_branch"
2371
- ? "operator_branch"
2372
- : executionWorkspaceMode === "agent_default"
2373
- ? "adapter_managed"
2374
- : "shared_workspace",
2375
- strategyType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "project_primary",
2376
- name: executionWorkspace.branchName ?? issueRef?.identifier ?? `workspace-${agent.id.slice(0, 8)}`,
2377
- status: "active",
2378
- cwd: executionWorkspace.cwd,
2379
- repoUrl: executionWorkspace.repoUrl,
2380
- baseRef: executionWorkspace.repoRef,
2381
- branchName: executionWorkspace.branchName,
2382
- providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs",
2383
- providerRef: executionWorkspace.worktreePath,
2384
- lastUsedAt: new Date(),
2385
- openedAt: new Date(),
2386
- metadata: {
2387
- source: executionWorkspace.source,
2388
- createdByRuntime: executionWorkspace.created,
2389
- },
2390
- })
2391
- : null;
2392
- }
2393
- catch (error) {
2394
- if (executionWorkspace.created) {
2395
- try {
2396
- await cleanupExecutionWorkspaceArtifacts({
2397
- workspace: {
2398
- id: existingExecutionWorkspace?.id ?? `transient-${run.id}`,
2399
- cwd: executionWorkspace.cwd,
2400
- providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs",
2401
- providerRef: executionWorkspace.worktreePath,
2402
- branchName: executionWorkspace.branchName,
2403
- repoUrl: executionWorkspace.repoUrl,
2404
- baseRef: executionWorkspace.repoRef,
2405
- projectId: resolvedProjectId,
2406
- projectWorkspaceId: resolvedProjectWorkspaceId,
2407
- sourceIssueId: issueRef?.id ?? null,
2408
- metadata: {
2409
- createdByRuntime: true,
2410
- source: executionWorkspace.source,
2411
- },
2412
- },
2413
- projectWorkspace: {
2414
- cwd: resolvedWorkspace.cwd,
2415
- cleanupCommand: null,
2416
- },
2417
- teardownCommand: projectExecutionWorkspacePolicy?.workspaceStrategy?.teardownCommand ?? null,
2418
- recorder: workspaceOperationRecorder,
2419
- });
2420
- }
2421
- catch (cleanupError) {
2422
- logger.warn({
2423
- runId: run.id,
2424
- issueId,
2425
- executionWorkspaceCwd: executionWorkspace.cwd,
2426
- cleanupError: cleanupError instanceof Error ? cleanupError.message : String(cleanupError),
2427
- }, "Failed to cleanup realized execution workspace after persistence failure");
2428
- }
2429
- }
2430
- throw error;
2431
- }
2432
- await workspaceOperationRecorder.attachExecutionWorkspaceId(persistedExecutionWorkspace?.id ?? null);
2433
- if (existingExecutionWorkspace &&
2434
- persistedExecutionWorkspace &&
2435
- existingExecutionWorkspace.id !== persistedExecutionWorkspace.id &&
2436
- existingExecutionWorkspace.status === "active") {
2437
- await executionWorkspacesSvc.update(existingExecutionWorkspace.id, {
2438
- status: "idle",
2439
- cleanupReason: null,
2440
- });
2441
- }
2442
- if (issueId && persistedExecutionWorkspace) {
2443
- const nextIssueWorkspaceMode = issueExecutionWorkspaceModeForPersistedWorkspace(persistedExecutionWorkspace.mode);
2444
- const shouldSwitchIssueToExistingWorkspace = issueRef?.executionWorkspacePreference === "reuse_existing" ||
2445
- executionWorkspaceMode === "isolated_workspace" ||
2446
- executionWorkspaceMode === "operator_branch";
2447
- const nextIssuePatch = {};
2448
- if (issueRef?.executionWorkspaceId !== persistedExecutionWorkspace.id) {
2449
- nextIssuePatch.executionWorkspaceId = persistedExecutionWorkspace.id;
2450
- }
2451
- if (resolvedProjectWorkspaceId && issueRef?.projectWorkspaceId !== resolvedProjectWorkspaceId) {
2452
- nextIssuePatch.projectWorkspaceId = resolvedProjectWorkspaceId;
2453
- }
2454
- if (shouldSwitchIssueToExistingWorkspace) {
2455
- nextIssuePatch.executionWorkspacePreference = "reuse_existing";
2456
- nextIssuePatch.executionWorkspaceSettings = {
2457
- ...(issueExecutionWorkspaceSettings ?? {}),
2458
- mode: nextIssueWorkspaceMode,
2459
- };
2460
- }
2461
- if (Object.keys(nextIssuePatch).length > 0) {
2462
- await issuesSvc.update(issueId, nextIssuePatch);
2463
- }
2464
- }
2465
- if (persistedExecutionWorkspace) {
2466
- context.executionWorkspaceId = persistedExecutionWorkspace.id;
2467
- await db
2468
- .update(heartbeatRuns)
2469
- .set({
2470
- contextSnapshot: context,
2471
- updatedAt: new Date(),
2472
- })
2473
- .where(eq(heartbeatRuns.id, run.id));
2474
- }
2475
- const runtimeSessionResolution = resolveRuntimeSessionParamsForWorkspace({
2476
- orgId: agent.orgId,
2477
- agent,
2478
- previousSessionParams,
2479
- resolvedWorkspace: {
2480
- ...resolvedWorkspace,
2481
- cwd: resolveDefaultAgentWorkspaceDir(agent.orgId, agent),
2482
- source: "agent_home",
2483
- },
2484
- });
2485
- const runtimeSessionParams = runtimeSessionResolution.sessionParams;
2486
- const runtimeWorkspaceWarnings = [
2487
- ...resolvedWorkspace.warnings,
2488
- ...executionWorkspace.warnings,
2489
- ...(runtimeSessionResolution.warning ? [runtimeSessionResolution.warning] : []),
2490
- ...(resetTaskSession && sessionResetReason
2491
- ? [
2492
- taskKey
2493
- ? `Skipping saved session resume for task "${taskKey}" because ${sessionResetReason}.`
2494
- : `Skipping saved session resume because ${sessionResetReason}.`,
2495
- ]
2496
- : []),
2497
- ];
2498
- const runtimeSceneContext = await runContextSvc.buildSceneContext({
2499
- scene: "heartbeat",
2500
- agent,
2501
- resolvedWorkspace,
2502
- runtimeConfig,
2503
- executionWorkspaceMode,
2504
- executionWorkspace: {
2505
- cwd: executionWorkspace.cwd,
2506
- source: executionWorkspace.source,
2507
- strategy: executionWorkspace.strategy,
2508
- projectId: executionWorkspace.projectId,
2509
- workspaceId: executionWorkspace.workspaceId,
2510
- repoUrl: executionWorkspace.repoUrl,
2511
- repoRef: executionWorkspace.repoRef,
2512
- branchName: executionWorkspace.branchName,
2513
- worktreePath: executionWorkspace.worktreePath,
2514
- },
2515
- });
2516
- context.rudderScene = runtimeSceneContext.rudderScene;
2517
- context.rudderWorkspace = runtimeSceneContext.rudderWorkspace;
2518
- context.rudderWorkspaces = runtimeSceneContext.rudderWorkspaces;
2519
- if (runtimeSceneContext.rudderRuntimeServiceIntents) {
2520
- context.rudderRuntimeServiceIntents = runtimeSceneContext.rudderRuntimeServiceIntents;
2521
- }
2522
- else {
2523
- delete context.rudderRuntimeServiceIntents;
2524
- }
2525
- if (executionWorkspace.projectId && !readNonEmptyString(context.projectId)) {
2526
- context.projectId = executionWorkspace.projectId;
2527
- }
2528
- const runtimeSessionFallback = taskKey || resetTaskSession ? null : runtime.sessionId;
2529
- let previousSessionDisplayId = truncateDisplayId(explicitResumeSessionDisplayId ??
2530
- taskSessionForRun?.sessionDisplayId ??
2531
- (sessionCodec.getDisplayId ? sessionCodec.getDisplayId(runtimeSessionParams) : null) ??
2532
- readNonEmptyString(runtimeSessionParams?.sessionId) ??
2533
- runtimeSessionFallback);
2534
- let runtimeSessionIdForAdapter = readNonEmptyString(runtimeSessionParams?.sessionId) ?? runtimeSessionFallback;
2535
- let runtimeSessionParamsForAdapter = runtimeSessionParams;
2536
- const sessionCompaction = await evaluateSessionCompaction({
2537
- agent,
2538
- sessionId: previousSessionDisplayId ?? runtimeSessionIdForAdapter,
2539
- issueId,
2540
- });
2541
- if (sessionCompaction.rotate) {
2542
- context.rudderSessionHandoffMarkdown = sessionCompaction.handoffMarkdown;
2543
- context.rudderSessionRotationReason = sessionCompaction.reason;
2544
- context.rudderPreviousSessionId = previousSessionDisplayId ?? runtimeSessionIdForAdapter;
2545
- runtimeSessionIdForAdapter = null;
2546
- runtimeSessionParamsForAdapter = null;
2547
- previousSessionDisplayId = null;
2548
- if (sessionCompaction.reason) {
2549
- runtimeWorkspaceWarnings.push(`Starting a fresh session because ${sessionCompaction.reason}.`);
2550
- }
2551
- }
2552
- else {
2553
- delete context.rudderSessionHandoffMarkdown;
2554
- delete context.rudderSessionRotationReason;
2555
- delete context.rudderPreviousSessionId;
2556
- }
2557
- const runtimeForAdapter = {
2558
- sessionId: runtimeSessionIdForAdapter,
2559
- sessionParams: runtimeSessionParamsForAdapter,
2560
- sessionDisplayId: previousSessionDisplayId,
2561
- taskKey,
2562
- };
2563
- let seq = 1;
2564
- let handle = null;
2565
- let stdoutExcerpt = "";
2566
- let stderrExcerpt = "";
2567
- try {
2568
- const startedAt = run.startedAt ?? new Date();
2569
- const runningWithSession = await db
2570
- .update(heartbeatRuns)
2571
- .set({
2572
- startedAt,
2573
- sessionIdBefore: runtimeForAdapter.sessionDisplayId ?? runtimeForAdapter.sessionId,
2574
- contextSnapshot: context,
2575
- updatedAt: new Date(),
2576
- })
2577
- .where(eq(heartbeatRuns.id, run.id))
2578
- .returning()
2579
- .then((rows) => rows[0] ?? null);
2580
- if (runningWithSession)
2581
- run = runningWithSession;
2582
- const runningAgent = await db
2583
- .update(agents)
2584
- .set({ status: "running", updatedAt: new Date() })
2585
- .where(eq(agents.id, agent.id))
2586
- .returning()
2587
- .then((rows) => rows[0] ?? null);
2588
- if (runningAgent) {
2589
- publishLiveEvent({
2590
- orgId: runningAgent.orgId,
2591
- type: "agent.status",
2592
- payload: {
2593
- agentId: runningAgent.id,
2594
- status: runningAgent.status,
2595
- outcome: "running",
2596
- },
2597
- });
2598
- }
2599
- const currentRun = run;
2600
- await appendRunEvent(currentRun, seq++, {
2601
- eventType: "lifecycle",
2602
- stream: "system",
2603
- level: "info",
2604
- message: "run started",
2605
- });
2606
- handle = await runLogStore.begin({
2607
- orgId: run.orgId,
2608
- agentId: run.agentId,
2609
- runId,
2610
- });
2611
- await db
2612
- .update(heartbeatRuns)
2613
- .set({
2614
- logStore: handle.store,
2615
- logRef: handle.logRef,
2616
- updatedAt: new Date(),
2617
- })
2618
- .where(eq(heartbeatRuns.id, runId));
2619
- const adapter = getServerAdapter(agent.agentRuntimeType);
2620
- stdoutTranscriptParser = adapter.parseStdoutLine ?? null;
2621
- const currentUserRedactionOptions = await getCurrentUserRedactionOptions();
2622
- const onLog = async (stream, chunk) => {
2623
- const sanitizedChunk = redactCurrentUserText(chunk, currentUserRedactionOptions);
2624
- if (stream === "stdout")
2625
- stdoutExcerpt = appendExcerpt(stdoutExcerpt, sanitizedChunk);
2626
- if (stream === "stderr")
2627
- stderrExcerpt = appendExcerpt(stderrExcerpt, sanitizedChunk);
2628
- const ts = new Date().toISOString();
2629
- if (handle) {
2630
- await runLogStore.append(handle, {
2631
- stream,
2632
- chunk: sanitizedChunk,
2633
- ts,
2634
- });
2635
- }
2636
- const payloadChunk = sanitizedChunk.length > MAX_LIVE_LOG_CHUNK_BYTES
2637
- ? sanitizedChunk.slice(sanitizedChunk.length - MAX_LIVE_LOG_CHUNK_BYTES)
2638
- : sanitizedChunk;
2639
- publishLiveEvent({
2640
- orgId: run.orgId,
2641
- type: "heartbeat.run.log",
2642
- payload: {
2643
- runId: run.id,
2644
- agentId: run.agentId,
2645
- ts,
2646
- stream,
2647
- chunk: payloadChunk,
2648
- truncated: payloadChunk.length !== sanitizedChunk.length,
2649
- },
2650
- });
2651
- if (stream === "stdout") {
2652
- stdoutTranscriptBuffer = appendTranscriptEntriesFromChunk({
2653
- buffer: stdoutTranscriptBuffer,
2654
- chunk: sanitizedChunk,
2655
- transcript: executionTranscript,
2656
- parser: stdoutTranscriptParser,
2657
- kind: "stdout",
2658
- });
2659
- return;
2660
- }
2661
- stderrTranscriptBuffer = appendTranscriptEntriesFromChunk({
2662
- buffer: stderrTranscriptBuffer,
2663
- chunk: sanitizedChunk,
2664
- transcript: executionTranscript,
2665
- kind: "stderr",
2666
- });
2667
- };
2668
- for (const warning of runtimeWorkspaceWarnings) {
2669
- const logEntry = formatRuntimeWorkspaceWarningLog(warning);
2670
- await onLog(logEntry.stream, logEntry.chunk);
2671
- }
2672
- const adapterEnv = Object.fromEntries(Object.entries(parseObject(resolvedConfig.env)).filter((entry) => typeof entry[0] === "string" && typeof entry[1] === "string"));
2673
- const runtimeServices = await ensureRuntimeServicesForRun({
2674
- db,
2675
- runId: run.id,
2676
- agent: {
2677
- id: agent.id,
2678
- name: agent.name,
2679
- orgId: agent.orgId,
2680
- },
2681
- issue: issueRef,
2682
- workspace: executionWorkspace,
2683
- executionWorkspaceId: persistedExecutionWorkspace?.id ?? issueRef?.executionWorkspaceId ?? null,
2684
- config: resolvedConfig,
2685
- adapterEnv,
2686
- onLog,
2687
- });
2688
- if (runtimeServices.length > 0) {
2689
- context.rudderRuntimeServices = runtimeServices;
2690
- context.rudderRuntimePrimaryUrl =
2691
- runtimeServices.find((service) => readNonEmptyString(service.url))?.url ?? null;
2692
- await db
2693
- .update(heartbeatRuns)
2694
- .set({
2695
- contextSnapshot: context,
2696
- updatedAt: new Date(),
2697
- })
2698
- .where(eq(heartbeatRuns.id, run.id));
2699
- }
2700
- if (issueId && (executionWorkspace.created || runtimeServices.some((service) => !service.reused))) {
2701
- try {
2702
- await issuesSvc.addComment(issueId, buildWorkspaceReadyComment({
2703
- workspace: executionWorkspace,
2704
- runtimeServices,
2705
- }), { agentId: agent.id });
2706
- }
2707
- catch (err) {
2708
- await onLog("stderr", `[rudder] Failed to post workspace-ready comment: ${err instanceof Error ? err.message : String(err)}\n`);
2709
- }
2710
- }
2711
- const onAdapterMeta = async (meta) => {
2712
- if (meta.env && secretKeys.size > 0) {
2713
- for (const key of secretKeys) {
2714
- if (key in meta.env)
2715
- meta.env[key] = "***REDACTED***";
2716
- }
2717
- }
2718
- heartbeatObservationContext.metadata = {
2719
- ...(heartbeatObservationContext.metadata ?? {}),
2720
- ...buildHeartbeatRuntimeTraceMetadata({
2721
- runtimeConfig,
2722
- runtimeSkills: runtimeSkillEntries,
2723
- adapterMeta: meta,
2724
- }),
2725
- };
2726
- updateExecutionObservation(observation, heartbeatObservationContext, {
2727
- input: rootObservationInput,
2728
- });
2729
- await appendRunEvent(currentRun, seq++, {
2730
- eventType: "adapter.invoke",
2731
- stream: "system",
2732
- level: "info",
2733
- message: "adapter invocation",
2734
- payload: buildHeartbeatAdapterInvokePayload({
2735
- meta,
2736
- runtimeSkills: runtimeSkillEntries,
2737
- }),
2738
- });
2739
- };
2740
- const authToken = adapter.supportsLocalAgentJwt
2741
- ? createLocalAgentJwt(agent.id, agent.orgId, agent.agentRuntimeType, run.id)
2742
- : null;
2743
- if (adapter.supportsLocalAgentJwt && !authToken) {
2744
- logger.warn({
2745
- orgId: agent.orgId,
2746
- agentId: agent.id,
2747
- runId: run.id,
2748
- agentRuntimeType: agent.agentRuntimeType,
2749
- }, "local agent jwt secret missing or invalid; running without injected RUDDER_API_KEY");
2750
- }
2751
- const adapterResult = await executeAdapterWithModelFallbacks(adapter, {
2752
- runId: run.id,
2753
- agent,
2754
- runtime: runtimeForAdapter,
2755
- config: runtimeConfig,
2756
- context,
2757
- onLog,
2758
- onMeta: onAdapterMeta,
2759
- onSpawn: async (meta) => {
2760
- await persistRunProcessMetadata(run.id, meta);
2761
- },
2762
- authToken: authToken ?? undefined,
2763
- }, {
2764
- resolveAdapter: findServerAdapter,
2765
- createAuthToken: (agentRuntimeType) => createLocalAgentJwt(agent.id, agent.orgId, agentRuntimeType, run.id) ?? undefined,
2766
- onAttemptStart: (_attempt, attemptAdapter) => {
2767
- stdoutTranscriptParser = attemptAdapter.parseStdoutLine ?? null;
2768
- },
2769
- });
2770
- const adapterManagedRuntimeServices = adapterResult.runtimeServices
2771
- ? await persistAdapterManagedRuntimeServices({
2772
- db,
2773
- agentRuntimeType: agent.agentRuntimeType,
2774
- runId: run.id,
2775
- agent: {
2776
- id: agent.id,
2777
- name: agent.name,
2778
- orgId: agent.orgId,
2779
- },
2780
- issue: issueRef,
2781
- workspace: executionWorkspace,
2782
- reports: adapterResult.runtimeServices,
2783
- })
2784
- : [];
2785
- if (adapterManagedRuntimeServices.length > 0) {
2786
- const combinedRuntimeServices = [
2787
- ...runtimeServices,
2788
- ...adapterManagedRuntimeServices,
2789
- ];
2790
- context.rudderRuntimeServices = combinedRuntimeServices;
2791
- context.rudderRuntimePrimaryUrl =
2792
- combinedRuntimeServices.find((service) => readNonEmptyString(service.url))?.url ?? null;
2793
- await db
2794
- .update(heartbeatRuns)
2795
- .set({
2796
- contextSnapshot: context,
2797
- updatedAt: new Date(),
2798
- })
2799
- .where(eq(heartbeatRuns.id, run.id));
2800
- if (issueId) {
2801
- try {
2802
- await issuesSvc.addComment(issueId, buildWorkspaceReadyComment({
2803
- workspace: executionWorkspace,
2804
- runtimeServices: adapterManagedRuntimeServices,
2805
- }), { agentId: agent.id });
2806
- }
2807
- catch (err) {
2808
- await onLog("stderr", `[rudder] Failed to post adapter-managed runtime comment: ${err instanceof Error ? err.message : String(err)}\n`);
2809
- }
2810
- }
2811
- }
2812
- const nextSessionState = resolveNextSessionState({
2813
- codec: sessionCodec,
2814
- adapterResult,
2815
- previousParams: previousSessionParams,
2816
- previousDisplayId: runtimeForAdapter.sessionDisplayId,
2817
- previousLegacySessionId: runtimeForAdapter.sessionId,
2818
- });
2819
- const rawUsage = normalizeUsageTotals(adapterResult.usage);
2820
- const sessionUsageResolution = await resolveNormalizedUsageForSession({
2821
- agentId: agent.id,
2822
- runId: run.id,
2823
- sessionId: nextSessionState.displayId ?? nextSessionState.legacySessionId,
2824
- rawUsage,
2825
- });
2826
- const normalizedUsage = sessionUsageResolution.normalizedUsage;
2827
- let outcome;
2828
- const latestRun = await getRun(run.id);
2829
- if (latestRun?.status === "cancelled") {
2830
- outcome = "cancelled";
2831
- }
2832
- else if (adapterResult.timedOut) {
2833
- outcome = "timed_out";
2834
- }
2835
- else if ((adapterResult.exitCode ?? 0) === 0 && !adapterResult.errorMessage) {
2836
- outcome = "succeeded";
2837
- }
2838
- else {
2839
- outcome = "failed";
2840
- }
2841
- let logSummary = null;
2842
- if (handle) {
2843
- logSummary = await runLogStore.finalize(handle);
2844
- }
2845
- const status = outcome === "succeeded"
2846
- ? "succeeded"
2847
- : outcome === "cancelled"
2848
- ? "cancelled"
2849
- : outcome === "timed_out"
2850
- ? "timed_out"
2851
- : "failed";
2852
- heartbeatObservationContext.status = status;
2853
- finalObservationStatus = status;
2854
- finalObservationSessionId = nextSessionState.displayId ?? nextSessionState.legacySessionId ?? finalObservationSessionId;
2855
- const adapterResultSummary = summarizeHeartbeatRunResultJson(adapterResult.resultJson);
2856
- transcriptFallbackResult = {
2857
- ts: new Date().toISOString(),
2858
- model: readNonEmptyString(adapterResult.model),
2859
- output: readNonEmptyString(adapterResult.summary)
2860
- ?? readNonEmptyString(adapterResultSummary?.result)
2861
- ?? readNonEmptyString(adapterResultSummary?.summary)
2862
- ?? readNonEmptyString(adapterResultSummary?.message)
2863
- ?? null,
2864
- usage: adapterResult.usage ?? null,
2865
- costUsd: typeof adapterResult.costUsd === "number" ? adapterResult.costUsd : null,
2866
- subtype: status,
2867
- isError: outcome !== "succeeded",
2868
- errors: adapterResult.errorMessage ? [adapterResult.errorMessage] : [],
2869
- };
2870
- const usageJson = normalizedUsage || adapterResult.costUsd != null
2871
- ? {
2872
- ...(normalizedUsage ?? {}),
2873
- ...(rawUsage ? {
2874
- rawInputTokens: rawUsage.inputTokens,
2875
- rawCachedInputTokens: rawUsage.cachedInputTokens,
2876
- rawOutputTokens: rawUsage.outputTokens,
2877
- } : {}),
2878
- ...(sessionUsageResolution.derivedFromSessionTotals ? { usageSource: "session_delta" } : {}),
2879
- ...((nextSessionState.displayId ?? nextSessionState.legacySessionId)
2880
- ? { persistedSessionId: nextSessionState.displayId ?? nextSessionState.legacySessionId }
2881
- : {}),
2882
- sessionReused: runtimeForAdapter.sessionId != null || runtimeForAdapter.sessionDisplayId != null,
2883
- taskSessionReused: taskSessionForRun != null,
2884
- freshSession: runtimeForAdapter.sessionId == null && runtimeForAdapter.sessionDisplayId == null,
2885
- sessionRotated: sessionCompaction.rotate,
2886
- sessionRotationReason: sessionCompaction.reason,
2887
- provider: readNonEmptyString(adapterResult.provider) ?? "unknown",
2888
- biller: resolveLedgerBiller(adapterResult),
2889
- model: readNonEmptyString(adapterResult.model) ?? "unknown",
2890
- ...(adapterResult.costUsd != null ? { costUsd: adapterResult.costUsd } : {}),
2891
- billingType: normalizeLedgerBillingType(adapterResult.billingType),
2892
- }
2893
- : null;
2894
- await setRunStatus(run.id, status, {
2895
- finishedAt: new Date(),
2896
- error: outcome === "succeeded"
2897
- ? null
2898
- : redactCurrentUserText(adapterResult.errorMessage ?? (outcome === "timed_out" ? "Timed out" : "Adapter failed"), currentUserRedactionOptions),
2899
- errorCode: outcome === "timed_out"
2900
- ? "timeout"
2901
- : outcome === "cancelled"
2902
- ? "cancelled"
2903
- : outcome === "failed"
2904
- ? (adapterResult.errorCode ?? "adapter_failed")
2905
- : null,
2906
- exitCode: adapterResult.exitCode,
2907
- signal: adapterResult.signal,
2908
- usageJson,
2909
- resultJson: adapterResult.resultJson ?? null,
2910
- sessionIdAfter: nextSessionState.displayId ?? nextSessionState.legacySessionId,
2911
- stdoutExcerpt,
2912
- stderrExcerpt,
2913
- logBytes: logSummary?.bytes,
2914
- logSha256: logSummary?.sha256,
2915
- logCompressed: logSummary?.compressed ?? false,
2916
- });
2917
- await setWakeupStatus(run.wakeupRequestId, outcome === "succeeded" ? "completed" : status, {
2918
- finishedAt: new Date(),
2919
- error: adapterResult.errorMessage ?? null,
2920
- });
2921
- const finalizedRun = await getRun(run.id);
2922
- if (finalizedRun) {
2923
- await appendRunEvent(finalizedRun, seq++, {
2924
- eventType: "lifecycle",
2925
- stream: "system",
2926
- level: outcome === "succeeded" ? "info" : "error",
2927
- message: `run ${outcome}`,
2928
- payload: {
2929
- status,
2930
- exitCode: adapterResult.exitCode,
2931
- },
2932
- });
2933
- await releaseIssueExecutionAndPromote(finalizedRun);
2934
- }
2935
- if (finalizedRun) {
2936
- await updateRuntimeState(agent, finalizedRun, adapterResult, {
2937
- legacySessionId: nextSessionState.legacySessionId,
2938
- }, normalizedUsage);
2939
- if (taskKey) {
2940
- if (adapterResult.clearSession || (!nextSessionState.params && !nextSessionState.displayId)) {
2941
- await clearTaskSessions(agent.orgId, agent.id, {
2942
- taskKey,
2943
- agentRuntimeType: agent.agentRuntimeType,
2944
- });
2945
- }
2946
- else {
2947
- await upsertTaskSession({
2948
- orgId: agent.orgId,
2949
- agentId: agent.id,
2950
- agentRuntimeType: agent.agentRuntimeType,
2951
- taskKey,
2952
- sessionParamsJson: nextSessionState.params,
2953
- sessionDisplayId: nextSessionState.displayId,
2954
- lastRunId: finalizedRun.id,
2955
- lastError: outcome === "succeeded" ? null : (adapterResult.errorMessage ?? "run_failed"),
2956
- });
2957
- }
2958
- }
2959
- await emitHeartbeatLiveEval(finalizedRun.id);
2960
- }
2961
- await finalizeAgentStatus(agent.id, outcome);
2962
- }
2963
- catch (err) {
2964
- const message = redactCurrentUserText(err instanceof Error ? err.message : "Unknown adapter failure", await getCurrentUserRedactionOptions());
2965
- heartbeatObservationContext.status = "failed";
2966
- finalObservationStatus = "failed";
2967
- transcriptFallbackResult = {
2968
- ts: new Date().toISOString(),
2969
- output: message,
2970
- subtype: "failed",
2971
- isError: true,
2972
- errors: [message],
2973
- };
2974
- logger.error({ err, runId }, "heartbeat execution failed");
2975
- let logSummary = null;
2976
- if (handle) {
2977
- try {
2978
- logSummary = await runLogStore.finalize(handle);
2979
- }
2980
- catch (finalizeErr) {
2981
- logger.warn({ err: finalizeErr, runId }, "failed to finalize run log after error");
2982
- }
2983
- }
2984
- const failedRun = await setRunStatus(run.id, "failed", {
2985
- error: message,
2986
- errorCode: "adapter_failed",
2987
- finishedAt: new Date(),
2988
- stdoutExcerpt,
2989
- stderrExcerpt,
2990
- logBytes: logSummary?.bytes,
2991
- logSha256: logSummary?.sha256,
2992
- logCompressed: logSummary?.compressed ?? false,
2993
- });
2994
- await setWakeupStatus(run.wakeupRequestId, "failed", {
2995
- finishedAt: new Date(),
2996
- error: message,
2997
- });
2998
- if (failedRun) {
2999
- await appendRunEvent(failedRun, seq++, {
3000
- eventType: "error",
3001
- stream: "system",
3002
- level: "error",
3003
- message,
3004
- });
3005
- await releaseIssueExecutionAndPromote(failedRun);
3006
- await updateRuntimeState(agent, failedRun, {
3007
- exitCode: null,
3008
- signal: null,
3009
- timedOut: false,
3010
- errorMessage: message,
3011
- }, {
3012
- legacySessionId: runtimeForAdapter.sessionId,
3013
- });
3014
- if (taskKey && (previousSessionParams || previousSessionDisplayId || taskSession)) {
3015
- await upsertTaskSession({
3016
- orgId: agent.orgId,
3017
- agentId: agent.id,
3018
- agentRuntimeType: agent.agentRuntimeType,
3019
- taskKey,
3020
- sessionParamsJson: previousSessionParams,
3021
- sessionDisplayId: previousSessionDisplayId,
3022
- lastRunId: failedRun.id,
3023
- lastError: message,
3024
- });
3025
- }
3026
- await emitHeartbeatLiveEval(failedRun.id);
3027
- }
3028
- await finalizeAgentStatus(agent.id, "failed");
3029
- }
3030
- finally {
3031
- stdoutTranscriptBuffer = appendTranscriptEntriesFromChunk({
3032
- buffer: stdoutTranscriptBuffer,
3033
- chunk: "",
3034
- transcript: executionTranscript,
3035
- parser: stdoutTranscriptParser,
3036
- finalize: true,
3037
- kind: "stdout",
3038
- });
3039
- stderrTranscriptBuffer = appendTranscriptEntriesFromChunk({
3040
- buffer: stderrTranscriptBuffer,
3041
- chunk: "",
3042
- transcript: executionTranscript,
3043
- finalize: true,
3044
- kind: "stderr",
3045
- });
3046
- try {
3047
- const transcriptStats = emitExecutionTranscriptTree({
3048
- context: heartbeatObservationContext,
3049
- parentObservation: observation,
3050
- transcript: executionTranscript,
3051
- fallbackResult: transcriptFallbackResult,
3052
- });
3053
- finalObservationOutput = transcriptStats.finalOutput ?? transcriptFallbackResult?.output ?? null;
3054
- finalObservationSessionId = transcriptStats.finalSessionId ?? finalObservationSessionId;
3055
- }
3056
- catch (error) {
3057
- logger.warn({
3058
- runId: run.id,
3059
- err: error instanceof Error ? error.message : String(error),
3060
- }, "Failed to export heartbeat transcript tree to Langfuse");
3061
- }
3062
- updateExecutionObservation(observation, heartbeatObservationContext, {
3063
- input: rootObservationInput,
3064
- output: finalObservationOutput,
3065
- level: finalObservationStatus === "failed" || finalObservationStatus === "timed_out" ? "ERROR" : "DEFAULT",
3066
- statusMessage: finalObservationStatus ?? undefined,
3067
- });
3068
- updateExecutionTraceIO(observation, {
3069
- input: rootObservationInput,
3070
- output: finalObservationOutput,
3071
- });
3072
- updateExecutionTraceSession(observation, finalObservationSessionId);
3073
- }
3074
- });
3075
- }
3076
- catch (outerErr) {
3077
- // Setup code before adapter.execute threw (e.g. ensureRuntimeState, resolveWorkspaceForRun).
3078
- // The inner catch did not fire, so we must record the failure here.
3079
- const message = outerErr instanceof Error ? outerErr.message : "Unknown setup failure";
3080
- logger.error({ err: outerErr, runId }, "heartbeat execution setup failed");
3081
- await setRunStatus(runId, "failed", {
3082
- error: message,
3083
- errorCode: "adapter_failed",
3084
- finishedAt: new Date(),
3085
- }).catch(() => undefined);
3086
- await setWakeupStatus(run.wakeupRequestId, "failed", {
3087
- finishedAt: new Date(),
3088
- error: message,
3089
- }).catch(() => undefined);
3090
- const failedRun = await getRun(runId).catch(() => null);
3091
- if (failedRun) {
3092
- // Emit a run-log event so the failure is visible in the run timeline,
3093
- // consistent with what the inner catch block does for adapter failures.
3094
- await appendRunEvent(failedRun, 1, {
3095
- eventType: "error",
3096
- stream: "system",
3097
- level: "error",
3098
- message,
3099
- }).catch(() => undefined);
3100
- await emitHeartbeatLiveEval(failedRun.id).catch(() => undefined);
3101
- await releaseIssueExecutionAndPromote(failedRun).catch(() => undefined);
3102
- }
3103
- // Ensure the agent is not left stuck in "running" if the inner catch handler's
3104
- // DB calls threw (e.g. a transient DB error in finalizeAgentStatus).
3105
- await finalizeAgentStatus(run.agentId, "failed").catch(() => undefined);
3106
- }
3107
- finally {
3108
- await releaseRuntimeServicesForRun(run.id).catch(() => undefined);
3109
- activeRunExecutions.delete(run.id);
3110
- await startNextQueuedRunForAgent(run.agentId);
3111
- }
3112
- }
3113
- async function releaseIssueExecutionAndPromote(run) {
3114
- const outcome = await db.transaction(async (tx) => {
3115
- await tx.execute(sql `select id from issues where org_id = ${run.orgId} and execution_run_id = ${run.id} for update`);
3116
- const issue = await tx
3117
- .select({
3118
- id: issues.id,
3119
- orgId: issues.orgId,
3120
- title: issues.title,
3121
- description: issues.description,
3122
- status: issues.status,
3123
- priority: issues.priority,
3124
- projectId: issues.projectId,
3125
- assigneeAgentId: issues.assigneeAgentId,
3126
- })
3127
- .from(issues)
3128
- .where(and(eq(issues.orgId, run.orgId), eq(issues.executionRunId, run.id)))
3129
- .then((rows) => rows[0] ?? null);
3130
- if (!issue)
3131
- return { promotedRun: null, passiveClosure: null };
3132
- const passiveClosure = await evaluatePassiveIssueClosureForLockedIssue({
3133
- tx,
3134
- run,
3135
- issue,
3136
- now: new Date(),
3137
- });
3138
- if (passiveClosure.kind === "queued") {
3139
- return { promotedRun: passiveClosure.run, passiveClosure };
3140
- }
3141
- await tx
3142
- .update(issues)
3143
- .set({
3144
- executionRunId: null,
3145
- executionAgentNameKey: null,
3146
- executionLockedAt: null,
3147
- updatedAt: new Date(),
3148
- })
3149
- .where(eq(issues.id, issue.id));
3150
- while (true) {
3151
- const deferred = await tx
3152
- .select()
3153
- .from(agentWakeupRequests)
3154
- .where(and(eq(agentWakeupRequests.orgId, issue.orgId), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql `${agentWakeupRequests.payload} ->> 'issueId' = ${issue.id}`))
3155
- .orderBy(asc(agentWakeupRequests.requestedAt))
3156
- .limit(1)
3157
- .then((rows) => rows[0] ?? null);
3158
- if (!deferred)
3159
- return { promotedRun: null, passiveClosure };
3160
- const deferredAgent = await tx
3161
- .select()
3162
- .from(agents)
3163
- .where(eq(agents.id, deferred.agentId))
3164
- .then((rows) => rows[0] ?? null);
3165
- if (!deferredAgent ||
3166
- deferredAgent.orgId !== issue.orgId ||
3167
- deferredAgent.status === "paused" ||
3168
- deferredAgent.status === "terminated" ||
3169
- deferredAgent.status === "pending_approval") {
3170
- await tx
3171
- .update(agentWakeupRequests)
3172
- .set({
3173
- status: "failed",
3174
- finishedAt: new Date(),
3175
- error: "Deferred wake could not be promoted: agent is not invokable",
3176
- updatedAt: new Date(),
3177
- })
3178
- .where(eq(agentWakeupRequests.id, deferred.id));
3179
- continue;
3180
- }
3181
- const deferredPayload = parseObject(deferred.payload);
3182
- const deferredContextSeed = parseObject(deferredPayload[DEFERRED_WAKE_CONTEXT_KEY]);
3183
- const promotedContextSeed = { ...deferredContextSeed };
3184
- const promotedReason = readNonEmptyString(deferred.reason) ?? "issue_execution_promoted";
3185
- const promotedSource = readNonEmptyString(deferred.source) ?? "automation";
3186
- const promotedTriggerDetail = readNonEmptyString(deferred.triggerDetail) ?? null;
3187
- const promotedPayload = deferredPayload;
3188
- delete promotedPayload[DEFERRED_WAKE_CONTEXT_KEY];
3189
- const { contextSnapshot: promotedContextSnapshot, taskKey: promotedTaskKey, } = enrichWakeContextSnapshot({
3190
- contextSnapshot: promotedContextSeed,
3191
- reason: promotedReason,
3192
- source: promotedSource,
3193
- triggerDetail: promotedTriggerDetail,
3194
- payload: promotedPayload,
3195
- });
3196
- const sessionBefore = readNonEmptyString(promotedContextSnapshot.resumeSessionDisplayId) ??
3197
- await resolveSessionBeforeForWakeup(deferredAgent, promotedTaskKey);
3198
- const now = new Date();
3199
- const newRun = await tx
3200
- .insert(heartbeatRuns)
3201
- .values({
3202
- orgId: deferredAgent.orgId,
3203
- agentId: deferredAgent.id,
3204
- invocationSource: promotedSource,
3205
- triggerDetail: promotedTriggerDetail,
3206
- status: "queued",
3207
- wakeupRequestId: deferred.id,
3208
- contextSnapshot: promotedContextSnapshot,
3209
- sessionIdBefore: sessionBefore,
3210
- })
3211
- .returning()
3212
- .then((rows) => rows[0]);
3213
- await tx
3214
- .update(agentWakeupRequests)
3215
- .set({
3216
- status: "queued",
3217
- reason: "issue_execution_promoted",
3218
- runId: newRun.id,
3219
- claimedAt: null,
3220
- finishedAt: null,
3221
- error: null,
3222
- updatedAt: now,
3223
- })
3224
- .where(eq(agentWakeupRequests.id, deferred.id));
3225
- await tx
3226
- .update(issues)
3227
- .set({
3228
- executionRunId: newRun.id,
3229
- executionAgentNameKey: normalizeAgentNameKey(deferredAgent.name),
3230
- executionLockedAt: now,
3231
- updatedAt: now,
3232
- })
3233
- .where(eq(issues.id, issue.id));
3234
- return { promotedRun: newRun, passiveClosure };
3235
- }
3236
- });
3237
- const passiveClosure = outcome.passiveClosure;
3238
- if (passiveClosure?.kind === "queued") {
3239
- await appendRunEvent(run, await nextRunEventSeq(run.id), {
3240
- eventType: "issue.passive_followup_queued",
3241
- stream: "system",
3242
- level: "warn",
3243
- message: `Queued passive issue follow-up ${passiveClosure.run.id}`,
3244
- payload: {
3245
- issueId: passiveClosure.issue.id,
3246
- followupRunId: passiveClosure.run.id,
3247
- originRunId: passiveClosure.originRunId,
3248
- previousRunId: passiveClosure.previousRunId,
3249
- attempt: passiveClosure.attempt,
3250
- maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
3251
- reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
3252
- requestedAt: passiveClosure.requestedAt.toISOString(),
3253
- },
3254
- });
3255
- await appendRunEvent(passiveClosure.run, await nextRunEventSeq(passiveClosure.run.id), {
3256
- eventType: "issue.passive_followup_queued",
3257
- stream: "system",
3258
- level: "warn",
3259
- message: `Passive follow-up queued because run ${run.id} ended without issue close-out`,
3260
- payload: {
3261
- issueId: passiveClosure.issue.id,
3262
- originRunId: passiveClosure.originRunId,
3263
- previousRunId: passiveClosure.previousRunId,
3264
- attempt: passiveClosure.attempt,
3265
- maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
3266
- reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
3267
- requestedAt: passiveClosure.requestedAt.toISOString(),
3268
- },
3269
- });
3270
- await logActivity(db, {
3271
- orgId: passiveClosure.issue.orgId,
3272
- actorType: "system",
3273
- actorId: "issue_closure_governance",
3274
- action: "issue.passive_followup_queued",
3275
- entityType: "issue",
3276
- entityId: passiveClosure.issue.id,
3277
- agentId: run.agentId,
3278
- runId: run.id,
3279
- details: {
3280
- issueId: passiveClosure.issue.id,
3281
- issueTitle: passiveClosure.issue.title,
3282
- followupRunId: passiveClosure.run.id,
3283
- originRunId: passiveClosure.originRunId,
3284
- previousRunId: passiveClosure.previousRunId,
3285
- attempt: passiveClosure.attempt,
3286
- maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
3287
- reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
3288
- requestedAt: passiveClosure.requestedAt.toISOString(),
3289
- },
3290
- });
3291
- }
3292
- else if (passiveClosure?.kind === "operator_review") {
3293
- await appendRunEvent(run, await nextRunEventSeq(run.id), {
3294
- eventType: "issue.closure_needs_operator_review",
3295
- stream: "system",
3296
- level: "warn",
3297
- message: "Passive issue follow-up stopped and needs operator review",
3298
- payload: {
3299
- issueId: passiveClosure.issue.id,
3300
- originRunId: passiveClosure.originRunId,
3301
- previousRunId: passiveClosure.previousRunId,
3302
- attempts: passiveClosure.attempts,
3303
- maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
3304
- reason: passiveClosure.reason,
3305
- },
3306
- });
3307
- await logActivity(db, {
3308
- orgId: passiveClosure.issue.orgId,
3309
- actorType: "system",
3310
- actorId: "issue_closure_governance",
3311
- action: "issue.closure_needs_operator_review",
3312
- entityType: "issue",
3313
- entityId: passiveClosure.issue.id,
3314
- agentId: run.agentId,
3315
- runId: run.id,
3316
- details: {
3317
- issueId: passiveClosure.issue.id,
3318
- issueTitle: passiveClosure.issue.title,
3319
- originRunId: passiveClosure.originRunId,
3320
- previousRunId: passiveClosure.previousRunId,
3321
- attempts: passiveClosure.attempts,
3322
- maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
3323
- reason: passiveClosure.reason,
3324
- },
3325
- });
3326
- }
3327
- const promotedRun = outcome.promotedRun;
3328
- if (!promotedRun)
3329
- return;
3330
- publishLiveEvent({
3331
- orgId: promotedRun.orgId,
3332
- type: "heartbeat.run.queued",
3333
- payload: {
3334
- runId: promotedRun.id,
3335
- agentId: promotedRun.agentId,
3336
- invocationSource: promotedRun.invocationSource,
3337
- triggerDetail: promotedRun.triggerDetail,
3338
- wakeupRequestId: promotedRun.wakeupRequestId,
3339
- },
3340
- });
3341
- await startNextQueuedRunForAgent(promotedRun.agentId);
3342
- }
3343
- async function enqueueWakeup(agentId, opts = {}) {
3344
- const source = opts.source ?? "on_demand";
3345
- const triggerDetail = opts.triggerDetail ?? null;
3346
- const contextSnapshot = { ...(opts.contextSnapshot ?? {}) };
3347
- const reason = opts.reason ?? null;
3348
- const payload = opts.payload ?? null;
3349
- const existingWakeupRequestId = readNonEmptyString(opts.existingWakeupRequestId);
3350
- const { contextSnapshot: enrichedContextSnapshot, issueIdFromPayload, taskKey, wakeCommentId, } = enrichWakeContextSnapshot({
3351
- contextSnapshot,
3352
- reason,
3353
- source,
3354
- triggerDetail,
3355
- payload,
3356
- });
3357
- let issueId = readNonEmptyString(enrichedContextSnapshot.issueId) ?? issueIdFromPayload;
3358
- const agent = await getAgent(agentId);
3359
- if (!agent)
3360
- throw notFound("Agent not found");
3361
- const explicitResumeSession = await resolveExplicitResumeSessionOverride(agent, payload, taskKey);
3362
- if (explicitResumeSession) {
3363
- enrichedContextSnapshot.resumeFromRunId = explicitResumeSession.resumeFromRunId;
3364
- enrichedContextSnapshot.resumeSessionDisplayId = explicitResumeSession.sessionDisplayId;
3365
- enrichedContextSnapshot.resumeSessionParams = explicitResumeSession.sessionParams;
3366
- if (!readNonEmptyString(enrichedContextSnapshot.issueId) && explicitResumeSession.issueId) {
3367
- enrichedContextSnapshot.issueId = explicitResumeSession.issueId;
3368
- }
3369
- if (!readNonEmptyString(enrichedContextSnapshot.taskId) && explicitResumeSession.taskId) {
3370
- enrichedContextSnapshot.taskId = explicitResumeSession.taskId;
3371
- }
3372
- if (!readNonEmptyString(enrichedContextSnapshot.taskKey) && explicitResumeSession.taskKey) {
3373
- enrichedContextSnapshot.taskKey = explicitResumeSession.taskKey;
3374
- }
3375
- issueId = readNonEmptyString(enrichedContextSnapshot.issueId) ?? issueId;
3376
- }
3377
- await hydrateWakeContextSnapshot(db, agent.orgId, enrichedContextSnapshot);
3378
- const effectiveTaskKey = readNonEmptyString(enrichedContextSnapshot.taskKey) ?? taskKey;
3379
- const sessionBefore = explicitResumeSession?.sessionDisplayId ??
3380
- await resolveSessionBeforeForWakeup(agent, effectiveTaskKey);
3381
- const writeSkippedRequest = async (skipReason) => {
3382
- if (existingWakeupRequestId) {
3383
- await setWakeupStatus(existingWakeupRequestId, "skipped", {
3384
- reason: skipReason,
3385
- finishedAt: new Date(),
3386
- runId: null,
3387
- claimedAt: null,
3388
- error: null,
3389
- });
3390
- return;
3391
- }
3392
- await db.insert(agentWakeupRequests).values({
3393
- orgId: agent.orgId,
3394
- agentId,
3395
- source,
3396
- triggerDetail,
3397
- reason: skipReason,
3398
- payload,
3399
- status: "skipped",
3400
- requestedByActorType: opts.requestedByActorType ?? null,
3401
- requestedByActorId: opts.requestedByActorId ?? null,
3402
- idempotencyKey: opts.idempotencyKey ?? null,
3403
- finishedAt: new Date(),
3404
- });
3405
- };
3406
- let projectId = readNonEmptyString(enrichedContextSnapshot.projectId);
3407
- if (!projectId && issueId) {
3408
- projectId = await db
3409
- .select({ projectId: issues.projectId })
3410
- .from(issues)
3411
- .where(and(eq(issues.id, issueId), eq(issues.orgId, agent.orgId)))
3412
- .then((rows) => rows[0]?.projectId ?? null);
3413
- }
3414
- const budgetBlock = await budgets.getInvocationBlock(agent.orgId, agentId, {
3415
- issueId,
3416
- projectId,
3417
- });
3418
- if (budgetBlock) {
3419
- await writeSkippedRequest("budget.blocked");
3420
- throw conflict(budgetBlock.reason, {
3421
- scopeType: budgetBlock.scopeType,
3422
- scopeId: budgetBlock.scopeId,
3423
- });
3424
- }
3425
- if (agent.status === "paused") {
3426
- const deferredPayload = buildDeferredWakePayload(payload, enrichedContextSnapshot, issueId);
3427
- if (existingWakeupRequestId) {
3428
- await setWakeupStatus(existingWakeupRequestId, "deferred_agent_paused", {
3429
- reason,
3430
- payload: deferredPayload,
3431
- runId: null,
3432
- claimedAt: null,
3433
- finishedAt: null,
3434
- error: null,
3435
- });
3436
- return null;
3437
- }
3438
- await db.transaction(async (tx) => {
3439
- const deferredRows = await tx
3440
- .select()
3441
- .from(agentWakeupRequests)
3442
- .where(and(eq(agentWakeupRequests.orgId, agent.orgId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_agent_paused"), sql `${agentWakeupRequests.runId} is null`))
3443
- .orderBy(asc(agentWakeupRequests.requestedAt));
3444
- const existingDeferred = deferredRows.find((candidate) => isSameTaskScope(deriveDeferredWakeTaskKey(candidate.payload), effectiveTaskKey));
3445
- if (existingDeferred) {
3446
- const mergedDeferredContext = mergeCoalescedContextSnapshot(readDeferredWakeContext(existingDeferred.payload), enrichedContextSnapshot);
3447
- await updateWakeupRequestRecord(tx, existingDeferred.id, {
3448
- payload: buildDeferredWakePayload({
3449
- ...readDeferredWakePayload(existingDeferred.payload),
3450
- ...(payload ?? {}),
3451
- }, mergedDeferredContext, issueId),
3452
- coalescedCount: (existingDeferred.coalescedCount ?? 0) + 1,
3453
- error: null,
3454
- finishedAt: null,
3455
- claimedAt: null,
3456
- runId: null,
3457
- });
3458
- return;
3459
- }
3460
- await insertWakeupRequestRecord(tx, {
3461
- orgId: agent.orgId,
3462
- agentId,
3463
- source,
3464
- triggerDetail,
3465
- reason,
3466
- payload: deferredPayload,
3467
- status: "deferred_agent_paused",
3468
- requestedByActorType: opts.requestedByActorType ?? null,
3469
- requestedByActorId: opts.requestedByActorId ?? null,
3470
- idempotencyKey: opts.idempotencyKey ?? null,
3471
- });
3472
- });
3473
- return null;
3474
- }
3475
- if (agent.status === "terminated" || agent.status === "pending_approval") {
3476
- throw conflict("Agent is not invokable in its current state", { status: agent.status });
3477
- }
3478
- const policy = parseHeartbeatPolicy(agent);
3479
- if (source === "timer" && !policy.enabled) {
3480
- await writeSkippedRequest("heartbeat.disabled");
3481
- return null;
3482
- }
3483
- if (source !== "timer" && !policy.wakeOnDemand) {
3484
- await writeSkippedRequest("heartbeat.wakeOnDemand.disabled");
3485
- return null;
3486
- }
3487
- const bypassIssueExecutionLock = reason === "issue_comment_mentioned" ||
3488
- readNonEmptyString(enrichedContextSnapshot.wakeReason) === "issue_comment_mentioned";
3489
- if (issueId && !bypassIssueExecutionLock) {
3490
- const agentNameKey = normalizeAgentNameKey(agent.name);
3491
- const outcome = await db.transaction(async (tx) => {
3492
- await tx.execute(sql `select id from issues where id = ${issueId} and org_id = ${agent.orgId} for update`);
3493
- const issue = await tx
3494
- .select({
3495
- id: issues.id,
3496
- orgId: issues.orgId,
3497
- executionRunId: issues.executionRunId,
3498
- executionAgentNameKey: issues.executionAgentNameKey,
3499
- })
3500
- .from(issues)
3501
- .where(and(eq(issues.id, issueId), eq(issues.orgId, agent.orgId)))
3502
- .then((rows) => rows[0] ?? null);
3503
- if (!issue) {
3504
- if (existingWakeupRequestId) {
3505
- await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
3506
- status: "skipped",
3507
- reason: "issue_execution_issue_not_found",
3508
- runId: null,
3509
- claimedAt: null,
3510
- finishedAt: new Date(),
3511
- error: null,
3512
- });
3513
- }
3514
- else {
3515
- await insertWakeupRequestRecord(tx, {
3516
- orgId: agent.orgId,
3517
- agentId,
3518
- source,
3519
- triggerDetail,
3520
- reason: "issue_execution_issue_not_found",
3521
- payload,
3522
- status: "skipped",
3523
- requestedByActorType: opts.requestedByActorType ?? null,
3524
- requestedByActorId: opts.requestedByActorId ?? null,
3525
- idempotencyKey: opts.idempotencyKey ?? null,
3526
- finishedAt: new Date(),
3527
- });
3528
- }
3529
- return { kind: "skipped" };
3530
- }
3531
- let activeExecutionRun = issue.executionRunId
3532
- ? await tx
3533
- .select()
3534
- .from(heartbeatRuns)
3535
- .where(eq(heartbeatRuns.id, issue.executionRunId))
3536
- .then((rows) => rows[0] ?? null)
3537
- : null;
3538
- if (activeExecutionRun && activeExecutionRun.status !== "queued" && activeExecutionRun.status !== "running") {
3539
- activeExecutionRun = null;
3540
- }
3541
- if (!activeExecutionRun && issue.executionRunId) {
3542
- await tx
3543
- .update(issues)
3544
- .set({
3545
- executionRunId: null,
3546
- executionAgentNameKey: null,
3547
- executionLockedAt: null,
3548
- updatedAt: new Date(),
3549
- })
3550
- .where(eq(issues.id, issue.id));
3551
- }
3552
- if (!activeExecutionRun) {
3553
- const legacyRun = await tx
3554
- .select()
3555
- .from(heartbeatRuns)
3556
- .where(and(eq(heartbeatRuns.orgId, issue.orgId), inArray(heartbeatRuns.status, ["queued", "running"]), sql `${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`))
3557
- .orderBy(sql `case when ${heartbeatRuns.status} = 'running' then 0 else 1 end`, asc(heartbeatRuns.createdAt))
3558
- .limit(1)
3559
- .then((rows) => rows[0] ?? null);
3560
- if (legacyRun) {
3561
- activeExecutionRun = legacyRun;
3562
- const legacyAgent = await tx
3563
- .select({ name: agents.name })
3564
- .from(agents)
3565
- .where(eq(agents.id, legacyRun.agentId))
3566
- .then((rows) => rows[0] ?? null);
3567
- await tx
3568
- .update(issues)
3569
- .set({
3570
- executionRunId: legacyRun.id,
3571
- executionAgentNameKey: normalizeAgentNameKey(legacyAgent?.name),
3572
- executionLockedAt: new Date(),
3573
- updatedAt: new Date(),
3574
- })
3575
- .where(eq(issues.id, issue.id));
3576
- }
3577
- }
3578
- if (activeExecutionRun) {
3579
- const executionAgent = await tx
3580
- .select({ name: agents.name })
3581
- .from(agents)
3582
- .where(eq(agents.id, activeExecutionRun.agentId))
3583
- .then((rows) => rows[0] ?? null);
3584
- const executionAgentNameKey = normalizeAgentNameKey(issue.executionAgentNameKey) ??
3585
- normalizeAgentNameKey(executionAgent?.name);
3586
- const isSameExecutionAgent = Boolean(executionAgentNameKey) && executionAgentNameKey === agentNameKey;
3587
- const shouldQueueFollowupForCommentWake = Boolean(wakeCommentId) &&
3588
- activeExecutionRun.status === "running" &&
3589
- isSameExecutionAgent;
3590
- if (isSameExecutionAgent && !shouldQueueFollowupForCommentWake) {
3591
- const mergedContextSnapshot = mergeCoalescedContextSnapshot(activeExecutionRun.contextSnapshot, enrichedContextSnapshot);
3592
- const mergedRun = await tx
3593
- .update(heartbeatRuns)
3594
- .set({
3595
- contextSnapshot: mergedContextSnapshot,
3596
- updatedAt: new Date(),
3597
- })
3598
- .where(eq(heartbeatRuns.id, activeExecutionRun.id))
3599
- .returning()
3600
- .then((rows) => rows[0] ?? activeExecutionRun);
3601
- if (existingWakeupRequestId) {
3602
- await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
3603
- status: "coalesced",
3604
- reason: "issue_execution_same_name",
3605
- runId: mergedRun.id,
3606
- claimedAt: null,
3607
- finishedAt: new Date(),
3608
- error: null,
3609
- });
3610
- }
3611
- else {
3612
- await insertWakeupRequestRecord(tx, {
3613
- orgId: agent.orgId,
3614
- agentId,
3615
- source,
3616
- triggerDetail,
3617
- reason: "issue_execution_same_name",
3618
- payload,
3619
- status: "coalesced",
3620
- coalescedCount: 1,
3621
- requestedByActorType: opts.requestedByActorType ?? null,
3622
- requestedByActorId: opts.requestedByActorId ?? null,
3623
- idempotencyKey: opts.idempotencyKey ?? null,
3624
- runId: mergedRun.id,
3625
- finishedAt: new Date(),
3626
- });
3627
- }
3628
- return { kind: "coalesced", run: mergedRun };
3629
- }
3630
- const deferredPayload = buildDeferredWakePayload(payload, enrichedContextSnapshot, issueId);
3631
- const existingDeferred = await tx
3632
- .select()
3633
- .from(agentWakeupRequests)
3634
- .where(and(eq(agentWakeupRequests.orgId, agent.orgId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql `${agentWakeupRequests.payload} ->> 'issueId' = ${issue.id}`))
3635
- .orderBy(asc(agentWakeupRequests.requestedAt))
3636
- .limit(1)
3637
- .then((rows) => rows[0] ?? null);
3638
- if (existingDeferred) {
3639
- const mergedDeferredContext = mergeCoalescedContextSnapshot(readDeferredWakeContext(existingDeferred.payload), enrichedContextSnapshot);
3640
- const mergedDeferredPayload = buildDeferredWakePayload({
3641
- ...readDeferredWakePayload(existingDeferred.payload),
3642
- ...(payload ?? {}),
3643
- }, mergedDeferredContext, issueId);
3644
- if (existingWakeupRequestId && existingDeferred.id !== existingWakeupRequestId) {
3645
- await updateWakeupRequestRecord(tx, existingDeferred.id, {
3646
- payload: mergedDeferredPayload,
3647
- coalescedCount: (existingDeferred.coalescedCount ?? 0) + 1,
3648
- });
3649
- await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
3650
- status: "coalesced",
3651
- reason: "issue_execution_deferred",
3652
- runId: null,
3653
- claimedAt: null,
3654
- finishedAt: new Date(),
3655
- error: null,
3656
- });
3657
- }
3658
- else {
3659
- await updateWakeupRequestRecord(tx, existingDeferred.id, {
3660
- payload: mergedDeferredPayload,
3661
- coalescedCount: (existingDeferred.coalescedCount ?? 0) + 1,
3662
- status: "deferred_issue_execution",
3663
- reason: "issue_execution_deferred",
3664
- runId: null,
3665
- claimedAt: null,
3666
- finishedAt: null,
3667
- error: null,
3668
- });
3669
- }
3670
- return { kind: "deferred" };
3671
- }
3672
- if (existingWakeupRequestId) {
3673
- await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
3674
- status: "deferred_issue_execution",
3675
- reason: "issue_execution_deferred",
3676
- payload: deferredPayload,
3677
- runId: null,
3678
- claimedAt: null,
3679
- finishedAt: null,
3680
- error: null,
3681
- });
3682
- }
3683
- else {
3684
- await insertWakeupRequestRecord(tx, {
3685
- orgId: agent.orgId,
3686
- agentId,
3687
- source,
3688
- triggerDetail,
3689
- reason: "issue_execution_deferred",
3690
- payload: deferredPayload,
3691
- status: "deferred_issue_execution",
3692
- requestedByActorType: opts.requestedByActorType ?? null,
3693
- requestedByActorId: opts.requestedByActorId ?? null,
3694
- idempotencyKey: opts.idempotencyKey ?? null,
3695
- });
3696
- }
3697
- return { kind: "deferred" };
3698
- }
3699
- const wakeupRequest = existingWakeupRequestId
3700
- ? await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
3701
- status: "queued",
3702
- runId: null,
3703
- claimedAt: null,
3704
- finishedAt: null,
3705
- error: null,
3706
- })
3707
- : await insertWakeupRequestRecord(tx, {
3708
- orgId: agent.orgId,
3709
- agentId,
3710
- source,
3711
- triggerDetail,
3712
- reason,
3713
- payload,
3714
- status: "queued",
3715
- requestedByActorType: opts.requestedByActorType ?? null,
3716
- requestedByActorId: opts.requestedByActorId ?? null,
3717
- idempotencyKey: opts.idempotencyKey ?? null,
3718
- });
3719
- const newRun = await tx
3720
- .insert(heartbeatRuns)
3721
- .values({
3722
- orgId: agent.orgId,
3723
- agentId,
3724
- invocationSource: source,
3725
- triggerDetail,
3726
- status: "queued",
3727
- wakeupRequestId: wakeupRequest.id,
3728
- contextSnapshot: enrichedContextSnapshot,
3729
- sessionIdBefore: sessionBefore,
3730
- })
3731
- .returning()
3732
- .then((rows) => rows[0]);
3733
- await updateWakeupRequestRecord(tx, wakeupRequest.id, {
3734
- runId: newRun.id,
3735
- status: "queued",
3736
- claimedAt: null,
3737
- finishedAt: null,
3738
- error: null,
3739
- });
3740
- await tx
3741
- .update(issues)
3742
- .set({
3743
- executionRunId: newRun.id,
3744
- executionAgentNameKey: agentNameKey,
3745
- executionLockedAt: new Date(),
3746
- updatedAt: new Date(),
3747
- })
3748
- .where(eq(issues.id, issue.id));
3749
- return { kind: "queued", run: newRun };
3750
- });
3751
- if (outcome.kind === "deferred" || outcome.kind === "skipped")
3752
- return null;
3753
- if (outcome.kind === "coalesced")
3754
- return outcome.run;
3755
- const newRun = outcome.run;
3756
- publishLiveEvent({
3757
- orgId: newRun.orgId,
3758
- type: "heartbeat.run.queued",
3759
- payload: {
3760
- runId: newRun.id,
3761
- agentId: newRun.agentId,
3762
- invocationSource: newRun.invocationSource,
3763
- triggerDetail: newRun.triggerDetail,
3764
- wakeupRequestId: newRun.wakeupRequestId,
3765
- },
3766
- });
3767
- await startNextQueuedRunForAgent(agent.id);
3768
- return newRun;
3769
- }
3770
- const activeRuns = await db
3771
- .select()
3772
- .from(heartbeatRuns)
3773
- .where(and(eq(heartbeatRuns.agentId, agentId), inArray(heartbeatRuns.status, ["queued", "running"])))
3774
- .orderBy(desc(heartbeatRuns.createdAt));
3775
- const sameScopeQueuedRun = activeRuns.find((candidate) => candidate.status === "queued" && isSameTaskScope(runTaskKey(candidate), taskKey));
3776
- const sameScopeRunningRun = activeRuns.find((candidate) => candidate.status === "running" && isSameTaskScope(runTaskKey(candidate), taskKey));
3777
- const shouldQueueFollowupForCommentWake = Boolean(wakeCommentId) && Boolean(sameScopeRunningRun) && !sameScopeQueuedRun;
3778
- const coalescedTargetRun = sameScopeQueuedRun ??
3779
- (shouldQueueFollowupForCommentWake ? null : sameScopeRunningRun ?? null);
3780
- if (coalescedTargetRun) {
3781
- const mergedContextSnapshot = mergeCoalescedContextSnapshot(coalescedTargetRun.contextSnapshot, contextSnapshot);
3782
- const mergedRun = await db
3783
- .update(heartbeatRuns)
3784
- .set({
3785
- contextSnapshot: mergedContextSnapshot,
3786
- updatedAt: new Date(),
3787
- })
3788
- .where(eq(heartbeatRuns.id, coalescedTargetRun.id))
3789
- .returning()
3790
- .then((rows) => rows[0] ?? coalescedTargetRun);
3791
- if (existingWakeupRequestId) {
3792
- await setWakeupStatus(existingWakeupRequestId, "coalesced", {
3793
- runId: mergedRun.id,
3794
- claimedAt: null,
3795
- finishedAt: new Date(),
3796
- error: null,
3797
- });
3798
- }
3799
- else {
3800
- await db.insert(agentWakeupRequests).values({
3801
- orgId: agent.orgId,
3802
- agentId,
3803
- source,
3804
- triggerDetail,
3805
- reason,
3806
- payload,
3807
- status: "coalesced",
3808
- coalescedCount: 1,
3809
- requestedByActorType: opts.requestedByActorType ?? null,
3810
- requestedByActorId: opts.requestedByActorId ?? null,
3811
- idempotencyKey: opts.idempotencyKey ?? null,
3812
- runId: mergedRun.id,
3813
- finishedAt: new Date(),
3814
- });
3815
- }
3816
- return mergedRun;
3817
- }
3818
- const wakeupRequest = existingWakeupRequestId
3819
- ? await updateWakeupRequestRecord(db, existingWakeupRequestId, {
3820
- status: "queued",
3821
- runId: null,
3822
- claimedAt: null,
3823
- finishedAt: null,
3824
- error: null,
3825
- })
3826
- : await insertWakeupRequestRecord(db, {
3827
- orgId: agent.orgId,
3828
- agentId,
3829
- source,
3830
- triggerDetail,
3831
- reason,
3832
- payload,
3833
- status: "queued",
3834
- requestedByActorType: opts.requestedByActorType ?? null,
3835
- requestedByActorId: opts.requestedByActorId ?? null,
3836
- idempotencyKey: opts.idempotencyKey ?? null,
3837
- });
3838
- const newRun = await db
3839
- .insert(heartbeatRuns)
3840
- .values({
3841
- orgId: agent.orgId,
3842
- agentId,
3843
- invocationSource: source,
3844
- triggerDetail,
3845
- status: "queued",
3846
- wakeupRequestId: wakeupRequest.id,
3847
- contextSnapshot: enrichedContextSnapshot,
3848
- sessionIdBefore: sessionBefore,
3849
- })
3850
- .returning()
3851
- .then((rows) => rows[0]);
3852
- await updateWakeupRequestRecord(db, wakeupRequest.id, {
3853
- status: "queued",
3854
- runId: newRun.id,
3855
- claimedAt: null,
3856
- finishedAt: null,
3857
- error: null,
3858
- });
3859
- publishLiveEvent({
3860
- orgId: newRun.orgId,
3861
- type: "heartbeat.run.queued",
3862
- payload: {
3863
- runId: newRun.id,
3864
- agentId: newRun.agentId,
3865
- invocationSource: newRun.invocationSource,
3866
- triggerDetail: newRun.triggerDetail,
3867
- wakeupRequestId: newRun.wakeupRequestId,
3868
- },
3869
- });
3870
- await startNextQueuedRunForAgent(agent.id);
3871
- return newRun;
3872
- }
3873
- async function resumeDeferredWakeupsForAgent(agentId) {
3874
- const agent = await getAgent(agentId);
3875
- if (!agent)
3876
- throw notFound("Agent not found");
3877
- const replayedRequestIds = [];
3878
- while (true) {
3879
- const deferred = await db
3880
- .select()
3881
- .from(agentWakeupRequests)
3882
- .where(and(eq(agentWakeupRequests.orgId, agent.orgId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_agent_paused"), sql `${agentWakeupRequests.runId} is null`))
3883
- .orderBy(asc(agentWakeupRequests.requestedAt))
3884
- .limit(1)
3885
- .then((rows) => rows[0] ?? null);
3886
- if (!deferred)
3887
- break;
3888
- const replayPayload = readDeferredWakePayload(deferred.payload);
3889
- const replayContextSnapshot = readDeferredWakeContext(deferred.payload);
3890
- try {
3891
- await enqueueWakeup(agentId, {
3892
- source: readNonEmptyString(deferred.source) ?? "on_demand",
3893
- triggerDetail: readNonEmptyString(deferred.triggerDetail) ?? undefined,
3894
- reason: readNonEmptyString(deferred.reason) ?? null,
3895
- payload: replayPayload,
3896
- idempotencyKey: deferred.idempotencyKey,
3897
- requestedByActorType: deferred.requestedByActorType ?? undefined,
3898
- requestedByActorId: deferred.requestedByActorId,
3899
- contextSnapshot: replayContextSnapshot,
3900
- existingWakeupRequestId: deferred.id,
3901
- });
3902
- }
3903
- catch (error) {
3904
- const current = await db
3905
- .select({ status: agentWakeupRequests.status })
3906
- .from(agentWakeupRequests)
3907
- .where(eq(agentWakeupRequests.id, deferred.id))
3908
- .then((rows) => rows[0] ?? null);
3909
- if (current?.status === "deferred_agent_paused") {
3910
- await setWakeupStatus(deferred.id, "failed", {
3911
- finishedAt: new Date(),
3912
- error: error instanceof Error ? error.message : String(error),
3913
- });
3914
- }
3915
- }
3916
- replayedRequestIds.push(deferred.id);
3917
- const current = await db
3918
- .select({ status: agentWakeupRequests.status })
3919
- .from(agentWakeupRequests)
3920
- .where(eq(agentWakeupRequests.id, deferred.id))
3921
- .then((rows) => rows[0] ?? null);
3922
- if (current?.status === "deferred_agent_paused")
3923
- break;
3924
- }
3925
- return {
3926
- replayed: replayedRequestIds.length,
3927
- wakeupRequestIds: replayedRequestIds,
3928
- };
3929
- }
3930
- async function listProjectScopedRunIds(orgId, projectId) {
3931
- const runIssueId = sql `${heartbeatRuns.contextSnapshot} ->> 'issueId'`;
3932
- const effectiveProjectId = sql `coalesce(${heartbeatRuns.contextSnapshot} ->> 'projectId', ${issues.projectId}::text)`;
3933
- const rows = await db
3934
- .selectDistinctOn([heartbeatRuns.id], { id: heartbeatRuns.id })
3935
- .from(heartbeatRuns)
3936
- .leftJoin(issues, and(eq(issues.orgId, orgId), sql `${issues.id}::text = ${runIssueId}`))
3937
- .where(and(eq(heartbeatRuns.orgId, orgId), inArray(heartbeatRuns.status, ["queued", "running"]), sql `${effectiveProjectId} = ${projectId}`));
3938
- return rows.map((row) => row.id);
3939
- }
3940
- async function listProjectScopedWakeupIds(orgId, projectId) {
3941
- const wakeIssueId = sql `${agentWakeupRequests.payload} ->> 'issueId'`;
3942
- const effectiveProjectId = sql `coalesce(${agentWakeupRequests.payload} ->> 'projectId', ${issues.projectId}::text)`;
3943
- const rows = await db
3944
- .selectDistinctOn([agentWakeupRequests.id], { id: agentWakeupRequests.id })
3945
- .from(agentWakeupRequests)
3946
- .leftJoin(issues, and(eq(issues.orgId, orgId), sql `${issues.id}::text = ${wakeIssueId}`))
3947
- .where(and(eq(agentWakeupRequests.orgId, orgId), inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), sql `${agentWakeupRequests.runId} is null`, sql `${effectiveProjectId} = ${projectId}`));
3948
- return rows.map((row) => row.id);
3949
- }
3950
- async function cancelPendingWakeupsForBudgetScope(scope) {
3951
- const now = new Date();
3952
- let wakeupIds = [];
3953
- if (scope.scopeType === "organization") {
3954
- wakeupIds = await db
3955
- .select({ id: agentWakeupRequests.id })
3956
- .from(agentWakeupRequests)
3957
- .where(and(eq(agentWakeupRequests.orgId, scope.orgId), inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), sql `${agentWakeupRequests.runId} is null`))
3958
- .then((rows) => rows.map((row) => row.id));
3959
- }
3960
- else if (scope.scopeType === "agent") {
3961
- wakeupIds = await db
3962
- .select({ id: agentWakeupRequests.id })
3963
- .from(agentWakeupRequests)
3964
- .where(and(eq(agentWakeupRequests.orgId, scope.orgId), eq(agentWakeupRequests.agentId, scope.scopeId), inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), sql `${agentWakeupRequests.runId} is null`))
3965
- .then((rows) => rows.map((row) => row.id));
3966
- }
3967
- else {
3968
- wakeupIds = await listProjectScopedWakeupIds(scope.orgId, scope.scopeId);
3969
- }
3970
- if (wakeupIds.length === 0)
3971
- return 0;
3972
- await db
3973
- .update(agentWakeupRequests)
3974
- .set({
3975
- status: "cancelled",
3976
- finishedAt: now,
3977
- error: "Cancelled due to budget pause",
3978
- updatedAt: now,
3979
- })
3980
- .where(inArray(agentWakeupRequests.id, wakeupIds));
3981
- return wakeupIds.length;
3982
- }
3983
- async function cancelRunInternal(runId, reason = "Cancelled by control plane") {
3984
- const run = await getRun(runId);
3985
- if (!run)
3986
- throw notFound("Heartbeat run not found");
3987
- if (run.status !== "running" && run.status !== "queued")
3988
- return run;
3989
- const running = runningProcesses.get(run.id);
3990
- if (running) {
3991
- running.child.kill("SIGTERM");
3992
- const graceMs = Math.max(1, running.graceSec) * 1000;
3993
- setTimeout(() => {
3994
- if (!running.child.killed) {
3995
- running.child.kill("SIGKILL");
3996
- }
3997
- }, graceMs);
3998
- }
3999
- const cancelled = await setRunStatus(run.id, "cancelled", {
4000
- finishedAt: new Date(),
4001
- error: reason,
4002
- errorCode: "cancelled",
4003
- });
4004
- await setWakeupStatus(run.wakeupRequestId, "cancelled", {
4005
- finishedAt: new Date(),
4006
- error: reason,
4007
- });
4008
- if (cancelled) {
4009
- await appendRunEvent(cancelled, 1, {
4010
- eventType: "lifecycle",
4011
- stream: "system",
4012
- level: "warn",
4013
- message: "run cancelled",
4014
- });
4015
- await releaseIssueExecutionAndPromote(cancelled);
4016
- }
4017
- runningProcesses.delete(run.id);
4018
- await finalizeAgentStatus(run.agentId, "cancelled");
4019
- await startNextQueuedRunForAgent(run.agentId);
4020
- return cancelled;
4021
- }
4022
- async function cancelActiveForAgentInternal(agentId, reason = "Cancelled due to agent pause") {
4023
- const runs = await db
4024
- .select()
4025
- .from(heartbeatRuns)
4026
- .where(and(eq(heartbeatRuns.agentId, agentId), inArray(heartbeatRuns.status, ["queued", "running"])));
4027
- for (const run of runs) {
4028
- await setRunStatus(run.id, "cancelled", {
4029
- finishedAt: new Date(),
4030
- error: reason,
4031
- errorCode: "cancelled",
4032
- });
4033
- await setWakeupStatus(run.wakeupRequestId, "cancelled", {
4034
- finishedAt: new Date(),
4035
- error: reason,
4036
- });
4037
- const running = runningProcesses.get(run.id);
4038
- if (running) {
4039
- running.child.kill("SIGTERM");
4040
- runningProcesses.delete(run.id);
4041
- }
4042
- await releaseIssueExecutionAndPromote(run);
4043
- }
4044
- return runs.length;
4045
- }
4046
- async function cancelBudgetScopeWork(scope) {
4047
- if (scope.scopeType === "agent") {
4048
- await cancelActiveForAgentInternal(scope.scopeId, "Cancelled due to budget pause");
4049
- await cancelPendingWakeupsForBudgetScope(scope);
4050
- return;
4051
- }
4052
- const runIds = scope.scopeType === "organization"
4053
- ? await db
4054
- .select({ id: heartbeatRuns.id })
4055
- .from(heartbeatRuns)
4056
- .where(and(eq(heartbeatRuns.orgId, scope.orgId), inArray(heartbeatRuns.status, ["queued", "running"])))
4057
- .then((rows) => rows.map((row) => row.id))
4058
- : await listProjectScopedRunIds(scope.orgId, scope.scopeId);
4059
- for (const runId of runIds) {
4060
- await cancelRunInternal(runId, "Cancelled due to budget pause");
4061
- }
4062
- await cancelPendingWakeupsForBudgetScope(scope);
4063
- }
4064
- async function retryRunInternal(runId, opts) {
4065
- const run = await getRun(runId);
4066
- if (!run)
4067
- throw notFound("Heartbeat run not found");
4068
- if (run.status !== "failed" && run.status !== "timed_out") {
4069
- throw conflict("Only failed or timed out runs can be retried", {
4070
- status: run.status,
4071
- });
4072
- }
4073
- const agent = await getAgent(run.agentId);
4074
- if (!agent)
4075
- throw notFound("Agent not found");
4076
- if (agent.status === "paused" ||
4077
- agent.status === "terminated" ||
4078
- agent.status === "pending_approval") {
4079
- throw conflict("Agent is not invokable in its current state", { status: agent.status });
4080
- }
4081
- const policy = parseHeartbeatPolicy(agent);
4082
- if (!policy.wakeOnDemand) {
4083
- throw conflict("Agent is not configured for on-demand wakeups");
4084
- }
4085
- const context = parseObject(run.contextSnapshot);
4086
- const issueId = readNonEmptyString(context.issueId);
4087
- let projectId = readNonEmptyString(context.projectId);
4088
- if (!projectId && issueId) {
4089
- projectId = await db
4090
- .select({ projectId: issues.projectId })
4091
- .from(issues)
4092
- .where(and(eq(issues.id, issueId), eq(issues.orgId, agent.orgId)))
4093
- .then((rows) => rows[0]?.projectId ?? null);
4094
- }
4095
- const budgetBlock = await budgets.getInvocationBlock(agent.orgId, agent.id, {
4096
- issueId,
4097
- projectId,
4098
- });
4099
- if (budgetBlock) {
4100
- throw conflict(budgetBlock.reason, {
4101
- scopeType: budgetBlock.scopeType,
4102
- scopeId: budgetBlock.scopeId,
4103
- });
4104
- }
4105
- return enqueueRecoveryRun(run, agent, {
4106
- recoveryTrigger: "manual",
4107
- source: "on_demand",
4108
- triggerDetail: "manual",
4109
- wakeReason: "retry_failed_run",
4110
- requestedByActorType: opts?.requestedByActorType ?? "user",
4111
- requestedByActorId: opts?.requestedByActorId ?? null,
4112
- now: opts?.now ?? new Date(),
4113
- });
4114
- }
4115
- async function buildSkillAnalytics(scope, opts) {
4116
- const now = opts?.now ?? new Date();
4117
- const customDateKeys = opts?.startDate && opts?.endDate
4118
- ? buildDateKeysBetween(opts.startDate, opts.endDate).slice(0, 120)
4119
- : [];
4120
- const windowDays = customDateKeys.length > 0
4121
- ? customDateKeys.length
4122
- : Math.max(1, Math.min(opts?.windowDays ?? 30, 90));
4123
- const dateKeys = customDateKeys.length > 0
4124
- ? customDateKeys
4125
- : buildRecentDateKeys(windowDays, now);
4126
- const startDate = dateKeys[0];
4127
- const endDate = dateKeys.at(-1);
4128
- const windowStart = new Date(`${startDate}T00:00:00.000Z`);
4129
- const windowEnd = new Date(`${endDate}T23:59:59.999Z`);
4130
- const rows = await db
4131
- .select({
4132
- createdAt: heartbeatRunEvents.createdAt,
4133
- payload: heartbeatRunEvents.payload,
4134
- })
4135
- .from(heartbeatRunEvents)
4136
- .where(and(eq(heartbeatRunEvents.orgId, scope.orgId), ...(scope.agentId ? [eq(heartbeatRunEvents.agentId, scope.agentId)] : []), eq(heartbeatRunEvents.eventType, "adapter.invoke"), gte(heartbeatRunEvents.createdAt, windowStart), lte(heartbeatRunEvents.createdAt, windowEnd)))
4137
- .orderBy(asc(heartbeatRunEvents.createdAt), asc(heartbeatRunEvents.id));
4138
- const days = new Map();
4139
- for (const date of dateKeys) {
4140
- days.set(date, { totalCount: 0, runCount: 0, skills: new Map() });
4141
- }
4142
- const overallSkills = new Map();
4143
- let totalCount = 0;
4144
- let totalRunsWithSkills = 0;
4145
- for (const row of rows) {
4146
- const date = new Date(row.createdAt).toISOString().slice(0, 10);
4147
- const bucket = days.get(date);
4148
- if (!bucket)
4149
- continue;
4150
- const payload = parseObject(row.payload);
4151
- const loadedSkills = Array.isArray(payload.loadedSkills) ? payload.loadedSkills : [];
4152
- if (loadedSkills.length === 0)
4153
- continue;
4154
- const eventSkills = new Map();
4155
- for (const entry of loadedSkills) {
4156
- const normalized = normalizeLoadedSkill(entry);
4157
- if (!normalized)
4158
- continue;
4159
- if (!eventSkills.has(normalized.key)) {
4160
- eventSkills.set(normalized.key, normalized.label);
4161
- }
4162
- }
4163
- if (eventSkills.size === 0)
4164
- continue;
4165
- bucket.runCount += 1;
4166
- totalRunsWithSkills += 1;
4167
- for (const [key, label] of eventSkills) {
4168
- bucket.totalCount += 1;
4169
- totalCount += 1;
4170
- const existingDaySkill = bucket.skills.get(key);
4171
- if (existingDaySkill) {
4172
- existingDaySkill.count += 1;
4173
- }
4174
- else {
4175
- bucket.skills.set(key, { key, label, count: 1 });
4176
- }
4177
- const existingOverallSkill = overallSkills.get(key);
4178
- if (existingOverallSkill) {
4179
- existingOverallSkill.count += 1;
4180
- }
4181
- else {
4182
- overallSkills.set(key, { key, label, count: 1 });
4183
- }
4184
- }
4185
- }
4186
- return {
4187
- agentId: scope.agentId ?? "__all__",
4188
- orgId: scope.orgId,
4189
- windowDays,
4190
- startDate,
4191
- endDate,
4192
- totalCount,
4193
- totalRunsWithSkills,
4194
- skills: Array.from(overallSkills.values()).sort((left, right) => (right.count - left.count
4195
- || left.label.localeCompare(right.label)
4196
- || left.key.localeCompare(right.key))),
4197
- days: dateKeys.map((date) => {
4198
- const bucket = days.get(date);
4199
- return {
4200
- date,
4201
- totalCount: bucket.totalCount,
4202
- runCount: bucket.runCount,
4203
- skills: Array.from(bucket.skills.values()).sort((left, right) => (right.count - left.count
4204
- || left.label.localeCompare(right.label)
4205
- || left.key.localeCompare(right.key))),
4206
- };
4207
- }),
4208
- };
4209
- }
4210
- return {
4211
- list: async (orgId, agentId, limit) => {
4212
- const query = db
4213
- .select(heartbeatRunListColumns)
4214
- .from(heartbeatRuns)
4215
- .where(agentId
4216
- ? and(eq(heartbeatRuns.orgId, orgId), eq(heartbeatRuns.agentId, agentId))
4217
- : eq(heartbeatRuns.orgId, orgId))
4218
- .orderBy(desc(heartbeatRuns.createdAt));
4219
- const rows = limit ? await query.limit(limit) : await query;
4220
- return rows.map((row) => ({
4221
- ...row,
4222
- resultJson: summarizeHeartbeatRunResultJson(row.resultJson),
4223
- }));
4224
- },
4225
- getAgentSkillAnalytics: async (agentId, opts) => {
4226
- const agent = await getAgent(agentId);
4227
- if (!agent)
4228
- throw notFound("Agent not found");
4229
- return buildSkillAnalytics({ orgId: agent.orgId, agentId: agent.id }, opts);
4230
- },
4231
- getOrganizationSkillAnalytics: async (orgId, opts) => {
4232
- const org = await db
4233
- .select({ id: organizations.id })
4234
- .from(organizations)
4235
- .where(eq(organizations.id, orgId))
4236
- .limit(1)
4237
- .then((rows) => rows[0] ?? null);
4238
- if (!org)
4239
- throw notFound("Organization not found");
4240
- return buildSkillAnalytics({ orgId }, opts);
4241
- },
4242
- getRun,
4243
- getRuntimeState: async (agentId) => {
4244
- const state = await getRuntimeState(agentId);
4245
- const agent = await getAgent(agentId);
4246
- if (!agent)
4247
- return null;
4248
- const ensured = state ?? (await ensureRuntimeState(agent));
4249
- const latestTaskSession = await db
4250
- .select()
4251
- .from(agentTaskSessions)
4252
- .where(and(eq(agentTaskSessions.orgId, agent.orgId), eq(agentTaskSessions.agentId, agent.id)))
4253
- .orderBy(desc(agentTaskSessions.updatedAt))
4254
- .limit(1)
4255
- .then((rows) => rows[0] ?? null);
4256
- return {
4257
- ...ensured,
4258
- sessionDisplayId: latestTaskSession?.sessionDisplayId ?? ensured.sessionId,
4259
- sessionParamsJson: latestTaskSession?.sessionParamsJson ?? null,
4260
- };
4261
- },
4262
- listTaskSessions: async (agentId) => {
4263
- const agent = await getAgent(agentId);
4264
- if (!agent)
4265
- throw notFound("Agent not found");
4266
- return db
4267
- .select()
4268
- .from(agentTaskSessions)
4269
- .where(and(eq(agentTaskSessions.orgId, agent.orgId), eq(agentTaskSessions.agentId, agentId)))
4270
- .orderBy(desc(agentTaskSessions.updatedAt), desc(agentTaskSessions.createdAt));
4271
- },
4272
- resetRuntimeSession: async (agentId, opts) => {
4273
- const agent = await getAgent(agentId);
4274
- if (!agent)
4275
- throw notFound("Agent not found");
4276
- await ensureRuntimeState(agent);
4277
- const taskKey = readNonEmptyString(opts?.taskKey);
4278
- const clearedTaskSessions = await clearTaskSessions(agent.orgId, agent.id, taskKey ? { taskKey, agentRuntimeType: agent.agentRuntimeType } : undefined);
4279
- const runtimePatch = {
4280
- sessionId: null,
4281
- lastError: null,
4282
- updatedAt: new Date(),
4283
- };
4284
- if (!taskKey) {
4285
- runtimePatch.stateJson = {};
4286
- }
4287
- const updated = await db
4288
- .update(agentRuntimeState)
4289
- .set(runtimePatch)
4290
- .where(eq(agentRuntimeState.agentId, agentId))
4291
- .returning()
4292
- .then((rows) => rows[0] ?? null);
4293
- if (!updated)
4294
- return null;
4295
- return {
4296
- ...updated,
4297
- sessionDisplayId: null,
4298
- sessionParamsJson: null,
4299
- clearedTaskSessions,
4300
- };
4301
- },
4302
- listEvents: (runId, afterSeq = 0, limit = 200) => db
4303
- .select()
4304
- .from(heartbeatRunEvents)
4305
- .where(and(eq(heartbeatRunEvents.runId, runId), gt(heartbeatRunEvents.seq, afterSeq)))
4306
- .orderBy(asc(heartbeatRunEvents.seq))
4307
- .limit(Math.max(1, Math.min(limit, 1000))),
4308
- readLog: async (runId, opts) => {
4309
- const run = await getRun(runId);
4310
- if (!run)
4311
- throw notFound("Heartbeat run not found");
4312
- if (!run.logStore || !run.logRef)
4313
- throw notFound("Run log not found");
4314
- const result = await runLogStore.read({
4315
- store: run.logStore,
4316
- logRef: run.logRef,
4317
- }, opts);
4318
- return {
4319
- runId,
4320
- store: run.logStore,
4321
- logRef: run.logRef,
4322
- ...result,
4323
- content: redactCurrentUserText(result.content, await getCurrentUserRedactionOptions()),
4324
- };
4325
- },
4326
- invoke: async (agentId, source = "on_demand", contextSnapshot = {}, triggerDetail = "manual", actor) => enqueueWakeup(agentId, {
4327
- source,
4328
- triggerDetail,
4329
- contextSnapshot,
4330
- requestedByActorType: actor?.actorType,
4331
- requestedByActorId: actor?.actorId ?? null,
4332
- }),
4333
- wakeup: enqueueWakeup,
4334
- resumeDeferredWakeupsForAgent,
4335
- retryRun: retryRunInternal,
4336
- reportRunActivity: clearDetachedRunWarning,
4337
- reapOrphanedRuns,
4338
- resumeQueuedRuns,
4339
- tickTimers: async (now = new Date()) => {
4340
- const allAgents = await db.select().from(agents);
4341
- let checked = 0;
4342
- let enqueued = 0;
4343
- let skipped = 0;
4344
- for (const agent of allAgents) {
4345
- if (agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval")
4346
- continue;
4347
- const policy = parseHeartbeatPolicy(agent);
4348
- if (!policy.enabled || policy.intervalSec <= 0)
4349
- continue;
4350
- checked += 1;
4351
- const baseline = new Date(agent.lastHeartbeatAt ?? agent.createdAt).getTime();
4352
- const elapsedMs = now.getTime() - baseline;
4353
- if (elapsedMs < policy.intervalSec * 1000)
4354
- continue;
4355
- const run = await enqueueWakeup(agent.id, {
4356
- source: "timer",
4357
- triggerDetail: "system",
4358
- reason: "heartbeat_timer",
4359
- requestedByActorType: "system",
4360
- requestedByActorId: "heartbeat_scheduler",
4361
- contextSnapshot: {
4362
- source: "scheduler",
4363
- reason: "interval_elapsed",
4364
- now: now.toISOString(),
4365
- },
4366
- });
4367
- if (run)
4368
- enqueued += 1;
4369
- else
4370
- skipped += 1;
4371
- }
4372
- return { checked, enqueued, skipped };
4373
- },
4374
- cancelRun: (runId) => cancelRunInternal(runId),
4375
- cancelActiveForAgent: (agentId) => cancelActiveForAgentInternal(agentId),
4376
- cancelBudgetScopeWork,
4377
- getActiveRunForAgent: async (agentId) => {
4378
- const [run] = await db
4379
- .select()
4380
- .from(heartbeatRuns)
4381
- .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "running")))
4382
- .orderBy(desc(heartbeatRuns.startedAt))
4383
- .limit(1);
4384
- return run ?? null;
4385
- },
4386
- };
4387
- }
1
+ export { buildExplicitResumeSessionOverride, buildHeartbeatAdapterInvokePayload, buildHeartbeatRuntimeTraceMetadata, buildIssueRunTraceName, formatRuntimeWorkspaceWarningLog, heartbeatService, parseSessionCompactionPolicy, prioritizeProjectWorkspaceCandidatesForRun, resolveHeartbeatObservabilitySurface, resolveRuntimeSessionParamsForWorkspace, shouldResetTaskSessionForWake, } from "./orchestrator.js";
4388
2
  //# sourceMappingURL=heartbeat.js.map