@rudderhq/server 0.1.0-canary.2 → 0.1.0-canary.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bootstrap/register-api-routes.js +1 -1
- package/dist/bootstrap/register-api-routes.js.map +1 -1
- package/dist/dev-server-status.d.ts +1 -7
- package/dist/dev-server-status.d.ts.map +1 -1
- package/dist/dev-server-status.js +1 -4
- package/dist/dev-server-status.js.map +1 -1
- package/dist/routes/agents.d.ts +2 -1
- package/dist/routes/agents.d.ts.map +1 -1
- package/dist/routes/agents.js +158 -1
- package/dist/routes/agents.js.map +1 -1
- package/dist/routes/chats.d.ts.map +1 -1
- package/dist/routes/chats.js +164 -21
- package/dist/routes/chats.js.map +1 -1
- package/dist/routes/dashboard.d.ts.map +1 -1
- package/dist/routes/dashboard.js +24 -0
- package/dist/routes/dashboard.js.map +1 -1
- package/dist/routes/health.d.ts.map +1 -1
- package/dist/routes/health.js +3 -12
- package/dist/routes/health.js.map +1 -1
- package/dist/routes/instance-settings.d.ts.map +1 -1
- package/dist/routes/instance-settings.js +1 -26
- package/dist/routes/instance-settings.js.map +1 -1
- package/dist/routes/issues.d.ts.map +1 -1
- package/dist/routes/issues.js +19 -17
- package/dist/routes/issues.js.map +1 -1
- package/dist/services/agent-run-context.d.ts +1 -1
- package/dist/services/agents.d.ts +13 -13
- package/dist/services/chat-assistant.d.ts +1 -1
- package/dist/services/chat-assistant.d.ts.map +1 -1
- package/dist/services/chat-assistant.js +244 -41
- package/dist/services/chat-assistant.js.map +1 -1
- package/dist/services/chat-generation-locks.d.ts +2 -1
- package/dist/services/chat-generation-locks.d.ts.map +1 -1
- package/dist/services/chat-generation-locks.js +12 -3
- package/dist/services/chat-generation-locks.js.map +1 -1
- package/dist/services/chats.d.ts +3 -3
- package/dist/services/chats.d.ts.map +1 -1
- package/dist/services/chats.js +21 -2
- package/dist/services/chats.js.map +1 -1
- package/dist/services/costs.d.ts +1 -1
- package/dist/services/documents.d.ts +23 -0
- package/dist/services/documents.d.ts.map +1 -1
- package/dist/services/documents.js +17 -1
- package/dist/services/documents.js.map +1 -1
- package/dist/services/finance.d.ts +2 -2
- package/dist/services/heartbeat.d.ts +1 -1
- package/dist/services/heartbeat.d.ts.map +1 -1
- package/dist/services/heartbeat.js +1 -1
- package/dist/services/heartbeat.js.map +1 -1
- package/dist/services/instance-settings.d.ts +1 -3
- package/dist/services/instance-settings.d.ts.map +1 -1
- package/dist/services/instance-settings.js +1 -38
- package/dist/services/instance-settings.js.map +1 -1
- package/dist/services/messenger.d.ts +1 -1
- package/dist/services/messenger.d.ts.map +1 -1
- package/dist/services/messenger.js +59 -12
- package/dist/services/messenger.js.map +1 -1
- package/dist/services/runtime-kernel/analytics.d.ts +7 -0
- package/dist/services/runtime-kernel/analytics.d.ts.map +1 -0
- package/dist/services/runtime-kernel/analytics.js +46 -0
- package/dist/services/runtime-kernel/analytics.js.map +1 -0
- package/dist/services/runtime-kernel/common.d.ts +4 -0
- package/dist/services/runtime-kernel/common.d.ts.map +1 -0
- package/dist/services/runtime-kernel/common.js +15 -0
- package/dist/services/runtime-kernel/common.js.map +1 -0
- package/dist/services/runtime-kernel/execution.d.ts +11 -0
- package/dist/services/runtime-kernel/execution.d.ts.map +1 -0
- package/dist/services/runtime-kernel/execution.js +28 -0
- package/dist/services/runtime-kernel/execution.js.map +1 -0
- package/dist/services/runtime-kernel/heartbeat.d.ts +1 -881
- package/dist/services/runtime-kernel/heartbeat.d.ts.map +1 -1
- package/dist/services/runtime-kernel/heartbeat.js +1 -4344
- package/dist/services/runtime-kernel/heartbeat.js.map +1 -1
- package/dist/services/runtime-kernel/model-fallback.d.ts +10 -0
- package/dist/services/runtime-kernel/model-fallback.d.ts.map +1 -0
- package/dist/services/runtime-kernel/model-fallback.js +147 -0
- package/dist/services/runtime-kernel/model-fallback.js.map +1 -0
- package/dist/services/runtime-kernel/orchestrator.d.ts +826 -0
- package/dist/services/runtime-kernel/orchestrator.d.ts.map +1 -0
- package/dist/services/runtime-kernel/orchestrator.js +4044 -0
- package/dist/services/runtime-kernel/orchestrator.js.map +1 -0
- package/dist/services/runtime-kernel/recovery.d.ts +8 -0
- package/dist/services/runtime-kernel/recovery.d.ts.map +1 -0
- package/dist/services/runtime-kernel/recovery.js +101 -0
- package/dist/services/runtime-kernel/recovery.js.map +1 -0
- package/dist/services/runtime-kernel/run-state.d.ts +45 -0
- package/dist/services/runtime-kernel/run-state.d.ts.map +1 -0
- package/dist/services/runtime-kernel/run-state.js +45 -0
- package/dist/services/runtime-kernel/run-state.js.map +1 -0
- package/dist/services/runtime-kernel/session-policy.d.ts +42 -0
- package/dist/services/runtime-kernel/session-policy.d.ts.map +1 -0
- package/dist/services/runtime-kernel/session-policy.js +116 -0
- package/dist/services/runtime-kernel/session-policy.js.map +1 -0
- package/dist/services/runtime-kernel/types.d.ts +13 -0
- package/dist/services/runtime-kernel/types.d.ts.map +1 -0
- package/dist/services/runtime-kernel/types.js +2 -0
- package/dist/services/runtime-kernel/types.js.map +1 -0
- package/dist/services/runtime-kernel/wakeup-queue.d.ts +9 -0
- package/dist/services/runtime-kernel/wakeup-queue.d.ts.map +1 -0
- package/dist/services/runtime-kernel/wakeup-queue.js +19 -0
- package/dist/services/runtime-kernel/wakeup-queue.js.map +1 -0
- package/package.json +14 -14
- package/resources/bundled-skills/rudder-create-agent/SKILL.md +1 -1
- package/resources/bundled-skills/rudder-create-agent/references/api-reference.md +4 -2
- package/resources/bundled-skills/rudder-create-agent/references/cli-reference.md +1 -1
- package/skills/rudder-create-agent/SKILL.md +1 -1
- package/skills/rudder-create-agent/references/api-reference.md +4 -2
- package/skills/rudder-create-agent/references/cli-reference.md +1 -1
- package/ui-dist/assets/{_basePickBy-C8ZhuJlI.js → _basePickBy-DAU2OOE3.js} +1 -1
- package/ui-dist/assets/{_baseUniq-BRCVYeJb.js → _baseUniq-BQLJfIXV.js} +1 -1
- package/ui-dist/assets/{arc-B1R0dxYm.js → arc-B2UTEkTB.js} +1 -1
- package/ui-dist/assets/{architectureDiagram-2XIMDMQ5-D7sRWBaf.js → architectureDiagram-2XIMDMQ5-DicHsSOp.js} +1 -1
- package/ui-dist/assets/{blockDiagram-WCTKOSBZ-Byw6t-Eq.js → blockDiagram-WCTKOSBZ-tbkC8ZCs.js} +1 -1
- package/ui-dist/assets/{c4Diagram-IC4MRINW-RsrU80Hc.js → c4Diagram-IC4MRINW-pPT_Xw2l.js} +1 -1
- package/ui-dist/assets/channel-D8qCsVGC.js +1 -0
- package/ui-dist/assets/{chunk-4BX2VUAB-D9oP4FJ_.js → chunk-4BX2VUAB-vX_EgGXp.js} +1 -1
- package/ui-dist/assets/{chunk-55IACEB6-DT8xQAKU.js → chunk-55IACEB6-DUyGlyCg.js} +1 -1
- package/ui-dist/assets/{chunk-FMBD7UC4-FwqsP6AV.js → chunk-FMBD7UC4-DPCYqV54.js} +1 -1
- package/ui-dist/assets/{chunk-JSJVCQXG-CTOvJxU3.js → chunk-JSJVCQXG-CRqpXPJG.js} +1 -1
- package/ui-dist/assets/{chunk-KX2RTZJC-8oze7Khf.js → chunk-KX2RTZJC-BWaSxc23.js} +1 -1
- package/ui-dist/assets/{chunk-NQ4KR5QH-D38LrSCR.js → chunk-NQ4KR5QH-BCIl9FTX.js} +1 -1
- package/ui-dist/assets/{chunk-QZHKN3VN-B3ZhJA7D.js → chunk-QZHKN3VN-BgTqyV62.js} +1 -1
- package/ui-dist/assets/{chunk-WL4C6EOR-CSTjCHX6.js → chunk-WL4C6EOR-COlN9kxA.js} +1 -1
- package/ui-dist/assets/classDiagram-VBA2DB6C-Cm8BRWIa.js +1 -0
- package/ui-dist/assets/classDiagram-v2-RAHNMMFH-Cm8BRWIa.js +1 -0
- package/ui-dist/assets/clone-B0pIfwDy.js +1 -0
- package/ui-dist/assets/{cose-bilkent-S5V4N54A-CI_3Qz62.js → cose-bilkent-S5V4N54A-Bbgtq4rD.js} +1 -1
- package/ui-dist/assets/{dagre-KLK3FWXG-BNakeJ5Q.js → dagre-KLK3FWXG-DgUnjcLS.js} +1 -1
- package/ui-dist/assets/{diagram-E7M64L7V-UubVu-IN.js → diagram-E7M64L7V-DC3DIXbu.js} +1 -1
- package/ui-dist/assets/{diagram-IFDJBPK2-DRu8FSPT.js → diagram-IFDJBPK2-qv7Ij_rH.js} +1 -1
- package/ui-dist/assets/{diagram-P4PSJMXO-t2_nA6WC.js → diagram-P4PSJMXO-Dw2WzhiN.js} +1 -1
- package/ui-dist/assets/{erDiagram-INFDFZHY-ET8QDHtO.js → erDiagram-INFDFZHY-BV5s3J3D.js} +1 -1
- package/ui-dist/assets/{flowDiagram-PKNHOUZH-QXQeM_4d.js → flowDiagram-PKNHOUZH-kmPgde9o.js} +1 -1
- package/ui-dist/assets/{ganttDiagram-A5KZAMGK-B6yFn5Hd.js → ganttDiagram-A5KZAMGK-C2hkJ2Th.js} +1 -1
- package/ui-dist/assets/{gitGraphDiagram-K3NZZRJ6-DZAeG6MG.js → gitGraphDiagram-K3NZZRJ6-K5e1NgIt.js} +1 -1
- package/ui-dist/assets/{graph-BaTTyfU1.js → graph-Cna3FnWY.js} +1 -1
- package/ui-dist/assets/{index-C7kEQtA_.js → index-6luhu7W5.js} +1 -1
- package/ui-dist/assets/index-B1iX9NWL.js +1364 -0
- package/ui-dist/assets/{index-D-mb6cn2.js → index-BdLts2oZ.js} +1 -1
- package/ui-dist/assets/{index-ym7ZDmXE.js → index-CBKnHGYE.js} +1 -1
- package/ui-dist/assets/{index-D_ZvBEXt.js → index-CDtBE3Ga.js} +1 -1
- package/ui-dist/assets/{index-Cftoq4bF.js → index-CFGJHREd.js} +1 -1
- package/ui-dist/assets/{index-Bo1sSJ7x.js → index-CTpC-gpc.js} +1 -1
- package/ui-dist/assets/{index-h_UqLbty.js → index-CVAmlnRr.js} +1 -1
- package/ui-dist/assets/{index-BWEA-ibM.js → index-CWbJ-pcg.js} +1 -1
- package/ui-dist/assets/{index-BSfIb9qw.js → index-ChU0nSCq.js} +1 -1
- package/ui-dist/assets/{index-DQeEMbWr.js → index-CypGJ__o.js} +1 -1
- package/ui-dist/assets/{index-BXxi8m3U.js → index-D-Fe0jjS.js} +1 -1
- package/ui-dist/assets/{index-BGOxzMvq.js → index-D8aBm25c.js} +1 -1
- package/ui-dist/assets/index-D9uKiQrM.css +1 -0
- package/ui-dist/assets/{index-DBsb_N5Q.js → index-DD0ggVK_.js} +1 -1
- package/ui-dist/assets/{index-MYVA1f40.js → index-DJecNMv2.js} +1 -1
- package/ui-dist/assets/{index-CAcJz5d9.js → index-DKSfpiAJ.js} +1 -1
- package/ui-dist/assets/{index-ZHLqhZdz.js → index-Dgj2bRIV.js} +1 -1
- package/ui-dist/assets/{index-DvQZ5FJk.js → index-DsydT3bX.js} +1 -1
- package/ui-dist/assets/{index-qVKE_HM2.js → index-U7gGuiCe.js} +1 -1
- package/ui-dist/assets/{index-CRtxHtSg.js → index-aL5hFrtS.js} +1 -1
- package/ui-dist/assets/{index-DPbDGs74.js → index-o_iEnzyw.js} +1 -1
- package/ui-dist/assets/{index-eessTUZm.js → index-qUFX4kXb.js} +1 -1
- package/ui-dist/assets/{index-C2T1FM7J.js → index-sV3mB3fg.js} +1 -1
- package/ui-dist/assets/{infoDiagram-LFFYTUFH-BOnKcq7S.js → infoDiagram-LFFYTUFH-BiQ6_Z3F.js} +1 -1
- package/ui-dist/assets/{ishikawaDiagram-PHBUUO56-etV-MroN.js → ishikawaDiagram-PHBUUO56-C_WqRu1B.js} +1 -1
- package/ui-dist/assets/{journeyDiagram-4ABVD52K-mvjGNrBx.js → journeyDiagram-4ABVD52K-BIpPercT.js} +1 -1
- package/ui-dist/assets/{kanban-definition-K7BYSVSG-CW-Dw_bU.js → kanban-definition-K7BYSVSG-DaFD_JyB.js} +1 -1
- package/ui-dist/assets/{layout-DUcWZ3H3.js → layout-Dj_oike0.js} +1 -1
- package/ui-dist/assets/{linear-B0r1V0oG.js → linear-tnVt1ugU.js} +1 -1
- package/ui-dist/assets/{mermaid.core-Bo_YuNee.js → mermaid.core-DflBh7Hv.js} +4 -4
- package/ui-dist/assets/{mindmap-definition-YRQLILUH-BmEjkBnz.js → mindmap-definition-YRQLILUH-C3H-gFSh.js} +1 -1
- package/ui-dist/assets/{pieDiagram-SKSYHLDU-BRER3VVx.js → pieDiagram-SKSYHLDU-DH53odzN.js} +1 -1
- package/ui-dist/assets/{quadrantDiagram-337W2JSQ-oYnSbQKi.js → quadrantDiagram-337W2JSQ-DCn4Dric.js} +1 -1
- package/ui-dist/assets/{requirementDiagram-Z7DCOOCP-BRLFZsts.js → requirementDiagram-Z7DCOOCP-KypOfBcY.js} +1 -1
- package/ui-dist/assets/{sankeyDiagram-WA2Y5GQK-YOKeMxIU.js → sankeyDiagram-WA2Y5GQK-DEmi0ryr.js} +1 -1
- package/ui-dist/assets/{sequenceDiagram-2WXFIKYE-Bj-YKQMs.js → sequenceDiagram-2WXFIKYE-sW6ojsVG.js} +1 -1
- package/ui-dist/assets/{stateDiagram-RAJIS63D-BYf-CeWe.js → stateDiagram-RAJIS63D-CmrX9BEC.js} +1 -1
- package/ui-dist/assets/stateDiagram-v2-FVOUBMTO-DJuCnZbk.js +1 -0
- package/ui-dist/assets/{timeline-definition-YZTLITO2-CG6r3GP4.js → timeline-definition-YZTLITO2-CzJ5Y8YU.js} +1 -1
- package/ui-dist/assets/{treemap-KZPCXAKY-DCX4FLGH.js → treemap-KZPCXAKY-BwUNliTA.js} +1 -1
- package/ui-dist/assets/{vennDiagram-LZ73GAT5-CpbbhHdY.js → vennDiagram-LZ73GAT5-5A50aIn1.js} +1 -1
- package/ui-dist/assets/{xychartDiagram-JWTSCODW-Biq5EV_L.js → xychartDiagram-JWTSCODW-D9qTFskE.js} +1 -1
- package/ui-dist/index.html +2 -2
- package/ui-dist/assets/channel-BExT12GI.js +0 -1
- package/ui-dist/assets/classDiagram-VBA2DB6C-B3FSZmLP.js +0 -1
- package/ui-dist/assets/classDiagram-v2-RAHNMMFH-B3FSZmLP.js +0 -1
- package/ui-dist/assets/clone-DxXNeJhJ.js +0 -1
- package/ui-dist/assets/index-2Hx2TlnA.css +0 -1
- package/ui-dist/assets/index-wzjO7KZw.js +0 -1343
- package/ui-dist/assets/stateDiagram-v2-FVOUBMTO-g0byx18e.js +0 -1
|
@@ -0,0 +1,4044 @@
|
|
|
1
|
+
import { and, asc, desc, eq, gt, gte, inArray, lte, sql } from "drizzle-orm";
|
|
2
|
+
import { agents, agentRuntimeState, agentTaskSessions, agentWakeupRequests, activityLog, heartbeatRunEvents, heartbeatRuns, issueComments, issues, organizations, projects, } from "@rudderhq/db";
|
|
3
|
+
import { conflict, notFound } from "../../errors.js";
|
|
4
|
+
import { createExecutionScores, observeExecutionEvent, updateExecutionObservation, updateExecutionTraceIO, updateExecutionTraceName, updateExecutionTraceSession, withExecutionObservation, } from "../../langfuse.js";
|
|
5
|
+
import { emitExecutionTranscriptTree } from "../../langfuse-transcript.js";
|
|
6
|
+
import { logger } from "../../middleware/logger.js";
|
|
7
|
+
import { publishLiveEvent } from "../live-events.js";
|
|
8
|
+
import { getRunLogStore } from "../run-log-store.js";
|
|
9
|
+
import { findServerAdapter, getServerAdapter, runningProcesses } from "../../agent-runtimes/index.js";
|
|
10
|
+
import { createLocalAgentJwt } from "../../agent-auth-jwt.js";
|
|
11
|
+
import { parseObject, asNumber } from "../../agent-runtimes/utils.js";
|
|
12
|
+
import { costService } from "../costs.js";
|
|
13
|
+
import { budgetService } from "../budgets.js";
|
|
14
|
+
import { agentRunContextService, } from "../agent-run-context.js";
|
|
15
|
+
import { resolveDefaultAgentWorkspaceDir, } from "../../home-paths.js";
|
|
16
|
+
import { summarizeHeartbeatRunResultJson } from "../heartbeat-run-summary.js";
|
|
17
|
+
import { buildWorkspaceReadyComment, cleanupExecutionWorkspaceArtifacts, ensureRuntimeServicesForRun, persistAdapterManagedRuntimeServices, realizeExecutionWorkspace, releaseRuntimeServicesForRun, } from "../workspace-runtime.js";
|
|
18
|
+
import { issueService } from "../issues.js";
|
|
19
|
+
import { executionWorkspaceService } from "../execution-workspaces.js";
|
|
20
|
+
import { buildObservedRunLangfuseScores } from "../run-intelligence.js";
|
|
21
|
+
import { workspaceOperationService } from "../workspace-operations.js";
|
|
22
|
+
import { buildExecutionWorkspaceAdapterConfig, issueExecutionWorkspaceModeForPersistedWorkspace, parseIssueExecutionWorkspaceSettings, parseProjectExecutionWorkspacePolicy, resolveExecutionWorkspaceMode, } from "../execution-workspace-policy.js";
|
|
23
|
+
import { instanceSettingsService } from "../instance-settings.js";
|
|
24
|
+
import { logActivity } from "../activity-log.js";
|
|
25
|
+
import { redactCurrentUserText, redactCurrentUserValue } from "../../log-redaction.js";
|
|
26
|
+
import { hasSessionCompactionThresholds, } from "@rudderhq/agent-runtime-utils";
|
|
27
|
+
import { buildCreateAgentBenchmarkTags, coerceCreateAgentBenchmarkMetadata, extractCreateAgentBenchmarkMetadata, } from "@rudderhq/run-intelligence-core";
|
|
28
|
+
import { executeAdapterWithModelFallbacks } from "./model-fallback.js";
|
|
29
|
+
import { normalizeAgentNameKey, readNonEmptyString, truncateDisplayId, } from "./common.js";
|
|
30
|
+
import { buildDateKeysBetween, buildRecentDateKeys, normalizeLoadedSkill, } from "./analytics.js";
|
|
31
|
+
import { buildHeartbeatAdapterInvokePayload, buildHeartbeatObservationName, buildHeartbeatRuntimeTraceMetadata, buildIssueRunTraceName, resolveHeartbeatObservabilitySurface, } from "./run-state.js";
|
|
32
|
+
import { buildExplicitResumeSessionOverride, defaultSessionCodec, formatRuntimeWorkspaceWarningLog, normalizeSessionParams, parseSessionCompactionPolicy, resolveRuntimeSessionParamsForWorkspace, shouldResetTaskSessionForWake, } from "./session-policy.js";
|
|
33
|
+
import { appendExcerpt, appendTranscriptEntriesFromChunk, } from "./execution.js";
|
|
34
|
+
import { isProcessAlive, isTrackedLocalChildProcessAdapter, terminateOrphanedProcess, } from "./recovery.js";
|
|
35
|
+
import { parseHeartbeatPolicy } from "./wakeup-queue.js";
|
|
36
|
+
export { prioritizeProjectWorkspaceCandidatesForRun } from "../agent-run-context.js";
|
|
37
|
+
export { buildDateKeysBetween, buildRecentDateKeys, normalizeLoadedSkill, } from "./analytics.js";
|
|
38
|
+
export { normalizeAgentNameKey, readNonEmptyString, truncateDisplayId, } from "./common.js";
|
|
39
|
+
export { buildHeartbeatAdapterInvokePayload, buildHeartbeatRuntimeTraceMetadata, buildIssueRunTraceName, resolveHeartbeatObservabilitySurface, } from "./run-state.js";
|
|
40
|
+
export { buildExplicitResumeSessionOverride, formatRuntimeWorkspaceWarningLog, parseSessionCompactionPolicy, resolveRuntimeSessionParamsForWorkspace, shouldResetTaskSessionForWake, } from "./session-policy.js";
|
|
41
|
+
export { appendExcerpt, appendTranscriptEntriesFromChunk, } from "./execution.js";
|
|
42
|
+
export { isProcessAlive, isTrackedLocalChildProcessAdapter, terminateOrphanedProcess, } from "./recovery.js";
|
|
43
|
+
export { normalizeMaxConcurrentRuns, parseHeartbeatPolicy } from "./wakeup-queue.js";
|
|
44
|
+
const MAX_LIVE_LOG_CHUNK_BYTES = 8 * 1024;
|
|
45
|
+
const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext";
|
|
46
|
+
const DETACHED_PROCESS_ERROR_CODE = "process_detached";
|
|
47
|
+
const startLocksByAgent = new Map();
|
|
48
|
+
const MAX_RECOVERY_CHAIN_DEPTH = 8;
|
|
49
|
+
const ISSUE_PASSIVE_FOLLOWUP_REASON = "issue_passive_followup";
|
|
50
|
+
const ISSUE_PASSIVE_FOLLOWUP_WAKE_SOURCE = "passive_issue_followup";
|
|
51
|
+
const ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON = "missing_closure";
|
|
52
|
+
const ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS = 2;
|
|
53
|
+
const ISSUE_PASSIVE_FOLLOWUP_COOLDOWN_MS_BY_ATTEMPT = new Map([
|
|
54
|
+
[1, 2 * 60 * 1000],
|
|
55
|
+
[2, 5 * 60 * 1000],
|
|
56
|
+
]);
|
|
57
|
+
const ISSUE_PASSIVE_FOLLOWUP_TIMER_CONTINUITY_MAX_WINDOW_MS = 15 * 60 * 1000;
|
|
58
|
+
const heartbeatRunListColumns = {
|
|
59
|
+
id: heartbeatRuns.id,
|
|
60
|
+
orgId: heartbeatRuns.orgId,
|
|
61
|
+
agentId: heartbeatRuns.agentId,
|
|
62
|
+
invocationSource: heartbeatRuns.invocationSource,
|
|
63
|
+
triggerDetail: heartbeatRuns.triggerDetail,
|
|
64
|
+
status: heartbeatRuns.status,
|
|
65
|
+
startedAt: heartbeatRuns.startedAt,
|
|
66
|
+
finishedAt: heartbeatRuns.finishedAt,
|
|
67
|
+
error: heartbeatRuns.error,
|
|
68
|
+
wakeupRequestId: heartbeatRuns.wakeupRequestId,
|
|
69
|
+
exitCode: heartbeatRuns.exitCode,
|
|
70
|
+
signal: heartbeatRuns.signal,
|
|
71
|
+
usageJson: heartbeatRuns.usageJson,
|
|
72
|
+
resultJson: heartbeatRuns.resultJson,
|
|
73
|
+
sessionIdBefore: heartbeatRuns.sessionIdBefore,
|
|
74
|
+
sessionIdAfter: heartbeatRuns.sessionIdAfter,
|
|
75
|
+
logStore: heartbeatRuns.logStore,
|
|
76
|
+
logRef: heartbeatRuns.logRef,
|
|
77
|
+
logBytes: heartbeatRuns.logBytes,
|
|
78
|
+
logSha256: heartbeatRuns.logSha256,
|
|
79
|
+
logCompressed: heartbeatRuns.logCompressed,
|
|
80
|
+
stdoutExcerpt: sql `NULL`.as("stdoutExcerpt"),
|
|
81
|
+
stderrExcerpt: sql `NULL`.as("stderrExcerpt"),
|
|
82
|
+
errorCode: heartbeatRuns.errorCode,
|
|
83
|
+
externalRunId: heartbeatRuns.externalRunId,
|
|
84
|
+
processPid: heartbeatRuns.processPid,
|
|
85
|
+
processStartedAt: heartbeatRuns.processStartedAt,
|
|
86
|
+
retryOfRunId: heartbeatRuns.retryOfRunId,
|
|
87
|
+
processLossRetryCount: heartbeatRuns.processLossRetryCount,
|
|
88
|
+
contextSnapshot: heartbeatRuns.contextSnapshot,
|
|
89
|
+
createdAt: heartbeatRuns.createdAt,
|
|
90
|
+
updatedAt: heartbeatRuns.updatedAt,
|
|
91
|
+
};
|
|
92
|
+
async function withAgentStartLock(agentId, fn) {
|
|
93
|
+
const previous = startLocksByAgent.get(agentId) ?? Promise.resolve();
|
|
94
|
+
const run = previous.then(fn);
|
|
95
|
+
const marker = run.then(() => undefined, () => undefined);
|
|
96
|
+
startLocksByAgent.set(agentId, marker);
|
|
97
|
+
try {
|
|
98
|
+
return await run;
|
|
99
|
+
}
|
|
100
|
+
finally {
|
|
101
|
+
if (startLocksByAgent.get(agentId) === marker) {
|
|
102
|
+
startLocksByAgent.delete(agentId);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
function normalizeLedgerBillingType(value) {
|
|
107
|
+
const raw = readNonEmptyString(value);
|
|
108
|
+
switch (raw) {
|
|
109
|
+
case "api":
|
|
110
|
+
case "metered_api":
|
|
111
|
+
return "metered_api";
|
|
112
|
+
case "subscription":
|
|
113
|
+
case "subscription_included":
|
|
114
|
+
return "subscription_included";
|
|
115
|
+
case "subscription_overage":
|
|
116
|
+
return "subscription_overage";
|
|
117
|
+
case "credits":
|
|
118
|
+
return "credits";
|
|
119
|
+
case "fixed":
|
|
120
|
+
return "fixed";
|
|
121
|
+
default:
|
|
122
|
+
return "unknown";
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
function resolveLedgerBiller(result) {
|
|
126
|
+
return readNonEmptyString(result.biller) ?? readNonEmptyString(result.provider) ?? "unknown";
|
|
127
|
+
}
|
|
128
|
+
function normalizeBilledCostCents(costUsd, billingType) {
|
|
129
|
+
if (billingType === "subscription_included")
|
|
130
|
+
return 0;
|
|
131
|
+
if (typeof costUsd !== "number" || !Number.isFinite(costUsd))
|
|
132
|
+
return 0;
|
|
133
|
+
return Math.max(0, Math.round(costUsd * 100));
|
|
134
|
+
}
|
|
135
|
+
async function resolveLedgerScopeForRun(db, orgId, run) {
|
|
136
|
+
const context = parseObject(run.contextSnapshot);
|
|
137
|
+
const contextIssueId = readNonEmptyString(context.issueId);
|
|
138
|
+
const contextProjectId = readNonEmptyString(context.projectId);
|
|
139
|
+
if (!contextIssueId) {
|
|
140
|
+
return {
|
|
141
|
+
issueId: null,
|
|
142
|
+
projectId: contextProjectId,
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
const issue = await db
|
|
146
|
+
.select({
|
|
147
|
+
id: issues.id,
|
|
148
|
+
projectId: issues.projectId,
|
|
149
|
+
})
|
|
150
|
+
.from(issues)
|
|
151
|
+
.where(and(eq(issues.id, contextIssueId), eq(issues.orgId, orgId)))
|
|
152
|
+
.then((rows) => rows[0] ?? null);
|
|
153
|
+
return {
|
|
154
|
+
issueId: issue?.id ?? null,
|
|
155
|
+
projectId: issue?.projectId ?? contextProjectId,
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
function normalizeUsageTotals(usage) {
|
|
159
|
+
if (!usage)
|
|
160
|
+
return null;
|
|
161
|
+
return {
|
|
162
|
+
inputTokens: Math.max(0, Math.floor(asNumber(usage.inputTokens, 0))),
|
|
163
|
+
cachedInputTokens: Math.max(0, Math.floor(asNumber(usage.cachedInputTokens, 0))),
|
|
164
|
+
outputTokens: Math.max(0, Math.floor(asNumber(usage.outputTokens, 0))),
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
function readRawUsageTotals(usageJson) {
|
|
168
|
+
const parsed = parseObject(usageJson);
|
|
169
|
+
if (Object.keys(parsed).length === 0)
|
|
170
|
+
return null;
|
|
171
|
+
const inputTokens = Math.max(0, Math.floor(asNumber(parsed.rawInputTokens, asNumber(parsed.inputTokens, 0))));
|
|
172
|
+
const cachedInputTokens = Math.max(0, Math.floor(asNumber(parsed.rawCachedInputTokens, asNumber(parsed.cachedInputTokens, 0))));
|
|
173
|
+
const outputTokens = Math.max(0, Math.floor(asNumber(parsed.rawOutputTokens, asNumber(parsed.outputTokens, 0))));
|
|
174
|
+
if (inputTokens <= 0 && cachedInputTokens <= 0 && outputTokens <= 0) {
|
|
175
|
+
return null;
|
|
176
|
+
}
|
|
177
|
+
return {
|
|
178
|
+
inputTokens,
|
|
179
|
+
cachedInputTokens,
|
|
180
|
+
outputTokens,
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
function deriveNormalizedUsageDelta(current, previous) {
|
|
184
|
+
if (!current)
|
|
185
|
+
return null;
|
|
186
|
+
if (!previous)
|
|
187
|
+
return { ...current };
|
|
188
|
+
const inputTokens = current.inputTokens >= previous.inputTokens
|
|
189
|
+
? current.inputTokens - previous.inputTokens
|
|
190
|
+
: current.inputTokens;
|
|
191
|
+
const cachedInputTokens = current.cachedInputTokens >= previous.cachedInputTokens
|
|
192
|
+
? current.cachedInputTokens - previous.cachedInputTokens
|
|
193
|
+
: current.cachedInputTokens;
|
|
194
|
+
const outputTokens = current.outputTokens >= previous.outputTokens
|
|
195
|
+
? current.outputTokens - previous.outputTokens
|
|
196
|
+
: current.outputTokens;
|
|
197
|
+
return {
|
|
198
|
+
inputTokens: Math.max(0, inputTokens),
|
|
199
|
+
cachedInputTokens: Math.max(0, cachedInputTokens),
|
|
200
|
+
outputTokens: Math.max(0, outputTokens),
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
function formatCount(value) {
|
|
204
|
+
if (typeof value !== "number" || !Number.isFinite(value))
|
|
205
|
+
return "0";
|
|
206
|
+
return value.toLocaleString("en-US");
|
|
207
|
+
}
|
|
208
|
+
function parseIssueAssigneeAgentRuntimeOverrides(raw) {
|
|
209
|
+
const parsed = parseObject(raw);
|
|
210
|
+
const parsedAdapterConfig = parseObject(parsed.agentRuntimeConfig);
|
|
211
|
+
const agentRuntimeConfig = Object.keys(parsedAdapterConfig).length > 0 ? parsedAdapterConfig : null;
|
|
212
|
+
const useProjectWorkspace = typeof parsed.useProjectWorkspace === "boolean"
|
|
213
|
+
? parsed.useProjectWorkspace
|
|
214
|
+
: null;
|
|
215
|
+
if (!agentRuntimeConfig && useProjectWorkspace === null)
|
|
216
|
+
return null;
|
|
217
|
+
return {
|
|
218
|
+
agentRuntimeConfig,
|
|
219
|
+
useProjectWorkspace,
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
function deriveTaskKey(contextSnapshot, payload) {
|
|
223
|
+
return (readNonEmptyString(contextSnapshot?.taskKey) ??
|
|
224
|
+
readNonEmptyString(contextSnapshot?.taskId) ??
|
|
225
|
+
readNonEmptyString(contextSnapshot?.issueId) ??
|
|
226
|
+
readNonEmptyString(payload?.taskKey) ??
|
|
227
|
+
readNonEmptyString(payload?.taskId) ??
|
|
228
|
+
readNonEmptyString(payload?.issueId) ??
|
|
229
|
+
null);
|
|
230
|
+
}
|
|
231
|
+
function describeSessionResetReason(contextSnapshot) {
|
|
232
|
+
if (contextSnapshot?.forceFreshSession === true)
|
|
233
|
+
return "forceFreshSession was requested";
|
|
234
|
+
const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason);
|
|
235
|
+
if (wakeReason === "issue_assigned")
|
|
236
|
+
return "wake reason is issue_assigned";
|
|
237
|
+
return null;
|
|
238
|
+
}
|
|
239
|
+
function deriveCommentId(contextSnapshot, payload) {
|
|
240
|
+
return (readNonEmptyString(contextSnapshot?.wakeCommentId) ??
|
|
241
|
+
readNonEmptyString(contextSnapshot?.commentId) ??
|
|
242
|
+
readNonEmptyString(payload?.commentId) ??
|
|
243
|
+
null);
|
|
244
|
+
}
|
|
245
|
+
function enrichWakeContextSnapshot(input) {
|
|
246
|
+
const { contextSnapshot, reason, source, triggerDetail, payload } = input;
|
|
247
|
+
const issueIdFromPayload = readNonEmptyString(payload?.["issueId"]);
|
|
248
|
+
const commentIdFromPayload = readNonEmptyString(payload?.["commentId"]);
|
|
249
|
+
const taskKey = deriveTaskKey(contextSnapshot, payload);
|
|
250
|
+
const wakeCommentId = deriveCommentId(contextSnapshot, payload);
|
|
251
|
+
if (!readNonEmptyString(contextSnapshot["wakeReason"]) && reason) {
|
|
252
|
+
contextSnapshot.wakeReason = reason;
|
|
253
|
+
}
|
|
254
|
+
if (!readNonEmptyString(contextSnapshot["issueId"]) && issueIdFromPayload) {
|
|
255
|
+
contextSnapshot.issueId = issueIdFromPayload;
|
|
256
|
+
}
|
|
257
|
+
if (!readNonEmptyString(contextSnapshot["taskId"]) && issueIdFromPayload) {
|
|
258
|
+
contextSnapshot.taskId = issueIdFromPayload;
|
|
259
|
+
}
|
|
260
|
+
if (!readNonEmptyString(contextSnapshot["taskKey"]) && taskKey) {
|
|
261
|
+
contextSnapshot.taskKey = taskKey;
|
|
262
|
+
}
|
|
263
|
+
if (!readNonEmptyString(contextSnapshot["commentId"]) && commentIdFromPayload) {
|
|
264
|
+
contextSnapshot.commentId = commentIdFromPayload;
|
|
265
|
+
}
|
|
266
|
+
if (!readNonEmptyString(contextSnapshot["wakeCommentId"]) && wakeCommentId) {
|
|
267
|
+
contextSnapshot.wakeCommentId = wakeCommentId;
|
|
268
|
+
}
|
|
269
|
+
if (!readNonEmptyString(contextSnapshot["wakeSource"]) && source) {
|
|
270
|
+
contextSnapshot.wakeSource = source;
|
|
271
|
+
}
|
|
272
|
+
if (!readNonEmptyString(contextSnapshot["wakeTriggerDetail"]) && triggerDetail) {
|
|
273
|
+
contextSnapshot.wakeTriggerDetail = triggerDetail;
|
|
274
|
+
}
|
|
275
|
+
return {
|
|
276
|
+
contextSnapshot,
|
|
277
|
+
issueIdFromPayload,
|
|
278
|
+
commentIdFromPayload,
|
|
279
|
+
taskKey,
|
|
280
|
+
wakeCommentId,
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
function mergeCoalescedContextSnapshot(existingRaw, incoming) {
|
|
284
|
+
const existing = parseObject(existingRaw);
|
|
285
|
+
const merged = {
|
|
286
|
+
...existing,
|
|
287
|
+
...incoming,
|
|
288
|
+
};
|
|
289
|
+
const commentId = deriveCommentId(incoming, null);
|
|
290
|
+
if (commentId) {
|
|
291
|
+
merged.commentId = commentId;
|
|
292
|
+
merged.wakeCommentId = commentId;
|
|
293
|
+
}
|
|
294
|
+
return merged;
|
|
295
|
+
}
|
|
296
|
+
function buildDeferredWakePayload(payload, contextSnapshot, issueId) {
|
|
297
|
+
const deferredPayload = { ...(payload ?? {}) };
|
|
298
|
+
if (issueId && !readNonEmptyString(deferredPayload.issueId)) {
|
|
299
|
+
deferredPayload.issueId = issueId;
|
|
300
|
+
}
|
|
301
|
+
deferredPayload[DEFERRED_WAKE_CONTEXT_KEY] = contextSnapshot;
|
|
302
|
+
return deferredPayload;
|
|
303
|
+
}
|
|
304
|
+
function readDeferredWakeContext(payloadRaw) {
|
|
305
|
+
const payload = parseObject(payloadRaw);
|
|
306
|
+
return parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]);
|
|
307
|
+
}
|
|
308
|
+
function readDeferredWakePayload(payloadRaw) {
|
|
309
|
+
const payload = parseObject(payloadRaw);
|
|
310
|
+
delete payload[DEFERRED_WAKE_CONTEXT_KEY];
|
|
311
|
+
return payload;
|
|
312
|
+
}
|
|
313
|
+
function deriveDeferredWakeTaskKey(payloadRaw) {
|
|
314
|
+
const payload = readDeferredWakePayload(payloadRaw);
|
|
315
|
+
const contextSnapshot = readDeferredWakeContext(payloadRaw);
|
|
316
|
+
return deriveTaskKey(contextSnapshot, payload);
|
|
317
|
+
}
|
|
318
|
+
async function hydrateWakeContextSnapshot(db, orgId, contextSnapshot) {
|
|
319
|
+
const issueId = readNonEmptyString(contextSnapshot.issueId);
|
|
320
|
+
const commentId = deriveCommentId(contextSnapshot, null);
|
|
321
|
+
const issueContext = parseObject(contextSnapshot.issue);
|
|
322
|
+
const commentContext = parseObject(contextSnapshot.comment);
|
|
323
|
+
const needsIssueContext = !!issueId &&
|
|
324
|
+
(!readNonEmptyString(issueContext.id) ||
|
|
325
|
+
!readNonEmptyString(issueContext.title) ||
|
|
326
|
+
!readNonEmptyString(issueContext.status) ||
|
|
327
|
+
!("priority" in issueContext) ||
|
|
328
|
+
!("description" in issueContext));
|
|
329
|
+
const needsProjectId = !!issueId && !readNonEmptyString(contextSnapshot.projectId);
|
|
330
|
+
const needsCommentContext = !!commentId &&
|
|
331
|
+
(!readNonEmptyString(commentContext.id) ||
|
|
332
|
+
!readNonEmptyString(commentContext.body));
|
|
333
|
+
if (!needsIssueContext && !needsProjectId && !needsCommentContext)
|
|
334
|
+
return;
|
|
335
|
+
if (issueId && (needsIssueContext || needsProjectId)) {
|
|
336
|
+
const issueRow = await db
|
|
337
|
+
.select({
|
|
338
|
+
id: issues.id,
|
|
339
|
+
title: issues.title,
|
|
340
|
+
description: issues.description,
|
|
341
|
+
status: issues.status,
|
|
342
|
+
priority: issues.priority,
|
|
343
|
+
projectId: issues.projectId,
|
|
344
|
+
})
|
|
345
|
+
.from(issues)
|
|
346
|
+
.where(and(eq(issues.id, issueId), eq(issues.orgId, orgId)))
|
|
347
|
+
.then((rows) => rows[0] ?? null);
|
|
348
|
+
if (issueRow) {
|
|
349
|
+
contextSnapshot.issue = {
|
|
350
|
+
...issueContext,
|
|
351
|
+
id: readNonEmptyString(issueContext.id) ?? issueRow.id,
|
|
352
|
+
title: readNonEmptyString(issueContext.title) ?? issueRow.title,
|
|
353
|
+
description: "description" in issueContext ? issueContext.description : issueRow.description,
|
|
354
|
+
status: readNonEmptyString(issueContext.status) ?? issueRow.status,
|
|
355
|
+
priority: "priority" in issueContext ? issueContext.priority : issueRow.priority,
|
|
356
|
+
};
|
|
357
|
+
if (!readNonEmptyString(contextSnapshot.projectId) && issueRow.projectId) {
|
|
358
|
+
contextSnapshot.projectId = issueRow.projectId;
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
if (commentId && needsCommentContext) {
|
|
363
|
+
const commentConditions = [eq(issueComments.id, commentId), eq(issueComments.orgId, orgId)];
|
|
364
|
+
if (issueId) {
|
|
365
|
+
commentConditions.push(eq(issueComments.issueId, issueId));
|
|
366
|
+
}
|
|
367
|
+
const commentRow = await db
|
|
368
|
+
.select({
|
|
369
|
+
id: issueComments.id,
|
|
370
|
+
body: issueComments.body,
|
|
371
|
+
authorAgentId: issueComments.authorAgentId,
|
|
372
|
+
authorUserId: issueComments.authorUserId,
|
|
373
|
+
})
|
|
374
|
+
.from(issueComments)
|
|
375
|
+
.where(and(...commentConditions))
|
|
376
|
+
.then((rows) => rows[0] ?? null);
|
|
377
|
+
if (commentRow) {
|
|
378
|
+
contextSnapshot.comment = {
|
|
379
|
+
...commentContext,
|
|
380
|
+
id: readNonEmptyString(commentContext.id) ?? commentRow.id,
|
|
381
|
+
body: readNonEmptyString(commentContext.body) ?? commentRow.body,
|
|
382
|
+
authorAgentId: "authorAgentId" in commentContext ? commentContext.authorAgentId : commentRow.authorAgentId,
|
|
383
|
+
authorUserId: "authorUserId" in commentContext ? commentContext.authorUserId : commentRow.authorUserId,
|
|
384
|
+
};
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
function firstNonEmptyLine(value) {
|
|
389
|
+
if (typeof value !== "string")
|
|
390
|
+
return null;
|
|
391
|
+
const line = value
|
|
392
|
+
.split("\n")
|
|
393
|
+
.map((chunk) => chunk.trim())
|
|
394
|
+
.find(Boolean);
|
|
395
|
+
return line ?? null;
|
|
396
|
+
}
|
|
397
|
+
function deriveRecoveryFailureKind(run) {
|
|
398
|
+
return (readNonEmptyString(run.errorCode) ??
|
|
399
|
+
(run.status === "timed_out" ? "timed_out" : null) ??
|
|
400
|
+
run.status);
|
|
401
|
+
}
|
|
402
|
+
function deriveRecoveryFailureSummary(run) {
|
|
403
|
+
return (firstNonEmptyLine(run.error) ??
|
|
404
|
+
firstNonEmptyLine(run.stderrExcerpt) ??
|
|
405
|
+
firstNonEmptyLine(run.stdoutExcerpt) ??
|
|
406
|
+
(run.status === "timed_out" ? "The run timed out before it completed." : null) ??
|
|
407
|
+
"The previous run failed before it completed.");
|
|
408
|
+
}
|
|
409
|
+
function mergeMissingRecoveryContextFields(target, source) {
|
|
410
|
+
const keysToBackfill = [
|
|
411
|
+
"issueId",
|
|
412
|
+
"taskId",
|
|
413
|
+
"taskKey",
|
|
414
|
+
"projectId",
|
|
415
|
+
"projectWorkspaceId",
|
|
416
|
+
"commentId",
|
|
417
|
+
"wakeCommentId",
|
|
418
|
+
"issue",
|
|
419
|
+
"comment",
|
|
420
|
+
"source",
|
|
421
|
+
"wakeSource",
|
|
422
|
+
"wakeTriggerDetail",
|
|
423
|
+
];
|
|
424
|
+
for (const key of keysToBackfill) {
|
|
425
|
+
if (!(key in target) || target[key] === null || target[key] === undefined || target[key] === "") {
|
|
426
|
+
const value = source[key];
|
|
427
|
+
if (value !== null && value !== undefined && value !== "") {
|
|
428
|
+
target[key] = value;
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
async function hydrateRecoveryBaseContextSnapshot(run, getRunById) {
|
|
434
|
+
const mergedContext = { ...parseObject(run.contextSnapshot) };
|
|
435
|
+
let ancestorRunId = readNonEmptyString(run.retryOfRunId);
|
|
436
|
+
let depth = 0;
|
|
437
|
+
while (ancestorRunId && depth < MAX_RECOVERY_CHAIN_DEPTH) {
|
|
438
|
+
const ancestorRun = await getRunById(ancestorRunId);
|
|
439
|
+
if (!ancestorRun)
|
|
440
|
+
break;
|
|
441
|
+
mergeMissingRecoveryContextFields(mergedContext, parseObject(ancestorRun.contextSnapshot));
|
|
442
|
+
ancestorRunId = readNonEmptyString(ancestorRun.retryOfRunId);
|
|
443
|
+
depth += 1;
|
|
444
|
+
}
|
|
445
|
+
return mergedContext;
|
|
446
|
+
}
|
|
447
|
+
function buildRecoveryContextSnapshot(input) {
|
|
448
|
+
const { baseContextSnapshot, run, recoveryTrigger, wakeReason, wakeSource, triggerDetail } = input;
|
|
449
|
+
const failureKind = deriveRecoveryFailureKind(run);
|
|
450
|
+
const failureSummary = deriveRecoveryFailureSummary(run);
|
|
451
|
+
const recovery = {
|
|
452
|
+
originalRunId: run.id,
|
|
453
|
+
failureKind,
|
|
454
|
+
failureSummary,
|
|
455
|
+
recoveryTrigger,
|
|
456
|
+
recoveryMode: "continue_preferred",
|
|
457
|
+
};
|
|
458
|
+
return {
|
|
459
|
+
...baseContextSnapshot,
|
|
460
|
+
wakeReason,
|
|
461
|
+
wakeSource,
|
|
462
|
+
wakeTriggerDetail: triggerDetail,
|
|
463
|
+
retryOfRunId: run.id,
|
|
464
|
+
retryReason: failureKind,
|
|
465
|
+
recovery,
|
|
466
|
+
};
|
|
467
|
+
}
|
|
468
|
+
function normalizePassiveFollowupContext(raw) {
|
|
469
|
+
const parsed = parseObject(raw);
|
|
470
|
+
const originRunId = readNonEmptyString(parsed.originRunId);
|
|
471
|
+
if (!originRunId)
|
|
472
|
+
return null;
|
|
473
|
+
const attempt = Math.max(0, Math.floor(asNumber(parsed.attempt, 0)));
|
|
474
|
+
return {
|
|
475
|
+
originRunId,
|
|
476
|
+
previousRunId: readNonEmptyString(parsed.previousRunId),
|
|
477
|
+
attempt,
|
|
478
|
+
maxAttempts: Math.max(1, Math.floor(asNumber(parsed.maxAttempts, ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS))),
|
|
479
|
+
reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
|
|
480
|
+
queuedAt: readNonEmptyString(parsed.queuedAt),
|
|
481
|
+
};
|
|
482
|
+
}
|
|
483
|
+
function passiveFollowupCooldownMs(attempt) {
|
|
484
|
+
return ISSUE_PASSIVE_FOLLOWUP_COOLDOWN_MS_BY_ATTEMPT.get(attempt) ?? 5 * 60 * 1000;
|
|
485
|
+
}
|
|
486
|
+
function isAgentEligibleForTimerContinuation(agent) {
|
|
487
|
+
return (agent.status !== "paused" &&
|
|
488
|
+
agent.status !== "terminated" &&
|
|
489
|
+
agent.status !== "pending_approval");
|
|
490
|
+
}
|
|
491
|
+
function hasCredibleTimerContinuation(input) {
|
|
492
|
+
if (!input.policy.enabled || input.policy.intervalSec <= 0)
|
|
493
|
+
return false;
|
|
494
|
+
if (!isAgentEligibleForTimerContinuation(input.agent))
|
|
495
|
+
return false;
|
|
496
|
+
const intervalMs = input.policy.intervalSec * 1000;
|
|
497
|
+
const nearTermWindowMs = Math.min(intervalMs * 2, ISSUE_PASSIVE_FOLLOWUP_TIMER_CONTINUITY_MAX_WINDOW_MS);
|
|
498
|
+
const lastHeartbeatMs = input.agent.lastHeartbeatAt
|
|
499
|
+
? new Date(input.agent.lastHeartbeatAt).getTime()
|
|
500
|
+
: new Date(input.agent.createdAt).getTime();
|
|
501
|
+
const runFinishedMs = input.run.finishedAt
|
|
502
|
+
? new Date(input.run.finishedAt).getTime()
|
|
503
|
+
: input.now.getTime();
|
|
504
|
+
const baselineMs = Math.max(lastHeartbeatMs, runFinishedMs);
|
|
505
|
+
const nextTimerMs = baselineMs + intervalMs;
|
|
506
|
+
return Math.max(0, nextTimerMs - input.now.getTime()) <= nearTermWindowMs;
|
|
507
|
+
}
|
|
508
|
+
function buildPassiveFollowupContextSnapshot(input) {
|
|
509
|
+
const baseContext = { ...parseObject(input.run.contextSnapshot) };
|
|
510
|
+
delete baseContext.recovery;
|
|
511
|
+
delete baseContext.retryOfRunId;
|
|
512
|
+
delete baseContext.retryReason;
|
|
513
|
+
const taskKey = deriveTaskKey(baseContext, { issueId: input.issue.id }) ?? input.issue.id;
|
|
514
|
+
return {
|
|
515
|
+
...baseContext,
|
|
516
|
+
issueId: input.issue.id,
|
|
517
|
+
taskId: input.issue.id,
|
|
518
|
+
taskKey,
|
|
519
|
+
projectId: readNonEmptyString(baseContext.projectId) ?? input.issue.projectId ?? undefined,
|
|
520
|
+
wakeReason: ISSUE_PASSIVE_FOLLOWUP_REASON,
|
|
521
|
+
wakeSource: ISSUE_PASSIVE_FOLLOWUP_WAKE_SOURCE,
|
|
522
|
+
wakeTriggerDetail: "system",
|
|
523
|
+
issue: {
|
|
524
|
+
id: input.issue.id,
|
|
525
|
+
title: input.issue.title,
|
|
526
|
+
description: input.issue.description,
|
|
527
|
+
status: input.issue.status,
|
|
528
|
+
priority: input.issue.priority,
|
|
529
|
+
},
|
|
530
|
+
passiveFollowup: {
|
|
531
|
+
originRunId: input.originRunId,
|
|
532
|
+
previousRunId: input.run.id,
|
|
533
|
+
attempt: input.attempt,
|
|
534
|
+
maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
|
|
535
|
+
reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
|
|
536
|
+
queuedAt: input.now.toISOString(),
|
|
537
|
+
},
|
|
538
|
+
};
|
|
539
|
+
}
|
|
540
|
+
function runTaskKey(run) {
|
|
541
|
+
return deriveTaskKey(run.contextSnapshot, null);
|
|
542
|
+
}
|
|
543
|
+
function isSameTaskScope(left, right) {
|
|
544
|
+
return (left ?? null) === (right ?? null);
|
|
545
|
+
}
|
|
546
|
+
function getAgentRuntimeSessionCodec(agentRuntimeType) {
|
|
547
|
+
const adapter = getServerAdapter(agentRuntimeType);
|
|
548
|
+
return adapter.sessionCodec ?? defaultSessionCodec;
|
|
549
|
+
}
|
|
550
|
+
function resolveNextSessionState(input) {
|
|
551
|
+
const { codec, adapterResult, previousParams, previousDisplayId, previousLegacySessionId } = input;
|
|
552
|
+
if (adapterResult.clearSession) {
|
|
553
|
+
return {
|
|
554
|
+
params: null,
|
|
555
|
+
displayId: null,
|
|
556
|
+
legacySessionId: null,
|
|
557
|
+
};
|
|
558
|
+
}
|
|
559
|
+
const explicitParams = adapterResult.sessionParams;
|
|
560
|
+
const hasExplicitParams = adapterResult.sessionParams !== undefined;
|
|
561
|
+
const hasExplicitSessionId = adapterResult.sessionId !== undefined;
|
|
562
|
+
const explicitSessionId = readNonEmptyString(adapterResult.sessionId);
|
|
563
|
+
const hasExplicitDisplay = adapterResult.sessionDisplayId !== undefined;
|
|
564
|
+
const explicitDisplayId = readNonEmptyString(adapterResult.sessionDisplayId);
|
|
565
|
+
const shouldUsePrevious = !hasExplicitParams && !hasExplicitSessionId && !hasExplicitDisplay;
|
|
566
|
+
const candidateParams = hasExplicitParams
|
|
567
|
+
? explicitParams
|
|
568
|
+
: hasExplicitSessionId
|
|
569
|
+
? (explicitSessionId ? { sessionId: explicitSessionId } : null)
|
|
570
|
+
: previousParams;
|
|
571
|
+
const serialized = normalizeSessionParams(codec.serialize(normalizeSessionParams(candidateParams) ?? null));
|
|
572
|
+
const deserialized = normalizeSessionParams(codec.deserialize(serialized));
|
|
573
|
+
const displayId = truncateDisplayId(explicitDisplayId ??
|
|
574
|
+
(codec.getDisplayId ? codec.getDisplayId(deserialized) : null) ??
|
|
575
|
+
readNonEmptyString(deserialized?.sessionId) ??
|
|
576
|
+
(shouldUsePrevious ? previousDisplayId : null) ??
|
|
577
|
+
explicitSessionId ??
|
|
578
|
+
(shouldUsePrevious ? previousLegacySessionId : null));
|
|
579
|
+
const legacySessionId = explicitSessionId ??
|
|
580
|
+
readNonEmptyString(deserialized?.sessionId) ??
|
|
581
|
+
displayId ??
|
|
582
|
+
(shouldUsePrevious ? previousLegacySessionId : null);
|
|
583
|
+
return {
|
|
584
|
+
params: serialized,
|
|
585
|
+
displayId,
|
|
586
|
+
legacySessionId,
|
|
587
|
+
};
|
|
588
|
+
}
|
|
589
|
+
export function heartbeatService(db) {
|
|
590
|
+
const instanceSettings = instanceSettingsService(db);
|
|
591
|
+
const getCurrentUserRedactionOptions = async () => ({
|
|
592
|
+
enabled: (await instanceSettings.getGeneral()).censorUsernameInLogs,
|
|
593
|
+
});
|
|
594
|
+
const runLogStore = getRunLogStore();
|
|
595
|
+
const runContextSvc = agentRunContextService(db);
|
|
596
|
+
const issuesSvc = issueService(db);
|
|
597
|
+
const executionWorkspacesSvc = executionWorkspaceService(db);
|
|
598
|
+
const workspaceOperationsSvc = workspaceOperationService(db);
|
|
599
|
+
const activeRunExecutions = new Set();
|
|
600
|
+
const budgetHooks = {
|
|
601
|
+
cancelWorkForScope: cancelBudgetScopeWork,
|
|
602
|
+
};
|
|
603
|
+
const budgets = budgetService(db, budgetHooks);
|
|
604
|
+
async function getAgent(agentId) {
|
|
605
|
+
return db
|
|
606
|
+
.select()
|
|
607
|
+
.from(agents)
|
|
608
|
+
.where(eq(agents.id, agentId))
|
|
609
|
+
.then((rows) => rows[0] ?? null);
|
|
610
|
+
}
|
|
611
|
+
async function getRun(runId) {
|
|
612
|
+
return db
|
|
613
|
+
.select()
|
|
614
|
+
.from(heartbeatRuns)
|
|
615
|
+
.where(eq(heartbeatRuns.id, runId))
|
|
616
|
+
.then((rows) => rows[0] ?? null);
|
|
617
|
+
}
|
|
618
|
+
async function getRuntimeState(agentId) {
|
|
619
|
+
return db
|
|
620
|
+
.select()
|
|
621
|
+
.from(agentRuntimeState)
|
|
622
|
+
.where(eq(agentRuntimeState.agentId, agentId))
|
|
623
|
+
.then((rows) => rows[0] ?? null);
|
|
624
|
+
}
|
|
625
|
+
async function getTaskSession(orgId, agentId, agentRuntimeType, taskKey) {
|
|
626
|
+
return db
|
|
627
|
+
.select()
|
|
628
|
+
.from(agentTaskSessions)
|
|
629
|
+
.where(and(eq(agentTaskSessions.orgId, orgId), eq(agentTaskSessions.agentId, agentId), eq(agentTaskSessions.agentRuntimeType, agentRuntimeType), eq(agentTaskSessions.taskKey, taskKey)))
|
|
630
|
+
.then((rows) => rows[0] ?? null);
|
|
631
|
+
}
|
|
632
|
+
async function getLatestRunForSession(agentId, sessionId, opts) {
|
|
633
|
+
const conditions = [
|
|
634
|
+
eq(heartbeatRuns.agentId, agentId),
|
|
635
|
+
eq(heartbeatRuns.sessionIdAfter, sessionId),
|
|
636
|
+
];
|
|
637
|
+
if (opts?.excludeRunId) {
|
|
638
|
+
conditions.push(sql `${heartbeatRuns.id} <> ${opts.excludeRunId}`);
|
|
639
|
+
}
|
|
640
|
+
return db
|
|
641
|
+
.select()
|
|
642
|
+
.from(heartbeatRuns)
|
|
643
|
+
.where(and(...conditions))
|
|
644
|
+
.orderBy(desc(heartbeatRuns.createdAt))
|
|
645
|
+
.limit(1)
|
|
646
|
+
.then((rows) => rows[0] ?? null);
|
|
647
|
+
}
|
|
648
|
+
async function getOldestRunForSession(agentId, sessionId) {
|
|
649
|
+
return db
|
|
650
|
+
.select({
|
|
651
|
+
id: heartbeatRuns.id,
|
|
652
|
+
createdAt: heartbeatRuns.createdAt,
|
|
653
|
+
})
|
|
654
|
+
.from(heartbeatRuns)
|
|
655
|
+
.where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.sessionIdAfter, sessionId)))
|
|
656
|
+
.orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id))
|
|
657
|
+
.limit(1)
|
|
658
|
+
.then((rows) => rows[0] ?? null);
|
|
659
|
+
}
|
|
660
|
+
async function resolveNormalizedUsageForSession(input) {
|
|
661
|
+
const { agentId, runId, sessionId, rawUsage } = input;
|
|
662
|
+
if (!sessionId || !rawUsage) {
|
|
663
|
+
return {
|
|
664
|
+
normalizedUsage: rawUsage,
|
|
665
|
+
previousRawUsage: null,
|
|
666
|
+
derivedFromSessionTotals: false,
|
|
667
|
+
};
|
|
668
|
+
}
|
|
669
|
+
const previousRun = await getLatestRunForSession(agentId, sessionId, { excludeRunId: runId });
|
|
670
|
+
const previousRawUsage = readRawUsageTotals(previousRun?.usageJson);
|
|
671
|
+
return {
|
|
672
|
+
normalizedUsage: deriveNormalizedUsageDelta(rawUsage, previousRawUsage),
|
|
673
|
+
previousRawUsage,
|
|
674
|
+
derivedFromSessionTotals: previousRawUsage !== null,
|
|
675
|
+
};
|
|
676
|
+
}
|
|
677
|
+
async function evaluateSessionCompaction(input) {
|
|
678
|
+
const { agent, sessionId, issueId } = input;
|
|
679
|
+
if (!sessionId) {
|
|
680
|
+
return {
|
|
681
|
+
rotate: false,
|
|
682
|
+
reason: null,
|
|
683
|
+
handoffMarkdown: null,
|
|
684
|
+
previousRunId: null,
|
|
685
|
+
};
|
|
686
|
+
}
|
|
687
|
+
const policy = parseSessionCompactionPolicy(agent);
|
|
688
|
+
if (!policy.enabled || !hasSessionCompactionThresholds(policy)) {
|
|
689
|
+
return {
|
|
690
|
+
rotate: false,
|
|
691
|
+
reason: null,
|
|
692
|
+
handoffMarkdown: null,
|
|
693
|
+
previousRunId: null,
|
|
694
|
+
};
|
|
695
|
+
}
|
|
696
|
+
const fetchLimit = Math.max(policy.maxSessionRuns > 0 ? policy.maxSessionRuns + 1 : 0, 4);
|
|
697
|
+
const runs = await db
|
|
698
|
+
.select({
|
|
699
|
+
id: heartbeatRuns.id,
|
|
700
|
+
createdAt: heartbeatRuns.createdAt,
|
|
701
|
+
usageJson: heartbeatRuns.usageJson,
|
|
702
|
+
resultJson: heartbeatRuns.resultJson,
|
|
703
|
+
error: heartbeatRuns.error,
|
|
704
|
+
})
|
|
705
|
+
.from(heartbeatRuns)
|
|
706
|
+
.where(and(eq(heartbeatRuns.agentId, agent.id), eq(heartbeatRuns.sessionIdAfter, sessionId)))
|
|
707
|
+
.orderBy(desc(heartbeatRuns.createdAt))
|
|
708
|
+
.limit(fetchLimit);
|
|
709
|
+
if (runs.length === 0) {
|
|
710
|
+
return {
|
|
711
|
+
rotate: false,
|
|
712
|
+
reason: null,
|
|
713
|
+
handoffMarkdown: null,
|
|
714
|
+
previousRunId: null,
|
|
715
|
+
};
|
|
716
|
+
}
|
|
717
|
+
const latestRun = runs[0] ?? null;
|
|
718
|
+
const oldestRun = policy.maxSessionAgeHours > 0
|
|
719
|
+
? await getOldestRunForSession(agent.id, sessionId)
|
|
720
|
+
: runs[runs.length - 1] ?? latestRun;
|
|
721
|
+
const latestRawUsage = readRawUsageTotals(latestRun?.usageJson);
|
|
722
|
+
const sessionAgeHours = latestRun && oldestRun
|
|
723
|
+
? Math.max(0, (new Date(latestRun.createdAt).getTime() - new Date(oldestRun.createdAt).getTime()) / (1000 * 60 * 60))
|
|
724
|
+
: 0;
|
|
725
|
+
let reason = null;
|
|
726
|
+
if (policy.maxSessionRuns > 0 && runs.length > policy.maxSessionRuns) {
|
|
727
|
+
reason = `session exceeded ${policy.maxSessionRuns} runs`;
|
|
728
|
+
}
|
|
729
|
+
else if (policy.maxRawInputTokens > 0 &&
|
|
730
|
+
latestRawUsage &&
|
|
731
|
+
latestRawUsage.inputTokens >= policy.maxRawInputTokens) {
|
|
732
|
+
reason =
|
|
733
|
+
`session raw input reached ${formatCount(latestRawUsage.inputTokens)} tokens ` +
|
|
734
|
+
`(threshold ${formatCount(policy.maxRawInputTokens)})`;
|
|
735
|
+
}
|
|
736
|
+
else if (policy.maxSessionAgeHours > 0 && sessionAgeHours >= policy.maxSessionAgeHours) {
|
|
737
|
+
reason = `session age reached ${Math.floor(sessionAgeHours)} hours`;
|
|
738
|
+
}
|
|
739
|
+
if (!reason || !latestRun) {
|
|
740
|
+
return {
|
|
741
|
+
rotate: false,
|
|
742
|
+
reason: null,
|
|
743
|
+
handoffMarkdown: null,
|
|
744
|
+
previousRunId: latestRun?.id ?? null,
|
|
745
|
+
};
|
|
746
|
+
}
|
|
747
|
+
const latestSummary = summarizeHeartbeatRunResultJson(latestRun.resultJson);
|
|
748
|
+
const latestTextSummary = readNonEmptyString(latestSummary?.summary) ??
|
|
749
|
+
readNonEmptyString(latestSummary?.result) ??
|
|
750
|
+
readNonEmptyString(latestSummary?.message) ??
|
|
751
|
+
readNonEmptyString(latestRun.error);
|
|
752
|
+
const handoffMarkdown = [
|
|
753
|
+
"Rudder session handoff:",
|
|
754
|
+
`- Previous session: ${sessionId}`,
|
|
755
|
+
issueId ? `- Issue: ${issueId}` : "",
|
|
756
|
+
`- Rotation reason: ${reason}`,
|
|
757
|
+
latestTextSummary ? `- Last run summary: ${latestTextSummary}` : "",
|
|
758
|
+
"Continue from the current task state. Rebuild only the minimum context you need.",
|
|
759
|
+
]
|
|
760
|
+
.filter(Boolean)
|
|
761
|
+
.join("\n");
|
|
762
|
+
return {
|
|
763
|
+
rotate: true,
|
|
764
|
+
reason,
|
|
765
|
+
handoffMarkdown,
|
|
766
|
+
previousRunId: latestRun.id,
|
|
767
|
+
};
|
|
768
|
+
}
|
|
769
|
+
async function resolveSessionBeforeForWakeup(agent, taskKey) {
|
|
770
|
+
if (taskKey) {
|
|
771
|
+
const codec = getAgentRuntimeSessionCodec(agent.agentRuntimeType);
|
|
772
|
+
const existingTaskSession = await getTaskSession(agent.orgId, agent.id, agent.agentRuntimeType, taskKey);
|
|
773
|
+
const parsedParams = normalizeSessionParams(codec.deserialize(existingTaskSession?.sessionParamsJson ?? null));
|
|
774
|
+
return truncateDisplayId(existingTaskSession?.sessionDisplayId ??
|
|
775
|
+
(codec.getDisplayId ? codec.getDisplayId(parsedParams) : null) ??
|
|
776
|
+
readNonEmptyString(parsedParams?.sessionId));
|
|
777
|
+
}
|
|
778
|
+
const runtimeForRun = await getRuntimeState(agent.id);
|
|
779
|
+
return runtimeForRun?.sessionId ?? null;
|
|
780
|
+
}
|
|
781
|
+
async function resolveExplicitResumeSessionOverride(agent, payload, taskKey) {
|
|
782
|
+
const resumeFromRunId = readNonEmptyString(payload?.resumeFromRunId);
|
|
783
|
+
if (!resumeFromRunId)
|
|
784
|
+
return null;
|
|
785
|
+
const resumeRun = await db
|
|
786
|
+
.select({
|
|
787
|
+
id: heartbeatRuns.id,
|
|
788
|
+
contextSnapshot: heartbeatRuns.contextSnapshot,
|
|
789
|
+
sessionIdBefore: heartbeatRuns.sessionIdBefore,
|
|
790
|
+
sessionIdAfter: heartbeatRuns.sessionIdAfter,
|
|
791
|
+
})
|
|
792
|
+
.from(heartbeatRuns)
|
|
793
|
+
.where(and(eq(heartbeatRuns.id, resumeFromRunId), eq(heartbeatRuns.orgId, agent.orgId), eq(heartbeatRuns.agentId, agent.id)))
|
|
794
|
+
.then((rows) => rows[0] ?? null);
|
|
795
|
+
if (!resumeRun)
|
|
796
|
+
return null;
|
|
797
|
+
const resumeContext = parseObject(resumeRun.contextSnapshot);
|
|
798
|
+
const resumeTaskKey = deriveTaskKey(resumeContext, null) ?? taskKey;
|
|
799
|
+
const resumeTaskSession = resumeTaskKey
|
|
800
|
+
? await getTaskSession(agent.orgId, agent.id, agent.agentRuntimeType, resumeTaskKey)
|
|
801
|
+
: null;
|
|
802
|
+
const sessionCodec = getAgentRuntimeSessionCodec(agent.agentRuntimeType);
|
|
803
|
+
const sessionOverride = buildExplicitResumeSessionOverride({
|
|
804
|
+
resumeFromRunId,
|
|
805
|
+
resumeRunSessionIdBefore: resumeRun.sessionIdBefore,
|
|
806
|
+
resumeRunSessionIdAfter: resumeRun.sessionIdAfter,
|
|
807
|
+
taskSession: resumeTaskSession,
|
|
808
|
+
sessionCodec,
|
|
809
|
+
});
|
|
810
|
+
if (!sessionOverride)
|
|
811
|
+
return null;
|
|
812
|
+
return {
|
|
813
|
+
resumeFromRunId,
|
|
814
|
+
taskKey: resumeTaskKey,
|
|
815
|
+
issueId: readNonEmptyString(resumeContext.issueId),
|
|
816
|
+
taskId: readNonEmptyString(resumeContext.taskId) ?? readNonEmptyString(resumeContext.issueId),
|
|
817
|
+
sessionDisplayId: sessionOverride.sessionDisplayId,
|
|
818
|
+
sessionParams: sessionOverride.sessionParams,
|
|
819
|
+
};
|
|
820
|
+
}
|
|
821
|
+
async function upsertTaskSession(input) {
|
|
822
|
+
const existing = await getTaskSession(input.orgId, input.agentId, input.agentRuntimeType, input.taskKey);
|
|
823
|
+
if (existing) {
|
|
824
|
+
return db
|
|
825
|
+
.update(agentTaskSessions)
|
|
826
|
+
.set({
|
|
827
|
+
sessionParamsJson: input.sessionParamsJson,
|
|
828
|
+
sessionDisplayId: input.sessionDisplayId,
|
|
829
|
+
lastRunId: input.lastRunId,
|
|
830
|
+
lastError: input.lastError,
|
|
831
|
+
updatedAt: new Date(),
|
|
832
|
+
})
|
|
833
|
+
.where(eq(agentTaskSessions.id, existing.id))
|
|
834
|
+
.returning()
|
|
835
|
+
.then((rows) => rows[0] ?? null);
|
|
836
|
+
}
|
|
837
|
+
return db
|
|
838
|
+
.insert(agentTaskSessions)
|
|
839
|
+
.values({
|
|
840
|
+
orgId: input.orgId,
|
|
841
|
+
agentId: input.agentId,
|
|
842
|
+
agentRuntimeType: input.agentRuntimeType,
|
|
843
|
+
taskKey: input.taskKey,
|
|
844
|
+
sessionParamsJson: input.sessionParamsJson,
|
|
845
|
+
sessionDisplayId: input.sessionDisplayId,
|
|
846
|
+
lastRunId: input.lastRunId,
|
|
847
|
+
lastError: input.lastError,
|
|
848
|
+
})
|
|
849
|
+
.returning()
|
|
850
|
+
.then((rows) => rows[0] ?? null);
|
|
851
|
+
}
|
|
852
|
+
async function clearTaskSessions(orgId, agentId, opts) {
|
|
853
|
+
const conditions = [
|
|
854
|
+
eq(agentTaskSessions.orgId, orgId),
|
|
855
|
+
eq(agentTaskSessions.agentId, agentId),
|
|
856
|
+
];
|
|
857
|
+
if (opts?.taskKey) {
|
|
858
|
+
conditions.push(eq(agentTaskSessions.taskKey, opts.taskKey));
|
|
859
|
+
}
|
|
860
|
+
if (opts?.agentRuntimeType) {
|
|
861
|
+
conditions.push(eq(agentTaskSessions.agentRuntimeType, opts.agentRuntimeType));
|
|
862
|
+
}
|
|
863
|
+
return db
|
|
864
|
+
.delete(agentTaskSessions)
|
|
865
|
+
.where(and(...conditions))
|
|
866
|
+
.returning()
|
|
867
|
+
.then((rows) => rows.length);
|
|
868
|
+
}
|
|
869
|
+
async function ensureRuntimeState(agent) {
|
|
870
|
+
const existing = await getRuntimeState(agent.id);
|
|
871
|
+
if (existing)
|
|
872
|
+
return existing;
|
|
873
|
+
const now = new Date();
|
|
874
|
+
return db
|
|
875
|
+
.insert(agentRuntimeState)
|
|
876
|
+
.values({
|
|
877
|
+
agentId: agent.id,
|
|
878
|
+
orgId: agent.orgId,
|
|
879
|
+
agentRuntimeType: agent.agentRuntimeType,
|
|
880
|
+
stateJson: {},
|
|
881
|
+
createdAt: now,
|
|
882
|
+
updatedAt: now,
|
|
883
|
+
})
|
|
884
|
+
.onConflictDoUpdate({
|
|
885
|
+
target: agentRuntimeState.agentId,
|
|
886
|
+
set: {
|
|
887
|
+
orgId: agent.orgId,
|
|
888
|
+
agentRuntimeType: agent.agentRuntimeType,
|
|
889
|
+
updatedAt: now,
|
|
890
|
+
},
|
|
891
|
+
})
|
|
892
|
+
.returning()
|
|
893
|
+
.then((rows) => rows[0]);
|
|
894
|
+
}
|
|
895
|
+
function buildHeartbeatObservabilityContext(run, overrides = {}) {
|
|
896
|
+
const contextSnapshot = parseObject(run.contextSnapshot);
|
|
897
|
+
const issueSnapshot = parseObject(contextSnapshot.issue);
|
|
898
|
+
const benchmarkMetadata = extractCreateAgentBenchmarkMetadata(readNonEmptyString(issueSnapshot.description))
|
|
899
|
+
?? coerceCreateAgentBenchmarkMetadata(parseObject(contextSnapshot.benchmark));
|
|
900
|
+
const baseMetadata = {
|
|
901
|
+
wakeupRequestId: run.wakeupRequestId,
|
|
902
|
+
errorCode: run.errorCode,
|
|
903
|
+
retryOfRunId: run.retryOfRunId,
|
|
904
|
+
processLossRetryCount: run.processLossRetryCount,
|
|
905
|
+
externalRunId: run.externalRunId,
|
|
906
|
+
executionWorkspaceId: readNonEmptyString(contextSnapshot.executionWorkspaceId),
|
|
907
|
+
...(benchmarkMetadata ?? {}),
|
|
908
|
+
};
|
|
909
|
+
const benchmarkTags = benchmarkMetadata ? buildCreateAgentBenchmarkTags(benchmarkMetadata) : [];
|
|
910
|
+
return {
|
|
911
|
+
surface: resolveHeartbeatObservabilitySurface(contextSnapshot),
|
|
912
|
+
rootExecutionId: run.id,
|
|
913
|
+
orgId: run.orgId,
|
|
914
|
+
agentId: run.agentId,
|
|
915
|
+
issueId: readNonEmptyString(contextSnapshot.issueId),
|
|
916
|
+
sessionKey: run.sessionIdAfter ??
|
|
917
|
+
run.sessionIdBefore ??
|
|
918
|
+
readNonEmptyString(contextSnapshot.sessionKey) ??
|
|
919
|
+
readNonEmptyString(contextSnapshot.taskKey),
|
|
920
|
+
runtime: readNonEmptyString(contextSnapshot.agentRuntimeType),
|
|
921
|
+
trigger: run.triggerDetail ?? run.invocationSource,
|
|
922
|
+
status: run.status,
|
|
923
|
+
metadata: {
|
|
924
|
+
...baseMetadata,
|
|
925
|
+
...(overrides.metadata ?? {}),
|
|
926
|
+
},
|
|
927
|
+
tags: [...benchmarkTags, ...(overrides.tags ?? [])],
|
|
928
|
+
...overrides,
|
|
929
|
+
};
|
|
930
|
+
}
|
|
931
|
+
async function emitHeartbeatObservationEvent(run, input, overrides = {}) {
|
|
932
|
+
try {
|
|
933
|
+
await observeExecutionEvent(buildHeartbeatObservabilityContext(run, overrides), input);
|
|
934
|
+
}
|
|
935
|
+
catch (error) {
|
|
936
|
+
logger.warn({
|
|
937
|
+
runId: run.id,
|
|
938
|
+
eventName: input.name,
|
|
939
|
+
err: error instanceof Error ? error.message : String(error),
|
|
940
|
+
}, "Failed to emit Langfuse heartbeat event");
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
async function emitHeartbeatLiveEval(runId) {
|
|
944
|
+
try {
|
|
945
|
+
const { detail, scores } = await buildObservedRunLangfuseScores(db, runId);
|
|
946
|
+
await createExecutionScores(buildHeartbeatObservabilityContext(detail.run, {
|
|
947
|
+
runtime: detail.bundle.agentRuntimeType,
|
|
948
|
+
metadata: {
|
|
949
|
+
agentName: detail.agentName,
|
|
950
|
+
orgName: detail.orgName,
|
|
951
|
+
},
|
|
952
|
+
}), scores.map((score) => ({
|
|
953
|
+
rootExecutionId: detail.run.id,
|
|
954
|
+
name: score.name,
|
|
955
|
+
value: score.value,
|
|
956
|
+
comment: score.comment,
|
|
957
|
+
metadata: score.metadata,
|
|
958
|
+
})));
|
|
959
|
+
}
|
|
960
|
+
catch (error) {
|
|
961
|
+
logger.warn({
|
|
962
|
+
runId,
|
|
963
|
+
err: error instanceof Error ? error.message : String(error),
|
|
964
|
+
}, "Failed to emit Langfuse heartbeat scores");
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
async function setRunStatus(runId, status, patch) {
|
|
968
|
+
const updated = await db
|
|
969
|
+
.update(heartbeatRuns)
|
|
970
|
+
.set({ status, ...patch, updatedAt: new Date() })
|
|
971
|
+
.where(eq(heartbeatRuns.id, runId))
|
|
972
|
+
.returning()
|
|
973
|
+
.then((rows) => rows[0] ?? null);
|
|
974
|
+
if (updated) {
|
|
975
|
+
publishLiveEvent({
|
|
976
|
+
orgId: updated.orgId,
|
|
977
|
+
type: "heartbeat.run.status",
|
|
978
|
+
payload: {
|
|
979
|
+
runId: updated.id,
|
|
980
|
+
agentId: updated.agentId,
|
|
981
|
+
status: updated.status,
|
|
982
|
+
invocationSource: updated.invocationSource,
|
|
983
|
+
triggerDetail: updated.triggerDetail,
|
|
984
|
+
error: updated.error ?? null,
|
|
985
|
+
errorCode: updated.errorCode ?? null,
|
|
986
|
+
startedAt: updated.startedAt ? new Date(updated.startedAt).toISOString() : null,
|
|
987
|
+
finishedAt: updated.finishedAt ? new Date(updated.finishedAt).toISOString() : null,
|
|
988
|
+
},
|
|
989
|
+
});
|
|
990
|
+
await emitHeartbeatObservationEvent(updated, {
|
|
991
|
+
name: `heartbeat.status.${status}`,
|
|
992
|
+
asType: "event",
|
|
993
|
+
output: {
|
|
994
|
+
status: updated.status,
|
|
995
|
+
error: updated.error,
|
|
996
|
+
errorCode: updated.errorCode,
|
|
997
|
+
startedAt: updated.startedAt ? new Date(updated.startedAt).toISOString() : null,
|
|
998
|
+
finishedAt: updated.finishedAt ? new Date(updated.finishedAt).toISOString() : null,
|
|
999
|
+
},
|
|
1000
|
+
}, {
|
|
1001
|
+
status: updated.status,
|
|
1002
|
+
});
|
|
1003
|
+
}
|
|
1004
|
+
return updated;
|
|
1005
|
+
}
|
|
1006
|
+
async function setWakeupStatus(wakeupRequestId, status, patch) {
|
|
1007
|
+
if (!wakeupRequestId)
|
|
1008
|
+
return;
|
|
1009
|
+
await db
|
|
1010
|
+
.update(agentWakeupRequests)
|
|
1011
|
+
.set({ status, ...patch, updatedAt: new Date() })
|
|
1012
|
+
.where(eq(agentWakeupRequests.id, wakeupRequestId));
|
|
1013
|
+
}
|
|
1014
|
+
async function updateWakeupRequestRecord(tx, wakeupRequestId, patch) {
|
|
1015
|
+
return tx
|
|
1016
|
+
.update(agentWakeupRequests)
|
|
1017
|
+
.set({ ...patch, updatedAt: new Date() })
|
|
1018
|
+
.where(eq(agentWakeupRequests.id, wakeupRequestId))
|
|
1019
|
+
.returning()
|
|
1020
|
+
.then((rows) => rows[0] ?? null);
|
|
1021
|
+
}
|
|
1022
|
+
async function insertWakeupRequestRecord(tx, values) {
|
|
1023
|
+
return tx
|
|
1024
|
+
.insert(agentWakeupRequests)
|
|
1025
|
+
.values(values)
|
|
1026
|
+
.returning()
|
|
1027
|
+
.then((rows) => rows[0] ?? null);
|
|
1028
|
+
}
|
|
1029
|
+
async function appendRunEvent(run, seq, event) {
|
|
1030
|
+
const currentUserRedactionOptions = await getCurrentUserRedactionOptions();
|
|
1031
|
+
const sanitizedMessage = event.message
|
|
1032
|
+
? redactCurrentUserText(event.message, currentUserRedactionOptions)
|
|
1033
|
+
: event.message;
|
|
1034
|
+
const sanitizedPayload = event.payload
|
|
1035
|
+
? redactCurrentUserValue(event.payload, currentUserRedactionOptions)
|
|
1036
|
+
: event.payload;
|
|
1037
|
+
await db.insert(heartbeatRunEvents).values({
|
|
1038
|
+
orgId: run.orgId,
|
|
1039
|
+
runId: run.id,
|
|
1040
|
+
agentId: run.agentId,
|
|
1041
|
+
seq,
|
|
1042
|
+
eventType: event.eventType,
|
|
1043
|
+
stream: event.stream,
|
|
1044
|
+
level: event.level,
|
|
1045
|
+
color: event.color,
|
|
1046
|
+
message: sanitizedMessage,
|
|
1047
|
+
payload: sanitizedPayload,
|
|
1048
|
+
});
|
|
1049
|
+
publishLiveEvent({
|
|
1050
|
+
orgId: run.orgId,
|
|
1051
|
+
type: "heartbeat.run.event",
|
|
1052
|
+
payload: {
|
|
1053
|
+
runId: run.id,
|
|
1054
|
+
agentId: run.agentId,
|
|
1055
|
+
seq,
|
|
1056
|
+
eventType: event.eventType,
|
|
1057
|
+
stream: event.stream ?? null,
|
|
1058
|
+
level: event.level ?? null,
|
|
1059
|
+
color: event.color ?? null,
|
|
1060
|
+
message: sanitizedMessage ?? null,
|
|
1061
|
+
payload: sanitizedPayload ?? null,
|
|
1062
|
+
},
|
|
1063
|
+
});
|
|
1064
|
+
await emitHeartbeatObservationEvent(run, {
|
|
1065
|
+
name: `heartbeat.event.${event.eventType}`,
|
|
1066
|
+
asType: "event",
|
|
1067
|
+
level: event.level === "error" ? "ERROR" : event.level === "warn" ? "WARNING" : "DEFAULT",
|
|
1068
|
+
output: {
|
|
1069
|
+
seq,
|
|
1070
|
+
eventType: event.eventType,
|
|
1071
|
+
stream: event.stream ?? null,
|
|
1072
|
+
level: event.level ?? null,
|
|
1073
|
+
color: event.color ?? null,
|
|
1074
|
+
message: sanitizedMessage ?? null,
|
|
1075
|
+
},
|
|
1076
|
+
metadata: sanitizedPayload ?? undefined,
|
|
1077
|
+
}, {
|
|
1078
|
+
status: run.status,
|
|
1079
|
+
});
|
|
1080
|
+
}
|
|
1081
|
+
async function nextRunEventSeq(runId) {
|
|
1082
|
+
const [row] = await db
|
|
1083
|
+
.select({ maxSeq: sql `max(${heartbeatRunEvents.seq})` })
|
|
1084
|
+
.from(heartbeatRunEvents)
|
|
1085
|
+
.where(eq(heartbeatRunEvents.runId, runId));
|
|
1086
|
+
return Number(row?.maxSeq ?? 0) + 1;
|
|
1087
|
+
}
|
|
1088
|
+
async function persistRunProcessMetadata(runId, meta) {
|
|
1089
|
+
const startedAt = new Date(meta.startedAt);
|
|
1090
|
+
const updated = await db
|
|
1091
|
+
.update(heartbeatRuns)
|
|
1092
|
+
.set({
|
|
1093
|
+
processPid: meta.pid,
|
|
1094
|
+
processStartedAt: Number.isNaN(startedAt.getTime()) ? new Date() : startedAt,
|
|
1095
|
+
updatedAt: new Date(),
|
|
1096
|
+
})
|
|
1097
|
+
.where(eq(heartbeatRuns.id, runId))
|
|
1098
|
+
.returning()
|
|
1099
|
+
.then((rows) => rows[0] ?? null);
|
|
1100
|
+
if (updated) {
|
|
1101
|
+
await emitHeartbeatObservationEvent(updated, {
|
|
1102
|
+
name: "heartbeat.process.spawn",
|
|
1103
|
+
asType: "event",
|
|
1104
|
+
output: {
|
|
1105
|
+
pid: meta.pid,
|
|
1106
|
+
startedAt: meta.startedAt,
|
|
1107
|
+
},
|
|
1108
|
+
});
|
|
1109
|
+
}
|
|
1110
|
+
return updated;
|
|
1111
|
+
}
|
|
1112
|
+
async function clearDetachedRunWarning(runId) {
|
|
1113
|
+
const updated = await db
|
|
1114
|
+
.update(heartbeatRuns)
|
|
1115
|
+
.set({
|
|
1116
|
+
error: null,
|
|
1117
|
+
errorCode: null,
|
|
1118
|
+
updatedAt: new Date(),
|
|
1119
|
+
})
|
|
1120
|
+
.where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.errorCode, DETACHED_PROCESS_ERROR_CODE)))
|
|
1121
|
+
.returning()
|
|
1122
|
+
.then((rows) => rows[0] ?? null);
|
|
1123
|
+
if (!updated)
|
|
1124
|
+
return null;
|
|
1125
|
+
await appendRunEvent(updated, await nextRunEventSeq(updated.id), {
|
|
1126
|
+
eventType: "lifecycle",
|
|
1127
|
+
stream: "system",
|
|
1128
|
+
level: "info",
|
|
1129
|
+
message: "Detached child process reported activity; cleared detached warning",
|
|
1130
|
+
});
|
|
1131
|
+
return updated;
|
|
1132
|
+
}
|
|
1133
|
+
async function enqueueRecoveryRun(run, agent, opts) {
|
|
1134
|
+
/**
|
|
1135
|
+
* Recovery runs intentionally clone the prior run's task context and then
|
|
1136
|
+
* layer explicit recovery metadata on top. This keeps retries visible and
|
|
1137
|
+
* auditable while preserving "continue preferred" semantics for issue work.
|
|
1138
|
+
*
|
|
1139
|
+
* Reasoning:
|
|
1140
|
+
* - Manual retry and automatic process-loss retry must assemble the same
|
|
1141
|
+
* recovery contract so prompts/runtime behavior stay aligned.
|
|
1142
|
+
* - We backfill missing context from the retry chain to recover from older
|
|
1143
|
+
* lossy retry runs without mutating the historical source run rows.
|
|
1144
|
+
*
|
|
1145
|
+
* Traceability:
|
|
1146
|
+
* - doc/developing/RUN-RECOVERY.md
|
|
1147
|
+
* - doc/DEVELOPING.md
|
|
1148
|
+
*/
|
|
1149
|
+
const baseContextSnapshot = await hydrateRecoveryBaseContextSnapshot(run, getRun);
|
|
1150
|
+
const recoveryContextSnapshot = buildRecoveryContextSnapshot({
|
|
1151
|
+
baseContextSnapshot,
|
|
1152
|
+
run,
|
|
1153
|
+
recoveryTrigger: opts.recoveryTrigger,
|
|
1154
|
+
wakeReason: opts.wakeReason,
|
|
1155
|
+
wakeSource: `recovery.${opts.recoveryTrigger}`,
|
|
1156
|
+
triggerDetail: opts.triggerDetail,
|
|
1157
|
+
});
|
|
1158
|
+
const issueId = readNonEmptyString(recoveryContextSnapshot.issueId);
|
|
1159
|
+
const taskKey = deriveTaskKey(recoveryContextSnapshot, null);
|
|
1160
|
+
const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey);
|
|
1161
|
+
const recovery = recoveryContextSnapshot.recovery;
|
|
1162
|
+
const requestPayload = {
|
|
1163
|
+
originalRunId: run.id,
|
|
1164
|
+
failureKind: recovery.failureKind,
|
|
1165
|
+
recoveryTrigger: recovery.recoveryTrigger,
|
|
1166
|
+
...(issueId ? { issueId } : {}),
|
|
1167
|
+
};
|
|
1168
|
+
const outcome = await db.transaction(async (tx) => {
|
|
1169
|
+
let issueRow = null;
|
|
1170
|
+
if (issueId) {
|
|
1171
|
+
await tx.execute(sql `select id from issues where id = ${issueId} and org_id = ${run.orgId} for update`);
|
|
1172
|
+
issueRow = await tx
|
|
1173
|
+
.select({
|
|
1174
|
+
id: issues.id,
|
|
1175
|
+
orgId: issues.orgId,
|
|
1176
|
+
executionRunId: issues.executionRunId,
|
|
1177
|
+
executionAgentNameKey: issues.executionAgentNameKey,
|
|
1178
|
+
})
|
|
1179
|
+
.from(issues)
|
|
1180
|
+
.where(and(eq(issues.id, issueId), eq(issues.orgId, run.orgId)))
|
|
1181
|
+
.then((rows) => rows[0] ?? null);
|
|
1182
|
+
}
|
|
1183
|
+
if (issueRow?.executionRunId) {
|
|
1184
|
+
const activeExecutionRun = await tx
|
|
1185
|
+
.select()
|
|
1186
|
+
.from(heartbeatRuns)
|
|
1187
|
+
.where(eq(heartbeatRuns.id, issueRow.executionRunId))
|
|
1188
|
+
.then((rows) => rows[0] ?? null);
|
|
1189
|
+
const isActiveExecutionRun = activeExecutionRun &&
|
|
1190
|
+
(activeExecutionRun.status === "queued" || activeExecutionRun.status === "running");
|
|
1191
|
+
if (!isActiveExecutionRun) {
|
|
1192
|
+
await tx
|
|
1193
|
+
.update(issues)
|
|
1194
|
+
.set({
|
|
1195
|
+
executionRunId: null,
|
|
1196
|
+
executionAgentNameKey: null,
|
|
1197
|
+
executionLockedAt: null,
|
|
1198
|
+
updatedAt: opts.now,
|
|
1199
|
+
})
|
|
1200
|
+
.where(eq(issues.id, issueRow.id));
|
|
1201
|
+
issueRow = {
|
|
1202
|
+
...issueRow,
|
|
1203
|
+
executionRunId: null,
|
|
1204
|
+
executionAgentNameKey: null,
|
|
1205
|
+
};
|
|
1206
|
+
}
|
|
1207
|
+
else if (activeExecutionRun) {
|
|
1208
|
+
const activeContext = parseObject(activeExecutionRun.contextSnapshot);
|
|
1209
|
+
const activeRecovery = parseObject(activeContext.recovery);
|
|
1210
|
+
if (activeExecutionRun.agentId === run.agentId &&
|
|
1211
|
+
(activeExecutionRun.retryOfRunId === run.id ||
|
|
1212
|
+
readNonEmptyString(activeRecovery.originalRunId) === run.id)) {
|
|
1213
|
+
return { kind: "existing", run: activeExecutionRun };
|
|
1214
|
+
}
|
|
1215
|
+
throw conflict("Issue already has an active execution run", {
|
|
1216
|
+
issueId: issueRow.id,
|
|
1217
|
+
executionRunId: activeExecutionRun.id,
|
|
1218
|
+
});
|
|
1219
|
+
}
|
|
1220
|
+
}
|
|
1221
|
+
const wakeupRequest = await tx
|
|
1222
|
+
.insert(agentWakeupRequests)
|
|
1223
|
+
.values({
|
|
1224
|
+
orgId: run.orgId,
|
|
1225
|
+
agentId: run.agentId,
|
|
1226
|
+
source: opts.source,
|
|
1227
|
+
triggerDetail: opts.triggerDetail,
|
|
1228
|
+
reason: opts.wakeReason,
|
|
1229
|
+
payload: requestPayload,
|
|
1230
|
+
status: "queued",
|
|
1231
|
+
requestedByActorType: opts.requestedByActorType ?? null,
|
|
1232
|
+
requestedByActorId: opts.requestedByActorId ?? null,
|
|
1233
|
+
updatedAt: opts.now,
|
|
1234
|
+
})
|
|
1235
|
+
.returning()
|
|
1236
|
+
.then((rows) => rows[0]);
|
|
1237
|
+
const recoveryRun = await tx
|
|
1238
|
+
.insert(heartbeatRuns)
|
|
1239
|
+
.values({
|
|
1240
|
+
orgId: run.orgId,
|
|
1241
|
+
agentId: run.agentId,
|
|
1242
|
+
invocationSource: opts.source,
|
|
1243
|
+
triggerDetail: opts.triggerDetail,
|
|
1244
|
+
status: "queued",
|
|
1245
|
+
wakeupRequestId: wakeupRequest.id,
|
|
1246
|
+
contextSnapshot: recoveryContextSnapshot,
|
|
1247
|
+
sessionIdBefore: sessionBefore,
|
|
1248
|
+
retryOfRunId: run.id,
|
|
1249
|
+
processLossRetryCount: opts.recoveryTrigger === "automatic"
|
|
1250
|
+
? (run.processLossRetryCount ?? 0) + 1
|
|
1251
|
+
: (run.processLossRetryCount ?? 0),
|
|
1252
|
+
updatedAt: opts.now,
|
|
1253
|
+
})
|
|
1254
|
+
.returning()
|
|
1255
|
+
.then((rows) => rows[0]);
|
|
1256
|
+
await tx
|
|
1257
|
+
.update(agentWakeupRequests)
|
|
1258
|
+
.set({
|
|
1259
|
+
runId: recoveryRun.id,
|
|
1260
|
+
updatedAt: opts.now,
|
|
1261
|
+
})
|
|
1262
|
+
.where(eq(agentWakeupRequests.id, wakeupRequest.id));
|
|
1263
|
+
if (issueRow) {
|
|
1264
|
+
await tx
|
|
1265
|
+
.update(issues)
|
|
1266
|
+
.set({
|
|
1267
|
+
executionRunId: recoveryRun.id,
|
|
1268
|
+
executionAgentNameKey: normalizeAgentNameKey(agent.name),
|
|
1269
|
+
executionLockedAt: opts.now,
|
|
1270
|
+
updatedAt: opts.now,
|
|
1271
|
+
})
|
|
1272
|
+
.where(eq(issues.id, issueRow.id));
|
|
1273
|
+
}
|
|
1274
|
+
return { kind: "queued", run: recoveryRun };
|
|
1275
|
+
});
|
|
1276
|
+
if (outcome.kind === "existing")
|
|
1277
|
+
return outcome.run;
|
|
1278
|
+
const recoveryRun = outcome.run;
|
|
1279
|
+
await appendRunEvent(recoveryRun, await nextRunEventSeq(recoveryRun.id), {
|
|
1280
|
+
eventType: "lifecycle",
|
|
1281
|
+
stream: "system",
|
|
1282
|
+
level: opts.recoveryTrigger === "automatic" ? "warn" : "info",
|
|
1283
|
+
message: `Recovery queued from run ${run.id}`,
|
|
1284
|
+
payload: {
|
|
1285
|
+
originalRunId: run.id,
|
|
1286
|
+
failureKind: recovery.failureKind,
|
|
1287
|
+
failureSummary: recovery.failureSummary,
|
|
1288
|
+
recoveryTrigger: recovery.recoveryTrigger,
|
|
1289
|
+
recoveryMode: recovery.recoveryMode,
|
|
1290
|
+
},
|
|
1291
|
+
});
|
|
1292
|
+
publishLiveEvent({
|
|
1293
|
+
orgId: recoveryRun.orgId,
|
|
1294
|
+
type: "heartbeat.run.queued",
|
|
1295
|
+
payload: {
|
|
1296
|
+
runId: recoveryRun.id,
|
|
1297
|
+
agentId: recoveryRun.agentId,
|
|
1298
|
+
invocationSource: recoveryRun.invocationSource,
|
|
1299
|
+
triggerDetail: recoveryRun.triggerDetail,
|
|
1300
|
+
wakeupRequestId: recoveryRun.wakeupRequestId,
|
|
1301
|
+
},
|
|
1302
|
+
});
|
|
1303
|
+
if (opts.startImmediately !== false) {
|
|
1304
|
+
await startNextQueuedRunForAgent(agent.id);
|
|
1305
|
+
}
|
|
1306
|
+
return recoveryRun;
|
|
1307
|
+
}
|
|
1308
|
+
async function enqueueProcessLossRetry(run, agent, now) {
|
|
1309
|
+
return enqueueRecoveryRun(run, agent, {
|
|
1310
|
+
recoveryTrigger: "automatic",
|
|
1311
|
+
source: "automation",
|
|
1312
|
+
triggerDetail: "system",
|
|
1313
|
+
wakeReason: "process_lost_retry",
|
|
1314
|
+
requestedByActorType: "system",
|
|
1315
|
+
requestedByActorId: null,
|
|
1316
|
+
startImmediately: false,
|
|
1317
|
+
now,
|
|
1318
|
+
});
|
|
1319
|
+
}
|
|
1320
|
+
async function runHasIssueClosureComment(tx, run, issueId) {
|
|
1321
|
+
const commentActivity = await tx
|
|
1322
|
+
.select({ id: activityLog.id })
|
|
1323
|
+
.from(activityLog)
|
|
1324
|
+
.where(and(eq(activityLog.orgId, run.orgId), eq(activityLog.action, "issue.comment_added"), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issueId), eq(activityLog.runId, run.id)))
|
|
1325
|
+
.limit(1)
|
|
1326
|
+
.then((rows) => rows[0] ?? null);
|
|
1327
|
+
return Boolean(commentActivity);
|
|
1328
|
+
}
|
|
1329
|
+
async function issueHasDeferredWake(tx, orgId, issueId) {
|
|
1330
|
+
const deferred = await tx
|
|
1331
|
+
.select({ id: agentWakeupRequests.id })
|
|
1332
|
+
.from(agentWakeupRequests)
|
|
1333
|
+
.where(and(eq(agentWakeupRequests.orgId, orgId), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql `${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`))
|
|
1334
|
+
.limit(1)
|
|
1335
|
+
.then((rows) => rows[0] ?? null);
|
|
1336
|
+
return Boolean(deferred);
|
|
1337
|
+
}
|
|
1338
|
+
async function passiveFollowupAlreadyRecorded(tx, runId) {
|
|
1339
|
+
const idempotencyKey = `${ISSUE_PASSIVE_FOLLOWUP_REASON}:${runId}`;
|
|
1340
|
+
const existingWake = await tx
|
|
1341
|
+
.select({ id: agentWakeupRequests.id })
|
|
1342
|
+
.from(agentWakeupRequests)
|
|
1343
|
+
.where(eq(agentWakeupRequests.idempotencyKey, idempotencyKey))
|
|
1344
|
+
.limit(1)
|
|
1345
|
+
.then((rows) => rows[0] ?? null);
|
|
1346
|
+
if (existingWake)
|
|
1347
|
+
return true;
|
|
1348
|
+
const existingReview = await tx
|
|
1349
|
+
.select({ id: activityLog.id })
|
|
1350
|
+
.from(activityLog)
|
|
1351
|
+
.where(and(eq(activityLog.runId, runId), eq(activityLog.action, "issue.closure_needs_operator_review")))
|
|
1352
|
+
.limit(1)
|
|
1353
|
+
.then((rows) => rows[0] ?? null);
|
|
1354
|
+
return Boolean(existingReview);
|
|
1355
|
+
}
|
|
1356
|
+
async function evaluatePassiveIssueClosureForLockedIssue(input) {
|
|
1357
|
+
const { tx, run, issue, now } = input;
|
|
1358
|
+
const context = parseObject(run.contextSnapshot);
|
|
1359
|
+
const runIssueId = readNonEmptyString(context.issueId);
|
|
1360
|
+
if (!runIssueId || runIssueId !== issue.id)
|
|
1361
|
+
return { kind: "none", reason: "run_not_issue_backed" };
|
|
1362
|
+
if (run.status !== "succeeded")
|
|
1363
|
+
return { kind: "none", reason: "run_not_successful" };
|
|
1364
|
+
if (issue.status !== "todo" && issue.status !== "in_progress") {
|
|
1365
|
+
return { kind: "none", reason: "issue_has_closure_status" };
|
|
1366
|
+
}
|
|
1367
|
+
if (issue.assigneeAgentId !== run.agentId) {
|
|
1368
|
+
return { kind: "none", reason: "issue_no_longer_assigned_to_run_agent" };
|
|
1369
|
+
}
|
|
1370
|
+
if (await runHasIssueClosureComment(tx, run, issue.id)) {
|
|
1371
|
+
return { kind: "none", reason: "run_authored_issue_comment" };
|
|
1372
|
+
}
|
|
1373
|
+
if (await issueHasDeferredWake(tx, issue.orgId, issue.id)) {
|
|
1374
|
+
return { kind: "none", reason: "deferred_issue_wake_exists" };
|
|
1375
|
+
}
|
|
1376
|
+
if (await passiveFollowupAlreadyRecorded(tx, run.id)) {
|
|
1377
|
+
return { kind: "none", reason: "passive_followup_already_recorded" };
|
|
1378
|
+
}
|
|
1379
|
+
const agent = await tx
|
|
1380
|
+
.select()
|
|
1381
|
+
.from(agents)
|
|
1382
|
+
.where(eq(agents.id, run.agentId))
|
|
1383
|
+
.then((rows) => rows[0] ?? null);
|
|
1384
|
+
if (!agent || agent.orgId !== run.orgId) {
|
|
1385
|
+
return { kind: "none", reason: "agent_not_found" };
|
|
1386
|
+
}
|
|
1387
|
+
const policy = parseHeartbeatPolicy(agent);
|
|
1388
|
+
if (hasCredibleTimerContinuation({ agent, policy, run, now })) {
|
|
1389
|
+
return { kind: "none", reason: "timer_continuity_expected" };
|
|
1390
|
+
}
|
|
1391
|
+
const passiveContext = normalizePassiveFollowupContext(context.passiveFollowup);
|
|
1392
|
+
const currentAttempt = passiveContext?.attempt ?? 0;
|
|
1393
|
+
const originRunId = passiveContext?.originRunId ?? run.id;
|
|
1394
|
+
if (currentAttempt >= ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS) {
|
|
1395
|
+
return {
|
|
1396
|
+
kind: "operator_review",
|
|
1397
|
+
issue,
|
|
1398
|
+
originRunId,
|
|
1399
|
+
previousRunId: run.id,
|
|
1400
|
+
attempts: currentAttempt,
|
|
1401
|
+
reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
|
|
1402
|
+
};
|
|
1403
|
+
}
|
|
1404
|
+
const nextAttempt = currentAttempt + 1;
|
|
1405
|
+
const requestedAt = new Date(now.getTime() + passiveFollowupCooldownMs(nextAttempt));
|
|
1406
|
+
const contextSnapshot = buildPassiveFollowupContextSnapshot({
|
|
1407
|
+
run,
|
|
1408
|
+
issue,
|
|
1409
|
+
originRunId,
|
|
1410
|
+
attempt: nextAttempt,
|
|
1411
|
+
now,
|
|
1412
|
+
});
|
|
1413
|
+
const taskKey = deriveTaskKey(contextSnapshot, { issueId: issue.id });
|
|
1414
|
+
const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey);
|
|
1415
|
+
const requestPayload = {
|
|
1416
|
+
issueId: issue.id,
|
|
1417
|
+
originRunId,
|
|
1418
|
+
previousRunId: run.id,
|
|
1419
|
+
attempt: nextAttempt,
|
|
1420
|
+
reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
|
|
1421
|
+
};
|
|
1422
|
+
const wakeupRequest = await tx
|
|
1423
|
+
.insert(agentWakeupRequests)
|
|
1424
|
+
.values({
|
|
1425
|
+
orgId: run.orgId,
|
|
1426
|
+
agentId: run.agentId,
|
|
1427
|
+
source: "automation",
|
|
1428
|
+
triggerDetail: "system",
|
|
1429
|
+
reason: ISSUE_PASSIVE_FOLLOWUP_REASON,
|
|
1430
|
+
payload: requestPayload,
|
|
1431
|
+
status: "queued",
|
|
1432
|
+
requestedByActorType: "system",
|
|
1433
|
+
requestedByActorId: "issue_closure_governance",
|
|
1434
|
+
idempotencyKey: `${ISSUE_PASSIVE_FOLLOWUP_REASON}:${run.id}`,
|
|
1435
|
+
requestedAt,
|
|
1436
|
+
updatedAt: now,
|
|
1437
|
+
})
|
|
1438
|
+
.returning()
|
|
1439
|
+
.then((rows) => rows[0]);
|
|
1440
|
+
const followupRun = await tx
|
|
1441
|
+
.insert(heartbeatRuns)
|
|
1442
|
+
.values({
|
|
1443
|
+
orgId: run.orgId,
|
|
1444
|
+
agentId: run.agentId,
|
|
1445
|
+
invocationSource: "automation",
|
|
1446
|
+
triggerDetail: "system",
|
|
1447
|
+
status: "queued",
|
|
1448
|
+
wakeupRequestId: wakeupRequest.id,
|
|
1449
|
+
contextSnapshot,
|
|
1450
|
+
sessionIdBefore: sessionBefore,
|
|
1451
|
+
updatedAt: now,
|
|
1452
|
+
})
|
|
1453
|
+
.returning()
|
|
1454
|
+
.then((rows) => rows[0]);
|
|
1455
|
+
await tx
|
|
1456
|
+
.update(agentWakeupRequests)
|
|
1457
|
+
.set({
|
|
1458
|
+
runId: followupRun.id,
|
|
1459
|
+
updatedAt: now,
|
|
1460
|
+
})
|
|
1461
|
+
.where(eq(agentWakeupRequests.id, wakeupRequest.id));
|
|
1462
|
+
await tx
|
|
1463
|
+
.update(issues)
|
|
1464
|
+
.set({
|
|
1465
|
+
executionRunId: followupRun.id,
|
|
1466
|
+
executionAgentNameKey: normalizeAgentNameKey(agent.name),
|
|
1467
|
+
executionLockedAt: now,
|
|
1468
|
+
updatedAt: now,
|
|
1469
|
+
})
|
|
1470
|
+
.where(eq(issues.id, issue.id));
|
|
1471
|
+
return {
|
|
1472
|
+
kind: "queued",
|
|
1473
|
+
run: followupRun,
|
|
1474
|
+
issue,
|
|
1475
|
+
originRunId,
|
|
1476
|
+
previousRunId: run.id,
|
|
1477
|
+
attempt: nextAttempt,
|
|
1478
|
+
requestedAt,
|
|
1479
|
+
};
|
|
1480
|
+
}
|
|
1481
|
+
async function countRunningRunsForAgent(agentId) {
|
|
1482
|
+
const [{ count }] = await db
|
|
1483
|
+
.select({ count: sql `count(*)` })
|
|
1484
|
+
.from(heartbeatRuns)
|
|
1485
|
+
.where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "running")));
|
|
1486
|
+
return Number(count ?? 0);
|
|
1487
|
+
}
|
|
1488
|
+
async function claimQueuedRun(run) {
|
|
1489
|
+
if (run.status !== "queued")
|
|
1490
|
+
return run;
|
|
1491
|
+
if (run.wakeupRequestId) {
|
|
1492
|
+
const wakeup = await db
|
|
1493
|
+
.select({ requestedAt: agentWakeupRequests.requestedAt })
|
|
1494
|
+
.from(agentWakeupRequests)
|
|
1495
|
+
.where(eq(agentWakeupRequests.id, run.wakeupRequestId))
|
|
1496
|
+
.then((rows) => rows[0] ?? null);
|
|
1497
|
+
if (wakeup && new Date(wakeup.requestedAt).getTime() > Date.now()) {
|
|
1498
|
+
return null;
|
|
1499
|
+
}
|
|
1500
|
+
}
|
|
1501
|
+
const agent = await getAgent(run.agentId);
|
|
1502
|
+
if (!agent) {
|
|
1503
|
+
await cancelRunInternal(run.id, "Cancelled because the agent no longer exists");
|
|
1504
|
+
return null;
|
|
1505
|
+
}
|
|
1506
|
+
if (agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval") {
|
|
1507
|
+
await cancelRunInternal(run.id, "Cancelled because the agent is not invokable");
|
|
1508
|
+
return null;
|
|
1509
|
+
}
|
|
1510
|
+
const context = parseObject(run.contextSnapshot);
|
|
1511
|
+
const budgetBlock = await budgets.getInvocationBlock(run.orgId, run.agentId, {
|
|
1512
|
+
issueId: readNonEmptyString(context.issueId),
|
|
1513
|
+
projectId: readNonEmptyString(context.projectId),
|
|
1514
|
+
});
|
|
1515
|
+
if (budgetBlock) {
|
|
1516
|
+
await cancelRunInternal(run.id, budgetBlock.reason);
|
|
1517
|
+
return null;
|
|
1518
|
+
}
|
|
1519
|
+
const claimedAt = new Date();
|
|
1520
|
+
const claimed = await db
|
|
1521
|
+
.update(heartbeatRuns)
|
|
1522
|
+
.set({
|
|
1523
|
+
status: "running",
|
|
1524
|
+
startedAt: run.startedAt ?? claimedAt,
|
|
1525
|
+
updatedAt: claimedAt,
|
|
1526
|
+
})
|
|
1527
|
+
.where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "queued")))
|
|
1528
|
+
.returning()
|
|
1529
|
+
.then((rows) => rows[0] ?? null);
|
|
1530
|
+
if (!claimed)
|
|
1531
|
+
return null;
|
|
1532
|
+
publishLiveEvent({
|
|
1533
|
+
orgId: claimed.orgId,
|
|
1534
|
+
type: "heartbeat.run.status",
|
|
1535
|
+
payload: {
|
|
1536
|
+
runId: claimed.id,
|
|
1537
|
+
agentId: claimed.agentId,
|
|
1538
|
+
status: claimed.status,
|
|
1539
|
+
invocationSource: claimed.invocationSource,
|
|
1540
|
+
triggerDetail: claimed.triggerDetail,
|
|
1541
|
+
error: claimed.error ?? null,
|
|
1542
|
+
errorCode: claimed.errorCode ?? null,
|
|
1543
|
+
startedAt: claimed.startedAt ? new Date(claimed.startedAt).toISOString() : null,
|
|
1544
|
+
finishedAt: claimed.finishedAt ? new Date(claimed.finishedAt).toISOString() : null,
|
|
1545
|
+
},
|
|
1546
|
+
});
|
|
1547
|
+
await setWakeupStatus(claimed.wakeupRequestId, "claimed", { claimedAt });
|
|
1548
|
+
return claimed;
|
|
1549
|
+
}
|
|
1550
|
+
async function finalizeAgentStatus(agentId, outcome) {
|
|
1551
|
+
const existing = await getAgent(agentId);
|
|
1552
|
+
if (!existing)
|
|
1553
|
+
return;
|
|
1554
|
+
if (existing.status === "paused" || existing.status === "terminated") {
|
|
1555
|
+
return;
|
|
1556
|
+
}
|
|
1557
|
+
const runningCount = await countRunningRunsForAgent(agentId);
|
|
1558
|
+
const nextStatus = runningCount > 0
|
|
1559
|
+
? "running"
|
|
1560
|
+
: outcome === "succeeded" || outcome === "cancelled"
|
|
1561
|
+
? "idle"
|
|
1562
|
+
: "error";
|
|
1563
|
+
const updated = await db
|
|
1564
|
+
.update(agents)
|
|
1565
|
+
.set({
|
|
1566
|
+
status: nextStatus,
|
|
1567
|
+
lastHeartbeatAt: new Date(),
|
|
1568
|
+
updatedAt: new Date(),
|
|
1569
|
+
})
|
|
1570
|
+
.where(eq(agents.id, agentId))
|
|
1571
|
+
.returning()
|
|
1572
|
+
.then((rows) => rows[0] ?? null);
|
|
1573
|
+
if (updated) {
|
|
1574
|
+
publishLiveEvent({
|
|
1575
|
+
orgId: updated.orgId,
|
|
1576
|
+
type: "agent.status",
|
|
1577
|
+
payload: {
|
|
1578
|
+
agentId: updated.id,
|
|
1579
|
+
status: updated.status,
|
|
1580
|
+
lastHeartbeatAt: updated.lastHeartbeatAt
|
|
1581
|
+
? new Date(updated.lastHeartbeatAt).toISOString()
|
|
1582
|
+
: null,
|
|
1583
|
+
outcome,
|
|
1584
|
+
},
|
|
1585
|
+
});
|
|
1586
|
+
}
|
|
1587
|
+
}
|
|
1588
|
+
async function reapOrphanedRuns(opts) {
|
|
1589
|
+
const staleThresholdMs = opts?.staleThresholdMs ?? 0;
|
|
1590
|
+
const now = new Date();
|
|
1591
|
+
// Find all runs stuck in "running" state (queued runs are legitimately waiting; resumeQueuedRuns handles them)
|
|
1592
|
+
const activeRuns = await db
|
|
1593
|
+
.select({
|
|
1594
|
+
run: heartbeatRuns,
|
|
1595
|
+
agentRuntimeType: agents.agentRuntimeType,
|
|
1596
|
+
})
|
|
1597
|
+
.from(heartbeatRuns)
|
|
1598
|
+
.innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
|
|
1599
|
+
.where(eq(heartbeatRuns.status, "running"));
|
|
1600
|
+
const reaped = [];
|
|
1601
|
+
for (const { run, agentRuntimeType } of activeRuns) {
|
|
1602
|
+
if (runningProcesses.has(run.id) || activeRunExecutions.has(run.id))
|
|
1603
|
+
continue;
|
|
1604
|
+
// Apply staleness threshold to avoid false positives
|
|
1605
|
+
if (staleThresholdMs > 0) {
|
|
1606
|
+
const refTime = run.updatedAt ? new Date(run.updatedAt).getTime() : 0;
|
|
1607
|
+
if (now.getTime() - refTime < staleThresholdMs)
|
|
1608
|
+
continue;
|
|
1609
|
+
}
|
|
1610
|
+
const tracksLocalChild = isTrackedLocalChildProcessAdapter(agentRuntimeType);
|
|
1611
|
+
let detachedTerminationMessage = null;
|
|
1612
|
+
if (tracksLocalChild && run.processPid && isProcessAlive(run.processPid)) {
|
|
1613
|
+
const termination = await terminateOrphanedProcess(run.processPid);
|
|
1614
|
+
if (termination.stillAlive) {
|
|
1615
|
+
const detachedMessage = termination.error
|
|
1616
|
+
? `Lost in-memory process handle, child pid ${run.processPid} is still alive, and Rudder could not terminate it: ${termination.error}`
|
|
1617
|
+
: `Lost in-memory process handle, but child pid ${run.processPid} is still alive`;
|
|
1618
|
+
const detachedRun = await setRunStatus(run.id, "running", {
|
|
1619
|
+
error: detachedMessage,
|
|
1620
|
+
errorCode: DETACHED_PROCESS_ERROR_CODE,
|
|
1621
|
+
});
|
|
1622
|
+
if (detachedRun) {
|
|
1623
|
+
await appendRunEvent(detachedRun, await nextRunEventSeq(detachedRun.id), {
|
|
1624
|
+
eventType: "lifecycle",
|
|
1625
|
+
stream: "system",
|
|
1626
|
+
level: "warn",
|
|
1627
|
+
message: detachedMessage,
|
|
1628
|
+
payload: {
|
|
1629
|
+
processPid: run.processPid,
|
|
1630
|
+
},
|
|
1631
|
+
});
|
|
1632
|
+
}
|
|
1633
|
+
continue;
|
|
1634
|
+
}
|
|
1635
|
+
detachedTerminationMessage = termination.terminationSignal
|
|
1636
|
+
? `Terminated detached child pid ${run.processPid} with ${termination.terminationSignal} after Rudder lost its process handle`
|
|
1637
|
+
: `Detached child pid ${run.processPid} exited before Rudder could terminate it`;
|
|
1638
|
+
}
|
|
1639
|
+
const shouldRetry = tracksLocalChild && !!run.processPid && (run.processLossRetryCount ?? 0) < 1;
|
|
1640
|
+
const baseMessage = run.processPid
|
|
1641
|
+
? `Process lost -- child pid ${run.processPid} is no longer running`
|
|
1642
|
+
: "Process lost -- server may have restarted";
|
|
1643
|
+
let finalizedRun = await setRunStatus(run.id, "failed", {
|
|
1644
|
+
error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
|
|
1645
|
+
errorCode: "process_lost",
|
|
1646
|
+
finishedAt: now,
|
|
1647
|
+
});
|
|
1648
|
+
await setWakeupStatus(run.wakeupRequestId, "failed", {
|
|
1649
|
+
finishedAt: now,
|
|
1650
|
+
error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
|
|
1651
|
+
});
|
|
1652
|
+
if (!finalizedRun)
|
|
1653
|
+
finalizedRun = await getRun(run.id);
|
|
1654
|
+
if (!finalizedRun)
|
|
1655
|
+
continue;
|
|
1656
|
+
if (detachedTerminationMessage) {
|
|
1657
|
+
await appendRunEvent(finalizedRun, await nextRunEventSeq(finalizedRun.id), {
|
|
1658
|
+
eventType: "lifecycle",
|
|
1659
|
+
stream: "system",
|
|
1660
|
+
level: "warn",
|
|
1661
|
+
message: detachedTerminationMessage,
|
|
1662
|
+
payload: {
|
|
1663
|
+
...(run.processPid ? { processPid: run.processPid } : {}),
|
|
1664
|
+
},
|
|
1665
|
+
});
|
|
1666
|
+
}
|
|
1667
|
+
let retriedRun = null;
|
|
1668
|
+
if (shouldRetry) {
|
|
1669
|
+
const agent = await getAgent(run.agentId);
|
|
1670
|
+
if (agent) {
|
|
1671
|
+
retriedRun = await enqueueProcessLossRetry(finalizedRun, agent, now);
|
|
1672
|
+
}
|
|
1673
|
+
}
|
|
1674
|
+
else {
|
|
1675
|
+
await releaseIssueExecutionAndPromote(finalizedRun);
|
|
1676
|
+
}
|
|
1677
|
+
await appendRunEvent(finalizedRun, await nextRunEventSeq(finalizedRun.id), {
|
|
1678
|
+
eventType: "lifecycle",
|
|
1679
|
+
stream: "system",
|
|
1680
|
+
level: "error",
|
|
1681
|
+
message: shouldRetry
|
|
1682
|
+
? `${baseMessage}; queued retry ${retriedRun?.id ?? ""}`.trim()
|
|
1683
|
+
: baseMessage,
|
|
1684
|
+
payload: {
|
|
1685
|
+
...(run.processPid ? { processPid: run.processPid } : {}),
|
|
1686
|
+
...(retriedRun ? { retryRunId: retriedRun.id } : {}),
|
|
1687
|
+
},
|
|
1688
|
+
});
|
|
1689
|
+
await finalizeAgentStatus(run.agentId, "failed");
|
|
1690
|
+
await startNextQueuedRunForAgent(run.agentId);
|
|
1691
|
+
runningProcesses.delete(run.id);
|
|
1692
|
+
reaped.push(run.id);
|
|
1693
|
+
}
|
|
1694
|
+
if (reaped.length > 0) {
|
|
1695
|
+
logger.warn({ reapedCount: reaped.length, runIds: reaped }, "reaped orphaned heartbeat runs");
|
|
1696
|
+
}
|
|
1697
|
+
return { reaped: reaped.length, runIds: reaped };
|
|
1698
|
+
}
|
|
1699
|
+
async function resumeQueuedRuns() {
|
|
1700
|
+
const queuedRuns = await db
|
|
1701
|
+
.select({ agentId: heartbeatRuns.agentId })
|
|
1702
|
+
.from(heartbeatRuns)
|
|
1703
|
+
.where(eq(heartbeatRuns.status, "queued"));
|
|
1704
|
+
const agentIds = [...new Set(queuedRuns.map((r) => r.agentId))];
|
|
1705
|
+
for (const agentId of agentIds) {
|
|
1706
|
+
await startNextQueuedRunForAgent(agentId);
|
|
1707
|
+
}
|
|
1708
|
+
}
|
|
1709
|
+
async function updateRuntimeState(agent, run, result, session, normalizedUsage) {
|
|
1710
|
+
await ensureRuntimeState(agent);
|
|
1711
|
+
const usage = normalizedUsage ?? normalizeUsageTotals(result.usage);
|
|
1712
|
+
const inputTokens = usage?.inputTokens ?? 0;
|
|
1713
|
+
const outputTokens = usage?.outputTokens ?? 0;
|
|
1714
|
+
const cachedInputTokens = usage?.cachedInputTokens ?? 0;
|
|
1715
|
+
const billingType = normalizeLedgerBillingType(result.billingType);
|
|
1716
|
+
const additionalCostCents = normalizeBilledCostCents(result.costUsd, billingType);
|
|
1717
|
+
const hasTokenUsage = inputTokens > 0 || outputTokens > 0 || cachedInputTokens > 0;
|
|
1718
|
+
const provider = result.provider ?? "unknown";
|
|
1719
|
+
const biller = resolveLedgerBiller(result);
|
|
1720
|
+
const ledgerScope = await resolveLedgerScopeForRun(db, agent.orgId, run);
|
|
1721
|
+
await db
|
|
1722
|
+
.update(agentRuntimeState)
|
|
1723
|
+
.set({
|
|
1724
|
+
agentRuntimeType: agent.agentRuntimeType,
|
|
1725
|
+
sessionId: session.legacySessionId,
|
|
1726
|
+
lastRunId: run.id,
|
|
1727
|
+
lastRunStatus: run.status,
|
|
1728
|
+
lastError: result.errorMessage ?? null,
|
|
1729
|
+
totalInputTokens: sql `${agentRuntimeState.totalInputTokens} + ${inputTokens}`,
|
|
1730
|
+
totalOutputTokens: sql `${agentRuntimeState.totalOutputTokens} + ${outputTokens}`,
|
|
1731
|
+
totalCachedInputTokens: sql `${agentRuntimeState.totalCachedInputTokens} + ${cachedInputTokens}`,
|
|
1732
|
+
totalCostCents: sql `${agentRuntimeState.totalCostCents} + ${additionalCostCents}`,
|
|
1733
|
+
updatedAt: new Date(),
|
|
1734
|
+
})
|
|
1735
|
+
.where(eq(agentRuntimeState.agentId, agent.id));
|
|
1736
|
+
if (additionalCostCents > 0 || hasTokenUsage) {
|
|
1737
|
+
const costs = costService(db, budgetHooks);
|
|
1738
|
+
await costs.createEvent(agent.orgId, {
|
|
1739
|
+
heartbeatRunId: run.id,
|
|
1740
|
+
agentId: agent.id,
|
|
1741
|
+
issueId: ledgerScope.issueId,
|
|
1742
|
+
projectId: ledgerScope.projectId,
|
|
1743
|
+
provider,
|
|
1744
|
+
biller,
|
|
1745
|
+
billingType,
|
|
1746
|
+
model: result.model ?? "unknown",
|
|
1747
|
+
inputTokens,
|
|
1748
|
+
cachedInputTokens,
|
|
1749
|
+
outputTokens,
|
|
1750
|
+
costCents: additionalCostCents,
|
|
1751
|
+
occurredAt: new Date(),
|
|
1752
|
+
});
|
|
1753
|
+
}
|
|
1754
|
+
}
|
|
1755
|
+
async function startNextQueuedRunForAgent(agentId) {
|
|
1756
|
+
return withAgentStartLock(agentId, async () => {
|
|
1757
|
+
const agent = await getAgent(agentId);
|
|
1758
|
+
if (!agent)
|
|
1759
|
+
return [];
|
|
1760
|
+
if (agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval") {
|
|
1761
|
+
return [];
|
|
1762
|
+
}
|
|
1763
|
+
const policy = parseHeartbeatPolicy(agent);
|
|
1764
|
+
const runningCount = await countRunningRunsForAgent(agentId);
|
|
1765
|
+
const availableSlots = Math.max(0, policy.maxConcurrentRuns - runningCount);
|
|
1766
|
+
if (availableSlots <= 0)
|
|
1767
|
+
return [];
|
|
1768
|
+
const queuedRuns = await db
|
|
1769
|
+
.select()
|
|
1770
|
+
.from(heartbeatRuns)
|
|
1771
|
+
.where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued"), sql `(
|
|
1772
|
+
${heartbeatRuns.wakeupRequestId} is null
|
|
1773
|
+
or exists (
|
|
1774
|
+
select 1
|
|
1775
|
+
from ${agentWakeupRequests}
|
|
1776
|
+
where ${agentWakeupRequests.id} = ${heartbeatRuns.wakeupRequestId}
|
|
1777
|
+
and ${agentWakeupRequests.requestedAt} <= now()
|
|
1778
|
+
)
|
|
1779
|
+
)`))
|
|
1780
|
+
.orderBy(asc(heartbeatRuns.createdAt))
|
|
1781
|
+
.limit(availableSlots);
|
|
1782
|
+
if (queuedRuns.length === 0)
|
|
1783
|
+
return [];
|
|
1784
|
+
const claimedRuns = [];
|
|
1785
|
+
for (const queuedRun of queuedRuns) {
|
|
1786
|
+
const claimed = await claimQueuedRun(queuedRun);
|
|
1787
|
+
if (claimed)
|
|
1788
|
+
claimedRuns.push(claimed);
|
|
1789
|
+
}
|
|
1790
|
+
if (claimedRuns.length === 0)
|
|
1791
|
+
return [];
|
|
1792
|
+
for (const claimedRun of claimedRuns) {
|
|
1793
|
+
void executeRun(claimedRun.id).catch((err) => {
|
|
1794
|
+
logger.error({ err, runId: claimedRun.id }, "queued heartbeat execution failed");
|
|
1795
|
+
});
|
|
1796
|
+
}
|
|
1797
|
+
return claimedRuns;
|
|
1798
|
+
});
|
|
1799
|
+
}
|
|
1800
|
+
async function executeRun(runId) {
|
|
1801
|
+
let run = await getRun(runId);
|
|
1802
|
+
if (!run)
|
|
1803
|
+
return;
|
|
1804
|
+
if (run.status !== "queued" && run.status !== "running")
|
|
1805
|
+
return;
|
|
1806
|
+
if (run.status === "queued") {
|
|
1807
|
+
const claimed = await claimQueuedRun(run);
|
|
1808
|
+
if (!claimed) {
|
|
1809
|
+
// Another worker has already claimed or finalized this run.
|
|
1810
|
+
return;
|
|
1811
|
+
}
|
|
1812
|
+
run = claimed;
|
|
1813
|
+
}
|
|
1814
|
+
activeRunExecutions.add(run.id);
|
|
1815
|
+
try {
|
|
1816
|
+
const agent = await getAgent(run.agentId);
|
|
1817
|
+
if (!agent) {
|
|
1818
|
+
await setRunStatus(runId, "failed", {
|
|
1819
|
+
error: "Agent not found",
|
|
1820
|
+
errorCode: "agent_not_found",
|
|
1821
|
+
finishedAt: new Date(),
|
|
1822
|
+
});
|
|
1823
|
+
await setWakeupStatus(run.wakeupRequestId, "failed", {
|
|
1824
|
+
finishedAt: new Date(),
|
|
1825
|
+
error: "Agent not found",
|
|
1826
|
+
});
|
|
1827
|
+
const failedRun = await getRun(runId);
|
|
1828
|
+
if (failedRun)
|
|
1829
|
+
await releaseIssueExecutionAndPromote(failedRun);
|
|
1830
|
+
return;
|
|
1831
|
+
}
|
|
1832
|
+
const heartbeatObservationContext = buildHeartbeatObservabilityContext(run, {
|
|
1833
|
+
runtime: agent.agentRuntimeType,
|
|
1834
|
+
metadata: {
|
|
1835
|
+
agentName: agent.name,
|
|
1836
|
+
invocationSource: run.invocationSource,
|
|
1837
|
+
triggerDetail: run.triggerDetail,
|
|
1838
|
+
},
|
|
1839
|
+
});
|
|
1840
|
+
await withExecutionObservation(heartbeatObservationContext, {
|
|
1841
|
+
name: buildHeartbeatObservationName(run, agent.name),
|
|
1842
|
+
asType: "agent",
|
|
1843
|
+
input: {
|
|
1844
|
+
agentId: agent.id,
|
|
1845
|
+
agentName: agent.name,
|
|
1846
|
+
invocationSource: run.invocationSource,
|
|
1847
|
+
triggerDetail: run.triggerDetail,
|
|
1848
|
+
issueId: readNonEmptyString(parseObject(run.contextSnapshot).issueId),
|
|
1849
|
+
},
|
|
1850
|
+
}, async (observation) => {
|
|
1851
|
+
const executionTranscript = [];
|
|
1852
|
+
let stdoutTranscriptBuffer = "";
|
|
1853
|
+
let stderrTranscriptBuffer = "";
|
|
1854
|
+
let stdoutTranscriptParser = null;
|
|
1855
|
+
let transcriptFallbackResult = null;
|
|
1856
|
+
let finalObservationOutput = null;
|
|
1857
|
+
let finalObservationStatus = run.status;
|
|
1858
|
+
let finalObservationSessionId = heartbeatObservationContext.sessionKey ?? null;
|
|
1859
|
+
const runtime = await ensureRuntimeState(agent);
|
|
1860
|
+
const context = parseObject(run.contextSnapshot);
|
|
1861
|
+
const taskKey = deriveTaskKey(context, null);
|
|
1862
|
+
const sessionCodec = getAgentRuntimeSessionCodec(agent.agentRuntimeType);
|
|
1863
|
+
const issueId = readNonEmptyString(context.issueId);
|
|
1864
|
+
const issueContext = issueId
|
|
1865
|
+
? await db
|
|
1866
|
+
.select({
|
|
1867
|
+
id: issues.id,
|
|
1868
|
+
identifier: issues.identifier,
|
|
1869
|
+
title: issues.title,
|
|
1870
|
+
projectId: issues.projectId,
|
|
1871
|
+
projectWorkspaceId: issues.projectWorkspaceId,
|
|
1872
|
+
executionWorkspaceId: issues.executionWorkspaceId,
|
|
1873
|
+
executionWorkspacePreference: issues.executionWorkspacePreference,
|
|
1874
|
+
assigneeAgentId: issues.assigneeAgentId,
|
|
1875
|
+
assigneeAgentRuntimeOverrides: issues.assigneeAgentRuntimeOverrides,
|
|
1876
|
+
executionWorkspaceSettings: issues.executionWorkspaceSettings,
|
|
1877
|
+
})
|
|
1878
|
+
.from(issues)
|
|
1879
|
+
.where(and(eq(issues.id, issueId), eq(issues.orgId, agent.orgId)))
|
|
1880
|
+
.then((rows) => rows[0] ?? null)
|
|
1881
|
+
: null;
|
|
1882
|
+
const issueAssigneeOverrides = issueContext && issueContext.assigneeAgentId === agent.id
|
|
1883
|
+
? parseIssueAssigneeAgentRuntimeOverrides(issueContext.assigneeAgentRuntimeOverrides)
|
|
1884
|
+
: null;
|
|
1885
|
+
const issueExecutionWorkspaceSettings = parseIssueExecutionWorkspaceSettings(issueContext?.executionWorkspaceSettings);
|
|
1886
|
+
const contextProjectId = readNonEmptyString(context.projectId);
|
|
1887
|
+
const executionProjectId = issueContext?.projectId ?? contextProjectId;
|
|
1888
|
+
const projectExecutionWorkspacePolicy = executionProjectId
|
|
1889
|
+
? await db
|
|
1890
|
+
.select({ executionWorkspacePolicy: projects.executionWorkspacePolicy })
|
|
1891
|
+
.from(projects)
|
|
1892
|
+
.where(and(eq(projects.id, executionProjectId), eq(projects.orgId, agent.orgId)))
|
|
1893
|
+
.then((rows) => parseProjectExecutionWorkspacePolicy(rows[0]?.executionWorkspacePolicy))
|
|
1894
|
+
: null;
|
|
1895
|
+
const taskSession = taskKey
|
|
1896
|
+
? await getTaskSession(agent.orgId, agent.id, agent.agentRuntimeType, taskKey)
|
|
1897
|
+
: null;
|
|
1898
|
+
const resetTaskSession = shouldResetTaskSessionForWake(context);
|
|
1899
|
+
const sessionResetReason = describeSessionResetReason(context);
|
|
1900
|
+
const taskSessionForRun = resetTaskSession ? null : taskSession;
|
|
1901
|
+
const explicitResumeSessionParams = normalizeSessionParams(sessionCodec.deserialize(parseObject(context.resumeSessionParams)));
|
|
1902
|
+
const explicitResumeSessionDisplayId = truncateDisplayId(readNonEmptyString(context.resumeSessionDisplayId) ??
|
|
1903
|
+
(sessionCodec.getDisplayId ? sessionCodec.getDisplayId(explicitResumeSessionParams) : null) ??
|
|
1904
|
+
readNonEmptyString(explicitResumeSessionParams?.sessionId));
|
|
1905
|
+
const previousSessionParams = explicitResumeSessionParams ??
|
|
1906
|
+
(explicitResumeSessionDisplayId ? { sessionId: explicitResumeSessionDisplayId } : null) ??
|
|
1907
|
+
normalizeSessionParams(sessionCodec.deserialize(taskSessionForRun?.sessionParamsJson ?? null));
|
|
1908
|
+
const config = parseObject(agent.agentRuntimeConfig);
|
|
1909
|
+
const executionWorkspaceMode = resolveExecutionWorkspaceMode({
|
|
1910
|
+
projectPolicy: projectExecutionWorkspacePolicy,
|
|
1911
|
+
issueSettings: issueExecutionWorkspaceSettings,
|
|
1912
|
+
legacyUseProjectWorkspace: issueAssigneeOverrides?.useProjectWorkspace ?? null,
|
|
1913
|
+
});
|
|
1914
|
+
const resolvedWorkspace = await runContextSvc.resolveWorkspaceForRun(agent, context, previousSessionParams, { useProjectWorkspace: executionWorkspaceMode !== "agent_default" });
|
|
1915
|
+
const workspaceManagedConfig = buildExecutionWorkspaceAdapterConfig({
|
|
1916
|
+
agentConfig: config,
|
|
1917
|
+
projectPolicy: projectExecutionWorkspacePolicy,
|
|
1918
|
+
issueSettings: issueExecutionWorkspaceSettings,
|
|
1919
|
+
mode: executionWorkspaceMode,
|
|
1920
|
+
legacyUseProjectWorkspace: issueAssigneeOverrides?.useProjectWorkspace ?? null,
|
|
1921
|
+
});
|
|
1922
|
+
const mergedConfig = issueAssigneeOverrides?.agentRuntimeConfig
|
|
1923
|
+
? { ...workspaceManagedConfig, ...issueAssigneeOverrides.agentRuntimeConfig }
|
|
1924
|
+
: workspaceManagedConfig;
|
|
1925
|
+
const { resolvedConfig, runtimeConfig, runtimeSkillEntries, secretKeys } = await runContextSvc.prepareRuntimeConfig({
|
|
1926
|
+
scene: "heartbeat",
|
|
1927
|
+
agent,
|
|
1928
|
+
baseConfig: mergedConfig,
|
|
1929
|
+
});
|
|
1930
|
+
heartbeatObservationContext.metadata = {
|
|
1931
|
+
...(heartbeatObservationContext.metadata ?? {}),
|
|
1932
|
+
...buildHeartbeatRuntimeTraceMetadata({
|
|
1933
|
+
runtimeConfig,
|
|
1934
|
+
runtimeSkills: runtimeSkillEntries,
|
|
1935
|
+
}),
|
|
1936
|
+
};
|
|
1937
|
+
const issueRef = issueContext
|
|
1938
|
+
? {
|
|
1939
|
+
id: issueContext.id,
|
|
1940
|
+
identifier: issueContext.identifier,
|
|
1941
|
+
title: issueContext.title,
|
|
1942
|
+
projectId: issueContext.projectId,
|
|
1943
|
+
projectWorkspaceId: issueContext.projectWorkspaceId,
|
|
1944
|
+
executionWorkspaceId: issueContext.executionWorkspaceId,
|
|
1945
|
+
executionWorkspacePreference: issueContext.executionWorkspacePreference,
|
|
1946
|
+
}
|
|
1947
|
+
: null;
|
|
1948
|
+
const rootObservationInput = {
|
|
1949
|
+
agentId: agent.id,
|
|
1950
|
+
agentName: agent.name,
|
|
1951
|
+
invocationSource: run.invocationSource,
|
|
1952
|
+
triggerDetail: run.triggerDetail,
|
|
1953
|
+
issue: issueRef
|
|
1954
|
+
? {
|
|
1955
|
+
id: issueRef.id,
|
|
1956
|
+
identifier: issueRef.identifier ?? null,
|
|
1957
|
+
title: issueRef.title ?? null,
|
|
1958
|
+
}
|
|
1959
|
+
: null,
|
|
1960
|
+
};
|
|
1961
|
+
updateExecutionObservation(observation, heartbeatObservationContext, {
|
|
1962
|
+
input: rootObservationInput,
|
|
1963
|
+
});
|
|
1964
|
+
updateExecutionTraceIO(observation, { input: rootObservationInput });
|
|
1965
|
+
if (issueRef) {
|
|
1966
|
+
updateExecutionTraceName(observation, buildIssueRunTraceName({
|
|
1967
|
+
issueTitle: issueRef.title,
|
|
1968
|
+
issueId: issueRef.id,
|
|
1969
|
+
}));
|
|
1970
|
+
}
|
|
1971
|
+
const existingExecutionWorkspace = issueRef?.executionWorkspaceId ? await executionWorkspacesSvc.getById(issueRef.executionWorkspaceId) : null;
|
|
1972
|
+
const workspaceOperationRecorder = workspaceOperationsSvc.createRecorder({
|
|
1973
|
+
orgId: agent.orgId,
|
|
1974
|
+
heartbeatRunId: run.id,
|
|
1975
|
+
executionWorkspaceId: existingExecutionWorkspace?.id ?? null,
|
|
1976
|
+
});
|
|
1977
|
+
const executionWorkspace = await realizeExecutionWorkspace({
|
|
1978
|
+
base: {
|
|
1979
|
+
baseCwd: resolvedWorkspace.cwd,
|
|
1980
|
+
source: resolvedWorkspace.source,
|
|
1981
|
+
projectId: resolvedWorkspace.projectId,
|
|
1982
|
+
workspaceId: resolvedWorkspace.workspaceId,
|
|
1983
|
+
repoUrl: resolvedWorkspace.repoUrl,
|
|
1984
|
+
repoRef: resolvedWorkspace.repoRef,
|
|
1985
|
+
},
|
|
1986
|
+
config: runtimeConfig,
|
|
1987
|
+
issue: issueRef,
|
|
1988
|
+
agent: {
|
|
1989
|
+
id: agent.id,
|
|
1990
|
+
name: agent.name,
|
|
1991
|
+
orgId: agent.orgId,
|
|
1992
|
+
},
|
|
1993
|
+
recorder: workspaceOperationRecorder,
|
|
1994
|
+
});
|
|
1995
|
+
const resolvedProjectId = executionWorkspace.projectId ?? issueRef?.projectId ?? executionProjectId ?? null;
|
|
1996
|
+
const resolvedProjectWorkspaceId = issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null;
|
|
1997
|
+
const shouldReuseExisting = issueRef?.executionWorkspacePreference === "reuse_existing" &&
|
|
1998
|
+
existingExecutionWorkspace &&
|
|
1999
|
+
existingExecutionWorkspace.status !== "archived";
|
|
2000
|
+
let persistedExecutionWorkspace = null;
|
|
2001
|
+
try {
|
|
2002
|
+
persistedExecutionWorkspace = shouldReuseExisting && existingExecutionWorkspace
|
|
2003
|
+
? await executionWorkspacesSvc.update(existingExecutionWorkspace.id, {
|
|
2004
|
+
cwd: executionWorkspace.cwd,
|
|
2005
|
+
repoUrl: executionWorkspace.repoUrl,
|
|
2006
|
+
baseRef: executionWorkspace.repoRef,
|
|
2007
|
+
branchName: executionWorkspace.branchName,
|
|
2008
|
+
providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs",
|
|
2009
|
+
providerRef: executionWorkspace.worktreePath,
|
|
2010
|
+
status: "active",
|
|
2011
|
+
lastUsedAt: new Date(),
|
|
2012
|
+
metadata: {
|
|
2013
|
+
...(existingExecutionWorkspace.metadata ?? {}),
|
|
2014
|
+
source: executionWorkspace.source,
|
|
2015
|
+
createdByRuntime: executionWorkspace.created,
|
|
2016
|
+
},
|
|
2017
|
+
})
|
|
2018
|
+
: resolvedProjectId
|
|
2019
|
+
? await executionWorkspacesSvc.create({
|
|
2020
|
+
orgId: agent.orgId,
|
|
2021
|
+
projectId: resolvedProjectId,
|
|
2022
|
+
projectWorkspaceId: resolvedProjectWorkspaceId,
|
|
2023
|
+
sourceIssueId: issueRef?.id ?? null,
|
|
2024
|
+
mode: executionWorkspaceMode === "isolated_workspace"
|
|
2025
|
+
? "isolated_workspace"
|
|
2026
|
+
: executionWorkspaceMode === "operator_branch"
|
|
2027
|
+
? "operator_branch"
|
|
2028
|
+
: executionWorkspaceMode === "agent_default"
|
|
2029
|
+
? "adapter_managed"
|
|
2030
|
+
: "shared_workspace",
|
|
2031
|
+
strategyType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "project_primary",
|
|
2032
|
+
name: executionWorkspace.branchName ?? issueRef?.identifier ?? `workspace-${agent.id.slice(0, 8)}`,
|
|
2033
|
+
status: "active",
|
|
2034
|
+
cwd: executionWorkspace.cwd,
|
|
2035
|
+
repoUrl: executionWorkspace.repoUrl,
|
|
2036
|
+
baseRef: executionWorkspace.repoRef,
|
|
2037
|
+
branchName: executionWorkspace.branchName,
|
|
2038
|
+
providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs",
|
|
2039
|
+
providerRef: executionWorkspace.worktreePath,
|
|
2040
|
+
lastUsedAt: new Date(),
|
|
2041
|
+
openedAt: new Date(),
|
|
2042
|
+
metadata: {
|
|
2043
|
+
source: executionWorkspace.source,
|
|
2044
|
+
createdByRuntime: executionWorkspace.created,
|
|
2045
|
+
},
|
|
2046
|
+
})
|
|
2047
|
+
: null;
|
|
2048
|
+
}
|
|
2049
|
+
catch (error) {
|
|
2050
|
+
if (executionWorkspace.created) {
|
|
2051
|
+
try {
|
|
2052
|
+
await cleanupExecutionWorkspaceArtifacts({
|
|
2053
|
+
workspace: {
|
|
2054
|
+
id: existingExecutionWorkspace?.id ?? `transient-${run.id}`,
|
|
2055
|
+
cwd: executionWorkspace.cwd,
|
|
2056
|
+
providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs",
|
|
2057
|
+
providerRef: executionWorkspace.worktreePath,
|
|
2058
|
+
branchName: executionWorkspace.branchName,
|
|
2059
|
+
repoUrl: executionWorkspace.repoUrl,
|
|
2060
|
+
baseRef: executionWorkspace.repoRef,
|
|
2061
|
+
projectId: resolvedProjectId,
|
|
2062
|
+
projectWorkspaceId: resolvedProjectWorkspaceId,
|
|
2063
|
+
sourceIssueId: issueRef?.id ?? null,
|
|
2064
|
+
metadata: {
|
|
2065
|
+
createdByRuntime: true,
|
|
2066
|
+
source: executionWorkspace.source,
|
|
2067
|
+
},
|
|
2068
|
+
},
|
|
2069
|
+
projectWorkspace: {
|
|
2070
|
+
cwd: resolvedWorkspace.cwd,
|
|
2071
|
+
cleanupCommand: null,
|
|
2072
|
+
},
|
|
2073
|
+
teardownCommand: projectExecutionWorkspacePolicy?.workspaceStrategy?.teardownCommand ?? null,
|
|
2074
|
+
recorder: workspaceOperationRecorder,
|
|
2075
|
+
});
|
|
2076
|
+
}
|
|
2077
|
+
catch (cleanupError) {
|
|
2078
|
+
logger.warn({
|
|
2079
|
+
runId: run.id,
|
|
2080
|
+
issueId,
|
|
2081
|
+
executionWorkspaceCwd: executionWorkspace.cwd,
|
|
2082
|
+
cleanupError: cleanupError instanceof Error ? cleanupError.message : String(cleanupError),
|
|
2083
|
+
}, "Failed to cleanup realized execution workspace after persistence failure");
|
|
2084
|
+
}
|
|
2085
|
+
}
|
|
2086
|
+
throw error;
|
|
2087
|
+
}
|
|
2088
|
+
await workspaceOperationRecorder.attachExecutionWorkspaceId(persistedExecutionWorkspace?.id ?? null);
|
|
2089
|
+
if (existingExecutionWorkspace &&
|
|
2090
|
+
persistedExecutionWorkspace &&
|
|
2091
|
+
existingExecutionWorkspace.id !== persistedExecutionWorkspace.id &&
|
|
2092
|
+
existingExecutionWorkspace.status === "active") {
|
|
2093
|
+
await executionWorkspacesSvc.update(existingExecutionWorkspace.id, {
|
|
2094
|
+
status: "idle",
|
|
2095
|
+
cleanupReason: null,
|
|
2096
|
+
});
|
|
2097
|
+
}
|
|
2098
|
+
if (issueId && persistedExecutionWorkspace) {
|
|
2099
|
+
const nextIssueWorkspaceMode = issueExecutionWorkspaceModeForPersistedWorkspace(persistedExecutionWorkspace.mode);
|
|
2100
|
+
const shouldSwitchIssueToExistingWorkspace = issueRef?.executionWorkspacePreference === "reuse_existing" ||
|
|
2101
|
+
executionWorkspaceMode === "isolated_workspace" ||
|
|
2102
|
+
executionWorkspaceMode === "operator_branch";
|
|
2103
|
+
const nextIssuePatch = {};
|
|
2104
|
+
if (issueRef?.executionWorkspaceId !== persistedExecutionWorkspace.id) {
|
|
2105
|
+
nextIssuePatch.executionWorkspaceId = persistedExecutionWorkspace.id;
|
|
2106
|
+
}
|
|
2107
|
+
if (resolvedProjectWorkspaceId && issueRef?.projectWorkspaceId !== resolvedProjectWorkspaceId) {
|
|
2108
|
+
nextIssuePatch.projectWorkspaceId = resolvedProjectWorkspaceId;
|
|
2109
|
+
}
|
|
2110
|
+
if (shouldSwitchIssueToExistingWorkspace) {
|
|
2111
|
+
nextIssuePatch.executionWorkspacePreference = "reuse_existing";
|
|
2112
|
+
nextIssuePatch.executionWorkspaceSettings = {
|
|
2113
|
+
...(issueExecutionWorkspaceSettings ?? {}),
|
|
2114
|
+
mode: nextIssueWorkspaceMode,
|
|
2115
|
+
};
|
|
2116
|
+
}
|
|
2117
|
+
if (Object.keys(nextIssuePatch).length > 0) {
|
|
2118
|
+
await issuesSvc.update(issueId, nextIssuePatch);
|
|
2119
|
+
}
|
|
2120
|
+
}
|
|
2121
|
+
if (persistedExecutionWorkspace) {
|
|
2122
|
+
context.executionWorkspaceId = persistedExecutionWorkspace.id;
|
|
2123
|
+
await db
|
|
2124
|
+
.update(heartbeatRuns)
|
|
2125
|
+
.set({
|
|
2126
|
+
contextSnapshot: context,
|
|
2127
|
+
updatedAt: new Date(),
|
|
2128
|
+
})
|
|
2129
|
+
.where(eq(heartbeatRuns.id, run.id));
|
|
2130
|
+
}
|
|
2131
|
+
const runtimeSessionResolution = resolveRuntimeSessionParamsForWorkspace({
|
|
2132
|
+
orgId: agent.orgId,
|
|
2133
|
+
agent,
|
|
2134
|
+
previousSessionParams,
|
|
2135
|
+
resolvedWorkspace: {
|
|
2136
|
+
...resolvedWorkspace,
|
|
2137
|
+
cwd: resolveDefaultAgentWorkspaceDir(agent.orgId, agent),
|
|
2138
|
+
source: "agent_home",
|
|
2139
|
+
},
|
|
2140
|
+
});
|
|
2141
|
+
const runtimeSessionParams = runtimeSessionResolution.sessionParams;
|
|
2142
|
+
const runtimeWorkspaceWarnings = [
|
|
2143
|
+
...resolvedWorkspace.warnings,
|
|
2144
|
+
...executionWorkspace.warnings,
|
|
2145
|
+
...(runtimeSessionResolution.warning ? [runtimeSessionResolution.warning] : []),
|
|
2146
|
+
...(resetTaskSession && sessionResetReason
|
|
2147
|
+
? [
|
|
2148
|
+
taskKey
|
|
2149
|
+
? `Skipping saved session resume for task "${taskKey}" because ${sessionResetReason}.`
|
|
2150
|
+
: `Skipping saved session resume because ${sessionResetReason}.`,
|
|
2151
|
+
]
|
|
2152
|
+
: []),
|
|
2153
|
+
];
|
|
2154
|
+
const runtimeSceneContext = await runContextSvc.buildSceneContext({
|
|
2155
|
+
scene: "heartbeat",
|
|
2156
|
+
agent,
|
|
2157
|
+
resolvedWorkspace,
|
|
2158
|
+
runtimeConfig,
|
|
2159
|
+
executionWorkspaceMode,
|
|
2160
|
+
executionWorkspace: {
|
|
2161
|
+
cwd: executionWorkspace.cwd,
|
|
2162
|
+
source: executionWorkspace.source,
|
|
2163
|
+
strategy: executionWorkspace.strategy,
|
|
2164
|
+
projectId: executionWorkspace.projectId,
|
|
2165
|
+
workspaceId: executionWorkspace.workspaceId,
|
|
2166
|
+
repoUrl: executionWorkspace.repoUrl,
|
|
2167
|
+
repoRef: executionWorkspace.repoRef,
|
|
2168
|
+
branchName: executionWorkspace.branchName,
|
|
2169
|
+
worktreePath: executionWorkspace.worktreePath,
|
|
2170
|
+
},
|
|
2171
|
+
});
|
|
2172
|
+
context.rudderScene = runtimeSceneContext.rudderScene;
|
|
2173
|
+
context.rudderWorkspace = runtimeSceneContext.rudderWorkspace;
|
|
2174
|
+
context.rudderWorkspaces = runtimeSceneContext.rudderWorkspaces;
|
|
2175
|
+
if (runtimeSceneContext.rudderRuntimeServiceIntents) {
|
|
2176
|
+
context.rudderRuntimeServiceIntents = runtimeSceneContext.rudderRuntimeServiceIntents;
|
|
2177
|
+
}
|
|
2178
|
+
else {
|
|
2179
|
+
delete context.rudderRuntimeServiceIntents;
|
|
2180
|
+
}
|
|
2181
|
+
if (executionWorkspace.projectId && !readNonEmptyString(context.projectId)) {
|
|
2182
|
+
context.projectId = executionWorkspace.projectId;
|
|
2183
|
+
}
|
|
2184
|
+
const runtimeSessionFallback = taskKey || resetTaskSession ? null : runtime.sessionId;
|
|
2185
|
+
let previousSessionDisplayId = truncateDisplayId(explicitResumeSessionDisplayId ??
|
|
2186
|
+
taskSessionForRun?.sessionDisplayId ??
|
|
2187
|
+
(sessionCodec.getDisplayId ? sessionCodec.getDisplayId(runtimeSessionParams) : null) ??
|
|
2188
|
+
readNonEmptyString(runtimeSessionParams?.sessionId) ??
|
|
2189
|
+
runtimeSessionFallback);
|
|
2190
|
+
let runtimeSessionIdForAdapter = readNonEmptyString(runtimeSessionParams?.sessionId) ?? runtimeSessionFallback;
|
|
2191
|
+
let runtimeSessionParamsForAdapter = runtimeSessionParams;
|
|
2192
|
+
const sessionCompaction = await evaluateSessionCompaction({
|
|
2193
|
+
agent,
|
|
2194
|
+
sessionId: previousSessionDisplayId ?? runtimeSessionIdForAdapter,
|
|
2195
|
+
issueId,
|
|
2196
|
+
});
|
|
2197
|
+
if (sessionCompaction.rotate) {
|
|
2198
|
+
context.rudderSessionHandoffMarkdown = sessionCompaction.handoffMarkdown;
|
|
2199
|
+
context.rudderSessionRotationReason = sessionCompaction.reason;
|
|
2200
|
+
context.rudderPreviousSessionId = previousSessionDisplayId ?? runtimeSessionIdForAdapter;
|
|
2201
|
+
runtimeSessionIdForAdapter = null;
|
|
2202
|
+
runtimeSessionParamsForAdapter = null;
|
|
2203
|
+
previousSessionDisplayId = null;
|
|
2204
|
+
if (sessionCompaction.reason) {
|
|
2205
|
+
runtimeWorkspaceWarnings.push(`Starting a fresh session because ${sessionCompaction.reason}.`);
|
|
2206
|
+
}
|
|
2207
|
+
}
|
|
2208
|
+
else {
|
|
2209
|
+
delete context.rudderSessionHandoffMarkdown;
|
|
2210
|
+
delete context.rudderSessionRotationReason;
|
|
2211
|
+
delete context.rudderPreviousSessionId;
|
|
2212
|
+
}
|
|
2213
|
+
const runtimeForAdapter = {
|
|
2214
|
+
sessionId: runtimeSessionIdForAdapter,
|
|
2215
|
+
sessionParams: runtimeSessionParamsForAdapter,
|
|
2216
|
+
sessionDisplayId: previousSessionDisplayId,
|
|
2217
|
+
taskKey,
|
|
2218
|
+
};
|
|
2219
|
+
let seq = 1;
|
|
2220
|
+
let handle = null;
|
|
2221
|
+
let stdoutExcerpt = "";
|
|
2222
|
+
let stderrExcerpt = "";
|
|
2223
|
+
try {
|
|
2224
|
+
const startedAt = run.startedAt ?? new Date();
|
|
2225
|
+
const runningWithSession = await db
|
|
2226
|
+
.update(heartbeatRuns)
|
|
2227
|
+
.set({
|
|
2228
|
+
startedAt,
|
|
2229
|
+
sessionIdBefore: runtimeForAdapter.sessionDisplayId ?? runtimeForAdapter.sessionId,
|
|
2230
|
+
contextSnapshot: context,
|
|
2231
|
+
updatedAt: new Date(),
|
|
2232
|
+
})
|
|
2233
|
+
.where(eq(heartbeatRuns.id, run.id))
|
|
2234
|
+
.returning()
|
|
2235
|
+
.then((rows) => rows[0] ?? null);
|
|
2236
|
+
if (runningWithSession)
|
|
2237
|
+
run = runningWithSession;
|
|
2238
|
+
const runningAgent = await db
|
|
2239
|
+
.update(agents)
|
|
2240
|
+
.set({ status: "running", updatedAt: new Date() })
|
|
2241
|
+
.where(eq(agents.id, agent.id))
|
|
2242
|
+
.returning()
|
|
2243
|
+
.then((rows) => rows[0] ?? null);
|
|
2244
|
+
if (runningAgent) {
|
|
2245
|
+
publishLiveEvent({
|
|
2246
|
+
orgId: runningAgent.orgId,
|
|
2247
|
+
type: "agent.status",
|
|
2248
|
+
payload: {
|
|
2249
|
+
agentId: runningAgent.id,
|
|
2250
|
+
status: runningAgent.status,
|
|
2251
|
+
outcome: "running",
|
|
2252
|
+
},
|
|
2253
|
+
});
|
|
2254
|
+
}
|
|
2255
|
+
const currentRun = run;
|
|
2256
|
+
await appendRunEvent(currentRun, seq++, {
|
|
2257
|
+
eventType: "lifecycle",
|
|
2258
|
+
stream: "system",
|
|
2259
|
+
level: "info",
|
|
2260
|
+
message: "run started",
|
|
2261
|
+
});
|
|
2262
|
+
handle = await runLogStore.begin({
|
|
2263
|
+
orgId: run.orgId,
|
|
2264
|
+
agentId: run.agentId,
|
|
2265
|
+
runId,
|
|
2266
|
+
});
|
|
2267
|
+
await db
|
|
2268
|
+
.update(heartbeatRuns)
|
|
2269
|
+
.set({
|
|
2270
|
+
logStore: handle.store,
|
|
2271
|
+
logRef: handle.logRef,
|
|
2272
|
+
updatedAt: new Date(),
|
|
2273
|
+
})
|
|
2274
|
+
.where(eq(heartbeatRuns.id, runId));
|
|
2275
|
+
const adapter = getServerAdapter(agent.agentRuntimeType);
|
|
2276
|
+
stdoutTranscriptParser = adapter.parseStdoutLine ?? null;
|
|
2277
|
+
const currentUserRedactionOptions = await getCurrentUserRedactionOptions();
|
|
2278
|
+
const onLog = async (stream, chunk) => {
|
|
2279
|
+
const sanitizedChunk = redactCurrentUserText(chunk, currentUserRedactionOptions);
|
|
2280
|
+
if (stream === "stdout")
|
|
2281
|
+
stdoutExcerpt = appendExcerpt(stdoutExcerpt, sanitizedChunk);
|
|
2282
|
+
if (stream === "stderr")
|
|
2283
|
+
stderrExcerpt = appendExcerpt(stderrExcerpt, sanitizedChunk);
|
|
2284
|
+
const ts = new Date().toISOString();
|
|
2285
|
+
if (handle) {
|
|
2286
|
+
await runLogStore.append(handle, {
|
|
2287
|
+
stream,
|
|
2288
|
+
chunk: sanitizedChunk,
|
|
2289
|
+
ts,
|
|
2290
|
+
});
|
|
2291
|
+
}
|
|
2292
|
+
const payloadChunk = sanitizedChunk.length > MAX_LIVE_LOG_CHUNK_BYTES
|
|
2293
|
+
? sanitizedChunk.slice(sanitizedChunk.length - MAX_LIVE_LOG_CHUNK_BYTES)
|
|
2294
|
+
: sanitizedChunk;
|
|
2295
|
+
publishLiveEvent({
|
|
2296
|
+
orgId: run.orgId,
|
|
2297
|
+
type: "heartbeat.run.log",
|
|
2298
|
+
payload: {
|
|
2299
|
+
runId: run.id,
|
|
2300
|
+
agentId: run.agentId,
|
|
2301
|
+
ts,
|
|
2302
|
+
stream,
|
|
2303
|
+
chunk: payloadChunk,
|
|
2304
|
+
truncated: payloadChunk.length !== sanitizedChunk.length,
|
|
2305
|
+
},
|
|
2306
|
+
});
|
|
2307
|
+
if (stream === "stdout") {
|
|
2308
|
+
stdoutTranscriptBuffer = appendTranscriptEntriesFromChunk({
|
|
2309
|
+
buffer: stdoutTranscriptBuffer,
|
|
2310
|
+
chunk: sanitizedChunk,
|
|
2311
|
+
transcript: executionTranscript,
|
|
2312
|
+
parser: stdoutTranscriptParser,
|
|
2313
|
+
kind: "stdout",
|
|
2314
|
+
});
|
|
2315
|
+
return;
|
|
2316
|
+
}
|
|
2317
|
+
stderrTranscriptBuffer = appendTranscriptEntriesFromChunk({
|
|
2318
|
+
buffer: stderrTranscriptBuffer,
|
|
2319
|
+
chunk: sanitizedChunk,
|
|
2320
|
+
transcript: executionTranscript,
|
|
2321
|
+
kind: "stderr",
|
|
2322
|
+
});
|
|
2323
|
+
};
|
|
2324
|
+
for (const warning of runtimeWorkspaceWarnings) {
|
|
2325
|
+
const logEntry = formatRuntimeWorkspaceWarningLog(warning);
|
|
2326
|
+
await onLog(logEntry.stream, logEntry.chunk);
|
|
2327
|
+
}
|
|
2328
|
+
const adapterEnv = Object.fromEntries(Object.entries(parseObject(resolvedConfig.env)).filter((entry) => typeof entry[0] === "string" && typeof entry[1] === "string"));
|
|
2329
|
+
const runtimeServices = await ensureRuntimeServicesForRun({
|
|
2330
|
+
db,
|
|
2331
|
+
runId: run.id,
|
|
2332
|
+
agent: {
|
|
2333
|
+
id: agent.id,
|
|
2334
|
+
name: agent.name,
|
|
2335
|
+
orgId: agent.orgId,
|
|
2336
|
+
},
|
|
2337
|
+
issue: issueRef,
|
|
2338
|
+
workspace: executionWorkspace,
|
|
2339
|
+
executionWorkspaceId: persistedExecutionWorkspace?.id ?? issueRef?.executionWorkspaceId ?? null,
|
|
2340
|
+
config: resolvedConfig,
|
|
2341
|
+
adapterEnv,
|
|
2342
|
+
onLog,
|
|
2343
|
+
});
|
|
2344
|
+
if (runtimeServices.length > 0) {
|
|
2345
|
+
context.rudderRuntimeServices = runtimeServices;
|
|
2346
|
+
context.rudderRuntimePrimaryUrl =
|
|
2347
|
+
runtimeServices.find((service) => readNonEmptyString(service.url))?.url ?? null;
|
|
2348
|
+
await db
|
|
2349
|
+
.update(heartbeatRuns)
|
|
2350
|
+
.set({
|
|
2351
|
+
contextSnapshot: context,
|
|
2352
|
+
updatedAt: new Date(),
|
|
2353
|
+
})
|
|
2354
|
+
.where(eq(heartbeatRuns.id, run.id));
|
|
2355
|
+
}
|
|
2356
|
+
if (issueId && (executionWorkspace.created || runtimeServices.some((service) => !service.reused))) {
|
|
2357
|
+
try {
|
|
2358
|
+
await issuesSvc.addComment(issueId, buildWorkspaceReadyComment({
|
|
2359
|
+
workspace: executionWorkspace,
|
|
2360
|
+
runtimeServices,
|
|
2361
|
+
}), { agentId: agent.id });
|
|
2362
|
+
}
|
|
2363
|
+
catch (err) {
|
|
2364
|
+
await onLog("stderr", `[rudder] Failed to post workspace-ready comment: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
2365
|
+
}
|
|
2366
|
+
}
|
|
2367
|
+
const onAdapterMeta = async (meta) => {
|
|
2368
|
+
if (meta.env && secretKeys.size > 0) {
|
|
2369
|
+
for (const key of secretKeys) {
|
|
2370
|
+
if (key in meta.env)
|
|
2371
|
+
meta.env[key] = "***REDACTED***";
|
|
2372
|
+
}
|
|
2373
|
+
}
|
|
2374
|
+
heartbeatObservationContext.metadata = {
|
|
2375
|
+
...(heartbeatObservationContext.metadata ?? {}),
|
|
2376
|
+
...buildHeartbeatRuntimeTraceMetadata({
|
|
2377
|
+
runtimeConfig,
|
|
2378
|
+
runtimeSkills: runtimeSkillEntries,
|
|
2379
|
+
adapterMeta: meta,
|
|
2380
|
+
}),
|
|
2381
|
+
};
|
|
2382
|
+
updateExecutionObservation(observation, heartbeatObservationContext, {
|
|
2383
|
+
input: rootObservationInput,
|
|
2384
|
+
});
|
|
2385
|
+
await appendRunEvent(currentRun, seq++, {
|
|
2386
|
+
eventType: "adapter.invoke",
|
|
2387
|
+
stream: "system",
|
|
2388
|
+
level: "info",
|
|
2389
|
+
message: "adapter invocation",
|
|
2390
|
+
payload: buildHeartbeatAdapterInvokePayload({
|
|
2391
|
+
meta,
|
|
2392
|
+
runtimeSkills: runtimeSkillEntries,
|
|
2393
|
+
}),
|
|
2394
|
+
});
|
|
2395
|
+
};
|
|
2396
|
+
const authToken = adapter.supportsLocalAgentJwt
|
|
2397
|
+
? createLocalAgentJwt(agent.id, agent.orgId, agent.agentRuntimeType, run.id)
|
|
2398
|
+
: null;
|
|
2399
|
+
if (adapter.supportsLocalAgentJwt && !authToken) {
|
|
2400
|
+
logger.warn({
|
|
2401
|
+
orgId: agent.orgId,
|
|
2402
|
+
agentId: agent.id,
|
|
2403
|
+
runId: run.id,
|
|
2404
|
+
agentRuntimeType: agent.agentRuntimeType,
|
|
2405
|
+
}, "local agent jwt secret missing or invalid; running without injected RUDDER_API_KEY");
|
|
2406
|
+
}
|
|
2407
|
+
const adapterResult = await executeAdapterWithModelFallbacks(adapter, {
|
|
2408
|
+
runId: run.id,
|
|
2409
|
+
agent,
|
|
2410
|
+
runtime: runtimeForAdapter,
|
|
2411
|
+
config: runtimeConfig,
|
|
2412
|
+
context,
|
|
2413
|
+
onLog,
|
|
2414
|
+
onMeta: onAdapterMeta,
|
|
2415
|
+
onSpawn: async (meta) => {
|
|
2416
|
+
await persistRunProcessMetadata(run.id, meta);
|
|
2417
|
+
},
|
|
2418
|
+
authToken: authToken ?? undefined,
|
|
2419
|
+
}, {
|
|
2420
|
+
resolveAdapter: findServerAdapter,
|
|
2421
|
+
createAuthToken: (agentRuntimeType) => createLocalAgentJwt(agent.id, agent.orgId, agentRuntimeType, run.id) ?? undefined,
|
|
2422
|
+
onAttemptStart: (_attempt, attemptAdapter) => {
|
|
2423
|
+
stdoutTranscriptParser = attemptAdapter.parseStdoutLine ?? null;
|
|
2424
|
+
},
|
|
2425
|
+
});
|
|
2426
|
+
const adapterManagedRuntimeServices = adapterResult.runtimeServices
|
|
2427
|
+
? await persistAdapterManagedRuntimeServices({
|
|
2428
|
+
db,
|
|
2429
|
+
agentRuntimeType: agent.agentRuntimeType,
|
|
2430
|
+
runId: run.id,
|
|
2431
|
+
agent: {
|
|
2432
|
+
id: agent.id,
|
|
2433
|
+
name: agent.name,
|
|
2434
|
+
orgId: agent.orgId,
|
|
2435
|
+
},
|
|
2436
|
+
issue: issueRef,
|
|
2437
|
+
workspace: executionWorkspace,
|
|
2438
|
+
reports: adapterResult.runtimeServices,
|
|
2439
|
+
})
|
|
2440
|
+
: [];
|
|
2441
|
+
if (adapterManagedRuntimeServices.length > 0) {
|
|
2442
|
+
const combinedRuntimeServices = [
|
|
2443
|
+
...runtimeServices,
|
|
2444
|
+
...adapterManagedRuntimeServices,
|
|
2445
|
+
];
|
|
2446
|
+
context.rudderRuntimeServices = combinedRuntimeServices;
|
|
2447
|
+
context.rudderRuntimePrimaryUrl =
|
|
2448
|
+
combinedRuntimeServices.find((service) => readNonEmptyString(service.url))?.url ?? null;
|
|
2449
|
+
await db
|
|
2450
|
+
.update(heartbeatRuns)
|
|
2451
|
+
.set({
|
|
2452
|
+
contextSnapshot: context,
|
|
2453
|
+
updatedAt: new Date(),
|
|
2454
|
+
})
|
|
2455
|
+
.where(eq(heartbeatRuns.id, run.id));
|
|
2456
|
+
if (issueId) {
|
|
2457
|
+
try {
|
|
2458
|
+
await issuesSvc.addComment(issueId, buildWorkspaceReadyComment({
|
|
2459
|
+
workspace: executionWorkspace,
|
|
2460
|
+
runtimeServices: adapterManagedRuntimeServices,
|
|
2461
|
+
}), { agentId: agent.id });
|
|
2462
|
+
}
|
|
2463
|
+
catch (err) {
|
|
2464
|
+
await onLog("stderr", `[rudder] Failed to post adapter-managed runtime comment: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
2465
|
+
}
|
|
2466
|
+
}
|
|
2467
|
+
}
|
|
2468
|
+
const nextSessionState = resolveNextSessionState({
|
|
2469
|
+
codec: sessionCodec,
|
|
2470
|
+
adapterResult,
|
|
2471
|
+
previousParams: previousSessionParams,
|
|
2472
|
+
previousDisplayId: runtimeForAdapter.sessionDisplayId,
|
|
2473
|
+
previousLegacySessionId: runtimeForAdapter.sessionId,
|
|
2474
|
+
});
|
|
2475
|
+
const rawUsage = normalizeUsageTotals(adapterResult.usage);
|
|
2476
|
+
const sessionUsageResolution = await resolveNormalizedUsageForSession({
|
|
2477
|
+
agentId: agent.id,
|
|
2478
|
+
runId: run.id,
|
|
2479
|
+
sessionId: nextSessionState.displayId ?? nextSessionState.legacySessionId,
|
|
2480
|
+
rawUsage,
|
|
2481
|
+
});
|
|
2482
|
+
const normalizedUsage = sessionUsageResolution.normalizedUsage;
|
|
2483
|
+
let outcome;
|
|
2484
|
+
const latestRun = await getRun(run.id);
|
|
2485
|
+
if (latestRun?.status === "cancelled") {
|
|
2486
|
+
outcome = "cancelled";
|
|
2487
|
+
}
|
|
2488
|
+
else if (adapterResult.timedOut) {
|
|
2489
|
+
outcome = "timed_out";
|
|
2490
|
+
}
|
|
2491
|
+
else if ((adapterResult.exitCode ?? 0) === 0 && !adapterResult.errorMessage) {
|
|
2492
|
+
outcome = "succeeded";
|
|
2493
|
+
}
|
|
2494
|
+
else {
|
|
2495
|
+
outcome = "failed";
|
|
2496
|
+
}
|
|
2497
|
+
let logSummary = null;
|
|
2498
|
+
if (handle) {
|
|
2499
|
+
logSummary = await runLogStore.finalize(handle);
|
|
2500
|
+
}
|
|
2501
|
+
const status = outcome === "succeeded"
|
|
2502
|
+
? "succeeded"
|
|
2503
|
+
: outcome === "cancelled"
|
|
2504
|
+
? "cancelled"
|
|
2505
|
+
: outcome === "timed_out"
|
|
2506
|
+
? "timed_out"
|
|
2507
|
+
: "failed";
|
|
2508
|
+
heartbeatObservationContext.status = status;
|
|
2509
|
+
finalObservationStatus = status;
|
|
2510
|
+
finalObservationSessionId = nextSessionState.displayId ?? nextSessionState.legacySessionId ?? finalObservationSessionId;
|
|
2511
|
+
const adapterResultSummary = summarizeHeartbeatRunResultJson(adapterResult.resultJson);
|
|
2512
|
+
transcriptFallbackResult = {
|
|
2513
|
+
ts: new Date().toISOString(),
|
|
2514
|
+
model: readNonEmptyString(adapterResult.model),
|
|
2515
|
+
output: readNonEmptyString(adapterResult.summary)
|
|
2516
|
+
?? readNonEmptyString(adapterResultSummary?.result)
|
|
2517
|
+
?? readNonEmptyString(adapterResultSummary?.summary)
|
|
2518
|
+
?? readNonEmptyString(adapterResultSummary?.message)
|
|
2519
|
+
?? null,
|
|
2520
|
+
usage: adapterResult.usage ?? null,
|
|
2521
|
+
costUsd: typeof adapterResult.costUsd === "number" ? adapterResult.costUsd : null,
|
|
2522
|
+
subtype: status,
|
|
2523
|
+
isError: outcome !== "succeeded",
|
|
2524
|
+
errors: adapterResult.errorMessage ? [adapterResult.errorMessage] : [],
|
|
2525
|
+
};
|
|
2526
|
+
const usageJson = normalizedUsage || adapterResult.costUsd != null
|
|
2527
|
+
? {
|
|
2528
|
+
...(normalizedUsage ?? {}),
|
|
2529
|
+
...(rawUsage ? {
|
|
2530
|
+
rawInputTokens: rawUsage.inputTokens,
|
|
2531
|
+
rawCachedInputTokens: rawUsage.cachedInputTokens,
|
|
2532
|
+
rawOutputTokens: rawUsage.outputTokens,
|
|
2533
|
+
} : {}),
|
|
2534
|
+
...(sessionUsageResolution.derivedFromSessionTotals ? { usageSource: "session_delta" } : {}),
|
|
2535
|
+
...((nextSessionState.displayId ?? nextSessionState.legacySessionId)
|
|
2536
|
+
? { persistedSessionId: nextSessionState.displayId ?? nextSessionState.legacySessionId }
|
|
2537
|
+
: {}),
|
|
2538
|
+
sessionReused: runtimeForAdapter.sessionId != null || runtimeForAdapter.sessionDisplayId != null,
|
|
2539
|
+
taskSessionReused: taskSessionForRun != null,
|
|
2540
|
+
freshSession: runtimeForAdapter.sessionId == null && runtimeForAdapter.sessionDisplayId == null,
|
|
2541
|
+
sessionRotated: sessionCompaction.rotate,
|
|
2542
|
+
sessionRotationReason: sessionCompaction.reason,
|
|
2543
|
+
provider: readNonEmptyString(adapterResult.provider) ?? "unknown",
|
|
2544
|
+
biller: resolveLedgerBiller(adapterResult),
|
|
2545
|
+
model: readNonEmptyString(adapterResult.model) ?? "unknown",
|
|
2546
|
+
...(adapterResult.costUsd != null ? { costUsd: adapterResult.costUsd } : {}),
|
|
2547
|
+
billingType: normalizeLedgerBillingType(adapterResult.billingType),
|
|
2548
|
+
}
|
|
2549
|
+
: null;
|
|
2550
|
+
await setRunStatus(run.id, status, {
|
|
2551
|
+
finishedAt: new Date(),
|
|
2552
|
+
error: outcome === "succeeded"
|
|
2553
|
+
? null
|
|
2554
|
+
: redactCurrentUserText(adapterResult.errorMessage ?? (outcome === "timed_out" ? "Timed out" : "Adapter failed"), currentUserRedactionOptions),
|
|
2555
|
+
errorCode: outcome === "timed_out"
|
|
2556
|
+
? "timeout"
|
|
2557
|
+
: outcome === "cancelled"
|
|
2558
|
+
? "cancelled"
|
|
2559
|
+
: outcome === "failed"
|
|
2560
|
+
? (adapterResult.errorCode ?? "adapter_failed")
|
|
2561
|
+
: null,
|
|
2562
|
+
exitCode: adapterResult.exitCode,
|
|
2563
|
+
signal: adapterResult.signal,
|
|
2564
|
+
usageJson,
|
|
2565
|
+
resultJson: adapterResult.resultJson ?? null,
|
|
2566
|
+
sessionIdAfter: nextSessionState.displayId ?? nextSessionState.legacySessionId,
|
|
2567
|
+
stdoutExcerpt,
|
|
2568
|
+
stderrExcerpt,
|
|
2569
|
+
logBytes: logSummary?.bytes,
|
|
2570
|
+
logSha256: logSummary?.sha256,
|
|
2571
|
+
logCompressed: logSummary?.compressed ?? false,
|
|
2572
|
+
});
|
|
2573
|
+
await setWakeupStatus(run.wakeupRequestId, outcome === "succeeded" ? "completed" : status, {
|
|
2574
|
+
finishedAt: new Date(),
|
|
2575
|
+
error: adapterResult.errorMessage ?? null,
|
|
2576
|
+
});
|
|
2577
|
+
const finalizedRun = await getRun(run.id);
|
|
2578
|
+
if (finalizedRun) {
|
|
2579
|
+
await appendRunEvent(finalizedRun, seq++, {
|
|
2580
|
+
eventType: "lifecycle",
|
|
2581
|
+
stream: "system",
|
|
2582
|
+
level: outcome === "succeeded" ? "info" : "error",
|
|
2583
|
+
message: `run ${outcome}`,
|
|
2584
|
+
payload: {
|
|
2585
|
+
status,
|
|
2586
|
+
exitCode: adapterResult.exitCode,
|
|
2587
|
+
},
|
|
2588
|
+
});
|
|
2589
|
+
await releaseIssueExecutionAndPromote(finalizedRun);
|
|
2590
|
+
}
|
|
2591
|
+
if (finalizedRun) {
|
|
2592
|
+
await updateRuntimeState(agent, finalizedRun, adapterResult, {
|
|
2593
|
+
legacySessionId: nextSessionState.legacySessionId,
|
|
2594
|
+
}, normalizedUsage);
|
|
2595
|
+
if (taskKey) {
|
|
2596
|
+
if (adapterResult.clearSession || (!nextSessionState.params && !nextSessionState.displayId)) {
|
|
2597
|
+
await clearTaskSessions(agent.orgId, agent.id, {
|
|
2598
|
+
taskKey,
|
|
2599
|
+
agentRuntimeType: agent.agentRuntimeType,
|
|
2600
|
+
});
|
|
2601
|
+
}
|
|
2602
|
+
else {
|
|
2603
|
+
await upsertTaskSession({
|
|
2604
|
+
orgId: agent.orgId,
|
|
2605
|
+
agentId: agent.id,
|
|
2606
|
+
agentRuntimeType: agent.agentRuntimeType,
|
|
2607
|
+
taskKey,
|
|
2608
|
+
sessionParamsJson: nextSessionState.params,
|
|
2609
|
+
sessionDisplayId: nextSessionState.displayId,
|
|
2610
|
+
lastRunId: finalizedRun.id,
|
|
2611
|
+
lastError: outcome === "succeeded" ? null : (adapterResult.errorMessage ?? "run_failed"),
|
|
2612
|
+
});
|
|
2613
|
+
}
|
|
2614
|
+
}
|
|
2615
|
+
await emitHeartbeatLiveEval(finalizedRun.id);
|
|
2616
|
+
}
|
|
2617
|
+
await finalizeAgentStatus(agent.id, outcome);
|
|
2618
|
+
}
|
|
2619
|
+
catch (err) {
|
|
2620
|
+
const message = redactCurrentUserText(err instanceof Error ? err.message : "Unknown adapter failure", await getCurrentUserRedactionOptions());
|
|
2621
|
+
heartbeatObservationContext.status = "failed";
|
|
2622
|
+
finalObservationStatus = "failed";
|
|
2623
|
+
transcriptFallbackResult = {
|
|
2624
|
+
ts: new Date().toISOString(),
|
|
2625
|
+
output: message,
|
|
2626
|
+
subtype: "failed",
|
|
2627
|
+
isError: true,
|
|
2628
|
+
errors: [message],
|
|
2629
|
+
};
|
|
2630
|
+
logger.error({ err, runId }, "heartbeat execution failed");
|
|
2631
|
+
let logSummary = null;
|
|
2632
|
+
if (handle) {
|
|
2633
|
+
try {
|
|
2634
|
+
logSummary = await runLogStore.finalize(handle);
|
|
2635
|
+
}
|
|
2636
|
+
catch (finalizeErr) {
|
|
2637
|
+
logger.warn({ err: finalizeErr, runId }, "failed to finalize run log after error");
|
|
2638
|
+
}
|
|
2639
|
+
}
|
|
2640
|
+
const failedRun = await setRunStatus(run.id, "failed", {
|
|
2641
|
+
error: message,
|
|
2642
|
+
errorCode: "adapter_failed",
|
|
2643
|
+
finishedAt: new Date(),
|
|
2644
|
+
stdoutExcerpt,
|
|
2645
|
+
stderrExcerpt,
|
|
2646
|
+
logBytes: logSummary?.bytes,
|
|
2647
|
+
logSha256: logSummary?.sha256,
|
|
2648
|
+
logCompressed: logSummary?.compressed ?? false,
|
|
2649
|
+
});
|
|
2650
|
+
await setWakeupStatus(run.wakeupRequestId, "failed", {
|
|
2651
|
+
finishedAt: new Date(),
|
|
2652
|
+
error: message,
|
|
2653
|
+
});
|
|
2654
|
+
if (failedRun) {
|
|
2655
|
+
await appendRunEvent(failedRun, seq++, {
|
|
2656
|
+
eventType: "error",
|
|
2657
|
+
stream: "system",
|
|
2658
|
+
level: "error",
|
|
2659
|
+
message,
|
|
2660
|
+
});
|
|
2661
|
+
await releaseIssueExecutionAndPromote(failedRun);
|
|
2662
|
+
await updateRuntimeState(agent, failedRun, {
|
|
2663
|
+
exitCode: null,
|
|
2664
|
+
signal: null,
|
|
2665
|
+
timedOut: false,
|
|
2666
|
+
errorMessage: message,
|
|
2667
|
+
}, {
|
|
2668
|
+
legacySessionId: runtimeForAdapter.sessionId,
|
|
2669
|
+
});
|
|
2670
|
+
if (taskKey && (previousSessionParams || previousSessionDisplayId || taskSession)) {
|
|
2671
|
+
await upsertTaskSession({
|
|
2672
|
+
orgId: agent.orgId,
|
|
2673
|
+
agentId: agent.id,
|
|
2674
|
+
agentRuntimeType: agent.agentRuntimeType,
|
|
2675
|
+
taskKey,
|
|
2676
|
+
sessionParamsJson: previousSessionParams,
|
|
2677
|
+
sessionDisplayId: previousSessionDisplayId,
|
|
2678
|
+
lastRunId: failedRun.id,
|
|
2679
|
+
lastError: message,
|
|
2680
|
+
});
|
|
2681
|
+
}
|
|
2682
|
+
await emitHeartbeatLiveEval(failedRun.id);
|
|
2683
|
+
}
|
|
2684
|
+
await finalizeAgentStatus(agent.id, "failed");
|
|
2685
|
+
}
|
|
2686
|
+
finally {
|
|
2687
|
+
stdoutTranscriptBuffer = appendTranscriptEntriesFromChunk({
|
|
2688
|
+
buffer: stdoutTranscriptBuffer,
|
|
2689
|
+
chunk: "",
|
|
2690
|
+
transcript: executionTranscript,
|
|
2691
|
+
parser: stdoutTranscriptParser,
|
|
2692
|
+
finalize: true,
|
|
2693
|
+
kind: "stdout",
|
|
2694
|
+
});
|
|
2695
|
+
stderrTranscriptBuffer = appendTranscriptEntriesFromChunk({
|
|
2696
|
+
buffer: stderrTranscriptBuffer,
|
|
2697
|
+
chunk: "",
|
|
2698
|
+
transcript: executionTranscript,
|
|
2699
|
+
finalize: true,
|
|
2700
|
+
kind: "stderr",
|
|
2701
|
+
});
|
|
2702
|
+
try {
|
|
2703
|
+
const transcriptStats = emitExecutionTranscriptTree({
|
|
2704
|
+
context: heartbeatObservationContext,
|
|
2705
|
+
parentObservation: observation,
|
|
2706
|
+
transcript: executionTranscript,
|
|
2707
|
+
fallbackResult: transcriptFallbackResult,
|
|
2708
|
+
});
|
|
2709
|
+
finalObservationOutput = transcriptStats.finalOutput ?? transcriptFallbackResult?.output ?? null;
|
|
2710
|
+
finalObservationSessionId = transcriptStats.finalSessionId ?? finalObservationSessionId;
|
|
2711
|
+
}
|
|
2712
|
+
catch (error) {
|
|
2713
|
+
logger.warn({
|
|
2714
|
+
runId: run.id,
|
|
2715
|
+
err: error instanceof Error ? error.message : String(error),
|
|
2716
|
+
}, "Failed to export heartbeat transcript tree to Langfuse");
|
|
2717
|
+
}
|
|
2718
|
+
updateExecutionObservation(observation, heartbeatObservationContext, {
|
|
2719
|
+
input: rootObservationInput,
|
|
2720
|
+
output: finalObservationOutput,
|
|
2721
|
+
level: finalObservationStatus === "failed" || finalObservationStatus === "timed_out" ? "ERROR" : "DEFAULT",
|
|
2722
|
+
statusMessage: finalObservationStatus ?? undefined,
|
|
2723
|
+
});
|
|
2724
|
+
updateExecutionTraceIO(observation, {
|
|
2725
|
+
input: rootObservationInput,
|
|
2726
|
+
output: finalObservationOutput,
|
|
2727
|
+
});
|
|
2728
|
+
updateExecutionTraceSession(observation, finalObservationSessionId);
|
|
2729
|
+
}
|
|
2730
|
+
});
|
|
2731
|
+
}
|
|
2732
|
+
catch (outerErr) {
|
|
2733
|
+
// Setup code before adapter.execute threw (e.g. ensureRuntimeState, resolveWorkspaceForRun).
|
|
2734
|
+
// The inner catch did not fire, so we must record the failure here.
|
|
2735
|
+
const message = outerErr instanceof Error ? outerErr.message : "Unknown setup failure";
|
|
2736
|
+
logger.error({ err: outerErr, runId }, "heartbeat execution setup failed");
|
|
2737
|
+
await setRunStatus(runId, "failed", {
|
|
2738
|
+
error: message,
|
|
2739
|
+
errorCode: "adapter_failed",
|
|
2740
|
+
finishedAt: new Date(),
|
|
2741
|
+
}).catch(() => undefined);
|
|
2742
|
+
await setWakeupStatus(run.wakeupRequestId, "failed", {
|
|
2743
|
+
finishedAt: new Date(),
|
|
2744
|
+
error: message,
|
|
2745
|
+
}).catch(() => undefined);
|
|
2746
|
+
const failedRun = await getRun(runId).catch(() => null);
|
|
2747
|
+
if (failedRun) {
|
|
2748
|
+
// Emit a run-log event so the failure is visible in the run timeline,
|
|
2749
|
+
// consistent with what the inner catch block does for adapter failures.
|
|
2750
|
+
await appendRunEvent(failedRun, 1, {
|
|
2751
|
+
eventType: "error",
|
|
2752
|
+
stream: "system",
|
|
2753
|
+
level: "error",
|
|
2754
|
+
message,
|
|
2755
|
+
}).catch(() => undefined);
|
|
2756
|
+
await emitHeartbeatLiveEval(failedRun.id).catch(() => undefined);
|
|
2757
|
+
await releaseIssueExecutionAndPromote(failedRun).catch(() => undefined);
|
|
2758
|
+
}
|
|
2759
|
+
// Ensure the agent is not left stuck in "running" if the inner catch handler's
|
|
2760
|
+
// DB calls threw (e.g. a transient DB error in finalizeAgentStatus).
|
|
2761
|
+
await finalizeAgentStatus(run.agentId, "failed").catch(() => undefined);
|
|
2762
|
+
}
|
|
2763
|
+
finally {
|
|
2764
|
+
await releaseRuntimeServicesForRun(run.id).catch(() => undefined);
|
|
2765
|
+
activeRunExecutions.delete(run.id);
|
|
2766
|
+
await startNextQueuedRunForAgent(run.agentId);
|
|
2767
|
+
}
|
|
2768
|
+
}
|
|
2769
|
+
async function releaseIssueExecutionAndPromote(run) {
|
|
2770
|
+
const outcome = await db.transaction(async (tx) => {
|
|
2771
|
+
await tx.execute(sql `select id from issues where org_id = ${run.orgId} and execution_run_id = ${run.id} for update`);
|
|
2772
|
+
const issue = await tx
|
|
2773
|
+
.select({
|
|
2774
|
+
id: issues.id,
|
|
2775
|
+
orgId: issues.orgId,
|
|
2776
|
+
title: issues.title,
|
|
2777
|
+
description: issues.description,
|
|
2778
|
+
status: issues.status,
|
|
2779
|
+
priority: issues.priority,
|
|
2780
|
+
projectId: issues.projectId,
|
|
2781
|
+
assigneeAgentId: issues.assigneeAgentId,
|
|
2782
|
+
})
|
|
2783
|
+
.from(issues)
|
|
2784
|
+
.where(and(eq(issues.orgId, run.orgId), eq(issues.executionRunId, run.id)))
|
|
2785
|
+
.then((rows) => rows[0] ?? null);
|
|
2786
|
+
if (!issue)
|
|
2787
|
+
return { promotedRun: null, passiveClosure: null };
|
|
2788
|
+
const passiveClosure = await evaluatePassiveIssueClosureForLockedIssue({
|
|
2789
|
+
tx,
|
|
2790
|
+
run,
|
|
2791
|
+
issue,
|
|
2792
|
+
now: new Date(),
|
|
2793
|
+
});
|
|
2794
|
+
if (passiveClosure.kind === "queued") {
|
|
2795
|
+
return { promotedRun: passiveClosure.run, passiveClosure };
|
|
2796
|
+
}
|
|
2797
|
+
await tx
|
|
2798
|
+
.update(issues)
|
|
2799
|
+
.set({
|
|
2800
|
+
executionRunId: null,
|
|
2801
|
+
executionAgentNameKey: null,
|
|
2802
|
+
executionLockedAt: null,
|
|
2803
|
+
updatedAt: new Date(),
|
|
2804
|
+
})
|
|
2805
|
+
.where(eq(issues.id, issue.id));
|
|
2806
|
+
while (true) {
|
|
2807
|
+
const deferred = await tx
|
|
2808
|
+
.select()
|
|
2809
|
+
.from(agentWakeupRequests)
|
|
2810
|
+
.where(and(eq(agentWakeupRequests.orgId, issue.orgId), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql `${agentWakeupRequests.payload} ->> 'issueId' = ${issue.id}`))
|
|
2811
|
+
.orderBy(asc(agentWakeupRequests.requestedAt))
|
|
2812
|
+
.limit(1)
|
|
2813
|
+
.then((rows) => rows[0] ?? null);
|
|
2814
|
+
if (!deferred)
|
|
2815
|
+
return { promotedRun: null, passiveClosure };
|
|
2816
|
+
const deferredAgent = await tx
|
|
2817
|
+
.select()
|
|
2818
|
+
.from(agents)
|
|
2819
|
+
.where(eq(agents.id, deferred.agentId))
|
|
2820
|
+
.then((rows) => rows[0] ?? null);
|
|
2821
|
+
if (!deferredAgent ||
|
|
2822
|
+
deferredAgent.orgId !== issue.orgId ||
|
|
2823
|
+
deferredAgent.status === "paused" ||
|
|
2824
|
+
deferredAgent.status === "terminated" ||
|
|
2825
|
+
deferredAgent.status === "pending_approval") {
|
|
2826
|
+
await tx
|
|
2827
|
+
.update(agentWakeupRequests)
|
|
2828
|
+
.set({
|
|
2829
|
+
status: "failed",
|
|
2830
|
+
finishedAt: new Date(),
|
|
2831
|
+
error: "Deferred wake could not be promoted: agent is not invokable",
|
|
2832
|
+
updatedAt: new Date(),
|
|
2833
|
+
})
|
|
2834
|
+
.where(eq(agentWakeupRequests.id, deferred.id));
|
|
2835
|
+
continue;
|
|
2836
|
+
}
|
|
2837
|
+
const deferredPayload = parseObject(deferred.payload);
|
|
2838
|
+
const deferredContextSeed = parseObject(deferredPayload[DEFERRED_WAKE_CONTEXT_KEY]);
|
|
2839
|
+
const promotedContextSeed = { ...deferredContextSeed };
|
|
2840
|
+
const promotedReason = readNonEmptyString(deferred.reason) ?? "issue_execution_promoted";
|
|
2841
|
+
const promotedSource = readNonEmptyString(deferred.source) ?? "automation";
|
|
2842
|
+
const promotedTriggerDetail = readNonEmptyString(deferred.triggerDetail) ?? null;
|
|
2843
|
+
const promotedPayload = deferredPayload;
|
|
2844
|
+
delete promotedPayload[DEFERRED_WAKE_CONTEXT_KEY];
|
|
2845
|
+
const { contextSnapshot: promotedContextSnapshot, taskKey: promotedTaskKey, } = enrichWakeContextSnapshot({
|
|
2846
|
+
contextSnapshot: promotedContextSeed,
|
|
2847
|
+
reason: promotedReason,
|
|
2848
|
+
source: promotedSource,
|
|
2849
|
+
triggerDetail: promotedTriggerDetail,
|
|
2850
|
+
payload: promotedPayload,
|
|
2851
|
+
});
|
|
2852
|
+
const sessionBefore = readNonEmptyString(promotedContextSnapshot.resumeSessionDisplayId) ??
|
|
2853
|
+
await resolveSessionBeforeForWakeup(deferredAgent, promotedTaskKey);
|
|
2854
|
+
const now = new Date();
|
|
2855
|
+
const newRun = await tx
|
|
2856
|
+
.insert(heartbeatRuns)
|
|
2857
|
+
.values({
|
|
2858
|
+
orgId: deferredAgent.orgId,
|
|
2859
|
+
agentId: deferredAgent.id,
|
|
2860
|
+
invocationSource: promotedSource,
|
|
2861
|
+
triggerDetail: promotedTriggerDetail,
|
|
2862
|
+
status: "queued",
|
|
2863
|
+
wakeupRequestId: deferred.id,
|
|
2864
|
+
contextSnapshot: promotedContextSnapshot,
|
|
2865
|
+
sessionIdBefore: sessionBefore,
|
|
2866
|
+
})
|
|
2867
|
+
.returning()
|
|
2868
|
+
.then((rows) => rows[0]);
|
|
2869
|
+
await tx
|
|
2870
|
+
.update(agentWakeupRequests)
|
|
2871
|
+
.set({
|
|
2872
|
+
status: "queued",
|
|
2873
|
+
reason: "issue_execution_promoted",
|
|
2874
|
+
runId: newRun.id,
|
|
2875
|
+
claimedAt: null,
|
|
2876
|
+
finishedAt: null,
|
|
2877
|
+
error: null,
|
|
2878
|
+
updatedAt: now,
|
|
2879
|
+
})
|
|
2880
|
+
.where(eq(agentWakeupRequests.id, deferred.id));
|
|
2881
|
+
await tx
|
|
2882
|
+
.update(issues)
|
|
2883
|
+
.set({
|
|
2884
|
+
executionRunId: newRun.id,
|
|
2885
|
+
executionAgentNameKey: normalizeAgentNameKey(deferredAgent.name),
|
|
2886
|
+
executionLockedAt: now,
|
|
2887
|
+
updatedAt: now,
|
|
2888
|
+
})
|
|
2889
|
+
.where(eq(issues.id, issue.id));
|
|
2890
|
+
return { promotedRun: newRun, passiveClosure };
|
|
2891
|
+
}
|
|
2892
|
+
});
|
|
2893
|
+
const passiveClosure = outcome.passiveClosure;
|
|
2894
|
+
if (passiveClosure?.kind === "queued") {
|
|
2895
|
+
await appendRunEvent(run, await nextRunEventSeq(run.id), {
|
|
2896
|
+
eventType: "issue.passive_followup_queued",
|
|
2897
|
+
stream: "system",
|
|
2898
|
+
level: "warn",
|
|
2899
|
+
message: `Queued passive issue follow-up ${passiveClosure.run.id}`,
|
|
2900
|
+
payload: {
|
|
2901
|
+
issueId: passiveClosure.issue.id,
|
|
2902
|
+
followupRunId: passiveClosure.run.id,
|
|
2903
|
+
originRunId: passiveClosure.originRunId,
|
|
2904
|
+
previousRunId: passiveClosure.previousRunId,
|
|
2905
|
+
attempt: passiveClosure.attempt,
|
|
2906
|
+
maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
|
|
2907
|
+
reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
|
|
2908
|
+
requestedAt: passiveClosure.requestedAt.toISOString(),
|
|
2909
|
+
},
|
|
2910
|
+
});
|
|
2911
|
+
await appendRunEvent(passiveClosure.run, await nextRunEventSeq(passiveClosure.run.id), {
|
|
2912
|
+
eventType: "issue.passive_followup_queued",
|
|
2913
|
+
stream: "system",
|
|
2914
|
+
level: "warn",
|
|
2915
|
+
message: `Passive follow-up queued because run ${run.id} ended without issue close-out`,
|
|
2916
|
+
payload: {
|
|
2917
|
+
issueId: passiveClosure.issue.id,
|
|
2918
|
+
originRunId: passiveClosure.originRunId,
|
|
2919
|
+
previousRunId: passiveClosure.previousRunId,
|
|
2920
|
+
attempt: passiveClosure.attempt,
|
|
2921
|
+
maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
|
|
2922
|
+
reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
|
|
2923
|
+
requestedAt: passiveClosure.requestedAt.toISOString(),
|
|
2924
|
+
},
|
|
2925
|
+
});
|
|
2926
|
+
await logActivity(db, {
|
|
2927
|
+
orgId: passiveClosure.issue.orgId,
|
|
2928
|
+
actorType: "system",
|
|
2929
|
+
actorId: "issue_closure_governance",
|
|
2930
|
+
action: "issue.passive_followup_queued",
|
|
2931
|
+
entityType: "issue",
|
|
2932
|
+
entityId: passiveClosure.issue.id,
|
|
2933
|
+
agentId: run.agentId,
|
|
2934
|
+
runId: run.id,
|
|
2935
|
+
details: {
|
|
2936
|
+
issueId: passiveClosure.issue.id,
|
|
2937
|
+
issueTitle: passiveClosure.issue.title,
|
|
2938
|
+
followupRunId: passiveClosure.run.id,
|
|
2939
|
+
originRunId: passiveClosure.originRunId,
|
|
2940
|
+
previousRunId: passiveClosure.previousRunId,
|
|
2941
|
+
attempt: passiveClosure.attempt,
|
|
2942
|
+
maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
|
|
2943
|
+
reason: ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON,
|
|
2944
|
+
requestedAt: passiveClosure.requestedAt.toISOString(),
|
|
2945
|
+
},
|
|
2946
|
+
});
|
|
2947
|
+
}
|
|
2948
|
+
else if (passiveClosure?.kind === "operator_review") {
|
|
2949
|
+
await appendRunEvent(run, await nextRunEventSeq(run.id), {
|
|
2950
|
+
eventType: "issue.closure_needs_operator_review",
|
|
2951
|
+
stream: "system",
|
|
2952
|
+
level: "warn",
|
|
2953
|
+
message: "Passive issue follow-up stopped and needs operator review",
|
|
2954
|
+
payload: {
|
|
2955
|
+
issueId: passiveClosure.issue.id,
|
|
2956
|
+
originRunId: passiveClosure.originRunId,
|
|
2957
|
+
previousRunId: passiveClosure.previousRunId,
|
|
2958
|
+
attempts: passiveClosure.attempts,
|
|
2959
|
+
maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
|
|
2960
|
+
reason: passiveClosure.reason,
|
|
2961
|
+
},
|
|
2962
|
+
});
|
|
2963
|
+
await logActivity(db, {
|
|
2964
|
+
orgId: passiveClosure.issue.orgId,
|
|
2965
|
+
actorType: "system",
|
|
2966
|
+
actorId: "issue_closure_governance",
|
|
2967
|
+
action: "issue.closure_needs_operator_review",
|
|
2968
|
+
entityType: "issue",
|
|
2969
|
+
entityId: passiveClosure.issue.id,
|
|
2970
|
+
agentId: run.agentId,
|
|
2971
|
+
runId: run.id,
|
|
2972
|
+
details: {
|
|
2973
|
+
issueId: passiveClosure.issue.id,
|
|
2974
|
+
issueTitle: passiveClosure.issue.title,
|
|
2975
|
+
originRunId: passiveClosure.originRunId,
|
|
2976
|
+
previousRunId: passiveClosure.previousRunId,
|
|
2977
|
+
attempts: passiveClosure.attempts,
|
|
2978
|
+
maxAttempts: ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS,
|
|
2979
|
+
reason: passiveClosure.reason,
|
|
2980
|
+
},
|
|
2981
|
+
});
|
|
2982
|
+
}
|
|
2983
|
+
const promotedRun = outcome.promotedRun;
|
|
2984
|
+
if (!promotedRun)
|
|
2985
|
+
return;
|
|
2986
|
+
publishLiveEvent({
|
|
2987
|
+
orgId: promotedRun.orgId,
|
|
2988
|
+
type: "heartbeat.run.queued",
|
|
2989
|
+
payload: {
|
|
2990
|
+
runId: promotedRun.id,
|
|
2991
|
+
agentId: promotedRun.agentId,
|
|
2992
|
+
invocationSource: promotedRun.invocationSource,
|
|
2993
|
+
triggerDetail: promotedRun.triggerDetail,
|
|
2994
|
+
wakeupRequestId: promotedRun.wakeupRequestId,
|
|
2995
|
+
},
|
|
2996
|
+
});
|
|
2997
|
+
await startNextQueuedRunForAgent(promotedRun.agentId);
|
|
2998
|
+
}
|
|
2999
|
+
async function enqueueWakeup(agentId, opts = {}) {
|
|
3000
|
+
const source = opts.source ?? "on_demand";
|
|
3001
|
+
const triggerDetail = opts.triggerDetail ?? null;
|
|
3002
|
+
const contextSnapshot = { ...(opts.contextSnapshot ?? {}) };
|
|
3003
|
+
const reason = opts.reason ?? null;
|
|
3004
|
+
const payload = opts.payload ?? null;
|
|
3005
|
+
const existingWakeupRequestId = readNonEmptyString(opts.existingWakeupRequestId);
|
|
3006
|
+
const { contextSnapshot: enrichedContextSnapshot, issueIdFromPayload, taskKey, wakeCommentId, } = enrichWakeContextSnapshot({
|
|
3007
|
+
contextSnapshot,
|
|
3008
|
+
reason,
|
|
3009
|
+
source,
|
|
3010
|
+
triggerDetail,
|
|
3011
|
+
payload,
|
|
3012
|
+
});
|
|
3013
|
+
let issueId = readNonEmptyString(enrichedContextSnapshot.issueId) ?? issueIdFromPayload;
|
|
3014
|
+
const agent = await getAgent(agentId);
|
|
3015
|
+
if (!agent)
|
|
3016
|
+
throw notFound("Agent not found");
|
|
3017
|
+
const explicitResumeSession = await resolveExplicitResumeSessionOverride(agent, payload, taskKey);
|
|
3018
|
+
if (explicitResumeSession) {
|
|
3019
|
+
enrichedContextSnapshot.resumeFromRunId = explicitResumeSession.resumeFromRunId;
|
|
3020
|
+
enrichedContextSnapshot.resumeSessionDisplayId = explicitResumeSession.sessionDisplayId;
|
|
3021
|
+
enrichedContextSnapshot.resumeSessionParams = explicitResumeSession.sessionParams;
|
|
3022
|
+
if (!readNonEmptyString(enrichedContextSnapshot.issueId) && explicitResumeSession.issueId) {
|
|
3023
|
+
enrichedContextSnapshot.issueId = explicitResumeSession.issueId;
|
|
3024
|
+
}
|
|
3025
|
+
if (!readNonEmptyString(enrichedContextSnapshot.taskId) && explicitResumeSession.taskId) {
|
|
3026
|
+
enrichedContextSnapshot.taskId = explicitResumeSession.taskId;
|
|
3027
|
+
}
|
|
3028
|
+
if (!readNonEmptyString(enrichedContextSnapshot.taskKey) && explicitResumeSession.taskKey) {
|
|
3029
|
+
enrichedContextSnapshot.taskKey = explicitResumeSession.taskKey;
|
|
3030
|
+
}
|
|
3031
|
+
issueId = readNonEmptyString(enrichedContextSnapshot.issueId) ?? issueId;
|
|
3032
|
+
}
|
|
3033
|
+
await hydrateWakeContextSnapshot(db, agent.orgId, enrichedContextSnapshot);
|
|
3034
|
+
const effectiveTaskKey = readNonEmptyString(enrichedContextSnapshot.taskKey) ?? taskKey;
|
|
3035
|
+
const sessionBefore = explicitResumeSession?.sessionDisplayId ??
|
|
3036
|
+
await resolveSessionBeforeForWakeup(agent, effectiveTaskKey);
|
|
3037
|
+
const writeSkippedRequest = async (skipReason) => {
|
|
3038
|
+
if (existingWakeupRequestId) {
|
|
3039
|
+
await setWakeupStatus(existingWakeupRequestId, "skipped", {
|
|
3040
|
+
reason: skipReason,
|
|
3041
|
+
finishedAt: new Date(),
|
|
3042
|
+
runId: null,
|
|
3043
|
+
claimedAt: null,
|
|
3044
|
+
error: null,
|
|
3045
|
+
});
|
|
3046
|
+
return;
|
|
3047
|
+
}
|
|
3048
|
+
await db.insert(agentWakeupRequests).values({
|
|
3049
|
+
orgId: agent.orgId,
|
|
3050
|
+
agentId,
|
|
3051
|
+
source,
|
|
3052
|
+
triggerDetail,
|
|
3053
|
+
reason: skipReason,
|
|
3054
|
+
payload,
|
|
3055
|
+
status: "skipped",
|
|
3056
|
+
requestedByActorType: opts.requestedByActorType ?? null,
|
|
3057
|
+
requestedByActorId: opts.requestedByActorId ?? null,
|
|
3058
|
+
idempotencyKey: opts.idempotencyKey ?? null,
|
|
3059
|
+
finishedAt: new Date(),
|
|
3060
|
+
});
|
|
3061
|
+
};
|
|
3062
|
+
let projectId = readNonEmptyString(enrichedContextSnapshot.projectId);
|
|
3063
|
+
if (!projectId && issueId) {
|
|
3064
|
+
projectId = await db
|
|
3065
|
+
.select({ projectId: issues.projectId })
|
|
3066
|
+
.from(issues)
|
|
3067
|
+
.where(and(eq(issues.id, issueId), eq(issues.orgId, agent.orgId)))
|
|
3068
|
+
.then((rows) => rows[0]?.projectId ?? null);
|
|
3069
|
+
}
|
|
3070
|
+
const budgetBlock = await budgets.getInvocationBlock(agent.orgId, agentId, {
|
|
3071
|
+
issueId,
|
|
3072
|
+
projectId,
|
|
3073
|
+
});
|
|
3074
|
+
if (budgetBlock) {
|
|
3075
|
+
await writeSkippedRequest("budget.blocked");
|
|
3076
|
+
throw conflict(budgetBlock.reason, {
|
|
3077
|
+
scopeType: budgetBlock.scopeType,
|
|
3078
|
+
scopeId: budgetBlock.scopeId,
|
|
3079
|
+
});
|
|
3080
|
+
}
|
|
3081
|
+
if (agent.status === "paused") {
|
|
3082
|
+
const deferredPayload = buildDeferredWakePayload(payload, enrichedContextSnapshot, issueId);
|
|
3083
|
+
if (existingWakeupRequestId) {
|
|
3084
|
+
await setWakeupStatus(existingWakeupRequestId, "deferred_agent_paused", {
|
|
3085
|
+
reason,
|
|
3086
|
+
payload: deferredPayload,
|
|
3087
|
+
runId: null,
|
|
3088
|
+
claimedAt: null,
|
|
3089
|
+
finishedAt: null,
|
|
3090
|
+
error: null,
|
|
3091
|
+
});
|
|
3092
|
+
return null;
|
|
3093
|
+
}
|
|
3094
|
+
await db.transaction(async (tx) => {
|
|
3095
|
+
const deferredRows = await tx
|
|
3096
|
+
.select()
|
|
3097
|
+
.from(agentWakeupRequests)
|
|
3098
|
+
.where(and(eq(agentWakeupRequests.orgId, agent.orgId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_agent_paused"), sql `${agentWakeupRequests.runId} is null`))
|
|
3099
|
+
.orderBy(asc(agentWakeupRequests.requestedAt));
|
|
3100
|
+
const existingDeferred = deferredRows.find((candidate) => isSameTaskScope(deriveDeferredWakeTaskKey(candidate.payload), effectiveTaskKey));
|
|
3101
|
+
if (existingDeferred) {
|
|
3102
|
+
const mergedDeferredContext = mergeCoalescedContextSnapshot(readDeferredWakeContext(existingDeferred.payload), enrichedContextSnapshot);
|
|
3103
|
+
await updateWakeupRequestRecord(tx, existingDeferred.id, {
|
|
3104
|
+
payload: buildDeferredWakePayload({
|
|
3105
|
+
...readDeferredWakePayload(existingDeferred.payload),
|
|
3106
|
+
...(payload ?? {}),
|
|
3107
|
+
}, mergedDeferredContext, issueId),
|
|
3108
|
+
coalescedCount: (existingDeferred.coalescedCount ?? 0) + 1,
|
|
3109
|
+
error: null,
|
|
3110
|
+
finishedAt: null,
|
|
3111
|
+
claimedAt: null,
|
|
3112
|
+
runId: null,
|
|
3113
|
+
});
|
|
3114
|
+
return;
|
|
3115
|
+
}
|
|
3116
|
+
await insertWakeupRequestRecord(tx, {
|
|
3117
|
+
orgId: agent.orgId,
|
|
3118
|
+
agentId,
|
|
3119
|
+
source,
|
|
3120
|
+
triggerDetail,
|
|
3121
|
+
reason,
|
|
3122
|
+
payload: deferredPayload,
|
|
3123
|
+
status: "deferred_agent_paused",
|
|
3124
|
+
requestedByActorType: opts.requestedByActorType ?? null,
|
|
3125
|
+
requestedByActorId: opts.requestedByActorId ?? null,
|
|
3126
|
+
idempotencyKey: opts.idempotencyKey ?? null,
|
|
3127
|
+
});
|
|
3128
|
+
});
|
|
3129
|
+
return null;
|
|
3130
|
+
}
|
|
3131
|
+
if (agent.status === "terminated" || agent.status === "pending_approval") {
|
|
3132
|
+
throw conflict("Agent is not invokable in its current state", { status: agent.status });
|
|
3133
|
+
}
|
|
3134
|
+
const policy = parseHeartbeatPolicy(agent);
|
|
3135
|
+
if (source === "timer" && !policy.enabled) {
|
|
3136
|
+
await writeSkippedRequest("heartbeat.disabled");
|
|
3137
|
+
return null;
|
|
3138
|
+
}
|
|
3139
|
+
if (source !== "timer" && !policy.wakeOnDemand) {
|
|
3140
|
+
await writeSkippedRequest("heartbeat.wakeOnDemand.disabled");
|
|
3141
|
+
return null;
|
|
3142
|
+
}
|
|
3143
|
+
const bypassIssueExecutionLock = reason === "issue_comment_mentioned" ||
|
|
3144
|
+
readNonEmptyString(enrichedContextSnapshot.wakeReason) === "issue_comment_mentioned";
|
|
3145
|
+
if (issueId && !bypassIssueExecutionLock) {
|
|
3146
|
+
const agentNameKey = normalizeAgentNameKey(agent.name);
|
|
3147
|
+
const outcome = await db.transaction(async (tx) => {
|
|
3148
|
+
await tx.execute(sql `select id from issues where id = ${issueId} and org_id = ${agent.orgId} for update`);
|
|
3149
|
+
const issue = await tx
|
|
3150
|
+
.select({
|
|
3151
|
+
id: issues.id,
|
|
3152
|
+
orgId: issues.orgId,
|
|
3153
|
+
executionRunId: issues.executionRunId,
|
|
3154
|
+
executionAgentNameKey: issues.executionAgentNameKey,
|
|
3155
|
+
})
|
|
3156
|
+
.from(issues)
|
|
3157
|
+
.where(and(eq(issues.id, issueId), eq(issues.orgId, agent.orgId)))
|
|
3158
|
+
.then((rows) => rows[0] ?? null);
|
|
3159
|
+
if (!issue) {
|
|
3160
|
+
if (existingWakeupRequestId) {
|
|
3161
|
+
await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
|
|
3162
|
+
status: "skipped",
|
|
3163
|
+
reason: "issue_execution_issue_not_found",
|
|
3164
|
+
runId: null,
|
|
3165
|
+
claimedAt: null,
|
|
3166
|
+
finishedAt: new Date(),
|
|
3167
|
+
error: null,
|
|
3168
|
+
});
|
|
3169
|
+
}
|
|
3170
|
+
else {
|
|
3171
|
+
await insertWakeupRequestRecord(tx, {
|
|
3172
|
+
orgId: agent.orgId,
|
|
3173
|
+
agentId,
|
|
3174
|
+
source,
|
|
3175
|
+
triggerDetail,
|
|
3176
|
+
reason: "issue_execution_issue_not_found",
|
|
3177
|
+
payload,
|
|
3178
|
+
status: "skipped",
|
|
3179
|
+
requestedByActorType: opts.requestedByActorType ?? null,
|
|
3180
|
+
requestedByActorId: opts.requestedByActorId ?? null,
|
|
3181
|
+
idempotencyKey: opts.idempotencyKey ?? null,
|
|
3182
|
+
finishedAt: new Date(),
|
|
3183
|
+
});
|
|
3184
|
+
}
|
|
3185
|
+
return { kind: "skipped" };
|
|
3186
|
+
}
|
|
3187
|
+
let activeExecutionRun = issue.executionRunId
|
|
3188
|
+
? await tx
|
|
3189
|
+
.select()
|
|
3190
|
+
.from(heartbeatRuns)
|
|
3191
|
+
.where(eq(heartbeatRuns.id, issue.executionRunId))
|
|
3192
|
+
.then((rows) => rows[0] ?? null)
|
|
3193
|
+
: null;
|
|
3194
|
+
if (activeExecutionRun && activeExecutionRun.status !== "queued" && activeExecutionRun.status !== "running") {
|
|
3195
|
+
activeExecutionRun = null;
|
|
3196
|
+
}
|
|
3197
|
+
if (!activeExecutionRun && issue.executionRunId) {
|
|
3198
|
+
await tx
|
|
3199
|
+
.update(issues)
|
|
3200
|
+
.set({
|
|
3201
|
+
executionRunId: null,
|
|
3202
|
+
executionAgentNameKey: null,
|
|
3203
|
+
executionLockedAt: null,
|
|
3204
|
+
updatedAt: new Date(),
|
|
3205
|
+
})
|
|
3206
|
+
.where(eq(issues.id, issue.id));
|
|
3207
|
+
}
|
|
3208
|
+
if (!activeExecutionRun) {
|
|
3209
|
+
const legacyRun = await tx
|
|
3210
|
+
.select()
|
|
3211
|
+
.from(heartbeatRuns)
|
|
3212
|
+
.where(and(eq(heartbeatRuns.orgId, issue.orgId), inArray(heartbeatRuns.status, ["queued", "running"]), sql `${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`))
|
|
3213
|
+
.orderBy(sql `case when ${heartbeatRuns.status} = 'running' then 0 else 1 end`, asc(heartbeatRuns.createdAt))
|
|
3214
|
+
.limit(1)
|
|
3215
|
+
.then((rows) => rows[0] ?? null);
|
|
3216
|
+
if (legacyRun) {
|
|
3217
|
+
activeExecutionRun = legacyRun;
|
|
3218
|
+
const legacyAgent = await tx
|
|
3219
|
+
.select({ name: agents.name })
|
|
3220
|
+
.from(agents)
|
|
3221
|
+
.where(eq(agents.id, legacyRun.agentId))
|
|
3222
|
+
.then((rows) => rows[0] ?? null);
|
|
3223
|
+
await tx
|
|
3224
|
+
.update(issues)
|
|
3225
|
+
.set({
|
|
3226
|
+
executionRunId: legacyRun.id,
|
|
3227
|
+
executionAgentNameKey: normalizeAgentNameKey(legacyAgent?.name),
|
|
3228
|
+
executionLockedAt: new Date(),
|
|
3229
|
+
updatedAt: new Date(),
|
|
3230
|
+
})
|
|
3231
|
+
.where(eq(issues.id, issue.id));
|
|
3232
|
+
}
|
|
3233
|
+
}
|
|
3234
|
+
if (activeExecutionRun) {
|
|
3235
|
+
const executionAgent = await tx
|
|
3236
|
+
.select({ name: agents.name })
|
|
3237
|
+
.from(agents)
|
|
3238
|
+
.where(eq(agents.id, activeExecutionRun.agentId))
|
|
3239
|
+
.then((rows) => rows[0] ?? null);
|
|
3240
|
+
const executionAgentNameKey = normalizeAgentNameKey(issue.executionAgentNameKey) ??
|
|
3241
|
+
normalizeAgentNameKey(executionAgent?.name);
|
|
3242
|
+
const isSameExecutionAgent = Boolean(executionAgentNameKey) && executionAgentNameKey === agentNameKey;
|
|
3243
|
+
const shouldQueueFollowupForCommentWake = Boolean(wakeCommentId) &&
|
|
3244
|
+
activeExecutionRun.status === "running" &&
|
|
3245
|
+
isSameExecutionAgent;
|
|
3246
|
+
if (isSameExecutionAgent && !shouldQueueFollowupForCommentWake) {
|
|
3247
|
+
const mergedContextSnapshot = mergeCoalescedContextSnapshot(activeExecutionRun.contextSnapshot, enrichedContextSnapshot);
|
|
3248
|
+
const mergedRun = await tx
|
|
3249
|
+
.update(heartbeatRuns)
|
|
3250
|
+
.set({
|
|
3251
|
+
contextSnapshot: mergedContextSnapshot,
|
|
3252
|
+
updatedAt: new Date(),
|
|
3253
|
+
})
|
|
3254
|
+
.where(eq(heartbeatRuns.id, activeExecutionRun.id))
|
|
3255
|
+
.returning()
|
|
3256
|
+
.then((rows) => rows[0] ?? activeExecutionRun);
|
|
3257
|
+
if (existingWakeupRequestId) {
|
|
3258
|
+
await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
|
|
3259
|
+
status: "coalesced",
|
|
3260
|
+
reason: "issue_execution_same_name",
|
|
3261
|
+
runId: mergedRun.id,
|
|
3262
|
+
claimedAt: null,
|
|
3263
|
+
finishedAt: new Date(),
|
|
3264
|
+
error: null,
|
|
3265
|
+
});
|
|
3266
|
+
}
|
|
3267
|
+
else {
|
|
3268
|
+
await insertWakeupRequestRecord(tx, {
|
|
3269
|
+
orgId: agent.orgId,
|
|
3270
|
+
agentId,
|
|
3271
|
+
source,
|
|
3272
|
+
triggerDetail,
|
|
3273
|
+
reason: "issue_execution_same_name",
|
|
3274
|
+
payload,
|
|
3275
|
+
status: "coalesced",
|
|
3276
|
+
coalescedCount: 1,
|
|
3277
|
+
requestedByActorType: opts.requestedByActorType ?? null,
|
|
3278
|
+
requestedByActorId: opts.requestedByActorId ?? null,
|
|
3279
|
+
idempotencyKey: opts.idempotencyKey ?? null,
|
|
3280
|
+
runId: mergedRun.id,
|
|
3281
|
+
finishedAt: new Date(),
|
|
3282
|
+
});
|
|
3283
|
+
}
|
|
3284
|
+
return { kind: "coalesced", run: mergedRun };
|
|
3285
|
+
}
|
|
3286
|
+
const deferredPayload = buildDeferredWakePayload(payload, enrichedContextSnapshot, issueId);
|
|
3287
|
+
const existingDeferred = await tx
|
|
3288
|
+
.select()
|
|
3289
|
+
.from(agentWakeupRequests)
|
|
3290
|
+
.where(and(eq(agentWakeupRequests.orgId, agent.orgId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql `${agentWakeupRequests.payload} ->> 'issueId' = ${issue.id}`))
|
|
3291
|
+
.orderBy(asc(agentWakeupRequests.requestedAt))
|
|
3292
|
+
.limit(1)
|
|
3293
|
+
.then((rows) => rows[0] ?? null);
|
|
3294
|
+
if (existingDeferred) {
|
|
3295
|
+
const mergedDeferredContext = mergeCoalescedContextSnapshot(readDeferredWakeContext(existingDeferred.payload), enrichedContextSnapshot);
|
|
3296
|
+
const mergedDeferredPayload = buildDeferredWakePayload({
|
|
3297
|
+
...readDeferredWakePayload(existingDeferred.payload),
|
|
3298
|
+
...(payload ?? {}),
|
|
3299
|
+
}, mergedDeferredContext, issueId);
|
|
3300
|
+
if (existingWakeupRequestId && existingDeferred.id !== existingWakeupRequestId) {
|
|
3301
|
+
await updateWakeupRequestRecord(tx, existingDeferred.id, {
|
|
3302
|
+
payload: mergedDeferredPayload,
|
|
3303
|
+
coalescedCount: (existingDeferred.coalescedCount ?? 0) + 1,
|
|
3304
|
+
});
|
|
3305
|
+
await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
|
|
3306
|
+
status: "coalesced",
|
|
3307
|
+
reason: "issue_execution_deferred",
|
|
3308
|
+
runId: null,
|
|
3309
|
+
claimedAt: null,
|
|
3310
|
+
finishedAt: new Date(),
|
|
3311
|
+
error: null,
|
|
3312
|
+
});
|
|
3313
|
+
}
|
|
3314
|
+
else {
|
|
3315
|
+
await updateWakeupRequestRecord(tx, existingDeferred.id, {
|
|
3316
|
+
payload: mergedDeferredPayload,
|
|
3317
|
+
coalescedCount: (existingDeferred.coalescedCount ?? 0) + 1,
|
|
3318
|
+
status: "deferred_issue_execution",
|
|
3319
|
+
reason: "issue_execution_deferred",
|
|
3320
|
+
runId: null,
|
|
3321
|
+
claimedAt: null,
|
|
3322
|
+
finishedAt: null,
|
|
3323
|
+
error: null,
|
|
3324
|
+
});
|
|
3325
|
+
}
|
|
3326
|
+
return { kind: "deferred" };
|
|
3327
|
+
}
|
|
3328
|
+
if (existingWakeupRequestId) {
|
|
3329
|
+
await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
|
|
3330
|
+
status: "deferred_issue_execution",
|
|
3331
|
+
reason: "issue_execution_deferred",
|
|
3332
|
+
payload: deferredPayload,
|
|
3333
|
+
runId: null,
|
|
3334
|
+
claimedAt: null,
|
|
3335
|
+
finishedAt: null,
|
|
3336
|
+
error: null,
|
|
3337
|
+
});
|
|
3338
|
+
}
|
|
3339
|
+
else {
|
|
3340
|
+
await insertWakeupRequestRecord(tx, {
|
|
3341
|
+
orgId: agent.orgId,
|
|
3342
|
+
agentId,
|
|
3343
|
+
source,
|
|
3344
|
+
triggerDetail,
|
|
3345
|
+
reason: "issue_execution_deferred",
|
|
3346
|
+
payload: deferredPayload,
|
|
3347
|
+
status: "deferred_issue_execution",
|
|
3348
|
+
requestedByActorType: opts.requestedByActorType ?? null,
|
|
3349
|
+
requestedByActorId: opts.requestedByActorId ?? null,
|
|
3350
|
+
idempotencyKey: opts.idempotencyKey ?? null,
|
|
3351
|
+
});
|
|
3352
|
+
}
|
|
3353
|
+
return { kind: "deferred" };
|
|
3354
|
+
}
|
|
3355
|
+
const wakeupRequest = existingWakeupRequestId
|
|
3356
|
+
? await updateWakeupRequestRecord(tx, existingWakeupRequestId, {
|
|
3357
|
+
status: "queued",
|
|
3358
|
+
runId: null,
|
|
3359
|
+
claimedAt: null,
|
|
3360
|
+
finishedAt: null,
|
|
3361
|
+
error: null,
|
|
3362
|
+
})
|
|
3363
|
+
: await insertWakeupRequestRecord(tx, {
|
|
3364
|
+
orgId: agent.orgId,
|
|
3365
|
+
agentId,
|
|
3366
|
+
source,
|
|
3367
|
+
triggerDetail,
|
|
3368
|
+
reason,
|
|
3369
|
+
payload,
|
|
3370
|
+
status: "queued",
|
|
3371
|
+
requestedByActorType: opts.requestedByActorType ?? null,
|
|
3372
|
+
requestedByActorId: opts.requestedByActorId ?? null,
|
|
3373
|
+
idempotencyKey: opts.idempotencyKey ?? null,
|
|
3374
|
+
});
|
|
3375
|
+
const newRun = await tx
|
|
3376
|
+
.insert(heartbeatRuns)
|
|
3377
|
+
.values({
|
|
3378
|
+
orgId: agent.orgId,
|
|
3379
|
+
agentId,
|
|
3380
|
+
invocationSource: source,
|
|
3381
|
+
triggerDetail,
|
|
3382
|
+
status: "queued",
|
|
3383
|
+
wakeupRequestId: wakeupRequest.id,
|
|
3384
|
+
contextSnapshot: enrichedContextSnapshot,
|
|
3385
|
+
sessionIdBefore: sessionBefore,
|
|
3386
|
+
})
|
|
3387
|
+
.returning()
|
|
3388
|
+
.then((rows) => rows[0]);
|
|
3389
|
+
await updateWakeupRequestRecord(tx, wakeupRequest.id, {
|
|
3390
|
+
runId: newRun.id,
|
|
3391
|
+
status: "queued",
|
|
3392
|
+
claimedAt: null,
|
|
3393
|
+
finishedAt: null,
|
|
3394
|
+
error: null,
|
|
3395
|
+
});
|
|
3396
|
+
await tx
|
|
3397
|
+
.update(issues)
|
|
3398
|
+
.set({
|
|
3399
|
+
executionRunId: newRun.id,
|
|
3400
|
+
executionAgentNameKey: agentNameKey,
|
|
3401
|
+
executionLockedAt: new Date(),
|
|
3402
|
+
updatedAt: new Date(),
|
|
3403
|
+
})
|
|
3404
|
+
.where(eq(issues.id, issue.id));
|
|
3405
|
+
return { kind: "queued", run: newRun };
|
|
3406
|
+
});
|
|
3407
|
+
if (outcome.kind === "deferred" || outcome.kind === "skipped")
|
|
3408
|
+
return null;
|
|
3409
|
+
if (outcome.kind === "coalesced")
|
|
3410
|
+
return outcome.run;
|
|
3411
|
+
const newRun = outcome.run;
|
|
3412
|
+
publishLiveEvent({
|
|
3413
|
+
orgId: newRun.orgId,
|
|
3414
|
+
type: "heartbeat.run.queued",
|
|
3415
|
+
payload: {
|
|
3416
|
+
runId: newRun.id,
|
|
3417
|
+
agentId: newRun.agentId,
|
|
3418
|
+
invocationSource: newRun.invocationSource,
|
|
3419
|
+
triggerDetail: newRun.triggerDetail,
|
|
3420
|
+
wakeupRequestId: newRun.wakeupRequestId,
|
|
3421
|
+
},
|
|
3422
|
+
});
|
|
3423
|
+
await startNextQueuedRunForAgent(agent.id);
|
|
3424
|
+
return newRun;
|
|
3425
|
+
}
|
|
3426
|
+
const activeRuns = await db
|
|
3427
|
+
.select()
|
|
3428
|
+
.from(heartbeatRuns)
|
|
3429
|
+
.where(and(eq(heartbeatRuns.agentId, agentId), inArray(heartbeatRuns.status, ["queued", "running"])))
|
|
3430
|
+
.orderBy(desc(heartbeatRuns.createdAt));
|
|
3431
|
+
const sameScopeQueuedRun = activeRuns.find((candidate) => candidate.status === "queued" && isSameTaskScope(runTaskKey(candidate), taskKey));
|
|
3432
|
+
const sameScopeRunningRun = activeRuns.find((candidate) => candidate.status === "running" && isSameTaskScope(runTaskKey(candidate), taskKey));
|
|
3433
|
+
const shouldQueueFollowupForCommentWake = Boolean(wakeCommentId) && Boolean(sameScopeRunningRun) && !sameScopeQueuedRun;
|
|
3434
|
+
const coalescedTargetRun = sameScopeQueuedRun ??
|
|
3435
|
+
(shouldQueueFollowupForCommentWake ? null : sameScopeRunningRun ?? null);
|
|
3436
|
+
if (coalescedTargetRun) {
|
|
3437
|
+
const mergedContextSnapshot = mergeCoalescedContextSnapshot(coalescedTargetRun.contextSnapshot, contextSnapshot);
|
|
3438
|
+
const mergedRun = await db
|
|
3439
|
+
.update(heartbeatRuns)
|
|
3440
|
+
.set({
|
|
3441
|
+
contextSnapshot: mergedContextSnapshot,
|
|
3442
|
+
updatedAt: new Date(),
|
|
3443
|
+
})
|
|
3444
|
+
.where(eq(heartbeatRuns.id, coalescedTargetRun.id))
|
|
3445
|
+
.returning()
|
|
3446
|
+
.then((rows) => rows[0] ?? coalescedTargetRun);
|
|
3447
|
+
if (existingWakeupRequestId) {
|
|
3448
|
+
await setWakeupStatus(existingWakeupRequestId, "coalesced", {
|
|
3449
|
+
runId: mergedRun.id,
|
|
3450
|
+
claimedAt: null,
|
|
3451
|
+
finishedAt: new Date(),
|
|
3452
|
+
error: null,
|
|
3453
|
+
});
|
|
3454
|
+
}
|
|
3455
|
+
else {
|
|
3456
|
+
await db.insert(agentWakeupRequests).values({
|
|
3457
|
+
orgId: agent.orgId,
|
|
3458
|
+
agentId,
|
|
3459
|
+
source,
|
|
3460
|
+
triggerDetail,
|
|
3461
|
+
reason,
|
|
3462
|
+
payload,
|
|
3463
|
+
status: "coalesced",
|
|
3464
|
+
coalescedCount: 1,
|
|
3465
|
+
requestedByActorType: opts.requestedByActorType ?? null,
|
|
3466
|
+
requestedByActorId: opts.requestedByActorId ?? null,
|
|
3467
|
+
idempotencyKey: opts.idempotencyKey ?? null,
|
|
3468
|
+
runId: mergedRun.id,
|
|
3469
|
+
finishedAt: new Date(),
|
|
3470
|
+
});
|
|
3471
|
+
}
|
|
3472
|
+
return mergedRun;
|
|
3473
|
+
}
|
|
3474
|
+
const wakeupRequest = existingWakeupRequestId
|
|
3475
|
+
? await updateWakeupRequestRecord(db, existingWakeupRequestId, {
|
|
3476
|
+
status: "queued",
|
|
3477
|
+
runId: null,
|
|
3478
|
+
claimedAt: null,
|
|
3479
|
+
finishedAt: null,
|
|
3480
|
+
error: null,
|
|
3481
|
+
})
|
|
3482
|
+
: await insertWakeupRequestRecord(db, {
|
|
3483
|
+
orgId: agent.orgId,
|
|
3484
|
+
agentId,
|
|
3485
|
+
source,
|
|
3486
|
+
triggerDetail,
|
|
3487
|
+
reason,
|
|
3488
|
+
payload,
|
|
3489
|
+
status: "queued",
|
|
3490
|
+
requestedByActorType: opts.requestedByActorType ?? null,
|
|
3491
|
+
requestedByActorId: opts.requestedByActorId ?? null,
|
|
3492
|
+
idempotencyKey: opts.idempotencyKey ?? null,
|
|
3493
|
+
});
|
|
3494
|
+
const newRun = await db
|
|
3495
|
+
.insert(heartbeatRuns)
|
|
3496
|
+
.values({
|
|
3497
|
+
orgId: agent.orgId,
|
|
3498
|
+
agentId,
|
|
3499
|
+
invocationSource: source,
|
|
3500
|
+
triggerDetail,
|
|
3501
|
+
status: "queued",
|
|
3502
|
+
wakeupRequestId: wakeupRequest.id,
|
|
3503
|
+
contextSnapshot: enrichedContextSnapshot,
|
|
3504
|
+
sessionIdBefore: sessionBefore,
|
|
3505
|
+
})
|
|
3506
|
+
.returning()
|
|
3507
|
+
.then((rows) => rows[0]);
|
|
3508
|
+
await updateWakeupRequestRecord(db, wakeupRequest.id, {
|
|
3509
|
+
status: "queued",
|
|
3510
|
+
runId: newRun.id,
|
|
3511
|
+
claimedAt: null,
|
|
3512
|
+
finishedAt: null,
|
|
3513
|
+
error: null,
|
|
3514
|
+
});
|
|
3515
|
+
publishLiveEvent({
|
|
3516
|
+
orgId: newRun.orgId,
|
|
3517
|
+
type: "heartbeat.run.queued",
|
|
3518
|
+
payload: {
|
|
3519
|
+
runId: newRun.id,
|
|
3520
|
+
agentId: newRun.agentId,
|
|
3521
|
+
invocationSource: newRun.invocationSource,
|
|
3522
|
+
triggerDetail: newRun.triggerDetail,
|
|
3523
|
+
wakeupRequestId: newRun.wakeupRequestId,
|
|
3524
|
+
},
|
|
3525
|
+
});
|
|
3526
|
+
await startNextQueuedRunForAgent(agent.id);
|
|
3527
|
+
return newRun;
|
|
3528
|
+
}
|
|
3529
|
+
async function resumeDeferredWakeupsForAgent(agentId) {
|
|
3530
|
+
const agent = await getAgent(agentId);
|
|
3531
|
+
if (!agent)
|
|
3532
|
+
throw notFound("Agent not found");
|
|
3533
|
+
const replayedRequestIds = [];
|
|
3534
|
+
while (true) {
|
|
3535
|
+
const deferred = await db
|
|
3536
|
+
.select()
|
|
3537
|
+
.from(agentWakeupRequests)
|
|
3538
|
+
.where(and(eq(agentWakeupRequests.orgId, agent.orgId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_agent_paused"), sql `${agentWakeupRequests.runId} is null`))
|
|
3539
|
+
.orderBy(asc(agentWakeupRequests.requestedAt))
|
|
3540
|
+
.limit(1)
|
|
3541
|
+
.then((rows) => rows[0] ?? null);
|
|
3542
|
+
if (!deferred)
|
|
3543
|
+
break;
|
|
3544
|
+
const replayPayload = readDeferredWakePayload(deferred.payload);
|
|
3545
|
+
const replayContextSnapshot = readDeferredWakeContext(deferred.payload);
|
|
3546
|
+
try {
|
|
3547
|
+
await enqueueWakeup(agentId, {
|
|
3548
|
+
source: readNonEmptyString(deferred.source) ?? "on_demand",
|
|
3549
|
+
triggerDetail: readNonEmptyString(deferred.triggerDetail) ?? undefined,
|
|
3550
|
+
reason: readNonEmptyString(deferred.reason) ?? null,
|
|
3551
|
+
payload: replayPayload,
|
|
3552
|
+
idempotencyKey: deferred.idempotencyKey,
|
|
3553
|
+
requestedByActorType: deferred.requestedByActorType ?? undefined,
|
|
3554
|
+
requestedByActorId: deferred.requestedByActorId,
|
|
3555
|
+
contextSnapshot: replayContextSnapshot,
|
|
3556
|
+
existingWakeupRequestId: deferred.id,
|
|
3557
|
+
});
|
|
3558
|
+
}
|
|
3559
|
+
catch (error) {
|
|
3560
|
+
const current = await db
|
|
3561
|
+
.select({ status: agentWakeupRequests.status })
|
|
3562
|
+
.from(agentWakeupRequests)
|
|
3563
|
+
.where(eq(agentWakeupRequests.id, deferred.id))
|
|
3564
|
+
.then((rows) => rows[0] ?? null);
|
|
3565
|
+
if (current?.status === "deferred_agent_paused") {
|
|
3566
|
+
await setWakeupStatus(deferred.id, "failed", {
|
|
3567
|
+
finishedAt: new Date(),
|
|
3568
|
+
error: error instanceof Error ? error.message : String(error),
|
|
3569
|
+
});
|
|
3570
|
+
}
|
|
3571
|
+
}
|
|
3572
|
+
replayedRequestIds.push(deferred.id);
|
|
3573
|
+
const current = await db
|
|
3574
|
+
.select({ status: agentWakeupRequests.status })
|
|
3575
|
+
.from(agentWakeupRequests)
|
|
3576
|
+
.where(eq(agentWakeupRequests.id, deferred.id))
|
|
3577
|
+
.then((rows) => rows[0] ?? null);
|
|
3578
|
+
if (current?.status === "deferred_agent_paused")
|
|
3579
|
+
break;
|
|
3580
|
+
}
|
|
3581
|
+
return {
|
|
3582
|
+
replayed: replayedRequestIds.length,
|
|
3583
|
+
wakeupRequestIds: replayedRequestIds,
|
|
3584
|
+
};
|
|
3585
|
+
}
|
|
3586
|
+
async function listProjectScopedRunIds(orgId, projectId) {
|
|
3587
|
+
const runIssueId = sql `${heartbeatRuns.contextSnapshot} ->> 'issueId'`;
|
|
3588
|
+
const effectiveProjectId = sql `coalesce(${heartbeatRuns.contextSnapshot} ->> 'projectId', ${issues.projectId}::text)`;
|
|
3589
|
+
const rows = await db
|
|
3590
|
+
.selectDistinctOn([heartbeatRuns.id], { id: heartbeatRuns.id })
|
|
3591
|
+
.from(heartbeatRuns)
|
|
3592
|
+
.leftJoin(issues, and(eq(issues.orgId, orgId), sql `${issues.id}::text = ${runIssueId}`))
|
|
3593
|
+
.where(and(eq(heartbeatRuns.orgId, orgId), inArray(heartbeatRuns.status, ["queued", "running"]), sql `${effectiveProjectId} = ${projectId}`));
|
|
3594
|
+
return rows.map((row) => row.id);
|
|
3595
|
+
}
|
|
3596
|
+
async function listProjectScopedWakeupIds(orgId, projectId) {
|
|
3597
|
+
const wakeIssueId = sql `${agentWakeupRequests.payload} ->> 'issueId'`;
|
|
3598
|
+
const effectiveProjectId = sql `coalesce(${agentWakeupRequests.payload} ->> 'projectId', ${issues.projectId}::text)`;
|
|
3599
|
+
const rows = await db
|
|
3600
|
+
.selectDistinctOn([agentWakeupRequests.id], { id: agentWakeupRequests.id })
|
|
3601
|
+
.from(agentWakeupRequests)
|
|
3602
|
+
.leftJoin(issues, and(eq(issues.orgId, orgId), sql `${issues.id}::text = ${wakeIssueId}`))
|
|
3603
|
+
.where(and(eq(agentWakeupRequests.orgId, orgId), inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), sql `${agentWakeupRequests.runId} is null`, sql `${effectiveProjectId} = ${projectId}`));
|
|
3604
|
+
return rows.map((row) => row.id);
|
|
3605
|
+
}
|
|
3606
|
+
async function cancelPendingWakeupsForBudgetScope(scope) {
|
|
3607
|
+
const now = new Date();
|
|
3608
|
+
let wakeupIds = [];
|
|
3609
|
+
if (scope.scopeType === "organization") {
|
|
3610
|
+
wakeupIds = await db
|
|
3611
|
+
.select({ id: agentWakeupRequests.id })
|
|
3612
|
+
.from(agentWakeupRequests)
|
|
3613
|
+
.where(and(eq(agentWakeupRequests.orgId, scope.orgId), inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), sql `${agentWakeupRequests.runId} is null`))
|
|
3614
|
+
.then((rows) => rows.map((row) => row.id));
|
|
3615
|
+
}
|
|
3616
|
+
else if (scope.scopeType === "agent") {
|
|
3617
|
+
wakeupIds = await db
|
|
3618
|
+
.select({ id: agentWakeupRequests.id })
|
|
3619
|
+
.from(agentWakeupRequests)
|
|
3620
|
+
.where(and(eq(agentWakeupRequests.orgId, scope.orgId), eq(agentWakeupRequests.agentId, scope.scopeId), inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), sql `${agentWakeupRequests.runId} is null`))
|
|
3621
|
+
.then((rows) => rows.map((row) => row.id));
|
|
3622
|
+
}
|
|
3623
|
+
else {
|
|
3624
|
+
wakeupIds = await listProjectScopedWakeupIds(scope.orgId, scope.scopeId);
|
|
3625
|
+
}
|
|
3626
|
+
if (wakeupIds.length === 0)
|
|
3627
|
+
return 0;
|
|
3628
|
+
await db
|
|
3629
|
+
.update(agentWakeupRequests)
|
|
3630
|
+
.set({
|
|
3631
|
+
status: "cancelled",
|
|
3632
|
+
finishedAt: now,
|
|
3633
|
+
error: "Cancelled due to budget pause",
|
|
3634
|
+
updatedAt: now,
|
|
3635
|
+
})
|
|
3636
|
+
.where(inArray(agentWakeupRequests.id, wakeupIds));
|
|
3637
|
+
return wakeupIds.length;
|
|
3638
|
+
}
|
|
3639
|
+
async function cancelRunInternal(runId, reason = "Cancelled by control plane") {
|
|
3640
|
+
const run = await getRun(runId);
|
|
3641
|
+
if (!run)
|
|
3642
|
+
throw notFound("Heartbeat run not found");
|
|
3643
|
+
if (run.status !== "running" && run.status !== "queued")
|
|
3644
|
+
return run;
|
|
3645
|
+
const running = runningProcesses.get(run.id);
|
|
3646
|
+
if (running) {
|
|
3647
|
+
running.child.kill("SIGTERM");
|
|
3648
|
+
const graceMs = Math.max(1, running.graceSec) * 1000;
|
|
3649
|
+
setTimeout(() => {
|
|
3650
|
+
if (!running.child.killed) {
|
|
3651
|
+
running.child.kill("SIGKILL");
|
|
3652
|
+
}
|
|
3653
|
+
}, graceMs);
|
|
3654
|
+
}
|
|
3655
|
+
const cancelled = await setRunStatus(run.id, "cancelled", {
|
|
3656
|
+
finishedAt: new Date(),
|
|
3657
|
+
error: reason,
|
|
3658
|
+
errorCode: "cancelled",
|
|
3659
|
+
});
|
|
3660
|
+
await setWakeupStatus(run.wakeupRequestId, "cancelled", {
|
|
3661
|
+
finishedAt: new Date(),
|
|
3662
|
+
error: reason,
|
|
3663
|
+
});
|
|
3664
|
+
if (cancelled) {
|
|
3665
|
+
await appendRunEvent(cancelled, 1, {
|
|
3666
|
+
eventType: "lifecycle",
|
|
3667
|
+
stream: "system",
|
|
3668
|
+
level: "warn",
|
|
3669
|
+
message: "run cancelled",
|
|
3670
|
+
});
|
|
3671
|
+
await releaseIssueExecutionAndPromote(cancelled);
|
|
3672
|
+
}
|
|
3673
|
+
runningProcesses.delete(run.id);
|
|
3674
|
+
await finalizeAgentStatus(run.agentId, "cancelled");
|
|
3675
|
+
await startNextQueuedRunForAgent(run.agentId);
|
|
3676
|
+
return cancelled;
|
|
3677
|
+
}
|
|
3678
|
+
async function cancelActiveForAgentInternal(agentId, reason = "Cancelled due to agent pause") {
|
|
3679
|
+
const runs = await db
|
|
3680
|
+
.select()
|
|
3681
|
+
.from(heartbeatRuns)
|
|
3682
|
+
.where(and(eq(heartbeatRuns.agentId, agentId), inArray(heartbeatRuns.status, ["queued", "running"])));
|
|
3683
|
+
for (const run of runs) {
|
|
3684
|
+
await setRunStatus(run.id, "cancelled", {
|
|
3685
|
+
finishedAt: new Date(),
|
|
3686
|
+
error: reason,
|
|
3687
|
+
errorCode: "cancelled",
|
|
3688
|
+
});
|
|
3689
|
+
await setWakeupStatus(run.wakeupRequestId, "cancelled", {
|
|
3690
|
+
finishedAt: new Date(),
|
|
3691
|
+
error: reason,
|
|
3692
|
+
});
|
|
3693
|
+
const running = runningProcesses.get(run.id);
|
|
3694
|
+
if (running) {
|
|
3695
|
+
running.child.kill("SIGTERM");
|
|
3696
|
+
runningProcesses.delete(run.id);
|
|
3697
|
+
}
|
|
3698
|
+
await releaseIssueExecutionAndPromote(run);
|
|
3699
|
+
}
|
|
3700
|
+
return runs.length;
|
|
3701
|
+
}
|
|
3702
|
+
async function cancelBudgetScopeWork(scope) {
|
|
3703
|
+
if (scope.scopeType === "agent") {
|
|
3704
|
+
await cancelActiveForAgentInternal(scope.scopeId, "Cancelled due to budget pause");
|
|
3705
|
+
await cancelPendingWakeupsForBudgetScope(scope);
|
|
3706
|
+
return;
|
|
3707
|
+
}
|
|
3708
|
+
const runIds = scope.scopeType === "organization"
|
|
3709
|
+
? await db
|
|
3710
|
+
.select({ id: heartbeatRuns.id })
|
|
3711
|
+
.from(heartbeatRuns)
|
|
3712
|
+
.where(and(eq(heartbeatRuns.orgId, scope.orgId), inArray(heartbeatRuns.status, ["queued", "running"])))
|
|
3713
|
+
.then((rows) => rows.map((row) => row.id))
|
|
3714
|
+
: await listProjectScopedRunIds(scope.orgId, scope.scopeId);
|
|
3715
|
+
for (const runId of runIds) {
|
|
3716
|
+
await cancelRunInternal(runId, "Cancelled due to budget pause");
|
|
3717
|
+
}
|
|
3718
|
+
await cancelPendingWakeupsForBudgetScope(scope);
|
|
3719
|
+
}
|
|
3720
|
+
async function retryRunInternal(runId, opts) {
|
|
3721
|
+
const run = await getRun(runId);
|
|
3722
|
+
if (!run)
|
|
3723
|
+
throw notFound("Heartbeat run not found");
|
|
3724
|
+
if (run.status !== "failed" && run.status !== "timed_out") {
|
|
3725
|
+
throw conflict("Only failed or timed out runs can be retried", {
|
|
3726
|
+
status: run.status,
|
|
3727
|
+
});
|
|
3728
|
+
}
|
|
3729
|
+
const agent = await getAgent(run.agentId);
|
|
3730
|
+
if (!agent)
|
|
3731
|
+
throw notFound("Agent not found");
|
|
3732
|
+
if (agent.status === "paused" ||
|
|
3733
|
+
agent.status === "terminated" ||
|
|
3734
|
+
agent.status === "pending_approval") {
|
|
3735
|
+
throw conflict("Agent is not invokable in its current state", { status: agent.status });
|
|
3736
|
+
}
|
|
3737
|
+
const policy = parseHeartbeatPolicy(agent);
|
|
3738
|
+
if (!policy.wakeOnDemand) {
|
|
3739
|
+
throw conflict("Agent is not configured for on-demand wakeups");
|
|
3740
|
+
}
|
|
3741
|
+
const context = parseObject(run.contextSnapshot);
|
|
3742
|
+
const issueId = readNonEmptyString(context.issueId);
|
|
3743
|
+
let projectId = readNonEmptyString(context.projectId);
|
|
3744
|
+
if (!projectId && issueId) {
|
|
3745
|
+
projectId = await db
|
|
3746
|
+
.select({ projectId: issues.projectId })
|
|
3747
|
+
.from(issues)
|
|
3748
|
+
.where(and(eq(issues.id, issueId), eq(issues.orgId, agent.orgId)))
|
|
3749
|
+
.then((rows) => rows[0]?.projectId ?? null);
|
|
3750
|
+
}
|
|
3751
|
+
const budgetBlock = await budgets.getInvocationBlock(agent.orgId, agent.id, {
|
|
3752
|
+
issueId,
|
|
3753
|
+
projectId,
|
|
3754
|
+
});
|
|
3755
|
+
if (budgetBlock) {
|
|
3756
|
+
throw conflict(budgetBlock.reason, {
|
|
3757
|
+
scopeType: budgetBlock.scopeType,
|
|
3758
|
+
scopeId: budgetBlock.scopeId,
|
|
3759
|
+
});
|
|
3760
|
+
}
|
|
3761
|
+
return enqueueRecoveryRun(run, agent, {
|
|
3762
|
+
recoveryTrigger: "manual",
|
|
3763
|
+
source: "on_demand",
|
|
3764
|
+
triggerDetail: "manual",
|
|
3765
|
+
wakeReason: "retry_failed_run",
|
|
3766
|
+
requestedByActorType: opts?.requestedByActorType ?? "user",
|
|
3767
|
+
requestedByActorId: opts?.requestedByActorId ?? null,
|
|
3768
|
+
now: opts?.now ?? new Date(),
|
|
3769
|
+
});
|
|
3770
|
+
}
|
|
3771
|
+
async function buildSkillAnalytics(scope, opts) {
|
|
3772
|
+
const now = opts?.now ?? new Date();
|
|
3773
|
+
const customDateKeys = opts?.startDate && opts?.endDate
|
|
3774
|
+
? buildDateKeysBetween(opts.startDate, opts.endDate).slice(0, 120)
|
|
3775
|
+
: [];
|
|
3776
|
+
const windowDays = customDateKeys.length > 0
|
|
3777
|
+
? customDateKeys.length
|
|
3778
|
+
: Math.max(1, Math.min(opts?.windowDays ?? 30, 90));
|
|
3779
|
+
const dateKeys = customDateKeys.length > 0
|
|
3780
|
+
? customDateKeys
|
|
3781
|
+
: buildRecentDateKeys(windowDays, now);
|
|
3782
|
+
const startDate = dateKeys[0];
|
|
3783
|
+
const endDate = dateKeys.at(-1);
|
|
3784
|
+
const windowStart = new Date(`${startDate}T00:00:00.000Z`);
|
|
3785
|
+
const windowEnd = new Date(`${endDate}T23:59:59.999Z`);
|
|
3786
|
+
const rows = await db
|
|
3787
|
+
.select({
|
|
3788
|
+
createdAt: heartbeatRunEvents.createdAt,
|
|
3789
|
+
payload: heartbeatRunEvents.payload,
|
|
3790
|
+
})
|
|
3791
|
+
.from(heartbeatRunEvents)
|
|
3792
|
+
.where(and(eq(heartbeatRunEvents.orgId, scope.orgId), ...(scope.agentId ? [eq(heartbeatRunEvents.agentId, scope.agentId)] : []), eq(heartbeatRunEvents.eventType, "adapter.invoke"), gte(heartbeatRunEvents.createdAt, windowStart), lte(heartbeatRunEvents.createdAt, windowEnd)))
|
|
3793
|
+
.orderBy(asc(heartbeatRunEvents.createdAt), asc(heartbeatRunEvents.id));
|
|
3794
|
+
const days = new Map();
|
|
3795
|
+
for (const date of dateKeys) {
|
|
3796
|
+
days.set(date, { totalCount: 0, runCount: 0, skills: new Map() });
|
|
3797
|
+
}
|
|
3798
|
+
const overallSkills = new Map();
|
|
3799
|
+
let totalCount = 0;
|
|
3800
|
+
let totalRunsWithSkills = 0;
|
|
3801
|
+
for (const row of rows) {
|
|
3802
|
+
const date = new Date(row.createdAt).toISOString().slice(0, 10);
|
|
3803
|
+
const bucket = days.get(date);
|
|
3804
|
+
if (!bucket)
|
|
3805
|
+
continue;
|
|
3806
|
+
const payload = parseObject(row.payload);
|
|
3807
|
+
const loadedSkills = Array.isArray(payload.loadedSkills) ? payload.loadedSkills : [];
|
|
3808
|
+
if (loadedSkills.length === 0)
|
|
3809
|
+
continue;
|
|
3810
|
+
const eventSkills = new Map();
|
|
3811
|
+
for (const entry of loadedSkills) {
|
|
3812
|
+
const normalized = normalizeLoadedSkill(entry);
|
|
3813
|
+
if (!normalized)
|
|
3814
|
+
continue;
|
|
3815
|
+
if (!eventSkills.has(normalized.key)) {
|
|
3816
|
+
eventSkills.set(normalized.key, normalized.label);
|
|
3817
|
+
}
|
|
3818
|
+
}
|
|
3819
|
+
if (eventSkills.size === 0)
|
|
3820
|
+
continue;
|
|
3821
|
+
bucket.runCount += 1;
|
|
3822
|
+
totalRunsWithSkills += 1;
|
|
3823
|
+
for (const [key, label] of eventSkills) {
|
|
3824
|
+
bucket.totalCount += 1;
|
|
3825
|
+
totalCount += 1;
|
|
3826
|
+
const existingDaySkill = bucket.skills.get(key);
|
|
3827
|
+
if (existingDaySkill) {
|
|
3828
|
+
existingDaySkill.count += 1;
|
|
3829
|
+
}
|
|
3830
|
+
else {
|
|
3831
|
+
bucket.skills.set(key, { key, label, count: 1 });
|
|
3832
|
+
}
|
|
3833
|
+
const existingOverallSkill = overallSkills.get(key);
|
|
3834
|
+
if (existingOverallSkill) {
|
|
3835
|
+
existingOverallSkill.count += 1;
|
|
3836
|
+
}
|
|
3837
|
+
else {
|
|
3838
|
+
overallSkills.set(key, { key, label, count: 1 });
|
|
3839
|
+
}
|
|
3840
|
+
}
|
|
3841
|
+
}
|
|
3842
|
+
return {
|
|
3843
|
+
agentId: scope.agentId ?? "__all__",
|
|
3844
|
+
orgId: scope.orgId,
|
|
3845
|
+
windowDays,
|
|
3846
|
+
startDate,
|
|
3847
|
+
endDate,
|
|
3848
|
+
totalCount,
|
|
3849
|
+
totalRunsWithSkills,
|
|
3850
|
+
skills: Array.from(overallSkills.values()).sort((left, right) => (right.count - left.count
|
|
3851
|
+
|| left.label.localeCompare(right.label)
|
|
3852
|
+
|| left.key.localeCompare(right.key))),
|
|
3853
|
+
days: dateKeys.map((date) => {
|
|
3854
|
+
const bucket = days.get(date);
|
|
3855
|
+
return {
|
|
3856
|
+
date,
|
|
3857
|
+
totalCount: bucket.totalCount,
|
|
3858
|
+
runCount: bucket.runCount,
|
|
3859
|
+
skills: Array.from(bucket.skills.values()).sort((left, right) => (right.count - left.count
|
|
3860
|
+
|| left.label.localeCompare(right.label)
|
|
3861
|
+
|| left.key.localeCompare(right.key))),
|
|
3862
|
+
};
|
|
3863
|
+
}),
|
|
3864
|
+
};
|
|
3865
|
+
}
|
|
3866
|
+
return {
|
|
3867
|
+
list: async (orgId, agentId, limit) => {
|
|
3868
|
+
const query = db
|
|
3869
|
+
.select(heartbeatRunListColumns)
|
|
3870
|
+
.from(heartbeatRuns)
|
|
3871
|
+
.where(agentId
|
|
3872
|
+
? and(eq(heartbeatRuns.orgId, orgId), eq(heartbeatRuns.agentId, agentId))
|
|
3873
|
+
: eq(heartbeatRuns.orgId, orgId))
|
|
3874
|
+
.orderBy(desc(heartbeatRuns.createdAt));
|
|
3875
|
+
const rows = limit ? await query.limit(limit) : await query;
|
|
3876
|
+
return rows.map((row) => ({
|
|
3877
|
+
...row,
|
|
3878
|
+
resultJson: summarizeHeartbeatRunResultJson(row.resultJson),
|
|
3879
|
+
}));
|
|
3880
|
+
},
|
|
3881
|
+
getAgentSkillAnalytics: async (agentId, opts) => {
|
|
3882
|
+
const agent = await getAgent(agentId);
|
|
3883
|
+
if (!agent)
|
|
3884
|
+
throw notFound("Agent not found");
|
|
3885
|
+
return buildSkillAnalytics({ orgId: agent.orgId, agentId: agent.id }, opts);
|
|
3886
|
+
},
|
|
3887
|
+
getOrganizationSkillAnalytics: async (orgId, opts) => {
|
|
3888
|
+
const org = await db
|
|
3889
|
+
.select({ id: organizations.id })
|
|
3890
|
+
.from(organizations)
|
|
3891
|
+
.where(eq(organizations.id, orgId))
|
|
3892
|
+
.limit(1)
|
|
3893
|
+
.then((rows) => rows[0] ?? null);
|
|
3894
|
+
if (!org)
|
|
3895
|
+
throw notFound("Organization not found");
|
|
3896
|
+
return buildSkillAnalytics({ orgId }, opts);
|
|
3897
|
+
},
|
|
3898
|
+
getRun,
|
|
3899
|
+
getRuntimeState: async (agentId) => {
|
|
3900
|
+
const state = await getRuntimeState(agentId);
|
|
3901
|
+
const agent = await getAgent(agentId);
|
|
3902
|
+
if (!agent)
|
|
3903
|
+
return null;
|
|
3904
|
+
const ensured = state ?? (await ensureRuntimeState(agent));
|
|
3905
|
+
const latestTaskSession = await db
|
|
3906
|
+
.select()
|
|
3907
|
+
.from(agentTaskSessions)
|
|
3908
|
+
.where(and(eq(agentTaskSessions.orgId, agent.orgId), eq(agentTaskSessions.agentId, agent.id)))
|
|
3909
|
+
.orderBy(desc(agentTaskSessions.updatedAt))
|
|
3910
|
+
.limit(1)
|
|
3911
|
+
.then((rows) => rows[0] ?? null);
|
|
3912
|
+
return {
|
|
3913
|
+
...ensured,
|
|
3914
|
+
sessionDisplayId: latestTaskSession?.sessionDisplayId ?? ensured.sessionId,
|
|
3915
|
+
sessionParamsJson: latestTaskSession?.sessionParamsJson ?? null,
|
|
3916
|
+
};
|
|
3917
|
+
},
|
|
3918
|
+
listTaskSessions: async (agentId) => {
|
|
3919
|
+
const agent = await getAgent(agentId);
|
|
3920
|
+
if (!agent)
|
|
3921
|
+
throw notFound("Agent not found");
|
|
3922
|
+
return db
|
|
3923
|
+
.select()
|
|
3924
|
+
.from(agentTaskSessions)
|
|
3925
|
+
.where(and(eq(agentTaskSessions.orgId, agent.orgId), eq(agentTaskSessions.agentId, agentId)))
|
|
3926
|
+
.orderBy(desc(agentTaskSessions.updatedAt), desc(agentTaskSessions.createdAt));
|
|
3927
|
+
},
|
|
3928
|
+
resetRuntimeSession: async (agentId, opts) => {
|
|
3929
|
+
const agent = await getAgent(agentId);
|
|
3930
|
+
if (!agent)
|
|
3931
|
+
throw notFound("Agent not found");
|
|
3932
|
+
await ensureRuntimeState(agent);
|
|
3933
|
+
const taskKey = readNonEmptyString(opts?.taskKey);
|
|
3934
|
+
const clearedTaskSessions = await clearTaskSessions(agent.orgId, agent.id, taskKey ? { taskKey, agentRuntimeType: agent.agentRuntimeType } : undefined);
|
|
3935
|
+
const runtimePatch = {
|
|
3936
|
+
sessionId: null,
|
|
3937
|
+
lastError: null,
|
|
3938
|
+
updatedAt: new Date(),
|
|
3939
|
+
};
|
|
3940
|
+
if (!taskKey) {
|
|
3941
|
+
runtimePatch.stateJson = {};
|
|
3942
|
+
}
|
|
3943
|
+
const updated = await db
|
|
3944
|
+
.update(agentRuntimeState)
|
|
3945
|
+
.set(runtimePatch)
|
|
3946
|
+
.where(eq(agentRuntimeState.agentId, agentId))
|
|
3947
|
+
.returning()
|
|
3948
|
+
.then((rows) => rows[0] ?? null);
|
|
3949
|
+
if (!updated)
|
|
3950
|
+
return null;
|
|
3951
|
+
return {
|
|
3952
|
+
...updated,
|
|
3953
|
+
sessionDisplayId: null,
|
|
3954
|
+
sessionParamsJson: null,
|
|
3955
|
+
clearedTaskSessions,
|
|
3956
|
+
};
|
|
3957
|
+
},
|
|
3958
|
+
listEvents: (runId, afterSeq = 0, limit = 200) => db
|
|
3959
|
+
.select()
|
|
3960
|
+
.from(heartbeatRunEvents)
|
|
3961
|
+
.where(and(eq(heartbeatRunEvents.runId, runId), gt(heartbeatRunEvents.seq, afterSeq)))
|
|
3962
|
+
.orderBy(asc(heartbeatRunEvents.seq))
|
|
3963
|
+
.limit(Math.max(1, Math.min(limit, 1000))),
|
|
3964
|
+
readLog: async (runId, opts) => {
|
|
3965
|
+
const run = await getRun(runId);
|
|
3966
|
+
if (!run)
|
|
3967
|
+
throw notFound("Heartbeat run not found");
|
|
3968
|
+
if (!run.logStore || !run.logRef)
|
|
3969
|
+
throw notFound("Run log not found");
|
|
3970
|
+
const result = await runLogStore.read({
|
|
3971
|
+
store: run.logStore,
|
|
3972
|
+
logRef: run.logRef,
|
|
3973
|
+
}, opts);
|
|
3974
|
+
return {
|
|
3975
|
+
runId,
|
|
3976
|
+
store: run.logStore,
|
|
3977
|
+
logRef: run.logRef,
|
|
3978
|
+
...result,
|
|
3979
|
+
content: redactCurrentUserText(result.content, await getCurrentUserRedactionOptions()),
|
|
3980
|
+
};
|
|
3981
|
+
},
|
|
3982
|
+
invoke: async (agentId, source = "on_demand", contextSnapshot = {}, triggerDetail = "manual", actor) => enqueueWakeup(agentId, {
|
|
3983
|
+
source,
|
|
3984
|
+
triggerDetail,
|
|
3985
|
+
contextSnapshot,
|
|
3986
|
+
requestedByActorType: actor?.actorType,
|
|
3987
|
+
requestedByActorId: actor?.actorId ?? null,
|
|
3988
|
+
}),
|
|
3989
|
+
wakeup: enqueueWakeup,
|
|
3990
|
+
resumeDeferredWakeupsForAgent,
|
|
3991
|
+
retryRun: retryRunInternal,
|
|
3992
|
+
reportRunActivity: clearDetachedRunWarning,
|
|
3993
|
+
reapOrphanedRuns,
|
|
3994
|
+
resumeQueuedRuns,
|
|
3995
|
+
tickTimers: async (now = new Date()) => {
|
|
3996
|
+
const allAgents = await db.select().from(agents);
|
|
3997
|
+
let checked = 0;
|
|
3998
|
+
let enqueued = 0;
|
|
3999
|
+
let skipped = 0;
|
|
4000
|
+
for (const agent of allAgents) {
|
|
4001
|
+
if (agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval")
|
|
4002
|
+
continue;
|
|
4003
|
+
const policy = parseHeartbeatPolicy(agent);
|
|
4004
|
+
if (!policy.enabled || policy.intervalSec <= 0)
|
|
4005
|
+
continue;
|
|
4006
|
+
checked += 1;
|
|
4007
|
+
const baseline = new Date(agent.lastHeartbeatAt ?? agent.createdAt).getTime();
|
|
4008
|
+
const elapsedMs = now.getTime() - baseline;
|
|
4009
|
+
if (elapsedMs < policy.intervalSec * 1000)
|
|
4010
|
+
continue;
|
|
4011
|
+
const run = await enqueueWakeup(agent.id, {
|
|
4012
|
+
source: "timer",
|
|
4013
|
+
triggerDetail: "system",
|
|
4014
|
+
reason: "heartbeat_timer",
|
|
4015
|
+
requestedByActorType: "system",
|
|
4016
|
+
requestedByActorId: "heartbeat_scheduler",
|
|
4017
|
+
contextSnapshot: {
|
|
4018
|
+
source: "scheduler",
|
|
4019
|
+
reason: "interval_elapsed",
|
|
4020
|
+
now: now.toISOString(),
|
|
4021
|
+
},
|
|
4022
|
+
});
|
|
4023
|
+
if (run)
|
|
4024
|
+
enqueued += 1;
|
|
4025
|
+
else
|
|
4026
|
+
skipped += 1;
|
|
4027
|
+
}
|
|
4028
|
+
return { checked, enqueued, skipped };
|
|
4029
|
+
},
|
|
4030
|
+
cancelRun: (runId) => cancelRunInternal(runId),
|
|
4031
|
+
cancelActiveForAgent: (agentId) => cancelActiveForAgentInternal(agentId),
|
|
4032
|
+
cancelBudgetScopeWork,
|
|
4033
|
+
getActiveRunForAgent: async (agentId) => {
|
|
4034
|
+
const [run] = await db
|
|
4035
|
+
.select()
|
|
4036
|
+
.from(heartbeatRuns)
|
|
4037
|
+
.where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "running")))
|
|
4038
|
+
.orderBy(desc(heartbeatRuns.startedAt))
|
|
4039
|
+
.limit(1);
|
|
4040
|
+
return run ?? null;
|
|
4041
|
+
},
|
|
4042
|
+
};
|
|
4043
|
+
}
|
|
4044
|
+
//# sourceMappingURL=orchestrator.js.map
|