@penclipai/server 2026.426.0 → 2026.505.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/builtin-adapter-types.d.ts.map +1 -1
- package/dist/adapters/builtin-adapter-types.js +3 -0
- package/dist/adapters/builtin-adapter-types.js.map +1 -1
- package/dist/adapters/index.d.ts +2 -2
- package/dist/adapters/index.d.ts.map +1 -1
- package/dist/adapters/index.js +1 -1
- package/dist/adapters/index.js.map +1 -1
- package/dist/adapters/registry.d.ts +2 -1
- package/dist/adapters/registry.d.ts.map +1 -1
- package/dist/adapters/registry.js +76 -6
- package/dist/adapters/registry.js.map +1 -1
- package/dist/adapters/types.d.ts +1 -1
- package/dist/adapters/types.d.ts.map +1 -1
- package/dist/adapters/utils.d.ts.map +1 -1
- package/dist/adapters/utils.js +2 -1
- package/dist/adapters/utils.js.map +1 -1
- package/dist/attachment-types.d.ts +1 -16
- package/dist/attachment-types.d.ts.map +1 -1
- package/dist/attachment-types.js +7 -0
- package/dist/attachment-types.js.map +1 -1
- package/dist/auth/better-auth.d.ts +3 -1
- package/dist/auth/better-auth.d.ts.map +1 -1
- package/dist/auth/better-auth.js +8 -2
- package/dist/auth/better-auth.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +27 -13
- package/dist/index.js.map +1 -1
- package/dist/middleware/auth.d.ts.map +1 -1
- package/dist/middleware/auth.js +143 -2
- package/dist/middleware/auth.js.map +1 -1
- package/dist/onboarding-assets/ceo/AGENTS.md +1 -1
- package/dist/onboarding-assets/ceo/HEARTBEAT.md +5 -5
- package/dist/redaction.d.ts.map +1 -1
- package/dist/redaction.js +30 -12
- package/dist/redaction.js.map +1 -1
- package/dist/routes/access.d.ts.map +1 -1
- package/dist/routes/access.js +10 -0
- package/dist/routes/access.js.map +1 -1
- package/dist/routes/activity.d.ts.map +1 -1
- package/dist/routes/activity.js +4 -2
- package/dist/routes/activity.js.map +1 -1
- package/dist/routes/adapters.d.ts.map +1 -1
- package/dist/routes/adapters.js +1 -0
- package/dist/routes/adapters.js.map +1 -1
- package/dist/routes/agents.d.ts.map +1 -1
- package/dist/routes/agents.js +317 -56
- package/dist/routes/agents.js.map +1 -1
- package/dist/routes/costs.d.ts.map +1 -1
- package/dist/routes/costs.js +21 -2
- package/dist/routes/costs.js.map +1 -1
- package/dist/routes/instance-settings.d.ts.map +1 -1
- package/dist/routes/instance-settings.js +37 -2
- package/dist/routes/instance-settings.js.map +1 -1
- package/dist/routes/issue-tree-control.d.ts.map +1 -1
- package/dist/routes/issue-tree-control.js +3 -1
- package/dist/routes/issue-tree-control.js.map +1 -1
- package/dist/routes/issues.d.ts.map +1 -1
- package/dist/routes/issues.js +257 -32
- package/dist/routes/issues.js.map +1 -1
- package/dist/routes/projects.d.ts.map +1 -1
- package/dist/routes/projects.js +10 -3
- package/dist/routes/projects.js.map +1 -1
- package/dist/routes/routines.d.ts.map +1 -1
- package/dist/routes/routines.js +6 -1
- package/dist/routes/routines.js.map +1 -1
- package/dist/routes/workspace-command-authz.d.ts +1 -1
- package/dist/routes/workspace-command-authz.d.ts.map +1 -1
- package/dist/routes/workspace-command-authz.js +2 -2
- package/dist/routes/workspace-command-authz.js.map +1 -1
- package/dist/runtime-api.d.ts +4 -0
- package/dist/runtime-api.d.ts.map +1 -1
- package/dist/runtime-api.js +38 -10
- package/dist/runtime-api.js.map +1 -1
- package/dist/services/companies.d.ts +6 -0
- package/dist/services/companies.d.ts.map +1 -1
- package/dist/services/companies.js +1 -0
- package/dist/services/companies.js.map +1 -1
- package/dist/services/company-portability.d.ts.map +1 -1
- package/dist/services/company-portability.js +16 -15
- package/dist/services/company-portability.js.map +1 -1
- package/dist/services/costs.d.ts +9 -0
- package/dist/services/costs.d.ts.map +1 -1
- package/dist/services/costs.js +45 -1
- package/dist/services/costs.js.map +1 -1
- package/dist/services/environment-execution-target.d.ts.map +1 -1
- package/dist/services/environment-execution-target.js +7 -13
- package/dist/services/environment-execution-target.js.map +1 -1
- package/dist/services/environment-run-orchestrator.d.ts.map +1 -1
- package/dist/services/environment-run-orchestrator.js +56 -0
- package/dist/services/environment-run-orchestrator.js.map +1 -1
- package/dist/services/environment-runtime.d.ts +2 -0
- package/dist/services/environment-runtime.d.ts.map +1 -1
- package/dist/services/environment-runtime.js +80 -39
- package/dist/services/environment-runtime.js.map +1 -1
- package/dist/services/heartbeat-stop-metadata.d.ts +2 -1
- package/dist/services/heartbeat-stop-metadata.d.ts.map +1 -1
- package/dist/services/heartbeat-stop-metadata.js +10 -1
- package/dist/services/heartbeat-stop-metadata.js.map +1 -1
- package/dist/services/heartbeat-stop-metadata.test.js +24 -0
- package/dist/services/heartbeat-stop-metadata.test.js.map +1 -1
- package/dist/services/heartbeat.d.ts +156 -5
- package/dist/services/heartbeat.d.ts.map +1 -1
- package/dist/services/heartbeat.js +1384 -112
- package/dist/services/heartbeat.js.map +1 -1
- package/dist/services/index.d.ts +1 -0
- package/dist/services/index.d.ts.map +1 -1
- package/dist/services/index.js +1 -0
- package/dist/services/index.js.map +1 -1
- package/dist/services/instance-settings.d.ts.map +1 -1
- package/dist/services/instance-settings.js +4 -1
- package/dist/services/instance-settings.js.map +1 -1
- package/dist/services/issue-execution-policy.d.ts +56 -1
- package/dist/services/issue-execution-policy.d.ts.map +1 -1
- package/dist/services/issue-execution-policy.js +400 -2
- package/dist/services/issue-execution-policy.js.map +1 -1
- package/dist/services/issue-thread-interactions.d.ts +5 -1
- package/dist/services/issue-thread-interactions.d.ts.map +1 -1
- package/dist/services/issue-thread-interactions.js +44 -1
- package/dist/services/issue-thread-interactions.js.map +1 -1
- package/dist/services/issue-tree-control.d.ts +1 -0
- package/dist/services/issue-tree-control.d.ts.map +1 -1
- package/dist/services/issue-tree-control.js +84 -4
- package/dist/services/issue-tree-control.js.map +1 -1
- package/dist/services/issues.d.ts +10 -1
- package/dist/services/issues.d.ts.map +1 -1
- package/dist/services/issues.js +452 -48
- package/dist/services/issues.js.map +1 -1
- package/dist/services/plugin-environment-driver.d.ts +4 -0
- package/dist/services/plugin-environment-driver.d.ts.map +1 -1
- package/dist/services/plugin-environment-driver.js +18 -1
- package/dist/services/plugin-environment-driver.js.map +1 -1
- package/dist/services/productivity-review.d.ts +83 -0
- package/dist/services/productivity-review.d.ts.map +1 -0
- package/dist/services/productivity-review.js +650 -0
- package/dist/services/productivity-review.js.map +1 -0
- package/dist/services/recovery/index.d.ts +1 -1
- package/dist/services/recovery/index.d.ts.map +1 -1
- package/dist/services/recovery/index.js +1 -1
- package/dist/services/recovery/index.js.map +1 -1
- package/dist/services/recovery/issue-graph-liveness.d.ts +13 -1
- package/dist/services/recovery/issue-graph-liveness.d.ts.map +1 -1
- package/dist/services/recovery/issue-graph-liveness.js +212 -92
- package/dist/services/recovery/issue-graph-liveness.js.map +1 -1
- package/dist/services/recovery/origins.d.ts +2 -0
- package/dist/services/recovery/origins.d.ts.map +1 -1
- package/dist/services/recovery/origins.js +4 -0
- package/dist/services/recovery/origins.js.map +1 -1
- package/dist/services/recovery/run-liveness-continuations.d.ts.map +1 -1
- package/dist/services/recovery/run-liveness-continuations.js.map +1 -1
- package/dist/services/recovery/service.d.ts +20 -2
- package/dist/services/recovery/service.d.ts.map +1 -1
- package/dist/services/recovery/service.js +405 -63
- package/dist/services/recovery/service.js.map +1 -1
- package/dist/services/routines.d.ts +5 -2
- package/dist/services/routines.d.ts.map +1 -1
- package/dist/services/routines.js +47 -3
- package/dist/services/routines.js.map +1 -1
- package/dist/worktree-config.d.ts.map +1 -1
- package/dist/worktree-config.js +2 -5
- package/dist/worktree-config.js.map +1 -1
- package/package.json +16 -15
- package/skills/diagnose-why-work-stopped/SKILL.md +161 -0
- package/skills/paperclip/SKILL.md +37 -26
- package/skills/paperclip/references/api-reference.md +6 -2
- package/skills/paperclip-converting-plans-to-tasks/SKILL.md +42 -0
- package/skills/paperclip-create-agent/SKILL.md +3 -2
- package/skills/paperclip-create-agent/references/agent-instruction-templates.md +1 -1
- package/skills/paperclip-create-agent/references/api-reference.md +7 -2
- package/skills/paperclip-create-agent/references/baseline-role-guide.md +1 -1
- package/skills/paperclip-create-agent/references/draft-review-checklist.md +2 -2
- package/skills/paperclip-dev/SKILL.md +267 -0
- package/skills/terminal-bench-loop/SKILL.md +236 -0
- package/ui-dist/assets/{_basePickBy-BRqa7PJ5.js → _basePickBy-BS0Fg_DB.js} +1 -1
- package/ui-dist/assets/{_baseUniq-DhE2yrXC.js → _baseUniq-Dtnt_4SE.js} +1 -1
- package/ui-dist/assets/{arc-7qnikTQ3.js → arc-BCoOPxh5.js} +1 -1
- package/ui-dist/assets/{architectureDiagram-VXUJARFQ-CH0wVUOM.js → architectureDiagram-VXUJARFQ-C6eX2QUo.js} +1 -1
- package/ui-dist/assets/{blockDiagram-VD42YOAC-CeeRyJQX.js → blockDiagram-VD42YOAC-aUueUD4B.js} +1 -1
- package/ui-dist/assets/browser-ponyfill-BlAfsWm_.js +2 -0
- package/ui-dist/assets/{c4Diagram-YG6GDRKO-C_cV0CGo.js → c4Diagram-YG6GDRKO-CfPWRlOF.js} +1 -1
- package/ui-dist/assets/channel-ChNSCFJf.js +1 -0
- package/ui-dist/assets/{chunk-4BX2VUAB-DQ6pxPVT.js → chunk-4BX2VUAB-BTD1apA4.js} +1 -1
- package/ui-dist/assets/{chunk-55IACEB6-L8pS0IoX.js → chunk-55IACEB6-BXXF_ClN.js} +1 -1
- package/ui-dist/assets/{chunk-B4BG7PRW-BZKGE88E.js → chunk-B4BG7PRW-hAZeWGP8.js} +1 -1
- package/ui-dist/assets/{chunk-DI55MBZ5-CefSoZ_K.js → chunk-DI55MBZ5-cOH3UoEl.js} +1 -1
- package/ui-dist/assets/{chunk-FMBD7UC4-Bc3qTTHB.js → chunk-FMBD7UC4-Cu2yZOcl.js} +1 -1
- package/ui-dist/assets/{chunk-QN33PNHL-CjWBr5bI.js → chunk-QN33PNHL-0DNN5aRU.js} +1 -1
- package/ui-dist/assets/{chunk-QZHKN3VN-C0JUdmmz.js → chunk-QZHKN3VN-B9_bhK2n.js} +1 -1
- package/ui-dist/assets/{chunk-TZMSLE5B-D4d4I82z.js → chunk-TZMSLE5B-Cr5xwxio.js} +1 -1
- package/ui-dist/assets/classDiagram-2ON5EDUG-4aK1QZU3.js +1 -0
- package/ui-dist/assets/classDiagram-v2-WZHVMYZB-4aK1QZU3.js +1 -0
- package/ui-dist/assets/clone-C8lk5Qbc.js +1 -0
- package/ui-dist/assets/{cose-bilkent-S5V4N54A-B09h9XGZ.js → cose-bilkent-S5V4N54A-6_Dw6gpQ.js} +1 -1
- package/ui-dist/assets/{dagre-6UL2VRFP-CA02PXuX.js → dagre-6UL2VRFP-CFBhlh5H.js} +1 -1
- package/ui-dist/assets/{diagram-PSM6KHXK-DaT9cnrY.js → diagram-PSM6KHXK-C88ftcah.js} +1 -1
- package/ui-dist/assets/{diagram-QEK2KX5R-Drwc3gBw.js → diagram-QEK2KX5R-9EUupcuH.js} +1 -1
- package/ui-dist/assets/{diagram-S2PKOQOG-CpsGCaT6.js → diagram-S2PKOQOG-Dsml0wWh.js} +1 -1
- package/ui-dist/assets/{erDiagram-Q2GNP2WA-CVkBh9TY.js → erDiagram-Q2GNP2WA-sM-XdfHS.js} +1 -1
- package/ui-dist/assets/{flowDiagram-NV44I4VS-De9sXvPR.js → flowDiagram-NV44I4VS-qll7oaoW.js} +1 -1
- package/ui-dist/assets/{ganttDiagram-JELNMOA3-CSFa0gXS.js → ganttDiagram-JELNMOA3-VWnJMcjC.js} +1 -1
- package/ui-dist/assets/{gitGraphDiagram-V2S2FVAM-DEJaChxa.js → gitGraphDiagram-V2S2FVAM-DFnocrfl.js} +1 -1
- package/ui-dist/assets/{graph-D2R4DCtu.js → graph-nq3Qye4Z.js} +1 -1
- package/ui-dist/assets/{index-DEG-9CFs.js → index-3Owzaheh.js} +1 -1
- package/ui-dist/assets/{index-DHnKx9xX.js → index-B2A-a635.js} +1 -1
- package/ui-dist/assets/{index-C1I0SGDm.js → index-BGFrRiqa.js} +1 -1
- package/ui-dist/assets/{index-B44EtLRv.js → index-BVC5UhRK.js} +1 -1
- package/ui-dist/assets/{index-C_dAXwxT.js → index-BrP1U_Hy.js} +1 -1
- package/ui-dist/assets/{index-flZjKn_n.js → index-CXXHGqM8.js} +1 -1
- package/ui-dist/assets/{index-ssM_UKPW.js → index-CgyPAauR.js} +1 -1
- package/ui-dist/assets/{index-Ct1AraKR.js → index-CksQ4Ytv.js} +1 -1
- package/ui-dist/assets/{index-DQ6I_vpd.js → index-CrNzj2vZ.js} +1 -1
- package/ui-dist/assets/{index-DzZID5RY.js → index-CxbZBH3M.js} +1 -1
- package/ui-dist/assets/{index-Cn6_RRY5.js → index-D-dSSrf-.js} +1 -1
- package/ui-dist/assets/{index-CVa2OHgx.js → index-D6uZ_7Vh.js} +1 -1
- package/ui-dist/assets/{index-BzjWQd50.js → index-D7JGmxas.js} +1 -1
- package/ui-dist/assets/{index-CnT1_9UF.js → index-DDqO9GAq.js} +1 -1
- package/ui-dist/assets/index-DEUtmlPm.js +513 -0
- package/ui-dist/assets/{index-D2fEhyQg.js → index-DF5RDSoK.js} +1 -1
- package/ui-dist/assets/{index-CZGNe8K3.js → index-DfI92epU.js} +1 -1
- package/ui-dist/assets/{index-ByamXtyB.js → index-Dukb9MDQ.js} +1 -1
- package/ui-dist/assets/index-HP73_6Vr.css +1 -0
- package/ui-dist/assets/{index-BJS4rvUh.js → index-NXDTW2n4.js} +1 -1
- package/ui-dist/assets/{index-Bad5Hy7e.js → index-SxPPG9ig.js} +1 -1
- package/ui-dist/assets/{index-CC51mhhA.js → index-lC4Yz3Gw.js} +1 -1
- package/ui-dist/assets/{index-BFzkl36p.js → index-q2RXGI2V.js} +1 -1
- package/ui-dist/assets/{index-40icqWwg.js → index-qjfdrS96.js} +1 -1
- package/ui-dist/assets/{infoDiagram-HS3SLOUP-CJcjzWkM.js → infoDiagram-HS3SLOUP-CTrK5xoS.js} +1 -1
- package/ui-dist/assets/{journeyDiagram-XKPGCS4Q-ByITI00s.js → journeyDiagram-XKPGCS4Q-YFC7FykG.js} +1 -1
- package/ui-dist/assets/{kanban-definition-3W4ZIXB7-DvEjKke-.js → kanban-definition-3W4ZIXB7-B3dlyva0.js} +1 -1
- package/ui-dist/assets/{layout-CZcd66hi.js → layout-DefunPTK.js} +1 -1
- package/ui-dist/assets/{linear-jTUy3iHu.js → linear-CIPvzeMv.js} +1 -1
- package/ui-dist/assets/{mermaid.core-DECSZPbJ.js → mermaid.core-zKYhmnnR.js} +4 -4
- package/ui-dist/assets/{mindmap-definition-VGOIOE7T-Twtu17_c.js → mindmap-definition-VGOIOE7T-BlU-ebRa.js} +1 -1
- package/ui-dist/assets/{pieDiagram-ADFJNKIX-DlbgZ010.js → pieDiagram-ADFJNKIX-Ceto4LXH.js} +1 -1
- package/ui-dist/assets/{quadrantDiagram-AYHSOK5B-CMAa3qAT.js → quadrantDiagram-AYHSOK5B-C6M6hkuE.js} +1 -1
- package/ui-dist/assets/{requirementDiagram-UZGBJVZJ-CXRTfJOe.js → requirementDiagram-UZGBJVZJ-B-bcG938.js} +1 -1
- package/ui-dist/assets/{sankeyDiagram-TZEHDZUN-DeyO4fer.js → sankeyDiagram-TZEHDZUN-CIqty6Qi.js} +1 -1
- package/ui-dist/assets/{sequenceDiagram-WL72ISMW-Ch8wlJIL.js → sequenceDiagram-WL72ISMW-CIt2R5tk.js} +1 -1
- package/ui-dist/assets/{stateDiagram-FKZM4ZOC-BgL_AAl9.js → stateDiagram-FKZM4ZOC-BC1RFlfg.js} +1 -1
- package/ui-dist/assets/stateDiagram-v2-4FDKWEC3-Iy6tYSSw.js +1 -0
- package/ui-dist/assets/{timeline-definition-IT6M3QCI-D1QWd7TQ.js → timeline-definition-IT6M3QCI-DZqvoU94.js} +1 -1
- package/ui-dist/assets/{treemap-GDKQZRPO-B5RkmUv8.js → treemap-GDKQZRPO-CSeKauwA.js} +1 -1
- package/ui-dist/assets/{xychartDiagram-PRI3JC2R-WtDhjZfk.js → xychartDiagram-PRI3JC2R-Ut3mCiEd.js} +1 -1
- package/ui-dist/index.html +2 -2
- package/ui-dist/locales/en/common.json +137 -1
- package/ui-dist/locales/zh-CN/common.json +111 -1
- package/ui-dist/assets/browser-ponyfill-Ct3hGqsr.js +0 -2
- package/ui-dist/assets/channel-pHFjGZL-.js +0 -1
- package/ui-dist/assets/classDiagram-2ON5EDUG-X4ZksqXl.js +0 -1
- package/ui-dist/assets/classDiagram-v2-WZHVMYZB-X4ZksqXl.js +0 -1
- package/ui-dist/assets/clone-DZzimpfG.js +0 -1
- package/ui-dist/assets/index-C1oE3J7o.css +0 -1
- package/ui-dist/assets/index-fSIlEIHr.js +0 -510
- package/ui-dist/assets/stateDiagram-v2-4FDKWEC3-gnLzrhSv.js +0 -1
|
@@ -3,14 +3,14 @@ import path from "node:path";
|
|
|
3
3
|
import { execFile as execFileCallback } from "node:child_process";
|
|
4
4
|
import { promisify } from "node:util";
|
|
5
5
|
import { randomUUID } from "node:crypto";
|
|
6
|
-
import { and, asc, desc, eq, getTableColumns, gt, inArray, isNull, lte, notInArray, or, sql } from "drizzle-orm";
|
|
7
|
-
import { AGENT_DEFAULT_MAX_CONCURRENT_RUNS, ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY, } from "@penclipai/shared";
|
|
6
|
+
import { and, asc, desc, eq, getTableColumns, gt, inArray, isNull, lt, lte, notInArray, or, sql } from "drizzle-orm";
|
|
7
|
+
import { AGENT_DEFAULT_MAX_CONCURRENT_RUNS, ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY, MODEL_PROFILE_KEYS, } from "@penclipai/shared";
|
|
8
8
|
import { agents, agentRuntimeState, agentTaskSessions, agentWakeupRequests, activityLog, companySkills as companySkillsTable, documentRevisions, issueDocuments, heartbeatRunEvents, heartbeatRuns, issueComments, issueRelations, issues, issueWorkProducts, projects, projectWorkspaces, workspaceOperations, } from "@penclipai/db";
|
|
9
9
|
import { conflict, HttpError, notFound } from "../errors.js";
|
|
10
10
|
import { logger } from "../middleware/logger.js";
|
|
11
11
|
import { publishLiveEvent } from "./live-events.js";
|
|
12
12
|
import { getRunLogStore } from "./run-log-store.js";
|
|
13
|
-
import { getServerAdapter, runningProcesses } from "../adapters/index.js";
|
|
13
|
+
import { getServerAdapter, listAdapterModelProfiles, runningProcesses } from "../adapters/index.js";
|
|
14
14
|
import { createLocalAgentJwt } from "../agent-auth-jwt.js";
|
|
15
15
|
import { parseObject, asBoolean, asNumber, appendWithByteCap, MAX_EXCERPT_BYTES } from "../adapters/utils.js";
|
|
16
16
|
import { costService } from "./costs.js";
|
|
@@ -21,11 +21,12 @@ import { budgetService } from "./budgets.js";
|
|
|
21
21
|
import { secretService } from "./secrets.js";
|
|
22
22
|
import { resolveDefaultAgentWorkspaceDir, resolveManagedProjectWorkspaceDir } from "../home-paths.js";
|
|
23
23
|
import { buildHeartbeatRunIssueComment, HEARTBEAT_RUN_RESULT_OUTPUT_MAX_CHARS, HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS, HEARTBEAT_RUN_SAFE_RESULT_JSON_MAX_BYTES, mergeHeartbeatRunResultJson, } from "./heartbeat-run-summary.js";
|
|
24
|
-
import { buildHeartbeatRunStopMetadata, mergeHeartbeatRunStopMetadata, } from "./heartbeat-stop-metadata.js";
|
|
24
|
+
import { buildHeartbeatRunStopMetadata, mergeHeartbeatRunStopMetadata, normalizeMaxTurnStopReason, } from "./heartbeat-stop-metadata.js";
|
|
25
25
|
import { classifyRunLiveness, } from "./run-liveness.js";
|
|
26
26
|
import { logActivity, publishPluginDomainEvent } from "./activity-log.js";
|
|
27
27
|
import { buildWorkspaceReadyComment, cleanupExecutionWorkspaceArtifacts, ensureRuntimeServicesForRun, persistAdapterManagedRuntimeServices, realizeExecutionWorkspace, releaseRuntimeServicesForRun, sanitizeRuntimeServiceBaseEnv, } from "./workspace-runtime.js";
|
|
28
28
|
import { issueService } from "./issues.js";
|
|
29
|
+
import { buildIssueMonitorClearedPatch, buildIssueMonitorTriggeredPatch, normalizeIssueExecutionPolicy, parseIssueExecutionState, } from "./issue-execution-policy.js";
|
|
29
30
|
import { ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS, isVerifiedIssueTreeControlInteractionWake, issueTreeControlService, } from "./issue-tree-control.js";
|
|
30
31
|
import { getIssueContinuationSummaryDocument, refreshIssueContinuationSummary, } from "./issue-continuation-summary.js";
|
|
31
32
|
import { executionWorkspaceService, mergeExecutionWorkspaceConfig } from "./execution-workspaces.js";
|
|
@@ -35,11 +36,13 @@ import { resolveRuntimeLocalizationPrompt, } from "./agent-runtime-localization.
|
|
|
35
36
|
import { canCoalesceWithRunLocale, materializeRuntimeUiLocaleContextSnapshot, resolveContextRuntimeUiLocale, } from "./heartbeat-runtime-locale.js";
|
|
36
37
|
import { buildExecutionWorkspaceAdapterConfig, gateProjectExecutionWorkspacePolicy, issueExecutionWorkspaceModeForPersistedWorkspace, parseIssueExecutionWorkspaceSettings, parseProjectExecutionWorkspacePolicy, resolveExecutionWorkspaceEnvironmentId, resolveExecutionWorkspaceMode, } from "./execution-workspace-policy.js";
|
|
37
38
|
import { instanceSettingsService } from "./instance-settings.js";
|
|
38
|
-
import { RUN_LIVENESS_CONTINUATION_REASON, buildRunLivenessContinuationIdempotencyKey, decideRunLivenessContinuation, findExistingRunLivenessContinuationWake, readContinuationAttempt, } from "./recovery/index.js";
|
|
39
|
+
import { RECOVERY_ORIGIN_KINDS, RUN_LIVENESS_CONTINUATION_REASON, buildRunLivenessContinuationIdempotencyKey, decideRunLivenessContinuation, findExistingRunLivenessContinuationWake, readContinuationAttempt, } from "./recovery/index.js";
|
|
39
40
|
import { isAutomaticRecoverySuppressedByPauseHold } from "./recovery/pause-hold-guard.js";
|
|
40
41
|
import { recoveryService } from "./recovery/service.js";
|
|
42
|
+
import { productivityReviewService } from "./productivity-review.js";
|
|
41
43
|
import { withAgentStartLock } from "./agent-start-lock.js";
|
|
42
44
|
import { redactCurrentUserText, redactCurrentUserValue } from "../log-redaction.js";
|
|
45
|
+
import { redactEventPayload } from "../redaction.js";
|
|
43
46
|
import { hasSessionCompactionThresholds, resolveSessionCompactionPolicy, } from "@penclipai/adapter-utils";
|
|
44
47
|
import { readPaperclipSkillSyncPreference, writePaperclipSkillSyncPreference, } from "@penclipai/adapter-utils/server-utils";
|
|
45
48
|
import { extractSkillMentionIds } from "@penclipai/shared";
|
|
@@ -53,7 +56,8 @@ const MAX_RUN_EVENT_PAYLOAD_ARRAY_ITEMS = 50;
|
|
|
53
56
|
const MAX_RUN_EVENT_PAYLOAD_OBJECT_KEYS = 100;
|
|
54
57
|
const MAX_RUN_EVENT_PAYLOAD_DEPTH = 6;
|
|
55
58
|
const HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT = AGENT_DEFAULT_MAX_CONCURRENT_RUNS;
|
|
56
|
-
const
|
|
59
|
+
const HEARTBEAT_MAX_CONCURRENT_RUNS_MIN = 1;
|
|
60
|
+
const HEARTBEAT_MAX_CONCURRENT_RUNS_MAX = 50;
|
|
57
61
|
const LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS = [
|
|
58
62
|
"environment.lease_acquired",
|
|
59
63
|
"environment.lease_released",
|
|
@@ -85,6 +89,13 @@ const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_JITTER_RATIO = 0.25;
|
|
|
85
89
|
const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON = "transient_failure";
|
|
86
90
|
const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry";
|
|
87
91
|
const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length;
|
|
92
|
+
export const MAX_TURN_CONTINUATION_RETRY_REASON = "max_turns_continuation";
|
|
93
|
+
export const MAX_TURN_CONTINUATION_WAKE_REASON = "max_turns_continuation_retry";
|
|
94
|
+
const MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS = 2;
|
|
95
|
+
const MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP = 10;
|
|
96
|
+
const MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS = 1_000;
|
|
97
|
+
const MAX_TURN_CONTINUATION_MAX_DELAY_MS = 5 * 60 * 1000;
|
|
98
|
+
const MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES = ["scheduled_retry", "queued", "running"];
|
|
88
99
|
function resolveCodexTransientFallbackMode(attempt) {
|
|
89
100
|
if (attempt <= 1)
|
|
90
101
|
return "same_session";
|
|
@@ -104,6 +115,11 @@ function readHeartbeatRunErrorFamily(run) {
|
|
|
104
115
|
}
|
|
105
116
|
return null;
|
|
106
117
|
}
|
|
118
|
+
function isMaxTurnExhaustionRun(run) {
|
|
119
|
+
const resultJson = parseObject(run.resultJson);
|
|
120
|
+
return Boolean(normalizeMaxTurnStopReason(resultJson.stopReason) ??
|
|
121
|
+
normalizeMaxTurnStopReason(run.errorCode));
|
|
122
|
+
}
|
|
107
123
|
function readTransientRetryNotBeforeFromRun(run) {
|
|
108
124
|
const resultJson = parseObject(run.resultJson);
|
|
109
125
|
const value = resultJson.retryNotBefore ?? resultJson.transientRetryNotBefore;
|
|
@@ -547,6 +563,8 @@ const heartbeatRunIssueSummaryColumns = {
|
|
|
547
563
|
status: heartbeatRuns.status,
|
|
548
564
|
invocationSource: heartbeatRuns.invocationSource,
|
|
549
565
|
triggerDetail: heartbeatRuns.triggerDetail,
|
|
566
|
+
contextCommentId: sql `${heartbeatRuns.contextSnapshot} ->> 'commentId'`.as("contextCommentId"),
|
|
567
|
+
contextWakeCommentId: sql `${heartbeatRuns.contextSnapshot} ->> 'wakeCommentId'`.as("contextWakeCommentId"),
|
|
550
568
|
startedAt: heartbeatRuns.startedAt,
|
|
551
569
|
finishedAt: heartbeatRuns.finishedAt,
|
|
552
570
|
createdAt: heartbeatRuns.createdAt,
|
|
@@ -653,7 +671,7 @@ function normalizeMaxConcurrentRuns(value) {
|
|
|
653
671
|
const parsed = Math.floor(asNumber(value, HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT));
|
|
654
672
|
if (!Number.isFinite(parsed))
|
|
655
673
|
return HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT;
|
|
656
|
-
return Math.max(
|
|
674
|
+
return Math.max(HEARTBEAT_MAX_CONCURRENT_RUNS_MIN, Math.min(HEARTBEAT_MAX_CONCURRENT_RUNS_MAX, parsed));
|
|
657
675
|
}
|
|
658
676
|
export function prioritizeProjectWorkspaceCandidatesForRun(rows, preferredWorkspaceId) {
|
|
659
677
|
if (!preferredWorkspaceId)
|
|
@@ -666,6 +684,113 @@ export function prioritizeProjectWorkspaceCandidatesForRun(rows, preferredWorksp
|
|
|
666
684
|
function readNonEmptyString(value) {
|
|
667
685
|
return typeof value === "string" && value.trim().length > 0 ? value : null;
|
|
668
686
|
}
|
|
687
|
+
function readModelProfileKey(value) {
|
|
688
|
+
return MODEL_PROFILE_KEYS.includes(value)
|
|
689
|
+
? value
|
|
690
|
+
: null;
|
|
691
|
+
}
|
|
692
|
+
function readContextModelProfile(contextSnapshot) {
|
|
693
|
+
return readModelProfileKey(contextSnapshot?.modelProfile);
|
|
694
|
+
}
|
|
695
|
+
export function normalizeModelProfileWakeContext(input) {
|
|
696
|
+
const modelProfileFromPayload = readModelProfileKey(input.payload?.modelProfile);
|
|
697
|
+
if (!readContextModelProfile(input.contextSnapshot) && modelProfileFromPayload) {
|
|
698
|
+
input.contextSnapshot.modelProfile = modelProfileFromPayload;
|
|
699
|
+
}
|
|
700
|
+
return input.contextSnapshot;
|
|
701
|
+
}
|
|
702
|
+
function readAgentRuntimeModelProfile(runtimeConfig, key) {
|
|
703
|
+
const modelProfiles = parseObject(parseObject(runtimeConfig).modelProfiles);
|
|
704
|
+
const profile = parseObject(modelProfiles[key]);
|
|
705
|
+
if (Object.keys(profile).length === 0) {
|
|
706
|
+
return { enabled: true, adapterConfig: {}, configured: false };
|
|
707
|
+
}
|
|
708
|
+
return {
|
|
709
|
+
enabled: profile.enabled !== false,
|
|
710
|
+
adapterConfig: parseObject(profile.adapterConfig),
|
|
711
|
+
configured: true,
|
|
712
|
+
};
|
|
713
|
+
}
|
|
714
|
+
export function resolveModelProfileApplication(input) {
|
|
715
|
+
const issueModelProfile = input.issueModelProfile ?? null;
|
|
716
|
+
const contextModelProfile = readContextModelProfile(input.contextSnapshot);
|
|
717
|
+
const requested = issueModelProfile ?? contextModelProfile;
|
|
718
|
+
const requestedBy = issueModelProfile
|
|
719
|
+
? "issue_override"
|
|
720
|
+
: contextModelProfile
|
|
721
|
+
? "wake_context"
|
|
722
|
+
: null;
|
|
723
|
+
if (!requested) {
|
|
724
|
+
return {
|
|
725
|
+
requested: null,
|
|
726
|
+
requestedBy: null,
|
|
727
|
+
applied: null,
|
|
728
|
+
configSource: null,
|
|
729
|
+
fallbackReason: null,
|
|
730
|
+
adapterConfig: null,
|
|
731
|
+
};
|
|
732
|
+
}
|
|
733
|
+
const adapterProfile = input.adapterModelProfiles.find((profile) => profile.key === requested) ?? null;
|
|
734
|
+
if (!adapterProfile) {
|
|
735
|
+
return {
|
|
736
|
+
requested,
|
|
737
|
+
requestedBy,
|
|
738
|
+
applied: null,
|
|
739
|
+
configSource: null,
|
|
740
|
+
fallbackReason: input.profileResolutionFallbackReason ?? "adapter_profile_not_supported",
|
|
741
|
+
adapterConfig: null,
|
|
742
|
+
};
|
|
743
|
+
}
|
|
744
|
+
const runtimeProfile = readAgentRuntimeModelProfile(input.agentRuntimeConfig, requested);
|
|
745
|
+
if (!runtimeProfile.enabled) {
|
|
746
|
+
return {
|
|
747
|
+
requested,
|
|
748
|
+
requestedBy,
|
|
749
|
+
applied: null,
|
|
750
|
+
configSource: null,
|
|
751
|
+
fallbackReason: "agent_runtime_profile_disabled",
|
|
752
|
+
adapterConfig: null,
|
|
753
|
+
};
|
|
754
|
+
}
|
|
755
|
+
return {
|
|
756
|
+
requested,
|
|
757
|
+
requestedBy,
|
|
758
|
+
applied: requested,
|
|
759
|
+
configSource: runtimeProfile.configured ? "agent_runtime" : "adapter_default",
|
|
760
|
+
fallbackReason: null,
|
|
761
|
+
adapterConfig: {
|
|
762
|
+
...parseObject(adapterProfile.adapterConfig),
|
|
763
|
+
...runtimeProfile.adapterConfig,
|
|
764
|
+
},
|
|
765
|
+
};
|
|
766
|
+
}
|
|
767
|
+
export function mergeModelProfileAdapterConfig(input) {
|
|
768
|
+
return {
|
|
769
|
+
...input.baseConfig,
|
|
770
|
+
...(input.modelProfile.adapterConfig ?? {}),
|
|
771
|
+
...(input.issueAdapterConfig ?? {}),
|
|
772
|
+
};
|
|
773
|
+
}
|
|
774
|
+
function modelProfileRunMetadata(modelProfile) {
|
|
775
|
+
if (!modelProfile.requested)
|
|
776
|
+
return null;
|
|
777
|
+
return {
|
|
778
|
+
requested: modelProfile.requested,
|
|
779
|
+
requestedBy: modelProfile.requestedBy,
|
|
780
|
+
applied: modelProfile.applied,
|
|
781
|
+
configSource: modelProfile.configSource,
|
|
782
|
+
fallbackReason: modelProfile.fallbackReason,
|
|
783
|
+
};
|
|
784
|
+
}
|
|
785
|
+
function mergeModelProfileRunMetadata(resultJson, modelProfile) {
|
|
786
|
+
const metadata = modelProfileRunMetadata(modelProfile);
|
|
787
|
+
if (!metadata)
|
|
788
|
+
return resultJson;
|
|
789
|
+
return {
|
|
790
|
+
...(resultJson ?? {}),
|
|
791
|
+
modelProfile: metadata,
|
|
792
|
+
};
|
|
793
|
+
}
|
|
669
794
|
export function summarizeHeartbeatRunContextSnapshot(contextSnapshot) {
|
|
670
795
|
const summary = {};
|
|
671
796
|
const allowedKeys = [
|
|
@@ -677,6 +802,7 @@ export function summarizeHeartbeatRunContextSnapshot(contextSnapshot) {
|
|
|
677
802
|
"wakeReason",
|
|
678
803
|
"wakeSource",
|
|
679
804
|
"wakeTriggerDetail",
|
|
805
|
+
"modelProfile",
|
|
680
806
|
];
|
|
681
807
|
for (const key of allowedKeys) {
|
|
682
808
|
const value = readNonEmptyString(contextSnapshot?.[key]);
|
|
@@ -931,14 +1057,18 @@ export function resolveRuntimeSessionParamsForWorkspace(input) {
|
|
|
931
1057
|
}
|
|
932
1058
|
function parseIssueAssigneeAdapterOverrides(raw) {
|
|
933
1059
|
const parsed = parseObject(raw);
|
|
1060
|
+
const modelProfile = MODEL_PROFILE_KEYS.includes(parsed.modelProfile)
|
|
1061
|
+
? parsed.modelProfile
|
|
1062
|
+
: null;
|
|
934
1063
|
const parsedAdapterConfig = parseObject(parsed.adapterConfig);
|
|
935
1064
|
const adapterConfig = Object.keys(parsedAdapterConfig).length > 0 ? parsedAdapterConfig : null;
|
|
936
1065
|
const useProjectWorkspace = typeof parsed.useProjectWorkspace === "boolean"
|
|
937
1066
|
? parsed.useProjectWorkspace
|
|
938
1067
|
: null;
|
|
939
|
-
if (!adapterConfig && useProjectWorkspace === null)
|
|
1068
|
+
if (!modelProfile && !adapterConfig && useProjectWorkspace === null)
|
|
940
1069
|
return null;
|
|
941
1070
|
return {
|
|
1071
|
+
modelProfile,
|
|
942
1072
|
adapterConfig,
|
|
943
1073
|
useProjectWorkspace,
|
|
944
1074
|
};
|
|
@@ -1162,6 +1292,7 @@ function enrichWakeContextSnapshot(input) {
|
|
|
1162
1292
|
if (!readNonEmptyString(contextSnapshot["wakeTriggerDetail"]) && triggerDetail) {
|
|
1163
1293
|
contextSnapshot.wakeTriggerDetail = triggerDetail;
|
|
1164
1294
|
}
|
|
1295
|
+
normalizeModelProfileWakeContext({ contextSnapshot, payload });
|
|
1165
1296
|
return {
|
|
1166
1297
|
contextSnapshot,
|
|
1167
1298
|
issueIdFromPayload,
|
|
@@ -1514,7 +1645,23 @@ export function heartbeatService(db, options = {}) {
|
|
|
1514
1645
|
};
|
|
1515
1646
|
const budgets = budgetService(db, budgetHooks);
|
|
1516
1647
|
const recovery = recoveryService(db, { enqueueWakeup });
|
|
1648
|
+
const productivityReviews = productivityReviewService(db, { enqueueWakeup });
|
|
1517
1649
|
let unsafeTextProjectionPromise = null;
|
|
1650
|
+
async function releaseEnvironmentLeasesForRun(input) {
|
|
1651
|
+
const releaseResult = await envOrchestrator.releaseForRun({
|
|
1652
|
+
heartbeatRunId: input.runId,
|
|
1653
|
+
companyId: input.companyId,
|
|
1654
|
+
agentId: input.agentId,
|
|
1655
|
+
status: leaseReleaseStatusForRunStatus(input.status),
|
|
1656
|
+
failureReason: input.failureReason ?? undefined,
|
|
1657
|
+
}).catch((err) => {
|
|
1658
|
+
logger.warn({ err, runId: input.runId }, "failed to release environment leases for heartbeat run");
|
|
1659
|
+
return null;
|
|
1660
|
+
});
|
|
1661
|
+
for (const releaseError of releaseResult?.errors ?? []) {
|
|
1662
|
+
logger.warn({ err: releaseError.error, leaseId: releaseError.leaseId, runId: input.runId }, "failed to release environment lease for heartbeat run");
|
|
1663
|
+
}
|
|
1664
|
+
}
|
|
1518
1665
|
async function hasUnsafeTextProjectionDatabase() {
|
|
1519
1666
|
if (!unsafeTextProjectionPromise) {
|
|
1520
1667
|
unsafeTextProjectionPromise = db
|
|
@@ -1613,6 +1760,522 @@ export function heartbeatService(db, options = {}) {
|
|
|
1613
1760
|
.limit(1)
|
|
1614
1761
|
.then((rows) => rows[0] ?? null);
|
|
1615
1762
|
}
|
|
1763
|
+
const issueMonitorDispatchColumns = {
|
|
1764
|
+
id: issues.id,
|
|
1765
|
+
companyId: issues.companyId,
|
|
1766
|
+
projectId: issues.projectId,
|
|
1767
|
+
goalId: issues.goalId,
|
|
1768
|
+
identifier: issues.identifier,
|
|
1769
|
+
title: issues.title,
|
|
1770
|
+
status: issues.status,
|
|
1771
|
+
priority: issues.priority,
|
|
1772
|
+
assigneeAgentId: issues.assigneeAgentId,
|
|
1773
|
+
assigneeUserId: issues.assigneeUserId,
|
|
1774
|
+
billingCode: issues.billingCode,
|
|
1775
|
+
executionPolicy: issues.executionPolicy,
|
|
1776
|
+
executionState: issues.executionState,
|
|
1777
|
+
monitorNextCheckAt: issues.monitorNextCheckAt,
|
|
1778
|
+
monitorWakeRequestedAt: issues.monitorWakeRequestedAt,
|
|
1779
|
+
monitorLastTriggeredAt: issues.monitorLastTriggeredAt,
|
|
1780
|
+
monitorAttemptCount: issues.monitorAttemptCount,
|
|
1781
|
+
monitorNotes: issues.monitorNotes,
|
|
1782
|
+
monitorScheduledBy: issues.monitorScheduledBy,
|
|
1783
|
+
};
|
|
1784
|
+
function parseMonitorDate(value) {
|
|
1785
|
+
if (!value)
|
|
1786
|
+
return null;
|
|
1787
|
+
const date = new Date(value);
|
|
1788
|
+
return Number.isNaN(date.getTime()) ? null : date;
|
|
1789
|
+
}
|
|
1790
|
+
function issueMonitorLimitClearReason(input) {
|
|
1791
|
+
const timeoutAt = parseMonitorDate(input.monitor?.timeoutAt ?? null);
|
|
1792
|
+
if (timeoutAt && input.now.getTime() >= timeoutAt.getTime()) {
|
|
1793
|
+
return "timeout_exceeded";
|
|
1794
|
+
}
|
|
1795
|
+
const maxAttempts = input.monitor?.maxAttempts ?? null;
|
|
1796
|
+
if (maxAttempts !== null && input.nextAttemptCount > maxAttempts) {
|
|
1797
|
+
return "max_attempts_exhausted";
|
|
1798
|
+
}
|
|
1799
|
+
return null;
|
|
1800
|
+
}
|
|
1801
|
+
function monitorRecoveryPolicy(monitor) {
|
|
1802
|
+
return monitor?.recoveryPolicy ?? "wake_owner";
|
|
1803
|
+
}
|
|
1804
|
+
function monitorRecoveryDetails(input) {
|
|
1805
|
+
return {
|
|
1806
|
+
identifier: input.claimed.identifier,
|
|
1807
|
+
nextCheckAt: input.scheduledAtIso,
|
|
1808
|
+
attemptedAttemptCount: input.nextAttemptCount,
|
|
1809
|
+
notes: input.claimed.monitorNotes ?? null,
|
|
1810
|
+
serviceName: input.monitor?.serviceName ?? null,
|
|
1811
|
+
timeoutAt: input.monitor?.timeoutAt ?? null,
|
|
1812
|
+
maxAttempts: input.monitor?.maxAttempts ?? null,
|
|
1813
|
+
clearReason: input.clearReason,
|
|
1814
|
+
recoveryPolicy: input.recoveryPolicy,
|
|
1815
|
+
source: input.source,
|
|
1816
|
+
};
|
|
1817
|
+
}
|
|
1818
|
+
function formatIssueIdentifierLink(identifier, fallback) {
|
|
1819
|
+
if (!identifier)
|
|
1820
|
+
return fallback;
|
|
1821
|
+
const prefix = identifier.split("-")[0];
|
|
1822
|
+
if (!prefix || !/^[A-Z][A-Z0-9]*-\d+$/.test(identifier))
|
|
1823
|
+
return identifier;
|
|
1824
|
+
return `[${identifier}](/${prefix}/issues/${identifier})`;
|
|
1825
|
+
}
|
|
1826
|
+
function monitorRecoveryComment(input) {
|
|
1827
|
+
const label = formatIssueIdentifierLink(input.issue.identifier, input.issue.id);
|
|
1828
|
+
const reason = input.clearReason === "timeout_exceeded"
|
|
1829
|
+
? "its timeout was reached"
|
|
1830
|
+
: "its maximum attempt count was reached";
|
|
1831
|
+
return [
|
|
1832
|
+
`Paperclip cleared the scheduled external-service monitor for ${label} because ${reason}.`,
|
|
1833
|
+
"",
|
|
1834
|
+
`- Attempt count: ${input.nextAttemptCount}`,
|
|
1835
|
+
`- Recovery policy: ${input.recoveryPolicy}`,
|
|
1836
|
+
"",
|
|
1837
|
+
"Next action: inspect the external service state, record the result on this issue, and restore an explicit execution or waiting path if more work remains.",
|
|
1838
|
+
].join("\n");
|
|
1839
|
+
}
|
|
1840
|
+
async function findOpenIssueMonitorRecoveryIssue(claimed) {
|
|
1841
|
+
return db
|
|
1842
|
+
.select()
|
|
1843
|
+
.from(issues)
|
|
1844
|
+
.where(and(eq(issues.companyId, claimed.companyId), eq(issues.originKind, RECOVERY_ORIGIN_KINDS.strandedIssueRecovery), eq(issues.originId, claimed.id), isNull(issues.hiddenAt), notInArray(issues.status, ["done", "cancelled"])))
|
|
1845
|
+
.orderBy(desc(issues.createdAt))
|
|
1846
|
+
.limit(1)
|
|
1847
|
+
.then((rows) => rows[0] ?? null);
|
|
1848
|
+
}
|
|
1849
|
+
async function performIssueMonitorRecovery(input) {
|
|
1850
|
+
const details = monitorRecoveryDetails({
|
|
1851
|
+
claimed: input.claimed,
|
|
1852
|
+
scheduledAtIso: input.scheduledAtIso,
|
|
1853
|
+
nextAttemptCount: input.nextAttemptCount,
|
|
1854
|
+
clearReason: input.clearReason,
|
|
1855
|
+
recoveryPolicy: input.recoveryPolicy,
|
|
1856
|
+
monitor: input.monitor,
|
|
1857
|
+
source: input.activitySource,
|
|
1858
|
+
});
|
|
1859
|
+
if (input.recoveryPolicy === "create_recovery_issue") {
|
|
1860
|
+
let recoveryIssue = await findOpenIssueMonitorRecoveryIssue(input.claimed);
|
|
1861
|
+
if (!recoveryIssue) {
|
|
1862
|
+
recoveryIssue = await issuesSvc.create(input.claimed.companyId, {
|
|
1863
|
+
title: `Recover external-service monitor for ${input.claimed.identifier ?? input.claimed.title}`,
|
|
1864
|
+
description: monitorRecoveryComment({
|
|
1865
|
+
issue: input.claimed,
|
|
1866
|
+
clearReason: input.clearReason,
|
|
1867
|
+
recoveryPolicy: input.recoveryPolicy,
|
|
1868
|
+
nextAttemptCount: input.nextAttemptCount,
|
|
1869
|
+
}),
|
|
1870
|
+
status: "todo",
|
|
1871
|
+
priority: "high",
|
|
1872
|
+
parentId: input.claimed.id,
|
|
1873
|
+
projectId: input.claimed.projectId,
|
|
1874
|
+
goalId: input.claimed.goalId,
|
|
1875
|
+
assigneeAgentId: input.claimed.assigneeAgentId,
|
|
1876
|
+
originKind: RECOVERY_ORIGIN_KINDS.strandedIssueRecovery,
|
|
1877
|
+
originId: input.claimed.id,
|
|
1878
|
+
originFingerprint: `issue_monitor:${input.clearReason}`,
|
|
1879
|
+
billingCode: input.claimed.billingCode,
|
|
1880
|
+
});
|
|
1881
|
+
}
|
|
1882
|
+
if (recoveryIssue.assigneeAgentId) {
|
|
1883
|
+
await enqueueWakeup(recoveryIssue.assigneeAgentId, {
|
|
1884
|
+
source: "automation",
|
|
1885
|
+
triggerDetail: "system",
|
|
1886
|
+
reason: "issue_monitor_recovery_issue",
|
|
1887
|
+
idempotencyKey: `issue-monitor-recovery-issue:${input.claimed.id}:${input.clearReason}:${input.scheduledAtIso}`,
|
|
1888
|
+
payload: { issueId: recoveryIssue.id, sourceIssueId: input.claimed.id },
|
|
1889
|
+
requestedByActorType: input.actorType,
|
|
1890
|
+
requestedByActorId: input.actorId,
|
|
1891
|
+
contextSnapshot: {
|
|
1892
|
+
issueId: recoveryIssue.id,
|
|
1893
|
+
sourceIssueId: input.claimed.id,
|
|
1894
|
+
source: "issue.monitor.recovery_issue",
|
|
1895
|
+
wakeReason: "issue_monitor_recovery_issue",
|
|
1896
|
+
},
|
|
1897
|
+
});
|
|
1898
|
+
}
|
|
1899
|
+
await logActivity(db, {
|
|
1900
|
+
companyId: input.claimed.companyId,
|
|
1901
|
+
actorType: input.actorType,
|
|
1902
|
+
actorId: input.actorId,
|
|
1903
|
+
agentId: input.agentId,
|
|
1904
|
+
runId: input.runId,
|
|
1905
|
+
action: "issue.monitor_recovery_issue_created",
|
|
1906
|
+
entityType: "issue",
|
|
1907
|
+
entityId: input.claimed.id,
|
|
1908
|
+
details: {
|
|
1909
|
+
...details,
|
|
1910
|
+
recoveryIssueId: recoveryIssue.id,
|
|
1911
|
+
recoveryIdentifier: recoveryIssue.identifier,
|
|
1912
|
+
},
|
|
1913
|
+
});
|
|
1914
|
+
return;
|
|
1915
|
+
}
|
|
1916
|
+
if (input.recoveryPolicy === "escalate_to_board") {
|
|
1917
|
+
await db.insert(issueComments).values({
|
|
1918
|
+
companyId: input.claimed.companyId,
|
|
1919
|
+
issueId: input.claimed.id,
|
|
1920
|
+
body: monitorRecoveryComment({
|
|
1921
|
+
issue: input.claimed,
|
|
1922
|
+
clearReason: input.clearReason,
|
|
1923
|
+
recoveryPolicy: input.recoveryPolicy,
|
|
1924
|
+
nextAttemptCount: input.nextAttemptCount,
|
|
1925
|
+
}),
|
|
1926
|
+
});
|
|
1927
|
+
await logActivity(db, {
|
|
1928
|
+
companyId: input.claimed.companyId,
|
|
1929
|
+
actorType: input.actorType,
|
|
1930
|
+
actorId: input.actorId,
|
|
1931
|
+
agentId: input.agentId,
|
|
1932
|
+
runId: input.runId,
|
|
1933
|
+
action: "issue.monitor_escalated_to_board",
|
|
1934
|
+
entityType: "issue",
|
|
1935
|
+
entityId: input.claimed.id,
|
|
1936
|
+
details,
|
|
1937
|
+
});
|
|
1938
|
+
return;
|
|
1939
|
+
}
|
|
1940
|
+
await enqueueWakeup(input.claimed.assigneeAgentId, {
|
|
1941
|
+
source: "automation",
|
|
1942
|
+
triggerDetail: "system",
|
|
1943
|
+
reason: "issue_monitor_recovery",
|
|
1944
|
+
idempotencyKey: `issue-monitor-recovery:${input.claimed.id}:${input.clearReason}:${input.scheduledAtIso}`,
|
|
1945
|
+
payload: {
|
|
1946
|
+
issueId: input.claimed.id,
|
|
1947
|
+
monitorAttemptCount: input.nextAttemptCount,
|
|
1948
|
+
monitorNotes: input.claimed.monitorNotes ?? null,
|
|
1949
|
+
clearReason: input.clearReason,
|
|
1950
|
+
serviceName: input.monitor?.serviceName ?? null,
|
|
1951
|
+
timeoutAt: input.monitor?.timeoutAt ?? null,
|
|
1952
|
+
maxAttempts: input.monitor?.maxAttempts ?? null,
|
|
1953
|
+
},
|
|
1954
|
+
requestedByActorType: input.actorType,
|
|
1955
|
+
requestedByActorId: input.actorId,
|
|
1956
|
+
contextSnapshot: {
|
|
1957
|
+
issueId: input.claimed.id,
|
|
1958
|
+
source: "issue.monitor.recovery",
|
|
1959
|
+
wakeReason: "issue_monitor_recovery",
|
|
1960
|
+
monitorAttemptCount: input.nextAttemptCount,
|
|
1961
|
+
monitorNotes: input.claimed.monitorNotes ?? null,
|
|
1962
|
+
clearReason: input.clearReason,
|
|
1963
|
+
serviceName: input.monitor?.serviceName ?? null,
|
|
1964
|
+
timeoutAt: input.monitor?.timeoutAt ?? null,
|
|
1965
|
+
maxAttempts: input.monitor?.maxAttempts ?? null,
|
|
1966
|
+
},
|
|
1967
|
+
});
|
|
1968
|
+
await logActivity(db, {
|
|
1969
|
+
companyId: input.claimed.companyId,
|
|
1970
|
+
actorType: input.actorType,
|
|
1971
|
+
actorId: input.actorId,
|
|
1972
|
+
agentId: input.agentId,
|
|
1973
|
+
runId: input.runId,
|
|
1974
|
+
action: "issue.monitor_recovery_wake_queued",
|
|
1975
|
+
entityType: "issue",
|
|
1976
|
+
entityId: input.claimed.id,
|
|
1977
|
+
details,
|
|
1978
|
+
});
|
|
1979
|
+
}
|
|
1980
|
+
async function clearIssueMonitorAndRecover(input) {
|
|
1981
|
+
await db
|
|
1982
|
+
.update(issues)
|
|
1983
|
+
.set({
|
|
1984
|
+
...buildIssueMonitorClearedPatch({
|
|
1985
|
+
issue: input.claimed,
|
|
1986
|
+
policy: input.policy,
|
|
1987
|
+
clearReason: input.clearReason,
|
|
1988
|
+
clearedAt: input.now,
|
|
1989
|
+
}),
|
|
1990
|
+
updatedAt: input.now,
|
|
1991
|
+
})
|
|
1992
|
+
.where(eq(issues.id, input.claimed.id));
|
|
1993
|
+
await logActivity(db, {
|
|
1994
|
+
companyId: input.claimed.companyId,
|
|
1995
|
+
actorType: input.actorType,
|
|
1996
|
+
actorId: input.actorId,
|
|
1997
|
+
agentId: input.agentId,
|
|
1998
|
+
runId: input.runId,
|
|
1999
|
+
action: "issue.monitor_exhausted",
|
|
2000
|
+
entityType: "issue",
|
|
2001
|
+
entityId: input.claimed.id,
|
|
2002
|
+
details: monitorRecoveryDetails({
|
|
2003
|
+
claimed: input.claimed,
|
|
2004
|
+
scheduledAtIso: input.scheduledAtIso,
|
|
2005
|
+
nextAttemptCount: input.nextAttemptCount,
|
|
2006
|
+
clearReason: input.clearReason,
|
|
2007
|
+
recoveryPolicy: input.recoveryPolicy,
|
|
2008
|
+
monitor: input.monitor,
|
|
2009
|
+
source: input.activitySource,
|
|
2010
|
+
}),
|
|
2011
|
+
});
|
|
2012
|
+
await performIssueMonitorRecovery({
|
|
2013
|
+
claimed: input.claimed,
|
|
2014
|
+
scheduledAtIso: input.scheduledAtIso,
|
|
2015
|
+
nextAttemptCount: input.nextAttemptCount,
|
|
2016
|
+
clearReason: input.clearReason,
|
|
2017
|
+
recoveryPolicy: input.recoveryPolicy,
|
|
2018
|
+
monitor: input.monitor,
|
|
2019
|
+
actorType: input.actorType,
|
|
2020
|
+
actorId: input.actorId,
|
|
2021
|
+
agentId: input.agentId,
|
|
2022
|
+
runId: input.runId,
|
|
2023
|
+
activitySource: input.activitySource,
|
|
2024
|
+
});
|
|
2025
|
+
return { outcome: "skipped", reason: input.clearReason };
|
|
2026
|
+
}
|
|
2027
|
+
async function dispatchClaimedIssueMonitor(claimed, input) {
|
|
2028
|
+
if (!claimed.assigneeAgentId || !claimed.monitorNextCheckAt) {
|
|
2029
|
+
throw conflict("Issue monitor is not ready to dispatch");
|
|
2030
|
+
}
|
|
2031
|
+
const scheduledAtIso = claimed.monitorNextCheckAt.toISOString();
|
|
2032
|
+
const nextAttemptCount = (claimed.monitorAttemptCount ?? 0) + 1;
|
|
2033
|
+
const policy = normalizeIssueExecutionPolicy(claimed.executionPolicy ?? null);
|
|
2034
|
+
const monitor = policy?.monitor ?? null;
|
|
2035
|
+
const clearReason = issueMonitorLimitClearReason({ monitor, nextAttemptCount, now: input.now });
|
|
2036
|
+
const recoveryPolicy = monitorRecoveryPolicy(monitor);
|
|
2037
|
+
const monitorMetadata = {
|
|
2038
|
+
serviceName: monitor?.serviceName ?? null,
|
|
2039
|
+
timeoutAt: monitor?.timeoutAt ?? null,
|
|
2040
|
+
maxAttempts: monitor?.maxAttempts ?? null,
|
|
2041
|
+
recoveryPolicy: monitor?.recoveryPolicy ?? null,
|
|
2042
|
+
};
|
|
2043
|
+
if (clearReason) {
|
|
2044
|
+
return clearIssueMonitorAndRecover({
|
|
2045
|
+
claimed,
|
|
2046
|
+
policy,
|
|
2047
|
+
scheduledAtIso,
|
|
2048
|
+
nextAttemptCount,
|
|
2049
|
+
clearReason,
|
|
2050
|
+
recoveryPolicy,
|
|
2051
|
+
monitor,
|
|
2052
|
+
now: input.now,
|
|
2053
|
+
actorType: input.actorType,
|
|
2054
|
+
actorId: input.actorId,
|
|
2055
|
+
agentId: input.agentId,
|
|
2056
|
+
runId: input.runId,
|
|
2057
|
+
activitySource: input.activitySource,
|
|
2058
|
+
});
|
|
2059
|
+
}
|
|
2060
|
+
try {
|
|
2061
|
+
await enqueueWakeup(claimed.assigneeAgentId, {
|
|
2062
|
+
source: input.source,
|
|
2063
|
+
triggerDetail: input.triggerDetail,
|
|
2064
|
+
reason: input.wakeReason,
|
|
2065
|
+
idempotencyKey: `issue-monitor:${claimed.id}:${scheduledAtIso}`,
|
|
2066
|
+
payload: {
|
|
2067
|
+
issueId: claimed.id,
|
|
2068
|
+
nextCheckAt: scheduledAtIso,
|
|
2069
|
+
monitorAttemptCount: nextAttemptCount,
|
|
2070
|
+
monitorNotes: claimed.monitorNotes ?? null,
|
|
2071
|
+
...monitorMetadata,
|
|
2072
|
+
source: input.activitySource,
|
|
2073
|
+
},
|
|
2074
|
+
requestedByActorType: input.actorType,
|
|
2075
|
+
requestedByActorId: input.actorId,
|
|
2076
|
+
contextSnapshot: {
|
|
2077
|
+
issueId: claimed.id,
|
|
2078
|
+
source: "issue.monitor",
|
|
2079
|
+
wakeReason: input.wakeReason,
|
|
2080
|
+
nextCheckAt: scheduledAtIso,
|
|
2081
|
+
monitorAttemptCount: nextAttemptCount,
|
|
2082
|
+
monitorNotes: claimed.monitorNotes ?? null,
|
|
2083
|
+
...monitorMetadata,
|
|
2084
|
+
manualTrigger: input.activitySource === "manual",
|
|
2085
|
+
},
|
|
2086
|
+
});
|
|
2087
|
+
await db
|
|
2088
|
+
.update(issues)
|
|
2089
|
+
.set({
|
|
2090
|
+
...buildIssueMonitorTriggeredPatch({
|
|
2091
|
+
issue: claimed,
|
|
2092
|
+
policy,
|
|
2093
|
+
triggeredAt: input.now,
|
|
2094
|
+
}),
|
|
2095
|
+
updatedAt: new Date(),
|
|
2096
|
+
})
|
|
2097
|
+
.where(eq(issues.id, claimed.id));
|
|
2098
|
+
await logActivity(db, {
|
|
2099
|
+
companyId: claimed.companyId,
|
|
2100
|
+
actorType: input.actorType,
|
|
2101
|
+
actorId: input.actorId,
|
|
2102
|
+
agentId: input.agentId,
|
|
2103
|
+
runId: input.runId,
|
|
2104
|
+
action: "issue.monitor_triggered",
|
|
2105
|
+
entityType: "issue",
|
|
2106
|
+
entityId: claimed.id,
|
|
2107
|
+
details: {
|
|
2108
|
+
identifier: claimed.identifier,
|
|
2109
|
+
nextCheckAt: scheduledAtIso,
|
|
2110
|
+
lastTriggeredAt: input.now.toISOString(),
|
|
2111
|
+
attemptCount: nextAttemptCount,
|
|
2112
|
+
notes: claimed.monitorNotes ?? null,
|
|
2113
|
+
...monitorMetadata,
|
|
2114
|
+
source: input.activitySource,
|
|
2115
|
+
},
|
|
2116
|
+
});
|
|
2117
|
+
return { outcome: "triggered" };
|
|
2118
|
+
}
|
|
2119
|
+
catch (err) {
|
|
2120
|
+
if (err instanceof HttpError && err.status >= 400 && err.status < 500) {
|
|
2121
|
+
if (input.clearOnClientError) {
|
|
2122
|
+
await db
|
|
2123
|
+
.update(issues)
|
|
2124
|
+
.set({
|
|
2125
|
+
...buildIssueMonitorClearedPatch({
|
|
2126
|
+
issue: claimed,
|
|
2127
|
+
policy,
|
|
2128
|
+
clearReason: "dispatch_skipped",
|
|
2129
|
+
clearedAt: input.now,
|
|
2130
|
+
}),
|
|
2131
|
+
updatedAt: new Date(),
|
|
2132
|
+
})
|
|
2133
|
+
.where(eq(issues.id, claimed.id));
|
|
2134
|
+
await logActivity(db, {
|
|
2135
|
+
companyId: claimed.companyId,
|
|
2136
|
+
actorType: input.actorType,
|
|
2137
|
+
actorId: input.actorId,
|
|
2138
|
+
agentId: input.agentId,
|
|
2139
|
+
runId: input.runId,
|
|
2140
|
+
action: "issue.monitor_skipped",
|
|
2141
|
+
entityType: "issue",
|
|
2142
|
+
entityId: claimed.id,
|
|
2143
|
+
details: {
|
|
2144
|
+
identifier: claimed.identifier,
|
|
2145
|
+
nextCheckAt: scheduledAtIso,
|
|
2146
|
+
attemptCount: nextAttemptCount,
|
|
2147
|
+
notes: claimed.monitorNotes ?? null,
|
|
2148
|
+
reason: err.message,
|
|
2149
|
+
source: input.activitySource,
|
|
2150
|
+
},
|
|
2151
|
+
});
|
|
2152
|
+
return { outcome: "skipped", reason: err.message };
|
|
2153
|
+
}
|
|
2154
|
+
await db
|
|
2155
|
+
.update(issues)
|
|
2156
|
+
.set({
|
|
2157
|
+
monitorWakeRequestedAt: null,
|
|
2158
|
+
updatedAt: new Date(),
|
|
2159
|
+
})
|
|
2160
|
+
.where(eq(issues.id, claimed.id));
|
|
2161
|
+
}
|
|
2162
|
+
else {
|
|
2163
|
+
await db
|
|
2164
|
+
.update(issues)
|
|
2165
|
+
.set({
|
|
2166
|
+
monitorWakeRequestedAt: null,
|
|
2167
|
+
updatedAt: new Date(),
|
|
2168
|
+
})
|
|
2169
|
+
.where(eq(issues.id, claimed.id));
|
|
2170
|
+
}
|
|
2171
|
+
throw err;
|
|
2172
|
+
}
|
|
2173
|
+
}
|
|
2174
|
+
async function triggerIssueMonitor(issueId, input) {
|
|
2175
|
+
const now = input?.now ?? new Date();
|
|
2176
|
+
const actorType = input?.actorType ?? "system";
|
|
2177
|
+
const actorId = input?.actorId ?? (actorType === "system" ? "heartbeat_scheduler" : null);
|
|
2178
|
+
if (!actorId) {
|
|
2179
|
+
throw conflict("Issue monitor trigger requires an actor");
|
|
2180
|
+
}
|
|
2181
|
+
const issue = await db
|
|
2182
|
+
.select(issueMonitorDispatchColumns)
|
|
2183
|
+
.from(issues)
|
|
2184
|
+
.where(eq(issues.id, issueId))
|
|
2185
|
+
.limit(1)
|
|
2186
|
+
.then((rows) => rows[0] ?? null);
|
|
2187
|
+
if (!issue) {
|
|
2188
|
+
throw notFound("Issue not found");
|
|
2189
|
+
}
|
|
2190
|
+
if (!issue.monitorNextCheckAt) {
|
|
2191
|
+
throw conflict("Issue has no scheduled monitor");
|
|
2192
|
+
}
|
|
2193
|
+
if (!issue.assigneeAgentId || issue.assigneeUserId) {
|
|
2194
|
+
throw conflict("Issue monitor requires an agent assignee");
|
|
2195
|
+
}
|
|
2196
|
+
if (!["in_progress", "in_review"].includes(issue.status)) {
|
|
2197
|
+
throw conflict("Issue monitor can only run while the issue is in progress or in review");
|
|
2198
|
+
}
|
|
2199
|
+
const staleClaimThreshold = new Date(now.getTime() - 5 * 60 * 1000);
|
|
2200
|
+
const claimed = await db.transaction(async (tx) => {
|
|
2201
|
+
const [updated] = await tx
|
|
2202
|
+
.update(issues)
|
|
2203
|
+
.set({
|
|
2204
|
+
monitorWakeRequestedAt: now,
|
|
2205
|
+
updatedAt: now,
|
|
2206
|
+
})
|
|
2207
|
+
.where(and(eq(issues.id, issueId), sql `${issues.monitorNextCheckAt} is not null`, isNull(issues.assigneeUserId), sql `${issues.assigneeAgentId} is not null`, inArray(issues.status, ["in_progress", "in_review"]), or(isNull(issues.monitorWakeRequestedAt), lt(issues.monitorWakeRequestedAt, staleClaimThreshold))))
|
|
2208
|
+
.returning();
|
|
2209
|
+
return (updated ?? null);
|
|
2210
|
+
});
|
|
2211
|
+
if (!claimed) {
|
|
2212
|
+
throw conflict("Issue monitor check is already in progress");
|
|
2213
|
+
}
|
|
2214
|
+
return dispatchClaimedIssueMonitor(claimed, {
|
|
2215
|
+
now,
|
|
2216
|
+
source: "on_demand",
|
|
2217
|
+
triggerDetail: "manual",
|
|
2218
|
+
wakeReason: input?.wakeReason ?? "issue_monitor_due",
|
|
2219
|
+
actorType,
|
|
2220
|
+
actorId,
|
|
2221
|
+
agentId: input?.agentId ?? null,
|
|
2222
|
+
runId: input?.runId ?? null,
|
|
2223
|
+
clearOnClientError: false,
|
|
2224
|
+
activitySource: "manual",
|
|
2225
|
+
});
|
|
2226
|
+
}
|
|
2227
|
+
async function tickDueIssueMonitors(now = new Date()) {
|
|
2228
|
+
const staleClaimThreshold = new Date(now.getTime() - 5 * 60 * 1000);
|
|
2229
|
+
const dueMonitors = await db
|
|
2230
|
+
.select(issueMonitorDispatchColumns)
|
|
2231
|
+
.from(issues)
|
|
2232
|
+
.where(and(sql `${issues.monitorNextCheckAt} is not null`, lte(issues.monitorNextCheckAt, now), isNull(issues.assigneeUserId), sql `${issues.assigneeAgentId} is not null`, inArray(issues.status, ["in_progress", "in_review"]), or(isNull(issues.monitorWakeRequestedAt), lt(issues.monitorWakeRequestedAt, staleClaimThreshold))))
|
|
2233
|
+
.orderBy(asc(issues.monitorNextCheckAt), asc(issues.updatedAt))
|
|
2234
|
+
.limit(50);
|
|
2235
|
+
let triggered = 0;
|
|
2236
|
+
let skipped = 0;
|
|
2237
|
+
for (const due of dueMonitors) {
|
|
2238
|
+
const claimed = await db.transaction(async (tx) => {
|
|
2239
|
+
const [updated] = await tx
|
|
2240
|
+
.update(issues)
|
|
2241
|
+
.set({
|
|
2242
|
+
monitorWakeRequestedAt: now,
|
|
2243
|
+
updatedAt: now,
|
|
2244
|
+
})
|
|
2245
|
+
.where(and(eq(issues.id, due.id), sql `${issues.monitorNextCheckAt} is not null`, lte(issues.monitorNextCheckAt, now), isNull(issues.assigneeUserId), sql `${issues.assigneeAgentId} is not null`, inArray(issues.status, ["in_progress", "in_review"]), or(isNull(issues.monitorWakeRequestedAt), lt(issues.monitorWakeRequestedAt, staleClaimThreshold))))
|
|
2246
|
+
.returning();
|
|
2247
|
+
return (updated ?? null);
|
|
2248
|
+
});
|
|
2249
|
+
if (!claimed)
|
|
2250
|
+
continue;
|
|
2251
|
+
try {
|
|
2252
|
+
const result = await dispatchClaimedIssueMonitor(claimed, {
|
|
2253
|
+
now,
|
|
2254
|
+
source: "automation",
|
|
2255
|
+
triggerDetail: "system",
|
|
2256
|
+
wakeReason: "issue_monitor_due",
|
|
2257
|
+
actorType: "system",
|
|
2258
|
+
actorId: "heartbeat_scheduler",
|
|
2259
|
+
agentId: null,
|
|
2260
|
+
runId: null,
|
|
2261
|
+
clearOnClientError: true,
|
|
2262
|
+
activitySource: "scheduled",
|
|
2263
|
+
});
|
|
2264
|
+
if (result.outcome === "triggered")
|
|
2265
|
+
triggered += 1;
|
|
2266
|
+
if (result.outcome === "skipped")
|
|
2267
|
+
skipped += 1;
|
|
2268
|
+
}
|
|
2269
|
+
catch (err) {
|
|
2270
|
+
logger.error({ err, issueId: claimed.id }, "issue monitor tick failed");
|
|
2271
|
+
}
|
|
2272
|
+
}
|
|
2273
|
+
return {
|
|
2274
|
+
checked: dueMonitors.length,
|
|
2275
|
+
triggered,
|
|
2276
|
+
skipped,
|
|
2277
|
+
};
|
|
2278
|
+
}
|
|
1616
2279
|
async function getOldestRunForSession(agentId, sessionId) {
|
|
1617
2280
|
return db
|
|
1618
2281
|
.select({
|
|
@@ -2022,16 +2685,26 @@ export function heartbeatService(db, options = {}) {
|
|
|
2022
2685
|
const existing = await getRuntimeState(agent.id);
|
|
2023
2686
|
if (existing)
|
|
2024
2687
|
return existing;
|
|
2025
|
-
|
|
2688
|
+
const inserted = await db
|
|
2026
2689
|
.insert(agentRuntimeState)
|
|
2027
2690
|
.values({
|
|
2028
2691
|
agentId: agent.id,
|
|
2029
2692
|
companyId: agent.companyId,
|
|
2030
2693
|
adapterType: agent.adapterType,
|
|
2031
2694
|
stateJson: {},
|
|
2695
|
+
})
|
|
2696
|
+
.onConflictDoNothing({
|
|
2697
|
+
target: agentRuntimeState.agentId,
|
|
2032
2698
|
})
|
|
2033
2699
|
.returning()
|
|
2034
|
-
.then((rows) => rows[0]);
|
|
2700
|
+
.then((rows) => rows[0] ?? null);
|
|
2701
|
+
if (inserted)
|
|
2702
|
+
return inserted;
|
|
2703
|
+
const ensured = await getRuntimeState(agent.id);
|
|
2704
|
+
if (!ensured) {
|
|
2705
|
+
throw new Error(`Failed to ensure runtime state for agent ${agent.id}`);
|
|
2706
|
+
}
|
|
2707
|
+
return ensured;
|
|
2035
2708
|
}
|
|
2036
2709
|
async function setRunStatus(runId, status, patch) {
|
|
2037
2710
|
const updated = await db
|
|
@@ -2158,6 +2831,28 @@ export function heartbeatService(db, options = {}) {
|
|
|
2158
2831
|
projectId: issue.projectId,
|
|
2159
2832
|
})
|
|
2160
2833
|
: null;
|
|
2834
|
+
if (issue) {
|
|
2835
|
+
const productivityHold = await productivityReviews.isProductivityReviewContinuationHoldActive({
|
|
2836
|
+
companyId: issue.companyId,
|
|
2837
|
+
issueId: issue.id,
|
|
2838
|
+
agentId: run.agentId,
|
|
2839
|
+
});
|
|
2840
|
+
if (productivityHold.held) {
|
|
2841
|
+
await setRunStatus(run.id, run.status, {
|
|
2842
|
+
livenessReason: `${run.livenessReason ?? "Run ended without concrete progress"}; continuation held by productivity review ${productivityHold.reviewIdentifier ?? productivityHold.reviewIssueId}`,
|
|
2843
|
+
});
|
|
2844
|
+
await productivityReviews.recordContinuationHold({
|
|
2845
|
+
companyId: issue.companyId,
|
|
2846
|
+
issueId: issue.id,
|
|
2847
|
+
runId: run.id,
|
|
2848
|
+
agentId: run.agentId,
|
|
2849
|
+
reviewIssueId: productivityHold.reviewIssueId,
|
|
2850
|
+
trigger: productivityHold.trigger,
|
|
2851
|
+
reason: productivityHold.reason,
|
|
2852
|
+
});
|
|
2853
|
+
return;
|
|
2854
|
+
}
|
|
2855
|
+
}
|
|
2161
2856
|
const nextAttempt = readContinuationAttempt(run.continuationAttempt) + 1;
|
|
2162
2857
|
const idempotencyKey = issue
|
|
2163
2858
|
? buildRunLivenessContinuationIdempotencyKey({
|
|
@@ -2224,9 +2919,10 @@ export function heartbeatService(db, options = {}) {
|
|
|
2224
2919
|
const boundedPayload = event.payload
|
|
2225
2920
|
? boundHeartbeatRunEventPayloadForStorage(event.payload)
|
|
2226
2921
|
: event.payload;
|
|
2227
|
-
const
|
|
2228
|
-
|
|
2229
|
-
|
|
2922
|
+
const secretSanitizedPayload = boundedPayload ? redactEventPayload(boundedPayload) : boundedPayload;
|
|
2923
|
+
const sanitizedPayload = secretSanitizedPayload
|
|
2924
|
+
? redactCurrentUserValue(secretSanitizedPayload, currentUserRedactionOptions)
|
|
2925
|
+
: secretSanitizedPayload;
|
|
2230
2926
|
await db.insert(heartbeatRunEvents).values({
|
|
2231
2927
|
companyId: run.companyId,
|
|
2232
2928
|
runId: run.id,
|
|
@@ -2621,19 +3317,235 @@ export function heartbeatService(db, options = {}) {
|
|
|
2621
3317
|
eventType: "lifecycle",
|
|
2622
3318
|
stream: "system",
|
|
2623
3319
|
level: "warn",
|
|
2624
|
-
message: "Queued automatic retry after orphaned child process was confirmed dead",
|
|
3320
|
+
message: "Queued automatic retry after orphaned child process was confirmed dead",
|
|
3321
|
+
payload: {
|
|
3322
|
+
retryOfRunId: run.id,
|
|
3323
|
+
},
|
|
3324
|
+
});
|
|
3325
|
+
return queued;
|
|
3326
|
+
}
|
|
3327
|
+
async function evaluateScheduledRetryGate(input) {
|
|
3328
|
+
const { run, agent, contextSnapshot } = input;
|
|
3329
|
+
const retryReason = input.retryReason ?? readNonEmptyString(contextSnapshot.retryReason) ?? run.scheduledRetryReason ?? null;
|
|
3330
|
+
const issueId = readNonEmptyString(contextSnapshot.issueId);
|
|
3331
|
+
const projectId = readNonEmptyString(contextSnapshot.projectId);
|
|
3332
|
+
const budgetBlock = await budgets.getInvocationBlock(run.companyId, run.agentId, {
|
|
3333
|
+
issueId,
|
|
3334
|
+
projectId,
|
|
3335
|
+
});
|
|
3336
|
+
if (budgetBlock) {
|
|
3337
|
+
return {
|
|
3338
|
+
allowed: false,
|
|
3339
|
+
reason: budgetBlock.reason,
|
|
3340
|
+
errorCode: "budget_blocked",
|
|
3341
|
+
issueId,
|
|
3342
|
+
details: {
|
|
3343
|
+
scopeType: budgetBlock.scopeType,
|
|
3344
|
+
scopeId: budgetBlock.scopeId,
|
|
3345
|
+
},
|
|
3346
|
+
};
|
|
3347
|
+
}
|
|
3348
|
+
if (agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval") {
|
|
3349
|
+
return {
|
|
3350
|
+
allowed: false,
|
|
3351
|
+
reason: "Scheduled retry suppressed because the agent is not invokable",
|
|
3352
|
+
errorCode: "agent_not_invokable",
|
|
3353
|
+
issueId,
|
|
3354
|
+
details: {
|
|
3355
|
+
agentId: agent.id,
|
|
3356
|
+
agentStatus: agent.status,
|
|
3357
|
+
},
|
|
3358
|
+
};
|
|
3359
|
+
}
|
|
3360
|
+
if (!issueId)
|
|
3361
|
+
return { allowed: true };
|
|
3362
|
+
const issue = await db
|
|
3363
|
+
.select({
|
|
3364
|
+
id: issues.id,
|
|
3365
|
+
status: issues.status,
|
|
3366
|
+
assigneeAgentId: issues.assigneeAgentId,
|
|
3367
|
+
executionRunId: issues.executionRunId,
|
|
3368
|
+
executionState: issues.executionState,
|
|
3369
|
+
})
|
|
3370
|
+
.from(issues)
|
|
3371
|
+
.where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))
|
|
3372
|
+
.then((rows) => rows[0] ?? null);
|
|
3373
|
+
if (!issue) {
|
|
3374
|
+
return {
|
|
3375
|
+
allowed: false,
|
|
3376
|
+
reason: "Scheduled retry suppressed because the target issue no longer exists",
|
|
3377
|
+
errorCode: "issue_not_found",
|
|
3378
|
+
issueId,
|
|
3379
|
+
details: { issueId },
|
|
3380
|
+
};
|
|
3381
|
+
}
|
|
3382
|
+
if (issue.assigneeAgentId !== run.agentId) {
|
|
3383
|
+
return {
|
|
3384
|
+
allowed: false,
|
|
3385
|
+
reason: "Scheduled retry suppressed because issue ownership changed",
|
|
3386
|
+
errorCode: "issue_reassigned",
|
|
3387
|
+
issueId,
|
|
3388
|
+
details: {
|
|
3389
|
+
issueId,
|
|
3390
|
+
previousAssigneeAgentId: run.agentId,
|
|
3391
|
+
currentAssigneeAgentId: issue.assigneeAgentId,
|
|
3392
|
+
},
|
|
3393
|
+
};
|
|
3394
|
+
}
|
|
3395
|
+
if (issue.status === "cancelled" || issue.status === "done") {
|
|
3396
|
+
return {
|
|
3397
|
+
allowed: false,
|
|
3398
|
+
reason: `Scheduled retry suppressed because issue reached terminal status (${issue.status})`,
|
|
3399
|
+
errorCode: issue.status === "cancelled" ? "issue_cancelled" : "issue_terminal_status",
|
|
3400
|
+
issueId,
|
|
3401
|
+
details: { issueId, currentStatus: issue.status },
|
|
3402
|
+
};
|
|
3403
|
+
}
|
|
3404
|
+
if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON && issue.status !== "in_progress") {
|
|
3405
|
+
return {
|
|
3406
|
+
allowed: false,
|
|
3407
|
+
reason: `Scheduled max-turn continuation suppressed because issue is no longer in_progress (current status: ${issue.status})`,
|
|
3408
|
+
errorCode: "issue_not_in_progress",
|
|
3409
|
+
issueId,
|
|
3410
|
+
details: { issueId, currentStatus: issue.status, requiredStatus: "in_progress" },
|
|
3411
|
+
};
|
|
3412
|
+
}
|
|
3413
|
+
if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON &&
|
|
3414
|
+
input.enforceIssueExecutionLock &&
|
|
3415
|
+
issue.executionRunId !== run.id) {
|
|
3416
|
+
return {
|
|
3417
|
+
allowed: false,
|
|
3418
|
+
reason: "Scheduled max-turn continuation suppressed because the issue execution lock belongs to a different run",
|
|
3419
|
+
errorCode: "issue_execution_lock_changed",
|
|
3420
|
+
issueId,
|
|
3421
|
+
details: {
|
|
3422
|
+
issueId,
|
|
3423
|
+
expectedExecutionRunId: run.id,
|
|
3424
|
+
currentExecutionRunId: issue.executionRunId,
|
|
3425
|
+
},
|
|
3426
|
+
};
|
|
3427
|
+
}
|
|
3428
|
+
if (issue.status === "in_review") {
|
|
3429
|
+
const executionState = parseIssueExecutionState(issue.executionState);
|
|
3430
|
+
const currentParticipant = executionState?.currentParticipant ?? null;
|
|
3431
|
+
if (currentParticipant) {
|
|
3432
|
+
const participantMatches = currentParticipant.type === "agent" && currentParticipant.agentId === run.agentId;
|
|
3433
|
+
if (!participantMatches) {
|
|
3434
|
+
return {
|
|
3435
|
+
allowed: false,
|
|
3436
|
+
reason: "Scheduled retry suppressed because the issue is waiting on another review participant",
|
|
3437
|
+
errorCode: "issue_review_participant_changed",
|
|
3438
|
+
issueId,
|
|
3439
|
+
details: {
|
|
3440
|
+
issueId,
|
|
3441
|
+
currentStageType: executionState?.currentStageType ?? null,
|
|
3442
|
+
currentParticipant,
|
|
3443
|
+
},
|
|
3444
|
+
};
|
|
3445
|
+
}
|
|
3446
|
+
}
|
|
3447
|
+
}
|
|
3448
|
+
const activePauseHold = await treeControlSvc.getActivePauseHoldGate(run.companyId, issueId);
|
|
3449
|
+
if (activePauseHold) {
|
|
3450
|
+
return {
|
|
3451
|
+
allowed: false,
|
|
3452
|
+
reason: "Scheduled retry suppressed because the issue is held by an active subtree pause hold",
|
|
3453
|
+
errorCode: "issue_paused",
|
|
3454
|
+
issueId,
|
|
3455
|
+
details: {
|
|
3456
|
+
issueId,
|
|
3457
|
+
holdId: activePauseHold.holdId,
|
|
3458
|
+
rootIssueId: activePauseHold.rootIssueId,
|
|
3459
|
+
},
|
|
3460
|
+
};
|
|
3461
|
+
}
|
|
3462
|
+
const dependencyReadiness = await issuesSvc.listDependencyReadiness(run.companyId, [issueId]);
|
|
3463
|
+
const readiness = dependencyReadiness.get(issueId);
|
|
3464
|
+
if (readiness && !readiness.isDependencyReady) {
|
|
3465
|
+
return {
|
|
3466
|
+
allowed: false,
|
|
3467
|
+
reason: "Scheduled retry suppressed because issue dependencies are still blocked",
|
|
3468
|
+
errorCode: "issue_dependencies_blocked",
|
|
3469
|
+
issueId,
|
|
3470
|
+
details: {
|
|
3471
|
+
issueId,
|
|
3472
|
+
unresolvedBlockerIssueIds: readiness.unresolvedBlockerIssueIds,
|
|
3473
|
+
unresolvedBlockerCount: readiness.unresolvedBlockerCount,
|
|
3474
|
+
},
|
|
3475
|
+
};
|
|
3476
|
+
}
|
|
3477
|
+
return { allowed: true };
|
|
3478
|
+
}
|
|
3479
|
+
async function cancelScheduledRetryForGate(run, gate, now) {
|
|
3480
|
+
const cancelled = await db
|
|
3481
|
+
.update(heartbeatRuns)
|
|
3482
|
+
.set({
|
|
3483
|
+
status: "cancelled",
|
|
3484
|
+
finishedAt: now,
|
|
3485
|
+
error: gate.reason,
|
|
3486
|
+
errorCode: gate.errorCode,
|
|
3487
|
+
updatedAt: now,
|
|
3488
|
+
})
|
|
3489
|
+
.where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "scheduled_retry"), lte(heartbeatRuns.scheduledRetryAt, now)))
|
|
3490
|
+
.returning()
|
|
3491
|
+
.then((rows) => rows[0] ?? null);
|
|
3492
|
+
if (!cancelled)
|
|
3493
|
+
return null;
|
|
3494
|
+
if (cancelled.wakeupRequestId) {
|
|
3495
|
+
await db
|
|
3496
|
+
.update(agentWakeupRequests)
|
|
3497
|
+
.set({
|
|
3498
|
+
status: "cancelled",
|
|
3499
|
+
finishedAt: now,
|
|
3500
|
+
error: gate.reason,
|
|
3501
|
+
updatedAt: now,
|
|
3502
|
+
})
|
|
3503
|
+
.where(eq(agentWakeupRequests.id, cancelled.wakeupRequestId));
|
|
3504
|
+
}
|
|
3505
|
+
if (gate.issueId) {
|
|
3506
|
+
await db
|
|
3507
|
+
.update(issues)
|
|
3508
|
+
.set({
|
|
3509
|
+
executionRunId: null,
|
|
3510
|
+
executionAgentNameKey: null,
|
|
3511
|
+
executionLockedAt: null,
|
|
3512
|
+
updatedAt: now,
|
|
3513
|
+
})
|
|
3514
|
+
.where(and(eq(issues.companyId, cancelled.companyId), eq(issues.id, gate.issueId), eq(issues.executionRunId, cancelled.id)));
|
|
3515
|
+
}
|
|
3516
|
+
await appendRunEvent(cancelled, await nextRunEventSeq(cancelled.id), {
|
|
3517
|
+
eventType: "lifecycle",
|
|
3518
|
+
stream: "system",
|
|
3519
|
+
level: "warn",
|
|
3520
|
+
message: gate.reason,
|
|
2625
3521
|
payload: {
|
|
2626
|
-
|
|
3522
|
+
...gate.details,
|
|
3523
|
+
scheduledRetryAttempt: cancelled.scheduledRetryAttempt,
|
|
3524
|
+
scheduledRetryAt: cancelled.scheduledRetryAt ? new Date(cancelled.scheduledRetryAt).toISOString() : null,
|
|
3525
|
+
scheduledRetryReason: cancelled.scheduledRetryReason,
|
|
2627
3526
|
},
|
|
2628
3527
|
});
|
|
2629
|
-
return
|
|
3528
|
+
return cancelled;
|
|
2630
3529
|
}
|
|
2631
3530
|
async function scheduleBoundedRetryForRun(run, agent, opts) {
|
|
2632
3531
|
const now = opts?.now ?? new Date();
|
|
2633
3532
|
const retryReason = opts?.retryReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON;
|
|
2634
3533
|
const wakeReason = opts?.wakeReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON;
|
|
3534
|
+
const maxAttempts = Math.max(0, Math.floor(opts?.maxAttempts ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS));
|
|
2635
3535
|
const nextAttempt = (run.scheduledRetryAttempt ?? 0) + 1;
|
|
2636
|
-
const baseSchedule =
|
|
3536
|
+
const baseSchedule = opts?.delayMs != null
|
|
3537
|
+
? nextAttempt <= maxAttempts
|
|
3538
|
+
? {
|
|
3539
|
+
attempt: nextAttempt,
|
|
3540
|
+
baseDelayMs: Math.max(0, Math.floor(opts.delayMs)),
|
|
3541
|
+
delayMs: Math.max(0, Math.floor(opts.delayMs)),
|
|
3542
|
+
dueAt: new Date(now.getTime() + Math.max(0, Math.floor(opts.delayMs))),
|
|
3543
|
+
maxAttempts,
|
|
3544
|
+
}
|
|
3545
|
+
: null
|
|
3546
|
+
: nextAttempt <= maxAttempts
|
|
3547
|
+
? computeBoundedTransientHeartbeatRetrySchedule(nextAttempt, now, opts?.random)
|
|
3548
|
+
: null;
|
|
2637
3549
|
const transientRecovery = retryReason === BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON
|
|
2638
3550
|
? readTransientRecoveryContractFromRun(run)
|
|
2639
3551
|
: null;
|
|
@@ -2650,13 +3562,13 @@ export function heartbeatService(db, options = {}) {
|
|
|
2650
3562
|
payload: {
|
|
2651
3563
|
retryReason,
|
|
2652
3564
|
scheduledRetryAttempt: run.scheduledRetryAttempt ?? 0,
|
|
2653
|
-
maxAttempts
|
|
3565
|
+
maxAttempts,
|
|
2654
3566
|
},
|
|
2655
3567
|
});
|
|
2656
3568
|
return {
|
|
2657
3569
|
outcome: "retry_exhausted",
|
|
2658
3570
|
attempt: nextAttempt,
|
|
2659
|
-
maxAttempts
|
|
3571
|
+
maxAttempts,
|
|
2660
3572
|
};
|
|
2661
3573
|
}
|
|
2662
3574
|
const schedule = transientRetryNotBefore && transientRetryNotBefore.getTime() > baseSchedule.dueAt.getTime()
|
|
@@ -2668,6 +3580,29 @@ export function heartbeatService(db, options = {}) {
|
|
|
2668
3580
|
: baseSchedule;
|
|
2669
3581
|
const contextSnapshot = parseObject(run.contextSnapshot);
|
|
2670
3582
|
const issueId = readNonEmptyString(contextSnapshot.issueId);
|
|
3583
|
+
if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON) {
|
|
3584
|
+
const gate = await evaluateScheduledRetryGate({ run, agent, contextSnapshot, retryReason });
|
|
3585
|
+
if (!gate.allowed) {
|
|
3586
|
+
await appendRunEvent(run, await nextRunEventSeq(run.id), {
|
|
3587
|
+
eventType: "lifecycle",
|
|
3588
|
+
stream: "system",
|
|
3589
|
+
level: "warn",
|
|
3590
|
+
message: gate.reason,
|
|
3591
|
+
payload: {
|
|
3592
|
+
retryReason,
|
|
3593
|
+
scheduledRetryAttempt: nextAttempt,
|
|
3594
|
+
maxAttempts,
|
|
3595
|
+
...gate.details,
|
|
3596
|
+
},
|
|
3597
|
+
});
|
|
3598
|
+
return {
|
|
3599
|
+
outcome: "not_scheduled",
|
|
3600
|
+
reason: gate.reason,
|
|
3601
|
+
errorCode: gate.errorCode,
|
|
3602
|
+
issueId: gate.issueId,
|
|
3603
|
+
};
|
|
3604
|
+
}
|
|
3605
|
+
}
|
|
2671
3606
|
const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null);
|
|
2672
3607
|
const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey);
|
|
2673
3608
|
const retryContextSnapshot = {
|
|
@@ -2681,7 +3616,113 @@ export function heartbeatService(db, options = {}) {
|
|
|
2681
3616
|
...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),
|
|
2682
3617
|
...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}),
|
|
2683
3618
|
};
|
|
2684
|
-
const
|
|
3619
|
+
const maxTurnContinuationIdempotencyKey = retryReason === MAX_TURN_CONTINUATION_RETRY_REASON
|
|
3620
|
+
? `max-turn-continuation:${run.companyId}:${issueId ?? "no-issue"}:${run.id}:${schedule.attempt}`
|
|
3621
|
+
: null;
|
|
3622
|
+
const scheduleResult = await db.transaction(async (tx) => {
|
|
3623
|
+
if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON) {
|
|
3624
|
+
if (issueId) {
|
|
3625
|
+
await tx.execute(sql `select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`);
|
|
3626
|
+
}
|
|
3627
|
+
else {
|
|
3628
|
+
await tx.execute(sql `select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`);
|
|
3629
|
+
}
|
|
3630
|
+
const existingContinuation = await tx
|
|
3631
|
+
.select()
|
|
3632
|
+
.from(heartbeatRuns)
|
|
3633
|
+
.where(and(eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.retryOfRunId, run.id), eq(heartbeatRuns.scheduledRetryReason, retryReason), eq(heartbeatRuns.scheduledRetryAttempt, schedule.attempt), inArray(heartbeatRuns.status, [...MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES]), issueId
|
|
3634
|
+
? sql `${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`
|
|
3635
|
+
: sql `${heartbeatRuns.contextSnapshot} ->> 'issueId' is null`))
|
|
3636
|
+
.orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id))
|
|
3637
|
+
.limit(1)
|
|
3638
|
+
.then((rows) => rows[0] ?? null);
|
|
3639
|
+
if (existingContinuation) {
|
|
3640
|
+
if (existingContinuation.wakeupRequestId) {
|
|
3641
|
+
const existingWakeup = await tx
|
|
3642
|
+
.select({ coalescedCount: agentWakeupRequests.coalescedCount })
|
|
3643
|
+
.from(agentWakeupRequests)
|
|
3644
|
+
.where(eq(agentWakeupRequests.id, existingContinuation.wakeupRequestId))
|
|
3645
|
+
.then((rows) => rows[0] ?? null);
|
|
3646
|
+
await tx
|
|
3647
|
+
.update(agentWakeupRequests)
|
|
3648
|
+
.set({
|
|
3649
|
+
coalescedCount: (existingWakeup?.coalescedCount ?? 0) + 1,
|
|
3650
|
+
updatedAt: now,
|
|
3651
|
+
})
|
|
3652
|
+
.where(eq(agentWakeupRequests.id, existingContinuation.wakeupRequestId));
|
|
3653
|
+
}
|
|
3654
|
+
return {
|
|
3655
|
+
outcome: "scheduled",
|
|
3656
|
+
run: existingContinuation,
|
|
3657
|
+
reusedExisting: true,
|
|
3658
|
+
};
|
|
3659
|
+
}
|
|
3660
|
+
if (issueId) {
|
|
3661
|
+
const lockedIssue = await tx
|
|
3662
|
+
.select({
|
|
3663
|
+
id: issues.id,
|
|
3664
|
+
status: issues.status,
|
|
3665
|
+
assigneeAgentId: issues.assigneeAgentId,
|
|
3666
|
+
executionRunId: issues.executionRunId,
|
|
3667
|
+
})
|
|
3668
|
+
.from(issues)
|
|
3669
|
+
.where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))
|
|
3670
|
+
.then((rows) => rows[0] ?? null);
|
|
3671
|
+
if (!lockedIssue) {
|
|
3672
|
+
return {
|
|
3673
|
+
outcome: "not_scheduled",
|
|
3674
|
+
reason: "Scheduled max-turn continuation suppressed because the target issue no longer exists",
|
|
3675
|
+
errorCode: "issue_not_found",
|
|
3676
|
+
issueId,
|
|
3677
|
+
details: { issueId },
|
|
3678
|
+
};
|
|
3679
|
+
}
|
|
3680
|
+
if (lockedIssue.assigneeAgentId !== run.agentId) {
|
|
3681
|
+
return {
|
|
3682
|
+
outcome: "not_scheduled",
|
|
3683
|
+
reason: "Scheduled max-turn continuation suppressed because issue ownership changed",
|
|
3684
|
+
errorCode: "issue_reassigned",
|
|
3685
|
+
issueId,
|
|
3686
|
+
details: {
|
|
3687
|
+
issueId,
|
|
3688
|
+
previousAssigneeAgentId: run.agentId,
|
|
3689
|
+
currentAssigneeAgentId: lockedIssue.assigneeAgentId,
|
|
3690
|
+
},
|
|
3691
|
+
};
|
|
3692
|
+
}
|
|
3693
|
+
if (lockedIssue.status === "cancelled" || lockedIssue.status === "done") {
|
|
3694
|
+
return {
|
|
3695
|
+
outcome: "not_scheduled",
|
|
3696
|
+
reason: `Scheduled max-turn continuation suppressed because issue reached terminal status (${lockedIssue.status})`,
|
|
3697
|
+
errorCode: lockedIssue.status === "cancelled" ? "issue_cancelled" : "issue_terminal_status",
|
|
3698
|
+
issueId,
|
|
3699
|
+
details: { issueId, currentStatus: lockedIssue.status },
|
|
3700
|
+
};
|
|
3701
|
+
}
|
|
3702
|
+
if (lockedIssue.status !== "in_progress") {
|
|
3703
|
+
return {
|
|
3704
|
+
outcome: "not_scheduled",
|
|
3705
|
+
reason: `Scheduled max-turn continuation suppressed because issue is no longer in_progress (current status: ${lockedIssue.status})`,
|
|
3706
|
+
errorCode: "issue_not_in_progress",
|
|
3707
|
+
issueId,
|
|
3708
|
+
details: { issueId, currentStatus: lockedIssue.status, requiredStatus: "in_progress" },
|
|
3709
|
+
};
|
|
3710
|
+
}
|
|
3711
|
+
if (lockedIssue.executionRunId !== run.id) {
|
|
3712
|
+
return {
|
|
3713
|
+
outcome: "not_scheduled",
|
|
3714
|
+
reason: "Scheduled max-turn continuation suppressed because the issue execution lock belongs to a different run",
|
|
3715
|
+
errorCode: "issue_execution_lock_changed",
|
|
3716
|
+
issueId,
|
|
3717
|
+
details: {
|
|
3718
|
+
issueId,
|
|
3719
|
+
expectedExecutionRunId: run.id,
|
|
3720
|
+
currentExecutionRunId: lockedIssue.executionRunId,
|
|
3721
|
+
},
|
|
3722
|
+
};
|
|
3723
|
+
}
|
|
3724
|
+
}
|
|
3725
|
+
}
|
|
2685
3726
|
const wakeupRequest = await tx
|
|
2686
3727
|
.insert(agentWakeupRequests)
|
|
2687
3728
|
.values({
|
|
@@ -2703,6 +3744,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
2703
3744
|
status: "queued",
|
|
2704
3745
|
requestedByActorType: "system",
|
|
2705
3746
|
requestedByActorId: null,
|
|
3747
|
+
idempotencyKey: maxTurnContinuationIdempotencyKey,
|
|
2706
3748
|
updatedAt: now,
|
|
2707
3749
|
})
|
|
2708
3750
|
.returning()
|
|
@@ -2745,8 +3787,57 @@ export function heartbeatService(db, options = {}) {
|
|
|
2745
3787
|
})
|
|
2746
3788
|
.where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId), eq(issues.executionRunId, run.id)));
|
|
2747
3789
|
}
|
|
2748
|
-
return
|
|
3790
|
+
return {
|
|
3791
|
+
outcome: "scheduled",
|
|
3792
|
+
run: scheduledRun,
|
|
3793
|
+
reusedExisting: false,
|
|
3794
|
+
};
|
|
2749
3795
|
});
|
|
3796
|
+
if (scheduleResult.outcome === "not_scheduled") {
|
|
3797
|
+
await appendRunEvent(run, await nextRunEventSeq(run.id), {
|
|
3798
|
+
eventType: "lifecycle",
|
|
3799
|
+
stream: "system",
|
|
3800
|
+
level: "warn",
|
|
3801
|
+
message: scheduleResult.reason,
|
|
3802
|
+
payload: {
|
|
3803
|
+
retryReason,
|
|
3804
|
+
scheduledRetryAttempt: nextAttempt,
|
|
3805
|
+
maxAttempts,
|
|
3806
|
+
...scheduleResult.details,
|
|
3807
|
+
},
|
|
3808
|
+
});
|
|
3809
|
+
return {
|
|
3810
|
+
outcome: "not_scheduled",
|
|
3811
|
+
reason: scheduleResult.reason,
|
|
3812
|
+
errorCode: scheduleResult.errorCode,
|
|
3813
|
+
issueId: scheduleResult.issueId,
|
|
3814
|
+
};
|
|
3815
|
+
}
|
|
3816
|
+
const retryRun = scheduleResult.run;
|
|
3817
|
+
const dueAt = retryRun.scheduledRetryAt ? new Date(retryRun.scheduledRetryAt) : schedule.dueAt;
|
|
3818
|
+
if (scheduleResult.reusedExisting) {
|
|
3819
|
+
await appendRunEvent(run, await nextRunEventSeq(run.id), {
|
|
3820
|
+
eventType: "lifecycle",
|
|
3821
|
+
stream: "system",
|
|
3822
|
+
level: "info",
|
|
3823
|
+
message: `Reused existing max-turn continuation ${retryRun.scheduledRetryAttempt}/${schedule.maxAttempts}`,
|
|
3824
|
+
payload: {
|
|
3825
|
+
retryRunId: retryRun.id,
|
|
3826
|
+
retryReason,
|
|
3827
|
+
idempotencyKey: maxTurnContinuationIdempotencyKey,
|
|
3828
|
+
scheduledRetryAttempt: retryRun.scheduledRetryAttempt,
|
|
3829
|
+
scheduledRetryAt: dueAt.toISOString(),
|
|
3830
|
+
},
|
|
3831
|
+
});
|
|
3832
|
+
return {
|
|
3833
|
+
outcome: "scheduled",
|
|
3834
|
+
run: retryRun,
|
|
3835
|
+
dueAt,
|
|
3836
|
+
attempt: retryRun.scheduledRetryAttempt,
|
|
3837
|
+
maxAttempts: schedule.maxAttempts,
|
|
3838
|
+
reusedExisting: true,
|
|
3839
|
+
};
|
|
3840
|
+
}
|
|
2750
3841
|
await appendRunEvent(run, await nextRunEventSeq(run.id), {
|
|
2751
3842
|
eventType: "lifecycle",
|
|
2752
3843
|
stream: "system",
|
|
@@ -2767,7 +3858,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
2767
3858
|
return {
|
|
2768
3859
|
outcome: "scheduled",
|
|
2769
3860
|
run: retryRun,
|
|
2770
|
-
dueAt
|
|
3861
|
+
dueAt,
|
|
2771
3862
|
attempt: schedule.attempt,
|
|
2772
3863
|
maxAttempts: schedule.maxAttempts,
|
|
2773
3864
|
};
|
|
@@ -2781,76 +3872,33 @@ export function heartbeatService(db, options = {}) {
|
|
|
2781
3872
|
.limit(50);
|
|
2782
3873
|
const promotedRunIds = [];
|
|
2783
3874
|
for (const dueRun of dueRuns) {
|
|
2784
|
-
const
|
|
2785
|
-
if (
|
|
2786
|
-
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
})
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
|
|
2800
|
-
|
|
2801
|
-
|
|
2802
|
-
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
|
|
2809
|
-
|
|
2810
|
-
|
|
2811
|
-
.returning()
|
|
2812
|
-
.then((rows) => rows[0] ?? null);
|
|
2813
|
-
if (!cancelled)
|
|
2814
|
-
continue;
|
|
2815
|
-
if (cancelled.wakeupRequestId) {
|
|
2816
|
-
await db
|
|
2817
|
-
.update(agentWakeupRequests)
|
|
2818
|
-
.set({
|
|
2819
|
-
status: "cancelled",
|
|
2820
|
-
finishedAt: now,
|
|
2821
|
-
error: reason,
|
|
2822
|
-
updatedAt: now,
|
|
2823
|
-
})
|
|
2824
|
-
.where(eq(agentWakeupRequests.id, cancelled.wakeupRequestId));
|
|
2825
|
-
}
|
|
2826
|
-
if (issue.executionRunId === cancelled.id) {
|
|
2827
|
-
await db
|
|
2828
|
-
.update(issues)
|
|
2829
|
-
.set({
|
|
2830
|
-
executionRunId: null,
|
|
2831
|
-
executionAgentNameKey: null,
|
|
2832
|
-
executionLockedAt: null,
|
|
2833
|
-
updatedAt: now,
|
|
2834
|
-
})
|
|
2835
|
-
.where(and(eq(issues.id, issue.id), eq(issues.executionRunId, cancelled.id)));
|
|
2836
|
-
}
|
|
2837
|
-
await appendRunEvent(cancelled, await nextRunEventSeq(cancelled.id), {
|
|
2838
|
-
eventType: "lifecycle",
|
|
2839
|
-
stream: "system",
|
|
2840
|
-
level: "warn",
|
|
2841
|
-
message: issueCancelled
|
|
2842
|
-
? "Scheduled retry cancelled because issue was cancelled before it became due"
|
|
2843
|
-
: "Scheduled retry cancelled because issue ownership changed before it became due",
|
|
2844
|
-
payload: {
|
|
2845
|
-
issueId: issue.id,
|
|
2846
|
-
issueStatus: issue.status,
|
|
2847
|
-
scheduledRetryAttempt: cancelled.scheduledRetryAttempt,
|
|
2848
|
-
scheduledRetryAt: cancelled.scheduledRetryAt ? new Date(cancelled.scheduledRetryAt).toISOString() : null,
|
|
2849
|
-
scheduledRetryReason: cancelled.scheduledRetryReason,
|
|
2850
|
-
previousRetryAgentId: cancelled.agentId,
|
|
2851
|
-
currentAssigneeAgentId: issue.assigneeAgentId,
|
|
2852
|
-
},
|
|
2853
|
-
});
|
|
3875
|
+
const agent = await getAgent(dueRun.agentId);
|
|
3876
|
+
if (!agent) {
|
|
3877
|
+
await cancelScheduledRetryForGate(dueRun, {
|
|
3878
|
+
allowed: false,
|
|
3879
|
+
reason: "Scheduled retry suppressed because the agent no longer exists",
|
|
3880
|
+
errorCode: "agent_not_invokable",
|
|
3881
|
+
issueId: readNonEmptyString(parseObject(dueRun.contextSnapshot).issueId),
|
|
3882
|
+
details: { agentId: dueRun.agentId },
|
|
3883
|
+
}, now);
|
|
3884
|
+
continue;
|
|
3885
|
+
}
|
|
3886
|
+
const contextSnapshot = parseObject(dueRun.contextSnapshot);
|
|
3887
|
+
const gate = await evaluateScheduledRetryGate({
|
|
3888
|
+
run: dueRun,
|
|
3889
|
+
agent,
|
|
3890
|
+
contextSnapshot,
|
|
3891
|
+
retryReason: dueRun.scheduledRetryReason,
|
|
3892
|
+
enforceIssueExecutionLock: dueRun.scheduledRetryReason === MAX_TURN_CONTINUATION_RETRY_REASON,
|
|
3893
|
+
});
|
|
3894
|
+
if (!gate.allowed) {
|
|
3895
|
+
if (gate.errorCode === "issue_not_found" &&
|
|
3896
|
+
dueRun.scheduledRetryReason !== MAX_TURN_CONTINUATION_RETRY_REASON) {
|
|
3897
|
+
// Preserve legacy transient retry behavior for runs that only carry a
|
|
3898
|
+
// loose task context rather than a persisted issue row.
|
|
3899
|
+
}
|
|
3900
|
+
else {
|
|
3901
|
+
await cancelScheduledRetryForGate(dueRun, gate, now);
|
|
2854
3902
|
continue;
|
|
2855
3903
|
}
|
|
2856
3904
|
}
|
|
@@ -2904,6 +3952,18 @@ export function heartbeatService(db, options = {}) {
|
|
|
2904
3952
|
maxConcurrentRuns: normalizeMaxConcurrentRuns(heartbeat.maxConcurrentRuns),
|
|
2905
3953
|
};
|
|
2906
3954
|
}
|
|
3955
|
+
function parseMaxTurnContinuationPolicy(agent) {
|
|
3956
|
+
const runtimeConfig = parseObject(agent.runtimeConfig);
|
|
3957
|
+
const heartbeat = parseObject(runtimeConfig.heartbeat);
|
|
3958
|
+
const configured = parseObject(heartbeat.maxTurnContinuation);
|
|
3959
|
+
const rawMaxAttempts = Math.floor(asNumber(configured.maxAttempts, MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS));
|
|
3960
|
+
const rawDelayMs = Math.floor(asNumber(configured.delayMs, MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS));
|
|
3961
|
+
return {
|
|
3962
|
+
enabled: asBoolean(configured.enabled, true),
|
|
3963
|
+
maxAttempts: Math.max(0, Math.min(MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP, rawMaxAttempts)),
|
|
3964
|
+
delayMs: Math.max(0, Math.min(MAX_TURN_CONTINUATION_MAX_DELAY_MS, rawDelayMs)),
|
|
3965
|
+
};
|
|
3966
|
+
}
|
|
2907
3967
|
function issueRunPriorityRank(priority) {
|
|
2908
3968
|
switch (priority) {
|
|
2909
3969
|
case "critical":
|
|
@@ -2995,6 +4055,12 @@ export function heartbeatService(db, options = {}) {
|
|
|
2995
4055
|
logger.info({ runId: run.id, issueId, unresolvedBlockerCount }, "claimQueuedRun: cancelled blocked queued run");
|
|
2996
4056
|
return null;
|
|
2997
4057
|
}
|
|
4058
|
+
const staleness = await evaluateQueuedRunStaleness(run, issueId, context);
|
|
4059
|
+
if (staleness.stale) {
|
|
4060
|
+
await cancelQueuedRunForStaleIssue(run, issueId, staleness);
|
|
4061
|
+
logger.info({ runId: run.id, issueId, errorCode: staleness.errorCode }, "claimQueuedRun: cancelled stale queued run");
|
|
4062
|
+
return null;
|
|
4063
|
+
}
|
|
2998
4064
|
}
|
|
2999
4065
|
const claimedAt = new Date();
|
|
3000
4066
|
const claimed = await db
|
|
@@ -3089,6 +4155,132 @@ export function heartbeatService(db, options = {}) {
|
|
|
3089
4155
|
});
|
|
3090
4156
|
return cancelled;
|
|
3091
4157
|
}
|
|
4158
|
+
async function evaluateQueuedRunStaleness(run, issueId, context) {
|
|
4159
|
+
const issue = await db
|
|
4160
|
+
.select({
|
|
4161
|
+
id: issues.id,
|
|
4162
|
+
status: issues.status,
|
|
4163
|
+
assigneeAgentId: issues.assigneeAgentId,
|
|
4164
|
+
executionRunId: issues.executionRunId,
|
|
4165
|
+
executionState: issues.executionState,
|
|
4166
|
+
})
|
|
4167
|
+
.from(issues)
|
|
4168
|
+
.where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))
|
|
4169
|
+
.then((rows) => rows[0] ?? null);
|
|
4170
|
+
if (!issue) {
|
|
4171
|
+
return {
|
|
4172
|
+
stale: true,
|
|
4173
|
+
errorCode: "issue_not_found",
|
|
4174
|
+
reason: "Cancelled because the target issue no longer exists",
|
|
4175
|
+
details: { issueId },
|
|
4176
|
+
};
|
|
4177
|
+
}
|
|
4178
|
+
const wakeCommentId = deriveCommentId(context, null);
|
|
4179
|
+
const isInteractionWake = allowsIssueInteractionWake(context);
|
|
4180
|
+
const resumeIntent = context.resumeIntent === true || context.followUpRequested === true;
|
|
4181
|
+
const retryReason = readNonEmptyString(context.retryReason) ?? run.scheduledRetryReason ?? null;
|
|
4182
|
+
if (issue.assigneeAgentId !== run.agentId && !isInteractionWake) {
|
|
4183
|
+
return {
|
|
4184
|
+
stale: true,
|
|
4185
|
+
errorCode: "issue_assignee_changed",
|
|
4186
|
+
reason: "Cancelled because issue assignee changed before the queued run could start; the new owner will be woken instead",
|
|
4187
|
+
details: {
|
|
4188
|
+
issueId,
|
|
4189
|
+
previousAssigneeAgentId: run.agentId,
|
|
4190
|
+
currentAssigneeAgentId: issue.assigneeAgentId,
|
|
4191
|
+
},
|
|
4192
|
+
};
|
|
4193
|
+
}
|
|
4194
|
+
if (issue.status === "done" || issue.status === "cancelled") {
|
|
4195
|
+
if (!resumeIntent && !wakeCommentId) {
|
|
4196
|
+
return {
|
|
4197
|
+
stale: true,
|
|
4198
|
+
errorCode: "issue_terminal_status",
|
|
4199
|
+
reason: `Cancelled because issue reached terminal status (${issue.status}) before the queued run could start`,
|
|
4200
|
+
details: { issueId, currentStatus: issue.status },
|
|
4201
|
+
};
|
|
4202
|
+
}
|
|
4203
|
+
}
|
|
4204
|
+
if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON && issue.status !== "in_progress") {
|
|
4205
|
+
return {
|
|
4206
|
+
stale: true,
|
|
4207
|
+
errorCode: "issue_not_in_progress",
|
|
4208
|
+
reason: `Cancelled because max-turn continuation issue is no longer in_progress (current status: ${issue.status}) before the queued run could start`,
|
|
4209
|
+
details: { issueId, currentStatus: issue.status, requiredStatus: "in_progress" },
|
|
4210
|
+
};
|
|
4211
|
+
}
|
|
4212
|
+
if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON && issue.executionRunId !== run.id) {
|
|
4213
|
+
return {
|
|
4214
|
+
stale: true,
|
|
4215
|
+
errorCode: "issue_execution_lock_changed",
|
|
4216
|
+
reason: "Cancelled because max-turn continuation no longer owns the issue execution lock before the queued run could start",
|
|
4217
|
+
details: {
|
|
4218
|
+
issueId,
|
|
4219
|
+
expectedExecutionRunId: run.id,
|
|
4220
|
+
currentExecutionRunId: issue.executionRunId,
|
|
4221
|
+
},
|
|
4222
|
+
};
|
|
4223
|
+
}
|
|
4224
|
+
if (issue.status === "in_review") {
|
|
4225
|
+
const executionState = parseIssueExecutionState(issue.executionState);
|
|
4226
|
+
const currentParticipant = executionState?.currentParticipant ?? null;
|
|
4227
|
+
if (currentParticipant) {
|
|
4228
|
+
const participantMatches = currentParticipant.type === "agent" && currentParticipant.agentId === run.agentId;
|
|
4229
|
+
if (!participantMatches && !wakeCommentId) {
|
|
4230
|
+
return {
|
|
4231
|
+
stale: true,
|
|
4232
|
+
errorCode: "issue_review_participant_changed",
|
|
4233
|
+
reason: "Cancelled because the in-review participant changed before the queued run could start; the current participant will be woken instead",
|
|
4234
|
+
details: {
|
|
4235
|
+
issueId,
|
|
4236
|
+
currentStageType: executionState?.currentStageType ?? null,
|
|
4237
|
+
currentParticipant,
|
|
4238
|
+
},
|
|
4239
|
+
};
|
|
4240
|
+
}
|
|
4241
|
+
}
|
|
4242
|
+
}
|
|
4243
|
+
return { stale: false };
|
|
4244
|
+
}
|
|
4245
|
+
async function cancelQueuedRunForStaleIssue(run, issueId, staleness) {
|
|
4246
|
+
const now = new Date();
|
|
4247
|
+
const cancelled = await setRunStatus(run.id, "cancelled", {
|
|
4248
|
+
finishedAt: now,
|
|
4249
|
+
error: staleness.reason,
|
|
4250
|
+
errorCode: staleness.errorCode,
|
|
4251
|
+
resultJson: {
|
|
4252
|
+
...parseObject(run.resultJson),
|
|
4253
|
+
stopReason: staleness.errorCode,
|
|
4254
|
+
effectiveTimeoutSec: 0,
|
|
4255
|
+
timeoutConfigured: false,
|
|
4256
|
+
timeoutSource: "stale_queued_run_gate",
|
|
4257
|
+
timeoutFired: false,
|
|
4258
|
+
},
|
|
4259
|
+
});
|
|
4260
|
+
if (!cancelled)
|
|
4261
|
+
return null;
|
|
4262
|
+
await setWakeupStatus(run.wakeupRequestId, "skipped", {
|
|
4263
|
+
finishedAt: now,
|
|
4264
|
+
error: staleness.reason,
|
|
4265
|
+
});
|
|
4266
|
+
await db
|
|
4267
|
+
.update(issues)
|
|
4268
|
+
.set({
|
|
4269
|
+
executionRunId: null,
|
|
4270
|
+
executionAgentNameKey: null,
|
|
4271
|
+
executionLockedAt: null,
|
|
4272
|
+
updatedAt: now,
|
|
4273
|
+
})
|
|
4274
|
+
.where(and(eq(issues.companyId, run.companyId), eq(issues.id, issueId), eq(issues.executionRunId, run.id)));
|
|
4275
|
+
await appendRunEvent(cancelled, await nextRunEventSeq(cancelled.id), {
|
|
4276
|
+
eventType: "lifecycle",
|
|
4277
|
+
stream: "system",
|
|
4278
|
+
level: "warn",
|
|
4279
|
+
message: staleness.reason,
|
|
4280
|
+
payload: staleness.details,
|
|
4281
|
+
});
|
|
4282
|
+
return cancelled;
|
|
4283
|
+
}
|
|
3092
4284
|
async function finalizeAgentStatus(agentId, outcome) {
|
|
3093
4285
|
const existing = await getAgent(agentId);
|
|
3094
4286
|
if (!existing)
|
|
@@ -3359,6 +4551,13 @@ export function heartbeatService(db, options = {}) {
|
|
|
3359
4551
|
if (!finalizedRun)
|
|
3360
4552
|
continue;
|
|
3361
4553
|
finalizedRun = await classifyAndPersistRunLiveness(finalizedRun, parseObject(finalizedRun.resultJson)) ?? finalizedRun;
|
|
4554
|
+
await releaseEnvironmentLeasesForRun({
|
|
4555
|
+
runId: finalizedRun.id,
|
|
4556
|
+
companyId: finalizedRun.companyId,
|
|
4557
|
+
agentId: finalizedRun.agentId,
|
|
4558
|
+
status: finalizedRun.status,
|
|
4559
|
+
failureReason: finalizedRun.error ?? undefined,
|
|
4560
|
+
});
|
|
3362
4561
|
let retriedRun = null;
|
|
3363
4562
|
if (shouldRetry) {
|
|
3364
4563
|
const agent = await getAgent(run.agentId);
|
|
@@ -3420,9 +4619,15 @@ export function heartbeatService(db, options = {}) {
|
|
|
3420
4619
|
async function scanSilentActiveRuns(opts) {
|
|
3421
4620
|
return recovery.scanSilentActiveRuns(opts);
|
|
3422
4621
|
}
|
|
4622
|
+
async function reconcileProductivityReviews(opts) {
|
|
4623
|
+
return productivityReviews.reconcileProductivityReviews(opts);
|
|
4624
|
+
}
|
|
3423
4625
|
async function buildRunOutputSilence(run, now = new Date()) {
|
|
3424
4626
|
return recovery.buildRunOutputSilence(run, now);
|
|
3425
4627
|
}
|
|
4628
|
+
async function buildIssueGraphLivenessAutoRecoveryPreview(opts) {
|
|
4629
|
+
return recovery.buildIssueGraphLivenessAutoRecoveryPreview(opts);
|
|
4630
|
+
}
|
|
3426
4631
|
async function reconcileIssueGraphLiveness(opts) {
|
|
3427
4632
|
return recovery.reconcileIssueGraphLiveness(opts);
|
|
3428
4633
|
}
|
|
@@ -3743,6 +4948,9 @@ export function heartbeatService(db, options = {}) {
|
|
|
3743
4948
|
const shouldReuseExisting = issueRef?.executionWorkspacePreference === "reuse_existing" &&
|
|
3744
4949
|
existingExecutionWorkspace !== null &&
|
|
3745
4950
|
existingExecutionWorkspace.status !== "archived";
|
|
4951
|
+
const reusableExecutionWorkspaceConfig = shouldReuseExisting
|
|
4952
|
+
? existingExecutionWorkspace?.config ?? null
|
|
4953
|
+
: null;
|
|
3746
4954
|
const persistedExecutionWorkspaceMode = shouldReuseExisting && existingExecutionWorkspace
|
|
3747
4955
|
? issueExecutionWorkspaceModeForPersistedWorkspace(existingExecutionWorkspace.mode)
|
|
3748
4956
|
: null;
|
|
@@ -3755,7 +4963,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
3755
4963
|
const selectedEnvironmentId = resolveExecutionWorkspaceEnvironmentId({
|
|
3756
4964
|
projectPolicy: projectExecutionWorkspacePolicy,
|
|
3757
4965
|
issueSettings: issueExecutionWorkspaceSettings,
|
|
3758
|
-
workspaceConfig:
|
|
4966
|
+
workspaceConfig: reusableExecutionWorkspaceConfig,
|
|
3759
4967
|
agentDefaultEnvironmentId: agent.defaultEnvironmentId,
|
|
3760
4968
|
defaultEnvironmentId: defaultEnvironment.id,
|
|
3761
4969
|
});
|
|
@@ -3770,12 +4978,45 @@ export function heartbeatService(db, options = {}) {
|
|
|
3770
4978
|
});
|
|
3771
4979
|
const persistedWorkspaceManagedConfig = applyPersistedExecutionWorkspaceConfig({
|
|
3772
4980
|
config: workspaceManagedConfig,
|
|
3773
|
-
workspaceConfig:
|
|
4981
|
+
workspaceConfig: reusableExecutionWorkspaceConfig,
|
|
3774
4982
|
mode: effectiveExecutionWorkspaceMode,
|
|
3775
4983
|
});
|
|
3776
|
-
|
|
3777
|
-
|
|
3778
|
-
|
|
4984
|
+
let adapterModelProfiles = [];
|
|
4985
|
+
let profileResolutionFallbackReason = null;
|
|
4986
|
+
try {
|
|
4987
|
+
adapterModelProfiles = await listAdapterModelProfiles(agent.adapterType);
|
|
4988
|
+
}
|
|
4989
|
+
catch (error) {
|
|
4990
|
+
profileResolutionFallbackReason = "adapter_profile_resolution_failed";
|
|
4991
|
+
logger.warn({
|
|
4992
|
+
err: error,
|
|
4993
|
+
companyId: agent.companyId,
|
|
4994
|
+
agentId: agent.id,
|
|
4995
|
+
adapterType: agent.adapterType,
|
|
4996
|
+
runId: run.id,
|
|
4997
|
+
}, "Failed to resolve adapter model profiles; falling back to primary adapter config");
|
|
4998
|
+
}
|
|
4999
|
+
const modelProfileApplication = resolveModelProfileApplication({
|
|
5000
|
+
adapterModelProfiles,
|
|
5001
|
+
agentRuntimeConfig: agent.runtimeConfig,
|
|
5002
|
+
issueModelProfile: issueAssigneeOverrides?.modelProfile ?? null,
|
|
5003
|
+
contextSnapshot: context,
|
|
5004
|
+
profileResolutionFallbackReason,
|
|
5005
|
+
});
|
|
5006
|
+
const modelProfileMetadata = modelProfileRunMetadata(modelProfileApplication);
|
|
5007
|
+
if (modelProfileMetadata) {
|
|
5008
|
+
context.paperclipModelProfile = modelProfileMetadata;
|
|
5009
|
+
if (modelProfileApplication.requested)
|
|
5010
|
+
context.modelProfile = modelProfileApplication.requested;
|
|
5011
|
+
}
|
|
5012
|
+
else {
|
|
5013
|
+
delete context.paperclipModelProfile;
|
|
5014
|
+
}
|
|
5015
|
+
const mergedConfig = mergeModelProfileAdapterConfig({
|
|
5016
|
+
baseConfig: persistedWorkspaceManagedConfig,
|
|
5017
|
+
modelProfile: modelProfileApplication,
|
|
5018
|
+
issueAdapterConfig: issueAssigneeOverrides?.adapterConfig ?? null,
|
|
5019
|
+
});
|
|
3779
5020
|
const configSnapshot = buildExecutionWorkspaceConfigSnapshot(mergedConfig, selectedEnvironmentId);
|
|
3780
5021
|
const executionRunConfig = stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig);
|
|
3781
5022
|
const { resolvedConfig, secretKeys } = await resolveExecutionRunAdapterConfig({
|
|
@@ -4298,12 +5539,16 @@ export function heartbeatService(db, options = {}) {
|
|
|
4298
5539
|
meta.env[key] = "***REDACTED***";
|
|
4299
5540
|
}
|
|
4300
5541
|
}
|
|
5542
|
+
const modelProfileMetadata = modelProfileRunMetadata(modelProfileApplication);
|
|
4301
5543
|
await appendRunEvent(currentRun, seq++, {
|
|
4302
5544
|
eventType: "adapter.invoke",
|
|
4303
5545
|
stream: "system",
|
|
4304
5546
|
level: "info",
|
|
4305
5547
|
message: "adapter invocation",
|
|
4306
|
-
payload:
|
|
5548
|
+
payload: {
|
|
5549
|
+
...meta,
|
|
5550
|
+
...(modelProfileMetadata ? { modelProfile: modelProfileMetadata } : {}),
|
|
5551
|
+
},
|
|
4307
5552
|
});
|
|
4308
5553
|
};
|
|
4309
5554
|
const adapter = getServerAdapter(agent.adapterType);
|
|
@@ -4324,6 +5569,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
4324
5569
|
runtime: runtimeForAdapter,
|
|
4325
5570
|
config: runtimeConfig,
|
|
4326
5571
|
context,
|
|
5572
|
+
runtimeCommandSpec: adapter.getRuntimeCommandSpec?.(runtimeConfig) ?? null,
|
|
4327
5573
|
executionTarget,
|
|
4328
5574
|
executionTransport: remoteExecution
|
|
4329
5575
|
? { remoteExecution: remoteExecution }
|
|
@@ -4465,11 +5711,11 @@ export function heartbeatService(db, options = {}) {
|
|
|
4465
5711
|
}
|
|
4466
5712
|
: null;
|
|
4467
5713
|
const persistedResultJson = mergeHeartbeatRunResultJson(mergeRunStopMetadataForAgent(agent, outcome, {
|
|
4468
|
-
resultJson: mergeAdapterRecoveryMetadata({
|
|
5714
|
+
resultJson: mergeModelProfileRunMetadata(mergeAdapterRecoveryMetadata({
|
|
4469
5715
|
resultJson: adapterResult.resultJson ?? null,
|
|
4470
5716
|
errorFamily: adapterResult.errorFamily ?? null,
|
|
4471
5717
|
retryNotBefore: adapterResult.retryNotBefore ?? null,
|
|
4472
|
-
}),
|
|
5718
|
+
}), modelProfileApplication),
|
|
4473
5719
|
errorCode: runErrorCode,
|
|
4474
5720
|
errorMessage: runErrorMessage,
|
|
4475
5721
|
}), adapterResult.summary ?? null);
|
|
@@ -4523,7 +5769,30 @@ export function heartbeatService(db, options = {}) {
|
|
|
4523
5769
|
await onLog("stderr", `[paperclip] Failed to post run summary comment: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
4524
5770
|
}
|
|
4525
5771
|
}
|
|
4526
|
-
if (outcome === "failed" &&
|
|
5772
|
+
if (outcome === "failed" && isMaxTurnExhaustionRun(livenessRun)) {
|
|
5773
|
+
const policy = parseMaxTurnContinuationPolicy(agent);
|
|
5774
|
+
if (policy.enabled && policy.maxAttempts > 0) {
|
|
5775
|
+
await scheduleBoundedRetryForRun(livenessRun, agent, {
|
|
5776
|
+
retryReason: MAX_TURN_CONTINUATION_RETRY_REASON,
|
|
5777
|
+
wakeReason: MAX_TURN_CONTINUATION_WAKE_REASON,
|
|
5778
|
+
maxAttempts: policy.maxAttempts,
|
|
5779
|
+
delayMs: policy.delayMs,
|
|
5780
|
+
});
|
|
5781
|
+
}
|
|
5782
|
+
else {
|
|
5783
|
+
await appendRunEvent(livenessRun, await nextRunEventSeq(livenessRun.id), {
|
|
5784
|
+
eventType: "lifecycle",
|
|
5785
|
+
stream: "system",
|
|
5786
|
+
level: "warn",
|
|
5787
|
+
message: "Max-turn continuation suppressed because the policy is disabled",
|
|
5788
|
+
payload: {
|
|
5789
|
+
retryReason: MAX_TURN_CONTINUATION_RETRY_REASON,
|
|
5790
|
+
policy,
|
|
5791
|
+
},
|
|
5792
|
+
});
|
|
5793
|
+
}
|
|
5794
|
+
}
|
|
5795
|
+
else if (outcome === "failed" && readTransientRecoveryContractFromRun(livenessRun)) {
|
|
4527
5796
|
await scheduleBoundedRetryForRun(livenessRun, agent);
|
|
4528
5797
|
}
|
|
4529
5798
|
await finalizeIssueCommentPolicy(livenessRun, agent);
|
|
@@ -4674,19 +5943,13 @@ export function heartbeatService(db, options = {}) {
|
|
|
4674
5943
|
}
|
|
4675
5944
|
finally {
|
|
4676
5945
|
const latestRun = await getRun(run.id).catch(() => null);
|
|
4677
|
-
|
|
4678
|
-
|
|
5946
|
+
await releaseEnvironmentLeasesForRun({
|
|
5947
|
+
runId: run.id,
|
|
4679
5948
|
companyId: run.companyId,
|
|
4680
5949
|
agentId: run.agentId,
|
|
4681
|
-
status:
|
|
5950
|
+
status: latestRun?.status,
|
|
4682
5951
|
failureReason: latestRun?.error ?? undefined,
|
|
4683
|
-
}).catch((err) => {
|
|
4684
|
-
logger.warn({ err, runId: run.id }, "failed to release environment leases for heartbeat run");
|
|
4685
|
-
return null;
|
|
4686
5952
|
});
|
|
4687
|
-
for (const releaseError of releaseResult?.errors ?? []) {
|
|
4688
|
-
logger.warn({ err: releaseError.error, leaseId: releaseError.leaseId, runId: run.id }, "failed to release environment lease for heartbeat run");
|
|
4689
|
-
}
|
|
4690
5953
|
await releaseRuntimeServicesForRun(run.id).catch(() => undefined);
|
|
4691
5954
|
activeRunExecutions.delete(run.id);
|
|
4692
5955
|
await startNextQueuedRunForAgent(run.agentId);
|
|
@@ -4933,7 +6196,8 @@ export function heartbeatService(db, options = {}) {
|
|
|
4933
6196
|
if (await isAutomaticRecoverySuppressedByPauseHold(db, issue.companyId, issue.id, treeControlSvc)) {
|
|
4934
6197
|
return { kind: "released" };
|
|
4935
6198
|
}
|
|
4936
|
-
const shouldBlockImmediately =
|
|
6199
|
+
const shouldBlockImmediately = issue.originKind === RECOVERY_ORIGIN_KINDS.strandedIssueRecovery ||
|
|
6200
|
+
!recoveryAgentInvokable ||
|
|
4937
6201
|
!recoveryAgent ||
|
|
4938
6202
|
didAutomaticRecoveryFail(run, issue.status === "todo" ? "assignment_recovery" : "issue_continuation_needed");
|
|
4939
6203
|
if (shouldBlockImmediately) {
|
|
@@ -5951,6 +7215,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
5951
7215
|
requestedByActorId: actor?.actorId ?? null,
|
|
5952
7216
|
}),
|
|
5953
7217
|
wakeup: enqueueWakeup,
|
|
7218
|
+
triggerIssueMonitor,
|
|
5954
7219
|
reportRunActivity: clearDetachedRunWarning,
|
|
5955
7220
|
reapOrphanedRuns,
|
|
5956
7221
|
promoteDueScheduledRetries,
|
|
@@ -5965,8 +7230,10 @@ export function heartbeatService(db, options = {}) {
|
|
|
5965
7230
|
return scheduleBoundedRetryForRun(run, agent, opts);
|
|
5966
7231
|
},
|
|
5967
7232
|
reconcileStrandedAssignedIssues,
|
|
7233
|
+
buildIssueGraphLivenessAutoRecoveryPreview,
|
|
5968
7234
|
reconcileIssueGraphLiveness,
|
|
5969
7235
|
scanSilentActiveRuns,
|
|
7236
|
+
reconcileProductivityReviews,
|
|
5970
7237
|
buildRunOutputSilence,
|
|
5971
7238
|
tickTimers: async (now = new Date()) => {
|
|
5972
7239
|
const allAgents = await db.select().from(agents);
|
|
@@ -6001,7 +7268,12 @@ export function heartbeatService(db, options = {}) {
|
|
|
6001
7268
|
else
|
|
6002
7269
|
skipped += 1;
|
|
6003
7270
|
}
|
|
6004
|
-
|
|
7271
|
+
const issueMonitors = await tickDueIssueMonitors(now);
|
|
7272
|
+
return {
|
|
7273
|
+
checked: checked + issueMonitors.checked,
|
|
7274
|
+
enqueued: enqueued + issueMonitors.triggered,
|
|
7275
|
+
skipped: skipped + issueMonitors.skipped,
|
|
7276
|
+
};
|
|
6005
7277
|
},
|
|
6006
7278
|
cancelRun: (runId) => cancelRunInternal(runId),
|
|
6007
7279
|
cancelActiveForAgent: (agentId) => cancelActiveForAgentInternal(agentId),
|