@penclipai/server 2026.704.0 → 2026.714.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/index.d.ts +1 -1
- package/dist/adapters/index.d.ts.map +1 -1
- package/dist/adapters/index.js.map +1 -1
- package/dist/adapters/process/execute.d.ts.map +1 -1
- package/dist/adapters/process/execute.js +1 -0
- package/dist/adapters/process/execute.js.map +1 -1
- package/dist/adapters/registry.d.ts.map +1 -1
- package/dist/adapters/registry.js +79 -51
- package/dist/adapters/registry.js.map +1 -1
- package/dist/adapters/utils.d.ts +5 -0
- package/dist/adapters/utils.d.ts.map +1 -1
- package/dist/adapters/utils.js.map +1 -1
- package/dist/agent-auth-jwt.d.ts +5 -1
- package/dist/agent-auth-jwt.d.ts.map +1 -1
- package/dist/agent-auth-jwt.js +72 -20
- package/dist/agent-auth-jwt.js.map +1 -1
- package/dist/app.d.ts +2 -0
- package/dist/app.d.ts.map +1 -1
- package/dist/app.js +10 -0
- package/dist/app.js.map +1 -1
- package/dist/built-ins/agents/reflection-coach/AGENTS.md +47 -0
- package/dist/built-ins/agents/reflection-coach/routines/recent-agent-reflection.md +77 -0
- package/dist/errors.d.ts +1 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +2 -2
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +208 -116
- package/dist/index.js.map +1 -1
- package/dist/middleware/api-compression.d.ts +8 -0
- package/dist/middleware/api-compression.d.ts.map +1 -0
- package/dist/middleware/api-compression.js +207 -0
- package/dist/middleware/api-compression.js.map +1 -0
- package/dist/middleware/auth.d.ts.map +1 -1
- package/dist/middleware/auth.js +130 -3
- package/dist/middleware/auth.js.map +1 -1
- package/dist/middleware/error-handler.d.ts.map +1 -1
- package/dist/middleware/error-handler.js +27 -0
- package/dist/middleware/error-handler.js.map +1 -1
- package/dist/onboarding-assets/default/AGENTS.md +1 -0
- package/dist/redaction.d.ts.map +1 -1
- package/dist/redaction.js +11 -0
- package/dist/redaction.js.map +1 -1
- package/dist/routes/access.d.ts.map +1 -1
- package/dist/routes/access.js +4 -3
- package/dist/routes/access.js.map +1 -1
- package/dist/routes/adapters.d.ts.map +1 -1
- package/dist/routes/adapters.js +2 -0
- package/dist/routes/adapters.js.map +1 -1
- package/dist/routes/agents.d.ts.map +1 -1
- package/dist/routes/agents.js +222 -87
- package/dist/routes/agents.js.map +1 -1
- package/dist/routes/attention.d.ts +3 -0
- package/dist/routes/attention.d.ts.map +1 -0
- package/dist/routes/attention.js +24 -0
- package/dist/routes/attention.js.map +1 -0
- package/dist/routes/authz.d.ts.map +1 -1
- package/dist/routes/authz.js +29 -1
- package/dist/routes/authz.js.map +1 -1
- package/dist/routes/board-chat.d.ts.map +1 -1
- package/dist/routes/board-chat.js +4 -1
- package/dist/routes/board-chat.js.map +1 -1
- package/dist/routes/built-in-agents.d.ts +3 -0
- package/dist/routes/built-in-agents.d.ts.map +1 -0
- package/dist/routes/built-in-agents.js +267 -0
- package/dist/routes/built-in-agents.js.map +1 -0
- package/dist/routes/cases.d.ts +46 -0
- package/dist/routes/cases.d.ts.map +1 -0
- package/dist/routes/cases.js +1395 -0
- package/dist/routes/cases.js.map +1 -0
- package/dist/routes/companies.d.ts.map +1 -1
- package/dist/routes/companies.js +4 -1
- package/dist/routes/companies.js.map +1 -1
- package/dist/routes/company-skills.d.ts.map +1 -1
- package/dist/routes/company-skills.js +453 -33
- package/dist/routes/company-skills.js.map +1 -1
- package/dist/routes/environments.d.ts.map +1 -1
- package/dist/routes/environments.js +71 -8
- package/dist/routes/environments.js.map +1 -1
- package/dist/routes/execution-workspaces.d.ts.map +1 -1
- package/dist/routes/execution-workspaces.js +89 -3
- package/dist/routes/execution-workspaces.js.map +1 -1
- package/dist/routes/health.d.ts +2 -0
- package/dist/routes/health.d.ts.map +1 -1
- package/dist/routes/health.js +30 -0
- package/dist/routes/health.js.map +1 -1
- package/dist/routes/inbox-dismissals.d.ts.map +1 -1
- package/dist/routes/inbox-dismissals.js +80 -19
- package/dist/routes/inbox-dismissals.js.map +1 -1
- package/dist/routes/index.d.ts +2 -0
- package/dist/routes/index.d.ts.map +1 -1
- package/dist/routes/index.js +2 -0
- package/dist/routes/index.js.map +1 -1
- package/dist/routes/issues.d.ts +25 -0
- package/dist/routes/issues.d.ts.map +1 -1
- package/dist/routes/issues.js +1455 -46
- package/dist/routes/issues.js.map +1 -1
- package/dist/routes/openapi.d.ts.map +1 -1
- package/dist/routes/openapi.js +448 -11
- package/dist/routes/openapi.js.map +1 -1
- package/dist/routes/pipelines.js +2 -2
- package/dist/routes/pipelines.js.map +1 -1
- package/dist/routes/resource-memberships.d.ts.map +1 -1
- package/dist/routes/resource-memberships.js +13 -5
- package/dist/routes/resource-memberships.js.map +1 -1
- package/dist/routes/secrets.d.ts.map +1 -1
- package/dist/routes/secrets.js +261 -1
- package/dist/routes/secrets.js.map +1 -1
- package/dist/routes/sidebar-badges.d.ts.map +1 -1
- package/dist/routes/sidebar-badges.js +18 -2
- package/dist/routes/sidebar-badges.js.map +1 -1
- package/dist/routes/user-profiles.d.ts.map +1 -1
- package/dist/routes/user-profiles.js +5 -4
- package/dist/routes/user-profiles.js.map +1 -1
- package/dist/server-info.d.ts +2 -0
- package/dist/server-info.d.ts.map +1 -1
- package/dist/server-info.js +20 -5
- package/dist/server-info.js.map +1 -1
- package/dist/services/access.d.ts +4 -4
- package/dist/services/access.d.ts.map +1 -1
- package/dist/services/activity.d.ts +3 -3
- package/dist/services/activity.d.ts.map +1 -1
- package/dist/services/activity.js +5 -4
- package/dist/services/activity.js.map +1 -1
- package/dist/services/adapter-registry-bootstrap.reconcile.test.js +1 -0
- package/dist/services/adapter-registry-bootstrap.reconcile.test.js.map +1 -1
- package/dist/services/agent-secret-bindings.d.ts +15 -0
- package/dist/services/agent-secret-bindings.d.ts.map +1 -1
- package/dist/services/agent-secret-bindings.js +43 -0
- package/dist/services/agent-secret-bindings.js.map +1 -1
- package/dist/services/agents.d.ts +194 -133
- package/dist/services/agents.d.ts.map +1 -1
- package/dist/services/agents.js +107 -4
- package/dist/services/agents.js.map +1 -1
- package/dist/services/approvals.d.ts +10 -10
- package/dist/services/approvals.d.ts.map +1 -1
- package/dist/services/approvals.js +9 -1
- package/dist/services/approvals.js.map +1 -1
- package/dist/services/assets.d.ts +5 -5
- package/dist/services/attention.d.ts +11 -0
- package/dist/services/attention.d.ts.map +1 -0
- package/dist/services/attention.js +1024 -0
- package/dist/services/attention.js.map +1 -0
- package/dist/services/authorization.d.ts +15 -2
- package/dist/services/authorization.d.ts.map +1 -1
- package/dist/services/authorization.js +324 -8
- package/dist/services/authorization.js.map +1 -1
- package/dist/services/board-auth.d.ts +2 -2
- package/dist/services/budgets.d.ts.map +1 -1
- package/dist/services/budgets.js +8 -7
- package/dist/services/budgets.js.map +1 -1
- package/dist/services/built-in-agent-metadata.d.ts +9 -0
- package/dist/services/built-in-agent-metadata.d.ts.map +1 -0
- package/dist/services/built-in-agent-metadata.js +39 -0
- package/dist/services/built-in-agent-metadata.js.map +1 -0
- package/dist/services/built-in-agents.d.ts +394 -0
- package/dist/services/built-in-agents.d.ts.map +1 -0
- package/dist/services/built-in-agents.js +1426 -0
- package/dist/services/built-in-agents.js.map +1 -0
- package/dist/services/change-consent-gate.d.ts +18 -0
- package/dist/services/change-consent-gate.d.ts.map +1 -0
- package/dist/services/change-consent-gate.js +170 -0
- package/dist/services/change-consent-gate.js.map +1 -0
- package/dist/services/companies.d.ts +14 -8
- package/dist/services/companies.d.ts.map +1 -1
- package/dist/services/companies.js +6 -1
- package/dist/services/companies.js.map +1 -1
- package/dist/services/company-artifacts.d.ts +4 -1
- package/dist/services/company-artifacts.d.ts.map +1 -1
- package/dist/services/company-artifacts.js +9 -1
- package/dist/services/company-artifacts.js.map +1 -1
- package/dist/services/company-member-roles.d.ts.map +1 -1
- package/dist/services/company-member-roles.js +2 -0
- package/dist/services/company-member-roles.js.map +1 -1
- package/dist/services/company-portability.d.ts +1 -1
- package/dist/services/company-portability.d.ts.map +1 -1
- package/dist/services/company-portability.js +113 -8
- package/dist/services/company-portability.js.map +1 -1
- package/dist/services/company-search.d.ts.map +1 -1
- package/dist/services/company-search.js +843 -294
- package/dist/services/company-search.js.map +1 -1
- package/dist/services/company-skills.d.ts +64 -3
- package/dist/services/company-skills.d.ts.map +1 -1
- package/dist/services/company-skills.js +1205 -20
- package/dist/services/company-skills.js.map +1 -1
- package/dist/services/costs.d.ts +9 -8
- package/dist/services/costs.d.ts.map +1 -1
- package/dist/services/costs.js +9 -2
- package/dist/services/costs.js.map +1 -1
- package/dist/services/dashboard.d.ts +2 -0
- package/dist/services/dashboard.d.ts.map +1 -1
- package/dist/services/dashboard.js +60 -17
- package/dist/services/dashboard.js.map +1 -1
- package/dist/services/database-backup-health.d.ts +34 -0
- package/dist/services/database-backup-health.d.ts.map +1 -0
- package/dist/services/database-backup-health.js +104 -0
- package/dist/services/database-backup-health.js.map +1 -0
- package/dist/services/document-annotations.d.ts +169 -1
- package/dist/services/document-annotations.d.ts.map +1 -1
- package/dist/services/document-annotations.js +272 -1
- package/dist/services/document-annotations.js.map +1 -1
- package/dist/services/documents.d.ts +368 -0
- package/dist/services/documents.d.ts.map +1 -1
- package/dist/services/documents.js +2 -2
- package/dist/services/documents.js.map +1 -1
- package/dist/services/environment-config.d.ts.map +1 -1
- package/dist/services/environment-config.js +6 -0
- package/dist/services/environment-config.js.map +1 -1
- package/dist/services/environment-custom-image-runtime.d.ts +28 -0
- package/dist/services/environment-custom-image-runtime.d.ts.map +1 -1
- package/dist/services/environment-custom-image-runtime.js +106 -7
- package/dist/services/environment-custom-image-runtime.js.map +1 -1
- package/dist/services/environment-custom-images.d.ts +31 -1
- package/dist/services/environment-custom-images.d.ts.map +1 -1
- package/dist/services/environment-custom-images.js +110 -4
- package/dist/services/environment-custom-images.js.map +1 -1
- package/dist/services/environment-execution-target.d.ts.map +1 -1
- package/dist/services/environment-execution-target.js +1 -3
- package/dist/services/environment-execution-target.js.map +1 -1
- package/dist/services/environment-probe.d.ts +1 -0
- package/dist/services/environment-probe.d.ts.map +1 -1
- package/dist/services/environment-probe.js +99 -5
- package/dist/services/environment-probe.js.map +1 -1
- package/dist/services/environments.d.ts +3 -1
- package/dist/services/environments.d.ts.map +1 -1
- package/dist/services/environments.js +98 -3
- package/dist/services/environments.js.map +1 -1
- package/dist/services/execution-workspace-policy.d.ts +29 -3
- package/dist/services/execution-workspace-policy.d.ts.map +1 -1
- package/dist/services/execution-workspace-policy.js +44 -1
- package/dist/services/execution-workspace-policy.js.map +1 -1
- package/dist/services/execution-workspaces.d.ts +64 -1
- package/dist/services/execution-workspaces.d.ts.map +1 -1
- package/dist/services/execution-workspaces.js +634 -3
- package/dist/services/execution-workspaces.js.map +1 -1
- package/dist/services/external-objects.d.ts +26 -26
- package/dist/services/feedback.d.ts +2 -2
- package/dist/services/finance.d.ts +5 -5
- package/dist/services/goals.d.ts +8 -8
- package/dist/services/heartbeat-stop-metadata.d.ts +2 -2
- package/dist/services/heartbeat-stop-metadata.d.ts.map +1 -1
- package/dist/services/heartbeat-stop-metadata.js +4 -0
- package/dist/services/heartbeat-stop-metadata.js.map +1 -1
- package/dist/services/heartbeat-stop-metadata.test.js +9 -0
- package/dist/services/heartbeat-stop-metadata.test.js.map +1 -1
- package/dist/services/heartbeat.d.ts +133 -28
- package/dist/services/heartbeat.d.ts.map +1 -1
- package/dist/services/heartbeat.js +1963 -239
- package/dist/services/heartbeat.js.map +1 -1
- package/dist/services/inbox-dismissals.d.ts +26 -0
- package/dist/services/inbox-dismissals.d.ts.map +1 -1
- package/dist/services/inbox-dismissals.js +33 -18
- package/dist/services/inbox-dismissals.js.map +1 -1
- package/dist/services/index.d.ts +3 -1
- package/dist/services/index.d.ts.map +1 -1
- package/dist/services/index.js +3 -1
- package/dist/services/index.js.map +1 -1
- package/dist/services/instance-settings.d.ts +25 -1
- package/dist/services/instance-settings.d.ts.map +1 -1
- package/dist/services/instance-settings.js +103 -5
- package/dist/services/instance-settings.js.map +1 -1
- package/dist/services/issue-continuation-summary.js +1 -1
- package/dist/services/issue-continuation-summary.js.map +1 -1
- package/dist/services/issue-recovery-actions.d.ts +3 -0
- package/dist/services/issue-recovery-actions.d.ts.map +1 -1
- package/dist/services/issue-recovery-actions.js +9 -0
- package/dist/services/issue-recovery-actions.js.map +1 -1
- package/dist/services/issue-rewake-throttle.d.ts +92 -0
- package/dist/services/issue-rewake-throttle.d.ts.map +1 -0
- package/dist/services/issue-rewake-throttle.js +139 -0
- package/dist/services/issue-rewake-throttle.js.map +1 -0
- package/dist/services/issue-thread-interactions.d.ts +8 -1
- package/dist/services/issue-thread-interactions.d.ts.map +1 -1
- package/dist/services/issue-thread-interactions.js +231 -16
- package/dist/services/issue-thread-interactions.js.map +1 -1
- package/dist/services/issue-visibility.d.ts +4 -0
- package/dist/services/issue-visibility.d.ts.map +1 -0
- package/dist/services/issue-visibility.js +9 -0
- package/dist/services/issue-visibility.js.map +1 -0
- package/dist/services/issues.d.ts +225 -105
- package/dist/services/issues.d.ts.map +1 -1
- package/dist/services/issues.js +540 -18
- package/dist/services/issues.js.map +1 -1
- package/dist/services/local-service-supervisor.d.ts +3 -0
- package/dist/services/local-service-supervisor.d.ts.map +1 -1
- package/dist/services/local-service-supervisor.js +51 -0
- package/dist/services/local-service-supervisor.js.map +1 -1
- package/dist/services/pipeline-case-outputs.d.ts.map +1 -1
- package/dist/services/pipeline-case-outputs.js +2 -1
- package/dist/services/pipeline-case-outputs.js.map +1 -1
- package/dist/services/pipelines-aggregation.d.ts.map +1 -1
- package/dist/services/pipelines-aggregation.js +3 -2
- package/dist/services/pipelines-aggregation.js.map +1 -1
- package/dist/services/pipelines.d.ts +165 -165
- package/dist/services/pipelines.d.ts.map +1 -1
- package/dist/services/pipelines.js +6 -4
- package/dist/services/pipelines.js.map +1 -1
- package/dist/services/plugin-database.d.ts +2 -2
- package/dist/services/plugin-environment-driver.d.ts +1 -1
- package/dist/services/plugin-host-services.d.ts.map +1 -1
- package/dist/services/plugin-host-services.js +3 -1
- package/dist/services/plugin-host-services.js.map +1 -1
- package/dist/services/plugin-managed-routines.d.ts +1 -0
- package/dist/services/plugin-managed-routines.d.ts.map +1 -1
- package/dist/services/plugin-registry.d.ts +21 -21
- package/dist/services/productivity-review.d.ts +1 -0
- package/dist/services/productivity-review.d.ts.map +1 -1
- package/dist/services/productivity-review.js +6 -5
- package/dist/services/productivity-review.js.map +1 -1
- package/dist/services/projects.d.ts +9 -9
- package/dist/services/recovery/service.d.ts +82 -54
- package/dist/services/recovery/service.d.ts.map +1 -1
- package/dist/services/recovery/service.js +200 -35
- package/dist/services/recovery/service.js.map +1 -1
- package/dist/services/recovery/successful-run-handoff.d.ts +1 -0
- package/dist/services/recovery/successful-run-handoff.d.ts.map +1 -1
- package/dist/services/recovery/successful-run-handoff.js +2 -0
- package/dist/services/recovery/successful-run-handoff.js.map +1 -1
- package/dist/services/recovery/successful-run-handoff.test.js +20 -0
- package/dist/services/recovery/successful-run-handoff.test.js.map +1 -1
- package/dist/services/resource-memberships.d.ts +13 -10
- package/dist/services/resource-memberships.d.ts.map +1 -1
- package/dist/services/resource-memberships.js +89 -20
- package/dist/services/resource-memberships.js.map +1 -1
- package/dist/services/responsible-user-denial-run-outcomes.d.ts +60 -0
- package/dist/services/responsible-user-denial-run-outcomes.d.ts.map +1 -0
- package/dist/services/responsible-user-denial-run-outcomes.js +56 -0
- package/dist/services/responsible-user-denial-run-outcomes.js.map +1 -0
- package/dist/services/responsible-user-denial-run-outcomes.test.d.ts +2 -0
- package/dist/services/responsible-user-denial-run-outcomes.test.d.ts.map +1 -0
- package/dist/services/responsible-user-denial-run-outcomes.test.js +75 -0
- package/dist/services/responsible-user-denial-run-outcomes.test.js.map +1 -0
- package/dist/services/routines.d.ts +30 -8
- package/dist/services/routines.d.ts.map +1 -1
- package/dist/services/routines.js +241 -18
- package/dist/services/routines.js.map +1 -1
- package/dist/services/run-liveness.d.ts.map +1 -1
- package/dist/services/run-liveness.js +21 -0
- package/dist/services/run-liveness.js.map +1 -1
- package/dist/services/run-log-store.d.ts +1 -0
- package/dist/services/run-log-store.d.ts.map +1 -1
- package/dist/services/run-log-store.js +4 -0
- package/dist/services/run-log-store.js.map +1 -1
- package/dist/services/run-scratch.d.ts +42 -0
- package/dist/services/run-scratch.d.ts.map +1 -0
- package/dist/services/run-scratch.js +111 -0
- package/dist/services/run-scratch.js.map +1 -0
- package/dist/services/run-scratch.test.d.ts +2 -0
- package/dist/services/run-scratch.test.d.ts.map +1 -0
- package/dist/services/run-scratch.test.js +98 -0
- package/dist/services/run-scratch.test.js.map +1 -0
- package/dist/services/secrets.d.ts +1261 -64
- package/dist/services/secrets.d.ts.map +1 -1
- package/dist/services/secrets.js +1089 -96
- package/dist/services/secrets.js.map +1 -1
- package/dist/services/task-watchdogs.d.ts +1 -0
- package/dist/services/task-watchdogs.d.ts.map +1 -1
- package/dist/services/task-watchdogs.js +78 -9
- package/dist/services/task-watchdogs.js.map +1 -1
- package/dist/services/work-timeline.d.ts.map +1 -1
- package/dist/services/work-timeline.js +34 -2
- package/dist/services/work-timeline.js.map +1 -1
- package/dist/services/workspace-runtime-read-model.d.ts +30 -29
- package/dist/services/workspace-runtime-read-model.d.ts.map +1 -1
- package/dist/services/workspace-runtime-read-model.js.map +1 -1
- package/dist/services/workspace-runtime.d.ts +59 -13
- package/dist/services/workspace-runtime.d.ts.map +1 -1
- package/dist/services/workspace-runtime.js +1121 -102
- package/dist/services/workspace-runtime.js.map +1 -1
- package/dist/skills-catalog/catalog/bundled/paperclip-operations/reflection-coach/SKILL.md +202 -0
- package/dist/skills-catalog/catalog/bundled/product/paperclip-capsules/SKILL.md +1 -1
- package/dist/skills-catalog/catalog/bundled/product/wireframe/SKILL.md +1 -1
- package/dist/skills-catalog/catalog/optional/content/release-announcement/SKILL.md +90 -0
- package/dist/skills-catalog/catalog/optional/finance/ramp/SKILL.md +98 -0
- package/dist/skills-catalog/generated/catalog.json +84 -12
- package/dist/ui-branding.d.ts +7 -0
- package/dist/ui-branding.d.ts.map +1 -1
- package/dist/ui-branding.js +8 -2
- package/dist/ui-branding.js.map +1 -1
- package/dist/version.d.ts +14 -0
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +114 -3
- package/dist/version.js.map +1 -1
- package/package.json +19 -20
- package/skills/paperclip/SKILL.md +45 -10
- package/skills/paperclip/references/api-reference.md +246 -1
- package/skills/paperclip/references/cases.md +295 -0
- package/skills/paperclip/references/company-skills.md +1 -1
- package/skills/paperclip-board/SKILL.md +3 -4
- package/skills/paperclip-converting-plans-to-tasks/SKILL.md +3 -7
- package/skills/para-memory-files/SKILL.md +3 -7
- package/ui-dist/assets/{abnfDiagram-VRR7QNED-BsFz2IiG.js → abnfDiagram-VRR7QNED-Dtgfy1PK.js} +1 -1
- package/ui-dist/assets/{arc-HZhAwbOs.js → arc-BGdvNvs8.js} +1 -1
- package/ui-dist/assets/{architectureDiagram-ZJ3FMSHR-BteMjuOn.js → architectureDiagram-ZJ3FMSHR-CK19SQDx.js} +1 -1
- package/ui-dist/assets/{blockDiagram-677ZJIJ3-Iq66uZjf.js → blockDiagram-677ZJIJ3-Kh4N3XG4.js} +1 -1
- package/ui-dist/assets/{browser-ponyfill-BBsuyZJt.js → browser-ponyfill-B3hXtEFW.js} +1 -1
- package/ui-dist/assets/{c4Diagram-LMCZKHZV-_nlglj7I.js → c4Diagram-LMCZKHZV-CBHpGrgU.js} +1 -1
- package/ui-dist/assets/channel-4fjB8e8N.js +1 -0
- package/ui-dist/assets/{chunk-2Q5K7J3B-Cvz7WwSQ.js → chunk-2Q5K7J3B-BdHbqi7k.js} +1 -1
- package/ui-dist/assets/{chunk-32BRIVSS-LLubniCS.js → chunk-32BRIVSS-DxtzZzpt.js} +1 -1
- package/ui-dist/assets/{chunk-5VM5RSS4-BOst5hcr.js → chunk-5VM5RSS4-DMKxBbgu.js} +1 -1
- package/ui-dist/assets/{chunk-EX3LRPZG-D5WNShVm.js → chunk-EX3LRPZG-BaIOPi4v.js} +1 -1
- package/ui-dist/assets/{chunk-JWPE2WC7-CA07x4av.js → chunk-JWPE2WC7-DGo3MPmr.js} +1 -1
- package/ui-dist/assets/{chunk-MOJQB5TN-Clfi83rK.js → chunk-MOJQB5TN-BEhQg0uD.js} +1 -1
- package/ui-dist/assets/{chunk-RYQCIY6F-BhZ6p7YJ.js → chunk-RYQCIY6F-HPJ6zRpJ.js} +1 -1
- package/ui-dist/assets/{chunk-V7JOEXUC-CdexJGoV.js → chunk-V7JOEXUC-Bl_6cAJD.js} +1 -1
- package/ui-dist/assets/{chunk-VR4S4FIN-8vjmBhr1.js → chunk-VR4S4FIN-KMjgGCsw.js} +1 -1
- package/ui-dist/assets/{chunk-XXDRQBXY-CqmCaplj.js → chunk-XXDRQBXY-oWGAIKjr.js} +1 -1
- package/ui-dist/assets/classDiagram-OUVF2IWQ-CTXxO5l9.js +1 -0
- package/ui-dist/assets/classDiagram-v2-EOCWNBFH-CTXxO5l9.js +1 -0
- package/ui-dist/assets/{cose-bilkent-JH36ORCC-CY-zAygY.js → cose-bilkent-JH36ORCC-Ckqse1pH.js} +1 -1
- package/ui-dist/assets/{cynefin-VYW2F7L2-BQSQ1HKr.js → cynefin-VYW2F7L2-DfZ8Hzqi.js} +1 -1
- package/ui-dist/assets/{cynefinDiagram-TSTJHNR4-NJCuFrZL.js → cynefinDiagram-TSTJHNR4-BkDQo9oi.js} +1 -1
- package/ui-dist/assets/{dagre-VKFMJZFB-DAUPIc8g.js → dagre-VKFMJZFB-ClghXQ6E.js} +1 -1
- package/ui-dist/assets/{diagram-FQU43EPY-DvMwJlCi.js → diagram-FQU43EPY-D5a0CwpV.js} +1 -1
- package/ui-dist/assets/{diagram-G47NLZAW-DPY0Jz_y.js → diagram-G47NLZAW-hVjYR_xl.js} +1 -1
- package/ui-dist/assets/{diagram-NH7WQ7WH-BpopeEhp.js → diagram-NH7WQ7WH-BlDPgD5Z.js} +1 -1
- package/ui-dist/assets/{diagram-OA4YK3LP-Qpu05z9X.js → diagram-OA4YK3LP-D0iZKIS3.js} +1 -1
- package/ui-dist/assets/{diagram-WEI45ONY-DHnvw9XU.js → diagram-WEI45ONY-m6hrfD0J.js} +1 -1
- package/ui-dist/assets/{ebnfDiagram-CCIWWBDH-D61S54_7.js → ebnfDiagram-CCIWWBDH-D4SQYLod.js} +1 -1
- package/ui-dist/assets/{erDiagram-Q63AITRT-pplAN_x2.js → erDiagram-Q63AITRT-Byzb6-I5.js} +1 -1
- package/ui-dist/assets/{flowDiagram-23GEKE2U-BaBcbyDq.js → flowDiagram-23GEKE2U-CaTUkFO6.js} +1 -1
- package/ui-dist/assets/{ganttDiagram-NO4QXBWP-BOJSII2b.js → ganttDiagram-NO4QXBWP-DmdMQ3ES.js} +1 -1
- package/ui-dist/assets/{gitGraphDiagram-IHSO6WYX-aYAaO4a2.js → gitGraphDiagram-IHSO6WYX-Bt-1g2CS.js} +1 -1
- package/ui-dist/assets/{index-CZxYajxF.js → index-2Dk9yO0W.js} +1 -1
- package/ui-dist/assets/{index-B8H0SHw1.js → index-8jLpgFNU.js} +1 -1
- package/ui-dist/assets/{index-3EKATXgc.js → index-AP_iimxs.js} +1 -1
- package/ui-dist/assets/{index-C2JgdjPa.js → index-BNJasQY9.js} +1 -1
- package/ui-dist/assets/{index-BaiskB3o.js → index-BO-aNmyX.js} +1 -1
- package/ui-dist/assets/{index-B5YenxAQ.js → index-BPCzCn1Y.js} +1 -1
- package/ui-dist/assets/{index-Da2kqT1M.js → index-BSLnLTb2.js} +1 -1
- package/ui-dist/assets/{index-By0oeEvz.js → index-CCZ2qgHN.js} +1 -1
- package/ui-dist/assets/{index-bUFCSEgv.js → index-CLZXR1k9.js} +1 -1
- package/ui-dist/assets/{index-CJlw4l9L.js → index-CTNfrE1s.js} +1 -1
- package/ui-dist/assets/{index-TmpY6Xh_.js → index-ChNd25s7.js} +1 -1
- package/ui-dist/assets/{index-BRgdHUNR.js → index-CjHiMOEO.js} +1 -1
- package/ui-dist/assets/{index-DrL6pHSJ.js → index-CmeJQpXZ.js} +1 -1
- package/ui-dist/assets/index-CyuYtn7U.css +1 -0
- package/ui-dist/assets/{index-CKKytnOd.js → index-CzZjM8eh.js} +1 -1
- package/ui-dist/assets/{index-Dceamza_.js → index-D-6T0Aft.js} +1 -1
- package/ui-dist/assets/{index-CjDqB1SX.js → index-DG_7V1pG.js} +1 -1
- package/ui-dist/assets/index-DIKcgxCt.js +676 -0
- package/ui-dist/assets/{index-BKVeJ3Jh.js → index-DiUerHmi.js} +1 -1
- package/ui-dist/assets/{index-C-nKjFZh.js → index-NZ7v5Fc2.js} +1 -1
- package/ui-dist/assets/{index-CvPPuNs1.js → index-Py70EvBg.js} +1 -1
- package/ui-dist/assets/{index-LHkZdu0h.js → index-R3aYn0qM.js} +1 -1
- package/ui-dist/assets/{index-BT8b79Bd.js → index-YXpm7gW-.js} +1 -1
- package/ui-dist/assets/{index-CemenVOf.js → index-n4q3gzki.js} +1 -1
- package/ui-dist/assets/{index-D3ODiUup.js → index-vlZ3GoYY.js} +1 -1
- package/ui-dist/assets/{infoDiagram-FWYZ7A6U-C7_90qGq.js → infoDiagram-FWYZ7A6U-jdi5ciVg.js} +1 -1
- package/ui-dist/assets/{ishikawaDiagram-FXEZZL3T-T-nPPdTt.js → ishikawaDiagram-FXEZZL3T-CMz5VvYp.js} +1 -1
- package/ui-dist/assets/{journeyDiagram-5HDEW3XC-C3hqQN0R.js → journeyDiagram-5HDEW3XC-DK6dk2R-.js} +1 -1
- package/ui-dist/assets/{kanban-definition-HUTT4EX6-CPoD9DUm.js → kanban-definition-HUTT4EX6-DX5cZL-D.js} +1 -1
- package/ui-dist/assets/{linear-D-S8yr3Z.js → linear-CI96ABKp.js} +1 -1
- package/ui-dist/assets/{mermaid.core-DhvH8shK.js → mermaid.core-DnsEVfbd.js} +4 -4
- package/ui-dist/assets/{mindmap-definition-LN4V7U3C-CxCIKrKo.js → mindmap-definition-LN4V7U3C-BdDCY7hf.js} +1 -1
- package/ui-dist/assets/{pegDiagram-2B236MQR-D4G-PC6N.js → pegDiagram-2B236MQR-DuY4TF-f.js} +1 -1
- package/ui-dist/assets/{pieDiagram-ENE6RG2P-N4X8UvqY.js → pieDiagram-ENE6RG2P-bgFw7dm3.js} +1 -1
- package/ui-dist/assets/{quadrantDiagram-ABIIQ3AL-CZohoDln.js → quadrantDiagram-ABIIQ3AL-BCZVWyWS.js} +1 -1
- package/ui-dist/assets/{railroadDiagram-RFXS5EU6-fMGQVhwF.js → railroadDiagram-RFXS5EU6-D9V5_Cgc.js} +1 -1
- package/ui-dist/assets/{requirementDiagram-TGXJPOKE-CK0IuUmk.js → requirementDiagram-TGXJPOKE-CItsya2r.js} +1 -1
- package/ui-dist/assets/{sankeyDiagram-HTMAVEWB-pSfb3CqG.js → sankeyDiagram-HTMAVEWB-Cz9IHsZO.js} +1 -1
- package/ui-dist/assets/{sequenceDiagram-DBY2YBRQ-CbhkqIeq.js → sequenceDiagram-DBY2YBRQ-hMSqTiwK.js} +1 -1
- package/ui-dist/assets/{sizeCapture-X5ZJPWSS-B6vqqPx-.js → sizeCapture-X5ZJPWSS-0g9M5wMf.js} +1 -1
- package/ui-dist/assets/{stateDiagram-2N3HPSRC-DIJf9h5C.js → stateDiagram-2N3HPSRC-F8_YhgWl.js} +1 -1
- package/ui-dist/assets/stateDiagram-v2-6OUMAXLB-fZImUBtq.js +1 -0
- package/ui-dist/assets/{swimlanes-5IMT3BWC-DIX_LV8O.js → swimlanes-5IMT3BWC-CCZqPbpE.js} +2 -2
- package/ui-dist/assets/swimlanesDiagram-G3AALYLV-9C1UT51R.js +8 -0
- package/ui-dist/assets/{timeline-definition-FHXFAJF6-CD-Pxb_g.js → timeline-definition-FHXFAJF6-BeUIr18x.js} +1 -1
- package/ui-dist/assets/{vennDiagram-L72KCM5P-BAZsWL0R.js → vennDiagram-L72KCM5P-D3d5h_G4.js} +1 -1
- package/ui-dist/assets/{wardleyDiagram-EHGQE667-CnNyfwX5.js → wardleyDiagram-EHGQE667-Bzt-As9A.js} +1 -1
- package/ui-dist/assets/{xychartDiagram-FW5EYKEG-j9YgsD3Y.js → xychartDiagram-FW5EYKEG-C2mly0xf.js} +1 -1
- package/ui-dist/fonts/InterVariable-Italic.woff2 +0 -0
- package/ui-dist/fonts/InterVariable.woff2 +0 -0
- package/ui-dist/fonts/NOTICE.md +17 -0
- package/ui-dist/index.html +2 -2
- package/ui-dist/locales/en/common.json +1142 -5
- package/ui-dist/locales/zh-CN/common.json +1140 -3
- package/ui-dist/assets/channel-CsiI2LYw.js +0 -1
- package/ui-dist/assets/classDiagram-OUVF2IWQ-CBn1Jqo8.js +0 -1
- package/ui-dist/assets/classDiagram-v2-EOCWNBFH-CBn1Jqo8.js +0 -1
- package/ui-dist/assets/index-BsL8K1vL.css +0 -1
- package/ui-dist/assets/index-D__fTsb2.js +0 -643
- package/ui-dist/assets/stateDiagram-v2-6OUMAXLB-DpwLD7YL.js +0 -1
- package/ui-dist/assets/swimlanesDiagram-G3AALYLV-Cklauutg.js +0 -8
|
@@ -5,10 +5,11 @@ import { promisify } from "node:util";
|
|
|
5
5
|
import { createHash, randomUUID } from "node:crypto";
|
|
6
6
|
import { and, asc, desc, eq, getTableColumns, gt, gte, inArray, isNull, lt, lte, notInArray, or, sql } from "drizzle-orm";
|
|
7
7
|
import { AGENT_DEFAULT_MAX_CONCURRENT_RUNS, ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY, MODEL_PROFILE_KEYS, envBindingSchema, } from "@penclipai/shared";
|
|
8
|
-
import { agents, agentConfigRevisions, agentRuntimeState, agentTaskSessions, agentWakeupRequests, activityLog, approvals, companySkills as companySkillsTable, companies, costEvents, documentAnnotationComments, documentAnnotationThreads, documentRevisions, issueDocuments, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueComments, issuePlanDecompositions, issueRelations, issueThreadInteractions, issues, issueWorkProducts, projects, projectWorkspaces, routineRevisions, routineRuns, routines, workspaceOperations, } from "@penclipai/db";
|
|
8
|
+
import { agents, agentConfigRevisions, agentRuntimeState, agentTaskSessions, agentWakeupRequests, activityLog, approvals, companyMemberships, companySkillTestRuns, companySkillVersions, companySkills as companySkillsTable, companies, costEvents, documentAnnotationComments, documentAnnotationThreads, documentRevisions, issueDocuments, executionWorkspaces, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueComments, issuePlanDecompositions, issueRecoveryActions, issueRelations, issueThreadInteractions, issues, issueWorkProducts, projects, projectWorkspaces, routineRevisions, routineRuns, routines, workspaceOperations, } from "@penclipai/db";
|
|
9
9
|
import { conflict, HttpError, notFound } from "../errors.js";
|
|
10
10
|
import { logger } from "../middleware/logger.js";
|
|
11
11
|
import { publishLiveEvent } from "./live-events.js";
|
|
12
|
+
import { normalizeResponsibleUserDenialCode } from "./responsible-user-denial-run-outcomes.js";
|
|
12
13
|
import { getRunLogStore } from "./run-log-store.js";
|
|
13
14
|
import { getServerAdapter, listAdapterModelProfiles, runningProcesses } from "../adapters/index.js";
|
|
14
15
|
import { createLocalAgentJwt } from "../agent-auth-jwt.js";
|
|
@@ -23,10 +24,12 @@ import { resolveDefaultAgentWorkspaceDir, resolveManagedProjectWorkspaceDir } fr
|
|
|
23
24
|
import { buildHeartbeatRunIssueComment, HEARTBEAT_RUN_RESULT_OUTPUT_MAX_CHARS, HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS, HEARTBEAT_RUN_SAFE_RESULT_JSON_MAX_BYTES, mergeHeartbeatRunResultJson, } from "./heartbeat-run-summary.js";
|
|
24
25
|
import { buildHeartbeatRunStopMetadata, mergeHeartbeatRunStopMetadata, normalizeMaxTurnStopReason, } from "./heartbeat-stop-metadata.js";
|
|
25
26
|
import { classifyRunLiveness, } from "./run-liveness.js";
|
|
27
|
+
import { ISSUE_NEW_INPUT_ACTIVITY_ACTIONS, ISSUE_PROGRESS_ACTIVITY_ACTIONS, ISSUE_REWAKE_LOOKBACK_MS, ISSUE_REWAKE_RUN_SAMPLE_LIMIT, evaluateIssueRewakeThrottle, isThrottleCandidateIssueRewake, } from "./issue-rewake-throttle.js";
|
|
26
28
|
import { logActivity, publishPluginDomainEvent } from "./activity-log.js";
|
|
27
29
|
import { buildWorkspaceReadyComment, cleanupExecutionWorkspaceArtifacts, ensureGitWorktreeBranchCoherent, ensurePersistedExecutionWorkspaceAvailable, ensureRuntimeServicesForRun, formatManagedGitWorktreeBranchInspection, inspectManagedGitWorktreeBranch, persistAdapterManagedRuntimeServices, realizeExecutionWorkspace, releaseRuntimeServicesForRun, sanitizeRuntimeServiceBaseEnv, } from "./workspace-runtime.js";
|
|
28
30
|
import { issueService } from "./issues.js";
|
|
29
|
-
import {
|
|
31
|
+
import { visibleIssueCondition } from "./issue-visibility.js";
|
|
32
|
+
import { ISSUE_BLOCKERS_RESOLVED_WAKE_REASON, } from "./issue-dependency-wakeups.js";
|
|
30
33
|
import { buildIssueMonitorClearedPatch, buildIssueMonitorTriggeredPatch, normalizeIssueExecutionPolicy, parseIssueExecutionState, } from "./issue-execution-policy.js";
|
|
31
34
|
import { ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS, isVerifiedIssueTreeControlInteractionWake, issueTreeControlService, } from "./issue-tree-control.js";
|
|
32
35
|
import { continuationSummaryParksExecutor, getIssueContinuationSummaryDocument, refreshIssueContinuationSummary, } from "./issue-continuation-summary.js";
|
|
@@ -34,8 +37,9 @@ import { buildPlanReviewContext } from "./plan-review-context.js";
|
|
|
34
37
|
import { executionWorkspaceService, mergeExecutionWorkspaceConfig } from "./execution-workspaces.js";
|
|
35
38
|
import { workspaceOperationService } from "./workspace-operations.js";
|
|
36
39
|
import { isProcessGroupAlive, terminateLocalService } from "./local-service-supervisor.js";
|
|
37
|
-
import {
|
|
38
|
-
import {
|
|
40
|
+
import { HEARTBEAT_RUN_SCRATCH_MARKER, buildHeartbeatRunScratchEnv, cleanupHeartbeatRunScratch, prepareHeartbeatRunScratch, } from "./run-scratch.js";
|
|
41
|
+
import { buildExecutionWorkspaceAdapterConfig, gateProjectExecutionWorkspacePolicy, issueExecutionWorkspaceModeForPersistedWorkspace, isUnrunnableWorktreeCombo, parseIssueExecutionWorkspaceSettings, parseProjectExecutionWorkspacePolicy, resolveEffectiveWorkspaceStrategyType, resolveExecutionWorkspaceEnvironmentId, resolveExecutionWorkspaceMode, WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE, WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION, } from "./execution-workspace-policy.js";
|
|
42
|
+
import { instanceSettingsService, resolveWorktreeRunExecutionActivation, } from "./instance-settings.js";
|
|
39
43
|
import { evaluateExecutionAllowlist, isExecutionForcedToKubernetes, } from "./execution-allowlist.js";
|
|
40
44
|
import { RECOVERY_ORIGIN_KINDS, FINISH_SUCCESSFUL_RUN_HANDOFF_REASON, SUCCESSFUL_RUN_MISSING_STATE_REASON, RUN_LIVENESS_CONTINUATION_REASON, buildRunLivenessContinuationIdempotencyKey, buildFinishSuccessfulRunHandoffIdempotencyKey, buildSuccessfulRunHandoffRequiredNotice, decideRunLivenessContinuation, decideSuccessfulRunHandoff, findExistingFinishSuccessfulRunHandoffWake, findExistingRunLivenessContinuationWake, SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY, readContinuationAttempt, } from "./recovery/index.js";
|
|
41
45
|
import { isAutomaticRecoverySuppressedByPauseHold } from "./recovery/pause-hold-guard.js";
|
|
@@ -49,8 +53,9 @@ import { redactQuarantinedBodyForHigherTrust, sanitizeQuarantinedCommentForHighe
|
|
|
49
53
|
import { redactCurrentUserText, redactCurrentUserValue, } from "../log-redaction.js";
|
|
50
54
|
import { redactEventPayload, redactSensitiveText } from "../redaction.js";
|
|
51
55
|
import { hasSessionCompactionThresholds, resolveSessionCompactionPolicy, } from "@penclipai/adapter-utils";
|
|
52
|
-
import { readPaperclipSkillSyncPreference, writePaperclipSkillSyncPreference, } from "@penclipai/adapter-utils/server-utils";
|
|
56
|
+
import { readPaperclipSkillSyncPreference, UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON, UNMANAGED_BACKGROUND_TASK_STOP_REASON, writePaperclipSkillSyncPreference, } from "@penclipai/adapter-utils/server-utils";
|
|
53
57
|
import { extractSkillMentionIds, isUuidLike } from "@penclipai/shared";
|
|
58
|
+
import { evaluateCodexCredentialReadiness } from "@penclipai/adapter-codex-local/server";
|
|
54
59
|
import { environmentService } from "./environments.js";
|
|
55
60
|
import { parseExecutionPolicyBootstrapEnv } from "./execution-policy-bootstrap.js";
|
|
56
61
|
import { environmentRuntimeService } from "./environment-runtime.js";
|
|
@@ -94,7 +99,7 @@ const MAX_INLINE_WAKE_COMMENT_BODY_TOTAL_CHARS = 12_000;
|
|
|
94
99
|
const execFile = promisify(execFileCallback);
|
|
95
100
|
const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"];
|
|
96
101
|
const CANCELLABLE_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"];
|
|
97
|
-
const HEARTBEAT_RUN_TERMINAL_STATUSES = ["succeeded", "failed", "cancelled", "timed_out"];
|
|
102
|
+
const HEARTBEAT_RUN_TERMINAL_STATUSES = ["succeeded", "interrupted", "failed", "cancelled", "timed_out"];
|
|
98
103
|
const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = ["failed", "cancelled", "timed_out"];
|
|
99
104
|
const TIMER_ACTIONABLE_ISSUE_STATUSES = ["todo", "in_progress"];
|
|
100
105
|
export { ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS, ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS, ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS, } from "./recovery/service.js";
|
|
@@ -110,6 +115,10 @@ const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_JITTER_RATIO = 0.25;
|
|
|
110
115
|
const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON = "transient_failure";
|
|
111
116
|
const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry";
|
|
112
117
|
const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length;
|
|
118
|
+
export const INTERACTION_CONTINUATION_INFRA_RETRY_REASON = "interaction_continuation_infra_retry";
|
|
119
|
+
export const INTERACTION_CONTINUATION_INFRA_WAKE_REASON = "interaction_continuation_infra_retry";
|
|
120
|
+
const INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS = 3;
|
|
121
|
+
const RESOLVED_INTERACTION_CONTINUATION_STATUSES = new Set(["accepted", "answered", "rejected"]);
|
|
113
122
|
const WORKSPACE_VALIDATION_FAILURE_CODE = "workspace_validation_failed";
|
|
114
123
|
const WORKSPACE_VALIDATION_RECOVERY_CAUSE = "workspace_validation_failed";
|
|
115
124
|
const CONFIGURATION_INCOMPLETE_FAILURE_CODE = "configuration_incomplete";
|
|
@@ -122,7 +131,6 @@ const GITHUB_PR_WORKFLOW_SKILL_SLUG = "github-pr-workflow";
|
|
|
122
131
|
const PUSH_CAPABILITY_ENV_KEYS = ["GH_TOKEN", "GITHUB_TOKEN"];
|
|
123
132
|
// Keep this in sync with local adapters that require a git workspace before launch.
|
|
124
133
|
const GIT_SENSITIVE_LOCAL_ADAPTER_TYPES = new Set([
|
|
125
|
-
"acpx_local",
|
|
126
134
|
"claude_local",
|
|
127
135
|
"codex_local",
|
|
128
136
|
"cursor",
|
|
@@ -174,6 +182,9 @@ function readHeartbeatRunErrorFamily(run) {
|
|
|
174
182
|
const persistedFamily = readNonEmptyString(resultJson.errorFamily);
|
|
175
183
|
if (persistedFamily)
|
|
176
184
|
return persistedFamily;
|
|
185
|
+
if (run.errorCode === "provider_quota") {
|
|
186
|
+
return "provider_quota";
|
|
187
|
+
}
|
|
177
188
|
if (run.errorCode === "codex_transient_upstream" || run.errorCode === "claude_transient_upstream") {
|
|
178
189
|
return "transient_upstream";
|
|
179
190
|
}
|
|
@@ -194,13 +205,45 @@ function readTransientRetryNotBeforeFromRun(run) {
|
|
|
194
205
|
return Number.isNaN(parsed.getTime()) ? null : parsed;
|
|
195
206
|
}
|
|
196
207
|
function readTransientRecoveryContractFromRun(run) {
|
|
197
|
-
|
|
208
|
+
const errorFamily = readHeartbeatRunErrorFamily(run);
|
|
209
|
+
return errorFamily === "transient_upstream" || errorFamily === "provider_quota"
|
|
198
210
|
? {
|
|
199
|
-
errorFamily
|
|
211
|
+
errorFamily,
|
|
200
212
|
retryNotBefore: readTransientRetryNotBeforeFromRun(run),
|
|
201
213
|
}
|
|
202
214
|
: null;
|
|
203
215
|
}
|
|
216
|
+
function isResolvedInteractionContinuationWakeContext(contextSnapshot) {
|
|
217
|
+
const context = parseObject(contextSnapshot);
|
|
218
|
+
const interactionId = readNonEmptyString(context.interactionId);
|
|
219
|
+
const interactionStatus = readNonEmptyString(context.interactionStatus);
|
|
220
|
+
if (!interactionId || !interactionStatus)
|
|
221
|
+
return false;
|
|
222
|
+
if (!RESOLVED_INTERACTION_CONTINUATION_STATUSES.has(interactionStatus))
|
|
223
|
+
return false;
|
|
224
|
+
const mutation = readNonEmptyString(context.mutation);
|
|
225
|
+
const wakeReason = readNonEmptyString(context.wakeReason);
|
|
226
|
+
const retryReason = readNonEmptyString(context.retryReason);
|
|
227
|
+
return ((mutation === "interaction" && wakeReason === "issue_commented") ||
|
|
228
|
+
wakeReason === INTERACTION_CONTINUATION_INFRA_WAKE_REASON ||
|
|
229
|
+
retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON);
|
|
230
|
+
}
|
|
231
|
+
function isSpawnLikeFailureMessage(value) {
|
|
232
|
+
if (typeof value !== "string")
|
|
233
|
+
return false;
|
|
234
|
+
return /failed to start command|spawn\b|\bENOENT\b/i.test(value);
|
|
235
|
+
}
|
|
236
|
+
function isRetryableInteractionContinuationInfrastructureFailure(run) {
|
|
237
|
+
if (run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE || run.errorCode === "process_lost") {
|
|
238
|
+
return true;
|
|
239
|
+
}
|
|
240
|
+
if (run.errorCode !== "adapter_failed" && run.errorCode !== "setup_failed")
|
|
241
|
+
return false;
|
|
242
|
+
const resultJson = parseObject(run.resultJson);
|
|
243
|
+
return (isSpawnLikeFailureMessage(run.error) ||
|
|
244
|
+
isSpawnLikeFailureMessage(resultJson.errorMessage) ||
|
|
245
|
+
isSpawnLikeFailureMessage(resultJson.message));
|
|
246
|
+
}
|
|
204
247
|
function mergeAdapterRecoveryMetadata(input) {
|
|
205
248
|
const errorFamily = readNonEmptyString(input.errorFamily);
|
|
206
249
|
const retryNotBefore = readNonEmptyString(input.retryNotBefore);
|
|
@@ -213,6 +256,7 @@ function mergeAdapterRecoveryMetadata(input) {
|
|
|
213
256
|
? {
|
|
214
257
|
retryNotBefore,
|
|
215
258
|
transientRetryNotBefore: retryNotBefore,
|
|
259
|
+
...(errorFamily === "provider_quota" ? { providerQuotaRetryNotBefore: retryNotBefore } : {}),
|
|
216
260
|
}
|
|
217
261
|
: {}),
|
|
218
262
|
};
|
|
@@ -221,6 +265,22 @@ const RUNNING_ISSUE_WAKE_REASONS_REQUIRING_FOLLOWUP = new Set([
|
|
|
221
265
|
"approval_approved",
|
|
222
266
|
ISSUE_BLOCKERS_RESOLVED_WAKE_REASON,
|
|
223
267
|
]);
|
|
268
|
+
const ISSUE_RESPONSIBLE_USER_WAKE_REASONS = new Set([
|
|
269
|
+
"issue_assigned",
|
|
270
|
+
"issue_checked_out",
|
|
271
|
+
"issue_commented",
|
|
272
|
+
"issue_comment_mentioned",
|
|
273
|
+
"issue_reopened_via_comment",
|
|
274
|
+
"issue_blockers_resolved",
|
|
275
|
+
"issue_children_completed",
|
|
276
|
+
"issue_status_changed",
|
|
277
|
+
"issue_tree_restored",
|
|
278
|
+
"issue_recovery_action_restored",
|
|
279
|
+
"execution_review_requested",
|
|
280
|
+
"execution_approval_requested",
|
|
281
|
+
"execution_changes_requested",
|
|
282
|
+
"approval_approved",
|
|
283
|
+
]);
|
|
224
284
|
const SESSIONED_LOCAL_ADAPTERS = new Set([
|
|
225
285
|
"claude_local",
|
|
226
286
|
"codex_local",
|
|
@@ -235,9 +295,18 @@ const SESSIONED_LOCAL_ADAPTERS = new Set([
|
|
|
235
295
|
const activeRunExecutions = new Set();
|
|
236
296
|
const INLINE_BASE64_IMAGE_DATA_RE = /("type":"image","source":\{"type":"base64","data":")([A-Za-z0-9+/=]{1024,})(")/g;
|
|
237
297
|
function formatMissingBindingForOperator(missing) {
|
|
298
|
+
if (missing.bindingType === "user_secret_ref") {
|
|
299
|
+
const definitionLabel = missing.userSecretDefinitionName
|
|
300
|
+
? `"${missing.userSecretDefinitionName}"`
|
|
301
|
+
: missing.userSecretDefinitionKey
|
|
302
|
+
? `"${missing.userSecretDefinitionKey}"`
|
|
303
|
+
: "declared user secret";
|
|
304
|
+
const ownerLabel = missing.responsibleUserId ? ` for responsible user ${missing.responsibleUserId}` : "";
|
|
305
|
+
return `user secret ${definitionLabel}${ownerLabel} not available at ${missing.consumerType} ${missing.configPath}`;
|
|
306
|
+
}
|
|
238
307
|
const secretLabel = missing.secretName
|
|
239
308
|
? `"${missing.secretName}"`
|
|
240
|
-
: missing.secretId;
|
|
309
|
+
: missing.secretId ?? "unknown";
|
|
241
310
|
return `secret ${secretLabel} not bound at ${missing.consumerType} ${missing.configPath}`;
|
|
242
311
|
}
|
|
243
312
|
function isConfiguredEnvBindingValue(binding) {
|
|
@@ -342,19 +411,39 @@ export async function resolveExecutionRunAdapterConfig(input) {
|
|
|
342
411
|
if (typeof input.secretsSvc.collectMissingRuntimeBindings === "function") {
|
|
343
412
|
const missingBindings = [];
|
|
344
413
|
if (environmentEnv && input.environmentId) {
|
|
345
|
-
missingBindings.push(...(await input.secretsSvc.collectMissingRuntimeBindings(input.companyId, environmentEnv, {
|
|
414
|
+
missingBindings.push(...(await input.secretsSvc.collectMissingRuntimeBindings(input.companyId, environmentEnv, {
|
|
415
|
+
consumerType: "environment",
|
|
416
|
+
consumerId: input.environmentId,
|
|
417
|
+
responsibleUserId: input.responsibleUserId ?? null,
|
|
418
|
+
})));
|
|
346
419
|
}
|
|
347
420
|
if (input.agentId) {
|
|
348
|
-
missingBindings.push(...(await input.secretsSvc.collectMissingRuntimeBindings(input.companyId, parseObject(executionRunConfig.env), {
|
|
421
|
+
missingBindings.push(...(await input.secretsSvc.collectMissingRuntimeBindings(input.companyId, parseObject(executionRunConfig.env), {
|
|
422
|
+
consumerType: "agent",
|
|
423
|
+
consumerId: input.agentId,
|
|
424
|
+
responsibleUserId: input.responsibleUserId ?? null,
|
|
425
|
+
})));
|
|
349
426
|
if (typeof input.secretsSvc.collectMissingAdapterConfigRuntimeBindings === "function") {
|
|
350
|
-
missingBindings.push(...(await input.secretsSvc.collectMissingAdapterConfigRuntimeBindings(input.companyId, executionRunConfig, input.adapterType ?? null, {
|
|
427
|
+
missingBindings.push(...(await input.secretsSvc.collectMissingAdapterConfigRuntimeBindings(input.companyId, executionRunConfig, input.adapterType ?? null, {
|
|
428
|
+
consumerType: "agent",
|
|
429
|
+
consumerId: input.agentId,
|
|
430
|
+
responsibleUserId: input.responsibleUserId ?? null,
|
|
431
|
+
})));
|
|
351
432
|
}
|
|
352
433
|
}
|
|
353
434
|
if (projectEnv && input.projectId) {
|
|
354
|
-
missingBindings.push(...(await input.secretsSvc.collectMissingRuntimeBindings(input.companyId, projectEnv, {
|
|
435
|
+
missingBindings.push(...(await input.secretsSvc.collectMissingRuntimeBindings(input.companyId, projectEnv, {
|
|
436
|
+
consumerType: "project",
|
|
437
|
+
consumerId: input.projectId,
|
|
438
|
+
responsibleUserId: input.responsibleUserId ?? null,
|
|
439
|
+
})));
|
|
355
440
|
}
|
|
356
441
|
if (routineEnv && input.routineId) {
|
|
357
|
-
missingBindings.push(...(await input.secretsSvc.collectMissingRuntimeBindings(input.companyId, routineEnv, {
|
|
442
|
+
missingBindings.push(...(await input.secretsSvc.collectMissingRuntimeBindings(input.companyId, routineEnv, {
|
|
443
|
+
consumerType: "routine",
|
|
444
|
+
consumerId: input.routineId,
|
|
445
|
+
responsibleUserId: input.responsibleUserId ?? null,
|
|
446
|
+
})));
|
|
358
447
|
}
|
|
359
448
|
if (requiredScopedEnvBinding) {
|
|
360
449
|
const requiredEnvKeys = new Set(requiredScopedEnvBinding.keys);
|
|
@@ -400,6 +489,7 @@ export async function resolveExecutionRunAdapterConfig(input) {
|
|
|
400
489
|
consumerId: input.environmentId,
|
|
401
490
|
actorType: "agent",
|
|
402
491
|
actorId: input.agentId ?? null,
|
|
492
|
+
responsibleUserId: input.responsibleUserId ?? null,
|
|
403
493
|
issueId: input.issueId ?? null,
|
|
404
494
|
heartbeatRunId: input.heartbeatRunId ?? null,
|
|
405
495
|
...(lowTrustAllowedBindingIds !== undefined ? { allowedBindingIds: lowTrustAllowedBindingIds } : {}),
|
|
@@ -412,6 +502,7 @@ export async function resolveExecutionRunAdapterConfig(input) {
|
|
|
412
502
|
consumerId: input.agentId,
|
|
413
503
|
actorType: "agent",
|
|
414
504
|
actorId: input.agentId,
|
|
505
|
+
responsibleUserId: input.responsibleUserId ?? null,
|
|
415
506
|
issueId: input.issueId ?? null,
|
|
416
507
|
heartbeatRunId: input.heartbeatRunId ?? null,
|
|
417
508
|
...(lowTrustAllowedBindingIds !== undefined ? { allowedBindingIds: lowTrustAllowedBindingIds } : {}),
|
|
@@ -433,6 +524,7 @@ export async function resolveExecutionRunAdapterConfig(input) {
|
|
|
433
524
|
consumerId: input.projectId,
|
|
434
525
|
actorType: "agent",
|
|
435
526
|
actorId: input.agentId ?? null,
|
|
527
|
+
responsibleUserId: input.responsibleUserId ?? null,
|
|
436
528
|
issueId: input.issueId ?? null,
|
|
437
529
|
heartbeatRunId: input.heartbeatRunId ?? null,
|
|
438
530
|
...(lowTrustAllowedBindingIds !== undefined ? { allowedBindingIds: lowTrustAllowedBindingIds } : {}),
|
|
@@ -455,6 +547,7 @@ export async function resolveExecutionRunAdapterConfig(input) {
|
|
|
455
547
|
consumerId: input.routineId,
|
|
456
548
|
actorType: "agent",
|
|
457
549
|
actorId: input.agentId ?? null,
|
|
550
|
+
responsibleUserId: input.responsibleUserId ?? null,
|
|
458
551
|
issueId: input.issueId ?? null,
|
|
459
552
|
heartbeatRunId: input.heartbeatRunId ?? null,
|
|
460
553
|
...(lowTrustAllowedBindingIds !== undefined ? { allowedBindingIds: lowTrustAllowedBindingIds } : {}),
|
|
@@ -470,6 +563,42 @@ export async function resolveExecutionRunAdapterConfig(input) {
|
|
|
470
563
|
secretKeys.add(key);
|
|
471
564
|
}
|
|
472
565
|
}
|
|
566
|
+
// Pre-dispatch credential gate for codex_local: a managed Codex home with no
|
|
567
|
+
// usable auth.json and an empty OPENAI_API_KEY would dispatch a run that
|
|
568
|
+
// immediately fails with "no Codex credentials provisioned" (adapter_failed),
|
|
569
|
+
// making a configuration problem look like a runtime failure. Surface it as a
|
|
570
|
+
// configuration-incomplete blocker instead, naming the missing credential
|
|
571
|
+
// action and owner without leaking any secret value. This runs after secret
|
|
572
|
+
// resolution so a per-agent OPENAI_API_KEY (plain or resolved secret) counts
|
|
573
|
+
// as satisfying the credential. It shares the exact readiness predicate the
|
|
574
|
+
// adapter uses at execute time, so the two cannot drift.
|
|
575
|
+
if ((input.adapterType ?? null) === "codex_local") {
|
|
576
|
+
const resolvedEnv = parseObject(resolvedConfig.env);
|
|
577
|
+
const readiness = await evaluateCodexCredentialReadiness({
|
|
578
|
+
env: process.env,
|
|
579
|
+
companyId: input.companyId,
|
|
580
|
+
configuredCodexHome: readNonEmptyString(resolvedEnv.CODEX_HOME),
|
|
581
|
+
configuredApiKey: readNonEmptyString(resolvedEnv.OPENAI_API_KEY),
|
|
582
|
+
});
|
|
583
|
+
if (readiness.managed && !readiness.ready) {
|
|
584
|
+
throw new ConfigurationIncompleteFailure(`configuration incomplete: no Codex credentials available for managed home "${readiness.effectiveHome}". ` +
|
|
585
|
+
`Sign in to Codex on the host with a ChatGPT subscription, or bind a per-agent OPENAI_API_KEY secret for this agent.`, {
|
|
586
|
+
configurationIncomplete: {
|
|
587
|
+
reason: "codex_credentials_missing",
|
|
588
|
+
companyId: input.companyId,
|
|
589
|
+
agentId: input.agentId ?? null,
|
|
590
|
+
issueId: input.issueId ?? null,
|
|
591
|
+
projectId: input.projectId ?? null,
|
|
592
|
+
routineId: input.routineId ?? null,
|
|
593
|
+
responsibleUserId: input.responsibleUserId ?? null,
|
|
594
|
+
adapterType: "codex_local",
|
|
595
|
+
requiredEnvKeys: ["OPENAI_API_KEY"],
|
|
596
|
+
effectiveCodexHome: readiness.effectiveHome,
|
|
597
|
+
missingBindings: [],
|
|
598
|
+
},
|
|
599
|
+
});
|
|
600
|
+
}
|
|
601
|
+
}
|
|
473
602
|
return {
|
|
474
603
|
resolvedConfig,
|
|
475
604
|
secretKeys,
|
|
@@ -787,6 +916,9 @@ function isWorkspaceValidationFailure(error) {
|
|
|
787
916
|
function isWorkspaceValidationFailedRun(run) {
|
|
788
917
|
return run?.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE;
|
|
789
918
|
}
|
|
919
|
+
function readWorkspaceValidationPayloadFromRun(run) {
|
|
920
|
+
return parseObject(parseObject(run?.resultJson).workspaceValidation);
|
|
921
|
+
}
|
|
790
922
|
function stableStringifyForFingerprint(value) {
|
|
791
923
|
if (Array.isArray(value)) {
|
|
792
924
|
return `[${value.map((entry) => stableStringifyForFingerprint(entry)).join(",")}]`;
|
|
@@ -828,6 +960,14 @@ async function hasGitMetadata(cwd) {
|
|
|
828
960
|
.then((entry) => entry.isDirectory() || entry.isFile())
|
|
829
961
|
.catch(() => false);
|
|
830
962
|
}
|
|
963
|
+
async function isGitCheckout(cwd) {
|
|
964
|
+
const normalized = readNonEmptyString(cwd);
|
|
965
|
+
if (!normalized)
|
|
966
|
+
return false;
|
|
967
|
+
return execFile("git", ["rev-parse", "--show-toplevel"], { cwd: normalized })
|
|
968
|
+
.then((result) => Boolean(readNonEmptyString(result.stdout)))
|
|
969
|
+
.catch(() => false);
|
|
970
|
+
}
|
|
831
971
|
function sameResolvedPath(left, right) {
|
|
832
972
|
const leftPath = readNonEmptyString(left);
|
|
833
973
|
const rightPath = readNonEmptyString(right);
|
|
@@ -854,6 +994,45 @@ async function hasGitPushRemote(cwd) {
|
|
|
854
994
|
}
|
|
855
995
|
return false;
|
|
856
996
|
}
|
|
997
|
+
export async function assertGitWorktreeBaseWorkspaceReady(input) {
|
|
998
|
+
if (!input.issue)
|
|
999
|
+
return;
|
|
1000
|
+
if (input.requestedExecutionWorkspaceMode !== "isolated_workspace" &&
|
|
1001
|
+
input.requestedExecutionWorkspaceMode !== "operator_branch") {
|
|
1002
|
+
return;
|
|
1003
|
+
}
|
|
1004
|
+
const strategyType = resolveEffectiveWorkspaceStrategyType(input.requestedExecutionWorkspaceMode, input.config);
|
|
1005
|
+
if (strategyType !== "git_worktree")
|
|
1006
|
+
return;
|
|
1007
|
+
const issueLabel = input.issue.identifier ?? input.issue.id;
|
|
1008
|
+
const remediation = "This task needs a project / project workspace or a reusable execution workspace before it can run.";
|
|
1009
|
+
const fail = (reason, message, extra = {}) => {
|
|
1010
|
+
throw new WorkspaceValidationFailure(message, {
|
|
1011
|
+
workspaceValidation: {
|
|
1012
|
+
reason,
|
|
1013
|
+
issueId: input.issue.id,
|
|
1014
|
+
issueIdentifier: input.issue.identifier,
|
|
1015
|
+
issueProjectId: input.issue.projectId,
|
|
1016
|
+
issueProjectWorkspaceId: input.issue.projectWorkspaceId,
|
|
1017
|
+
issueExecutionWorkspaceId: input.issue.executionWorkspaceId ?? null,
|
|
1018
|
+
issueExecutionWorkspacePreference: input.issue.executionWorkspacePreference ?? null,
|
|
1019
|
+
requestedExecutionWorkspaceMode: input.requestedExecutionWorkspaceMode,
|
|
1020
|
+
workspaceStrategyType: strategyType,
|
|
1021
|
+
resolvedWorkspaceSource: input.base.source,
|
|
1022
|
+
resolvedProjectId: input.base.projectId,
|
|
1023
|
+
resolvedProjectWorkspaceId: input.base.workspaceId,
|
|
1024
|
+
resolvedWorkspaceCwd: input.base.baseCwd,
|
|
1025
|
+
...extra,
|
|
1026
|
+
},
|
|
1027
|
+
});
|
|
1028
|
+
};
|
|
1029
|
+
if (input.base.source === "agent_home") {
|
|
1030
|
+
fail("git_worktree_base_agent_home", `Issue ${issueLabel} requested ${input.requestedExecutionWorkspaceMode} with git_worktree, but no project or reusable execution workspace was resolved; refusing to create a git worktree from agent fallback cwd "${input.base.baseCwd}". ${remediation}`);
|
|
1031
|
+
}
|
|
1032
|
+
if (!await isGitCheckout(input.base.baseCwd)) {
|
|
1033
|
+
fail("git_worktree_base_not_git_checkout", `Issue ${issueLabel} requested ${input.requestedExecutionWorkspaceMode} with git_worktree, but base workspace "${input.base.baseCwd}" is not a git checkout. ${remediation}`);
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
857
1036
|
export async function assertPushCapabilityCheckoutValid(input) {
|
|
858
1037
|
if (!input.enabled || !input.issue)
|
|
859
1038
|
return;
|
|
@@ -959,8 +1138,8 @@ export async function assertGitSensitiveAdapterWorkspaceValid(input) {
|
|
|
959
1138
|
if (workspaceExpectation && effectiveCwd && !await hasGitMetadata(effectiveCwd)) {
|
|
960
1139
|
fail("missing_git_metadata", `Issue ${issue.identifier ?? issue.id} expected a git workspace for ${input.adapterType}, but "${effectiveCwd}" has no .git metadata.`);
|
|
961
1140
|
}
|
|
962
|
-
const expectedManagedBranchName = readNonEmptyString(input.
|
|
963
|
-
readNonEmptyString(input.
|
|
1141
|
+
const expectedManagedBranchName = readNonEmptyString(input.executionWorkspace.branchName) ??
|
|
1142
|
+
readNonEmptyString(input.persistedExecutionWorkspace?.branchName);
|
|
964
1143
|
if (input.persistedExecutionWorkspace?.strategyType === "git_worktree" &&
|
|
965
1144
|
effectiveCwd &&
|
|
966
1145
|
expectedManagedBranchName) {
|
|
@@ -1468,6 +1647,10 @@ function normalizeBilledCostCents(costUsd, billingType) {
|
|
|
1468
1647
|
return 0;
|
|
1469
1648
|
return Math.max(0, Math.round(costUsd * 100));
|
|
1470
1649
|
}
|
|
1650
|
+
export function resolveLedgerCostStatus(input) {
|
|
1651
|
+
const hasTokenUsage = input.inputTokens > 0 || input.cachedInputTokens > 0 || input.outputTokens > 0;
|
|
1652
|
+
return input.costUsd == null && hasTokenUsage ? "unpriced" : "reported";
|
|
1653
|
+
}
|
|
1471
1654
|
async function resolveLedgerScopeForRun(db, companyId, run) {
|
|
1472
1655
|
const context = parseObject(run.contextSnapshot);
|
|
1473
1656
|
const contextIssueId = readNonEmptyString(context.issueId);
|
|
@@ -1727,6 +1910,8 @@ export function shouldResetTaskSessionForWake(contextSnapshot) {
|
|
|
1727
1910
|
return false;
|
|
1728
1911
|
}
|
|
1729
1912
|
function shouldRequireIssueCommentForWake(contextSnapshot) {
|
|
1913
|
+
if (contextSnapshot?.skipIssueComment === true)
|
|
1914
|
+
return false;
|
|
1730
1915
|
const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason);
|
|
1731
1916
|
return (wakeReason === "issue_assigned" ||
|
|
1732
1917
|
wakeReason === "execution_review_requested" ||
|
|
@@ -1854,6 +2039,96 @@ const EFFECTIVE_RUN_WORKSPACE_CONFIG_CATEGORIES = [
|
|
|
1854
2039
|
"environment",
|
|
1855
2040
|
"realization",
|
|
1856
2041
|
];
|
|
2042
|
+
export function resolveExecutionWorkspaceReuseRequestForIssue(input) {
|
|
2043
|
+
const requestedExecutionWorkspaceId = readNonEmptyString(input.issueExecutionWorkspaceId);
|
|
2044
|
+
const requestedShouldReuseExisting = input.issueExecutionWorkspacePreference === "reuse_existing" && requestedExecutionWorkspaceId !== null;
|
|
2045
|
+
return {
|
|
2046
|
+
requestedExecutionWorkspaceId,
|
|
2047
|
+
requestedShouldReuseExisting,
|
|
2048
|
+
existingExecutionWorkspaceAvailable: requestedShouldReuseExisting &&
|
|
2049
|
+
input.existingExecutionWorkspaceStatus !== null &&
|
|
2050
|
+
input.existingExecutionWorkspaceStatus !== undefined &&
|
|
2051
|
+
input.existingExecutionWorkspaceStatus !== "archived",
|
|
2052
|
+
};
|
|
2053
|
+
}
|
|
2054
|
+
export function resolveExecutionWorkspaceReuseProvisioningPolicy(input) {
|
|
2055
|
+
const shouldRestoreExistingWorkspace = input.requestedShouldReuseExisting;
|
|
2056
|
+
const replacementClassDrift = input.requestedShouldReuseExisting && input.workspaceConfigFreshness.action === "replace";
|
|
2057
|
+
return {
|
|
2058
|
+
shouldRestoreExistingWorkspace,
|
|
2059
|
+
shouldRefreshWorkspaceConfigSnapshot: shouldRestoreExistingWorkspace &&
|
|
2060
|
+
!replacementClassDrift &&
|
|
2061
|
+
input.workspaceConfigFreshness.shouldRefreshConfigSnapshot,
|
|
2062
|
+
shouldPersistLatestWorkspaceConfigMetadata: !replacementClassDrift,
|
|
2063
|
+
};
|
|
2064
|
+
}
|
|
2065
|
+
function formatInheritedExecutionWorkspaceReuseFailure(input) {
|
|
2066
|
+
const issueLabel = input.issueRef?.identifier ?? input.issueRef?.id ?? input.runId;
|
|
2067
|
+
const workspaceLabel = input.executionWorkspaceId ?? "unknown workspace";
|
|
2068
|
+
const causeMessage = input.cause instanceof Error
|
|
2069
|
+
? input.cause.message
|
|
2070
|
+
: input.cause != null
|
|
2071
|
+
? String(input.cause)
|
|
2072
|
+
: null;
|
|
2073
|
+
const remediation = input.reason === "inherited_workspace_reuse_failed"
|
|
2074
|
+
? "Inspect the referenced execution workspace restore/provision logs, repair or unarchive the workspace, or intentionally clear the issue's reuse_existing workspace binding before retrying."
|
|
2075
|
+
: "Repair or unarchive the referenced execution workspace, or intentionally clear the issue's reuse_existing workspace binding before retrying.";
|
|
2076
|
+
const message = causeMessage
|
|
2077
|
+
? `Issue ${issueLabel} requested inherited execution workspace reuse for ${workspaceLabel}, but the workspace could not be restored because ${causeMessage}.`
|
|
2078
|
+
: `Issue ${issueLabel} requested inherited execution workspace reuse for ${workspaceLabel}, but the workspace could not be restored.`;
|
|
2079
|
+
return `${message} ${remediation}`;
|
|
2080
|
+
}
|
|
2081
|
+
export async function provisionExecutionWorkspaceForFreshnessDecision(input) {
|
|
2082
|
+
const policy = resolveExecutionWorkspaceReuseProvisioningPolicy({
|
|
2083
|
+
requestedShouldReuseExisting: input.requestedShouldReuseExisting,
|
|
2084
|
+
workspaceConfigFreshness: input.workspaceConfigFreshness,
|
|
2085
|
+
});
|
|
2086
|
+
if (!policy.shouldRestoreExistingWorkspace) {
|
|
2087
|
+
const executionWorkspace = await input.realizeWorkspace();
|
|
2088
|
+
return {
|
|
2089
|
+
executionWorkspace,
|
|
2090
|
+
reusedExecutionWorkspace: null,
|
|
2091
|
+
policy,
|
|
2092
|
+
};
|
|
2093
|
+
}
|
|
2094
|
+
let restored = null;
|
|
2095
|
+
let reuseFailure = null;
|
|
2096
|
+
try {
|
|
2097
|
+
restored = (await input.restoreExistingWorkspace?.()) ?? null;
|
|
2098
|
+
}
|
|
2099
|
+
catch (error) {
|
|
2100
|
+
if (isWorkspaceValidationFailure(error)) {
|
|
2101
|
+
throw error;
|
|
2102
|
+
}
|
|
2103
|
+
reuseFailure = formatInheritedExecutionWorkspaceReuseFailure({
|
|
2104
|
+
reason: "inherited_workspace_reuse_failed",
|
|
2105
|
+
issueRef: input.issueRef,
|
|
2106
|
+
runId: input.runId,
|
|
2107
|
+
executionWorkspaceId: input.existingExecutionWorkspaceId,
|
|
2108
|
+
workspaceConfigFreshness: input.workspaceConfigFreshness,
|
|
2109
|
+
cause: error,
|
|
2110
|
+
});
|
|
2111
|
+
}
|
|
2112
|
+
if (!restored) {
|
|
2113
|
+
reuseFailure = reuseFailure ?? formatInheritedExecutionWorkspaceReuseFailure({
|
|
2114
|
+
reason: "inherited_workspace_reuse_unavailable",
|
|
2115
|
+
issueRef: input.issueRef,
|
|
2116
|
+
runId: input.runId,
|
|
2117
|
+
executionWorkspaceId: input.existingExecutionWorkspaceId,
|
|
2118
|
+
workspaceConfigFreshness: input.workspaceConfigFreshness,
|
|
2119
|
+
});
|
|
2120
|
+
}
|
|
2121
|
+
if (reuseFailure)
|
|
2122
|
+
throw new Error(reuseFailure);
|
|
2123
|
+
if (!restored) {
|
|
2124
|
+
throw new Error("Expected restored execution workspace after reuse fallback handling");
|
|
2125
|
+
}
|
|
2126
|
+
return {
|
|
2127
|
+
executionWorkspace: restored,
|
|
2128
|
+
reusedExecutionWorkspace: restored,
|
|
2129
|
+
policy,
|
|
2130
|
+
};
|
|
2131
|
+
}
|
|
1857
2132
|
const EFFECTIVE_RUN_SESSION_CONFIG_CATEGORY_LABELS = {
|
|
1858
2133
|
adapter: "adapter",
|
|
1859
2134
|
adapterConfig: "adapter config",
|
|
@@ -2421,11 +2696,14 @@ export function resolveTaskSessionConfigFreshness(input) {
|
|
|
2421
2696
|
nextFingerprint: input.configMetadata?.fingerprint ?? null,
|
|
2422
2697
|
};
|
|
2423
2698
|
}
|
|
2424
|
-
function shouldAutoCheckoutIssueForWake(input) {
|
|
2699
|
+
export function shouldAutoCheckoutIssueForWake(input) {
|
|
2425
2700
|
if (input.issueAssigneeAgentId !== input.agentId)
|
|
2426
2701
|
return false;
|
|
2427
2702
|
if (!input.isDependencyReady)
|
|
2428
2703
|
return false;
|
|
2704
|
+
const executionState = parseIssueExecutionState(input.issueExecutionState);
|
|
2705
|
+
if (executionState?.status === "pending")
|
|
2706
|
+
return false;
|
|
2429
2707
|
const issueStatus = readNonEmptyString(input.issueStatus);
|
|
2430
2708
|
if (issueStatus !== "todo" &&
|
|
2431
2709
|
issueStatus !== "backlog" &&
|
|
@@ -2560,6 +2838,8 @@ const INTERACTION_CONTINUATION_CONTEXT_KEYS = [
|
|
|
2560
2838
|
"interactionStatus",
|
|
2561
2839
|
"continuationPolicy",
|
|
2562
2840
|
"checkboxSelection",
|
|
2841
|
+
"itemVerdicts",
|
|
2842
|
+
"newlyResolvedItemIds",
|
|
2563
2843
|
];
|
|
2564
2844
|
function isInteractionResolutionWakePayload(payload) {
|
|
2565
2845
|
return readNonEmptyString(payload?.mutation) === "interaction";
|
|
@@ -2839,6 +3119,7 @@ export async function buildPaperclipWakePayload(input) {
|
|
|
2839
3119
|
: [],
|
|
2840
3120
|
executionStage: Object.keys(executionStage).length > 0 ? executionStage : null,
|
|
2841
3121
|
taskWatchdog: (input.contextSnapshot.taskWatchdog ?? null),
|
|
3122
|
+
skillTest: (input.contextSnapshot.paperclipSkillTest ?? null),
|
|
2842
3123
|
continuationSummary: safeContinuationSummary
|
|
2843
3124
|
? {
|
|
2844
3125
|
key: safeContinuationSummary.key,
|
|
@@ -3067,6 +3348,9 @@ export function buildPaperclipTaskMarkdown(input) {
|
|
|
3067
3348
|
}
|
|
3068
3349
|
lines.push(`- Work mode: ${quoteTaskScalar("planning")}`, "", "Planning mode directive:", directive);
|
|
3069
3350
|
}
|
|
3351
|
+
else if (issue.workMode === "skill_test") {
|
|
3352
|
+
lines.push(`- Work mode: ${quoteTaskScalar("skill_test")}`, "", "Skill test mode directive:", "You are testing a pinned skill revision. Make no durable changes outside this issue. Do not push, publish, send external messages, or mutate other issues. Write your final output as issue document `output`, then finish by marking this issue done.");
|
|
3353
|
+
}
|
|
3070
3354
|
else if (acceptedPlanContinuation) {
|
|
3071
3355
|
lines.push("", "Accepted plan directive:", "Create child issues from the approved plan only. Do not write code or perform implementation work on the source issue.");
|
|
3072
3356
|
}
|
|
@@ -3178,6 +3462,30 @@ export function normalizeSessionParams(params) {
|
|
|
3178
3462
|
return null;
|
|
3179
3463
|
return Object.keys(params).length > 0 ? params : null;
|
|
3180
3464
|
}
|
|
3465
|
+
export function resolveSkillTestRunCompletionForHeartbeatOutcome(outcome, error) {
|
|
3466
|
+
if (outcome === "cancelled") {
|
|
3467
|
+
return {
|
|
3468
|
+
outcome: "cancelled",
|
|
3469
|
+
error: error ?? "Harness run was cancelled",
|
|
3470
|
+
heartbeatOutcome: outcome,
|
|
3471
|
+
};
|
|
3472
|
+
}
|
|
3473
|
+
if (outcome === "timed_out") {
|
|
3474
|
+
return {
|
|
3475
|
+
outcome: "failed",
|
|
3476
|
+
error: error ?? "Timed out",
|
|
3477
|
+
heartbeatOutcome: outcome,
|
|
3478
|
+
};
|
|
3479
|
+
}
|
|
3480
|
+
if (outcome === "failed") {
|
|
3481
|
+
return {
|
|
3482
|
+
outcome: "failed",
|
|
3483
|
+
error: error ?? "Adapter failed",
|
|
3484
|
+
heartbeatOutcome: outcome,
|
|
3485
|
+
};
|
|
3486
|
+
}
|
|
3487
|
+
return null;
|
|
3488
|
+
}
|
|
3181
3489
|
const HERMES_ADAPTER_TYPE = "hermes_local";
|
|
3182
3490
|
const HERMES_SESSION_ID_REGEX = /^(?:\d{8}_\d{6}_[A-Za-z0-9_-]{4,}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$/;
|
|
3183
3491
|
function requiresCanonicalSessionIds(adapterType) {
|
|
@@ -3289,11 +3597,70 @@ export function resolveNextSessionState(input) {
|
|
|
3289
3597
|
legacySessionId,
|
|
3290
3598
|
};
|
|
3291
3599
|
}
|
|
3600
|
+
function isTruthyRuntimeEnvValue(value) {
|
|
3601
|
+
return value === "true" || value === "1" || value === "yes" || value === "on";
|
|
3602
|
+
}
|
|
3603
|
+
export function resolveHeartbeatSchedulingSuppression(env = process.env, overrides = {}) {
|
|
3604
|
+
if (isTruthyRuntimeEnvValue(env.PAPERCLIP_IN_WORKTREE) && !overrides.allowWorktreeRunExecution) {
|
|
3605
|
+
return { suppressed: true, reason: "worktree_instance" };
|
|
3606
|
+
}
|
|
3607
|
+
if (isTruthyRuntimeEnvValue(env.PAPERCLIP_DATABASE_RESTORE_IN_PROGRESS) ||
|
|
3608
|
+
isTruthyRuntimeEnvValue(env.PAPERCLIP_RESTORE_IN_PROGRESS)) {
|
|
3609
|
+
return { suppressed: true, reason: "database_restore_in_progress" };
|
|
3610
|
+
}
|
|
3611
|
+
return { suppressed: false, reason: null };
|
|
3612
|
+
}
|
|
3292
3613
|
export function heartbeatService(db, options = {}) {
|
|
3293
3614
|
const instanceSettings = instanceSettingsService(db);
|
|
3294
3615
|
const getCurrentUserRedactionOptions = async () => ({
|
|
3295
3616
|
enabled: (await instanceSettings.getGeneral()).censorUsernameInLogs,
|
|
3296
3617
|
});
|
|
3618
|
+
const runtimeEnv = options.runtimeEnv ?? process.env;
|
|
3619
|
+
const inWorktreeRuntime = isTruthyRuntimeEnvValue(runtimeEnv.PAPERCLIP_IN_WORKTREE);
|
|
3620
|
+
// Preview worktree instances suppress the run engine by default. Users can lift
|
|
3621
|
+
// that per-worktree via the `enableWorktreeRunExecution` experimental setting
|
|
3622
|
+
// (worktree instances have their own isolated DB, so it can't affect the parent).
|
|
3623
|
+
// Only worktree runtimes ever read the setting; a short TTL keeps the hot-path
|
|
3624
|
+
// suppression checks off the DB, and a read failure falls back to prior/default
|
|
3625
|
+
// (fail closed to suppression).
|
|
3626
|
+
let cachedWorktreeRunExecutionOverride = {
|
|
3627
|
+
allowed: false,
|
|
3628
|
+
cutoff: null,
|
|
3629
|
+
at: 0,
|
|
3630
|
+
};
|
|
3631
|
+
const WORKTREE_RUN_EXECUTION_OVERRIDE_TTL_MS = 3_000;
|
|
3632
|
+
const resolveWorktreeRunExecutionOverride = async () => {
|
|
3633
|
+
if (!inWorktreeRuntime)
|
|
3634
|
+
return { allowed: false, cutoff: null };
|
|
3635
|
+
const now = Date.now();
|
|
3636
|
+
if (now - cachedWorktreeRunExecutionOverride.at < WORKTREE_RUN_EXECUTION_OVERRIDE_TTL_MS) {
|
|
3637
|
+
return cachedWorktreeRunExecutionOverride;
|
|
3638
|
+
}
|
|
3639
|
+
try {
|
|
3640
|
+
const activation = resolveWorktreeRunExecutionActivation(await instanceSettings.getExperimental(), runtimeEnv.PAPERCLIP_INSTANCE_ID?.trim() || null);
|
|
3641
|
+
const cutoff = activation.armed ? new Date(activation.cutoff) : null;
|
|
3642
|
+
cachedWorktreeRunExecutionOverride = {
|
|
3643
|
+
allowed: Boolean(activation.armed && cutoff && !Number.isNaN(cutoff.getTime())),
|
|
3644
|
+
cutoff: cutoff && !Number.isNaN(cutoff.getTime()) ? cutoff : null,
|
|
3645
|
+
at: now,
|
|
3646
|
+
};
|
|
3647
|
+
}
|
|
3648
|
+
catch {
|
|
3649
|
+
// Keep the prior (default-false) value so a settings read failure fails
|
|
3650
|
+
// closed to the safe suppressed state.
|
|
3651
|
+
}
|
|
3652
|
+
return cachedWorktreeRunExecutionOverride;
|
|
3653
|
+
};
|
|
3654
|
+
const getSchedulingSuppression = async () => {
|
|
3655
|
+
const override = await resolveWorktreeRunExecutionOverride();
|
|
3656
|
+
return resolveHeartbeatSchedulingSuppression(runtimeEnv, {
|
|
3657
|
+
allowWorktreeRunExecution: override.allowed,
|
|
3658
|
+
});
|
|
3659
|
+
};
|
|
3660
|
+
const getWorktreeExecutionCutoff = async () => {
|
|
3661
|
+
const override = await resolveWorktreeRunExecutionOverride();
|
|
3662
|
+
return override.allowed ? override.cutoff : null;
|
|
3663
|
+
};
|
|
3297
3664
|
const runLogStore = getRunLogStore();
|
|
3298
3665
|
const secretsSvc = secretService(db);
|
|
3299
3666
|
const companySkills = companySkillService(db);
|
|
@@ -3319,9 +3686,219 @@ export function heartbeatService(db, options = {}) {
|
|
|
3319
3686
|
};
|
|
3320
3687
|
const budgets = budgetService(db, budgetHooks);
|
|
3321
3688
|
const recovery = recoveryService(db, { enqueueWakeup });
|
|
3689
|
+
function isPlanApprovalConfirmationPayload(payload) {
|
|
3690
|
+
const target = parseObject(parseObject(payload).target);
|
|
3691
|
+
return readNonEmptyString(target.type) === "issue_document" &&
|
|
3692
|
+
readNonEmptyString(target.key) === "plan";
|
|
3693
|
+
}
|
|
3694
|
+
async function getAcceptedPlanApprovalInteractionForRun(run, issueId) {
|
|
3695
|
+
const context = parseObject(run.contextSnapshot);
|
|
3696
|
+
const interactionId = readNonEmptyString(context.interactionId);
|
|
3697
|
+
if (!issueId || !interactionId)
|
|
3698
|
+
return null;
|
|
3699
|
+
const interaction = await db
|
|
3700
|
+
.select({
|
|
3701
|
+
id: issueThreadInteractions.id,
|
|
3702
|
+
kind: issueThreadInteractions.kind,
|
|
3703
|
+
status: issueThreadInteractions.status,
|
|
3704
|
+
payload: issueThreadInteractions.payload,
|
|
3705
|
+
result: issueThreadInteractions.result,
|
|
3706
|
+
})
|
|
3707
|
+
.from(issueThreadInteractions)
|
|
3708
|
+
.where(and(eq(issueThreadInteractions.companyId, run.companyId), eq(issueThreadInteractions.issueId, issueId), eq(issueThreadInteractions.id, interactionId)))
|
|
3709
|
+
.then((rows) => rows[0] ?? null);
|
|
3710
|
+
if (!interaction)
|
|
3711
|
+
return null;
|
|
3712
|
+
if (interaction.kind !== "request_confirmation" || interaction.status !== "accepted")
|
|
3713
|
+
return null;
|
|
3714
|
+
return isPlanApprovalConfirmationPayload(interaction.payload) ? interaction : null;
|
|
3715
|
+
}
|
|
3716
|
+
function planApprovalResumeFailureErrorCode(run) {
|
|
3717
|
+
return readNonEmptyString(run.errorCode) ?? "unknown_error";
|
|
3718
|
+
}
|
|
3719
|
+
function buildPlanApprovalResumeFailureComment(input) {
|
|
3720
|
+
const errorCode = planApprovalResumeFailureErrorCode(input.run);
|
|
3721
|
+
if (input.status === "retrying") {
|
|
3722
|
+
return `Agent failed to resume after approval: \`${errorCode}\` — retrying (attempt ${input.attempt}/${input.maxAttempts})`;
|
|
3723
|
+
}
|
|
3724
|
+
return `Agent failed to resume after approval: \`${errorCode}\` — needs attention`;
|
|
3725
|
+
}
|
|
3726
|
+
function buildPlanApprovalResumeFailureResult(input) {
|
|
3727
|
+
return {
|
|
3728
|
+
status: input.status,
|
|
3729
|
+
errorCode: planApprovalResumeFailureErrorCode(input.run),
|
|
3730
|
+
attempt: input.attempt,
|
|
3731
|
+
maxAttempts: input.maxAttempts,
|
|
3732
|
+
runId: input.run.id,
|
|
3733
|
+
retryRunId: input.retryRunId ?? null,
|
|
3734
|
+
recoveryActionId: input.recoveryActionId ?? null,
|
|
3735
|
+
updatedAt: new Date().toISOString(),
|
|
3736
|
+
};
|
|
3737
|
+
}
|
|
3738
|
+
async function updatePlanApprovalInteractionResumeFailure(input) {
|
|
3739
|
+
const result = parseObject(input.interaction.result);
|
|
3740
|
+
const nextResult = {
|
|
3741
|
+
...result,
|
|
3742
|
+
version: 1,
|
|
3743
|
+
outcome: "accepted",
|
|
3744
|
+
resumeFailure: input.failure,
|
|
3745
|
+
};
|
|
3746
|
+
await db
|
|
3747
|
+
.update(issueThreadInteractions)
|
|
3748
|
+
.set({
|
|
3749
|
+
result: nextResult,
|
|
3750
|
+
updatedAt: new Date(),
|
|
3751
|
+
})
|
|
3752
|
+
.where(eq(issueThreadInteractions.id, input.interaction.id));
|
|
3753
|
+
}
|
|
3754
|
+
async function addPlanApprovalResumeFailureCommentOnce(input) {
|
|
3755
|
+
const existing = await db
|
|
3756
|
+
.select({ id: issueComments.id })
|
|
3757
|
+
.from(issueComments)
|
|
3758
|
+
.where(and(eq(issueComments.companyId, input.run.companyId), eq(issueComments.issueId, input.issueId), or(eq(issueComments.body, input.body), sql `${issueComments.body} like ${`${input.body}\n%`}`)))
|
|
3759
|
+
.limit(1)
|
|
3760
|
+
.then((rows) => rows[0] ?? null);
|
|
3761
|
+
if (existing)
|
|
3762
|
+
return null;
|
|
3763
|
+
return issuesSvc.addComment(input.issueId, input.body, { runId: input.run.id }, { authorType: "system" });
|
|
3764
|
+
}
|
|
3765
|
+
async function getActiveRecoveryActionId(companyId, sourceIssueId) {
|
|
3766
|
+
return db
|
|
3767
|
+
.select({ id: issueRecoveryActions.id })
|
|
3768
|
+
.from(issueRecoveryActions)
|
|
3769
|
+
.where(and(eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, sourceIssueId), inArray(issueRecoveryActions.status, ["active", "escalated"])))
|
|
3770
|
+
.orderBy(desc(issueRecoveryActions.updatedAt))
|
|
3771
|
+
.limit(1)
|
|
3772
|
+
.then((rows) => rows[0]?.id ?? null);
|
|
3773
|
+
}
|
|
3774
|
+
async function recordPlanApprovalResumeFailureRetry(input) {
|
|
3775
|
+
const interaction = await getAcceptedPlanApprovalInteractionForRun(input.run, input.issueId);
|
|
3776
|
+
if (!interaction || !input.issueId)
|
|
3777
|
+
return null;
|
|
3778
|
+
const body = buildPlanApprovalResumeFailureComment({
|
|
3779
|
+
run: input.run,
|
|
3780
|
+
status: "retrying",
|
|
3781
|
+
attempt: input.attempt,
|
|
3782
|
+
maxAttempts: input.maxAttempts,
|
|
3783
|
+
});
|
|
3784
|
+
await addPlanApprovalResumeFailureCommentOnce({
|
|
3785
|
+
issueId: input.issueId,
|
|
3786
|
+
run: input.run,
|
|
3787
|
+
body,
|
|
3788
|
+
});
|
|
3789
|
+
await updatePlanApprovalInteractionResumeFailure({
|
|
3790
|
+
interaction,
|
|
3791
|
+
failure: buildPlanApprovalResumeFailureResult({
|
|
3792
|
+
run: input.run,
|
|
3793
|
+
status: "retrying",
|
|
3794
|
+
attempt: input.attempt,
|
|
3795
|
+
maxAttempts: input.maxAttempts,
|
|
3796
|
+
retryRunId: input.retryRunId,
|
|
3797
|
+
}),
|
|
3798
|
+
});
|
|
3799
|
+
return interaction.id;
|
|
3800
|
+
}
|
|
3801
|
+
async function escalatePlanApprovalResumeFailureNeedsAttention(input) {
|
|
3802
|
+
const interaction = await getAcceptedPlanApprovalInteractionForRun(input.run, input.issueId);
|
|
3803
|
+
if (!interaction || !input.issueId)
|
|
3804
|
+
return null;
|
|
3805
|
+
const issue = await db
|
|
3806
|
+
.select()
|
|
3807
|
+
.from(issues)
|
|
3808
|
+
.where(and(eq(issues.companyId, input.run.companyId), eq(issues.id, input.issueId)))
|
|
3809
|
+
.then((rows) => rows[0] ?? null);
|
|
3810
|
+
if (!issue)
|
|
3811
|
+
return null;
|
|
3812
|
+
if (issue.status !== "todo" && issue.status !== "in_progress" && issue.status !== "in_review")
|
|
3813
|
+
return null;
|
|
3814
|
+
const body = buildPlanApprovalResumeFailureComment({
|
|
3815
|
+
run: input.run,
|
|
3816
|
+
status: "needs_attention",
|
|
3817
|
+
attempt: input.attempt,
|
|
3818
|
+
maxAttempts: input.maxAttempts,
|
|
3819
|
+
});
|
|
3820
|
+
await recovery.escalateStrandedAssignedIssue({
|
|
3821
|
+
issue,
|
|
3822
|
+
previousStatus: issue.status,
|
|
3823
|
+
latestRun: input.run,
|
|
3824
|
+
comment: body,
|
|
3825
|
+
});
|
|
3826
|
+
await addPlanApprovalResumeFailureCommentOnce({
|
|
3827
|
+
issueId: issue.id,
|
|
3828
|
+
run: input.run,
|
|
3829
|
+
body,
|
|
3830
|
+
});
|
|
3831
|
+
const recoveryActionId = await getActiveRecoveryActionId(issue.companyId, issue.id);
|
|
3832
|
+
await updatePlanApprovalInteractionResumeFailure({
|
|
3833
|
+
interaction,
|
|
3834
|
+
failure: buildPlanApprovalResumeFailureResult({
|
|
3835
|
+
run: input.run,
|
|
3836
|
+
status: "needs_attention",
|
|
3837
|
+
attempt: input.attempt,
|
|
3838
|
+
maxAttempts: input.maxAttempts,
|
|
3839
|
+
recoveryActionId,
|
|
3840
|
+
}),
|
|
3841
|
+
});
|
|
3842
|
+
return interaction.id;
|
|
3843
|
+
}
|
|
3322
3844
|
const productivityReviews = productivityReviewService(db, { enqueueWakeup });
|
|
3323
3845
|
const taskWatchdogs = taskWatchdogService(db, { enqueueWakeup });
|
|
3324
3846
|
let unsafeTextProjectionPromise = null;
|
|
3847
|
+
async function completeSkillTestRunForHeartbeatOutcome(input) {
|
|
3848
|
+
const completion = resolveSkillTestRunCompletionForHeartbeatOutcome(input.outcome, input.error);
|
|
3849
|
+
if (!completion || !input.issueId)
|
|
3850
|
+
return null;
|
|
3851
|
+
let isSkillTestIssue = input.issueWorkMode === "skill_test";
|
|
3852
|
+
if (!isSkillTestIssue && input.issueWorkMode === undefined) {
|
|
3853
|
+
const issueRow = await db
|
|
3854
|
+
.select({
|
|
3855
|
+
workMode: issues.workMode,
|
|
3856
|
+
harnessKind: issues.harnessKind,
|
|
3857
|
+
})
|
|
3858
|
+
.from(issues)
|
|
3859
|
+
.where(and(eq(issues.companyId, input.run.companyId), eq(issues.id, input.issueId)))
|
|
3860
|
+
.then((rows) => rows[0] ?? null);
|
|
3861
|
+
isSkillTestIssue = issueRow?.workMode === "skill_test" || issueRow?.harnessKind === "skill_test";
|
|
3862
|
+
}
|
|
3863
|
+
if (!isSkillTestIssue)
|
|
3864
|
+
return null;
|
|
3865
|
+
const existingRun = await db
|
|
3866
|
+
.select({
|
|
3867
|
+
id: companySkillTestRuns.id,
|
|
3868
|
+
status: companySkillTestRuns.status,
|
|
3869
|
+
})
|
|
3870
|
+
.from(companySkillTestRuns)
|
|
3871
|
+
.where(and(eq(companySkillTestRuns.companyId, input.run.companyId), eq(companySkillTestRuns.issueId, input.issueId)))
|
|
3872
|
+
.then((rows) => rows[0] ?? null);
|
|
3873
|
+
if (!existingRun || ["succeeded", "failed", "cancelled"].includes(existingRun.status))
|
|
3874
|
+
return null;
|
|
3875
|
+
const completedRun = await companySkills.completeTestRunForIssue({
|
|
3876
|
+
companyId: input.run.companyId,
|
|
3877
|
+
issueId: input.issueId,
|
|
3878
|
+
outcome: completion.outcome,
|
|
3879
|
+
error: completion.error,
|
|
3880
|
+
});
|
|
3881
|
+
if (!completedRun)
|
|
3882
|
+
return null;
|
|
3883
|
+
await logActivity(db, {
|
|
3884
|
+
companyId: input.run.companyId,
|
|
3885
|
+
actorType: "system",
|
|
3886
|
+
actorId: "heartbeat_finalize",
|
|
3887
|
+
agentId: input.run.agentId,
|
|
3888
|
+
runId: input.run.id,
|
|
3889
|
+
action: "company.skill_test_run_completed",
|
|
3890
|
+
entityType: "company_skill_test_run",
|
|
3891
|
+
entityId: completedRun.id,
|
|
3892
|
+
details: {
|
|
3893
|
+
issueId: input.issueId,
|
|
3894
|
+
status: completedRun.status,
|
|
3895
|
+
outputDocumentKey: completedRun.outputDocumentKey,
|
|
3896
|
+
heartbeatOutcome: completion.heartbeatOutcome,
|
|
3897
|
+
source: "heartbeat.run_finalized",
|
|
3898
|
+
},
|
|
3899
|
+
});
|
|
3900
|
+
return completedRun;
|
|
3901
|
+
}
|
|
3325
3902
|
async function releaseEnvironmentLeasesForRun(input) {
|
|
3326
3903
|
const releaseResult = await envOrchestrator.releaseForRun({
|
|
3327
3904
|
heartbeatRunId: input.runId,
|
|
@@ -3447,7 +4024,11 @@ export function heartbeatService(db, options = {}) {
|
|
|
3447
4024
|
assigneeAgentId: issues.assigneeAgentId,
|
|
3448
4025
|
assigneeAdapterOverrides: issues.assigneeAdapterOverrides,
|
|
3449
4026
|
executionPolicy: issues.executionPolicy,
|
|
4027
|
+
executionState: issues.executionState,
|
|
3450
4028
|
executionWorkspaceSettings: issues.executionWorkspaceSettings,
|
|
4029
|
+
parentId: issues.parentId,
|
|
4030
|
+
createdByUserId: issues.createdByUserId,
|
|
4031
|
+
responsibleUserId: issues.responsibleUserId,
|
|
3451
4032
|
originKind: issues.originKind,
|
|
3452
4033
|
originId: issues.originId,
|
|
3453
4034
|
originRunId: issues.originRunId,
|
|
@@ -3457,14 +4038,59 @@ export function heartbeatService(db, options = {}) {
|
|
|
3457
4038
|
.where(and(eq(issues.id, issueId), eq(issues.companyId, companyId)))
|
|
3458
4039
|
.then((rows) => rows[0] ?? null);
|
|
3459
4040
|
}
|
|
4041
|
+
async function getPinnedSkillTestContext(companyId, issueId) {
|
|
4042
|
+
const row = await db
|
|
4043
|
+
.select({
|
|
4044
|
+
testRunId: companySkillTestRuns.id,
|
|
4045
|
+
skillId: companySkillTestRuns.skillId,
|
|
4046
|
+
inputId: companySkillTestRuns.inputId,
|
|
4047
|
+
skillVersionId: companySkillTestRuns.skillVersionId,
|
|
4048
|
+
outputDocumentKey: companySkillTestRuns.outputDocumentKey,
|
|
4049
|
+
fileInventory: companySkillVersions.fileInventory,
|
|
4050
|
+
revisionNumber: companySkillVersions.revisionNumber,
|
|
4051
|
+
label: companySkillVersions.label,
|
|
4052
|
+
})
|
|
4053
|
+
.from(companySkillTestRuns)
|
|
4054
|
+
.innerJoin(companySkillVersions, and(eq(companySkillVersions.id, companySkillTestRuns.skillVersionId), eq(companySkillVersions.companyId, companySkillTestRuns.companyId)))
|
|
4055
|
+
.where(and(eq(companySkillTestRuns.companyId, companyId), eq(companySkillTestRuns.issueId, issueId)))
|
|
4056
|
+
.then((rows) => rows[0] ?? null);
|
|
4057
|
+
if (!row)
|
|
4058
|
+
return null;
|
|
4059
|
+
const fileInventory = Array.isArray(row.fileInventory)
|
|
4060
|
+
? row.fileInventory.flatMap((entry) => {
|
|
4061
|
+
if (!entry || typeof entry !== "object" || Array.isArray(entry))
|
|
4062
|
+
return [];
|
|
4063
|
+
const record = entry;
|
|
4064
|
+
const path = typeof record.path === "string" ? record.path : "";
|
|
4065
|
+
if (!path)
|
|
4066
|
+
return [];
|
|
4067
|
+
return [{
|
|
4068
|
+
path,
|
|
4069
|
+
kind: typeof record.kind === "string" ? record.kind : "other",
|
|
4070
|
+
content: typeof record.content === "string" ? record.content : "",
|
|
4071
|
+
}];
|
|
4072
|
+
})
|
|
4073
|
+
: [];
|
|
4074
|
+
return {
|
|
4075
|
+
testRunId: row.testRunId,
|
|
4076
|
+
skillId: row.skillId,
|
|
4077
|
+
inputId: row.inputId ?? null,
|
|
4078
|
+
skillVersionId: row.skillVersionId,
|
|
4079
|
+
revisionNumber: row.revisionNumber,
|
|
4080
|
+
label: row.label ?? null,
|
|
4081
|
+
outputDocumentKey: row.outputDocumentKey,
|
|
4082
|
+
fileInventory,
|
|
4083
|
+
};
|
|
4084
|
+
}
|
|
3460
4085
|
async function getRoutineEnvForExecutionIssue(companyId, issueContext) {
|
|
3461
4086
|
if (!issueContext || issueContext.originKind !== "routine_execution" || !issueContext.originId) {
|
|
3462
|
-
return { routineId: null, env: null };
|
|
4087
|
+
return { routineId: null, env: null, responsibleUserId: null };
|
|
3463
4088
|
}
|
|
3464
4089
|
const routineRun = issueContext.originRunId
|
|
3465
4090
|
? await db
|
|
3466
4091
|
.select({
|
|
3467
4092
|
routineRevisionId: routineRuns.routineRevisionId,
|
|
4093
|
+
responsibleUserId: routineRuns.responsibleUserId,
|
|
3468
4094
|
})
|
|
3469
4095
|
.from(routineRuns)
|
|
3470
4096
|
.where(and(eq(routineRuns.id, issueContext.originRunId), eq(routineRuns.companyId, companyId), eq(routineRuns.routineId, issueContext.originId)))
|
|
@@ -3474,21 +4100,136 @@ export function heartbeatService(db, options = {}) {
|
|
|
3474
4100
|
const revision = await db
|
|
3475
4101
|
.select({
|
|
3476
4102
|
snapshot: routineRevisions.snapshot,
|
|
4103
|
+
responsibleUserId: routineRevisions.responsibleUserId,
|
|
3477
4104
|
})
|
|
3478
4105
|
.from(routineRevisions)
|
|
3479
4106
|
.where(and(eq(routineRevisions.id, routineRun.routineRevisionId), eq(routineRevisions.companyId, companyId), eq(routineRevisions.routineId, issueContext.originId)))
|
|
3480
4107
|
.then((rows) => rows[0] ?? null);
|
|
3481
4108
|
const snapshot = revision?.snapshot;
|
|
3482
4109
|
if (snapshot?.version === 1) {
|
|
3483
|
-
return {
|
|
4110
|
+
return {
|
|
4111
|
+
routineId: issueContext.originId,
|
|
4112
|
+
env: snapshot.routine.env ?? null,
|
|
4113
|
+
responsibleUserId: revision?.responsibleUserId ?? snapshot.routine.responsibleUserId ?? null,
|
|
4114
|
+
};
|
|
3484
4115
|
}
|
|
3485
4116
|
}
|
|
3486
4117
|
const routine = await db
|
|
3487
|
-
.select({ env: routines.env })
|
|
4118
|
+
.select({ env: routines.env, responsibleUserId: routines.responsibleUserId })
|
|
3488
4119
|
.from(routines)
|
|
3489
4120
|
.where(and(eq(routines.id, issueContext.originId), eq(routines.companyId, companyId)))
|
|
3490
4121
|
.then((rows) => rows[0] ?? null);
|
|
3491
|
-
return {
|
|
4122
|
+
return {
|
|
4123
|
+
routineId: issueContext.originId,
|
|
4124
|
+
env: routine?.env ?? null,
|
|
4125
|
+
responsibleUserId: routineRun?.responsibleUserId ?? routine?.responsibleUserId ?? null,
|
|
4126
|
+
};
|
|
4127
|
+
}
|
|
4128
|
+
async function resolveCompanyDefaultResponsibleUserId(companyId) {
|
|
4129
|
+
const company = await db
|
|
4130
|
+
.select({ defaultResponsibleUserId: companies.defaultResponsibleUserId })
|
|
4131
|
+
.from(companies)
|
|
4132
|
+
.where(eq(companies.id, companyId))
|
|
4133
|
+
.then((rows) => rows[0] ?? null);
|
|
4134
|
+
const explicitDefault = readNonEmptyString(company?.defaultResponsibleUserId);
|
|
4135
|
+
if (explicitDefault)
|
|
4136
|
+
return explicitDefault;
|
|
4137
|
+
const owner = await db
|
|
4138
|
+
.select({ userId: companyMemberships.principalId })
|
|
4139
|
+
.from(companyMemberships)
|
|
4140
|
+
.where(and(eq(companyMemberships.companyId, companyId), eq(companyMemberships.principalType, "user"), eq(companyMemberships.status, "active"), eq(companyMemberships.membershipRole, "owner")))
|
|
4141
|
+
.orderBy(asc(companyMemberships.createdAt), asc(companyMemberships.id))
|
|
4142
|
+
.limit(1)
|
|
4143
|
+
.then((rows) => rows[0] ?? null);
|
|
4144
|
+
if (owner?.userId)
|
|
4145
|
+
return owner.userId;
|
|
4146
|
+
const firstUser = await db
|
|
4147
|
+
.select({ userId: companyMemberships.principalId })
|
|
4148
|
+
.from(companyMemberships)
|
|
4149
|
+
.where(and(eq(companyMemberships.companyId, companyId), eq(companyMemberships.principalType, "user"), eq(companyMemberships.status, "active")))
|
|
4150
|
+
.orderBy(asc(companyMemberships.createdAt), asc(companyMemberships.id))
|
|
4151
|
+
.limit(1)
|
|
4152
|
+
.then((rows) => rows[0] ?? null);
|
|
4153
|
+
return firstUser?.userId ?? null;
|
|
4154
|
+
}
|
|
4155
|
+
async function resolveParentIssueResponsibleUserId(companyId, parentId) {
|
|
4156
|
+
if (!parentId)
|
|
4157
|
+
return null;
|
|
4158
|
+
const parent = await db
|
|
4159
|
+
.select({
|
|
4160
|
+
responsibleUserId: issues.responsibleUserId,
|
|
4161
|
+
createdByUserId: issues.createdByUserId,
|
|
4162
|
+
})
|
|
4163
|
+
.from(issues)
|
|
4164
|
+
.where(and(eq(issues.companyId, companyId), eq(issues.id, parentId)))
|
|
4165
|
+
.then((rows) => rows[0] ?? null);
|
|
4166
|
+
return parent?.responsibleUserId ?? null;
|
|
4167
|
+
}
|
|
4168
|
+
function isManualUserRun(input) {
|
|
4169
|
+
if (input.requestedByActorType !== "user")
|
|
4170
|
+
return false;
|
|
4171
|
+
const wakeReason = readNonEmptyString(input.contextSnapshot.wakeReason);
|
|
4172
|
+
if (wakeReason && ISSUE_RESPONSIBLE_USER_WAKE_REASONS.has(wakeReason))
|
|
4173
|
+
return false;
|
|
4174
|
+
return input.source === "on_demand" || input.triggerDetail === "manual";
|
|
4175
|
+
}
|
|
4176
|
+
async function resolveResponsibleUserIdForRunSeed(input) {
|
|
4177
|
+
const contextResponsibleUserId = readNonEmptyString(input.contextSnapshot.responsibleUserId);
|
|
4178
|
+
const requestedUserId = input.requestedByActorType === "user"
|
|
4179
|
+
? readNonEmptyString(input.requestedByActorId)
|
|
4180
|
+
: null;
|
|
4181
|
+
if (contextResponsibleUserId)
|
|
4182
|
+
return contextResponsibleUserId;
|
|
4183
|
+
if (input.existingRunResponsibleUserId)
|
|
4184
|
+
return input.existingRunResponsibleUserId;
|
|
4185
|
+
if (input.routineEnvContext.responsibleUserId)
|
|
4186
|
+
return input.routineEnvContext.responsibleUserId;
|
|
4187
|
+
if (isManualUserRun(input) && requestedUserId)
|
|
4188
|
+
return requestedUserId;
|
|
4189
|
+
if (input.issueContext?.responsibleUserId)
|
|
4190
|
+
return input.issueContext.responsibleUserId;
|
|
4191
|
+
const parentResponsibleUserId = await resolveParentIssueResponsibleUserId(input.companyId, input.issueContext?.parentId);
|
|
4192
|
+
if (parentResponsibleUserId)
|
|
4193
|
+
return parentResponsibleUserId;
|
|
4194
|
+
if (input.issueContext)
|
|
4195
|
+
return resolveCompanyDefaultResponsibleUserId(input.companyId);
|
|
4196
|
+
if (requestedUserId)
|
|
4197
|
+
return requestedUserId;
|
|
4198
|
+
return resolveCompanyDefaultResponsibleUserId(input.companyId);
|
|
4199
|
+
}
|
|
4200
|
+
async function resolveResponsibleUserIdForRun(input) {
|
|
4201
|
+
const responsibleUserId = await resolveResponsibleUserIdForRunSeed({
|
|
4202
|
+
companyId: input.run.companyId,
|
|
4203
|
+
contextSnapshot: input.contextSnapshot,
|
|
4204
|
+
issueContext: input.issueContext,
|
|
4205
|
+
routineEnvContext: input.routineEnvContext,
|
|
4206
|
+
existingRunResponsibleUserId: input.run.responsibleUserId,
|
|
4207
|
+
source: input.run.invocationSource,
|
|
4208
|
+
triggerDetail: input.run.triggerDetail,
|
|
4209
|
+
});
|
|
4210
|
+
if (!responsibleUserId) {
|
|
4211
|
+
throw new HttpError(422, "Unable to resolve responsible user for heartbeat run dispatch", {
|
|
4212
|
+
code: "responsible_user_unresolved",
|
|
4213
|
+
runId: input.run.id,
|
|
4214
|
+
agentId: input.run.agentId,
|
|
4215
|
+
companyId: input.run.companyId,
|
|
4216
|
+
issueId: input.issueContext?.id ?? null,
|
|
4217
|
+
invocationSource: input.run.invocationSource,
|
|
4218
|
+
triggerDetail: input.run.triggerDetail,
|
|
4219
|
+
wakeReason: readNonEmptyString(input.contextSnapshot.wakeReason),
|
|
4220
|
+
});
|
|
4221
|
+
}
|
|
4222
|
+
return responsibleUserId;
|
|
4223
|
+
}
|
|
4224
|
+
async function resolveResponsibleUserIdForRunContext(run, contextSnapshot) {
|
|
4225
|
+
const issueId = readNonEmptyString(contextSnapshot.issueId) ?? readNonEmptyString(contextSnapshot.taskId);
|
|
4226
|
+
const issueContext = issueId ? await getIssueExecutionContext(run.companyId, issueId) : null;
|
|
4227
|
+
return resolveResponsibleUserIdForRun({
|
|
4228
|
+
run,
|
|
4229
|
+
contextSnapshot,
|
|
4230
|
+
issueContext,
|
|
4231
|
+
routineEnvContext: await getRoutineEnvForExecutionIssue(run.companyId, issueContext),
|
|
4232
|
+
});
|
|
3492
4233
|
}
|
|
3493
4234
|
async function getRuntimeState(agentId) {
|
|
3494
4235
|
return db
|
|
@@ -3617,7 +4358,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
3617
4358
|
return db
|
|
3618
4359
|
.select()
|
|
3619
4360
|
.from(issues)
|
|
3620
|
-
.where(and(eq(issues.companyId, claimed.companyId), eq(issues.originKind, RECOVERY_ORIGIN_KINDS.strandedIssueRecovery), eq(issues.originId, claimed.id),
|
|
4361
|
+
.where(and(eq(issues.companyId, claimed.companyId), eq(issues.originKind, RECOVERY_ORIGIN_KINDS.strandedIssueRecovery), eq(issues.originId, claimed.id), visibleIssueCondition(), notInArray(issues.status, ["done", "cancelled"])))
|
|
3621
4362
|
.orderBy(desc(issues.createdAt))
|
|
3622
4363
|
.limit(1)
|
|
3623
4364
|
.then((rows) => rows[0] ?? null);
|
|
@@ -4067,8 +4808,11 @@ export function heartbeatService(db, options = {}) {
|
|
|
4067
4808
|
.then((rows) => rows[0] ?? null);
|
|
4068
4809
|
}
|
|
4069
4810
|
async function resolveNormalizedUsageForSession(input) {
|
|
4070
|
-
const { agentId, runId, sessionId, rawUsage } = input;
|
|
4071
|
-
|
|
4811
|
+
const { agentId, runId, sessionId, rawUsage, usageBasis } = input;
|
|
4812
|
+
// Adapters that declare per-run usage (e.g. the ACPX lane reports each
|
|
4813
|
+
// turn's tokens, not session totals) must not be session-delta'd, or
|
|
4814
|
+
// consecutive runs would be undercounted.
|
|
4815
|
+
if (!sessionId || !rawUsage || usageBasis === "per_run") {
|
|
4072
4816
|
return {
|
|
4073
4817
|
normalizedUsage: rawUsage,
|
|
4074
4818
|
previousRawUsage: null,
|
|
@@ -4198,6 +4942,27 @@ export function heartbeatService(db, options = {}) {
|
|
|
4198
4942
|
const runtimeForRun = await getRuntimeState(agent.id);
|
|
4199
4943
|
return runtimeForRun?.sessionId ?? null;
|
|
4200
4944
|
}
|
|
4945
|
+
async function hasResolvableSessionWorkspaceCwd(sessionParams) {
|
|
4946
|
+
const cwd = readNonEmptyString(sessionParams?.cwd);
|
|
4947
|
+
if (!cwd || isUnsafeSessionWorkspaceCwd(cwd))
|
|
4948
|
+
return false;
|
|
4949
|
+
return fs
|
|
4950
|
+
.stat(cwd)
|
|
4951
|
+
.then((stats) => stats.isDirectory())
|
|
4952
|
+
.catch(() => false);
|
|
4953
|
+
}
|
|
4954
|
+
async function hasResolvablePriorSessionWorkspaceForWake(input) {
|
|
4955
|
+
if (await hasResolvableSessionWorkspaceCwd(input.explicitResumeSession?.sessionParams))
|
|
4956
|
+
return true;
|
|
4957
|
+
if (shouldResetTaskSessionForWake(input.contextSnapshot))
|
|
4958
|
+
return false;
|
|
4959
|
+
if (!input.taskKey)
|
|
4960
|
+
return false;
|
|
4961
|
+
const codec = getAdapterSessionCodec(input.agent.adapterType);
|
|
4962
|
+
const taskSession = await getTaskSession(input.agent.companyId, input.agent.id, input.agent.adapterType, input.taskKey);
|
|
4963
|
+
const taskSessionParams = normalizeResumeParamsForAdapter(input.agent.adapterType, codec.deserialize(taskSession?.sessionParamsJson ?? null));
|
|
4964
|
+
return hasResolvableSessionWorkspaceCwd(taskSessionParams);
|
|
4965
|
+
}
|
|
4201
4966
|
async function resolveExplicitResumeSessionOverride(agent, payload, taskKey) {
|
|
4202
4967
|
const resumeFromRunId = readNonEmptyString(payload?.resumeFromRunId);
|
|
4203
4968
|
if (!resumeFromRunId)
|
|
@@ -4744,9 +5509,22 @@ export function heartbeatService(db, options = {}) {
|
|
|
4744
5509
|
const prefix = issue.identifier?.split("-")[0] || "PAP";
|
|
4745
5510
|
return `[${label}](/${prefix}/issues/${label})`;
|
|
4746
5511
|
}
|
|
5512
|
+
function hasUnmanagedBackgroundTaskEvidence(resultJson) {
|
|
5513
|
+
const evidence = parseObject(resultJson?.unmanagedBackgroundTask);
|
|
5514
|
+
return evidence.stopped === true &&
|
|
5515
|
+
(evidence.stopReason === UNMANAGED_BACKGROUND_TASK_STOP_REASON ||
|
|
5516
|
+
evidence.reason === UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON);
|
|
5517
|
+
}
|
|
5518
|
+
function withUnmanagedBackgroundTaskStopReason(resultJson) {
|
|
5519
|
+
return {
|
|
5520
|
+
...(resultJson ?? {}),
|
|
5521
|
+
stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON,
|
|
5522
|
+
};
|
|
5523
|
+
}
|
|
4747
5524
|
async function buildDetectedSuccessfulRunProgressSummary(run) {
|
|
4748
5525
|
const resultJson = parseObject(run.resultJson);
|
|
4749
5526
|
const candidates = [
|
|
5527
|
+
hasUnmanagedBackgroundTaskEvidence(resultJson) ? UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON : null,
|
|
4750
5528
|
readNonEmptyString(run.nextAction) ? `Next action noted: ${readNonEmptyString(run.nextAction)}` : null,
|
|
4751
5529
|
readNonEmptyString(run.livenessReason),
|
|
4752
5530
|
readNonEmptyString(resultJson.summary),
|
|
@@ -4791,6 +5569,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
4791
5569
|
assigneeAgentId: issues.assigneeAgentId,
|
|
4792
5570
|
assigneeUserId: issues.assigneeUserId,
|
|
4793
5571
|
executionState: issues.executionState,
|
|
5572
|
+
monitorNextCheckAt: issues.monitorNextCheckAt,
|
|
4794
5573
|
projectId: issues.projectId,
|
|
4795
5574
|
})
|
|
4796
5575
|
.from(issues)
|
|
@@ -4868,7 +5647,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
4868
5647
|
.where(and(eq(issues.companyId, issue.companyId), inArray(issues.originKind, [
|
|
4869
5648
|
RECOVERY_ORIGIN_KINDS.strandedIssueRecovery,
|
|
4870
5649
|
RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation,
|
|
4871
|
-
]), eq(issues.originId, issue.id),
|
|
5650
|
+
]), eq(issues.originId, issue.id), visibleIssueCondition(), notInArray(issues.status, ["done", "cancelled"])))
|
|
4872
5651
|
.limit(1)
|
|
4873
5652
|
.then((rows) => rows[0] ?? null)
|
|
4874
5653
|
: Promise.resolve(null),
|
|
@@ -4906,6 +5685,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
4906
5685
|
hasActiveExecutionPath: Boolean(activeExecutionPath),
|
|
4907
5686
|
hasQueuedWake: Boolean(queuedWake),
|
|
4908
5687
|
hasPendingInteractionOrApproval: Boolean(pendingInteraction || pendingApproval),
|
|
5688
|
+
hasPersistedMonitor: Boolean(issue?.monitorNextCheckAt),
|
|
4909
5689
|
hasExplicitBlockerPath: Boolean(explicitBlocker),
|
|
4910
5690
|
hasOpenRecoveryIssue: Boolean(openRecoveryIssue),
|
|
4911
5691
|
hasPauseHold: Boolean(pauseHold),
|
|
@@ -4915,6 +5695,16 @@ export function heartbeatService(db, options = {}) {
|
|
|
4915
5695
|
});
|
|
4916
5696
|
if (decision.kind !== "enqueue" || !issue)
|
|
4917
5697
|
return;
|
|
5698
|
+
if (hasUnmanagedBackgroundTaskEvidence(parseObject(run.resultJson))) {
|
|
5699
|
+
await db
|
|
5700
|
+
.update(heartbeatRuns)
|
|
5701
|
+
.set({
|
|
5702
|
+
livenessReason: UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON,
|
|
5703
|
+
resultJson: withUnmanagedBackgroundTaskStopReason(parseObject(run.resultJson)),
|
|
5704
|
+
updatedAt: new Date(),
|
|
5705
|
+
})
|
|
5706
|
+
.where(eq(heartbeatRuns.id, run.id));
|
|
5707
|
+
}
|
|
4918
5708
|
const handoffRun = await enqueueWakeup(run.agentId, {
|
|
4919
5709
|
source: "automation",
|
|
4920
5710
|
triggerDetail: "system",
|
|
@@ -5144,6 +5934,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
5144
5934
|
retryReason: "missing_issue_comment",
|
|
5145
5935
|
missingIssueCommentForRunId: run.id,
|
|
5146
5936
|
}, "status_only");
|
|
5937
|
+
const responsibleUserId = await resolveResponsibleUserIdForRunContext(run, retryContextSnapshot);
|
|
5147
5938
|
const now = new Date();
|
|
5148
5939
|
const retryRun = await db.transaction(async (tx) => {
|
|
5149
5940
|
await tx.execute(sql `select id from issues where company_id = ${run.companyId} and execution_run_id = ${run.id} for update`);
|
|
@@ -5184,6 +5975,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
5184
5975
|
status: "queued",
|
|
5185
5976
|
wakeupRequestId: wakeupRequest.id,
|
|
5186
5977
|
contextSnapshot: retryContextSnapshot,
|
|
5978
|
+
responsibleUserId,
|
|
5187
5979
|
sessionIdBefore: sessionBefore,
|
|
5188
5980
|
retryOfRunId: run.id,
|
|
5189
5981
|
issueCommentStatus: "not_applicable",
|
|
@@ -5319,17 +6111,37 @@ export function heartbeatService(db, options = {}) {
|
|
|
5319
6111
|
return { outcome: "retry_exhausted", queuedRun: null };
|
|
5320
6112
|
}
|
|
5321
6113
|
async function enqueueProcessLossRetry(run, agent, now) {
|
|
5322
|
-
const
|
|
5323
|
-
|
|
6114
|
+
const existingRetry = await db
|
|
6115
|
+
.select()
|
|
6116
|
+
.from(heartbeatRuns)
|
|
6117
|
+
.where(and(eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.retryOfRunId, run.id)))
|
|
6118
|
+
.orderBy(asc(heartbeatRuns.createdAt))
|
|
6119
|
+
.limit(1)
|
|
6120
|
+
.then((rows) => rows[0] ?? null);
|
|
6121
|
+
if (existingRetry) {
|
|
5324
6122
|
await appendRunEvent(run, await nextRunEventSeq(run.id), {
|
|
5325
6123
|
eventType: "lifecycle",
|
|
5326
6124
|
stream: "system",
|
|
5327
6125
|
level: "warn",
|
|
5328
|
-
message: "Process-loss retry
|
|
6126
|
+
message: "Process-loss retry already exists; skipping duplicate retry enqueue",
|
|
5329
6127
|
payload: {
|
|
5330
|
-
|
|
5331
|
-
|
|
5332
|
-
|
|
6128
|
+
retryRunId: existingRetry.id,
|
|
6129
|
+
retryRunStatus: existingRetry.status,
|
|
6130
|
+
},
|
|
6131
|
+
});
|
|
6132
|
+
return existingRetry;
|
|
6133
|
+
}
|
|
6134
|
+
const invokability = await getAgentInvokability(agent);
|
|
6135
|
+
if (!invokability.invokable) {
|
|
6136
|
+
await appendRunEvent(run, await nextRunEventSeq(run.id), {
|
|
6137
|
+
eventType: "lifecycle",
|
|
6138
|
+
stream: "system",
|
|
6139
|
+
level: "warn",
|
|
6140
|
+
message: "Process-loss retry suppressed because the agent is not invokable",
|
|
6141
|
+
payload: {
|
|
6142
|
+
reason: invokability.reason,
|
|
6143
|
+
invalidOrgChain: invokability.invalidOrgChain,
|
|
6144
|
+
...invokability.details,
|
|
5333
6145
|
},
|
|
5334
6146
|
});
|
|
5335
6147
|
await releaseIssueExecutionAndPromote(run);
|
|
@@ -5345,6 +6157,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
5345
6157
|
wakeReason: "process_lost_retry",
|
|
5346
6158
|
retryReason: "process_lost",
|
|
5347
6159
|
}, "normal_model");
|
|
6160
|
+
const responsibleUserId = await resolveResponsibleUserIdForRunContext(run, retryContextSnapshot);
|
|
5348
6161
|
const queued = await db.transaction(async (tx) => {
|
|
5349
6162
|
const wakeupRequest = await tx
|
|
5350
6163
|
.insert(agentWakeupRequests)
|
|
@@ -5375,6 +6188,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
5375
6188
|
status: "queued",
|
|
5376
6189
|
wakeupRequestId: wakeupRequest.id,
|
|
5377
6190
|
contextSnapshot: retryContextSnapshot,
|
|
6191
|
+
responsibleUserId,
|
|
5378
6192
|
sessionIdBefore: sessionBefore,
|
|
5379
6193
|
retryOfRunId: run.id,
|
|
5380
6194
|
processLossRetryCount: (run.processLossRetryCount ?? 0) + 1,
|
|
@@ -5425,6 +6239,95 @@ export function heartbeatService(db, options = {}) {
|
|
|
5425
6239
|
});
|
|
5426
6240
|
return queued;
|
|
5427
6241
|
}
|
|
6242
|
+
async function drainRunningRunsForShutdown(signal, now = new Date()) {
|
|
6243
|
+
const activeRuns = await db
|
|
6244
|
+
.select({
|
|
6245
|
+
run: heartbeatRuns,
|
|
6246
|
+
agent: agents,
|
|
6247
|
+
})
|
|
6248
|
+
.from(heartbeatRuns)
|
|
6249
|
+
.innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
|
|
6250
|
+
.where(eq(heartbeatRuns.status, "running"));
|
|
6251
|
+
const interruptedRunIds = [];
|
|
6252
|
+
const retryRunIds = [];
|
|
6253
|
+
for (const { run, agent } of activeRuns) {
|
|
6254
|
+
const running = runningProcesses.get(run.id);
|
|
6255
|
+
try {
|
|
6256
|
+
if (running) {
|
|
6257
|
+
await terminateHeartbeatRunProcess({
|
|
6258
|
+
pid: running.child.pid ?? run.processPid,
|
|
6259
|
+
processGroupId: running.processGroupId ?? run.processGroupId,
|
|
6260
|
+
graceMs: Math.max(1, running.graceSec) * 1000,
|
|
6261
|
+
});
|
|
6262
|
+
}
|
|
6263
|
+
else if (run.processPid || run.processGroupId) {
|
|
6264
|
+
await terminateHeartbeatRunProcess({
|
|
6265
|
+
pid: run.processPid,
|
|
6266
|
+
processGroupId: run.processGroupId,
|
|
6267
|
+
});
|
|
6268
|
+
}
|
|
6269
|
+
}
|
|
6270
|
+
finally {
|
|
6271
|
+
runningProcesses.delete(run.id);
|
|
6272
|
+
}
|
|
6273
|
+
const message = `Interrupted by graceful server shutdown (${signal}); retry queued for restart recovery`;
|
|
6274
|
+
const interruptedStatus = await setRunStatusIfRunning(run.id, "interrupted", {
|
|
6275
|
+
finishedAt: now,
|
|
6276
|
+
error: message,
|
|
6277
|
+
errorCode: "server_shutdown_interrupted",
|
|
6278
|
+
signal,
|
|
6279
|
+
resultJson: mergeRunStopMetadataForAgent(agent, "interrupted", {
|
|
6280
|
+
resultJson: parseObject(run.resultJson),
|
|
6281
|
+
errorCode: "server_shutdown_interrupted",
|
|
6282
|
+
errorMessage: message,
|
|
6283
|
+
}),
|
|
6284
|
+
});
|
|
6285
|
+
if (!interruptedStatus.updated || !interruptedStatus.run)
|
|
6286
|
+
continue;
|
|
6287
|
+
let interrupted = interruptedStatus.run;
|
|
6288
|
+
await setWakeupStatus(run.wakeupRequestId, "cancelled", {
|
|
6289
|
+
finishedAt: now,
|
|
6290
|
+
error: null,
|
|
6291
|
+
});
|
|
6292
|
+
interrupted = await classifyAndPersistRunLiveness(interrupted, parseObject(interrupted.resultJson)) ?? interrupted;
|
|
6293
|
+
await releaseEnvironmentLeasesForRun({
|
|
6294
|
+
runId: interrupted.id,
|
|
6295
|
+
companyId: interrupted.companyId,
|
|
6296
|
+
agentId: interrupted.agentId,
|
|
6297
|
+
status: interrupted.status,
|
|
6298
|
+
failureReason: interrupted.error ?? undefined,
|
|
6299
|
+
});
|
|
6300
|
+
const retry = await enqueueProcessLossRetry(interrupted, agent, now);
|
|
6301
|
+
if (!retry) {
|
|
6302
|
+
await releaseIssueExecutionAndPromote(interrupted);
|
|
6303
|
+
}
|
|
6304
|
+
else {
|
|
6305
|
+
retryRunIds.push(retry.id);
|
|
6306
|
+
}
|
|
6307
|
+
await appendRunEvent(interrupted, await nextRunEventSeq(interrupted.id), {
|
|
6308
|
+
eventType: "lifecycle",
|
|
6309
|
+
stream: "system",
|
|
6310
|
+
level: "warn",
|
|
6311
|
+
message,
|
|
6312
|
+
payload: {
|
|
6313
|
+
signal,
|
|
6314
|
+
...(run.processPid ? { processPid: run.processPid } : {}),
|
|
6315
|
+
...(run.processGroupId ? { processGroupId: run.processGroupId } : {}),
|
|
6316
|
+
...(retry ? { retryRunId: retry.id } : {}),
|
|
6317
|
+
},
|
|
6318
|
+
});
|
|
6319
|
+
await finalizeAgentStatus(run.agentId, "interrupted", message);
|
|
6320
|
+
interruptedRunIds.push(interrupted.id);
|
|
6321
|
+
}
|
|
6322
|
+
if (interruptedRunIds.length > 0) {
|
|
6323
|
+
logger.warn({ signal, interrupted: interruptedRunIds.length, interruptedRunIds, retryRunIds }, "interrupted running heartbeat runs for graceful shutdown");
|
|
6324
|
+
}
|
|
6325
|
+
return {
|
|
6326
|
+
interrupted: interruptedRunIds.length,
|
|
6327
|
+
interruptedRunIds,
|
|
6328
|
+
retryRunIds,
|
|
6329
|
+
};
|
|
6330
|
+
}
|
|
5428
6331
|
async function evaluateScheduledRetryGate(input) {
|
|
5429
6332
|
const { run, agent, contextSnapshot } = input;
|
|
5430
6333
|
const retryReason = input.retryReason ?? readNonEmptyString(contextSnapshot.retryReason) ?? run.scheduledRetryReason ?? null;
|
|
@@ -5716,7 +6619,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
5716
6619
|
const wakeReason = opts?.wakeReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON;
|
|
5717
6620
|
const maxAttempts = Math.max(0, Math.floor(opts?.maxAttempts ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS));
|
|
5718
6621
|
const nextAttempt = (run.scheduledRetryAttempt ?? 0) + 1;
|
|
5719
|
-
const
|
|
6622
|
+
const computedBaseSchedule = opts?.delayMs != null
|
|
5720
6623
|
? nextAttempt <= maxAttempts
|
|
5721
6624
|
? {
|
|
5722
6625
|
attempt: nextAttempt,
|
|
@@ -5729,13 +6632,16 @@ export function heartbeatService(db, options = {}) {
|
|
|
5729
6632
|
: nextAttempt <= maxAttempts
|
|
5730
6633
|
? computeBoundedTransientHeartbeatRetrySchedule(nextAttempt, now, opts?.random)
|
|
5731
6634
|
: null;
|
|
6635
|
+
const baseSchedule = computedBaseSchedule ? { ...computedBaseSchedule, maxAttempts } : null;
|
|
5732
6636
|
const transientRecovery = retryReason === BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON
|
|
5733
6637
|
? readTransientRecoveryContractFromRun(run)
|
|
5734
6638
|
: null;
|
|
5735
|
-
const codexTransientFallbackMode = agent.adapterType === "codex_local" && transientRecovery
|
|
6639
|
+
const codexTransientFallbackMode = agent.adapterType === "codex_local" && transientRecovery?.errorFamily === "transient_upstream"
|
|
5736
6640
|
? resolveCodexTransientFallbackMode(nextAttempt)
|
|
5737
6641
|
: null;
|
|
5738
6642
|
const transientRetryNotBefore = transientRecovery?.retryNotBefore ?? null;
|
|
6643
|
+
const contextSnapshot = parseObject(run.contextSnapshot);
|
|
6644
|
+
const issueId = readNonEmptyString(contextSnapshot.issueId);
|
|
5739
6645
|
if (!baseSchedule) {
|
|
5740
6646
|
await appendRunEvent(run, await nextRunEventSeq(run.id), {
|
|
5741
6647
|
eventType: "lifecycle",
|
|
@@ -5748,6 +6654,16 @@ export function heartbeatService(db, options = {}) {
|
|
|
5748
6654
|
maxAttempts,
|
|
5749
6655
|
},
|
|
5750
6656
|
});
|
|
6657
|
+
if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) {
|
|
6658
|
+
await escalatePlanApprovalResumeFailureNeedsAttention({
|
|
6659
|
+
run,
|
|
6660
|
+
issueId,
|
|
6661
|
+
attempt: Math.min(run.scheduledRetryAttempt ?? maxAttempts, maxAttempts),
|
|
6662
|
+
maxAttempts,
|
|
6663
|
+
}).catch((error) => {
|
|
6664
|
+
logger.warn({ err: error, runId: run.id, issueId }, "failed to escalate exhausted plan-approval resume failure");
|
|
6665
|
+
});
|
|
6666
|
+
}
|
|
5751
6667
|
return {
|
|
5752
6668
|
outcome: "retry_exhausted",
|
|
5753
6669
|
attempt: nextAttempt,
|
|
@@ -5757,8 +6673,6 @@ export function heartbeatService(db, options = {}) {
|
|
|
5757
6673
|
if (retryReason !== MAX_TURN_CONTINUATION_RETRY_REASON) {
|
|
5758
6674
|
const invokability = await getAgentInvokability(agent);
|
|
5759
6675
|
if (!invokability.invokable) {
|
|
5760
|
-
const contextSnapshot = parseObject(run.contextSnapshot);
|
|
5761
|
-
const issueId = readNonEmptyString(contextSnapshot.issueId);
|
|
5762
6676
|
await appendRunEvent(run, await nextRunEventSeq(run.id), {
|
|
5763
6677
|
eventType: "lifecycle",
|
|
5764
6678
|
stream: "system",
|
|
@@ -5788,10 +6702,16 @@ export function heartbeatService(db, options = {}) {
|
|
|
5788
6702
|
delayMs: Math.max(0, transientRetryNotBefore.getTime() - now.getTime()),
|
|
5789
6703
|
}
|
|
5790
6704
|
: baseSchedule;
|
|
5791
|
-
const
|
|
5792
|
-
|
|
5793
|
-
if (
|
|
5794
|
-
const gate = await evaluateScheduledRetryGate({
|
|
6705
|
+
const requiresIssueGate = retryReason === MAX_TURN_CONTINUATION_RETRY_REASON ||
|
|
6706
|
+
retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON;
|
|
6707
|
+
if (requiresIssueGate) {
|
|
6708
|
+
const gate = await evaluateScheduledRetryGate({
|
|
6709
|
+
run,
|
|
6710
|
+
agent,
|
|
6711
|
+
contextSnapshot,
|
|
6712
|
+
retryReason,
|
|
6713
|
+
enforceIssueExecutionLock: retryReason === MAX_TURN_CONTINUATION_RETRY_REASON,
|
|
6714
|
+
});
|
|
5795
6715
|
if (!gate.allowed) {
|
|
5796
6716
|
await appendRunEvent(run, await nextRunEventSeq(run.id), {
|
|
5797
6717
|
eventType: "lifecycle",
|
|
@@ -5815,21 +6735,90 @@ export function heartbeatService(db, options = {}) {
|
|
|
5815
6735
|
}
|
|
5816
6736
|
const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null);
|
|
5817
6737
|
const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey);
|
|
6738
|
+
const interactionContinuationPayload = retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON
|
|
6739
|
+
? {
|
|
6740
|
+
mutation: "interaction",
|
|
6741
|
+
interactionId: readNonEmptyString(contextSnapshot.interactionId),
|
|
6742
|
+
interactionKind: readNonEmptyString(contextSnapshot.interactionKind),
|
|
6743
|
+
interactionStatus: readNonEmptyString(contextSnapshot.interactionStatus),
|
|
6744
|
+
continuationPolicy: readNonEmptyString(contextSnapshot.continuationPolicy),
|
|
6745
|
+
}
|
|
6746
|
+
: {};
|
|
6747
|
+
const workspaceValidationRetryPayload = retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON && isWorkspaceValidationFailedRun(run)
|
|
6748
|
+
? readWorkspaceValidationPayloadFromRun(run)
|
|
6749
|
+
: null;
|
|
6750
|
+
const shouldQuarantineWorkspaceForRetry = workspaceValidationRetryPayload !== null &&
|
|
6751
|
+
Object.keys(workspaceValidationRetryPayload).length > 0;
|
|
5818
6752
|
const retryContextSnapshot = withRecoveryModelProfileHint({
|
|
5819
6753
|
...contextSnapshot,
|
|
5820
6754
|
retryOfRunId: run.id,
|
|
5821
6755
|
wakeReason,
|
|
5822
6756
|
retryReason,
|
|
6757
|
+
...(shouldQuarantineWorkspaceForRetry
|
|
6758
|
+
? {
|
|
6759
|
+
workspaceValidationRecovery: {
|
|
6760
|
+
strategy: "quarantine_failed_workspace_and_retry_clean",
|
|
6761
|
+
sourceRunId: run.id,
|
|
6762
|
+
reason: readNonEmptyString(workspaceValidationRetryPayload?.reason) ?? WORKSPACE_VALIDATION_FAILURE_CODE,
|
|
6763
|
+
fingerprint: readNonEmptyString(workspaceValidationRetryPayload?.fingerprint),
|
|
6764
|
+
failedExecutionWorkspaceId: readNonEmptyString(workspaceValidationRetryPayload?.executionWorkspaceId),
|
|
6765
|
+
},
|
|
6766
|
+
}
|
|
6767
|
+
: {}),
|
|
5823
6768
|
...(transientRecovery ? { errorFamily: transientRecovery.errorFamily } : {}),
|
|
5824
6769
|
scheduledRetryAttempt: schedule.attempt,
|
|
5825
6770
|
scheduledRetryAt: schedule.dueAt.toISOString(),
|
|
5826
6771
|
...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),
|
|
6772
|
+
...(transientRecovery?.errorFamily === "provider_quota" && transientRetryNotBefore
|
|
6773
|
+
? { providerQuotaRetryNotBefore: transientRetryNotBefore.toISOString() }
|
|
6774
|
+
: {}),
|
|
5827
6775
|
...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}),
|
|
5828
6776
|
}, "normal_model");
|
|
5829
|
-
const
|
|
6777
|
+
const responsibleUserId = await resolveResponsibleUserIdForRunContext(run, retryContextSnapshot);
|
|
6778
|
+
const continuationRetryIdempotencyKey = retryReason === MAX_TURN_CONTINUATION_RETRY_REASON
|
|
5830
6779
|
? `max-turn-continuation:${run.companyId}:${issueId ?? "no-issue"}:${run.id}:${schedule.attempt}`
|
|
5831
|
-
:
|
|
6780
|
+
: retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON
|
|
6781
|
+
? `interaction-continuation:${run.companyId}:${issueId ?? "no-issue"}:${run.id}:${schedule.attempt}`
|
|
6782
|
+
: null;
|
|
5832
6783
|
const scheduleResult = await db.transaction(async (tx) => {
|
|
6784
|
+
if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) {
|
|
6785
|
+
if (issueId) {
|
|
6786
|
+
await tx.execute(sql `select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`);
|
|
6787
|
+
}
|
|
6788
|
+
else {
|
|
6789
|
+
await tx.execute(sql `select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`);
|
|
6790
|
+
}
|
|
6791
|
+
const existingContinuation = await tx
|
|
6792
|
+
.select()
|
|
6793
|
+
.from(heartbeatRuns)
|
|
6794
|
+
.where(and(eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.retryOfRunId, run.id), eq(heartbeatRuns.scheduledRetryReason, retryReason), eq(heartbeatRuns.scheduledRetryAttempt, schedule.attempt), inArray(heartbeatRuns.status, [...MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES]), issueId
|
|
6795
|
+
? sql `${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`
|
|
6796
|
+
: sql `${heartbeatRuns.contextSnapshot} ->> 'issueId' is null`))
|
|
6797
|
+
.orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id))
|
|
6798
|
+
.limit(1)
|
|
6799
|
+
.then((rows) => rows[0] ?? null);
|
|
6800
|
+
if (existingContinuation) {
|
|
6801
|
+
if (existingContinuation.wakeupRequestId) {
|
|
6802
|
+
const existingWakeup = await tx
|
|
6803
|
+
.select({ coalescedCount: agentWakeupRequests.coalescedCount })
|
|
6804
|
+
.from(agentWakeupRequests)
|
|
6805
|
+
.where(eq(agentWakeupRequests.id, existingContinuation.wakeupRequestId))
|
|
6806
|
+
.then((rows) => rows[0] ?? null);
|
|
6807
|
+
await tx
|
|
6808
|
+
.update(agentWakeupRequests)
|
|
6809
|
+
.set({
|
|
6810
|
+
coalescedCount: (existingWakeup?.coalescedCount ?? 0) + 1,
|
|
6811
|
+
updatedAt: now,
|
|
6812
|
+
})
|
|
6813
|
+
.where(eq(agentWakeupRequests.id, existingContinuation.wakeupRequestId));
|
|
6814
|
+
}
|
|
6815
|
+
return {
|
|
6816
|
+
outcome: "scheduled",
|
|
6817
|
+
run: existingContinuation,
|
|
6818
|
+
reusedExisting: true,
|
|
6819
|
+
};
|
|
6820
|
+
}
|
|
6821
|
+
}
|
|
5833
6822
|
if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON) {
|
|
5834
6823
|
if (issueId) {
|
|
5835
6824
|
await tx.execute(sql `select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`);
|
|
@@ -5944,17 +6933,21 @@ export function heartbeatService(db, options = {}) {
|
|
|
5944
6933
|
payload: withRecoveryModelProfileHint({
|
|
5945
6934
|
...(issueId ? { issueId } : {}),
|
|
5946
6935
|
retryOfRunId: run.id,
|
|
6936
|
+
...interactionContinuationPayload,
|
|
5947
6937
|
retryReason,
|
|
5948
6938
|
...(transientRecovery ? { errorFamily: transientRecovery.errorFamily } : {}),
|
|
5949
6939
|
scheduledRetryAttempt: schedule.attempt,
|
|
5950
6940
|
scheduledRetryAt: schedule.dueAt.toISOString(),
|
|
5951
6941
|
...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),
|
|
6942
|
+
...(transientRecovery?.errorFamily === "provider_quota" && transientRetryNotBefore
|
|
6943
|
+
? { providerQuotaRetryNotBefore: transientRetryNotBefore.toISOString() }
|
|
6944
|
+
: {}),
|
|
5952
6945
|
...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}),
|
|
5953
6946
|
}, "normal_model"),
|
|
5954
6947
|
status: "queued",
|
|
5955
6948
|
requestedByActorType: "system",
|
|
5956
6949
|
requestedByActorId: null,
|
|
5957
|
-
idempotencyKey:
|
|
6950
|
+
idempotencyKey: continuationRetryIdempotencyKey,
|
|
5958
6951
|
updatedAt: now,
|
|
5959
6952
|
})
|
|
5960
6953
|
.returning()
|
|
@@ -5969,6 +6962,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
5969
6962
|
status: "scheduled_retry",
|
|
5970
6963
|
wakeupRequestId: wakeupRequest.id,
|
|
5971
6964
|
contextSnapshot: retryContextSnapshot,
|
|
6965
|
+
responsibleUserId,
|
|
5972
6966
|
sessionIdBefore: sessionBefore,
|
|
5973
6967
|
retryOfRunId: run.id,
|
|
5974
6968
|
scheduledRetryAt: schedule.dueAt,
|
|
@@ -5986,6 +6980,79 @@ export function heartbeatService(db, options = {}) {
|
|
|
5986
6980
|
updatedAt: now,
|
|
5987
6981
|
})
|
|
5988
6982
|
.where(eq(agentWakeupRequests.id, wakeupRequest.id));
|
|
6983
|
+
let detachWorkspaceFromIssue = false;
|
|
6984
|
+
if (issueId && shouldQuarantineWorkspaceForRetry) {
|
|
6985
|
+
const issueWorkspace = await tx
|
|
6986
|
+
.select({
|
|
6987
|
+
id: issues.id,
|
|
6988
|
+
companyId: issues.companyId,
|
|
6989
|
+
executionWorkspaceId: issues.executionWorkspaceId,
|
|
6990
|
+
})
|
|
6991
|
+
.from(issues)
|
|
6992
|
+
.where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))
|
|
6993
|
+
.for("update")
|
|
6994
|
+
.then((rows) => rows[0] ?? null);
|
|
6995
|
+
const failedExecutionWorkspaceId = readNonEmptyString(workspaceValidationRetryPayload?.executionWorkspaceId) ??
|
|
6996
|
+
readNonEmptyString(issueWorkspace?.executionWorkspaceId);
|
|
6997
|
+
if (issueWorkspace && failedExecutionWorkspaceId) {
|
|
6998
|
+
const failedWorkspace = await tx
|
|
6999
|
+
.select({
|
|
7000
|
+
id: executionWorkspaces.id,
|
|
7001
|
+
companyId: executionWorkspaces.companyId,
|
|
7002
|
+
sourceIssueId: executionWorkspaces.sourceIssueId,
|
|
7003
|
+
status: executionWorkspaces.status,
|
|
7004
|
+
metadata: executionWorkspaces.metadata,
|
|
7005
|
+
})
|
|
7006
|
+
.from(executionWorkspaces)
|
|
7007
|
+
.where(and(eq(executionWorkspaces.id, failedExecutionWorkspaceId), eq(executionWorkspaces.companyId, run.companyId)))
|
|
7008
|
+
.for("update")
|
|
7009
|
+
.then((rows) => rows[0] ?? null);
|
|
7010
|
+
const workspaceBelongsToIssue = failedWorkspace
|
|
7011
|
+
? failedWorkspace.sourceIssueId === issueId
|
|
7012
|
+
: false;
|
|
7013
|
+
if (failedWorkspace &&
|
|
7014
|
+
workspaceBelongsToIssue &&
|
|
7015
|
+
issueWorkspace.executionWorkspaceId === failedExecutionWorkspaceId) {
|
|
7016
|
+
const existingMetadata = parseObject(failedWorkspace.metadata);
|
|
7017
|
+
const quarantine = {
|
|
7018
|
+
reason: WORKSPACE_VALIDATION_FAILURE_CODE,
|
|
7019
|
+
retryReason,
|
|
7020
|
+
sourceRunId: run.id,
|
|
7021
|
+
retryRunId: scheduledRun.id,
|
|
7022
|
+
issueId,
|
|
7023
|
+
sourceIssueId: failedWorkspace.sourceIssueId ?? null,
|
|
7024
|
+
quarantinedAt: now.toISOString(),
|
|
7025
|
+
workspaceValidation: workspaceValidationRetryPayload ?? {},
|
|
7026
|
+
};
|
|
7027
|
+
await tx
|
|
7028
|
+
.update(executionWorkspaces)
|
|
7029
|
+
.set({
|
|
7030
|
+
status: "archived",
|
|
7031
|
+
closedAt: now,
|
|
7032
|
+
cleanupEligibleAt: null,
|
|
7033
|
+
cleanupReason: WORKSPACE_VALIDATION_FAILURE_CODE,
|
|
7034
|
+
metadata: {
|
|
7035
|
+
...existingMetadata,
|
|
7036
|
+
workspaceValidationQuarantine: quarantine,
|
|
7037
|
+
},
|
|
7038
|
+
updatedAt: now,
|
|
7039
|
+
})
|
|
7040
|
+
.where(and(eq(executionWorkspaces.id, failedWorkspace.id), eq(executionWorkspaces.companyId, run.companyId)));
|
|
7041
|
+
await logActivity(tx, {
|
|
7042
|
+
companyId: run.companyId,
|
|
7043
|
+
actorType: "system",
|
|
7044
|
+
actorId: "heartbeat",
|
|
7045
|
+
agentId: run.agentId,
|
|
7046
|
+
runId: run.id,
|
|
7047
|
+
action: "execution_workspace.workspace_validation_quarantined",
|
|
7048
|
+
entityType: "execution_workspace",
|
|
7049
|
+
entityId: failedWorkspace.id,
|
|
7050
|
+
details: quarantine,
|
|
7051
|
+
});
|
|
7052
|
+
detachWorkspaceFromIssue = issueWorkspace.executionWorkspaceId === failedExecutionWorkspaceId;
|
|
7053
|
+
}
|
|
7054
|
+
}
|
|
7055
|
+
}
|
|
5989
7056
|
if (issueId) {
|
|
5990
7057
|
await tx
|
|
5991
7058
|
.update(issues)
|
|
@@ -5993,6 +7060,12 @@ export function heartbeatService(db, options = {}) {
|
|
|
5993
7060
|
executionRunId: scheduledRun.id,
|
|
5994
7061
|
executionAgentNameKey: normalizeAgentNameKey(agent.name),
|
|
5995
7062
|
executionLockedAt: now,
|
|
7063
|
+
...(detachWorkspaceFromIssue
|
|
7064
|
+
? {
|
|
7065
|
+
executionWorkspaceId: null,
|
|
7066
|
+
executionWorkspacePreference: null,
|
|
7067
|
+
}
|
|
7068
|
+
: {}),
|
|
5996
7069
|
updatedAt: now,
|
|
5997
7070
|
})
|
|
5998
7071
|
.where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId), eq(issues.executionRunId, run.id)));
|
|
@@ -6030,11 +7103,11 @@ export function heartbeatService(db, options = {}) {
|
|
|
6030
7103
|
eventType: "lifecycle",
|
|
6031
7104
|
stream: "system",
|
|
6032
7105
|
level: "info",
|
|
6033
|
-
message: `Reused existing
|
|
7106
|
+
message: `Reused existing continuation retry ${retryRun.scheduledRetryAttempt}/${schedule.maxAttempts}`,
|
|
6034
7107
|
payload: {
|
|
6035
7108
|
retryRunId: retryRun.id,
|
|
6036
7109
|
retryReason,
|
|
6037
|
-
idempotencyKey:
|
|
7110
|
+
idempotencyKey: continuationRetryIdempotencyKey,
|
|
6038
7111
|
scheduledRetryAttempt: retryRun.scheduledRetryAttempt,
|
|
6039
7112
|
scheduledRetryAt: dueAt.toISOString(),
|
|
6040
7113
|
},
|
|
@@ -6062,9 +7135,23 @@ export function heartbeatService(db, options = {}) {
|
|
|
6062
7135
|
baseDelayMs: schedule.baseDelayMs,
|
|
6063
7136
|
delayMs: schedule.delayMs,
|
|
6064
7137
|
...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),
|
|
7138
|
+
...(transientRecovery?.errorFamily === "provider_quota" && transientRetryNotBefore
|
|
7139
|
+
? { providerQuotaRetryNotBefore: transientRetryNotBefore.toISOString() }
|
|
7140
|
+
: {}),
|
|
6065
7141
|
...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}),
|
|
6066
7142
|
},
|
|
6067
7143
|
});
|
|
7144
|
+
if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) {
|
|
7145
|
+
await recordPlanApprovalResumeFailureRetry({
|
|
7146
|
+
run,
|
|
7147
|
+
issueId,
|
|
7148
|
+
retryRunId: retryRun.id,
|
|
7149
|
+
attempt: schedule.attempt,
|
|
7150
|
+
maxAttempts: schedule.maxAttempts,
|
|
7151
|
+
}).catch((error) => {
|
|
7152
|
+
logger.warn({ err: error, runId: run.id, issueId, retryRunId: retryRun.id }, "failed to record plan-approval resume retry failure");
|
|
7153
|
+
});
|
|
7154
|
+
}
|
|
6068
7155
|
return {
|
|
6069
7156
|
outcome: "scheduled",
|
|
6070
7157
|
run: retryRun,
|
|
@@ -6073,11 +7160,36 @@ export function heartbeatService(db, options = {}) {
|
|
|
6073
7160
|
maxAttempts: schedule.maxAttempts,
|
|
6074
7161
|
};
|
|
6075
7162
|
}
|
|
7163
|
+
async function scheduleInteractionContinuationInfrastructureRetryIfEligible(run, agent) {
|
|
7164
|
+
if (!run.wakeupRequestId)
|
|
7165
|
+
return null;
|
|
7166
|
+
if (!isResolvedInteractionContinuationWakeContext(run.contextSnapshot))
|
|
7167
|
+
return null;
|
|
7168
|
+
if (!isRetryableInteractionContinuationInfrastructureFailure(run)) {
|
|
7169
|
+
const context = parseObject(run.contextSnapshot);
|
|
7170
|
+
const issueId = readNonEmptyString(context.issueId);
|
|
7171
|
+
await escalatePlanApprovalResumeFailureNeedsAttention({
|
|
7172
|
+
run,
|
|
7173
|
+
issueId,
|
|
7174
|
+
attempt: Math.min(run.scheduledRetryAttempt ?? INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS),
|
|
7175
|
+
maxAttempts: INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS,
|
|
7176
|
+
}).catch((error) => {
|
|
7177
|
+
logger.warn({ err: error, runId: run.id, issueId }, "failed to escalate non-retryable plan-approval resume failure");
|
|
7178
|
+
});
|
|
7179
|
+
return null;
|
|
7180
|
+
}
|
|
7181
|
+
return scheduleBoundedRetryForRun(run, agent, {
|
|
7182
|
+
retryReason: INTERACTION_CONTINUATION_INFRA_RETRY_REASON,
|
|
7183
|
+
wakeReason: INTERACTION_CONTINUATION_INFRA_WAKE_REASON,
|
|
7184
|
+
maxAttempts: INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS,
|
|
7185
|
+
});
|
|
7186
|
+
}
|
|
6076
7187
|
async function promoteDueScheduledRetries(now = new Date()) {
|
|
7188
|
+
const cutoff = await getWorktreeExecutionCutoff();
|
|
6077
7189
|
const dueRuns = await db
|
|
6078
7190
|
.select()
|
|
6079
7191
|
.from(heartbeatRuns)
|
|
6080
|
-
.where(and(eq(heartbeatRuns.status, "scheduled_retry"), lte(heartbeatRuns.scheduledRetryAt, now)))
|
|
7192
|
+
.where(and(eq(heartbeatRuns.status, "scheduled_retry"), lte(heartbeatRuns.scheduledRetryAt, now), cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined))
|
|
6081
7193
|
.orderBy(asc(heartbeatRuns.scheduledRetryAt), asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id))
|
|
6082
7194
|
.limit(50);
|
|
6083
7195
|
const promotedRunIds = [];
|
|
@@ -6494,10 +7606,17 @@ export function heartbeatService(db, options = {}) {
|
|
|
6494
7606
|
}
|
|
6495
7607
|
}
|
|
6496
7608
|
const claimedAt = new Date();
|
|
7609
|
+
const responsibleUserId = await resolveResponsibleUserIdForRun({
|
|
7610
|
+
run,
|
|
7611
|
+
contextSnapshot: context,
|
|
7612
|
+
issueContext: issueId ? await getIssueExecutionContext(run.companyId, issueId) : null,
|
|
7613
|
+
routineEnvContext: { routineId: null, env: null, responsibleUserId: null },
|
|
7614
|
+
});
|
|
6497
7615
|
const claimed = await db
|
|
6498
7616
|
.update(heartbeatRuns)
|
|
6499
7617
|
.set({
|
|
6500
7618
|
status: "running",
|
|
7619
|
+
responsibleUserId,
|
|
6501
7620
|
startedAt: run.startedAt ?? claimedAt,
|
|
6502
7621
|
updatedAt: claimedAt,
|
|
6503
7622
|
})
|
|
@@ -6747,7 +7866,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
6747
7866
|
return null;
|
|
6748
7867
|
return trimmed.length > 500 ? `${trimmed.slice(0, 499)}…` : trimmed;
|
|
6749
7868
|
}
|
|
6750
|
-
async function finalizeAgentStatus(agentId, outcome, failureReason) {
|
|
7869
|
+
async function finalizeAgentStatus(agentId, outcome, failureReason, options) {
|
|
6751
7870
|
const existing = await getAgent(agentId);
|
|
6752
7871
|
if (!existing)
|
|
6753
7872
|
return;
|
|
@@ -6758,7 +7877,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
6758
7877
|
const runningCount = await countRunningRunsForAgent(agentId);
|
|
6759
7878
|
const nextStatus = runningCount > 0
|
|
6760
7879
|
? "running"
|
|
6761
|
-
: outcome === "succeeded" || outcome === "cancelled"
|
|
7880
|
+
: outcome === "succeeded" || outcome === "interrupted" || outcome === "cancelled" || (outcome === "failed" && options?.keepIdleOnFailure)
|
|
6762
7881
|
? "idle"
|
|
6763
7882
|
: "error";
|
|
6764
7883
|
const updated = await db
|
|
@@ -7002,15 +8121,34 @@ export function heartbeatService(db, options = {}) {
|
|
|
7002
8121
|
}
|
|
7003
8122
|
const shouldRetry = tracksLocalChild && (!!run.processPid || !!run.processGroupId) && (run.processLossRetryCount ?? 0) < 1;
|
|
7004
8123
|
const baseMessage = buildProcessLossMessage(run, descendantOnlyCleanup ? { descendantOnly: true } : undefined);
|
|
8124
|
+
const unmanagedBackgroundTaskEvidence = descendantOnlyCleanup
|
|
8125
|
+
? {
|
|
8126
|
+
kind: "orphaned_process_group_cleanup",
|
|
8127
|
+
stopped: true,
|
|
8128
|
+
stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON,
|
|
8129
|
+
reason: UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON,
|
|
8130
|
+
processPid: run.processPid ?? null,
|
|
8131
|
+
processGroupId: run.processGroupId ?? null,
|
|
8132
|
+
}
|
|
8133
|
+
: null;
|
|
7005
8134
|
let finalizedRun = await setRunStatus(run.id, "failed", {
|
|
7006
8135
|
error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
|
|
7007
8136
|
errorCode: "process_lost",
|
|
7008
8137
|
finishedAt: now,
|
|
7009
|
-
resultJson:
|
|
7010
|
-
|
|
7011
|
-
|
|
7012
|
-
|
|
7013
|
-
|
|
8138
|
+
resultJson: (() => {
|
|
8139
|
+
const result = mergeRunStopMetadataForAgent({ adapterType, adapterConfig }, "failed", {
|
|
8140
|
+
resultJson: parseObject(run.resultJson),
|
|
8141
|
+
errorCode: "process_lost",
|
|
8142
|
+
errorMessage: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
|
|
8143
|
+
});
|
|
8144
|
+
return unmanagedBackgroundTaskEvidence
|
|
8145
|
+
? {
|
|
8146
|
+
...result,
|
|
8147
|
+
stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON,
|
|
8148
|
+
unmanagedBackgroundTask: unmanagedBackgroundTaskEvidence,
|
|
8149
|
+
}
|
|
8150
|
+
: result;
|
|
8151
|
+
})(),
|
|
7014
8152
|
});
|
|
7015
8153
|
await setWakeupStatus(run.wakeupRequestId, "failed", {
|
|
7016
8154
|
finishedAt: now,
|
|
@@ -7029,13 +8167,17 @@ export function heartbeatService(db, options = {}) {
|
|
|
7029
8167
|
failureReason: finalizedRun.error ?? undefined,
|
|
7030
8168
|
});
|
|
7031
8169
|
let retriedRun = null;
|
|
8170
|
+
const retryAgent = await getAgent(run.agentId);
|
|
7032
8171
|
if (shouldRetry) {
|
|
7033
|
-
|
|
7034
|
-
|
|
7035
|
-
retriedRun = await enqueueProcessLossRetry(finalizedRun, agent, now);
|
|
8172
|
+
if (retryAgent) {
|
|
8173
|
+
retriedRun = await enqueueProcessLossRetry(finalizedRun, retryAgent, now);
|
|
7036
8174
|
}
|
|
7037
8175
|
}
|
|
7038
|
-
else {
|
|
8176
|
+
else if (retryAgent) {
|
|
8177
|
+
const scheduled = await scheduleInteractionContinuationInfrastructureRetryIfEligible(finalizedRun, retryAgent);
|
|
8178
|
+
retriedRun = scheduled?.outcome === "scheduled" ? scheduled.run : null;
|
|
8179
|
+
}
|
|
8180
|
+
if (!retriedRun) {
|
|
7039
8181
|
await releaseIssueExecutionAndPromote(finalizedRun);
|
|
7040
8182
|
}
|
|
7041
8183
|
await appendRunEvent(finalizedRun, await nextRunEventSeq(finalizedRun.id), {
|
|
@@ -7063,18 +8205,21 @@ export function heartbeatService(db, options = {}) {
|
|
|
7063
8205
|
return { reaped: reaped.length, runIds: reaped };
|
|
7064
8206
|
}
|
|
7065
8207
|
async function resumeQueuedRuns() {
|
|
8208
|
+
if ((await getSchedulingSuppression()).suppressed)
|
|
8209
|
+
return;
|
|
8210
|
+
const cutoff = await getWorktreeExecutionCutoff();
|
|
7066
8211
|
const queuedRuns = await db
|
|
7067
8212
|
.select({ agentId: heartbeatRuns.agentId })
|
|
7068
8213
|
.from(heartbeatRuns)
|
|
7069
8214
|
.innerJoin(companies, eq(companies.id, heartbeatRuns.companyId))
|
|
7070
|
-
.where(and(eq(heartbeatRuns.status, "queued"), eq(companies.status, "active")));
|
|
8215
|
+
.where(and(eq(heartbeatRuns.status, "queued"), eq(companies.status, "active"), cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined));
|
|
7071
8216
|
const agentIds = [...new Set(queuedRuns.map((r) => r.agentId))];
|
|
7072
8217
|
for (const agentId of agentIds) {
|
|
7073
8218
|
await startNextQueuedRunForAgent(agentId);
|
|
7074
8219
|
}
|
|
7075
8220
|
}
|
|
7076
8221
|
async function reconcileStrandedAssignedIssues() {
|
|
7077
|
-
return recovery.reconcileStrandedAssignedIssues();
|
|
8222
|
+
return recovery.reconcileStrandedAssignedIssues({ issueCreatedAtGte: await getWorktreeExecutionCutoff() });
|
|
7078
8223
|
}
|
|
7079
8224
|
async function sweepStaleIssueLocks() {
|
|
7080
8225
|
return recovery.sweepStaleIssueLocks();
|
|
@@ -7091,13 +8236,13 @@ export function heartbeatService(db, options = {}) {
|
|
|
7091
8236
|
readNonEmptyString(nestedContext.taskId);
|
|
7092
8237
|
}
|
|
7093
8238
|
async function scanSilentActiveRuns(opts) {
|
|
7094
|
-
return recovery.scanSilentActiveRuns(opts);
|
|
8239
|
+
return recovery.scanSilentActiveRuns({ ...opts, issueCreatedAtGte: await getWorktreeExecutionCutoff() });
|
|
7095
8240
|
}
|
|
7096
8241
|
async function reconcileProductivityReviews(opts) {
|
|
7097
|
-
return productivityReviews.reconcileProductivityReviews(opts);
|
|
8242
|
+
return productivityReviews.reconcileProductivityReviews({ ...opts, issueCreatedAtGte: await getWorktreeExecutionCutoff() });
|
|
7098
8243
|
}
|
|
7099
8244
|
async function reconcileTaskWatchdogs(opts) {
|
|
7100
|
-
return taskWatchdogs.reconcileTaskWatchdogs(opts);
|
|
8245
|
+
return taskWatchdogs.reconcileTaskWatchdogs({ ...opts, issueCreatedAtGte: await getWorktreeExecutionCutoff() });
|
|
7101
8246
|
}
|
|
7102
8247
|
async function buildRunOutputSilence(run, now = new Date()) {
|
|
7103
8248
|
return recovery.buildRunOutputSilence(run, now);
|
|
@@ -7106,7 +8251,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
7106
8251
|
return recovery.buildIssueGraphLivenessAutoRecoveryPreview(opts);
|
|
7107
8252
|
}
|
|
7108
8253
|
async function reconcileIssueGraphLiveness(opts) {
|
|
7109
|
-
return recovery.reconcileIssueGraphLiveness(opts);
|
|
8254
|
+
return recovery.reconcileIssueGraphLiveness({ ...opts, issueCreatedAtGte: await getWorktreeExecutionCutoff() });
|
|
7110
8255
|
}
|
|
7111
8256
|
async function updateRuntimeState(agent, run, result, session, normalizedUsage) {
|
|
7112
8257
|
await ensureRuntimeState(agent);
|
|
@@ -7117,6 +8262,12 @@ export function heartbeatService(db, options = {}) {
|
|
|
7117
8262
|
const billingType = normalizeLedgerBillingType(result.billingType);
|
|
7118
8263
|
const additionalCostCents = normalizeBilledCostCents(result.costUsd, billingType);
|
|
7119
8264
|
const hasTokenUsage = inputTokens > 0 || outputTokens > 0 || cachedInputTokens > 0;
|
|
8265
|
+
const costStatus = resolveLedgerCostStatus({
|
|
8266
|
+
costUsd: result.costUsd,
|
|
8267
|
+
inputTokens,
|
|
8268
|
+
cachedInputTokens,
|
|
8269
|
+
outputTokens,
|
|
8270
|
+
});
|
|
7120
8271
|
const provider = result.provider ?? "unknown";
|
|
7121
8272
|
const biller = resolveLedgerBiller(result);
|
|
7122
8273
|
const ledgerScope = await resolveLedgerScopeForRun(db, agent.companyId, run);
|
|
@@ -7145,6 +8296,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
7145
8296
|
provider,
|
|
7146
8297
|
biller,
|
|
7147
8298
|
billingType,
|
|
8299
|
+
costStatus,
|
|
7148
8300
|
model: result.model ?? "unknown",
|
|
7149
8301
|
inputTokens,
|
|
7150
8302
|
cachedInputTokens,
|
|
@@ -7155,6 +8307,9 @@ export function heartbeatService(db, options = {}) {
|
|
|
7155
8307
|
}
|
|
7156
8308
|
}
|
|
7157
8309
|
async function startNextQueuedRunForAgent(agentId) {
|
|
8310
|
+
if ((await getSchedulingSuppression()).suppressed)
|
|
8311
|
+
return [];
|
|
8312
|
+
const cutoff = await getWorktreeExecutionCutoff();
|
|
7158
8313
|
return withAgentStartLock(agentId, async () => {
|
|
7159
8314
|
const agent = await getAgent(agentId);
|
|
7160
8315
|
if (!agent)
|
|
@@ -7174,7 +8329,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
7174
8329
|
const queuedRuns = await db
|
|
7175
8330
|
.select()
|
|
7176
8331
|
.from(heartbeatRuns)
|
|
7177
|
-
.where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued")))
|
|
8332
|
+
.where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued"), cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined))
|
|
7178
8333
|
.orderBy(asc(heartbeatRuns.createdAt));
|
|
7179
8334
|
if (queuedRuns.length === 0)
|
|
7180
8335
|
return [];
|
|
@@ -7232,6 +8387,8 @@ export function heartbeatService(db, options = {}) {
|
|
|
7232
8387
|
});
|
|
7233
8388
|
}
|
|
7234
8389
|
async function executeRun(runId) {
|
|
8390
|
+
if ((await getSchedulingSuppression()).suppressed)
|
|
8391
|
+
return;
|
|
7235
8392
|
let run = await getRun(runId);
|
|
7236
8393
|
if (!run)
|
|
7237
8394
|
return;
|
|
@@ -7246,6 +8403,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
7246
8403
|
run = claimed;
|
|
7247
8404
|
}
|
|
7248
8405
|
activeRunExecutions.add(run.id);
|
|
8406
|
+
let runScratch = null;
|
|
7249
8407
|
try {
|
|
7250
8408
|
const agent = await getAgent(run.agentId);
|
|
7251
8409
|
if (!agent) {
|
|
@@ -7282,6 +8440,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
7282
8440
|
contextSnapshot: context,
|
|
7283
8441
|
issueStatus: issueContext.status,
|
|
7284
8442
|
issueAssigneeAgentId: issueContext.assigneeAgentId,
|
|
8443
|
+
issueExecutionState: issueContext.executionState,
|
|
7285
8444
|
isDependencyReady: issueDependencyReadiness?.isDependencyReady ?? true,
|
|
7286
8445
|
agentId: agent.id,
|
|
7287
8446
|
})) {
|
|
@@ -7382,6 +8541,26 @@ export function heartbeatService(db, options = {}) {
|
|
|
7382
8541
|
delete context.acceptedPlanWakeRouting;
|
|
7383
8542
|
}
|
|
7384
8543
|
const routineEnvContext = await getRoutineEnvForExecutionIssue(agent.companyId, issueContext);
|
|
8544
|
+
const responsibleUserId = await resolveResponsibleUserIdForRun({
|
|
8545
|
+
run,
|
|
8546
|
+
contextSnapshot: context,
|
|
8547
|
+
issueContext,
|
|
8548
|
+
routineEnvContext,
|
|
8549
|
+
});
|
|
8550
|
+
if (responsibleUserId && run.responsibleUserId !== responsibleUserId) {
|
|
8551
|
+
await db
|
|
8552
|
+
.update(heartbeatRuns)
|
|
8553
|
+
.set({ responsibleUserId, updatedAt: new Date() })
|
|
8554
|
+
.where(eq(heartbeatRuns.id, run.id));
|
|
8555
|
+
run = { ...run, responsibleUserId };
|
|
8556
|
+
}
|
|
8557
|
+
if (responsibleUserId && issueContext && !issueContext.responsibleUserId) {
|
|
8558
|
+
await db
|
|
8559
|
+
.update(issues)
|
|
8560
|
+
.set({ responsibleUserId, updatedAt: new Date() })
|
|
8561
|
+
.where(and(eq(issues.companyId, agent.companyId), eq(issues.id, issueContext.id), isNull(issues.responsibleUserId)));
|
|
8562
|
+
issueContext = { ...issueContext, responsibleUserId };
|
|
8563
|
+
}
|
|
7385
8564
|
const projectExecutionWorkspacePolicy = gateProjectExecutionWorkspacePolicy(parseProjectExecutionWorkspacePolicy(projectContext?.executionWorkspacePolicy), isolatedWorkspacesEnabled);
|
|
7386
8565
|
const trustPreset = resolveCoreTrustPreset({
|
|
7387
8566
|
companyId: agent.companyId,
|
|
@@ -7459,6 +8638,18 @@ export function heartbeatService(db, options = {}) {
|
|
|
7459
8638
|
else {
|
|
7460
8639
|
delete context.paperclipContinuationSummary;
|
|
7461
8640
|
}
|
|
8641
|
+
const pinnedSkillTestContext = issueRef?.workMode === "skill_test"
|
|
8642
|
+
? await getPinnedSkillTestContext(agent.companyId, issueRef.id)
|
|
8643
|
+
: null;
|
|
8644
|
+
if (pinnedSkillTestContext) {
|
|
8645
|
+
context.paperclipSkillTest = {
|
|
8646
|
+
...pinnedSkillTestContext,
|
|
8647
|
+
directive: "Use this pinned file inventory as the exact skill revision under test, regardless of synced runtime skills.",
|
|
8648
|
+
};
|
|
8649
|
+
}
|
|
8650
|
+
else {
|
|
8651
|
+
delete context.paperclipSkillTest;
|
|
8652
|
+
}
|
|
7462
8653
|
const paperclipWakePayload = await buildPaperclipWakePayload({
|
|
7463
8654
|
db,
|
|
7464
8655
|
companyId: agent.companyId,
|
|
@@ -7527,15 +8718,20 @@ export function heartbeatService(db, options = {}) {
|
|
|
7527
8718
|
else {
|
|
7528
8719
|
delete context.paperclipTaskMarkdown;
|
|
7529
8720
|
}
|
|
7530
|
-
const
|
|
7531
|
-
const
|
|
7532
|
-
|
|
7533
|
-
|
|
7534
|
-
|
|
7535
|
-
|
|
8721
|
+
const requestedExecutionWorkspaceId = readNonEmptyString(issueRef?.executionWorkspaceId);
|
|
8722
|
+
const existingExecutionWorkspace = requestedExecutionWorkspaceId ? await executionWorkspacesSvc.getById(requestedExecutionWorkspaceId) : null;
|
|
8723
|
+
const workspaceReuseRequest = resolveExecutionWorkspaceReuseRequestForIssue({
|
|
8724
|
+
issueExecutionWorkspaceId: requestedExecutionWorkspaceId,
|
|
8725
|
+
issueExecutionWorkspacePreference: issueRef?.executionWorkspacePreference ?? null,
|
|
8726
|
+
existingExecutionWorkspaceStatus: existingExecutionWorkspace?.status ?? null,
|
|
8727
|
+
});
|
|
8728
|
+
const requestedShouldReuseExisting = workspaceReuseRequest.requestedShouldReuseExisting;
|
|
8729
|
+
const reusableExistingExecutionWorkspace = workspaceReuseRequest.existingExecutionWorkspaceAvailable
|
|
8730
|
+
? existingExecutionWorkspace
|
|
7536
8731
|
: null;
|
|
7537
|
-
const
|
|
7538
|
-
|
|
8732
|
+
const requestedReusableExecutionWorkspaceConfig = reusableExistingExecutionWorkspace?.config ?? null;
|
|
8733
|
+
const persistedExecutionWorkspaceMode = requestedShouldReuseExisting && reusableExistingExecutionWorkspace
|
|
8734
|
+
? issueExecutionWorkspaceModeForPersistedWorkspace(reusableExistingExecutionWorkspace.mode)
|
|
7539
8735
|
: null;
|
|
7540
8736
|
const localEnvironment = await environmentsSvc.ensureLocalEnvironment(agent.companyId);
|
|
7541
8737
|
const resolvedInstanceSettings = await instanceSettings.get();
|
|
@@ -7674,6 +8870,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
7674
8870
|
environmentEnv: selectedEnvironmentForConfig?.envVars ?? null,
|
|
7675
8871
|
projectId: projectContext?.id ?? null,
|
|
7676
8872
|
routineId: routineEnvContext.routineId,
|
|
8873
|
+
responsibleUserId,
|
|
7677
8874
|
executionRunConfig,
|
|
7678
8875
|
projectEnv: projectContext?.env ?? null,
|
|
7679
8876
|
routineEnv: routineEnvContext.env,
|
|
@@ -7724,16 +8921,16 @@ export function heartbeatService(db, options = {}) {
|
|
|
7724
8921
|
projectPolicy: projectExecutionWorkspacePolicy,
|
|
7725
8922
|
issueSettings: issueExecutionWorkspaceSettings,
|
|
7726
8923
|
reusableExecutionWorkspaceConfig: requestedReusableExecutionWorkspaceConfig,
|
|
7727
|
-
existingExecutionWorkspace:
|
|
8924
|
+
existingExecutionWorkspace: reusableExistingExecutionWorkspace
|
|
7728
8925
|
? {
|
|
7729
|
-
id:
|
|
7730
|
-
mode:
|
|
7731
|
-
strategyType:
|
|
7732
|
-
projectWorkspaceId:
|
|
7733
|
-
repoUrl:
|
|
7734
|
-
baseRef:
|
|
7735
|
-
branchName:
|
|
7736
|
-
config:
|
|
8926
|
+
id: reusableExistingExecutionWorkspace.id,
|
|
8927
|
+
mode: reusableExistingExecutionWorkspace.mode,
|
|
8928
|
+
strategyType: reusableExistingExecutionWorkspace.strategyType,
|
|
8929
|
+
projectWorkspaceId: reusableExistingExecutionWorkspace.projectWorkspaceId,
|
|
8930
|
+
repoUrl: reusableExistingExecutionWorkspace.repoUrl,
|
|
8931
|
+
baseRef: reusableExistingExecutionWorkspace.baseRef,
|
|
8932
|
+
branchName: reusableExistingExecutionWorkspace.branchName,
|
|
8933
|
+
config: reusableExistingExecutionWorkspace.config,
|
|
7737
8934
|
}
|
|
7738
8935
|
: null,
|
|
7739
8936
|
},
|
|
@@ -7818,15 +9015,15 @@ export function heartbeatService(db, options = {}) {
|
|
|
7818
9015
|
repoUrl: resolvedWorkspace.repoUrl,
|
|
7819
9016
|
repoRef: resolvedWorkspace.repoRef,
|
|
7820
9017
|
};
|
|
9018
|
+
await assertGitWorktreeBaseWorkspaceReady({
|
|
9019
|
+
requestedExecutionWorkspaceMode,
|
|
9020
|
+
config: hostExecutionWorkspaceConfig,
|
|
9021
|
+
issue: issueRef,
|
|
9022
|
+
base: executionWorkspaceBase,
|
|
9023
|
+
});
|
|
7821
9024
|
const workspaceStrategyForFingerprint = parseObject(hostExecutionWorkspaceConfig.workspaceStrategy);
|
|
7822
9025
|
const workspaceStrategyFingerprintValue = Object.keys(workspaceStrategyForFingerprint).length > 0 ? workspaceStrategyForFingerprint : null;
|
|
7823
|
-
const latestWorkspaceStrategyType =
|
|
7824
|
-
(requestedExecutionWorkspaceMode === "agent_default"
|
|
7825
|
-
? "adapter_managed"
|
|
7826
|
-
: requestedExecutionWorkspaceMode === "isolated_workspace" ||
|
|
7827
|
-
requestedExecutionWorkspaceMode === "operator_branch"
|
|
7828
|
-
? "git_worktree"
|
|
7829
|
-
: "project_primary");
|
|
9026
|
+
const latestWorkspaceStrategyType = resolveEffectiveWorkspaceStrategyType(requestedExecutionWorkspaceMode, hostExecutionWorkspaceConfig);
|
|
7830
9027
|
const selectedEnvironmentConfigForFingerprint = parseObject(selectedEnvironmentForConfig?.config);
|
|
7831
9028
|
const workspaceEnvironmentFingerprint = selectedEnvironmentForConfig
|
|
7832
9029
|
? {
|
|
@@ -7860,24 +9057,24 @@ export function heartbeatService(db, options = {}) {
|
|
|
7860
9057
|
realization: workspaceRealizationFingerprint,
|
|
7861
9058
|
secretManifest,
|
|
7862
9059
|
});
|
|
7863
|
-
const inferredExistingWorkspaceConfigMetadata =
|
|
9060
|
+
const inferredExistingWorkspaceConfigMetadata = reusableExistingExecutionWorkspace
|
|
7864
9061
|
? buildEffectiveRunWorkspaceConfigMetadata({
|
|
7865
|
-
mode: issueExecutionWorkspaceModeForPersistedWorkspace(
|
|
7866
|
-
projectId:
|
|
7867
|
-
projectWorkspaceId:
|
|
7868
|
-
strategyType:
|
|
9062
|
+
mode: issueExecutionWorkspaceModeForPersistedWorkspace(reusableExistingExecutionWorkspace.mode),
|
|
9063
|
+
projectId: reusableExistingExecutionWorkspace.projectId,
|
|
9064
|
+
projectWorkspaceId: reusableExistingExecutionWorkspace.projectWorkspaceId,
|
|
9065
|
+
strategyType: reusableExistingExecutionWorkspace.strategyType,
|
|
7869
9066
|
workspaceStrategy: workspaceStrategyFingerprintValue
|
|
7870
9067
|
? {
|
|
7871
9068
|
...workspaceStrategyFingerprintValue,
|
|
7872
|
-
type:
|
|
7873
|
-
...(
|
|
7874
|
-
? { baseRef:
|
|
9069
|
+
type: reusableExistingExecutionWorkspace.strategyType,
|
|
9070
|
+
...(reusableExistingExecutionWorkspace.baseRef
|
|
9071
|
+
? { baseRef: reusableExistingExecutionWorkspace.baseRef }
|
|
7875
9072
|
: {}),
|
|
7876
9073
|
}
|
|
7877
|
-
: { type:
|
|
7878
|
-
repoUrl:
|
|
7879
|
-
repoRef:
|
|
7880
|
-
configSnapshot:
|
|
9074
|
+
: { type: reusableExistingExecutionWorkspace.strategyType },
|
|
9075
|
+
repoUrl: reusableExistingExecutionWorkspace.repoUrl,
|
|
9076
|
+
repoRef: reusableExistingExecutionWorkspace.baseRef,
|
|
9077
|
+
configSnapshot: reusableExistingExecutionWorkspace.config,
|
|
7881
9078
|
environment: workspaceEnvironmentFingerprint,
|
|
7882
9079
|
realization: workspaceRealizationFingerprint,
|
|
7883
9080
|
secretManifest,
|
|
@@ -7885,82 +9082,107 @@ export function heartbeatService(db, options = {}) {
|
|
|
7885
9082
|
})
|
|
7886
9083
|
: null;
|
|
7887
9084
|
const workspaceConfigFreshness = resolveExecutionWorkspaceConfigFreshness({
|
|
7888
|
-
hasExistingWorkspace: requestedShouldReuseExisting && Boolean(
|
|
7889
|
-
existingWorkspaceMetadata:
|
|
9085
|
+
hasExistingWorkspace: requestedShouldReuseExisting && Boolean(reusableExistingExecutionWorkspace),
|
|
9086
|
+
existingWorkspaceMetadata: reusableExistingExecutionWorkspace?.metadata ?? null,
|
|
7890
9087
|
inferredMetadata: inferredExistingWorkspaceConfigMetadata,
|
|
7891
9088
|
nextMetadata: latestWorkspaceConfigMetadata,
|
|
7892
9089
|
});
|
|
7893
|
-
const
|
|
7894
|
-
|
|
9090
|
+
const workspaceReuseProvisioningPolicy = resolveExecutionWorkspaceReuseProvisioningPolicy({
|
|
9091
|
+
requestedShouldReuseExisting,
|
|
9092
|
+
workspaceConfigFreshness,
|
|
9093
|
+
});
|
|
7895
9094
|
const workspaceOperationRecorder = workspaceOperationsSvc.createRecorder({
|
|
7896
9095
|
companyId: agent.companyId,
|
|
7897
9096
|
heartbeatRunId: run.id,
|
|
7898
|
-
executionWorkspaceId:
|
|
9097
|
+
executionWorkspaceId: workspaceReuseProvisioningPolicy.shouldRestoreExistingWorkspace
|
|
9098
|
+
? workspaceReuseRequest.requestedExecutionWorkspaceId
|
|
9099
|
+
: null,
|
|
7899
9100
|
issueId,
|
|
7900
9101
|
});
|
|
7901
|
-
const reusedExecutionWorkspace =
|
|
7902
|
-
|
|
7903
|
-
|
|
7904
|
-
|
|
7905
|
-
|
|
7906
|
-
|
|
7907
|
-
|
|
7908
|
-
|
|
7909
|
-
|
|
7910
|
-
|
|
7911
|
-
|
|
7912
|
-
|
|
7913
|
-
|
|
7914
|
-
|
|
7915
|
-
|
|
7916
|
-
|
|
7917
|
-
|
|
7918
|
-
|
|
7919
|
-
|
|
7920
|
-
|
|
9102
|
+
const { executionWorkspace, reusedExecutionWorkspace, policy: resolvedWorkspaceReusePolicy } = await provisionExecutionWorkspaceForFreshnessDecision({
|
|
9103
|
+
requestedShouldReuseExisting,
|
|
9104
|
+
existingExecutionWorkspaceId: workspaceReuseRequest.requestedExecutionWorkspaceId,
|
|
9105
|
+
issueRef,
|
|
9106
|
+
runId: run.id,
|
|
9107
|
+
workspaceConfigFreshness,
|
|
9108
|
+
restoreExistingWorkspace: reusableExistingExecutionWorkspace
|
|
9109
|
+
? () => ensurePersistedExecutionWorkspaceAvailable({
|
|
9110
|
+
db,
|
|
9111
|
+
base: executionWorkspaceBase,
|
|
9112
|
+
workspace: {
|
|
9113
|
+
id: reusableExistingExecutionWorkspace.id,
|
|
9114
|
+
mode: reusableExistingExecutionWorkspace.mode,
|
|
9115
|
+
strategyType: reusableExistingExecutionWorkspace.strategyType,
|
|
9116
|
+
cwd: reusableExistingExecutionWorkspace.cwd,
|
|
9117
|
+
providerRef: reusableExistingExecutionWorkspace.providerRef,
|
|
9118
|
+
projectId: reusableExistingExecutionWorkspace.projectId,
|
|
9119
|
+
projectWorkspaceId: reusableExistingExecutionWorkspace.projectWorkspaceId,
|
|
9120
|
+
repoUrl: reusableExistingExecutionWorkspace.repoUrl,
|
|
9121
|
+
baseRef: reusableExistingExecutionWorkspace.baseRef,
|
|
9122
|
+
branchName: reusableExistingExecutionWorkspace.branchName,
|
|
9123
|
+
metadata: reusableExistingExecutionWorkspace.metadata,
|
|
9124
|
+
config: {
|
|
9125
|
+
provisionCommand: configSnapshot?.provisionCommand
|
|
9126
|
+
?? reusableExistingExecutionWorkspace.config?.provisionCommand
|
|
9127
|
+
?? projectExecutionWorkspacePolicy?.workspaceStrategy?.provisionCommand
|
|
9128
|
+
?? null,
|
|
9129
|
+
},
|
|
7921
9130
|
},
|
|
7922
|
-
|
|
9131
|
+
issue: issueRef,
|
|
9132
|
+
agent: {
|
|
9133
|
+
id: agent.id,
|
|
9134
|
+
name: agent.name,
|
|
9135
|
+
companyId: agent.companyId,
|
|
9136
|
+
},
|
|
9137
|
+
heartbeatRunId: run.id,
|
|
9138
|
+
enableWorkspaceBranchReconcileForward: resolvedInstanceSettings.experimental.enableWorkspaceBranchReconcileForward,
|
|
9139
|
+
enableWorkspaceDirtyQuarantineRepair: resolvedInstanceSettings.experimental.enableWorkspaceDirtyQuarantineRepair,
|
|
9140
|
+
recorder: workspaceOperationRecorder,
|
|
9141
|
+
})
|
|
9142
|
+
: null,
|
|
9143
|
+
realizeWorkspace: () => realizeExecutionWorkspace({
|
|
9144
|
+
db,
|
|
9145
|
+
base: executionWorkspaceBase,
|
|
9146
|
+
config: hostExecutionWorkspaceConfig,
|
|
7923
9147
|
issue: issueRef,
|
|
7924
9148
|
agent: {
|
|
7925
9149
|
id: agent.id,
|
|
7926
9150
|
name: agent.name,
|
|
7927
9151
|
companyId: agent.companyId,
|
|
7928
9152
|
},
|
|
9153
|
+
heartbeatRunId: run.id,
|
|
9154
|
+
enableWorkspaceBranchReconcileForward: resolvedInstanceSettings.experimental.enableWorkspaceBranchReconcileForward,
|
|
9155
|
+
enableWorkspaceDirtyQuarantineRepair: resolvedInstanceSettings.experimental.enableWorkspaceDirtyQuarantineRepair,
|
|
7929
9156
|
recorder: workspaceOperationRecorder,
|
|
7930
|
-
})
|
|
7931
|
-
: null;
|
|
7932
|
-
const executionWorkspace = reusedExecutionWorkspace ?? await realizeExecutionWorkspace({
|
|
7933
|
-
base: executionWorkspaceBase,
|
|
7934
|
-
config: hostExecutionWorkspaceConfig,
|
|
7935
|
-
issue: issueRef,
|
|
7936
|
-
agent: {
|
|
7937
|
-
id: agent.id,
|
|
7938
|
-
name: agent.name,
|
|
7939
|
-
companyId: agent.companyId,
|
|
7940
|
-
},
|
|
7941
|
-
recorder: workspaceOperationRecorder,
|
|
9157
|
+
}),
|
|
7942
9158
|
});
|
|
7943
9159
|
const resolvedProjectId = executionWorkspace.projectId ?? issueRef?.projectId ?? executionProjectId ?? null;
|
|
7944
9160
|
const resolvedProjectWorkspaceId = issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null;
|
|
7945
9161
|
let persistedExecutionWorkspace = null;
|
|
7946
9162
|
const nextExecutionWorkspaceMetadata = mergeExecutionWorkspaceMetadataForPersistence({
|
|
7947
|
-
existingMetadata:
|
|
9163
|
+
existingMetadata: resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace
|
|
9164
|
+
? reusableExistingExecutionWorkspace?.metadata ?? null
|
|
9165
|
+
: null,
|
|
7948
9166
|
source: executionWorkspace.source,
|
|
7949
9167
|
createdByRuntime: executionWorkspace.created,
|
|
7950
9168
|
configSnapshot,
|
|
7951
|
-
shouldReuseExisting,
|
|
7952
|
-
shouldRefreshConfigSnapshot: shouldRefreshWorkspaceConfigSnapshot,
|
|
7953
|
-
workspaceConfigMetadata:
|
|
9169
|
+
shouldReuseExisting: resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace,
|
|
9170
|
+
shouldRefreshConfigSnapshot: resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot,
|
|
9171
|
+
workspaceConfigMetadata: resolvedWorkspaceReusePolicy.shouldPersistLatestWorkspaceConfigMetadata
|
|
9172
|
+
? latestWorkspaceConfigMetadata
|
|
9173
|
+
: null,
|
|
7954
9174
|
baseRef: executionWorkspace.repoRef,
|
|
7955
9175
|
baseRefSha: executionWorkspace.baseRefSha ?? null,
|
|
7956
9176
|
});
|
|
9177
|
+
const pendingForwardBranchReconcile = executionWorkspace.pendingForwardBranchReconcile ?? null;
|
|
9178
|
+
const branchNameForInitialPersistence = pendingForwardBranchReconcile?.recordedBranchName ?? executionWorkspace.branchName;
|
|
7957
9179
|
try {
|
|
7958
|
-
persistedExecutionWorkspace =
|
|
7959
|
-
? await executionWorkspacesSvc.update(
|
|
9180
|
+
persistedExecutionWorkspace = resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace && reusableExistingExecutionWorkspace
|
|
9181
|
+
? await executionWorkspacesSvc.update(reusableExistingExecutionWorkspace.id, {
|
|
7960
9182
|
cwd: executionWorkspace.cwd,
|
|
7961
9183
|
repoUrl: executionWorkspace.repoUrl,
|
|
7962
9184
|
baseRef: executionWorkspace.repoRef,
|
|
7963
|
-
branchName:
|
|
9185
|
+
branchName: branchNameForInitialPersistence,
|
|
7964
9186
|
providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs",
|
|
7965
9187
|
providerRef: executionWorkspace.worktreePath,
|
|
7966
9188
|
status: "active",
|
|
@@ -7981,12 +9203,12 @@ export function heartbeatService(db, options = {}) {
|
|
|
7981
9203
|
? "adapter_managed"
|
|
7982
9204
|
: "shared_workspace",
|
|
7983
9205
|
strategyType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "project_primary",
|
|
7984
|
-
name:
|
|
9206
|
+
name: branchNameForInitialPersistence ?? issueRef?.identifier ?? `workspace-${agent.id.slice(0, 8)}`,
|
|
7985
9207
|
status: "active",
|
|
7986
9208
|
cwd: executionWorkspace.cwd,
|
|
7987
9209
|
repoUrl: executionWorkspace.repoUrl,
|
|
7988
9210
|
baseRef: executionWorkspace.repoRef,
|
|
7989
|
-
branchName:
|
|
9211
|
+
branchName: branchNameForInitialPersistence,
|
|
7990
9212
|
providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs",
|
|
7991
9213
|
providerRef: executionWorkspace.worktreePath,
|
|
7992
9214
|
lastUsedAt: new Date(),
|
|
@@ -8000,7 +9222,9 @@ export function heartbeatService(db, options = {}) {
|
|
|
8000
9222
|
try {
|
|
8001
9223
|
await cleanupExecutionWorkspaceArtifacts({
|
|
8002
9224
|
workspace: {
|
|
8003
|
-
id:
|
|
9225
|
+
id: reusableExistingExecutionWorkspace?.id
|
|
9226
|
+
?? workspaceReuseRequest.requestedExecutionWorkspaceId
|
|
9227
|
+
?? `transient-${run.id}`,
|
|
8004
9228
|
cwd: executionWorkspace.cwd,
|
|
8005
9229
|
providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs",
|
|
8006
9230
|
providerRef: executionWorkspace.worktreePath,
|
|
@@ -8040,18 +9264,18 @@ export function heartbeatService(db, options = {}) {
|
|
|
8040
9264
|
recorder: workspaceOperationRecorder,
|
|
8041
9265
|
runId: run.id,
|
|
8042
9266
|
decision: workspaceConfigFreshness,
|
|
8043
|
-
hasExistingWorkspace: Boolean(
|
|
9267
|
+
hasExistingWorkspace: Boolean(reusableExistingExecutionWorkspace),
|
|
8044
9268
|
reuseRequested: requestedShouldReuseExisting,
|
|
8045
9269
|
workspaceReused: Boolean(reusedExecutionWorkspace),
|
|
8046
|
-
configSnapshotRefreshed: shouldRefreshWorkspaceConfigSnapshot,
|
|
8047
|
-
previousWorkspaceId:
|
|
9270
|
+
configSnapshotRefreshed: resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot,
|
|
9271
|
+
previousWorkspaceId: workspaceReuseRequest.requestedExecutionWorkspaceId,
|
|
8048
9272
|
activeWorkspaceId: persistedExecutionWorkspace?.id ?? null,
|
|
8049
9273
|
});
|
|
8050
|
-
if (
|
|
9274
|
+
if (reusableExistingExecutionWorkspace &&
|
|
8051
9275
|
persistedExecutionWorkspace &&
|
|
8052
|
-
|
|
8053
|
-
|
|
8054
|
-
await executionWorkspacesSvc.update(
|
|
9276
|
+
reusableExistingExecutionWorkspace.id !== persistedExecutionWorkspace.id &&
|
|
9277
|
+
reusableExistingExecutionWorkspace.status === "active") {
|
|
9278
|
+
await executionWorkspacesSvc.update(reusableExistingExecutionWorkspace.id, {
|
|
8055
9279
|
status: "idle",
|
|
8056
9280
|
cleanupReason: null,
|
|
8057
9281
|
});
|
|
@@ -8144,6 +9368,46 @@ export function heartbeatService(db, options = {}) {
|
|
|
8144
9368
|
const workspaceRealization = realizationResult.workspaceRealization;
|
|
8145
9369
|
const executionTarget = realizationResult.executionTarget;
|
|
8146
9370
|
const remoteExecution = realizationResult.remoteExecution;
|
|
9371
|
+
if (!executionTarget || executionTarget.kind === "local") {
|
|
9372
|
+
try {
|
|
9373
|
+
runScratch = await prepareHeartbeatRunScratch({
|
|
9374
|
+
companyId: agent.companyId,
|
|
9375
|
+
agentId: agent.id,
|
|
9376
|
+
runId: run.id,
|
|
9377
|
+
issueId: issueRef?.id ?? null,
|
|
9378
|
+
issueIdentifier: issueRef?.identifier ?? null,
|
|
9379
|
+
});
|
|
9380
|
+
const existingRuntimeEnv = parseObject(runtimeConfig.env);
|
|
9381
|
+
const scratchEnv = buildHeartbeatRunScratchEnv(existingRuntimeEnv, runScratch);
|
|
9382
|
+
runtimeConfig = {
|
|
9383
|
+
...runtimeConfig,
|
|
9384
|
+
env: {
|
|
9385
|
+
...existingRuntimeEnv,
|
|
9386
|
+
...scratchEnv.env,
|
|
9387
|
+
},
|
|
9388
|
+
};
|
|
9389
|
+
context.paperclipScratch = {
|
|
9390
|
+
type: "heartbeat_run",
|
|
9391
|
+
dir: runScratch.dir,
|
|
9392
|
+
cleanupPolicy: "terminal_run",
|
|
9393
|
+
marker: HEARTBEAT_RUN_SCRATCH_MARKER,
|
|
9394
|
+
tempKeysApplied: scratchEnv.tempKeysApplied,
|
|
9395
|
+
};
|
|
9396
|
+
}
|
|
9397
|
+
catch (scratchPrepareError) {
|
|
9398
|
+
runScratch = null;
|
|
9399
|
+
delete context.paperclipScratch;
|
|
9400
|
+
logger.warn({
|
|
9401
|
+
err: scratchPrepareError,
|
|
9402
|
+
runId: run.id,
|
|
9403
|
+
issueId,
|
|
9404
|
+
agentId: agent.id,
|
|
9405
|
+
}, "failed to prepare heartbeat run scratch directory; continuing without scratch env");
|
|
9406
|
+
}
|
|
9407
|
+
}
|
|
9408
|
+
else {
|
|
9409
|
+
delete context.paperclipScratch;
|
|
9410
|
+
}
|
|
8147
9411
|
context.paperclipEnvironment = {
|
|
8148
9412
|
id: selectedEnvironment.id,
|
|
8149
9413
|
name: selectedEnvironment.name,
|
|
@@ -8217,6 +9481,16 @@ export function heartbeatService(db, options = {}) {
|
|
|
8217
9481
|
})(),
|
|
8218
9482
|
};
|
|
8219
9483
|
context.paperclipWorkspaces = resolvedWorkspace.workspaceHints;
|
|
9484
|
+
// The wake payload is built before the execution workspace is resolved, so
|
|
9485
|
+
// attach the branch pin here; the shared wake-prompt renderer surfaces it as
|
|
9486
|
+
// a one-time "stay on this branch" hint on non-resumed sessions.
|
|
9487
|
+
if (executionWorkspace.branchName) {
|
|
9488
|
+
const wakePayloadForWorkspace = parseObject(context[PAPERCLIP_WAKE_PAYLOAD_KEY]);
|
|
9489
|
+
context[PAPERCLIP_WAKE_PAYLOAD_KEY] = {
|
|
9490
|
+
...wakePayloadForWorkspace,
|
|
9491
|
+
executionWorkspace: { branchName: executionWorkspace.branchName },
|
|
9492
|
+
};
|
|
9493
|
+
}
|
|
8220
9494
|
const runtimeServiceIntents = (() => {
|
|
8221
9495
|
const runtimeConfig = parseObject(hostExecutionWorkspaceConfig.workspaceRuntime);
|
|
8222
9496
|
return Array.isArray(runtimeConfig.services)
|
|
@@ -8302,12 +9576,12 @@ export function heartbeatService(db, options = {}) {
|
|
|
8302
9576
|
reasons: workspaceConfigFreshness.reasons,
|
|
8303
9577
|
reuseRequested: requestedShouldReuseExisting,
|
|
8304
9578
|
workspaceReused: Boolean(reusedExecutionWorkspace),
|
|
8305
|
-
configSnapshotRefreshed: shouldRefreshWorkspaceConfigSnapshot,
|
|
9579
|
+
configSnapshotRefreshed: resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot,
|
|
8306
9580
|
storedFingerprintPresent: workspaceConfigFreshness.storedFingerprintPresent,
|
|
8307
9581
|
storedFingerprint: workspaceConfigFreshness.storedFingerprint,
|
|
8308
9582
|
inferredFingerprint: workspaceConfigFreshness.inferredFingerprint,
|
|
8309
9583
|
nextFingerprint: workspaceConfigFreshness.nextFingerprint,
|
|
8310
|
-
previousWorkspaceId:
|
|
9584
|
+
previousWorkspaceId: workspaceReuseRequest.requestedExecutionWorkspaceId,
|
|
8311
9585
|
activeWorkspaceId: persistedExecutionWorkspace?.id ?? null,
|
|
8312
9586
|
},
|
|
8313
9587
|
};
|
|
@@ -8424,19 +9698,21 @@ export function heartbeatService(db, options = {}) {
|
|
|
8424
9698
|
if (stream === "stderr")
|
|
8425
9699
|
stderrExcerpt = appendExcerpt(stderrExcerpt, sanitizedChunk);
|
|
8426
9700
|
const ts = new Date().toISOString();
|
|
9701
|
+
outputSeq += 1;
|
|
9702
|
+
const chunkSeq = outputSeq;
|
|
8427
9703
|
let appendedBytes = 0;
|
|
8428
9704
|
if (handle) {
|
|
8429
9705
|
appendedBytes = await runLogStore.append(handle, {
|
|
8430
9706
|
stream,
|
|
8431
9707
|
chunk: sanitizedChunk,
|
|
8432
9708
|
ts,
|
|
9709
|
+
seq: chunkSeq,
|
|
8433
9710
|
});
|
|
8434
9711
|
persistedLogBytes += appendedBytes;
|
|
8435
9712
|
}
|
|
8436
|
-
outputSeq += 1;
|
|
8437
9713
|
outputProgressState.pending = {
|
|
8438
9714
|
at: new Date(ts),
|
|
8439
|
-
seq:
|
|
9715
|
+
seq: chunkSeq,
|
|
8440
9716
|
stream,
|
|
8441
9717
|
bytes: persistedLogBytes,
|
|
8442
9718
|
};
|
|
@@ -8472,6 +9748,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
8472
9748
|
agentId: run.agentId,
|
|
8473
9749
|
issueId,
|
|
8474
9750
|
ts,
|
|
9751
|
+
seq: chunkSeq,
|
|
8475
9752
|
stream,
|
|
8476
9753
|
chunk: payloadChunk,
|
|
8477
9754
|
truncated: payloadChunk.length !== sanitizedChunk.length,
|
|
@@ -8572,8 +9849,11 @@ export function heartbeatService(db, options = {}) {
|
|
|
8572
9849
|
});
|
|
8573
9850
|
};
|
|
8574
9851
|
const adapter = getServerAdapter(agent.adapterType);
|
|
9852
|
+
const localAgentJwtScope = issueRef?.workMode === "skill_test"
|
|
9853
|
+
? { kind: "skill_test", issueId: issueRef.id }
|
|
9854
|
+
: { kind: "standard" };
|
|
8575
9855
|
const authToken = adapter.supportsLocalAgentJwt
|
|
8576
|
-
? createLocalAgentJwt(agent.id, agent.companyId, agent.adapterType, run.id)
|
|
9856
|
+
? createLocalAgentJwt(agent.id, agent.companyId, agent.adapterType, run.id, run.responsibleUserId, localAgentJwtScope)
|
|
8577
9857
|
: null;
|
|
8578
9858
|
if (adapter.supportsLocalAgentJwt && !authToken) {
|
|
8579
9859
|
logger.warn({
|
|
@@ -8615,8 +9895,10 @@ export function heartbeatService(db, options = {}) {
|
|
|
8615
9895
|
let inspection = branchInspection.inspection;
|
|
8616
9896
|
const initialManagedGitWorktreeBranch = formatManagedGitWorktreeBranchInspection(inspection);
|
|
8617
9897
|
if (!inspection.valid && inspection.reasonCode === "branch_mismatch" && inspection.repoRoot) {
|
|
9898
|
+
let repairedExpectedBranchName = inspection.expectedBranchName;
|
|
8618
9899
|
try {
|
|
8619
|
-
await ensureGitWorktreeBranchCoherent({
|
|
9900
|
+
const coherence = await ensureGitWorktreeBranchCoherent({
|
|
9901
|
+
db,
|
|
8620
9902
|
repoRoot: inspection.repoRoot,
|
|
8621
9903
|
worktreePath: inspection.worktreePath,
|
|
8622
9904
|
expectedBranchName: inspection.expectedBranchName,
|
|
@@ -8630,8 +9912,18 @@ export function heartbeatService(db, options = {}) {
|
|
|
8630
9912
|
}
|
|
8631
9913
|
: null,
|
|
8632
9914
|
executionWorkspaceId: branchInspection.workspaceRecord.id,
|
|
9915
|
+
heartbeatRunId: run.id,
|
|
9916
|
+
enableWorkspaceBranchReconcileForward: resolvedInstanceSettings.experimental.enableWorkspaceBranchReconcileForward,
|
|
9917
|
+
enableWorkspaceDirtyQuarantineRepair: resolvedInstanceSettings.experimental.enableWorkspaceDirtyQuarantineRepair,
|
|
9918
|
+
persistForwardReconcile: false,
|
|
9919
|
+
reconcileOperationPhase: "workspace_finalize",
|
|
8633
9920
|
recorder: workspaceOperationRecorder,
|
|
8634
9921
|
});
|
|
9922
|
+
if (coherence.branchName && coherence.branchName !== branchInspection.workspaceRecord.branchName) {
|
|
9923
|
+
repairedExpectedBranchName = coherence.branchName;
|
|
9924
|
+
executionWorkspace.branchName = coherence.branchName;
|
|
9925
|
+
executionWorkspace.warnings.push(...coherence.warnings);
|
|
9926
|
+
}
|
|
8635
9927
|
}
|
|
8636
9928
|
catch (repairErr) {
|
|
8637
9929
|
const workspaceValidationFailure = isWorkspaceValidationFailure(repairErr) ? repairErr : null;
|
|
@@ -8668,7 +9960,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
8668
9960
|
}
|
|
8669
9961
|
const repairedInspection = await inspectManagedGitWorktreeBranch({
|
|
8670
9962
|
worktreePath: inspection.worktreePath,
|
|
8671
|
-
expectedBranchName:
|
|
9963
|
+
expectedBranchName: repairedExpectedBranchName,
|
|
8672
9964
|
repoRoot: inspection.repoRoot,
|
|
8673
9965
|
});
|
|
8674
9966
|
finalizeBranchRepairMetadata = {
|
|
@@ -8868,6 +10160,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
8868
10160
|
runId: run.id,
|
|
8869
10161
|
sessionId: nextSessionState.displayId ?? nextSessionState.legacySessionId,
|
|
8870
10162
|
rawUsage,
|
|
10163
|
+
usageBasis: adapterResult.usageBasis ?? null,
|
|
8871
10164
|
});
|
|
8872
10165
|
const normalizedUsage = sessionUsageResolution.normalizedUsage;
|
|
8873
10166
|
const runErrorMessage = outcome === "cancelled"
|
|
@@ -8875,12 +10168,13 @@ export function heartbeatService(db, options = {}) {
|
|
|
8875
10168
|
: outcome === "succeeded"
|
|
8876
10169
|
? null
|
|
8877
10170
|
: redactCurrentUserText(adapterResult.errorMessage ?? (outcome === "timed_out" ? "Timed out" : "Adapter failed"), currentUserRedactionOptions);
|
|
10171
|
+
const recordedResponsibleUserDenialCode = normalizeResponsibleUserDenialCode(latestRun?.errorCode);
|
|
8878
10172
|
const runErrorCode = outcome === "timed_out"
|
|
8879
10173
|
? "timeout"
|
|
8880
10174
|
: outcome === "cancelled"
|
|
8881
10175
|
? (latestRun?.errorCode ?? "cancelled")
|
|
8882
10176
|
: outcome === "failed"
|
|
8883
|
-
? (adapterResult.errorCode ?? "adapter_failed")
|
|
10177
|
+
? (adapterResult.errorCode ?? recordedResponsibleUserDenialCode ?? "adapter_failed")
|
|
8884
10178
|
: null;
|
|
8885
10179
|
let logSummary = null;
|
|
8886
10180
|
if (handle) {
|
|
@@ -8906,7 +10200,11 @@ export function heartbeatService(db, options = {}) {
|
|
|
8906
10200
|
rawCachedInputTokens: rawUsage.cachedInputTokens,
|
|
8907
10201
|
rawOutputTokens: rawUsage.outputTokens,
|
|
8908
10202
|
} : {}),
|
|
8909
|
-
...(sessionUsageResolution.derivedFromSessionTotals
|
|
10203
|
+
...(sessionUsageResolution.derivedFromSessionTotals
|
|
10204
|
+
? { usageSource: "session_delta" }
|
|
10205
|
+
: adapterResult.usageBasis === "per_run"
|
|
10206
|
+
? { usageSource: "per_run" }
|
|
10207
|
+
: {}),
|
|
8910
10208
|
...((nextSessionState.displayId ?? nextSessionState.legacySessionId)
|
|
8911
10209
|
? { persistedSessionId: nextSessionState.displayId ?? nextSessionState.legacySessionId }
|
|
8912
10210
|
: {}),
|
|
@@ -8920,6 +10218,12 @@ export function heartbeatService(db, options = {}) {
|
|
|
8920
10218
|
biller: resolveLedgerBiller(adapterResult),
|
|
8921
10219
|
model: readNonEmptyString(adapterResult.model) ?? "unknown",
|
|
8922
10220
|
...(adapterResult.costUsd != null ? { costUsd: adapterResult.costUsd } : {}),
|
|
10221
|
+
costStatus: resolveLedgerCostStatus({
|
|
10222
|
+
costUsd: adapterResult.costUsd,
|
|
10223
|
+
inputTokens: normalizedUsage?.inputTokens ?? 0,
|
|
10224
|
+
cachedInputTokens: normalizedUsage?.cachedInputTokens ?? 0,
|
|
10225
|
+
outputTokens: normalizedUsage?.outputTokens ?? 0,
|
|
10226
|
+
}),
|
|
8923
10227
|
billingType: normalizeLedgerBillingType(adapterResult.billingType),
|
|
8924
10228
|
}
|
|
8925
10229
|
: null;
|
|
@@ -8978,6 +10282,19 @@ export function heartbeatService(db, options = {}) {
|
|
|
8978
10282
|
exitCode: adapterResult.exitCode,
|
|
8979
10283
|
},
|
|
8980
10284
|
});
|
|
10285
|
+
try {
|
|
10286
|
+
await completeSkillTestRunForHeartbeatOutcome({
|
|
10287
|
+
run: finalizedRun,
|
|
10288
|
+
issueId,
|
|
10289
|
+
issueWorkMode: issueRef?.workMode ?? null,
|
|
10290
|
+
outcome,
|
|
10291
|
+
error: runErrorMessage,
|
|
10292
|
+
});
|
|
10293
|
+
}
|
|
10294
|
+
catch (err) {
|
|
10295
|
+
logger.warn({ err, runId: finalizedRun.id, issueId }, "failed to complete skill test run after heartbeat finalization");
|
|
10296
|
+
await onLog("stderr", `[paperclip] Failed to complete skill test run: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
10297
|
+
}
|
|
8981
10298
|
const livenessRun = finalizedRun;
|
|
8982
10299
|
await refreshContinuationSummaryForRun(livenessRun, agent);
|
|
8983
10300
|
const skipRunIssueComment = parseObject(livenessRun.contextSnapshot).skipIssueComment === true;
|
|
@@ -9032,11 +10349,9 @@ export function heartbeatService(db, options = {}) {
|
|
|
9032
10349
|
: livenessRun, agent);
|
|
9033
10350
|
// Dependency wake re-check: if this run's issue was marked done mid-run,
|
|
9034
10351
|
// the route-time `issue_blockers_resolved` wake may have been gated by
|
|
9035
|
-
// workspace finalization or merged into this run.
|
|
9036
|
-
//
|
|
9037
|
-
//
|
|
9038
|
-
// only returns dependents whose done blockers have crossed the successful
|
|
9039
|
-
// `workspace_finalize` barrier.
|
|
10352
|
+
// workspace finalization or merged into this run. Reuse the level-triggered
|
|
10353
|
+
// dependency backstop so finalize and periodic recovery share idempotency,
|
|
10354
|
+
// readiness, active-path, and observability rules.
|
|
9040
10355
|
if (issueId && finalizedRun) {
|
|
9041
10356
|
try {
|
|
9042
10357
|
const blockerIssueStatus = await db
|
|
@@ -9045,59 +10360,12 @@ export function heartbeatService(db, options = {}) {
|
|
|
9045
10360
|
.where(eq(issues.id, issueId))
|
|
9046
10361
|
.then((rows) => rows[0]?.status ?? null);
|
|
9047
10362
|
if (blockerIssueStatus === "done") {
|
|
9048
|
-
|
|
9049
|
-
|
|
9050
|
-
|
|
9051
|
-
|
|
9052
|
-
|
|
9053
|
-
|
|
9054
|
-
const existingWake = await findExistingIssueBlockersResolvedWake(db, {
|
|
9055
|
-
companyId: finalizedRun.companyId,
|
|
9056
|
-
idempotencyKey,
|
|
9057
|
-
});
|
|
9058
|
-
if (existingWake)
|
|
9059
|
-
continue;
|
|
9060
|
-
await enqueueWakeup(dependent.assigneeAgentId, {
|
|
9061
|
-
source: "automation",
|
|
9062
|
-
triggerDetail: "system",
|
|
9063
|
-
reason: ISSUE_BLOCKERS_RESOLVED_WAKE_REASON,
|
|
9064
|
-
payload: {
|
|
9065
|
-
issueId: dependent.id,
|
|
9066
|
-
resolvedBlockerIssueId: issueId,
|
|
9067
|
-
blockerIssueIds: dependent.blockerIssueIds,
|
|
9068
|
-
deferredFor: "workspace_finalize",
|
|
9069
|
-
},
|
|
9070
|
-
idempotencyKey,
|
|
9071
|
-
requestedByActorType: "system",
|
|
9072
|
-
requestedByActorId: "heartbeat_finalize",
|
|
9073
|
-
contextSnapshot: {
|
|
9074
|
-
issueId: dependent.id,
|
|
9075
|
-
taskId: dependent.id,
|
|
9076
|
-
wakeReason: ISSUE_BLOCKERS_RESOLVED_WAKE_REASON,
|
|
9077
|
-
source: "workspace.finalize",
|
|
9078
|
-
resolvedBlockerIssueId: issueId,
|
|
9079
|
-
blockerIssueIds: dependent.blockerIssueIds,
|
|
9080
|
-
},
|
|
9081
|
-
}).then((wakeRun) => logActivity(db, {
|
|
9082
|
-
companyId: finalizedRun.companyId,
|
|
9083
|
-
actorType: "system",
|
|
9084
|
-
actorId: "heartbeat_finalize",
|
|
9085
|
-
agentId: dependent.assigneeAgentId,
|
|
9086
|
-
runId: finalizedRun.id,
|
|
9087
|
-
action: "issue.blockers_resolved_wake_emitted",
|
|
9088
|
-
entityType: "issue",
|
|
9089
|
-
entityId: dependent.id,
|
|
9090
|
-
details: {
|
|
9091
|
-
source: "workspace.finalize",
|
|
9092
|
-
wakeupRunId: wakeRun?.id ?? null,
|
|
9093
|
-
idempotencyKey,
|
|
9094
|
-
resolvedBlockerIssueId: issueId,
|
|
9095
|
-
blockerIssueIds: dependent.blockerIssueIds,
|
|
9096
|
-
},
|
|
9097
|
-
})).catch((wakeErr) => {
|
|
9098
|
-
logger.warn({ err: wakeErr, issueId, dependentIssueId: dependent.id, agentId: dependent.assigneeAgentId }, "failed to fire deferred dependent wake after workspace_finalize");
|
|
9099
|
-
});
|
|
9100
|
-
}
|
|
10363
|
+
await recovery.reconcileResolvedDependencyWakeBackstop({
|
|
10364
|
+
runId: finalizedRun.id,
|
|
10365
|
+
companyId: finalizedRun.companyId,
|
|
10366
|
+
blockerIssueId: issueId,
|
|
10367
|
+
source: "workspace.finalize",
|
|
10368
|
+
});
|
|
9101
10369
|
}
|
|
9102
10370
|
}
|
|
9103
10371
|
catch (finalizeWakeErr) {
|
|
@@ -9130,13 +10398,20 @@ export function heartbeatService(db, options = {}) {
|
|
|
9130
10398
|
}
|
|
9131
10399
|
}
|
|
9132
10400
|
}
|
|
9133
|
-
await finalizeAgentStatus(agent.id, outcome, outcome === "succeeded" ? null : (adapterResult.errorMessage ?? null)
|
|
10401
|
+
await finalizeAgentStatus(agent.id, outcome, outcome === "succeeded" ? null : (adapterResult.errorMessage ?? null), {
|
|
10402
|
+
keepIdleOnFailure: outcome === "failed" &&
|
|
10403
|
+
(finalizedRun ? readHeartbeatRunErrorFamily(finalizedRun) === "provider_quota" : runErrorCode === "provider_quota"),
|
|
10404
|
+
});
|
|
9134
10405
|
}
|
|
9135
10406
|
catch (err) {
|
|
9136
10407
|
const message = redactCurrentUserText(err instanceof Error ? err.message : "Unknown adapter failure", await getCurrentUserRedactionOptions());
|
|
9137
10408
|
const workspaceValidationFailure = isWorkspaceValidationFailure(err) ? err : null;
|
|
9138
10409
|
const configurationIncompleteFailure = isConfigurationIncompleteFailure(err) ? err : null;
|
|
9139
|
-
const
|
|
10410
|
+
const recordedResponsibleUserDenialCode = normalizeResponsibleUserDenialCode((await getRun(run.id).catch(() => null))?.errorCode);
|
|
10411
|
+
const failureErrorCode = workspaceValidationFailure?.code
|
|
10412
|
+
?? configurationIncompleteFailure?.code
|
|
10413
|
+
?? recordedResponsibleUserDenialCode
|
|
10414
|
+
?? "adapter_failed";
|
|
9140
10415
|
logger.error({ err, runId }, "heartbeat execution failed");
|
|
9141
10416
|
let logSummary = null;
|
|
9142
10417
|
if (handle) {
|
|
@@ -9190,10 +10465,23 @@ export function heartbeatService(db, options = {}) {
|
|
|
9190
10465
|
message,
|
|
9191
10466
|
});
|
|
9192
10467
|
const livenessRun = await classifyAndPersistRunLiveness(failedRun) ?? failedRun;
|
|
10468
|
+
try {
|
|
10469
|
+
await completeSkillTestRunForHeartbeatOutcome({
|
|
10470
|
+
run: livenessRun,
|
|
10471
|
+
issueId,
|
|
10472
|
+
issueWorkMode: issueRef?.workMode ?? null,
|
|
10473
|
+
outcome: "failed",
|
|
10474
|
+
error: message,
|
|
10475
|
+
});
|
|
10476
|
+
}
|
|
10477
|
+
catch (err) {
|
|
10478
|
+
logger.warn({ err, runId: livenessRun.id, issueId }, "failed to complete skill test run after heartbeat adapter failure");
|
|
10479
|
+
}
|
|
9193
10480
|
await refreshContinuationSummaryForRun(livenessRun, agent);
|
|
9194
10481
|
if (!isWorkspaceValidationFailedRun(livenessRun) && !isConfigurationIncompleteFailedRun(livenessRun)) {
|
|
9195
10482
|
await finalizeIssueCommentPolicy(livenessRun, agent);
|
|
9196
10483
|
}
|
|
10484
|
+
await scheduleInteractionContinuationInfrastructureRetryIfEligible(livenessRun, agent);
|
|
9197
10485
|
await releaseIssueExecutionAndPromote(livenessRun);
|
|
9198
10486
|
await updateRuntimeState(agent, livenessRun, {
|
|
9199
10487
|
exitCode: null,
|
|
@@ -9228,7 +10516,11 @@ export function heartbeatService(db, options = {}) {
|
|
|
9228
10516
|
// recovery path routes it to a human owner instead of looping retries.
|
|
9229
10517
|
const workspaceValidationSetupFailure = isWorkspaceValidationFailure(outerErr) ? outerErr : null;
|
|
9230
10518
|
const configurationIncompleteSetupFailure = isConfigurationIncompleteFailure(outerErr) ? outerErr : null;
|
|
9231
|
-
const
|
|
10519
|
+
const recordedResponsibleUserDenialCode = normalizeResponsibleUserDenialCode((await getRun(runId).catch(() => null))?.errorCode);
|
|
10520
|
+
const setupFailureErrorCode = workspaceValidationSetupFailure?.code ??
|
|
10521
|
+
configurationIncompleteSetupFailure?.code ??
|
|
10522
|
+
recordedResponsibleUserDenialCode ??
|
|
10523
|
+
"setup_failed";
|
|
9232
10524
|
logger.error({ err: outerErr, runId }, "heartbeat execution setup failed");
|
|
9233
10525
|
const setupFailureAgent = await getAgent(run.agentId).catch(() => null);
|
|
9234
10526
|
const setupFailureWrite = await setRunStatusIfRunning(runId, "failed", {
|
|
@@ -9267,14 +10559,30 @@ export function heartbeatService(db, options = {}) {
|
|
|
9267
10559
|
message,
|
|
9268
10560
|
}).catch(() => undefined);
|
|
9269
10561
|
const livenessRun = await classifyAndPersistRunLiveness(failedRun).catch(() => failedRun);
|
|
10562
|
+
const setupFailureIssueId = readNonEmptyString(parseObject(livenessRun.contextSnapshot).issueId);
|
|
10563
|
+
if (setupFailureIssueId) {
|
|
10564
|
+
await completeSkillTestRunForHeartbeatOutcome({
|
|
10565
|
+
run: livenessRun,
|
|
10566
|
+
issueId: setupFailureIssueId,
|
|
10567
|
+
outcome: "failed",
|
|
10568
|
+
error: message,
|
|
10569
|
+
}).catch((completionErr) => {
|
|
10570
|
+
logger.warn({ err: completionErr, runId: livenessRun.id, issueId: setupFailureIssueId }, "failed to complete skill test run after heartbeat setup failure");
|
|
10571
|
+
});
|
|
10572
|
+
}
|
|
9270
10573
|
const failedAgent = setupFailureAgent ?? await getAgent(run.agentId).catch(() => null);
|
|
9271
10574
|
if (failedAgent) {
|
|
9272
10575
|
await refreshContinuationSummaryForRun(livenessRun, failedAgent).catch(() => undefined);
|
|
9273
10576
|
if (!isWorkspaceValidationFailedRun(livenessRun) && !isConfigurationIncompleteFailedRun(livenessRun)) {
|
|
9274
10577
|
await finalizeIssueCommentPolicy(livenessRun, failedAgent).catch(() => undefined);
|
|
9275
10578
|
}
|
|
10579
|
+
await scheduleInteractionContinuationInfrastructureRetryIfEligible(livenessRun, failedAgent).catch((retryError) => {
|
|
10580
|
+
logger.warn({ err: retryError, runId: livenessRun.id }, "failed to schedule interaction continuation retry after setup failure");
|
|
10581
|
+
});
|
|
9276
10582
|
}
|
|
9277
|
-
await releaseIssueExecutionAndPromote(livenessRun).catch(() =>
|
|
10583
|
+
await releaseIssueExecutionAndPromote(livenessRun).catch((releaseError) => {
|
|
10584
|
+
logger.error({ err: releaseError, runId }, "failed to release issue execution after heartbeat setup failure");
|
|
10585
|
+
});
|
|
9278
10586
|
}
|
|
9279
10587
|
// Ensure the agent is not left stuck in "running" if the setup-failure
|
|
9280
10588
|
// path owned the terminal transition. If another path already finalized
|
|
@@ -9293,6 +10601,53 @@ export function heartbeatService(db, options = {}) {
|
|
|
9293
10601
|
failureReason: latestRun?.error ?? undefined,
|
|
9294
10602
|
});
|
|
9295
10603
|
await releaseRuntimeServicesForRun(run.id).catch(() => undefined);
|
|
10604
|
+
if (runScratch && latestRun && isHeartbeatRunTerminalStatus(latestRun.status)) {
|
|
10605
|
+
const scratchForCleanup = runScratch;
|
|
10606
|
+
let scratchCleanup = null;
|
|
10607
|
+
try {
|
|
10608
|
+
scratchCleanup = await cleanupHeartbeatRunScratch({
|
|
10609
|
+
scratch: scratchForCleanup,
|
|
10610
|
+
processGroupId: latestRun.processGroupId,
|
|
10611
|
+
isProcessGroupAlive,
|
|
10612
|
+
});
|
|
10613
|
+
}
|
|
10614
|
+
catch (scratchCleanupError) {
|
|
10615
|
+
logger.warn({
|
|
10616
|
+
err: scratchCleanupError,
|
|
10617
|
+
runId: run.id,
|
|
10618
|
+
scratchDir: scratchForCleanup.dir,
|
|
10619
|
+
}, "failed to clean heartbeat run scratch directory");
|
|
10620
|
+
await appendRunEvent(latestRun, await nextRunEventSeq(latestRun.id), {
|
|
10621
|
+
eventType: "error",
|
|
10622
|
+
stream: "system",
|
|
10623
|
+
level: "warn",
|
|
10624
|
+
message: "run scratch cleanup failed",
|
|
10625
|
+
payload: {
|
|
10626
|
+
dir: scratchForCleanup.dir,
|
|
10627
|
+
error: scratchCleanupError instanceof Error
|
|
10628
|
+
? scratchCleanupError.message
|
|
10629
|
+
: String(scratchCleanupError),
|
|
10630
|
+
},
|
|
10631
|
+
}).catch(() => undefined);
|
|
10632
|
+
}
|
|
10633
|
+
if (scratchCleanup) {
|
|
10634
|
+
await appendRunEvent(latestRun, await nextRunEventSeq(latestRun.id), {
|
|
10635
|
+
eventType: "lifecycle",
|
|
10636
|
+
stream: "system",
|
|
10637
|
+
level: scratchCleanup.removed ? "info" : "warn",
|
|
10638
|
+
message: scratchCleanup.removed
|
|
10639
|
+
? "run scratch cleaned"
|
|
10640
|
+
: `run scratch cleanup skipped: ${scratchCleanup.reason}`,
|
|
10641
|
+
payload: scratchCleanup,
|
|
10642
|
+
}).catch((scratchCleanupEventError) => {
|
|
10643
|
+
logger.warn({
|
|
10644
|
+
err: scratchCleanupEventError,
|
|
10645
|
+
runId: run.id,
|
|
10646
|
+
scratchDir: scratchForCleanup.dir,
|
|
10647
|
+
}, "failed to record heartbeat run scratch cleanup event");
|
|
10648
|
+
});
|
|
10649
|
+
}
|
|
10650
|
+
}
|
|
9296
10651
|
activeRunExecutions.delete(run.id);
|
|
9297
10652
|
await startNextQueuedRunForAgent(run.agentId);
|
|
9298
10653
|
}
|
|
@@ -9595,6 +10950,27 @@ export function heartbeatService(db, options = {}) {
|
|
|
9595
10950
|
const sessionBefore = readNonEmptyString(promotedContextSnapshot.resumeSessionDisplayId) ??
|
|
9596
10951
|
await resolveSessionBeforeForWakeup(deferredAgent, promotedTaskKey);
|
|
9597
10952
|
const promotedContinuationAttempt = readContinuationAttempt(promotedContextSnapshot.livenessContinuationAttempt);
|
|
10953
|
+
const promotedResponsibleUserId = await resolveResponsibleUserIdForRunSeed({
|
|
10954
|
+
companyId: deferredAgent.companyId,
|
|
10955
|
+
contextSnapshot: promotedContextSnapshot,
|
|
10956
|
+
issueContext: issue,
|
|
10957
|
+
routineEnvContext: await getRoutineEnvForExecutionIssue(deferredAgent.companyId, issue),
|
|
10958
|
+
requestedByActorType: deferred.requestedByActorType,
|
|
10959
|
+
requestedByActorId: deferred.requestedByActorId,
|
|
10960
|
+
source: promotedSource,
|
|
10961
|
+
triggerDetail: promotedTriggerDetail,
|
|
10962
|
+
existingRunResponsibleUserId: run.responsibleUserId,
|
|
10963
|
+
});
|
|
10964
|
+
if (!promotedResponsibleUserId) {
|
|
10965
|
+
throw new HttpError(422, "Unable to resolve responsible user for promoted heartbeat run", {
|
|
10966
|
+
code: "responsible_user_unresolved",
|
|
10967
|
+
runId: run.id,
|
|
10968
|
+
agentId: deferredAgent.id,
|
|
10969
|
+
companyId: deferredAgent.companyId,
|
|
10970
|
+
issueId: issue.id,
|
|
10971
|
+
wakeReason: readNonEmptyString(promotedContextSnapshot.wakeReason),
|
|
10972
|
+
});
|
|
10973
|
+
}
|
|
9598
10974
|
const now = new Date();
|
|
9599
10975
|
const newRun = await tx
|
|
9600
10976
|
.insert(heartbeatRuns)
|
|
@@ -9606,6 +10982,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
9606
10982
|
status: "queued",
|
|
9607
10983
|
wakeupRequestId: deferred.id,
|
|
9608
10984
|
contextSnapshot: promotedContextSnapshot,
|
|
10985
|
+
responsibleUserId: promotedResponsibleUserId,
|
|
9609
10986
|
sessionIdBefore: sessionBefore,
|
|
9610
10987
|
continuationAttempt: promotedContinuationAttempt,
|
|
9611
10988
|
})
|
|
@@ -9645,8 +11022,14 @@ export function heartbeatService(db, options = {}) {
|
|
|
9645
11022
|
.where(and(eq(heartbeatRuns.companyId, issue.companyId), inArray(heartbeatRuns.status, [...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES]), sql `${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`, sql `${heartbeatRuns.id} <> ${run.id}`, agentId ? eq(heartbeatRuns.agentId, agentId) : sql `true`))
|
|
9646
11023
|
.limit(1)
|
|
9647
11024
|
.then((rows) => rows[0] ?? null);
|
|
9648
|
-
const
|
|
9649
|
-
|
|
11025
|
+
const issueHasPersistedMonitor = Boolean(issue.monitorNextCheckAt);
|
|
11026
|
+
const findExplicitBlockerPath = () => tx
|
|
11027
|
+
.select({ id: issueRelations.issueId })
|
|
11028
|
+
.from(issueRelations)
|
|
11029
|
+
.innerJoin(issues, eq(issueRelations.issueId, issues.id))
|
|
11030
|
+
.where(and(eq(issueRelations.companyId, issue.companyId), eq(issueRelations.relatedIssueId, issue.id), eq(issueRelations.type, "blocks"), eq(issues.companyId, issue.companyId), notInArray(issues.status, ["done", "cancelled"]), isNull(issues.hiddenAt)))
|
|
11031
|
+
.limit(1)
|
|
11032
|
+
.then((rows) => rows[0] ?? null);
|
|
9650
11033
|
const executionState = parseIssueExecutionState(issue.executionState);
|
|
9651
11034
|
const currentParticipant = executionState?.status === "pending"
|
|
9652
11035
|
? executionState.currentParticipant
|
|
@@ -9661,7 +11044,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
9661
11044
|
const existingReviewParticipantExecutionPath = await findExistingExecutionPath(currentParticipant.agentId);
|
|
9662
11045
|
if (options.suppressImmediateRecovery ||
|
|
9663
11046
|
existingReviewParticipantExecutionPath ||
|
|
9664
|
-
|
|
11047
|
+
issueHasPersistedMonitor ||
|
|
9665
11048
|
await isAutomaticRecoverySuppressedByPauseHold(db, issue.companyId, issue.id, treeControlSvc)) {
|
|
9666
11049
|
return { kind: "released" };
|
|
9667
11050
|
}
|
|
@@ -9766,7 +11149,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
9766
11149
|
return { kind: "released" };
|
|
9767
11150
|
}
|
|
9768
11151
|
const existingExecutionPath = await findExistingExecutionPath();
|
|
9769
|
-
if (existingExecutionPath) {
|
|
11152
|
+
if (existingExecutionPath || issueHasPersistedMonitor || await findExplicitBlockerPath()) {
|
|
9770
11153
|
return { kind: "released" };
|
|
9771
11154
|
}
|
|
9772
11155
|
if (await isAutomaticRecoverySuppressedByPauseHold(db, issue.companyId, issue.id, treeControlSvc)) {
|
|
@@ -9811,6 +11194,35 @@ export function heartbeatService(db, options = {}) {
|
|
|
9811
11194
|
const recoveryReason = issue.status === "todo" ? "issue_assignment_recovery" : "issue_continuation_needed";
|
|
9812
11195
|
const recoverySource = issue.status === "todo" ? "issue.assignment_recovery" : "issue.continuation_recovery";
|
|
9813
11196
|
const now = new Date();
|
|
11197
|
+
const recoveryContextSnapshot = withRecoveryModelProfileHint({
|
|
11198
|
+
issueId: issue.id,
|
|
11199
|
+
taskId: issue.id,
|
|
11200
|
+
wakeReason: recoveryReason,
|
|
11201
|
+
retryReason,
|
|
11202
|
+
source: recoverySource,
|
|
11203
|
+
retryOfRunId: run.id,
|
|
11204
|
+
}, "normal_model");
|
|
11205
|
+
const responsibleUserId = await resolveResponsibleUserIdForRunSeed({
|
|
11206
|
+
companyId: issue.companyId,
|
|
11207
|
+
contextSnapshot: recoveryContextSnapshot,
|
|
11208
|
+
issueContext: issue,
|
|
11209
|
+
routineEnvContext: await getRoutineEnvForExecutionIssue(issue.companyId, issue),
|
|
11210
|
+
requestedByActorType: "system",
|
|
11211
|
+
requestedByActorId: null,
|
|
11212
|
+
source: "automation",
|
|
11213
|
+
triggerDetail: "system",
|
|
11214
|
+
existingRunResponsibleUserId: run.responsibleUserId,
|
|
11215
|
+
});
|
|
11216
|
+
if (!responsibleUserId) {
|
|
11217
|
+
throw new HttpError(422, "Unable to resolve responsible user for recovery heartbeat run", {
|
|
11218
|
+
code: "responsible_user_unresolved",
|
|
11219
|
+
runId: run.id,
|
|
11220
|
+
agentId: recoveryAgent.id,
|
|
11221
|
+
companyId: issue.companyId,
|
|
11222
|
+
issueId: issue.id,
|
|
11223
|
+
wakeReason: recoveryReason,
|
|
11224
|
+
});
|
|
11225
|
+
}
|
|
9814
11226
|
const wakeupRequest = await tx
|
|
9815
11227
|
.insert(agentWakeupRequests)
|
|
9816
11228
|
.values({
|
|
@@ -9839,14 +11251,8 @@ export function heartbeatService(db, options = {}) {
|
|
|
9839
11251
|
triggerDetail: "system",
|
|
9840
11252
|
status: "queued",
|
|
9841
11253
|
wakeupRequestId: wakeupRequest.id,
|
|
9842
|
-
contextSnapshot:
|
|
9843
|
-
|
|
9844
|
-
taskId: issue.id,
|
|
9845
|
-
wakeReason: recoveryReason,
|
|
9846
|
-
retryReason,
|
|
9847
|
-
source: recoverySource,
|
|
9848
|
-
retryOfRunId: run.id,
|
|
9849
|
-
}, "normal_model"),
|
|
11254
|
+
contextSnapshot: recoveryContextSnapshot,
|
|
11255
|
+
responsibleUserId,
|
|
9850
11256
|
sessionIdBefore: recoverySessionBefore,
|
|
9851
11257
|
retryOfRunId: run.id,
|
|
9852
11258
|
updatedAt: now,
|
|
@@ -9961,6 +11367,16 @@ export function heartbeatService(db, options = {}) {
|
|
|
9961
11367
|
},
|
|
9962
11368
|
});
|
|
9963
11369
|
};
|
|
11370
|
+
const schedulingSuppression = await getSchedulingSuppression();
|
|
11371
|
+
if (schedulingSuppression.suppressed) {
|
|
11372
|
+
await writeSkippedHeartbeatRequest("heartbeat.scheduling_suppressed", {
|
|
11373
|
+
reason: schedulingSuppression.reason,
|
|
11374
|
+
});
|
|
11375
|
+
return null;
|
|
11376
|
+
}
|
|
11377
|
+
const worktreeExecutionCutoff = opts.requestedByActorType === "user"
|
|
11378
|
+
? null
|
|
11379
|
+
: await getWorktreeExecutionCutoff();
|
|
9964
11380
|
const company = await db
|
|
9965
11381
|
.select({ status: companies.status })
|
|
9966
11382
|
.from(companies)
|
|
@@ -9995,6 +11411,20 @@ export function heartbeatService(db, options = {}) {
|
|
|
9995
11411
|
const effectiveTaskKey = readNonEmptyString(enrichedContextSnapshot.taskKey) ?? taskKey;
|
|
9996
11412
|
const sessionBefore = explicitResumeSession?.sessionDisplayId ??
|
|
9997
11413
|
await resolveSessionBeforeForWakeup(agent, effectiveTaskKey);
|
|
11414
|
+
let hasResolvablePriorSessionWorkspace = null;
|
|
11415
|
+
const resolveHasResolvablePriorSessionWorkspace = async () => {
|
|
11416
|
+
if (hasResolvablePriorSessionWorkspace !== null)
|
|
11417
|
+
return hasResolvablePriorSessionWorkspace;
|
|
11418
|
+
hasResolvablePriorSessionWorkspace = issueId
|
|
11419
|
+
? await hasResolvablePriorSessionWorkspaceForWake({
|
|
11420
|
+
agent,
|
|
11421
|
+
contextSnapshot: enrichedContextSnapshot,
|
|
11422
|
+
taskKey: effectiveTaskKey,
|
|
11423
|
+
explicitResumeSession,
|
|
11424
|
+
})
|
|
11425
|
+
: false;
|
|
11426
|
+
return hasResolvablePriorSessionWorkspace;
|
|
11427
|
+
};
|
|
9998
11428
|
const continuationAttempt = readContinuationAttempt(enrichedContextSnapshot.livenessContinuationAttempt);
|
|
9999
11429
|
let projectId = readNonEmptyString(enrichedContextSnapshot.projectId);
|
|
10000
11430
|
if (!projectId && issueId) {
|
|
@@ -10009,11 +11439,19 @@ export function heartbeatService(db, options = {}) {
|
|
|
10009
11439
|
? or(eq(issues.id, issueId), eq(issues.identifier, issueId.toUpperCase()))
|
|
10010
11440
|
: eq(issues.identifier, issueId.toUpperCase());
|
|
10011
11441
|
const resolvedIssue = await db
|
|
10012
|
-
.select({ id: issues.id, projectId: issues.projectId })
|
|
11442
|
+
.select({ id: issues.id, projectId: issues.projectId, createdAt: issues.createdAt })
|
|
10013
11443
|
.from(issues)
|
|
10014
11444
|
.where(and(eq(issues.companyId, agent.companyId), idMatch))
|
|
10015
11445
|
.then((rows) => rows[0] ?? null);
|
|
10016
11446
|
if (resolvedIssue) {
|
|
11447
|
+
if (worktreeExecutionCutoff && resolvedIssue.createdAt < worktreeExecutionCutoff) {
|
|
11448
|
+
await writeSkippedHeartbeatRequest("heartbeat.worktree_execution_cutoff", {
|
|
11449
|
+
reason: "worktree_execution_cutoff",
|
|
11450
|
+
cutoff: worktreeExecutionCutoff.toISOString(),
|
|
11451
|
+
issueId: resolvedIssue.id,
|
|
11452
|
+
});
|
|
11453
|
+
return null;
|
|
11454
|
+
}
|
|
10017
11455
|
projectId = resolvedIssue.projectId ?? null;
|
|
10018
11456
|
// Canonicalize context to the UUID so downstream lookups always use UUID
|
|
10019
11457
|
if (resolvedIssue.id !== issueId) {
|
|
@@ -10030,6 +11468,39 @@ export function heartbeatService(db, options = {}) {
|
|
|
10030
11468
|
if (projectId && !readNonEmptyString(enrichedContextSnapshot.projectId)) {
|
|
10031
11469
|
enrichedContextSnapshot.projectId = projectId;
|
|
10032
11470
|
}
|
|
11471
|
+
const isolatedWorkspacesEnabled = issueId
|
|
11472
|
+
? (await instanceSettings.getExperimental()).enableIsolatedWorkspaces
|
|
11473
|
+
: false;
|
|
11474
|
+
let queuedResponsibleUserIdPromise = null;
|
|
11475
|
+
const resolveQueuedResponsibleUserId = () => {
|
|
11476
|
+
queuedResponsibleUserIdPromise ??= (async () => {
|
|
11477
|
+
const queuedIssueContext = issueId ? await getIssueExecutionContext(agent.companyId, issueId) : null;
|
|
11478
|
+
const queuedRoutineEnvContext = await getRoutineEnvForExecutionIssue(agent.companyId, queuedIssueContext);
|
|
11479
|
+
const queuedResponsibleUserId = await resolveResponsibleUserIdForRunSeed({
|
|
11480
|
+
companyId: agent.companyId,
|
|
11481
|
+
contextSnapshot: enrichedContextSnapshot,
|
|
11482
|
+
issueContext: queuedIssueContext,
|
|
11483
|
+
routineEnvContext: queuedRoutineEnvContext,
|
|
11484
|
+
requestedByActorType: opts.requestedByActorType ?? null,
|
|
11485
|
+
requestedByActorId: opts.requestedByActorId ?? null,
|
|
11486
|
+
source,
|
|
11487
|
+
triggerDetail,
|
|
11488
|
+
});
|
|
11489
|
+
if (!queuedResponsibleUserId) {
|
|
11490
|
+
throw new HttpError(422, "Unable to resolve responsible user for heartbeat run dispatch", {
|
|
11491
|
+
code: "responsible_user_unresolved",
|
|
11492
|
+
agentId,
|
|
11493
|
+
companyId: agent.companyId,
|
|
11494
|
+
issueId: issueId ?? null,
|
|
11495
|
+
source,
|
|
11496
|
+
triggerDetail,
|
|
11497
|
+
wakeReason: readNonEmptyString(enrichedContextSnapshot.wakeReason),
|
|
11498
|
+
});
|
|
11499
|
+
}
|
|
11500
|
+
return queuedResponsibleUserId;
|
|
11501
|
+
})();
|
|
11502
|
+
return queuedResponsibleUserIdPromise;
|
|
11503
|
+
};
|
|
10033
11504
|
const budgetBlock = await budgets.getInvocationBlock(agent.companyId, agentId, {
|
|
10034
11505
|
issueId,
|
|
10035
11506
|
projectId,
|
|
@@ -10131,10 +11602,17 @@ export function heartbeatService(db, options = {}) {
|
|
|
10131
11602
|
.select({
|
|
10132
11603
|
id: issues.id,
|
|
10133
11604
|
companyId: issues.companyId,
|
|
11605
|
+
identifier: issues.identifier,
|
|
10134
11606
|
status: issues.status,
|
|
11607
|
+
projectId: issues.projectId,
|
|
11608
|
+
projectWorkspaceId: issues.projectWorkspaceId,
|
|
11609
|
+
executionWorkspaceId: issues.executionWorkspaceId,
|
|
11610
|
+
executionWorkspacePreference: issues.executionWorkspacePreference,
|
|
11611
|
+
executionWorkspaceSettings: issues.executionWorkspaceSettings,
|
|
10135
11612
|
assigneeAgentId: issues.assigneeAgentId,
|
|
10136
11613
|
executionRunId: issues.executionRunId,
|
|
10137
11614
|
executionAgentNameKey: issues.executionAgentNameKey,
|
|
11615
|
+
createdAt: issues.createdAt,
|
|
10138
11616
|
})
|
|
10139
11617
|
.from(issues)
|
|
10140
11618
|
.where(and(eq(issues.id, issueId), eq(issues.companyId, agent.companyId)))
|
|
@@ -10155,6 +11633,29 @@ export function heartbeatService(db, options = {}) {
|
|
|
10155
11633
|
});
|
|
10156
11634
|
return { kind: "skipped" };
|
|
10157
11635
|
}
|
|
11636
|
+
if (worktreeExecutionCutoff && issue.createdAt < worktreeExecutionCutoff) {
|
|
11637
|
+
await tx.insert(agentWakeupRequests).values({
|
|
11638
|
+
companyId: agent.companyId,
|
|
11639
|
+
agentId,
|
|
11640
|
+
source,
|
|
11641
|
+
triggerDetail,
|
|
11642
|
+
reason: "heartbeat.worktree_execution_cutoff",
|
|
11643
|
+
payload: {
|
|
11644
|
+
...(payload ?? {}),
|
|
11645
|
+
heartbeatSkip: {
|
|
11646
|
+
reason: "worktree_execution_cutoff",
|
|
11647
|
+
cutoff: worktreeExecutionCutoff.toISOString(),
|
|
11648
|
+
issueId: issue.id,
|
|
11649
|
+
},
|
|
11650
|
+
},
|
|
11651
|
+
status: "skipped",
|
|
11652
|
+
requestedByActorType: opts.requestedByActorType ?? null,
|
|
11653
|
+
requestedByActorId: opts.requestedByActorId ?? null,
|
|
11654
|
+
idempotencyKey: opts.idempotencyKey ?? null,
|
|
11655
|
+
finishedAt: new Date(),
|
|
11656
|
+
});
|
|
11657
|
+
return { kind: "skipped" };
|
|
11658
|
+
}
|
|
10158
11659
|
const cancelStaleScheduledRetry = async (scheduledRun) => {
|
|
10159
11660
|
const issueCancelled = issue.status === "cancelled";
|
|
10160
11661
|
if (scheduledRun.status !== "scheduled_retry" ||
|
|
@@ -10362,6 +11863,118 @@ export function heartbeatService(db, options = {}) {
|
|
|
10362
11863
|
});
|
|
10363
11864
|
return { kind: "skipped" };
|
|
10364
11865
|
}
|
|
11866
|
+
if (isolatedWorkspacesEnabled && !activeExecutionRun && issue.status !== "done" && issue.status !== "cancelled") {
|
|
11867
|
+
const issueSettings = parseIssueExecutionWorkspaceSettings(issue.executionWorkspaceSettings);
|
|
11868
|
+
const resolvedMode = resolveExecutionWorkspaceMode({
|
|
11869
|
+
projectPolicy: null,
|
|
11870
|
+
issueSettings,
|
|
11871
|
+
legacyUseProjectWorkspace: null,
|
|
11872
|
+
});
|
|
11873
|
+
const workspaceManagedConfig = buildExecutionWorkspaceAdapterConfig({
|
|
11874
|
+
agentConfig: parseObject(agent.adapterConfig),
|
|
11875
|
+
projectPolicy: null,
|
|
11876
|
+
issueSettings,
|
|
11877
|
+
mode: resolvedMode,
|
|
11878
|
+
legacyUseProjectWorkspace: null,
|
|
11879
|
+
});
|
|
11880
|
+
const resolvedStrategy = resolveEffectiveWorkspaceStrategyType(resolvedMode, workspaceManagedConfig);
|
|
11881
|
+
const existingExecutionWorkspaceStatus = issue.executionWorkspaceId
|
|
11882
|
+
? await tx
|
|
11883
|
+
.select({ status: executionWorkspaces.status })
|
|
11884
|
+
.from(executionWorkspaces)
|
|
11885
|
+
.where(and(eq(executionWorkspaces.id, issue.executionWorkspaceId), eq(executionWorkspaces.companyId, issue.companyId)))
|
|
11886
|
+
.then((rows) => rows[0]?.status ?? null)
|
|
11887
|
+
: null;
|
|
11888
|
+
const reuseRequest = resolveExecutionWorkspaceReuseRequestForIssue({
|
|
11889
|
+
issueExecutionWorkspaceId: issue.executionWorkspaceId,
|
|
11890
|
+
issueExecutionWorkspacePreference: issue.executionWorkspacePreference,
|
|
11891
|
+
existingExecutionWorkspaceStatus,
|
|
11892
|
+
});
|
|
11893
|
+
const hasResolvablePriorSessionWorkspace = await resolveHasResolvablePriorSessionWorkspace();
|
|
11894
|
+
if (isUnrunnableWorktreeCombo({
|
|
11895
|
+
issue: {
|
|
11896
|
+
projectId: issue.projectId ?? projectId ?? null,
|
|
11897
|
+
projectWorkspaceId: issue.projectWorkspaceId,
|
|
11898
|
+
executionWorkspaceId: issue.executionWorkspaceId,
|
|
11899
|
+
executionWorkspacePreference: issue.executionWorkspacePreference,
|
|
11900
|
+
},
|
|
11901
|
+
resolvedMode,
|
|
11902
|
+
resolvedStrategy,
|
|
11903
|
+
reusableExecutionWorkspaceAvailable: reuseRequest.existingExecutionWorkspaceAvailable,
|
|
11904
|
+
hasResolvablePriorSessionWorkspace,
|
|
11905
|
+
})) {
|
|
11906
|
+
const now = new Date();
|
|
11907
|
+
const issueLabel = formatIssueIdentifierLink(issue.identifier, issue.id);
|
|
11908
|
+
const blockedComment = [
|
|
11909
|
+
`Paperclip blocked ${issueLabel} before dispatch because its workspace settings are not runnable.`,
|
|
11910
|
+
"",
|
|
11911
|
+
`- Code: \`${WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE}\``,
|
|
11912
|
+
`- Reason: ${WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE}`,
|
|
11913
|
+
`- Next action: ${WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION}`,
|
|
11914
|
+
].join("\n");
|
|
11915
|
+
await tx
|
|
11916
|
+
.update(issues)
|
|
11917
|
+
.set({
|
|
11918
|
+
status: "blocked",
|
|
11919
|
+
checkoutRunId: null,
|
|
11920
|
+
executionRunId: null,
|
|
11921
|
+
executionAgentNameKey: null,
|
|
11922
|
+
executionLockedAt: null,
|
|
11923
|
+
updatedAt: now,
|
|
11924
|
+
})
|
|
11925
|
+
.where(eq(issues.id, issue.id));
|
|
11926
|
+
await tx.insert(issueComments).values({
|
|
11927
|
+
companyId: issue.companyId,
|
|
11928
|
+
issueId: issue.id,
|
|
11929
|
+
body: blockedComment,
|
|
11930
|
+
createdAt: now,
|
|
11931
|
+
updatedAt: now,
|
|
11932
|
+
});
|
|
11933
|
+
await tx.insert(agentWakeupRequests).values({
|
|
11934
|
+
companyId: agent.companyId,
|
|
11935
|
+
agentId,
|
|
11936
|
+
source,
|
|
11937
|
+
triggerDetail,
|
|
11938
|
+
reason: WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE,
|
|
11939
|
+
payload: {
|
|
11940
|
+
...(payload ?? {}),
|
|
11941
|
+
issueId,
|
|
11942
|
+
heartbeatSkip: {
|
|
11943
|
+
code: WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE,
|
|
11944
|
+
reason: WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE,
|
|
11945
|
+
remediation: WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION,
|
|
11946
|
+
},
|
|
11947
|
+
},
|
|
11948
|
+
status: "skipped",
|
|
11949
|
+
requestedByActorType: opts.requestedByActorType ?? null,
|
|
11950
|
+
requestedByActorId: opts.requestedByActorId ?? null,
|
|
11951
|
+
idempotencyKey: opts.idempotencyKey ?? null,
|
|
11952
|
+
finishedAt: now,
|
|
11953
|
+
});
|
|
11954
|
+
await logActivity(tx, {
|
|
11955
|
+
companyId: issue.companyId,
|
|
11956
|
+
actorType: "system",
|
|
11957
|
+
actorId: "system",
|
|
11958
|
+
agentId,
|
|
11959
|
+
runId: null,
|
|
11960
|
+
action: "issue.workspace_preflight_blocked",
|
|
11961
|
+
entityType: "issue",
|
|
11962
|
+
entityId: issue.id,
|
|
11963
|
+
details: {
|
|
11964
|
+
code: WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE,
|
|
11965
|
+
reason: WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE,
|
|
11966
|
+
remediation: WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION,
|
|
11967
|
+
requestedReason: reason,
|
|
11968
|
+
source,
|
|
11969
|
+
triggerDetail,
|
|
11970
|
+
resolvedMode,
|
|
11971
|
+
resolvedStrategy,
|
|
11972
|
+
hasResolvablePriorSessionWorkspace,
|
|
11973
|
+
},
|
|
11974
|
+
});
|
|
11975
|
+
return { kind: "skipped" };
|
|
11976
|
+
}
|
|
11977
|
+
}
|
|
10365
11978
|
if (activeExecutionRun) {
|
|
10366
11979
|
const executionAgent = await tx
|
|
10367
11980
|
.select({ name: agents.name })
|
|
@@ -10469,6 +12082,81 @@ export function heartbeatService(db, options = {}) {
|
|
|
10469
12082
|
return { kind: "deferred" };
|
|
10470
12083
|
}
|
|
10471
12084
|
}
|
|
12085
|
+
// PAP-13775: no live run holds the lock, so this wake would start a
|
|
12086
|
+
// fresh adapter session. If this agent's recent runs on this issue
|
|
12087
|
+
// keep succeeding without any issue-visible progress and the wake
|
|
12088
|
+
// carries no new information, hold it back for an escalating cooldown
|
|
12089
|
+
// so external pollers/reconcilers can't storm full-price sessions.
|
|
12090
|
+
// Server-side recovery retries insert runs directly and never reach
|
|
12091
|
+
// this gate.
|
|
12092
|
+
if (isThrottleCandidateIssueRewake({
|
|
12093
|
+
reason,
|
|
12094
|
+
wakeCommentId: wakeCommentId ?? null,
|
|
12095
|
+
forceFreshSession: enrichedContextSnapshot.forceFreshSession === true,
|
|
12096
|
+
hasExplicitResume: Boolean(explicitResumeSession),
|
|
12097
|
+
})) {
|
|
12098
|
+
const throttleNow = new Date();
|
|
12099
|
+
const recentTerminalRuns = await tx
|
|
12100
|
+
.select({
|
|
12101
|
+
id: heartbeatRuns.id,
|
|
12102
|
+
status: heartbeatRuns.status,
|
|
12103
|
+
finishedAt: heartbeatRuns.finishedAt,
|
|
12104
|
+
})
|
|
12105
|
+
.from(heartbeatRuns)
|
|
12106
|
+
.where(and(eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agentId), sql `${heartbeatRuns.finishedAt} is not null`, gte(heartbeatRuns.finishedAt, new Date(throttleNow.getTime() - ISSUE_REWAKE_LOOKBACK_MS)), sql `${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`))
|
|
12107
|
+
.orderBy(desc(heartbeatRuns.finishedAt))
|
|
12108
|
+
.limit(ISSUE_REWAKE_RUN_SAMPLE_LIMIT);
|
|
12109
|
+
if (recentTerminalRuns.length > 0) {
|
|
12110
|
+
const sampleRunIds = recentTerminalRuns.map((sampleRun) => sampleRun.id);
|
|
12111
|
+
const progressRows = await tx
|
|
12112
|
+
.select({ runId: activityLog.runId })
|
|
12113
|
+
.from(activityLog)
|
|
12114
|
+
.where(and(eq(activityLog.companyId, agent.companyId), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issue.id), inArray(activityLog.runId, sampleRunIds), inArray(activityLog.action, ISSUE_PROGRESS_ACTIVITY_ACTIONS)));
|
|
12115
|
+
const lastRunFinishedAt = recentTerminalRuns[0]?.finishedAt ?? null;
|
|
12116
|
+
const newInputRows = lastRunFinishedAt
|
|
12117
|
+
? await tx
|
|
12118
|
+
.select({ id: activityLog.id })
|
|
12119
|
+
.from(activityLog)
|
|
12120
|
+
.where(and(eq(activityLog.companyId, agent.companyId), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issue.id), gt(activityLog.createdAt, lastRunFinishedAt), inArray(activityLog.action, ISSUE_NEW_INPUT_ACTIVITY_ACTIONS)))
|
|
12121
|
+
.limit(1)
|
|
12122
|
+
: [];
|
|
12123
|
+
const throttleDecision = evaluateIssueRewakeThrottle({
|
|
12124
|
+
now: throttleNow,
|
|
12125
|
+
recentTerminalRuns,
|
|
12126
|
+
runIdsWithIssueProgress: new Set(progressRows
|
|
12127
|
+
.map((row) => row.runId)
|
|
12128
|
+
.filter((runId) => Boolean(runId))),
|
|
12129
|
+
hasNewIssueInputSinceLastRun: newInputRows.length > 0,
|
|
12130
|
+
});
|
|
12131
|
+
if (throttleDecision.blocked) {
|
|
12132
|
+
await tx.insert(agentWakeupRequests).values({
|
|
12133
|
+
companyId: agent.companyId,
|
|
12134
|
+
agentId,
|
|
12135
|
+
source,
|
|
12136
|
+
triggerDetail,
|
|
12137
|
+
reason: "issue_rewake_throttled",
|
|
12138
|
+
payload: {
|
|
12139
|
+
...(payload ?? {}),
|
|
12140
|
+
issueId,
|
|
12141
|
+
heartbeatSkip: {
|
|
12142
|
+
reason: "issue_rewake_throttled",
|
|
12143
|
+
requestedReason: reason,
|
|
12144
|
+
noProgressStreak: throttleDecision.noProgressStreak,
|
|
12145
|
+
cooldownMs: throttleDecision.cooldownMs,
|
|
12146
|
+
lastRunFinishedAt: throttleDecision.lastRunFinishedAt.toISOString(),
|
|
12147
|
+
nextAllowedAt: throttleDecision.nextAllowedAt.toISOString(),
|
|
12148
|
+
},
|
|
12149
|
+
},
|
|
12150
|
+
status: "skipped",
|
|
12151
|
+
requestedByActorType: opts.requestedByActorType ?? null,
|
|
12152
|
+
requestedByActorId: opts.requestedByActorId ?? null,
|
|
12153
|
+
idempotencyKey: opts.idempotencyKey ?? null,
|
|
12154
|
+
finishedAt: throttleNow,
|
|
12155
|
+
});
|
|
12156
|
+
return { kind: "skipped" };
|
|
12157
|
+
}
|
|
12158
|
+
}
|
|
12159
|
+
}
|
|
10472
12160
|
const dailyCapBlock = await getHeartbeatDailyCapBlock(agent, policy, {}, tx);
|
|
10473
12161
|
if (dailyCapBlock) {
|
|
10474
12162
|
const now = new Date();
|
|
@@ -10527,6 +12215,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
10527
12215
|
invocationSource: source,
|
|
10528
12216
|
triggerDetail,
|
|
10529
12217
|
status: "queued",
|
|
12218
|
+
responsibleUserId: await resolveQueuedResponsibleUserId(),
|
|
10530
12219
|
wakeupRequestId: wakeupRequest.id,
|
|
10531
12220
|
contextSnapshot: enrichedContextSnapshot,
|
|
10532
12221
|
sessionIdBefore: sessionBefore,
|
|
@@ -10679,6 +12368,7 @@ export function heartbeatService(db, options = {}) {
|
|
|
10679
12368
|
invocationSource: source,
|
|
10680
12369
|
triggerDetail,
|
|
10681
12370
|
status: "queued",
|
|
12371
|
+
responsibleUserId: await resolveQueuedResponsibleUserId(),
|
|
10682
12372
|
wakeupRequestId: wakeupRequest.id,
|
|
10683
12373
|
contextSnapshot: enrichedContextSnapshot,
|
|
10684
12374
|
sessionIdBefore: sessionBefore,
|
|
@@ -10924,6 +12614,17 @@ export function heartbeatService(db, options = {}) {
|
|
|
10924
12614
|
await cancelPendingWakeupsForBudgetScope(scope);
|
|
10925
12615
|
}
|
|
10926
12616
|
return {
|
|
12617
|
+
waitForRunExecutionDrain: async (runId, options = {}) => {
|
|
12618
|
+
const timeoutMs = options.timeoutMs ?? 5_000;
|
|
12619
|
+
const intervalMs = options.intervalMs ?? 25;
|
|
12620
|
+
const deadline = Date.now() + timeoutMs;
|
|
12621
|
+
while (liveRunExecutions.has(runId)) {
|
|
12622
|
+
if (Date.now() >= deadline) {
|
|
12623
|
+
throw new Error(`Timed out waiting for heartbeat run ${runId} execution to drain`);
|
|
12624
|
+
}
|
|
12625
|
+
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
|
12626
|
+
}
|
|
12627
|
+
},
|
|
10927
12628
|
list: async (companyId, agentId, limit, options = {}) => {
|
|
10928
12629
|
const safeForLegacyEncoding = await hasUnsafeTextProjectionDatabase();
|
|
10929
12630
|
const summary = options.summary === true;
|
|
@@ -11092,6 +12793,11 @@ export function heartbeatService(db, options = {}) {
|
|
|
11092
12793
|
triggerIssueMonitor,
|
|
11093
12794
|
reportRunActivity: clearDetachedRunWarning,
|
|
11094
12795
|
reapOrphanedRuns,
|
|
12796
|
+
// Override-aware scheduling-suppression check (honors the worktree
|
|
12797
|
+
// run-execution experimental setting). Callers outside the service that
|
|
12798
|
+
// gate on suppression should prefer this over the env-only resolver.
|
|
12799
|
+
resolveSchedulingSuppression: getSchedulingSuppression,
|
|
12800
|
+
drainRunningRunsForShutdown,
|
|
11095
12801
|
promoteDueScheduledRetries,
|
|
11096
12802
|
retryScheduledRetryNow,
|
|
11097
12803
|
resumeQueuedRuns,
|
|
@@ -11113,6 +12819,14 @@ export function heartbeatService(db, options = {}) {
|
|
|
11113
12819
|
reconcileTaskWatchdogs,
|
|
11114
12820
|
buildRunOutputSilence,
|
|
11115
12821
|
tickTimers: async (now = new Date()) => {
|
|
12822
|
+
if ((await getSchedulingSuppression()).suppressed) {
|
|
12823
|
+
return {
|
|
12824
|
+
checked: 0,
|
|
12825
|
+
enqueued: 0,
|
|
12826
|
+
skipped: 0,
|
|
12827
|
+
};
|
|
12828
|
+
}
|
|
12829
|
+
const cutoff = await getWorktreeExecutionCutoff();
|
|
11116
12830
|
const allAgents = await db
|
|
11117
12831
|
.select({ ...getTableColumns(agents) })
|
|
11118
12832
|
.from(agents)
|
|
@@ -11129,6 +12843,16 @@ export function heartbeatService(db, options = {}) {
|
|
|
11129
12843
|
const policy = parseHeartbeatPolicy(agent);
|
|
11130
12844
|
if (!policy.enabled || policy.intervalSec <= 0)
|
|
11131
12845
|
continue;
|
|
12846
|
+
if (cutoff) {
|
|
12847
|
+
const eligibleIssue = await db
|
|
12848
|
+
.select({ id: issues.id })
|
|
12849
|
+
.from(issues)
|
|
12850
|
+
.where(and(eq(issues.companyId, agent.companyId), eq(issues.assigneeAgentId, agent.id), inArray(issues.status, ["todo", "in_progress"]), gte(issues.createdAt, cutoff)))
|
|
12851
|
+
.limit(1)
|
|
12852
|
+
.then((rows) => rows[0] ?? null);
|
|
12853
|
+
if (!eligibleIssue)
|
|
12854
|
+
continue;
|
|
12855
|
+
}
|
|
11132
12856
|
checked += 1;
|
|
11133
12857
|
const baseline = new Date(agent.lastHeartbeatAt ?? agent.createdAt).getTime();
|
|
11134
12858
|
const elapsedMs = now.getTime() - baseline;
|