@rudderhq/server 0.7.2-canary.0 → 0.7.3

package/dist/services/runtime-kernel/heartbeat.js

@@ -8,7 +8,9 @@ import { killChildProcessTree, parseObject } from "../../agent-runtimes/utils.js
 import { notFound } from "../../errors.js";
 import { redactCurrentUserText, redactCurrentUserValue } from "../../log-redaction.js";
 import { logger } from "../../middleware/logger.js";
+import { agentIssueCreationService, getAgentIssueCreationRequestIdFromRunContext, } from "../agent-issue-creation.js";
 import { agentRunContextService } from "../agent-run-context.js";
+import { approvalService } from "../approvals.js";
 import { publishAutomationRunOutputToChat } from "../automation-chat-output.js";
 import { budgetService } from "../budgets.js";
 import { costService } from "../costs.js";
@@ -31,7 +33,7 @@ import { createHeartbeatExecuteHandlers } from "./heartbeat.execute.js";
 import { createHeartbeatMiscHandlers } from "./heartbeat.misc.js";
 import { createHeartbeatRecoveryHandlers } from "./heartbeat.recovery.js";
 import { createHeartbeatReleaseHandlers } from "./heartbeat.release.js";
-import { checkpointHeartbeatRunTerminalEffect, claimExpiredHeartbeatRunExecution, claimHeartbeatRunTerminalEffects, completeHeartbeatRunTerminalEffects, failHeartbeatRunTerminalEffect, markHeartbeatRunProcessExited, reconcileHeartbeatRunEvidence, reconcileHeartbeatRunTerminalEffectsIntent, releaseHeartbeatRunTerminalEffectsClaim, renewHeartbeatRunExecutionLease, renewHeartbeatRunTerminalEffectsClaim, RUN_EXECUTION_LEASE_MS, setWakeupStatusMonotonic, terminalEffectNames, transitionHeartbeatRunToTerminal, } from "./heartbeat.terminal.js";
+import { checkpointHeartbeatRunTerminalEffect, claimExpiredHeartbeatRunExecution, claimHeartbeatRunTerminalEffects, completeHeartbeatRunTerminalEffects, failHeartbeatRunTerminalEffect, markHeartbeatRunProcessExited, reconcileHeartbeatRunEvidence, reconcileHeartbeatRunTerminalEffectsIntent, releaseHeartbeatRunTerminalEffectsClaim, renewHeartbeatRunExecutionLease, renewHeartbeatRunTerminalEffectsClaim, RUN_EXECUTION_LEASE_MS, RUN_EXECUTION_LEASE_RENEW_INTERVAL_MS, setWakeupStatusMonotonic, terminalEffectNames, transitionHeartbeatRunToTerminal, } from "./heartbeat.terminal.js";
 import { createHeartbeatWakeupHandlers } from "./heartbeat.wakeup.js";
 const DEFAULT_HEARTBEAT_RUN_TIMEOUT_MS = 12 * 60 * 60 * 1000;
 const DEFAULT_HEARTBEAT_RUN_INACTIVITY_TIMEOUT_MS = 30 * 60 * 1000;
@@ -41,6 +43,49 @@ const TERMINAL_EFFECT_CLAIM_RENEW_INTERVAL_MS = 60_000;
 // server process.
 const activeRunExecutions = new Set();
 const runAbortControllers = new Map();
+const localExecutionLeaseStates = new Map();
+const LOCAL_EXECUTION_SLEEP_GAP_MS = RUN_EXECUTION_LEASE_RENEW_INTERVAL_MS * 2;
+const heartbeatRecoveryTails = new WeakMap();
+function getLocalExecutionLeaseState(runId, nowMs) {
+    const existing = localExecutionLeaseStates.get(runId);
+    if (existing)
+        return existing;
+    const state = {
+        executionOwnerToken: null,
+        lastRenewedAtMs: nowMs,
+        sleepRecoveryUsed: false,
+        sleepRecoveryPending: false,
+        sleepRecoveryGraceUntilMs: null,
+    };
+    localExecutionLeaseStates.set(runId, state);
+    return state;
+}
+function localExecutionOwnerMatches(runId, ownerToken) {
+    const state = localExecutionLeaseStates.get(runId);
+    return !state || state.executionOwnerToken === null || state.executionOwnerToken === ownerToken;
+}
+function observeLocalExecutionLease(runId, now) {
+    const nowMs = now.getTime();
+    const existing = localExecutionLeaseStates.get(runId);
+    const state = getLocalExecutionLeaseState(runId, nowMs);
+    const sleepRecoveryDetected = Boolean(existing
+        && nowMs > existing.lastRenewedAtMs
+        && nowMs - existing.lastRenewedAtMs > LOCAL_EXECUTION_SLEEP_GAP_MS
+        && !state.sleepRecoveryUsed);
+    if (sleepRecoveryDetected) {
+        state.sleepRecoveryUsed = true;
+        state.sleepRecoveryPending = true;
+    }
+    state.lastRenewedAtMs = nowMs;
+    return sleepRecoveryDetected;
+}
+function pruneLocalExecutionLeaseStates() {
+    for (const runId of localExecutionLeaseStates.keys()) {
+        if (!activeRunExecutions.has(runId) && !runningProcesses.has(runId)) {
+            localExecutionLeaseStates.delete(runId);
+        }
+    }
+}
 function formatDurationMs(ms) {
     const totalSeconds = Math.max(0, Math.round(ms / 1000));
     const hours = Math.floor(totalSeconds / 3600);
@@ -59,7 +104,9 @@ export function heartbeatService(db, testHooks) {
     });
     const runLogStore = getRunLogStore();
     const runContextSvc = agentRunContextService(db);
+    const approvalsSvc = approvalService(db);
     const issuesSvc = issueService(db);
+    const agentIssueCreationSvc = agentIssueCreationService(db);
     const executionWorkspacesSvc = runWorkspaceService(db);
     const workspaceOperationsSvc = workspaceOperationService(db);
     const budgetHooks = {
@@ -448,6 +495,44 @@ export function heartbeatService(db, testHooks) {
         return updated;
     }
     async function transitionRunToTerminal(runId, status, patch, opts) {
+        const runBeforeTerminal = await getRun(runId);
+        const agentIssueCreationSettlement = runBeforeTerminal
+            ? await (async () => {
+                const contextRequestId = getAgentIssueCreationRequestIdFromRunContext(runBeforeTerminal.contextSnapshot);
+                if (contextRequestId) {
+                    return {
+                        orgId: runBeforeTerminal.orgId,
+                        agentId: runBeforeTerminal.agentId,
+                        runId: runBeforeTerminal.id,
+                        requestId: contextRequestId,
+                    };
+                }
+                return await agentIssueCreationSvc
+                    .getSettlementIntentForRun(runBeforeTerminal.orgId, runBeforeTerminal.agentId, runBeforeTerminal.id)
+                    .catch((error) => {
+                    logger.warn({ err: error, runId }, "failed to resolve Agent Issue creation settlement terminal effect");
+                    return null;
+                });
+            })()
+            : null;
+        const agentIssueCreationNotification = runBeforeTerminal
+            ? await agentIssueCreationSvc
+                .getNotificationIntentForRun(runBeforeTerminal.orgId, runBeforeTerminal.agentId, runId)
+                .catch((error) => {
+                logger.warn({ err: error, runId }, "failed to resolve Agent Issue creation notification terminal effect");
+                return null;
+            })
+            : null;
+        const terminalEffectsIntent = {
+            version: 1,
+            ...(opts?.terminalEffectsIntent ?? {}),
+            ...(agentIssueCreationSettlement
+                ? { agentIssueCreationSettlement }
+                : {}),
+            ...(agentIssueCreationNotification
+                ? { agentIssueCreationNotification }
+                : {}),
+        };
         const updated = await transitionHeartbeatRunToTerminal(db, {
             runId,
             status,
@@ -455,12 +540,43 @@ export function heartbeatService(db, testHooks) {
             expectedStatuses: opts?.expectedStatuses,
             activityWatermark: opts?.activityWatermark,
             terminalEffectsPending: opts?.terminalEffectsPending,
-            terminalEffectsIntent: opts?.terminalEffectsIntent,
+            terminalEffectsIntent,
             processExitedAt: opts?.processExitedAt,
             expectedExecutionOwnerToken: opts?.expectedExecutionOwnerToken,
         });
-        if (updated)
+        if (updated) {
             publishRunStatus(updated);
+            const context = parseObject(updated.contextSnapshot);
+            const requestId = agentIssueCreationSettlement?.requestId
+                ?? getAgentIssueCreationRequestIdFromRunContext(context);
+            if (requestId) {
+                const settledRequest = await agentIssueCreationSvc.settleForRun({
+                    orgId: updated.orgId,
+                    agentId: updated.agentId,
+                    runId: updated.id,
+                    requestId,
+                    runStatus: updated.status,
+                    error: updated.error,
+                }).catch((error) => {
+                    logger.warn({ err: error, orgId: updated.orgId, agentId: updated.agentId, runId: updated.id, requestId }, "failed to settle Agent Issue creation request after run termination");
+                    return null;
+                });
+                if (settledRequest?.status === "succeeded" && settledRequest.createdIssueId) {
+                    await reconcileTerminalEffectsIntent(updated.id, {
+                        version: 1,
+                        agentIssueCreationNotification: {
+                            orgId: updated.orgId,
+                            agentId: updated.agentId,
+                            runId: updated.id,
+                            requestId: settledRequest.id,
+                            issueId: settledRequest.createdIssueId,
+                        },
+                    }).catch((error) => {
+                        logger.warn({ err: error, orgId: updated.orgId, agentId: updated.agentId, runId: updated.id, requestId }, "failed to record Agent Issue creation notification terminal effect");
+                    });
+                }
+            }
+        }
         return updated;
     }
     async function reconcileRunEvidence(runId, patch) {
@@ -590,8 +706,105 @@ export function heartbeatService(db, testHooks) {
     async function acknowledgeRunProcessExit(runId) {
         return markHeartbeatRunProcessExited(db, runId);
     }
-    async function renewRunExecutionLease(runId, ownerToken) {
-        return renewHeartbeatRunExecutionLease(db, runId, ownerToken);
+    function abortRunExecution(runId) {
+        const controller = runAbortControllers.get(runId);
+        if (controller) {
+            controller.abort();
+            return;
+        }
+        const tracked = runningProcesses.get(runId);
+        if (tracked)
+            killChildProcessTree(tracked.child, false);
+    }
+    async function renewRunExecutionLease(runId, ownerToken, now = new Date(), opts) {
+        await testHooks?.beforeRunExecutionLeaseRenewal?.({ runId, ownerToken });
+        const knownState = localExecutionLeaseStates.get(runId);
+        if (knownState && knownState.executionOwnerToken !== null && knownState.executionOwnerToken !== ownerToken) {
+            knownState.sleepRecoveryPending = false;
+            return null;
+        }
+        const sleepRecoveryDetected = opts?.observeSleep !== false
+            ? observeLocalExecutionLease(runId, now)
+            : false;
+        const state = localExecutionLeaseStates.get(runId);
+        if (state && state.executionOwnerToken === null) {
+            state.executionOwnerToken = ownerToken;
+        }
+        if (state && state.executionOwnerToken !== ownerToken) {
+            state.sleepRecoveryPending = false;
+            return null;
+        }
+        const renewed = await renewHeartbeatRunExecutionLease(db, runId, ownerToken, now, {
+            touchActivity: opts?.touchActivity || sleepRecoveryDetected,
+        });
+        if (!renewed) {
+            if (state)
+                state.sleepRecoveryPending = false;
+        }
+        return renewed;
+    }
+    async function preserveLiveExecutionAfterSleep(run, now) {
+        if (!activeRunExecutions.has(run.id) && !runningProcesses.has(run.id))
+            return false;
+        if (!localExecutionOwnerMatches(run.id, run.executionOwnerToken)) {
+            abortRunExecution(run.id);
+            return true;
+        }
+        if (!run.executionOwnerToken || !run.executionLeaseExpiresAt)
+            return false;
+        observeLocalExecutionLease(run.id, now);
+        const state = localExecutionLeaseStates.get(run.id);
+        if (state && state.executionOwnerToken === null) {
+            state.executionOwnerToken = run.executionOwnerToken;
+        }
+        const nowMs = now.getTime();
+        if (state
+            && state.sleepRecoveryGraceUntilMs !== null
+            && nowMs < state.sleepRecoveryGraceUntilMs) {
+            return true;
+        }
+        const wakeGracePending = state?.sleepRecoveryPending === true;
+        if (wakeGracePending) {
+            state.sleepRecoveryPending = false;
+        }
+        else if (new Date(run.executionLeaseExpiresAt).getTime() > now.getTime()) {
+            return false;
+        }
+        else if (state?.sleepRecoveryUsed) {
+            return false;
+        }
+        else if (state) {
+            state.sleepRecoveryUsed = true;
+        }
+        const renewed = await renewRunExecutionLease(run.id, run.executionOwnerToken, now, {
+            observeSleep: false,
+            touchActivity: true,
+        });
+        if (!renewed) {
+            abortRunExecution(run.id);
+        }
+        else if (state) {
+            state.sleepRecoveryGraceUntilMs = nowMs + RUN_EXECUTION_LEASE_RENEW_INTERVAL_MS;
+        }
+        return true;
+    }
+    async function withHeartbeatRecoveryLock(task) {
+        const dbKey = db;
+        const previous = heartbeatRecoveryTails.get(dbKey) ?? Promise.resolve();
+        let release;
+        const current = new Promise((resolve) => {
+            release = resolve;
+        });
+        heartbeatRecoveryTails.set(dbKey, current);
+        await previous;
+        try {
+            return await task();
+        }
+        finally {
+            release();
+            if (heartbeatRecoveryTails.get(dbKey) === current)
+                heartbeatRecoveryTails.delete(dbKey);
+        }
     }
     async function countRunningRunsForAgent(agentId, excludeRunId) {
         const pendingExecutionIds = [...activeRunExecutions].filter((runId) => runId !== excludeRunId);
@@ -836,126 +1049,152 @@ export function heartbeatService(db, testHooks) {
             });
         }
     }
-    async function reapOrphanedRuns(opts) {
+    async function reapOrphanedRunsLocked(opts) {
+        pruneLocalExecutionLeaseStates();
         const staleThresholdMs = opts?.staleThresholdMs ?? 0;
         const now = opts?.now ?? new Date();
         const recoveryCutoff = opts?.recoveryCutoff ?? now;
-        // Find all runs stuck in "running" state (queued runs are legitimately waiting; resumeQueuedRuns handles them)
-        const activeRuns = await db
-            .select({
-            run: heartbeatRuns,
-            agentRuntimeType: agents.agentRuntimeType,
-        })
-            .from(heartbeatRuns)
-            .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
-            .where(or(eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.terminalEffectsPending, true)));
         const reaped = [];
-        for (const { run, agentRuntimeType } of activeRuns) {
-            if (run.terminalEffectsPending) {
-                if (!run.processExitedAt) {
-                    const exited = await terminateRunProcessAndWait(run, agentRuntimeType);
-                    if (!exited || activeRunExecutions.has(run.id))
+        try {
+            // Find all runs stuck in "running" state (queued runs are legitimately waiting; resumeQueuedRuns handles them)
+            const activeRuns = await db
+                .select({
+                run: heartbeatRuns,
+                agentRuntimeType: agents.agentRuntimeType,
+            })
+                .from(heartbeatRuns)
+                .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
+                .where(or(eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.terminalEffectsPending, true)));
+            for (const { run, agentRuntimeType } of activeRuns) {
+                if (run.terminalEffectsPending) {
+                    if (!run.processExitedAt) {
+                        const exited = await terminateRunProcessAndWait(run, agentRuntimeType);
+                        if (!exited || activeRunExecutions.has(run.id))
+                            continue;
+                        await acknowledgeRunProcessExit(run.id);
+                    }
+                    const completed = await completeTerminalControlEffects(run);
+                    if (!completed)
+                        continue;
+                    reaped.push(run.id);
+                    continue;
+                }
+                // Apply staleness threshold to avoid false positives
+                if (staleThresholdMs > 0) {
+                    const refTime = run.updatedAt ? new Date(run.updatedAt).getTime() : 0;
+                    if (now.getTime() - refTime < staleThresholdMs)
                         continue;
-                    await acknowledgeRunProcessExit(run.id);
                 }
-                const completed = await completeTerminalControlEffects(run);
-                if (!completed)
+                // The lease timer pauses while the host sleeps, but the child process
+                // and this server's execution registry can remain alive. Preserve that
+                // local execution instead of turning it into a process-loss retry.
+                if (activeRunExecutions.has(run.id) || runningProcesses.has(run.id)) {
+                    if (!localExecutionOwnerMatches(run.id, run.executionOwnerToken)) {
+                        abortRunExecution(run.id);
+                        continue;
+                    }
+                    if (run.executionOwnerToken) {
+                        const renewed = await renewRunExecutionLease(run.id, run.executionOwnerToken, now);
+                        if (!renewed)
+                            abortRunExecution(run.id);
+                    }
                     continue;
-                reaped.push(run.id);
-                continue;
-            }
-            // Apply staleness threshold to avoid false positives
-            if (staleThresholdMs > 0) {
-                const refTime = run.updatedAt ? new Date(run.updatedAt).getTime() : 0;
-                if (now.getTime() - refTime < staleThresholdMs)
+                }
+                const recoveryClaim = await claimExpiredHeartbeatRunExecution(db, run.id, { now, recoveryCutoff });
+                if (!recoveryClaim)
                     continue;
-            }
-            const recoveryClaim = await claimExpiredHeartbeatRunExecution(db, run.id, { now, recoveryCutoff });
-            if (!recoveryClaim)
-                continue;
-            const claimedRun = recoveryClaim.run;
-            const tracksLocalChild = isTrackedLocalChildProcessAdapter(agentRuntimeType);
-            let detachedTerminationMessage = null;
-            if (!claimedRun.processExitedAt && tracksLocalChild && claimedRun.processPid && isProcessAlive(claimedRun.processPid)) {
-                const termination = await terminateOrphanedProcess(claimedRun.processPid);
-                if (termination.stillAlive) {
-                    const detachedMessage = termination.error
-                        ? `Lost in-memory process handle, child pid ${claimedRun.processPid} is still alive, and Rudder could not terminate it: ${termination.error}`
-                        : `Lost in-memory process handle, but child pid ${claimedRun.processPid} is still alive`;
-                    const detachedRun = await setRunStatus(claimedRun.id, "running", {
-                        error: detachedMessage,
-                        errorCode: DETACHED_PROCESS_ERROR_CODE,
-                    });
-                    if (detachedRun) {
-                        await appendRunEvent(detachedRun, {
-                            eventType: "lifecycle",
-                            stream: "system",
-                            level: "warn",
-                            message: detachedMessage,
-                            payload: {
-                                processPid: claimedRun.processPid,
-                            },
+                const claimedRun = recoveryClaim.run;
+                const tracksLocalChild = isTrackedLocalChildProcessAdapter(agentRuntimeType);
+                let detachedTerminationMessage = null;
+                if (!claimedRun.processExitedAt && tracksLocalChild && claimedRun.processPid && isProcessAlive(claimedRun.processPid)) {
+                    const termination = await terminateOrphanedProcess(claimedRun.processPid);
+                    if (termination.stillAlive) {
+                        const detachedMessage = termination.error
+                            ? `Lost in-memory process handle, child pid ${claimedRun.processPid} is still alive, and Rudder could not terminate it: ${termination.error}`
+                            : `Lost in-memory process handle, but child pid ${claimedRun.processPid} is still alive`;
+                        const detachedRun = await setRunStatus(claimedRun.id, "running", {
+                            error: detachedMessage,
+                            errorCode: DETACHED_PROCESS_ERROR_CODE,
                         });
+                        if (detachedRun) {
+                            await appendRunEvent(detachedRun, {
+                                eventType: "lifecycle",
+                                stream: "system",
+                                level: "warn",
+                                message: detachedMessage,
+                                payload: {
+                                    processPid: claimedRun.processPid,
+                                },
+                            });
+                        }
+                        continue;
                     }
+                    detachedTerminationMessage = termination.terminationSignal
+                        ? `Terminated detached child pid ${claimedRun.processPid} with ${termination.terminationSignal} after Rudder lost its process handle`
+                        : `Detached child pid ${claimedRun.processPid} exited before Rudder could terminate it`;
+                }
+                const shouldRetry = tracksLocalChild && !!claimedRun.processPid && (claimedRun.processLossRetryCount ?? 0) < 1;
+                const baseMessage = claimedRun.processPid
+                    ? `Process lost -- child pid ${claimedRun.processPid} is no longer running`
+                    : "Process lost -- server may have restarted";
+                let finalizedRun = await transitionRunToTerminal(claimedRun.id, "failed", {
+                    error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
+                    errorCode: "process_lost",
+                    finishedAt: now,
+                }, {
+                    processExitedAt: now,
+                    terminalEffectsIntent: shouldRetry ? { version: 1, processLossRetry: true } : { version: 1 },
+                    expectedExecutionOwnerToken: recoveryClaim.ownerToken,
+                });
+                if (!finalizedRun)
                     continue;
+                await setWakeupStatus(claimedRun.wakeupRequestId, "failed", {
+                    finishedAt: now,
+                    error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
+                });
+                if (detachedTerminationMessage) {
+                    await appendRunEvent(finalizedRun, {
+                        eventType: "lifecycle",
+                        stream: "system",
+                        level: "warn",
+                        message: detachedTerminationMessage,
+                        payload: {
+                            ...(claimedRun.processPid ? { processPid: claimedRun.processPid } : {}),
+                        },
+                    });
                 }
-                detachedTerminationMessage = termination.terminationSignal
-                    ? `Terminated detached child pid ${claimedRun.processPid} with ${termination.terminationSignal} after Rudder lost its process handle`
-                    : `Detached child pid ${claimedRun.processPid} exited before Rudder could terminate it`;
-            }
-            const shouldRetry = tracksLocalChild && !!claimedRun.processPid && (claimedRun.processLossRetryCount ?? 0) < 1;
-            const baseMessage = claimedRun.processPid
-                ? `Process lost -- child pid ${claimedRun.processPid} is no longer running`
-                : "Process lost -- server may have restarted";
-            let finalizedRun = await transitionRunToTerminal(claimedRun.id, "failed", {
-                error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
-                errorCode: "process_lost",
-                finishedAt: now,
-            }, {
-                processExitedAt: now,
-                terminalEffectsIntent: shouldRetry ? { version: 1, processLossRetry: true } : { version: 1 },
-                expectedExecutionOwnerToken: recoveryClaim.ownerToken,
-            });
-            if (!finalizedRun)
-                continue;
-            await setWakeupStatus(claimedRun.wakeupRequestId, "failed", {
-                finishedAt: now,
-                error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
-            });
-            if (detachedTerminationMessage) {
                 await appendRunEvent(finalizedRun, {
                     eventType: "lifecycle",
                     stream: "system",
-                    level: "warn",
-                    message: detachedTerminationMessage,
+                    level: "error",
+                    message: shouldRetry
+                        ? `${baseMessage}; retry will be queued after terminal effects complete`
+                        : baseMessage,
                     payload: {
                         ...(claimedRun.processPid ? { processPid: claimedRun.processPid } : {}),
                     },
                 });
+                const completed = await completeTerminalControlEffects(finalizedRun);
+                runningProcesses.delete(claimedRun.id);
+                if (!completed)
+                    continue;
+                reaped.push(claimedRun.id);
             }
-            await appendRunEvent(finalizedRun, {
-                eventType: "lifecycle",
-                stream: "system",
-                level: "error",
-                message: shouldRetry
-                    ? `${baseMessage}; retry will be queued after terminal effects complete`
-                    : baseMessage,
-                payload: {
-                    ...(claimedRun.processPid ? { processPid: claimedRun.processPid } : {}),
-                },
+        }
+        finally {
+            // Terminal effects are the fast path. Keep a request-scoped sweep as a
+            // second durable path for runs whose effect was lost or dead-lettered.
+            await agentIssueCreationSvc.reconcileTerminalSettlements().catch((error) => {
+                logger.warn({ err: error }, "failed to reconcile terminal Agent Issue creation requests");
             });
-            const completed = await completeTerminalControlEffects(finalizedRun);
-            runningProcesses.delete(claimedRun.id);
-            if (!completed)
-                continue;
-            reaped.push(claimedRun.id);
         }
         if (reaped.length > 0) {
             logger.warn({ reapedCount: reaped.length, runIds: reaped }, "reaped orphaned heartbeat runs");
         }
         return { reaped: reaped.length, runIds: reaped };
     }
-    async function reapInactiveRuns(opts) {
+    async function reapInactiveRunsLocked(opts) {
+        pruneLocalExecutionLeaseStates();
         const maxInactivityMs = opts?.maxInactivityMs ?? DEFAULT_HEARTBEAT_RUN_INACTIVITY_TIMEOUT_MS;
         if (!Number.isFinite(maxInactivityMs) || maxInactivityMs <= 0) {
             return { timedOut: 0, runIds: [] };
@@ -977,6 +1216,8 @@ export function heartbeatService(db, testHooks) {
         for (const { run, agentRuntimeType, lastEventAt, eventCount } of activeRuns) {
             if (opts?.recoveryCutoff && new Date(run.createdAt).getTime() >= opts.recoveryCutoff.getTime())
                 continue;
+            if (await preserveLiveExecutionAfterSleep(run, now))
+                continue;
             const activityTimes = [
                 run.updatedAt,
                 lastEventAt,
@@ -1041,7 +1282,8 @@ export function heartbeatService(db, testHooks) {
         }
         return { timedOut: timedOut.length, runIds: timedOut };
     }
-    async function reapTimedOutRuns(opts) {
+    async function reapTimedOutRunsLocked(opts) {
+        pruneLocalExecutionLeaseStates();
         const maxRuntimeMs = opts?.maxRuntimeMs ?? DEFAULT_HEARTBEAT_RUN_TIMEOUT_MS;
         if (!Number.isFinite(maxRuntimeMs) || maxRuntimeMs <= 0) {
             return { timedOut: 0, runIds: [] };
@@ -1059,6 +1301,8 @@ export function heartbeatService(db, testHooks) {
         for (const { run, agentRuntimeType } of activeRuns) {
             if (opts?.recoveryCutoff && new Date(run.createdAt).getTime() >= opts.recoveryCutoff.getTime())
                 continue;
+            if (await preserveLiveExecutionAfterSleep(run, now))
+                continue;
             const startedAt = run.startedAt ? new Date(run.startedAt).getTime() : null;
             if (!startedAt || !Number.isFinite(startedAt))
                 continue;
@@ -1110,6 +1354,15 @@ export function heartbeatService(db, testHooks) {
         }
         return { timedOut: timedOut.length, runIds: timedOut };
     }
+    async function reapOrphanedRuns(opts) {
+        return withHeartbeatRecoveryLock(() => reapOrphanedRunsLocked(opts));
+    }
+    async function reapInactiveRuns(opts) {
+        return withHeartbeatRecoveryLock(() => reapInactiveRunsLocked(opts));
+    }
+    async function reapTimedOutRuns(opts) {
+        return withHeartbeatRecoveryLock(() => reapTimedOutRunsLocked(opts));
+    }
     async function resumeQueuedRuns() {
         const queuedRuns = await db
             .select({ agentId: heartbeatRuns.agentId })
@@ -1300,6 +1553,45 @@ export function heartbeatService(db, testHooks) {
                     lastError: taskSessionIntent.lastError ?? null,
                 }));
             }
+            const agentIssueCreationSettlement = intent.agentIssueCreationSettlement;
+            if (agentIssueCreationSettlement) {
+                await runClaimedEffect("agent_issue_creation_settlement", async () => {
+                    const settledRequest = await agentIssueCreationSvc.settleForRun({
+                        orgId: agentIssueCreationSettlement.orgId,
+                        agentId: agentIssueCreationSettlement.agentId,
+                        runId: agentIssueCreationSettlement.runId,
+                        requestId: agentIssueCreationSettlement.requestId,
+                        runStatus: current.status,
+                        error: current.error,
+                    });
+                    const request = settledRequest
+                        ?? await agentIssueCreationSvc.getById(agentIssueCreationSettlement.orgId, agentIssueCreationSettlement.requestId);
+                    if (request?.status === "succeeded" && request.createdIssueId) {
+                        const notificationIntent = {
+                            orgId: agentIssueCreationSettlement.orgId,
+                            agentId: agentIssueCreationSettlement.agentId,
+                            runId: agentIssueCreationSettlement.runId,
+                            requestId: request.id,
+                            issueId: request.createdIssueId,
+                        };
+                        intent.agentIssueCreationNotification ??= notificationIntent;
+                        await reconcileTerminalEffectsIntent(current.id, {
+                            version: 1,
+                            agentIssueCreationNotification: notificationIntent,
+                        });
+                    }
+                    else if (request
+                        && !request.createdIssueId
+                        && ["queued", "running", "deferred"].includes(request.status)) {
+                        throw new Error("Agent Issue creation request remained active after terminal settlement");
+                    }
+                    return settledRequest;
+                });
+            }
+            const agentIssueCreationNotification = intent.agentIssueCreationNotification;
+            if (agentIssueCreationNotification) {
+                await runClaimedEffect("agent_issue_creation_notification", () => agentIssueCreationSvc.publishNotificationForRequest(agentIssueCreationNotification));
+            }
             if (intent.processLossRetry) {
                 await runClaimedEffect("process_loss_retry", () => {
                     if (!agent)
@@ -1407,8 +1699,8 @@ export function heartbeatService(db, testHooks) {
         });
     }
     const baseContext = {
-        db, instanceSettings, getCurrentUserRedactionOptions, runLogStore, runContextSvc, issuesSvc, executionWorkspacesSvc, workspaceOperationsSvc, activeRunExecutions, runAbortControllers, budgetHooks, budgets,
-        getAgent, getRun, getRuntimeState, getTaskSession, getLatestRunForSession, getOldestRunForSession, resolveNormalizedUsageForSession, evaluateSessionCompaction, resolveSessionBeforeForWakeup, resolveExplicitResumeSessionOverride, upsertTaskSession, clearTaskSessions, ensureRuntimeState, setRunStatus, transitionRunToTerminal, reconcileRunEvidence, reconcileTerminalEffectsIntent, setWakeupStatus, updateWakeupRequestRecord, insertWakeupRequestRecord, appendRunEvent, persistRunProcessMetadata, clearDetachedRunWarning, terminateRunProcessAndWait, acknowledgeRunProcessExit, renewRunExecutionLease, countRunningRunsForAgent, claimQueuedRun, finalizeAgentStatus, completeTerminalControlEffects, reapOrphanedRuns, reapInactiveRuns, reapTimedOutRuns, resumeQueuedRuns, updateRuntimeState, startNextQueuedRunForAgent,
+        db, approvalsSvc, instanceSettings, getCurrentUserRedactionOptions, runLogStore, runContextSvc, issuesSvc, executionWorkspacesSvc, workspaceOperationsSvc, activeRunExecutions, runAbortControllers, budgetHooks, budgets,
+        getAgent, getRun, getRuntimeState, getTaskSession, getLatestRunForSession, getOldestRunForSession, resolveNormalizedUsageForSession, evaluateSessionCompaction, resolveSessionBeforeForWakeup, resolveExplicitResumeSessionOverride, upsertTaskSession, clearTaskSessions, ensureRuntimeState, setRunStatus, transitionRunToTerminal, reconcileRunEvidence, reconcileTerminalEffectsIntent, setWakeupStatus, updateWakeupRequestRecord, insertWakeupRequestRecord, appendRunEvent, persistRunProcessMetadata, clearDetachedRunWarning, terminateRunProcessAndWait, acknowledgeRunProcessExit, abortRunExecution, renewRunExecutionLease, countRunningRunsForAgent, claimQueuedRun, finalizeAgentStatus, completeTerminalControlEffects, reapOrphanedRuns, reapInactiveRuns, reapTimedOutRuns, resumeQueuedRuns, updateRuntimeState, startNextQueuedRunForAgent,
     };
     const recoveryHandlers = createHeartbeatRecoveryHandlers({ ...baseContext, startNextQueuedRunForAgent });
     const wakeupHandlers = createHeartbeatWakeupHandlers({
@@ -1423,7 +1715,12 @@ export function heartbeatService(db, testHooks) {
         ...wakeupHandlers,
         afterIssuePromotionCommitted: testHooks?.afterIssuePromotionCommitted,
     });
-    const executeHandlers = createHeartbeatExecuteHandlers({ ...baseContext, ...recoveryHandlers, ...releaseHandlers, ...wakeupHandlers });
+    const executeHandlers = createHeartbeatExecuteHandlers({
+        ...baseContext,
+        ...recoveryHandlers,
+        ...releaseHandlers,
+        ...wakeupHandlers,
+    });
     const miscHandlers = createHeartbeatMiscHandlers({ ...baseContext, ...recoveryHandlers, ...releaseHandlers, ...wakeupHandlers, ...executeHandlers });
     const { enqueueRecoveryRun, enqueueProcessLossRetry, evaluatePassiveIssueClosureForLockedIssue, parseHeartbeatPolicy } = recoveryHandlers;
     const { enqueueWakeup } = wakeupHandlers;
@@ -1649,6 +1946,7 @@ export function heartbeatService(db, testHooks) {
         reapOrphanedRuns,
         reapInactiveRuns,
         reapTimedOutRuns,
+        startNextQueuedRunForAgent,
         resumeQueuedRuns,
         tickTimers: async (now = new Date()) => {
             const allAgents = await db.select().from(agents);