@rudderhq/server 0.4.6-canary.9 → 0.5.0

package/dist/services/runtime-kernel/heartbeat.js

@@ -1,13 +1,15 @@
 import { hasSessionCompactionThresholds } from "@rudderhq/agent-runtime-utils";
 import { agentRuntimeState, agents, agentTaskSessions, agentWakeupRequests, heartbeatRunEvents, heartbeatRuns, issues, organizations } from "@rudderhq/db";
-import { summarizeTokenUsage } from "@rudderhq/shared";
-import { and, asc, desc, eq, gt, gte, inArray, lte, sql } from "drizzle-orm";
+import { summarizeTokenUsage, toHeartbeatRun } from "@rudderhq/shared";
+import { and, asc, desc, eq, gt, gte, inArray, lte, or, sql } from "drizzle-orm";
+import { randomUUID } from "node:crypto";
 import { runningProcesses } from "../../agent-runtimes/index.js";
-import { parseObject } from "../../agent-runtimes/utils.js";
+import { killChildProcessTree, parseObject } from "../../agent-runtimes/utils.js";
 import { notFound } from "../../errors.js";
 import { redactCurrentUserText, redactCurrentUserValue } from "../../log-redaction.js";
 import { logger } from "../../middleware/logger.js";
 import { agentRunContextService } from "../agent-run-context.js";
+import { publishAutomationRunOutputToChat } from "../automation-chat-output.js";
 import { budgetService } from "../budgets.js";
 import { costService } from "../costs.js";
 import { runWorkspaceService } from "../execution-workspaces.js";
@@ -15,6 +17,7 @@ import { summarizeHeartbeatRunResultJson } from "../heartbeat-run-summary.js";
 import { instanceSettingsService } from "../instance-settings.js";
 import { issueService } from "../issues.js";
 import { publishLiveEvent } from "../live-events.js";
+import { appendHeartbeatRunEvent } from "../run-events.js";
 import { getRunLogStore } from "../run-log-store.js";
 import { workspaceOperationService } from "../workspace-operations.js";
 export { prioritizeProjectWorkspaceCandidatesForRun } from "../agent-run-context.js";
@@ -26,9 +29,16 @@ import { createHeartbeatExecuteHandlers } from "./heartbeat.execute.js";
 import { createHeartbeatMiscHandlers } from "./heartbeat.misc.js";
 import { createHeartbeatRecoveryHandlers } from "./heartbeat.recovery.js";
 import { createHeartbeatReleaseHandlers } from "./heartbeat.release.js";
+import { checkpointHeartbeatRunTerminalEffect, claimExpiredHeartbeatRunExecution, claimHeartbeatRunTerminalEffects, completeHeartbeatRunTerminalEffects, failHeartbeatRunTerminalEffect, markHeartbeatRunProcessExited, reconcileHeartbeatRunEvidence, reconcileHeartbeatRunTerminalEffectsIntent, releaseHeartbeatRunTerminalEffectsClaim, renewHeartbeatRunExecutionLease, renewHeartbeatRunTerminalEffectsClaim, RUN_EXECUTION_LEASE_MS, setWakeupStatusMonotonic, terminalEffectNames, transitionHeartbeatRunToTerminal, } from "./heartbeat.terminal.js";
 import { createHeartbeatWakeupHandlers } from "./heartbeat.wakeup.js";
 const DEFAULT_HEARTBEAT_RUN_TIMEOUT_MS = 12 * 60 * 60 * 1000;
 const DEFAULT_HEARTBEAT_RUN_INACTIVITY_TIMEOUT_MS = 30 * 60 * 1000;
+const TERMINAL_EFFECT_CLAIM_RENEW_INTERVAL_MS = 60_000;
+// heartbeatService is instantiated by routes and the scheduler. Execution
+// ownership must therefore be shared across every service instance in this
+// server process.
+const activeRunExecutions = new Set();
+const runAbortControllers = new Map();
 function formatDurationMs(ms) {
     const totalSeconds = Math.max(0, Math.round(ms / 1000));
     const hours = Math.floor(totalSeconds / 3600);
@@ -40,7 +50,7 @@ function formatDurationMs(ms) {
         return `${minutes}m ${seconds}s`;
     return `${seconds}s`;
 }
-export function heartbeatService(db) {
+export function heartbeatService(db, testHooks) {
     const instanceSettings = instanceSettingsService(db);
     const getCurrentUserRedactionOptions = async () => ({
         enabled: (await instanceSettings.getGeneral()).censorUsernameInLogs,
@@ -50,7 +60,6 @@ export function heartbeatService(db) {
     const issuesSvc = issueService(db);
     const executionWorkspacesSvc = runWorkspaceService(db);
     const workspaceOperationsSvc = workspaceOperationService(db);
-    const activeRunExecutions = new Set();
     const budgetHooks = {
         cancelWorkForScope: (scope) => cancelBudgetScopeWork(scope),
     };
@@ -229,8 +238,7 @@ export function heartbeatService(db) {
                 (codec.getDisplayId ? codec.getDisplayId(parsedParams) : null) ??
                 readNonEmptyString(parsedParams?.sessionId));
         }
-        const runtimeForRun = await getRuntimeState(agent.id);
-        return runtimeForRun?.sessionId ?? null;
+        return null;
     }
     async function resolveExplicitResumeSessionOverride(agent, payload, taskKey) {
         const resumeFromRunId = readNonEmptyString(payload?.resumeFromRunId);
@@ -242,6 +250,8 @@ export function heartbeatService(db) {
             contextSnapshot: heartbeatRuns.contextSnapshot,
             sessionIdBefore: heartbeatRuns.sessionIdBefore,
             sessionIdAfter: heartbeatRuns.sessionIdAfter,
+            sessionParamsBeforeJson: heartbeatRuns.sessionParamsBeforeJson,
+            sessionParamsAfterJson: heartbeatRuns.sessionParamsAfterJson,
         })
             .from(heartbeatRuns)
             .where(and(eq(heartbeatRuns.id, resumeFromRunId), eq(heartbeatRuns.orgId, agent.orgId), eq(heartbeatRuns.agentId, agent.id)))
@@ -254,10 +264,21 @@ export function heartbeatService(db) {
             ? await getTaskSession(agent.orgId, agent.id, agent.agentRuntimeType, resumeTaskKey)
             : null;
         const sessionCodec = getAgentRuntimeSessionCodec(agent.agentRuntimeType);
+        const persistedSessionSuppression = heartbeatSessions.readSessionReuseSuppression(resumeContext);
+        const resumeRunHasSessionAfter = Boolean(readNonEmptyString(resumeRun.sessionIdAfter)) ||
+            Boolean(resumeRun.sessionParamsAfterJson && Object.keys(resumeRun.sessionParamsAfterJson).length > 0);
         const sessionOverride = buildExplicitResumeSessionOverride({
             resumeFromRunId,
             resumeRunSessionIdBefore: resumeRun.sessionIdBefore,
             resumeRunSessionIdAfter: resumeRun.sessionIdAfter,
+            resumeRunSessionParamsBefore: resumeRun.sessionParamsBeforeJson,
+            resumeRunSessionParamsAfter: resumeRun.sessionParamsAfterJson,
+            resumeRunSessionCleared: resumeRun.sessionIdAfter == null &&
+                resumeRun.sessionParamsAfterJson != null &&
+                Object.keys(resumeRun.sessionParamsAfterJson).length === 0,
+            resumeRunSessionSuppression: resumeRunHasSessionAfter ? null : persistedSessionSuppression,
+            resumeContextSessionParams: parseObject(resumeContext.resumeSessionParams),
+            resumeContextSessionDisplayId: readNonEmptyString(resumeContext.resumeSessionDisplayId),
             taskSession: resumeTaskSession,
             sessionCodec,
         });
@@ -270,38 +291,64 @@ export function heartbeatService(db) {
             taskId: readNonEmptyString(resumeContext.taskId) ?? readNonEmptyString(resumeContext.issueId),
             sessionDisplayId: sessionOverride.sessionDisplayId,
             sessionParams: sessionOverride.sessionParams,
+            sessionCleared: "sessionCleared" in sessionOverride && sessionOverride.sessionCleared === true,
+            sessionReuseSuppression: "sessionReuseSuppression" in sessionOverride
+                ? sessionOverride.sessionReuseSuppression
+                : null,
         };
     }
     async function upsertTaskSession(input) {
-        const existing = await getTaskSession(input.orgId, input.agentId, input.agentRuntimeType, input.taskKey);
-        if (existing) {
-            return db
-                .update(agentTaskSessions)
-                .set({
+        return db.transaction(async (tx) => {
+            await tx.execute(sql `select pg_advisory_xact_lock(hashtext(${`task-session:${input.orgId}:${input.agentId}:${input.agentRuntimeType}:${input.taskKey}`}))`);
+            const existing = await tx
+                .select()
+                .from(agentTaskSessions)
+                .where(and(eq(agentTaskSessions.orgId, input.orgId), eq(agentTaskSessions.agentId, input.agentId), eq(agentTaskSessions.agentRuntimeType, input.agentRuntimeType), eq(agentTaskSessions.taskKey, input.taskKey)))
+                .then((rows) => rows[0] ?? null);
+            if (existing) {
+                if (existing.lastRunId && input.lastRunId && existing.lastRunId !== input.lastRunId) {
+                    const runDates = await tx
+                        .select({ id: heartbeatRuns.id, createdAt: heartbeatRuns.createdAt })
+                        .from(heartbeatRuns)
+                        .where(inArray(heartbeatRuns.id, [existing.lastRunId, input.lastRunId]));
+                    const existingDate = runDates.find((row) => row.id === existing.lastRunId)?.createdAt;
+                    const incomingDate = runDates.find((row) => row.id === input.lastRunId)?.createdAt;
+                    if (existingDate
+                        && incomingDate
+                        && (existingDate.getTime() > incomingDate.getTime()
+                            || (existingDate.getTime() === incomingDate.getTime()
+                                && existing.lastRunId.localeCompare(input.lastRunId) > 0))) {
+                        return existing;
+                    }
+                }
+                return tx
+                    .update(agentTaskSessions)
+                    .set({
+                    sessionParamsJson: input.sessionParamsJson,
+                    sessionDisplayId: input.sessionDisplayId,
+                    lastRunId: input.lastRunId,
+                    lastError: input.lastError,
+                    updatedAt: new Date(),
+                })
+                    .where(eq(agentTaskSessions.id, existing.id))
+                    .returning()
+                    .then((rows) => rows[0] ?? null);
+            }
+            return tx
+                .insert(agentTaskSessions)
+                .values({
+                orgId: input.orgId,
+                agentId: input.agentId,
+                agentRuntimeType: input.agentRuntimeType,
+                taskKey: input.taskKey,
                 sessionParamsJson: input.sessionParamsJson,
                 sessionDisplayId: input.sessionDisplayId,
                 lastRunId: input.lastRunId,
                 lastError: input.lastError,
-                updatedAt: new Date(),
             })
-                .where(eq(agentTaskSessions.id, existing.id))
                 .returning()
                 .then((rows) => rows[0] ?? null);
-        }
-        return db
-            .insert(agentTaskSessions)
-            .values({
-            orgId: input.orgId,
-            agentId: input.agentId,
-            agentRuntimeType: input.agentRuntimeType,
-            taskKey: input.taskKey,
-            sessionParamsJson: input.sessionParamsJson,
-            sessionDisplayId: input.sessionDisplayId,
-            lastRunId: input.lastRunId,
-            lastError: input.lastError,
-        })
-            .returning()
-            .then((rows) => rows[0] ?? null);
+        });
     }
     async function clearTaskSessions(orgId, agentId, opts) {
         const conditions = [
@@ -314,6 +361,27 @@ export function heartbeatService(db) {
         if (opts?.agentRuntimeType) {
             conditions.push(eq(agentTaskSessions.agentRuntimeType, opts.agentRuntimeType));
         }
+        if (opts?.sourceRunId) {
+            const sourceRun = await getRun(opts.sourceRunId);
+            if (sourceRun) {
+                conditions.push(sql `(
+          ${agentTaskSessions.lastRunId} is null
+          or ${agentTaskSessions.lastRunId} = ${opts.sourceRunId}
+          or not exists (
+            select 1
+            from heartbeat_runs previous_run
+            where previous_run.id = ${agentTaskSessions.lastRunId}
+              and (
+                previous_run.created_at > ${sourceRun.createdAt.toISOString()}::timestamptz
+                or (
+                  previous_run.created_at = ${sourceRun.createdAt.toISOString()}::timestamptz
+                  and previous_run.id > ${sourceRun.id}
+                )
+              )
+          )
+        )`);
+            }
+        }
         return db
             .delete(agentTaskSessions)
             .where(and(...conditions))
@@ -346,39 +414,61 @@ export function heartbeatService(db) {
             .returning()
             .then((rows) => rows[0]);
     }
+    function publishRunStatus(updated) {
+        publishLiveEvent({
+            orgId: updated.orgId,
+            type: "heartbeat.run.status",
+            payload: {
+                runId: updated.id,
+                agentId: updated.agentId,
+                status: updated.status,
+                invocationSource: updated.invocationSource,
+                triggerDetail: updated.triggerDetail,
+                error: updated.error ?? null,
+                errorCode: updated.errorCode ?? null,
+                startedAt: updated.startedAt ? new Date(updated.startedAt).toISOString() : null,
+                finishedAt: updated.finishedAt ? new Date(updated.finishedAt).toISOString() : null,
+            },
+        });
+    }
     async function setRunStatus(runId, status, patch) {
+        if (["succeeded", "failed", "cancelled", "timed_out"].includes(status)) {
+            throw new Error("Terminal heartbeat run updates must use transitionRunToTerminal");
+        }
         const updated = await db
             .update(heartbeatRuns)
             .set({ status, ...patch, updatedAt: new Date() })
             .where(eq(heartbeatRuns.id, runId))
             .returning()
             .then((rows) => rows[0] ?? null);
-        if (updated) {
-            publishLiveEvent({
-                orgId: updated.orgId,
-                type: "heartbeat.run.status",
-                payload: {
-                    runId: updated.id,
-                    agentId: updated.agentId,
-                    status: updated.status,
-                    invocationSource: updated.invocationSource,
-                    triggerDetail: updated.triggerDetail,
-                    error: updated.error ?? null,
-                    errorCode: updated.errorCode ?? null,
-                    startedAt: updated.startedAt ? new Date(updated.startedAt).toISOString() : null,
-                    finishedAt: updated.finishedAt ? new Date(updated.finishedAt).toISOString() : null,
-                },
-            });
-        }
+        if (updated)
+            publishRunStatus(updated);
+        return updated;
+    }
+    async function transitionRunToTerminal(runId, status, patch, opts) {
+        const updated = await transitionHeartbeatRunToTerminal(db, {
+            runId,
+            status,
+            patch,
+            expectedStatuses: opts?.expectedStatuses,
+            activityWatermark: opts?.activityWatermark,
+            terminalEffectsPending: opts?.terminalEffectsPending,
+            terminalEffectsIntent: opts?.terminalEffectsIntent,
+            processExitedAt: opts?.processExitedAt,
+            expectedExecutionOwnerToken: opts?.expectedExecutionOwnerToken,
+        });
+        if (updated)
+            publishRunStatus(updated);
         return updated;
     }
+    async function reconcileRunEvidence(runId, patch) {
+        return reconcileHeartbeatRunEvidence(db, runId, patch);
+    }
+    async function reconcileTerminalEffectsIntent(runId, intent) {
+        return reconcileHeartbeatRunTerminalEffectsIntent(db, runId, intent);
+    }
     async function setWakeupStatus(wakeupRequestId, status, patch) {
-        if (!wakeupRequestId)
-            return;
-        await db
-            .update(agentWakeupRequests)
-            .set({ status, ...patch, updatedAt: new Date() })
-            .where(eq(agentWakeupRequests.id, wakeupRequestId));
+        return setWakeupStatusMonotonic(db, wakeupRequestId, status, patch);
     }
     async function updateWakeupRequestRecord(tx, wakeupRequestId, patch) {
         return tx
@@ -389,13 +479,21 @@ export function heartbeatService(db) {
             .then((rows) => rows[0] ?? null);
     }
     async function insertWakeupRequestRecord(tx, values) {
-        return tx
+        const inserted = await tx
             .insert(agentWakeupRequests)
             .values(values)
+            .onConflictDoNothing()
             .returning()
             .then((rows) => rows[0] ?? null);
+        if (inserted || !values.idempotencyKey)
+            return inserted;
+        return tx
+            .select()
+            .from(agentWakeupRequests)
+            .where(and(eq(agentWakeupRequests.orgId, values.orgId), eq(agentWakeupRequests.agentId, values.agentId), eq(agentWakeupRequests.idempotencyKey, values.idempotencyKey)))
+            .then((rows) => rows[0] ?? null);
     }
-    async function appendRunEvent(run, seq, event) {
+    async function appendRunEvent(run, event) {
         const currentUserRedactionOptions = await getCurrentUserRedactionOptions();
         const sanitizedMessage = event.message
             ? redactCurrentUserText(event.message, currentUserRedactionOptions)
@@ -403,17 +501,17 @@ export function heartbeatService(db) {
         const sanitizedPayload = event.payload
             ? redactCurrentUserValue(event.payload, currentUserRedactionOptions)
             : event.payload;
-        await db.insert(heartbeatRunEvents).values({
+        const inserted = await appendHeartbeatRunEvent(db, {
             orgId: run.orgId,
             runId: run.id,
             agentId: run.agentId,
-            seq,
             eventType: event.eventType,
             stream: event.stream,
             level: event.level,
             color: event.color,
             message: sanitizedMessage,
             payload: sanitizedPayload,
+            idempotencyKey: event.idempotencyKey ?? null,
         });
         publishLiveEvent({
             orgId: run.orgId,
@@ -421,7 +519,7 @@ export function heartbeatService(db) {
             payload: {
                 runId: run.id,
                 agentId: run.agentId,
-                seq,
+                seq: inserted.seq,
                 eventType: event.eventType,
                 stream: event.stream ?? null,
                 level: event.level ?? null,
@@ -431,13 +529,6 @@ export function heartbeatService(db) {
             },
         });
     }
-    async function nextRunEventSeq(runId) {
-        const [row] = await db
-            .select({ maxSeq: sql `max(${heartbeatRunEvents.seq})` })
-            .from(heartbeatRunEvents)
-            .where(eq(heartbeatRunEvents.runId, runId));
-        return Number(row?.maxSeq ?? 0) + 1;
-    }
     async function persistRunProcessMetadata(runId, meta) {
         const startedAt = new Date(meta.startedAt);
         const updated = await db
@@ -447,7 +538,7 @@ export function heartbeatService(db) {
             processStartedAt: Number.isNaN(startedAt.getTime()) ? new Date() : startedAt,
             updatedAt: new Date(),
         })
-            .where(eq(heartbeatRuns.id, runId))
+            .where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.status, "running")))
             .returning()
             .then((rows) => rows[0] ?? null);
         return updated;
@@ -465,7 +556,7 @@ export function heartbeatService(db) {
             .then((rows) => rows[0] ?? null);
         if (!updated)
             return null;
-        await appendRunEvent(updated, await nextRunEventSeq(updated.id), {
+        await appendRunEvent(updated, {
             eventType: "lifecycle",
             stream: "system",
             level: "info",
@@ -473,11 +564,39 @@ export function heartbeatService(db) {
         });
         return updated;
     }
-    async function countRunningRunsForAgent(agentId) {
+    async function terminateRunProcessAndWait(run, agentRuntimeType) {
+        if (run.processExitedAt)
+            return true;
+        runAbortControllers.get(run.id)?.abort();
+        const tracked = runningProcesses.get(run.id);
+        if (tracked) {
+            const pid = tracked.child.pid;
+            killChildProcessTree(tracked.child, false);
+            if (typeof pid !== "number" || pid <= 0)
+                return true;
+            const graceMs = Math.max(1, tracked.graceSec) * 1000;
+            if (await waitForProcessExit(pid, graceMs))
+                return true;
+            killChildProcessTree(tracked.child, true);
+            return waitForProcessExit(pid, ORPHANED_PROCESS_KILL_WAIT_MS);
+        }
+        if (isTrackedLocalChildProcessAdapter(agentRuntimeType) && run.processPid && isProcessAlive(run.processPid)) {
+            return !(await terminateOrphanedProcess(run.processPid)).stillAlive;
+        }
+        return !activeRunExecutions.has(run.id);
+    }
+    async function acknowledgeRunProcessExit(runId) {
+        return markHeartbeatRunProcessExited(db, runId);
+    }
+    async function renewRunExecutionLease(runId, ownerToken) {
+        return renewHeartbeatRunExecutionLease(db, runId, ownerToken);
+    }
+    async function countRunningRunsForAgent(agentId, excludeRunId) {
+        const pendingExecutionIds = [...activeRunExecutions].filter((runId) => runId !== excludeRunId);
         const [{ count }] = await db
             .select({ count: sql `count(*)` })
             .from(heartbeatRuns)
-            .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "running")));
+            .where(and(eq(heartbeatRuns.agentId, agentId), or(eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.terminalEffectsPending, true), ...(pendingExecutionIds.length > 0 ? [inArray(heartbeatRuns.id, pendingExecutionIds)] : [])), ...(excludeRunId ? [sql `${heartbeatRuns.id} <> ${excludeRunId}`] : [])));
         return Number(count ?? 0);
     }
     async function claimQueuedRun(run) {
@@ -499,25 +618,26 @@ export function heartbeatService(db) {
             }
         }
         async function cancelQueuedRunDuringClaim(reason) {
-            const cancelled = await setRunStatus(run.id, "cancelled", {
+            const cancelled = await transitionRunToTerminal(run.id, "cancelled", {
                 finishedAt: new Date(),
                 error: reason,
                 errorCode: "cancelled",
-            });
+            }, { expectedStatuses: ["queued"], processExitedAt: new Date() });
+            if (!cancelled)
+                return null;
             await setWakeupStatus(run.wakeupRequestId, "cancelled", {
                 finishedAt: new Date(),
                 error: reason,
             });
             if (cancelled) {
-                await appendRunEvent(cancelled, 1, {
+                await appendRunEvent(cancelled, {
                     eventType: "lifecycle",
                     stream: "system",
                     level: "warn",
                     message: "run cancelled",
                 });
-                await releaseIssueExecutionAndPromote(cancelled);
+                await completeTerminalControlEffects(cancelled, { startNext: false });
             }
-            await finalizeAgentStatus(run.agentId, "cancelled");
             return null;
         }
         const agent = await getAgent(run.agentId);
@@ -554,16 +674,82 @@ export function heartbeatService(db) {
             return await cancelQueuedRunDuringClaim(budgetBlock.reason);
         }
         const claimedAt = new Date();
-        const claimed = await db
-            .update(heartbeatRuns)
-            .set({
-            status: "running",
-            startedAt: run.startedAt ?? claimedAt,
-            updatedAt: claimedAt,
-        })
-            .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "queued")))
-            .returning()
-            .then((rows) => rows[0] ?? null);
+        const claimed = await db.transaction(async (tx) => {
+            await tx.execute(sql `select pg_advisory_xact_lock(hashtext(${`agent-run-state:${run.agentId}`}))`);
+            await tx.execute(sql `select id from agents where id = ${run.agentId} for update`);
+            const currentAgent = await tx
+                .select({ status: agents.status })
+                .from(agents)
+                .where(eq(agents.id, run.agentId))
+                .then((rows) => rows[0] ?? null);
+            if (!currentAgent || ["paused", "terminated", "pending_approval"].includes(currentAgent.status)) {
+                return null;
+            }
+            const pendingTerminalEffects = await tx
+                .select({ id: heartbeatRuns.id })
+                .from(heartbeatRuns)
+                .where(and(eq(heartbeatRuns.agentId, run.agentId), eq(heartbeatRuns.terminalEffectsPending, true)))
+                .limit(1)
+                .then((rows) => rows[0] ?? null);
+            if (pendingTerminalEffects)
+                return null;
+            await tx.execute(sql `select id from heartbeat_runs where id = ${run.id} for update`);
+            const currentRun = await tx
+                .select()
+                .from(heartbeatRuns)
+                .where(eq(heartbeatRuns.id, run.id))
+                .then((rows) => rows[0] ?? null);
+            if (!currentRun || currentRun.status !== "queued")
+                return null;
+            const currentContext = parseObject(currentRun.contextSnapshot);
+            const currentTaskKey = deriveTaskKey(currentContext, null);
+            const resetTaskSession = shouldResetTaskSessionForWake(currentContext);
+            const sessionCodec = getAgentRuntimeSessionCodec(agent.agentRuntimeType);
+            const explicitSessionParams = normalizeSessionParams(sessionCodec.deserialize(parseObject(currentContext.resumeSessionParams)));
+            const explicitSessionDisplayId = truncateDisplayId(readNonEmptyString(currentContext.resumeSessionDisplayId) ??
+                (sessionCodec.getDisplayId ? sessionCodec.getDisplayId(explicitSessionParams) : null) ??
+                readNonEmptyString(explicitSessionParams?.sessionId));
+            const taskSession = currentTaskKey && !resetTaskSession
+                ? await tx
+                    .select()
+                    .from(agentTaskSessions)
+                    .where(and(eq(agentTaskSessions.orgId, currentRun.orgId), eq(agentTaskSessions.agentId, currentRun.agentId), eq(agentTaskSessions.agentRuntimeType, agent.agentRuntimeType), eq(agentTaskSessions.taskKey, currentTaskKey)))
+                    .then((rows) => rows[0] ?? null)
+                : null;
+            const taskSessionParams = normalizeSessionParams(sessionCodec.deserialize(taskSession?.sessionParamsJson ?? null));
+            const taskSessionDisplayId = truncateDisplayId(taskSession?.sessionDisplayId ??
+                (sessionCodec.getDisplayId ? sessionCodec.getDisplayId(taskSessionParams) : null) ??
+                readNonEmptyString(taskSessionParams?.sessionId));
+            const sessionSelection = heartbeatSessions.selectRunSessionLineage({
+                forceFresh: Boolean(heartbeatSessions.readSessionReuseSuppression(currentContext)),
+                explicitSessionParams,
+                explicitSessionDisplayId,
+                taskSessionParams,
+                taskSessionDisplayId,
+            });
+            const claimedRun = await tx
+                .update(heartbeatRuns)
+                .set({
+                status: "running",
+                startedAt: currentRun.startedAt ?? claimedAt,
+                sessionIdBefore: sessionSelection.sessionDisplayId ?? readNonEmptyString(sessionSelection.sessionParams?.sessionId),
+                sessionParamsBeforeJson: sessionSelection.sessionParams,
+                sessionReuseScope: sessionSelection.reuseScope,
+                executionOwnerToken: randomUUID(),
+                executionLeaseExpiresAt: new Date(claimedAt.getTime() + RUN_EXECUTION_LEASE_MS),
+                updatedAt: claimedAt,
+            })
+                .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "queued")))
+                .returning()
+                .then((rows) => rows[0] ?? null);
+            if (!claimedRun)
+                return null;
+            await tx
+                .update(agents)
+                .set({ status: "running", updatedAt: claimedAt })
+                .where(eq(agents.id, run.agentId));
+            return claimedRun;
+        });
         if (!claimed)
             return null;
         publishLiveEvent({
@@ -584,14 +770,14 @@ export function heartbeatService(db) {
         await setWakeupStatus(claimed.wakeupRequestId, "claimed", { claimedAt });
         return claimed;
     }
-    async function finalizeAgentStatus(agentId, outcome) {
+    async function finalizeAgentStatus(agentId, outcome, runId) {
         const existing = await getAgent(agentId);
         if (!existing)
             return;
         if (existing.status === "paused" || existing.status === "terminated") {
             return;
         }
-        const runningCount = await countRunningRunsForAgent(agentId);
+        const runningCount = await countRunningRunsForAgent(agentId, runId);
         const nextStatus = runningCount > 0
             ? "running"
             : outcome === "succeeded" || outcome === "cancelled"
@@ -624,7 +810,8 @@ export function heartbeatService(db) {
     }
     async function reapOrphanedRuns(opts) {
         const staleThresholdMs = opts?.staleThresholdMs ?? 0;
-        const now = new Date();
+        const now = opts?.now ?? new Date();
+        const recoveryCutoff = opts?.recoveryCutoff ?? now;
         // Find all runs stuck in "running" state (queued runs are legitimately waiting; resumeQueuedRuns handles them)
         const activeRuns = await db
             .select({
@@ -633,100 +820,107 @@ export function heartbeatService(db) {
         })
             .from(heartbeatRuns)
             .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
-            .where(eq(heartbeatRuns.status, "running"));
+            .where(or(eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.terminalEffectsPending, true)));
         const reaped = [];
         for (const { run, agentRuntimeType } of activeRuns) {
-            if (runningProcesses.has(run.id) || activeRunExecutions.has(run.id))
+            if (run.terminalEffectsPending) {
+                if (!run.processExitedAt) {
+                    const exited = await terminateRunProcessAndWait(run, agentRuntimeType);
+                    if (!exited || activeRunExecutions.has(run.id))
+                        continue;
+                    await acknowledgeRunProcessExit(run.id);
+                }
+                const completed = await completeTerminalControlEffects(run);
+                if (!completed)
+                    continue;
+                reaped.push(run.id);
                 continue;
+            }
             // Apply staleness threshold to avoid false positives
             if (staleThresholdMs > 0) {
                 const refTime = run.updatedAt ? new Date(run.updatedAt).getTime() : 0;
                 if (now.getTime() - refTime < staleThresholdMs)
                     continue;
             }
+            const recoveryClaim = await claimExpiredHeartbeatRunExecution(db, run.id, { now, recoveryCutoff });
+            if (!recoveryClaim)
+                continue;
+            const claimedRun = recoveryClaim.run;
             const tracksLocalChild = isTrackedLocalChildProcessAdapter(agentRuntimeType);
             let detachedTerminationMessage = null;
-            if (tracksLocalChild && run.processPid && isProcessAlive(run.processPid)) {
-                const termination = await terminateOrphanedProcess(run.processPid);
+            if (!claimedRun.processExitedAt && tracksLocalChild && claimedRun.processPid && isProcessAlive(claimedRun.processPid)) {
+                const termination = await terminateOrphanedProcess(claimedRun.processPid);
                 if (termination.stillAlive) {
                     const detachedMessage = termination.error
-                        ? `Lost in-memory process handle, child pid ${run.processPid} is still alive, and Rudder could not terminate it: ${termination.error}`
-                        : `Lost in-memory process handle, but child pid ${run.processPid} is still alive`;
-                    const detachedRun = await setRunStatus(run.id, "running", {
+                        ? `Lost in-memory process handle, child pid ${claimedRun.processPid} is still alive, and Rudder could not terminate it: ${termination.error}`
+                        : `Lost in-memory process handle, but child pid ${claimedRun.processPid} is still alive`;
+                    const detachedRun = await setRunStatus(claimedRun.id, "running", {
                         error: detachedMessage,
                         errorCode: DETACHED_PROCESS_ERROR_CODE,
                     });
                     if (detachedRun) {
-                        await appendRunEvent(detachedRun, await nextRunEventSeq(detachedRun.id), {
+                        await appendRunEvent(detachedRun, {
                             eventType: "lifecycle",
                             stream: "system",
                             level: "warn",
                             message: detachedMessage,
                             payload: {
-                                processPid: run.processPid,
+                                processPid: claimedRun.processPid,
                             },
                         });
                     }
                     continue;
                 }
                 detachedTerminationMessage = termination.terminationSignal
-                    ? `Terminated detached child pid ${run.processPid} with ${termination.terminationSignal} after Rudder lost its process handle`
-                    : `Detached child pid ${run.processPid} exited before Rudder could terminate it`;
+                    ? `Terminated detached child pid ${claimedRun.processPid} with ${termination.terminationSignal} after Rudder lost its process handle`
+                    : `Detached child pid ${claimedRun.processPid} exited before Rudder could terminate it`;
             }
-            const shouldRetry = tracksLocalChild && !!run.processPid && (run.processLossRetryCount ?? 0) < 1;
-            const baseMessage = run.processPid
-                ? `Process lost -- child pid ${run.processPid} is no longer running`
+            const shouldRetry = tracksLocalChild && !!claimedRun.processPid && (claimedRun.processLossRetryCount ?? 0) < 1;
+            const baseMessage = claimedRun.processPid
+                ? `Process lost -- child pid ${claimedRun.processPid} is no longer running`
                 : "Process lost -- server may have restarted";
-            let finalizedRun = await setRunStatus(run.id, "failed", {
+            let finalizedRun = await transitionRunToTerminal(claimedRun.id, "failed", {
                 error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
                 errorCode: "process_lost",
                 finishedAt: now,
+            }, {
+                processExitedAt: now,
+                terminalEffectsIntent: shouldRetry ? { version: 1, processLossRetry: true } : { version: 1 },
+                expectedExecutionOwnerToken: recoveryClaim.ownerToken,
             });
-            await setWakeupStatus(run.wakeupRequestId, "failed", {
+            if (!finalizedRun)
+                continue;
+            await setWakeupStatus(claimedRun.wakeupRequestId, "failed", {
                 finishedAt: now,
                 error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
             });
-            if (!finalizedRun)
-                finalizedRun = await getRun(run.id);
-            if (!finalizedRun)
-                continue;
             if (detachedTerminationMessage) {
-                await appendRunEvent(finalizedRun, await nextRunEventSeq(finalizedRun.id), {
+                await appendRunEvent(finalizedRun, {
                     eventType: "lifecycle",
                     stream: "system",
                     level: "warn",
                     message: detachedTerminationMessage,
                     payload: {
-                        ...(run.processPid ? { processPid: run.processPid } : {}),
+                        ...(claimedRun.processPid ? { processPid: claimedRun.processPid } : {}),
                     },
                 });
             }
-            let retriedRun = null;
-            if (shouldRetry) {
-                const agent = await getAgent(run.agentId);
-                if (agent) {
-                    retriedRun = await enqueueProcessLossRetry(finalizedRun, agent, now);
-                }
-            }
-            else {
-                await releaseIssueExecutionAndPromote(finalizedRun);
-            }
-            await appendRunEvent(finalizedRun, await nextRunEventSeq(finalizedRun.id), {
+            await appendRunEvent(finalizedRun, {
                 eventType: "lifecycle",
                 stream: "system",
                 level: "error",
                 message: shouldRetry
-                    ? `${baseMessage}; queued retry ${retriedRun?.id ?? ""}`.trim()
+                    ? `${baseMessage}; retry will be queued after terminal effects complete`
                     : baseMessage,
                 payload: {
-                    ...(run.processPid ? { processPid: run.processPid } : {}),
-                    ...(retriedRun ? { retryRunId: retriedRun.id } : {}),
+                    ...(claimedRun.processPid ? { processPid: claimedRun.processPid } : {}),
                 },
             });
-            await finalizeAgentStatus(run.agentId, "failed");
-            await startNextQueuedRunForAgent(run.agentId);
-            runningProcesses.delete(run.id);
-            reaped.push(run.id);
+            const completed = await completeTerminalControlEffects(finalizedRun);
+            runningProcesses.delete(claimedRun.id);
+            if (!completed)
+                continue;
+            reaped.push(claimedRun.id);
         }
         if (reaped.length > 0) {
             logger.warn({ reapedCount: reaped.length, runIds: reaped }, "reaped orphaned heartbeat runs");
@@ -744,6 +938,7 @@ export function heartbeatService(db) {
             run: heartbeatRuns,
             agentRuntimeType: agents.agentRuntimeType,
             lastEventAt: sql `max(${heartbeatRunEvents.createdAt})`,
+            eventCount: sql `count(${heartbeatRunEvents.id})::int`,
         })
             .from(heartbeatRuns)
             .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
@@ -751,7 +946,9 @@ export function heartbeatService(db) {
             .where(eq(heartbeatRuns.status, "running"))
             .groupBy(heartbeatRuns.id, agents.agentRuntimeType);
         const timedOut = [];
-        for (const { run, agentRuntimeType, lastEventAt } of activeRuns) {
+        for (const { run, agentRuntimeType, lastEventAt, eventCount } of activeRuns) {
+            if (opts?.recoveryCutoff && new Date(run.createdAt).getTime() >= opts.recoveryCutoff.getTime())
+                continue;
             const activityTimes = [
                 run.updatedAt,
                 lastEventAt,
@@ -768,33 +965,29 @@ export function heartbeatService(db) {
             if (inactiveMs < maxInactivityMs)
                 continue;
             const message = `Run had no recorded activity for ${formatDurationMs(maxInactivityMs)}`;
-            const running = runningProcesses.get(run.id);
-            if (running) {
-                const pid = running.child.pid;
-                running.child.kill("SIGTERM");
-                const graceMs = Math.max(1, running.graceSec) * 1000;
-                setTimeout(() => {
-                    if (typeof pid === "number" && isProcessAlive(pid)) {
-                        running.child.kill("SIGKILL");
-                    }
-                }, graceMs);
-            }
-            else if (isTrackedLocalChildProcessAdapter(agentRuntimeType) && run.processPid && isProcessAlive(run.processPid)) {
-                await terminateOrphanedProcess(run.processPid);
-            }
-            const finalizedRun = await setRunStatus(run.id, "timed_out", {
+            const processExitPending = activeRunExecutions.has(run.id)
+                || runningProcesses.has(run.id)
+                || (!run.processExitedAt && isTrackedLocalChildProcessAdapter(agentRuntimeType) && !!run.processPid && isProcessAlive(run.processPid));
+            const finalizedRun = await transitionRunToTerminal(run.id, "timed_out", {
                 finishedAt: now,
                 error: message,
                 errorCode: "inactivity_timeout",
+                terminalEffectsPending: true,
+            }, {
+                activityWatermark: {
+                    updatedAt: run.updatedAt,
+                    eventCount: Number(eventCount ?? 0),
+                },
+                processExitedAt: processExitPending ? null : now,
+                expectedExecutionOwnerToken: run.executionOwnerToken,
             });
+            if (!finalizedRun)
+                continue;
             await setWakeupStatus(run.wakeupRequestId, "timed_out", {
                 finishedAt: now,
                 error: message,
             });
-            const terminalRun = finalizedRun ?? await getRun(run.id);
-            if (!terminalRun)
-                continue;
-            await appendRunEvent(terminalRun, await nextRunEventSeq(terminalRun.id), {
+            await appendRunEvent(finalizedRun, {
                 eventType: "lifecycle",
                 stream: "system",
                 level: "error",
@@ -807,10 +1000,12 @@ export function heartbeatService(db) {
                     ...(run.processPid ? { processPid: run.processPid } : {}),
                 },
             });
-            await releaseIssueExecutionAndPromote(terminalRun);
-            await finalizeAgentStatus(run.agentId, "timed_out");
-            await startNextQueuedRunForAgent(run.agentId);
-            runningProcesses.delete(run.id);
+            const processExited = !processExitPending || await terminateRunProcessAndWait(finalizedRun, agentRuntimeType);
+            if (processExited && !activeRunExecutions.has(finalizedRun.id)) {
+                await acknowledgeRunProcessExit(finalizedRun.id);
+                await completeTerminalControlEffects(finalizedRun);
+                runningProcesses.delete(run.id);
+            }
             timedOut.push(run.id);
         }
         if (timedOut.length > 0) {
@@ -834,6 +1029,8 @@ export function heartbeatService(db) {
             .where(eq(heartbeatRuns.status, "running"));
         const timedOut = [];
         for (const { run, agentRuntimeType } of activeRuns) {
+            if (opts?.recoveryCutoff && new Date(run.createdAt).getTime() >= opts.recoveryCutoff.getTime())
+                continue;
             const startedAt = run.startedAt ? new Date(run.startedAt).getTime() : null;
             if (!startedAt || !Number.isFinite(startedAt))
                 continue;
@@ -841,33 +1038,25 @@ export function heartbeatService(db) {
             if (runtimeMs < maxRuntimeMs)
                 continue;
             const message = `Run exceeded maximum duration of ${formatDurationMs(maxRuntimeMs)}`;
-            const running = runningProcesses.get(run.id);
-            if (running) {
-                const pid = running.child.pid;
-                running.child.kill("SIGTERM");
-                const graceMs = Math.max(1, running.graceSec) * 1000;
-                setTimeout(() => {
-                    if (typeof pid === "number" && isProcessAlive(pid)) {
-                        running.child.kill("SIGKILL");
-                    }
-                }, graceMs);
-            }
-            else if (isTrackedLocalChildProcessAdapter(agentRuntimeType) && run.processPid && isProcessAlive(run.processPid)) {
-                await terminateOrphanedProcess(run.processPid);
-            }
-            const finalizedRun = await setRunStatus(run.id, "timed_out", {
+            const processExitPending = activeRunExecutions.has(run.id)
+                || runningProcesses.has(run.id)
+                || (!run.processExitedAt && isTrackedLocalChildProcessAdapter(agentRuntimeType) && !!run.processPid && isProcessAlive(run.processPid));
+            const finalizedRun = await transitionRunToTerminal(run.id, "timed_out", {
                 finishedAt: now,
                 error: message,
                 errorCode: "timeout",
+                terminalEffectsPending: true,
+            }, {
+                processExitedAt: processExitPending ? null : now,
+                expectedExecutionOwnerToken: run.executionOwnerToken,
             });
+            if (!finalizedRun)
+                continue;
             await setWakeupStatus(run.wakeupRequestId, "timed_out", {
                 finishedAt: now,
                 error: message,
             });
-            const terminalRun = finalizedRun ?? await getRun(run.id);
-            if (!terminalRun)
-                continue;
-            await appendRunEvent(terminalRun, await nextRunEventSeq(terminalRun.id), {
+            await appendRunEvent(finalizedRun, {
                 eventType: "lifecycle",
                 stream: "system",
                 level: "error",
@@ -880,10 +1069,12 @@ export function heartbeatService(db) {
                     ...(run.processPid ? { processPid: run.processPid } : {}),
                 },
             });
-            await releaseIssueExecutionAndPromote(terminalRun);
-            await finalizeAgentStatus(run.agentId, "timed_out");
-            await startNextQueuedRunForAgent(run.agentId);
-            runningProcesses.delete(run.id);
+            const processExited = !processExitPending || await terminateRunProcessAndWait(finalizedRun, agentRuntimeType);
+            if (processExited && !activeRunExecutions.has(finalizedRun.id)) {
+                await acknowledgeRunProcessExit(finalizedRun.id);
+                await completeTerminalControlEffects(finalizedRun);
+                runningProcesses.delete(run.id);
+            }
             timedOut.push(run.id);
         }
         if (timedOut.length > 0) {
@@ -901,7 +1092,7 @@ export function heartbeatService(db) {
             await startNextQueuedRunForAgent(agentId);
         }
     }
-    async function updateRuntimeState(agent, run, result, session, normalizedUsage) {
+    async function updateRuntimeState(agent, run, result, session, normalizedUsage, opts) {
         await ensureRuntimeState(agent);
         const usage = normalizedUsage ?? normalizeUsageTotals(result.usage);
         const rawInputTokens = usage?.inputTokens ?? 0;
@@ -919,24 +1110,37 @@ export function heartbeatService(db) {
         });
         const biller = resolveLedgerBiller(result);
         const ledgerScope = await resolveLedgerScopeForRun(db, agent.orgId, run);
-        await db
-            .update(agentRuntimeState)
-            .set({
-            agentRuntimeType: agent.agentRuntimeType,
-            sessionId: session.legacySessionId,
-            lastRunId: run.id,
-            lastRunStatus: run.status,
-            lastError: result.errorMessage ?? null,
-            totalInputTokens: sql `${agentRuntimeState.totalInputTokens} + ${tokenSummary.promptTokens}`,
-            totalOutputTokens: sql `${agentRuntimeState.totalOutputTokens} + ${outputTokens}`,
-            totalCachedInputTokens: sql `${agentRuntimeState.totalCachedInputTokens} + ${cachedInputTokens}`,
-            totalCostCents: sql `${agentRuntimeState.totalCostCents} + ${additionalCostCents}`,
-            updatedAt: new Date(),
-        })
-            .where(eq(agentRuntimeState.agentId, agent.id));
+        if (opts?.ownsTerminal) {
+            await db
+                .update(agentRuntimeState)
+                .set({
+                agentRuntimeType: agent.agentRuntimeType,
+                sessionId: session.legacySessionId,
+                lastRunId: run.id,
+                lastRunStatus: run.status,
+                lastError: result.errorMessage ?? null,
+                updatedAt: new Date(),
+            })
+                .where(and(eq(agentRuntimeState.agentId, agent.id), sql `(
+            ${agentRuntimeState.lastRunId} is null
+            or ${agentRuntimeState.lastRunId} = ${run.id}
+            or not exists (
+              select 1
+              from heartbeat_runs previous_run
+              where previous_run.id = ${agentRuntimeState.lastRunId}
+                and (
+                  previous_run.created_at > ${run.createdAt.toISOString()}::timestamptz
+                  or (
+                    previous_run.created_at = ${run.createdAt.toISOString()}::timestamptz
+                    and previous_run.id > ${run.id}
+                  )
+                )
+            )
+          )`));
+        }
         if (additionalCostCents > 0 || hasTokenUsage) {
             const costs = costService(db, budgetHooks);
-            await costs.createEvent(agent.orgId, {
+            await costs.createHeartbeatRunEventOnce(agent.orgId, {
                 heartbeatRunId: run.id,
                 agentId: agent.id,
                 issueId: ledgerScope.issueId,
@@ -950,9 +1154,163 @@ export function heartbeatService(db) {
                 outputTokens,
                 costCents: additionalCostCents,
                 occurredAt: new Date(),
+            }, async (tx) => {
+                await tx
+                    .update(agentRuntimeState)
+                    .set({
+                    totalInputTokens: sql `${agentRuntimeState.totalInputTokens} + ${tokenSummary.promptTokens}`,
+                    totalOutputTokens: sql `${agentRuntimeState.totalOutputTokens} + ${outputTokens}`,
+                    totalCachedInputTokens: sql `${agentRuntimeState.totalCachedInputTokens} + ${cachedInputTokens}`,
+                    totalCostCents: sql `${agentRuntimeState.totalCostCents} + ${additionalCostCents}`,
+                    updatedAt: new Date(),
+                })
+                    .where(eq(agentRuntimeState.agentId, agent.id));
             });
         }
     }
+    async function completeTerminalControlEffects(run, opts) {
+        const claim = await claimHeartbeatRunTerminalEffects(db, run.id);
+        if (!claim)
+            return;
+        const current = claim.run;
+        let claimReleased = false;
+        try {
+            const runContext = parseObject(current.contextSnapshot);
+            const intent = parseObject(current.terminalEffectsJson);
+            const completedEffects = new Set(current.terminalEffectsCompletedJson ?? []);
+            const deadLetteredEffects = new Set(current.terminalEffectsDeadLetteredJson ?? []);
+            const runClaimedEffect = async (effectName, effect) => {
+                if (completedEffects.has(effectName) || deadLetteredEffects.has(effectName))
+                    return undefined;
+                const renewClaim = async () => {
+                    if (!await renewHeartbeatRunTerminalEffectsClaim(db, current.id, claim.claimToken)) {
+                        throw new Error("Heartbeat terminal effects claim was lost");
+                    }
+                };
+                await renewClaim();
+                let renewalError = null;
+                let renewal = Promise.resolve();
+                const renewalTimer = setInterval(() => {
+                    renewal = renewal
+                        .then(renewClaim)
+                        .catch((error) => {
+                        renewalError ??= error;
+                    });
+                }, TERMINAL_EFFECT_CLAIM_RENEW_INTERVAL_MS);
+                renewalTimer.unref?.();
+                try {
+                    const result = await effect();
+                    await renewal;
+                    if (renewalError)
+                        throw renewalError;
+                    await renewClaim();
+                    if (!await checkpointHeartbeatRunTerminalEffect(db, current.id, claim.claimToken, effectName)) {
+                        throw new Error(`Heartbeat terminal effect ${effectName} checkpoint was lost`);
+                    }
+                    completedEffects.add(effectName);
+                    return result;
+                }
+                catch (error) {
+                    const failure = await failHeartbeatRunTerminalEffect(db, current.id, claim.claimToken, effectName, error);
+                    if (!failure)
+                        throw error;
+                    if (failure.deadLettered) {
+                        deadLetteredEffects.add(effectName);
+                        return undefined;
+                    }
+                    claimReleased = true;
+                    throw error;
+                }
+                finally {
+                    clearInterval(renewalTimer);
+                }
+            };
+            if (intent.automation) {
+                await runClaimedEffect("automation_chat", () => publishAutomationRunOutputToChat(db, {
+                    issueId: readNonEmptyString(runContext.issueId),
+                    output: opts?.automationOutput ?? intent.automation?.output ?? current.error,
+                    status: current.status,
+                    transcript: opts?.automationTranscript ?? [],
+                }));
+            }
+            const agent = await getAgent(current.agentId);
+            const runtimeIntent = intent.runtime;
+            if (runtimeIntent) {
+                await runClaimedEffect("runtime_cost", () => {
+                    if (!agent)
+                        throw new Error("Agent not found while replaying runtime terminal effect");
+                    return updateRuntimeState(agent, current, {
+                        exitCode: null,
+                        signal: null,
+                        timedOut: false,
+                        errorMessage: runtimeIntent.errorMessage,
+                        provider: runtimeIntent.provider,
+                        biller: runtimeIntent.biller,
+                        model: runtimeIntent.model,
+                        billingType: runtimeIntent.billingType,
+                        costUsd: runtimeIntent.costUsd,
+                    }, { legacySessionId: runtimeIntent.legacySessionId }, runtimeIntent.normalizedUsage, { ownsTerminal: runtimeIntent.ownsTerminal !== false });
+                });
+            }
+            const taskSessionIntent = intent.taskSession;
+            if (taskSessionIntent?.operation === "clear") {
+                await runClaimedEffect("task_session", () => clearTaskSessions(taskSessionIntent.orgId, taskSessionIntent.agentId, {
+                    taskKey: taskSessionIntent.taskKey,
+                    agentRuntimeType: taskSessionIntent.agentRuntimeType,
+                    sourceRunId: taskSessionIntent.lastRunId ?? current.id,
+                }));
+            }
+            else if (taskSessionIntent?.operation === "upsert") {
+                await runClaimedEffect("task_session", () => upsertTaskSession({
+                    orgId: taskSessionIntent.orgId,
+                    agentId: taskSessionIntent.agentId,
+                    agentRuntimeType: taskSessionIntent.agentRuntimeType,
+                    taskKey: taskSessionIntent.taskKey,
+                    sessionParamsJson: taskSessionIntent.sessionParamsJson ?? null,
+                    sessionDisplayId: taskSessionIntent.sessionDisplayId ?? null,
+                    lastRunId: taskSessionIntent.lastRunId ?? current.id,
+                    lastError: taskSessionIntent.lastError ?? null,
+                }));
+            }
+            if (intent.processLossRetry) {
+                await runClaimedEffect("process_loss_retry", () => {
+                    if (!agent)
+                        throw new Error("Agent not found while replaying process-loss retry");
+                    return enqueueProcessLossRetry(current, agent, current.finishedAt ?? new Date());
+                });
+            }
+            await runClaimedEffect("issue_release", () => releaseIssueExecutionAndPromote(current, { startNext: false }));
+            const requiredEffects = terminalEffectNames(intent);
+            if (!requiredEffects.every((effect) => completedEffects.has(effect) || deadLetteredEffects.has(effect))) {
+                throw new Error("Heartbeat terminal effects did not converge");
+            }
+            const completed = await completeHeartbeatRunTerminalEffects(db, current.id, claim.claimToken);
+            if (!completed)
+                return;
+            if (completed.agent) {
+                publishLiveEvent({
+                    orgId: completed.agent.orgId,
+                    type: "agent.status",
+                    payload: {
+                        agentId: completed.agent.id,
+                        status: completed.agent.status,
+                        lastHeartbeatAt: completed.agent.lastHeartbeatAt
+                            ? new Date(completed.agent.lastHeartbeatAt).toISOString()
+                            : null,
+                    },
+                });
+            }
+            if (opts?.startNext !== false)
+                await resumeQueuedRuns();
+            return completed;
+        }
+        catch (error) {
+            if (!claimReleased) {
+                await releaseHeartbeatRunTerminalEffectsClaim(db, current.id, claim.claimToken, error);
+            }
+            throw error;
+        }
+    }
     async function startNextQueuedRunForAgent(agentId) {
         return withAgentStartLock(agentId, async () => {
             const agent = await getAgent(agentId);
@@ -961,6 +1319,14 @@ export function heartbeatService(db) {
             if (agent.status === "paused" || agent.status === "terminated" || agent.status === "pending_approval") {
                 return [];
             }
+            const pendingTerminalEffects = await db
+                .select({ id: heartbeatRuns.id })
+                .from(heartbeatRuns)
+                .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.terminalEffectsPending, true)))
+                .limit(1)
+                .then((rows) => rows[0] ?? null);
+            if (pendingTerminalEffects)
+                return [];
             const policy = parseHeartbeatPolicy(agent);
             const runningCount = await countRunningRunsForAgent(agentId);
             const availableSlots = Math.max(0, policy.maxConcurrentRuns - runningCount);
@@ -984,27 +1350,51 @@ export function heartbeatService(db) {
                 return [];
             const claimedRuns = [];
             for (const queuedRun of queuedRuns) {
-                const claimed = await claimQueuedRun(queuedRun);
-                if (claimed)
-                    claimedRuns.push(claimed);
+                if (activeRunExecutions.has(queuedRun.id))
+                    continue;
+                activeRunExecutions.add(queuedRun.id);
+                runAbortControllers.set(queuedRun.id, new AbortController());
+                try {
+                    const claimed = await claimQueuedRun(queuedRun);
+                    if (claimed) {
+                        claimedRuns.push(claimed);
+                        void executeRun(claimed.id, { executionReserved: true }).catch((err) => {
+                            logger.error({ err, runId: claimed.id }, "queued heartbeat execution failed");
+                        });
+                    }
+                    else {
+                        runAbortControllers.delete(queuedRun.id);
+                        activeRunExecutions.delete(queuedRun.id);
+                    }
+                }
+                catch (error) {
+                    runAbortControllers.delete(queuedRun.id);
+                    activeRunExecutions.delete(queuedRun.id);
+                    throw error;
+                }
             }
             if (claimedRuns.length === 0)
                 return [];
-            for (const claimedRun of claimedRuns) {
-                void executeRun(claimedRun.id).catch((err) => {
-                    logger.error({ err, runId: claimedRun.id }, "queued heartbeat execution failed");
-                });
-            }
             return claimedRuns;
         });
     }
     const baseContext = {
-        db, instanceSettings, getCurrentUserRedactionOptions, runLogStore, runContextSvc, issuesSvc, executionWorkspacesSvc, workspaceOperationsSvc, activeRunExecutions, budgetHooks, budgets,
-        getAgent, getRun, getRuntimeState, getTaskSession, getLatestRunForSession, getOldestRunForSession, resolveNormalizedUsageForSession, evaluateSessionCompaction, resolveSessionBeforeForWakeup, resolveExplicitResumeSessionOverride, upsertTaskSession, clearTaskSessions, ensureRuntimeState, setRunStatus, setWakeupStatus, updateWakeupRequestRecord, insertWakeupRequestRecord, appendRunEvent, nextRunEventSeq, persistRunProcessMetadata, clearDetachedRunWarning, countRunningRunsForAgent, claimQueuedRun, finalizeAgentStatus, reapOrphanedRuns, reapInactiveRuns, reapTimedOutRuns, resumeQueuedRuns, updateRuntimeState, startNextQueuedRunForAgent,
+        db, instanceSettings, getCurrentUserRedactionOptions, runLogStore, runContextSvc, issuesSvc, executionWorkspacesSvc, workspaceOperationsSvc, activeRunExecutions, runAbortControllers, budgetHooks, budgets,
+        getAgent, getRun, getRuntimeState, getTaskSession, getLatestRunForSession, getOldestRunForSession, resolveNormalizedUsageForSession, evaluateSessionCompaction, resolveSessionBeforeForWakeup, resolveExplicitResumeSessionOverride, upsertTaskSession, clearTaskSessions, ensureRuntimeState, setRunStatus, transitionRunToTerminal, reconcileRunEvidence, reconcileTerminalEffectsIntent, setWakeupStatus, updateWakeupRequestRecord, insertWakeupRequestRecord, appendRunEvent, persistRunProcessMetadata, clearDetachedRunWarning, terminateRunProcessAndWait, acknowledgeRunProcessExit, renewRunExecutionLease, countRunningRunsForAgent, claimQueuedRun, finalizeAgentStatus, completeTerminalControlEffects, reapOrphanedRuns, reapInactiveRuns, reapTimedOutRuns, resumeQueuedRuns, updateRuntimeState, startNextQueuedRunForAgent,
     };
     const recoveryHandlers = createHeartbeatRecoveryHandlers({ ...baseContext, startNextQueuedRunForAgent });
-    const wakeupHandlers = createHeartbeatWakeupHandlers({ ...baseContext, ...recoveryHandlers, startNextQueuedRunForAgent });
-    const releaseHandlers = createHeartbeatReleaseHandlers({ ...baseContext, ...recoveryHandlers, ...wakeupHandlers });
+    const wakeupHandlers = createHeartbeatWakeupHandlers({
+        ...baseContext,
+        ...recoveryHandlers,
+        startNextQueuedRunForAgent,
+        afterIssuePromotionCommitted: testHooks?.afterIssuePromotionCommitted,
+    });
+    const releaseHandlers = createHeartbeatReleaseHandlers({
+        ...baseContext,
+        ...recoveryHandlers,
+        ...wakeupHandlers,
+        afterIssuePromotionCommitted: testHooks?.afterIssuePromotionCommitted,
+    });
     const executeHandlers = createHeartbeatExecuteHandlers({ ...baseContext, ...recoveryHandlers, ...releaseHandlers, ...wakeupHandlers });
     const miscHandlers = createHeartbeatMiscHandlers({ ...baseContext, ...recoveryHandlers, ...releaseHandlers, ...wakeupHandlers, ...executeHandlers });
     const { enqueueRecoveryRun, enqueueProcessLossRetry, evaluatePassiveIssueClosureForLockedIssue, parseHeartbeatPolicy } = recoveryHandlers;
@@ -1058,7 +1448,7 @@ export function heartbeatService(db) {
                         usedSkillsByRun.set(event.runId, runSkills);
                 }
             }
-            return rows.map((row) => ({
+            return rows.map((row) => toHeartbeatRun({
                 ...row,
                 resultJson: (() => {
                     const summary = summarizeHeartbeatRunResultJson(row.resultJson);
@@ -1249,7 +1639,7 @@ export function heartbeatService(db) {
             const [run] = await db
                 .select()
                 .from(heartbeatRuns)
-                .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "running")))
+                .where(and(eq(heartbeatRuns.agentId, agentId), or(eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.terminalEffectsPending, true))))
                 .orderBy(desc(heartbeatRuns.startedAt))
                 .limit(1);
             return run ?? null;