@rudderhq/server 0.7.2 → 0.7.4

package/dist/services/runtime-kernel/heartbeat.js

@@ -1,5 +1,5 @@
 import { hasSessionCompactionThresholds } from "@rudderhq/agent-runtime-utils";
-import { agentRuntimeState, agents, agentTaskSessions, agentWakeupRequests, heartbeatRunEvents, heartbeatRuns, issues, organizations } from "@rudderhq/db";
+import { agentRuntimeState, agents, agentTaskSessions, agentWakeupRequests, goals, heartbeatRunEvents, heartbeatRuns, issues, organizations } from "@rudderhq/db";
 import { summarizeTokenUsage, toHeartbeatRun } from "@rudderhq/shared";
 import { and, asc, desc, eq, gt, gte, inArray, lte, ne, or, sql } from "drizzle-orm";
 import { randomUUID } from "node:crypto";
@@ -8,6 +8,7 @@ import { killChildProcessTree, parseObject } from "../../agent-runtimes/utils.js
 import { notFound } from "../../errors.js";
 import { redactCurrentUserText, redactCurrentUserValue } from "../../log-redaction.js";
 import { logger } from "../../middleware/logger.js";
+import { agentIssueCreationService, getAgentIssueCreationRequestIdFromRunContext, } from "../agent-issue-creation.js";
 import { agentRunContextService } from "../agent-run-context.js";
 import { approvalService } from "../approvals.js";
 import { publishAutomationRunOutputToChat } from "../automation-chat-output.js";
@@ -105,6 +106,7 @@ export function heartbeatService(db, testHooks) {
     const runContextSvc = agentRunContextService(db);
     const approvalsSvc = approvalService(db);
     const issuesSvc = issueService(db);
+    const agentIssueCreationSvc = agentIssueCreationService(db);
     const executionWorkspacesSvc = runWorkspaceService(db);
     const workspaceOperationsSvc = workspaceOperationService(db);
     const budgetHooks = {
@@ -493,6 +495,44 @@ export function heartbeatService(db, testHooks) {
         return updated;
     }
     async function transitionRunToTerminal(runId, status, patch, opts) {
+        const runBeforeTerminal = await getRun(runId);
+        const agentIssueCreationSettlement = runBeforeTerminal
+            ? await (async () => {
+                const contextRequestId = getAgentIssueCreationRequestIdFromRunContext(runBeforeTerminal.contextSnapshot);
+                if (contextRequestId) {
+                    return {
+                        orgId: runBeforeTerminal.orgId,
+                        agentId: runBeforeTerminal.agentId,
+                        runId: runBeforeTerminal.id,
+                        requestId: contextRequestId,
+                    };
+                }
+                return await agentIssueCreationSvc
+                    .getSettlementIntentForRun(runBeforeTerminal.orgId, runBeforeTerminal.agentId, runBeforeTerminal.id)
+                    .catch((error) => {
+                    logger.warn({ err: error, runId }, "failed to resolve Agent Issue creation settlement terminal effect");
+                    return null;
+                });
+            })()
+            : null;
+        const agentIssueCreationNotification = runBeforeTerminal
+            ? await agentIssueCreationSvc
+                .getNotificationIntentForRun(runBeforeTerminal.orgId, runBeforeTerminal.agentId, runId)
+                .catch((error) => {
+                logger.warn({ err: error, runId }, "failed to resolve Agent Issue creation notification terminal effect");
+                return null;
+            })
+            : null;
+        const terminalEffectsIntent = {
+            version: 1,
+            ...(opts?.terminalEffectsIntent ?? {}),
+            ...(agentIssueCreationSettlement
+                ? { agentIssueCreationSettlement }
+                : {}),
+            ...(agentIssueCreationNotification
+                ? { agentIssueCreationNotification }
+                : {}),
+        };
         const updated = await transitionHeartbeatRunToTerminal(db, {
             runId,
             status,
@@ -500,12 +540,43 @@ export function heartbeatService(db, testHooks) {
             expectedStatuses: opts?.expectedStatuses,
             activityWatermark: opts?.activityWatermark,
             terminalEffectsPending: opts?.terminalEffectsPending,
-            terminalEffectsIntent: opts?.terminalEffectsIntent,
+            terminalEffectsIntent,
             processExitedAt: opts?.processExitedAt,
             expectedExecutionOwnerToken: opts?.expectedExecutionOwnerToken,
         });
-        if (updated)
+        if (updated) {
             publishRunStatus(updated);
+            const context = parseObject(updated.contextSnapshot);
+            const requestId = agentIssueCreationSettlement?.requestId
+                ?? getAgentIssueCreationRequestIdFromRunContext(context);
+            if (requestId) {
+                const settledRequest = await agentIssueCreationSvc.settleForRun({
+                    orgId: updated.orgId,
+                    agentId: updated.agentId,
+                    runId: updated.id,
+                    requestId,
+                    runStatus: updated.status,
+                    error: updated.error,
+                }).catch((error) => {
+                    logger.warn({ err: error, orgId: updated.orgId, agentId: updated.agentId, runId: updated.id, requestId }, "failed to settle Agent Issue creation request after run termination");
+                    return null;
+                });
+                if (settledRequest?.status === "succeeded" && settledRequest.createdIssueId) {
+                    await reconcileTerminalEffectsIntent(updated.id, {
+                        version: 1,
+                        agentIssueCreationNotification: {
+                            orgId: updated.orgId,
+                            agentId: updated.agentId,
+                            runId: updated.id,
+                            requestId: settledRequest.id,
+                            issueId: settledRequest.createdIssueId,
+                        },
+                    }).catch((error) => {
+                        logger.warn({ err: error, orgId: updated.orgId, agentId: updated.agentId, runId: updated.id, requestId }, "failed to record Agent Issue creation notification terminal effect");
+                    });
+                }
+            }
+        }
         return updated;
     }
     async function reconcileRunEvidence(runId, patch) {
@@ -846,6 +917,30 @@ export function heartbeatService(db, testHooks) {
             if (!currentRun || currentRun.status !== "queued")
                 return null;
             const currentContext = parseObject(currentRun.contextSnapshot);
+            const goalContext = parseObject(currentContext.goal);
+            const goalId = readNonEmptyString(currentContext.goalId) ?? readNonEmptyString(goalContext.id);
+            const wakeReason = readNonEmptyString(currentContext.wakeReason) ?? readNonEmptyString(wakeup?.reason);
+            if (currentRun.wakeupRequestId
+                && goalId
+                && wakeReason
+                && ["goal_started", "goal_feedback", "goal_change_decided"].includes(wakeReason)) {
+                await tx.execute(sql `select id from organizations where id = ${currentRun.orgId} for update`);
+                const focusedGoal = await tx.select({ id: goals.id }).from(goals).where(and(eq(goals.orgId, currentRun.orgId), eq(goals.lifecycle, "active"), eq(goals.focus, true))).limit(1).then((rows) => rows[0] ?? null);
+                if (focusedGoal && focusedGoal.id !== goalId) {
+                    const deferred = await tx.update(agentWakeupRequests).set({
+                        status: "deferred_goal_focus",
+                        runId: null,
+                        claimedAt: null,
+                        finishedAt: null,
+                        error: "goal.focused_elsewhere",
+                        updatedAt: new Date(),
+                    }).where(and(eq(agentWakeupRequests.id, currentRun.wakeupRequestId), eq(agentWakeupRequests.runId, currentRun.id), eq(agentWakeupRequests.status, "queued"))).returning().then((rows) => rows[0] ?? null);
+                    if (!deferred)
+                        return null;
+                    await tx.delete(heartbeatRuns).where(and(eq(heartbeatRuns.id, currentRun.id), eq(heartbeatRuns.status, "queued")));
+                    return null;
+                }
+            }
             const currentTaskKey = deriveTaskKey(currentContext, null);
             const resetTaskSession = shouldResetTaskSessionForWake(currentContext);
             const sessionCodec = getAgentRuntimeSessionCodec(agent.agentRuntimeType);
@@ -983,130 +1078,139 @@ export function heartbeatService(db, testHooks) {
         const staleThresholdMs = opts?.staleThresholdMs ?? 0;
         const now = opts?.now ?? new Date();
         const recoveryCutoff = opts?.recoveryCutoff ?? now;
-        // Find all runs stuck in "running" state (queued runs are legitimately waiting; resumeQueuedRuns handles them)
-        const activeRuns = await db
-            .select({
-            run: heartbeatRuns,
-            agentRuntimeType: agents.agentRuntimeType,
-        })
-            .from(heartbeatRuns)
-            .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
-            .where(or(eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.terminalEffectsPending, true)));
         const reaped = [];
-        for (const { run, agentRuntimeType } of activeRuns) {
-            if (run.terminalEffectsPending) {
-                if (!run.processExitedAt) {
-                    const exited = await terminateRunProcessAndWait(run, agentRuntimeType);
-                    if (!exited || activeRunExecutions.has(run.id))
+        try {
+            // Find all runs stuck in "running" state (queued runs are legitimately waiting; resumeQueuedRuns handles them)
+            const activeRuns = await db
+                .select({
+                run: heartbeatRuns,
+                agentRuntimeType: agents.agentRuntimeType,
+            })
+                .from(heartbeatRuns)
+                .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
+                .where(or(eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.terminalEffectsPending, true)));
+            for (const { run, agentRuntimeType } of activeRuns) {
+                if (run.terminalEffectsPending) {
+                    if (!run.processExitedAt) {
+                        const exited = await terminateRunProcessAndWait(run, agentRuntimeType);
+                        if (!exited || activeRunExecutions.has(run.id))
+                            continue;
+                        await acknowledgeRunProcessExit(run.id);
+                    }
+                    const completed = await completeTerminalControlEffects(run);
+                    if (!completed)
                         continue;
-                    await acknowledgeRunProcessExit(run.id);
-                }
-                const completed = await completeTerminalControlEffects(run);
-                if (!completed)
-                    continue;
-                reaped.push(run.id);
-                continue;
-            }
-            // Apply staleness threshold to avoid false positives
-            if (staleThresholdMs > 0) {
-                const refTime = run.updatedAt ? new Date(run.updatedAt).getTime() : 0;
-                if (now.getTime() - refTime < staleThresholdMs)
-                    continue;
-            }
-            // The lease timer pauses while the host sleeps, but the child process
-            // and this server's execution registry can remain alive. Preserve that
-            // local execution instead of turning it into a process-loss retry.
-            if (activeRunExecutions.has(run.id) || runningProcesses.has(run.id)) {
-                if (!localExecutionOwnerMatches(run.id, run.executionOwnerToken)) {
-                    abortRunExecution(run.id);
+                    reaped.push(run.id);
                     continue;
                 }
-                if (run.executionOwnerToken) {
-                    const renewed = await renewRunExecutionLease(run.id, run.executionOwnerToken, now);
-                    if (!renewed)
+                // Apply staleness threshold to avoid false positives
+                if (staleThresholdMs > 0) {
+                    const refTime = run.updatedAt ? new Date(run.updatedAt).getTime() : 0;
+                    if (now.getTime() - refTime < staleThresholdMs)
+                        continue;
+                }
+                // The lease timer pauses while the host sleeps, but the child process
+                // and this server's execution registry can remain alive. Preserve that
+                // local execution instead of turning it into a process-loss retry.
+                if (activeRunExecutions.has(run.id) || runningProcesses.has(run.id)) {
+                    if (!localExecutionOwnerMatches(run.id, run.executionOwnerToken)) {
                         abortRunExecution(run.id);
+                        continue;
+                    }
+                    if (run.executionOwnerToken) {
+                        const renewed = await renewRunExecutionLease(run.id, run.executionOwnerToken, now);
+                        if (!renewed)
+                            abortRunExecution(run.id);
+                    }
+                    continue;
                 }
-                continue;
-            }
-            const recoveryClaim = await claimExpiredHeartbeatRunExecution(db, run.id, { now, recoveryCutoff });
-            if (!recoveryClaim)
-                continue;
-            const claimedRun = recoveryClaim.run;
-            const tracksLocalChild = isTrackedLocalChildProcessAdapter(agentRuntimeType);
-            let detachedTerminationMessage = null;
-            if (!claimedRun.processExitedAt && tracksLocalChild && claimedRun.processPid && isProcessAlive(claimedRun.processPid)) {
-                const termination = await terminateOrphanedProcess(claimedRun.processPid);
-                if (termination.stillAlive) {
-                    const detachedMessage = termination.error
-                        ? `Lost in-memory process handle, child pid ${claimedRun.processPid} is still alive, and Rudder could not terminate it: ${termination.error}`
-                        : `Lost in-memory process handle, but child pid ${claimedRun.processPid} is still alive`;
-                    const detachedRun = await setRunStatus(claimedRun.id, "running", {
-                        error: detachedMessage,
-                        errorCode: DETACHED_PROCESS_ERROR_CODE,
-                    });
-                    if (detachedRun) {
-                        await appendRunEvent(detachedRun, {
-                            eventType: "lifecycle",
-                            stream: "system",
-                            level: "warn",
-                            message: detachedMessage,
-                            payload: {
-                                processPid: claimedRun.processPid,
-                            },
+                const recoveryClaim = await claimExpiredHeartbeatRunExecution(db, run.id, { now, recoveryCutoff });
+                if (!recoveryClaim)
+                    continue;
+                const claimedRun = recoveryClaim.run;
+                const tracksLocalChild = isTrackedLocalChildProcessAdapter(agentRuntimeType);
+                let detachedTerminationMessage = null;
+                if (!claimedRun.processExitedAt && tracksLocalChild && claimedRun.processPid && isProcessAlive(claimedRun.processPid)) {
+                    const termination = await terminateOrphanedProcess(claimedRun.processPid);
+                    if (termination.stillAlive) {
+                        const detachedMessage = termination.error
+                            ? `Lost in-memory process handle, child pid ${claimedRun.processPid} is still alive, and Rudder could not terminate it: ${termination.error}`
+                            : `Lost in-memory process handle, but child pid ${claimedRun.processPid} is still alive`;
+                        const detachedRun = await setRunStatus(claimedRun.id, "running", {
+                            error: detachedMessage,
+                            errorCode: DETACHED_PROCESS_ERROR_CODE,
                         });
+                        if (detachedRun) {
+                            await appendRunEvent(detachedRun, {
+                                eventType: "lifecycle",
+                                stream: "system",
+                                level: "warn",
+                                message: detachedMessage,
+                                payload: {
+                                    processPid: claimedRun.processPid,
+                                },
+                            });
+                        }
+                        continue;
                     }
+                    detachedTerminationMessage = termination.terminationSignal
+                        ? `Terminated detached child pid ${claimedRun.processPid} with ${termination.terminationSignal} after Rudder lost its process handle`
+                        : `Detached child pid ${claimedRun.processPid} exited before Rudder could terminate it`;
+                }
+                const shouldRetry = tracksLocalChild && !!claimedRun.processPid && (claimedRun.processLossRetryCount ?? 0) < 1;
+                const baseMessage = claimedRun.processPid
+                    ? `Process lost -- child pid ${claimedRun.processPid} is no longer running`
+                    : "Process lost -- server may have restarted";
+                let finalizedRun = await transitionRunToTerminal(claimedRun.id, "failed", {
+                    error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
+                    errorCode: "process_lost",
+                    finishedAt: now,
+                }, {
+                    processExitedAt: now,
+                    terminalEffectsIntent: shouldRetry ? { version: 1, processLossRetry: true } : { version: 1 },
+                    expectedExecutionOwnerToken: recoveryClaim.ownerToken,
+                });
+                if (!finalizedRun)
                     continue;
+                await setWakeupStatus(claimedRun.wakeupRequestId, "failed", {
+                    finishedAt: now,
+                    error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
+                });
+                if (detachedTerminationMessage) {
+                    await appendRunEvent(finalizedRun, {
+                        eventType: "lifecycle",
+                        stream: "system",
+                        level: "warn",
+                        message: detachedTerminationMessage,
+                        payload: {
+                            ...(claimedRun.processPid ? { processPid: claimedRun.processPid } : {}),
+                        },
+                    });
                 }
-                detachedTerminationMessage = termination.terminationSignal
-                    ? `Terminated detached child pid ${claimedRun.processPid} with ${termination.terminationSignal} after Rudder lost its process handle`
-                    : `Detached child pid ${claimedRun.processPid} exited before Rudder could terminate it`;
-            }
-            const shouldRetry = tracksLocalChild && !!claimedRun.processPid && (claimedRun.processLossRetryCount ?? 0) < 1;
-            const baseMessage = claimedRun.processPid
-                ? `Process lost -- child pid ${claimedRun.processPid} is no longer running`
-                : "Process lost -- server may have restarted";
-            let finalizedRun = await transitionRunToTerminal(claimedRun.id, "failed", {
-                error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
-                errorCode: "process_lost",
-                finishedAt: now,
-            }, {
-                processExitedAt: now,
-                terminalEffectsIntent: shouldRetry ? { version: 1, processLossRetry: true } : { version: 1 },
-                expectedExecutionOwnerToken: recoveryClaim.ownerToken,
-            });
-            if (!finalizedRun)
-                continue;
-            await setWakeupStatus(claimedRun.wakeupRequestId, "failed", {
-                finishedAt: now,
-                error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
-            });
-            if (detachedTerminationMessage) {
                 await appendRunEvent(finalizedRun, {
                     eventType: "lifecycle",
                     stream: "system",
-                    level: "warn",
-                    message: detachedTerminationMessage,
+                    level: "error",
+                    message: shouldRetry
+                        ? `${baseMessage}; retry will be queued after terminal effects complete`
+                        : baseMessage,
                     payload: {
                         ...(claimedRun.processPid ? { processPid: claimedRun.processPid } : {}),
                     },
                 });
+                const completed = await completeTerminalControlEffects(finalizedRun);
+                runningProcesses.delete(claimedRun.id);
+                if (!completed)
+                    continue;
+                reaped.push(claimedRun.id);
             }
-            await appendRunEvent(finalizedRun, {
-                eventType: "lifecycle",
-                stream: "system",
-                level: "error",
-                message: shouldRetry
-                    ? `${baseMessage}; retry will be queued after terminal effects complete`
-                    : baseMessage,
-                payload: {
-                    ...(claimedRun.processPid ? { processPid: claimedRun.processPid } : {}),
-                },
+        }
+        finally {
+            // Terminal effects are the fast path. Keep a request-scoped sweep as a
+            // second durable path for runs whose effect was lost or dead-lettered.
+            await agentIssueCreationSvc.reconcileTerminalSettlements().catch((error) => {
+                logger.warn({ err: error }, "failed to reconcile terminal Agent Issue creation requests");
             });
-            const completed = await completeTerminalControlEffects(finalizedRun);
-            runningProcesses.delete(claimedRun.id);
-            if (!completed)
-                continue;
-            reaped.push(claimedRun.id);
         }
         if (reaped.length > 0) {
             logger.warn({ reapedCount: reaped.length, runIds: reaped }, "reaped orphaned heartbeat runs");
@@ -1283,7 +1387,9 @@ export function heartbeatService(db, testHooks) {
     async function reapTimedOutRuns(opts) {
         return withHeartbeatRecoveryLock(() => reapTimedOutRunsLocked(opts));
     }
+    let recoverPendingWakeups = null;
     async function resumeQueuedRuns() {
+        await recoverPendingWakeups?.();
         const queuedRuns = await db
             .select({ agentId: heartbeatRuns.agentId })
             .from(heartbeatRuns)
@@ -1473,6 +1579,45 @@ export function heartbeatService(db, testHooks) {
                     lastError: taskSessionIntent.lastError ?? null,
                 }));
             }
+            const agentIssueCreationSettlement = intent.agentIssueCreationSettlement;
+            if (agentIssueCreationSettlement) {
+                await runClaimedEffect("agent_issue_creation_settlement", async () => {
+                    const settledRequest = await agentIssueCreationSvc.settleForRun({
+                        orgId: agentIssueCreationSettlement.orgId,
+                        agentId: agentIssueCreationSettlement.agentId,
+                        runId: agentIssueCreationSettlement.runId,
+                        requestId: agentIssueCreationSettlement.requestId,
+                        runStatus: current.status,
+                        error: current.error,
+                    });
+                    const request = settledRequest
+                        ?? await agentIssueCreationSvc.getById(agentIssueCreationSettlement.orgId, agentIssueCreationSettlement.requestId);
+                    if (request?.status === "succeeded" && request.createdIssueId) {
+                        const notificationIntent = {
+                            orgId: agentIssueCreationSettlement.orgId,
+                            agentId: agentIssueCreationSettlement.agentId,
+                            runId: agentIssueCreationSettlement.runId,
+                            requestId: request.id,
+                            issueId: request.createdIssueId,
+                        };
+                        intent.agentIssueCreationNotification ??= notificationIntent;
+                        await reconcileTerminalEffectsIntent(current.id, {
+                            version: 1,
+                            agentIssueCreationNotification: notificationIntent,
+                        });
+                    }
+                    else if (request
+                        && !request.createdIssueId
+                        && ["queued", "running", "deferred"].includes(request.status)) {
+                        throw new Error("Agent Issue creation request remained active after terminal settlement");
+                    }
+                    return settledRequest;
+                });
+            }
+            const agentIssueCreationNotification = intent.agentIssueCreationNotification;
+            if (agentIssueCreationNotification) {
+                await runClaimedEffect("agent_issue_creation_notification", () => agentIssueCreationSvc.publishNotificationForRequest(agentIssueCreationNotification));
+            }
             if (intent.processLossRetry) {
                 await runClaimedEffect("process_loss_retry", () => {
                     if (!agent)
@@ -1556,6 +1701,7 @@ export function heartbeatService(db, testHooks) {
                 activeRunExecutions.add(queuedRun.id);
                 runAbortControllers.set(queuedRun.id, new AbortController());
                 try {
+                    await testHooks?.beforeRunClaim?.(queuedRun);
                     const claimed = await claimQueuedRun(queuedRun);
                     if (claimed) {
                         claimedRuns.push(claimed);
@@ -1590,6 +1736,7 @@ export function heartbeatService(db, testHooks) {
         startNextQueuedRunForAgent,
         afterIssuePromotionCommitted: testHooks?.afterIssuePromotionCommitted,
     });
+    recoverPendingWakeups = wakeupHandlers.recoverPendingWakeups;
     const releaseHandlers = createHeartbeatReleaseHandlers({
         ...baseContext,
         ...recoveryHandlers,
@@ -1608,6 +1755,69 @@ export function heartbeatService(db, testHooks) {
     const { releaseIssueExecutionAndPromote } = releaseHandlers;
     const { executeRun } = executeHandlers;
     const { resumeDeferredWakeupsForAgent, listProjectScopedRunIds, listProjectScopedWakeupIds, cancelPendingWakeupsForBudgetScope, cancelRunInternal, cancelActiveForAgentInternal, cancelBudgetScopeWork, retryRunInternal, buildSkillAnalytics } = miscHandlers;
+    async function resumePendingWakeupRequests(opts = {}) {
+        const pendingWakeups = await db.select().from(agentWakeupRequests).where(and(opts.orgId ? eq(agentWakeupRequests.orgId, opts.orgId) : undefined, inArray(agentWakeupRequests.status, [
+            "queued",
+            "deferred_agent_paused",
+            "deferred_goal_focus",
+            "deferred_goal_blocked",
+        ]), sql `${agentWakeupRequests.runId} is null`, lte(agentWakeupRequests.requestedAt, new Date()))).orderBy(asc(agentWakeupRequests.requestedAt)).limit(100);
+        let resumed = 0;
+        let deferred = 0;
+        let terminalized = 0;
+        for (const pending of pendingWakeups) {
+            const source = ["timer", "assignment", "review", "on_demand", "automation"].includes(pending.source)
+                ? pending.source
+                : "on_demand";
+            const triggerDetail = pending.triggerDetail
+                && ["manual", "ping", "callback", "system"].includes(pending.triggerDetail)
+                ? pending.triggerDetail
+                : undefined;
+            const requestedByActorType = pending.requestedByActorType
+                && ["user", "agent", "system"].includes(pending.requestedByActorType)
+                ? pending.requestedByActorType
+                : undefined;
+            try {
+                const run = await enqueueWakeup(pending.agentId, {
+                    existingWakeupRequestId: pending.id,
+                    source,
+                    triggerDetail,
+                    reason: pending.reason,
+                    payload: readDeferredWakePayload(pending.payload),
+                    contextSnapshot: readDeferredWakeContext(pending.payload),
+                    requestedByActorType,
+                    requestedByActorId: pending.requestedByActorId,
+                    idempotencyKey: pending.idempotencyKey,
+                    startImmediately: opts.startImmediately,
+                });
+                if (run)
+                    resumed += 1;
+                else
+                    deferred += 1;
+            }
+            catch (error) {
+                const current = await db.select({ status: agentWakeupRequests.status })
+                    .from(agentWakeupRequests)
+                    .where(eq(agentWakeupRequests.id, pending.id))
+                    .then((rows) => rows[0] ?? null);
+                if (current && [
+                    "queued",
+                    "deferred_agent_paused",
+                    "deferred_goal_focus",
+                    "deferred_goal_blocked",
+                ].includes(current.status)) {
+                    await setWakeupStatus(pending.id, "failed", {
+                        finishedAt: new Date(),
+                        error: error instanceof Error ? error.message : String(error),
+                        runId: null,
+                        claimedAt: null,
+                    });
+                }
+                terminalized += 1;
+            }
+        }
+        return { checked: pendingWakeups.length, resumed, deferred, terminalized };
+    }
     return {
         overview: async (orgId) => {
             const [latestRows, recentRows] = await Promise.all([
@@ -1827,6 +2037,8 @@ export function heartbeatService(db, testHooks) {
         reapOrphanedRuns,
         reapInactiveRuns,
         reapTimedOutRuns,
+        startNextQueuedRunForAgent,
+        resumePendingWakeupRequests,
         resumeQueuedRuns,
         tickTimers: async (now = new Date()) => {
             const allAgents = await db.select().from(agents);