@rudderhq/server 0.4.6-canary.9 → 0.5.0

package/dist/services/runtime-kernel/heartbeat.execute.js

@@ -1,13 +1,12 @@
-import { agents, heartbeatRuns, issues, projects } from "@rudderhq/db";
+import { heartbeatRuns, issues, projects } from "@rudderhq/db";
 import { resolveAgentRunScene, } from "@rudderhq/shared";
-import { and, eq } from "drizzle-orm";
+import { and, eq, sql } from "drizzle-orm";
 import { createLocalAgentJwt } from "../../agent-auth-jwt.js";
 import { findServerAdapter, getServerAdapter } from "../../agent-runtimes/index.js";
 import { parseObject } from "../../agent-runtimes/utils.js";
 import { resolveDefaultAgentWorkspaceDir, } from "../../home-paths.js";
 import { redactCurrentUserText } from "../../log-redaction.js";
 import { logger } from "../../middleware/logger.js";
-import { publishAutomationRunOutputToChat } from "../automation-chat-output.js";
 import { buildExecutionWorkspaceAdapterConfig, issueExecutionWorkspaceModeForPersistedWorkspace, parseIssueExecutionWorkspaceSettings, parseProjectExecutionWorkspacePolicy, resolveExecutionWorkspaceMode, } from "../execution-workspace-policy.js";
 import { summarizeHeartbeatRunResultJson } from "../heartbeat-run-summary.js";
 import { publishLiveEvent } from "../live-events.js";
@@ -18,10 +17,25 @@ export { prioritizeProjectWorkspaceCandidatesForRun } from "../agent-run-context
 import * as heartbeatCore from "./heartbeat.core.js";
 import * as heartbeatSessions from "./heartbeat.sessions.js";
 const { MAX_LIVE_LOG_CHUNK_BYTES, HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT, HEARTBEAT_MAX_CONCURRENT_RUNS_MIN, HEARTBEAT_MAX_CONCURRENT_RUNS_MAX, DEFERRED_WAKE_CONTEXT_KEY, DETACHED_PROCESS_ERROR_CODE, ORPHANED_PROCESS_TERMINATION_GRACE_MS, ORPHANED_PROCESS_KILL_WAIT_MS, ORPHANED_PROCESS_POLL_INTERVAL_MS, startLocksByAgent, MAX_RECOVERY_CHAIN_DEPTH, ISSUE_PASSIVE_FOLLOWUP_REASON, ISSUE_PASSIVE_FOLLOWUP_WAKE_SOURCE, ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON, ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS, ISSUE_REVIEW_CLOSEOUT_REASON, ISSUE_REVIEW_CLOSEOUT_FAILURE_REASON, ISSUE_REVIEW_CLOSEOUT_MAX_ATTEMPTS, ISSUE_PASSIVE_FOLLOWUP_COOLDOWN_MS_BY_ATTEMPT, ISSUE_PASSIVE_FOLLOWUP_TIMER_CONTINUITY_MAX_WINDOW_MS, SESSIONED_LOCAL_ADAPTERS, heartbeatRunListColumns, appendExcerpt, appendTranscriptEntriesFromChunk, normalizeMaxConcurrentRuns, withAgentStartLock, readNonEmptyString, buildHeartbeatAdapterInvokePayload, sanitizeStartupContextContextForPersistence, sanitizeStartupContextPromptForPersistence, buildRecentDateKeys, buildDateKeysBetween, fallbackSkillLabel, normalizeLoadedSkill, normalizeLoadedSkillForPayload, emptySkillEvidenceCounts, incrementSkillEvidenceCount, strongestSkillEvidence, resolveSkillEvidence, readSkillEvidenceFromPayload, extractSkillSlugFromPath, collectSkillPathsFromText, collectStringValues, normalizeSkillUseFromPath, dedupeSkillUses, collectSkillUsesFromText, readToolCommandInput, isCommandTranscriptTool, isReadTranscriptTool, inferUsedSkillsFromTranscript, normalizeSkillCandidate, addSkillCandidate, readSkillReferenceSlug, collectSkillReferences, inferUsedSkillsFromPrompt, resolveForbiddenRuntimeSkillMarkers, detectForbiddenRuntimeSkillMarker, normalizeLedgerBillingType, resolveLedgerBiller, normalizeBilledCostCents, resolveLedgerScopeForRun } = heartbeatCore;
-const { buildExplicitResumeSessionOverride, normalizeUsageTotals, readRawUsageTotals, deriveNormalizedUsageDelta, formatCount, parseSessionCompactionPolicy, resolveRuntimeSessionParamsForWorkspace, parseIssueAssigneeAgentRuntimeOverrides, deriveTaskKey, shouldResetTaskSessionForWake, formatRuntimeWorkspaceWarningLog, describeSessionResetReason, deriveCommentId, enrichWakeContextSnapshot, mergeCoalescedContextSnapshot, issueCommentAuthorKind, issueCommentAuthorLabel, buildDeferredWakePayload, readDeferredWakeContext, readDeferredWakePayload, deriveDeferredWakeTaskKey, hydrateWakeContextSnapshot, firstNonEmptyLine, deriveRecoveryFailureKind, deriveRecoveryFailureSummary, mergeMissingRecoveryContextFields, hydrateRecoveryBaseContextSnapshot, buildRecoveryContextSnapshot, normalizePassiveFollowupContext, normalizeReviewCloseoutContext, passiveFollowupCooldownMs, issueHasReviewer, isAgentEligibleForTimerContinuation, hasCredibleTimerContinuation, buildPassiveFollowupContextSnapshot, runTaskKey, isSameTaskScope, isTrackedLocalChildProcessAdapter, isProcessAlive, waitForProcessExit, terminateOrphanedProcess, truncateDisplayId, normalizeAgentNameKey, defaultSessionCodec, getAgentRuntimeSessionCodec, normalizeSessionParams, resolveNextSessionState } = heartbeatSessions;
+const { buildExplicitResumeSessionOverride, selectRunSessionLineage, normalizeUsageTotals, readRawUsageTotals, deriveNormalizedUsageDelta, formatCount, parseSessionCompactionPolicy, resolveRuntimeSessionParamsForWorkspace, parseIssueAssigneeAgentRuntimeOverrides, deriveTaskKey, shouldResetTaskSessionForWake, formatRuntimeWorkspaceWarningLog, describeSessionResetReason, deriveCommentId, enrichWakeContextSnapshot, mergeCoalescedContextSnapshot, issueCommentAuthorKind, issueCommentAuthorLabel, buildDeferredWakePayload, readDeferredWakeContext, readDeferredWakePayload, deriveDeferredWakeTaskKey, hydrateWakeContextSnapshot, firstNonEmptyLine, deriveRecoveryFailureKind, deriveRecoveryFailureSummary, mergeMissingRecoveryContextFields, hydrateRecoveryBaseContextSnapshot, buildRecoveryContextSnapshot, normalizePassiveFollowupContext, normalizeReviewCloseoutContext, passiveFollowupCooldownMs, issueHasReviewer, isAgentEligibleForTimerContinuation, hasCredibleTimerContinuation, buildPassiveFollowupContextSnapshot, runTaskKey, isSameTaskScope, isTrackedLocalChildProcessAdapter, isProcessAlive, waitForProcessExit, terminateOrphanedProcess, truncateDisplayId, normalizeAgentNameKey, defaultSessionCodec, getAgentRuntimeSessionCodec, normalizeSessionParams, resolveNextSessionState } = heartbeatSessions;
 function buildPersistableHeartbeatContext(context) {
     return sanitizeStartupContextContextForPersistence(context) ?? {};
 }
+const EXECUTOR_OWNED_CONTEXT_KEYS = [
+    "executionWorkspaceId",
+    "rudderGitIdentity",
+    "rudderScene",
+    "rudderWorkspace",
+    "rudderWorkspaces",
+    "rudderStartupContext",
+    "rudderStartupContextMetrics",
+    "rudderRuntimeServiceIntents",
+    "rudderSessionHandoffMarkdown",
+    "rudderSessionRotationReason",
+    "rudderPreviousSessionId",
+    "rudderRuntimeServices",
+    "rudderRuntimePrimaryUrl",
+];
 function resolveRuntimeSceneForRun(run) {
     return resolveAgentRunScene({
         ...run,
@@ -32,37 +46,114 @@ function resolveRuntimeSceneForRun(run) {
     });
 }
 export function createHeartbeatExecuteHandlers(context) {
-    const { db, instanceSettings, getCurrentUserRedactionOptions, runLogStore, runContextSvc, issuesSvc, executionWorkspacesSvc, workspaceOperationsSvc, activeRunExecutions, budgetHooks, budgets, getAgent, getRun, getRuntimeState, getTaskSession, getLatestRunForSession, getOldestRunForSession, resolveNormalizedUsageForSession, evaluateSessionCompaction, resolveSessionBeforeForWakeup, resolveExplicitResumeSessionOverride, upsertTaskSession, clearTaskSessions, ensureRuntimeState, setRunStatus, setWakeupStatus, updateWakeupRequestRecord, insertWakeupRequestRecord, appendRunEvent, nextRunEventSeq, persistRunProcessMetadata, clearDetachedRunWarning, enqueueRecoveryRun, enqueueProcessLossRetry, parseHeartbeatPolicy, markAgentHeartbeatChecked, evaluateTimerPreflight, runHasIssueClosureComment, runHasIssueReviewDecision, issueHasDeferredWake, passiveFollowupAlreadyRecorded, reviewerCloseoutAlreadyRecorded, issueHasRecordedBlockedReviewerDecision, evaluatePassiveIssueClosureForLockedIssue, countRunningRunsForAgent, claimQueuedRun, finalizeAgentStatus, reapOrphanedRuns, resumeQueuedRuns, updateRuntimeState, startNextQueuedRunForAgent, releaseIssueExecutionAndPromote, enqueueWakeup, resumeDeferredWakeupsForAgent, listProjectScopedRunIds, listProjectScopedWakeupIds, cancelPendingWakeupsForBudgetScope, cancelRunInternal, cancelActiveForAgentInternal, cancelBudgetScopeWork, retryRunInternal, buildSkillAnalytics } = context;
-    async function executeRun(runId) {
-        let run = await getRun(runId);
-        if (!run)
+    const { db, instanceSettings, getCurrentUserRedactionOptions, runLogStore, runContextSvc, issuesSvc, executionWorkspacesSvc, workspaceOperationsSvc, activeRunExecutions, runAbortControllers, budgetHooks, budgets, getAgent, getRun, getRuntimeState, getTaskSession, getLatestRunForSession, getOldestRunForSession, resolveNormalizedUsageForSession, evaluateSessionCompaction, resolveSessionBeforeForWakeup, resolveExplicitResumeSessionOverride, upsertTaskSession, clearTaskSessions, ensureRuntimeState, setRunStatus, transitionRunToTerminal, reconcileRunEvidence, reconcileTerminalEffectsIntent, setWakeupStatus, updateWakeupRequestRecord, insertWakeupRequestRecord, appendRunEvent, persistRunProcessMetadata, clearDetachedRunWarning, acknowledgeRunProcessExit, renewRunExecutionLease, enqueueRecoveryRun, enqueueProcessLossRetry, parseHeartbeatPolicy, markAgentHeartbeatChecked, evaluateTimerPreflight, runHasIssueClosureComment, runHasIssueReviewDecision, issueHasDeferredWake, passiveFollowupAlreadyRecorded, reviewerCloseoutAlreadyRecorded, issueHasRecordedBlockedReviewerDecision, evaluatePassiveIssueClosureForLockedIssue, countRunningRunsForAgent, claimQueuedRun, finalizeAgentStatus, completeTerminalControlEffects, reapOrphanedRuns, resumeQueuedRuns, updateRuntimeState, startNextQueuedRunForAgent, releaseIssueExecutionAndPromote, enqueueWakeup, resumeDeferredWakeupsForAgent, listProjectScopedRunIds, listProjectScopedWakeupIds, cancelPendingWakeupsForBudgetScope, cancelRunInternal, cancelActiveForAgentInternal, cancelBudgetScopeWork, retryRunInternal, buildSkillAnalytics } = context;
+    async function persistRunningExecutionContext(runId, desiredContext, patch = {}) {
+        return db.transaction(async (tx) => {
+            await tx.execute(sql `select id from heartbeat_runs where id = ${runId} for update`);
+            const currentRun = await tx
+                .select()
+                .from(heartbeatRuns)
+                .where(eq(heartbeatRuns.id, runId))
+                .then((rows) => rows[0] ?? null);
+            if (!currentRun || currentRun.status !== "running")
+                return null;
+            const persistableContext = buildPersistableHeartbeatContext(desiredContext);
+            const mergedContext = mergeCoalescedContextSnapshot(persistableContext, parseObject(currentRun.contextSnapshot));
+            for (const key of EXECUTOR_OWNED_CONTEXT_KEYS) {
+                if (Object.prototype.hasOwnProperty.call(persistableContext, key)) {
+                    mergedContext[key] = persistableContext[key];
+                }
+                else {
+                    delete mergedContext[key];
+                }
+            }
+            return tx
+                .update(heartbeatRuns)
+                .set({
+                ...patch,
+                contextSnapshot: mergedContext,
+                updatedAt: new Date(),
+            })
+                .where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.status, "running")))
+                .returning()
+                .then((rows) => rows[0] ?? null);
+        });
+    }
+    async function executeRun(runId, opts) {
+        const executionReserved = opts?.executionReserved === true;
+        if (executionReserved && !activeRunExecutions.has(runId))
             return;
-        if (run.status !== "queued" && run.status !== "running")
+        let run;
+        let executionLeaseTimer = null;
+        let executionOwnerToken = null;
+        try {
+            run = await getRun(runId);
+        }
+        catch (error) {
+            if (executionReserved) {
+                runAbortControllers.delete(runId);
+                activeRunExecutions.delete(runId);
+            }
+            throw error;
+        }
+        if (!run || (run.status !== "queued" && run.status !== "running")) {
+            if (executionReserved) {
+                runAbortControllers.delete(runId);
+                activeRunExecutions.delete(runId);
+                if (run?.terminalEffectsPending) {
+                    await acknowledgeRunProcessExit(run.id);
+                    await completeTerminalControlEffects(run);
+                }
+                if (run)
+                    await startNextQueuedRunForAgent(run.agentId);
+            }
             return;
-        if (run.status === "queued") {
-            const claimed = await claimQueuedRun(run);
-            if (!claimed) {
-                // Another worker has already claimed or finalized this run.
+        }
+        if (executionReserved) {
+            if (!activeRunExecutions.has(run.id))
                 return;
-            }
-            run = claimed;
         }
-        activeRunExecutions.add(run.id);
+        else {
+            if (activeRunExecutions.has(run.id))
+                return;
+            activeRunExecutions.add(run.id);
+        }
+        const executionAbortController = runAbortControllers.get(run.id) ?? new AbortController();
+        runAbortControllers.set(run.id, executionAbortController);
         try {
+            if (run.status === "queued") {
+                const claimed = await claimQueuedRun(run);
+                if (!claimed) {
+                    // Another worker has already claimed or finalized this run.
+                    return;
+                }
+                run = claimed;
+            }
+            executionOwnerToken = run.executionOwnerToken;
+            if (executionOwnerToken) {
+                await renewRunExecutionLease(run.id, executionOwnerToken);
+                executionLeaseTimer = setInterval(() => {
+                    void renewRunExecutionLease(run.id, executionOwnerToken).catch(() => undefined);
+                }, 60_000);
+                executionLeaseTimer.unref?.();
+            }
             const agent = await getAgent(run.agentId);
             if (!agent) {
-                await setRunStatus(runId, "failed", {
+                const failed = await transitionRunToTerminal(runId, "failed", {
                     error: "Agent not found",
                     errorCode: "agent_not_found",
                     finishedAt: new Date(),
+                }, {
+                    processExitedAt: new Date(),
+                    expectedExecutionOwnerToken: executionOwnerToken,
                 });
-                await setWakeupStatus(run.wakeupRequestId, "failed", {
-                    finishedAt: new Date(),
-                    error: "Agent not found",
-                });
-                const failedRun = await getRun(runId);
-                if (failedRun)
-                    await releaseIssueExecutionAndPromote(failedRun);
+                if (failed) {
+                    await setWakeupStatus(run.wakeupRequestId, "failed", {
+                        finishedAt: new Date(),
+                        error: "Agent not found",
+                    });
+                    await completeTerminalControlEffects(failed);
+                }
                 return;
             }
             const executionTranscript = [];
@@ -73,8 +164,26 @@ export function createHeartbeatExecuteHandlers(context) {
             let latestAdapterMeta = null;
             let adapterForbiddenMarkerObserved = false;
             let finalRunOutput = null;
-            let finalRunStatus = run.status;
-            const runtime = await ensureRuntimeState(agent);
+            let ownsTerminalState = false;
+            let shouldCompleteTerminalEffects = false;
+            const finalizeExecutionTranscript = () => {
+                stdoutTranscriptBuffer = appendTranscriptEntriesFromChunk({
+                    buffer: stdoutTranscriptBuffer,
+                    chunk: "",
+                    transcript: executionTranscript,
+                    parser: stdoutTranscriptParser,
+                    finalize: true,
+                    kind: "stdout",
+                });
+                stderrTranscriptBuffer = appendTranscriptEntriesFromChunk({
+                    buffer: stderrTranscriptBuffer,
+                    chunk: "",
+                    transcript: executionTranscript,
+                    finalize: true,
+                    kind: "stderr",
+                });
+            };
+            await ensureRuntimeState(agent);
             const context = parseObject(run.contextSnapshot);
             delete context.rudderGitIdentity;
             const taskKey = deriveTaskKey(context, null);
@@ -115,16 +224,37 @@ export function createHeartbeatExecuteHandlers(context) {
             const taskSession = taskKey
                 ? await getTaskSession(agent.orgId, agent.id, agent.agentRuntimeType, taskKey)
                 : null;
-            const resetTaskSession = shouldResetTaskSessionForWake(context);
+            const resetTaskSession = run.sessionReuseScope === "unknown"
+                ? shouldResetTaskSessionForWake(context)
+                : run.sessionReuseScope === "none" && shouldResetTaskSessionForWake(context);
             const sessionResetReason = describeSessionResetReason(context);
             const taskSessionForRun = resetTaskSession ? null : taskSession;
             const explicitResumeSessionParams = normalizeSessionParams(sessionCodec.deserialize(parseObject(context.resumeSessionParams)));
             const explicitResumeSessionDisplayId = truncateDisplayId(readNonEmptyString(context.resumeSessionDisplayId) ??
                 (sessionCodec.getDisplayId ? sessionCodec.getDisplayId(explicitResumeSessionParams) : null) ??
                 readNonEmptyString(explicitResumeSessionParams?.sessionId));
-            const previousSessionParams = explicitResumeSessionParams ??
-                (explicitResumeSessionDisplayId ? { sessionId: explicitResumeSessionDisplayId } : null) ??
-                normalizeSessionParams(sessionCodec.deserialize(taskSessionForRun?.sessionParamsJson ?? null));
+            const taskSessionParams = normalizeSessionParams(sessionCodec.deserialize(taskSessionForRun?.sessionParamsJson ?? null));
+            const taskSessionDisplayId = truncateDisplayId(taskSessionForRun?.sessionDisplayId ??
+                (sessionCodec.getDisplayId ? sessionCodec.getDisplayId(taskSessionParams) : null) ??
+                readNonEmptyString(taskSessionParams?.sessionId));
+            const frozenSessionParams = normalizeSessionParams(sessionCodec.deserialize(run.sessionParamsBeforeJson ?? null));
+            const frozenSessionDisplayId = truncateDisplayId(run.sessionIdBefore ??
+                (sessionCodec.getDisplayId ? sessionCodec.getDisplayId(frozenSessionParams) : null) ??
+                readNonEmptyString(frozenSessionParams?.sessionId));
+            const sessionSelection = run.sessionReuseScope === "unknown"
+                ? selectRunSessionLineage({
+                    forceFresh: Boolean(heartbeatSessions.readSessionReuseSuppression(context)),
+                    explicitSessionParams: explicitResumeSessionParams,
+                    explicitSessionDisplayId: explicitResumeSessionDisplayId,
+                    taskSessionParams,
+                    taskSessionDisplayId,
+                })
+                : {
+                    reuseScope: run.sessionReuseScope,
+                    sessionParams: run.sessionReuseScope === "none" ? null : frozenSessionParams,
+                    sessionDisplayId: run.sessionReuseScope === "none" ? null : frozenSessionDisplayId,
+                };
+            const previousSessionParams = sessionSelection.sessionParams;
             const config = await runContextSvc.materializeManagedInstructionsForRun({
                 ...agent,
                 agentRuntimeConfig: parseObject(agent.agentRuntimeConfig),
@@ -313,13 +443,9 @@ export function createHeartbeatExecuteHandlers(context) {
             }
             if (persistedExecutionWorkspace) {
                 context.executionWorkspaceId = persistedExecutionWorkspace.id;
-                await db
-                    .update(heartbeatRuns)
-                    .set({
-                    contextSnapshot: buildPersistableHeartbeatContext(context),
-                    updatedAt: new Date(),
-                })
-                    .where(eq(heartbeatRuns.id, run.id));
+                const workspaceContextRun = await persistRunningExecutionContext(run.id, context);
+                if (workspaceContextRun)
+                    run = workspaceContextRun;
             }
             const runtimeSessionResolution = resolveRuntimeSessionParamsForWorkspace({
                 orgId: agent.orgId,
@@ -378,13 +504,10 @@ export function createHeartbeatExecuteHandlers(context) {
             if (executionWorkspace.projectId && !readNonEmptyString(context.projectId)) {
                 context.projectId = executionWorkspace.projectId;
             }
-            const runtimeSessionFallback = taskKey || resetTaskSession ? null : runtime.sessionId;
-            let previousSessionDisplayId = truncateDisplayId(explicitResumeSessionDisplayId ??
-                taskSessionForRun?.sessionDisplayId ??
+            let previousSessionDisplayId = truncateDisplayId(sessionSelection.sessionDisplayId ??
                 (sessionCodec.getDisplayId ? sessionCodec.getDisplayId(runtimeSessionParams) : null) ??
-                readNonEmptyString(runtimeSessionParams?.sessionId) ??
-                runtimeSessionFallback);
-            let runtimeSessionIdForAdapter = readNonEmptyString(runtimeSessionParams?.sessionId) ?? runtimeSessionFallback;
+                readNonEmptyString(runtimeSessionParams?.sessionId));
+            let runtimeSessionIdForAdapter = readNonEmptyString(runtimeSessionParams?.sessionId);
             let runtimeSessionParamsForAdapter = runtimeSessionParams;
             const sessionCompaction = await evaluateSessionCompaction({
                 agent,
@@ -407,13 +530,13 @@ export function createHeartbeatExecuteHandlers(context) {
                 delete context.rudderSessionRotationReason;
                 delete context.rudderPreviousSessionId;
             }
+            const sessionReuseScope = sessionCompaction.rotate ? "none" : sessionSelection.reuseScope;
             const runtimeForAdapter = {
                 sessionId: runtimeSessionIdForAdapter,
                 sessionParams: runtimeSessionParamsForAdapter,
                 sessionDisplayId: previousSessionDisplayId,
                 taskKey,
             };
-            let seq = 1;
             let handle = null;
             let stdoutExcerpt = "";
             let stderrExcerpt = "";
@@ -429,7 +552,7 @@ export function createHeartbeatExecuteHandlers(context) {
             const appendForbiddenMarkerEvent = async (eventRun, scan) => {
                 if (!scan.observed)
                     return;
-                await appendRunEvent(eventRun, seq++, {
+                await appendRunEvent(eventRun, {
                     eventType: "adapter.forbidden_marker",
                     stream: "system",
                     level: "error",
@@ -451,38 +574,16 @@ export function createHeartbeatExecuteHandlers(context) {
                     skillsDir: readNonEmptyString(runtimeSceneContext.rudderWorkspace.agentSkillsDir) ?? "",
                 });
                 const startedAt = run.startedAt ?? new Date();
-                const runningWithSession = await db
-                    .update(heartbeatRuns)
-                    .set({
+                const runningWithSession = await persistRunningExecutionContext(run.id, context, {
                     startedAt,
                     sessionIdBefore: runtimeForAdapter.sessionDisplayId ?? runtimeForAdapter.sessionId,
-                    contextSnapshot: buildPersistableHeartbeatContext(context),
-                    updatedAt: new Date(),
-                })
-                    .where(eq(heartbeatRuns.id, run.id))
-                    .returning()
-                    .then((rows) => rows[0] ?? null);
+                    sessionParamsBeforeJson: runtimeForAdapter.sessionParams,
+                    sessionReuseScope,
+                });
                 if (runningWithSession)
                     run = runningWithSession;
-                const runningAgent = await db
-                    .update(agents)
-                    .set({ status: "running", updatedAt: new Date() })
-                    .where(eq(agents.id, agent.id))
-                    .returning()
-                    .then((rows) => rows[0] ?? null);
-                if (runningAgent) {
-                    publishLiveEvent({
-                        orgId: runningAgent.orgId,
-                        type: "agent.status",
-                        payload: {
-                            agentId: runningAgent.id,
-                            status: runningAgent.status,
-                            outcome: "running",
-                        },
-                    });
-                }
                 const currentRun = run;
-                await appendRunEvent(currentRun, seq++, {
+                await appendRunEvent(currentRun, {
                     eventType: "lifecycle",
                     stream: "system",
                     level: "info",
@@ -524,7 +625,7 @@ export function createHeartbeatExecuteHandlers(context) {
                         await db
                             .update(heartbeatRuns)
                             .set({ updatedAt: new Date(nowMs) })
-                            .where(eq(heartbeatRuns.id, run.id));
+                            .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running")));
                     }
                     const payloadChunk = sanitizedChunk.length > MAX_LIVE_LOG_CHUNK_BYTES
                         ? sanitizedChunk.slice(sanitizedChunk.length - MAX_LIVE_LOG_CHUNK_BYTES)
@@ -582,13 +683,7 @@ export function createHeartbeatExecuteHandlers(context) {
                     context.rudderRuntimeServices = runtimeServices;
                     context.rudderRuntimePrimaryUrl =
                         runtimeServices.find((service) => readNonEmptyString(service.url))?.url ?? null;
-                    await db
-                        .update(heartbeatRuns)
-                        .set({
-                        contextSnapshot: buildPersistableHeartbeatContext(context),
-                        updatedAt: new Date(),
-                    })
-                        .where(eq(heartbeatRuns.id, run.id));
+                    await persistRunningExecutionContext(run.id, context);
                 }
                 if (issueId && (executionWorkspace.created || runtimeServices.some((service) => !service.reused))) {
                     try {
@@ -610,7 +705,7 @@ export function createHeartbeatExecuteHandlers(context) {
                                 meta.env[key] = "***REDACTED***";
                         }
                     }
-                    await appendRunEvent(currentRun, seq++, {
+                    await appendRunEvent(currentRun, {
                         eventType: "adapter.invoke",
                         stream: "system",
                         level: "info",
@@ -632,6 +727,10 @@ export function createHeartbeatExecuteHandlers(context) {
                         agentRuntimeType: agent.agentRuntimeType,
                     }, "local agent jwt secret missing or invalid; running without injected RUDDER_API_KEY");
                 }
+                const runBeforeAdapter = await getRun(run.id);
+                if (runBeforeAdapter?.status !== "running") {
+                    throw new Error("Run was finalized before adapter invocation");
+                }
                 const adapterResult = await executeAdapterWithModelFallbacks(adapter, {
                     runId: run.id,
                     agent,
@@ -643,6 +742,7 @@ export function createHeartbeatExecuteHandlers(context) {
                     onSpawn: async (meta) => {
                         await persistRunProcessMetadata(run.id, meta);
                     },
+                    abortSignal: executionAbortController.signal,
                     authToken: authToken ?? undefined,
                 }, {
                     resolveAdapter: findServerAdapter,
@@ -674,13 +774,7 @@ export function createHeartbeatExecuteHandlers(context) {
                     context.rudderRuntimeServices = combinedRuntimeServices;
                     context.rudderRuntimePrimaryUrl =
                         combinedRuntimeServices.find((service) => readNonEmptyString(service.url))?.url ?? null;
-                    await db
-                        .update(heartbeatRuns)
-                        .set({
-                        contextSnapshot: buildPersistableHeartbeatContext(context),
-                        updatedAt: new Date(),
-                    })
-                        .where(eq(heartbeatRuns.id, run.id));
+                    await persistRunningExecutionContext(run.id, context);
                     if (issueId) {
                         try {
                             await issuesSvc.addComment(issueId, buildWorkspaceReadyComment({
@@ -742,7 +836,6 @@ export function createHeartbeatExecuteHandlers(context) {
                         : outcome === "timed_out"
                             ? "timed_out"
                             : "failed";
-                finalRunStatus = status;
                 const adapterResultSummary = summarizeHeartbeatRunResultJson(adapterResult.resultJson);
                 const persistedResultSummary = summarizeHeartbeatRunResultJson({
                     ...(adapterResult.resultJson ?? {}),
@@ -774,9 +867,10 @@ export function createHeartbeatExecuteHandlers(context) {
                         ...((nextSessionState.displayId ?? nextSessionState.legacySessionId)
                             ? { persistedSessionId: nextSessionState.displayId ?? nextSessionState.legacySessionId }
                             : {}),
-                        sessionReused: runtimeForAdapter.sessionId != null || runtimeForAdapter.sessionDisplayId != null,
-                        taskSessionReused: taskSessionForRun != null,
-                        freshSession: runtimeForAdapter.sessionId == null && runtimeForAdapter.sessionDisplayId == null,
+                        sessionReused: sessionReuseScope !== "none",
+                        taskSessionReused: sessionReuseScope === "task",
+                        freshSession: sessionReuseScope === "none",
+                        sessionReuseScope,
                         sessionRotated: sessionCompaction.rotate,
                         sessionRotationReason: sessionCompaction.reason,
                         provider: readNonEmptyString(adapterResult.provider) ?? "unknown",
@@ -786,7 +880,8 @@ export function createHeartbeatExecuteHandlers(context) {
                         billingType: normalizeLedgerBillingType(adapterResult.billingType),
                     }
                     : null;
-                await setRunStatus(run.id, status, {
+                finalizeExecutionTranscript();
+                const terminalEvidence = {
                     finishedAt: new Date(),
                     error: outcome === "succeeded"
                         ? null
@@ -808,24 +903,85 @@ export function createHeartbeatExecuteHandlers(context) {
                     resultJson: adapterResult.resultJson ?? null,
                     resultSummaryJson: persistedResultSummary,
                     sessionIdAfter: nextSessionState.displayId ?? nextSessionState.legacySessionId,
+                    sessionParamsAfterJson: adapterResult.clearSession ? {} : nextSessionState.params,
                     stdoutExcerpt,
                     stderrExcerpt,
                     logBytes: logSummary?.bytes,
                     logSha256: logSummary?.sha256,
                     logCompressed: logSummary?.compressed ?? false,
+                };
+                const terminalEffectsIntent = {
+                    version: 1,
+                    automation: {
+                        output: transcriptFallbackResult?.output ?? terminalEvidence.error,
+                        transcript: executionTranscript,
+                    },
+                    runtime: {
+                        adapterResult: adapterResult,
+                        legacySessionId: nextSessionState.legacySessionId,
+                        normalizedUsage: normalizedUsage,
+                        ownsTerminal: true,
+                    },
+                    ...(taskKey
+                        ? {
+                            taskSession: adapterResult.clearSession || (!nextSessionState.params && !nextSessionState.displayId)
+                                ? {
+                                    operation: "clear",
+                                    orgId: agent.orgId,
+                                    agentId: agent.id,
+                                    agentRuntimeType: agent.agentRuntimeType,
+                                    taskKey,
+                                    lastRunId: run.id,
+                                }
+                                : {
+                                    operation: "upsert",
+                                    orgId: agent.orgId,
+                                    agentId: agent.id,
+                                    agentRuntimeType: agent.agentRuntimeType,
+                                    taskKey,
+                                    sessionParamsJson: nextSessionState.params,
+                                    sessionDisplayId: nextSessionState.displayId,
+                                    lastRunId: run.id,
+                                    lastError: outcome === "succeeded" ? null : (adapterResult.errorMessage ?? "run_failed"),
+                                },
+                        }
+                        : {}),
+                };
+                const claimedTerminalRun = await transitionRunToTerminal(run.id, status, terminalEvidence, {
+                    terminalEffectsIntent,
+                    processExitedAt: new Date(),
+                    expectedExecutionOwnerToken: executionOwnerToken,
                 });
-                await setWakeupStatus(run.wakeupRequestId, outcome === "succeeded" ? "completed" : status, {
-                    finishedAt: new Date(),
-                    error: failureCausedByForbiddenMarker
-                        ? "Forbidden runtime skill marker observed"
-                        : adapterResult.errorMessage ?? null,
-                });
-                const finalizedRun = await getRun(run.id);
+                ownsTerminalState = Boolean(claimedTerminalRun);
+                if (!claimedTerminalRun) {
+                    await reconcileRunEvidence(run.id, terminalEvidence);
+                    await reconcileTerminalEffectsIntent(run.id, {
+                        version: 1,
+                        automation: terminalEffectsIntent.automation,
+                        runtime: {
+                            adapterResult: adapterResult,
+                            legacySessionId: null,
+                            normalizedUsage: normalizedUsage,
+                            ownsTerminal: false,
+                        },
+                    });
+                }
+                const finalizedRun = claimedTerminalRun ?? await getRun(run.id);
                 if (finalizedRun) {
+                    transcriptFallbackResult.subtype = finalizedRun.status;
+                    transcriptFallbackResult.isError = finalizedRun.status !== "succeeded";
+                    if (ownsTerminalState) {
+                        await setWakeupStatus(run.wakeupRequestId, outcome === "succeeded" ? "completed" : status, {
+                            finishedAt: new Date(),
+                            error: failureCausedByForbiddenMarker
+                                ? "Forbidden runtime skill marker observed"
+                                : adapterResult.errorMessage ?? null,
+                        });
+                    }
                     await appendForbiddenMarkerEvent(finalizedRun, forbiddenMarkerScan);
                     const transcriptUsedSkills = inferUsedSkillsFromTranscript(executionTranscript);
                     if (transcriptUsedSkills.length > 0) {
-                        await appendRunEvent(finalizedRun, seq++, {
+                        await appendRunEvent(finalizedRun, {
                             eventType: "adapter.skill_usage",
                             stream: "system",
                             level: "info",
@@ -842,50 +998,29 @@ export function createHeartbeatExecuteHandlers(context) {
                             },
                         });
                     }
-                    await appendRunEvent(finalizedRun, seq++, {
-                        eventType: "lifecycle",
-                        stream: "system",
-                        level: outcome === "succeeded" ? "info" : "error",
-                        message: `run ${outcome}`,
-                        payload: {
-                            status,
-                            exitCode: adapterResult.exitCode,
-                        },
-                    });
-                    await releaseIssueExecutionAndPromote(finalizedRun);
+                    if (ownsTerminalState) {
+                        await appendRunEvent(finalizedRun, {
+                            eventType: "lifecycle",
+                            stream: "system",
+                            level: outcome === "succeeded" ? "info" : "error",
+                            message: `run ${outcome}`,
+                            payload: {
+                                status,
+                                exitCode: adapterResult.exitCode,
+                            },
+                        });
+                    }
                 }
                 if (finalizedRun) {
-                    await updateRuntimeState(agent, finalizedRun, adapterResult, {
-                        legacySessionId: nextSessionState.legacySessionId,
-                    }, normalizedUsage);
-                    if (taskKey) {
-                        if (adapterResult.clearSession || (!nextSessionState.params && !nextSessionState.displayId)) {
-                            await clearTaskSessions(agent.orgId, agent.id, {
-                                taskKey,
-                                agentRuntimeType: agent.agentRuntimeType,
-                            });
-                        }
-                        else {
-                            await upsertTaskSession({
-                                orgId: agent.orgId,
-                                agentId: agent.id,
-                                agentRuntimeType: agent.agentRuntimeType,
-                                taskKey,
-                                sessionParamsJson: nextSessionState.params,
-                                sessionDisplayId: nextSessionState.displayId,
-                                lastRunId: finalizedRun.id,
-                                lastError: outcome === "succeeded" ? null : (adapterResult.errorMessage ?? "run_failed"),
-                            });
-                        }
+                    if (ownsTerminalState || finalizedRun.terminalEffectsPending) {
+                        shouldCompleteTerminalEffects = true;
                     }
                 }
-                await finalizeAgentStatus(agent.id, outcome);
             }
             catch (err) {
                 const isWorkspacePreflightFailure = isWorkspacePermissionPreflightError(err) ||
                     isManagedWorkspaceConfigurationError(err);
                 const message = redactCurrentUserText(err instanceof Error ? err.message : "Unknown adapter failure", await getCurrentUserRedactionOptions());
-                finalRunStatus = "failed";
                 transcriptFallbackResult = {
                     ts: new Date().toISOString(),
                     output: message,
@@ -895,17 +1030,32 @@ export function createHeartbeatExecuteHandlers(context) {
                 };
                 logger.error({ err, runId }, "heartbeat execution failed");
                 const latestRun = await getRun(run.id);
-                if (latestRun?.status === "cancelled" || latestRun?.status === "timed_out") {
+                if (latestRun?.status === "succeeded"
+                    || latestRun?.status === "failed"
+                    || latestRun?.status === "cancelled"
+                    || latestRun?.status === "timed_out") {
                     const terminalStatus = latestRun.status;
-                    finalRunStatus = terminalStatus;
                     transcriptFallbackResult = {
                         ts: new Date().toISOString(),
                         output: latestRun.error ?? message,
                         subtype: terminalStatus,
-                        isError: terminalStatus === "timed_out",
+                        isError: terminalStatus !== "succeeded",
                         errors: latestRun.error ? [latestRun.error] : [],
                     };
-                    await finalizeAgentStatus(agent.id, terminalStatus);
+                    let lateLogSummary = null;
+                    if (handle) {
+                        lateLogSummary = await runLogStore.finalize(handle).catch(() => null);
+                    }
+                    await reconcileRunEvidence(run.id, {
+                        stdoutExcerpt,
+                        stderrExcerpt,
+                        logBytes: lateLogSummary?.bytes,
+                        logSha256: lateLogSummary?.sha256,
+                        logCompressed: lateLogSummary?.compressed,
+                    });
+                    if (latestRun.terminalEffectsPending) {
+                        shouldCompleteTerminalEffects = true;
+                    }
                     return;
                 }
                 let logSummary = null;
@@ -917,7 +1067,7 @@ export function createHeartbeatExecuteHandlers(context) {
                         logger.warn({ err: finalizeErr, runId }, "failed to finalize run log after error");
                     }
                 }
-                const failedRun = await setRunStatus(run.id, "failed", {
+                const failureEvidence = {
                     error: message,
                     errorCode: isWorkspacePreflightFailure ? err.errorCode : "adapter_failed",
                     finishedAt: new Date(),
@@ -926,82 +1076,93 @@ export function createHeartbeatExecuteHandlers(context) {
                     logBytes: logSummary?.bytes,
                     logSha256: logSummary?.sha256,
                     logCompressed: logSummary?.compressed ?? false,
-                });
-                await setWakeupStatus(run.wakeupRequestId, "failed", {
-                    finishedAt: new Date(),
-                    error: message,
-                });
-                if (failedRun) {
-                    await appendForbiddenMarkerEvent(failedRun, buildForbiddenMarkerScan(null));
-                    await appendRunEvent(failedRun, seq++, {
-                        eventType: isWorkspacePreflightFailure ? "runtime.workspace_preflight_failed" : "error",
-                        stream: "system",
-                        level: "error",
-                        message,
-                        ...(isWorkspacePreflightFailure
-                            ? {
-                                payload: {
-                                    errorCode: err.errorCode,
-                                    failure: err.failure,
+                };
+                finalizeExecutionTranscript();
+                const failureIntent = {
+                    version: 1,
+                    automation: { output: message, transcript: executionTranscript },
+                    ...(!isWorkspacePreflightFailure
+                        ? {
+                            runtime: {
+                                adapterResult: {
+                                    exitCode: null,
+                                    signal: null,
+                                    timedOut: false,
+                                    errorMessage: message,
                                 },
-                            }
-                            : {}),
-                    });
-                    await releaseIssueExecutionAndPromote(failedRun);
-                    if (!isWorkspacePreflightFailure) {
-                        await updateRuntimeState(agent, failedRun, {
-                            exitCode: null,
-                            signal: null,
-                            timedOut: false,
-                            errorMessage: message,
-                        }, {
-                            legacySessionId: runtimeForAdapter.sessionId,
-                        });
-                        if (taskKey && (previousSessionParams || previousSessionDisplayId || taskSession)) {
-                            await upsertTaskSession({
+                                legacySessionId: runtimeForAdapter.sessionId,
+                            },
+                        }
+                        : {}),
+                    ...(taskKey && !isWorkspacePreflightFailure && (previousSessionParams || previousSessionDisplayId || taskSession)
+                        ? {
+                            taskSession: {
+                                operation: "upsert",
                                 orgId: agent.orgId,
                                 agentId: agent.id,
                                 agentRuntimeType: agent.agentRuntimeType,
                                 taskKey,
                                 sessionParamsJson: previousSessionParams,
                                 sessionDisplayId: previousSessionDisplayId,
-                                lastRunId: failedRun.id,
+                                lastRunId: run.id,
                                 lastError: message,
-                            });
+                            },
                         }
+                        : {}),
+                };
+                const claimedFailedRun = await transitionRunToTerminal(run.id, "failed", failureEvidence, {
+                    terminalEffectsIntent: failureIntent,
+                    processExitedAt: new Date(),
+                    expectedExecutionOwnerToken: executionOwnerToken,
+                });
+                ownsTerminalState = Boolean(claimedFailedRun);
+                if (!claimedFailedRun)
+                    await reconcileRunEvidence(run.id, failureEvidence);
+                const failedRun = claimedFailedRun ?? await getRun(run.id);
+                if (ownsTerminalState) {
+                    await setWakeupStatus(run.wakeupRequestId, "failed", {
+                        finishedAt: new Date(),
+                        error: message,
+                    });
+                }
+                if (failedRun) {
+                    if (ownsTerminalState) {
+                        await appendForbiddenMarkerEvent(failedRun, buildForbiddenMarkerScan(null));
+                        await appendRunEvent(failedRun, {
+                            eventType: isWorkspacePreflightFailure ? "runtime.workspace_preflight_failed" : "error",
+                            stream: "system",
+                            level: "error",
+                            message,
+                            ...(isWorkspacePreflightFailure
+                                ? {
+                                    payload: {
+                                        errorCode: err.errorCode,
+                                        failure: err.failure,
+                                    },
+                                }
+                                : {}),
+                        });
+                    }
+                    if (ownsTerminalState || failedRun.terminalEffectsPending) {
+                        shouldCompleteTerminalEffects = true;
                     }
                 }
-                await finalizeAgentStatus(agent.id, "failed");
             }
             finally {
-                stdoutTranscriptBuffer = appendTranscriptEntriesFromChunk({
-                    buffer: stdoutTranscriptBuffer,
-                    chunk: "",
-                    transcript: executionTranscript,
-                    parser: stdoutTranscriptParser,
-                    finalize: true,
-                    kind: "stdout",
-                });
-                stderrTranscriptBuffer = appendTranscriptEntriesFromChunk({
-                    buffer: stderrTranscriptBuffer,
-                    chunk: "",
-                    transcript: executionTranscript,
-                    finalize: true,
-                    kind: "stderr",
-                });
+                finalizeExecutionTranscript();
                 finalRunOutput = transcriptFallbackResult?.output ?? null;
-                await publishAutomationRunOutputToChat(db, {
-                    issueId,
-                    output: finalRunOutput,
-                    status: finalRunStatus,
-                    transcript: executionTranscript,
-                }).catch((error) => {
-                    logger.warn({
-                        runId: run.id,
-                        issueId,
-                        err: error instanceof Error ? error.message : String(error),
-                    }, "Failed to publish automation run output to chat");
-                });
+                if (ownsTerminalState || shouldCompleteTerminalEffects) {
+                    const terminalRun = await getRun(run.id).catch(() => null);
+                    if (terminalRun?.terminalEffectsPending) {
+                        await acknowledgeRunProcessExit(terminalRun.id);
+                        await completeTerminalControlEffects(terminalRun, ownsTerminalState
+                            ? {
+                                automationOutput: finalRunOutput,
+                                automationTranscript: executionTranscript,
+                            }
+                            : undefined);
+                    }
+                }
             }
         }
         catch (outerErr) {
@@ -1009,33 +1170,42 @@ export function createHeartbeatExecuteHandlers(context) {
             // The inner catch did not fire, so we must record the failure here.
             const message = outerErr instanceof Error ? outerErr.message : "Unknown setup failure";
             logger.error({ err: outerErr, runId }, "heartbeat execution setup failed");
-            await setRunStatus(runId, "failed", {
-                error: message,
-                errorCode: "adapter_failed",
-                finishedAt: new Date(),
-            }).catch(() => undefined);
-            await setWakeupStatus(run.wakeupRequestId, "failed", {
-                finishedAt: new Date(),
-                error: message,
-            }).catch(() => undefined);
-            const failedRun = await getRun(runId).catch(() => null);
-            if (failedRun) {
+            const latestRun = await getRun(runId).catch(() => null);
+            const claimedFailedRun = latestRun?.status === "running"
+                ? await transitionRunToTerminal(runId, "failed", {
+                    error: message,
+                    errorCode: "adapter_failed",
+                    finishedAt: new Date(),
+                }, {
+                    processExitedAt: new Date(),
+                    expectedExecutionOwnerToken: executionOwnerToken,
+                }).catch(() => undefined)
+                : null;
+            const terminalRun = claimedFailedRun ?? await getRun(runId).catch(() => latestRun);
+            if (claimedFailedRun) {
+                await setWakeupStatus(run.wakeupRequestId, "failed", {
+                    finishedAt: new Date(),
+                    error: message,
+                }).catch(() => undefined);
                 // Emit a run-log event so the failure is visible in the run timeline,
                 // consistent with what the inner catch block does for adapter failures.
-                await appendRunEvent(failedRun, 1, {
+                await appendRunEvent(claimedFailedRun, {
                     eventType: "error",
                     stream: "system",
                     level: "error",
                     message,
                 }).catch(() => undefined);
-                await releaseIssueExecutionAndPromote(failedRun).catch(() => undefined);
             }
-            // Ensure the agent is not left stuck in "running" if the inner catch handler's
-            // DB calls threw (e.g. a transient DB error in finalizeAgentStatus).
-            await finalizeAgentStatus(run.agentId, "failed").catch(() => undefined);
+            if (terminalRun?.terminalEffectsPending) {
+                await acknowledgeRunProcessExit(terminalRun.id).catch(() => undefined);
+                await completeTerminalControlEffects(terminalRun).catch(() => undefined);
+            }
         }
         finally {
+            if (executionLeaseTimer)
+                clearInterval(executionLeaseTimer);
             await releaseRuntimeServicesForRun(run.id).catch(() => undefined);
+            runAbortControllers.delete(run.id);
             activeRunExecutions.delete(run.id);
             await startNextQueuedRunForAgent(run.agentId);
         }