@rudderhq/server 0.7.10 → 0.7.12

package/dist/services/runtime-kernel/heartbeat.execute.js

@@ -1,3 +1,13 @@
+// @ts-nocheck
+/**
+ * @fileoverview Executes claimed heartbeat runs through runtime adapters,
+ * workspace realization, transcript persistence, and close-out release.
+ *
+ * @see doc/product/domains/execution/agent-runs.md - run execution state and evidence
+ * @see doc/product/domains/execution/run-admission-and-recovery.md - retry and recovery behavior
+ * @see doc/product/domains/agents/instruction-loading.md - AGENT.INSTRUCTIONS.001 runtime instruction frame
+ */
+import { buildModelAttemptSpecs, isAgentRuntimeNetworkSuspension, } from "@rudderhq/agent-runtime-utils";
 import { goals, heartbeatRuns, issues, projects } from "@rudderhq/db";
 import { resolveAgentRunScene, } from "@rudderhq/shared";
 import { and, eq, sql } from "drizzle-orm";
@@ -13,12 +23,13 @@ import { publishLiveEvent } from "../live-events.js";
 import { isManagedWorkspaceConfigurationError, isWorkspacePermissionPreflightError, preflightManagedAgentWorkspace, } from "../managed-workspace-preflight.js";
 import { listProjectResourceAttachments } from "../resource-catalog.js";
 import { buildWorkspaceReadyComment, cleanupExecutionWorkspaceArtifacts, ensureRuntimeServicesForRun, persistAdapterManagedRuntimeServices, realizeExecutionWorkspace, releaseRuntimeServicesForRun } from "../workspace-runtime.js";
-import { createAssignmentRunFailureBudget, inspectAssignmentRunWorkspace, resolveProjectWorkingSetCwd, } from "./assignment-run-guardrail.js";
+import { AssignmentDependencyPreflightError, createAssignmentRunFailureBudget, inspectAssignmentRunWorkspace, repairAssignmentRunWorkspace, resolveProjectWorkingSetCwd, } from "./assignment-run-guardrail.js";
+import { beginHeartbeatRunAttempt, finishHeartbeatRunAttempt, markHeartbeatRunAttemptWaiting, } from "./heartbeat-attempt-ledger.js";
 import { executeAdapterWithModelFallbacks } from "./model-fallback.js";
 export { prioritizeProjectWorkspaceCandidatesForRun } from "../agent-run-context.js";
 import * as heartbeatCore from "./heartbeat.core.js";
 import * as heartbeatSessions from "./heartbeat.sessions.js";
-const { MAX_LIVE_LOG_CHUNK_BYTES, HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT, HEARTBEAT_MAX_CONCURRENT_RUNS_MIN, HEARTBEAT_MAX_CONCURRENT_RUNS_MAX, DEFERRED_WAKE_CONTEXT_KEY, DETACHED_PROCESS_ERROR_CODE, ORPHANED_PROCESS_TERMINATION_GRACE_MS, ORPHANED_PROCESS_KILL_WAIT_MS, ORPHANED_PROCESS_POLL_INTERVAL_MS, startLocksByAgent, MAX_RECOVERY_CHAIN_DEPTH, ISSUE_PASSIVE_FOLLOWUP_REASON, ISSUE_PASSIVE_FOLLOWUP_WAKE_SOURCE, ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON, ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS, ISSUE_REVIEW_CLOSEOUT_REASON, ISSUE_REVIEW_CLOSEOUT_FAILURE_REASON, ISSUE_REVIEW_CLOSEOUT_MAX_ATTEMPTS, ISSUE_PASSIVE_FOLLOWUP_COOLDOWN_MS_BY_ATTEMPT, ISSUE_PASSIVE_FOLLOWUP_TIMER_CONTINUITY_MAX_WINDOW_MS, SESSIONED_LOCAL_ADAPTERS, heartbeatRunListColumns, appendExcerpt, appendTranscriptEntriesFromChunk, normalizeMaxConcurrentRuns, withAgentStartLock, readNonEmptyString, buildHeartbeatAdapterInvokePayload, sanitizeStartupContextContextForPersistence, sanitizeStartupContextPromptForPersistence, buildRecentDateKeys, buildDateKeysBetween, fallbackSkillLabel, normalizeLoadedSkill, normalizeLoadedSkillForPayload, emptySkillEvidenceCounts, incrementSkillEvidenceCount, strongestSkillEvidence, resolveSkillEvidence, readSkillEvidenceFromPayload, extractSkillSlugFromPath, collectSkillPathsFromText, collectStringValues, normalizeSkillUseFromPath, dedupeSkillUses, collectSkillUsesFromText, readToolCommandInput, isCommandTranscriptTool, isReadTranscriptTool, inferUsedSkillsFromTranscript, normalizeSkillCandidate, addSkillCandidate, readSkillReferenceSlug, collectSkillReferences, inferUsedSkillsFromPrompt, resolveForbiddenRuntimeSkillMarkers, detectForbiddenRuntimeSkillMarker, normalizeLedgerBillingType, resolveLedgerBiller, normalizeBilledCostCents, resolveLedgerScopeForRun } = heartbeatCore;
+const { MAX_LIVE_LOG_CHUNK_BYTES, HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT, HEARTBEAT_MAX_CONCURRENT_RUNS_MIN, HEARTBEAT_MAX_CONCURRENT_RUNS_MAX, DEFERRED_WAKE_CONTEXT_KEY, DETACHED_PROCESS_ERROR_CODE, ORPHANED_PROCESS_TERMINATION_GRACE_MS, ORPHANED_PROCESS_KILL_WAIT_MS, ORPHANED_PROCESS_POLL_INTERVAL_MS, startLocksByAgent, MAX_RECOVERY_CHAIN_DEPTH, ISSUE_PASSIVE_FOLLOWUP_REASON, ISSUE_PASSIVE_FOLLOWUP_WAKE_SOURCE, ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON, ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS, ISSUE_REVIEW_CLOSEOUT_REASON, ISSUE_REVIEW_CLOSEOUT_FAILURE_REASON, ISSUE_REVIEW_CLOSEOUT_MAX_ATTEMPTS, ISSUE_PASSIVE_FOLLOWUP_COOLDOWN_MS_BY_ATTEMPT, ISSUE_PASSIVE_FOLLOWUP_TIMER_CONTINUITY_MAX_WINDOW_MS, networkWaitBackoffMs, SESSIONED_LOCAL_ADAPTERS, heartbeatRunListColumns, appendExcerpt, appendTranscriptEntriesFromChunk, normalizeMaxConcurrentRuns, withAgentStartLock, readNonEmptyString, buildHeartbeatAdapterInvokePayload, sanitizeStartupContextContextForPersistence, sanitizeStartupContextPromptForPersistence, buildRecentDateKeys, buildDateKeysBetween, fallbackSkillLabel, normalizeLoadedSkill, normalizeLoadedSkillForPayload, emptySkillEvidenceCounts, incrementSkillEvidenceCount, strongestSkillEvidence, resolveSkillEvidence, readSkillEvidenceFromPayload, extractSkillSlugFromPath, collectSkillPathsFromText, collectStringValues, normalizeSkillUseFromPath, dedupeSkillUses, collectSkillUsesFromText, readToolCommandInput, isCommandTranscriptTool, isReadTranscriptTool, inferUsedSkillsFromTranscript, normalizeSkillCandidate, addSkillCandidate, readSkillReferenceSlug, collectSkillReferences, inferUsedSkillsFromPrompt, resolveForbiddenRuntimeSkillMarkers, detectForbiddenRuntimeSkillMarker, normalizeLedgerBillingType, resolveLedgerBiller, normalizeBilledCostCents, resolveLedgerScopeForRun } = heartbeatCore;
 const { buildExplicitResumeSessionOverride, selectRunSessionLineage, normalizeUsageTotals, readRawUsageTotals, deriveNormalizedUsageDelta, formatCount, parseSessionCompactionPolicy, resolveRuntimeSessionParamsForWorkspace, parseIssueAssigneeAgentRuntimeOverrides, deriveTaskKey, shouldResetTaskSessionForWake, formatRuntimeWorkspaceWarningLog, describeSessionResetReason, deriveCommentId, enrichWakeContextSnapshot, mergeCoalescedContextSnapshot, issueCommentAuthorKind, issueCommentAuthorLabel, buildDeferredWakePayload, readDeferredWakeContext, readDeferredWakePayload, deriveDeferredWakeTaskKey, hydrateWakeContextSnapshot, firstNonEmptyLine, deriveRecoveryFailureKind, deriveRecoveryFailureSummary, mergeMissingRecoveryContextFields, hydrateRecoveryBaseContextSnapshot, buildRecoveryContextSnapshot, normalizePassiveFollowupContext, normalizeReviewCloseoutContext, passiveFollowupCooldownMs, issueHasReviewer, isAgentEligibleForTimerContinuation, hasCredibleTimerContinuation, buildPassiveFollowupContextSnapshot, runTaskKey, isSameTaskScope, isTrackedLocalChildProcessAdapter, isProcessAlive, waitForProcessExit, terminateOrphanedProcess, truncateDisplayId, normalizeAgentNameKey, defaultSessionCodec, getAgentRuntimeSessionCodec, normalizeSessionParams, resolveNextSessionState } = heartbeatSessions;
 function buildPersistableHeartbeatContext(context) {
     return sanitizeStartupContextContextForPersistence(context) ?? {};
@@ -123,6 +134,20 @@ export function createHeartbeatExecuteHandlers(context) {
         }
         const executionAbortController = runAbortControllers.get(run.id) ?? new AbortController();
         runAbortControllers.set(run.id, executionAbortController);
+        let activeAttemptRef = null;
+        let networkSuspended = false;
+        const finishActiveAttempt = async (input) => {
+            const ref = activeAttemptRef;
+            activeAttemptRef = null;
+            if (!ref)
+                return;
+            try {
+                await finishHeartbeatRunAttempt(db, ref, input);
+            }
+            catch (error) {
+                logger.warn({ err: error, runId: runId, attemptIndex: ref.attemptIndex }, "failed to persist heartbeat attempt terminal state");
+            }
+        };
         try {
             if (run.status === "queued") {
                 const claimed = await claimQueuedRun(run);
@@ -178,6 +203,7 @@ export function createHeartbeatExecuteHandlers(context) {
             let finalRunOutput = null;
             let ownsTerminalState = false;
             let shouldCompleteTerminalEffects = false;
+            let activeAttemptSpec = null;
             const finalizeExecutionTranscript = () => {
                 stdoutTranscriptBuffer = appendTranscriptEntriesFromChunk({
                     buffer: stdoutTranscriptBuffer,
@@ -592,6 +618,27 @@ export function createHeartbeatExecuteHandlers(context) {
                 sessionDisplayId: previousSessionDisplayId,
                 taskKey,
             };
+            const attemptStride = Math.max(1, buildModelAttemptSpecs(runtimeConfig, agent.agentRuntimeType).length);
+            const recoveryAttemptOrdinal = Math.max(0, Math.floor(Number(run.networkWaitAttemptCount) || 0));
+            const attemptResumeSource = recoveryAttemptOrdinal === 0
+                ? "fresh"
+                : run.recoveryCheckpoint?.continuation === "resume_same_session"
+                    ? "same_session"
+                    : "pristine_replay";
+            const resolveLedgerAttemptIndex = (attempt) => recoveryAttemptOrdinal * attemptStride + attempt.index;
+            const recoveryStartAttemptIndex = recoveryAttemptOrdinal > 0
+                && typeof run.recoveryCheckpoint?.fallbackIndex === "number"
+                ? Math.max(0, Math.floor(run.recoveryCheckpoint.fallbackIndex))
+                : 0;
+            const persistAttempt = async (label, operation) => {
+                try {
+                    return await operation();
+                }
+                catch (error) {
+                    logger.warn({ err: error, runId: run.id, label }, "failed to persist heartbeat attempt ledger state");
+                    return null;
+                }
+            };
             let handle = null;
             let stdoutExcerpt = "";
             let stderrExcerpt = "";
@@ -631,6 +678,7 @@ export function createHeartbeatExecuteHandlers(context) {
                 const startedAt = run.startedAt ?? new Date();
                 const runningWithSession = await persistRunningExecutionContext(run.id, context, {
                     startedAt,
+                    runningSubstate: "executing",
                     sessionIdBefore: runtimeForAdapter.sessionDisplayId ?? runtimeForAdapter.sessionId,
                     sessionParamsBeforeJson: runtimeForAdapter.sessionParams,
                     sessionReuseScope,
@@ -645,29 +693,92 @@ export function createHeartbeatExecuteHandlers(context) {
                     message: "run started",
                 });
                 if (assignmentGuardrailEnabled) {
-                    const workspacePreflight = await inspectAssignmentRunWorkspace({
+                    const initialWorkspacePreflight = await inspectAssignmentRunWorkspace({
                         actualCwd: executionWorkspace.cwd,
                         projectWorkingSetCwd: configuredProjectWorkingSetCwd ?? resolvedWorkspace.cwd,
                     });
+                    const dependencyRepair = initialWorkspacePreflight.ready
+                        ? null
+                        : await repairAssignmentRunWorkspace({ preflight: initialWorkspacePreflight });
+                    const workspacePreflight = dependencyRepair?.final ?? initialWorkspacePreflight;
+                    if (dependencyRepair && !dependencyRepair.succeeded) {
+                        await appendRunEvent(currentRun, {
+                            eventType: "runtime.assignment_dependency_repair",
+                            stream: "system",
+                            level: "warn",
+                            message: dependencyRepair.skippedReason
+                                ? `assignment dependency repair skipped: ${dependencyRepair.skippedReason}`
+                                : "assignment dependency repair did not make the workspace ready",
+                            payload: {
+                                command: dependencyRepair.command,
+                                attempted: dependencyRepair.attempted,
+                                coalesced: dependencyRepair.coalesced,
+                                rechecked: dependencyRepair.rechecked,
+                                succeeded: dependencyRepair.succeeded,
+                                output: dependencyRepair.output,
+                                skippedReason: dependencyRepair.skippedReason,
+                                workspaceFingerprint: dependencyRepair.workspaceFingerprint,
+                                readinessFingerprint: dependencyRepair.readinessFingerprint,
+                                initialReady: dependencyRepair.initial.ready,
+                                finalReady: dependencyRepair.final.ready,
+                            },
+                        });
+                    }
+                    else if (dependencyRepair?.attempted) {
+                        await appendRunEvent(currentRun, {
+                            eventType: "runtime.assignment_dependency_repair",
+                            stream: "system",
+                            level: "info",
+                            message: "assignment dependency repair completed",
+                            payload: {
+                                command: dependencyRepair.command,
+                                attempted: dependencyRepair.attempted,
+                                coalesced: dependencyRepair.coalesced,
+                                rechecked: dependencyRepair.rechecked,
+                                succeeded: dependencyRepair.succeeded,
+                                output: dependencyRepair.output,
+                                skippedReason: dependencyRepair.skippedReason,
+                                workspaceFingerprint: dependencyRepair.workspaceFingerprint,
+                                readinessFingerprint: dependencyRepair.readinessFingerprint,
+                                initialReady: dependencyRepair.initial.ready,
+                                finalReady: dependencyRepair.final.ready,
+                            },
+                        });
+                    }
                     await appendRunEvent(currentRun, {
                         eventType: "runtime.assignment_preflight",
                         stream: "system",
-                        level: workspacePreflight.cwdMatchesProjectWorkingSet ? "info" : "warn",
-                        message: workspacePreflight.cwdMatchesProjectWorkingSet
-                            ? "assignment run workspace preflight passed"
-                            : "assignment run workspace differs from project working set",
-                        payload: workspacePreflight,
+                        level: workspacePreflight.ready ? "info" : "error",
+                        message: workspacePreflight.ready
+                            ? dependencyRepair?.attempted
+                                ? "assignment run workspace preflight passed after dependency repair"
+                                : "assignment run workspace preflight passed"
+                            : "assignment run workspace preflight failed",
+                        payload: {
+                            ...workspacePreflight,
+                            initialReady: dependencyRepair?.initial.ready ?? workspacePreflight.ready,
+                            dependencyRepair: dependencyRepair
+                                ? {
+                                    command: dependencyRepair.command,
+                                    attempted: dependencyRepair.attempted,
+                                    coalesced: dependencyRepair.coalesced,
+                                    rechecked: dependencyRepair.rechecked,
+                                    succeeded: dependencyRepair.succeeded,
+                                    output: dependencyRepair.output,
+                                    skippedReason: dependencyRepair.skippedReason,
+                                }
+                                : null,
+                        },
                     });
                     if (!workspacePreflight.ready) {
-                        throw new Error(workspacePreflight.recoveryCommand
-                            ? `Assignment startup preflight failed in ${workspacePreflight.actualCwd}. Run this recovery command once before retrying: ${workspacePreflight.recoveryCommand}`
-                            : `Assignment project working set is not available: ${workspacePreflight.projectWorkingSetCwd}. Restore or remount that directory before retrying.`);
+                        throw new AssignmentDependencyPreflightError(workspacePreflight, dependencyRepair);
                     }
                 }
                 handle = await runLogStore.begin({
                     orgId: run.orgId,
                     agentId: run.agentId,
                     runId,
+                    append: Boolean(run.logRef),
                 });
                 await db
                     .update(heartbeatRuns)
@@ -884,11 +995,43 @@ export function createHeartbeatExecuteHandlers(context) {
                     requestApproval,
                     waitForApproval,
                 }, {
+                    startAttemptIndex: recoveryStartAttemptIndex,
                     resolveAdapter: findServerAdapter,
                     createAuthToken: (agentRuntimeType) => createLocalAgentJwt(agent.id, agent.orgId, agentRuntimeType, run.id) ?? undefined,
-                    onAttemptStart: (_attempt, attemptAdapter) => {
+                    onAttemptStart: async (attempt, attemptAdapter) => {
+                        activeAttemptSpec = {
+                            index: attempt.index,
+                            fallbackIndex: attempt.fallbackIndex,
+                        };
+                        activeAttemptRef = await persistAttempt("started", () => beginHeartbeatRunAttempt(db, {
+                            orgId: run.orgId,
+                            runId: run.id,
+                            agentId: run.agentId,
+                            attemptIndex: resolveLedgerAttemptIndex(attempt),
+                            fallbackIndex: attempt.fallbackIndex,
+                            runtimeType: attempt.agentRuntimeType ?? agent.agentRuntimeType,
+                            model: attempt.model,
+                            isFallback: attempt.isFallback,
+                            resumeSource: attemptResumeSource,
+                        }));
                         stdoutTranscriptParser = attemptAdapter.parseStdoutLine ?? null;
                     },
+                    onAttemptFailure: async (_attempt, failure) => {
+                        const failureRecord = failure && typeof failure === "object" ? failure : null;
+                        const failureMessage = failure instanceof Error
+                            ? failure.message
+                            : readNonEmptyString(failureRecord?.errorMessage) ?? "Adapter fallback attempt failed";
+                        await finishActiveAttempt({
+                            status: "failed",
+                            errorCode: readNonEmptyString(failureRecord?.errorCode) ?? "adapter_failed",
+                            error: failureMessage,
+                            usageDeltaJson: failureRecord?.usage,
+                            costUsd: failureRecord?.costUsd,
+                            sessionDisplayId: readNonEmptyString(failureRecord?.sessionDisplayId)
+                                ?? readNonEmptyString(failureRecord?.sessionId),
+                            sessionParamsJson: failureRecord?.sessionParams,
+                        });
+                    },
                 });
                 if (assignmentGuardrailCheckpoint) {
                     adapterResult.errorMessage = "Assignment run stopped after reaching its tool failure budget";
@@ -935,6 +1078,123 @@ export function createHeartbeatExecuteHandlers(context) {
                         }
                     }
                 }
+                const networkSuspension = isAgentRuntimeNetworkSuspension(adapterResult.networkSuspension)
+                    ? adapterResult.networkSuspension
+                    : isAgentRuntimeNetworkSuspension(adapterResult.suspension)
+                        ? adapterResult.suspension
+                        : null;
+                if (networkSuspension) {
+                    // A transport suspension is a non-terminal transition. Persist the
+                    // provider checkpoint before releasing the local execution lease so a
+                    // restart or another scheduler tick can safely claim this same Run.
+                    const now = new Date();
+                    const currentRunForWait = await getRun(run.id);
+                    const waitAttempt = (currentRunForWait?.networkWaitAttemptCount ?? 0) + 1;
+                    const backoff = networkWaitBackoffMs(waitAttempt);
+                    const jitteredBackoff = Math.max(1_000, Math.round(backoff * (0.9 + Math.random() * 0.2)));
+                    const nextRetryAt = new Date(now.getTime() + jitteredBackoff);
+                    const checkpoint = {
+                        kind: networkSuspension.kind,
+                        code: networkSuspension.code ?? "provider_transport_unavailable",
+                        transport: networkSuspension.transport,
+                        provider: networkSuspension.provider ?? adapterResult.provider ?? null,
+                        model: networkSuspension.model ?? adapterResult.model ?? null,
+                        submissionPhase: networkSuspension.submissionPhase,
+                        continuation: networkSuspension.continuation,
+                        sessionId: networkSuspension.sessionId ?? adapterResult.sessionDisplayId ?? adapterResult.sessionId ?? null,
+                        sessionParams: networkSuspension.sessionParams ?? adapterResult.sessionParams ?? null,
+                        progress: networkSuspension.progress ?? {
+                            modelOutputObserved: networkSuspension.modelOutputObserved,
+                            toolActivityObserved: networkSuspension.toolActivityObserved,
+                        },
+                        modelOutputObserved: networkSuspension.modelOutputObserved
+                            ?? networkSuspension.progress?.modelOutputObserved
+                            ?? false,
+                        toolActivityObserved: networkSuspension.toolActivityObserved
+                            ?? networkSuspension.progress?.toolActivityObserved
+                            ?? false,
+                        sideEffectRisk: networkSuspension.sideEffectRisk ?? null,
+                        attemptIndex: activeAttemptSpec?.index ?? 0,
+                        fallbackIndex: activeAttemptSpec?.fallbackIndex ?? null,
+                        message: networkSuspension.message,
+                        observedAt: now.toISOString(),
+                    };
+                    const resumedSessionId = readNonEmptyString(networkSuspension.sessionId)
+                        ?? readNonEmptyString(adapterResult.sessionDisplayId)
+                        ?? readNonEmptyString(adapterResult.sessionId)
+                        ?? previousSessionDisplayId
+                        ?? null;
+                    const resumedSessionParams = networkSuspension.sessionParams
+                        ?? adapterResult.sessionParams
+                        ?? runtimeForAdapter.sessionParams
+                        ?? null;
+                    const waitingRun = await db
+                        .update(heartbeatRuns)
+                        .set({
+                        runningSubstate: "waiting_for_network",
+                        networkWaitStartedAt: currentRunForWait?.networkWaitStartedAt ?? now,
+                        networkWaitNextRetryAt: nextRetryAt,
+                        networkWaitAttemptCount: waitAttempt,
+                        recoveryCheckpoint: checkpoint,
+                        sessionIdBefore: resumedSessionId,
+                        sessionParamsBeforeJson: resumedSessionParams,
+                        sessionReuseScope: resumedSessionParams || resumedSessionId ? "explicit" : "none",
+                        processExitedAt: now,
+                        processPid: null,
+                        processStartedAt: null,
+                        executionOwnerToken: null,
+                        executionLeaseExpiresAt: null,
+                        updatedAt: now,
+                    })
+                        .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ...(executionOwnerToken ? [eq(heartbeatRuns.executionOwnerToken, executionOwnerToken)] : [])))
+                        .returning()
+                        .then((rows) => rows[0] ?? null);
+                    if (!waitingRun)
+                        return;
+                    networkSuspended = true;
+                    const waitingAttemptRef = activeAttemptRef;
+                    activeAttemptRef = null;
+                    await persistAttempt("waiting_for_network", () => markHeartbeatRunAttemptWaiting(db, waitingAttemptRef, {
+                        submissionPhase: networkSuspension.submissionPhase,
+                        providerThreadId: networkSuspension.providerThreadId ?? adapterResult.providerThreadId,
+                        providerTurnId: networkSuspension.providerTurnId ?? adapterResult.providerTurnId,
+                        sessionDisplayId: resumedSessionId,
+                        sessionParamsJson: resumedSessionParams,
+                        checkpointJson: checkpoint,
+                        errorCode: networkSuspension.code ?? "provider_transport_unavailable",
+                        error: networkSuspension.message ?? null,
+                        suspendedAt: now,
+                    }));
+                    await appendRunEvent(waitingRun, {
+                        eventType: "network.waiting",
+                        stream: "system",
+                        level: "info",
+                        message: "run waiting for network",
+                        payload: {
+                            attempt: waitAttempt,
+                            nextRetryAt: nextRetryAt.toISOString(),
+                            backoffMs: jitteredBackoff,
+                            suspension: checkpoint,
+                        },
+                    });
+                    publishLiveEvent({
+                        orgId: waitingRun.orgId,
+                        type: "heartbeat.run.status",
+                        payload: {
+                            runId: waitingRun.id,
+                            agentId: waitingRun.agentId,
+                            status: waitingRun.status,
+                            executionPhase: "waiting_for_network",
+                            networkWaitAttemptCount: waitAttempt,
+                            networkWaitNextRetryAt: nextRetryAt.toISOString(),
+                        },
+                    });
+                    if (handle) {
+                        await runLogStore.finalize(handle).catch(() => undefined);
+                        handle = null;
+                    }
+                    return;
+                }
                 const nextSessionState = resolveNextSessionState({
                     codec: sessionCodec,
                     adapterResult,
@@ -1101,6 +1361,19 @@ export function createHeartbeatExecuteHandlers(context) {
                     expectedExecutionOwnerToken: executionOwnerToken,
                 });
                 ownsTerminalState = Boolean(claimedTerminalRun);
+                await finishActiveAttempt({
+                    status,
+                    submissionPhase: adapterResult.submissionPhase,
+                    providerThreadId: adapterResult.providerThreadId,
+                    providerTurnId: adapterResult.providerTurnId,
+                    sessionDisplayId: nextSessionState.displayId ?? nextSessionState.legacySessionId,
+                    sessionParamsJson: adapterResult.clearSession ? {} : nextSessionState.params,
+                    usageDeltaJson: usageJson ?? adapterResult.usage,
+                    costUsd: adapterResult.costUsd,
+                    errorCode: terminalEvidence.errorCode,
+                    error: terminalEvidence.error,
+                    finishedAt: terminalEvidence.finishedAt,
+                });
                 if (!claimedTerminalRun) {
                     await reconcileRunEvidence(run.id, terminalEvidence);
                     await reconcileTerminalEffectsIntent(run.id, {
@@ -1199,7 +1472,9 @@ export function createHeartbeatExecuteHandlers(context) {
                 }
             }
             catch (err) {
-                const isWorkspacePreflightFailure = isWorkspacePermissionPreflightError(err) ||
+                const isAssignmentDependencyPreflightFailure = err instanceof AssignmentDependencyPreflightError;
+                const isWorkspacePreflightFailure = isAssignmentDependencyPreflightFailure ||
+                    isWorkspacePermissionPreflightError(err) ||
                     isManagedWorkspaceConfigurationError(err);
                 const message = redactCurrentUserText(err instanceof Error ? err.message : "Unknown adapter failure", await getCurrentUserRedactionOptions());
                 transcriptFallbackResult = {
@@ -1297,6 +1572,14 @@ export function createHeartbeatExecuteHandlers(context) {
                     expectedExecutionOwnerToken: executionOwnerToken,
                 });
                 ownsTerminalState = Boolean(claimedFailedRun);
+                await finishActiveAttempt({
+                    status: "failed",
+                    errorCode: failureEvidence.errorCode,
+                    error: failureEvidence.error,
+                    sessionDisplayId: previousSessionDisplayId,
+                    sessionParamsJson: previousSessionParams,
+                    finishedAt: failureEvidence.finishedAt,
+                });
                 if (!claimedFailedRun)
                     await reconcileRunEvidence(run.id, failureEvidence);
                 const failedRun = claimedFailedRun ?? await getRun(run.id);
@@ -1310,7 +1593,11 @@ export function createHeartbeatExecuteHandlers(context) {
                     if (ownsTerminalState) {
                         await appendForbiddenMarkerEvent(failedRun, buildForbiddenMarkerScan(null));
                         await appendRunEvent(failedRun, {
-                            eventType: isWorkspacePreflightFailure ? "runtime.workspace_preflight_failed" : "error",
+                            eventType: isAssignmentDependencyPreflightFailure
+                                ? "runtime.assignment_preflight_failed"
+                                : isWorkspacePreflightFailure
+                                    ? "runtime.workspace_preflight_failed"
+                                    : "error",
                             stream: "system",
                             level: "error",
                             message,
@@ -1318,7 +1605,9 @@ export function createHeartbeatExecuteHandlers(context) {
                                 ? {
                                     payload: {
                                         errorCode: err.errorCode,
-                                        failure: err.failure,
+                                        ...(isAssignmentDependencyPreflightFailure
+                                            ? err.failure
+                                            : { failure: err.failure }),
                                     },
                                 }
                                 : {}),
@@ -1351,6 +1640,12 @@ export function createHeartbeatExecuteHandlers(context) {
             // The inner catch did not fire, so we must record the failure here.
             const message = outerErr instanceof Error ? outerErr.message : "Unknown setup failure";
             logger.error({ err: outerErr, runId }, "heartbeat execution setup failed");
+            await finishActiveAttempt({
+                status: "failed",
+                errorCode: "adapter_failed",
+                error: message,
+                finishedAt: new Date(),
+            });
             const latestRun = await getRun(runId).catch(() => null);
             const claimedFailedRun = latestRun?.status === "running"
                 ? await transitionRunToTerminal(runId, "failed", {
@@ -1388,7 +1683,8 @@ export function createHeartbeatExecuteHandlers(context) {
             await releaseRuntimeServicesForRun(run.id).catch(() => undefined);
             runAbortControllers.delete(run.id);
             activeRunExecutions.delete(run.id);
-            await startNextQueuedRunForAgent(run.agentId);
+            if (!networkSuspended)
+                await startNextQueuedRunForAgent(run.agentId);
         }
     }
     return { executeRun };