@tickernelz/paperclip-pro-server 2026.930.6 → 2026.1001.1

package/dist/services/heartbeat.js

@@ -24,7 +24,7 @@ import { claimQueuedNativeReviewRun } from "./native-runtime/native-review-dispa
 import { buildNativeReviewRequest } from "./native-runtime/native-review-prompt.js";
 import { isGracefulShutdownInterruptedRun, legacyExecutionNeedsReconciliation, terminalizeLegacyExecution, } from "./legacy-execution-recovery.js";
 import { adapterExecutionControls, captureAdapterStopOwnership, createAdapterExecutionControl, registerAdapterExecutionControl, waitForAdapterStop, } from "./adapter-execution-control.js";
-import { executionFailureRetryCount } from "./execution-recovery-attempt.js";
+import { executionFailureRetryCount, processLossContinuationCount, } from "./execution-recovery-attempt.js";
 import { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js";
 export { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js";
 import { buildExecutionContinuation } from "./execution-continuation.js";
@@ -42,6 +42,7 @@ import path from "node:path";
 import { execFile as execFileCallback } from "node:child_process";
 import { promisify } from "node:util";
 import { createHash, randomUUID } from "node:crypto";
+import { setTimeout as delay } from "node:timers/promises";
 import { and, asc, desc, eq, exists, getTableColumns, gt, gte, inArray, isNull, isNotNull, lt, lte, ne, notInArray, or, sql, } from "drizzle-orm";
 import { AGENT_DEFAULT_MAX_CONCURRENT_RUNS, INSTANCE_DEFAULT_LOCAL_START_WAIT_BYPASS_SEC, INSTANCE_DEFAULT_MAX_CONCURRENT_LOCAL_RUNS, INSTANCE_DEFAULT_MAX_CONCURRENT_LOCAL_STARTS, INSTANCE_LOCAL_START_WAIT_BYPASS_SEC_MAX, INSTANCE_LOCAL_START_WAIT_BYPASS_SEC_MIN, INSTANCE_MAX_CONCURRENT_LOCAL_RUNS_MAX, INSTANCE_MAX_CONCURRENT_LOCAL_RUNS_MIN, INSTANCE_MAX_CONCURRENT_LOCAL_STARTS_MAX, INSTANCE_MAX_CONCURRENT_LOCAL_STARTS_MIN, LOCAL_CLI_ADAPTER_TYPES, CHAT_PROVIDERS, CONNECTION_INTENT_AGENT_GUIDANCE, CONNECTION_RUNTIME_TOOL_NAMES, ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY, PROVIDER_QUOTA_MONITOR_SERVICE_NAME, envBindingSchema, isToolConnectionAttentionHealth, } from "@tickernelz/paperclip-pro-shared";
 import { agents, agentConfigRevisions, agentRuntimeState, agentSessionGoalActions, agentTaskSessions, agentWakeupRequests, activityLog, approvals, assets, chatConversations, chatDeliveries, chatEndpoints, chatMessageLinks, companyMemberships, companySkillTestRuns, companySkillVersions, companySkills as companySkillsTable, companies, completionContracts, costEvents, documentAnnotationComments, documentAnnotationThreads, documentRevisions, environmentLeases, issueDocuments, executionWorkspaces, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueAttachments, issueComments, issuePlanDecompositions, issueRecoveryActions, issueRelations, issueThreadInteractions, issues, issueWorkProducts, nativeRunFinalizations, projects, projectWorkspaces, routineRevisions, routineRuns, routines, toolMcpGateways, toolMcpGatewayTokens, toolCatalogEntries, toolConnectionInstalls, toolConnections, toolProfileEntries, toolProfiles, workspaceOperations, } from "@tickernelz/paperclip-pro-db";
@@ -323,9 +324,23 @@ const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_JITTER_RATIO = 0;
 const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON = "transient_failure";
 const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry";
 const PROCESS_LOST_RETRY_WAKE_REASON = "process_lost_retry";
+const PROCESS_LOSS_RETRY_REASON = "process_lost";
+const PROCESS_LOSS_CONTINUATION_MAX_ATTEMPTS = 2;
+const PROCESS_LOSS_CONTINUATION_DELAY_MS = 30_000;
+const PROCESS_LOSS_ERROR_CODES = {
+    orphaned_running_run: true,
+    server_shutdown_interrupted: true,
+    process_lost: true,
+    [DETACHED_PROCESS_ERROR_CODE]: true,
+};
 function isTransientWorkspaceGitScanCode(code) {
     return code === WORKSPACE_GIT_SCAN_ERROR_CODES.timeout || code === WORKSPACE_GIT_SCAN_ERROR_CODES.saturated;
 }
+const ADAPTER_SELF_SIGNALLED_ERROR_CODES = {
+    codex_output_inactivity_monitor: true,
+    paperclip_mcp_unavailable: true,
+    hermes_gateway_cancelled: true,
+};
 const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length;
 export { INTERACTION_CONTINUATION_INFRA_RETRY_REASON, INTERACTION_CONTINUATION_INFRA_WAKE_REASON, };
 const INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS = 2;
@@ -798,6 +813,15 @@ const nativeSessionResumeDispatchTimers = new Map();
 // resolveHeartbeatSchedulingSuppression() check and every heartbeatService()
 // instance see the same drain.
 let taskDrainState = null;
+let serverShutdownSignal = null;
+const shutdownTerminatedRunIds = new Set();
+export function beginHeartbeatServerShutdown(signal) {
+    serverShutdownSignal ??= signal;
+}
+export function resetHeartbeatServerShutdownForTests() {
+    serverShutdownSignal = null;
+    shutdownTerminatedRunIds.clear();
+}
 function readTaskDrain(now) {
     if (taskDrainState &&
         taskDrainState.expiresAt !== null &&
@@ -3917,6 +3941,7 @@ const SESSION_CONFIG_FINGERPRINT_KEY = "__paperclipConfigFingerprint";
 const SESSION_CONFIG_FINGERPRINT_VERSION_KEY = "__paperclipConfigFingerprintVersion";
 const SESSION_CONFIG_CATEGORIES_KEY = "__paperclipConfigCategories";
 const SESSION_CONFIG_CATEGORY_FINGERPRINTS_KEY = "__paperclipConfigCategoryFingerprints";
+const SESSION_CONFIG_RESUME_FINGERPRINT_KEY = "__paperclipConfigResumeFingerprint";
 const PAPERCLIP_SESSION_METADATA_KEYS = new Set([
     SESSION_AI_CREDENTIAL_IDENTITY_KEY,
     SESSION_CONFIGURED_MODEL_KEY,
@@ -3924,6 +3949,7 @@ const PAPERCLIP_SESSION_METADATA_KEYS = new Set([
     SESSION_CONFIG_FINGERPRINT_VERSION_KEY,
     SESSION_CONFIG_CATEGORIES_KEY,
     SESSION_CONFIG_CATEGORY_FINGERPRINTS_KEY,
+    SESSION_CONFIG_RESUME_FINGERPRINT_KEY,
 ]);
 const WORKSPACE_CONFIG_FINGERPRINT_METADATA_KEY = "configFingerprint";
 const EFFECTIVE_RUN_SESSION_CONFIG_CATEGORIES = [
@@ -3948,6 +3974,19 @@ const EFFECTIVE_RUN_WORKSPACE_CONFIG_CATEGORIES = [
     "environment",
     "realization",
 ];
+const RESUME_NEUTRAL_ADAPTER_CONFIG_KEYS = [
+    "instructionsBundleMode",
+    "instructionsRootPath",
+    "instructionsEntryFile",
+    "instructionsFilePath",
+    "promptTemplate",
+    "bootstrapPromptTemplate",
+    "paperclipSkillSync",
+    "paperclipRuntimeSkills",
+    "paperclipConnectorSkillDigest",
+];
+const PROCESS_LOSS_RESUME_NEUTRAL_SESSION_CONFIG_CATEGORIES = { instructions: true, runtimeSkills: true };
+const PROCESS_LOSS_RESUME_TARGET_SESSION_CONFIG_CATEGORIES = { adapter: true, adapterConfig: true };
 /**
  * Projectless native runs bind their immutable envelope to the run id even
  * though no project-scoped execution-workspace row exists. Treat that value
@@ -4118,6 +4157,7 @@ function readConfigFingerprintFromSessionParams(sessionParams) {
         version,
         categories: readConfigCategoriesFromSessionParams(sessionParams),
         categoryFingerprints: parseStoredConfigCategoryFingerprints(sessionParams[SESSION_CONFIG_CATEGORY_FINGERPRINTS_KEY]),
+        resumeFingerprint: readNonEmptyString(sessionParams[SESSION_CONFIG_RESUME_FINGERPRINT_KEY]),
     };
 }
 function describeEffectiveRunConfigCategories(categories) {
@@ -4366,9 +4406,10 @@ function buildSessionConfigCategoryValues(input) {
 export async function buildEffectiveRunSessionConfigMetadata(input) {
     const secretManifest = input.secretManifest ?? [];
     const instructions = await resolveInstructionsConfigFingerprintMetadata(input.effectiveAdapterConfig);
+    const fingerprintAdapterConfig = managedAiSessionFingerprintConfig(input.effectiveAdapterConfig, input.managedAiHome);
     const categoryValues = buildSessionConfigCategoryValues({
         adapterType: input.adapterType,
-        effectiveAdapterConfig: managedAiSessionFingerprintConfig(input.effectiveAdapterConfig, input.managedAiHome),
+        effectiveAdapterConfig: fingerprintAdapterConfig,
         agentRuntimeConfig: input.agentRuntimeConfig,
         instructions,
         issueOverrides: input.issueOverrides,
@@ -4391,11 +4432,26 @@ export async function buildEffectiveRunSessionConfigMetadata(input) {
         subcategories: EFFECTIVE_RUN_SESSION_CONFIG_CATEGORIES,
         secretManifest,
     });
+    const resumeAdapterConfig = { ...fingerprintAdapterConfig };
+    for (const key of RESUME_NEUTRAL_ADAPTER_CONFIG_KEYS)
+        delete resumeAdapterConfig[key];
+    const { resume: resumeFingerprint } = createEffectiveRunConfigSubcategoryFingerprints({
+        category: "session",
+        value: {
+            resume: {
+                adapterType: input.adapterType,
+                adapterConfig: resumeAdapterConfig,
+            },
+        },
+        subcategories: ["resume"],
+        secretManifest,
+    });
     return {
         version: EFFECTIVE_RUN_CONFIG_FINGERPRINT_VERSION,
         fingerprint: fingerprints.sessionFingerprint.fingerprint,
         categories: [...EFFECTIVE_RUN_SESSION_CONFIG_CATEGORIES],
         categoryFingerprints,
+        resumeFingerprint,
         fingerprints,
     };
 }
@@ -4609,6 +4665,8 @@ function attachPaperclipSessionMetadataToSessionParams(sessionParams, configured
         next[SESSION_CONFIG_CATEGORIES_KEY] = configMetadata.categories;
         next[SESSION_CONFIG_CATEGORY_FINGERPRINTS_KEY] =
             configMetadata.categoryFingerprints;
+        next[SESSION_CONFIG_RESUME_FINGERPRINT_KEY] =
+            configMetadata.resumeFingerprint;
     }
     return next;
 }
@@ -4638,6 +4696,17 @@ export function stripPaperclipSessionMetadataFromSessionParams(sessionParams) {
     }
     return next;
 }
+function isProcessLossContinuationRun(run, contextSnapshot) {
+    return (run.scheduledRetryReason === PROCESS_LOSS_RETRY_REASON ||
+        readNonEmptyString(contextSnapshot?.wakeReason) ===
+            PROCESS_LOST_RETRY_WAKE_REASON);
+}
+function processLossContinuationCanResumeAcross(input) {
+    const resumeTargetUnchanged = input.storedResumeFingerprint === input.nextResumeFingerprint;
+    return input.changedCategories.every((category) => PROCESS_LOSS_RESUME_NEUTRAL_SESSION_CONFIG_CATEGORIES[category] === true ||
+        (resumeTargetUnchanged &&
+            PROCESS_LOSS_RESUME_TARGET_SESSION_CONFIG_CATEGORIES[category] === true));
+}
 export function resolveTaskSessionConfigFreshness(input) {
     if (!input.hasTaskSession) {
         return {
@@ -4675,7 +4744,14 @@ export function resolveTaskSessionConfigFreshness(input) {
                 previous: storedConfig.categoryFingerprints,
                 next: input.configMetadata.categoryFingerprints,
             });
-            reasons.push(`effective run configuration changed: ${describeEffectiveRunConfigCategories(changedCategories)}`);
+            if (!input.processLossContinuation ||
+                !processLossContinuationCanResumeAcross({
+                    changedCategories,
+                    storedResumeFingerprint: storedConfig.resumeFingerprint,
+                    nextResumeFingerprint: input.configMetadata.resumeFingerprint,
+                })) {
+                reasons.push(`effective run configuration changed: ${describeEffectiveRunConfigCategories(changedCategories)}`);
+            }
         }
     }
     if (input.wakeResetReason)
@@ -6257,18 +6333,20 @@ export function resolveHeartbeatSchedulingSuppression(env = process.env, overrid
         isTruthyRuntimeEnvValue(env.PAPERCLIP_RESTORE_IN_PROGRESS)) {
         return { suppressed: true, reason: "database_restore_in_progress" };
     }
+    if (serverShutdownSignal !== null) {
+        return { suppressed: true, reason: "server_shutdown" };
+    }
     if (readTaskDrain(new Date()) !== null) {
         return { suppressed: true, reason: "task_drain" };
     }
     return { suppressed: false, reason: null };
 }
 export function heartbeatService(db, options = {}) {
-    let shutdownInProgress = false;
+    const runtimeEnv = options.runtimeEnv ?? process.env;
     const instanceSettings = instanceSettingsService(db);
     const getCurrentUserRedactionOptions = async () => ({
         enabled: (await instanceSettings.getGeneral()).censorUsernameInLogs,
     });
-    const runtimeEnv = options.runtimeEnv ?? process.env;
     const inWorktreeRuntime = isTruthyRuntimeEnvValue(runtimeEnv.PAPERCLIP_IN_WORKTREE);
     // Preview worktree instances suppress the run engine by default. Users can lift
     // that per-worktree via the `enableWorktreeRunExecution` experimental setting
@@ -6357,14 +6435,14 @@ export function heartbeatService(db, options = {}) {
             const agent = await getAgent(run.agentId);
             if (!agent || agent.companyId !== run.companyId)
                 return null;
+            if (PROCESS_LOSS_ERROR_CODES[run.errorCode ?? ""])
+                return enqueueProcessLossRetry(run, agent, new Date());
             const result = await scheduleBoundedRetryForRun(run, agent);
             return result.outcome === "scheduled" ? result.run : null;
         },
-        // Mirrors scheduleBoundedRetryForRun's transient budget check: a failed
-        // or interrupted run that has already consumed every bounded transient
-        // attempt cannot be retried again through this lane.
-        transientRetryBudgetSpent: (run) => executionFailureRetryCount(run) >=
-            BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS,
+        retryBudgetSpent: (run) => PROCESS_LOSS_ERROR_CODES[run.errorCode ?? ""]
+            ? processLossContinuationCount(run) >= PROCESS_LOSS_CONTINUATION_MAX_ATTEMPTS
+            : executionFailureRetryCount(run) >= BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS,
         continueProcessGoneRun,
     });
     const runDispatch = createRunDispatch(db, {
@@ -6547,10 +6625,15 @@ export function heartbeatService(db, options = {}) {
                 const [source] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, effect.companyId), eq(heartbeatRuns.id, effect.runId)));
                 const agent = source ? await getAgent(source.agentId) : null;
                 if (source && agent && agent.companyId === source.companyId) {
-                    await scheduleBoundedRetryForRun(source, agent, effect.reviewParticipant ? {
-                        retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON,
-                        wakeReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON,
-                    } : undefined);
+                    if (PROCESS_LOSS_ERROR_CODES[source.errorCode ?? ""]) {
+                        await enqueueProcessLossRetry(source, agent, new Date());
+                    }
+                    else {
+                        await scheduleBoundedRetryForRun(source, agent, effect.reviewParticipant ? {
+                            retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON,
+                            wakeReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON,
+                        } : undefined);
+                    }
                 }
             }
             else if (effect.kind === "run_queued") {
@@ -9970,15 +10053,18 @@ export function heartbeatService(db, options = {}) {
         return { outcome: "retry_exhausted", queuedRun: null };
     }
     async function enqueueProcessLossRetry(run, agent, now, opts) {
-        // Native sessions have their own fenced same-run controller. Legacy
-        // bootstrap recovery shares the durable delay and incident counter with
-        // transient retries; process loss must not open a second retry budget.
         if (run.runtimeMode === "native")
             return null;
         if (!isGracefulShutdownInterruptedRun(run) &&
             legacyExecutionNeedsReconciliation(run))
             return null;
-        const scheduled = await scheduleBoundedRetryForRun(run, agent, { now, wakeReason: opts?.wakeReason });
+        const scheduled = await scheduleBoundedRetryForRun(run, agent, {
+            now,
+            wakeReason: opts?.wakeReason,
+            retryReason: PROCESS_LOSS_RETRY_REASON,
+            maxAttempts: PROCESS_LOSS_CONTINUATION_MAX_ATTEMPTS,
+            delayMs: PROCESS_LOSS_CONTINUATION_DELAY_MS,
+        });
         return scheduled.outcome === "scheduled" ? scheduled.run : null;
     }
     async function continueProcessGoneRun(run) {
@@ -10061,7 +10147,7 @@ export function heartbeatService(db, options = {}) {
         return (readNonEmptyString(parseObject(input.adapterConfig).engine) !== "cli");
     }
     async function prepareHotRestartShutdown(signal, now = new Date()) {
-        shutdownInProgress = true;
+        beginHeartbeatServerShutdown(signal);
         const idleSessions = await closeIdleWarmNativeSessionsForRestart();
         if (idleSessions.failed > 0) {
             logger.warn({ idleSessions }, "idle native sessions could not checkpoint before controller shutdown");
@@ -10112,6 +10198,25 @@ export function heartbeatService(db, options = {}) {
             ...intent,
             previousServerVersion: intent.previousServerVersion ?? serverVersion,
         };
+        if (process.platform === "linux" && runtimeEnv.PAPERCLIP_SERVICE_MANAGED === "1") {
+            const drainRunIds = snapshotRuns.map((run) => run.runId);
+            await writeHotRestartShutdownSnapshot({
+                intent: intentWithVersion,
+                signal,
+                activeRuns: snapshotRuns,
+                drainReason: "service_stop_kills_children",
+                drainRunIds,
+                capturedAt: now,
+            });
+            logger.warn({ signal, previousServerPid: intent.previousServerPid, drainRunIds }, "systemd stops every process left in the service cgroup; draining live runs into process-loss continuations instead of adopting them");
+            return {
+                mode: "service_drain_required",
+                skipDrain: false,
+                activeRunIds: drainRunIds,
+                drainRunIds,
+                drainReason: "service_stop_kills_children",
+            };
+        }
         const serverStdioRuns = activeRuns.filter(isServerStdioBoundHotRestartRun);
         if (serverStdioRuns.length > 0) {
             const activeServerStdioRunIds = serverStdioRuns.map(({ run }) => run.id);
@@ -10288,18 +10393,15 @@ export function heartbeatService(db, options = {}) {
                 classify(candidate, "finalized_while_down", `run_status_${run.status}`, patch);
                 continue;
             }
-            const hasSelectiveAcpDrain = intent.drainReason === "active_acp_run" &&
+            const hasSelectiveDrain = (intent.drainReason === "active_acp_run" ||
+                intent.drainReason === "service_stop_kills_children") &&
                 (intent.drainRunIds?.length ?? 0) > 0;
-            if (hasSelectiveAcpDrain &&
+            if (hasSelectiveDrain &&
                 intent.drainRunIds?.includes(candidate.runId)) {
-                // A selective ACP drain is expected to persist a terminal row before
-                // the new server starts. If the process was terminated but that write
-                // failed, surface the run as lost instead of hiding it as an expected
-                // drain skip.
                 classify(candidate, "lost", "selective_drain_not_finalized", patch);
                 continue;
             }
-            if (intent.drainRequired && !hasSelectiveAcpDrain) {
+            if (intent.drainRequired && !hasSelectiveDrain) {
                 classify(candidate, "skipped", "drain_required", patch);
                 continue;
             }
@@ -10527,6 +10629,7 @@ export function heartbeatService(db, options = {}) {
         };
     }
     async function drainRunningRunsForShutdown(signal, now = new Date(), runIds = null) {
+        beginHeartbeatServerShutdown(signal);
         const selectedRunIds = runIds ? [...new Set(runIds)] : null;
         if (selectedRunIds?.length === 0) {
             return {
@@ -10549,6 +10652,27 @@ export function heartbeatService(db, options = {}) {
         const interruptedRunIds = [];
         const retryRunIds = [];
         const restartSuspendedRunIds = [];
+        const legacyTerminations = new Map();
+        for (const { run } of activeRuns) {
+            const running = runningProcesses.get(run.id);
+            if (run.runtimeMode === "native" ||
+                !running ||
+                (run.controllerBootId && run.controllerBootId !== legacyControllerBootId) ||
+                isNativeRunnerOwnershipHeld(run))
+                continue;
+            shutdownTerminatedRunIds.add(run.id);
+            legacyTerminations.set(run.id, terminateHeartbeatRunProcess({
+                pid: running.child.pid,
+                processGroupId: running.processGroupId,
+                graceMs: Math.max(1, running.graceSec) * 1000,
+            }).then(() => null, (error) => error ?? new Error("Run process termination failed")));
+        }
+        await Promise.all(legacyTerminations.values());
+        const executorDeadline = Date.now() + (options.shutdownExecutorSettleTimeoutMs ?? 15_000);
+        while ([...legacyTerminations.keys()].some((runId) => activeRunExecutions.has(runId)) &&
+            Date.now() < executorDeadline) {
+            await delay(50);
+        }
         for (const { run, agent } of activeRuns) {
             // Shutdown owns only this boot's legacy executions. Expired foreign
             // owners belong to the reaper, not another container's drain.
@@ -10622,7 +10746,12 @@ export function heartbeatService(db, options = {}) {
                         runtimeMode: run.runtimeMode,
                     });
                 }
-                if (running) {
+                if (legacyTerminations.has(run.id)) {
+                    const terminationError = await legacyTerminations.get(run.id);
+                    if (terminationError)
+                        throw terminationError;
+                }
+                else if (running) {
                     await terminateHeartbeatRunProcess({
                         pid: running.child.pid,
                         processGroupId: running.processGroupId,
@@ -10648,8 +10777,20 @@ export function heartbeatService(db, options = {}) {
                     errorMessage: message,
                 }),
             });
-            if (!interruptedStatus.updated || !interruptedStatus.run)
+            if (!interruptedStatus.updated || !interruptedStatus.run) {
+                if (interruptedStatus.run &&
+                    isGracefulShutdownInterruptedRun(interruptedStatus.run)) {
+                    interruptedRunIds.push(run.id);
+                    const [executorRetry] = await db
+                        .select({ id: heartbeatRuns.id })
+                        .from(heartbeatRuns)
+                        .where(eq(heartbeatRuns.retryOfRunId, run.id))
+                        .limit(1);
+                    if (executorRetry)
+                        retryRunIds.push(executorRetry.id);
+                }
                 continue;
+            }
             let interrupted = interruptedStatus.run;
             await setWakeupStatus(run.wakeupRequestId, "cancelled", {
                 finishedAt: now,
@@ -10708,11 +10849,13 @@ export function heartbeatService(db, options = {}) {
         const retryReason = opts?.retryReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON;
         const wakeReason = opts?.wakeReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON;
         const maxAttempts = Math.max(0, Math.floor(opts?.maxAttempts ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS));
-        const nextAttempt = (retryReason === WORKSPACE_BUSY_RETRY_REASON ||
-            retryReason === AI_CONNECTION_BUSY_RETRY_REASON ||
-            retryReason === MAX_TURN_CONTINUATION_RETRY_REASON
-            ? (run.scheduledRetryAttempt ?? 0)
-            : executionFailureRetryCount(run)) + 1;
+        const nextAttempt = retryReason === PROCESS_LOSS_RETRY_REASON
+            ? processLossContinuationCount(run) + 1
+            : (retryReason === WORKSPACE_BUSY_RETRY_REASON ||
+                retryReason === AI_CONNECTION_BUSY_RETRY_REASON ||
+                retryReason === MAX_TURN_CONTINUATION_RETRY_REASON
+                ? (run.scheduledRetryAttempt ?? 0)
+                : executionFailureRetryCount(run)) + 1;
         const computedBaseSchedule = opts?.delayMs != null
             ? nextAttempt <= maxAttempts
                 ? {
@@ -10740,16 +10883,19 @@ export function heartbeatService(db, options = {}) {
         const contextSnapshot = parseObject(run.contextSnapshot);
         const issueId = readNonEmptyString(contextSnapshot.issueId);
         if (!baseSchedule) {
+            const spentAttempts = retryReason === PROCESS_LOSS_RETRY_REASON
+                ? nextAttempt - 1
+                : (run.scheduledRetryAttempt ?? 0);
             const exhaustion = {
                 retryReason,
-                scheduledRetryAttempt: run.scheduledRetryAttempt ?? 0,
+                scheduledRetryAttempt: spentAttempts,
                 maxAttempts,
             };
             await appendRunEvent(run, {
                 eventType: "lifecycle",
                 stream: "system",
                 level: "warn",
-                message: `Bounded retry exhausted after ${run.scheduledRetryAttempt ?? 0} scheduled attempts; no further automatic retry will be queued`,
+                message: `Bounded retry exhausted after ${spentAttempts} scheduled attempts; no further automatic retry will be queued`,
                 payload: exhaustion,
                 retryExhaustion: exhaustion,
             });
@@ -10908,6 +11054,14 @@ export function heartbeatService(db, options = {}) {
             ...(retryReason === AI_CONNECTION_BUSY_RETRY_REASON
                 ? { failureRetriesBeforeAiConnectionWait: executionFailureRetryCount(run) }
                 : {}),
+            ...(retryReason === PROCESS_LOSS_RETRY_REASON
+                ? {
+                    failureRetriesBeforeProcessLoss: executionFailureRetryCount(run),
+                    processLossContinuations: nextAttempt,
+                }
+                : retryReason === MAX_TURN_CONTINUATION_RETRY_REASON
+                    ? { processLossContinuations: undefined }
+                    : {}),
             ...(shouldQuarantineWorkspaceForRetry
                 ? {
                     workspaceValidationRecovery: {
@@ -15595,6 +15749,7 @@ export function heartbeatService(db, options = {}) {
                 taskSessionParams: taskSession?.sessionParamsJson ?? taskSessionDecodedParams,
                 configMetadata: sessionConfigMetadata,
                 wakeResetReason: wakeSessionResetReason,
+                processLossContinuation: isProcessLossContinuationRun(run, context),
                 preserveLegacySessionWithoutConfigMetadata: acceptedPlanContinuationWake && !acceptedPlanWakeRoutingDecision,
             });
             const resetTaskSession = shouldResetTaskSessionForWake(context) || sessionConfigFreshness.reset;
@@ -18272,7 +18427,12 @@ export function heartbeatService(db, options = {}) {
                     failedProcessRunCancellations.get(run.id);
                 await processCancellation?.settled;
                 let outcome;
+                let processLossErrorCode = null;
                 const latestRun = await getRun(run.id);
+                const latestResult = parseObject(latestRun?.resultJson);
+                const paperclipCancellationRecorded = Boolean(processCancellation) ||
+                    Boolean(latestResult.executionCancellation) ||
+                    Boolean(latestResult.startupCancellation);
                 if (isHeartbeatRunTerminalStatus(latestRun?.status)) {
                     outcome = latestRun.status;
                 }
@@ -18288,6 +18448,11 @@ export function heartbeatService(db, options = {}) {
                                 ? "cancelled"
                                 : "failed";
                 }
+                else if (shutdownTerminatedRunIds.has(run.id) &&
+                    !paperclipCancellationRecorded) {
+                    outcome = "interrupted";
+                    processLossErrorCode = "server_shutdown_interrupted";
+                }
                 else if (adapterResult.timedOut) {
                     outcome = "timed_out";
                 }
@@ -18299,6 +18464,20 @@ export function heartbeatService(db, options = {}) {
                 }
                 else {
                     outcome = "failed";
+                    if (!paperclipCancellationRecorded &&
+                        (adapterResult.signal === "SIGTERM" ||
+                            adapterResult.signal === "SIGINT" ||
+                            adapterResult.exitCode === 143 ||
+                            adapterResult.exitCode === 130) &&
+                        !ADAPTER_SELF_SIGNALLED_ERROR_CODES[adapterResult.errorCode ?? ""] &&
+                        !hasUnmanagedBackgroundTaskEvidence(parseObject(adapterResult.resultJson))) {
+                        processLossErrorCode =
+                            serverShutdownSignal === null
+                                ? "process_lost"
+                                : "server_shutdown_interrupted";
+                        if (serverShutdownSignal !== null)
+                            outcome = "interrupted";
+                    }
                 }
                 const nextSessionState = resolveNextSessionState({
                     adapterType: agent.adapterType,
@@ -18322,18 +18501,21 @@ export function heartbeatService(db, options = {}) {
                     ? (latestRun?.error ?? adapterResult.errorMessage ?? "Cancelled")
                     : outcome === "succeeded"
                         ? null
-                        : redactCurrentUserText(adapterResult.errorMessage ??
-                            (outcome === "timed_out" ? "Timed out" : "Adapter failed"), currentUserRedactionOptions);
+                        : outcome === "interrupted" && serverShutdownSignal !== null
+                            ? `Interrupted by graceful server shutdown (${serverShutdownSignal})`
+                            : redactCurrentUserText(adapterResult.errorMessage ??
+                                (outcome === "timed_out" ? "Timed out" : "Adapter failed"), currentUserRedactionOptions);
                 const recordedResponsibleUserDenialCode = normalizeResponsibleUserDenialCode(latestRun?.errorCode);
-                const runErrorCode = outcome === "timed_out"
-                    ? "timeout"
-                    : outcome === "cancelled"
-                        ? (latestRun?.errorCode ?? "cancelled")
-                        : outcome === "failed"
-                            ? (adapterResult.errorCode ??
-                                recordedResponsibleUserDenialCode ??
-                                "adapter_failed")
-                            : null;
+                const runErrorCode = processLossErrorCode ??
+                    (outcome === "timed_out"
+                        ? "timeout"
+                        : outcome === "cancelled"
+                            ? (latestRun?.errorCode ?? "cancelled")
+                            : outcome === "failed"
+                                ? (adapterResult.errorCode ??
+                                    recordedResponsibleUserDenialCode ??
+                                    "adapter_failed")
+                                : null);
                 let logSummary = null;
                 if (handle) {
                     logSummary = await runLogStore.finalize(handle);
@@ -18358,7 +18540,9 @@ export function heartbeatService(db, options = {}) {
                         ? "cancelled"
                         : outcome === "timed_out"
                             ? "timed_out"
-                            : "failed";
+                            : outcome === "interrupted"
+                                ? "interrupted"
+                                : "failed";
                 const cacheAdjustedCostUsd = resolveCacheAdjustedCostUsd(adapterResult);
                 const usageJson = normalizedUsage ||
                     adapterResult.costUsd != null ||
@@ -18482,7 +18666,11 @@ export function heartbeatService(db, options = {}) {
                     persistedRun =
                         (await classifyAndPersistRunLiveness(persistedRun, persistedResultJson)) ?? persistedRun;
                 }
-                await setWakeupStatus(run.wakeupRequestId, outcome === "succeeded" ? "completed" : status, {
+                await setWakeupStatus(run.wakeupRequestId, outcome === "succeeded"
+                    ? "completed"
+                    : outcome === "interrupted"
+                        ? "cancelled"
+                        : status, {
                     finishedAt: new Date(),
                     error: runErrorMessage,
                 });
@@ -18624,7 +18812,10 @@ export function heartbeatService(db, options = {}) {
                     catch (err) {
                         await onLog("stderr", `[paperclip] Failed to resolve run presentation: ${err instanceof Error ? err.message : String(err)}\n`);
                     }
-                    if (outcome === "failed" && isMaxTurnExhaustionRun(livenessRun)) {
+                    if (processLossErrorCode) {
+                        await enqueueProcessLossRetry(livenessRun, agent, new Date());
+                    }
+                    else if (outcome === "failed" && isMaxTurnExhaustionRun(livenessRun)) {
                         const policy = parseMaxTurnContinuationPolicy(agent);
                         if (policy.enabled && policy.maxAttempts > 0) {
                             await scheduleBoundedRetryForRun(livenessRun, agent, {
@@ -19340,6 +19531,7 @@ export function heartbeatService(db, options = {}) {
             finally {
                 controllerLease.stop();
                 activeRunExecutions.delete(run.id);
+                shutdownTerminatedRunIds.delete(run.id);
                 clearLocalCliRunStarting(run.id);
                 // A failed owned Stop remains visible until this exact executor settles,
                 // including a graceful exit result arriving after the cancellation error.
@@ -19363,7 +19555,7 @@ export function heartbeatService(db, options = {}) {
             }
             if (!nativeSessionResumeScheduled &&
                 !nativeWorkspaceFinalizeScheduled &&
-                !shutdownInProgress) {
+                serverShutdownSignal === null) {
                 if (latestRun)
                     await resumeRemoteStopComments(latestRun).catch(err => {
                         logger.warn({ err, runId: run.id }, "failed to resume user messages after remote Stop");
@@ -19621,7 +19813,8 @@ export function heartbeatService(db, options = {}) {
         // continuation wake landed mid-drain left its issue in `todo` with no
         // run and no path until a person noticed.
         if (schedulingSuppression.suppressed &&
-            schedulingSuppression.reason !== "task_drain") {
+            schedulingSuppression.reason !== "task_drain" &&
+            schedulingSuppression.reason !== "server_shutdown") {
             await writeSkippedHeartbeatRequest("heartbeat.scheduling_suppressed", {
                 reason: schedulingSuppression.reason,
             });
@@ -21960,6 +22153,7 @@ export function heartbeatService(db, options = {}) {
         dispatchPendingNativeStatusWakeups,
         triggerIssueMonitor,
         reportRunActivity: clearDetachedRunWarning,
+        beginServerShutdown: beginHeartbeatServerShutdown,
         prepareHotRestartShutdown,
         reconcileHotRestartAdoption,
         recoverNativeRunsAfterRestart,