@desplega.ai/agent-swarm 1.131.0 → 1.133.1

package/src/heartbeat/heartbeat.ts

@@ -36,6 +36,11 @@ import {
   updateAgentStatus,
 } from "../be/db";
 import { repointTrackerSyncBySwarmId } from "../be/db-queries/tracker";
+import {
+  agentsWithLiveRuntime,
+  countActiveRuntimeInstancesForAgent,
+  expireStaleRuntimeInstances,
+} from "../be/multi-runtime";
 import { promotePendingSteeringForTask } from "../be/steering";
 import { resolveTemplate } from "../prompts/resolver";
 import {
@@ -48,6 +53,7 @@ import {
   REBOOT_RETRY_PIN_TAG,
 } from "../tasks/worker-follow-up";
 import type { AgentTask } from "../types";
+import { isMultiRuntimeEnabled } from "../utils/multi-runtime";
 import { scrubSecrets } from "../utils/secret-scrubber";
 import { getExecutorRegistry } from "../workflows";
 import { recoverIncompleteRuns } from "../workflows/recovery";
@@ -212,6 +218,7 @@ export interface HeartbeatFindings {
     mentionProcessing: number;
     inboxProcessing: number;
     workflowRuns: number;
+    staleRuntimes: number;
   };
 }
 
@@ -285,9 +292,15 @@ export async function codeLevelTriage(): Promise<HeartbeatFindings> {
       mentionProcessing: 0,
       inboxProcessing: 0,
       workflowRuns: 0,
+      staleRuntimes: 0,
     },
   };
 
+  // 0. Retire runtimes that stopped pinging, before anything reasons about
+  // which workers are alive — otherwise auto-assignment below can hand a task
+  // to an agent this same sweep is about to mark offline, stranding it.
+  findings.staleCleanup.staleRuntimes = expireStaleRuntimeInstances().expired;
+
   // 1. Detect and remediate stalled tasks (tiered: auto-fail dead workers)
   detectAndRemediateStalledTasks(findings);
 
@@ -423,7 +436,7 @@ function remediateCrashedWorkerTask(
         `[Heartbeat] Workflow-step task ${task.id.slice(0, 8)} failed — engine will handle retry (${opts.shortLabel})`,
       );
       const remaining = getActiveTaskCount(task.agentId);
-      if (remaining === 0) updateAgentStatus(task.agentId, "idle");
+      if (remaining === 0) restoreAgentIdleAfterRemediation(task.agentId);
     }
     return;
   }
@@ -443,7 +456,7 @@ function remediateCrashedWorkerTask(
         })`,
       );
       const remaining = getActiveTaskCount(task.agentId);
-      if (remaining === 0) updateAgentStatus(task.agentId, "idle");
+      if (remaining === 0) restoreAgentIdleAfterRemediation(task.agentId);
     }
     return;
   }
@@ -462,7 +475,7 @@ function remediateCrashedWorkerTask(
         `[Heartbeat] Auto-failed task ${task.id.slice(0, 8)} — ${RESUME_BUDGET_EXHAUSTED_REASON} (${opts.shortLabel})`,
       );
       const remaining = getActiveTaskCount(task.agentId);
-      if (remaining === 0) updateAgentStatus(task.agentId, "idle");
+      if (remaining === 0) restoreAgentIdleAfterRemediation(task.agentId);
     }
     return;
   }
@@ -537,7 +550,7 @@ function remediateCrashedWorkerTask(
   if (opts.cleanupActiveSession) deleteActiveSession(task.id);
 
   const remaining = getActiveTaskCount(task.agentId);
-  if (remaining === 0) updateAgentStatus(task.agentId, "idle");
+  if (remaining === 0) restoreAgentIdleAfterRemediation(task.agentId);
 }
 
 /**
@@ -622,7 +635,7 @@ export async function runRebootSweep(): Promise<void> {
 
       // Fix agent status
       if (getActiveTaskCount(task.agentId) === 0) {
-        updateAgentStatus(task.agentId, "idle");
+        restoreAgentIdleAfterRemediation(task.agentId);
       }
 
       // Don't retry system-generated heartbeat tasks
@@ -760,8 +773,20 @@ function autoAssignPoolTasks(findings: HeartbeatFindings): void {
     // assigning to them would just have them exit on their next poll. Filter on
     // the rows already returned (emptyPollCount is populated) rather than
     // re-querying per worker via shouldBlockPolling().
+    // A multi-runtime agent whose runtimes have all died still reads as idle
+    // until its rows expire; assigning to it would strand the task, since
+    // nothing is left to poll for it.
+    // While the mode is on, only an agent with a live runtime can poll, so
+    // anything else — dead runtimes, or a worker that has not re-registered
+    // since the flag was enabled — would have the task stranded on it. With
+    // the flag off this is inert: legacy workers stop refreshing their
+    // retained rows, and filtering on them would park tasks on healthy agents.
+    const multiRuntime = isMultiRuntimeEnabled();
+    const withLiveRuntime = multiRuntime ? agentsWithLiveRuntime() : null;
     const idleWorkers = getIdleWorkersWithCapacity().filter(
-      (w) => (w.emptyPollCount ?? 0) < MAX_EMPTY_POLLS,
+      (w) =>
+        (w.emptyPollCount ?? 0) < MAX_EMPTY_POLLS &&
+        (withLiveRuntime === null || withLiveRuntime.has(w.id)),
     );
     if (idleWorkers.length === 0) return;
 
@@ -959,6 +984,16 @@ function escalateStarvedPoolTasks(findings: HeartbeatFindings): void {
   }
 }
 
+/**
+ * Return a remediated agent to idle — unless multi-runtime liveness says no
+ * process is serving it. Recovery still runs for the task; the agent just
+ * must not be advertised as available when nothing can poll for it.
+ */
+function restoreAgentIdleAfterRemediation(agentId: string): void {
+  if (isMultiRuntimeEnabled() && countActiveRuntimeInstancesForAgent(agentId) === 0) return;
+  updateAgentStatus(agentId, "idle");
+}
+
 /**
  * Call existing stale resource cleanup functions.
  */
@@ -1271,8 +1306,13 @@ export async function runHeartbeatSweep(): Promise<void> {
           mentionProcessing: 0,
           inboxProcessing: 0,
           workflowRuns: 0,
+          staleRuntimes: 0,
         },
       };
+      // Expiry runs even on a cleanup-only tick: an idle agent whose runtime
+      // stopped pinging is exactly the case the preflight gate sees as
+      // "nothing actionable", and it would otherwise stay available forever.
+      cleanupOnlyFindings.staleCleanup.staleRuntimes = expireStaleRuntimeInstances().expired;
       await cleanupStaleResources(cleanupOnlyFindings);
       logFindings(cleanupOnlyFindings);
       return; // Nothing actionable — bail early
@@ -1345,14 +1385,28 @@ export function startHeartbeat(intervalMs = defaultIntervalMs()): void {
 
   console.log(`[Heartbeat] Starting with ${intervalMs}ms interval`);
 
-  // Run aggressive reboot sweep first (no thresholds), then normal sweep cycle
-  setTimeout(async () => {
-    await runRebootSweep();
-    runHeartbeatSweep();
+  // Run aggressive reboot sweep first (no thresholds), then normal sweep cycle.
+  // Both sweeps are best-effort: `runHeartbeatSweep` resets `isSweeping` in a
+  // `finally` but has no `catch`, so a throw anywhere inside it escapes as an
+  // unhandled rejection unless it is caught here.
+  setTimeout(() => {
+    runRebootSweep()
+      .then(() => runHeartbeatSweep())
+      .catch((err) =>
+        console.error(
+          "[Heartbeat] boot sweep failed:",
+          scrubSecrets(err instanceof Error ? err.message : String(err)),
+        ),
+      );
   }, 5000);
 
   heartbeatInterval = setInterval(() => {
-    runHeartbeatSweep();
+    runHeartbeatSweep().catch((err) =>
+      console.error(
+        "[Heartbeat] sweep failed:",
+        scrubSecrets(err instanceof Error ? err.message : String(err)),
+      ),
+    );
   }, intervalMs);
 
   // Also start the checklist interval
@@ -1434,7 +1488,12 @@ export function startHeartbeatChecklist(intervalMs = HEARTBEAT_CHECKLIST_INTERVA
 
   // Recurring checklist starts from the second interval onward
   checklistInterval = setInterval(() => {
-    checkHeartbeatChecklist();
+    checkHeartbeatChecklist().catch((err) =>
+      console.error(
+        "[Heartbeat] checklist check failed:",
+        scrubSecrets(err instanceof Error ? err.message : String(err)),
+      ),
+    );
   }, intervalMs);
 }
 

package/src/hooks/hook.ts

@@ -10,6 +10,7 @@ import {
   type RetrievalRow,
 } from "../be/memory/raters/llm";
 import {
+  buildIndependentIdentityPayloads,
   contentSha256,
   readIdentityBaselines,
   warnProfileFileTooLarge,
@@ -110,6 +111,42 @@ export function shouldShowRegistrationNudge(opts: {
   return true;
 }
 
+export async function postHookProfileUpdate({
+  url,
+  headers,
+  body,
+  label,
+  fetchImpl = fetch,
+}: {
+  url: string;
+  headers: Record<string, string>;
+  body: Record<string, string>;
+  label: string;
+  fetchImpl?: typeof fetch;
+}): Promise<void> {
+  try {
+    const response = await fetchImpl(url, {
+      method: "PUT",
+      headers: {
+        ...headers,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(body),
+    });
+    if (!response.ok) {
+      const responseBody = await response.text();
+      console.error(
+        scrubSecrets(
+          `[hook] ${label} profile sync failed: HTTP ${response.status}${responseBody ? ` — ${responseBody}` : ""}`,
+        ),
+      );
+    }
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    console.error(scrubSecrets(`[hook] ${label} profile sync errored: ${message}`));
+  }
+}
+
 /**
  * Hook response for blocking actions
  * See: https://code.claude.com/docs/en/hooks
@@ -637,23 +674,12 @@ export async function handleHook(): Promise<void> {
     const content = await file.text();
 
     if (!content.trim()) return;
-    if (content.length > MAX_PROFILE_FILE_LENGTH) {
-      warnProfileFileTooLarge(agentId, "claudeMd", content.length, "hook");
-      return;
-    }
-
-    try {
-      await fetch(`${getBaseUrl()}/api/agents/${agentId}/profile`, {
-        method: "PUT",
-        headers: {
-          ...mcpConfig.headers,
-          "Content-Type": "application/json",
-        },
-        body: JSON.stringify({ claudeMd: content, changeSource: "session_sync" }),
-      });
-    } catch {
-      // Silently fail - don't block shutdown
-    }
+    await postHookProfileUpdate({
+      url: `${getBaseUrl()}/api/agents/${agentId}/profile`,
+      headers: mcpConfig.headers,
+      body: { claudeMd: content, changeSource: "session_sync" },
+      label: "claudeMd",
+    });
   };
 
   // Minimum length for SOUL.md and IDENTITY.md to prevent accidental corruption.
@@ -684,8 +710,6 @@ export async function handleHook(): Promise<void> {
       const content = await soulFile.text();
       if (baselines?.soulMd && contentSha256(content) === baselines.soulMd) {
         // Unchanged during session — skip to preserve Lead's DB edits
-      } else if (content.length > MAX_PROFILE_FILE_LENGTH) {
-        warnProfileFileTooLarge(agentId, "soulMd", content.length, "hook");
       } else if (content.trim()) {
         if (content.length < IDENTITY_FILE_MIN_LENGTH) {
           console.error(
@@ -702,8 +726,6 @@ export async function handleHook(): Promise<void> {
       const content = await identityFile.text();
       if (baselines?.identityMd && contentSha256(content) === baselines.identityMd) {
         // Unchanged during session — skip
-      } else if (content.length > MAX_PROFILE_FILE_LENGTH) {
-        warnProfileFileTooLarge(agentId, "identityMd", content.length, "hook");
       } else if (content.trim()) {
         if (content.length < IDENTITY_FILE_MIN_LENGTH) {
           console.error(
@@ -720,8 +742,6 @@ export async function handleHook(): Promise<void> {
       const content = await toolsMdFile.text();
       if (baselines?.toolsMd && contentSha256(content) === baselines.toolsMd) {
         // Unchanged during session — skip
-      } else if (content.length > MAX_PROFILE_FILE_LENGTH) {
-        warnProfileFileTooLarge(agentId, "toolsMd", content.length, "hook");
       } else if (content.trim()) {
         updates.toolsMd = content;
       }
@@ -741,17 +761,13 @@ export async function handleHook(): Promise<void> {
 
     if (Object.keys(updates).length === 0) return;
 
-    try {
-      await fetch(`${getBaseUrl()}/api/agents/${agentId}/profile`, {
-        method: "PUT",
-        headers: {
-          ...mcpConfig.headers,
-          "Content-Type": "application/json",
-        },
-        body: JSON.stringify({ ...updates, changeSource }),
+    for (const payload of buildIndependentIdentityPayloads(updates, changeSource)) {
+      await postHookProfileUpdate({
+        url: `${getBaseUrl()}/api/agents/${agentId}/profile`,
+        headers: mcpConfig.headers,
+        body: payload.body,
+        label: payload.label,
       });
-    } catch {
-      // Silently fail
     }
   };
 

package/src/http/active-sessions.ts

@@ -12,6 +12,7 @@ import {
   updateActiveSessionProviderSessionId,
 } from "../be/db";
 import { ActiveSessionSchema, AgentTaskSchema } from "../types";
+import { isMultiRuntimeEnabled } from "../utils/multi-runtime";
 import { route } from "./route-def";
 import { jsonError } from "./utils";
 
@@ -47,6 +48,7 @@ const createActiveSession = route({
     inboxMessageId: z.string().optional(),
     taskDescription: z.string().optional(),
     runnerSessionId: z.string().optional(),
+    runtimeInstanceId: z.string().optional(),
   }),
   responses: {
     201: { description: "Session created", schema: z.object({ session: ActiveSessionSchema }) },
@@ -166,6 +168,7 @@ export async function handleActiveSessions(
       inboxMessageId: parsed.body.inboxMessageId,
       taskDescription: parsed.body.taskDescription,
       runnerSessionId: parsed.body.runnerSessionId,
+      runtimeInstanceId: parsed.body.runtimeInstanceId,
     });
     createActiveSession.respond(res, 201, { session });
     return true;
@@ -211,7 +214,14 @@ export async function handleActiveSessions(
     if (!parsed) return true;
     let cleaned = 0;
     if (parsed.body?.agentId) {
-      cleaned = cleanupAgentSessions(parsed.body.agentId);
+      // Multi-runtime: several processes share this agent id, and a booting
+      // worker has no evidence distinguishing a crashed predecessor's session
+      // from a live-but-quiet sibling's — sessions heartbeat on tool activity
+      // only, and during the activation window a live worker's runtime may
+      // have no row at all. Reclamation stays with the heartbeat's
+      // stalled-task classifier (stale session AND stale task), backstopped
+      // by the sweep's stale-session cleanup; boot cleanup deletes nothing.
+      cleaned = isMultiRuntimeEnabled() ? 0 : cleanupAgentSessions(parsed.body.agentId);
     } else {
       cleaned = cleanupStaleSessions(parsed.body?.maxAgeMinutes ?? 30);
     }