@desplega.ai/agent-swarm 1.89.0 → 1.90.0

package/src/heartbeat/heartbeat.ts

@@ -1,5 +1,6 @@
 import {
   assignUnassignedTaskPending,
+  backfillSupersedeTaskResumeTaskId,
   cleanupStaleSessions,
   createTaskExtended,
   deleteActiveSession,
@@ -25,7 +26,7 @@ import {
   updateAgentStatus,
 } from "../be/db";
 import { resolveTemplate } from "../prompts/resolver";
-import { createResumeFollowUp } from "../tasks/worker-follow-up";
+import { createResumeFollowUp, getNextResumeGeneration } from "../tasks/worker-follow-up";
 import type { AgentTask } from "../types";
 import { getExecutorRegistry } from "../workflows";
 import { recoverIncompleteRuns } from "../workflows/recovery";
@@ -60,6 +61,11 @@ const STALE_CLEANUP_THRESHOLD_MINUTES = Number(process.env.HEARTBEAT_STALE_CLEAN
 /** Max pool tasks to auto-assign per sweep */
 const MAX_AUTO_ASSIGN_PER_SWEEP = Number(process.env.HEARTBEAT_MAX_AUTO_ASSIGN) || 5;
 
+/** Max crash-recovery resume generations before failing for lead triage */
+export const MAX_RESUME_GENERATIONS = Number(process.env.HEARTBEAT_MAX_RESUME_GENERATIONS) || 3;
+
+export const RESUME_BUDGET_EXHAUSTED_REASON = "resume_budget_exhausted";
+
 /** Heartbeat checklist interval: how often to check HEARTBEAT.md (default: 30 min) */
 const HEARTBEAT_CHECKLIST_INTERVAL_MS =
   Number(process.env.HEARTBEAT_CHECKLIST_INTERVAL_MS) || 30 * 60 * 1000;
@@ -98,10 +104,17 @@ export interface HeartbeatFindings {
 let heartbeatInterval: ReturnType<typeof setInterval> | null = null;
 let checklistInterval: ReturnType<typeof setInterval> | null = null;
 let isSweeping = false;
+let beforeHeartbeatSupersedeForTests: ((task: AgentTask) => void) | null = null;
 
 /** Tasks auto-failed during the reboot sweep, consumed by boot triage */
 let rebootAffectedTasks: Array<{ original: AgentTask; retryTaskId: string | null }> = [];
 
+export function setBeforeHeartbeatSupersedeForTests(
+  hook: ((task: AgentTask) => void) | null,
+): void {
+  beforeHeartbeatSupersedeForTests = hook;
+}
+
 // ============================================================================
 // Tier 1: Preflight Gate
 // ============================================================================
@@ -300,16 +313,40 @@ function remediateCrashedWorkerTask(
     return;
   }
 
-  // Supersede + resume path.
+  const nextResumeGeneration = getNextResumeGeneration(task);
+  if (nextResumeGeneration > MAX_RESUME_GENERATIONS) {
+    const failed = failTask(task.id, RESUME_BUDGET_EXHAUSTED_REASON);
+    if (failed) {
+      findings.autoFailedTasks.push({
+        taskId: task.id,
+        agentId: task.agentId,
+        reason: RESUME_BUDGET_EXHAUSTED_REASON,
+      });
+      if (opts.cleanupActiveSession) deleteActiveSession(task.id);
+      console.warn(
+        `[Heartbeat] Auto-failed task ${task.id.slice(0, 8)} — ${RESUME_BUDGET_EXHAUSTED_REASON} (${opts.shortLabel})`,
+      );
+      const remaining = getActiveTaskCount(task.agentId);
+      if (remaining === 0) updateAgentStatus(task.agentId, "idle");
+    }
+    return;
+  }
+
+  beforeHeartbeatSupersedeForTests?.(task);
+
   const superseded = supersedeTask(task.id, {
     reason: opts.supersedeReason,
     resumeTaskId: null,
   });
-  if (!superseded) return;
+  if (!superseded) {
+    return;
+  }
 
   const resume = createResumeFollowUp({ parentId: task.id, reason: "crash_recovery" });
 
   if (resume.kind === "created") {
+    backfillSupersedeTaskResumeTaskId(task.id, resume.task.id);
+
     findings.autoResumedTasks.push({
       taskId: task.id,
       resumeTaskId: resume.task.id,
@@ -320,10 +357,20 @@ function remediateCrashedWorkerTask(
       `[Heartbeat] Auto-superseded task ${task.id.slice(0, 8)} — created resume ${resume.task.id.slice(0, 8)} (${opts.shortLabel})`,
     );
   } else {
-    // `workflow-skip` is unreachable here (handled above). `skipped` covers
-    // parent-not-found / lead-not-found edge cases — just log for operators.
-    console.log(
-      `[Heartbeat] Task ${task.id.slice(0, 8)} superseded but no resume created (${
+    const reason =
+      resume.kind === "skipped"
+        ? `resume_creation_skipped_${resume.reason}`
+        : "resume_creation_skipped_workflow";
+    const failed = failTask(task.id, reason);
+    if (failed) {
+      findings.autoFailedTasks.push({
+        taskId: task.id,
+        agentId: task.agentId,
+        reason,
+      });
+    }
+    console.warn(
+      `[Heartbeat] Task ${task.id.slice(0, 8)} failed because no resume was created (${
         resume.kind === "skipped" ? resume.reason : "workflow-skip"
       })`,
     );

package/src/http/tasks.ts

@@ -2,6 +2,7 @@ import type { IncomingMessage, ServerResponse } from "node:http";
 import { ensure } from "@desplega.ai/business-use";
 import { z } from "zod";
 import {
+  backfillSupersedeTaskResumeTaskId,
   cancelTask,
   completeTask,
   failTask,
@@ -905,6 +906,7 @@ export async function handleTasks(
     }
 
     const resumeTaskId = followUp.task.id;
+    backfillSupersedeTaskResumeTaskId(parsed.params.id, resumeTaskId);
 
     ensure({
       id: "task.superseded",

package/src/tasks/worker-follow-up.ts

@@ -22,6 +22,20 @@ export const WORKER_LIVENESS_WINDOW_SECONDS = Number(
   process.env.WORKER_LIVENESS_WINDOW_SECONDS || "30",
 );
 
+export const RESUME_GENERATION_TAG_PREFIX = "resume-generation:";
+
+export function getResumeGeneration(task: Pick<AgentTask, "tags">): number {
+  const tag = task.tags.find((value) => value.startsWith(RESUME_GENERATION_TAG_PREFIX));
+  if (!tag) return 0;
+
+  const parsed = Number(tag.slice(RESUME_GENERATION_TAG_PREFIX.length));
+  return Number.isInteger(parsed) && parsed > 0 ? parsed : 0;
+}
+
+export function getNextResumeGeneration(parent: Pick<AgentTask, "tags">): number {
+  return getResumeGeneration(parent) + 1;
+}
+
 function attachmentPointer(a: TaskAttachment): string {
   switch (a.kind) {
     case "url":
@@ -205,7 +219,11 @@ export function createResumeFollowUp(args: {
   ].join("\n");
 
   const priority = Math.min(100, (parent.priority ?? 50) + 10);
-  const tags = ["auto-resume", `reason:${args.reason}`];
+  const tags = [
+    "auto-resume",
+    `reason:${args.reason}`,
+    `${RESUME_GENERATION_TAG_PREFIX}${getNextResumeGeneration(parent)}`,
+  ];
 
   // Identity-shaped fields (dir, VCS provider/repo/number/url/etc.,
   // outputSchema, slack channel/thread/user, agentmail, mention, contextKey,

package/src/tests/heartbeat-supersede-resume.test.ts

@@ -10,16 +10,29 @@ import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:tes
 import { unlink } from "node:fs/promises";
 import {
   closeDb,
+  completeTask,
   createAgent,
   createTaskExtended,
   getChildTasks,
   getDb,
+  getLogsByTaskId,
   getTaskById,
   initDb,
   insertActiveSession,
   startTask,
 } from "../be/db";
-import { codeLevelTriage } from "../heartbeat/heartbeat";
+import {
+  createTrackerSync,
+  getTrackerSync,
+  getTrackerSyncByExternalId,
+} from "../be/db-queries/tracker";
+import {
+  codeLevelTriage,
+  MAX_RESUME_GENERATIONS,
+  RESUME_BUDGET_EXHAUSTED_REASON,
+  setBeforeHeartbeatSupersedeForTests,
+} from "../heartbeat/heartbeat";
+import { RESUME_GENERATION_TAG_PREFIX } from "../tasks/worker-follow-up";
 
 const TEST_DB_PATH = "./test-heartbeat-supersede-resume.sqlite";
 
@@ -46,6 +59,8 @@ describe("Heartbeat — supersede + resume (DES-523)", () => {
   });
 
   beforeEach(() => {
+    setBeforeHeartbeatSupersedeForTests(null);
+    getDb().run("DELETE FROM tracker_sync");
     getDb().run("DELETE FROM agent_tasks");
     getDb().run("DELETE FROM agents");
     getDb().run("DELETE FROM active_sessions");
@@ -81,7 +96,82 @@ describe("Heartbeat — supersede + resume (DES-523)", () => {
     expect(resume.taskType).toBe("resume");
     expect(resume.tags).toContain("auto-resume");
     expect(resume.tags).toContain("reason:crash_recovery");
+    expect(resume.tags).toContain(`${RESUME_GENERATION_TAG_PREFIX}1`);
     expect(resume.id).toBe(findings.autoResumedTasks[0]!.resumeTaskId);
+
+    const supersedeLog = getLogsByTaskId(parent.id).find(
+      (log) => log.eventType === "task_superseded",
+    );
+    expect(supersedeLog).toBeTruthy();
+    const metadata = JSON.parse(supersedeLog!.metadata ?? "{}") as { resumeTaskId?: string };
+    expect(metadata.resumeTaskId).toBe(resume.id);
+  });
+
+  test("Case A: crash-recovery resume chain stops at the generation cap", async () => {
+    const agent = createAgent({ name: "dead-resume-worker", isLead: false, status: "busy" });
+    const parent = createTaskExtended("Resume at generation cap", {
+      agentId: agent.id,
+      taskType: "resume",
+      tags: [
+        "auto-resume",
+        "reason:crash_recovery",
+        `${RESUME_GENERATION_TAG_PREFIX}${MAX_RESUME_GENERATIONS}`,
+      ],
+    });
+    startTask(parent.id);
+
+    const oldTime = new Date(Date.now() - 10 * 60 * 1000).toISOString();
+    getDb().run("UPDATE agent_tasks SET lastUpdatedAt = ? WHERE id = ?", [oldTime, parent.id]);
+
+    const findings = await codeLevelTriage();
+
+    expect(findings.autoResumedTasks.length).toBe(0);
+    expect(findings.autoFailedTasks.length).toBe(1);
+    expect(findings.autoFailedTasks[0]!.taskId).toBe(parent.id);
+    expect(findings.autoFailedTasks[0]!.reason).toBe(RESUME_BUDGET_EXHAUSTED_REASON);
+
+    const updatedParent = getTaskById(parent.id);
+    expect(updatedParent?.status).toBe("failed");
+    expect(updatedParent?.failureReason).toBe(RESUME_BUDGET_EXHAUSTED_REASON);
+    expect(getChildTasks(parent.id).length).toBe(0);
+  });
+
+  test("Case A: supersede race does not create a resume child or repoint tracker_sync", async () => {
+    const agent = createAgent({ name: "dead-worker-race", isLead: false, status: "busy" });
+    const parent = createTaskExtended("Tracked parent that finishes during heartbeat", {
+      agentId: agent.id,
+    });
+    startTask(parent.id);
+
+    createTrackerSync({
+      provider: "linear",
+      entityType: "task",
+      swarmId: parent.id,
+      externalId: "linear-race-issue",
+      externalIdentifier: "ENG-637",
+      externalUrl: "https://linear.app/test/issue/ENG-637",
+    });
+
+    const oldTime = new Date(Date.now() - 10 * 60 * 1000).toISOString();
+    getDb().run("UPDATE agent_tasks SET lastUpdatedAt = ? WHERE id = ?", [oldTime, parent.id]);
+
+    setBeforeHeartbeatSupersedeForTests((task) => {
+      expect(task.id).toBe(parent.id);
+      completeTask(parent.id, "finished by racing worker");
+    });
+
+    const findings = await codeLevelTriage();
+
+    expect(findings.autoResumedTasks.length).toBe(0);
+    expect(findings.autoFailedTasks.length).toBe(0);
+
+    const updatedParent = getTaskById(parent.id);
+    expect(updatedParent?.status).toBe("completed");
+    expect(getChildTasks(parent.id).length).toBe(0);
+
+    expect(getTrackerSync("linear", "task", parent.id)).not.toBeNull();
+    const byExternal = getTrackerSyncByExternalId("linear", "task", "linear-race-issue");
+    expect(byExternal?.swarmId).toBe(parent.id);
   });
 
   // --------------------------------------------------------------------------