@desplega.ai/agent-swarm 1.86.0 → 1.87.0

package/src/tests/e2b-dispatch.test.ts

@@ -0,0 +1,330 @@
+import { describe, expect, test } from "bun:test";
+import {
+  buildDetachedShell,
+  buildImageTemplate,
+  buildTemplateArgs,
+  deleteTemplate,
+  type E2BSandboxInfo,
+  e2bSdkConnectionOptions,
+  sandboxPortHost,
+  setTemplateVisibility,
+  waitForAgentRegistration,
+} from "../e2b/dispatch";
+import {
+  parseDotenv,
+  parseKeyValue,
+  redactObjectWithEnv,
+  redactWithEnv,
+  resolveSwarmApiKey,
+  selectEnv,
+} from "../e2b/env";
+
+describe("E2B env helpers", () => {
+  test("parses common dotenv forms", () => {
+    expect(
+      parseDotenv(`
+        # ignored
+        export API_KEY=abc123
+        QUOTED="hello\\nworld"
+        SINGLE='literal value'
+        INLINE=value # comment
+        QUOTED_COMMENT="bar" # comment
+        SINGLE_COMMENT='baz' # comment
+        QUOTED_HASH="bar # keep"
+      `),
+    ).toEqual({
+      API_KEY: "abc123",
+      QUOTED: "hello\nworld",
+      SINGLE: "literal value",
+      INLINE: "value",
+      QUOTED_COMMENT: "bar",
+      SINGLE_COMMENT: "baz",
+      QUOTED_HASH: "bar # keep",
+    });
+  });
+
+  test("validates KEY=VALUE inputs", () => {
+    expect(parseKeyValue("FOO=bar=baz", "--secret")).toEqual(["FOO", "bar=baz"]);
+    expect(() => parseKeyValue("bad-key=value", "--secret")).toThrow("invalid env key");
+    expect(() => parseKeyValue("NOVALUE", "--secret")).toThrow("KEY=VALUE");
+  });
+
+  test("resolves swarm API key with env-file precedence before process default", () => {
+    const previousApiKey = process.env.API_KEY;
+    const previousNamespacedKey = process.env.AGENT_SWARM_API_KEY;
+    delete process.env.API_KEY;
+    delete process.env.AGENT_SWARM_API_KEY;
+    try {
+      expect(resolveSwarmApiKey({ API_KEY: "legacy", AGENT_SWARM_API_KEY: "preferred" })).toBe(
+        "preferred",
+      );
+      expect(resolveSwarmApiKey({ API_KEY: "legacy" }, "explicit")).toBe("explicit");
+      expect(() => resolveSwarmApiKey({})).toThrow("Missing swarm API key");
+    } finally {
+      if (previousApiKey === undefined) {
+        delete process.env.API_KEY;
+      } else {
+        process.env.API_KEY = previousApiKey;
+      }
+      if (previousNamespacedKey === undefined) {
+        delete process.env.AGENT_SWARM_API_KEY;
+      } else {
+        process.env.AGENT_SWARM_API_KEY = previousNamespacedKey;
+      }
+    }
+  });
+
+  test("selectEnv and redactWithEnv keep secrets out of logs", () => {
+    const selected = selectEnv(
+      {
+        API_KEY: "super-secret-value",
+        E2B_API_KEY: "controller-secret",
+        PATH: "/bin",
+      },
+      ["API_KEY", "PATH"],
+    );
+    expect(selected).toEqual({ API_KEY: "super-secret-value", PATH: "/bin" });
+    expect(redactWithEnv("token=super-secret-value", selected)).toContain("[REDACTED:API_KEY]");
+  });
+
+  test("redactObjectWithEnv redacts token-like response fields", () => {
+    expect(
+      redactObjectWithEnv(
+        {
+          sandboxID: "sbx123",
+          envdAccessToken: "controller-token-that-should-not-print",
+          nested: { trafficAccessToken: "traffic-token-that-should-not-print" },
+        },
+        {},
+      ),
+    ).toEqual({
+      sandboxID: "sbx123",
+      envdAccessToken: "[REDACTED:envdAccessToken]",
+      nested: { trafficAccessToken: "[REDACTED:trafficAccessToken]" },
+    });
+  });
+});
+
+describe("E2B dispatch helpers", () => {
+  test("computes public port host from sandbox domain shapes", () => {
+    const bareDomain: E2BSandboxInfo = {
+      sandboxID: "sbx123",
+      templateID: "tpl",
+      domain: "e2b.app",
+    };
+    const sandboxDomain: E2BSandboxInfo = {
+      sandboxID: "sbx123",
+      templateID: "tpl",
+      domain: "sbx123.e2b.app",
+    };
+
+    expect(sandboxPortHost(bareDomain, 3013)).toBe("3013-sbx123.e2b.app");
+    expect(sandboxPortHost(sandboxDomain, 3013)).toBe("3013-sbx123.e2b.app");
+  });
+
+  test("computes public port host from configured E2B endpoints when API domain is absent", () => {
+    const missingDomain: E2BSandboxInfo = {
+      sandboxID: "sbx123",
+      templateID: "tpl",
+      domain: null,
+    };
+
+    expect(sandboxPortHost(missingDomain, 3013, { E2B_DOMAIN: "self-hosted.e2b.test" })).toBe(
+      "3013-sbx123.self-hosted.e2b.test",
+    );
+    expect(
+      sandboxPortHost(missingDomain, 3013, {
+        E2B_SANDBOX_URL: "https://sandbox.private.e2b.test",
+      }),
+    ).toBe("3013-sbx123.private.e2b.test");
+    expect(
+      sandboxPortHost(missingDomain, 3013, {
+        E2B_SANDBOX_URL: "https://sandboxes.internal:8443",
+      }),
+    ).toBe("3013-sbx123.sandboxes.internal:8443");
+  });
+
+  test("buildDetachedShell backgrounds command and captures pid without invalid shell chaining", () => {
+    const shell = buildDetachedShell("/api-entrypoint.sh", "/tmp/api.log", "/tmp/api.pid");
+
+    expect(shell).toContain("nohup /api-entrypoint.sh >/tmp/api.log 2>&1 </dev/null & pid=$!");
+    expect(shell).toContain("sleep 2");
+    expect(shell).toContain('kill -0 "$pid"');
+    expect(shell).toContain("cat /tmp/api.log >&2");
+    expect(shell).toContain("pid=$!");
+    expect(shell).not.toContain("&;");
+    expect(shell).not.toContain("& &&");
+  });
+
+  test("E2B SDK connection options preserve loaded controller endpoints", () => {
+    expect(
+      e2bSdkConnectionOptions(
+        "controller-key",
+        {
+          E2B_DOMAIN: "sandbox.example.com",
+          E2B_SANDBOX_URL: "https://sandbox.sandbox.example.com",
+        },
+        "https://api.sandbox.example.com",
+      ),
+    ).toEqual({
+      apiKey: "controller-key",
+      domain: "sandbox.example.com",
+      sandboxUrl: "https://sandbox.sandbox.example.com",
+      apiUrl: "https://api.sandbox.example.com",
+    });
+  });
+
+  test("waitForAgentRegistration checks the worker registration endpoint with bearer auth", async () => {
+    const originalFetch = globalThis.fetch;
+    const calls: Array<{ url: string; authorization: string | null }> = [];
+
+    try {
+      globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+        calls.push({
+          url: String(input),
+          authorization: new Headers(init?.headers).get("authorization"),
+        });
+        return new Response("{}", { status: 200 });
+      }) as typeof fetch;
+
+      await waitForAgentRegistration("https://api.example.com/", "worker/id", "swarm-secret", 10);
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+
+    expect(calls).toEqual([
+      {
+        url: "https://api.example.com/api/agents/worker%2Fid",
+        authorization: "Bearer swarm-secret",
+      },
+    ]);
+  });
+
+  test("buildTemplateArgs uses current Dockerfile template create command", () => {
+    expect(
+      buildTemplateArgs({
+        role: "worker",
+        name: "agent-swarm-worker",
+        dockerfile: "Dockerfile.worker",
+        cwd: "/repo",
+        cpuCount: 4,
+        memoryMb: 8192,
+        noCache: true,
+        e2bEnv: { E2B_API_KEY: "secret" },
+      }),
+    ).toEqual([
+      "template",
+      "create",
+      "-p",
+      "/repo",
+      "-d",
+      "Dockerfile.worker",
+      "-c",
+      "sleep infinity",
+      "--ready-cmd",
+      "sleep 0",
+      "--cpu-count",
+      "4",
+      "--memory-mb",
+      "8192",
+      "--no-cache",
+      "agent-swarm-worker",
+    ]);
+  });
+
+  test("deleteTemplate supports dry-run cleanup", async () => {
+    await expect(
+      deleteTemplate({
+        name: "agent-swarm-worker-e2e",
+        e2bEnv: { E2B_API_KEY: "secret" },
+        dryRun: true,
+      }),
+    ).resolves.toMatchObject({
+      exitCode: 0,
+      stdout: "e2b template delete agent-swarm-worker-e2e -y\n",
+    });
+  });
+
+  test("setTemplateVisibility supports dry-run publish and unpublish", async () => {
+    await expect(
+      setTemplateVisibility({
+        name: "agent-swarm-worker-latest",
+        public: true,
+        e2bEnv: { E2B_API_KEY: "secret" },
+        dryRun: true,
+      }),
+    ).resolves.toMatchObject({
+      exitCode: 0,
+      stdout: 'PATCH /v2/templates/agent-swarm-worker-latest {"public":true}\n',
+    });
+
+    await expect(
+      setTemplateVisibility({
+        name: "agent-swarm-worker-latest",
+        public: false,
+        e2bEnv: { E2B_API_KEY: "secret" },
+        dryRun: true,
+      }),
+    ).resolves.toMatchObject({
+      exitCode: 0,
+      stdout: 'PATCH /v2/templates/agent-swarm-worker-latest {"public":false}\n',
+    });
+  });
+
+  test("setTemplateVisibility updates templates through the E2B API key path", async () => {
+    const originalFetch = globalThis.fetch;
+    const calls: Array<{ url: string; init?: RequestInit }> = [];
+    globalThis.fetch = async (url: string | URL | Request, init?: RequestInit) => {
+      calls.push({ url: String(url), init });
+      return new Response(JSON.stringify({ names: ["workspace/agent-swarm-worker-latest"] }), {
+        status: 200,
+        headers: { "Content-Type": "application/json" },
+      });
+    };
+
+    try {
+      const result = await setTemplateVisibility({
+        name: "agent swarm/worker",
+        public: true,
+        e2bEnv: {
+          E2B_API_KEY: "controller-secret",
+          E2B_API_URL: "https://api.e2b.example",
+        },
+      });
+
+      expect(calls).toHaveLength(1);
+      expect(calls[0]?.url).toBe("https://api.e2b.example/v2/templates/agent%20swarm%2Fworker");
+      expect(calls[0]?.init?.method).toBe("PATCH");
+      expect(calls[0]?.init?.headers).toMatchObject({
+        "Content-Type": "application/json",
+        "X-API-Key": "controller-secret",
+      });
+      expect(calls[0]?.init?.body).toBe(JSON.stringify({ public: true }));
+      expect(result.stdout).toBe(
+        "Set E2B template agent swarm/worker visibility to public (workspace/agent-swarm-worker-latest)\n",
+      );
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+
+  test("buildImageTemplate dry-run uses the Dockerless SDK path", async () => {
+    await expect(
+      buildImageTemplate({
+        role: "api",
+        name: "agent-swarm-api",
+        image: "ghcr.io/desplega-ai/agent-swarm:latest",
+        cpuCount: 2,
+        memoryMb: 2048,
+        noCache: false,
+        e2bEnv: { E2B_API_KEY: "secret" },
+        dryRun: true,
+      }),
+    ).resolves.toMatchObject({
+      exitCode: 0,
+      stdout: expect.stringContaining(
+        "e2b-sdk template build --from-image ghcr.io/desplega-ai/agent-swarm:latest",
+      ),
+    });
+  });
+});

package/src/tests/heartbeat-supersede-resume.test.ts

@@ -0,0 +1,285 @@
+/**
+ * DES-523: heartbeat sweep should auto-supersede + resume crashed-worker tasks.
+ *
+ * Mirrors the test-setup pattern from `heartbeat.test.ts` (own sqlite file,
+ * full DB reset between tests). Each test exercises one branch of
+ * `remediateCrashedWorkerTask` inside `detectAndRemediateStalledTasks`.
+ */
+
+import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { unlink } from "node:fs/promises";
+import {
+  closeDb,
+  createAgent,
+  createTaskExtended,
+  getChildTasks,
+  getDb,
+  getTaskById,
+  initDb,
+  insertActiveSession,
+  startTask,
+} from "../be/db";
+import { codeLevelTriage } from "../heartbeat/heartbeat";
+
+const TEST_DB_PATH = "./test-heartbeat-supersede-resume.sqlite";
+
+describe("Heartbeat — supersede + resume (DES-523)", () => {
+  beforeAll(async () => {
+    try {
+      await unlink(TEST_DB_PATH);
+    } catch {
+      // File doesn't exist
+    }
+    closeDb();
+    initDb(TEST_DB_PATH);
+  });
+
+  afterAll(async () => {
+    closeDb();
+    for (const path of [TEST_DB_PATH, `${TEST_DB_PATH}-wal`, `${TEST_DB_PATH}-shm`]) {
+      try {
+        await unlink(path);
+      } catch {
+        // Files may not exist
+      }
+    }
+  });
+
+  beforeEach(() => {
+    getDb().run("DELETE FROM agent_tasks");
+    getDb().run("DELETE FROM agents");
+    getDb().run("DELETE FROM active_sessions");
+  });
+
+  // --------------------------------------------------------------------------
+  // Case A — no active session
+  // --------------------------------------------------------------------------
+
+  test("Case A: regular task with no active session is auto-superseded and resumed", async () => {
+    const agent = createAgent({ name: "dead-worker", isLead: false, status: "busy" });
+    const parent = createTaskExtended("Long-running parent work", { agentId: agent.id });
+    startTask(parent.id);
+
+    // 10 min stale — past the 5 min no-session threshold.
+    const oldTime = new Date(Date.now() - 10 * 60 * 1000).toISOString();
+    getDb().run("UPDATE agent_tasks SET lastUpdatedAt = ? WHERE id = ?", [oldTime, parent.id]);
+
+    const findings = await codeLevelTriage();
+
+    expect(findings.autoResumedTasks.length).toBe(1);
+    expect(findings.autoResumedTasks[0]!.taskId).toBe(parent.id);
+    expect(findings.autoFailedTasks.length).toBe(0);
+
+    // Parent transitioned to `superseded` (NOT `failed`).
+    const updatedParent = getTaskById(parent.id);
+    expect(updatedParent?.status).toBe("superseded");
+
+    // A resume follow-up child exists.
+    const children = getChildTasks(parent.id);
+    expect(children.length).toBe(1);
+    const resume = children[0]!;
+    expect(resume.taskType).toBe("resume");
+    expect(resume.tags).toContain("auto-resume");
+    expect(resume.tags).toContain("reason:crash_recovery");
+    expect(resume.id).toBe(findings.autoResumedTasks[0]!.resumeTaskId);
+  });
+
+  // --------------------------------------------------------------------------
+  // Case A — system task: must fall back to failTask, never resume
+  // --------------------------------------------------------------------------
+
+  test("Case A: system task (taskType=heartbeat) is failed, not resumed", async () => {
+    const lead = createAgent({ name: "lead", isLead: true, status: "busy" });
+    const parent = createTaskExtended("Periodic heartbeat checklist", {
+      agentId: lead.id,
+      taskType: "heartbeat",
+    });
+    startTask(parent.id);
+
+    const oldTime = new Date(Date.now() - 10 * 60 * 1000).toISOString();
+    getDb().run("UPDATE agent_tasks SET lastUpdatedAt = ? WHERE id = ?", [oldTime, parent.id]);
+
+    const findings = await codeLevelTriage();
+
+    expect(findings.autoFailedTasks.length).toBe(1);
+    expect(findings.autoFailedTasks[0]!.taskId).toBe(parent.id);
+    expect(findings.autoResumedTasks.length).toBe(0);
+
+    const updatedParent = getTaskById(parent.id);
+    expect(updatedParent?.status).toBe("failed");
+
+    // No resume child was created.
+    const children = getChildTasks(parent.id);
+    expect(children.length).toBe(0);
+  });
+
+  // --------------------------------------------------------------------------
+  // Case A — idempotency: a non-terminal child already exists → fail, no 2nd resume
+  // --------------------------------------------------------------------------
+
+  test("Case A: idempotency — non-terminal child already exists, parent is failed (no 2nd resume)", async () => {
+    const agent = createAgent({ name: "dead-worker", isLead: false, status: "busy" });
+    const parent = createTaskExtended("Parent with existing child", { agentId: agent.id });
+    startTask(parent.id);
+
+    // Pre-insert a non-terminal child. `createTaskExtended` defaults to
+    // `unassigned` without an agentId — we assign the same agent so the child
+    // lands in `pending`, mirroring what a prior sweep would have produced.
+    const preexisting = createTaskExtended("Existing pending child", {
+      parentTaskId: parent.id,
+      agentId: agent.id,
+      tags: ["auto-resume", "reason:crash_recovery"],
+      taskType: "resume",
+    });
+    expect(preexisting.status).toBe("pending");
+
+    const oldTime = new Date(Date.now() - 10 * 60 * 1000).toISOString();
+    getDb().run("UPDATE agent_tasks SET lastUpdatedAt = ? WHERE id = ?", [oldTime, parent.id]);
+
+    const findings = await codeLevelTriage();
+
+    // Idempotency path: parent gets the legacy `failTask` treatment.
+    expect(findings.autoFailedTasks.length).toBe(1);
+    expect(findings.autoFailedTasks[0]!.taskId).toBe(parent.id);
+    expect(findings.autoResumedTasks.length).toBe(0);
+
+    const updatedParent = getTaskById(parent.id);
+    expect(updatedParent?.status).toBe("failed");
+
+    // Only the original pre-existing child remains — no second resume was created.
+    const children = getChildTasks(parent.id);
+    expect(children.length).toBe(1);
+    expect(children[0]!.id).toBe(preexisting.id);
+  });
+
+  // --------------------------------------------------------------------------
+  // Case A — delegation children must NOT block the resume path
+  // --------------------------------------------------------------------------
+
+  test("Case A: ordinary delegation child does NOT block resume (only taskType=resume children count)", async () => {
+    // PR #594 review: `send-task` auto-defaults `parentTaskId` to the
+    // caller's current task. So a crashed worker that had delegated subtasks
+    // has non-terminal children that are NOT resume tasks. The idempotency
+    // guard must only count taskType=resume children — otherwise the parent
+    // is silently failed and the original work is dropped.
+    const agent = createAgent({ name: "dead-delegator", isLead: false, status: "busy" });
+    const otherAgent = createAgent({ name: "subtask-worker", isLead: false, status: "busy" });
+    const parent = createTaskExtended("Parent that delegated", { agentId: agent.id });
+    startTask(parent.id);
+
+    // A delegated subtask — `taskType` is NOT "resume". `send-task` auto-sets
+    // parentTaskId to the delegator's current task, so this models reality.
+    const delegated = createTaskExtended("Delegated subtask", {
+      parentTaskId: parent.id,
+      agentId: otherAgent.id,
+      taskType: "delegation",
+    });
+    expect(delegated.status).toBe("pending");
+
+    const oldTime = new Date(Date.now() - 10 * 60 * 1000).toISOString();
+    getDb().run("UPDATE agent_tasks SET lastUpdatedAt = ? WHERE id = ?", [oldTime, parent.id]);
+
+    const findings = await codeLevelTriage();
+
+    // Resume path should fire — the delegated child does not count.
+    expect(findings.autoResumedTasks.length).toBe(1);
+    expect(findings.autoResumedTasks[0]!.taskId).toBe(parent.id);
+    expect(findings.autoFailedTasks.length).toBe(0);
+
+    const updatedParent = getTaskById(parent.id);
+    expect(updatedParent?.status).toBe("superseded");
+
+    // Children now: the original delegation + the new resume.
+    const children = getChildTasks(parent.id);
+    expect(children.length).toBe(2);
+    const resumeChild = children.find((c) => c.taskType === "resume");
+    expect(resumeChild).not.toBeUndefined();
+  });
+
+  // --------------------------------------------------------------------------
+  // Case B — stale session heartbeat
+  // --------------------------------------------------------------------------
+
+  test("Case B: stale session heartbeat is auto-superseded and resumed", async () => {
+    const agent = createAgent({ name: "crashed-worker", isLead: false, status: "busy" });
+    const parent = createTaskExtended("Crashed worker's task", { agentId: agent.id });
+    startTask(parent.id);
+
+    insertActiveSession({
+      agentId: agent.id,
+      taskId: parent.id,
+      triggerType: "task_assigned",
+    });
+
+    // Make both task and session heartbeat 20 min stale — past the 15 min threshold.
+    const oldTime = new Date(Date.now() - 20 * 60 * 1000).toISOString();
+    getDb().run("UPDATE agent_tasks SET lastUpdatedAt = ? WHERE id = ?", [oldTime, parent.id]);
+    getDb().run("UPDATE active_sessions SET lastHeartbeatAt = ? WHERE taskId = ?", [
+      oldTime,
+      parent.id,
+    ]);
+
+    const findings = await codeLevelTriage();
+
+    expect(findings.autoResumedTasks.length).toBe(1);
+    expect(findings.autoResumedTasks[0]!.taskId).toBe(parent.id);
+    expect(findings.autoFailedTasks.length).toBe(0);
+
+    const updatedParent = getTaskById(parent.id);
+    expect(updatedParent?.status).toBe("superseded");
+
+    const children = getChildTasks(parent.id);
+    expect(children.length).toBe(1);
+    const resume = children[0]!;
+    expect(resume.taskType).toBe("resume");
+    expect(resume.tags).toContain("auto-resume");
+    expect(resume.tags).toContain("reason:crash_recovery");
+
+    // Orphan active_session row was cleaned up.
+    const remainingSessions = getDb()
+      .query("SELECT COUNT(*) as count FROM active_sessions WHERE taskId = ?")
+      .get(parent.id) as { count: number };
+    expect(remainingSessions.count).toBe(0);
+  });
+
+  // --------------------------------------------------------------------------
+  // Workflow carve-out — workflowRunStepId set → workflow-skip → failTask path
+  // --------------------------------------------------------------------------
+
+  test("Workflow-step parent: failed with workflow reason, no supersede or resume", async () => {
+    const agent = createAgent({ name: "workflow-worker", isLead: false, status: "busy" });
+    const parent = createTaskExtended("Workflow-step task", { agentId: agent.id });
+    startTask(parent.id);
+
+    // Backfill workflowRunStepId — createTaskExtended doesn't accept it.
+    // FKs are toggled off because this test only exercises the heartbeat path,
+    // not the workflow engine itself (same pattern as task-supersede-resume.test.ts).
+    const stepId = crypto.randomUUID();
+    getDb().exec("PRAGMA foreign_keys = OFF");
+    try {
+      getDb().run("UPDATE agent_tasks SET workflowRunStepId = ? WHERE id = ?", [stepId, parent.id]);
+    } finally {
+      getDb().exec("PRAGMA foreign_keys = ON");
+    }
+
+    const oldTime = new Date(Date.now() - 10 * 60 * 1000).toISOString();
+    getDb().run("UPDATE agent_tasks SET lastUpdatedAt = ? WHERE id = ?", [oldTime, parent.id]);
+
+    const findings = await codeLevelTriage();
+
+    // Workflow carve-out: the engine owns retry. Skip the supersede flow and
+    // mark the dedicated workflow-task failure so the engine can react.
+    expect(findings.autoResumedTasks.length).toBe(0);
+    expect(findings.autoFailedTasks.length).toBe(1);
+    expect(findings.autoFailedTasks[0]!.taskId).toBe(parent.id);
+    expect(findings.autoFailedTasks[0]!.reason).toBe("superseded_workflow_task");
+
+    // No resume child was created.
+    const children = getChildTasks(parent.id);
+    expect(children.length).toBe(0);
+
+    const updatedParent = getTaskById(parent.id);
+    expect(updatedParent?.status).toBe("failed");
+    expect(updatedParent?.failureReason).toBe("superseded_workflow_task");
+  });
+});