@desplega.ai/agent-swarm 2.5.3 → 2.6.0

package/src/tests/provider-key-probes.test.ts

@@ -0,0 +1,181 @@
+import { afterEach, describe, expect, mock, test } from "bun:test";
+import { checkProviderPoolKey } from "../utils/provider-key-probes";
+
+describe("API pool key probes", () => {
+  const realFetch = globalThis.fetch;
+  const originalNodeEnv = process.env.NODE_ENV;
+  const originalPrivateUrls = process.env.ALLOW_PRIVATE_NETWORK_URLS;
+
+  afterEach(() => {
+    globalThis.fetch = realFetch;
+    if (originalNodeEnv === undefined) delete process.env.NODE_ENV;
+    else process.env.NODE_ENV = originalNodeEnv;
+    if (originalPrivateUrls === undefined) delete process.env.ALLOW_PRIVATE_NETWORK_URLS;
+    else process.env.ALLOW_PRIVATE_NETWORK_URLS = originalPrivateUrls;
+  });
+
+  test("uses the supplied OpenRouter URL and forbids redirects", async () => {
+    const calls: Array<{ url: string; init?: RequestInit }> = [];
+    globalThis.fetch = mock(async (url: string | URL | Request, init?: RequestInit) => {
+      calls.push({ url: String(url), init });
+      return new Response("{}", { status: calls.length === 1 ? 404 : 200 });
+    }) as typeof fetch;
+    expect(
+      await checkProviderPoolKey("OPENROUTER_API_KEY", "fake-abcde", {
+        OPENROUTER_BASE_URL: "https://gateway.example/v1/",
+      }),
+    ).toMatchObject({ ok: true });
+    expect(calls.map((call) => call.url)).toEqual([
+      "https://gateway.example/v1/key",
+      "https://gateway.example/v1/models",
+    ]);
+    expect(calls.every((call) => call.init?.redirect === "manual")).toBe(true);
+  });
+
+  test("rejects a private registry URL before sending credentials", async () => {
+    process.env.NODE_ENV = "production";
+    delete process.env.ALLOW_PRIVATE_NETWORK_URLS;
+    const fetch = mock(async () => new Response("{}"));
+    globalThis.fetch = fetch as unknown as typeof globalThis.fetch;
+    const result = await checkProviderPoolKey("NAN_API_KEY", "fake-abcde", {
+      NAN_BASE_URL: "https://127.0.0.1/v1",
+    });
+    expect(result.ok).toBe(false);
+    expect(result.httpStatus).toBeUndefined();
+    expect(fetch).not.toHaveBeenCalled();
+  });
+
+  test("supports registry aliases and configured URLs", async () => {
+    const fetch = mock(async (url: string | URL | Request, init?: RequestInit) => {
+      expect(String(url)).toBe("https://gateway.example/v1/models");
+      expect(init?.headers).toEqual({ Authorization: "Bearer fake-abcde" });
+      expect(init?.redirect).toBe("manual");
+      return new Response("Unauthorized", { status: 401 });
+    });
+    globalThis.fetch = fetch as typeof globalThis.fetch;
+    expect(
+      await checkProviderPoolKey("HELMCODE_API_NAN", "fake-abcde", {
+        NAN_BASE_URL: "https://gateway.example/v1",
+      }),
+    ).toMatchObject({ ok: false, httpStatus: 401 });
+  });
+
+  test.each([
+    [200, "{}", true, false],
+    [403, '{"error":{"type":"permission_error"}}', true, false],
+    [403, '{"error":{"type":"other"}}', false, false],
+    [401, "Unauthorized", false, false],
+    [429, "Throttled", true, true],
+    [503, "Unavailable", true, true],
+  ])("preserves Claude OAuth semantics for HTTP %i", async (status, body, ok, skipped) => {
+    globalThis.fetch = mock(async (url: string | URL | Request, init?: RequestInit) => {
+      expect(String(url)).toBe("https://api.anthropic.com/api/oauth/profile");
+      expect(init?.redirect).toBe("manual");
+      expect(init?.signal).toBeInstanceOf(AbortSignal);
+      return new Response(body, { status });
+    }) as typeof fetch;
+    const result = await checkProviderPoolKey("CLAUDE_CODE_OAUTH_TOKEN", "fake-abcde", {});
+    expect(result.ok).toBe(ok);
+    expect(result.skipped === true).toBe(skipped);
+    if (!ok) expect(result.httpStatus).toBe(status);
+  });
+
+  test("checks an Anthropic key at its gateway, keeping the URL path prefix", async () => {
+    const calls: Array<{ url: string; init?: RequestInit }> = [];
+    globalThis.fetch = mock(async (url: string | URL | Request, init?: RequestInit) => {
+      calls.push({ url: String(url), init });
+      return new Response("Unauthorized", { status: 401 });
+    }) as unknown as typeof fetch;
+    const result = await checkProviderPoolKey("ANTHROPIC_API_KEY", "fake-abcde", {
+      ANTHROPIC_BASE_URL: "https://gateway.example/api",
+    });
+    expect(result).toMatchObject({ ok: false, httpStatus: 401 });
+    expect(calls.map((call) => call.url)).toEqual(["https://gateway.example/api/v1/models"]);
+    expect(new Headers(calls[0]?.init?.headers).get("x-api-key")).toBe("fake-abcde");
+    expect(calls[0]?.init?.redirect).toBe("manual");
+
+    globalThis.fetch = mock(async () => new Response("{}")) as unknown as typeof fetch;
+    expect(
+      await checkProviderPoolKey("ANTHROPIC_API_KEY", "fake-abcde", {
+        ANTHROPIC_BASE_URL: "https://gateway.example/api",
+      }),
+    ).toMatchObject({ ok: true });
+  });
+
+  test("a gateway that lists no models leaves the Anthropic key unchecked", async () => {
+    globalThis.fetch = mock(
+      async () => new Response("Not found", { status: 404 }),
+    ) as unknown as typeof fetch;
+    expect(
+      await checkProviderPoolKey("ANTHROPIC_API_KEY", "fake-abcde", {
+        ANTHROPIC_BASE_URL: "https://gateway.example",
+      }),
+    ).toMatchObject({ ok: true, skipped: true });
+  });
+
+  test("an Anthropic key stays on api.anthropic.com when the gateway uses ANTHROPIC_AUTH_TOKEN", async () => {
+    const urls: string[] = [];
+    globalThis.fetch = mock(async (url: string | URL | Request) => {
+      urls.push(String(url));
+      return new Response("{}");
+    }) as unknown as typeof fetch;
+    expect(
+      await checkProviderPoolKey("ANTHROPIC_API_KEY", "fake-abcde", {
+        ANTHROPIC_BASE_URL: "https://gateway.example",
+        ANTHROPIC_AUTH_TOKEN: "gateway-token",
+      }),
+    ).toMatchObject({ ok: true });
+    expect(urls).toEqual(["https://api.anthropic.com/v1/models"]);
+  });
+
+  test("a private Anthropic gateway gets no credentials and no rejection", async () => {
+    process.env.NODE_ENV = "production";
+    delete process.env.ALLOW_PRIVATE_NETWORK_URLS;
+    const fetch = mock(async () => new Response("{}"));
+    globalThis.fetch = fetch as unknown as typeof globalThis.fetch;
+    const result = await checkProviderPoolKey("ANTHROPIC_API_KEY", "fake-abcde", {
+      ANTHROPIC_BASE_URL: "http://127.0.0.1:4000",
+    });
+    expect(result.ok).toBe(false);
+    expect(result.httpStatus).toBeUndefined();
+    expect(fetch).not.toHaveBeenCalled();
+  });
+
+  test("skips keys without an API-safe probe", async () => {
+    const fetch = mock(async () => new Response("{}"));
+    globalThis.fetch = fetch as unknown as typeof globalThis.fetch;
+    for (const keyType of ["DEVIN_API_KEY", "AMP_API_KEY", "CURSOR_API_KEY"]) {
+      expect(await checkProviderPoolKey(keyType, "fake-abcde", {})).toMatchObject({
+        ok: true,
+        skipped: true,
+      });
+    }
+    expect(fetch).not.toHaveBeenCalled();
+  });
+
+  test("reports network errors without exposing the submitted key", async () => {
+    globalThis.fetch = mock(async () => {
+      throw new Error("Rejected fake-abcde");
+    }) as unknown as typeof fetch;
+    const result = await checkProviderPoolKey("OPENAI_API_KEY", "fake-abcde", {});
+    expect(result.ok).toBe(false);
+    expect(result.error).not.toContain("fake-abcde");
+  });
+
+  test.each([
+    "ANTHROPIC_API_KEY",
+    "OPENAI_API_KEY",
+    "OPENROUTER_API_KEY",
+    "NAN_API_KEY",
+    "CLAUDE_CODE_OAUTH_TOKEN",
+  ])("redacts a long echoed %s before truncating the provider error", async (keyType) => {
+    const apiKey = "Z".repeat(240);
+    globalThis.fetch = mock(
+      async () => new Response(`Rejected ${apiKey}`, { status: 403 }),
+    ) as unknown as typeof fetch;
+    const result = await checkProviderPoolKey(keyType, apiKey, {});
+    expect(result).toMatchObject({ ok: false, httpStatus: 403 });
+    expect(result.error).toContain("[REDACTED]");
+    expect(result.error).not.toContain(apiKey.slice(0, 40));
+  });
+});

package/src/tests/rbac-e2e-helpers.test.ts

@@ -1,9 +1,23 @@
 import { expect, test } from "bun:test";
 import { closeSync, openSync } from "node:fs";
 import { mkdtemp, rm } from "node:fs/promises";
+import { createServer } from "node:net";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { type SwarmServer, waitForListen } from "./rbac-e2e-helpers";
+import {
+  makeScratchDir,
+  removeScratchDir,
+  type SwarmServer,
+  spawnSwarmServer,
+  waitForListen,
+} from "./rbac-e2e-helpers";
+import {
+  getFreePort,
+  listenOnFreePort,
+  PortInUseError,
+  SERVER_BOOT_HOOK_TIMEOUT_MS,
+  startOnFreePort,
+} from "./test-net";
 
 test("waitForListen includes a bounded server log when the process exits early", async () => {
   const dir = await mkdtemp(join(tmpdir(), "rbac-e2e-helper-"));
@@ -51,3 +65,76 @@ test("waitForListen includes a bounded server log when the process exits early",
     await rm(dir, { recursive: true, force: true });
   }
 });
+
+test("startOnFreePort retries only PortInUseError, up to its attempt bound", async () => {
+  const ports = [41001, 41002, 41003, 41004];
+  const getPort = async () => ports.shift() ?? 0;
+  const tried: number[] = [];
+
+  const port = await startOnFreePort(
+    async (p) => {
+      tried.push(p);
+      if (p === 41001) throw new PortInUseError("taken");
+      return p;
+    },
+    { getPort },
+  );
+  expect(port).toBe(41002);
+  expect(tried).toEqual([41001, 41002]);
+
+  const other = await startOnFreePort(
+    async () => {
+      throw new Error("boot crashed");
+    },
+    { getPort },
+  ).catch((error: unknown) => error);
+  expect((other as Error).message).toBe("boot crashed");
+  expect(ports).toEqual([41004]);
+
+  let calls = 0;
+  const exhausted = await startOnFreePort(
+    async () => {
+      calls++;
+      throw new PortInUseError(`taken ${calls}`);
+    },
+    { getPort: async () => 0 },
+  ).catch((error: unknown) => error);
+  expect(exhausted).toBeInstanceOf(PortInUseError);
+  expect((exhausted as Error).message).toBe("taken 3");
+  expect(calls).toBe(3);
+});
+
+test(
+  "spawnSwarmServer starts again on a new port when its first port is taken",
+  async () => {
+    const dir = await makeScratchDir();
+    const holder = createServer();
+    const heldPort = await listenOnFreePort(holder);
+    let server: SwarmServer | undefined;
+    try {
+      const offered: number[] = [];
+      server = await spawnSwarmServer({
+        dbPath: join(dir, "test.sqlite"),
+        logPath: join(dir, "server.log"),
+        getPort: async () => {
+          const port = offered.length === 0 ? heldPort : await getFreePort();
+          offered.push(port);
+          return port;
+        },
+      });
+
+      expect(offered).toHaveLength(2);
+      expect(offered[0]).toBe(heldPort);
+      expect(server.port).toBe(offered[1]);
+      expect(server.port).not.toBe(heldPort);
+      expect((await fetch(`${server.base}/docs`)).ok).toBe(true);
+      const log = await Bun.file(server.logPath).text();
+      expect(log).toContain(`Is port ${heldPort} in use?`);
+    } finally {
+      await server?.stop();
+      holder.close();
+      await removeScratchDir(dir);
+    }
+  },
+  SERVER_BOOT_HOOK_TIMEOUT_MS * 2,
+);

package/src/tests/rbac-e2e-helpers.ts

@@ -14,7 +14,7 @@ import { openSync } from "node:fs";
 import { mkdtemp, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { dirname, join } from "node:path";
-import { getFreePort } from "./test-net";
+import { getFreePort, PORT_IN_USE_LOG, PortInUseError, startOnFreePort } from "./test-net";
 
 // Re-exported for the suites that imported it from here before test-net.ts existed.
 export { getFreePort };
@@ -50,6 +50,9 @@ export async function removeScratchDir(dir: string): Promise<void> {
  * Spawn `bun src/http.ts` against `dbPath` on a free port. Integrations are
  * disabled; both API-key env vars are pinned so a developer's shell env can't
  * leak in (AGENT_SWARM_API_KEY takes precedence in getApiKey()).
+ *
+ * With waitForListen, a child that exits because another process took its
+ * port during boot is started again on a new port (startOnFreePort).
  */
 export async function spawnSwarmServer(opts: {
   dbPath: string;
@@ -57,9 +60,38 @@ export async function spawnSwarmServer(opts: {
   env?: Record<string, string>;
   /** Poll /docs until the server accepts requests (default true). */
   waitForListen?: boolean;
+  /** Port source for each attempt (default getFreePort). Tests use it to force a taken port. */
+  getPort?: () => Promise<number>;
 }): Promise<SwarmServer> {
-  const port = await getFreePort();
   const logFd = openSync(opts.logPath, "a");
+  return startOnFreePort(
+    async (port) => {
+      const log = Bun.file(opts.logPath);
+      const logStart = (await log.exists()) ? log.size : 0;
+      const server = launchSwarmServer(opts, port, logFd);
+      if (opts.waitForListen === false) return server;
+      try {
+        await waitForListen(server);
+      } catch (error) {
+        if (server.proc.exitCode !== null) {
+          const attemptLog = await Bun.file(opts.logPath).slice(logStart).text();
+          if (PORT_IN_USE_LOG.test(attemptLog)) {
+            throw new PortInUseError((error as Error).message, { cause: error });
+          }
+        }
+        throw error;
+      }
+      return server;
+    },
+    { getPort: opts.getPort },
+  );
+}
+
+function launchSwarmServer(
+  opts: { dbPath: string; logPath: string; env?: Record<string, string> },
+  port: number,
+  logFd: number,
+): SwarmServer {
   const proc = Bun.spawn(["bun", "src/http.ts"], {
     cwd: REPO_ROOT,
     stdout: logFd,
@@ -93,12 +125,10 @@ export async function spawnSwarmServer(opts: {
       ...opts.env,
     },
   });
-  const base = `http://localhost:${port}`;
-
-  const server: SwarmServer = {
+  return {
     proc,
     port,
-    base,
+    base: `http://localhost:${port}`,
     dbPath: opts.dbPath,
     logPath: opts.logPath,
     async stop() {
@@ -107,11 +137,6 @@ export async function spawnSwarmServer(opts: {
       return proc.exitCode;
     },
   };
-
-  if (opts.waitForListen !== false) {
-    await waitForListen(server);
-  }
-  return server;
 }
 
 export async function waitForListen(server: SwarmServer, deadlineMs = 90_000): Promise<void> {

package/src/tests/runner-push-delivery.test.ts

@@ -139,6 +139,31 @@ async function nudge(kind: WorkerNudge["kind"], taskId?: string) {
   expect((await waiting)?.kind).toBe(kind);
 }
 
+test("shutdown stops an open poll window before it can claim a resume", async () => {
+  let shuttingDown = false;
+  // The second poll would claim the resume created during shutdown.
+  pollTriggerAfter = 2;
+  const sleep = spyOn(Bun, "sleep").mockImplementation(async () => {
+    expect(requests.filter((r) => r === "GET /api/poll")).toHaveLength(1);
+    shuttingDown = true;
+  });
+  try {
+    const trigger = await pollForTrigger({
+      apiUrl,
+      apiKey,
+      agentId,
+      pollInterval: 2,
+      pollTimeout: 5000,
+      shouldStop: () => shuttingDown,
+    });
+    expect(shuttingDown).toBe(true);
+    expect(trigger).toBeNull();
+    expect(requests.filter((r) => r === "GET /api/poll")).toHaveLength(1);
+  } finally {
+    sleep.mockRestore();
+  }
+});
+
 async function poll(now: number, mode: DeliveryMode = "push") {
   const checks = delivery.checks(mode, now);
   const schedule = delivery.schedule({

package/src/tests/runtime-controller-liveness.test.ts

@@ -0,0 +1,133 @@
+import { afterEach, expect, spyOn, test } from "bun:test";
+import { createHash } from "node:crypto";
+import { createServer as createHttpServer } from "node:http";
+import { createServer as createNetServer, type Socket } from "node:net";
+import { WebSocketServer } from "ws";
+import { startControllerClient } from "../commands/controller";
+import { CONTROLLER_PROTOCOL_VERSION } from "../runtime-providers/controller-protocol";
+import type { RuntimeProvider } from "../runtime-providers/types";
+import { listenOnFreePort } from "./test-net";
+
+const LIVENESS_MS = 300;
+const WELCOME = JSON.stringify({
+  type: "welcome",
+  controllerId: "controller-1",
+  version: CONTROLLER_PROTOCOL_VERSION,
+});
+
+const provider = {
+  kind: "docker",
+  traits: { canProvision: true, ephemeralDisk: true, supportsPause: false },
+  checkConfig: () => ({ ready: true, missing: [] }),
+} as unknown as RuntimeProvider;
+
+const cleanups: Array<() => void> = [];
+afterEach(() => {
+  for (const cleanup of cleanups.splice(0).reverse()) cleanup();
+});
+
+// A raw TCP WebSocket peer. Under Bun, `ws` maps to a native shim that always answers
+// pings, so it cannot model a dead link. This peer completes the handshake, sends
+// welcome, and then never answers a frame: what a cut network looks like to the client.
+async function startRawServer(options: { pingEveryMs?: number } = {}) {
+  let connections = 0;
+  const sockets: Socket[] = [];
+  const server = createNetServer((socket) => {
+    sockets.push(socket);
+    socket.once("data", (data) => {
+      const key = /sec-websocket-key: *(\S+)/i.exec(data.toString())?.[1];
+      const accept = createHash("sha1")
+        .update(`${key}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`)
+        .digest("base64");
+      socket.write(
+        `HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ${accept}\r\n\r\n`,
+      );
+      const payload = Buffer.from(WELCOME);
+      socket.write(Buffer.concat([Buffer.from([0x81, payload.length]), payload]));
+      connections++;
+      if (options.pingEveryMs) {
+        const timer = setInterval(
+          () => socket.write(Buffer.from([0x89, 0x00])),
+          options.pingEveryMs,
+        );
+        socket.once("close", () => clearInterval(timer));
+      }
+    });
+    socket.on("error", () => {});
+  });
+  const port = await listenOnFreePort(server);
+  cleanups.push(() => {
+    for (const socket of sockets) socket.destroy();
+    server.close();
+  });
+  return { apiUrl: `http://127.0.0.1:${port}`, connections: () => connections };
+}
+
+function startClient(apiUrl: string) {
+  const logSpy = spyOn(console, "log").mockImplementation(() => {});
+  const client = startControllerClient({
+    apiUrl,
+    token: "aseph_test-token",
+    provider,
+    target: {},
+    livenessTimeoutMs: LIVENESS_MS,
+  });
+  cleanups.push(() => {
+    client.stop();
+    logSpy.mockRestore();
+  });
+  return { logs: () => logSpy.mock.calls.flat().join("\n") };
+}
+
+async function until(check: () => boolean, timeoutMs: number) {
+  const deadline = Date.now() + timeoutMs;
+  while (!check()) {
+    if (Date.now() > deadline) throw new Error("Timed out waiting for condition");
+    await Bun.sleep(20);
+  }
+}
+
+test("a server that goes silent without closing makes the client reconnect", async () => {
+  const server = await startRawServer();
+  const { logs } = startClient(server.apiUrl);
+  await until(() => logs().includes("Connected as controller-1"), 2000);
+  const silentAt = Date.now();
+  // Detection within the timeout plus one check interval, then the first backoff (at most 1000ms).
+  await until(() => server.connections() === 2, LIVENESS_MS * 2 + 1500);
+  expect(logs()).toContain(`No traffic for ${LIVENESS_MS}ms`);
+  expect(logs()).toContain("Disconnected. Reconnect in");
+  expect(Date.now() - silentAt).toBeGreaterThanOrEqual(LIVENESS_MS);
+});
+
+test("server ping frames keep an idle controller connected", async () => {
+  // Matches the API: it pings on an interval and sends no app message to an idle controller.
+  // This peer never answers the client's pings, so only the server pings count.
+  const server = await startRawServer({ pingEveryMs: LIVENESS_MS / 3 });
+  const { logs } = startClient(server.apiUrl);
+  await until(() => logs().includes("Connected as controller-1"), 2000);
+  await Bun.sleep(LIVENESS_MS * 4);
+  expect(server.connections()).toBe(1);
+  expect(logs()).not.toContain("Disconnected");
+});
+
+test("pongs to client pings keep an idle controller connected", async () => {
+  // A healthy peer that sends nothing on its own still answers pings.
+  const http = createHttpServer();
+  const wss = new WebSocketServer({ server: http });
+  let connections = 0;
+  wss.on("connection", (ws) => {
+    connections++;
+    ws.send(WELCOME);
+  });
+  const port = await listenOnFreePort(http);
+  cleanups.push(() => {
+    for (const ws of wss.clients) ws.terminate();
+    wss.close();
+    http.close();
+  });
+  const { logs } = startClient(`http://127.0.0.1:${port}`);
+  await until(() => logs().includes("Connected as controller-1"), 2000);
+  await Bun.sleep(LIVENESS_MS * 4);
+  expect(connections).toBe(1);
+  expect(logs()).not.toContain("Disconnected");
+});

package/src/tests/runtime-providers-docker.test.ts

@@ -310,6 +310,7 @@ describe("Docker runtime provider", () => {
     await provider.terminate(target, handle);
     expect(calls[0]?.init.method).toBe("DELETE");
     expect(calls[0]?.url.searchParams.get("force")).toBe("true");
+    expect(calls[0]?.url.searchParams.get("v")).toBe("true");
   });
 
   test("termination succeeds when the container is already absent", async () => {
@@ -376,6 +377,7 @@ describe("Docker runtime provider", () => {
         "DELETE /v1.47/containers/container-1",
       ]);
       expect(calls[2]?.url.searchParams.get("force")).toBe("true");
+      expect(calls[2]?.url.searchParams.get("v")).toBe("true");
     }
   });
 

package/src/tests/script-connections.test.ts

@@ -570,6 +570,9 @@ describe("script connections", () => {
       markMigrationApplied(database, "151_repair_scheduled_task_required_params.sql");
       // 152 indexes task routing context absent from this migration-112-only fixture.
       markMigrationApplied(database, "152_routing_decisions.sql");
+      // 153 drops a column, so SQLite revalidates 144's agent_tasks triggers
+      // against this fixture's id-only agent_tasks (Bun < 1.4.4 hid the error).
+      markMigrationApplied(database, "153_deferred_task_wait_members.sql");
       // 156 alters inbox_item_state, which this migration-112-only fixture does not create.
       markMigrationApplied(database, "156_inbox_item_state_read_at.sql");
       // 157 alters scheduled_tasks and slack_messages, neither of which this
@@ -648,6 +651,8 @@ describe("script connections", () => {
       markMigrationApplied(database, "240_workflow_run_parent_step_attempt.sql");
       // 241 indexes skills, which this migration-112-only fixture does not create.
       markMigrationApplied(database, "241_skills_meta_fts.sql");
+      // 244 alters api_key_status, which this migration-112-only fixture does not create.
+      markMigrationApplied(database, "244_api_key_validation.sql");
 
       const id = crypto.randomUUID();
       const now = new Date().toISOString();