@desplega.ai/agent-swarm 1.158.0 → 1.160.0

package/src/tests/runner-key-outcome-late-reset.test.ts

@@ -0,0 +1,224 @@
+/**
+ * A success reset that times out on the worker can still finish on the API
+ * later. These tests run the production reporting code against the real
+ * api-keys handler and a real DB, delay the success handler, and check the
+ * state AFTER the late handler finishes: it must not clear auth failures
+ * recorded after the fence the task read with its key draw, from this runner
+ * or any other.
+ */
+import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { unlink } from "node:fs/promises";
+import { createServer as createHttpServer, type Server } from "node:http";
+import { closeDb, getKeyStatuses, getKv, initDb } from "../be/db";
+import {
+  type ApiConfig,
+  checkCompletedProcesses,
+  type RunnerState,
+  type RunningTask,
+  reportKeyCompletionOutcome,
+} from "../commands/runner";
+import { handleApiKeys } from "../http/api-keys";
+import type { ProviderResult } from "../providers/types";
+import { listenOnFreePort } from "./test-net";
+
+const TEST_DB = `./test-runner-key-outcome-late-reset-${Date.now()}.sqlite`;
+const authFailure =
+  "[auth-error] Codex authentication failed — check OPENAI_API_KEY or ChatGPT login. Original error: workspace routing discovery unauthorized (401)";
+
+let api: Server;
+let proxy: Server;
+let apiUrl = "";
+let clearDelayMs = 0;
+/** Credential reports in the order the API FINISHED handling them. */
+const handled: string[] = [];
+/** Success resets still in flight on the API side. */
+let pendingClears: Promise<void>[] = [];
+
+const status = async (keySuffix: string) =>
+  (await getKeyStatuses("CODEX_OAUTH")).find((s) => s.keySuffix === keySuffix)!;
+/** The `authFailureFence` a runner reads with its key draw, before the task starts. */
+const drawFence = async () => {
+  const resp = await fetch(`${apiUrl}/api/keys/available?keyType=CODEX_OAUTH&totalKeys=1`);
+  return ((await resp.json()) as { authFailureFence: number }).authFailureFence;
+};
+
+beforeAll(async () => {
+  process.env.DB_PATH = TEST_DB;
+  initDb(TEST_DB);
+  // The real api-keys handler on a real DB.
+  api = createHttpServer(async (req, res) => {
+    const url = new URL(req.url || "/", "http://localhost");
+    const pathSegments = url.pathname.split("/").filter(Boolean);
+    if (await handleApiKeys(req, res, pathSegments, url.searchParams)) {
+      if (url.pathname === "/api/keys/clear-rate-limit") handled.push("clear");
+      if (url.pathname === "/api/keys/report-auth-failure") handled.push("auth-failure");
+      return;
+    }
+    // Every other worker call (finish, active sessions, ...) is out of scope.
+    res.writeHead(200, { "Content-Type": "application/json" });
+    res.end("{}");
+  });
+  const apiPort = await listenOnFreePort(api);
+  // A proxy in front of it. It reads the whole request, then forwards it; a
+  // success reset is forwarded late. The worker's abort never cancels it.
+  proxy = createHttpServer(async (req, res) => {
+    const chunks: Buffer[] = [];
+    for await (const chunk of req) chunks.push(chunk as Buffer);
+    const forward = async () => {
+      if (req.url === "/api/keys/clear-rate-limit") await Bun.sleep(clearDelayMs);
+      const upstream = await fetch(`http://127.0.0.1:${apiPort}${req.url}`, {
+        method: req.method,
+        headers: { "Content-Type": "application/json" },
+        body: req.method === "GET" ? undefined : Buffer.concat(chunks),
+      });
+      const text = await upstream.text();
+      if (!res.destroyed) {
+        res.writeHead(upstream.status, { "Content-Type": "application/json" });
+        res.end(text);
+      }
+    };
+    const done = forward();
+    if (req.url === "/api/keys/clear-rate-limit") pendingClears.push(done);
+    await done;
+  });
+  const proxyPort = await listenOnFreePort(proxy);
+  apiUrl = `http://127.0.0.1:${proxyPort}`;
+});
+
+afterAll(async () => {
+  await Promise.all(pendingClears);
+  for (const s of [proxy, api]) await new Promise<void>((resolve) => s.close(() => resolve()));
+  closeDb();
+  await unlink(TEST_DB).catch(() => {});
+  await unlink(`${TEST_DB}-wal`).catch(() => {});
+  await unlink(`${TEST_DB}-shm`).catch(() => {});
+});
+
+beforeEach(() => {
+  handled.length = 0;
+  pendingClears = [];
+  clearDelayMs = 0;
+});
+
+describe("late success reset", () => {
+  test("a timed-out success reset that finishes late keeps the later auth bench", async () => {
+    const credential = { keyType: "CODEX_OAUTH", keySuffix: "late1", keyIndex: 0 };
+    const authFence = await drawFence();
+    const report = (i: number, exitCode: number, failureReason?: string, timeoutMs?: number) =>
+      reportKeyCompletionOutcome({
+        apiUrl,
+        apiKey: "test-key",
+        credential: { ...credential, authFence },
+        taskId: `00000000-0000-4000-8000-00000000010${i}`,
+        exitCode,
+        failureReason,
+        timeoutMs,
+      });
+
+    clearDelayMs = 400;
+    const started = Date.now();
+    await report(0, 0, undefined, 100);
+    expect(Date.now() - started).toBeLessThan(400);
+    await report(1, 1, authFailure);
+    await report(2, 1, authFailure);
+
+    let row = await status("late1");
+    expect(row.consecutiveAuthFailures).toBe(2);
+    expect(row.status).toBe("rate_limited");
+
+    // Let the late success handler finish on the API, then check again.
+    await Promise.all(pendingClears);
+    expect(handled).toEqual(["auth-failure", "auth-failure", "clear"]);
+    row = await status("late1");
+    expect(row.consecutiveAuthFailures).toBe(2);
+    expect(row.status).toBe("rate_limited");
+    expect(await getKv("codex-auth-watch", "bench:late1")).not.toBeNull();
+  });
+
+  test("two runners: A's late success cannot clear B's later failures, whatever A's clock says", async () => {
+    const credential = { keyType: "CODEX_OAUTH", keySuffix: "late3", keyIndex: 2 };
+    const report = (
+      runner: string,
+      authFence: number,
+      exitCode: number,
+      failureReason?: string,
+      timeoutMs?: number,
+    ) =>
+      reportKeyCompletionOutcome({
+        apiUrl,
+        apiKey: "test-key",
+        credential: { ...credential, authFence },
+        taskId: crypto.randomUUID(),
+        exitCode,
+        failureReason,
+        timeoutMs,
+      }).then(() => runner);
+
+    // Runner A draws the key first; its clock runs an hour ahead. Nothing it sends
+    // carries a clock value, so the skew cannot move its success after B's failures.
+    const fenceA = await drawFence();
+    const realNow = Date.now;
+    Date.now = () => realNow() + 3_600_000;
+    clearDelayMs = 400;
+    try {
+      await report("A", fenceA, 0, undefined, 100);
+    } finally {
+      Date.now = realNow;
+    }
+    // Runner B draws after A and fails twice on the same login.
+    const fenceB = await drawFence();
+    await report("B", fenceB, 1, authFailure);
+    await report("B", fenceB, 1, authFailure);
+
+    await Promise.all(pendingClears);
+    expect(handled).toEqual(["auth-failure", "auth-failure", "clear"]);
+    const row = await status("late3");
+    expect(row.consecutiveAuthFailures).toBe(2);
+    expect(row.status).toBe("rate_limited");
+    expect(await getKv("codex-auth-watch", "bench:late3")).not.toBeNull();
+  });
+
+  test("checkCompletedProcesses lands each outcome before it handles the next completion", async () => {
+    const credentialInfo = {
+      keyType: "CODEX_OAUTH",
+      keySuffix: "late2",
+      keyIndex: 1,
+      authFence: await drawFence(),
+    };
+    const running = (i: number, result: ProviderResult) =>
+      ({
+        taskId: `00000000-0000-4000-8000-00000000020${i}`,
+        startTime: new Date(),
+        result,
+        credentialInfo,
+        harnessProvider: "codex",
+        hasLocalEnvironment: false,
+      }) as unknown as RunningTask;
+    const state = {
+      activeTasks: new Map<string, RunningTask>(),
+      maxConcurrent: 3,
+      startedAt: Date.now(),
+      tasksProcessed: 0,
+      harnessProvider: "codex",
+      codexCreditsExhaustedCooldownMs: 7_200_000,
+      modelWindowBlocks: new Map(),
+    } as RunnerState;
+    const tasks = [
+      running(0, { exitCode: 0, isError: false } as ProviderResult),
+      running(1, { exitCode: 1, isError: true, failureReason: authFailure } as ProviderResult),
+      running(2, { exitCode: 1, isError: true, failureReason: authFailure } as ProviderResult),
+    ];
+    for (const t of tasks) state.activeTasks.set(t.taskId, t);
+    const apiConfig: ApiConfig = { apiUrl, apiKey: "test-key", agentId: crypto.randomUUID() };
+
+    clearDelayMs = 200;
+    await checkCompletedProcesses(state, "worker", apiConfig);
+
+    // Awaited in order: the slow success reset finished before the failures.
+    expect(handled).toEqual(["clear", "auth-failure", "auth-failure"]);
+    expect(state.activeTasks.size).toBe(0);
+    const row = await status("late2");
+    expect(row.consecutiveAuthFailures).toBe(2);
+    expect(row.status).toBe("rate_limited");
+  });
+});

package/src/tests/runner-registration-retry.test.ts

@@ -0,0 +1,149 @@
+import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { createServer, type Server } from "node:http";
+import {
+  AgentRegistrationHttpError,
+  isRetryableRegistrationError,
+  registerAgent,
+  registerAgentWithRetry,
+} from "../commands/runner";
+import { getFreePort, listenOnFreePort } from "./test-net";
+
+/**
+ * After the 2026-09-30 deploy every worker got `500 {"error":"database is
+ * locked"}` from `POST /api/agents` for ~4 minutes and exited on the first
+ * failure. Boot registration now retries transient failures inside a budget
+ * and still exits on a 4xx. These tests drive the real `registerAgent` against
+ * a fake API, with an injected clock so no real time passes.
+ */
+
+const AGENT_ID = "3f0b7a52-8c1d-4e6a-9b27-5d4c1e8a7f10";
+
+let server: Server;
+let baseUrl: string;
+/** Responses served in order; the last one repeats. */
+let script: Array<{ status: number; body: unknown }>;
+let calls: number;
+
+beforeAll(async () => {
+  server = createServer((req, res) => {
+    req.resume();
+    const url = new URL(req.url ?? "/", "http://localhost");
+    if (req.method === "POST" && url.pathname === "/api/agents") {
+      const next = script[Math.min(calls, script.length - 1)] ?? { status: 200, body: {} };
+      calls++;
+      res.writeHead(next.status, { "Content-Type": "application/json" });
+      res.end(JSON.stringify(next.body));
+      return;
+    }
+    res.writeHead(404, { "Content-Type": "application/json" });
+    res.end(JSON.stringify({ error: "not found" }));
+  });
+  const port = await listenOnFreePort(server);
+  baseUrl = `http://127.0.0.1:${port}`;
+});
+
+afterAll(() => {
+  server.close();
+});
+
+beforeEach(() => {
+  script = [];
+  calls = 0;
+});
+
+function fakeClock() {
+  let t = 0;
+  const waits: number[] = [];
+  return {
+    waits,
+    now: () => t,
+    sleep: async (ms: number) => {
+      waits.push(ms);
+      t += ms;
+    },
+  };
+}
+
+function register(apiUrl = baseUrl) {
+  return registerAgent({
+    apiUrl,
+    apiKey: "example-test-swarm-key",
+    agentId: AGENT_ID,
+    name: "retry-test-worker",
+    isLead: false,
+    harnessProvider: "claude",
+  });
+}
+
+describe("boot registration retry", () => {
+  test("rides out 'database is locked' 500s, then registers", async () => {
+    script = [
+      { status: 500, body: { error: "database is locked" } },
+      { status: 500, body: { error: "database is locked" } },
+      { status: 503, body: { error: "unavailable" } },
+      { status: 200, body: { enabledCapabilities: ["core"] } },
+    ];
+    const clock = fakeClock();
+    const reg = await registerAgentWithRetry(() => register(), { label: "test", ...clock });
+    expect(reg.serverCapabilities).toEqual(["core"]);
+    expect(calls).toBe(4);
+    expect(clock.waits).toHaveLength(3);
+    // Backoff grows (2s, 4s, 8s nominal, +/-20% jitter).
+    expect(clock.waits[0]).toBeGreaterThanOrEqual(1_600);
+    expect(clock.waits[2]).toBeGreaterThanOrEqual(6_400);
+  });
+
+  test("exits on the first 4xx without retrying", async () => {
+    script = [{ status: 401, body: { error: "unauthorized" } }];
+    const clock = fakeClock();
+    const err = await registerAgentWithRetry(() => register(), { label: "test", ...clock }).catch(
+      (e: unknown) => e,
+    );
+    expect(err).toBeInstanceOf(AgentRegistrationHttpError);
+    expect((err as AgentRegistrationHttpError).status).toBe(401);
+    expect(calls).toBe(1);
+    expect(clock.waits).toHaveLength(0);
+  });
+
+  test("gives up once the budget is spent", async () => {
+    script = [{ status: 500, body: { error: "database is locked" } }];
+    const clock = fakeClock();
+    const err = await registerAgentWithRetry(() => register(), {
+      label: "test",
+      budgetMs: 60_000,
+      ...clock,
+    }).catch((e: unknown) => e);
+    expect(err).toBeInstanceOf(AgentRegistrationHttpError);
+    expect(clock.now()).toBeLessThanOrEqual(60_000);
+    // 2 + 4 + 8 + 16 + 30 nominal fits only partly in 60s; more than one try, bounded.
+    expect(calls).toBeGreaterThan(1);
+    expect(calls).toBeLessThan(10);
+  });
+
+  test("retries a connection error", async () => {
+    const deadPort = await getFreePort();
+    const clock = fakeClock();
+    let attempts = 0;
+    const reg = await registerAgentWithRetry(
+      () => {
+        attempts++;
+        return attempts < 3 ? register(`http://127.0.0.1:${deadPort}`) : register();
+      },
+      { label: "test", ...clock },
+    );
+    expect(reg).toEqual({});
+    expect(attempts).toBe(3);
+    expect(clock.waits).toHaveLength(2);
+  });
+
+  test("classifies statuses", () => {
+    const http = (status: number, body = "{}") => new AgentRegistrationHttpError(status, body);
+    expect(isRetryableRegistrationError(http(500))).toBe(true);
+    expect(isRetryableRegistrationError(http(502))).toBe(true);
+    expect(isRetryableRegistrationError(http(429))).toBe(true);
+    expect(isRetryableRegistrationError(http(400))).toBe(false);
+    expect(isRetryableRegistrationError(http(403))).toBe(false);
+    expect(isRetryableRegistrationError(http(409, '{"error":"database is locked"}'))).toBe(true);
+    expect(isRetryableRegistrationError(new TypeError("fetch failed"))).toBe(true);
+  });
+});

package/src/tests/runtime-model-catalog.test.ts

@@ -0,0 +1,86 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import {
+  getClaudeManagedModelPricing,
+  listClaudeManagedModels,
+} from "../providers/claude-managed-models";
+import {
+  getCodexContextWindow,
+  getCodexModelPricing,
+  listCodexModels,
+} from "../providers/codex-models";
+import {
+  refreshRuntimeModelCatalog,
+  resetRuntimeModelCatalogForTests,
+  setRuntimeModelCatalog,
+} from "../utils/runtime-model-catalog";
+
+const NEW_CODEX = "gpt-9-test";
+const NEW_CLAUDE = "claude-opus-9";
+
+const liveCatalog = {
+  openai: {
+    id: "openai",
+    models: {
+      [NEW_CODEX]: {
+        id: NEW_CODEX,
+        cost: { input: 3, output: 12 },
+        limit: { context: 2_000_000 },
+        release_date: "2099-01-01",
+        reasoning: true,
+      },
+    },
+  },
+  anthropic: {
+    id: "anthropic",
+    models: {
+      [NEW_CLAUDE]: {
+        id: NEW_CLAUDE,
+        cost: { input: 6, output: 30, cache_read: 0.6, cache_write: 7.5 },
+        limit: { context: 1_000_000 },
+        release_date: "2099-01-01",
+      },
+    },
+  },
+};
+
+afterEach(() => resetRuntimeModelCatalogForTests());
+
+describe("runtime model catalog", () => {
+  test("a model unknown to the snapshot becomes listed, priced and windowed once the live catalog has it", () => {
+    expect(listCodexModels()).not.toContain(NEW_CODEX);
+    expect(getCodexModelPricing(NEW_CODEX)).toBeUndefined();
+
+    setRuntimeModelCatalog(liveCatalog);
+
+    expect(listCodexModels()[0]).toBe(NEW_CODEX);
+    expect(getCodexModelPricing(NEW_CODEX)).toMatchObject({
+      inputPerMillion: 3,
+      outputPerMillion: 12,
+    });
+    expect(getCodexContextWindow(NEW_CODEX)).toBe(2_000_000);
+    expect(listClaudeManagedModels()).toContain(NEW_CLAUDE);
+    expect(getClaudeManagedModelPricing(NEW_CLAUDE)).toEqual({
+      inputPerMillion: 6,
+      outputPerMillion: 30,
+      cacheReadPerMillion: 0.6,
+      cacheWritePerMillion: 7.5,
+    });
+  });
+
+  test("worker pull from GET /api/models-catalog installs the live layer; failures keep the snapshot", async () => {
+    const failing = (async () => new Response("nope", { status: 503 })) as unknown as typeof fetch;
+    expect(await refreshRuntimeModelCatalog({ apiUrl: "http://x", fetchImpl: failing })).toBe(
+      false,
+    );
+    expect(getCodexModelPricing(NEW_CODEX)).toBeUndefined();
+
+    let url = "";
+    const ok = (async (input: string | URL | Request) => {
+      url = String(input);
+      return Response.json({ source: "live", providers: liveCatalog });
+    }) as unknown as typeof fetch;
+    expect(await refreshRuntimeModelCatalog({ apiUrl: "http://x", fetchImpl: ok })).toBe(true);
+    expect(url).toBe("http://x/api/models-catalog");
+    expect(getCodexModelPricing(NEW_CODEX)?.inputPerMillion).toBe(3);
+  });
+});

package/src/tests/script-connections.test.ts

@@ -585,6 +585,15 @@ describe("script connections", () => {
       markMigrationApplied(database, "175_workflow_runs_list_index.sql");
       // 177 indexes events, which this migration-112-only fixture does not create.
       markMigrationApplied(database, "177_events_event_agent_created_index.sql");
+      // 182 alters agent_tasks, which this migration-112-only fixture does not create.
+      markMigrationApplied(database, "182_task_human_free_flag.sql");
+      // 184 alters agents and agent_tasks, which this migration-112-only fixture
+      // does not create.
+      markMigrationApplied(database, "184_model_catalog.sql");
+      // 186 alters api_key_status, which this migration-112-only fixture does not create.
+      markMigrationApplied(database, "186_api_key_seat_mismatch.sql");
+      // 185 alters api_key_status, which this migration-112-only fixture does not create.
+      markMigrationApplied(database, "185_api_key_auth_failures.sql");
 
       const id = crypto.randomUUID();
       const now = new Date().toISOString();

package/src/tests/seat-mismatch-report.test.ts

@@ -0,0 +1,85 @@
+import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { reportSeatMismatchOutcome } from "../commands/seat-mismatch-report";
+
+/**
+ * Report helper for a `credits_required` seat mismatch: the report sends the
+ * credential and model family, resolves only after the API responds, and
+ * never throws. The completion ordering is covered by
+ * `credential-outcome-report.test.ts`.
+ */
+
+interface RecordedRequest {
+  method: string;
+  path: string;
+  auth: string | null;
+  body: Record<string, unknown>;
+}
+
+const SEAT_REPORT_DELAY_MS = 150;
+let server: ReturnType<typeof Bun.serve>;
+let apiUrl: string;
+let requests: RecordedRequest[] = [];
+let respondedAt: number | undefined;
+let status = 200;
+
+beforeAll(() => {
+  server = Bun.serve({
+    port: 0,
+    async fetch(req) {
+      const url = new URL(req.url);
+      requests.push({
+        method: req.method,
+        path: url.pathname,
+        auth: req.headers.get("authorization"),
+        body: (await req.json()) as Record<string, unknown>,
+      });
+      await Bun.sleep(SEAT_REPORT_DELAY_MS);
+      respondedAt = Date.now();
+      return Response.json({ success: status === 200 }, { status });
+    },
+  });
+  apiUrl = server.url.toString().replace(/\/$/, "");
+});
+
+afterAll(() => {
+  server.stop(true);
+});
+
+beforeEach(() => {
+  requests = [];
+  respondedAt = undefined;
+  status = 200;
+});
+
+const credential = { keyType: "CLAUDE_CODE_OAUTH_TOKEN", keySuffix: "TJAAA", keyIndex: 3 };
+
+describe("reportSeatMismatchOutcome", () => {
+  test("posts the seat mismatch and resolves only after the API responds", async () => {
+    await reportSeatMismatchOutcome(apiUrl, "example-test-key", credential, "fable");
+    const resolvedAt = Date.now();
+
+    expect(requests).toEqual([
+      {
+        method: "POST",
+        path: "/api/keys/report-seat-mismatch",
+        auth: "Bearer example-test-key",
+        body: {
+          keyType: "CLAUDE_CODE_OAUTH_TOKEN",
+          keySuffix: "TJAAA",
+          keyIndex: 3,
+          model: "fable",
+        },
+      },
+    ]);
+    expect(respondedAt).toBeDefined();
+    expect(resolvedAt).toBeGreaterThanOrEqual(respondedAt!);
+  });
+
+  test("a failed report is logged and never throws", async () => {
+    status = 500;
+    await expect(
+      reportSeatMismatchOutcome(apiUrl, "example-test-key", credential, "fable"),
+    ).resolves.toBeUndefined();
+    expect(requests).toHaveLength(1);
+  });
+});