@desplega.ai/agent-swarm 1.158.0 → 1.159.0

package/src/tests/harness-model-support.test.ts

@@ -0,0 +1,374 @@
+// Model-catalog phase 4: harness CLI model support, CLI-unsupported fallback
+// and fail-fast at claim, and the worker-side outcome reporter.
+
+import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { unlink } from "node:fs/promises";
+import { closeDb, createAgent, createTaskExtended, getDbClient, initDb } from "../be/db";
+import {
+  getHarnessModelSupport,
+  recordHarnessModelSupport,
+  setAgentHarnessCliVersion,
+} from "../be/harness-model-support";
+import { type ModelCatalogEntry, replaceModelCatalog } from "../be/model-catalog-store";
+import { invalidateTierResolutionCatalog } from "../be/model-tier-resolution";
+import { handlePoll } from "../http/poll";
+import {
+  parseCliVersion,
+  reportHarnessModelOutcome,
+  resetHarnessCliVersionForTests,
+} from "../utils/harness-cli-version";
+import { isUnknownModelError } from "../utils/harness-model-error";
+
+const TEST_DB_PATH = "./test-harness-model-support.sqlite";
+const DAY = 24 * 60 * 60 * 1000;
+const CLI = "2.1.283";
+
+async function removeDbFiles(path: string): Promise<void> {
+  for (const suffix of ["", "-wal", "-shm"]) {
+    try {
+      await unlink(path + suffix);
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+    }
+  }
+}
+
+function entry(modelId: string, daysOld: number): ModelCatalogEntry {
+  return {
+    provider: "anthropic",
+    modelId,
+    releaseDate: new Date(Date.now() - daysOld * DAY).toISOString().slice(0, 10),
+    pricing: { input: 5, output: 25 },
+    checkedAt: Date.now(),
+  };
+}
+
+type Trigger = { type: string; taskId: string; task: Record<string, unknown> };
+
+async function callPoll(agentId: string): Promise<Trigger | null> {
+  let bodyStr = "";
+  const req = {
+    method: "GET",
+    url: "/api/poll",
+    headers: { "x-agent-id": agentId },
+  } as unknown as Parameters<typeof handlePoll>[0];
+  const res = {
+    setHeader() {},
+    writeHead() {},
+    end(body?: string) {
+      bodyStr = body ?? "";
+    },
+  } as unknown as Parameters<typeof handlePoll>[1];
+  await handlePoll(req, res, ["api", "poll"], new URLSearchParams(), agentId);
+  return (JSON.parse(bodyStr) as { trigger: Trigger | null }).trigger;
+}
+
+async function worker(name: string) {
+  const agent = await createAgent({
+    name,
+    isLead: false,
+    status: "idle",
+    maxTasks: 1,
+    harnessProvider: "claude",
+  });
+  await setAgentHarnessCliVersion(agent.id, CLI);
+  return agent;
+}
+
+beforeAll(async () => {
+  await removeDbFiles(TEST_DB_PATH);
+  initDb(TEST_DB_PATH);
+});
+
+afterAll(async () => {
+  closeDb();
+  await removeDbFiles(TEST_DB_PATH);
+});
+
+beforeEach(async () => {
+  const client = getDbClient();
+  await client.run("DELETE FROM agent_tasks");
+  await client.run("DELETE FROM agents");
+  await client.run("DELETE FROM harness_model_support");
+  await client.run("DELETE FROM model_alias_resolutions");
+  await replaceModelCatalog([
+    entry("claude-opus-5", 200),
+    entry("claude-opus-5-5", 30),
+    entry("claude-sonnet-5-5", 10),
+  ]);
+  invalidateTierResolutionCatalog();
+  delete process.env.MODEL_TIER_CLAUDE_SMART;
+});
+
+describe("claim-time CLI support", () => {
+  test("alias resolving to an unsupported model falls back within the family", async () => {
+    process.env.MODEL_TIER_CLAUDE_SMART = "latest:anthropic/opus@stable";
+    await recordHarnessModelSupport({
+      harness: "claude",
+      cliVersion: CLI,
+      modelId: "claude-opus-5-5",
+      status: "unsupported",
+      error: "There's an issue with the selected model (claude-opus-5-5).",
+    });
+    const w = await worker("w-fallback");
+    const task = await createTaskExtended("smart work", { agentId: w.id, modelTier: "smart" });
+
+    const trigger = await callPoll(w.id);
+    expect(trigger?.task.resolvedModel).toBe("claude-opus-5");
+    expect(trigger?.task.modelSource).toBe("fallback:cli-unsupported");
+    const row = await getDbClient().get<{ resolvedModel: string; modelSource: string }>(
+      "SELECT resolvedModel, modelSource FROM agent_tasks WHERE id = ?",
+      [task.id],
+    );
+    expect(row).toEqual({
+      resolvedModel: "claude-opus-5",
+      modelSource: "fallback:cli-unsupported",
+    });
+    delete process.env.MODEL_TIER_CLAUDE_SMART;
+  });
+
+  test("the fallback never picks an unpriced sibling, which would run outside cost accounting", async () => {
+    process.env.MODEL_TIER_CLAUDE_SMART = "latest:anthropic/opus@stable";
+    // Newer than the priced fallback (claude-opus-5) but with no price yet.
+    await replaceModelCatalog([
+      entry("claude-opus-5", 200),
+      entry("claude-opus-5-5", 30),
+      { ...entry("claude-opus-5-4", 60), pricing: null },
+    ]);
+    invalidateTierResolutionCatalog();
+    await recordHarnessModelSupport({
+      harness: "claude",
+      cliVersion: CLI,
+      modelId: "claude-opus-5-5",
+      status: "unsupported",
+    });
+    const w = await worker("w-unpriced-fallback");
+    await createTaskExtended("smart work", { agentId: w.id, modelTier: "smart" });
+
+    const trigger = await callPoll(w.id);
+    expect(trigger?.task.resolvedModel).toBe("claude-opus-5");
+    expect(trigger?.task.modelSource).toBe("fallback:cli-unsupported");
+    delete process.env.MODEL_TIER_CLAUDE_SMART;
+  });
+
+  test("explicit unsupported model fails fast: trigger carries modelUnsupported", async () => {
+    await recordHarnessModelSupport({
+      harness: "claude",
+      cliVersion: CLI,
+      modelId: "claude-opus-5-5",
+      status: "unsupported",
+    });
+    const w = await worker("w-explicit");
+    await createTaskExtended("pinned work", { agentId: w.id, model: "claude-opus-5-5" });
+    const trigger = await callPoll(w.id);
+    expect(String(trigger?.task.modelUnsupported)).toContain("claude-opus-5-5");
+    expect(trigger?.task.resolvedModel).toBeUndefined();
+  });
+
+  test("an explicit latest: alias on an unsupported model falls back, like a tier alias", async () => {
+    await recordHarnessModelSupport({
+      harness: "claude",
+      cliVersion: CLI,
+      modelId: "claude-opus-5-5",
+      status: "unsupported",
+    });
+    const w = await worker("w-explicit-alias");
+    const task = await createTaskExtended("aliased work", {
+      agentId: w.id,
+      model: "latest:anthropic/opus",
+    });
+    const trigger = await callPoll(w.id);
+    expect(trigger?.task.modelUnsupported).toBeUndefined();
+    expect(trigger?.task.resolvedModel).toBe("claude-opus-5");
+    expect(trigger?.task.modelSource).toBe("fallback:cli-unsupported");
+    // The alias the task named is still on the record.
+    expect(trigger?.task.modelAlias).toBe("latest:anthropic/opus");
+    const row = await getDbClient().get<{ modelAlias: string }>(
+      "SELECT modelAlias FROM agent_tasks WHERE id = ?",
+      [task.id],
+    );
+    expect(row?.modelAlias).toBe("latest:anthropic/opus");
+  });
+
+  test("an alias with no usable sibling keeps its resolution instead of failing the task", async () => {
+    for (const modelId of ["claude-opus-5-5", "claude-opus-5"]) {
+      await recordHarnessModelSupport({
+        harness: "claude",
+        cliVersion: CLI,
+        modelId,
+        status: "unsupported",
+      });
+    }
+    const w = await worker("w-no-sibling");
+    await createTaskExtended("aliased work", { agentId: w.id, model: "latest:anthropic/opus" });
+    const trigger = await callPoll(w.id);
+    expect(trigger?.task.modelUnsupported).toBeUndefined();
+    expect(trigger?.task.resolvedModel).toBe("claude-opus-5-5");
+    expect(trigger?.task.modelSource).toBe("model");
+  });
+
+  test("a tier default (CLI shortname) the CLI rejected falls back within its family", async () => {
+    await recordHarnessModelSupport({
+      harness: "claude",
+      cliVersion: CLI,
+      modelId: "opus",
+      status: "unsupported",
+    });
+    const w = await worker("w-tier-default");
+    await createTaskExtended("smart work", { agentId: w.id, modelTier: "smart" });
+    const trigger = await callPoll(w.id);
+    expect(trigger?.task.modelUnsupported).toBeUndefined();
+    expect(trigger?.task.resolvedModel).toBe("claude-opus-5-5");
+    expect(trigger?.task.modelSource).toBe("fallback:cli-unsupported");
+  });
+
+  test("a tier default shortname stays when only a catalog id was rejected: the CLI resolves it", async () => {
+    await recordHarnessModelSupport({
+      harness: "claude",
+      cliVersion: CLI,
+      modelId: "claude-opus-5-5",
+      status: "unsupported",
+    });
+    const w = await worker("w-shortname-ok");
+    await createTaskExtended("smart work", { agentId: w.id, modelTier: "smart" });
+    const trigger = await callPoll(w.id);
+    expect(trigger?.task.resolvedModel).toBe("opus");
+    expect(trigger?.task.modelSource).toBe("tier-default");
+  });
+
+  test("a concrete id from a tier config value falls back; only a pinned task model fails fast", async () => {
+    process.env.MODEL_TIER_CLAUDE_SMART = "claude-opus-5-5";
+    await recordHarnessModelSupport({
+      harness: "claude",
+      cliVersion: CLI,
+      modelId: "claude-opus-5-5",
+      status: "unsupported",
+    });
+    const w = await worker("w-tier-config");
+    await createTaskExtended("smart work", { agentId: w.id, modelTier: "smart" });
+    const trigger = await callPoll(w.id);
+    expect(trigger?.task.resolvedModel).toBe("claude-opus-5");
+    expect(trigger?.task.modelSource).toBe("fallback:cli-unsupported");
+    delete process.env.MODEL_TIER_CLAUDE_SMART;
+  });
+
+  test("unknown support (no row) is allowed; other CLI versions are unaffected", async () => {
+    await recordHarnessModelSupport({
+      harness: "claude",
+      cliVersion: "2.0.0",
+      modelId: "claude-opus-5-5",
+      status: "unsupported",
+    });
+    const w = await worker("w-unknown");
+    await createTaskExtended("pinned work", { agentId: w.id, model: "claude-opus-5-5" });
+    const trigger = await callPoll(w.id);
+    expect(trigger?.task.resolvedModel).toBe("claude-opus-5-5");
+    expect(trigger?.task.modelUnsupported).toBeUndefined();
+  });
+
+  test("an ok row is never downgraded to unknown", async () => {
+    const base = { harness: "claude", cliVersion: CLI, modelId: "claude-opus-5-5" };
+    await recordHarnessModelSupport({ ...base, status: "ok" });
+    await recordHarnessModelSupport({ ...base, status: "unknown" });
+    expect(await getHarnessModelSupport("claude", CLI, "claude-opus-5-5")).toBe("ok");
+  });
+});
+
+describe("worker outcome reporting", () => {
+  test("parses CLI versions and recognizes unknown-model errors", () => {
+    expect(parseCliVersion("2.1.283 (Claude Code)")).toBe("2.1.283");
+    expect(parseCliVersion("codex-cli 0.157.1")).toBe("0.157.1");
+    expect(isUnknownModelError("There's an issue with the selected model (claude-x).")).toBe(true);
+    expect(isUnknownModelError("The model `gpt-9` does not exist or you do not have access")).toBe(
+      true,
+    );
+    expect(isUnknownModelError("rate limit reached")).toBe(false);
+  });
+
+  test("with a model id, only a rejection that names that model counts", () => {
+    const rejection = "There's an issue with the selected model (claude-x). It may not exist.";
+    expect(isUnknownModelError(rejection, "claude-x")).toBe(true);
+    expect(isUnknownModelError(rejection, "CLAUDE-X")).toBe(true);
+    expect(isUnknownModelError("The model `gpt-9` does not exist", "gpt-9")).toBe(true);
+    expect(isUnknownModelError("unknown model 'gpt-9'\nstack trace", "gpt-9")).toBe(true);
+    // Rejection text for another model, or phrase-only text from an unrelated failure.
+    expect(isUnknownModelError(rejection, "claude-y")).toBe(false);
+    expect(isUnknownModelError("tool call failed: invalid model output", "claude-x")).toBe(false);
+    expect(isUnknownModelError("unknown model type in schema\nran claude-x fine", "claude-x")).toBe(
+      false,
+    );
+  });
+
+  test("an unrelated failure containing a rejection phrase does not report unsupported", async () => {
+    resetHarnessCliVersionForTests({ claude: CLI });
+    const bodies: unknown[] = [];
+    const fetchImpl = (async (_url: string, init: RequestInit) => {
+      bodies.push(JSON.parse(String(init.body)));
+      return new Response("{}", { status: 200 });
+    }) as unknown as typeof fetch;
+
+    await reportHarnessModelOutcome({
+      apiUrl: "http://x",
+      agentId: "a",
+      harness: "claude",
+      fetchImpl,
+      model: "m1",
+      exitCode: 1,
+      failureReason: "Tool call failed: invalid model output from the summarizer",
+    });
+
+    expect(bodies).toEqual([]);
+    resetHarnessCliVersionForTests();
+  });
+
+  test("sends unsupported on a model rejection, ok once on success, nothing on other failures", async () => {
+    resetHarnessCliVersionForTests({ claude: CLI });
+    const bodies: unknown[] = [];
+    const fetchImpl = (async (_url: string, init: RequestInit) => {
+      bodies.push(JSON.parse(String(init.body)));
+      return new Response("{}", { status: 200 });
+    }) as unknown as typeof fetch;
+    const base = { apiUrl: "http://x", agentId: "a", harness: "claude", fetchImpl };
+
+    await reportHarnessModelOutcome({ ...base, model: "m1", exitCode: 1, failureReason: "boom" });
+    await reportHarnessModelOutcome({
+      ...base,
+      model: "m1",
+      exitCode: 1,
+      failureReason: "There's an issue with the selected model (m1).",
+    });
+    await reportHarnessModelOutcome({ ...base, model: "m2", exitCode: 0 });
+    await reportHarnessModelOutcome({ ...base, model: "m2", exitCode: 0 });
+
+    expect(bodies).toMatchObject([
+      { modelId: "m1", status: "unsupported", cliVersion: CLI },
+      { modelId: "m2", status: "ok" },
+    ]);
+    resetHarnessCliVersionForTests();
+  });
+
+  test("scrubs secrets from the failure text before it leaves the worker", async () => {
+    resetHarnessCliVersionForTests({ claude: CLI });
+    const token = "ghp_abcdefghijklmnopqrstuvwxyz0123456789";
+    const bodies: Array<{ error?: string }> = [];
+    const fetchImpl = (async (_url: string, init: RequestInit) => {
+      bodies.push(JSON.parse(String(init.body)));
+      return new Response("{}", { status: 200 });
+    }) as unknown as typeof fetch;
+
+    await reportHarnessModelOutcome({
+      apiUrl: "http://x",
+      agentId: "a",
+      harness: "claude",
+      fetchImpl,
+      model: "m1",
+      exitCode: 1,
+      failureReason: `There's an issue with the selected model (m1). token=${token}`,
+    });
+
+    expect(bodies).toHaveLength(1);
+    expect(bodies[0]?.error).toContain("selected model");
+    expect(bodies[0]?.error).not.toContain(token);
+    resetHarnessCliVersionForTests();
+  });
+});

package/src/tests/heartbeat-checklist-config.test.ts

@@ -0,0 +1,71 @@
+import { afterEach, describe, expect, spyOn, test } from "bun:test";
+import { startHeartbeatChecklist, stopHeartbeatChecklist } from "../heartbeat/heartbeat";
+
+// startHeartbeatChecklist reads HEARTBEAT_CHECKLIST_DISABLE and
+// HEARTBEAT_CHECKLIST_INTERVAL_MS at call time. These cases pin how each raw
+// env string is interpreted, observed through the startup log line.
+const ENV_KEYS = ["HEARTBEAT_CHECKLIST_DISABLE", "HEARTBEAT_CHECKLIST_INTERVAL_MS"] as const;
+const saved = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]]));
+
+function startWithEnv(env: Partial<Record<(typeof ENV_KEYS)[number], string>>): string[] {
+  for (const key of ENV_KEYS) {
+    if (env[key] === undefined) delete process.env[key];
+    else process.env[key] = env[key];
+  }
+  const spy = spyOn(console, "log").mockImplementation(() => {});
+  try {
+    startHeartbeatChecklist();
+    return spy.mock.calls.map((args) => String(args[0]));
+  } finally {
+    spy.mockRestore();
+  }
+}
+
+const DISABLED = "[Heartbeat] Checklist disabled via HEARTBEAT_CHECKLIST_DISABLE";
+const TICK_OFF =
+  "[Heartbeat] Recurring checklist off (HEARTBEAT_CHECKLIST_INTERVAL_MS=0); boot triage still scheduled";
+const starting = (ms: number) => `[Heartbeat] Checklist starting with ${ms}ms interval`;
+
+describe("startHeartbeatChecklist env parsing", () => {
+  afterEach(() => {
+    stopHeartbeatChecklist();
+    for (const key of ENV_KEYS) {
+      if (saved[key] === undefined) delete process.env[key];
+      else process.env[key] = saved[key];
+    }
+  });
+
+  test.each([
+    ["unset", undefined],
+    ["empty", ""],
+    ["false", "false"],
+    ["0", "0"],
+    ["FALSE", "FALSE"],
+    ["typo falls back to on", "treu"],
+  ])("HEARTBEAT_CHECKLIST_DISABLE %s keeps the checklist on", (_label, value) => {
+    expect(startWithEnv({ HEARTBEAT_CHECKLIST_DISABLE: value })).toEqual([starting(1_800_000)]);
+  });
+
+  test.each([
+    ["true"],
+    ["1"],
+    ["TRUE"],
+  ])("HEARTBEAT_CHECKLIST_DISABLE %s turns the checklist off", (value) => {
+    expect(startWithEnv({ HEARTBEAT_CHECKLIST_DISABLE: value })).toEqual([DISABLED]);
+  });
+
+  test.each([
+    ["0", [TICK_OFF]],
+    ["-5", [TICK_OFF]],
+    ["60000", [starting(60_000)]],
+    ["abc", [starting(1_800_000)]],
+    ["", [starting(1_800_000)]],
+  ])("HEARTBEAT_CHECKLIST_INTERVAL_MS=%p", (value, expected) => {
+    expect(startWithEnv({ HEARTBEAT_CHECKLIST_INTERVAL_MS: value })).toEqual(expected);
+  });
+
+  test("interval 0: a second start does not schedule boot triage again", () => {
+    expect(startWithEnv({ HEARTBEAT_CHECKLIST_INTERVAL_MS: "0" })).toEqual([TICK_OFF]);
+    expect(startWithEnv({ HEARTBEAT_CHECKLIST_INTERVAL_MS: "0" })).toEqual([]);
+  });
+});

package/src/tests/heartbeat-checklist.test.ts

@@ -290,8 +290,10 @@ describe("Heartbeat Checklist", () => {
       expect(tasks[0]!.task).toContain("≤10 items");
       expect(tasks[0]!.task).toContain("20 is the absolute max");
       expect(tasks[0]!.task).toContain("script-run");
-      expect(tasks[0]!.task).toContain("Heartbeat Audit");
-      expect(tasks[0]!.task).toContain("Rule #11");
+      expect(tasks[0]!.task).toContain("schedule-health");
+      expect(tasks[0]!.task).toContain("task-failure-audit");
+      expect(tasks[0]!.task).not.toContain("Heartbeat Audit");
+      expect(tasks[0]!.task).not.toMatch(/Rules? #\d/);
     });
 
     test("created task has correct tags", async () => {

package/src/tests/heartbeat-orphan-resume.test.ts

@@ -24,6 +24,7 @@ import {
   runRebootSweep,
   setBeforeHeartbeatSupersedeForTests,
 } from "../heartbeat/heartbeat";
+import { createResumeFollowUp } from "../tasks/worker-follow-up";
 
 const TEST_DB_PATH = "./test-heartbeat-orphan-resume.sqlite";
 
@@ -81,4 +82,51 @@ describe("Heartbeat orphaned supersede repair (TLA+ counterexample)", () => {
     expect((await getTaskById(task.id))?.status).toBe("superseded");
     expect(await getChildTasks(task.id)).toHaveLength(1);
   });
+
+  // Crash after createResumeFollowUp, before backfillSupersedeTaskResumeTaskId.
+  test("a dependent left waiting on a superseded task with a resume is re-pointed on the next sweep", async () => {
+    const agent = await createAgent({ name: "crashed-worker", isLead: false, status: "idle" });
+    const task = await createTaskExtended("Work interrupted mid-backfill", { agentId: agent.id });
+    const dependent = await createTaskExtended("Waits on the interrupted work", {
+      agentId: agent.id,
+      dependsOn: [task.id],
+    });
+    await startTask(task.id);
+
+    await supersedeTask(task.id, { reason: "crash_recovery", resumeTaskId: null });
+    const resume = await createResumeFollowUp({ parentId: task.id, reason: "crash_recovery" });
+    if (resume.kind !== "created") throw new Error(`resume not created: ${resume.kind}`);
+    await getDbClient().run("UPDATE agent_tasks SET finishedAt = ? WHERE id = ?", [
+      new Date(Date.now() - 10 * 60 * 1000).toISOString(),
+      task.id,
+    ]);
+
+    await runRebootSweep();
+    await codeLevelTriage();
+
+    const after = await getTaskById(dependent.id);
+    expect(after?.status).toBe("pending");
+    expect(after?.dependsOn).toEqual([resume.task.id]);
+    expect(await getChildTasks(task.id)).toHaveLength(1);
+  });
+
+  test("dependents of a superseded task that gets no resume are cascade-failed by the sweep", async () => {
+    const task = await createTaskExtended("Unassigned work superseded before any resume");
+    const dependent = await createTaskExtended("Waits on unassigned work", {
+      dependsOn: [task.id],
+    });
+
+    await supersedeTask(task.id, { reason: "manual", resumeTaskId: null });
+    expect((await getTaskById(dependent.id))?.status).toBe("unassigned");
+    await getDbClient().run("UPDATE agent_tasks SET finishedAt = ? WHERE id = ?", [
+      new Date(Date.now() - 10 * 60 * 1000).toISOString(),
+      task.id,
+    ]);
+
+    await codeLevelTriage();
+
+    const after = await getTaskById(dependent.id);
+    expect(after?.status).toBe("failed");
+    expect(after?.failureReason).toContain("was superseded");
+  });
 });