@desplega.ai/agent-swarm 1.158.0 → 1.159.0

package/src/tests/model-catalog-migration.test.ts

@@ -0,0 +1,134 @@
+import { Database } from "bun:sqlite";
+import { afterEach, describe, expect, test } from "bun:test";
+import { runMigrations } from "../be/migrations/runner";
+
+const DB_PATH = "./test-model-catalog-migration.sqlite";
+
+async function removeDb(): Promise<void> {
+  for (const suffix of ["", "-wal", "-shm"]) {
+    try {
+      await Bun.file(DB_PATH + suffix).delete();
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+    }
+  }
+}
+
+afterEach(removeDb);
+
+function columnNames(db: Database, table: string): string[] {
+  return db
+    .query<{ name: string }, []>(`PRAGMA table_info(${table})`)
+    .all()
+    .map((column) => column.name);
+}
+
+function tableExists(db: Database, table: string): boolean {
+  return (
+    db
+      .query<{ name: string }, [string]>(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name = ?",
+      )
+      .get(table) !== null
+  );
+}
+
+describe("migration 184 model catalog, tier resolution and harness support", () => {
+  test("upgrades a pre-184 database in place and keeps existing agent and task rows", async () => {
+    await removeDb();
+    const db = new Database(DB_PATH, { create: true });
+    try {
+      runMigrations(db);
+
+      // Roll the database back to the state before migration 184.
+      for (const table of [
+        "model_catalog",
+        "model_catalog_overlay",
+        "model_catalog_meta",
+        "model_alias_resolutions",
+        "harness_model_support",
+      ]) {
+        db.run(`DROP TABLE ${table}`);
+      }
+      for (const column of ["resolvedModel", "modelSource", "modelAlias"]) {
+        db.run(`ALTER TABLE agent_tasks DROP COLUMN ${column}`);
+      }
+      for (const column of ["modelTierOverrides", "harnessCliVersion"]) {
+        db.run(`ALTER TABLE agents DROP COLUMN ${column}`);
+      }
+      db.run("DELETE FROM _migrations WHERE version = 184");
+
+      db.run(
+        `INSERT INTO agents (id, name, status, createdAt, lastUpdatedAt)
+         VALUES ('agent-1', 'worker', 'idle', '2026-09-01', '2026-09-01')`,
+      );
+      db.run(
+        `INSERT INTO agent_tasks (id, task, status, createdAt, lastUpdatedAt)
+         VALUES ('task-1', 'existing task', 'pending', '2026-09-01', '2026-09-01')`,
+      );
+
+      runMigrations(db);
+
+      expect(
+        db
+          .query<{ version: number }, []>("SELECT version FROM _migrations WHERE version = 184")
+          .get(),
+      ).not.toBeNull();
+      for (const table of [
+        "model_catalog",
+        "model_catalog_overlay",
+        "model_catalog_meta",
+        "model_alias_resolutions",
+        "harness_model_support",
+      ]) {
+        expect(tableExists(db, table)).toBe(true);
+      }
+      expect(columnNames(db, "agent_tasks")).toEqual(
+        expect.arrayContaining(["resolvedModel", "modelSource", "modelAlias"]),
+      );
+      expect(columnNames(db, "agents")).toEqual(
+        expect.arrayContaining(["modelTierOverrides", "harnessCliVersion"]),
+      );
+
+      const task = db
+        .query<Record<string, unknown>, []>(
+          "SELECT task, resolvedModel, modelSource, modelAlias FROM agent_tasks WHERE id = 'task-1'",
+        )
+        .get();
+      expect(task).toEqual({
+        task: "existing task",
+        resolvedModel: null,
+        modelSource: null,
+        modelAlias: null,
+      });
+      const agent = db
+        .query<Record<string, unknown>, []>(
+          "SELECT name, modelTierOverrides, harnessCliVersion FROM agents WHERE id = 'agent-1'",
+        )
+        .get();
+      expect(agent).toEqual({ name: "worker", modelTierOverrides: null, harnessCliVersion: null });
+    } finally {
+      db.close();
+    }
+  });
+
+  test("harness_model_support only accepts ok, unsupported and unknown", async () => {
+    await removeDb();
+    const db = new Database(DB_PATH, { create: true });
+    try {
+      runMigrations(db);
+      const insert = (status: string) =>
+        db.run(
+          `INSERT INTO harness_model_support (harness, cliVersion, modelId, status, checkedAt)
+           VALUES ('claude', '2.1.0', ?, ?, 1)`,
+          [`m-${status}`, status],
+        );
+      expect(() => insert("ok")).not.toThrow();
+      expect(() => insert("unsupported")).not.toThrow();
+      expect(() => insert("unknown")).not.toThrow();
+      expect(() => insert("maybe")).toThrow(/CHECK/);
+    } finally {
+      db.close();
+    }
+  });
+});

package/src/tests/model-catalog-table.test.ts

@@ -0,0 +1,249 @@
+import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
+import { unlink } from "node:fs/promises";
+import { closeDb, getActivePricingRow, getDbClient, initDb } from "../be/db";
+import {
+  listModelCatalogOverlay,
+  loadModelsCatalog,
+  markModelsCatalogStale,
+  upsertModelCatalogOverlay,
+} from "../be/model-catalog-store";
+import { getModelsCatalog, resetModelsCatalogForTests } from "../be/models-catalog";
+import type { ModelsDevCache } from "../be/modelsdev-cache";
+import { MODEL_CATALOG_FRESH_MS, refreshModelCatalog } from "../be/pricing-refresh";
+
+const TEST_DB_PATH = "./test-model-catalog-table.sqlite";
+
+async function removeDbFiles(path: string): Promise<void> {
+  for (const suffix of ["", "-wal", "-shm"]) {
+    try {
+      await unlink(path + suffix);
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+    }
+  }
+}
+
+function payload(extraModels: Record<string, unknown> = {}): ModelsDevCache {
+  return {
+    openai: {
+      name: "OpenAI",
+      models: {
+        "gpt-base": {
+          name: "GPT Base",
+          cost: { input: 1, output: 4 },
+          limit: { context: 100_000 },
+        },
+        ...extraModels,
+      },
+    },
+  } as ModelsDevCache;
+}
+
+function ok(cache: ModelsDevCache, etag: string): typeof fetch {
+  return (async () =>
+    new Response(JSON.stringify(cache), {
+      status: 200,
+      headers: { "content-type": "application/json", etag },
+    })) as unknown as typeof fetch;
+}
+
+beforeAll(async () => {
+  await removeDbFiles(TEST_DB_PATH);
+  initDb(TEST_DB_PATH);
+  // Module state may carry over from other files sharing this process.
+  resetModelsCatalogForTests();
+  markModelsCatalogStale();
+});
+
+afterAll(async () => {
+  closeDb();
+  await removeDbFiles(TEST_DB_PATH);
+});
+
+afterEach(async () => {
+  const client = getDbClient();
+  await client.run("DELETE FROM model_catalog");
+  await client.run("DELETE FROM model_catalog_overlay");
+  await client.run("DELETE FROM model_catalog_meta");
+  await client.run("DELETE FROM pricing WHERE model LIKE 'gpt-%'");
+  resetModelsCatalogForTests();
+  markModelsCatalogStale();
+});
+
+describe("persistent model catalog", () => {
+  test("empty table serves the vendored snapshot", async () => {
+    const result = await loadModelsCatalog();
+    expect(result.source).toBe("snapshot");
+    expect(result.updatedAt).toBeNull();
+  });
+
+  test("forced refresh adds a new model to the catalog and pricing without restart", async () => {
+    await refreshModelCatalog({ force: true, now: 1_000, fetchImpl: ok(payload(), '"v1"') });
+
+    const result = await refreshModelCatalog({
+      force: true,
+      now: 2_000,
+      fetchImpl: ok(
+        payload({ "gpt-brand-new": { name: "Brand New", cost: { input: 3, output: 9 } } }),
+        '"v2"',
+      ),
+    });
+
+    expect(result.status).toBe("updated");
+    expect(result.added).toEqual(["openai/gpt-brand-new"]);
+    expect(result.models).toBe(2);
+    const catalog = await loadModelsCatalog();
+    expect(catalog.source).toBe("live");
+    expect(catalog.providers.openai?.models["gpt-brand-new"]?.cost?.input).toBe(3);
+    const price = await getActivePricingRow("codex", "gpt-brand-new", "input", 2_000);
+    expect(price?.pricePerMillionUsd).toBe(3);
+  });
+
+  test("304 keeps the catalog and bumps checkedAt", async () => {
+    await refreshModelCatalog({ force: true, now: 1_000, fetchImpl: ok(payload(), '"v1"') });
+    let sent: string | null = null;
+    const result = await refreshModelCatalog({
+      force: true,
+      now: 5_000,
+      fetchImpl: (async (_url: unknown, init?: RequestInit) => {
+        sent = new Headers(init?.headers).get("if-none-match");
+        return new Response(null, { status: 304 });
+      }) as unknown as typeof fetch,
+    });
+    expect(sent).toBe('"v1"');
+    expect(result.status).toBe("not-modified");
+    expect(result.models).toBe(1);
+    expect(result.checkedAt).toBe(5_000);
+    const row = await getDbClient().get<{ checkedAt: number }>(
+      "SELECT checkedAt FROM model_catalog WHERE modelId = 'gpt-base'",
+    );
+    expect(row?.checkedAt).toBe(5_000);
+    expect(getModelsCatalog().providers.openai?.models["gpt-base"]).toBeDefined();
+  });
+
+  test("unforced refresh within 4h skips the network", async () => {
+    await refreshModelCatalog({ now: 1_000, fetchImpl: ok(payload(), '"v1"') });
+    let called = false;
+    const fetchImpl = (async () => {
+      called = true;
+      return new Response(null, { status: 304 });
+    }) as unknown as typeof fetch;
+    const skipped = await refreshModelCatalog({ now: 1_000 + 60_000, fetchImpl });
+    expect(skipped.status).toBe("skipped-fresh");
+    expect(called).toBe(false);
+
+    const stale = await refreshModelCatalog({ now: 1_000 + MODEL_CATALOG_FRESH_MS + 1, fetchImpl });
+    expect(stale.status).toBe("not-modified");
+    expect(called).toBe(true);
+  });
+
+  test("overlay-only model appears in the catalog and gets a pricing row", async () => {
+    await refreshModelCatalog({ force: true, now: 1_000, fetchImpl: ok(payload(), '"v1"') });
+    await upsertModelCatalogOverlay(
+      {
+        provider: "openai",
+        modelId: "gpt-overlay-only",
+        name: "Overlay Only",
+        contextWindow: 400_000,
+        pricing: { input: 7, output: 21 },
+        reason: "launch blog",
+      },
+      { now: 2_000 },
+    );
+    const catalog = await loadModelsCatalog();
+    const model = catalog.providers.openai?.models["gpt-overlay-only"];
+    expect(model?.name).toBe("Overlay Only");
+    expect(model?.limit?.context).toBe(400_000);
+    const price = await getActivePricingRow("codex", "gpt-overlay-only", "output", 2_000);
+    expect(price?.pricePerMillionUsd).toBe(21);
+  });
+
+  test("a corrected overlay-only price supersedes the earlier pricing row", async () => {
+    await refreshModelCatalog({ force: true, now: 1_000, fetchImpl: ok(payload(), '"v1"') });
+    const overlay = {
+      provider: "openai",
+      modelId: "gpt-overlay-only",
+      pricing: { input: 7, output: 21 },
+      reason: "launch blog",
+    };
+    await upsertModelCatalogOverlay(overlay, { now: 2_000 });
+    await upsertModelCatalogOverlay(
+      { ...overlay, pricing: { input: 7, output: 28 } },
+      { now: 3_000 },
+    );
+    expect(
+      (await getActivePricingRow("codex", "gpt-overlay-only", "output", 2_500))?.pricePerMillionUsd,
+    ).toBe(21);
+    expect(
+      (await getActivePricingRow("codex", "gpt-overlay-only", "output", 3_000))?.pricePerMillionUsd,
+    ).toBe(28);
+    expect(
+      (await getActivePricingRow("codex", "gpt-overlay-only", "input", 3_000))?.effectiveFrom,
+    ).toBe(2_000);
+  });
+
+  test("an overlay price never overrides an upstream-priced model", async () => {
+    await refreshModelCatalog({ force: true, now: 1_000, fetchImpl: ok(payload(), '"v1"') });
+    const before = await getActivePricingRow("codex", "gpt-base", "output", 1_000);
+    expect(before).not.toBeNull();
+    await upsertModelCatalogOverlay(
+      {
+        provider: "openai",
+        modelId: "gpt-base",
+        pricing: { input: 99, output: 999 },
+        reason: "upstream wrong",
+      },
+      { now: 2_000 },
+    );
+    const after = await getActivePricingRow("codex", "gpt-base", "output", 2_000);
+    expect(after?.pricePerMillionUsd).toBe(before?.pricePerMillionUsd);
+  });
+
+  test("overlay wins per field over upstream", async () => {
+    await refreshModelCatalog({ force: true, now: 1_000, fetchImpl: ok(payload(), '"v1"') });
+    await upsertModelCatalogOverlay({
+      provider: "openai",
+      modelId: "gpt-base",
+      contextWindow: 200_000,
+      reason: "upstream wrong",
+    });
+    const model = (await loadModelsCatalog()).providers.openai?.models["gpt-base"];
+    expect(model?.limit?.context).toBe(200_000);
+    expect(model?.name).toBe("GPT Base");
+  });
+
+  test("overlay auto-expires once upstream matches", async () => {
+    await refreshModelCatalog({ force: true, now: 1_000, fetchImpl: ok(payload(), '"v1"') });
+    await upsertModelCatalogOverlay({
+      provider: "openai",
+      modelId: "gpt-late",
+      name: "Late",
+      pricing: { input: 2 },
+      reason: "not on models.dev yet",
+    });
+    await upsertModelCatalogOverlay({
+      provider: "openai",
+      modelId: "gpt-pinned",
+      name: "Pinned",
+      reason: "keep",
+      expiresWhenUpstreamMatches: false,
+    });
+
+    await refreshModelCatalog({
+      force: true,
+      now: 2_000,
+      fetchImpl: ok(
+        payload({
+          "gpt-late": { name: "Late", cost: { input: 2, output: 8 } },
+          "gpt-pinned": { name: "Pinned" },
+        }),
+        '"v2"',
+      ),
+    });
+
+    const remaining = (await listModelCatalogOverlay()).map((o) => o.modelId);
+    expect(remaining).toEqual(["gpt-pinned"]);
+    const late = (await loadModelsCatalog()).providers.openai?.models["gpt-late"];
+    expect(late?.cost?.output).toBe(8);
+  });
+});