@desplega.ai/agent-swarm 1.158.0 → 1.160.0

package/src/tests/memory-fts-populate.test.ts

@@ -0,0 +1,139 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { unlink } from "node:fs/promises";
+import { closeDb, createAgent, getDb, getDbClient, initDb } from "../be/db";
+import {
+  FTS_DELETE_EXTRA_SQL,
+  FTS_MISSING_IDS_SQL,
+  SqliteMemoryStore,
+} from "../be/memory/providers/sqlite-store";
+
+const TEST_DB_PATH = "./test-memory-fts-populate.sqlite";
+const agentId = "aaaa0000-0000-4000-8000-000000000171";
+
+// The pre-fix backfill. Kept here as the positive control for the plan check:
+// if SQLite ever stops reporting it as CORRELATED, the assertion below would
+// pass vacuously, so the control has to fail first.
+const QUADRATIC_BACKFILL_SQL = `INSERT INTO memory_fts(memory_id, name, content)
+  SELECT m.id, m.name, m.content
+  FROM agent_memory m
+  WHERE NOT EXISTS (SELECT 1 FROM memory_fts f WHERE f.memory_id = m.id)`;
+
+// EXPLAIN goes through a prepared statement that is finalized right away: the
+// client's cached `db.query()` statement for an EXPLAIN stays "in progress" and
+// wedges the connection's next COMMIT.
+function planOf(sql: string): string[] {
+  const stmt = getDb().prepare(`EXPLAIN QUERY PLAN ${sql}`);
+  try {
+    return (stmt.all() as { detail: string }[]).map((row) => row.detail);
+  } finally {
+    stmt.finalize();
+  }
+}
+
+async function count(sql: string, params: string[] = []): Promise<number> {
+  return (await getDbClient().get<{ count: number }>(sql, params))?.count ?? -1;
+}
+
+describe("memory_fts boot backfill", () => {
+  let store: SqliteMemoryStore;
+
+  beforeAll(async () => {
+    for (const suffix of ["", "-wal", "-shm"]) {
+      try {
+        await unlink(TEST_DB_PATH + suffix);
+      } catch {}
+    }
+    initDb(TEST_DB_PATH);
+    await createAgent({ id: agentId, name: "FTS Populate Agent", isLead: false, status: "idle" });
+    store = new SqliteMemoryStore();
+    await store.whenFtsPopulated();
+  });
+
+  afterAll(async () => {
+    closeDb();
+    for (const suffix of ["", "-wal", "-shm"]) {
+      try {
+        await unlink(TEST_DB_PATH + suffix);
+      } catch {}
+    }
+  });
+
+  test("diff queries are not correlated against the UNINDEXED memory_id", async () => {
+    // Positive control: the old anti-join scans memory_fts once per memory.
+    expect(planOf(QUADRATIC_BACKFILL_SQL).join(" | ")).toContain("CORRELATED");
+
+    for (const sql of [FTS_MISSING_IDS_SQL, FTS_DELETE_EXTRA_SQL]) {
+      const plan = planOf(sql).join(" | ");
+      expect(plan).not.toContain("CORRELATED");
+      // Exactly one pass over memory_fts per statement.
+      expect(plan.match(/SCAN memory_fts/g)?.length).toBe(1);
+    }
+  });
+
+  test("a new store backfills missing rows across batches and drops orphans", async () => {
+    const total = 1050; // crosses two 500-row batch boundaries
+    const seed = await store.store({
+      agentId,
+      scope: "agent",
+      name: "fts backfill 0",
+      content: "backfill-token-0",
+      source: "manual",
+    });
+    // Clone the seed row in one statement: 1,050 store() calls outrun the
+    // default test timeout, and the backfill only reads id/name/content.
+    const columns = (
+      await getDbClient().query<{ name: string }>("PRAGMA table_info(agent_memory)")
+    ).map((col) => col.name);
+    const projection = columns
+      .map((col) => {
+        if (col === "id") return "'clone-' || n.i";
+        if (col === "name") return "'fts backfill ' || n.i";
+        if (col === "content") return "'backfill-token-' || n.i";
+        if (col === "key") return "'fts-backfill-' || n.i";
+        return `s."${col}"`;
+      })
+      .join(", ");
+    await getDbClient().run(
+      `WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < ?)
+       INSERT INTO agent_memory(${columns.map((col) => `"${col}"`).join(", ")})
+       SELECT ${projection} FROM n, agent_memory s WHERE s.id = ?`,
+      [total - 1, seed.id],
+    );
+    const ids = [seed.id];
+    expect(await count("SELECT COUNT(*) AS count FROM agent_memory")).toBe(total);
+    // Only the seed went through store(), so only it is indexed so far.
+    expect(await count("SELECT COUNT(*) AS count FROM memory_fts")).toBe(1);
+    await getDbClient().run(
+      "INSERT INTO memory_fts(memory_id, name, content) SELECT id, name, content FROM agent_memory WHERE id != ?",
+      [seed.id],
+    );
+    expect(await count("SELECT COUNT(*) AS count FROM memory_fts")).toBe(total);
+
+    // Simulate an index that lost most of its rows and gained an orphan.
+    await getDbClient().run("DELETE FROM memory_fts WHERE memory_id != ?", [ids[0] as string]);
+    await getDbClient().run(
+      "INSERT INTO memory_fts(memory_id, name, content) VALUES ('orphan-id', 'orphan', 'orphan')",
+    );
+
+    const rebuilt = new SqliteMemoryStore();
+    await rebuilt.whenFtsPopulated();
+
+    expect(await count("SELECT COUNT(*) AS count FROM memory_fts")).toBe(total);
+    expect(await count("SELECT COUNT(DISTINCT memory_id) AS count FROM memory_fts")).toBe(total);
+    expect(
+      await count("SELECT COUNT(*) AS count FROM memory_fts WHERE memory_id = 'orphan-id'"),
+    ).toBe(0);
+    expect(
+      await count("SELECT COUNT(*) AS count FROM memory_fts WHERE memory_fts MATCH ?", [
+        '"backfill-token-1049"',
+      ]),
+    ).toBe(1);
+  });
+
+  test("a steady-state boot inserts nothing", async () => {
+    const before = await count("SELECT COUNT(*) AS count FROM memory_fts");
+    const again = new SqliteMemoryStore();
+    await again.whenFtsPopulated();
+    expect(await count("SELECT COUNT(*) AS count FROM memory_fts")).toBe(before);
+  });
+});

package/src/tests/memory-http-recall-gating.test.ts

@@ -5,6 +5,9 @@ import type { IncomingMessage, ServerResponse } from "node:http";
 import { Readable } from "node:stream";
 import { closeDb, createAgent, getDbClient, initDb } from "../be/db";
 import * as realMemoryModule from "../be/memory";
+import { buildRecallQuery } from "../memory/recall-query";
+import { getTemplateDefinition } from "../prompts/registry";
+import "../tools/templates";
 import { SIMILARITY_THRESHOLD } from "../prompts/memories";
 import type { AgentMemory } from "../types";
 
@@ -72,11 +75,17 @@ function candidate(memoryFixture: AgentMemory, similarity: number) {
   };
 }
 
+let embeddedQuery = "";
+let searchedQuery = "";
+
 mock.module("../be/memory", () => ({
   getEmbeddingProvider: () => ({
     name: "test-embedding",
     dimensions: 3,
-    embed: async () => new Float32Array([1, 0, 0]),
+    embed: async (query: string) => {
+      embeddedQuery = query;
+      return new Float32Array([1, 0, 0]);
+    },
     embedBatch: async (texts: string[]) => texts.map(() => new Float32Array([1, 0, 0])),
   }),
   getMemoryStore: () => ({
@@ -104,6 +113,7 @@ mock.module("../be/memory", () => ({
       _agentId: string,
       options: import("../be/memory/types").MemorySearchOptions,
     ) => {
+      searchedQuery = options.queryText ?? "";
       if (options.queryText === "document recall") {
         return [candidate(memory, 0.95), candidate(memoryChunk, 0.9)];
       }
@@ -324,3 +334,117 @@ describe("memory HTTP recall capture gating", () => {
     expect(await countRetrievals()).toBe(0);
   });
 });
+
+function workerWrapper(event: "completed" | "failed", task = "", output = ""): string {
+  return getTemplateDefinition(`task.worker.${event}`)!.defaultBody.replace(
+    /{{(\w+)}}/g,
+    (_, key) =>
+      ({ task_desc: task, output_summary: output, failure_reason: output })[key as "task_desc"] ??
+      "",
+  );
+}
+
+describe("pre-task recall query", () => {
+  for (const event of ["completed", "failed"] as const) {
+    test(`${event} wrapper retains only task and output`, () => {
+      expect(buildRecallQuery(workerWrapper(event, "Fix memory recall", "Fixed the query"))).toBe(
+        "Fix memory recall\n\nFixed the query",
+      );
+      expect(buildRecallQuery(workerWrapper(event))).toBe("");
+    });
+  }
+
+  test("preserves output headings and embedded thread context once", () => {
+    const output =
+      "Result\n\nIMPORTANT: actual output heading\n<thread_context>Context</thread_context>";
+    expect(buildRecallQuery(workerWrapper("completed", "Task", output))).toBe(`Task\n\n${output}`);
+    expect(
+      buildRecallQuery(
+        workerWrapper(
+          "failed",
+          "Task",
+          "Reason\n\nDecide whether to reassign, retry, or handle the failure. Actual result",
+        ),
+      ),
+    ).toContain("Actual result");
+  });
+
+  test("creator instructions do not become recall content", () => {
+    for (const event of ["completed", "failed"] as const) {
+      const wrapper = workerWrapper(event).replace(
+        event === "completed" ? "Output:\n" : "Failure reason: ",
+        `${event === "completed" ? "Output:\n" : "Failure reason: "}\nAdditional instructions from the task creator:\nReview quickly\n`,
+      );
+      expect(buildRecallQuery(wrapper)).toBe("");
+    }
+  });
+
+  test("removes sibling context and preserves thread context", () => {
+    const thread = "<thread_context>Earlier user intent</thread_context>";
+    const query = `<sibling_tasks_in_progress>Unrelated task</sibling_tasks_in_progress>\n\n${workerWrapper("completed", "Fix recall", "Done")}\n\n${thread}`;
+    expect(buildRecallQuery(query)).toBe(`Fix recall\n\nDone\n\n${thread}`);
+    expect(buildRecallQuery(`User request\n${thread}`)).toBe(`User request\n${thread}`);
+    expect(buildRecallQuery("Worker task completed — partial text")).toBe(
+      "Worker task completed — partial text",
+    );
+  });
+
+  test("bounds long ASCII and multilingual queries without splitting code points", () => {
+    expect(buildRecallQuery("a".repeat(40000))).toBe("a".repeat(8191));
+    const query = buildRecallQuery("你好🙂مرحبا".repeat(10000));
+    expect(Buffer.byteLength(query)).toBeLessThanOrEqual(8191);
+    expect(query).not.toContain("�");
+    expect(Buffer.from(query).toString("utf8")).toBe(query);
+  });
+
+  test("bounds malformed near-wrappers before parsing", () => {
+    const malformed = workerWrapper("completed", "Task", '"\n\nOutput:\n'.repeat(200000)).replace(
+      '" for full details.',
+      '" missing suffix.',
+    );
+    const bounded = malformed.slice(0, 65536);
+    expect(buildRecallQuery(malformed)).toBe(buildRecallQuery(bounded));
+    expect(Buffer.byteLength(buildRecallQuery(malformed))).toBeLessThanOrEqual(8191);
+  });
+
+  test("long valid wrappers preserve content and truncate the embedding query", () => {
+    expect(buildRecallQuery(workerWrapper("completed", "Task", "a".repeat(20000)))).toBe(
+      `Task\n\n${"a".repeat(8185)}`,
+    );
+  });
+
+  test("HTTP prompt recall uses content for embedding and text search only", async () => {
+    const original = workerWrapper("completed", "Fix recall", "Done");
+    await callMemoryRoute(
+      "POST",
+      "/api/memory/search",
+      ["api", "memory", "search"],
+      { query: original },
+      { "x-memory-consumption": "prompt" },
+    );
+    expect(embeddedQuery).toBe("Fix recall\n\nDone");
+    expect(searchedQuery).toBe(embeddedQuery);
+    await callMemoryRoute("POST", "/api/memory/search", ["api", "memory", "search"], {
+      query: original,
+    });
+    expect(embeddedQuery).toBe(original);
+    expect(searchedQuery).toBe(original);
+  });
+
+  test("blank wrapper skips search and retrieval records", async () => {
+    embeddedQuery = "untouched";
+    searchedQuery = "untouched";
+    const before = await countRetrievals();
+    const response = await callMemoryRoute(
+      "POST",
+      "/api/memory/search",
+      ["api", "memory", "search"],
+      { query: workerWrapper("completed"), intent: "pre-task memory recall" },
+      { "x-memory-consumption": "prompt", "x-source-task-id": sourceTaskId },
+    );
+    expect(response.body.results).toEqual([]);
+    expect(embeddedQuery).toBe("untouched");
+    expect(searchedQuery).toBe("untouched");
+    expect(await countRetrievals()).toBe(before);
+  });
+});

package/src/tests/migration-046-budgets.test.ts

@@ -3,7 +3,7 @@ import { unlink } from "node:fs/promises";
 import { closeDb, getDbClient, initDb } from "../be/db";
 import { loadModelsDevCache } from "../be/modelsdev-cache";
 import { seedPricingFromModelsDev } from "../be/seed-pricing";
-import { CODEX_MODEL_PRICING } from "../providers/codex-models";
+import { codexModelPricingTable } from "../providers/codex-models";
 
 const TEST_DB_PATH = "./test-migration-046.sqlite";
 
@@ -133,7 +133,7 @@ describe("migration 046 — budgets and pricing", () => {
   });
 
   test("pricing seed includes every known Codex model/token class at effective_from=0", async () => {
-    const minimumCodexRows = Object.keys(CODEX_MODEL_PRICING).length * 3;
+    const minimumCodexRows = Object.keys(codexModelPricingTable()).length * 3;
 
     const seedRows = await getDbClient().get<CountRow>(
       "SELECT COUNT(*) as cnt FROM pricing WHERE provider = 'codex' AND effective_from = 0",
@@ -151,7 +151,7 @@ describe("migration 046 — budgets and pricing", () => {
     // time comes from newer effective_from rows (runtime refresh); the
     // worker-local table is advisory and agentswarm.cost.drift.usd is the
     // watchdog. This test pins seed COVERAGE, not rate sync.
-    for (const model of Object.keys(CODEX_MODEL_PRICING)) {
+    for (const model of Object.keys(codexModelPricingTable())) {
       for (const tokenClass of ["input", "cached_input", "output"] as const) {
         const row = await client.get<PricingRow>(
           "SELECT * FROM pricing WHERE provider = 'codex' AND model = ? AND token_class = ? AND effective_from = ?",

package/src/tests/model-catalog-migration.test.ts

@@ -0,0 +1,134 @@
+import { Database } from "bun:sqlite";
+import { afterEach, describe, expect, test } from "bun:test";
+import { runMigrations } from "../be/migrations/runner";
+
+const DB_PATH = "./test-model-catalog-migration.sqlite";
+
+async function removeDb(): Promise<void> {
+  for (const suffix of ["", "-wal", "-shm"]) {
+    try {
+      await Bun.file(DB_PATH + suffix).delete();
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+    }
+  }
+}
+
+afterEach(removeDb);
+
+function columnNames(db: Database, table: string): string[] {
+  return db
+    .query<{ name: string }, []>(`PRAGMA table_info(${table})`)
+    .all()
+    .map((column) => column.name);
+}
+
+function tableExists(db: Database, table: string): boolean {
+  return (
+    db
+      .query<{ name: string }, [string]>(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name = ?",
+      )
+      .get(table) !== null
+  );
+}
+
+describe("migration 184 model catalog, tier resolution and harness support", () => {
+  test("upgrades a pre-184 database in place and keeps existing agent and task rows", async () => {
+    await removeDb();
+    const db = new Database(DB_PATH, { create: true });
+    try {
+      runMigrations(db);
+
+      // Roll the database back to the state before migration 184.
+      for (const table of [
+        "model_catalog",
+        "model_catalog_overlay",
+        "model_catalog_meta",
+        "model_alias_resolutions",
+        "harness_model_support",
+      ]) {
+        db.run(`DROP TABLE ${table}`);
+      }
+      for (const column of ["resolvedModel", "modelSource", "modelAlias"]) {
+        db.run(`ALTER TABLE agent_tasks DROP COLUMN ${column}`);
+      }
+      for (const column of ["modelTierOverrides", "harnessCliVersion"]) {
+        db.run(`ALTER TABLE agents DROP COLUMN ${column}`);
+      }
+      db.run("DELETE FROM _migrations WHERE version = 184");
+
+      db.run(
+        `INSERT INTO agents (id, name, status, createdAt, lastUpdatedAt)
+         VALUES ('agent-1', 'worker', 'idle', '2026-09-01', '2026-09-01')`,
+      );
+      db.run(
+        `INSERT INTO agent_tasks (id, task, status, createdAt, lastUpdatedAt)
+         VALUES ('task-1', 'existing task', 'pending', '2026-09-01', '2026-09-01')`,
+      );
+
+      runMigrations(db);
+
+      expect(
+        db
+          .query<{ version: number }, []>("SELECT version FROM _migrations WHERE version = 184")
+          .get(),
+      ).not.toBeNull();
+      for (const table of [
+        "model_catalog",
+        "model_catalog_overlay",
+        "model_catalog_meta",
+        "model_alias_resolutions",
+        "harness_model_support",
+      ]) {
+        expect(tableExists(db, table)).toBe(true);
+      }
+      expect(columnNames(db, "agent_tasks")).toEqual(
+        expect.arrayContaining(["resolvedModel", "modelSource", "modelAlias"]),
+      );
+      expect(columnNames(db, "agents")).toEqual(
+        expect.arrayContaining(["modelTierOverrides", "harnessCliVersion"]),
+      );
+
+      const task = db
+        .query<Record<string, unknown>, []>(
+          "SELECT task, resolvedModel, modelSource, modelAlias FROM agent_tasks WHERE id = 'task-1'",
+        )
+        .get();
+      expect(task).toEqual({
+        task: "existing task",
+        resolvedModel: null,
+        modelSource: null,
+        modelAlias: null,
+      });
+      const agent = db
+        .query<Record<string, unknown>, []>(
+          "SELECT name, modelTierOverrides, harnessCliVersion FROM agents WHERE id = 'agent-1'",
+        )
+        .get();
+      expect(agent).toEqual({ name: "worker", modelTierOverrides: null, harnessCliVersion: null });
+    } finally {
+      db.close();
+    }
+  });
+
+  test("harness_model_support only accepts ok, unsupported and unknown", async () => {
+    await removeDb();
+    const db = new Database(DB_PATH, { create: true });
+    try {
+      runMigrations(db);
+      const insert = (status: string) =>
+        db.run(
+          `INSERT INTO harness_model_support (harness, cliVersion, modelId, status, checkedAt)
+           VALUES ('claude', '2.1.0', ?, ?, 1)`,
+          [`m-${status}`, status],
+        );
+      expect(() => insert("ok")).not.toThrow();
+      expect(() => insert("unsupported")).not.toThrow();
+      expect(() => insert("unknown")).not.toThrow();
+      expect(() => insert("maybe")).toThrow(/CHECK/);
+    } finally {
+      db.close();
+    }
+  });
+});