@desplega.ai/agent-swarm 1.64.0 → 1.65.0

package/src/http/memory.ts

@@ -1,14 +1,9 @@
 import type { IncomingMessage, ServerResponse } from "node:http";
 import { z } from "zod";
 import { chunkContent } from "../be/chunking";
-import {
-  createMemory,
-  deleteMemoriesBySourcePath,
-  getDb,
-  searchMemoriesByVector,
-  updateMemoryEmbedding,
-} from "../be/db";
-import { getEmbedding, serializeEmbedding } from "../be/embedding";
+import { getEmbeddingProvider, getMemoryStore } from "../be/memory";
+import { CANDIDATE_SET_MULTIPLIER } from "../be/memory/constants";
+import { rerank } from "../be/memory/reranker";
 import { AgentMemoryScopeSchema, AgentMemorySourceSchema } from "../types";
 import { route } from "./route-def";
 import { json, jsonError } from "./utils";
@@ -54,6 +49,26 @@ const searchMemory = route({
   },
 });
 
+const reEmbedMemory = route({
+  method: "post",
+  path: "/api/memory/re-embed",
+  pattern: ["api", "memory", "re-embed"],
+  summary: "Re-embed all memories using the current embedding provider",
+  tags: ["Memory"],
+  auth: { apiKey: true },
+  body: z.object({
+    agentId: z
+      .string()
+      .uuid()
+      .optional()
+      .describe("Re-embed only this agent's memories. Omit for all."),
+    batchSize: z.number().int().min(1).max(100).default(20).describe("Memories per batch"),
+  }),
+  responses: {
+    202: { description: "Re-embedding started" },
+  },
+});
+
 // ─── Handler ─────────────────────────────────────────────────────────────────
 
 export async function handleMemory(
@@ -68,10 +83,9 @@ export async function handleMemory(
 
     const { agentId, content, name, scope, source, sourceTaskId, sourcePath, tags } = parsed.body;
 
-    // Chunk content and create memories in a transaction (with dedup)
+    // Chunk content and create memories
     const contentChunks = chunkContent(content);
     if (contentChunks.length === 0) {
-      // Content too small to chunk — create a single memory
       contentChunks.push({
         content: content.trim(),
         chunkIndex: 0,
@@ -80,46 +94,45 @@ export async function handleMemory(
       });
     }
 
-    const memoryIds = getDb().transaction(() => {
-      // Delete old chunks if re-indexing same file
-      if (sourcePath && agentId) {
-        deleteMemoriesBySourcePath(sourcePath, agentId);
-      }
+    const store = getMemoryStore();
+    const provider = getEmbeddingProvider();
 
-      const ids: string[] = [];
-      for (const chunk of contentChunks) {
-        const memory = createMemory({
-          agentId: agentId || null,
-          content: chunk.content,
-          name,
-          scope,
-          source,
-          sourcePath: sourcePath || null,
-          sourceTaskId: sourceTaskId || null,
-          chunkIndex: chunk.chunkIndex,
-          totalChunks: chunk.totalChunks,
-          tags: tags || [],
-        });
-        ids.push(memory.id);
-      }
-      return ids;
-    })();
+    // Dedup — delete old chunks for this source path
+    if (sourcePath && agentId) {
+      store.deleteBySourcePath(sourcePath, agentId);
+    }
 
-    // Async embedding — fire and forget
+    // Atomic batch insert — all chunks or none
+    const memories = store.storeBatch(
+      contentChunks.map((chunk) => ({
+        agentId: agentId || null,
+        content: chunk.content,
+        name,
+        scope,
+        source,
+        sourcePath: sourcePath || null,
+        sourceTaskId: sourceTaskId || null,
+        chunkIndex: chunk.chunkIndex,
+        totalChunks: chunk.totalChunks,
+        tags: tags || [],
+      })),
+    );
+
+    // Async batch embed (fire and forget)
     (async () => {
-      for (let i = 0; i < contentChunks.length; i++) {
-        try {
-          const embedding = await getEmbedding(contentChunks[i]!.content);
-          if (embedding) {
-            updateMemoryEmbedding(memoryIds[i]!, serializeEmbedding(embedding));
+      try {
+        const embeddings = await provider.embedBatch(contentChunks.map((c) => c.content));
+        for (let i = 0; i < embeddings.length; i++) {
+          if (embeddings[i]) {
+            store.updateEmbedding(memories[i]!.id, embeddings[i]!, provider.name);
           }
-        } catch (err) {
-          console.error(`[memory] Failed to embed chunk ${memoryIds[i]}:`, (err as Error).message);
         }
+      } catch (err) {
+        console.error("[memory] Batch embedding failed:", (err as Error).message);
       }
     })();
 
-    json(res, { queued: true, memoryIds }, 202);
+    json(res, { queued: true, memoryIds: memories.map((m) => m.id) }, 202);
     return true;
   }
 
@@ -135,20 +148,25 @@ export async function handleMemory(
     const { query, limit } = parsed.body;
 
     try {
-      const queryEmbedding = await getEmbedding(query);
+      const provider = getEmbeddingProvider();
+      const store = getMemoryStore();
+      const queryEmbedding = await provider.embed(query);
+
       if (!queryEmbedding) {
         json(res, { results: [] });
         return true;
       }
 
-      const results = searchMemoriesByVector(queryEmbedding, myAgentId, {
+      const candidateLimit = Math.min(limit, 20) * CANDIDATE_SET_MULTIPLIER;
+      const candidates = store.search(queryEmbedding, myAgentId, {
         scope: "all",
-        limit: Math.min(limit, 20),
+        limit: candidateLimit,
         isLead: false,
       });
+      const ranked = rerank(candidates, { limit: Math.min(limit, 20) });
 
       json(res, {
-        results: results.map((r) => ({
+        results: ranked.map((r) => ({
           id: r.id,
           name: r.name,
           content: r.content,
@@ -164,5 +182,40 @@ export async function handleMemory(
     return true;
   }
 
+  if (reEmbedMemory.match(req.method, pathSegments)) {
+    const parsed = await reEmbedMemory.parse(req, res, pathSegments, new URLSearchParams());
+    if (!parsed) return true;
+
+    const { agentId, batchSize } = parsed.body;
+    const store = getMemoryStore();
+    const provider = getEmbeddingProvider();
+    const memories = store.listForReembedding(agentId ? { agentId } : undefined);
+
+    json(res, { started: true, totalMemories: memories.length }, 202);
+
+    // Async re-embed in batches
+    (async () => {
+      for (let i = 0; i < memories.length; i += batchSize) {
+        const batch = memories.slice(i, i + batchSize);
+        try {
+          const embeddings = await provider.embedBatch(batch.map((m) => m.content));
+          for (let j = 0; j < embeddings.length; j++) {
+            if (embeddings[j]) {
+              store.updateEmbedding(batch[j]!.id, embeddings[j]!, provider.name);
+            }
+          }
+          console.log(
+            `[memory] Re-embedded batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(memories.length / batchSize)}`,
+          );
+        } catch (err) {
+          console.error("[memory] Re-embed batch failed:", (err as Error).message);
+        }
+      }
+      console.log(`[memory] Re-embedding complete: ${memories.length} memories`);
+    })();
+
+    return true;
+  }
+
   return false;
 }

package/src/server.ts

@@ -27,6 +27,7 @@ import {
   registerMcpServerUpdateTool,
 } from "./tools/mcp-servers";
 // Memory capability
+import { registerMemoryDeleteTool } from "./tools/memory-delete";
 import { registerMemoryGetTool } from "./tools/memory-get";
 import { registerMemorySearchTool } from "./tools/memory-search";
 import { registerMyAgentInfoTool } from "./tools/my-agent-info";
@@ -237,6 +238,7 @@ export function createServer() {
   if (hasCapability("memory")) {
     registerMemorySearchTool(server);
     registerMemoryGetTool(server);
+    registerMemoryDeleteTool(server);
     registerInjectLearningTool(server);
   }
 

package/src/tests/artifact-sdk.test.ts

@@ -47,7 +47,8 @@ describe("getAvailablePort", () => {
     for (let i = 0; i < 10; i++) {
       ports.add(await getAvailablePort());
     }
-    expect(ports.size).toBe(10);
+    // Allow minor collisions — OS can recycle ports between bind/release cycles
+    expect(ports.size).toBeGreaterThanOrEqual(8);
   });
 });
 

package/src/tests/memory-e2e.test.ts

@@ -0,0 +1,453 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { unlink } from "node:fs/promises";
+import { closeDb, createAgent, getDb, initDb } from "../be/db";
+import { SqliteMemoryStore } from "../be/memory/providers/sqlite-store";
+import { rerank } from "../be/memory/reranker";
+
+const TEST_DB_PATH = "./test-memory-e2e.sqlite";
+
+describe("Memory E2E Lifecycle", () => {
+  const agentA = "aaaa0000-0000-4000-8000-000000000e01";
+  const agentB = "bbbb0000-0000-4000-8000-000000000e02";
+  let store: SqliteMemoryStore;
+
+  beforeAll(async () => {
+    for (const suffix of ["", "-wal", "-shm"]) {
+      try {
+        await unlink(TEST_DB_PATH + suffix);
+      } catch {}
+    }
+    initDb(TEST_DB_PATH);
+    createAgent({ id: agentA, name: "E2E Agent A", isLead: false, status: "idle" });
+    createAgent({ id: agentB, name: "E2E Agent B", isLead: true, status: "idle" });
+    store = new SqliteMemoryStore();
+  });
+
+  afterAll(async () => {
+    closeDb();
+    for (const suffix of ["", "-wal", "-shm"]) {
+      try {
+        await unlink(TEST_DB_PATH + suffix);
+      } catch {}
+    }
+  });
+
+  // ==========================================================================
+  // Full lifecycle: store → search → get → delete
+  // ==========================================================================
+
+  describe("store → search → get → delete lifecycle", () => {
+    let memoryId: string;
+
+    test("store creates a memory with correct fields", () => {
+      const memory = store.store({
+        agentId: agentA,
+        scope: "agent",
+        name: "deployment info",
+        content: "The deployment pipeline uses GitHub Actions with staging on Fly.io",
+        source: "manual",
+        tags: ["deployment", "ci"],
+      });
+      memoryId = memory.id;
+      expect(memory.id).toBeDefined();
+      expect(memory.agentId).toBe(agentA);
+      expect(memory.content).toContain("GitHub Actions");
+    });
+
+    test("updateEmbedding stores vector and model name", () => {
+      const embedding = new Float32Array([0.8, 0.2, 0.1]);
+      store.updateEmbedding(memoryId, embedding, "test-model-v1");
+
+      // Verify via raw SQL
+      const row = getDb()
+        .prepare("SELECT embeddingModel FROM agent_memory WHERE id = ?")
+        .get(memoryId) as { embeddingModel: string | null };
+      expect(row.embeddingModel).toBe("test-model-v1");
+    });
+
+    test("search returns the memory with similarity score", () => {
+      const query = new Float32Array([0.8, 0.2, 0.1]); // same as stored
+      const results = store.search(query, agentA, { limit: 5 });
+      expect(results.length).toBeGreaterThan(0);
+
+      const found = results.find((r) => r.id === memoryId);
+      expect(found).toBeDefined();
+      expect(found!.similarity).toBeCloseTo(1.0, 3);
+    });
+
+    test("get retrieves memory and increments accessCount", () => {
+      const mem1 = store.get(memoryId);
+      expect(mem1).not.toBeNull();
+      expect(mem1!.name).toBe("deployment info");
+
+      // Access count should increment on each get
+      const row1 = getDb()
+        .prepare("SELECT accessCount FROM agent_memory WHERE id = ?")
+        .get(memoryId) as { accessCount: number };
+      const count1 = row1.accessCount;
+
+      store.get(memoryId);
+      const row2 = getDb()
+        .prepare("SELECT accessCount FROM agent_memory WHERE id = ?")
+        .get(memoryId) as { accessCount: number };
+      expect(row2.accessCount).toBe(count1 + 1);
+    });
+
+    test("peek reads without incrementing accessCount", () => {
+      const rowBefore = getDb()
+        .prepare("SELECT accessCount FROM agent_memory WHERE id = ?")
+        .get(memoryId) as { accessCount: number };
+
+      const mem = store.peek(memoryId);
+      expect(mem).not.toBeNull();
+      expect(mem!.name).toBe("deployment info");
+
+      const rowAfter = getDb()
+        .prepare("SELECT accessCount FROM agent_memory WHERE id = ?")
+        .get(memoryId) as { accessCount: number };
+      expect(rowAfter.accessCount).toBe(rowBefore.accessCount);
+    });
+
+    test("delete removes the memory", () => {
+      const deleted = store.delete(memoryId);
+      expect(deleted).toBe(true);
+
+      const found = store.get(memoryId);
+      expect(found).toBeNull();
+    });
+  });
+
+  // ==========================================================================
+  // Reranking: newer memories rank higher with similar embeddings
+  // ==========================================================================
+
+  describe("reranking affects result order", () => {
+    test("newer memory with same embedding ranks higher", () => {
+      // Create old memory
+      const old = store.store({
+        agentId: agentA,
+        scope: "agent",
+        name: "old knowledge",
+        content: "Old deployment docs",
+        source: "manual",
+      });
+      store.updateEmbedding(old.id, new Float32Array([0.5, 0.5, 0.0]), "test-model");
+
+      // Backdate the old memory's createdAt
+      getDb()
+        .prepare("UPDATE agent_memory SET createdAt = datetime('now', '-30 days') WHERE id = ?")
+        .run(old.id);
+
+      // Create fresh memory with similar embedding
+      const fresh = store.store({
+        agentId: agentA,
+        scope: "agent",
+        name: "fresh knowledge",
+        content: "New deployment docs",
+        source: "manual",
+      });
+      store.updateEmbedding(fresh.id, new Float32Array([0.5, 0.5, 0.0]), "test-model");
+
+      // Search with matching query
+      const candidates = store.search(new Float32Array([0.5, 0.5, 0.0]), agentA, {
+        limit: 20,
+      });
+      const ranked = rerank(candidates, { limit: 10 });
+
+      const freshIdx = ranked.findIndex((r) => r.id === fresh.id);
+      const oldIdx = ranked.findIndex((r) => r.id === old.id);
+
+      expect(freshIdx).toBeGreaterThanOrEqual(0);
+      expect(oldIdx).toBeGreaterThanOrEqual(0);
+      expect(freshIdx).toBeLessThan(oldIdx); // fresh ranks higher
+
+      // Cleanup
+      store.delete(old.id);
+      store.delete(fresh.id);
+    });
+  });
+
+  // ==========================================================================
+  // TTL expiry filtering
+  // ==========================================================================
+
+  describe("TTL expiry filtering", () => {
+    test("task_completion has ~7d TTL", () => {
+      const memory = store.store({
+        agentId: agentA,
+        scope: "agent",
+        name: "task result",
+        content: "Completed task output",
+        source: "task_completion",
+      });
+
+      const row = getDb()
+        .prepare("SELECT expiresAt FROM agent_memory WHERE id = ?")
+        .get(memory.id) as { expiresAt: string | null };
+      expect(row.expiresAt).not.toBeNull();
+
+      const expiresAt = new Date(row.expiresAt!).getTime();
+      const expectedMin = Date.now() + 6 * 24 * 60 * 60 * 1000; // ~6d
+      const expectedMax = Date.now() + 8 * 24 * 60 * 60 * 1000; // ~8d
+      expect(expiresAt).toBeGreaterThan(expectedMin);
+      expect(expiresAt).toBeLessThan(expectedMax);
+
+      store.delete(memory.id);
+    });
+
+    test("manual source has no TTL (null expiresAt)", () => {
+      const memory = store.store({
+        agentId: agentA,
+        scope: "agent",
+        name: "permanent note",
+        content: "This should never expire",
+        source: "manual",
+      });
+
+      const row = getDb()
+        .prepare("SELECT expiresAt FROM agent_memory WHERE id = ?")
+        .get(memory.id) as { expiresAt: string | null };
+      expect(row.expiresAt).toBeNull();
+
+      store.delete(memory.id);
+    });
+
+    test("expired memories are excluded from search", () => {
+      const memory = store.store({
+        agentId: agentA,
+        scope: "agent",
+        name: "expired item",
+        content: "This should be hidden from search",
+        source: "session_summary",
+      });
+      store.updateEmbedding(memory.id, new Float32Array([0.9, 0.1, 0.0]), "test-model");
+
+      // Force expiry by backdating expiresAt
+      getDb()
+        .prepare("UPDATE agent_memory SET expiresAt = datetime('now', '-1 day') WHERE id = ?")
+        .run(memory.id);
+
+      // Search should NOT return expired memory
+      const results = store.search(new Float32Array([0.9, 0.1, 0.0]), agentA, { limit: 20 });
+      const found = results.find((r) => r.id === memory.id);
+      expect(found).toBeUndefined();
+
+      // But get() still returns it (lazy expiry — no hard delete)
+      const direct = store.get(memory.id);
+      expect(direct).not.toBeNull();
+      expect(direct!.name).toBe("expired item");
+
+      // includeExpired: true should return it in search
+      const withExpired = store.search(new Float32Array([0.9, 0.1, 0.0]), agentA, {
+        limit: 20,
+        includeExpired: true,
+      });
+      const foundExpired = withExpired.find((r) => r.id === memory.id);
+      expect(foundExpired).toBeDefined();
+
+      store.delete(memory.id);
+    });
+  });
+
+  // ==========================================================================
+  // Batch operations
+  // ==========================================================================
+
+  describe("storeBatch atomicity", () => {
+    test("stores multiple chunks atomically", () => {
+      const chunks = [
+        { content: "Chunk 0 content", chunkIndex: 0, totalChunks: 3 },
+        { content: "Chunk 1 content", chunkIndex: 1, totalChunks: 3 },
+        { content: "Chunk 2 content", chunkIndex: 2, totalChunks: 3 },
+      ];
+
+      const memories = store.storeBatch(
+        chunks.map((c) => ({
+          agentId: agentA,
+          scope: "agent" as const,
+          name: "batch-test",
+          content: c.content,
+          source: "file_index" as const,
+          sourcePath: "/test/batch.md",
+          chunkIndex: c.chunkIndex,
+          totalChunks: c.totalChunks,
+        })),
+      );
+
+      expect(memories.length).toBe(3);
+      for (let i = 0; i < 3; i++) {
+        expect(memories[i].chunkIndex).toBe(i);
+        expect(memories[i].totalChunks).toBe(3);
+      }
+
+      // Cleanup
+      store.deleteBySourcePath("/test/batch.md", agentA);
+    });
+  });
+
+  // ==========================================================================
+  // Scope visibility
+  // ==========================================================================
+
+  describe("scope visibility rules", () => {
+    let agentMemId: string;
+    let swarmMemId: string;
+    let otherAgentMemId: string;
+
+    beforeAll(() => {
+      const m1 = store.store({
+        agentId: agentA,
+        scope: "agent",
+        name: "A's private",
+        content: "Agent A only",
+        source: "manual",
+      });
+      store.updateEmbedding(m1.id, new Float32Array([1, 0, 0]), "test-model");
+      agentMemId = m1.id;
+
+      const m2 = store.store({
+        agentId: agentA,
+        scope: "swarm",
+        name: "shared knowledge",
+        content: "Visible to all",
+        source: "manual",
+      });
+      store.updateEmbedding(m2.id, new Float32Array([0, 1, 0]), "test-model");
+      swarmMemId = m2.id;
+
+      const m3 = store.store({
+        agentId: agentB,
+        scope: "agent",
+        name: "B's private",
+        content: "Agent B only",
+        source: "manual",
+      });
+      store.updateEmbedding(m3.id, new Float32Array([0, 0, 1]), "test-model");
+      otherAgentMemId = m3.id;
+    });
+
+    afterAll(() => {
+      store.delete(agentMemId);
+      store.delete(swarmMemId);
+      store.delete(otherAgentMemId);
+    });
+
+    test("worker sees own agent-scoped + swarm memories", () => {
+      const results = store.search(new Float32Array([1, 1, 1]), agentA, { isLead: false });
+      const names = results.map((r) => r.name);
+      expect(names).toContain("A's private");
+      expect(names).toContain("shared knowledge");
+      expect(names).not.toContain("B's private");
+    });
+
+    test("lead sees all memories", () => {
+      const results = store.search(new Float32Array([1, 1, 1]), agentA, { isLead: true });
+      const names = results.map((r) => r.name);
+      expect(names).toContain("A's private");
+      expect(names).toContain("shared knowledge");
+      expect(names).toContain("B's private");
+    });
+  });
+
+  // ==========================================================================
+  // Stats including expired count
+  // ==========================================================================
+
+  describe("getStats includes expired count", () => {
+    test("reports expired memories", () => {
+      const mem = store.store({
+        agentId: agentA,
+        scope: "agent",
+        name: "stats-expired",
+        content: "Will expire",
+        source: "session_summary",
+      });
+
+      // Force expiry
+      getDb()
+        .prepare("UPDATE agent_memory SET expiresAt = datetime('now', '-1 hour') WHERE id = ?")
+        .run(mem.id);
+
+      const stats = store.getStats(agentA);
+      expect(stats.expired).toBeGreaterThanOrEqual(1);
+
+      store.delete(mem.id);
+    });
+  });
+
+  // ==========================================================================
+  // Re-embed updates embeddingModel
+  // ==========================================================================
+
+  describe("updateEmbedding tracks model", () => {
+    test("re-embedding updates embeddingModel column", () => {
+      const mem = store.store({
+        agentId: agentA,
+        scope: "agent",
+        name: "model-track",
+        content: "Track model changes",
+        source: "manual",
+      });
+
+      // First embed
+      store.updateEmbedding(mem.id, new Float32Array([0.1, 0.2, 0.3]), "model-v1");
+      let row = getDb()
+        .prepare("SELECT embeddingModel FROM agent_memory WHERE id = ?")
+        .get(mem.id) as { embeddingModel: string | null };
+      expect(row.embeddingModel).toBe("model-v1");
+
+      // Re-embed with new model
+      store.updateEmbedding(mem.id, new Float32Array([0.3, 0.2, 0.1]), "model-v2");
+      row = getDb().prepare("SELECT embeddingModel FROM agent_memory WHERE id = ?").get(mem.id) as {
+        embeddingModel: string | null;
+      };
+      expect(row.embeddingModel).toBe("model-v2");
+
+      store.delete(mem.id);
+    });
+  });
+
+  // ==========================================================================
+  // listForReembedding
+  // ==========================================================================
+
+  describe("listForReembedding", () => {
+    test("returns id and content for all memories", () => {
+      const mem = store.store({
+        agentId: agentA,
+        scope: "agent",
+        name: "reembed-list",
+        content: "Content for re-embedding",
+        source: "manual",
+      });
+
+      const list = store.listForReembedding();
+      expect(list.length).toBeGreaterThan(0);
+
+      const found = list.find((item) => item.id === mem.id);
+      expect(found).toBeDefined();
+      expect(found!.content).toBe("Content for re-embedding");
+
+      store.delete(mem.id);
+    });
+
+    test("filters by agentId", () => {
+      const mem = store.store({
+        agentId: agentB,
+        scope: "agent",
+        name: "reembed-filtered",
+        content: "B's content",
+        source: "manual",
+      });
+
+      const listAll = store.listForReembedding();
+      const listB = store.listForReembedding({ agentId: agentB });
+      expect(listB.length).toBeLessThanOrEqual(listAll.length);
+
+      const found = listB.find((item) => item.id === mem.id);
+      expect(found).toBeDefined();
+
+      store.delete(mem.id);
+    });
+  });
+});