@cortexmemory/cli 0.27.3 → 0.28.0

package/templates/vercel-ai-quickstart/tests/e2e/chat-memory-flow.test.ts

@@ -0,0 +1,483 @@
+/**
+ * E2E Tests: Chat API Memory Flow
+ *
+ * These tests verify the FULL memory flow through the chat API routes:
+ * - Fact storage from conversations
+ * - Belief revision (superseding facts)
+ * - Memory recall across conversations
+ * - Conversation lifecycle (create, chat, delete)
+ *
+ * REQUIRES:
+ * - CONVEX_URL: Real Convex deployment
+ * - OPENAI_API_KEY: For LLM calls and embeddings
+ * - CORTEX_FACT_EXTRACTION=true: Enable fact extraction
+ *
+ * These tests use real HTTP requests to the quickstart server,
+ * so the server must be running on localhost:3000.
+ */
+
+import { Cortex } from "@cortexmemory/sdk";
+
+// Skip if required env vars not set
+const SKIP_E2E =
+  !process.env.CONVEX_URL ||
+  !process.env.OPENAI_API_KEY ||
+  !process.env.QUICKSTART_URL;
+
+const BASE_URL = process.env.QUICKSTART_URL || "http://localhost:3000";
+
+// Generate unique IDs for test isolation
+function generateTestId(prefix: string): string {
+  const timestamp = Date.now();
+  const random = Math.random().toString(36).slice(2, 8);
+  return `${prefix}-e2e-${timestamp}-${random}`;
+}
+
+/**
+ * Make a chat request to the API
+ */
+async function sendChatMessage(
+  endpoint: string,
+  messages: Array<{ role: string; content: string }>,
+  options: {
+    userId: string;
+    memorySpaceId: string;
+    conversationId?: string;
+  },
+): Promise<{
+  response: string;
+  conversationId?: string;
+}> {
+  const response = await fetch(`${BASE_URL}/api/${endpoint}`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      messages: messages.map((m, i) => ({
+        id: `msg-${i}`,
+        role: m.role,
+        content: m.content,
+        createdAt: new Date().toISOString(),
+      })),
+      userId: options.userId,
+      memorySpaceId: options.memorySpaceId,
+      conversationId: options.conversationId,
+    }),
+  });
+
+  if (!response.ok) {
+    const error = await response.text();
+    throw new Error(`Chat API error: ${response.status} - ${error}`);
+  }
+
+  // Parse streaming response
+  const text = await response.text();
+
+  // Extract text content from the stream (simplified parsing)
+  let fullResponse = "";
+  let conversationId: string | undefined;
+
+  const lines = text.split("\n");
+  for (const line of lines) {
+    if (line.startsWith("0:")) {
+      // Text content
+      try {
+        const content = JSON.parse(line.slice(2));
+        if (typeof content === "string") {
+          fullResponse += content;
+        }
+      } catch {
+        // Ignore parse errors
+      }
+    } else if (line.includes("data-conversation-update")) {
+      // Extract conversation ID
+      try {
+        const match = line.match(/"conversationId":"([^"]+)"/);
+        if (match) {
+          conversationId = match[1];
+        }
+      } catch {
+        // Ignore parse errors
+      }
+    }
+  }
+
+  return { response: fullResponse, conversationId };
+}
+
+describe("Chat Memory Flow E2E", () => {
+  let cortex: Cortex;
+  let testUserId: string;
+  let testMemorySpaceId: string;
+
+  beforeAll(() => {
+    if (SKIP_E2E) {
+      console.log(
+        "Skipping E2E tests - CONVEX_URL, OPENAI_API_KEY, or QUICKSTART_URL not configured",
+      );
+      return;
+    }
+    cortex = new Cortex({ convexUrl: process.env.CONVEX_URL! });
+  });
+
+  beforeEach(() => {
+    if (SKIP_E2E) return;
+    testUserId = generateTestId("user");
+    testMemorySpaceId = generateTestId("space");
+  });
+
+  afterAll(async () => {
+    if (cortex) {
+      cortex.close();
+    }
+  });
+
+  // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+  // V5 Route Tests
+  // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+  (SKIP_E2E ? describe.skip : describe)("v5 route (/api/chat)", () => {
+    it("should store facts from conversation", async () => {
+      // Send a message with a fact
+      await sendChatMessage(
+        "chat",
+        [
+          {
+            role: "user",
+            content: "My name is Alice and I work as a software engineer",
+          },
+        ],
+        { userId: testUserId, memorySpaceId: testMemorySpaceId },
+      );
+
+      // Wait for fact extraction
+      await new Promise((r) => setTimeout(r, 5000));
+
+      // Verify facts were stored
+      const facts = await cortex.facts.list({
+        memorySpaceId: testMemorySpaceId,
+        userId: testUserId,
+        includeSuperseded: false,
+      });
+
+      console.log(`[V5] Stored facts: ${facts.length}`);
+      facts.forEach((f) => console.log(`  - ${f.fact}`));
+
+      expect(facts.length).toBeGreaterThan(0);
+    }, 60000);
+
+    it("should supersede facts through belief revision", async () => {
+      // First message: establish a preference
+      await sendChatMessage(
+        "chat",
+        [{ role: "user", content: "My favorite color is blue" }],
+        { userId: testUserId, memorySpaceId: testMemorySpaceId },
+      );
+
+      await new Promise((r) => setTimeout(r, 5000));
+
+      // Second message: change the preference
+      await sendChatMessage(
+        "chat",
+        [
+          { role: "user", content: "My favorite color is blue" },
+          {
+            role: "assistant",
+            content: "Got it, blue is your favorite color!",
+          },
+          {
+            role: "user",
+            content: "Actually, my favorite color is purple now",
+          },
+        ],
+        { userId: testUserId, memorySpaceId: testMemorySpaceId },
+      );
+
+      await new Promise((r) => setTimeout(r, 5000));
+
+      // Check facts
+      const allFacts = await cortex.facts.list({
+        memorySpaceId: testMemorySpaceId,
+        userId: testUserId,
+        includeSuperseded: true,
+      });
+
+      const activeFacts = await cortex.facts.list({
+        memorySpaceId: testMemorySpaceId,
+        userId: testUserId,
+        includeSuperseded: false,
+      });
+
+      console.log(
+        `[V5] All facts: ${allFacts.length}, Active: ${activeFacts.length}`,
+      );
+      allFacts.forEach((f) => {
+        const status = f.supersededBy ? "SUPERSEDED" : "ACTIVE";
+        console.log(`  [${status}] ${f.fact}`);
+      });
+
+      // Should have superseded the old color preference
+      const colorFacts = activeFacts.filter(
+        (f) =>
+          f.fact.toLowerCase().includes("color") ||
+          f.fact.toLowerCase().includes("purple") ||
+          f.fact.toLowerCase().includes("blue"),
+      );
+
+      // Ideally only one active color fact (purple)
+      expect(colorFacts.length).toBeLessThanOrEqual(2);
+    }, 90000);
+
+    it("should recall facts in subsequent conversations", async () => {
+      // First conversation: store a fact
+      const conv1Result = await sendChatMessage(
+        "chat",
+        [{ role: "user", content: "I have a dog named Max" }],
+        { userId: testUserId, memorySpaceId: testMemorySpaceId },
+      );
+
+      await new Promise((r) => setTimeout(r, 5000));
+
+      // Second conversation: ask about the fact
+      const conv2Result = await sendChatMessage(
+        "chat",
+        [{ role: "user", content: "What do you remember about my pets?" }],
+        { userId: testUserId, memorySpaceId: testMemorySpaceId },
+      );
+
+      console.log(
+        `[V5] Recall response: ${conv2Result.response.slice(0, 200)}...`,
+      );
+
+      // Response should mention Max (the dog)
+      const responseText = conv2Result.response.toLowerCase();
+      const mentionsPet =
+        responseText.includes("max") || responseText.includes("dog");
+
+      // Note: LLM responses are non-deterministic, so we just verify we got a response
+      expect(conv2Result.response.length).toBeGreaterThan(0);
+    }, 90000);
+  });
+
+  // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+  // V6 Route Tests
+  // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+  (SKIP_E2E ? describe.skip : describe)("v6 route (/api/chat-v6)", () => {
+    it("should store facts from conversation", async () => {
+      // Send a message with a fact
+      await sendChatMessage(
+        "chat-v6",
+        [{ role: "user", content: "My name is Bob and I'm a data scientist" }],
+        { userId: testUserId, memorySpaceId: testMemorySpaceId },
+      );
+
+      // Wait for fact extraction
+      await new Promise((r) => setTimeout(r, 5000));
+
+      // Verify facts were stored
+      const facts = await cortex.facts.list({
+        memorySpaceId: testMemorySpaceId,
+        userId: testUserId,
+        includeSuperseded: false,
+      });
+
+      console.log(`[V6] Stored facts: ${facts.length}`);
+      facts.forEach((f) => console.log(`  - ${f.fact}`));
+
+      expect(facts.length).toBeGreaterThan(0);
+    }, 60000);
+
+    it("should supersede facts through belief revision", async () => {
+      // First message: establish a preference
+      await sendChatMessage(
+        "chat-v6",
+        [{ role: "user", content: "I prefer tea over coffee" }],
+        { userId: testUserId, memorySpaceId: testMemorySpaceId },
+      );
+
+      await new Promise((r) => setTimeout(r, 5000));
+
+      // Second message: change the preference
+      await sendChatMessage(
+        "chat-v6",
+        [
+          { role: "user", content: "I prefer tea over coffee" },
+          { role: "assistant", content: "Got it, you prefer tea!" },
+          {
+            role: "user",
+            content: "Actually I've switched to coffee now, it helps me focus",
+          },
+        ],
+        { userId: testUserId, memorySpaceId: testMemorySpaceId },
+      );
+
+      await new Promise((r) => setTimeout(r, 5000));
+
+      // Check facts
+      const allFacts = await cortex.facts.list({
+        memorySpaceId: testMemorySpaceId,
+        userId: testUserId,
+        includeSuperseded: true,
+      });
+
+      const activeFacts = await cortex.facts.list({
+        memorySpaceId: testMemorySpaceId,
+        userId: testUserId,
+        includeSuperseded: false,
+      });
+
+      console.log(
+        `[V6] All facts: ${allFacts.length}, Active: ${activeFacts.length}`,
+      );
+      allFacts.forEach((f) => {
+        const status = f.supersededBy ? "SUPERSEDED" : "ACTIVE";
+        console.log(`  [${status}] ${f.fact}`);
+      });
+
+      // Should have at least one fact about beverages
+      const beverageFacts = allFacts.filter(
+        (f) =>
+          f.fact.toLowerCase().includes("tea") ||
+          f.fact.toLowerCase().includes("coffee"),
+      );
+      expect(beverageFacts.length).toBeGreaterThan(0);
+    }, 90000);
+
+    it("should recall facts in subsequent conversations", async () => {
+      // First conversation: store a fact
+      await sendChatMessage(
+        "chat-v6",
+        [{ role: "user", content: "I live in San Francisco" }],
+        { userId: testUserId, memorySpaceId: testMemorySpaceId },
+      );
+
+      await new Promise((r) => setTimeout(r, 5000));
+
+      // Second conversation: ask about the fact
+      const conv2Result = await sendChatMessage(
+        "chat-v6",
+        [{ role: "user", content: "Where do I live?" }],
+        { userId: testUserId, memorySpaceId: testMemorySpaceId },
+      );
+
+      console.log(
+        `[V6] Recall response: ${conv2Result.response.slice(0, 200)}...`,
+      );
+
+      // Verify we got a response
+      expect(conv2Result.response.length).toBeGreaterThan(0);
+    }, 90000);
+  });
+
+  // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+  // Feature Parity Tests
+  // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+  (SKIP_E2E ? describe.skip : describe)("v5 vs v6 feature parity", () => {
+    it("both routes should store facts for the same message", async () => {
+      const v5UserId = generateTestId("user-v5");
+      const v6UserId = generateTestId("user-v6");
+      const sharedSpaceId = testMemorySpaceId;
+
+      // Send same message to both routes
+      const message = "I am a TypeScript developer with 5 years of experience";
+
+      await Promise.all([
+        sendChatMessage("chat", [{ role: "user", content: message }], {
+          userId: v5UserId,
+          memorySpaceId: sharedSpaceId,
+        }),
+        sendChatMessage("chat-v6", [{ role: "user", content: message }], {
+          userId: v6UserId,
+          memorySpaceId: sharedSpaceId,
+        }),
+      ]);
+
+      // Wait for fact extraction
+      await new Promise((r) => setTimeout(r, 7000));
+
+      // Check facts for both users
+      const [v5Facts, v6Facts] = await Promise.all([
+        cortex.facts.list({
+          memorySpaceId: sharedSpaceId,
+          userId: v5UserId,
+          includeSuperseded: false,
+        }),
+        cortex.facts.list({
+          memorySpaceId: sharedSpaceId,
+          userId: v6UserId,
+          includeSuperseded: false,
+        }),
+      ]);
+
+      console.log(`V5 facts: ${v5Facts.length}, V6 facts: ${v6Facts.length}`);
+      console.log(
+        "V5 facts:",
+        v5Facts.map((f) => f.fact),
+      );
+      console.log(
+        "V6 facts:",
+        v6Facts.map((f) => f.fact),
+      );
+
+      // CRITICAL: Both routes should store facts
+      expect(v5Facts.length).toBeGreaterThan(0);
+      expect(v6Facts.length).toBeGreaterThan(0);
+
+      // Fact counts should be similar (allow some variance due to LLM non-determinism)
+      const diff = Math.abs(v5Facts.length - v6Facts.length);
+      expect(diff).toBeLessThanOrEqual(2);
+    }, 90000);
+  });
+
+  // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+  // Conversation Lifecycle
+  // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+  (SKIP_E2E ? describe.skip : describe)("conversation lifecycle", () => {
+    it("should create, list, and delete conversations", async () => {
+      // Create a conversation via chat
+      const chatResult = await sendChatMessage(
+        "chat",
+        [{ role: "user", content: "Hello, this is a test conversation" }],
+        { userId: testUserId, memorySpaceId: testMemorySpaceId },
+      );
+
+      // Wait for conversation to be created
+      await new Promise((r) => setTimeout(r, 2000));
+
+      // List conversations
+      const listResponse = await fetch(
+        `${BASE_URL}/api/conversations?userId=${testUserId}&memorySpaceId=${testMemorySpaceId}`,
+      );
+      const listData = await listResponse.json();
+
+      console.log(
+        `Conversations: ${JSON.stringify(listData.conversations, null, 2)}`,
+      );
+      expect(listData.conversations).toBeDefined();
+      expect(listData.conversations.length).toBeGreaterThan(0);
+
+      // Delete conversation
+      const convId = listData.conversations[0].id;
+      const deleteResponse = await fetch(
+        `${BASE_URL}/api/conversations?conversationId=${convId}`,
+        { method: "DELETE" },
+      );
+      const deleteData = await deleteResponse.json();
+
+      expect(deleteData.success).toBe(true);
+
+      // Verify deletion
+      const listAfterDelete = await fetch(
+        `${BASE_URL}/api/conversations?userId=${testUserId}&memorySpaceId=${testMemorySpaceId}`,
+      );
+      const listAfterDeleteData = await listAfterDelete.json();
+
+      // Should have one less conversation
+      expect(listAfterDeleteData.conversations.length).toBeLessThan(
+        listData.conversations.length,
+      );
+    }, 60000);
+  });
+});

package/templates/vercel-ai-quickstart/tests/helpers/mock-cortex.ts

@@ -48,42 +48,40 @@ export function createMockCortex() {
           const ns = mutableStore.get(namespace);
           if (!ns) return null;
           return ns.get(key) ?? null;
-        }
+        },
       ),
       set: jest.fn(
         async (
           namespace: string,
           key: string,
-          value: unknown
+          value: unknown,
         ): Promise<void> => {
           if (!mutableStore.has(namespace)) {
             mutableStore.set(namespace, new Map());
           }
           mutableStore.get(namespace)!.set(key, value);
-        }
-      ),
-      delete: jest.fn(
-        async (namespace: string, key: string): Promise<void> => {
-          const ns = mutableStore.get(namespace);
-          if (ns) ns.delete(key);
-        }
+        },
       ),
+      delete: jest.fn(async (namespace: string, key: string): Promise<void> => {
+        const ns = mutableStore.get(namespace);
+        if (ns) ns.delete(key);
+      }),
     },
     users: {
       get: jest.fn(
         async (
-          userId: string
+          userId: string,
         ): Promise<{
           userId: string;
           data: Record<string, unknown>;
         } | null> => {
           return usersStore.get(userId) ?? null;
-        }
+        },
       ),
       update: jest.fn(
         async (
           userId: string,
-          data: Record<string, unknown>
+          data: Record<string, unknown>,
         ): Promise<{ userId: string; data: Record<string, unknown> }> => {
           const existing = usersStore.get(userId);
           const user = {
@@ -92,7 +90,7 @@ export function createMockCortex() {
           };
           usersStore.set(userId, user);
           return user;
-        }
+        },
       ),
       delete: jest.fn(async (userId: string): Promise<void> => {
         usersStore.delete(userId);
@@ -115,17 +113,14 @@ export function createMockCortex() {
               ) {
                 return false;
               }
-              if (
-                params.userId &&
-                conv.participants.userId !== params.userId
-              ) {
+              if (params.userId && conv.participants.userId !== params.userId) {
                 return false;
               }
               return true;
-            }
+            },
           );
           return { conversations, hasMore: false };
-        }
+        },
       ),
       create: jest.fn(
         async (params: {
@@ -148,31 +143,36 @@ export function createMockCortex() {
           };
           conversationsStore.set(params.conversationId, conversation);
           return conversation;
-        }
+        },
+      ),
+      get: jest.fn(
+        async (
+          conversationId: string,
+          options?: { includeMessages?: boolean; messageLimit?: number },
+        ) => {
+          const conv = conversationsStore.get(conversationId);
+          if (!conv) return null;
+
+          // If includeMessages is requested, return with messages
+          if (options?.includeMessages) {
+            return {
+              ...conv,
+              messages: conv.messages || [],
+            };
+          }
+
+          // Otherwise return without messages
+          const { messages: _messages, ...convWithoutMessages } = conv;
+          return convWithoutMessages;
+        },
       ),
-      get: jest.fn(async (conversationId: string, options?: { includeMessages?: boolean; messageLimit?: number }) => {
-        const conv = conversationsStore.get(conversationId);
-        if (!conv) return null;
-        
-        // If includeMessages is requested, return with messages
-        if (options?.includeMessages) {
-          return {
-            ...conv,
-            messages: conv.messages || [],
-          };
-        }
-        
-        // Otherwise return without messages
-        const { messages: _messages, ...convWithoutMessages } = conv;
-        return convWithoutMessages;
-      }),
       delete: jest.fn(async (conversationId: string): Promise<void> => {
         conversationsStore.delete(conversationId);
       }),
       update: jest.fn(
         async (
           conversationId: string,
-          updates: { metadata?: Record<string, unknown> }
+          updates: { metadata?: Record<string, unknown> },
         ) => {
           const conv = conversationsStore.get(conversationId);
           if (!conv) throw new Error("Conversation not found");
@@ -183,7 +183,7 @@ export function createMockCortex() {
           };
           conversationsStore.set(conversationId, updated);
           return updated;
-        }
+        },
       ),
     },
     memory: {
@@ -215,7 +215,7 @@ export type MockCortex = ReturnType<typeof createMockCortex>;
 export const seedTestData = {
   user: (
     userId: string,
-    data: Record<string, unknown> = {}
+    data: Record<string, unknown> = {},
   ): { userId: string; data: Record<string, unknown> } => {
     const user = { userId, data };
     usersStore.set(userId, user);
@@ -234,7 +234,7 @@ export const seedTestData = {
       userId?: string;
       title?: string;
       messages?: Array<{ role: string; content: string }>;
-    } = {}
+    } = {},
   ) => {
     const now = Date.now();
     const messages = params.messages?.map((msg, i) => ({