@usewhisper/mcp-server 0.2.3 → 0.4.0

package/dist/chunk-JO3ORBZD.js

@@ -0,0 +1,616 @@
+import {
+  extractEventDate
+} from "./chunk-5KBZQHDL.js";
+import {
+  db,
+  embedSingle
+} from "./chunk-MEFLJ4PV.js";
+
+// ../src/engine/memory/extractor.ts
+import OpenAI from "openai";
+var openai = new OpenAI({
+  apiKey: process.env.OPENAI_API_KEY || ""
+});
+var EXTRACTION_PROMPT = `You are an expert memory extraction system. Your job is to extract atomic, unambiguous memories from conversation chunks.
+
+**Critical Rules:**
+1. Each memory must be a SINGLE fact/preference/event/relationship
+2. Resolve ALL pronouns (he/she/it/they/them) to actual names using context
+3. Resolve ALL ambiguous references ("the company", "that project") to specific entities
+4. Extract temporal information when events occurred (not when mentioned)
+5. Be conservative - only extract high-confidence memories
+
+**Memory Types:**
+- factual: Objective facts ("John works at Google")
+- preference: User preferences ("Sarah prefers dark mode")
+- event: Events with timestamps ("Team met on Jan 15, 2024")
+- relationship: Relationships ("Alex reports to Maria")
+- opinion: Subjective views ("User thinks Python is easier than Rust")
+- goal: Future intentions ("User wants to learn machine learning")
+- instruction: Persistent instructions ("Always use formal tone with clients")
+
+**Disambiguation:**
+- Replace "he" \u2192 actual name using context
+- Replace "she" \u2192 actual name using context
+- Replace "it" \u2192 specific thing using context
+- Replace "the company" \u2192 company name
+- Replace "that project" \u2192 project name
+
+**Example:**
+Input: "He said he prefers using React. The project will launch next week."
+Context: Previous message: "Alex joined the team yesterday. He's working on the dashboard project."
+
+Bad Output:
+- "He prefers React" \u274C (ambiguous)
+- "The project launches next week" \u274C (what project?)
+
+Good Output:
+- content: "Alex prefers using React for development"
+  type: preference
+  entities: ["Alex", "React"]
+  eventDate: null
+
+- content: "Dashboard project launch scheduled for [specific date if mentioned]"
+  type: event
+  entities: ["Dashboard project"]
+  eventDate: [calculated date] or null if not specific`;
+async function extractMemories(chunk, context) {
+  const contextStr = buildContextString(context);
+  const prompt = `${EXTRACTION_PROMPT}
+
+${contextStr}
+
+**Current chunk to analyze:**
+${chunk}
+
+**Document Date (when this was said):** ${context.documentDate.toISOString()}
+
+Extract memories and return a JSON array. For each memory:
+{
+  "content": "clear, unambiguous statement with no pronouns",
+  "memoryType": "factual|preference|event|relationship|opinion|goal|instruction",
+  "entityMentions": ["list", "of", "entities", "mentioned"],
+  "eventDate": "ISO date string or null",
+  "confidence": 0.0-1.0,
+  "reasoning": "brief explanation of extraction"
+}
+
+Return ONLY the JSON array, no other text.`;
+  try {
+    const response = await openai.chat.completions.create({
+      model: "gpt-4o",
+      max_tokens: 4096,
+      temperature: 0,
+      // Deterministic for extraction
+      messages: [
+        {
+          role: "user",
+          content: prompt
+        }
+      ],
+      response_format: { type: "json_object" }
+    });
+    const text = response.choices[0]?.message?.content?.trim();
+    if (!text) {
+      throw new Error("No text response from OpenAI");
+    }
+    const jsonMatch = text.match(/```json\n?([\s\S]*?)\n?```/) || text.match(/\[[\s\S]*\]/);
+    const jsonStr = jsonMatch ? jsonMatch[1] || jsonMatch[0] : text;
+    const rawMemories = JSON.parse(jsonStr);
+    if (!Array.isArray(rawMemories)) {
+      console.error("Expected array of memories, got:", rawMemories);
+      return [];
+    }
+    return rawMemories.map((m) => ({
+      content: m.content,
+      memoryType: m.memoryType,
+      entityMentions: m.entityMentions || [],
+      eventDate: m.eventDate ? new Date(m.eventDate) : null,
+      confidence: m.confidence || 0.7,
+      reasoning: m.reasoning
+    }));
+  } catch (error) {
+    console.error("Memory extraction failed:", error);
+    return [];
+  }
+}
+function buildContextString(context) {
+  const parts = [];
+  if (context.previousMessages && context.previousMessages.length > 0) {
+    parts.push("**Context from previous messages:**");
+    parts.push(context.previousMessages.slice(-5).join("\n"));
+  }
+  if (context.entityContext && context.entityContext.size > 0) {
+    parts.push("\n**Known entities:**");
+    context.entityContext.forEach((name, pronoun) => {
+      parts.push(`- "${pronoun}" refers to ${name}`);
+    });
+  }
+  if (parts.length === 0) {
+    return "**Context:** None available";
+  }
+  return parts.join("\n");
+}
+function buildEntityContext(recentMemories) {
+  const entityMap = /* @__PURE__ */ new Map();
+  for (const memory of recentMemories) {
+    for (const entity of memory.entityMentions) {
+      if (/^[A-Z][a-z]+(?:\s[A-Z][a-z]+)*$/.test(entity)) {
+        entityMap.set("he", entity);
+        entityMap.set("she", entity);
+        entityMap.set("they", entity);
+      }
+    }
+  }
+  return entityMap;
+}
+function validateMemory(memory) {
+  if (memory.confidence < 0.6) {
+    return false;
+  }
+  if (memory.content.length < 10) {
+    return false;
+  }
+  const pronouns = /\b(he|she|it|they|them|his|her|their)\b/i;
+  if (pronouns.test(memory.content)) {
+    console.warn("Memory contains unresolved pronouns:", memory.content);
+    return false;
+  }
+  const vagueRefs = /\b(the company|that project|this thing|the system)\b/i;
+  if (vagueRefs.test(memory.content)) {
+    console.warn("Memory contains vague references:", memory.content);
+    return false;
+  }
+  return true;
+}
+
+// ../src/engine/memory/relations.ts
+import OpenAI2 from "openai";
+var openai2 = new OpenAI2({
+  apiKey: process.env.OPENAI_API_KEY || ""
+});
+var RELATION_DETECTION_PROMPT = `You are an expert at detecting relationships between memories in a knowledge graph.
+
+**Relation Types:**
+
+1. **updates** - New memory supersedes/replaces old memory (state mutation)
+   Example:
+   - Old: "User's favorite color is blue"
+   - New: "User's favorite color is green"
+   - Relation: updates (green replaces blue)
+
+2. **extends** - New memory adds detail to existing memory without contradiction (refinement)
+   Example:
+   - Old: "John works at Google"
+   - New: "John works at Google as a Senior Engineer"
+   - Relation: extends (adds job title)
+
+3. **derives** - New memory is inferred from existing memory/memories (inference)
+   Example:
+   - Memory 1: "User prefers dark mode"
+   - Memory 2: "User prefers high contrast"
+   - New: "User likely has vision preferences for accessibility"
+   - Relation: derives (inferred from both)
+
+4. **contradicts** - New memory conflicts with existing memory (conflict detection)
+   Example:
+   - Old: "Meeting scheduled for 3pm"
+   - New: "Meeting scheduled for 4pm"
+   - Relation: contradicts (should trigger update)
+
+5. **supports** - New memory provides evidence/support for existing memory
+   Example:
+   - Memory 1: "User is interested in ML"
+   - New: "User enrolled in ML course"
+   - Relation: supports (confirms interest)
+
+**Important:**
+- Only detect relations when there's a clear, meaningful connection
+- Be conservative - if unsure, don't create a relation
+- "updates" should invalidate the old memory (set validUntil)
+- "extends" keeps the old memory valid but adds information
+- "contradicts" should flag for review/resolution`;
+async function detectRelations(newMemory, existingMemories) {
+  if (existingMemories.length === 0) {
+    return [];
+  }
+  const relevantMemories = filterRelevantMemories(newMemory, existingMemories);
+  if (relevantMemories.length === 0) {
+    return [];
+  }
+  const prompt = `${RELATION_DETECTION_PROMPT}
+
+**New memory:**
+"${newMemory.content}"
+Type: ${newMemory.memoryType}
+Entities: ${newMemory.entityMentions.join(", ")}
+
+**Existing memories to check against:**
+${relevantMemories.map((m, i) => `${i}. "${m.content}" (Type: ${m.memoryType}, Date: ${m.documentDate?.toISOString() || "unknown"})`).join("\n")}
+
+Analyze if the new memory relates to any existing memories.
+
+Return a JSON array of relations:
+[{
+  "toMemoryIndex": 0,
+  "relationType": "updates|extends|derives|contradicts|supports",
+  "confidence": 0.0-1.0,
+  "reasoning": "brief explanation why this relation exists"
+}]
+
+Return ONLY the JSON array. If no relations found, return [].`;
+  try {
+    const response = await openai2.chat.completions.create({
+      model: "gpt-4o",
+      max_tokens: 2048,
+      temperature: 0,
+      messages: [{ role: "user", content: prompt }],
+      response_format: { type: "json_object" }
+    });
+    const text = response.choices[0]?.message?.content?.trim();
+    if (!text) {
+      return [];
+    }
+    const jsonMatch = text.match(/```json\n?([\s\S]*?)\n?```/) || text.match(/\[[\s\S]*\]/);
+    const jsonStr = jsonMatch ? jsonMatch[1] || jsonMatch[0] : text;
+    const relations = JSON.parse(jsonStr);
+    if (!Array.isArray(relations)) {
+      return [];
+    }
+    return relations.filter((r) => r.confidence >= 0.7).map((r) => ({
+      toMemoryId: relevantMemories[r.toMemoryIndex].id,
+      relationType: r.relationType,
+      confidence: r.confidence,
+      reasoning: r.reasoning
+    }));
+  } catch (error) {
+    console.error("Relation detection failed:", error);
+    return [];
+  }
+}
+function filterRelevantMemories(newMemory, existingMemories) {
+  return existingMemories.filter((existing) => {
+    const sharedEntities = newMemory.entityMentions.some(
+      (entity) => existing.entityMentions.includes(entity)
+    );
+    if (sharedEntities) {
+      return true;
+    }
+    const newWords = extractKeywords(newMemory.content);
+    const existingWords = extractKeywords(existing.content);
+    const overlap = newWords.filter((w) => existingWords.includes(w));
+    return overlap.length >= 2;
+  });
+}
+function extractKeywords(text) {
+  const stopWords = /* @__PURE__ */ new Set([
+    "the",
+    "a",
+    "an",
+    "is",
+    "are",
+    "was",
+    "were",
+    "be",
+    "been",
+    "being",
+    "have",
+    "has",
+    "had",
+    "do",
+    "does",
+    "did",
+    "will",
+    "would",
+    "could",
+    "should",
+    "may",
+    "might",
+    "must",
+    "can",
+    "to",
+    "of",
+    "in",
+    "for",
+    "on",
+    "at",
+    "by",
+    "from",
+    "with",
+    "about"
+  ]);
+  return text.toLowerCase().split(/\W+/).filter((word) => word.length > 3 && !stopWords.has(word)).slice(0, 10);
+}
+function shouldInvalidateMemory(relationType) {
+  return relationType === "updates" || relationType === "contradicts";
+}
+
+// ../src/engine/memory/ingest.ts
+async function ingestSession(params) {
+  const { sessionId, projectId, orgId, userId, messages } = params;
+  const result = {
+    memoriesCreated: 0,
+    relationsCreated: 0,
+    memoriesInvalidated: 0,
+    errors: []
+  };
+  if (messages.length === 0) {
+    return result;
+  }
+  try {
+    const context = {
+      sessionId,
+      userId: userId || "unknown",
+      projectId,
+      orgId,
+      documentDate: messages[messages.length - 1].timestamp,
+      previousMessages: messages.slice(0, -1).map((m) => `${m.role}: ${m.content}`)
+    };
+    const recentMemories = await db.memory.findMany({
+      where: {
+        sessionId,
+        projectId,
+        isActive: true
+      },
+      orderBy: {
+        createdAt: "desc"
+      },
+      take: 20,
+      select: {
+        content: true,
+        entityMentions: true
+      }
+    });
+    context.entityContext = buildEntityContext(recentMemories);
+    const latestMessage = messages[messages.length - 1].content;
+    const extractedMemories = await extractMemories(latestMessage, context);
+    const validMemories = extractedMemories.filter(validateMemory);
+    if (validMemories.length === 0) {
+      return result;
+    }
+    const existingMemories = await db.memory.findMany({
+      where: {
+        projectId,
+        userId,
+        isActive: true
+      },
+      orderBy: {
+        createdAt: "desc"
+      },
+      take: 100,
+      // Check against last 100 memories
+      select: {
+        id: true,
+        content: true,
+        memoryType: true,
+        entityMentions: true,
+        documentDate: true
+      }
+    });
+    for (const extracted of validMemories) {
+      try {
+        const relations = await detectRelations(
+          {
+            content: extracted.content,
+            memoryType: extracted.memoryType,
+            entityMentions: extracted.entityMentions
+          },
+          existingMemories
+        );
+        const eventDate = extracted.eventDate || await extractEventDate(
+          extracted.content,
+          context.documentDate
+        );
+        const embedding = await embedSingle(extracted.content);
+        const embeddingStr = `[${embedding.join(",")}]`;
+        const [memory] = await db.$queryRaw`
+          INSERT INTO "memories" (
+            id, "projectId", "orgId", "userId", "sessionId", "memoryType", 
+            content, embedding, entity_mentions, confidence, "documentDate", 
+            "eventDate", "validFrom", metadata, "createdAt", "updatedAt"
+          )
+          VALUES (
+            gen_random_uuid(), ${projectId}, ${orgId || null}, ${userId || null}, 
+            ${sessionId}, ${extracted.memoryType}, ${extracted.content}, 
+            ${embeddingStr}::vector, ${JSON.stringify(extracted.entityMentions)}::jsonb, ${extracted.confidence}, 
+            ${context.documentDate}, ${eventDate}, NOW(), 
+            ${JSON.stringify({
+          reasoning: extracted.reasoning,
+          extractedFrom: "session_ingestion"
+        })}::jsonb, NOW(), NOW()
+          )
+          RETURNING id, "projectId", "orgId", "userId", "sessionId", "memoryType", 
+            content, confidence, "documentDate", 
+            "eventDate", "validFrom", metadata, "createdAt", "updatedAt"
+        `;
+        result.memoriesCreated++;
+        for (const relation of relations) {
+          try {
+            await db.memoryRelation.create({
+              data: {
+                fromMemoryId: memory.id,
+                toMemoryId: relation.toMemoryId,
+                relationType: relation.relationType,
+                confidence: relation.confidence,
+                reasoning: relation.reasoning
+              }
+            });
+            result.relationsCreated++;
+            if (shouldInvalidateMemory(relation.relationType)) {
+              await db.memory.update({
+                where: { id: relation.toMemoryId },
+                data: {
+                  validUntil: /* @__PURE__ */ new Date(),
+                  supersededBy: memory.id
+                }
+              });
+              const oldMemory = await db.memory.findUnique({
+                where: { id: relation.toMemoryId },
+                select: { version: true }
+              });
+              if (oldMemory) {
+                await db.memory.update({
+                  where: { id: memory.id },
+                  data: { version: oldMemory.version + 1 }
+                });
+              }
+              result.memoriesInvalidated++;
+            }
+          } catch (error) {
+            result.errors.push(`Failed to create relation: ${error}`);
+          }
+        }
+      } catch (error) {
+        result.errors.push(`Failed to process memory: ${error}`);
+      }
+    }
+    return result;
+  } catch (error) {
+    result.errors.push(`Ingestion failed: ${error}`);
+    return result;
+  }
+}
+async function ingestChunk(params) {
+  const { chunkId, chunkContent, projectId, orgId, documentDate, metadata } = params;
+  const result = {
+    memoriesCreated: 0,
+    relationsCreated: 0,
+    memoriesInvalidated: 0,
+    errors: []
+  };
+  try {
+    const context = {
+      sessionId: `chunk_${chunkId}`,
+      userId: "system",
+      projectId,
+      orgId,
+      documentDate
+    };
+    const extractedMemories = await extractMemories(chunkContent, context);
+    const validMemories = extractedMemories.filter(validateMemory);
+    for (const extracted of validMemories) {
+      const eventDate = extracted.eventDate || await extractEventDate(
+        extracted.content,
+        documentDate
+      );
+      const embedding = await embedSingle(extracted.content);
+      await db.memory.create({
+        data: {
+          projectId,
+          orgId,
+          memoryType: extracted.memoryType,
+          content: extracted.content,
+          embedding,
+          entityMentions: extracted.entityMentions,
+          confidence: extracted.confidence,
+          documentDate,
+          eventDate,
+          validFrom: /* @__PURE__ */ new Date(),
+          sourceChunkId: chunkId,
+          scope: "DOCUMENT",
+          // Document-level scope
+          metadata: {
+            ...metadata,
+            reasoning: extracted.reasoning
+          }
+        }
+      });
+      result.memoriesCreated++;
+    }
+    return result;
+  } catch (error) {
+    result.errors.push(`Chunk ingestion failed: ${error}`);
+    return result;
+  }
+}
+async function ingestChunksBatch(params) {
+  const { chunks, projectId, orgId, documentDate } = params;
+  const aggregateResult = {
+    memoriesCreated: 0,
+    relationsCreated: 0,
+    memoriesInvalidated: 0,
+    errors: []
+  };
+  const batchSize = 10;
+  for (let i = 0; i < chunks.length; i += batchSize) {
+    const batch = chunks.slice(i, i + batchSize);
+    const results = await Promise.all(
+      batch.map(
+        (chunk) => ingestChunk({
+          chunkId: chunk.id,
+          chunkContent: chunk.content,
+          projectId,
+          orgId,
+          documentDate,
+          metadata: chunk.metadata
+        })
+      )
+    );
+    for (const result of results) {
+      aggregateResult.memoriesCreated += result.memoriesCreated;
+      aggregateResult.relationsCreated += result.relationsCreated;
+      aggregateResult.memoriesInvalidated += result.memoriesInvalidated;
+      aggregateResult.errors.push(...result.errors);
+    }
+  }
+  return aggregateResult;
+}
+async function updateMemory(params) {
+  const { memoryId, newContent, reasoning } = params;
+  const oldMemory = await db.memory.findUnique({
+    where: { id: memoryId }
+  });
+  if (!oldMemory) {
+    throw new Error("Memory not found");
+  }
+  const embedding = await embedSingle(newContent);
+  const newMemory = await db.memory.create({
+    data: {
+      projectId: oldMemory.projectId,
+      orgId: oldMemory.orgId,
+      userId: oldMemory.userId,
+      sessionId: oldMemory.sessionId,
+      memoryType: oldMemory.memoryType,
+      content: newContent,
+      embedding,
+      entityMentions: oldMemory.entityMentions,
+      confidence: oldMemory.confidence,
+      documentDate: oldMemory.documentDate,
+      eventDate: oldMemory.eventDate,
+      validFrom: /* @__PURE__ */ new Date(),
+      version: oldMemory.version + 1,
+      scope: oldMemory.scope,
+      metadata: {
+        ...oldMemory.metadata,
+        updateReasoning: reasoning
+      }
+    }
+  });
+  await db.memory.update({
+    where: { id: memoryId },
+    data: {
+      validUntil: /* @__PURE__ */ new Date(),
+      supersededBy: newMemory.id
+    }
+  });
+  await db.memoryRelation.create({
+    data: {
+      fromMemoryId: newMemory.id,
+      toMemoryId: memoryId,
+      relationType: "updates",
+      confidence: 1,
+      reasoning: reasoning || "Manual update"
+    }
+  });
+  return {
+    newMemoryId: newMemory.id,
+    oldMemoryId: memoryId
+  };
+}
+
+export {
+  ingestSession,
+  ingestChunk,
+  ingestChunksBatch,
+  updateMemory
+};