@theromans/convex-dobty 0.1.9 → 0.1.11

package/src/component/knowledge.ts

@@ -13,8 +13,10 @@ export const seedKnowledgeChunks = mutation({
       v.object({
         topic: v.string(),
         chunkKey: v.string(),
-        title: v.string(),
-        content: v.string(),
+        title_English: v.string(),
+        title_Deutsch: v.string(),
+        content_English: v.string(),
+        content_Deutsch: v.string(),
       }),
     ),
   },
@@ -23,6 +25,9 @@ export const seedKnowledgeChunks = mutation({
     let updated = 0;
     let skipped = 0;
 
+    // Collect all existing chunkKeys for this topic to detect deletions
+    const incomingKeys = new Set(chunks.map((c) => `${c.topic}::${c.chunkKey}`));
+
     for (const chunk of chunks) {
       const existing = await ctx.db
         .query("knowledgeChunks")
@@ -32,10 +37,21 @@ export const seedKnowledgeChunks = mutation({
         .unique();
 
       if (existing) {
-        if (existing.content !== chunk.content || existing.title !== chunk.title) {
+        const contentChanged =
+          existing.content_English !== chunk.content_English ||
+          existing.content_Deutsch !== chunk.content_Deutsch ||
+          existing.title_English !== chunk.title_English ||
+          existing.title_Deutsch !== chunk.title_Deutsch;
+
+        if (contentChanged) {
           await ctx.db.patch(existing._id, {
-            title: chunk.title,
-            content: chunk.content,
+            title_English: chunk.title_English,
+            title_Deutsch: chunk.title_Deutsch,
+            content_English: chunk.content_English,
+            content_Deutsch: chunk.content_Deutsch,
+            // Invalidate both embeddings so they get re-embedded
+            embeddedAt_en: undefined,
+            embeddedAt_de: undefined,
             updatedAt: Date.now(),
           });
           updated++;
@@ -46,15 +62,27 @@ export const seedKnowledgeChunks = mutation({
         await ctx.db.insert("knowledgeChunks", {
           topic: chunk.topic,
           chunkKey: chunk.chunkKey,
-          title: chunk.title,
-          content: chunk.content,
+          title_English: chunk.title_English,
+          title_Deutsch: chunk.title_Deutsch,
+          content_English: chunk.content_English,
+          content_Deutsch: chunk.content_Deutsch,
           updatedAt: Date.now(),
         });
         inserted++;
       }
     }
 
-    return { inserted, updated, skipped };
+    // Remove chunks that are no longer in the incoming set (full replacement semantics)
+    const allExisting = await ctx.db.query("knowledgeChunks").collect();
+    let deleted = 0;
+    for (const existing of allExisting) {
+      if (!incomingKeys.has(`${existing.topic}::${existing.chunkKey}`)) {
+        await ctx.db.delete(existing._id);
+        deleted++;
+      }
+    }
+
+    return { inserted, updated, skipped, deleted };
   },
 });
 
@@ -79,49 +107,104 @@ export const listTopics = query({
   },
 });
 
-/** List chunks that need (re-)embedding. Returns metadata without embedding vectors. */
-export const listStaleChunks = query({
+/** List chunks that need (re-)embedding for English. */
+export const listStaleChunksEn = query({
   args: {},
   handler: async (ctx) => {
     const all = await ctx.db.query("knowledgeChunks").collect();
     return all
-      .filter((c) => !c.embedding || !c.embeddedAt || c.embeddedAt < c.updatedAt)
+      .filter(
+        (c) =>
+          !c.embedding_en ||
+          !c.embeddedAt_en ||
+          c.embeddedAt_en < c.updatedAt,
+      )
       .map((c) => ({
         _id: c._id,
         topic: c.topic,
         chunkKey: c.chunkKey,
-        title: c.title,
-        content: c.content,
+        title: c.title_English,
+        content: c.content_English,
       }));
   },
 });
 
-/** Update a chunk's embedding by ID (used by host-side Node.js embedding). */
-export const updateChunkEmbedding = mutation({
+/** List chunks that need (re-)embedding for Deutsch. */
+export const listStaleChunksDe = query({
+  args: {},
+  handler: async (ctx) => {
+    const all = await ctx.db.query("knowledgeChunks").collect();
+    return all
+      .filter(
+        (c) =>
+          !c.embedding_de ||
+          !c.embeddedAt_de ||
+          c.embeddedAt_de < c.updatedAt,
+      )
+      .map((c) => ({
+        _id: c._id,
+        topic: c.topic,
+        chunkKey: c.chunkKey,
+        title: c.title_Deutsch,
+        content: c.content_Deutsch,
+      }));
+  },
+});
+
+/** Update a chunk's English embedding by ID. */
+export const updateChunkEmbeddingEn = mutation({
   args: {
     chunkId: v.id("knowledgeChunks"),
     embedding: v.array(v.float64()),
   },
   handler: async (ctx, { chunkId, embedding }) => {
     await ctx.db.patch(chunkId, {
-      embedding,
-      embeddedAt: Date.now(),
+      embedding_en: embedding,
+      embeddedAt_en: Date.now(),
+    });
+  },
+});
+
+/** Update a chunk's Deutsch embedding by ID. */
+export const updateChunkEmbeddingDe = mutation({
+  args: {
+    chunkId: v.id("knowledgeChunks"),
+    embedding: v.array(v.float64()),
+  },
+  handler: async (ctx, { chunkId, embedding }) => {
+    await ctx.db.patch(chunkId, {
+      embedding_de: embedding,
+      embeddedAt_de: Date.now(),
     });
   },
 });
 
 // ── Internal functions (called by knowledgeSearch.ts) ──
 
-/** Patch embedding + embeddedAt for a single chunk. */
-export const updateEmbedding = internalMutation({
+/** Patch English embedding for a single chunk. */
+export const updateEmbeddingEn = internalMutation({
+  args: {
+    chunkId: v.id("knowledgeChunks"),
+    embedding: v.array(v.float64()),
+  },
+  handler: async (ctx, { chunkId, embedding }) => {
+    await ctx.db.patch(chunkId, {
+      embedding_en: embedding,
+      embeddedAt_en: Date.now(),
+    });
+  },
+});
+
+/** Patch Deutsch embedding for a single chunk. */
+export const updateEmbeddingDe = internalMutation({
   args: {
     chunkId: v.id("knowledgeChunks"),
     embedding: v.array(v.float64()),
   },
   handler: async (ctx, { chunkId, embedding }) => {
     await ctx.db.patch(chunkId, {
-      embedding,
-      embeddedAt: Date.now(),
+      embedding_de: embedding,
+      embeddedAt_de: Date.now(),
     });
   },
 });

package/src/component/knowledgeSearch.ts

@@ -8,6 +8,7 @@ const EMBEDDING_MODEL = "BAAI/bge-m3";
 const EMBEDDING_DIMENSIONS = 1024;
 
 const EMBEDDING_TIMEOUT_MS = 10_000;
+const DEFAULT_SEARCH_LIMIT = 5;
 
 async function getEmbedding(
   text: string,
@@ -48,110 +49,215 @@ async function getEmbedding(
   }
 }
 
-interface KnowledgeResult {
+export interface KnowledgeResult {
   title: string;
   content: string;
+  title_English: string;
+  title_Deutsch: string;
+  content_English: string;
+  content_Deutsch: string;
   score: number;
+  language: "en" | "de";
+  embeddingLanguage: "en" | "de";
 }
 
-/** Search knowledge chunks by embedding similarity. */
+/**
+ * Search knowledge chunks by embedding similarity.
+ *
+ * - language: the requested language for returned title/content ("en" | "de")
+ * - Uses the embedding index for the requested language.
+ * - Falls back to the other language's index if the primary has no vectors.
+ * - Returns up to `limit` results (default 5).
+ */
 export const searchKnowledge = action({
   args: {
     topic: v.string(),
     query: v.string(),
+    language: v.union(v.literal("en"), v.literal("de")),
     limit: v.optional(v.number()),
     apiToken: v.string(),
   },
-  handler: async (ctx, { topic, query, limit = 3, apiToken }): Promise<KnowledgeResult[]> => {
+  handler: async (
+    ctx,
+    { topic, query, language, limit = DEFAULT_SEARCH_LIMIT, apiToken },
+  ): Promise<KnowledgeResult[]> => {
     const embedding = await getEmbedding(query, apiToken);
 
-    const results = await ctx.vectorSearch("knowledgeChunks", "by_embedding", {
-      vector: embedding,
-      limit,
-      filter: (q) => q.eq("topic", topic),
-    });
-
-    if (results.length === 0) return [];
-
-    const docs: (KnowledgeResult | null)[] = await Promise.all(
-      results.map(async (r): Promise<KnowledgeResult | null> => {
-        const doc: any = await ctx.runQuery(internal.knowledge.getChunk, {
-          chunkId: r._id,
-        });
-        return doc ? { title: doc.title, content: doc.content, score: r._score } : null;
-      }),
-    );
-
-    return docs.filter((d): d is KnowledgeResult => d !== null);
+    return _searchByVector(ctx, { topic, embedding, language, limit });
   },
 });
 
-/** Search knowledge chunks by pre-computed embedding vector (no fetch needed). */
+/**
+ * Search knowledge chunks by pre-computed embedding vector (no outbound fetch needed).
+ *
+ * - language: the requested language for returned title/content ("en" | "de")
+ * - Falls back to the other language's index if primary has no vectors.
+ * - Returns up to `limit` results (default 5).
+ */
 export const searchByVector = action({
   args: {
     topic: v.string(),
     embedding: v.array(v.float64()),
+    language: v.union(v.literal("en"), v.literal("de")),
     limit: v.optional(v.number()),
   },
-  handler: async (ctx, { topic, embedding, limit = 3 }): Promise<KnowledgeResult[]> => {
-    const results = await ctx.vectorSearch("knowledgeChunks", "by_embedding", {
+  handler: async (
+    ctx,
+    { topic, embedding, language, limit = DEFAULT_SEARCH_LIMIT },
+  ): Promise<KnowledgeResult[]> => {
+    return _searchByVector(ctx, { topic, embedding, language, limit });
+  },
+});
+
+async function _searchByVector(
+  ctx: any,
+  {
+    topic,
+    embedding,
+    language,
+    limit,
+  }: {
+    topic: string;
+    embedding: number[];
+    language: "en" | "de";
+    limit: number;
+  },
+): Promise<KnowledgeResult[]> {
+  const primaryIndex = language === "en" ? "by_embedding_en" : "by_embedding_de";
+  const fallbackIndex = language === "en" ? "by_embedding_de" : "by_embedding_en";
+  const fallbackLanguage: "en" | "de" = language === "en" ? "de" : "en";
+
+  // Try primary language index first
+  let results = await ctx.vectorSearch("knowledgeChunks", primaryIndex, {
+    vector: embedding,
+    limit,
+    filter: (q: any) => q.eq("topic", topic),
+  });
+
+  let embeddingLanguage: "en" | "de" = language;
+
+  // Fall back to the other language's index if primary returned nothing
+  if (results.length === 0) {
+    results = await ctx.vectorSearch("knowledgeChunks", fallbackIndex, {
       vector: embedding,
       limit,
-      filter: (q) => q.eq("topic", topic),
+      filter: (q: any) => q.eq("topic", topic),
     });
+    embeddingLanguage = fallbackLanguage;
+  }
 
-    if (results.length === 0) return [];
+  if (results.length === 0) return [];
 
-    const docs: (KnowledgeResult | null)[] = await Promise.all(
-      results.map(async (r): Promise<KnowledgeResult | null> => {
-        const doc: any = await ctx.runQuery(internal.knowledge.getChunk, {
-          chunkId: r._id,
-        });
-        return doc ? { title: doc.title, content: doc.content, score: r._score } : null;
-      }),
-    );
+  const docs: (KnowledgeResult | null)[] = await Promise.all(
+    results.map(async (r: any): Promise<KnowledgeResult | null> => {
+      const doc: any = await ctx.runQuery(internal.knowledge.getChunk, {
+        chunkId: r._id,
+      });
+      if (!doc) return null;
 
-    return docs.filter((d): d is KnowledgeResult => d !== null);
-  },
-});
+      const title = language === "en" ? doc.title_English : doc.title_Deutsch;
+      const content =
+        language === "en" ? doc.content_English : doc.content_Deutsch;
+
+      return {
+        title,
+        content,
+        title_English: doc.title_English,
+        title_Deutsch: doc.title_Deutsch,
+        content_English: doc.content_English,
+        content_Deutsch: doc.content_Deutsch,
+        score: r._score,
+        language,
+        embeddingLanguage,
+      };
+    }),
+  );
+
+  return docs.filter((d): d is KnowledgeResult => d !== null);
+}
 
-/** Re-embed stale chunks (embedding missing or content changed since last embed). */
+/**
+ * Re-embed stale chunks for both languages.
+ * Embeds English and Deutsch separately, storing embedding_en and embedding_de.
+ */
 export const refreshEmbeddings = action({
   args: {
     apiToken: v.string(),
   },
-  handler: async (ctx, { apiToken }): Promise<{ total: number; stale: number; embedded: number; errors: number }> => {
+  handler: async (
+    ctx,
+    { apiToken },
+  ): Promise<{
+    total: number;
+    staleEn: number;
+    staleDe: number;
+    embeddedEn: number;
+    embeddedDe: number;
+    errors: number;
+  }> => {
     const allChunks: any[] = await ctx.runQuery(
       internal.knowledge.listAllChunks,
       {},
     );
 
-    const stale = allChunks.filter(
-      (c) => !c.embedding || !c.embeddedAt || c.embeddedAt < c.updatedAt,
+    const staleEn = allChunks.filter(
+      (c) =>
+        !c.embedding_en || !c.embeddedAt_en || c.embeddedAt_en < c.updatedAt,
+    );
+    const staleDe = allChunks.filter(
+      (c) =>
+        !c.embedding_de || !c.embeddedAt_de || c.embeddedAt_de < c.updatedAt,
     );
 
-    let embedded = 0;
+    let embeddedEn = 0;
+    let embeddedDe = 0;
     let errors = 0;
 
-    for (const chunk of stale) {
+    for (const chunk of staleEn) {
+      try {
+        const embedding = await getEmbedding(
+          `${chunk.title_English}\n\n${chunk.content_English}`,
+          apiToken,
+        );
+        await ctx.runMutation(internal.knowledge.updateEmbeddingEn, {
+          chunkId: chunk._id,
+          embedding,
+        });
+        embeddedEn++;
+      } catch (e: any) {
+        console.error(
+          `Failed to embed EN chunk ${chunk.chunkKey}: ${e.message}`,
+        );
+        errors++;
+      }
+    }
+
+    for (const chunk of staleDe) {
       try {
         const embedding = await getEmbedding(
-          `${chunk.title}\n\n${chunk.content}`,
+          `${chunk.title_Deutsch}\n\n${chunk.content_Deutsch}`,
           apiToken,
         );
-        await ctx.runMutation(internal.knowledge.updateEmbedding, {
+        await ctx.runMutation(internal.knowledge.updateEmbeddingDe, {
           chunkId: chunk._id,
           embedding,
         });
-        embedded++;
+        embeddedDe++;
       } catch (e: any) {
         console.error(
-          `Failed to embed chunk ${chunk.chunkKey}: ${e.message}`,
+          `Failed to embed DE chunk ${chunk.chunkKey}: ${e.message}`,
         );
         errors++;
       }
     }
 
-    return { total: allChunks.length, stale: stale.length, embedded, errors };
+    return {
+      total: allChunks.length,
+      staleEn: staleEn.length,
+      staleDe: staleDe.length,
+      embeddedEn,
+      embeddedDe,
+      errors,
+    };
   },
 });

package/src/component/schema.ts

@@ -215,16 +215,34 @@ export default defineSchema({
   knowledgeChunks: defineTable({
     topic: v.string(),
     chunkKey: v.string(),
-    title: v.string(),
-    content: v.string(),
+    // New bilingual fields (required going forward)
+    title_English: v.optional(v.string()),
+    title_Deutsch: v.optional(v.string()),
+    content_English: v.optional(v.string()),
+    content_Deutsch: v.optional(v.string()),
+    // Legacy single-language fields — kept optional to allow migration deploy
+    title: v.optional(v.string()),
+    content: v.optional(v.string()),
+    /** Embedding of title_English + content_English */
+    embedding_en: v.optional(v.array(v.float64())),
+    /** Embedding of title_Deutsch + content_Deutsch */
+    embedding_de: v.optional(v.array(v.float64())),
+    // Legacy single embedding — kept optional for migration
     embedding: v.optional(v.array(v.float64())),
+    embeddedAt_en: v.optional(v.number()),
+    embeddedAt_de: v.optional(v.number()),
     embeddedAt: v.optional(v.number()),
     updatedAt: v.number(),
   })
     .index("by_topic", ["topic"])
     .index("by_topic_chunkKey", ["topic", "chunkKey"])
-    .vectorIndex("by_embedding", {
-      vectorField: "embedding",
+    .vectorIndex("by_embedding_en", {
+      vectorField: "embedding_en",
+      dimensions: 1024,
+      filterFields: ["topic"],
+    })
+    .vectorIndex("by_embedding_de", {
+      vectorField: "embedding_de",
       dimensions: 1024,
       filterFields: ["topic"],
     }),