@convex-dev/rag 0.7.0 → 0.7.1

package/src/component/_generated/component.ts

@@ -409,12 +409,17 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
         "internal",
         {
           chunkContext?: { after: number; before: number };
-          embedding: Array<number>;
+          dimension?: number;
+          embedding?: Array<number>;
           filters: Array<{ name: string; value: any }>;
           limit: number;
           modelId: string;
           namespace: string;
+          searchType?: "vector" | "text" | "hybrid";
+          textQuery?: string;
+          textWeight?: number;
           vectorScoreThreshold?: number;
+          vectorWeight?: number;
         },
         {
           entries: Array<{

package/src/component/_generated/server.ts

@@ -107,11 +107,6 @@ export const internalAction: ActionBuilder<DataModel, "internal"> =
  */
 export const httpAction: HttpActionBuilder = httpActionGeneric;
 
-type GenericCtx =
-  | GenericActionCtx<DataModel>
-  | GenericMutationCtx<DataModel>
-  | GenericQueryCtx<DataModel>;
-
 /**
  * A set of services for use within Convex query functions.
  *

package/src/component/chunks.ts

@@ -311,6 +311,107 @@ export const vRangeResult = v.object({
   ),
 });
 
+export async function buildRanges(
+  ctx: QueryCtx,
+  chunks: (Doc<"chunks"> | null)[],
+  chunkContext: { before: number; after: number },
+): Promise<{
+  ranges: (null | Infer<typeof vRangeResult>)[];
+  entries: Entry[];
+}> {
+  // Note: This preserves order of entries as they first appeared.
+  const entryDocs = (
+    await Promise.all(
+      Array.from(
+        new Set(chunks.filter((c) => c !== null).map((c) => c.entryId)),
+      ).map((id) => ctx.db.get(id)),
+    )
+  ).filter((d): d is Doc<"entries"> => d !== null);
+  const entries = entryDocs.map(publicEntry);
+  const entryDocById = new Map(entryDocs.map((d) => [d._id, d]));
+
+  const entryOrders = chunks
+    .filter((c) => c !== null)
+    .map((c) => [c.entryId, c.order] as const)
+    .reduce(
+      (acc, [entryId, order]) => {
+        if (acc[entryId]?.includes(order)) {
+          // De-dupe orders
+          return acc;
+        }
+        acc[entryId] = [...(acc[entryId] ?? []), order].sort((a, b) => a - b);
+        return acc;
+      },
+      {} as Record<Id<"entries">, number[]>,
+    );
+
+  const result: Array<Infer<typeof vRangeResult> | null> = [];
+
+  for (const chunk of chunks) {
+    if (chunk === null) {
+      result.push(null);
+      continue;
+    }
+    // Note: if we parallelize this in the future, we could have a race
+    // instead we'd check that other chunks are not the same doc/order
+    if (
+      result.find(
+        (r) => r?.entryId === chunk.entryId && r?.order === chunk.order,
+      )
+    ) {
+      // De-dupe chunks
+      result.push(null);
+      continue;
+    }
+    const entryId = chunk.entryId;
+    const entry = entryDocById.get(entryId);
+    assert(entry, `Entry ${entryId} not found`);
+    const otherOrders = entryOrders[entryId] ?? [chunk.order];
+    const ourOrderIndex = otherOrders.indexOf(chunk.order);
+    const previousOrder = otherOrders[ourOrderIndex - 1] ?? -Infinity;
+    const nextOrder = otherOrders[ourOrderIndex + 1] ?? Infinity;
+    // We absorb all previous context up to the previous chunk.
+    const startOrder = Math.max(
+      chunk.order - chunkContext.before,
+      0,
+      Math.min(previousOrder + 1, chunk.order),
+    );
+    // We stop short if the next chunk order's "before" context will cover it.
+    const endOrder = Math.min(
+      chunk.order + chunkContext.after + 1,
+      Math.max(nextOrder - chunkContext.before, chunk.order + 1),
+    );
+    const contentIds: Id<"content">[] = [];
+    if (startOrder === chunk.order && endOrder === chunk.order + 1) {
+      contentIds.push(chunk.contentId);
+    } else {
+      const rangeChunks = await ctx.db
+        .query("chunks")
+        .withIndex("entryId_order", (q) =>
+          q
+            .eq("entryId", entryId)
+            .gte("order", startOrder)
+            .lt("order", endOrder),
+        )
+        .collect();
+      for (const c of rangeChunks) {
+        contentIds.push(c.contentId);
+      }
+    }
+    const content = await Promise.all(
+      contentIds.map(async (contentId) => {
+        const content = await ctx.db.get(contentId);
+        assert(content, `Content ${contentId} not found`);
+        return { text: content.text, metadata: content.metadata };
+      }),
+    );
+
+    result.push({ entryId, order: chunk.order, startOrder, content });
+  }
+
+  return { ranges: result, entries };
+}
+
 export const getRangesOfChunks = internalQuery({
   args: {
     embeddingIds: v.array(vVectorId),
@@ -339,98 +440,7 @@ export const getRangesOfChunks = internalQuery({
           .first(),
       ),
     );
-
-    // Note: This preserves order of entries as they first appeared.
-    const entries = (
-      await Promise.all(
-        Array.from(
-          new Set(chunks.filter((c) => c !== null).map((c) => c.entryId)),
-        ).map((id) => ctx.db.get(id)),
-      )
-    )
-      .filter((d) => d !== null)
-      .map(publicEntry);
-
-    const entryOders = chunks
-      .filter((c) => c !== null)
-      .map((c) => [c.entryId, c.order] as const)
-      .reduce(
-        (acc, [entryId, order]) => {
-          if (acc[entryId]?.includes(order)) {
-            // De-dupe orders
-            return acc;
-          }
-          acc[entryId] = [...(acc[entryId] ?? []), order].sort((a, b) => a - b);
-          return acc;
-        },
-        {} as Record<Id<"entries">, number[]>,
-      );
-
-    const result: Array<Infer<typeof vRangeResult> | null> = [];
-
-    for (const chunk of chunks) {
-      if (chunk === null) {
-        result.push(null);
-        continue;
-      }
-      // Note: if we parallelize this in the future, we could have a race
-      // instead we'd check that other chunks are not the same doc/order
-      if (
-        result.find(
-          (r) => r?.entryId === chunk.entryId && r?.order === chunk.order,
-        )
-      ) {
-        // De-dupe chunks
-        result.push(null);
-        continue;
-      }
-      const entryId = chunk.entryId;
-      const entry = await ctx.db.get(entryId);
-      assert(entry, `Entry ${entryId} not found`);
-      const otherOrders = entryOders[entryId] ?? [chunk.order];
-      const ourOrderIndex = otherOrders.indexOf(chunk.order);
-      const previousOrder = otherOrders[ourOrderIndex - 1] ?? -Infinity;
-      const nextOrder = otherOrders[ourOrderIndex + 1] ?? Infinity;
-      // We absorb all previous context up to the previous chunk.
-      const startOrder = Math.max(
-        chunk.order - chunkContext.before,
-        0,
-        Math.min(previousOrder + 1, chunk.order),
-      );
-      // We stop short if the next chunk order's "before" context will cover it.
-      const endOrder = Math.min(
-        chunk.order + chunkContext.after + 1,
-        Math.max(nextOrder - chunkContext.before, chunk.order + 1),
-      );
-      const contentIds: Id<"content">[] = [];
-      if (startOrder === chunk.order && endOrder === chunk.order + 1) {
-        contentIds.push(chunk.contentId);
-      } else {
-        const chunks = await ctx.db
-          .query("chunks")
-          .withIndex("entryId_order", (q) =>
-            q
-              .eq("entryId", entryId)
-              .gte("order", startOrder)
-              .lt("order", endOrder),
-          )
-          .collect();
-        for (const chunk of chunks) {
-          contentIds.push(chunk.contentId);
-        }
-      }
-      const content = await Promise.all(
-        contentIds.map(async (contentId) => {
-          const content = await ctx.db.get(contentId);
-          assert(content, `Content ${contentId} not found`);
-          return { text: content.text, metadata: content.metadata };
-        }),
-      );
-
-      result.push({ entryId, order: chunk.order, startOrder, content });
-    }
-
-    return { ranges: result, entries };
+    return buildRanges(ctx, chunks, chunkContext);
   },
 });
 

package/src/component/schema.ts

@@ -71,7 +71,6 @@ export const schema = defineSchema({
       v.object({
         kind: v.literal("ready"),
         embeddingId: vVectorId,
-        // TODO: text search
         searchableText: v.optional(v.string()),
       }),
       v.object({

package/src/component/search.test.ts

@@ -3,7 +3,7 @@
 import { describe, expect, test } from "vitest";
 import { convexTest, type TestConvex } from "convex-test";
 import schema from "./schema.js";
-import { api } from "./_generated/api.js";
+import { api, internal } from "./_generated/api.js";
 import { modules } from "./setup.test.js";
 import { insertChunks } from "./chunks.js";
 import type { Id } from "./_generated/dataModel.js";
@@ -442,4 +442,306 @@ describe("search", () => {
       );
     }
   });
+
+  describe("hybrid search", () => {
+    function createSearchableChunks(texts: string[], baseEmbedding = 0.1) {
+      return texts.map((text, i) => ({
+        content: { text, metadata: { index: i } },
+        embedding: [...Array(127).fill(0.01), baseEmbedding + i * 0.01],
+        searchableText: text,
+      }));
+    }
+
+    test("textSearch internal query finds chunks by text content", async () => {
+      const t = convexTest(schema, modules);
+      const namespaceId = await setupTestNamespace(t);
+      const entryId = await setupTestEntry(t, namespaceId);
+
+      const chunks = createSearchableChunks([
+        "The quick brown fox jumps over the lazy dog",
+        "A fast red car drives on the highway",
+        "The brown bear sleeps in the forest",
+      ]);
+
+      await t.run(async (ctx) => {
+        await insertChunks(ctx, { entryId, startOrder: 0, chunks });
+      });
+
+      const results = await t.query(internal.search.textSearch, {
+        query: "brown",
+        namespaceId,
+        filters: [],
+        limit: 10,
+      });
+
+      expect(results.length).toBeGreaterThan(0);
+      for (const r of results) {
+        expect(r.entryId).toBe(entryId);
+      }
+    });
+
+    test("textSearch scopes results to the given namespace", async () => {
+      const t = convexTest(schema, modules);
+      const ns1Id = await setupTestNamespace(t, "namespace-1");
+      const ns2Id = await setupTestNamespace(t, "namespace-2");
+      const entry1Id = await setupTestEntry(t, ns1Id, "entry-1");
+      const entry2Id = await setupTestEntry(t, ns2Id, "entry-2");
+
+      await t.run(async (ctx) => {
+        await insertChunks(ctx, {
+          entryId: entry1Id,
+          startOrder: 0,
+          chunks: createSearchableChunks(["alpha bravo charlie"]),
+        });
+        await insertChunks(ctx, {
+          entryId: entry2Id,
+          startOrder: 0,
+          chunks: createSearchableChunks(["alpha delta echo"]),
+        });
+      });
+
+      const ns1Results = await t.query(internal.search.textSearch, {
+        query: "alpha",
+        namespaceId: ns1Id,
+        filters: [],
+        limit: 10,
+      });
+
+      // All results should belong to namespace-1's entry.
+      for (const r of ns1Results) {
+        expect(r.entryId).toBe(entry1Id);
+      }
+    });
+
+    test("textSearch applies numbered filters", async () => {
+      const t = convexTest(schema, modules);
+      const namespaceId = await setupTestNamespace(t, "filtered-ns", 128, [
+        "category",
+      ]);
+
+      const cat1Entry = await setupTestEntry(t, namespaceId, "cat1", 0, [
+        { name: "category", value: "docs" },
+      ]);
+      const cat2Entry = await setupTestEntry(t, namespaceId, "cat2", 0, [
+        { name: "category", value: "blogs" },
+      ]);
+
+      await t.run(async (ctx) => {
+        await insertChunks(ctx, {
+          entryId: cat1Entry,
+          startOrder: 0,
+          chunks: createSearchableChunks(["shared keyword content"]),
+        });
+        await insertChunks(ctx, {
+          entryId: cat2Entry,
+          startOrder: 0,
+          chunks: createSearchableChunks(["shared keyword content"]),
+        });
+      });
+
+      // Filter to "docs" category only (filter index 0 = "category").
+      const results = await t.query(internal.search.textSearch, {
+        query: "shared keyword",
+        namespaceId,
+        filters: [{ 0: "docs" }],
+        limit: 10,
+      });
+
+      expect(results.length).toBeGreaterThan(0);
+      for (const r of results) {
+        expect(r.entryId).toBe(cat1Entry);
+      }
+    });
+
+    test("text-only search returns results via dimension arg", async () => {
+      const t = convexTest(schema, modules);
+      const namespaceId = await setupTestNamespace(t);
+      const entryId = await setupTestEntry(t, namespaceId);
+
+      const chunks = createSearchableChunks([
+        "Machine learning is a subset of artificial intelligence",
+        "Deep learning uses neural networks with many layers",
+        "Natural language processing handles text data",
+      ]);
+
+      await t.run(async (ctx) => {
+        await insertChunks(ctx, { entryId, startOrder: 0, chunks });
+      });
+
+      // Text-only: no embedding, provide dimension instead.
+      const result = await t.action(api.search.search, {
+        namespace: "test-namespace",
+        dimension: 128,
+        modelId: "test-model",
+        filters: [],
+        limit: 10,
+        textQuery: "neural networks",
+      });
+
+      expect(result.results.length).toBeGreaterThan(0);
+      expect(result.entries).toHaveLength(1);
+
+      // Text-only scores are position-based.
+      expect(result.results[0].score).toBe(1.0);
+      for (let i = 1; i < result.results.length; i++) {
+        expect(result.results[i].score).toBeLessThan(
+          result.results[i - 1].score,
+        );
+      }
+    });
+
+    test("hybrid search returns results when textQuery is provided", async () => {
+      const t = convexTest(schema, modules);
+      const namespaceId = await setupTestNamespace(t);
+      const entryId = await setupTestEntry(t, namespaceId);
+
+      const chunks = createSearchableChunks([
+        "Machine learning is a subset of artificial intelligence",
+        "Deep learning uses neural networks with many layers",
+        "Natural language processing handles text data",
+      ]);
+
+      await t.run(async (ctx) => {
+        await insertChunks(ctx, { entryId, startOrder: 0, chunks });
+      });
+
+      const result = await t.action(api.search.search, {
+        namespace: "test-namespace",
+        embedding: [...Array(127).fill(0.01), 0.1],
+        modelId: "test-model",
+        filters: [],
+        limit: 10,
+        textQuery: "neural networks",
+      });
+
+      expect(result.results.length).toBeGreaterThan(0);
+      expect(result.entries).toHaveLength(1);
+
+      // Hybrid scores are position-based (1.0 for top, decreasing linearly).
+      expect(result.results[0].score).toBe(1.0);
+      for (let i = 1; i < result.results.length; i++) {
+        expect(result.results[i].score).toBeLessThan(
+          result.results[i - 1].score,
+        );
+      }
+    });
+
+    test("hybrid search deduplicates results from vector and text paths", async () => {
+      const t = convexTest(schema, modules);
+      const namespaceId = await setupTestNamespace(t);
+      const entryId = await setupTestEntry(t, namespaceId);
+
+      const chunks = createSearchableChunks([
+        "Unique content about quantum computing",
+        "Another chunk about classical physics",
+      ]);
+
+      await t.run(async (ctx) => {
+        await insertChunks(ctx, { entryId, startOrder: 0, chunks });
+      });
+
+      const result = await t.action(api.search.search, {
+        namespace: "test-namespace",
+        embedding: [...Array(127).fill(0.01), 0.1],
+        modelId: "test-model",
+        filters: [],
+        limit: 10,
+        textQuery: "quantum computing",
+      });
+
+      // Each chunk should appear at most once in the results.
+      const entryOrderPairs = result.results.map(
+        (r) => `${r.entryId}:${r.order}`,
+      );
+      const uniquePairs = new Set(entryOrderPairs);
+      expect(uniquePairs.size).toBe(entryOrderPairs.length);
+    });
+
+    test("vector-only search is unchanged when textQuery is not provided", async () => {
+      const t = convexTest(schema, modules);
+      const namespaceId = await setupTestNamespace(t);
+      const entryId = await setupTestEntry(t, namespaceId);
+
+      const targetEmbedding = [...Array(127).fill(0.5), 1];
+      const chunks = [
+        {
+          content: { text: "Target chunk", metadata: {} },
+          embedding: targetEmbedding,
+          searchableText: "Target chunk",
+        },
+        {
+          content: { text: "Other chunk", metadata: {} },
+          embedding: [...Array(127).fill(0.1), 0],
+          searchableText: "Other chunk",
+        },
+      ];
+
+      await t.run(async (ctx) => {
+        await insertChunks(ctx, { entryId, startOrder: 0, chunks });
+      });
+
+      const result = await t.action(api.search.search, {
+        namespace: "test-namespace",
+        embedding: targetEmbedding,
+        modelId: "test-model",
+        filters: [],
+        limit: 10,
+      });
+
+      // Without textQuery, scores should be cosine similarity (not position-based).
+      expect(result.results).toHaveLength(2);
+      expect(result.results[0].score).toBeGreaterThan(result.results[1].score);
+      // Cosine similarity scores are typically between -1 and 1, not exactly 1.0.
+      // Position-based would give exactly 1.0 for the first result.
+      // With cosine similarity the first result can be 1.0 if exact match,
+      // but the second should not follow the linear decrease pattern.
+      expect(result.results[0].content[0].text).toBe("Target chunk");
+    });
+
+    test("textWeight and vectorWeight influence hybrid ranking", async () => {
+      const t = convexTest(schema, modules);
+      const namespaceId = await setupTestNamespace(t);
+      const entryId = await setupTestEntry(t, namespaceId);
+
+      const chunks = createSearchableChunks([
+        "Alpha topic with specific terminology",
+        "Beta topic with different keywords",
+        "Gamma topic about something else entirely",
+      ]);
+
+      await t.run(async (ctx) => {
+        await insertChunks(ctx, { entryId, startOrder: 0, chunks });
+      });
+
+      const embedding = [...Array(127).fill(0.01), 0.1];
+
+      // Search with heavy text weight.
+      const textHeavy = await t.action(api.search.search, {
+        namespace: "test-namespace",
+        embedding,
+        modelId: "test-model",
+        filters: [],
+        limit: 10,
+        textQuery: "specific terminology",
+        textWeight: 10,
+        vectorWeight: 1,
+      });
+
+      // Search with heavy vector weight.
+      const vectorHeavy = await t.action(api.search.search, {
+        namespace: "test-namespace",
+        embedding,
+        modelId: "test-model",
+        filters: [],
+        limit: 10,
+        textQuery: "specific terminology",
+        textWeight: 1,
+        vectorWeight: 10,
+      });
+
+      // Both should return results.
+      expect(textHeavy.results.length).toBeGreaterThan(0);
+      expect(vectorHeavy.results.length).toBeGreaterThan(0);
+    });
+  });
 });