@convex-dev/rag 0.7.0 → 0.7.2

package/src/component/namespaces.ts

@@ -1,3 +1,18 @@
+import { assert } from "convex-helpers";
+import { paginator } from "convex-helpers/server/pagination";
+import { mergedStream, stream } from "convex-helpers/server/stream";
+import { paginationOptsValidator, PaginationResult } from "convex/server";
+import type { Infer } from "convex/values";
+import {
+  statuses,
+  vActiveStatus,
+  vEntry,
+  vNamespace,
+  vPaginationResult,
+  vStatus,
+  type OnCompleteNamespace,
+} from "../shared.js";
+import { api } from "./_generated/api.js";
 import type { Doc, Id } from "./_generated/dataModel.js";
 import {
   action,
@@ -5,59 +20,15 @@ import {
   mutation,
   query,
   type MutationCtx,
-  type QueryCtx,
 } from "./_generated/server.js";
-import { schema, v } from "./schema.js";
+import { deleteEntrySync } from "./entries.js";
 import {
-  vNamespace,
-  vPaginationResult,
-  vActiveStatus,
-  type Namespace,
-  type NamespaceId,
-  type OnCompleteNamespace,
-  vStatus,
-  statuses,
-  filterNamesContain,
-  vEntry,
-} from "../shared.js";
-import { paginationOptsValidator, PaginationResult } from "convex/server";
-import { paginator } from "convex-helpers/server/pagination";
-import type { Infer, ObjectType } from "convex/values";
-import { mergedStream, stream } from "convex-helpers/server/stream";
-import { assert } from "convex-helpers";
-import { api } from "./_generated/api.js";
-
-function namespaceIsCompatible(
-  existing: Doc<"namespaces">,
-  args: {
-    modelId: string;
-    dimension: number;
-    filterNames: string[];
-  },
-) {
-  // Check basic compatibility
-  if (
-    existing.modelId !== args.modelId ||
-    existing.dimension !== args.dimension
-  ) {
-    return false;
-  }
-
-  // For filter names, the namespace must support all requested filters
-  // but can support additional filters (superset is OK)
-  if (!filterNamesContain(existing.filterNames, args.filterNames)) {
-    return false;
-  }
-
-  return true;
-}
-
-export const vNamespaceLookupArgs = {
-  namespace: v.string(),
-  modelId: v.string(),
-  dimension: v.number(),
-  filterNames: v.array(v.string()),
-};
+  getCompatibleNamespaceHandler,
+  namespaceIsCompatible,
+  publicNamespace,
+  vNamespaceLookupArgs,
+} from "./helpers.js";
+import { schema, v } from "./schema.js";
 
 export const get = query({
   args: vNamespaceLookupArgs,
@@ -77,24 +48,6 @@ export const getCompatibleNamespace = internalQuery({
   handler: getCompatibleNamespaceHandler,
 });
 
-export async function getCompatibleNamespaceHandler(
-  ctx: QueryCtx,
-  args: ObjectType<typeof vNamespaceLookupArgs>,
-) {
-  const iter = ctx.db
-    .query("namespaces")
-    .withIndex("status_namespace_version", (q) =>
-      q.eq("status.kind", "ready").eq("namespace", args.namespace),
-    )
-    .order("desc");
-  for await (const existing of iter) {
-    if (namespaceIsCompatible(existing, args)) {
-      return existing;
-    }
-  }
-  return null;
-}
-
 export const lookup = query({
   args: {
     namespace: v.string(),
@@ -312,16 +265,6 @@ export const listNamespaceVersions = query({
   },
 });
 
-export function publicNamespace(namespace: Doc<"namespaces">): Namespace {
-  const { _id, _creationTime, status, ...rest } = namespace;
-  return {
-    namespaceId: _id as unknown as NamespaceId,
-    createdAt: _creationTime,
-    ...rest,
-    status: status.kind,
-  };
-}
-
 export const deleteNamespace = mutation({
   args: { namespaceId: v.id("namespaces") },
   returns: v.object({
@@ -368,13 +311,13 @@ export const deleteNamespaceSync = action({
             cursor,
           },
         })) as PaginationResult<Infer<typeof vEntry>>;
+        for (const entry of entries.page) {
+          await deleteEntrySync(ctx, entry.entryId as unknown as Id<"entries">);
+        }
         if (entries.isDone) {
           break;
         }
         cursor = entries.continueCursor;
-        await ctx.runAction(api.entries.deleteSync, {
-          entryId: entries.page[0].entryId as unknown as Id<"entries">,
-        });
       }
     }
     await ctx.runMutation(api.namespaces.deleteNamespace, {

package/src/component/schema.ts

@@ -71,7 +71,6 @@ export const schema = defineSchema({
       v.object({
         kind: v.literal("ready"),
         embeddingId: vVectorId,
-        // TODO: text search
         searchableText: v.optional(v.string()),
       }),
       v.object({

package/src/component/search.test.ts

@@ -3,7 +3,7 @@
 import { describe, expect, test } from "vitest";
 import { convexTest, type TestConvex } from "convex-test";
 import schema from "./schema.js";
-import { api } from "./_generated/api.js";
+import { api, internal } from "./_generated/api.js";
 import { modules } from "./setup.test.js";
 import { insertChunks } from "./chunks.js";
 import type { Id } from "./_generated/dataModel.js";
@@ -442,4 +442,306 @@ describe("search", () => {
       );
     }
   });
+
+  describe("hybrid search", () => {
+    function createSearchableChunks(texts: string[], baseEmbedding = 0.1) {
+      return texts.map((text, i) => ({
+        content: { text, metadata: { index: i } },
+        embedding: [...Array(127).fill(0.01), baseEmbedding + i * 0.01],
+        searchableText: text,
+      }));
+    }
+
+    test("textSearch internal query finds chunks by text content", async () => {
+      const t = convexTest(schema, modules);
+      const namespaceId = await setupTestNamespace(t);
+      const entryId = await setupTestEntry(t, namespaceId);
+
+      const chunks = createSearchableChunks([
+        "The quick brown fox jumps over the lazy dog",
+        "A fast red car drives on the highway",
+        "The brown bear sleeps in the forest",
+      ]);
+
+      await t.run(async (ctx) => {
+        await insertChunks(ctx, { entryId, startOrder: 0, chunks });
+      });
+
+      const results = await t.query(internal.search.textSearch, {
+        query: "brown",
+        namespaceId,
+        filters: [],
+        limit: 10,
+      });
+
+      expect(results.length).toBeGreaterThan(0);
+      for (const r of results) {
+        expect(r.entryId).toBe(entryId);
+      }
+    });
+
+    test("textSearch scopes results to the given namespace", async () => {
+      const t = convexTest(schema, modules);
+      const ns1Id = await setupTestNamespace(t, "namespace-1");
+      const ns2Id = await setupTestNamespace(t, "namespace-2");
+      const entry1Id = await setupTestEntry(t, ns1Id, "entry-1");
+      const entry2Id = await setupTestEntry(t, ns2Id, "entry-2");
+
+      await t.run(async (ctx) => {
+        await insertChunks(ctx, {
+          entryId: entry1Id,
+          startOrder: 0,
+          chunks: createSearchableChunks(["alpha bravo charlie"]),
+        });
+        await insertChunks(ctx, {
+          entryId: entry2Id,
+          startOrder: 0,
+          chunks: createSearchableChunks(["alpha delta echo"]),
+        });
+      });
+
+      const ns1Results = await t.query(internal.search.textSearch, {
+        query: "alpha",
+        namespaceId: ns1Id,
+        filters: [],
+        limit: 10,
+      });
+
+      // All results should belong to namespace-1's entry.
+      for (const r of ns1Results) {
+        expect(r.entryId).toBe(entry1Id);
+      }
+    });
+
+    test("textSearch applies numbered filters", async () => {
+      const t = convexTest(schema, modules);
+      const namespaceId = await setupTestNamespace(t, "filtered-ns", 128, [
+        "category",
+      ]);
+
+      const cat1Entry = await setupTestEntry(t, namespaceId, "cat1", 0, [
+        { name: "category", value: "docs" },
+      ]);
+      const cat2Entry = await setupTestEntry(t, namespaceId, "cat2", 0, [
+        { name: "category", value: "blogs" },
+      ]);
+
+      await t.run(async (ctx) => {
+        await insertChunks(ctx, {
+          entryId: cat1Entry,
+          startOrder: 0,
+          chunks: createSearchableChunks(["shared keyword content"]),
+        });
+        await insertChunks(ctx, {
+          entryId: cat2Entry,
+          startOrder: 0,
+          chunks: createSearchableChunks(["shared keyword content"]),
+        });
+      });
+
+      // Filter to "docs" category only (filter index 0 = "category").
+      const results = await t.query(internal.search.textSearch, {
+        query: "shared keyword",
+        namespaceId,
+        filters: [{ 0: "docs" }],
+        limit: 10,
+      });
+
+      expect(results.length).toBeGreaterThan(0);
+      for (const r of results) {
+        expect(r.entryId).toBe(cat1Entry);
+      }
+    });
+
+    test("text-only search returns results via dimension arg", async () => {
+      const t = convexTest(schema, modules);
+      const namespaceId = await setupTestNamespace(t);
+      const entryId = await setupTestEntry(t, namespaceId);
+
+      const chunks = createSearchableChunks([
+        "Machine learning is a subset of artificial intelligence",
+        "Deep learning uses neural networks with many layers",
+        "Natural language processing handles text data",
+      ]);
+
+      await t.run(async (ctx) => {
+        await insertChunks(ctx, { entryId, startOrder: 0, chunks });
+      });
+
+      // Text-only: no embedding, provide dimension instead.
+      const result = await t.action(api.search.search, {
+        namespace: "test-namespace",
+        dimension: 128,
+        modelId: "test-model",
+        filters: [],
+        limit: 10,
+        textQuery: "neural networks",
+      });
+
+      expect(result.results.length).toBeGreaterThan(0);
+      expect(result.entries).toHaveLength(1);
+
+      // Text-only scores are position-based.
+      expect(result.results[0].score).toBe(1.0);
+      for (let i = 1; i < result.results.length; i++) {
+        expect(result.results[i].score).toBeLessThan(
+          result.results[i - 1].score,
+        );
+      }
+    });
+
+    test("hybrid search returns results when textQuery is provided", async () => {
+      const t = convexTest(schema, modules);
+      const namespaceId = await setupTestNamespace(t);
+      const entryId = await setupTestEntry(t, namespaceId);
+
+      const chunks = createSearchableChunks([
+        "Machine learning is a subset of artificial intelligence",
+        "Deep learning uses neural networks with many layers",
+        "Natural language processing handles text data",
+      ]);
+
+      await t.run(async (ctx) => {
+        await insertChunks(ctx, { entryId, startOrder: 0, chunks });
+      });
+
+      const result = await t.action(api.search.search, {
+        namespace: "test-namespace",
+        embedding: [...Array(127).fill(0.01), 0.1],
+        modelId: "test-model",
+        filters: [],
+        limit: 10,
+        textQuery: "neural networks",
+      });
+
+      expect(result.results.length).toBeGreaterThan(0);
+      expect(result.entries).toHaveLength(1);
+
+      // Hybrid scores are position-based (1.0 for top, decreasing linearly).
+      expect(result.results[0].score).toBe(1.0);
+      for (let i = 1; i < result.results.length; i++) {
+        expect(result.results[i].score).toBeLessThan(
+          result.results[i - 1].score,
+        );
+      }
+    });
+
+    test("hybrid search deduplicates results from vector and text paths", async () => {
+      const t = convexTest(schema, modules);
+      const namespaceId = await setupTestNamespace(t);
+      const entryId = await setupTestEntry(t, namespaceId);
+
+      const chunks = createSearchableChunks([
+        "Unique content about quantum computing",
+        "Another chunk about classical physics",
+      ]);
+
+      await t.run(async (ctx) => {
+        await insertChunks(ctx, { entryId, startOrder: 0, chunks });
+      });
+
+      const result = await t.action(api.search.search, {
+        namespace: "test-namespace",
+        embedding: [...Array(127).fill(0.01), 0.1],
+        modelId: "test-model",
+        filters: [],
+        limit: 10,
+        textQuery: "quantum computing",
+      });
+
+      // Each chunk should appear at most once in the results.
+      const entryOrderPairs = result.results.map(
+        (r) => `${r.entryId}:${r.order}`,
+      );
+      const uniquePairs = new Set(entryOrderPairs);
+      expect(uniquePairs.size).toBe(entryOrderPairs.length);
+    });
+
+    test("vector-only search is unchanged when textQuery is not provided", async () => {
+      const t = convexTest(schema, modules);
+      const namespaceId = await setupTestNamespace(t);
+      const entryId = await setupTestEntry(t, namespaceId);
+
+      const targetEmbedding = [...Array(127).fill(0.5), 1];
+      const chunks = [
+        {
+          content: { text: "Target chunk", metadata: {} },
+          embedding: targetEmbedding,
+          searchableText: "Target chunk",
+        },
+        {
+          content: { text: "Other chunk", metadata: {} },
+          embedding: [...Array(127).fill(0.1), 0],
+          searchableText: "Other chunk",
+        },
+      ];
+
+      await t.run(async (ctx) => {
+        await insertChunks(ctx, { entryId, startOrder: 0, chunks });
+      });
+
+      const result = await t.action(api.search.search, {
+        namespace: "test-namespace",
+        embedding: targetEmbedding,
+        modelId: "test-model",
+        filters: [],
+        limit: 10,
+      });
+
+      // Without textQuery, scores should be cosine similarity (not position-based).
+      expect(result.results).toHaveLength(2);
+      expect(result.results[0].score).toBeGreaterThan(result.results[1].score);
+      // Cosine similarity scores are typically between -1 and 1, not exactly 1.0.
+      // Position-based would give exactly 1.0 for the first result.
+      // With cosine similarity the first result can be 1.0 if exact match,
+      // but the second should not follow the linear decrease pattern.
+      expect(result.results[0].content[0].text).toBe("Target chunk");
+    });
+
+    test("textWeight and vectorWeight influence hybrid ranking", async () => {
+      const t = convexTest(schema, modules);
+      const namespaceId = await setupTestNamespace(t);
+      const entryId = await setupTestEntry(t, namespaceId);
+
+      const chunks = createSearchableChunks([
+        "Alpha topic with specific terminology",
+        "Beta topic with different keywords",
+        "Gamma topic about something else entirely",
+      ]);
+
+      await t.run(async (ctx) => {
+        await insertChunks(ctx, { entryId, startOrder: 0, chunks });
+      });
+
+      const embedding = [...Array(127).fill(0.01), 0.1];
+
+      // Search with heavy text weight.
+      const textHeavy = await t.action(api.search.search, {
+        namespace: "test-namespace",
+        embedding,
+        modelId: "test-model",
+        filters: [],
+        limit: 10,
+        textQuery: "specific terminology",
+        textWeight: 10,
+        vectorWeight: 1,
+      });
+
+      // Search with heavy vector weight.
+      const vectorHeavy = await t.action(api.search.search, {
+        namespace: "test-namespace",
+        embedding,
+        modelId: "test-model",
+        filters: [],
+        limit: 10,
+        textQuery: "specific terminology",
+        textWeight: 1,
+        vectorWeight: 10,
+      });
+
+      // Both should return results.
+      expect(textHeavy.results.length).toBeGreaterThan(0);
+      expect(vectorHeavy.results.length).toBeGreaterThan(0);
+    });
+  });
 });