@convex-dev/rag 0.7.0 → 0.7.2

package/src/component/_generated/component.ts

@@ -409,12 +409,17 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
         "internal",
         {
           chunkContext?: { after: number; before: number };
-          embedding: Array<number>;
+          dimension?: number;
+          embedding?: Array<number>;
           filters: Array<{ name: string; value: any }>;
           limit: number;
           modelId: string;
           namespace: string;
+          searchType?: "vector" | "text" | "hybrid";
+          textQuery?: string;
+          textWeight?: number;
           vectorScoreThreshold?: number;
+          vectorWeight?: number;
         },
         {
           entries: Array<{

package/src/component/_generated/server.ts

@@ -107,11 +107,6 @@ export const internalAction: ActionBuilder<DataModel, "internal"> =
  */
 export const httpAction: HttpActionBuilder = httpActionGeneric;
 
-type GenericCtx =
-  | GenericActionCtx<DataModel>
-  | GenericMutationCtx<DataModel>
-  | GenericQueryCtx<DataModel>;
-
 /**
  * A set of services for use within Convex query functions.
  *

package/src/component/chunks.ts

@@ -24,7 +24,7 @@ import {
 import { insertEmbedding } from "./embeddings/index.js";
 import { vVectorId, type VectorTableName } from "./embeddings/tables.js";
 import { schema, v } from "./schema.js";
-import { getPreviousEntry, publicEntry } from "./entries.js";
+import { getPreviousEntry, publicEntry } from "./helpers.js";
 import {
   filterFieldsFromNumbers,
   numberedFilterFromNamedFilters,
@@ -311,6 +311,107 @@ export const vRangeResult = v.object({
   ),
 });
 
+export async function buildRanges(
+  ctx: QueryCtx,
+  chunks: (Doc<"chunks"> | null)[],
+  chunkContext: { before: number; after: number },
+): Promise<{
+  ranges: (null | Infer<typeof vRangeResult>)[];
+  entries: Entry[];
+}> {
+  // Note: This preserves order of entries as they first appeared.
+  const entryDocs = (
+    await Promise.all(
+      Array.from(
+        new Set(chunks.filter((c) => c !== null).map((c) => c.entryId)),
+      ).map((id) => ctx.db.get(id)),
+    )
+  ).filter((d): d is Doc<"entries"> => d !== null);
+  const entries = entryDocs.map(publicEntry);
+  const entryDocById = new Map(entryDocs.map((d) => [d._id, d]));
+
+  const entryOrders = chunks
+    .filter((c) => c !== null)
+    .map((c) => [c.entryId, c.order] as const)
+    .reduce(
+      (acc, [entryId, order]) => {
+        if (acc[entryId]?.includes(order)) {
+          // De-dupe orders
+          return acc;
+        }
+        acc[entryId] = [...(acc[entryId] ?? []), order].sort((a, b) => a - b);
+        return acc;
+      },
+      {} as Record<Id<"entries">, number[]>,
+    );
+
+  const result: Array<Infer<typeof vRangeResult> | null> = [];
+
+  for (const chunk of chunks) {
+    if (chunk === null) {
+      result.push(null);
+      continue;
+    }
+    // Note: if we parallelize this in the future, we could have a race
+    // instead we'd check that other chunks are not the same doc/order
+    if (
+      result.find(
+        (r) => r?.entryId === chunk.entryId && r?.order === chunk.order,
+      )
+    ) {
+      // De-dupe chunks
+      result.push(null);
+      continue;
+    }
+    const entryId = chunk.entryId;
+    const entry = entryDocById.get(entryId);
+    assert(entry, `Entry ${entryId} not found`);
+    const otherOrders = entryOrders[entryId] ?? [chunk.order];
+    const ourOrderIndex = otherOrders.indexOf(chunk.order);
+    const previousOrder = otherOrders[ourOrderIndex - 1] ?? -Infinity;
+    const nextOrder = otherOrders[ourOrderIndex + 1] ?? Infinity;
+    // We absorb all previous context up to the previous chunk.
+    const startOrder = Math.max(
+      chunk.order - chunkContext.before,
+      0,
+      Math.min(previousOrder + 1, chunk.order),
+    );
+    // We stop short if the next chunk order's "before" context will cover it.
+    const endOrder = Math.min(
+      chunk.order + chunkContext.after + 1,
+      Math.max(nextOrder - chunkContext.before, chunk.order + 1),
+    );
+    const contentIds: Id<"content">[] = [];
+    if (startOrder === chunk.order && endOrder === chunk.order + 1) {
+      contentIds.push(chunk.contentId);
+    } else {
+      const rangeChunks = await ctx.db
+        .query("chunks")
+        .withIndex("entryId_order", (q) =>
+          q
+            .eq("entryId", entryId)
+            .gte("order", startOrder)
+            .lt("order", endOrder),
+        )
+        .collect();
+      for (const c of rangeChunks) {
+        contentIds.push(c.contentId);
+      }
+    }
+    const content = await Promise.all(
+      contentIds.map(async (contentId) => {
+        const content = await ctx.db.get(contentId);
+        assert(content, `Content ${contentId} not found`);
+        return { text: content.text, metadata: content.metadata };
+      }),
+    );
+
+    result.push({ entryId, order: chunk.order, startOrder, content });
+  }
+
+  return { ranges: result, entries };
+}
+
 export const getRangesOfChunks = internalQuery({
   args: {
     embeddingIds: v.array(vVectorId),
@@ -339,98 +440,7 @@ export const getRangesOfChunks = internalQuery({
           .first(),
       ),
     );
-
-    // Note: This preserves order of entries as they first appeared.
-    const entries = (
-      await Promise.all(
-        Array.from(
-          new Set(chunks.filter((c) => c !== null).map((c) => c.entryId)),
-        ).map((id) => ctx.db.get(id)),
-      )
-    )
-      .filter((d) => d !== null)
-      .map(publicEntry);
-
-    const entryOders = chunks
-      .filter((c) => c !== null)
-      .map((c) => [c.entryId, c.order] as const)
-      .reduce(
-        (acc, [entryId, order]) => {
-          if (acc[entryId]?.includes(order)) {
-            // De-dupe orders
-            return acc;
-          }
-          acc[entryId] = [...(acc[entryId] ?? []), order].sort((a, b) => a - b);
-          return acc;
-        },
-        {} as Record<Id<"entries">, number[]>,
-      );
-
-    const result: Array<Infer<typeof vRangeResult> | null> = [];
-
-    for (const chunk of chunks) {
-      if (chunk === null) {
-        result.push(null);
-        continue;
-      }
-      // Note: if we parallelize this in the future, we could have a race
-      // instead we'd check that other chunks are not the same doc/order
-      if (
-        result.find(
-          (r) => r?.entryId === chunk.entryId && r?.order === chunk.order,
-        )
-      ) {
-        // De-dupe chunks
-        result.push(null);
-        continue;
-      }
-      const entryId = chunk.entryId;
-      const entry = await ctx.db.get(entryId);
-      assert(entry, `Entry ${entryId} not found`);
-      const otherOrders = entryOders[entryId] ?? [chunk.order];
-      const ourOrderIndex = otherOrders.indexOf(chunk.order);
-      const previousOrder = otherOrders[ourOrderIndex - 1] ?? -Infinity;
-      const nextOrder = otherOrders[ourOrderIndex + 1] ?? Infinity;
-      // We absorb all previous context up to the previous chunk.
-      const startOrder = Math.max(
-        chunk.order - chunkContext.before,
-        0,
-        Math.min(previousOrder + 1, chunk.order),
-      );
-      // We stop short if the next chunk order's "before" context will cover it.
-      const endOrder = Math.min(
-        chunk.order + chunkContext.after + 1,
-        Math.max(nextOrder - chunkContext.before, chunk.order + 1),
-      );
-      const contentIds: Id<"content">[] = [];
-      if (startOrder === chunk.order && endOrder === chunk.order + 1) {
-        contentIds.push(chunk.contentId);
-      } else {
-        const chunks = await ctx.db
-          .query("chunks")
-          .withIndex("entryId_order", (q) =>
-            q
-              .eq("entryId", entryId)
-              .gte("order", startOrder)
-              .lt("order", endOrder),
-          )
-          .collect();
-        for (const chunk of chunks) {
-          contentIds.push(chunk.contentId);
-        }
-      }
-      const content = await Promise.all(
-        contentIds.map(async (contentId) => {
-          const content = await ctx.db.get(contentId);
-          assert(content, `Content ${contentId} not found`);
-          return { text: content.text, metadata: content.metadata };
-        }),
-      );
-
-      result.push({ entryId, order: chunk.order, startOrder, content });
-    }
-
-    return { ranges: result, entries };
+    return buildRanges(ctx, chunks, chunkContext);
   },
 });
 

package/src/component/entries.ts

@@ -1,11 +1,18 @@
+import {
+  vResultValidator,
+  vWorkIdValidator,
+  Workpool,
+} from "@convex-dev/workpool";
 import { assert, omit } from "convex-helpers";
+import { mergedStream, stream } from "convex-helpers/server/stream";
+import { doc } from "convex-helpers/validators";
 import {
   createFunctionHandle,
   paginationOptsValidator,
   PaginationResult,
 } from "convex/server";
-import { v, type Value } from "convex/values";
-import type { ChunkerAction, EntryFilter, EntryId } from "../shared.js";
+import { v } from "convex/values";
+import type { ChunkerAction, OnComplete } from "../shared.js";
 import {
   statuses,
   vActiveStatus,
@@ -15,7 +22,7 @@ import {
   vStatus,
   type Entry,
 } from "../shared.js";
-import { api, internal } from "./_generated/api.js";
+import { api, components, internal } from "./_generated/api.js";
 import type { Doc, Id } from "./_generated/dataModel.js";
 import {
   action,
@@ -23,26 +30,19 @@ import {
   internalQuery,
   mutation,
   query,
+  type ActionCtx,
   type MutationCtx,
   type QueryCtx,
 } from "./_generated/server.js";
 import { deleteChunksPageHandler, insertChunks } from "./chunks.js";
-import schema, { type StatusWithOnComplete } from "./schema.js";
-import { mergedStream } from "convex-helpers/server/stream";
-import { stream } from "convex-helpers/server/stream";
 import {
   getCompatibleNamespaceHandler,
+  getPreviousEntry,
+  publicEntry,
   publicNamespace,
   vNamespaceLookupArgs,
-} from "./namespaces.js";
-import type { OnComplete } from "../shared.js";
-import {
-  vResultValidator,
-  vWorkIdValidator,
-  Workpool,
-} from "@convex-dev/workpool";
-import { components } from "./_generated/api.js";
-import { doc } from "convex-helpers/validators";
+} from "./helpers.js";
+import schema, { type StatusWithOnComplete } from "./schema.js";
 
 const workpool = new Workpool(components.workpool, {
   retryActionsByDefault: true,
@@ -479,58 +479,6 @@ async function promoteToReadyHandler(
   };
 }
 
-export async function getPreviousEntry(ctx: QueryCtx, entry: Doc<"entries">) {
-  if (!entry.key) {
-    return null;
-  }
-  const previousEntry = await ctx.db
-    .query("entries")
-    .withIndex("namespaceId_status_key_version", (q) =>
-      q
-        .eq("namespaceId", entry.namespaceId)
-        .eq("status.kind", "ready")
-        .eq("key", entry.key),
-    )
-    .unique();
-  if (previousEntry?._id === entry._id) return null;
-  return previousEntry;
-}
-
-export function publicEntry(entry: {
-  _id: Id<"entries">;
-  key?: string | undefined;
-  importance: number;
-  filterValues: EntryFilter[];
-  contentHash?: string | undefined;
-  title?: string | undefined;
-  metadata?: Record<string, Value> | undefined;
-  status: StatusWithOnComplete;
-}): Entry {
-  const { key, importance, filterValues, contentHash, title, metadata } = entry;
-
-  const fields = {
-    entryId: entry._id as unknown as EntryId,
-    key,
-    title,
-    metadata,
-    importance,
-    filterValues,
-    contentHash,
-  };
-  if (entry.status.kind === "replaced") {
-    return {
-      ...fields,
-      status: "replaced" as const,
-      replacedAt: entry.status.replacedAt,
-    };
-  } else {
-    return {
-      ...fields,
-      status: entry.status.kind,
-    };
-  }
-}
-
 export const deleteAsync = mutation({
   args: v.object({
     entryId: v.id("entries"),
@@ -563,22 +511,24 @@ async function deleteAsyncHandler(
 export const deleteSync = action({
   args: { entryId: v.id("entries") },
   returns: v.null(),
-  handler: async (ctx, { entryId }) => {
-    let startOrder = 0;
-    while (true) {
-      const status = await ctx.runMutation(internal.chunks.deleteChunksPage, {
-        entryId,
-        startOrder,
-      });
-      if (status.isDone) {
-        await ctx.runMutation(internal.entries._del, { entryId });
-        break;
-      }
-      startOrder = status.nextStartOrder;
-    }
-  },
+  handler: async (ctx, { entryId }) => deleteEntrySync(ctx, entryId),
 });
 
+export async function deleteEntrySync(ctx: ActionCtx, entryId: Id<"entries">) {
+  let startOrder = 0;
+  while (true) {
+    const status = await ctx.runMutation(internal.chunks.deleteChunksPage, {
+      entryId,
+      startOrder,
+    });
+    if (status.isDone) {
+      await ctx.runMutation(internal.entries._del, { entryId });
+      break;
+    }
+    startOrder = status.nextStartOrder;
+  }
+}
+
 export const _del = internalMutation({
   args: { entryId: v.id("entries") },
   returns: v.null(),
@@ -656,9 +606,7 @@ export const deleteByKeySync = action({
         { namespaceId: args.namespaceId, key: args.key },
       );
       for await (const entry of entries) {
-        await ctx.runAction(api.entries.deleteSync, {
-          entryId: entry._id,
-        });
+        await deleteEntrySync(ctx, entry._id);
       }
       if (entries.length <= 100) {
         break;

package/src/component/helpers.ts

@@ -0,0 +1,124 @@
+import type { ObjectType, Value } from "convex/values";
+import {
+  Entry,
+  EntryFilter,
+  EntryId,
+  filterNamesContain,
+  type Namespace,
+  type NamespaceId,
+} from "../shared.js";
+import type { Doc, Id } from "./_generated/dataModel.js";
+import { type QueryCtx } from "./_generated/server.js";
+import { StatusWithOnComplete, v } from "./schema.js";
+
+export function publicNamespace(namespace: Doc<"namespaces">): Namespace {
+  const { _id, _creationTime, status, ...rest } = namespace;
+  return {
+    namespaceId: _id as unknown as NamespaceId,
+    createdAt: _creationTime,
+    ...rest,
+    status: status.kind,
+  };
+}
+
+export function publicEntry(entry: {
+  _id: Id<"entries">;
+  key?: string | undefined;
+  importance: number;
+  filterValues: EntryFilter[];
+  contentHash?: string | undefined;
+  title?: string | undefined;
+  metadata?: Record<string, Value> | undefined;
+  status: StatusWithOnComplete;
+}): Entry {
+  const { key, importance, filterValues, contentHash, title, metadata } = entry;
+
+  const fields = {
+    entryId: entry._id as unknown as EntryId,
+    key,
+    title,
+    metadata,
+    importance,
+    filterValues,
+    contentHash,
+  };
+  if (entry.status.kind === "replaced") {
+    return {
+      ...fields,
+      status: "replaced" as const,
+      replacedAt: entry.status.replacedAt,
+    };
+  } else {
+    return {
+      ...fields,
+      status: entry.status.kind,
+    };
+  }
+}
+
+export const vNamespaceLookupArgs = {
+  namespace: v.string(),
+  modelId: v.string(),
+  dimension: v.number(),
+  filterNames: v.array(v.string()),
+};
+
+export async function getCompatibleNamespaceHandler(
+  ctx: QueryCtx,
+  args: ObjectType<typeof vNamespaceLookupArgs>,
+) {
+  const iter = ctx.db
+    .query("namespaces")
+    .withIndex("status_namespace_version", (q) =>
+      q.eq("status.kind", "ready").eq("namespace", args.namespace),
+    )
+    .order("desc");
+  for await (const existing of iter) {
+    if (namespaceIsCompatible(existing, args)) {
+      return existing;
+    }
+  }
+  return null;
+}
+
+export function namespaceIsCompatible(
+  existing: Doc<"namespaces">,
+  args: {
+    modelId: string;
+    dimension: number;
+    filterNames: string[];
+  },
+) {
+  // Check basic compatibility
+  if (
+    existing.modelId !== args.modelId ||
+    existing.dimension !== args.dimension
+  ) {
+    return false;
+  }
+
+  // For filter names, the namespace must support all requested filters
+  // but can support additional filters (superset is OK)
+  if (!filterNamesContain(existing.filterNames, args.filterNames)) {
+    return false;
+  }
+
+  return true;
+}
+
+export async function getPreviousEntry(ctx: QueryCtx, entry: Doc<"entries">) {
+  if (!entry.key) {
+    return null;
+  }
+  const previousEntry = await ctx.db
+    .query("entries")
+    .withIndex("namespaceId_status_key_version", (q) =>
+      q
+        .eq("namespaceId", entry.namespaceId)
+        .eq("status.kind", "ready")
+        .eq("key", entry.key),
+    )
+    .unique();
+  if (previousEntry?._id === entry._id) return null;
+  return previousEntry;
+}

package/src/component/namespaces.test.ts

@@ -0,0 +1,97 @@
+/// <reference types="vite/client" />
+
+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+import { type TestConvex } from "convex-test";
+import schema from "./schema.js";
+import { api } from "./_generated/api.js";
+import { initConvexTest } from "./setup.test.js";
+import type { Id } from "./_generated/dataModel.js";
+
+type ConvexTest = TestConvex<typeof schema>;
+
+describe("namespaces", () => {
+  async function setupTestNamespace(t: ConvexTest) {
+    const namespace = await t.mutation(api.namespaces.getOrCreate, {
+      namespace: "test-namespace",
+      status: "ready",
+      modelId: "test-model",
+      dimension: 128,
+      filterNames: [],
+    });
+    return namespace.namespaceId;
+  }
+
+  function testEntryArgs(namespaceId: Id<"namespaces">, key: string) {
+    return {
+      namespaceId,
+      key,
+      importance: 0.5,
+      filterValues: [],
+      contentHash: `hash-${key}`,
+      title: `Entry ${key}`,
+    };
+  }
+
+  function testChunk() {
+    return {
+      content: { text: "test chunk" },
+      embedding: Array.from({ length: 128 }, () => Math.random()),
+      searchableText: "test chunk",
+    };
+  }
+
+  beforeEach(async () => {
+    vi.useFakeTimers();
+  });
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  test("deleteNamespaceSync deletes all entries then the namespace", async () => {
+    const t = initConvexTest();
+    const namespaceId = await setupTestNamespace(t);
+
+    // Create multiple entries so the pagination loop must handle them all
+    const entryIds: Id<"entries">[] = [];
+    for (let i = 0; i < 3; i++) {
+      const result = await t.mutation(api.entries.add, {
+        entry: testEntryArgs(namespaceId, `key-${i}`),
+        allChunks: [testChunk()],
+      });
+      expect(result.status).toBe("ready");
+      entryIds.push(result.entryId);
+    }
+
+    // Verify entries exist
+    const entriesBefore = await t.run(async (ctx) => {
+      return ctx.db
+        .query("entries")
+        .filter((q) => q.eq(q.field("namespaceId"), namespaceId))
+        .collect();
+    });
+    expect(entriesBefore).toHaveLength(3);
+
+    // This should delete all entries and then the namespace.
+    // Before the fix, this threw "cannot delete, has entries" because
+    // the pagination loop checked isDone before processing the page,
+    // skipping deletion when entries fit in a single page.
+    await t.action(api.namespaces.deleteNamespaceSync, {
+      namespaceId,
+    });
+
+    // Verify all entries are deleted
+    const entriesAfter = await t.run(async (ctx) => {
+      return ctx.db
+        .query("entries")
+        .filter((q) => q.eq(q.field("namespaceId"), namespaceId))
+        .collect();
+    });
+    expect(entriesAfter).toHaveLength(0);
+
+    // Verify namespace is deleted
+    const namespaceAfter = await t.run(async (ctx) => {
+      return ctx.db.get(namespaceId);
+    });
+    expect(namespaceAfter).toBeNull();
+  });
+});