@convex-dev/rag 0.7.1 → 0.7.2

package/src/component/entries.ts

@@ -1,11 +1,18 @@
+import {
+  vResultValidator,
+  vWorkIdValidator,
+  Workpool,
+} from "@convex-dev/workpool";
 import { assert, omit } from "convex-helpers";
+import { mergedStream, stream } from "convex-helpers/server/stream";
+import { doc } from "convex-helpers/validators";
 import {
   createFunctionHandle,
   paginationOptsValidator,
   PaginationResult,
 } from "convex/server";
-import { v, type Value } from "convex/values";
-import type { ChunkerAction, EntryFilter, EntryId } from "../shared.js";
+import { v } from "convex/values";
+import type { ChunkerAction, OnComplete } from "../shared.js";
 import {
   statuses,
   vActiveStatus,
@@ -15,7 +22,7 @@ import {
   vStatus,
   type Entry,
 } from "../shared.js";
-import { api, internal } from "./_generated/api.js";
+import { api, components, internal } from "./_generated/api.js";
 import type { Doc, Id } from "./_generated/dataModel.js";
 import {
   action,
@@ -23,26 +30,19 @@ import {
   internalQuery,
   mutation,
   query,
+  type ActionCtx,
   type MutationCtx,
   type QueryCtx,
 } from "./_generated/server.js";
 import { deleteChunksPageHandler, insertChunks } from "./chunks.js";
-import schema, { type StatusWithOnComplete } from "./schema.js";
-import { mergedStream } from "convex-helpers/server/stream";
-import { stream } from "convex-helpers/server/stream";
 import {
   getCompatibleNamespaceHandler,
+  getPreviousEntry,
+  publicEntry,
   publicNamespace,
   vNamespaceLookupArgs,
-} from "./namespaces.js";
-import type { OnComplete } from "../shared.js";
-import {
-  vResultValidator,
-  vWorkIdValidator,
-  Workpool,
-} from "@convex-dev/workpool";
-import { components } from "./_generated/api.js";
-import { doc } from "convex-helpers/validators";
+} from "./helpers.js";
+import schema, { type StatusWithOnComplete } from "./schema.js";
 
 const workpool = new Workpool(components.workpool, {
   retryActionsByDefault: true,
@@ -479,58 +479,6 @@ async function promoteToReadyHandler(
   };
 }
 
-export async function getPreviousEntry(ctx: QueryCtx, entry: Doc<"entries">) {
-  if (!entry.key) {
-    return null;
-  }
-  const previousEntry = await ctx.db
-    .query("entries")
-    .withIndex("namespaceId_status_key_version", (q) =>
-      q
-        .eq("namespaceId", entry.namespaceId)
-        .eq("status.kind", "ready")
-        .eq("key", entry.key),
-    )
-    .unique();
-  if (previousEntry?._id === entry._id) return null;
-  return previousEntry;
-}
-
-export function publicEntry(entry: {
-  _id: Id<"entries">;
-  key?: string | undefined;
-  importance: number;
-  filterValues: EntryFilter[];
-  contentHash?: string | undefined;
-  title?: string | undefined;
-  metadata?: Record<string, Value> | undefined;
-  status: StatusWithOnComplete;
-}): Entry {
-  const { key, importance, filterValues, contentHash, title, metadata } = entry;
-
-  const fields = {
-    entryId: entry._id as unknown as EntryId,
-    key,
-    title,
-    metadata,
-    importance,
-    filterValues,
-    contentHash,
-  };
-  if (entry.status.kind === "replaced") {
-    return {
-      ...fields,
-      status: "replaced" as const,
-      replacedAt: entry.status.replacedAt,
-    };
-  } else {
-    return {
-      ...fields,
-      status: entry.status.kind,
-    };
-  }
-}
-
 export const deleteAsync = mutation({
   args: v.object({
     entryId: v.id("entries"),
@@ -563,22 +511,24 @@ async function deleteAsyncHandler(
 export const deleteSync = action({
   args: { entryId: v.id("entries") },
   returns: v.null(),
-  handler: async (ctx, { entryId }) => {
-    let startOrder = 0;
-    while (true) {
-      const status = await ctx.runMutation(internal.chunks.deleteChunksPage, {
-        entryId,
-        startOrder,
-      });
-      if (status.isDone) {
-        await ctx.runMutation(internal.entries._del, { entryId });
-        break;
-      }
-      startOrder = status.nextStartOrder;
-    }
-  },
+  handler: async (ctx, { entryId }) => deleteEntrySync(ctx, entryId),
 });
 
+export async function deleteEntrySync(ctx: ActionCtx, entryId: Id<"entries">) {
+  let startOrder = 0;
+  while (true) {
+    const status = await ctx.runMutation(internal.chunks.deleteChunksPage, {
+      entryId,
+      startOrder,
+    });
+    if (status.isDone) {
+      await ctx.runMutation(internal.entries._del, { entryId });
+      break;
+    }
+    startOrder = status.nextStartOrder;
+  }
+}
+
 export const _del = internalMutation({
   args: { entryId: v.id("entries") },
   returns: v.null(),
@@ -656,9 +606,7 @@ export const deleteByKeySync = action({
         { namespaceId: args.namespaceId, key: args.key },
       );
       for await (const entry of entries) {
-        await ctx.runAction(api.entries.deleteSync, {
-          entryId: entry._id,
-        });
+        await deleteEntrySync(ctx, entry._id);
       }
       if (entries.length <= 100) {
         break;

package/src/component/helpers.ts

@@ -0,0 +1,124 @@
+import type { ObjectType, Value } from "convex/values";
+import {
+  Entry,
+  EntryFilter,
+  EntryId,
+  filterNamesContain,
+  type Namespace,
+  type NamespaceId,
+} from "../shared.js";
+import type { Doc, Id } from "./_generated/dataModel.js";
+import { type QueryCtx } from "./_generated/server.js";
+import { StatusWithOnComplete, v } from "./schema.js";
+
+export function publicNamespace(namespace: Doc<"namespaces">): Namespace {
+  const { _id, _creationTime, status, ...rest } = namespace;
+  return {
+    namespaceId: _id as unknown as NamespaceId,
+    createdAt: _creationTime,
+    ...rest,
+    status: status.kind,
+  };
+}
+
+export function publicEntry(entry: {
+  _id: Id<"entries">;
+  key?: string | undefined;
+  importance: number;
+  filterValues: EntryFilter[];
+  contentHash?: string | undefined;
+  title?: string | undefined;
+  metadata?: Record<string, Value> | undefined;
+  status: StatusWithOnComplete;
+}): Entry {
+  const { key, importance, filterValues, contentHash, title, metadata } = entry;
+
+  const fields = {
+    entryId: entry._id as unknown as EntryId,
+    key,
+    title,
+    metadata,
+    importance,
+    filterValues,
+    contentHash,
+  };
+  if (entry.status.kind === "replaced") {
+    return {
+      ...fields,
+      status: "replaced" as const,
+      replacedAt: entry.status.replacedAt,
+    };
+  } else {
+    return {
+      ...fields,
+      status: entry.status.kind,
+    };
+  }
+}
+
+export const vNamespaceLookupArgs = {
+  namespace: v.string(),
+  modelId: v.string(),
+  dimension: v.number(),
+  filterNames: v.array(v.string()),
+};
+
+export async function getCompatibleNamespaceHandler(
+  ctx: QueryCtx,
+  args: ObjectType<typeof vNamespaceLookupArgs>,
+) {
+  const iter = ctx.db
+    .query("namespaces")
+    .withIndex("status_namespace_version", (q) =>
+      q.eq("status.kind", "ready").eq("namespace", args.namespace),
+    )
+    .order("desc");
+  for await (const existing of iter) {
+    if (namespaceIsCompatible(existing, args)) {
+      return existing;
+    }
+  }
+  return null;
+}
+
+export function namespaceIsCompatible(
+  existing: Doc<"namespaces">,
+  args: {
+    modelId: string;
+    dimension: number;
+    filterNames: string[];
+  },
+) {
+  // Check basic compatibility
+  if (
+    existing.modelId !== args.modelId ||
+    existing.dimension !== args.dimension
+  ) {
+    return false;
+  }
+
+  // For filter names, the namespace must support all requested filters
+  // but can support additional filters (superset is OK)
+  if (!filterNamesContain(existing.filterNames, args.filterNames)) {
+    return false;
+  }
+
+  return true;
+}
+
+export async function getPreviousEntry(ctx: QueryCtx, entry: Doc<"entries">) {
+  if (!entry.key) {
+    return null;
+  }
+  const previousEntry = await ctx.db
+    .query("entries")
+    .withIndex("namespaceId_status_key_version", (q) =>
+      q
+        .eq("namespaceId", entry.namespaceId)
+        .eq("status.kind", "ready")
+        .eq("key", entry.key),
+    )
+    .unique();
+  if (previousEntry?._id === entry._id) return null;
+  return previousEntry;
+}

package/src/component/namespaces.test.ts

@@ -0,0 +1,97 @@
+/// <reference types="vite/client" />
+
+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+import { type TestConvex } from "convex-test";
+import schema from "./schema.js";
+import { api } from "./_generated/api.js";
+import { initConvexTest } from "./setup.test.js";
+import type { Id } from "./_generated/dataModel.js";
+
+type ConvexTest = TestConvex<typeof schema>;
+
+describe("namespaces", () => {
+  async function setupTestNamespace(t: ConvexTest) {
+    const namespace = await t.mutation(api.namespaces.getOrCreate, {
+      namespace: "test-namespace",
+      status: "ready",
+      modelId: "test-model",
+      dimension: 128,
+      filterNames: [],
+    });
+    return namespace.namespaceId;
+  }
+
+  function testEntryArgs(namespaceId: Id<"namespaces">, key: string) {
+    return {
+      namespaceId,
+      key,
+      importance: 0.5,
+      filterValues: [],
+      contentHash: `hash-${key}`,
+      title: `Entry ${key}`,
+    };
+  }
+
+  function testChunk() {
+    return {
+      content: { text: "test chunk" },
+      embedding: Array.from({ length: 128 }, () => Math.random()),
+      searchableText: "test chunk",
+    };
+  }
+
+  beforeEach(async () => {
+    vi.useFakeTimers();
+  });
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  test("deleteNamespaceSync deletes all entries then the namespace", async () => {
+    const t = initConvexTest();
+    const namespaceId = await setupTestNamespace(t);
+
+    // Create multiple entries so the pagination loop must handle them all
+    const entryIds: Id<"entries">[] = [];
+    for (let i = 0; i < 3; i++) {
+      const result = await t.mutation(api.entries.add, {
+        entry: testEntryArgs(namespaceId, `key-${i}`),
+        allChunks: [testChunk()],
+      });
+      expect(result.status).toBe("ready");
+      entryIds.push(result.entryId);
+    }
+
+    // Verify entries exist
+    const entriesBefore = await t.run(async (ctx) => {
+      return ctx.db
+        .query("entries")
+        .filter((q) => q.eq(q.field("namespaceId"), namespaceId))
+        .collect();
+    });
+    expect(entriesBefore).toHaveLength(3);
+
+    // This should delete all entries and then the namespace.
+    // Before the fix, this threw "cannot delete, has entries" because
+    // the pagination loop checked isDone before processing the page,
+    // skipping deletion when entries fit in a single page.
+    await t.action(api.namespaces.deleteNamespaceSync, {
+      namespaceId,
+    });
+
+    // Verify all entries are deleted
+    const entriesAfter = await t.run(async (ctx) => {
+      return ctx.db
+        .query("entries")
+        .filter((q) => q.eq(q.field("namespaceId"), namespaceId))
+        .collect();
+    });
+    expect(entriesAfter).toHaveLength(0);
+
+    // Verify namespace is deleted
+    const namespaceAfter = await t.run(async (ctx) => {
+      return ctx.db.get(namespaceId);
+    });
+    expect(namespaceAfter).toBeNull();
+  });
+});

package/src/component/namespaces.ts

@@ -1,3 +1,18 @@
+import { assert } from "convex-helpers";
+import { paginator } from "convex-helpers/server/pagination";
+import { mergedStream, stream } from "convex-helpers/server/stream";
+import { paginationOptsValidator, PaginationResult } from "convex/server";
+import type { Infer } from "convex/values";
+import {
+  statuses,
+  vActiveStatus,
+  vEntry,
+  vNamespace,
+  vPaginationResult,
+  vStatus,
+  type OnCompleteNamespace,
+} from "../shared.js";
+import { api } from "./_generated/api.js";
 import type { Doc, Id } from "./_generated/dataModel.js";
 import {
   action,
@@ -5,59 +20,15 @@ import {
   mutation,
   query,
   type MutationCtx,
-  type QueryCtx,
 } from "./_generated/server.js";
-import { schema, v } from "./schema.js";
+import { deleteEntrySync } from "./entries.js";
 import {
-  vNamespace,
-  vPaginationResult,
-  vActiveStatus,
-  type Namespace,
-  type NamespaceId,
-  type OnCompleteNamespace,
-  vStatus,
-  statuses,
-  filterNamesContain,
-  vEntry,
-} from "../shared.js";
-import { paginationOptsValidator, PaginationResult } from "convex/server";
-import { paginator } from "convex-helpers/server/pagination";
-import type { Infer, ObjectType } from "convex/values";
-import { mergedStream, stream } from "convex-helpers/server/stream";
-import { assert } from "convex-helpers";
-import { api } from "./_generated/api.js";
-
-function namespaceIsCompatible(
-  existing: Doc<"namespaces">,
-  args: {
-    modelId: string;
-    dimension: number;
-    filterNames: string[];
-  },
-) {
-  // Check basic compatibility
-  if (
-    existing.modelId !== args.modelId ||
-    existing.dimension !== args.dimension
-  ) {
-    return false;
-  }
-
-  // For filter names, the namespace must support all requested filters
-  // but can support additional filters (superset is OK)
-  if (!filterNamesContain(existing.filterNames, args.filterNames)) {
-    return false;
-  }
-
-  return true;
-}
-
-export const vNamespaceLookupArgs = {
-  namespace: v.string(),
-  modelId: v.string(),
-  dimension: v.number(),
-  filterNames: v.array(v.string()),
-};
+  getCompatibleNamespaceHandler,
+  namespaceIsCompatible,
+  publicNamespace,
+  vNamespaceLookupArgs,
+} from "./helpers.js";
+import { schema, v } from "./schema.js";
 
 export const get = query({
   args: vNamespaceLookupArgs,
@@ -77,24 +48,6 @@ export const getCompatibleNamespace = internalQuery({
   handler: getCompatibleNamespaceHandler,
 });
 
-export async function getCompatibleNamespaceHandler(
-  ctx: QueryCtx,
-  args: ObjectType<typeof vNamespaceLookupArgs>,
-) {
-  const iter = ctx.db
-    .query("namespaces")
-    .withIndex("status_namespace_version", (q) =>
-      q.eq("status.kind", "ready").eq("namespace", args.namespace),
-    )
-    .order("desc");
-  for await (const existing of iter) {
-    if (namespaceIsCompatible(existing, args)) {
-      return existing;
-    }
-  }
-  return null;
-}
-
 export const lookup = query({
   args: {
     namespace: v.string(),
@@ -312,16 +265,6 @@ export const listNamespaceVersions = query({
   },
 });
 
-export function publicNamespace(namespace: Doc<"namespaces">): Namespace {
-  const { _id, _creationTime, status, ...rest } = namespace;
-  return {
-    namespaceId: _id as unknown as NamespaceId,
-    createdAt: _creationTime,
-    ...rest,
-    status: status.kind,
-  };
-}
-
 export const deleteNamespace = mutation({
   args: { namespaceId: v.id("namespaces") },
   returns: v.object({
@@ -368,13 +311,13 @@ export const deleteNamespaceSync = action({
             cursor,
           },
         })) as PaginationResult<Infer<typeof vEntry>>;
+        for (const entry of entries.page) {
+          await deleteEntrySync(ctx, entry.entryId as unknown as Id<"entries">);
+        }
         if (entries.isDone) {
           break;
         }
         cursor = entries.continueCursor;
-        await ctx.runAction(api.entries.deleteSync, {
-          entryId: entries.page[0].entryId as unknown as Id<"entries">,
-        });
       }
     }
     await ctx.runMutation(api.namespaces.deleteNamespace, {