@convex-dev/rag 0.5.4 → 0.6.1

package/src/component/entries.ts

@@ -111,7 +111,7 @@ export const addAsync = mutation({
         name: workpoolName(namespace.namespace, args.entry.key, entryId),
         onComplete: internal.entries.addAsyncOnComplete,
         context: entryId,
-      }
+      },
     );
     return { entryId, status: status.kind, created: true };
   },
@@ -120,7 +120,7 @@ export const addAsync = mutation({
 function workpoolName(
   namespace: string,
   key: string | undefined,
-  entryId: Id<"entries">
+  entryId: Id<"entries">,
 ) {
   return `rag-async-${namespace}-${key ? key + "-" + entryId : entryId}`;
 }
@@ -137,7 +137,7 @@ export const addAsyncOnComplete = internalMutation({
     const entry = await ctx.db.get(args.context);
     if (!entry) {
       console.error(
-        `Entry ${args.context} not found when trying to complete chunker for async add`
+        `Entry ${args.context} not found when trying to complete chunker for async add`,
       );
       return;
     }
@@ -154,7 +154,7 @@ export const addAsyncOnComplete = internalMutation({
           namespace,
           entry,
           null,
-          args.result.kind === "canceled" ? "Canceled" : args.result.error
+          args.result.kind === "canceled" ? "Canceled" : args.result.error,
         );
       }
     }
@@ -169,7 +169,7 @@ type AddEntryArgs = Pick<
 async function findExistingEntry(
   ctx: MutationCtx,
   namespaceId: Id<"namespaces">,
-  key: string | undefined
+  key: string | undefined,
 ) {
   if (!key) {
     return null;
@@ -182,11 +182,11 @@ async function findExistingEntry(
           q
             .eq("namespaceId", namespaceId)
             .eq("status.kind", status)
-            .eq("key", key)
+            .eq("key", key),
         )
-        .order("desc")
+        .order("desc"),
     ),
-    ["version"]
+    ["version"],
   ).first();
   return existing;
 }
@@ -256,7 +256,7 @@ async function runOnComplete(
   namespace: Doc<"namespaces">,
   entry: Doc<"entries">,
   replacedEntry: Doc<"entries"> | null,
-  error?: string
+  error?: string,
 ) {
   await ctx.runMutation(onComplete as unknown as OnComplete, {
     namespace: publicNamespace(namespace),
@@ -282,8 +282,8 @@ function entryIsSame(existing: Doc<"entries">, newEntry: AddEntryArgs) {
   if (
     !existing.filterValues.every((filter) =>
       newEntry.filterValues.some(
-        (f) => f.name === filter.name && f.value === filter.value
-      )
+        (f) => f.name === filter.name && f.value === filter.value,
+      ),
     )
   ) {
     return false;
@@ -309,7 +309,7 @@ export const list = query({
       .withIndex("status_namespaceId", (q) =>
         namespaceId
           ? q.eq("status.kind", args.status).eq("namespaceId", namespaceId)
-          : q.eq("status.kind", args.status)
+          : q.eq("status.kind", args.status),
       )
       .order(args.order ?? "asc")
       .paginate(args.paginationOpts);
@@ -359,16 +359,16 @@ export const findByContentHash = query({
             q
               .eq("namespaceId", namespace._id)
               .eq("status.kind", status)
-              .eq("key", args.key)
+              .eq("key", args.key),
           )
-          .order("desc")
+          .order("desc"),
       ),
-      ["version"]
+      ["version"],
     )) {
       attempts++;
       if (attempts > 20) {
         console.debug(
-          `Giving up after checking ${attempts} entries for ${args.key} content hash ${args.contentHash}, returning null`
+          `Giving up after checking ${attempts} entries for ${args.key} content hash ${args.contentHash}, returning null`,
         );
         return null;
       }
@@ -407,7 +407,7 @@ export const promoteToReady = mutation({
 
 async function promoteToReadyHandler(
   ctx: MutationCtx,
-  args: { entryId: Id<"entries"> }
+  args: { entryId: Id<"entries"> },
 ) {
   const entry = await ctx.db.get(args.entryId);
   assert(entry, `Entry ${args.entryId} not found`);
@@ -418,7 +418,7 @@ async function promoteToReadyHandler(
     return { replacedEntry: null };
   } else if (entry.status.kind === "replaced") {
     console.debug(
-      `Entry ${args.entryId} is already replaced, returning the current version...`
+      `Entry ${args.entryId} is already replaced, returning the current version...`,
     );
     return { replacedEntry: publicEntry(entry) };
   }
@@ -441,7 +441,7 @@ async function promoteToReadyHandler(
       previousStatus.onComplete,
       namespace,
       entry,
-      previousEntry
+      previousEntry,
     );
   }
   // Then mark all previous pending entries as replaced,
@@ -454,7 +454,7 @@ async function promoteToReadyHandler(
           .eq("namespaceId", entry.namespaceId)
           .eq("status.kind", "pending")
           .eq("key", entry.key)
-          .lt("version", entry.version)
+          .lt("version", entry.version),
       )
       .collect();
     await Promise.all(
@@ -468,10 +468,10 @@ async function promoteToReadyHandler(
             previousStatus.onComplete,
             namespace,
             entry,
-            null
+            null,
           );
         }
-      })
+      }),
     );
   }
   return {
@@ -489,7 +489,7 @@ export async function getPreviousEntry(ctx: QueryCtx, entry: Doc<"entries">) {
       q
         .eq("namespaceId", entry.namespaceId)
         .eq("status.kind", "ready")
-        .eq("key", entry.key)
+        .eq("key", entry.key),
     )
     .unique();
   if (previousEntry?._id === entry._id) return null;
@@ -542,7 +542,7 @@ export const deleteAsync = mutation({
 
 async function deleteAsyncHandler(
   ctx: MutationCtx,
-  args: { entryId: Id<"entries">; startOrder: number }
+  args: { entryId: Id<"entries">; startOrder: number },
 ) {
   const { entryId, startOrder } = args;
   const entry = await ctx.db.get(entryId);
@@ -614,7 +614,7 @@ export const deleteByKeyAsync = mutation({
 
 async function getEntriesByKey(
   ctx: QueryCtx,
-  args: { namespaceId: Id<"namespaces">; key: string; beforeVersion?: number }
+  args: { namespaceId: Id<"namespaces">; key: string; beforeVersion?: number },
 ): Promise<Doc<"entries">[]> {
   return mergedStream(
     statuses.map((status) =>
@@ -625,11 +625,11 @@ async function getEntriesByKey(
             .eq("namespaceId", args.namespaceId)
             .eq("status.kind", status)
             .eq("key", args.key)
-            .lt("version", args.beforeVersion ?? Infinity)
+            .lt("version", args.beforeVersion ?? Infinity),
         )
-        .order("desc")
+        .order("desc"),
     ),
-    ["version"]
+    ["version"],
   ).take(100);
 }
 
@@ -653,7 +653,7 @@ export const deleteByKeySync = action({
     while (true) {
       const entries: Doc<"entries">[] = await ctx.runQuery(
         internal.entries.getEntriesForNamespaceByKey,
-        { namespaceId: args.namespaceId, key: args.key }
+        { namespaceId: args.namespaceId, key: args.key },
       );
       for await (const entry of entries) {
         await ctx.runAction(api.entries.deleteSync, {

package/src/component/filters.ts

@@ -55,7 +55,7 @@ export type NamedFilter<K extends string = string, V = Value> = {
  */
 export function filterFieldsFromNumbers(
   namespaceId: GenericId<"namespaces">,
-  filters: NumberedFilter | undefined
+  filters: NumberedFilter | undefined,
 ): NamedFilterField {
   const filterFields: NamedFilterField = {};
   if (!filters) return filterFields;
@@ -63,7 +63,7 @@ export function filterFieldsFromNumbers(
     const index = Number(i);
     if (isNaN(index) || index < 0 || index >= filterFieldNames.length) {
       console.warn(
-        `Unknown filter index: ${index} for value ${JSON.stringify(filter)}`
+        `Unknown filter index: ${index} for value ${JSON.stringify(filter)}`,
       );
       break;
     }
@@ -78,7 +78,7 @@ export function filterFieldsFromNumbers(
  */
 export function numberedFilterFromNamedFilters(
   namedFilters: Array<{ name: string; value: Value }>,
-  filterNames: string[]
+  filterNames: string[],
 ): NumberedFilter {
   const numberedFilter: NumberedFilter = {};
   for (const namedFilter of namedFilters) {
@@ -86,8 +86,8 @@ export function numberedFilterFromNamedFilters(
     if (index === -1) {
       throw new Error(
         `Unknown filter name: ${namedFilter.name} for namespace with names ${filterNames.join(
-          ", "
-        )}`
+          ", ",
+        )}`,
       );
     }
     numberedFilter[index] = namedFilter.value;
@@ -101,7 +101,7 @@ export function numberedFilterFromNamedFilters(
  */
 export function numberedFiltersFromNamedFilters(
   filters: NamedFilter[],
-  filterNames: string[]
+  filterNames: string[],
 ): Array<NumberedFilter> {
   const filterFields: Array<NumberedFilter> = [];
   for (const filter of filters) {
@@ -109,8 +109,8 @@ export function numberedFiltersFromNamedFilters(
     if (index === -1) {
       throw new Error(
         `Unknown filter name: ${filter.name} for namespace with names ${filterNames.join(
-          ", "
-        )}`
+          ", ",
+        )}`,
       );
     }
     filterFields.push({ [index]: filter.value });

package/src/component/namespaces.ts

@@ -18,11 +18,11 @@ import {
   vStatus,
   statuses,
   filterNamesContain,
-  Entry,
+  vEntry,
 } from "../shared.js";
 import { paginationOptsValidator, PaginationResult } from "convex/server";
 import { paginator } from "convex-helpers/server/pagination";
-import type { ObjectType } from "convex/values";
+import type { Infer, ObjectType } from "convex/values";
 import { mergedStream, stream } from "convex-helpers/server/stream";
 import { assert } from "convex-helpers";
 import { api } from "./_generated/api.js";
@@ -33,7 +33,7 @@ function namespaceIsCompatible(
     modelId: string;
     dimension: number;
     filterNames: string[];
-  }
+  },
 ) {
   // Check basic compatibility
   if (
@@ -79,12 +79,12 @@ export const getCompatibleNamespace = internalQuery({
 
 export async function getCompatibleNamespaceHandler(
   ctx: QueryCtx,
-  args: ObjectType<typeof vNamespaceLookupArgs>
+  args: ObjectType<typeof vNamespaceLookupArgs>,
 ) {
   const iter = ctx.db
     .query("namespaces")
     .withIndex("status_namespace_version", (q) =>
-      q.eq("status.kind", "ready").eq("namespace", args.namespace)
+      q.eq("status.kind", "ready").eq("namespace", args.namespace),
     )
     .order("desc");
   for await (const existing of iter) {
@@ -132,11 +132,11 @@ export const getOrCreate = mutation({
         stream(ctx.db, schema)
           .query("namespaces")
           .withIndex("status_namespace_version", (q) =>
-            q.eq("status.kind", status).eq("namespace", args.namespace)
+            q.eq("status.kind", status).eq("namespace", args.namespace),
           )
-          .order("desc")
+          .order("desc"),
       ),
-      ["version"]
+      ["version"],
     );
 
     let version: number = 0;
@@ -172,7 +172,7 @@ async function runOnComplete(
   ctx: MutationCtx,
   onComplete: string | undefined,
   namespace: Doc<"namespaces">,
-  replacedNamespace: Doc<"namespaces"> | null
+  replacedNamespace: Doc<"namespaces"> | null,
 ) {
   const onCompleteFn = onComplete as unknown as OnCompleteNamespace;
   if (!onCompleteFn) {
@@ -198,25 +198,25 @@ export const promoteToReady = mutation({
 
 async function promoteToReadyHandler(
   ctx: MutationCtx,
-  args: { namespaceId: Id<"namespaces"> }
+  args: { namespaceId: Id<"namespaces"> },
 ) {
   const namespace = await ctx.db.get(args.namespaceId);
   assert(namespace, `Namespace ${args.namespaceId} not found`);
   if (namespace.status.kind === "ready") {
     console.debug(
-      `Namespace ${args.namespaceId} is already ready, not promoting`
+      `Namespace ${args.namespaceId} is already ready, not promoting`,
     );
     return { replacedNamespace: null };
   } else if (namespace.status.kind === "replaced") {
     console.debug(
-      `Namespace ${args.namespaceId} is already replaced, not promoting and returning itself`
+      `Namespace ${args.namespaceId} is already replaced, not promoting and returning itself`,
     );
     return { replacedNamespace: publicNamespace(namespace) };
   }
   const previousNamespace = await ctx.db
     .query("namespaces")
     .withIndex("status_namespace_version", (q) =>
-      q.eq("status.kind", "ready").eq("namespace", namespace.namespace)
+      q.eq("status.kind", "ready").eq("namespace", namespace.namespace),
     )
     .order("desc")
     .unique();
@@ -237,7 +237,7 @@ async function promoteToReadyHandler(
       ctx,
       previousStatus.onComplete,
       namespace,
-      previousNamespace
+      previousNamespace,
     );
   }
   const previousPendingNamespaces = await ctx.db
@@ -246,7 +246,7 @@ async function promoteToReadyHandler(
       q
         .eq("status.kind", "pending")
         .eq("namespace", namespace.namespace)
-        .lt("version", namespace.version)
+        .lt("version", namespace.version),
     )
     .collect();
   // Then mark all previous pending namespaces as replaced,
@@ -259,7 +259,7 @@ async function promoteToReadyHandler(
       if (previousStatus.kind === "pending" && previousStatus.onComplete) {
         await runOnComplete(ctx, previousStatus.onComplete, namespace, null);
       }
-    })
+    }),
   );
   return {
     replacedNamespace: previousNamespace
@@ -278,7 +278,7 @@ export const list = query({
     const namespaces = await paginator(ctx.db, schema)
       .query("namespaces")
       .withIndex("status_namespace_version", (q) =>
-        q.eq("status.kind", args.status ?? "ready")
+        q.eq("status.kind", args.status ?? "ready"),
       )
       .order("desc")
       .paginate(args.paginationOpts);
@@ -298,11 +298,11 @@ export const listNamespaceVersions = query({
         stream(ctx.db, schema)
           .query("namespaces")
           .withIndex("status_namespace_version", (q) =>
-            q.eq("status.kind", status).eq("namespace", args.namespace)
+            q.eq("status.kind", status).eq("namespace", args.namespace),
           )
-          .order("desc")
+          .order("desc"),
       ),
-      ["version"]
+      ["version"],
     ).paginate(args.paginationOpts);
 
     return {
@@ -332,21 +332,21 @@ export const deleteNamespace = mutation({
 
 async function deleteHandler(
   ctx: MutationCtx,
-  args: { namespaceId: Id<"namespaces"> }
+  args: { namespaceId: Id<"namespaces"> },
 ) {
   const namespace = await ctx.db.get(args.namespaceId);
   assert(namespace, `Namespace ${args.namespaceId} not found`);
   const anyEntry = await ctx.db
     .query("entries")
     .withIndex("namespaceId_status_key_version", (q) =>
-      q.eq("namespaceId", args.namespaceId)
+      q.eq("namespaceId", args.namespaceId),
     )
     .first();
   if (anyEntry) {
     throw new Error(
       `Namespace ${args.namespaceId} cannot delete, has entries` +
         "First delete all entries." +
-        `Entry: ${anyEntry.key} id ${anyEntry._id} (${anyEntry.status.kind})`
+        `Entry: ${anyEntry.key} id ${anyEntry._id} (${anyEntry.status.kind})`,
     );
   }
   await ctx.db.delete(args.namespaceId);
@@ -360,17 +360,14 @@ export const deleteNamespaceSync = action({
     for (const status of statuses) {
       let cursor: string | null = null;
       while (true) {
-        const entries: PaginationResult<Entry> = await ctx.runQuery(
-          api.entries.list,
-          {
-            namespaceId: args.namespaceId,
-            status: status,
-            paginationOpts: {
-              numItems: 100,
-              cursor,
-            },
-          }
-        );
+        const entries = (await ctx.runQuery(api.entries.list, {
+          namespaceId: args.namespaceId,
+          status: status,
+          paginationOpts: {
+            numItems: 100,
+            cursor,
+          },
+        })) as PaginationResult<Infer<typeof vEntry>>;
         if (entries.isDone) {
           break;
         }

package/src/component/schema.ts

@@ -20,7 +20,7 @@ export const vStatusWithOnComplete = v.union(
   v.object({
     kind: v.literal("replaced"),
     replacedAt: v.number(),
-  })
+  }),
 );
 
 export type StatusWithOnComplete = Infer<typeof vStatusWithOnComplete>;
@@ -79,7 +79,7 @@ export const schema = defineSchema({
         embeddingId: vVectorId,
         vector: v.array(v.number()),
         pendingSearchableText: v.optional(v.string()),
-      })
+      }),
     ),
     // TODO: should content be inline?
     contentId: v.id("content"),

package/src/component/search.test.ts

@@ -16,7 +16,7 @@ describe("search", () => {
     t: ConvexTest,
     namespace = "test-namespace",
     dimension = 128,
-    filterNames: string[] = []
+    filterNames: string[] = [],
   ) {
     return await t.run(async (ctx) => {
       return ctx.db.insert("namespaces", {
@@ -35,7 +35,7 @@ describe("search", () => {
     namespaceId: Id<"namespaces">,
     key = "test-entry",
     version = 0,
-    filterValues: Array<{ name: string; value: Value }> = []
+    filterValues: Array<{ name: string; value: Value }> = [],
   ) {
     return await t.run(async (ctx) => {
       return ctx.db.insert("entries", {
@@ -207,7 +207,7 @@ describe("search", () => {
 
     // With threshold should return fewer results
     expect(resultWithThreshold.results.length).toBeLessThan(
-      resultWithoutThreshold.results.length
+      resultWithoutThreshold.results.length,
     );
     expect(resultWithoutThreshold.results).toHaveLength(2);
 
@@ -305,7 +305,7 @@ describe("search", () => {
       t,
       "multi-filter-namespace",
       128,
-      ["category", "priority_category"]
+      ["category", "priority_category"],
     );
 
     // Create entries with different filter combinations
@@ -438,7 +438,7 @@ describe("search", () => {
     // Results should be sorted by score (best first)
     for (let i = 1; i < result.results.length; i++) {
       expect(result.results[i - 1].score).toBeGreaterThanOrEqual(
-        result.results[i].score
+        result.results[i].score,
       );
     }
   });

package/src/component/search.ts

@@ -5,7 +5,6 @@ import { numberedFiltersFromNamedFilters, vNamedFilter } from "./filters.js";
 import { internal } from "./_generated/api.js";
 import {
   vEntry,
-  type Entry,
   vSearchResult,
   type SearchResult,
   type EntryId,
@@ -22,7 +21,7 @@ export const search = action({
     limit: v.number(),
     vectorScoreThreshold: v.optional(v.number()),
     chunkContext: v.optional(
-      v.object({ before: v.number(), after: v.number() })
+      v.object({ before: v.number(), after: v.number() }),
     ),
   },
   returns: v.object({
@@ -31,10 +30,10 @@ export const search = action({
   }),
   handler: async (
     ctx,
-    args
+    args,
   ): Promise<{
     results: SearchResult[];
-    entries: Entry[];
+    entries: Infer<typeof vEntry>[];
   }> => {
     const { modelId, embedding, filters, limit } = args;
     const namespace = await ctx.runQuery(
@@ -44,11 +43,11 @@ export const search = action({
         modelId,
         dimension: embedding.length,
         filterNames: filters.map((f) => f.name),
-      }
+      },
     );
     if (!namespace) {
       console.debug(
-        `No compatible namespace found for ${args.namespace} with model ${args.modelId} and dimension ${embedding.length} and filters ${filters.map((f) => f.name).join(", ")}.`
+        `No compatible namespace found for ${args.namespace} with model ${args.modelId} and dimension ${embedding.length} and filters ${filters.map((f) => f.name).join(", ")}.`,
       );
       return {
         results: [],
@@ -71,20 +70,20 @@ export const search = action({
       {
         embeddingIds: aboveThreshold.map((r) => r._id),
         chunkContext,
-      }
+      },
     );
     return {
       results: ranges
         .map((r, i) => publicSearchResult(r, aboveThreshold[i]._score))
         .filter((r) => r !== null),
-      entries,
+      entries: entries as Infer<typeof vEntry>[],
     };
   },
 });
 
 function publicSearchResult(
   r: Infer<typeof vRangeResult> | null,
-  score: number
+  score: number,
 ): SearchResult | null {
   if (r === null) {
     return null;

package/src/component/setup.test.ts

@@ -3,17 +3,11 @@ import { test } from "vitest";
 import { convexTest } from "convex-test";
 import schema from "./schema.js";
 export const modules = import.meta.glob("./**/*.*s");
-
-// Sorry about everything
-import componentSchema from "../../node_modules/@convex-dev/workpool/src/component/schema.js";
-export { componentSchema };
-export const componentModules = import.meta.glob(
-  "../../node_modules/@convex-dev/workpool/src/component/**/*.ts"
-);
+import workpool from "@convex-dev/workpool/test";
 
 export function initConvexTest() {
   const t = convexTest(schema, modules);
-  t.registerComponent("workpool", componentSchema, componentModules);
+  t.registerComponent("workpool", workpool.schema, workpool.modules);
   return t;
 }