@convex-dev/rag 0.5.4 → 0.6.0

package/src/component/embeddings/index.ts

@@ -47,7 +47,7 @@ export async function insertEmbedding(
   embedding: number[],
   namespaceId: Id<"namespaces">,
   importance: number | undefined,
-  filters: NumberedFilter | undefined
+  filters: NumberedFilter | undefined,
 ) {
   const filterFields = filterFieldsFromNumbers(namespaceId, filters);
   const dimension = validateVectorDimension(embedding.length);
@@ -74,12 +74,12 @@ export async function searchEmbeddings(
     // filter3, filter1, or filter2 is present.
     filters: Array<NumberedFilter>;
     limit: number;
-  }
+  },
 ) {
   const dimension = validateVectorDimension(embedding.length);
   const tableName = getVectorTableName(dimension);
   const orFilters = filters.flatMap((filter) =>
-    filterFieldsFromNumbers(namespaceId, filter)
+    filterFieldsFromNumbers(namespaceId, filter),
   );
   return ctx.vectorSearch(tableName, "vector", {
     vector: searchVector(embedding),
@@ -89,9 +89,9 @@ export async function searchEmbeddings(
         : q.or(
             ...orFilters.flatMap((namedFilter) =>
               Object.entries(namedFilter).map(([filterField, filter]) =>
-                q.eq(filterField as keyof (typeof orFilters)[number], filter)
-              )
-            )
+                q.eq(filterField as keyof (typeof orFilters)[number], filter),
+              ),
+            ),
           ),
     limit,
   });

package/src/component/embeddings/tables.ts

@@ -1,6 +1,7 @@
 import { literals } from "convex-helpers/validators";
 import {
   defineTable,
+  GenericTableIndexes,
   type GenericTableSearchIndexes,
   type SchemaDefinition,
   type TableDefinition,
@@ -32,7 +33,7 @@ function table(dimensions: VectorDimension): Table {
 
 type Table = TableDefinition<
   VObject<ObjectType<typeof embeddingsFields>, typeof embeddingsFields>,
-  { model_table_threadId: ["model", "table", "threadId", "_creationTime"] },
+  GenericTableIndexes,
   GenericTableSearchIndexes,
   VectorIndex
 >;
@@ -41,7 +42,7 @@ type VectorIndex = {
   vector: {
     vectorField: "vector";
     dimensions: number;
-    filterFields: string;
+    filterFields: keyof typeof vAllFilterFields & string;
   };
 };
 
@@ -55,11 +56,11 @@ export const VectorDimensions = [
 ] as const;
 
 export function assertVectorDimension(
-  dimension: number
+  dimension: number,
 ): asserts dimension is VectorDimension {
   if (!VectorDimensions.includes(dimension as VectorDimension)) {
     throw new Error(
-      `Unsupported vector dimension${dimension}. Supported: ${VectorDimensions.join(", ")}`
+      `Unsupported vector dimension${dimension}. Supported: ${VectorDimensions.join(", ")}`,
     );
   }
 }
@@ -67,14 +68,14 @@ export function assertVectorDimension(
 export function validateVectorDimension(dimension: number): VectorDimension {
   if (!VectorDimensions.includes(dimension as VectorDimension)) {
     throw new Error(
-      `Unsupported vector dimension${dimension}. Supported: ${VectorDimensions.join(", ")}`
+      `Unsupported vector dimension${dimension}. Supported: ${VectorDimensions.join(", ")}`,
     );
   }
   return dimension as VectorDimension;
 }
 export type VectorDimension = (typeof VectorDimensions)[number];
 export const VectorTableNames = VectorDimensions.map(
-  (d) => `vectors_${d}`
+  (d) => `vectors_${d}`,
 ) as `vectors_${(typeof VectorDimensions)[number]}`[];
 export type VectorTableName = (typeof VectorTableNames)[number];
 export type VectorTableId = GenericId<(typeof VectorTableNames)[number]>;
@@ -82,7 +83,7 @@ export type VectorTableId = GenericId<(typeof VectorTableNames)[number]>;
 export const vVectorDimension = literals(...VectorDimensions);
 export const vVectorTableName = literals(...VectorTableNames);
 export const vVectorId = v.union(
-  ...VectorTableNames.map((name) => v.id(name))
+  ...VectorTableNames.map((name) => v.id(name)),
 ) as VUnion<
   GenericId<(typeof VectorTableNames)[number]>,
   VId<(typeof VectorTableNames)[number]>[]
@@ -108,7 +109,7 @@ const tables: {
   VectorDimensions.map((dimensions) => [
     `vectors_${dimensions}`,
     table(dimensions),
-  ])
+  ]),
 ) as Record<`vectors_${(typeof VectorDimensions)[number]}`, Table>;
 
 export default tables;

package/src/component/entries.test.ts

@@ -97,8 +97,8 @@ describe("entries", () => {
         .filter((q) =>
           q.and(
             q.eq(q.field("namespaceId"), namespaceId),
-            q.eq(q.field("key"), entry.key)
-          )
+            q.eq(q.field("key"), entry.key),
+          ),
         )
         .collect();
     });
@@ -143,8 +143,8 @@ describe("entries", () => {
         .filter((q) =>
           q.and(
             q.eq(q.field("namespaceId"), namespaceId),
-            q.eq(q.field("key"), entry.key)
-          )
+            q.eq(q.field("key"), entry.key),
+          ),
         )
         .collect();
     });
@@ -537,7 +537,7 @@ describe("entries", () => {
       {
         namespaceId,
         key: "shared-key",
-      }
+      },
     );
     expect(sharedBefore).toHaveLength(2);
 
@@ -563,7 +563,7 @@ describe("entries", () => {
 
     const sharedAfter = await t.query(
       internal.entries.getEntriesForNamespaceByKey,
-      { namespaceId, key: "shared-key" }
+      { namespaceId, key: "shared-key" },
     );
     expect(sharedAfter).toHaveLength(0);
 
@@ -722,8 +722,8 @@ describe("entries", () => {
         .filter((q) =>
           q.and(
             q.eq(q.field("namespaceId"), namespaceId),
-            q.eq(q.field("key"), "versioned-key")
-          )
+            q.eq(q.field("key"), "versioned-key"),
+          ),
         )
         .collect();
     });
@@ -747,8 +747,8 @@ describe("entries", () => {
         .filter((q) =>
           q.and(
             q.eq(q.field("namespaceId"), namespaceId),
-            q.eq(q.field("key"), "versioned-key")
-          )
+            q.eq(q.field("key"), "versioned-key"),
+          ),
         )
         .collect();
     });

package/src/component/entries.ts

@@ -111,7 +111,7 @@ export const addAsync = mutation({
         name: workpoolName(namespace.namespace, args.entry.key, entryId),
         onComplete: internal.entries.addAsyncOnComplete,
         context: entryId,
-      }
+      },
     );
     return { entryId, status: status.kind, created: true };
   },
@@ -120,7 +120,7 @@ export const addAsync = mutation({
 function workpoolName(
   namespace: string,
   key: string | undefined,
-  entryId: Id<"entries">
+  entryId: Id<"entries">,
 ) {
   return `rag-async-${namespace}-${key ? key + "-" + entryId : entryId}`;
 }
@@ -137,7 +137,7 @@ export const addAsyncOnComplete = internalMutation({
     const entry = await ctx.db.get(args.context);
     if (!entry) {
       console.error(
-        `Entry ${args.context} not found when trying to complete chunker for async add`
+        `Entry ${args.context} not found when trying to complete chunker for async add`,
       );
       return;
     }
@@ -154,7 +154,7 @@ export const addAsyncOnComplete = internalMutation({
           namespace,
           entry,
           null,
-          args.result.kind === "canceled" ? "Canceled" : args.result.error
+          args.result.kind === "canceled" ? "Canceled" : args.result.error,
         );
       }
     }
@@ -169,7 +169,7 @@ type AddEntryArgs = Pick<
 async function findExistingEntry(
   ctx: MutationCtx,
   namespaceId: Id<"namespaces">,
-  key: string | undefined
+  key: string | undefined,
 ) {
   if (!key) {
     return null;
@@ -182,11 +182,11 @@ async function findExistingEntry(
           q
             .eq("namespaceId", namespaceId)
             .eq("status.kind", status)
-            .eq("key", key)
+            .eq("key", key),
         )
-        .order("desc")
+        .order("desc"),
     ),
-    ["version"]
+    ["version"],
   ).first();
   return existing;
 }
@@ -256,7 +256,7 @@ async function runOnComplete(
   namespace: Doc<"namespaces">,
   entry: Doc<"entries">,
   replacedEntry: Doc<"entries"> | null,
-  error?: string
+  error?: string,
 ) {
   await ctx.runMutation(onComplete as unknown as OnComplete, {
     namespace: publicNamespace(namespace),
@@ -282,8 +282,8 @@ function entryIsSame(existing: Doc<"entries">, newEntry: AddEntryArgs) {
   if (
     !existing.filterValues.every((filter) =>
       newEntry.filterValues.some(
-        (f) => f.name === filter.name && f.value === filter.value
-      )
+        (f) => f.name === filter.name && f.value === filter.value,
+      ),
     )
   ) {
     return false;
@@ -309,7 +309,7 @@ export const list = query({
       .withIndex("status_namespaceId", (q) =>
         namespaceId
           ? q.eq("status.kind", args.status).eq("namespaceId", namespaceId)
-          : q.eq("status.kind", args.status)
+          : q.eq("status.kind", args.status),
       )
       .order(args.order ?? "asc")
       .paginate(args.paginationOpts);
@@ -359,16 +359,16 @@ export const findByContentHash = query({
             q
               .eq("namespaceId", namespace._id)
               .eq("status.kind", status)
-              .eq("key", args.key)
+              .eq("key", args.key),
           )
-          .order("desc")
+          .order("desc"),
       ),
-      ["version"]
+      ["version"],
     )) {
       attempts++;
       if (attempts > 20) {
         console.debug(
-          `Giving up after checking ${attempts} entries for ${args.key} content hash ${args.contentHash}, returning null`
+          `Giving up after checking ${attempts} entries for ${args.key} content hash ${args.contentHash}, returning null`,
         );
         return null;
       }
@@ -407,7 +407,7 @@ export const promoteToReady = mutation({
 
 async function promoteToReadyHandler(
   ctx: MutationCtx,
-  args: { entryId: Id<"entries"> }
+  args: { entryId: Id<"entries"> },
 ) {
   const entry = await ctx.db.get(args.entryId);
   assert(entry, `Entry ${args.entryId} not found`);
@@ -418,7 +418,7 @@ async function promoteToReadyHandler(
     return { replacedEntry: null };
   } else if (entry.status.kind === "replaced") {
     console.debug(
-      `Entry ${args.entryId} is already replaced, returning the current version...`
+      `Entry ${args.entryId} is already replaced, returning the current version...`,
     );
     return { replacedEntry: publicEntry(entry) };
   }
@@ -441,7 +441,7 @@ async function promoteToReadyHandler(
       previousStatus.onComplete,
       namespace,
       entry,
-      previousEntry
+      previousEntry,
     );
   }
   // Then mark all previous pending entries as replaced,
@@ -454,7 +454,7 @@ async function promoteToReadyHandler(
           .eq("namespaceId", entry.namespaceId)
           .eq("status.kind", "pending")
           .eq("key", entry.key)
-          .lt("version", entry.version)
+          .lt("version", entry.version),
       )
       .collect();
     await Promise.all(
@@ -468,10 +468,10 @@ async function promoteToReadyHandler(
             previousStatus.onComplete,
             namespace,
             entry,
-            null
+            null,
           );
         }
-      })
+      }),
     );
   }
   return {
@@ -489,7 +489,7 @@ export async function getPreviousEntry(ctx: QueryCtx, entry: Doc<"entries">) {
       q
         .eq("namespaceId", entry.namespaceId)
         .eq("status.kind", "ready")
-        .eq("key", entry.key)
+        .eq("key", entry.key),
     )
     .unique();
   if (previousEntry?._id === entry._id) return null;
@@ -542,7 +542,7 @@ export const deleteAsync = mutation({
 
 async function deleteAsyncHandler(
   ctx: MutationCtx,
-  args: { entryId: Id<"entries">; startOrder: number }
+  args: { entryId: Id<"entries">; startOrder: number },
 ) {
   const { entryId, startOrder } = args;
   const entry = await ctx.db.get(entryId);
@@ -614,7 +614,7 @@ export const deleteByKeyAsync = mutation({
 
 async function getEntriesByKey(
   ctx: QueryCtx,
-  args: { namespaceId: Id<"namespaces">; key: string; beforeVersion?: number }
+  args: { namespaceId: Id<"namespaces">; key: string; beforeVersion?: number },
 ): Promise<Doc<"entries">[]> {
   return mergedStream(
     statuses.map((status) =>
@@ -625,11 +625,11 @@ async function getEntriesByKey(
             .eq("namespaceId", args.namespaceId)
             .eq("status.kind", status)
             .eq("key", args.key)
-            .lt("version", args.beforeVersion ?? Infinity)
+            .lt("version", args.beforeVersion ?? Infinity),
         )
-        .order("desc")
+        .order("desc"),
     ),
-    ["version"]
+    ["version"],
   ).take(100);
 }
 
@@ -653,7 +653,7 @@ export const deleteByKeySync = action({
     while (true) {
       const entries: Doc<"entries">[] = await ctx.runQuery(
         internal.entries.getEntriesForNamespaceByKey,
-        { namespaceId: args.namespaceId, key: args.key }
+        { namespaceId: args.namespaceId, key: args.key },
       );
       for await (const entry of entries) {
         await ctx.runAction(api.entries.deleteSync, {

package/src/component/filters.ts

@@ -55,7 +55,7 @@ export type NamedFilter<K extends string = string, V = Value> = {
  */
 export function filterFieldsFromNumbers(
   namespaceId: GenericId<"namespaces">,
-  filters: NumberedFilter | undefined
+  filters: NumberedFilter | undefined,
 ): NamedFilterField {
   const filterFields: NamedFilterField = {};
   if (!filters) return filterFields;
@@ -63,7 +63,7 @@ export function filterFieldsFromNumbers(
     const index = Number(i);
     if (isNaN(index) || index < 0 || index >= filterFieldNames.length) {
       console.warn(
-        `Unknown filter index: ${index} for value ${JSON.stringify(filter)}`
+        `Unknown filter index: ${index} for value ${JSON.stringify(filter)}`,
       );
       break;
     }
@@ -78,7 +78,7 @@ export function filterFieldsFromNumbers(
  */
 export function numberedFilterFromNamedFilters(
   namedFilters: Array<{ name: string; value: Value }>,
-  filterNames: string[]
+  filterNames: string[],
 ): NumberedFilter {
   const numberedFilter: NumberedFilter = {};
   for (const namedFilter of namedFilters) {
@@ -86,8 +86,8 @@ export function numberedFilterFromNamedFilters(
     if (index === -1) {
       throw new Error(
         `Unknown filter name: ${namedFilter.name} for namespace with names ${filterNames.join(
-          ", "
-        )}`
+          ", ",
+        )}`,
       );
     }
     numberedFilter[index] = namedFilter.value;
@@ -101,7 +101,7 @@ export function numberedFilterFromNamedFilters(
  */
 export function numberedFiltersFromNamedFilters(
   filters: NamedFilter[],
-  filterNames: string[]
+  filterNames: string[],
 ): Array<NumberedFilter> {
   const filterFields: Array<NumberedFilter> = [];
   for (const filter of filters) {
@@ -109,8 +109,8 @@ export function numberedFiltersFromNamedFilters(
     if (index === -1) {
       throw new Error(
         `Unknown filter name: ${filter.name} for namespace with names ${filterNames.join(
-          ", "
-        )}`
+          ", ",
+        )}`,
       );
     }
     filterFields.push({ [index]: filter.value });

package/src/component/namespaces.ts

@@ -18,11 +18,11 @@ import {
   vStatus,
   statuses,
   filterNamesContain,
-  Entry,
+  vEntry,
 } from "../shared.js";
 import { paginationOptsValidator, PaginationResult } from "convex/server";
 import { paginator } from "convex-helpers/server/pagination";
-import type { ObjectType } from "convex/values";
+import type { Infer, ObjectType } from "convex/values";
 import { mergedStream, stream } from "convex-helpers/server/stream";
 import { assert } from "convex-helpers";
 import { api } from "./_generated/api.js";
@@ -33,7 +33,7 @@ function namespaceIsCompatible(
     modelId: string;
     dimension: number;
     filterNames: string[];
-  }
+  },
 ) {
   // Check basic compatibility
   if (
@@ -79,12 +79,12 @@ export const getCompatibleNamespace = internalQuery({
 
 export async function getCompatibleNamespaceHandler(
   ctx: QueryCtx,
-  args: ObjectType<typeof vNamespaceLookupArgs>
+  args: ObjectType<typeof vNamespaceLookupArgs>,
 ) {
   const iter = ctx.db
     .query("namespaces")
     .withIndex("status_namespace_version", (q) =>
-      q.eq("status.kind", "ready").eq("namespace", args.namespace)
+      q.eq("status.kind", "ready").eq("namespace", args.namespace),
     )
     .order("desc");
   for await (const existing of iter) {
@@ -132,11 +132,11 @@ export const getOrCreate = mutation({
         stream(ctx.db, schema)
           .query("namespaces")
           .withIndex("status_namespace_version", (q) =>
-            q.eq("status.kind", status).eq("namespace", args.namespace)
+            q.eq("status.kind", status).eq("namespace", args.namespace),
           )
-          .order("desc")
+          .order("desc"),
       ),
-      ["version"]
+      ["version"],
     );
 
     let version: number = 0;
@@ -172,7 +172,7 @@ async function runOnComplete(
   ctx: MutationCtx,
   onComplete: string | undefined,
   namespace: Doc<"namespaces">,
-  replacedNamespace: Doc<"namespaces"> | null
+  replacedNamespace: Doc<"namespaces"> | null,
 ) {
   const onCompleteFn = onComplete as unknown as OnCompleteNamespace;
   if (!onCompleteFn) {
@@ -198,25 +198,25 @@ export const promoteToReady = mutation({
 
 async function promoteToReadyHandler(
   ctx: MutationCtx,
-  args: { namespaceId: Id<"namespaces"> }
+  args: { namespaceId: Id<"namespaces"> },
 ) {
   const namespace = await ctx.db.get(args.namespaceId);
   assert(namespace, `Namespace ${args.namespaceId} not found`);
   if (namespace.status.kind === "ready") {
     console.debug(
-      `Namespace ${args.namespaceId} is already ready, not promoting`
+      `Namespace ${args.namespaceId} is already ready, not promoting`,
     );
     return { replacedNamespace: null };
   } else if (namespace.status.kind === "replaced") {
     console.debug(
-      `Namespace ${args.namespaceId} is already replaced, not promoting and returning itself`
+      `Namespace ${args.namespaceId} is already replaced, not promoting and returning itself`,
     );
     return { replacedNamespace: publicNamespace(namespace) };
   }
   const previousNamespace = await ctx.db
     .query("namespaces")
     .withIndex("status_namespace_version", (q) =>
-      q.eq("status.kind", "ready").eq("namespace", namespace.namespace)
+      q.eq("status.kind", "ready").eq("namespace", namespace.namespace),
     )
     .order("desc")
     .unique();
@@ -237,7 +237,7 @@ async function promoteToReadyHandler(
       ctx,
       previousStatus.onComplete,
       namespace,
-      previousNamespace
+      previousNamespace,
     );
   }
   const previousPendingNamespaces = await ctx.db
@@ -246,7 +246,7 @@ async function promoteToReadyHandler(
       q
         .eq("status.kind", "pending")
         .eq("namespace", namespace.namespace)
-        .lt("version", namespace.version)
+        .lt("version", namespace.version),
     )
     .collect();
   // Then mark all previous pending namespaces as replaced,
@@ -259,7 +259,7 @@ async function promoteToReadyHandler(
       if (previousStatus.kind === "pending" && previousStatus.onComplete) {
         await runOnComplete(ctx, previousStatus.onComplete, namespace, null);
       }
-    })
+    }),
   );
   return {
     replacedNamespace: previousNamespace
@@ -278,7 +278,7 @@ export const list = query({
     const namespaces = await paginator(ctx.db, schema)
       .query("namespaces")
       .withIndex("status_namespace_version", (q) =>
-        q.eq("status.kind", args.status ?? "ready")
+        q.eq("status.kind", args.status ?? "ready"),
       )
       .order("desc")
       .paginate(args.paginationOpts);
@@ -298,11 +298,11 @@ export const listNamespaceVersions = query({
         stream(ctx.db, schema)
           .query("namespaces")
           .withIndex("status_namespace_version", (q) =>
-            q.eq("status.kind", status).eq("namespace", args.namespace)
+            q.eq("status.kind", status).eq("namespace", args.namespace),
           )
-          .order("desc")
+          .order("desc"),
       ),
-      ["version"]
+      ["version"],
     ).paginate(args.paginationOpts);
 
     return {
@@ -332,21 +332,21 @@ export const deleteNamespace = mutation({
 
 async function deleteHandler(
   ctx: MutationCtx,
-  args: { namespaceId: Id<"namespaces"> }
+  args: { namespaceId: Id<"namespaces"> },
 ) {
   const namespace = await ctx.db.get(args.namespaceId);
   assert(namespace, `Namespace ${args.namespaceId} not found`);
   const anyEntry = await ctx.db
     .query("entries")
     .withIndex("namespaceId_status_key_version", (q) =>
-      q.eq("namespaceId", args.namespaceId)
+      q.eq("namespaceId", args.namespaceId),
     )
     .first();
   if (anyEntry) {
     throw new Error(
       `Namespace ${args.namespaceId} cannot delete, has entries` +
         "First delete all entries." +
-        `Entry: ${anyEntry.key} id ${anyEntry._id} (${anyEntry.status.kind})`
+        `Entry: ${anyEntry.key} id ${anyEntry._id} (${anyEntry.status.kind})`,
     );
   }
   await ctx.db.delete(args.namespaceId);
@@ -360,17 +360,14 @@ export const deleteNamespaceSync = action({
     for (const status of statuses) {
       let cursor: string | null = null;
       while (true) {
-        const entries: PaginationResult<Entry> = await ctx.runQuery(
-          api.entries.list,
-          {
-            namespaceId: args.namespaceId,
-            status: status,
-            paginationOpts: {
-              numItems: 100,
-              cursor,
-            },
-          }
-        );
+        const entries = (await ctx.runQuery(api.entries.list, {
+          namespaceId: args.namespaceId,
+          status: status,
+          paginationOpts: {
+            numItems: 100,
+            cursor,
+          },
+        })) as PaginationResult<Infer<typeof vEntry>>;
         if (entries.isDone) {
           break;
         }

package/src/component/schema.ts

@@ -20,7 +20,7 @@ export const vStatusWithOnComplete = v.union(
   v.object({
     kind: v.literal("replaced"),
     replacedAt: v.number(),
-  })
+  }),
 );
 
 export type StatusWithOnComplete = Infer<typeof vStatusWithOnComplete>;
@@ -79,7 +79,7 @@ export const schema = defineSchema({
         embeddingId: vVectorId,
         vector: v.array(v.number()),
         pendingSearchableText: v.optional(v.string()),
-      })
+      }),
     ),
     // TODO: should content be inline?
     contentId: v.id("content"),

package/src/component/search.test.ts

@@ -16,7 +16,7 @@ describe("search", () => {
     t: ConvexTest,
     namespace = "test-namespace",
     dimension = 128,
-    filterNames: string[] = []
+    filterNames: string[] = [],
   ) {
     return await t.run(async (ctx) => {
       return ctx.db.insert("namespaces", {
@@ -35,7 +35,7 @@ describe("search", () => {
     namespaceId: Id<"namespaces">,
     key = "test-entry",
     version = 0,
-    filterValues: Array<{ name: string; value: Value }> = []
+    filterValues: Array<{ name: string; value: Value }> = [],
   ) {
     return await t.run(async (ctx) => {
       return ctx.db.insert("entries", {
@@ -207,7 +207,7 @@ describe("search", () => {
 
     // With threshold should return fewer results
     expect(resultWithThreshold.results.length).toBeLessThan(
-      resultWithoutThreshold.results.length
+      resultWithoutThreshold.results.length,
     );
     expect(resultWithoutThreshold.results).toHaveLength(2);
 
@@ -305,7 +305,7 @@ describe("search", () => {
       t,
       "multi-filter-namespace",
       128,
-      ["category", "priority_category"]
+      ["category", "priority_category"],
     );
 
     // Create entries with different filter combinations
@@ -438,7 +438,7 @@ describe("search", () => {
     // Results should be sorted by score (best first)
     for (let i = 1; i < result.results.length; i++) {
       expect(result.results[i - 1].score).toBeGreaterThanOrEqual(
-        result.results[i].score
+        result.results[i].score,
       );
     }
   });