@convex-dev/rag 0.5.4 → 0.6.1

package/src/component/chunks.ts

@@ -50,7 +50,7 @@ export const insert = mutation({
 
 export async function insertChunks(
   ctx: MutationCtx,
-  { entryId, startOrder, chunks }: InsertChunksArgs
+  { entryId, startOrder, chunks }: InsertChunksArgs,
 ) {
   const entry = await ctx.db.get(entryId);
   if (!entry) {
@@ -71,12 +71,12 @@ export async function insertChunks(
       q
         .eq("entryId", entryId)
         .gte("order", startOrder)
-        .lt("order", startOrder + chunks.length)
+        .lt("order", startOrder + chunks.length),
     )
     .collect();
   if (existingChunks.length > 0) {
     console.debug(
-      `Deleting ${existingChunks.length} existing chunks for entry ${entryId} at version ${entry.version}`
+      `Deleting ${existingChunks.length} existing chunks for entry ${entryId} at version ${entry.version}`,
     );
   }
   // TODO: avoid writing if they're the same
@@ -87,11 +87,11 @@ export async function insertChunks(
       }
       await ctx.db.delete(c.contentId);
       await ctx.db.delete(c._id);
-    })
+    }),
   );
   const numberedFilter = numberedFilterFromNamedFilters(
     entry.filterValues,
-    namespace!.filterNames
+    namespace!.filterNames,
   );
   for (const chunk of chunks) {
     const contentId = await ctx.db.insert("content", {
@@ -110,7 +110,7 @@ export async function insertChunks(
         chunk.embedding,
         entry.namespaceId,
         entry.importance,
-        numberedFilter
+        numberedFilter,
       );
       state = {
         kind: "ready",
@@ -126,13 +126,11 @@ export async function insertChunks(
         contentId,
         namespaceId: entry.namespaceId,
         ...filterFieldsFromNumbers(entry.namespaceId, numberedFilter),
-      })
+      }),
     );
     order++;
   }
-  return {
-    status: previousEntry ? ("pending" as const) : ("ready" as const),
-  };
+  return { status: previousEntry ? ("pending" as const) : ("ready" as const) };
 }
 
 async function ensureLatestEntryVersion(ctx: QueryCtx, entry: Doc<"entries">) {
@@ -148,14 +146,14 @@ async function ensureLatestEntryVersion(ctx: QueryCtx, entry: Doc<"entries">) {
             .eq("namespaceId", entry.namespaceId)
             .eq("status.kind", status)
             .eq("key", entry.key)
-            .gt("version", entry.version)
-        )
+            .gt("version", entry.version),
+        ),
     ),
-    ["version"]
+    ["version"],
   ).first();
   if (newerEntry) {
     console.warn(
-      `Bailing from inserting chunks for entry ${entry.key} at version ${entry.version} since there's a newer version ${newerEntry.version} (status ${newerEntry.status}) creation time difference ${(newerEntry._creationTime - entry._creationTime).toFixed(0)}ms`
+      `Bailing from inserting chunks for entry ${entry.key} at version ${entry.version} since there's a newer version ${newerEntry.version} (status ${newerEntry.status}) creation time difference ${(newerEntry._creationTime - entry._creationTime).toFixed(0)}ms`,
     );
     return false;
   }
@@ -163,14 +161,8 @@ async function ensureLatestEntryVersion(ctx: QueryCtx, entry: Doc<"entries">) {
 }
 
 export const replaceChunksPage = mutation({
-  args: v.object({
-    entryId: v.id("entries"),
-    startOrder: v.number(),
-  }),
-  returns: v.object({
-    status: vStatus,
-    nextStartOrder: v.number(),
-  }),
+  args: v.object({ entryId: v.id("entries"), startOrder: v.number() }),
+  returns: v.object({ status: vStatus, nextStartOrder: v.number() }),
   handler: async (ctx, args) => {
     const { entryId, startOrder } = args;
     const entryOrNull = await ctx.db.get(entryId);
@@ -180,10 +172,7 @@ export const replaceChunksPage = mutation({
     const entry = entryOrNull;
     const isLatest = await ensureLatestEntryVersion(ctx, entry);
     if (!isLatest) {
-      return {
-        status: "replaced" as const,
-        nextStartOrder: startOrder,
-      };
+      return { status: "replaced" as const, nextStartOrder: startOrder };
     }
 
     // Get the namespace for filter conversion
@@ -200,7 +189,7 @@ export const replaceChunksPage = mutation({
                 q
                   .eq("namespaceId", entry.namespaceId)
                   .eq("status.kind", "pending")
-                  .eq("key", entry.key)
+                  .eq("key", entry.key),
               )
               .collect()
           ).filter((e) => e._id !== entry._id)
@@ -212,29 +201,27 @@ export const replaceChunksPage = mutation({
           stream(ctx.db, schema)
             .query("chunks")
             .withIndex("entryId_order", (q) =>
-              q.eq("entryId", entry._id).gte("order", startOrder)
-            )
+              q.eq("entryId", entry._id).gte("order", startOrder),
+            ),
         ),
-      ["order"]
+      ["order"],
     );
     const namespaceId = entry.namespaceId;
     const namedFilters = numberedFilterFromNamedFilters(
       entry.filterValues,
-      namespace!.filterNames
+      namespace!.filterNames,
     );
     async function addChunk(
-      chunk: Doc<"chunks"> & { state: { kind: "pending" } }
+      chunk: Doc<"chunks"> & { state: { kind: "pending" } },
     ) {
       const embeddingId = await insertEmbedding(
         ctx,
         chunk.state.embedding,
         namespaceId,
         entry.importance,
-        namedFilters
+        namedFilters,
       );
-      await ctx.db.patch(chunk._id, {
-        state: { kind: "ready", embeddingId },
-      });
+      await ctx.db.patch(chunk._id, { state: { kind: "ready", embeddingId } });
     }
     let dataUsedSoFar = 0;
     let indexToDelete = startOrder;
@@ -247,6 +234,8 @@ export const replaceChunksPage = mutation({
           assert(chunk.state.kind === "ready");
           const vector = await ctx.db.get(chunk.state.embeddingId);
           assert(vector, `Vector ${chunk.state.embeddingId} not found`);
+          // get and delete both count as bandwidth reads
+          dataUsedSoFar += estimateEmbeddingSize(vector) * 2;
           await ctx.db.delete(chunk.state.embeddingId);
           await ctx.db.patch(chunk._id, {
             state: {
@@ -256,7 +245,7 @@ export const replaceChunksPage = mutation({
               pendingSearchableText: chunk.state.searchableText,
             },
           });
-        })
+        }),
       );
       chunksToDeleteEmbeddings = [];
       if (chunkToAdd) {
@@ -265,9 +254,9 @@ export const replaceChunksPage = mutation({
       chunkToAdd = null;
     }
     for await (const chunk of chunkStream) {
-      if (chunk.state.kind === "pending") {
-        dataUsedSoFar += await estimateChunkSize(chunk);
-      } else {
+      // one for the stream read, one for patching / replacing
+      dataUsedSoFar += estimateChunkSize(chunk) * 2;
+      if (chunk.state.kind !== "pending") {
         dataUsedSoFar += 17 * KB; // embedding conservative estimate
       }
       if (chunk.order > indexToDelete) {
@@ -277,23 +266,17 @@ export const replaceChunksPage = mutation({
         // check if we're close to the limit
         // if so, bail and pick up on this chunk.order.
         if (dataUsedSoFar > BANDWIDTH_PER_TRANSACTION_SOFT_LIMIT) {
-          return {
-            status: "pending" as const,
-            nextStartOrder: indexToDelete,
-          };
+          return { status: "pending" as const, nextStartOrder: indexToDelete };
         }
       }
       if (dataUsedSoFar > BANDWIDTH_PER_TRANSACTION_HARD_LIMIT) {
-        return {
-          status: "pending" as const,
-          nextStartOrder: indexToDelete,
-        };
+        return { status: "pending" as const, nextStartOrder: indexToDelete };
       }
       if (chunk.state.kind === "pending") {
         if (chunk.entryId === entryId) {
           if (chunkToAdd) {
             console.warn(
-              `Multiple pending chunks before changing order ${chunk.order} for entry ${entryId} version ${entry.version}: ${chunkToAdd._id} and ${chunk._id}`
+              `Multiple pending chunks before changing order ${chunk.order} for entry ${entryId} version ${entry.version}: ${chunkToAdd._id} and ${chunk._id}`,
             );
             await addChunk(chunkToAdd);
           }
@@ -304,7 +287,7 @@ export const replaceChunksPage = mutation({
           chunksToDeleteEmbeddings.push(chunk);
         } else {
           console.debug(
-            `Skipping adding chunk ${chunk._id} for entry ${entryId} version ${entry.version} since it's already ready`
+            `Skipping adding chunk ${chunk._id} for entry ${entryId} version ${entry.version} since it's already ready`,
           );
         }
       }
@@ -312,10 +295,7 @@ export const replaceChunksPage = mutation({
     // handle the last batch
     await handleBatch();
 
-    return {
-      status: "ready" as const,
-      nextStartOrder: 0,
-    };
+    return { status: "ready" as const, nextStartOrder: 0 };
   },
 });
 
@@ -327,7 +307,7 @@ export const vRangeResult = v.object({
     v.object({
       text: v.string(),
       metadata: v.optional(v.record(v.string(), v.any())),
-    })
+    }),
   ),
 });
 
@@ -342,7 +322,7 @@ export const getRangesOfChunks = internalQuery({
   }),
   handler: async (
     ctx,
-    args
+    args,
   ): Promise<{
     ranges: (null | Infer<typeof vRangeResult>)[];
     entries: Entry[];
@@ -353,19 +333,19 @@ export const getRangesOfChunks = internalQuery({
         ctx.db
           .query("chunks")
           .withIndex("embeddingId", (q) =>
-            q.eq("state.embeddingId", embeddingId)
+            q.eq("state.embeddingId", embeddingId),
           )
           .order("desc")
-          .first()
-      )
+          .first(),
+      ),
     );
 
     // Note: This preserves order of entries as they first appeared.
     const entries = (
       await Promise.all(
         Array.from(
-          new Set(chunks.filter((c) => c !== null).map((c) => c.entryId))
-        ).map((id) => ctx.db.get(id))
+          new Set(chunks.filter((c) => c !== null).map((c) => c.entryId)),
+        ).map((id) => ctx.db.get(id)),
       )
     )
       .filter((d) => d !== null)
@@ -383,7 +363,7 @@ export const getRangesOfChunks = internalQuery({
           acc[entryId] = [...(acc[entryId] ?? []), order].sort((a, b) => a - b);
           return acc;
         },
-        {} as Record<Id<"entries">, number[]>
+        {} as Record<Id<"entries">, number[]>,
       );
 
     const result: Array<Infer<typeof vRangeResult> | null> = [];
@@ -397,7 +377,7 @@ export const getRangesOfChunks = internalQuery({
       // instead we'd check that other chunks are not the same doc/order
       if (
         result.find(
-          (r) => r?.entryId === chunk.entryId && r?.order === chunk.order
+          (r) => r?.entryId === chunk.entryId && r?.order === chunk.order,
         )
       ) {
         // De-dupe chunks
@@ -415,12 +395,12 @@ export const getRangesOfChunks = internalQuery({
       const startOrder = Math.max(
         chunk.order - chunkContext.before,
         0,
-        Math.min(previousOrder + 1, chunk.order)
+        Math.min(previousOrder + 1, chunk.order),
       );
       // We stop short if the next chunk order's "before" context will cover it.
       const endOrder = Math.min(
         chunk.order + chunkContext.after + 1,
-        Math.max(nextOrder - chunkContext.before, chunk.order + 1)
+        Math.max(nextOrder - chunkContext.before, chunk.order + 1),
       );
       const contentIds: Id<"content">[] = [];
       if (startOrder === chunk.order && endOrder === chunk.order + 1) {
@@ -432,7 +412,7 @@ export const getRangesOfChunks = internalQuery({
             q
               .eq("entryId", entryId)
               .gte("order", startOrder)
-              .lt("order", endOrder)
+              .lt("order", endOrder),
           )
           .collect();
         for (const chunk of chunks) {
@@ -444,21 +424,13 @@ export const getRangesOfChunks = internalQuery({
           const content = await ctx.db.get(contentId);
           assert(content, `Content ${contentId} not found`);
           return { text: content.text, metadata: content.metadata };
-        })
+        }),
       );
 
-      result.push({
-        entryId,
-        order: chunk.order,
-        startOrder,
-        content,
-      });
+      result.push({ entryId, order: chunk.order, startOrder, content });
     }
 
-    return {
-      ranges: result,
-      entries,
-    };
+    return { ranges: result, entries };
   },
 });
 
@@ -483,7 +455,7 @@ export const list = query({
           const content = await ctx.db.get(chunk.contentId);
           assert(content, `Content ${chunk.contentId} not found`);
           return publicChunk(chunk, content);
-        })
+        }),
       ),
     };
   },
@@ -516,31 +488,30 @@ async function publicChunk(chunk: Doc<"chunks">, content: Doc<"content">) {
 }
 
 export const deleteChunksPage = internalMutation({
-  args: v.object({
-    entryId: v.id("entries"),
-    startOrder: v.number(),
-  }),
+  args: v.object({ entryId: v.id("entries"), startOrder: v.number() }),
   returns: v.object({ isDone: v.boolean(), nextStartOrder: v.number() }),
   handler: deleteChunksPageHandler,
 });
 
 export async function deleteChunksPageHandler(
   ctx: MutationCtx,
-  { entryId, startOrder }: { entryId: Id<"entries">; startOrder: number }
+  { entryId, startOrder }: { entryId: Id<"entries">; startOrder: number },
 ) {
   const chunkStream = ctx.db
     .query("chunks")
     .withIndex("entryId_order", (q) =>
-      q.eq("entryId", entryId).gte("order", startOrder)
+      q.eq("entryId", entryId).gte("order", startOrder),
     );
   let dataUsedSoFar = 0;
   for await (const chunk of chunkStream) {
-    dataUsedSoFar += await estimateChunkSize(chunk);
+    // one for the stream read, one for deleting
+    dataUsedSoFar += estimateChunkSize(chunk) * 2;
     await ctx.db.delete(chunk._id);
     if (chunk.state.kind === "ready") {
       const embedding = await ctx.db.get(chunk.state.embeddingId);
       if (embedding) {
-        dataUsedSoFar += estimateEmbeddingSize(embedding);
+        // get and delete both count as bandwidth reads
+        dataUsedSoFar += estimateEmbeddingSize(embedding) * 2;
         await ctx.db.delete(chunk.state.embeddingId);
       }
     }
@@ -572,7 +543,7 @@ function estimateEmbeddingSize(embedding: Doc<VectorTableName>) {
   return dataUsedSoFar;
 }
 
-async function estimateChunkSize(chunk: Doc<"chunks">) {
+function estimateChunkSize(chunk: Doc<"chunks">) {
   let dataUsedSoFar = 100; // constant metadata - roughly
   if (chunk.state.kind === "pending") {
     dataUsedSoFar += chunk.state.embedding.length * 8;
@@ -590,7 +561,7 @@ async function estimateContentSize(ctx: QueryCtx, contentId: Id<"content">) {
   if (content) {
     dataUsedSoFar += content.text.length;
     dataUsedSoFar += JSON.stringify(
-      convexToJson(content.metadata ?? {})
+      convexToJson(content.metadata ?? {}),
     ).length;
   }
   return dataUsedSoFar;

package/src/component/embeddings/importance.test.ts

@@ -45,7 +45,7 @@ describe("importance.ts", () => {
       expect(result[0]).toBeCloseTo(embedding[0] * importance);
       expect(result[1]).toBeCloseTo(embedding[1] * importance);
       expect(
-        Math.sqrt(result[0] ** 2 + result[1] ** 2 + result[2] ** 2)
+        Math.sqrt(result[0] ** 2 + result[1] ** 2 + result[2] ** 2),
       ).toBeCloseTo(1);
     });
 
@@ -175,7 +175,7 @@ describe("importance.ts", () => {
 
         expect(retrievedImportance).toBeCloseTo(importance, 3);
         expect(Math.abs(retrievedImportance - importance)).toBeLessThan(
-          tolerance
+          tolerance,
         );
       });
     });
@@ -198,13 +198,13 @@ describe("importance.ts", () => {
       // Create vector with initial importance
       const vectorWithInitialImp = vectorWithImportance(
         embedding,
-        initialImportance
+        initialImportance,
       );
 
       // Modify importance
       const vectorWithModifiedImp = modifyImportance(
         vectorWithInitialImp,
-        newImportance
+        newImportance,
       );
 
       // Retrieve and verify

package/src/component/embeddings/importance.ts

@@ -35,7 +35,7 @@ export function vectorWithImportance(embedding: number[], importance: number) {
   // We drop the final dimension if it'd make it larger than 4096.
   // Unfortunate current limitation of Convex vector search.
   const vectorToModify = normalizeVector(
-    embedding.length === 4096 ? embedding.slice(0, 4095) : embedding
+    embedding.length === 4096 ? embedding.slice(0, 4095) : embedding,
   );
   const scaled = scaleVector(vectorToModify, importance);
 

package/src/component/embeddings/index.test.ts

@@ -25,7 +25,6 @@ const testApi: ApiFromModules<{
   fns: {
     search: typeof search;
   };
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
 }>["fns"] = anyApi["embeddings"]["index.test"] as any;
 
 describe("embeddings", () => {
@@ -62,7 +61,7 @@ describe("embeddings", () => {
     expect(insertedVector).toBeDefined();
     expect(insertedVector!.namespaceId).toBe(namespaceId);
     expect(insertedVector!.vector).toHaveLength(
-      vectorWithImportanceDimension(128)
+      vectorWithImportanceDimension(128),
     );
     expect(insertedVector!.filter0).toBeUndefined();
     expect(insertedVector!.filter1).toBeUndefined();
@@ -94,7 +93,7 @@ describe("embeddings", () => {
         embedding,
         namespaceId,
         importance,
-        undefined
+        undefined,
       );
     });
 
@@ -116,7 +115,7 @@ describe("embeddings", () => {
 
     // Vectors should be different due to importance scaling
     expect(insertedVector!.vector).not.toEqual(
-      vectorWithoutImportanceData!.vector
+      vectorWithoutImportanceData!.vector,
     );
 
     // The last element should be the weight: sqrt(1 - importance^2)

package/src/component/embeddings/index.ts

@@ -47,7 +47,7 @@ export async function insertEmbedding(
   embedding: number[],
   namespaceId: Id<"namespaces">,
   importance: number | undefined,
-  filters: NumberedFilter | undefined
+  filters: NumberedFilter | undefined,
 ) {
   const filterFields = filterFieldsFromNumbers(namespaceId, filters);
   const dimension = validateVectorDimension(embedding.length);
@@ -74,12 +74,12 @@ export async function searchEmbeddings(
     // filter3, filter1, or filter2 is present.
     filters: Array<NumberedFilter>;
     limit: number;
-  }
+  },
 ) {
   const dimension = validateVectorDimension(embedding.length);
   const tableName = getVectorTableName(dimension);
   const orFilters = filters.flatMap((filter) =>
-    filterFieldsFromNumbers(namespaceId, filter)
+    filterFieldsFromNumbers(namespaceId, filter),
   );
   return ctx.vectorSearch(tableName, "vector", {
     vector: searchVector(embedding),
@@ -89,9 +89,9 @@ export async function searchEmbeddings(
         : q.or(
             ...orFilters.flatMap((namedFilter) =>
               Object.entries(namedFilter).map(([filterField, filter]) =>
-                q.eq(filterField as keyof (typeof orFilters)[number], filter)
-              )
-            )
+                q.eq(filterField as keyof (typeof orFilters)[number], filter),
+              ),
+            ),
           ),
     limit,
   });

package/src/component/embeddings/tables.ts

@@ -1,6 +1,7 @@
 import { literals } from "convex-helpers/validators";
 import {
   defineTable,
+  GenericTableIndexes,
   type GenericTableSearchIndexes,
   type SchemaDefinition,
   type TableDefinition,
@@ -32,7 +33,7 @@ function table(dimensions: VectorDimension): Table {
 
 type Table = TableDefinition<
   VObject<ObjectType<typeof embeddingsFields>, typeof embeddingsFields>,
-  { model_table_threadId: ["model", "table", "threadId", "_creationTime"] },
+  GenericTableIndexes,
   GenericTableSearchIndexes,
   VectorIndex
 >;
@@ -41,7 +42,7 @@ type VectorIndex = {
   vector: {
     vectorField: "vector";
     dimensions: number;
-    filterFields: string;
+    filterFields: keyof typeof vAllFilterFields & string;
   };
 };
 
@@ -55,11 +56,11 @@ export const VectorDimensions = [
 ] as const;
 
 export function assertVectorDimension(
-  dimension: number
+  dimension: number,
 ): asserts dimension is VectorDimension {
   if (!VectorDimensions.includes(dimension as VectorDimension)) {
     throw new Error(
-      `Unsupported vector dimension${dimension}. Supported: ${VectorDimensions.join(", ")}`
+      `Unsupported vector dimension${dimension}. Supported: ${VectorDimensions.join(", ")}`,
     );
   }
 }
@@ -67,14 +68,14 @@ export function assertVectorDimension(
 export function validateVectorDimension(dimension: number): VectorDimension {
   if (!VectorDimensions.includes(dimension as VectorDimension)) {
     throw new Error(
-      `Unsupported vector dimension${dimension}. Supported: ${VectorDimensions.join(", ")}`
+      `Unsupported vector dimension${dimension}. Supported: ${VectorDimensions.join(", ")}`,
     );
   }
   return dimension as VectorDimension;
 }
 export type VectorDimension = (typeof VectorDimensions)[number];
 export const VectorTableNames = VectorDimensions.map(
-  (d) => `vectors_${d}`
+  (d) => `vectors_${d}`,
 ) as `vectors_${(typeof VectorDimensions)[number]}`[];
 export type VectorTableName = (typeof VectorTableNames)[number];
 export type VectorTableId = GenericId<(typeof VectorTableNames)[number]>;
@@ -82,7 +83,7 @@ export type VectorTableId = GenericId<(typeof VectorTableNames)[number]>;
 export const vVectorDimension = literals(...VectorDimensions);
 export const vVectorTableName = literals(...VectorTableNames);
 export const vVectorId = v.union(
-  ...VectorTableNames.map((name) => v.id(name))
+  ...VectorTableNames.map((name) => v.id(name)),
 ) as VUnion<
   GenericId<(typeof VectorTableNames)[number]>,
   VId<(typeof VectorTableNames)[number]>[]
@@ -108,7 +109,7 @@ const tables: {
   VectorDimensions.map((dimensions) => [
     `vectors_${dimensions}`,
     table(dimensions),
-  ])
+  ]),
 ) as Record<`vectors_${(typeof VectorDimensions)[number]}`, Table>;
 
 export default tables;

package/src/component/entries.test.ts

@@ -97,8 +97,8 @@ describe("entries", () => {
         .filter((q) =>
           q.and(
             q.eq(q.field("namespaceId"), namespaceId),
-            q.eq(q.field("key"), entry.key)
-          )
+            q.eq(q.field("key"), entry.key),
+          ),
         )
         .collect();
     });
@@ -143,8 +143,8 @@ describe("entries", () => {
         .filter((q) =>
           q.and(
             q.eq(q.field("namespaceId"), namespaceId),
-            q.eq(q.field("key"), entry.key)
-          )
+            q.eq(q.field("key"), entry.key),
+          ),
         )
         .collect();
     });
@@ -537,7 +537,7 @@ describe("entries", () => {
       {
         namespaceId,
         key: "shared-key",
-      }
+      },
     );
     expect(sharedBefore).toHaveLength(2);
 
@@ -563,7 +563,7 @@ describe("entries", () => {
 
     const sharedAfter = await t.query(
       internal.entries.getEntriesForNamespaceByKey,
-      { namespaceId, key: "shared-key" }
+      { namespaceId, key: "shared-key" },
     );
     expect(sharedAfter).toHaveLength(0);
 
@@ -722,8 +722,8 @@ describe("entries", () => {
         .filter((q) =>
           q.and(
             q.eq(q.field("namespaceId"), namespaceId),
-            q.eq(q.field("key"), "versioned-key")
-          )
+            q.eq(q.field("key"), "versioned-key"),
+          ),
         )
         .collect();
     });
@@ -747,8 +747,8 @@ describe("entries", () => {
         .filter((q) =>
           q.and(
             q.eq(q.field("namespaceId"), namespaceId),
-            q.eq(q.field("key"), "versioned-key")
-          )
+            q.eq(q.field("key"), "versioned-key"),
+          ),
         )
         .collect();
     });