@convex-dev/rag 0.3.0 → 0.3.2

package/src/client/fileUtils.ts

@@ -133,13 +133,17 @@ export function guessMimeTypeFromContents(buf: ArrayBuffer | string): string {
 /**
  * Make a contentHash of a Blob that matches the File Storage metadata, allowing
  * identifying when content is identical.
+ * By default, uses SHA-256 (which is what Convex File Storage tracks).
+ * Git / GitHub use SHA-1.
  * @param blob The contents to hash
- * @returns sha256 hash of the contents
+ * @returns hash of the contents
  */
-
-export async function contentHashFromArrayBuffer(buffer: ArrayBuffer) {
+export async function contentHashFromArrayBuffer(
+  buffer: ArrayBuffer,
+  algorithm: "SHA-256" | "SHA-1" = "SHA-256"
+) {
   return Array.from(
-    new Uint8Array(await crypto.subtle.digest("SHA-256", buffer))
+    new Uint8Array(await crypto.subtle.digest(algorithm, buffer))
   )
     .map((b) => b.toString(16).padStart(2, "0"))
     .join("");

package/src/client/index.test.ts

@@ -32,10 +32,10 @@ const rag = new RAG(components.rag, {
   filterNames: ["simpleString", "arrayOfStrings", "customObject"],
 });
 
-export const findExistingEntryByContentHash = query({
+export const findEntryByContentHash = query({
   args: { namespace: v.string(), key: v.string(), contentHash: v.string() },
   handler: async (ctx, args) => {
-    return rag.findExistingEntryByContentHash(ctx, {
+    return rag.findEntryByContentHash(ctx, {
       namespace: args.namespace,
       key: args.key,
       contentHash: args.contentHash,
@@ -95,7 +95,7 @@ export const search = action({
   },
   handler: async (ctx, args) => {
     const { results, entries, text } = await rag.search(ctx, {
-      embedding: args.embedding,
+      query: args.embedding,
       namespace: args.namespace,
       limit: args.limit ?? 10,
       chunkContext: args.chunkContext ?? { before: 0, after: 0 },
@@ -111,7 +111,7 @@ export const search = action({
 
 const testApi: ApiFromModules<{
   fns: {
-    findExistingEntryByContentHash: typeof findExistingEntryByContentHash;
+    findEntryByContentHash: typeof findEntryByContentHash;
     add: typeof add;
     search: typeof search;
   };
@@ -200,16 +200,16 @@ describe("RAG thick client", () => {
         limit: 10,
       });
 
-      // Should match README format: "# Title:\n{entry.text}"
-      expect(text).toContain("# Test Document:");
+      // Should match README format: "## Title:\n{entry.text}"
+      expect(text).toContain("## Test Document:");
       expect(entries).toHaveLength(1);
       expect(entries[0].text).toBe(
         "Chunk 1 content\nChunk 2 content\nChunk 3 content"
       );
 
-      // Overall text should be: "# Test Document:\nChunk 1 content\nChunk 2 content\nChunk 3 content"
+      // Overall text should be: "## Test Document:\nChunk 1 content\nChunk 2 content\nChunk 3 content"
       expect(text).toBe(
-        "# Test Document:\nChunk 1 content\nChunk 2 content\nChunk 3 content"
+        "## Test Document:\n\nChunk 1 content\nChunk 2 content\nChunk 3 content"
       );
     });
 
@@ -235,8 +235,8 @@ describe("RAG thick client", () => {
         limit: 10,
       });
 
-      // Should not have "# " prefix since no title
-      expect(text).not.toContain("# ");
+      // Should not have "## " prefix since no title
+      expect(text).not.toContain("## ");
       expect(entries).toHaveLength(1);
       expect(entries[0].text).toBe("Content without title");
       expect(text).toBe("Content without title");
@@ -337,11 +337,11 @@ describe("RAG thick client", () => {
 
       // Should have entries separated by "\n---\n" as per README
       expect(text).toContain("---");
-      expect(text).toMatch(/# .+:\n.+\n---\n# .+:\n.+/);
+      expect(text).toMatch(/## .+:\n\n.+\n\n---\n\n## .+:\n\n.+/);
 
-      // Should have both titles prefixed with "# "
-      expect(text).toContain("# First Document:");
-      expect(text).toContain("# Second Document:");
+      // Should have both titles prefixed with "## "
+      expect(text).toContain("## First Document:");
+      expect(text).toContain("## Second Document:");
 
       expect(entries).toHaveLength(2);
     });
@@ -384,14 +384,14 @@ describe("RAG thick client", () => {
 
       // Should properly handle mixed formatting
       expect(text).toContain("---"); // Entries should be separated
-      expect(text).toContain("# Titled Document:"); // Titled entry should have prefix
+      expect(text).toContain("## Titled Document:"); // Titled entry should have prefix
 
       // One entry should have title format, one should not
       const parts = text.split("\n---\n");
       expect(parts).toHaveLength(2);
 
-      const hasTitle = parts.some((part) => part.startsWith("# "));
-      const hasNoTitle = parts.some((part) => !part.startsWith("# "));
+      const hasTitle = parts.some((part) => part.startsWith("## "));
+      const hasNoTitle = parts.some((part) => !part.startsWith("## "));
       expect(hasTitle).toBe(true);
       expect(hasNoTitle).toBe(true);
 
@@ -445,28 +445,32 @@ describe("RAG thick client", () => {
       });
 
       // Verify basic structure matches README
-      expect(text).toContain("# Title 1:");
-      expect(text).toContain("# Title 2:");
+      expect(text).toContain("## Title 1:");
+      expect(text).toContain("## Title 2:");
       expect(text).toContain("---");
 
       // Should have proper entry separation
-      const parts = text.split("\n---\n");
+      const parts = text.split("\n\n---\n\n");
       expect(parts).toHaveLength(2);
 
-      // Each part should start with "# Title X:"
+      // Each part should start with "## Title X:"
       parts.forEach((part) => {
-        expect(part).toMatch(/^# Title \d+:/);
+        expect(part).toMatch(/^## Title \d+:/);
       });
 
       expect(entries).toHaveLength(2);
 
       // Individual entry texts should follow the pattern
       expect(text).toBe(
-        `# Title 1:
+        `## Title 1:
+
 Chunk 1 contents
 Chunk 2 contents
+
 ---
-# Title 2:
+
+## Title 2:
+
 Chunk 3 contents
 Chunk 4 contents`
       );

package/src/client/index.ts

@@ -27,7 +27,7 @@ import {
   type Chunk,
   type CreateChunkArgs,
   type Entry,
-  type EntryFilterValues,
+  type EntryFilter,
   type EntryId,
   type Namespace,
   type NamespaceId,
@@ -66,6 +66,7 @@ export type {
 export {
   type VEntry,
   type VSearchEntry,
+  type EntryFilter,
   vEntry,
   vSearchEntry,
   vSearchResult,
@@ -155,7 +156,7 @@ export class RAG<
     entryId: EntryId;
     status: Status;
     created: boolean;
-    replacedVersion: Entry<FitlerSchemas, EntryMetadata> | null;
+    replacedEntry: Entry<FitlerSchemas, EntryMetadata> | null;
   }> {
     let namespaceId: NamespaceId;
     if ("namespaceId" in args) {
@@ -182,7 +183,7 @@ export class RAG<
     const onComplete =
       args.onComplete && (await createFunctionHandle(args.onComplete));
 
-    const { entryId, status, created, replacedVersion } = await ctx.runMutation(
+    const { entryId, status, created, replacedEntry } = await ctx.runMutation(
       this.component.entries.add,
       {
         entry: {
@@ -203,7 +204,7 @@ export class RAG<
         entryId: entryId as EntryId,
         status,
         created,
-        replacedVersion: replacedVersion as Entry<
+        replacedEntry: replacedEntry as Entry<
           FitlerSchemas,
           EntryMetadata
         > | null,
@@ -243,7 +244,7 @@ export class RAG<
             entryId: entryId as EntryId,
             status: "replaced" as const,
             created: false,
-            replacedVersion: null,
+            replacedEntry: null,
           };
         }
         startOrder = nextStartOrder;
@@ -256,7 +257,7 @@ export class RAG<
     return {
       entryId: entryId as EntryId,
       status: "ready" as const,
-      replacedVersion: promoted.replacedVersion as Entry<
+      replacedEntry: promoted.replacedEntry as Entry<
         FitlerSchemas,
         EntryMetadata
       > | null,
@@ -362,7 +363,7 @@ export class RAG<
       /**
        * The query to search for. Optional if embedding is provided.
        */
-      query?: string;
+      query: string | Array<number>;
     } & SearchOptions<FitlerSchemas>
   ): Promise<{
     results: SearchResult[];
@@ -376,7 +377,7 @@ export class RAG<
       chunkContext = { before: 0, after: 0 },
       vectorScoreThreshold,
     } = args;
-    let embedding = args.embedding;
+    let embedding = Array.isArray(args.query) ? args.query : undefined;
     if (!embedding) {
       const embedResult = await embed({
         model: this.options.textEmbeddingModel,
@@ -405,7 +406,7 @@ export class RAG<
       for (const range of ranges) {
         if (previousEnd !== 0) {
           if (range.startOrder !== previousEnd) {
-            text += "\n...\n";
+            text += "\n\n...\n\n";
           } else {
             text += "\n";
           }
@@ -419,8 +420,8 @@ export class RAG<
     return {
       results: results as SearchResult[],
       text: entriesWithTexts
-        .map((e) => (e.title ? `# ${e.title}:\n${e.text}` : e.text))
-        .join(`\n---\n`),
+        .map((e) => (e.title ? `## ${e.title}:\n\n${e.text}` : e.text))
+        .join(`\n\n---\n\n`),
       entries: entriesWithTexts,
     };
   }
@@ -446,6 +447,11 @@ export class RAG<
          * The namespace to search in. e.g. a userId if entries are per-user.
          */
         namespace: string;
+        /**
+         * The text or embedding to search for. If provided, it will be used
+         * instead of the prompt for vector search.
+         */
+        query?: string | Array<number>;
       };
       /**
        * Required. The prompt to use for context search, as well as the final
@@ -551,15 +557,18 @@ export class RAG<
   async list(
     ctx: RunQueryCtx,
     args: {
-      namespaceId: NamespaceId;
-      paginationOpts: PaginationOptions;
+      namespaceId?: NamespaceId;
       order?: "desc" | "asc";
       status?: Status;
-    }
+    } & ({ paginationOpts: PaginationOptions } | { limit: number })
   ): Promise<PaginationResult<Entry<FitlerSchemas, EntryMetadata>>> {
+    const paginationOpts =
+      "paginationOpts" in args
+        ? args.paginationOpts
+        : { cursor: null, numItems: args.limit };
     const results = await ctx.runQuery(this.component.entries.list, {
       namespaceId: args.namespaceId,
-      paginationOpts: args.paginationOpts,
+      paginationOpts,
       order: args.order ?? "asc",
       status: args.status ?? "ready",
     });
@@ -586,7 +595,7 @@ export class RAG<
    * new results into a new entry when migrating, or avoiding duplicating work
    * when updating content.
    */
-  async findExistingEntryByContentHash(
+  async findEntryByContentHash(
     ctx: RunQueryCtx,
     args: {
       namespace: string;
@@ -679,11 +688,13 @@ export class RAG<
     args: {
       paginationOpts: PaginationOptions;
       entryId: EntryId;
+      order?: "desc" | "asc";
     }
   ): Promise<PaginationResult<Chunk>> {
     return ctx.runQuery(this.component.chunks.list, {
       entryId: args.entryId,
       paginationOpts: args.paginationOpts,
+      order: args.order ?? "asc",
     });
   }
 
@@ -927,7 +938,7 @@ async function createChunkArgsBatch(
   for (const batch of makeBatches(missingEmbeddingsWithIndex, 100)) {
     const { embeddings } = await embedMany({
       model: embedModel,
-      values: batch.map((b) => b.text),
+      values: batch.map((b) => b.text.trim() || "<empty>"),
     });
     for (const [index, embedding] of embeddings.entries()) {
       argsMaybeMissingEmbeddings[batch[index].index].embedding = embedding;
@@ -1060,7 +1071,7 @@ type EntryArgs<
    * and searching with the same value will return entries that match that
    * value exactly.
    */
-  filterValues?: EntryFilterValues<FitlerSchemas>[];
+  filterValues?: EntryFilter<FitlerSchemas>[];
   /**
    * The importance of the entry. This is used to scale the vector search
    * score of each chunk.
@@ -1080,11 +1091,6 @@ type EntryArgs<
 };
 
 type SearchOptions<FitlerSchemas extends Record<string, Value>> = {
-  /**
-   * The embedding to search for. If provided, it will be used instead
-   * of the query for vector search.
-   */
-  embedding?: Array<number>;
   /**
    * Filters to apply to the search. These are OR'd together. To represent
    * AND logic, your filter can be an object or array with multiple values.
@@ -1097,7 +1103,7 @@ type SearchOptions<FitlerSchemas extends Record<string, Value>> = {
    * `{ team_user: { team: "team1", user: "user1" } }`, it will not match
    * `{ team_user: { team: "team1" } }` but it will match
    */
-  filters?: EntryFilterValues<FitlerSchemas>[];
+  filters?: EntryFilter<FitlerSchemas>[];
   /**
    * The maximum number of messages to fetch. Default is 10.
    * This is the number *before* the chunkContext is applied.

package/src/component/_generated/api.d.ts

@@ -62,6 +62,7 @@ export type Mounts = {
       "public",
       {
         entryId: string;
+        order: "desc" | "asc";
         paginationOpts: {
           cursor: string | null;
           endCursor?: string | null;
@@ -115,13 +116,14 @@ export type Mounts = {
       {
         created: boolean;
         entryId: string;
-        replacedVersion: {
+        replacedEntry: {
           contentHash?: string;
           entryId: string;
           filterValues: Array<{ name: string; value: any }>;
           importance: number;
           key?: string;
           metadata?: Record<string, any>;
+          replacedAt?: number;
           status: "pending" | "ready" | "replaced";
           title?: string;
         } | null;
@@ -170,6 +172,7 @@ export type Mounts = {
         importance: number;
         key?: string;
         metadata?: Record<string, any>;
+        replacedAt?: number;
         status: "pending" | "ready" | "replaced";
         title?: string;
       } | null
@@ -185,6 +188,7 @@ export type Mounts = {
         importance: number;
         key?: string;
         metadata?: Record<string, any>;
+        replacedAt?: number;
         status: "pending" | "ready" | "replaced";
         title?: string;
       } | null
@@ -193,7 +197,7 @@ export type Mounts = {
       "query",
       "public",
       {
-        namespaceId: string;
+        namespaceId?: string;
         order?: "desc" | "asc";
         paginationOpts: {
           cursor: string | null;
@@ -215,6 +219,7 @@ export type Mounts = {
           importance: number;
           key?: string;
           metadata?: Record<string, any>;
+          replacedAt?: number;
           status: "pending" | "ready" | "replaced";
           title?: string;
         }>;
@@ -227,13 +232,14 @@ export type Mounts = {
       "public",
       { entryId: string },
       {
-        replacedVersion: {
+        replacedEntry: {
           contentHash?: string;
           entryId: string;
           filterValues: Array<{ name: string; value: any }>;
           importance: number;
           key?: string;
           metadata?: Record<string, any>;
+          replacedAt?: number;
           status: "pending" | "ready" | "replaced";
           title?: string;
         } | null;
@@ -321,7 +327,7 @@ export type Mounts = {
       "public",
       { namespaceId: string },
       {
-        replacedVersion: null | {
+        replacedNamespace: null | {
           createdAt: number;
           dimension: number;
           filterNames: Array<string>;
@@ -355,6 +361,7 @@ export type Mounts = {
           importance: number;
           key?: string;
           metadata?: Record<string, any>;
+          replacedAt?: number;
           status: "pending" | "ready" | "replaced";
           title?: string;
         }>;

package/src/component/chunks.test.ts

@@ -397,6 +397,7 @@ describe("chunks", () => {
     // Test listing with pagination
     const result = await t.query(api.chunks.list, {
       entryId,
+      order: "asc",
       paginationOpts: { numItems: 3, cursor: null },
     });
 
@@ -417,6 +418,7 @@ describe("chunks", () => {
     // Get next page
     const nextResult = await t.query(api.chunks.list, {
       entryId,
+      order: "asc",
       paginationOpts: { numItems: 3, cursor: result.continueCursor },
     });
 

package/src/component/chunks.ts

@@ -463,6 +463,7 @@ export const list = query({
   args: v.object({
     entryId: v.id("entries"),
     paginationOpts: paginationOptsValidator,
+    order: v.union(v.literal("desc"), v.literal("asc")),
   }),
   returns: vPaginationResult(vChunk),
   handler: async (ctx, args) => {
@@ -470,7 +471,7 @@ export const list = query({
     const chunks = await paginator(ctx.db, schema)
       .query("chunks")
       .withIndex("entryId_order", (q) => q.eq("entryId", entryId))
-      .order("asc")
+      .order(args.order)
       .paginate(paginationOpts);
     return {
       ...chunks,

package/src/component/entries.test.ts

@@ -46,7 +46,7 @@ describe("entries", () => {
     expect(result.created).toBe(true);
     expect(result.status).toBe("ready");
     expect(result.entryId).toBeDefined();
-    expect(result.replacedVersion).toBeNull();
+    expect(result.replacedEntry).toBeNull();
 
     // Verify the entry was actually created
     const createdDoc = await t.run(async (ctx) => {
@@ -73,7 +73,7 @@ describe("entries", () => {
 
     expect(firstResult.created).toBe(true);
     expect(firstResult.status).toBe("ready");
-    expect(firstResult.replacedVersion).toBeNull();
+    expect(firstResult.replacedEntry).toBeNull();
 
     // Second add with identical content
     const secondResult = await t.mutation(api.entries.add, {
@@ -84,7 +84,7 @@ describe("entries", () => {
     expect(secondResult.created).toBe(false);
     expect(secondResult.status).toBe("ready");
     expect(secondResult.entryId).toBe(firstResult.entryId);
-    expect(secondResult.replacedVersion).toBeNull();
+    expect(secondResult.replacedEntry).toBeNull();
 
     // Verify no new entry was created
     const allDocs = await t.run(async (ctx) => {
@@ -116,7 +116,7 @@ describe("entries", () => {
     });
 
     expect(firstResult.created).toBe(true);
-    expect(firstResult.replacedVersion).toBeNull();
+    expect(firstResult.replacedEntry).toBeNull();
 
     // Second add with different content hash
     const modifiedEntry = {
@@ -131,9 +131,9 @@ describe("entries", () => {
 
     expect(secondResult.created).toBe(true);
     expect(secondResult.entryId).not.toBe(firstResult.entryId);
-    // When creating a entry as "ready" initially, replacedVersion is null
+    // When creating a entry as "ready" initially, replacedEntry is null
     // Replacement only happens during pending -> ready transitions
-    expect(secondResult.replacedVersion).toMatchObject({
+    expect(secondResult.replacedEntry).toMatchObject({
       entryId: firstResult.entryId,
     });
 
@@ -181,7 +181,7 @@ describe("entries", () => {
 
     expect(secondResult.created).toBe(true);
     expect(secondResult.entryId).not.toBe(firstResult.entryId);
-    expect(secondResult.replacedVersion).toMatchObject({
+    expect(secondResult.replacedEntry).toMatchObject({
       entryId: firstResult.entryId,
     });
 
@@ -219,7 +219,7 @@ describe("entries", () => {
 
     expect(secondResult.created).toBe(true);
     expect(secondResult.entryId).not.toBe(firstResult.entryId);
-    expect(secondResult.replacedVersion).toMatchObject({
+    expect(secondResult.replacedEntry).toMatchObject({
       entryId: firstResult.entryId,
     });
 
@@ -247,7 +247,7 @@ describe("entries", () => {
 
     expect(result.created).toBe(true);
     expect(result.status).toBe("pending");
-    expect(result.replacedVersion).toBeNull();
+    expect(result.replacedEntry).toBeNull();
 
     // Verify the entry was created with pending status
     const createdDoc = await t.run(async (ctx) => {
@@ -277,8 +277,8 @@ describe("entries", () => {
     expect(result1.created).toBe(true);
     expect(result2.created).toBe(true);
     expect(result1.entryId).not.toBe(result2.entryId);
-    expect(result1.replacedVersion).toBeNull();
-    expect(result2.replacedVersion).toBeNull();
+    expect(result1.replacedEntry).toBeNull();
+    expect(result2.replacedEntry).toBeNull();
 
     // Verify both entries exist
     const allDocs = await t.run(async (ctx) => {
@@ -293,7 +293,7 @@ describe("entries", () => {
     expect(keys).toEqual(["doc1", "doc2"]);
   });
 
-  test("pending to ready transition populates replacedVersion", async () => {
+  test("pending to ready transition populates replacedEntry", async () => {
     const t = convexTest(schema, modules);
     const namespaceId = await setupTestNamespace(t);
 
@@ -307,7 +307,7 @@ describe("entries", () => {
 
     expect(firstResult.created).toBe(true);
     expect(firstResult.status).toBe("ready");
-    expect(firstResult.replacedVersion).toBeNull();
+    expect(firstResult.replacedEntry).toBeNull();
 
     // Second add - create as pending (no allChunks)
     const modifiedEntry = {
@@ -322,15 +322,15 @@ describe("entries", () => {
 
     expect(pendingResult.created).toBe(true);
     expect(pendingResult.status).toBe("pending");
-    expect(pendingResult.replacedVersion).toBeNull();
+    expect(pendingResult.replacedEntry).toBeNull();
 
     // Promote to ready - this should replace the first entry
     const promoteResult = await t.mutation(api.entries.promoteToReady, {
       entryId: pendingResult.entryId,
     });
 
-    expect(promoteResult.replacedVersion).not.toBeNull();
-    expect(promoteResult.replacedVersion!.entryId).toBe(firstResult.entryId);
+    expect(promoteResult.replacedEntry).not.toBeNull();
+    expect(promoteResult.replacedEntry!.entryId).toBe(firstResult.entryId);
 
     // Verify the first entry is now replaced
     const firstDoc = await t.run(async (ctx) => {