@convex-dev/rag 0.1.7

package/package.json

@@ -0,0 +1,97 @@
+{
+  "name": "@convex-dev/rag",
+  "description": "A rag component for Convex.",
+  "repository": "github:get-convex/rag",
+  "homepage": "https://github.com/get-convex/rag#readme",
+  "bugs": {
+    "email": "support@convex.dev",
+    "url": "https://github.com/get-convex/rag/issues"
+  },
+  "version": "0.1.7",
+  "license": "Apache-2.0",
+  "keywords": [
+    "convex",
+    "component",
+    "document",
+    "embeddings",
+    "rag",
+    "search",
+    "semantic",
+    "vector"
+  ],
+  "type": "module",
+  "scripts": {
+    "example": "cd example && npm run dev",
+    "dev": "run-p -r 'example' 'build:watch'",
+    "dashboard": "cd example && npx convex dashboard",
+    "all": "run-p -r 'example' 'build:watch' 'test:watch'",
+    "setup": "npm i && npm run build && cd example && npm i && npx convex dev --once && printf 'VITE_CONVEX_SITE_URL=' >> .env.local && npx convex env get CONVEX_SITE_URL >> .env.local",
+    "build:watch": "cd src && npx chokidar -d 1000 '../tsconfig.json' '**/*.ts' -c 'npm run build' --initial",
+    "build": "tsc --project ./tsconfig.build.json && npm run copy:dts && echo '{\\n  \"type\": \"module\"\\n}' > dist/package.json",
+    "copy:dts": "rsync -a --include='*/' --include='*.d.ts' --exclude='*' src/ dist/ || cpy 'src/**/*.d.ts' 'dist/' --parents",
+    "typecheck": "tsc --noEmit",
+    "clean": "rm -rf dist tsconfig.build.tsbuildinfo",
+    "alpha": "npm run clean && npm run build && run-p test lint typecheck && npm version prerelease --preid alpha && npm publish --tag alpha && git push --tags",
+    "release": "npm run clean && npm run build && run-p test lint typecheck && npm version patch && npm publish && git push --tags",
+    "test": "vitest run --typecheck --config ./src/vitest.config.ts",
+    "test:watch": "vitest --typecheck --config ./src/vitest.config.ts",
+    "test:debug": "vitest --inspect-brk --no-file-parallelism --config ./src/vitest.config.ts",
+    "test:coverage": "vitest run --coverage --coverage.reporter=text",
+    "lint": "eslint src",
+    "version": "pbcopy <<<$npm_package_version; vim CHANGELOG.md && git add CHANGELOG.md"
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "@convex-dev/component-source": "./src/client/index.ts",
+      "types": "./dist/client/index.d.ts",
+      "default": "./dist/client/index.js"
+    },
+    "./react": {
+      "@convex-dev/component-source": "./src/react/index.ts",
+      "types": "./dist/react/index.d.ts",
+      "default": "./dist/react/index.js"
+    },
+    "./convex.config": {
+      "@convex-dev/component-source": "./src/component/convex.config.ts",
+      "types": "./dist/component/convex.config.d.ts",
+      "default": "./dist/component/convex.config.js"
+    }
+  },
+  "peerDependencies": {
+    "@ai-sdk/provider": "^1.1.3",
+    "@convex-dev/workpool": "^0.2.14",
+    "ai": "^4.3.16",
+    "convex": "^1.24.8",
+    "convex-helpers": "^0.1.94"
+  },
+  "devDependencies": {
+    "@ai-sdk/openai": "^1.3.22",
+    "@arethetypeswrong/cli": "^0.17.4",
+    "@edge-runtime/vm": "^5.0.0",
+    "@eslint/js": "9.29.0",
+    "@types/node": "18.17.0",
+    "@typescript-eslint/eslint-plugin": "8.35.0",
+    "@typescript-eslint/parser": "8.35.0",
+    "chokidar-cli": "3.0.0",
+    "convex-helpers": "0.1.95",
+    "convex-test": "0.0.37",
+    "cpy-cli": "5.0.0",
+    "eslint": "9.29.0",
+    "eslint-plugin-react": "^7.37.5",
+    "eslint-plugin-react-hooks": "^5.2.0",
+    "globals": "^15.9.0",
+    "npm-run-all2": "7.0.2",
+    "prettier": "3.2.5",
+    "typescript": "5.5",
+    "typescript-eslint": "8.4.0",
+    "vitest": "3.2.4"
+  },
+  "main": "./dist/client/index.js",
+  "types": "./dist/client/index.d.ts",
+  "module": "./dist/client/index.js"
+}

package/src/client/_generated/_ignore.ts

@@ -0,0 +1 @@
+// This is only here so convex-test can detect a _generated folder

package/src/client/defaultChunker.test.ts

@@ -0,0 +1,243 @@
+import { defaultChunker } from "./defaultChunker.js";
+import { describe, test, expect } from "vitest";
+
+describe("defaultChunker", () => {
+  test("handles empty text", () => {
+    expect(defaultChunker("")).toEqual([]);
+    expect(defaultChunker("   ")).toEqual(["   "]);
+  });
+
+  test("chunks paragraphs that fit within limits", () => {
+    const text = `This is the first paragraph with about 100 characters. It should be combined with others.
+
+This is the second paragraph with similar length to make a good chunk together.
+
+This is the third paragraph that will likely be in the next chunk.`;
+
+    const chunks = defaultChunker(text);
+
+    // Should combine all paragraphs since total length (238 chars) is well within limits
+    expect(chunks.length).toBe(1);
+    expect(chunks[0]).toBe(text);
+    chunks.forEach((chunk: string) => {
+      expect(chunk.length).toBeGreaterThan(0);
+      expect(chunk.length).toBeLessThanOrEqual(2000);
+    });
+    expect(chunks.join("\n")).toBe(text);
+  });
+
+  test("combines small paragraphs to meet minimum character limit", () => {
+    const text = `Short para 1.
+
+Short para 2.
+
+Short para 3.
+
+Short para 4.`;
+
+    const chunks = defaultChunker(text, {
+      minCharsSoftLimit: 50,
+      maxCharsSoftLimit: 200,
+    });
+
+    // Should combine multiple short paragraphs
+    chunks.forEach((chunk: string) => {
+      expect(chunk.length).toBeGreaterThanOrEqual(50);
+      expect(chunk.length).toBeLessThanOrEqual(200);
+    });
+    expect(chunks.length).toBe(1);
+    expect(chunks[0]).toBe(
+      "Short para 1.\n\nShort para 2.\n\nShort para 3.\n\nShort para 4."
+    );
+    expect(chunks.join("\n")).toBe(text);
+  });
+
+  test("splits large paragraphs by lines", () => {
+    const longParagraph = Array(50)
+      .fill("This is a line that makes the paragraph very long.")
+      .join("\n");
+
+    const chunks = defaultChunker(longParagraph, {
+      minLines: 2,
+      minCharsSoftLimit: 200,
+      maxCharsSoftLimit: 500,
+    });
+
+    expect(chunks.length).toBeGreaterThan(1);
+    chunks.forEach((chunk: string) => {
+      expect(chunk.length).toBeLessThanOrEqual(500);
+      // Each chunk should have at least 2 lines (minLines)
+      expect(chunk.split("\n").length).toBeGreaterThanOrEqual(2);
+    });
+    expect(chunks.join("\n")).toBe(longParagraph);
+  });
+
+  test("respects minLines constraint when splitting", () => {
+    const text =
+      "Line 1\nLine 2\nLine 3\nLine 4\nLine 5\nLine 6\nLine 7\nLine 8";
+
+    const chunks = defaultChunker(text, {
+      minLines: 3,
+      minCharsSoftLimit: 10,
+      maxCharsSoftLimit: 30, // Very small to force splitting
+    });
+
+    chunks.forEach((chunk: string) => {
+      const lineCount = chunk.split("\n").length;
+      expect(lineCount).toBeGreaterThanOrEqual(3);
+    });
+    expect(chunks.join("\n")).toBe(text);
+  });
+
+  test("handles mixed content with paragraphs and large sections", () => {
+    const text = `Small paragraph 1.
+
+Small paragraph 2.
+
+This is a very long paragraph that definitely exceeds the maximum character limit and should be split by lines instead of being treated as a single paragraph unit.
+Line 2 of the long paragraph.
+Line 3 of the long paragraph.
+Line 4 of the long paragraph.
+Line 5 of the long paragraph.
+
+Another small paragraph at the end.`;
+
+    const chunks = defaultChunker(text, {
+      minLines: 1,
+      minCharsSoftLimit: 100,
+      maxCharsSoftLimit: 300,
+    });
+
+    expect(chunks.length).toBeGreaterThan(1);
+    chunks.forEach((chunk: string) => {
+      expect(chunk.length).toBeLessThanOrEqual(300);
+      expect(chunk.trim().length).toBeGreaterThan(0);
+    });
+    expect(chunks.join("\n")).toBe(text);
+  });
+
+  test("uses custom delimiter", () => {
+    const text = "Section 1\n---\nSection 2\n---\nSection 3";
+
+    const chunks = defaultChunker(text, {
+      delimiter: "\n---\n",
+      minCharsSoftLimit: 5,
+      maxCharsSoftLimit: 50,
+    });
+
+    expect(chunks.length).toBeGreaterThan(0);
+    // Should be able to reconstruct original text with join("\n")
+    expect(chunks.join("\n")).toBe(text);
+  });
+
+  test("handles single line that exceeds max limit", () => {
+    const veryLongLine = "A".repeat(3000);
+
+    const chunks = defaultChunker(veryLongLine, {
+      minLines: 1,
+      minCharsSoftLimit: 200,
+      maxCharsSoftLimit: 1000,
+    });
+
+    // Should split even a single line if it's too long
+    expect(chunks.length).toBe(1);
+    expect(chunks.join("\n")).toBe(veryLongLine);
+  });
+
+  test("splits single line exceeding hard limit with custom hard limit", () => {
+    const longLine = "A".repeat(15000);
+
+    const chunks = defaultChunker(longLine, {
+      minLines: 1,
+      minCharsSoftLimit: 200,
+      maxCharsSoftLimit: 1000,
+      maxCharsHardLimit: 5000,
+    });
+
+    // Should be split into multiple chunks
+    expect(chunks.length).toBeGreaterThan(1);
+
+    // Each chunk should not exceed the hard limit
+    chunks.forEach((chunk: string) => {
+      expect(chunk.length).toBeLessThanOrEqual(5000);
+    });
+
+    // Content should be preserved when joined back together
+    expect(chunks.join("")).toBe(longLine);
+  });
+
+  test("splits extremely long single line with default hard limit", () => {
+    // Create a line that exceeds the default hard limit of 10000
+    const extremelyLongLine = "B".repeat(25000);
+
+    const chunks = defaultChunker(extremelyLongLine, {
+      minLines: 1,
+      minCharsSoftLimit: 200,
+      maxCharsSoftLimit: 1000,
+      // Using default maxCharsHardLimit of 10000
+    });
+
+    // Should be split into multiple chunks
+    expect(chunks.length).toBeGreaterThan(1);
+
+    // Each chunk should not exceed the default hard limit
+    chunks.forEach((chunk: string) => {
+      expect(chunk.length).toBeLessThanOrEqual(10000);
+    });
+
+    // Content should be preserved when joined back together
+    expect(chunks.join("")).toBe(extremelyLongLine);
+
+    // Should have at least 3 chunks for 25000 characters with 10000 limit
+    expect(chunks.length).toBeGreaterThanOrEqual(3);
+  });
+
+  test("verifies hard limit splitting with different character patterns", () => {
+    const longLine = "A".repeat(15000);
+
+    const chunks = defaultChunker(longLine, {
+      minLines: 1,
+      minCharsSoftLimit: 200,
+      maxCharsSoftLimit: 1000,
+      maxCharsHardLimit: 5000,
+    });
+
+    // Should be split into multiple chunks
+    expect(chunks.length).toBeGreaterThan(1);
+
+    // Each chunk should not exceed the hard limit
+    chunks.forEach((chunk: string) => {
+      expect(chunk.length).toBeLessThanOrEqual(5000);
+    });
+
+    // Content should be preserved when joined back together
+    expect(chunks.join("")).toBe(longLine);
+  });
+
+  test("preserves content without losing text", () => {
+    const originalText = `Paragraph 1 with some content.
+
+Paragraph 2 with different content.
+
+
+Paragraph 3 with more content.`;
+
+    const chunks = defaultChunker(originalText);
+    const reconstructed = chunks.join("\n");
+
+    // Should be able to reconstruct original text with join("\n")
+    expect(reconstructed).toBe(originalText);
+
+    // All original words should be preserved
+    const originalWords = originalText.split(/\s+/).filter((w) => w.length > 0);
+    const reconstructedWords = reconstructed
+      .split(/\s+/)
+      .filter((w) => w.length > 0);
+
+    expect(reconstructedWords.length).toBe(originalWords.length);
+    originalWords.forEach((word) => {
+      expect(reconstructed).toContain(word);
+    });
+    expect(chunks.join("\n")).toBe(originalText);
+  });
+});

package/src/client/defaultChunker.ts

@@ -0,0 +1,183 @@
+/**
+ * Chunk text for embedding.
+ *
+ * By default, it will chunk into paragraphs and target
+ * 200-2000 characters per chunk (only less than 1 line if the hard limit is reached).
+ */
+export function defaultChunker(
+  text: string,
+  {
+    minLines = 1,
+    minCharsSoftLimit = 200,
+    maxCharsSoftLimit = 2000,
+    maxCharsHardLimit = 10000,
+    delimiter = "\n\n",
+  }: {
+    minLines?: number;
+    minCharsSoftLimit?: number;
+    maxCharsSoftLimit?: number;
+    maxCharsHardLimit?: number;
+    delimiter?: string;
+  } = {}
+): string[] {
+  if (!text) return [];
+
+  // Split text into individual lines
+  const lines = text.split("\n");
+  const chunks: string[] = [];
+
+  let currentChunk: string[] = [];
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+
+    // Check if this line starts a new section (based on delimiter pattern)
+    const isNewSection = shouldStartNewSection(lines, i, delimiter);
+
+    // Calculate potential chunk if we add this line
+    const potentialChunk = [...currentChunk, line].join("\n");
+
+    // If adding this line would exceed max chars, finalize current chunk first
+    if (potentialChunk.length > maxCharsSoftLimit && currentChunk.length > 0) {
+      const trimmedChunk = removeTrailingEmptyLines(currentChunk);
+      chunks.push(trimmedChunk.join("\n"));
+
+      // Split the line if it exceeds hard limit
+      const splitLines = maybeSplitLine(line, maxCharsHardLimit);
+      // Add all but the last split piece as separate chunks
+      for (let j = 0; j < splitLines.length - 1; j++) {
+        chunks.push(splitLines[j]);
+      }
+      // Keep the last piece for potential combination with next lines
+      currentChunk = [splitLines[splitLines.length - 1]];
+      continue;
+    }
+
+    // If we're starting a new section and current chunk meets minimum requirements
+    if (
+      isNewSection &&
+      currentChunk.length >= minLines &&
+      currentChunk.join("\n").length >= Math.min(minCharsSoftLimit * 0.8, 150)
+    ) {
+      // Simple logic: only split if potential chunk would exceed the soft max limit
+      if (potentialChunk.length > maxCharsSoftLimit) {
+        // When splitting at delimiter boundary, preserve natural empty lines (don't remove trailing empty lines)
+        chunks.push(currentChunk.join("\n"));
+        currentChunk = [line];
+        continue;
+      }
+    }
+
+    // Add line to current chunk
+    currentChunk.push(line);
+
+    // If current chunk is too big, split it
+    if (currentChunk.join("\n").length > maxCharsSoftLimit) {
+      if (currentChunk.length === 1) {
+        // Single line too long - split it if it exceeds hard limit
+        const splitLines = maybeSplitLine(line, maxCharsHardLimit);
+        if (splitLines.length > 1) {
+          // Line was split - add all but the last piece as separate chunks
+          for (let j = 0; j < splitLines.length - 1; j++) {
+            chunks.push(splitLines[j]);
+          }
+          // Keep the last piece for potential combination with next lines
+          currentChunk = [splitLines[splitLines.length - 1]];
+        } else {
+          // Line doesn't exceed hard limit, keep it as is
+          chunks.push(line);
+          currentChunk = [];
+        }
+      } else {
+        // Remove last line and finalize chunk
+        const lastLine = currentChunk.pop()!;
+        const trimmedChunk = removeTrailingEmptyLines(currentChunk);
+        chunks.push(trimmedChunk.join("\n"));
+        currentChunk = [lastLine];
+      }
+    }
+  }
+
+  // Add remaining chunk, splitting if it exceeds hard limit
+  if (currentChunk.length > 0) {
+    const remainingText = currentChunk.join("\n");
+    if (remainingText.length > maxCharsHardLimit) {
+      // Split the remaining chunk if it exceeds hard limit
+      const splitLines = maybeSplitLine(remainingText, maxCharsHardLimit);
+      chunks.push(...splitLines);
+    } else {
+      const trimmedChunk = removeTrailingEmptyLines(currentChunk);
+      chunks.push(trimmedChunk.join("\n"));
+    }
+  }
+
+  return chunks;
+}
+
+function maybeSplitLine(line: string, maxCharsHardLimit: number): string[] {
+  const inputs = [line]; // in reverse order
+  const lines: string[] = [];
+  while (inputs.length > 0) {
+    const input = inputs.pop()!;
+    if (input.length <= maxCharsHardLimit) {
+      lines.push(input);
+      continue;
+    }
+    // split it in half
+    const splitIndex = Math.floor(input.length / 2);
+    const candidate = input.slice(0, splitIndex);
+    const rest = input.slice(splitIndex);
+    if (candidate.length < maxCharsHardLimit) {
+      lines.push(candidate, rest);
+    } else {
+      inputs.push(rest, candidate);
+    }
+  }
+  return lines;
+}
+
+function shouldStartNewSection(
+  lines: string[],
+  index: number,
+  delimiter: string
+): boolean {
+  if (index === 0) return false;
+
+  // For default "\n\n" delimiter, check for blank lines
+  if (delimiter === "\n\n") {
+    return lines[index - 1] === "";
+  }
+
+  // For custom delimiters, check if previous lines match the delimiter pattern
+  const delimiterLines = delimiter.split("\n");
+  if (delimiterLines.length <= 1) return false;
+
+  // Check if the delimiter pattern appears before this line
+  for (let i = 0; i < delimiterLines.length - 1; i++) {
+    const checkIndex = index - delimiterLines.length + 1 + i;
+    if (checkIndex < 0 || lines[checkIndex] !== delimiterLines[i]) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+function removeTrailingEmptyLines(lines: string[]): string[] {
+  // Don't remove anything if there's only one line
+  if (lines.length <= 1) {
+    return lines;
+  }
+
+  // Find the last non-empty line
+  for (let i = lines.length - 1; i >= 0; i--) {
+    if (lines[i].trim() !== "") {
+      return lines.slice(0, i + 1);
+    }
+  }
+
+  // If all lines are empty, keep at least one
+  return lines.length > 0 ? [lines[0]] : [];
+}
+
+export default defaultChunker;