@mastra/rag 0.10.3 → 1.0.0

package/.turbo/turbo-build.log

@@ -1,23 +1,23 @@
 
-> @mastra/rag@0.10.3-alpha.1 build /home/runner/work/mastra/mastra/packages/rag
+> @mastra/rag@1.0.0-alpha.0 build /home/runner/work/mastra/mastra/packages/rag
 > tsup src/index.ts --format esm,cjs --experimental-dts --clean --treeshake=smallest --splitting
 
 CLI Building entry: src/index.ts
 CLI Using tsconfig: tsconfig.json
 CLI tsup v8.5.0
 TSC Build start
-TSC ⚡️ Build success in 15594ms
+TSC ⚡️ Build success in 13246ms
 DTS Build start
 CLI Target: es2022
 Analysis will use the bundled TypeScript version 5.8.3
 Writing package typings: /home/runner/work/mastra/mastra/packages/rag/dist/_tsup-dts-rollup.d.ts
 Analysis will use the bundled TypeScript version 5.8.3
 Writing package typings: /home/runner/work/mastra/mastra/packages/rag/dist/_tsup-dts-rollup.d.cts
-DTS ⚡️ Build success in 12613ms
+DTS ⚡️ Build success in 12527ms
 CLI Cleaning output folder
 ESM Build start
 CJS Build start
-ESM dist/index.js 241.64 KB
-ESM ⚡️ Build success in 4247ms
-CJS dist/index.cjs 243.35 KB
-CJS ⚡️ Build success in 4248ms
+CJS dist/index.cjs 243.73 KB
+CJS ⚡️ Build success in 4842ms
+ESM dist/index.js 242.02 KB
+ESM ⚡️ Build success in 4846ms

package/CHANGELOG.md

@@ -1,5 +1,48 @@
 # @mastra/rag
 
+## 1.0.0
+
+### Major Changes
+
+- 75136cd: Add LaTeX chunking support by adding a case for Language.LATEX in getSeparatorsForLanguage.
+
+### Patch Changes
+
+- 63f6b7d: dependencies updates:
+  - Updated dependency [`zod@^3.25.57` ↗︎](https://www.npmjs.com/package/zod/v/3.25.57) (from `^3.25.56`, in `dependencies`)
+- Updated dependencies [63f6b7d]
+- Updated dependencies [12a95fc]
+- Updated dependencies [4b0f8a6]
+- Updated dependencies [51264a5]
+- Updated dependencies [8e6f677]
+- Updated dependencies [d70c420]
+- Updated dependencies [ee9af57]
+- Updated dependencies [36f1c36]
+- Updated dependencies [2a16996]
+- Updated dependencies [10d352e]
+- Updated dependencies [9589624]
+- Updated dependencies [53d3c37]
+- Updated dependencies [751c894]
+- Updated dependencies [577ce3a]
+- Updated dependencies [9260b3a]
+  - @mastra/core@0.10.6
+
+## 1.0.0-alpha.0
+
+### Major Changes
+
+- 75136cd: Add LaTeX chunking support by adding a case for Language.LATEX in getSeparatorsForLanguage.
+
+### Patch Changes
+
+- 63f6b7d: dependencies updates:
+  - Updated dependency [`zod@^3.25.57` ↗︎](https://www.npmjs.com/package/zod/v/3.25.57) (from `^3.25.56`, in `dependencies`)
+- Updated dependencies [63f6b7d]
+- Updated dependencies [36f1c36]
+- Updated dependencies [10d352e]
+- Updated dependencies [53d3c37]
+  - @mastra/core@0.10.6-alpha.0
+
 ## 0.10.3
 
 ### Patch Changes

package/dist/index.cjs

@@ -729,7 +729,7 @@ var customAlphabet = (alphabet, defaultSize = 21) => {
   };
 };
 
-// ../../node_modules/.pnpm/@ai-sdk+provider-utils@2.2.8_zod@3.25.56/node_modules/@ai-sdk/provider-utils/dist/index.mjs
+// ../../node_modules/.pnpm/@ai-sdk+provider-utils@2.2.8_zod@3.25.57/node_modules/@ai-sdk/provider-utils/dist/index.mjs
 var import_secure_json_parse = __toESM(require_secure_json_parse());
 function combineHeaders(...headers) {
   return headers.reduce(
@@ -4767,6 +4767,21 @@ var RecursiveCharacterTransformer = class _RecursiveCharacterTransformer extends
           " ",
           ""
         ];
+      case "latex" /* LATEX */:
+        return [
+          "\\\\part\\*?\\{",
+          "\\\\chapter\\*?\\{",
+          "\\\\section\\*?\\{",
+          "\\\\subsection\\*?\\{",
+          "\\\\subsubsection\\*?\\{",
+          "\\\\begin\\{.*?\\}",
+          "\\\\end\\{.*?\\}",
+          "\\\\[a-zA-Z]+\\{.*?\\}",
+          "\n\n",
+          "\n",
+          " ",
+          ""
+        ];
       // ... (add other language cases following the same pattern)
       default:
         throw new Error(`Language ${language} is not supported! Please choose from ${Object.values(Language)}`);

package/dist/index.js

@@ -727,7 +727,7 @@ var customAlphabet = (alphabet, defaultSize = 21) => {
   };
 };
 
-// ../../node_modules/.pnpm/@ai-sdk+provider-utils@2.2.8_zod@3.25.56/node_modules/@ai-sdk/provider-utils/dist/index.mjs
+// ../../node_modules/.pnpm/@ai-sdk+provider-utils@2.2.8_zod@3.25.57/node_modules/@ai-sdk/provider-utils/dist/index.mjs
 var import_secure_json_parse = __toESM(require_secure_json_parse());
 function combineHeaders(...headers) {
   return headers.reduce(
@@ -4765,6 +4765,21 @@ var RecursiveCharacterTransformer = class _RecursiveCharacterTransformer extends
           " ",
           ""
         ];
+      case "latex" /* LATEX */:
+        return [
+          "\\\\part\\*?\\{",
+          "\\\\chapter\\*?\\{",
+          "\\\\section\\*?\\{",
+          "\\\\subsection\\*?\\{",
+          "\\\\subsubsection\\*?\\{",
+          "\\\\begin\\{.*?\\}",
+          "\\\\end\\{.*?\\}",
+          "\\\\[a-zA-Z]+\\{.*?\\}",
+          "\n\n",
+          "\n",
+          " ",
+          ""
+        ];
       // ... (add other language cases following the same pattern)
       default:
         throw new Error(`Language ${language} is not supported! Please choose from ${Object.values(Language)}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mastra/rag",
-  "version": "0.10.3",
+  "version": "1.0.0",
   "description": "",
   "type": "module",
   "main": "dist/index.js",
@@ -26,26 +26,26 @@
     "js-tiktoken": "^1.0.20",
     "node-html-better-parser": "^1.4.11",
     "pathe": "^2.0.3",
-    "zod": "^3.25.56"
+    "zod": "^3.25.57"
   },
   "peerDependencies": {
-    "ai": "^4.0.0",
-    "@mastra/core": "^0.10.0-alpha.0"
+    "@mastra/core": "^0.10.0-alpha.0",
+    "ai": "^4.0.0"
   },
   "devDependencies": {
     "@ai-sdk/cohere": "latest",
     "@ai-sdk/openai": "latest",
     "@microsoft/api-extractor": "^7.52.8",
     "@types/big.js": "^6.2.2",
-    "@types/node": "^20.17.57",
+    "@types/node": "^20.19.0",
     "ai": "^4.3.16",
     "dotenv": "^16.5.0",
     "eslint": "^9.28.0",
     "tsup": "^8.5.0",
-    "typescript": "^5.8.2",
-    "vitest": "^3.2.2",
-    "@internal/lint": "0.0.11",
-    "@mastra/core": "0.10.4"
+    "typescript": "^5.8.3",
+    "vitest": "^3.2.3",
+    "@internal/lint": "0.0.13",
+    "@mastra/core": "0.10.6"
   },
   "keywords": [
     "rag",

package/src/document/document.test.ts

@@ -1505,6 +1505,111 @@ describe('MDocument', () => {
     });
   });
 
+  describe('chunkLaTeX', () => {
+    it('should split LaTeX text correctly based on sections', async () => {
+      const text = `\\section{Introduction}
+      
+      This is the introduction section.
+      
+      \\subsection{Background}
+      
+      Some background information.
+      
+      \\subsubsection{Details}
+      
+      Even more detailed explanation.
+      
+      \\section{Conclusion}
+      
+      Final thoughts here.`;
+
+      const doc = MDocument.fromText(text, { meta: 'data' });
+
+      await doc.chunk({
+        strategy: 'latex',
+        size: 100,
+        overlap: 10,
+        keepSeparator: 'start',
+      });
+
+      const chunks = doc.getText();
+      expect(chunks.length).toBeGreaterThan(1);
+      expect(chunks[0]).toContain('\\section{Introduction}');
+    });
+
+    it('should handle environments like equations or itemize', async () => {
+      const text = `\\section{Math Section}
+  
+      Here is an equation:
+      
+      \\[
+      E = mc^2
+      \\]
+      
+      \\begin{itemize}
+        \\item First item
+        \\item Second item
+      \\end{itemize}
+      
+      End of the section.`;
+
+      const doc = MDocument.fromText(text, { meta: 'data' });
+
+      await doc.chunk({
+        strategy: 'latex',
+        size: 100,
+        overlap: 10,
+        keepSeparator: 'start',
+      });
+
+      const chunks = doc.getText();
+      expect(chunks.some(chunk => chunk.includes('\\begin{itemize}'))).toBe(true);
+      expect(chunks.some(chunk => chunk.includes('E = mc^2'))).toBe(true);
+    });
+
+    it('should split with keepSeparator at end', async () => {
+      const text = `Intro text here.
+        \\section{First}
+        Content A.
+
+        \\section{Second}
+        Content B.`;
+
+      const doc = MDocument.fromText(text, { meta: 'data' });
+
+      await doc.chunk({
+        strategy: 'latex',
+        size: 50,
+        overlap: 0,
+        keepSeparator: 'end',
+      });
+
+      const chunks = doc.getText();
+      expect(chunks.length).toBe(3);
+      expect(chunks[0].trimEnd().includes('\\section{')).toBe(true);
+      expect(chunks[1].trimEnd().includes('\\section{')).toBe(true);
+    });
+
+    it('should strip whitespace correctly', async () => {
+      const text = `\\section{Whitespace}
+      
+        Content with leading and trailing whitespace.  
+      `;
+
+      const doc = MDocument.fromText(text, { meta: 'data' });
+
+      await doc.chunk({
+        strategy: 'latex',
+        size: 100,
+        overlap: 0,
+        stripWhitespace: true,
+      });
+
+      const chunks = doc.getText();
+      expect(chunks.every(chunk => chunk === chunk.trim())).toBe(true);
+    });
+  });
+
   describe('MarkdownHeader', () => {
     it('should split on headers and preserve metadata', async () => {
       const text = `# Main Title

package/src/document/transformers/character.ts

@@ -272,6 +272,21 @@ export class RecursiveCharacterTransformer extends TextTransformer {
           ' ',
           '',
         ];
+      case Language.LATEX:
+        return [
+          '\\\\part\\*?\\{',
+          '\\\\chapter\\*?\\{',
+          '\\\\section\\*?\\{',
+          '\\\\subsection\\*?\\{',
+          '\\\\subsubsection\\*?\\{',
+          '\\\\begin\\{.*?\\}',
+          '\\\\end\\{.*?\\}',
+          '\\\\[a-zA-Z]+\\{.*?\\}',
+          '\n\n',
+          '\n',
+          ' ',
+          '',
+        ];
       // ... (add other language cases following the same pattern)
       default:
         throw new Error(`Language ${language} is not supported! Please choose from ${Object.values(Language)}`);