@mastra/rag 0.2.0-alpha.1 → 0.10.1-alpha.0

package/.turbo/turbo-build.log

@@ -1,23 +1,23 @@
 
-> @mastra/rag@0.2.0-alpha.1 build /home/runner/work/mastra/mastra/packages/rag
+> @mastra/rag@0.10.1-alpha.0 build /home/runner/work/mastra/mastra/packages/rag
 > tsup src/index.ts --format esm,cjs --experimental-dts --clean --treeshake=smallest --splitting
 
 CLI Building entry: src/index.ts
 CLI Using tsconfig: tsconfig.json
 CLI tsup v8.4.0
 TSC Build start
-TSC ⚡️ Build success in 13960ms
+TSC ⚡️ Build success in 17241ms
 DTS Build start
 CLI Target: es2022
 Analysis will use the bundled TypeScript version 5.8.3
 Writing package typings: /home/runner/work/mastra/mastra/packages/rag/dist/_tsup-dts-rollup.d.ts
 Analysis will use the bundled TypeScript version 5.8.3
 Writing package typings: /home/runner/work/mastra/mastra/packages/rag/dist/_tsup-dts-rollup.d.cts
-DTS ⚡️ Build success in 14823ms
+DTS ⚡️ Build success in 16153ms
 CLI Cleaning output folder
 ESM Build start
 CJS Build start
-ESM dist/index.js 238.92 KB
-ESM ⚡️ Build success in 4442ms
-CJS dist/index.cjs 240.69 KB
-CJS ⚡️ Build success in 4451ms
+CJS dist/index.cjs 240.87 KB
+CJS ⚡️ Build success in 4013ms
+ESM dist/index.js 239.11 KB
+ESM ⚡️ Build success in 4026ms

package/CHANGELOG.md

@@ -1,5 +1,44 @@
 # @mastra/rag
 
+## 0.10.1-alpha.0
+
+### Patch Changes
+
+- 8784cef: Changed stripHeaders for markdown chunking to strip headers correctly from output when true
+- Updated dependencies [6d16390]
+- Updated dependencies [1e4a421]
+  - @mastra/core@0.10.1-alpha.0
+
+## 0.10.0
+
+### Minor Changes
+
+- 83da932: Move @mastra/core to peerdeps
+
+### Patch Changes
+
+- 4424836: [MASTRA-2591] Rag Tool Return Types
+- 8cdd799: [MASTRA-3078] added sources to return for vector query tool
+- 4f62987: update rerank weight sum to use big.js
+- Updated dependencies [b3a3d63]
+- Updated dependencies [344f453]
+- Updated dependencies [0a3ae6d]
+- Updated dependencies [95911be]
+- Updated dependencies [f53a6ac]
+- Updated dependencies [5eb5a99]
+- Updated dependencies [7e632c5]
+- Updated dependencies [1e9fbfa]
+- Updated dependencies [eabdcd9]
+- Updated dependencies [90be034]
+- Updated dependencies [99f050a]
+- Updated dependencies [d0ee3c6]
+- Updated dependencies [b2ae5aa]
+- Updated dependencies [23f258c]
+- Updated dependencies [a7292b0]
+- Updated dependencies [0dcb9f0]
+- Updated dependencies [2672a05]
+  - @mastra/core@0.10.0
+
 ## 0.2.0-alpha.1
 
 ### Minor Changes

package/dist/index.cjs

@@ -5459,10 +5459,12 @@ var MarkdownHeaderTransformer = class {
     }
     const aggregatedChunks = [];
     for (const line of lines) {
+      const lastLine = aggregatedChunks[aggregatedChunks.length - 1]?.content?.split("\n")?.slice(-1)[0]?.trim();
+      const lastChunkIsHeader = lastLine ? this.headersToSplitOn.some(([sep]) => lastLine.startsWith(sep)) : false;
       if (aggregatedChunks.length > 0 && JSON.stringify(aggregatedChunks?.[aggregatedChunks.length - 1].metadata) === JSON.stringify(line.metadata)) {
         const aggChunk = aggregatedChunks[aggregatedChunks.length - 1];
         aggChunk.content += "  \n" + line.content;
-      } else if (aggregatedChunks.length > 0 && JSON.stringify(aggregatedChunks?.[aggregatedChunks.length - 1].metadata) !== JSON.stringify(line.metadata) && Object.keys(aggregatedChunks?.[aggregatedChunks.length - 1].metadata).length < Object.keys(line.metadata).length && aggregatedChunks?.[aggregatedChunks.length - 1]?.content?.split("\n")?.slice(-1)[0][0] === "#" && !this.stripHeaders) {
+      } else if (aggregatedChunks.length > 0 && JSON.stringify(aggregatedChunks?.[aggregatedChunks.length - 1].metadata) !== JSON.stringify(line.metadata) && Object.keys(aggregatedChunks?.[aggregatedChunks.length - 1].metadata).length < Object.keys(line.metadata).length && lastChunkIsHeader) {
         if (aggregatedChunks && aggregatedChunks?.[aggregatedChunks.length - 1]) {
           const aggChunk = aggregatedChunks[aggregatedChunks.length - 1];
           if (aggChunk) {
@@ -5535,10 +5537,12 @@ var MarkdownHeaderTransformer = class {
             headerStack.push(header);
             initialMetadata[name14] = header.data;
           }
-          linesWithMetadata.push({
-            content: line,
-            metadata: { ...currentMetadata, ...initialMetadata }
-          });
+          if (!this.stripHeaders) {
+            linesWithMetadata.push({
+              content: line,
+              metadata: { ...currentMetadata, ...initialMetadata }
+            });
+          }
           break;
         }
       }

package/dist/index.js

@@ -5457,10 +5457,12 @@ var MarkdownHeaderTransformer = class {
     }
     const aggregatedChunks = [];
     for (const line of lines) {
+      const lastLine = aggregatedChunks[aggregatedChunks.length - 1]?.content?.split("\n")?.slice(-1)[0]?.trim();
+      const lastChunkIsHeader = lastLine ? this.headersToSplitOn.some(([sep]) => lastLine.startsWith(sep)) : false;
       if (aggregatedChunks.length > 0 && JSON.stringify(aggregatedChunks?.[aggregatedChunks.length - 1].metadata) === JSON.stringify(line.metadata)) {
         const aggChunk = aggregatedChunks[aggregatedChunks.length - 1];
         aggChunk.content += "  \n" + line.content;
-      } else if (aggregatedChunks.length > 0 && JSON.stringify(aggregatedChunks?.[aggregatedChunks.length - 1].metadata) !== JSON.stringify(line.metadata) && Object.keys(aggregatedChunks?.[aggregatedChunks.length - 1].metadata).length < Object.keys(line.metadata).length && aggregatedChunks?.[aggregatedChunks.length - 1]?.content?.split("\n")?.slice(-1)[0][0] === "#" && !this.stripHeaders) {
+      } else if (aggregatedChunks.length > 0 && JSON.stringify(aggregatedChunks?.[aggregatedChunks.length - 1].metadata) !== JSON.stringify(line.metadata) && Object.keys(aggregatedChunks?.[aggregatedChunks.length - 1].metadata).length < Object.keys(line.metadata).length && lastChunkIsHeader) {
         if (aggregatedChunks && aggregatedChunks?.[aggregatedChunks.length - 1]) {
           const aggChunk = aggregatedChunks[aggregatedChunks.length - 1];
           if (aggChunk) {
@@ -5533,10 +5535,12 @@ var MarkdownHeaderTransformer = class {
             headerStack.push(header);
             initialMetadata[name14] = header.data;
           }
-          linesWithMetadata.push({
-            content: line,
-            metadata: { ...currentMetadata, ...initialMetadata }
-          });
+          if (!this.stripHeaders) {
+            linesWithMetadata.push({
+              content: line,
+              metadata: { ...currentMetadata, ...initialMetadata }
+            });
+          }
           break;
         }
       }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mastra/rag",
-  "version": "0.2.0-alpha.1",
+  "version": "0.10.1-alpha.0",
   "description": "",
   "type": "module",
   "main": "dist/index.js",
@@ -30,7 +30,7 @@
   },
   "peerDependencies": {
     "ai": "^4.0.0",
-    "@mastra/core": "^0.9.4"
+    "@mastra/core": "^0.10.0"
   },
   "devDependencies": {
     "@ai-sdk/cohere": "latest",
@@ -44,8 +44,8 @@
     "tsup": "^8.4.0",
     "typescript": "^5.8.2",
     "vitest": "^3.1.2",
-    "@internal/lint": "0.0.5",
-    "@mastra/core": "0.10.0-alpha.1"
+    "@internal/lint": "0.0.6",
+    "@mastra/core": "0.10.1-alpha.0"
   },
   "keywords": [
     "rag",

package/src/document/document.test.ts

@@ -1611,6 +1611,7 @@ describe('MDocument', () => {
         strategy: 'markdown',
         headers: [['#', 'Header 1']],
         returnEachLine: true,
+        stripHeaders: false,
       });
 
       expect(doc.getDocs().length).toBe(4); // Title + 3 lines
@@ -1639,6 +1640,125 @@ describe('MDocument', () => {
       const docs = doc.getDocs();
       expect(docs?.[0]?.text).toContain('# Title');
     });
+
+    it('should remove headers when stripHeaders: true is set in markdown chunker', async () => {
+      const markdown = [
+        '# H1 Title',
+        'Some intro text.',
+        '## H2 Subtitle',
+        'More details.',
+        '### H3 Section',
+        'Final content.',
+      ].join('\n');
+
+      const doc = MDocument.fromMarkdown(markdown);
+      const chunks = await doc.chunk({
+        strategy: 'markdown',
+        size: 500,
+        overlap: 0,
+        headers: [
+          ['#', 'h1'],
+          ['##', 'h2'],
+          ['###', 'h3'],
+        ],
+        stripHeaders: true,
+      });
+      // None of the chunk texts should start with the header patterns
+      const headerPatterns = [/^#\s/, /^##\s/, /^###\s/];
+      for (const chunk of chunks) {
+        for (const pattern of headerPatterns) {
+          expect(pattern.test(chunk.text)).toBe(false);
+        }
+      }
+    });
+
+    it('should support custom header prefixes', async () => {
+      const text = `!!! Important\nThis is important.\n--- Section\nSection content.`;
+      const doc = MDocument.fromMarkdown(text);
+      await doc.chunk({
+        strategy: 'markdown',
+        headers: [
+          ['!!!', 'important'],
+          ['---', 'section'],
+        ],
+        stripHeaders: true,
+      });
+      const texts = doc.getText();
+      expect(texts.some(t => t.startsWith('!!!'))).toBe(false);
+      expect(texts.some(t => t.startsWith('---'))).toBe(false);
+    });
+
+    it('should attach correct metadata for nested headers', async () => {
+      const text = `# H1\n## H2\n### H3\nContent`;
+      const doc = MDocument.fromMarkdown(text);
+      await doc.chunk({
+        strategy: 'markdown',
+        headers: [
+          ['#', 'h1'],
+          ['##', 'h2'],
+          ['###', 'h3'],
+        ],
+        stripHeaders: true,
+      });
+      const chunk = doc.getDocs().find(c => c.text.includes('Content'));
+      expect(chunk?.metadata?.h1).toBe('H1');
+      expect(chunk?.metadata?.h2).toBe('H2');
+      expect(chunk?.metadata?.h3).toBe('H3');
+    });
+
+    it('should include header lines as chunks if stripHeaders is false', async () => {
+      const text = `# H1\nContent`;
+      const doc = MDocument.fromMarkdown(text);
+      await doc.chunk({
+        strategy: 'markdown',
+        headers: [['#', 'h1']],
+        stripHeaders: false,
+      });
+      const texts = doc.getText();
+      expect(texts.some(t => t.startsWith('# H1'))).toBe(true);
+    });
+
+    it('should handle multiple adjacent headers correctly', async () => {
+      const text = `# H1\n## H2\n### H3\nContent`;
+      const doc = MDocument.fromMarkdown(text);
+      await doc.chunk({
+        strategy: 'markdown',
+        headers: [
+          ['#', 'h1'],
+          ['##', 'h2'],
+          ['###', 'h3'],
+        ],
+        stripHeaders: true,
+      });
+      const texts = doc.getText();
+      expect(texts.some(t => t === 'Content')).toBe(true);
+      expect(texts.some(t => t === '')).toBe(false);
+    });
+
+    it('should handle content before any header', async () => {
+      const text = `Intro before header\n# H1\nContent`;
+      const doc = MDocument.fromMarkdown(text);
+      await doc.chunk({
+        strategy: 'markdown',
+        headers: [['#', 'h1']],
+        stripHeaders: true,
+      });
+      const preHeaderChunk = doc.getDocs().find(c => c.text.includes('Intro before header'));
+      expect(preHeaderChunk?.metadata?.h1).toBeUndefined();
+    });
+
+    it('should not treat headers inside code blocks as headers', async () => {
+      const text = ['# Real Header', '```', '# Not a header', '```', 'Content'].join('\n');
+      const doc = MDocument.fromMarkdown(text);
+      await doc.chunk({
+        strategy: 'markdown',
+        headers: [['#', 'h1']],
+        stripHeaders: true,
+      });
+      const texts = doc.getText();
+      expect(texts.some(t => t.includes('# Not a header'))).toBe(true);
+      expect(texts.some(t => t.startsWith('# Real Header'))).toBe(false);
+    });
   });
 
   describe('metadata extraction', () => {

package/src/document/transformers/markdown.ts

@@ -61,6 +61,8 @@ export class MarkdownHeaderTransformer {
     const aggregatedChunks: LineType[] = [];
 
     for (const line of lines) {
+      const lastLine = aggregatedChunks[aggregatedChunks.length - 1]?.content?.split('\n')?.slice(-1)[0]?.trim();
+      const lastChunkIsHeader = lastLine ? this.headersToSplitOn.some(([sep]) => lastLine.startsWith(sep)) : false;
       if (
         aggregatedChunks.length > 0 &&
         JSON.stringify(aggregatedChunks?.[aggregatedChunks.length - 1]!.metadata) === JSON.stringify(line.metadata)
@@ -72,8 +74,7 @@ export class MarkdownHeaderTransformer {
         JSON.stringify(aggregatedChunks?.[aggregatedChunks.length - 1]!.metadata) !== JSON.stringify(line.metadata) &&
         Object.keys(aggregatedChunks?.[aggregatedChunks.length - 1]!.metadata).length <
           Object.keys(line.metadata).length &&
-        aggregatedChunks?.[aggregatedChunks.length - 1]?.content?.split('\n')?.slice(-1)[0]![0] === '#' &&
-        !this.stripHeaders
+        lastChunkIsHeader
       ) {
         if (aggregatedChunks && aggregatedChunks?.[aggregatedChunks.length - 1]) {
           const aggChunk = aggregatedChunks[aggregatedChunks.length - 1];
@@ -166,12 +167,13 @@ export class MarkdownHeaderTransformer {
             initialMetadata[name] = header.data;
           }
 
-          // Always create a separate chunk for the header
-          linesWithMetadata.push({
-            content: line,
-            metadata: { ...currentMetadata, ...initialMetadata },
-          });
-
+          // Only add header to linesWithMetadata if stripHeaders is false
+          if (!this.stripHeaders) {
+            linesWithMetadata.push({
+              content: line,
+              metadata: { ...currentMetadata, ...initialMetadata },
+            });
+          }
           break;
         }
       }
@@ -197,6 +199,7 @@ export class MarkdownHeaderTransformer {
         }
       }
 
+      // Reset metadata for next line
       currentMetadata = { ...initialMetadata };
     }