@mastra/rag 0.1.14-alpha.3 → 0.1.15-alpha.1

package/.turbo/turbo-build.log

@@ -1,23 +1,23 @@
 
-> @mastra/rag@0.1.14-alpha.3 build /home/runner/work/mastra/mastra/packages/rag
+> @mastra/rag@0.1.15-alpha.1 build /home/runner/work/mastra/mastra/packages/rag
 > tsup src/index.ts --format esm,cjs --experimental-dts --clean --treeshake=smallest --splitting
 
 CLI Building entry: src/index.ts
 CLI Using tsconfig: tsconfig.json
 CLI tsup v8.4.0
 TSC Build start
-TSC ⚡️ Build success in 27617ms
+TSC ⚡️ Build success in 27112ms
 DTS Build start
 CLI Target: es2022
 Analysis will use the bundled TypeScript version 5.8.2
 Writing package typings: /home/runner/work/mastra/mastra/packages/rag/dist/_tsup-dts-rollup.d.ts
 Analysis will use the bundled TypeScript version 5.8.2
 Writing package typings: /home/runner/work/mastra/mastra/packages/rag/dist/_tsup-dts-rollup.d.cts
-DTS ⚡️ Build success in 40031ms
+DTS ⚡️ Build success in 35973ms
 CLI Cleaning output folder
 ESM Build start
 CJS Build start
-ESM dist/index.js 92.31 KB
-ESM ⚡️ Build success in 2414ms
-CJS dist/index.cjs 93.03 KB
-CJS ⚡️ Build success in 2414ms
+CJS dist/index.cjs 93.37 KB
+CJS ⚡️ Build success in 1563ms
+ESM dist/index.js 92.65 KB
+ESM ⚡️ Build success in 1563ms

package/CHANGELOG.md

@@ -1,5 +1,47 @@
 # @mastra/rag
 
+## 0.1.15-alpha.1
+
+### Patch Changes
+
+- e47f529: Updated KeywordExtraction in chunk
+- Updated dependencies [619c39d]
+- Updated dependencies [fe56be0]
+- Updated dependencies [a0967a0]
+- Updated dependencies [fca3b21]
+- Updated dependencies [0118361]
+- Updated dependencies [619c39d]
+  - @mastra/core@0.8.0-alpha.1
+
+## 0.1.15-alpha.0
+
+### Patch Changes
+
+- 7599d77: fix(deps): update ai sdk to ^4.2.2
+- Updated dependencies [107bcfe]
+- Updated dependencies [5b4e19f]
+- Updated dependencies [7599d77]
+- Updated dependencies [cafae83]
+- Updated dependencies [8076ecf]
+- Updated dependencies [304397c]
+  - @mastra/core@0.7.1-alpha.0
+
+## 0.1.14
+
+### Patch Changes
+
+- Updated dependencies [b4fbc59]
+- Updated dependencies [a838fde]
+- Updated dependencies [a8bd4cf]
+- Updated dependencies [7a3eeb0]
+- Updated dependencies [0b54522]
+- Updated dependencies [b3b34f5]
+- Updated dependencies [1af25d5]
+- Updated dependencies [a4686e8]
+- Updated dependencies [6530ad1]
+- Updated dependencies [27439ad]
+  - @mastra/core@0.7.0
+
 ## 0.1.14-alpha.3
 
 ### Patch Changes

package/dist/_tsup-dts-rollup.d.cts

@@ -138,7 +138,7 @@ declare type ExtractParams = {
     title?: TitleExtractorsArgs | boolean;
     summary?: SummaryExtractArgs | boolean;
     questions?: QuestionAnswerExtractArgs | boolean;
-    keywords?: boolean | Record<string, any>;
+    keywords?: KeywordExtractArgs | boolean;
 };
 export { ExtractParams }
 export { ExtractParams as ExtractParams_alias_1 }

package/dist/_tsup-dts-rollup.d.ts

@@ -138,7 +138,7 @@ declare type ExtractParams = {
     title?: TitleExtractorsArgs | boolean;
     summary?: SummaryExtractArgs | boolean;
     questions?: QuestionAnswerExtractArgs | boolean;
-    keywords?: boolean | Record<string, any>;
+    keywords?: KeywordExtractArgs | boolean;
 };
 export { ExtractParams }
 export { ExtractParams as ExtractParams_alias_1 }

package/dist/index.cjs

@@ -1305,6 +1305,18 @@ var MDocument = class _MDocument {
     }
     if (typeof title !== "undefined") {
       transformations.push(new llamaindex.TitleExtractor(typeof title === "boolean" ? {} : title));
+      this.chunks = this.chunks.map(
+        (doc, i) => new llamaindex.Document({
+          ...doc,
+          relationships: {
+            [llamaindex.NodeRelationship.SOURCE]: {
+              nodeId: `doc-${i}`,
+              nodeType: llamaindex.ObjectType.DOCUMENT,
+              metadata: doc.metadata
+            }
+          }
+        })
+      );
     }
     const pipeline = new llamaindex.IngestionPipeline({
       transformations

package/dist/index.js

@@ -1,4 +1,4 @@
-import { Document, SummaryExtractor, QuestionsAnsweredExtractor, KeywordExtractor, TitleExtractor, IngestionPipeline } from 'llamaindex';
+import { Document, SummaryExtractor, QuestionsAnsweredExtractor, KeywordExtractor, TitleExtractor, ObjectType, NodeRelationship, IngestionPipeline } from 'llamaindex';
 import { parse } from 'node-html-better-parser';
 import { encodingForModel, getEncoding } from 'js-tiktoken';
 import { CohereRelevanceScorer, MastraAgentRelevanceScorer } from '@mastra/core/relevance';
@@ -1303,6 +1303,18 @@ var MDocument = class _MDocument {
     }
     if (typeof title !== "undefined") {
       transformations.push(new TitleExtractor(typeof title === "boolean" ? {} : title));
+      this.chunks = this.chunks.map(
+        (doc, i) => new Document({
+          ...doc,
+          relationships: {
+            [NodeRelationship.SOURCE]: {
+              nodeId: `doc-${i}`,
+              nodeType: ObjectType.DOCUMENT,
+              metadata: doc.metadata
+            }
+          }
+        })
+      );
     }
     const pipeline = new IngestionPipeline({
       transformations

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mastra/rag",
-  "version": "0.1.14-alpha.3",
+  "version": "0.1.15-alpha.1",
   "description": "",
   "type": "module",
   "main": "dist/index.js",
@@ -29,7 +29,7 @@
     "node-html-better-parser": "^1.4.7",
     "pathe": "^2.0.3",
     "zod": "^3.24.2",
-    "@mastra/core": "^0.7.0-alpha.3"
+    "@mastra/core": "^0.8.0-alpha.1"
   },
   "peerDependencies": {
     "ai": "^4.0.0"
@@ -39,7 +39,7 @@
     "@ai-sdk/openai": "latest",
     "@microsoft/api-extractor": "^7.52.1",
     "@types/node": "^20.17.27",
-    "ai": "^4.1.54",
+    "ai": "^4.2.2",
     "dotenv": "^16.4.7",
     "eslint": "^9.23.0",
     "tsup": "^8.4.0",

package/src/document/document.test.ts

@@ -1638,6 +1638,79 @@ describe('MDocument', () => {
       expect(docs?.[0]?.text).toContain('# Title');
     });
   });
+
+  describe('metadata extraction', () => {
+    it('should extract metadata with default settings', async () => {
+      const doc = MDocument.fromMarkdown(
+        '# AI and Machine Learning\n\nThis is a test document about artificial intelligence and machine learning.',
+      );
+
+      const chunks = await doc.chunk({
+        strategy: 'markdown',
+        extract: {
+          title: true,
+          summary: true,
+          keywords: true,
+        },
+      });
+
+      const metadata = chunks[0].metadata;
+      expect(metadata).toBeDefined();
+      expect(metadata.documentTitle).toBeDefined();
+      expect(metadata.sectionSummary).toBeDefined();
+      expect(metadata.excerptKeywords).toMatch(/^KEYWORDS: .*/);
+    }, 15000);
+
+    it('should extract metadata with custom settings', async () => {
+      const doc = MDocument.fromMarkdown(
+        '# AI and Machine Learning\n\nThis is a test document about artificial intelligence and machine learning.',
+      );
+
+      const chunks = await doc.chunk({
+        strategy: 'markdown',
+        extract: {
+          title: {
+            nodes: 2,
+            nodeTemplate: 'Generate a title for this: {context}',
+            combineTemplate: 'Combine these titles: {context}',
+          },
+          summary: {
+            summaries: ['self'],
+            promptTemplate: 'Summarize this: {context}',
+          },
+          questions: {
+            questions: 2,
+            promptTemplate: 'Generate {numQuestions} questions about: {context}',
+          },
+          keywords: {
+            keywords: 3,
+            promptTemplate: 'Extract {maxKeywords} key terms from: {context}',
+          },
+        },
+      });
+
+      const metadata = chunks[0].metadata;
+      expect(metadata).toBeDefined();
+      expect(metadata.documentTitle).toBeDefined();
+      expect(metadata.sectionSummary).toBeDefined();
+      expect(metadata.questionsThisExcerptCanAnswer).toMatch(/^1\. .*\?2\. .*\?$/);
+      expect(metadata.excerptKeywords).toMatch(/^1\. .*\n2\. .*\n3\. .*$/);
+    }, 15000);
+
+    it('should handle invalid summary types', async () => {
+      const doc = MDocument.fromText('Test document');
+
+      await expect(
+        doc.chunk({
+          extract: {
+            summary: {
+              summaries: ['invalid'],
+            },
+          },
+        }),
+      ).rejects.toThrow("Summaries must be one of 'self', 'prev', 'next'");
+    }, 15000);
+  });
 });
 
 // Helper function to find the longest common substring between two strings

package/src/document/document.ts

@@ -5,6 +5,8 @@ import {
   QuestionsAnsweredExtractor,
   SummaryExtractor,
   TitleExtractor,
+  ObjectType,
+  NodeRelationship,
 } from 'llamaindex';
 
 import { CharacterTransformer, RecursiveCharacterTransformer } from './transformers/character';
@@ -43,6 +45,19 @@ export class MDocument {
 
     if (typeof title !== 'undefined') {
       transformations.push(new TitleExtractor(typeof title === 'boolean' ? {} : title));
+      this.chunks = this.chunks.map(
+        (doc, i) =>
+          new Chunk({
+            ...doc,
+            relationships: {
+              [NodeRelationship.SOURCE]: {
+                nodeId: `doc-${i}`,
+                nodeType: ObjectType.DOCUMENT,
+                metadata: doc.metadata,
+              },
+            },
+          }),
+      );
     }
 
     const pipeline = new IngestionPipeline({

package/src/document/types.ts

@@ -41,7 +41,7 @@ export type ExtractParams = {
   title?: TitleExtractorsArgs | boolean;
   summary?: SummaryExtractArgs | boolean;
   questions?: QuestionAnswerExtractArgs | boolean;
-  keywords?: boolean | Record<string, any>;
+  keywords?: KeywordExtractArgs | boolean;
 };
 
 export type ChunkOptions = {