@mastra/rag 2.0.0-beta.6 → 2.0.0

package/dist/index.js

@@ -4444,6 +4444,44 @@ var KeywordExtractor = class extends BaseExtractor {
     return results;
   }
 };
+var SchemaExtractor = class extends BaseExtractor {
+  schema;
+  llm;
+  instructions;
+  metadataKey;
+  constructor(options) {
+    super();
+    this.schema = options.schema;
+    this.llm = options.llm;
+    this.instructions = options.instructions;
+    this.metadataKey = options.metadataKey;
+  }
+  async extract(nodes) {
+    const agent = new Agent({
+      name: "schema-extractor",
+      id: "schema-extractor",
+      instructions: this.instructions ?? "Extract structured data from the provided text.",
+      model: this.llm ?? baseLLM
+    });
+    const results = await Promise.all(
+      nodes.map(async (node) => {
+        try {
+          const result = await agent.generate([{ role: "user", content: node.getContent() }], {
+            structuredOutput: { schema: this.schema }
+          });
+          if (this.metadataKey) {
+            return { [this.metadataKey]: result.object };
+          }
+          return result.object;
+        } catch (error) {
+          console.error("Schema extraction failed:", error);
+          return {};
+        }
+      })
+    );
+    return results;
+  }
+};
 
 // src/document/types.ts
 var Language = /* @__PURE__ */ ((Language2) => {
@@ -4481,14 +4519,14 @@ var TextTransformer = class {
   maxSize;
   overlap;
   lengthFunction;
-  keepSeparator;
+  separatorPosition;
   addStartIndex;
   stripWhitespace;
   constructor({
     maxSize = 4e3,
     overlap = 200,
     lengthFunction = (text) => text.length,
-    keepSeparator = false,
+    separatorPosition,
     addStartIndex = false,
     stripWhitespace = true
   }) {
@@ -4498,7 +4536,7 @@ var TextTransformer = class {
     this.maxSize = maxSize;
     this.overlap = overlap;
     this.lengthFunction = lengthFunction;
-    this.keepSeparator = keepSeparator;
+    this.separatorPosition = separatorPosition;
     this.addStartIndex = addStartIndex;
     this.stripWhitespace = stripWhitespace;
   }
@@ -4604,11 +4642,11 @@ var TextTransformer = class {
 };
 
 // src/document/transformers/character.ts
-function splitTextWithRegex(text, separator, keepSeparator) {
+function splitTextWithRegex(text, separator, separatorPosition) {
   if (!separator) {
     return text.split("");
   }
-  if (!keepSeparator) {
+  if (!separatorPosition) {
     return text.split(new RegExp(separator)).filter((s) => s !== "");
   }
   if (!text) {
@@ -4616,7 +4654,7 @@ function splitTextWithRegex(text, separator, keepSeparator) {
   }
   const splits = text.split(new RegExp(`(${separator})`));
   const result = [];
-  if (keepSeparator === "end") {
+  if (separatorPosition === "end") {
     for (let i = 0; i < splits.length - 1; i += 2) {
       if (i + 1 < splits.length) {
         const chunk = splits[i] + (splits[i + 1] || "");
@@ -4648,7 +4686,7 @@ var CharacterTransformer = class extends TextTransformer {
   }
   splitText({ text }) {
     const separator = this.isSeparatorRegex ? this.separator : this.separator.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-    const initialSplits = splitTextWithRegex(text, separator, this.keepSeparator);
+    const initialSplits = splitTextWithRegex(text, separator, this.separatorPosition);
     const chunks = [];
     for (const split of initialSplits) {
       if (this.lengthFunction(split) <= this.maxSize) {
@@ -4703,9 +4741,9 @@ var RecursiveCharacterTransformer = class _RecursiveCharacterTransformer extends
       }
     }
     const _separator = this.isSeparatorRegex ? separator : separator?.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-    const splits = splitTextWithRegex(text, _separator, this.keepSeparator);
+    const splits = splitTextWithRegex(text, _separator, this.separatorPosition);
     const goodSplits = [];
-    const mergeSeparator = this.keepSeparator ? "" : separator;
+    const mergeSeparator = this.separatorPosition ? "" : separator;
     for (const s of splits) {
       if (this.lengthFunction(s) < this.maxSize) {
         goodSplits.push(s);
@@ -5851,7 +5889,6 @@ var SentenceTransformer = class extends TextTransformer {
   sentenceEnders;
   fallbackToWords;
   fallbackToCharacters;
-  keepSeparator;
   constructor(options) {
     const parentOverlap = Math.min(options.overlap ?? 0, options.maxSize - 1);
     const baseOptions = {
@@ -5866,7 +5903,6 @@ var SentenceTransformer = class extends TextTransformer {
     this.sentenceEnders = options.sentenceEnders ?? [".", "!", "?"];
     this.fallbackToWords = options.fallbackToWords ?? true;
     this.fallbackToCharacters = options.fallbackToCharacters ?? true;
-    this.keepSeparator = options.keepSeparator ?? false;
     this.overlap = options.overlap ?? 0;
   }
   detectSentenceBoundaries(text) {
@@ -6167,8 +6203,8 @@ var TokenTransformer = class _TokenTransformer extends TextTransformer {
 var baseChunkOptionsSchema = z.object({
   maxSize: z.number().positive().optional(),
   overlap: z.number().min(0).optional(),
-  lengthFunction: z.function().optional(),
-  keepSeparator: z.union([z.boolean(), z.literal("start"), z.literal("end")]).optional(),
+  lengthFunction: z.optional(z.function()),
+  separatorPosition: z.enum(["start", "end"]).optional(),
   addStartIndex: z.boolean().optional(),
   stripWhitespace: z.boolean().optional()
 });
@@ -6263,8 +6299,11 @@ var MDocument = class _MDocument {
     });
     this.type = type;
   }
-  async extractMetadata({ title, summary, questions, keywords }) {
+  async extractMetadata({ title, summary, questions, keywords, schema }) {
     const transformations = [];
+    if (schema) {
+      transformations.push(new SchemaExtractor(schema));
+    }
     if (typeof summary !== "undefined") {
       transformations.push(new SummaryExtractor(typeof summary === "boolean" ? {} : summary));
     }
@@ -6415,7 +6454,7 @@ var MDocument = class _MDocument {
         const textSplitter = new RecursiveCharacterTransformer({
           maxSize: options.maxSize,
           overlap: options.overlap,
-          keepSeparator: options.keepSeparator,
+          separatorPosition: options.separatorPosition,
           addStartIndex: options.addStartIndex,
           stripWhitespace: options.stripWhitespace
         });
@@ -6431,7 +6470,7 @@ var MDocument = class _MDocument {
         const textSplitter = new RecursiveCharacterTransformer({
           maxSize: options.maxSize,
           overlap: options.overlap,
-          keepSeparator: options.keepSeparator,
+          separatorPosition: options.separatorPosition,
           addStartIndex: options.addStartIndex,
           stripWhitespace: options.stripWhitespace
         });
@@ -6494,7 +6533,7 @@ var MDocument = class _MDocument {
       sentenceEnders: options?.sentenceEnders,
       fallbackToWords: options?.fallbackToWords,
       fallbackToCharacters: options?.fallbackToCharacters,
-      keepSeparator: options?.keepSeparator,
+      separatorPosition: options?.separatorPosition,
       lengthFunction: options?.lengthFunction,
       addStartIndex: options?.addStartIndex,
       stripWhitespace: options?.stripWhitespace