@mastra/rag 0.1.20-alpha.0 → 0.1.20-alpha.2

package/dist/index.js

@@ -1,4 +1,4 @@
-import { Document, ObjectType, NodeRelationship, IngestionPipeline, BaseExtractor, PromptTemplate, defaultSummaryPrompt, TextNode, defaultQuestionExtractPrompt, defaultKeywordExtractPrompt, MetadataMode, defaultTitleExtractorPromptTemplate, defaultTitleCombinePromptTemplate } from 'llamaindex';
+import { randomUUID, createHash } from 'crypto';
 import { z } from 'zod';
 import { parse } from 'node-html-better-parser';
 import { encodingForModel, getEncoding } from 'js-tiktoken';
@@ -12,6 +12,11 @@ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
 var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __knownSymbol = (name14, symbol15) => (symbol15 = Symbol[name14]) ? symbol15 : Symbol.for("Symbol." + name14);
+var __typeError = (msg) => {
+  throw TypeError(msg);
+};
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
 var __commonJS = (cb, mod) => function __require() {
   return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
 };
@@ -31,6 +36,43 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
   __defProp(target, "default", { value: mod, enumerable: true }) ,
   mod
 ));
+var __decoratorStart = (base) => [, , , __create(null)];
+var __decoratorStrings = ["class", "method", "getter", "setter", "accessor", "field", "value", "get", "set"];
+var __expectFn = (fn) => fn !== void 0 && typeof fn !== "function" ? __typeError("Function expected") : fn;
+var __decoratorContext = (kind, name14, done, metadata, fns) => ({ kind: __decoratorStrings[kind], name: name14, metadata, addInitializer: (fn) => done._ ? __typeError("Already initialized") : fns.push(__expectFn(fn || null)) });
+var __decoratorMetadata = (array, target) => __defNormalProp(target, __knownSymbol("metadata"), array[3]);
+var __runInitializers = (array, flags, self, value) => {
+  for (var i = 0, fns = array[flags >> 1], n = fns && fns.length; i < n; i++) flags & 1 ? fns[i].call(self) : value = fns[i].call(self, value);
+  return value;
+};
+var __decorateElement = (array, flags, name14, decorators, target, extra) => {
+  var fn, it, done, ctx, access, k = flags & 7, s = false, p = false;
+  var j = array.length + 1 , key = __decoratorStrings[k + 5];
+  var initializers = (array[j - 1] = []), extraInitializers = array[j] || (array[j] = []);
+  var desc = ((target = target.prototype), __getOwnPropDesc({ get [name14]() {
+    return __privateGet(this, extra);
+  }, set [name14](x) {
+    return __privateSet(this, extra, x);
+  } }, name14));
+  for (var i = decorators.length - 1; i >= 0; i--) {
+    ctx = __decoratorContext(k, name14, done = {}, array[3], extraInitializers);
+    {
+      ctx.static = s, ctx.private = p, access = ctx.access = { has: (x) => name14 in x };
+      access.get = (x) => x[name14];
+      access.set = (x, y) => x[name14] = y;
+    }
+    it = (0, decorators[i])({ get: desc.get, set: desc.set } , ctx), done._ = 1;
+    if (it === void 0) __expectFn(it) && (desc[key] = it );
+    else if (typeof it !== "object" || it === null) __typeError("Object expected");
+    else __expectFn(fn = it.get) && (desc.get = fn), __expectFn(fn = it.set) && (desc.set = fn), __expectFn(fn = it.init) && initializers.unshift(fn);
+  }
+  return desc && __defProp(target, name14, desc), target;
+};
+var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
+var __accessCheck = (obj, member, msg) => member.has(obj) || __typeError("Cannot " + msg);
+var __privateGet = (obj, member, getter) => (__accessCheck(obj, member, "read from private field"), member.get(obj));
+var __privateAdd = (obj, member, value) => member.has(obj) ? __typeError("Cannot add the same private member more than once") : member instanceof WeakSet ? member.add(obj) : member.set(obj, value);
+var __privateSet = (obj, member, value, setter) => (__accessCheck(obj, member, "write to private field"), member.set(obj, value), value);
 
 // ../../node_modules/.pnpm/secure-json-parse@2.7.0/node_modules/secure-json-parse/index.js
 var require_secure_json_parse = __commonJS({
@@ -135,6 +177,275 @@ var require_secure_json_parse = __commonJS({
   }
 });
 
+// src/document/prompts/format.ts
+function format(str, params) {
+  return str.replace(/{(\w+)}/g, (_, k) => params[k] ?? "");
+}
+
+// src/document/prompts/base.ts
+var BasePromptTemplate = class {
+  templateVars = /* @__PURE__ */ new Set();
+  options = {};
+  constructor(options) {
+    const { templateVars } = options;
+    if (templateVars) {
+      this.templateVars = new Set(templateVars);
+    }
+    if (options.options) {
+      this.options = options.options;
+    }
+  }
+};
+var PromptTemplate = class _PromptTemplate extends BasePromptTemplate {
+  #template;
+  constructor(options) {
+    const { template, ...rest } = options;
+    super(rest);
+    this.#template = template;
+  }
+  partialFormat(options) {
+    const prompt = new _PromptTemplate({
+      template: this.template,
+      templateVars: [...this.templateVars],
+      options: this.options
+    });
+    prompt.options = {
+      ...prompt.options,
+      ...options
+    };
+    return prompt;
+  }
+  format(options) {
+    const allOptions = {
+      ...this.options,
+      ...options
+    };
+    return format(this.template, allOptions);
+  }
+  formatMessages(options) {
+    const prompt = this.format(options);
+    return [
+      {
+        role: "user",
+        content: prompt
+      }
+    ];
+  }
+  get template() {
+    return this.#template;
+  }
+};
+
+// src/document/prompts/prompt.ts
+var defaultSummaryPrompt = new PromptTemplate({
+  templateVars: ["context"],
+  template: `Write a summary of the following. Try to use only the information provided. Try to include as many key details as possible.
+
+
+{context}
+
+
+SUMMARY:"""
+`
+});
+var defaultKeywordExtractPrompt = new PromptTemplate({
+  templateVars: ["maxKeywords", "context"],
+  template: `
+Some text is provided below. Given the text, extract up to {maxKeywords} keywords from the text. Avoid stopwords.
+---------------------
+{context}
+---------------------
+Provide keywords in the following comma-separated format: 'KEYWORDS: <keywords>'
+`
+}).partialFormat({
+  maxKeywords: "10"
+});
+var defaultQuestionExtractPrompt = new PromptTemplate({
+  templateVars: ["numQuestions", "context"],
+  template: `(
+  "Given the contextual informations below, generate {numQuestions} questions this context can provides specific answers to which are unlikely to be found else where. Higher-level summaries of surrounding context may be provided as well. "
+  "Try using these summaries to generate better questions that this context can answer."
+  "---------------------"
+  "{context}"
+  "---------------------"
+  "Provide questions in the following format: 'QUESTIONS: <questions>'"
+)`
+}).partialFormat({
+  numQuestions: "5"
+});
+var defaultTitleExtractorPromptTemplate = new PromptTemplate({
+  templateVars: ["context"],
+  template: `{context}
+Give a title that summarizes all of the unique entities, titles or themes found in the context. 
+Title: `
+});
+var defaultTitleCombinePromptTemplate = new PromptTemplate({
+  templateVars: ["context"],
+  template: `{context} 
+Based on the above candidate titles and contents, what is the comprehensive title for this document? 
+Title: `
+});
+var _hash_dec, _init, _hash;
+_hash_dec = [lazyInitHash];
+var BaseNode = class {
+  constructor(init) {
+    __publicField(this, "id_");
+    __publicField(this, "metadata");
+    __publicField(this, "relationships");
+    __privateAdd(this, _hash, __runInitializers(_init, 8, this, "")), __runInitializers(_init, 11, this);
+    const { id_, metadata, relationships } = init || {};
+    this.id_ = id_ ?? randomUUID();
+    this.metadata = metadata ?? {};
+    this.relationships = relationships ?? {};
+  }
+  get sourceNode() {
+    const relationship = this.relationships["SOURCE" /* SOURCE */];
+    if (Array.isArray(relationship)) {
+      throw new Error("Source object must be a single RelatedNodeInfo object");
+    }
+    return relationship;
+  }
+  get prevNode() {
+    const relationship = this.relationships["PREVIOUS" /* PREVIOUS */];
+    if (Array.isArray(relationship)) {
+      throw new Error("Previous object must be a single RelatedNodeInfo object");
+    }
+    return relationship;
+  }
+  get nextNode() {
+    const relationship = this.relationships["NEXT" /* NEXT */];
+    if (Array.isArray(relationship)) {
+      throw new Error("Next object must be a single RelatedNodeInfo object");
+    }
+    return relationship;
+  }
+  get parentNode() {
+    const relationship = this.relationships["PARENT" /* PARENT */];
+    if (Array.isArray(relationship)) {
+      throw new Error("Parent object must be a single RelatedNodeInfo object");
+    }
+    return relationship;
+  }
+  get childNodes() {
+    const relationship = this.relationships["CHILD" /* CHILD */];
+    if (!Array.isArray(relationship)) {
+      throw new Error("Child object must be a an array of RelatedNodeInfo objects");
+    }
+    return relationship;
+  }
+};
+_init = __decoratorStart();
+_hash = new WeakMap();
+__decorateElement(_init, 4, "hash", _hash_dec, BaseNode, _hash);
+__decoratorMetadata(_init, BaseNode);
+var TextNode = class extends BaseNode {
+  text;
+  startCharIdx;
+  endCharIdx;
+  metadataSeparator;
+  constructor(init = {}) {
+    super(init);
+    const { text, startCharIdx, endCharIdx, metadataSeparator } = init;
+    this.text = text ?? "";
+    if (startCharIdx) {
+      this.startCharIdx = startCharIdx;
+    }
+    if (endCharIdx) {
+      this.endCharIdx = endCharIdx;
+    }
+    this.metadataSeparator = metadataSeparator ?? "\n";
+  }
+  /**
+   * Generate a hash of the text node.
+   * The ID is not part of the hash as it can change independent of content.
+   * @returns
+   */
+  generateHash() {
+    const hashFunction = createSHA256();
+    hashFunction.update(`type=${this.type}`);
+    hashFunction.update(`startCharIdx=${this.startCharIdx} endCharIdx=${this.endCharIdx}`);
+    hashFunction.update(this.getContent());
+    return hashFunction.digest();
+  }
+  get type() {
+    return "TEXT" /* TEXT */;
+  }
+  getContent() {
+    const metadataStr = this.getMetadataStr().trim();
+    return `${metadataStr}
+
+${this.text}`.trim();
+  }
+  getMetadataStr() {
+    const usableMetadataKeys = new Set(Object.keys(this.metadata).sort());
+    return [...usableMetadataKeys].map((key) => `${key}: ${this.metadata[key]}`).join(this.metadataSeparator);
+  }
+  getNodeInfo() {
+    return { start: this.startCharIdx, end: this.endCharIdx };
+  }
+  getText() {
+    return this.text;
+  }
+};
+var Document = class extends TextNode {
+  constructor(init) {
+    super(init);
+  }
+  get type() {
+    return "DOCUMENT" /* DOCUMENT */;
+  }
+};
+function lazyInitHash(value, _context) {
+  return {
+    get() {
+      const oldValue = value.get.call(this);
+      if (oldValue === "") {
+        const hash = this.generateHash();
+        value.set.call(this, hash);
+      }
+      return value.get.call(this);
+    },
+    set(newValue) {
+      value.set.call(this, newValue);
+    },
+    init(value2) {
+      return value2;
+    }
+  };
+}
+function createSHA256() {
+  const hash = createHash("sha256");
+  return {
+    update(data) {
+      hash.update(data);
+    },
+    digest() {
+      return hash.digest("base64");
+    }
+  };
+}
+
+// src/document/extractors/base.ts
+var BaseExtractor = class {
+  isTextNodeOnly = true;
+  /**
+   *
+   * @param nodes Nodes to extract metadata from.
+   * @returns Metadata extracted from the nodes.
+   */
+  async processNodes(nodes) {
+    let newNodes = nodes;
+    const curMetadataList = await this.extract(newNodes);
+    for (const idx in newNodes) {
+      newNodes[idx].metadata = {
+        ...newNodes[idx].metadata,
+        ...curMetadataList[idx]
+      };
+    }
+    return newNodes;
+  }
+};
+
 // ../../node_modules/.pnpm/@ai-sdk+provider@1.1.3/node_modules/@ai-sdk/provider/dist/index.mjs
 var marker = "vercel.ai.error";
 var symbol = Symbol.for(marker);
@@ -2428,8 +2739,10 @@ var openaiTextEmbeddingResponseSchema = z.object({
 });
 var modelMaxImagesPerCall = {
   "dall-e-3": 1,
-  "dall-e-2": 10
+  "dall-e-2": 10,
+  "gpt-image-1": 10
 };
+var hasDefaultResponseFormat = /* @__PURE__ */ new Set(["gpt-image-1"]);
 var OpenAIImageModel = class {
   constructor(modelId, settings, config) {
     this.modelId = modelId;
@@ -2479,7 +2792,7 @@ var OpenAIImageModel = class {
         n,
         size,
         ...(_d = providerOptions.openai) != null ? _d : {},
-        response_format: "b64_json"
+        ...!hasDefaultResponseFormat.has(this.modelId) ? { response_format: "b64_json" } : {}
       },
       failedResponseHandler: openaiFailedResponseHandler,
       successfulResponseHandler: createJsonResponseHandler(
@@ -2974,8 +3287,15 @@ var OpenAIResponsesLanguageModel = class {
       user: openaiOptions == null ? void 0 : openaiOptions.user,
       instructions: openaiOptions == null ? void 0 : openaiOptions.instructions,
       // model-specific settings:
-      ...modelConfig.isReasoningModel && (openaiOptions == null ? void 0 : openaiOptions.reasoningEffort) != null && {
-        reasoning: { effort: openaiOptions == null ? void 0 : openaiOptions.reasoningEffort }
+      ...modelConfig.isReasoningModel && ((openaiOptions == null ? void 0 : openaiOptions.reasoningEffort) != null || (openaiOptions == null ? void 0 : openaiOptions.reasoningSummary) != null) && {
+        reasoning: {
+          ...(openaiOptions == null ? void 0 : openaiOptions.reasoningEffort) != null && {
+            effort: openaiOptions.reasoningEffort
+          },
+          ...(openaiOptions == null ? void 0 : openaiOptions.reasoningSummary) != null && {
+            summary: openaiOptions.reasoningSummary
+          }
+        }
       },
       ...modelConfig.requiredAutoTruncation && {
         truncation: "auto"
@@ -3057,7 +3377,7 @@ var OpenAIResponsesLanguageModel = class {
     }
   }
   async doGenerate(options) {
-    var _a15, _b, _c, _d, _e;
+    var _a15, _b, _c, _d, _e, _f, _g;
     const { args: body, warnings } = this.getArgs(options);
     const {
       responseHeaders,
@@ -3110,7 +3430,13 @@ var OpenAIResponsesLanguageModel = class {
                 type: z.literal("computer_call")
               }),
               z.object({
-                type: z.literal("reasoning")
+                type: z.literal("reasoning"),
+                summary: z.array(
+                  z.object({
+                    type: z.literal("summary_text"),
+                    text: z.string()
+                  })
+                )
               })
             ])
           ),
@@ -3128,6 +3454,7 @@ var OpenAIResponsesLanguageModel = class {
       toolName: output.name,
       args: output.arguments
     }));
+    const reasoningSummary = (_b = (_a15 = response.output.find((item) => item.type === "reasoning")) == null ? void 0 : _a15.summary) != null ? _b : null;
     return {
       text: outputTextElements.map((content) => content.text).join("\n"),
       sources: outputTextElements.flatMap(
@@ -3142,10 +3469,14 @@ var OpenAIResponsesLanguageModel = class {
         })
       ),
       finishReason: mapOpenAIResponseFinishReason({
-        finishReason: (_a15 = response.incomplete_details) == null ? void 0 : _a15.reason,
+        finishReason: (_c = response.incomplete_details) == null ? void 0 : _c.reason,
         hasToolCalls: toolCalls.length > 0
       }),
       toolCalls: toolCalls.length > 0 ? toolCalls : void 0,
+      reasoning: reasoningSummary ? reasoningSummary.map((summary) => ({
+        type: "text",
+        text: summary.text
+      })) : void 0,
       usage: {
         promptTokens: response.usage.input_tokens,
         completionTokens: response.usage.output_tokens
@@ -3169,8 +3500,8 @@ var OpenAIResponsesLanguageModel = class {
       providerMetadata: {
         openai: {
           responseId: response.id,
-          cachedPromptTokens: (_c = (_b = response.usage.input_tokens_details) == null ? void 0 : _b.cached_tokens) != null ? _c : null,
-          reasoningTokens: (_e = (_d = response.usage.output_tokens_details) == null ? void 0 : _d.reasoning_tokens) != null ? _e : null
+          cachedPromptTokens: (_e = (_d = response.usage.input_tokens_details) == null ? void 0 : _d.cached_tokens) != null ? _e : null,
+          reasoningTokens: (_g = (_f = response.usage.output_tokens_details) == null ? void 0 : _f.reasoning_tokens) != null ? _g : null
         }
       },
       warnings
@@ -3253,6 +3584,11 @@ var OpenAIResponsesLanguageModel = class {
                 type: "text-delta",
                 textDelta: value.delta
               });
+            } else if (isResponseReasoningSummaryTextDeltaChunk(value)) {
+              controller.enqueue({
+                type: "reasoning",
+                textDelta: value.delta
+              });
             } else if (isResponseOutputItemDoneChunk(value) && value.item.type === "function_call") {
               ongoingToolCalls[value.output_index] = void 0;
               hasToolCalls = true;
@@ -3384,6 +3720,13 @@ var responseAnnotationAddedSchema = z.object({
     title: z.string()
   })
 });
+var responseReasoningSummaryTextDeltaSchema = z.object({
+  type: z.literal("response.reasoning_summary_text.delta"),
+  item_id: z.string(),
+  output_index: z.number(),
+  summary_index: z.number(),
+  delta: z.string()
+});
 var openaiResponsesChunkSchema = z.union([
   textDeltaChunkSchema,
   responseFinishedChunkSchema,
@@ -3392,6 +3735,7 @@ var openaiResponsesChunkSchema = z.union([
   responseFunctionCallArgumentsDeltaSchema,
   responseOutputItemAddedSchema,
   responseAnnotationAddedSchema,
+  responseReasoningSummaryTextDeltaSchema,
   z.object({ type: z.string() }).passthrough()
   // fallback for unknown chunks
 ]);
@@ -3416,6 +3760,9 @@ function isResponseOutputItemAddedChunk(chunk) {
 function isResponseAnnotationAddedChunk(chunk) {
   return chunk.type === "response.output_text.annotation.added";
 }
+function isResponseReasoningSummaryTextDeltaChunk(chunk) {
+  return chunk.type === "response.reasoning_summary_text.delta";
+}
 function getResponsesModelConfig(modelId) {
   if (modelId.startsWith("o")) {
     if (modelId.startsWith("o1-mini") || modelId.startsWith("o1-preview")) {
@@ -3445,7 +3792,8 @@ var openaiResponsesProviderOptionsSchema = z.object({
   user: z.string().nullish(),
   reasoningEffort: z.string().nullish(),
   strictSchemas: z.boolean().nullish(),
-  instructions: z.string().nullish()
+  instructions: z.string().nullish(),
+  reasoningSummary: z.string().nullish()
 });
 var WebSearchPreviewParameters = z.object({});
 function webSearchPreviewTool({
@@ -3666,40 +4014,11 @@ var baseLLM = openai2("gpt-4o");
 
 // src/document/extractors/title.ts
 var TitleExtractor = class extends BaseExtractor {
-  /**
-   * MastraLanguageModel instance.
-   * @type {MastraLanguageModel}
-   */
   llm;
-  /**
-   * Can work for mixture of text and non-text nodes
-   * @type {boolean}
-   * @default false
-   */
   isTextNodeOnly = false;
-  /**
-   * Number of nodes to extrct titles from.
-   * @type {number}
-   * @default 5
-   */
   nodes = 5;
-  /**
-   * The prompt template to use for the title extractor.
-   * @type {string}
-   */
   nodeTemplate;
-  /**
-   * The prompt template to merge title with..
-   * @type {string}
-   */
   combineTemplate;
-  /**
-   * Constructor for the TitleExtractor class.
-   * @param {MastraLanguageModel} llm MastraLanguageModel instance.
-   * @param {number} nodes Number of nodes to extract titles from.
-   * @param {TitleExtractorPrompt} nodeTemplate The prompt template to use for the title extractor.
-   * @param {string} combineTemplate The prompt template to merge title with..
-   */
   constructor(options) {
     super();
     this.llm = options?.llm ?? baseLLM;
@@ -3723,7 +4042,7 @@ var TitleExtractor = class extends BaseExtractor {
     const nodesToExtractTitle = [];
     const nodeIndexes = [];
     nodes.forEach((node, idx) => {
-      const text = node.getContent(this.metadataMode);
+      const text = node.getContent();
       if (!text || text.trim() === "") {
         results[idx] = { documentTitle: "" };
       } else {
@@ -3810,7 +4129,7 @@ var TitleExtractor = class extends BaseExtractor {
               {
                 type: "text",
                 text: this.nodeTemplate.format({
-                  context: node.getContent(MetadataMode.ALL)
+                  context: node.getContent()
                 })
               }
             ]
@@ -3827,21 +4146,11 @@ var TitleExtractor = class extends BaseExtractor {
     return await Promise.all(titleJobs);
   }
 };
+
+// src/document/extractors/summary.ts
 var SummaryExtractor = class extends BaseExtractor {
-  /**
-   * MastraLanguageModel instance.
-   * @type {MastraLanguageModel}
-   */
   llm;
-  /**
-   * List of summaries to extract: 'self', 'prev', 'next'
-   * @type {string[]}
-   */
   summaries;
-  /**
-   * The prompt template to use for the summary extractor.
-   * @type {string}
-   */
   promptTemplate;
   selfSummary;
   prevSummary;
@@ -3867,14 +4176,14 @@ var SummaryExtractor = class extends BaseExtractor {
    * @returns {Promise<string>} Summary extracted from the node.
    */
   async generateNodeSummary(node) {
-    const text = node.getContent(this.metadataMode);
+    const text = node.getContent();
     if (!text || text.trim() === "") {
       return "";
     }
     if (this.isTextNodeOnly && !(node instanceof TextNode)) {
       return "";
     }
-    const context = node.getContent(this.metadataMode);
+    const context = node.getContent();
     const prompt = this.promptTemplate.format({
       context
     });
@@ -3919,28 +4228,12 @@ var SummaryExtractor = class extends BaseExtractor {
     return metadataList;
   }
 };
+
+// src/document/extractors/questions.ts
 var QuestionsAnsweredExtractor = class extends BaseExtractor {
-  /**
-   * MastraLanguageModel instance.
-   * @type {MastraLanguageModel}
-   */
   llm;
-  /**
-   * Number of questions to generate.
-   * @type {number}
-   * @default 5
-   */
   questions = 5;
-  /**
-   * The prompt template to use for the question extractor.
-   * @type {string}
-   */
   promptTemplate;
-  /**
-   * Wheter to use metadata for embeddings only
-   * @type {boolean}
-   * @default false
-   */
   embeddingOnly = false;
   /**
    * Constructor for the QuestionsAnsweredExtractor class.
@@ -3968,14 +4261,14 @@ var QuestionsAnsweredExtractor = class extends BaseExtractor {
    * @returns {Promise<Array<ExtractQuestion> | Array<{}>>} Questions extracted from the node.
    */
   async extractQuestionsFromNode(node) {
-    const text = node.getContent(this.metadataMode);
+    const text = node.getContent();
     if (!text || text.trim() === "") {
       return { questionsThisExcerptCanAnswer: "" };
     }
     if (this.isTextNodeOnly && !(node instanceof TextNode)) {
       return { questionsThisExcerptCanAnswer: "" };
     }
-    const contextStr = node.getContent(this.metadataMode);
+    const contextStr = node.getContent();
     const prompt = this.promptTemplate.format({
       context: contextStr,
       numQuestions: this.questions.toString()
@@ -4014,22 +4307,11 @@ var QuestionsAnsweredExtractor = class extends BaseExtractor {
     return results;
   }
 };
+
+// src/document/extractors/keywords.ts
 var KeywordExtractor = class extends BaseExtractor {
-  /**
-   * MastraLanguageModel instance.
-   * @type {MastraLanguageModel}
-   */
   llm;
-  /**
-   * Number of keywords to extract.
-   * @type {number}
-   * @default 5
-   */
   keywords = 5;
-  /**
-   * The prompt template to use for the question extractor.
-   * @type {string}
-   */
   promptTemplate;
   /**
    * Constructor for the KeywordExtractor class.
@@ -4058,7 +4340,7 @@ var KeywordExtractor = class extends BaseExtractor {
    * Adds error handling for malformed/empty LLM output.
    */
   async extractKeywordsFromNodes(node) {
-    const text = node.getContent(this.metadataMode);
+    const text = node.getContent();
     if (!text || text.trim() === "") {
       return { excerptKeywords: "" };
     }
@@ -4077,7 +4359,7 @@ var KeywordExtractor = class extends BaseExtractor {
               {
                 type: "text",
                 text: this.promptTemplate.format({
-                  context: node.getContent(MetadataMode.ALL),
+                  context: node.getContent(),
                   maxKeywords: this.keywords.toString()
                 })
               }
@@ -4142,6 +4424,8 @@ var Language = /* @__PURE__ */ ((Language2) => {
   Language2["POWERSHELL"] = "powershell";
   return Language2;
 })(Language || {});
+
+// src/document/transformers/text.ts
 var TextTransformer = class {
   size;
   overlap;
@@ -4720,6 +5004,8 @@ var HTMLSectionTransformer = class {
     return this.createDocuments(texts, metadatas);
   }
 };
+
+// src/document/transformers/json.ts
 var RecursiveJsonTransformer = class _RecursiveJsonTransformer {
   maxSize;
   minSize;
@@ -5127,6 +5413,8 @@ var LatexTransformer = class extends RecursiveCharacterTransformer {
     super({ separators, isSeparatorRegex: true, options });
   }
 };
+
+// src/document/transformers/markdown.ts
 var MarkdownTransformer = class extends RecursiveCharacterTransformer {
   constructor(options = {}) {
     const separators = RecursiveCharacterTransformer.getSeparatorsForLanguage("markdown" /* MARKDOWN */);
@@ -5411,21 +5699,19 @@ var MDocument = class _MDocument {
         (doc) => doc?.metadata?.docId ? new Document({
           ...doc,
           relationships: {
-            [NodeRelationship.SOURCE]: {
+            ["SOURCE" /* SOURCE */]: {
               nodeId: doc.metadata.docId,
-              nodeType: ObjectType.DOCUMENT,
+              nodeType: "DOCUMENT" /* DOCUMENT */,
               metadata: doc.metadata
             }
           }
         }) : doc
       );
     }
-    const pipeline = new IngestionPipeline({
-      transformations
-    });
-    const nodes = await pipeline.run({
-      documents: this.chunks
-    });
+    let nodes = this.chunks;
+    for (const extractor of transformations) {
+      nodes = await extractor.processNodes(nodes);
+    }
     this.chunks = this.chunks.map((doc, i) => {
       return new Document({
         text: doc.text,