@mastra/rag 2.1.1 → 2.1.2

package/dist/document/transformers/token.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"token.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/token.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,gBAAgB,EAAY,MAAM,aAAa,CAAC;AAE7E,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAElD,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC;AAEzC,UAAU,SAAS;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,MAAM,CAAC;IACrC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;CACpC;AAED,wBAAgB,iBAAiB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,SAAS,CAAA;CAAE,GAAG,MAAM,EAAE,CAkBvG;AAED,qBAAa,gBAAiB,SAAQ,eAAe;IACnD,OAAO,CAAC,SAAS,CAAW;IAC5B,OAAO,CAAC,cAAc,CAAsB;IAC5C,OAAO,CAAC,iBAAiB,CAAsB;gBAEnC,EACV,YAA4B,EAC5B,SAAS,EACT,cAA0B,EAC1B,iBAAyB,EACzB,OAAY,GACb,EAAE;QACD,YAAY,CAAC,EAAE,gBAAgB,CAAC;QAChC,SAAS,CAAC,EAAE,aAAa,CAAC;QAC1B,cAAc,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;QACrC,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;QACxC,OAAO,EAAE,iBAAiB,CAAC;KAC5B;IAaD,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE;IA0B/C,MAAM,CAAC,YAAY,CAAC,EAClB,YAA4B,EAC5B,SAAS,EACT,OAAY,GACb,EAAE;QACD,YAAY,CAAC,EAAE,gBAAgB,CAAC;QAChC,SAAS,CAAC,EAAE,aAAa,CAAC;QAC1B,OAAO,CAAC,EAAE,iBAAiB,CAAC;KAC7B,GAAG,gBAAgB;CAuCrB"}
+{"version":3,"file":"token.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/token.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE7E,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAElD,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC;AAEzC,UAAU,SAAS;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,MAAM,CAAC;IACrC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;CACpC;AAED,wBAAgB,iBAAiB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,SAAS,CAAA;CAAE,GAAG,MAAM,EAAE,CAkBvG;AAED,qBAAa,gBAAiB,SAAQ,eAAe;IACnD,OAAO,CAAC,SAAS,CAAW;IAC5B,OAAO,CAAC,YAAY,CAAmB;IACvC,OAAO,CAAC,eAAe,CAAmB;gBAE9B,EACV,YAA4B,EAC5B,SAAS,EACT,SAAS,EAAE,iBAAiB,EAC5B,cAA0B,EAC1B,iBAAyB,EACzB,OAAY,GACb,EAAE;QACD,YAAY,CAAC,EAAE,gBAAgB,CAAC;QAChC,SAAS,CAAC,EAAE,aAAa,CAAC;QAC1B,SAAS,CAAC,EAAE,QAAQ,CAAC;QACrB,cAAc,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;QACrC,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;QACxC,OAAO,EAAE,iBAAiB,CAAC;KAC5B;IAiBD,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE;IAqB/C,MAAM,CAAC,YAAY,CAAC,EAClB,YAA4B,EAC5B,SAAS,EACT,OAAY,GACb,EAAE;QACD,YAAY,CAAC,EAAE,gBAAgB,CAAC;QAChC,SAAS,CAAC,EAAE,aAAa,CAAC;QAC1B,OAAO,CAAC,EAAE,iBAAiB,CAAC;KAC7B,GAAG,gBAAgB;CAwCrB"}

package/dist/index.cjs

@@ -6072,31 +6072,34 @@ var MarkdownHeaderTransformer = class {
 var SemanticMarkdownTransformer = class _SemanticMarkdownTransformer extends TextTransformer {
   tokenizer;
   joinThreshold;
-  allowedSpecial;
-  disallowedSpecial;
+  allowedArray;
+  disallowedArray;
   constructor({
     joinThreshold = 500,
     encodingName = "cl100k_base",
     modelName,
+    tokenizer: existingTokenizer,
     allowedSpecial = /* @__PURE__ */ new Set(),
     disallowedSpecial = "all",
     ...baseOptions
   } = {}) {
     super(baseOptions);
     this.joinThreshold = joinThreshold;
-    this.allowedSpecial = allowedSpecial;
-    this.disallowedSpecial = disallowedSpecial;
-    try {
-      this.tokenizer = modelName ? jsTiktoken.encodingForModel(modelName) : jsTiktoken.getEncoding(encodingName);
-    } catch {
-      throw new Error("Could not load tiktoken encoding. Please install it with `npm install js-tiktoken`.");
+    this.allowedArray = allowedSpecial === "all" ? "all" : Array.from(allowedSpecial);
+    this.disallowedArray = disallowedSpecial === "all" ? "all" : Array.from(disallowedSpecial);
+    if (existingTokenizer) {
+      this.tokenizer = existingTokenizer;
+    } else {
+      try {
+        this.tokenizer = modelName ? jsTiktoken.encodingForModel(modelName) : jsTiktoken.getEncoding(encodingName);
+      } catch {
+        throw new Error("Could not load tiktoken encoding. Please install it with `npm install js-tiktoken`.");
+      }
     }
   }
   countTokens(text) {
-    const allowed = this.allowedSpecial === "all" ? "all" : Array.from(this.allowedSpecial);
-    const disallowed = this.disallowedSpecial === "all" ? "all" : Array.from(this.disallowedSpecial);
     const processedText = this.stripWhitespace ? text.trim() : text;
-    return this.tokenizer.encode(processedText, allowed, disallowed).length;
+    return this.tokenizer.encode(processedText, this.allowedArray, this.disallowedArray).length;
   }
   splitMarkdownByHeaders(markdown) {
     const sections = [];
@@ -6150,14 +6153,21 @@ var SemanticMarkdownTransformer = class _SemanticMarkdownTransformer extends Tex
         const current = workingSections[j];
         if (current.depth === depth) {
           const prev = workingSections[j - 1];
-          if (prev.length + current.length < this.joinThreshold && prev.depth <= current.depth) {
-            const title = `${"#".repeat(current.depth)} ${current.title}`;
-            const formattedTitle = `
+          const title = `${"#".repeat(current.depth)} ${current.title}`;
+          const formattedTitle = `
 
 ${title}`;
+          const headerLength = this.tokenizer.encode(
+            `${formattedTitle}
+`,
+            this.allowedArray,
+            this.disallowedArray
+          ).length;
+          const mergedLength = prev.length + current.length + headerLength;
+          if (mergedLength < this.joinThreshold && prev.depth <= current.depth) {
             prev.content += `${formattedTitle}
 ${current.content}`;
-            prev.length = this.countTokens(prev.content);
+            prev.length = mergedLength;
             workingSections.splice(j, 1);
             j--;
           }
@@ -6227,6 +6237,7 @@ ${section.content}`;
       ...options,
       encodingName,
       modelName,
+      tokenizer,
       lengthFunction: tikTokenCounter
     });
   }
@@ -6481,30 +6492,33 @@ function splitTextOnTokens({ text, tokenizer }) {
 }
 var TokenTransformer = class _TokenTransformer extends TextTransformer {
   tokenizer;
-  allowedSpecial;
-  disallowedSpecial;
+  allowedArray;
+  disallowedArray;
   constructor({
     encodingName = "cl100k_base",
     modelName,
+    tokenizer: existingTokenizer,
     allowedSpecial = /* @__PURE__ */ new Set(),
     disallowedSpecial = "all",
     options = {}
   }) {
     super(options);
-    try {
-      this.tokenizer = modelName ? jsTiktoken.encodingForModel(modelName) : jsTiktoken.getEncoding(encodingName);
-    } catch {
-      throw new Error("Could not load tiktoken encoding. Please install it with `npm install js-tiktoken`.");
+    if (existingTokenizer) {
+      this.tokenizer = existingTokenizer;
+    } else {
+      try {
+        this.tokenizer = modelName ? jsTiktoken.encodingForModel(modelName) : jsTiktoken.getEncoding(encodingName);
+      } catch {
+        throw new Error("Could not load tiktoken encoding. Please install it with `npm install js-tiktoken`.");
+      }
     }
-    this.allowedSpecial = allowedSpecial;
-    this.disallowedSpecial = disallowedSpecial;
+    this.allowedArray = allowedSpecial === "all" ? "all" : Array.from(allowedSpecial);
+    this.disallowedArray = disallowedSpecial === "all" ? "all" : Array.from(disallowedSpecial);
   }
   splitText({ text }) {
     const encode = (text2) => {
-      const allowed = this.allowedSpecial === "all" ? "all" : Array.from(this.allowedSpecial);
-      const disallowed = this.disallowedSpecial === "all" ? "all" : Array.from(this.disallowedSpecial);
       const processedText = this.stripWhitespace ? text2.trim() : text2;
-      return Array.from(this.tokenizer.encode(processedText, allowed, disallowed));
+      return Array.from(this.tokenizer.encode(processedText, this.allowedArray, this.disallowedArray));
     };
     const decode = (tokens) => {
       const text2 = this.tokenizer.decode(tokens);
@@ -6541,6 +6555,7 @@ var TokenTransformer = class _TokenTransformer extends TextTransformer {
     return new _TokenTransformer({
       encodingName,
       modelName,
+      tokenizer,
       allowedSpecial: options.allowedSpecial,
       disallowedSpecial: options.disallowedSpecial,
       options: {