@mastra/rag 2.6.4 → 2.6.5-alpha.1

package/dist/docs/SKILL.md

@@ -3,7 +3,7 @@ name: mastra-rag
 description: Documentation for @mastra/rag. Use when working with @mastra/rag APIs, configuration, or implementation.
 metadata:
   package: "@mastra/rag"
-  version: "2.6.4"
+  version: "2.6.5-alpha.1"
 ---
 
 ## When to use

package/dist/docs/assets/SOURCE_MAP.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.6.4",
+  "version": "2.6.5-alpha.1",
   "package": "@mastra/rag",
   "exports": {},
   "modules": {}

package/dist/document/transformers/html.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"html.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/html.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAWjD,qBAAa,qBAAqB;IAChC,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,iBAAiB,CAAU;IAEnC,YAAY,OAAO,EAAE,gBAAgB,GAAG;QAAE,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAA;KAAE,EAGtE;IAED,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,QAAQ,EAAE,CAsDhD;IAED,OAAO,CAAC,QAAQ;IA2BhB,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,yBAAyB;IAyBjC,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE,CA4B9E;IAED,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,CAUpD;CACF;AAED,qBAAa,sBAAsB;IACjC,OAAO,CAAC,gBAAgB,CAAyB;IACjD,OAAO,CAAC,YAAY,CAAgC;IAEpD,YAAY,OAAO,EAAE,gBAAgB,GAAG;QAAE,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAA;KAAE,EAGvE;IAED,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,EAAE,CAalC;IAED,OAAO,CAAC,QAAQ;IAwBhB,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,kBAAkB;IA6DpB,cAAc,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAW/D;IAED,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE,CA6B9E;IAED,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,CAUpD;CACF"}
+{"version":3,"file":"html.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/html.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAwEjD,qBAAa,qBAAqB;IAChC,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,iBAAiB,CAAU;IAEnC,YAAY,OAAO,EAAE,gBAAgB,GAAG;QAAE,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAA;KAAE,EAGtE;IAED,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,QAAQ,EAAE,CAsDhD;IAED,OAAO,CAAC,QAAQ;IA2BhB,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,yBAAyB;IAyBjC,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE,CA4B9E;IAED,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,CAUpD;CACF;AAED,qBAAa,sBAAsB;IACjC,OAAO,CAAC,gBAAgB,CAAyB;IACjD,OAAO,CAAC,YAAY,CAAgC;IAEpD,YAAY,OAAO,EAAE,gBAAgB,GAAG;QAAE,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAA;KAAE,EAGvE;IAED,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,EAAE,CAalC;IAED,OAAO,CAAC,QAAQ;IAwBhB,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,kBAAkB;IA6DpB,cAAc,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAW/D;IAED,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE,CA6B9E;IAED,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,CAUpD;CACF"}

package/dist/document/transformers/sentence.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sentence.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/sentence.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC;AAEzC,qBAAa,mBAAoB,SAAQ,eAAe;IACtD,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC;IAC7B,SAAS,CAAC,cAAc,EAAE,MAAM,EAAE,CAAC;IACnC,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,oBAAoB,EAAE,OAAO,CAAC;IAExC,YAAY,OAAO,EAAE,oBAAoB,EAoBxC;IAED,OAAO,CAAC,wBAAwB;IA+BhC,OAAO,CAAC,sBAAsB;IAqB9B,OAAO,CAAC,oBAAoB;IA8B5B;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAsDhC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAqB/B,OAAO,CAAC,sBAAsB;IAmC9B,OAAO,CAAC,2BAA2B;IAsBnC,OAAO,CAAC,wBAAwB;IA4BhC,OAAO,CAAC,kBAAkB;IAqB1B,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE,CAQ9C;CACF"}
+{"version":3,"file":"sentence.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/sentence.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC;AAEzC,qBAAa,mBAAoB,SAAQ,eAAe;IACtD,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC;IAC7B,SAAS,CAAC,cAAc,EAAE,MAAM,EAAE,CAAC;IACnC,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,oBAAoB,EAAE,OAAO,CAAC;IAExC,YAAY,OAAO,EAAE,oBAAoB,EAoBxC;IAED,OAAO,CAAC,wBAAwB;IA+BhC,OAAO,CAAC,sBAAsB;IAqB9B,OAAO,CAAC,oBAAoB;IA8B5B;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAmEhC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAqB/B,OAAO,CAAC,sBAAsB;IAmC9B,OAAO,CAAC,2BAA2B;IAsBnC,OAAO,CAAC,wBAAwB;IA4BhC,OAAO,CAAC,kBAAkB;IAqB1B,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE,CAQ9C;CACF"}

package/dist/index.cjs

@@ -4549,6 +4549,61 @@ var RecursiveCharacterTransformer = class RecursiveCharacterTransformer extends
 };
 //#endregion
 //#region src/document/transformers/html.ts
+const BLOCK_TAGS = /* @__PURE__ */ new Set([
+	"address",
+	"article",
+	"aside",
+	"blockquote",
+	"br",
+	"caption",
+	"center",
+	"dd",
+	"details",
+	"dialog",
+	"div",
+	"dl",
+	"dt",
+	"fieldset",
+	"figcaption",
+	"figure",
+	"footer",
+	"form",
+	"h1",
+	"h2",
+	"h3",
+	"h4",
+	"h5",
+	"h6",
+	"header",
+	"hgroup",
+	"hr",
+	"legend",
+	"li",
+	"main",
+	"menu",
+	"nav",
+	"ol",
+	"p",
+	"pre",
+	"search",
+	"section",
+	"summary",
+	"table",
+	"tbody",
+	"td",
+	"tfoot",
+	"th",
+	"thead",
+	"tr",
+	"ul"
+]);
+function extractText(node) {
+	if (!node) return "";
+	if (!node.tagName) return node.text || "";
+	let content = "";
+	for (const child of node.childNodes ?? []) content += extractText(child);
+	return BLOCK_TAGS.has(node.tagName.toLowerCase()) ? " " + content + " " : content;
+}
 var HTMLHeaderTransformer = class {
 	headersToSplitOn;
 	returnEachElement;
@@ -4607,14 +4662,7 @@ var HTMLHeaderTransformer = class {
 		return "/" + parts.join("/");
 	}
 	getTextContent(element) {
-		if (!element) return "";
-		if (!element.tagName) return element.text || "";
-		let content = element.text || "";
-		if (element.childNodes) for (const child of element.childNodes) {
-			const childText = this.getTextContent(child);
-			if (childText) content += " " + childText;
-		}
-		return content.trim();
+		return extractText(element).replace(/\s+/g, " ").trim();
 	}
 	aggregateElementsToChunks(elements) {
 		const aggregatedChunks = [];
@@ -4692,14 +4740,7 @@ var HTMLSectionTransformer = class {
 		return "/" + parts.join("/");
 	}
 	getTextContent(element) {
-		if (!element) return "";
-		if (!element.tagName) return element.text || "";
-		let content = element.text || "";
-		if (element.childNodes) for (const child of element.childNodes) {
-			const childText = this.getTextContent(child);
-			if (childText) content += " " + childText;
-		}
-		return content.trim();
+		return extractText(element).replace(/\s+/g, " ").trim();
 	}
 	splitHtmlByHeaders(htmlDoc) {
 		const sections = [];
@@ -5503,35 +5544,44 @@ var SentenceTransformer = class extends TextTransformer {
 		const chunks = [];
 		let currentChunk = [];
 		let currentSize = 0;
+		let carriedCount = 0;
 		const separator = " ";
+		const sepLength = this.lengthFunction(separator);
 		for (const sentence of sentences) {
 			const sentenceLength = this.lengthFunction(sentence);
-			const separatorLength = currentChunk.length > 0 ? this.lengthFunction(separator) : 0;
-			const totalLength = currentSize + sentenceLength + separatorLength;
 			if (sentenceLength > this.maxSize) {
-				if (currentChunk.length > 0) {
-					chunks.push(currentChunk.join(separator));
-					currentChunk = [];
-					currentSize = 0;
-				}
+				if (currentChunk.length > carriedCount) chunks.push(currentChunk.join(separator));
+				currentChunk = [];
+				currentSize = 0;
+				carriedCount = 0;
 				const fallbackChunks = this.handleOversizedSentence(sentence);
 				chunks.push(...fallbackChunks);
 				continue;
 			}
-			if (currentChunk.length > 0 && totalLength > this.maxSize) {
-				chunks.push(currentChunk.join(separator));
-				currentChunk = this.calculateSentenceOverlap(currentChunk);
-				currentSize = this.calculateChunkSize(currentChunk);
+			if (currentChunk.length > 0 && currentSize + sepLength + sentenceLength > this.maxSize) {
+				if (currentChunk.length > carriedCount) chunks.push(currentChunk.join(separator));
+				const overlapSentences = this.calculateSentenceOverlap(currentChunk);
+				let overlapSize = this.calculateChunkSize(overlapSentences);
+				let start = 0;
+				while (start < overlapSentences.length && overlapSize + sepLength + sentenceLength > this.maxSize) {
+					overlapSize -= this.lengthFunction(overlapSentences[start]);
+					if (start < overlapSentences.length - 1) overlapSize -= sepLength;
+					start++;
+				}
+				currentChunk = overlapSentences.slice(start);
+				carriedCount = currentChunk.length;
+				currentSize = currentChunk.length > 0 ? overlapSize : 0;
 			}
+			currentSize += sentenceLength + (currentChunk.length > 0 ? sepLength : 0);
 			currentChunk.push(sentence);
-			currentSize += sentenceLength + separatorLength;
 			if (currentSize >= this.targetSize) {
 				chunks.push(currentChunk.join(separator));
 				currentChunk = this.calculateSentenceOverlap(currentChunk);
+				carriedCount = currentChunk.length;
 				currentSize = this.calculateChunkSize(currentChunk);
 			}
 		}
-		if (currentChunk.length > 0) chunks.push(currentChunk.join(separator));
+		if (currentChunk.length > carriedCount) chunks.push(currentChunk.join(separator));
 		return chunks;
 	}
 	/**