@mastra/rag 2.6.4 → 2.6.5-alpha.1

package/dist/index.js

@@ -4547,6 +4547,61 @@ var RecursiveCharacterTransformer = class RecursiveCharacterTransformer extends
 };
 //#endregion
 //#region src/document/transformers/html.ts
+const BLOCK_TAGS = /* @__PURE__ */ new Set([
+	"address",
+	"article",
+	"aside",
+	"blockquote",
+	"br",
+	"caption",
+	"center",
+	"dd",
+	"details",
+	"dialog",
+	"div",
+	"dl",
+	"dt",
+	"fieldset",
+	"figcaption",
+	"figure",
+	"footer",
+	"form",
+	"h1",
+	"h2",
+	"h3",
+	"h4",
+	"h5",
+	"h6",
+	"header",
+	"hgroup",
+	"hr",
+	"legend",
+	"li",
+	"main",
+	"menu",
+	"nav",
+	"ol",
+	"p",
+	"pre",
+	"search",
+	"section",
+	"summary",
+	"table",
+	"tbody",
+	"td",
+	"tfoot",
+	"th",
+	"thead",
+	"tr",
+	"ul"
+]);
+function extractText(node) {
+	if (!node) return "";
+	if (!node.tagName) return node.text || "";
+	let content = "";
+	for (const child of node.childNodes ?? []) content += extractText(child);
+	return BLOCK_TAGS.has(node.tagName.toLowerCase()) ? " " + content + " " : content;
+}
 var HTMLHeaderTransformer = class {
 	headersToSplitOn;
 	returnEachElement;
@@ -4605,14 +4660,7 @@ var HTMLHeaderTransformer = class {
 		return "/" + parts.join("/");
 	}
 	getTextContent(element) {
-		if (!element) return "";
-		if (!element.tagName) return element.text || "";
-		let content = element.text || "";
-		if (element.childNodes) for (const child of element.childNodes) {
-			const childText = this.getTextContent(child);
-			if (childText) content += " " + childText;
-		}
-		return content.trim();
+		return extractText(element).replace(/\s+/g, " ").trim();
 	}
 	aggregateElementsToChunks(elements) {
 		const aggregatedChunks = [];
@@ -4690,14 +4738,7 @@ var HTMLSectionTransformer = class {
 		return "/" + parts.join("/");
 	}
 	getTextContent(element) {
-		if (!element) return "";
-		if (!element.tagName) return element.text || "";
-		let content = element.text || "";
-		if (element.childNodes) for (const child of element.childNodes) {
-			const childText = this.getTextContent(child);
-			if (childText) content += " " + childText;
-		}
-		return content.trim();
+		return extractText(element).replace(/\s+/g, " ").trim();
 	}
 	splitHtmlByHeaders(htmlDoc) {
 		const sections = [];
@@ -5501,35 +5542,44 @@ var SentenceTransformer = class extends TextTransformer {
 		const chunks = [];
 		let currentChunk = [];
 		let currentSize = 0;
+		let carriedCount = 0;
 		const separator = " ";
+		const sepLength = this.lengthFunction(separator);
 		for (const sentence of sentences) {
 			const sentenceLength = this.lengthFunction(sentence);
-			const separatorLength = currentChunk.length > 0 ? this.lengthFunction(separator) : 0;
-			const totalLength = currentSize + sentenceLength + separatorLength;
 			if (sentenceLength > this.maxSize) {
-				if (currentChunk.length > 0) {
-					chunks.push(currentChunk.join(separator));
-					currentChunk = [];
-					currentSize = 0;
-				}
+				if (currentChunk.length > carriedCount) chunks.push(currentChunk.join(separator));
+				currentChunk = [];
+				currentSize = 0;
+				carriedCount = 0;
 				const fallbackChunks = this.handleOversizedSentence(sentence);
 				chunks.push(...fallbackChunks);
 				continue;
 			}
-			if (currentChunk.length > 0 && totalLength > this.maxSize) {
-				chunks.push(currentChunk.join(separator));
-				currentChunk = this.calculateSentenceOverlap(currentChunk);
-				currentSize = this.calculateChunkSize(currentChunk);
+			if (currentChunk.length > 0 && currentSize + sepLength + sentenceLength > this.maxSize) {
+				if (currentChunk.length > carriedCount) chunks.push(currentChunk.join(separator));
+				const overlapSentences = this.calculateSentenceOverlap(currentChunk);
+				let overlapSize = this.calculateChunkSize(overlapSentences);
+				let start = 0;
+				while (start < overlapSentences.length && overlapSize + sepLength + sentenceLength > this.maxSize) {
+					overlapSize -= this.lengthFunction(overlapSentences[start]);
+					if (start < overlapSentences.length - 1) overlapSize -= sepLength;
+					start++;
+				}
+				currentChunk = overlapSentences.slice(start);
+				carriedCount = currentChunk.length;
+				currentSize = currentChunk.length > 0 ? overlapSize : 0;
 			}
+			currentSize += sentenceLength + (currentChunk.length > 0 ? sepLength : 0);
 			currentChunk.push(sentence);
-			currentSize += sentenceLength + separatorLength;
 			if (currentSize >= this.targetSize) {
 				chunks.push(currentChunk.join(separator));
 				currentChunk = this.calculateSentenceOverlap(currentChunk);
+				carriedCount = currentChunk.length;
 				currentSize = this.calculateChunkSize(currentChunk);
 			}
 		}
-		if (currentChunk.length > 0) chunks.push(currentChunk.join(separator));
+		if (currentChunk.length > carriedCount) chunks.push(currentChunk.join(separator));
 		return chunks;
 	}
 	/**