@mastra/rag 2.6.5-alpha.0 → 2.6.5-alpha.2

package/dist/index.js

@@ -3958,46 +3958,66 @@ var TextTransformer = class {
 		if (this.stripWhitespace) text = text.trim();
 		return text === "" ? null : text;
 	}
-	mergeSplits(splits, separator) {
+	/**
+	* Merges splits into chunks of at most `maxSize`.
+	* `joiners[i]`, when given, is the text placed before `splits[i]` when it is
+	* joined to a preceding piece; otherwise `separator` is used for every piece.
+	*/
+	mergeSplits(splits, separator, joiners) {
 		const docs = [];
 		let currentDoc = [];
+		let currentJoiners = [];
 		let total = 0;
-		for (const d of splits) {
+		const lengthOf = (s) => s ? this.lengthFunction(s) : 0;
+		const join = (pieces, pieceJoiners) => {
+			if (!joiners) return this.joinDocs(pieces, separator);
+			const text = pieces.map((piece, i) => i === 0 ? piece : pieceJoiners[i] + piece).join("");
+			return this.joinDocs([text], "");
+		};
+		splits.forEach((d, index) => {
 			const len = this.lengthFunction(d);
-			const separatorLen = separator ? this.lengthFunction(separator) : 0;
+			const joiner = joiners ? joiners[index] ?? separator : separator;
+			const separatorLen = lengthOf(joiner);
 			if (total + len + (currentDoc.length > 0 ? separatorLen : 0) > this.maxSize) {
 				if (total > this.maxSize) console.warn(`Created a chunk of size ${total}, which is longer than the specified ${this.maxSize}`);
 				if (currentDoc.length > 0) {
-					const doc = this.joinDocs(currentDoc, separator);
+					const doc = join(currentDoc, currentJoiners);
 					if (doc !== null) docs.push(doc);
 					if (this.overlap > 0) {
-						let overlapContent = [];
+						const overlapContent = [];
+						const overlapJoiners = [];
 						let overlapSize = 0;
 						for (let i = currentDoc.length - 1; i >= 0; i--) {
 							const piece = currentDoc[i];
 							const pieceLen = this.lengthFunction(piece);
-							if (overlapSize + pieceLen > this.overlap) break;
+							const connectorLen = overlapContent.length > 0 ? lengthOf(currentJoiners[i + 1]) : 0;
+							if (overlapSize + pieceLen + connectorLen > this.overlap) break;
 							overlapContent.unshift(piece);
-							overlapSize += pieceLen + (overlapContent.length > 1 ? separatorLen : 0);
+							overlapJoiners.unshift(currentJoiners[i]);
+							overlapSize += pieceLen + connectorLen;
 						}
 						while (overlapContent.length > 0 && overlapSize + len + separatorLen > this.maxSize) {
 							const removed = overlapContent.shift();
+							overlapJoiners.shift();
 							const removedLen = this.lengthFunction(removed);
-							overlapSize -= removedLen + (overlapContent.length > 0 ? separatorLen : 0);
+							overlapSize -= removedLen + (overlapContent.length > 0 ? lengthOf(overlapJoiners[0]) : 0);
 						}
 						currentDoc = overlapContent;
+						currentJoiners = overlapJoiners;
 						total = overlapSize;
 					} else {
 						currentDoc = [];
+						currentJoiners = [];
 						total = 0;
 					}
 				}
 			}
 			currentDoc.push(d);
+			currentJoiners.push(joiner);
 			total += len + (currentDoc.length > 1 ? separatorLen : 0);
-		}
+		});
 		if (currentDoc.length > 0) {
-			const doc = this.joinDocs(currentDoc, separator);
+			const doc = join(currentDoc, currentJoiners);
 			if (doc !== null) docs.push(doc);
 		}
 		return docs;
@@ -4027,6 +4047,30 @@ function splitTextWithRegex(text, separator, separatorPosition) {
 	}
 	return result.filter((s) => s !== "");
 }
+function splitTextWithMatchedSeparators(text, separator) {
+	const splits = [];
+	const joiners = [];
+	let pending = "";
+	let lastIndex = 0;
+	const push = (piece) => {
+		if (piece !== "") {
+			splits.push(piece);
+			joiners.push(pending);
+			pending = "";
+		}
+	};
+	for (const match of text.matchAll(new RegExp(separator, "g"))) {
+		if (match[0] === "" && (match.index === 0 || match.index === text.length)) continue;
+		push(text.slice(lastIndex, match.index));
+		pending += match[0];
+		lastIndex = match.index + match[0].length;
+	}
+	push(text.slice(lastIndex));
+	return {
+		splits,
+		joiners
+	};
+}
 var CharacterTransformer = class extends TextTransformer {
 	separator;
 	isSeparatorRegex;
@@ -4096,26 +4140,33 @@ var RecursiveCharacterTransformer = class RecursiveCharacterTransformer extends
 				break;
 			}
 		}
-		const splits = splitTextWithRegex(text, this.isSeparatorRegex ? separator : separator?.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), this.separatorPosition);
+		const _separator = this.isSeparatorRegex ? separator : separator?.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+		const { splits, joiners } = this.isSeparatorRegex && !this.separatorPosition && !!_separator ? splitTextWithMatchedSeparators(text, _separator) : {
+			splits: splitTextWithRegex(text, _separator, this.separatorPosition),
+			joiners: void 0
+		};
 		const goodSplits = [];
+		const goodJoiners = [];
 		const mergeSeparator = this.separatorPosition ? "" : separator;
-		for (const s of splits) if (this.lengthFunction(s) < this.maxSize) goodSplits.push(s);
-		else {
-			if (goodSplits.length > 0) {
-				const mergedText = this.mergeSplits(goodSplits, mergeSeparator);
-				finalChunks.push(...mergedText);
-				goodSplits.length = 0;
-			}
-			if (newSeparators.length === 0) finalChunks.push(s);
-			else {
-				const otherInfo = this._splitText(s, newSeparators);
-				finalChunks.push(...otherInfo);
+		const merge = () => this.mergeSplits(goodSplits, mergeSeparator, joiners ? goodJoiners : void 0);
+		splits.forEach((s, index) => {
+			if (this.lengthFunction(s) < this.maxSize) {
+				goodSplits.push(s);
+				goodJoiners.push(joiners?.[index] ?? "");
+			} else {
+				if (goodSplits.length > 0) {
+					finalChunks.push(...merge());
+					goodSplits.length = 0;
+					goodJoiners.length = 0;
+				}
+				if (newSeparators.length === 0) finalChunks.push(s);
+				else {
+					const otherInfo = this._splitText(s, newSeparators);
+					finalChunks.push(...otherInfo);
+				}
 			}
-		}
-		if (goodSplits.length > 0) {
-			const mergedText = this.mergeSplits(goodSplits, mergeSeparator);
-			finalChunks.push(...mergedText);
-		}
+		});
+		if (goodSplits.length > 0) finalChunks.push(...merge());
 		return finalChunks;
 	}
 	splitText({ text }) {
@@ -5542,35 +5593,44 @@ var SentenceTransformer = class extends TextTransformer {
 		const chunks = [];
 		let currentChunk = [];
 		let currentSize = 0;
+		let carriedCount = 0;
 		const separator = " ";
+		const sepLength = this.lengthFunction(separator);
 		for (const sentence of sentences) {
 			const sentenceLength = this.lengthFunction(sentence);
-			const separatorLength = currentChunk.length > 0 ? this.lengthFunction(separator) : 0;
-			const totalLength = currentSize + sentenceLength + separatorLength;
 			if (sentenceLength > this.maxSize) {
-				if (currentChunk.length > 0) {
-					chunks.push(currentChunk.join(separator));
-					currentChunk = [];
-					currentSize = 0;
-				}
+				if (currentChunk.length > carriedCount) chunks.push(currentChunk.join(separator));
+				currentChunk = [];
+				currentSize = 0;
+				carriedCount = 0;
 				const fallbackChunks = this.handleOversizedSentence(sentence);
 				chunks.push(...fallbackChunks);
 				continue;
 			}
-			if (currentChunk.length > 0 && totalLength > this.maxSize) {
-				chunks.push(currentChunk.join(separator));
-				currentChunk = this.calculateSentenceOverlap(currentChunk);
-				currentSize = this.calculateChunkSize(currentChunk);
+			if (currentChunk.length > 0 && currentSize + sepLength + sentenceLength > this.maxSize) {
+				if (currentChunk.length > carriedCount) chunks.push(currentChunk.join(separator));
+				const overlapSentences = this.calculateSentenceOverlap(currentChunk);
+				let overlapSize = this.calculateChunkSize(overlapSentences);
+				let start = 0;
+				while (start < overlapSentences.length && overlapSize + sepLength + sentenceLength > this.maxSize) {
+					overlapSize -= this.lengthFunction(overlapSentences[start]);
+					if (start < overlapSentences.length - 1) overlapSize -= sepLength;
+					start++;
+				}
+				currentChunk = overlapSentences.slice(start);
+				carriedCount = currentChunk.length;
+				currentSize = currentChunk.length > 0 ? overlapSize : 0;
 			}
+			currentSize += sentenceLength + (currentChunk.length > 0 ? sepLength : 0);
 			currentChunk.push(sentence);
-			currentSize += sentenceLength + separatorLength;
 			if (currentSize >= this.targetSize) {
 				chunks.push(currentChunk.join(separator));
 				currentChunk = this.calculateSentenceOverlap(currentChunk);
+				carriedCount = currentChunk.length;
 				currentSize = this.calculateChunkSize(currentChunk);
 			}
 		}
-		if (currentChunk.length > 0) chunks.push(currentChunk.join(separator));
+		if (currentChunk.length > carriedCount) chunks.push(currentChunk.join(separator));
 		return chunks;
 	}
 	/**