@mastra/rag 2.6.5-alpha.0 → 2.6.5-alpha.1

package/dist/index.js

@@ -5542,35 +5542,44 @@ var SentenceTransformer = class extends TextTransformer {
 		const chunks = [];
 		let currentChunk = [];
 		let currentSize = 0;
+		let carriedCount = 0;
 		const separator = " ";
+		const sepLength = this.lengthFunction(separator);
 		for (const sentence of sentences) {
 			const sentenceLength = this.lengthFunction(sentence);
-			const separatorLength = currentChunk.length > 0 ? this.lengthFunction(separator) : 0;
-			const totalLength = currentSize + sentenceLength + separatorLength;
 			if (sentenceLength > this.maxSize) {
-				if (currentChunk.length > 0) {
-					chunks.push(currentChunk.join(separator));
-					currentChunk = [];
-					currentSize = 0;
-				}
+				if (currentChunk.length > carriedCount) chunks.push(currentChunk.join(separator));
+				currentChunk = [];
+				currentSize = 0;
+				carriedCount = 0;
 				const fallbackChunks = this.handleOversizedSentence(sentence);
 				chunks.push(...fallbackChunks);
 				continue;
 			}
-			if (currentChunk.length > 0 && totalLength > this.maxSize) {
-				chunks.push(currentChunk.join(separator));
-				currentChunk = this.calculateSentenceOverlap(currentChunk);
-				currentSize = this.calculateChunkSize(currentChunk);
+			if (currentChunk.length > 0 && currentSize + sepLength + sentenceLength > this.maxSize) {
+				if (currentChunk.length > carriedCount) chunks.push(currentChunk.join(separator));
+				const overlapSentences = this.calculateSentenceOverlap(currentChunk);
+				let overlapSize = this.calculateChunkSize(overlapSentences);
+				let start = 0;
+				while (start < overlapSentences.length && overlapSize + sepLength + sentenceLength > this.maxSize) {
+					overlapSize -= this.lengthFunction(overlapSentences[start]);
+					if (start < overlapSentences.length - 1) overlapSize -= sepLength;
+					start++;
+				}
+				currentChunk = overlapSentences.slice(start);
+				carriedCount = currentChunk.length;
+				currentSize = currentChunk.length > 0 ? overlapSize : 0;
 			}
+			currentSize += sentenceLength + (currentChunk.length > 0 ? sepLength : 0);
 			currentChunk.push(sentence);
-			currentSize += sentenceLength + separatorLength;
 			if (currentSize >= this.targetSize) {
 				chunks.push(currentChunk.join(separator));
 				currentChunk = this.calculateSentenceOverlap(currentChunk);
+				carriedCount = currentChunk.length;
 				currentSize = this.calculateChunkSize(currentChunk);
 			}
 		}
-		if (currentChunk.length > 0) chunks.push(currentChunk.join(separator));
+		if (currentChunk.length > carriedCount) chunks.push(currentChunk.join(separator));
 		return chunks;
 	}
 	/**