@mastra/rag 2.6.5-alpha.0 → 2.6.5-alpha.1

package/dist/docs/SKILL.md

@@ -3,7 +3,7 @@ name: mastra-rag
 description: Documentation for @mastra/rag. Use when working with @mastra/rag APIs, configuration, or implementation.
 metadata:
   package: "@mastra/rag"
-  version: "2.6.5-alpha.0"
+  version: "2.6.5-alpha.1"
 ---
 
 ## When to use

package/dist/docs/assets/SOURCE_MAP.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.6.5-alpha.0",
+  "version": "2.6.5-alpha.1",
   "package": "@mastra/rag",
   "exports": {},
   "modules": {}

package/dist/document/transformers/sentence.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sentence.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/sentence.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC;AAEzC,qBAAa,mBAAoB,SAAQ,eAAe;IACtD,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC;IAC7B,SAAS,CAAC,cAAc,EAAE,MAAM,EAAE,CAAC;IACnC,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,oBAAoB,EAAE,OAAO,CAAC;IAExC,YAAY,OAAO,EAAE,oBAAoB,EAoBxC;IAED,OAAO,CAAC,wBAAwB;IA+BhC,OAAO,CAAC,sBAAsB;IAqB9B,OAAO,CAAC,oBAAoB;IA8B5B;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAsDhC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAqB/B,OAAO,CAAC,sBAAsB;IAmC9B,OAAO,CAAC,2BAA2B;IAsBnC,OAAO,CAAC,wBAAwB;IA4BhC,OAAO,CAAC,kBAAkB;IAqB1B,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE,CAQ9C;CACF"}
+{"version":3,"file":"sentence.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/sentence.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC;AAEzC,qBAAa,mBAAoB,SAAQ,eAAe;IACtD,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC;IAC7B,SAAS,CAAC,cAAc,EAAE,MAAM,EAAE,CAAC;IACnC,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,oBAAoB,EAAE,OAAO,CAAC;IAExC,YAAY,OAAO,EAAE,oBAAoB,EAoBxC;IAED,OAAO,CAAC,wBAAwB;IA+BhC,OAAO,CAAC,sBAAsB;IAqB9B,OAAO,CAAC,oBAAoB;IA8B5B;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAmEhC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAqB/B,OAAO,CAAC,sBAAsB;IAmC9B,OAAO,CAAC,2BAA2B;IAsBnC,OAAO,CAAC,wBAAwB;IA4BhC,OAAO,CAAC,kBAAkB;IAqB1B,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE,CAQ9C;CACF"}

package/dist/index.cjs

@@ -5544,35 +5544,44 @@ var SentenceTransformer = class extends TextTransformer {
 		const chunks = [];
 		let currentChunk = [];
 		let currentSize = 0;
+		let carriedCount = 0;
 		const separator = " ";
+		const sepLength = this.lengthFunction(separator);
 		for (const sentence of sentences) {
 			const sentenceLength = this.lengthFunction(sentence);
-			const separatorLength = currentChunk.length > 0 ? this.lengthFunction(separator) : 0;
-			const totalLength = currentSize + sentenceLength + separatorLength;
 			if (sentenceLength > this.maxSize) {
-				if (currentChunk.length > 0) {
-					chunks.push(currentChunk.join(separator));
-					currentChunk = [];
-					currentSize = 0;
-				}
+				if (currentChunk.length > carriedCount) chunks.push(currentChunk.join(separator));
+				currentChunk = [];
+				currentSize = 0;
+				carriedCount = 0;
 				const fallbackChunks = this.handleOversizedSentence(sentence);
 				chunks.push(...fallbackChunks);
 				continue;
 			}
-			if (currentChunk.length > 0 && totalLength > this.maxSize) {
-				chunks.push(currentChunk.join(separator));
-				currentChunk = this.calculateSentenceOverlap(currentChunk);
-				currentSize = this.calculateChunkSize(currentChunk);
+			if (currentChunk.length > 0 && currentSize + sepLength + sentenceLength > this.maxSize) {
+				if (currentChunk.length > carriedCount) chunks.push(currentChunk.join(separator));
+				const overlapSentences = this.calculateSentenceOverlap(currentChunk);
+				let overlapSize = this.calculateChunkSize(overlapSentences);
+				let start = 0;
+				while (start < overlapSentences.length && overlapSize + sepLength + sentenceLength > this.maxSize) {
+					overlapSize -= this.lengthFunction(overlapSentences[start]);
+					if (start < overlapSentences.length - 1) overlapSize -= sepLength;
+					start++;
+				}
+				currentChunk = overlapSentences.slice(start);
+				carriedCount = currentChunk.length;
+				currentSize = currentChunk.length > 0 ? overlapSize : 0;
 			}
+			currentSize += sentenceLength + (currentChunk.length > 0 ? sepLength : 0);
 			currentChunk.push(sentence);
-			currentSize += sentenceLength + separatorLength;
 			if (currentSize >= this.targetSize) {
 				chunks.push(currentChunk.join(separator));
 				currentChunk = this.calculateSentenceOverlap(currentChunk);
+				carriedCount = currentChunk.length;
 				currentSize = this.calculateChunkSize(currentChunk);
 			}
 		}
-		if (currentChunk.length > 0) chunks.push(currentChunk.join(separator));
+		if (currentChunk.length > carriedCount) chunks.push(currentChunk.join(separator));
 		return chunks;
 	}
 	/**