@mastra/rag 2.6.5-alpha.1 → 2.6.5-alpha.2

package/dist/docs/SKILL.md

@@ -3,7 +3,7 @@ name: mastra-rag
 description: Documentation for @mastra/rag. Use when working with @mastra/rag APIs, configuration, or implementation.
 metadata:
   package: "@mastra/rag"
-  version: "2.6.5-alpha.1"
+  version: "2.6.5-alpha.2"
 ---
 
 ## When to use

package/dist/docs/assets/SOURCE_MAP.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.6.5-alpha.1",
+  "version": "2.6.5-alpha.2",
   "package": "@mastra/rag",
   "exports": {},
   "modules": {}

package/dist/document/transformers/character.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"character.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/character.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACpC,OAAO,KAAK,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAE/F,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC;AA+CzC,qBAAa,oBAAqB,SAAQ,eAAe;IACvD,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC;IAC5B,SAAS,CAAC,gBAAgB,EAAE,OAAO,CAAC;IAEpC,YAAY,EAAE,SAAkB,EAAE,gBAAwB,EAAE,GAAG,WAAW,EAAE,GAAE,qBAA0B,EAIvG;IAED,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE,CAmB9C;IAED,OAAO,CAAC,YAAY;CAyDrB;AAED,qBAAa,6BAA8B,SAAQ,eAAe;IAChE,SAAS,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;IAC/B,SAAS,CAAC,gBAAgB,EAAE,OAAO,CAAC;IAEpC,YAAY,EAAE,UAAU,EAAE,gBAAwB,EAAE,QAAQ,EAAE,GAAG,WAAW,EAAE,GAAE,qBAA0B,EAIzG;IAED,OAAO,CAAC,UAAU;IAuDlB,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE,CAE9C;IAED,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,GAAE,gBAAqB,GAAG,6BAA6B,CAQrG;IAED,MAAM,CAAC,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,EAAE,CAua5D;CACF"}
+{"version":3,"file":"character.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/character.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACpC,OAAO,KAAK,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAE/F,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC;AA6EzC,qBAAa,oBAAqB,SAAQ,eAAe;IACvD,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC;IAC5B,SAAS,CAAC,gBAAgB,EAAE,OAAO,CAAC;IAEpC,YAAY,EAAE,SAAkB,EAAE,gBAAwB,EAAE,GAAG,WAAW,EAAE,GAAE,qBAA0B,EAIvG;IAED,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE,CAmB9C;IAED,OAAO,CAAC,YAAY;CAyDrB;AAED,qBAAa,6BAA8B,SAAQ,eAAe;IAChE,SAAS,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;IAC/B,SAAS,CAAC,gBAAgB,EAAE,OAAO,CAAC;IAEpC,YAAY,EAAE,UAAU,EAAE,gBAAwB,EAAE,QAAQ,EAAE,GAAG,WAAW,EAAE,GAAE,qBAA0B,EAIzG;IAED,OAAO,CAAC,UAAU;IA8DlB,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE,CAE9C;IAED,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,GAAE,gBAAqB,GAAG,6BAA6B,CAQrG;IAED,MAAM,CAAC,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,EAAE,CAua5D;CACF"}

package/dist/document/transformers/text.d.ts

@@ -17,6 +17,11 @@ export declare abstract class TextTransformer implements Transformer {
     splitDocuments(documents: Document[]): Document[];
     transformDocuments(documents: Document[]): Document[];
     protected joinDocs(docs: string[], separator: string): string | null;
-    protected mergeSplits(splits: string[], separator: string): string[];
+    /**
+     * Merges splits into chunks of at most `maxSize`.
+     * `joiners[i]`, when given, is the text placed before `splits[i]` when it is
+     * joined to a preceding piece; otherwise `separator` is used for every piece.
+     */
+    protected mergeSplits(splits: string[], separator: string, joiners?: string[]): string[];
 }
 //# sourceMappingURL=text.d.ts.map

package/dist/document/transformers/text.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"text.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEjD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD,8BAAsB,eAAgB,YAAW,WAAW;IAC1D,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,cAAc,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;IACnD,SAAS,CAAC,iBAAiB,CAAC,EAAE,OAAO,GAAG,KAAK,CAAC;IAC9C,SAAS,CAAC,aAAa,EAAE,OAAO,CAAC;IACjC,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IAEnC,YAAY,EACV,OAAc,EACd,OAAa,EACb,cAA8C,EAC9C,iBAAiB,EACjB,aAAqB,EACrB,eAAsB,GACvB,EAAE,gBAAgB,EAUlB;IAED,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAErC;IAED,QAAQ,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE,CAAC;IAEzD,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE,CA0B9E;IAED,cAAc,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,CAQhD;IAED,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,CAUpD;IAED,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAMnE;IAED,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAmEnE;CACF"}
+{"version":3,"file":"text.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEjD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD,8BAAsB,eAAgB,YAAW,WAAW;IAC1D,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,cAAc,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;IACnD,SAAS,CAAC,iBAAiB,CAAC,EAAE,OAAO,GAAG,KAAK,CAAC;IAC9C,SAAS,CAAC,aAAa,EAAE,OAAO,CAAC;IACjC,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IAEnC,YAAY,EACV,OAAc,EACd,OAAa,EACb,cAA8C,EAC9C,iBAAiB,EACjB,aAAqB,EACrB,eAAsB,GACvB,EAAE,gBAAgB,EAUlB;IAED,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAErC;IAED,QAAQ,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE,CAAC;IAEzD,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE,CA0B9E;IAED,cAAc,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,CAQhD;IAED,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,CAUpD;IAED,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAMnE;IAED;;;;OAIG;IACH,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAkFvF;CACF"}

package/dist/index.cjs

@@ -3960,46 +3960,66 @@ var TextTransformer = class {
 		if (this.stripWhitespace) text = text.trim();
 		return text === "" ? null : text;
 	}
-	mergeSplits(splits, separator) {
+	/**
+	* Merges splits into chunks of at most `maxSize`.
+	* `joiners[i]`, when given, is the text placed before `splits[i]` when it is
+	* joined to a preceding piece; otherwise `separator` is used for every piece.
+	*/
+	mergeSplits(splits, separator, joiners) {
 		const docs = [];
 		let currentDoc = [];
+		let currentJoiners = [];
 		let total = 0;
-		for (const d of splits) {
+		const lengthOf = (s) => s ? this.lengthFunction(s) : 0;
+		const join = (pieces, pieceJoiners) => {
+			if (!joiners) return this.joinDocs(pieces, separator);
+			const text = pieces.map((piece, i) => i === 0 ? piece : pieceJoiners[i] + piece).join("");
+			return this.joinDocs([text], "");
+		};
+		splits.forEach((d, index) => {
 			const len = this.lengthFunction(d);
-			const separatorLen = separator ? this.lengthFunction(separator) : 0;
+			const joiner = joiners ? joiners[index] ?? separator : separator;
+			const separatorLen = lengthOf(joiner);
 			if (total + len + (currentDoc.length > 0 ? separatorLen : 0) > this.maxSize) {
 				if (total > this.maxSize) console.warn(`Created a chunk of size ${total}, which is longer than the specified ${this.maxSize}`);
 				if (currentDoc.length > 0) {
-					const doc = this.joinDocs(currentDoc, separator);
+					const doc = join(currentDoc, currentJoiners);
 					if (doc !== null) docs.push(doc);
 					if (this.overlap > 0) {
-						let overlapContent = [];
+						const overlapContent = [];
+						const overlapJoiners = [];
 						let overlapSize = 0;
 						for (let i = currentDoc.length - 1; i >= 0; i--) {
 							const piece = currentDoc[i];
 							const pieceLen = this.lengthFunction(piece);
-							if (overlapSize + pieceLen > this.overlap) break;
+							const connectorLen = overlapContent.length > 0 ? lengthOf(currentJoiners[i + 1]) : 0;
+							if (overlapSize + pieceLen + connectorLen > this.overlap) break;
 							overlapContent.unshift(piece);
-							overlapSize += pieceLen + (overlapContent.length > 1 ? separatorLen : 0);
+							overlapJoiners.unshift(currentJoiners[i]);
+							overlapSize += pieceLen + connectorLen;
 						}
 						while (overlapContent.length > 0 && overlapSize + len + separatorLen > this.maxSize) {
 							const removed = overlapContent.shift();
+							overlapJoiners.shift();
 							const removedLen = this.lengthFunction(removed);
-							overlapSize -= removedLen + (overlapContent.length > 0 ? separatorLen : 0);
+							overlapSize -= removedLen + (overlapContent.length > 0 ? lengthOf(overlapJoiners[0]) : 0);
 						}
 						currentDoc = overlapContent;
+						currentJoiners = overlapJoiners;
 						total = overlapSize;
 					} else {
 						currentDoc = [];
+						currentJoiners = [];
 						total = 0;
 					}
 				}
 			}
 			currentDoc.push(d);
+			currentJoiners.push(joiner);
 			total += len + (currentDoc.length > 1 ? separatorLen : 0);
-		}
+		});
 		if (currentDoc.length > 0) {
-			const doc = this.joinDocs(currentDoc, separator);
+			const doc = join(currentDoc, currentJoiners);
 			if (doc !== null) docs.push(doc);
 		}
 		return docs;
@@ -4029,6 +4049,30 @@ function splitTextWithRegex(text, separator, separatorPosition) {
 	}
 	return result.filter((s) => s !== "");
 }
+function splitTextWithMatchedSeparators(text, separator) {
+	const splits = [];
+	const joiners = [];
+	let pending = "";
+	let lastIndex = 0;
+	const push = (piece) => {
+		if (piece !== "") {
+			splits.push(piece);
+			joiners.push(pending);
+			pending = "";
+		}
+	};
+	for (const match of text.matchAll(new RegExp(separator, "g"))) {
+		if (match[0] === "" && (match.index === 0 || match.index === text.length)) continue;
+		push(text.slice(lastIndex, match.index));
+		pending += match[0];
+		lastIndex = match.index + match[0].length;
+	}
+	push(text.slice(lastIndex));
+	return {
+		splits,
+		joiners
+	};
+}
 var CharacterTransformer = class extends TextTransformer {
 	separator;
 	isSeparatorRegex;
@@ -4098,26 +4142,33 @@ var RecursiveCharacterTransformer = class RecursiveCharacterTransformer extends
 				break;
 			}
 		}
-		const splits = splitTextWithRegex(text, this.isSeparatorRegex ? separator : separator?.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), this.separatorPosition);
+		const _separator = this.isSeparatorRegex ? separator : separator?.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+		const { splits, joiners } = this.isSeparatorRegex && !this.separatorPosition && !!_separator ? splitTextWithMatchedSeparators(text, _separator) : {
+			splits: splitTextWithRegex(text, _separator, this.separatorPosition),
+			joiners: void 0
+		};
 		const goodSplits = [];
+		const goodJoiners = [];
 		const mergeSeparator = this.separatorPosition ? "" : separator;
-		for (const s of splits) if (this.lengthFunction(s) < this.maxSize) goodSplits.push(s);
-		else {
-			if (goodSplits.length > 0) {
-				const mergedText = this.mergeSplits(goodSplits, mergeSeparator);
-				finalChunks.push(...mergedText);
-				goodSplits.length = 0;
-			}
-			if (newSeparators.length === 0) finalChunks.push(s);
-			else {
-				const otherInfo = this._splitText(s, newSeparators);
-				finalChunks.push(...otherInfo);
+		const merge = () => this.mergeSplits(goodSplits, mergeSeparator, joiners ? goodJoiners : void 0);
+		splits.forEach((s, index) => {
+			if (this.lengthFunction(s) < this.maxSize) {
+				goodSplits.push(s);
+				goodJoiners.push(joiners?.[index] ?? "");
+			} else {
+				if (goodSplits.length > 0) {
+					finalChunks.push(...merge());
+					goodSplits.length = 0;
+					goodJoiners.length = 0;
+				}
+				if (newSeparators.length === 0) finalChunks.push(s);
+				else {
+					const otherInfo = this._splitText(s, newSeparators);
+					finalChunks.push(...otherInfo);
+				}
 			}
-		}
-		if (goodSplits.length > 0) {
-			const mergedText = this.mergeSplits(goodSplits, mergeSeparator);
-			finalChunks.push(...mergedText);
-		}
+		});
+		if (goodSplits.length > 0) finalChunks.push(...merge());
 		return finalChunks;
 	}
 	splitText({ text }) {