@mastra/rag 2.7.1-alpha.0 → 2.7.2-alpha.0

package/dist/index.js

@@ -4007,24 +4007,22 @@ var TextTransformer = class {
 // src/document/transformers/character.ts
 function splitTextWithRegex(text, separator, separatorPosition) {
 	if (!separator) return text.split("");
-	if (!separatorPosition) return text.split(new RegExp(separator)).filter((s) => s !== "");
 	if (!text) return [];
-	const splits = text.split(new RegExp(`(${separator})`));
 	const result = [];
-	if (separatorPosition === "end") {
-		for (let i = 0; i < splits.length - 1; i += 2) if (i + 1 < splits.length) {
-			const chunk = splits[i] + (splits[i + 1] || "");
-			if (chunk) result.push(chunk);
-		}
-		if (splits.length % 2 === 1 && splits[splits.length - 1]) result.push(splits?.[splits.length - 1]);
-	} else {
-		if (splits[0]) result.push(splits[0]);
-		for (let i = 1; i < splits.length - 1; i += 2) {
-			const separator2 = splits[i];
-			const text2 = splits[i + 1] ?? "";
-			if (separator2) result.push(separator2 + text2);
-		}
-	}
+	let lastIndex = 0;
+	let pendingSeparator = "";
+	for (const match of text.matchAll(new RegExp(separator, "g"))) {
+		const matched = match[0];
+		const piece = text.slice(lastIndex, match.index);
+		if (separatorPosition === "end") result.push(piece + matched);
+		else if (separatorPosition === "start") {
+			result.push(pendingSeparator + piece);
+			pendingSeparator = matched;
+		} else result.push(piece);
+		lastIndex = match.index + matched.length;
+	}
+	const tail = text.slice(lastIndex);
+	result.push(separatorPosition === "start" ? pendingSeparator + tail : tail);
 	return result.filter((s) => s !== "");
 }
 function splitTextWithMatchedSeparators(text, separator) {
@@ -4846,12 +4844,10 @@ var HTMLSectionTransformer = class {
 // src/document/transformers/json.ts
 var RecursiveJsonTransformer = class RecursiveJsonTransformer {
 	maxSize;
-	minSize;
 	ensureAscii;
 	convertLists;
-	constructor({ maxSize = 2e3, minSize, ensureAscii = false, convertLists = true }) {
+	constructor({ maxSize = 2e3, ensureAscii = false, convertLists = true }) {
 		this.maxSize = maxSize;
-		this.minSize = minSize ?? Math.max(maxSize - 200, 50);
 		this.ensureAscii = ensureAscii;
 		this.convertLists = convertLists;
 	}
@@ -4931,7 +4927,7 @@ var RecursiveJsonTransformer = class RecursiveJsonTransformer {
 				},
 				chunks
 			};
-			else return {
+			return {
 				currentChunk: testValue,
 				chunks: [...chunks, currentChunk]
 			};
@@ -4978,14 +4974,11 @@ var RecursiveJsonTransformer = class RecursiveJsonTransformer {
 		let currentGroup = [];
 		const saveCurrentGroup = () => {
 			if (currentGroup.length > 0) {
-				const groupChunk = this.createChunk(currentGroup, path);
-				if (RecursiveJsonTransformer.jsonSize(groupChunk) >= this.minSize) {
-					chunks.push(groupChunk);
-					currentGroup = [];
-				}
+				chunks.push(this.createChunk(currentGroup, path));
+				currentGroup = [];
 			}
 		};
-		for (const item of value) {
+		for (const [itemIndex, item] of value.entries()) {
 			const testGroup = [...currentGroup, item];
 			const testChunk = this.createChunk(testGroup, path);
 			if (this.isWithinSizeLimit(testChunk)) {
@@ -4998,7 +4991,7 @@ var RecursiveJsonTransformer = class RecursiveJsonTransformer {
 				const singleItemChunk = this.createChunk(singleItemArray, path);
 				if (this.isWithinSizeLimit(singleItemChunk)) currentGroup = singleItemArray;
 				else {
-					const itemPath = [...path, String(chunks.length)];
+					const itemPath = [...path, String(itemIndex)];
 					const nestedChunks = this.handleNestedObject(item, itemPath, depth + 1, maxDepth);
 					chunks.push(...nestedChunks);
 				}
@@ -5023,11 +5016,8 @@ var RecursiveJsonTransformer = class RecursiveJsonTransformer {
 		let currentChunk = {};
 		const saveCurrentChunk = () => {
 			if (Object.keys(currentChunk).length > 0) {
-				const objChunk = this.createChunk(currentChunk, path);
-				if (RecursiveJsonTransformer.jsonSize(objChunk) >= this.minSize) {
-					chunks.push(objChunk);
-					currentChunk = {};
-				}
+				chunks.push(this.createChunk(currentChunk, path));
+				currentChunk = {};
 			}
 		};
 		for (const [key, val] of Object.entries(value)) {
@@ -5115,30 +5105,15 @@ var RecursiveJsonTransformer = class RecursiveJsonTransformer {
 		return chunks;
 	}
 	/**
-	* Converts Unicode characters to their escaped ASCII representation
-	* e.g., 'café' becomes 'caf\u00e9'
-	*/
-	escapeNonAscii(obj) {
-		if (typeof obj === "string") return obj.replace(/[\u0080-\uffff]/g, (char) => {
-			return `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`;
-		});
-		if (Array.isArray(obj)) return obj.map((item) => this.escapeNonAscii(item));
-		if (typeof obj === "object" && obj !== null) return Object.fromEntries(Object.entries(obj).map(([key, value]) => [key, this.escapeNonAscii(value)]));
-		return obj;
-	}
-	/**
 	* Splits JSON into a list of JSON formatted strings
 	*/
 	splitText({ jsonData, convertLists = false, ensureAscii = true }) {
-		const chunks = this.splitJson({
+		const serialized = this.splitJson({
 			jsonData,
 			convertLists
-		});
-		if (ensureAscii) return chunks.map((chunk) => this.escapeNonAscii(chunk)).map((chunk) => JSON.stringify(chunk));
-		return chunks.map((chunk) => JSON.stringify(chunk, (key, value) => {
-			if (typeof value === "string") return value.replace(/\\u[\da-f]{4}/gi, (match) => String.fromCharCode(parseInt(match.slice(2), 16)));
-			return value;
-		}));
+		}).map((chunk) => JSON.stringify(chunk));
+		if (!ensureAscii) return serialized;
+		return serialized.map((text) => text.replace(/[\u0080-\uffff]/g, (char) => `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`));
 	}
 	/**
 	* Create documents from a list of json objects
@@ -5250,11 +5225,12 @@ var MarkdownHeaderTransformer = class {
 			const line = lines[i];
 			const strippedLine = line.trim();
 			if (!inCodeBlock) {
-				if (strippedLine.startsWith("```") && strippedLine.split("```").length === 2 || strippedLine.startsWith("~~~")) {
+				const fence = strippedLine.match(/^(`{3,}|~{3,})/);
+				if (fence && (fence[1][0] !== "`" || !strippedLine.slice(fence[1].length).includes("`"))) {
 					inCodeBlock = true;
-					openingFence = strippedLine.startsWith("```") ? "```" : "~~~";
+					openingFence = fence[1];
 				}
-			} else if (strippedLine.startsWith(openingFence)) {
+			} else if (strippedLine.startsWith(openingFence) && new RegExp(`^${openingFence[0]}{${openingFence.length},}\\s*$`).test(strippedLine)) {
 				inCodeBlock = false;
 				openingFence = "";
 			}
@@ -5262,13 +5238,9 @@ var MarkdownHeaderTransformer = class {
 				currentContent.push(line);
 				continue;
 			}
-			if (strippedLine.includes("|") && strippedLine.length > 0) {
-				if (!inTable) inTable = true;
-				currentContent.push(line);
-				continue;
-			} else if (inTable) inTable = false;
+			const isIndentedCode = strippedLine.includes("|") && /^(?: {4}|\t)/.test(line);
 			let headerMatched = false;
-			for (const [sep, name] of this.headersToSplitOn) if (strippedLine.startsWith(sep) && (strippedLine.length === sep.length || strippedLine[sep.length] === " ")) {
+			for (const [sep, name] of this.headersToSplitOn) if (!isIndentedCode && strippedLine.startsWith(sep) && (strippedLine.length === sep.length || strippedLine[sep.length] === " ")) {
 				headerMatched = true;
 				if (currentContent.length > 0) {
 					linesWithMetadata.push({
@@ -5300,7 +5272,14 @@ var MarkdownHeaderTransformer = class {
 				});
 				break;
 			}
-			if (!headerMatched) {
+			if (headerMatched) inTable = false;
+			else {
+				if (strippedLine.includes("|")) {
+					inTable = true;
+					currentContent.push(line);
+					continue;
+				}
+				if (inTable) inTable = false;
 				if (strippedLine || this.returnEachLine) {
 					currentContent.push(line);
 					if (this.returnEachLine) {