mdream 0.13.0 → 0.13.2

package/dist/splitter.mjs

@@ -62,10 +62,15 @@ function htmlToMarkdownSplitChunks(html, options = {}) {
 	let currentHeaderText = "";
 	let lineNumber = 1;
 	let lastChunkEndPosition = 0;
-	function flushChunk() {
+	let lastSplitPosition = 0;
+	function flushChunk(endPosition, applyOverlap = false) {
 		const currentMd = getCurrentMarkdown(processor.state);
-		const chunkContent = currentMd.slice(lastChunkEndPosition);
-		if (!chunkContent.trim()) return;
+		const chunkEnd = endPosition ?? currentMd.length;
+		const chunkContent = currentMd.slice(lastChunkEndPosition, chunkEnd);
+		if (!chunkContent.trim()) {
+			lastChunkEndPosition = chunkEnd;
+			return;
+		}
 		const chunk = {
 			content: chunkContent.trimEnd(),
 			metadata: { loc: { lines: {
@@ -83,10 +88,12 @@ function htmlToMarkdownSplitChunks(html, options = {}) {
 		if (currentChunkCodeLanguage) chunk.metadata.code = currentChunkCodeLanguage;
 		chunks.push(chunk);
 		currentChunkCodeLanguage = "";
-		if (opts.chunkOverlap > 0) {
-			const overlapText = chunkContent.slice(-opts.chunkOverlap);
-			lastChunkEndPosition = currentMd.length - overlapText.length;
-		} else lastChunkEndPosition = currentMd.length;
+		lastSplitPosition = chunkEnd;
+		if (applyOverlap && opts.chunkOverlap > 0) {
+			const maxOverlap = Math.max(0, chunkContent.length - 1);
+			const actualOverlap = Math.min(opts.chunkOverlap, maxOverlap);
+			lastChunkEndPosition = chunkEnd - actualOverlap;
+		} else lastChunkEndPosition = chunkEnd;
 		lineNumber += (chunkContent.match(/\n/g) || []).length;
 	}
 	const parseState = {
@@ -133,7 +140,36 @@ function htmlToMarkdownSplitChunks(html, options = {}) {
 		if (!opts.returnEachLine) {
 			const currentMd = getCurrentMarkdown(processor.state);
 			const currentChunkSize = opts.lengthFunction(currentMd.slice(lastChunkEndPosition));
-			if (currentChunkSize > opts.chunkSize) flushChunk();
+			if (currentChunkSize > opts.chunkSize) {
+				const idealSplitPos = lastChunkEndPosition + opts.chunkSize;
+				const separators = [
+					"\n\n",
+					"```\n",
+					"\n",
+					" "
+				];
+				let splitPosition = -1;
+				for (const sep of separators) {
+					const idx = currentMd.lastIndexOf(sep, idealSplitPos);
+					const candidateSplitPos = idx + sep.length;
+					if (idx >= 0) {
+						const beforeSplit = currentMd.slice(0, candidateSplitPos);
+						let backtickCount = 0;
+						let pos = 0;
+						while ((pos = beforeSplit.indexOf("```", pos)) !== -1) {
+							backtickCount++;
+							pos += 3;
+						}
+						if (backtickCount % 2 === 1) continue;
+					}
+					if (idx >= 0 && candidateSplitPos > lastSplitPosition) {
+						splitPosition = candidateSplitPos;
+						break;
+					}
+				}
+				if (splitPosition === -1 || splitPosition <= lastChunkEndPosition) splitPosition = currentMd.length;
+				flushChunk(splitPosition, true);
+			}
 		}
 	});
 	flushChunk();
@@ -159,7 +195,7 @@ function htmlToMarkdownSplitChunks(html, options = {}) {
 		return lineChunks;
 	}
 	if (opts.stripHeaders) for (const chunk of chunks) chunk.content = chunk.content.split("\n").filter((line) => !line.match(/^#{1,6}\s+/)).join("\n").trim();
-	return chunks;
+	return chunks.filter((chunk) => chunk.content.length > 0);
 }
 
 //#endregion

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "mdream",
   "type": "module",
-  "version": "0.13.0",
+  "version": "0.13.2",
   "description": "Ultra-performant HTML to Markdown Convertor Optimized for LLMs and llm.txt artifacts.",
   "author": {
     "name": "Harlan Wilton",