mdream 0.15.2 → 0.16.0

package/dist/llms-txt.mjs

@@ -1,5 +1,383 @@
-import "./_chunks/markdown-processor-D26Uo5td.mjs";
-import "./_chunks/src-BJpipdul.mjs";
-import { n as generateLlmsTxtArtifacts, t as createLlmsTxtStream } from "./_chunks/llms-txt-Czb_M48B.mjs";
-
-export { createLlmsTxtStream, generateLlmsTxtArtifacts };
+import "./_chunks/markdown-processor.mjs";
+import "./_chunks/stream.mjs";
+import { htmlToMarkdown } from "./index.mjs";
+import { t as extractionPlugin } from "./_chunks/extraction.mjs";
+import { mkdir, open, readFile } from "node:fs/promises";
+import { basename, dirname, join, relative, sep } from "pathe";
+import { glob } from "tinyglobby";
+function extractMetadata(html, url) {
+	let title = "";
+	let description = "";
+	let keywords = "";
+	let author = "";
+	htmlToMarkdown(html, {
+		plugins: [extractionPlugin({
+			"title": (element) => {
+				if (!title && element.textContent) title = element.textContent.trim();
+			},
+			"meta[name=\"description\"]": (element) => {
+				if (!description && element.attributes?.content) description = element.attributes.content.trim();
+			},
+			"meta[property=\"og:description\"]": (element) => {
+				if (!description && element.attributes?.content) description = element.attributes.content.trim();
+			},
+			"meta[name=\"keywords\"]": (element) => {
+				if (!keywords && element.attributes?.content) keywords = element.attributes.content.trim();
+			},
+			"meta[name=\"author\"]": (element) => {
+				if (!author && element.attributes?.content) author = element.attributes.content.trim();
+			},
+			"meta[property=\"og:title\"]": (element) => {
+				if (!title && element.attributes?.content) title = element.attributes.content.trim();
+			}
+		})],
+		origin: url
+	});
+	return {
+		title: title || void 0,
+		description: description || void 0,
+		keywords: keywords || void 0,
+		author: author || void 0
+	};
+}
+function pathToUrl(filePath, baseDir) {
+	let url = relative(baseDir, filePath);
+	url = url.split(sep).join("/");
+	if (url.endsWith(".html")) url = url.slice(0, -5);
+	if (url.endsWith("/index")) url = url.slice(0, -6);
+	if (url === "index") return "/";
+	if (!url.startsWith("/")) url = `/${url}`;
+	return url;
+}
+async function processHtmlFiles(patterns, origin) {
+	const allPatterns = Array.isArray(patterns) ? patterns : [patterns];
+	const allFiles = [];
+	for (const pattern of allPatterns) {
+		const files = await glob(pattern);
+		allFiles.push(...files);
+	}
+	const uniqueFiles = [...new Set(allFiles)];
+	const results = [];
+	const baseDir = uniqueFiles.length > 0 ? dirname(uniqueFiles[0]) : ".";
+	for (const filePath of uniqueFiles) try {
+		const html = await readFile(filePath, "utf-8");
+		const metadata = extractMetadata(html, origin || filePath);
+		const content = htmlToMarkdown(html, { origin });
+		const url = pathToUrl(filePath, baseDir);
+		results.push({
+			filePath,
+			title: metadata?.title || basename(filePath, ".html"),
+			content,
+			url,
+			metadata
+		});
+	} catch (error) {
+		console.error(`Error processing ${filePath}:`, error);
+	}
+	return results;
+}
+function generateLlmsTxtContent(files, options) {
+	const { siteName = "Site", description, origin = "", sections, notes } = options;
+	let content = `# ${siteName}\n\n`;
+	if (description) content += `> ${description}\n\n`;
+	if (origin) content += `Canonical Origin: ${origin}\n\n`;
+	if (sections) for (const section of sections) content += formatSection(section);
+	if (files.length > 0) {
+		content += `## Pages\n\n`;
+		for (const file of files) {
+			const desc = file.metadata?.description;
+			const descText = desc ? `: ${desc.substring(0, 100)}${desc.length > 100 ? "..." : ""}` : "";
+			if (file.filePath && options.outputDir && file.filePath.endsWith(".md")) {
+				const relativePath = relative(options.outputDir, file.filePath);
+				content += `- [${file.title}](${relativePath})${descText}\n`;
+			} else {
+				const url = file.url.startsWith("http://") || file.url.startsWith("https://") ? file.url : origin ? origin + file.url : file.url;
+				content += `- [${file.title}](${url})${descText}\n`;
+			}
+		}
+	}
+	if (notes) content += `\n${formatNotes(notes)}`;
+	return content;
+}
+function parseFrontmatter(content) {
+	const match = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
+	if (!match) return {
+		frontmatter: null,
+		body: content
+	};
+	const frontmatterContent = match[1];
+	const body = match[2];
+	const frontmatter = {};
+	const lines = frontmatterContent.split("\n");
+	for (const line of lines) {
+		const colonIndex = line.indexOf(":");
+		if (colonIndex > 0) {
+			const key = line.substring(0, colonIndex).trim();
+			frontmatter[key] = line.substring(colonIndex + 1).trim();
+		}
+	}
+	return {
+		frontmatter,
+		body
+	};
+}
+function serializeFrontmatter(data) {
+	const lines = [];
+	for (const [key, value] of Object.entries(data)) if (value !== void 0 && value !== null) lines.push(`${key}: ${String(value)}`);
+	return lines.join("\n");
+}
+function generateLlmsFullTxtContent(files, options) {
+	const { siteName = "Site", description, origin = "", sections, notes } = options;
+	let content = `# ${siteName}\n\n`;
+	if (description) content += `> ${description}\n\n`;
+	if (origin) content += `Canonical Origin: ${origin}\n\n`;
+	if (sections) for (const section of sections) content += formatSection(section);
+	if (files.length > 0) {
+		content += `## Table of Contents\n\n`;
+		for (const file of files) {
+			const anchor = file.title.toLowerCase().replace(/[^a-z0-9]/g, "-");
+			content += `- [${file.title}](#${anchor})\n`;
+		}
+		content += `\n---\n\n`;
+		for (const file of files) {
+			const url = file.url.startsWith("http://") || file.url.startsWith("https://") ? file.url : origin ? origin + file.url : file.url;
+			const { frontmatter, body } = parseFrontmatter(file.content);
+			const metadata = {
+				title: file.title,
+				url
+			};
+			if (file.filePath && options.outputDir) metadata.file = relative(options.outputDir, file.filePath);
+			else if (file.filePath) metadata.file = file.filePath;
+			if (file.metadata) {
+				if (file.metadata.description) metadata.description = file.metadata.description;
+				if (file.metadata.keywords) metadata.keywords = file.metadata.keywords;
+				if (file.metadata.author) metadata.author = file.metadata.author;
+			}
+			const frontmatterString = serializeFrontmatter(frontmatter ? {
+				...frontmatter,
+				...metadata
+			} : metadata);
+			let contentBody = frontmatter ? body : file.content;
+			const titleLine = contentBody.trim().split("\n")[0];
+			if (titleLine === file.title || titleLine === `# ${file.title}`) contentBody = contentBody.trim().split("\n").slice(1).join("\n").trimStart();
+			content += `---\n${frontmatterString}\n---\n\n${contentBody}\n\n---\n\n`;
+		}
+	}
+	if (notes) content += `\n${formatNotes(notes)}`;
+	return content;
+}
+function generateMarkdownFilesContent(files) {
+	const markdownFiles = [];
+	for (const file of files) {
+		const mdPath = `md/${file.url === "/" ? "index" : file.url.replace(/^\//, "").replace(/\/$/, "")}.md`;
+		markdownFiles.push({
+			path: mdPath,
+			content: file.content
+		});
+	}
+	return markdownFiles;
+}
+async function generateLlmsTxtArtifacts(options) {
+	let files;
+	if (options.files) files = options.files;
+	else if (options.patterns) files = await processHtmlFiles(options.patterns, options.origin);
+	else throw new Error("Either patterns or files must be provided");
+	const llmsTxt = generateLlmsTxtContent(files, options);
+	let llmsFullTxt;
+	if (options.generateFull) llmsFullTxt = generateLlmsFullTxtContent(files, options);
+	let markdownFiles;
+	if (options.generateMarkdown) markdownFiles = generateMarkdownFilesContent(files);
+	return {
+		llmsTxt,
+		llmsFullTxt,
+		markdownFiles,
+		processedFiles: files
+	};
+}
+function formatSection(section) {
+	let content = `## ${section.title}\n\n`;
+	if (section.description) {
+		const descriptions = Array.isArray(section.description) ? section.description : [section.description];
+		for (const desc of descriptions) content += `${desc}\n\n`;
+	}
+	if (section.links?.length) {
+		for (const link of section.links) {
+			const desc = link.description ? `: ${link.description}` : "";
+			content += `- [${link.title}](${link.href})${desc}\n`;
+		}
+		content += "\n";
+	}
+	return content;
+}
+function formatNotes(notes) {
+	const noteLines = Array.isArray(notes) ? notes : [notes];
+	let content = "";
+	for (const note of noteLines) content += `${note}\n\n`;
+	return content;
+}
+function getGroupPrefix(url, depth) {
+	const segments = url.split("/").filter(Boolean);
+	if (segments.length === 0) return "/";
+	if (depth === 1 || segments.length === 1) return `/${segments[0]}`;
+	return `/${segments[0]}/${segments[1]}`;
+}
+function sortPagesByPath(pages) {
+	const twoSegmentCount = /* @__PURE__ */ new Map();
+	for (const page of pages) {
+		const prefix = getGroupPrefix(page.url, 2);
+		twoSegmentCount.set(prefix, (twoSegmentCount.get(prefix) || 0) + 1);
+	}
+	const segmentHasNested = /* @__PURE__ */ new Map();
+	for (const page of pages) {
+		const segments = page.url.split("/").filter(Boolean);
+		const firstSegment = segments.length > 0 ? segments[0] : "";
+		if (!segmentHasNested.has(firstSegment)) segmentHasNested.set(firstSegment, false);
+		if (segments.length > 1) segmentHasNested.set(firstSegment, true);
+	}
+	return pages.sort((a, b) => {
+		const segmentsA = a.url.split("/").filter(Boolean);
+		const segmentsB = b.url.split("/").filter(Boolean);
+		const firstSegmentA = segmentsA.length > 0 ? segmentsA[0] : "";
+		const firstSegmentB = segmentsB.length > 0 ? segmentsB[0] : "";
+		const twoSegPrefixA = getGroupPrefix(a.url, 2);
+		const twoSegPrefixB = getGroupPrefix(b.url, 2);
+		const twoSegCountA = twoSegmentCount.get(twoSegPrefixA) || 0;
+		const twoSegCountB = twoSegmentCount.get(twoSegPrefixB) || 0;
+		let groupKeyA = twoSegCountA > 1 ? twoSegPrefixA : `/${firstSegmentA}`;
+		let groupKeyB = twoSegCountB > 1 ? twoSegPrefixB : `/${firstSegmentB}`;
+		const isRootLevelA = segmentsA.length <= 1;
+		const isRootLevelB = segmentsB.length <= 1;
+		const hasNestedA = segmentHasNested.get(firstSegmentA);
+		const hasNestedB = segmentHasNested.get(firstSegmentB);
+		if (isRootLevelA && !hasNestedA) groupKeyA = "";
+		if (isRootLevelB && !hasNestedB) groupKeyB = "";
+		if (groupKeyA === "" && groupKeyB !== "") return -1;
+		if (groupKeyA !== "" && groupKeyB === "") return 1;
+		if (groupKeyA !== groupKeyB) return groupKeyA.localeCompare(groupKeyB);
+		if (segmentsA.length === 0) return -1;
+		if (segmentsB.length === 0) return 1;
+		const minLen = Math.min(segmentsA.length, segmentsB.length);
+		for (let i = 0; i < minLen; i++) {
+			const cmp = segmentsA[i].localeCompare(segmentsB[i]);
+			if (cmp !== 0) return cmp;
+		}
+		return segmentsA.length - segmentsB.length;
+	});
+}
+function createLlmsTxtStream(options = {}) {
+	const { siteName = "Site", description, origin = "", generateFull, outputDir = process.cwd(), sections, notes } = options;
+	let llmsTxtHandle;
+	let llmsFullTxtHandle;
+	const bufferedPages = [];
+	return new WritableStream({
+		async start() {
+			await mkdir(outputDir, { recursive: true });
+			llmsTxtHandle = await open(join(outputDir, "llms.txt"), "w");
+			let header = `# ${siteName}\n\n`;
+			if (description) header += `> ${description}\n\n`;
+			if (origin) header += `Canonical Origin: ${origin}\n\n`;
+			if (sections) for (const section of sections) header += formatSection(section);
+			await llmsTxtHandle.write(header);
+			if (generateFull) {
+				llmsFullTxtHandle = await open(join(outputDir, "llms-full.txt"), "w");
+				let fullHeader = `# ${siteName}\n\n`;
+				if (description) fullHeader += `> ${description}\n\n`;
+				if (origin) fullHeader += `Canonical Origin: ${origin}\n\n`;
+				if (sections) for (const section of sections) fullHeader += formatSection(section);
+				await llmsFullTxtHandle.write(fullHeader);
+			}
+		},
+		async write(file) {
+			const desc = file.metadata?.description;
+			bufferedPages.push({
+				url: file.url,
+				title: file.title,
+				description: desc,
+				filePath: file.filePath
+			});
+			if (generateFull && llmsFullTxtHandle) {
+				const url = file.url.startsWith("http://") || file.url.startsWith("https://") ? file.url : origin ? origin + file.url : file.url;
+				const { frontmatter, body } = parseFrontmatter(file.content);
+				const metadata = {
+					title: file.title,
+					url
+				};
+				if (file.filePath) metadata.file = relative(outputDir, file.filePath);
+				if (file.metadata) {
+					if (file.metadata.description) metadata.description = file.metadata.description;
+					if (file.metadata.keywords) metadata.keywords = file.metadata.keywords;
+					if (file.metadata.author) metadata.author = file.metadata.author;
+				}
+				const frontmatterString = serializeFrontmatter(frontmatter ? {
+					...frontmatter,
+					...metadata
+				} : metadata);
+				let contentBody = frontmatter ? body : file.content;
+				const titleLine = contentBody.trim().split("\n")[0];
+				if (titleLine === file.title || titleLine === `# ${file.title}`) contentBody = contentBody.trim().split("\n").slice(1).join("\n").trimStart();
+				const fullChunk = `---\n${frontmatterString}\n---\n\n${contentBody}\n\n---\n\n`;
+				await llmsFullTxtHandle.write(fullChunk);
+			}
+		},
+		async close() {
+			const sortedPages = sortPagesByPath(bufferedPages);
+			const twoSegmentCount = /* @__PURE__ */ new Map();
+			for (const page of sortedPages) {
+				const prefix = getGroupPrefix(page.url, 2);
+				twoSegmentCount.set(prefix, (twoSegmentCount.get(prefix) || 0) + 1);
+			}
+			const segmentHasNested = /* @__PURE__ */ new Map();
+			for (const page of sortedPages) {
+				const segments = page.url.split("/").filter(Boolean);
+				const firstSegment = segments.length > 0 ? segments[0] : "";
+				if (!segmentHasNested.has(firstSegment)) segmentHasNested.set(firstSegment, false);
+				if (segments.length > 1) segmentHasNested.set(firstSegment, true);
+			}
+			await llmsTxtHandle?.write(`## Pages\n\n`);
+			let currentGroup = "";
+			let segmentGroupIndex = 0;
+			let urlsInCurrentGroup = 0;
+			for (let i = 0; i < sortedPages.length; i++) {
+				const page = sortedPages[i];
+				const segments = page.url.split("/").filter(Boolean);
+				const firstSegment = segments.length > 0 ? segments[0] : "";
+				const twoSegPrefix = getGroupPrefix(page.url, 2);
+				let groupKey = (twoSegmentCount.get(twoSegPrefix) || 0) > 1 ? twoSegPrefix : `/${firstSegment}`;
+				const isRootLevel = segments.length <= 1;
+				const hasNested = segmentHasNested.get(firstSegment);
+				if (isRootLevel && !hasNested) groupKey = "";
+				if (groupKey !== currentGroup) {
+					if (urlsInCurrentGroup > 0) {
+						if (segmentGroupIndex === 0 || segmentGroupIndex >= 1 && segmentGroupIndex <= 2 && urlsInCurrentGroup > 1) await llmsTxtHandle?.write("\n");
+					}
+					currentGroup = groupKey;
+					segmentGroupIndex++;
+					urlsInCurrentGroup = 0;
+				}
+				urlsInCurrentGroup++;
+				const descText = page.description ? `: ${page.description.substring(0, 160)}${page.description.length > 160 ? "..." : ""}` : "";
+				let chunk = "";
+				if (page.filePath && page.filePath.endsWith(".md")) {
+					const relativePath = relative(outputDir, page.filePath);
+					chunk = `- [${page.title}](${relativePath})${descText}\n`;
+				} else {
+					const url = page.url.startsWith("http://") || page.url.startsWith("https://") ? page.url : origin ? origin + page.url : page.url;
+					chunk = `- [${page.title}](${url})${descText}\n`;
+				}
+				await llmsTxtHandle?.write(chunk);
+			}
+			if (notes) {
+				const notesContent = formatNotes(notes);
+				await llmsTxtHandle?.write(`\n${notesContent}`);
+				if (generateFull && llmsFullTxtHandle) await llmsFullTxtHandle.write(`\n${notesContent}`);
+			}
+			await llmsTxtHandle?.close();
+			await llmsFullTxtHandle?.close();
+		},
+		async abort(_reason) {
+			await llmsTxtHandle?.close();
+			await llmsFullTxtHandle?.close();
+		}
+	});
+}
+export { createLlmsTxtStream, generateLlmsTxtArtifacts };

package/dist/plugins.d.mts

@@ -1,8 +1,7 @@
-import { d as Plugin, x as extractionPlugin } from "./_chunks/types-CT4ZxeOH.mjs";
-import { t as createPlugin } from "./_chunks/plugin-D5soyEXm.mjs";
+import { b as extractionPlugin, u as Plugin } from "./_chunks/types.mjs";
+import { t as createPlugin } from "./_chunks/plugin.mjs";
 
 //#region src/plugins/filter.d.ts
-
 /**
  * Plugin that filters nodes based on CSS selectors.
  * Allows including or excluding nodes based on selectors.
@@ -16,11 +15,8 @@ import { t as createPlugin } from "./_chunks/plugin-D5soyEXm.mjs";
  * withQuerySelectorPlugin({ exclude: ['nav', '#sidebar', '.footer'] })
  */
 declare function filterPlugin(options?: {
-  /** CSS selectors (or Tag Ids) for elements to include (all others will be excluded) */
-  include?: (string | number)[];
-  /** CSS selectors (or Tag Ids) for elements to exclude */
-  exclude?: (string | number)[];
-  /** Whether to also process the children of matching elements */
+  /** CSS selectors (or Tag Ids) for elements to include (all others will be excluded) */include?: (string | number)[]; /** CSS selectors (or Tag Ids) for elements to exclude */
+  exclude?: (string | number)[]; /** Whether to also process the children of matching elements */
   processChildren?: boolean;
   keepAbsolute?: boolean;
 }): Plugin;
@@ -72,17 +68,10 @@ declare function frontmatterPlugin(options?: FrontmatterPluginOptions): Plugin;
  */
 declare function isolateMainPlugin(): Plugin;
 //#endregion
-//#region src/plugins/readability.d.ts
-/**
- * Creates a plugin that implements readability.js style heuristics for content quality assessment
- * Controls content inclusion/exclusion using buffer regions
- */
-declare function readabilityPlugin(): Plugin;
-//#endregion
 //#region src/plugins/tailwind.d.ts
 /**
  * Creates a plugin that adds Tailwind class processing
  */
 declare function tailwindPlugin(): Plugin;
 //#endregion
-export { createPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, readabilityPlugin, tailwindPlugin };
+export { createPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin };

package/dist/plugins.mjs

@@ -1,5 +1,4 @@
-import { t as createPlugin } from "./_chunks/plugin-CjWWQTuL.mjs";
-import { t as extractionPlugin } from "./_chunks/extraction-BA9MDtq3.mjs";
-import { a as filterPlugin, i as frontmatterPlugin, n as readabilityPlugin, r as isolateMainPlugin, t as tailwindPlugin } from "./_chunks/plugins-DJnqR2fA.mjs";
-
-export { createPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, readabilityPlugin, tailwindPlugin };
+import { t as createPlugin } from "./_chunks/plugin.mjs";
+import { t as extractionPlugin } from "./_chunks/extraction.mjs";
+import { i as filterPlugin, n as isolateMainPlugin, r as frontmatterPlugin, t as tailwindPlugin } from "./_chunks/tailwind.mjs";
+export { createPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin };

package/dist/preset/minimal.d.mts

@@ -1,7 +1,6 @@
-import { i as HTMLToMarkdownOptions } from "../_chunks/types-CT4ZxeOH.mjs";
+import { r as HTMLToMarkdownOptions } from "../_chunks/types.mjs";
 
 //#region src/preset/minimal.d.ts
-
 /**
  * Creates a configurable minimal preset with advanced options
  *

package/dist/preset/minimal.mjs

@@ -1,4 +1,30 @@
-import "../_chunks/plugins-DJnqR2fA.mjs";
-import { t as withMinimalPreset } from "../_chunks/minimal-BiDhcwif.mjs";
-
-export { withMinimalPreset };
+import { Ct as TAG_OBJECT, G as TAG_FIELDSET, J as TAG_FORM, K as TAG_FIGURE, Lt as TAG_SELECT, W as TAG_EMBED, Zt as TAG_TEXTAREA, at as TAG_IFRAME, bt as TAG_NAV, k as TAG_BUTTON, q as TAG_FOOTER, st as TAG_INPUT, x as TAG_ASIDE } from "../_chunks/const.mjs";
+import { i as filterPlugin, n as isolateMainPlugin, r as frontmatterPlugin, t as tailwindPlugin } from "../_chunks/tailwind.mjs";
+function withMinimalPreset(options = {}) {
+	const plugins = [
+		frontmatterPlugin(),
+		isolateMainPlugin(),
+		tailwindPlugin(),
+		filterPlugin({ exclude: [
+			TAG_FORM,
+			TAG_FIELDSET,
+			TAG_OBJECT,
+			TAG_EMBED,
+			TAG_FIGURE,
+			TAG_FOOTER,
+			TAG_ASIDE,
+			TAG_IFRAME,
+			TAG_INPUT,
+			TAG_TEXTAREA,
+			TAG_SELECT,
+			TAG_BUTTON,
+			TAG_NAV
+		] })
+	];
+	if (options.plugins) plugins.push(...options.plugins);
+	return {
+		...options,
+		plugins
+	};
+}
+export { withMinimalPreset };

package/dist/splitter.d.mts

@@ -1,7 +1,6 @@
-import { h as SplitterOptions, o as MarkdownChunk } from "./_chunks/types-CT4ZxeOH.mjs";
+import { a as MarkdownChunk, m as SplitterOptions } from "./_chunks/types.mjs";
 
 //#region src/splitter.d.ts
-
 /**
  * Convert HTML to Markdown and split into chunks in single pass
  * Yields chunks during HTML event processing for better memory efficiency

package/dist/splitter.mjs

@@ -1,7 +1,5 @@
-import { $ as TAG_H2, F as TAG_CODE, Nt as TAG_PRE, Q as TAG_H1, et as TAG_H3, gn as TEXT_NODE, h as NodeEventExit, m as NodeEventEnter, nt as TAG_H5, ot as TAG_HR, r as ELEMENT_NODE, rt as TAG_H6, tt as TAG_H4 } from "./_chunks/const-Bf_XN9U9.mjs";
-import { a as parseHtmlStream, n as createMarkdownProcessor, r as processPluginsForEvent } from "./_chunks/markdown-processor-D26Uo5td.mjs";
-
-//#region src/splitter.ts
+import { $ as TAG_H5, N as TAG_CODE, Q as TAG_H4, X as TAG_H2, Y as TAG_H1, Z as TAG_H3, dn as TEXT_NODE, et as TAG_H6, h as NodeEventExit, kt as TAG_PRE, m as NodeEventEnter, r as ELEMENT_NODE, rt as TAG_HR } from "./_chunks/const.mjs";
+import { a as parseHtmlStream, n as createMarkdownProcessor, r as processPluginsForEvent } from "./_chunks/markdown-processor.mjs";
 const DEFAULT_HEADERS_TO_SPLIT_ON = [
 	TAG_H2,
 	TAG_H3,
@@ -31,18 +29,9 @@ function getCodeLanguage(node) {
 function shouldSplitOnHeader(tagId, options) {
 	return options.headersToSplitOn.includes(tagId);
 }
-/**
-* Get current markdown content WITHOUT clearing buffers
-*/
 function getCurrentMarkdown(state) {
-	const fragments = [];
-	for (const [regionId, content] of state.regionContentBuffers.entries()) if (state.regionToggles.get(regionId)) fragments.push(...content);
-	return fragments.join("").trimStart();
+	return state.buffer.join("").trimStart();
 }
-/**
-* Convert HTML to Markdown and split into chunks in single pass
-* Yields chunks during HTML event processing for better memory efficiency
-*/
 function* htmlToMarkdownSplitChunksStream(html, options = {}) {
 	const opts = createOptions(options);
 	if (opts.chunkOverlap >= opts.chunkSize) throw new Error("chunkOverlap must be less than chunkSize");
@@ -158,10 +147,10 @@ function* htmlToMarkdownSplitChunksStream(html, options = {}) {
 					if (idx >= 0) {
 						const beforeSplit = currentMd.slice(0, candidateSplitPos);
 						let backtickCount = 0;
-						let pos = 0;
-						while ((pos = beforeSplit.indexOf("```", pos)) !== -1) {
+						let pos = beforeSplit.indexOf("```", 0);
+						while (pos !== -1) {
 							backtickCount++;
-							pos += 3;
+							pos = beforeSplit.indexOf("```", pos + 3);
 						}
 						if (backtickCount % 2 === 1) continue;
 					}
@@ -177,10 +166,6 @@ function* htmlToMarkdownSplitChunksStream(html, options = {}) {
 	}
 	yield* flushChunk();
 }
-/**
-* Convert HTML to Markdown and split into chunks in single pass
-* Chunks are created during HTML event processing
-*/
 function htmlToMarkdownSplitChunks(html, options = {}) {
 	const opts = createOptions(options);
 	const chunks = [];
@@ -208,6 +193,4 @@ function htmlToMarkdownSplitChunks(html, options = {}) {
 	}
 	return chunks;
 }
-
-//#endregion
-export { htmlToMarkdownSplitChunks, htmlToMarkdownSplitChunksStream };
+export { htmlToMarkdownSplitChunks, htmlToMarkdownSplitChunksStream };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "mdream",
   "type": "module",
-  "version": "0.15.2",
+  "version": "0.16.0",
   "description": "Ultra-performant HTML to Markdown Convertor Optimized for LLMs and llm.txt artifacts.",
   "author": {
     "name": "Harlan Wilton",