@mdream/js 0.17.0

package/dist/_chunks/resolve-plugins.mjs

@@ -0,0 +1,302 @@
+import { n as DEFAULT_BLOCK_SPACING, o as NO_SPACING } from "./const.mjs";
+import { r as parseHtmlStream } from "./parse.mjs";
+import { a as extractionCollectorPlugin, i as filterPlugin, n as isolateMainPlugin, r as frontmatterPlugin, t as tailwindPlugin } from "./plugins.mjs";
+//#region src/plugin-processor.ts
+/**
+* Processes plugins for a given node event
+* Shared logic between markdown-processor.ts and stream.ts
+*
+* @param event - The node event to process
+* @param plugins - Array of plugins to apply
+* @param state - The current runtime state
+* @param processEvent - Callback to process the event after plugin processing
+* @returns true if the event should be skipped, false to continue processing
+*/
+function processPluginsForEvent(event, plugins, state, processEvent) {
+	if (plugins?.length) {
+		if (event.node.type === 1 && event.type === 0) {
+			const element = event.node;
+			for (const plugin of plugins) if (plugin.processAttributes) plugin.processAttributes(element, state);
+		}
+		let shouldSkip = false;
+		for (const plugin of plugins) {
+			const res = plugin.beforeNodeProcess?.(event, state);
+			if (typeof res === "object") shouldSkip = res.skip;
+		}
+		if (shouldSkip) return true;
+		if (event.node.type === 1) {
+			const element = event.node;
+			const fn = event.type === 0 ? "onNodeEnter" : "onNodeExit";
+			const pluginOutputs = [];
+			for (const plugin of plugins) if (plugin[fn]) {
+				const result = plugin[fn](element, state);
+				if (result) pluginOutputs.push(result);
+			}
+			if (pluginOutputs.length > 0) element.pluginOutput = [...element.pluginOutput || [], ...pluginOutputs];
+		} else if (event.node.type === 2 && event.type === 0) {
+			const textNode = event.node;
+			for (const plugin of plugins) if (plugin.processTextNode) {
+				const result = plugin.processTextNode(textNode, state);
+				if (result) {
+					if (result.skip) return true;
+					textNode.value = result.content;
+				}
+			}
+		}
+	}
+	processEvent(event);
+	return false;
+}
+//#endregion
+//#region src/markdown-processor.ts
+/**
+* Determines if spacing is needed between two characters
+*/
+function needsSpacing(lastChar, firstChar, state) {
+	if (lastChar === " " || lastChar === "\n" || lastChar === "	") return false;
+	if (firstChar === " " || firstChar === "\n" || firstChar === "	") return false;
+	const noSpaceAfter = new Set([
+		"[",
+		"(",
+		">",
+		"*",
+		"_",
+		"`"
+	]);
+	const noSpaceBefore = new Set([
+		"]",
+		")",
+		"<",
+		".",
+		",",
+		"!",
+		"?",
+		":",
+		";",
+		"*",
+		"_",
+		"`"
+	]);
+	if (lastChar === "|" && firstChar === "<" && state && (state.depthMap[28] || 0) > 0) return true;
+	if (noSpaceAfter.has(lastChar) || noSpaceBefore.has(firstChar)) return false;
+	return true;
+}
+/**
+* Determines if spacing should be added before text content
+*/
+function shouldAddSpacingBeforeText(lastChar, lastNode, textNode) {
+	if (!lastChar || lastChar === "\n" || lastChar === " " || lastChar === "[" || lastChar === ">") return false;
+	if (lastNode?.tagHandler?.isInline) return false;
+	const firstChar = textNode.value[0];
+	if (firstChar === " ") return false;
+	if (firstChar === "." || firstChar === "," || firstChar === "!" || firstChar === "?" || firstChar === ":" || firstChar === ";" || firstChar === "_" || firstChar === "*" || firstChar === "`" || firstChar === ")" || firstChar === "]") return false;
+	return true;
+}
+/**
+* Calculate newline configuration based on tag handler spacing config
+*/
+function calculateNewLineConfig(node) {
+	const tagId = node.tagId;
+	const depthMap = node.depthMap;
+	if (tagId !== 25 && (depthMap[25] || 0) > 0 || tagId !== 22 && (depthMap[22] || 0) > 0) return NO_SPACING;
+	const isBlockElement = tagId !== void 0 && (tagId >= 7 && tagId <= 12 || tagId === 35 || tagId === 36);
+	let currParent = node.parent;
+	while (currParent) {
+		if (currParent.tagHandler?.collapsesInnerWhiteSpace) {
+			if (isBlockElement && currParent.tagId === 37) {
+				currParent = currParent.parent;
+				continue;
+			}
+			return NO_SPACING;
+		}
+		currParent = currParent.parent;
+	}
+	if (node.tagHandler?.spacing) return node.tagHandler?.spacing;
+	return DEFAULT_BLOCK_SPACING;
+}
+/**
+* Creates a markdown processor that consumes DOM events and generates markdown
+*/
+function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverrideHandlers) {
+	const state = {
+		options,
+		buffer: [],
+		depthMap: new Uint8Array(108)
+	};
+	let lastYieldedLength = 0;
+	/**
+	* Process a DOM event and generate markdown
+	*/
+	function processEvent(event) {
+		const { type: eventType, node } = event;
+		const lastNode = state.lastNode;
+		state.lastNode = event.node;
+		state.depth = node.depth;
+		const buff = state.buffer;
+		const lastBuffEntry = buff.at(-1);
+		const lastChar = lastBuffEntry?.charAt(lastBuffEntry.length - 1) || "";
+		let secondLastChar;
+		if (lastBuffEntry && lastBuffEntry.length > 1) secondLastChar = lastBuffEntry.charAt(lastBuffEntry.length - 2);
+		else if (buff.length > 1) {
+			const prevBuff = buff[buff.length - 2];
+			if (prevBuff) secondLastChar = prevBuff.charAt(prevBuff.length - 1);
+		}
+		if (node.type === 2 && eventType === 0) {
+			const textNode = node;
+			if (textNode.value) {
+				if (textNode.excludedFromMarkdown) return;
+				if (textNode.value === " " && lastChar === "\n") return;
+				if (shouldAddSpacingBeforeText(lastChar, lastNode, textNode)) textNode.value = ` ${textNode.value}`;
+				state.buffer.push(textNode.value);
+				state.lastContentCache = textNode.value;
+			}
+			state.lastTextNode = textNode;
+			return;
+		}
+		if (node.type !== 1) return;
+		const context = {
+			node,
+			state
+		};
+		const output = [];
+		const element = node;
+		if (element.pluginOutput?.length) {
+			output.push(...element.pluginOutput);
+			element.pluginOutput = [];
+		}
+		const lastFragment = state.lastContentCache;
+		let lastNewLines = 0;
+		if (lastChar === "\n") lastNewLines++;
+		if (secondLastChar === "\n") lastNewLines++;
+		const eventFn = eventType === 0 ? "enter" : "exit";
+		const handler = node.tagHandler;
+		if (!output.length && handler?.[eventFn]) {
+			const res = handler[eventFn](context);
+			if (res) output.push(res);
+		}
+		const configuredNewLines = calculateNewLineConfig(node)[eventType] || 0;
+		const newLines = Math.max(0, configuredNewLines - lastNewLines);
+		if (newLines > 0) {
+			if (!buff.length) {
+				for (const fragment of output) if (fragment) {
+					state.buffer.push(fragment);
+					state.lastContentCache = fragment;
+				}
+				return;
+			}
+			const newlinesStr = "\n".repeat(newLines);
+			if (lastChar === " " && buff?.length) buff[buff.length - 1] = buff.at(-1).substring(0, buff.at(-1).length - 1);
+			if (eventType === 0) output.unshift(newlinesStr);
+			else output.push(newlinesStr);
+		} else if (lastFragment && state.lastTextNode?.containsWhitespace && !!node.parent && "value" in state.lastTextNode && typeof state.lastTextNode.value === "string") {
+			if (!node.parent.depthMap[34] || node.parent.tagId === 34) {
+				const isInlineElement = node.tagHandler?.isInline;
+				const collapsesWhiteSpace = node.tagHandler?.collapsesInnerWhiteSpace;
+				const hasSpacing = node.tagHandler?.spacing && Array.isArray(node.tagHandler.spacing);
+				if ((!isInlineElement || eventType === 1) && !(!isInlineElement && !collapsesWhiteSpace && configuredNewLines > 0) && !(collapsesWhiteSpace && eventType === 0) && !(hasSpacing && eventType === 0)) {
+					const originalLength = lastFragment.length;
+					const trimmed = lastFragment.trimEnd();
+					if (originalLength - trimmed.length > 0) {
+						if (buff?.length && buff.at(-1) === lastFragment) buff[buff.length - 1] = trimmed;
+					}
+				}
+				state.lastTextNode = void 0;
+			}
+		}
+		if (output[0]?.[0] && eventType === 0 && lastChar && needsSpacing(lastChar, output[0][0], state)) {
+			state.buffer.push(" ");
+			state.lastContentCache = " ";
+		}
+		for (const fragment of output) if (fragment) {
+			state.buffer.push(fragment);
+			state.lastContentCache = fragment;
+		}
+	}
+	/**
+	* Process HTML string and generate events
+	*/
+	function processHtml(html) {
+		parseHtmlStream(html, {
+			depthMap: state.depthMap,
+			depth: 0,
+			resolvedPlugins,
+			tagOverrideHandlers
+		}, (event) => {
+			processPluginsForEvent(event, resolvedPlugins, state, processEvent);
+		});
+	}
+	/**
+	* Get the final markdown output
+	*/
+	function getMarkdown() {
+		const result = state.buffer.join("").trimStart();
+		state.buffer.length = 0;
+		return result.trimEnd();
+	}
+	/**
+	* Get new markdown content since the last call (for streaming)
+	*/
+	function getMarkdownChunk() {
+		const currentContent = state.buffer.join("").trimStart();
+		const newContent = currentContent.slice(lastYieldedLength);
+		lastYieldedLength = currentContent.length;
+		return newContent;
+	}
+	return {
+		processEvent,
+		processHtml,
+		getMarkdown,
+		getMarkdownChunk,
+		state
+	};
+}
+//#endregion
+//#region src/resolve-plugins.ts
+function resolveFrontmatterOpt(opt) {
+	if (typeof opt === "function") return {
+		config: {},
+		callback: opt
+	};
+	if (typeof opt === "object") return {
+		config: opt,
+		callback: opt.onExtract
+	};
+	return { config: {} };
+}
+/**
+* Resolves declarative BuiltinPlugins config into a flat TransformPlugin array.
+* Optionally appends imperative transform plugins.
+*/
+function resolvePlugins(options, hooks) {
+	const plugins = [];
+	let callExtractionHandlers;
+	let getFrontmatter;
+	let frontmatterCallback;
+	const config = options.plugins;
+	if (config) {
+		if (config.frontmatter) {
+			const fm = resolveFrontmatterOpt(config.frontmatter);
+			const fmPlugin = frontmatterPlugin(fm.config);
+			plugins.push(fmPlugin);
+			getFrontmatter = fmPlugin.getFrontmatter;
+			frontmatterCallback = fm.callback;
+		}
+		if (config.isolateMain) plugins.push(isolateMainPlugin());
+		if (config.tailwind) plugins.push(tailwindPlugin());
+		if (config.filter) plugins.push(filterPlugin(config.filter));
+		if (config.extraction) {
+			const collector = extractionCollectorPlugin(config.extraction);
+			plugins.push(collector.plugin);
+			callExtractionHandlers = collector.callHandlers;
+		}
+	}
+	if (hooks) plugins.push(...hooks);
+	return {
+		plugins,
+		callExtractionHandlers,
+		getFrontmatter,
+		frontmatterCallback
+	};
+}
+//#endregion
+export { createMarkdownProcessor as n, processPluginsForEvent as r, resolvePlugins as t };

package/dist/_chunks/src.mjs

@@ -0,0 +1,344 @@
+import { i as buildTagOverrideHandlers, r as parseHtmlStream } from "./parse.mjs";
+import { n as createMarkdownProcessor, r as processPluginsForEvent, t as resolvePlugins } from "./resolve-plugins.mjs";
+//#region src/clean.ts
+function resolveClean(clean) {
+	if (clean === true) return {
+		urls: true,
+		fragments: true,
+		emptyLinks: true,
+		redundantLinks: true,
+		selfLinkHeadings: true,
+		emptyImages: true,
+		emptyLinkText: true
+	};
+	if (clean === false) return {};
+	return clean;
+}
+/**
+* Try to parse a markdown link `[text](url)` starting at position `start` (the `[`).
+* Returns { text, url, end } or null if not a link.
+* Handles balanced parens in URLs (e.g. `javascript:void(0)`).
+*/
+function parseLink(md, start) {
+	const len = md.length;
+	let j = start + 1;
+	let depth = 1;
+	while (j < len && depth > 0) {
+		const c = md.charCodeAt(j);
+		if (c === 91) depth++;
+		else if (c === 93) depth--;
+		j++;
+	}
+	if (depth !== 0) return null;
+	const textEnd = j - 1;
+	if (j >= len || md.charCodeAt(j) !== 40) return null;
+	j++;
+	const urlStart = j;
+	let parenDepth = 1;
+	while (j < len && parenDepth > 0) {
+		const c = md.charCodeAt(j);
+		if (c === 40) parenDepth++;
+		else if (c === 41) parenDepth--;
+		j++;
+	}
+	if (parenDepth !== 0) return null;
+	const urlEnd = j - 1;
+	return {
+		text: md.slice(start + 1, textEnd),
+		url: md.slice(urlStart, urlEnd),
+		end: j
+	};
+}
+function slugify(text) {
+	let slug = "";
+	let lastWasDash = false;
+	for (let i = 0; i < text.length; i++) {
+		const c = text.charCodeAt(i);
+		if (c >= 97 && c <= 122) {
+			slug += text[i];
+			lastWasDash = false;
+		} else if (c >= 65 && c <= 90) {
+			slug += String.fromCharCode(c + 32);
+			lastWasDash = false;
+		} else if (c >= 48 && c <= 57) {
+			slug += text[i];
+			lastWasDash = false;
+		} else if (c === 95) {
+			slug += "_";
+			lastWasDash = false;
+		} else if (c === 32 || c === 9 || c === 45) {
+			if (!lastWasDash && slug.length > 0) {
+				slug += "-";
+				lastWasDash = true;
+			}
+		}
+	}
+	if (lastWasDash) slug = slug.slice(0, -1);
+	return slug;
+}
+/** Strip inline markdown formatting from heading text for slug generation */
+function stripHeadingFormatting(text) {
+	let result = "";
+	const len = text.length;
+	let i = 0;
+	while (i < len) {
+		const c = text.charCodeAt(i);
+		if (c === 91) {
+			const link = parseLink(text, i);
+			if (link) {
+				result += link.text;
+				i = link.end;
+				continue;
+			}
+		}
+		if (c === 42 || c === 95 || c === 96 || c === 126) {
+			i++;
+			continue;
+		}
+		result += text[i];
+		i++;
+	}
+	return result.trim();
+}
+function collectHeadingSlugs(md) {
+	const slugs = /* @__PURE__ */ new Set();
+	const len = md.length;
+	let i = 0;
+	while (i < len) {
+		if (i === 0 || md.charCodeAt(i - 1) === 10) {
+			let hashes = 0;
+			let j = i;
+			while (j < len && md.charCodeAt(j) === 35) {
+				hashes++;
+				j++;
+			}
+			if (hashes >= 1 && hashes <= 6 && j < len && md.charCodeAt(j) === 32) {
+				j++;
+				const lineEnd = md.indexOf("\n", j);
+				const cleaned = stripHeadingFormatting(lineEnd === -1 ? md.slice(j) : md.slice(j, lineEnd));
+				if (cleaned) slugs.add(slugify(cleaned));
+			}
+		}
+		const nl = md.indexOf("\n", i);
+		if (nl === -1) break;
+		i = nl + 1;
+	}
+	return slugs;
+}
+function cleanFragments(md) {
+	const slugs = collectHeadingSlugs(md);
+	const len = md.length;
+	let result = "";
+	let i = 0;
+	while (i < len) {
+		if (md.charCodeAt(i) === 91) {
+			const link = parseLink(md, i);
+			if (link && link.url.charCodeAt(0) === 35 && link.url.length > 1) {
+				const fragment = link.url.slice(1).split(" ")[0];
+				if (slugs.size > 0 ? !slugs.has(fragment) : true) {
+					result += link.text;
+					i = link.end;
+					continue;
+				}
+			}
+		}
+		result += md[i];
+		i++;
+	}
+	return result;
+}
+function cleanEmptyLinks(md) {
+	const len = md.length;
+	let result = "";
+	let i = 0;
+	while (i < len) {
+		if (md.charCodeAt(i) === 91) {
+			const link = parseLink(md, i);
+			if (link) {
+				const url = link.url;
+				if (url === "#" || url.startsWith("javascript:") || url.startsWith("data:") || url.startsWith("vbscript:")) {
+					result += link.text;
+					i = link.end;
+					continue;
+				}
+			}
+		}
+		result += md[i];
+		i++;
+	}
+	return result;
+}
+function cleanRedundantLinks(md) {
+	const len = md.length;
+	let result = "";
+	let i = 0;
+	while (i < len) {
+		if (md.charCodeAt(i) === 91) {
+			const link = parseLink(md, i);
+			if (link && link.text === link.url) {
+				result += link.text;
+				i = link.end;
+				continue;
+			}
+		}
+		result += md[i];
+		i++;
+	}
+	return result;
+}
+function cleanSelfLinkHeadings(md) {
+	const len = md.length;
+	let result = "";
+	let i = 0;
+	while (i < len) {
+		if (i === 0 || md.charCodeAt(i - 1) === 10) {
+			let hashes = 0;
+			let j = i;
+			while (j < len && md.charCodeAt(j) === 35) {
+				hashes++;
+				j++;
+			}
+			if (hashes >= 1 && hashes <= 6 && j < len && md.charCodeAt(j) === 32) {
+				j++;
+				if (j < len && md.charCodeAt(j) === 91) {
+					const link = parseLink(md, j);
+					if (link && link.url.charCodeAt(0) === 35) {
+						result += md.slice(i, j);
+						result += link.text;
+						i = link.end;
+						continue;
+					}
+				}
+			}
+		}
+		result += md[i];
+		i++;
+	}
+	return result;
+}
+function cleanEmptyImages(md) {
+	const len = md.length;
+	let result = "";
+	let i = 0;
+	while (i < len) {
+		if (md.charCodeAt(i) === 33 && i + 1 < len && md.charCodeAt(i + 1) === 91) {
+			let j = i + 2;
+			const altStart = j;
+			while (j < len && md.charCodeAt(j) !== 93) j++;
+			if (j < len) {
+				if (md.slice(altStart, j).trim().length === 0) {
+					j++;
+					if (j < len && md.charCodeAt(j) === 40) {
+						let parenDepth = 1;
+						j++;
+						while (j < len && parenDepth > 0) {
+							if (md.charCodeAt(j) === 40) parenDepth++;
+							else if (md.charCodeAt(j) === 41) parenDepth--;
+							j++;
+						}
+						i = j;
+						continue;
+					}
+				}
+			}
+		}
+		result += md[i];
+		i++;
+	}
+	return result;
+}
+function cleanEmptyLinkText(md) {
+	const len = md.length;
+	let result = "";
+	let i = 0;
+	while (i < len) {
+		if (md.charCodeAt(i) === 91) {
+			const link = parseLink(md, i);
+			if (link && link.text.trim().length === 0) {
+				i = link.end;
+				continue;
+			}
+		}
+		result += md[i];
+		i++;
+	}
+	return result;
+}
+function applyClean(md, opts) {
+	if (opts.emptyImages) md = cleanEmptyImages(md);
+	if (opts.emptyLinks) md = cleanEmptyLinks(md);
+	if (opts.emptyLinkText) md = cleanEmptyLinkText(md);
+	if (opts.redundantLinks) md = cleanRedundantLinks(md);
+	if (opts.selfLinkHeadings) md = cleanSelfLinkHeadings(md);
+	if (opts.fragments) md = cleanFragments(md);
+	return md;
+}
+//#endregion
+//#region src/stream.ts
+/**
+* Creates a markdown stream from an HTML stream
+* @param htmlStream - ReadableStream of HTML content (as Uint8Array or string)
+* @param options - Configuration options for conversion
+* @param resolvedPlugins - Pre-resolved plugin instances
+* @param tagOverrideHandlers - Tag override handlers from declarative config
+* @returns An async generator yielding markdown chunks
+*/
+async function* streamHtmlToMarkdown$1(htmlStream, options = {}, resolvedPlugins = [], tagOverrideHandlers) {
+	if (!htmlStream) throw new Error("Invalid HTML stream provided");
+	const decoder = new TextDecoder();
+	const reader = htmlStream.getReader();
+	const processor = createMarkdownProcessor(options, resolvedPlugins, tagOverrideHandlers);
+	const parseState = {
+		depthMap: new Uint8Array(1024),
+		depth: 0,
+		resolvedPlugins,
+		tagOverrideHandlers
+	};
+	let remainingHtml = "";
+	try {
+		while (true) {
+			const { done, value } = await reader.read();
+			if (done) break;
+			remainingHtml = parseHtmlStream(`${remainingHtml}${typeof value === "string" ? value : decoder.decode(value, { stream: true })}`, parseState, (event) => {
+				processPluginsForEvent(event, resolvedPlugins, processor.state, processor.processEvent);
+			});
+			const chunk = processor.getMarkdownChunk();
+			if (chunk) yield chunk;
+		}
+		if (remainingHtml) parseHtmlStream(remainingHtml, parseState, (event) => {
+			processPluginsForEvent(event, resolvedPlugins, processor.state, processor.processEvent);
+		});
+		const finalChunk = processor.getMarkdownChunk();
+		if (finalChunk) yield finalChunk;
+	} finally {
+		if (remainingHtml) decoder.decode(new Uint8Array(0), { stream: false });
+		reader.releaseLock();
+	}
+}
+//#endregion
+//#region src/index.ts
+function resolveHooks(options) {
+	return options.hooks?.length ? options.hooks : void 0;
+}
+function convert(html, options, hooks) {
+	const { plugins, callExtractionHandlers, getFrontmatter, frontmatterCallback } = resolvePlugins(options, hooks);
+	const processor = createMarkdownProcessor(options, plugins, options.plugins?.tagOverrides ? buildTagOverrideHandlers(options.plugins.tagOverrides) : void 0);
+	processor.processHtml(html);
+	if (getFrontmatter && frontmatterCallback) {
+		const fm = getFrontmatter();
+		if (fm) frontmatterCallback(fm);
+	}
+	callExtractionHandlers?.();
+	return processor.getMarkdown();
+}
+function htmlToMarkdown(html, options = {}) {
+	const markdown = convert(html, options, resolveHooks(options));
+	if (options.clean) return applyClean(markdown, resolveClean(options.clean));
+	return markdown;
+}
+function streamHtmlToMarkdown(htmlStream, options = {}) {
+	const { plugins } = resolvePlugins(options, resolveHooks(options));
+	return streamHtmlToMarkdown$1(htmlStream, options, plugins, options.plugins?.tagOverrides ? buildTagOverrideHandlers(options.plugins.tagOverrides) : void 0);
+}
+//#endregion
+export { streamHtmlToMarkdown as n, htmlToMarkdown as t };