mdream 0.2.7 → 0.3.0

package/README.md

@@ -22,12 +22,10 @@
 
 - 🧠 Content Extraction: [Readability.js]() scoring heuristics for [~50% fewer tokens*]() and improved accuracy.
 - 🔍 GitHub Flavored Markdown: Frontmatter, Nested & HTML markup support.
-- Tailwind CSS: Converts Tailwind CSS classes to Markdown for better readability.
 
 **Ultra Performant**
 - 🚀 Convert 1.4MB of HTML in [~50ms*]() with advanced streaming support, including content-based buffering.
 - ⚡ 5kB gzip, zero dependencies.
-- Streaming support
 
 **Adaptable**
 
@@ -46,7 +44,7 @@ human readability.
 
 Other LLM specific convertors focus on supporting _all_ document formats, resulting in larger bundles and lower quality Markdown output.
 
-Mdream is an ultra-performant HTML to Markdown converter built specifically for producing high-quality Markdown for LLMs as quickly as possible. It provides
+Mdream is HTML parser + Markdown generator built specifically for producing high-quality Markdown for LLMs as quickly as possible. It provides
 a powerful plugin system to customize the conversion process, allowing you to extract, transform, and filter content as needed.
 
 ## CLI Usage

package/dist/_chunks/index-VTwTBxk0.d.mts

@@ -0,0 +1,58 @@
+import { HTMLToMarkdownOptions, NodeEvent, Plugin } from "./types-BHoibuoP.mjs";
+import { ReadableStream } from "node:stream/web";
+
+//#region src/const.d.ts
+
+declare const TagIdMap: Record<string, number>;
+//#endregion
+//#region src/markdown-processor.d.ts
+
+/**
+ * Creates a markdown processor that consumes DOM events and generates markdown
+ */
+declare function createMarkdownProcessor(options?: HTMLToMarkdownOptions): {
+  processEvent: (event: NodeEvent) => void;
+  processHtml: (html: string) => void;
+  getMarkdown: () => string;
+  getMarkdownChunk: () => string;
+};
+declare const MarkdownProcessor: typeof createMarkdownProcessor;
+//#endregion
+//#region src/parse.d.ts
+interface ParseOptions {
+  plugins?: Plugin[];
+}
+interface ParseResult {
+  events: NodeEvent[];
+  remainingHtml: string;
+}
+/**
+ * Pure HTML parser that emits DOM events
+ * Completely decoupled from markdown generation
+ */
+declare function parseHtml(html: string, options?: ParseOptions): ParseResult;
+/**
+ * Streaming HTML parser - calls onEvent for each DOM event
+ */
+//#endregion
+//#region src/pluggable/plugin.d.ts
+/**
+ * Create a plugin that implements the Plugin interface with improved type inference
+ *
+ * @returns A complete plugin implementation
+ */
+declare function createPlugin<T extends Partial<Plugin>>(plugin: T): Plugin;
+//#endregion
+//#region src/stream.d.ts
+/**
+ * Creates a markdown stream from an HTML stream
+ * @param htmlStream - ReadableStream of HTML content (as Uint8Array or string)
+ * @param options - Configuration options for conversion
+ * @returns An async generator yielding markdown chunks
+ */
+declare function streamHtmlToMarkdown(htmlStream: ReadableStream | null, options?: HTMLToMarkdownOptions): AsyncIterable<string>;
+//#endregion
+//#region src/index.d.ts
+declare function htmlToMarkdown(html: string, options?: HTMLToMarkdownOptions): string;
+//#endregion
+export { MarkdownProcessor, TagIdMap, createPlugin, htmlToMarkdown, parseHtml, streamHtmlToMarkdown };

package/dist/_chunks/plugin-DCJFRZej.mjs

@@ -0,0 +1,299 @@
+//#region src/buffer-region.ts
+/**
+* Creates a new buffer region
+* Returns null if node already has a region assigned
+*/
+function createBufferRegion(node, state, include) {
+	if (node.regionId) return null;
+	const id = state.regionToggles.size + 1;
+	node.regionId = id;
+	state.regionToggles.set(id, include);
+	state.regionContentBuffers.set(id, []);
+	return id;
+}
+/**
+* Collects content for a node into appropriate buffer (optimized)
+*/
+function collectNodeContent(node, content, state) {
+	if (!content) return;
+	const regionId = node.regionId || 0;
+	const targetBuffer = state.regionContentBuffers.get(regionId);
+	if (targetBuffer) {
+		targetBuffer.push(content);
+		state.lastContentCache = content;
+	}
+}
+/**
+* Assembles final content from buffer regions and clears them after use
+* Ensures frontmatter (regionId -1) appears first, followed by other included regions
+*/
+function assembleBufferedContent(state) {
+	const fragments = [];
+	for (const [regionId, content] of Array.from(state.regionContentBuffers.entries())) {
+		const include = state.regionToggles.get(regionId);
+		if (include) fragments.push(...content);
+	}
+	state.regionToggles.clear();
+	state.regionContentBuffers.clear();
+	return fragments.join("").trimStart();
+}
+
+//#endregion
+//#region src/const.ts
+const TAG_HTML = 0;
+const TAG_HEAD = 1;
+const TAG_DETAILS = 2;
+const TAG_SUMMARY = 3;
+const TAG_TITLE = 4;
+const TAG_META = 5;
+const TAG_BR = 6;
+const TAG_H1 = 7;
+const TAG_H2 = 8;
+const TAG_H3 = 9;
+const TAG_H4 = 10;
+const TAG_H5 = 11;
+const TAG_H6 = 12;
+const TAG_HR = 13;
+const TAG_STRONG = 14;
+const TAG_B = 15;
+const TAG_EM = 16;
+const TAG_I = 17;
+const TAG_DEL = 18;
+const TAG_SUB = 19;
+const TAG_SUP = 20;
+const TAG_INS = 21;
+const TAG_BLOCKQUOTE = 22;
+const TAG_CODE = 23;
+const TAG_UL = 24;
+const TAG_LI = 25;
+const TAG_A = 26;
+const TAG_IMG = 27;
+const TAG_TABLE = 28;
+const TAG_THEAD = 29;
+const TAG_TR = 30;
+const TAG_TH = 31;
+const TAG_TD = 32;
+const TAG_OL = 33;
+const TAG_PRE = 34;
+const TAG_P = 35;
+const TAG_DIV = 36;
+const TAG_SPAN = 37;
+const TAG_TBODY = 38;
+const TAG_TFOOT = 39;
+const TAG_FORM = 40;
+const TAG_NAV = 41;
+const TAG_LABEL = 42;
+const TAG_BUTTON = 43;
+const TAG_BODY = 44;
+const TAG_CENTER = 45;
+const TAG_KBD = 46;
+const TAG_FOOTER = 47;
+const TAG_PATH = 48;
+const TAG_SVG = 49;
+const TAG_ARTICLE = 50;
+const TAG_SECTION = 51;
+const TAG_SCRIPT = 52;
+const TAG_STYLE = 53;
+const TAG_LINK = 54;
+const TAG_AREA = 55;
+const TAG_BASE = 56;
+const TAG_COL = 57;
+const TAG_EMBED = 58;
+const TAG_INPUT = 59;
+const TAG_KEYGEN = 60;
+const TAG_PARAM = 61;
+const TAG_SOURCE = 62;
+const TAG_TRACK = 63;
+const TAG_WBR = 64;
+const TAG_SELECT = 65;
+const TAG_TEXTAREA = 66;
+const TAG_OPTION = 67;
+const TAG_FIELDSET = 68;
+const TAG_LEGEND = 69;
+const TAG_AUDIO = 70;
+const TAG_VIDEO = 71;
+const TAG_CANVAS = 72;
+const TAG_IFRAME = 73;
+const TAG_MAP = 74;
+const TAG_DIALOG = 75;
+const TAG_METER = 76;
+const TAG_PROGRESS = 77;
+const TAG_TEMPLATE = 78;
+const TAG_ABBR = 79;
+const TAG_MARK = 80;
+const TAG_Q = 81;
+const TAG_SAMP = 82;
+const TAG_SMALL = 83;
+const TAG_NOSCRIPT = 84;
+const TAG_NOFRAMES = 85;
+const TAG_XMP = 86;
+const TAG_PLAINTEXT = 87;
+const TAG_ASIDE = 88;
+const TAG_U = 89;
+const TAG_CITE = 90;
+const TAG_DFN = 91;
+const TAG_VAR = 92;
+const TAG_TIME = 93;
+const TAG_BDO = 94;
+const TAG_RUBY = 95;
+const TAG_RT = 96;
+const TAG_RP = 97;
+const TAG_DD = 98;
+const TAG_DT = 99;
+const TAG_ADDRESS = 100;
+const TAG_DL = 101;
+const TAG_FIGURE = 102;
+const TAG_OBJECT = 103;
+const TAG_MAIN = 104;
+const TAG_HEADER = 105;
+const TAG_FIGCAPTION = 106;
+const TAG_CAPTION = 107;
+const MAX_TAG_ID = 108;
+const HTML_ENTITIES = {
+	"&": "&",
+	"&lt;": "<",
+	"&gt;": ">",
+	"&quot;": "\"",
+	"&#39;": "'",
+	"&apos;": "'",
+	"&nbsp;": " "
+};
+const ELEMENT_NODE = 1;
+const TEXT_NODE = 2;
+const NodeEventEnter = 0;
+const NodeEventExit = 1;
+const TagIdMap = {
+	html: TAG_HTML,
+	head: TAG_HEAD,
+	details: TAG_DETAILS,
+	summary: TAG_SUMMARY,
+	title: TAG_TITLE,
+	meta: TAG_META,
+	br: TAG_BR,
+	h1: TAG_H1,
+	h2: TAG_H2,
+	h3: TAG_H3,
+	h4: TAG_H4,
+	h5: TAG_H5,
+	h6: TAG_H6,
+	hr: TAG_HR,
+	strong: TAG_STRONG,
+	b: TAG_B,
+	em: TAG_EM,
+	i: TAG_I,
+	del: TAG_DEL,
+	sub: TAG_SUB,
+	sup: TAG_SUP,
+	ins: TAG_INS,
+	blockquote: TAG_BLOCKQUOTE,
+	code: TAG_CODE,
+	ul: TAG_UL,
+	li: TAG_LI,
+	a: TAG_A,
+	img: TAG_IMG,
+	table: TAG_TABLE,
+	thead: TAG_THEAD,
+	tr: TAG_TR,
+	th: TAG_TH,
+	td: TAG_TD,
+	ol: TAG_OL,
+	pre: TAG_PRE,
+	p: TAG_P,
+	div: TAG_DIV,
+	span: TAG_SPAN,
+	tbody: TAG_TBODY,
+	tfoot: TAG_TFOOT,
+	form: TAG_FORM,
+	nav: TAG_NAV,
+	label: TAG_LABEL,
+	button: TAG_BUTTON,
+	body: TAG_BODY,
+	center: TAG_CENTER,
+	kbd: TAG_KBD,
+	footer: TAG_FOOTER,
+	path: TAG_PATH,
+	svg: TAG_SVG,
+	article: TAG_ARTICLE,
+	section: TAG_SECTION,
+	script: TAG_SCRIPT,
+	style: TAG_STYLE,
+	link: TAG_LINK,
+	area: TAG_AREA,
+	base: TAG_BASE,
+	col: TAG_COL,
+	embed: TAG_EMBED,
+	input: TAG_INPUT,
+	keygen: TAG_KEYGEN,
+	param: TAG_PARAM,
+	source: TAG_SOURCE,
+	track: TAG_TRACK,
+	wbr: TAG_WBR,
+	select: TAG_SELECT,
+	textarea: TAG_TEXTAREA,
+	option: TAG_OPTION,
+	fieldset: TAG_FIELDSET,
+	legend: TAG_LEGEND,
+	audio: TAG_AUDIO,
+	video: TAG_VIDEO,
+	canvas: TAG_CANVAS,
+	iframe: TAG_IFRAME,
+	map: TAG_MAP,
+	dialog: TAG_DIALOG,
+	meter: TAG_METER,
+	progress: TAG_PROGRESS,
+	template: TAG_TEMPLATE,
+	abbr: TAG_ABBR,
+	mark: TAG_MARK,
+	q: TAG_Q,
+	samp: TAG_SAMP,
+	small: TAG_SMALL,
+	noscript: TAG_NOSCRIPT,
+	noframes: TAG_NOFRAMES,
+	xmp: TAG_XMP,
+	plaintext: TAG_PLAINTEXT,
+	aside: TAG_ASIDE,
+	u: TAG_U,
+	cite: TAG_CITE,
+	dfn: TAG_DFN,
+	var: TAG_VAR,
+	time: TAG_TIME,
+	bdo: TAG_BDO,
+	ruby: TAG_RUBY,
+	rt: TAG_RT,
+	rp: TAG_RP,
+	dd: TAG_DD,
+	dt: TAG_DT,
+	dl: TAG_DL,
+	address: TAG_ADDRESS,
+	figure: TAG_FIGURE,
+	object: TAG_OBJECT,
+	main: TAG_MAIN,
+	header: TAG_HEADER,
+	figcaption: TAG_FIGCAPTION,
+	caption: TAG_CAPTION
+};
+const MARKDOWN_STRONG = "**";
+const MARKDOWN_EMPHASIS = "_";
+const MARKDOWN_STRIKETHROUGH = "~~";
+const MARKDOWN_CODE_BLOCK = "```";
+const MARKDOWN_INLINE_CODE = "`";
+const MARKDOWN_HORIZONTAL_RULE = "---";
+const NO_SPACING = [0, 0];
+const DEFAULT_BLOCK_SPACING = [2, 2];
+const BLOCKQUOTE_SPACING = [1, 1];
+const LIST_ITEM_SPACING = [1, 0];
+const TABLE_ROW_SPACING = [0, 1];
+
+//#endregion
+//#region src/pluggable/plugin.ts
+/**
+* Create a plugin that implements the Plugin interface with improved type inference
+*
+* @returns A complete plugin implementation
+*/
+function createPlugin(plugin) {
+	return plugin;
+}
+
+//#endregion
+export { BLOCKQUOTE_SPACING, DEFAULT_BLOCK_SPACING, ELEMENT_NODE, HTML_ENTITIES, LIST_ITEM_SPACING, MARKDOWN_CODE_BLOCK, MARKDOWN_EMPHASIS, MARKDOWN_HORIZONTAL_RULE, MARKDOWN_INLINE_CODE, MARKDOWN_STRIKETHROUGH, MARKDOWN_STRONG, MAX_TAG_ID, NO_SPACING, NodeEventEnter, NodeEventExit, TABLE_ROW_SPACING, TAG_A, TAG_ABBR, TAG_ADDRESS, TAG_AREA, TAG_ARTICLE, TAG_ASIDE, TAG_AUDIO, TAG_B, TAG_BASE, TAG_BDO, TAG_BLOCKQUOTE, TAG_BODY, TAG_BR, TAG_BUTTON, TAG_CANVAS, TAG_CAPTION, TAG_CENTER, TAG_CITE, TAG_CODE, TAG_COL, TAG_DD, TAG_DEL, TAG_DETAILS, TAG_DFN, TAG_DIALOG, TAG_DIV, TAG_DL, TAG_DT, TAG_EM, TAG_EMBED, TAG_FIELDSET, TAG_FIGCAPTION, TAG_FIGURE, TAG_FOOTER, TAG_FORM, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TAG_HEAD, TAG_HEADER, TAG_HR, TAG_HTML, TAG_I, TAG_IFRAME, TAG_IMG, TAG_INPUT, TAG_INS, TAG_KBD, TAG_KEYGEN, TAG_LABEL, TAG_LEGEND, TAG_LI, TAG_LINK, TAG_MAIN, TAG_MAP, TAG_MARK, TAG_META, TAG_METER, TAG_NAV, TAG_NOFRAMES, TAG_NOSCRIPT, TAG_OBJECT, TAG_OL, TAG_OPTION, TAG_P, TAG_PARAM, TAG_PLAINTEXT, TAG_PRE, TAG_PROGRESS, TAG_Q, TAG_RP, TAG_RT, TAG_RUBY, TAG_SAMP, TAG_SCRIPT, TAG_SECTION, TAG_SELECT, TAG_SMALL, TAG_SOURCE, TAG_SPAN, TAG_STRONG, TAG_STYLE, TAG_SUB, TAG_SUMMARY, TAG_SUP, TAG_SVG, TAG_TABLE, TAG_TBODY, TAG_TD, TAG_TEMPLATE, TAG_TEXTAREA, TAG_TFOOT, TAG_TH, TAG_THEAD, TAG_TIME, TAG_TITLE, TAG_TR, TAG_TRACK, TAG_U, TAG_UL, TAG_VAR, TAG_VIDEO, TAG_WBR, TAG_XMP, TEXT_NODE, TagIdMap as TagIdMap$1, assembleBufferedContent, collectNodeContent, createBufferRegion, createPlugin as createPlugin$1 };