mdream 0.12.3 → 0.13.1

package/dist/_chunks/{plugin-Bqz9GKOA.mjs → const-BOAJ1T5c.mjs}

@@ -285,15 +285,4 @@ const LIST_ITEM_SPACING = [1, 0];
 const TABLE_ROW_SPACING = [0, 1];
 
 //#endregion
-//#region src/pluggable/plugin.ts
-/**
-* Create a plugin that implements the Plugin interface with improved type inference
-*
-* @returns A complete plugin implementation
-*/
-function createPlugin(plugin) {
-	return plugin;
-}
-
-//#endregion
-export { BLOCKQUOTE_SPACING, DEFAULT_BLOCK_SPACING, ELEMENT_NODE, HTML_ENTITIES, LIST_ITEM_SPACING, MARKDOWN_CODE_BLOCK, MARKDOWN_EMPHASIS, MARKDOWN_HORIZONTAL_RULE, MARKDOWN_INLINE_CODE, MARKDOWN_STRIKETHROUGH, MARKDOWN_STRONG, MAX_TAG_ID, NO_SPACING, NodeEventEnter, NodeEventExit, TABLE_ROW_SPACING, TAG_A, TAG_ABBR, TAG_ADDRESS, TAG_AREA, TAG_ARTICLE, TAG_ASIDE, TAG_AUDIO, TAG_B, TAG_BASE, TAG_BDO, TAG_BLOCKQUOTE, TAG_BODY, TAG_BR, TAG_BUTTON, TAG_CANVAS, TAG_CAPTION, TAG_CENTER, TAG_CITE, TAG_CODE, TAG_COL, TAG_DD, TAG_DEL, TAG_DETAILS, TAG_DFN, TAG_DIALOG, TAG_DIV, TAG_DL, TAG_DT, TAG_EM, TAG_EMBED, TAG_FIELDSET, TAG_FIGCAPTION, TAG_FIGURE, TAG_FOOTER, TAG_FORM, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TAG_HEAD, TAG_HEADER, TAG_HR, TAG_HTML, TAG_I, TAG_IFRAME, TAG_IMG, TAG_INPUT, TAG_INS, TAG_KBD, TAG_KEYGEN, TAG_LABEL, TAG_LEGEND, TAG_LI, TAG_LINK, TAG_MAIN, TAG_MAP, TAG_MARK, TAG_META, TAG_METER, TAG_NAV, TAG_NOFRAMES, TAG_NOSCRIPT, TAG_OBJECT, TAG_OL, TAG_OPTION, TAG_P, TAG_PARAM, TAG_PLAINTEXT, TAG_PRE, TAG_PROGRESS, TAG_Q, TAG_RP, TAG_RT, TAG_RUBY, TAG_SAMP, TAG_SCRIPT, TAG_SECTION, TAG_SELECT, TAG_SMALL, TAG_SOURCE, TAG_SPAN, TAG_STRONG, TAG_STYLE, TAG_SUB, TAG_SUMMARY, TAG_SUP, TAG_SVG, TAG_TABLE, TAG_TBODY, TAG_TD, TAG_TEMPLATE, TAG_TEXTAREA, TAG_TFOOT, TAG_TH, TAG_THEAD, TAG_TIME, TAG_TITLE, TAG_TR, TAG_TRACK, TAG_U, TAG_UL, TAG_VAR, TAG_VIDEO, TAG_WBR, TAG_XMP, TEXT_NODE, TagIdMap, assembleBufferedContent, collectNodeContent, createBufferRegion, createPlugin };
+export { BLOCKQUOTE_SPACING, DEFAULT_BLOCK_SPACING, ELEMENT_NODE, HTML_ENTITIES, LIST_ITEM_SPACING, MARKDOWN_CODE_BLOCK, MARKDOWN_EMPHASIS, MARKDOWN_HORIZONTAL_RULE, MARKDOWN_INLINE_CODE, MARKDOWN_STRIKETHROUGH, MARKDOWN_STRONG, MAX_TAG_ID, NO_SPACING, NodeEventEnter, NodeEventExit, TABLE_ROW_SPACING, TAG_A, TAG_ABBR, TAG_ADDRESS, TAG_AREA, TAG_ARTICLE, TAG_ASIDE, TAG_AUDIO, TAG_B, TAG_BASE, TAG_BDO, TAG_BLOCKQUOTE, TAG_BODY, TAG_BR, TAG_BUTTON, TAG_CANVAS, TAG_CAPTION, TAG_CENTER, TAG_CITE, TAG_CODE, TAG_COL, TAG_DD, TAG_DEL, TAG_DETAILS, TAG_DFN, TAG_DIALOG, TAG_DIV, TAG_DL, TAG_DT, TAG_EM, TAG_EMBED, TAG_FIELDSET, TAG_FIGCAPTION, TAG_FIGURE, TAG_FOOTER, TAG_FORM, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TAG_HEAD, TAG_HEADER, TAG_HR, TAG_HTML, TAG_I, TAG_IFRAME, TAG_IMG, TAG_INPUT, TAG_INS, TAG_KBD, TAG_KEYGEN, TAG_LABEL, TAG_LEGEND, TAG_LI, TAG_LINK, TAG_MAIN, TAG_MAP, TAG_MARK, TAG_META, TAG_METER, TAG_NAV, TAG_NOFRAMES, TAG_NOSCRIPT, TAG_OBJECT, TAG_OL, TAG_OPTION, TAG_P, TAG_PARAM, TAG_PLAINTEXT, TAG_PRE, TAG_PROGRESS, TAG_Q, TAG_RP, TAG_RT, TAG_RUBY, TAG_SAMP, TAG_SCRIPT, TAG_SECTION, TAG_SELECT, TAG_SMALL, TAG_SOURCE, TAG_SPAN, TAG_STRONG, TAG_STYLE, TAG_SUB, TAG_SUMMARY, TAG_SUP, TAG_SVG, TAG_TABLE, TAG_TBODY, TAG_TD, TAG_TEMPLATE, TAG_TEXTAREA, TAG_TFOOT, TAG_TH, TAG_THEAD, TAG_TIME, TAG_TITLE, TAG_TR, TAG_TRACK, TAG_U, TAG_UL, TAG_VAR, TAG_VIDEO, TAG_WBR, TAG_XMP, TEXT_NODE, TagIdMap, assembleBufferedContent, collectNodeContent, createBufferRegion };

package/dist/_chunks/{extraction-BSOWm6fo.mjs → extraction-BPaDGYvv.mjs}

@@ -1,4 +1,4 @@
-import { createPlugin } from "./plugin-Bqz9GKOA.mjs";
+import { createPlugin } from "./plugin-DrovQriD.mjs";
 
 //#region src/libs/query-selector.ts
 /**

package/dist/_chunks/{llms-txt-CQ4yFagU.mjs → llms-txt-XvDQwYbj.mjs}

@@ -1,5 +1,5 @@
-import { htmlToMarkdown } from "./src-DBqiXz8C.mjs";
-import { extractionPlugin } from "./extraction-BSOWm6fo.mjs";
+import { htmlToMarkdown } from "./src-D-NT7shY.mjs";
+import { extractionPlugin } from "./extraction-BPaDGYvv.mjs";
 import { readFile } from "node:fs/promises";
 import { basename, dirname, relative, sep } from "pathe";
 import { glob } from "tinyglobby";

package/dist/_chunks/{src-DBqiXz8C.mjs → markdown-processor-BWVPNlvD.mjs}

@@ -1,4 +1,4 @@
-import { BLOCKQUOTE_SPACING, DEFAULT_BLOCK_SPACING, ELEMENT_NODE, HTML_ENTITIES, LIST_ITEM_SPACING, MARKDOWN_CODE_BLOCK, MARKDOWN_EMPHASIS, MARKDOWN_HORIZONTAL_RULE, MARKDOWN_INLINE_CODE, MARKDOWN_STRIKETHROUGH, MARKDOWN_STRONG, MAX_TAG_ID, NO_SPACING, NodeEventEnter, NodeEventExit, TABLE_ROW_SPACING, TAG_A, TAG_ABBR, TAG_ADDRESS, TAG_AREA, TAG_ASIDE, TAG_AUDIO, TAG_B, TAG_BASE, TAG_BDO, TAG_BLOCKQUOTE, TAG_BODY, TAG_BR, TAG_BUTTON, TAG_CANVAS, TAG_CENTER, TAG_CITE, TAG_CODE, TAG_COL, TAG_DD, TAG_DEL, TAG_DETAILS, TAG_DFN, TAG_DIALOG, TAG_DIV, TAG_DL, TAG_DT, TAG_EM, TAG_EMBED, TAG_FIELDSET, TAG_FOOTER, TAG_FORM, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TAG_HEAD, TAG_HR, TAG_I, TAG_IFRAME, TAG_IMG, TAG_INPUT, TAG_INS, TAG_KBD, TAG_KEYGEN, TAG_LABEL, TAG_LEGEND, TAG_LI, TAG_LINK, TAG_MAP, TAG_MARK, TAG_META, TAG_METER, TAG_NAV, TAG_NOFRAMES, TAG_NOSCRIPT, TAG_OL, TAG_OPTION, TAG_P, TAG_PARAM, TAG_PLAINTEXT, TAG_PRE, TAG_PROGRESS, TAG_Q, TAG_RP, TAG_RT, TAG_RUBY, TAG_SAMP, TAG_SCRIPT, TAG_SELECT, TAG_SMALL, TAG_SOURCE, TAG_SPAN, TAG_STRONG, TAG_STYLE, TAG_SUB, TAG_SUMMARY, TAG_SUP, TAG_SVG, TAG_TABLE, TAG_TBODY, TAG_TD, TAG_TEMPLATE, TAG_TEXTAREA, TAG_TFOOT, TAG_TH, TAG_THEAD, TAG_TIME, TAG_TITLE, TAG_TR, TAG_TRACK, TAG_U, TAG_UL, TAG_VAR, TAG_VIDEO, TAG_WBR, TAG_XMP, TEXT_NODE, TagIdMap, assembleBufferedContent, collectNodeContent } from "./plugin-Bqz9GKOA.mjs";
+import { BLOCKQUOTE_SPACING, DEFAULT_BLOCK_SPACING, ELEMENT_NODE, HTML_ENTITIES, LIST_ITEM_SPACING, MARKDOWN_CODE_BLOCK, MARKDOWN_EMPHASIS, MARKDOWN_HORIZONTAL_RULE, MARKDOWN_INLINE_CODE, MARKDOWN_STRIKETHROUGH, MARKDOWN_STRONG, MAX_TAG_ID, NO_SPACING, NodeEventEnter, NodeEventExit, TABLE_ROW_SPACING, TAG_A, TAG_ABBR, TAG_ADDRESS, TAG_AREA, TAG_ASIDE, TAG_AUDIO, TAG_B, TAG_BASE, TAG_BDO, TAG_BLOCKQUOTE, TAG_BODY, TAG_BR, TAG_BUTTON, TAG_CANVAS, TAG_CENTER, TAG_CITE, TAG_CODE, TAG_COL, TAG_DD, TAG_DEL, TAG_DETAILS, TAG_DFN, TAG_DIALOG, TAG_DIV, TAG_DL, TAG_DT, TAG_EM, TAG_EMBED, TAG_FIELDSET, TAG_FOOTER, TAG_FORM, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TAG_HEAD, TAG_HR, TAG_I, TAG_IFRAME, TAG_IMG, TAG_INPUT, TAG_INS, TAG_KBD, TAG_KEYGEN, TAG_LABEL, TAG_LEGEND, TAG_LI, TAG_LINK, TAG_MAP, TAG_MARK, TAG_META, TAG_METER, TAG_NAV, TAG_NOFRAMES, TAG_NOSCRIPT, TAG_OL, TAG_OPTION, TAG_P, TAG_PARAM, TAG_PLAINTEXT, TAG_PRE, TAG_PROGRESS, TAG_Q, TAG_RP, TAG_RT, TAG_RUBY, TAG_SAMP, TAG_SCRIPT, TAG_SELECT, TAG_SMALL, TAG_SOURCE, TAG_SPAN, TAG_STRONG, TAG_STYLE, TAG_SUB, TAG_SUMMARY, TAG_SUP, TAG_SVG, TAG_TABLE, TAG_TBODY, TAG_TD, TAG_TEMPLATE, TAG_TEXTAREA, TAG_TFOOT, TAG_TH, TAG_THEAD, TAG_TIME, TAG_TITLE, TAG_TR, TAG_TRACK, TAG_U, TAG_UL, TAG_VAR, TAG_VIDEO, TAG_WBR, TAG_XMP, TEXT_NODE, TagIdMap, assembleBufferedContent, collectNodeContent } from "./const-BOAJ1T5c.mjs";
 
 //#region src/tags.ts
 function resolveUrl(url, origin) {
@@ -1386,53 +1386,4 @@ function createMarkdownProcessor(options = {}) {
 const MarkdownProcessor = createMarkdownProcessor;
 
 //#endregion
-//#region src/stream.ts
-/**
-* Creates a markdown stream from an HTML stream
-* @param htmlStream - ReadableStream of HTML content (as Uint8Array or string)
-* @param options - Configuration options for conversion
-* @returns An async generator yielding markdown chunks
-*/
-async function* streamHtmlToMarkdown(htmlStream, options = {}) {
-	if (!htmlStream) throw new Error("Invalid HTML stream provided");
-	const decoder = new TextDecoder();
-	const reader = htmlStream.getReader();
-	const processor = createMarkdownProcessor(options);
-	const parseState = {
-		depthMap: new Uint8Array(1024),
-		depth: 0,
-		plugins: options.plugins || []
-	};
-	let remainingHtml = "";
-	try {
-		while (true) {
-			const { done, value } = await reader.read();
-			if (done) break;
-			const htmlContent = `${remainingHtml}${typeof value === "string" ? value : decoder.decode(value, { stream: true })}`;
-			remainingHtml = parseHtmlStream(htmlContent, parseState, (event) => {
-				processPluginsForEvent(event, options.plugins, processor.state, processor.processEvent);
-			});
-			const chunk = processor.getMarkdownChunk();
-			if (chunk) yield chunk;
-		}
-		if (remainingHtml) parseHtmlStream(remainingHtml, parseState, (event) => {
-			processPluginsForEvent(event, options.plugins, processor.state, processor.processEvent);
-		});
-		const finalChunk = processor.getMarkdownChunk();
-		if (finalChunk) yield finalChunk;
-	} finally {
-		if (remainingHtml) decoder.decode(new Uint8Array(0), { stream: false });
-		reader.releaseLock();
-	}
-}
-
-//#endregion
-//#region src/index.ts
-function htmlToMarkdown(html, options = {}) {
-	const processor = createMarkdownProcessor(options);
-	processor.processHtml(html);
-	return processor.getMarkdown();
-}
-
-//#endregion
-export { MarkdownProcessor, htmlToMarkdown, parseHtml, streamHtmlToMarkdown };
+export { MarkdownProcessor, createMarkdownProcessor, parseHtml, parseHtmlStream, processPluginsForEvent };

package/dist/_chunks/{minimal-DSW9dhXV.mjs → minimal-co1tIZYm.mjs}

@@ -1,5 +1,5 @@
-import { TAG_ASIDE, TAG_BUTTON, TAG_EMBED, TAG_FIELDSET, TAG_FIGURE, TAG_FOOTER, TAG_FORM, TAG_IFRAME, TAG_INPUT, TAG_NAV, TAG_OBJECT, TAG_SELECT, TAG_TEXTAREA } from "./plugin-Bqz9GKOA.mjs";
-import { filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin } from "./plugins-TeB1_RYL.mjs";
+import { TAG_ASIDE, TAG_BUTTON, TAG_EMBED, TAG_FIELDSET, TAG_FIGURE, TAG_FOOTER, TAG_FORM, TAG_IFRAME, TAG_INPUT, TAG_NAV, TAG_OBJECT, TAG_SELECT, TAG_TEXTAREA } from "./const-BOAJ1T5c.mjs";
+import { filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin } from "./plugins-C5_irVJs.mjs";
 
 //#region src/preset/minimal.ts
 /**

package/dist/_chunks/{plugin-BUiqQb0v.d.mts → plugin-CgnpSqtP.d.mts}

@@ -1,4 +1,4 @@
-import { Plugin } from "./types-B94khc0C.mjs";
+import { Plugin } from "./types-DqiI86yW.mjs";
 
 //#region src/pluggable/plugin.d.ts
 

package/dist/_chunks/plugin-DrovQriD.mjs

@@ -0,0 +1,12 @@
+//#region src/pluggable/plugin.ts
+/**
+* Create a plugin that implements the Plugin interface with improved type inference
+*
+* @returns A complete plugin implementation
+*/
+function createPlugin(plugin) {
+	return plugin;
+}
+
+//#endregion
+export { createPlugin };

package/dist/_chunks/{plugins-TeB1_RYL.mjs → plugins-C5_irVJs.mjs}

@@ -1,5 +1,6 @@
-import { ELEMENT_NODE, TAG_A, TAG_ADDRESS, TAG_ARTICLE, TAG_ASIDE, TAG_AUDIO, TAG_B, TAG_BLOCKQUOTE, TAG_BODY, TAG_BR, TAG_BUTTON, TAG_CAPTION, TAG_CODE, TAG_DD, TAG_DETAILS, TAG_DIV, TAG_DL, TAG_DT, TAG_EM, TAG_EMBED, TAG_FIELDSET, TAG_FIGCAPTION, TAG_FIGURE, TAG_FOOTER, TAG_FORM, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TAG_HEAD, TAG_HEADER, TAG_HR, TAG_HTML, TAG_I, TAG_IFRAME, TAG_IMG, TAG_INPUT, TAG_LI, TAG_MAIN, TAG_META, TAG_NAV, TAG_OBJECT, TAG_OL, TAG_P, TAG_PRE, TAG_SCRIPT, TAG_SECTION, TAG_SELECT, TAG_SPAN, TAG_STRONG, TAG_STYLE, TAG_SUMMARY, TAG_SVG, TAG_TABLE, TAG_TBODY, TAG_TD, TAG_TEXTAREA, TAG_TFOOT, TAG_TH, TAG_THEAD, TAG_TITLE, TAG_TR, TAG_UL, TAG_VIDEO, TEXT_NODE, collectNodeContent, createBufferRegion, createPlugin } from "./plugin-Bqz9GKOA.mjs";
-import { parseSelector } from "./extraction-BSOWm6fo.mjs";
+import { ELEMENT_NODE, TAG_A, TAG_ADDRESS, TAG_ARTICLE, TAG_ASIDE, TAG_AUDIO, TAG_B, TAG_BLOCKQUOTE, TAG_BODY, TAG_BR, TAG_BUTTON, TAG_CAPTION, TAG_CODE, TAG_DD, TAG_DETAILS, TAG_DIV, TAG_DL, TAG_DT, TAG_EM, TAG_EMBED, TAG_FIELDSET, TAG_FIGCAPTION, TAG_FIGURE, TAG_FOOTER, TAG_FORM, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TAG_HEAD, TAG_HEADER, TAG_HR, TAG_HTML, TAG_I, TAG_IFRAME, TAG_IMG, TAG_INPUT, TAG_LI, TAG_MAIN, TAG_META, TAG_NAV, TAG_OBJECT, TAG_OL, TAG_P, TAG_PRE, TAG_SCRIPT, TAG_SECTION, TAG_SELECT, TAG_SPAN, TAG_STRONG, TAG_STYLE, TAG_SUMMARY, TAG_SVG, TAG_TABLE, TAG_TBODY, TAG_TD, TAG_TEXTAREA, TAG_TFOOT, TAG_TH, TAG_THEAD, TAG_TITLE, TAG_TR, TAG_UL, TAG_VIDEO, TEXT_NODE, collectNodeContent, createBufferRegion } from "./const-BOAJ1T5c.mjs";
+import { createPlugin } from "./plugin-DrovQriD.mjs";
+import { parseSelector } from "./extraction-BPaDGYvv.mjs";
 
 //#region src/plugins/filter.ts
 /**

package/dist/_chunks/src-D-NT7shY.mjs

@@ -0,0 +1,52 @@
+import { createMarkdownProcessor, parseHtmlStream, processPluginsForEvent } from "./markdown-processor-BWVPNlvD.mjs";
+
+//#region src/stream.ts
+/**
+* Creates a markdown stream from an HTML stream
+* @param htmlStream - ReadableStream of HTML content (as Uint8Array or string)
+* @param options - Configuration options for conversion
+* @returns An async generator yielding markdown chunks
+*/
+async function* streamHtmlToMarkdown(htmlStream, options = {}) {
+	if (!htmlStream) throw new Error("Invalid HTML stream provided");
+	const decoder = new TextDecoder();
+	const reader = htmlStream.getReader();
+	const processor = createMarkdownProcessor(options);
+	const parseState = {
+		depthMap: new Uint8Array(1024),
+		depth: 0,
+		plugins: options.plugins || []
+	};
+	let remainingHtml = "";
+	try {
+		while (true) {
+			const { done, value } = await reader.read();
+			if (done) break;
+			const htmlContent = `${remainingHtml}${typeof value === "string" ? value : decoder.decode(value, { stream: true })}`;
+			remainingHtml = parseHtmlStream(htmlContent, parseState, (event) => {
+				processPluginsForEvent(event, options.plugins, processor.state, processor.processEvent);
+			});
+			const chunk = processor.getMarkdownChunk();
+			if (chunk) yield chunk;
+		}
+		if (remainingHtml) parseHtmlStream(remainingHtml, parseState, (event) => {
+			processPluginsForEvent(event, options.plugins, processor.state, processor.processEvent);
+		});
+		const finalChunk = processor.getMarkdownChunk();
+		if (finalChunk) yield finalChunk;
+	} finally {
+		if (remainingHtml) decoder.decode(new Uint8Array(0), { stream: false });
+		reader.releaseLock();
+	}
+}
+
+//#endregion
+//#region src/index.ts
+function htmlToMarkdown(html, options = {}) {
+	const processor = createMarkdownProcessor(options);
+	processor.processHtml(html);
+	return processor.getMarkdown();
+}
+
+//#endregion
+export { htmlToMarkdown, streamHtmlToMarkdown };

package/dist/_chunks/{types-B94khc0C.d.mts → types-DqiI86yW.d.mts}

@@ -251,5 +251,70 @@ interface PluginContext {
   tailwind?: TailwindContext;
   [key: string]: unknown;
 }
+/**
+ * Markdown chunk with content and metadata
+ * Compatible with LangChain Document structure
+ */
+interface MarkdownChunk {
+  /** The markdown content of the chunk */
+  content: string;
+  /** Metadata extracted during chunking */
+  metadata: {
+    /** Header hierarchy at this chunk position */
+    headers?: Record<string, string>;
+    /** Code block language if chunk is/contains code */
+    code?: string;
+    /** Line number range in original document */
+    loc?: {
+      lines: {
+        from: number;
+        to: number;
+      };
+    };
+  };
+}
+/**
+ * Options for HTML to Markdown chunking
+ * Extends HTMLToMarkdownOptions with chunking-specific settings
+ */
+interface SplitterOptions extends HTMLToMarkdownOptions {
+  /**
+   * Header tag IDs to split on (TAG_H1, TAG_H2, etc.)
+   * @example [TAG_H1, TAG_H2]
+   * @default [TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6]
+   */
+  headersToSplitOn?: number[];
+  /**
+   * Return each line as individual chunk
+   * @default false
+   */
+  returnEachLine?: boolean;
+  /**
+   * Strip headers from chunk content
+   * @default true
+   */
+  stripHeaders?: boolean;
+  /**
+   * Maximum chunk size
+   * @default 1000
+   */
+  chunkSize?: number;
+  /**
+   * Overlap between chunks for context preservation
+   * @default 200
+   */
+  chunkOverlap?: number;
+  /**
+   * Function to measure chunk length (default: character count)
+   * Can be replaced with token counter for LLM applications
+   * @default (text) => text.length
+   */
+  lengthFunction?: (text: string) => number;
+  /**
+   * Keep separators in the split chunks
+   * @default false
+   */
+  keepSeparator?: boolean;
+}
 //#endregion
-export { BufferRegion, ELEMENT_NODE as ELEMENT_NODE$1, ElementNode, type ExtractedElement, HTMLToMarkdownOptions, HandlerContext, MdreamProcessingState, MdreamRuntimeState, Node, NodeEvent, Plugin, PluginContext, PluginCreationOptions, ReadabilityContext, TEXT_NODE as TEXT_NODE$1, TagHandler, TailwindContext, TextNode, extractionPlugin as extractionPlugin$1 };
+export { BufferRegion, ELEMENT_NODE as ELEMENT_NODE$1, ElementNode, type ExtractedElement, HTMLToMarkdownOptions, HandlerContext, MarkdownChunk, MdreamProcessingState, MdreamRuntimeState, Node, NodeEvent, Plugin, PluginContext, PluginCreationOptions, ReadabilityContext, SplitterOptions, TEXT_NODE as TEXT_NODE$1, TagHandler, TailwindContext, TextNode, extractionPlugin as extractionPlugin$1 };

package/dist/cli.mjs

@@ -1,9 +1,11 @@
-import "./_chunks/plugin-Bqz9GKOA.mjs";
-import { streamHtmlToMarkdown } from "./_chunks/src-DBqiXz8C.mjs";
-import "./_chunks/extraction-BSOWm6fo.mjs";
-import { generateLlmsTxtArtifacts } from "./_chunks/llms-txt-CQ4yFagU.mjs";
-import "./_chunks/plugins-TeB1_RYL.mjs";
-import { withMinimalPreset } from "./_chunks/minimal-DSW9dhXV.mjs";
+import "./_chunks/const-BOAJ1T5c.mjs";
+import "./_chunks/markdown-processor-BWVPNlvD.mjs";
+import "./_chunks/plugin-DrovQriD.mjs";
+import { streamHtmlToMarkdown } from "./_chunks/src-D-NT7shY.mjs";
+import "./_chunks/extraction-BPaDGYvv.mjs";
+import { generateLlmsTxtArtifacts } from "./_chunks/llms-txt-XvDQwYbj.mjs";
+import "./_chunks/plugins-C5_irVJs.mjs";
+import { withMinimalPreset } from "./_chunks/minimal-co1tIZYm.mjs";
 import { readFileSync } from "node:fs";
 import { mkdir, writeFile } from "node:fs/promises";
 import { Readable } from "node:stream";

package/dist/index.d.mts

@@ -1,5 +1,5 @@
-import { BufferRegion, ELEMENT_NODE$1 as ELEMENT_NODE, ElementNode, ExtractedElement, HTMLToMarkdownOptions, HandlerContext, MdreamProcessingState, MdreamRuntimeState, Node, NodeEvent, Plugin, PluginContext, PluginCreationOptions, ReadabilityContext, TEXT_NODE$1 as TEXT_NODE, TagHandler, TailwindContext, TextNode } from "./_chunks/types-B94khc0C.mjs";
-import { createPlugin$1 as createPlugin } from "./_chunks/plugin-BUiqQb0v.mjs";
+import { BufferRegion, ELEMENT_NODE$1 as ELEMENT_NODE, ElementNode, ExtractedElement, HTMLToMarkdownOptions, HandlerContext, MarkdownChunk, MdreamProcessingState, MdreamRuntimeState, Node, NodeEvent, Plugin, PluginContext, PluginCreationOptions, ReadabilityContext, SplitterOptions, TEXT_NODE$1 as TEXT_NODE, TagHandler, TailwindContext, TextNode } from "./_chunks/types-DqiI86yW.mjs";
+import { createPlugin$1 as createPlugin } from "./_chunks/plugin-CgnpSqtP.mjs";
 import { ReadableStream } from "node:stream/web";
 
 //#region src/const.d.ts
@@ -181,4 +181,4 @@ declare function streamHtmlToMarkdown(htmlStream: ReadableStream | null, options
 //#region src/index.d.ts
 declare function htmlToMarkdown(html: string, options?: HTMLToMarkdownOptions): string;
 //#endregion
-export { BufferRegion, ELEMENT_NODE, ElementNode, ExtractedElement, HTMLToMarkdownOptions, HandlerContext, MarkdownProcessor, MdreamProcessingState, MdreamRuntimeState, Node, NodeEvent, Plugin, PluginContext, PluginCreationOptions, ReadabilityContext, TEXT_NODE, TagHandler, TagIdMap, TailwindContext, TextNode, createPlugin, htmlToMarkdown, parseHtml, streamHtmlToMarkdown };
+export { BufferRegion, ELEMENT_NODE, ElementNode, ExtractedElement, HTMLToMarkdownOptions, HandlerContext, MarkdownChunk, MarkdownProcessor, MdreamProcessingState, MdreamRuntimeState, Node, NodeEvent, Plugin, PluginContext, PluginCreationOptions, ReadabilityContext, SplitterOptions, TEXT_NODE, TagHandler, TagIdMap, TailwindContext, TextNode, createPlugin, htmlToMarkdown, parseHtml, streamHtmlToMarkdown };

package/dist/index.mjs

@@ -1,4 +1,6 @@
-import { TagIdMap, createPlugin } from "./_chunks/plugin-Bqz9GKOA.mjs";
-import { MarkdownProcessor, htmlToMarkdown, parseHtml, streamHtmlToMarkdown } from "./_chunks/src-DBqiXz8C.mjs";
+import { TagIdMap } from "./_chunks/const-BOAJ1T5c.mjs";
+import { MarkdownProcessor, parseHtml } from "./_chunks/markdown-processor-BWVPNlvD.mjs";
+import { createPlugin } from "./_chunks/plugin-DrovQriD.mjs";
+import { htmlToMarkdown, streamHtmlToMarkdown } from "./_chunks/src-D-NT7shY.mjs";
 
 export { MarkdownProcessor, TagIdMap, createPlugin, htmlToMarkdown, parseHtml, streamHtmlToMarkdown };

package/dist/llms-txt.mjs

@@ -1,6 +1,8 @@
-import "./_chunks/plugin-Bqz9GKOA.mjs";
-import "./_chunks/src-DBqiXz8C.mjs";
-import "./_chunks/extraction-BSOWm6fo.mjs";
-import { generateLlmsTxtArtifacts } from "./_chunks/llms-txt-CQ4yFagU.mjs";
+import "./_chunks/const-BOAJ1T5c.mjs";
+import "./_chunks/markdown-processor-BWVPNlvD.mjs";
+import "./_chunks/plugin-DrovQriD.mjs";
+import "./_chunks/src-D-NT7shY.mjs";
+import "./_chunks/extraction-BPaDGYvv.mjs";
+import { generateLlmsTxtArtifacts } from "./_chunks/llms-txt-XvDQwYbj.mjs";
 
 export { generateLlmsTxtArtifacts };

package/dist/plugins.d.mts

@@ -1,5 +1,5 @@
-import { Plugin, extractionPlugin$1 as extractionPlugin } from "./_chunks/types-B94khc0C.mjs";
-import { createPlugin$1 as createPlugin } from "./_chunks/plugin-BUiqQb0v.mjs";
+import { Plugin, extractionPlugin$1 as extractionPlugin } from "./_chunks/types-DqiI86yW.mjs";
+import { createPlugin$1 as createPlugin } from "./_chunks/plugin-CgnpSqtP.mjs";
 
 //#region src/plugins/filter.d.ts
 

package/dist/plugins.mjs

@@ -1,5 +1,6 @@
-import { createPlugin } from "./_chunks/plugin-Bqz9GKOA.mjs";
-import { extractionPlugin } from "./_chunks/extraction-BSOWm6fo.mjs";
-import { filterPlugin, frontmatterPlugin, isolateMainPlugin, readabilityPlugin, tailwindPlugin } from "./_chunks/plugins-TeB1_RYL.mjs";
+import "./_chunks/const-BOAJ1T5c.mjs";
+import { createPlugin } from "./_chunks/plugin-DrovQriD.mjs";
+import { extractionPlugin } from "./_chunks/extraction-BPaDGYvv.mjs";
+import { filterPlugin, frontmatterPlugin, isolateMainPlugin, readabilityPlugin, tailwindPlugin } from "./_chunks/plugins-C5_irVJs.mjs";
 
 export { createPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, readabilityPlugin, tailwindPlugin };

package/dist/preset/minimal.d.mts

@@ -1,4 +1,4 @@
-import { HTMLToMarkdownOptions } from "../_chunks/types-B94khc0C.mjs";
+import { HTMLToMarkdownOptions } from "../_chunks/types-DqiI86yW.mjs";
 
 //#region src/preset/minimal.d.ts
 

package/dist/preset/minimal.mjs

@@ -1,6 +1,7 @@
-import "../_chunks/plugin-Bqz9GKOA.mjs";
-import "../_chunks/extraction-BSOWm6fo.mjs";
-import "../_chunks/plugins-TeB1_RYL.mjs";
-import { withMinimalPreset } from "../_chunks/minimal-DSW9dhXV.mjs";
+import "../_chunks/const-BOAJ1T5c.mjs";
+import "../_chunks/plugin-DrovQriD.mjs";
+import "../_chunks/extraction-BPaDGYvv.mjs";
+import "../_chunks/plugins-C5_irVJs.mjs";
+import { withMinimalPreset } from "../_chunks/minimal-co1tIZYm.mjs";
 
 export { withMinimalPreset };

package/dist/splitter.d.mts

@@ -0,0 +1,11 @@
+import { MarkdownChunk, SplitterOptions } from "./_chunks/types-DqiI86yW.mjs";
+
+//#region src/splitter.d.ts
+
+/**
+ * Convert HTML to Markdown and split into chunks in single pass
+ * Chunks are created during HTML event processing
+ */
+declare function htmlToMarkdownSplitChunks(html: string, options?: SplitterOptions): MarkdownChunk[];
+//#endregion
+export { type MarkdownChunk, type SplitterOptions, htmlToMarkdownSplitChunks };

package/dist/splitter.mjs

@@ -0,0 +1,202 @@
+import { ELEMENT_NODE, NodeEventEnter, NodeEventExit, TAG_CODE, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TAG_HR, TAG_PRE, TEXT_NODE } from "./_chunks/const-BOAJ1T5c.mjs";
+import { createMarkdownProcessor, parseHtmlStream, processPluginsForEvent } from "./_chunks/markdown-processor-BWVPNlvD.mjs";
+
+//#region src/splitter.ts
+const DEFAULT_HEADERS_TO_SPLIT_ON = [
+	TAG_H2,
+	TAG_H3,
+	TAG_H4,
+	TAG_H5,
+	TAG_H6
+];
+function createOptions(options) {
+	return {
+		headersToSplitOn: options.headersToSplitOn ?? DEFAULT_HEADERS_TO_SPLIT_ON,
+		returnEachLine: options.returnEachLine ?? false,
+		stripHeaders: options.stripHeaders ?? true,
+		chunkSize: options.chunkSize ?? 1e3,
+		chunkOverlap: options.chunkOverlap ?? 200,
+		lengthFunction: options.lengthFunction ?? ((text) => text.length),
+		keepSeparator: options.keepSeparator ?? false,
+		origin: options.origin,
+		plugins: options.plugins ?? []
+	};
+}
+function getCodeLanguage(node) {
+	const className = node.attributes?.class;
+	if (!className) return "";
+	const langParts = className.split(" ").map((c) => c.split("language-")[1]).filter(Boolean);
+	return langParts.length > 0 ? langParts[0].trim() : "";
+}
+function shouldSplitOnHeader(tagId, options) {
+	return options.headersToSplitOn.includes(tagId);
+}
+/**
+* Get current markdown content WITHOUT clearing buffers
+*/
+function getCurrentMarkdown(state) {
+	const fragments = [];
+	for (const [regionId, content] of state.regionContentBuffers.entries()) {
+		const include = state.regionToggles.get(regionId);
+		if (include) fragments.push(...content);
+	}
+	return fragments.join("").trimStart();
+}
+/**
+* Convert HTML to Markdown and split into chunks in single pass
+* Chunks are created during HTML event processing
+*/
+function htmlToMarkdownSplitChunks(html, options = {}) {
+	const opts = createOptions(options);
+	if (opts.chunkOverlap >= opts.chunkSize) throw new Error("chunkOverlap must be less than chunkSize");
+	const processor = createMarkdownProcessor({
+		origin: opts.origin,
+		plugins: opts.plugins
+	});
+	const chunks = [];
+	const headerHierarchy = new Map();
+	const seenSplitHeaders = new Set();
+	let currentChunkCodeLanguage = "";
+	let collectingHeaderText = false;
+	let currentHeaderTagId = null;
+	let currentHeaderText = "";
+	let lineNumber = 1;
+	let lastChunkEndPosition = 0;
+	let lastSplitPosition = 0;
+	function flushChunk(endPosition) {
+		const currentMd = getCurrentMarkdown(processor.state);
+		const chunkEnd = endPosition ?? currentMd.length;
+		const chunkContent = currentMd.slice(lastChunkEndPosition, chunkEnd);
+		if (!chunkContent.trim()) {
+			lastChunkEndPosition = chunkEnd;
+			return;
+		}
+		const chunk = {
+			content: chunkContent.trimEnd(),
+			metadata: { loc: { lines: {
+				from: lineNumber,
+				to: lineNumber + (chunkContent.match(/\n/g) || []).length
+			} } }
+		};
+		if (headerHierarchy.size > 0) {
+			chunk.metadata.headers = {};
+			for (const [tagId, text] of headerHierarchy.entries()) {
+				const level = `h${tagId - TAG_H1 + 1}`;
+				chunk.metadata.headers[level] = text;
+			}
+		}
+		if (currentChunkCodeLanguage) chunk.metadata.code = currentChunkCodeLanguage;
+		chunks.push(chunk);
+		currentChunkCodeLanguage = "";
+		lastSplitPosition = chunkEnd;
+		if (opts.chunkOverlap > 0) {
+			const maxOverlap = Math.max(0, chunkContent.length - 1);
+			const actualOverlap = Math.min(opts.chunkOverlap, maxOverlap);
+			lastChunkEndPosition = chunkEnd - actualOverlap;
+		} else lastChunkEndPosition = chunkEnd;
+		lineNumber += (chunkContent.match(/\n/g) || []).length;
+	}
+	const parseState = {
+		depthMap: processor.state.depthMap,
+		depth: 0,
+		plugins: opts.plugins
+	};
+	parseHtmlStream(html, parseState, (event) => {
+		const { type: eventType, node } = event;
+		if (node.type === ELEMENT_NODE) {
+			const element = node;
+			const tagId = element.tagId;
+			if (tagId && tagId >= TAG_H1 && tagId <= TAG_H6) {
+				if (eventType === NodeEventEnter) {
+					collectingHeaderText = true;
+					currentHeaderTagId = tagId;
+					currentHeaderText = "";
+					if (shouldSplitOnHeader(tagId, opts)) {
+						if (seenSplitHeaders.has(tagId)) {
+							flushChunk();
+							for (let i = tagId; i <= TAG_H6; i++) headerHierarchy.delete(i);
+						}
+						seenSplitHeaders.add(tagId);
+					}
+				} else if (eventType === NodeEventExit && currentHeaderTagId === tagId) {
+					headerHierarchy.set(tagId, currentHeaderText.trim());
+					collectingHeaderText = false;
+					currentHeaderTagId = null;
+				}
+			}
+			if (tagId === TAG_CODE && element.depthMap[TAG_PRE] > 0) {
+				if (eventType === NodeEventEnter) {
+					const lang = getCodeLanguage(element);
+					if (lang && !currentChunkCodeLanguage) currentChunkCodeLanguage = lang;
+				}
+			}
+			if (tagId === TAG_HR && eventType === NodeEventEnter) flushChunk();
+		}
+		if (collectingHeaderText && node.type === TEXT_NODE) {
+			const textNode = node;
+			currentHeaderText += textNode.value;
+		}
+		processPluginsForEvent(event, opts.plugins, processor.state, processor.processEvent);
+		if (!opts.returnEachLine) {
+			const currentMd = getCurrentMarkdown(processor.state);
+			const currentChunkSize = opts.lengthFunction(currentMd.slice(lastChunkEndPosition));
+			if (currentChunkSize > opts.chunkSize) {
+				const idealSplitPos = lastChunkEndPosition + opts.chunkSize;
+				const separators = [
+					"\n\n",
+					"```\n",
+					"\n",
+					" "
+				];
+				let splitPosition = -1;
+				for (const sep of separators) {
+					const idx = currentMd.lastIndexOf(sep, idealSplitPos);
+					const candidateSplitPos = idx + sep.length;
+					if (idx >= 0) {
+						const beforeSplit = currentMd.slice(0, candidateSplitPos);
+						let backtickCount = 0;
+						let pos = 0;
+						while ((pos = beforeSplit.indexOf("```", pos)) !== -1) {
+							backtickCount++;
+							pos += 3;
+						}
+						if (backtickCount % 2 === 1) continue;
+					}
+					if (idx >= 0 && candidateSplitPos > lastSplitPosition) {
+						splitPosition = candidateSplitPos;
+						break;
+					}
+				}
+				if (splitPosition === -1 || splitPosition <= lastChunkEndPosition) splitPosition = currentMd.length;
+				flushChunk(splitPosition);
+			}
+		}
+	});
+	flushChunk();
+	if (opts.returnEachLine && chunks.length > 0) {
+		const lineChunks = [];
+		for (const chunk of chunks) {
+			const lines = chunk.content.split("\n");
+			const chunkStartLine = chunk.metadata.loc?.lines.from || 1;
+			for (let i = 0; i < lines.length; i++) {
+				const line = lines[i];
+				if (line.trim()) lineChunks.push({
+					content: line,
+					metadata: {
+						...chunk.metadata,
+						loc: { lines: {
+							from: chunkStartLine + i,
+							to: chunkStartLine + i
+						} }
+					}
+				});
+			}
+		}
+		return lineChunks;
+	}
+	if (opts.stripHeaders) for (const chunk of chunks) chunk.content = chunk.content.split("\n").filter((line) => !line.match(/^#{1,6}\s+/)).join("\n").trim();
+	return chunks.filter((chunk) => chunk.content.length > 0);
+}
+
+//#endregion
+export { htmlToMarkdownSplitChunks };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "mdream",
   "type": "module",
-  "version": "0.12.3",
+  "version": "0.13.1",
   "description": "Ultra-performant HTML to Markdown Convertor Optimized for LLMs and llm.txt artifacts.",
   "author": {
     "name": "Harlan Wilton",
@@ -54,6 +54,14 @@
         "default": "./dist/preset/minimal.mjs"
       },
       "default": "./dist/preset/minimal.mjs"
+    },
+    "./splitter": {
+      "types": "./dist/splitter.d.mts",
+      "import": {
+        "types": "./dist/splitter.d.mts",
+        "default": "./dist/splitter.mjs"
+      },
+      "default": "./dist/splitter.mjs"
     }
   },
   "main": "./dist/index.mjs",