@mdream/js 0.17.0

package/dist/splitter.mjs

@@ -0,0 +1,215 @@
+import "./_chunks/const.mjs";
+import { r as parseHtmlStream } from "./_chunks/parse.mjs";
+import { n as createMarkdownProcessor, r as processPluginsForEvent, t as resolvePlugins } from "./_chunks/resolve-plugins.mjs";
+import "./_chunks/plugins.mjs";
+//#region src/splitter.ts
+const MARKDOWN_HEADER_LINE_RE = /^#{1,6}\s+/;
+const NEWLINE_RE = /\n/g;
+const DEFAULT_HEADERS_TO_SPLIT_ON = [
+	8,
+	9,
+	10,
+	11,
+	12
+];
+function createOptions(options) {
+	return {
+		headersToSplitOn: options.headersToSplitOn ?? DEFAULT_HEADERS_TO_SPLIT_ON,
+		returnEachLine: options.returnEachLine ?? false,
+		stripHeaders: options.stripHeaders ?? true,
+		chunkSize: options.chunkSize ?? 1e3,
+		chunkOverlap: options.chunkOverlap ?? 200,
+		lengthFunction: options.lengthFunction ?? ((text) => text.length),
+		keepSeparator: options.keepSeparator ?? false,
+		resolvedPlugins: resolvePlugins(options).plugins
+	};
+}
+function getCodeLanguage(node) {
+	const className = node.attributes?.class;
+	if (!className) return "";
+	const langParts = className.split(" ").map((c) => c.split("language-")[1]).filter(Boolean);
+	return langParts && langParts.length > 0 ? langParts[0].trim() : "";
+}
+function shouldSplitOnHeader(tagId, options) {
+	return options.headersToSplitOn.includes(tagId);
+}
+/**
+* Get current markdown content WITHOUT clearing buffers
+*/
+function getCurrentMarkdown(state) {
+	return state.buffer.join("").trimStart();
+}
+/**
+* Convert HTML to Markdown and split into chunks in single pass.
+* Yields chunks during HTML event processing for better memory efficiency.
+*
+* **JavaScript engine only** — uses the JS engine's internal processing pipeline.
+* Not compatible with the Rust engine.
+*/
+function* htmlToMarkdownSplitChunksStream(html, options = {}) {
+	const opts = createOptions(options);
+	if (opts.chunkOverlap >= opts.chunkSize) throw new Error("chunkOverlap must be less than chunkSize");
+	const processor = createMarkdownProcessor(options, opts.resolvedPlugins);
+	const headerHierarchy = /* @__PURE__ */ new Map();
+	const seenSplitHeaders = /* @__PURE__ */ new Set();
+	let currentChunkCodeLanguage = "";
+	let collectingHeaderText = false;
+	let currentHeaderTagId = null;
+	let currentHeaderText = "";
+	let lineNumber = 1;
+	let lastChunkEndPosition = 0;
+	let lastSplitPosition = 0;
+	function* flushChunk(endPosition, applyOverlap = false) {
+		const currentMd = getCurrentMarkdown(processor.state);
+		const chunkEnd = endPosition ?? currentMd.length;
+		const originalChunkContent = currentMd.slice(lastChunkEndPosition, chunkEnd);
+		if (!originalChunkContent.trim()) {
+			lastChunkEndPosition = chunkEnd;
+			return;
+		}
+		let chunkContent = originalChunkContent;
+		if (opts.stripHeaders) {
+			chunkContent = chunkContent.split("\n").filter((line) => !MARKDOWN_HEADER_LINE_RE.test(line)).join("\n").trim();
+			if (!chunkContent) {
+				lastChunkEndPosition = chunkEnd;
+				return;
+			}
+		}
+		const chunk = {
+			content: chunkContent.trimEnd(),
+			metadata: { loc: { lines: {
+				from: lineNumber,
+				to: lineNumber + (originalChunkContent.match(NEWLINE_RE) || []).length
+			} } }
+		};
+		if (headerHierarchy.size > 0) {
+			chunk.metadata.headers = {};
+			for (const [tagId, text] of headerHierarchy.entries()) {
+				const level = `h${tagId - 7 + 1}`;
+				chunk.metadata.headers[level] = text;
+			}
+		}
+		if (currentChunkCodeLanguage) chunk.metadata.code = currentChunkCodeLanguage;
+		yield chunk;
+		currentChunkCodeLanguage = "";
+		lastSplitPosition = chunkEnd;
+		if (applyOverlap && opts.chunkOverlap > 0) {
+			const maxOverlap = Math.max(0, originalChunkContent.length - 1);
+			lastChunkEndPosition = chunkEnd - Math.min(opts.chunkOverlap, maxOverlap);
+		} else lastChunkEndPosition = chunkEnd;
+		lineNumber += (originalChunkContent.match(NEWLINE_RE) || []).length;
+	}
+	const parseState = {
+		depthMap: processor.state.depthMap,
+		depth: 0,
+		resolvedPlugins: opts.resolvedPlugins
+	};
+	const eventBuffer = [];
+	parseHtmlStream(html, parseState, (event) => {
+		eventBuffer.push(event);
+	});
+	for (const event of eventBuffer) {
+		const { type: eventType, node } = event;
+		if (node.type === 1) {
+			const element = node;
+			const tagId = element.tagId;
+			if (tagId && tagId >= 7 && tagId <= 12) {
+				if (eventType === 0) {
+					collectingHeaderText = true;
+					currentHeaderTagId = tagId;
+					currentHeaderText = "";
+					if (shouldSplitOnHeader(tagId, opts)) {
+						if (seenSplitHeaders.has(tagId)) {
+							yield* flushChunk();
+							for (let i = tagId; i <= 12; i++) headerHierarchy.delete(i);
+						}
+						seenSplitHeaders.add(tagId);
+					}
+				} else if (eventType === 1 && currentHeaderTagId === tagId) {
+					headerHierarchy.set(tagId, currentHeaderText.trim());
+					collectingHeaderText = false;
+					currentHeaderTagId = null;
+				}
+			}
+			if (tagId === 23 && (element.depthMap[34] || 0) > 0) {
+				if (eventType === 0) {
+					const lang = getCodeLanguage(element);
+					if (lang && !currentChunkCodeLanguage) currentChunkCodeLanguage = lang;
+				}
+			}
+			if (tagId === 13 && eventType === 0) yield* flushChunk();
+		}
+		if (collectingHeaderText && node.type === 2) currentHeaderText += node.value;
+		processPluginsForEvent(event, opts.resolvedPlugins, processor.state, processor.processEvent);
+		if (!opts.returnEachLine) {
+			const currentMd = getCurrentMarkdown(processor.state);
+			if (opts.lengthFunction(currentMd.slice(lastChunkEndPosition)) > opts.chunkSize) {
+				const idealSplitPos = lastChunkEndPosition + opts.chunkSize;
+				const separators = [
+					"\n\n",
+					"```\n",
+					"\n",
+					" "
+				];
+				let splitPosition = -1;
+				for (const sep of separators) {
+					const idx = currentMd.lastIndexOf(sep, idealSplitPos);
+					const candidateSplitPos = idx + sep.length;
+					if (idx >= 0) {
+						const beforeSplit = currentMd.slice(0, candidateSplitPos);
+						let backtickCount = 0;
+						let pos = beforeSplit.indexOf("```", 0);
+						while (pos !== -1) {
+							backtickCount++;
+							pos = beforeSplit.indexOf("```", pos + 3);
+						}
+						if (backtickCount % 2 === 1) continue;
+					}
+					if (idx >= 0 && candidateSplitPos > lastSplitPosition) {
+						splitPosition = candidateSplitPos;
+						break;
+					}
+				}
+				if (splitPosition === -1 || splitPosition <= lastChunkEndPosition) splitPosition = currentMd.length;
+				yield* flushChunk(splitPosition, true);
+			}
+		}
+	}
+	yield* flushChunk();
+}
+/**
+* Convert HTML to Markdown and split into chunks in single pass.
+* Chunks are created during HTML event processing.
+*
+* **JavaScript engine only** — uses the JS engine's internal processing pipeline.
+* Not compatible with the Rust engine.
+*/
+function htmlToMarkdownSplitChunks(html, options = {}) {
+	const opts = createOptions(options);
+	const chunks = [];
+	for (const chunk of htmlToMarkdownSplitChunksStream(html, options)) chunks.push(chunk);
+	if (opts.returnEachLine && chunks.length > 0) {
+		const lineChunks = [];
+		for (const chunk of chunks) {
+			const lines = chunk.content.split("\n");
+			const chunkStartLine = chunk.metadata.loc?.lines.from || 1;
+			for (let i = 0; i < lines.length; i++) {
+				const line = lines[i];
+				if (line && line.trim()) lineChunks.push({
+					content: line,
+					metadata: {
+						...chunk.metadata,
+						loc: { lines: {
+							from: chunkStartLine + i,
+							to: chunkStartLine + i
+						} }
+					}
+				});
+			}
+		}
+		return lineChunks;
+	}
+	return chunks;
+}
+//#endregion
+export { htmlToMarkdownSplitChunks, htmlToMarkdownSplitChunksStream };

package/package.json

@@ -0,0 +1,93 @@
+{
+  "name": "@mdream/js",
+  "type": "module",
+  "version": "0.17.0",
+  "description": "JavaScript HTML-to-Markdown engine for mdream. Escape hatch for hooks and edge runtimes.",
+  "author": {
+    "name": "Harlan Wilton",
+    "email": "harlan@harlanzw.com",
+    "url": "https://harlanzw.com/"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/harlan-zw/mdream",
+    "directory": "packages/js"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": {
+        "types": "./dist/index.d.mts",
+        "default": "./dist/index.mjs"
+      },
+      "default": "./dist/index.mjs"
+    },
+    "./negotiate": {
+      "types": "./dist/negotiate.d.mts",
+      "import": {
+        "types": "./dist/negotiate.d.mts",
+        "default": "./dist/negotiate.mjs"
+      },
+      "default": "./dist/negotiate.mjs"
+    },
+    "./plugins": {
+      "types": "./dist/plugins.d.mts",
+      "import": {
+        "types": "./dist/plugins.d.mts",
+        "default": "./dist/plugins.mjs"
+      },
+      "default": "./dist/plugins.mjs"
+    },
+    "./preset/minimal": {
+      "types": "./dist/preset/minimal.d.mts",
+      "import": {
+        "types": "./dist/preset/minimal.d.mts",
+        "default": "./dist/preset/minimal.mjs"
+      },
+      "default": "./dist/preset/minimal.mjs"
+    },
+    "./splitter": {
+      "types": "./dist/splitter.d.mts",
+      "import": {
+        "types": "./dist/splitter.d.mts",
+        "default": "./dist/splitter.mjs"
+      },
+      "default": "./dist/splitter.mjs"
+    },
+    "./parse": {
+      "types": "./dist/parse.d.mts",
+      "import": {
+        "types": "./dist/parse.d.mts",
+        "default": "./dist/parse.mjs"
+      },
+      "default": "./dist/parse.mjs"
+    },
+    "./llms-txt": {
+      "types": "./dist/llms-txt.d.mts",
+      "import": {
+        "types": "./dist/llms-txt.d.mts",
+        "default": "./dist/llms-txt.mjs"
+      },
+      "default": "./dist/llms-txt.mjs"
+    }
+  },
+  "main": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "bin": {
+    "mdream-js": "./bin/mdream.mjs"
+  },
+  "files": [
+    "bin",
+    "dist"
+  ],
+  "dependencies": {
+    "cac": "^7.0.0",
+    "pathe": "^2.0.3"
+  },
+  "scripts": {
+    "build": "obuild",
+    "typecheck": "tsc --noEmit",
+    "dev:prepare": "obuild --stub"
+  }
+}