mdream 0.1.1

package/LICENSE.md

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2025 Harlan Wilton
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,185 @@
+<h1>mdream</h1>
+
+[![npm version](https://img.shields.io/npm/v/mdream?color=yellow)](https://npmjs.com/package/mdream)
+[![npm downloads](https://img.shields.io/npm/dm/mdream?color=yellow)](https://npm.chart.dev/mdream)
+[![license](https://img.shields.io/github/license/harlan-zw/mdream?color=yellow)](https://github.com/harlan-zw/mdream/blob/main/LICENSE.md)
+
+> Ultra-performant JavaScript HTML to Markdown converter optimized for LLMs.
+
+<img src=".github/logo.png" alt="mdream logo" width="200">
+
+<p align="center">
+<table>
+<tbody>
+<td align="center">
+<sub>Made possible by my <a href="https://github.com/sponsors/harlan-zw">Sponsor Program 💖</a><br> Follow me <a href="https://twitter.com/harlan_zw">@harlan_zw</a> 🐦 • Join <a href="https://discord.gg/275MBUBvgP">Discord</a> for help</sub><br>
+</td>
+</tbody>
+</table>
+</p>
+
+## Features
+
+- 🧠 Content Extraction: [Readability.js]() scoring heuristics for [~50% fewer tokens*]() and improved accuracy.
+- 🔍 GitHub Flavored Markdown: Frontmatter, Nested & HTML markup support.
+- Tailwind CSS: Converts Tailwind CSS classes to Markdown for better readability.
+
+**Ultra Performant**
+- 🚀 Convert 1.4MB of HTML in [~50ms*]() with advanced streaming support, including content-based buffering.
+- ⚡ 5kB gzip, zero dependencies.
+- Streaming support
+
+**Adaptable**
+
+- ⚙️ Run anywhere: CLI, edge workers, browsers, Node, etc.
+- 🔌 Extensible: [Plugin system](#plugin-system) for customizing and extending functionality.
+
+**CLI**
+
+- integrates with crawlee to provide entire site markdown dumps
+- Run a MCP web server
+
+## Why Mdream?
+
+Traditional HTML to Markdown converters were not built for LLMs or humans. They tend to be slow and bloated and produce output that's poorly suited for LLMs token usage or for
+human readability.
+
+Mdream is an ultra-performant HTML to Markdown converter built specifically for LLM Content Analysis & Human Readability. With zero dependencies, streaming built-in and opinionated output optimized for both human readability and AI processing.
+
+Perfect for: RAG systems, web scraping, content extraction, ChatGPT/Claude integration, and large-scale document processing.
+
+## CLI Usage
+
+The Mdream CLI is designed to work exclusively with Unix pipes, providing flexibility and freedom to integrate with other tools.
+
+**Pipe Site to Markdown**
+
+Fetches the [Markdown Wikipedia page](https://en.wikipedia.org/wiki/Markdown) and converts it to Markdown preserving the original links and images.
+
+```bash
+curl -s https://en.wikipedia.org/wiki/Markdown \
+ | npx mdream --origin https://en.wikipedia.org --filters minimal-from-first-header \
+  | tee streaming.md
+```
+
+_Tip: The `--origin` flag will fix relative image and link paths_
+
+**Local File to Markdown**
+
+Converts a local HTML file to a Markdown file, using `tee` to write the output to a file and display it in the terminal.
+
+```bash
+cat index.html \
+ | npx mdream \
+  | tee streaming.md
+```
+
+### CLI Options
+
+- `--origin <url>`: Base URL for resolving relative links and images
+- `-v, --verbose`: Enable verbose debug logging to stderr
+- `--help`: Display help information
+- `--version`: Display version information
+
+## API Usage
+
+### Installation
+
+```bash
+# npm
+npm install mdream
+
+# yarn
+yarn add mdream
+
+# pnpm
+pnpm add mdream
+```
+
+### Usage
+
+Mdream provides two utils for working with HTML, both will process content as a stream.
+- `syncHtmlToMarkdown`: Useful if you already have the entire HTML payload you want to convert.
+- `streamHtmlToMarkdown`: Best practice if you are fetching or reading from a local file.
+
+**Convert existing HTML**
+
+```ts
+import { syncHtmlToMarkdown } from 'mdream'
+
+// Simple conversion
+const markdown = syncHtmlToMarkdown('<h1>Hello World</h1>')
+console.log(markdown) // # Hello World
+````
+
+**Convert from Fetch**
+
+```ts
+import { streamHtmlToMarkdown } from 'mdream'
+
+// Using fetch with streaming
+const response = await fetch('https://example.com')
+const htmlStream = response.body
+const markdownGenerator = streamHtmlToMarkdown(htmlStream, {
+  origin: 'https://example.com',
+  filters: 'minimal-from-first-header'
+})
+
+// Process chunks as they arrive
+for await (const chunk of markdownGenerator) {
+  console.log(chunk)
+}
+```
+
+## Documentation
+
+### Plugin System
+
+Mdream now features a powerful plugin system that allows you to customize and extend the HTML-to-Markdown conversion process.
+
+```ts
+import { createPlugin, filterUnsupportedTags, syncHtmlToMarkdown, withTailwind } from 'mdream'
+
+// Create a custom plugin
+const myPlugin = createPlugin({
+  name: 'my-plugin',
+  transformContent: (content, node) => {
+    if (node.type === 1 && node.name === 'div' && node.attributes?.role === 'alert') {
+      return `⚠️ ${content} ⚠️`
+    }
+    return content
+  }
+})
+
+// Use multiple plugins together
+const html = '<div role="alert" class="font-bold">Important message</div>'
+const markdown = syncHtmlToMarkdown(html, {
+  plugins: [
+    withTailwind(), // Apply Tailwind class processing
+    filterUnsupportedTags(), // Filter out unsupported tags
+    myPlugin // Apply custom transformations
+  ]
+})
+
+console.log(markdown) // "⚠️ **Important message** ⚠️"
+```
+
+For more details, see the [plugin documentation](./docs/plugins.md).
+
+## Credits
+
+- [ultrahtml](https://github.com/natemoo-re/ultrahtml): HTML parsing inspiration
+
+## License
+
+Licensed under the [MIT license](https://github.com/harlan-zw/mdream/blob/main/LICENSE.md).
+
+<!-- Badges -->
+[npm-version-src]: https://img.shields.io/npm/v/mdream/latest.svg?style=flat&colorA=18181B&colorB=4C9BE0
+[npm-version-href]: https://npmjs.com/package/mdream
+
+[npm-downloads-src]: https://img.shields.io/npm/dm/mdream.svg?style=flat&colorA=18181B&colorB=4C9BE0
+[npm-downloads-href]: https://npmjs.com/package/mdream
+
+[license-src]: https://img.shields.io/github/license/harlan-zw/mdream.svg?style=flat&colorA=18181B&colorB=4C9BE0
+[license-href]: https://github.com/harlan-zw/mdream/blob/main/LICENSE.md

package/bin/mdream.mjs

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+import('../dist/cli.mjs')

package/dist/cli.d.mts

@@ -0,0 +1,2 @@
+
+export { };

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+
+export { };

package/dist/cli.mjs

@@ -0,0 +1,25 @@
+import { Readable } from 'node:stream';
+import { cac } from 'cac';
+import { f as frontmatterPlugin } from './shared/mdream.cpEmpxyh.mjs';
+import { r as readabilityPlugin } from './shared/mdream.C8ruysN5.mjs';
+import { s as streamHtmlToMarkdown } from './shared/mdream.DUeWbUFG.mjs';
+import './shared/mdream.-hdaPj9a.mjs';
+
+async function streamingConvert(options = {}) {
+  const outputStream = process.stdout;
+  const conversionOptions = { origin: options.origin };
+  conversionOptions.plugins = conversionOptions.plugins || [];
+  conversionOptions.plugins.push(readabilityPlugin());
+  conversionOptions.plugins.push(frontmatterPlugin());
+  const markdownGenerator = streamHtmlToMarkdown(Readable.toWeb(process.stdin), conversionOptions);
+  for await (const markdownChunk of markdownGenerator) {
+    if (markdownChunk && markdownChunk.length > 0) {
+      outputStream.write(markdownChunk);
+    }
+  }
+}
+const cli = cac();
+cli.command("[options]", "Convert HTML from stdin to Markdown on stdout").option("--origin <url>", "Origin URL for resolving relative image paths").option("--preset <preset>", "Conversion presets: minimal").action(async (_, opts) => {
+  await streamingConvert(opts);
+});
+cli.help().version("1.0.0").parse();

package/dist/index.d.mts

@@ -0,0 +1,15 @@
+import { H as HTMLToMarkdownOptions } from './shared/mdream.a2AvjJLp.mjs';
+export { B as BufferRegion, E as ELEMENT_NODE, b as ElementNode, f as HandlerContext, M as MdreamProcessingState, d as MdreamRuntimeState, N as Node, e as NodeEvent, P as Plugin, a as PluginCreationOptions, T as TEXT_NODE, g as TagHandler, c as TextNode } from './shared/mdream.a2AvjJLp.mjs';
+import { ReadableStream } from 'node:stream/web';
+
+/**
+ * Creates a markdown stream from an HTML stream
+ * @param htmlStream - ReadableStream of HTML content (as Uint8Array or string)
+ * @param options - Configuration options for conversion
+ * @returns An async generator yielding markdown chunks
+ */
+declare function streamHtmlToMarkdown(htmlStream: ReadableStream | null, options?: HTMLToMarkdownOptions): AsyncIterable<string>;
+
+declare function syncHtmlToMarkdown(html: string, options?: HTMLToMarkdownOptions): string;
+
+export { HTMLToMarkdownOptions, streamHtmlToMarkdown, syncHtmlToMarkdown };

package/dist/index.d.ts

@@ -0,0 +1,15 @@
+import { H as HTMLToMarkdownOptions } from './shared/mdream.a2AvjJLp.js';
+export { B as BufferRegion, E as ELEMENT_NODE, b as ElementNode, f as HandlerContext, M as MdreamProcessingState, d as MdreamRuntimeState, N as Node, e as NodeEvent, P as Plugin, a as PluginCreationOptions, T as TEXT_NODE, g as TagHandler, c as TextNode } from './shared/mdream.a2AvjJLp.js';
+import { ReadableStream } from 'node:stream/web';
+
+/**
+ * Creates a markdown stream from an HTML stream
+ * @param htmlStream - ReadableStream of HTML content (as Uint8Array or string)
+ * @param options - Configuration options for conversion
+ * @returns An async generator yielding markdown chunks
+ */
+declare function streamHtmlToMarkdown(htmlStream: ReadableStream | null, options?: HTMLToMarkdownOptions): AsyncIterable<string>;
+
+declare function syncHtmlToMarkdown(html: string, options?: HTMLToMarkdownOptions): string;
+
+export { HTMLToMarkdownOptions, streamHtmlToMarkdown, syncHtmlToMarkdown };

package/dist/index.mjs

@@ -0,0 +1,13 @@
+import { p as processPartialHTMLToMarkdown } from './shared/mdream.DUeWbUFG.mjs';
+export { s as streamHtmlToMarkdown } from './shared/mdream.DUeWbUFG.mjs';
+import './shared/mdream.-hdaPj9a.mjs';
+
+function syncHtmlToMarkdown(html, options = {}) {
+  const state = {
+    options
+  };
+  const result = processPartialHTMLToMarkdown(html, state).chunk;
+  return result.trimEnd();
+}
+
+export { syncHtmlToMarkdown };

package/dist/plugins.d.mts

@@ -0,0 +1,88 @@
+import { P as Plugin } from './shared/mdream.a2AvjJLp.mjs';
+
+/**
+ * Create a plugin that implements the Plugin interface with improved type inference
+ *
+ * @returns A complete plugin implementation
+ */
+declare function createPlugin<T extends Partial<Plugin>>(plugin: T): Plugin;
+
+/**
+ * Plugin that filters nodes based on CSS selectors.
+ * Allows including or excluding nodes based on selectors.
+ *
+ * @example
+ * // Include only heading elements and their children
+ * withQuerySelectorPlugin({ include: ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'] })
+ *
+ * @example
+ * // Exclude navigation, sidebar, and footer
+ * withQuerySelectorPlugin({ exclude: ['nav', '#sidebar', '.footer'] })
+ */
+declare function filterPlugin(options?: {
+    /** CSS selectors (or Tag Ids) for elements to include (all others will be excluded) */
+    include?: (string | number)[];
+    /** CSS selectors (or Tag Ids) for elements to exclude */
+    exclude?: (string | number)[];
+    /** Whether to also process the children of matching elements */
+    processChildren?: boolean;
+    keepAbsolute?: boolean;
+}): Plugin;
+
+interface FrontmatterPluginOptions {
+    /** Additional frontmatter fields to include */
+    additionalFields?: Record<string, string>;
+    /** Meta tag names to extract (beyond the standard ones) */
+    metaFields?: string[];
+    /** Custom formatter for frontmatter values */
+    formatValue?: (name: string, value: string) => string;
+}
+/**
+ * A plugin that manages frontmatter generation from HTML head elements
+ * Extracts metadata from meta tags and title and generates YAML frontmatter
+ */
+declare function frontmatterPlugin(options?: FrontmatterPluginOptions): Plugin;
+
+/**
+ * Plugin that isolates main content using the following priority order:
+ * 1. If an explicit <main> element exists (within 5 depth levels), use its content exclusively
+ * 2. Otherwise, find content between the first header tag (h1-h6) and first footer
+ * 3. If footer is within 5 levels of nesting from the header, use it as the end boundary
+ * 4. Exclude all content before the start marker and after the end marker
+ *
+ * @example
+ * ```html
+ * <body>
+ *   <nav>Navigation (excluded)</nav>
+ *   <main>
+ *     <h1>Main Title (included)</h1>
+ *     <p>Main content (included)</p>
+ *   </main>
+ *   <footer>Footer (excluded)</footer>
+ * </body>
+ * ```
+ *
+ * @example
+ * ```html
+ * <body>
+ *   <nav>Navigation (excluded)</nav>
+ *   <h1>Main Title (included)</h1>
+ *   <p>Main content (included)</p>
+ *   <footer>Footer (excluded)</footer>
+ * </body>
+ * ```
+ */
+declare function isolateMainPlugin(): Plugin;
+
+/**
+ * Creates a plugin that implements readability.js style heuristics for content quality assessment
+ * Controls content inclusion/exclusion using buffer regions
+ */
+declare function readabilityPlugin(): Plugin;
+
+/**
+ * Creates a plugin that adds Tailwind class processing
+ */
+declare function tailwindPlugin(): Plugin;
+
+export { createPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, readabilityPlugin, tailwindPlugin };

package/dist/plugins.d.ts

@@ -0,0 +1,88 @@
+import { P as Plugin } from './shared/mdream.a2AvjJLp.js';
+
+/**
+ * Create a plugin that implements the Plugin interface with improved type inference
+ *
+ * @returns A complete plugin implementation
+ */
+declare function createPlugin<T extends Partial<Plugin>>(plugin: T): Plugin;
+
+/**
+ * Plugin that filters nodes based on CSS selectors.
+ * Allows including or excluding nodes based on selectors.
+ *
+ * @example
+ * // Include only heading elements and their children
+ * withQuerySelectorPlugin({ include: ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'] })
+ *
+ * @example
+ * // Exclude navigation, sidebar, and footer
+ * withQuerySelectorPlugin({ exclude: ['nav', '#sidebar', '.footer'] })
+ */
+declare function filterPlugin(options?: {
+    /** CSS selectors (or Tag Ids) for elements to include (all others will be excluded) */
+    include?: (string | number)[];
+    /** CSS selectors (or Tag Ids) for elements to exclude */
+    exclude?: (string | number)[];
+    /** Whether to also process the children of matching elements */
+    processChildren?: boolean;
+    keepAbsolute?: boolean;
+}): Plugin;
+
+interface FrontmatterPluginOptions {
+    /** Additional frontmatter fields to include */
+    additionalFields?: Record<string, string>;
+    /** Meta tag names to extract (beyond the standard ones) */
+    metaFields?: string[];
+    /** Custom formatter for frontmatter values */
+    formatValue?: (name: string, value: string) => string;
+}
+/**
+ * A plugin that manages frontmatter generation from HTML head elements
+ * Extracts metadata from meta tags and title and generates YAML frontmatter
+ */
+declare function frontmatterPlugin(options?: FrontmatterPluginOptions): Plugin;
+
+/**
+ * Plugin that isolates main content using the following priority order:
+ * 1. If an explicit <main> element exists (within 5 depth levels), use its content exclusively
+ * 2. Otherwise, find content between the first header tag (h1-h6) and first footer
+ * 3. If footer is within 5 levels of nesting from the header, use it as the end boundary
+ * 4. Exclude all content before the start marker and after the end marker
+ *
+ * @example
+ * ```html
+ * <body>
+ *   <nav>Navigation (excluded)</nav>
+ *   <main>
+ *     <h1>Main Title (included)</h1>
+ *     <p>Main content (included)</p>
+ *   </main>
+ *   <footer>Footer (excluded)</footer>
+ * </body>
+ * ```
+ *
+ * @example
+ * ```html
+ * <body>
+ *   <nav>Navigation (excluded)</nav>
+ *   <h1>Main Title (included)</h1>
+ *   <p>Main content (included)</p>
+ *   <footer>Footer (excluded)</footer>
+ * </body>
+ * ```
+ */
+declare function isolateMainPlugin(): Plugin;
+
+/**
+ * Creates a plugin that implements readability.js style heuristics for content quality assessment
+ * Controls content inclusion/exclusion using buffer regions
+ */
+declare function readabilityPlugin(): Plugin;
+
+/**
+ * Creates a plugin that adds Tailwind class processing
+ */
+declare function tailwindPlugin(): Plugin;
+
+export { createPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, readabilityPlugin, tailwindPlugin };

package/dist/plugins.mjs

@@ -0,0 +1,4 @@
+export { c as createPlugin, f as frontmatterPlugin } from './shared/mdream.cpEmpxyh.mjs';
+export { f as filterPlugin, i as isolateMainPlugin, t as tailwindPlugin } from './shared/mdream.5zaIXVJz.mjs';
+export { r as readabilityPlugin } from './shared/mdream.C8ruysN5.mjs';
+import './shared/mdream.-hdaPj9a.mjs';

package/dist/preset/minimal.d.mts

@@ -0,0 +1,11 @@
+import { H as HTMLToMarkdownOptions } from '../shared/mdream.a2AvjJLp.mjs';
+
+/**
+ * Creates a configurable minimal preset with advanced options
+ *
+ * @param options HTML to Markdown options
+ * @returns HTML to Markdown options with configured plugins
+ */
+declare function withMinimalPreset(options?: HTMLToMarkdownOptions): HTMLToMarkdownOptions;
+
+export { withMinimalPreset };

package/dist/preset/minimal.d.ts

@@ -0,0 +1,11 @@
+import { H as HTMLToMarkdownOptions } from '../shared/mdream.a2AvjJLp.js';
+
+/**
+ * Creates a configurable minimal preset with advanced options
+ *
+ * @param options HTML to Markdown options
+ * @returns HTML to Markdown options with configured plugins
+ */
+declare function withMinimalPreset(options?: HTMLToMarkdownOptions): HTMLToMarkdownOptions;
+
+export { withMinimalPreset };

package/dist/preset/minimal.mjs

@@ -0,0 +1,39 @@
+import { y as TAG_FORM, t as TAG_FIELDSET, q as TAG_OBJECT, r as TAG_EMBED, a0 as TAG_FIGURE, B as TAG_FOOTER, z as TAG_ASIDE, s as TAG_IFRAME, w as TAG_INPUT, v as TAG_TEXTAREA, u as TAG_SELECT, x as TAG_BUTTON, A as TAG_NAV } from '../shared/mdream.-hdaPj9a.mjs';
+import { i as isolateMainPlugin, t as tailwindPlugin, f as filterPlugin } from '../shared/mdream.5zaIXVJz.mjs';
+import { f as frontmatterPlugin } from '../shared/mdream.cpEmpxyh.mjs';
+
+function withMinimalPreset(options = {}) {
+  const plugins = [
+    isolateMainPlugin(),
+    frontmatterPlugin(),
+    tailwindPlugin(),
+    // First apply readability plugin to extract main content
+    // Then filter out unwanted tags
+    filterPlugin({
+      exclude: [
+        TAG_FORM,
+        TAG_FIELDSET,
+        TAG_OBJECT,
+        TAG_EMBED,
+        TAG_FIGURE,
+        TAG_FOOTER,
+        TAG_ASIDE,
+        TAG_IFRAME,
+        TAG_INPUT,
+        TAG_TEXTAREA,
+        TAG_SELECT,
+        TAG_BUTTON,
+        TAG_NAV
+      ]
+    })
+  ];
+  if (options.plugins) {
+    plugins.push(...options.plugins);
+  }
+  return {
+    ...options,
+    plugins
+  };
+}
+
+export { withMinimalPreset };