mdream 1.7.1 → 2.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -3,6 +3,13 @@
3
3
  [![npm version](https://img.shields.io/npm/v/mdream?color=yellow)](https://npmjs.com/package/mdream)
4
4
  [![npm downloads](https://img.shields.io/npm/dm/mdream?color=yellow)](https://npm.chart.dev/mdream)
5
5
  [![license](https://img.shields.io/github/license/harlan-zw/mdream?color=yellow)](https://github.com/harlan-zw/mdream/blob/main/LICENSE.md)
6
+ <a href="https://skilld.dev/gh/harlan-zw/mdream">
7
+ <picture>
8
+ <source media="(prefers-color-scheme: dark)" srcset="https://skilld.dev/b/harlan-zw/mdream?theme=dark">
9
+ <source media="(prefers-color-scheme: light)" srcset="https://skilld.dev/b/harlan-zw/mdream?theme=light">
10
+ <img alt="Skill repository on skilld.dev" src="https://skilld.dev/b/harlan-zw/mdream?theme=light">
11
+ </picture>
12
+ </a>
6
13
 
7
14
  ## Installation
8
15
 
@@ -17,6 +24,9 @@ pnpm add mdream
17
24
  yarn add mdream
18
25
  ```
19
26
 
27
+ > [!TIP]
28
+ > Using an AI agent? Get the mdream Skill on [skilld.dev/gh/harlan-zw/mdream](https://skilld.dev/gh/harlan-zw/mdream).
29
+
20
30
  For the JavaScript-only engine (hook-based plugins, splitter, pure HTML parser):
21
31
 
22
32
  ```bash
@@ -89,7 +99,7 @@ externals: ['mdream']
89
99
 
90
100
  ### `htmlToMarkdown()`
91
101
 
92
- Converts a complete HTML string to Markdown, plain text, or HTML synchronously.
102
+ Converts a complete HTML string to Markdown, plain text, or HTML. It is synchronous in Node and on edge runtimes.
93
103
 
94
104
  **Rust engine** (`mdream`):
95
105
 
@@ -97,6 +107,9 @@ Converts a complete HTML string to Markdown, plain text, or HTML synchronously.
97
107
  import { htmlToMarkdown } from 'mdream'
98
108
 
99
109
  function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string
110
+
111
+ // Browser bundles and the CDN script load the WASM binary first
112
+ function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): Promise<string>
100
113
  ```
101
114
 
102
115
  **JS engine** (`@mdream/js`):
@@ -132,6 +145,8 @@ const html = htmlToMarkdown('<h1>Hello <strong>World</strong></h1>', {
132
145
 
133
146
  Converts an HTML `ReadableStream` to Markdown, plain text, or HTML incrementally. Returns an `AsyncIterable<string>` that yields output chunks as they are processed.
134
147
 
148
+ It takes the same options as `htmlToMarkdown()`. The `frontmatter` callback and the `extraction` handlers run once, after the stream ends and before the last chunk is yielded.
149
+
135
150
 
136
151
  ```ts
137
152
  import { streamHtmlToMarkdown } from 'mdream'
@@ -176,7 +191,7 @@ import { htmlToMarkdown } from '@mdream/js'
176
191
  import { htmlToMarkdown } from 'mdream'
177
192
  ```
178
193
 
179
- Both engines accept the same declarative plugin configuration (`origin`, `minimal`, `frontmatter`, `isolateMain`, `tailwind`, `filter`, `extraction`, `tagOverrides`, `clean`). The JS engine additionally supports `hooks` for imperative plugin transforms.
194
+ Both engines support the same declarative plugins, but the option shape differs. The Rust engine takes `minimal`, `frontmatter`, `isolateMain`, `tailwind`, `filter`, `extraction`, and `tagOverrides` at the top level. The JS engine reads the plugins from `options.plugins` and has no `minimal` option: use `withMinimalPreset()` from `@mdream/js/preset/minimal`. If the JS engine gets one of these options at the top level, it throws a `TypeError` that names the fix. The JS engine also supports `hooks` for imperative plugin transforms.
180
195
 
181
196
  ## Options
182
197
 
@@ -216,7 +231,7 @@ interface MdreamOptions {
216
231
  tailwind?: boolean
217
232
 
218
233
  /** Filter elements by CSS selectors. Default when minimal: excludes form, nav, footer, etc. */
219
- filter?: { include?: string[], exclude?: string[], processChildren?: boolean }
234
+ filter?: false | { include?: string[], exclude?: string[], processChildren?: boolean }
220
235
 
221
236
  /** Extract elements matching CSS selectors during conversion. */
222
237
  extraction?: Record<string, (element: ExtractedElement) => void>
@@ -263,7 +278,7 @@ interface EngineOptions {
263
278
  }
264
279
 
265
280
  interface BuiltinPlugins {
266
- filter?: { include?: (string | number)[], exclude?: (string | number)[], processChildren?: boolean }
281
+ filter?: false | { include?: (string | number)[], exclude?: (string | number)[], processChildren?: boolean }
267
282
  frontmatter?: boolean | ((fm: Record<string, string>) => void) | FrontmatterConfig
268
283
  isolateMain?: boolean
269
284
  tailwind?: boolean
@@ -336,7 +351,7 @@ Override how specific HTML tags are rendered in Markdown. String values act as a
336
351
  >
337
352
  > ```ts
338
353
  > htmlToMarkdown('<my-em>hi</my-em>', { tagOverrides: { 'my-em': 'em' } })
339
- > // → "_hi_"
354
+ > // → "*hi*"
340
355
  > ```
341
356
 
342
357
  ```ts
@@ -375,12 +390,12 @@ const markdown = htmlToMarkdown(html, {
375
390
  })
376
391
  ```
377
392
 
378
- **Output is GitHub Flavored Markdown.** Mdream emits a fixed GFM dialect tuned for LLM input: ATX headings (`#`), fenced code blocks (` ``` `), `-` bullets, `_` emphasis, `**` strong, `---` horizontal rules, inline links. These are not configurable. For simple delimiter swaps you can use `tagOverrides`:
393
+ **Output is GitHub Flavored Markdown.** Mdream emits a fixed GFM dialect tuned for LLM input: ATX headings (`#`), fenced code blocks (` ``` `), `-` bullets, `*` emphasis, `**` strong, `---` horizontal rules, inline links. These are not configurable. For simple delimiter swaps you can use `tagOverrides`:
379
394
 
380
395
  ```ts
381
396
  htmlToMarkdown(html, {
382
397
  tagOverrides: {
383
- em: { enter: '*', exit: '*', isInline: true }, // _x_ → *x*
398
+ em: { enter: '_', exit: '_', isInline: true }, // *x* → _x_
384
399
  strong: { enter: '__', exit: '__', isInline: true }, // **x** → __x__
385
400
  hr: { enter: '* * *', exit: '' }, // --- → * * *
386
401
  },
@@ -414,6 +429,16 @@ The `minimal` preset enables the following plugins together:
414
429
  - **filter**: Excludes `form`, `fieldset`, `object`, `embed`, `footer`, `aside`, `iframe`, `input`, `textarea`, `select`, `button`, `nav`
415
430
  - **clean**: All post-processing cleanup enabled
416
431
 
432
+ A `filter` passed with `minimal` adds to the preset's excludes. It does not replace them. To turn the preset's filter off, pass `filter: false`. `withMinimalPreset()` in the JS engine works the same way.
433
+
434
+ ```ts
435
+ // Excludes h1 as well as form, nav, footer, and the rest
436
+ htmlToMarkdown(html, { minimal: true, filter: { exclude: ['h1'] } })
437
+
438
+ // Keeps forms and nav
439
+ htmlToMarkdown(html, { minimal: true, filter: false })
440
+ ```
441
+
417
442
  **Rust engine:**
418
443
 
419
444
  ```ts
@@ -436,13 +461,13 @@ const markdown = htmlToMarkdown(html, withMinimalPreset({
436
461
  }))
437
462
  ```
438
463
 
439
- `withMinimalPreset()` returns an `EngineOptions` object with all plugin defaults applied. You can override individual plugins:
464
+ `withMinimalPreset()` returns an `EngineOptions` object with all plugin defaults applied. You can override individual plugins. A `filter` adds to the preset's excludes:
440
465
 
441
466
  ```ts
442
467
  const markdown = htmlToMarkdown(html, withMinimalPreset({
443
468
  plugins: {
444
469
  frontmatter: false,
445
- filter: { exclude: ['nav'] },
470
+ filter: { exclude: ['.cookie-banner'] },
446
471
  },
447
472
  }))
448
473
  ```
@@ -927,6 +952,15 @@ The CLI reads HTML from stdin and writes Markdown, plain text, or HTML to stdout
927
952
 
928
953
  ## Browser and Edge Usage
929
954
 
955
+ Every entry takes the same `MdreamOptions` and produces the same Markdown string. In Node and edge runtimes, `htmlToMarkdown` is synchronous. In browser bundles, the CDN script, and `mdream/worker`, it returns `Promise<string>`, because the WASM binary loads first. Each export condition ships its own types, so TypeScript shows the right return type when it resolves the `browser` condition (for example, with `customConditions: ["browser"]`).
956
+
957
+ ```ts
958
+ import { htmlToMarkdown } from 'mdream'
959
+
960
+ // In a browser bundle
961
+ const markdown = await htmlToMarkdown('<h1>Hello</h1>', { minimal: true })
962
+ ```
963
+
930
964
  ### Edge / Cloudflare Workers
931
965
 
932
966
  For edge runtimes (Cloudflare Workers, Vercel Edge), `mdream` automatically selects the WASM build via export conditions (`workerd`, `edge-light`). Both `htmlToMarkdown` and `streamHtmlToMarkdown` are available:
@@ -957,61 +991,30 @@ catch (error) {
957
991
  }
958
992
  ```
959
993
 
960
- If your toolchain doesn't resolve the export conditions, the raw wasm-bindgen build (web target) is exposed at `mdream/wasm` for manual initialization:
994
+ If your toolchain doesn't resolve the export conditions, the wasm-bindgen build (web target) is exposed at `mdream/wasm` for manual initialization. Its `htmlToMarkdown` takes the same `MdreamOptions` as the `mdream` entry. Its `MarkdownStream` runs the callbacks in `finish()`.
961
995
 
962
996
  ```ts
963
997
  import init, { htmlToMarkdown } from 'mdream/wasm'
964
998
  import wasmModule from 'mdream/wasm/mdream_edge_bg.wasm'
965
999
 
966
1000
  await init({ module_or_path: wasmModule })
967
- const markdown = htmlToMarkdown('<h1>Hello</h1>', {})
968
- ```
969
-
970
- You can also import the edge entry point directly:
971
-
972
- ```ts
973
- import { htmlToMarkdown } from 'mdream/worker'
1001
+ const markdown = htmlToMarkdown('<h1>Hello</h1>', { minimal: true })
974
1002
  ```
975
1003
 
976
- The `mdream/worker` entry provides an async API since WASM must be initialized first:
977
-
978
- ```ts
979
- import { htmlToMarkdown, initWorker, terminateWorker } from 'mdream/worker'
980
-
981
- // Initialize once with the WASM URL
982
- await initWorker('https://cdn.example.com/mdream_edge_bg.wasm')
983
-
984
- // Convert (returns Promise<string>)
985
- const markdown = await htmlToMarkdown('<h1>Hello</h1>')
986
-
987
- // Clean up when done
988
- terminateWorker()
989
- ```
1004
+ `mdream/worker` is not an edge entry. It runs the conversion in a browser Web Worker; see [Web Worker](#web-worker).
990
1005
 
991
1006
  ### Browser CDN (IIFE)
992
1007
 
993
- Use mdream directly via CDN with no build step. Call `init()` once to load the WASM binary, then use `htmlToMarkdown()` synchronously.
1008
+ Use mdream directly via CDN with no build step. The script inlines the WASM binary and initializes it on load, so `window.mdream.htmlToMarkdown()` is ready at once. It takes the same `MdreamOptions` as the `mdream` entry and returns `Promise<string>`, the same as the browser bundle.
994
1009
 
995
1010
  ```html
996
1011
  <script src="https://unpkg.com/mdream/dist/iife.js"></script>
997
1012
  <script>
998
- await window.mdream.init()
999
- const markdown = window.mdream.htmlToMarkdown('<h1>Hello</h1><p>World</p>')
1000
- console.log(markdown) // # Hello\n\nWorld
1013
+ window.mdream.htmlToMarkdown('<h1>Hello</h1><p>World</p>')
1014
+ .then(markdown => console.log(markdown)) // # Hello\n\nWorld
1001
1015
  </script>
1002
1016
  ```
1003
1017
 
1004
- You can pass a custom WASM URL or `ArrayBuffer` to `init()`:
1005
-
1006
- ```js
1007
- // Custom URL
1008
- await window.mdream.init('https://cdn.example.com/mdream_edge_bg.wasm')
1009
-
1010
- // Pre-loaded ArrayBuffer
1011
- const wasmBytes = await fetch('/wasm/mdream_edge_bg.wasm').then(r => r.arrayBuffer())
1012
- await window.mdream.init(wasmBytes)
1013
- ```
1014
-
1015
1018
  **CDN Options:**
1016
1019
  - **unpkg**: `https://unpkg.com/mdream/dist/iife.js`
1017
1020
  - **jsDelivr**: `https://cdn.jsdelivr.net/npm/mdream/dist/iife.js`
@@ -1025,7 +1028,8 @@ import { htmlToMarkdown, initWorker, terminateWorker } from 'mdream/worker'
1025
1028
 
1026
1029
  await initWorker('/path/to/mdream_edge_bg.wasm')
1027
1030
 
1028
- const markdown = await htmlToMarkdown('<h1>Hello</h1>')
1031
+ // Takes the same options as the mdream entry. Callbacks run on this thread.
1032
+ const markdown = await htmlToMarkdown('<h1>Hello</h1>', { minimal: true })
1029
1033
 
1030
1034
  // Clean up
1031
1035
  terminateWorker()
@@ -1051,10 +1055,10 @@ if (article) {
1051
1055
 
1052
1056
  ## llms.txt Generation
1053
1057
 
1054
- For llms.txt artifact generation, use the separate `@mdream/llms-txt` package. It accepts pre-converted Markdown and generates `llms.txt` and `llms-full.txt` artifacts.
1058
+ For llms.txt artifact generation, use `@mdream/js/llms-txt` from the `@mdream/js` package. It accepts pre-converted Markdown and generates `llms.txt` and `llms-full.txt` artifacts.
1055
1059
 
1056
1060
  ```ts
1057
- import { generateLlmsTxtArtifacts } from '@mdream/llms-txt'
1061
+ import { generateLlmsTxtArtifacts } from '@mdream/js/llms-txt'
1058
1062
  import { htmlToMarkdown } from 'mdream'
1059
1063
 
1060
1064
  const result = await generateLlmsTxtArtifacts({
@@ -1076,8 +1080,7 @@ console.log(result.llmsFullTxt) // llms-full.txt content
1076
1080
  | Package | Description |
1077
1081
  |---------|-------------|
1078
1082
  | [`mdream`](https://npmjs.com/package/mdream) | Core HTML to Markdown converter (Rust + WASM engine) |
1079
- | [`@mdream/js`](https://npmjs.com/package/@mdream/js) | JavaScript engine with hook-based plugins and splitter |
1080
- | [`@mdream/llms-txt`](https://github.com/harlan-zw/mdream/tree/main/packages/llms-txt) | Engine-agnostic llms.txt artifact generation |
1083
+ | [`@mdream/js`](https://npmjs.com/package/@mdream/js) | JavaScript engine with hook-based plugins, splitter, and llms.txt generation (`@mdream/js/llms-txt`) |
1081
1084
  | [`@mdream/crawl`](https://github.com/harlan-zw/mdream/tree/main/packages/crawl) | Site-wide crawler for llms.txt generation |
1082
1085
  | [`@mdream/vite`](https://github.com/harlan-zw/mdream/tree/main/packages/vite) | Vite plugin integration |
1083
1086
  | [`@mdream/nuxt`](https://github.com/harlan-zw/mdream/tree/main/packages/nuxt) | Nuxt module integration |
@@ -1,11 +1,105 @@
1
- import { HtmlToMarkdownOptions, MdreamNapiResult } from "../napi/index.js";
2
- declare function htmlToMarkdown(html: string, options?: HtmlToMarkdownOptions): Promise<MdreamNapiResult>;
3
- declare function createMarkdownStream(options?: HtmlToMarkdownOptions): Promise<MarkdownStream>;
4
- declare class MarkdownStream {
1
+ interface CleanOptions {
2
+ /** Strip tracking query parameters (utm_*, fbclid, gclid, etc.) from URLs */
3
+ urls?: boolean;
4
+ /** Strip fragment-only links that don't match any heading in the output */
5
+ fragments?: boolean;
6
+ /** Strip links with meaningless hrefs (#, javascript:void(0)) → plain text */
7
+ emptyLinks?: boolean;
8
+ /** Collapse 3+ consecutive blank lines to 2 */
9
+ blankLines?: boolean;
10
+ /** Strip links where text equals URL: [https://x.com](https://x.com) → https://x.com */
11
+ redundantLinks?: boolean;
12
+ /** Strip self-referencing heading anchors: ## [Title](#title) → ## Title */
13
+ selfLinkHeadings?: boolean;
14
+ /** Strip images with no alt text (decorative/tracking pixels) */
15
+ emptyImages?: boolean;
16
+ /** Drop links that produce no visible text: [](url) → nothing */
17
+ emptyLinkText?: boolean;
18
+ }
19
+ interface ExtractedElement {
20
+ selector: string;
21
+ tagName: string;
22
+ textContent: string;
23
+ attributes: Record<string, string>;
24
+ }
25
+ interface FrontmatterConfig {
26
+ additionalFields?: Record<string, string>;
27
+ metaFields?: string[];
28
+ /** Callback to receive structured frontmatter data after conversion */
29
+ onExtract?: (frontmatter: Record<string, string>) => void;
30
+ }
31
+ interface TagOverride {
32
+ enter?: string;
33
+ exit?: string;
34
+ spacing?: number[];
35
+ isInline?: boolean;
36
+ isSelfClosing?: boolean;
37
+ collapsesInnerWhiteSpace?: boolean;
38
+ alias?: string;
39
+ }
40
+ interface MdreamOptions {
41
+ /** Origin URL for resolving relative image paths and internal links. */
42
+ origin?: string;
43
+ /**
44
+ * Clean up the markdown output. Pass `true` for all cleanup or an object
45
+ * to enable specific features. Both `htmlToMarkdown` and
46
+ * `streamHtmlToMarkdown` apply it.
47
+ */
48
+ clean?: boolean | CleanOptions;
49
+ /** Enable minimal preset (frontmatter, isolateMain, tailwind, filter). Default: false */
50
+ minimal?: boolean;
51
+ /**
52
+ * Extract frontmatter from HTML head.
53
+ * - `true`: enable with defaults
54
+ * - `(fm) => void`: enable and receive structured data via callback
55
+ * - `FrontmatterConfig`: enable with config options and optional callback
56
+ */
57
+ frontmatter?: boolean | ((frontmatter: Record<string, string>) => void) | FrontmatterConfig;
58
+ /** Isolate main content area. Default when minimal: true */
59
+ isolateMain?: boolean;
60
+ /** Convert Tailwind utility classes. Default when minimal: true */
61
+ tailwind?: boolean;
62
+ /**
63
+ * Filter elements by CSS selector. `minimal` excludes form, nav, footer, and
64
+ * similar; a filter passed with `minimal` adds to those excludes. `false`
65
+ * turns filtering off, including the `minimal` one.
66
+ */
67
+ filter?: false | {
68
+ include?: string[];
69
+ exclude?: string[];
70
+ processChildren?: boolean;
71
+ };
72
+ /** Extract elements matching CSS selectors */
73
+ extraction?: Record<string, (element: ExtractedElement) => void>;
74
+ /** Tag overrides. String values act as aliases */
75
+ tagOverrides?: Record<string, TagOverride | string>;
76
+ /**
77
+ * Hard-wrap prose at this many characters, breaking on word boundaries.
78
+ * Applied inline during conversion (zero-cost when unset). Code blocks,
79
+ * tables, and headings are never wrapped. `0` disables wrapping.
80
+ */
81
+ wrapWidth?: number;
82
+ /**
83
+ * Output format. Defaults to `markdown`; use `text` for readable plain text,
84
+ * or `html` for allowlisted semantic HTML.
85
+ */
86
+ format?: 'markdown' | 'text' | 'html';
87
+ }
88
+ /**
89
+ * Browser builds fetch the WASM binary on first use, so this returns a
90
+ * Promise. The `browser` export condition ships types that say so.
91
+ */
92
+ export declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): Promise<string>;
93
+ export declare function createMarkdownStream(options?: Partial<MdreamOptions>): Promise<MarkdownStream>;
94
+ /** Streaming converter. Runs the frontmatter and extraction callbacks in `finish()`. */
95
+ export declare class MarkdownStream {
5
96
  private _inner;
6
- constructor(options?: HtmlToMarkdownOptions);
97
+ private _carry;
98
+ private _callbacks;
99
+ constructor(options?: Partial<MdreamOptions>);
7
100
  processChunk(chunk: string): string;
101
+ processChunkBytes(chunk: Uint8Array): string;
8
102
  finish(): string;
9
103
  }
10
- declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: HtmlToMarkdownOptions): AsyncIterable<string>;
11
- export { MarkdownStream, createMarkdownStream, htmlToMarkdown, streamHtmlToMarkdown };
104
+ export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
105
+ export type { CleanOptions, ExtractedElement, FrontmatterConfig, MdreamOptions, TagOverride };
package/dist/browser.mjs CHANGED
@@ -1,5 +1,169 @@
1
1
  import init, { MarkdownStream as MarkdownStream$1, __mdreamTakePanicMessage, htmlToMarkdownResult } from "../wasm/mdream_edge.js";
2
- function wasmPanicError(error, takeMessage = __mdreamTakePanicMessage) {
2
+ const MINIMAL_FILTER_EXCLUDE = [
3
+ "form",
4
+ "fieldset",
5
+ "object",
6
+ "embed",
7
+ "footer",
8
+ "aside",
9
+ "iframe",
10
+ "input",
11
+ "textarea",
12
+ "select",
13
+ "button",
14
+ "nav"
15
+ ];
16
+ const CLEAN_ALL = {
17
+ urls: true,
18
+ fragments: true,
19
+ emptyLinks: true,
20
+ redundantLinks: true,
21
+ selfLinkHeadings: true,
22
+ emptyImages: true,
23
+ emptyLinkText: true
24
+ };
25
+ function resolveCleanConfig(options, minimal) {
26
+ const cleanUrls = options.cleanUrls === true;
27
+ let cleanOpt = options.clean;
28
+ if (cleanOpt === void 0) {
29
+ if (!minimal) return { cleanUrls };
30
+ cleanOpt = true;
31
+ }
32
+ if (!cleanOpt) return { cleanUrls };
33
+ const resolved = cleanOpt === true ? CLEAN_ALL : cleanOpt;
34
+ return {
35
+ cleanUrls: cleanUrls || resolved.urls === true,
36
+ clean: resolved
37
+ };
38
+ }
39
+ function resolveFrontmatter(opt) {
40
+ if (typeof opt === "function") return {
41
+ config: {},
42
+ callback: opt
43
+ };
44
+ if (opt && typeof opt === "object") {
45
+ const { onExtract, ...config } = opt;
46
+ return {
47
+ config,
48
+ callback: onExtract
49
+ };
50
+ }
51
+ return { config: {} };
52
+ }
53
+ function resolveOptions(options) {
54
+ if (Array.isArray(options.plugins)) throw new TypeError("Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://mdream.dev/v1-migration#custom-plugins");
55
+ const minimal = options.minimal === true;
56
+ const plugins = options.plugins ? { ...options.plugins } : {};
57
+ let frontmatterCallback;
58
+ const frontmatterDisabled = options.frontmatter === false;
59
+ if (minimal ? !frontmatterDisabled : !!options.frontmatter) {
60
+ const fm = resolveFrontmatter(options.frontmatter);
61
+ plugins.frontmatter = fm.config;
62
+ frontmatterCallback = fm.callback;
63
+ } else if (frontmatterDisabled) delete plugins.frontmatter;
64
+ const isolateMainDisabled = options.isolateMain === false;
65
+ if (minimal ? !isolateMainDisabled : options.isolateMain) plugins.isolateMain = true;
66
+ else if (isolateMainDisabled) delete plugins.isolateMain;
67
+ const tailwindDisabled = options.tailwind === false;
68
+ if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
69
+ else if (tailwindDisabled) delete plugins.tailwind;
70
+ if (options.filter === false) delete plugins.filter;
71
+ else if (minimal) plugins.filter = {
72
+ ...options.filter,
73
+ exclude: [...MINIMAL_FILTER_EXCLUDE, ...options.filter?.exclude ?? []]
74
+ };
75
+ else if (options.filter) plugins.filter = options.filter;
76
+ const extraction = options.extraction;
77
+ let extractionHandlers;
78
+ if (extraction) {
79
+ plugins.extraction = { selectors: Object.keys(extraction) };
80
+ extractionHandlers = extraction;
81
+ }
82
+ const tagOverrides = options.tagOverrides;
83
+ if (tagOverrides) {
84
+ const overrides = {};
85
+ for (const tag in tagOverrides) {
86
+ const value = tagOverrides[tag];
87
+ if (value) overrides[tag] = typeof value === "string" ? { alias: value } : value;
88
+ }
89
+ plugins.tagOverrides = overrides;
90
+ }
91
+ const { cleanUrls, clean } = resolveCleanConfig(options, minimal);
92
+ return {
93
+ napiOpts: {
94
+ origin: options.origin,
95
+ cleanUrls,
96
+ clean,
97
+ plugins,
98
+ wrapWidth: options.wrapWidth,
99
+ format: options.format
100
+ },
101
+ extractionHandlers,
102
+ frontmatterCallback
103
+ };
104
+ }
105
+ function deliverPluginData(data, { frontmatterCallback, extractionHandlers }) {
106
+ if (data.frontmatter && frontmatterCallback) frontmatterCallback(data.frontmatter);
107
+ if (data.extracted?.length && extractionHandlers) for (const element of data.extracted) extractionHandlers[element.selector]?.(element);
108
+ }
109
+ function hasCallbacks({ frontmatterCallback, extractionHandlers }) {
110
+ return !!frontmatterCallback || !!extractionHandlers;
111
+ }
112
+ function convertResult(convert, html, options = {}) {
113
+ const resolved = resolveOptions(options);
114
+ const result = convert(html, resolved.napiOpts);
115
+ deliverPluginData(result, resolved);
116
+ return result;
117
+ }
118
+ async function* pumpStream(stream, htmlStream, callbacks, { decodeInJs = false, mapError = (error) => error } = {}) {
119
+ const reader = htmlStream.getReader();
120
+ const decoder = decodeInJs ? new TextDecoder() : void 0;
121
+ try {
122
+ while (true) {
123
+ const { done, value } = await reader.read();
124
+ if (done) break;
125
+ let processed;
126
+ if (typeof value === "string") processed = stream.processChunk(decoder ? decoder.decode() + value : value);
127
+ else processed = decoder ? stream.processChunk(decoder.decode(value, { stream: true })) : stream.processChunkBytes(value);
128
+ if (processed) yield processed;
129
+ }
130
+ const decoderTail = decoder?.decode();
131
+ if (decoderTail) {
132
+ const processed = stream.processChunk(decoderTail);
133
+ if (processed) yield processed;
134
+ }
135
+ const final_ = stream.finish();
136
+ if (hasCallbacks(callbacks)) deliverPluginData(stream.takeData(), callbacks);
137
+ if (final_) yield final_;
138
+ } catch (error) {
139
+ throw mapError(error);
140
+ } finally {
141
+ reader.releaseLock();
142
+ }
143
+ }
144
+ function createSurrogateCarry() {
145
+ let held = "";
146
+ return {
147
+ take(chunk) {
148
+ if (held) {
149
+ chunk = held + chunk;
150
+ held = "";
151
+ }
152
+ const last = chunk.charCodeAt(chunk.length - 1);
153
+ if (last >= 55296 && last <= 56319) {
154
+ held = chunk.slice(-1);
155
+ return chunk.slice(0, -1);
156
+ }
157
+ return chunk;
158
+ },
159
+ flush() {
160
+ const value = held;
161
+ held = "";
162
+ return value;
163
+ }
164
+ };
165
+ }
166
+ function wasmPanicError(error, takeMessage) {
3
167
  if (!(error instanceof WebAssembly.RuntimeError)) return error;
4
168
  let message;
5
169
  try {
@@ -16,67 +180,83 @@ function ensureInit() {
16
180
  return _initPromise;
17
181
  }
18
182
  ensureInit();
19
- async function htmlToMarkdown(html, options) {
20
- await ensureInit();
183
+ function convert(html, napiOpts) {
21
184
  try {
22
- return htmlToMarkdownResult(html, options || {});
185
+ return htmlToMarkdownResult(html, napiOpts);
23
186
  } catch (error) {
24
- throw wasmPanicError(error);
187
+ throw wasmPanicError(error, __mdreamTakePanicMessage);
25
188
  }
26
189
  }
190
+ async function htmlToMarkdown(html, options = {}) {
191
+ await ensureInit();
192
+ return convertResult(convert, html, options).markdown || "";
193
+ }
27
194
  async function createMarkdownStream(options) {
28
195
  await ensureInit();
29
196
  return new MarkdownStream(options);
30
197
  }
31
198
  var MarkdownStream = class {
32
199
  _inner;
33
- constructor(options) {
200
+ _carry = createSurrogateCarry();
201
+ _callbacks;
202
+ constructor(options = {}) {
203
+ const resolved = resolveOptions(options);
204
+ this._callbacks = resolved;
34
205
  try {
35
- this._inner = new MarkdownStream$1(options || {});
206
+ this._inner = new MarkdownStream$1(resolved.napiOpts);
36
207
  } catch (error) {
37
- throw wasmPanicError(error);
208
+ throw wasmPanicError(error, __mdreamTakePanicMessage);
38
209
  }
39
210
  }
40
211
  processChunk(chunk) {
41
212
  try {
42
- return this._inner.processChunk(chunk);
213
+ return this._inner.processChunk(this._carry.take(chunk));
214
+ } catch (error) {
215
+ throw wasmPanicError(error, __mdreamTakePanicMessage);
216
+ }
217
+ }
218
+ processChunkBytes(chunk) {
219
+ try {
220
+ const held = this._carry.flush();
221
+ return held ? this._inner.processChunk(held) + this._inner.processChunkBytes(chunk) : this._inner.processChunkBytes(chunk);
43
222
  } catch (error) {
44
- throw wasmPanicError(error);
223
+ throw wasmPanicError(error, __mdreamTakePanicMessage);
45
224
  }
46
225
  }
47
226
  finish() {
227
+ let markdown;
48
228
  try {
49
- return this._inner.finish();
229
+ const held = this._carry.flush();
230
+ markdown = held ? this._inner.processChunk(held) + this._inner.finish() : this._inner.finish();
50
231
  } catch (error) {
51
- throw wasmPanicError(error);
232
+ throw wasmPanicError(error, __mdreamTakePanicMessage);
52
233
  }
234
+ if (this._callbacks.frontmatterCallback || this._callbacks.extractionHandlers) deliverPluginData(this._inner.takeData(), this._callbacks);
235
+ return markdown;
53
236
  }
54
237
  };
55
- async function* streamHtmlToMarkdown(htmlStream, options) {
238
+ async function* streamHtmlToMarkdown(htmlStream, options = {}) {
56
239
  if (!htmlStream) throw new Error("Invalid HTML stream provided");
57
240
  await ensureInit();
58
- const stream = new MarkdownStream$1(options || {});
59
- const reader = htmlStream.getReader();
60
- const decoder = new TextDecoder();
241
+ const resolved = resolveOptions(options);
242
+ const carry = createSurrogateCarry();
243
+ let stream;
61
244
  try {
62
- while (true) {
63
- const { done, value } = await reader.read();
64
- if (done) break;
65
- const chunk = typeof value === "string" ? decoder.decode() + value : decoder.decode(value, { stream: true });
66
- const processed = stream.processChunk(chunk);
67
- if (processed) yield processed;
68
- }
69
- const decoderTail = decoder.decode();
70
- if (decoderTail) {
71
- const processed = stream.processChunk(decoderTail);
72
- if (processed) yield processed;
73
- }
74
- const final_ = stream.finish();
75
- if (final_) yield final_;
245
+ stream = new MarkdownStream$1(resolved.napiOpts);
76
246
  } catch (error) {
77
- throw wasmPanicError(error);
78
- } finally {
79
- reader.releaseLock();
247
+ throw wasmPanicError(error, __mdreamTakePanicMessage);
80
248
  }
249
+ yield* pumpStream({
250
+ processChunk: (chunk) => stream.processChunk(carry.take(chunk)),
251
+ processChunkBytes: (chunk) => {
252
+ const held = carry.flush();
253
+ return held ? stream.processChunk(held) + stream.processChunkBytes(chunk) : stream.processChunkBytes(chunk);
254
+ },
255
+ finish: () => {
256
+ const held = carry.flush();
257
+ return held ? stream.processChunk(held) + stream.finish() : stream.finish();
258
+ },
259
+ takeData: () => stream.takeData()
260
+ }, htmlStream, resolved, { mapError: (error) => wasmPanicError(error, __mdreamTakePanicMessage) });
81
261
  }
82
262
  export { MarkdownStream, createMarkdownStream, htmlToMarkdown, streamHtmlToMarkdown };