mdream 1.7.1 → 2.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -51
- package/dist/browser.d.mts +101 -7
- package/dist/browser.mjs +213 -33
- package/dist/edge.d.mts +96 -6
- package/dist/edge.mjs +114 -43
- package/dist/iife.js +232 -23
- package/dist/index.d.mts +15 -12
- package/dist/index.mjs +68 -17
- package/dist/wasm.d.mts +100 -0
- package/dist/wasm.mjs +140 -0
- package/dist/worker.d.mts +11 -8
- package/dist/worker.mjs +121 -4
- package/napi/index.d.mts +20 -0
- package/napi/index.d.ts +20 -0
- package/napi/index.mjs +47 -33
- package/package.json +33 -22
- package/skills/mdream/SKILL.md +124 -0
- package/skills/mdream/references/runtimes.md +49 -0
- package/wasm/mdream_edge.d.ts +27 -0
- package/wasm/mdream_edge.js +91 -19
- package/wasm/mdream_edge_bg.wasm +0 -0
- package/wasm/mdream_edge_bg.wasm.d.ts +3 -0
- package/wasm/package.json +1 -1
- package/wasm-bundler/mdream_edge.d.ts +24 -0
- package/wasm-bundler/mdream_edge.js +6 -2
- package/wasm-bundler/mdream_edge_bg.js +91 -19
- package/wasm-bundler/mdream_edge_bg.wasm +0 -0
- package/wasm-bundler/mdream_edge_bg.wasm.d.ts +3 -0
- package/wasm-bundler/package.json +1 -1
package/README.md
CHANGED
|
@@ -3,6 +3,13 @@
|
|
|
3
3
|
[](https://npmjs.com/package/mdream)
|
|
4
4
|
[](https://npm.chart.dev/mdream)
|
|
5
5
|
[](https://github.com/harlan-zw/mdream/blob/main/LICENSE.md)
|
|
6
|
+
<a href="https://skilld.dev/gh/harlan-zw/mdream">
|
|
7
|
+
<picture>
|
|
8
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://skilld.dev/b/harlan-zw/mdream?theme=dark">
|
|
9
|
+
<source media="(prefers-color-scheme: light)" srcset="https://skilld.dev/b/harlan-zw/mdream?theme=light">
|
|
10
|
+
<img alt="Skill repository on skilld.dev" src="https://skilld.dev/b/harlan-zw/mdream?theme=light">
|
|
11
|
+
</picture>
|
|
12
|
+
</a>
|
|
6
13
|
|
|
7
14
|
## Installation
|
|
8
15
|
|
|
@@ -17,6 +24,9 @@ pnpm add mdream
|
|
|
17
24
|
yarn add mdream
|
|
18
25
|
```
|
|
19
26
|
|
|
27
|
+
> [!TIP]
|
|
28
|
+
> Using an AI agent? Get the mdream Skill on [skilld.dev/gh/harlan-zw/mdream](https://skilld.dev/gh/harlan-zw/mdream).
|
|
29
|
+
|
|
20
30
|
For the JavaScript-only engine (hook-based plugins, splitter, pure HTML parser):
|
|
21
31
|
|
|
22
32
|
```bash
|
|
@@ -89,7 +99,7 @@ externals: ['mdream']
|
|
|
89
99
|
|
|
90
100
|
### `htmlToMarkdown()`
|
|
91
101
|
|
|
92
|
-
Converts a complete HTML string to Markdown, plain text, or HTML
|
|
102
|
+
Converts a complete HTML string to Markdown, plain text, or HTML. It is synchronous in Node and on edge runtimes.
|
|
93
103
|
|
|
94
104
|
**Rust engine** (`mdream`):
|
|
95
105
|
|
|
@@ -97,6 +107,9 @@ Converts a complete HTML string to Markdown, plain text, or HTML synchronously.
|
|
|
97
107
|
import { htmlToMarkdown } from 'mdream'
|
|
98
108
|
|
|
99
109
|
function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string
|
|
110
|
+
|
|
111
|
+
// Browser bundles and the CDN script load the WASM binary first
|
|
112
|
+
function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): Promise<string>
|
|
100
113
|
```
|
|
101
114
|
|
|
102
115
|
**JS engine** (`@mdream/js`):
|
|
@@ -132,6 +145,8 @@ const html = htmlToMarkdown('<h1>Hello <strong>World</strong></h1>', {
|
|
|
132
145
|
|
|
133
146
|
Converts an HTML `ReadableStream` to Markdown, plain text, or HTML incrementally. Returns an `AsyncIterable<string>` that yields output chunks as they are processed.
|
|
134
147
|
|
|
148
|
+
It takes the same options as `htmlToMarkdown()`. The `frontmatter` callback and the `extraction` handlers run once, after the stream ends and before the last chunk is yielded.
|
|
149
|
+
|
|
135
150
|
|
|
136
151
|
```ts
|
|
137
152
|
import { streamHtmlToMarkdown } from 'mdream'
|
|
@@ -176,7 +191,7 @@ import { htmlToMarkdown } from '@mdream/js'
|
|
|
176
191
|
import { htmlToMarkdown } from 'mdream'
|
|
177
192
|
```
|
|
178
193
|
|
|
179
|
-
Both engines
|
|
194
|
+
Both engines support the same declarative plugins, but the option shape differs. The Rust engine takes `minimal`, `frontmatter`, `isolateMain`, `tailwind`, `filter`, `extraction`, and `tagOverrides` at the top level. The JS engine reads the plugins from `options.plugins` and has no `minimal` option: use `withMinimalPreset()` from `@mdream/js/preset/minimal`. If the JS engine gets one of these options at the top level, it throws a `TypeError` that names the fix. The JS engine also supports `hooks` for imperative plugin transforms.
|
|
180
195
|
|
|
181
196
|
## Options
|
|
182
197
|
|
|
@@ -216,7 +231,7 @@ interface MdreamOptions {
|
|
|
216
231
|
tailwind?: boolean
|
|
217
232
|
|
|
218
233
|
/** Filter elements by CSS selectors. Default when minimal: excludes form, nav, footer, etc. */
|
|
219
|
-
filter?: { include?: string[], exclude?: string[], processChildren?: boolean }
|
|
234
|
+
filter?: false | { include?: string[], exclude?: string[], processChildren?: boolean }
|
|
220
235
|
|
|
221
236
|
/** Extract elements matching CSS selectors during conversion. */
|
|
222
237
|
extraction?: Record<string, (element: ExtractedElement) => void>
|
|
@@ -263,7 +278,7 @@ interface EngineOptions {
|
|
|
263
278
|
}
|
|
264
279
|
|
|
265
280
|
interface BuiltinPlugins {
|
|
266
|
-
filter?: { include?: (string | number)[], exclude?: (string | number)[], processChildren?: boolean }
|
|
281
|
+
filter?: false | { include?: (string | number)[], exclude?: (string | number)[], processChildren?: boolean }
|
|
267
282
|
frontmatter?: boolean | ((fm: Record<string, string>) => void) | FrontmatterConfig
|
|
268
283
|
isolateMain?: boolean
|
|
269
284
|
tailwind?: boolean
|
|
@@ -336,7 +351,7 @@ Override how specific HTML tags are rendered in Markdown. String values act as a
|
|
|
336
351
|
>
|
|
337
352
|
> ```ts
|
|
338
353
|
> htmlToMarkdown('<my-em>hi</my-em>', { tagOverrides: { 'my-em': 'em' } })
|
|
339
|
-
> // → "
|
|
354
|
+
> // → "*hi*"
|
|
340
355
|
> ```
|
|
341
356
|
|
|
342
357
|
```ts
|
|
@@ -375,12 +390,12 @@ const markdown = htmlToMarkdown(html, {
|
|
|
375
390
|
})
|
|
376
391
|
```
|
|
377
392
|
|
|
378
|
-
**Output is GitHub Flavored Markdown.** Mdream emits a fixed GFM dialect tuned for LLM input: ATX headings (`#`), fenced code blocks (` ``` `), `-` bullets,
|
|
393
|
+
**Output is GitHub Flavored Markdown.** Mdream emits a fixed GFM dialect tuned for LLM input: ATX headings (`#`), fenced code blocks (` ``` `), `-` bullets, `*` emphasis, `**` strong, `---` horizontal rules, inline links. These are not configurable. For simple delimiter swaps you can use `tagOverrides`:
|
|
379
394
|
|
|
380
395
|
```ts
|
|
381
396
|
htmlToMarkdown(html, {
|
|
382
397
|
tagOverrides: {
|
|
383
|
-
em: { enter: '
|
|
398
|
+
em: { enter: '_', exit: '_', isInline: true }, // *x* → _x_
|
|
384
399
|
strong: { enter: '__', exit: '__', isInline: true }, // **x** → __x__
|
|
385
400
|
hr: { enter: '* * *', exit: '' }, // --- → * * *
|
|
386
401
|
},
|
|
@@ -414,6 +429,16 @@ The `minimal` preset enables the following plugins together:
|
|
|
414
429
|
- **filter**: Excludes `form`, `fieldset`, `object`, `embed`, `footer`, `aside`, `iframe`, `input`, `textarea`, `select`, `button`, `nav`
|
|
415
430
|
- **clean**: All post-processing cleanup enabled
|
|
416
431
|
|
|
432
|
+
A `filter` passed with `minimal` adds to the preset's excludes. It does not replace them. To turn the preset's filter off, pass `filter: false`. `withMinimalPreset()` in the JS engine works the same way.
|
|
433
|
+
|
|
434
|
+
```ts
|
|
435
|
+
// Excludes h1 as well as form, nav, footer, and the rest
|
|
436
|
+
htmlToMarkdown(html, { minimal: true, filter: { exclude: ['h1'] } })
|
|
437
|
+
|
|
438
|
+
// Keeps forms and nav
|
|
439
|
+
htmlToMarkdown(html, { minimal: true, filter: false })
|
|
440
|
+
```
|
|
441
|
+
|
|
417
442
|
**Rust engine:**
|
|
418
443
|
|
|
419
444
|
```ts
|
|
@@ -436,13 +461,13 @@ const markdown = htmlToMarkdown(html, withMinimalPreset({
|
|
|
436
461
|
}))
|
|
437
462
|
```
|
|
438
463
|
|
|
439
|
-
`withMinimalPreset()` returns an `EngineOptions` object with all plugin defaults applied. You can override individual plugins:
|
|
464
|
+
`withMinimalPreset()` returns an `EngineOptions` object with all plugin defaults applied. You can override individual plugins. A `filter` adds to the preset's excludes:
|
|
440
465
|
|
|
441
466
|
```ts
|
|
442
467
|
const markdown = htmlToMarkdown(html, withMinimalPreset({
|
|
443
468
|
plugins: {
|
|
444
469
|
frontmatter: false,
|
|
445
|
-
filter: { exclude: ['
|
|
470
|
+
filter: { exclude: ['.cookie-banner'] },
|
|
446
471
|
},
|
|
447
472
|
}))
|
|
448
473
|
```
|
|
@@ -927,6 +952,15 @@ The CLI reads HTML from stdin and writes Markdown, plain text, or HTML to stdout
|
|
|
927
952
|
|
|
928
953
|
## Browser and Edge Usage
|
|
929
954
|
|
|
955
|
+
Every entry takes the same `MdreamOptions` and produces the same Markdown string. In Node and edge runtimes, `htmlToMarkdown` is synchronous. In browser bundles, the CDN script, and `mdream/worker`, it returns `Promise<string>`, because the WASM binary loads first. Each export condition ships its own types, so TypeScript shows the right return type when it resolves the `browser` condition (for example, with `customConditions: ["browser"]`).
|
|
956
|
+
|
|
957
|
+
```ts
|
|
958
|
+
import { htmlToMarkdown } from 'mdream'
|
|
959
|
+
|
|
960
|
+
// In a browser bundle
|
|
961
|
+
const markdown = await htmlToMarkdown('<h1>Hello</h1>', { minimal: true })
|
|
962
|
+
```
|
|
963
|
+
|
|
930
964
|
### Edge / Cloudflare Workers
|
|
931
965
|
|
|
932
966
|
For edge runtimes (Cloudflare Workers, Vercel Edge), `mdream` automatically selects the WASM build via export conditions (`workerd`, `edge-light`). Both `htmlToMarkdown` and `streamHtmlToMarkdown` are available:
|
|
@@ -957,61 +991,30 @@ catch (error) {
|
|
|
957
991
|
}
|
|
958
992
|
```
|
|
959
993
|
|
|
960
|
-
If your toolchain doesn't resolve the export conditions, the
|
|
994
|
+
If your toolchain doesn't resolve the export conditions, the wasm-bindgen build (web target) is exposed at `mdream/wasm` for manual initialization. Its `htmlToMarkdown` takes the same `MdreamOptions` as the `mdream` entry. Its `MarkdownStream` runs the callbacks in `finish()`.
|
|
961
995
|
|
|
962
996
|
```ts
|
|
963
997
|
import init, { htmlToMarkdown } from 'mdream/wasm'
|
|
964
998
|
import wasmModule from 'mdream/wasm/mdream_edge_bg.wasm'
|
|
965
999
|
|
|
966
1000
|
await init({ module_or_path: wasmModule })
|
|
967
|
-
const markdown = htmlToMarkdown('<h1>Hello</h1>', {})
|
|
968
|
-
```
|
|
969
|
-
|
|
970
|
-
You can also import the edge entry point directly:
|
|
971
|
-
|
|
972
|
-
```ts
|
|
973
|
-
import { htmlToMarkdown } from 'mdream/worker'
|
|
1001
|
+
const markdown = htmlToMarkdown('<h1>Hello</h1>', { minimal: true })
|
|
974
1002
|
```
|
|
975
1003
|
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
```ts
|
|
979
|
-
import { htmlToMarkdown, initWorker, terminateWorker } from 'mdream/worker'
|
|
980
|
-
|
|
981
|
-
// Initialize once with the WASM URL
|
|
982
|
-
await initWorker('https://cdn.example.com/mdream_edge_bg.wasm')
|
|
983
|
-
|
|
984
|
-
// Convert (returns Promise<string>)
|
|
985
|
-
const markdown = await htmlToMarkdown('<h1>Hello</h1>')
|
|
986
|
-
|
|
987
|
-
// Clean up when done
|
|
988
|
-
terminateWorker()
|
|
989
|
-
```
|
|
1004
|
+
`mdream/worker` is not an edge entry. It runs the conversion in a browser Web Worker; see [Web Worker](#web-worker).
|
|
990
1005
|
|
|
991
1006
|
### Browser CDN (IIFE)
|
|
992
1007
|
|
|
993
|
-
Use mdream directly via CDN with no build step.
|
|
1008
|
+
Use mdream directly via CDN with no build step. The script inlines the WASM binary and initializes it on load, so `window.mdream.htmlToMarkdown()` is ready at once. It takes the same `MdreamOptions` as the `mdream` entry and returns `Promise<string>`, the same as the browser bundle.
|
|
994
1009
|
|
|
995
1010
|
```html
|
|
996
1011
|
<script src="https://unpkg.com/mdream/dist/iife.js"></script>
|
|
997
1012
|
<script>
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
console.log(markdown) // # Hello\n\nWorld
|
|
1013
|
+
window.mdream.htmlToMarkdown('<h1>Hello</h1><p>World</p>')
|
|
1014
|
+
.then(markdown => console.log(markdown)) // # Hello\n\nWorld
|
|
1001
1015
|
</script>
|
|
1002
1016
|
```
|
|
1003
1017
|
|
|
1004
|
-
You can pass a custom WASM URL or `ArrayBuffer` to `init()`:
|
|
1005
|
-
|
|
1006
|
-
```js
|
|
1007
|
-
// Custom URL
|
|
1008
|
-
await window.mdream.init('https://cdn.example.com/mdream_edge_bg.wasm')
|
|
1009
|
-
|
|
1010
|
-
// Pre-loaded ArrayBuffer
|
|
1011
|
-
const wasmBytes = await fetch('/wasm/mdream_edge_bg.wasm').then(r => r.arrayBuffer())
|
|
1012
|
-
await window.mdream.init(wasmBytes)
|
|
1013
|
-
```
|
|
1014
|
-
|
|
1015
1018
|
**CDN Options:**
|
|
1016
1019
|
- **unpkg**: `https://unpkg.com/mdream/dist/iife.js`
|
|
1017
1020
|
- **jsDelivr**: `https://cdn.jsdelivr.net/npm/mdream/dist/iife.js`
|
|
@@ -1025,7 +1028,8 @@ import { htmlToMarkdown, initWorker, terminateWorker } from 'mdream/worker'
|
|
|
1025
1028
|
|
|
1026
1029
|
await initWorker('/path/to/mdream_edge_bg.wasm')
|
|
1027
1030
|
|
|
1028
|
-
|
|
1031
|
+
// Takes the same options as the mdream entry. Callbacks run on this thread.
|
|
1032
|
+
const markdown = await htmlToMarkdown('<h1>Hello</h1>', { minimal: true })
|
|
1029
1033
|
|
|
1030
1034
|
// Clean up
|
|
1031
1035
|
terminateWorker()
|
|
@@ -1051,10 +1055,10 @@ if (article) {
|
|
|
1051
1055
|
|
|
1052
1056
|
## llms.txt Generation
|
|
1053
1057
|
|
|
1054
|
-
For llms.txt artifact generation, use
|
|
1058
|
+
For llms.txt artifact generation, use `@mdream/js/llms-txt` from the `@mdream/js` package. It accepts pre-converted Markdown and generates `llms.txt` and `llms-full.txt` artifacts.
|
|
1055
1059
|
|
|
1056
1060
|
```ts
|
|
1057
|
-
import { generateLlmsTxtArtifacts } from '@mdream/llms-txt'
|
|
1061
|
+
import { generateLlmsTxtArtifacts } from '@mdream/js/llms-txt'
|
|
1058
1062
|
import { htmlToMarkdown } from 'mdream'
|
|
1059
1063
|
|
|
1060
1064
|
const result = await generateLlmsTxtArtifacts({
|
|
@@ -1076,8 +1080,7 @@ console.log(result.llmsFullTxt) // llms-full.txt content
|
|
|
1076
1080
|
| Package | Description |
|
|
1077
1081
|
|---------|-------------|
|
|
1078
1082
|
| [`mdream`](https://npmjs.com/package/mdream) | Core HTML to Markdown converter (Rust + WASM engine) |
|
|
1079
|
-
| [`@mdream/js`](https://npmjs.com/package/@mdream/js) | JavaScript engine with hook-based plugins and
|
|
1080
|
-
| [`@mdream/llms-txt`](https://github.com/harlan-zw/mdream/tree/main/packages/llms-txt) | Engine-agnostic llms.txt artifact generation |
|
|
1083
|
+
| [`@mdream/js`](https://npmjs.com/package/@mdream/js) | JavaScript engine with hook-based plugins, splitter, and llms.txt generation (`@mdream/js/llms-txt`) |
|
|
1081
1084
|
| [`@mdream/crawl`](https://github.com/harlan-zw/mdream/tree/main/packages/crawl) | Site-wide crawler for llms.txt generation |
|
|
1082
1085
|
| [`@mdream/vite`](https://github.com/harlan-zw/mdream/tree/main/packages/vite) | Vite plugin integration |
|
|
1083
1086
|
| [`@mdream/nuxt`](https://github.com/harlan-zw/mdream/tree/main/packages/nuxt) | Nuxt module integration |
|
package/dist/browser.d.mts
CHANGED
|
@@ -1,11 +1,105 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
1
|
+
interface CleanOptions {
|
|
2
|
+
/** Strip tracking query parameters (utm_*, fbclid, gclid, etc.) from URLs */
|
|
3
|
+
urls?: boolean;
|
|
4
|
+
/** Strip fragment-only links that don't match any heading in the output */
|
|
5
|
+
fragments?: boolean;
|
|
6
|
+
/** Strip links with meaningless hrefs (#, javascript:void(0)) → plain text */
|
|
7
|
+
emptyLinks?: boolean;
|
|
8
|
+
/** Collapse 3+ consecutive blank lines to 2 */
|
|
9
|
+
blankLines?: boolean;
|
|
10
|
+
/** Strip links where text equals URL: [https://x.com](https://x.com) → https://x.com */
|
|
11
|
+
redundantLinks?: boolean;
|
|
12
|
+
/** Strip self-referencing heading anchors: ## [Title](#title) → ## Title */
|
|
13
|
+
selfLinkHeadings?: boolean;
|
|
14
|
+
/** Strip images with no alt text (decorative/tracking pixels) */
|
|
15
|
+
emptyImages?: boolean;
|
|
16
|
+
/** Drop links that produce no visible text: [](url) → nothing */
|
|
17
|
+
emptyLinkText?: boolean;
|
|
18
|
+
}
|
|
19
|
+
interface ExtractedElement {
|
|
20
|
+
selector: string;
|
|
21
|
+
tagName: string;
|
|
22
|
+
textContent: string;
|
|
23
|
+
attributes: Record<string, string>;
|
|
24
|
+
}
|
|
25
|
+
interface FrontmatterConfig {
|
|
26
|
+
additionalFields?: Record<string, string>;
|
|
27
|
+
metaFields?: string[];
|
|
28
|
+
/** Callback to receive structured frontmatter data after conversion */
|
|
29
|
+
onExtract?: (frontmatter: Record<string, string>) => void;
|
|
30
|
+
}
|
|
31
|
+
interface TagOverride {
|
|
32
|
+
enter?: string;
|
|
33
|
+
exit?: string;
|
|
34
|
+
spacing?: number[];
|
|
35
|
+
isInline?: boolean;
|
|
36
|
+
isSelfClosing?: boolean;
|
|
37
|
+
collapsesInnerWhiteSpace?: boolean;
|
|
38
|
+
alias?: string;
|
|
39
|
+
}
|
|
40
|
+
interface MdreamOptions {
|
|
41
|
+
/** Origin URL for resolving relative image paths and internal links. */
|
|
42
|
+
origin?: string;
|
|
43
|
+
/**
|
|
44
|
+
* Clean up the markdown output. Pass `true` for all cleanup or an object
|
|
45
|
+
* to enable specific features. Both `htmlToMarkdown` and
|
|
46
|
+
* `streamHtmlToMarkdown` apply it.
|
|
47
|
+
*/
|
|
48
|
+
clean?: boolean | CleanOptions;
|
|
49
|
+
/** Enable minimal preset (frontmatter, isolateMain, tailwind, filter). Default: false */
|
|
50
|
+
minimal?: boolean;
|
|
51
|
+
/**
|
|
52
|
+
* Extract frontmatter from HTML head.
|
|
53
|
+
* - `true`: enable with defaults
|
|
54
|
+
* - `(fm) => void`: enable and receive structured data via callback
|
|
55
|
+
* - `FrontmatterConfig`: enable with config options and optional callback
|
|
56
|
+
*/
|
|
57
|
+
frontmatter?: boolean | ((frontmatter: Record<string, string>) => void) | FrontmatterConfig;
|
|
58
|
+
/** Isolate main content area. Default when minimal: true */
|
|
59
|
+
isolateMain?: boolean;
|
|
60
|
+
/** Convert Tailwind utility classes. Default when minimal: true */
|
|
61
|
+
tailwind?: boolean;
|
|
62
|
+
/**
|
|
63
|
+
* Filter elements by CSS selector. `minimal` excludes form, nav, footer, and
|
|
64
|
+
* similar; a filter passed with `minimal` adds to those excludes. `false`
|
|
65
|
+
* turns filtering off, including the `minimal` one.
|
|
66
|
+
*/
|
|
67
|
+
filter?: false | {
|
|
68
|
+
include?: string[];
|
|
69
|
+
exclude?: string[];
|
|
70
|
+
processChildren?: boolean;
|
|
71
|
+
};
|
|
72
|
+
/** Extract elements matching CSS selectors */
|
|
73
|
+
extraction?: Record<string, (element: ExtractedElement) => void>;
|
|
74
|
+
/** Tag overrides. String values act as aliases */
|
|
75
|
+
tagOverrides?: Record<string, TagOverride | string>;
|
|
76
|
+
/**
|
|
77
|
+
* Hard-wrap prose at this many characters, breaking on word boundaries.
|
|
78
|
+
* Applied inline during conversion (zero-cost when unset). Code blocks,
|
|
79
|
+
* tables, and headings are never wrapped. `0` disables wrapping.
|
|
80
|
+
*/
|
|
81
|
+
wrapWidth?: number;
|
|
82
|
+
/**
|
|
83
|
+
* Output format. Defaults to `markdown`; use `text` for readable plain text,
|
|
84
|
+
* or `html` for allowlisted semantic HTML.
|
|
85
|
+
*/
|
|
86
|
+
format?: 'markdown' | 'text' | 'html';
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Browser builds fetch the WASM binary on first use, so this returns a
|
|
90
|
+
* Promise. The `browser` export condition ships types that say so.
|
|
91
|
+
*/
|
|
92
|
+
export declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): Promise<string>;
|
|
93
|
+
export declare function createMarkdownStream(options?: Partial<MdreamOptions>): Promise<MarkdownStream>;
|
|
94
|
+
/** Streaming converter. Runs the frontmatter and extraction callbacks in `finish()`. */
|
|
95
|
+
export declare class MarkdownStream {
|
|
5
96
|
private _inner;
|
|
6
|
-
|
|
97
|
+
private _carry;
|
|
98
|
+
private _callbacks;
|
|
99
|
+
constructor(options?: Partial<MdreamOptions>);
|
|
7
100
|
processChunk(chunk: string): string;
|
|
101
|
+
processChunkBytes(chunk: Uint8Array): string;
|
|
8
102
|
finish(): string;
|
|
9
103
|
}
|
|
10
|
-
declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?:
|
|
11
|
-
export {
|
|
104
|
+
export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
|
|
105
|
+
export type { CleanOptions, ExtractedElement, FrontmatterConfig, MdreamOptions, TagOverride };
|
package/dist/browser.mjs
CHANGED
|
@@ -1,5 +1,169 @@
|
|
|
1
1
|
import init, { MarkdownStream as MarkdownStream$1, __mdreamTakePanicMessage, htmlToMarkdownResult } from "../wasm/mdream_edge.js";
|
|
2
|
-
|
|
2
|
+
const MINIMAL_FILTER_EXCLUDE = [
|
|
3
|
+
"form",
|
|
4
|
+
"fieldset",
|
|
5
|
+
"object",
|
|
6
|
+
"embed",
|
|
7
|
+
"footer",
|
|
8
|
+
"aside",
|
|
9
|
+
"iframe",
|
|
10
|
+
"input",
|
|
11
|
+
"textarea",
|
|
12
|
+
"select",
|
|
13
|
+
"button",
|
|
14
|
+
"nav"
|
|
15
|
+
];
|
|
16
|
+
const CLEAN_ALL = {
|
|
17
|
+
urls: true,
|
|
18
|
+
fragments: true,
|
|
19
|
+
emptyLinks: true,
|
|
20
|
+
redundantLinks: true,
|
|
21
|
+
selfLinkHeadings: true,
|
|
22
|
+
emptyImages: true,
|
|
23
|
+
emptyLinkText: true
|
|
24
|
+
};
|
|
25
|
+
function resolveCleanConfig(options, minimal) {
|
|
26
|
+
const cleanUrls = options.cleanUrls === true;
|
|
27
|
+
let cleanOpt = options.clean;
|
|
28
|
+
if (cleanOpt === void 0) {
|
|
29
|
+
if (!minimal) return { cleanUrls };
|
|
30
|
+
cleanOpt = true;
|
|
31
|
+
}
|
|
32
|
+
if (!cleanOpt) return { cleanUrls };
|
|
33
|
+
const resolved = cleanOpt === true ? CLEAN_ALL : cleanOpt;
|
|
34
|
+
return {
|
|
35
|
+
cleanUrls: cleanUrls || resolved.urls === true,
|
|
36
|
+
clean: resolved
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
function resolveFrontmatter(opt) {
|
|
40
|
+
if (typeof opt === "function") return {
|
|
41
|
+
config: {},
|
|
42
|
+
callback: opt
|
|
43
|
+
};
|
|
44
|
+
if (opt && typeof opt === "object") {
|
|
45
|
+
const { onExtract, ...config } = opt;
|
|
46
|
+
return {
|
|
47
|
+
config,
|
|
48
|
+
callback: onExtract
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
return { config: {} };
|
|
52
|
+
}
|
|
53
|
+
function resolveOptions(options) {
|
|
54
|
+
if (Array.isArray(options.plugins)) throw new TypeError("Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://mdream.dev/v1-migration#custom-plugins");
|
|
55
|
+
const minimal = options.minimal === true;
|
|
56
|
+
const plugins = options.plugins ? { ...options.plugins } : {};
|
|
57
|
+
let frontmatterCallback;
|
|
58
|
+
const frontmatterDisabled = options.frontmatter === false;
|
|
59
|
+
if (minimal ? !frontmatterDisabled : !!options.frontmatter) {
|
|
60
|
+
const fm = resolveFrontmatter(options.frontmatter);
|
|
61
|
+
plugins.frontmatter = fm.config;
|
|
62
|
+
frontmatterCallback = fm.callback;
|
|
63
|
+
} else if (frontmatterDisabled) delete plugins.frontmatter;
|
|
64
|
+
const isolateMainDisabled = options.isolateMain === false;
|
|
65
|
+
if (minimal ? !isolateMainDisabled : options.isolateMain) plugins.isolateMain = true;
|
|
66
|
+
else if (isolateMainDisabled) delete plugins.isolateMain;
|
|
67
|
+
const tailwindDisabled = options.tailwind === false;
|
|
68
|
+
if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
|
|
69
|
+
else if (tailwindDisabled) delete plugins.tailwind;
|
|
70
|
+
if (options.filter === false) delete plugins.filter;
|
|
71
|
+
else if (minimal) plugins.filter = {
|
|
72
|
+
...options.filter,
|
|
73
|
+
exclude: [...MINIMAL_FILTER_EXCLUDE, ...options.filter?.exclude ?? []]
|
|
74
|
+
};
|
|
75
|
+
else if (options.filter) plugins.filter = options.filter;
|
|
76
|
+
const extraction = options.extraction;
|
|
77
|
+
let extractionHandlers;
|
|
78
|
+
if (extraction) {
|
|
79
|
+
plugins.extraction = { selectors: Object.keys(extraction) };
|
|
80
|
+
extractionHandlers = extraction;
|
|
81
|
+
}
|
|
82
|
+
const tagOverrides = options.tagOverrides;
|
|
83
|
+
if (tagOverrides) {
|
|
84
|
+
const overrides = {};
|
|
85
|
+
for (const tag in tagOverrides) {
|
|
86
|
+
const value = tagOverrides[tag];
|
|
87
|
+
if (value) overrides[tag] = typeof value === "string" ? { alias: value } : value;
|
|
88
|
+
}
|
|
89
|
+
plugins.tagOverrides = overrides;
|
|
90
|
+
}
|
|
91
|
+
const { cleanUrls, clean } = resolveCleanConfig(options, minimal);
|
|
92
|
+
return {
|
|
93
|
+
napiOpts: {
|
|
94
|
+
origin: options.origin,
|
|
95
|
+
cleanUrls,
|
|
96
|
+
clean,
|
|
97
|
+
plugins,
|
|
98
|
+
wrapWidth: options.wrapWidth,
|
|
99
|
+
format: options.format
|
|
100
|
+
},
|
|
101
|
+
extractionHandlers,
|
|
102
|
+
frontmatterCallback
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
function deliverPluginData(data, { frontmatterCallback, extractionHandlers }) {
|
|
106
|
+
if (data.frontmatter && frontmatterCallback) frontmatterCallback(data.frontmatter);
|
|
107
|
+
if (data.extracted?.length && extractionHandlers) for (const element of data.extracted) extractionHandlers[element.selector]?.(element);
|
|
108
|
+
}
|
|
109
|
+
function hasCallbacks({ frontmatterCallback, extractionHandlers }) {
|
|
110
|
+
return !!frontmatterCallback || !!extractionHandlers;
|
|
111
|
+
}
|
|
112
|
+
function convertResult(convert, html, options = {}) {
|
|
113
|
+
const resolved = resolveOptions(options);
|
|
114
|
+
const result = convert(html, resolved.napiOpts);
|
|
115
|
+
deliverPluginData(result, resolved);
|
|
116
|
+
return result;
|
|
117
|
+
}
|
|
118
|
+
async function* pumpStream(stream, htmlStream, callbacks, { decodeInJs = false, mapError = (error) => error } = {}) {
|
|
119
|
+
const reader = htmlStream.getReader();
|
|
120
|
+
const decoder = decodeInJs ? new TextDecoder() : void 0;
|
|
121
|
+
try {
|
|
122
|
+
while (true) {
|
|
123
|
+
const { done, value } = await reader.read();
|
|
124
|
+
if (done) break;
|
|
125
|
+
let processed;
|
|
126
|
+
if (typeof value === "string") processed = stream.processChunk(decoder ? decoder.decode() + value : value);
|
|
127
|
+
else processed = decoder ? stream.processChunk(decoder.decode(value, { stream: true })) : stream.processChunkBytes(value);
|
|
128
|
+
if (processed) yield processed;
|
|
129
|
+
}
|
|
130
|
+
const decoderTail = decoder?.decode();
|
|
131
|
+
if (decoderTail) {
|
|
132
|
+
const processed = stream.processChunk(decoderTail);
|
|
133
|
+
if (processed) yield processed;
|
|
134
|
+
}
|
|
135
|
+
const final_ = stream.finish();
|
|
136
|
+
if (hasCallbacks(callbacks)) deliverPluginData(stream.takeData(), callbacks);
|
|
137
|
+
if (final_) yield final_;
|
|
138
|
+
} catch (error) {
|
|
139
|
+
throw mapError(error);
|
|
140
|
+
} finally {
|
|
141
|
+
reader.releaseLock();
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
function createSurrogateCarry() {
|
|
145
|
+
let held = "";
|
|
146
|
+
return {
|
|
147
|
+
take(chunk) {
|
|
148
|
+
if (held) {
|
|
149
|
+
chunk = held + chunk;
|
|
150
|
+
held = "";
|
|
151
|
+
}
|
|
152
|
+
const last = chunk.charCodeAt(chunk.length - 1);
|
|
153
|
+
if (last >= 55296 && last <= 56319) {
|
|
154
|
+
held = chunk.slice(-1);
|
|
155
|
+
return chunk.slice(0, -1);
|
|
156
|
+
}
|
|
157
|
+
return chunk;
|
|
158
|
+
},
|
|
159
|
+
flush() {
|
|
160
|
+
const value = held;
|
|
161
|
+
held = "";
|
|
162
|
+
return value;
|
|
163
|
+
}
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
function wasmPanicError(error, takeMessage) {
|
|
3
167
|
if (!(error instanceof WebAssembly.RuntimeError)) return error;
|
|
4
168
|
let message;
|
|
5
169
|
try {
|
|
@@ -16,67 +180,83 @@ function ensureInit() {
|
|
|
16
180
|
return _initPromise;
|
|
17
181
|
}
|
|
18
182
|
ensureInit();
|
|
19
|
-
|
|
20
|
-
await ensureInit();
|
|
183
|
+
function convert(html, napiOpts) {
|
|
21
184
|
try {
|
|
22
|
-
return htmlToMarkdownResult(html,
|
|
185
|
+
return htmlToMarkdownResult(html, napiOpts);
|
|
23
186
|
} catch (error) {
|
|
24
|
-
throw wasmPanicError(error);
|
|
187
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
25
188
|
}
|
|
26
189
|
}
|
|
190
|
+
async function htmlToMarkdown(html, options = {}) {
|
|
191
|
+
await ensureInit();
|
|
192
|
+
return convertResult(convert, html, options).markdown || "";
|
|
193
|
+
}
|
|
27
194
|
async function createMarkdownStream(options) {
|
|
28
195
|
await ensureInit();
|
|
29
196
|
return new MarkdownStream(options);
|
|
30
197
|
}
|
|
31
198
|
var MarkdownStream = class {
|
|
32
199
|
_inner;
|
|
33
|
-
|
|
200
|
+
_carry = createSurrogateCarry();
|
|
201
|
+
_callbacks;
|
|
202
|
+
constructor(options = {}) {
|
|
203
|
+
const resolved = resolveOptions(options);
|
|
204
|
+
this._callbacks = resolved;
|
|
34
205
|
try {
|
|
35
|
-
this._inner = new MarkdownStream$1(
|
|
206
|
+
this._inner = new MarkdownStream$1(resolved.napiOpts);
|
|
36
207
|
} catch (error) {
|
|
37
|
-
throw wasmPanicError(error);
|
|
208
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
38
209
|
}
|
|
39
210
|
}
|
|
40
211
|
processChunk(chunk) {
|
|
41
212
|
try {
|
|
42
|
-
return this._inner.processChunk(chunk);
|
|
213
|
+
return this._inner.processChunk(this._carry.take(chunk));
|
|
214
|
+
} catch (error) {
|
|
215
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
processChunkBytes(chunk) {
|
|
219
|
+
try {
|
|
220
|
+
const held = this._carry.flush();
|
|
221
|
+
return held ? this._inner.processChunk(held) + this._inner.processChunkBytes(chunk) : this._inner.processChunkBytes(chunk);
|
|
43
222
|
} catch (error) {
|
|
44
|
-
throw wasmPanicError(error);
|
|
223
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
45
224
|
}
|
|
46
225
|
}
|
|
47
226
|
finish() {
|
|
227
|
+
let markdown;
|
|
48
228
|
try {
|
|
49
|
-
|
|
229
|
+
const held = this._carry.flush();
|
|
230
|
+
markdown = held ? this._inner.processChunk(held) + this._inner.finish() : this._inner.finish();
|
|
50
231
|
} catch (error) {
|
|
51
|
-
throw wasmPanicError(error);
|
|
232
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
52
233
|
}
|
|
234
|
+
if (this._callbacks.frontmatterCallback || this._callbacks.extractionHandlers) deliverPluginData(this._inner.takeData(), this._callbacks);
|
|
235
|
+
return markdown;
|
|
53
236
|
}
|
|
54
237
|
};
|
|
55
|
-
async function* streamHtmlToMarkdown(htmlStream, options) {
|
|
238
|
+
async function* streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
56
239
|
if (!htmlStream) throw new Error("Invalid HTML stream provided");
|
|
57
240
|
await ensureInit();
|
|
58
|
-
const
|
|
59
|
-
const
|
|
60
|
-
|
|
241
|
+
const resolved = resolveOptions(options);
|
|
242
|
+
const carry = createSurrogateCarry();
|
|
243
|
+
let stream;
|
|
61
244
|
try {
|
|
62
|
-
|
|
63
|
-
const { done, value } = await reader.read();
|
|
64
|
-
if (done) break;
|
|
65
|
-
const chunk = typeof value === "string" ? decoder.decode() + value : decoder.decode(value, { stream: true });
|
|
66
|
-
const processed = stream.processChunk(chunk);
|
|
67
|
-
if (processed) yield processed;
|
|
68
|
-
}
|
|
69
|
-
const decoderTail = decoder.decode();
|
|
70
|
-
if (decoderTail) {
|
|
71
|
-
const processed = stream.processChunk(decoderTail);
|
|
72
|
-
if (processed) yield processed;
|
|
73
|
-
}
|
|
74
|
-
const final_ = stream.finish();
|
|
75
|
-
if (final_) yield final_;
|
|
245
|
+
stream = new MarkdownStream$1(resolved.napiOpts);
|
|
76
246
|
} catch (error) {
|
|
77
|
-
throw wasmPanicError(error);
|
|
78
|
-
} finally {
|
|
79
|
-
reader.releaseLock();
|
|
247
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
80
248
|
}
|
|
249
|
+
yield* pumpStream({
|
|
250
|
+
processChunk: (chunk) => stream.processChunk(carry.take(chunk)),
|
|
251
|
+
processChunkBytes: (chunk) => {
|
|
252
|
+
const held = carry.flush();
|
|
253
|
+
return held ? stream.processChunk(held) + stream.processChunkBytes(chunk) : stream.processChunkBytes(chunk);
|
|
254
|
+
},
|
|
255
|
+
finish: () => {
|
|
256
|
+
const held = carry.flush();
|
|
257
|
+
return held ? stream.processChunk(held) + stream.finish() : stream.finish();
|
|
258
|
+
},
|
|
259
|
+
takeData: () => stream.takeData()
|
|
260
|
+
}, htmlStream, resolved, { mapError: (error) => wasmPanicError(error, __mdreamTakePanicMessage) });
|
|
81
261
|
}
|
|
82
262
|
export { MarkdownStream, createMarkdownStream, htmlToMarkdown, streamHtmlToMarkdown };
|