mdream 1.7.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +357 -163
- package/bin/mdream.mjs +45 -24
- package/dist/browser-stub.mjs +7 -0
- package/dist/browser.d.mts +98 -5
- package/dist/browser.mjs +224 -33
- package/dist/edge.d.mts +93 -4
- package/dist/edge.mjs +151 -69
- package/dist/iife.js +1 -396
- package/dist/index.d.mts +8 -6
- package/dist/index.mjs +104 -43
- package/dist/{worker.d.mts → wasm.d.mts} +20 -20
- package/dist/wasm.mjs +150 -0
- package/napi/index.d.mts +20 -33
- package/napi/index.d.ts +20 -33
- package/napi/index.mjs +48 -36
- package/package.json +47 -37
- package/skills/mdream/SKILL.md +128 -0
- package/skills/mdream/references/runtimes.md +57 -0
- package/wasm/mdream_edge.d.ts +27 -0
- package/wasm/mdream_edge.js +94 -20
- package/wasm/mdream_edge_bg.wasm +0 -0
- package/wasm/mdream_edge_bg.wasm.d.ts +3 -0
- package/wasm/package.json +1 -1
- package/wasm-bundler/mdream_edge.d.ts +24 -0
- package/wasm-bundler/mdream_edge.js +6 -2
- package/wasm-bundler/mdream_edge_bg.js +94 -20
- package/wasm-bundler/mdream_edge_bg.wasm +0 -0
- package/wasm-bundler/mdream_edge_bg.wasm.d.ts +3 -0
- package/wasm-bundler/package.json +1 -1
- package/dist/worker.mjs +0 -100
package/napi/index.mjs
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
// prettier-ignore
|
|
2
1
|
/* eslint-disable */
|
|
3
2
|
// @ts-nocheck
|
|
4
3
|
/* auto-generated by NAPI-RS */
|
|
@@ -8,6 +7,10 @@ const require = createRequire(import.meta.url)
|
|
|
8
7
|
|
|
9
8
|
const { readFileSync } = require('fs')
|
|
10
9
|
let nativeBinding = null
|
|
10
|
+
// Which artifact actually loaded. The WASI fallback chain overwrites it with
|
|
11
|
+
// the flavor it resolved; the late native retry below leaves it alone because
|
|
12
|
+
// it only runs while no WASI candidate has been loaded.
|
|
13
|
+
let __napiLoadedBindingTarget = 'native'
|
|
11
14
|
const loadErrors = []
|
|
12
15
|
|
|
13
16
|
const isMusl = () => {
|
|
@@ -66,7 +69,16 @@ const isMuslFromChildProcess = () => {
|
|
|
66
69
|
function requireNative() {
|
|
67
70
|
if (process.env.NAPI_RS_NATIVE_LIBRARY_PATH) {
|
|
68
71
|
try {
|
|
69
|
-
|
|
72
|
+
const overrideBinding = require(process.env.NAPI_RS_NATIVE_LIBRARY_PATH)
|
|
73
|
+
// The override may be a generated WASI loader, which already reports its
|
|
74
|
+
// own flavor. Adopt it: `module.exports` aliases this object, so claiming
|
|
75
|
+
// 'native' would both misreport the artifact and overwrite the loader's
|
|
76
|
+
// marker through the alias.
|
|
77
|
+
__napiLoadedBindingTarget =
|
|
78
|
+
overrideBinding && typeof overrideBinding.__napiBindingTarget === 'string'
|
|
79
|
+
? overrideBinding.__napiBindingTarget
|
|
80
|
+
: 'native'
|
|
81
|
+
return overrideBinding
|
|
70
82
|
} catch (err) {
|
|
71
83
|
loadErrors.push(err)
|
|
72
84
|
}
|
|
@@ -110,36 +122,36 @@ function requireNative() {
|
|
|
110
122
|
if (process.arch === 'x64') {
|
|
111
123
|
if ((process.config && process.config.variables && process.config.variables.shlib_suffix === 'dll.a') || (process.config && process.config.variables && process.config.variables.node_target_type === 'shared_library')) {
|
|
112
124
|
try {
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
125
|
+
return require('./rust.win32-x64-gnu.node')
|
|
126
|
+
} catch (e) {
|
|
127
|
+
loadErrors.push(e)
|
|
128
|
+
}
|
|
129
|
+
try {
|
|
130
|
+
const binding = require('@mdream/rust-win32-x64-gnu')
|
|
131
|
+
const bindingPackageVersion = require('@mdream/rust-win32-x64-gnu/package.json').version
|
|
132
|
+
if (bindingPackageVersion !== '0.17.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
133
|
+
throw new Error(`Native binding package version mismatch, expected 0.17.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
134
|
+
}
|
|
135
|
+
return binding
|
|
136
|
+
} catch (e) {
|
|
137
|
+
loadErrors.push(e)
|
|
122
138
|
}
|
|
123
|
-
return binding
|
|
124
|
-
} catch (e) {
|
|
125
|
-
loadErrors.push(e)
|
|
126
|
-
}
|
|
127
139
|
} else {
|
|
128
140
|
try {
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
141
|
+
return require('./rust.win32-x64-msvc.node')
|
|
142
|
+
} catch (e) {
|
|
143
|
+
loadErrors.push(e)
|
|
144
|
+
}
|
|
145
|
+
try {
|
|
146
|
+
const binding = require('@mdream/rust-win32-x64-msvc')
|
|
147
|
+
const bindingPackageVersion = require('@mdream/rust-win32-x64-msvc/package.json').version
|
|
148
|
+
if (bindingPackageVersion !== '0.17.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
149
|
+
throw new Error(`Native binding package version mismatch, expected 0.17.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
150
|
+
}
|
|
151
|
+
return binding
|
|
152
|
+
} catch (e) {
|
|
153
|
+
loadErrors.push(e)
|
|
138
154
|
}
|
|
139
|
-
return binding
|
|
140
|
-
} catch (e) {
|
|
141
|
-
loadErrors.push(e)
|
|
142
|
-
}
|
|
143
155
|
}
|
|
144
156
|
} else if (process.arch === 'ia32') {
|
|
145
157
|
try {
|
|
@@ -553,7 +565,7 @@ function createLoadErrorChain(errors) {
|
|
|
553
565
|
//
|
|
554
566
|
// NAPI_RS_WASI_FLAVOR selects one exact generated flavor and implies strict
|
|
555
567
|
// WASI loading. It never crosses into another flavor or falls back to native.
|
|
556
|
-
const __napiWasiFlavors = [
|
|
568
|
+
const __napiWasiFlavors = ['wasm32-wasi']
|
|
557
569
|
const __napiWasiFlavor = process.env.NAPI_RS_WASI_FLAVOR
|
|
558
570
|
const __napiWasiFlavorRequested =
|
|
559
571
|
typeof __napiWasiFlavor === 'string' && __napiWasiFlavor.length > 0
|
|
@@ -622,15 +634,16 @@ if (!nativeBinding || forceWasi) {
|
|
|
622
634
|
}
|
|
623
635
|
return null
|
|
624
636
|
}
|
|
625
|
-
if (!wasiBindingLoaded && (!__napiWasiFlavorRequested || __napiWasiFlavor ===
|
|
637
|
+
if (!wasiBindingLoaded && (!__napiWasiFlavorRequested || __napiWasiFlavor === 'wasm32-wasi')) {
|
|
626
638
|
let candidateError = null
|
|
627
639
|
let candidateFailed = false
|
|
628
640
|
try {
|
|
629
|
-
candidateError = __napiWasiResolveCandidate('./rust.wasi.cjs', false, [
|
|
641
|
+
candidateError = __napiWasiResolveCandidate('./rust.wasi.cjs', false, ['./rust.wasm32-wasi.debug.wasm', './rust.wasm32-wasi.wasm'])
|
|
630
642
|
candidateFailed = candidateError !== null
|
|
631
643
|
if (!candidateFailed) {
|
|
632
644
|
wasiBinding = require('./rust.wasi.cjs')
|
|
633
645
|
nativeBinding = wasiBinding
|
|
646
|
+
__napiLoadedBindingTarget = 'wasm32-wasi'
|
|
634
647
|
wasiBindingLoaded = true
|
|
635
648
|
}
|
|
636
649
|
} catch (err) {
|
|
@@ -642,7 +655,7 @@ if (!nativeBinding || forceWasi) {
|
|
|
642
655
|
loadErrors.push(candidateError)
|
|
643
656
|
}
|
|
644
657
|
}
|
|
645
|
-
if (!wasiBindingLoaded && (!__napiWasiFlavorRequested || __napiWasiFlavor ===
|
|
658
|
+
if (!wasiBindingLoaded && (!__napiWasiFlavorRequested || __napiWasiFlavor === 'wasm32-wasi')) {
|
|
646
659
|
let candidateError = null
|
|
647
660
|
let candidateFailed = false
|
|
648
661
|
try {
|
|
@@ -657,6 +670,7 @@ if (!nativeBinding || forceWasi) {
|
|
|
657
670
|
}
|
|
658
671
|
wasiBinding = require('@mdream/rust-wasm32-wasi')
|
|
659
672
|
nativeBinding = wasiBinding
|
|
673
|
+
__napiLoadedBindingTarget = 'wasm32-wasi'
|
|
660
674
|
wasiBindingLoaded = true
|
|
661
675
|
}
|
|
662
676
|
} catch (err) {
|
|
@@ -702,9 +716,7 @@ if (!nativeBinding) {
|
|
|
702
716
|
throw new Error(`Failed to load native binding`)
|
|
703
717
|
}
|
|
704
718
|
|
|
705
|
-
const { MarkdownStream, htmlToMarkdown
|
|
719
|
+
const { MarkdownStream, htmlToMarkdown } = nativeBinding
|
|
706
720
|
export { MarkdownStream }
|
|
707
721
|
export { htmlToMarkdown }
|
|
708
|
-
export
|
|
709
|
-
export { htmlToMarkdownChunks }
|
|
710
|
-
export { splitMarkdown }
|
|
722
|
+
export const __napiBindingTarget = __napiLoadedBindingTarget
|
package/package.json
CHANGED
|
@@ -1,42 +1,57 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mdream",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "
|
|
4
|
+
"version": "2.0.0",
|
|
5
5
|
"description": "Ultra-performant HTML to Markdown Convertor Optimized for LLMs and llm.txt artifacts.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
8
8
|
"email": "harlan@harlanzw.com",
|
|
9
9
|
"url": "https://harlanzw.com/"
|
|
10
10
|
},
|
|
11
|
+
"contributors": [
|
|
12
|
+
{
|
|
13
|
+
"name": "André Cruz",
|
|
14
|
+
"url": "https://github.com/edevil"
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"name": "Bernardo Pereira",
|
|
18
|
+
"url": "https://github.com/BernardoPe"
|
|
19
|
+
}
|
|
20
|
+
],
|
|
11
21
|
"license": "MIT",
|
|
12
22
|
"repository": {
|
|
13
23
|
"type": "git",
|
|
14
24
|
"url": "https://github.com/harlan-zw/mdream",
|
|
15
25
|
"directory": "packages/mdream"
|
|
16
26
|
},
|
|
27
|
+
"sideEffects": [
|
|
28
|
+
"./wasm-bundler/mdream_edge.js"
|
|
29
|
+
],
|
|
17
30
|
"exports": {
|
|
18
31
|
".": {
|
|
32
|
+
"workerd": {
|
|
33
|
+
"types": "./dist/edge.d.mts",
|
|
34
|
+
"default": "./dist/edge.mjs"
|
|
35
|
+
},
|
|
36
|
+
"edge-light": {
|
|
37
|
+
"types": "./dist/edge.d.mts",
|
|
38
|
+
"default": "./dist/edge.mjs"
|
|
39
|
+
},
|
|
19
40
|
"types": "./dist/index.d.mts",
|
|
20
|
-
"
|
|
21
|
-
"edge-light": "./dist/edge.mjs",
|
|
22
|
-
"browser": "./dist/browser.mjs",
|
|
41
|
+
"browser": "./dist/browser-stub.mjs",
|
|
23
42
|
"import": {
|
|
24
43
|
"types": "./dist/index.d.mts",
|
|
25
44
|
"default": "./dist/index.mjs"
|
|
26
45
|
},
|
|
27
46
|
"default": "./dist/index.mjs"
|
|
28
47
|
},
|
|
29
|
-
"./
|
|
30
|
-
"types": "./dist/
|
|
31
|
-
"
|
|
32
|
-
"types": "./dist/worker.d.mts",
|
|
33
|
-
"default": "./dist/worker.mjs"
|
|
34
|
-
},
|
|
35
|
-
"default": "./dist/worker.mjs"
|
|
48
|
+
"./browser": {
|
|
49
|
+
"types": "./dist/browser.d.mts",
|
|
50
|
+
"default": "./dist/browser.mjs"
|
|
36
51
|
},
|
|
37
52
|
"./wasm": {
|
|
38
|
-
"types": "./wasm
|
|
39
|
-
"default": "./wasm
|
|
53
|
+
"types": "./dist/wasm.d.mts",
|
|
54
|
+
"default": "./dist/wasm.mjs"
|
|
40
55
|
},
|
|
41
56
|
"./wasm/*": "./wasm/*"
|
|
42
57
|
},
|
|
@@ -51,36 +66,32 @@
|
|
|
51
66
|
"bin",
|
|
52
67
|
"dist",
|
|
53
68
|
"napi/",
|
|
69
|
+
"skills",
|
|
54
70
|
"wasm-bundler/",
|
|
55
71
|
"wasm/"
|
|
56
72
|
],
|
|
57
|
-
"browser": "./dist/browser.mjs",
|
|
58
73
|
"dependencies": {},
|
|
59
74
|
"optionalDependencies": {
|
|
60
|
-
"@mdream/rust-android-arm-eabi": "
|
|
61
|
-
"@mdream/rust-android-arm64": "
|
|
62
|
-
"@mdream/rust-darwin-arm64": "
|
|
63
|
-
"@mdream/rust-darwin-x64": "
|
|
64
|
-
"@mdream/rust-freebsd-x64": "
|
|
65
|
-
"@mdream/rust-linux-arm-gnueabihf": "
|
|
66
|
-
"@mdream/rust-linux-arm64-gnu": "
|
|
67
|
-
"@mdream/rust-linux-arm64-musl": "
|
|
68
|
-
"@mdream/rust-linux-x64-gnu": "
|
|
69
|
-
"@mdream/rust-linux-x64-musl": "
|
|
70
|
-
"@mdream/rust-wasm32-wasi": "
|
|
71
|
-
"@mdream/rust-win32-arm64-msvc": "
|
|
72
|
-
"@mdream/rust-win32-x64-msvc": "
|
|
75
|
+
"@mdream/rust-android-arm-eabi": "2.0.0",
|
|
76
|
+
"@mdream/rust-android-arm64": "2.0.0",
|
|
77
|
+
"@mdream/rust-darwin-arm64": "2.0.0",
|
|
78
|
+
"@mdream/rust-darwin-x64": "2.0.0",
|
|
79
|
+
"@mdream/rust-freebsd-x64": "2.0.0",
|
|
80
|
+
"@mdream/rust-linux-arm-gnueabihf": "2.0.0",
|
|
81
|
+
"@mdream/rust-linux-arm64-gnu": "2.0.0",
|
|
82
|
+
"@mdream/rust-linux-arm64-musl": "2.0.0",
|
|
83
|
+
"@mdream/rust-linux-x64-gnu": "2.0.0",
|
|
84
|
+
"@mdream/rust-linux-x64-musl": "2.0.0",
|
|
85
|
+
"@mdream/rust-wasm32-wasi": "2.0.0",
|
|
86
|
+
"@mdream/rust-win32-arm64-msvc": "2.0.0",
|
|
87
|
+
"@mdream/rust-win32-x64-msvc": "2.0.0"
|
|
73
88
|
},
|
|
74
89
|
"devDependencies": {
|
|
75
|
-
"@mdream/js": "
|
|
76
|
-
"@mdream/rust-build": "0.17.0"
|
|
90
|
+
"@mdream/js": "2.0.0",
|
|
91
|
+
"@mdream/rust-build": "0.17.0",
|
|
92
|
+
"rolldown": "^1.2.12"
|
|
77
93
|
},
|
|
78
94
|
"scripts": {
|
|
79
|
-
"flame": "pnpm build && unbuild bench/bundle && clinic flame -- node bench/bundle/dist/string.mjs 10",
|
|
80
|
-
"bench:build": "pnpm build && unbuild bench/bundle",
|
|
81
|
-
"bench:stream": "pnpm build && unbuild bench/bundle && hyperfine --runs 100 'node bench/bundle/dist/stream.mjs' --warmup 3",
|
|
82
|
-
"bench:await": "pnpm build && unbuild bench/bundle && hyperfine --runs 100 'node bench/bundle/dist/await.mjs' --warmup 3",
|
|
83
|
-
"bench:string": "pnpm build && unbuild bench/bundle && hyperfine --runs 100 'node bench/bundle/dist/string.mjs' --warmup 3",
|
|
84
95
|
"ts": "node --experimental-strip-types",
|
|
85
96
|
"test:live": "curl -s https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax | node ./bin/mdream.mjs --origin https://docs.github.com | tee test/github-markdown.md",
|
|
86
97
|
"test:images": "cat test/fixtures/test-origin.html | node ./bin/mdream.mjs --origin https://docs.github.com",
|
|
@@ -89,12 +100,11 @@
|
|
|
89
100
|
"test:wiki:file": "pnpm build && cat test/fixtures/wikipedia-largest.html | node ./bin/mdream.mjs --origin https://en.wikipedia.org | tee test/wiki-markdown.md",
|
|
90
101
|
"test:wiki-small:file": "cat test/fixtures/wikipedia-small.html | node ./bin/mdream.mjs --origin https://en.wikipedia.org | tee test/wiki-markdown.md",
|
|
91
102
|
"build": "obuild",
|
|
92
|
-
"build:native": "
|
|
93
|
-
"build:edge": "
|
|
103
|
+
"build:native": "pnpm --dir ../../crates/node run build:platform",
|
|
104
|
+
"build:edge": "node ../../scripts/build-wasm-edge.mjs --target web --out-dir ../../packages/mdream/wasm --out-name mdream_edge && node ../../scripts/build-wasm-edge.mjs --target bundler --out-dir ../../packages/mdream/wasm-bundler --out-name mdream_edge",
|
|
94
105
|
"typecheck": "tsc --noEmit",
|
|
95
106
|
"dev:prepare": "obuild --stub",
|
|
96
107
|
"test": "vitest test",
|
|
97
|
-
"test:browser": "vitest test --project=browser",
|
|
98
108
|
"test:attw": "attw --pack"
|
|
99
109
|
}
|
|
100
110
|
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: mdream
|
|
3
|
+
description: Convert HTML to Markdown, plain text, or safe HTML with mdream, the Rust engine for Node, edge, and the browser. Use when a task mentions mdream, htmlToMarkdown, streamHtmlToMarkdown, the minimal preset, frontmatter or extraction callbacks, tagOverrides, clean options, the mdream CLI, or errors such as "Custom hook plugins require @mdream/js" or ERR_PACKAGE_PATH_NOT_EXPORTED for mdream/plugins.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# mdream
|
|
7
|
+
|
|
8
|
+
This Skill covers v2.
|
|
9
|
+
For an earlier release, read the Skill at its release tag.
|
|
10
|
+
The package wraps a Rust converter: NAPI in Node, WASM on edge runtimes and in the browser.
|
|
11
|
+
`@mdream/js` is a separate pure JS engine. It has hook plugins, the splitter, `llms.txt` generation, and content negotiation. Its option shape differs; see [Traps](#traps).
|
|
12
|
+
|
|
13
|
+
## Setup
|
|
14
|
+
|
|
15
|
+
```ts
|
|
16
|
+
import { htmlToMarkdown } from 'mdream'
|
|
17
|
+
|
|
18
|
+
const markdown = htmlToMarkdown('<main><h1>Docs</h1></main>', {
|
|
19
|
+
origin: 'https://example.com',
|
|
20
|
+
minimal: true,
|
|
21
|
+
})
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
- `htmlToMarkdown` is synchronous in Node and on edge runtimes. It returns a string.
|
|
25
|
+
- Set `origin` to make relative `href` and `src` values absolute. Without it, they stay relative.
|
|
26
|
+
- If a bundler fails on the native binding, mark `mdream` as external. In Next.js, add it to `serverExternalPackages`. `@mdream/vite` does this for you.
|
|
27
|
+
|
|
28
|
+
## Automatic behaviour
|
|
29
|
+
|
|
30
|
+
Default output keeps everything: navigation, forms, footers, and text hidden with CSS classes.
|
|
31
|
+
The `<title>` text never appears in the body. Read it with `frontmatter` or `extraction`.
|
|
32
|
+
|
|
33
|
+
`minimal: true` turns on these options. Each one is off without it.
|
|
34
|
+
|
|
35
|
+
| Option | Effect under `minimal` |
|
|
36
|
+
|---|---|
|
|
37
|
+
| `frontmatter` | YAML frontmatter from `<title>` and `<meta>`. |
|
|
38
|
+
| `isolateMain` | Keeps `<main>`, else the content from the first heading to the first `<footer>`. |
|
|
39
|
+
| `tailwind` | `font-bold` becomes `**bold**`. `hidden` and `absolute` content is dropped. |
|
|
40
|
+
| `filter` | Excludes `form`, `fieldset`, `object`, `embed`, `footer`, `aside`, `iframe`, `input`, `textarea`, `select`, `button`, `nav`. |
|
|
41
|
+
| `clean` | All cleanup: tracking parameters, `#` links, images without alt, and more. |
|
|
42
|
+
|
|
43
|
+
To turn one off, pass it as `false`: `{ minimal: true, frontmatter: false, clean: false }`.
|
|
44
|
+
A `filter` under `minimal` adds to the excludes above: `{ minimal: true, filter: { exclude: ['h1'] } }` drops `h1` and every default tag. `filter: false` keeps forms and `nav`.
|
|
45
|
+
|
|
46
|
+
## Common tasks
|
|
47
|
+
|
|
48
|
+
Read metadata and matched elements in one pass:
|
|
49
|
+
|
|
50
|
+
```ts
|
|
51
|
+
import { htmlToMarkdown } from 'mdream'
|
|
52
|
+
|
|
53
|
+
const links: string[] = []
|
|
54
|
+
let title = ''
|
|
55
|
+
const markdown = htmlToMarkdown('<html><head><title>T</title></head><body><a href="/a">A</a></body></html>', {
|
|
56
|
+
frontmatter: (fm) => { title = fm.title ?? '' },
|
|
57
|
+
extraction: {
|
|
58
|
+
'a[href]': (el) => { links.push(el.attributes.href ?? '') },
|
|
59
|
+
},
|
|
60
|
+
})
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Stream a large response with the same options.
|
|
64
|
+
Callbacks run once, after the last chunk is read:
|
|
65
|
+
|
|
66
|
+
```ts
|
|
67
|
+
import { streamHtmlToMarkdown } from 'mdream'
|
|
68
|
+
|
|
69
|
+
const response = await fetch('https://example.com')
|
|
70
|
+
let markdown = ''
|
|
71
|
+
let title = ''
|
|
72
|
+
for await (const chunk of streamHtmlToMarkdown(response.body, {
|
|
73
|
+
origin: 'https://example.com',
|
|
74
|
+
minimal: true,
|
|
75
|
+
frontmatter: (fm) => { title = fm.title ?? '' },
|
|
76
|
+
})) {
|
|
77
|
+
markdown += chunk
|
|
78
|
+
}
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Render a custom element with Markdown semantics. Unknown tags output only their text.
|
|
82
|
+
|
|
83
|
+
```ts
|
|
84
|
+
import { htmlToMarkdown } from 'mdream'
|
|
85
|
+
|
|
86
|
+
htmlToMarkdown('<x-heading>Title</x-heading><callout>Read this</callout>', {
|
|
87
|
+
tagOverrides: {
|
|
88
|
+
'x-heading': 'h2',
|
|
89
|
+
'callout': { enter: '> **Note:** ', exit: '', spacing: [2, 2] },
|
|
90
|
+
},
|
|
91
|
+
})
|
|
92
|
+
// ## Title\n\n> **Note:** Read this
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Other output: `format: 'text'` gives plain text, `format: 'html'` gives allowlisted semantic HTML. `wrapWidth: 80` wraps prose only, never code, tables, or headings.
|
|
96
|
+
|
|
97
|
+
CLI: `curl -s URL | mdream --origin URL --preset minimal`. It streams stdin to stdout. Flags: `--format`, `--text`, `--wrap-width`.
|
|
98
|
+
|
|
99
|
+
## Traps
|
|
100
|
+
|
|
101
|
+
- **Unknown options throw.** A key outside the [Config](#config) list throws a `TypeError` that names the fix. Pass `{ minimal: true }`, not `preset`. Pass `clean: { urls: true }`, not `cleanUrls`. Pass plugin options at the top level, not in a `plugins` object. A key set to `undefined` does not throw. The CLI exits with code 1 on an unknown flag.
|
|
102
|
+
- **Hook plugins are not in this package.** An array in `plugins` throws `Custom hook plugins require @mdream/js`. Use `@mdream/js` with `plugins: [createPlugin({...})]` from `@mdream/js/plugins`.
|
|
103
|
+
- **`@mdream/js` takes plugins as an array.** It throws a `TypeError` on `minimal`, `format`, `frontmatter`, `isolateMain`, `tailwind`, `filter`, or `extraction` at the top level. Use `withMinimalPreset()` from `@mdream/js/preset/minimal`, or `{ plugins: [frontmatterPlugin()] }` from `@mdream/js/plugins`. For text or HTML output, use `@mdream/js/text` or `@mdream/js/html`.
|
|
104
|
+
- **Emphasis is `*`, not `_`.** Headings are ATX, bullets are `-`, rules are `---`. Only `tagOverrides` changes a delimiter, for example `em: { enter: '_', exit: '_', isInline: true }`.
|
|
105
|
+
- **Filter selectors also drop inline `position: absolute` and `position: fixed` elements.**
|
|
106
|
+
- **Browser bundles import `mdream/browser`.** In a browser bundle, the root `mdream` entry throws a `TypeError` when called. Every entry takes the same options and produces the same string. `mdream/browser` and the CDN script return `Promise<string>`, so `await` them. Read [references/runtimes.md](references/runtimes.md) before you use `mdream` outside Node or a Cloudflare Worker.
|
|
107
|
+
|
|
108
|
+
## Version limits
|
|
109
|
+
|
|
110
|
+
Code from mdream 0.x fails on 1.x and 2.x. The subpaths `mdream/plugins`, `mdream/preset/minimal`, `mdream/splitter`, `mdream/llms-txt`, and `mdream/negotiate` throw `ERR_PACKAGE_PATH_NOT_EXPORTED`.
|
|
111
|
+
|
|
112
|
+
```ts
|
|
113
|
+
// Old (0.x):
|
|
114
|
+
// import { htmlToMarkdown } from 'mdream'
|
|
115
|
+
// import { withMinimalPreset } from 'mdream/preset/minimal'
|
|
116
|
+
// htmlToMarkdown(html, withMinimalPreset({ origin }))
|
|
117
|
+
|
|
118
|
+
import { htmlToMarkdown } from 'mdream'
|
|
119
|
+
|
|
120
|
+
// New:
|
|
121
|
+
htmlToMarkdown('<p>x</p>', { minimal: true, origin: 'https://example.com' })
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
The splitter, `llms.txt` generation, and `shouldServeMarkdown` moved to `@mdream/js/splitter`, `@mdream/js/llms-txt`, and `@mdream/js/negotiate`.
|
|
125
|
+
|
|
126
|
+
## Config
|
|
127
|
+
|
|
128
|
+
Options: `origin`, `minimal`, `clean` (`true` or per rule), `frontmatter`, `isolateMain`, `tailwind`, `filter`, `extraction`, `tagOverrides`, `wrapWidth`, `format`. Types: `MdreamOptions` in the [package README](https://github.com/harlan-zw/mdream/tree/main/packages/mdream#options).
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# mdream outside Node
|
|
2
|
+
|
|
3
|
+
`mdream` picks an entry by export condition.
|
|
4
|
+
Its conversion entries take the same `MdreamOptions`, including `minimal`, `frontmatter`, `filter`, `extraction`, and `tagOverrides`.
|
|
5
|
+
Callbacks work in every supported runtime.
|
|
6
|
+
Some entries return a string; others return `Promise<string>`.
|
|
7
|
+
|
|
8
|
+
| Import | Resolved by | Returns |
|
|
9
|
+
|---|---|---|
|
|
10
|
+
| `mdream` | Node | `string` |
|
|
11
|
+
| `mdream` | `workerd`, `edge-light` | `string` |
|
|
12
|
+
| `mdream` | `browser` | throws a `TypeError` when called |
|
|
13
|
+
| `mdream/browser` | direct import | `Promise<string>` |
|
|
14
|
+
| `mdream/wasm` | direct import | `string`, after `init()` |
|
|
15
|
+
| `dist/iife.js` (CDN) | script tag | `Promise<string>` |
|
|
16
|
+
|
|
17
|
+
The `mdream` types always describe the synchronous API. The `mdream/browser` types return `Promise<string>` and need no `customConditions` setting.
|
|
18
|
+
|
|
19
|
+
## Cloudflare Workers and Vercel Edge
|
|
20
|
+
|
|
21
|
+
The `workerd` and `edge-light` conditions load the WASM build and instantiate it at import. The API matches Node.
|
|
22
|
+
|
|
23
|
+
```ts
|
|
24
|
+
import { htmlToMarkdown } from 'mdream'
|
|
25
|
+
|
|
26
|
+
export default {
|
|
27
|
+
async fetch(request: Request): Promise<Response> {
|
|
28
|
+
const html = await request.text()
|
|
29
|
+
return new Response(htmlToMarkdown(html, { minimal: true }))
|
|
30
|
+
},
|
|
31
|
+
}
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
A Rust panic becomes a normal `Error` with the message `mdream WASM panic, please report this at ...`. The instance stays usable, so catch it per request.
|
|
35
|
+
|
|
36
|
+
## Browser bundles
|
|
37
|
+
|
|
38
|
+
In a Vite, webpack, or esbuild client build, import from `mdream/browser`.
|
|
39
|
+
This entry fetches the WASM binary on first use and returns `Promise<string>`.
|
|
40
|
+
Await the result.
|
|
41
|
+
Options and callbacks work as in Node.
|
|
42
|
+
|
|
43
|
+
```ts
|
|
44
|
+
import { htmlToMarkdown } from 'mdream/browser'
|
|
45
|
+
|
|
46
|
+
const markdown = await htmlToMarkdown(html, { minimal: true })
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
In a browser bundle, the root `mdream` entry throws a `TypeError` when called. The import itself does not throw. A module that runs on the server and the client can import `mdream` and call it only on the server.
|
|
50
|
+
|
|
51
|
+
For browser code that wants a plain string with no WASM, `@mdream/js` is synchronous. It takes plugins as an array of imports from `@mdream/js/plugins`.
|
|
52
|
+
|
|
53
|
+
## Web Worker, raw WASM, and CDN
|
|
54
|
+
|
|
55
|
+
- Web Worker: v2 has no worker entry. Write a module worker that imports `mdream/browser`, and start it with `new Worker(new URL('./md.worker.ts', import.meta.url), { type: 'module' })`. Functions cannot cross `postMessage`, so set callbacks inside the worker.
|
|
56
|
+
- `mdream/wasm`: the wasm-bindgen build. Await the default export `init({ module_or_path })` before the first `htmlToMarkdown(html, options)` call. Its `MarkdownStream` runs the callbacks in `finish()`.
|
|
57
|
+
- `https://unpkg.com/mdream/dist/iife.js`: the WASM is inlined and ready at load. `window.mdream` has only `htmlToMarkdown`. There is no `init()`. The call returns `Promise<string>`, the same as `mdream/browser`.
|
package/wasm/mdream_edge.d.ts
CHANGED
|
@@ -6,7 +6,23 @@ export class MarkdownStream {
|
|
|
6
6
|
[Symbol.dispose](): void;
|
|
7
7
|
finish(): string;
|
|
8
8
|
constructor(options: any);
|
|
9
|
+
/**
|
|
10
|
+
* Byte chunk in, string out, skipping the transcode per chunk. A sequence
|
|
11
|
+
* split across chunks carries to the next call rather than decoding early,
|
|
12
|
+
* matching `TextDecoder({ stream: true })`; invalid bytes become U+FFFD.
|
|
13
|
+
*/
|
|
14
|
+
processChunkBytes(chunk: Uint8Array): string;
|
|
15
|
+
/**
|
|
16
|
+
* A carried byte tail cannot be completed by a string chunk, so it is
|
|
17
|
+
* flushed first. Byte and string chunks may be mixed freely.
|
|
18
|
+
*/
|
|
9
19
|
processChunk(chunk: string): string;
|
|
20
|
+
/**
|
|
21
|
+
* Frontmatter and extracted elements collected so far, as
|
|
22
|
+
* `{ frontmatter?, extracted? }`. Call after `finish()` for the complete
|
|
23
|
+
* data. Extracted elements drain on each call.
|
|
24
|
+
*/
|
|
25
|
+
takeData(): any;
|
|
10
26
|
}
|
|
11
27
|
|
|
12
28
|
/**
|
|
@@ -16,6 +32,14 @@ export function __mdreamTakePanicMessage(): string | undefined;
|
|
|
16
32
|
|
|
17
33
|
export function htmlToMarkdown(html: string, options: any): string;
|
|
18
34
|
|
|
35
|
+
/**
|
|
36
|
+
* Bytes in, string out: skips the transcode (~14% of a convert) for a
|
|
37
|
+
* caller already holding UTF-8 bytes. Encoding a string to get here
|
|
38
|
+
* measures 1-16% *slower* than [`html_to_markdown`] — only worth it when
|
|
39
|
+
* the bytes are already in hand.
|
|
40
|
+
*/
|
|
41
|
+
export function htmlToMarkdownBytes(html: Uint8Array, options: any): string;
|
|
42
|
+
|
|
19
43
|
export function htmlToMarkdownResult(html: string, options: any): any;
|
|
20
44
|
|
|
21
45
|
export function install_panic_hook(): void;
|
|
@@ -27,11 +51,14 @@ export interface InitOutput {
|
|
|
27
51
|
readonly __mdreamTakePanicMessage: (a: number) => void;
|
|
28
52
|
readonly __wbg_markdownstream_free: (a: number, b: number) => void;
|
|
29
53
|
readonly htmlToMarkdown: (a: number, b: number, c: number, d: number) => void;
|
|
54
|
+
readonly htmlToMarkdownBytes: (a: number, b: number, c: number, d: number) => void;
|
|
30
55
|
readonly htmlToMarkdownResult: (a: number, b: number, c: number) => number;
|
|
31
56
|
readonly install_panic_hook: () => void;
|
|
32
57
|
readonly markdownstream_finish: (a: number, b: number) => void;
|
|
33
58
|
readonly markdownstream_new: (a: number) => number;
|
|
34
59
|
readonly markdownstream_processChunk: (a: number, b: number, c: number, d: number) => void;
|
|
60
|
+
readonly markdownstream_processChunkBytes: (a: number, b: number, c: number, d: number) => void;
|
|
61
|
+
readonly markdownstream_takeData: (a: number) => number;
|
|
35
62
|
readonly __wbindgen_export: (a: number, b: number) => number;
|
|
36
63
|
readonly __wbindgen_export2: (a: number, b: number, c: number, d: number) => number;
|
|
37
64
|
readonly __wbindgen_export3: (a: number) => void;
|