mdream 1.4.0 → 1.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -1
- package/bin/mdream.mjs +18 -2
- package/dist/browser.mjs +6 -1
- package/dist/edge.mjs +116 -5
- package/dist/iife.js +1 -1
- package/dist/index.d.mts +5 -0
- package/dist/index.mjs +42 -22
- package/dist/worker.d.mts +5 -0
- package/napi/index.d.mts +1 -0
- package/napi/index.d.ts +1 -0
- package/napi/index.mjs +12 -11
- package/package.json +21 -16
- package/wasm/mdream_edge_bg.wasm +0 -0
- package/wasm/package.json +1 -1
- package/wasm-bundler/mdream_edge_bg.wasm +0 -0
- package/wasm-bundler/package.json +1 -1
package/README.md
CHANGED
|
@@ -116,6 +116,11 @@ const markdown = htmlToMarkdown('<h1>Hello World</h1><p>Some content.</p>')
|
|
|
116
116
|
// # Hello World
|
|
117
117
|
//
|
|
118
118
|
// Some content.
|
|
119
|
+
|
|
120
|
+
const text = htmlToMarkdown('<h1>Hello <strong>World</strong></h1>', {
|
|
121
|
+
format: 'text',
|
|
122
|
+
})
|
|
123
|
+
// Hello World
|
|
119
124
|
```
|
|
120
125
|
|
|
121
126
|
### `streamHtmlToMarkdown()`
|
|
@@ -220,6 +225,9 @@ interface MdreamOptions {
|
|
|
220
225
|
* tables, and headings are never wrapped. `0` (or unset) disables wrapping.
|
|
221
226
|
*/
|
|
222
227
|
wrapWidth?: number
|
|
228
|
+
|
|
229
|
+
/** Output Markdown or plain text. Default: 'markdown' */
|
|
230
|
+
format?: 'markdown' | 'text'
|
|
223
231
|
}
|
|
224
232
|
```
|
|
225
233
|
|
|
@@ -244,6 +252,9 @@ interface EngineOptions {
|
|
|
244
252
|
* disables wrapping.
|
|
245
253
|
*/
|
|
246
254
|
wrapWidth?: number
|
|
255
|
+
|
|
256
|
+
/** Output Markdown or plain text. Default: 'markdown' */
|
|
257
|
+
format?: 'markdown' | 'text'
|
|
247
258
|
}
|
|
248
259
|
|
|
249
260
|
interface BuiltinPlugins {
|
|
@@ -880,6 +891,14 @@ cat index.html \
|
|
|
880
891
|
| tee output.md
|
|
881
892
|
```
|
|
882
893
|
|
|
894
|
+
**Plain text output:**
|
|
895
|
+
|
|
896
|
+
```bash
|
|
897
|
+
cat index.html \
|
|
898
|
+
| npx mdream --format text \
|
|
899
|
+
| tee output.txt
|
|
900
|
+
```
|
|
901
|
+
|
|
883
902
|
### CLI Options
|
|
884
903
|
|
|
885
904
|
| Option | Description |
|
|
@@ -887,9 +906,11 @@ cat index.html \
|
|
|
887
906
|
| `--origin <url>` | Base URL for resolving relative links and images |
|
|
888
907
|
| `--preset minimal` | Enable the minimal preset |
|
|
889
908
|
| `--wrap-width <n>` | Hard-wrap prose at `n` characters (code, tables, and headings are never wrapped) |
|
|
909
|
+
| `--format <format>` | Output format: `markdown`, `text` |
|
|
910
|
+
| `--text` | Alias for `--format text` |
|
|
890
911
|
| `-h`, `--help` | Display help information |
|
|
891
912
|
|
|
892
|
-
The CLI reads HTML from stdin and writes Markdown to stdout. It uses the streaming API internally.
|
|
913
|
+
The CLI reads HTML from stdin and writes Markdown or plain text to stdout. It uses the streaming API internally.
|
|
893
914
|
|
|
894
915
|
## Browser and Edge Usage
|
|
895
916
|
|
|
@@ -910,6 +931,16 @@ for await (const chunk of streamHtmlToMarkdown(response.body)) {
|
|
|
910
931
|
}
|
|
911
932
|
```
|
|
912
933
|
|
|
934
|
+
If your toolchain doesn't resolve the export conditions, the raw wasm-bindgen build (web target) is exposed at `mdream/wasm` for manual initialization:
|
|
935
|
+
|
|
936
|
+
```ts
|
|
937
|
+
import init, { htmlToMarkdown } from 'mdream/wasm'
|
|
938
|
+
import wasmModule from 'mdream/wasm/mdream_edge_bg.wasm'
|
|
939
|
+
|
|
940
|
+
await init({ module_or_path: wasmModule })
|
|
941
|
+
const markdown = htmlToMarkdown('<h1>Hello</h1>', {})
|
|
942
|
+
```
|
|
943
|
+
|
|
913
944
|
You can also import the edge entry point directly:
|
|
914
945
|
|
|
915
946
|
```ts
|
package/bin/mdream.mjs
CHANGED
|
@@ -6,6 +6,7 @@ const args = process.argv.slice(2)
|
|
|
6
6
|
let origin
|
|
7
7
|
let preset
|
|
8
8
|
let wrapWidth
|
|
9
|
+
let format
|
|
9
10
|
for (let i = 0; i < args.length; i++) {
|
|
10
11
|
if (args[i] === '--origin' && args[i + 1]) {
|
|
11
12
|
origin = args[++i]
|
|
@@ -16,13 +17,28 @@ for (let i = 0; i < args.length; i++) {
|
|
|
16
17
|
else if (args[i] === '--wrap-width' && args[i + 1]) {
|
|
17
18
|
wrapWidth = Number.parseInt(args[++i], 10) || undefined
|
|
18
19
|
}
|
|
20
|
+
else if (args[i] === '--format') {
|
|
21
|
+
const value = args[++i]
|
|
22
|
+
if (!value) {
|
|
23
|
+
process.stderr.write('--format requires a value: markdown or text\n')
|
|
24
|
+
process.exit(1)
|
|
25
|
+
}
|
|
26
|
+
if (value !== 'markdown' && value !== 'text') {
|
|
27
|
+
process.stderr.write(`Unknown format: ${value}\n`)
|
|
28
|
+
process.exit(1)
|
|
29
|
+
}
|
|
30
|
+
format = value
|
|
31
|
+
}
|
|
32
|
+
else if (args[i] === '--text') {
|
|
33
|
+
format = 'text'
|
|
34
|
+
}
|
|
19
35
|
else if (args[i] === '-h' || args[i] === '--help') {
|
|
20
|
-
process.stdout.write('Usage: mdream [--origin <url>] [--preset minimal] [--wrap-width <n>]\nPipe HTML via stdin, outputs Markdown to stdout.\n')
|
|
36
|
+
process.stdout.write('Usage: mdream [--origin <url>] [--preset minimal] [--wrap-width <n>] [--format markdown|text] [--text]\nPipe HTML via stdin, outputs Markdown or plain text to stdout.\n')
|
|
21
37
|
process.exit(0)
|
|
22
38
|
}
|
|
23
39
|
}
|
|
24
40
|
|
|
25
|
-
const options = { origin, minimal: preset === 'minimal', wrapWidth }
|
|
41
|
+
const options = { origin, minimal: preset === 'minimal', wrapWidth, format }
|
|
26
42
|
const stream = Readable.toWeb(process.stdin)
|
|
27
43
|
for await (const chunk of streamHtmlToMarkdown(stream, options)) {
|
|
28
44
|
if (chunk?.length)
|
package/dist/browser.mjs
CHANGED
|
@@ -35,10 +35,15 @@ async function* streamHtmlToMarkdown(htmlStream, options) {
|
|
|
35
35
|
while (true) {
|
|
36
36
|
const { done, value } = await reader.read();
|
|
37
37
|
if (done) break;
|
|
38
|
-
const chunk = typeof value === "string" ? value : decoder.decode(value);
|
|
38
|
+
const chunk = typeof value === "string" ? decoder.decode() + value : decoder.decode(value, { stream: true });
|
|
39
39
|
const processed = stream.processChunk(chunk);
|
|
40
40
|
if (processed) yield processed;
|
|
41
41
|
}
|
|
42
|
+
const decoderTail = decoder.decode();
|
|
43
|
+
if (decoderTail) {
|
|
44
|
+
const processed = stream.processChunk(decoderTail);
|
|
45
|
+
if (processed) yield processed;
|
|
46
|
+
}
|
|
42
47
|
const final_ = stream.finish();
|
|
43
48
|
if (final_) yield final_;
|
|
44
49
|
} finally {
|
package/dist/edge.mjs
CHANGED
|
@@ -1,11 +1,117 @@
|
|
|
1
|
-
import { MarkdownStream as MarkdownStream$1, htmlToMarkdownResult } from "../wasm
|
|
1
|
+
import { MarkdownStream as MarkdownStream$1, htmlToMarkdownResult, initSync } from "../wasm/mdream_edge.js";
|
|
2
|
+
import wasmModule from "../wasm/mdream_edge_bg.wasm";
|
|
3
|
+
const MINIMAL_FILTER_DEFAULT = { exclude: [
|
|
4
|
+
"form",
|
|
5
|
+
"fieldset",
|
|
6
|
+
"object",
|
|
7
|
+
"embed",
|
|
8
|
+
"footer",
|
|
9
|
+
"aside",
|
|
10
|
+
"iframe",
|
|
11
|
+
"input",
|
|
12
|
+
"textarea",
|
|
13
|
+
"select",
|
|
14
|
+
"button",
|
|
15
|
+
"nav"
|
|
16
|
+
] };
|
|
17
|
+
const CLEAN_ALL = {
|
|
18
|
+
urls: true,
|
|
19
|
+
fragments: true,
|
|
20
|
+
emptyLinks: true,
|
|
21
|
+
redundantLinks: true,
|
|
22
|
+
selfLinkHeadings: true,
|
|
23
|
+
emptyImages: true,
|
|
24
|
+
emptyLinkText: true
|
|
25
|
+
};
|
|
26
|
+
function resolveCleanConfig(options, minimal) {
|
|
27
|
+
const cleanUrls = options.cleanUrls === true;
|
|
28
|
+
let cleanOpt = options.clean;
|
|
29
|
+
if (cleanOpt === void 0) {
|
|
30
|
+
if (!minimal) return { cleanUrls };
|
|
31
|
+
cleanOpt = true;
|
|
32
|
+
}
|
|
33
|
+
if (!cleanOpt) return { cleanUrls };
|
|
34
|
+
const resolved = cleanOpt === true ? CLEAN_ALL : cleanOpt;
|
|
35
|
+
return {
|
|
36
|
+
cleanUrls: cleanUrls || resolved.urls === true,
|
|
37
|
+
clean: resolved
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
function resolveFrontmatter(opt) {
|
|
41
|
+
if (typeof opt === "function") return {
|
|
42
|
+
config: {},
|
|
43
|
+
callback: opt
|
|
44
|
+
};
|
|
45
|
+
if (opt && typeof opt === "object") {
|
|
46
|
+
const { onExtract, ...config } = opt;
|
|
47
|
+
return {
|
|
48
|
+
config,
|
|
49
|
+
callback: onExtract
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
return { config: {} };
|
|
53
|
+
}
|
|
54
|
+
function resolveOptions(options) {
|
|
55
|
+
if (Array.isArray(options.plugins)) throw new TypeError("Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://mdream.dev/v1-migration#custom-plugins");
|
|
56
|
+
const minimal = options.minimal === true;
|
|
57
|
+
const plugins = options.plugins ? { ...options.plugins } : {};
|
|
58
|
+
let frontmatterCallback;
|
|
59
|
+
const frontmatterDisabled = options.frontmatter === false;
|
|
60
|
+
if (minimal ? !frontmatterDisabled : !!options.frontmatter) {
|
|
61
|
+
const fm = resolveFrontmatter(options.frontmatter);
|
|
62
|
+
plugins.frontmatter = fm.config;
|
|
63
|
+
frontmatterCallback = fm.callback;
|
|
64
|
+
} else if (frontmatterDisabled) delete plugins.frontmatter;
|
|
65
|
+
const isolateMainDisabled = options.isolateMain === false;
|
|
66
|
+
if (minimal ? !isolateMainDisabled : options.isolateMain) plugins.isolateMain = true;
|
|
67
|
+
else if (isolateMainDisabled) delete plugins.isolateMain;
|
|
68
|
+
const tailwindDisabled = options.tailwind === false;
|
|
69
|
+
if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
|
|
70
|
+
else if (tailwindDisabled) delete plugins.tailwind;
|
|
71
|
+
if (minimal) plugins.filter = options.filter || MINIMAL_FILTER_DEFAULT;
|
|
72
|
+
else if (options.filter) plugins.filter = options.filter;
|
|
73
|
+
const extraction = options.extraction;
|
|
74
|
+
let extractionHandlers;
|
|
75
|
+
if (extraction) {
|
|
76
|
+
plugins.extraction = { selectors: Object.keys(extraction) };
|
|
77
|
+
extractionHandlers = extraction;
|
|
78
|
+
}
|
|
79
|
+
const tagOverrides = options.tagOverrides;
|
|
80
|
+
if (tagOverrides) {
|
|
81
|
+
const overrides = {};
|
|
82
|
+
for (const tag in tagOverrides) {
|
|
83
|
+
const value = tagOverrides[tag];
|
|
84
|
+
if (value) overrides[tag] = typeof value === "string" ? { alias: value } : value;
|
|
85
|
+
}
|
|
86
|
+
plugins.tagOverrides = overrides;
|
|
87
|
+
}
|
|
88
|
+
const { cleanUrls, clean } = resolveCleanConfig(options, minimal);
|
|
89
|
+
return {
|
|
90
|
+
napiOpts: {
|
|
91
|
+
origin: options.origin,
|
|
92
|
+
cleanUrls,
|
|
93
|
+
clean,
|
|
94
|
+
plugins,
|
|
95
|
+
wrapWidth: options.wrapWidth,
|
|
96
|
+
format: options.format
|
|
97
|
+
},
|
|
98
|
+
extractionHandlers,
|
|
99
|
+
frontmatterCallback
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
initSync({ module: wasmModule });
|
|
2
103
|
function htmlToMarkdown(html, options) {
|
|
3
|
-
return htmlToMarkdownResult(html,
|
|
104
|
+
if (!options) return htmlToMarkdownResult(html, void 0).markdown || "";
|
|
105
|
+
const { napiOpts, extractionHandlers, frontmatterCallback } = resolveOptions(options);
|
|
106
|
+
const result = htmlToMarkdownResult(html, napiOpts);
|
|
107
|
+
if (result.frontmatter && frontmatterCallback) frontmatterCallback(result.frontmatter);
|
|
108
|
+
if (result.extracted?.length && extractionHandlers) for (const element of result.extracted) extractionHandlers[element.selector]?.(element);
|
|
109
|
+
return result.markdown || "";
|
|
4
110
|
}
|
|
5
111
|
var MarkdownStream = class {
|
|
6
112
|
_inner;
|
|
7
113
|
constructor(options) {
|
|
8
|
-
this._inner = new MarkdownStream$1(options
|
|
114
|
+
this._inner = new MarkdownStream$1(options);
|
|
9
115
|
}
|
|
10
116
|
processChunk(chunk) {
|
|
11
117
|
return this._inner.processChunk(chunk);
|
|
@@ -16,17 +122,22 @@ var MarkdownStream = class {
|
|
|
16
122
|
};
|
|
17
123
|
async function* streamHtmlToMarkdown(htmlStream, options) {
|
|
18
124
|
if (!htmlStream) throw new Error("Invalid HTML stream provided");
|
|
19
|
-
const stream = new MarkdownStream$1(options
|
|
125
|
+
const stream = new MarkdownStream$1(options ? resolveOptions(options).napiOpts : void 0);
|
|
20
126
|
const reader = htmlStream.getReader();
|
|
21
127
|
const decoder = new TextDecoder();
|
|
22
128
|
try {
|
|
23
129
|
while (true) {
|
|
24
130
|
const { done, value } = await reader.read();
|
|
25
131
|
if (done) break;
|
|
26
|
-
const chunk = typeof value === "string" ? value : decoder.decode(value);
|
|
132
|
+
const chunk = typeof value === "string" ? decoder.decode() + value : decoder.decode(value, { stream: true });
|
|
27
133
|
const processed = stream.processChunk(chunk);
|
|
28
134
|
if (processed) yield processed;
|
|
29
135
|
}
|
|
136
|
+
const decoderTail = decoder.decode();
|
|
137
|
+
if (decoderTail) {
|
|
138
|
+
const processed = stream.processChunk(decoderTail);
|
|
139
|
+
if (processed) yield processed;
|
|
140
|
+
}
|
|
30
141
|
const final_ = stream.finish();
|
|
31
142
|
if (final_) yield final_;
|
|
32
143
|
} finally {
|