mdream 2.0.0-beta.1 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +319 -128
- package/bin/mdream.mjs +45 -24
- package/dist/browser-stub.mjs +7 -0
- package/dist/browser.d.mts +1 -3
- package/dist/browser.mjs +41 -30
- package/dist/edge.d.mts +0 -2
- package/dist/edge.mjs +41 -30
- package/dist/iife.js +1 -605
- package/dist/index.d.mts +0 -2
- package/dist/index.mjs +42 -32
- package/dist/wasm.d.mts +2 -5
- package/dist/wasm.mjs +41 -31
- package/napi/index.d.mts +0 -33
- package/napi/index.d.ts +0 -33
- package/napi/index.mjs +55 -58
- package/package.json +34 -34
- package/skills/mdream/SKILL.md +9 -5
- package/skills/mdream/references/runtimes.md +17 -9
- package/wasm/mdream_edge.js +3 -1
- package/wasm/mdream_edge_bg.wasm +0 -0
- package/wasm/package.json +1 -1
- package/wasm-bundler/mdream_edge_bg.js +3 -1
- package/wasm-bundler/mdream_edge_bg.wasm +0 -0
- package/wasm-bundler/package.json +1 -1
- package/dist/worker.d.mts +0 -101
- package/dist/worker.mjs +0 -217
package/bin/mdream.mjs
CHANGED
|
@@ -2,40 +2,61 @@
|
|
|
2
2
|
import { Readable } from 'node:stream'
|
|
3
3
|
import { streamHtmlToMarkdown } from 'mdream'
|
|
4
4
|
|
|
5
|
+
const USAGE = 'Usage: mdream [--origin <url>] [--preset minimal] [--wrap-width <n>] [--format markdown|text|html] [--text]\nPipe HTML via stdin, outputs Markdown, plain text, or safe HTML to stdout.\n'
|
|
6
|
+
|
|
7
|
+
function fail(message) {
|
|
8
|
+
process.stderr.write(`${message}\n`)
|
|
9
|
+
process.exit(1)
|
|
10
|
+
}
|
|
11
|
+
|
|
5
12
|
const args = process.argv.slice(2)
|
|
6
13
|
let origin
|
|
7
14
|
let preset
|
|
8
15
|
let wrapWidth
|
|
9
16
|
let format
|
|
10
17
|
for (let i = 0; i < args.length; i++) {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
18
|
+
const arg = args[i]
|
|
19
|
+
// `--flag=value` and `--flag value` both work.
|
|
20
|
+
const eq = arg.startsWith('--') ? arg.indexOf('=') : -1
|
|
21
|
+
const flag = eq === -1 ? arg : arg.slice(0, eq)
|
|
22
|
+
const takeValue = () => {
|
|
23
|
+
const value = eq === -1 ? args[++i] : arg.slice(eq + 1)
|
|
24
|
+
// A following flag, such as `--text` or `-h`, is not a value.
|
|
25
|
+
if (!value || (eq === -1 && (value.startsWith('--') || value === '-h')))
|
|
26
|
+
fail(`The ${flag} option needs a value.`)
|
|
27
|
+
return value
|
|
28
|
+
}
|
|
29
|
+
if (flag === '--origin') {
|
|
30
|
+
origin = takeValue()
|
|
31
|
+
}
|
|
32
|
+
else if (flag === '--preset') {
|
|
33
|
+
preset = takeValue()
|
|
34
|
+
if (preset !== 'minimal')
|
|
35
|
+
fail(`Unknown preset: ${preset}. Use --preset minimal.`)
|
|
36
|
+
}
|
|
37
|
+
else if (flag === '--wrap-width') {
|
|
38
|
+
wrapWidth = Number.parseInt(takeValue(), 10) || undefined
|
|
39
|
+
}
|
|
40
|
+
else if (flag === '--format') {
|
|
41
|
+
format = takeValue()
|
|
42
|
+
if (format !== 'markdown' && format !== 'text' && format !== 'html')
|
|
43
|
+
fail(`Unknown format: ${format}. Use markdown, text, or html.`)
|
|
44
|
+
}
|
|
45
|
+
else if (flag === '--text') {
|
|
46
|
+
if (eq !== -1)
|
|
47
|
+
fail('The --text option takes no value.')
|
|
33
48
|
format = 'text'
|
|
34
49
|
}
|
|
35
|
-
else if (
|
|
36
|
-
process.stdout.write(
|
|
50
|
+
else if (arg === '-h' || arg === '--help') {
|
|
51
|
+
process.stdout.write(USAGE)
|
|
37
52
|
process.exit(0)
|
|
38
53
|
}
|
|
54
|
+
else if (arg.startsWith('-')) {
|
|
55
|
+
fail(`Unknown option: ${flag}. Run mdream --help to list the options.`)
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
fail(`Unknown argument: ${arg}. Pipe the HTML to mdream on stdin.`)
|
|
59
|
+
}
|
|
39
60
|
}
|
|
40
61
|
|
|
41
62
|
const options = { origin, minimal: preset === 'minimal', wrapWidth, format }
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
function htmlToMarkdown() {
|
|
2
|
+
throw new TypeError("In a browser bundle, import htmlToMarkdown from 'mdream/browser'. It returns a Promise.");
|
|
3
|
+
}
|
|
4
|
+
function streamHtmlToMarkdown() {
|
|
5
|
+
throw new TypeError("In a browser bundle, import streamHtmlToMarkdown from 'mdream/browser'.");
|
|
6
|
+
}
|
|
7
|
+
export { htmlToMarkdown, streamHtmlToMarkdown };
|
package/dist/browser.d.mts
CHANGED
|
@@ -5,8 +5,6 @@ interface CleanOptions {
|
|
|
5
5
|
fragments?: boolean;
|
|
6
6
|
/** Strip links with meaningless hrefs (#, javascript:void(0)) → plain text */
|
|
7
7
|
emptyLinks?: boolean;
|
|
8
|
-
/** Collapse 3+ consecutive blank lines to 2 */
|
|
9
|
-
blankLines?: boolean;
|
|
10
8
|
/** Strip links where text equals URL: [https://x.com](https://x.com) → https://x.com */
|
|
11
9
|
redundantLinks?: boolean;
|
|
12
10
|
/** Strip self-referencing heading anchors: ## [Title](#title) → ## Title */
|
|
@@ -87,7 +85,7 @@ interface MdreamOptions {
|
|
|
87
85
|
}
|
|
88
86
|
/**
|
|
89
87
|
* Browser builds fetch the WASM binary on first use, so this returns a
|
|
90
|
-
* Promise.
|
|
88
|
+
* Promise. `mdream/browser` ships types that say so.
|
|
91
89
|
*/
|
|
92
90
|
export declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): Promise<string>;
|
|
93
91
|
export declare function createMarkdownStream(options?: Partial<MdreamOptions>): Promise<MarkdownStream>;
|
package/dist/browser.mjs
CHANGED
|
@@ -22,19 +22,38 @@ const CLEAN_ALL = {
|
|
|
22
22
|
emptyImages: true,
|
|
23
23
|
emptyLinkText: true
|
|
24
24
|
};
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
25
|
+
const VALID_OPTIONS = "origin, clean, minimal, frontmatter, isolateMain, tailwind, filter, extraction, tagOverrides, wrapWidth, format";
|
|
26
|
+
function unknownOptionMessage(key, value) {
|
|
27
|
+
switch (key) {
|
|
28
|
+
case "preset": return "mdream has no `preset` option. Pass { minimal: true }.";
|
|
29
|
+
case "cleanUrls": return "mdream has no `cleanUrls` option. Pass { clean: { urls: true } }.";
|
|
30
|
+
case "plugins": return Array.isArray(value) ? "Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://github.com/harlan-zw/mdream/tree/main/packages/js#migrating-from-v1" : "mdream takes plugin options at the top level, such as { frontmatter: true }.";
|
|
31
31
|
}
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
32
|
+
return `mdream has no \`${key}\` option. Valid options: ${VALID_OPTIONS}.`;
|
|
33
|
+
}
|
|
34
|
+
function assertKnownOptions(options) {
|
|
35
|
+
for (const key in options) {
|
|
36
|
+
switch (key) {
|
|
37
|
+
case "origin":
|
|
38
|
+
case "clean":
|
|
39
|
+
case "minimal":
|
|
40
|
+
case "frontmatter":
|
|
41
|
+
case "isolateMain":
|
|
42
|
+
case "tailwind":
|
|
43
|
+
case "filter":
|
|
44
|
+
case "extraction":
|
|
45
|
+
case "tagOverrides":
|
|
46
|
+
case "wrapWidth":
|
|
47
|
+
case "format": continue;
|
|
48
|
+
}
|
|
49
|
+
const value = options[key];
|
|
50
|
+
if (value !== void 0) throw new TypeError(unknownOptionMessage(key, value));
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
function resolveClean(clean, minimal) {
|
|
54
|
+
if (clean === void 0) return minimal ? CLEAN_ALL : void 0;
|
|
55
|
+
if (!clean) return void 0;
|
|
56
|
+
return clean === true ? CLEAN_ALL : clean;
|
|
38
57
|
}
|
|
39
58
|
function resolveFrontmatter(opt) {
|
|
40
59
|
if (typeof opt === "function") return {
|
|
@@ -51,24 +70,18 @@ function resolveFrontmatter(opt) {
|
|
|
51
70
|
return { config: {} };
|
|
52
71
|
}
|
|
53
72
|
function resolveOptions(options) {
|
|
54
|
-
|
|
73
|
+
assertKnownOptions(options);
|
|
55
74
|
const minimal = options.minimal === true;
|
|
56
|
-
const plugins =
|
|
75
|
+
const plugins = {};
|
|
57
76
|
let frontmatterCallback;
|
|
58
|
-
|
|
59
|
-
if (minimal ? !frontmatterDisabled : !!options.frontmatter) {
|
|
77
|
+
if (minimal ? options.frontmatter !== false : options.frontmatter) {
|
|
60
78
|
const fm = resolveFrontmatter(options.frontmatter);
|
|
61
79
|
plugins.frontmatter = fm.config;
|
|
62
80
|
frontmatterCallback = fm.callback;
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
if (minimal ?
|
|
66
|
-
|
|
67
|
-
const tailwindDisabled = options.tailwind === false;
|
|
68
|
-
if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
|
|
69
|
-
else if (tailwindDisabled) delete plugins.tailwind;
|
|
70
|
-
if (options.filter === false) delete plugins.filter;
|
|
71
|
-
else if (minimal) plugins.filter = {
|
|
81
|
+
}
|
|
82
|
+
if (minimal ? options.isolateMain !== false : options.isolateMain) plugins.isolateMain = true;
|
|
83
|
+
if (minimal ? options.tailwind !== false : options.tailwind) plugins.tailwind = true;
|
|
84
|
+
if (minimal && options.filter !== false) plugins.filter = {
|
|
72
85
|
...options.filter,
|
|
73
86
|
exclude: [...MINIMAL_FILTER_EXCLUDE, ...options.filter?.exclude ?? []]
|
|
74
87
|
};
|
|
@@ -88,12 +101,10 @@ function resolveOptions(options) {
|
|
|
88
101
|
}
|
|
89
102
|
plugins.tagOverrides = overrides;
|
|
90
103
|
}
|
|
91
|
-
const { cleanUrls, clean } = resolveCleanConfig(options, minimal);
|
|
92
104
|
return {
|
|
93
105
|
napiOpts: {
|
|
94
106
|
origin: options.origin,
|
|
95
|
-
|
|
96
|
-
clean,
|
|
107
|
+
clean: resolveClean(options.clean, minimal),
|
|
97
108
|
plugins,
|
|
98
109
|
wrapWidth: options.wrapWidth,
|
|
99
110
|
format: options.format
|
|
@@ -115,9 +126,9 @@ function convertResult(convert, html, options = {}) {
|
|
|
115
126
|
deliverPluginData(result, resolved);
|
|
116
127
|
return result;
|
|
117
128
|
}
|
|
118
|
-
async function* pumpStream(stream, htmlStream, callbacks, {
|
|
129
|
+
async function* pumpStream(stream, htmlStream, callbacks, { mapError = (error) => error } = {}) {
|
|
119
130
|
const reader = htmlStream.getReader();
|
|
120
|
-
const decoder =
|
|
131
|
+
const decoder = stream.processChunkBytes ? void 0 : new TextDecoder();
|
|
121
132
|
try {
|
|
122
133
|
while (true) {
|
|
123
134
|
const { done, value } = await reader.read();
|
package/dist/edge.d.mts
CHANGED
|
@@ -5,8 +5,6 @@ interface CleanOptions {
|
|
|
5
5
|
fragments?: boolean;
|
|
6
6
|
/** Strip links with meaningless hrefs (#, javascript:void(0)) → plain text */
|
|
7
7
|
emptyLinks?: boolean;
|
|
8
|
-
/** Collapse 3+ consecutive blank lines to 2 */
|
|
9
|
-
blankLines?: boolean;
|
|
10
8
|
/** Strip links where text equals URL: [https://x.com](https://x.com) → https://x.com */
|
|
11
9
|
redundantLinks?: boolean;
|
|
12
10
|
/** Strip self-referencing heading anchors: ## [Title](#title) → ## Title */
|
package/dist/edge.mjs
CHANGED
|
@@ -22,19 +22,38 @@ const CLEAN_ALL = {
|
|
|
22
22
|
emptyImages: true,
|
|
23
23
|
emptyLinkText: true
|
|
24
24
|
};
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
25
|
+
const VALID_OPTIONS = "origin, clean, minimal, frontmatter, isolateMain, tailwind, filter, extraction, tagOverrides, wrapWidth, format";
|
|
26
|
+
function unknownOptionMessage(key, value) {
|
|
27
|
+
switch (key) {
|
|
28
|
+
case "preset": return "mdream has no `preset` option. Pass { minimal: true }.";
|
|
29
|
+
case "cleanUrls": return "mdream has no `cleanUrls` option. Pass { clean: { urls: true } }.";
|
|
30
|
+
case "plugins": return Array.isArray(value) ? "Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://github.com/harlan-zw/mdream/tree/main/packages/js#migrating-from-v1" : "mdream takes plugin options at the top level, such as { frontmatter: true }.";
|
|
31
31
|
}
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
32
|
+
return `mdream has no \`${key}\` option. Valid options: ${VALID_OPTIONS}.`;
|
|
33
|
+
}
|
|
34
|
+
function assertKnownOptions(options) {
|
|
35
|
+
for (const key in options) {
|
|
36
|
+
switch (key) {
|
|
37
|
+
case "origin":
|
|
38
|
+
case "clean":
|
|
39
|
+
case "minimal":
|
|
40
|
+
case "frontmatter":
|
|
41
|
+
case "isolateMain":
|
|
42
|
+
case "tailwind":
|
|
43
|
+
case "filter":
|
|
44
|
+
case "extraction":
|
|
45
|
+
case "tagOverrides":
|
|
46
|
+
case "wrapWidth":
|
|
47
|
+
case "format": continue;
|
|
48
|
+
}
|
|
49
|
+
const value = options[key];
|
|
50
|
+
if (value !== void 0) throw new TypeError(unknownOptionMessage(key, value));
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
function resolveClean(clean, minimal) {
|
|
54
|
+
if (clean === void 0) return minimal ? CLEAN_ALL : void 0;
|
|
55
|
+
if (!clean) return void 0;
|
|
56
|
+
return clean === true ? CLEAN_ALL : clean;
|
|
38
57
|
}
|
|
39
58
|
function resolveFrontmatter(opt) {
|
|
40
59
|
if (typeof opt === "function") return {
|
|
@@ -51,24 +70,18 @@ function resolveFrontmatter(opt) {
|
|
|
51
70
|
return { config: {} };
|
|
52
71
|
}
|
|
53
72
|
function resolveOptions(options) {
|
|
54
|
-
|
|
73
|
+
assertKnownOptions(options);
|
|
55
74
|
const minimal = options.minimal === true;
|
|
56
|
-
const plugins =
|
|
75
|
+
const plugins = {};
|
|
57
76
|
let frontmatterCallback;
|
|
58
|
-
|
|
59
|
-
if (minimal ? !frontmatterDisabled : !!options.frontmatter) {
|
|
77
|
+
if (minimal ? options.frontmatter !== false : options.frontmatter) {
|
|
60
78
|
const fm = resolveFrontmatter(options.frontmatter);
|
|
61
79
|
plugins.frontmatter = fm.config;
|
|
62
80
|
frontmatterCallback = fm.callback;
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
if (minimal ?
|
|
66
|
-
|
|
67
|
-
const tailwindDisabled = options.tailwind === false;
|
|
68
|
-
if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
|
|
69
|
-
else if (tailwindDisabled) delete plugins.tailwind;
|
|
70
|
-
if (options.filter === false) delete plugins.filter;
|
|
71
|
-
else if (minimal) plugins.filter = {
|
|
81
|
+
}
|
|
82
|
+
if (minimal ? options.isolateMain !== false : options.isolateMain) plugins.isolateMain = true;
|
|
83
|
+
if (minimal ? options.tailwind !== false : options.tailwind) plugins.tailwind = true;
|
|
84
|
+
if (minimal && options.filter !== false) plugins.filter = {
|
|
72
85
|
...options.filter,
|
|
73
86
|
exclude: [...MINIMAL_FILTER_EXCLUDE, ...options.filter?.exclude ?? []]
|
|
74
87
|
};
|
|
@@ -88,12 +101,10 @@ function resolveOptions(options) {
|
|
|
88
101
|
}
|
|
89
102
|
plugins.tagOverrides = overrides;
|
|
90
103
|
}
|
|
91
|
-
const { cleanUrls, clean } = resolveCleanConfig(options, minimal);
|
|
92
104
|
return {
|
|
93
105
|
napiOpts: {
|
|
94
106
|
origin: options.origin,
|
|
95
|
-
|
|
96
|
-
clean,
|
|
107
|
+
clean: resolveClean(options.clean, minimal),
|
|
97
108
|
plugins,
|
|
98
109
|
wrapWidth: options.wrapWidth,
|
|
99
110
|
format: options.format
|
|
@@ -115,9 +126,9 @@ function convertResult(convert, html, options = {}) {
|
|
|
115
126
|
deliverPluginData(result, resolved);
|
|
116
127
|
return result;
|
|
117
128
|
}
|
|
118
|
-
async function* pumpStream(stream, htmlStream, callbacks, {
|
|
129
|
+
async function* pumpStream(stream, htmlStream, callbacks, { mapError = (error) => error } = {}) {
|
|
119
130
|
const reader = htmlStream.getReader();
|
|
120
|
-
const decoder =
|
|
131
|
+
const decoder = stream.processChunkBytes ? void 0 : new TextDecoder();
|
|
121
132
|
try {
|
|
122
133
|
while (true) {
|
|
123
134
|
const { done, value } = await reader.read();
|