mdream 2.0.0-beta.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +319 -128
- package/bin/mdream.mjs +45 -24
- package/dist/browser-stub.mjs +7 -0
- package/dist/browser.d.mts +1 -3
- package/dist/browser.mjs +41 -30
- package/dist/edge.d.mts +0 -2
- package/dist/edge.mjs +41 -30
- package/dist/iife.js +1 -605
- package/dist/index.d.mts +0 -2
- package/dist/index.mjs +42 -32
- package/dist/wasm.d.mts +2 -5
- package/dist/wasm.mjs +41 -31
- package/napi/index.d.mts +0 -33
- package/napi/index.d.ts +0 -33
- package/napi/index.mjs +1 -4
- package/package.json +34 -34
- package/skills/mdream/SKILL.md +9 -5
- package/skills/mdream/references/runtimes.md +17 -9
- package/wasm/mdream_edge.js +3 -1
- package/wasm/mdream_edge_bg.wasm +0 -0
- package/wasm/package.json +1 -1
- package/wasm-bundler/mdream_edge_bg.js +3 -1
- package/wasm-bundler/mdream_edge_bg.wasm +0 -0
- package/wasm-bundler/package.json +1 -1
- package/dist/worker.d.mts +0 -101
- package/dist/worker.mjs +0 -217
package/dist/index.d.mts
CHANGED
|
@@ -5,8 +5,6 @@ export interface CleanOptions {
|
|
|
5
5
|
fragments?: boolean;
|
|
6
6
|
/** Strip links with meaningless hrefs (#, javascript:void(0)) → plain text */
|
|
7
7
|
emptyLinks?: boolean;
|
|
8
|
-
/** Collapse 3+ consecutive blank lines to 2 */
|
|
9
|
-
blankLines?: boolean;
|
|
10
8
|
/** Strip links where text equals URL: [https://x.com](https://x.com) → https://x.com */
|
|
11
9
|
redundantLinks?: boolean;
|
|
12
10
|
/** Strip self-referencing heading anchors: ## [Title](#title) → ## Title */
|
package/dist/index.mjs
CHANGED
|
@@ -22,19 +22,38 @@ const CLEAN_ALL = {
|
|
|
22
22
|
emptyImages: true,
|
|
23
23
|
emptyLinkText: true
|
|
24
24
|
};
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
25
|
+
const VALID_OPTIONS = "origin, clean, minimal, frontmatter, isolateMain, tailwind, filter, extraction, tagOverrides, wrapWidth, format";
|
|
26
|
+
function unknownOptionMessage(key, value) {
|
|
27
|
+
switch (key) {
|
|
28
|
+
case "preset": return "mdream has no `preset` option. Pass { minimal: true }.";
|
|
29
|
+
case "cleanUrls": return "mdream has no `cleanUrls` option. Pass { clean: { urls: true } }.";
|
|
30
|
+
case "plugins": return Array.isArray(value) ? "Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://github.com/harlan-zw/mdream/tree/main/packages/js#migrating-from-v1" : "mdream takes plugin options at the top level, such as { frontmatter: true }.";
|
|
31
31
|
}
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
32
|
+
return `mdream has no \`${key}\` option. Valid options: ${VALID_OPTIONS}.`;
|
|
33
|
+
}
|
|
34
|
+
function assertKnownOptions(options) {
|
|
35
|
+
for (const key in options) {
|
|
36
|
+
switch (key) {
|
|
37
|
+
case "origin":
|
|
38
|
+
case "clean":
|
|
39
|
+
case "minimal":
|
|
40
|
+
case "frontmatter":
|
|
41
|
+
case "isolateMain":
|
|
42
|
+
case "tailwind":
|
|
43
|
+
case "filter":
|
|
44
|
+
case "extraction":
|
|
45
|
+
case "tagOverrides":
|
|
46
|
+
case "wrapWidth":
|
|
47
|
+
case "format": continue;
|
|
48
|
+
}
|
|
49
|
+
const value = options[key];
|
|
50
|
+
if (value !== void 0) throw new TypeError(unknownOptionMessage(key, value));
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
function resolveClean(clean, minimal) {
|
|
54
|
+
if (clean === void 0) return minimal ? CLEAN_ALL : void 0;
|
|
55
|
+
if (!clean) return void 0;
|
|
56
|
+
return clean === true ? CLEAN_ALL : clean;
|
|
38
57
|
}
|
|
39
58
|
function resolveFrontmatter(opt) {
|
|
40
59
|
if (typeof opt === "function") return {
|
|
@@ -51,24 +70,18 @@ function resolveFrontmatter(opt) {
|
|
|
51
70
|
return { config: {} };
|
|
52
71
|
}
|
|
53
72
|
function resolveOptions(options) {
|
|
54
|
-
|
|
73
|
+
assertKnownOptions(options);
|
|
55
74
|
const minimal = options.minimal === true;
|
|
56
|
-
const plugins =
|
|
75
|
+
const plugins = {};
|
|
57
76
|
let frontmatterCallback;
|
|
58
|
-
|
|
59
|
-
if (minimal ? !frontmatterDisabled : !!options.frontmatter) {
|
|
77
|
+
if (minimal ? options.frontmatter !== false : options.frontmatter) {
|
|
60
78
|
const fm = resolveFrontmatter(options.frontmatter);
|
|
61
79
|
plugins.frontmatter = fm.config;
|
|
62
80
|
frontmatterCallback = fm.callback;
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
if (minimal ?
|
|
66
|
-
|
|
67
|
-
const tailwindDisabled = options.tailwind === false;
|
|
68
|
-
if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
|
|
69
|
-
else if (tailwindDisabled) delete plugins.tailwind;
|
|
70
|
-
if (options.filter === false) delete plugins.filter;
|
|
71
|
-
else if (minimal) plugins.filter = {
|
|
81
|
+
}
|
|
82
|
+
if (minimal ? options.isolateMain !== false : options.isolateMain) plugins.isolateMain = true;
|
|
83
|
+
if (minimal ? options.tailwind !== false : options.tailwind) plugins.tailwind = true;
|
|
84
|
+
if (minimal && options.filter !== false) plugins.filter = {
|
|
72
85
|
...options.filter,
|
|
73
86
|
exclude: [...MINIMAL_FILTER_EXCLUDE, ...options.filter?.exclude ?? []]
|
|
74
87
|
};
|
|
@@ -88,12 +101,10 @@ function resolveOptions(options) {
|
|
|
88
101
|
}
|
|
89
102
|
plugins.tagOverrides = overrides;
|
|
90
103
|
}
|
|
91
|
-
const { cleanUrls, clean } = resolveCleanConfig(options, minimal);
|
|
92
104
|
return {
|
|
93
105
|
napiOpts: {
|
|
94
106
|
origin: options.origin,
|
|
95
|
-
|
|
96
|
-
clean,
|
|
107
|
+
clean: resolveClean(options.clean, minimal),
|
|
97
108
|
plugins,
|
|
98
109
|
wrapWidth: options.wrapWidth,
|
|
99
110
|
format: options.format
|
|
@@ -115,9 +126,9 @@ function convertResult(convert, html, options = {}) {
|
|
|
115
126
|
deliverPluginData(result, resolved);
|
|
116
127
|
return result;
|
|
117
128
|
}
|
|
118
|
-
async function* pumpStream(stream, htmlStream, callbacks, {
|
|
129
|
+
async function* pumpStream(stream, htmlStream, callbacks, { mapError = (error) => error } = {}) {
|
|
119
130
|
const reader = htmlStream.getReader();
|
|
120
|
-
const decoder =
|
|
131
|
+
const decoder = stream.processChunkBytes ? void 0 : new TextDecoder();
|
|
121
132
|
try {
|
|
122
133
|
while (true) {
|
|
123
134
|
const { done, value } = await reader.read();
|
|
@@ -173,12 +184,11 @@ async function* streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
|
173
184
|
const stream = new MarkdownStream(resolved.napiOpts);
|
|
174
185
|
yield* pumpStream({
|
|
175
186
|
processChunk: (chunk) => stream.processChunk(carry.take(chunk)),
|
|
176
|
-
processChunkBytes: (chunk) => stream.processChunkBytes(chunk),
|
|
177
187
|
finish: () => {
|
|
178
188
|
const held = carry.flush();
|
|
179
189
|
return held ? stream.processChunk(held) + stream.finish() : stream.finish();
|
|
180
190
|
},
|
|
181
191
|
takeData: () => stream.takeData()
|
|
182
|
-
}, htmlStream, resolved
|
|
192
|
+
}, htmlStream, resolved);
|
|
183
193
|
}
|
|
184
194
|
export { htmlToMarkdown, streamHtmlToMarkdown };
|
package/dist/wasm.d.mts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import mdream_edge_default, { MarkdownStream as MarkdownStream$1 } from "../wasm/mdream_edge.js";
|
|
1
|
+
import mdream_edge_default, { InitInput, InitOutput, MarkdownStream as MarkdownStream$1, SyncInitInput, initSync } from "../wasm/mdream_edge.js";
|
|
2
2
|
import { MdreamNapiResult } from "../napi/index.js";
|
|
3
|
-
export * from "../wasm/mdream_edge.js";
|
|
4
3
|
interface CleanOptions {
|
|
5
4
|
/** Strip tracking query parameters (utm_*, fbclid, gclid, etc.) from URLs */
|
|
6
5
|
urls?: boolean;
|
|
@@ -8,8 +7,6 @@ interface CleanOptions {
|
|
|
8
7
|
fragments?: boolean;
|
|
9
8
|
/** Strip links with meaningless hrefs (#, javascript:void(0)) → plain text */
|
|
10
9
|
emptyLinks?: boolean;
|
|
11
|
-
/** Collapse 3+ consecutive blank lines to 2 */
|
|
12
|
-
blankLines?: boolean;
|
|
13
10
|
/** Strip links where text equals URL: [https://x.com](https://x.com) → https://x.com */
|
|
14
11
|
redundantLinks?: boolean;
|
|
15
12
|
/** Strip self-referencing heading anchors: ## [Title](#title) → ## Title */
|
|
@@ -97,4 +94,4 @@ export declare class MarkdownStream extends MarkdownStream$1 {
|
|
|
97
94
|
constructor(options?: Partial<MdreamOptions>);
|
|
98
95
|
finish(): string;
|
|
99
96
|
}
|
|
100
|
-
export { mdream_edge_default as default };
|
|
97
|
+
export { type InitInput, type InitOutput, type SyncInitInput, mdream_edge_default as default, initSync };
|
package/dist/wasm.mjs
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import mdream_edge_default, { MarkdownStream as MarkdownStream$1, htmlToMarkdownBytes as htmlToMarkdownBytes$1, htmlToMarkdownResult as htmlToMarkdownResult$1 } from "../wasm/mdream_edge.js";
|
|
2
|
-
export * from "../wasm/mdream_edge.js";
|
|
1
|
+
import mdream_edge_default, { MarkdownStream as MarkdownStream$1, htmlToMarkdownBytes as htmlToMarkdownBytes$1, htmlToMarkdownResult as htmlToMarkdownResult$1, initSync } from "../wasm/mdream_edge.js";
|
|
3
2
|
const MINIMAL_FILTER_EXCLUDE = [
|
|
4
3
|
"form",
|
|
5
4
|
"fieldset",
|
|
@@ -23,19 +22,38 @@ const CLEAN_ALL = {
|
|
|
23
22
|
emptyImages: true,
|
|
24
23
|
emptyLinkText: true
|
|
25
24
|
};
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
25
|
+
const VALID_OPTIONS = "origin, clean, minimal, frontmatter, isolateMain, tailwind, filter, extraction, tagOverrides, wrapWidth, format";
|
|
26
|
+
function unknownOptionMessage(key, value) {
|
|
27
|
+
switch (key) {
|
|
28
|
+
case "preset": return "mdream has no `preset` option. Pass { minimal: true }.";
|
|
29
|
+
case "cleanUrls": return "mdream has no `cleanUrls` option. Pass { clean: { urls: true } }.";
|
|
30
|
+
case "plugins": return Array.isArray(value) ? "Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://github.com/harlan-zw/mdream/tree/main/packages/js#migrating-from-v1" : "mdream takes plugin options at the top level, such as { frontmatter: true }.";
|
|
32
31
|
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
32
|
+
return `mdream has no \`${key}\` option. Valid options: ${VALID_OPTIONS}.`;
|
|
33
|
+
}
|
|
34
|
+
function assertKnownOptions(options) {
|
|
35
|
+
for (const key in options) {
|
|
36
|
+
switch (key) {
|
|
37
|
+
case "origin":
|
|
38
|
+
case "clean":
|
|
39
|
+
case "minimal":
|
|
40
|
+
case "frontmatter":
|
|
41
|
+
case "isolateMain":
|
|
42
|
+
case "tailwind":
|
|
43
|
+
case "filter":
|
|
44
|
+
case "extraction":
|
|
45
|
+
case "tagOverrides":
|
|
46
|
+
case "wrapWidth":
|
|
47
|
+
case "format": continue;
|
|
48
|
+
}
|
|
49
|
+
const value = options[key];
|
|
50
|
+
if (value !== void 0) throw new TypeError(unknownOptionMessage(key, value));
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
function resolveClean(clean, minimal) {
|
|
54
|
+
if (clean === void 0) return minimal ? CLEAN_ALL : void 0;
|
|
55
|
+
if (!clean) return void 0;
|
|
56
|
+
return clean === true ? CLEAN_ALL : clean;
|
|
39
57
|
}
|
|
40
58
|
function resolveFrontmatter(opt) {
|
|
41
59
|
if (typeof opt === "function") return {
|
|
@@ -52,24 +70,18 @@ function resolveFrontmatter(opt) {
|
|
|
52
70
|
return { config: {} };
|
|
53
71
|
}
|
|
54
72
|
function resolveOptions(options) {
|
|
55
|
-
|
|
73
|
+
assertKnownOptions(options);
|
|
56
74
|
const minimal = options.minimal === true;
|
|
57
|
-
const plugins =
|
|
75
|
+
const plugins = {};
|
|
58
76
|
let frontmatterCallback;
|
|
59
|
-
|
|
60
|
-
if (minimal ? !frontmatterDisabled : !!options.frontmatter) {
|
|
77
|
+
if (minimal ? options.frontmatter !== false : options.frontmatter) {
|
|
61
78
|
const fm = resolveFrontmatter(options.frontmatter);
|
|
62
79
|
plugins.frontmatter = fm.config;
|
|
63
80
|
frontmatterCallback = fm.callback;
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
if (minimal ?
|
|
67
|
-
|
|
68
|
-
const tailwindDisabled = options.tailwind === false;
|
|
69
|
-
if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
|
|
70
|
-
else if (tailwindDisabled) delete plugins.tailwind;
|
|
71
|
-
if (options.filter === false) delete plugins.filter;
|
|
72
|
-
else if (minimal) plugins.filter = {
|
|
81
|
+
}
|
|
82
|
+
if (minimal ? options.isolateMain !== false : options.isolateMain) plugins.isolateMain = true;
|
|
83
|
+
if (minimal ? options.tailwind !== false : options.tailwind) plugins.tailwind = true;
|
|
84
|
+
if (minimal && options.filter !== false) plugins.filter = {
|
|
73
85
|
...options.filter,
|
|
74
86
|
exclude: [...MINIMAL_FILTER_EXCLUDE, ...options.filter?.exclude ?? []]
|
|
75
87
|
};
|
|
@@ -89,12 +101,10 @@ function resolveOptions(options) {
|
|
|
89
101
|
}
|
|
90
102
|
plugins.tagOverrides = overrides;
|
|
91
103
|
}
|
|
92
|
-
const { cleanUrls, clean } = resolveCleanConfig(options, minimal);
|
|
93
104
|
return {
|
|
94
105
|
napiOpts: {
|
|
95
106
|
origin: options.origin,
|
|
96
|
-
|
|
97
|
-
clean,
|
|
107
|
+
clean: resolveClean(options.clean, minimal),
|
|
98
108
|
plugins,
|
|
99
109
|
wrapWidth: options.wrapWidth,
|
|
100
110
|
format: options.format
|
|
@@ -137,4 +147,4 @@ var MarkdownStream = class extends MarkdownStream$1 {
|
|
|
137
147
|
return markdown;
|
|
138
148
|
}
|
|
139
149
|
};
|
|
140
|
-
export { MarkdownStream, mdream_edge_default as default, htmlToMarkdown, htmlToMarkdownBytes, htmlToMarkdownResult };
|
|
150
|
+
export { MarkdownStream, mdream_edge_default as default, htmlToMarkdown, htmlToMarkdownBytes, htmlToMarkdownResult, initSync };
|
package/napi/index.d.mts
CHANGED
|
@@ -12,7 +12,6 @@ export declare const __napiBindingTarget: 'native' | 'wasm32-wasi' | 'wasm32-was
|
|
|
12
12
|
export declare class MarkdownStream {
|
|
13
13
|
constructor(options?: HtmlToMarkdownOptions | undefined | null)
|
|
14
14
|
processChunk(chunk: string): string
|
|
15
|
-
processChunkBytes(chunk: Uint8Array): string
|
|
16
15
|
finish(): string
|
|
17
16
|
/**
|
|
18
17
|
* Frontmatter and extracted elements collected so far. Call after
|
|
@@ -21,22 +20,10 @@ export declare class MarkdownStream {
|
|
|
21
20
|
takeData(): MdreamStreamData
|
|
22
21
|
}
|
|
23
22
|
|
|
24
|
-
export interface ChunkLocNapi {
|
|
25
|
-
from: number
|
|
26
|
-
to: number
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
export interface ChunkMetadataNapi {
|
|
30
|
-
headers?: Record<string, string>
|
|
31
|
-
code?: string
|
|
32
|
-
loc?: ChunkLocNapi
|
|
33
|
-
}
|
|
34
|
-
|
|
35
23
|
export interface CleanOptionsNapi {
|
|
36
24
|
urls?: boolean
|
|
37
25
|
fragments?: boolean
|
|
38
26
|
emptyLinks?: boolean
|
|
39
|
-
blankLines?: boolean
|
|
40
27
|
redundantLinks?: boolean
|
|
41
28
|
selfLinkHeadings?: boolean
|
|
42
29
|
emptyImages?: boolean
|
|
@@ -67,24 +54,14 @@ export interface FrontmatterOptions {
|
|
|
67
54
|
|
|
68
55
|
export declare function htmlToMarkdown(html: string, options?: HtmlToMarkdownOptions | undefined | null): MdreamNapiResult
|
|
69
56
|
|
|
70
|
-
export declare function htmlToMarkdownBytes(html: Uint8Array, options?: HtmlToMarkdownOptions | undefined | null): MdreamNapiResult
|
|
71
|
-
|
|
72
|
-
export declare function htmlToMarkdownChunks(html: string, options?: HtmlToMarkdownOptions | undefined | null, splitterOptions?: SplitterOptionsNapi | undefined | null): Array<MarkdownChunkNapi>
|
|
73
|
-
|
|
74
57
|
export interface HtmlToMarkdownOptions {
|
|
75
58
|
origin?: string
|
|
76
|
-
cleanUrls?: boolean
|
|
77
59
|
clean?: CleanOptionsNapi
|
|
78
60
|
plugins?: PluginOptions
|
|
79
61
|
wrapWidth?: number
|
|
80
62
|
format?: "markdown" | "text" | "html"
|
|
81
63
|
}
|
|
82
64
|
|
|
83
|
-
export interface MarkdownChunkNapi {
|
|
84
|
-
content: string
|
|
85
|
-
metadata: ChunkMetadataNapi
|
|
86
|
-
}
|
|
87
|
-
|
|
88
65
|
export interface MdreamNapiResult {
|
|
89
66
|
markdown: string
|
|
90
67
|
extracted?: Array<ExtractedElementNapi>
|
|
@@ -106,16 +83,6 @@ export interface PluginOptions {
|
|
|
106
83
|
tagOverrides?: Record<string, TagOverrideNapi>
|
|
107
84
|
}
|
|
108
85
|
|
|
109
|
-
export declare function splitMarkdown(markdown: string, options?: SplitterOptionsNapi | undefined | null): Array<MarkdownChunkNapi>
|
|
110
|
-
|
|
111
|
-
export interface SplitterOptionsNapi {
|
|
112
|
-
headersToSplitOn?: Array<number>
|
|
113
|
-
returnEachLine?: boolean
|
|
114
|
-
stripHeaders?: boolean
|
|
115
|
-
chunkSize?: number
|
|
116
|
-
chunkOverlap?: number
|
|
117
|
-
}
|
|
118
|
-
|
|
119
86
|
export interface TagOverrideNapi {
|
|
120
87
|
enter?: string
|
|
121
88
|
exit?: string
|
package/napi/index.d.ts
CHANGED
|
@@ -12,7 +12,6 @@ export declare const __napiBindingTarget: 'native' | 'wasm32-wasi' | 'wasm32-was
|
|
|
12
12
|
export declare class MarkdownStream {
|
|
13
13
|
constructor(options?: HtmlToMarkdownOptions | undefined | null)
|
|
14
14
|
processChunk(chunk: string): string
|
|
15
|
-
processChunkBytes(chunk: Uint8Array): string
|
|
16
15
|
finish(): string
|
|
17
16
|
/**
|
|
18
17
|
* Frontmatter and extracted elements collected so far. Call after
|
|
@@ -21,22 +20,10 @@ export declare class MarkdownStream {
|
|
|
21
20
|
takeData(): MdreamStreamData
|
|
22
21
|
}
|
|
23
22
|
|
|
24
|
-
export interface ChunkLocNapi {
|
|
25
|
-
from: number
|
|
26
|
-
to: number
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
export interface ChunkMetadataNapi {
|
|
30
|
-
headers?: Record<string, string>
|
|
31
|
-
code?: string
|
|
32
|
-
loc?: ChunkLocNapi
|
|
33
|
-
}
|
|
34
|
-
|
|
35
23
|
export interface CleanOptionsNapi {
|
|
36
24
|
urls?: boolean
|
|
37
25
|
fragments?: boolean
|
|
38
26
|
emptyLinks?: boolean
|
|
39
|
-
blankLines?: boolean
|
|
40
27
|
redundantLinks?: boolean
|
|
41
28
|
selfLinkHeadings?: boolean
|
|
42
29
|
emptyImages?: boolean
|
|
@@ -67,24 +54,14 @@ export interface FrontmatterOptions {
|
|
|
67
54
|
|
|
68
55
|
export declare function htmlToMarkdown(html: string, options?: HtmlToMarkdownOptions | undefined | null): MdreamNapiResult
|
|
69
56
|
|
|
70
|
-
export declare function htmlToMarkdownBytes(html: Uint8Array, options?: HtmlToMarkdownOptions | undefined | null): MdreamNapiResult
|
|
71
|
-
|
|
72
|
-
export declare function htmlToMarkdownChunks(html: string, options?: HtmlToMarkdownOptions | undefined | null, splitterOptions?: SplitterOptionsNapi | undefined | null): Array<MarkdownChunkNapi>
|
|
73
|
-
|
|
74
57
|
export interface HtmlToMarkdownOptions {
|
|
75
58
|
origin?: string
|
|
76
|
-
cleanUrls?: boolean
|
|
77
59
|
clean?: CleanOptionsNapi
|
|
78
60
|
plugins?: PluginOptions
|
|
79
61
|
wrapWidth?: number
|
|
80
62
|
format?: "markdown" | "text" | "html"
|
|
81
63
|
}
|
|
82
64
|
|
|
83
|
-
export interface MarkdownChunkNapi {
|
|
84
|
-
content: string
|
|
85
|
-
metadata: ChunkMetadataNapi
|
|
86
|
-
}
|
|
87
|
-
|
|
88
65
|
export interface MdreamNapiResult {
|
|
89
66
|
markdown: string
|
|
90
67
|
extracted?: Array<ExtractedElementNapi>
|
|
@@ -106,16 +83,6 @@ export interface PluginOptions {
|
|
|
106
83
|
tagOverrides?: Record<string, TagOverrideNapi>
|
|
107
84
|
}
|
|
108
85
|
|
|
109
|
-
export declare function splitMarkdown(markdown: string, options?: SplitterOptionsNapi | undefined | null): Array<MarkdownChunkNapi>
|
|
110
|
-
|
|
111
|
-
export interface SplitterOptionsNapi {
|
|
112
|
-
headersToSplitOn?: Array<number>
|
|
113
|
-
returnEachLine?: boolean
|
|
114
|
-
stripHeaders?: boolean
|
|
115
|
-
chunkSize?: number
|
|
116
|
-
chunkOverlap?: number
|
|
117
|
-
}
|
|
118
|
-
|
|
119
86
|
export interface TagOverrideNapi {
|
|
120
87
|
enter?: string
|
|
121
88
|
exit?: string
|
package/napi/index.mjs
CHANGED
|
@@ -716,10 +716,7 @@ if (!nativeBinding) {
|
|
|
716
716
|
throw new Error(`Failed to load native binding`)
|
|
717
717
|
}
|
|
718
718
|
|
|
719
|
-
const { MarkdownStream, htmlToMarkdown
|
|
719
|
+
const { MarkdownStream, htmlToMarkdown } = nativeBinding
|
|
720
720
|
export { MarkdownStream }
|
|
721
721
|
export { htmlToMarkdown }
|
|
722
|
-
export { htmlToMarkdownBytes }
|
|
723
|
-
export { htmlToMarkdownChunks }
|
|
724
|
-
export { splitMarkdown }
|
|
725
722
|
export const __napiBindingTarget = __napiLoadedBindingTarget
|
package/package.json
CHANGED
|
@@ -1,19 +1,32 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mdream",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "2.0.0
|
|
4
|
+
"version": "2.0.0",
|
|
5
5
|
"description": "Ultra-performant HTML to Markdown Convertor Optimized for LLMs and llm.txt artifacts.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
8
8
|
"email": "harlan@harlanzw.com",
|
|
9
9
|
"url": "https://harlanzw.com/"
|
|
10
10
|
},
|
|
11
|
+
"contributors": [
|
|
12
|
+
{
|
|
13
|
+
"name": "André Cruz",
|
|
14
|
+
"url": "https://github.com/edevil"
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"name": "Bernardo Pereira",
|
|
18
|
+
"url": "https://github.com/BernardoPe"
|
|
19
|
+
}
|
|
20
|
+
],
|
|
11
21
|
"license": "MIT",
|
|
12
22
|
"repository": {
|
|
13
23
|
"type": "git",
|
|
14
24
|
"url": "https://github.com/harlan-zw/mdream",
|
|
15
25
|
"directory": "packages/mdream"
|
|
16
26
|
},
|
|
27
|
+
"sideEffects": [
|
|
28
|
+
"./wasm-bundler/mdream_edge.js"
|
|
29
|
+
],
|
|
17
30
|
"exports": {
|
|
18
31
|
".": {
|
|
19
32
|
"workerd": {
|
|
@@ -24,24 +37,17 @@
|
|
|
24
37
|
"types": "./dist/edge.d.mts",
|
|
25
38
|
"default": "./dist/edge.mjs"
|
|
26
39
|
},
|
|
27
|
-
"browser": {
|
|
28
|
-
"types": "./dist/browser.d.mts",
|
|
29
|
-
"default": "./dist/browser.mjs"
|
|
30
|
-
},
|
|
31
40
|
"types": "./dist/index.d.mts",
|
|
41
|
+
"browser": "./dist/browser-stub.mjs",
|
|
32
42
|
"import": {
|
|
33
43
|
"types": "./dist/index.d.mts",
|
|
34
44
|
"default": "./dist/index.mjs"
|
|
35
45
|
},
|
|
36
46
|
"default": "./dist/index.mjs"
|
|
37
47
|
},
|
|
38
|
-
"./
|
|
39
|
-
"types": "./dist/
|
|
40
|
-
"
|
|
41
|
-
"types": "./dist/worker.d.mts",
|
|
42
|
-
"default": "./dist/worker.mjs"
|
|
43
|
-
},
|
|
44
|
-
"default": "./dist/worker.mjs"
|
|
48
|
+
"./browser": {
|
|
49
|
+
"types": "./dist/browser.d.mts",
|
|
50
|
+
"default": "./dist/browser.mjs"
|
|
45
51
|
},
|
|
46
52
|
"./wasm": {
|
|
47
53
|
"types": "./dist/wasm.d.mts",
|
|
@@ -64,33 +70,28 @@
|
|
|
64
70
|
"wasm-bundler/",
|
|
65
71
|
"wasm/"
|
|
66
72
|
],
|
|
67
|
-
"browser": "./dist/browser.mjs",
|
|
68
73
|
"dependencies": {},
|
|
69
74
|
"optionalDependencies": {
|
|
70
|
-
"@mdream/rust-android-arm-eabi": "2.0.0
|
|
71
|
-
"@mdream/rust-android-arm64": "2.0.0
|
|
72
|
-
"@mdream/rust-darwin-arm64": "2.0.0
|
|
73
|
-
"@mdream/rust-darwin-x64": "2.0.0
|
|
74
|
-
"@mdream/rust-freebsd-x64": "2.0.0
|
|
75
|
-
"@mdream/rust-linux-arm-gnueabihf": "2.0.0
|
|
76
|
-
"@mdream/rust-linux-arm64-gnu": "2.0.0
|
|
77
|
-
"@mdream/rust-linux-arm64-musl": "2.0.0
|
|
78
|
-
"@mdream/rust-linux-x64-gnu": "2.0.0
|
|
79
|
-
"@mdream/rust-linux-x64-musl": "2.0.0
|
|
80
|
-
"@mdream/rust-wasm32-wasi": "2.0.0
|
|
81
|
-
"@mdream/rust-win32-arm64-msvc": "2.0.0
|
|
82
|
-
"@mdream/rust-win32-x64-msvc": "2.0.0
|
|
75
|
+
"@mdream/rust-android-arm-eabi": "2.0.0",
|
|
76
|
+
"@mdream/rust-android-arm64": "2.0.0",
|
|
77
|
+
"@mdream/rust-darwin-arm64": "2.0.0",
|
|
78
|
+
"@mdream/rust-darwin-x64": "2.0.0",
|
|
79
|
+
"@mdream/rust-freebsd-x64": "2.0.0",
|
|
80
|
+
"@mdream/rust-linux-arm-gnueabihf": "2.0.0",
|
|
81
|
+
"@mdream/rust-linux-arm64-gnu": "2.0.0",
|
|
82
|
+
"@mdream/rust-linux-arm64-musl": "2.0.0",
|
|
83
|
+
"@mdream/rust-linux-x64-gnu": "2.0.0",
|
|
84
|
+
"@mdream/rust-linux-x64-musl": "2.0.0",
|
|
85
|
+
"@mdream/rust-wasm32-wasi": "2.0.0",
|
|
86
|
+
"@mdream/rust-win32-arm64-msvc": "2.0.0",
|
|
87
|
+
"@mdream/rust-win32-x64-msvc": "2.0.0"
|
|
83
88
|
},
|
|
84
89
|
"devDependencies": {
|
|
85
|
-
"@mdream/js": "2.0.0
|
|
86
|
-
"@mdream/rust-build": "0.17.0"
|
|
90
|
+
"@mdream/js": "2.0.0",
|
|
91
|
+
"@mdream/rust-build": "0.17.0",
|
|
92
|
+
"rolldown": "^1.2.12"
|
|
87
93
|
},
|
|
88
94
|
"scripts": {
|
|
89
|
-
"flame": "pnpm build && unbuild bench/bundle && clinic flame -- node bench/bundle/dist/string.mjs 10",
|
|
90
|
-
"bench:build": "pnpm build && unbuild bench/bundle",
|
|
91
|
-
"bench:stream": "pnpm build && unbuild bench/bundle && hyperfine --runs 100 'node bench/bundle/dist/stream.mjs' --warmup 3",
|
|
92
|
-
"bench:await": "pnpm build && unbuild bench/bundle && hyperfine --runs 100 'node bench/bundle/dist/await.mjs' --warmup 3",
|
|
93
|
-
"bench:string": "pnpm build && unbuild bench/bundle && hyperfine --runs 100 'node bench/bundle/dist/string.mjs' --warmup 3",
|
|
94
95
|
"ts": "node --experimental-strip-types",
|
|
95
96
|
"test:live": "curl -s https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax | node ./bin/mdream.mjs --origin https://docs.github.com | tee test/github-markdown.md",
|
|
96
97
|
"test:images": "cat test/fixtures/test-origin.html | node ./bin/mdream.mjs --origin https://docs.github.com",
|
|
@@ -104,7 +105,6 @@
|
|
|
104
105
|
"typecheck": "tsc --noEmit",
|
|
105
106
|
"dev:prepare": "obuild --stub",
|
|
106
107
|
"test": "vitest test",
|
|
107
|
-
"test:browser": "vitest test --project=browser",
|
|
108
108
|
"test:attw": "attw --pack"
|
|
109
109
|
}
|
|
110
110
|
}
|
package/skills/mdream/SKILL.md
CHANGED
|
@@ -5,7 +5,9 @@ description: Convert HTML to Markdown, plain text, or safe HTML with mdream, the
|
|
|
5
5
|
|
|
6
6
|
# mdream
|
|
7
7
|
|
|
8
|
-
This Skill covers
|
|
8
|
+
This Skill covers v2.
|
|
9
|
+
For an earlier release, read the Skill at its release tag.
|
|
10
|
+
The package wraps a Rust converter: NAPI in Node, WASM on edge runtimes and in the browser.
|
|
9
11
|
`@mdream/js` is a separate pure JS engine. It has hook plugins, the splitter, `llms.txt` generation, and content negotiation. Its option shape differs; see [Traps](#traps).
|
|
10
12
|
|
|
11
13
|
## Setup
|
|
@@ -58,7 +60,8 @@ const markdown = htmlToMarkdown('<html><head><title>T</title></head><body><a hre
|
|
|
58
60
|
})
|
|
59
61
|
```
|
|
60
62
|
|
|
61
|
-
Stream a large response
|
|
63
|
+
Stream a large response with the same options.
|
|
64
|
+
Callbacks run once, after the last chunk is read:
|
|
62
65
|
|
|
63
66
|
```ts
|
|
64
67
|
import { streamHtmlToMarkdown } from 'mdream'
|
|
@@ -95,11 +98,12 @@ CLI: `curl -s URL | mdream --origin URL --preset minimal`. It streams stdin to s
|
|
|
95
98
|
|
|
96
99
|
## Traps
|
|
97
100
|
|
|
98
|
-
- **
|
|
99
|
-
-
|
|
101
|
+
- **Unknown options throw.** A key outside the [Config](#config) list throws a `TypeError` that names the fix. Pass `{ minimal: true }`, not `preset`. Pass `clean: { urls: true }`, not `cleanUrls`. Pass plugin options at the top level, not in a `plugins` object. A key set to `undefined` does not throw. The CLI exits with code 1 on an unknown flag.
|
|
102
|
+
- **Hook plugins are not in this package.** An array in `plugins` throws `Custom hook plugins require @mdream/js`. Use `@mdream/js` with `plugins: [createPlugin({...})]` from `@mdream/js/plugins`.
|
|
103
|
+
- **`@mdream/js` takes plugins as an array.** It throws a `TypeError` on `minimal`, `format`, `frontmatter`, `isolateMain`, `tailwind`, `filter`, or `extraction` at the top level. Use `withMinimalPreset()` from `@mdream/js/preset/minimal`, or `{ plugins: [frontmatterPlugin()] }` from `@mdream/js/plugins`. For text or HTML output, use `@mdream/js/text` or `@mdream/js/html`.
|
|
100
104
|
- **Emphasis is `*`, not `_`.** Headings are ATX, bullets are `-`, rules are `---`. Only `tagOverrides` changes a delimiter, for example `em: { enter: '_', exit: '_', isInline: true }`.
|
|
101
105
|
- **Filter selectors also drop inline `position: absolute` and `position: fixed` elements.**
|
|
102
|
-
- **Browser
|
|
106
|
+
- **Browser bundles import `mdream/browser`.** In a browser bundle, the root `mdream` entry throws a `TypeError` when called. Every entry takes the same options and produces the same string. `mdream/browser` and the CDN script return `Promise<string>`, so `await` them. Read [references/runtimes.md](references/runtimes.md) before you use `mdream` outside Node or a Cloudflare Worker.
|
|
103
107
|
|
|
104
108
|
## Version limits
|
|
105
109
|
|
|
@@ -1,17 +1,20 @@
|
|
|
1
1
|
# mdream outside Node
|
|
2
2
|
|
|
3
|
-
`mdream` picks an entry by export condition.
|
|
3
|
+
`mdream` picks an entry by export condition.
|
|
4
|
+
Its conversion entries take the same `MdreamOptions`, including `minimal`, `frontmatter`, `filter`, `extraction`, and `tagOverrides`.
|
|
5
|
+
Callbacks work in every supported runtime.
|
|
6
|
+
Some entries return a string; others return `Promise<string>`.
|
|
4
7
|
|
|
5
8
|
| Import | Resolved by | Returns |
|
|
6
9
|
|---|---|---|
|
|
7
10
|
| `mdream` | Node | `string` |
|
|
8
11
|
| `mdream` | `workerd`, `edge-light` | `string` |
|
|
9
|
-
| `mdream` | `browser` | `
|
|
10
|
-
| `mdream/
|
|
12
|
+
| `mdream` | `browser` | throws a `TypeError` when called |
|
|
13
|
+
| `mdream/browser` | direct import | `Promise<string>` |
|
|
11
14
|
| `mdream/wasm` | direct import | `string`, after `init()` |
|
|
12
15
|
| `dist/iife.js` (CDN) | script tag | `Promise<string>` |
|
|
13
16
|
|
|
14
|
-
|
|
17
|
+
The `mdream` types always describe the synchronous API. The `mdream/browser` types return `Promise<string>` and need no `customConditions` setting.
|
|
15
18
|
|
|
16
19
|
## Cloudflare Workers and Vercel Edge
|
|
17
20
|
|
|
@@ -32,18 +35,23 @@ A Rust panic becomes a normal `Error` with the message `mdream WASM panic, pleas
|
|
|
32
35
|
|
|
33
36
|
## Browser bundles
|
|
34
37
|
|
|
35
|
-
In a Vite, webpack, or esbuild client build,
|
|
38
|
+
In a Vite, webpack, or esbuild client build, import from `mdream/browser`.
|
|
39
|
+
This entry fetches the WASM binary on first use and returns `Promise<string>`.
|
|
40
|
+
Await the result.
|
|
41
|
+
Options and callbacks work as in Node.
|
|
36
42
|
|
|
37
43
|
```ts
|
|
38
|
-
import { htmlToMarkdown } from 'mdream'
|
|
44
|
+
import { htmlToMarkdown } from 'mdream/browser'
|
|
39
45
|
|
|
40
46
|
const markdown = await htmlToMarkdown(html, { minimal: true })
|
|
41
47
|
```
|
|
42
48
|
|
|
43
|
-
|
|
49
|
+
In a browser bundle, the root `mdream` entry throws a `TypeError` when called. The import itself does not throw. A module that runs on the server and the client can import `mdream` and call it only on the server.
|
|
50
|
+
|
|
51
|
+
For browser code that wants a plain string with no WASM, `@mdream/js` is synchronous. It takes plugins as an array of imports from `@mdream/js/plugins`.
|
|
44
52
|
|
|
45
53
|
## Web Worker, raw WASM, and CDN
|
|
46
54
|
|
|
47
|
-
- `mdream/
|
|
55
|
+
- Web Worker: v2 has no worker entry. Write a module worker that imports `mdream/browser`, and start it with `new Worker(new URL('./md.worker.ts', import.meta.url), { type: 'module' })`. Functions cannot cross `postMessage`, so set callbacks inside the worker.
|
|
48
56
|
- `mdream/wasm`: the wasm-bindgen build. Await the default export `init({ module_or_path })` before the first `htmlToMarkdown(html, options)` call. Its `MarkdownStream` runs the callbacks in `finish()`.
|
|
49
|
-
- `https://unpkg.com/mdream/dist/iife.js`: the WASM is inlined and ready at load. `window.mdream` has only `htmlToMarkdown`. There is no `init()`. The call returns `Promise<string>`, the same as
|
|
57
|
+
- `https://unpkg.com/mdream/dist/iife.js`: the WASM is inlined and ready at load. `window.mdream` has only `htmlToMarkdown`. There is no `init()`. The call returns `Promise<string>`, the same as `mdream/browser`.
|