mdream 1.7.2 → 2.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -51
- package/dist/browser.d.mts +100 -5
- package/dist/browser.mjs +213 -33
- package/dist/edge.d.mts +95 -4
- package/dist/edge.mjs +114 -43
- package/dist/iife.js +232 -23
- package/dist/index.d.mts +8 -4
- package/dist/index.mjs +68 -17
- package/dist/wasm.d.mts +100 -0
- package/dist/wasm.mjs +140 -0
- package/dist/worker.d.mts +8 -4
- package/dist/worker.mjs +121 -4
- package/napi/index.d.mts +20 -0
- package/napi/index.d.ts +20 -0
- package/napi/index.mjs +47 -32
- package/package.json +32 -22
- package/skills/mdream/SKILL.md +124 -0
- package/skills/mdream/references/runtimes.md +49 -0
- package/wasm/mdream_edge.d.ts +27 -0
- package/wasm/mdream_edge.js +91 -19
- package/wasm/mdream_edge_bg.wasm +0 -0
- package/wasm/mdream_edge_bg.wasm.d.ts +3 -0
- package/wasm/package.json +1 -1
- package/wasm-bundler/mdream_edge.d.ts +24 -0
- package/wasm-bundler/mdream_edge.js +6 -2
- package/wasm-bundler/mdream_edge_bg.js +91 -19
- package/wasm-bundler/mdream_edge_bg.wasm +0 -0
- package/wasm-bundler/mdream_edge_bg.wasm.d.ts +3 -0
- package/wasm-bundler/package.json +1 -1
package/dist/index.d.mts
CHANGED
|
@@ -42,8 +42,8 @@ export interface MdreamOptions {
|
|
|
42
42
|
origin?: string;
|
|
43
43
|
/**
|
|
44
44
|
* Clean up the markdown output. Pass `true` for all cleanup or an object
|
|
45
|
-
* to enable specific features. `
|
|
46
|
-
*
|
|
45
|
+
* to enable specific features. Both `htmlToMarkdown` and
|
|
46
|
+
* `streamHtmlToMarkdown` apply it.
|
|
47
47
|
*/
|
|
48
48
|
clean?: boolean | CleanOptions;
|
|
49
49
|
/** Enable minimal preset (frontmatter, isolateMain, tailwind, filter). Default: false */
|
|
@@ -59,8 +59,12 @@ export interface MdreamOptions {
|
|
|
59
59
|
isolateMain?: boolean;
|
|
60
60
|
/** Convert Tailwind utility classes. Default when minimal: true */
|
|
61
61
|
tailwind?: boolean;
|
|
62
|
-
/**
|
|
63
|
-
|
|
62
|
+
/**
|
|
63
|
+
* Filter elements by CSS selector. `minimal` excludes form, nav, footer, and
|
|
64
|
+
* similar; a filter passed with `minimal` adds to those excludes. `false`
|
|
65
|
+
* turns filtering off, including the `minimal` one.
|
|
66
|
+
*/
|
|
67
|
+
filter?: false | {
|
|
64
68
|
include?: string[];
|
|
65
69
|
exclude?: string[];
|
|
66
70
|
processChildren?: boolean;
|
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { MarkdownStream, htmlToMarkdown as htmlToMarkdown$1 } from "../napi/index.mjs";
|
|
2
|
-
const
|
|
2
|
+
const MINIMAL_FILTER_EXCLUDE = [
|
|
3
3
|
"form",
|
|
4
4
|
"fieldset",
|
|
5
5
|
"object",
|
|
@@ -12,7 +12,7 @@ const MINIMAL_FILTER_DEFAULT = { exclude: [
|
|
|
12
12
|
"select",
|
|
13
13
|
"button",
|
|
14
14
|
"nav"
|
|
15
|
-
]
|
|
15
|
+
];
|
|
16
16
|
const CLEAN_ALL = {
|
|
17
17
|
urls: true,
|
|
18
18
|
fragments: true,
|
|
@@ -67,7 +67,11 @@ function resolveOptions(options) {
|
|
|
67
67
|
const tailwindDisabled = options.tailwind === false;
|
|
68
68
|
if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
|
|
69
69
|
else if (tailwindDisabled) delete plugins.tailwind;
|
|
70
|
-
if (
|
|
70
|
+
if (options.filter === false) delete plugins.filter;
|
|
71
|
+
else if (minimal) plugins.filter = {
|
|
72
|
+
...options.filter,
|
|
73
|
+
exclude: [...MINIMAL_FILTER_EXCLUDE, ...options.filter?.exclude ?? []]
|
|
74
|
+
};
|
|
71
75
|
else if (options.filter) plugins.filter = options.filter;
|
|
72
76
|
const extraction = options.extraction;
|
|
73
77
|
let extractionHandlers;
|
|
@@ -98,36 +102,83 @@ function resolveOptions(options) {
|
|
|
98
102
|
frontmatterCallback
|
|
99
103
|
};
|
|
100
104
|
}
|
|
101
|
-
function
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
if (napiResult.frontmatter && frontmatterCallback) frontmatterCallback(napiResult.frontmatter);
|
|
105
|
-
if (napiResult.extracted?.length && extractionHandlers) for (const el of napiResult.extracted) extractionHandlers[el.selector]?.(el);
|
|
106
|
-
return napiResult.markdown;
|
|
105
|
+
function deliverPluginData(data, { frontmatterCallback, extractionHandlers }) {
|
|
106
|
+
if (data.frontmatter && frontmatterCallback) frontmatterCallback(data.frontmatter);
|
|
107
|
+
if (data.extracted?.length && extractionHandlers) for (const element of data.extracted) extractionHandlers[element.selector]?.(element);
|
|
107
108
|
}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
109
|
+
function hasCallbacks({ frontmatterCallback, extractionHandlers }) {
|
|
110
|
+
return !!frontmatterCallback || !!extractionHandlers;
|
|
111
|
+
}
|
|
112
|
+
function convertResult(convert, html, options = {}) {
|
|
113
|
+
const resolved = resolveOptions(options);
|
|
114
|
+
const result = convert(html, resolved.napiOpts);
|
|
115
|
+
deliverPluginData(result, resolved);
|
|
116
|
+
return result;
|
|
117
|
+
}
|
|
118
|
+
async function* pumpStream(stream, htmlStream, callbacks, { decodeInJs = false, mapError = (error) => error } = {}) {
|
|
112
119
|
const reader = htmlStream.getReader();
|
|
113
|
-
const decoder = new TextDecoder();
|
|
120
|
+
const decoder = decodeInJs ? new TextDecoder() : void 0;
|
|
114
121
|
try {
|
|
115
122
|
while (true) {
|
|
116
123
|
const { done, value } = await reader.read();
|
|
117
124
|
if (done) break;
|
|
118
|
-
|
|
119
|
-
|
|
125
|
+
let processed;
|
|
126
|
+
if (typeof value === "string") processed = stream.processChunk(decoder ? decoder.decode() + value : value);
|
|
127
|
+
else processed = decoder ? stream.processChunk(decoder.decode(value, { stream: true })) : stream.processChunkBytes(value);
|
|
120
128
|
if (processed) yield processed;
|
|
121
129
|
}
|
|
122
|
-
const decoderTail = decoder
|
|
130
|
+
const decoderTail = decoder?.decode();
|
|
123
131
|
if (decoderTail) {
|
|
124
132
|
const processed = stream.processChunk(decoderTail);
|
|
125
133
|
if (processed) yield processed;
|
|
126
134
|
}
|
|
127
135
|
const final_ = stream.finish();
|
|
136
|
+
if (hasCallbacks(callbacks)) deliverPluginData(stream.takeData(), callbacks);
|
|
128
137
|
if (final_) yield final_;
|
|
138
|
+
} catch (error) {
|
|
139
|
+
throw mapError(error);
|
|
129
140
|
} finally {
|
|
130
141
|
reader.releaseLock();
|
|
131
142
|
}
|
|
132
143
|
}
|
|
144
|
+
function createSurrogateCarry() {
|
|
145
|
+
let held = "";
|
|
146
|
+
return {
|
|
147
|
+
take(chunk) {
|
|
148
|
+
if (held) {
|
|
149
|
+
chunk = held + chunk;
|
|
150
|
+
held = "";
|
|
151
|
+
}
|
|
152
|
+
const last = chunk.charCodeAt(chunk.length - 1);
|
|
153
|
+
if (last >= 55296 && last <= 56319) {
|
|
154
|
+
held = chunk.slice(-1);
|
|
155
|
+
return chunk.slice(0, -1);
|
|
156
|
+
}
|
|
157
|
+
return chunk;
|
|
158
|
+
},
|
|
159
|
+
flush() {
|
|
160
|
+
const value = held;
|
|
161
|
+
held = "";
|
|
162
|
+
return value;
|
|
163
|
+
}
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
function htmlToMarkdown(html, options = {}) {
|
|
167
|
+
return convertResult(htmlToMarkdown$1, html, options).markdown;
|
|
168
|
+
}
|
|
169
|
+
async function* streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
170
|
+
if (!htmlStream) throw new Error("Invalid HTML stream provided");
|
|
171
|
+
const resolved = resolveOptions(options);
|
|
172
|
+
const carry = createSurrogateCarry();
|
|
173
|
+
const stream = new MarkdownStream(resolved.napiOpts);
|
|
174
|
+
yield* pumpStream({
|
|
175
|
+
processChunk: (chunk) => stream.processChunk(carry.take(chunk)),
|
|
176
|
+
processChunkBytes: (chunk) => stream.processChunkBytes(chunk),
|
|
177
|
+
finish: () => {
|
|
178
|
+
const held = carry.flush();
|
|
179
|
+
return held ? stream.processChunk(held) + stream.finish() : stream.finish();
|
|
180
|
+
},
|
|
181
|
+
takeData: () => stream.takeData()
|
|
182
|
+
}, htmlStream, resolved, { decodeInJs: true });
|
|
183
|
+
}
|
|
133
184
|
export { htmlToMarkdown, streamHtmlToMarkdown };
|
package/dist/wasm.d.mts
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import mdream_edge_default, { MarkdownStream as MarkdownStream$1 } from "../wasm/mdream_edge.js";
|
|
2
|
+
import { MdreamNapiResult } from "../napi/index.js";
|
|
3
|
+
export * from "../wasm/mdream_edge.js";
|
|
4
|
+
interface CleanOptions {
|
|
5
|
+
/** Strip tracking query parameters (utm_*, fbclid, gclid, etc.) from URLs */
|
|
6
|
+
urls?: boolean;
|
|
7
|
+
/** Strip fragment-only links that don't match any heading in the output */
|
|
8
|
+
fragments?: boolean;
|
|
9
|
+
/** Strip links with meaningless hrefs (#, javascript:void(0)) → plain text */
|
|
10
|
+
emptyLinks?: boolean;
|
|
11
|
+
/** Collapse 3+ consecutive blank lines to 2 */
|
|
12
|
+
blankLines?: boolean;
|
|
13
|
+
/** Strip links where text equals URL: [https://x.com](https://x.com) → https://x.com */
|
|
14
|
+
redundantLinks?: boolean;
|
|
15
|
+
/** Strip self-referencing heading anchors: ## [Title](#title) → ## Title */
|
|
16
|
+
selfLinkHeadings?: boolean;
|
|
17
|
+
/** Strip images with no alt text (decorative/tracking pixels) */
|
|
18
|
+
emptyImages?: boolean;
|
|
19
|
+
/** Drop links that produce no visible text: [](url) → nothing */
|
|
20
|
+
emptyLinkText?: boolean;
|
|
21
|
+
}
|
|
22
|
+
interface ExtractedElement {
|
|
23
|
+
selector: string;
|
|
24
|
+
tagName: string;
|
|
25
|
+
textContent: string;
|
|
26
|
+
attributes: Record<string, string>;
|
|
27
|
+
}
|
|
28
|
+
interface FrontmatterConfig {
|
|
29
|
+
additionalFields?: Record<string, string>;
|
|
30
|
+
metaFields?: string[];
|
|
31
|
+
/** Callback to receive structured frontmatter data after conversion */
|
|
32
|
+
onExtract?: (frontmatter: Record<string, string>) => void;
|
|
33
|
+
}
|
|
34
|
+
interface TagOverride {
|
|
35
|
+
enter?: string;
|
|
36
|
+
exit?: string;
|
|
37
|
+
spacing?: number[];
|
|
38
|
+
isInline?: boolean;
|
|
39
|
+
isSelfClosing?: boolean;
|
|
40
|
+
collapsesInnerWhiteSpace?: boolean;
|
|
41
|
+
alias?: string;
|
|
42
|
+
}
|
|
43
|
+
interface MdreamOptions {
|
|
44
|
+
/** Origin URL for resolving relative image paths and internal links. */
|
|
45
|
+
origin?: string;
|
|
46
|
+
/**
|
|
47
|
+
* Clean up the markdown output. Pass `true` for all cleanup or an object
|
|
48
|
+
* to enable specific features. Both `htmlToMarkdown` and
|
|
49
|
+
* `streamHtmlToMarkdown` apply it.
|
|
50
|
+
*/
|
|
51
|
+
clean?: boolean | CleanOptions;
|
|
52
|
+
/** Enable minimal preset (frontmatter, isolateMain, tailwind, filter). Default: false */
|
|
53
|
+
minimal?: boolean;
|
|
54
|
+
/**
|
|
55
|
+
* Extract frontmatter from HTML head.
|
|
56
|
+
* - `true`: enable with defaults
|
|
57
|
+
* - `(fm) => void`: enable and receive structured data via callback
|
|
58
|
+
* - `FrontmatterConfig`: enable with config options and optional callback
|
|
59
|
+
*/
|
|
60
|
+
frontmatter?: boolean | ((frontmatter: Record<string, string>) => void) | FrontmatterConfig;
|
|
61
|
+
/** Isolate main content area. Default when minimal: true */
|
|
62
|
+
isolateMain?: boolean;
|
|
63
|
+
/** Convert Tailwind utility classes. Default when minimal: true */
|
|
64
|
+
tailwind?: boolean;
|
|
65
|
+
/**
|
|
66
|
+
* Filter elements by CSS selector. `minimal` excludes form, nav, footer, and
|
|
67
|
+
* similar; a filter passed with `minimal` adds to those excludes. `false`
|
|
68
|
+
* turns filtering off, including the `minimal` one.
|
|
69
|
+
*/
|
|
70
|
+
filter?: false | {
|
|
71
|
+
include?: string[];
|
|
72
|
+
exclude?: string[];
|
|
73
|
+
processChildren?: boolean;
|
|
74
|
+
};
|
|
75
|
+
/** Extract elements matching CSS selectors */
|
|
76
|
+
extraction?: Record<string, (element: ExtractedElement) => void>;
|
|
77
|
+
/** Tag overrides. String values act as aliases */
|
|
78
|
+
tagOverrides?: Record<string, TagOverride | string>;
|
|
79
|
+
/**
|
|
80
|
+
* Hard-wrap prose at this many characters, breaking on word boundaries.
|
|
81
|
+
* Applied inline during conversion (zero-cost when unset). Code blocks,
|
|
82
|
+
* tables, and headings are never wrapped. `0` disables wrapping.
|
|
83
|
+
*/
|
|
84
|
+
wrapWidth?: number;
|
|
85
|
+
/**
|
|
86
|
+
* Output format. Defaults to `markdown`; use `text` for readable plain text,
|
|
87
|
+
* or `html` for allowlisted semantic HTML.
|
|
88
|
+
*/
|
|
89
|
+
format?: 'markdown' | 'text' | 'html';
|
|
90
|
+
}
|
|
91
|
+
export declare function htmlToMarkdownResult(html: string, options?: Partial<MdreamOptions>): MdreamNapiResult;
|
|
92
|
+
export declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string;
|
|
93
|
+
export declare function htmlToMarkdownBytes(html: Uint8Array, options?: Partial<MdreamOptions>): string;
|
|
94
|
+
/** Streaming converter. Runs the frontmatter and extraction callbacks in `finish()`. */
|
|
95
|
+
export declare class MarkdownStream extends MarkdownStream$1 {
|
|
96
|
+
private _callbacks;
|
|
97
|
+
constructor(options?: Partial<MdreamOptions>);
|
|
98
|
+
finish(): string;
|
|
99
|
+
}
|
|
100
|
+
export { mdream_edge_default as default };
|
package/dist/wasm.mjs
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import mdream_edge_default, { MarkdownStream as MarkdownStream$1, htmlToMarkdownBytes as htmlToMarkdownBytes$1, htmlToMarkdownResult as htmlToMarkdownResult$1 } from "../wasm/mdream_edge.js";
|
|
2
|
+
export * from "../wasm/mdream_edge.js";
|
|
3
|
+
const MINIMAL_FILTER_EXCLUDE = [
|
|
4
|
+
"form",
|
|
5
|
+
"fieldset",
|
|
6
|
+
"object",
|
|
7
|
+
"embed",
|
|
8
|
+
"footer",
|
|
9
|
+
"aside",
|
|
10
|
+
"iframe",
|
|
11
|
+
"input",
|
|
12
|
+
"textarea",
|
|
13
|
+
"select",
|
|
14
|
+
"button",
|
|
15
|
+
"nav"
|
|
16
|
+
];
|
|
17
|
+
const CLEAN_ALL = {
|
|
18
|
+
urls: true,
|
|
19
|
+
fragments: true,
|
|
20
|
+
emptyLinks: true,
|
|
21
|
+
redundantLinks: true,
|
|
22
|
+
selfLinkHeadings: true,
|
|
23
|
+
emptyImages: true,
|
|
24
|
+
emptyLinkText: true
|
|
25
|
+
};
|
|
26
|
+
function resolveCleanConfig(options, minimal) {
|
|
27
|
+
const cleanUrls = options.cleanUrls === true;
|
|
28
|
+
let cleanOpt = options.clean;
|
|
29
|
+
if (cleanOpt === void 0) {
|
|
30
|
+
if (!minimal) return { cleanUrls };
|
|
31
|
+
cleanOpt = true;
|
|
32
|
+
}
|
|
33
|
+
if (!cleanOpt) return { cleanUrls };
|
|
34
|
+
const resolved = cleanOpt === true ? CLEAN_ALL : cleanOpt;
|
|
35
|
+
return {
|
|
36
|
+
cleanUrls: cleanUrls || resolved.urls === true,
|
|
37
|
+
clean: resolved
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
function resolveFrontmatter(opt) {
|
|
41
|
+
if (typeof opt === "function") return {
|
|
42
|
+
config: {},
|
|
43
|
+
callback: opt
|
|
44
|
+
};
|
|
45
|
+
if (opt && typeof opt === "object") {
|
|
46
|
+
const { onExtract, ...config } = opt;
|
|
47
|
+
return {
|
|
48
|
+
config,
|
|
49
|
+
callback: onExtract
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
return { config: {} };
|
|
53
|
+
}
|
|
54
|
+
function resolveOptions(options) {
|
|
55
|
+
if (Array.isArray(options.plugins)) throw new TypeError("Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://mdream.dev/v1-migration#custom-plugins");
|
|
56
|
+
const minimal = options.minimal === true;
|
|
57
|
+
const plugins = options.plugins ? { ...options.plugins } : {};
|
|
58
|
+
let frontmatterCallback;
|
|
59
|
+
const frontmatterDisabled = options.frontmatter === false;
|
|
60
|
+
if (minimal ? !frontmatterDisabled : !!options.frontmatter) {
|
|
61
|
+
const fm = resolveFrontmatter(options.frontmatter);
|
|
62
|
+
plugins.frontmatter = fm.config;
|
|
63
|
+
frontmatterCallback = fm.callback;
|
|
64
|
+
} else if (frontmatterDisabled) delete plugins.frontmatter;
|
|
65
|
+
const isolateMainDisabled = options.isolateMain === false;
|
|
66
|
+
if (minimal ? !isolateMainDisabled : options.isolateMain) plugins.isolateMain = true;
|
|
67
|
+
else if (isolateMainDisabled) delete plugins.isolateMain;
|
|
68
|
+
const tailwindDisabled = options.tailwind === false;
|
|
69
|
+
if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
|
|
70
|
+
else if (tailwindDisabled) delete plugins.tailwind;
|
|
71
|
+
if (options.filter === false) delete plugins.filter;
|
|
72
|
+
else if (minimal) plugins.filter = {
|
|
73
|
+
...options.filter,
|
|
74
|
+
exclude: [...MINIMAL_FILTER_EXCLUDE, ...options.filter?.exclude ?? []]
|
|
75
|
+
};
|
|
76
|
+
else if (options.filter) plugins.filter = options.filter;
|
|
77
|
+
const extraction = options.extraction;
|
|
78
|
+
let extractionHandlers;
|
|
79
|
+
if (extraction) {
|
|
80
|
+
plugins.extraction = { selectors: Object.keys(extraction) };
|
|
81
|
+
extractionHandlers = extraction;
|
|
82
|
+
}
|
|
83
|
+
const tagOverrides = options.tagOverrides;
|
|
84
|
+
if (tagOverrides) {
|
|
85
|
+
const overrides = {};
|
|
86
|
+
for (const tag in tagOverrides) {
|
|
87
|
+
const value = tagOverrides[tag];
|
|
88
|
+
if (value) overrides[tag] = typeof value === "string" ? { alias: value } : value;
|
|
89
|
+
}
|
|
90
|
+
plugins.tagOverrides = overrides;
|
|
91
|
+
}
|
|
92
|
+
const { cleanUrls, clean } = resolveCleanConfig(options, minimal);
|
|
93
|
+
return {
|
|
94
|
+
napiOpts: {
|
|
95
|
+
origin: options.origin,
|
|
96
|
+
cleanUrls,
|
|
97
|
+
clean,
|
|
98
|
+
plugins,
|
|
99
|
+
wrapWidth: options.wrapWidth,
|
|
100
|
+
format: options.format
|
|
101
|
+
},
|
|
102
|
+
extractionHandlers,
|
|
103
|
+
frontmatterCallback
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
function deliverPluginData(data, { frontmatterCallback, extractionHandlers }) {
|
|
107
|
+
if (data.frontmatter && frontmatterCallback) frontmatterCallback(data.frontmatter);
|
|
108
|
+
if (data.extracted?.length && extractionHandlers) for (const element of data.extracted) extractionHandlers[element.selector]?.(element);
|
|
109
|
+
}
|
|
110
|
+
function convertResult(convert, html, options = {}) {
|
|
111
|
+
const resolved = resolveOptions(options);
|
|
112
|
+
const result = convert(html, resolved.napiOpts);
|
|
113
|
+
deliverPluginData(result, resolved);
|
|
114
|
+
return result;
|
|
115
|
+
}
|
|
116
|
+
function htmlToMarkdownResult(html, options = {}) {
|
|
117
|
+
return convertResult(htmlToMarkdownResult$1, html, options);
|
|
118
|
+
}
|
|
119
|
+
function htmlToMarkdown(html, options = {}) {
|
|
120
|
+
return htmlToMarkdownResult(html, options).markdown || "";
|
|
121
|
+
}
|
|
122
|
+
function htmlToMarkdownBytes(html, options = {}) {
|
|
123
|
+
const resolved = resolveOptions(options);
|
|
124
|
+
if (resolved.frontmatterCallback || resolved.extractionHandlers) return htmlToMarkdown(new TextDecoder().decode(html), options);
|
|
125
|
+
return htmlToMarkdownBytes$1(html, resolved.napiOpts);
|
|
126
|
+
}
|
|
127
|
+
var MarkdownStream = class extends MarkdownStream$1 {
|
|
128
|
+
_callbacks;
|
|
129
|
+
constructor(options = {}) {
|
|
130
|
+
const resolved = resolveOptions(options);
|
|
131
|
+
super(resolved.napiOpts);
|
|
132
|
+
this._callbacks = resolved;
|
|
133
|
+
}
|
|
134
|
+
finish() {
|
|
135
|
+
const markdown = super.finish();
|
|
136
|
+
deliverPluginData(this.takeData(), this._callbacks);
|
|
137
|
+
return markdown;
|
|
138
|
+
}
|
|
139
|
+
};
|
|
140
|
+
export { MarkdownStream, mdream_edge_default as default, htmlToMarkdown, htmlToMarkdownBytes, htmlToMarkdownResult };
|
package/dist/worker.d.mts
CHANGED
|
@@ -42,8 +42,8 @@ interface MdreamOptions {
|
|
|
42
42
|
origin?: string;
|
|
43
43
|
/**
|
|
44
44
|
* Clean up the markdown output. Pass `true` for all cleanup or an object
|
|
45
|
-
* to enable specific features. `
|
|
46
|
-
*
|
|
45
|
+
* to enable specific features. Both `htmlToMarkdown` and
|
|
46
|
+
* `streamHtmlToMarkdown` apply it.
|
|
47
47
|
*/
|
|
48
48
|
clean?: boolean | CleanOptions;
|
|
49
49
|
/** Enable minimal preset (frontmatter, isolateMain, tailwind, filter). Default: false */
|
|
@@ -59,8 +59,12 @@ interface MdreamOptions {
|
|
|
59
59
|
isolateMain?: boolean;
|
|
60
60
|
/** Convert Tailwind utility classes. Default when minimal: true */
|
|
61
61
|
tailwind?: boolean;
|
|
62
|
-
/**
|
|
63
|
-
|
|
62
|
+
/**
|
|
63
|
+
* Filter elements by CSS selector. `minimal` excludes form, nav, footer, and
|
|
64
|
+
* similar; a filter passed with `minimal` adds to those excludes. `false`
|
|
65
|
+
* turns filtering off, including the `minimal` one.
|
|
66
|
+
*/
|
|
67
|
+
filter?: false | {
|
|
64
68
|
include?: string[];
|
|
65
69
|
exclude?: string[];
|
|
66
70
|
processChildren?: boolean;
|
package/dist/worker.mjs
CHANGED
|
@@ -1,3 +1,110 @@
|
|
|
1
|
+
const MINIMAL_FILTER_EXCLUDE = [
|
|
2
|
+
"form",
|
|
3
|
+
"fieldset",
|
|
4
|
+
"object",
|
|
5
|
+
"embed",
|
|
6
|
+
"footer",
|
|
7
|
+
"aside",
|
|
8
|
+
"iframe",
|
|
9
|
+
"input",
|
|
10
|
+
"textarea",
|
|
11
|
+
"select",
|
|
12
|
+
"button",
|
|
13
|
+
"nav"
|
|
14
|
+
];
|
|
15
|
+
const CLEAN_ALL = {
|
|
16
|
+
urls: true,
|
|
17
|
+
fragments: true,
|
|
18
|
+
emptyLinks: true,
|
|
19
|
+
redundantLinks: true,
|
|
20
|
+
selfLinkHeadings: true,
|
|
21
|
+
emptyImages: true,
|
|
22
|
+
emptyLinkText: true
|
|
23
|
+
};
|
|
24
|
+
function resolveCleanConfig(options, minimal) {
|
|
25
|
+
const cleanUrls = options.cleanUrls === true;
|
|
26
|
+
let cleanOpt = options.clean;
|
|
27
|
+
if (cleanOpt === void 0) {
|
|
28
|
+
if (!minimal) return { cleanUrls };
|
|
29
|
+
cleanOpt = true;
|
|
30
|
+
}
|
|
31
|
+
if (!cleanOpt) return { cleanUrls };
|
|
32
|
+
const resolved = cleanOpt === true ? CLEAN_ALL : cleanOpt;
|
|
33
|
+
return {
|
|
34
|
+
cleanUrls: cleanUrls || resolved.urls === true,
|
|
35
|
+
clean: resolved
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
function resolveFrontmatter(opt) {
|
|
39
|
+
if (typeof opt === "function") return {
|
|
40
|
+
config: {},
|
|
41
|
+
callback: opt
|
|
42
|
+
};
|
|
43
|
+
if (opt && typeof opt === "object") {
|
|
44
|
+
const { onExtract, ...config } = opt;
|
|
45
|
+
return {
|
|
46
|
+
config,
|
|
47
|
+
callback: onExtract
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
return { config: {} };
|
|
51
|
+
}
|
|
52
|
+
function resolveOptions(options) {
|
|
53
|
+
if (Array.isArray(options.plugins)) throw new TypeError("Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://mdream.dev/v1-migration#custom-plugins");
|
|
54
|
+
const minimal = options.minimal === true;
|
|
55
|
+
const plugins = options.plugins ? { ...options.plugins } : {};
|
|
56
|
+
let frontmatterCallback;
|
|
57
|
+
const frontmatterDisabled = options.frontmatter === false;
|
|
58
|
+
if (minimal ? !frontmatterDisabled : !!options.frontmatter) {
|
|
59
|
+
const fm = resolveFrontmatter(options.frontmatter);
|
|
60
|
+
plugins.frontmatter = fm.config;
|
|
61
|
+
frontmatterCallback = fm.callback;
|
|
62
|
+
} else if (frontmatterDisabled) delete plugins.frontmatter;
|
|
63
|
+
const isolateMainDisabled = options.isolateMain === false;
|
|
64
|
+
if (minimal ? !isolateMainDisabled : options.isolateMain) plugins.isolateMain = true;
|
|
65
|
+
else if (isolateMainDisabled) delete plugins.isolateMain;
|
|
66
|
+
const tailwindDisabled = options.tailwind === false;
|
|
67
|
+
if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
|
|
68
|
+
else if (tailwindDisabled) delete plugins.tailwind;
|
|
69
|
+
if (options.filter === false) delete plugins.filter;
|
|
70
|
+
else if (minimal) plugins.filter = {
|
|
71
|
+
...options.filter,
|
|
72
|
+
exclude: [...MINIMAL_FILTER_EXCLUDE, ...options.filter?.exclude ?? []]
|
|
73
|
+
};
|
|
74
|
+
else if (options.filter) plugins.filter = options.filter;
|
|
75
|
+
const extraction = options.extraction;
|
|
76
|
+
let extractionHandlers;
|
|
77
|
+
if (extraction) {
|
|
78
|
+
plugins.extraction = { selectors: Object.keys(extraction) };
|
|
79
|
+
extractionHandlers = extraction;
|
|
80
|
+
}
|
|
81
|
+
const tagOverrides = options.tagOverrides;
|
|
82
|
+
if (tagOverrides) {
|
|
83
|
+
const overrides = {};
|
|
84
|
+
for (const tag in tagOverrides) {
|
|
85
|
+
const value = tagOverrides[tag];
|
|
86
|
+
if (value) overrides[tag] = typeof value === "string" ? { alias: value } : value;
|
|
87
|
+
}
|
|
88
|
+
plugins.tagOverrides = overrides;
|
|
89
|
+
}
|
|
90
|
+
const { cleanUrls, clean } = resolveCleanConfig(options, minimal);
|
|
91
|
+
return {
|
|
92
|
+
napiOpts: {
|
|
93
|
+
origin: options.origin,
|
|
94
|
+
cleanUrls,
|
|
95
|
+
clean,
|
|
96
|
+
plugins,
|
|
97
|
+
wrapWidth: options.wrapWidth,
|
|
98
|
+
format: options.format
|
|
99
|
+
},
|
|
100
|
+
extractionHandlers,
|
|
101
|
+
frontmatterCallback
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
function deliverPluginData(data, { frontmatterCallback, extractionHandlers }) {
|
|
105
|
+
if (data.frontmatter && frontmatterCallback) frontmatterCallback(data.frontmatter);
|
|
106
|
+
if (data.extracted?.length && extractionHandlers) for (const element of data.extracted) extractionHandlers[element.selector]?.(element);
|
|
107
|
+
}
|
|
1
108
|
let _worker = null;
|
|
2
109
|
let _ready = null;
|
|
3
110
|
let _idCounter = 0;
|
|
@@ -30,7 +137,7 @@ self.onmessage = function(e) {
|
|
|
30
137
|
}
|
|
31
138
|
try {
|
|
32
139
|
const result = htmlToMarkdownResult(msg.html, msg.options || {});
|
|
33
|
-
self.postMessage({ id: msg.id, type: 'result', data: result
|
|
140
|
+
self.postMessage({ id: msg.id, type: 'result', data: result });
|
|
34
141
|
} catch (err) {
|
|
35
142
|
// WASM aborts on panic (#195): the message is stashed by the panic hook
|
|
36
143
|
const panic = engine.__mdreamTakePanicMessage?.();
|
|
@@ -47,7 +154,12 @@ function onMessage(e) {
|
|
|
47
154
|
const pending = _pending.get(msg.id);
|
|
48
155
|
if (!pending) return;
|
|
49
156
|
_pending.delete(msg.id);
|
|
50
|
-
if (msg.type === "result")
|
|
157
|
+
if (msg.type === "result") try {
|
|
158
|
+
deliverPluginData(msg.data, pending.callbacks);
|
|
159
|
+
pending.resolve(msg.data.markdown || "");
|
|
160
|
+
} catch (error) {
|
|
161
|
+
pending.reject(error);
|
|
162
|
+
}
|
|
51
163
|
else pending.reject(new Error(msg.message));
|
|
52
164
|
}
|
|
53
165
|
}
|
|
@@ -73,17 +185,22 @@ function initWorker(wasmUrl) {
|
|
|
73
185
|
function htmlToMarkdown(html, options) {
|
|
74
186
|
if (!_worker || !_ready) return Promise.reject(/* @__PURE__ */ new Error("Call initWorker() before htmlToMarkdown()"));
|
|
75
187
|
return _ready.then(() => {
|
|
188
|
+
const { napiOpts, frontmatterCallback, extractionHandlers } = resolveOptions(options ?? {});
|
|
76
189
|
const id = _idCounter++;
|
|
77
190
|
return new Promise((resolve, reject) => {
|
|
78
191
|
_pending.set(id, {
|
|
79
192
|
resolve,
|
|
80
|
-
reject
|
|
193
|
+
reject,
|
|
194
|
+
callbacks: {
|
|
195
|
+
frontmatterCallback,
|
|
196
|
+
extractionHandlers
|
|
197
|
+
}
|
|
81
198
|
});
|
|
82
199
|
_worker.postMessage({
|
|
83
200
|
id,
|
|
84
201
|
type: "convert",
|
|
85
202
|
html,
|
|
86
|
-
options
|
|
203
|
+
options: napiOpts
|
|
87
204
|
});
|
|
88
205
|
});
|
|
89
206
|
});
|
package/napi/index.d.mts
CHANGED
|
@@ -1,10 +1,24 @@
|
|
|
1
1
|
/* auto-generated by NAPI-RS */
|
|
2
2
|
/* eslint-disable */
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Which binding artifact the generated loader actually loaded: `'native'` for
|
|
6
|
+
* a native addon, otherwise the `platformArchABI` of the WASI flavor. Every
|
|
7
|
+
* flavor napi-rs can build is listed, because `NAPI_RS_NATIVE_LIBRARY_PATH`
|
|
8
|
+
* can point the loader at a WASI artifact this package does not build itself.
|
|
9
|
+
*/
|
|
10
|
+
export declare const __napiBindingTarget: 'native' | 'wasm32-wasi' | 'wasm32-wasip1'
|
|
11
|
+
|
|
3
12
|
export declare class MarkdownStream {
|
|
4
13
|
constructor(options?: HtmlToMarkdownOptions | undefined | null)
|
|
5
14
|
processChunk(chunk: string): string
|
|
6
15
|
processChunkBytes(chunk: Uint8Array): string
|
|
7
16
|
finish(): string
|
|
17
|
+
/**
|
|
18
|
+
* Frontmatter and extracted elements collected so far. Call after
|
|
19
|
+
* `finish()` for the complete data. Extracted elements drain on each call.
|
|
20
|
+
*/
|
|
21
|
+
takeData(): MdreamStreamData
|
|
8
22
|
}
|
|
9
23
|
|
|
10
24
|
export interface ChunkLocNapi {
|
|
@@ -77,6 +91,12 @@ export interface MdreamNapiResult {
|
|
|
77
91
|
frontmatter?: Record<string, string>
|
|
78
92
|
}
|
|
79
93
|
|
|
94
|
+
/** Plugin data a stream collected: frontmatter and extracted elements. */
|
|
95
|
+
export interface MdreamStreamData {
|
|
96
|
+
extracted?: Array<ExtractedElementNapi>
|
|
97
|
+
frontmatter?: Record<string, string>
|
|
98
|
+
}
|
|
99
|
+
|
|
80
100
|
export interface PluginOptions {
|
|
81
101
|
filter?: FilterOptions
|
|
82
102
|
isolateMain?: boolean
|
package/napi/index.d.ts
CHANGED
|
@@ -1,10 +1,24 @@
|
|
|
1
1
|
/* auto-generated by NAPI-RS */
|
|
2
2
|
/* eslint-disable */
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Which binding artifact the generated loader actually loaded: `'native'` for
|
|
6
|
+
* a native addon, otherwise the `platformArchABI` of the WASI flavor. Every
|
|
7
|
+
* flavor napi-rs can build is listed, because `NAPI_RS_NATIVE_LIBRARY_PATH`
|
|
8
|
+
* can point the loader at a WASI artifact this package does not build itself.
|
|
9
|
+
*/
|
|
10
|
+
export declare const __napiBindingTarget: 'native' | 'wasm32-wasi' | 'wasm32-wasip1'
|
|
11
|
+
|
|
3
12
|
export declare class MarkdownStream {
|
|
4
13
|
constructor(options?: HtmlToMarkdownOptions | undefined | null)
|
|
5
14
|
processChunk(chunk: string): string
|
|
6
15
|
processChunkBytes(chunk: Uint8Array): string
|
|
7
16
|
finish(): string
|
|
17
|
+
/**
|
|
18
|
+
* Frontmatter and extracted elements collected so far. Call after
|
|
19
|
+
* `finish()` for the complete data. Extracted elements drain on each call.
|
|
20
|
+
*/
|
|
21
|
+
takeData(): MdreamStreamData
|
|
8
22
|
}
|
|
9
23
|
|
|
10
24
|
export interface ChunkLocNapi {
|
|
@@ -77,6 +91,12 @@ export interface MdreamNapiResult {
|
|
|
77
91
|
frontmatter?: Record<string, string>
|
|
78
92
|
}
|
|
79
93
|
|
|
94
|
+
/** Plugin data a stream collected: frontmatter and extracted elements. */
|
|
95
|
+
export interface MdreamStreamData {
|
|
96
|
+
extracted?: Array<ExtractedElementNapi>
|
|
97
|
+
frontmatter?: Record<string, string>
|
|
98
|
+
}
|
|
99
|
+
|
|
80
100
|
export interface PluginOptions {
|
|
81
101
|
filter?: FilterOptions
|
|
82
102
|
isolateMain?: boolean
|