mdream 1.7.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +357 -163
- package/bin/mdream.mjs +45 -24
- package/dist/browser-stub.mjs +7 -0
- package/dist/browser.d.mts +98 -5
- package/dist/browser.mjs +224 -33
- package/dist/edge.d.mts +93 -4
- package/dist/edge.mjs +151 -69
- package/dist/iife.js +1 -396
- package/dist/index.d.mts +8 -6
- package/dist/index.mjs +104 -43
- package/dist/{worker.d.mts → wasm.d.mts} +20 -20
- package/dist/wasm.mjs +150 -0
- package/napi/index.d.mts +20 -33
- package/napi/index.d.ts +20 -33
- package/napi/index.mjs +48 -36
- package/package.json +47 -37
- package/skills/mdream/SKILL.md +128 -0
- package/skills/mdream/references/runtimes.md +57 -0
- package/wasm/mdream_edge.d.ts +27 -0
- package/wasm/mdream_edge.js +94 -20
- package/wasm/mdream_edge_bg.wasm +0 -0
- package/wasm/mdream_edge_bg.wasm.d.ts +3 -0
- package/wasm/package.json +1 -1
- package/wasm-bundler/mdream_edge.d.ts +24 -0
- package/wasm-bundler/mdream_edge.js +6 -2
- package/wasm-bundler/mdream_edge_bg.js +94 -20
- package/wasm-bundler/mdream_edge_bg.wasm +0 -0
- package/wasm-bundler/mdream_edge_bg.wasm.d.ts +3 -0
- package/wasm-bundler/package.json +1 -1
- package/dist/worker.mjs +0 -100
package/dist/edge.mjs
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import { MarkdownStream as MarkdownStream$1, __mdreamTakePanicMessage, htmlToMarkdownResult
|
|
2
|
-
|
|
3
|
-
const MINIMAL_FILTER_DEFAULT = { exclude: [
|
|
1
|
+
import { MarkdownStream as MarkdownStream$1, __mdreamTakePanicMessage, htmlToMarkdownResult } from "../wasm-bundler/mdream_edge.js";
|
|
2
|
+
const MINIMAL_FILTER_EXCLUDE = [
|
|
4
3
|
"form",
|
|
5
4
|
"fieldset",
|
|
6
5
|
"object",
|
|
@@ -13,7 +12,7 @@ const MINIMAL_FILTER_DEFAULT = { exclude: [
|
|
|
13
12
|
"select",
|
|
14
13
|
"button",
|
|
15
14
|
"nav"
|
|
16
|
-
]
|
|
15
|
+
];
|
|
17
16
|
const CLEAN_ALL = {
|
|
18
17
|
urls: true,
|
|
19
18
|
fragments: true,
|
|
@@ -23,19 +22,38 @@ const CLEAN_ALL = {
|
|
|
23
22
|
emptyImages: true,
|
|
24
23
|
emptyLinkText: true
|
|
25
24
|
};
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
25
|
+
const VALID_OPTIONS = "origin, clean, minimal, frontmatter, isolateMain, tailwind, filter, extraction, tagOverrides, wrapWidth, format";
|
|
26
|
+
function unknownOptionMessage(key, value) {
|
|
27
|
+
switch (key) {
|
|
28
|
+
case "preset": return "mdream has no `preset` option. Pass { minimal: true }.";
|
|
29
|
+
case "cleanUrls": return "mdream has no `cleanUrls` option. Pass { clean: { urls: true } }.";
|
|
30
|
+
case "plugins": return Array.isArray(value) ? "Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://github.com/harlan-zw/mdream/tree/main/packages/js#migrating-from-v1" : "mdream takes plugin options at the top level, such as { frontmatter: true }.";
|
|
32
31
|
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
32
|
+
return `mdream has no \`${key}\` option. Valid options: ${VALID_OPTIONS}.`;
|
|
33
|
+
}
|
|
34
|
+
function assertKnownOptions(options) {
|
|
35
|
+
for (const key in options) {
|
|
36
|
+
switch (key) {
|
|
37
|
+
case "origin":
|
|
38
|
+
case "clean":
|
|
39
|
+
case "minimal":
|
|
40
|
+
case "frontmatter":
|
|
41
|
+
case "isolateMain":
|
|
42
|
+
case "tailwind":
|
|
43
|
+
case "filter":
|
|
44
|
+
case "extraction":
|
|
45
|
+
case "tagOverrides":
|
|
46
|
+
case "wrapWidth":
|
|
47
|
+
case "format": continue;
|
|
48
|
+
}
|
|
49
|
+
const value = options[key];
|
|
50
|
+
if (value !== void 0) throw new TypeError(unknownOptionMessage(key, value));
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
function resolveClean(clean, minimal) {
|
|
54
|
+
if (clean === void 0) return minimal ? CLEAN_ALL : void 0;
|
|
55
|
+
if (!clean) return void 0;
|
|
56
|
+
return clean === true ? CLEAN_ALL : clean;
|
|
39
57
|
}
|
|
40
58
|
function resolveFrontmatter(opt) {
|
|
41
59
|
if (typeof opt === "function") return {
|
|
@@ -52,23 +70,21 @@ function resolveFrontmatter(opt) {
|
|
|
52
70
|
return { config: {} };
|
|
53
71
|
}
|
|
54
72
|
function resolveOptions(options) {
|
|
55
|
-
|
|
73
|
+
assertKnownOptions(options);
|
|
56
74
|
const minimal = options.minimal === true;
|
|
57
|
-
const plugins =
|
|
75
|
+
const plugins = {};
|
|
58
76
|
let frontmatterCallback;
|
|
59
|
-
|
|
60
|
-
if (minimal ? !frontmatterDisabled : !!options.frontmatter) {
|
|
77
|
+
if (minimal ? options.frontmatter !== false : options.frontmatter) {
|
|
61
78
|
const fm = resolveFrontmatter(options.frontmatter);
|
|
62
79
|
plugins.frontmatter = fm.config;
|
|
63
80
|
frontmatterCallback = fm.callback;
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
if (minimal ?
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
if (minimal) plugins.filter = options.filter || MINIMAL_FILTER_DEFAULT;
|
|
81
|
+
}
|
|
82
|
+
if (minimal ? options.isolateMain !== false : options.isolateMain) plugins.isolateMain = true;
|
|
83
|
+
if (minimal ? options.tailwind !== false : options.tailwind) plugins.tailwind = true;
|
|
84
|
+
if (minimal && options.filter !== false) plugins.filter = {
|
|
85
|
+
...options.filter,
|
|
86
|
+
exclude: [...MINIMAL_FILTER_EXCLUDE, ...options.filter?.exclude ?? []]
|
|
87
|
+
};
|
|
72
88
|
else if (options.filter) plugins.filter = options.filter;
|
|
73
89
|
const extraction = options.extraction;
|
|
74
90
|
let extractionHandlers;
|
|
@@ -85,12 +101,10 @@ function resolveOptions(options) {
|
|
|
85
101
|
}
|
|
86
102
|
plugins.tagOverrides = overrides;
|
|
87
103
|
}
|
|
88
|
-
const { cleanUrls, clean } = resolveCleanConfig(options, minimal);
|
|
89
104
|
return {
|
|
90
105
|
napiOpts: {
|
|
91
106
|
origin: options.origin,
|
|
92
|
-
|
|
93
|
-
clean,
|
|
107
|
+
clean: resolveClean(options.clean, minimal),
|
|
94
108
|
plugins,
|
|
95
109
|
wrapWidth: options.wrapWidth,
|
|
96
110
|
format: options.format
|
|
@@ -99,7 +113,68 @@ function resolveOptions(options) {
|
|
|
99
113
|
frontmatterCallback
|
|
100
114
|
};
|
|
101
115
|
}
|
|
102
|
-
function
|
|
116
|
+
function deliverPluginData(data, { frontmatterCallback, extractionHandlers }) {
|
|
117
|
+
if (data.frontmatter && frontmatterCallback) frontmatterCallback(data.frontmatter);
|
|
118
|
+
if (data.extracted?.length && extractionHandlers) for (const element of data.extracted) extractionHandlers[element.selector]?.(element);
|
|
119
|
+
}
|
|
120
|
+
function hasCallbacks({ frontmatterCallback, extractionHandlers }) {
|
|
121
|
+
return !!frontmatterCallback || !!extractionHandlers;
|
|
122
|
+
}
|
|
123
|
+
function convertResult(convert, html, options = {}) {
|
|
124
|
+
const resolved = resolveOptions(options);
|
|
125
|
+
const result = convert(html, resolved.napiOpts);
|
|
126
|
+
deliverPluginData(result, resolved);
|
|
127
|
+
return result;
|
|
128
|
+
}
|
|
129
|
+
async function* pumpStream(stream, htmlStream, callbacks, { mapError = (error) => error } = {}) {
|
|
130
|
+
const reader = htmlStream.getReader();
|
|
131
|
+
const decoder = stream.processChunkBytes ? void 0 : new TextDecoder();
|
|
132
|
+
try {
|
|
133
|
+
while (true) {
|
|
134
|
+
const { done, value } = await reader.read();
|
|
135
|
+
if (done) break;
|
|
136
|
+
let processed;
|
|
137
|
+
if (typeof value === "string") processed = stream.processChunk(decoder ? decoder.decode() + value : value);
|
|
138
|
+
else processed = decoder ? stream.processChunk(decoder.decode(value, { stream: true })) : stream.processChunkBytes(value);
|
|
139
|
+
if (processed) yield processed;
|
|
140
|
+
}
|
|
141
|
+
const decoderTail = decoder?.decode();
|
|
142
|
+
if (decoderTail) {
|
|
143
|
+
const processed = stream.processChunk(decoderTail);
|
|
144
|
+
if (processed) yield processed;
|
|
145
|
+
}
|
|
146
|
+
const final_ = stream.finish();
|
|
147
|
+
if (hasCallbacks(callbacks)) deliverPluginData(stream.takeData(), callbacks);
|
|
148
|
+
if (final_) yield final_;
|
|
149
|
+
} catch (error) {
|
|
150
|
+
throw mapError(error);
|
|
151
|
+
} finally {
|
|
152
|
+
reader.releaseLock();
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
function createSurrogateCarry() {
|
|
156
|
+
let held = "";
|
|
157
|
+
return {
|
|
158
|
+
take(chunk) {
|
|
159
|
+
if (held) {
|
|
160
|
+
chunk = held + chunk;
|
|
161
|
+
held = "";
|
|
162
|
+
}
|
|
163
|
+
const last = chunk.charCodeAt(chunk.length - 1);
|
|
164
|
+
if (last >= 55296 && last <= 56319) {
|
|
165
|
+
held = chunk.slice(-1);
|
|
166
|
+
return chunk.slice(0, -1);
|
|
167
|
+
}
|
|
168
|
+
return chunk;
|
|
169
|
+
},
|
|
170
|
+
flush() {
|
|
171
|
+
const value = held;
|
|
172
|
+
held = "";
|
|
173
|
+
return value;
|
|
174
|
+
}
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
function wasmPanicError(error, takeMessage) {
|
|
103
178
|
if (!(error instanceof WebAssembly.RuntimeError)) return error;
|
|
104
179
|
let message;
|
|
105
180
|
try {
|
|
@@ -110,70 +185,77 @@ function wasmPanicError(error, takeMessage = __mdreamTakePanicMessage) {
|
|
|
110
185
|
if (!message) return error;
|
|
111
186
|
return new Error(`mdream WASM panic, please report this at https://github.com/harlan-zw/mdream/issues\n${message}`, { cause: error });
|
|
112
187
|
}
|
|
113
|
-
initSync({ module: wasmModule });
|
|
114
188
|
function convert(html, napiOpts) {
|
|
115
189
|
try {
|
|
116
190
|
return htmlToMarkdownResult(html, napiOpts);
|
|
117
191
|
} catch (error) {
|
|
118
|
-
throw wasmPanicError(error);
|
|
192
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
119
193
|
}
|
|
120
194
|
}
|
|
121
|
-
function htmlToMarkdown(html, options) {
|
|
122
|
-
|
|
123
|
-
const { napiOpts, extractionHandlers, frontmatterCallback } = resolveOptions(options);
|
|
124
|
-
const result = convert(html, napiOpts);
|
|
125
|
-
if (result.frontmatter && frontmatterCallback) frontmatterCallback(result.frontmatter);
|
|
126
|
-
if (result.extracted?.length && extractionHandlers) for (const element of result.extracted) extractionHandlers[element.selector]?.(element);
|
|
127
|
-
return result.markdown || "";
|
|
195
|
+
function htmlToMarkdown(html, options = {}) {
|
|
196
|
+
return convertResult(convert, html, options).markdown || "";
|
|
128
197
|
}
|
|
129
198
|
var MarkdownStream = class {
|
|
130
199
|
_inner;
|
|
131
|
-
|
|
200
|
+
_carry = createSurrogateCarry();
|
|
201
|
+
_callbacks;
|
|
202
|
+
constructor(options = {}) {
|
|
203
|
+
const resolved = resolveOptions(options);
|
|
204
|
+
this._callbacks = resolved;
|
|
132
205
|
try {
|
|
133
|
-
this._inner = new MarkdownStream$1(
|
|
206
|
+
this._inner = new MarkdownStream$1(resolved.napiOpts);
|
|
134
207
|
} catch (error) {
|
|
135
|
-
throw wasmPanicError(error);
|
|
208
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
136
209
|
}
|
|
137
210
|
}
|
|
138
211
|
processChunk(chunk) {
|
|
139
212
|
try {
|
|
140
|
-
return this._inner.processChunk(chunk);
|
|
213
|
+
return this._inner.processChunk(this._carry.take(chunk));
|
|
141
214
|
} catch (error) {
|
|
142
|
-
throw wasmPanicError(error);
|
|
215
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
processChunkBytes(chunk) {
|
|
219
|
+
try {
|
|
220
|
+
const held = this._carry.flush();
|
|
221
|
+
return held ? this._inner.processChunk(held) + this._inner.processChunkBytes(chunk) : this._inner.processChunkBytes(chunk);
|
|
222
|
+
} catch (error) {
|
|
223
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
143
224
|
}
|
|
144
225
|
}
|
|
145
226
|
finish() {
|
|
227
|
+
let markdown;
|
|
146
228
|
try {
|
|
147
|
-
|
|
229
|
+
const held = this._carry.flush();
|
|
230
|
+
markdown = held ? this._inner.processChunk(held) + this._inner.finish() : this._inner.finish();
|
|
148
231
|
} catch (error) {
|
|
149
|
-
throw wasmPanicError(error);
|
|
232
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
150
233
|
}
|
|
234
|
+
if (this._callbacks.frontmatterCallback || this._callbacks.extractionHandlers) deliverPluginData(this._inner.takeData(), this._callbacks);
|
|
235
|
+
return markdown;
|
|
151
236
|
}
|
|
152
237
|
};
|
|
153
|
-
async function* streamHtmlToMarkdown(htmlStream, options) {
|
|
238
|
+
async function* streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
154
239
|
if (!htmlStream) throw new Error("Invalid HTML stream provided");
|
|
155
|
-
const
|
|
156
|
-
const
|
|
157
|
-
|
|
240
|
+
const resolved = resolveOptions(options);
|
|
241
|
+
const carry = createSurrogateCarry();
|
|
242
|
+
let stream;
|
|
158
243
|
try {
|
|
159
|
-
|
|
160
|
-
const { done, value } = await reader.read();
|
|
161
|
-
if (done) break;
|
|
162
|
-
const chunk = typeof value === "string" ? decoder.decode() + value : decoder.decode(value, { stream: true });
|
|
163
|
-
const processed = stream.processChunk(chunk);
|
|
164
|
-
if (processed) yield processed;
|
|
165
|
-
}
|
|
166
|
-
const decoderTail = decoder.decode();
|
|
167
|
-
if (decoderTail) {
|
|
168
|
-
const processed = stream.processChunk(decoderTail);
|
|
169
|
-
if (processed) yield processed;
|
|
170
|
-
}
|
|
171
|
-
const final_ = stream.finish();
|
|
172
|
-
if (final_) yield final_;
|
|
244
|
+
stream = new MarkdownStream$1(resolved.napiOpts);
|
|
173
245
|
} catch (error) {
|
|
174
|
-
throw wasmPanicError(error);
|
|
175
|
-
} finally {
|
|
176
|
-
reader.releaseLock();
|
|
246
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
177
247
|
}
|
|
248
|
+
yield* pumpStream({
|
|
249
|
+
processChunk: (chunk) => stream.processChunk(carry.take(chunk)),
|
|
250
|
+
processChunkBytes: (chunk) => {
|
|
251
|
+
const held = carry.flush();
|
|
252
|
+
return held ? stream.processChunk(held) + stream.processChunkBytes(chunk) : stream.processChunkBytes(chunk);
|
|
253
|
+
},
|
|
254
|
+
finish: () => {
|
|
255
|
+
const held = carry.flush();
|
|
256
|
+
return held ? stream.processChunk(held) + stream.finish() : stream.finish();
|
|
257
|
+
},
|
|
258
|
+
takeData: () => stream.takeData()
|
|
259
|
+
}, htmlStream, resolved, { mapError: (error) => wasmPanicError(error, __mdreamTakePanicMessage) });
|
|
178
260
|
}
|
|
179
261
|
export { MarkdownStream, htmlToMarkdown, streamHtmlToMarkdown };
|