mdream 1.7.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/edge.mjs CHANGED
@@ -1,6 +1,5 @@
1
- import { MarkdownStream as MarkdownStream$1, __mdreamTakePanicMessage, htmlToMarkdownResult, initSync } from "../wasm/mdream_edge.js";
2
- import wasmModule from "../wasm/mdream_edge_bg.wasm";
3
- const MINIMAL_FILTER_DEFAULT = { exclude: [
1
+ import { MarkdownStream as MarkdownStream$1, __mdreamTakePanicMessage, htmlToMarkdownResult } from "../wasm-bundler/mdream_edge.js";
2
+ const MINIMAL_FILTER_EXCLUDE = [
4
3
  "form",
5
4
  "fieldset",
6
5
  "object",
@@ -13,7 +12,7 @@ const MINIMAL_FILTER_DEFAULT = { exclude: [
13
12
  "select",
14
13
  "button",
15
14
  "nav"
16
- ] };
15
+ ];
17
16
  const CLEAN_ALL = {
18
17
  urls: true,
19
18
  fragments: true,
@@ -23,19 +22,38 @@ const CLEAN_ALL = {
23
22
  emptyImages: true,
24
23
  emptyLinkText: true
25
24
  };
26
- function resolveCleanConfig(options, minimal) {
27
- const cleanUrls = options.cleanUrls === true;
28
- let cleanOpt = options.clean;
29
- if (cleanOpt === void 0) {
30
- if (!minimal) return { cleanUrls };
31
- cleanOpt = true;
25
+ const VALID_OPTIONS = "origin, clean, minimal, frontmatter, isolateMain, tailwind, filter, extraction, tagOverrides, wrapWidth, format";
26
+ function unknownOptionMessage(key, value) {
27
+ switch (key) {
28
+ case "preset": return "mdream has no `preset` option. Pass { minimal: true }.";
29
+ case "cleanUrls": return "mdream has no `cleanUrls` option. Pass { clean: { urls: true } }.";
30
+ case "plugins": return Array.isArray(value) ? "Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://github.com/harlan-zw/mdream/tree/main/packages/js#migrating-from-v1" : "mdream takes plugin options at the top level, such as { frontmatter: true }.";
32
31
  }
33
- if (!cleanOpt) return { cleanUrls };
34
- const resolved = cleanOpt === true ? CLEAN_ALL : cleanOpt;
35
- return {
36
- cleanUrls: cleanUrls || resolved.urls === true,
37
- clean: resolved
38
- };
32
+ return `mdream has no \`${key}\` option. Valid options: ${VALID_OPTIONS}.`;
33
+ }
34
+ function assertKnownOptions(options) {
35
+ for (const key in options) {
36
+ switch (key) {
37
+ case "origin":
38
+ case "clean":
39
+ case "minimal":
40
+ case "frontmatter":
41
+ case "isolateMain":
42
+ case "tailwind":
43
+ case "filter":
44
+ case "extraction":
45
+ case "tagOverrides":
46
+ case "wrapWidth":
47
+ case "format": continue;
48
+ }
49
+ const value = options[key];
50
+ if (value !== void 0) throw new TypeError(unknownOptionMessage(key, value));
51
+ }
52
+ }
53
+ function resolveClean(clean, minimal) {
54
+ if (clean === void 0) return minimal ? CLEAN_ALL : void 0;
55
+ if (!clean) return void 0;
56
+ return clean === true ? CLEAN_ALL : clean;
39
57
  }
40
58
  function resolveFrontmatter(opt) {
41
59
  if (typeof opt === "function") return {
@@ -52,23 +70,21 @@ function resolveFrontmatter(opt) {
52
70
  return { config: {} };
53
71
  }
54
72
  function resolveOptions(options) {
55
- if (Array.isArray(options.plugins)) throw new TypeError("Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://mdream.dev/v1-migration#custom-plugins");
73
+ assertKnownOptions(options);
56
74
  const minimal = options.minimal === true;
57
- const plugins = options.plugins ? { ...options.plugins } : {};
75
+ const plugins = {};
58
76
  let frontmatterCallback;
59
- const frontmatterDisabled = options.frontmatter === false;
60
- if (minimal ? !frontmatterDisabled : !!options.frontmatter) {
77
+ if (minimal ? options.frontmatter !== false : options.frontmatter) {
61
78
  const fm = resolveFrontmatter(options.frontmatter);
62
79
  plugins.frontmatter = fm.config;
63
80
  frontmatterCallback = fm.callback;
64
- } else if (frontmatterDisabled) delete plugins.frontmatter;
65
- const isolateMainDisabled = options.isolateMain === false;
66
- if (minimal ? !isolateMainDisabled : options.isolateMain) plugins.isolateMain = true;
67
- else if (isolateMainDisabled) delete plugins.isolateMain;
68
- const tailwindDisabled = options.tailwind === false;
69
- if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
70
- else if (tailwindDisabled) delete plugins.tailwind;
71
- if (minimal) plugins.filter = options.filter || MINIMAL_FILTER_DEFAULT;
81
+ }
82
+ if (minimal ? options.isolateMain !== false : options.isolateMain) plugins.isolateMain = true;
83
+ if (minimal ? options.tailwind !== false : options.tailwind) plugins.tailwind = true;
84
+ if (minimal && options.filter !== false) plugins.filter = {
85
+ ...options.filter,
86
+ exclude: [...MINIMAL_FILTER_EXCLUDE, ...options.filter?.exclude ?? []]
87
+ };
72
88
  else if (options.filter) plugins.filter = options.filter;
73
89
  const extraction = options.extraction;
74
90
  let extractionHandlers;
@@ -85,12 +101,10 @@ function resolveOptions(options) {
85
101
  }
86
102
  plugins.tagOverrides = overrides;
87
103
  }
88
- const { cleanUrls, clean } = resolveCleanConfig(options, minimal);
89
104
  return {
90
105
  napiOpts: {
91
106
  origin: options.origin,
92
- cleanUrls,
93
- clean,
107
+ clean: resolveClean(options.clean, minimal),
94
108
  plugins,
95
109
  wrapWidth: options.wrapWidth,
96
110
  format: options.format
@@ -99,7 +113,68 @@ function resolveOptions(options) {
99
113
  frontmatterCallback
100
114
  };
101
115
  }
102
- function wasmPanicError(error, takeMessage = __mdreamTakePanicMessage) {
116
+ function deliverPluginData(data, { frontmatterCallback, extractionHandlers }) {
117
+ if (data.frontmatter && frontmatterCallback) frontmatterCallback(data.frontmatter);
118
+ if (data.extracted?.length && extractionHandlers) for (const element of data.extracted) extractionHandlers[element.selector]?.(element);
119
+ }
120
+ function hasCallbacks({ frontmatterCallback, extractionHandlers }) {
121
+ return !!frontmatterCallback || !!extractionHandlers;
122
+ }
123
+ function convertResult(convert, html, options = {}) {
124
+ const resolved = resolveOptions(options);
125
+ const result = convert(html, resolved.napiOpts);
126
+ deliverPluginData(result, resolved);
127
+ return result;
128
+ }
129
+ async function* pumpStream(stream, htmlStream, callbacks, { mapError = (error) => error } = {}) {
130
+ const reader = htmlStream.getReader();
131
+ const decoder = stream.processChunkBytes ? void 0 : new TextDecoder();
132
+ try {
133
+ while (true) {
134
+ const { done, value } = await reader.read();
135
+ if (done) break;
136
+ let processed;
137
+ if (typeof value === "string") processed = stream.processChunk(decoder ? decoder.decode() + value : value);
138
+ else processed = decoder ? stream.processChunk(decoder.decode(value, { stream: true })) : stream.processChunkBytes(value);
139
+ if (processed) yield processed;
140
+ }
141
+ const decoderTail = decoder?.decode();
142
+ if (decoderTail) {
143
+ const processed = stream.processChunk(decoderTail);
144
+ if (processed) yield processed;
145
+ }
146
+ const final_ = stream.finish();
147
+ if (hasCallbacks(callbacks)) deliverPluginData(stream.takeData(), callbacks);
148
+ if (final_) yield final_;
149
+ } catch (error) {
150
+ throw mapError(error);
151
+ } finally {
152
+ reader.releaseLock();
153
+ }
154
+ }
155
+ function createSurrogateCarry() {
156
+ let held = "";
157
+ return {
158
+ take(chunk) {
159
+ if (held) {
160
+ chunk = held + chunk;
161
+ held = "";
162
+ }
163
+ const last = chunk.charCodeAt(chunk.length - 1);
164
+ if (last >= 55296 && last <= 56319) {
165
+ held = chunk.slice(-1);
166
+ return chunk.slice(0, -1);
167
+ }
168
+ return chunk;
169
+ },
170
+ flush() {
171
+ const value = held;
172
+ held = "";
173
+ return value;
174
+ }
175
+ };
176
+ }
177
+ function wasmPanicError(error, takeMessage) {
103
178
  if (!(error instanceof WebAssembly.RuntimeError)) return error;
104
179
  let message;
105
180
  try {
@@ -110,70 +185,77 @@ function wasmPanicError(error, takeMessage = __mdreamTakePanicMessage) {
110
185
  if (!message) return error;
111
186
  return new Error(`mdream WASM panic, please report this at https://github.com/harlan-zw/mdream/issues\n${message}`, { cause: error });
112
187
  }
113
- initSync({ module: wasmModule });
114
188
  function convert(html, napiOpts) {
115
189
  try {
116
190
  return htmlToMarkdownResult(html, napiOpts);
117
191
  } catch (error) {
118
- throw wasmPanicError(error);
192
+ throw wasmPanicError(error, __mdreamTakePanicMessage);
119
193
  }
120
194
  }
121
- function htmlToMarkdown(html, options) {
122
- if (!options) return convert(html, void 0).markdown || "";
123
- const { napiOpts, extractionHandlers, frontmatterCallback } = resolveOptions(options);
124
- const result = convert(html, napiOpts);
125
- if (result.frontmatter && frontmatterCallback) frontmatterCallback(result.frontmatter);
126
- if (result.extracted?.length && extractionHandlers) for (const element of result.extracted) extractionHandlers[element.selector]?.(element);
127
- return result.markdown || "";
195
+ function htmlToMarkdown(html, options = {}) {
196
+ return convertResult(convert, html, options).markdown || "";
128
197
  }
129
198
  var MarkdownStream = class {
130
199
  _inner;
131
- constructor(options) {
200
+ _carry = createSurrogateCarry();
201
+ _callbacks;
202
+ constructor(options = {}) {
203
+ const resolved = resolveOptions(options);
204
+ this._callbacks = resolved;
132
205
  try {
133
- this._inner = new MarkdownStream$1(options);
206
+ this._inner = new MarkdownStream$1(resolved.napiOpts);
134
207
  } catch (error) {
135
- throw wasmPanicError(error);
208
+ throw wasmPanicError(error, __mdreamTakePanicMessage);
136
209
  }
137
210
  }
138
211
  processChunk(chunk) {
139
212
  try {
140
- return this._inner.processChunk(chunk);
213
+ return this._inner.processChunk(this._carry.take(chunk));
141
214
  } catch (error) {
142
- throw wasmPanicError(error);
215
+ throw wasmPanicError(error, __mdreamTakePanicMessage);
216
+ }
217
+ }
218
+ processChunkBytes(chunk) {
219
+ try {
220
+ const held = this._carry.flush();
221
+ return held ? this._inner.processChunk(held) + this._inner.processChunkBytes(chunk) : this._inner.processChunkBytes(chunk);
222
+ } catch (error) {
223
+ throw wasmPanicError(error, __mdreamTakePanicMessage);
143
224
  }
144
225
  }
145
226
  finish() {
227
+ let markdown;
146
228
  try {
147
- return this._inner.finish();
229
+ const held = this._carry.flush();
230
+ markdown = held ? this._inner.processChunk(held) + this._inner.finish() : this._inner.finish();
148
231
  } catch (error) {
149
- throw wasmPanicError(error);
232
+ throw wasmPanicError(error, __mdreamTakePanicMessage);
150
233
  }
234
+ if (this._callbacks.frontmatterCallback || this._callbacks.extractionHandlers) deliverPluginData(this._inner.takeData(), this._callbacks);
235
+ return markdown;
151
236
  }
152
237
  };
153
- async function* streamHtmlToMarkdown(htmlStream, options) {
238
+ async function* streamHtmlToMarkdown(htmlStream, options = {}) {
154
239
  if (!htmlStream) throw new Error("Invalid HTML stream provided");
155
- const stream = new MarkdownStream$1(options ? resolveOptions(options).napiOpts : void 0);
156
- const reader = htmlStream.getReader();
157
- const decoder = new TextDecoder();
240
+ const resolved = resolveOptions(options);
241
+ const carry = createSurrogateCarry();
242
+ let stream;
158
243
  try {
159
- while (true) {
160
- const { done, value } = await reader.read();
161
- if (done) break;
162
- const chunk = typeof value === "string" ? decoder.decode() + value : decoder.decode(value, { stream: true });
163
- const processed = stream.processChunk(chunk);
164
- if (processed) yield processed;
165
- }
166
- const decoderTail = decoder.decode();
167
- if (decoderTail) {
168
- const processed = stream.processChunk(decoderTail);
169
- if (processed) yield processed;
170
- }
171
- const final_ = stream.finish();
172
- if (final_) yield final_;
244
+ stream = new MarkdownStream$1(resolved.napiOpts);
173
245
  } catch (error) {
174
- throw wasmPanicError(error);
175
- } finally {
176
- reader.releaseLock();
246
+ throw wasmPanicError(error, __mdreamTakePanicMessage);
177
247
  }
248
+ yield* pumpStream({
249
+ processChunk: (chunk) => stream.processChunk(carry.take(chunk)),
250
+ processChunkBytes: (chunk) => {
251
+ const held = carry.flush();
252
+ return held ? stream.processChunk(held) + stream.processChunkBytes(chunk) : stream.processChunkBytes(chunk);
253
+ },
254
+ finish: () => {
255
+ const held = carry.flush();
256
+ return held ? stream.processChunk(held) + stream.finish() : stream.finish();
257
+ },
258
+ takeData: () => stream.takeData()
259
+ }, htmlStream, resolved, { mapError: (error) => wasmPanicError(error, __mdreamTakePanicMessage) });
178
260
  }
179
261
  export { MarkdownStream, htmlToMarkdown, streamHtmlToMarkdown };