mdream 2.1.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -96,6 +96,7 @@ externals: ['mdream']
96
96
  - [CLI Usage](#cli-usage)
97
97
  - [Browser and Edge Usage](#browser-and-edge-usage)
98
98
  - [Edge / Cloudflare Workers](#edge--cloudflare-workers)
99
+ - [Smaller Markdown WASM build](#smaller-markdown-wasm-build)
99
100
  - [Browser CDN (IIFE)](#browser-cdn-iife)
100
101
  - [llms.txt Generation](#llmstxt-generation)
101
102
  - [Related Packages](#related-packages)
@@ -1140,7 +1141,7 @@ If you pass an unknown flag, a flag without a value, or an unknown preset, the C
1140
1141
 
1141
1142
  ## Browser and Edge Usage
1142
1143
 
1143
- Every entry takes the same `MdreamOptions` and produces the same Markdown string.
1144
+ By default, every entry takes the same `MdreamOptions` and produces the same Markdown string.
1144
1145
  In Node and edge runtimes, `htmlToMarkdown` from `mdream` is synchronous.
1145
1146
  In a browser bundle, import from `mdream/browser`.
1146
1147
  Its `htmlToMarkdown` returns `Promise<string>`, because the WASM binary loads first.
@@ -1207,6 +1208,30 @@ await init({ module_or_path: wasmModule })
1207
1208
  const markdown = htmlToMarkdown('<h1>Hello</h1>', { minimal: true })
1208
1209
  ```
1209
1210
 
1211
+ ### Smaller Markdown WASM build
1212
+
1213
+ Enable the `mdream-markdown` export condition to bundle a smaller WASM engine.
1214
+ Keep importing from `mdream/browser` in browsers or `mdream` on the edge.
1215
+ One-shot conversion, `MarkdownStream`, and `streamHtmlToMarkdown` all work.
1216
+
1217
+ In Vite, add the condition to your client config.
1218
+
1219
+ ```ts
1220
+ import { defaultClientConditions, defineConfig } from 'vite'
1221
+
1222
+ export default defineConfig({
1223
+ resolve: { conditions: [...defaultClientConditions, 'mdream-markdown'] },
1224
+ })
1225
+ ```
1226
+
1227
+ In Wrangler, set `WRANGLER_BUILD_CONDITIONS=workerd,worker,browser,mdream-markdown` in the environment that builds the Worker.
1228
+ For matching TypeScript types, add `mdream-markdown` to `compilerOptions.customConditions`.
1229
+ For an edge import, also include your runtime's condition, such as `workerd`.
1230
+
1231
+ This build accepts `origin` and `format: 'markdown'`.
1232
+ It throws on other options, including plugins, cleanup, and `wrapWidth`.
1233
+ Remove the condition if your app needs those options or text or safe HTML output.
1234
+
1210
1235
  ### Browser CDN (IIFE)
1211
1236
 
1212
1237
  Use mdream from a CDN without a build step.
@@ -0,0 +1,129 @@
1
+ import { HtmlToMarkdownOptions, MdreamNapiResult } from "../napi/index.js";
2
+ interface CleanOptions {
3
+ /** Strip tracking query parameters (utm_*, fbclid, gclid, etc.) from URLs */
4
+ urls?: boolean;
5
+ /** Strip fragment-only links that don't match any heading in the output */
6
+ fragments?: boolean;
7
+ /** Strip links with meaningless hrefs (#, javascript:void(0)) → plain text */
8
+ emptyLinks?: boolean;
9
+ /** Strip links where text equals URL: [https://x.com](https://x.com) → https://x.com */
10
+ redundantLinks?: boolean;
11
+ /** Strip self-referencing heading anchors: ## [Title](#title) → ## Title */
12
+ selfLinkHeadings?: boolean;
13
+ /** Strip images with no alt text (decorative/tracking pixels) */
14
+ emptyImages?: boolean;
15
+ /** Drop links that produce no visible text: [](url) → nothing */
16
+ emptyLinkText?: boolean;
17
+ }
18
+ interface ExtractedElement {
19
+ selector: string;
20
+ tagName: string;
21
+ textContent: string;
22
+ attributes: Record<string, string>;
23
+ }
24
+ interface FrontmatterConfig {
25
+ additionalFields?: Record<string, string>;
26
+ metaFields?: string[];
27
+ /** Callback to receive structured frontmatter data after conversion */
28
+ onExtract?: (frontmatter: Record<string, string>) => void;
29
+ }
30
+ interface TagOverride {
31
+ enter?: string;
32
+ exit?: string;
33
+ spacing?: number[];
34
+ isInline?: boolean;
35
+ isSelfClosing?: boolean;
36
+ collapsesInnerWhiteSpace?: boolean;
37
+ alias?: string;
38
+ }
39
+ interface MdreamOptions {
40
+ /** Origin URL for resolving relative image paths and internal links. */
41
+ origin?: string;
42
+ /**
43
+ * Clean up the markdown output. Pass `true` for all cleanup or an object
44
+ * to enable specific features. Both `htmlToMarkdown` and
45
+ * `streamHtmlToMarkdown` apply it.
46
+ */
47
+ clean?: boolean | CleanOptions;
48
+ /** Enable minimal preset (frontmatter, isolateMain, tailwind, filter). Default: false */
49
+ minimal?: boolean;
50
+ /**
51
+ * Extract frontmatter from HTML head.
52
+ * - `true`: enable with defaults
53
+ * - `(fm) => void`: enable and receive structured data via callback
54
+ * - `FrontmatterConfig`: enable with config options and optional callback
55
+ */
56
+ frontmatter?: boolean | ((frontmatter: Record<string, string>) => void) | FrontmatterConfig;
57
+ /** Isolate main content area. Default when minimal: true */
58
+ isolateMain?: boolean;
59
+ /** Convert Tailwind utility classes. Default when minimal: true */
60
+ tailwind?: boolean;
61
+ /**
62
+ * Filter elements by CSS selector. `minimal` excludes form, nav, footer, and
63
+ * similar; a filter passed with `minimal` adds to those excludes. `false`
64
+ * turns filtering off, including the `minimal` one.
65
+ */
66
+ filter?: false | {
67
+ include?: string[];
68
+ exclude?: string[];
69
+ processChildren?: boolean;
70
+ };
71
+ /** Extract elements matching CSS selectors */
72
+ extraction?: Record<string, (element: ExtractedElement) => void>;
73
+ /** Tag overrides. String values act as aliases */
74
+ tagOverrides?: Record<string, TagOverride | string>;
75
+ /**
76
+ * Hard-wrap prose at this many characters, breaking on word boundaries.
77
+ * Applied inline during conversion (zero-cost when unset). Code blocks,
78
+ * tables, and headings are never wrapped. `0` disables wrapping.
79
+ */
80
+ wrapWidth?: number;
81
+ /**
82
+ * Output format. Defaults to `markdown`; use `text` for readable plain text,
83
+ * or `html` for allowlisted semantic HTML.
84
+ */
85
+ format?: 'markdown' | 'text' | 'html';
86
+ }
87
+ interface ResolvedOptions {
88
+ napiOpts: HtmlToMarkdownOptions;
89
+ extractionHandlers?: Record<string, (el: ExtractedElement) => void>;
90
+ frontmatterCallback?: (fm: Record<string, string>) => void;
91
+ }
92
+ type FullBuildOption = Exclude<keyof MdreamOptions, 'origin' | 'format'>;
93
+ type MarkdownOptions = {
94
+ origin?: string;
95
+ format?: 'markdown';
96
+ } & Partial<Record<FullBuildOption, undefined>>;
97
+ interface WasmStream {
98
+ processChunk: (chunk: string) => string;
99
+ processChunkBytes: (chunk: Uint8Array) => string;
100
+ finish: () => string;
101
+ takeData: () => Omit<MdreamNapiResult, 'markdown'>;
102
+ free: () => void;
103
+ }
104
+ /** The bindings of one WASM instance a conversion may use. */
105
+ interface WasmEngine {
106
+ oneShot: (html: string, napiOpts: HtmlToMarkdownOptions) => MdreamNapiResult;
107
+ /** Only the entries that take bytes pass it; without it, bytes stream. */
108
+ bytes?: (html: Uint8Array, napiOpts: HtmlToMarkdownOptions) => string;
109
+ Stream: new (napiOpts: HtmlToMarkdownOptions) => WasmStream;
110
+ }
111
+ interface WasmBindings extends WasmEngine {
112
+ takePanicMessage: () => string | undefined;
113
+ }
114
+ declare class WasmMarkdownStream {
115
+ private readonly _bindings;
116
+ private readonly _callbacks;
117
+ private readonly _chunks;
118
+ protected constructor(_bindings: WasmBindings, _callbacks: ResolvedOptions);
119
+ processChunk(chunk: string): string;
120
+ processChunkBytes(chunk: Uint8Array): string;
121
+ finish(): string;
122
+ }
123
+ export declare function htmlToMarkdown(html: string, options?: MarkdownOptions): Promise<string>;
124
+ export declare function createMarkdownStream(options?: MarkdownOptions): Promise<MarkdownStream>;
125
+ export declare class MarkdownStream extends WasmMarkdownStream {
126
+ constructor(options?: MarkdownOptions);
127
+ }
128
+ export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: MarkdownOptions): AsyncIterable<string>;
129
+ export type { CleanOptions, ExtractedElement, FrontmatterConfig, MarkdownOptions as MdreamOptions, TagOverride };
@@ -0,0 +1,229 @@
1
+ import init, { MarkdownStream as MarkdownStream$1, __mdreamTakePanicMessage, htmlToMarkdownResult } from "../wasm-bundler/mdream_markdown_web.js";
2
+ const FULL_BUILD_OPTIONS = {
3
+ clean: true,
4
+ minimal: true,
5
+ frontmatter: true,
6
+ isolateMain: true,
7
+ tailwind: true,
8
+ filter: true,
9
+ extraction: true,
10
+ tagOverrides: true,
11
+ wrapWidth: true
12
+ };
13
+ const fullBuildKeys = Object.keys(FULL_BUILD_OPTIONS);
14
+ function unsupportedOption(key) {
15
+ throw new TypeError(`The mdream Markdown build does not support the \`${key}\` option. Remove the mdream-markdown condition to use it.`);
16
+ }
17
+ function rejectUnknownOption(values, key) {
18
+ if (key !== "origin" && key !== "format" && !Object.hasOwn(FULL_BUILD_OPTIONS, key) && values[key] !== void 0) unsupportedOption(key);
19
+ }
20
+ function resolveMarkdownOptions(options = {}) {
21
+ const values = options;
22
+ const { format } = values;
23
+ if (format !== void 0 && format !== "markdown") unsupportedOption("format");
24
+ for (const key of fullBuildKeys) if (values[key] !== void 0) unsupportedOption(key);
25
+ for (const key of Object.getOwnPropertyNames(values)) rejectUnknownOption(values, key);
26
+ for (const key in values) if (!Object.hasOwn(values, key)) rejectUnknownOption(values, key);
27
+ return { napiOpts: { origin: options.origin } };
28
+ }
29
+ function deliverPluginData(data, { frontmatterCallback, extractionHandlers }) {
30
+ if (data.frontmatter && frontmatterCallback) frontmatterCallback(data.frontmatter);
31
+ if (data.extracted?.length && extractionHandlers) for (const element of data.extracted) extractionHandlers[element.selector]?.(element);
32
+ }
33
+ function hasCallbacks({ frontmatterCallback, extractionHandlers }) {
34
+ return !!frontmatterCallback || !!extractionHandlers;
35
+ }
36
+ const utf8 = new TextEncoder();
37
+ const WASM_CHUNK_BYTES = 16384;
38
+ const WASM_ONE_SHOT_MAX = 256;
39
+ function convertWasm(html, napiOpts, engine) {
40
+ if (html.length <= WASM_ONE_SHOT_MAX) return engine.oneShot(html, napiOpts);
41
+ return withWasmStream(engine, napiOpts, (stream) => {
42
+ let markdown = "";
43
+ if (html.charCodeAt(0) === 65279) {
44
+ markdown += stream.processChunk(String.fromCharCode(65279));
45
+ html = html.slice(1);
46
+ }
47
+ const bytes = new Uint8Array(WASM_CHUNK_BYTES);
48
+ for (let start = 0; start < html.length;) {
49
+ const { read, written } = utf8.encodeInto(html.slice(start, start + WASM_CHUNK_BYTES), bytes);
50
+ markdown += stream.processChunkBytes(bytes.subarray(0, written));
51
+ start += read;
52
+ }
53
+ return {
54
+ markdown: markdown + stream.finish(),
55
+ ...stream.takeData()
56
+ };
57
+ });
58
+ }
59
+ function withWasmStream(engine, napiOpts, use) {
60
+ const stream = new engine.Stream(napiOpts);
61
+ try {
62
+ return use(stream);
63
+ } finally {
64
+ try {
65
+ stream.free();
66
+ } catch {}
67
+ }
68
+ }
69
+ function freeStream(stream) {
70
+ try {
71
+ stream.free?.();
72
+ } catch {}
73
+ }
74
+ async function* pumpStream(stream, htmlStream, callbacks, { mapError = (error) => error } = {}) {
75
+ const decoder = stream.processChunkBytes ? void 0 : new TextDecoder();
76
+ let reader;
77
+ try {
78
+ reader = htmlStream.getReader();
79
+ while (true) {
80
+ const { done, value } = await reader.read();
81
+ if (done) break;
82
+ let processed;
83
+ if (typeof value === "string") processed = stream.processChunk(decoder ? decoder.decode() + value : value);
84
+ else processed = decoder ? stream.processChunk(decoder.decode(value, { stream: true })) : stream.processChunkBytes(value);
85
+ if (processed) yield processed;
86
+ }
87
+ const decoderTail = decoder?.decode();
88
+ if (decoderTail) {
89
+ const processed = stream.processChunk(decoderTail);
90
+ if (processed) yield processed;
91
+ }
92
+ const final_ = stream.finish();
93
+ if (hasCallbacks(callbacks)) deliverPluginData(stream.takeData(), callbacks);
94
+ if (final_) yield final_;
95
+ } catch (error) {
96
+ throw mapError(error);
97
+ } finally {
98
+ freeStream(stream);
99
+ reader?.releaseLock();
100
+ }
101
+ }
102
+ function createSurrogateCarry() {
103
+ let held = "";
104
+ return {
105
+ take(chunk) {
106
+ if (held) {
107
+ chunk = held + chunk;
108
+ held = "";
109
+ }
110
+ const last = chunk.charCodeAt(chunk.length - 1);
111
+ if (last >= 55296 && last <= 56319) {
112
+ held = chunk.slice(-1);
113
+ return chunk.slice(0, -1);
114
+ }
115
+ return chunk;
116
+ },
117
+ flush() {
118
+ const value = held;
119
+ held = "";
120
+ return value;
121
+ }
122
+ };
123
+ }
124
+ function wasmPanicError(error, takeMessage) {
125
+ if (!(error instanceof WebAssembly.RuntimeError)) return error;
126
+ let message;
127
+ try {
128
+ message = takeMessage();
129
+ } catch {
130
+ return error;
131
+ }
132
+ if (!message) return error;
133
+ return new Error(`mdream WASM panic, please report this at https://github.com/harlan-zw/mdream/issues\n${message}`, { cause: error });
134
+ }
135
+ function convertWasmHtml(html, bindings, resolved) {
136
+ let result;
137
+ try {
138
+ result = convertWasm(html, resolved.napiOpts, bindings);
139
+ } catch (error) {
140
+ throw wasmPanicError(error, bindings.takePanicMessage);
141
+ }
142
+ deliverPluginData(result, resolved);
143
+ return result.markdown || "";
144
+ }
145
+ function streamChunks(inner) {
146
+ const carry = createSurrogateCarry();
147
+ return {
148
+ processChunk: (chunk) => inner.processChunk(carry.take(chunk)),
149
+ processChunkBytes: (chunk) => {
150
+ const held = carry.flush();
151
+ return held ? inner.processChunk(held) + inner.processChunkBytes(chunk) : inner.processChunkBytes(chunk);
152
+ },
153
+ finish: () => {
154
+ const held = carry.flush();
155
+ return held ? inner.processChunk(held) + inner.finish() : inner.finish();
156
+ },
157
+ takeData: () => inner.takeData(),
158
+ free: () => inner.free()
159
+ };
160
+ }
161
+ function openStream(bindings, { napiOpts }) {
162
+ try {
163
+ return streamChunks(new bindings.Stream(napiOpts));
164
+ } catch (error) {
165
+ throw wasmPanicError(error, bindings.takePanicMessage);
166
+ }
167
+ }
168
+ var WasmMarkdownStream = class {
169
+ _bindings;
170
+ _callbacks;
171
+ _chunks;
172
+ constructor(_bindings, _callbacks) {
173
+ this._bindings = _bindings;
174
+ this._callbacks = _callbacks;
175
+ this._chunks = openStream(_bindings, _callbacks);
176
+ }
177
+ processChunk(chunk) {
178
+ try {
179
+ return this._chunks.processChunk(chunk);
180
+ } catch (error) {
181
+ throw wasmPanicError(error, this._bindings.takePanicMessage);
182
+ }
183
+ }
184
+ processChunkBytes(chunk) {
185
+ try {
186
+ return this._chunks.processChunkBytes(chunk);
187
+ } catch (error) {
188
+ throw wasmPanicError(error, this._bindings.takePanicMessage);
189
+ }
190
+ }
191
+ finish() {
192
+ let markdown;
193
+ try {
194
+ markdown = this._chunks.finish();
195
+ } catch (error) {
196
+ throw wasmPanicError(error, this._bindings.takePanicMessage);
197
+ }
198
+ if (this._callbacks.frontmatterCallback || this._callbacks.extractionHandlers) deliverPluginData(this._chunks.takeData(), this._callbacks);
199
+ return markdown;
200
+ }
201
+ };
202
+ function pumpWasmStream(htmlStream, bindings, resolved) {
203
+ return pumpStream(openStream(bindings, resolved), htmlStream, resolved, { mapError: (error) => wasmPanicError(error, bindings.takePanicMessage) });
204
+ }
205
+ const bindings = {
206
+ oneShot: htmlToMarkdownResult,
207
+ Stream: MarkdownStream$1,
208
+ takePanicMessage: __mdreamTakePanicMessage
209
+ };
210
+ const ready = init();
211
+ async function htmlToMarkdown(html, options = {}) {
212
+ await ready;
213
+ return convertWasmHtml(html, bindings, resolveMarkdownOptions(options));
214
+ }
215
+ async function createMarkdownStream(options) {
216
+ await ready;
217
+ return new MarkdownStream(options);
218
+ }
219
+ var MarkdownStream = class extends WasmMarkdownStream {
220
+ constructor(options = {}) {
221
+ super(bindings, resolveMarkdownOptions(options));
222
+ }
223
+ };
224
+ async function* streamHtmlToMarkdown(htmlStream, options = {}) {
225
+ if (!htmlStream) throw new Error("Invalid HTML stream provided");
226
+ await ready;
227
+ yield* pumpWasmStream(htmlStream, bindings, resolveMarkdownOptions(options));
228
+ }
229
+ export { MarkdownStream, createMarkdownStream, htmlToMarkdown, streamHtmlToMarkdown };
@@ -1,3 +1,4 @@
1
+ import { HtmlToMarkdownOptions, MdreamNapiResult } from "../napi/index.js";
1
2
  interface CleanOptions {
2
3
  /** Strip tracking query parameters (utm_*, fbclid, gclid, etc.) from URLs */
3
4
  urls?: boolean;
@@ -83,21 +84,41 @@ interface MdreamOptions {
83
84
  */
84
85
  format?: 'markdown' | 'text' | 'html';
85
86
  }
86
- /**
87
- * Browser builds fetch the WASM binary on first use, so this returns a
88
- * Promise. `mdream/browser` ships types that say so.
89
- */
90
- export declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): Promise<string>;
91
- export declare function createMarkdownStream(options?: Partial<MdreamOptions>): Promise<MarkdownStream>;
92
- /** Streaming converter. Runs the frontmatter and extraction callbacks in `finish()`. */
93
- export declare class MarkdownStream {
94
- private _inner;
95
- private _carry;
96
- private _callbacks;
97
- constructor(options?: Partial<MdreamOptions>);
87
+ interface ResolvedOptions {
88
+ napiOpts: HtmlToMarkdownOptions;
89
+ extractionHandlers?: Record<string, (el: ExtractedElement) => void>;
90
+ frontmatterCallback?: (fm: Record<string, string>) => void;
91
+ }
92
+ interface WasmStream {
93
+ processChunk: (chunk: string) => string;
94
+ processChunkBytes: (chunk: Uint8Array) => string;
95
+ finish: () => string;
96
+ takeData: () => Omit<MdreamNapiResult, 'markdown'>;
97
+ free: () => void;
98
+ }
99
+ /** The bindings of one WASM instance a conversion may use. */
100
+ interface WasmEngine {
101
+ oneShot: (html: string, napiOpts: HtmlToMarkdownOptions) => MdreamNapiResult;
102
+ /** Only the entries that take bytes pass it; without it, bytes stream. */
103
+ bytes?: (html: Uint8Array, napiOpts: HtmlToMarkdownOptions) => string;
104
+ Stream: new (napiOpts: HtmlToMarkdownOptions) => WasmStream;
105
+ }
106
+ interface WasmBindings extends WasmEngine {
107
+ takePanicMessage: () => string | undefined;
108
+ }
109
+ declare class WasmMarkdownStream {
110
+ private readonly _bindings;
111
+ private readonly _callbacks;
112
+ private readonly _chunks;
113
+ protected constructor(_bindings: WasmBindings, _callbacks: ResolvedOptions);
98
114
  processChunk(chunk: string): string;
99
115
  processChunkBytes(chunk: Uint8Array): string;
100
116
  finish(): string;
101
117
  }
118
+ export declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): Promise<string>;
119
+ export declare function createMarkdownStream(options?: Partial<MdreamOptions>): Promise<MarkdownStream>;
120
+ export declare class MarkdownStream extends WasmMarkdownStream {
121
+ constructor(options?: Partial<MdreamOptions>);
122
+ }
102
123
  export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
103
124
  export type { CleanOptions, ExtractedElement, FrontmatterConfig, MdreamOptions, TagOverride };
package/dist/browser.mjs CHANGED
@@ -120,16 +120,49 @@ function deliverPluginData(data, { frontmatterCallback, extractionHandlers }) {
120
120
  function hasCallbacks({ frontmatterCallback, extractionHandlers }) {
121
121
  return !!frontmatterCallback || !!extractionHandlers;
122
122
  }
123
- function convertResult(convert, html, options = {}) {
124
- const resolved = resolveOptions(options);
125
- const result = convert(html, resolved.napiOpts);
126
- deliverPluginData(result, resolved);
127
- return result;
123
+ const utf8 = new TextEncoder();
124
+ const WASM_CHUNK_BYTES = 16384;
125
+ const WASM_ONE_SHOT_MAX = 256;
126
+ function convertWasm(html, napiOpts, engine) {
127
+ if (html.length <= WASM_ONE_SHOT_MAX) return engine.oneShot(html, napiOpts);
128
+ return withWasmStream(engine, napiOpts, (stream) => {
129
+ let markdown = "";
130
+ if (html.charCodeAt(0) === 65279) {
131
+ markdown += stream.processChunk(String.fromCharCode(65279));
132
+ html = html.slice(1);
133
+ }
134
+ const bytes = new Uint8Array(WASM_CHUNK_BYTES);
135
+ for (let start = 0; start < html.length;) {
136
+ const { read, written } = utf8.encodeInto(html.slice(start, start + WASM_CHUNK_BYTES), bytes);
137
+ markdown += stream.processChunkBytes(bytes.subarray(0, written));
138
+ start += read;
139
+ }
140
+ return {
141
+ markdown: markdown + stream.finish(),
142
+ ...stream.takeData()
143
+ };
144
+ });
145
+ }
146
+ function withWasmStream(engine, napiOpts, use) {
147
+ const stream = new engine.Stream(napiOpts);
148
+ try {
149
+ return use(stream);
150
+ } finally {
151
+ try {
152
+ stream.free();
153
+ } catch {}
154
+ }
155
+ }
156
+ function freeStream(stream) {
157
+ try {
158
+ stream.free?.();
159
+ } catch {}
128
160
  }
129
161
  async function* pumpStream(stream, htmlStream, callbacks, { mapError = (error) => error } = {}) {
130
- const reader = htmlStream.getReader();
131
162
  const decoder = stream.processChunkBytes ? void 0 : new TextDecoder();
163
+ let reader;
132
164
  try {
165
+ reader = htmlStream.getReader();
133
166
  while (true) {
134
167
  const { done, value } = await reader.read();
135
168
  if (done) break;
@@ -149,7 +182,8 @@ async function* pumpStream(stream, htmlStream, callbacks, { mapError = (error) =
149
182
  } catch (error) {
150
183
  throw mapError(error);
151
184
  } finally {
152
- reader.releaseLock();
185
+ freeStream(stream);
186
+ reader?.releaseLock();
153
187
  }
154
188
  }
155
189
  function createSurrogateCarry() {
@@ -185,89 +219,98 @@ function wasmPanicError(error, takeMessage) {
185
219
  if (!message) return error;
186
220
  return new Error(`mdream WASM panic, please report this at https://github.com/harlan-zw/mdream/issues\n${message}`, { cause: error });
187
221
  }
188
- let _initPromise;
189
- function ensureInit() {
190
- if (!_initPromise) _initPromise = init();
191
- return _initPromise;
192
- }
193
- ensureInit();
194
- function convert(html, napiOpts) {
222
+ function convertWasmHtml(html, bindings, resolved) {
223
+ let result;
195
224
  try {
196
- return htmlToMarkdownResult(html, napiOpts);
225
+ result = convertWasm(html, resolved.napiOpts, bindings);
197
226
  } catch (error) {
198
- throw wasmPanicError(error, __mdreamTakePanicMessage);
227
+ throw wasmPanicError(error, bindings.takePanicMessage);
199
228
  }
229
+ deliverPluginData(result, resolved);
230
+ return result.markdown || "";
200
231
  }
201
- async function htmlToMarkdown(html, options = {}) {
202
- await ensureInit();
203
- return convertResult(convert, html, options).markdown || "";
232
+ function streamChunks(inner) {
233
+ const carry = createSurrogateCarry();
234
+ return {
235
+ processChunk: (chunk) => inner.processChunk(carry.take(chunk)),
236
+ processChunkBytes: (chunk) => {
237
+ const held = carry.flush();
238
+ return held ? inner.processChunk(held) + inner.processChunkBytes(chunk) : inner.processChunkBytes(chunk);
239
+ },
240
+ finish: () => {
241
+ const held = carry.flush();
242
+ return held ? inner.processChunk(held) + inner.finish() : inner.finish();
243
+ },
244
+ takeData: () => inner.takeData(),
245
+ free: () => inner.free()
246
+ };
204
247
  }
205
- async function createMarkdownStream(options) {
206
- await ensureInit();
207
- return new MarkdownStream(options);
248
+ function openStream(bindings, { napiOpts }) {
249
+ try {
250
+ return streamChunks(new bindings.Stream(napiOpts));
251
+ } catch (error) {
252
+ throw wasmPanicError(error, bindings.takePanicMessage);
253
+ }
208
254
  }
209
- var MarkdownStream = class {
210
- _inner;
211
- _carry = createSurrogateCarry();
255
+ var WasmMarkdownStream = class {
256
+ _bindings;
212
257
  _callbacks;
213
- constructor(options = {}) {
214
- const resolved = resolveOptions(options);
215
- this._callbacks = resolved;
216
- try {
217
- this._inner = new MarkdownStream$1(resolved.napiOpts);
218
- } catch (error) {
219
- throw wasmPanicError(error, __mdreamTakePanicMessage);
220
- }
258
+ _chunks;
259
+ constructor(_bindings, _callbacks) {
260
+ this._bindings = _bindings;
261
+ this._callbacks = _callbacks;
262
+ this._chunks = openStream(_bindings, _callbacks);
221
263
  }
222
264
  processChunk(chunk) {
223
265
  try {
224
- return this._inner.processChunk(this._carry.take(chunk));
266
+ return this._chunks.processChunk(chunk);
225
267
  } catch (error) {
226
- throw wasmPanicError(error, __mdreamTakePanicMessage);
268
+ throw wasmPanicError(error, this._bindings.takePanicMessage);
227
269
  }
228
270
  }
229
271
  processChunkBytes(chunk) {
230
272
  try {
231
- const held = this._carry.flush();
232
- return held ? this._inner.processChunk(held) + this._inner.processChunkBytes(chunk) : this._inner.processChunkBytes(chunk);
273
+ return this._chunks.processChunkBytes(chunk);
233
274
  } catch (error) {
234
- throw wasmPanicError(error, __mdreamTakePanicMessage);
275
+ throw wasmPanicError(error, this._bindings.takePanicMessage);
235
276
  }
236
277
  }
237
278
  finish() {
238
279
  let markdown;
239
280
  try {
240
- const held = this._carry.flush();
241
- markdown = held ? this._inner.processChunk(held) + this._inner.finish() : this._inner.finish();
281
+ markdown = this._chunks.finish();
242
282
  } catch (error) {
243
- throw wasmPanicError(error, __mdreamTakePanicMessage);
283
+ throw wasmPanicError(error, this._bindings.takePanicMessage);
244
284
  }
245
- if (this._callbacks.frontmatterCallback || this._callbacks.extractionHandlers) deliverPluginData(this._inner.takeData(), this._callbacks);
285
+ if (this._callbacks.frontmatterCallback || this._callbacks.extractionHandlers) deliverPluginData(this._chunks.takeData(), this._callbacks);
246
286
  return markdown;
247
287
  }
248
288
  };
289
+ function pumpWasmStream(htmlStream, bindings, resolved) {
290
+ return pumpStream(openStream(bindings, resolved), htmlStream, resolved, { mapError: (error) => wasmPanicError(error, bindings.takePanicMessage) });
291
+ }
292
+ const bindings = {
293
+ oneShot: htmlToMarkdownResult,
294
+ Stream: MarkdownStream$1,
295
+ takePanicMessage: __mdreamTakePanicMessage
296
+ };
297
+ const ready = init();
298
+ async function htmlToMarkdown(html, options = {}) {
299
+ await ready;
300
+ return convertWasmHtml(html, bindings, resolveOptions(options));
301
+ }
302
+ async function createMarkdownStream(options) {
303
+ await ready;
304
+ return new MarkdownStream(options);
305
+ }
306
+ var MarkdownStream = class extends WasmMarkdownStream {
307
+ constructor(options = {}) {
308
+ super(bindings, resolveOptions(options));
309
+ }
310
+ };
249
311
  async function* streamHtmlToMarkdown(htmlStream, options = {}) {
250
312
  if (!htmlStream) throw new Error("Invalid HTML stream provided");
251
- await ensureInit();
252
- const resolved = resolveOptions(options);
253
- const carry = createSurrogateCarry();
254
- let stream;
255
- try {
256
- stream = new MarkdownStream$1(resolved.napiOpts);
257
- } catch (error) {
258
- throw wasmPanicError(error, __mdreamTakePanicMessage);
259
- }
260
- yield* pumpStream({
261
- processChunk: (chunk) => stream.processChunk(carry.take(chunk)),
262
- processChunkBytes: (chunk) => {
263
- const held = carry.flush();
264
- return held ? stream.processChunk(held) + stream.processChunkBytes(chunk) : stream.processChunkBytes(chunk);
265
- },
266
- finish: () => {
267
- const held = carry.flush();
268
- return held ? stream.processChunk(held) + stream.finish() : stream.finish();
269
- },
270
- takeData: () => stream.takeData()
271
- }, htmlStream, resolved, { mapError: (error) => wasmPanicError(error, __mdreamTakePanicMessage) });
313
+ await ready;
314
+ yield* pumpWasmStream(htmlStream, bindings, resolveOptions(options));
272
315
  }
273
316
  export { MarkdownStream, createMarkdownStream, htmlToMarkdown, streamHtmlToMarkdown };