mdream 1.7.1 → 2.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/edge.d.mts CHANGED
@@ -1,10 +1,100 @@
1
- import { HtmlToMarkdownOptions } from "../napi/index.js";
2
- declare function htmlToMarkdown(html: string, options?: HtmlToMarkdownOptions): string;
3
- declare class MarkdownStream {
1
+ interface CleanOptions {
2
+ /** Strip tracking query parameters (utm_*, fbclid, gclid, etc.) from URLs */
3
+ urls?: boolean;
4
+ /** Strip fragment-only links that don't match any heading in the output */
5
+ fragments?: boolean;
6
+ /** Strip links with meaningless hrefs (#, javascript:void(0)) → plain text */
7
+ emptyLinks?: boolean;
8
+ /** Collapse 3+ consecutive blank lines to 2 */
9
+ blankLines?: boolean;
10
+ /** Strip links where text equals URL: [https://x.com](https://x.com) → https://x.com */
11
+ redundantLinks?: boolean;
12
+ /** Strip self-referencing heading anchors: ## [Title](#title) → ## Title */
13
+ selfLinkHeadings?: boolean;
14
+ /** Strip images with no alt text (decorative/tracking pixels) */
15
+ emptyImages?: boolean;
16
+ /** Drop links that produce no visible text: [](url) → nothing */
17
+ emptyLinkText?: boolean;
18
+ }
19
+ interface ExtractedElement {
20
+ selector: string;
21
+ tagName: string;
22
+ textContent: string;
23
+ attributes: Record<string, string>;
24
+ }
25
+ interface FrontmatterConfig {
26
+ additionalFields?: Record<string, string>;
27
+ metaFields?: string[];
28
+ /** Callback to receive structured frontmatter data after conversion */
29
+ onExtract?: (frontmatter: Record<string, string>) => void;
30
+ }
31
+ interface TagOverride {
32
+ enter?: string;
33
+ exit?: string;
34
+ spacing?: number[];
35
+ isInline?: boolean;
36
+ isSelfClosing?: boolean;
37
+ collapsesInnerWhiteSpace?: boolean;
38
+ alias?: string;
39
+ }
40
+ interface MdreamOptions {
41
+ /** Origin URL for resolving relative image paths and internal links. */
42
+ origin?: string;
43
+ /**
44
+ * Clean up the markdown output. Pass `true` for all cleanup or an object
45
+ * to enable specific features. Both `htmlToMarkdown` and
46
+ * `streamHtmlToMarkdown` apply it.
47
+ */
48
+ clean?: boolean | CleanOptions;
49
+ /** Enable minimal preset (frontmatter, isolateMain, tailwind, filter). Default: false */
50
+ minimal?: boolean;
51
+ /**
52
+ * Extract frontmatter from HTML head.
53
+ * - `true`: enable with defaults
54
+ * - `(fm) => void`: enable and receive structured data via callback
55
+ * - `FrontmatterConfig`: enable with config options and optional callback
56
+ */
57
+ frontmatter?: boolean | ((frontmatter: Record<string, string>) => void) | FrontmatterConfig;
58
+ /** Isolate main content area. Default when minimal: true */
59
+ isolateMain?: boolean;
60
+ /** Convert Tailwind utility classes. Default when minimal: true */
61
+ tailwind?: boolean;
62
+ /**
63
+ * Filter elements by CSS selector. `minimal` excludes form, nav, footer, and
64
+ * similar; a filter passed with `minimal` adds to those excludes. `false`
65
+ * turns filtering off, including the `minimal` one.
66
+ */
67
+ filter?: false | {
68
+ include?: string[];
69
+ exclude?: string[];
70
+ processChildren?: boolean;
71
+ };
72
+ /** Extract elements matching CSS selectors */
73
+ extraction?: Record<string, (element: ExtractedElement) => void>;
74
+ /** Tag overrides. String values act as aliases */
75
+ tagOverrides?: Record<string, TagOverride | string>;
76
+ /**
77
+ * Hard-wrap prose at this many characters, breaking on word boundaries.
78
+ * Applied inline during conversion (zero-cost when unset). Code blocks,
79
+ * tables, and headings are never wrapped. `0` disables wrapping.
80
+ */
81
+ wrapWidth?: number;
82
+ /**
83
+ * Output format. Defaults to `markdown`; use `text` for readable plain text,
84
+ * or `html` for allowlisted semantic HTML.
85
+ */
86
+ format?: 'markdown' | 'text' | 'html';
87
+ }
88
+ export declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string;
89
+ /** Streaming converter. Runs the frontmatter and extraction callbacks in `finish()`. */
90
+ export declare class MarkdownStream {
4
91
  private _inner;
5
- constructor(options?: HtmlToMarkdownOptions);
92
+ private _carry;
93
+ private _callbacks;
94
+ constructor(options?: Partial<MdreamOptions>);
6
95
  processChunk(chunk: string): string;
96
+ processChunkBytes(chunk: Uint8Array): string;
7
97
  finish(): string;
8
98
  }
9
- declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: HtmlToMarkdownOptions): AsyncIterable<string>;
10
- export { MarkdownStream, htmlToMarkdown, streamHtmlToMarkdown };
99
+ export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
100
+ export type { CleanOptions, ExtractedElement, FrontmatterConfig, MdreamOptions, TagOverride };
package/dist/edge.mjs CHANGED
@@ -1,6 +1,5 @@
1
- import { MarkdownStream as MarkdownStream$1, __mdreamTakePanicMessage, htmlToMarkdownResult, initSync } from "../wasm/mdream_edge.js";
2
- import wasmModule from "../wasm/mdream_edge_bg.wasm";
3
- const MINIMAL_FILTER_DEFAULT = { exclude: [
1
+ import { MarkdownStream as MarkdownStream$1, __mdreamTakePanicMessage, htmlToMarkdownResult } from "../wasm-bundler/mdream_edge.js";
2
+ const MINIMAL_FILTER_EXCLUDE = [
4
3
  "form",
5
4
  "fieldset",
6
5
  "object",
@@ -13,7 +12,7 @@ const MINIMAL_FILTER_DEFAULT = { exclude: [
13
12
  "select",
14
13
  "button",
15
14
  "nav"
16
- ] };
15
+ ];
17
16
  const CLEAN_ALL = {
18
17
  urls: true,
19
18
  fragments: true,
@@ -68,7 +67,11 @@ function resolveOptions(options) {
68
67
  const tailwindDisabled = options.tailwind === false;
69
68
  if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
70
69
  else if (tailwindDisabled) delete plugins.tailwind;
71
- if (minimal) plugins.filter = options.filter || MINIMAL_FILTER_DEFAULT;
70
+ if (options.filter === false) delete plugins.filter;
71
+ else if (minimal) plugins.filter = {
72
+ ...options.filter,
73
+ exclude: [...MINIMAL_FILTER_EXCLUDE, ...options.filter?.exclude ?? []]
74
+ };
72
75
  else if (options.filter) plugins.filter = options.filter;
73
76
  const extraction = options.extraction;
74
77
  let extractionHandlers;
@@ -99,7 +102,68 @@ function resolveOptions(options) {
99
102
  frontmatterCallback
100
103
  };
101
104
  }
102
- function wasmPanicError(error, takeMessage = __mdreamTakePanicMessage) {
105
+ function deliverPluginData(data, { frontmatterCallback, extractionHandlers }) {
106
+ if (data.frontmatter && frontmatterCallback) frontmatterCallback(data.frontmatter);
107
+ if (data.extracted?.length && extractionHandlers) for (const element of data.extracted) extractionHandlers[element.selector]?.(element);
108
+ }
109
+ function hasCallbacks({ frontmatterCallback, extractionHandlers }) {
110
+ return !!frontmatterCallback || !!extractionHandlers;
111
+ }
112
+ function convertResult(convert, html, options = {}) {
113
+ const resolved = resolveOptions(options);
114
+ const result = convert(html, resolved.napiOpts);
115
+ deliverPluginData(result, resolved);
116
+ return result;
117
+ }
118
+ async function* pumpStream(stream, htmlStream, callbacks, { decodeInJs = false, mapError = (error) => error } = {}) {
119
+ const reader = htmlStream.getReader();
120
+ const decoder = decodeInJs ? new TextDecoder() : void 0;
121
+ try {
122
+ while (true) {
123
+ const { done, value } = await reader.read();
124
+ if (done) break;
125
+ let processed;
126
+ if (typeof value === "string") processed = stream.processChunk(decoder ? decoder.decode() + value : value);
127
+ else processed = decoder ? stream.processChunk(decoder.decode(value, { stream: true })) : stream.processChunkBytes(value);
128
+ if (processed) yield processed;
129
+ }
130
+ const decoderTail = decoder?.decode();
131
+ if (decoderTail) {
132
+ const processed = stream.processChunk(decoderTail);
133
+ if (processed) yield processed;
134
+ }
135
+ const final_ = stream.finish();
136
+ if (hasCallbacks(callbacks)) deliverPluginData(stream.takeData(), callbacks);
137
+ if (final_) yield final_;
138
+ } catch (error) {
139
+ throw mapError(error);
140
+ } finally {
141
+ reader.releaseLock();
142
+ }
143
+ }
144
+ function createSurrogateCarry() {
145
+ let held = "";
146
+ return {
147
+ take(chunk) {
148
+ if (held) {
149
+ chunk = held + chunk;
150
+ held = "";
151
+ }
152
+ const last = chunk.charCodeAt(chunk.length - 1);
153
+ if (last >= 55296 && last <= 56319) {
154
+ held = chunk.slice(-1);
155
+ return chunk.slice(0, -1);
156
+ }
157
+ return chunk;
158
+ },
159
+ flush() {
160
+ const value = held;
161
+ held = "";
162
+ return value;
163
+ }
164
+ };
165
+ }
166
+ function wasmPanicError(error, takeMessage) {
103
167
  if (!(error instanceof WebAssembly.RuntimeError)) return error;
104
168
  let message;
105
169
  try {
@@ -110,70 +174,77 @@ function wasmPanicError(error, takeMessage = __mdreamTakePanicMessage) {
110
174
  if (!message) return error;
111
175
  return new Error(`mdream WASM panic, please report this at https://github.com/harlan-zw/mdream/issues\n${message}`, { cause: error });
112
176
  }
113
- initSync({ module: wasmModule });
114
177
  function convert(html, napiOpts) {
115
178
  try {
116
179
  return htmlToMarkdownResult(html, napiOpts);
117
180
  } catch (error) {
118
- throw wasmPanicError(error);
181
+ throw wasmPanicError(error, __mdreamTakePanicMessage);
119
182
  }
120
183
  }
121
- function htmlToMarkdown(html, options) {
122
- if (!options) return convert(html, void 0).markdown || "";
123
- const { napiOpts, extractionHandlers, frontmatterCallback } = resolveOptions(options);
124
- const result = convert(html, napiOpts);
125
- if (result.frontmatter && frontmatterCallback) frontmatterCallback(result.frontmatter);
126
- if (result.extracted?.length && extractionHandlers) for (const element of result.extracted) extractionHandlers[element.selector]?.(element);
127
- return result.markdown || "";
184
+ function htmlToMarkdown(html, options = {}) {
185
+ return convertResult(convert, html, options).markdown || "";
128
186
  }
129
187
  var MarkdownStream = class {
130
188
  _inner;
131
- constructor(options) {
189
+ _carry = createSurrogateCarry();
190
+ _callbacks;
191
+ constructor(options = {}) {
192
+ const resolved = resolveOptions(options);
193
+ this._callbacks = resolved;
132
194
  try {
133
- this._inner = new MarkdownStream$1(options);
195
+ this._inner = new MarkdownStream$1(resolved.napiOpts);
134
196
  } catch (error) {
135
- throw wasmPanicError(error);
197
+ throw wasmPanicError(error, __mdreamTakePanicMessage);
136
198
  }
137
199
  }
138
200
  processChunk(chunk) {
139
201
  try {
140
- return this._inner.processChunk(chunk);
202
+ return this._inner.processChunk(this._carry.take(chunk));
203
+ } catch (error) {
204
+ throw wasmPanicError(error, __mdreamTakePanicMessage);
205
+ }
206
+ }
207
+ processChunkBytes(chunk) {
208
+ try {
209
+ const held = this._carry.flush();
210
+ return held ? this._inner.processChunk(held) + this._inner.processChunkBytes(chunk) : this._inner.processChunkBytes(chunk);
141
211
  } catch (error) {
142
- throw wasmPanicError(error);
212
+ throw wasmPanicError(error, __mdreamTakePanicMessage);
143
213
  }
144
214
  }
145
215
  finish() {
216
+ let markdown;
146
217
  try {
147
- return this._inner.finish();
218
+ const held = this._carry.flush();
219
+ markdown = held ? this._inner.processChunk(held) + this._inner.finish() : this._inner.finish();
148
220
  } catch (error) {
149
- throw wasmPanicError(error);
221
+ throw wasmPanicError(error, __mdreamTakePanicMessage);
150
222
  }
223
+ if (this._callbacks.frontmatterCallback || this._callbacks.extractionHandlers) deliverPluginData(this._inner.takeData(), this._callbacks);
224
+ return markdown;
151
225
  }
152
226
  };
153
- async function* streamHtmlToMarkdown(htmlStream, options) {
227
+ async function* streamHtmlToMarkdown(htmlStream, options = {}) {
154
228
  if (!htmlStream) throw new Error("Invalid HTML stream provided");
155
- const stream = new MarkdownStream$1(options ? resolveOptions(options).napiOpts : void 0);
156
- const reader = htmlStream.getReader();
157
- const decoder = new TextDecoder();
229
+ const resolved = resolveOptions(options);
230
+ const carry = createSurrogateCarry();
231
+ let stream;
158
232
  try {
159
- while (true) {
160
- const { done, value } = await reader.read();
161
- if (done) break;
162
- const chunk = typeof value === "string" ? decoder.decode() + value : decoder.decode(value, { stream: true });
163
- const processed = stream.processChunk(chunk);
164
- if (processed) yield processed;
165
- }
166
- const decoderTail = decoder.decode();
167
- if (decoderTail) {
168
- const processed = stream.processChunk(decoderTail);
169
- if (processed) yield processed;
170
- }
171
- const final_ = stream.finish();
172
- if (final_) yield final_;
233
+ stream = new MarkdownStream$1(resolved.napiOpts);
173
234
  } catch (error) {
174
- throw wasmPanicError(error);
175
- } finally {
176
- reader.releaseLock();
235
+ throw wasmPanicError(error, __mdreamTakePanicMessage);
177
236
  }
237
+ yield* pumpStream({
238
+ processChunk: (chunk) => stream.processChunk(carry.take(chunk)),
239
+ processChunkBytes: (chunk) => {
240
+ const held = carry.flush();
241
+ return held ? stream.processChunk(held) + stream.processChunkBytes(chunk) : stream.processChunkBytes(chunk);
242
+ },
243
+ finish: () => {
244
+ const held = carry.flush();
245
+ return held ? stream.processChunk(held) + stream.finish() : stream.finish();
246
+ },
247
+ takeData: () => stream.takeData()
248
+ }, htmlStream, resolved, { mapError: (error) => wasmPanicError(error, __mdreamTakePanicMessage) });
178
249
  }
179
250
  export { MarkdownStream, htmlToMarkdown, streamHtmlToMarkdown };