mdream 1.7.2 → 2.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -51
- package/dist/browser.d.mts +100 -5
- package/dist/browser.mjs +213 -33
- package/dist/edge.d.mts +95 -4
- package/dist/edge.mjs +114 -43
- package/dist/iife.js +232 -23
- package/dist/index.d.mts +8 -4
- package/dist/index.mjs +68 -17
- package/dist/wasm.d.mts +100 -0
- package/dist/wasm.mjs +140 -0
- package/dist/worker.d.mts +8 -4
- package/dist/worker.mjs +121 -4
- package/napi/index.d.mts +20 -0
- package/napi/index.d.ts +20 -0
- package/napi/index.mjs +47 -32
- package/package.json +32 -22
- package/skills/mdream/SKILL.md +124 -0
- package/skills/mdream/references/runtimes.md +49 -0
- package/wasm/mdream_edge.d.ts +27 -0
- package/wasm/mdream_edge.js +91 -19
- package/wasm/mdream_edge_bg.wasm +0 -0
- package/wasm/mdream_edge_bg.wasm.d.ts +3 -0
- package/wasm/package.json +1 -1
- package/wasm-bundler/mdream_edge.d.ts +24 -0
- package/wasm-bundler/mdream_edge.js +6 -2
- package/wasm-bundler/mdream_edge_bg.js +91 -19
- package/wasm-bundler/mdream_edge_bg.wasm +0 -0
- package/wasm-bundler/mdream_edge_bg.wasm.d.ts +3 -0
- package/wasm-bundler/package.json +1 -1
package/dist/edge.d.mts
CHANGED
|
@@ -1,9 +1,100 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
interface CleanOptions {
|
|
2
|
+
/** Strip tracking query parameters (utm_*, fbclid, gclid, etc.) from URLs */
|
|
3
|
+
urls?: boolean;
|
|
4
|
+
/** Strip fragment-only links that don't match any heading in the output */
|
|
5
|
+
fragments?: boolean;
|
|
6
|
+
/** Strip links with meaningless hrefs (#, javascript:void(0)) → plain text */
|
|
7
|
+
emptyLinks?: boolean;
|
|
8
|
+
/** Collapse 3+ consecutive blank lines to 2 */
|
|
9
|
+
blankLines?: boolean;
|
|
10
|
+
/** Strip links where text equals URL: [https://x.com](https://x.com) → https://x.com */
|
|
11
|
+
redundantLinks?: boolean;
|
|
12
|
+
/** Strip self-referencing heading anchors: ## [Title](#title) → ## Title */
|
|
13
|
+
selfLinkHeadings?: boolean;
|
|
14
|
+
/** Strip images with no alt text (decorative/tracking pixels) */
|
|
15
|
+
emptyImages?: boolean;
|
|
16
|
+
/** Drop links that produce no visible text: [](url) → nothing */
|
|
17
|
+
emptyLinkText?: boolean;
|
|
18
|
+
}
|
|
19
|
+
interface ExtractedElement {
|
|
20
|
+
selector: string;
|
|
21
|
+
tagName: string;
|
|
22
|
+
textContent: string;
|
|
23
|
+
attributes: Record<string, string>;
|
|
24
|
+
}
|
|
25
|
+
interface FrontmatterConfig {
|
|
26
|
+
additionalFields?: Record<string, string>;
|
|
27
|
+
metaFields?: string[];
|
|
28
|
+
/** Callback to receive structured frontmatter data after conversion */
|
|
29
|
+
onExtract?: (frontmatter: Record<string, string>) => void;
|
|
30
|
+
}
|
|
31
|
+
interface TagOverride {
|
|
32
|
+
enter?: string;
|
|
33
|
+
exit?: string;
|
|
34
|
+
spacing?: number[];
|
|
35
|
+
isInline?: boolean;
|
|
36
|
+
isSelfClosing?: boolean;
|
|
37
|
+
collapsesInnerWhiteSpace?: boolean;
|
|
38
|
+
alias?: string;
|
|
39
|
+
}
|
|
40
|
+
interface MdreamOptions {
|
|
41
|
+
/** Origin URL for resolving relative image paths and internal links. */
|
|
42
|
+
origin?: string;
|
|
43
|
+
/**
|
|
44
|
+
* Clean up the markdown output. Pass `true` for all cleanup or an object
|
|
45
|
+
* to enable specific features. Both `htmlToMarkdown` and
|
|
46
|
+
* `streamHtmlToMarkdown` apply it.
|
|
47
|
+
*/
|
|
48
|
+
clean?: boolean | CleanOptions;
|
|
49
|
+
/** Enable minimal preset (frontmatter, isolateMain, tailwind, filter). Default: false */
|
|
50
|
+
minimal?: boolean;
|
|
51
|
+
/**
|
|
52
|
+
* Extract frontmatter from HTML head.
|
|
53
|
+
* - `true`: enable with defaults
|
|
54
|
+
* - `(fm) => void`: enable and receive structured data via callback
|
|
55
|
+
* - `FrontmatterConfig`: enable with config options and optional callback
|
|
56
|
+
*/
|
|
57
|
+
frontmatter?: boolean | ((frontmatter: Record<string, string>) => void) | FrontmatterConfig;
|
|
58
|
+
/** Isolate main content area. Default when minimal: true */
|
|
59
|
+
isolateMain?: boolean;
|
|
60
|
+
/** Convert Tailwind utility classes. Default when minimal: true */
|
|
61
|
+
tailwind?: boolean;
|
|
62
|
+
/**
|
|
63
|
+
* Filter elements by CSS selector. `minimal` excludes form, nav, footer, and
|
|
64
|
+
* similar; a filter passed with `minimal` adds to those excludes. `false`
|
|
65
|
+
* turns filtering off, including the `minimal` one.
|
|
66
|
+
*/
|
|
67
|
+
filter?: false | {
|
|
68
|
+
include?: string[];
|
|
69
|
+
exclude?: string[];
|
|
70
|
+
processChildren?: boolean;
|
|
71
|
+
};
|
|
72
|
+
/** Extract elements matching CSS selectors */
|
|
73
|
+
extraction?: Record<string, (element: ExtractedElement) => void>;
|
|
74
|
+
/** Tag overrides. String values act as aliases */
|
|
75
|
+
tagOverrides?: Record<string, TagOverride | string>;
|
|
76
|
+
/**
|
|
77
|
+
* Hard-wrap prose at this many characters, breaking on word boundaries.
|
|
78
|
+
* Applied inline during conversion (zero-cost when unset). Code blocks,
|
|
79
|
+
* tables, and headings are never wrapped. `0` disables wrapping.
|
|
80
|
+
*/
|
|
81
|
+
wrapWidth?: number;
|
|
82
|
+
/**
|
|
83
|
+
* Output format. Defaults to `markdown`; use `text` for readable plain text,
|
|
84
|
+
* or `html` for allowlisted semantic HTML.
|
|
85
|
+
*/
|
|
86
|
+
format?: 'markdown' | 'text' | 'html';
|
|
87
|
+
}
|
|
88
|
+
export declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string;
|
|
89
|
+
/** Streaming converter. Runs the frontmatter and extraction callbacks in `finish()`. */
|
|
3
90
|
export declare class MarkdownStream {
|
|
4
91
|
private _inner;
|
|
5
|
-
|
|
92
|
+
private _carry;
|
|
93
|
+
private _callbacks;
|
|
94
|
+
constructor(options?: Partial<MdreamOptions>);
|
|
6
95
|
processChunk(chunk: string): string;
|
|
96
|
+
processChunkBytes(chunk: Uint8Array): string;
|
|
7
97
|
finish(): string;
|
|
8
98
|
}
|
|
9
|
-
export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?:
|
|
99
|
+
export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
|
|
100
|
+
export type { CleanOptions, ExtractedElement, FrontmatterConfig, MdreamOptions, TagOverride };
|
package/dist/edge.mjs
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import { MarkdownStream as MarkdownStream$1, __mdreamTakePanicMessage, htmlToMarkdownResult
|
|
2
|
-
|
|
3
|
-
const MINIMAL_FILTER_DEFAULT = { exclude: [
|
|
1
|
+
import { MarkdownStream as MarkdownStream$1, __mdreamTakePanicMessage, htmlToMarkdownResult } from "../wasm-bundler/mdream_edge.js";
|
|
2
|
+
const MINIMAL_FILTER_EXCLUDE = [
|
|
4
3
|
"form",
|
|
5
4
|
"fieldset",
|
|
6
5
|
"object",
|
|
@@ -13,7 +12,7 @@ const MINIMAL_FILTER_DEFAULT = { exclude: [
|
|
|
13
12
|
"select",
|
|
14
13
|
"button",
|
|
15
14
|
"nav"
|
|
16
|
-
]
|
|
15
|
+
];
|
|
17
16
|
const CLEAN_ALL = {
|
|
18
17
|
urls: true,
|
|
19
18
|
fragments: true,
|
|
@@ -68,7 +67,11 @@ function resolveOptions(options) {
|
|
|
68
67
|
const tailwindDisabled = options.tailwind === false;
|
|
69
68
|
if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
|
|
70
69
|
else if (tailwindDisabled) delete plugins.tailwind;
|
|
71
|
-
if (
|
|
70
|
+
if (options.filter === false) delete plugins.filter;
|
|
71
|
+
else if (minimal) plugins.filter = {
|
|
72
|
+
...options.filter,
|
|
73
|
+
exclude: [...MINIMAL_FILTER_EXCLUDE, ...options.filter?.exclude ?? []]
|
|
74
|
+
};
|
|
72
75
|
else if (options.filter) plugins.filter = options.filter;
|
|
73
76
|
const extraction = options.extraction;
|
|
74
77
|
let extractionHandlers;
|
|
@@ -99,7 +102,68 @@ function resolveOptions(options) {
|
|
|
99
102
|
frontmatterCallback
|
|
100
103
|
};
|
|
101
104
|
}
|
|
102
|
-
function
|
|
105
|
+
function deliverPluginData(data, { frontmatterCallback, extractionHandlers }) {
|
|
106
|
+
if (data.frontmatter && frontmatterCallback) frontmatterCallback(data.frontmatter);
|
|
107
|
+
if (data.extracted?.length && extractionHandlers) for (const element of data.extracted) extractionHandlers[element.selector]?.(element);
|
|
108
|
+
}
|
|
109
|
+
function hasCallbacks({ frontmatterCallback, extractionHandlers }) {
|
|
110
|
+
return !!frontmatterCallback || !!extractionHandlers;
|
|
111
|
+
}
|
|
112
|
+
function convertResult(convert, html, options = {}) {
|
|
113
|
+
const resolved = resolveOptions(options);
|
|
114
|
+
const result = convert(html, resolved.napiOpts);
|
|
115
|
+
deliverPluginData(result, resolved);
|
|
116
|
+
return result;
|
|
117
|
+
}
|
|
118
|
+
async function* pumpStream(stream, htmlStream, callbacks, { decodeInJs = false, mapError = (error) => error } = {}) {
|
|
119
|
+
const reader = htmlStream.getReader();
|
|
120
|
+
const decoder = decodeInJs ? new TextDecoder() : void 0;
|
|
121
|
+
try {
|
|
122
|
+
while (true) {
|
|
123
|
+
const { done, value } = await reader.read();
|
|
124
|
+
if (done) break;
|
|
125
|
+
let processed;
|
|
126
|
+
if (typeof value === "string") processed = stream.processChunk(decoder ? decoder.decode() + value : value);
|
|
127
|
+
else processed = decoder ? stream.processChunk(decoder.decode(value, { stream: true })) : stream.processChunkBytes(value);
|
|
128
|
+
if (processed) yield processed;
|
|
129
|
+
}
|
|
130
|
+
const decoderTail = decoder?.decode();
|
|
131
|
+
if (decoderTail) {
|
|
132
|
+
const processed = stream.processChunk(decoderTail);
|
|
133
|
+
if (processed) yield processed;
|
|
134
|
+
}
|
|
135
|
+
const final_ = stream.finish();
|
|
136
|
+
if (hasCallbacks(callbacks)) deliverPluginData(stream.takeData(), callbacks);
|
|
137
|
+
if (final_) yield final_;
|
|
138
|
+
} catch (error) {
|
|
139
|
+
throw mapError(error);
|
|
140
|
+
} finally {
|
|
141
|
+
reader.releaseLock();
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
function createSurrogateCarry() {
|
|
145
|
+
let held = "";
|
|
146
|
+
return {
|
|
147
|
+
take(chunk) {
|
|
148
|
+
if (held) {
|
|
149
|
+
chunk = held + chunk;
|
|
150
|
+
held = "";
|
|
151
|
+
}
|
|
152
|
+
const last = chunk.charCodeAt(chunk.length - 1);
|
|
153
|
+
if (last >= 55296 && last <= 56319) {
|
|
154
|
+
held = chunk.slice(-1);
|
|
155
|
+
return chunk.slice(0, -1);
|
|
156
|
+
}
|
|
157
|
+
return chunk;
|
|
158
|
+
},
|
|
159
|
+
flush() {
|
|
160
|
+
const value = held;
|
|
161
|
+
held = "";
|
|
162
|
+
return value;
|
|
163
|
+
}
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
function wasmPanicError(error, takeMessage) {
|
|
103
167
|
if (!(error instanceof WebAssembly.RuntimeError)) return error;
|
|
104
168
|
let message;
|
|
105
169
|
try {
|
|
@@ -110,70 +174,77 @@ function wasmPanicError(error, takeMessage = __mdreamTakePanicMessage) {
|
|
|
110
174
|
if (!message) return error;
|
|
111
175
|
return new Error(`mdream WASM panic, please report this at https://github.com/harlan-zw/mdream/issues\n${message}`, { cause: error });
|
|
112
176
|
}
|
|
113
|
-
initSync({ module: wasmModule });
|
|
114
177
|
function convert(html, napiOpts) {
|
|
115
178
|
try {
|
|
116
179
|
return htmlToMarkdownResult(html, napiOpts);
|
|
117
180
|
} catch (error) {
|
|
118
|
-
throw wasmPanicError(error);
|
|
181
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
119
182
|
}
|
|
120
183
|
}
|
|
121
|
-
function htmlToMarkdown(html, options) {
|
|
122
|
-
|
|
123
|
-
const { napiOpts, extractionHandlers, frontmatterCallback } = resolveOptions(options);
|
|
124
|
-
const result = convert(html, napiOpts);
|
|
125
|
-
if (result.frontmatter && frontmatterCallback) frontmatterCallback(result.frontmatter);
|
|
126
|
-
if (result.extracted?.length && extractionHandlers) for (const element of result.extracted) extractionHandlers[element.selector]?.(element);
|
|
127
|
-
return result.markdown || "";
|
|
184
|
+
function htmlToMarkdown(html, options = {}) {
|
|
185
|
+
return convertResult(convert, html, options).markdown || "";
|
|
128
186
|
}
|
|
129
187
|
var MarkdownStream = class {
|
|
130
188
|
_inner;
|
|
131
|
-
|
|
189
|
+
_carry = createSurrogateCarry();
|
|
190
|
+
_callbacks;
|
|
191
|
+
constructor(options = {}) {
|
|
192
|
+
const resolved = resolveOptions(options);
|
|
193
|
+
this._callbacks = resolved;
|
|
132
194
|
try {
|
|
133
|
-
this._inner = new MarkdownStream$1(
|
|
195
|
+
this._inner = new MarkdownStream$1(resolved.napiOpts);
|
|
134
196
|
} catch (error) {
|
|
135
|
-
throw wasmPanicError(error);
|
|
197
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
136
198
|
}
|
|
137
199
|
}
|
|
138
200
|
processChunk(chunk) {
|
|
139
201
|
try {
|
|
140
|
-
return this._inner.processChunk(chunk);
|
|
202
|
+
return this._inner.processChunk(this._carry.take(chunk));
|
|
203
|
+
} catch (error) {
|
|
204
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
processChunkBytes(chunk) {
|
|
208
|
+
try {
|
|
209
|
+
const held = this._carry.flush();
|
|
210
|
+
return held ? this._inner.processChunk(held) + this._inner.processChunkBytes(chunk) : this._inner.processChunkBytes(chunk);
|
|
141
211
|
} catch (error) {
|
|
142
|
-
throw wasmPanicError(error);
|
|
212
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
143
213
|
}
|
|
144
214
|
}
|
|
145
215
|
finish() {
|
|
216
|
+
let markdown;
|
|
146
217
|
try {
|
|
147
|
-
|
|
218
|
+
const held = this._carry.flush();
|
|
219
|
+
markdown = held ? this._inner.processChunk(held) + this._inner.finish() : this._inner.finish();
|
|
148
220
|
} catch (error) {
|
|
149
|
-
throw wasmPanicError(error);
|
|
221
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
150
222
|
}
|
|
223
|
+
if (this._callbacks.frontmatterCallback || this._callbacks.extractionHandlers) deliverPluginData(this._inner.takeData(), this._callbacks);
|
|
224
|
+
return markdown;
|
|
151
225
|
}
|
|
152
226
|
};
|
|
153
|
-
async function* streamHtmlToMarkdown(htmlStream, options) {
|
|
227
|
+
async function* streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
154
228
|
if (!htmlStream) throw new Error("Invalid HTML stream provided");
|
|
155
|
-
const
|
|
156
|
-
const
|
|
157
|
-
|
|
229
|
+
const resolved = resolveOptions(options);
|
|
230
|
+
const carry = createSurrogateCarry();
|
|
231
|
+
let stream;
|
|
158
232
|
try {
|
|
159
|
-
|
|
160
|
-
const { done, value } = await reader.read();
|
|
161
|
-
if (done) break;
|
|
162
|
-
const chunk = typeof value === "string" ? decoder.decode() + value : decoder.decode(value, { stream: true });
|
|
163
|
-
const processed = stream.processChunk(chunk);
|
|
164
|
-
if (processed) yield processed;
|
|
165
|
-
}
|
|
166
|
-
const decoderTail = decoder.decode();
|
|
167
|
-
if (decoderTail) {
|
|
168
|
-
const processed = stream.processChunk(decoderTail);
|
|
169
|
-
if (processed) yield processed;
|
|
170
|
-
}
|
|
171
|
-
const final_ = stream.finish();
|
|
172
|
-
if (final_) yield final_;
|
|
233
|
+
stream = new MarkdownStream$1(resolved.napiOpts);
|
|
173
234
|
} catch (error) {
|
|
174
|
-
throw wasmPanicError(error);
|
|
175
|
-
} finally {
|
|
176
|
-
reader.releaseLock();
|
|
235
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
177
236
|
}
|
|
237
|
+
yield* pumpStream({
|
|
238
|
+
processChunk: (chunk) => stream.processChunk(carry.take(chunk)),
|
|
239
|
+
processChunkBytes: (chunk) => {
|
|
240
|
+
const held = carry.flush();
|
|
241
|
+
return held ? stream.processChunk(held) + stream.processChunkBytes(chunk) : stream.processChunkBytes(chunk);
|
|
242
|
+
},
|
|
243
|
+
finish: () => {
|
|
244
|
+
const held = carry.flush();
|
|
245
|
+
return held ? stream.processChunk(held) + stream.finish() : stream.finish();
|
|
246
|
+
},
|
|
247
|
+
takeData: () => stream.takeData()
|
|
248
|
+
}, htmlStream, resolved, { mapError: (error) => wasmPanicError(error, __mdreamTakePanicMessage) });
|
|
178
249
|
}
|
|
179
250
|
export { MarkdownStream, htmlToMarkdown, streamHtmlToMarkdown };
|