mdream 2.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -1
- package/dist/browser-markdown.d.mts +129 -0
- package/dist/browser-markdown.mjs +229 -0
- package/dist/browser.d.mts +33 -12
- package/dist/browser.mjs +106 -63
- package/dist/edge-markdown.d.mts +128 -0
- package/dist/edge-markdown.mjs +222 -0
- package/dist/edge.d.mts +32 -7
- package/dist/edge.mjs +100 -52
- package/dist/iife.js +1 -1
- package/dist/index.mjs +10 -2
- package/dist/wasm.mjs +48 -2
- package/napi/index.mjs +56 -56
- package/package.json +33 -19
- package/wasm/mdream_edge.js +9 -0
- package/{wasm-bundler → wasm}/mdream_edge_bg.js +9 -0
- package/wasm/mdream_edge_bg.wasm +0 -0
- package/wasm/package.json +1 -1
- package/wasm-bundler/mdream_edge.js +4 -4
- package/wasm-bundler/mdream_markdown.d.ts +45 -0
- package/wasm-bundler/mdream_markdown.js +15 -0
- package/wasm-bundler/mdream_markdown_bg.js +372 -0
- package/wasm-bundler/mdream_markdown_bg.wasm +0 -0
- package/wasm-bundler/mdream_markdown_web.d.ts +90 -0
- package/wasm-bundler/mdream_markdown_web.js +476 -0
- package/wasm-bundler/mdream_edge_bg.wasm +0 -0
- package/wasm-bundler/mdream_edge_bg.wasm.d.ts +0 -20
- package/wasm-bundler/package.json +0 -19
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { HtmlToMarkdownOptions, MdreamNapiResult } from "../napi/index.js";
|
|
2
|
+
interface CleanOptions {
|
|
3
|
+
/** Strip tracking query parameters (utm_*, fbclid, gclid, etc.) from URLs */
|
|
4
|
+
urls?: boolean;
|
|
5
|
+
/** Strip fragment-only links that don't match any heading in the output */
|
|
6
|
+
fragments?: boolean;
|
|
7
|
+
/** Strip links with meaningless hrefs (#, javascript:void(0)) → plain text */
|
|
8
|
+
emptyLinks?: boolean;
|
|
9
|
+
/** Strip links where text equals URL: [https://x.com](https://x.com) → https://x.com */
|
|
10
|
+
redundantLinks?: boolean;
|
|
11
|
+
/** Strip self-referencing heading anchors: ## [Title](#title) → ## Title */
|
|
12
|
+
selfLinkHeadings?: boolean;
|
|
13
|
+
/** Strip images with no alt text (decorative/tracking pixels) */
|
|
14
|
+
emptyImages?: boolean;
|
|
15
|
+
/** Drop links that produce no visible text: [](url) → nothing */
|
|
16
|
+
emptyLinkText?: boolean;
|
|
17
|
+
}
|
|
18
|
+
interface ExtractedElement {
|
|
19
|
+
selector: string;
|
|
20
|
+
tagName: string;
|
|
21
|
+
textContent: string;
|
|
22
|
+
attributes: Record<string, string>;
|
|
23
|
+
}
|
|
24
|
+
interface FrontmatterConfig {
|
|
25
|
+
additionalFields?: Record<string, string>;
|
|
26
|
+
metaFields?: string[];
|
|
27
|
+
/** Callback to receive structured frontmatter data after conversion */
|
|
28
|
+
onExtract?: (frontmatter: Record<string, string>) => void;
|
|
29
|
+
}
|
|
30
|
+
interface TagOverride {
|
|
31
|
+
enter?: string;
|
|
32
|
+
exit?: string;
|
|
33
|
+
spacing?: number[];
|
|
34
|
+
isInline?: boolean;
|
|
35
|
+
isSelfClosing?: boolean;
|
|
36
|
+
collapsesInnerWhiteSpace?: boolean;
|
|
37
|
+
alias?: string;
|
|
38
|
+
}
|
|
39
|
+
interface MdreamOptions {
|
|
40
|
+
/** Origin URL for resolving relative image paths and internal links. */
|
|
41
|
+
origin?: string;
|
|
42
|
+
/**
|
|
43
|
+
* Clean up the markdown output. Pass `true` for all cleanup or an object
|
|
44
|
+
* to enable specific features. Both `htmlToMarkdown` and
|
|
45
|
+
* `streamHtmlToMarkdown` apply it.
|
|
46
|
+
*/
|
|
47
|
+
clean?: boolean | CleanOptions;
|
|
48
|
+
/** Enable minimal preset (frontmatter, isolateMain, tailwind, filter). Default: false */
|
|
49
|
+
minimal?: boolean;
|
|
50
|
+
/**
|
|
51
|
+
* Extract frontmatter from HTML head.
|
|
52
|
+
* - `true`: enable with defaults
|
|
53
|
+
* - `(fm) => void`: enable and receive structured data via callback
|
|
54
|
+
* - `FrontmatterConfig`: enable with config options and optional callback
|
|
55
|
+
*/
|
|
56
|
+
frontmatter?: boolean | ((frontmatter: Record<string, string>) => void) | FrontmatterConfig;
|
|
57
|
+
/** Isolate main content area. Default when minimal: true */
|
|
58
|
+
isolateMain?: boolean;
|
|
59
|
+
/** Convert Tailwind utility classes. Default when minimal: true */
|
|
60
|
+
tailwind?: boolean;
|
|
61
|
+
/**
|
|
62
|
+
* Filter elements by CSS selector. `minimal` excludes form, nav, footer, and
|
|
63
|
+
* similar; a filter passed with `minimal` adds to those excludes. `false`
|
|
64
|
+
* turns filtering off, including the `minimal` one.
|
|
65
|
+
*/
|
|
66
|
+
filter?: false | {
|
|
67
|
+
include?: string[];
|
|
68
|
+
exclude?: string[];
|
|
69
|
+
processChildren?: boolean;
|
|
70
|
+
};
|
|
71
|
+
/** Extract elements matching CSS selectors */
|
|
72
|
+
extraction?: Record<string, (element: ExtractedElement) => void>;
|
|
73
|
+
/** Tag overrides. String values act as aliases */
|
|
74
|
+
tagOverrides?: Record<string, TagOverride | string>;
|
|
75
|
+
/**
|
|
76
|
+
* Hard-wrap prose at this many characters, breaking on word boundaries.
|
|
77
|
+
* Applied inline during conversion (zero-cost when unset). Code blocks,
|
|
78
|
+
* tables, and headings are never wrapped. `0` disables wrapping.
|
|
79
|
+
*/
|
|
80
|
+
wrapWidth?: number;
|
|
81
|
+
/**
|
|
82
|
+
* Output format. Defaults to `markdown`; use `text` for readable plain text,
|
|
83
|
+
* or `html` for allowlisted semantic HTML.
|
|
84
|
+
*/
|
|
85
|
+
format?: 'markdown' | 'text' | 'html';
|
|
86
|
+
}
|
|
87
|
+
interface ResolvedOptions {
|
|
88
|
+
napiOpts: HtmlToMarkdownOptions;
|
|
89
|
+
extractionHandlers?: Record<string, (el: ExtractedElement) => void>;
|
|
90
|
+
frontmatterCallback?: (fm: Record<string, string>) => void;
|
|
91
|
+
}
|
|
92
|
+
type FullBuildOption = Exclude<keyof MdreamOptions, 'origin' | 'format'>;
|
|
93
|
+
type MarkdownOptions = {
|
|
94
|
+
origin?: string;
|
|
95
|
+
format?: 'markdown';
|
|
96
|
+
} & Partial<Record<FullBuildOption, undefined>>;
|
|
97
|
+
interface WasmStream {
|
|
98
|
+
processChunk: (chunk: string) => string;
|
|
99
|
+
processChunkBytes: (chunk: Uint8Array) => string;
|
|
100
|
+
finish: () => string;
|
|
101
|
+
takeData: () => Omit<MdreamNapiResult, 'markdown'>;
|
|
102
|
+
free: () => void;
|
|
103
|
+
}
|
|
104
|
+
/** The bindings of one WASM instance a conversion may use. */
|
|
105
|
+
interface WasmEngine {
|
|
106
|
+
oneShot: (html: string, napiOpts: HtmlToMarkdownOptions) => MdreamNapiResult;
|
|
107
|
+
/** Only the entries that take bytes pass it; without it, bytes stream. */
|
|
108
|
+
bytes?: (html: Uint8Array, napiOpts: HtmlToMarkdownOptions) => string;
|
|
109
|
+
Stream: new (napiOpts: HtmlToMarkdownOptions) => WasmStream;
|
|
110
|
+
}
|
|
111
|
+
interface WasmBindings extends WasmEngine {
|
|
112
|
+
takePanicMessage: () => string | undefined;
|
|
113
|
+
}
|
|
114
|
+
declare class WasmMarkdownStream {
|
|
115
|
+
private readonly _bindings;
|
|
116
|
+
private readonly _callbacks;
|
|
117
|
+
private readonly _chunks;
|
|
118
|
+
protected constructor(_bindings: WasmBindings, _callbacks: ResolvedOptions);
|
|
119
|
+
processChunk(chunk: string): string;
|
|
120
|
+
processChunkBytes(chunk: Uint8Array): string;
|
|
121
|
+
finish(): string;
|
|
122
|
+
}
|
|
123
|
+
export declare function htmlToMarkdown(html: string, options?: MarkdownOptions): string;
|
|
124
|
+
export declare class MarkdownStream extends WasmMarkdownStream {
|
|
125
|
+
constructor(options?: MarkdownOptions);
|
|
126
|
+
}
|
|
127
|
+
export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: MarkdownOptions): AsyncIterable<string>;
|
|
128
|
+
export type { CleanOptions, ExtractedElement, FrontmatterConfig, MarkdownOptions as MdreamOptions, TagOverride };
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import { MarkdownStream as MarkdownStream$1, __mdreamTakePanicMessage, htmlToMarkdownResult } from "../wasm-bundler/mdream_markdown.js";
|
|
2
|
+
const FULL_BUILD_OPTIONS = {
|
|
3
|
+
clean: true,
|
|
4
|
+
minimal: true,
|
|
5
|
+
frontmatter: true,
|
|
6
|
+
isolateMain: true,
|
|
7
|
+
tailwind: true,
|
|
8
|
+
filter: true,
|
|
9
|
+
extraction: true,
|
|
10
|
+
tagOverrides: true,
|
|
11
|
+
wrapWidth: true
|
|
12
|
+
};
|
|
13
|
+
const fullBuildKeys = Object.keys(FULL_BUILD_OPTIONS);
|
|
14
|
+
function unsupportedOption(key) {
|
|
15
|
+
throw new TypeError(`The mdream Markdown build does not support the \`${key}\` option. Remove the mdream-markdown condition to use it.`);
|
|
16
|
+
}
|
|
17
|
+
function rejectUnknownOption(values, key) {
|
|
18
|
+
if (key !== "origin" && key !== "format" && !Object.hasOwn(FULL_BUILD_OPTIONS, key) && values[key] !== void 0) unsupportedOption(key);
|
|
19
|
+
}
|
|
20
|
+
function resolveMarkdownOptions(options = {}) {
|
|
21
|
+
const values = options;
|
|
22
|
+
const { format } = values;
|
|
23
|
+
if (format !== void 0 && format !== "markdown") unsupportedOption("format");
|
|
24
|
+
for (const key of fullBuildKeys) if (values[key] !== void 0) unsupportedOption(key);
|
|
25
|
+
for (const key of Object.getOwnPropertyNames(values)) rejectUnknownOption(values, key);
|
|
26
|
+
for (const key in values) if (!Object.hasOwn(values, key)) rejectUnknownOption(values, key);
|
|
27
|
+
return { napiOpts: { origin: options.origin } };
|
|
28
|
+
}
|
|
29
|
+
function deliverPluginData(data, { frontmatterCallback, extractionHandlers }) {
|
|
30
|
+
if (data.frontmatter && frontmatterCallback) frontmatterCallback(data.frontmatter);
|
|
31
|
+
if (data.extracted?.length && extractionHandlers) for (const element of data.extracted) extractionHandlers[element.selector]?.(element);
|
|
32
|
+
}
|
|
33
|
+
function hasCallbacks({ frontmatterCallback, extractionHandlers }) {
|
|
34
|
+
return !!frontmatterCallback || !!extractionHandlers;
|
|
35
|
+
}
|
|
36
|
+
const utf8 = new TextEncoder();
|
|
37
|
+
const WASM_CHUNK_BYTES = 16384;
|
|
38
|
+
const WASM_ONE_SHOT_MAX = 256;
|
|
39
|
+
function convertWasm(html, napiOpts, engine) {
|
|
40
|
+
if (html.length <= WASM_ONE_SHOT_MAX) return engine.oneShot(html, napiOpts);
|
|
41
|
+
return withWasmStream(engine, napiOpts, (stream) => {
|
|
42
|
+
let markdown = "";
|
|
43
|
+
if (html.charCodeAt(0) === 65279) {
|
|
44
|
+
markdown += stream.processChunk(String.fromCharCode(65279));
|
|
45
|
+
html = html.slice(1);
|
|
46
|
+
}
|
|
47
|
+
const bytes = new Uint8Array(WASM_CHUNK_BYTES);
|
|
48
|
+
for (let start = 0; start < html.length;) {
|
|
49
|
+
const { read, written } = utf8.encodeInto(html.slice(start, start + WASM_CHUNK_BYTES), bytes);
|
|
50
|
+
markdown += stream.processChunkBytes(bytes.subarray(0, written));
|
|
51
|
+
start += read;
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
markdown: markdown + stream.finish(),
|
|
55
|
+
...stream.takeData()
|
|
56
|
+
};
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
function withWasmStream(engine, napiOpts, use) {
|
|
60
|
+
const stream = new engine.Stream(napiOpts);
|
|
61
|
+
try {
|
|
62
|
+
return use(stream);
|
|
63
|
+
} finally {
|
|
64
|
+
try {
|
|
65
|
+
stream.free();
|
|
66
|
+
} catch {}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
function freeStream(stream) {
|
|
70
|
+
try {
|
|
71
|
+
stream.free?.();
|
|
72
|
+
} catch {}
|
|
73
|
+
}
|
|
74
|
+
async function* pumpStream(stream, htmlStream, callbacks, { mapError = (error) => error } = {}) {
|
|
75
|
+
const decoder = stream.processChunkBytes ? void 0 : new TextDecoder();
|
|
76
|
+
let reader;
|
|
77
|
+
try {
|
|
78
|
+
reader = htmlStream.getReader();
|
|
79
|
+
while (true) {
|
|
80
|
+
const { done, value } = await reader.read();
|
|
81
|
+
if (done) break;
|
|
82
|
+
let processed;
|
|
83
|
+
if (typeof value === "string") processed = stream.processChunk(decoder ? decoder.decode() + value : value);
|
|
84
|
+
else processed = decoder ? stream.processChunk(decoder.decode(value, { stream: true })) : stream.processChunkBytes(value);
|
|
85
|
+
if (processed) yield processed;
|
|
86
|
+
}
|
|
87
|
+
const decoderTail = decoder?.decode();
|
|
88
|
+
if (decoderTail) {
|
|
89
|
+
const processed = stream.processChunk(decoderTail);
|
|
90
|
+
if (processed) yield processed;
|
|
91
|
+
}
|
|
92
|
+
const final_ = stream.finish();
|
|
93
|
+
if (hasCallbacks(callbacks)) deliverPluginData(stream.takeData(), callbacks);
|
|
94
|
+
if (final_) yield final_;
|
|
95
|
+
} catch (error) {
|
|
96
|
+
throw mapError(error);
|
|
97
|
+
} finally {
|
|
98
|
+
freeStream(stream);
|
|
99
|
+
reader?.releaseLock();
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
function createSurrogateCarry() {
|
|
103
|
+
let held = "";
|
|
104
|
+
return {
|
|
105
|
+
take(chunk) {
|
|
106
|
+
if (held) {
|
|
107
|
+
chunk = held + chunk;
|
|
108
|
+
held = "";
|
|
109
|
+
}
|
|
110
|
+
const last = chunk.charCodeAt(chunk.length - 1);
|
|
111
|
+
if (last >= 55296 && last <= 56319) {
|
|
112
|
+
held = chunk.slice(-1);
|
|
113
|
+
return chunk.slice(0, -1);
|
|
114
|
+
}
|
|
115
|
+
return chunk;
|
|
116
|
+
},
|
|
117
|
+
flush() {
|
|
118
|
+
const value = held;
|
|
119
|
+
held = "";
|
|
120
|
+
return value;
|
|
121
|
+
}
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
function wasmPanicError(error, takeMessage) {
|
|
125
|
+
if (!(error instanceof WebAssembly.RuntimeError)) return error;
|
|
126
|
+
let message;
|
|
127
|
+
try {
|
|
128
|
+
message = takeMessage();
|
|
129
|
+
} catch {
|
|
130
|
+
return error;
|
|
131
|
+
}
|
|
132
|
+
if (!message) return error;
|
|
133
|
+
return new Error(`mdream WASM panic, please report this at https://github.com/harlan-zw/mdream/issues\n${message}`, { cause: error });
|
|
134
|
+
}
|
|
135
|
+
function convertWasmHtml(html, bindings, resolved) {
|
|
136
|
+
let result;
|
|
137
|
+
try {
|
|
138
|
+
result = convertWasm(html, resolved.napiOpts, bindings);
|
|
139
|
+
} catch (error) {
|
|
140
|
+
throw wasmPanicError(error, bindings.takePanicMessage);
|
|
141
|
+
}
|
|
142
|
+
deliverPluginData(result, resolved);
|
|
143
|
+
return result.markdown || "";
|
|
144
|
+
}
|
|
145
|
+
function streamChunks(inner) {
|
|
146
|
+
const carry = createSurrogateCarry();
|
|
147
|
+
return {
|
|
148
|
+
processChunk: (chunk) => inner.processChunk(carry.take(chunk)),
|
|
149
|
+
processChunkBytes: (chunk) => {
|
|
150
|
+
const held = carry.flush();
|
|
151
|
+
return held ? inner.processChunk(held) + inner.processChunkBytes(chunk) : inner.processChunkBytes(chunk);
|
|
152
|
+
},
|
|
153
|
+
finish: () => {
|
|
154
|
+
const held = carry.flush();
|
|
155
|
+
return held ? inner.processChunk(held) + inner.finish() : inner.finish();
|
|
156
|
+
},
|
|
157
|
+
takeData: () => inner.takeData(),
|
|
158
|
+
free: () => inner.free()
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
function openStream(bindings, { napiOpts }) {
|
|
162
|
+
try {
|
|
163
|
+
return streamChunks(new bindings.Stream(napiOpts));
|
|
164
|
+
} catch (error) {
|
|
165
|
+
throw wasmPanicError(error, bindings.takePanicMessage);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
var WasmMarkdownStream = class {
|
|
169
|
+
_bindings;
|
|
170
|
+
_callbacks;
|
|
171
|
+
_chunks;
|
|
172
|
+
constructor(_bindings, _callbacks) {
|
|
173
|
+
this._bindings = _bindings;
|
|
174
|
+
this._callbacks = _callbacks;
|
|
175
|
+
this._chunks = openStream(_bindings, _callbacks);
|
|
176
|
+
}
|
|
177
|
+
processChunk(chunk) {
|
|
178
|
+
try {
|
|
179
|
+
return this._chunks.processChunk(chunk);
|
|
180
|
+
} catch (error) {
|
|
181
|
+
throw wasmPanicError(error, this._bindings.takePanicMessage);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
processChunkBytes(chunk) {
|
|
185
|
+
try {
|
|
186
|
+
return this._chunks.processChunkBytes(chunk);
|
|
187
|
+
} catch (error) {
|
|
188
|
+
throw wasmPanicError(error, this._bindings.takePanicMessage);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
finish() {
|
|
192
|
+
let markdown;
|
|
193
|
+
try {
|
|
194
|
+
markdown = this._chunks.finish();
|
|
195
|
+
} catch (error) {
|
|
196
|
+
throw wasmPanicError(error, this._bindings.takePanicMessage);
|
|
197
|
+
}
|
|
198
|
+
if (this._callbacks.frontmatterCallback || this._callbacks.extractionHandlers) deliverPluginData(this._chunks.takeData(), this._callbacks);
|
|
199
|
+
return markdown;
|
|
200
|
+
}
|
|
201
|
+
};
|
|
202
|
+
function pumpWasmStream(htmlStream, bindings, resolved) {
|
|
203
|
+
return pumpStream(openStream(bindings, resolved), htmlStream, resolved, { mapError: (error) => wasmPanicError(error, bindings.takePanicMessage) });
|
|
204
|
+
}
|
|
205
|
+
const bindings = {
|
|
206
|
+
oneShot: htmlToMarkdownResult,
|
|
207
|
+
Stream: MarkdownStream$1,
|
|
208
|
+
takePanicMessage: __mdreamTakePanicMessage
|
|
209
|
+
};
|
|
210
|
+
function htmlToMarkdown(html, options = {}) {
|
|
211
|
+
return convertWasmHtml(html, bindings, resolveMarkdownOptions(options));
|
|
212
|
+
}
|
|
213
|
+
var MarkdownStream = class extends WasmMarkdownStream {
|
|
214
|
+
constructor(options = {}) {
|
|
215
|
+
super(bindings, resolveMarkdownOptions(options));
|
|
216
|
+
}
|
|
217
|
+
};
|
|
218
|
+
async function* streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
219
|
+
if (!htmlStream) throw new Error("Invalid HTML stream provided");
|
|
220
|
+
yield* pumpWasmStream(htmlStream, bindings, resolveMarkdownOptions(options));
|
|
221
|
+
}
|
|
222
|
+
export { MarkdownStream, htmlToMarkdown, streamHtmlToMarkdown };
|
package/dist/edge.d.mts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { HtmlToMarkdownOptions, MdreamNapiResult } from "../napi/index.js";
|
|
1
2
|
interface CleanOptions {
|
|
2
3
|
/** Strip tracking query parameters (utm_*, fbclid, gclid, etc.) from URLs */
|
|
3
4
|
urls?: boolean;
|
|
@@ -83,16 +84,40 @@ interface MdreamOptions {
|
|
|
83
84
|
*/
|
|
84
85
|
format?: 'markdown' | 'text' | 'html';
|
|
85
86
|
}
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
87
|
+
interface ResolvedOptions {
|
|
88
|
+
napiOpts: HtmlToMarkdownOptions;
|
|
89
|
+
extractionHandlers?: Record<string, (el: ExtractedElement) => void>;
|
|
90
|
+
frontmatterCallback?: (fm: Record<string, string>) => void;
|
|
91
|
+
}
|
|
92
|
+
interface WasmStream {
|
|
93
|
+
processChunk: (chunk: string) => string;
|
|
94
|
+
processChunkBytes: (chunk: Uint8Array) => string;
|
|
95
|
+
finish: () => string;
|
|
96
|
+
takeData: () => Omit<MdreamNapiResult, 'markdown'>;
|
|
97
|
+
free: () => void;
|
|
98
|
+
}
|
|
99
|
+
/** The bindings of one WASM instance a conversion may use. */
|
|
100
|
+
interface WasmEngine {
|
|
101
|
+
oneShot: (html: string, napiOpts: HtmlToMarkdownOptions) => MdreamNapiResult;
|
|
102
|
+
/** Only the entries that take bytes pass it; without it, bytes stream. */
|
|
103
|
+
bytes?: (html: Uint8Array, napiOpts: HtmlToMarkdownOptions) => string;
|
|
104
|
+
Stream: new (napiOpts: HtmlToMarkdownOptions) => WasmStream;
|
|
105
|
+
}
|
|
106
|
+
interface WasmBindings extends WasmEngine {
|
|
107
|
+
takePanicMessage: () => string | undefined;
|
|
108
|
+
}
|
|
109
|
+
declare class WasmMarkdownStream {
|
|
110
|
+
private readonly _bindings;
|
|
111
|
+
private readonly _callbacks;
|
|
112
|
+
private readonly _chunks;
|
|
113
|
+
protected constructor(_bindings: WasmBindings, _callbacks: ResolvedOptions);
|
|
93
114
|
processChunk(chunk: string): string;
|
|
94
115
|
processChunkBytes(chunk: Uint8Array): string;
|
|
95
116
|
finish(): string;
|
|
96
117
|
}
|
|
118
|
+
export declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string;
|
|
119
|
+
export declare class MarkdownStream extends WasmMarkdownStream {
|
|
120
|
+
constructor(options?: Partial<MdreamOptions>);
|
|
121
|
+
}
|
|
97
122
|
export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
|
|
98
123
|
export type { CleanOptions, ExtractedElement, FrontmatterConfig, MdreamOptions, TagOverride };
|
package/dist/edge.mjs
CHANGED
|
@@ -120,16 +120,49 @@ function deliverPluginData(data, { frontmatterCallback, extractionHandlers }) {
|
|
|
120
120
|
function hasCallbacks({ frontmatterCallback, extractionHandlers }) {
|
|
121
121
|
return !!frontmatterCallback || !!extractionHandlers;
|
|
122
122
|
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
return
|
|
123
|
+
const utf8 = new TextEncoder();
|
|
124
|
+
const WASM_CHUNK_BYTES = 16384;
|
|
125
|
+
const WASM_ONE_SHOT_MAX = 256;
|
|
126
|
+
function convertWasm(html, napiOpts, engine) {
|
|
127
|
+
if (html.length <= WASM_ONE_SHOT_MAX) return engine.oneShot(html, napiOpts);
|
|
128
|
+
return withWasmStream(engine, napiOpts, (stream) => {
|
|
129
|
+
let markdown = "";
|
|
130
|
+
if (html.charCodeAt(0) === 65279) {
|
|
131
|
+
markdown += stream.processChunk(String.fromCharCode(65279));
|
|
132
|
+
html = html.slice(1);
|
|
133
|
+
}
|
|
134
|
+
const bytes = new Uint8Array(WASM_CHUNK_BYTES);
|
|
135
|
+
for (let start = 0; start < html.length;) {
|
|
136
|
+
const { read, written } = utf8.encodeInto(html.slice(start, start + WASM_CHUNK_BYTES), bytes);
|
|
137
|
+
markdown += stream.processChunkBytes(bytes.subarray(0, written));
|
|
138
|
+
start += read;
|
|
139
|
+
}
|
|
140
|
+
return {
|
|
141
|
+
markdown: markdown + stream.finish(),
|
|
142
|
+
...stream.takeData()
|
|
143
|
+
};
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
function withWasmStream(engine, napiOpts, use) {
|
|
147
|
+
const stream = new engine.Stream(napiOpts);
|
|
148
|
+
try {
|
|
149
|
+
return use(stream);
|
|
150
|
+
} finally {
|
|
151
|
+
try {
|
|
152
|
+
stream.free();
|
|
153
|
+
} catch {}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
function freeStream(stream) {
|
|
157
|
+
try {
|
|
158
|
+
stream.free?.();
|
|
159
|
+
} catch {}
|
|
128
160
|
}
|
|
129
161
|
async function* pumpStream(stream, htmlStream, callbacks, { mapError = (error) => error } = {}) {
|
|
130
|
-
const reader = htmlStream.getReader();
|
|
131
162
|
const decoder = stream.processChunkBytes ? void 0 : new TextDecoder();
|
|
163
|
+
let reader;
|
|
132
164
|
try {
|
|
165
|
+
reader = htmlStream.getReader();
|
|
133
166
|
while (true) {
|
|
134
167
|
const { done, value } = await reader.read();
|
|
135
168
|
if (done) break;
|
|
@@ -149,7 +182,8 @@ async function* pumpStream(stream, htmlStream, callbacks, { mapError = (error) =
|
|
|
149
182
|
} catch (error) {
|
|
150
183
|
throw mapError(error);
|
|
151
184
|
} finally {
|
|
152
|
-
|
|
185
|
+
freeStream(stream);
|
|
186
|
+
reader?.releaseLock();
|
|
153
187
|
}
|
|
154
188
|
}
|
|
155
189
|
function createSurrogateCarry() {
|
|
@@ -185,77 +219,91 @@ function wasmPanicError(error, takeMessage) {
|
|
|
185
219
|
if (!message) return error;
|
|
186
220
|
return new Error(`mdream WASM panic, please report this at https://github.com/harlan-zw/mdream/issues\n${message}`, { cause: error });
|
|
187
221
|
}
|
|
188
|
-
function
|
|
222
|
+
function convertWasmHtml(html, bindings, resolved) {
|
|
223
|
+
let result;
|
|
189
224
|
try {
|
|
190
|
-
|
|
225
|
+
result = convertWasm(html, resolved.napiOpts, bindings);
|
|
191
226
|
} catch (error) {
|
|
192
|
-
throw wasmPanicError(error,
|
|
227
|
+
throw wasmPanicError(error, bindings.takePanicMessage);
|
|
193
228
|
}
|
|
229
|
+
deliverPluginData(result, resolved);
|
|
230
|
+
return result.markdown || "";
|
|
194
231
|
}
|
|
195
|
-
function
|
|
196
|
-
|
|
232
|
+
function streamChunks(inner) {
|
|
233
|
+
const carry = createSurrogateCarry();
|
|
234
|
+
return {
|
|
235
|
+
processChunk: (chunk) => inner.processChunk(carry.take(chunk)),
|
|
236
|
+
processChunkBytes: (chunk) => {
|
|
237
|
+
const held = carry.flush();
|
|
238
|
+
return held ? inner.processChunk(held) + inner.processChunkBytes(chunk) : inner.processChunkBytes(chunk);
|
|
239
|
+
},
|
|
240
|
+
finish: () => {
|
|
241
|
+
const held = carry.flush();
|
|
242
|
+
return held ? inner.processChunk(held) + inner.finish() : inner.finish();
|
|
243
|
+
},
|
|
244
|
+
takeData: () => inner.takeData(),
|
|
245
|
+
free: () => inner.free()
|
|
246
|
+
};
|
|
247
|
+
}
|
|
248
|
+
function openStream(bindings, { napiOpts }) {
|
|
249
|
+
try {
|
|
250
|
+
return streamChunks(new bindings.Stream(napiOpts));
|
|
251
|
+
} catch (error) {
|
|
252
|
+
throw wasmPanicError(error, bindings.takePanicMessage);
|
|
253
|
+
}
|
|
197
254
|
}
|
|
198
|
-
var
|
|
199
|
-
|
|
200
|
-
_carry = createSurrogateCarry();
|
|
255
|
+
var WasmMarkdownStream = class {
|
|
256
|
+
_bindings;
|
|
201
257
|
_callbacks;
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
this.
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
} catch (error) {
|
|
208
|
-
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
209
|
-
}
|
|
258
|
+
_chunks;
|
|
259
|
+
constructor(_bindings, _callbacks) {
|
|
260
|
+
this._bindings = _bindings;
|
|
261
|
+
this._callbacks = _callbacks;
|
|
262
|
+
this._chunks = openStream(_bindings, _callbacks);
|
|
210
263
|
}
|
|
211
264
|
processChunk(chunk) {
|
|
212
265
|
try {
|
|
213
|
-
return this.
|
|
266
|
+
return this._chunks.processChunk(chunk);
|
|
214
267
|
} catch (error) {
|
|
215
|
-
throw wasmPanicError(error,
|
|
268
|
+
throw wasmPanicError(error, this._bindings.takePanicMessage);
|
|
216
269
|
}
|
|
217
270
|
}
|
|
218
271
|
processChunkBytes(chunk) {
|
|
219
272
|
try {
|
|
220
|
-
|
|
221
|
-
return held ? this._inner.processChunk(held) + this._inner.processChunkBytes(chunk) : this._inner.processChunkBytes(chunk);
|
|
273
|
+
return this._chunks.processChunkBytes(chunk);
|
|
222
274
|
} catch (error) {
|
|
223
|
-
throw wasmPanicError(error,
|
|
275
|
+
throw wasmPanicError(error, this._bindings.takePanicMessage);
|
|
224
276
|
}
|
|
225
277
|
}
|
|
226
278
|
finish() {
|
|
227
279
|
let markdown;
|
|
228
280
|
try {
|
|
229
|
-
|
|
230
|
-
markdown = held ? this._inner.processChunk(held) + this._inner.finish() : this._inner.finish();
|
|
281
|
+
markdown = this._chunks.finish();
|
|
231
282
|
} catch (error) {
|
|
232
|
-
throw wasmPanicError(error,
|
|
283
|
+
throw wasmPanicError(error, this._bindings.takePanicMessage);
|
|
233
284
|
}
|
|
234
|
-
if (this._callbacks.frontmatterCallback || this._callbacks.extractionHandlers) deliverPluginData(this.
|
|
285
|
+
if (this._callbacks.frontmatterCallback || this._callbacks.extractionHandlers) deliverPluginData(this._chunks.takeData(), this._callbacks);
|
|
235
286
|
return markdown;
|
|
236
287
|
}
|
|
237
288
|
};
|
|
289
|
+
function pumpWasmStream(htmlStream, bindings, resolved) {
|
|
290
|
+
return pumpStream(openStream(bindings, resolved), htmlStream, resolved, { mapError: (error) => wasmPanicError(error, bindings.takePanicMessage) });
|
|
291
|
+
}
|
|
292
|
+
const bindings = {
|
|
293
|
+
oneShot: htmlToMarkdownResult,
|
|
294
|
+
Stream: MarkdownStream$1,
|
|
295
|
+
takePanicMessage: __mdreamTakePanicMessage
|
|
296
|
+
};
|
|
297
|
+
function htmlToMarkdown(html, options = {}) {
|
|
298
|
+
return convertWasmHtml(html, bindings, resolveOptions(options));
|
|
299
|
+
}
|
|
300
|
+
var MarkdownStream = class extends WasmMarkdownStream {
|
|
301
|
+
constructor(options = {}) {
|
|
302
|
+
super(bindings, resolveOptions(options));
|
|
303
|
+
}
|
|
304
|
+
};
|
|
238
305
|
async function* streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
239
306
|
if (!htmlStream) throw new Error("Invalid HTML stream provided");
|
|
240
|
-
|
|
241
|
-
const carry = createSurrogateCarry();
|
|
242
|
-
let stream;
|
|
243
|
-
try {
|
|
244
|
-
stream = new MarkdownStream$1(resolved.napiOpts);
|
|
245
|
-
} catch (error) {
|
|
246
|
-
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
247
|
-
}
|
|
248
|
-
yield* pumpStream({
|
|
249
|
-
processChunk: (chunk) => stream.processChunk(carry.take(chunk)),
|
|
250
|
-
processChunkBytes: (chunk) => {
|
|
251
|
-
const held = carry.flush();
|
|
252
|
-
return held ? stream.processChunk(held) + stream.processChunkBytes(chunk) : stream.processChunkBytes(chunk);
|
|
253
|
-
},
|
|
254
|
-
finish: () => {
|
|
255
|
-
const held = carry.flush();
|
|
256
|
-
return held ? stream.processChunk(held) + stream.finish() : stream.finish();
|
|
257
|
-
},
|
|
258
|
-
takeData: () => stream.takeData()
|
|
259
|
-
}, htmlStream, resolved, { mapError: (error) => wasmPanicError(error, __mdreamTakePanicMessage) });
|
|
307
|
+
yield* pumpWasmStream(htmlStream, bindings, resolveOptions(options));
|
|
260
308
|
}
|
|
261
309
|
export { MarkdownStream, htmlToMarkdown, streamHtmlToMarkdown };
|