mdream 2.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -1
- package/dist/browser-markdown.d.mts +129 -0
- package/dist/browser-markdown.mjs +229 -0
- package/dist/browser.d.mts +33 -12
- package/dist/browser.mjs +106 -63
- package/dist/edge-markdown.d.mts +128 -0
- package/dist/edge-markdown.mjs +222 -0
- package/dist/edge.d.mts +32 -7
- package/dist/edge.mjs +100 -52
- package/dist/iife.js +1 -1
- package/dist/index.mjs +10 -2
- package/dist/wasm.mjs +48 -2
- package/napi/index.mjs +56 -56
- package/package.json +33 -19
- package/wasm/mdream_edge.js +9 -0
- package/{wasm-bundler → wasm}/mdream_edge_bg.js +9 -0
- package/wasm/mdream_edge_bg.wasm +0 -0
- package/wasm/package.json +1 -1
- package/wasm-bundler/mdream_edge.js +4 -4
- package/wasm-bundler/mdream_markdown.d.ts +45 -0
- package/wasm-bundler/mdream_markdown.js +15 -0
- package/wasm-bundler/mdream_markdown_bg.js +372 -0
- package/wasm-bundler/mdream_markdown_bg.wasm +0 -0
- package/wasm-bundler/mdream_markdown_web.d.ts +90 -0
- package/wasm-bundler/mdream_markdown_web.js +476 -0
- package/wasm-bundler/mdream_edge_bg.wasm +0 -0
- package/wasm-bundler/mdream_edge_bg.wasm.d.ts +0 -20
- package/wasm-bundler/package.json +0 -19
package/README.md
CHANGED
|
@@ -96,6 +96,7 @@ externals: ['mdream']
|
|
|
96
96
|
- [CLI Usage](#cli-usage)
|
|
97
97
|
- [Browser and Edge Usage](#browser-and-edge-usage)
|
|
98
98
|
- [Edge / Cloudflare Workers](#edge--cloudflare-workers)
|
|
99
|
+
- [Smaller Markdown WASM build](#smaller-markdown-wasm-build)
|
|
99
100
|
- [Browser CDN (IIFE)](#browser-cdn-iife)
|
|
100
101
|
- [llms.txt Generation](#llmstxt-generation)
|
|
101
102
|
- [Related Packages](#related-packages)
|
|
@@ -1140,7 +1141,7 @@ If you pass an unknown flag, a flag without a value, or an unknown preset, the C
|
|
|
1140
1141
|
|
|
1141
1142
|
## Browser and Edge Usage
|
|
1142
1143
|
|
|
1143
|
-
|
|
1144
|
+
By default, every entry takes the same `MdreamOptions` and produces the same Markdown string.
|
|
1144
1145
|
In Node and edge runtimes, `htmlToMarkdown` from `mdream` is synchronous.
|
|
1145
1146
|
In a browser bundle, import from `mdream/browser`.
|
|
1146
1147
|
Its `htmlToMarkdown` returns `Promise<string>`, because the WASM binary loads first.
|
|
@@ -1207,6 +1208,30 @@ await init({ module_or_path: wasmModule })
|
|
|
1207
1208
|
const markdown = htmlToMarkdown('<h1>Hello</h1>', { minimal: true })
|
|
1208
1209
|
```
|
|
1209
1210
|
|
|
1211
|
+
### Smaller Markdown WASM build
|
|
1212
|
+
|
|
1213
|
+
Enable the `mdream-markdown` export condition to bundle a smaller WASM engine.
|
|
1214
|
+
Keep importing from `mdream/browser` in browsers or `mdream` on the edge.
|
|
1215
|
+
One-shot conversion, `MarkdownStream`, and `streamHtmlToMarkdown` all work.
|
|
1216
|
+
|
|
1217
|
+
In Vite, add the condition to your client config.
|
|
1218
|
+
|
|
1219
|
+
```ts
|
|
1220
|
+
import { defaultClientConditions, defineConfig } from 'vite'
|
|
1221
|
+
|
|
1222
|
+
export default defineConfig({
|
|
1223
|
+
resolve: { conditions: [...defaultClientConditions, 'mdream-markdown'] },
|
|
1224
|
+
})
|
|
1225
|
+
```
|
|
1226
|
+
|
|
1227
|
+
In Wrangler, set `WRANGLER_BUILD_CONDITIONS=workerd,worker,browser,mdream-markdown` in the environment that builds the Worker.
|
|
1228
|
+
For matching TypeScript types, add `mdream-markdown` to `compilerOptions.customConditions`.
|
|
1229
|
+
For an edge import, also include your runtime's condition, such as `workerd`.
|
|
1230
|
+
|
|
1231
|
+
This build accepts `origin` and `format: 'markdown'`.
|
|
1232
|
+
It throws on other options, including plugins, cleanup, and `wrapWidth`.
|
|
1233
|
+
Remove the condition if your app needs those options or text or safe HTML output.
|
|
1234
|
+
|
|
1210
1235
|
### Browser CDN (IIFE)
|
|
1211
1236
|
|
|
1212
1237
|
Use mdream from a CDN without a build step.
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { HtmlToMarkdownOptions, MdreamNapiResult } from "../napi/index.js";
|
|
2
|
+
interface CleanOptions {
|
|
3
|
+
/** Strip tracking query parameters (utm_*, fbclid, gclid, etc.) from URLs */
|
|
4
|
+
urls?: boolean;
|
|
5
|
+
/** Strip fragment-only links that don't match any heading in the output */
|
|
6
|
+
fragments?: boolean;
|
|
7
|
+
/** Strip links with meaningless hrefs (#, javascript:void(0)) → plain text */
|
|
8
|
+
emptyLinks?: boolean;
|
|
9
|
+
/** Strip links where text equals URL: [https://x.com](https://x.com) → https://x.com */
|
|
10
|
+
redundantLinks?: boolean;
|
|
11
|
+
/** Strip self-referencing heading anchors: ## [Title](#title) → ## Title */
|
|
12
|
+
selfLinkHeadings?: boolean;
|
|
13
|
+
/** Strip images with no alt text (decorative/tracking pixels) */
|
|
14
|
+
emptyImages?: boolean;
|
|
15
|
+
/** Drop links that produce no visible text: [](url) → nothing */
|
|
16
|
+
emptyLinkText?: boolean;
|
|
17
|
+
}
|
|
18
|
+
interface ExtractedElement {
|
|
19
|
+
selector: string;
|
|
20
|
+
tagName: string;
|
|
21
|
+
textContent: string;
|
|
22
|
+
attributes: Record<string, string>;
|
|
23
|
+
}
|
|
24
|
+
interface FrontmatterConfig {
|
|
25
|
+
additionalFields?: Record<string, string>;
|
|
26
|
+
metaFields?: string[];
|
|
27
|
+
/** Callback to receive structured frontmatter data after conversion */
|
|
28
|
+
onExtract?: (frontmatter: Record<string, string>) => void;
|
|
29
|
+
}
|
|
30
|
+
interface TagOverride {
|
|
31
|
+
enter?: string;
|
|
32
|
+
exit?: string;
|
|
33
|
+
spacing?: number[];
|
|
34
|
+
isInline?: boolean;
|
|
35
|
+
isSelfClosing?: boolean;
|
|
36
|
+
collapsesInnerWhiteSpace?: boolean;
|
|
37
|
+
alias?: string;
|
|
38
|
+
}
|
|
39
|
+
interface MdreamOptions {
|
|
40
|
+
/** Origin URL for resolving relative image paths and internal links. */
|
|
41
|
+
origin?: string;
|
|
42
|
+
/**
|
|
43
|
+
* Clean up the markdown output. Pass `true` for all cleanup or an object
|
|
44
|
+
* to enable specific features. Both `htmlToMarkdown` and
|
|
45
|
+
* `streamHtmlToMarkdown` apply it.
|
|
46
|
+
*/
|
|
47
|
+
clean?: boolean | CleanOptions;
|
|
48
|
+
/** Enable minimal preset (frontmatter, isolateMain, tailwind, filter). Default: false */
|
|
49
|
+
minimal?: boolean;
|
|
50
|
+
/**
|
|
51
|
+
* Extract frontmatter from HTML head.
|
|
52
|
+
* - `true`: enable with defaults
|
|
53
|
+
* - `(fm) => void`: enable and receive structured data via callback
|
|
54
|
+
* - `FrontmatterConfig`: enable with config options and optional callback
|
|
55
|
+
*/
|
|
56
|
+
frontmatter?: boolean | ((frontmatter: Record<string, string>) => void) | FrontmatterConfig;
|
|
57
|
+
/** Isolate main content area. Default when minimal: true */
|
|
58
|
+
isolateMain?: boolean;
|
|
59
|
+
/** Convert Tailwind utility classes. Default when minimal: true */
|
|
60
|
+
tailwind?: boolean;
|
|
61
|
+
/**
|
|
62
|
+
* Filter elements by CSS selector. `minimal` excludes form, nav, footer, and
|
|
63
|
+
* similar; a filter passed with `minimal` adds to those excludes. `false`
|
|
64
|
+
* turns filtering off, including the `minimal` one.
|
|
65
|
+
*/
|
|
66
|
+
filter?: false | {
|
|
67
|
+
include?: string[];
|
|
68
|
+
exclude?: string[];
|
|
69
|
+
processChildren?: boolean;
|
|
70
|
+
};
|
|
71
|
+
/** Extract elements matching CSS selectors */
|
|
72
|
+
extraction?: Record<string, (element: ExtractedElement) => void>;
|
|
73
|
+
/** Tag overrides. String values act as aliases */
|
|
74
|
+
tagOverrides?: Record<string, TagOverride | string>;
|
|
75
|
+
/**
|
|
76
|
+
* Hard-wrap prose at this many characters, breaking on word boundaries.
|
|
77
|
+
* Applied inline during conversion (zero-cost when unset). Code blocks,
|
|
78
|
+
* tables, and headings are never wrapped. `0` disables wrapping.
|
|
79
|
+
*/
|
|
80
|
+
wrapWidth?: number;
|
|
81
|
+
/**
|
|
82
|
+
* Output format. Defaults to `markdown`; use `text` for readable plain text,
|
|
83
|
+
* or `html` for allowlisted semantic HTML.
|
|
84
|
+
*/
|
|
85
|
+
format?: 'markdown' | 'text' | 'html';
|
|
86
|
+
}
|
|
87
|
+
interface ResolvedOptions {
|
|
88
|
+
napiOpts: HtmlToMarkdownOptions;
|
|
89
|
+
extractionHandlers?: Record<string, (el: ExtractedElement) => void>;
|
|
90
|
+
frontmatterCallback?: (fm: Record<string, string>) => void;
|
|
91
|
+
}
|
|
92
|
+
type FullBuildOption = Exclude<keyof MdreamOptions, 'origin' | 'format'>;
|
|
93
|
+
type MarkdownOptions = {
|
|
94
|
+
origin?: string;
|
|
95
|
+
format?: 'markdown';
|
|
96
|
+
} & Partial<Record<FullBuildOption, undefined>>;
|
|
97
|
+
interface WasmStream {
|
|
98
|
+
processChunk: (chunk: string) => string;
|
|
99
|
+
processChunkBytes: (chunk: Uint8Array) => string;
|
|
100
|
+
finish: () => string;
|
|
101
|
+
takeData: () => Omit<MdreamNapiResult, 'markdown'>;
|
|
102
|
+
free: () => void;
|
|
103
|
+
}
|
|
104
|
+
/** The bindings of one WASM instance a conversion may use. */
|
|
105
|
+
interface WasmEngine {
|
|
106
|
+
oneShot: (html: string, napiOpts: HtmlToMarkdownOptions) => MdreamNapiResult;
|
|
107
|
+
/** Only the entries that take bytes pass it; without it, bytes stream. */
|
|
108
|
+
bytes?: (html: Uint8Array, napiOpts: HtmlToMarkdownOptions) => string;
|
|
109
|
+
Stream: new (napiOpts: HtmlToMarkdownOptions) => WasmStream;
|
|
110
|
+
}
|
|
111
|
+
interface WasmBindings extends WasmEngine {
|
|
112
|
+
takePanicMessage: () => string | undefined;
|
|
113
|
+
}
|
|
114
|
+
declare class WasmMarkdownStream {
|
|
115
|
+
private readonly _bindings;
|
|
116
|
+
private readonly _callbacks;
|
|
117
|
+
private readonly _chunks;
|
|
118
|
+
protected constructor(_bindings: WasmBindings, _callbacks: ResolvedOptions);
|
|
119
|
+
processChunk(chunk: string): string;
|
|
120
|
+
processChunkBytes(chunk: Uint8Array): string;
|
|
121
|
+
finish(): string;
|
|
122
|
+
}
|
|
123
|
+
export declare function htmlToMarkdown(html: string, options?: MarkdownOptions): Promise<string>;
|
|
124
|
+
export declare function createMarkdownStream(options?: MarkdownOptions): Promise<MarkdownStream>;
|
|
125
|
+
export declare class MarkdownStream extends WasmMarkdownStream {
|
|
126
|
+
constructor(options?: MarkdownOptions);
|
|
127
|
+
}
|
|
128
|
+
export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: MarkdownOptions): AsyncIterable<string>;
|
|
129
|
+
export type { CleanOptions, ExtractedElement, FrontmatterConfig, MarkdownOptions as MdreamOptions, TagOverride };
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
import init, { MarkdownStream as MarkdownStream$1, __mdreamTakePanicMessage, htmlToMarkdownResult } from "../wasm-bundler/mdream_markdown_web.js";
|
|
2
|
+
const FULL_BUILD_OPTIONS = {
|
|
3
|
+
clean: true,
|
|
4
|
+
minimal: true,
|
|
5
|
+
frontmatter: true,
|
|
6
|
+
isolateMain: true,
|
|
7
|
+
tailwind: true,
|
|
8
|
+
filter: true,
|
|
9
|
+
extraction: true,
|
|
10
|
+
tagOverrides: true,
|
|
11
|
+
wrapWidth: true
|
|
12
|
+
};
|
|
13
|
+
const fullBuildKeys = Object.keys(FULL_BUILD_OPTIONS);
|
|
14
|
+
function unsupportedOption(key) {
|
|
15
|
+
throw new TypeError(`The mdream Markdown build does not support the \`${key}\` option. Remove the mdream-markdown condition to use it.`);
|
|
16
|
+
}
|
|
17
|
+
function rejectUnknownOption(values, key) {
|
|
18
|
+
if (key !== "origin" && key !== "format" && !Object.hasOwn(FULL_BUILD_OPTIONS, key) && values[key] !== void 0) unsupportedOption(key);
|
|
19
|
+
}
|
|
20
|
+
function resolveMarkdownOptions(options = {}) {
|
|
21
|
+
const values = options;
|
|
22
|
+
const { format } = values;
|
|
23
|
+
if (format !== void 0 && format !== "markdown") unsupportedOption("format");
|
|
24
|
+
for (const key of fullBuildKeys) if (values[key] !== void 0) unsupportedOption(key);
|
|
25
|
+
for (const key of Object.getOwnPropertyNames(values)) rejectUnknownOption(values, key);
|
|
26
|
+
for (const key in values) if (!Object.hasOwn(values, key)) rejectUnknownOption(values, key);
|
|
27
|
+
return { napiOpts: { origin: options.origin } };
|
|
28
|
+
}
|
|
29
|
+
function deliverPluginData(data, { frontmatterCallback, extractionHandlers }) {
|
|
30
|
+
if (data.frontmatter && frontmatterCallback) frontmatterCallback(data.frontmatter);
|
|
31
|
+
if (data.extracted?.length && extractionHandlers) for (const element of data.extracted) extractionHandlers[element.selector]?.(element);
|
|
32
|
+
}
|
|
33
|
+
function hasCallbacks({ frontmatterCallback, extractionHandlers }) {
|
|
34
|
+
return !!frontmatterCallback || !!extractionHandlers;
|
|
35
|
+
}
|
|
36
|
+
const utf8 = new TextEncoder();
|
|
37
|
+
const WASM_CHUNK_BYTES = 16384;
|
|
38
|
+
const WASM_ONE_SHOT_MAX = 256;
|
|
39
|
+
function convertWasm(html, napiOpts, engine) {
|
|
40
|
+
if (html.length <= WASM_ONE_SHOT_MAX) return engine.oneShot(html, napiOpts);
|
|
41
|
+
return withWasmStream(engine, napiOpts, (stream) => {
|
|
42
|
+
let markdown = "";
|
|
43
|
+
if (html.charCodeAt(0) === 65279) {
|
|
44
|
+
markdown += stream.processChunk(String.fromCharCode(65279));
|
|
45
|
+
html = html.slice(1);
|
|
46
|
+
}
|
|
47
|
+
const bytes = new Uint8Array(WASM_CHUNK_BYTES);
|
|
48
|
+
for (let start = 0; start < html.length;) {
|
|
49
|
+
const { read, written } = utf8.encodeInto(html.slice(start, start + WASM_CHUNK_BYTES), bytes);
|
|
50
|
+
markdown += stream.processChunkBytes(bytes.subarray(0, written));
|
|
51
|
+
start += read;
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
markdown: markdown + stream.finish(),
|
|
55
|
+
...stream.takeData()
|
|
56
|
+
};
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
function withWasmStream(engine, napiOpts, use) {
|
|
60
|
+
const stream = new engine.Stream(napiOpts);
|
|
61
|
+
try {
|
|
62
|
+
return use(stream);
|
|
63
|
+
} finally {
|
|
64
|
+
try {
|
|
65
|
+
stream.free();
|
|
66
|
+
} catch {}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
function freeStream(stream) {
|
|
70
|
+
try {
|
|
71
|
+
stream.free?.();
|
|
72
|
+
} catch {}
|
|
73
|
+
}
|
|
74
|
+
async function* pumpStream(stream, htmlStream, callbacks, { mapError = (error) => error } = {}) {
|
|
75
|
+
const decoder = stream.processChunkBytes ? void 0 : new TextDecoder();
|
|
76
|
+
let reader;
|
|
77
|
+
try {
|
|
78
|
+
reader = htmlStream.getReader();
|
|
79
|
+
while (true) {
|
|
80
|
+
const { done, value } = await reader.read();
|
|
81
|
+
if (done) break;
|
|
82
|
+
let processed;
|
|
83
|
+
if (typeof value === "string") processed = stream.processChunk(decoder ? decoder.decode() + value : value);
|
|
84
|
+
else processed = decoder ? stream.processChunk(decoder.decode(value, { stream: true })) : stream.processChunkBytes(value);
|
|
85
|
+
if (processed) yield processed;
|
|
86
|
+
}
|
|
87
|
+
const decoderTail = decoder?.decode();
|
|
88
|
+
if (decoderTail) {
|
|
89
|
+
const processed = stream.processChunk(decoderTail);
|
|
90
|
+
if (processed) yield processed;
|
|
91
|
+
}
|
|
92
|
+
const final_ = stream.finish();
|
|
93
|
+
if (hasCallbacks(callbacks)) deliverPluginData(stream.takeData(), callbacks);
|
|
94
|
+
if (final_) yield final_;
|
|
95
|
+
} catch (error) {
|
|
96
|
+
throw mapError(error);
|
|
97
|
+
} finally {
|
|
98
|
+
freeStream(stream);
|
|
99
|
+
reader?.releaseLock();
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
function createSurrogateCarry() {
|
|
103
|
+
let held = "";
|
|
104
|
+
return {
|
|
105
|
+
take(chunk) {
|
|
106
|
+
if (held) {
|
|
107
|
+
chunk = held + chunk;
|
|
108
|
+
held = "";
|
|
109
|
+
}
|
|
110
|
+
const last = chunk.charCodeAt(chunk.length - 1);
|
|
111
|
+
if (last >= 55296 && last <= 56319) {
|
|
112
|
+
held = chunk.slice(-1);
|
|
113
|
+
return chunk.slice(0, -1);
|
|
114
|
+
}
|
|
115
|
+
return chunk;
|
|
116
|
+
},
|
|
117
|
+
flush() {
|
|
118
|
+
const value = held;
|
|
119
|
+
held = "";
|
|
120
|
+
return value;
|
|
121
|
+
}
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
function wasmPanicError(error, takeMessage) {
|
|
125
|
+
if (!(error instanceof WebAssembly.RuntimeError)) return error;
|
|
126
|
+
let message;
|
|
127
|
+
try {
|
|
128
|
+
message = takeMessage();
|
|
129
|
+
} catch {
|
|
130
|
+
return error;
|
|
131
|
+
}
|
|
132
|
+
if (!message) return error;
|
|
133
|
+
return new Error(`mdream WASM panic, please report this at https://github.com/harlan-zw/mdream/issues\n${message}`, { cause: error });
|
|
134
|
+
}
|
|
135
|
+
function convertWasmHtml(html, bindings, resolved) {
|
|
136
|
+
let result;
|
|
137
|
+
try {
|
|
138
|
+
result = convertWasm(html, resolved.napiOpts, bindings);
|
|
139
|
+
} catch (error) {
|
|
140
|
+
throw wasmPanicError(error, bindings.takePanicMessage);
|
|
141
|
+
}
|
|
142
|
+
deliverPluginData(result, resolved);
|
|
143
|
+
return result.markdown || "";
|
|
144
|
+
}
|
|
145
|
+
function streamChunks(inner) {
|
|
146
|
+
const carry = createSurrogateCarry();
|
|
147
|
+
return {
|
|
148
|
+
processChunk: (chunk) => inner.processChunk(carry.take(chunk)),
|
|
149
|
+
processChunkBytes: (chunk) => {
|
|
150
|
+
const held = carry.flush();
|
|
151
|
+
return held ? inner.processChunk(held) + inner.processChunkBytes(chunk) : inner.processChunkBytes(chunk);
|
|
152
|
+
},
|
|
153
|
+
finish: () => {
|
|
154
|
+
const held = carry.flush();
|
|
155
|
+
return held ? inner.processChunk(held) + inner.finish() : inner.finish();
|
|
156
|
+
},
|
|
157
|
+
takeData: () => inner.takeData(),
|
|
158
|
+
free: () => inner.free()
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
function openStream(bindings, { napiOpts }) {
|
|
162
|
+
try {
|
|
163
|
+
return streamChunks(new bindings.Stream(napiOpts));
|
|
164
|
+
} catch (error) {
|
|
165
|
+
throw wasmPanicError(error, bindings.takePanicMessage);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
var WasmMarkdownStream = class {
|
|
169
|
+
_bindings;
|
|
170
|
+
_callbacks;
|
|
171
|
+
_chunks;
|
|
172
|
+
constructor(_bindings, _callbacks) {
|
|
173
|
+
this._bindings = _bindings;
|
|
174
|
+
this._callbacks = _callbacks;
|
|
175
|
+
this._chunks = openStream(_bindings, _callbacks);
|
|
176
|
+
}
|
|
177
|
+
processChunk(chunk) {
|
|
178
|
+
try {
|
|
179
|
+
return this._chunks.processChunk(chunk);
|
|
180
|
+
} catch (error) {
|
|
181
|
+
throw wasmPanicError(error, this._bindings.takePanicMessage);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
processChunkBytes(chunk) {
|
|
185
|
+
try {
|
|
186
|
+
return this._chunks.processChunkBytes(chunk);
|
|
187
|
+
} catch (error) {
|
|
188
|
+
throw wasmPanicError(error, this._bindings.takePanicMessage);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
finish() {
|
|
192
|
+
let markdown;
|
|
193
|
+
try {
|
|
194
|
+
markdown = this._chunks.finish();
|
|
195
|
+
} catch (error) {
|
|
196
|
+
throw wasmPanicError(error, this._bindings.takePanicMessage);
|
|
197
|
+
}
|
|
198
|
+
if (this._callbacks.frontmatterCallback || this._callbacks.extractionHandlers) deliverPluginData(this._chunks.takeData(), this._callbacks);
|
|
199
|
+
return markdown;
|
|
200
|
+
}
|
|
201
|
+
};
|
|
202
|
+
function pumpWasmStream(htmlStream, bindings, resolved) {
|
|
203
|
+
return pumpStream(openStream(bindings, resolved), htmlStream, resolved, { mapError: (error) => wasmPanicError(error, bindings.takePanicMessage) });
|
|
204
|
+
}
|
|
205
|
+
const bindings = {
|
|
206
|
+
oneShot: htmlToMarkdownResult,
|
|
207
|
+
Stream: MarkdownStream$1,
|
|
208
|
+
takePanicMessage: __mdreamTakePanicMessage
|
|
209
|
+
};
|
|
210
|
+
const ready = init();
|
|
211
|
+
async function htmlToMarkdown(html, options = {}) {
|
|
212
|
+
await ready;
|
|
213
|
+
return convertWasmHtml(html, bindings, resolveMarkdownOptions(options));
|
|
214
|
+
}
|
|
215
|
+
async function createMarkdownStream(options) {
|
|
216
|
+
await ready;
|
|
217
|
+
return new MarkdownStream(options);
|
|
218
|
+
}
|
|
219
|
+
var MarkdownStream = class extends WasmMarkdownStream {
|
|
220
|
+
constructor(options = {}) {
|
|
221
|
+
super(bindings, resolveMarkdownOptions(options));
|
|
222
|
+
}
|
|
223
|
+
};
|
|
224
|
+
async function* streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
225
|
+
if (!htmlStream) throw new Error("Invalid HTML stream provided");
|
|
226
|
+
await ready;
|
|
227
|
+
yield* pumpWasmStream(htmlStream, bindings, resolveMarkdownOptions(options));
|
|
228
|
+
}
|
|
229
|
+
export { MarkdownStream, createMarkdownStream, htmlToMarkdown, streamHtmlToMarkdown };
|
package/dist/browser.d.mts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { HtmlToMarkdownOptions, MdreamNapiResult } from "../napi/index.js";
|
|
1
2
|
interface CleanOptions {
|
|
2
3
|
/** Strip tracking query parameters (utm_*, fbclid, gclid, etc.) from URLs */
|
|
3
4
|
urls?: boolean;
|
|
@@ -83,21 +84,41 @@ interface MdreamOptions {
|
|
|
83
84
|
*/
|
|
84
85
|
format?: 'markdown' | 'text' | 'html';
|
|
85
86
|
}
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
87
|
+
interface ResolvedOptions {
|
|
88
|
+
napiOpts: HtmlToMarkdownOptions;
|
|
89
|
+
extractionHandlers?: Record<string, (el: ExtractedElement) => void>;
|
|
90
|
+
frontmatterCallback?: (fm: Record<string, string>) => void;
|
|
91
|
+
}
|
|
92
|
+
interface WasmStream {
|
|
93
|
+
processChunk: (chunk: string) => string;
|
|
94
|
+
processChunkBytes: (chunk: Uint8Array) => string;
|
|
95
|
+
finish: () => string;
|
|
96
|
+
takeData: () => Omit<MdreamNapiResult, 'markdown'>;
|
|
97
|
+
free: () => void;
|
|
98
|
+
}
|
|
99
|
+
/** The bindings of one WASM instance a conversion may use. */
|
|
100
|
+
interface WasmEngine {
|
|
101
|
+
oneShot: (html: string, napiOpts: HtmlToMarkdownOptions) => MdreamNapiResult;
|
|
102
|
+
/** Only the entries that take bytes pass it; without it, bytes stream. */
|
|
103
|
+
bytes?: (html: Uint8Array, napiOpts: HtmlToMarkdownOptions) => string;
|
|
104
|
+
Stream: new (napiOpts: HtmlToMarkdownOptions) => WasmStream;
|
|
105
|
+
}
|
|
106
|
+
interface WasmBindings extends WasmEngine {
|
|
107
|
+
takePanicMessage: () => string | undefined;
|
|
108
|
+
}
|
|
109
|
+
declare class WasmMarkdownStream {
|
|
110
|
+
private readonly _bindings;
|
|
111
|
+
private readonly _callbacks;
|
|
112
|
+
private readonly _chunks;
|
|
113
|
+
protected constructor(_bindings: WasmBindings, _callbacks: ResolvedOptions);
|
|
98
114
|
processChunk(chunk: string): string;
|
|
99
115
|
processChunkBytes(chunk: Uint8Array): string;
|
|
100
116
|
finish(): string;
|
|
101
117
|
}
|
|
118
|
+
export declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): Promise<string>;
|
|
119
|
+
export declare function createMarkdownStream(options?: Partial<MdreamOptions>): Promise<MarkdownStream>;
|
|
120
|
+
export declare class MarkdownStream extends WasmMarkdownStream {
|
|
121
|
+
constructor(options?: Partial<MdreamOptions>);
|
|
122
|
+
}
|
|
102
123
|
export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
|
|
103
124
|
export type { CleanOptions, ExtractedElement, FrontmatterConfig, MdreamOptions, TagOverride };
|
package/dist/browser.mjs
CHANGED
|
@@ -120,16 +120,49 @@ function deliverPluginData(data, { frontmatterCallback, extractionHandlers }) {
|
|
|
120
120
|
function hasCallbacks({ frontmatterCallback, extractionHandlers }) {
|
|
121
121
|
return !!frontmatterCallback || !!extractionHandlers;
|
|
122
122
|
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
return
|
|
123
|
+
const utf8 = new TextEncoder();
|
|
124
|
+
const WASM_CHUNK_BYTES = 16384;
|
|
125
|
+
const WASM_ONE_SHOT_MAX = 256;
|
|
126
|
+
function convertWasm(html, napiOpts, engine) {
|
|
127
|
+
if (html.length <= WASM_ONE_SHOT_MAX) return engine.oneShot(html, napiOpts);
|
|
128
|
+
return withWasmStream(engine, napiOpts, (stream) => {
|
|
129
|
+
let markdown = "";
|
|
130
|
+
if (html.charCodeAt(0) === 65279) {
|
|
131
|
+
markdown += stream.processChunk(String.fromCharCode(65279));
|
|
132
|
+
html = html.slice(1);
|
|
133
|
+
}
|
|
134
|
+
const bytes = new Uint8Array(WASM_CHUNK_BYTES);
|
|
135
|
+
for (let start = 0; start < html.length;) {
|
|
136
|
+
const { read, written } = utf8.encodeInto(html.slice(start, start + WASM_CHUNK_BYTES), bytes);
|
|
137
|
+
markdown += stream.processChunkBytes(bytes.subarray(0, written));
|
|
138
|
+
start += read;
|
|
139
|
+
}
|
|
140
|
+
return {
|
|
141
|
+
markdown: markdown + stream.finish(),
|
|
142
|
+
...stream.takeData()
|
|
143
|
+
};
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
function withWasmStream(engine, napiOpts, use) {
|
|
147
|
+
const stream = new engine.Stream(napiOpts);
|
|
148
|
+
try {
|
|
149
|
+
return use(stream);
|
|
150
|
+
} finally {
|
|
151
|
+
try {
|
|
152
|
+
stream.free();
|
|
153
|
+
} catch {}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
function freeStream(stream) {
|
|
157
|
+
try {
|
|
158
|
+
stream.free?.();
|
|
159
|
+
} catch {}
|
|
128
160
|
}
|
|
129
161
|
async function* pumpStream(stream, htmlStream, callbacks, { mapError = (error) => error } = {}) {
|
|
130
|
-
const reader = htmlStream.getReader();
|
|
131
162
|
const decoder = stream.processChunkBytes ? void 0 : new TextDecoder();
|
|
163
|
+
let reader;
|
|
132
164
|
try {
|
|
165
|
+
reader = htmlStream.getReader();
|
|
133
166
|
while (true) {
|
|
134
167
|
const { done, value } = await reader.read();
|
|
135
168
|
if (done) break;
|
|
@@ -149,7 +182,8 @@ async function* pumpStream(stream, htmlStream, callbacks, { mapError = (error) =
|
|
|
149
182
|
} catch (error) {
|
|
150
183
|
throw mapError(error);
|
|
151
184
|
} finally {
|
|
152
|
-
|
|
185
|
+
freeStream(stream);
|
|
186
|
+
reader?.releaseLock();
|
|
153
187
|
}
|
|
154
188
|
}
|
|
155
189
|
function createSurrogateCarry() {
|
|
@@ -185,89 +219,98 @@ function wasmPanicError(error, takeMessage) {
|
|
|
185
219
|
if (!message) return error;
|
|
186
220
|
return new Error(`mdream WASM panic, please report this at https://github.com/harlan-zw/mdream/issues\n${message}`, { cause: error });
|
|
187
221
|
}
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
if (!_initPromise) _initPromise = init();
|
|
191
|
-
return _initPromise;
|
|
192
|
-
}
|
|
193
|
-
ensureInit();
|
|
194
|
-
function convert(html, napiOpts) {
|
|
222
|
+
function convertWasmHtml(html, bindings, resolved) {
|
|
223
|
+
let result;
|
|
195
224
|
try {
|
|
196
|
-
|
|
225
|
+
result = convertWasm(html, resolved.napiOpts, bindings);
|
|
197
226
|
} catch (error) {
|
|
198
|
-
throw wasmPanicError(error,
|
|
227
|
+
throw wasmPanicError(error, bindings.takePanicMessage);
|
|
199
228
|
}
|
|
229
|
+
deliverPluginData(result, resolved);
|
|
230
|
+
return result.markdown || "";
|
|
200
231
|
}
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
return
|
|
232
|
+
function streamChunks(inner) {
|
|
233
|
+
const carry = createSurrogateCarry();
|
|
234
|
+
return {
|
|
235
|
+
processChunk: (chunk) => inner.processChunk(carry.take(chunk)),
|
|
236
|
+
processChunkBytes: (chunk) => {
|
|
237
|
+
const held = carry.flush();
|
|
238
|
+
return held ? inner.processChunk(held) + inner.processChunkBytes(chunk) : inner.processChunkBytes(chunk);
|
|
239
|
+
},
|
|
240
|
+
finish: () => {
|
|
241
|
+
const held = carry.flush();
|
|
242
|
+
return held ? inner.processChunk(held) + inner.finish() : inner.finish();
|
|
243
|
+
},
|
|
244
|
+
takeData: () => inner.takeData(),
|
|
245
|
+
free: () => inner.free()
|
|
246
|
+
};
|
|
204
247
|
}
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
248
|
+
function openStream(bindings, { napiOpts }) {
|
|
249
|
+
try {
|
|
250
|
+
return streamChunks(new bindings.Stream(napiOpts));
|
|
251
|
+
} catch (error) {
|
|
252
|
+
throw wasmPanicError(error, bindings.takePanicMessage);
|
|
253
|
+
}
|
|
208
254
|
}
|
|
209
|
-
var
|
|
210
|
-
|
|
211
|
-
_carry = createSurrogateCarry();
|
|
255
|
+
var WasmMarkdownStream = class {
|
|
256
|
+
_bindings;
|
|
212
257
|
_callbacks;
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
this.
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
} catch (error) {
|
|
219
|
-
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
220
|
-
}
|
|
258
|
+
_chunks;
|
|
259
|
+
constructor(_bindings, _callbacks) {
|
|
260
|
+
this._bindings = _bindings;
|
|
261
|
+
this._callbacks = _callbacks;
|
|
262
|
+
this._chunks = openStream(_bindings, _callbacks);
|
|
221
263
|
}
|
|
222
264
|
processChunk(chunk) {
|
|
223
265
|
try {
|
|
224
|
-
return this.
|
|
266
|
+
return this._chunks.processChunk(chunk);
|
|
225
267
|
} catch (error) {
|
|
226
|
-
throw wasmPanicError(error,
|
|
268
|
+
throw wasmPanicError(error, this._bindings.takePanicMessage);
|
|
227
269
|
}
|
|
228
270
|
}
|
|
229
271
|
processChunkBytes(chunk) {
|
|
230
272
|
try {
|
|
231
|
-
|
|
232
|
-
return held ? this._inner.processChunk(held) + this._inner.processChunkBytes(chunk) : this._inner.processChunkBytes(chunk);
|
|
273
|
+
return this._chunks.processChunkBytes(chunk);
|
|
233
274
|
} catch (error) {
|
|
234
|
-
throw wasmPanicError(error,
|
|
275
|
+
throw wasmPanicError(error, this._bindings.takePanicMessage);
|
|
235
276
|
}
|
|
236
277
|
}
|
|
237
278
|
finish() {
|
|
238
279
|
let markdown;
|
|
239
280
|
try {
|
|
240
|
-
|
|
241
|
-
markdown = held ? this._inner.processChunk(held) + this._inner.finish() : this._inner.finish();
|
|
281
|
+
markdown = this._chunks.finish();
|
|
242
282
|
} catch (error) {
|
|
243
|
-
throw wasmPanicError(error,
|
|
283
|
+
throw wasmPanicError(error, this._bindings.takePanicMessage);
|
|
244
284
|
}
|
|
245
|
-
if (this._callbacks.frontmatterCallback || this._callbacks.extractionHandlers) deliverPluginData(this.
|
|
285
|
+
if (this._callbacks.frontmatterCallback || this._callbacks.extractionHandlers) deliverPluginData(this._chunks.takeData(), this._callbacks);
|
|
246
286
|
return markdown;
|
|
247
287
|
}
|
|
248
288
|
};
|
|
289
|
+
function pumpWasmStream(htmlStream, bindings, resolved) {
|
|
290
|
+
return pumpStream(openStream(bindings, resolved), htmlStream, resolved, { mapError: (error) => wasmPanicError(error, bindings.takePanicMessage) });
|
|
291
|
+
}
|
|
292
|
+
const bindings = {
|
|
293
|
+
oneShot: htmlToMarkdownResult,
|
|
294
|
+
Stream: MarkdownStream$1,
|
|
295
|
+
takePanicMessage: __mdreamTakePanicMessage
|
|
296
|
+
};
|
|
297
|
+
const ready = init();
|
|
298
|
+
async function htmlToMarkdown(html, options = {}) {
|
|
299
|
+
await ready;
|
|
300
|
+
return convertWasmHtml(html, bindings, resolveOptions(options));
|
|
301
|
+
}
|
|
302
|
+
async function createMarkdownStream(options) {
|
|
303
|
+
await ready;
|
|
304
|
+
return new MarkdownStream(options);
|
|
305
|
+
}
|
|
306
|
+
var MarkdownStream = class extends WasmMarkdownStream {
|
|
307
|
+
constructor(options = {}) {
|
|
308
|
+
super(bindings, resolveOptions(options));
|
|
309
|
+
}
|
|
310
|
+
};
|
|
249
311
|
async function* streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
250
312
|
if (!htmlStream) throw new Error("Invalid HTML stream provided");
|
|
251
|
-
await
|
|
252
|
-
|
|
253
|
-
const carry = createSurrogateCarry();
|
|
254
|
-
let stream;
|
|
255
|
-
try {
|
|
256
|
-
stream = new MarkdownStream$1(resolved.napiOpts);
|
|
257
|
-
} catch (error) {
|
|
258
|
-
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
259
|
-
}
|
|
260
|
-
yield* pumpStream({
|
|
261
|
-
processChunk: (chunk) => stream.processChunk(carry.take(chunk)),
|
|
262
|
-
processChunkBytes: (chunk) => {
|
|
263
|
-
const held = carry.flush();
|
|
264
|
-
return held ? stream.processChunk(held) + stream.processChunkBytes(chunk) : stream.processChunkBytes(chunk);
|
|
265
|
-
},
|
|
266
|
-
finish: () => {
|
|
267
|
-
const held = carry.flush();
|
|
268
|
-
return held ? stream.processChunk(held) + stream.finish() : stream.finish();
|
|
269
|
-
},
|
|
270
|
-
takeData: () => stream.takeData()
|
|
271
|
-
}, htmlStream, resolved, { mapError: (error) => wasmPanicError(error, __mdreamTakePanicMessage) });
|
|
313
|
+
await ready;
|
|
314
|
+
yield* pumpWasmStream(htmlStream, bindings, resolveOptions(options));
|
|
272
315
|
}
|
|
273
316
|
export { MarkdownStream, createMarkdownStream, htmlToMarkdown, streamHtmlToMarkdown };
|