mdream 1.7.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +357 -163
- package/bin/mdream.mjs +45 -24
- package/dist/browser-stub.mjs +7 -0
- package/dist/browser.d.mts +98 -5
- package/dist/browser.mjs +224 -33
- package/dist/edge.d.mts +93 -4
- package/dist/edge.mjs +151 -69
- package/dist/iife.js +1 -396
- package/dist/index.d.mts +8 -6
- package/dist/index.mjs +104 -43
- package/dist/{worker.d.mts → wasm.d.mts} +20 -20
- package/dist/wasm.mjs +150 -0
- package/napi/index.d.mts +20 -33
- package/napi/index.d.ts +20 -33
- package/napi/index.mjs +48 -36
- package/package.json +47 -37
- package/skills/mdream/SKILL.md +128 -0
- package/skills/mdream/references/runtimes.md +57 -0
- package/wasm/mdream_edge.d.ts +27 -0
- package/wasm/mdream_edge.js +94 -20
- package/wasm/mdream_edge_bg.wasm +0 -0
- package/wasm/mdream_edge_bg.wasm.d.ts +3 -0
- package/wasm/package.json +1 -1
- package/wasm-bundler/mdream_edge.d.ts +24 -0
- package/wasm-bundler/mdream_edge.js +6 -2
- package/wasm-bundler/mdream_edge_bg.js +94 -20
- package/wasm-bundler/mdream_edge_bg.wasm +0 -0
- package/wasm-bundler/mdream_edge_bg.wasm.d.ts +3 -0
- package/wasm-bundler/package.json +1 -1
- package/dist/worker.mjs +0 -100
package/bin/mdream.mjs
CHANGED
|
@@ -2,40 +2,61 @@
|
|
|
2
2
|
import { Readable } from 'node:stream'
|
|
3
3
|
import { streamHtmlToMarkdown } from 'mdream'
|
|
4
4
|
|
|
5
|
+
const USAGE = 'Usage: mdream [--origin <url>] [--preset minimal] [--wrap-width <n>] [--format markdown|text|html] [--text]\nPipe HTML via stdin, outputs Markdown, plain text, or safe HTML to stdout.\n'
|
|
6
|
+
|
|
7
|
+
function fail(message) {
|
|
8
|
+
process.stderr.write(`${message}\n`)
|
|
9
|
+
process.exit(1)
|
|
10
|
+
}
|
|
11
|
+
|
|
5
12
|
const args = process.argv.slice(2)
|
|
6
13
|
let origin
|
|
7
14
|
let preset
|
|
8
15
|
let wrapWidth
|
|
9
16
|
let format
|
|
10
17
|
for (let i = 0; i < args.length; i++) {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
18
|
+
const arg = args[i]
|
|
19
|
+
// `--flag=value` and `--flag value` both work.
|
|
20
|
+
const eq = arg.startsWith('--') ? arg.indexOf('=') : -1
|
|
21
|
+
const flag = eq === -1 ? arg : arg.slice(0, eq)
|
|
22
|
+
const takeValue = () => {
|
|
23
|
+
const value = eq === -1 ? args[++i] : arg.slice(eq + 1)
|
|
24
|
+
// A following flag, such as `--text` or `-h`, is not a value.
|
|
25
|
+
if (!value || (eq === -1 && (value.startsWith('--') || value === '-h')))
|
|
26
|
+
fail(`The ${flag} option needs a value.`)
|
|
27
|
+
return value
|
|
28
|
+
}
|
|
29
|
+
if (flag === '--origin') {
|
|
30
|
+
origin = takeValue()
|
|
31
|
+
}
|
|
32
|
+
else if (flag === '--preset') {
|
|
33
|
+
preset = takeValue()
|
|
34
|
+
if (preset !== 'minimal')
|
|
35
|
+
fail(`Unknown preset: ${preset}. Use --preset minimal.`)
|
|
36
|
+
}
|
|
37
|
+
else if (flag === '--wrap-width') {
|
|
38
|
+
wrapWidth = Number.parseInt(takeValue(), 10) || undefined
|
|
39
|
+
}
|
|
40
|
+
else if (flag === '--format') {
|
|
41
|
+
format = takeValue()
|
|
42
|
+
if (format !== 'markdown' && format !== 'text' && format !== 'html')
|
|
43
|
+
fail(`Unknown format: ${format}. Use markdown, text, or html.`)
|
|
44
|
+
}
|
|
45
|
+
else if (flag === '--text') {
|
|
46
|
+
if (eq !== -1)
|
|
47
|
+
fail('The --text option takes no value.')
|
|
33
48
|
format = 'text'
|
|
34
49
|
}
|
|
35
|
-
else if (
|
|
36
|
-
process.stdout.write(
|
|
50
|
+
else if (arg === '-h' || arg === '--help') {
|
|
51
|
+
process.stdout.write(USAGE)
|
|
37
52
|
process.exit(0)
|
|
38
53
|
}
|
|
54
|
+
else if (arg.startsWith('-')) {
|
|
55
|
+
fail(`Unknown option: ${flag}. Run mdream --help to list the options.`)
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
fail(`Unknown argument: ${arg}. Pipe the HTML to mdream on stdin.`)
|
|
59
|
+
}
|
|
39
60
|
}
|
|
40
61
|
|
|
41
62
|
const options = { origin, minimal: preset === 'minimal', wrapWidth, format }
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
function htmlToMarkdown() {
|
|
2
|
+
throw new TypeError("In a browser bundle, import htmlToMarkdown from 'mdream/browser'. It returns a Promise.");
|
|
3
|
+
}
|
|
4
|
+
function streamHtmlToMarkdown() {
|
|
5
|
+
throw new TypeError("In a browser bundle, import streamHtmlToMarkdown from 'mdream/browser'.");
|
|
6
|
+
}
|
|
7
|
+
export { htmlToMarkdown, streamHtmlToMarkdown };
|
package/dist/browser.d.mts
CHANGED
|
@@ -1,10 +1,103 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
interface CleanOptions {
|
|
2
|
+
/** Strip tracking query parameters (utm_*, fbclid, gclid, etc.) from URLs */
|
|
3
|
+
urls?: boolean;
|
|
4
|
+
/** Strip fragment-only links that don't match any heading in the output */
|
|
5
|
+
fragments?: boolean;
|
|
6
|
+
/** Strip links with meaningless hrefs (#, javascript:void(0)) → plain text */
|
|
7
|
+
emptyLinks?: boolean;
|
|
8
|
+
/** Strip links where text equals URL: [https://x.com](https://x.com) → https://x.com */
|
|
9
|
+
redundantLinks?: boolean;
|
|
10
|
+
/** Strip self-referencing heading anchors: ## [Title](#title) → ## Title */
|
|
11
|
+
selfLinkHeadings?: boolean;
|
|
12
|
+
/** Strip images with no alt text (decorative/tracking pixels) */
|
|
13
|
+
emptyImages?: boolean;
|
|
14
|
+
/** Drop links that produce no visible text: [](url) → nothing */
|
|
15
|
+
emptyLinkText?: boolean;
|
|
16
|
+
}
|
|
17
|
+
interface ExtractedElement {
|
|
18
|
+
selector: string;
|
|
19
|
+
tagName: string;
|
|
20
|
+
textContent: string;
|
|
21
|
+
attributes: Record<string, string>;
|
|
22
|
+
}
|
|
23
|
+
interface FrontmatterConfig {
|
|
24
|
+
additionalFields?: Record<string, string>;
|
|
25
|
+
metaFields?: string[];
|
|
26
|
+
/** Callback to receive structured frontmatter data after conversion */
|
|
27
|
+
onExtract?: (frontmatter: Record<string, string>) => void;
|
|
28
|
+
}
|
|
29
|
+
interface TagOverride {
|
|
30
|
+
enter?: string;
|
|
31
|
+
exit?: string;
|
|
32
|
+
spacing?: number[];
|
|
33
|
+
isInline?: boolean;
|
|
34
|
+
isSelfClosing?: boolean;
|
|
35
|
+
collapsesInnerWhiteSpace?: boolean;
|
|
36
|
+
alias?: string;
|
|
37
|
+
}
|
|
38
|
+
interface MdreamOptions {
|
|
39
|
+
/** Origin URL for resolving relative image paths and internal links. */
|
|
40
|
+
origin?: string;
|
|
41
|
+
/**
|
|
42
|
+
* Clean up the markdown output. Pass `true` for all cleanup or an object
|
|
43
|
+
* to enable specific features. Both `htmlToMarkdown` and
|
|
44
|
+
* `streamHtmlToMarkdown` apply it.
|
|
45
|
+
*/
|
|
46
|
+
clean?: boolean | CleanOptions;
|
|
47
|
+
/** Enable minimal preset (frontmatter, isolateMain, tailwind, filter). Default: false */
|
|
48
|
+
minimal?: boolean;
|
|
49
|
+
/**
|
|
50
|
+
* Extract frontmatter from HTML head.
|
|
51
|
+
* - `true`: enable with defaults
|
|
52
|
+
* - `(fm) => void`: enable and receive structured data via callback
|
|
53
|
+
* - `FrontmatterConfig`: enable with config options and optional callback
|
|
54
|
+
*/
|
|
55
|
+
frontmatter?: boolean | ((frontmatter: Record<string, string>) => void) | FrontmatterConfig;
|
|
56
|
+
/** Isolate main content area. Default when minimal: true */
|
|
57
|
+
isolateMain?: boolean;
|
|
58
|
+
/** Convert Tailwind utility classes. Default when minimal: true */
|
|
59
|
+
tailwind?: boolean;
|
|
60
|
+
/**
|
|
61
|
+
* Filter elements by CSS selector. `minimal` excludes form, nav, footer, and
|
|
62
|
+
* similar; a filter passed with `minimal` adds to those excludes. `false`
|
|
63
|
+
* turns filtering off, including the `minimal` one.
|
|
64
|
+
*/
|
|
65
|
+
filter?: false | {
|
|
66
|
+
include?: string[];
|
|
67
|
+
exclude?: string[];
|
|
68
|
+
processChildren?: boolean;
|
|
69
|
+
};
|
|
70
|
+
/** Extract elements matching CSS selectors */
|
|
71
|
+
extraction?: Record<string, (element: ExtractedElement) => void>;
|
|
72
|
+
/** Tag overrides. String values act as aliases */
|
|
73
|
+
tagOverrides?: Record<string, TagOverride | string>;
|
|
74
|
+
/**
|
|
75
|
+
* Hard-wrap prose at this many characters, breaking on word boundaries.
|
|
76
|
+
* Applied inline during conversion (zero-cost when unset). Code blocks,
|
|
77
|
+
* tables, and headings are never wrapped. `0` disables wrapping.
|
|
78
|
+
*/
|
|
79
|
+
wrapWidth?: number;
|
|
80
|
+
/**
|
|
81
|
+
* Output format. Defaults to `markdown`; use `text` for readable plain text,
|
|
82
|
+
* or `html` for allowlisted semantic HTML.
|
|
83
|
+
*/
|
|
84
|
+
format?: 'markdown' | 'text' | 'html';
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Browser builds fetch the WASM binary on first use, so this returns a
|
|
88
|
+
* Promise. `mdream/browser` ships types that say so.
|
|
89
|
+
*/
|
|
90
|
+
export declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): Promise<string>;
|
|
91
|
+
export declare function createMarkdownStream(options?: Partial<MdreamOptions>): Promise<MarkdownStream>;
|
|
92
|
+
/** Streaming converter. Runs the frontmatter and extraction callbacks in `finish()`. */
|
|
4
93
|
export declare class MarkdownStream {
|
|
5
94
|
private _inner;
|
|
6
|
-
|
|
95
|
+
private _carry;
|
|
96
|
+
private _callbacks;
|
|
97
|
+
constructor(options?: Partial<MdreamOptions>);
|
|
7
98
|
processChunk(chunk: string): string;
|
|
99
|
+
processChunkBytes(chunk: Uint8Array): string;
|
|
8
100
|
finish(): string;
|
|
9
101
|
}
|
|
10
|
-
export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?:
|
|
102
|
+
export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
|
|
103
|
+
export type { CleanOptions, ExtractedElement, FrontmatterConfig, MdreamOptions, TagOverride };
|
package/dist/browser.mjs
CHANGED
|
@@ -1,5 +1,180 @@
|
|
|
1
1
|
import init, { MarkdownStream as MarkdownStream$1, __mdreamTakePanicMessage, htmlToMarkdownResult } from "../wasm/mdream_edge.js";
|
|
2
|
-
|
|
2
|
+
const MINIMAL_FILTER_EXCLUDE = [
|
|
3
|
+
"form",
|
|
4
|
+
"fieldset",
|
|
5
|
+
"object",
|
|
6
|
+
"embed",
|
|
7
|
+
"footer",
|
|
8
|
+
"aside",
|
|
9
|
+
"iframe",
|
|
10
|
+
"input",
|
|
11
|
+
"textarea",
|
|
12
|
+
"select",
|
|
13
|
+
"button",
|
|
14
|
+
"nav"
|
|
15
|
+
];
|
|
16
|
+
const CLEAN_ALL = {
|
|
17
|
+
urls: true,
|
|
18
|
+
fragments: true,
|
|
19
|
+
emptyLinks: true,
|
|
20
|
+
redundantLinks: true,
|
|
21
|
+
selfLinkHeadings: true,
|
|
22
|
+
emptyImages: true,
|
|
23
|
+
emptyLinkText: true
|
|
24
|
+
};
|
|
25
|
+
const VALID_OPTIONS = "origin, clean, minimal, frontmatter, isolateMain, tailwind, filter, extraction, tagOverrides, wrapWidth, format";
|
|
26
|
+
function unknownOptionMessage(key, value) {
|
|
27
|
+
switch (key) {
|
|
28
|
+
case "preset": return "mdream has no `preset` option. Pass { minimal: true }.";
|
|
29
|
+
case "cleanUrls": return "mdream has no `cleanUrls` option. Pass { clean: { urls: true } }.";
|
|
30
|
+
case "plugins": return Array.isArray(value) ? "Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://github.com/harlan-zw/mdream/tree/main/packages/js#migrating-from-v1" : "mdream takes plugin options at the top level, such as { frontmatter: true }.";
|
|
31
|
+
}
|
|
32
|
+
return `mdream has no \`${key}\` option. Valid options: ${VALID_OPTIONS}.`;
|
|
33
|
+
}
|
|
34
|
+
function assertKnownOptions(options) {
|
|
35
|
+
for (const key in options) {
|
|
36
|
+
switch (key) {
|
|
37
|
+
case "origin":
|
|
38
|
+
case "clean":
|
|
39
|
+
case "minimal":
|
|
40
|
+
case "frontmatter":
|
|
41
|
+
case "isolateMain":
|
|
42
|
+
case "tailwind":
|
|
43
|
+
case "filter":
|
|
44
|
+
case "extraction":
|
|
45
|
+
case "tagOverrides":
|
|
46
|
+
case "wrapWidth":
|
|
47
|
+
case "format": continue;
|
|
48
|
+
}
|
|
49
|
+
const value = options[key];
|
|
50
|
+
if (value !== void 0) throw new TypeError(unknownOptionMessage(key, value));
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
function resolveClean(clean, minimal) {
|
|
54
|
+
if (clean === void 0) return minimal ? CLEAN_ALL : void 0;
|
|
55
|
+
if (!clean) return void 0;
|
|
56
|
+
return clean === true ? CLEAN_ALL : clean;
|
|
57
|
+
}
|
|
58
|
+
function resolveFrontmatter(opt) {
|
|
59
|
+
if (typeof opt === "function") return {
|
|
60
|
+
config: {},
|
|
61
|
+
callback: opt
|
|
62
|
+
};
|
|
63
|
+
if (opt && typeof opt === "object") {
|
|
64
|
+
const { onExtract, ...config } = opt;
|
|
65
|
+
return {
|
|
66
|
+
config,
|
|
67
|
+
callback: onExtract
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
return { config: {} };
|
|
71
|
+
}
|
|
72
|
+
function resolveOptions(options) {
|
|
73
|
+
assertKnownOptions(options);
|
|
74
|
+
const minimal = options.minimal === true;
|
|
75
|
+
const plugins = {};
|
|
76
|
+
let frontmatterCallback;
|
|
77
|
+
if (minimal ? options.frontmatter !== false : options.frontmatter) {
|
|
78
|
+
const fm = resolveFrontmatter(options.frontmatter);
|
|
79
|
+
plugins.frontmatter = fm.config;
|
|
80
|
+
frontmatterCallback = fm.callback;
|
|
81
|
+
}
|
|
82
|
+
if (minimal ? options.isolateMain !== false : options.isolateMain) plugins.isolateMain = true;
|
|
83
|
+
if (minimal ? options.tailwind !== false : options.tailwind) plugins.tailwind = true;
|
|
84
|
+
if (minimal && options.filter !== false) plugins.filter = {
|
|
85
|
+
...options.filter,
|
|
86
|
+
exclude: [...MINIMAL_FILTER_EXCLUDE, ...options.filter?.exclude ?? []]
|
|
87
|
+
};
|
|
88
|
+
else if (options.filter) plugins.filter = options.filter;
|
|
89
|
+
const extraction = options.extraction;
|
|
90
|
+
let extractionHandlers;
|
|
91
|
+
if (extraction) {
|
|
92
|
+
plugins.extraction = { selectors: Object.keys(extraction) };
|
|
93
|
+
extractionHandlers = extraction;
|
|
94
|
+
}
|
|
95
|
+
const tagOverrides = options.tagOverrides;
|
|
96
|
+
if (tagOverrides) {
|
|
97
|
+
const overrides = {};
|
|
98
|
+
for (const tag in tagOverrides) {
|
|
99
|
+
const value = tagOverrides[tag];
|
|
100
|
+
if (value) overrides[tag] = typeof value === "string" ? { alias: value } : value;
|
|
101
|
+
}
|
|
102
|
+
plugins.tagOverrides = overrides;
|
|
103
|
+
}
|
|
104
|
+
return {
|
|
105
|
+
napiOpts: {
|
|
106
|
+
origin: options.origin,
|
|
107
|
+
clean: resolveClean(options.clean, minimal),
|
|
108
|
+
plugins,
|
|
109
|
+
wrapWidth: options.wrapWidth,
|
|
110
|
+
format: options.format
|
|
111
|
+
},
|
|
112
|
+
extractionHandlers,
|
|
113
|
+
frontmatterCallback
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
function deliverPluginData(data, { frontmatterCallback, extractionHandlers }) {
|
|
117
|
+
if (data.frontmatter && frontmatterCallback) frontmatterCallback(data.frontmatter);
|
|
118
|
+
if (data.extracted?.length && extractionHandlers) for (const element of data.extracted) extractionHandlers[element.selector]?.(element);
|
|
119
|
+
}
|
|
120
|
+
function hasCallbacks({ frontmatterCallback, extractionHandlers }) {
|
|
121
|
+
return !!frontmatterCallback || !!extractionHandlers;
|
|
122
|
+
}
|
|
123
|
+
function convertResult(convert, html, options = {}) {
|
|
124
|
+
const resolved = resolveOptions(options);
|
|
125
|
+
const result = convert(html, resolved.napiOpts);
|
|
126
|
+
deliverPluginData(result, resolved);
|
|
127
|
+
return result;
|
|
128
|
+
}
|
|
129
|
+
async function* pumpStream(stream, htmlStream, callbacks, { mapError = (error) => error } = {}) {
|
|
130
|
+
const reader = htmlStream.getReader();
|
|
131
|
+
const decoder = stream.processChunkBytes ? void 0 : new TextDecoder();
|
|
132
|
+
try {
|
|
133
|
+
while (true) {
|
|
134
|
+
const { done, value } = await reader.read();
|
|
135
|
+
if (done) break;
|
|
136
|
+
let processed;
|
|
137
|
+
if (typeof value === "string") processed = stream.processChunk(decoder ? decoder.decode() + value : value);
|
|
138
|
+
else processed = decoder ? stream.processChunk(decoder.decode(value, { stream: true })) : stream.processChunkBytes(value);
|
|
139
|
+
if (processed) yield processed;
|
|
140
|
+
}
|
|
141
|
+
const decoderTail = decoder?.decode();
|
|
142
|
+
if (decoderTail) {
|
|
143
|
+
const processed = stream.processChunk(decoderTail);
|
|
144
|
+
if (processed) yield processed;
|
|
145
|
+
}
|
|
146
|
+
const final_ = stream.finish();
|
|
147
|
+
if (hasCallbacks(callbacks)) deliverPluginData(stream.takeData(), callbacks);
|
|
148
|
+
if (final_) yield final_;
|
|
149
|
+
} catch (error) {
|
|
150
|
+
throw mapError(error);
|
|
151
|
+
} finally {
|
|
152
|
+
reader.releaseLock();
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
function createSurrogateCarry() {
|
|
156
|
+
let held = "";
|
|
157
|
+
return {
|
|
158
|
+
take(chunk) {
|
|
159
|
+
if (held) {
|
|
160
|
+
chunk = held + chunk;
|
|
161
|
+
held = "";
|
|
162
|
+
}
|
|
163
|
+
const last = chunk.charCodeAt(chunk.length - 1);
|
|
164
|
+
if (last >= 55296 && last <= 56319) {
|
|
165
|
+
held = chunk.slice(-1);
|
|
166
|
+
return chunk.slice(0, -1);
|
|
167
|
+
}
|
|
168
|
+
return chunk;
|
|
169
|
+
},
|
|
170
|
+
flush() {
|
|
171
|
+
const value = held;
|
|
172
|
+
held = "";
|
|
173
|
+
return value;
|
|
174
|
+
}
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
function wasmPanicError(error, takeMessage) {
|
|
3
178
|
if (!(error instanceof WebAssembly.RuntimeError)) return error;
|
|
4
179
|
let message;
|
|
5
180
|
try {
|
|
@@ -16,67 +191,83 @@ function ensureInit() {
|
|
|
16
191
|
return _initPromise;
|
|
17
192
|
}
|
|
18
193
|
ensureInit();
|
|
19
|
-
|
|
20
|
-
await ensureInit();
|
|
194
|
+
function convert(html, napiOpts) {
|
|
21
195
|
try {
|
|
22
|
-
return htmlToMarkdownResult(html,
|
|
196
|
+
return htmlToMarkdownResult(html, napiOpts);
|
|
23
197
|
} catch (error) {
|
|
24
|
-
throw wasmPanicError(error);
|
|
198
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
25
199
|
}
|
|
26
200
|
}
|
|
201
|
+
async function htmlToMarkdown(html, options = {}) {
|
|
202
|
+
await ensureInit();
|
|
203
|
+
return convertResult(convert, html, options).markdown || "";
|
|
204
|
+
}
|
|
27
205
|
async function createMarkdownStream(options) {
|
|
28
206
|
await ensureInit();
|
|
29
207
|
return new MarkdownStream(options);
|
|
30
208
|
}
|
|
31
209
|
var MarkdownStream = class {
|
|
32
210
|
_inner;
|
|
33
|
-
|
|
211
|
+
_carry = createSurrogateCarry();
|
|
212
|
+
_callbacks;
|
|
213
|
+
constructor(options = {}) {
|
|
214
|
+
const resolved = resolveOptions(options);
|
|
215
|
+
this._callbacks = resolved;
|
|
34
216
|
try {
|
|
35
|
-
this._inner = new MarkdownStream$1(
|
|
217
|
+
this._inner = new MarkdownStream$1(resolved.napiOpts);
|
|
36
218
|
} catch (error) {
|
|
37
|
-
throw wasmPanicError(error);
|
|
219
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
38
220
|
}
|
|
39
221
|
}
|
|
40
222
|
processChunk(chunk) {
|
|
41
223
|
try {
|
|
42
|
-
return this._inner.processChunk(chunk);
|
|
224
|
+
return this._inner.processChunk(this._carry.take(chunk));
|
|
225
|
+
} catch (error) {
|
|
226
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
processChunkBytes(chunk) {
|
|
230
|
+
try {
|
|
231
|
+
const held = this._carry.flush();
|
|
232
|
+
return held ? this._inner.processChunk(held) + this._inner.processChunkBytes(chunk) : this._inner.processChunkBytes(chunk);
|
|
43
233
|
} catch (error) {
|
|
44
|
-
throw wasmPanicError(error);
|
|
234
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
45
235
|
}
|
|
46
236
|
}
|
|
47
237
|
finish() {
|
|
238
|
+
let markdown;
|
|
48
239
|
try {
|
|
49
|
-
|
|
240
|
+
const held = this._carry.flush();
|
|
241
|
+
markdown = held ? this._inner.processChunk(held) + this._inner.finish() : this._inner.finish();
|
|
50
242
|
} catch (error) {
|
|
51
|
-
throw wasmPanicError(error);
|
|
243
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
52
244
|
}
|
|
245
|
+
if (this._callbacks.frontmatterCallback || this._callbacks.extractionHandlers) deliverPluginData(this._inner.takeData(), this._callbacks);
|
|
246
|
+
return markdown;
|
|
53
247
|
}
|
|
54
248
|
};
|
|
55
|
-
async function* streamHtmlToMarkdown(htmlStream, options) {
|
|
249
|
+
async function* streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
56
250
|
if (!htmlStream) throw new Error("Invalid HTML stream provided");
|
|
57
251
|
await ensureInit();
|
|
58
|
-
const
|
|
59
|
-
const
|
|
60
|
-
|
|
252
|
+
const resolved = resolveOptions(options);
|
|
253
|
+
const carry = createSurrogateCarry();
|
|
254
|
+
let stream;
|
|
61
255
|
try {
|
|
62
|
-
|
|
63
|
-
const { done, value } = await reader.read();
|
|
64
|
-
if (done) break;
|
|
65
|
-
const chunk = typeof value === "string" ? decoder.decode() + value : decoder.decode(value, { stream: true });
|
|
66
|
-
const processed = stream.processChunk(chunk);
|
|
67
|
-
if (processed) yield processed;
|
|
68
|
-
}
|
|
69
|
-
const decoderTail = decoder.decode();
|
|
70
|
-
if (decoderTail) {
|
|
71
|
-
const processed = stream.processChunk(decoderTail);
|
|
72
|
-
if (processed) yield processed;
|
|
73
|
-
}
|
|
74
|
-
const final_ = stream.finish();
|
|
75
|
-
if (final_) yield final_;
|
|
256
|
+
stream = new MarkdownStream$1(resolved.napiOpts);
|
|
76
257
|
} catch (error) {
|
|
77
|
-
throw wasmPanicError(error);
|
|
78
|
-
} finally {
|
|
79
|
-
reader.releaseLock();
|
|
258
|
+
throw wasmPanicError(error, __mdreamTakePanicMessage);
|
|
80
259
|
}
|
|
260
|
+
yield* pumpStream({
|
|
261
|
+
processChunk: (chunk) => stream.processChunk(carry.take(chunk)),
|
|
262
|
+
processChunkBytes: (chunk) => {
|
|
263
|
+
const held = carry.flush();
|
|
264
|
+
return held ? stream.processChunk(held) + stream.processChunkBytes(chunk) : stream.processChunkBytes(chunk);
|
|
265
|
+
},
|
|
266
|
+
finish: () => {
|
|
267
|
+
const held = carry.flush();
|
|
268
|
+
return held ? stream.processChunk(held) + stream.finish() : stream.finish();
|
|
269
|
+
},
|
|
270
|
+
takeData: () => stream.takeData()
|
|
271
|
+
}, htmlStream, resolved, { mapError: (error) => wasmPanicError(error, __mdreamTakePanicMessage) });
|
|
81
272
|
}
|
|
82
273
|
export { MarkdownStream, createMarkdownStream, htmlToMarkdown, streamHtmlToMarkdown };
|
package/dist/edge.d.mts
CHANGED
|
@@ -1,9 +1,98 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
interface CleanOptions {
|
|
2
|
+
/** Strip tracking query parameters (utm_*, fbclid, gclid, etc.) from URLs */
|
|
3
|
+
urls?: boolean;
|
|
4
|
+
/** Strip fragment-only links that don't match any heading in the output */
|
|
5
|
+
fragments?: boolean;
|
|
6
|
+
/** Strip links with meaningless hrefs (#, javascript:void(0)) → plain text */
|
|
7
|
+
emptyLinks?: boolean;
|
|
8
|
+
/** Strip links where text equals URL: [https://x.com](https://x.com) → https://x.com */
|
|
9
|
+
redundantLinks?: boolean;
|
|
10
|
+
/** Strip self-referencing heading anchors: ## [Title](#title) → ## Title */
|
|
11
|
+
selfLinkHeadings?: boolean;
|
|
12
|
+
/** Strip images with no alt text (decorative/tracking pixels) */
|
|
13
|
+
emptyImages?: boolean;
|
|
14
|
+
/** Drop links that produce no visible text: [](url) → nothing */
|
|
15
|
+
emptyLinkText?: boolean;
|
|
16
|
+
}
|
|
17
|
+
interface ExtractedElement {
|
|
18
|
+
selector: string;
|
|
19
|
+
tagName: string;
|
|
20
|
+
textContent: string;
|
|
21
|
+
attributes: Record<string, string>;
|
|
22
|
+
}
|
|
23
|
+
interface FrontmatterConfig {
|
|
24
|
+
additionalFields?: Record<string, string>;
|
|
25
|
+
metaFields?: string[];
|
|
26
|
+
/** Callback to receive structured frontmatter data after conversion */
|
|
27
|
+
onExtract?: (frontmatter: Record<string, string>) => void;
|
|
28
|
+
}
|
|
29
|
+
interface TagOverride {
|
|
30
|
+
enter?: string;
|
|
31
|
+
exit?: string;
|
|
32
|
+
spacing?: number[];
|
|
33
|
+
isInline?: boolean;
|
|
34
|
+
isSelfClosing?: boolean;
|
|
35
|
+
collapsesInnerWhiteSpace?: boolean;
|
|
36
|
+
alias?: string;
|
|
37
|
+
}
|
|
38
|
+
interface MdreamOptions {
|
|
39
|
+
/** Origin URL for resolving relative image paths and internal links. */
|
|
40
|
+
origin?: string;
|
|
41
|
+
/**
|
|
42
|
+
* Clean up the markdown output. Pass `true` for all cleanup or an object
|
|
43
|
+
* to enable specific features. Both `htmlToMarkdown` and
|
|
44
|
+
* `streamHtmlToMarkdown` apply it.
|
|
45
|
+
*/
|
|
46
|
+
clean?: boolean | CleanOptions;
|
|
47
|
+
/** Enable minimal preset (frontmatter, isolateMain, tailwind, filter). Default: false */
|
|
48
|
+
minimal?: boolean;
|
|
49
|
+
/**
|
|
50
|
+
* Extract frontmatter from HTML head.
|
|
51
|
+
* - `true`: enable with defaults
|
|
52
|
+
* - `(fm) => void`: enable and receive structured data via callback
|
|
53
|
+
* - `FrontmatterConfig`: enable with config options and optional callback
|
|
54
|
+
*/
|
|
55
|
+
frontmatter?: boolean | ((frontmatter: Record<string, string>) => void) | FrontmatterConfig;
|
|
56
|
+
/** Isolate main content area. Default when minimal: true */
|
|
57
|
+
isolateMain?: boolean;
|
|
58
|
+
/** Convert Tailwind utility classes. Default when minimal: true */
|
|
59
|
+
tailwind?: boolean;
|
|
60
|
+
/**
|
|
61
|
+
* Filter elements by CSS selector. `minimal` excludes form, nav, footer, and
|
|
62
|
+
* similar; a filter passed with `minimal` adds to those excludes. `false`
|
|
63
|
+
* turns filtering off, including the `minimal` one.
|
|
64
|
+
*/
|
|
65
|
+
filter?: false | {
|
|
66
|
+
include?: string[];
|
|
67
|
+
exclude?: string[];
|
|
68
|
+
processChildren?: boolean;
|
|
69
|
+
};
|
|
70
|
+
/** Extract elements matching CSS selectors */
|
|
71
|
+
extraction?: Record<string, (element: ExtractedElement) => void>;
|
|
72
|
+
/** Tag overrides. String values act as aliases */
|
|
73
|
+
tagOverrides?: Record<string, TagOverride | string>;
|
|
74
|
+
/**
|
|
75
|
+
* Hard-wrap prose at this many characters, breaking on word boundaries.
|
|
76
|
+
* Applied inline during conversion (zero-cost when unset). Code blocks,
|
|
77
|
+
* tables, and headings are never wrapped. `0` disables wrapping.
|
|
78
|
+
*/
|
|
79
|
+
wrapWidth?: number;
|
|
80
|
+
/**
|
|
81
|
+
* Output format. Defaults to `markdown`; use `text` for readable plain text,
|
|
82
|
+
* or `html` for allowlisted semantic HTML.
|
|
83
|
+
*/
|
|
84
|
+
format?: 'markdown' | 'text' | 'html';
|
|
85
|
+
}
|
|
86
|
+
export declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string;
|
|
87
|
+
/** Streaming converter. Runs the frontmatter and extraction callbacks in `finish()`. */
|
|
3
88
|
export declare class MarkdownStream {
|
|
4
89
|
private _inner;
|
|
5
|
-
|
|
90
|
+
private _carry;
|
|
91
|
+
private _callbacks;
|
|
92
|
+
constructor(options?: Partial<MdreamOptions>);
|
|
6
93
|
processChunk(chunk: string): string;
|
|
94
|
+
processChunkBytes(chunk: Uint8Array): string;
|
|
7
95
|
finish(): string;
|
|
8
96
|
}
|
|
9
|
-
export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?:
|
|
97
|
+
export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
|
|
98
|
+
export type { CleanOptions, ExtractedElement, FrontmatterConfig, MdreamOptions, TagOverride };
|