mdream 1.4.1 → 1.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -1
- package/bin/mdream.mjs +18 -2
- package/dist/browser.mjs +6 -1
- package/dist/edge.mjs +116 -5
- package/dist/iife.js +1 -1
- package/dist/index.d.mts +5 -0
- package/dist/index.mjs +42 -22
- package/dist/worker.d.mts +5 -0
- package/napi/index.d.mts +1 -0
- package/napi/index.d.ts +1 -0
- package/package.json +21 -16
- package/wasm/mdream_edge_bg.wasm +0 -0
- package/wasm/package.json +1 -1
- package/wasm-bundler/mdream_edge_bg.wasm +0 -0
- package/wasm-bundler/package.json +1 -1
package/dist/index.d.mts
CHANGED
|
@@ -75,6 +75,11 @@ interface MdreamOptions {
|
|
|
75
75
|
* tables, and headings are never wrapped. `0` disables wrapping.
|
|
76
76
|
*/
|
|
77
77
|
wrapWidth?: number;
|
|
78
|
+
/**
|
|
79
|
+
* Output format. Defaults to `markdown`; use `text` to omit Markdown/HTML
|
|
80
|
+
* markup while preserving readable text and block spacing.
|
|
81
|
+
*/
|
|
82
|
+
format?: 'markdown' | 'text';
|
|
78
83
|
}
|
|
79
84
|
declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string;
|
|
80
85
|
declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
|
package/dist/index.mjs
CHANGED
|
@@ -23,12 +23,16 @@ const CLEAN_ALL = {
|
|
|
23
23
|
emptyLinkText: true
|
|
24
24
|
};
|
|
25
25
|
function resolveCleanConfig(options, minimal) {
|
|
26
|
+
const cleanUrls = options.cleanUrls === true;
|
|
26
27
|
let cleanOpt = options.clean;
|
|
27
|
-
if (cleanOpt === void 0
|
|
28
|
-
|
|
28
|
+
if (cleanOpt === void 0) {
|
|
29
|
+
if (!minimal) return { cleanUrls };
|
|
30
|
+
cleanOpt = true;
|
|
31
|
+
}
|
|
32
|
+
if (!cleanOpt) return { cleanUrls };
|
|
29
33
|
const resolved = cleanOpt === true ? CLEAN_ALL : cleanOpt;
|
|
30
34
|
return {
|
|
31
|
-
cleanUrls: resolved.urls
|
|
35
|
+
cleanUrls: cleanUrls || resolved.urls === true,
|
|
32
36
|
clean: resolved
|
|
33
37
|
};
|
|
34
38
|
}
|
|
@@ -37,35 +41,46 @@ function resolveFrontmatter(opt) {
|
|
|
37
41
|
config: {},
|
|
38
42
|
callback: opt
|
|
39
43
|
};
|
|
40
|
-
if (typeof opt === "object")
|
|
41
|
-
config
|
|
42
|
-
|
|
43
|
-
|
|
44
|
+
if (opt && typeof opt === "object") {
|
|
45
|
+
const { onExtract, ...config } = opt;
|
|
46
|
+
return {
|
|
47
|
+
config,
|
|
48
|
+
callback: onExtract
|
|
49
|
+
};
|
|
50
|
+
}
|
|
44
51
|
return { config: {} };
|
|
45
52
|
}
|
|
46
53
|
function resolveOptions(options) {
|
|
54
|
+
if (Array.isArray(options.plugins)) throw new TypeError("Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://mdream.dev/v1-migration#custom-plugins");
|
|
47
55
|
const minimal = options.minimal === true;
|
|
48
|
-
const plugins = {};
|
|
56
|
+
const plugins = options.plugins ? { ...options.plugins } : {};
|
|
49
57
|
let frontmatterCallback;
|
|
50
|
-
|
|
58
|
+
const frontmatterDisabled = options.frontmatter === false;
|
|
59
|
+
if (minimal ? !frontmatterDisabled : !!options.frontmatter) {
|
|
51
60
|
const fm = resolveFrontmatter(options.frontmatter);
|
|
52
61
|
plugins.frontmatter = fm.config;
|
|
53
62
|
frontmatterCallback = fm.callback;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
if (minimal ?
|
|
63
|
+
} else if (frontmatterDisabled) delete plugins.frontmatter;
|
|
64
|
+
const isolateMainDisabled = options.isolateMain === false;
|
|
65
|
+
if (minimal ? !isolateMainDisabled : options.isolateMain) plugins.isolateMain = true;
|
|
66
|
+
else if (isolateMainDisabled) delete plugins.isolateMain;
|
|
67
|
+
const tailwindDisabled = options.tailwind === false;
|
|
68
|
+
if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
|
|
69
|
+
else if (tailwindDisabled) delete plugins.tailwind;
|
|
57
70
|
if (minimal) plugins.filter = options.filter || MINIMAL_FILTER_DEFAULT;
|
|
58
71
|
else if (options.filter) plugins.filter = options.filter;
|
|
72
|
+
const extraction = options.extraction;
|
|
59
73
|
let extractionHandlers;
|
|
60
|
-
if (
|
|
61
|
-
plugins.extraction = { selectors: Object.keys(
|
|
62
|
-
extractionHandlers =
|
|
74
|
+
if (extraction) {
|
|
75
|
+
plugins.extraction = { selectors: Object.keys(extraction) };
|
|
76
|
+
extractionHandlers = extraction;
|
|
63
77
|
}
|
|
64
|
-
|
|
78
|
+
const tagOverrides = options.tagOverrides;
|
|
79
|
+
if (tagOverrides) {
|
|
65
80
|
const overrides = {};
|
|
66
|
-
for (const tag in
|
|
67
|
-
const
|
|
68
|
-
if (
|
|
81
|
+
for (const tag in tagOverrides) {
|
|
82
|
+
const value = tagOverrides[tag];
|
|
83
|
+
if (value) overrides[tag] = typeof value === "string" ? { alias: value } : value;
|
|
69
84
|
}
|
|
70
85
|
plugins.tagOverrides = overrides;
|
|
71
86
|
}
|
|
@@ -76,14 +91,14 @@ function resolveOptions(options) {
|
|
|
76
91
|
cleanUrls,
|
|
77
92
|
clean,
|
|
78
93
|
plugins,
|
|
79
|
-
wrapWidth: options.wrapWidth
|
|
94
|
+
wrapWidth: options.wrapWidth,
|
|
95
|
+
format: options.format
|
|
80
96
|
},
|
|
81
97
|
extractionHandlers,
|
|
82
98
|
frontmatterCallback
|
|
83
99
|
};
|
|
84
100
|
}
|
|
85
101
|
function htmlToMarkdown(html, options = {}) {
|
|
86
|
-
if (Array.isArray(options.plugins)) throw new TypeError("Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://mdream.dev/v1-migration#custom-plugins");
|
|
87
102
|
const { napiOpts, extractionHandlers, frontmatterCallback } = resolveOptions(options);
|
|
88
103
|
const napiResult = htmlToMarkdown$1(html, napiOpts);
|
|
89
104
|
if (napiResult.frontmatter && frontmatterCallback) frontmatterCallback(napiResult.frontmatter);
|
|
@@ -100,10 +115,15 @@ async function* streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
|
100
115
|
while (true) {
|
|
101
116
|
const { done, value } = await reader.read();
|
|
102
117
|
if (done) break;
|
|
103
|
-
const chunk = typeof value === "string" ? value : decoder.decode(value);
|
|
118
|
+
const chunk = typeof value === "string" ? decoder.decode() + value : decoder.decode(value, { stream: true });
|
|
104
119
|
const processed = stream.processChunk(chunk);
|
|
105
120
|
if (processed) yield processed;
|
|
106
121
|
}
|
|
122
|
+
const decoderTail = decoder.decode();
|
|
123
|
+
if (decoderTail) {
|
|
124
|
+
const processed = stream.processChunk(decoderTail);
|
|
125
|
+
if (processed) yield processed;
|
|
126
|
+
}
|
|
107
127
|
const final_ = stream.finish();
|
|
108
128
|
if (final_) yield final_;
|
|
109
129
|
} finally {
|
package/dist/worker.d.mts
CHANGED
|
@@ -75,6 +75,11 @@ interface MdreamOptions {
|
|
|
75
75
|
* tables, and headings are never wrapped. `0` disables wrapping.
|
|
76
76
|
*/
|
|
77
77
|
wrapWidth?: number;
|
|
78
|
+
/**
|
|
79
|
+
* Output format. Defaults to `markdown`; use `text` to omit Markdown/HTML
|
|
80
|
+
* markup while preserving readable text and block spacing.
|
|
81
|
+
*/
|
|
82
|
+
format?: 'markdown' | 'text';
|
|
78
83
|
}
|
|
79
84
|
/**
|
|
80
85
|
* Initialize the mdream web worker. Must be called before htmlToMarkdown.
|
package/napi/index.d.mts
CHANGED
package/napi/index.d.ts
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mdream",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.
|
|
4
|
+
"version": "1.5.3",
|
|
5
5
|
"description": "Ultra-performant HTML to Markdown Convertor Optimized for LLMs and llm.txt artifacts.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -33,7 +33,12 @@
|
|
|
33
33
|
"default": "./dist/worker.mjs"
|
|
34
34
|
},
|
|
35
35
|
"default": "./dist/worker.mjs"
|
|
36
|
-
}
|
|
36
|
+
},
|
|
37
|
+
"./wasm": {
|
|
38
|
+
"types": "./wasm/mdream_edge.d.ts",
|
|
39
|
+
"default": "./wasm/mdream_edge.js"
|
|
40
|
+
},
|
|
41
|
+
"./wasm/*": "./wasm/*"
|
|
37
42
|
},
|
|
38
43
|
"main": "./dist/index.mjs",
|
|
39
44
|
"unpkg": "./dist/iife.js",
|
|
@@ -52,22 +57,22 @@
|
|
|
52
57
|
"browser": "./dist/browser.mjs",
|
|
53
58
|
"dependencies": {},
|
|
54
59
|
"optionalDependencies": {
|
|
55
|
-
"@mdream/rust-android-
|
|
56
|
-
"@mdream/rust-darwin-
|
|
57
|
-
"@mdream/rust-darwin-
|
|
58
|
-
"@mdream/rust-
|
|
59
|
-
"@mdream/rust-linux-arm-gnueabihf": "1.
|
|
60
|
-
"@mdream/rust-linux-arm64-gnu": "1.
|
|
61
|
-
"@mdream/rust-
|
|
62
|
-
"@mdream/rust-linux-
|
|
63
|
-
"@mdream/rust-linux-x64-
|
|
64
|
-
"@mdream/rust-
|
|
65
|
-
"@mdream/rust-
|
|
66
|
-
"@mdream/rust-win32-arm64-msvc": "1.
|
|
67
|
-
"@mdream/rust-win32-x64-msvc": "1.
|
|
60
|
+
"@mdream/rust-android-arm-eabi": "1.5.3",
|
|
61
|
+
"@mdream/rust-darwin-x64": "1.5.3",
|
|
62
|
+
"@mdream/rust-darwin-arm64": "1.5.3",
|
|
63
|
+
"@mdream/rust-freebsd-x64": "1.5.3",
|
|
64
|
+
"@mdream/rust-linux-arm-gnueabihf": "1.5.3",
|
|
65
|
+
"@mdream/rust-linux-arm64-gnu": "1.5.3",
|
|
66
|
+
"@mdream/rust-linux-arm64-musl": "1.5.3",
|
|
67
|
+
"@mdream/rust-linux-x64-gnu": "1.5.3",
|
|
68
|
+
"@mdream/rust-linux-x64-musl": "1.5.3",
|
|
69
|
+
"@mdream/rust-android-arm64": "1.5.3",
|
|
70
|
+
"@mdream/rust-wasm32-wasi": "1.5.3",
|
|
71
|
+
"@mdream/rust-win32-arm64-msvc": "1.5.3",
|
|
72
|
+
"@mdream/rust-win32-x64-msvc": "1.5.3"
|
|
68
73
|
},
|
|
69
74
|
"devDependencies": {
|
|
70
|
-
"@mdream/js": "1.
|
|
75
|
+
"@mdream/js": "1.5.3"
|
|
71
76
|
},
|
|
72
77
|
"scripts": {
|
|
73
78
|
"flame": "pnpm build && unbuild bench/bundle && clinic flame -- node bench/bundle/dist/string.mjs 10",
|
package/wasm/mdream_edge_bg.wasm
CHANGED
|
Binary file
|
package/wasm/package.json
CHANGED
|
Binary file
|