mdream 1.4.0 → 1.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -1
- package/bin/mdream.mjs +18 -2
- package/dist/browser.mjs +6 -1
- package/dist/edge.mjs +116 -5
- package/dist/iife.js +1 -1
- package/dist/index.d.mts +5 -0
- package/dist/index.mjs +42 -22
- package/dist/worker.d.mts +5 -0
- package/napi/index.d.mts +1 -0
- package/napi/index.d.ts +1 -0
- package/napi/index.mjs +12 -11
- package/package.json +21 -16
- package/wasm/mdream_edge_bg.wasm +0 -0
- package/wasm/package.json +1 -1
- package/wasm-bundler/mdream_edge_bg.wasm +0 -0
- package/wasm-bundler/package.json +1 -1
package/dist/index.d.mts
CHANGED
|
@@ -75,6 +75,11 @@ interface MdreamOptions {
|
|
|
75
75
|
* tables, and headings are never wrapped. `0` disables wrapping.
|
|
76
76
|
*/
|
|
77
77
|
wrapWidth?: number;
|
|
78
|
+
/**
|
|
79
|
+
* Output format. Defaults to `markdown`; use `text` to omit Markdown/HTML
|
|
80
|
+
* markup while preserving readable text and block spacing.
|
|
81
|
+
*/
|
|
82
|
+
format?: 'markdown' | 'text';
|
|
78
83
|
}
|
|
79
84
|
declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string;
|
|
80
85
|
declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
|
package/dist/index.mjs
CHANGED
|
@@ -23,12 +23,16 @@ const CLEAN_ALL = {
|
|
|
23
23
|
emptyLinkText: true
|
|
24
24
|
};
|
|
25
25
|
function resolveCleanConfig(options, minimal) {
|
|
26
|
+
const cleanUrls = options.cleanUrls === true;
|
|
26
27
|
let cleanOpt = options.clean;
|
|
27
|
-
if (cleanOpt === void 0
|
|
28
|
-
|
|
28
|
+
if (cleanOpt === void 0) {
|
|
29
|
+
if (!minimal) return { cleanUrls };
|
|
30
|
+
cleanOpt = true;
|
|
31
|
+
}
|
|
32
|
+
if (!cleanOpt) return { cleanUrls };
|
|
29
33
|
const resolved = cleanOpt === true ? CLEAN_ALL : cleanOpt;
|
|
30
34
|
return {
|
|
31
|
-
cleanUrls: resolved.urls
|
|
35
|
+
cleanUrls: cleanUrls || resolved.urls === true,
|
|
32
36
|
clean: resolved
|
|
33
37
|
};
|
|
34
38
|
}
|
|
@@ -37,35 +41,46 @@ function resolveFrontmatter(opt) {
|
|
|
37
41
|
config: {},
|
|
38
42
|
callback: opt
|
|
39
43
|
};
|
|
40
|
-
if (typeof opt === "object")
|
|
41
|
-
config
|
|
42
|
-
|
|
43
|
-
|
|
44
|
+
if (opt && typeof opt === "object") {
|
|
45
|
+
const { onExtract, ...config } = opt;
|
|
46
|
+
return {
|
|
47
|
+
config,
|
|
48
|
+
callback: onExtract
|
|
49
|
+
};
|
|
50
|
+
}
|
|
44
51
|
return { config: {} };
|
|
45
52
|
}
|
|
46
53
|
function resolveOptions(options) {
|
|
54
|
+
if (Array.isArray(options.plugins)) throw new TypeError("Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://mdream.dev/v1-migration#custom-plugins");
|
|
47
55
|
const minimal = options.minimal === true;
|
|
48
|
-
const plugins = {};
|
|
56
|
+
const plugins = options.plugins ? { ...options.plugins } : {};
|
|
49
57
|
let frontmatterCallback;
|
|
50
|
-
|
|
58
|
+
const frontmatterDisabled = options.frontmatter === false;
|
|
59
|
+
if (minimal ? !frontmatterDisabled : !!options.frontmatter) {
|
|
51
60
|
const fm = resolveFrontmatter(options.frontmatter);
|
|
52
61
|
plugins.frontmatter = fm.config;
|
|
53
62
|
frontmatterCallback = fm.callback;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
if (minimal ?
|
|
63
|
+
} else if (frontmatterDisabled) delete plugins.frontmatter;
|
|
64
|
+
const isolateMainDisabled = options.isolateMain === false;
|
|
65
|
+
if (minimal ? !isolateMainDisabled : options.isolateMain) plugins.isolateMain = true;
|
|
66
|
+
else if (isolateMainDisabled) delete plugins.isolateMain;
|
|
67
|
+
const tailwindDisabled = options.tailwind === false;
|
|
68
|
+
if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
|
|
69
|
+
else if (tailwindDisabled) delete plugins.tailwind;
|
|
57
70
|
if (minimal) plugins.filter = options.filter || MINIMAL_FILTER_DEFAULT;
|
|
58
71
|
else if (options.filter) plugins.filter = options.filter;
|
|
72
|
+
const extraction = options.extraction;
|
|
59
73
|
let extractionHandlers;
|
|
60
|
-
if (
|
|
61
|
-
plugins.extraction = { selectors: Object.keys(
|
|
62
|
-
extractionHandlers =
|
|
74
|
+
if (extraction) {
|
|
75
|
+
plugins.extraction = { selectors: Object.keys(extraction) };
|
|
76
|
+
extractionHandlers = extraction;
|
|
63
77
|
}
|
|
64
|
-
|
|
78
|
+
const tagOverrides = options.tagOverrides;
|
|
79
|
+
if (tagOverrides) {
|
|
65
80
|
const overrides = {};
|
|
66
|
-
for (const tag in
|
|
67
|
-
const
|
|
68
|
-
if (
|
|
81
|
+
for (const tag in tagOverrides) {
|
|
82
|
+
const value = tagOverrides[tag];
|
|
83
|
+
if (value) overrides[tag] = typeof value === "string" ? { alias: value } : value;
|
|
69
84
|
}
|
|
70
85
|
plugins.tagOverrides = overrides;
|
|
71
86
|
}
|
|
@@ -76,14 +91,14 @@ function resolveOptions(options) {
|
|
|
76
91
|
cleanUrls,
|
|
77
92
|
clean,
|
|
78
93
|
plugins,
|
|
79
|
-
wrapWidth: options.wrapWidth
|
|
94
|
+
wrapWidth: options.wrapWidth,
|
|
95
|
+
format: options.format
|
|
80
96
|
},
|
|
81
97
|
extractionHandlers,
|
|
82
98
|
frontmatterCallback
|
|
83
99
|
};
|
|
84
100
|
}
|
|
85
101
|
function htmlToMarkdown(html, options = {}) {
|
|
86
|
-
if (Array.isArray(options.plugins)) throw new TypeError("Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://mdream.dev/v1-migration#custom-plugins");
|
|
87
102
|
const { napiOpts, extractionHandlers, frontmatterCallback } = resolveOptions(options);
|
|
88
103
|
const napiResult = htmlToMarkdown$1(html, napiOpts);
|
|
89
104
|
if (napiResult.frontmatter && frontmatterCallback) frontmatterCallback(napiResult.frontmatter);
|
|
@@ -100,10 +115,15 @@ async function* streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
|
100
115
|
while (true) {
|
|
101
116
|
const { done, value } = await reader.read();
|
|
102
117
|
if (done) break;
|
|
103
|
-
const chunk = typeof value === "string" ? value : decoder.decode(value);
|
|
118
|
+
const chunk = typeof value === "string" ? decoder.decode() + value : decoder.decode(value, { stream: true });
|
|
104
119
|
const processed = stream.processChunk(chunk);
|
|
105
120
|
if (processed) yield processed;
|
|
106
121
|
}
|
|
122
|
+
const decoderTail = decoder.decode();
|
|
123
|
+
if (decoderTail) {
|
|
124
|
+
const processed = stream.processChunk(decoderTail);
|
|
125
|
+
if (processed) yield processed;
|
|
126
|
+
}
|
|
107
127
|
const final_ = stream.finish();
|
|
108
128
|
if (final_) yield final_;
|
|
109
129
|
} finally {
|
package/dist/worker.d.mts
CHANGED
|
@@ -75,6 +75,11 @@ interface MdreamOptions {
|
|
|
75
75
|
* tables, and headings are never wrapped. `0` disables wrapping.
|
|
76
76
|
*/
|
|
77
77
|
wrapWidth?: number;
|
|
78
|
+
/**
|
|
79
|
+
* Output format. Defaults to `markdown`; use `text` to omit Markdown/HTML
|
|
80
|
+
* markup while preserving readable text and block spacing.
|
|
81
|
+
*/
|
|
82
|
+
format?: 'markdown' | 'text';
|
|
78
83
|
}
|
|
79
84
|
/**
|
|
80
85
|
* Initialize the mdream web worker. Must be called before htmlToMarkdown.
|
package/napi/index.d.mts
CHANGED
package/napi/index.d.ts
CHANGED
package/napi/index.mjs
CHANGED
|
@@ -3,11 +3,11 @@
|
|
|
3
3
|
// @ts-nocheck
|
|
4
4
|
/* auto-generated by NAPI-RS */
|
|
5
5
|
|
|
6
|
-
import { createRequire } from '
|
|
6
|
+
import { createRequire } from 'module'
|
|
7
7
|
const require = createRequire(import.meta.url)
|
|
8
8
|
const __dirname = new URL('.', import.meta.url).pathname
|
|
9
9
|
|
|
10
|
-
const { readFileSync } = require('
|
|
10
|
+
const { readFileSync } = require('fs')
|
|
11
11
|
let nativeBinding = null
|
|
12
12
|
const loadErrors = []
|
|
13
13
|
|
|
@@ -37,7 +37,7 @@ const isMuslFromFilesystem = () => {
|
|
|
37
37
|
|
|
38
38
|
const isMuslFromReport = () => {
|
|
39
39
|
let report = null
|
|
40
|
-
if (typeof process.report
|
|
40
|
+
if (process.report && typeof process.report.getReport === 'function') {
|
|
41
41
|
process.report.excludeNetwork = true
|
|
42
42
|
report = process.report.getReport()
|
|
43
43
|
}
|
|
@@ -109,7 +109,7 @@ function requireNative() {
|
|
|
109
109
|
}
|
|
110
110
|
} else if (process.platform === 'win32') {
|
|
111
111
|
if (process.arch === 'x64') {
|
|
112
|
-
if (process.config
|
|
112
|
+
if ((process.config && process.config.variables && process.config.variables.shlib_suffix === 'dll.a') || (process.config && process.config.variables && process.config.variables.node_target_type === 'shared_library')) {
|
|
113
113
|
try {
|
|
114
114
|
return require('./rust.win32-x64-gnu.node')
|
|
115
115
|
} catch (e) {
|
|
@@ -574,17 +574,18 @@ if (!nativeBinding || forceWasi) {
|
|
|
574
574
|
|
|
575
575
|
if (!nativeBinding) {
|
|
576
576
|
if (loadErrors.length > 0) {
|
|
577
|
-
|
|
577
|
+
const error = new Error(
|
|
578
578
|
`Cannot find native binding. ` +
|
|
579
579
|
`npm has a bug related to optional dependencies (https://github.com/npm/cli/issues/4828). ` +
|
|
580
580
|
'Please try `npm i` again after removing both package-lock.json and node_modules directory.',
|
|
581
|
-
{
|
|
582
|
-
cause: loadErrors.reduce((err, cur) => {
|
|
583
|
-
cur.cause = err
|
|
584
|
-
return cur
|
|
585
|
-
}),
|
|
586
|
-
},
|
|
587
581
|
)
|
|
582
|
+
// assign instead of the `new Error(message, { cause })` options form,
|
|
583
|
+
// which Node < 16.9 silently ignores
|
|
584
|
+
error.cause = loadErrors.reduce((err, cur) => {
|
|
585
|
+
cur.cause = err
|
|
586
|
+
return cur
|
|
587
|
+
})
|
|
588
|
+
throw error
|
|
588
589
|
}
|
|
589
590
|
throw new Error(`Failed to load native binding`)
|
|
590
591
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mdream",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.
|
|
4
|
+
"version": "1.5.2",
|
|
5
5
|
"description": "Ultra-performant HTML to Markdown Convertor Optimized for LLMs and llm.txt artifacts.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -33,7 +33,12 @@
|
|
|
33
33
|
"default": "./dist/worker.mjs"
|
|
34
34
|
},
|
|
35
35
|
"default": "./dist/worker.mjs"
|
|
36
|
-
}
|
|
36
|
+
},
|
|
37
|
+
"./wasm": {
|
|
38
|
+
"types": "./wasm/mdream_edge.d.ts",
|
|
39
|
+
"default": "./wasm/mdream_edge.js"
|
|
40
|
+
},
|
|
41
|
+
"./wasm/*": "./wasm/*"
|
|
37
42
|
},
|
|
38
43
|
"main": "./dist/index.mjs",
|
|
39
44
|
"unpkg": "./dist/iife.js",
|
|
@@ -52,22 +57,22 @@
|
|
|
52
57
|
"browser": "./dist/browser.mjs",
|
|
53
58
|
"dependencies": {},
|
|
54
59
|
"optionalDependencies": {
|
|
55
|
-
"@mdream/rust-
|
|
56
|
-
"@mdream/rust-android-arm64": "1.
|
|
57
|
-
"@mdream/rust-darwin-arm64": "1.
|
|
58
|
-
"@mdream/rust-
|
|
59
|
-
"@mdream/rust-
|
|
60
|
-
"@mdream/rust-linux-arm-gnueabihf": "1.
|
|
61
|
-
"@mdream/rust-linux-arm64-gnu": "1.
|
|
62
|
-
"@mdream/rust-linux-arm64-musl": "1.
|
|
63
|
-
"@mdream/rust-linux-x64-
|
|
64
|
-
"@mdream/rust-
|
|
65
|
-
"@mdream/rust-win32-arm64-msvc": "1.
|
|
66
|
-
"@mdream/rust-
|
|
67
|
-
"@mdream/rust-
|
|
60
|
+
"@mdream/rust-darwin-x64": "1.5.2",
|
|
61
|
+
"@mdream/rust-android-arm64": "1.5.2",
|
|
62
|
+
"@mdream/rust-darwin-arm64": "1.5.2",
|
|
63
|
+
"@mdream/rust-freebsd-x64": "1.5.2",
|
|
64
|
+
"@mdream/rust-android-arm-eabi": "1.5.2",
|
|
65
|
+
"@mdream/rust-linux-arm-gnueabihf": "1.5.2",
|
|
66
|
+
"@mdream/rust-linux-arm64-gnu": "1.5.2",
|
|
67
|
+
"@mdream/rust-linux-arm64-musl": "1.5.2",
|
|
68
|
+
"@mdream/rust-linux-x64-gnu": "1.5.2",
|
|
69
|
+
"@mdream/rust-linux-x64-musl": "1.5.2",
|
|
70
|
+
"@mdream/rust-win32-arm64-msvc": "1.5.2",
|
|
71
|
+
"@mdream/rust-wasm32-wasi": "1.5.2",
|
|
72
|
+
"@mdream/rust-win32-x64-msvc": "1.5.2"
|
|
68
73
|
},
|
|
69
74
|
"devDependencies": {
|
|
70
|
-
"@mdream/js": "1.
|
|
75
|
+
"@mdream/js": "1.5.2"
|
|
71
76
|
},
|
|
72
77
|
"scripts": {
|
|
73
78
|
"flame": "pnpm build && unbuild bench/bundle && clinic flame -- node bench/bundle/dist/string.mjs 10",
|
package/wasm/mdream_edge_bg.wasm
CHANGED
|
Binary file
|
package/wasm/package.json
CHANGED
|
Binary file
|