mdream 1.4.1 → 1.5.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.mts CHANGED
@@ -75,6 +75,11 @@ interface MdreamOptions {
75
75
  * tables, and headings are never wrapped. `0` disables wrapping.
76
76
  */
77
77
  wrapWidth?: number;
78
+ /**
79
+ * Output format. Defaults to `markdown`; use `text` to omit Markdown/HTML
80
+ * markup while preserving readable text and block spacing.
81
+ */
82
+ format?: 'markdown' | 'text';
78
83
  }
79
84
  declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string;
80
85
  declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
package/dist/index.mjs CHANGED
@@ -23,12 +23,16 @@ const CLEAN_ALL = {
23
23
  emptyLinkText: true
24
24
  };
25
25
  function resolveCleanConfig(options, minimal) {
26
+ const cleanUrls = options.cleanUrls === true;
26
27
  let cleanOpt = options.clean;
27
- if (cleanOpt === void 0 && minimal) cleanOpt = true;
28
- if (!cleanOpt) return { cleanUrls: false };
28
+ if (cleanOpt === void 0) {
29
+ if (!minimal) return { cleanUrls };
30
+ cleanOpt = true;
31
+ }
32
+ if (!cleanOpt) return { cleanUrls };
29
33
  const resolved = cleanOpt === true ? CLEAN_ALL : cleanOpt;
30
34
  return {
31
- cleanUrls: resolved.urls || false,
35
+ cleanUrls: cleanUrls || resolved.urls === true,
32
36
  clean: resolved
33
37
  };
34
38
  }
@@ -37,35 +41,46 @@ function resolveFrontmatter(opt) {
37
41
  config: {},
38
42
  callback: opt
39
43
  };
40
- if (typeof opt === "object") return {
41
- config: opt,
42
- callback: opt.onExtract
43
- };
44
+ if (opt && typeof opt === "object") {
45
+ const { onExtract, ...config } = opt;
46
+ return {
47
+ config,
48
+ callback: onExtract
49
+ };
50
+ }
44
51
  return { config: {} };
45
52
  }
46
53
  function resolveOptions(options) {
54
+ if (Array.isArray(options.plugins)) throw new TypeError("Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://mdream.dev/v1-migration#custom-plugins");
47
55
  const minimal = options.minimal === true;
48
- const plugins = {};
56
+ const plugins = options.plugins ? { ...options.plugins } : {};
49
57
  let frontmatterCallback;
50
- if (minimal ? options.frontmatter !== false : !!options.frontmatter) {
58
+ const frontmatterDisabled = options.frontmatter === false;
59
+ if (minimal ? !frontmatterDisabled : !!options.frontmatter) {
51
60
  const fm = resolveFrontmatter(options.frontmatter);
52
61
  plugins.frontmatter = fm.config;
53
62
  frontmatterCallback = fm.callback;
54
- }
55
- if (minimal ? options.isolateMain !== false : options.isolateMain) plugins.isolateMain = true;
56
- if (minimal ? options.tailwind !== false : options.tailwind) plugins.tailwind = true;
63
+ } else if (frontmatterDisabled) delete plugins.frontmatter;
64
+ const isolateMainDisabled = options.isolateMain === false;
65
+ if (minimal ? !isolateMainDisabled : options.isolateMain) plugins.isolateMain = true;
66
+ else if (isolateMainDisabled) delete plugins.isolateMain;
67
+ const tailwindDisabled = options.tailwind === false;
68
+ if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
69
+ else if (tailwindDisabled) delete plugins.tailwind;
57
70
  if (minimal) plugins.filter = options.filter || MINIMAL_FILTER_DEFAULT;
58
71
  else if (options.filter) plugins.filter = options.filter;
72
+ const extraction = options.extraction;
59
73
  let extractionHandlers;
60
- if (options.extraction) {
61
- plugins.extraction = { selectors: Object.keys(options.extraction) };
62
- extractionHandlers = options.extraction;
74
+ if (extraction) {
75
+ plugins.extraction = { selectors: Object.keys(extraction) };
76
+ extractionHandlers = extraction;
63
77
  }
64
- if (options.tagOverrides) {
78
+ const tagOverrides = options.tagOverrides;
79
+ if (tagOverrides) {
65
80
  const overrides = {};
66
- for (const tag in options.tagOverrides) {
67
- const v = options.tagOverrides[tag];
68
- if (v) overrides[tag] = typeof v === "string" ? { alias: v } : v;
81
+ for (const tag in tagOverrides) {
82
+ const value = tagOverrides[tag];
83
+ if (value) overrides[tag] = typeof value === "string" ? { alias: value } : value;
69
84
  }
70
85
  plugins.tagOverrides = overrides;
71
86
  }
@@ -76,14 +91,14 @@ function resolveOptions(options) {
76
91
  cleanUrls,
77
92
  clean,
78
93
  plugins,
79
- wrapWidth: options.wrapWidth
94
+ wrapWidth: options.wrapWidth,
95
+ format: options.format
80
96
  },
81
97
  extractionHandlers,
82
98
  frontmatterCallback
83
99
  };
84
100
  }
85
101
  function htmlToMarkdown(html, options = {}) {
86
- if (Array.isArray(options.plugins)) throw new TypeError("Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://mdream.dev/v1-migration#custom-plugins");
87
102
  const { napiOpts, extractionHandlers, frontmatterCallback } = resolveOptions(options);
88
103
  const napiResult = htmlToMarkdown$1(html, napiOpts);
89
104
  if (napiResult.frontmatter && frontmatterCallback) frontmatterCallback(napiResult.frontmatter);
@@ -100,10 +115,15 @@ async function* streamHtmlToMarkdown(htmlStream, options = {}) {
100
115
  while (true) {
101
116
  const { done, value } = await reader.read();
102
117
  if (done) break;
103
- const chunk = typeof value === "string" ? value : decoder.decode(value);
118
+ const chunk = typeof value === "string" ? decoder.decode() + value : decoder.decode(value, { stream: true });
104
119
  const processed = stream.processChunk(chunk);
105
120
  if (processed) yield processed;
106
121
  }
122
+ const decoderTail = decoder.decode();
123
+ if (decoderTail) {
124
+ const processed = stream.processChunk(decoderTail);
125
+ if (processed) yield processed;
126
+ }
107
127
  const final_ = stream.finish();
108
128
  if (final_) yield final_;
109
129
  } finally {
package/dist/worker.d.mts CHANGED
@@ -75,6 +75,11 @@ interface MdreamOptions {
75
75
  * tables, and headings are never wrapped. `0` disables wrapping.
76
76
  */
77
77
  wrapWidth?: number;
78
+ /**
79
+ * Output format. Defaults to `markdown`; use `text` to omit Markdown/HTML
80
+ * markup while preserving readable text and block spacing.
81
+ */
82
+ format?: 'markdown' | 'text';
78
83
  }
79
84
  /**
80
85
  * Initialize the mdream web worker. Must be called before htmlToMarkdown.
package/napi/index.d.mts CHANGED
@@ -63,6 +63,7 @@ export interface HtmlToMarkdownOptions {
63
63
  clean?: CleanOptionsNapi
64
64
  plugins?: PluginOptions
65
65
  wrapWidth?: number
66
+ format?: "markdown" | "text"
66
67
  }
67
68
 
68
69
  export interface MarkdownChunkNapi {
package/napi/index.d.ts CHANGED
@@ -63,6 +63,7 @@ export interface HtmlToMarkdownOptions {
63
63
  clean?: CleanOptionsNapi
64
64
  plugins?: PluginOptions
65
65
  wrapWidth?: number
66
+ format?: "markdown" | "text"
66
67
  }
67
68
 
68
69
  export interface MarkdownChunkNapi {
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "mdream",
3
3
  "type": "module",
4
- "version": "1.4.1",
4
+ "version": "1.5.3",
5
5
  "description": "Ultra-performant HTML to Markdown Convertor Optimized for LLMs and llm.txt artifacts.",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",
@@ -33,7 +33,12 @@
33
33
  "default": "./dist/worker.mjs"
34
34
  },
35
35
  "default": "./dist/worker.mjs"
36
- }
36
+ },
37
+ "./wasm": {
38
+ "types": "./wasm/mdream_edge.d.ts",
39
+ "default": "./wasm/mdream_edge.js"
40
+ },
41
+ "./wasm/*": "./wasm/*"
37
42
  },
38
43
  "main": "./dist/index.mjs",
39
44
  "unpkg": "./dist/iife.js",
@@ -52,22 +57,22 @@
52
57
  "browser": "./dist/browser.mjs",
53
58
  "dependencies": {},
54
59
  "optionalDependencies": {
55
- "@mdream/rust-android-arm64": "1.4.1",
56
- "@mdream/rust-darwin-arm64": "1.4.1",
57
- "@mdream/rust-darwin-x64": "1.4.1",
58
- "@mdream/rust-android-arm-eabi": "1.4.1",
59
- "@mdream/rust-linux-arm-gnueabihf": "1.4.1",
60
- "@mdream/rust-linux-arm64-gnu": "1.4.1",
61
- "@mdream/rust-freebsd-x64": "1.4.1",
62
- "@mdream/rust-linux-arm64-musl": "1.4.1",
63
- "@mdream/rust-linux-x64-gnu": "1.4.1",
64
- "@mdream/rust-wasm32-wasi": "1.4.1",
65
- "@mdream/rust-linux-x64-musl": "1.4.1",
66
- "@mdream/rust-win32-arm64-msvc": "1.4.1",
67
- "@mdream/rust-win32-x64-msvc": "1.4.1"
60
+ "@mdream/rust-android-arm-eabi": "1.5.3",
61
+ "@mdream/rust-darwin-x64": "1.5.3",
62
+ "@mdream/rust-darwin-arm64": "1.5.3",
63
+ "@mdream/rust-freebsd-x64": "1.5.3",
64
+ "@mdream/rust-linux-arm-gnueabihf": "1.5.3",
65
+ "@mdream/rust-linux-arm64-gnu": "1.5.3",
66
+ "@mdream/rust-linux-arm64-musl": "1.5.3",
67
+ "@mdream/rust-linux-x64-gnu": "1.5.3",
68
+ "@mdream/rust-linux-x64-musl": "1.5.3",
69
+ "@mdream/rust-android-arm64": "1.5.3",
70
+ "@mdream/rust-wasm32-wasi": "1.5.3",
71
+ "@mdream/rust-win32-arm64-msvc": "1.5.3",
72
+ "@mdream/rust-win32-x64-msvc": "1.5.3"
68
73
  },
69
74
  "devDependencies": {
70
- "@mdream/js": "1.4.1"
75
+ "@mdream/js": "1.5.3"
71
76
  },
72
77
  "scripts": {
73
78
  "flame": "pnpm build && unbuild bench/bundle && clinic flame -- node bench/bundle/dist/string.mjs 10",
Binary file
package/wasm/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "mdream-edge",
3
3
  "type": "module",
4
4
  "description": "WebAssembly edge runtime bindings for mdream HTML-to-Markdown converter",
5
- "version": "1.4.1",
5
+ "version": "1.5.3",
6
6
  "license": "MIT",
7
7
  "files": [
8
8
  "mdream_edge_bg.wasm",
Binary file
@@ -2,7 +2,7 @@
2
2
  "name": "mdream-edge",
3
3
  "type": "module",
4
4
  "description": "WebAssembly edge runtime bindings for mdream HTML-to-Markdown converter",
5
- "version": "1.4.1",
5
+ "version": "1.5.3",
6
6
  "license": "MIT",
7
7
  "files": [
8
8
  "mdream_edge_bg.wasm",