mdream 1.4.0 → 1.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -116,6 +116,11 @@ const markdown = htmlToMarkdown('<h1>Hello World</h1><p>Some content.</p>')
116
116
  // # Hello World
117
117
  //
118
118
  // Some content.
119
+
120
+ const text = htmlToMarkdown('<h1>Hello <strong>World</strong></h1>', {
121
+ format: 'text',
122
+ })
123
+ // Hello World
119
124
  ```
120
125
 
121
126
  ### `streamHtmlToMarkdown()`
@@ -220,6 +225,9 @@ interface MdreamOptions {
220
225
  * tables, and headings are never wrapped. `0` (or unset) disables wrapping.
221
226
  */
222
227
  wrapWidth?: number
228
+
229
+ /** Output Markdown or plain text. Default: 'markdown' */
230
+ format?: 'markdown' | 'text'
223
231
  }
224
232
  ```
225
233
 
@@ -244,6 +252,9 @@ interface EngineOptions {
244
252
  * disables wrapping.
245
253
  */
246
254
  wrapWidth?: number
255
+
256
+ /** Output Markdown or plain text. Default: 'markdown' */
257
+ format?: 'markdown' | 'text'
247
258
  }
248
259
 
249
260
  interface BuiltinPlugins {
@@ -880,6 +891,14 @@ cat index.html \
880
891
  | tee output.md
881
892
  ```
882
893
 
894
+ **Plain text output:**
895
+
896
+ ```bash
897
+ cat index.html \
898
+ | npx mdream --format text \
899
+ | tee output.txt
900
+ ```
901
+
883
902
  ### CLI Options
884
903
 
885
904
  | Option | Description |
@@ -887,9 +906,11 @@ cat index.html \
887
906
  | `--origin <url>` | Base URL for resolving relative links and images |
888
907
  | `--preset minimal` | Enable the minimal preset |
889
908
  | `--wrap-width <n>` | Hard-wrap prose at `n` characters (code, tables, and headings are never wrapped) |
909
+ | `--format <format>` | Output format: `markdown`, `text` |
910
+ | `--text` | Alias for `--format text` |
890
911
  | `-h`, `--help` | Display help information |
891
912
 
892
- The CLI reads HTML from stdin and writes Markdown to stdout. It uses the streaming API internally.
913
+ The CLI reads HTML from stdin and writes Markdown or plain text to stdout. It uses the streaming API internally.
893
914
 
894
915
  ## Browser and Edge Usage
895
916
 
@@ -910,6 +931,16 @@ for await (const chunk of streamHtmlToMarkdown(response.body)) {
910
931
  }
911
932
  ```
912
933
 
934
+ If your toolchain doesn't resolve the export conditions, the raw wasm-bindgen build (web target) is exposed at `mdream/wasm` for manual initialization:
935
+
936
+ ```ts
937
+ import init, { htmlToMarkdown } from 'mdream/wasm'
938
+ import wasmModule from 'mdream/wasm/mdream_edge_bg.wasm'
939
+
940
+ await init({ module_or_path: wasmModule })
941
+ const markdown = htmlToMarkdown('<h1>Hello</h1>', {})
942
+ ```
943
+
913
944
  You can also import the edge entry point directly:
914
945
 
915
946
  ```ts
package/bin/mdream.mjs CHANGED
@@ -6,6 +6,7 @@ const args = process.argv.slice(2)
6
6
  let origin
7
7
  let preset
8
8
  let wrapWidth
9
+ let format
9
10
  for (let i = 0; i < args.length; i++) {
10
11
  if (args[i] === '--origin' && args[i + 1]) {
11
12
  origin = args[++i]
@@ -16,13 +17,28 @@ for (let i = 0; i < args.length; i++) {
16
17
  else if (args[i] === '--wrap-width' && args[i + 1]) {
17
18
  wrapWidth = Number.parseInt(args[++i], 10) || undefined
18
19
  }
20
+ else if (args[i] === '--format') {
21
+ const value = args[++i]
22
+ if (!value) {
23
+ process.stderr.write('--format requires a value: markdown or text\n')
24
+ process.exit(1)
25
+ }
26
+ if (value !== 'markdown' && value !== 'text') {
27
+ process.stderr.write(`Unknown format: ${value}\n`)
28
+ process.exit(1)
29
+ }
30
+ format = value
31
+ }
32
+ else if (args[i] === '--text') {
33
+ format = 'text'
34
+ }
19
35
  else if (args[i] === '-h' || args[i] === '--help') {
20
- process.stdout.write('Usage: mdream [--origin <url>] [--preset minimal] [--wrap-width <n>]\nPipe HTML via stdin, outputs Markdown to stdout.\n')
36
+ process.stdout.write('Usage: mdream [--origin <url>] [--preset minimal] [--wrap-width <n>] [--format markdown|text] [--text]\nPipe HTML via stdin, outputs Markdown or plain text to stdout.\n')
21
37
  process.exit(0)
22
38
  }
23
39
  }
24
40
 
25
- const options = { origin, minimal: preset === 'minimal', wrapWidth }
41
+ const options = { origin, minimal: preset === 'minimal', wrapWidth, format }
26
42
  const stream = Readable.toWeb(process.stdin)
27
43
  for await (const chunk of streamHtmlToMarkdown(stream, options)) {
28
44
  if (chunk?.length)
package/dist/browser.mjs CHANGED
@@ -35,10 +35,15 @@ async function* streamHtmlToMarkdown(htmlStream, options) {
35
35
  while (true) {
36
36
  const { done, value } = await reader.read();
37
37
  if (done) break;
38
- const chunk = typeof value === "string" ? value : decoder.decode(value);
38
+ const chunk = typeof value === "string" ? decoder.decode() + value : decoder.decode(value, { stream: true });
39
39
  const processed = stream.processChunk(chunk);
40
40
  if (processed) yield processed;
41
41
  }
42
+ const decoderTail = decoder.decode();
43
+ if (decoderTail) {
44
+ const processed = stream.processChunk(decoderTail);
45
+ if (processed) yield processed;
46
+ }
42
47
  const final_ = stream.finish();
43
48
  if (final_) yield final_;
44
49
  } finally {
package/dist/edge.mjs CHANGED
@@ -1,11 +1,117 @@
1
- import { MarkdownStream as MarkdownStream$1, htmlToMarkdownResult } from "../wasm-bundler/mdream_edge.js";
1
+ import { MarkdownStream as MarkdownStream$1, htmlToMarkdownResult, initSync } from "../wasm/mdream_edge.js";
2
+ import wasmModule from "../wasm/mdream_edge_bg.wasm";
3
+ const MINIMAL_FILTER_DEFAULT = { exclude: [
4
+ "form",
5
+ "fieldset",
6
+ "object",
7
+ "embed",
8
+ "footer",
9
+ "aside",
10
+ "iframe",
11
+ "input",
12
+ "textarea",
13
+ "select",
14
+ "button",
15
+ "nav"
16
+ ] };
17
+ const CLEAN_ALL = {
18
+ urls: true,
19
+ fragments: true,
20
+ emptyLinks: true,
21
+ redundantLinks: true,
22
+ selfLinkHeadings: true,
23
+ emptyImages: true,
24
+ emptyLinkText: true
25
+ };
26
+ function resolveCleanConfig(options, minimal) {
27
+ const cleanUrls = options.cleanUrls === true;
28
+ let cleanOpt = options.clean;
29
+ if (cleanOpt === void 0) {
30
+ if (!minimal) return { cleanUrls };
31
+ cleanOpt = true;
32
+ }
33
+ if (!cleanOpt) return { cleanUrls };
34
+ const resolved = cleanOpt === true ? CLEAN_ALL : cleanOpt;
35
+ return {
36
+ cleanUrls: cleanUrls || resolved.urls === true,
37
+ clean: resolved
38
+ };
39
+ }
40
+ function resolveFrontmatter(opt) {
41
+ if (typeof opt === "function") return {
42
+ config: {},
43
+ callback: opt
44
+ };
45
+ if (opt && typeof opt === "object") {
46
+ const { onExtract, ...config } = opt;
47
+ return {
48
+ config,
49
+ callback: onExtract
50
+ };
51
+ }
52
+ return { config: {} };
53
+ }
54
+ function resolveOptions(options) {
55
+ if (Array.isArray(options.plugins)) throw new TypeError("Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://mdream.dev/v1-migration#custom-plugins");
56
+ const minimal = options.minimal === true;
57
+ const plugins = options.plugins ? { ...options.plugins } : {};
58
+ let frontmatterCallback;
59
+ const frontmatterDisabled = options.frontmatter === false;
60
+ if (minimal ? !frontmatterDisabled : !!options.frontmatter) {
61
+ const fm = resolveFrontmatter(options.frontmatter);
62
+ plugins.frontmatter = fm.config;
63
+ frontmatterCallback = fm.callback;
64
+ } else if (frontmatterDisabled) delete plugins.frontmatter;
65
+ const isolateMainDisabled = options.isolateMain === false;
66
+ if (minimal ? !isolateMainDisabled : options.isolateMain) plugins.isolateMain = true;
67
+ else if (isolateMainDisabled) delete plugins.isolateMain;
68
+ const tailwindDisabled = options.tailwind === false;
69
+ if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
70
+ else if (tailwindDisabled) delete plugins.tailwind;
71
+ if (minimal) plugins.filter = options.filter || MINIMAL_FILTER_DEFAULT;
72
+ else if (options.filter) plugins.filter = options.filter;
73
+ const extraction = options.extraction;
74
+ let extractionHandlers;
75
+ if (extraction) {
76
+ plugins.extraction = { selectors: Object.keys(extraction) };
77
+ extractionHandlers = extraction;
78
+ }
79
+ const tagOverrides = options.tagOverrides;
80
+ if (tagOverrides) {
81
+ const overrides = {};
82
+ for (const tag in tagOverrides) {
83
+ const value = tagOverrides[tag];
84
+ if (value) overrides[tag] = typeof value === "string" ? { alias: value } : value;
85
+ }
86
+ plugins.tagOverrides = overrides;
87
+ }
88
+ const { cleanUrls, clean } = resolveCleanConfig(options, minimal);
89
+ return {
90
+ napiOpts: {
91
+ origin: options.origin,
92
+ cleanUrls,
93
+ clean,
94
+ plugins,
95
+ wrapWidth: options.wrapWidth,
96
+ format: options.format
97
+ },
98
+ extractionHandlers,
99
+ frontmatterCallback
100
+ };
101
+ }
102
+ initSync({ module: wasmModule });
2
103
  function htmlToMarkdown(html, options) {
3
- return htmlToMarkdownResult(html, options || {}).markdown || "";
104
+ if (!options) return htmlToMarkdownResult(html, void 0).markdown || "";
105
+ const { napiOpts, extractionHandlers, frontmatterCallback } = resolveOptions(options);
106
+ const result = htmlToMarkdownResult(html, napiOpts);
107
+ if (result.frontmatter && frontmatterCallback) frontmatterCallback(result.frontmatter);
108
+ if (result.extracted?.length && extractionHandlers) for (const element of result.extracted) extractionHandlers[element.selector]?.(element);
109
+ return result.markdown || "";
4
110
  }
5
111
  var MarkdownStream = class {
6
112
  _inner;
7
113
  constructor(options) {
8
- this._inner = new MarkdownStream$1(options || {});
114
+ this._inner = new MarkdownStream$1(options);
9
115
  }
10
116
  processChunk(chunk) {
11
117
  return this._inner.processChunk(chunk);
@@ -16,17 +122,22 @@ var MarkdownStream = class {
16
122
  };
17
123
  async function* streamHtmlToMarkdown(htmlStream, options) {
18
124
  if (!htmlStream) throw new Error("Invalid HTML stream provided");
19
- const stream = new MarkdownStream$1(options || {});
125
+ const stream = new MarkdownStream$1(options ? resolveOptions(options).napiOpts : void 0);
20
126
  const reader = htmlStream.getReader();
21
127
  const decoder = new TextDecoder();
22
128
  try {
23
129
  while (true) {
24
130
  const { done, value } = await reader.read();
25
131
  if (done) break;
26
- const chunk = typeof value === "string" ? value : decoder.decode(value);
132
+ const chunk = typeof value === "string" ? decoder.decode() + value : decoder.decode(value, { stream: true });
27
133
  const processed = stream.processChunk(chunk);
28
134
  if (processed) yield processed;
29
135
  }
136
+ const decoderTail = decoder.decode();
137
+ if (decoderTail) {
138
+ const processed = stream.processChunk(decoderTail);
139
+ if (processed) yield processed;
140
+ }
30
141
  const final_ = stream.finish();
31
142
  if (final_) yield final_;
32
143
  } finally {