mdream 2.0.0-beta.1 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/bin/mdream.mjs CHANGED
@@ -2,40 +2,61 @@
2
2
  import { Readable } from 'node:stream'
3
3
  import { streamHtmlToMarkdown } from 'mdream'
4
4
 
5
+ const USAGE = 'Usage: mdream [--origin <url>] [--preset minimal] [--wrap-width <n>] [--format markdown|text|html] [--text]\nPipe HTML via stdin, outputs Markdown, plain text, or safe HTML to stdout.\n'
6
+
7
+ function fail(message) {
8
+ process.stderr.write(`${message}\n`)
9
+ process.exit(1)
10
+ }
11
+
5
12
  const args = process.argv.slice(2)
6
13
  let origin
7
14
  let preset
8
15
  let wrapWidth
9
16
  let format
10
17
  for (let i = 0; i < args.length; i++) {
11
- if (args[i] === '--origin' && args[i + 1]) {
12
- origin = args[++i]
13
- }
14
- else if (args[i] === '--preset' && args[i + 1]) {
15
- preset = args[++i]
16
- }
17
- else if (args[i] === '--wrap-width' && args[i + 1]) {
18
- wrapWidth = Number.parseInt(args[++i], 10) || undefined
19
- }
20
- else if (args[i] === '--format') {
21
- const value = args[++i]
22
- if (!value) {
23
- process.stderr.write('--format requires a value: markdown, text, or html\n')
24
- process.exit(1)
25
- }
26
- if (value !== 'markdown' && value !== 'text' && value !== 'html') {
27
- process.stderr.write(`Unknown format: ${value}\n`)
28
- process.exit(1)
29
- }
30
- format = value
31
- }
32
- else if (args[i] === '--text') {
18
+ const arg = args[i]
19
+ // `--flag=value` and `--flag value` both work.
20
+ const eq = arg.startsWith('--') ? arg.indexOf('=') : -1
21
+ const flag = eq === -1 ? arg : arg.slice(0, eq)
22
+ const takeValue = () => {
23
+ const value = eq === -1 ? args[++i] : arg.slice(eq + 1)
24
+ // A following flag, such as `--text` or `-h`, is not a value.
25
+ if (!value || (eq === -1 && (value.startsWith('--') || value === '-h')))
26
+ fail(`The ${flag} option needs a value.`)
27
+ return value
28
+ }
29
+ if (flag === '--origin') {
30
+ origin = takeValue()
31
+ }
32
+ else if (flag === '--preset') {
33
+ preset = takeValue()
34
+ if (preset !== 'minimal')
35
+ fail(`Unknown preset: ${preset}. Use --preset minimal.`)
36
+ }
37
+ else if (flag === '--wrap-width') {
38
+ wrapWidth = Number.parseInt(takeValue(), 10) || undefined
39
+ }
40
+ else if (flag === '--format') {
41
+ format = takeValue()
42
+ if (format !== 'markdown' && format !== 'text' && format !== 'html')
43
+ fail(`Unknown format: ${format}. Use markdown, text, or html.`)
44
+ }
45
+ else if (flag === '--text') {
46
+ if (eq !== -1)
47
+ fail('The --text option takes no value.')
33
48
  format = 'text'
34
49
  }
35
- else if (args[i] === '-h' || args[i] === '--help') {
36
- process.stdout.write('Usage: mdream [--origin <url>] [--preset minimal] [--wrap-width <n>] [--format markdown|text|html] [--text]\nPipe HTML via stdin, outputs Markdown, plain text, or safe HTML to stdout.\n')
50
+ else if (arg === '-h' || arg === '--help') {
51
+ process.stdout.write(USAGE)
37
52
  process.exit(0)
38
53
  }
54
+ else if (arg.startsWith('-')) {
55
+ fail(`Unknown option: ${flag}. Run mdream --help to list the options.`)
56
+ }
57
+ else {
58
+ fail(`Unknown argument: ${arg}. Pipe the HTML to mdream on stdin.`)
59
+ }
39
60
  }
40
61
 
41
62
  const options = { origin, minimal: preset === 'minimal', wrapWidth, format }
@@ -0,0 +1,7 @@
1
+ function htmlToMarkdown() {
2
+ throw new TypeError("In a browser bundle, import htmlToMarkdown from 'mdream/browser'. It returns a Promise.");
3
+ }
4
+ function streamHtmlToMarkdown() {
5
+ throw new TypeError("In a browser bundle, import streamHtmlToMarkdown from 'mdream/browser'.");
6
+ }
7
+ export { htmlToMarkdown, streamHtmlToMarkdown };
@@ -5,8 +5,6 @@ interface CleanOptions {
5
5
  fragments?: boolean;
6
6
  /** Strip links with meaningless hrefs (#, javascript:void(0)) → plain text */
7
7
  emptyLinks?: boolean;
8
- /** Collapse 3+ consecutive blank lines to 2 */
9
- blankLines?: boolean;
10
8
  /** Strip links where text equals URL: [https://x.com](https://x.com) → https://x.com */
11
9
  redundantLinks?: boolean;
12
10
  /** Strip self-referencing heading anchors: ## [Title](#title) → ## Title */
@@ -87,7 +85,7 @@ interface MdreamOptions {
87
85
  }
88
86
  /**
89
87
  * Browser builds fetch the WASM binary on first use, so this returns a
90
- * Promise. The `browser` export condition ships types that say so.
88
+ * Promise. `mdream/browser` ships types that say so.
91
89
  */
92
90
  export declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): Promise<string>;
93
91
  export declare function createMarkdownStream(options?: Partial<MdreamOptions>): Promise<MarkdownStream>;
package/dist/browser.mjs CHANGED
@@ -22,19 +22,38 @@ const CLEAN_ALL = {
22
22
  emptyImages: true,
23
23
  emptyLinkText: true
24
24
  };
25
- function resolveCleanConfig(options, minimal) {
26
- const cleanUrls = options.cleanUrls === true;
27
- let cleanOpt = options.clean;
28
- if (cleanOpt === void 0) {
29
- if (!minimal) return { cleanUrls };
30
- cleanOpt = true;
25
+ const VALID_OPTIONS = "origin, clean, minimal, frontmatter, isolateMain, tailwind, filter, extraction, tagOverrides, wrapWidth, format";
26
+ function unknownOptionMessage(key, value) {
27
+ switch (key) {
28
+ case "preset": return "mdream has no `preset` option. Pass { minimal: true }.";
29
+ case "cleanUrls": return "mdream has no `cleanUrls` option. Pass { clean: { urls: true } }.";
30
+ case "plugins": return Array.isArray(value) ? "Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://github.com/harlan-zw/mdream/tree/main/packages/js#migrating-from-v1" : "mdream takes plugin options at the top level, such as { frontmatter: true }.";
31
31
  }
32
- if (!cleanOpt) return { cleanUrls };
33
- const resolved = cleanOpt === true ? CLEAN_ALL : cleanOpt;
34
- return {
35
- cleanUrls: cleanUrls || resolved.urls === true,
36
- clean: resolved
37
- };
32
+ return `mdream has no \`${key}\` option. Valid options: ${VALID_OPTIONS}.`;
33
+ }
34
+ function assertKnownOptions(options) {
35
+ for (const key in options) {
36
+ switch (key) {
37
+ case "origin":
38
+ case "clean":
39
+ case "minimal":
40
+ case "frontmatter":
41
+ case "isolateMain":
42
+ case "tailwind":
43
+ case "filter":
44
+ case "extraction":
45
+ case "tagOverrides":
46
+ case "wrapWidth":
47
+ case "format": continue;
48
+ }
49
+ const value = options[key];
50
+ if (value !== void 0) throw new TypeError(unknownOptionMessage(key, value));
51
+ }
52
+ }
53
+ function resolveClean(clean, minimal) {
54
+ if (clean === void 0) return minimal ? CLEAN_ALL : void 0;
55
+ if (!clean) return void 0;
56
+ return clean === true ? CLEAN_ALL : clean;
38
57
  }
39
58
  function resolveFrontmatter(opt) {
40
59
  if (typeof opt === "function") return {
@@ -51,24 +70,18 @@ function resolveFrontmatter(opt) {
51
70
  return { config: {} };
52
71
  }
53
72
  function resolveOptions(options) {
54
- if (Array.isArray(options.plugins)) throw new TypeError("Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://mdream.dev/v1-migration#custom-plugins");
73
+ assertKnownOptions(options);
55
74
  const minimal = options.minimal === true;
56
- const plugins = options.plugins ? { ...options.plugins } : {};
75
+ const plugins = {};
57
76
  let frontmatterCallback;
58
- const frontmatterDisabled = options.frontmatter === false;
59
- if (minimal ? !frontmatterDisabled : !!options.frontmatter) {
77
+ if (minimal ? options.frontmatter !== false : options.frontmatter) {
60
78
  const fm = resolveFrontmatter(options.frontmatter);
61
79
  plugins.frontmatter = fm.config;
62
80
  frontmatterCallback = fm.callback;
63
- } else if (frontmatterDisabled) delete plugins.frontmatter;
64
- const isolateMainDisabled = options.isolateMain === false;
65
- if (minimal ? !isolateMainDisabled : options.isolateMain) plugins.isolateMain = true;
66
- else if (isolateMainDisabled) delete plugins.isolateMain;
67
- const tailwindDisabled = options.tailwind === false;
68
- if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
69
- else if (tailwindDisabled) delete plugins.tailwind;
70
- if (options.filter === false) delete plugins.filter;
71
- else if (minimal) plugins.filter = {
81
+ }
82
+ if (minimal ? options.isolateMain !== false : options.isolateMain) plugins.isolateMain = true;
83
+ if (minimal ? options.tailwind !== false : options.tailwind) plugins.tailwind = true;
84
+ if (minimal && options.filter !== false) plugins.filter = {
72
85
  ...options.filter,
73
86
  exclude: [...MINIMAL_FILTER_EXCLUDE, ...options.filter?.exclude ?? []]
74
87
  };
@@ -88,12 +101,10 @@ function resolveOptions(options) {
88
101
  }
89
102
  plugins.tagOverrides = overrides;
90
103
  }
91
- const { cleanUrls, clean } = resolveCleanConfig(options, minimal);
92
104
  return {
93
105
  napiOpts: {
94
106
  origin: options.origin,
95
- cleanUrls,
96
- clean,
107
+ clean: resolveClean(options.clean, minimal),
97
108
  plugins,
98
109
  wrapWidth: options.wrapWidth,
99
110
  format: options.format
@@ -115,9 +126,9 @@ function convertResult(convert, html, options = {}) {
115
126
  deliverPluginData(result, resolved);
116
127
  return result;
117
128
  }
118
- async function* pumpStream(stream, htmlStream, callbacks, { decodeInJs = false, mapError = (error) => error } = {}) {
129
+ async function* pumpStream(stream, htmlStream, callbacks, { mapError = (error) => error } = {}) {
119
130
  const reader = htmlStream.getReader();
120
- const decoder = decodeInJs ? new TextDecoder() : void 0;
131
+ const decoder = stream.processChunkBytes ? void 0 : new TextDecoder();
121
132
  try {
122
133
  while (true) {
123
134
  const { done, value } = await reader.read();
package/dist/edge.d.mts CHANGED
@@ -5,8 +5,6 @@ interface CleanOptions {
5
5
  fragments?: boolean;
6
6
  /** Strip links with meaningless hrefs (#, javascript:void(0)) → plain text */
7
7
  emptyLinks?: boolean;
8
- /** Collapse 3+ consecutive blank lines to 2 */
9
- blankLines?: boolean;
10
8
  /** Strip links where text equals URL: [https://x.com](https://x.com) → https://x.com */
11
9
  redundantLinks?: boolean;
12
10
  /** Strip self-referencing heading anchors: ## [Title](#title) → ## Title */
package/dist/edge.mjs CHANGED
@@ -22,19 +22,38 @@ const CLEAN_ALL = {
22
22
  emptyImages: true,
23
23
  emptyLinkText: true
24
24
  };
25
- function resolveCleanConfig(options, minimal) {
26
- const cleanUrls = options.cleanUrls === true;
27
- let cleanOpt = options.clean;
28
- if (cleanOpt === void 0) {
29
- if (!minimal) return { cleanUrls };
30
- cleanOpt = true;
25
+ const VALID_OPTIONS = "origin, clean, minimal, frontmatter, isolateMain, tailwind, filter, extraction, tagOverrides, wrapWidth, format";
26
+ function unknownOptionMessage(key, value) {
27
+ switch (key) {
28
+ case "preset": return "mdream has no `preset` option. Pass { minimal: true }.";
29
+ case "cleanUrls": return "mdream has no `cleanUrls` option. Pass { clean: { urls: true } }.";
30
+ case "plugins": return Array.isArray(value) ? "Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://github.com/harlan-zw/mdream/tree/main/packages/js#migrating-from-v1" : "mdream takes plugin options at the top level, such as { frontmatter: true }.";
31
31
  }
32
- if (!cleanOpt) return { cleanUrls };
33
- const resolved = cleanOpt === true ? CLEAN_ALL : cleanOpt;
34
- return {
35
- cleanUrls: cleanUrls || resolved.urls === true,
36
- clean: resolved
37
- };
32
+ return `mdream has no \`${key}\` option. Valid options: ${VALID_OPTIONS}.`;
33
+ }
34
+ function assertKnownOptions(options) {
35
+ for (const key in options) {
36
+ switch (key) {
37
+ case "origin":
38
+ case "clean":
39
+ case "minimal":
40
+ case "frontmatter":
41
+ case "isolateMain":
42
+ case "tailwind":
43
+ case "filter":
44
+ case "extraction":
45
+ case "tagOverrides":
46
+ case "wrapWidth":
47
+ case "format": continue;
48
+ }
49
+ const value = options[key];
50
+ if (value !== void 0) throw new TypeError(unknownOptionMessage(key, value));
51
+ }
52
+ }
53
+ function resolveClean(clean, minimal) {
54
+ if (clean === void 0) return minimal ? CLEAN_ALL : void 0;
55
+ if (!clean) return void 0;
56
+ return clean === true ? CLEAN_ALL : clean;
38
57
  }
39
58
  function resolveFrontmatter(opt) {
40
59
  if (typeof opt === "function") return {
@@ -51,24 +70,18 @@ function resolveFrontmatter(opt) {
51
70
  return { config: {} };
52
71
  }
53
72
  function resolveOptions(options) {
54
- if (Array.isArray(options.plugins)) throw new TypeError("Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://mdream.dev/v1-migration#custom-plugins");
73
+ assertKnownOptions(options);
55
74
  const minimal = options.minimal === true;
56
- const plugins = options.plugins ? { ...options.plugins } : {};
75
+ const plugins = {};
57
76
  let frontmatterCallback;
58
- const frontmatterDisabled = options.frontmatter === false;
59
- if (minimal ? !frontmatterDisabled : !!options.frontmatter) {
77
+ if (minimal ? options.frontmatter !== false : options.frontmatter) {
60
78
  const fm = resolveFrontmatter(options.frontmatter);
61
79
  plugins.frontmatter = fm.config;
62
80
  frontmatterCallback = fm.callback;
63
- } else if (frontmatterDisabled) delete plugins.frontmatter;
64
- const isolateMainDisabled = options.isolateMain === false;
65
- if (minimal ? !isolateMainDisabled : options.isolateMain) plugins.isolateMain = true;
66
- else if (isolateMainDisabled) delete plugins.isolateMain;
67
- const tailwindDisabled = options.tailwind === false;
68
- if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
69
- else if (tailwindDisabled) delete plugins.tailwind;
70
- if (options.filter === false) delete plugins.filter;
71
- else if (minimal) plugins.filter = {
81
+ }
82
+ if (minimal ? options.isolateMain !== false : options.isolateMain) plugins.isolateMain = true;
83
+ if (minimal ? options.tailwind !== false : options.tailwind) plugins.tailwind = true;
84
+ if (minimal && options.filter !== false) plugins.filter = {
72
85
  ...options.filter,
73
86
  exclude: [...MINIMAL_FILTER_EXCLUDE, ...options.filter?.exclude ?? []]
74
87
  };
@@ -88,12 +101,10 @@ function resolveOptions(options) {
88
101
  }
89
102
  plugins.tagOverrides = overrides;
90
103
  }
91
- const { cleanUrls, clean } = resolveCleanConfig(options, minimal);
92
104
  return {
93
105
  napiOpts: {
94
106
  origin: options.origin,
95
- cleanUrls,
96
- clean,
107
+ clean: resolveClean(options.clean, minimal),
97
108
  plugins,
98
109
  wrapWidth: options.wrapWidth,
99
110
  format: options.format
@@ -115,9 +126,9 @@ function convertResult(convert, html, options = {}) {
115
126
  deliverPluginData(result, resolved);
116
127
  return result;
117
128
  }
118
- async function* pumpStream(stream, htmlStream, callbacks, { decodeInJs = false, mapError = (error) => error } = {}) {
129
+ async function* pumpStream(stream, htmlStream, callbacks, { mapError = (error) => error } = {}) {
119
130
  const reader = htmlStream.getReader();
120
- const decoder = decodeInJs ? new TextDecoder() : void 0;
131
+ const decoder = stream.processChunkBytes ? void 0 : new TextDecoder();
121
132
  try {
122
133
  while (true) {
123
134
  const { done, value } = await reader.read();