@mdream/js 1.5.13 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -38,7 +38,7 @@ plugins.
38
38
 
39
39
  ### `htmlToMarkdown(html, options?)`
40
40
 
41
- Converts an HTML string to Markdown synchronously.
41
+ Converts an HTML string synchronously.
42
42
 
43
43
  ```typescript
44
44
  import { htmlToMarkdown } from '@mdream/js'
@@ -50,6 +50,11 @@ const text = htmlToMarkdown('<h1>Hello</h1><p><strong>World</strong></p>', {
50
50
  format: 'text',
51
51
  })
52
52
  // Hello\n\nWorld
53
+
54
+ const html = htmlToMarkdown('<h1>Hello</h1><p><strong>World</strong></p>', {
55
+ format: 'html',
56
+ })
57
+ // <h1 id="hello">Hello</h1><p><strong>World</strong></p>
53
58
  ```
54
59
 
55
60
  **Parameters:**
@@ -63,7 +68,7 @@ const text = htmlToMarkdown('<h1>Hello</h1><p><strong>World</strong></p>', {
63
68
 
64
69
  ### `streamHtmlToMarkdown(htmlStream, options?)`
65
70
 
66
- Converts an HTML stream to Markdown incrementally. Useful for large documents or streaming HTTP responses.
71
+ Converts an HTML stream incrementally. Useful for large documents or streaming HTTP responses.
67
72
 
68
73
  ```typescript
69
74
  import { streamHtmlToMarkdown } from '@mdream/js'
@@ -113,7 +118,7 @@ const markdown = htmlToMarkdown(html, {
113
118
  | `clean` | `boolean \| CleanOptions` | `undefined` | Post-processing cleanup. Pass `true` for all cleanup rules or an object for specific features (see [CleanOptions](#cleanoptions)). Sync API only for `fragments`. |
114
119
  | `hooks` | `TransformPlugin[]` | `undefined` | Imperative hook-based transform plugins for custom behavior (see [Plugins](#plugins)) |
115
120
  | `wrapWidth` | `number` | `undefined` | Hard-wrap prose at this many characters on word boundaries |
116
- | `format` | `'markdown' \| 'text'` | `'markdown'` | Output Markdown or plain text with Markdown/HTML markup omitted |
121
+ | `format` | `'markdown' \| 'text' \| 'html'` | `'markdown'` | Output Markdown, plain text, or HTML |
117
122
 
118
123
  ### `BuiltinPlugins`
119
124
 
@@ -579,7 +584,7 @@ const remaining = parseHtmlStream(htmlChunk, state, (event) => {
579
584
 
580
585
  ## CLI
581
586
 
582
- Reads HTML from stdin and outputs Markdown to stdout.
587
+ Reads HTML from stdin and writes the selected format to stdout.
583
588
 
584
589
  ```bash
585
590
  # Basic conversion
@@ -593,6 +598,9 @@ curl -s https://example.com | npx @mdream/js --origin https://example.com --pres
593
598
 
594
599
  # Plain text output
595
600
  curl -s https://example.com | npx @mdream/js --format text
601
+
602
+ # HTML output
603
+ curl -s https://example.com | npx @mdream/js --format html
596
604
  ```
597
605
 
598
606
  ### CLI Options
@@ -602,7 +610,7 @@ curl -s https://example.com | npx @mdream/js --format text
602
610
  | `--origin <url>` | Origin URL for resolving relative image paths and links |
603
611
  | `--preset <preset>` | Conversion preset. Currently supports: `minimal` |
604
612
  | `--wrap-width <n>` | Hard-wrap prose at `n` characters |
605
- | `--format <format>` | Output format: `markdown`, `text` |
613
+ | `--format <format>` | Output format: `markdown`, `text`, `html` |
606
614
  | `--text` | Alias for `--format text` |
607
615
  | `-v, --version` | Show version number |
608
616
  | `-h, --help` | Show help |
@@ -1,5 +1,150 @@
1
- import { DEFAULT_BLOCK_SPACING, NO_SPACING, isInsideRawHtmlBlock } from "./const.mjs";
2
- import { blockOpenPrefix, breakHandler, continuationPrefix, finalizeParse, isCharacterReferenceTail, isInsideHeading, isInsideTableCell, listMarkerLineStart, orderedItemNumber, parseHtmlStream, renderBreak, resolveUrl } from "./parse.mjs";
1
+ import { DEFAULT_BLOCK_SPACING, NO_SPACING, TagIdMap, isInsideRawHtmlBlock } from "./const.mjs";
2
+ import { blockOpenPrefix, breakHandler, continuationPrefix, finalizeParse, getLanguageFromClass, isCharacterReferenceTail, isInsideHeading, isInsideTableCell, isSafeHtmlUrl, listMarkerLineStart, orderedItemNumber, parseHtmlStream, parseUnsignedInteger, renderBreak, resolveUrl } from "./parse.mjs";
3
+ function createHtmlOutputState() {
4
+ return { frames: [] };
5
+ }
6
+ function escapeHtml(value, attribute = false) {
7
+ return value.replace(attribute ? /[&<>"]/g : /[&<>]/g, (character) => character === "&" ? "&amp;" : character === "<" ? "&lt;" : character === ">" ? "&gt;" : "&quot;");
8
+ }
9
+ function slugifyHeading(text) {
10
+ return text.toLowerCase().replace(/[^a-z\d \t\n\r-]/g, "").trim().replace(/[ \t\n\r-]+/g, "-");
11
+ }
12
+ function tagName(node, tagId) {
13
+ switch (tagId) {
14
+ case 14:
15
+ case 15: return "strong";
16
+ case 16:
17
+ case 17: return "em";
18
+ case 18:
19
+ case 110:
20
+ case 111: return "del";
21
+ }
22
+ if (!(tagId === 2 || tagId === 3 || tagId >= 14 && tagId <= 26 || tagId >= 28 && tagId <= 33 || tagId >= 35 && tagId <= 39 || tagId === 41 || tagId >= 46 && tagId <= 47 || tagId >= 50 && tagId <= 51 || tagId >= 79 && tagId <= 83 || tagId >= 88 && tagId <= 93 || tagId >= 95 && tagId <= 102 || tagId >= 104 && tagId <= 107)) return;
23
+ if (node.tagHandler?.aliasTagId === void 0) return node.name;
24
+ for (const name in TagIdMap) if (TagIdMap[name] === tagId) return name;
25
+ }
26
+ function resolveTagId(node) {
27
+ return node.tagHandler?.aliasTagId ?? node.tagId;
28
+ }
29
+ function resolveSafeUrl(url, options, image = false) {
30
+ if (!isSafeHtmlUrl(url, image)) return;
31
+ const resolved = resolveUrl(url, options.origin, options.clean);
32
+ return isSafeHtmlUrl(resolved, image) ? resolved : void 0;
33
+ }
34
+ function safeAttributes(node, tagId, options) {
35
+ const attributes = node.attributes;
36
+ if (tagId === 26) {
37
+ const href = attributes.href && resolveSafeUrl(attributes.href, options);
38
+ if (!href) return void 0;
39
+ const title = attributes.title === void 0 ? "" : ` title="${escapeHtml(attributes.title, true)}"`;
40
+ return ` href="${escapeHtml(href, true)}"${title}`;
41
+ }
42
+ if (tagId === 33) {
43
+ const start = parseUnsignedInteger(attributes.start);
44
+ return start === void 0 ? "" : ` start="${start}"`;
45
+ }
46
+ if (tagId === 23) {
47
+ const language = getLanguageFromClass(attributes.class);
48
+ return language ? ` class="language-${escapeHtml(language, true)}"` : "";
49
+ }
50
+ if (tagId === 31 || tagId === 32) {
51
+ let output = "";
52
+ const colspan = parseUnsignedInteger(attributes.colspan);
53
+ if (colspan !== void 0 && colspan > 0) output += ` colspan="${colspan}"`;
54
+ if (tagId === 31) {
55
+ const align = attributes.align?.toLowerCase();
56
+ if (align === "left" || align === "center" || align === "right") output += ` align="${align}"`;
57
+ }
58
+ return output;
59
+ }
60
+ return "";
61
+ }
62
+ function append(state, output, value) {
63
+ const frame = state.frames.at(-1);
64
+ if (frame) frame.output += value;
65
+ else output.push(value);
66
+ }
67
+ function processText(node, state, output) {
68
+ if (node.excludedFromMarkdown || !node.value) return;
69
+ for (let index = state.frames.length - 1; index >= 0; index--) {
70
+ const frame = state.frames[index];
71
+ if (frame._tag === 0) {
72
+ frame.text += node.value;
73
+ break;
74
+ }
75
+ }
76
+ append(state, output, escapeHtml(node.value));
77
+ }
78
+ function closeFrame(state, output, frame) {
79
+ state.frames.pop();
80
+ if (frame._tag === 0) {
81
+ const slug = slugifyHeading(frame.text);
82
+ const id = slug ? ` id="${slug}"` : "";
83
+ append(state, output, `<h${frame.level}${id}>${frame.output}</h${frame.level}>`);
84
+ return;
85
+ }
86
+ append(state, output, `<pre tabindex="0"><code${frame.language ? ` class="language-${escapeHtml(frame.language, true)}"` : ""}>${frame.output}</code></pre>`);
87
+ }
88
+ function processHtmlOutputEvent(event, state, output, options) {
89
+ const { node, type } = event;
90
+ if (node.type === 2) {
91
+ if (type === 0) processText(node, state, output);
92
+ return;
93
+ }
94
+ if (node.type !== 1) return;
95
+ const element = node;
96
+ const tagId = resolveTagId(element);
97
+ const frame = state.frames.at(-1);
98
+ if (frame?._tag === 1) {
99
+ if (type === 0 && tagId === 23 && !frame.language) frame.language = getLanguageFromClass(element.attributes.class);
100
+ else if (type === 0 && tagId === 6) frame.output += "\n";
101
+ else if (type !== 0 && tagId === 34 && element.depth === frame.depth) closeFrame(state, output, frame);
102
+ return;
103
+ }
104
+ if (type === 0 && tagId !== void 0 && tagId >= 7 && tagId <= 12) {
105
+ state.frames.push({
106
+ _tag: 0,
107
+ level: tagId - 7 + 1,
108
+ output: "",
109
+ text: ""
110
+ });
111
+ return;
112
+ }
113
+ if (type !== 0 && frame?._tag === 0 && tagId !== void 0 && tagId >= 7 && tagId <= 12) {
114
+ closeFrame(state, output, frame);
115
+ return;
116
+ }
117
+ if (type === 0 && tagId === 34) {
118
+ state.frames.push({
119
+ _tag: 1,
120
+ depth: element.depth,
121
+ language: getLanguageFromClass(element.attributes.class),
122
+ output: ""
123
+ });
124
+ return;
125
+ }
126
+ if (tagId === 6) {
127
+ if (type === 0) append(state, output, "<br>");
128
+ return;
129
+ }
130
+ if (tagId === 13) {
131
+ if (type === 0) append(state, output, "<hr>");
132
+ return;
133
+ }
134
+ if (tagId === 27) {
135
+ if (type === 0) {
136
+ const src = element.attributes.src && resolveSafeUrl(element.attributes.src, options, true);
137
+ if (src) append(state, output, `<img src="${escapeHtml(src, true)}" alt="${escapeHtml(element.attributes.alt || "", true)}"${element.attributes.title === void 0 ? "" : ` title="${escapeHtml(element.attributes.title, true)}"`}>`);
138
+ }
139
+ return;
140
+ }
141
+ if (tagId === void 0) return;
142
+ const name = tagName(element, tagId);
143
+ if (!name) return;
144
+ const attributes = safeAttributes(element, tagId, options);
145
+ if (attributes === void 0) return;
146
+ append(state, output, type === 0 ? `<${name}${attributes}>` : `</${name}>`);
147
+ }
3
148
  function processPluginsForEvent(event, plugins, state, processEvent) {
4
149
  if (plugins?.length) {
5
150
  if (event.node.type === 1 && event.type === 0) {
@@ -623,6 +768,8 @@ function getPlainTextOutput(node, eventType, state) {
623
768
  if (tagId === 81) return "\"";
624
769
  }
625
770
  function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverrideHandlers) {
771
+ const outputFormat = options.format === "html" || options.format === "text" ? options.format : "markdown";
772
+ const htmlState = outputFormat === "html" ? createHtmlOutputState() : void 0;
626
773
  const state = {
627
774
  options,
628
775
  buffer: [],
@@ -690,6 +837,10 @@ function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverride
690
837
  const { type: eventType, node } = event;
691
838
  state.depth = node.depth;
692
839
  if (node.type === 1 ? node.excludedFromMarkdown : node.parent?.excludedFromMarkdown) return;
840
+ if (htmlState) {
841
+ processHtmlOutputEvent(event, htmlState, state.buffer, options);
842
+ return;
843
+ }
693
844
  if (state.listRulePending !== void 0) {
694
845
  const isVisibleText = node.type === 2 && eventType === 0 && hasNonWhitespace(node.value);
695
846
  const opensVisibleElement = node.type === 1 && eventType === 0 && !node.tagHandler?.excludesTextNodes;
@@ -911,19 +1062,28 @@ function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverride
911
1062
  depth: 0,
912
1063
  resolvedPlugins,
913
1064
  tagOverrideHandlers,
914
- plainText: state.plainText
1065
+ plainText: outputFormat !== "markdown"
915
1066
  };
916
1067
  const handleEvent = resolvedPlugins.length ? (event) => processPluginsForEvent(event, resolvedPlugins, state, processEvent) : processEvent;
917
1068
  finalizeParse(parseHtmlStream(html, parseState, handleEvent), parseState, handleEvent);
918
1069
  }
919
1070
  function getMarkdown() {
920
1071
  const content = state.buffer.join("");
1072
+ if (htmlState) {
1073
+ state.buffer.length = 0;
1074
+ return content;
1075
+ }
921
1076
  const result = state.plainText && preserveLeadingWhitespace ? content : content.trimStart();
922
1077
  state.buffer.length = 0;
923
1078
  preserveLeadingWhitespace = false;
924
1079
  return result.trimEnd();
925
1080
  }
926
1081
  function getMarkdownChunk() {
1082
+ if (htmlState) {
1083
+ const chunk = state.buffer.join("");
1084
+ state.buffer.length = 0;
1085
+ return chunk;
1086
+ }
927
1087
  resolveItemMarker(state, false);
928
1088
  const content = state.buffer.join("");
929
1089
  const currentContent = hasYieldedContent || state.plainText && preserveLeadingWhitespace ? content : content.trimStart();
@@ -287,6 +287,28 @@ function isEmptyLinkHref(href) {
287
287
  default: return false;
288
288
  }
289
289
  }
290
+ function isSafeHtmlUrl(href, image = false) {
291
+ let start = 0;
292
+ while (start < href.length && href.charCodeAt(start) <= 32) start++;
293
+ let end = href.length;
294
+ while (end > start && href.charCodeAt(end - 1) <= 32) end--;
295
+ if (start === end) return false;
296
+ for (let index = start; index < end; index++) {
297
+ const code = href.charCodeAt(index);
298
+ if (code === 9 || code === 10 || code === 13) continue;
299
+ if (code === 58) switch (lowerAscii(href.charCodeAt(start))) {
300
+ case 104: return schemeMatches(href, start, end, "http:") || schemeMatches(href, start, end, "https:");
301
+ case 109: return !image && schemeMatches(href, start, end, "mailto:");
302
+ case 116: return !image && schemeMatches(href, start, end, "tel:");
303
+ case 102: return !image && schemeMatches(href, start, end, "ftp:");
304
+ default: return false;
305
+ }
306
+ if (code === 47 || code === 63 || code === 35) return true;
307
+ const lower = lowerAscii(code);
308
+ if (!(lower >= 97 && lower <= 122 || code >= 48 && code <= 57 || code === 43 || code === 45 || code === 46)) return true;
309
+ }
310
+ return true;
311
+ }
290
312
  function continuationPrefix(node, listIndentWidths, includeBlockquotes = true) {
291
313
  const chain = [];
292
314
  let current = node.parent;
@@ -358,7 +380,9 @@ function getLanguageFromClass(className) {
358
380
  return "";
359
381
  }
360
382
  function parseUnsignedInteger(raw) {
361
- return raw !== void 0 && /^[\t\n\f\r ]*\+?\d+[\t\n\f\r ]*$/.test(raw) ? Number(raw) : void 0;
383
+ if (raw === void 0 || !/^[\t\n\f\r ]*\+?\d+[\t\n\f\r ]*$/.test(raw)) return void 0;
384
+ const value = Number(raw);
385
+ return value <= 4294967295 ? value : void 0;
362
386
  }
363
387
  function lastOutputChar(buffer) {
364
388
  for (let index = buffer.length - 1; index >= 0; index--) {
@@ -2577,4 +2601,4 @@ function parseAttributes(attrStr) {
2577
2601
  if (result._tag === "incomplete") throw new Error("Isolated attribute scans always complete at EOF");
2578
2602
  return result.attributes === EMPTY_ATTRIBUTES ? {} : result.attributes;
2579
2603
  }
2580
- export { blockOpenPrefix, breakHandler, buildTagOverrideHandlers, continuationPrefix, finalizeParse, isCharacterReferenceTail, isEmptyLinkHref, isInsideHeading, isInsideTableCell, listMarkerLineStart, orderedItemNumber, parseAttributes, parseHtml, parseHtmlStream, renderBreak, resolveUrl };
2604
+ export { blockOpenPrefix, breakHandler, buildTagOverrideHandlers, continuationPrefix, finalizeParse, getLanguageFromClass, isCharacterReferenceTail, isEmptyLinkHref, isInsideHeading, isInsideTableCell, isSafeHtmlUrl, listMarkerLineStart, orderedItemNumber, parseAttributes, parseHtml, parseHtmlStream, parseUnsignedInteger, renderBreak, resolveUrl };
@@ -306,7 +306,7 @@ function convert(html, options, hooks) {
306
306
  }
307
307
  function htmlToMarkdown(html, options = {}) {
308
308
  const markdown = convert(html, options, resolveHooks(options));
309
- if (options.clean && options.format !== "text") return applyClean(markdown, resolveClean(options.clean));
309
+ if (options.clean && (options.format === void 0 || options.format === "markdown")) return applyClean(markdown, resolveClean(options.clean));
310
310
  return markdown;
311
311
  }
312
312
  function streamHtmlToMarkdown(htmlStream, options = {}) {
@@ -10,7 +10,7 @@ async function* streamHtmlToMarkdown(htmlStream, options = {}, resolvedPlugins =
10
10
  depth: 0,
11
11
  resolvedPlugins,
12
12
  tagOverrideHandlers,
13
- plainText: processor.state.plainText
13
+ plainText: options.format === "text" || options.format === "html"
14
14
  };
15
15
  const handleEvent = resolvedPlugins.length ? (event) => processPluginsForEvent(event, resolvedPlugins, processor.state, processor.processEvent) : processor.processEvent;
16
16
  let remainingHtml = "";
@@ -150,10 +150,10 @@ interface EngineOptions {
150
150
  */
151
151
  wrapWidth?: number;
152
152
  /**
153
- * Output format. Defaults to `markdown`; use `text` to omit Markdown/HTML
154
- * markup while preserving readable text and block spacing.
153
+ * Output format. Defaults to `markdown`; use `text` for readable plain text,
154
+ * or `html` for allowlisted semantic HTML.
155
155
  */
156
- format?: 'markdown' | 'text';
156
+ format?: 'markdown' | 'text' | 'html';
157
157
  }
158
158
  interface ElementNode extends Node {
159
159
  /** Element tag name (for ELEMENT_NODE) */
package/dist/cli.mjs CHANGED
@@ -7,7 +7,7 @@ import { cac } from "cac";
7
7
  import { dirname, join } from "pathe";
8
8
  async function streamingConvert(options = {}) {
9
9
  const format = options.text ? "text" : options.format;
10
- if (format && format !== "markdown" && format !== "text") {
10
+ if (format && format !== "markdown" && format !== "text" && format !== "html") {
11
11
  process.stderr.write(`Unknown format: ${format}\n`);
12
12
  process.exitCode = 1;
13
13
  return;
@@ -24,7 +24,7 @@ async function streamingConvert(options = {}) {
24
24
  const packageJsonPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
25
25
  const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf-8"));
26
26
  const cli = cac();
27
- cli.command("[options]", "Convert HTML from stdin to Markdown on stdout (JS engine)").option("--origin <url>", "Origin URL for resolving relative image paths").option("--preset <preset>", "Conversion presets: minimal").option("--wrap-width <n>", "Hard-wrap prose at <n> characters on word boundaries").option("--format <format>", "Output format: markdown, text").option("--text", "Alias for --format text").action(async (_, opts) => {
27
+ cli.command("[options]", "Convert HTML from stdin to Markdown on stdout (JS engine)").option("--origin <url>", "Origin URL for resolving relative image paths").option("--preset <preset>", "Conversion presets: minimal").option("--wrap-width <n>", "Hard-wrap prose at <n> characters on word boundaries").option("--format <format>", "Output format: markdown, text, html").option("--text", "Alias for --format text").action(async (_, opts) => {
28
28
  await streamingConvert(opts);
29
29
  });
30
30
  cli.help().version(packageJson.version).parse();
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@mdream/js",
3
3
  "type": "module",
4
- "version": "1.5.13",
4
+ "version": "1.6.0",
5
5
  "description": "JavaScript HTML-to-Markdown engine for mdream. Escape hatch for hooks and edge runtimes.",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",