@mdream/js 1.3.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -982,6 +982,205 @@ const BACKTICK_CHAR = 96;
982
982
  const PIPE_CHAR = 124;
983
983
  const OPEN_BRACKET_CHAR = 91;
984
984
  const CLOSE_BRACKET_CHAR = 93;
985
+ const HEAD_CONTENT_TAGS = new Set([
986
+ 4,
987
+ 5,
988
+ 54,
989
+ 56,
990
+ 53,
991
+ 52,
992
+ 84,
993
+ 78
994
+ ]);
995
+ const CLOSES_P = (() => {
996
+ const t = new Uint8Array(109);
997
+ for (const id of [
998
+ 36,
999
+ 35,
1000
+ 24,
1001
+ 33,
1002
+ 101,
1003
+ 25,
1004
+ 98,
1005
+ 99,
1006
+ 28,
1007
+ 7,
1008
+ 8,
1009
+ 9,
1010
+ 10,
1011
+ 11,
1012
+ 12,
1013
+ 22,
1014
+ 51,
1015
+ 50,
1016
+ 105,
1017
+ 47,
1018
+ 41,
1019
+ 88,
1020
+ 34,
1021
+ 13,
1022
+ 40,
1023
+ 68,
1024
+ 102,
1025
+ 106,
1026
+ 100,
1027
+ 104,
1028
+ 45,
1029
+ 2,
1030
+ 3,
1031
+ 75
1032
+ ]) t[id] = 1;
1033
+ return t;
1034
+ })();
1035
+ const NEEDS_IMPLIED_END_RECOVERY = (() => {
1036
+ const t = CLOSES_P.slice();
1037
+ for (const id of [
1038
+ 26,
1039
+ 32,
1040
+ 31,
1041
+ 30,
1042
+ 29,
1043
+ 38,
1044
+ 39
1045
+ ]) t[id] = 1;
1046
+ return t;
1047
+ })();
1048
+ const P_SCOPE_BOUNDARY = new Set([
1049
+ 43,
1050
+ 32,
1051
+ 31,
1052
+ 107,
1053
+ 28,
1054
+ 78,
1055
+ 0,
1056
+ 24,
1057
+ 33,
1058
+ 101,
1059
+ 25
1060
+ ]);
1061
+ const LI_SCOPE_BOUNDARY = new Set([
1062
+ 24,
1063
+ 33,
1064
+ 28,
1065
+ 32,
1066
+ 31,
1067
+ 107,
1068
+ 78,
1069
+ 0
1070
+ ]);
1071
+ const DL_SCOPE_BOUNDARY = new Set([
1072
+ 101,
1073
+ 24,
1074
+ 33,
1075
+ 25,
1076
+ 28,
1077
+ 32,
1078
+ 31,
1079
+ 107,
1080
+ 78,
1081
+ 0
1082
+ ]);
1083
+ const CELL_SCOPE_BOUNDARY = new Set([
1084
+ 30,
1085
+ 29,
1086
+ 38,
1087
+ 39,
1088
+ 28,
1089
+ 107,
1090
+ 78,
1091
+ 0
1092
+ ]);
1093
+ const A_SCOPE_BOUNDARY = new Set([
1094
+ 35,
1095
+ 36,
1096
+ 25,
1097
+ 24,
1098
+ 33,
1099
+ 101,
1100
+ 98,
1101
+ 99,
1102
+ 28,
1103
+ 32,
1104
+ 31,
1105
+ 30,
1106
+ 107,
1107
+ 22,
1108
+ 51,
1109
+ 50,
1110
+ 105,
1111
+ 47,
1112
+ 41,
1113
+ 88,
1114
+ 104,
1115
+ 40,
1116
+ 68,
1117
+ 102,
1118
+ 43,
1119
+ 7,
1120
+ 8,
1121
+ 9,
1122
+ 10,
1123
+ 11,
1124
+ 12,
1125
+ 78,
1126
+ 0
1127
+ ]);
1128
+ const HEADINGS = new Set([
1129
+ 7,
1130
+ 8,
1131
+ 9,
1132
+ 10,
1133
+ 11,
1134
+ 12
1135
+ ]);
1136
+ const SINGLE_P = new Set([35]);
1137
+ const SINGLE_LI = new Set([25]);
1138
+ const SINGLE_A = new Set([26]);
1139
+ const DT_DD = new Set([99, 98]);
1140
+ const TD_TH = new Set([32, 31]);
1141
+ const TR_CELLS = new Set([
1142
+ 32,
1143
+ 31,
1144
+ 30
1145
+ ]);
1146
+ const SECTION_CELLS = new Set([
1147
+ 32,
1148
+ 31,
1149
+ 30,
1150
+ 29,
1151
+ 38,
1152
+ 39,
1153
+ 107
1154
+ ]);
1155
+ function closeImpliedTo(state, target, boundary, handleEvent) {
1156
+ let found = false;
1157
+ for (let node = state.currentNode; node; node = node.parent) {
1158
+ const id = node.tagId;
1159
+ if (id !== void 0 && target.has(id)) {
1160
+ found = true;
1161
+ break;
1162
+ }
1163
+ if (id !== void 0 && boundary.has(id)) break;
1164
+ }
1165
+ if (!found) return;
1166
+ while (state.currentNode) {
1167
+ const id = state.currentNode.tagId;
1168
+ const isTarget = id !== void 0 && target.has(id);
1169
+ closeNode(state.currentNode, state, handleEvent);
1170
+ if (isTarget) break;
1171
+ }
1172
+ }
1173
+ function closeTableContext(state, closeable, handleEvent) {
1174
+ while (state.currentNode) {
1175
+ const id = state.currentNode.tagId;
1176
+ if (id === void 0 || !closeable.has(id)) break;
1177
+ closeNode(state.currentNode, state, handleEvent);
1178
+ }
1179
+ }
1180
+ function finalizeParse(leftover, state, handleEvent) {
1181
+ if (leftover.length > 0 && leftover.charCodeAt(0) !== LT_CHAR) processTextBuffer(leftover, state, handleEvent);
1182
+ while (state.currentNode) closeNode(state.currentNode, state, handleEvent);
1183
+ }
985
1184
  const EMPTY_ATTRIBUTES = Object.freeze({});
986
1185
  function copyDepthMap(depthMap) {
987
1186
  return new Uint8Array(depthMap);
@@ -1379,6 +1578,30 @@ function processOpeningTag(tagName, tagId, htmlChunk, i, state, handleEvent) {
1379
1578
  remainingText: `<${tagName}${result.attrBuffer}`,
1380
1579
  selfClosing: false
1381
1580
  };
1581
+ if ((state.depthMap[1] || 0) > 0 && !HEAD_CONTENT_TAGS.has(tagId)) {
1582
+ while (state.currentNode && state.currentNode.tagId !== 1) closeNode(state.currentNode, state, handleEvent);
1583
+ const headNode = state.currentNode;
1584
+ if (headNode && headNode.tagId === 1) closeNode(headNode, state, handleEvent);
1585
+ }
1586
+ if (tagId >= 0 && tagId < 109 && NEEDS_IMPLIED_END_RECOVERY[tagId] === 1) if (tagId === 26) {
1587
+ if ((state.depthMap[26] || 0) > 0) closeImpliedTo(state, SINGLE_A, A_SCOPE_BOUNDARY, handleEvent);
1588
+ } else if (tagId === 32 || tagId === 31 || tagId === 30 || tagId === 29 || tagId === 38 || tagId === 39) {
1589
+ if ((state.depthMap[28] || 0) > 0) if (tagId === 32 || tagId === 31) {
1590
+ if ((state.depthMap[32] || 0) > 0 || (state.depthMap[31] || 0) > 0) closeImpliedTo(state, TD_TH, CELL_SCOPE_BOUNDARY, handleEvent);
1591
+ } else if (tagId === 30) {
1592
+ if ((state.depthMap[30] || 0) > 0) closeTableContext(state, TR_CELLS, handleEvent);
1593
+ } else closeTableContext(state, SECTION_CELLS, handleEvent);
1594
+ } else {
1595
+ if ((state.depthMap[35] || 0) > 0) closeImpliedTo(state, SINGLE_P, P_SCOPE_BOUNDARY, handleEvent);
1596
+ if (HEADINGS.has(tagId)) {
1597
+ const top = state.currentNode;
1598
+ if (top && top.tagId !== void 0 && HEADINGS.has(top.tagId)) closeNode(top, state, handleEvent);
1599
+ } else if (tagId === 25) {
1600
+ if ((state.depthMap[25] || 0) > 0) closeImpliedTo(state, SINGLE_LI, LI_SCOPE_BOUNDARY, handleEvent);
1601
+ } else if (tagId === 99 || tagId === 98) {
1602
+ if ((state.depthMap[99] || 0) > 0 || (state.depthMap[98] || 0) > 0) closeImpliedTo(state, DT_DD, DL_SCOPE_BOUNDARY, handleEvent);
1603
+ }
1604
+ }
1382
1605
  const currentTagCount = result.attributes && result.attributes.id ? void 0 : state.depthMap[tagId] || 0;
1383
1606
  state.depthMap[tagId] = (currentTagCount || 0) + 1;
1384
1607
  state.depth++;
@@ -1554,4 +1777,4 @@ function parseAttributes(attrStr) {
1554
1777
  }
1555
1778
  return result;
1556
1779
  }
1557
- export { buildTagOverrideHandlers, parseAttributes, parseHtml, parseHtmlStream };
1780
+ export { buildTagOverrideHandlers, finalizeParse, parseAttributes, parseHtml, parseHtmlStream };
@@ -1,5 +1,5 @@
1
1
  import { DEFAULT_BLOCK_SPACING, NO_SPACING } from "./const.mjs";
2
- import { parseHtmlStream } from "./parse.mjs";
2
+ import { finalizeParse, parseHtmlStream } from "./parse.mjs";
3
3
  import { extractionCollectorPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin } from "./plugins.mjs";
4
4
  function processPluginsForEvent(event, plugins, state, processEvent) {
5
5
  if (plugins?.length) {
@@ -85,6 +85,73 @@ function shouldAddSpacingBeforeText(lastChar, lastNode, textNode) {
85
85
  if (firstChar === "." || firstChar === "," || firstChar === "!" || firstChar === "?" || firstChar === ":" || firstChar === ";" || firstChar === "_" || firstChar === "*" || firstChar === "`" || firstChar === ")" || firstChar === "]") return false;
86
86
  return true;
87
87
  }
88
+ function canWrapHere(depthMap) {
89
+ if (depthMap[34] || depthMap[23] || depthMap[32] || depthMap[31]) return false;
90
+ for (let h = 7; h <= 12; h++) if (depthMap[h]) return false;
91
+ return true;
92
+ }
93
+ function currentColumn(buffer) {
94
+ let col = 0;
95
+ for (let i = buffer.length - 1; i >= 0; i--) {
96
+ const s = buffer[i];
97
+ const nl = s.lastIndexOf("\n");
98
+ if (nl >= 0) return col + [...s.slice(nl + 1)].length;
99
+ col += [...s].length;
100
+ }
101
+ return col;
102
+ }
103
+ function wrapContinuationPrefix(state, node) {
104
+ const chain = [];
105
+ let cur = node.parent;
106
+ while (cur) {
107
+ chain.push(cur);
108
+ cur = cur.parent;
109
+ }
110
+ let p = "";
111
+ let liIdx = 0;
112
+ for (let i = chain.length - 1; i >= 0; i--) {
113
+ const tagId = chain[i].tagId;
114
+ if (tagId === 22) p += "> ";
115
+ else if (tagId === 25) {
116
+ const w = state.listIndentWidths[liIdx] ?? 2;
117
+ p += " ".repeat(w);
118
+ liIdx++;
119
+ }
120
+ }
121
+ return p;
122
+ }
123
+ function wrapText(value, col, width, prefix) {
124
+ const leading = value.charCodeAt(0) === 32;
125
+ const trailing = value.charCodeAt(value.length - 1) === 32;
126
+ const prefixLen = [...prefix].length;
127
+ let out = "";
128
+ let first = true;
129
+ let i = 0;
130
+ const len = value.length;
131
+ while (i < len) {
132
+ let next = value.indexOf(" ", i);
133
+ if (next === -1) next = len;
134
+ if (next > i) {
135
+ const word = value.slice(i, next);
136
+ const wordLen = [...word].length;
137
+ const needSpace = first ? leading : true;
138
+ if (needSpace && col > prefixLen && col + 1 + wordLen > width) {
139
+ out += `\n${prefix}`;
140
+ col = prefixLen;
141
+ } else if (needSpace) {
142
+ out += " ";
143
+ col += 1;
144
+ }
145
+ out += word;
146
+ col += wordLen;
147
+ first = false;
148
+ }
149
+ i = next + 1;
150
+ }
151
+ if (trailing && out !== "" && !out.endsWith(" ") && !out.endsWith("\n")) out += " ";
152
+ if (out === "" && (leading || trailing)) out = " ";
153
+ return out;
154
+ }
88
155
  function calculateNewLineConfig(node) {
89
156
  const tagId = node.tagId;
90
157
  const depthMap = node.depthMap;
@@ -163,8 +230,15 @@ function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverride
163
230
  if (lastChar === "\n" && value[0] && value[0] !== "\n") value = indent + value;
164
231
  textNode.value = value;
165
232
  }
166
- state.buffer.push(textNode.value);
167
- state.lastContentCache = textNode.value;
233
+ const wrapWidth = state.options?.wrapWidth;
234
+ if (wrapWidth && canWrapHere(state.depthMap)) {
235
+ const wrapped = wrapText(textNode.value, currentColumn(state.buffer), wrapWidth, wrapContinuationPrefix(state, textNode));
236
+ state.buffer.push(wrapped);
237
+ state.lastContentCache = wrapped;
238
+ } else {
239
+ state.buffer.push(textNode.value);
240
+ state.lastContentCache = textNode.value;
241
+ }
168
242
  }
169
243
  state.lastTextNode = textNode;
170
244
  return;
@@ -231,14 +305,16 @@ function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverride
231
305
  updateListIndent(state, element, eventType);
232
306
  }
233
307
  function processHtml(html) {
234
- parseHtmlStream(html, {
308
+ const parseState = {
235
309
  depthMap: state.depthMap,
236
310
  depth: 0,
237
311
  resolvedPlugins,
238
312
  tagOverrideHandlers
239
- }, (event) => {
313
+ };
314
+ const handleEvent = (event) => {
240
315
  processPluginsForEvent(event, resolvedPlugins, state, processEvent);
241
- });
316
+ };
317
+ finalizeParse(parseHtmlStream(html, parseState, handleEvent), parseState, handleEvent);
242
318
  }
243
319
  function getMarkdown() {
244
320
  const result = state.buffer.join("").trimStart();
@@ -1,5 +1,5 @@
1
1
  import "./const.mjs";
2
- import { buildTagOverrideHandlers, parseHtmlStream } from "./parse.mjs";
2
+ import { buildTagOverrideHandlers, finalizeParse, parseHtmlStream } from "./parse.mjs";
3
3
  import { createMarkdownProcessor, processPluginsForEvent, resolvePlugins } from "./resolve-plugins.mjs";
4
4
  import "./plugins.mjs";
5
5
  import "../preset/minimal.mjs";
@@ -310,9 +310,10 @@ async function* streamHtmlToMarkdown$1(htmlStream, options = {}, resolvedPlugins
310
310
  const chunk = processor.getMarkdownChunk();
311
311
  if (chunk) yield chunk;
312
312
  }
313
- if (remainingHtml) parseHtmlStream(remainingHtml, parseState, (event) => {
313
+ const handleEvent = (event) => {
314
314
  processPluginsForEvent(event, resolvedPlugins, processor.state, processor.processEvent);
315
- });
315
+ };
316
+ finalizeParse(remainingHtml ? parseHtmlStream(remainingHtml, parseState, handleEvent) : "", parseState, handleEvent);
316
317
  const finalChunk = processor.getMarkdownChunk();
317
318
  if (finalChunk) yield finalChunk;
318
319
  } finally {
@@ -137,6 +137,13 @@ interface EngineOptions {
137
137
  * final markdown (sync API only for `fragments`).
138
138
  */
139
139
  clean?: boolean | CleanOptions;
140
+ /**
141
+ * Hard-wrap prose at this many characters, breaking on word boundaries.
142
+ * Applied inline during conversion (zero-cost when unset). Code blocks
143
+ * (`<pre>`/`<code>`), tables, and headings are never wrapped. `0` disables
144
+ * wrapping.
145
+ */
146
+ wrapWidth?: number;
140
147
  }
141
148
  interface ElementNode extends Node {
142
149
  /** Element tag name (for ELEMENT_NODE) */
package/dist/cli.mjs CHANGED
@@ -6,7 +6,10 @@ import { fileURLToPath } from "node:url";
6
6
  import { cac } from "cac";
7
7
  import { dirname, join } from "pathe";
8
8
  async function streamingConvert(options = {}) {
9
- let conversionOptions = { origin: options.origin };
9
+ let conversionOptions = {
10
+ origin: options.origin,
11
+ wrapWidth: options.wrapWidth ? Number(options.wrapWidth) || void 0 : void 0
12
+ };
10
13
  if (options.preset === "minimal") conversionOptions = withMinimalPreset(conversionOptions);
11
14
  const markdownGenerator = streamHtmlToMarkdown(Readable.toWeb(process.stdin), conversionOptions);
12
15
  for await (const markdownChunk of markdownGenerator) if (markdownChunk && markdownChunk.length > 0) process.stdout.write(markdownChunk);
@@ -14,7 +17,7 @@ async function streamingConvert(options = {}) {
14
17
  const packageJsonPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
15
18
  const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf-8"));
16
19
  const cli = cac();
17
- cli.command("[options]", "Convert HTML from stdin to Markdown on stdout (JS engine)").option("--origin <url>", "Origin URL for resolving relative image paths").option("--preset <preset>", "Conversion presets: minimal").action(async (_, opts) => {
20
+ cli.command("[options]", "Convert HTML from stdin to Markdown on stdout (JS engine)").option("--origin <url>", "Origin URL for resolving relative image paths").option("--preset <preset>", "Conversion presets: minimal").option("--wrap-width <n>", "Hard-wrap prose at <n> characters on word boundaries").action(async (_, opts) => {
18
21
  await streamingConvert(opts);
19
22
  });
20
23
  cli.help().version(packageJson.version).parse();
package/dist/parse.d.mts CHANGED
@@ -1,4 +1,18 @@
1
1
  import { ElementNode, Node, NodeEvent, TagHandler, TransformPlugin } from "./_chunks/types.mjs";
2
+ /**
3
+ * Commit end-of-input state: flush trailing buffered text and close any open
4
+ * elements. The streaming parser keeps trailing text and unclosed elements
5
+ * pending (a later chunk might continue them); at true EOF they must be
6
+ * committed so trailing content is not dropped (e.g. `<p>a<p>b`).
7
+ *
8
+ * `leftover` is the residual returned by the final `parseHtmlStream`. Pure
9
+ * trailing text (no leading `<`) is emitted; a residual that is an incomplete
10
+ * start tag (leading `<`) is dropped, matching the browser tokenizer's
11
+ * EOF-in-tag behaviour. The text-buffer flags set while the trailing text was
12
+ * scanned persist on `state`, so `processTextBuffer` commits it as if the next
13
+ * tag had triggered the flush.
14
+ */
15
+ declare function finalizeParse(leftover: string, state: ParseState, handleEvent: (event: NodeEvent) => void): void;
2
16
  interface ParseOptions {
3
17
  resolvedPlugins?: TransformPlugin[];
4
18
  }
@@ -58,4 +72,4 @@ declare function parseHtmlStream(html: string, state: ParseState, onEvent: (even
58
72
  * Parse HTML attributes string into key-value object
59
73
  */
60
74
  declare function parseAttributes(attrStr: string): Record<string, string>;
61
- export { ParseOptions, ParseResult, ParseState, parseAttributes, parseHtml, parseHtmlStream };
75
+ export { ParseOptions, ParseResult, ParseState, finalizeParse, parseAttributes, parseHtml, parseHtmlStream };
package/dist/parse.mjs CHANGED
@@ -1,2 +1,2 @@
1
- import { parseAttributes, parseHtml, parseHtmlStream } from "./_chunks/parse.mjs";
2
- export { parseAttributes, parseHtml, parseHtmlStream };
1
+ import { finalizeParse, parseAttributes, parseHtml, parseHtmlStream } from "./_chunks/parse.mjs";
2
+ export { finalizeParse, parseAttributes, parseHtml, parseHtmlStream };
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@mdream/js",
3
3
  "type": "module",
4
- "version": "1.3.0",
4
+ "version": "1.4.1",
5
5
  "description": "JavaScript HTML-to-Markdown engine for mdream. Escape hatch for hooks and edge runtimes.",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",